Refactor spreadsheet handling to track corrections and defaults

Improved the backend's value cleaning to differentiate between corrections and defaults, logging metadata for clearer traceability. Updated frontend to display corrected/defaulted fields with visual cues and tooltips for better user feedback. Enhanced data models and response structures to support this richer metadata.

backend/app/routers/spreadsheet.py

@@ -96,9 +96,9 @@ async def upload_file(file: UploadFile = File(...)):
                     cleaned_value = importer._clean_value(
                         original_value, expected_type, col_name
                     )
-
+                    corrected = False
                     # Check if a correction was applied
-                    if cleaned_value != original_value:
+                    if cleaned_value[0] != original_value:
                         corrected = True
                         corrected_columns.append(col_name)
 
@@ -156,10 +156,78 @@ async def upload_file(file: UploadFile = File(...)):
             f"Rows corrected: {sum(1 for r in updated_raw_data if r.get('corrected'))}"
         )
 
+        updated_addinfo = [
+            {
+                "row_num": row["row_num"],  # Identify row for the frontend
+                "corrected_columns": row.get("corrected_columns", []),
+                "default_set": [
+                    col_name
+                    for col_name in row.get("corrected_columns", [])
+                    if row.get("default_set", False) and col_name == "directory"
+                ],  # Specify which keys are explicitly `default_set`
+            }
+            for row in updated_raw_data
+            if row.get("corrected")
+            or row.get("default_set")  # Only include rows with changes
+        ]
+        logger.debug(f"Constructed addinfo: {updated_addinfo}")
+        # Clean updated raw data in place
+        for row in updated_raw_data:
+            # Remove unwanted metadata fields
+            row.pop("corrected", None)
+            row.pop("corrected_columns", None)
+            row.pop("default_set", None)
+            row.pop("defaulted_columns", None)
+            row.pop("directory", None)
+
+            # Sanitize nested data (e.g., replace directory tuples with strings)
+            if "data" in row:
+                for idx, value in enumerate(row["data"]):
+                    if isinstance(value, tuple):
+                        row["data"][idx] = value[0]  # Extract the first item (string)
+
+        # Confirm cleanup worked
+        for row in updated_raw_data:
+            unexpected_keys = [
+                k
+                for k in row.keys()
+                if k
+                in [
+                    "corrected",
+                    "corrected_columns",
+                    "default_set",
+                    "defaulted_columns",
+                    "directory",
+                ]
+            ]
+            if unexpected_keys:
+                logger.error(f"Unexpected keys persist: {unexpected_keys}")
+
+        # Construct stripped_raw_data from the cleaned updated_raw_data
+        stripped_raw_data = [
+            {
+                k: v
+                for k, v in row.items()
+                if k
+                not in [
+                    "corrected",
+                    "corrected_columns",
+                    "default_set",
+                    "defaulted_columns",
+                    "directory",
+                ]
+            }
+            for row in updated_raw_data
+        ]
+
+        # Verify the final stripped raw data before returning
+        # logger.debug(f"Sanitized raw_data for response: {stripped_raw_data}")
+
         response_data = SpreadsheetResponse(
             data=validated_model,
             errors=errors,
-            raw_data=updated_raw_data,
+            raw_data=stripped_raw_data,  # Final submission data
+            addinfo=updated_addinfo,  # Metadata for frontend display
             dewars_count=len(dewars),
             dewars=list(dewars),
             pucks_count=len(pucks),
@@ -168,7 +236,6 @@ async def upload_file(file: UploadFile = File(...)):
             samples=list(samples),
             headers=headers,
         )
-
         logger.debug(f"Final updated_raw_data sent in response: {updated_raw_data}")
         # Store row data for future use
         for idx, row in enumerate(validated_model):