added error recognition in spreadsheet

backend/app/routers/spreadsheet.py

@@ -1,3 +1,4 @@
+from app.sample_models import SpreadsheetModel, SpreadsheetResponse
 from fastapi import APIRouter, UploadFile, File, HTTPException
 import logging
 from app.services.spreadsheet_service import SampleSpreadsheetImporter, SpreadsheetImportError
@@ -7,56 +8,50 @@ import os
 router = APIRouter()
 logger = logging.getLogger(__name__)
 
-
 @router.get("/download-template", response_class=FileResponse)
 async def download_template():
-    # Constructing the absolute path
+    # No changes here; just serves a static file
     current_dir = os.path.dirname(__file__)
     template_path = os.path.join(current_dir, "../../downloads/V7_TELLSamplesSpreadsheetTemplate.xlsx")
-
     if not os.path.exists(template_path):
         raise HTTPException(status_code=404, detail="Template file not found.")
-
+    return FileResponse(template_path, filename="template.xlsx",
-    return FileResponse(template_path, filename="V7_TELLSamplesSpreadsheetTemplate.xlsx",
                         media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
 
-
+@router.post("/upload", response_model=SpreadsheetResponse)
-@router.post("/upload")
 async def upload_file(file: UploadFile = File(...)):
     try:
         logger.info(f"Received file: {file.filename}")
 
-        # File type check
+        # Validate file type
         if not file.filename.endswith('.xlsx'):
             logger.error("Invalid file format")
             raise HTTPException(status_code=400, detail="Invalid file format. Please upload an .xlsx file.")
 
-        # Reading file
+        # Process spreadsheet
         importer = SampleSpreadsheetImporter()
-        validated_model = importer.import_spreadsheet(file)
+        validated_model, errors, raw_data = importer.import_spreadsheet_with_errors(file)
-        logger.info(f"Validated model: {validated_model}")
 
+        # Collect dewar, puck, and sample names
         dewars = {sample.dewarname for sample in validated_model if sample.dewarname}
         pucks = {sample.puckname for sample in validated_model if sample.puckname}
         samples = {sample.crystalname for sample in validated_model if sample.crystalname}
 
-        # Logging the sets of names
+        # Construct response data
-        logger.info(f"Dewar Names: {dewars}")
+        response_data = SpreadsheetResponse(
-        logger.info(f"Puck Names: {pucks}")
+            data=validated_model,
-        logger.info(f"Sample Names: {samples}")
+            errors=errors,
+            raw_data=raw_data,
+            dewars_count=len(dewars),
+            dewars=list(dewars),
+            pucks_count=len(pucks),
+            pucks=list(pucks),
+            samples_count=len(samples),
+            samples=list(samples)
+        )
 
-        # Forming structured response
+        logger.info(f"Returning response: {response_data.dict()}")
-        response = {
+        return response_data
-            "dewars_count": len(dewars),
-            "dewars": list(dewars),
-            "pucks_count": len(pucks),
-            "pucks": list(pucks),
-            "samples_count": len(samples),
-            "samples": list(samples)
-        }
-
-        logger.info(f"Returning response: {response}")
-        return response
     except SpreadsheetImportError as e:
         logger.error(f"Spreadsheet import error: {str(e)}")
         raise HTTPException(status_code=400, detail=str(e))

backend/app/sample_models.py

@@ -1,5 +1,6 @@
 import re
-from typing import Any, Optional
+from typing import Any, Optional, List, Dict
+
 from pydantic import BaseModel, Field, field_validator
 from typing_extensions import Annotated
 
@@ -267,4 +268,17 @@ class SpreadsheetModel(BaseModel):
             username: str
             puck_number: int
             prefix: Optional[str]
-            folder: Optional[str]
+            folder: Optional[str]
+
+class SpreadsheetResponse(BaseModel):
+    data: List[SpreadsheetModel]  # Validated data rows as SpreadsheetModel instances
+    errors: List[Dict[str, Any]]  # Errors encountered during validation
+    raw_data: List[Dict[str, Any]]  # Raw data extracted from the spreadsheet
+    dewars_count: int
+    dewars: List[str]
+    pucks_count: int
+    pucks: List[str]
+    samples_count: int
+    samples: List[str]
+
+__all__ = ['SpreadsheetModel', 'SpreadsheetResponse']

backend/app/services/spreadsheet_service.py

@@ -1,7 +1,9 @@
+# sample_spreadsheet_importer.py
+
 import logging
 import openpyxl
 from pydantic import ValidationError
-from typing import Union
+from typing import Union, List, Tuple
 from io import BytesIO
 from app.sample_models import SpreadsheetModel
 
@@ -40,6 +42,9 @@ class SampleSpreadsheetImporter:
         return value
 
     def import_spreadsheet(self, file):
+        return self.import_spreadsheet_with_errors(file)
+
+    def import_spreadsheet_with_errors(self, file) -> Tuple[List[SpreadsheetModel], List[dict], List[dict]]:
         self.model = []
         self.filename = file.filename
         logger.info(f"Importing spreadsheet from .xlsx file: {self.filename}")
@@ -64,8 +69,10 @@ class SampleSpreadsheetImporter:
 
         return self.process_spreadsheet(sheet)
 
-    def process_spreadsheet(self, sheet):
+    def process_spreadsheet(self, sheet) -> Tuple[List[SpreadsheetModel], List[dict], List[dict]]:
         model = []
+        errors = []
+        raw_data = []
 
         # Skip the first 3 rows
         rows = list(sheet.iter_rows(min_row=4, values_only=True))
@@ -82,6 +89,9 @@ class SampleSpreadsheetImporter:
                 logger.debug(f"Skipping empty row at index {index}")
                 continue
 
+            # Record raw data for later use
+            raw_data.append({"row_num": index + 4, "data": row})
+
             # Pad the row to ensure it has the expected number of columns
             if len(row) < expected_columns:
                 row = list(row) + [None] * (expected_columns - len(row))
@@ -126,10 +136,54 @@ class SampleSpreadsheetImporter:
                 model.append(validated_record)
                 logger.debug(f"Row {index + 4} processed and validated successfully")
             except ValidationError as e:
-                error_message = f"Validation error in row {index + 4}: {e}"
+                logger.error(f"Validation error in row {index + 4}: {e}")
-                logger.error(error_message)
+                for error in e.errors():
-                raise SpreadsheetImportError(error_message)
+                    field = error['loc'][0]
+                    msg = error['msg']
+                    # Map field name (which is the key in `record`) to its index in the row
+                    field_to_col = {
+                        'dewarname': 0,
+                        'puckname': 1,
+                        'pucktype': 2,
+                        'crystalname': 3,
+                        'positioninpuck': 4,
+                        'priority': 5,
+                        'comments': 6,
+                        'directory': 7,
+                        'proteinname': 8,
+                        'oscillation': 9,
+                        'aperture': 10,
+                        'exposure': 11,
+                        'totalrange': 12,
+                        'transmission': 13,
+                        'dose': 14,
+                        'targetresolution': 15,
+                        'datacollectiontype': 16,
+                        'processingpipeline': 17,
+                        'spacegroupnumber': 18,
+                        'cellparameters': 19,
+                        'rescutkey': 20,
+                        'rescutvalue': 21,
+                        'userresolution': 22,
+                        'pdbid': 23,
+                        'autoprocfull': 24,
+                        'procfull': 25,
+                        'adpenabled': 26,
+                        'noano': 27,
+                        'ffcscampaign': 28,
+                        'trustedhigh': 29,
+                        'autoprocextraparams': 30,
+                        'chiphiangles': 31
+                    }
+                    column_index = field_to_col[field]
+                    error_info = {
+                        'row': index + 4,
+                        'cell': column_index,
+                        'value': row[column_index],  # Value that caused the error
+                        'message': msg
+                    }
+                    errors.append(error_info)
 
         self.model = model
-        logger.info(f"Finished processing {len(model)} records")
+        logger.info(f"Finished processing {len(model)} records with {len(errors)} errors")
-        return self.model
+        return self.model, errors, raw_data

frontend/src/components/SpreadsheetTable.tsx

@@ -0,0 +1,43 @@
+import React from 'react';
+import { Table, TableBody, TableCell, TableContainer, TableHead, TableRow, Paper, Tooltip } from '@mui/material';
+
+const SpreadsheetTable = ({ raw_data, errors }) => {
+    const getErrorForCell = (rowIdx, colIdx) => {
+        return errors.find(e => e.row === rowIdx && e.cell === colIdx);
+    };
+
+    return (
+        <TableContainer component={Paper}>
+            <Table>
+                <TableHead>
+                    <TableRow>
+                        {raw_data.length > 0 && Object.keys(raw_data[0].data).map((col, colIdx) => (
+                            <TableCell key={colIdx}>{`Column ${colIdx + 1}`}</TableCell>
+                        ))}
+                    </TableRow>
+                </TableHead>
+                <TableBody>
+                    {raw_data.map((rowItem, rowIndex) => (
+                        <TableRow key={rowIndex}>
+                            {Object.values(rowItem.data).map((cellValue, cellIndex) => {
+                                const cellError = getErrorForCell(rowItem.row_num, cellIndex);
+                                return (
+                                    <TableCell
+                                        key={cellIndex}
+                                        style={{ backgroundColor: cellError ? 'red' : 'white' }}
+                                    >
+                                        <Tooltip title={cellError ? cellError.message : ''} arrow>
+                                            <span>{cellValue}</span>
+                                        </Tooltip>
+                                    </TableCell>
+                                )
+                            })}
+                        </TableRow>
+                    ))}
+                </TableBody>
+            </Table>
+        </TableContainer>
+    );
+};
+
+export default SpreadsheetTable;

frontend/src/components/UploadDialog.tsx

@@ -1,4 +1,4 @@
-import { useState, useEffect } from 'react';
+import React, { useState, useEffect } from 'react';
 import {
     Dialog,
     DialogTitle,
@@ -8,6 +8,7 @@ import {
     Typography,
     IconButton,
     Box,
+    Tooltip
 } from '@mui/material';
 import CloseIcon from '@mui/icons-material/Close';
 import DownloadIcon from '@mui/icons-material/Download';
@@ -15,6 +16,7 @@ import UploadFileIcon from '@mui/icons-material/UploadFile';
 import logo from '../assets/Heidi-logo.png';
 import { OpenAPI, SpreadsheetService } from '../../openapi';
 import type { Body_upload_file_upload_post } from '../../openapi/models/Body_upload_file_upload_post';
+import SpreadsheetTable from './SpreadsheetTable';  // Ensure the path is correct
 
 interface UploadDialogProps {
     open: boolean;
@@ -24,11 +26,15 @@ interface UploadDialogProps {
 const UploadDialog: React.FC<UploadDialogProps> = ({ open, onClose }) => {
     const [uploadError, setUploadError] = useState<string | null>(null);
     const [fileSummary, setFileSummary] = useState<{
+        data: any[];
+        errors: { row: number, cell: number, value: any, message: string }[];
+        raw_data: { row_num: number, data: any[] }[];
         dewars_count: number;
         dewars: string[];
         pucks_count: number;
         pucks: string[];
-        samples_count: string[];
+        samples_count: number;
+        samples: string[];
     } | null>(null);
 
     useEffect(() => {
@@ -41,30 +47,21 @@ const UploadDialog: React.FC<UploadDialogProps> = ({ open, onClose }) => {
             return;
         }
 
-        // Reset the previous state
         setUploadError(null);
         setFileSummary(null);
 
-        // Example file type check: only allow .xlsx files
         if (!file.name.endsWith('.xlsx')) {
             setUploadError('Invalid file format. Please upload an .xlsx file.');
             return;
         }
 
-        // Create the formData object compliant with the type definition
         const formData: Body_upload_file_upload_post = {
-            file: file, // TypeScript understands that file is a Blob
+            file: file,
         } as Body_upload_file_upload_post;
 
         try {
-            // Use the generated OpenAPI client UploadService method
             const response = await SpreadsheetService.uploadFileUploadPost(formData);
-
             console.log('File summary response from backend:', response);
-            console.log('Dewars:', response.dewars);
-            console.log('Pucks:', response.pucks);
-            console.log('Samples:', response.samples);
-
             setFileSummary(response);
         } catch (error) {
             console.error('File upload error:', error);
@@ -72,6 +69,13 @@ const UploadDialog: React.FC<UploadDialogProps> = ({ open, onClose }) => {
         }
     };
 
+    const handleRawDataChange = (updatedRawData) => {
+        setFileSummary((prevSummary) => ({
+            ...prevSummary,
+            raw_data: updatedRawData
+        }));
+    };
+
     return (
         <Dialog open={open} onClose={onClose} fullWidth maxWidth="sm">
             <DialogTitle>
@@ -85,8 +89,8 @@ const UploadDialog: React.FC<UploadDialogProps> = ({ open, onClose }) => {
             <DialogContent dividers>
                 <Box display="flex" flexDirection="column" alignItems="center" mb={2}>
                     <img src={logo} alt="Logo" style={{ width: 200, marginBottom: 16 }} />
-                    <Typography variant="subtitle1">Latest Spreadsheet Template Version 7</Typography>
+                    <Typography variant="subtitle1">Latest Spreadsheet Template Version 6</Typography>
-                    <Typography variant="body2" color="textSecondary">Last update: November 6, 2024</Typography>
+                    <Typography variant="body2" color="textSecondary">Last update: October 18, 2024</Typography>
                     <Button variant="outlined" startIcon={<DownloadIcon />} href="http://127.0.0.1:8000/download-template" download sx={{ mt: 1 }}>
                         Download XLSX
                     </Button>
@@ -112,18 +116,14 @@ const UploadDialog: React.FC<UploadDialogProps> = ({ open, onClose }) => {
                             <Typography variant="body1">
                                 <strong>File Summary:</strong>
                             </Typography>
+                            <SpreadsheetTable
+                                raw_data={fileSummary.raw_data}
+                                errors={fileSummary.errors}
+                                setRawData={handleRawDataChange}
+                            />
                             <Typography>Dewars: {fileSummary.dewars_count}</Typography>
                             <Typography>Pucks: {fileSummary.pucks_count}</Typography>
                             <Typography>Samples: {fileSummary.samples_count}</Typography>
-                            <Typography variant="body2">
-                                <strong>Dewar Names:</strong> {Array.isArray(fileSummary.dewars) ? fileSummary.dewars.join(', ') : 'N/A'}
-                            </Typography>
-                            <Typography variant="body2">
-                                <strong>Puck Names:</strong> {Array.isArray(fileSummary.pucks) ? fileSummary.pucks.join(', ') : 'N/A'}
-                            </Typography>
-                            <Typography variant="body2">
-                                <strong>Sample Names:</strong> {Array.isArray(fileSummary.samples) ? fileSummary.samples.join(', ') : 'N/A'}
-                            </Typography>
                         </Box>
                     )}
                 </Box>