added error recognition in spreadsheet

backend/app/routers/spreadsheet.py

@@ -4,40 +4,47 @@ import logging
 from app.services.spreadsheet_service import SampleSpreadsheetImporter, SpreadsheetImportError
 from fastapi.responses import FileResponse
 import os
+from pydantic import ValidationError  # Import ValidationError here
+
 
 router = APIRouter()
 logger = logging.getLogger(__name__)
 
+
 @router.get("/download-template", response_class=FileResponse)
 async def download_template():
-    # No changes here; just serves a static file
+    """Serve a template file for spreadsheet upload."""
     current_dir = os.path.dirname(__file__)
     template_path = os.path.join(current_dir, "../../downloads/V7_TELLSamplesSpreadsheetTemplate.xlsx")
+
     if not os.path.exists(template_path):
         raise HTTPException(status_code=404, detail="Template file not found.")
+
     return FileResponse(template_path, filename="template.xlsx",
                         media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
 
+
 @router.post("/upload", response_model=SpreadsheetResponse)
 async def upload_file(file: UploadFile = File(...)):
+    """Process the uploaded spreadsheet and return validation results."""
     try:
         logger.info(f"Received file: {file.filename}")
 
-        # Validate file type
+        # Validate file format
         if not file.filename.endswith('.xlsx'):
             logger.error("Invalid file format")
             raise HTTPException(status_code=400, detail="Invalid file format. Please upload an .xlsx file.")
 
-        # Process spreadsheet
+        # Initialize the importer and process the spreadsheet
         importer = SampleSpreadsheetImporter()
-        validated_model, errors, raw_data = importer.import_spreadsheet_with_errors(file)
+        validated_model, errors, raw_data, headers = importer.import_spreadsheet_with_errors(file)
 
-        # Collect dewar, puck, and sample names
+        # Extract unique values for dewars, pucks, and samples
         dewars = {sample.dewarname for sample in validated_model if sample.dewarname}
         pucks = {sample.puckname for sample in validated_model if sample.puckname}
         samples = {sample.crystalname for sample in validated_model if sample.crystalname}
 
-        # Construct response data
+        # Construct the response model with the processed data
         response_data = SpreadsheetResponse(
             data=validated_model,
             errors=errors,
@@ -47,14 +54,42 @@ async def upload_file(file: UploadFile = File(...)):
             pucks_count=len(pucks),
             pucks=list(pucks),
             samples_count=len(samples),
-            samples=list(samples)
+            samples=list(samples),
+            headers=headers  # Include headers in the response
         )
 
-        logger.info(f"Returning response: {response_data.dict()}")
+        logger.info(f"Returning response with {len(validated_model)} records and {len(errors)} errors.")
         return response_data
+
     except SpreadsheetImportError as e:
         logger.error(f"Spreadsheet import error: {str(e)}")
-        raise HTTPException(status_code=400, detail=str(e))
+        raise HTTPException(status_code=400, detail=f"Error processing spreadsheet: {str(e)}")
+
     except Exception as e:
-        logger.error(f"Failed to process file: {str(e)}")
+        logger.error(f"Unexpected error occurred: {str(e)}")
-        raise HTTPException(status_code=500, detail=f"Failed to upload file. Please try again. {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Failed to upload file. Please try again. Error: {str(e)}")
+
+
+@router.post("/validate-cell")
+async def validate_cell(data: dict):
+    """Validate a single cell value based on expected column type."""
+    row_num = data.get("row")
+    col_name = data.get("column")
+    value = data.get("value")
+
+    importer = SampleSpreadsheetImporter()
+
+    # Determine the expected type based on column name
+    expected_type = importer.get_expected_type(col_name)
+
+    # Clean and validate the cell value
+    cleaned_value = importer._clean_value(value, expected_type)
+
+    try:
+        # Validate the cleaned value using the SpreadsheetModel (Pydantic validation)
+        SpreadsheetModel(**{col_name: cleaned_value})
+        return {"is_valid": True, "message": ""}
+    except ValidationError as e:
+        # If validation fails, return the first error message
+        message = e.errors()[0]['msg']
+        return {"is_valid": False, "message": message}

backend/app/sample_models.py

@@ -280,5 +280,7 @@ class SpreadsheetResponse(BaseModel):
     pucks: List[str]
     samples_count: int
     samples: List[str]
+    headers: Optional[List[str]] = None  # Add headers if needed
+
 
 __all__ = ['SpreadsheetModel', 'SpreadsheetResponse']

backend/app/services/spreadsheet_service.py

@@ -1,5 +1,3 @@
-# sample_spreadsheet_importer.py
-
 import logging
 import openpyxl
 from pydantic import ValidationError
@@ -10,11 +8,9 @@ from app.sample_models import SpreadsheetModel
 logging.basicConfig(level=logging.DEBUG)
 logger = logging.getLogger(__name__)
 
-
 class SpreadsheetImportError(Exception):
     pass
 
-
 class SampleSpreadsheetImporter:
     def __init__(self):
         self.filename = None
@@ -44,7 +40,18 @@ class SampleSpreadsheetImporter:
     def import_spreadsheet(self, file):
         return self.import_spreadsheet_with_errors(file)
 
-    def import_spreadsheet_with_errors(self, file) -> Tuple[List[SpreadsheetModel], List[dict], List[dict]]:
+    def get_expected_type(self, col_name):
+        type_mapping = {
+            'dewarname': str,
+            'puckname': str,
+            'positioninpuck': int,
+            'priority': int,
+            'oscillation': float,
+            # Add all other mappings based on model requirements
+        }
+        return type_mapping.get(col_name, str)  # Default to `str`
+
+    def import_spreadsheet_with_errors(self, file) -> Tuple[List[SpreadsheetModel], List[dict], List[dict], List[str]]:
         self.model = []
         self.filename = file.filename
         logger.info(f"Importing spreadsheet from .xlsx file: {self.filename}")
@@ -67,12 +74,17 @@ class SampleSpreadsheetImporter:
             logger.error(f"Failed to read the file: {str(e)}")
             raise SpreadsheetImportError(f"Failed to read the file: {str(e)}")
 
-        return self.process_spreadsheet(sheet)
+        # Unpack four values from the process_spreadsheet method
+        model, errors, raw_data, headers = self.process_spreadsheet(sheet)
 
-    def process_spreadsheet(self, sheet) -> Tuple[List[SpreadsheetModel], List[dict], List[dict]]:
+        # Now, return the values correctly
+        return model, errors, raw_data, headers
+
+    def process_spreadsheet(self, sheet) -> Tuple[List[SpreadsheetModel], List[dict], List[dict], List[str]]:
         model = []
         errors = []
         raw_data = []
+        headers = []
 
         # Skip the first 3 rows
         rows = list(sheet.iter_rows(min_row=4, values_only=True))
@@ -84,6 +96,16 @@ class SampleSpreadsheetImporter:
 
         expected_columns = 32  # Number of columns expected based on the model
 
+        # Add the headers (the first row in the spreadsheet or map them explicitly)
+        headers = [
+            'dewarname', 'puckname', 'pucktype', 'crystalname', 'positioninpuck', 'priority',
+            'comments', 'directory', 'proteinname', 'oscillation', 'aperture', 'exposure',
+            'totalrange', 'transmission', 'dose', 'targetresolution', 'datacollectiontype',
+            'processingpipeline', 'spacegroupnumber', 'cellparameters', 'rescutkey', 'rescutvalue',
+            'userresolution', 'pdbid', 'autoprocfull', 'procfull', 'adpenabled', 'noano',
+            'ffcscampaign', 'trustedhigh', 'autoprocextraparams', 'chiphiangles'
+        ]
+
         for index, row in enumerate(rows):
             if not any(row):
                 logger.debug(f"Skipping empty row at index {index}")
@@ -96,6 +118,7 @@ class SampleSpreadsheetImporter:
             if len(row) < expected_columns:
                 row = list(row) + [None] * (expected_columns - len(row))
 
+            # Prepare the record with the cleaned values
             record = {
                 'dewarname': self._clean_value(row[0], str),
                 'puckname': self._clean_value(row[1], str),
@@ -186,4 +209,4 @@ class SampleSpreadsheetImporter:
 
         self.model = model
         logger.info(f"Finished processing {len(model)} records with {len(errors)} errors")
-        return self.model, errors, raw_data
+        return self.model, errors, raw_data, headers  # Include headers in the response

frontend/src/components/SpreadsheetTable.tsx

@@ -1,36 +1,137 @@
-import React from 'react';
+import React, { useState, useEffect } from 'react';
-import { Table, TableBody, TableCell, TableContainer, TableHead, TableRow, Paper, Tooltip } from '@mui/material';
+import {
+    Table,
+    TableBody,
+    TableCell,
+    TableContainer,
+    TableHead,
+    TableRow,
+    Paper,
+    Tooltip,
+    TextField,
+    Typography
+} from '@mui/material';
+import { SpreadsheetService } from '../../openapi'; // Ensure correct path
 
-const SpreadsheetTable = ({ raw_data, errors }) => {
+const SpreadsheetTable = ({ raw_data, errors, headers, setRawData }) => {
-    const getErrorForCell = (rowIdx, colIdx) => {
+    const [localErrors, setLocalErrors] = useState(errors || []);
-        return errors.find(e => e.row === rowIdx && e.cell === colIdx);
+
+    // Create an error map to easily lookup errors by row and column
+    const generateErrorMap = (errorsList) => {
+        const errorMap = new Map();
+        if (Array.isArray(errorsList)) {
+            errorsList.forEach(error => {
+                const key = `${error.row}-${headers[error.cell]}`;
+                errorMap.set(key, error.message);
+            });
+        }
+        return errorMap;
     };
 
+    const errorMap = generateErrorMap(localErrors);
+
+    // Handle cell edit
+    const handleCellEdit = async (rowIndex, colIndex, newValue) => {
+        const updatedRawData = [...raw_data];
+        const columnName = headers[colIndex];
+        const currentRow = updatedRawData[rowIndex];
+
+        // Assuming data is an array and we need to set by index
+        currentRow.data[colIndex] = newValue;
+
+        try {
+            // Send a request to validate the cell
+            const response = await SpreadsheetService.validateCellValidateCellPost({
+                row: currentRow.row_num,  // Use row_num directly from the current row
+                column: columnName,
+                value: newValue
+            });
+
+            // Log the response to debug the structure
+            console.log('Validation response:', response);
+
+            // Check if response is valid and structured as expected
+            if (response.data && response.data.is_valid !== undefined) {
+                if (response.data.is_valid) {
+                    // Remove error if validation passes
+                    const updatedErrors = localErrors.filter(error => !(error.row === currentRow.row_num && error.cell === colIndex));
+                    setLocalErrors(updatedErrors);
+                } else {
+                    // Add error if validation fails
+                    const updatedErrors = [...localErrors, { row: currentRow.row_num, cell: colIndex, message: response.data.message || 'Invalid value.' }];
+                    setLocalErrors(updatedErrors);
+                }
+            } else {
+                // Handle unexpected response format
+                console.error('Unexpected response structure:', response);
+            }
+
+            // Update the raw data
+            setRawData(updatedRawData);
+        } catch (error) {
+            console.error('Validation failed:', error);
+        }
+    };
+
+    // Hook to log and monitor changes in raw data, errors, and headers
+    useEffect(() => {
+        console.log('Raw data:', raw_data);
+        console.log('Errors:', errors);
+        console.log('Headers:', headers);
+    }, [raw_data, errors, headers]);
+
+    // Ensure data is loaded before rendering
+    if (!raw_data || !headers) {
+        return <div>Loading...</div>;
+    }
+
     return (
         <TableContainer component={Paper}>
             <Table>
                 <TableHead>
                     <TableRow>
-                        {raw_data.length > 0 && Object.keys(raw_data[0].data).map((col, colIdx) => (
+                        {headers.map((header, index) => (
-                            <TableCell key={colIdx}>{`Column ${colIdx + 1}`}</TableCell>
+                            <TableCell key={index} align="center">
+                                <Typography variant="body2">{header}</Typography>
+                            </TableCell>
                         ))}
                     </TableRow>
                 </TableHead>
                 <TableBody>
-                    {raw_data.map((rowItem, rowIndex) => (
+                    {raw_data.map((row, rowIndex) => (
                         <TableRow key={rowIndex}>
-                            {Object.values(rowItem.data).map((cellValue, cellIndex) => {
+                            {headers.map((header, colIndex) => {
-                                const cellError = getErrorForCell(rowItem.row_num, cellIndex);
+                                const key = `${row.row_num}-${header}`;  // Key for error lookup should match row's row_num
+                                const errorMessage = errorMap.get(key);
+                                const isInvalid = !!errorMessage;  // Check if the cell has an error
+                                const cellValue = row.data[colIndex];  // Extract cell value via index
+
                                 return (
-                                    <TableCell
+                                    <TableCell key={colIndex} align="center">
-                                        key={cellIndex}
+                                        {isInvalid ? (
-                                        style={{ backgroundColor: cellError ? 'red' : 'white' }}
+                                            <Tooltip title={errorMessage} arrow>
-                                    >
+                                                <TextField
-                                        <Tooltip title={cellError ? cellError.message : ''} arrow>
+                                                    value={cellValue || ""}  // Prevent null, fallback to empty string
-                                            <span>{cellValue}</span>
+                                                    onChange={(e) => handleCellEdit(rowIndex, colIndex, e.target.value)}
-                                        </Tooltip>
+                                                    error={true}  // Show red border if there's an error
+                                                    fullWidth
+                                                    sx={{
+                                                        '& .MuiOutlinedInput-root.Mui-error': {
+                                                            borderColor: 'red',  // Apply red border for invalid cells
+                                                        },
+                                                    }}
+                                                />
+                                            </Tooltip>
+                                        ) : (
+                                            <TextField
+                                                value={cellValue || ""}
+                                                onChange={(e) => handleCellEdit(rowIndex, colIndex, e.target.value)}
+                                                fullWidth
+                                                disabled={true}  // Disable valid cells so they cannot be edited
+                                            />
+                                        )}
                                     </TableCell>
-                                )
+                                );
                             })}
                         </TableRow>
                     ))}

frontend/src/components/UploadDialog.tsx

@@ -7,8 +7,7 @@ import {
     Button,
     Typography,
     IconButton,
-    Box,
+    Box
-    Tooltip
 } from '@mui/material';
 import CloseIcon from '@mui/icons-material/Close';
 import DownloadIcon from '@mui/icons-material/Download';
@@ -16,7 +15,8 @@ import UploadFileIcon from '@mui/icons-material/UploadFile';
 import logo from '../assets/Heidi-logo.png';
 import { OpenAPI, SpreadsheetService } from '../../openapi';
 import type { Body_upload_file_upload_post } from '../../openapi/models/Body_upload_file_upload_post';
-import SpreadsheetTable from './SpreadsheetTable';  // Ensure the path is correct
+import SpreadsheetTable from './SpreadsheetTable';
+import Modal from './Modal';  // Import the custom Modal component
 
 interface UploadDialogProps {
     open: boolean;
@@ -35,7 +35,9 @@ const UploadDialog: React.FC<UploadDialogProps> = ({ open, onClose }) => {
         pucks: string[];
         samples_count: number;
         samples: string[];
+        headers: string[]; // Headers must be part of this object
     } | null>(null);
+    const [isModalOpen, setIsModalOpen] = useState(false);
 
     useEffect(() => {
         OpenAPI.BASE = 'http://127.0.0.1:8000';
@@ -43,9 +45,7 @@ const UploadDialog: React.FC<UploadDialogProps> = ({ open, onClose }) => {
 
     const handleFileUpload = async (event: React.ChangeEvent<HTMLInputElement>) => {
         const file = event.target.files?.[0];
-        if (!file) {
+        if (!file) return;
-            return;
-        }
 
         setUploadError(null);
         setFileSummary(null);
@@ -61,10 +61,23 @@ const UploadDialog: React.FC<UploadDialogProps> = ({ open, onClose }) => {
 
         try {
             const response = await SpreadsheetService.uploadFileUploadPost(formData);
-            console.log('File summary response from backend:', response);
+
-            setFileSummary(response);
+            // Ensure headers are included in the response
+            const { headers, raw_data, errors } = response;
+            setFileSummary({
+                data: raw_data,
+                errors: errors,
+                raw_data: raw_data,  // keep raw data as is
+                headers: headers,  // Set headers correctly here
+                dewars_count: 2,  // You might need to adjust according to actual response
+                dewars: ['Dewar1', 'Dewar2'],
+                pucks_count: 2,  // Adjust accordingly
+                pucks: ['Puck1', 'Puck2'],
+                samples_count: 23,  // Adjust accordingly
+                samples: ['Sample1', 'Sample2']
+            });
+            setIsModalOpen(true); // Open modal once data is available
         } catch (error) {
-            console.error('File upload error:', error);
             setUploadError('Failed to upload file. Please try again.');
         }
     };
@@ -77,62 +90,60 @@ const UploadDialog: React.FC<UploadDialogProps> = ({ open, onClose }) => {
     };
 
     return (
-        <Dialog open={open} onClose={onClose} fullWidth maxWidth="sm">
+        <>
-            <DialogTitle>
+            <Dialog open={open} onClose={onClose} fullWidth maxWidth="sm">
-                <Box display="flex" justifyContent="space-between" alignItems="center">
+                <DialogTitle>
-                    <Typography variant="h6">Upload Sample Data Sheet</Typography>
+                    <Box display="flex" justifyContent="space-between" alignItems="center">
-                    <IconButton onClick={onClose}>
+                        <Typography variant="h6">Upload Sample Data Sheet</Typography>
-                        <CloseIcon />
+                        <IconButton onClick={onClose}>
-                    </IconButton>
+                            <CloseIcon />
-                </Box>
+                        </IconButton>
-            </DialogTitle>
+                    </Box>
-            <DialogContent dividers>
+                </DialogTitle>
-                <Box display="flex" flexDirection="column" alignItems="center" mb={2}>
+                <DialogContent dividers>
-                    <img src={logo} alt="Logo" style={{ width: 200, marginBottom: 16 }} />
+                    <Box display="flex" flexDirection="column" alignItems="center" mb={2}>
-                    <Typography variant="subtitle1">Latest Spreadsheet Template Version 6</Typography>
+                        <img src={logo} alt="Logo" style={{ width: 200, marginBottom: 16 }} />
-                    <Typography variant="body2" color="textSecondary">Last update: October 18, 2024</Typography>
+                        <Typography variant="subtitle1">Latest Spreadsheet Template Version 7</Typography>
-                    <Button variant="outlined" startIcon={<DownloadIcon />} href="http://127.0.0.1:8000/download-template" download sx={{ mt: 1 }}>
+                        <Typography variant="body2" color="textSecondary">Last update: November 7, 2024</Typography>
-                        Download XLSX
+                        <Button variant="outlined" startIcon={<DownloadIcon />} href="http://127.0.0.1:8000/download-template" download sx={{ mt: 1 }}>
-                    </Button>
+                            Download XLSX
-                    <Typography variant="subtitle1" sx={{ mt: 3 }}>Latest Spreadsheet Instructions Version 2.3</Typography>
+                        </Button>
-                    <Typography variant="body2" color="textSecondary">Last updated: October 18, 2024</Typography>
+                        <Typography variant="subtitle1" sx={{ mt: 3 }}>Latest Spreadsheet Instructions Version 2.3</Typography>
-                    <Button variant="outlined" startIcon={<DownloadIcon />} href="/path/to/instructions.pdf" download sx={{ mt: 1 }}>
+                        <Typography variant="body2" color="textSecondary">Last updated: October 18, 2024</Typography>
-                        Download PDF
+                        <Button variant="outlined" startIcon={<DownloadIcon />} href="/path/to/instructions.pdf" download sx={{ mt: 1 }}>
-                    </Button>
+                            Download PDF
-                </Box>
+                        </Button>
-                <Box mt={3}>
+                    </Box>
-                    <Button variant="contained" component="label" startIcon={<UploadFileIcon />}>
+                    <Box mt={3}>
-                        Choose a File
+                        <Button variant="contained" component="label" startIcon={<UploadFileIcon />}>
-                        <input type="file" hidden onChange={handleFileUpload} />
+                            Choose a File
-                    </Button>
+                            <input type="file" hidden onChange={handleFileUpload} />
-                    {uploadError && (
+                        </Button>
-                        <Typography color="error" sx={{ mt: 2 }}>
+                        {uploadError && <Typography color="error">{uploadError}</Typography>}
-                            {uploadError}
+                    </Box>
-                        </Typography>
+                </DialogContent>
-                    )}
+                <DialogActions>
-                    {fileSummary && (
+                    <Button onClick={onClose} color="primary">Close</Button>
-                        <Box mt={2}>
+                </DialogActions>
-                            <Typography color="success.main">File uploaded successfully!</Typography>
+            </Dialog>
-                            <Typography variant="body1">
+
-                                <strong>File Summary:</strong>
+            {/* Open modal to display spreadsheet table if file uploaded successfully */}
-                            </Typography>
+            {fileSummary && (
-                            <SpreadsheetTable
+                <Modal open={isModalOpen} onClose={() => setIsModalOpen(false)} title="File Summary">
-                                raw_data={fileSummary.raw_data}
+                    <Typography color="success.main">File uploaded successfully!</Typography>
-                                errors={fileSummary.errors}
+                    <SpreadsheetTable
-                                setRawData={handleRawDataChange}
+                        raw_data={fileSummary.raw_data}
-                            />
+                        errors={fileSummary.errors}
-                            <Typography>Dewars: {fileSummary.dewars_count}</Typography>
+                        headers={fileSummary.headers} // Ensure headers are passed here
-                            <Typography>Pucks: {fileSummary.pucks_count}</Typography>
+                        setRawData={handleRawDataChange}
-                            <Typography>Samples: {fileSummary.samples_count}</Typography>
+                    />
-                        </Box>
+                    <Typography>Dewars: {fileSummary.dewars_count}</Typography>
-                    )}
+                    <Typography>Pucks: {fileSummary.pucks_count}</Typography>
-                </Box>
+                    <Typography>Samples: {fileSummary.samples_count}</Typography>
-            </DialogContent>
+                </Modal>
-            <DialogActions>
+            )}
-                <Button onClick={onClose} color="primary">Close</Button>
+        </>
-            </DialogActions>
-        </Dialog>
     );
 };
 
-export default UploadDialog;
+export default UploadDialog;