gotthard_g/aaredb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added error recognition in spreadsheet

Browse Source
This commit is contained in:
GotthardG 2024-11-07 14:27:49 +01:00
parent 8f82a3b7fe
commit 501d09e6aa
5 changed files with 274 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
backend/app/routers/spreadsheet.py
View File

@ -4,40 +4,47 @@ import logging
from app.services.spreadsheet_service import SampleSpreadsheetImporter, SpreadsheetImportError
from fastapi.responses import FileResponse
import os
from pydantic import ValidationError # Import ValidationError here
router = APIRouter()
logger = logging.getLogger(__name__)
@router.get("/download-template", response_class=FileResponse)
async def download_template():
# No changes here; just serves a static file
"""Serve a template file for spreadsheet upload."""
current_dir = os.path.dirname(__file__)
template_path = os.path.join(current_dir, "../../downloads/V7_TELLSamplesSpreadsheetTemplate.xlsx")
if not os.path.exists(template_path):
raise HTTPException(status_code=404, detail="Template file not found.")
return FileResponse(template_path, filename="template.xlsx",
media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
@router.post("/upload", response_model=SpreadsheetResponse)
async def upload_file(file: UploadFile = File(...)):
"""Process the uploaded spreadsheet and return validation results."""
try:
logger.info(f"Received file: {file.filename}")
# Validate file type
# Validate file format
if not file.filename.endswith('.xlsx'):
logger.error("Invalid file format")
raise HTTPException(status_code=400, detail="Invalid file format. Please upload an .xlsx file.")
# Process spreadsheet
# Initialize the importer and process the spreadsheet
importer = SampleSpreadsheetImporter()
validated_model, errors, raw_data = importer.import_spreadsheet_with_errors(file)
validated_model, errors, raw_data, headers = importer.import_spreadsheet_with_errors(file)
# Collect dewar, puck, and sample names
# Extract unique values for dewars, pucks, and samples
dewars = {sample.dewarname for sample in validated_model if sample.dewarname}
pucks = {sample.puckname for sample in validated_model if sample.puckname}
samples = {sample.crystalname for sample in validated_model if sample.crystalname}
# Construct response data
# Construct the response model with the processed data
response_data = SpreadsheetResponse(
data=validated_model,
errors=errors,
@ -47,14 +54,42 @@ async def upload_file(file: UploadFile = File(...)):
pucks_count=len(pucks),
pucks=list(pucks),
samples_count=len(samples),
samples=list(samples)
samples=list(samples),
headers=headers # Include headers in the response
)
logger.info(f"Returning response: {response_data.dict()}")
logger.info(f"Returning response with {len(validated_model)} records and {len(errors)} errors.")
return response_data
except SpreadsheetImportError as e:
logger.error(f"Spreadsheet import error: {str(e)}")
raise HTTPException(status_code=400, detail=str(e))
raise HTTPException(status_code=400, detail=f"Error processing spreadsheet: {str(e)}")
except Exception as e:
logger.error(f"Failed to process file: {str(e)}")
raise HTTPException(status_code=500, detail=f"Failed to upload file. Please try again. {str(e)}")
logger.error(f"Unexpected error occurred: {str(e)}")
raise HTTPException(status_code=500, detail=f"Failed to upload file. Please try again. Error: {str(e)}")
@router.post("/validate-cell")
async def validate_cell(data: dict):
"""Validate a single cell value based on expected column type."""
row_num = data.get("row")
col_name = data.get("column")
value = data.get("value")
importer = SampleSpreadsheetImporter()
# Determine the expected type based on column name
expected_type = importer.get_expected_type(col_name)
# Clean and validate the cell value
cleaned_value = importer._clean_value(value, expected_type)
try:
# Validate the cleaned value using the SpreadsheetModel (Pydantic validation)
SpreadsheetModel(**{col_name: cleaned_value})
return {"is_valid": True, "message": ""}
except ValidationError as e:
# If validation fails, return the first error message
message = e.errors()[0]['msg']
return {"is_valid": False, "message": message}

2
backend/app/sample_models.py
View File

@ -280,5 +280,7 @@ class SpreadsheetResponse(BaseModel):
pucks: List[str]
samples_count: int
samples: List[str]
headers: Optional[List[str]] = None # Add headers if needed
__all__ = ['SpreadsheetModel', 'SpreadsheetResponse']

39
backend/app/services/spreadsheet_service.py
View File

@ -1,5 +1,3 @@
# sample_spreadsheet_importer.py
import logging
import openpyxl
from pydantic import ValidationError
@ -10,11 +8,9 @@ from app.sample_models import SpreadsheetModel
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
class SpreadsheetImportError(Exception):
pass
class SampleSpreadsheetImporter:
def __init__(self):
self.filename = None
@ -44,7 +40,18 @@ class SampleSpreadsheetImporter:
def import_spreadsheet(self, file):
return self.import_spreadsheet_with_errors(file)
def import_spreadsheet_with_errors(self, file) -> Tuple[List[SpreadsheetModel], List[dict], List[dict]]:
def get_expected_type(self, col_name):
type_mapping = {
'dewarname': str,
'puckname': str,
'positioninpuck': int,
'priority': int,
'oscillation': float,
# Add all other mappings based on model requirements
}
return type_mapping.get(col_name, str) # Default to `str`
def import_spreadsheet_with_errors(self, file) -> Tuple[List[SpreadsheetModel], List[dict], List[dict], List[str]]:
self.model = []
self.filename = file.filename
logger.info(f"Importing spreadsheet from .xlsx file: {self.filename}")
@ -67,12 +74,17 @@ class SampleSpreadsheetImporter:
logger.error(f"Failed to read the file: {str(e)}")
raise SpreadsheetImportError(f"Failed to read the file: {str(e)}")
return self.process_spreadsheet(sheet)
# Unpack four values from the process_spreadsheet method
model, errors, raw_data, headers = self.process_spreadsheet(sheet)
def process_spreadsheet(self, sheet) -> Tuple[List[SpreadsheetModel], List[dict], List[dict]]:
# Now, return the values correctly
return model, errors, raw_data, headers
def process_spreadsheet(self, sheet) -> Tuple[List[SpreadsheetModel], List[dict], List[dict], List[str]]:
model = []
errors = []
raw_data = []
headers = []
# Skip the first 3 rows
rows = list(sheet.iter_rows(min_row=4, values_only=True))
@ -84,6 +96,16 @@ class SampleSpreadsheetImporter:
expected_columns = 32 # Number of columns expected based on the model
# Add the headers (the first row in the spreadsheet or map them explicitly)
headers = [
'dewarname', 'puckname', 'pucktype', 'crystalname', 'positioninpuck', 'priority',
'comments', 'directory', 'proteinname', 'oscillation', 'aperture', 'exposure',
'totalrange', 'transmission', 'dose', 'targetresolution', 'datacollectiontype',
'processingpipeline', 'spacegroupnumber', 'cellparameters', 'rescutkey', 'rescutvalue',
'userresolution', 'pdbid', 'autoprocfull', 'procfull', 'adpenabled', 'noano',
'ffcscampaign', 'trustedhigh', 'autoprocextraparams', 'chiphiangles'
]
for index, row in enumerate(rows):
if not any(row):
logger.debug(f"Skipping empty row at index {index}")
@ -96,6 +118,7 @@ class SampleSpreadsheetImporter:
if len(row) < expected_columns:
row = list(row) + [None] * (expected_columns - len(row))
# Prepare the record with the cleaned values
record = {
'dewarname': self._clean_value(row[0], str),
'puckname': self._clean_value(row[1], str),
@ -186,4 +209,4 @@ class SampleSpreadsheetImporter:
self.model = model
logger.info(f"Finished processing {len(model)} records with {len(errors)} errors")
return self.model, errors, raw_data
return self.model, errors, raw_data, headers # Include headers in the response

135
frontend/src/components/SpreadsheetTable.tsx
View File

@ -1,36 +1,137 @@
import React from 'react';
import { Table, TableBody, TableCell, TableContainer, TableHead, TableRow, Paper, Tooltip } from '@mui/material';
import React, { useState, useEffect } from 'react';
import {
Table,
TableBody,
TableCell,
TableContainer,
TableHead,
TableRow,
Paper,
Tooltip,
TextField,
Typography
} from '@mui/material';
import { SpreadsheetService } from '../../openapi'; // Ensure correct path
const SpreadsheetTable = ({ raw_data, errors }) => {
const getErrorForCell = (rowIdx, colIdx) => {
return errors.find(e => e.row === rowIdx && e.cell === colIdx);
const SpreadsheetTable = ({ raw_data, errors, headers, setRawData }) => {
const [localErrors, setLocalErrors] = useState(errors || []);
// Create an error map to easily lookup errors by row and column
const generateErrorMap = (errorsList) => {
const errorMap = new Map();
if (Array.isArray(errorsList)) {
errorsList.forEach(error => {
const key = `${error.row}-${headers[error.cell]}`;
errorMap.set(key, error.message);
});
}
return errorMap;
};
const errorMap = generateErrorMap(localErrors);
// Handle cell edit
const handleCellEdit = async (rowIndex, colIndex, newValue) => {
const updatedRawData = [...raw_data];
const columnName = headers[colIndex];
const currentRow = updatedRawData[rowIndex];
// Assuming data is an array and we need to set by index
currentRow.data[colIndex] = newValue;
try {
// Send a request to validate the cell
const response = await SpreadsheetService.validateCellValidateCellPost({
row: currentRow.row_num, // Use row_num directly from the current row
column: columnName,
value: newValue
});
// Log the response to debug the structure
console.log('Validation response:', response);
// Check if response is valid and structured as expected
if (response.data && response.data.is_valid !== undefined) {
if (response.data.is_valid) {
// Remove error if validation passes
const updatedErrors = localErrors.filter(error => !(error.row === currentRow.row_num && error.cell === colIndex));
setLocalErrors(updatedErrors);
} else {
// Add error if validation fails
const updatedErrors = [...localErrors, { row: currentRow.row_num, cell: colIndex, message: response.data.message || 'Invalid value.' }];
setLocalErrors(updatedErrors);
}
} else {
// Handle unexpected response format
console.error('Unexpected response structure:', response);
}
// Update the raw data
setRawData(updatedRawData);
} catch (error) {
console.error('Validation failed:', error);
}
};
// Hook to log and monitor changes in raw data, errors, and headers
useEffect(() => {
console.log('Raw data:', raw_data);
console.log('Errors:', errors);
console.log('Headers:', headers);
}, [raw_data, errors, headers]);
// Ensure data is loaded before rendering
if (!raw_data || !headers) {
return <div>Loading...</div>;
}
return (
<TableContainer component={Paper}>
<Table>
<TableHead>
<TableRow>
{raw_data.length > 0 && Object.keys(raw_data[0].data).map((col, colIdx) => (
<TableCell key={colIdx}>{`Column ${colIdx + 1}`}</TableCell>
{headers.map((header, index) => (
<TableCell key={index} align="center">
<Typography variant="body2">{header}</Typography>
</TableCell>
))}
</TableRow>
</TableHead>
<TableBody>
{raw_data.map((rowItem, rowIndex) => (
{raw_data.map((row, rowIndex) => (
<TableRow key={rowIndex}>
{Object.values(rowItem.data).map((cellValue, cellIndex) => {
const cellError = getErrorForCell(rowItem.row_num, cellIndex);
{headers.map((header, colIndex) => {
const key = `${row.row_num}-${header}`; // Key for error lookup should match row's row_num
const errorMessage = errorMap.get(key);
const isInvalid = !!errorMessage; // Check if the cell has an error
const cellValue = row.data[colIndex]; // Extract cell value via index
return (
<TableCell
key={cellIndex}
style={{ backgroundColor: cellError ? 'red' : 'white' }}
>
<Tooltip title={cellError ? cellError.message : ''} arrow>
<span>{cellValue}</span>
<TableCell key={colIndex} align="center">
{isInvalid ? (
<Tooltip title={errorMessage} arrow>
<TextField
value={cellValue || ""} // Prevent null, fallback to empty string
onChange={(e) => handleCellEdit(rowIndex, colIndex, e.target.value)}
error={true} // Show red border if there's an error
fullWidth
sx={{
'& .MuiOutlinedInput-root.Mui-error': {
borderColor: 'red', // Apply red border for invalid cells
},
}}
/>
</Tooltip>
) : (
<TextField
value={cellValue || ""}
onChange={(e) => handleCellEdit(rowIndex, colIndex, e.target.value)}
fullWidth
disabled={true} // Disable valid cells so they cannot be edited
/>
)}
</TableCell>
)
);
})}
</TableRow>
))}

75
frontend/src/components/UploadDialog.tsx
View File

@ -7,8 +7,7 @@ import {
Button,
Typography,
IconButton,
Box,
Tooltip
Box
} from '@mui/material';
import CloseIcon from '@mui/icons-material/Close';
import DownloadIcon from '@mui/icons-material/Download';
@ -16,7 +15,8 @@ import UploadFileIcon from '@mui/icons-material/UploadFile';
import logo from '../assets/Heidi-logo.png';
import { OpenAPI, SpreadsheetService } from '../../openapi';
import type { Body_upload_file_upload_post } from '../../openapi/models/Body_upload_file_upload_post';
import SpreadsheetTable from './SpreadsheetTable'; // Ensure the path is correct
import SpreadsheetTable from './SpreadsheetTable';
import Modal from './Modal'; // Import the custom Modal component
interface UploadDialogProps {
open: boolean;
@ -35,7 +35,9 @@ const UploadDialog: React.FC<UploadDialogProps> = ({ open, onClose }) => {
pucks: string[];
samples_count: number;
samples: string[];
headers: string[]; // Headers must be part of this object
} | null>(null);
const [isModalOpen, setIsModalOpen] = useState(false);
useEffect(() => {
OpenAPI.BASE = 'http://127.0.0.1:8000';
@ -43,9 +45,7 @@ const UploadDialog: React.FC<UploadDialogProps> = ({ open, onClose }) => {
const handleFileUpload = async (event: React.ChangeEvent<HTMLInputElement>) => {
const file = event.target.files?.[0];
if (!file) {
return;
}
if (!file) return;
setUploadError(null);
setFileSummary(null);
@ -61,10 +61,23 @@ const UploadDialog: React.FC<UploadDialogProps> = ({ open, onClose }) => {
try {
const response = await SpreadsheetService.uploadFileUploadPost(formData);
console.log('File summary response from backend:', response);
setFileSummary(response);
// Ensure headers are included in the response
const { headers, raw_data, errors } = response;
setFileSummary({
data: raw_data,
errors: errors,
raw_data: raw_data, // keep raw data as is
headers: headers, // Set headers correctly here
dewars_count: 2, // You might need to adjust according to actual response
dewars: ['Dewar1', 'Dewar2'],
pucks_count: 2, // Adjust accordingly
pucks: ['Puck1', 'Puck2'],
samples_count: 23, // Adjust accordingly
samples: ['Sample1', 'Sample2']
});
setIsModalOpen(true); // Open modal once data is available
} catch (error) {
console.error('File upload error:', error);
setUploadError('Failed to upload file. Please try again.');
}
};
@ -77,6 +90,7 @@ const UploadDialog: React.FC<UploadDialogProps> = ({ open, onClose }) => {
};
return (
<>
<Dialog open={open} onClose={onClose} fullWidth maxWidth="sm">
<DialogTitle>
<Box display="flex" justifyContent="space-between" alignItems="center">
@ -89,8 +103,8 @@ const UploadDialog: React.FC<UploadDialogProps> = ({ open, onClose }) => {
<DialogContent dividers>
<Box display="flex" flexDirection="column" alignItems="center" mb={2}>
<img src={logo} alt="Logo" style={{ width: 200, marginBottom: 16 }} />
<Typography variant="subtitle1">Latest Spreadsheet Template Version 6</Typography>
<Typography variant="body2" color="textSecondary">Last update: October 18, 2024</Typography>
<Typography variant="subtitle1">Latest Spreadsheet Template Version 7</Typography>
<Typography variant="body2" color="textSecondary">Last update: November 7, 2024</Typography>
<Button variant="outlined" startIcon={<DownloadIcon />} href="http://127.0.0.1:8000/download-template" download sx={{ mt: 1 }}>
Download XLSX
</Button>
@ -105,33 +119,30 @@ const UploadDialog: React.FC<UploadDialogProps> = ({ open, onClose }) => {
Choose a File
<input type="file" hidden onChange={handleFileUpload} />
</Button>
{uploadError && (
<Typography color="error" sx={{ mt: 2 }}>
{uploadError}
</Typography>
)}
{fileSummary && (
<Box mt={2}>
<Typography color="success.main">File uploaded successfully!</Typography>
<Typography variant="body1">
<strong>File Summary:</strong>
</Typography>
<SpreadsheetTable
raw_data={fileSummary.raw_data}
errors={fileSummary.errors}
setRawData={handleRawDataChange}
/>
<Typography>Dewars: {fileSummary.dewars_count}</Typography>
<Typography>Pucks: {fileSummary.pucks_count}</Typography>
<Typography>Samples: {fileSummary.samples_count}</Typography>
</Box>
)}
{uploadError && <Typography color="error">{uploadError}</Typography>}
</Box>
</DialogContent>
<DialogActions>
<Button onClick={onClose} color="primary">Close</Button>
</DialogActions>
</Dialog>
{/* Open modal to display spreadsheet table if file uploaded successfully */}
{fileSummary && (
<Modal open={isModalOpen} onClose={() => setIsModalOpen(false)} title="File Summary">
<Typography color="success.main">File uploaded successfully!</Typography>
<SpreadsheetTable
raw_data={fileSummary.raw_data}
errors={fileSummary.errors}
headers={fileSummary.headers} // Ensure headers are passed here
setRawData={handleRawDataChange}
/>
<Typography>Dewars: {fileSummary.dewars_count}</Typography>
<Typography>Pucks: {fileSummary.pucks_count}</Typography>
<Typography>Samples: {fileSummary.samples_count}</Typography>
</Modal>
)}
</>
);
};