From 8f82a3b7fe44baff76f376f520a91f68f7da2b93 Mon Sep 17 00:00:00 2001
From: GotthardG <51994228+GotthardG@users.noreply.github.com>
Date: Thu, 7 Nov 2024 10:10:53 +0100
Subject: [PATCH] added error recognition in spreadsheet
---
backend/app/routers/spreadsheet.py | 49 +++++++-------
backend/app/sample_models.py | 18 +++++-
backend/app/services/spreadsheet_service.py | 68 ++++++++++++++++++--
frontend/src/components/SpreadsheetTable.tsx | 43 +++++++++++++
frontend/src/components/UploadDialog.tsx | 46 ++++++-------
5 files changed, 165 insertions(+), 59 deletions(-)
create mode 100644 frontend/src/components/SpreadsheetTable.tsx
diff --git a/backend/app/routers/spreadsheet.py b/backend/app/routers/spreadsheet.py
index 0c2e9a3..0808e0b 100644
--- a/backend/app/routers/spreadsheet.py
+++ b/backend/app/routers/spreadsheet.py
@@ -1,3 +1,4 @@
+from app.sample_models import SpreadsheetModel, SpreadsheetResponse
from fastapi import APIRouter, UploadFile, File, HTTPException
import logging
from app.services.spreadsheet_service import SampleSpreadsheetImporter, SpreadsheetImportError
@@ -7,56 +8,50 @@ import os
router = APIRouter()
logger = logging.getLogger(__name__)
-
@router.get("/download-template", response_class=FileResponse)
async def download_template():
- # Constructing the absolute path
+ # No changes here; just serves a static file
current_dir = os.path.dirname(__file__)
template_path = os.path.join(current_dir, "../../downloads/V7_TELLSamplesSpreadsheetTemplate.xlsx")
-
if not os.path.exists(template_path):
raise HTTPException(status_code=404, detail="Template file not found.")
-
- return FileResponse(template_path, filename="V7_TELLSamplesSpreadsheetTemplate.xlsx",
+ return FileResponse(template_path, filename="template.xlsx",
media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
-
-@router.post("/upload")
+@router.post("/upload", response_model=SpreadsheetResponse)
async def upload_file(file: UploadFile = File(...)):
try:
logger.info(f"Received file: {file.filename}")
- # File type check
+ # Validate file type
if not file.filename.endswith('.xlsx'):
logger.error("Invalid file format")
raise HTTPException(status_code=400, detail="Invalid file format. Please upload an .xlsx file.")
- # Reading file
+ # Process spreadsheet
importer = SampleSpreadsheetImporter()
- validated_model = importer.import_spreadsheet(file)
- logger.info(f"Validated model: {validated_model}")
+ validated_model, errors, raw_data = importer.import_spreadsheet_with_errors(file)
+ # Collect dewar, puck, and sample names
dewars = {sample.dewarname for sample in validated_model if sample.dewarname}
pucks = {sample.puckname for sample in validated_model if sample.puckname}
samples = {sample.crystalname for sample in validated_model if sample.crystalname}
- # Logging the sets of names
- logger.info(f"Dewar Names: {dewars}")
- logger.info(f"Puck Names: {pucks}")
- logger.info(f"Sample Names: {samples}")
+ # Construct response data
+ response_data = SpreadsheetResponse(
+ data=validated_model,
+ errors=errors,
+ raw_data=raw_data,
+ dewars_count=len(dewars),
+ dewars=list(dewars),
+ pucks_count=len(pucks),
+ pucks=list(pucks),
+ samples_count=len(samples),
+ samples=list(samples)
+ )
- # Forming structured response
- response = {
- "dewars_count": len(dewars),
- "dewars": list(dewars),
- "pucks_count": len(pucks),
- "pucks": list(pucks),
- "samples_count": len(samples),
- "samples": list(samples)
- }
-
- logger.info(f"Returning response: {response}")
- return response
+ logger.info(f"Returning response: {response_data.dict()}")
+ return response_data
except SpreadsheetImportError as e:
logger.error(f"Spreadsheet import error: {str(e)}")
raise HTTPException(status_code=400, detail=str(e))
diff --git a/backend/app/sample_models.py b/backend/app/sample_models.py
index ffa5b1d..3fcc3e3 100644
--- a/backend/app/sample_models.py
+++ b/backend/app/sample_models.py
@@ -1,5 +1,6 @@
import re
-from typing import Any, Optional
+from typing import Any, Optional, List, Dict
+
from pydantic import BaseModel, Field, field_validator
from typing_extensions import Annotated
@@ -267,4 +268,17 @@ class SpreadsheetModel(BaseModel):
username: str
puck_number: int
prefix: Optional[str]
- folder: Optional[str]
\ No newline at end of file
+ folder: Optional[str]
+
+class SpreadsheetResponse(BaseModel):
+ data: List[SpreadsheetModel] # Validated data rows as SpreadsheetModel instances
+ errors: List[Dict[str, Any]] # Errors encountered during validation
+ raw_data: List[Dict[str, Any]] # Raw data extracted from the spreadsheet
+ dewars_count: int
+ dewars: List[str]
+ pucks_count: int
+ pucks: List[str]
+ samples_count: int
+ samples: List[str]
+
+__all__ = ['SpreadsheetModel', 'SpreadsheetResponse']
diff --git a/backend/app/services/spreadsheet_service.py b/backend/app/services/spreadsheet_service.py
index 0ef4562..d2ad81b 100644
--- a/backend/app/services/spreadsheet_service.py
+++ b/backend/app/services/spreadsheet_service.py
@@ -1,7 +1,9 @@
+# sample_spreadsheet_importer.py
+
import logging
import openpyxl
from pydantic import ValidationError
-from typing import Union
+from typing import Union, List, Tuple
from io import BytesIO
from app.sample_models import SpreadsheetModel
@@ -40,6 +42,9 @@ class SampleSpreadsheetImporter:
return value
def import_spreadsheet(self, file):
+ return self.import_spreadsheet_with_errors(file)
+
+ def import_spreadsheet_with_errors(self, file) -> Tuple[List[SpreadsheetModel], List[dict], List[dict]]:
self.model = []
self.filename = file.filename
logger.info(f"Importing spreadsheet from .xlsx file: {self.filename}")
@@ -64,8 +69,10 @@ class SampleSpreadsheetImporter:
return self.process_spreadsheet(sheet)
- def process_spreadsheet(self, sheet):
+ def process_spreadsheet(self, sheet) -> Tuple[List[SpreadsheetModel], List[dict], List[dict]]:
model = []
+ errors = []
+ raw_data = []
# Skip the first 3 rows
rows = list(sheet.iter_rows(min_row=4, values_only=True))
@@ -82,6 +89,9 @@ class SampleSpreadsheetImporter:
logger.debug(f"Skipping empty row at index {index}")
continue
+ # Record raw data for later use
+ raw_data.append({"row_num": index + 4, "data": row})
+
# Pad the row to ensure it has the expected number of columns
if len(row) < expected_columns:
row = list(row) + [None] * (expected_columns - len(row))
@@ -126,10 +136,54 @@ class SampleSpreadsheetImporter:
model.append(validated_record)
logger.debug(f"Row {index + 4} processed and validated successfully")
except ValidationError as e:
- error_message = f"Validation error in row {index + 4}: {e}"
- logger.error(error_message)
- raise SpreadsheetImportError(error_message)
+ logger.error(f"Validation error in row {index + 4}: {e}")
+ for error in e.errors():
+ field = error['loc'][0]
+ msg = error['msg']
+ # Map field name (which is the key in `record`) to its index in the row
+ field_to_col = {
+ 'dewarname': 0,
+ 'puckname': 1,
+ 'pucktype': 2,
+ 'crystalname': 3,
+ 'positioninpuck': 4,
+ 'priority': 5,
+ 'comments': 6,
+ 'directory': 7,
+ 'proteinname': 8,
+ 'oscillation': 9,
+ 'aperture': 10,
+ 'exposure': 11,
+ 'totalrange': 12,
+ 'transmission': 13,
+ 'dose': 14,
+ 'targetresolution': 15,
+ 'datacollectiontype': 16,
+ 'processingpipeline': 17,
+ 'spacegroupnumber': 18,
+ 'cellparameters': 19,
+ 'rescutkey': 20,
+ 'rescutvalue': 21,
+ 'userresolution': 22,
+ 'pdbid': 23,
+ 'autoprocfull': 24,
+ 'procfull': 25,
+ 'adpenabled': 26,
+ 'noano': 27,
+ 'ffcscampaign': 28,
+ 'trustedhigh': 29,
+ 'autoprocextraparams': 30,
+ 'chiphiangles': 31
+ }
+ column_index = field_to_col[field]
+ error_info = {
+ 'row': index + 4,
+ 'cell': column_index,
+ 'value': row[column_index], # Value that caused the error
+ 'message': msg
+ }
+ errors.append(error_info)
self.model = model
- logger.info(f"Finished processing {len(model)} records")
- return self.model
+ logger.info(f"Finished processing {len(model)} records with {len(errors)} errors")
+ return self.model, errors, raw_data
diff --git a/frontend/src/components/SpreadsheetTable.tsx b/frontend/src/components/SpreadsheetTable.tsx
new file mode 100644
index 0000000..adb088d
--- /dev/null
+++ b/frontend/src/components/SpreadsheetTable.tsx
@@ -0,0 +1,43 @@
+import React from 'react';
+import { Table, TableBody, TableCell, TableContainer, TableHead, TableRow, Paper, Tooltip } from '@mui/material';
+
+const SpreadsheetTable = ({ raw_data, errors }) => {
+ const getErrorForCell = (rowIdx, colIdx) => {
+ return errors.find(e => e.row === rowIdx && e.cell === colIdx);
+ };
+
+ return (
+
+
+
+
+ {raw_data.length > 0 && Object.keys(raw_data[0].data).map((col, colIdx) => (
+ {`Column ${colIdx + 1}`}
+ ))}
+
+
+
+ {raw_data.map((rowItem, rowIndex) => (
+
+ {Object.values(rowItem.data).map((cellValue, cellIndex) => {
+ const cellError = getErrorForCell(rowItem.row_num, cellIndex);
+ return (
+
+
+ {cellValue}
+
+
+ )
+ })}
+
+ ))}
+
+
+
+ );
+};
+
+export default SpreadsheetTable;
\ No newline at end of file
diff --git a/frontend/src/components/UploadDialog.tsx b/frontend/src/components/UploadDialog.tsx
index ffa3da6..97a376b 100644
--- a/frontend/src/components/UploadDialog.tsx
+++ b/frontend/src/components/UploadDialog.tsx
@@ -1,4 +1,4 @@
-import { useState, useEffect } from 'react';
+import React, { useState, useEffect } from 'react';
import {
Dialog,
DialogTitle,
@@ -8,6 +8,7 @@ import {
Typography,
IconButton,
Box,
+ Tooltip
} from '@mui/material';
import CloseIcon from '@mui/icons-material/Close';
import DownloadIcon from '@mui/icons-material/Download';
@@ -15,6 +16,7 @@ import UploadFileIcon from '@mui/icons-material/UploadFile';
import logo from '../assets/Heidi-logo.png';
import { OpenAPI, SpreadsheetService } from '../../openapi';
import type { Body_upload_file_upload_post } from '../../openapi/models/Body_upload_file_upload_post';
+import SpreadsheetTable from './SpreadsheetTable'; // Ensure the path is correct
interface UploadDialogProps {
open: boolean;
@@ -24,11 +26,15 @@ interface UploadDialogProps {
const UploadDialog: React.FC = ({ open, onClose }) => {
const [uploadError, setUploadError] = useState(null);
const [fileSummary, setFileSummary] = useState<{
+ data: any[];
+ errors: { row: number, cell: number, value: any, message: string }[];
+ raw_data: { row_num: number, data: any[] }[];
dewars_count: number;
dewars: string[];
pucks_count: number;
pucks: string[];
- samples_count: string[];
+ samples_count: number;
+ samples: string[];
} | null>(null);
useEffect(() => {
@@ -41,30 +47,21 @@ const UploadDialog: React.FC = ({ open, onClose }) => {
return;
}
- // Reset the previous state
setUploadError(null);
setFileSummary(null);
- // Example file type check: only allow .xlsx files
if (!file.name.endsWith('.xlsx')) {
setUploadError('Invalid file format. Please upload an .xlsx file.');
return;
}
- // Create the formData object compliant with the type definition
const formData: Body_upload_file_upload_post = {
- file: file, // TypeScript understands that file is a Blob
+ file: file,
} as Body_upload_file_upload_post;
try {
- // Use the generated OpenAPI client UploadService method
const response = await SpreadsheetService.uploadFileUploadPost(formData);
-
console.log('File summary response from backend:', response);
- console.log('Dewars:', response.dewars);
- console.log('Pucks:', response.pucks);
- console.log('Samples:', response.samples);
-
setFileSummary(response);
} catch (error) {
console.error('File upload error:', error);
@@ -72,6 +69,13 @@ const UploadDialog: React.FC = ({ open, onClose }) => {
}
};
+ const handleRawDataChange = (updatedRawData) => {
+ setFileSummary((prevSummary) => ({
+ ...prevSummary,
+ raw_data: updatedRawData
+ }));
+ };
+
return (