diff --git a/backend/__init__.py b/backend/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/main.py b/backend/app/main.py index c23fd08..5a3bea2 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -3,7 +3,7 @@ from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from app.routers import address, contact, proposal, dewar, shipment, upload, puck +from app.routers import address, contact, proposal, dewar, shipment, upload, puck, spreadsheet from app.database import Base, engine, SessionLocal, load_sample_data app = FastAPI() @@ -17,6 +17,7 @@ app.add_middleware( allow_headers=["*"], ) + @app.on_event("startup") def on_startup(): # Drop and recreate database schema @@ -29,15 +30,18 @@ def on_startup(): finally: db.close() + # Include routers with correct configuration app.include_router(contact.router, prefix="/contacts", tags=["contacts"]) app.include_router(address.router, prefix="/addresses", tags=["addresses"]) app.include_router(proposal.router, prefix="/proposals", tags=["proposals"]) app.include_router(dewar.router, prefix="/dewars", tags=["dewars"]) app.include_router(shipment.router, prefix="/shipments", tags=["shipments"]) -app.include_router(upload.router, tags=["upload"]) # Removed the trailing '/' from the prefix +app.include_router(upload.router, tags=["upload"]) app.include_router(puck.router, prefix="/pucks", tags=["pucks"]) +app.include_router(spreadsheet.router, tags=["spreadsheet"]) if __name__ == "__main__": import uvicorn - uvicorn.run(app, host="127.0.0.1", port=8000, log_level="debug") \ No newline at end of file + + uvicorn.run(app, host="127.0.0.1", port=8000, log_level="debug") diff --git a/backend/app/routers/spreadsheet.py b/backend/app/routers/spreadsheet.py new file mode 100644 index 0000000..09da6d2 --- /dev/null +++ b/backend/app/routers/spreadsheet.py @@ -0,0 +1,59 @@ +# app/routers/spreadsheet.py + +from fastapi import APIRouter, UploadFile, File, HTTPException +from app.services.spreadsheet_service import SampleSpreadsheetImporter, SpreadsheetImportError +import logging + +router = APIRouter() +logger = logging.getLogger(__name__) + + +@router.post("/upload") +async def upload_file(file: UploadFile = File(...)): + importer = SampleSpreadsheetImporter() + try: + result = importer.import_spreadsheet(file) + + if not result: + logger.warning("No data extracted from spreadsheet.") + return { + "dewars_count": 0, + "dewars": [], + "pucks_count": 0, + "pucks": [], + "samples_count": 0, + "samples": [] + } + + # Logging the raw results for debugging. + logger.info(f"Extracted Result: {result}") + + # Extract and respond with detailed information. + dewars = list(set(sample['dewarname'] for sample in result)) + pucks = list(set(sample['puckname'] for sample in result)) + samples = list(set(sample['crystalname'] for sample in result)) + + # Log the extracted names. + logger.info(f"Dewars: {dewars}") + logger.info(f"Pucks: {pucks}") + logger.info(f"Samples: {samples}") + + response_data = { + "dewars_count": len(dewars), + "dewars": dewars, + "pucks_count": len(pucks), + "pucks": pucks, + "samples_count": len(samples), + "samples": samples, # Ensure lists include detailed names + "dewar_names": dewars, # Redundant but for clarity in the frontend + "puck_names": pucks, # Redundant but for clarity in the frontend + "crystal_names": samples # Redundant but for clarity in the frontend + } + + # Log the final response for debugging. + logger.info(f"Final response: {response_data}") + + return response_data + except SpreadsheetImportError as e: + logger.error(f"Failed to process file: {str(e)}") + raise HTTPException(status_code=400, detail="Failed to upload file. Please try again.") diff --git a/backend/app/sample_models.py b/backend/app/sample_models.py new file mode 100644 index 0000000..ea61dae --- /dev/null +++ b/backend/app/sample_models.py @@ -0,0 +1,425 @@ +""" +Data model and validation for mandatory and single sample rows from +spreadsheet. Can be imported by sample_importer.py and database services. +""" + +import re +from typing import Any, Optional, Union + +from pydantic import BaseModel, Field, validator + +from typing_extensions import Annotated + + +class SpreadsheetModel(BaseModel): + dewarname: str = Field(..., alias='dewarname') + puckname: str = Field(..., alias='puckname') + pucktype: Optional[str] = "unipuck" + pucklocationindewar: Optional[Union[int, str]] + crystalname: Annotated[ + str, + Field(..., + max_length=64, + title="Crystal Name", + description="""max_length imposed by MTZ file header format + https://www.ccp4.ac.uk/html/mtzformat.html""", + alias='crystalname' + ), + ] + positioninpuck: int + priority: Optional[str] + comments: Optional[str] + pinbarcode: Optional[str] + directory: Optional[str] + proteinname: Any = "" + oscillation: Any = "" + exposure: Any = "" + totalrange: Any = "" + transmission: Any = "" + targetresolution: Any = "" + aperture: Any = "" + datacollectiontype: Any = "" + processingpipeline: Any = "" + spacegroupnumber: Any = "" + cellparameters: Any = "" + rescutkey: Any = "" + rescutvalue: Any = "" + userresolution: Any = "" + pdbmodel: Any = "" + autoprocfull: Any = "" + procfull: Any = "" + adpenabled: Any = "" + noano: Any = "" + trustedhigh: Any = "" + ffcscampaign: Any = "" + autoprocextraparams: Any = "" + chiphiangles: Any = "" + + @validator("dewarname", "puckname") + def dewarname_puckname_characters(cls, v, **kwargs): + assert ( + len(str(v)) > 0 + ), f"""" {v} " is not valid. + value must be provided for all samples in spreadsheet.""" + v = str(v).replace(" ", "_") + if re.search("\n", v): + assert v.isalnum(), "is not valid. newline character detected." + v = re.sub(r"\.0$", "", v) + return v.upper() + + @validator("crystalname") + def parameter_characters(cls, v, **kwargs): + v = str(v).replace(" ", "_") + if re.search("\n", v): + assert v.isalnum(), "is not valid. newline character detected." + characters = re.sub("[._+-]", "", v) + assert characters.isalnum(), f"""" {v} " is not valid. + must contain only alphanumeric and . _ + - characters""" + v = re.sub(r"\.0$", "", v) + return v + + @validator("directory") + def directory_characters(cls, v, **kwargs): + if v: + v = str(v).strip("/").replace(" ", "_") + if re.search("\n", v): + raise ValueError( + f"""" {v} " is not valid. + newline character detected.""" + ) + ok = "[a-z0-9_.+-]" + directory_re = re.compile("^((%s*|{%s+})*/?)*$" % (ok, ok), re.IGNORECASE) + if not directory_re.match(v): + raise ValueError( + f"' {v} ' is not valid. value must be a path or macro." + ) + + these_macros = re.findall(r"(\{[^}]+\})", v) + valid_macros = [ + "{date}", + "{prefix}", + "{sgpuck}", + "{puck}", + "{beamline}", + "{sgprefix}", + "{sgpriority}", + "{sgposition}", + "{protein}", + "{method}", + ] + for m in these_macros: + if m.lower() not in valid_macros: + raise ValueError( + f"""" {m} " is not a valid macro, please re-check documentation; + allowed macros: date, prefix, sgpuck, puck, beamline, sgprefix, + sgpriority, sgposition, protein, method""" + ) + return v + + @validator("positioninpuck", pre=True) + def positioninpuck_possible(cls, v, **kwargs): + if v: + try: + v = int(float(v)) + if v < 1 or v > 16: + raise ValueError( + f"""" {v} " is not valid. value must be from 1 to 16.""" + ) + except (ValueError, TypeError) as e: + raise ValueError( + f"""" {v} " is not valid. + Value must be a numeric type and from 1 to 16.""" + ) from e + else: + raise ValueError("Value must be provided. Value must be from 1 to 16.") + return v + + @validator("pucklocationindewar") + def pucklocationindewar_convert_to_int(cls, v, **kwargs): + return int(float(v)) if v else v + + @validator("priority") + def priority_positive(cls, v, **kwargs): + if v: + v = re.sub(r"\.0$", "", v) + try: + if not int(v) > 0: + raise ValueError( + f"""" {v} " is not valid. + value must be a positive integer.""" + ) + elif int(v) > 0: + v = int(v) + except (ValueError, TypeError) as e: + raise ValueError( + f"""" {v} " is not valid. + value must be a positive integer.""" + ) from e + return v + + @validator("aperture") + def aperture_selection(cls, v, **kwargs): + if v: + try: + v = int(float(v)) + if v not in [1, 2, 3]: + raise ValueError( + f"""" {v} " is not valid. + value must be integer 1, 2 or 3.""" + ) + except (ValueError, TypeError) as e: + raise ValueError( + f"""" {v} " is not valid. + value must be integer 1, 2 or 3.""" + ) from e + return v + + @validator( + "oscillation", + "exposure", + "totalrange", + "targetresolution", + "rescutvalue", + "userresolution", + ) + def parameter_positive_float(cls, v, **kwargs): + if v: + try: + v = float(v) + if not v > 0: + raise ValueError( + f"""" {v} " is not valid. + value must be a positive float.""" + ) + except (ValueError, TypeError) as e: + raise ValueError( + f"""" {v} " is not valid. + value must be a positive float.""" + ) from e + return v + + @validator("transmission") + def tranmission_fraction(cls, v, **kwargs): + if v: + try: + v = float(v) + if 100 >= v > 0: + v = v + else: + raise ValueError( + f"""" {v} " is not valid. + value must be a float between 0 and 100.""" + ) + except (ValueError, TypeError) as e: + raise ValueError( + f"""" {v} " is not valid. + value must be a float between 0 and 100.""" + ) from e + return v + + @validator("datacollectiontype") + def datacollectiontype_allowed(cls, v, **kwargs): + if v: + v = v.lower() + allowed = ["standard", "serial-xtal", "multi-orientation"] + if str(v) not in allowed: + raise ValueError( + f"""" {v} " is not valid. + value must be one of" {allowed} ".""" + ) + return v + + @validator("processingpipeline") + def processingpipeline_allowed(cls, v, **kwargs): + if v: + v = v.lower() + allowed = ["gopy", "autoproc", "xia2dials"] + if str(v) not in allowed: + raise ValueError( + f"""" {v} " is not valid. + value must be one of " {allowed} ".""" + ) + return v + + @validator("spacegroupnumber") + def spacegroupnumber_integer(cls, v, **kwargs): + if v: + try: + v = int(float(v)) + if not v > 0 or not v < 231: + raise ValueError( + f"""" {v} " is not valid. + value must be a positive integer between 1 and 230.""" + ) + except (ValueError, TypeError) as e: + raise ValueError( + f"""" {v} " is not valid. + value must be a positive integer between 1 and 230.""" + ) from e + return v + + @validator("cellparameters") + def cellparameters_positive_floats(cls, v, **kwargs): + if v: + splitted = str(v).split(" ") + if len(splitted) != 6: + raise ValueError( + f"' {v} ' is not valid. value must be a set of six numbers." + ) + for el in splitted: + try: + el = float(el) + if not el > 0: + raise ValueError( + f"' {el} ' is not valid. value must be a positive float." + ) + except (ValueError, TypeError) as e: + raise ValueError( + f"' {el} ' is not valid. value must be a positive float." + ) from e + return v + + @validator("rescutkey") + def rescutkey_allowed(cls, v, **kwargs): + if v: + v = v.lower() + allowed = ["is", "cchalf"] + if str(v) not in allowed: + raise ValueError(f"' {v} ' is not valid. value must be ' {allowed} '.") + return v + + @validator("autoprocfull", "procfull", "adpenabled", "noano", "ffcscampaign") + def boolean_allowed(cls, v, **kwargs): + if v: + v = v.title() + allowed = ["False", "True"] + if str(v) not in allowed: + raise ValueError( + f"""" {v} " is not valid. + value must be ' {allowed} '.""" + ) + return v + + @validator("trustedhigh") + def trusted_float(cls, v, **kwargs): + if v: + try: + v = float(v) + if 2.0 >= v > 0: + v = v + else: + raise ValueError( + f"""" {v} " is not valid. + value must be a float between 0 and 2.0.""" + ) + except (ValueError, TypeError) as e: + raise ValueError( + f"""" {v} " is not valid. + value must be a float between 0 and 2.0.""" + ) from e + return v + + @validator("proteinname") + def proteinname_characters(cls, v, **kwargs): + if v: + v = str(v).replace(" ", "_") + if re.search("\n", v): + assert v.isalnum(), "is not valid. newline character detected." + characters = re.sub("[._+-]", "", v) + assert characters.isalnum(), f"""" {v} " is not valid. + must contain only alphanumeric and . _ + - characters""" + v = re.sub(r"\.0$", "", v) + return v + + @validator("chiphiangles") + def chiphiangles_value(cls, v, **kwargs): + if v: + try: + v = str(v) + re.sub(r"(^\s*\[\s*|\s*\]\s*$)", "", v.strip()) + list_of_strings = re.findall(r"\(.*?\)", v) + list_of_tuples = [] + for el in list_of_strings: + first = re.findall(r"\(.*?\,", el)[0].replace(" ", "")[1:-1] + second = re.findall(r"\,.*?\)", el)[0].replace(" ", "")[1:-1] + my_tuple = (float(first), float(second)) + list_of_tuples.append(my_tuple) + v = list_of_tuples + except (ValueError, TypeError) as e: + raise ValueError( + f"""" {v} " is not valid. Example format is + (0.0, 0.0), (20.0, 0.0), (30, 0.0)""" + ) from e + return v + + @validator( + "priority", + "comments", + "pinbarcode", + "directory", + "proteinname", + "oscillation", + "exposure", + "totalrange", + "transmission", + "targetresolution", + "aperture", + "datacollectiontype", + "processingpipeline", + "spacegroupnumber", + "cellparameters", + "rescutkey", + "rescutvalue", + "userresolution", + "pdbmodel", + "autoprocfull", + "procfull", + "adpenabled", + "noano", + "trustedhigh", + "ffcscampaign", + "autoprocextraparams", + "chiphiangles", + ) + def set_default_emptystring(cls, v, **kwargs): + return v or "" + + class Config: + anystr_strip_whitespace = True + + +class TELLModel(SpreadsheetModel): + input_order: int + samplemountcount: int = 0 + samplestatus: str = "not present" + puckaddress: str = "---" + username: str + puck_number: int + prefix: Optional[str] + folder: Optional[str] + + +""" +Following params appended in teller.py for updating SDU sample model +class SDUTELLModel(TELLModel): + sdudaq: str + sdudiffcenter: str + sduopticalcenter: str + sdumount: str + sdusafetycheck: str + +Following params returned in the format expected by tell.set_samples_info() +{ + "userName": user, + "dewarName": sample["dewarname"], + "puckName": "", # FIXME at the moment this field is useless + "puckType": "Unipuck", + "puckAddress": sample["puckaddress"], + "puckBarcode": sample["puckname"], + "sampleBarcode": sample.get("pinbarcode", ""), + "sampleMountCount": sample["samplemountcount"], + "sampleName": sample["crystalname"], + "samplePosition": sample["positioninpuck"], + "sampleStatus": sample["samplestatus"], +} +""" \ No newline at end of file diff --git a/backend/app/services/__init__.py b/backend/app/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/services/spreadsheet_service.py b/backend/app/services/spreadsheet_service.py new file mode 100644 index 0000000..924bbe8 --- /dev/null +++ b/backend/app/services/spreadsheet_service.py @@ -0,0 +1,106 @@ +# app/services/spreadsheet_service.py + +import logging +import openpyxl +from pydantic import ValidationError, parse_obj_as +from typing import List +from app.sample_models import SpreadsheetModel, TELLModel +from io import BytesIO + +UNASSIGNED_PUCKADDRESS = "---" +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +class SpreadsheetImportError(Exception): + pass + + +class SampleSpreadsheetImporter: + def __init__(self): + self.filename = None + self.model = None + self.available_puck_positions = [] + + def import_spreadsheet(self, file): + self.available_puck_positions = [ + f"{s}{p}" for s in list("ABCDEF") for p in range(1, 6) + ] + self.available_puck_positions.append(UNASSIGNED_PUCKADDRESS) + self.filename = file.filename + + try: + logger.info(f"Importing spreadsheet from .xlsx file: {self.filename}") + contents = file.file.read() # Read the file contents into memory + file.file.seek(0) # Reset file pointer to the beginning + workbook = openpyxl.load_workbook(BytesIO(contents)) + sheet = workbook["Samples"] + except KeyError: + raise SpreadsheetImportError("The file is missing 'Samples' worksheet.") + except Exception as e: + raise SpreadsheetImportError(f"Failed to read the file: {str(e)}") + + return self.process_spreadsheet(sheet) + + def process_spreadsheet(self, sheet): + model = [] + + # Skip the first 3 rows + rows = list(sheet.iter_rows(min_row=4, values_only=True)) + + for row in rows: + sample = { + 'dewarname': self._clean_value(row[0]), + 'puckname': self._clean_value(row[1]), + 'crystalname': self._clean_value(row[4]) + } + + if not sample['dewarname'] or not sample['puckname'] or not sample['crystalname']: + # Skip rows with missing required fields + continue + + model.append(sample) + logger.info(f"Sample processed: {sample}") # Adding log for each processed sample + + logger.info(f"...finished import, got {len(model)} samples") + self.model = model + + try: + validated_model = self.validate() + except SpreadsheetImportError as e: + logger.error(f"Failed to validate spreadsheet: {str(e)}") + raise + + return validated_model + + def validate(self): + model = self.model + logger.info(f"...validating {len(model)} samples") + + # Log the model before validation + for sample in model: + logger.info(f"Validating sample: {sample}") + + validated_model = self.data_model_validation(SpreadsheetModel, model) + + # Log the validated model after validation + for sample in validated_model: + logger.info(f"Validated sample: {sample}") + + return validated_model + + @staticmethod + def data_model_validation(data_model, model): + try: + validated = parse_obj_as(List[data_model], model) + except ValidationError as e: + raise SpreadsheetImportError(f"{e.errors()[0]['loc'][2]} => {e.errors()[0]['msg']}") + + validated_model = [dict(value) for value in validated] + return validated_model + + @staticmethod + def _clean_value(value): + if isinstance(value, str): + return value.strip() + return value # For other types (int, float, None, etc.), return value as is or handle accordingly diff --git a/frontend/src/components/UploadDialog.tsx b/frontend/src/components/UploadDialog.tsx index c0921f6..b24e011 100644 --- a/frontend/src/components/UploadDialog.tsx +++ b/frontend/src/components/UploadDialog.tsx @@ -1,5 +1,4 @@ -import * as React from 'react'; -import { useState } from 'react'; +import { useState, useEffect } from 'react'; import { Dialog, DialogTitle, @@ -14,6 +13,8 @@ import CloseIcon from '@mui/icons-material/Close'; import DownloadIcon from '@mui/icons-material/Download'; import UploadFileIcon from '@mui/icons-material/UploadFile'; import logo from '../assets/Heidi-logo.png'; +import { OpenAPI, UploadService } from '../../openapi'; +import type { Body_upload_file_upload_post } from '../../openapi/models/Body_upload_file_upload_post'; interface UploadDialogProps { open: boolean; @@ -23,12 +24,19 @@ interface UploadDialogProps { const UploadDialog: React.FC = ({ open, onClose }) => { const [uploadError, setUploadError] = useState(null); const [fileSummary, setFileSummary] = useState<{ - dewars: number; - pucks: number; - samples: number; + dewars_count: number; + dewars: string[]; + pucks_count: number; + pucks: string[]; + samples_count: number; + samples: string[]; } | null>(null); - const handleFileUpload = (event: React.ChangeEvent) => { + useEffect(() => { + OpenAPI.BASE = 'http://127.0.0.1:8000'; + }, []); + + const handleFileUpload = async (event: React.ChangeEvent) => { const file = event.target.files?.[0]; if (!file) { return; @@ -44,22 +52,25 @@ const UploadDialog: React.FC = ({ open, onClose }) => { return; } - // Simulate file reading and validation - const reader = new FileReader(); - reader.onload = () => { - // Here, parse the file content and validate - // For the demo, we'll mock the summary - const mockSummary = { - dewars: 5, - pucks: 10, - samples: 100, - }; - setFileSummary(mockSummary); - }; - reader.onerror = () => { - setUploadError('Failed to read the file. Please try again.'); - }; - reader.readAsArrayBuffer(file); + // Create the formData object compliant with the type definition + const formData: Body_upload_file_upload_post = { + file: file, // TypeScript understands that file is a Blob + } as Body_upload_file_upload_post; + + try { + // Use the generated OpenAPI client UploadService method + const response = await UploadService.uploadFileUploadPost(formData); + + console.log('File summary response from backend:', response); + console.log('Dewars:', response.dewars); + console.log('Pucks:', response.pucks); + console.log('Samples:', response.samples); + + setFileSummary(response); + } catch (error) { + console.error('File upload error:', error); + setUploadError('Failed to upload file. Please try again.'); + } }; return ( @@ -74,14 +85,8 @@ const UploadDialog: React.FC = ({ open, onClose }) => { - Logo - - Latest Spreadsheet Template Version 6 - + Logo + Latest Spreadsheet Template Version 6 Last update: October 18, 2024 @@ -111,17 +116,9 @@ const UploadDialog: React.FC = ({ open, onClose }) => { - {uploadError && ( @@ -130,15 +127,22 @@ const UploadDialog: React.FC = ({ open, onClose }) => { )} {fileSummary && ( - - File uploaded successfully! - + File uploaded successfully! File Summary: - Dewars: {fileSummary.dewars} - Pucks: {fileSummary.pucks} - Samples: {fileSummary.samples} + Dewars: {fileSummary.dewars_count} + Pucks: {fileSummary.pucks_count} + Samples: {fileSummary.samples_count} + + Dewar Names: {Array.isArray(fileSummary.dewars) ? fileSummary.dewars.join(', ') : 'N/A'} + + + Puck Names: {Array.isArray(fileSummary.pucks) ? fileSummary.pucks.join(', ') : 'N/A'} + + + Sample Names: {Array.isArray(fileSummary.samples) ? fileSummary.samples.join(', ') : 'N/A'} + )} @@ -152,4 +156,4 @@ const UploadDialog: React.FC = ({ open, onClose }) => { ); }; -export default UploadDialog; +export default UploadDialog; \ No newline at end of file