upload dialog is uploading a file

This commit is contained in:
GotthardG 2024-11-05 14:08:34 +01:00
parent 976cdc1a0a
commit 8cec4cb8df
7 changed files with 648 additions and 50 deletions

0
backend/__init__.py Normal file
View File

View File

@ -3,7 +3,7 @@
from fastapi import FastAPI from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.cors import CORSMiddleware
from app.routers import address, contact, proposal, dewar, shipment, upload, puck from app.routers import address, contact, proposal, dewar, shipment, upload, puck, spreadsheet
from app.database import Base, engine, SessionLocal, load_sample_data from app.database import Base, engine, SessionLocal, load_sample_data
app = FastAPI() app = FastAPI()
@ -17,6 +17,7 @@ app.add_middleware(
allow_headers=["*"], allow_headers=["*"],
) )
@app.on_event("startup") @app.on_event("startup")
def on_startup(): def on_startup():
# Drop and recreate database schema # Drop and recreate database schema
@ -29,15 +30,18 @@ def on_startup():
finally: finally:
db.close() db.close()
# Include routers with correct configuration # Include routers with correct configuration
app.include_router(contact.router, prefix="/contacts", tags=["contacts"]) app.include_router(contact.router, prefix="/contacts", tags=["contacts"])
app.include_router(address.router, prefix="/addresses", tags=["addresses"]) app.include_router(address.router, prefix="/addresses", tags=["addresses"])
app.include_router(proposal.router, prefix="/proposals", tags=["proposals"]) app.include_router(proposal.router, prefix="/proposals", tags=["proposals"])
app.include_router(dewar.router, prefix="/dewars", tags=["dewars"]) app.include_router(dewar.router, prefix="/dewars", tags=["dewars"])
app.include_router(shipment.router, prefix="/shipments", tags=["shipments"]) app.include_router(shipment.router, prefix="/shipments", tags=["shipments"])
app.include_router(upload.router, tags=["upload"]) # Removed the trailing '/' from the prefix app.include_router(upload.router, tags=["upload"])
app.include_router(puck.router, prefix="/pucks", tags=["pucks"]) app.include_router(puck.router, prefix="/pucks", tags=["pucks"])
app.include_router(spreadsheet.router, tags=["spreadsheet"])
if __name__ == "__main__": if __name__ == "__main__":
import uvicorn import uvicorn
uvicorn.run(app, host="127.0.0.1", port=8000, log_level="debug")
uvicorn.run(app, host="127.0.0.1", port=8000, log_level="debug")

View File

@ -0,0 +1,59 @@
# app/routers/spreadsheet.py
from fastapi import APIRouter, UploadFile, File, HTTPException
from app.services.spreadsheet_service import SampleSpreadsheetImporter, SpreadsheetImportError
import logging
router = APIRouter()
logger = logging.getLogger(__name__)
@router.post("/upload")
async def upload_file(file: UploadFile = File(...)):
importer = SampleSpreadsheetImporter()
try:
result = importer.import_spreadsheet(file)
if not result:
logger.warning("No data extracted from spreadsheet.")
return {
"dewars_count": 0,
"dewars": [],
"pucks_count": 0,
"pucks": [],
"samples_count": 0,
"samples": []
}
# Logging the raw results for debugging.
logger.info(f"Extracted Result: {result}")
# Extract and respond with detailed information.
dewars = list(set(sample['dewarname'] for sample in result))
pucks = list(set(sample['puckname'] for sample in result))
samples = list(set(sample['crystalname'] for sample in result))
# Log the extracted names.
logger.info(f"Dewars: {dewars}")
logger.info(f"Pucks: {pucks}")
logger.info(f"Samples: {samples}")
response_data = {
"dewars_count": len(dewars),
"dewars": dewars,
"pucks_count": len(pucks),
"pucks": pucks,
"samples_count": len(samples),
"samples": samples, # Ensure lists include detailed names
"dewar_names": dewars, # Redundant but for clarity in the frontend
"puck_names": pucks, # Redundant but for clarity in the frontend
"crystal_names": samples # Redundant but for clarity in the frontend
}
# Log the final response for debugging.
logger.info(f"Final response: {response_data}")
return response_data
except SpreadsheetImportError as e:
logger.error(f"Failed to process file: {str(e)}")
raise HTTPException(status_code=400, detail="Failed to upload file. Please try again.")

View File

@ -0,0 +1,425 @@
"""
Data model and validation for mandatory and single sample rows from
spreadsheet. Can be imported by sample_importer.py and database services.
"""
import re
from typing import Any, Optional, Union
from pydantic import BaseModel, Field, validator
from typing_extensions import Annotated
class SpreadsheetModel(BaseModel):
dewarname: str = Field(..., alias='dewarname')
puckname: str = Field(..., alias='puckname')
pucktype: Optional[str] = "unipuck"
pucklocationindewar: Optional[Union[int, str]]
crystalname: Annotated[
str,
Field(...,
max_length=64,
title="Crystal Name",
description="""max_length imposed by MTZ file header format
https://www.ccp4.ac.uk/html/mtzformat.html""",
alias='crystalname'
),
]
positioninpuck: int
priority: Optional[str]
comments: Optional[str]
pinbarcode: Optional[str]
directory: Optional[str]
proteinname: Any = ""
oscillation: Any = ""
exposure: Any = ""
totalrange: Any = ""
transmission: Any = ""
targetresolution: Any = ""
aperture: Any = ""
datacollectiontype: Any = ""
processingpipeline: Any = ""
spacegroupnumber: Any = ""
cellparameters: Any = ""
rescutkey: Any = ""
rescutvalue: Any = ""
userresolution: Any = ""
pdbmodel: Any = ""
autoprocfull: Any = ""
procfull: Any = ""
adpenabled: Any = ""
noano: Any = ""
trustedhigh: Any = ""
ffcscampaign: Any = ""
autoprocextraparams: Any = ""
chiphiangles: Any = ""
@validator("dewarname", "puckname")
def dewarname_puckname_characters(cls, v, **kwargs):
assert (
len(str(v)) > 0
), f"""" {v} " is not valid.
value must be provided for all samples in spreadsheet."""
v = str(v).replace(" ", "_")
if re.search("\n", v):
assert v.isalnum(), "is not valid. newline character detected."
v = re.sub(r"\.0$", "", v)
return v.upper()
@validator("crystalname")
def parameter_characters(cls, v, **kwargs):
v = str(v).replace(" ", "_")
if re.search("\n", v):
assert v.isalnum(), "is not valid. newline character detected."
characters = re.sub("[._+-]", "", v)
assert characters.isalnum(), f"""" {v} " is not valid.
must contain only alphanumeric and . _ + - characters"""
v = re.sub(r"\.0$", "", v)
return v
@validator("directory")
def directory_characters(cls, v, **kwargs):
if v:
v = str(v).strip("/").replace(" ", "_")
if re.search("\n", v):
raise ValueError(
f"""" {v} " is not valid.
newline character detected."""
)
ok = "[a-z0-9_.+-]"
directory_re = re.compile("^((%s*|{%s+})*/?)*$" % (ok, ok), re.IGNORECASE)
if not directory_re.match(v):
raise ValueError(
f"' {v} ' is not valid. value must be a path or macro."
)
these_macros = re.findall(r"(\{[^}]+\})", v)
valid_macros = [
"{date}",
"{prefix}",
"{sgpuck}",
"{puck}",
"{beamline}",
"{sgprefix}",
"{sgpriority}",
"{sgposition}",
"{protein}",
"{method}",
]
for m in these_macros:
if m.lower() not in valid_macros:
raise ValueError(
f"""" {m} " is not a valid macro, please re-check documentation;
allowed macros: date, prefix, sgpuck, puck, beamline, sgprefix,
sgpriority, sgposition, protein, method"""
)
return v
@validator("positioninpuck", pre=True)
def positioninpuck_possible(cls, v, **kwargs):
if v:
try:
v = int(float(v))
if v < 1 or v > 16:
raise ValueError(
f"""" {v} " is not valid. value must be from 1 to 16."""
)
except (ValueError, TypeError) as e:
raise ValueError(
f"""" {v} " is not valid.
Value must be a numeric type and from 1 to 16."""
) from e
else:
raise ValueError("Value must be provided. Value must be from 1 to 16.")
return v
@validator("pucklocationindewar")
def pucklocationindewar_convert_to_int(cls, v, **kwargs):
return int(float(v)) if v else v
@validator("priority")
def priority_positive(cls, v, **kwargs):
if v:
v = re.sub(r"\.0$", "", v)
try:
if not int(v) > 0:
raise ValueError(
f"""" {v} " is not valid.
value must be a positive integer."""
)
elif int(v) > 0:
v = int(v)
except (ValueError, TypeError) as e:
raise ValueError(
f"""" {v} " is not valid.
value must be a positive integer."""
) from e
return v
@validator("aperture")
def aperture_selection(cls, v, **kwargs):
if v:
try:
v = int(float(v))
if v not in [1, 2, 3]:
raise ValueError(
f"""" {v} " is not valid.
value must be integer 1, 2 or 3."""
)
except (ValueError, TypeError) as e:
raise ValueError(
f"""" {v} " is not valid.
value must be integer 1, 2 or 3."""
) from e
return v
@validator(
"oscillation",
"exposure",
"totalrange",
"targetresolution",
"rescutvalue",
"userresolution",
)
def parameter_positive_float(cls, v, **kwargs):
if v:
try:
v = float(v)
if not v > 0:
raise ValueError(
f"""" {v} " is not valid.
value must be a positive float."""
)
except (ValueError, TypeError) as e:
raise ValueError(
f"""" {v} " is not valid.
value must be a positive float."""
) from e
return v
@validator("transmission")
def tranmission_fraction(cls, v, **kwargs):
if v:
try:
v = float(v)
if 100 >= v > 0:
v = v
else:
raise ValueError(
f"""" {v} " is not valid.
value must be a float between 0 and 100."""
)
except (ValueError, TypeError) as e:
raise ValueError(
f"""" {v} " is not valid.
value must be a float between 0 and 100."""
) from e
return v
@validator("datacollectiontype")
def datacollectiontype_allowed(cls, v, **kwargs):
if v:
v = v.lower()
allowed = ["standard", "serial-xtal", "multi-orientation"]
if str(v) not in allowed:
raise ValueError(
f"""" {v} " is not valid.
value must be one of" {allowed} "."""
)
return v
@validator("processingpipeline")
def processingpipeline_allowed(cls, v, **kwargs):
if v:
v = v.lower()
allowed = ["gopy", "autoproc", "xia2dials"]
if str(v) not in allowed:
raise ValueError(
f"""" {v} " is not valid.
value must be one of " {allowed} "."""
)
return v
@validator("spacegroupnumber")
def spacegroupnumber_integer(cls, v, **kwargs):
if v:
try:
v = int(float(v))
if not v > 0 or not v < 231:
raise ValueError(
f"""" {v} " is not valid.
value must be a positive integer between 1 and 230."""
)
except (ValueError, TypeError) as e:
raise ValueError(
f"""" {v} " is not valid.
value must be a positive integer between 1 and 230."""
) from e
return v
@validator("cellparameters")
def cellparameters_positive_floats(cls, v, **kwargs):
if v:
splitted = str(v).split(" ")
if len(splitted) != 6:
raise ValueError(
f"' {v} ' is not valid. value must be a set of six numbers."
)
for el in splitted:
try:
el = float(el)
if not el > 0:
raise ValueError(
f"' {el} ' is not valid. value must be a positive float."
)
except (ValueError, TypeError) as e:
raise ValueError(
f"' {el} ' is not valid. value must be a positive float."
) from e
return v
@validator("rescutkey")
def rescutkey_allowed(cls, v, **kwargs):
if v:
v = v.lower()
allowed = ["is", "cchalf"]
if str(v) not in allowed:
raise ValueError(f"' {v} ' is not valid. value must be ' {allowed} '.")
return v
@validator("autoprocfull", "procfull", "adpenabled", "noano", "ffcscampaign")
def boolean_allowed(cls, v, **kwargs):
if v:
v = v.title()
allowed = ["False", "True"]
if str(v) not in allowed:
raise ValueError(
f"""" {v} " is not valid.
value must be ' {allowed} '."""
)
return v
@validator("trustedhigh")
def trusted_float(cls, v, **kwargs):
if v:
try:
v = float(v)
if 2.0 >= v > 0:
v = v
else:
raise ValueError(
f"""" {v} " is not valid.
value must be a float between 0 and 2.0."""
)
except (ValueError, TypeError) as e:
raise ValueError(
f"""" {v} " is not valid.
value must be a float between 0 and 2.0."""
) from e
return v
@validator("proteinname")
def proteinname_characters(cls, v, **kwargs):
if v:
v = str(v).replace(" ", "_")
if re.search("\n", v):
assert v.isalnum(), "is not valid. newline character detected."
characters = re.sub("[._+-]", "", v)
assert characters.isalnum(), f"""" {v} " is not valid.
must contain only alphanumeric and . _ + - characters"""
v = re.sub(r"\.0$", "", v)
return v
@validator("chiphiangles")
def chiphiangles_value(cls, v, **kwargs):
if v:
try:
v = str(v)
re.sub(r"(^\s*\[\s*|\s*\]\s*$)", "", v.strip())
list_of_strings = re.findall(r"\(.*?\)", v)
list_of_tuples = []
for el in list_of_strings:
first = re.findall(r"\(.*?\,", el)[0].replace(" ", "")[1:-1]
second = re.findall(r"\,.*?\)", el)[0].replace(" ", "")[1:-1]
my_tuple = (float(first), float(second))
list_of_tuples.append(my_tuple)
v = list_of_tuples
except (ValueError, TypeError) as e:
raise ValueError(
f"""" {v} " is not valid. Example format is
(0.0, 0.0), (20.0, 0.0), (30, 0.0)"""
) from e
return v
@validator(
"priority",
"comments",
"pinbarcode",
"directory",
"proteinname",
"oscillation",
"exposure",
"totalrange",
"transmission",
"targetresolution",
"aperture",
"datacollectiontype",
"processingpipeline",
"spacegroupnumber",
"cellparameters",
"rescutkey",
"rescutvalue",
"userresolution",
"pdbmodel",
"autoprocfull",
"procfull",
"adpenabled",
"noano",
"trustedhigh",
"ffcscampaign",
"autoprocextraparams",
"chiphiangles",
)
def set_default_emptystring(cls, v, **kwargs):
return v or ""
class Config:
anystr_strip_whitespace = True
class TELLModel(SpreadsheetModel):
input_order: int
samplemountcount: int = 0
samplestatus: str = "not present"
puckaddress: str = "---"
username: str
puck_number: int
prefix: Optional[str]
folder: Optional[str]
"""
Following params appended in teller.py for updating SDU sample model
class SDUTELLModel(TELLModel):
sdudaq: str
sdudiffcenter: str
sduopticalcenter: str
sdumount: str
sdusafetycheck: str
Following params returned in the format expected by tell.set_samples_info()
{
"userName": user,
"dewarName": sample["dewarname"],
"puckName": "", # FIXME at the moment this field is useless
"puckType": "Unipuck",
"puckAddress": sample["puckaddress"],
"puckBarcode": sample["puckname"],
"sampleBarcode": sample.get("pinbarcode", ""),
"sampleMountCount": sample["samplemountcount"],
"sampleName": sample["crystalname"],
"samplePosition": sample["positioninpuck"],
"sampleStatus": sample["samplestatus"],
}
"""

View File

View File

@ -0,0 +1,106 @@
# app/services/spreadsheet_service.py
import logging
import openpyxl
from pydantic import ValidationError, parse_obj_as
from typing import List
from app.sample_models import SpreadsheetModel, TELLModel
from io import BytesIO
UNASSIGNED_PUCKADDRESS = "---"
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
class SpreadsheetImportError(Exception):
pass
class SampleSpreadsheetImporter:
def __init__(self):
self.filename = None
self.model = None
self.available_puck_positions = []
def import_spreadsheet(self, file):
self.available_puck_positions = [
f"{s}{p}" for s in list("ABCDEF") for p in range(1, 6)
]
self.available_puck_positions.append(UNASSIGNED_PUCKADDRESS)
self.filename = file.filename
try:
logger.info(f"Importing spreadsheet from .xlsx file: {self.filename}")
contents = file.file.read() # Read the file contents into memory
file.file.seek(0) # Reset file pointer to the beginning
workbook = openpyxl.load_workbook(BytesIO(contents))
sheet = workbook["Samples"]
except KeyError:
raise SpreadsheetImportError("The file is missing 'Samples' worksheet.")
except Exception as e:
raise SpreadsheetImportError(f"Failed to read the file: {str(e)}")
return self.process_spreadsheet(sheet)
def process_spreadsheet(self, sheet):
model = []
# Skip the first 3 rows
rows = list(sheet.iter_rows(min_row=4, values_only=True))
for row in rows:
sample = {
'dewarname': self._clean_value(row[0]),
'puckname': self._clean_value(row[1]),
'crystalname': self._clean_value(row[4])
}
if not sample['dewarname'] or not sample['puckname'] or not sample['crystalname']:
# Skip rows with missing required fields
continue
model.append(sample)
logger.info(f"Sample processed: {sample}") # Adding log for each processed sample
logger.info(f"...finished import, got {len(model)} samples")
self.model = model
try:
validated_model = self.validate()
except SpreadsheetImportError as e:
logger.error(f"Failed to validate spreadsheet: {str(e)}")
raise
return validated_model
def validate(self):
model = self.model
logger.info(f"...validating {len(model)} samples")
# Log the model before validation
for sample in model:
logger.info(f"Validating sample: {sample}")
validated_model = self.data_model_validation(SpreadsheetModel, model)
# Log the validated model after validation
for sample in validated_model:
logger.info(f"Validated sample: {sample}")
return validated_model
@staticmethod
def data_model_validation(data_model, model):
try:
validated = parse_obj_as(List[data_model], model)
except ValidationError as e:
raise SpreadsheetImportError(f"{e.errors()[0]['loc'][2]} => {e.errors()[0]['msg']}")
validated_model = [dict(value) for value in validated]
return validated_model
@staticmethod
def _clean_value(value):
if isinstance(value, str):
return value.strip()
return value # For other types (int, float, None, etc.), return value as is or handle accordingly

View File

@ -1,5 +1,4 @@
import * as React from 'react'; import { useState, useEffect } from 'react';
import { useState } from 'react';
import { import {
Dialog, Dialog,
DialogTitle, DialogTitle,
@ -14,6 +13,8 @@ import CloseIcon from '@mui/icons-material/Close';
import DownloadIcon from '@mui/icons-material/Download'; import DownloadIcon from '@mui/icons-material/Download';
import UploadFileIcon from '@mui/icons-material/UploadFile'; import UploadFileIcon from '@mui/icons-material/UploadFile';
import logo from '../assets/Heidi-logo.png'; import logo from '../assets/Heidi-logo.png';
import { OpenAPI, UploadService } from '../../openapi';
import type { Body_upload_file_upload_post } from '../../openapi/models/Body_upload_file_upload_post';
interface UploadDialogProps { interface UploadDialogProps {
open: boolean; open: boolean;
@ -23,12 +24,19 @@ interface UploadDialogProps {
const UploadDialog: React.FC<UploadDialogProps> = ({ open, onClose }) => { const UploadDialog: React.FC<UploadDialogProps> = ({ open, onClose }) => {
const [uploadError, setUploadError] = useState<string | null>(null); const [uploadError, setUploadError] = useState<string | null>(null);
const [fileSummary, setFileSummary] = useState<{ const [fileSummary, setFileSummary] = useState<{
dewars: number; dewars_count: number;
pucks: number; dewars: string[];
samples: number; pucks_count: number;
pucks: string[];
samples_count: number;
samples: string[];
} | null>(null); } | null>(null);
const handleFileUpload = (event: React.ChangeEvent<HTMLInputElement>) => { useEffect(() => {
OpenAPI.BASE = 'http://127.0.0.1:8000';
}, []);
const handleFileUpload = async (event: React.ChangeEvent<HTMLInputElement>) => {
const file = event.target.files?.[0]; const file = event.target.files?.[0];
if (!file) { if (!file) {
return; return;
@ -44,22 +52,25 @@ const UploadDialog: React.FC<UploadDialogProps> = ({ open, onClose }) => {
return; return;
} }
// Simulate file reading and validation // Create the formData object compliant with the type definition
const reader = new FileReader(); const formData: Body_upload_file_upload_post = {
reader.onload = () => { file: file, // TypeScript understands that file is a Blob
// Here, parse the file content and validate } as Body_upload_file_upload_post;
// For the demo, we'll mock the summary
const mockSummary = { try {
dewars: 5, // Use the generated OpenAPI client UploadService method
pucks: 10, const response = await UploadService.uploadFileUploadPost(formData);
samples: 100,
}; console.log('File summary response from backend:', response);
setFileSummary(mockSummary); console.log('Dewars:', response.dewars);
}; console.log('Pucks:', response.pucks);
reader.onerror = () => { console.log('Samples:', response.samples);
setUploadError('Failed to read the file. Please try again.');
}; setFileSummary(response);
reader.readAsArrayBuffer(file); } catch (error) {
console.error('File upload error:', error);
setUploadError('Failed to upload file. Please try again.');
}
}; };
return ( return (
@ -74,14 +85,8 @@ const UploadDialog: React.FC<UploadDialogProps> = ({ open, onClose }) => {
</DialogTitle> </DialogTitle>
<DialogContent dividers> <DialogContent dividers>
<Box display="flex" flexDirection="column" alignItems="center" mb={2}> <Box display="flex" flexDirection="column" alignItems="center" mb={2}>
<img <img src={logo} alt="Logo" style={{ width: 200, marginBottom: 16 }} />
src={logo} <Typography variant="subtitle1">Latest Spreadsheet Template Version 6</Typography>
alt="Logo"
style={{ width: 200, marginBottom: 16 }}
/>
<Typography variant="subtitle1">
Latest Spreadsheet Template Version 6
</Typography>
<Typography variant="body2" color="textSecondary"> <Typography variant="body2" color="textSecondary">
Last update: October 18, 2024 Last update: October 18, 2024
</Typography> </Typography>
@ -111,17 +116,9 @@ const UploadDialog: React.FC<UploadDialogProps> = ({ open, onClose }) => {
</Button> </Button>
</Box> </Box>
<Box mt={3}> <Box mt={3}>
<Button <Button variant="contained" component="label" startIcon={<UploadFileIcon />}>
variant="contained"
component="label"
startIcon={<UploadFileIcon />}
>
Choose a File Choose a File
<input <input type="file" hidden onChange={handleFileUpload} />
type="file"
hidden
onChange={handleFileUpload}
/>
</Button> </Button>
{uploadError && ( {uploadError && (
<Typography color="error" sx={{ mt: 2 }}> <Typography color="error" sx={{ mt: 2 }}>
@ -130,15 +127,22 @@ const UploadDialog: React.FC<UploadDialogProps> = ({ open, onClose }) => {
)} )}
{fileSummary && ( {fileSummary && (
<Box mt={2}> <Box mt={2}>
<Typography color="success.main"> <Typography color="success.main">File uploaded successfully!</Typography>
File uploaded successfully!
</Typography>
<Typography variant="body1"> <Typography variant="body1">
<strong>File Summary:</strong> <strong>File Summary:</strong>
</Typography> </Typography>
<Typography>Dewars: {fileSummary.dewars}</Typography> <Typography>Dewars: {fileSummary.dewars_count}</Typography>
<Typography>Pucks: {fileSummary.pucks}</Typography> <Typography>Pucks: {fileSummary.pucks_count}</Typography>
<Typography>Samples: {fileSummary.samples}</Typography> <Typography>Samples: {fileSummary.samples_count}</Typography>
<Typography variant="body2">
<strong>Dewar Names:</strong> {Array.isArray(fileSummary.dewars) ? fileSummary.dewars.join(', ') : 'N/A'}
</Typography>
<Typography variant="body2">
<strong>Puck Names:</strong> {Array.isArray(fileSummary.pucks) ? fileSummary.pucks.join(', ') : 'N/A'}
</Typography>
<Typography variant="body2">
<strong>Sample Names:</strong> {Array.isArray(fileSummary.samples) ? fileSummary.samples.join(', ') : 'N/A'}
</Typography>
</Box> </Box>
)} )}
</Box> </Box>
@ -152,4 +156,4 @@ const UploadDialog: React.FC<UploadDialogProps> = ({ open, onClose }) => {
); );
}; };
export default UploadDialog; export default UploadDialog;