now with the heidi v1 spreadsheet validator
This commit is contained in:
@ -3,7 +3,7 @@
|
|||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
from fastapi.middleware.cors import CORSMiddleware
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
|
|
||||||
from app.routers import address, contact, proposal, dewar, shipment, upload, puck, spreadsheet
|
from app.routers import address, contact, proposal, dewar, shipment, puck, spreadsheet
|
||||||
from app.database import Base, engine, SessionLocal, load_sample_data
|
from app.database import Base, engine, SessionLocal, load_sample_data
|
||||||
|
|
||||||
app = FastAPI()
|
app = FastAPI()
|
||||||
@ -37,7 +37,6 @@ app.include_router(address.router, prefix="/addresses", tags=["addresses"])
|
|||||||
app.include_router(proposal.router, prefix="/proposals", tags=["proposals"])
|
app.include_router(proposal.router, prefix="/proposals", tags=["proposals"])
|
||||||
app.include_router(dewar.router, prefix="/dewars", tags=["dewars"])
|
app.include_router(dewar.router, prefix="/dewars", tags=["dewars"])
|
||||||
app.include_router(shipment.router, prefix="/shipments", tags=["shipments"])
|
app.include_router(shipment.router, prefix="/shipments", tags=["shipments"])
|
||||||
app.include_router(upload.router, tags=["upload"])
|
|
||||||
app.include_router(puck.router, prefix="/pucks", tags=["pucks"])
|
app.include_router(puck.router, prefix="/pucks", tags=["pucks"])
|
||||||
app.include_router(spreadsheet.router, tags=["spreadsheet"])
|
app.include_router(spreadsheet.router, tags=["spreadsheet"])
|
||||||
|
|
||||||
|
@ -1,59 +1,51 @@
|
|||||||
# app/routers/spreadsheet.py
|
# app/routes/spreadsheet.py
|
||||||
|
|
||||||
from fastapi import APIRouter, UploadFile, File, HTTPException
|
from fastapi import APIRouter, UploadFile, File, HTTPException
|
||||||
from app.services.spreadsheet_service import SampleSpreadsheetImporter, SpreadsheetImportError
|
|
||||||
import logging
|
import logging
|
||||||
|
from app.services.spreadsheet_service import SampleSpreadsheetImporter, SpreadsheetImportError
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/upload")
|
@router.post("/upload")
|
||||||
async def upload_file(file: UploadFile = File(...)):
|
async def upload_file(file: UploadFile = File(...)):
|
||||||
importer = SampleSpreadsheetImporter()
|
|
||||||
try:
|
try:
|
||||||
result = importer.import_spreadsheet(file)
|
logger.info(f"Received file: {file.filename}")
|
||||||
|
|
||||||
if not result:
|
# File type check
|
||||||
logger.warning("No data extracted from spreadsheet.")
|
if not file.filename.endswith('.xlsx'):
|
||||||
return {
|
logger.error("Invalid file format")
|
||||||
"dewars_count": 0,
|
raise HTTPException(status_code=400, detail="Invalid file format. Please upload an .xlsx file.")
|
||||||
"dewars": [],
|
|
||||||
"pucks_count": 0,
|
|
||||||
"pucks": [],
|
|
||||||
"samples_count": 0,
|
|
||||||
"samples": []
|
|
||||||
}
|
|
||||||
|
|
||||||
# Logging the raw results for debugging.
|
# Reading file
|
||||||
logger.info(f"Extracted Result: {result}")
|
importer = SampleSpreadsheetImporter()
|
||||||
|
validated_model = importer.import_spreadsheet(file)
|
||||||
|
logger.info(f"Validated model: {validated_model}")
|
||||||
|
|
||||||
# Extract and respond with detailed information.
|
dewars = {sample['dewarname'] for sample in validated_model if 'dewarname' in sample}
|
||||||
dewars = list(set(sample['dewarname'] for sample in result))
|
pucks = {sample['puckname'] for sample in validated_model if 'puckname' in sample}
|
||||||
pucks = list(set(sample['puckname'] for sample in result))
|
samples = {sample['crystalname'] for sample in validated_model if 'crystalname' in sample}
|
||||||
samples = list(set(sample['crystalname'] for sample in result))
|
|
||||||
|
|
||||||
# Log the extracted names.
|
# Logging the sets of names
|
||||||
logger.info(f"Dewars: {dewars}")
|
logger.info(f"Dewar Names: {dewars}")
|
||||||
logger.info(f"Pucks: {pucks}")
|
logger.info(f"Puck Names: {pucks}")
|
||||||
logger.info(f"Samples: {samples}")
|
logger.info(f"Sample Names: {samples}")
|
||||||
|
|
||||||
response_data = {
|
# Forming structured response
|
||||||
|
response = {
|
||||||
"dewars_count": len(dewars),
|
"dewars_count": len(dewars),
|
||||||
"dewars": dewars,
|
"dewars": list(dewars),
|
||||||
"pucks_count": len(pucks),
|
"pucks_count": len(pucks),
|
||||||
"pucks": pucks,
|
"pucks": list(pucks),
|
||||||
"samples_count": len(samples),
|
"samples_count": len(samples),
|
||||||
"samples": samples, # Ensure lists include detailed names
|
"samples": list(samples)
|
||||||
"dewar_names": dewars, # Redundant but for clarity in the frontend
|
|
||||||
"puck_names": pucks, # Redundant but for clarity in the frontend
|
|
||||||
"crystal_names": samples # Redundant but for clarity in the frontend
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# Log the final response for debugging.
|
logger.info(f"Returning response: {response}")
|
||||||
logger.info(f"Final response: {response_data}")
|
return response
|
||||||
|
|
||||||
return response_data
|
|
||||||
except SpreadsheetImportError as e:
|
except SpreadsheetImportError as e:
|
||||||
|
logger.error(f"Spreadsheet import error: {str(e)}")
|
||||||
|
raise HTTPException(status_code=400, detail=str(e))
|
||||||
|
except Exception as e:
|
||||||
logger.error(f"Failed to process file: {str(e)}")
|
logger.error(f"Failed to process file: {str(e)}")
|
||||||
raise HTTPException(status_code=400, detail="Failed to upload file. Please try again.")
|
raise HTTPException(status_code=500, detail=f"Failed to upload file. Please try again. {str(e)}")
|
@ -1,35 +0,0 @@
|
|||||||
# app/routers/upload.py
|
|
||||||
|
|
||||||
from fastapi import APIRouter, UploadFile, File, HTTPException
|
|
||||||
import os
|
|
||||||
|
|
||||||
router = APIRouter()
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/upload")
|
|
||||||
async def upload_file(file: UploadFile = File(...)):
|
|
||||||
if not file.filename.endswith('.xlsx'):
|
|
||||||
raise HTTPException(status_code=400, detail="Invalid file format. Please upload an .xlsx file.")
|
|
||||||
|
|
||||||
save_path = os.path.join("uploads", file.filename)
|
|
||||||
os.makedirs(os.path.dirname(save_path), exist_ok=True)
|
|
||||||
with open(save_path, "wb") as buffer:
|
|
||||||
buffer.write(await file.read())
|
|
||||||
|
|
||||||
# Validate the file (add your validation logic here)
|
|
||||||
is_valid, summary, error = validate_file(save_path)
|
|
||||||
if not is_valid:
|
|
||||||
raise HTTPException(status_code=400, detail=error)
|
|
||||||
|
|
||||||
return summary
|
|
||||||
|
|
||||||
|
|
||||||
def validate_file(file_path: str):
|
|
||||||
# Implement your file validation logic here
|
|
||||||
# For demo purpose, assuming it always succeeds
|
|
||||||
summary = {
|
|
||||||
"dewars": 5,
|
|
||||||
"pucks": 10,
|
|
||||||
"samples": 100,
|
|
||||||
}
|
|
||||||
return True, summary, None
|
|
@ -1,13 +1,6 @@
|
|||||||
"""
|
|
||||||
Data model and validation for mandatory and single sample rows from
|
|
||||||
spreadsheet. Can be imported by sample_importer.py and database services.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import re
|
import re
|
||||||
from typing import Any, Optional, Union
|
from typing import Any, Optional, Union
|
||||||
|
from pydantic import BaseModel, Field, field_validator, AliasChoices
|
||||||
from pydantic import BaseModel, Field, validator
|
|
||||||
|
|
||||||
from typing_extensions import Annotated
|
from typing_extensions import Annotated
|
||||||
|
|
||||||
|
|
||||||
@ -19,15 +12,15 @@ class SpreadsheetModel(BaseModel):
|
|||||||
crystalname: Annotated[
|
crystalname: Annotated[
|
||||||
str,
|
str,
|
||||||
Field(...,
|
Field(...,
|
||||||
max_length=64,
|
max_length=64,
|
||||||
title="Crystal Name",
|
title="Crystal Name",
|
||||||
description="""max_length imposed by MTZ file header format
|
description="""max_length imposed by MTZ file header format
|
||||||
https://www.ccp4.ac.uk/html/mtzformat.html""",
|
https://www.ccp4.ac.uk/html/mtzformat.html""",
|
||||||
alias='crystalname'
|
alias='crystalname'
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
positioninpuck: int
|
positioninpuck: int
|
||||||
priority: Optional[str]
|
priority: Optional[int]
|
||||||
comments: Optional[str]
|
comments: Optional[str]
|
||||||
pinbarcode: Optional[str]
|
pinbarcode: Optional[str]
|
||||||
directory: Optional[str]
|
directory: Optional[str]
|
||||||
@ -55,20 +48,22 @@ class SpreadsheetModel(BaseModel):
|
|||||||
autoprocextraparams: Any = ""
|
autoprocextraparams: Any = ""
|
||||||
chiphiangles: Any = ""
|
chiphiangles: Any = ""
|
||||||
|
|
||||||
@validator("dewarname", "puckname")
|
@field_validator('dewarname', 'puckname', mode="before")
|
||||||
def dewarname_puckname_characters(cls, v, **kwargs):
|
@classmethod
|
||||||
assert (
|
def dewarname_puckname_characters(cls, v):
|
||||||
len(str(v)) > 0
|
if v:
|
||||||
), f"""" {v} " is not valid.
|
assert (
|
||||||
value must be provided for all samples in spreadsheet."""
|
len(str(v)) > 0
|
||||||
v = str(v).replace(" ", "_")
|
), f"""" {v} " is not valid. Value must be provided for all samples in the spreadsheet."""
|
||||||
if re.search("\n", v):
|
v = str(v).replace(" ", "_")
|
||||||
assert v.isalnum(), "is not valid. newline character detected."
|
if re.search("\n", v):
|
||||||
v = re.sub(r"\.0$", "", v)
|
assert v.isalnum(), "is not valid. newline character detected."
|
||||||
return v.upper()
|
v = re.sub(r"\.0$", "", v)
|
||||||
|
return v.upper()
|
||||||
|
|
||||||
@validator("crystalname")
|
@field_validator('crystalname', mode="before")
|
||||||
def parameter_characters(cls, v, **kwargs):
|
@classmethod
|
||||||
|
def parameter_characters(cls, v):
|
||||||
v = str(v).replace(" ", "_")
|
v = str(v).replace(" ", "_")
|
||||||
if re.search("\n", v):
|
if re.search("\n", v):
|
||||||
assert v.isalnum(), "is not valid. newline character detected."
|
assert v.isalnum(), "is not valid. newline character detected."
|
||||||
@ -78,8 +73,9 @@ class SpreadsheetModel(BaseModel):
|
|||||||
v = re.sub(r"\.0$", "", v)
|
v = re.sub(r"\.0$", "", v)
|
||||||
return v
|
return v
|
||||||
|
|
||||||
@validator("directory")
|
@field_validator('directory', mode="before")
|
||||||
def directory_characters(cls, v, **kwargs):
|
@classmethod
|
||||||
|
def directory_characters(cls, v):
|
||||||
if v:
|
if v:
|
||||||
v = str(v).strip("/").replace(" ", "_")
|
v = str(v).strip("/").replace(" ", "_")
|
||||||
if re.search("\n", v):
|
if re.search("\n", v):
|
||||||
@ -116,8 +112,9 @@ class SpreadsheetModel(BaseModel):
|
|||||||
)
|
)
|
||||||
return v
|
return v
|
||||||
|
|
||||||
@validator("positioninpuck", pre=True)
|
@field_validator('positioninpuck', mode="before")
|
||||||
def positioninpuck_possible(cls, v, **kwargs):
|
@classmethod
|
||||||
|
def positioninpuck_possible(cls, v):
|
||||||
if v:
|
if v:
|
||||||
try:
|
try:
|
||||||
v = int(float(v))
|
v = int(float(v))
|
||||||
@ -134,31 +131,37 @@ class SpreadsheetModel(BaseModel):
|
|||||||
raise ValueError("Value must be provided. Value must be from 1 to 16.")
|
raise ValueError("Value must be provided. Value must be from 1 to 16.")
|
||||||
return v
|
return v
|
||||||
|
|
||||||
@validator("pucklocationindewar")
|
@field_validator('pucklocationindewar', mode="before")
|
||||||
def pucklocationindewar_convert_to_int(cls, v, **kwargs):
|
@classmethod
|
||||||
return int(float(v)) if v else v
|
def pucklocationindewar_convert_to_str(cls, v):
|
||||||
|
if v == "Unipuck":
|
||||||
|
return v
|
||||||
|
try:
|
||||||
|
return str(int(float(v)))
|
||||||
|
except ValueError:
|
||||||
|
raise ValueError(f"Value error, could not convert string to float: '{v}'")
|
||||||
|
|
||||||
@validator("priority")
|
@field_validator('priority', mode="before")
|
||||||
def priority_positive(cls, v, **kwargs):
|
@classmethod
|
||||||
if v:
|
def priority_positive(cls, v):
|
||||||
|
if v is not None:
|
||||||
|
v = str(v).strip()
|
||||||
v = re.sub(r"\.0$", "", v)
|
v = re.sub(r"\.0$", "", v)
|
||||||
try:
|
try:
|
||||||
if not int(v) > 0:
|
if int(v) <= 0:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"""" {v} " is not valid.
|
f" '{v}' is not valid. Value must be a positive integer."
|
||||||
value must be a positive integer."""
|
|
||||||
)
|
)
|
||||||
elif int(v) > 0:
|
v = int(v)
|
||||||
v = int(v)
|
|
||||||
except (ValueError, TypeError) as e:
|
except (ValueError, TypeError) as e:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"""" {v} " is not valid.
|
f" '{v}' is not valid. Value must be a positive integer."
|
||||||
value must be a positive integer."""
|
|
||||||
) from e
|
) from e
|
||||||
return v
|
return v
|
||||||
|
|
||||||
@validator("aperture")
|
@field_validator('aperture', mode="before")
|
||||||
def aperture_selection(cls, v, **kwargs):
|
@classmethod
|
||||||
|
def aperture_selection(cls, v):
|
||||||
if v:
|
if v:
|
||||||
try:
|
try:
|
||||||
v = int(float(v))
|
v = int(float(v))
|
||||||
@ -174,15 +177,17 @@ class SpreadsheetModel(BaseModel):
|
|||||||
) from e
|
) from e
|
||||||
return v
|
return v
|
||||||
|
|
||||||
@validator(
|
@field_validator(
|
||||||
"oscillation",
|
"oscillation",
|
||||||
"exposure",
|
"exposure",
|
||||||
"totalrange",
|
"totalrange",
|
||||||
"targetresolution",
|
"targetresolution",
|
||||||
"rescutvalue",
|
"rescutvalue",
|
||||||
"userresolution",
|
"userresolution",
|
||||||
|
mode="before"
|
||||||
)
|
)
|
||||||
def parameter_positive_float(cls, v, **kwargs):
|
@classmethod
|
||||||
|
def parameter_positive_float(cls, v):
|
||||||
if v:
|
if v:
|
||||||
try:
|
try:
|
||||||
v = float(v)
|
v = float(v)
|
||||||
@ -198,8 +203,9 @@ class SpreadsheetModel(BaseModel):
|
|||||||
) from e
|
) from e
|
||||||
return v
|
return v
|
||||||
|
|
||||||
@validator("transmission")
|
@field_validator('transmission', mode="before")
|
||||||
def tranmission_fraction(cls, v, **kwargs):
|
@classmethod
|
||||||
|
def tranmission_fraction(cls, v):
|
||||||
if v:
|
if v:
|
||||||
try:
|
try:
|
||||||
v = float(v)
|
v = float(v)
|
||||||
@ -217,8 +223,9 @@ class SpreadsheetModel(BaseModel):
|
|||||||
) from e
|
) from e
|
||||||
return v
|
return v
|
||||||
|
|
||||||
@validator("datacollectiontype")
|
@field_validator('datacollectiontype', mode="before")
|
||||||
def datacollectiontype_allowed(cls, v, **kwargs):
|
@classmethod
|
||||||
|
def datacollectiontype_allowed(cls, v):
|
||||||
if v:
|
if v:
|
||||||
v = v.lower()
|
v = v.lower()
|
||||||
allowed = ["standard", "serial-xtal", "multi-orientation"]
|
allowed = ["standard", "serial-xtal", "multi-orientation"]
|
||||||
@ -229,8 +236,9 @@ class SpreadsheetModel(BaseModel):
|
|||||||
)
|
)
|
||||||
return v
|
return v
|
||||||
|
|
||||||
@validator("processingpipeline")
|
@field_validator('processingpipeline', mode="before")
|
||||||
def processingpipeline_allowed(cls, v, **kwargs):
|
@classmethod
|
||||||
|
def processingpipeline_allowed(cls, v):
|
||||||
if v:
|
if v:
|
||||||
v = v.lower()
|
v = v.lower()
|
||||||
allowed = ["gopy", "autoproc", "xia2dials"]
|
allowed = ["gopy", "autoproc", "xia2dials"]
|
||||||
@ -241,8 +249,9 @@ class SpreadsheetModel(BaseModel):
|
|||||||
)
|
)
|
||||||
return v
|
return v
|
||||||
|
|
||||||
@validator("spacegroupnumber")
|
@field_validator('spacegroupnumber', mode="before")
|
||||||
def spacegroupnumber_integer(cls, v, **kwargs):
|
@classmethod
|
||||||
|
def spacegroupnumber_integer(cls, v):
|
||||||
if v:
|
if v:
|
||||||
try:
|
try:
|
||||||
v = int(float(v))
|
v = int(float(v))
|
||||||
@ -258,8 +267,9 @@ class SpreadsheetModel(BaseModel):
|
|||||||
) from e
|
) from e
|
||||||
return v
|
return v
|
||||||
|
|
||||||
@validator("cellparameters")
|
@field_validator('cellparameters', mode="before")
|
||||||
def cellparameters_positive_floats(cls, v, **kwargs):
|
@classmethod
|
||||||
|
def cellparameters_positive_floats(cls, v):
|
||||||
if v:
|
if v:
|
||||||
splitted = str(v).split(" ")
|
splitted = str(v).split(" ")
|
||||||
if len(splitted) != 6:
|
if len(splitted) != 6:
|
||||||
@ -279,8 +289,9 @@ class SpreadsheetModel(BaseModel):
|
|||||||
) from e
|
) from e
|
||||||
return v
|
return v
|
||||||
|
|
||||||
@validator("rescutkey")
|
@field_validator('rescutkey', mode="before")
|
||||||
def rescutkey_allowed(cls, v, **kwargs):
|
@classmethod
|
||||||
|
def rescutkey_allowed(cls, v):
|
||||||
if v:
|
if v:
|
||||||
v = v.lower()
|
v = v.lower()
|
||||||
allowed = ["is", "cchalf"]
|
allowed = ["is", "cchalf"]
|
||||||
@ -288,8 +299,9 @@ class SpreadsheetModel(BaseModel):
|
|||||||
raise ValueError(f"' {v} ' is not valid. value must be ' {allowed} '.")
|
raise ValueError(f"' {v} ' is not valid. value must be ' {allowed} '.")
|
||||||
return v
|
return v
|
||||||
|
|
||||||
@validator("autoprocfull", "procfull", "adpenabled", "noano", "ffcscampaign")
|
@field_validator('autoprocfull', 'procfull', 'adpenabled', 'noano', 'ffcscampaign', mode="before")
|
||||||
def boolean_allowed(cls, v, **kwargs):
|
@classmethod
|
||||||
|
def boolean_allowed(cls, v):
|
||||||
if v:
|
if v:
|
||||||
v = v.title()
|
v = v.title()
|
||||||
allowed = ["False", "True"]
|
allowed = ["False", "True"]
|
||||||
@ -300,8 +312,9 @@ class SpreadsheetModel(BaseModel):
|
|||||||
)
|
)
|
||||||
return v
|
return v
|
||||||
|
|
||||||
@validator("trustedhigh")
|
@field_validator('trustedhigh', mode="before")
|
||||||
def trusted_float(cls, v, **kwargs):
|
@classmethod
|
||||||
|
def trusted_float(cls, v):
|
||||||
if v:
|
if v:
|
||||||
try:
|
try:
|
||||||
v = float(v)
|
v = float(v)
|
||||||
@ -319,8 +332,9 @@ class SpreadsheetModel(BaseModel):
|
|||||||
) from e
|
) from e
|
||||||
return v
|
return v
|
||||||
|
|
||||||
@validator("proteinname")
|
@field_validator('proteinname', mode="before")
|
||||||
def proteinname_characters(cls, v, **kwargs):
|
@classmethod
|
||||||
|
def proteinname_characters(cls, v):
|
||||||
if v:
|
if v:
|
||||||
v = str(v).replace(" ", "_")
|
v = str(v).replace(" ", "_")
|
||||||
if re.search("\n", v):
|
if re.search("\n", v):
|
||||||
@ -331,12 +345,13 @@ class SpreadsheetModel(BaseModel):
|
|||||||
v = re.sub(r"\.0$", "", v)
|
v = re.sub(r"\.0$", "", v)
|
||||||
return v
|
return v
|
||||||
|
|
||||||
@validator("chiphiangles")
|
@field_validator('chiphiangles', mode="before")
|
||||||
def chiphiangles_value(cls, v, **kwargs):
|
@classmethod
|
||||||
|
def chiphiangles_value(cls, v):
|
||||||
if v:
|
if v:
|
||||||
try:
|
try:
|
||||||
v = str(v)
|
v = str(v)
|
||||||
re.sub(r"(^\s*\[\s*|\s*\]\s*$)", "", v.strip())
|
v = re.sub(r"(^\s*\[\s*|\s*\]\s*$)", "", v.strip())
|
||||||
list_of_strings = re.findall(r"\(.*?\)", v)
|
list_of_strings = re.findall(r"\(.*?\)", v)
|
||||||
list_of_tuples = []
|
list_of_tuples = []
|
||||||
for el in list_of_strings:
|
for el in list_of_strings:
|
||||||
@ -352,7 +367,7 @@ class SpreadsheetModel(BaseModel):
|
|||||||
) from e
|
) from e
|
||||||
return v
|
return v
|
||||||
|
|
||||||
@validator(
|
@field_validator(
|
||||||
"priority",
|
"priority",
|
||||||
"comments",
|
"comments",
|
||||||
"pinbarcode",
|
"pinbarcode",
|
||||||
@ -380,12 +395,19 @@ class SpreadsheetModel(BaseModel):
|
|||||||
"ffcscampaign",
|
"ffcscampaign",
|
||||||
"autoprocextraparams",
|
"autoprocextraparams",
|
||||||
"chiphiangles",
|
"chiphiangles",
|
||||||
|
mode="before"
|
||||||
)
|
)
|
||||||
def set_default_emptystring(cls, v, **kwargs):
|
@classmethod
|
||||||
|
def set_default_emptystring(cls, v):
|
||||||
return v or ""
|
return v or ""
|
||||||
|
|
||||||
class Config:
|
class Config:
|
||||||
anystr_strip_whitespace = True
|
str_strip_whitespace = True
|
||||||
|
aliases = {
|
||||||
|
'dewarname': 'dewarname',
|
||||||
|
'puckname': 'puckname',
|
||||||
|
'crystalname': 'crystalname',
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class TELLModel(SpreadsheetModel):
|
class TELLModel(SpreadsheetModel):
|
||||||
@ -397,29 +419,3 @@ class TELLModel(SpreadsheetModel):
|
|||||||
puck_number: int
|
puck_number: int
|
||||||
prefix: Optional[str]
|
prefix: Optional[str]
|
||||||
folder: Optional[str]
|
folder: Optional[str]
|
||||||
|
|
||||||
|
|
||||||
"""
|
|
||||||
Following params appended in teller.py for updating SDU sample model
|
|
||||||
class SDUTELLModel(TELLModel):
|
|
||||||
sdudaq: str
|
|
||||||
sdudiffcenter: str
|
|
||||||
sduopticalcenter: str
|
|
||||||
sdumount: str
|
|
||||||
sdusafetycheck: str
|
|
||||||
|
|
||||||
Following params returned in the format expected by tell.set_samples_info()
|
|
||||||
{
|
|
||||||
"userName": user,
|
|
||||||
"dewarName": sample["dewarname"],
|
|
||||||
"puckName": "", # FIXME at the moment this field is useless
|
|
||||||
"puckType": "Unipuck",
|
|
||||||
"puckAddress": sample["puckaddress"],
|
|
||||||
"puckBarcode": sample["puckname"],
|
|
||||||
"sampleBarcode": sample.get("pinbarcode", ""),
|
|
||||||
"sampleMountCount": sample["samplemountcount"],
|
|
||||||
"sampleName": sample["crystalname"],
|
|
||||||
"samplePosition": sample["positioninpuck"],
|
|
||||||
"sampleStatus": sample["samplestatus"],
|
|
||||||
}
|
|
||||||
"""
|
|
@ -1,43 +1,58 @@
|
|||||||
# app/services/spreadsheet_service.py
|
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import openpyxl
|
import openpyxl
|
||||||
from pydantic import ValidationError, parse_obj_as
|
from pydantic import ValidationError, parse_obj_as
|
||||||
from typing import List
|
from typing import List
|
||||||
from app.sample_models import SpreadsheetModel, TELLModel
|
from app.sample_models import SpreadsheetModel
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
|
|
||||||
UNASSIGNED_PUCKADDRESS = "---"
|
UNASSIGNED_PUCKADDRESS = "---"
|
||||||
logging.basicConfig(level=logging.INFO)
|
logging.basicConfig(level=logging.DEBUG) # Change to DEBUG level to see more logs
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class SpreadsheetImportError(Exception):
|
class SpreadsheetImportError(Exception):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
class SampleSpreadsheetImporter:
|
class SampleSpreadsheetImporter:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.filename = None
|
self.filename = None
|
||||||
self.model = None
|
self.model = None
|
||||||
self.available_puck_positions = []
|
self.available_puck_positions = []
|
||||||
|
|
||||||
|
def _clean_value(self, value):
|
||||||
|
"""Clean value by converting it to the expected type and stripping whitespace for strings."""
|
||||||
|
if isinstance(value, str):
|
||||||
|
return value.strip()
|
||||||
|
elif isinstance(value, (float, int)):
|
||||||
|
return str(value) # Always return strings for priority field validation
|
||||||
|
return value
|
||||||
|
|
||||||
def import_spreadsheet(self, file):
|
def import_spreadsheet(self, file):
|
||||||
|
# Reinitialize state
|
||||||
self.available_puck_positions = [
|
self.available_puck_positions = [
|
||||||
f"{s}{p}" for s in list("ABCDEF") for p in range(1, 6)
|
f"{s}{p}" for s in list("ABCDEF") for p in range(1, 6)
|
||||||
]
|
]
|
||||||
self.available_puck_positions.append(UNASSIGNED_PUCKADDRESS)
|
self.available_puck_positions.append(UNASSIGNED_PUCKADDRESS)
|
||||||
|
self.model = []
|
||||||
|
|
||||||
self.filename = file.filename
|
self.filename = file.filename
|
||||||
|
logger.info(f"Importing spreadsheet from .xlsx file: {self.filename}")
|
||||||
|
|
||||||
|
contents = file.file.read()
|
||||||
|
file.file.seek(0) # Reset file pointer to the beginning
|
||||||
|
|
||||||
|
if not contents:
|
||||||
|
logger.error("The uploaded file is empty.")
|
||||||
|
raise SpreadsheetImportError("The uploaded file is empty.")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
logger.info(f"Importing spreadsheet from .xlsx file: {self.filename}")
|
|
||||||
contents = file.file.read() # Read the file contents into memory
|
|
||||||
file.file.seek(0) # Reset file pointer to the beginning
|
|
||||||
workbook = openpyxl.load_workbook(BytesIO(contents))
|
workbook = openpyxl.load_workbook(BytesIO(contents))
|
||||||
|
logger.debug("Workbook loaded successfully")
|
||||||
|
if "Samples" not in workbook.sheetnames:
|
||||||
|
logger.error("The file is missing 'Samples' worksheet.")
|
||||||
|
raise SpreadsheetImportError("The file is missing 'Samples' worksheet.")
|
||||||
sheet = workbook["Samples"]
|
sheet = workbook["Samples"]
|
||||||
except KeyError:
|
|
||||||
raise SpreadsheetImportError("The file is missing 'Samples' worksheet.")
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to read the file: {str(e)}")
|
||||||
raise SpreadsheetImportError(f"Failed to read the file: {str(e)}")
|
raise SpreadsheetImportError(f"Failed to read the file: {str(e)}")
|
||||||
|
|
||||||
return self.process_spreadsheet(sheet)
|
return self.process_spreadsheet(sheet)
|
||||||
@ -47,22 +62,47 @@ class SampleSpreadsheetImporter:
|
|||||||
|
|
||||||
# Skip the first 3 rows
|
# Skip the first 3 rows
|
||||||
rows = list(sheet.iter_rows(min_row=4, values_only=True))
|
rows = list(sheet.iter_rows(min_row=4, values_only=True))
|
||||||
|
logger.debug(f"Starting to process {len(rows)} rows from the sheet")
|
||||||
|
|
||||||
for row in rows:
|
if not rows:
|
||||||
sample = {
|
logger.error("The 'Samples' worksheet is empty.")
|
||||||
'dewarname': self._clean_value(row[0]),
|
raise SpreadsheetImportError("The 'Samples' worksheet is empty.")
|
||||||
'puckname': self._clean_value(row[1]),
|
|
||||||
'crystalname': self._clean_value(row[4])
|
|
||||||
}
|
|
||||||
|
|
||||||
|
for index, row in enumerate(rows):
|
||||||
|
if not row or all(value is None for value in row):
|
||||||
|
logger.debug(f"Skipping empty row or row with all None values at index {index}.")
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
sample = {
|
||||||
|
'dewarname': self._clean_value(row[0]),
|
||||||
|
'puckname': self._clean_value(row[1]),
|
||||||
|
'pucklocationindewar': self._clean_value(row[2]) if len(row) > 2 else None,
|
||||||
|
'positioninpuck': self._clean_value(row[3]) if len(row) > 3 else None,
|
||||||
|
'crystalname': self._clean_value(row[4]),
|
||||||
|
'priority': self._clean_value(row[5]) if len(row) > 5 else None,
|
||||||
|
'comments': self._clean_value(row[6]) if len(row) > 6 else None,
|
||||||
|
'pinbarcode': self._clean_value(row[7]) if len(row) > 7 else None,
|
||||||
|
'directory': self._clean_value(row[8]) if len(row) > 8 else None,
|
||||||
|
}
|
||||||
|
except IndexError:
|
||||||
|
logger.error(f"Index error processing row at index {index}: Row has missing values.")
|
||||||
|
raise SpreadsheetImportError(f"Index error processing row at index {index}: Row has missing values.")
|
||||||
|
|
||||||
|
# Skip rows missing essential fields
|
||||||
if not sample['dewarname'] or not sample['puckname'] or not sample['crystalname']:
|
if not sample['dewarname'] or not sample['puckname'] or not sample['crystalname']:
|
||||||
# Skip rows with missing required fields
|
logger.debug(f"Skipping row due to missing essential fields: {row}")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
model.append(sample)
|
model.append(sample)
|
||||||
logger.info(f"Sample processed: {sample}") # Adding log for each processed sample
|
logger.info(f"Sample processed: {sample}")
|
||||||
|
|
||||||
|
if not model:
|
||||||
|
logger.error("No valid samples found in the spreadsheet.")
|
||||||
|
raise SpreadsheetImportError("No valid samples found in the spreadsheet.")
|
||||||
|
|
||||||
logger.info(f"...finished import, got {len(model)} samples")
|
logger.info(f"...finished import, got {len(model)} samples")
|
||||||
|
logger.debug(f"Model data: {model}")
|
||||||
self.model = model
|
self.model = model
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@ -77,16 +117,15 @@ class SampleSpreadsheetImporter:
|
|||||||
model = self.model
|
model = self.model
|
||||||
logger.info(f"...validating {len(model)} samples")
|
logger.info(f"...validating {len(model)} samples")
|
||||||
|
|
||||||
# Log the model before validation
|
|
||||||
for sample in model:
|
for sample in model:
|
||||||
logger.info(f"Validating sample: {sample}")
|
logger.info(f"Validating sample: {sample}")
|
||||||
|
|
||||||
validated_model = self.data_model_validation(SpreadsheetModel, model)
|
validated_model = self.data_model_validation(SpreadsheetModel, model)
|
||||||
|
|
||||||
# Log the validated model after validation
|
|
||||||
for sample in validated_model:
|
for sample in validated_model:
|
||||||
logger.info(f"Validated sample: {sample}")
|
logger.info(f"Validated sample: {sample}")
|
||||||
|
|
||||||
|
logger.debug(f"Validated model data: {validated_model}")
|
||||||
return validated_model
|
return validated_model
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@ -94,13 +133,8 @@ class SampleSpreadsheetImporter:
|
|||||||
try:
|
try:
|
||||||
validated = parse_obj_as(List[data_model], model)
|
validated = parse_obj_as(List[data_model], model)
|
||||||
except ValidationError as e:
|
except ValidationError as e:
|
||||||
raise SpreadsheetImportError(f"{e.errors()[0]['loc'][2]} => {e.errors()[0]['msg']}")
|
logger.error(f"Validation error: {e.errors()}")
|
||||||
|
raise SpreadsheetImportError(f"{e.errors()[0]['loc']} => {e.errors()[0]['msg']}")
|
||||||
|
|
||||||
validated_model = [dict(value) for value in validated]
|
validated_model = [dict(value) for value in validated]
|
||||||
return validated_model
|
return validated_model
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _clean_value(value):
|
|
||||||
if isinstance(value, str):
|
|
||||||
return value.strip()
|
|
||||||
return value # For other types (int, float, None, etc.), return value as is or handle accordingly
|
|
||||||
|
BIN
backend/test.db
BIN
backend/test.db
Binary file not shown.
@ -13,7 +13,7 @@ import CloseIcon from '@mui/icons-material/Close';
|
|||||||
import DownloadIcon from '@mui/icons-material/Download';
|
import DownloadIcon from '@mui/icons-material/Download';
|
||||||
import UploadFileIcon from '@mui/icons-material/UploadFile';
|
import UploadFileIcon from '@mui/icons-material/UploadFile';
|
||||||
import logo from '../assets/Heidi-logo.png';
|
import logo from '../assets/Heidi-logo.png';
|
||||||
import { OpenAPI, UploadService } from '../../openapi';
|
import { OpenAPI, SpreadsheetService } from '../../openapi';
|
||||||
import type { Body_upload_file_upload_post } from '../../openapi/models/Body_upload_file_upload_post';
|
import type { Body_upload_file_upload_post } from '../../openapi/models/Body_upload_file_upload_post';
|
||||||
|
|
||||||
interface UploadDialogProps {
|
interface UploadDialogProps {
|
||||||
@ -59,7 +59,7 @@ const UploadDialog: React.FC<UploadDialogProps> = ({ open, onClose }) => {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
// Use the generated OpenAPI client UploadService method
|
// Use the generated OpenAPI client UploadService method
|
||||||
const response = await UploadService.uploadFileUploadPost(formData);
|
const response = await SpreadsheetService.uploadFileUploadPost(formData);
|
||||||
|
|
||||||
console.log('File summary response from backend:', response);
|
console.log('File summary response from backend:', response);
|
||||||
console.log('Dewars:', response.dewars);
|
console.log('Dewars:', response.dewars);
|
||||||
@ -87,31 +87,13 @@ const UploadDialog: React.FC<UploadDialogProps> = ({ open, onClose }) => {
|
|||||||
<Box display="flex" flexDirection="column" alignItems="center" mb={2}>
|
<Box display="flex" flexDirection="column" alignItems="center" mb={2}>
|
||||||
<img src={logo} alt="Logo" style={{ width: 200, marginBottom: 16 }} />
|
<img src={logo} alt="Logo" style={{ width: 200, marginBottom: 16 }} />
|
||||||
<Typography variant="subtitle1">Latest Spreadsheet Template Version 6</Typography>
|
<Typography variant="subtitle1">Latest Spreadsheet Template Version 6</Typography>
|
||||||
<Typography variant="body2" color="textSecondary">
|
<Typography variant="body2" color="textSecondary">Last update: October 18, 2024</Typography>
|
||||||
Last update: October 18, 2024
|
<Button variant="outlined" startIcon={<DownloadIcon />} href="/path/to/template.xlsx" download sx={{ mt: 1 }}>
|
||||||
</Typography>
|
|
||||||
<Button
|
|
||||||
variant="outlined"
|
|
||||||
startIcon={<DownloadIcon />}
|
|
||||||
href="/path/to/template.xlsx"
|
|
||||||
download
|
|
||||||
sx={{ mt: 1 }}
|
|
||||||
>
|
|
||||||
Download XLSX
|
Download XLSX
|
||||||
</Button>
|
</Button>
|
||||||
<Typography variant="subtitle1" sx={{ mt: 3 }}>
|
<Typography variant="subtitle1" sx={{ mt: 3 }}>Latest Spreadsheet Instructions Version 2.3</Typography>
|
||||||
Latest Spreadsheet Instructions Version 2.3
|
<Typography variant="body2" color="textSecondary">Last updated: October 18, 2024</Typography>
|
||||||
</Typography>
|
<Button variant="outlined" startIcon={<DownloadIcon />} href="/path/to/instructions.pdf" download sx={{ mt: 1 }}>
|
||||||
<Typography variant="body2" color="textSecondary">
|
|
||||||
Last updated: October 18, 2024
|
|
||||||
</Typography>
|
|
||||||
<Button
|
|
||||||
variant="outlined"
|
|
||||||
startIcon={<DownloadIcon />}
|
|
||||||
href="/path/to/instructions.pdf"
|
|
||||||
download
|
|
||||||
sx={{ mt: 1 }}
|
|
||||||
>
|
|
||||||
Download PDF
|
Download PDF
|
||||||
</Button>
|
</Button>
|
||||||
</Box>
|
</Box>
|
||||||
@ -148,9 +130,7 @@ const UploadDialog: React.FC<UploadDialogProps> = ({ open, onClose }) => {
|
|||||||
</Box>
|
</Box>
|
||||||
</DialogContent>
|
</DialogContent>
|
||||||
<DialogActions>
|
<DialogActions>
|
||||||
<Button onClick={onClose} color="primary">
|
<Button onClick={onClose} color="primary">Close</Button>
|
||||||
Close
|
|
||||||
</Button>
|
|
||||||
</DialogActions>
|
</DialogActions>
|
||||||
</Dialog>
|
</Dialog>
|
||||||
);
|
);
|
||||||
|
Reference in New Issue
Block a user