Add job cancellation handling and periodic cleanup logic
Introduce new statuses, "to_cancel" and "cancelled", to improve job state tracking. Implement logic to nullify `slurm_id` for cancelled jobs and a background thread to clean up cancelled jobs older than 2 hours. Ensure periodic cleanup runs hourly to maintain database hygiene.
This commit is contained in:
GotthardG
@ -4,13 +4,14 @@ import RunDetails from './RunDetails';
|
||||
import './SampleImage.css';
|
||||
import './ResultGrid.css';
|
||||
import { OpenAPI, SamplesService } from '../../openapi';
|
||||
import CheckCircleIcon from '@mui/icons-material/CheckCircle';
|
||||
import CancelIcon from '@mui/icons-material/Cancel';
|
||||
import ScheduleIcon from '@mui/icons-material/Schedule';
|
||||
import AutorenewIcon from '@mui/icons-material/Autorenew';
|
||||
import TaskAltIcon from '@mui/icons-material/TaskAlt';
|
||||
import ErrorOutlineIcon from '@mui/icons-material/ErrorOutline';
|
||||
import InfoOutlinedIcon from '@mui/icons-material/InfoOutlined';
|
||||
import HourglassEmptyIcon from '@mui/icons-material/HourglassEmpty';
|
||||
import RemoveCircleOutlineIcon from '@mui/icons-material/RemoveCircleOutline';
|
||||
// import ErrorIcon from '@mui/icons-material/Error';
|
||||
// import AccessTimeIcon from '@mui/icons-material/AccessTime';
|
||||
|
||||
|
||||
|
||||
|
||||
// Extend your image info interface if needed.
|
||||
@ -102,6 +103,8 @@ interface TreeRow {
|
||||
beamline_parameters?: ExperimentParameters['beamline_parameters'];
|
||||
experimentType?: string;
|
||||
numberOfImages?: number;
|
||||
hasResults: boolean;
|
||||
jobStatus?: string;
|
||||
}
|
||||
|
||||
interface ResultGridProps {
|
||||
@ -113,9 +116,9 @@ const useJobStream = (onJobs: (jobs: any[]) => void) => {
|
||||
|
||||
useEffect(() => {
|
||||
eventSourceRef.current = new EventSource(`${OpenAPI.BASE}/processing/jobs/stream`);
|
||||
eventSourceRef.current.onmessage = (event) => {
|
||||
// Receives: data: [{job_id, run_id, status, ...}, ...]
|
||||
const jobs = JSON.parse(event.data);
|
||||
eventSourceRef.current.onmessage = async (event) => {
|
||||
const jobs = JSON.parse(event.data); // Updated job data
|
||||
|
||||
onJobs(jobs);
|
||||
};
|
||||
|
||||
@ -128,25 +131,30 @@ const useJobStream = (onJobs: (jobs: any[]) => void) => {
|
||||
};
|
||||
|
||||
|
||||
|
||||
const ResultGrid: React.FC<ResultGridProps> = ({ activePgroup }) => {
|
||||
const [rows, setRows] = useState<TreeRow[]>([]);
|
||||
const [basePath, setBasePath] = useState('');
|
||||
const [detailPanelHeights, setDetailPanelHeights] = useState<{ [key: string]: number }>({}); // Store dynamic heights
|
||||
const [jobStatusMap, setJobStatusMap] = useState<{ [runId: number]: string }>({});
|
||||
|
||||
const getStatusIcon = (status: string) => {
|
||||
const getStatusIcon = (status: string, hasResults: boolean = false) => {
|
||||
switch (status) {
|
||||
case 'todo':
|
||||
return <HourglassEmptyIcon color="action" titleAccess="Todo" />;
|
||||
return <ScheduleIcon color="action" titleAccess="Todo" />;
|
||||
case 'submitted':
|
||||
return <AutorenewIcon color="primary" className="spin" titleAccess="Submitted" />;
|
||||
return <HourglassEmptyIcon color="primary" className="spin" titleAccess="Submitted" />;
|
||||
case 'completed':
|
||||
return <CheckCircleIcon color="success" titleAccess="Completed" />;
|
||||
return hasResults ? (
|
||||
<TaskAltIcon color="success" titleAccess="Completed" />
|
||||
) : (
|
||||
<InfoOutlinedIcon color="warning" titleAccess="Completed - No Results" />
|
||||
);
|
||||
case 'failed':
|
||||
return <CancelIcon color="error" titleAccess="Failed" />;
|
||||
return <ErrorOutlineIcon color="error" titleAccess="Failed" />;
|
||||
case 'no job':
|
||||
default:
|
||||
return <RemoveCircleOutlineIcon color="disabled" titleAccess="No job" />;
|
||||
return <InfoOutlinedIcon color="disabled" titleAccess="No job" />;
|
||||
}
|
||||
};
|
||||
|
||||
@ -160,6 +168,37 @@ const ResultGrid: React.FC<ResultGridProps> = ({ activePgroup }) => {
|
||||
setJobStatusMap(map);
|
||||
});
|
||||
|
||||
const handleJobs = async (jobs: any[]) => {
|
||||
console.log('Jobs received from the job stream:', jobs);
|
||||
|
||||
// Fetch results for each run based on the job stream
|
||||
const updatedRows = await Promise.all(
|
||||
rows.map(async (row) => {
|
||||
if (row.type === 'run' && row.experimentId) {
|
||||
try {
|
||||
const results = await SamplesService.getResultsForRunAndSample(
|
||||
row.sample_id,
|
||||
row.experimentId
|
||||
);
|
||||
const hasResults = results.length > 0;
|
||||
console.log(`Fetching results for experimentId: ${row.experimentId}, hasResults: ${hasResults}`);
|
||||
return { ...row, hasResults }; // Update `hasResults` for the run
|
||||
} catch (error) {
|
||||
console.error(`Error fetching results for experimentId: ${row.experimentId}`, error);
|
||||
return row; // Return unchanged row on error
|
||||
}
|
||||
}
|
||||
return row; // Return unchanged for non-run rows
|
||||
})
|
||||
);
|
||||
|
||||
// Update the rows state with new `hasResults` values
|
||||
setRows(updatedRows);
|
||||
};
|
||||
|
||||
useJobStream(handleJobs);
|
||||
|
||||
|
||||
const hasProcessingResults = (row: TreeRow): boolean => {
|
||||
// You can later replace this placeholder with actual logic.
|
||||
// Mocking the logic by returning `true` for demonstration.
|
||||
@ -269,8 +308,8 @@ const ResultGrid: React.FC<ResultGridProps> = ({ activePgroup }) => {
|
||||
beamline_parameters: run.beamline_parameters,
|
||||
experimentType,
|
||||
numberOfImages: numImages,
|
||||
images: sample.images.filter(img =>
|
||||
img.event_type === "Collecting" ),
|
||||
images: sample.images.filter(img => img.event_type === "Collecting"),
|
||||
hasResults: false, // Default to false until verified
|
||||
};
|
||||
treeRows.push(runRow);
|
||||
});
|
||||
@ -295,11 +334,28 @@ const ResultGrid: React.FC<ResultGridProps> = ({ activePgroup }) => {
|
||||
field: 'jobStatus',
|
||||
headerName: 'Job Status',
|
||||
width: 120,
|
||||
renderCell: (params) =>
|
||||
params.row.type === 'run'
|
||||
? getStatusIcon(jobStatusMap[params.row.experimentId] || 'no job')
|
||||
: null,
|
||||
renderCell: (params) => {
|
||||
if (params.row.type === 'run') {
|
||||
const hasResults = params.row.hasResults || false; // Check for results
|
||||
const jobStatus = jobStatusMap[params.row.experimentId] || 'no job'; // Fetch job status
|
||||
|
||||
// If there are results, only show the TaskAltIcon (no need for job status tracking)
|
||||
if (hasResults) {
|
||||
return (
|
||||
<div style={{ display: 'flex', alignItems: 'center', gap: '5px' }}>
|
||||
<TaskAltIcon color="success" titleAccess="Results available" />
|
||||
<span style={{ fontSize: '0.75rem', color: '#4caf50' }}>Results</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// Otherwise, show the job tracking status icon
|
||||
return getStatusIcon(jobStatus, hasResults);
|
||||
}
|
||||
return null; // No rendering for non-run rows
|
||||
},
|
||||
}
|
||||
,
|
||||
{
|
||||
field: 'puck_name',
|
||||
headerName: 'Puck Name',
|
||||
@ -366,6 +422,20 @@ const ResultGrid: React.FC<ResultGridProps> = ({ activePgroup }) => {
|
||||
});
|
||||
};
|
||||
|
||||
const handleResultsFetched = (runId: number, hasResults: boolean) => {
|
||||
console.log(`handleResultsFetched called for RunId ${runId}, hasResults: ${hasResults}`);
|
||||
setRows((prevRows) =>
|
||||
prevRows.map((row) => {
|
||||
if (row.type === 'run' && row.experimentId === runId) {
|
||||
console.log(`Updating row for runId ${runId}, setting hasResults=${hasResults}`);
|
||||
return { ...row, hasResults };
|
||||
}
|
||||
return row;
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
|
||||
const getDetailPanelContent = (params: any) => {
|
||||
if (params.row.type === 'run') {
|
||||
return (
|
||||
@ -374,13 +444,16 @@ const ResultGrid: React.FC<ResultGridProps> = ({ activePgroup }) => {
|
||||
runId={params.row.experimentId}
|
||||
sample_id={params.row.sample_id}
|
||||
basePath={basePath}
|
||||
onHeightChange={height => handleDetailPanelHeightChange(params.row.id, height)}
|
||||
onHeightChange={(height) => handleDetailPanelHeightChange(params.row.id, height)}
|
||||
onResultsFetched={(runId, hasResults) => handleResultsFetched(runId, hasResults)}
|
||||
/>
|
||||
);
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
|
||||
|
||||
const getDetailPanelHeight = (params: any) => {
|
||||
if (params.row.type === 'run') {
|
||||
// Use the dynamically calculated height from state
|
||||
@ -393,6 +466,7 @@ const ResultGrid: React.FC<ResultGridProps> = ({ activePgroup }) => {
|
||||
|
||||
return (
|
||||
<DataGridPremium
|
||||
key={JSON.stringify(rows)}
|
||||
rows={rows}
|
||||
columns={columns}
|
||||
getRowId={(row) => row.id}
|
||||
@ -423,4 +497,3 @@ const ResultGrid: React.FC<ResultGridProps> = ({ activePgroup }) => {
|
||||
};
|
||||
|
||||
export default ResultGrid;
|
||||
|
||||
|
||||
@ -16,6 +16,7 @@ interface RunDetailsProps {
|
||||
sample_id: number;
|
||||
basePath: string;
|
||||
onHeightChange?: (height: number) => void;
|
||||
onResultsFetched: (runId: number, hasResults: boolean) => void; // New callback
|
||||
}
|
||||
|
||||
interface CCPoint {
|
||||
@ -52,7 +53,9 @@ interface ProcessingResults {
|
||||
}
|
||||
|
||||
|
||||
const RunDetails: React.FC<RunDetailsProps> = ({ run, onHeightChange, basePath, runId, sample_id }) => {
|
||||
const RunDetails: React.FC<RunDetailsProps> = ({ run, onHeightChange, basePath, runId, sample_id, onResultsFetched }) => {
|
||||
console.log('onResultsFetched is available:', onResultsFetched);
|
||||
|
||||
const containerRef = useRef<HTMLDivElement | null>(null);
|
||||
const [currentHeight, setCurrentHeight] = useState<number>(0);
|
||||
const [modalOpen, setModalOpen] = useState<boolean>(false);
|
||||
@ -68,14 +71,14 @@ const RunDetails: React.FC<RunDetailsProps> = ({ run, onHeightChange, basePath,
|
||||
}, [runId]);
|
||||
|
||||
const fetchResults = async (sample_id: number, runId: number) => {
|
||||
console.log(`Fetching results for sample_id: ${sample_id}, runId: ${runId}`);
|
||||
try {
|
||||
const results = await SamplesService.getResultsForRunAndSample(sample_id, runId);
|
||||
|
||||
// Explicitly handle nested results
|
||||
const mappedResults: ProcessingResults[] = results.map((res): ProcessingResults => ({
|
||||
id: res.id,
|
||||
pipeline: res.result?.pipeline || 'N/A',
|
||||
resolution: res.result.resolution ?? 0,
|
||||
resolution: res.result?.resolution ?? 0,
|
||||
unit_cell: res.result?.unit_cell || 'N/A',
|
||||
spacegroup: res.result?.spacegroup || 'N/A',
|
||||
rmerge: res.result?.rmerge ?? 0,
|
||||
@ -92,8 +95,13 @@ const RunDetails: React.FC<RunDetailsProps> = ({ run, onHeightChange, basePath,
|
||||
}));
|
||||
|
||||
setProcessingResult(mappedResults);
|
||||
|
||||
console.log(`Mapped results for runId ${runId}:`, mappedResults);
|
||||
console.log(`Boolean value for hasResults: ${mappedResults.length > 0}`);
|
||||
onResultsFetched(runId, mappedResults.length > 0);
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error fetching results:', error);
|
||||
console.error(`Error fetching results for RunId ${runId}:`, error);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user