Refactor job model and endpoints for improved structure
Updated the job model to include `sample_id` and `run_id` fields, replacing `experiment_parameters_id`. Adjusted relationships and modified routers to reflect these changes. Added an endpoint for updating job status and restructured job streaming logic to include detailed experiment and sample data.
This commit is contained in:
GotthardG
@ -9,7 +9,7 @@
|
||||
"is_executing": true
|
||||
},
|
||||
"ExecuteTime": {
|
||||
"start_time": "2025-04-29T07:44:02.103530Z"
|
||||
"start_time": "2025-04-29T12:21:15.813326Z"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
@ -18,13 +18,24 @@
|
||||
"import json\n",
|
||||
"\n",
|
||||
"SSE_URL = \"https://127.0.0.1:8000/processing/jobs/stream\"\n",
|
||||
"UPDATE_URL = \"https://127.0.0.1:8000/processing/jobs/update_status\"\n",
|
||||
"\n",
|
||||
"def submit_job_update(job_id, status):\n",
|
||||
" payload = {\n",
|
||||
" \"job_id\": job_id,\n",
|
||||
" \"status\": status\n",
|
||||
" }\n",
|
||||
" try:\n",
|
||||
" response = requests.post(UPDATE_URL, json=payload, verify=False)\n",
|
||||
" if response.status_code == 200:\n",
|
||||
" print(f\"✅ Job {job_id} status updated to '{status}'. Response: {response.json()}\")\n",
|
||||
" else:\n",
|
||||
" print(f\"❌ Failed to update job {job_id}. Status: {response.status_code}. Response: {response.text}\")\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"Failed to submit update for Job {job_id}: {e}\")\n",
|
||||
"\n",
|
||||
"#SSE_URL = \"https://mx-aare-test.psi.ch:1492/processing/jobs/stream\"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def listen_and_process_jobs(url):\n",
|
||||
" print(\"Starting processing pipeline...\")\n",
|
||||
"def listen_and_update_jobs(url):\n",
|
||||
" print(\"Starting job status updater...\")\n",
|
||||
" with requests.get(url, stream=True, verify=False) as response:\n",
|
||||
" if response.status_code != 200:\n",
|
||||
" print(f\"Failed to connect with status code: {response.status_code}\")\n",
|
||||
@ -37,69 +48,25 @@
|
||||
" jobs = json.loads(event.data)\n",
|
||||
" print(f\"Jobs received: {jobs}\")\n",
|
||||
"\n",
|
||||
" for job in jobs:\n",
|
||||
" job_id = job.get(\"id\")\n",
|
||||
" parameters = job.get(\"parameters\")\n",
|
||||
" if parameters is None:\n",
|
||||
" print(\n",
|
||||
" f\"⚠️ Job {job_id if job_id is not None else '[unknown id]'} has no 'parameters'; skipping.\")\n",
|
||||
" continue\n",
|
||||
"\n",
|
||||
" print(f\"Processing job ID: {job_id} with parameters: {parameters}\")\n",
|
||||
"\n",
|
||||
" # Your custom job-processing logic goes here\n",
|
||||
" process_job_logic(job_id, parameters)\n",
|
||||
"\n",
|
||||
" # After job processing completes, send results & update job status\n",
|
||||
" submit_job_result(job_id, processing_result={'success': True})\n",
|
||||
" #for job in jobs:\n",
|
||||
" # job_id = job.get(\"job_id\")\n",
|
||||
" # print(f\"Job ID: {job_id}, Current status: {job.get('status')}\")\n",
|
||||
" # # Immediately update status to \"submitted\"\n",
|
||||
" # submit_job_update(job_id, \"submitted\")\n",
|
||||
" except json.JSONDecodeError as e:\n",
|
||||
" print(f\"Error decoding event data: {e}\")\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"Unexpected error while processing event: {e}\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def process_job_logic(job_id, parameters):\n",
|
||||
" print(f\"📦 {job_id} - Processing detailed logic here: {parameters}\")\n",
|
||||
" import time\n",
|
||||
" time.sleep(5)\n",
|
||||
" print(f\"✅ Job {job_id} processing complete\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def submit_job_result(job_id, processing_result):\n",
|
||||
" import backend.aareDBclient as aareDBclient\n",
|
||||
" from aareDBclient.models import ResultCreate\n",
|
||||
"\n",
|
||||
" configuration = aareDBclient.Configuration(\n",
|
||||
" host=\"https://127.0.0.1:8000\",\n",
|
||||
" verify_ssl=False\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" with aareDBclient.ApiClient(configuration) as api_client:\n",
|
||||
" api_instance = aareDBclient.ProcessingApi(api_client)\n",
|
||||
"\n",
|
||||
" result = ResultCreate(\n",
|
||||
" job_id=job_id,\n",
|
||||
" result=processing_result\n",
|
||||
" )\n",
|
||||
" try:\n",
|
||||
" res = api_instance.submit_processing_result(result_create=result)\n",
|
||||
" print(f\"Job {job_id} result submitted successfully! Backend Response: {res}\")\n",
|
||||
" except aareDBclient.rest.ApiException as e:\n",
|
||||
" print(f\"Failed to submit result for Job {job_id}: {e}\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"if __name__ == \"__main__\":\n",
|
||||
" listen_and_process_jobs(SSE_URL)"
|
||||
" listen_and_update_jobs(SSE_URL)\n"
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Starting processing pipeline...\n",
|
||||
"Jobs received: [{'status': 'todo', 'updated_at': None, 'created_at': '2025-04-29T07:40:46.419291', 'id': 1, 'experiment_parameters_id': 1}, {'status': 'todo', 'updated_at': None, 'created_at': '2025-04-29T07:41:44.451862', 'id': 2, 'experiment_parameters_id': 2}]\n",
|
||||
"⚠️ Job 1 has no 'parameters'; skipping.\n",
|
||||
"⚠️ Job 2 has no 'parameters'; skipping.\n"
|
||||
"Starting job status updater...\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -109,39 +76,6 @@
|
||||
"/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages/urllib3/connectionpool.py:1103: InsecureRequestWarning: Unverified HTTPS request is being made to host '127.0.0.1'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings\n",
|
||||
" warnings.warn(\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Jobs received: [{'status': 'todo', 'updated_at': None, 'created_at': '2025-04-29T07:40:46.419291', 'id': 1, 'experiment_parameters_id': 1}, {'status': 'todo', 'updated_at': None, 'created_at': '2025-04-29T07:41:44.451862', 'id': 2, 'experiment_parameters_id': 2}]\n",
|
||||
"⚠️ Job 1 has no 'parameters'; skipping.\n",
|
||||
"⚠️ Job 2 has no 'parameters'; skipping.\n",
|
||||
"Jobs received: [{'status': 'todo', 'updated_at': None, 'created_at': '2025-04-29T07:40:46.419291', 'id': 1, 'experiment_parameters_id': 1}, {'status': 'todo', 'updated_at': None, 'created_at': '2025-04-29T07:41:44.451862', 'id': 2, 'experiment_parameters_id': 2}]\n",
|
||||
"⚠️ Job 1 has no 'parameters'; skipping.\n",
|
||||
"⚠️ Job 2 has no 'parameters'; skipping.\n",
|
||||
"Jobs received: [{'status': 'todo', 'updated_at': None, 'created_at': '2025-04-29T07:40:46.419291', 'id': 1, 'experiment_parameters_id': 1}, {'status': 'todo', 'updated_at': None, 'created_at': '2025-04-29T07:41:44.451862', 'id': 2, 'experiment_parameters_id': 2}]\n",
|
||||
"⚠️ Job 1 has no 'parameters'; skipping.\n",
|
||||
"⚠️ Job 2 has no 'parameters'; skipping.\n",
|
||||
"Jobs received: [{'status': 'todo', 'updated_at': None, 'created_at': '2025-04-29T07:40:46.419291', 'id': 1, 'experiment_parameters_id': 1}, {'status': 'todo', 'updated_at': None, 'created_at': '2025-04-29T07:41:44.451862', 'id': 2, 'experiment_parameters_id': 2}]\n",
|
||||
"⚠️ Job 1 has no 'parameters'; skipping.\n",
|
||||
"⚠️ Job 2 has no 'parameters'; skipping.\n",
|
||||
"Jobs received: [{'status': 'todo', 'updated_at': None, 'created_at': '2025-04-29T07:40:46.419291', 'id': 1, 'experiment_parameters_id': 1}, {'status': 'todo', 'updated_at': None, 'created_at': '2025-04-29T07:41:44.451862', 'id': 2, 'experiment_parameters_id': 2}]\n",
|
||||
"⚠️ Job 1 has no 'parameters'; skipping.\n",
|
||||
"⚠️ Job 2 has no 'parameters'; skipping.\n",
|
||||
"Jobs received: [{'status': 'todo', 'updated_at': None, 'created_at': '2025-04-29T07:40:46.419291', 'id': 1, 'experiment_parameters_id': 1}, {'status': 'todo', 'updated_at': None, 'created_at': '2025-04-29T07:41:44.451862', 'id': 2, 'experiment_parameters_id': 2}]\n",
|
||||
"⚠️ Job 1 has no 'parameters'; skipping.\n",
|
||||
"⚠️ Job 2 has no 'parameters'; skipping.\n",
|
||||
"Jobs received: [{'status': 'todo', 'updated_at': None, 'created_at': '2025-04-29T07:40:46.419291', 'id': 1, 'experiment_parameters_id': 1}, {'status': 'todo', 'updated_at': None, 'created_at': '2025-04-29T07:41:44.451862', 'id': 2, 'experiment_parameters_id': 2}]\n",
|
||||
"⚠️ Job 1 has no 'parameters'; skipping.\n",
|
||||
"⚠️ Job 2 has no 'parameters'; skipping.\n",
|
||||
"Jobs received: [{'status': 'todo', 'updated_at': None, 'created_at': '2025-04-29T07:40:46.419291', 'id': 1, 'experiment_parameters_id': 1}, {'status': 'todo', 'updated_at': None, 'created_at': '2025-04-29T07:41:44.451862', 'id': 2, 'experiment_parameters_id': 2}]\n",
|
||||
"⚠️ Job 1 has no 'parameters'; skipping.\n",
|
||||
"⚠️ Job 2 has no 'parameters'; skipping.\n",
|
||||
"Jobs received: [{'status': 'todo', 'updated_at': None, 'created_at': '2025-04-29T07:40:46.419291', 'id': 1, 'experiment_parameters_id': 1}, {'status': 'todo', 'updated_at': None, 'created_at': '2025-04-29T07:41:44.451862', 'id': 2, 'experiment_parameters_id': 2}]\n",
|
||||
"⚠️ Job 1 has no 'parameters'; skipping.\n",
|
||||
"⚠️ Job 2 has no 'parameters'; skipping.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"execution_count": null
|
||||
|
||||
Reference in New Issue
Block a user