aaredb/backend/propipe_sim.ipynb

In [14]:

import requests
import sseclient
import json

SSE_URL = "https://127.0.0.1:8000/processing/jobs/stream"
UPDATE_URL = "https://127.0.0.1:8000/processing/jobs/update_status"

def submit_job_update(job_id, status, slurm_id):
    payload = {
        "job_id": job_id,
        "status": status,
        "slurm_id": slurm_id,
    }
    try:
        response = requests.post(UPDATE_URL, json=payload, verify=False)
        if response.status_code == 200:
            print(f"✅ Job {job_id} status updated to '{status}'. Response: {response.json()}")
        else:
            print(f"❌ Failed to update job {job_id}. Status: {response.status_code}. Response: {response.text}")
    except Exception as e:
        print(f"Failed to submit update for Job {job_id}: {e}")

def listen_and_update_jobs(url):
    print("Starting job status updater...")
    with requests.get(url, stream=True, verify=False) as response:
        if response.status_code != 200:
            print(f"Failed to connect with status code: {response.status_code}")
            return

        client = sseclient.SSEClient(response)

        for event in client.events():
            try:
                jobs = json.loads(event.data)
                print(f"Jobs received: {jobs}")

                for job in jobs:
                    job_id = job.get("job_id")
                    print(f"Job ID: {job_id}, Current status: {job.get('status')}")
                    # Immediately update status to "submitted"
                    submit_job_update(job_id, "submitted", 76545678)
            except json.JSONDecodeError as e:
                print(f"Error decoding event data: {e}")
            except Exception as e:
                print(f"Unexpected error while processing event: {e}")

if __name__ == "__main__":
    listen_and_update_jobs(SSE_URL)

Starting job status updater...
Jobs received: [{'job_id': 4, 'sample_id': 204, 'run_id': 1, 'sample_name': 'Sample204', 'status': 'todo', 'type': 'standard', 'created_at': '2025-04-30T09:05:14.901478', 'updated_at': None, 'data_collection_parameters': None, 'experiment_parameters': {'synchrotron': 'Swiss Light Source', 'beamline': 'PXIII', 'detector': {'manufacturer': 'DECTRIS', 'model': 'PILATUS4 2M', 'type': 'photon-counting', 'serialNumber': '16684dscsd668468', 'detectorDistance_mm': 95.0, 'beamCenterX_px': 512.0, 'beamCenterY_px': 512.0, 'pixelSizeX_um': 150.0, 'pixelSizeY_um': 150.0}, 'wavelength': 1.0, 'ringCurrent_A': 0.0, 'ringMode': 'Machine Down', 'undulator': None, 'undulatorgap_mm': None, 'monochromator': 'Si111', 'transmission': 1.0, 'focusingOptic': 'Kirkpatrick-Baez', 'beamlineFluxAtSample_ph_s': 0.0, 'beamSizeWidth': 30.0, 'beamSizeHeight': 30.0, 'characterization': None, 'rotation': {'omegaStart_deg': 0.0, 'omegaStep': 0.1, 'chi': 0.0, 'phi': 10.0, 'numberOfImages': 3600, 'exposureTime_s': 0.02}, 'gridScan': None, 'jet': None, 'cryojetTemperature_K': None, 'humidifierTemperature_K': None, 'humidifierHumidity': None}, 'filepath': '/das/work/p11/p11206/raw_data/vincent/20250415_6D_SLS2_1st_data/20250415_fullbeam_dtz220_Lyso102_again_360deg', 'slurm_id': None}]
Job ID: 4, Current status: todo
✅ Job 4 status updated to 'submitted'. Response: {'job_id': 4, 'status': 'submitted', 'slurm_id': 76545678}

/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages/urllib3/connectionpool.py:1103: InsecureRequestWarning: Unverified HTTPS request is being made to host '127.0.0.1'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings
  warnings.warn(
/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages/urllib3/connectionpool.py:1103: InsecureRequestWarning: Unverified HTTPS request is being made to host '127.0.0.1'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings
  warnings.warn(

---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
Cell In[14], line 48
     45                 print(f"Unexpected error while processing event: {e}")
     47 if __name__ == "__main__":
---> 48     listen_and_update_jobs(SSE_URL)

Cell In[14], line 32, in listen_and_update_jobs(url)
     28     return
     30 client = sseclient.SSEClient(response)
---> 32 for event in client.events():
     33     try:
     34         jobs = json.loads(event.data)

File /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages/sseclient/__init__.py:55, in SSEClient.events(self)
     54 def events(self):
---> 55     for chunk in self._read():
     56         event = Event()
     57         # Split before decoding so splitlines() only uses \r and \n

File /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages/sseclient/__init__.py:45, in SSEClient._read(self)
     38 """Read the incoming event source stream and yield event chunks.
     39 
     40 Unfortunately it is possible for some servers to decide to break an
     41 event into multiple HTTP chunks in the response. It is thus necessary
     42 to correctly stitch together consecutive response chunks and find the
     43 SSE delimiter (empty new line) to yield full, correct event chunks."""
     44 data = b''
---> 45 for chunk in self._event_source:
     46     for line in chunk.splitlines(True):
     47         data += line

File /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages/requests/models.py:820, in Response.iter_content.<locals>.generate()
    818 if hasattr(self.raw, "stream"):
    819     try:
--> 820         yield from self.raw.stream(chunk_size, decode_content=True)
    821     except ProtocolError as e:
    822         raise ChunkedEncodingError(e)

File /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages/urllib3/response.py:1040, in HTTPResponse.stream(self, amt, decode_content)
   1024 """
   1025 A generator wrapper for the read() method. A call will block until
   1026 ``amt`` bytes have been read from the connection or until the
   (...)
   1037     'content-encoding' header.
   1038 """
   1039 if self.chunked and self.supports_chunked_reads():
-> 1040     yield from self.read_chunked(amt, decode_content=decode_content)
   1041 else:
   1042     while not is_fp_closed(self._fp) or len(self._decoded_buffer) > 0:

File /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages/urllib3/response.py:1184, in HTTPResponse.read_chunked(self, amt, decode_content)
   1181     return None
   1183 while True:
-> 1184     self._update_chunk_length()
   1185     if self.chunk_left == 0:
   1186         break

File /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages/urllib3/response.py:1108, in HTTPResponse._update_chunk_length(self)
   1106 if self.chunk_left is not None:
   1107     return None
-> 1108 line = self._fp.fp.readline()  # type: ignore[union-attr]
   1109 line = line.split(b";", 1)[0]
   1110 try:

File /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/socket.py:707, in SocketIO.readinto(self, b)
    705 while True:
    706     try:
--> 707         return self._sock.recv_into(b)
    708     except timeout:
    709         self._timeout_occurred = True

File /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/ssl.py:1216, in SSLSocket.recv_into(self, buffer, nbytes, flags)
   1212     if flags != 0:
   1213         raise ValueError(
   1214           "non-zero flags not allowed in calls to recv_into() on %s" %
   1215           self.__class__)
-> 1216     return self.read(nbytes, buffer)
   1217 else:
   1218     return super().recv_into(buffer, nbytes, flags)

File /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/ssl.py:1072, in SSLSocket.read(self, len, buffer)
   1070 try:
   1071     if buffer is not None:
-> 1072         return self._sslobj.read(len, buffer)
   1073     else:
   1074         return self._sslobj.read(len)

KeyboardInterrupt: