From 634c045523f63be135a3d874b624a86df70ee3e0 Mon Sep 17 00:00:00 2001 From: romainsacchi Date: Sat, 19 Sep 2026 22:35:13 +0200 Subject: [PATCH 1/4] Update CLIC image to reviewed archive release 5bbfcaf --- docker-compose.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index cf60dd1..35cc312 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -147,7 +147,7 @@ services: # Romain Sacchi # CLIC Django platform clic-web: - image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:754f9f8} + image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:5bbfcaf} container_name: clic-web restart: always env_file: @@ -183,7 +183,7 @@ services: - clic_backend clic-package-exports: - image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:754f9f8} + image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:5bbfcaf} container_name: clic-package-exports restart: always env_file: @@ -209,7 +209,7 @@ services: - clic_backend clic-brightway-json-exports: - image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:754f9f8} + image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:5bbfcaf} container_name: clic-brightway-json-exports restart: always env_file: From 64959d3d29dbb1d625a271a678449fdb119ff658 Mon Sep 17 00:00:00 2001 From: romainsacchi Date: Sat, 19 Sep 2026 23:40:38 +0200 Subject: [PATCH 2/4] Update CLIC image for verified Zenodo nightly uploads --- docker-compose.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 35cc312..bb8309b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -147,7 +147,7 @@ services: # Romain Sacchi # CLIC Django platform clic-web: - image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:5bbfcaf} + image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:58e902a} container_name: clic-web restart: always env_file: @@ -183,7 +183,7 @@ services: - clic_backend clic-package-exports: - image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:5bbfcaf} + image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:58e902a} container_name: clic-package-exports restart: always env_file: @@ -209,7 +209,7 @@ services: - clic_backend clic-brightway-json-exports: - image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:5bbfcaf} + image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:58e902a} container_name: clic-brightway-json-exports restart: always env_file: From fea43ce1cbef61f64f510b5ce1c651a7ee0d53f3 Mon Sep 17 00:00:00 2001 From: romainsacchi Date: Sun, 20 Sep 2026 20:17:35 +0200 Subject: [PATCH 3/4] Automate CLIC archive recovery and nightly backups in Compose --- CLIC-ARCHIVE.md | 61 ++++++++++++++++++++++++++++++++++++++ docker-compose.yaml | 71 +++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 127 insertions(+), 5 deletions(-) create mode 100644 CLIC-ARCHIVE.md diff --git a/CLIC-ARCHIVE.md b/CLIC-ARCHIVE.md new file mode 100644 index 0000000..2d6464e --- /dev/null +++ b/CLIC-ARCHIVE.md @@ -0,0 +1,61 @@ +# CLIC archive cutover: operator actions + +This deployment enables the approved inventory-only Zenodo recovery and nightly backup route. +Heroku is frozen; its database and media are retained for rollback. Do not reopen its writers +or enable another archive publisher while PSI is authoritative. + +1. Add `INVENTORY_ARCHIVE_TOKEN` to the existing protected + `/mount/webhosting_dmz_write/clic/clic.env`, using the production Zenodo token supplied + securely by Romain. Keep all existing database, Django and SMTP settings. Do not attach the + filled env file or token to this repository, pull request or issue. +2. Merge/deploy the Compose change and recreate all four CLIC application services. Ensure + any host `CLIC_IMAGE` override selects `gitea.psi.ch/images/clic:ae59c15` or is removed. + No metadata-file copy, new host mount, cron job or systemd archive timer is required. +3. Check web and archive-worker health, the public catalog and SMTP delivery. Monitor unhealthy + containers and host outages through normal PSI monitoring. Create local admin credentials + separately; accounts and private data are not in the public archive. + +## Automatic behavior + +- Web applies migrations and restores only an empty inventory catalog, from the latest + published record owned by `86122` in series `22850223`. It verifies checksums/schema and + imports atomically. A failure prevents web startup; existing catalogs are not overwritten. +- Current cutover record: https://zenodo.org/records/22850224, containing **3,377 datasets and + 3,916 versions**. SHA-256: + `169fe991599e138ddd15eae7aa56b2c58ab80bf865e8087dfd4c303f868386ea`. +- Both bootstrap pins are intentionally empty. After future nightly uploads, an empty database + must recover the latest trusted version, not the original cutover snapshot. +- Both export workers and `clic-inventory-archive` wait for web readiness before starting; + the web healthcheck allows ten minutes for first bootstrap. +- Approved collection metadata is bundled at `/app/config/inventory-archive-metadata.json`. + Restored per-version redistribution approvals require no separate authorization file. +- Only the archive application service receives the publisher token. Its first check must + match the trusted archive and report `unchanged`; a mismatch creates no draft and alerts. + After that handover, daily checks start at **03:00 UTC**, polling every five minutes with + catch-up/retry. Unchanged inventories produce no new Zenodo version. +- Its independent healthcheck emails `romain.sacchi@psi.ch` through the existing SMTP settings + on publication failure or checks older than 26 hours. A down host or broken SMTP needs + external monitoring; Docker does not automatically restart an unhealthy running process. + +The image is built from application revision `ae59c154309d4c4ccd44ae2b22f6e81a7e0dcffb`. +Registry digest: `sha256:a3cba10de3822b848cadbbedfd2ec63f70c94f580ae585ca8ff50d8250528374` +(`linux/amd64`). Full verification passed: 855 backend tests, 20 browser tests and 108 +PostgreSQL archive/worker tests. The exact image passed token-free production restoration, +restart idempotence, a real automatic sandbox upload (record `606103`), unchanged-content +skipping and recovery of that new snapshot into a second empty database. +Archive migrations `0041` and `0042` run automatically; this scheduler adds no migration. +Private users, uploaded originals, reviews and impact scores are not restored. Keep ordinary +private operational backups and provision licensed impact factors independently if required. + +From the existing hosting project, inspect: + +```sh +docker compose logs --tail=100 clic-web clic-inventory-archive +docker compose exec -T clic-inventory-archive python manage.py inventory_archive_status +``` + +Old restoration-only/combined env overlays and host-timer instructions are superseded by this +Compose configuration. Do not enable legacy archive timers alongside this container. Env changes +require recreation, not just restart. Before rollback, stop all PSI application services including +the archive publisher, remove its token, and coordinate with Romain before reopening Heroku. +Independent PSI edits must be reconciled before any rollback. diff --git a/docker-compose.yaml b/docker-compose.yaml index bb8309b..1d33573 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -147,7 +147,7 @@ services: # Romain Sacchi # CLIC Django platform clic-web: - image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:58e902a} + image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:ae59c15} container_name: clic-web restart: always env_file: @@ -159,6 +159,15 @@ services: - CLIC_CACHE_BACKEND=redis - CLIC_REDIS_URL=redis://clic-redis:6379/0 - CLIC_REQUIRE_SHARED_CACHE=1 + - INVENTORY_ARCHIVE_ENVIRONMENT=production + - INVENTORY_ARCHIVE_AUTO_RESTORE=1 + - INVENTORY_ARCHIVE_PUBLISH_MODE=off + - INVENTORY_ARCHIVE_TERMS_REVIEWED=0 + - INVENTORY_ARCHIVE_TOKEN= + - INVENTORY_ARCHIVE_OWNER_ID=86122 + - INVENTORY_ARCHIVE_SERIES_ID=22850223 + - INVENTORY_ARCHIVE_BOOTSTRAP_RECORD_ID= + - INVENTORY_ARCHIVE_BOOTSTRAP_SHA256= volumes: - /mount/webhosting_dmz_write/clic/media:/app/media - /mount/webhosting_dmz_write/clic/impact_factors:/app/data/impact_factors:ro @@ -173,17 +182,17 @@ services: - CMD - python - -c - - "import urllib.request; r = urllib.request.Request('http://127.0.0.1:8000/health/live/', headers={'Host': 'clic.psi.ch', 'X-Forwarded-Proto': 'https'}); urllib.request.urlopen(r, timeout=5)" + - "import urllib.request; r = urllib.request.Request('http://127.0.0.1:8000/health/ready/', headers={'Host': 'clic.psi.ch', 'X-Forwarded-Proto': 'https'}); urllib.request.urlopen(r, timeout=5)" interval: 30s timeout: 5s retries: 5 - start_period: 30s + start_period: 10m networks: - backend - clic_backend clic-package-exports: - image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:58e902a} + image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:ae59c15} container_name: clic-package-exports restart: always env_file: @@ -195,12 +204,17 @@ services: - CLIC_CACHE_BACKEND=redis - CLIC_REDIS_URL=redis://clic-redis:6379/0 - CLIC_REQUIRE_SHARED_CACHE=1 + - INVENTORY_ARCHIVE_AUTO_RESTORE=0 + - INVENTORY_ARCHIVE_PUBLISH_MODE=off + - INVENTORY_ARCHIVE_TOKEN= volumes: - /mount/webhosting_dmz_write/clic/media:/app/media - /mount/webhosting_dmz_write/clic/impact_factors:/app/data/impact_factors:ro entrypoint: ["python"] command: ["manage.py", "process_package_exports"] depends_on: + clic-web: + condition: service_healthy clic-db: condition: service_healthy clic-redis: @@ -209,7 +223,7 @@ services: - clic_backend clic-brightway-json-exports: - image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:58e902a} + image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:ae59c15} container_name: clic-brightway-json-exports restart: always env_file: @@ -221,6 +235,9 @@ services: - CLIC_CACHE_BACKEND=redis - CLIC_REDIS_URL=redis://clic-redis:6379/0 - CLIC_REQUIRE_SHARED_CACHE=1 + - INVENTORY_ARCHIVE_AUTO_RESTORE=0 + - INVENTORY_ARCHIVE_PUBLISH_MODE=off + - INVENTORY_ARCHIVE_TOKEN= volumes: - /mount/webhosting_dmz_write/clic/media:/app/media - /mount/webhosting_dmz_write/clic/impact_factors:/app/data/impact_factors:ro @@ -234,6 +251,8 @@ services: "${CLIC_API_EXPORT_POLL_SECONDS:-2}", ] depends_on: + clic-web: + condition: service_healthy clic-db: condition: service_healthy clic-redis: @@ -241,12 +260,54 @@ services: networks: - clic_backend + clic-inventory-archive: + image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:ae59c15} + container_name: clic-inventory-archive + restart: always + env_file: + - /mount/webhosting_dmz_write/clic/clic.env + environment: + - DJANGO_SETTINGS_MODULE=config.settings.production + - POSTGRES_HOST=clic-db + - POSTGRES_PORT=5432 + - CLIC_CACHE_BACKEND=redis + - CLIC_REDIS_URL=redis://clic-redis:6379/0 + - CLIC_REQUIRE_SHARED_CACHE=1 + - IMPACT_SCORES_ENABLED=0 + - INVENTORY_ARCHIVE_ENVIRONMENT=production + - INVENTORY_ARCHIVE_AUTO_RESTORE=0 + - INVENTORY_ARCHIVE_PUBLISH_MODE=scheduled + - INVENTORY_ARCHIVE_TERMS_REVIEWED=1 + - INVENTORY_ARCHIVE_OWNER_ID=86122 + - INVENTORY_ARCHIVE_SERIES_ID=22850223 + - INVENTORY_ARCHIVE_BOOTSTRAP_RECORD_ID= + - INVENTORY_ARCHIVE_BOOTSTRAP_SHA256= + - INVENTORY_ARCHIVE_METADATA_FILE=/app/config/inventory-archive-metadata.json + - INVENTORY_ARCHIVE_REDISTRIBUTION_FILE= + - INVENTORY_ARCHIVE_ALERT_EMAIL=romain.sacchi@psi.ch + entrypoint: ["python"] + command: ["manage.py", "run_inventory_archive_worker"] + depends_on: + clic-web: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "manage.py", "inventory_archive_status", "--alert"] + interval: 5m + timeout: 60s + retries: 1 + start_period: 10m + stop_grace_period: 3m + networks: + - clic_backend + clic-db: image: postgres:16 container_name: clic-db restart: always env_file: - /mount/webhosting_dmz_write/clic/clic.env + environment: + - INVENTORY_ARCHIVE_TOKEN= volumes: - /mount/webhosting_dmz_write/clic/postgres:/var/lib/postgresql/data healthcheck: From cedd87127c6e238d170afcf0dab9af1cd26643da Mon Sep 17 00:00:00 2001 From: romainsacchi Date: Mon, 21 Sep 2026 13:11:22 +0200 Subject: [PATCH 4/4] Remove CLIC archive handoff document from PR --- CLIC-ARCHIVE.md | 61 ------------------------------------------------- 1 file changed, 61 deletions(-) delete mode 100644 CLIC-ARCHIVE.md diff --git a/CLIC-ARCHIVE.md b/CLIC-ARCHIVE.md deleted file mode 100644 index 2d6464e..0000000 --- a/CLIC-ARCHIVE.md +++ /dev/null @@ -1,61 +0,0 @@ -# CLIC archive cutover: operator actions - -This deployment enables the approved inventory-only Zenodo recovery and nightly backup route. -Heroku is frozen; its database and media are retained for rollback. Do not reopen its writers -or enable another archive publisher while PSI is authoritative. - -1. Add `INVENTORY_ARCHIVE_TOKEN` to the existing protected - `/mount/webhosting_dmz_write/clic/clic.env`, using the production Zenodo token supplied - securely by Romain. Keep all existing database, Django and SMTP settings. Do not attach the - filled env file or token to this repository, pull request or issue. -2. Merge/deploy the Compose change and recreate all four CLIC application services. Ensure - any host `CLIC_IMAGE` override selects `gitea.psi.ch/images/clic:ae59c15` or is removed. - No metadata-file copy, new host mount, cron job or systemd archive timer is required. -3. Check web and archive-worker health, the public catalog and SMTP delivery. Monitor unhealthy - containers and host outages through normal PSI monitoring. Create local admin credentials - separately; accounts and private data are not in the public archive. - -## Automatic behavior - -- Web applies migrations and restores only an empty inventory catalog, from the latest - published record owned by `86122` in series `22850223`. It verifies checksums/schema and - imports atomically. A failure prevents web startup; existing catalogs are not overwritten. -- Current cutover record: https://zenodo.org/records/22850224, containing **3,377 datasets and - 3,916 versions**. SHA-256: - `169fe991599e138ddd15eae7aa56b2c58ab80bf865e8087dfd4c303f868386ea`. -- Both bootstrap pins are intentionally empty. After future nightly uploads, an empty database - must recover the latest trusted version, not the original cutover snapshot. -- Both export workers and `clic-inventory-archive` wait for web readiness before starting; - the web healthcheck allows ten minutes for first bootstrap. -- Approved collection metadata is bundled at `/app/config/inventory-archive-metadata.json`. - Restored per-version redistribution approvals require no separate authorization file. -- Only the archive application service receives the publisher token. Its first check must - match the trusted archive and report `unchanged`; a mismatch creates no draft and alerts. - After that handover, daily checks start at **03:00 UTC**, polling every five minutes with - catch-up/retry. Unchanged inventories produce no new Zenodo version. -- Its independent healthcheck emails `romain.sacchi@psi.ch` through the existing SMTP settings - on publication failure or checks older than 26 hours. A down host or broken SMTP needs - external monitoring; Docker does not automatically restart an unhealthy running process. - -The image is built from application revision `ae59c154309d4c4ccd44ae2b22f6e81a7e0dcffb`. -Registry digest: `sha256:a3cba10de3822b848cadbbedfd2ec63f70c94f580ae585ca8ff50d8250528374` -(`linux/amd64`). Full verification passed: 855 backend tests, 20 browser tests and 108 -PostgreSQL archive/worker tests. The exact image passed token-free production restoration, -restart idempotence, a real automatic sandbox upload (record `606103`), unchanged-content -skipping and recovery of that new snapshot into a second empty database. -Archive migrations `0041` and `0042` run automatically; this scheduler adds no migration. -Private users, uploaded originals, reviews and impact scores are not restored. Keep ordinary -private operational backups and provision licensed impact factors independently if required. - -From the existing hosting project, inspect: - -```sh -docker compose logs --tail=100 clic-web clic-inventory-archive -docker compose exec -T clic-inventory-archive python manage.py inventory_archive_status -``` - -Old restoration-only/combined env overlays and host-timer instructions are superseded by this -Compose configuration. Do not enable legacy archive timers alongside this container. Env changes -require recreation, not just restart. Before rollback, stop all PSI application services including -the archive publisher, remove its token, and coordinate with Romain before reopening Heroku. -Independent PSI edits must be reconciled before any rollback.