diff --git a/CLIC-ARCHIVE.md b/CLIC-ARCHIVE.md new file mode 100644 index 0000000..2d6464e --- /dev/null +++ b/CLIC-ARCHIVE.md @@ -0,0 +1,61 @@ +# CLIC archive cutover: operator actions + +This deployment enables the approved inventory-only Zenodo recovery and nightly backup route. +Heroku is frozen; its database and media are retained for rollback. Do not reopen its writers +or enable another archive publisher while PSI is authoritative. + +1. Add `INVENTORY_ARCHIVE_TOKEN` to the existing protected + `/mount/webhosting_dmz_write/clic/clic.env`, using the production Zenodo token supplied + securely by Romain. Keep all existing database, Django and SMTP settings. Do not attach the + filled env file or token to this repository, pull request or issue. +2. Merge/deploy the Compose change and recreate all four CLIC application services. Ensure + any host `CLIC_IMAGE` override selects `gitea.psi.ch/images/clic:ae59c15` or is removed. + No metadata-file copy, new host mount, cron job or systemd archive timer is required. +3. Check web and archive-worker health, the public catalog and SMTP delivery. Monitor unhealthy + containers and host outages through normal PSI monitoring. Create local admin credentials + separately; accounts and private data are not in the public archive. + +## Automatic behavior + +- Web applies migrations and restores only an empty inventory catalog, from the latest + published record owned by `86122` in series `22850223`. It verifies checksums/schema and + imports atomically. A failure prevents web startup; existing catalogs are not overwritten. +- Current cutover record: https://zenodo.org/records/22850224, containing **3,377 datasets and + 3,916 versions**. SHA-256: + `169fe991599e138ddd15eae7aa56b2c58ab80bf865e8087dfd4c303f868386ea`. +- Both bootstrap pins are intentionally empty. After future nightly uploads, an empty database + must recover the latest trusted version, not the original cutover snapshot. +- Both export workers and `clic-inventory-archive` wait for web readiness before starting; + the web healthcheck allows ten minutes for first bootstrap. +- Approved collection metadata is bundled at `/app/config/inventory-archive-metadata.json`. + Restored per-version redistribution approvals require no separate authorization file. +- Only the archive application service receives the publisher token. Its first check must + match the trusted archive and report `unchanged`; a mismatch creates no draft and alerts. + After that handover, daily checks start at **03:00 UTC**, polling every five minutes with + catch-up/retry. Unchanged inventories produce no new Zenodo version. +- Its independent healthcheck emails `romain.sacchi@psi.ch` through the existing SMTP settings + on publication failure or checks older than 26 hours. A down host or broken SMTP needs + external monitoring; Docker does not automatically restart an unhealthy running process. + +The image is built from application revision `ae59c154309d4c4ccd44ae2b22f6e81a7e0dcffb`. +Registry digest: `sha256:a3cba10de3822b848cadbbedfd2ec63f70c94f580ae585ca8ff50d8250528374` +(`linux/amd64`). Full verification passed: 855 backend tests, 20 browser tests and 108 +PostgreSQL archive/worker tests. The exact image passed token-free production restoration, +restart idempotence, a real automatic sandbox upload (record `606103`), unchanged-content +skipping and recovery of that new snapshot into a second empty database. +Archive migrations `0041` and `0042` run automatically; this scheduler adds no migration. +Private users, uploaded originals, reviews and impact scores are not restored. Keep ordinary +private operational backups and provision licensed impact factors independently if required. + +From the existing hosting project, inspect: + +```sh +docker compose logs --tail=100 clic-web clic-inventory-archive +docker compose exec -T clic-inventory-archive python manage.py inventory_archive_status +``` + +Old restoration-only/combined env overlays and host-timer instructions are superseded by this +Compose configuration. Do not enable legacy archive timers alongside this container. Env changes +require recreation, not just restart. Before rollback, stop all PSI application services including +the archive publisher, remove its token, and coordinate with Romain before reopening Heroku. +Independent PSI edits must be reconciled before any rollback. diff --git a/docker-compose.yaml b/docker-compose.yaml index bb8309b..1d33573 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -147,7 +147,7 @@ services: # Romain Sacchi # CLIC Django platform clic-web: - image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:58e902a} + image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:ae59c15} container_name: clic-web restart: always env_file: @@ -159,6 +159,15 @@ services: - CLIC_CACHE_BACKEND=redis - CLIC_REDIS_URL=redis://clic-redis:6379/0 - CLIC_REQUIRE_SHARED_CACHE=1 + - INVENTORY_ARCHIVE_ENVIRONMENT=production + - INVENTORY_ARCHIVE_AUTO_RESTORE=1 + - INVENTORY_ARCHIVE_PUBLISH_MODE=off + - INVENTORY_ARCHIVE_TERMS_REVIEWED=0 + - INVENTORY_ARCHIVE_TOKEN= + - INVENTORY_ARCHIVE_OWNER_ID=86122 + - INVENTORY_ARCHIVE_SERIES_ID=22850223 + - INVENTORY_ARCHIVE_BOOTSTRAP_RECORD_ID= + - INVENTORY_ARCHIVE_BOOTSTRAP_SHA256= volumes: - /mount/webhosting_dmz_write/clic/media:/app/media - /mount/webhosting_dmz_write/clic/impact_factors:/app/data/impact_factors:ro @@ -173,17 +182,17 @@ services: - CMD - python - -c - - "import urllib.request; r = urllib.request.Request('http://127.0.0.1:8000/health/live/', headers={'Host': 'clic.psi.ch', 'X-Forwarded-Proto': 'https'}); urllib.request.urlopen(r, timeout=5)" + - "import urllib.request; r = urllib.request.Request('http://127.0.0.1:8000/health/ready/', headers={'Host': 'clic.psi.ch', 'X-Forwarded-Proto': 'https'}); urllib.request.urlopen(r, timeout=5)" interval: 30s timeout: 5s retries: 5 - start_period: 30s + start_period: 10m networks: - backend - clic_backend clic-package-exports: - image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:58e902a} + image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:ae59c15} container_name: clic-package-exports restart: always env_file: @@ -195,12 +204,17 @@ services: - CLIC_CACHE_BACKEND=redis - CLIC_REDIS_URL=redis://clic-redis:6379/0 - CLIC_REQUIRE_SHARED_CACHE=1 + - INVENTORY_ARCHIVE_AUTO_RESTORE=0 + - INVENTORY_ARCHIVE_PUBLISH_MODE=off + - INVENTORY_ARCHIVE_TOKEN= volumes: - /mount/webhosting_dmz_write/clic/media:/app/media - /mount/webhosting_dmz_write/clic/impact_factors:/app/data/impact_factors:ro entrypoint: ["python"] command: ["manage.py", "process_package_exports"] depends_on: + clic-web: + condition: service_healthy clic-db: condition: service_healthy clic-redis: @@ -209,7 +223,7 @@ services: - clic_backend clic-brightway-json-exports: - image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:58e902a} + image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:ae59c15} container_name: clic-brightway-json-exports restart: always env_file: @@ -221,6 +235,9 @@ services: - CLIC_CACHE_BACKEND=redis - CLIC_REDIS_URL=redis://clic-redis:6379/0 - CLIC_REQUIRE_SHARED_CACHE=1 + - INVENTORY_ARCHIVE_AUTO_RESTORE=0 + - INVENTORY_ARCHIVE_PUBLISH_MODE=off + - INVENTORY_ARCHIVE_TOKEN= volumes: - /mount/webhosting_dmz_write/clic/media:/app/media - /mount/webhosting_dmz_write/clic/impact_factors:/app/data/impact_factors:ro @@ -234,6 +251,8 @@ services: "${CLIC_API_EXPORT_POLL_SECONDS:-2}", ] depends_on: + clic-web: + condition: service_healthy clic-db: condition: service_healthy clic-redis: @@ -241,12 +260,54 @@ services: networks: - clic_backend + clic-inventory-archive: + image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:ae59c15} + container_name: clic-inventory-archive + restart: always + env_file: + - /mount/webhosting_dmz_write/clic/clic.env + environment: + - DJANGO_SETTINGS_MODULE=config.settings.production + - POSTGRES_HOST=clic-db + - POSTGRES_PORT=5432 + - CLIC_CACHE_BACKEND=redis + - CLIC_REDIS_URL=redis://clic-redis:6379/0 + - CLIC_REQUIRE_SHARED_CACHE=1 + - IMPACT_SCORES_ENABLED=0 + - INVENTORY_ARCHIVE_ENVIRONMENT=production + - INVENTORY_ARCHIVE_AUTO_RESTORE=0 + - INVENTORY_ARCHIVE_PUBLISH_MODE=scheduled + - INVENTORY_ARCHIVE_TERMS_REVIEWED=1 + - INVENTORY_ARCHIVE_OWNER_ID=86122 + - INVENTORY_ARCHIVE_SERIES_ID=22850223 + - INVENTORY_ARCHIVE_BOOTSTRAP_RECORD_ID= + - INVENTORY_ARCHIVE_BOOTSTRAP_SHA256= + - INVENTORY_ARCHIVE_METADATA_FILE=/app/config/inventory-archive-metadata.json + - INVENTORY_ARCHIVE_REDISTRIBUTION_FILE= + - INVENTORY_ARCHIVE_ALERT_EMAIL=romain.sacchi@psi.ch + entrypoint: ["python"] + command: ["manage.py", "run_inventory_archive_worker"] + depends_on: + clic-web: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "manage.py", "inventory_archive_status", "--alert"] + interval: 5m + timeout: 60s + retries: 1 + start_period: 10m + stop_grace_period: 3m + networks: + - clic_backend + clic-db: image: postgres:16 container_name: clic-db restart: always env_file: - /mount/webhosting_dmz_write/clic/clic.env + environment: + - INVENTORY_ARCHIVE_TOKEN= volumes: - /mount/webhosting_dmz_write/clic/postgres:/var/lib/postgresql/data healthcheck: