Merge branch 'ci/init_setup' into 'main'

refactor: cleanup; added more ci jobs

See merge request bec/bec_atlas!2

.gitlab-ci.yml

@@ -1,24 +1,89 @@
 image: $CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX/python:3.11
 
-stages:
-  - test
-
 variables:
   SCYLLA_HOST: scylla
   SCYLLA_PORT: 9042
   SCYLLA_KEYSPACE: bec_atlas
 
+workflow:
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+    - if: '$CI_PIPELINE_SOURCE == "web"'
+    - if: '$CI_PIPELINE_SOURCE == "pipeline"'
+    - if: '$CI_PIPELINE_SOURCE == "parent_pipeline"'
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
+      when: never
+    - if: $CI_COMMIT_BRANCH
+  auto_cancel:
+    on_new_commit: interruptible
+
+stages:
+  - Formatter
+  - test 
+
+include:
+  - template: Security/Secret-Detection.gitlab-ci.yml
+
 services:
   - name: $CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX/scylladb/scylla:latest
     alias: scylla
 
-# before_script:
-#   - pip install ./backend
-#   - python -u ./backend/bec_atlas/utils/setup_database.py
+formatter:
+  stage: Formatter
+  needs: []
+  script:
+    - pip install black isort
+    - isort --check --diff --line-length=100 --profile=black --multi-line=3 --trailing-comma ./backend/bec_atlas
+    - black --check --diff --color --line-length=100 --skip-magic-trailing-comma ./backend/bec_atlas
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+    - if: '$CI_PIPELINE_SOURCE == "web"'
+    - if: '$CI_PIPELINE_SOURCE == "pipeline"'
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'  
+    - if: '$CI_PIPELINE_SOURCE == "parent_pipeline"'
+      when: never
+    - if: $CI_COMMIT_BRANCH
+  interruptible: true
 
+pylint:
+  stage: Formatter
+  needs: []
+  script:
+    - pip install pylint pylint-exit anybadge
+    - mkdir ./pylint
+    - pylint ./backend/bec_atlas --output-format=text | tee ./pylint/pylint.log || pylint-exit $?
+    - PYLINT_SCORE=$(sed -n 's/^Your code has been rated at \([-0-9.]*\)\/.*/\1/p' ./pylint/pylint.log)
+    - anybadge --label=Pylint --file=pylint/pylint.svg --value=$PYLINT_SCORE 2=red 4=orange 8=yellow 10=green
+    - echo "Pylint score is $PYLINT_SCORE"
+  artifacts:
+    paths:
+      - ./pylint/
+    expire_in: 1 week
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+    - if: '$CI_PIPELINE_SOURCE == "web"'
+    - if: '$CI_PIPELINE_SOURCE == "pipeline"'
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'  
+    - if: '$CI_PIPELINE_SOURCE == "parent_pipeline"'
+      when: never
+    - if: $CI_COMMIT_BRANCH
+  interruptible: true
 
-test:
+backend_pytest:
   stage: test
+  needs: []
   script:
     - pip install ./backend[dev]
-    - pytest --skip-docker --random-order ./backend/tests
+    - pip install coverage pytest-asyncio
+    - coverage run --concurrency=thread --source=./backend --omit=*/backend/tests/* -m pytest -v --junitxml=report.xml --skip-docker --random-order --full-trace ./backend/tests
+    - coverage report
+    - coverage xml
+  coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
+  artifacts:
+    reports:
+      junit: report.xml
+      coverage_report:
+        coverage_format: cobertura
+        path: coverage.xml
+  interruptible: true

backend/bec_atlas/authentication.py

@@ -3,12 +3,13 @@ from datetime import datetime, timedelta
 from typing import Annotated
 
 import jwt
-from bec_atlas.datasources.scylladb import scylladb_schema as schema
 from fastapi import Depends, HTTPException, status
 from fastapi.security import OAuth2PasswordBearer
 from jwt.exceptions import InvalidTokenError
 from pwdlib import PasswordHash
 
+from bec_atlas.datasources.scylladb import scylladb_schema as schema
+
 ALGORITHM = "HS256"
 ACCESS_TOKEN_EXPIRE_MINUTES = 30
 

backend/bec_atlas/datasources/scylladb/scylladb.py

@@ -1,13 +1,14 @@
 import json
 import os
 
-from bec_atlas.authentication import get_password_hash
-from bec_atlas.datasources.scylladb import scylladb_schema as schema
 from cassandra.cluster import Cluster
 from cassandra.cqlengine import connection
 from cassandra.cqlengine.management import create_keyspace_simple, sync_table
 from pydantic import BaseModel
 
+from bec_atlas.authentication import get_password_hash
+from bec_atlas.datasources.scylladb import scylladb_schema as schema
+
 
 class ScylladbDatasource:
     KEYSPACE = "bec_atlas"

backend/bec_atlas/main.py

@@ -1,10 +1,11 @@
 import socketio
 import uvicorn
+from fastapi import FastAPI
+
 from bec_atlas.datasources.datasource_manager import DatasourceManager
 from bec_atlas.router.redis_router import RedisRouter, RedisWebsocket
 from bec_atlas.router.scan_router import ScanRouter
 from bec_atlas.router.user import UserRouter
-from fastapi import FastAPI
 
 CONFIG = {"redis": {"host": "localhost", "port": 6380}, "scylla": {"hosts": ["localhost"]}}
 

backend/bec_atlas/router/redis_router.py

@@ -4,10 +4,11 @@ import json
 from typing import TYPE_CHECKING
 
 import socketio
-from bec_atlas.router.base_router import BaseRouter
 from bec_lib.endpoints import MessageEndpoints
 from fastapi import APIRouter, WebSocket, WebSocketDisconnect
 
+from bec_atlas.router.base_router import BaseRouter
+
 if TYPE_CHECKING:
     from bec_lib.redis_connector import RedisConnector
 

backend/bec_atlas/router/scan_router.py

@@ -1,7 +1,8 @@
+from fastapi import APIRouter, Depends
+
 from bec_atlas.authentication import get_current_user
 from bec_atlas.datasources.scylladb import scylladb_schema as schema
 from bec_atlas.router.base_router import BaseRouter
-from fastapi import APIRouter, Depends
 
 
 class ScanRouter(BaseRouter):

backend/bec_atlas/router/user.py

@@ -1,13 +1,14 @@
 from typing import Annotated
 
-from bec_atlas.authentication import create_access_token, get_current_user, verify_password
-from bec_atlas.datasources.scylladb import scylladb_schema as schema
-from bec_atlas.router.base_router import BaseRouter
 from fastapi import APIRouter, Depends
 from fastapi.exceptions import HTTPException
 from fastapi.security import OAuth2PasswordRequestForm
 from pydantic import BaseModel
 
+from bec_atlas.authentication import create_access_token, get_current_user, verify_password
+from bec_atlas.datasources.scylladb import scylladb_schema as schema
+from bec_atlas.router.base_router import BaseRouter
+
 
 class UserLoginRequest(BaseModel):
     username: str

backend/bec_atlas/utils/launch.py

@@ -2,6 +2,7 @@ import argparse
 import os
 
 import libtmux
+
 from bec_atlas.utils.service_handler import ServiceHandler
 
 

backend/tests/test_login.py

@@ -1,5 +1,4 @@
 import os
-import socket
 
 import pytest
 from bec_atlas.main import AtlasApp