499 lines
16 KiB
YAML
499 lines
16 KiB
YAML
name: Pytest CI Workflow
|
|
description: Run pytest with coverage and report upload to Gitea
|
|
inputs:
|
|
ci-branch:
|
|
description: 'Target branch where test reports will be committed'
|
|
required: true
|
|
|
|
module-dir:
|
|
description: 'Directory (folder) containing the Python module to test'
|
|
required: true
|
|
|
|
gitea-domain:
|
|
description: 'Domain name of your Gitea instance without protocoL'
|
|
required: true
|
|
|
|
repo-path:
|
|
description: 'Relative path to the root of your repository'
|
|
required: true
|
|
|
|
ci-token:
|
|
description: ' CI authentication token (used for pushing to Gitea)'
|
|
required: true
|
|
|
|
apt_packages:
|
|
description: "List of system packages to install via apt-get"
|
|
required: false
|
|
default: ""
|
|
|
|
env:
|
|
description: "Environment type: pixi (default), pip or conda"
|
|
required: false
|
|
default: "pixi"
|
|
|
|
enable_pixi_post_link_scripts:
|
|
description: 'Set to true to enable post-link scripts for Pixi'
|
|
required: false
|
|
default: 'false'
|
|
|
|
custom_shell_commands_before:
|
|
description: "Additional shell commands to run"
|
|
required: false
|
|
default: ""
|
|
|
|
test-run-cmd:
|
|
description: "Wrapper command to run pytest"
|
|
required: false
|
|
default: "$CMD"
|
|
|
|
custom_shell_commands_after:
|
|
description: "Additional shell commands to run"
|
|
required: false
|
|
default: ""
|
|
|
|
test-files:
|
|
description: "Comma-separated list of test files or directories to run"
|
|
required: false
|
|
default: ""
|
|
|
|
runs:
|
|
using: "composite"
|
|
steps:
|
|
- name: Checkout
|
|
run: |
|
|
OUTPUT_DIR="outputs"
|
|
LOG_FILE="${OUTPUT_DIR}/checkout.log"
|
|
mkdir -p "$OUTPUT_DIR"
|
|
|
|
git clone -q https://gitea.psi.ch/tligui_y/test-ci.git ./temp-ci
|
|
|
|
echo "1 - Code checkout..."
|
|
|
|
# Init repo
|
|
git init . >> LOG_FILE 2>&1
|
|
git remote add origin ${{ github.server_url }}/${{ github.repository }} >> $LOG_FILE 2>&1
|
|
|
|
# Fetch uniquement le commit courant
|
|
git fetch --depth=1 origin ${{ github.sha }} >> $LOG_FILE 2>&1
|
|
|
|
# Checkout le commit
|
|
git checkout FETCH_HEAD >> $LOG_FILE 2>&1
|
|
|
|
echo "Exit Code: 0" >> $LOG_FILE
|
|
|
|
|
|
- name: Install optional apt packages
|
|
if: inputs.apt_packages != ''
|
|
run: |
|
|
OUTPUT_DIR="outputs"
|
|
LOG_FILE="${OUTPUT_DIR}/sudo-install.log"
|
|
VERSIONS_FILE="${OUTPUT_DIR}/sudo-versions.log"
|
|
|
|
echo "2 - Starting APT package installation..."
|
|
|
|
# Run install, capture everything into the log
|
|
sudo DEBIAN_FRONTEND=noninteractive apt-get update &>> "$LOG_FILE"
|
|
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y ${{ inputs.apt_packages }} &>> "$LOG_FILE"
|
|
EXIT_CODE=$?
|
|
|
|
if [ $EXIT_CODE -eq 0 ]; then
|
|
|
|
echo "📋 Installed APT packages (full list):" >> "$VERSIONS_FILE"
|
|
dpkg -l >> "$VERSIONS_FILE"
|
|
|
|
echo "Exit Code: 0" >> "$LOG_FILE"
|
|
echo "Exit Code: 0" >> "$VERSIONS_FILE"
|
|
fi
|
|
shell: bash
|
|
|
|
- name: Setup environment
|
|
shell: bash
|
|
run: |
|
|
OUTPUT_DIR="outputs"
|
|
LOG_FILE="${OUTPUT_DIR}/env-setup.log"
|
|
VERSIONS_FILE="${OUTPUT_DIR}/env-versions.log"
|
|
|
|
case "${{ inputs.env }}" in
|
|
pixi)
|
|
echo "3 - Installing Pixi environment..."
|
|
if [ "${{ inputs.enable_pixi_post_link_scripts }}" == "true" ]; then
|
|
pixi config set --local run-post-link-scripts insecure
|
|
fi
|
|
|
|
curl -fsSL https://pixi.sh/install.sh | bash >> $LOG_FILE 2>&1 || {
|
|
exit 1
|
|
}
|
|
|
|
echo "$HOME/.pixi/bin" >> $GITHUB_PATH
|
|
export PATH="$HOME/.pixi/bin:$PATH"
|
|
|
|
if [ ! -f "pixi.toml" ]; then
|
|
echo "⚠️ No pixi.toml found, skipping" >> "$LOG_FILE"
|
|
echo "Exit Code: 0" >> "$LOG_FILE"
|
|
else
|
|
echo -ne "Installing dependencies from pixi.toml... \r"
|
|
pixi install >> $LOG_FILE 2>&1 || {
|
|
exit 1
|
|
}
|
|
|
|
echo "📋 Pixi environment details:" >> "$VERSIONS_FILE"
|
|
pixi list >> $VERSIONS_FILE 2>&1
|
|
|
|
echo "Exit Code: 0" >> $LOG_FILE
|
|
echo "Exit Code: 0" >> "$VERSIONS_FILE"
|
|
fi
|
|
;;
|
|
|
|
pip)
|
|
echo "3 - Installing pip dependencies..."
|
|
python -m pip install --upgrade pip >> $LOG_FILE 2>&1
|
|
if [ -f "requirements.txt" ]; then
|
|
pip install -r requirements.txt >> $LOG_FILE 2>&1 || {
|
|
exit 1
|
|
}
|
|
echo "📋 Pip environment details:" >> "$VERSIONS_FILE"
|
|
pip freeze >> $VERSIONS_FILE 2>&1
|
|
|
|
echo "Exit Code: 0" >> $LOG_FILE
|
|
echo "Exit Code: 0" >> "$VERSIONS_FILE"
|
|
else
|
|
echo "⚠️ No requirements.txt found, skipping" >> "$LOG_FILE"
|
|
echo "Exit Code: 0" >> $LOG_FILE
|
|
fi
|
|
;;
|
|
|
|
conda)
|
|
echo "3 - Setting up conda environment..."
|
|
if [ -f "environment.yml" ]; then
|
|
conda env create -f environment.yml >> $LOG_FILE 2>&1 || {
|
|
exit 1
|
|
}
|
|
echo "📋 Conda environment details:" >> "$VERSIONS_FILE"
|
|
conda list >> $VERSIONS_FILE 2>&1
|
|
|
|
echo "Exit Code: 0" >> $LOG_FILE
|
|
echo "Exit Code: 0" >> "$VERSIONS_FILE"
|
|
else
|
|
echo "⚠️ No environment.yml found, skipping" >> "$LOG_FILE"
|
|
echo "Exit Code: 0" >> $LOG_FILE
|
|
fi
|
|
;;
|
|
esac
|
|
|
|
- name: Install CI test tooling
|
|
shell: bash
|
|
run: |
|
|
OUTPUT_DIR="outputs"
|
|
LOG_FILE="${OUTPUT_DIR}/ci-tooling.log"
|
|
|
|
mkdir -p "$OUTPUT_DIR"
|
|
|
|
echo "4 - Installing pytest + coverage tooling..."
|
|
|
|
case "${{ inputs.env }}" in
|
|
pixi)
|
|
pixi add pytest pytest-cov pytest-json-report coverage >> "$LOG_FILE" 2>&1 || {
|
|
exit 1
|
|
}
|
|
;;
|
|
conda)
|
|
conda install -y pytest pytest-cov pytest-json-report coverage >> "$LOG_FILE" 2>&1 || {
|
|
exit 1
|
|
}
|
|
;;
|
|
pip)
|
|
pip install pytest pytest-cov pytest-json-report coverage >> "$LOG_FILE" 2>&1 || {
|
|
exit 1
|
|
}
|
|
;;
|
|
esac
|
|
echo "Exit Code: 0" >> "$LOG_FILE"
|
|
|
|
|
|
- name: Run custom shell commands before
|
|
shell: bash
|
|
run: |
|
|
OUTPUT_DIR="outputs"
|
|
LOG_FILE="${OUTPUT_DIR}/custom-shell-before.log"
|
|
mkdir -p "$OUTPUT_DIR"
|
|
|
|
if [ -z "$(echo "${{ inputs.custom_shell_commands_before }}" | xargs)" ]; then
|
|
echo "💡 No custom shell commands found" >> "$LOG_FILE"
|
|
echo "Exit Code: 0" >> "$LOG_FILE"
|
|
exit 0
|
|
fi
|
|
|
|
echo "5 - Executing pre-custom shell commands..."
|
|
|
|
export LOG_FILE="$LOG_FILE"
|
|
{
|
|
echo "set -e"
|
|
echo "set -x"
|
|
echo "${{ inputs.custom_shell_commands_before }}"
|
|
} > custom_commands.sh
|
|
|
|
chmod +x custom_commands.sh
|
|
|
|
./custom_commands.sh >> "$LOG_FILE" 2>&1
|
|
EXIT_CODE=$?
|
|
|
|
if [ $EXIT_CODE -eq 0 ]; then
|
|
echo "Exit Code: 0" >> "$LOG_FILE"
|
|
else
|
|
exit 1
|
|
fi
|
|
|
|
- name: Run tests and generate reports
|
|
shell: bash
|
|
run: |
|
|
OUTPUT_DIR="outputs"
|
|
LOG_FILE="${OUTPUT_DIR}/test-ci.log"
|
|
PRINTS="${OUTPUT_DIR}/test-prints.log"
|
|
|
|
export LOG_FILE
|
|
|
|
echo "6 - Running tests and generating reports..."
|
|
cp temp-ci/conftest.py ./
|
|
|
|
if [ "${{ inputs.test-files }}" = "none" ]; then
|
|
TEST_PATH="--ignore=tests/"
|
|
elif [ -n "${{ inputs.test-files }}" ]; then
|
|
TEST_PATH="${{ inputs.test-files }}"
|
|
else
|
|
TEST_PATH="."
|
|
fi
|
|
|
|
PYTEST_ARGS="--continue-on-collection-errors \
|
|
--cov=${{ inputs.module-dir }} \
|
|
--json-report \
|
|
--json-report-file=markdown/pytest-report.json \
|
|
--capture=no"
|
|
|
|
case "${{ inputs.env }}" in
|
|
pixi) CMD="pixi run pytest -s $TEST_PATH $PYTEST_ARGS > outputs/pytest-output.log 2>&1 || true" ;;
|
|
conda) CMD="conda run -n myenv pytest -s $TEST_PATH $PYTEST_ARGS > outputs/pytest-output.log 2>&1 || true" ;;
|
|
pip) CMD="pytest -s $TEST_PATH $PYTEST_ARGS > outputs/pytest-output.log 2>&1 || true" ;;
|
|
esac
|
|
|
|
exec 1>>"$PRINTS"
|
|
exec 2>>"$LOG_FILE"
|
|
|
|
cat > custom_cmd.sh <<'EOS'
|
|
${{ inputs.test-run-cmd }}
|
|
EOS
|
|
|
|
chmod +x custom_cmd.sh
|
|
export CMD
|
|
./custom_cmd.sh
|
|
|
|
echo "Exit Code: 0" >> $PRINTS
|
|
|
|
awk '/=+ test session starts =+/{flag=1} /-+ JSON report -+/{flag=0} flag' outputs/pytest-output.log > outputs/raw-test-output.log
|
|
awk '/=+ short test summary info =+/,/=+.* in .*s =+/' outputs/pytest-output.log > outputs/short-test-output.log
|
|
|
|
set -e
|
|
|
|
echo "" >> $LOG_FILE
|
|
echo "📋 Short test summary:" >> "$LOG_FILE"
|
|
if [ -s outputs/short-test-output.log ]; then
|
|
cat outputs/short-test-output.log >> "$LOG_FILE"
|
|
else
|
|
echo "✅ All tests passed, nothing to report" >> "$LOG_FILE"
|
|
fi
|
|
|
|
mkdir -p markdown
|
|
|
|
echo "" >> $LOG_FILE
|
|
echo "📈 Generating coverage summary..." >> $LOG_FILE
|
|
pixi run coverage combine > /dev/null 2>> outputs/coverage-error.log || true
|
|
pixi run coverage report --format=markdown > markdown/coverage-summary.md 2>> outputs/coverage-error.log && {
|
|
echo "✅ Coverage summary generated" >> $LOG_FILE
|
|
} || {
|
|
echo "❌ Coverage summary generation failed" >> $LOG_FILE
|
|
echo "🪵 Extracting coverage error:" >> $LOG_FILE
|
|
cat outputs/coverage-error.log >> $LOG_FILE
|
|
exit 1
|
|
}
|
|
|
|
echo "" >> $LOG_FILE
|
|
echo "📊 Generating tests markdown summary..." >> $LOG_FILE
|
|
|
|
pixi run python temp-ci/json_to_md.py \
|
|
--input "markdown/pytest-report.json" \
|
|
--output "markdown/TEST-REPORT.md" \
|
|
--log_long "outputs/raw-test-output.log" \
|
|
--log_short "outputs/short-test-output.log" \
|
|
--params "markdown/runtime_params.json" \
|
|
>> "outputs/md-report.log" 2>&1 && {
|
|
echo "✅ Tests markdown generated" >> $LOG_FILE
|
|
} || {
|
|
echo "❌ Tests markdown generation failed" >> $LOG_FILE
|
|
echo "🪵 Extracting markdown generation error:" >> $LOG_FILE
|
|
cat outputs/md-report.log >> $LOG_FILE
|
|
exit 1
|
|
}
|
|
|
|
echo "" >> $LOG_FILE
|
|
echo "🌳 Generating JSON tree view..." >> $LOG_FILE
|
|
pixi run python temp-ci/json_to_tree.py \
|
|
"markdown/pytest-report.json" \
|
|
>> "outputs/json-tree-gen.log" 2>&1 && {
|
|
echo "✅ JSON tree view generated" >> $LOG_FILE
|
|
} || {
|
|
echo "❌ JSON tree generation failed" >> $LOG_FILE
|
|
echo "🪵 Extracting json file error:" >> $LOG_FILE
|
|
cat outputs/json-tree-gen.log >> $LOG_FILE
|
|
exit 1
|
|
}
|
|
|
|
rm -rf ./temp-test-ci
|
|
echo "Exit Code: 0" >> $LOG_FILE
|
|
|
|
- name: Run custom shell commands after
|
|
shell: bash
|
|
run: |
|
|
OUTPUT_DIR="outputs"
|
|
LOG_FILE="${OUTPUT_DIR}/custom-shell-after.log"
|
|
mkdir -p "$OUTPUT_DIR"
|
|
|
|
if [ -z "$(echo "${{ inputs.custom_shell_commands_after }}" | xargs)" ]; then
|
|
echo "💡 No custom shell commands found" >> "$LOG_FILE"
|
|
echo "Exit Code: 0" >> "$LOG_FILE"
|
|
exit 0
|
|
fi
|
|
|
|
echo "7 - Executing post-custom shell commands..."
|
|
|
|
export LOG_FILE="$LOG_FILE"
|
|
{
|
|
echo "set -e"
|
|
echo "set -x"
|
|
echo "${{ inputs.custom_shell_commands }}"
|
|
} > custom_commands.sh
|
|
|
|
chmod +x custom_commands.sh
|
|
|
|
./custom_commands.sh >> "$LOG_FILE" 2>&1
|
|
EXIT_CODE=$?
|
|
|
|
if [ $EXIT_CODE -eq 0 ]; then
|
|
echo "Exit Code: 0" >> "$LOG_FILE"
|
|
else
|
|
exit 1
|
|
fi
|
|
|
|
|
|
- name: Clone Gitea Wiki
|
|
shell: bash
|
|
env:
|
|
CI_TOKEN: ${{ inputs.ci-token }}
|
|
run: |
|
|
REPO_NAME="${{ github.repository }}"
|
|
DOMAIN="${{ inputs.gitea-domain }}"
|
|
WIKI_REPO="https://ci-token:${CI_TOKEN}@${DOMAIN}/${REPO_NAME}.wiki.git"
|
|
|
|
OUTPUT_DIR="outputs"
|
|
LOG_FILE="${OUTPUT_DIR}/wiki-clone.log"
|
|
|
|
mkdir -p "$OUTPUT_DIR"
|
|
|
|
echo "8 - Cloning wiki repo: $WIKI_REPO"
|
|
|
|
echo "Cloning wiki repo: $WIKI_REPO" >> $LOG_FILE
|
|
echo "" >> $LOG_FILE
|
|
if ! git clone "$WIKI_REPO" wiki >> wiki_clone.log 2>&1; then
|
|
cat wiki_clone.log >> $LOG_FILE
|
|
exit 1
|
|
fi
|
|
|
|
SHORT_SHA=$(git rev-parse --short HEAD)
|
|
REPO_URL="https://${{ inputs.gitea-domain }}/${{ github.repository }}"
|
|
RUN_LINK="${REPO_URL}/actions/runs/${GITHUB_RUN_NUMBER}"
|
|
|
|
echo "9 - Generating wiki reports..."
|
|
|
|
TEST_FILE="run-${GITHUB_RUN_NUMBER}-TEST-commit-${SHORT_SHA}.md"
|
|
COVERAGE_FILE="run-${GITHUB_RUN_NUMBER}-COVERAGE-commit-${SHORT_SHA}.md"
|
|
|
|
echo "<h1 style='font-size: 42px;'>Test Report</h1>" > "wiki/${TEST_FILE}"
|
|
echo "" >> "wiki/${TEST_FILE}"
|
|
echo "[View CI Run ${GITHUB_RUN_NUMBER}](${RUN_LINK}) | [Commit ${SHORT_SHA}](${REPO_URL}/commit/${SHORT_SHA})" >> "wiki/${TEST_FILE}"
|
|
cat markdown/TEST-REPORT.md >> "wiki/${TEST_FILE}"
|
|
|
|
echo "## Coverage Report" > "wiki/${COVERAGE_FILE}"
|
|
echo "[View CI Run ${GITHUB_RUN_NUMBER}](${RUN_LINK}) | [Commit ${SHORT_SHA}](${REPO_URL}/commit/${SHORT_SHA})" >> "wiki/${COVERAGE_FILE}"
|
|
cat markdown/coverage-summary.md >> "wiki/${COVERAGE_FILE}"
|
|
|
|
HOME_FILE="wiki/Home.md"
|
|
TEMP_CONTENT=$(mktemp)
|
|
|
|
{
|
|
echo "# Test Reports History"
|
|
echo ""
|
|
echo "## Latest Run"
|
|
echo "- [run ${GITHUB_RUN_NUMBER} TEST commit ${SHORT_SHA}](${TEST_FILE})"
|
|
echo "- [run ${GITHUB_RUN_NUMBER} COVERAGE commit ${SHORT_SHA}](${COVERAGE_FILE})"
|
|
echo ""
|
|
echo "## Previous Runs"
|
|
|
|
if [[ -f "${HOME_FILE}" && -s "${HOME_FILE}" ]]; then
|
|
sed -n '/## Latest Run/{n;p;n;p}' "${HOME_FILE}"
|
|
sed -n '/## Previous Runs/,$p' "${HOME_FILE}" | tail -n +2 | grep -v "^#" || true
|
|
fi
|
|
} > "${TEMP_CONTENT}" 2> reports.log 2>&1 || {
|
|
echo ""
|
|
echo "Generating wiki reports..." >> $LOG_FILE
|
|
cat reports.log >> $LOG_FILE
|
|
exit 1
|
|
}
|
|
|
|
mv "${TEMP_CONTENT}" "${HOME_FILE}"
|
|
echo "Exit Code: 0" >> $LOG_FILE
|
|
|
|
|
|
- name: Commit and push wiki
|
|
shell: bash
|
|
env:
|
|
CI_TOKEN: ${{ inputs.ci-token }}
|
|
run: |
|
|
OUTPUT_DIR="$(pwd)/outputs"
|
|
LOG_FILE="${OUTPUT_DIR}/commit-push-wiki.log"
|
|
|
|
mkdir -p "$OUTPUT_DIR"
|
|
|
|
echo "10 - Pushing wiki changes..."
|
|
|
|
cd wiki
|
|
|
|
git config user.name "ci-bot"
|
|
git config user.email "ci-bot@example.com"
|
|
git add .
|
|
|
|
if git commit -m "Update wiki with run ${GITHUB_RUN_NUMBER}" >> $LOG_FILE 2>&1; then
|
|
REPO_NAME="${{ github.repository }}"
|
|
DOMAIN="${{ inputs.gitea-domain }}"
|
|
WIKI_REPO="https://ci-token:${CI_TOKEN}@${DOMAIN}/${REPO_NAME}.wiki.git"
|
|
git push "$WIKI_REPO" HEAD:main >> git.log 2>&1 || {
|
|
cat git.log >> $LOG_FILE
|
|
exit 1
|
|
}
|
|
else
|
|
echo "⚠️ No changes to commit" >> $LOG_FILE
|
|
fi
|
|
echo "Exit Code: 0" >> $LOG_FILE
|
|
|
|
- name: Log wiki report URLs
|
|
shell: bash
|
|
run: |
|
|
OUTPUT_DIR="outputs"
|
|
LOG_FILE=""$OUTPUT_DIR"/wiki-report-urls.log"
|
|
|
|
mkdir -p "$OUTPUT_DIR"
|
|
|
|
REPO_URL="https://${{ inputs.gitea-domain }}/${{ github.repository }}"
|
|
SHORT_SHA=$(git rev-parse --short HEAD)
|
|
echo "🔗 Wiki Test Report URL:" >> $LOG_FILE
|
|
echo " ${REPO_URL}/wiki/run-${GITHUB_RUN_NUMBER}-TEST-commit-${SHORT_SHA}.md" >> $LOG_FILE
|
|
echo "🔗 Wiki Coverage Report URL:" >> $LOG_FILE
|
|
echo " ${REPO_URL}/wiki/run-${GITHUB_RUN_NUMBER}-COVERAGE-commit-${SHORT_SHA}.md" >> $LOG_FILE
|
|
echo "Exit Code: 0" >> $LOG_FILE |