diff --git a/.github/workflows/mirror.yaml b/.github/workflows/mirror.yaml index d9bca11b..6d8de170 100644 --- a/.github/workflows/mirror.yaml +++ b/.github/workflows/mirror.yaml @@ -9,6 +9,10 @@ on: description: 'common_bench branch/tag to use' required: false default: '' + BENCHMARK_RUNNER: + description: 'Runner backend to use (htcondor or empty for local)' + required: false + default: 'htcondor' SOURCE_REPOSITORY: description: 'Repository to report status to (owner/repo)' required: false @@ -65,6 +69,7 @@ jobs: GITHUB_PR=${{ inputs.SOURCE_PR || github.event.pull_request.number }} PIPELINE_NAME=${{ inputs.SOURCE_REPOSITORY || github.repository }}: ${{ inputs.SOURCE_TITLE || github.event.pull_request.title || github.ref_name }} COMMON_BENCH_VERSION=${{ inputs.COMMON_BENCH_VERSION }} + BENCHMARK_RUNNER=${{ inputs.BENCHMARK_RUNNER || 'htcondor' }} - name: Set pending EICweb status if: ${{ github.event_name != 'delete' && !inputs.SOURCE_SHA }} env: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ae1a36c4..95d2d1ea 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,6 +38,9 @@ default: reports: dotenv: .env when: always + id_tokens: + CI_JOB_JWT: + aud: https://eicweb.phy.anl.gov stages: - status-pending @@ -116,6 +119,9 @@ common:setup: - mkdir "${DETECTOR_CONFIG}" - ln -s "${LOCAL_DATA_PATH}/sim_output" "${DETECTOR_CONFIG}/sim_output" - ln -s "../results" "${DETECTOR_CONFIG}/results" + - if [ ! -w "$SNAKEMAKE_OUTPUT_CACHE" ] ; then + export SNAKEMAKE_OUTPUT_CACHE=${LOCAL_DATA_PATH}/snakemake-cache ; + fi - mkdir -p "$SNAKEMAKE_OUTPUT_CACHE" - ls -lrtha retry: @@ -123,6 +129,12 @@ common:setup: when: - runner_system_failure +.det_benchmark:htcondor: + extends: .det_benchmark + tags: + - osg + - jacamar + include: - local: 'benchmarks/backgrounds/config.yml' - local: 'benchmarks/backwards_ecal/config.yml' diff --git a/benchmarks/campaign/config.yml b/benchmarks/campaign/config.yml index 3d1942b2..6a05d472 100644 --- a/benchmarks/campaign/config.yml +++ b/benchmarks/campaign/config.yml @@ -1,11 +1,11 @@ bench:campaign: - extends: .det_benchmark + extends: .det_benchmark:htcondor stage: benchmarks script: - snakemake $SNAKEMAKE_FLAGS --cores 1 campaign_benchmark collect_results:campaign: - extends: .det_benchmark + extends: .det_benchmark:htcondor stage: collect needs: - "bench:campaign" diff --git a/benchmarks/material_scan/config.yml b/benchmarks/material_scan/config.yml index 894a369f..67794a6a 100644 --- a/benchmarks/material_scan/config.yml +++ b/benchmarks/material_scan/config.yml @@ -1,11 +1,11 @@ bench:material_scan: - extends: .det_benchmark + extends: .det_benchmark:htcondor stage: benchmarks script: - snakemake $SNAKEMAKE_FLAGS --cores 1 results/material_scan/epic_craterlake/material_scan_details.pdf collect_results:material_scan: - extends: .det_benchmark + extends: .det_benchmark:htcondor stage: collect needs: - ["bench:material_scan"] diff --git a/benchmarks/rich/config.yml b/benchmarks/rich/config.yml index 57a92b9a..eefcb646 100644 --- a/benchmarks/rich/config.yml +++ b/benchmarks/rich/config.yml @@ -1,5 +1,5 @@ bench:drich_fixed_eta: - extends: .det_benchmark + extends: .det_benchmark:htcondor stage: benchmarks script: - cmake -S benchmarks/rich -B build -DCMAKE_INSTALL_PREFIX=.local @@ -11,7 +11,7 @@ bench:drich_fixed_eta: done collect_results:rich: - extends: .det_benchmark + extends: .det_benchmark:htcondor stage: collect needs: - "bench:drich_fixed_eta"