|
| 1 | +name: "Hotspots Build/Validate" |
| 2 | + |
| 3 | +# Build/validate-only counterpart to `hotspots.yml` (which is a manual |
| 4 | +# `workflow_dispatch` build *and publish* of the `githubsecuritylab/hotspots-*` packs). |
| 5 | +# |
| 6 | +# This workflow runs the exact same `ql/hotspots` pipeline - generate the per-language |
| 7 | +# `Hotspots-<lang>.ql` queries with QL-4-QL, patch a copy of the `github/codeql` distribution, |
| 8 | +# then build the resulting packs - but stops at `codeql pack create` and never publishes |
| 9 | +# anything. It exists so PRs that touch `ql/hotspots/**` (notably Dependabot bumps of |
| 10 | +# `ql/hotspots/requirements.txt`, which the generator/patch scripts run on) get the same |
| 11 | +# CI signal every other pack in this repo already gets from `ci.yml`. |
| 12 | +# |
| 13 | +# Unlike `hotspots.yml`, which builds against the CodeQL CLI bundled with the CodeQL Action and |
| 14 | +# `github/codeql@main`, this workflow is deliberately deterministic: it uses this repo's pinned |
| 15 | +# `.codeqlversion` CLI and the matching `codeql-cli/v<version>` tag of `github/codeql`, so a red |
| 16 | +# run means *this PR* broke something rather than upstream moving underneath us. |
| 17 | +# |
| 18 | +# Security note: this workflow checks out and executes code from another repository |
| 19 | +# (`github/codeql`), so it is deliberately confined to the `pull_request` event - where the Actions |
| 20 | +# cache scope is the PR's own branch - with no `workflow_dispatch` (which would run it on the |
| 21 | +# default branch, whose cache is shared with every other workflow). For the same reason the |
| 22 | +# `github/codeql` ref is always derived from `.codeqlversion` rather than a caller-supplied input. |
| 23 | +# To build against a different `github/codeql` ref, |
| 24 | +# run the `hotspots.yml` publish workflow or the pipeline locally (see `ql/hotspots/README.md`). |
| 25 | + |
| 26 | +on: |
| 27 | + pull_request: |
| 28 | + branches: [main] |
| 29 | + paths: |
| 30 | + - "ql/hotspots/**" |
| 31 | + - ".github/actions/install-codeql/**" |
| 32 | + - ".github/workflows/hotspots-ci.yml" |
| 33 | + - ".codeqlversion" |
| 34 | + |
| 35 | +permissions: |
| 36 | + contents: read |
| 37 | + |
| 38 | +concurrency: |
| 39 | + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} |
| 40 | + cancel-in-progress: true |
| 41 | + |
| 42 | +env: |
| 43 | + # `swift` is deliberately absent: `generate-hotspots-queries.py` has no Swift support (it isn't |
| 44 | + # in that script's `supported_languages`), so no `Hotspots-swift.ql` is ever generated and the |
| 45 | + # patched Swift query pack has nothing to compile. `patch-codeql.py` still processes Swift (it |
| 46 | + # prints `[-] Hotspots queries not found ...` and moves on), and `hotspots.yml` still publishes |
| 47 | + # whatever that produces - validating it here would only ever fail on that known gap. |
| 48 | + HOTSPOTS_LANGUAGES: '["cpp", "csharp", "go", "java", "javascript", "python", "ruby"]' |
| 49 | + |
| 50 | +jobs: |
| 51 | + generate: |
| 52 | + name: Generate hotspots queries |
| 53 | + runs-on: ubuntu-latest |
| 54 | + outputs: |
| 55 | + codeql-ref: ${{ steps.ql-ref.outputs.ref }} |
| 56 | + languages: ${{ steps.languages.outputs.languages }} |
| 57 | + steps: |
| 58 | + - uses: actions/checkout@v7 |
| 59 | + with: |
| 60 | + persist-credentials: false |
| 61 | + |
| 62 | + - name: Expose language matrix |
| 63 | + id: languages |
| 64 | + run: echo "languages=${HOTSPOTS_LANGUAGES}" >> "$GITHUB_OUTPUT" |
| 65 | + |
| 66 | + - name: Setup CodeQL |
| 67 | + id: install-codeql |
| 68 | + uses: ./.github/actions/install-codeql |
| 69 | + |
| 70 | + - name: Resolve github/codeql ref |
| 71 | + id: ql-ref |
| 72 | + env: |
| 73 | + CODEQL_CLI_VERSION: ${{ steps.install-codeql.outputs.codeql-cli-version }} |
| 74 | + run: echo "ref=codeql-cli/v${CODEQL_CLI_VERSION}" >> "$GITHUB_OUTPUT" |
| 75 | + |
| 76 | + - name: Checkout github/codeql |
| 77 | + uses: actions/checkout@v7 |
| 78 | + with: |
| 79 | + repository: github/codeql |
| 80 | + ref: ${{ steps.ql-ref.outputs.ref }} |
| 81 | + path: codeql |
| 82 | + persist-credentials: false |
| 83 | + |
| 84 | + - name: Setup Python |
| 85 | + uses: actions/setup-python@v7 |
| 86 | + with: |
| 87 | + # `requirements.txt` is `pip-compile --generate-hashes`d against Python 3.12 and pins |
| 88 | + # cp312-only wheels, so the interpreter version has to match. |
| 89 | + python-version: "3.12" |
| 90 | + |
| 91 | + - name: Install Python dependencies |
| 92 | + run: python -m pip install --require-hashes -r ql/hotspots/requirements.txt |
| 93 | + |
| 94 | + - name: Validate scripts |
| 95 | + run: | |
| 96 | + python -m compileall -q ql/hotspots/scripts |
| 97 | + python ql/hotspots/scripts/generate-hotspots-queries.py --help > /dev/null |
| 98 | + python ql/hotspots/scripts/patch-codeql.py --help > /dev/null |
| 99 | +
|
| 100 | + # ~1 minute of `cargo build --release`; deliberately not cached, to keep this job free of |
| 101 | + # Actions cache writes of anything derived from the `github/codeql` checkout. |
| 102 | + - name: Build QL extractor |
| 103 | + working-directory: codeql/ql |
| 104 | + run: ./scripts/create-extractor-pack.sh |
| 105 | + |
| 106 | + - name: Generate hotspots queries |
| 107 | + run: | |
| 108 | + python ql/hotspots/scripts/generate-hotspots-queries.py \ |
| 109 | + --ql-extractor "$GITHUB_WORKSPACE/codeql/ql/extractor-pack" \ |
| 110 | + --ql-path "$GITHUB_WORKSPACE/codeql" |
| 111 | +
|
| 112 | + - name: Check generated queries |
| 113 | + env: |
| 114 | + LANGUAGES: ${{ env.HOTSPOTS_LANGUAGES }} |
| 115 | + run: | |
| 116 | + set -euo pipefail |
| 117 | + failed=0 |
| 118 | + for lang in $(echo "$LANGUAGES" | jq --raw-output '.[]'); do |
| 119 | + query="ql/hotspots/output/Hotspots-${lang}.ql" |
| 120 | + if [[ ! -f "$query" ]]; then |
| 121 | + echo "::error::No query generated for ${lang} (${query} is missing)" |
| 122 | + failed=1 |
| 123 | + continue |
| 124 | + fi |
| 125 | + # A generated query with no `import ... as P<hash>` lines means the QL-4-QL generator |
| 126 | + # matched no TaintTracking configurations at all for that language - the query would |
| 127 | + # still compile, but it would be an empty, useless hotspots query. |
| 128 | + imports=$(grep --count --extended-regexp '^import .* as P[0-9a-f]+' "$query" || true) |
| 129 | + if [[ "$imports" -eq 0 ]]; then |
| 130 | + echo "::error::${query} contains no taint-tracking configuration imports" |
| 131 | + failed=1 |
| 132 | + continue |
| 133 | + fi |
| 134 | + echo "[+] ${lang}: ${imports} configuration import(s)" |
| 135 | + done |
| 136 | + exit "$failed" |
| 137 | +
|
| 138 | + - name: Upload generated hotspots queries |
| 139 | + uses: actions/upload-artifact@v7 |
| 140 | + with: |
| 141 | + name: hotspots-generated-queries |
| 142 | + path: ql/hotspots/output |
| 143 | + if-no-files-found: error |
| 144 | + |
| 145 | + build-packs: |
| 146 | + name: Build ${{ matrix.language }} hotspots packs |
| 147 | + needs: generate |
| 148 | + runs-on: ubuntu-latest |
| 149 | + permissions: |
| 150 | + contents: read |
| 151 | + packages: read # `codeql pack install` reads from GHCR |
| 152 | + |
| 153 | + strategy: |
| 154 | + fail-fast: false |
| 155 | + matrix: |
| 156 | + language: ${{ fromJSON(needs.generate.outputs.languages) }} |
| 157 | + |
| 158 | + steps: |
| 159 | + - uses: actions/checkout@v7 |
| 160 | + with: |
| 161 | + persist-credentials: false |
| 162 | + |
| 163 | + - name: Setup CodeQL |
| 164 | + uses: ./.github/actions/install-codeql |
| 165 | + |
| 166 | + - name: Checkout github/codeql |
| 167 | + uses: actions/checkout@v7 |
| 168 | + with: |
| 169 | + repository: github/codeql |
| 170 | + ref: ${{ needs.generate.outputs.codeql-ref }} |
| 171 | + path: codeql |
| 172 | + persist-credentials: false |
| 173 | + |
| 174 | + - name: Setup Python |
| 175 | + uses: actions/setup-python@v7 |
| 176 | + with: |
| 177 | + python-version: "3.12" |
| 178 | + |
| 179 | + - name: Install Python dependencies |
| 180 | + run: python -m pip install --require-hashes -r ql/hotspots/requirements.txt |
| 181 | + |
| 182 | + - name: Download generated hotspots queries |
| 183 | + uses: actions/download-artifact@v8 |
| 184 | + with: |
| 185 | + name: hotspots-generated-queries |
| 186 | + path: ql/hotspots/output |
| 187 | + |
| 188 | + - name: Patch the CodeQL distribution |
| 189 | + run: | |
| 190 | + python ql/hotspots/scripts/patch-codeql.py \ |
| 191 | + --hotspots "$GITHUB_WORKSPACE/ql/hotspots/output" \ |
| 192 | + --ql "$GITHUB_WORKSPACE/codeql" \ |
| 193 | + --dest "$GITHUB_WORKSPACE/codeql-patched" \ |
| 194 | + --qlpack-version 0.0.0 |
| 195 | +
|
| 196 | + - name: Build hotspots packs (no publish) |
| 197 | + working-directory: codeql-patched |
| 198 | + env: |
| 199 | + GITHUB_TOKEN: ${{ github.token }} |
| 200 | + LANGUAGE: ${{ matrix.language }} |
| 201 | + # `pack create` is the compile step `pack publish` performs internally, so this is the |
| 202 | + # same build `hotspots.yml` does - just without the upload to GHCR. |
| 203 | + PACK_OUTPUT: ${{ runner.temp }}/hotspots-packs |
| 204 | + run: | |
| 205 | + set -euo pipefail |
| 206 | + for dir in lib src; do |
| 207 | + echo "::group::${LANGUAGE}/ql/${dir}" |
| 208 | + codeql pack install "${LANGUAGE}/ql/${dir}" |
| 209 | + codeql pack create "${LANGUAGE}/ql/${dir}" --output="${PACK_OUTPUT}" |
| 210 | + echo "::endgroup::" |
| 211 | + done |
0 commit comments