diff --git a/.github/workflows/betterleaks.yml b/.github/workflows/betterleaks.yml new file mode 100644 index 0000000..e3ee2df --- /dev/null +++ b/.github/workflows/betterleaks.yml @@ -0,0 +1,133 @@ +name: Betterleaks + +on: + push: + pull_request: + +permissions: + contents: read + +jobs: + scan: + name: Scan for secrets + runs-on: ubuntu-latest + env: + SMTP_URL: ${{ secrets.SMTP_URL }} + SMTP_PORT: ${{ secrets.SMTP_PORT || '25' }} + SMTP_EMAIL: ${{ secrets.SMTP_EMAIL }} + SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 1 + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Run Betterleaks + id: betterleaks + continue-on-error: true + uses: dortort/betterleaks-action@v0.1.0 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + scan-mode: dir + scan-path: . + config: .gitleaks.toml + report-format: json + report-path: betterleaks-report.json + redact: "true" + no-color: "true" + no-banner: "true" + fail-on-leak: "true" + + - name: Upload Betterleaks report + if: always() + uses: actions/upload-artifact@v4 + with: + name: betterleaks-report + path: betterleaks-report.json + if-no-files-found: ignore + + - name: Build Betterleaks email summary + id: leak_summary + if: steps.betterleaks.outcome == 'failure' + shell: bash + run: | + if [[ -s betterleaks-report.json ]]; then + jq -r ' + def one_line: + tostring + | gsub("[\r\n]+"; " ") + | if length > 240 then .[0:240] + "..." else . end; + + .[:20][] + | "- " + (.RuleID // "unknown-rule") + + " at " + (.File // "unknown-file") + + ":" + ((.StartLine // 0) | tostring) + + "\n match: " + ((.Match // .Secret // "REDACTED") | one_line) + ' betterleaks-report.json > betterleaks-email-summary.txt + + count="$(jq 'length' betterleaks-report.json)" + if (( count > 20 )); then + { + echo "" + echo "... and $((count - 20)) more finding(s). Download the artifact for full details." + } >> betterleaks-email-summary.txt + fi + else + echo "No JSON report was generated. Download the workflow logs for details." > betterleaks-email-summary.txt + fi + + { + echo "text<> "$GITHUB_OUTPUT" + + - name: Resolve committer email + id: committer + if: steps.betterleaks.outcome == 'failure' + shell: bash + run: | + committer_email="$(git log -1 --format='%ce')" + author_email="$(git log -1 --format='%ae')" + email="$committer_email" + if [[ -z "$email" || "$email" == *"noreply.github.com"* ]]; then + email="$author_email" + fi + if [[ "$email" =~ ^[^[:space:]@]+@[^[:space:]@]+\.[^[:space:]@]+$ && "$email" != *"noreply.github.com"* ]]; then + echo "email=$email" >> "$GITHUB_OUTPUT" + else + echo "No deliverable committer email found; skipping Betterleaks email notification." + echo "email=" >> "$GITHUB_OUTPUT" + fi + + - name: Email committer on Betterleaks failure + if: steps.betterleaks.outcome == 'failure' && steps.committer.outputs.email != '' && env.SMTP_URL != '' && env.SMTP_EMAIL != '' + uses: dawidd6/action-send-mail@v18 + with: + server_address: ${{ env.SMTP_URL }} + server_port: ${{ env.SMTP_PORT }} + secure: ${{ env.SMTP_PORT == '465' }} + username: ${{ env.SMTP_EMAIL }} + password: ${{ env.SMTP_PASSWORD }} + from: ${{ env.SMTP_EMAIL }} + to: ${{ steps.committer.outputs.email }} + subject: "[Betterleaks] Secret scan failed in ${{ github.repository }}" + body: | + Betterleaks detected one or more potential secrets. + + Repository: ${{ github.repository }} + Branch: ${{ github.ref_name }} + Commit: ${{ github.sha }} + Workflow run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + Findings: + ${{ steps.leak_summary.outputs.text }} + + Download the betterleaks-report artifact from the workflow run for details. + + - name: Fail if Betterleaks found leaks + if: steps.betterleaks.outcome == 'failure' + run: | + echo "Betterleaks detected one or more secrets. Download the betterleaks-report artifact from this workflow run for details." + exit 1 diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 0000000..4f89ba0 --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,100 @@ +title = "Betterleaks config" + +[extend] +useDefault = true + +[[rules]] +id = "secret-literal-assignment" +description = "Secret-like literal assignment in source, YAML, env and config files" +regex = '''(?i)([A-Za-z0-9_.-]*(?:password|passwd|pwd|api[_-]?key|apikey|access[_-]?key|secret[_-]?access[_-]?key|private[_-]?key|client[_-]?secret|consumer[_-]?key|consumer[_-]?secret|(?:auth|access|refresh|session|api|bearer|id|jwt|csrf|xsrf|oauth)[_-]?token|translate[_-]?auth|translation[_-]?auth|auth[_-]?password|auth[_-]?key|auth[_-]?credentials|database[_-]?url|connection[_-]?string)[A-Za-z0-9_.-]*)[ \t]*[:=][ \t]*["'']?([^"''[:space:]#{}$.][^"''\n#{}]{2,})["'']?''' +secretGroup = 2 +keywords = [ + "password", + "passwd", + "pwd", + "api_key", + "apikey", + "access_key", + "private_key", + "client_secret", + "consumer_key", + "consumer_secret", + "auth_token", + "access_token", + "refresh_token", + "session_token", + "api_token", + "bearer_token", + "id_token", + "jwt_token", + "csrf_token", + "xsrf_token", + "oauth_token", + "translate_auth", + "translation_auth", + "auth_password", + "auth_key", + "auth_credentials", + "database_url", + "connection_string" +] +tags = ["literal-secret"] + +[[rules]] +id = "env-short-secret-assignment" +description = "Uppercase env-style PASS/TOKEN/SECRET assignment" +regex = '''\b((?:PASS|TOKEN|SECRET|[A-Z0-9_]*(?:_PASS|_TOKEN|_SECRET|PASS_|TOKEN_|SECRET_)[A-Z0-9_]*))[ \t]*[:=][ \t]*["'']?([^"''[:space:]#{}$.][^"''\n#{}]{2,})["'']?''' +secretGroup = 2 +keywords = [ + "PASS", + "TOKEN", + "SECRET" +] +tags = ["env", "literal-secret"] + +[[rules]] +id = "standalone-sk-token" +description = "Standalone sk-* token not attached to a secret-like variable name" +regex = '''(?i)\b(sk-[A-Za-z0-9][A-Za-z0-9_-]{20,})\b''' +secretGroup = 1 +entropy = 2.5 +keywords = ["sk-"] +tags = ["standalone-token", "generic"] + +[[rules]] +id = "dotenv-only-jest-setup" +description = ".env may only contain the committed Jest setup lines" +path = '''(?i)(^|/)\.env$''' +regex = '''(?m)^(.+)$''' +secretGroup = 1 +tags = ["file", "dotenv"] +[[rules.allowlists]] +regexTarget = "match" +regexes = [ + '''^JEST_USE_SETUP=(ON|OFF) # Jest configuration variables: ON, OFF\r?$''', +] + +[[rules]] +id = "forbidden-secret-file" +description = "Forbidden secret-bearing file committed to repository" +path = '''(?i)(^|/)(\.env\..*|\.npmrc|\.pypirc|id_rsa|id_ed25519|.*\.(pem|key|p12|pfx|jks|kubeconfig))$''' +regex = '''(?s).{1,}''' +tags = ["file", "secret-file"] + +[[allowlists]] +description = "Allow Jenkins SonarQube token environment variable reference" +regexTarget = "match" +regexes = [ + '''SONAR_AUTH_TOKEN''', +] + +[[allowlists]] +description = "Allow local development RelStorage password defaults" +paths = [ + '''(?i)(^|/)develop/etc/(relstorage|alpha)\.conf$''', +] +regexTarget = "match" +regexes = [ + '''password='zope''', +] + diff --git a/Jenkinsfile b/Jenkinsfile index 7bc7dc1..ae5c4a6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -37,7 +37,7 @@ pipeline { } steps{ node(label: 'docker') { - withCredentials([string(credentialsId: 'eea-jenkins-token', variable: 'GITHUB_TOKEN'), string(credentialsId: 'bise-backend-trigger', variable: 'TRIGGER_MAIN_URL'), usernamePassword(credentialsId: 'jekinsdockerhub', usernameVariable: 'DOCKERHUB_USER', passwordVariable: 'DOCKERHUB_PASS')]) { + withCredentials([string(credentialsId: 'eea-jenkins-token', variable: 'GITHUB_TOKEN'), string(credentialsId: 'bise-backend-trigger', variable: 'TRIGGER_MAIN_URL'), usernamePassword(credentialsId: 'jekinsdockerhub', usernameVariable: 'DOCKERHUB_USER', passwordVariable: 'DOCKERHUB_PASS')]) { //betterleaks:allow sh '''docker pull eeacms/gitflow; docker run -i --rm --name="$BUILD_TAG" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_NAME="$GIT_NAME" -e DOCKERHUB_REPO="${IMAGE_NAME}" -e GIT_TOKEN="$GITHUB_TOKEN" -e DOCKERHUB_USER="$DOCKERHUB_USER" -e DOCKERHUB_PASS="$DOCKERHUB_PASS" -e TRIGGER_MAIN_URL="$TRIGGER_MAIN_URL" -e DEPENDENT_DOCKERFILE_URL="" -e GITFLOW_BEHAVIOR="RUN_ON_TAG" eeacms/gitflow''' } diff --git a/README.md b/README.md index 537f8e8..1364bdf 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,65 @@ See [develop](https://github.com/eea/bise-backend/tree/master/develop) See [release](https://github.com/eea/bise-backend/tree/master/RELEASE.md) +## Secret Scanning + +This repository uses the Betterleaks GitHub Action to scan the current +repository content on every push and pull request. The scan uses the rules in +`.gitleaks.toml` and uploads a `betterleaks-report` artifact when a finding is +detected. + +If the optional SMTP secrets are configured, failed scans also send an email to +the last commit committer. The workflow expects these repository or +organization secrets: + +- `SMTP_URL` +- `SMTP_PORT` (optional, defaults to `25`) +- `SMTP_EMAIL` +- `SMTP_PASSWORD` (optional if the SMTP server does not require authentication) + +Port `465` is sent with direct TLS; other ports use the default SMTP handshake. +The email includes a short finding summary from the redacted Betterleaks report, +including the redacted matched line from each finding. + +There are three common outcomes: + +1. **Everything is OK.** The `Betterleaks / Scan for secrets` check is green and + no action is needed. Regular references to runtime values are OK, for example: + + ```js + const tokenFromCookie = req.universalCookies.get('auth_token'); + ``` + +2. **A real secret was found.** The check is red and the workflow log asks you to + download the `betterleaks-report` artifact. Open the artifact from the GitHub + Actions run and check the reported file, line and rule. Remove the committed + value, move it to the proper secret store, and rotate it if it was exposed. + A report entry looks like this: + + ```json + { + "RuleID": "secret-literal-assignment", + "File": "src/config.js", + "StartLine": 12, + "Secret": "[REDACTED]" + } + ``` + +3. **The finding is a false positive.** Keep the value only if it is clearly not + sensitive, such as a test fixture, placeholder, or public example. Add + `betterleaks:allow` on the same line and include a short explanation in the + pull request. + + ```js + const testPassword = 'admin'; //betterleaks:allow + ``` + + ```yaml + password: "admin" #betterleaks:allow + ``` + +Do not add `betterleaks:allow` to real credentials. + ## Copyright and license The Initial Owner of the Original Code is European Environment Agency (EEA). diff --git a/test/tests/no-hard-coded-passwords/run.sh b/test/tests/no-hard-coded-passwords/run.sh index 9f326b4..eba6919 100755 --- a/test/tests/no-hard-coded-passwords/run.sh +++ b/test/tests/no-hard-coded-passwords/run.sh @@ -2,7 +2,7 @@ set -e IFS=$'\n' -userPasswds=( $(docker run --rm --user 0:0 --entrypoint cut "$1" -d: -f1-2 /etc/passwd) ) +userPasswds=( $(docker run --rm --user 0:0 --entrypoint cut "$1" -d: -f1-2 /etc/passwd) ) #betterleaks:allow userShadows=() if grep -qE ':x$' <<<"${userPasswds[*]}"; then userShadows=( $(docker run --rm --user 0:0 --entrypoint cut "$1" -d: -f1-2 /etc/shadow || true) ) diff --git a/test/tests/plone-relstorage/run.sh b/test/tests/plone-relstorage/run.sh index 1402621..0f29862 100755 --- a/test/tests/plone-relstorage/run.sh +++ b/test/tests/plone-relstorage/run.sh @@ -11,11 +11,11 @@ PLONE_TEST_TRIES=10 # Start Postgres zname="relstorage-container-$RANDOM-$RANDOM" zpull="$(docker pull postgres:12-alpine)" -zid="$(docker run -d --name "$zname" -e POSTGRES_USER=plone -e POSTGRES_PASSWORD=plone -e POSTGRES_DB=plone postgres:12-alpine)" +zid="$(docker run -d --name "$zname" -e POSTGRES_USER=plone -e POSTGRES_PASSWORD=plone -e POSTGRES_DB=plone postgres:12-alpine)" #betterleaks:allow # Start Plone as RelStorage Client pname="plone-container-$RANDOM-$RANDOM" -pid="$(docker run -d --name "$pname" --link=$zname:db -e RELSTORAGE_DSN="dbname='plone' user='plone' host='db' password='plone'" "$image")" +pid="$(docker run -d --name "$pname" --link=$zname:db -e RELSTORAGE_DSN="dbname='plone' user='plone' host='db' password='plone'" "$image")" #betterleaks:allow # Tear down trap "docker rm -vf $pid $zid > /dev/null" EXIT