Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 133 additions & 0 deletions .github/workflows/betterleaks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
name: Betterleaks

on:
push:
pull_request:

permissions:
contents: read

jobs:
scan:
name: Scan for secrets
runs-on: ubuntu-latest
env:
SMTP_URL: ${{ secrets.SMTP_URL }}
SMTP_PORT: ${{ secrets.SMTP_PORT || '25' }}
SMTP_EMAIL: ${{ secrets.SMTP_EMAIL }}
SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 1
ref: ${{ github.event.pull_request.head.sha || github.sha }}

- name: Run Betterleaks
id: betterleaks
continue-on-error: true
uses: dortort/betterleaks-action@v0.1.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
scan-mode: dir
scan-path: .
config: .gitleaks.toml
report-format: json
report-path: betterleaks-report.json
redact: "true"
no-color: "true"
no-banner: "true"
fail-on-leak: "true"

- name: Upload Betterleaks report
if: always()
uses: actions/upload-artifact@v4
with:
name: betterleaks-report
path: betterleaks-report.json
if-no-files-found: ignore

- name: Build Betterleaks email summary
id: leak_summary
if: steps.betterleaks.outcome == 'failure'
shell: bash
run: |
if [[ -s betterleaks-report.json ]]; then
jq -r '
def one_line:
tostring
| gsub("[\r\n]+"; " ")
| if length > 240 then .[0:240] + "..." else . end;

.[:20][]
| "- " + (.RuleID // "unknown-rule")
+ " at " + (.File // "unknown-file")
+ ":" + ((.StartLine // 0) | tostring)
+ "\n match: " + ((.Match // .Secret // "REDACTED") | one_line)
' betterleaks-report.json > betterleaks-email-summary.txt

count="$(jq 'length' betterleaks-report.json)"
if (( count > 20 )); then
{
echo ""
echo "... and $((count - 20)) more finding(s). Download the artifact for full details."
} >> betterleaks-email-summary.txt
fi
else
echo "No JSON report was generated. Download the workflow logs for details." > betterleaks-email-summary.txt
fi

{
echo "text<<BETTERLEAKS_SUMMARY"
cat betterleaks-email-summary.txt
echo "BETTERLEAKS_SUMMARY"
} >> "$GITHUB_OUTPUT"

- name: Resolve committer email
id: committer
if: steps.betterleaks.outcome == 'failure'
shell: bash
run: |
committer_email="$(git log -1 --format='%ce')"
author_email="$(git log -1 --format='%ae')"
email="$committer_email"
if [[ -z "$email" || "$email" == *"noreply.github.com"* ]]; then
email="$author_email"
fi
if [[ "$email" =~ ^[^[:space:]@]+@[^[:space:]@]+\.[^[:space:]@]+$ && "$email" != *"noreply.github.com"* ]]; then
echo "email=$email" >> "$GITHUB_OUTPUT"
else
echo "No deliverable committer email found; skipping Betterleaks email notification."
echo "email=" >> "$GITHUB_OUTPUT"
fi

- name: Email committer on Betterleaks failure
if: steps.betterleaks.outcome == 'failure' && steps.committer.outputs.email != '' && env.SMTP_URL != '' && env.SMTP_EMAIL != ''
uses: dawidd6/action-send-mail@v18
with:
server_address: ${{ env.SMTP_URL }}
server_port: ${{ env.SMTP_PORT }}
secure: ${{ env.SMTP_PORT == '465' }}
username: ${{ env.SMTP_EMAIL }}
password: ${{ env.SMTP_PASSWORD }}
from: ${{ env.SMTP_EMAIL }}
to: ${{ steps.committer.outputs.email }}
subject: "[Betterleaks] Secret scan failed in ${{ github.repository }}"
body: |
Betterleaks detected one or more potential secrets.

Repository: ${{ github.repository }}
Branch: ${{ github.ref_name }}
Commit: ${{ github.sha }}
Workflow run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

Findings:
${{ steps.leak_summary.outputs.text }}

Download the betterleaks-report artifact from the workflow run for details.

- name: Fail if Betterleaks found leaks
if: steps.betterleaks.outcome == 'failure'
run: |
echo "Betterleaks detected one or more secrets. Download the betterleaks-report artifact from this workflow run for details."
exit 1
100 changes: 100 additions & 0 deletions .gitleaks.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
title = "Betterleaks config"

[extend]
useDefault = true

[[rules]]
id = "secret-literal-assignment"
description = "Secret-like literal assignment in source, YAML, env and config files"
regex = '''(?i)([A-Za-z0-9_.-]*(?:password|passwd|pwd|api[_-]?key|apikey|access[_-]?key|secret[_-]?access[_-]?key|private[_-]?key|client[_-]?secret|consumer[_-]?key|consumer[_-]?secret|(?:auth|access|refresh|session|api|bearer|id|jwt|csrf|xsrf|oauth)[_-]?token|translate[_-]?auth|translation[_-]?auth|auth[_-]?password|auth[_-]?key|auth[_-]?credentials|database[_-]?url|connection[_-]?string)[A-Za-z0-9_.-]*)[ \t]*[:=][ \t]*["'']?([^"''[:space:]#{}$.][^"''\n#{}]{2,})["'']?'''
secretGroup = 2
keywords = [
"password",
"passwd",
"pwd",
"api_key",
"apikey",
"access_key",
"private_key",
"client_secret",
"consumer_key",
"consumer_secret",
"auth_token",
"access_token",
"refresh_token",
"session_token",
"api_token",
"bearer_token",
"id_token",
"jwt_token",
"csrf_token",
"xsrf_token",
"oauth_token",
"translate_auth",
"translation_auth",
"auth_password",
"auth_key",
"auth_credentials",
"database_url",
"connection_string"
]
tags = ["literal-secret"]

[[rules]]
id = "env-short-secret-assignment"
description = "Uppercase env-style PASS/TOKEN/SECRET assignment"
regex = '''\b((?:PASS|TOKEN|SECRET|[A-Z0-9_]*(?:_PASS|_TOKEN|_SECRET|PASS_|TOKEN_|SECRET_)[A-Z0-9_]*))[ \t]*[:=][ \t]*["'']?([^"''[:space:]#{}$.][^"''\n#{}]{2,})["'']?'''
secretGroup = 2
keywords = [
"PASS",
"TOKEN",
"SECRET"
]
tags = ["env", "literal-secret"]

[[rules]]
id = "standalone-sk-token"
description = "Standalone sk-* token not attached to a secret-like variable name"
regex = '''(?i)\b(sk-[A-Za-z0-9][A-Za-z0-9_-]{20,})\b'''
secretGroup = 1
entropy = 2.5
keywords = ["sk-"]
tags = ["standalone-token", "generic"]

[[rules]]
id = "dotenv-only-jest-setup"
description = ".env may only contain the committed Jest setup lines"
path = '''(?i)(^|/)\.env$'''
regex = '''(?m)^(.+)$'''
secretGroup = 1
tags = ["file", "dotenv"]
[[rules.allowlists]]
regexTarget = "match"
regexes = [
'''^JEST_USE_SETUP=(ON|OFF) # Jest configuration variables: ON, OFF\r?$''',
]

[[rules]]
id = "forbidden-secret-file"
description = "Forbidden secret-bearing file committed to repository"
path = '''(?i)(^|/)(\.env\..*|\.npmrc|\.pypirc|id_rsa|id_ed25519|.*\.(pem|key|p12|pfx|jks|kubeconfig))$'''
regex = '''(?s).{1,}'''
tags = ["file", "secret-file"]

[[allowlists]]
description = "Allow Jenkins SonarQube token environment variable reference"
regexTarget = "match"
regexes = [
'''SONAR_AUTH_TOKEN''',
]

[[allowlists]]
description = "Allow local development RelStorage password defaults"
paths = [
'''(?i)(^|/)develop/etc/(relstorage|alpha)\.conf$''',
]
regexTarget = "match"
regexes = [
'''password='zope''',
]

4 changes: 2 additions & 2 deletions Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ pipeline {
}
steps{
node(label: 'docker') {
withCredentials([string(credentialsId: 'eea-jenkins-token', variable: 'GITHUB_TOKEN'), string(credentialsId: 'ied-backend-trigger', variable: 'TRIGGER_MAIN_URL'), usernamePassword(credentialsId: 'jekinsdockerhub', usernameVariable: 'DOCKERHUB_USER', passwordVariable: 'DOCKERHUB_PASS')]) {
withCredentials([string(credentialsId: 'eea-jenkins-token', variable: 'GITHUB_TOKEN'), string(credentialsId: 'ied-backend-trigger', variable: 'TRIGGER_MAIN_URL'), usernamePassword(credentialsId: 'jekinsdockerhub', usernameVariable: 'DOCKERHUB_USER', passwordVariable: 'DOCKERHUB_PASS')]) { //betterleaks:allow
sh '''docker pull eeacms/gitflow; docker run -i --rm --name="$BUILD_TAG" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_NAME="$GIT_NAME" -e DOCKERHUB_REPO="eeacms/ied-backend" -e GIT_TOKEN="$GITHUB_TOKEN" -e DOCKERHUB_USER="$DOCKERHUB_USER" -e DOCKERHUB_PASS="$DOCKERHUB_PASS" -e TRIGGER_MAIN_URL="$TRIGGER_MAIN_URL" -e DEPENDENT_DOCKERFILE_URL="" -e GITFLOW_BEHAVIOR="RUN_ON_TAG" eeacms/gitflow'''
}

Expand All @@ -59,7 +59,7 @@ pipeline {
}
steps {
node(label: 'docker') {
withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'Rancher_dev_token', usernameVariable: 'RANCHER_ACCESS', passwordVariable: 'RANCHER_SECRET'],string(credentialsId: 'Rancher_dev_url', variable: 'RANCHER_URL')]) {
withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'Rancher_dev_token', usernameVariable: 'RANCHER_ACCESS', passwordVariable: 'RANCHER_SECRET'],string(credentialsId: 'Rancher_dev_url', variable: 'RANCHER_URL')]) { //betterleaks:allow
sh '''wget -O rancher_upgrade.sh https://raw.githubusercontent.com/eea/eea.docker.gitflow/master/src/rancher_upgrade.sh'''
sh '''chmod 755 rancher_upgrade.sh'''
sh '''./rancher_upgrade.sh'''
Expand Down
59 changes: 59 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,65 @@ See [develop](https://github.com/eea/ied-backend/tree/master/develop)

See [release](https://github.com/eea/ied-backend/tree/master/RELEASE.md)

## Secret Scanning

This repository uses the Betterleaks GitHub Action to scan the current
repository content on every push and pull request. The scan uses the rules in
`.gitleaks.toml` and uploads a `betterleaks-report` artifact when a finding is
detected.

If the optional SMTP secrets are configured, failed scans also send an email to
the last commit committer. The workflow expects these repository or
organization secrets:

- `SMTP_URL`
- `SMTP_PORT` (optional, defaults to `25`)
- `SMTP_EMAIL`
- `SMTP_PASSWORD` (optional if the SMTP server does not require authentication)

Port `465` is sent with direct TLS; other ports use the default SMTP handshake.
The email includes a short finding summary from the redacted Betterleaks report,
including the redacted matched line from each finding.

There are three common outcomes:

1. **Everything is OK.** The `Betterleaks / Scan for secrets` check is green and
no action is needed. Regular references to runtime values are OK, for example:

```js
const tokenFromCookie = req.universalCookies.get('auth_token');
```

2. **A real secret was found.** The check is red and the workflow log asks you to
download the `betterleaks-report` artifact. Open the artifact from the GitHub
Actions run and check the reported file, line and rule. Remove the committed
value, move it to the proper secret store, and rotate it if it was exposed.
A report entry looks like this:

```json
{
"RuleID": "secret-literal-assignment",
"File": "src/config.js",
"StartLine": 12,
"Secret": "[REDACTED]"
}
```

3. **The finding is a false positive.** Keep the value only if it is clearly not
sensitive, such as a test fixture, placeholder, or public example. Add
`betterleaks:allow` on the same line and include a short explanation in the
pull request.

```js
const testPassword = 'admin'; //betterleaks:allow
```

```yaml
password: "admin" #betterleaks:allow
```

Do not add `betterleaks:allow` to real credentials.

## Copyright and license

The Initial Owner of the Original Code is European Environment Agency (EEA).
Expand Down
4 changes: 2 additions & 2 deletions develop/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ services:
POSTGRES_DBPASS: zope
POSTGRES_DBUSER: zope
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_PASSWORD: postgres #betterleaks:allow
TZ: Europe/Bucharest
volumes:
- postgres_dump:/postgresql.backup
Expand All @@ -37,7 +37,7 @@ services:
- postgres
environment:
PGADMIN_DEFAULT_EMAIL: "alin@eaudeweb.ro"
PGADMIN_DEFAULT_PASSWORD: "alin"
PGADMIN_DEFAULT_PASSWORD: "alin" #betterleaks:allow

mailtrap:
image: eaudeweb/mailtrap
Expand Down
2 changes: 1 addition & 1 deletion test/tests/no-hard-coded-passwords/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
set -e

IFS=$'\n'
userPasswds=( $(docker run --rm --user 0:0 --entrypoint cut "$1" -d: -f1-2 /etc/passwd) )
userPasswds=( $(docker run --rm --user 0:0 --entrypoint cut "$1" -d: -f1-2 /etc/passwd) ) #betterleaks:allow
userShadows=()
if grep -qE ':x$' <<<"${userPasswds[*]}"; then
userShadows=( $(docker run --rm --user 0:0 --entrypoint cut "$1" -d: -f1-2 /etc/shadow || true) )
Expand Down
4 changes: 2 additions & 2 deletions test/tests/plone-relstorage/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,11 @@ PLONE_TEST_TRIES=10
# Start Postgres
zname="relstorage-container-$RANDOM-$RANDOM"
zpull="$(docker pull postgres:12-alpine)"
zid="$(docker run -d --name "$zname" -e POSTGRES_USER=plone -e POSTGRES_PASSWORD=plone -e POSTGRES_DB=plone postgres:12-alpine)"
zid="$(docker run -d --name "$zname" -e POSTGRES_USER=plone -e POSTGRES_PASSWORD=plone -e POSTGRES_DB=plone postgres:12-alpine)" #betterleaks:allow

# Start Plone as RelStorage Client
pname="plone-container-$RANDOM-$RANDOM"
pid="$(docker run -d --name "$pname" --link=$zname:db -e RELSTORAGE_DSN="dbname='plone' user='plone' host='db' password='plone'" "$image")"
pid="$(docker run -d --name "$pname" --link=$zname:db -e RELSTORAGE_DSN="dbname='plone' user='plone' host='db' password='plone'" "$image")" #betterleaks:allow

# Tear down
trap "docker rm -vf $pid $zid > /dev/null" EXIT
Expand Down
Loading