Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 32 additions & 3 deletions .github/actions/composite/setupNode/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ inputs:
description: 'Indicates if node is set up for hybrid app'
required: false
default: 'false'
WORKSPACES:
description: 'JSON array of npm workspace names or paths to install alongside the root project. Defaults to root dependencies only.'
required: false
default: '[]'
SEED_STICKY_DISKS:
description: |
'true' only in the seedCache workflow, which installs dependencies and nothing else.
Expand Down Expand Up @@ -37,11 +41,21 @@ runs:
shell: bash
run: echo "use-sticky-disk=${{ runner.os == 'Linux' && contains(runner.name, 'blacksmith') }}" >> "$GITHUB_OUTPUT"

- id: workspace-config
shell: bash
env:
WORKSPACES: ${{ inputs.WORKSPACES }}
run: |
workspaces=$(jq -ce 'if type == "array" and all(.[]; type == "string" and length > 0) then sort | unique else error("WORKSPACES must be an array of nonempty workspace names") end' <<< "$WORKSPACES")
echo "workspaces=$workspaces" >> "$GITHUB_OUTPUT"
# Equivalent selections must share a cache even if callers list them in a different order.
echo "cache-key=$(printf '%s' "$workspaces" | shasum -a 256 | cut -d ' ' -f 1)" >> "$GITHUB_OUTPUT"

- id: stickydisk-node-modules
if: steps.runner-info.outputs.use-sticky-disk == 'true'
uses: useblacksmith/stickydisk@74f3f01ab1392726dd6ee06904f0452b0ec1e151 # v1.6.0
with:
key: ${{ format('{0}-{1}-{2}-node-modules-{3}', github.repository, runner.os, runner.arch, inputs.IS_HYBRID_BUILD == 'true' && hashFiles('normalized-package-lock.json', 'patches/**', 'Mobile-Expensify/patches/**') || hashFiles('normalized-package-lock.json', 'patches/**')) }}
key: ${{ format('{0}-{1}-{2}-node-modules-workspaces-{3}-{4}', github.repository, runner.os, runner.arch, steps.workspace-config.outputs.cache-key, inputs.IS_HYBRID_BUILD == 'true' && hashFiles('normalized-package-lock.json', 'patches/**', 'Mobile-Expensify/patches/**') || hashFiles('normalized-package-lock.json', 'patches/**')) }}
path: node_modules
commit: ${{ inputs.SEED_STICKY_DISKS == 'true' && 'if-missing' || 'false' }}

Expand All @@ -67,7 +81,7 @@ runs:
uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb
with:
path: node_modules
key: ${{ format('{0}-node-modules-{1}', runner.os, inputs.IS_HYBRID_BUILD == 'true' && hashFiles('package-lock.json', 'patches/**', 'Mobile-Expensify/patches/**') || hashFiles('package-lock.json', 'patches/**')) }}
key: ${{ format('{0}-node-modules-workspaces-{1}-{2}', runner.os, steps.workspace-config.outputs.cache-key, inputs.IS_HYBRID_BUILD == 'true' && hashFiles('package-lock.json', 'patches/**', 'Mobile-Expensify/patches/**') || hashFiles('package-lock.json', 'patches/**')) }}

- id: cache-old-dot-node-modules
if: steps.runner-info.outputs.use-sticky-disk != 'true' && inputs.IS_HYBRID_BUILD == 'true'
Expand Down Expand Up @@ -123,10 +137,25 @@ runs:
- name: Install root project node packages
if: steps.check-node-modules.outputs.cache-hit != 'true' || (inputs.IS_HYBRID_BUILD == 'true' && steps.check-old-dot-node-modules.outputs.cache-hit != 'true')
uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4
env:
WORKSPACES: ${{ steps.workspace-config.outputs.workspaces }}
with:
timeout_minutes: 30
max_attempts: 3
command: npm ci
command: |
args=(--workspaces=false)
if [[ "$WORKSPACES" != '[]' ]]; then
# Selecting workspaces excludes the root project unless it is explicitly included.
args=(--include-workspace-root)
while IFS= read -r workspace; do
args+=(--workspace="$workspace")
done < <(jq -r '.[]' <<< "$WORKSPACES")
fi
npm ci "${args[@]}"

- name: Add project binaries to PATH
shell: bash
run: echo "$PWD/node_modules/.bin" >> "$GITHUB_PATH"

- name: Mark sticky disks as installed
if: steps.runner-info.outputs.use-sticky-disk == 'true'
Expand Down
4 changes: 1 addition & 3 deletions .github/workflows/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,9 +96,7 @@ The GitHub workflows require a large list of secrets to deploy, notify and test
1. `APPLE_ID` - Required for notarizing desktop code in `desktop/notarize.js`
1. `APPLE_TEAM_ID` - Required for notarizing desktop code in `desktop/notarize.js`
1. `APPLE_ID_PASSWORD` - Required for notarizing desktop code in `desktop/notarize.js`
1. `AWS_ACCESS_KEY_ID` - Required for hosting website and desktop compiled code
1. `AWS_SECRET_ACCESS_KEY` - Required for hosting website and desktop compiled code
1. `CLOUDFLARE_TOKEN` - Required for hosting website
1. `CLOUDFLARE_TOKEN` - Purges the help-site Cloudflare cache
1. `APPLE_CONTACT_EMAIL` - Email used for contact between Expensify and Apple for https://appstoreconnect.apple.com/
1. `APPLE_CONTACT_PHONE` - Phone number used for contact between Expensify and Apple for https://appstoreconnect.apple.com/
1. `APPLE_DEMO_EMAIL` - Demo account email used for https://appstoreconnect.apple.com/
Expand Down
73 changes: 38 additions & 35 deletions .github/workflows/buildAdHoc.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ on:
type: string
default: 'false'

concurrency:
group: adhoc-${{ github.event_name }}-${{ format('app-{0}-mobile-{1}', inputs.APP_PR_NUMBER || inputs.APP_REF, inputs.MOBILE_EXPENSIFY_PR || inputs.MOBILE_EXPENSIFY_REF) }}

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

don't we need platform in this key as well?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could add it but I don't think we need to. I was thinking we only manage concurrency on a per-PR basis, rather than per-platform-per-PR. Basically, if you start a web-only AdHoc build on a PR, then before it finishes start an all-platforms AdHoc build, it's expected that the latter would interrupt the former.

cancel-in-progress: ${{ github.event_name != 'push' }}

jobs:
postGitHubCommentBuildStarted:
name: Post build started comment
Expand Down Expand Up @@ -105,49 +109,47 @@ jobs:
ref: ${{ inputs.APP_REF }}
environment: adhoc
pull-request-number: ${{ inputs.APP_PR_NUMBER }}
build-preview-worker: true
secrets: inherit

deployWebAdHoc:
name: Deploy Web to S3 (adhoc)
name: Deploy the PR Worker Preview and web assets
needs: [buildWeb]
if: ${{ fromJSON(inputs.BUILD_WEB) && inputs.APP_PR_NUMBER != '' }}
runs-on: blacksmith-4vcpu-ubuntu-2404
runs-on: blacksmith-2vcpu-ubuntu-2404
Comment on lines -114 to +118

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was it intentional to switch to smaller vm here?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah all this job does is download some shit and upload some shit. So it can use the smallest runner

env:
WRANGLER_SEND_METRICS: false
steps:
- name: Download web build artifact
# v7
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131
- name: Checkout trusted preview tooling, not the PR branch
uses: useblacksmith/checkout@0647fdbab2614a5eb86d2971070e325060d91056 # v1.7.0
with:
name: web-build-tar-gz-artifact
path: ./

- name: Extract web build
run: tar -xzvf "${{ needs.buildWeb.outputs.TAR_FILENAME }}"
ref: ${{ github.workflow_sha }}
persist-credentials: false

- name: Configure AWS Credentials
# v6
uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7
- name: Setup Node
uses: ./.github/actions/composite/setupNode
with:
role-to-assume: arn:aws:iam::949552126497:role/app-adhoc-web-deploy-gh-actions
role-session-name: gh-${{ github.run_id }}-${{ github.run_attempt }}
aws-region: us-east-1
WORKSPACES: '["web/cloudflare-worker"]'

- name: Deploy to S3
run: |
BUCKET="s3://ad-hoc-expensify-cash/web/${{ inputs.APP_PR_NUMBER }}"
- name: Download the matching web assets
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
with:
name: web-build-tar-gz-artifact
path: ${{ runner.temp }}/adhoc-artifacts/assets

aws s3 sync dist "$BUCKET" --delete --acl public-read \
--exclude "*.br" \
--exclude "index.html" \
--exclude "service-worker.js" \
--exclude "workbox-*.js" \
--cache-control "public,max-age=31536000,immutable"
- name: Download the branch Worker bundle as data
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
with:
name: adhoc-worker-artifact
path: ${{ runner.temp }}/adhoc-artifacts/worker

aws s3 sync dist "$BUCKET" --acl public-read \
--exclude "*" \
--include "index.html" \
--include "service-worker.js" \
--include "workbox-*.js" \
--cache-control "no-cache,no-store,must-revalidate"
- name: Deploy and verify the PR Preview and assets together
run: |
bun scripts/deployAdhocCloudflareWorker.ts \
--pr-number ${{ inputs.APP_PR_NUMBER }} --sha ${{ needs.buildWeb.outputs.BUILD_SHA }} \
--artifact-directory "${{ runner.temp }}/adhoc-artifacts"
env:
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_WORKERS_DEPLOY_TOKEN }}
CLOUDFLARE_ACCOUNT_ID: 3eb3df6f8c190fa13c3fc0a53c30af1a

postGithubComment:
runs-on: blacksmith-4vcpu-ubuntu-2404
Expand Down Expand Up @@ -175,7 +177,7 @@ jobs:
WEB: ${{ needs.buildWeb.result == 'failure' && 'failure' || needs.deployWebAdHoc.result }}
ANDROID_LINK: ${{ needs.buildAndroid.outputs.ROCK_ARTIFACT_URL }}
IOS_LINK: ${{ needs.buildIOS.outputs.ROCK_ARTIFACT_URL }}
WEB_LINK: https://${{ inputs.APP_PR_NUMBER }}.pr-testing.expensify.com
WEB_LINK: https://${{ inputs.APP_PR_NUMBER }}.adhoc.expensify.com

- name: Publish links to apps for download on Expensify/Mobile-Expensify PR
if: ${{ inputs.MOBILE_EXPENSIFY_PR != '' }}
Expand Down Expand Up @@ -219,12 +221,13 @@ jobs:
github-token: ${{ github.token }}
script: |
const prNumber = '${{ inputs.APP_PR_NUMBER }}';
const webLink = prNumber && '${{ needs.deployWebAdHoc.result }}' === 'success' ? `https://${prNumber}.pr-testing.expensify.com` : '';
const webResult = '${{ needs.deployWebAdHoc.result }}';
const webLink = prNumber && webResult === 'success' ? `https://${prNumber}.adhoc.expensify.com` : '';
const androidLink = '${{ needs.buildAndroid.outputs.ROCK_ARTIFACT_URL }}' || '';
const iosLink = '${{ needs.buildIOS.outputs.ROCK_ARTIFACT_URL }}' || '';
const mobileExpensifySHA = '${{ steps.getMobileExpensifySHA.outputs.SHA }}' || '';

const webFailed = '${{ needs.buildWeb.result }}' === 'failure' || '${{ needs.deployWebAdHoc.result }}' === 'failure';
const webFailed = '${{ needs.buildWeb.result }}' === 'failure' || webResult === 'failure';
const webStatus = webLink ? '✅ Success' : webFailed ? '❌ Failed' : '⏭️ Skipped';
const androidStatus = androidLink ? '✅ Success' : '${{ needs.buildAndroid.result }}' === 'failure' ? '❌ Failed' : '⏭️ Skipped';
const iosStatus = iosLink ? '✅ Success' : '${{ needs.buildIOS.result }}' === 'failure' ? '❌ Failed' : '⏭️ Skipped';
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/buildVictoryChartRenderer.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ jobs:

- name: Setup Node
uses: ./.github/actions/composite/setupNode
with:
WORKSPACES: '["server/victory-chart-renderer"]'

- name: Build victory-chart-renderer Linux x64 binary
run: npm run server:vcr:build:linux
Expand Down
41 changes: 41 additions & 0 deletions .github/workflows/buildWeb.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,15 @@ on:
description: Pull request number (used for adhoc builds)
type: string
default: ''
build-preview-worker:
description: Bundle the branch Worker for a per-PR deployment (after the existing reviewed-code gate)
type: boolean
default: false

outputs:
BUILD_SHA:
description: Immutable App commit used for the web build and Worker bundle
value: ${{ jobs.build.outputs.BUILD_SHA }}
TAR_FILENAME:
description: Filename of the .tar.gz web build artifact
value: ${{ jobs.build.outputs.TAR_FILENAME }}
Expand All @@ -32,13 +39,15 @@ jobs:
name: Build Web
runs-on: blacksmith-8vcpu-ubuntu-2404
outputs:
BUILD_SHA: ${{ steps.checkout.outputs.commit }}
TAR_FILENAME: webBuild.tar.gz
ZIP_FILENAME: webBuild.zip
SOURCEMAP_FILENAME: merged-source-map.js.map
env:
PULL_REQUEST_NUMBER: ${{ inputs.pull-request-number }}
steps:
- name: Checkout
id: checkout
uses: useblacksmith/checkout@0647fdbab2614a5eb86d2971070e325060d91056 # v1.7.0
with:
ref: ${{ inputs.ref }}
Expand All @@ -55,6 +64,8 @@ jobs:

- name: Setup Node
uses: ./.github/actions/composite/setupNode
with:
WORKSPACES: ${{ inputs.build-preview-worker && '["web/cloudflare-worker"]' || '[]' }}

- name: Build web
run: |
Expand All @@ -68,6 +79,36 @@ jobs:
env:
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}

# No Cloudflare credentials are available to branch dependencies, build hooks or the bundler.
- name: Bundle the branch Worker for this web build
if: ${{ inputs.build-preview-worker }}
working-directory: web/cloudflare-worker
run: |
if ! node -e 'const fs = require("node:fs"); const expected = JSON.parse(fs.readFileSync("package.json", "utf8")).devDependencies.wrangler; process.exit(require("wrangler/package.json").version === expected ? 0 : 1)' 2>/dev/null; then
npm ci --workspaces=false
fi
npx wrangler deploy --dry-run --env staging --outdir ../../adhoc-worker
node --input-type=module <<'NODE'
import {writeFileSync} from 'node:fs';
import {experimental_readRawConfig} from 'wrangler';
const {rawConfig} = experimental_readRawConfig({config: 'wrangler.jsonc'});
const runtime = {...rawConfig, ...rawConfig.env?.staging};
writeFileSync('../../adhoc-worker/runtime.json', JSON.stringify({
compatibility_date: runtime.compatibility_date,
compatibility_flags: runtime.compatibility_flags ?? [],
}));
NODE
env:
WRANGLER_SEND_METRICS: false

- name: Upload the branch Worker bundle
if: ${{ inputs.build-preview-worker }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: adhoc-worker-artifact
path: ./adhoc-worker
if-no-files-found: error

- name: Upload web sourcemaps artifact
# v6
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/bunTests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ jobs:

- name: Setup Node
uses: ./.github/actions/composite/setupNode
with:
WORKSPACES: '["server/victory-chart-renderer", "web/cloudflare-worker"]'

- name: Run Bun tests
run: npm run test:bun
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/claude-review.yml
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,8 @@ jobs:
- name: Setup Node
if: steps.skip.outputs.skip != 'true' && (steps.filter.outputs.code == 'true' || steps.filter.outputs.docs == 'true')
uses: ./.github/actions/composite/setupNode
with:
WORKSPACES: '["server/victory-chart-renderer", "web/cloudflare-worker"]'

- name: Run Claude Code (code)
id: code-review
Expand Down
12 changes: 11 additions & 1 deletion .github/workflows/cloudflareWorkerTests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,11 @@ on:
- 'web/index.html'
- '.nvmrc'
- '.github/workflows/cloudflareWorkerTests.yml'
- '.github/workflows/buildWeb.yml'
- '.github/workflows/buildAdHoc.yml'
- 'scripts/deployAdhocCloudflareWorker.ts'
- 'scripts/extractAdhocAssets.ts'
- '.bun-version'

concurrency:
group: ${{ github.ref }}-cloudflare-worker-tests
Expand Down Expand Up @@ -41,7 +46,12 @@ jobs:
cache-dependency-path: web/cloudflare-worker/package-lock.json

- name: Install dependencies
run: npm ci
run: npm ci --workspaces=false

- name: Setup Bun for preview archive integration tests
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version-file: '.bun-version'

- name: Type check
run: npm run typecheck
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -762,7 +762,7 @@ jobs:

- name: Install Cloudflare Worker dependencies
working-directory: web/cloudflare-worker
run: npm ci
run: npm ci --workspaces=false

- name: Deploy to Cloudflare Workers
working-directory: web/cloudflare-worker
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/deployExpensifyHelp.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ jobs:

- name: Setup NodeJS
uses: ./.github/actions/composite/setupNode
with:
WORKSPACES: '["web/cloudflare-worker"]'

- name: Create docs routes file
run: ./.github/scripts/createDocsRoutes.sh
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/knip.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ jobs:

- name: Setup Node
uses: ./.github/actions/composite/setupNode
with:
WORKSPACES: '["server/victory-chart-renderer", "web/cloudflare-worker"]'

- name: Run knip on PR
run: KNIP=true npx knip --reporter json --no-exit-code > /tmp/knip-pr.json
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ jobs:

- name: Setup Node
uses: ./.github/actions/composite/setupNode
with:
WORKSPACES: '["server/victory-chart-renderer", "web/cloudflare-worker"]'

# oxlint runs JS plugins on one thread (oxc#26621), so OxlintLinter splits them into 4 shards on this 8 vCPU runner.
- name: Lint JavaScript and TypeScript with Oxlint
Expand Down
10 changes: 9 additions & 1 deletion .github/workflows/seedCache.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,15 @@ concurrency:

jobs:
seedStandalone:
name: Seed standalone node_modules
name: Seed standalone node_modules (${{ matrix.workspaces }})
runs-on: blacksmith-2vcpu-ubuntu-2404
strategy:
matrix:
workspaces:
- '[]'
- '["web/cloudflare-worker"]'
- '["server/victory-chart-renderer"]'
- '["server/victory-chart-renderer", "web/cloudflare-worker"]'
steps:
- name: Checkout
uses: useblacksmith/checkout@0647fdbab2614a5eb86d2971070e325060d91056 # v1.7.0
Expand All @@ -21,6 +28,7 @@ jobs:
uses: ./.github/actions/composite/setupNode
with:
SEED_STICKY_DISKS: 'true'
WORKSPACES: ${{ matrix.workspaces }}

seedHybrid:
name: Seed hybrid node_modules
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/typecheck.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ jobs:
- uses: useblacksmith/checkout@0647fdbab2614a5eb86d2971070e325060d91056 # v1.7.0

- uses: ./.github/actions/composite/setupNode
with:
WORKSPACES: '["server/victory-chart-renderer", "web/cloudflare-worker"]'

- name: Type check with TypeScript
run: npm run typecheck
Expand Down
Loading
Loading