ROCm · prasanna-amd · Aug 5, 2026 · Aug 6, 2026 · Aug 11, 2026 · Aug 18, 2026
@@ -28,6 +28,10 @@ on:
       notify_webhook:
         type: string
         default: ''
+      madengine_nodes:
+        description: 'Number of SLURM nodes for MADEngine workloads'
+        type: string
+        default: '2'
 
 permissions:
   contents: read
@@ -208,3 +212,23 @@ jobs:
       artifact_run_id: ${{ inputs.artifact_run_id || github.run_id }}
       notify_email: ${{ inputs.notify_email }}
       notify_webhook: ${{ inputs.notify_webhook }}
+
+  therock-test-madengine:
+    name: "Test MADEngine workloads (scheduled)"
+    if: ${{ !cancelled() && inputs.amdgpu_families == 'gfx950-dcgpu' && (inputs.event_name == 'schedule' || inputs.event_name == 'workflow_dispatch') }}
+    needs: [therock-build-linux]
+    permissions:
+      contents: read
+      id-token: write
+      packages: write
+    uses: ./.github/workflows/therock-rccl-test-madengine.yml
+    secrets: inherit
+    with:
+      amdgpu_families: ${{ inputs.amdgpu_families }}
+      artifact_group: ${{ inputs.artifact_group }}
+      test_runs_on: ruby-linux-slurm-scale-runner
+      artifact_run_id: ${{ inputs.artifact_run_id || github.run_id }}
+      workload: llama-3.1-70b-training
+      nodes: ${{ inputs.madengine_nodes || '2' }}
+      notify_email: ${{ inputs.notify_email }}
+      teams_webhook: ${{ inputs.notify_webhook }}
@@ -2,7 +2,7 @@ name: TheRock CI for rccl
 
 on:
   schedule:
-    - cron: '17 6 * * 0'  # Weekly, Sunday 06:17 UTC
+    - cron: '17 6 * * *'  # Nightly, 06:17 UTC
   workflow_dispatch:
     inputs:
       artifact_run_id:
@@ -21,6 +21,10 @@ on:
         description: 'Teams webhook URL for test summary notifications'
         type: string
         default: ''
+      madengine_nodes:
+        description: 'Number of SLURM nodes for MADEngine workloads'
+        type: string
+        default: '2'
 
 permissions:
   contents: read
@@ -89,6 +93,7 @@ jobs:
       test_scope: ${{ inputs.test_scope || 'smoke' }}
       notify_email: ${{ inputs.notify_email || 'collectives-ci@amd.com' }}
       notify_webhook: ${{ inputs.notify_webhook }}
+      madengine_nodes: ${{ inputs.madengine_nodes || '2' }}
       cmake_options: >
         -DTHEROCK_ENABLE_ALL=OFF
         -DTHEROCK_BUILD_TESTING=ON

@@ -0,0 +1,187 @@
+name: TheRock Test MADEngine Workloads for rccl
+
+on:
+  workflow_call:
+    inputs:
+      amdgpu_families:
+        type: string
+      artifact_group:
+        type: string
+      test_runs_on:
+        type: string
+      artifact_run_id:
+        type: string
+      workload:
+        type: string
+        default: 'llama-3.1-70b-training'
+      nodes:
+        type: string
+        default: '2'
+      notify_email:
+        type: string
+        default: ''
+      teams_webhook:
+        type: string
+        default: ''
+  workflow_dispatch:
+    inputs:
+      amdgpu_families:
+        type: string
+      artifact_group:
+        type: string
+      test_runs_on:
+        type: string
+        default: 'ruby-linux-slurm-scale-runner'
+      artifact_run_id:
+        description: 'Reuse artifacts from a previous build run ID'
+        type: string
+      workload:
+        description: 'MADEngine workload to run'
+        type: string
+        default: 'llama-3.1-70b-training'
+      nodes:
+        description: 'Number of SLURM nodes to allocate'
+        type: string
+        default: '2'
+      notify_email:
+        description: 'Email address to send test summary report'
+        type: string
+        default: ''
+      teams_webhook:
+        description: 'Teams webhook URL for notifications'
+        type: string
+        default: ''
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  test_madengine:
+    name: 'Test MADEngine ${{ inputs.workload }} (${{ inputs.nodes }}N)'
+    runs-on: ${{ inputs.test_runs_on }}
+    defaults:
+      run:
+        shell: bash
+    env:
+      ARTIFACT_RUN_ID: "${{ inputs.artifact_run_id }}"
+      RESULTS_DIR: "/apps/rccl-ci/perf"
+      WORK_DIR: "/apps/rccl-ci/workdir/${{ github.run_id }}"
+    steps:
+      - name: Checkout TheRock repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          repository: "ROCm/TheRock"
+          ref: 2e7c190b99caa226a8644eda6eca720b3db102d5 # 2026-07-22
+
+      - name: Checkout rocm-systems scripts
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          sparse-checkout: projects/rccl/ci/scripts
+          path: rocm-systems
+
+      - name: Set arch-specific paths
+        run: |
+          case "${{ inputs.amdgpu_families }}" in
+            gfx950-dcgpu)
+              echo "OUTPUT_ARTIFACTS_DIR=/apps/cvs_tests/dist_new/dist/rocm" >> $GITHUB_ENV
+              echo "AWS_SHARED_CREDENTIALS_FILE=/apps/cvs_tests/awsconfig/credentials.ini" >> $GITHUB_ENV
+              echo "CLUSTER=ruby" >> $GITHUB_ENV
+              # Use /apps NFS for workdir — /home has root_squash which blocks Docker container writes
+              echo "WORK_DIR=/apps/rccl-ci/workdir/${{ github.run_id }}" >> $GITHUB_ENV
+              ;;
+            *)
+              echo "Unsupported arch for MADEngine: ${{ inputs.amdgpu_families }}"
+              exit 1
+              ;;
+          esac
+
+      - name: Resolve TheRock artifact run
+        if: ${{ inputs.artifact_run_id == '' }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          pip3 install -r requirements.txt
+          run_id="$(python3 -c "
+          import sys, contextlib
+          sys.path.insert(0, 'build_tools')
+          from find_latest_artifacts import find_latest_artifacts
+          with contextlib.redirect_stdout(sys.stderr):
+              results = find_latest_artifacts(
+                  artifact_groups=['${{ inputs.artifact_group }}'],
+                  verbose=True,
+              )
+          if not results:
+              print('No artifacts found for ${{ inputs.artifact_group }}', file=sys.stderr)
+              sys.exit(1)
+          print(results[0].workflow_run_id)
+          ")"
+          echo "Resolved TheRock run_id=${run_id}"
+          echo "ARTIFACT_RUN_ID=${run_id}" >> $GITHUB_ENV
+
+      - name: Fetch RCCL artifacts
+        run: |
+          pip3 install -r requirements.txt
+          python3 build_tools/install_rocm_from_artifacts.py \
+            --run-id="${ARTIFACT_RUN_ID}" \
+            --artifact-group="${{ inputs.artifact_group }}" \
+            --output-dir="${OUTPUT_ARTIFACTS_DIR}" \
+            --rccl
+
+      - name: Setup Python environment
+        run: |
+          mkdir -p "${WORK_DIR}"
+          python3 -m venv "${WORK_DIR}/venv"
+          source "${WORK_DIR}/venv/bin/activate"
+          pip install --upgrade pip
+
+      - name: Login to container registry
+        run: |
+          echo "${{ github.token }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
+
+      - name: Run MADEngine workload
+        timeout-minutes: 210
+        env:
+          NCCL_DEBUG: WARN
+          MAD_SECRETS_HFTOKEN: ${{ secrets.HF_TOKEN || '' }}
+        run: |
+          source "${WORK_DIR}/venv/bin/activate"
+          export PYTHONPATH="rocm-systems/projects/rccl/ci/scripts:${PYTHONPATH}"
+
+          NOTIFY_ARGS=""
+          if [ -n "${{ inputs.notify_email }}" ]; then
+            NOTIFY_ARGS="$NOTIFY_ARGS --notify-email ${{ inputs.notify_email }}"
+          fi
+          if [ -n "${{ inputs.teams_webhook }}" ]; then
+            NOTIFY_ARGS="$NOTIFY_ARGS --teams-webhook ${{ inputs.teams_webhook }}"
+          fi
+
+          python3 rocm-systems/projects/rccl/ci/scripts/test_madengine.py \
+            --artifact-dir "${OUTPUT_ARTIFACTS_DIR}" \
+            --workload "${{ inputs.workload }}" \
+            --cluster "${CLUSTER}" \
+            --nodes "${{ inputs.nodes }}" \
+            --results-dir "${RESULTS_DIR}" \
+            --work-dir "${WORK_DIR}" \
+            --registry "ghcr.io/rocm/rocm-systems" \
+            $NOTIFY_ARGS
+
+      - name: Upload results
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: madengine-results-${{ inputs.workload }}-${{ inputs.nodes }}N-${{ inputs.amdgpu_families }}
+          path: |
+            ${{ env.WORK_DIR }}/perf.csv
+            ${{ env.WORK_DIR }}/perf_entry_super.csv
+            ${{ env.WORK_DIR }}/perf_entry_super.json
+            ${{ env.WORK_DIR }}/perf_super.csv
+            ${{ env.WORK_DIR }}/madengine_summary.txt
+            ${{ env.WORK_DIR }}/manifest.json
+          retention-days: 30
+
+      - name: Cleanup work directory
+        if: always()
+        run: |
+          rm -rf "${WORK_DIR}/madengine" "${WORK_DIR}/MAD" "${WORK_DIR}/venv" "${WORK_DIR}/rccl_libs"