Moving build system to pixi

.devcontainer/devcontainer.json

@@ -0,0 +1,26 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+{
+  "name": "ODM",
+  "build": {
+    "context": "..",
+    "dockerfile": "../Dockerfile",
+    "target": "dev"
+  },
+  "features": {
+    // https://github.com/devcontainers/features/blob/main/src/common-utils/main.sh
+    // Add `odm` user for consistency with the GPU image
+    "ghcr.io/devcontainers/features/common-utils:2": {
+      "installZsh": "true",
+      "username": "odm",
+      "upgradePackages": "true"
+    }
+  },
+  "remoteUser": "odm",
+  "postCreateCommand": "pixi install --locked",
+  // Use the `code` directory for compatibility with direct Docker workflows
+  "workspaceMount": "source=${localWorkspaceFolder},target=/code,type=bind",
+  "workspaceFolder": "/code",
+  "mounts": [
+    "source=${localEnv:HOME}${localEnv:USERPROFILE},target=/home/odm,type=bind,consistency=cached"
+  ]
+}

.devcontainer/gpu/devcontainer.json

@@ -0,0 +1,29 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+{
+  "name": "ODM GPU",
+  "build": {
+    "context": "../..",
+    "dockerfile": "../../gpu.Dockerfile",
+    "target": "dev"
+  },
+  "features": {
+    // https://github.com/devcontainers/features/blob/main/src/common-utils/main.sh
+    // Add `odm` user for consistency with the CPU image
+    "ghcr.io/devcontainers/features/common-utils:2": {
+      "installZsh": "true",
+      "username": "odm",
+      "upgradePackages": "true"
+    }
+  },
+  "remoteUser": "odm",
+  "postCreateCommand": "pixi install --locked -e gpu",
+  // Use the `code` directory for compatibility with direct Docker workflows
+  "workspaceMount": "source=${localWorkspaceFolder},target=/code,type=bind",
+  "workspaceFolder": "/code",
+  "mounts": [
+    "source=${localEnv:HOME}${localEnv:USERPROFILE},target=/home/odm,type=bind,consistency=cached"
+  ],
+  "hostRequirements": {
+    "gpu": "true"
+  }
+}

.dockerignore

@@ -1,23 +1,22 @@
-Dockerfile
-SuperBuild/build
-SuperBuild/download
-SuperBuild/install
-SuperBuild/src
-build
-opensfm
-odm_orthophoto
-odm_texturing
-odm_meshing
-odm_georeferencing
-images_resize
-.git
-
-/contrib
-/docs
-/hooks
-/img
-/license
-settings.yml
-code_of_conduct.md
-configure_18_04.sh
-index.html
+SuperBuild/build
+SuperBuild/download
+SuperBuild/install
+SuperBuild/src
+build
+opensfm
+odm_orthophoto
+odm_texturing
+odm_meshing
+odm_georeferencing
+images_resize
+.git
+venv
+.pixi
+scripts/pixi-shell-hook
+/docs
+/hooks
+/img
+/license
+settings.yml
+code_of_conduct.md
+index.html

.gitattributes

@@ -0,0 +1,2 @@
+# SCM syntax highlighting & preventing 3-way merges
+pixi.lock merge=binary linguist-language=YAML linguist-generated=true -diff

.github/workflows/publish-docker-gpu.yaml

@@ -4,22 +4,33 @@ on:
   push:
     branches:
     - master
-    tags:
-    - v*
+  workflow_dispatch:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     timeout-minutes: 2880
     steps:
     - name: Checkout
       uses: actions/checkout@v2
     - name: Set up QEMU
       uses: docker/setup-qemu-action@v1
+    - name: Free space before build
+      run: |
+        docker builder prune -af
+        docker image prune -f
+        echo "--- free before build"
+        df -h /
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v1
+      with:
+        # One worker at a time, as the builder and prod-env stages each carry a
+        # multi-GB conda environment and would otherwise be built together.
+        config-inline: |
+          [worker.oci]
+            max-parallelism = 1
     - name: Login to DockerHub
-      uses: docker/login-action@v1 
+      uses: docker/login-action@v1
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -33,16 +44,23 @@ jobs:
       uses: docker/build-push-action@v2
       with:
         file: ./gpu.Dockerfile
+        target: runtime
         platforms: linux/amd64
         push: true
         no-cache: true
         build-args: |
           GIT_COMMIT=${{ github.sha }}
           BUILD_DATE=${{ steps.buildmeta.outputs.date }}
           IMAGE_VERSION=${{ steps.buildmeta.outputs.version }}
-        tags: opendronemap/odm:gpu
-    # Trigger NodeODM build
-    - name: Dispatch NodeODM Build Event
-      id: nodeodm_dispatch
+        tags: opendronemap/odm:edge-gpu
+    - name: Report and reclaim space
+      # The runner is persistent and shared, so leave nothing behind: the image
+      # is in the registry by this point.
+      if: always()
       run: |
-        curl -X POST -u "${{secrets.PAT_USERNAME}}:${{secrets.PAT_TOKEN}}" -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" https://api.github.com/repos/OpenDroneMap/NodeODM/actions/workflows/publish-docker-gpu.yaml/dispatches --data '{"ref": "master"}'
+        echo "--- free after build (difference from before is what it used)"
+        df -h /
+        docker builder prune -af
+        docker image prune -f
+        echo "--- free after reclaim"
+        df -h /

.github/workflows/publish-docker.yaml

@@ -4,8 +4,7 @@ on:
   push:
     branches:
     - master
-    tags:
-    - v*
+  workflow_dispatch:
 
 jobs:
   build:
@@ -23,18 +22,10 @@ jobs:
           [worker.oci]
             max-parallelism = 1
     - name: Login to DockerHub
-      uses: docker/login-action@v1 
+      uses: docker/login-action@v1
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
-    # Use the repository information of the checked-out code to format docker tags
-    - name: Docker meta
-      id: docker_meta
-      uses: crazy-max/ghaction-docker-meta@v1
-      with:
-        images: opendronemap/odm
-        tag-semver: |
-          {{version}}
     - name: Compute build metadata
       id: buildmeta
       run: |
@@ -44,19 +35,22 @@ jobs:
       id: docker_build
       uses: docker/build-push-action@v2
       with:
-        file: ./portable.Dockerfile
+        file: ./Dockerfile
+        target: runtime
         platforms: linux/amd64,linux/arm64
         push: true
-        no-cache: true
+        cache-from: type=registry,ref=opendronemap/odm:buildcache
+        cache-to: type=registry,ref=opendronemap/odm:buildcache,mode=max
         build-args: |
           GIT_COMMIT=${{ github.sha }}
           BUILD_DATE=${{ steps.buildmeta.outputs.date }}
           IMAGE_VERSION=${{ steps.buildmeta.outputs.version }}
-        tags: |
-          ${{ steps.docker_meta.outputs.tags }}
-          opendronemap/odm:latest
-    # Trigger NodeODM build
-    - name: Dispatch NodeODM Build Event
-      id: nodeodm_dispatch
+        tags: opendronemap/odm:edge
+    - name: Reclaim space
+      # The runner is persistent and shared, so leave nothing behind: the image
+      # is in the registry by this point.
+      if: always()
       run: |
-        curl -X POST -u "${{secrets.PAT_USERNAME}}:${{secrets.PAT_TOKEN}}" -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" https://api.github.com/repos/OpenDroneMap/NodeODM/actions/workflows/publish-docker.yaml/dispatches --data '{"ref": "master"}'
+        docker builder prune -af
+        docker image prune -f
+        df -h /

.github/workflows/publish-windows.yml

@@ -6,6 +6,7 @@ on:
     - master
     tags:
     - v*
+  workflow_dispatch:
 
 permissions:
   contents: write
@@ -21,33 +22,27 @@ jobs:
   build:
     runs-on: windows-2022
     environment: azuresigning
+    # The runner has no NVIDIA driver, so pixi cannot detect the __cuda virtual
+    # package the gpu environments require. The installer's users supply the
+    # driver; mock it here to match the cuda floor in pixi.toml.
+    env:
+      CONDA_OVERRIDE_CUDA: "12.0"
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
-    - name: Setup Python
-      uses: actions/setup-python@v2
+      uses: actions/checkout@v6
+    - name: Setup pixi
+      uses: prefix-dev/setup-pixi@v0.9.3
       with:
-        python-version: '3.12.9'
-        architecture: 'x64'
-    - uses: Jimver/cuda-toolkit@v0.2.24
-      id: cuda-toolkit
-      with:
-        cuda: '12.8.1'
-    - name: Setup cmake
-      uses: jwlawson/actions-setup-cmake@v1.13
-      with:
-        cmake-version: '3.24.x'
-    - name: Install venv
-      run: |
-        python -m pip install virtualenv
-    - name: Build sources
-      run: |
-        python configure.py build
+        pixi-version: latest
+        locked: true
+        # The build env (gpu) and the shipped runtime env (gpu-prod) are both needed.
+        environments: >-
+          gpu
+          gpu-prod
+    - name: Build SuperBuild (with CUDA)
+      run: pixi run -e gpu build
     - name: Free up space
-      run: |
-        rmdir SuperBuild\download /s /q
-        rmdir SuperBuild\build /s /q
-      shell: cmd
+      run: cmd /c "rd /s /q SuperBuild\build 2>nul & rd /s /q SuperBuild\src 2>nul & exit /b 0"
     - name: Retrieve the metadata and decode it to a file
       env:
         AZURESIGNING_METADATA: ${{ secrets.AZURESIGNING_METADATA }}
@@ -61,15 +56,25 @@ jobs:
         tenant-id: ${{ secrets.AZURE_TENANT_ID }}
         subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
     - name: Create setup
-      run: |
-        python configure.py dist --signtool-path "C:\Program Files (x86)\Windows Kits\10\bin\10.0.26100.0\x64\signtool.exe" --azure-signing-metadata "%RUNNER_TEMP%\metadata.json"
+      run: pixi run -e gpu dist -- --signtool-path "C:\Program Files (x86)\Windows Kits\10\bin\10.0.26100.0\x64\signtool.exe" --azure-signing-metadata "%RUNNER_TEMP%\metadata.json"
       shell: cmd
     - name: Upload Setup File
       if: always()
       uses: actions/upload-artifact@v4
       with:
         name: Setup
         path: dist\*.exe
+    # Silent-install to a path different from the build dir and run the CLI in a
+    # shell with no inherited CONDA_PREFIX. This is the real test that the bundled
+    # conda env relocates: it fails here if activate.d doesn't honor %CONDA_PREFIX%.
+    - name: Verify relocated install
+      shell: cmd
+      run: |
+        for %%f in (dist\ODM_Setup_*.exe) do start /wait "" "%%f" /SILENT /SUPPRESSMSGBOXES /DIR=C:\ODMtest /LOG=C:\install.log
+        if not exist C:\ODMtest\run.py exit /b 1
+        cd /d C:\ODMtest
+        call win32env.bat
+        python -X utf8 run.py --help
     - name: Upload Setup to Release
       uses: svenstaro/upload-release-action@v2
       if: startsWith(github.ref, 'refs/tags/')
@@ -79,4 +84,3 @@ jobs:
         file_glob: true
         tag: ${{ github.ref }}
         overwrite: true
-