Skip to content

Fix build issues and parallelize build process #75

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
TSRBerry merged 5 commits into from
Apr 29, 2025
Merged

Fix build issues and parallelize build process #75

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
298880a
Partially revert #71
TSRBerry Apr 28, 2025
5a8ad75
Force build older cmake projects
TSRBerry Apr 28, 2025
70dd7d8
Pin git commits
TSRBerry Apr 28, 2025
4be129c
Build images in parallel
TSRBerry Apr 28, 2025
81d4689
Move labels to the top
TSRBerry Apr 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
115 changes: 61 additions & 54 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,23 @@ FROM archlinux:base-devel AS base

ARG MAKE_JOBS=1

# Pinned commit hashes and tags
ARG BUILDSCRIPTS_HASH=d707f1e4f987c6fdb5af05c557e26c1cc868f734
ARG GENERALTOOLS_HASH=46086605cdc63fb02ba0ed08cdc00801ba00c6f0
ARG SWITCHTOOLS_HASH=fb3204d69c51c44c3bba67027ee7c8295fd4f985
ARG LIBNX32_HASH=be0f3aade5d3a6fd67c70a8e16a1f7dc8ab2cd30
ARG DEKOTOOLS_HASH=aebf6e299383668ff1d337ad6cb3daca0d4c3754
ARG DEKO3D_HASH=9900322a40957fa47bed764b20ec00cb4e870f66
ARG UAM_HASH=97177458b362e6ed8848c8db0db2c31c58234df2
ARG SPIRV_CROSS_VER=sdk-1.3.261.1
ARG FMTLIB_VER=10.1.1
ARG GLSLANG_VER=sdk-1.3.261.1
ARG MINIZ_VER=3.0.2

# Use labels to make images easier to organize
LABEL libnx32.version="${LIBNX32_HASH}"
LABEL buildscripts.version="${BUILDSCRIPTS_HASH}"

# Prepare devkitpro env
ENV DEVKITPRO=/opt/devkitpro
ENV DEVKITARM=/opt/devkitpro/devkitARM
Expand Down Expand Up @@ -33,7 +50,11 @@ RUN echo "export DEVKITPRO=${DEVKITPRO}" > /etc/profile.d/devkit-env.sh \
&& echo "export PATH=${DEVKITPRO}/tools/bin:$PATH" >> /etc/profile.d/devkit-env.sh

# Create devkitpro dir
RUN mkdir -p -m 0755 ${DEVKITPRO}
RUN mkdir -p -m 0775 ${DEVKITPRO} && chown -R vita2hos:vita2hos ${DEVKITPRO}

# Copy devkitPro cmake files from the official docker images
COPY --from=devkitpro/devkitarm --chown=vita2hos:vita2hos ${DEVKITPRO}/cmake ${DEVKITPRO}/cmake
COPY --from=devkitpro/devkita64 --chown=vita2hos:vita2hos ${DEVKITPRO}/cmake ${DEVKITPRO}/cmake

# ------- Information about apt packages --------
# Mako: (python3, python3-pip, python3-setuptools)
Expand All @@ -48,7 +69,7 @@ RUN mkdir -p -m 0755 ${DEVKITPRO}
# dkARM_rules: (wget, tar, gzip), build-essential
# dkARM_crt0: (wget, tar, gzip), build-essential
# dkARM_gdb (py3): (git), python3-dev, build-essential, texinfo
# libnx (xerpi): (git), devkitARM, build-essential
# libnx (xerpi): (git), devkitARM, dkp_general-tools, build-essential
# switch-tools (xerpi): (git), libnx, autotools-dev, automake, autoconf, build-essential, liblz4-dev, libelf-dev
# dekotools: (git), meson, ninja-build
# deko3d (xerpi): (git), dekotools, build-essential
Expand All @@ -69,98 +90,80 @@ RUN pacman -Syu --needed --noconfirm \
libmpc libtool automake autoconf lz4 libelf xz bzip2 \
&& pacman -Scc --noconfirm

# Add devkitPro pacman repository
RUN echo "[dkp-libs]" >> /etc/pacman.conf && \
echo "Server = https://pkg.devkitpro.org/packages" >> /etc/pacman.conf && \
echo "[dkp-linux]" >> /etc/pacman.conf && \
echo "Server = https://pkg.devkitpro.org/packages/linux/\$arch/" >> /etc/pacman.conf

# Import and sign the devkitPro GPG key, install devkitPro keyring package and populate the keyring
RUN pacman-key --init && \
pacman-key --recv BC26F752D25B92CE272E0F44F7FD5492264BB9D0 --keyserver keyserver.ubuntu.com && \
pacman-key --lsign BC26F752D25B92CE272E0F44F7FD5492264BB9D0 && \
pacman -U --noconfirm https://pkg.devkitpro.org/devkitpro-keyring.pkg.tar.zst && \
pacman-key --populate devkitpro && \
pacman -Syu

# Install devkitPro's general-tools and switch-cmake
RUN pacman -Sy --noconfirm general-tools dkp-cmake-common-utils devkitarm-cmake switch-cmake

# Change ownership of the devkitPro directory
RUN chown -R vita2hos:vita2hos ${DEVKITPRO}

FROM base AS prepare

# Download public key for github.com
RUN mkdir -p -m 0700 ~/.ssh && ssh-keyscan github.com >> ~/.ssh/known_hosts

RUN ls -la /home

# Switch to vita2hos user
USER vita2hos
WORKDIR /home/vita2hos

FROM prepare AS buildscripts

# Run devkitPro's buildscripts to install GCC, binutils and newlib (1 = devkitARM)
ARG BUILDSCRIPTS_HASH=d707f1e4f987c6fdb5af05c557e26c1cc868f734
RUN git clone https://github.com/xerpi/buildscripts.git \
&& cd buildscripts && git checkout ${BUILDSCRIPTS_HASH} \
&& MAKEFLAGS="-j ${MAKE_JOBS}" BUILD_DKPRO_AUTOMATED=1 BUILD_DKPRO_PACKAGE=1 ./build-devkit.sh

FROM buildscripts AS switch-tools

# Clone switch-tools fork and install it
RUN git clone https://github.com/xerpi/switch-tools.git --branch arm-32-bit-support
RUN git clone https://github.com/xerpi/switch-tools.git \
&& cd switch-tools && git checkout ${SWITCHTOOLS_HASH}
RUN cd switch-tools && ./autogen.sh \
&& ./configure --prefix=${DEVKITPRO}/tools/ \
&& ./configure --prefix=${DEVKITPRO}/tools \
&& make -j $MAKE_JOBS install

FROM switch-tools AS general-tools

# Clone devkitPro's general-tools and install it
RUN git clone https://github.com/devkitPro/general-tools.git \
&& cd general-tools && git checkout ${GENERALTOOLS_HASH} \
&& ./autogen.sh \
&& ./configure --prefix=${DEVKITPRO}/tools \
&& make -j $MAKE_JOBS install

FROM switch-tools AS libnx
FROM general-tools AS libnx

# Clone libnx fork and install it
ARG LIBNX32_HASH=be0f3aade5d3a6fd67c70a8e16a1f7dc8ab2cd30
RUN git clone https://github.com/xerpi/libnx.git
RUN cd libnx && git checkout ${LIBNX32_HASH} \
&& make -j $MAKE_JOBS -C nx/ -f Makefile.32 install
RUN git clone https://github.com/xerpi/libnx.git \
&& cd libnx && git checkout ${LIBNX32_HASH}
RUN cd libnx && make -j $MAKE_JOBS -C nx/ -f Makefile.32 install

FROM libnx AS dekotools

# Clone and install dekotools
RUN git clone https://github.com/fincs/dekotools
RUN git clone https://github.com/fincs/dekotools \
&& cd dekotools && git checkout ${DEKOTOOLS_HASH}
RUN cd dekotools && meson build --prefix $DEVKITPRO/tools
RUN cd dekotools/build && ninja install -j $MAKE_JOBS

FROM dekotools AS deko3d

# Clone deko3d fork and install it
ARG DEKO3D_HASH=9900322a40957fa47bed764b20ec00cb4e870f66
RUN git clone https://github.com/xerpi/deko3d.git
RUN cd deko3d && git checkout ${DEKO3D_HASH} \
&& make -f Makefile.32 -j $MAKE_JOBS install

FROM deko3d AS portlibs-prepare
FROM libnx AS portlibs-prepare

# Prepare portlibs
ARG SPIRV_CROSS_VER=sdk-1.3.261.1
ARG FMTLIB_VER=10.1.1
ARG GLSLANG_VER=sdk-1.3.261.1
ARG MINIZ_VER=3.0.2
RUN git clone https://github.com/KhronosGroup/SPIRV-Cross \
&& cd SPIRV-Cross && git checkout tags/${SPIRV_CROSS_VER} -b ${SPIRV_CROSS_VER} && cd .. \
&& git clone https://github.com/fmtlib/fmt \
&& cd fmt && git checkout tags/${FMTLIB_VER} -b ${FMTLIB_VER} && cd .. \
&& git clone https://github.com/KhronosGroup/glslang \
&& cd glslang && git checkout tags/${GLSLANG_VER} -b ${GLSLANG_VER} && cd .. \
&& git clone https://github.com/xerpi/uam --branch switch-32 \
&& git clone https://github.com/richgel999/miniz.git --branch ${MINIZ_VER}
&& git clone https://github.com/xerpi/uam \
&& cd uam && git checkout ${UAM_HASH} && cd .. \
&& git clone https://github.com/richgel999/miniz \
&& cd miniz && git checkout tags/${MINIZ_VER} -b ${MINIZ_VER} && cd ..

FROM portlibs-prepare AS spirv

# Build and install SPIRV-Cross
RUN cd SPIRV-Cross \
&& mkdir build && cd build \
&& cmake .. \
-DCMAKE_POLICY_VERSION_MINIMUM=3.5 \
-DCMAKE_TOOLCHAIN_FILE=${DEVKITPRO}/cmake/devkitARM.cmake \
-DCMAKE_PROJECT_TOP_LEVEL_INCLUDES=${DEVKITPRO}/cmake/Platform/NintendoSwitch.cmake \
-DCMAKE_EXE_LINKER_FLAGS="-specs=${NX_ROOT}/switch32.specs" \
Expand All @@ -173,7 +176,7 @@ RUN cd SPIRV-Cross \
-DSPIRV_CROSS_CLI:BOOL=OFF \
&& cmake --build . --target install --parallel $MAKE_JOBS

FROM spirv AS fmt
FROM portlibs-prepare AS fmt

# Build and install fmt
RUN cd fmt \
Expand All @@ -187,7 +190,7 @@ RUN cd fmt \
-DFMT_TEST:BOOL=OFF \
&& cmake --build . --target install --parallel $MAKE_JOBS

FROM fmt AS glslang
FROM portlibs-prepare AS glslang

# Build and install glslang
RUN cd glslang \
Expand All @@ -204,7 +207,7 @@ RUN cd glslang \
-DENABLE_SPVREMAPPER:BOOL=OFF \
&& cmake --build . --target install --parallel $MAKE_JOBS

FROM glslang AS uam-host
FROM portlibs-prepare AS uam-host

# Build and install uam as a host executable
RUN cd uam \
Expand All @@ -213,7 +216,7 @@ RUN cd uam \
build_host
RUN cd uam/build_host && ninja -j $MAKE_JOBS install

FROM uam-host AS uam-switch
FROM portlibs-prepare AS uam-switch

# Add meson cross file for uam
COPY cross_file_switch32.txt cross_file_switch32.txt
Expand All @@ -226,12 +229,13 @@ RUN cd uam \
build
RUN cd uam/build && ninja -j $MAKE_JOBS install

FROM uam-switch AS miniz
FROM portlibs-prepare AS miniz

# Build and install miniz
RUN cd miniz \
&& mkdir build && cd build \
&& cmake .. \
-DCMAKE_POLICY_VERSION_MINIMUM=3.5 \
-DCMAKE_TOOLCHAIN_FILE=${DEVKITPRO}/cmake/devkitARM.cmake \
-DCMAKE_PROJECT_TOP_LEVEL_INCLUDES=${DEVKITPRO}/cmake/Platform/NintendoSwitch.cmake \
-DCMAKE_EXE_LINKER_FLAGS="-specs=${NX_ROOT}/switch32.specs" \
Expand All @@ -243,12 +247,15 @@ RUN cd miniz \

FROM base AS final

# Copy the entire $DEVKITPRO directory from the last build stage
# Copy the entire $DEVKITPRO directory from build stages
COPY --from=libnx --chown=vita2hos:vita2hos $DEVKITPRO $DEVKITPRO
COPY --from=deko3d --chown=vita2hos:vita2hos $DEVKITPRO $DEVKITPRO
COPY --from=spirv --chown=vita2hos:vita2hos $DEVKITPRO $DEVKITPRO
COPY --from=glslang --chown=vita2hos:vita2hos $DEVKITPRO $DEVKITPRO
COPY --from=fmt --chown=vita2hos:vita2hos $DEVKITPRO $DEVKITPRO
COPY --from=uam-host --chown=vita2hos:vita2hos $DEVKITPRO $DEVKITPRO
COPY --from=uam-switch --chown=vita2hos:vita2hos $DEVKITPRO $DEVKITPRO
COPY --from=miniz --chown=vita2hos:vita2hos $DEVKITPRO $DEVKITPRO

# Use labels to make images easier to organize
LABEL libnx32.version="${LIBNX32_HASH}"
LABEL buildscripts.version="${BUILDSCRIPTS_HASH}"

USER vita2hos
WORKDIR /home/vita2hos