diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index dd8c5eb7f3..0885ffda8a 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -3,7 +3,19 @@ 🚨 Please review the [guidelines for contributing](https://github.com/mendersoftware/mender/blob/master/CONTRIBUTING.md) to this repository. -- [ ] Make sure that all commits have a [`Changelog`](https://github.com/mendersoftware/mender/blob/master/CONTRIBUTING.md#changelog-tags) tag. If nothing should be added to the Changelog, add a `Changelog: None` tag. If there is a change, add `Changelog: Commit`, or `Changelog: Title`, depending on what should be included in the changelog. +- [ ] Make sure that all commits follow the conventional commit [specification](https://www.github.com/mendersoftware/mendertesting/commitlint/grammar.md) for the Mender project. + +The majority of our contributions are fixes, which means your commit should have +the form below: + +``` +fix: + + + +Changelog: or +Ticket: or +``` - [ ] Make sure that all commits are signed with [`git --signoff`](https://git-scm.com/book/en/v2/Git-Tools-Signing-Your-Work). Also note that the signoff author must match the author of the commit. diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4694569188..59190f7cf3 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,13 +1,13 @@ version: 2 updates: - commit-message: - prefix: "Changelog:All" + prefix: "chore" directory: / package-ecosystem: gitsubmodule schedule: interval: weekly - commit-message: - prefix: 'Changelog:All' + prefix: 'chore' directory: /scripts/linkbot/ package-ecosystem: npm schedule: diff --git a/.github/workflows/automatically-update-test-links.yml b/.github/workflows/automatically-update-test-links.yml index d30630b560..88a4f57f27 100644 --- a/.github/workflows/automatically-update-test-links.yml +++ b/.github/workflows/automatically-update-test-links.yml @@ -17,6 +17,7 @@ jobs: node monitor-bb-sd.js node monitor-rpi.js node monitor-ub-server.js + node monitor-raspbian-os.js working-directory: scripts/linkbot - name: Create Pull Request @@ -27,9 +28,9 @@ jobs: body: | New images are available for update commit-message: | - Image-Bot - Image updates + chore: Image-Bot image updates - Changelog: None + Base image updates for our provided images. committer: mender-test-bot author: mender-test-bot signoff: true diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 67a457dc87..8060515cef 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,16 +2,28 @@ variables: DOCKER_REPOSITORY: mendersoftware/mender-convert S3_BUCKET_NAME: mender-convert-images + # These variables are present elsewhere in the repository too. Make sure to # search for and change them too. MENDER_ARTIFACT_VERSION: master MENDER_CLIENT_VERSION: latest MENDER_ADDON_CONNECT_VERSION: latest MENDER_ADDON_CONFIGURE_VERSION: latest - # Make sure to update the link in mender-docs to the new one when changing - # this. - RASPBIAN_URL: https://downloads.raspberrypi.org/raspios_lite_armhf/images/raspios_lite_armhf-2021-01-12/2021-01-11-raspios-buster-armhf-lite.zip - RASPBIAN_NAME: 2021-01-11-raspios-buster-armhf-lite + + ## Auto-update + RASPBIAN_URL: "https://downloads.raspberrypi.org/raspios_lite_armhf/images/raspios_lite_armhf-2022-01-28/2022-01-28-raspios-bullseye-armhf-lite.zip" + RASPBIAN_NAME: 2022-01-28-raspios-bullseye-armhf-lite + + # Which version of mender-convert to use in published image name. Normally + # empty, in which case the branch or tag name will be used, but this variable + # is available so that it's possible to make small build fixes on an already + # tagged version. + MENDER_CONVERT_PUBLISH_VERSION: "" + + # Whether to run acceptance tests + TEST_MENDER_CONVERT: "true" + # Whether to publish packages automatically - they can always be published manually + PUBLISH_MENDER_CONVERT_AUTOMATIC: "false" DEBIAN_FRONTEND: noninteractive @@ -38,9 +50,29 @@ stages: - test - build - convert + - flash - test_acceptance - publish +test:check-commits: + only: + variables: + - '$TEST_MENDER_CONVERT == "true"' + +test:check-license: + rules: + - if: '$TEST_MENDER_CONVERT == "true"' + +test:check-shell-formatting: + rules: + - if: '$TEST_MENDER_CONVERT == "true"' + before_script: + - SHELL_SCRIPTS=$(find mender-convert* modules configs scripts + -type f + -not -name "*.md" + -not -path "scripts/linkbot/*" + -not -wholename "scripts/test/*") + build: stage: build needs: [] @@ -55,14 +87,6 @@ build: paths: - image.tar -test:check-shell-formatting: - before_script: - - SHELL_SCRIPTS=$(find mender-convert* modules configs scripts - -type f - -not -name "*.md" - -not -path "scripts/linkbot/*" - -not -wholename "scripts/test/*") - .template_convert_raspbian: &convert_raspbian stage: convert needs: @@ -74,31 +98,34 @@ test:check-shell-formatting: tags: - mender-qa-slave before_script: - - export AWS_ACCESS_KEY_ID=$TMP_STORAGE_AWS_ACCESS_KEY_ID - - export AWS_SECRET_ACCESS_KEY=$TMP_STORAGE_AWS_SECRET_ACCESS_KEY - - - apt update && apt install -yy bash wget unzip awscli docker.io + - apt update && apt install -yy bash wget unzip awscli docker.io curl - export IMAGE_NAME=$DOCKER_REPOSITORY:pr - docker load -i image.tar + - mkdir -p input + - cd input - wget -q ${RASPBIAN_URL} - unzip ${RASPBIAN_NAME}.zip + - cd .. + + - eval "$(curl https://raw.githubusercontent.com/mendersoftware/mendertesting/master/mender-ci-common.sh)" script: - - echo "MENDER_CLIENT_VERSION=${MENDER_CLIENT_VERSION}" > versions_override_config - - echo "MENDER_ADDON_CONNECT_VERSION=${MENDER_ADDON_CONNECT_VERSION}" >> versions_override_config - - echo "MENDER_ADDON_CONFIGURE_VERSION=${MENDER_ADDON_CONFIGURE_VERSION}" >> versions_override_config - - env MENDER_ARTIFACT_NAME=${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION} - ./docker-mender-convert -d ${RASPBIAN_NAME}.img - -c configs/${RASPBERRYPI_PLATFORM}_config - -c configs/images/raspberrypi_raspbian_config - -c versions_override_config + - env MENDER_ARTIFACT_NAME=${RASPBIAN_NAME}-mender + ./docker-mender-convert --disk-image input/${RASPBIAN_NAME}.img + --config configs/${RASPBERRYPI_PLATFORM}_config + --config configs/images/raspberrypi_raspbian_config - # Upload to temporary S3 bucket + # Collect artifacts. - mv deploy ${RASPBERRYPI_PLATFORM} - tar czf ${RASPBERRYPI_PLATFORM}.tar.gz ${RASPBERRYPI_PLATFORM} - - aws s3 cp ${RASPBERRYPI_PLATFORM}.tar.gz s3://mender-gitlab-tmp-storage/$CI_PROJECT_NAME/$CI_PIPELINE_ID/${RASPBERRYPI_PLATFORM}.tar.gz + # Upload to temporary S3 bucket + - mender_ci_save_tmp_artifact ${RASPBERRYPI_PLATFORM}.tar.gz + + artifacts: + paths: + - checksums convert_raspbian_raspberrypi3: <<: *convert_raspbian @@ -122,13 +149,11 @@ convert_raspbian_raspberrypi4: - mender-qa-slave timeout: 2h before_script: - - export AWS_ACCESS_KEY_ID=$TMP_STORAGE_AWS_ACCESS_KEY_ID - - export AWS_SECRET_ACCESS_KEY=$TMP_STORAGE_AWS_SECRET_ACCESS_KEY # Install dependencies - apt update - apt install -qyy bash wget git util-linux mtools python3 python3-pip gcc python3-dev libffi-dev liblzo2-dev libc-dev libssl-dev make sudo - awscli unzip qemu-system-x86 ovmf curl docker.io + awscli unzip qemu-system-x86 ovmf curl docker.io s3cmd # Python3 dependencies - python3 -m pip install -U pip - pip3 install --ignore-installed -r https://raw.githubusercontent.com/mendersoftware/meta-mender/master/tests/acceptance/requirements_py3.txt @@ -141,6 +166,9 @@ convert_raspbian_raspberrypi4: - curl -f -O https://mender.s3.amazonaws.com/mender-artifact/$MENDER_ARTIFACT_VERSION/linux/mender-artifact - chmod ugo+x mender-artifact - mv mender-artifact /usr/bin/ + - echo "MENDER_CLIENT_VERSION=${MENDER_CLIENT_VERSION}" > versions_override_config + - echo "MENDER_ADDON_CONNECT_VERSION=${MENDER_ADDON_CONNECT_VERSION}" >> versions_override_config + - echo "MENDER_ADDON_CONFIGURE_VERSION=${MENDER_ADDON_CONFIGURE_VERSION}" >> versions_override_config # Get Update Modules Artifact generators for the tests - if [ "${MENDER_CLIENT_VERSION}" = "latest" ]; then - mender_version=master @@ -161,12 +189,15 @@ convert_raspbian_raspberrypi4: - report_*.html reports: junit: results_*.xml + rules: + - if: '$TEST_MENDER_CONVERT == "true"' .template_test_acceptance_prebuilt_raspberrypi: &test_acceptance_prebuilt_raspberrypi <<: *test_acceptance script: + - eval "$(curl https://raw.githubusercontent.com/mendersoftware/mendertesting/master/mender-ci-common.sh)" # Fetch artifacts from temporary S3 bucket - - aws s3 cp s3://mender-gitlab-tmp-storage/$CI_PROJECT_NAME/$CI_PIPELINE_ID/${RASPBERRYPI_PLATFORM}.tar.gz ${RASPBERRYPI_PLATFORM}.tar.gz + - mender_ci_load_tmp_artifact ${RASPBERRYPI_PLATFORM}.tar.gz - tar xzf ${RASPBERRYPI_PLATFORM}.tar.gz - mv ${RASPBERRYPI_PLATFORM} deploy # Extract converted Raspbian artifacts @@ -196,52 +227,61 @@ test_acceptance_prebuilt_raspberrypi4: test_acceptance_qemux86_64: <<: *test_acceptance script: - - ./scripts/test/run-tests.sh --only qemux86_64 + - mkdir -p input/config + - cp versions_override_config input/config/versions_override_config + - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only ubuntu-qemux86-64 + +test_acceptance_debian_qemux86_64: + <<: *test_acceptance + script: + - mkdir -p input/config + - cp versions_override_config input/config/versions_override_config + - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only debian-qemux86-64 test_acceptance_raspberrypi: <<: *test_acceptance script: - - ./scripts/test/run-tests.sh --only raspberrypi3 + - mkdir -p input/config + - cp versions_override_config input/config/versions_override_config + - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only raspberrypi3 test_acceptance_beaglebone: <<: *test_acceptance script: - - ./scripts/test/run-tests.sh --only beaglebone + - mkdir -p input/config + - cp versions_override_config input/config/versions_override_config + - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only beaglebone test_acceptance_ubuntu: <<: *test_acceptance script: - - ./scripts/test/run-tests.sh --only ubuntu + - mkdir -p input/config + - cp versions_override_config input/config/versions_override_config + - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only ubuntu -publish:s3: - when: manual +.template:publish:s3: stage: publish image: debian:buster before_script: - - export PUBLISH_AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID - - export PUBLISH_AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY - - export AWS_ACCESS_KEY_ID=$TMP_STORAGE_AWS_ACCESS_KEY_ID - - export AWS_SECRET_ACCESS_KEY=$TMP_STORAGE_AWS_SECRET_ACCESS_KEY - - apt update && apt install -yyq awscli + - apt update && apt install -yyq awscli curl + - eval "$(curl https://raw.githubusercontent.com/mendersoftware/mendertesting/master/mender-ci-common.sh)" # Fetch artifacts from temporary S3 bucket - for RASPBERRYPI_PLATFORM in raspberrypi3 raspberrypi4; do - - aws s3 cp s3://mender-gitlab-tmp-storage/$CI_PROJECT_NAME/$CI_PIPELINE_ID/${RASPBERRYPI_PLATFORM}.tar.gz ${RASPBERRYPI_PLATFORM}.tar.gz + - mender_ci_load_tmp_artifact ${RASPBERRYPI_PLATFORM}.tar.gz - tar xzf ${RASPBERRYPI_PLATFORM}.tar.gz - done script: - # Prepare high privilege S3 keys (the TMP_STORAGE keys are for the tmp storage only) - - export AWS_ACCESS_KEY_ID=$PUBLISH_AWS_ACCESS_KEY_ID - - export AWS_SECRET_ACCESS_KEY=$PUBLISH_AWS_SECRET_ACCESS_KEY + - IMAGE_VERSION=${MENDER_CONVERT_PUBLISH_VERSION:-${CI_COMMIT_REF_NAME}} - for RASPBERRYPI_PLATFORM in raspberrypi3 raspberrypi4; do - - PUBLISH_NAME=${RASPBIAN_NAME}-${RASPBERRYPI_PLATFORM}-mender-${MENDER_CLIENT_VERSION}.img.xz + - PUBLISH_NAME=${RASPBIAN_NAME}-${RASPBERRYPI_PLATFORM}-mender-convert-${IMAGE_VERSION}.img.xz - echo "Publishing ${PUBLISH_NAME} version to S3" - aws s3 cp ${RASPBERRYPI_PLATFORM}/${RASPBIAN_NAME}-${RASPBERRYPI_PLATFORM}-mender.img.xz s3://$S3_BUCKET_NAME/${RASPBIAN_NAME}/arm/${PUBLISH_NAME} - aws s3api put-object-acl --acl public-read --bucket $S3_BUCKET_NAME --key ${RASPBIAN_NAME}/arm/${PUBLISH_NAME} - - PUBLISH_NAME=${RASPBIAN_NAME}-${RASPBERRYPI_PLATFORM}-mender-${MENDER_CLIENT_VERSION}.mender + - PUBLISH_NAME=${RASPBIAN_NAME}-${RASPBERRYPI_PLATFORM}-mender-convert-${IMAGE_VERSION}.mender - echo "Publishing ${PUBLISH_NAME} version to S3" - aws s3 cp ${RASPBERRYPI_PLATFORM}/${RASPBIAN_NAME}-${RASPBERRYPI_PLATFORM}-mender.mender s3://$S3_BUCKET_NAME/${RASPBIAN_NAME}/arm/${PUBLISH_NAME} @@ -249,5 +289,114 @@ publish:s3: --key ${RASPBIAN_NAME}/arm/${PUBLISH_NAME} - done - only: - - /^(master|[0-9]+\.[0-9]+\.x)$/ +.template:publish:docker-image: + stage: publish + tags: + - docker + image: docker + services: + - name: docker:20.10.8-dind + alias: docker + command: ["--mtu=1440"] # https://gitlab.com/gitlab-com/gl-infra/production/-/issues/5590 + before_script: + - docker login -u $DOCKER_HUB_USERNAME -p $DOCKER_HUB_PASSWORD + - docker load -i image.tar + script: + - IMAGE_VERSION=${MENDER_CONVERT_PUBLISH_VERSION:-${CI_COMMIT_REF_NAME}} + + - docker tag $DOCKER_REPOSITORY:pr $DOCKER_REPOSITORY:$IMAGE_VERSION + - docker push $DOCKER_REPOSITORY:$IMAGE_VERSION + - echo "PUBLISH_IMAGE_DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' $DOCKER_REPOSITORY:$IMAGE_VERSION)" >> publish.env + artifacts: + reports: + dotenv: publish.env + +publish:s3:manual: + when: manual + extends: .template:publish:s3 + +publish:s3:automatic: + rules: + - if: '$PUBLISH_MENDER_CONVERT_AUTOMATIC == "true"' + extends: .template:publish:s3 + +publish:docker-image:manual: + when: manual + extends: .template:publish:docker-image + +publish:docker-image:automatic: + rules: + - if: '$CI_COMMIT_BRANCH =~ /^(master|staging|production|feature-.+|[0-9]+\.[0-9]+\.([0-9]+|x))$/' + extends: .template:publish:docker-image + +#################### +# Hardware testing # +#################### + +test:hardware:convert: + <<: *convert_raspbian + variables: + RASPBERRYPI_PLATFORM: raspberrypi4 + when: manual + stage: convert + script: + # Convert the image for the hardware tester + - env MENDER_ARTIFACT_NAME=${RASPBIAN_NAME}-mender + ./docker-mender-convert --disk-image input/${RASPBIAN_NAME}.img + --config configs/${RASPBERRYPI_PLATFORM}_config + --config configs/images/raspberrypi_raspbian_config + --config configs/testing/hardware-testing + --overlay tests/ssh-public-key-overlay + # Collect artifacts. + - mv deploy ${RASPBERRYPI_PLATFORM} + - tar czf ${RASPBERRYPI_PLATFORM}.tar.gz ${RASPBERRYPI_PLATFORM} + # Upload to temporary S3 bucket + - mender_ci_save_tmp_artifact ${RASPBERRYPI_PLATFORM}.tar.gz + artifacts: + paths: + - checksums + +test:hardware:flash: + needs: + - job: test:hardware:convert + artifacts: true + variables: + RASPBERRYPI_PLATFORM: raspberrypi4 + tags: + - hw + stage: flash + script: + - eval "$(curl https://raw.githubusercontent.com/mendersoftware/mendertesting/master/mender-ci-common.sh)" + # Fetch artifacts from temporary S3 bucket + - mender_ci_load_tmp_artifact ${RASPBERRYPI_PLATFORM}.tar.gz + # Flash and boot the device under test + - slave-control mode flash + - tar -xvzf ${RASPBERRYPI_PLATFORM}.tar.gz --wildcards --no-anchored '*.img' --to-stdout | dd of=/dev/sdmux bs=8M + - slave-control mode test + +test:hardware:acceptance: + <<: *test_acceptance + variables: + RASPBERRYPI_PLATFORM: raspberrypi4 + stage: test_acceptance + needs: [] + needs: + - job: build + artifacts: true + - job: test:hardware:convert + artifacts: true + - job: test:hardware:flash + script: + # Fetch artifacts from temporary S3 bucket + - eval "$(curl https://raw.githubusercontent.com/mendersoftware/mendertesting/master/mender-ci-common.sh)" + - mender_ci_load_tmp_artifact ${RASPBERRYPI_PLATFORM}.tar.gz + # Extract the configuration + - tar xzf ${RASPBERRYPI_PLATFORM}.tar.gz + - mv ${RASPBERRYPI_PLATFORM} deploy + # Run the tests + - ./scripts/test/run-tests.sh + --prebuilt-image + raspberrypi4 + "${RASPBIAN_NAME}-${RASPBERRYPI_PLATFORM}-mender + --hardware-testing + --host ${SSH_JUMP_HOST_IP}:${SSH_JUMP_HOST_PORT} mender-image-tests -k 'not test_network_based_image_update and not test_image_update_broken_kernel'" diff --git a/Dockerfile b/Dockerfile index 7766d0a959..167d14e552 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ RUN cd /root/pxz && make FROM ubuntu:20.04 -ARG MENDER_ARTIFACT_VERSION=master +ARG MENDER_ARTIFACT_VERSION=3.7.x RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install -y \ # For 'ar' command to unpack .deb @@ -51,7 +51,11 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install -y \ zip \ unzip \ # manipulate binary and hex - xxd + xxd \ +# JSON power tool + jq \ +# GRUB command line tools, primarily grub-probe + grub-common COPY --from=build /root/pxz/pxz /usr/bin/pxz @@ -70,5 +74,15 @@ RUN wget -q -O /usr/bin/mender-artifact https://downloads.mender.io/mender-artif WORKDIR / -COPY docker-entrypoint.sh /usr/local/bin/ -ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] +COPY . /mender-convert + +RUN mkdir -p /mender-convert/work +RUN mkdir -p /mender-convert/input +RUN mkdir -p /mender-convert/deploy +RUN mkdir -p /mender-convert/logs + +VOLUME ["/mender-convert/input"] +VOLUME ["/mender-convert/deploy"] +VOLUME ["/mender-convert/logs"] + +ENTRYPOINT ["/mender-convert/docker-entrypoint.sh"] diff --git a/LICENSE b/LICENSE index 9f2ba3e1aa..c58aee758a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2021 Northern.tech AS +Copyright 2022 Northern.tech AS All content in this project is licensed under the Apache License v2, unless indicated otherwise. diff --git a/LIC_FILES_CHKSUM.sha256 b/LIC_FILES_CHKSUM.sha256 index ef08ffac52..0c4ef152b3 100644 --- a/LIC_FILES_CHKSUM.sha256 +++ b/LIC_FILES_CHKSUM.sha256 @@ -1 +1 @@ -b4acfcfa2a0ba1a8c82ec3965fbcee886cff8394ca4214e0ddac0a36beb1e05a LICENSE +1033348db7606a7e61b6484f293847cf8d7a35766efebb97e304d4bd5d7f3f6b LICENSE diff --git a/README.md b/README.md index 4a71ef5d94..27d8cf0141 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,19 @@ on your device. ![Mender logo](https://github.com/mendersoftware/mender/raw/master/mender_logo.png) +## Requisites + +`mender-convert` is supported on the following development platform(s): + + - Ubuntu 18.04 x86, 64bit + +Other platforms may work, but are not under active testing. Patches to add additional +platforms or fix compatibility issues are welcome. + +The storage used during the process, specifically the work directories (usually below the clone of +`mender-convert`) and the output directory should be located on local, Unix-style filesystem. +Using network storage or emulated filesystems might cause permission or owner issues. + ## Getting started @@ -31,9 +44,39 @@ For more detailed information about `mender-convert` please visit the [Debian family](https://docs.mender.io/artifacts/debian-family) section in [the Mender documentation](https://docs.mender.io/). +## Included configurations + +### Core configurations + +These configurations are officially supported. + +| Configuration | Supported OS / hardware | +| :------------ | :---------------------- | +| generic_x86-64_config | Generic x86, 64bit distribtion *use this as a starting point only!* | +| raspberrypi0w_config | RaspberryPi 0w, Raspbian 32bit | +| raspberrypi3_config | RaspberryPi 3, Raspbian 32bit | +| raspberrypi4_config | RaspberryPi 4, Raspbian 32bit | +| raspberrypi4_ubuntu_config | RaspberryPi 4, Ubuntu 32bit | + +### Contributed configurations + +These configurations have been submitted by community contributors. + +| Configuration | Supported OS / hardware | +| :------------ | :---------------------- | +| beaglebone_black_debian_emmc_config | BeagleBone Black 4GB, Debian 32bit on internal eMMC storage | +| beaglebone_black_debian_sdcard_config | BeagleBone Black, Debian 32bit on external SD card storage | +| comfile_pi_config | Comfile Pi, Raspbian 32bit | +| rockpro64_emmc_config | RockPro64, Debian 32bit on internal eMMC storage | +| rockpro64_sd_config | BeagleBone Black, Debian 32bit on external SD card storage | + +## Example usage: Raspberry Pi 3, Raspbian 32bit ### Prepare image and configuration +The following steps give a quick start using Raspbian. For a more detailed guide, +especially concerning version compatibilities, please visit [the the corresponding thread](https://hub.mender.io/t/raspberry-pi-3-model-b-b-raspbian/140) on the Mender Hub. + Download the raw Raspberry Pi disk image into a subdirectory input: ```bash @@ -109,15 +152,22 @@ necessary dependencies. Run mender-convert from inside the container with your desired options, e.g. ```bash +mkdir -p input/image +cp $PATH_TO_DISK_IMAGE/$INPUT_DISK_IMAGE input/image + +mkdir -p input/config +cp $PATH_TO_MY_OWN_CONFIG/$CUSTOM_CONFIG input/config + MENDER_ARTIFACT_NAME=release-1 ./docker-mender-convert \ - --disk-image input/$INPUT_DISK_IMAGE \ + --disk-image input/image/$INPUT_DISK_IMAGE \ --config configs/raspberrypi3_config \ + --config input/config/$CUSTOM_CONFIG \ --overlay rootfs_overlay_demo/ ``` Conversion will take 10-30 minutes, depending on image size and resources -available. You can watch `WORKDIR/convert.log` for progress and diagnostics -information. +available. You can watch `log/convert.log.XXXXX` for progress and diagnostics +information. The exact log file path is printed before the conversion starts. After it finishes, you can find your images in the `deploy` directory on your host machine! @@ -145,7 +195,7 @@ MENDER_ARTIFACT_NAME=release-1 ./mender-convert \ **NOTE!** You will be prompted to enter `sudo` password during the conversion process. This is required to be able to loopback mount images and for modifying them. Our recommendation is to use the provided Docker container, to run the -tool in a isolated environment. +tool in an isolated environment. ------------------------------------------------------------------------------- @@ -179,7 +229,7 @@ issue. We thank you in advance for your cooperation. * Fork us on [Github](https://github.com/mendersoftware) * Create an issue in the [bugtracker](https://tracker.mender.io/projects/MEN) * Email us at [contact@mender.io](mailto:contact@mender.io) -* Connect to the [#mender IRC channel on Freenode](http://webchat.freenode.net/?channels=mender) +* Connect to the [#mender IRC channel on Libera](https://web.libera.chat/?#mender) ## Authors diff --git a/configs/debian-qemux86-64_config b/configs/debian-qemux86-64_config new file mode 100644 index 0000000000..48c6a40323 --- /dev/null +++ b/configs/debian-qemux86-64_config @@ -0,0 +1,21 @@ +# This configuration can be used to run on a qemux86-64 machine. +# +# The image is generated with the following command: +# +# mender-convert/scripts/test/generate-image.sh debian +# +# Then manually uploaded to Mender's AWS S3 bucket for CI testing +# +# Locally, converted with the following command: +# +# MENDER_ARTIFACT_NAME=release-1 ./docker-mender-convert --disk-image input/Debian-11-x86-64.img --overlay rootfs_overlay_demo --config configs/qemux86-64_config +# +# and qemu is executed with the following command: +# +# qemu-system-x86_64 -enable-kvm -m 512 -smp 2 -bios /usr/share/OVMF/OVMF_CODE.fd -drive format=raw,file=deploy/Debian-11-x86-64-qemux86-64-mender.img + +MENDER_STORAGE_DEVICE_BASE=/dev/sda +MENDER_DEVICE_TYPE="qemux86-64" + +# Nothing to copy +MENDER_COPY_BOOT_GAP="n" diff --git a/configs/images/raspberrypi_raspbian_config b/configs/images/raspberrypi_raspbian_config index d0b4368dbf..4675902009 100644 --- a/configs/images/raspberrypi_raspbian_config +++ b/configs/images/raspberrypi_raspbian_config @@ -2,7 +2,7 @@ # often use a base of 1000 instead of 1024, and even then they are often smaller # than advertised. The number below is based on a conservative target of 7.9GB # (that's mathematical GB, base 1000), converted to MiB, rounding down. -MENDER_STORAGE_TOTAL_SIZE_MB=7534 +MENDER_STORAGE_TOTAL_SIZE_MB=30136 # Use all there is, which gets us almost, but not quite, to 500MiB free space # (about 480MiB at the time of writing). @@ -49,3 +49,18 @@ EOF } PLATFORM_MODIFY_HOOKS+=(grow_data_partition) + +# +# Allow APT Suite change in the release info +# +# Upstream repository http://raspbian.raspberrypi.org/raspbian has changed buster +# release suite from "stable" to "oldstable". The following configuration option +# allows APT to update without user interaction. +# This is the default un newer versions of apt, see: +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=931566 +# +function apt_allow_release_change_suite() { + run_and_log_cmd "echo 'Acquire::AllowReleaseInfoChange::Suite \"true\";' \ + | sudo tee work/rootfs/etc/apt/apt.conf.d/99release-change-suite" +} +PLATFORM_MODIFY_HOOKS+=(apt_allow_release_change_suite) diff --git a/configs/mender_convert_config b/configs/mender_convert_config index 4357caf7ba..c6e0ae50f5 100644 --- a/configs/mender_convert_config +++ b/configs/mender_convert_config @@ -32,6 +32,9 @@ MENDER_ARTIFACT_COMPRESSION="gzip" # this. MENDER_ENABLE_SYSTEMD=y +# Set the fstab options for mounting the boot partition +MENDER_BOOT_PART_FSTAB_OPTS="defaults,sync" + # Custom mkfs options for creating the rootfs partition MENDER_ROOT_PART_MKFS_OPTS="" @@ -94,11 +97,11 @@ MENDER_PARTITION_SCHEME="" # (see MENDER_BOOT_PART_SIZE_MB) and data partitions # (see MENDER_DATA_PART_SIZE_MB). # -MENDER_STORAGE_TOTAL_SIZE_MB="8192" +MENDER_STORAGE_TOTAL_SIZE_MB="30136" # The size of the boot partition in the generated .biosimg, .sdimg or .uefiimg # file. -MENDER_BOOT_PART_SIZE_MB="40" +MENDER_BOOT_PART_SIZE_MB="256" # The size of the persistent data partition in the generated .biosimg, .sdimg # or .uefiimg file. @@ -146,10 +149,14 @@ MENDER_ADDON_CONFIGURE_VERSION="latest" # what you are doing. MENDER_STORAGE_URL="https://downloads.mender.io" -# Mender APT repo, containing binary files, do not modify this unless you know +# Mender APT repo url, containing binary files, do not modify this unless you know # what you are doing. MENDER_APT_REPO_URL="${MENDER_STORAGE_URL}/repos/debian" +# Mender APT repo available distributions, do not modify this unless you know +# what you are doing. +MENDER_APT_REPO_DISTS="debian/buster debian/bullseye ubuntu/bionic ubuntu/focal" + # Mender GitHub organization URL prefix MENDER_GITHUB_ORG="https://github.com/mendersoftware" @@ -206,7 +213,7 @@ MENDER_COPY_BOOT_GAP="y" # # This variable directly mirrors the variable from the Yocto Project, which is # why it is missing a "MENDER_" prefix. -IMAGE_ROOTFS_SIZE="0" +IMAGE_ROOTFS_SIZE="13671875" # The amount of extra free space requested on the rootfs, in KiB. This is added # to the value of IMAGE_ROOTFS_SIZE. The size is further modified by diff --git a/configs/mender_convert_demo_config b/configs/mender_convert_demo_config new file mode 100644 index 0000000000..20f93caf87 --- /dev/null +++ b/configs/mender_convert_demo_config @@ -0,0 +1,35 @@ +# Install all the add-on's by default +MENDER_ADDON_CONNECT_INSTALL="y" +MENDER_ADDON_CONFIGURE_INSTALL="y" + +enable_demo_configuration_in_file() { + + log_info "Modifying mender.conf to follow a demo setup" + + run_and_log_cmd "mkdir -p work/rootfs/etc/mender" + + if [ ! -f work/rootfs/etc/mender/mender.conf ]; then + log_error "No Mender configuration file found in the work folder." + exit 1 + fi + + if [ ! -f work/rootfs/etc/mender/mender-connect.conf ]; then + log_info "Installing the mender-connect.conf file" + cat <<- EOF > work/rootfs/etc/mender/mender-connect.conf + { + "ShellCommand": "/bin/sh", + "User": "root" + } +EOF + run_and_log_cmd "sudo chmod 0600 work/rootfs/etc/mender/mender-connect.conf" + fi + + log_info "Adding the demo configuration to the Mender client" + tmpfile=work/mender.conf.bak + run_and_log_cmd "cat work/rootfs/etc/mender/mender.conf | jq '.InventoryPollIntervalSeconds=5 | .RetryPollIntervalSeconds=30 | .UpdatePollIntervalSeconds=5' > ${tmpfile}" + run_and_log_cmd "cp ${tmpfile} work/rootfs/etc/mender/mender.conf" + run_and_log_cmd "sudo chmod 0600 work/rootfs/etc/mender/mender.conf" + log_warn "New mender.conf contents: $(cat work/rootfs/etc/mender/mender.conf)" + +} +PLATFORM_MODIFY_HOOKS+=(enable_demo_configuration_in_file) diff --git a/configs/mender_grub_config b/configs/mender_grub_config index 1c8b9565a8..275eee8672 100644 --- a/configs/mender_grub_config +++ b/configs/mender_grub_config @@ -1,8 +1,17 @@ -# Use U-boot -> GRUB -> EFI boot-loader integration +# Use UEFI -> GRUB boot-loader integration +# +# On ARM, U-boot is used as UEFI (EFI) provider # # Only disable this if you know what you are doing MENDER_GRUB_EFI_INTEGRATION=y +# Integrate with /etc/grub.d boot scripts. +# +# This has no effect unless MENDER_GRUB_EFI_INTEGRATION is enabled. Default is +# auto, in which case it will be detected. This is only supported for x86_64 +# platforms at the moment. +MENDER_GRUB_D_INTEGRATION=auto + # Version of GRUB to use. Note that there needs to be a precompiled version # available at the MENDER_STORAGE_URL download source. GRUB_VERSION=2.04 @@ -17,7 +26,7 @@ GRUB_VERSION=2.04 MENDER_GRUB_KERNEL_BOOT_ARGS="" # grub-mender-grubenv is the Mender integration for the GRUB bootloader -MENDER_GRUBENV_VERSION="aa7e8f8c76c6aca6dca1820aaa42dc2cbf9762a1" +MENDER_GRUBENV_VERSION="2ac898f5924d5870f8394ad8ecd3ef1ab1422e3b" MENDER_GRUBENV_URL="${MENDER_GITHUB_ORG}/grub-mender-grubenv/archive/${MENDER_GRUBENV_VERSION}.tar.gz" # Name of the storage device containing root filesystem partitions in GRUB diff --git a/configs/qemux86-64_config b/configs/qemux86-64_config deleted file mode 100644 index 6b0599d527..0000000000 --- a/configs/qemux86-64_config +++ /dev/null @@ -1,19 +0,0 @@ -# This configuration can be used to run on a qemux86-64 machine. -# -# This has been tested on images generated with the following command: -# -# mkosi -d ubuntu -r bionic -t gpt_ext4 -b --checksum --password password --package=openssh-server,dhcpcd5 -o image.raw -# -# Converted with the following command: -# -# MENDER_ARTIFACT_NAME=release-1 ./mender-convert --disk-image input/image.raw --overlay rootfs_overlay_demo --config configs/qemux86-64_config -# -# and qemu is executed with the following command: -# -# qemu-system-x86_64 -enable-kvm -m 512 -smp 2 -bios /usr/share/ovmf/x64/OVMF_CODE.fd -drive format=raw,file=qemux86_64-release-1.sdimg - -MENDER_STORAGE_DEVICE_BASE=/dev/sda -MENDER_DEVICE_TYPE="qemux86_64" - -# Nothing to copy -MENDER_COPY_BOOT_GAP="n" diff --git a/configs/raspberrypi4_ubuntu_config b/configs/raspberrypi4_ubuntu_config index 2d0b7a52ee..9708f5e793 100644 --- a/configs/raspberrypi4_ubuntu_config +++ b/configs/raspberrypi4_ubuntu_config @@ -1,7 +1,4 @@ -# Binaries generated with the following script: -# -# https://github.com/drewmoseley/mender-convert-integration-scripts/blob/master/build-uboot-rpi.shraspberrypi-integration-scripts.tar.gz -RASPBERRYPI_BINARIES="raspberrypi4-integration-2019.01.tar.gz" +RASPBERRYPI_CONFIG="raspberrypi4" RASPBERRYPI_KERNEL_IMAGE="vmlinuz" MENDER_KERNEL_IMAGETYPE="zImage" MENDER_DEVICE_TYPE="raspberrypi4" diff --git a/configs/raspberrypi_config b/configs/raspberrypi_config index 8ec9f1ac44..7b4fb66d4e 100644 --- a/configs/raspberrypi_config +++ b/configs/raspberrypi_config @@ -94,9 +94,9 @@ function platform_modify() { # Raspberry Pi headless configuration expects boot partition to be mounted # at /boot, so replace these services with /uboot - run_and_log_cmd "find work/rootfs/lib/ -type f -name *.service | xargs sed -i.bak 's|/boot/|/uboot/|g'" + run_and_log_cmd "find work/rootfs/lib/ -type f -name *.service | xargs sudo sed -i.bak 's|/boot/|/uboot/|g'" log_debug "Diff from modified service files:\n$(find work/rootfs/lib/ -type f -name '*.service' | xargs -I@ bash -c 'diff -u @.bak @')" - run_and_log_cmd "find work/rootfs/lib/ -type f -name *.service.bak | xargs rm" + run_and_log_cmd "find work/rootfs/lib/ -type f -name *.service.bak | xargs sudo rm" log_info "Certain service files have been changed to align with our /uboot boot partition mount point. See convert.log for more information" run_and_log_cmd "sudo install -m 755 work/rpi/binaries/fw_printenv work/rootfs/sbin/fw_printenv" @@ -111,9 +111,9 @@ function platform_modify() { # Enable UART. This is required for U-Boot to operate correctly, see # https://source.denx.de/u-boot/u-boot/-/blob/master/arch/arm/mach-bcm283x/Kconfig log_info "Enabling UART in U-Boot configuration" - cat <<- EOF >> work/boot/config.txt + sudo bash -c 'cat <<- EOF >> work/boot/config.txt # Enable UART enable_uart=1 -EOF +EOF' } diff --git a/configs/testing/hardware-testing b/configs/testing/hardware-testing new file mode 100644 index 0000000000..38d55df899 --- /dev/null +++ b/configs/testing/hardware-testing @@ -0,0 +1,24 @@ +MENDER_COMPRESS_DISK_IMAGE="none" +MENDER_ARTIFACT_COMPRESSION="none" +# Install the mender-client +MENDER_CLIENT_INSTALL="y" +# Enable sshd +function enable_sshd() { + log_warn "Enabling sshd by default" + touch work/boot/ssh +} +PLATFORM_MODIFY_HOOKS+=(enable_sshd) +# Allow root ssh logins +function allow_root_ssh_logins() { + log_warn "Allowing ssh root logins" + echo "PermitRootLogin yes" >> work/rootfs/etc/ssh/sshd_config +} +PLATFORM_MODIFY_HOOKS+=(allow_root_ssh_logins) +# Configure the ssh key permissions in the overlay +function set_root_ssh_permissions() { + sudo chown -R root:root work/rootfs/root + sudo chmod 755 work/rootfs/root + sudo chmod 700 work/rootfs/root/.ssh + sudo chmod 600 work/rootfs/root/.ssh/authorized_keys +} +OVERLAY_MODIFY_HOOKS+=(set_root_ssh_permissions) diff --git a/configs/ubuntu-qemux86-64_config b/configs/ubuntu-qemux86-64_config new file mode 100644 index 0000000000..9d62ef05a1 --- /dev/null +++ b/configs/ubuntu-qemux86-64_config @@ -0,0 +1,21 @@ +# This configuration can be used to run on a qemux86-64 machine. +# +# The image is generated with the following command: +# +# mender-convert/scripts/test/generate-image.sh ubuntu +# +# Then manually uploaded to Mender's AWS S3 bucket for CI testing +# +# Locally, converted with the following command: +# +# MENDER_ARTIFACT_NAME=release-1 ./docker-mender-convert --disk-image input/Ubuntu-Focal-x86-64.img --overlay rootfs_overlay_demo --config configs/qemux86-64_config +# +# and qemu is executed with the following command: +# +# qemu-system-x86_64 -enable-kvm -m 512 -smp 2 -bios /usr/share/OVMF/OVMF_CODE.fd -drive format=raw,file=deploy/Ubuntu-Focal-x86-64-qemux86-64-mender.img + +MENDER_STORAGE_DEVICE_BASE=/dev/sda +MENDER_DEVICE_TYPE="qemux86-64" + +# Nothing to copy +MENDER_COPY_BOOT_GAP="n" diff --git a/docker-build b/docker-build index 8aa5f25728..fd7cf116f4 100755 --- a/docker-build +++ b/docker-build @@ -16,6 +16,13 @@ set -e -IMAGE_NAME=${IMAGE_NAME:-mender-convert} +GIT_PROVIDED_TAG_NAME="$(git describe --exact HEAD 2>/dev/null || \ + git for-each-ref "refs/heads/*" --format '%(refname:short)' --points-at HEAD 2>/dev/null)" -eval docker build . -t ${IMAGE_NAME} "$@" +if [ -z "$IMAGE_NAME" -a -z "$GIT_PROVIDED_TAG_NAME" ]; then + echo "Could not deduce mendersoftware/mender-convert container version from currently checked out commit. Using latest." 1>&2 +fi + +IMAGE_NAME=${IMAGE_NAME:-"mendersoftware/mender-convert${GIT_PROVIDED_TAG_NAME:+:$GIT_PROVIDED_TAG_NAME}"} + +eval docker build . -t "${IMAGE_NAME}" "$*" diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 092334ddec..f4bea4e24a 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -20,9 +20,9 @@ set -e cd /mender-convert -echo "Running mender-convert "$@"" +echo "Running mender-convert $*" ./mender-convert "$@" -# Set owner and group to same as launch directory. -[ -d deploy ] && chown -R --reference=. deploy +# Set owner and group to same as input directory. +[ -d deploy ] && chown -R --reference=input deploy diff --git a/docker-mender-convert b/docker-mender-convert index a2dad60ddf..119655356b 100755 --- a/docker-mender-convert +++ b/docker-mender-convert @@ -16,38 +16,42 @@ set -e -. modules/git.sh +GIT_PROVIDED_TAG_NAME="$(git describe --exact HEAD 2>/dev/null || \ + git for-each-ref "refs/heads/*" --format '%(refname:short)' --points-at HEAD 2>/dev/null)" -IMAGE_NAME=${IMAGE_NAME:-mender-convert} +if [ -z "$IMAGE_NAME" -a -z "$GIT_PROVIDED_TAG_NAME" ]; then + echo "Could not deduce mendersoftware/mender-convert container version from currently checked out commit. Using latest." 1>&2 +fi -MENDER_CONVERT_DIR="$(pwd)" +IMAGE_NAME=${IMAGE_NAME:-"mendersoftware/mender-convert${GIT_PROVIDED_TAG_NAME:+:$GIT_PROVIDED_TAG_NAME}"} +DEPLOY_DIRECTORY=${DEPLOY_DIRECTORY:-"$(pwd)/deploy"} +INPUT_DIRECTORY=${INPUT_DIRECTORY:-"$(pwd)/input"} +LOGS_DIRECTORY=${LOGS_DIRECTORY:-"$(pwd)/logs"} -#Pass in version in-case we are added as a git submodule -MENDER_CONVERT_VERSION=$(git_mender_convert_version) +mkdir -p "$DEPLOY_DIRECTORY" +mkdir -p "$LOGS_DIRECTORY" -# Create a unique work directory so we can run multiple containers in parallel -WORK_DIR=$(mktemp -d -p $(pwd) -t work.XXXX) -WORK_SUFFIX=$(echo $WORK_DIR | awk -F. '{print $NF}') -LOG_FILE="convert.log.${WORK_SUFFIX}" +# Create a unique log file so we can run multiple containers in parallel +LOG_FILE="convert.log.$(date +%s)-$$" + +echo "using log file at: ${LOGS_DIRECTORY}/${LOG_FILE}" set +e docker run \ --rm \ - -v $MENDER_CONVERT_DIR:/mender-convert \ - -v $WORK_DIR:/mender-convert/work \ + -v "$INPUT_DIRECTORY":/mender-convert/input \ + -v "$LOGS_DIRECTORY":/mender-convert/logs \ + -v "$DEPLOY_DIRECTORY":/mender-convert/deploy \ --privileged=true \ --cap-add=SYS_MODULE \ -v /dev:/dev \ -v /lib/modules:/lib/modules:ro \ - --env MENDER_ARTIFACT_NAME=${MENDER_ARTIFACT_NAME} \ - --env MENDER_CONVERT_VERSION=${MENDER_CONVERT_VERSION} \ - --env MENDER_CONVERT_LOG_FILE=logs/${LOG_FILE} \ - $IMAGE_NAME "$@" + --env MENDER_ARTIFACT_NAME="${MENDER_ARTIFACT_NAME}" \ + --env MENDER_CONVERT_LOG_FILE=logs/"${LOG_FILE}" \ + "$IMAGE_NAME" "$@" exit_code=$? -[ ${exit_code} -eq 0 ] && rmdir ${WORK_DIR} - -echo "Log file available at: logs/${LOG_FILE}" +echo "Log file available at: ${LOGS_DIRECTORY}/${LOG_FILE}" exit ${exit_code} diff --git a/rootfs_overlay_demo/.gitkeep b/input/rootfs_overlay_demo/.gitkeep similarity index 100% rename from rootfs_overlay_demo/.gitkeep rename to input/rootfs_overlay_demo/.gitkeep diff --git a/mender-convert-modify b/mender-convert-modify index 9810f1771e..4623a085a8 100755 --- a/mender-convert-modify +++ b/mender-convert-modify @@ -51,6 +51,7 @@ source modules/disk.sh source modules/probe.sh source modules/deb.sh source modules/testscfg.sh +source modules/grub.sh # The mender_convert_config is always used and provides all the defaults declare -a configs=("configs/mender_convert_config") @@ -97,30 +98,14 @@ mkdir -p work/rootfs sudo mount ${boot_part} work/boot sudo mount ${root_part} work/rootfs -mkdir -p work/deb-packages - log_info "Installing Mender client and related files" -deb_arch=$(probe_debian_arch_name) - if [ "${MENDER_CLIENT_INSTALL}" = "y" ]; then - - log_info "Installing Mender Client version ${MENDER_CLIENT_VERSION}" - - if [ "${MENDER_CLIENT_VERSION}" = "latest" ]; then - deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "stable" "mender-client") - elif [ "${MENDER_CLIENT_VERSION}" = "master" ]; then - deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "experimental" "mender-client") - else - DEBIAN_REVISION="-1" - deb_name=$(deb_from_repo_pool_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "mender-client" "${MENDER_CLIENT_VERSION}${DEBIAN_REVISION}") - fi - - deb_extract_package "work/deb-packages/${deb_name}" "work/rootfs/" + log_info "Installing Mender client version ${MENDER_CLIENT_VERSION}" + deb_get_and_install_pacakge mender-client "${MENDER_CLIENT_VERSION}" # Save installed client version for tests in Yocto variable format - testscfg_add "PREFERRED_VERSION_mender-client" "$(echo ${deb_name} | sed -r 's/.*_([0-9]+\.[0-9]+\.[0-9]+).*/\1/')" - + testscfg_add "PREFERRED_VERSION_mender-client" "$(echo ${DEB_NAME} | sed -r 's/.*_([0-9]+\.[0-9]+\.[0-9]+).*/\1/')" fi if [ "${MENDER_ENABLE_SYSTEMD}" == "y" ]; then @@ -129,153 +114,57 @@ if [ "${MENDER_ENABLE_SYSTEMD}" == "y" ]; then fi if [ "${MENDER_ADDON_CONNECT_INSTALL}" = "y" ]; then - log_info "Installing Mender Connect addon" - - if [ "${MENDER_ADDON_CONNECT_VERSION}" = "latest" ]; then - deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "stable" "mender-connect") - elif [ "${MENDER_ADDON_CONNECT_VERSION}" = "master" ]; then - deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "experimental" "mender-connect") - else - DEBIAN_REVISION="-1" - deb_name=$(deb_from_repo_pool_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "mender-connect" "${MENDER_ADDON_CONNECT_VERSION}${DEBIAN_REVISION}") - fi - - deb_extract_package "work/deb-packages/${deb_name}" "work/rootfs/" + deb_get_and_install_pacakge mender-connect "${MENDER_ADDON_CONNECT_VERSION}" run_and_log_cmd "sudo ln -sf /lib/systemd/system/mender-connect.service \ work/rootfs/etc/systemd/system/multi-user.target.wants/mender-connect.service" fi if [ "${MENDER_ADDON_CONFIGURE_INSTALL}" = "y" ]; then - log_info "Installing Mender Configure addon" - - if [ "${MENDER_ADDON_CONFIGURE_VERSION}" = "latest" ]; then - deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "stable" "mender-configure") - elif [ "${MENDER_ADDON_CONFIGURE_VERSION}" = "master" ]; then - deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "experimental" "mender-configure") - else - DEBIAN_REVISION="-1" - deb_name=$(deb_from_repo_pool_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "mender-configure" "${MENDER_ADDON_CONFIGURE_VERSION}${DEBIAN_REVISION}") - fi - - deb_extract_package "work/deb-packages/${deb_name}" "work/rootfs/" + deb_get_and_install_pacakge mender-configure "${MENDER_ADDON_CONFIGURE_VERSION}" "true" fi -# Do this unconditionally even if not installing mender-configure. The reason is -# that if the Debian package is installed later, this folder has to preexist. It -# is part of the rootfs setup, independently of the software. -log_info "Creating the Mender Configure state folder in the data partition" +# Do this unconditionally even if not installing add-ons. The reason is that if +# the Debian package is installed later, this folder has to preexist. It is part +# of the rootfs setup, independently of the software. +log_info "Creating state folder in the data partition for Mender add-ons" +run_and_log_cmd "sudo mkdir -p work/rootfs/var/lib" run_and_log_cmd "sudo mkdir -p work/rootfs/data/mender-configure" -run_and_log_cmd "sudo ln -sf /data/mender-configure work/rootfs/var/lib/mender-configure" +run_and_log_cmd "sudo ln -sf /data/mender-configure work/rootfs/var/lib" +run_and_log_cmd "sudo mkdir -p work/rootfs/data/mender-monitor" +run_and_log_cmd "sudo ln -sf /data/mender-monitor work/rootfs/var/lib" if [ "${MENDER_GRUB_EFI_INTEGRATION}" == "y" ]; then # Check for known U-Boot problems in all files on the boot partition. check_for_broken_uboot_uefi_support work/boot - run_and_log_cmd "wget -Nq '${MENDER_GRUBENV_URL}' -P work/" - run_and_log_cmd "tar xzvf work/${MENDER_GRUBENV_VERSION}.tar.gz -C work/" + if [ "$MENDER_GRUB_D_INTEGRATION" = y ] || ( [ "$MENDER_GRUB_D_INTEGRATION" = auto ] && supports_grub_d "work/rootfs" ); then + # No need to install Grub, use the one already present, and only install + # our tools. + log_info "Not installing GRUB EFI bootloader, relying on platform provided one." - if [ -z "${MENDER_GRUB_KERNEL_IMAGETYPE}" ]; then - kernel_imagetype=$(probe_kernel_in_boot_and_root) - else - kernel_imagetype="${MENDER_GRUB_KERNEL_IMAGETYPE}" - fi + log_info "Installing Mender GRUB tools..." + grub_install_grub_editenv_binary - if [ -z "${MENDER_GRUB_INITRD_IMAGETYPE}" ]; then - initrd_imagetype=$(probe_initrd_in_boot_and_root) - else - initrd_imagetype="${MENDER_GRUB_INITRD_IMAGETYPE}" - fi - - cat <<- EOF > work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION}/mender_grubenv_defines -mender_rootfsa_part=${MENDER_ROOTFS_PART_A_NUMBER} -mender_rootfsb_part=${MENDER_ROOTFS_PART_B_NUMBER} -mender_grub_storage_device=${MENDER_GRUB_STORAGE_DEVICE} -kernel_imagetype=kernel -initrd_imagetype=initrd -EOF - - run_and_log_cmd "sudo ln -s ${kernel_imagetype} work/rootfs/boot/kernel" - if [ "${initrd_imagetype}" != "" ]; then - run_and_log_cmd "sudo ln -s ${initrd_imagetype} work/rootfs/boot/initrd" - fi - - # For partuuid support grub.cfg expects dedicated variables to be added - if [ "${MENDER_ENABLE_PARTUUID}" == "y" ]; then - rootfsa_partuuid=$(disk_get_partuuid_from_device "${root_part_a_device}") - rootfsb_partuuid=$(disk_get_partuuid_from_device "${root_part_b_device}") - log_info "Using root partition A partuuid in grubenv: $rootfsa_partuuid" - log_info "Using root partition B partuuid in grubenv: $rootfsb_partuuid" - cat <<- EOF >> work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION}/mender_grubenv_defines -mender_rootfsa_uuid=${rootfsa_partuuid} -mender_rootfsb_uuid=${rootfsb_partuuid} -EOF + log_info "Generating grub config using update-grub..." + grub_create_grub_config + grub_install_grub_d_config else - cat <<- EOF >> work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION}/mender_grubenv_defines -mender_kernel_root_base=${MENDER_STORAGE_DEVICE_BASE} -EOF - fi + log_info "Installing GRUB EFI bootloader..." + grub_install_mender_grub - if [ -n "${MENDER_GRUB_KERNEL_BOOT_ARGS}" ]; then - cat <<- EOF > work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION}/11_bootargs_grub.cfg -set bootargs="${MENDER_GRUB_KERNEL_BOOT_ARGS}" -EOF + log_info "Generating the mender-grub config..." + grub_create_grub_config + grub_install_standalone_grub_config fi - cd work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION} - run_and_log_cmd "make 2>&1" - run_and_log_cmd "sudo make DESTDIR=../ BOOT_DIR=boot install-boot-files" - run_and_log_cmd "sudo make DESTDIR=../rootfs install-tools" - cd - > /dev/null 2>&1 - - # Remove conflicting boot files. These files do not necessarily effect the - # functionality, but lets get rid of them to avoid confusion. - # - # There is no Mender integration for EFI boot or systemd-boot. - sudo rm -rf work/boot/loader work/rootfs/boot/loader - sudo rm -rf work/boot/EFI/Linux - sudo rm -rf work/boot/EFI/systemd - sudo rm -rf work/boot/NvVars - for empty_dir in $( - cd work/boot - find . -maxdepth 1 -type d -empty - ); do - sudo rmdir work/boot/$empty_dir - done - - log_info "Installing GRUB" - - arch=$(probe_arch) - efi_name=$(probe_grub_efi_name) - efi_target_name=$(probe_grub_efi_target_name) - - log_info "GRUB EFI: ${efi_target_name}" - - run_and_log_cmd "wget -Nq ${MENDER_GRUB_BINARY_STORAGE_URL}/${arch}/${efi_name} -P work/" - run_and_log_cmd "wget -Nq ${MENDER_GRUB_BINARY_STORAGE_URL}/${arch}/grub-editenv -P work/" - - run_and_log_cmd "sudo install -m 751 work/grub-editenv work/rootfs/usr/bin/" - - run_and_log_cmd "sudo mkdir -p work/boot/EFI/BOOT" - run_and_log_cmd "sudo cp work/${efi_name} -P work/boot/EFI/BOOT/${efi_target_name}" -fi - -# Copy dtb directory to the boot partition for use by the bootloader. -if [ -d work/rootfs/boot/dtbs ]; then - # Look for the first directory that has dtb files. First check the base - # folder, then any subfolders in versioned order. - for candidate in work/rootfs/boot/dtbs $(find work/rootfs/boot/dtbs/ -maxdepth 1 -type d | sort -V -r); do - if [ $(find $candidate -maxdepth 1 -name '*.dtb' | wc -l) -gt 0 ]; then - run_and_log_cmd "sudo cp -r $candidate work/boot/dtb" - break - fi - done fi run_and_log_cmd "sudo mkdir -p work/rootfs/data/mender" -run_and_log_cmd "sudo ln -sf /data/mender work/rootfs/var/lib/mender" +run_and_log_cmd "sudo mkdir -p work/rootfs/var/lib" +run_and_log_cmd "sudo ln -sf /data/mender work/rootfs/var/lib" log_info "Using root device A in mender.conf: $root_part_a_device" log_info "Using root device B in mender.conf: $root_part_b_device" @@ -290,6 +179,15 @@ EOF run_and_log_cmd "sudo cp work/mender.conf.data work/rootfs/data/mender/mender.conf" run_and_log_cmd "sudo chmod 600 work/rootfs/data/mender/mender.conf" +if [ -f input/resources/mender.conf ]; then + log_info "Installing the local mender.conf file" + run_and_log_cmd "mkdir -p work/rootfs/etc/mender" + run_and_log_cmd "cp input/resources/mender.conf work/rootfs/etc/mender" + run_and_log_cmd "sudo chmod 600 work/rootfs/etc/mender/mender.conf" +else + log_warn "No mender.conf file found in input/resources. Have you remembered to run the bootstrap script?" +fi + if [ -z "${MENDER_DEVICE_TYPE}" ]; then # Observed systems who do not have this file, e.g images generated with mkosi if [ -f work/rootfs/etc/hostname ]; then @@ -343,14 +241,17 @@ fi log_info "Using boot partition device in fstab: $boot_part_device" log_info "Using data partition device in fstab: $data_part_device" -sudo bash -c "cat <<- EOF > work/rootfs/etc/fstab -# stock fstab - you probably want to override this with a machine specific one +fstab=$(sed '/\/boot/d' work/rootfs/etc/fstab) +fstab=$(sed '/.*\s\/\s.*/d' <<< "$fstab") +sudo bash -c 'cat <<- EOF > work/rootfs/etc/fstab +${boot_part_device} ${boot_part_mountpoint} auto ${MENDER_BOOT_PART_FSTAB_OPTS} 0 0 /dev/root / auto ${MENDER_ROOT_PART_FSTAB_OPTS} 1 ${MENDER_ROOT_PART_FS_PASSNO} -proc /proc proc defaults 0 0 -${boot_part_device} ${boot_part_mountpoint} auto defaults,sync 0 0 ${data_part_device} /data auto ${MENDER_DATA_PART_FSTAB_OPTS} 0 0 -EOF" + +# entries kept from original fstab +$fstab +EOF' # # Make sure to re-label rootfs when selinux is in enforcing mode diff --git a/mender-convert-package b/mender-convert-package index 7d30bb1584..30b8e80133 100755 --- a/mender-convert-package +++ b/mender-convert-package @@ -256,6 +256,10 @@ if [ "${partition_scheme}" == "gpt" ]; then log_info "Writing GPT partition table" run_and_log_cmd "${PARTED} -s ${img_path} mklabel gpt" run_and_log_cmd "${PARTED} -s ${img_path} unit s mkpart ESP fat32 ${boot_part_start} ${boot_part_end}" + # Keep the uuid of the original gpt table for the boot partition + boot_partuuid=$(disk_get_part_value "${disk_image}" 1 UUID) + log_info "Updating GPT partition uuid of the boot partition in image: ${img_path} to ${boot_partuuid}" + run_and_log_cmd "sgdisk -u ${MENDER_BOOT_PART_NUMBER}:${boot_partuuid} '${img_path}'" else log_info "Writing DOS (MBR) partition table" run_and_log_cmd "${PARTED} -s ${img_path} mklabel msdos" diff --git a/modules/deb.sh b/modules/deb.sh index e8686ca7ab..346de7e326 100644 --- a/modules/deb.sh +++ b/modules/deb.sh @@ -1,5 +1,5 @@ # -# Copyright 2020 Northern.tech AS +# Copyright 2022 Northern.tech AS # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,6 +14,7 @@ # limitations under the License. source modules/log.sh +source modules/probe.sh . # Download of latest deb package for the given distribution of an APT repository # @@ -47,7 +48,8 @@ function deb_from_repo_dist_get() { fi local -r filename=$(basename $deb_package_path) - run_and_log_cmd "wget -Nq ${repo_url}/${deb_package_path} -P ${download_dir}" + local -r deb_package_url=$(echo ${repo_url}/${deb_package_path} | sed 's/+/%2B/g') + run_and_log_cmd "wget -Nq ${deb_package_url} -P ${download_dir}" rm -f /tmp/Packages log_info "Successfully downloaded ${filename}" @@ -80,11 +82,18 @@ function deb_from_repo_pool_get() { local -r deb_package_path="pool/${component}/${initial}/${package}/${package}_${version}_${architecture}.deb" local -r filename=$(basename $deb_package_path) - run_and_log_cmd "wget -Nq ${repo_url}/${deb_package_path} -P ${download_dir}" + local -r deb_package_url=$(echo ${repo_url}/${deb_package_path} | sed 's/+/%2B/g') + run_and_log_cmd_noexit "wget -Nq ${deb_package_url} -P ${download_dir}" + local exit_code=$? rm -f /tmp/Packages - log_info "Successfully downloaded ${filename}" - echo ${filename} + if [[ ${exit_code} -ne 0 ]]; then + log_warn "Could not download ${filename}" + echo "" + else + log_info "Successfully downloaded ${filename}" + echo ${filename} + fi } # Extract the binary files of a deb package into a directory @@ -110,3 +119,56 @@ function deb_extract_package() { log_info "Successfully installed $(basename ${deb_package}) into ${dest_dir}" } + +# Download and install the binary files of a deb package into work/deb-packages +# This is the main entry point of deb.sh +# Defines variable DEB_NAME with the actual filename installed +# +# $1 - Package name +# $2 - Package version +# $3 - Arch independent (optional, default "false") +# +function deb_get_and_install_pacakge() { + if ! [[ $# -eq 2 || $# -eq 3 ]]; then + log_fatal "deb_get_and_install_pacakge() requires 2 or 3 arguments" + fi + local package="$1" + local version="$2" + local arch_indep="${3:-false}" + + mkdir -p work/deb-packages + + local deb_arch=$(probe_debian_arch_name) + local deb_distro=$(probe_debian_distro_name) + local deb_codename=$(probe_debian_distro_codename) + if ! [[ "$MENDER_APT_REPO_DISTS" == *"${deb_distro}/${deb_codename}"* ]]; then + deb_distro="debian" + deb_codename="buster" + fi + + DEB_NAME="" + if [ "${version}" = "latest" ]; then + DEB_NAME=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "${deb_distro}/${deb_codename}/stable" "${package}") + elif [ "${version}" = "master" ]; then + DEB_NAME=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "${deb_distro}/${deb_codename}/experimental" "${package}") + else + # On direct downloads, the architecture suffix will be "all" for arch independent packages + local pool_arch="" + if [[ "$arch_indep" == "true" ]]; then + pool_arch="all" + else + pool_arch="$deb_arch" + fi + + local debian_version="-1+${deb_distro}+${deb_codename}" + DEB_NAME=$(deb_from_repo_pool_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${pool_arch} "${package}" "${version}${debian_version}") + if [[ -z "${DEB_NAME}" ]]; then + local debian_version_fallback="-1" + DEB_NAME=$(deb_from_repo_pool_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${pool_arch} "${package}" "${version}${debian_version_fallback}") + if [[ -z "${DEB_NAME}" ]]; then + log_fatal "Specified version for ${package} cannot be found, tried ${version}${debian_version} and ${version}${debian_version_fallback}" + fi + fi + fi + deb_extract_package "work/deb-packages/${DEB_NAME}" "work/rootfs/" +} diff --git a/modules/grub.sh b/modules/grub.sh new file mode 100644 index 0000000000..904f96f93d --- /dev/null +++ b/modules/grub.sh @@ -0,0 +1,207 @@ +#!/usr/bin/env bash +# +# Copyright 2022 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# grub_create_grub_config +# +# +function grub_create_grub_config() { + + run_and_log_cmd "wget -Nq '${MENDER_GRUBENV_URL}' -P work/" + run_and_log_cmd "tar xzvf work/${MENDER_GRUBENV_VERSION}.tar.gz -C work/" + + cat <<- EOF > work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION}/mender_grubenv_defines +mender_rootfsa_part=${MENDER_ROOTFS_PART_A_NUMBER} +mender_rootfsb_part=${MENDER_ROOTFS_PART_B_NUMBER} +mender_grub_storage_device=${MENDER_GRUB_STORAGE_DEVICE} +kernel_imagetype=kernel +initrd_imagetype=initrd +EOF + + # For partuuid support grub.cfg expects dedicated variables to be added + if [ "${MENDER_ENABLE_PARTUUID}" == "y" ]; then + rootfsa_partuuid=$(disk_get_partuuid_from_device "${root_part_a_device}") + rootfsb_partuuid=$(disk_get_partuuid_from_device "${root_part_b_device}") + log_info "Using root partition A partuuid in grubenv: $rootfsa_partuuid" + log_info "Using root partition B partuuid in grubenv: $rootfsb_partuuid" + cat <<- EOF >> work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION}/mender_grubenv_defines +mender_rootfsa_uuid=${rootfsa_partuuid} +mender_rootfsb_uuid=${rootfsb_partuuid} +EOF + else + cat <<- EOF >> work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION}/mender_grubenv_defines +mender_kernel_root_base=${MENDER_STORAGE_DEVICE_BASE} +EOF + fi +} + +# grub_install_standalone_grub_config +# +# +function grub_install_standalone_grub_config() { + if [ -n "${MENDER_GRUB_KERNEL_BOOT_ARGS}" ]; then + cat <<- EOF > work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION}/11_bootargs_grub.cfg +set bootargs="${MENDER_GRUB_KERNEL_BOOT_ARGS}" +EOF + fi + + ( + cd work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION} + run_and_log_cmd "make 2>&1" + run_and_log_cmd "sudo make DESTDIR=$PWD/../ BOOT_DIR=boot install-standalone-boot-files" + run_and_log_cmd "sudo make DESTDIR=$PWD/../rootfs install-tools" + ) + +} + +# grub_install_grub_d_config +# +# +function grub_install_grub_d_config() { + if [ -n "${MENDER_GRUB_KERNEL_BOOT_ARGS}" ]; then + log_warn "MENDER_GRUB_KERNEL_BOOT_ARGS is ignored when MENDER_GRUB_D_INTEGRATION is enabled. Set it in the GRUB configuration instead." + fi + + # When using grub.d integration, /boot/efi must point to the boot partition, + # and /boot/grub must point to grub-mender-grubenv on the boot partition. + if [ ! -d work/rootfs/boot/efi ]; then + run_and_log_cmd "sudo mkdir work/rootfs/boot/efi" + fi + run_and_log_cmd "sudo mkdir work/boot/grub-mender-grubenv" + run_and_log_cmd "sudo mv work/rootfs/boot/grub/* work/boot/grub-mender-grubenv/" + run_and_log_cmd "sudo rmdir work/rootfs/boot/grub" + run_and_log_cmd "sudo ln -s efi/grub-mender-grubenv work/rootfs/boot/grub" + + ( + cd work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION} + run_and_log_cmd "make 2>&1" + run_and_log_cmd "sudo make DESTDIR=$PWD/../ BOOT_DIR=boot install-boot-env" + run_and_log_cmd "sudo make DESTDIR=$PWD/../rootfs install-grub.d-boot-scripts" + run_and_log_cmd "sudo make DESTDIR=$PWD/../rootfs install-tools" + # We need this for running the scripts once. + run_and_log_cmd "sudo make DESTDIR=$PWD/../rootfs install-offline-files" + ) + + # Mender-convert usually runs in a container. It's difficult to launch + # additional containers from within an existing one, but we need to run + # `update-grub` on a simulated device using some sort of container. Use good + # old `chroot`, which doesn't provide perfect containment, but it is good + # enough for our purposes, and doesn't require special containment + # capabilities. This will not work for foreign architectures, but we could + # probably use something like qemu-aarch64-static to get around that. + run_and_log_cmd "sudo mount work/boot work/rootfs/boot/efi -o bind" + run_and_log_cmd "sudo mount /dev work/rootfs/dev -o bind,ro" + run_and_log_cmd "sudo mount /proc work/rootfs/proc -o bind,ro" + run_and_log_cmd "sudo mount /sys work/rootfs/sys -o bind,ro" + + local ret=0 + + # Use `--no-nvram`, since we cannot update firmware memory in an offline + # build. Instead, use `--removable`, which creates entries that automate + # booting if you put the image into a new device, which you almost certainly + # will after using mender-convert. + run_and_log_cmd_noexit "sudo chroot work/rootfs grub-install --removable --no-nvram" || ret=$? + if [ $ret -eq 0 ]; then + run_and_log_cmd_noexit "sudo chroot work/rootfs grub-install --no-nvram" || ret=$? + fi + + if [ $ret -eq 0 ]; then + run_and_log_cmd_noexit "sudo chroot work/rootfs update-grub" || ret=$? + fi + + # Very important that these are unmounted, otherwise Docker may start to + # remove files inside them while tearing down the container. You can guess + # how I found that out... We run without the logger because otherwise the + # message from the previous command, which is the important one, is lost. + sudo umount -l work/rootfs/boot/efi || true + sudo umount -l work/rootfs/dev || true + sudo umount -l work/rootfs/proc || true + sudo umount -l work/rootfs/sys || true + + [ $ret -ne 0 ] && exit $ret + + ( + cd work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION} + # Should be removed after running. + run_and_log_cmd "sudo make DESTDIR=$PWD/../rootfs uninstall-offline-files" + ) +} + +# grub_install_grub_editenv_binary +# +# Install the editenv binary +function grub_install_grub_editenv_binary() { + log_info "Installing the GRUB editenv binary" + + arch=$(probe_arch) + + run_and_log_cmd "wget -Nq ${MENDER_GRUB_BINARY_STORAGE_URL}/${arch}/grub-editenv -P work/" + run_and_log_cmd "sudo install -m 751 work/grub-editenv work/rootfs/usr/bin/" + +} + +# grub_install_mender_grub +# +# Install mender-grub on the converted boot partition +function grub_install_mender_grub() { + kernel_imagetype=${MENDER_GRUB_KERNEL_IMAGETYPE:-$(probe_kernel_in_boot_and_root)} + initrd_imagetype=${MENDER_GRUB_INITRD_IMAGETYPE:-$(probe_initrd_in_boot_and_root)} + + run_and_log_cmd "sudo ln -s ${kernel_imagetype} work/rootfs/boot/kernel" + if [ "${initrd_imagetype}" != "" ]; then + run_and_log_cmd "sudo ln -s ${initrd_imagetype} work/rootfs/boot/initrd" + fi + + # Remove conflicting boot files. These files do not necessarily effect the + # functionality, but lets get rid of them to avoid confusion. + # + # There is no Mender integration for EFI boot or systemd-boot. + sudo rm -rf work/boot/loader work/rootfs/boot/loader + sudo rm -rf work/boot/EFI/Linux + sudo rm -rf work/boot/EFI/systemd + sudo rm -rf work/boot/NvVars + for empty_dir in $( + cd work/boot && find . -maxdepth 1 -type d -empty -not -name . + ); do + sudo rmdir work/boot/$empty_dir + done + + log_info "Installing GRUB..." + + log_info "Installing mender-grub-editenv" + grub_install_grub_editenv_binary + + local -r arch=$(probe_arch) + local -r efi_name=$(probe_grub_efi_name) + local -r efi_target_name=$(probe_grub_efi_target_name) + + log_info "GRUB EFI: ${efi_target_name}" + + run_and_log_cmd "wget -Nq ${MENDER_GRUB_BINARY_STORAGE_URL}/${arch}/${efi_name} -P work/" + run_and_log_cmd "sudo mkdir -p work/boot/EFI/BOOT" + run_and_log_cmd "sudo cp work/${efi_name} -P work/boot/EFI/BOOT/${efi_target_name}" + + # Copy dtb directory to the boot partition for use by the bootloader. + if [ -d work/rootfs/boot/dtbs ]; then + # Look for the first directory that has dtb files. First check the base + # folder, then any subfolders in versioned order. + for candidate in work/rootfs/boot/dtbs $(find work/rootfs/boot/dtbs/ -maxdepth 1 -type d | sort -V -r); do + if [ $(find $candidate -maxdepth 1 -name '*.dtb' | wc -l) -gt 0 ]; then + run_and_log_cmd "sudo cp -r $candidate work/boot/dtb" + break + fi + done + fi +} diff --git a/modules/probe.sh b/modules/probe.sh index 4a71aabc38..92233828de 100644 --- a/modules/probe.sh +++ b/modules/probe.sh @@ -38,7 +38,7 @@ probe_arch() { target_arch="unknown" if grep -q x86-64 <<< "${file_info}"; then - target_arch="x86-64" + target_arch="x86_64" elif grep -Eq "ELF 32-bit.*ARM" <<< "${file_info}"; then target_arch="arm" elif grep -Eq "ELF 64-bit.*aarch64" <<< "${file_info}"; then @@ -54,10 +54,10 @@ probe_arch() { # No input parameters and these work on the assumption that boot and root parts # are mounted at work/boot and work/rootfs probe_grub_efi_name() { - efi_name="" - arch=$(probe_arch) + local efi_name="" + local -r arch=$(probe_arch) case "${arch}" in - "x86-64") + "x86_64") efi_name="grub-efi-bootx64.efi" ;; "arm") @@ -81,7 +81,7 @@ probe_debian_arch_name() { deb_arch="" arch=$(probe_arch) case "${arch}" in - "x86-64") + "x86_64") deb_arch="amd64" ;; "arm") @@ -97,6 +97,29 @@ probe_debian_arch_name() { echo "${deb_arch}" } +# Prints Debian distro name based on ID from /etc/os-release +# +# Special handling for raspbian, where ID_LIKE is used instead +# +# No input parameters and these work on the assumption that boot and root parts +# are mounted at work/boot and work/rootfs +probe_debian_distro_name() { + local distro_name="$(. work/rootfs/etc/os-release && echo "$ID")" + if [[ "$distro_name" == "raspbian" ]]; then + distro_name="$(. work/rootfs/etc/os-release && echo "$ID_LIKE")" + fi + echo "${distro_name}" +} + +# Prints Debian distro codename based on VERSION_CODENAME from /etc/os-release +# +# No input parameters and these work on the assumption that boot and root parts +# are mounted at work/boot and work/rootfs +probe_debian_distro_codename() { + local -r distro_codename="$(. work/rootfs/etc/os-release && echo "$VERSION_CODENAME")" + echo "${distro_codename}" +} + # Prints GRUB EFI target name depending on target architecture # # This is what the file name should be when put on target boot part. @@ -104,10 +127,10 @@ probe_debian_arch_name() { # No input parameters and these work on the assumption that boot and root parts # are mounted at work/boot and work/rootfs probe_grub_efi_target_name() { - efi_target_name="" - arch=$(probe_arch) + local efi_target_name="" + local -r arch=$(probe_arch) case "$arch" in - "x86-64") + "x86_64") efi_target_name="bootx64.efi" ;; "arm") @@ -128,7 +151,7 @@ probe_grub_efi_target_name() { # $1 - directory in which the search is performed # probe_kernel_image() { - kernel_image_path="" + local kernel_image_path="" for image in vmlinuz zImage bzImage; do # Linux kernel image type and naming varies between different platforms. # @@ -165,7 +188,7 @@ probe_kernel_image() { # $1 - directory in which the search is performed # probe_initrd_image() { - initrd_image_path="" + local initrd_image_path="" for image in initramfs initrd; do # initrd/initramfs naming varies between different platforms. # @@ -321,3 +344,14 @@ is_efi_compatible_kernel() { esac return 0 } + +supports_grub_d() { + test -d "$1"/etc/grub.d || return 1 + + # Because we are executing programs inside a chroot in the image, we cannot + # currently convert non-native architectures to use grub.d integration. See + # relevant section on chroot inside grub_install_grub_d_config. + [ "$(probe_arch)" = "$(uname -m)" ] || return 1 + + return 0 +} diff --git a/modules/run.sh b/modules/run.sh index a52efd3ea1..93f9159da6 100644 --- a/modules/run.sh +++ b/modules/run.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright 2019 Northern.tech AS +# Copyright 2022 Northern.tech AS # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Run a command, capture output and log it +# Run a command, capture and log output, and exit on non-zero return code # # $1 - command to run function run_and_log_cmd() { @@ -31,3 +31,19 @@ function run_and_log_cmd() { exit ${exit_code} fi } + +# Run a command, capture and log output, and return the command's return code +# +# $1 - command to run +function run_and_log_cmd_noexit() { + local -r cmd="${1}" + local -r position="(${BASH_SOURCE[1]}:${BASH_LINENO[0]}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }" + local exit_code=0 + output="$({ eval ${cmd}; } 2>&1)" || exit_code=$? + local log_msg="Running: ${position} \n\r\n\r\t${cmd}" + if [[ "${output}" != "" ]]; then + log_msg="${log_msg}\n\t${output}\n" + fi + log_debug "${log_msg}" + return ${exit_code} +} diff --git a/scripts/bootstrap-rootfs-overlay-demo-server.sh b/scripts/bootstrap-rootfs-overlay-demo-server.sh index 57879e95f0..d2d22a8bac 100755 --- a/scripts/bootstrap-rootfs-overlay-demo-server.sh +++ b/scripts/bootstrap-rootfs-overlay-demo-server.sh @@ -56,28 +56,16 @@ if [ -e ${output_dir} ]; then sudo chown -R $(id -u) ${output_dir} sudo chgrp -R $(id -g) ${output_dir} fi + mkdir -p ${output_dir}/etc/mender -cat <<- EOF > ${output_dir}/etc/mender/mender.conf +mkdir -p ${root_dir}/input/resources +cat <<- EOF > ${root_dir}/input/resources/mender.conf { - "InventoryPollIntervalSeconds": 5, - "RetryPollIntervalSeconds": 30, "ServerURL": "https://docker.mender.io", - "ServerCertificate": "/etc/mender/server.crt", - "UpdatePollIntervalSeconds": 5 + "ServerCertificate": "/etc/mender/server.crt" } EOF -chmod 600 ${output_dir}/etc/mender/mender.conf - -cat <<- EOF > ${output_dir}/etc/mender/mender-connect.conf -{ - "ShellCommand": "/bin/sh", - "User": "root" -} -EOF - -chmod 600 ${output_dir}/etc/mender/mender-connect.conf - cat <<- EOF > ${output_dir}/etc/hosts 127.0.0.1 localhost @@ -93,4 +81,4 @@ wget -q "https://raw.githubusercontent.com/mendersoftware/mender/master/support/ sudo chown -R 0 ${output_dir} sudo chgrp -R 0 ${output_dir} -echo "Configuration file for using Demo Mender Server written to: ${output_dir}/etc/mender" +echo "Configuration file for using Demo Mender Server written to: ${root_dir}/input/resources/mender.conf" diff --git a/scripts/bootstrap-rootfs-overlay-hosted-server.sh b/scripts/bootstrap-rootfs-overlay-hosted-server.sh index a6fdb92740..897768d982 100755 --- a/scripts/bootstrap-rootfs-overlay-hosted-server.sh +++ b/scripts/bootstrap-rootfs-overlay-hosted-server.sh @@ -37,7 +37,7 @@ while (("$#")); do ;; *) echo "Sorry but the provided option is not supported: $1" - echo "Usage: $(basename $0) --output-dir ./rootfs_overlay_demo --tenant-token " + echo "Usage: $(basename $0) --output-dir ./input/rootfs_overlay_demo --tenant-token " exit 1 ;; esac @@ -57,20 +57,15 @@ if [ -e ${output_dir} ]; then sudo chown -R $(id -u) ${output_dir} sudo chgrp -R $(id -g) ${output_dir} fi -mkdir -p ${output_dir}/etc/mender -cat <<- EOF > ${output_dir}/etc/mender/mender.conf +mkdir -p ${root_dir}/input/resources +cat <<- EOF > ${root_dir}/input/resources/mender.conf { - "InventoryPollIntervalSeconds": 5, - "RetryPollIntervalSeconds": 30, "ServerURL": "https://hosted.mender.io/", - "TenantToken": "${tenant_token}", - "UpdatePollIntervalSeconds": 5 + "TenantToken": "${tenant_token}" } EOF -chmod 600 ${output_dir}/etc/mender/mender.conf - sudo chown -R 0 ${output_dir} sudo chgrp -R 0 ${output_dir} -echo "Configuration file for using Hosted Mender written to: ${output_dir}/etc/mender" +echo "Configuration file for using Hosted Mender written to: ${root_dir}/input/resources/mender.conf" diff --git a/scripts/bootstrap-rootfs-overlay-production-server.sh b/scripts/bootstrap-rootfs-overlay-production-server.sh index 212468c17a..dd21786207 100755 --- a/scripts/bootstrap-rootfs-overlay-production-server.sh +++ b/scripts/bootstrap-rootfs-overlay-production-server.sh @@ -61,29 +61,25 @@ if [ -e ${output_dir} ]; then sudo chown -R $(id -u) ${output_dir} sudo chgrp -R $(id -g) ${output_dir} fi -mkdir -p ${output_dir}/etc/mender -cat <<- EOF > ${output_dir}/etc/mender/mender.conf + +mkdir -p ${root_dir}/input/resources +cat <<- EOF > ${root_dir}/input/resources/mender.conf { - "InventoryPollIntervalSeconds": 5, - "RetryPollIntervalSeconds": 30, "ServerURL": "${server_url}", EOF if [ -n "${server_cert}" ]; then - cat <<- EOF >> ${output_dir}/etc/mender/mender.conf - "ServerCertificate": "/etc/mender/server.crt", + cat <<- EOF >> ${root_dir}/input/resources/mender.conf + "ServerCertificate": "/etc/mender/server.crt" EOF cp -f "${server_cert}" ${output_dir}/etc/mender/server.crt fi -cat <<- EOF >> ${output_dir}/etc/mender/mender.conf - "UpdatePollIntervalSeconds": 5 +cat <<- EOF >> ${root_dir}/input/resources/mender.conf } EOF -chmod 600 ${output_dir}/etc/mender/mender.conf - sudo chown -R 0 ${output_dir} sudo chgrp -R 0 ${output_dir} -echo "Configuration file for using Production Mender Server written to: ${output_dir}/etc/mender" +echo "Configuration file for using Production Mender Server written to: ${root_dir}/input/resources/mender.conf" diff --git a/scripts/linkbot/common.js b/scripts/linkbot/common.js index 402024ab8e..6603031f50 100644 --- a/scripts/linkbot/common.js +++ b/scripts/linkbot/common.js @@ -1,18 +1,18 @@ -const fs = require('fs') +const fs = require('fs'); const updateURLLink = (newLine, target) => { try { const data = fs.readFileSync('../test/run-tests.sh', 'utf8') - .replace(RegExp(`## Auto-update\n${target}=.*`), `## Auto-update\n${newLine}`) + .replace(RegExp(`## Auto-update\n${target}=.*`), `## Auto-update\n${newLine}`); fs.writeFile('../test/run-tests.sh', data, (err, data) => { if (err) { - console.error(err) + console.error(err); } - }) + }); } catch (err) { - console.error(err) - process.exit(1) + console.error(err); + process.exit(1); } -} +}; module.exports = { updateURLLink -} +}; diff --git a/scripts/linkbot/monitor-raspbian-os.js b/scripts/linkbot/monitor-raspbian-os.js new file mode 100644 index 0000000000..c684b329bc --- /dev/null +++ b/scripts/linkbot/monitor-raspbian-os.js @@ -0,0 +1,115 @@ +const jsdom = require("jsdom"); +const { JSDOM } = jsdom; +const fs = require("fs"); +const { updateURLLink } = require("./common"); + +const target = "RASPBIAN_URL"; + +// Read the input file, and parse the variable input +try { + const data = fs + .readFileSync("../../.gitlab-ci.yml", "utf8") + .split("\n") + .filter((line) => line.match(`${target}: .*`)); + var line = data[0]; + console.log(line); + var reg = + "raspios_lite_armhf-(?[0-9]{4}-[0-9]{2}-[0-9]{1,2})/(?[0-9]{4}-[0-9]{2}-[0-9]{1,2}).*$"; + var m = line.match( + ".*: \"(?[a-zA-Z-://._]*)(?raspios_lite_armhf-[0-9]{4}-[0-9]{2}-[0-9]{1,2})/(?[0-9]{4}-[0-9]{2}-[0-9]{1,2}).*\"$" + ); + console.log(m); + var url = m.groups.url; + var imageName = m.groups.imageName; + var updated = m.groups.updated; +} catch (err) { + console.error(err); + process.exit(1); +} + + +function getNewRaspbian(target, url, separator) { + console.log("getNewRaspbian"); + console.log(`${url}`); + return JSDOM.fromURL(url, {}).then((dom) => { + var document = dom.window.document; + var refs = document.getElementsByTagName("a"); + var test = Array.from(refs) + .filter((ref) => ref.textContent.match("raspios-.*-lite.*\.img\.(zip|xz)$")) + .reduce((acc, element) => { + acc.push(element.textContent.match("raspios-.*-lite.*\.img\.(zip|xz)$").input); + return acc; + }, [])[0]; + console.log(`Test var: ${test}`); + const imageName = test.substring(0, test.lastIndexOf('.img')); + if (separator) { + return { + replacementLine: `${target}${separator} \"${url}/${test}\"`, + imageName: imageName, + }; + } + return { replacementLine: `${target}=\"${url}/${test}\"`, imageName: imageName }; + }); +} + + +function updateGitlabCILink(newLine, newURL, newName) { + try { + const data = fs + .readFileSync("../../.gitlab-ci.yml", "utf8") + .replace( + RegExp(` *## Auto-update\n *${newURL}:.*\n *RASPBIAN_NAME: .*`), + ` ## Auto-update\n ${newLine}\n RASPBIAN_NAME: ${newName}` + ); + fs.writeFile("../../.gitlab-ci.yml", data, (err, data) => { + if (err) { + console.error(err); + } + }); + } catch (err) { + console.error(err); + process.exit(1); + } +} + +JSDOM.fromURL(url, {}).then((dom) => { + var document = dom.window.document; + var table = document.getElementsByTagName("table"); + var rows = table[0].rows; + var matches = []; + for (var i = 0; i < rows.length; i++) { + var rowText = rows[i].textContent; + var regMatch = rowText.match(reg); + if (regMatch) { + matches.push(regMatch); + } + } + // Sort the accumulated matches + matches.sort(function (a, b) { + let al = Date.parse(a.groups.date); + let bl = Date.parse(b.groups.date); + if (al == bl) { + let ad = Date.parse(a.groups.updated); + let bd = Date.parse(b.groups.updated); + return bd - ad; + } + return bl - al; + }); + var matchOn = matches[0].input.split("/")[0]; + if (matchOn !== imageName) { + // We also need to extract the new image name from the index folder, as + // dated folders contain images with different dates in them o_O + console.error("We've got a new release! \\o/"); + + // Update the GitlabCI link unconditionally + console.log("Updating the GitlabCI Link"); + var newVar = getNewRaspbian( + "RASPBIAN_URL", + `${url}${matches[0].input.split(" ")[0].split("/").slice(0, -1)}`, + (separator = ":") + ).then(({ replacementLine, imageName }) => { + console.log(`New release: ${replacementLine}`); + updateGitlabCILink(replacementLine, "RASPBIAN_URL", imageName); + }); + } +}); diff --git a/scripts/linkbot/monitor-rpi.js b/scripts/linkbot/monitor-rpi.js index 3deef387b6..c9760973d8 100644 --- a/scripts/linkbot/monitor-rpi.js +++ b/scripts/linkbot/monitor-rpi.js @@ -1,40 +1,40 @@ const jsdom = require("jsdom"); const { JSDOM } = jsdom; -const fs = require('fs') +const fs = require('fs'); const { updateURLLink } = require('./common'); -const target = "RASPBIAN_IMAGE_URL" +const target = "RASPBIAN_IMAGE_URL"; // Read the input file, and parse the variable input try { const data = fs.readFileSync('../test/run-tests.sh', 'utf8') .split('\n') - .filter(line => line.match(`${target}=.*`)) - var line = data[0] - console.log(line) - var reg = "raspbian_lite-(?[0-9]{4}-[0-9]{2}-[0-9]{1,2})/(?[0-9]{4}-[0-9]{2}-[0-9]{1,2}).*$" - var m = line.match(".*=\"(?[a-zA-Z-://\._]*)(?raspbian_lite-[0-9]{4}-[0-9]{2}-[0-9]{1,2})/(?[0-9]{4}-[0-9]{2}-[0-9]{1,2}).*$") - console.log(m) - var url = m.groups.url - var imageName = m.groups.imageName - var updated = m.groups.updated + .filter(line => line.match(`${target}=.*`)); + var line = data[0]; + console.log(line); + var reg = "raspbian_lite-(?[0-9]{4}-[0-9]{2}-[0-9]{1,2})/(?[0-9]{4}-[0-9]{2}-[0-9]{1,2}).*$"; + var m = line.match(".*=\"(?[a-zA-Z-://\._]*)(?raspbian_lite-[0-9]{4}-[0-9]{2}-[0-9]{1,2})/(?[0-9]{4}-[0-9]{2}-[0-9]{1,2}).*$"); + console.log(m); + var url = m.groups.url; + var imageName = m.groups.imageName; + var updated = m.groups.updated; } catch (err) { - console.error(err) - process.exit(1) + console.error(err); + process.exit(1); } function getNewRaspbian(url) { - console.log("getNewRaspbian") - console.log(`${url}`) + console.log("getNewRaspbian"); + console.log(`${url}`); return JSDOM.fromURL(url, {}).then(dom => { var document = dom.window.document; var refs = document.getElementsByTagName("a"); var test = Array.from(refs) .filter(ref => ref.textContent.match(("raspbian-.*-lite.*\.zip$"))) .reduce((acc, element) => { - acc.push(element.textContent.match("raspbian-.*-lite.*\.zip$").input) - return acc - }, [])[0] + acc.push(element.textContent.match("raspbian-.*-lite.*\.zip$").input); + return acc; + }, [])[0]; return `${target}=\"${url}/${test}\"`; }); } @@ -63,15 +63,15 @@ JSDOM.fromURL(url, {}).then(dom => { } return bl - al; }); - var matchOn = matches[0].input.split("/")[0] + var matchOn = matches[0].input.split("/")[0]; if (matchOn !== imageName) { // We also need to extract the new image name from the index folder, as // dated folders contain images with different dates in them o_O console.error("We've got a new release! \\o/"); var newVar = getNewRaspbian(`${url}${matches[0].input.split(" ")[0].split("/").slice(0, -1)}`) .then(res => { - console.log(`New release: ${res}`) - updateURLLink(res, target) - }) + console.log(`New release: ${res}`); + updateURLLink(res, target); + }); } }); diff --git a/scripts/linkbot/monitor-ub-server.js b/scripts/linkbot/monitor-ub-server.js index 39db954c30..dc40016b62 100644 --- a/scripts/linkbot/monitor-ub-server.js +++ b/scripts/linkbot/monitor-ub-server.js @@ -1,23 +1,24 @@ const jsdom = require("jsdom"); const { JSDOM } = jsdom; -const fs = require('fs') +const fs = require('fs'); const { updateURLLink } = require('./common'); -const url = "http://cdimage.ubuntu.com/ubuntu/releases/" -const reg = ".*(?[0-9]{2})\.04\.?(?[0-9]{1})?.*" +const target = "UBUNTU_SERVER_RPI_IMAGE_URL"; +const url = "http://cdimage.ubuntu.com/ubuntu/releases/"; +const reg = ".*(?[0-9][02468])\.04\.?(?[0-9]{1})?.*"; // Read the input file, and parse the variable input try { const data = fs.readFileSync('../test/run-tests.sh', 'utf8') .split('\n') - .filter(line => line.match("UBUNTU_SERVER_RPI_IMAGE_URL=.*")) - var line = data[0] - var m = line.match(`.*=\"${reg}\"`) - var imageName = m.groups.release - var minor = m.groups.minor || 0 + .filter(line => line.match(`${target}=.*`)); + var line = data[0]; + var m = line.match(`.*=\"${reg}\"`); + var imageName = m.groups.release; + var minor = m.groups.minor || 0; } catch (err) { - console.error(err) - process.exit(1) + console.error(err); + process.exit(1); } JSDOM.fromURL(url, {}).then(dom => { @@ -25,22 +26,43 @@ JSDOM.fromURL(url, {}).then(dom => { var refs = document.getElementsByTagName("a"); var matches = Array.from(refs) .filter(ref => ref.textContent.match(reg)) + .filter(ref => !ref.textContent.match(reg).groups.minor) // Ignore the minor number .reduce((acc, ref) => { - acc.push(ref.textContent.match(reg)) - return acc + acc.push(ref.textContent.match(reg)); + return acc; }, []) .sort((a,b) => { - return parseInt(b.groups.release) - parseInt(a.groups.release) || parseFloat(b.groups.minor) - parseFloat(a.groups.minor) - }) - var matchOn = matches[0].input - if (matchOn !== imageName) { - console.log("We've got a new release! \\o/"); - var newLine = "" - if (matches[0].groups.minor) { - newLine = `UBUNTU_SERVER_RPI_IMAGE_URL=\"${url}${matches[0].groups.release}.04.${matches[0].groups.minor}/release/ubuntu-${matches[0].groups.release}.04.${matches[0].groups.minor}-preinstalled-server-armhf+raspi.img.xz\"` - } else { - newLine = `UBUNTU_SERVER_RPI_IMAGE_URL=\"${url}${matches[0].groups.release}.04/release/ubuntu-${matches[0].groups.release}.04-preinstalled-server-armhf+raspi.img.xz\"` + return parseInt(b.groups.release) - parseInt(a.groups.release); + }); + var matchOn = matches[0].input; + + return matchOn; + + // Get the release image url from the releases (sub)-page + // const url = "http://cdimage.ubuntu.com/ubuntu/releases/" +}).then(releaseVersion => { + var releasedVersion = releaseVersion.replace(/\s/g, "").replace(/\//g, ""); + JSDOM.fromURL(`${url}${releasedVersion}/release/`, {}).then(dom => { + var document = dom.window.document; + var refs = document.getElementsByTagName("a"); + const match = Array.from(refs).find(ref => ref.href.match(`.*ubuntu-${releasedVersion}\.?[0-9]+-preinstalled-server-armhf.*\.img\.xz$`)); + if (match) { + console.log(`Ubuntu server image has a new release: ${match}`); + updateURLLink(`${target}=\"${match}\"`, target); } - updateURLLink(newLine) - } -}); + }) + .catch(err => { + console.log('Failed to get the update URL'); + console.log(`${url}${releasedVersion}/release/`); + JSDOM.fromURL(`${url}${releasedVersion}/beta/`, {}) + .then(dom => { + console.log('Only the beta is out still'); + }); + }); +}) + .catch(err => { + console.log('Failed to get the URL'); + console.log(url); + console.log(err); + throw err; + }); diff --git a/scripts/linkbot/package-lock.json b/scripts/linkbot/package-lock.json index 4905072357..6f9c63c533 100644 --- a/scripts/linkbot/package-lock.json +++ b/scripts/linkbot/package-lock.json @@ -6,15 +6,15 @@ "": { "dependencies": { "bent": "^7.3.12", - "jsdom": "^16.6.0" + "jsdom": "^19.0.0" } }, "node_modules/@tootallnate/once": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", - "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", + "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", "engines": { - "node": ">= 6" + "node": ">= 10" } }, "node_modules/abab": { @@ -103,9 +103,9 @@ } }, "node_modules/cssom": { - "version": "0.4.4", - "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.4.4.tgz", - "integrity": "sha512-p3pvU7r1MyyqbTk+WbNJIgJjG2VmTIaB10rI93LzVPrmDJKkzKYMtxxyAvQXR/NS6otuzveI7+7BBq3SjBS2mw==" + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz", + "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==" }, "node_modules/cssstyle": { "version": "2.3.0", @@ -124,16 +124,16 @@ "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==" }, "node_modules/data-urls": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz", - "integrity": "sha512-X5eWTSXO/BJmpdIKCRuKUgSCgAN0OwliVK3yPKbwIWU1Tdw5BRajxlzMidvh+gwko9AfQ9zIj52pzF91Q3YAvQ==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.1.tgz", + "integrity": "sha512-Ds554NeT5Gennfoo9KN50Vh6tpgtvYEwraYjejXnyTpu1C7oXKxdFk75REooENHE8ndTVOJuv+BEs4/J/xcozw==", "dependencies": { "abab": "^2.0.3", - "whatwg-mimetype": "^2.3.0", - "whatwg-url": "^8.0.0" + "whatwg-mimetype": "^3.0.0", + "whatwg-url": "^10.0.0" }, "engines": { - "node": ">=10" + "node": ">=12" } }, "node_modules/debug": { @@ -153,9 +153,9 @@ } }, "node_modules/decimal.js": { - "version": "10.2.1", - "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.2.1.tgz", - "integrity": "sha512-KaL7+6Fw6i5A2XSnsbhm/6B+NuEA7TZ4vqxnd5tXz9sbKtrN9Srj8ab4vKVdK8YAqZO9P1kg45Y6YLoduPf+kw==" + "version": "10.3.1", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.3.1.tgz", + "integrity": "sha512-V0pfhfr8suzyPGOx3nmq4aHqabehUZn6Ch9kyFpV79TGDTWFmHqUqXdabR7QHqxzrYolF4+tVmJhUG4OURg5dQ==" }, "node_modules/deep-is": { "version": "0.1.3", @@ -171,22 +171,14 @@ } }, "node_modules/domexception": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/domexception/-/domexception-2.0.1.tgz", - "integrity": "sha512-yxJ2mFy/sibVQlu5qHjOkf9J3K6zgmCxgJ94u2EdvDOV09H+32LtRswEcUsmUWN72pVLOEnTSRaIVVzVQgS0dg==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/domexception/-/domexception-4.0.0.tgz", + "integrity": "sha512-A2is4PLG+eeSfoTMA95/s4pvAoSo2mKtiM5jlHkAVewmiO8ISFTFKZjH7UAM1Atli/OT/7JHOrJRJiMKUZKYBw==", "dependencies": { - "webidl-conversions": "^5.0.0" + "webidl-conversions": "^7.0.0" }, "engines": { - "node": ">=8" - } - }, - "node_modules/domexception/node_modules/webidl-conversions": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-5.0.0.tgz", - "integrity": "sha512-VlZwKPCkYKxQgeSbH5EyngOmRp7Ww7I9rQLERETtf5ofd9pGeswWiOtogpEO850jziPRarreGxn5QIiTqpb2wA==", - "engines": { - "node": ">=8" + "node": ">=12" } }, "node_modules/escodegen": { @@ -244,9 +236,9 @@ "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=" }, "node_modules/form-data": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", - "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", @@ -257,22 +249,22 @@ } }, "node_modules/html-encoding-sniffer": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz", - "integrity": "sha512-D5JbOMBIR/TVZkubHT+OyT2705QvogUW4IBn6nHd756OwieSF9aDYFj4dv6HHEVGYbHaLETa3WggZYWWMyy3ZQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz", + "integrity": "sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==", "dependencies": { - "whatwg-encoding": "^1.0.5" + "whatwg-encoding": "^2.0.0" }, "engines": { - "node": ">=10" + "node": ">=12" } }, "node_modules/http-proxy-agent": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", - "integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", + "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", "dependencies": { - "@tootallnate/once": "1", + "@tootallnate/once": "2", "agent-base": "6", "debug": "4" }, @@ -293,11 +285,11 @@ } }, "node_modules/iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", "dependencies": { - "safer-buffer": ">= 2.1.2 < 3" + "safer-buffer": ">= 2.1.2 < 3.0.0" }, "engines": { "node": ">=0.10.0" @@ -317,22 +309,22 @@ } }, "node_modules/jsdom": { - "version": "16.6.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.6.0.tgz", - "integrity": "sha512-Ty1vmF4NHJkolaEmdjtxTfSfkdb8Ywarwf63f+F8/mDD1uLSSWDxDuMiZxiPhwunLrn9LOSVItWj4bLYsLN3Dg==", + "version": "19.0.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-19.0.0.tgz", + "integrity": "sha512-RYAyjCbxy/vri/CfnjUWJQQtZ3LKlLnDqj+9XLNnJPgEGeirZs3hllKR20re8LUZ6o1b1X4Jat+Qd26zmP41+A==", "dependencies": { "abab": "^2.0.5", - "acorn": "^8.2.4", + "acorn": "^8.5.0", "acorn-globals": "^6.0.0", - "cssom": "^0.4.4", + "cssom": "^0.5.0", "cssstyle": "^2.3.0", - "data-urls": "^2.0.0", - "decimal.js": "^10.2.1", - "domexception": "^2.0.1", + "data-urls": "^3.0.1", + "decimal.js": "^10.3.1", + "domexception": "^4.0.0", "escodegen": "^2.0.0", - "form-data": "^3.0.0", - "html-encoding-sniffer": "^2.0.1", - "http-proxy-agent": "^4.0.1", + "form-data": "^4.0.0", + "html-encoding-sniffer": "^3.0.0", + "http-proxy-agent": "^5.0.0", "https-proxy-agent": "^5.0.0", "is-potential-custom-element-name": "^1.0.1", "nwsapi": "^2.2.0", @@ -341,16 +333,16 @@ "symbol-tree": "^3.2.4", "tough-cookie": "^4.0.0", "w3c-hr-time": "^1.0.2", - "w3c-xmlserializer": "^2.0.0", - "webidl-conversions": "^6.1.0", - "whatwg-encoding": "^1.0.5", - "whatwg-mimetype": "^2.3.0", - "whatwg-url": "^8.5.0", - "ws": "^7.4.5", - "xml-name-validator": "^3.0.0" + "w3c-xmlserializer": "^3.0.0", + "webidl-conversions": "^7.0.0", + "whatwg-encoding": "^2.0.0", + "whatwg-mimetype": "^3.0.0", + "whatwg-url": "^10.0.0", + "ws": "^8.2.3", + "xml-name-validator": "^4.0.0" }, "engines": { - "node": ">=10" + "node": ">=12" }, "peerDependencies": { "canvas": "^2.5.0" @@ -362,9 +354,9 @@ } }, "node_modules/jsdom/node_modules/acorn": { - "version": "8.2.4", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.2.4.tgz", - "integrity": "sha512-Ibt84YwBDDA890eDiDCEqcbwvHlBvzzDkU2cGBBDDI1QWT12jTiXIOn2CIw5KK4i6N5Z2HUxwYjzriDyqaqqZg==", + "version": "8.5.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.5.0.tgz", + "integrity": "sha512-yXbYeFy+jUuYd3/CDcg2NkIYE991XYX/bje7LmjJigUciaeO1JR4XxXgCIV1/Zc/dRuFEyw1L0pbA+qynJkW5Q==", "bin": { "acorn": "bin/acorn" }, @@ -384,25 +376,20 @@ "node": ">= 0.8.0" } }, - "node_modules/lodash": { - "version": "4.17.20", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" - }, "node_modules/mime-db": { - "version": "1.47.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.47.0.tgz", - "integrity": "sha512-QBmA/G2y+IfeS4oktet3qRZ+P5kPhCKRXxXnQEudYqUaEioAU1/Lq2us3D/t1Jfo4hE9REQPrbB7K5sOczJVIw==", + "version": "1.49.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.49.0.tgz", + "integrity": "sha512-CIc8j9URtOVApSFCQIF+VBkX1RwXp/oMMOrqdyXSBXq5RWNEsRfyj1kiRnQgmNXmHxPoFIxOroKA3zcU9P+nAA==", "engines": { "node": ">= 0.6" } }, "node_modules/mime-types": { - "version": "2.1.30", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.30.tgz", - "integrity": "sha512-crmjA4bLtR8m9qLpHvgxSChT+XoSlZi8J4n/aIdn3z92e/U47Z0V/yl+Wh9W046GgFVAmoNR/fmdbZYcSSIUeg==", + "version": "2.1.32", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.32.tgz", + "integrity": "sha512-hJGaVS4G4c9TSMYh2n6SQAGrC4RnfU+daP8G7cSCmaqNjiOoUY0VHCMS42pxnQmVF1GWwFhbHWn3RIxCqTmZ9A==", "dependencies": { - "mime-db": "1.47.0" + "mime-db": "1.49.0" }, "engines": { "node": ">= 0.6" @@ -504,14 +491,14 @@ } }, "node_modules/tr46": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-2.0.2.tgz", - "integrity": "sha512-3n1qG+/5kg+jrbTzwAykB5yRYtQCTqOGKq5U5PE3b0a1/mzo6snDhjGS0zJVJunO0NrT3Dg1MLy5TjWP/UJppg==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", + "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==", "dependencies": { "punycode": "^2.1.1" }, "engines": { - "node": ">=8" + "node": ">=12" } }, "node_modules/type-check": { @@ -542,48 +529,53 @@ } }, "node_modules/w3c-xmlserializer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-2.0.0.tgz", - "integrity": "sha512-4tzD0mF8iSiMiNs30BiLO3EpfGLZUT2MSX/G+o7ZywDzliWQ3OPtTZ0PTC3B3ca1UAf4cJMHB+2Bf56EriJuRA==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-3.0.0.tgz", + "integrity": "sha512-3WFqGEgSXIyGhOmAFtlicJNMjEps8b1MG31NCA0/vOF9+nKMUW1ckhi9cnNHmf88Rzw5V+dwIwsm2C7X8k9aQg==", "dependencies": { - "xml-name-validator": "^3.0.0" + "xml-name-validator": "^4.0.0" }, "engines": { - "node": ">=10" + "node": ">=12" } }, "node_modules/webidl-conversions": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-6.1.0.tgz", - "integrity": "sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", "engines": { - "node": ">=10.4" + "node": ">=12" } }, "node_modules/whatwg-encoding": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz", - "integrity": "sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz", + "integrity": "sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==", "dependencies": { - "iconv-lite": "0.4.24" + "iconv-lite": "0.6.3" + }, + "engines": { + "node": ">=12" } }, "node_modules/whatwg-mimetype": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz", - "integrity": "sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==" + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz", + "integrity": "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==", + "engines": { + "node": ">=12" + } }, "node_modules/whatwg-url": { - "version": "8.5.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-8.5.0.tgz", - "integrity": "sha512-fy+R77xWv0AiqfLl4nuGUlQ3/6b5uNfQ4WAbGQVMYshCTCCPK9psC1nWh3XHuxGVCtlcDDQPQW1csmmIQo+fwg==", + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-10.0.0.tgz", + "integrity": "sha512-CLxxCmdUby142H5FZzn4D8ikO1cmypvXVQktsgosNy4a4BHrDHeciBBGZhb0bNoR5/MltoCatso+vFjjGx8t0w==", "dependencies": { - "lodash": "^4.7.0", - "tr46": "^2.0.2", - "webidl-conversions": "^6.1.0" + "tr46": "^3.0.0", + "webidl-conversions": "^7.0.0" }, "engines": { - "node": ">=10" + "node": ">=12" } }, "node_modules/word-wrap": { @@ -595,11 +587,11 @@ } }, "node_modules/ws": { - "version": "7.4.5", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.5.tgz", - "integrity": "sha512-xzyu3hFvomRfXKH8vOFMU3OguG6oOvhXMo3xsGy3xWExqaM2dxBbVxuD99O7m3ZUFMvvscsZDqxfgMaRr/Nr1g==", + "version": "8.2.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.2.3.tgz", + "integrity": "sha512-wBuoj1BDpC6ZQ1B7DWQBYVLphPWkm8i9Y0/3YdHjHKHiohOJ1ws+3OccDWtH+PoC9DZD5WOTrJvNbWvjS6JWaA==", "engines": { - "node": ">=8.3.0" + "node": ">=10.0.0" }, "peerDependencies": { "bufferutil": "^4.0.1", @@ -615,9 +607,12 @@ } }, "node_modules/xml-name-validator": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz", - "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==" + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-4.0.0.tgz", + "integrity": "sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw==", + "engines": { + "node": ">=12" + } }, "node_modules/xmlchars": { "version": "2.2.0", @@ -627,9 +622,9 @@ }, "dependencies": { "@tootallnate/once": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", - "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", + "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==" }, "abab": { "version": "2.0.5", @@ -702,9 +697,9 @@ } }, "cssom": { - "version": "0.4.4", - "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.4.4.tgz", - "integrity": "sha512-p3pvU7r1MyyqbTk+WbNJIgJjG2VmTIaB10rI93LzVPrmDJKkzKYMtxxyAvQXR/NS6otuzveI7+7BBq3SjBS2mw==" + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz", + "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==" }, "cssstyle": { "version": "2.3.0", @@ -722,13 +717,13 @@ } }, "data-urls": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz", - "integrity": "sha512-X5eWTSXO/BJmpdIKCRuKUgSCgAN0OwliVK3yPKbwIWU1Tdw5BRajxlzMidvh+gwko9AfQ9zIj52pzF91Q3YAvQ==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.1.tgz", + "integrity": "sha512-Ds554NeT5Gennfoo9KN50Vh6tpgtvYEwraYjejXnyTpu1C7oXKxdFk75REooENHE8ndTVOJuv+BEs4/J/xcozw==", "requires": { "abab": "^2.0.3", - "whatwg-mimetype": "^2.3.0", - "whatwg-url": "^8.0.0" + "whatwg-mimetype": "^3.0.0", + "whatwg-url": "^10.0.0" } }, "debug": { @@ -740,9 +735,9 @@ } }, "decimal.js": { - "version": "10.2.1", - "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.2.1.tgz", - "integrity": "sha512-KaL7+6Fw6i5A2XSnsbhm/6B+NuEA7TZ4vqxnd5tXz9sbKtrN9Srj8ab4vKVdK8YAqZO9P1kg45Y6YLoduPf+kw==" + "version": "10.3.1", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.3.1.tgz", + "integrity": "sha512-V0pfhfr8suzyPGOx3nmq4aHqabehUZn6Ch9kyFpV79TGDTWFmHqUqXdabR7QHqxzrYolF4+tVmJhUG4OURg5dQ==" }, "deep-is": { "version": "0.1.3", @@ -755,18 +750,11 @@ "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" }, "domexception": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/domexception/-/domexception-2.0.1.tgz", - "integrity": "sha512-yxJ2mFy/sibVQlu5qHjOkf9J3K6zgmCxgJ94u2EdvDOV09H+32LtRswEcUsmUWN72pVLOEnTSRaIVVzVQgS0dg==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/domexception/-/domexception-4.0.0.tgz", + "integrity": "sha512-A2is4PLG+eeSfoTMA95/s4pvAoSo2mKtiM5jlHkAVewmiO8ISFTFKZjH7UAM1Atli/OT/7JHOrJRJiMKUZKYBw==", "requires": { - "webidl-conversions": "^5.0.0" - }, - "dependencies": { - "webidl-conversions": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-5.0.0.tgz", - "integrity": "sha512-VlZwKPCkYKxQgeSbH5EyngOmRp7Ww7I9rQLERETtf5ofd9pGeswWiOtogpEO850jziPRarreGxn5QIiTqpb2wA==" - } + "webidl-conversions": "^7.0.0" } }, "escodegen": { @@ -802,9 +790,9 @@ "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=" }, "form-data": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", - "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", "requires": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", @@ -812,19 +800,19 @@ } }, "html-encoding-sniffer": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz", - "integrity": "sha512-D5JbOMBIR/TVZkubHT+OyT2705QvogUW4IBn6nHd756OwieSF9aDYFj4dv6HHEVGYbHaLETa3WggZYWWMyy3ZQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz", + "integrity": "sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==", "requires": { - "whatwg-encoding": "^1.0.5" + "whatwg-encoding": "^2.0.0" } }, "http-proxy-agent": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", - "integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", + "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", "requires": { - "@tootallnate/once": "1", + "@tootallnate/once": "2", "agent-base": "6", "debug": "4" } @@ -839,11 +827,11 @@ } }, "iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", "requires": { - "safer-buffer": ">= 2.1.2 < 3" + "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "is-potential-custom-element-name": { @@ -857,22 +845,22 @@ "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==" }, "jsdom": { - "version": "16.6.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.6.0.tgz", - "integrity": "sha512-Ty1vmF4NHJkolaEmdjtxTfSfkdb8Ywarwf63f+F8/mDD1uLSSWDxDuMiZxiPhwunLrn9LOSVItWj4bLYsLN3Dg==", + "version": "19.0.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-19.0.0.tgz", + "integrity": "sha512-RYAyjCbxy/vri/CfnjUWJQQtZ3LKlLnDqj+9XLNnJPgEGeirZs3hllKR20re8LUZ6o1b1X4Jat+Qd26zmP41+A==", "requires": { "abab": "^2.0.5", - "acorn": "^8.2.4", + "acorn": "^8.5.0", "acorn-globals": "^6.0.0", - "cssom": "^0.4.4", + "cssom": "^0.5.0", "cssstyle": "^2.3.0", - "data-urls": "^2.0.0", - "decimal.js": "^10.2.1", - "domexception": "^2.0.1", + "data-urls": "^3.0.1", + "decimal.js": "^10.3.1", + "domexception": "^4.0.0", "escodegen": "^2.0.0", - "form-data": "^3.0.0", - "html-encoding-sniffer": "^2.0.1", - "http-proxy-agent": "^4.0.1", + "form-data": "^4.0.0", + "html-encoding-sniffer": "^3.0.0", + "http-proxy-agent": "^5.0.0", "https-proxy-agent": "^5.0.0", "is-potential-custom-element-name": "^1.0.1", "nwsapi": "^2.2.0", @@ -881,19 +869,19 @@ "symbol-tree": "^3.2.4", "tough-cookie": "^4.0.0", "w3c-hr-time": "^1.0.2", - "w3c-xmlserializer": "^2.0.0", - "webidl-conversions": "^6.1.0", - "whatwg-encoding": "^1.0.5", - "whatwg-mimetype": "^2.3.0", - "whatwg-url": "^8.5.0", - "ws": "^7.4.5", - "xml-name-validator": "^3.0.0" + "w3c-xmlserializer": "^3.0.0", + "webidl-conversions": "^7.0.0", + "whatwg-encoding": "^2.0.0", + "whatwg-mimetype": "^3.0.0", + "whatwg-url": "^10.0.0", + "ws": "^8.2.3", + "xml-name-validator": "^4.0.0" }, "dependencies": { "acorn": { - "version": "8.2.4", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.2.4.tgz", - "integrity": "sha512-Ibt84YwBDDA890eDiDCEqcbwvHlBvzzDkU2cGBBDDI1QWT12jTiXIOn2CIw5KK4i6N5Z2HUxwYjzriDyqaqqZg==" + "version": "8.5.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.5.0.tgz", + "integrity": "sha512-yXbYeFy+jUuYd3/CDcg2NkIYE991XYX/bje7LmjJigUciaeO1JR4XxXgCIV1/Zc/dRuFEyw1L0pbA+qynJkW5Q==" } } }, @@ -906,22 +894,17 @@ "type-check": "~0.3.2" } }, - "lodash": { - "version": "4.17.20", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" - }, "mime-db": { - "version": "1.47.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.47.0.tgz", - "integrity": "sha512-QBmA/G2y+IfeS4oktet3qRZ+P5kPhCKRXxXnQEudYqUaEioAU1/Lq2us3D/t1Jfo4hE9REQPrbB7K5sOczJVIw==" + "version": "1.49.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.49.0.tgz", + "integrity": "sha512-CIc8j9URtOVApSFCQIF+VBkX1RwXp/oMMOrqdyXSBXq5RWNEsRfyj1kiRnQgmNXmHxPoFIxOroKA3zcU9P+nAA==" }, "mime-types": { - "version": "2.1.30", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.30.tgz", - "integrity": "sha512-crmjA4bLtR8m9qLpHvgxSChT+XoSlZi8J4n/aIdn3z92e/U47Z0V/yl+Wh9W046GgFVAmoNR/fmdbZYcSSIUeg==", + "version": "2.1.32", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.32.tgz", + "integrity": "sha512-hJGaVS4G4c9TSMYh2n6SQAGrC4RnfU+daP8G7cSCmaqNjiOoUY0VHCMS42pxnQmVF1GWwFhbHWn3RIxCqTmZ9A==", "requires": { - "mime-db": "1.47.0" + "mime-db": "1.49.0" } }, "ms": { @@ -1002,9 +985,9 @@ } }, "tr46": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-2.0.2.tgz", - "integrity": "sha512-3n1qG+/5kg+jrbTzwAykB5yRYtQCTqOGKq5U5PE3b0a1/mzo6snDhjGS0zJVJunO0NrT3Dg1MLy5TjWP/UJppg==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", + "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==", "requires": { "punycode": "^2.1.1" } @@ -1031,39 +1014,38 @@ } }, "w3c-xmlserializer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-2.0.0.tgz", - "integrity": "sha512-4tzD0mF8iSiMiNs30BiLO3EpfGLZUT2MSX/G+o7ZywDzliWQ3OPtTZ0PTC3B3ca1UAf4cJMHB+2Bf56EriJuRA==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-3.0.0.tgz", + "integrity": "sha512-3WFqGEgSXIyGhOmAFtlicJNMjEps8b1MG31NCA0/vOF9+nKMUW1ckhi9cnNHmf88Rzw5V+dwIwsm2C7X8k9aQg==", "requires": { - "xml-name-validator": "^3.0.0" + "xml-name-validator": "^4.0.0" } }, "webidl-conversions": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-6.1.0.tgz", - "integrity": "sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w==" + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==" }, "whatwg-encoding": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz", - "integrity": "sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz", + "integrity": "sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==", "requires": { - "iconv-lite": "0.4.24" + "iconv-lite": "0.6.3" } }, "whatwg-mimetype": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz", - "integrity": "sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==" + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz", + "integrity": "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==" }, "whatwg-url": { - "version": "8.5.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-8.5.0.tgz", - "integrity": "sha512-fy+R77xWv0AiqfLl4nuGUlQ3/6b5uNfQ4WAbGQVMYshCTCCPK9psC1nWh3XHuxGVCtlcDDQPQW1csmmIQo+fwg==", + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-10.0.0.tgz", + "integrity": "sha512-CLxxCmdUby142H5FZzn4D8ikO1cmypvXVQktsgosNy4a4BHrDHeciBBGZhb0bNoR5/MltoCatso+vFjjGx8t0w==", "requires": { - "lodash": "^4.7.0", - "tr46": "^2.0.2", - "webidl-conversions": "^6.1.0" + "tr46": "^3.0.0", + "webidl-conversions": "^7.0.0" } }, "word-wrap": { @@ -1072,15 +1054,15 @@ "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==" }, "ws": { - "version": "7.4.5", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.5.tgz", - "integrity": "sha512-xzyu3hFvomRfXKH8vOFMU3OguG6oOvhXMo3xsGy3xWExqaM2dxBbVxuD99O7m3ZUFMvvscsZDqxfgMaRr/Nr1g==", + "version": "8.2.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.2.3.tgz", + "integrity": "sha512-wBuoj1BDpC6ZQ1B7DWQBYVLphPWkm8i9Y0/3YdHjHKHiohOJ1ws+3OccDWtH+PoC9DZD5WOTrJvNbWvjS6JWaA==", "requires": {} }, "xml-name-validator": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz", - "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==" + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-4.0.0.tgz", + "integrity": "sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw==" }, "xmlchars": { "version": "2.2.0", diff --git a/scripts/linkbot/package.json b/scripts/linkbot/package.json index 4b96a33d60..060bf24868 100644 --- a/scripts/linkbot/package.json +++ b/scripts/linkbot/package.json @@ -1,6 +1,6 @@ { "dependencies": { "bent": "^7.3.12", - "jsdom": "^16.6.0" + "jsdom": "^19.0.0" } } diff --git a/scripts/test/generate-image.sh b/scripts/test/generate-image.sh new file mode 100755 index 0000000000..92cf96ced5 --- /dev/null +++ b/scripts/test/generate-image.sh @@ -0,0 +1,152 @@ +#!/bin/bash + +set -e + +usage() { + cat 1>&2 <&2 + exec sudo "$0" "$@" +fi + +while [ -n "$1" ]; do + case "$1" in + "ubuntu") + GENERATE_VARIANT=generate_ubuntu + ;; + "debian") + GENERATE_VARIANT=generate_debian + ;; + *) + usage + exit 1 + ;; + esac + shift +done + +cleanup_losetup() { + set +e + for dev in ${LO_DEVICE}p*; do + umount $dev + done + losetup -d $LO_DEVICE + rmdir tmp-p1 + rmdir tmp-p2 +} + +generate_debian() { + local -r image="Debian-11-x86-64.img" + + mkosi --root-size=2G --distribution=debian --release=bullseye --format=gpt_ext4 --bootable --checksum \ + --password password --package=openssh-server,dhcpcd5 --package grub-efi-amd64-signed \ + --package shim-signed --package lsb-release --output="$image" build + + post_process_image "$image" + + echo "Image successfully generated!" 1>&2 +} + +generate_ubuntu() { + local -r image="Ubuntu-Focal-x86-64.img" + + mkosi --root-size=2G --distribution=ubuntu --release=focal --format=gpt_ext4 --bootable --checksum \ + --password password --package=openssh-server,dhcpcd5 --package grub-efi-amd64-signed \ + --package shim-signed --package lsb-release --output="$image" build + + post_process_image "$image" + + echo "Image successfully generated!" 1>&2 +} + +post_process_image() { + local -r image="$1" + + mkdir -p tmp-p1 + mkdir -p tmp-p2 + + LO_DEVICE=$(losetup --find --show --partscan "$image") + trap cleanup_losetup EXIT + mount ${LO_DEVICE}p1 tmp-p1 + mount ${LO_DEVICE}p2 tmp-p2 + + pre_tweaks tmp-p1 tmp-p2 + + create_grub_regeneration_service tmp-p1 tmp-p2 + umount tmp-p1 + umount tmp-p2 + regenerate_grub_live "$image" + mount ${LO_DEVICE}p1 tmp-p1 + mount ${LO_DEVICE}p2 tmp-p2 +} + +pre_tweaks() { + local -r boot="$1" + local -r root="$2" + + # Fstab is missing for some reason. I'm not exactly sure why systemd-boot + # works without this, and GRUB doesn't. + cat > "$root/etc/fstab" < /dev/null`' > $root/etc/default/grub + fi + + sed -E -i -e 's/^#? *PermitRootLogin .*/PermitRootLogin yes/' $root/etc/ssh/sshd_config +} + +# Unfortunately installing grub scripts is something which is not really +# possible when offline. This is something which is easier with systemd-boot, so +# longterm GRUB will probably follow, or systemd-boot will take over. Anyway, +# let's do it by using a systemd service to perform the job, and then shut down. +create_grub_regeneration_service() { + local -r boot="$1" + local -r root="$2" + + cat > "$root/etc/systemd/system/mender-regenerate-grub-and-shutdown.service" <&2 + qemu-system-x86_64 \ + $maybe_kvm \ + -drive file="$image",if=ide,format=raw \ + -drive file=/usr/share/OVMF/OVMF_CODE.fd,if=pflash,format=raw,unit=0,readonly=on \ + -drive file="$nvvars",if=pflash,format=raw,unit=1 \ + -display vnc=:23 \ + -m 512 \ + || ret=$? + [ $ret -eq 0 ] && break + done + + return $ret +} + +$GENERATE_VARIANT diff --git a/scripts/test/mender-convert-qemu b/scripts/test/mender-convert-qemu index 80a9b08acc..809ab25e79 100755 --- a/scripts/test/mender-convert-qemu +++ b/scripts/test/mender-convert-qemu @@ -17,10 +17,11 @@ fi qemu-system-x86_64 \ -enable-kvm \ -nographic \ - -m 256 \ + -m 512 \ -net user,hostfwd=tcp::8822-:22 \ -net nic,macaddr=52:54:00$(od -txC -An -N3 /dev/urandom|tr \ :) \ - -bios ${ovmf_file} \ + -drive file=${ovmf_file},if=pflash,format=raw,unit=0,readonly=on \ + -drive file="$(dirname "$0")/../../tests/uefi-nvram/OVMF_VARS.fd",if=pflash,format=raw,unit=1,readonly=on \ -drive format=raw,file=${DISK_IMG} & qemu_pid=$! diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index 727fea657c..b85fc698b8 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -3,7 +3,7 @@ set -e usage() { - echo "$0 <--all | --only DEVICE_TYPE | --prebuilt-image DEVICE_TYPE IMAGE_NAME>" + echo "$0 [--config EXTRA_CONFIG_FILE] <--all | --only DEVICE_TYPE | --prebuilt-image DEVICE_TYPE IMAGE_NAME>" exit 1 } @@ -21,15 +21,17 @@ BBB_DEBIAN_SDCARD_IMAGE_URL="https://debian.beagleboard.org/images/bone-debian-1 # Not on official home page, but found via https://elinux.org/Beagleboard:BeagleBoneBlack_Debian: ## Auto-update -BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-05-28/buster-console/bone-debian-10.9-console-armhf-2021-05-28-1gb.img.xz" +BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2022-03-02/buster-console/bone-debian-10.11-console-armhf-2022-03-02-1gb.img.xz" ## Auto-update RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2020-02-14/2020-02-13-raspbian-buster-lite.zip" -UBUNTU_IMAGE_URL="https://downloads.mender.io/mender-convert/images/Ubuntu-Bionic-x86-64.img.gz" +UBUNTU_IMAGE_URL="https://downloads.mender.io/mender-convert/images/Ubuntu-Focal-x86-64.img.gz" + +DEBIAN_IMAGE_URL="https://downloads.mender.io/mender-convert/images/Debian-11-x86-64.img.gz" ## Auto-update -UBUNTU_SERVER_RPI_IMAGE_URL="http://cdimage.ubuntu.com/ubuntu/releases/18.04.5/release/ubuntu-18.04.5-preinstalled-server-armhf+raspi3.img.xz" +UBUNTU_SERVER_RPI_IMAGE_URL="http://cdimage.ubuntu.com/ubuntu/releases/20.04/release/ubuntu-20.04.4-preinstalled-server-armhf+raspi.img.xz" # Keep common function declarations in separate utils script UTILS_PATH=${0/$(basename $0)/test-utils.sh} @@ -44,92 +46,128 @@ get_pytest_files prepare_ssh_keys -if ! [ "$1" == "--all" -o "$1" == "--only" -a -n "$2" -o "$1" == "--prebuilt-image" -a -n "$3" ]; then - usage -fi +usage_if_empty() { + if [ -z "$1" ]; then + usage + fi +} + +PREBUILT_IMAGE= +TEST_PLATFORM= +TEST_ALL=0 +EXTRA_CONFIG= +while [ -n "$1" ]; do + case "$1" in + --prebuilt-image) + usage_if_empty "$3" + PREBUILT_IMAGE="$2 $3" + ;; + --all) + TEST_ALL=1 + ;; + --only) + usage_if_empty "$2" + TEST_PLATFORM="$2" + shift + ;; + --config) + usage_if_empty "$2" + EXTRA_CONFIG="$EXTRA_CONFIG --config $2" + shift + ;; + esac + shift +done test_result=0 -if [ "$1" == "--prebuilt-image" ]; then - run_tests "$2" "$3" \ - "-k" "'not test_update'" \ +if [ -n "$PREBUILT_IMAGE" ]; then + run_tests $PREBUILT_IMAGE \ || test_result=$? exit $test_result else - if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "qemux86_64" ]; then - wget --progress=dot:giga -N ${UBUNTU_IMAGE_URL} -P input/ - convert_and_test "qemux86_64" \ + if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "ubuntu-qemux86-64" ]; then + wget --progress=dot:giga -N ${UBUNTU_IMAGE_URL} -P input/image/ + mkdir -p input/tests + cp -r "tests/ssh-public-key-overlay" "input/tests/ssh-public-key-overlay" + convert_and_test "qemux86-64" \ "release-1" \ - "input/Ubuntu-Bionic-x86-64.img.gz" \ - "--overlay tests/ssh-public-key-overlay" \ - "--config configs/qemux86-64_config" \ + "input/image/Ubuntu-Focal-x86-64.img.gz" \ + "--overlay input/tests/ssh-public-key-overlay" \ + "--config configs/ubuntu-qemux86-64_config $EXTRA_CONFIG" \ || test_result=$? echo >&2 "----------------------------------------" echo >&2 "Running the uncompressed test" echo >&2 "----------------------------------------" rm -rf deploy - gunzip --force "input/Ubuntu-Bionic-x86-64.img.gz" + gunzip --force "input/image/Ubuntu-Focal-x86-64.img.gz" run_convert "release-2" \ - "input/Ubuntu-Bionic-x86-64.img" \ - "--config configs/qemux86-64_config" || test_result=$? + "input/image/Ubuntu-Focal-x86-64.img" \ + "--config configs/ubuntu-qemux86-64_config $EXTRA_CONFIG" || test_result=$? ret=0 - test -f deploy/Ubuntu-Bionic-x86-64-qemux86_64-mender.img || ret=$? - assert "${ret}" "0" "Expected uncompressed file deploy/Ubuntu-Bionic-x86-64-qemux86_64-mender.img" + test -f deploy/Ubuntu-Focal-x86-64-qemux86-64-mender.img || ret=$? + assert "${ret}" "0" "Expected uncompressed file deploy/Ubuntu-Focal-x86-64-qemux86-64-mender.img" fi - if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "raspberrypi3" ]; then - wget --progress=dot:giga -N ${RASPBIAN_IMAGE_URL} -P input/ + if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "raspberrypi3" ]; then + wget --progress=dot:giga -N ${RASPBIAN_IMAGE_URL} -P input/image/ RASPBIAN_IMAGE="${RASPBIAN_IMAGE_URL##*/}" convert_and_test "raspberrypi3" \ "release-1" \ - "input/${RASPBIAN_IMAGE}" \ - "--config configs/raspberrypi3_config" \ - -- \ - "-k" "'not test_update'" \ + "input/image/${RASPBIAN_IMAGE}" \ + "--config configs/raspberrypi3_config $EXTRA_CONFIG" \ || test_result=$? fi - if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "beaglebone" ]; then - wget --progress=dot:giga -N ${BBB_DEBIAN_SDCARD_IMAGE_URL} -P input/ + if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "beaglebone" ]; then + wget --progress=dot:giga -N ${BBB_DEBIAN_SDCARD_IMAGE_URL} -P input/image/ BBB_DEBIAN_SDCARD_IMAGE_COMPRESSED="${BBB_DEBIAN_SDCARD_IMAGE_URL##*/}" BBB_DEBIAN_SDCARD_IMAGE_UNCOMPRESSED="${BBB_DEBIAN_SDCARD_IMAGE_COMPRESSED%.xz}" # Convert uncompressed images to speed up this job - unxz --force "input/${BBB_DEBIAN_SDCARD_IMAGE_COMPRESSED}" + unxz --force "input/image/${BBB_DEBIAN_SDCARD_IMAGE_COMPRESSED}" convert_and_test "beaglebone-sdcard" \ "release-1" \ - "input/${BBB_DEBIAN_SDCARD_IMAGE_UNCOMPRESSED}" \ - "--config configs/beaglebone_black_debian_sdcard_config" \ - -- \ - "-k" "'not test_update'" \ + "input/image/${BBB_DEBIAN_SDCARD_IMAGE_UNCOMPRESSED}" \ + "--config configs/beaglebone_black_debian_sdcard_config $EXTRA_CONFIG" \ || test_result=$? rm -rf deploy - wget --progress=dot:giga -N ${BBB_DEBIAN_EMMC_IMAGE_URL} -P input/ + wget --progress=dot:giga -N ${BBB_DEBIAN_EMMC_IMAGE_URL} -P input/image/ BBB_DEBIAN_EMMC_IMAGE_COMPRESSED="${BBB_DEBIAN_EMMC_IMAGE_URL##*/}" BBB_DEBIAN_EMMC_IMAGE_UNCOMPRESSED="${BBB_DEBIAN_EMMC_IMAGE_COMPRESSED%.xz}" - unxz --force "input/${BBB_DEBIAN_EMMC_IMAGE_COMPRESSED}" + unxz --force "input/image/${BBB_DEBIAN_EMMC_IMAGE_COMPRESSED}" convert_and_test "beaglebone-emmc" \ "release-1" \ - "input/${BBB_DEBIAN_EMMC_IMAGE_UNCOMPRESSED}" \ - "--config configs/beaglebone_black_debian_emmc_config" \ - -- \ - "-k" "'not test_update'" \ + "input/image/${BBB_DEBIAN_EMMC_IMAGE_UNCOMPRESSED}" \ + "--config configs/beaglebone_black_debian_emmc_config $EXTRA_CONFIG" \ || test_result=$? fi - if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "ubuntu" ]; then - wget --progress=dot:giga -N ${UBUNTU_SERVER_RPI_IMAGE_URL} -P input/ + if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "ubuntu" ]; then + wget --progress=dot:giga -N ${UBUNTU_SERVER_RPI_IMAGE_URL} -P input/image/ UBUNTU_SERVER_RPI_IMAGE_COMPRESSED="${UBUNTU_SERVER_RPI_IMAGE_URL##*/}" convert_and_test "raspberrypi3" \ "release-1" \ - "input/${UBUNTU_SERVER_RPI_IMAGE_COMPRESSED}" \ - "--config configs/raspberrypi3_config" \ - -- \ - "-k" "'not test_update'" \ + "input/image/${UBUNTU_SERVER_RPI_IMAGE_COMPRESSED}" \ + "--config configs/raspberrypi3_config $EXTRA_CONFIG" \ || test_result=$? fi + if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "debian-qemux86-64" ]; then + wget --progress=dot:giga -N ${DEBIAN_IMAGE_URL} -P input/image/ + mkdir -p input/tests + cp -r "tests/ssh-public-key-overlay" "input/tests/ssh-public-key-overlay" + convert_and_test "qemux86-64" \ + "release-1" \ + "input/image/Debian-11-x86-64.img.gz" \ + "--overlay input/tests/ssh-public-key-overlay" \ + "--config configs/debian-qemux86-64_config $EXTRA_CONFIG" \ + || test_result=$? + fi + + + exit $test_result fi diff --git a/scripts/test/test-utils.sh b/scripts/test/test-utils.sh index 1ef608b3c8..8e23583962 100644 --- a/scripts/test/test-utils.sh +++ b/scripts/test/test-utils.sh @@ -140,7 +140,6 @@ run_tests() { --sdimg-location="${MENDER_CONVERT_DIR}/deploy" \ --ssh-priv-key="./ssh-priv-key/key" \ --qemu-wrapper="../scripts/test/mender-convert-qemu" \ - mender-image-tests \ ${pytest_extra_args} exitcode=$? @@ -163,15 +162,15 @@ prepare_ssh_keys() { sudo chown -R root:root tests/ssh-public-key-overlay/root fi if [ "$(stat -c %a tests/ssh-public-key-overlay/root)" != "755" ]; then - chmod 755 tests/ssh-public-key-overlay/root + sudo chmod 755 tests/ssh-public-key-overlay/root fi if [ "$(stat -c %a tests/ssh-public-key-overlay/root/.ssh)" != "755" ]; then - chmod 700 tests/ssh-public-key-overlay/root/.ssh + sudo chmod 700 tests/ssh-public-key-overlay/root/.ssh fi if [ "$(stat -c %a tests/ssh-public-key-overlay/root/.ssh/authorized_keys)" != "755" ]; then - chmod 600 tests/ssh-public-key-overlay/root/.ssh/authorized_keys + sudo chmod 600 tests/ssh-public-key-overlay/root/.ssh/authorized_keys fi if [ "$(stat -c %a tests/ssh-priv-key/key)" != "600" ]; then - chmod 600 tests/ssh-priv-key/key + sudo chmod 600 tests/ssh-priv-key/key fi } diff --git a/tests/conftest.py b/tests/conftest.py index 5344b09056..c00bfff55f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,4 @@ -# Copyright 2021 Northern.tech AS +# Copyright 2022 Northern.tech AS # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,3 +22,5 @@ # Load the parser for our custom option flags pytest_plugins = "utils.parseropts.parseropts" + +from utils.fixtures import * diff --git a/tests/mender-image-tests b/tests/mender-image-tests index 80bd05390e..1d76120611 160000 --- a/tests/mender-image-tests +++ b/tests/mender-image-tests @@ -1 +1 @@ -Subproject commit 80bd05390e1c1a5a77265053c0bf2a956cf35470 +Subproject commit 1d7612061138d9102ffb6a86ef362565a0e18937 diff --git a/tests/test_grub_integration.py b/tests/test_grub_integration.py new file mode 100644 index 0000000000..4eaaa56f9f --- /dev/null +++ b/tests/test_grub_integration.py @@ -0,0 +1,199 @@ +#!/usr/bin/python +# Copyright 2022 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import re +import os +import subprocess + +from utils.common import ( + extract_partition, + get_no_sftp, +) + + +@pytest.fixture(scope="function") +def cleanup_boot_scripts(request, connection): + """Take a backup of the various grub.cfg files and restore them after the + test. This is recommended for tests that call `grub-install` and/or + `update-grub`, so that other tests can also run them from a pristine + state.""" + + connection.run( + "cp $(find /boot/efi/EFI/ -name grub.cfg -not -path '*/EFI/BOOT/*') /data/grub-efi.cfg" + ) + connection.run("cp /boot/grub/grub.cfg /data/grub-main.cfg") + connection.run("cp /boot/grub-mender-grubenv.cfg /data/grub-mender-grubenv.cfg") + + def cleanup(): + connection.run( + "mv /data/grub-efi.cfg $(find /boot/efi/EFI/ -name grub.cfg -not -path '*/EFI/BOOT/*')" + ) + connection.run("mv /data/grub-main.cfg /boot/grub/grub.cfg") + connection.run("mv /data/grub-mender-grubenv.cfg /boot/grub-mender-grubenv.cfg") + + request.addfinalizer(cleanup) + + +def check_all_root_occurrences_valid(grub_cfg): + found_expected = False + inside_10_header = False + # One of the functions we define and use. + expected = "mender_check_and_restore_env" + with open(grub_cfg) as fd: + lineno = 0 + for line in fd.readlines(): + lineno += 1 + if re.match(r'^\s*root="\$\{mender_grub_storage_device\}', line): + continue + + if line.strip() == "### BEGIN /etc/grub.d/00_header ###": + # We allow root references inside the 00_header, because they + # are overriden by Mender later. + inside_10_header = True + elif line.strip() == "### END /etc/grub.d/00_header ###": + inside_10_header = False + if not inside_10_header and re.match(r"^\s*(set +)?root=", line): + pytest.fail( + "Found unexpected occurrence of `root=` in grub boot script\n" + "%d:%s" % (lineno, line) + ) + + if line.find(expected) >= 0: + found_expected = True + + assert found_expected, "Expected content (%s) not found" % expected + + +@pytest.mark.usefixtures("setup_board", "cleanup_boot_scripts") +class TestGrubIntegration: + @pytest.mark.min_mender_version("1.0.0") + def test_no_root_occurrences(self, connection, latest_part_image): + """Test that the generated grub scripts do not contain any occurrences of + `root=` except for known instances that we control. This is + important because Mender needs to keep tight control of when this + variable is set, in order to boot from, and mount, the correct root + partition.""" + + # First, check that the offline generated scripts don't have any. + extract_partition(latest_part_image, 1) + try: + subprocess.check_call( + ["mcopy", "-i", "img1.fs", "::/grub-mender-grubenv/grub.cfg", "."] + ) + check_all_root_occurrences_valid("grub.cfg") + finally: + os.remove("img1.fs") + os.remove("grub.cfg") + + extract_partition(latest_part_image, 2) + try: + subprocess.check_call( + [ + "debugfs", + "-R", + "dump -p /boot/grub-mender-grubenv.cfg grub-mender-grubenv.cfg", + "img2.fs", + ] + ) + check_all_root_occurrences_valid("grub-mender-grubenv.cfg") + finally: + os.remove("img2.fs") + os.remove("grub-mender-grubenv.cfg") + + # Then, check that the runtime generated scripts don't have any. + get_no_sftp("/boot/grub/grub.cfg", connection) + try: + check_all_root_occurrences_valid("grub.cfg") + finally: + os.remove("grub.cfg") + + get_no_sftp("/boot/grub-mender-grubenv.cfg", connection) + try: + check_all_root_occurrences_valid("grub-mender-grubenv.cfg") + finally: + os.remove("grub-mender-grubenv.cfg") + + # Check again after running `update-grub`. + connection.run("grub-install && update-grub") + get_no_sftp("/boot/grub/grub.cfg", connection) + try: + check_all_root_occurrences_valid("grub.cfg") + finally: + os.remove("grub.cfg") + + get_no_sftp("/boot/grub-mender-grubenv.cfg", connection) + try: + check_all_root_occurrences_valid("grub-mender-grubenv.cfg") + finally: + os.remove("grub-mender-grubenv.cfg") + + @pytest.mark.min_mender_version("1.0.0") + def test_offline_and_runtime_boot_scripts_identical(self, connection): + # Update scripts at runtime. + connection.run("grub-install && update-grub") + + # Take advantage of the copies already made by cleanup_boot_scripts + # fixture above, and use the copies in /data. + + # Take into account some known, but harmless differences. The "hd0,gpt1" + # style location is missing from the offline generated grub-efi.cfg + # file, but it is harmless because the filesystem UUID is being used + # instead. + connection.run( + r"sed -Ee 's/ *hd[0-9]+,gpt[0-9]+//' " + "$(find /boot/efi/EFI/ -name grub.cfg -not -path '*/EFI/BOOT/*') " + "> /data/new-grub-efi-modified.cfg" + ) + try: + connection.run("diff -u /data/grub-efi.cfg /data/new-grub-efi-modified.cfg") + finally: + connection.run("rm -f /data/new-grub-efi-modified.cfg") + + # Another few differences we work around in the main grub files: + # * `--hint` parameters are not generated in offline copy. + # * `root` variable is not set in offline copy. + # * `fwsetup` is added somewhat randomly depending on availability both + # on build host and device. + try: + connection.run("cp /data/grub-main.cfg /data/old-grub-modified.cfg") + connection.run("cp /boot/grub/grub.cfg /data/new-grub-modified.cfg") + connection.run( + r"sed -i -En -e '/\bsearch\b/{s/ --hint[^ ]*//g;}' " + "-e \"/^set root='hd0,gpt1'$/d\" " + r"-e '\,### BEGIN /etc/grub.d/30_uefi-firmware ###,{p; n; :loop; \,### END /etc/grub.d/30_uefi-firmware ###,b end; n; b loop; :end;}' " + "-e p " + "/data/old-grub-modified.cfg /data/new-grub-modified.cfg" + ) + connection.run("diff -u /data/old-grub-modified.cfg /data/new-grub-modified.cfg") + finally: + connection.run("rm -f /data/old-grub-modified.cfg /data/new-grub-modified.cfg") + + # Same differences as in previous check. + try: + connection.run("cp /data/grub-mender-grubenv.cfg /data/old-grub-mender-grubenv-modified.cfg") + connection.run("cp /boot/grub-mender-grubenv.cfg /data/new-grub-mender-grubenv-modified.cfg") + connection.run( + r"sed -i -En -e '/\bsearch\b/{s/ --hint[^ ]*//g;}' " + "-e \"/^set root='hd0,gpt1'$/d\" " + r"-e '\,### BEGIN /etc/grub.d/30_uefi-firmware ###,{p; n; :loop; \,### END /etc/grub.d/30_uefi-firmware ###,b end; n; b loop; :end;}' " + "-e p " + "/data/old-grub-mender-grubenv-modified.cfg /data/new-grub-mender-grubenv-modified.cfg" + ) + connection.run( + "diff -u /data/old-grub-mender-grubenv-modified.cfg /data/new-grub-mender-grubenv-modified.cfg" + ) + finally: + connection.run("rm -f /data/old-grub-mender-grubenv-modified.cfg /data/new-grub-mender-grubenv-modified.cfg") diff --git a/tests/uefi-nvram/MicCorThiParMarRoo_2010-10-05.crt b/tests/uefi-nvram/MicCorThiParMarRoo_2010-10-05.crt new file mode 100644 index 0000000000..d6a50b817a Binary files /dev/null and b/tests/uefi-nvram/MicCorThiParMarRoo_2010-10-05.crt differ diff --git a/tests/uefi-nvram/OVMF_VARS.fd b/tests/uefi-nvram/OVMF_VARS.fd new file mode 100644 index 0000000000..fbb72b7206 Binary files /dev/null and b/tests/uefi-nvram/OVMF_VARS.fd differ diff --git a/tests/uefi-nvram/README.md b/tests/uefi-nvram/README.md new file mode 100644 index 0000000000..3c9cdd495e --- /dev/null +++ b/tests/uefi-nvram/README.md @@ -0,0 +1,71 @@ +UEFI NVRAM +========== + +This directory holds the NVRAM file which is used as the firmware memory of the UEFI software which +runs under QEMU. It's main purpose is to start the UEFI software with certificates pre-loaded into +the firmware memory, and Secure Boot enabled. + +How to recreate the `OVMF_VARS.fd` file +-------------------------------- + +1. Create the `OVMF_VARS.fd` file: + ```bash + cp /usr/share/OVMF/OVMF_VARS.fd OVMF_VARS.fd + ``` + +2. Create a filesystem which contains the UEFI certificates: + ```bash + dd if=/dev/zero of=/tmp/cert-filesystem.fs bs=1M count=10; \ + mkfs.vfat /tmp/cert-filesystem.fs; \ + mkdir cert-filesystem; \ + sudo mount /tmp/cert-filesystem.fs cert-filesystem -o loop,uid=$UID; \ + cp *.crt cert-filesystem; \ + sudo umount cert-filesystem; \ + rmdir cert-filesystem + ``` + + Tip: If you ever need to re-fetch the certificate files, run `mokutil --db` on your own + computer. This lists your installed certificates, and they come with URLs which say where you can + download them. Make sure you download the `.crt`, not the `.crl`. + +3. Launch QEMU with the NVRAM and the filesystem containing the certificates. *Make sure to press F2 + quickly after the window appears to enter the firmware menu*: + ```bash + qemu-system-x86_64 \ + -drive file=/usr/share/OVMF/OVMF_CODE.fd,if=pflash,format=raw,unit=0,readonly=on \ + -drive file=./OVMF_VARS.fd,if=pflash,format=raw,unit=1 \ + -drive file=/tmp/cert-filesystem.fs,if=ide,format=raw + ``` + +4. After having entered the firmware menu, perform the following steps: + + 1. Enter "Device Manager". + + 2. Enter "Secure Boot Configuration". + + 3. Switch "Secure Boot Mode" to "Custom Mode". + + 4. Enter "Custom Secure Boot Options". + + 5. Enter "PK Options". + + 6. Enter "Enroll OK". + + 7. Enter "Enroll PK Using File". + + 8. Locate the certificate file in the filesystem that you created in main step 2, and add it. + + 9. Make sure to select "Commit Changes". + + 10. Repeat the same process starting from sub step 5, except for "DB Options" instead. + + 11. Go back to the "Secure Boot Configuration" screen. "Attempt Secure Boot" should now have + been auto-selected. If it's not, enable it and save the change. + + 12. Go back to the main manu and select "Reset". After the setup has been exited, you can kill + QEMU. + +5. Clean up: + ```bash + rm -f /tmp/cert-filesystem.fs + ```