From ca5d8b0329ac797c431a3f4fbe026fd197ef9693 Mon Sep 17 00:00:00 2001 From: stuffandthings Date: Thu, 8 Oct 2020 11:51:16 -0400 Subject: [PATCH 001/117] update configs --- configs/mender_convert_config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/mender_convert_config b/configs/mender_convert_config index 7ed997bd98..a465f348f8 100644 --- a/configs/mender_convert_config +++ b/configs/mender_convert_config @@ -89,11 +89,11 @@ MENDER_PARTITION_SCHEME="" # (see MENDER_BOOT_PART_SIZE_MB) and data partitions # (see MENDER_DATA_PART_SIZE_MB). # -MENDER_STORAGE_TOTAL_SIZE_MB="8192" +MENDER_STORAGE_TOTAL_SIZE_MB="16384" # The size of the boot partition in the generated .biosimg, .sdimg or .uefiimg # file. -MENDER_BOOT_PART_SIZE_MB="40" +MENDER_BOOT_PART_SIZE_MB="256" # The size of the persistent data partition in the generated .biosimg, .sdimg # or .uefiimg file. From 67e77d1abe42d4c786184aa168fe692911324c2a Mon Sep 17 00:00:00 2001 From: stuffandthings Date: Thu, 15 Oct 2020 15:20:11 -0400 Subject: [PATCH 002/117] 32gb drive --- configs/mender_convert_config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/mender_convert_config b/configs/mender_convert_config index a465f348f8..07d93a5f5f 100644 --- a/configs/mender_convert_config +++ b/configs/mender_convert_config @@ -89,7 +89,7 @@ MENDER_PARTITION_SCHEME="" # (see MENDER_BOOT_PART_SIZE_MB) and data partitions # (see MENDER_DATA_PART_SIZE_MB). # -MENDER_STORAGE_TOTAL_SIZE_MB="16384" +MENDER_STORAGE_TOTAL_SIZE_MB="31914" # The size of the boot partition in the generated .biosimg, .sdimg or .uefiimg # file. From afd820daceb75d98dbed3bc10c885815e27def05 Mon Sep 17 00:00:00 2001 From: stuffandthings Date: Thu, 15 Oct 2020 15:34:27 -0400 Subject: [PATCH 003/117] finalize fs sizing --- configs/images/raspberrypi_raspbian_config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/images/raspberrypi_raspbian_config b/configs/images/raspberrypi_raspbian_config index 7f83bab481..df03b31fcb 100644 --- a/configs/images/raspberrypi_raspbian_config +++ b/configs/images/raspberrypi_raspbian_config @@ -2,7 +2,7 @@ # often use a base of 1000 instead of 1024, and even then they are often smaller # than advertised. The number below is based on a conservative target of 7.9GB # (that's mathematical GB, base 1000), converted to MiB, rounding down. -MENDER_STORAGE_TOTAL_SIZE_MB=7534 +MENDER_STORAGE_TOTAL_SIZE_MB=31914 # Use all there is, which gets us almost, but not quite, to 500MiB free space # (about 480MiB at the time of writing). From e1915ba6cc9d65abad2d9360075c0b6070164829 Mon Sep 17 00:00:00 2001 From: stuffandthings Date: Thu, 15 Oct 2020 17:18:53 -0400 Subject: [PATCH 004/117] use max size in fs --- configs/mender_convert_config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/mender_convert_config b/configs/mender_convert_config index 07d93a5f5f..581027f1d6 100644 --- a/configs/mender_convert_config +++ b/configs/mender_convert_config @@ -173,7 +173,7 @@ MENDER_COPY_BOOT_GAP="y" # # This variable directly mirrors the variable from the Yocto Project, which is # why it is missing a "MENDER_" prefix. -IMAGE_ROOTFS_SIZE="0" +IMAGE_ROOTFS_SIZE="-1" # The amount of extra free space requested on the rootfs, in KiB. This is added # to the value of IMAGE_ROOTFS_SIZE. The size is further modified by From 327f29bc1dbea945fd49e94bb4dae4a51476e5a3 Mon Sep 17 00:00:00 2001 From: stuffandthings Date: Thu, 29 Oct 2020 16:48:09 -0400 Subject: [PATCH 005/117] .. --- configs/images/raspberrypi_raspbian_config | 2 +- configs/mender_convert_config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/images/raspberrypi_raspbian_config b/configs/images/raspberrypi_raspbian_config index df03b31fcb..1d2572e305 100644 --- a/configs/images/raspberrypi_raspbian_config +++ b/configs/images/raspberrypi_raspbian_config @@ -2,7 +2,7 @@ # often use a base of 1000 instead of 1024, and even then they are often smaller # than advertised. The number below is based on a conservative target of 7.9GB # (that's mathematical GB, base 1000), converted to MiB, rounding down. -MENDER_STORAGE_TOTAL_SIZE_MB=31914 +MENDER_STORAGE_TOTAL_SIZE_MB=30000 # Use all there is, which gets us almost, but not quite, to 500MiB free space # (about 480MiB at the time of writing). diff --git a/configs/mender_convert_config b/configs/mender_convert_config index 581027f1d6..b82502a647 100644 --- a/configs/mender_convert_config +++ b/configs/mender_convert_config @@ -89,7 +89,7 @@ MENDER_PARTITION_SCHEME="" # (see MENDER_BOOT_PART_SIZE_MB) and data partitions # (see MENDER_DATA_PART_SIZE_MB). # -MENDER_STORAGE_TOTAL_SIZE_MB="31914" +MENDER_STORAGE_TOTAL_SIZE_MB="30000" # The size of the boot partition in the generated .biosimg, .sdimg or .uefiimg # file. From 04f70d043f9393c0891e54a8dda5aaed4b76153b Mon Sep 17 00:00:00 2001 From: stuffandthings Date: Mon, 9 Nov 2020 20:33:34 -0500 Subject: [PATCH 006/117] tweak settings --- configs/images/raspberrypi_raspbian_config | 2 +- configs/mender_convert_config | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/images/raspberrypi_raspbian_config b/configs/images/raspberrypi_raspbian_config index 1d2572e305..75fb69eff5 100644 --- a/configs/images/raspberrypi_raspbian_config +++ b/configs/images/raspberrypi_raspbian_config @@ -2,7 +2,7 @@ # often use a base of 1000 instead of 1024, and even then they are often smaller # than advertised. The number below is based on a conservative target of 7.9GB # (that's mathematical GB, base 1000), converted to MiB, rounding down. -MENDER_STORAGE_TOTAL_SIZE_MB=30000 +MENDER_STORAGE_TOTAL_SIZE_MB=30136 # Use all there is, which gets us almost, but not quite, to 500MiB free space # (about 480MiB at the time of writing). diff --git a/configs/mender_convert_config b/configs/mender_convert_config index b82502a647..db4c9e1fdf 100644 --- a/configs/mender_convert_config +++ b/configs/mender_convert_config @@ -89,7 +89,7 @@ MENDER_PARTITION_SCHEME="" # (see MENDER_BOOT_PART_SIZE_MB) and data partitions # (see MENDER_DATA_PART_SIZE_MB). # -MENDER_STORAGE_TOTAL_SIZE_MB="30000" +MENDER_STORAGE_TOTAL_SIZE_MB="30136" # The size of the boot partition in the generated .biosimg, .sdimg or .uefiimg # file. @@ -173,7 +173,7 @@ MENDER_COPY_BOOT_GAP="y" # # This variable directly mirrors the variable from the Yocto Project, which is # why it is missing a "MENDER_" prefix. -IMAGE_ROOTFS_SIZE="-1" +IMAGE_ROOTFS_SIZE="0" # The amount of extra free space requested on the rootfs, in KiB. This is added # to the value of IMAGE_ROOTFS_SIZE. The size is further modified by From 9b653f77787db2dd81fa756f25a3ed54e0a8f6cf Mon Sep 17 00:00:00 2001 From: stuffandthings Date: Fri, 13 Nov 2020 14:05:01 -0500 Subject: [PATCH 007/117] testing --- configs/images/raspberrypi_raspbian_config | 2 +- configs/mender_convert_config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/images/raspberrypi_raspbian_config b/configs/images/raspberrypi_raspbian_config index 75fb69eff5..3c64160abc 100644 --- a/configs/images/raspberrypi_raspbian_config +++ b/configs/images/raspberrypi_raspbian_config @@ -6,7 +6,7 @@ MENDER_STORAGE_TOTAL_SIZE_MB=30136 # Use all there is, which gets us almost, but not quite, to 500MiB free space # (about 480MiB at the time of writing). -IMAGE_ROOTFS_SIZE=-1 +IMAGE_ROOTFS_SIZE=0 IMAGE_ROOTFS_EXTRA_SPACE=0 IMAGE_OVERHEAD_FACTOR=1.0 diff --git a/configs/mender_convert_config b/configs/mender_convert_config index db4c9e1fdf..e5dc3ae74b 100644 --- a/configs/mender_convert_config +++ b/configs/mender_convert_config @@ -173,7 +173,7 @@ MENDER_COPY_BOOT_GAP="y" # # This variable directly mirrors the variable from the Yocto Project, which is # why it is missing a "MENDER_" prefix. -IMAGE_ROOTFS_SIZE="0" +IMAGE_ROOTFS_SIZE="-1" # The amount of extra free space requested on the rootfs, in KiB. This is added # to the value of IMAGE_ROOTFS_SIZE. The size is further modified by From f779eccf731de53863928294e249d7e2cf0eb198 Mon Sep 17 00:00:00 2001 From: stuffandthings Date: Fri, 13 Nov 2020 14:24:45 -0500 Subject: [PATCH 008/117] .. --- configs/images/raspberrypi_raspbian_config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/images/raspberrypi_raspbian_config b/configs/images/raspberrypi_raspbian_config index 3c64160abc..75fb69eff5 100644 --- a/configs/images/raspberrypi_raspbian_config +++ b/configs/images/raspberrypi_raspbian_config @@ -6,7 +6,7 @@ MENDER_STORAGE_TOTAL_SIZE_MB=30136 # Use all there is, which gets us almost, but not quite, to 500MiB free space # (about 480MiB at the time of writing). -IMAGE_ROOTFS_SIZE=0 +IMAGE_ROOTFS_SIZE=-1 IMAGE_ROOTFS_EXTRA_SPACE=0 IMAGE_OVERHEAD_FACTOR=1.0 From 814617ce98e6dceec51942301b4ba365d65f8055 Mon Sep 17 00:00:00 2001 From: stuffandthings Date: Fri, 13 Nov 2020 20:46:27 -0500 Subject: [PATCH 009/117] .. --- configs/mender_convert_config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/mender_convert_config b/configs/mender_convert_config index e5dc3ae74b..ddc9640ebc 100644 --- a/configs/mender_convert_config +++ b/configs/mender_convert_config @@ -173,7 +173,7 @@ MENDER_COPY_BOOT_GAP="y" # # This variable directly mirrors the variable from the Yocto Project, which is # why it is missing a "MENDER_" prefix. -IMAGE_ROOTFS_SIZE="-1" +IMAGE_ROOTFS_SIZE="13671875" # The amount of extra free space requested on the rootfs, in KiB. This is added # to the value of IMAGE_ROOTFS_SIZE. The size is further modified by From 3672c63ed88b7aec702f06716094de93863ce639 Mon Sep 17 00:00:00 2001 From: Ole Petter Date: Fri, 18 Jun 2021 17:55:53 +0200 Subject: [PATCH 010/117] MEN-4462: Add a demo configuration config script This moves the device configuration to a config script in configs/, from the previous setup in the bootstrap scripts. It does so by creating a new resources/ folder in the top-level directory, which is then populated with the configuration file, and the server parameters only. When the conversion is run with the demo configuration configs/mender_convert_demo_config the conversion will install all the add-on's available, and modify the configuration file in resources to align with our regular Mender demo configuration. Changelog: Add a demo configuration in configs/mender_convert_demo_config which when added at run time creates a Mender demo setup in the converted image. This includes short polling intervals, and all add-on's installed by default. Signed-off-by: Ole Petter --- Dockerfile | 4 ++- configs/mender_convert_demo_config | 35 +++++++++++++++++++ mender-convert-modify | 9 +++++ .../bootstrap-rootfs-overlay-demo-server.sh | 22 +++--------- .../bootstrap-rootfs-overlay-hosted-server.sh | 13 +++---- ...tstrap-rootfs-overlay-production-server.sh | 20 ++++------- 6 files changed, 62 insertions(+), 41 deletions(-) create mode 100644 configs/mender_convert_demo_config diff --git a/Dockerfile b/Dockerfile index 7766d0a959..b54cc9b760 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,7 +51,9 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install -y \ zip \ unzip \ # manipulate binary and hex - xxd + xxd \ +# JSON power tool + jq COPY --from=build /root/pxz/pxz /usr/bin/pxz diff --git a/configs/mender_convert_demo_config b/configs/mender_convert_demo_config new file mode 100644 index 0000000000..20f93caf87 --- /dev/null +++ b/configs/mender_convert_demo_config @@ -0,0 +1,35 @@ +# Install all the add-on's by default +MENDER_ADDON_CONNECT_INSTALL="y" +MENDER_ADDON_CONFIGURE_INSTALL="y" + +enable_demo_configuration_in_file() { + + log_info "Modifying mender.conf to follow a demo setup" + + run_and_log_cmd "mkdir -p work/rootfs/etc/mender" + + if [ ! -f work/rootfs/etc/mender/mender.conf ]; then + log_error "No Mender configuration file found in the work folder." + exit 1 + fi + + if [ ! -f work/rootfs/etc/mender/mender-connect.conf ]; then + log_info "Installing the mender-connect.conf file" + cat <<- EOF > work/rootfs/etc/mender/mender-connect.conf + { + "ShellCommand": "/bin/sh", + "User": "root" + } +EOF + run_and_log_cmd "sudo chmod 0600 work/rootfs/etc/mender/mender-connect.conf" + fi + + log_info "Adding the demo configuration to the Mender client" + tmpfile=work/mender.conf.bak + run_and_log_cmd "cat work/rootfs/etc/mender/mender.conf | jq '.InventoryPollIntervalSeconds=5 | .RetryPollIntervalSeconds=30 | .UpdatePollIntervalSeconds=5' > ${tmpfile}" + run_and_log_cmd "cp ${tmpfile} work/rootfs/etc/mender/mender.conf" + run_and_log_cmd "sudo chmod 0600 work/rootfs/etc/mender/mender.conf" + log_warn "New mender.conf contents: $(cat work/rootfs/etc/mender/mender.conf)" + +} +PLATFORM_MODIFY_HOOKS+=(enable_demo_configuration_in_file) diff --git a/mender-convert-modify b/mender-convert-modify index 9810f1771e..e68999d396 100755 --- a/mender-convert-modify +++ b/mender-convert-modify @@ -290,6 +290,15 @@ EOF run_and_log_cmd "sudo cp work/mender.conf.data work/rootfs/data/mender/mender.conf" run_and_log_cmd "sudo chmod 600 work/rootfs/data/mender/mender.conf" +if [ -f resources/mender.conf ]; then + log_info "Installing the local mender.conf file" + run_and_log_cmd "mkdir -p work/rootfs/etc/mender" + run_and_log_cmd "cp resources/mender.conf work/rootfs/etc/mender" + run_and_log_cmd "sudo chmod 600 work/rootfs/etc/mender/mender.conf" +else + log_warn "No mender.conf file found in resources. Have you remembered to run the bootstrap script?" +fi + if [ -z "${MENDER_DEVICE_TYPE}" ]; then # Observed systems who do not have this file, e.g images generated with mkosi if [ -f work/rootfs/etc/hostname ]; then diff --git a/scripts/bootstrap-rootfs-overlay-demo-server.sh b/scripts/bootstrap-rootfs-overlay-demo-server.sh index 57879e95f0..5b3e4eca13 100755 --- a/scripts/bootstrap-rootfs-overlay-demo-server.sh +++ b/scripts/bootstrap-rootfs-overlay-demo-server.sh @@ -56,28 +56,16 @@ if [ -e ${output_dir} ]; then sudo chown -R $(id -u) ${output_dir} sudo chgrp -R $(id -g) ${output_dir} fi + mkdir -p ${output_dir}/etc/mender -cat <<- EOF > ${output_dir}/etc/mender/mender.conf +mkdir -p ${root_dir}/resources +cat <<- EOF > ${root_dir}/resources/mender.conf { - "InventoryPollIntervalSeconds": 5, - "RetryPollIntervalSeconds": 30, "ServerURL": "https://docker.mender.io", - "ServerCertificate": "/etc/mender/server.crt", - "UpdatePollIntervalSeconds": 5 + "ServerCertificate": "/etc/mender/server.crt" } EOF -chmod 600 ${output_dir}/etc/mender/mender.conf - -cat <<- EOF > ${output_dir}/etc/mender/mender-connect.conf -{ - "ShellCommand": "/bin/sh", - "User": "root" -} -EOF - -chmod 600 ${output_dir}/etc/mender/mender-connect.conf - cat <<- EOF > ${output_dir}/etc/hosts 127.0.0.1 localhost @@ -93,4 +81,4 @@ wget -q "https://raw.githubusercontent.com/mendersoftware/mender/master/support/ sudo chown -R 0 ${output_dir} sudo chgrp -R 0 ${output_dir} -echo "Configuration file for using Demo Mender Server written to: ${output_dir}/etc/mender" +echo "Configuration file for using Demo Mender Server written to: ${root_dir}/resources/mender.conf" diff --git a/scripts/bootstrap-rootfs-overlay-hosted-server.sh b/scripts/bootstrap-rootfs-overlay-hosted-server.sh index a6fdb92740..01b5b91393 100755 --- a/scripts/bootstrap-rootfs-overlay-hosted-server.sh +++ b/scripts/bootstrap-rootfs-overlay-hosted-server.sh @@ -57,20 +57,15 @@ if [ -e ${output_dir} ]; then sudo chown -R $(id -u) ${output_dir} sudo chgrp -R $(id -g) ${output_dir} fi -mkdir -p ${output_dir}/etc/mender -cat <<- EOF > ${output_dir}/etc/mender/mender.conf +mkdir -p ${root_dir}/resources +cat <<- EOF > ${root_dir}/resources/mender.conf { - "InventoryPollIntervalSeconds": 5, - "RetryPollIntervalSeconds": 30, "ServerURL": "https://hosted.mender.io/", - "TenantToken": "${tenant_token}", - "UpdatePollIntervalSeconds": 5 + "TenantToken": "${tenant_token}" } EOF -chmod 600 ${output_dir}/etc/mender/mender.conf - sudo chown -R 0 ${output_dir} sudo chgrp -R 0 ${output_dir} -echo "Configuration file for using Hosted Mender written to: ${output_dir}/etc/mender" +echo "Configuration file for using Hosted Mender written to: ${root_dir}/resources/mender.conf" diff --git a/scripts/bootstrap-rootfs-overlay-production-server.sh b/scripts/bootstrap-rootfs-overlay-production-server.sh index 212468c17a..deb28d0a0d 100755 --- a/scripts/bootstrap-rootfs-overlay-production-server.sh +++ b/scripts/bootstrap-rootfs-overlay-production-server.sh @@ -61,29 +61,21 @@ if [ -e ${output_dir} ]; then sudo chown -R $(id -u) ${output_dir} sudo chgrp -R $(id -g) ${output_dir} fi -mkdir -p ${output_dir}/etc/mender -cat <<- EOF > ${output_dir}/etc/mender/mender.conf + +mkdir -p ${root_dir}/resources +cat <<- EOF > ${root_dir}/resources/mender.conf { - "InventoryPollIntervalSeconds": 5, - "RetryPollIntervalSeconds": 30, "ServerURL": "${server_url}", EOF if [ -n "${server_cert}" ]; then - cat <<- EOF >> ${output_dir}/etc/mender/mender.conf - "ServerCertificate": "/etc/mender/server.crt", + cat <<- EOF >> ${root_dir}/resources/mender.conf + "ServerCertificate": "/etc/mender/server.crt" EOF cp -f "${server_cert}" ${output_dir}/etc/mender/server.crt fi -cat <<- EOF >> ${output_dir}/etc/mender/mender.conf - "UpdatePollIntervalSeconds": 5 -} -EOF - -chmod 600 ${output_dir}/etc/mender/mender.conf - sudo chown -R 0 ${output_dir} sudo chgrp -R 0 ${output_dir} -echo "Configuration file for using Production Mender Server written to: ${output_dir}/etc/mender" +echo "Configuration file for using Production Mender Server written to: ${root_dir}/resources/mender.conf" From d5f78cb2818b8af87e5bbd2b3034b3b989d58322 Mon Sep 17 00:00:00 2001 From: Ole Petter Date: Fri, 18 Jun 2021 17:59:35 +0200 Subject: [PATCH 011/117] Log: lowercase client Changelog: None Signed-off-by: Ole Petter --- mender-convert-modify | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mender-convert-modify b/mender-convert-modify index e68999d396..8022fbeaba 100755 --- a/mender-convert-modify +++ b/mender-convert-modify @@ -105,7 +105,7 @@ deb_arch=$(probe_debian_arch_name) if [ "${MENDER_CLIENT_INSTALL}" = "y" ]; then - log_info "Installing Mender Client version ${MENDER_CLIENT_VERSION}" + log_info "Installing Mender client version ${MENDER_CLIENT_VERSION}" if [ "${MENDER_CLIENT_VERSION}" = "latest" ]; then deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "stable" "mender-client") From 836a76b534f3539e6c65bdca86423d8b9e6a6686 Mon Sep 17 00:00:00 2001 From: Lluis Campos Date: Tue, 22 Jun 2021 12:38:16 +0200 Subject: [PATCH 012/117] run-tests.sh: Update Ubuntu Rpi image to 20.04 Changelog: None Signed-off-by: Lluis Campos --- scripts/test/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index 727fea657c..9c2d4b9025 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -29,7 +29,7 @@ RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbi UBUNTU_IMAGE_URL="https://downloads.mender.io/mender-convert/images/Ubuntu-Bionic-x86-64.img.gz" ## Auto-update -UBUNTU_SERVER_RPI_IMAGE_URL="http://cdimage.ubuntu.com/ubuntu/releases/18.04.5/release/ubuntu-18.04.5-preinstalled-server-armhf+raspi3.img.xz" +UBUNTU_SERVER_RPI_IMAGE_URL="http://cdimage.ubuntu.com/ubuntu/releases/20.04.2/release/ubuntu-20.04.2-preinstalled-server-armhf+raspi.img.xz" # Keep common function declarations in separate utils script UTILS_PATH=${0/$(basename $0)/test-utils.sh} From 54c6299d44e1399730eb5dd2ea45e4c680a89ec2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Jul 2021 07:01:54 +0000 Subject: [PATCH 013/117] Changelog:All: Bump tests/mender-image-tests from `80bd053` to `fd1d2d6` Bumps [tests/mender-image-tests](https://github.com/mendersoftware/mender-image-tests) from `80bd053` to `fd1d2d6`. - [Release notes](https://github.com/mendersoftware/mender-image-tests/releases) - [Commits](https://github.com/mendersoftware/mender-image-tests/compare/80bd05390e1c1a5a77265053c0bf2a956cf35470...fd1d2d6eb473602a14896e26cd017e088376ea1e) --- updated-dependencies: - dependency-name: tests/mender-image-tests dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/mender-image-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mender-image-tests b/tests/mender-image-tests index 80bd05390e..fd1d2d6eb4 160000 --- a/tests/mender-image-tests +++ b/tests/mender-image-tests @@ -1 +1 @@ -Subproject commit 80bd05390e1c1a5a77265053c0bf2a956cf35470 +Subproject commit fd1d2d6eb473602a14896e26cd017e088376ea1e From 81e8e938f04c459f4b1b8c6eaea1d88de48ed4b9 Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Fri, 9 Jul 2021 16:14:53 +0200 Subject: [PATCH 014/117] Update IRC links to Libera. Changelog: None Signed-off-by: Kristian Amlie --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4a71ef5d94..e06241b507 100644 --- a/README.md +++ b/README.md @@ -179,7 +179,7 @@ issue. We thank you in advance for your cooperation. * Fork us on [Github](https://github.com/mendersoftware) * Create an issue in the [bugtracker](https://tracker.mender.io/projects/MEN) * Email us at [contact@mender.io](mailto:contact@mender.io) -* Connect to the [#mender IRC channel on Freenode](http://webchat.freenode.net/?channels=mender) +* Connect to the [#mender IRC channel on Libera](https://web.libera.chat/?#mender) ## Authors From 441ccfaf5746966f5362c2a7fe868439a8afb2dc Mon Sep 17 00:00:00 2001 From: mender-test-bot Date: Mon, 12 Jul 2021 03:19:01 +0000 Subject: [PATCH 015/117] Image-Bot - Image updates Changelog: None Signed-off-by: mender-test-bot --- scripts/test/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index 9c2d4b9025..fa4df5b47d 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -21,7 +21,7 @@ BBB_DEBIAN_SDCARD_IMAGE_URL="https://debian.beagleboard.org/images/bone-debian-1 # Not on official home page, but found via https://elinux.org/Beagleboard:BeagleBoneBlack_Debian: ## Auto-update -BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-05-28/buster-console/bone-debian-10.9-console-armhf-2021-05-28-1gb.img.xz" +BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-07-09/buster-console/bone-debian-10.10-console-armhf-2021-07-09-1gb.img.xz" ## Auto-update RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2020-02-14/2020-02-13-raspbian-buster-lite.zip" From d26306a93253e6bd677f8f1b74a8e448ea1a36dd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jul 2021 07:01:51 +0000 Subject: [PATCH 016/117] Changelog:All: Bump tests/mender-image-tests from `fd1d2d6` to `798d07e` Bumps [tests/mender-image-tests](https://github.com/mendersoftware/mender-image-tests) from `fd1d2d6` to `798d07e`. - [Release notes](https://github.com/mendersoftware/mender-image-tests/releases) - [Commits](https://github.com/mendersoftware/mender-image-tests/compare/fd1d2d6eb473602a14896e26cd017e088376ea1e...798d07e60138b25cd9b7590a208c02371d68414a) --- updated-dependencies: - dependency-name: tests/mender-image-tests dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/mender-image-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mender-image-tests b/tests/mender-image-tests index fd1d2d6eb4..798d07e601 160000 --- a/tests/mender-image-tests +++ b/tests/mender-image-tests @@ -1 +1 @@ -Subproject commit fd1d2d6eb473602a14896e26cd017e088376ea1e +Subproject commit 798d07e60138b25cd9b7590a208c02371d68414a From daa7f81f011c71462f3089f069af7308e29c9a7b Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Thu, 22 Jul 2021 14:37:51 +0200 Subject: [PATCH 017/117] Switch to publish Raspberry Pi images using mender-convert version. This is done for several reasons: * The images do not contain the client anymore. * We want to have the mender-convert version in the name, so that we don't overwrite old, working images. We also add the build variable `MENDER_CONVERT_PUBLISH_VERSION`, to allow publishing of tagged mender-convert versions with small fixes on top. And finally, we introduce the `PUBLISH_MENDER_CONVERT` variable to publish images from tags. Branches are always published, even without this flag. Changelog: None Signed-off-by: Kristian Amlie --- .gitlab-ci.yml | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 67a457dc87..01463533a7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,6 +13,16 @@ variables: RASPBIAN_URL: https://downloads.raspberrypi.org/raspios_lite_armhf/images/raspios_lite_armhf-2021-01-12/2021-01-11-raspios-buster-armhf-lite.zip RASPBIAN_NAME: 2021-01-11-raspios-buster-armhf-lite + # Which version of mender-convert to use in published image name. Normally + # empty, in which case the branch or tag name will be used, but this variable + # is available so that it's possible to make small build fixes on an already + # tagged version. + MENDER_CONVERT_PUBLISH_VERSION: "" + + # Whether to publish images. Note that images from branches are always + # published, even if this is false. + PUBLISH_MENDER_CONVERT_IMAGES: "false" + DEBIAN_FRONTEND: noninteractive # Docker dind configuration. @@ -214,7 +224,6 @@ test_acceptance_ubuntu: - ./scripts/test/run-tests.sh --only ubuntu publish:s3: - when: manual stage: publish image: debian:buster before_script: @@ -233,15 +242,17 @@ publish:s3: - export AWS_ACCESS_KEY_ID=$PUBLISH_AWS_ACCESS_KEY_ID - export AWS_SECRET_ACCESS_KEY=$PUBLISH_AWS_SECRET_ACCESS_KEY + - IMAGE_VERSION=${MENDER_CONVERT_PUBLISH_VERSION:-${CI_COMMIT_REF_NAME}} + - for RASPBERRYPI_PLATFORM in raspberrypi3 raspberrypi4; do - - PUBLISH_NAME=${RASPBIAN_NAME}-${RASPBERRYPI_PLATFORM}-mender-${MENDER_CLIENT_VERSION}.img.xz + - PUBLISH_NAME=${RASPBIAN_NAME}-${RASPBERRYPI_PLATFORM}-mender-convert-${IMAGE_VERSION}.img.xz - echo "Publishing ${PUBLISH_NAME} version to S3" - aws s3 cp ${RASPBERRYPI_PLATFORM}/${RASPBIAN_NAME}-${RASPBERRYPI_PLATFORM}-mender.img.xz s3://$S3_BUCKET_NAME/${RASPBIAN_NAME}/arm/${PUBLISH_NAME} - aws s3api put-object-acl --acl public-read --bucket $S3_BUCKET_NAME --key ${RASPBIAN_NAME}/arm/${PUBLISH_NAME} - - PUBLISH_NAME=${RASPBIAN_NAME}-${RASPBERRYPI_PLATFORM}-mender-${MENDER_CLIENT_VERSION}.mender + - PUBLISH_NAME=${RASPBIAN_NAME}-${RASPBERRYPI_PLATFORM}-mender-convert-${IMAGE_VERSION}.mender - echo "Publishing ${PUBLISH_NAME} version to S3" - aws s3 cp ${RASPBERRYPI_PLATFORM}/${RASPBIAN_NAME}-${RASPBERRYPI_PLATFORM}-mender.mender s3://$S3_BUCKET_NAME/${RASPBIAN_NAME}/arm/${PUBLISH_NAME} @@ -249,5 +260,7 @@ publish:s3: --key ${RASPBIAN_NAME}/arm/${PUBLISH_NAME} - done - only: - - /^(master|[0-9]+\.[0-9]+\.x)$/ + rules: + - if: '$CI_COMMIT_REF_NAME =~ /^[0-9]+\.[0-9]+\.[0-9]+(-build[0-9]+)?$/ && $PUBLISH_MENDER_CONVERT_IMAGES == "true"' + - if: '$CI_COMMIT_REF_NAME =~ /^(master|[0-9]+\.[0-9]+\.x)$/' + - if: '$CI_COMMIT_REF_NAME !~ /^pr_[0-9]+$/ && $MENDER_CONVERT_PUBLISH_VERSION != "" && $PUBLISH_MENDER_CONVERT_IMAGES == "true"' From a3bcfd2a086d00e7fca8b56d2dad6fece2e0a3dc Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Thu, 22 Jul 2021 15:02:01 +0200 Subject: [PATCH 018/117] Make it possible to disable tests. Changelog: None Signed-off-by: Kristian Amlie --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 01463533a7..1817330ee8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,6 +13,8 @@ variables: RASPBIAN_URL: https://downloads.raspberrypi.org/raspios_lite_armhf/images/raspios_lite_armhf-2021-01-12/2021-01-11-raspios-buster-armhf-lite.zip RASPBIAN_NAME: 2021-01-11-raspios-buster-armhf-lite + TEST_MENDER_CONVERT: "true" + # Which version of mender-convert to use in published image name. Normally # empty, in which case the branch or tag name will be used, but this variable # is available so that it's possible to make small build fixes on an already @@ -171,6 +173,8 @@ convert_raspbian_raspberrypi4: - report_*.html reports: junit: results_*.xml + rules: + - if: '$TEST_MENDER_CONVERT == "true"' .template_test_acceptance_prebuilt_raspberrypi: &test_acceptance_prebuilt_raspberrypi <<: *test_acceptance From 9d18a8717d8812dba135c3e2231e786afeb1ac6c Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Fri, 23 Jul 2021 10:40:43 +0200 Subject: [PATCH 019/117] Add checksumming to artifacts from temporary S3 bucket. The temporary S3 bucket is not perfectly secure, because credentials can be obtained by outsiders by submitting a malicious pull request, and then later they can be used to manipulate objects while a privileged pipeline is running (during a release). Fix this by submitting a checksum file using the standard Gitlab artifact mechanism, and check that what we uploaded in one job, is what we get in the next one. Changelog: None Signed-off-by: Kristian Amlie --- .gitlab-ci.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1817330ee8..c75b4774a3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -107,11 +107,19 @@ test:check-shell-formatting: -c configs/images/raspberrypi_raspbian_config -c versions_override_config - # Upload to temporary S3 bucket + # Collect artifacts. - mv deploy ${RASPBERRYPI_PLATFORM} - tar czf ${RASPBERRYPI_PLATFORM}.tar.gz ${RASPBERRYPI_PLATFORM} + # Compute checksum + - mkdir checksums + - sha256sum ${RASPBERRYPI_PLATFORM}.tar.gz > checksums/${RASPBERRYPI_PLATFORM}.tar.gz.sha256 + # Upload to temporary S3 bucket - aws s3 cp ${RASPBERRYPI_PLATFORM}.tar.gz s3://mender-gitlab-tmp-storage/$CI_PROJECT_NAME/$CI_PIPELINE_ID/${RASPBERRYPI_PLATFORM}.tar.gz + artifacts: + paths: + - checksums + convert_raspbian_raspberrypi3: <<: *convert_raspbian variables: @@ -181,6 +189,8 @@ convert_raspbian_raspberrypi4: script: # Fetch artifacts from temporary S3 bucket - aws s3 cp s3://mender-gitlab-tmp-storage/$CI_PROJECT_NAME/$CI_PIPELINE_ID/${RASPBERRYPI_PLATFORM}.tar.gz ${RASPBERRYPI_PLATFORM}.tar.gz + # Check checksum + - sha256sum -c checksums/${RASPBERRYPI_PLATFORM}.tar.gz.sha256 - tar xzf ${RASPBERRYPI_PLATFORM}.tar.gz - mv ${RASPBERRYPI_PLATFORM} deploy # Extract converted Raspbian artifacts @@ -239,6 +249,8 @@ publish:s3: # Fetch artifacts from temporary S3 bucket - for RASPBERRYPI_PLATFORM in raspberrypi3 raspberrypi4; do - aws s3 cp s3://mender-gitlab-tmp-storage/$CI_PROJECT_NAME/$CI_PIPELINE_ID/${RASPBERRYPI_PLATFORM}.tar.gz ${RASPBERRYPI_PLATFORM}.tar.gz + # Check checksum + - sha256sum -c checksums/${RASPBERRYPI_PLATFORM}.tar.gz.sha256 - tar xzf ${RASPBERRYPI_PLATFORM}.tar.gz - done script: From 6ebe9dd913912944fd58056f0a47028af17ee398 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jul 2021 07:01:50 +0000 Subject: [PATCH 020/117] Changelog:All: Bump tests/mender-image-tests from `798d07e` to `9585db2` Bumps [tests/mender-image-tests](https://github.com/mendersoftware/mender-image-tests) from `798d07e` to `9585db2`. - [Release notes](https://github.com/mendersoftware/mender-image-tests/releases) - [Commits](https://github.com/mendersoftware/mender-image-tests/compare/798d07e60138b25cd9b7590a208c02371d68414a...9585db20894713083e48aae56db316761e8426fc) --- updated-dependencies: - dependency-name: tests/mender-image-tests dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/mender-image-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mender-image-tests b/tests/mender-image-tests index 798d07e601..9585db2089 160000 --- a/tests/mender-image-tests +++ b/tests/mender-image-tests @@ -1 +1 @@ -Subproject commit 798d07e60138b25cd9b7590a208c02371d68414a +Subproject commit 9585db20894713083e48aae56db316761e8426fc From 1666baa0e2d9b1c248bef8809d7d86fbe8e3a7fa Mon Sep 17 00:00:00 2001 From: mender-test-bot Date: Mon, 2 Aug 2021 03:18:11 +0000 Subject: [PATCH 021/117] Image-Bot - Image updates Changelog: None Signed-off-by: mender-test-bot --- scripts/test/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index fa4df5b47d..daf476eee9 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -21,7 +21,7 @@ BBB_DEBIAN_SDCARD_IMAGE_URL="https://debian.beagleboard.org/images/bone-debian-1 # Not on official home page, but found via https://elinux.org/Beagleboard:BeagleBoneBlack_Debian: ## Auto-update -BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-07-09/buster-console/bone-debian-10.10-console-armhf-2021-07-09-1gb.img.xz" +BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-07-26/buster-console/bone-debian-10.10-console-armhf-2021-07-26-1gb.img.xz" ## Auto-update RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2020-02-14/2020-02-13-raspbian-buster-lite.zip" From e11d02b8c176da2abc4df3661f86f5344bab702c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Aug 2021 07:05:14 +0000 Subject: [PATCH 022/117] Changelog:All: Bump jsdom from 16.6.0 to 16.7.0 in /scripts/linkbot Bumps [jsdom](https://github.com/jsdom/jsdom) from 16.6.0 to 16.7.0. - [Release notes](https://github.com/jsdom/jsdom/releases) - [Changelog](https://github.com/jsdom/jsdom/blob/master/Changelog.md) - [Commits](https://github.com/jsdom/jsdom/compare/16.6.0...16.7.0) --- updated-dependencies: - dependency-name: jsdom dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- scripts/linkbot/package-lock.json | 30 +++++++++++++++--------------- scripts/linkbot/package.json | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/scripts/linkbot/package-lock.json b/scripts/linkbot/package-lock.json index 4905072357..1464ce76e7 100644 --- a/scripts/linkbot/package-lock.json +++ b/scripts/linkbot/package-lock.json @@ -6,7 +6,7 @@ "": { "dependencies": { "bent": "^7.3.12", - "jsdom": "^16.6.0" + "jsdom": "^16.7.0" } }, "node_modules/@tootallnate/once": { @@ -317,9 +317,9 @@ } }, "node_modules/jsdom": { - "version": "16.6.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.6.0.tgz", - "integrity": "sha512-Ty1vmF4NHJkolaEmdjtxTfSfkdb8Ywarwf63f+F8/mDD1uLSSWDxDuMiZxiPhwunLrn9LOSVItWj4bLYsLN3Dg==", + "version": "16.7.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.7.0.tgz", + "integrity": "sha512-u9Smc2G1USStM+s/x1ru5Sxrl6mPYCbByG1U/hUmqaVsm4tbNyS7CicOSRyuGQYZhTu0h84qkZZQ/I+dzizSVw==", "dependencies": { "abab": "^2.0.5", "acorn": "^8.2.4", @@ -346,7 +346,7 @@ "whatwg-encoding": "^1.0.5", "whatwg-mimetype": "^2.3.0", "whatwg-url": "^8.5.0", - "ws": "^7.4.5", + "ws": "^7.4.6", "xml-name-validator": "^3.0.0" }, "engines": { @@ -595,9 +595,9 @@ } }, "node_modules/ws": { - "version": "7.4.5", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.5.tgz", - "integrity": "sha512-xzyu3hFvomRfXKH8vOFMU3OguG6oOvhXMo3xsGy3xWExqaM2dxBbVxuD99O7m3ZUFMvvscsZDqxfgMaRr/Nr1g==", + "version": "7.5.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.3.tgz", + "integrity": "sha512-kQ/dHIzuLrS6Je9+uv81ueZomEwH0qVYstcAQ4/Z93K8zeko9gtAbttJWzoC5ukqXY1PpoouV3+VSOqEAFt5wg==", "engines": { "node": ">=8.3.0" }, @@ -857,9 +857,9 @@ "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==" }, "jsdom": { - "version": "16.6.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.6.0.tgz", - "integrity": "sha512-Ty1vmF4NHJkolaEmdjtxTfSfkdb8Ywarwf63f+F8/mDD1uLSSWDxDuMiZxiPhwunLrn9LOSVItWj4bLYsLN3Dg==", + "version": "16.7.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.7.0.tgz", + "integrity": "sha512-u9Smc2G1USStM+s/x1ru5Sxrl6mPYCbByG1U/hUmqaVsm4tbNyS7CicOSRyuGQYZhTu0h84qkZZQ/I+dzizSVw==", "requires": { "abab": "^2.0.5", "acorn": "^8.2.4", @@ -886,7 +886,7 @@ "whatwg-encoding": "^1.0.5", "whatwg-mimetype": "^2.3.0", "whatwg-url": "^8.5.0", - "ws": "^7.4.5", + "ws": "^7.4.6", "xml-name-validator": "^3.0.0" }, "dependencies": { @@ -1072,9 +1072,9 @@ "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==" }, "ws": { - "version": "7.4.5", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.5.tgz", - "integrity": "sha512-xzyu3hFvomRfXKH8vOFMU3OguG6oOvhXMo3xsGy3xWExqaM2dxBbVxuD99O7m3ZUFMvvscsZDqxfgMaRr/Nr1g==", + "version": "7.5.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.3.tgz", + "integrity": "sha512-kQ/dHIzuLrS6Je9+uv81ueZomEwH0qVYstcAQ4/Z93K8zeko9gtAbttJWzoC5ukqXY1PpoouV3+VSOqEAFt5wg==", "requires": {} }, "xml-name-validator": { diff --git a/scripts/linkbot/package.json b/scripts/linkbot/package.json index 4b96a33d60..edb5932d70 100644 --- a/scripts/linkbot/package.json +++ b/scripts/linkbot/package.json @@ -1,6 +1,6 @@ { "dependencies": { "bent": "^7.3.12", - "jsdom": "^16.6.0" + "jsdom": "^16.7.0" } } From 9520332ed18242adb67b4ba52cb10ad28f6955ff Mon Sep 17 00:00:00 2001 From: Ole Petter Date: Tue, 29 Jun 2021 14:56:00 +0200 Subject: [PATCH 023/117] Improve the Ubuntu Server linkbot monitoring Changelog: None Signed-off-by: Ole Petter --- scripts/linkbot/monitor-ub-server.js | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/scripts/linkbot/monitor-ub-server.js b/scripts/linkbot/monitor-ub-server.js index 39db954c30..836166d30a 100644 --- a/scripts/linkbot/monitor-ub-server.js +++ b/scripts/linkbot/monitor-ub-server.js @@ -3,6 +3,7 @@ const { JSDOM } = jsdom; const fs = require('fs') const { updateURLLink } = require('./common'); +const target = "UBUNTU_SERVER_RPI_IMAGE_URL" const url = "http://cdimage.ubuntu.com/ubuntu/releases/" const reg = ".*(?[0-9]{2})\.04\.?(?[0-9]{1})?.*" @@ -10,7 +11,7 @@ const reg = ".*(?[0-9]{2})\.04\.?(?[0-9]{1})?.*" try { const data = fs.readFileSync('../test/run-tests.sh', 'utf8') .split('\n') - .filter(line => line.match("UBUNTU_SERVER_RPI_IMAGE_URL=.*")) + .filter(line => line.match(`${target}=.*`)) var line = data[0] var m = line.match(`.*=\"${reg}\"`) var imageName = m.groups.release @@ -33,14 +34,20 @@ JSDOM.fromURL(url, {}).then(dom => { return parseInt(b.groups.release) - parseInt(a.groups.release) || parseFloat(b.groups.minor) - parseFloat(a.groups.minor) }) var matchOn = matches[0].input - if (matchOn !== imageName) { - console.log("We've got a new release! \\o/"); - var newLine = "" - if (matches[0].groups.minor) { - newLine = `UBUNTU_SERVER_RPI_IMAGE_URL=\"${url}${matches[0].groups.release}.04.${matches[0].groups.minor}/release/ubuntu-${matches[0].groups.release}.04.${matches[0].groups.minor}-preinstalled-server-armhf+raspi.img.xz\"` - } else { - newLine = `UBUNTU_SERVER_RPI_IMAGE_URL=\"${url}${matches[0].groups.release}.04/release/ubuntu-${matches[0].groups.release}.04-preinstalled-server-armhf+raspi.img.xz\"` + + return matchOn; + + // Get the release image url from the releases (sub)-page + // const url = "http://cdimage.ubuntu.com/ubuntu/releases/" +}).then(releaseVersion => { + var releaseVersion = releaseVersion.replace(/\s/g, "").replace(/\//g, "") + JSDOM.fromURL(`${url}${releaseVersion}/release/`, {}).then(dom => { + var document = dom.window.document; + var refs = document.getElementsByTagName("a"); + const match = Array.from(refs).find(ref => ref.href.match(`.*ubuntu-${releaseVersion}-preinstalled-server-armhf.*\.img\.xz$`)) + if (match) { + console.log(`Ubuntu server image has a new release: ${match}`) + updateURLLink(`${target}=${match}`, target) } - updateURLLink(newLine) - } + }) }); From 2ecfbaabe24c61840ec8a8ac4771b7a73bf353e5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Aug 2021 07:05:37 +0000 Subject: [PATCH 024/117] Changelog:All: Bump jsdom from 16.7.0 to 17.0.0 in /scripts/linkbot Bumps [jsdom](https://github.com/jsdom/jsdom) from 16.7.0 to 17.0.0. - [Release notes](https://github.com/jsdom/jsdom/releases) - [Changelog](https://github.com/jsdom/jsdom/blob/master/Changelog.md) - [Commits](https://github.com/jsdom/jsdom/compare/16.7.0...17.0.0) --- updated-dependencies: - dependency-name: jsdom dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- scripts/linkbot/package-lock.json | 194 ++++++++++++++---------------- scripts/linkbot/package.json | 2 +- 2 files changed, 92 insertions(+), 104 deletions(-) diff --git a/scripts/linkbot/package-lock.json b/scripts/linkbot/package-lock.json index 1464ce76e7..6f52a220bf 100644 --- a/scripts/linkbot/package-lock.json +++ b/scripts/linkbot/package-lock.json @@ -6,7 +6,7 @@ "": { "dependencies": { "bent": "^7.3.12", - "jsdom": "^16.7.0" + "jsdom": "^17.0.0" } }, "node_modules/@tootallnate/once": { @@ -103,9 +103,9 @@ } }, "node_modules/cssom": { - "version": "0.4.4", - "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.4.4.tgz", - "integrity": "sha512-p3pvU7r1MyyqbTk+WbNJIgJjG2VmTIaB10rI93LzVPrmDJKkzKYMtxxyAvQXR/NS6otuzveI7+7BBq3SjBS2mw==" + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz", + "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==" }, "node_modules/cssstyle": { "version": "2.3.0", @@ -124,16 +124,16 @@ "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==" }, "node_modules/data-urls": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz", - "integrity": "sha512-X5eWTSXO/BJmpdIKCRuKUgSCgAN0OwliVK3yPKbwIWU1Tdw5BRajxlzMidvh+gwko9AfQ9zIj52pzF91Q3YAvQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.0.tgz", + "integrity": "sha512-4AefxbTTdFtxDUdh0BuMBs2qJVL25Mow2zlcuuePegQwgD6GEmQao42LLEeksOui8nL4RcNEugIpFP7eRd33xg==", "dependencies": { "abab": "^2.0.3", "whatwg-mimetype": "^2.3.0", - "whatwg-url": "^8.0.0" + "whatwg-url": "^9.0.0" }, "engines": { - "node": ">=10" + "node": ">=12" } }, "node_modules/debug": { @@ -153,9 +153,9 @@ } }, "node_modules/decimal.js": { - "version": "10.2.1", - "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.2.1.tgz", - "integrity": "sha512-KaL7+6Fw6i5A2XSnsbhm/6B+NuEA7TZ4vqxnd5tXz9sbKtrN9Srj8ab4vKVdK8YAqZO9P1kg45Y6YLoduPf+kw==" + "version": "10.3.1", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.3.1.tgz", + "integrity": "sha512-V0pfhfr8suzyPGOx3nmq4aHqabehUZn6Ch9kyFpV79TGDTWFmHqUqXdabR7QHqxzrYolF4+tVmJhUG4OURg5dQ==" }, "node_modules/deep-is": { "version": "0.1.3", @@ -244,9 +244,9 @@ "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=" }, "node_modules/form-data": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", - "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", @@ -317,20 +317,20 @@ } }, "node_modules/jsdom": { - "version": "16.7.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.7.0.tgz", - "integrity": "sha512-u9Smc2G1USStM+s/x1ru5Sxrl6mPYCbByG1U/hUmqaVsm4tbNyS7CicOSRyuGQYZhTu0h84qkZZQ/I+dzizSVw==", + "version": "17.0.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-17.0.0.tgz", + "integrity": "sha512-MUq4XdqwtNurZDVeKScENMPHnkgmdIvMzZ1r1NSwHkDuaqI6BouPjr+17COo4/19oLNnmdpFDPOHVpgIZmZ+VA==", "dependencies": { "abab": "^2.0.5", - "acorn": "^8.2.4", + "acorn": "^8.4.1", "acorn-globals": "^6.0.0", - "cssom": "^0.4.4", + "cssom": "^0.5.0", "cssstyle": "^2.3.0", - "data-urls": "^2.0.0", - "decimal.js": "^10.2.1", + "data-urls": "^3.0.0", + "decimal.js": "^10.3.1", "domexception": "^2.0.1", "escodegen": "^2.0.0", - "form-data": "^3.0.0", + "form-data": "^4.0.0", "html-encoding-sniffer": "^2.0.1", "http-proxy-agent": "^4.0.1", "https-proxy-agent": "^5.0.0", @@ -345,12 +345,12 @@ "webidl-conversions": "^6.1.0", "whatwg-encoding": "^1.0.5", "whatwg-mimetype": "^2.3.0", - "whatwg-url": "^8.5.0", - "ws": "^7.4.6", + "whatwg-url": "^9.0.0", + "ws": "^8.0.0", "xml-name-validator": "^3.0.0" }, "engines": { - "node": ">=10" + "node": ">=12" }, "peerDependencies": { "canvas": "^2.5.0" @@ -362,9 +362,9 @@ } }, "node_modules/jsdom/node_modules/acorn": { - "version": "8.2.4", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.2.4.tgz", - "integrity": "sha512-Ibt84YwBDDA890eDiDCEqcbwvHlBvzzDkU2cGBBDDI1QWT12jTiXIOn2CIw5KK4i6N5Z2HUxwYjzriDyqaqqZg==", + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.4.1.tgz", + "integrity": "sha512-asabaBSkEKosYKMITunzX177CXxQ4Q8BSSzMTKD+FefUhipQC70gfW5SiUDhYQ3vk8G+81HqQk7Fv9OXwwn9KA==", "bin": { "acorn": "bin/acorn" }, @@ -384,25 +384,20 @@ "node": ">= 0.8.0" } }, - "node_modules/lodash": { - "version": "4.17.20", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" - }, "node_modules/mime-db": { - "version": "1.47.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.47.0.tgz", - "integrity": "sha512-QBmA/G2y+IfeS4oktet3qRZ+P5kPhCKRXxXnQEudYqUaEioAU1/Lq2us3D/t1Jfo4hE9REQPrbB7K5sOczJVIw==", + "version": "1.49.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.49.0.tgz", + "integrity": "sha512-CIc8j9URtOVApSFCQIF+VBkX1RwXp/oMMOrqdyXSBXq5RWNEsRfyj1kiRnQgmNXmHxPoFIxOroKA3zcU9P+nAA==", "engines": { "node": ">= 0.6" } }, "node_modules/mime-types": { - "version": "2.1.30", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.30.tgz", - "integrity": "sha512-crmjA4bLtR8m9qLpHvgxSChT+XoSlZi8J4n/aIdn3z92e/U47Z0V/yl+Wh9W046GgFVAmoNR/fmdbZYcSSIUeg==", + "version": "2.1.32", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.32.tgz", + "integrity": "sha512-hJGaVS4G4c9TSMYh2n6SQAGrC4RnfU+daP8G7cSCmaqNjiOoUY0VHCMS42pxnQmVF1GWwFhbHWn3RIxCqTmZ9A==", "dependencies": { - "mime-db": "1.47.0" + "mime-db": "1.49.0" }, "engines": { "node": ">= 0.6" @@ -504,9 +499,9 @@ } }, "node_modules/tr46": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-2.0.2.tgz", - "integrity": "sha512-3n1qG+/5kg+jrbTzwAykB5yRYtQCTqOGKq5U5PE3b0a1/mzo6snDhjGS0zJVJunO0NrT3Dg1MLy5TjWP/UJppg==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-2.1.0.tgz", + "integrity": "sha512-15Ih7phfcdP5YxqiB+iDtLoaTz4Nd35+IiAv0kQ5FNKHzXgdWqPoTIqEDDJmXceQt4JZk6lVPT8lnDlPpGDppw==", "dependencies": { "punycode": "^2.1.1" }, @@ -574,16 +569,15 @@ "integrity": "sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==" }, "node_modules/whatwg-url": { - "version": "8.5.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-8.5.0.tgz", - "integrity": "sha512-fy+R77xWv0AiqfLl4nuGUlQ3/6b5uNfQ4WAbGQVMYshCTCCPK9psC1nWh3XHuxGVCtlcDDQPQW1csmmIQo+fwg==", + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-9.1.0.tgz", + "integrity": "sha512-CQ0UcrPHyomtlOCot1TL77WyMIm/bCwrJ2D6AOKGwEczU9EpyoqAokfqrf/MioU9kHcMsmJZcg1egXix2KYEsA==", "dependencies": { - "lodash": "^4.7.0", - "tr46": "^2.0.2", + "tr46": "^2.1.0", "webidl-conversions": "^6.1.0" }, "engines": { - "node": ">=10" + "node": ">=12" } }, "node_modules/word-wrap": { @@ -595,11 +589,11 @@ } }, "node_modules/ws": { - "version": "7.5.3", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.3.tgz", - "integrity": "sha512-kQ/dHIzuLrS6Je9+uv81ueZomEwH0qVYstcAQ4/Z93K8zeko9gtAbttJWzoC5ukqXY1PpoouV3+VSOqEAFt5wg==", + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.1.0.tgz", + "integrity": "sha512-0UWlCD2s3RSclw8FN+D0zDTUyMO+1kHwJQQJzkgUh16S8d3NYON0AKCEQPffE0ez4JyRFu76QDA9KR5bOG/7jw==", "engines": { - "node": ">=8.3.0" + "node": ">=10.0.0" }, "peerDependencies": { "bufferutil": "^4.0.1", @@ -702,9 +696,9 @@ } }, "cssom": { - "version": "0.4.4", - "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.4.4.tgz", - "integrity": "sha512-p3pvU7r1MyyqbTk+WbNJIgJjG2VmTIaB10rI93LzVPrmDJKkzKYMtxxyAvQXR/NS6otuzveI7+7BBq3SjBS2mw==" + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz", + "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==" }, "cssstyle": { "version": "2.3.0", @@ -722,13 +716,13 @@ } }, "data-urls": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz", - "integrity": "sha512-X5eWTSXO/BJmpdIKCRuKUgSCgAN0OwliVK3yPKbwIWU1Tdw5BRajxlzMidvh+gwko9AfQ9zIj52pzF91Q3YAvQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.0.tgz", + "integrity": "sha512-4AefxbTTdFtxDUdh0BuMBs2qJVL25Mow2zlcuuePegQwgD6GEmQao42LLEeksOui8nL4RcNEugIpFP7eRd33xg==", "requires": { "abab": "^2.0.3", "whatwg-mimetype": "^2.3.0", - "whatwg-url": "^8.0.0" + "whatwg-url": "^9.0.0" } }, "debug": { @@ -740,9 +734,9 @@ } }, "decimal.js": { - "version": "10.2.1", - "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.2.1.tgz", - "integrity": "sha512-KaL7+6Fw6i5A2XSnsbhm/6B+NuEA7TZ4vqxnd5tXz9sbKtrN9Srj8ab4vKVdK8YAqZO9P1kg45Y6YLoduPf+kw==" + "version": "10.3.1", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.3.1.tgz", + "integrity": "sha512-V0pfhfr8suzyPGOx3nmq4aHqabehUZn6Ch9kyFpV79TGDTWFmHqUqXdabR7QHqxzrYolF4+tVmJhUG4OURg5dQ==" }, "deep-is": { "version": "0.1.3", @@ -802,9 +796,9 @@ "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=" }, "form-data": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", - "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", "requires": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", @@ -857,20 +851,20 @@ "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==" }, "jsdom": { - "version": "16.7.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.7.0.tgz", - "integrity": "sha512-u9Smc2G1USStM+s/x1ru5Sxrl6mPYCbByG1U/hUmqaVsm4tbNyS7CicOSRyuGQYZhTu0h84qkZZQ/I+dzizSVw==", + "version": "17.0.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-17.0.0.tgz", + "integrity": "sha512-MUq4XdqwtNurZDVeKScENMPHnkgmdIvMzZ1r1NSwHkDuaqI6BouPjr+17COo4/19oLNnmdpFDPOHVpgIZmZ+VA==", "requires": { "abab": "^2.0.5", - "acorn": "^8.2.4", + "acorn": "^8.4.1", "acorn-globals": "^6.0.0", - "cssom": "^0.4.4", + "cssom": "^0.5.0", "cssstyle": "^2.3.0", - "data-urls": "^2.0.0", - "decimal.js": "^10.2.1", + "data-urls": "^3.0.0", + "decimal.js": "^10.3.1", "domexception": "^2.0.1", "escodegen": "^2.0.0", - "form-data": "^3.0.0", + "form-data": "^4.0.0", "html-encoding-sniffer": "^2.0.1", "http-proxy-agent": "^4.0.1", "https-proxy-agent": "^5.0.0", @@ -885,15 +879,15 @@ "webidl-conversions": "^6.1.0", "whatwg-encoding": "^1.0.5", "whatwg-mimetype": "^2.3.0", - "whatwg-url": "^8.5.0", - "ws": "^7.4.6", + "whatwg-url": "^9.0.0", + "ws": "^8.0.0", "xml-name-validator": "^3.0.0" }, "dependencies": { "acorn": { - "version": "8.2.4", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.2.4.tgz", - "integrity": "sha512-Ibt84YwBDDA890eDiDCEqcbwvHlBvzzDkU2cGBBDDI1QWT12jTiXIOn2CIw5KK4i6N5Z2HUxwYjzriDyqaqqZg==" + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.4.1.tgz", + "integrity": "sha512-asabaBSkEKosYKMITunzX177CXxQ4Q8BSSzMTKD+FefUhipQC70gfW5SiUDhYQ3vk8G+81HqQk7Fv9OXwwn9KA==" } } }, @@ -906,22 +900,17 @@ "type-check": "~0.3.2" } }, - "lodash": { - "version": "4.17.20", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" - }, "mime-db": { - "version": "1.47.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.47.0.tgz", - "integrity": "sha512-QBmA/G2y+IfeS4oktet3qRZ+P5kPhCKRXxXnQEudYqUaEioAU1/Lq2us3D/t1Jfo4hE9REQPrbB7K5sOczJVIw==" + "version": "1.49.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.49.0.tgz", + "integrity": "sha512-CIc8j9URtOVApSFCQIF+VBkX1RwXp/oMMOrqdyXSBXq5RWNEsRfyj1kiRnQgmNXmHxPoFIxOroKA3zcU9P+nAA==" }, "mime-types": { - "version": "2.1.30", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.30.tgz", - "integrity": "sha512-crmjA4bLtR8m9qLpHvgxSChT+XoSlZi8J4n/aIdn3z92e/U47Z0V/yl+Wh9W046GgFVAmoNR/fmdbZYcSSIUeg==", + "version": "2.1.32", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.32.tgz", + "integrity": "sha512-hJGaVS4G4c9TSMYh2n6SQAGrC4RnfU+daP8G7cSCmaqNjiOoUY0VHCMS42pxnQmVF1GWwFhbHWn3RIxCqTmZ9A==", "requires": { - "mime-db": "1.47.0" + "mime-db": "1.49.0" } }, "ms": { @@ -1002,9 +991,9 @@ } }, "tr46": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-2.0.2.tgz", - "integrity": "sha512-3n1qG+/5kg+jrbTzwAykB5yRYtQCTqOGKq5U5PE3b0a1/mzo6snDhjGS0zJVJunO0NrT3Dg1MLy5TjWP/UJppg==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-2.1.0.tgz", + "integrity": "sha512-15Ih7phfcdP5YxqiB+iDtLoaTz4Nd35+IiAv0kQ5FNKHzXgdWqPoTIqEDDJmXceQt4JZk6lVPT8lnDlPpGDppw==", "requires": { "punycode": "^2.1.1" } @@ -1057,12 +1046,11 @@ "integrity": "sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==" }, "whatwg-url": { - "version": "8.5.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-8.5.0.tgz", - "integrity": "sha512-fy+R77xWv0AiqfLl4nuGUlQ3/6b5uNfQ4WAbGQVMYshCTCCPK9psC1nWh3XHuxGVCtlcDDQPQW1csmmIQo+fwg==", + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-9.1.0.tgz", + "integrity": "sha512-CQ0UcrPHyomtlOCot1TL77WyMIm/bCwrJ2D6AOKGwEczU9EpyoqAokfqrf/MioU9kHcMsmJZcg1egXix2KYEsA==", "requires": { - "lodash": "^4.7.0", - "tr46": "^2.0.2", + "tr46": "^2.1.0", "webidl-conversions": "^6.1.0" } }, @@ -1072,9 +1060,9 @@ "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==" }, "ws": { - "version": "7.5.3", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.3.tgz", - "integrity": "sha512-kQ/dHIzuLrS6Je9+uv81ueZomEwH0qVYstcAQ4/Z93K8zeko9gtAbttJWzoC5ukqXY1PpoouV3+VSOqEAFt5wg==", + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.1.0.tgz", + "integrity": "sha512-0UWlCD2s3RSclw8FN+D0zDTUyMO+1kHwJQQJzkgUh16S8d3NYON0AKCEQPffE0ez4JyRFu76QDA9KR5bOG/7jw==", "requires": {} }, "xml-name-validator": { diff --git a/scripts/linkbot/package.json b/scripts/linkbot/package.json index edb5932d70..145a096192 100644 --- a/scripts/linkbot/package.json +++ b/scripts/linkbot/package.json @@ -1,6 +1,6 @@ { "dependencies": { "bent": "^7.3.12", - "jsdom": "^16.7.0" + "jsdom": "^17.0.0" } } From f23e1a06ee069c53afe8e28c279ddeaafc2fd73b Mon Sep 17 00:00:00 2001 From: Ole Petter Date: Mon, 16 Aug 2021 11:31:15 +0200 Subject: [PATCH 025/117] Linkbot: Have the Ubuntu images monitored only by LTS releases We do not want non-LTS releases covered by the linkbot. Changelog: None Signed-off-by: Ole Petter --- scripts/linkbot/monitor-ub-server.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/linkbot/monitor-ub-server.js b/scripts/linkbot/monitor-ub-server.js index 836166d30a..acf7ce71aa 100644 --- a/scripts/linkbot/monitor-ub-server.js +++ b/scripts/linkbot/monitor-ub-server.js @@ -26,12 +26,14 @@ JSDOM.fromURL(url, {}).then(dom => { var refs = document.getElementsByTagName("a"); var matches = Array.from(refs) .filter(ref => ref.textContent.match(reg)) + .filter(ref => parseInt(ref.textContent.match(reg).groups.release) % 2 == 0) // Only LTS + .filter(ref => !ref.textContent.match(reg).groups.minor) // Ignore the minor number .reduce((acc, ref) => { acc.push(ref.textContent.match(reg)) return acc }, []) .sort((a,b) => { - return parseInt(b.groups.release) - parseInt(a.groups.release) || parseFloat(b.groups.minor) - parseFloat(a.groups.minor) + return parseInt(b.groups.release) - parseInt(a.groups.release) }) var matchOn = matches[0].input @@ -44,7 +46,7 @@ JSDOM.fromURL(url, {}).then(dom => { JSDOM.fromURL(`${url}${releaseVersion}/release/`, {}).then(dom => { var document = dom.window.document; var refs = document.getElementsByTagName("a"); - const match = Array.from(refs).find(ref => ref.href.match(`.*ubuntu-${releaseVersion}-preinstalled-server-armhf.*\.img\.xz$`)) + const match = Array.from(refs).find(ref => ref.href.match(`.*ubuntu-${releaseVersion}\.?[0-9]+-preinstalled-server-armhf.*\.img\.xz$`)) if (match) { console.log(`Ubuntu server image has a new release: ${match}`) updateURLLink(`${target}=${match}`, target) From 49b9409c39921dc62fae63debaeb8f53bfed9517 Mon Sep 17 00:00:00 2001 From: Ole Petter Date: Mon, 16 Aug 2021 14:11:48 +0200 Subject: [PATCH 026/117] Linkbot: Quote the URL string in Bash Changelog: None Signed-off-by: Ole Petter --- scripts/linkbot/monitor-ub-server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/linkbot/monitor-ub-server.js b/scripts/linkbot/monitor-ub-server.js index acf7ce71aa..06edc43b01 100644 --- a/scripts/linkbot/monitor-ub-server.js +++ b/scripts/linkbot/monitor-ub-server.js @@ -49,7 +49,7 @@ JSDOM.fromURL(url, {}).then(dom => { const match = Array.from(refs).find(ref => ref.href.match(`.*ubuntu-${releaseVersion}\.?[0-9]+-preinstalled-server-armhf.*\.img\.xz$`)) if (match) { console.log(`Ubuntu server image has a new release: ${match}`) - updateURLLink(`${target}=${match}`, target) + updateURLLink(`${target}=\"${match}\"`, target) } }) }); From f075b8f137c39697389468c28dfcf5dcdc210cb5 Mon Sep 17 00:00:00 2001 From: Ole Petter Date: Mon, 16 Aug 2021 14:12:11 +0200 Subject: [PATCH 027/117] Linkbot: Move to using Ubuntu major.minor versions, ignore the patch Changelog: None Signed-off-by: Ole Petter --- scripts/test/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index daf476eee9..9b5d993d30 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -29,7 +29,7 @@ RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbi UBUNTU_IMAGE_URL="https://downloads.mender.io/mender-convert/images/Ubuntu-Bionic-x86-64.img.gz" ## Auto-update -UBUNTU_SERVER_RPI_IMAGE_URL="http://cdimage.ubuntu.com/ubuntu/releases/20.04.2/release/ubuntu-20.04.2-preinstalled-server-armhf+raspi.img.xz" +UBUNTU_SERVER_RPI_IMAGE_URL="http://cdimage.ubuntu.com/ubuntu/releases/20.04/release/ubuntu-20.04.2-preinstalled-server-armhf+raspi.img.xz" # Keep common function declarations in separate utils script UTILS_PATH=${0/$(basename $0)/test-utils.sh} From 924fe709504dc0ecd3fa0137e349b50ab4d9d454 Mon Sep 17 00:00:00 2001 From: Ole Petter Date: Tue, 17 Aug 2021 10:27:41 +0200 Subject: [PATCH 028/117] linkbot: Move the LTS verification from the filter to the regexp Changelog: None Signed-off-by: Ole Petter --- scripts/linkbot/monitor-ub-server.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/linkbot/monitor-ub-server.js b/scripts/linkbot/monitor-ub-server.js index 06edc43b01..28bfb6e468 100644 --- a/scripts/linkbot/monitor-ub-server.js +++ b/scripts/linkbot/monitor-ub-server.js @@ -5,7 +5,7 @@ const { updateURLLink } = require('./common'); const target = "UBUNTU_SERVER_RPI_IMAGE_URL" const url = "http://cdimage.ubuntu.com/ubuntu/releases/" -const reg = ".*(?[0-9]{2})\.04\.?(?[0-9]{1})?.*" +const reg = ".*(?[0-9][02468])\.04\.?(?[0-9]{1})?.*" // Read the input file, and parse the variable input try { @@ -26,7 +26,6 @@ JSDOM.fromURL(url, {}).then(dom => { var refs = document.getElementsByTagName("a"); var matches = Array.from(refs) .filter(ref => ref.textContent.match(reg)) - .filter(ref => parseInt(ref.textContent.match(reg).groups.release) % 2 == 0) // Only LTS .filter(ref => !ref.textContent.match(reg).groups.minor) // Ignore the minor number .reduce((acc, ref) => { acc.push(ref.textContent.match(reg)) From 57a0b0d7999cde1882218a6f6d4890e95c0a135d Mon Sep 17 00:00:00 2001 From: mender-test-bot Date: Mon, 23 Aug 2021 03:17:06 +0000 Subject: [PATCH 029/117] Image-Bot - Image updates Changelog: None Signed-off-by: mender-test-bot --- scripts/test/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index 9b5d993d30..8b0915340d 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -21,7 +21,7 @@ BBB_DEBIAN_SDCARD_IMAGE_URL="https://debian.beagleboard.org/images/bone-debian-1 # Not on official home page, but found via https://elinux.org/Beagleboard:BeagleBoneBlack_Debian: ## Auto-update -BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-07-26/buster-console/bone-debian-10.10-console-armhf-2021-07-26-1gb.img.xz" +BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-08-16/buster-console/bone-debian-10.10-console-armhf-2021-08-16-1gb.img.xz" ## Auto-update RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2020-02-14/2020-02-13-raspbian-buster-lite.zip" From 4ec9e26055a0c1080f3afe6c335a9600b47a6958 Mon Sep 17 00:00:00 2001 From: mender-test-bot Date: Mon, 30 Aug 2021 03:16:57 +0000 Subject: [PATCH 030/117] Image-Bot - Image updates Changelog: None Signed-off-by: mender-test-bot --- scripts/test/run-tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index 8b0915340d..f0e74a4241 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -21,7 +21,7 @@ BBB_DEBIAN_SDCARD_IMAGE_URL="https://debian.beagleboard.org/images/bone-debian-1 # Not on official home page, but found via https://elinux.org/Beagleboard:BeagleBoneBlack_Debian: ## Auto-update -BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-08-16/buster-console/bone-debian-10.10-console-armhf-2021-08-16-1gb.img.xz" +BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-08-23/buster-console/bone-debian-10.10-console-armhf-2021-08-23-1gb.img.xz" ## Auto-update RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2020-02-14/2020-02-13-raspbian-buster-lite.zip" @@ -29,7 +29,7 @@ RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbi UBUNTU_IMAGE_URL="https://downloads.mender.io/mender-convert/images/Ubuntu-Bionic-x86-64.img.gz" ## Auto-update -UBUNTU_SERVER_RPI_IMAGE_URL="http://cdimage.ubuntu.com/ubuntu/releases/20.04/release/ubuntu-20.04.2-preinstalled-server-armhf+raspi.img.xz" +UBUNTU_SERVER_RPI_IMAGE_URL="http://cdimage.ubuntu.com/ubuntu/releases/20.04/release/ubuntu-20.04.3-preinstalled-server-armhf+raspi.img.xz" # Keep common function declarations in separate utils script UTILS_PATH=${0/$(basename $0)/test-utils.sh} From 83122cc22a635c76004fe63a0b758a2b3324c2d1 Mon Sep 17 00:00:00 2001 From: Mark Schulte Date: Thu, 2 Sep 2021 00:04:07 -0700 Subject: [PATCH 031/117] Adding sudo when needed --- configs/raspberrypi_config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/raspberrypi_config b/configs/raspberrypi_config index 8ec9f1ac44..3dcbca4530 100644 --- a/configs/raspberrypi_config +++ b/configs/raspberrypi_config @@ -94,9 +94,9 @@ function platform_modify() { # Raspberry Pi headless configuration expects boot partition to be mounted # at /boot, so replace these services with /uboot - run_and_log_cmd "find work/rootfs/lib/ -type f -name *.service | xargs sed -i.bak 's|/boot/|/uboot/|g'" + run_and_log_cmd "find work/rootfs/lib/ -type f -name *.service | xargs sudo sed -i.bak 's|/boot/|/uboot/|g'" log_debug "Diff from modified service files:\n$(find work/rootfs/lib/ -type f -name '*.service' | xargs -I@ bash -c 'diff -u @.bak @')" - run_and_log_cmd "find work/rootfs/lib/ -type f -name *.service.bak | xargs rm" + run_and_log_cmd "find work/rootfs/lib/ -type f -name *.service.bak | xargs sudo rm" log_info "Certain service files have been changed to align with our /uboot boot partition mount point. See convert.log for more information" run_and_log_cmd "sudo install -m 755 work/rpi/binaries/fw_printenv work/rootfs/sbin/fw_printenv" From 7a6beaab2e400f513e56800da36c37144ea7b3e0 Mon Sep 17 00:00:00 2001 From: Mark Schulte Date: Thu, 2 Sep 2021 00:17:33 -0700 Subject: [PATCH 032/117] Sudo to bash line --- configs/raspberrypi_config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/raspberrypi_config b/configs/raspberrypi_config index 3dcbca4530..7b4fb66d4e 100644 --- a/configs/raspberrypi_config +++ b/configs/raspberrypi_config @@ -111,9 +111,9 @@ function platform_modify() { # Enable UART. This is required for U-Boot to operate correctly, see # https://source.denx.de/u-boot/u-boot/-/blob/master/arch/arm/mach-bcm283x/Kconfig log_info "Enabling UART in U-Boot configuration" - cat <<- EOF >> work/boot/config.txt + sudo bash -c 'cat <<- EOF >> work/boot/config.txt # Enable UART enable_uart=1 -EOF +EOF' } From bd695873af8acbc5e644fb3780e154b196a90309 Mon Sep 17 00:00:00 2001 From: mender-test-bot Date: Mon, 6 Sep 2021 03:17:17 +0000 Subject: [PATCH 033/117] Image-Bot - Image updates Changelog: None Signed-off-by: mender-test-bot --- scripts/test/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index f0e74a4241..15c14fb452 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -21,7 +21,7 @@ BBB_DEBIAN_SDCARD_IMAGE_URL="https://debian.beagleboard.org/images/bone-debian-1 # Not on official home page, but found via https://elinux.org/Beagleboard:BeagleBoneBlack_Debian: ## Auto-update -BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-08-23/buster-console/bone-debian-10.10-console-armhf-2021-08-23-1gb.img.xz" +BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-09-01/buster-console/bone-debian-10.10-console-armhf-2021-09-01-1gb.img.xz" ## Auto-update RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2020-02-14/2020-02-13-raspbian-buster-lite.zip" From 23680167a50ed9909fa6191c17102a5dadd5aefd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Sep 2021 07:01:57 +0000 Subject: [PATCH 034/117] Changelog:All: Bump tests/mender-image-tests from `9585db2` to `49f2152` Bumps [tests/mender-image-tests](https://github.com/mendersoftware/mender-image-tests) from `9585db2` to `49f2152`. - [Release notes](https://github.com/mendersoftware/mender-image-tests/releases) - [Commits](https://github.com/mendersoftware/mender-image-tests/compare/9585db20894713083e48aae56db316761e8426fc...49f21521304d059408e4416e8361dc3d4542292d) --- updated-dependencies: - dependency-name: tests/mender-image-tests dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/mender-image-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mender-image-tests b/tests/mender-image-tests index 9585db2089..49f2152130 160000 --- a/tests/mender-image-tests +++ b/tests/mender-image-tests @@ -1 +1 @@ -Subproject commit 9585db20894713083e48aae56db316761e8426fc +Subproject commit 49f21521304d059408e4416e8361dc3d4542292d From f4650ffd1d5f169620372b6e905c5826ebee27ce Mon Sep 17 00:00:00 2001 From: Lluis Campos Date: Mon, 13 Sep 2021 15:27:49 +0200 Subject: [PATCH 035/117] MEN-5057: configs/images/raspberrypi_raspbian_config: Allow APT Suite change As per today, our pre-converted image ships with Debian buster, while Raspberry Pi OS APT repository has moved already to Debian bullseye, moving buster from "stable" to "oldstable". This configuration allows apt to update the information without interaction from the user. This is the default behaviour in newer versions of apt, see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=931566 Changelog: Title Signed-off-by: Lluis Campos --- configs/images/raspberrypi_raspbian_config | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/configs/images/raspberrypi_raspbian_config b/configs/images/raspberrypi_raspbian_config index d0b4368dbf..0738306892 100644 --- a/configs/images/raspberrypi_raspbian_config +++ b/configs/images/raspberrypi_raspbian_config @@ -49,3 +49,18 @@ EOF } PLATFORM_MODIFY_HOOKS+=(grow_data_partition) + +# +# Allow APT Suite change in the release info +# +# Upstream repository http://raspbian.raspberrypi.org/raspbian has changed buster +# release suite from "stable" to "oldstable". The following configuration option +# allows APT to update without user interaction. +# This is the default un newer versions of apt, see: +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=931566 +# +function apt_allow_release_change_suite() { + run_and_log_cmd "echo 'Acquire::AllowReleaseInfoChange::Suite \"true\";' \ + | sudo tee work/rootfs/etc/apt/apt.conf.d/99release-change-suite" +} +PLATFORM_MODIFY_HOOKS+=(apt_allow_release_change_suite) From 6580e27f9274185fac8691c32f93ed1d10011985 Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Tue, 7 Sep 2021 16:53:56 +0200 Subject: [PATCH 036/117] Make sure version overrides are used in tests as well as production. Changelog: None Signed-off-by: Kristian Amlie --- .gitlab-ci.yml | 11 ++++--- scripts/test/run-tests.sh | 61 +++++++++++++++++++++++++++++---------- 2 files changed, 52 insertions(+), 20 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c75b4774a3..0c42b003c5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -161,6 +161,9 @@ convert_raspbian_raspberrypi4: - curl -f -O https://mender.s3.amazonaws.com/mender-artifact/$MENDER_ARTIFACT_VERSION/linux/mender-artifact - chmod ugo+x mender-artifact - mv mender-artifact /usr/bin/ + - echo "MENDER_CLIENT_VERSION=${MENDER_CLIENT_VERSION}" > versions_override_config + - echo "MENDER_ADDON_CONNECT_VERSION=${MENDER_ADDON_CONNECT_VERSION}" >> versions_override_config + - echo "MENDER_ADDON_CONFIGURE_VERSION=${MENDER_ADDON_CONFIGURE_VERSION}" >> versions_override_config # Get Update Modules Artifact generators for the tests - if [ "${MENDER_CLIENT_VERSION}" = "latest" ]; then - mender_version=master @@ -220,22 +223,22 @@ test_acceptance_prebuilt_raspberrypi4: test_acceptance_qemux86_64: <<: *test_acceptance script: - - ./scripts/test/run-tests.sh --only qemux86_64 + - ./scripts/test/run-tests.sh --config versions_override_config --only qemux86_64 test_acceptance_raspberrypi: <<: *test_acceptance script: - - ./scripts/test/run-tests.sh --only raspberrypi3 + - ./scripts/test/run-tests.sh --config versions_override_config --only raspberrypi3 test_acceptance_beaglebone: <<: *test_acceptance script: - - ./scripts/test/run-tests.sh --only beaglebone + - ./scripts/test/run-tests.sh --config versions_override_config --only beaglebone test_acceptance_ubuntu: <<: *test_acceptance script: - - ./scripts/test/run-tests.sh --only ubuntu + - ./scripts/test/run-tests.sh --config versions_override_config --only ubuntu publish:s3: stage: publish diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index 15c14fb452..454ceb0aa3 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -3,7 +3,7 @@ set -e usage() { - echo "$0 <--all | --only DEVICE_TYPE | --prebuilt-image DEVICE_TYPE IMAGE_NAME>" + echo "$0 [--config EXTRA_CONFIG_FILE] <--all | --only DEVICE_TYPE | --prebuilt-image DEVICE_TYPE IMAGE_NAME>" exit 1 } @@ -44,26 +44,55 @@ get_pytest_files prepare_ssh_keys -if ! [ "$1" == "--all" -o "$1" == "--only" -a -n "$2" -o "$1" == "--prebuilt-image" -a -n "$3" ]; then - usage -fi +usage_if_empty() { + if [ -z "$1" ]; then + usage + fi +} + +PREBUILT_IMAGE= +TEST_PLATFORM= +TEST_ALL=0 +EXTRA_CONFIG= +while [ -n "$1" ]; do + case "$1" in + --prebuilt-image) + usage_if_empty "$3" + PREBUILT_IMAGE="$2 $3" + ;; + --all) + TEST_ALL=1 + ;; + --only) + usage_if_empty "$2" + TEST_PLATFORM="$2" + shift + ;; + --config) + usage_if_empty "$2" + EXTRA_CONFIG="$EXTRA_CONFIG --config $2" + shift + ;; + esac + shift +done test_result=0 -if [ "$1" == "--prebuilt-image" ]; then - run_tests "$2" "$3" \ +if [ -n "$PREBUILT_IMAGE" ]; then + run_tests $PREBUILT_IMAGE \ "-k" "'not test_update'" \ || test_result=$? exit $test_result else - if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "qemux86_64" ]; then + if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "qemux86_64" ]; then wget --progress=dot:giga -N ${UBUNTU_IMAGE_URL} -P input/ convert_and_test "qemux86_64" \ "release-1" \ "input/Ubuntu-Bionic-x86-64.img.gz" \ "--overlay tests/ssh-public-key-overlay" \ - "--config configs/qemux86-64_config" \ + "--config configs/qemux86-64_config $EXTRA_CONFIG" \ || test_result=$? echo >&2 "----------------------------------------" @@ -73,25 +102,25 @@ else gunzip --force "input/Ubuntu-Bionic-x86-64.img.gz" run_convert "release-2" \ "input/Ubuntu-Bionic-x86-64.img" \ - "--config configs/qemux86-64_config" || test_result=$? + "--config configs/qemux86-64_config $EXTRA_CONFIG" || test_result=$? ret=0 test -f deploy/Ubuntu-Bionic-x86-64-qemux86_64-mender.img || ret=$? assert "${ret}" "0" "Expected uncompressed file deploy/Ubuntu-Bionic-x86-64-qemux86_64-mender.img" fi - if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "raspberrypi3" ]; then + if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "raspberrypi3" ]; then wget --progress=dot:giga -N ${RASPBIAN_IMAGE_URL} -P input/ RASPBIAN_IMAGE="${RASPBIAN_IMAGE_URL##*/}" convert_and_test "raspberrypi3" \ "release-1" \ "input/${RASPBIAN_IMAGE}" \ - "--config configs/raspberrypi3_config" \ + "--config configs/raspberrypi3_config $EXTRA_CONFIG" \ -- \ "-k" "'not test_update'" \ || test_result=$? fi - if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "beaglebone" ]; then + if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "beaglebone" ]; then wget --progress=dot:giga -N ${BBB_DEBIAN_SDCARD_IMAGE_URL} -P input/ BBB_DEBIAN_SDCARD_IMAGE_COMPRESSED="${BBB_DEBIAN_SDCARD_IMAGE_URL##*/}" BBB_DEBIAN_SDCARD_IMAGE_UNCOMPRESSED="${BBB_DEBIAN_SDCARD_IMAGE_COMPRESSED%.xz}" @@ -100,7 +129,7 @@ else convert_and_test "beaglebone-sdcard" \ "release-1" \ "input/${BBB_DEBIAN_SDCARD_IMAGE_UNCOMPRESSED}" \ - "--config configs/beaglebone_black_debian_sdcard_config" \ + "--config configs/beaglebone_black_debian_sdcard_config $EXTRA_CONFIG" \ -- \ "-k" "'not test_update'" \ || test_result=$? @@ -113,19 +142,19 @@ else convert_and_test "beaglebone-emmc" \ "release-1" \ "input/${BBB_DEBIAN_EMMC_IMAGE_UNCOMPRESSED}" \ - "--config configs/beaglebone_black_debian_emmc_config" \ + "--config configs/beaglebone_black_debian_emmc_config $EXTRA_CONFIG" \ -- \ "-k" "'not test_update'" \ || test_result=$? fi - if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "ubuntu" ]; then + if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "ubuntu" ]; then wget --progress=dot:giga -N ${UBUNTU_SERVER_RPI_IMAGE_URL} -P input/ UBUNTU_SERVER_RPI_IMAGE_COMPRESSED="${UBUNTU_SERVER_RPI_IMAGE_URL##*/}" convert_and_test "raspberrypi3" \ "release-1" \ "input/${UBUNTU_SERVER_RPI_IMAGE_COMPRESSED}" \ - "--config configs/raspberrypi3_config" \ + "--config configs/raspberrypi3_config $EXTRA_CONFIG" \ -- \ "-k" "'not test_update'" \ || test_result=$? From ed5da386679c2995c18c522d4e2abff3614f4139 Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Wed, 8 Sep 2021 08:49:10 +0200 Subject: [PATCH 037/117] URL encode '+' characters in package names, or else we get Forbidden. Changelog: Fix broken download of packages when `*_VERSION` are set to `master`. Signed-off-by: Kristian Amlie --- modules/deb.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/deb.sh b/modules/deb.sh index e8686ca7ab..57299629fa 100644 --- a/modules/deb.sh +++ b/modules/deb.sh @@ -47,7 +47,8 @@ function deb_from_repo_dist_get() { fi local -r filename=$(basename $deb_package_path) - run_and_log_cmd "wget -Nq ${repo_url}/${deb_package_path} -P ${download_dir}" + local -r deb_package_url=$(echo ${repo_url}/${deb_package_path} | sed 's/+/%2B/g') + run_and_log_cmd "wget -Nq ${deb_package_url} -P ${download_dir}" rm -f /tmp/Packages log_info "Successfully downloaded ${filename}" From 50da54d8bf5899448c3c95d0aa646b76bbde2e88 Mon Sep 17 00:00:00 2001 From: Lluis Campos Date: Wed, 15 Sep 2021 14:49:31 +0200 Subject: [PATCH 038/117] Update testing image Ubuntu x86-64 to Focal release Changelog: None Signed-off-by: Lluis Campos --- configs/qemux86-64_config | 13 ++++++++----- scripts/test/run-tests.sh | 12 ++++++------ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/configs/qemux86-64_config b/configs/qemux86-64_config index 6b0599d527..2f21b14659 100644 --- a/configs/qemux86-64_config +++ b/configs/qemux86-64_config @@ -1,16 +1,19 @@ # This configuration can be used to run on a qemux86-64 machine. # -# This has been tested on images generated with the following command: +# The image is generated with the following command: # -# mkosi -d ubuntu -r bionic -t gpt_ext4 -b --checksum --password password --package=openssh-server,dhcpcd5 -o image.raw +# mkosi --root-size=2G --distribution=ubuntu --release=focal --format=gpt_ext4 --bootable --checksum +# --password password --package=openssh-server,dhcpcd5 --output=Ubuntu-Focal-x86-64.img build # -# Converted with the following command: +# Then manually uploaded to Mender's AWS S3 bucket for CI testing # -# MENDER_ARTIFACT_NAME=release-1 ./mender-convert --disk-image input/image.raw --overlay rootfs_overlay_demo --config configs/qemux86-64_config +# Locally, converted with the following command: +# +# MENDER_ARTIFACT_NAME=release-1 ./docker-mender-convert --disk-image input/Ubuntu-Focal-x86-64.img --overlay rootfs_overlay_demo --config configs/qemux86-64_config # # and qemu is executed with the following command: # -# qemu-system-x86_64 -enable-kvm -m 512 -smp 2 -bios /usr/share/ovmf/x64/OVMF_CODE.fd -drive format=raw,file=qemux86_64-release-1.sdimg +# qemu-system-x86_64 -enable-kvm -m 512 -smp 2 -bios /usr/share/OVMF/OVMF_CODE.fd -drive format=raw,file=deploy/Ubuntu-Focal-x86-64-qemux86_64-mender.img MENDER_STORAGE_DEVICE_BASE=/dev/sda MENDER_DEVICE_TYPE="qemux86_64" diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index 454ceb0aa3..6bf52b1f50 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -26,7 +26,7 @@ BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-09-01/b ## Auto-update RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2020-02-14/2020-02-13-raspbian-buster-lite.zip" -UBUNTU_IMAGE_URL="https://downloads.mender.io/mender-convert/images/Ubuntu-Bionic-x86-64.img.gz" +UBUNTU_IMAGE_URL="https://downloads.mender.io/mender-convert/images/Ubuntu-Focal-x86-64.img.gz" ## Auto-update UBUNTU_SERVER_RPI_IMAGE_URL="http://cdimage.ubuntu.com/ubuntu/releases/20.04/release/ubuntu-20.04.3-preinstalled-server-armhf+raspi.img.xz" @@ -90,7 +90,7 @@ else wget --progress=dot:giga -N ${UBUNTU_IMAGE_URL} -P input/ convert_and_test "qemux86_64" \ "release-1" \ - "input/Ubuntu-Bionic-x86-64.img.gz" \ + "input/Ubuntu-Focal-x86-64.img.gz" \ "--overlay tests/ssh-public-key-overlay" \ "--config configs/qemux86-64_config $EXTRA_CONFIG" \ || test_result=$? @@ -99,13 +99,13 @@ else echo >&2 "Running the uncompressed test" echo >&2 "----------------------------------------" rm -rf deploy - gunzip --force "input/Ubuntu-Bionic-x86-64.img.gz" + gunzip --force "input/Ubuntu-Focal-x86-64.img.gz" run_convert "release-2" \ - "input/Ubuntu-Bionic-x86-64.img" \ + "input/Ubuntu-Focal-x86-64.img" \ "--config configs/qemux86-64_config $EXTRA_CONFIG" || test_result=$? ret=0 - test -f deploy/Ubuntu-Bionic-x86-64-qemux86_64-mender.img || ret=$? - assert "${ret}" "0" "Expected uncompressed file deploy/Ubuntu-Bionic-x86-64-qemux86_64-mender.img" + test -f deploy/Ubuntu-Focal-x86-64-qemux86_64-mender.img || ret=$? + assert "${ret}" "0" "Expected uncompressed file deploy/Ubuntu-Focal-x86-64-qemux86_64-mender.img" fi if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "raspberrypi3" ]; then From 6332c815348194a9fd85a6c6da33049d1af9aaa9 Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Thu, 26 Aug 2021 14:46:10 +0200 Subject: [PATCH 039/117] Upgrade to grub-mender-grubenv tools with new names. Changelog: grub-mender-grubenv: User space tools, "fw_printenv" and "fw_setenv" have been renamed to "grub-mender-grubenv-print" and "grub-mender-grubenv-set", respectively. This has been done in order to avoid conflict with U-Boot's user space tools, which have the same names. Signed-off-by: Kristian Amlie --- configs/mender_grub_config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/mender_grub_config b/configs/mender_grub_config index 1c8b9565a8..69382ca11b 100644 --- a/configs/mender_grub_config +++ b/configs/mender_grub_config @@ -17,7 +17,7 @@ GRUB_VERSION=2.04 MENDER_GRUB_KERNEL_BOOT_ARGS="" # grub-mender-grubenv is the Mender integration for the GRUB bootloader -MENDER_GRUBENV_VERSION="aa7e8f8c76c6aca6dca1820aaa42dc2cbf9762a1" +MENDER_GRUBENV_VERSION="1a7db967495bbe8be53b7a69dcb42822f39d9a74" MENDER_GRUBENV_URL="${MENDER_GITHUB_ORG}/grub-mender-grubenv/archive/${MENDER_GRUBENV_VERSION}.tar.gz" # Name of the storage device containing root filesystem partitions in GRUB From b92f1a33fda78fa5e6e5bd25d1e32dc198a2cef0 Mon Sep 17 00:00:00 2001 From: Lluis Campos Date: Tue, 14 Sep 2021 10:28:42 +0200 Subject: [PATCH 040/117] pre-converted images: remove client version from Artifact name The client is not being installed anymore in the pre-converted images, so this version string baked in into the Artifact name is misleading. Changelog: RPi pre-converted images: remove Mender client version from Artifact name. Signed-off-by: Lluis Campos --- .gitlab-ci.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0c42b003c5..91c9000264 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -98,14 +98,10 @@ test:check-shell-formatting: - unzip ${RASPBIAN_NAME}.zip script: - - echo "MENDER_CLIENT_VERSION=${MENDER_CLIENT_VERSION}" > versions_override_config - - echo "MENDER_ADDON_CONNECT_VERSION=${MENDER_ADDON_CONNECT_VERSION}" >> versions_override_config - - echo "MENDER_ADDON_CONFIGURE_VERSION=${MENDER_ADDON_CONFIGURE_VERSION}" >> versions_override_config - - env MENDER_ARTIFACT_NAME=${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION} - ./docker-mender-convert -d ${RASPBIAN_NAME}.img - -c configs/${RASPBERRYPI_PLATFORM}_config - -c configs/images/raspberrypi_raspbian_config - -c versions_override_config + - env MENDER_ARTIFACT_NAME=${RASPBIAN_NAME}-mender + ./docker-mender-convert --disk-image ${RASPBIAN_NAME}.img + --config configs/${RASPBERRYPI_PLATFORM}_config + --config configs/images/raspberrypi_raspbian_config # Collect artifacts. - mv deploy ${RASPBERRYPI_PLATFORM} From 873fb52c7910535b10c1a281b5a169dd9d4d50c1 Mon Sep 17 00:00:00 2001 From: Lluis Campos Date: Tue, 14 Sep 2021 10:32:08 +0200 Subject: [PATCH 041/117] QA-322: Align pipeline with Mender release process The idea is two have two publish jobs: one manual and one automatic controlled via env variable. The manual one should cover all special use cases that we were considering before. Changelog: None Signed-off-by: Lluis Campos --- .gitlab-ci.yml | 50 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 91c9000264..56f217c376 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,28 +2,29 @@ variables: DOCKER_REPOSITORY: mendersoftware/mender-convert S3_BUCKET_NAME: mender-convert-images + # These variables are present elsewhere in the repository too. Make sure to # search for and change them too. MENDER_ARTIFACT_VERSION: master MENDER_CLIENT_VERSION: latest MENDER_ADDON_CONNECT_VERSION: latest MENDER_ADDON_CONFIGURE_VERSION: latest + # Make sure to update the link in mender-docs to the new one when changing # this. RASPBIAN_URL: https://downloads.raspberrypi.org/raspios_lite_armhf/images/raspios_lite_armhf-2021-01-12/2021-01-11-raspios-buster-armhf-lite.zip RASPBIAN_NAME: 2021-01-11-raspios-buster-armhf-lite - TEST_MENDER_CONVERT: "true" - # Which version of mender-convert to use in published image name. Normally # empty, in which case the branch or tag name will be used, but this variable # is available so that it's possible to make small build fixes on an already # tagged version. MENDER_CONVERT_PUBLISH_VERSION: "" - # Whether to publish images. Note that images from branches are always - # published, even if this is false. - PUBLISH_MENDER_CONVERT_IMAGES: "false" + # Whether to run acceptance tests + TEST_MENDER_CONVERT: "true" + # Whether to publish packages automatically - they can always be published manually + PUBLISH_MENDER_CONVERT_AUTOMATIC: "false" DEBIAN_FRONTEND: noninteractive @@ -53,6 +54,25 @@ stages: - test_acceptance - publish +test:check-commits: + only: + variables: + - '$TEST_MENDER_CONVERT == "true"' + +test:check-license: + rules: + - if: '$TEST_MENDER_CONVERT == "true"' + +test:check-shell-formatting: + rules: + - if: '$TEST_MENDER_CONVERT == "true"' + before_script: + - SHELL_SCRIPTS=$(find mender-convert* modules configs scripts + -type f + -not -name "*.md" + -not -path "scripts/linkbot/*" + -not -wholename "scripts/test/*") + build: stage: build needs: [] @@ -67,14 +87,6 @@ build: paths: - image.tar -test:check-shell-formatting: - before_script: - - SHELL_SCRIPTS=$(find mender-convert* modules configs scripts - -type f - -not -name "*.md" - -not -path "scripts/linkbot/*" - -not -wholename "scripts/test/*") - .template_convert_raspbian: &convert_raspbian stage: convert needs: @@ -236,7 +248,7 @@ test_acceptance_ubuntu: script: - ./scripts/test/run-tests.sh --config versions_override_config --only ubuntu -publish:s3: +.template:publish:s3: stage: publish image: debian:buster before_script: @@ -275,7 +287,11 @@ publish:s3: --key ${RASPBIAN_NAME}/arm/${PUBLISH_NAME} - done +publish:s3:manual: + when: manual + extends: .template:publish:s3 + +publish:s3:automatic: rules: - - if: '$CI_COMMIT_REF_NAME =~ /^[0-9]+\.[0-9]+\.[0-9]+(-build[0-9]+)?$/ && $PUBLISH_MENDER_CONVERT_IMAGES == "true"' - - if: '$CI_COMMIT_REF_NAME =~ /^(master|[0-9]+\.[0-9]+\.x)$/' - - if: '$CI_COMMIT_REF_NAME !~ /^pr_[0-9]+$/ && $MENDER_CONVERT_PUBLISH_VERSION != "" && $PUBLISH_MENDER_CONVERT_IMAGES == "true"' + - if: '$PUBLISH_MENDER_CONVERT_AUTOMATIC == "true"' + extends: .template:publish:s3 From 2046fc6e6192c642304ae777ad85e0a6353388e6 Mon Sep 17 00:00:00 2001 From: mender-test-bot Date: Mon, 20 Sep 2021 03:17:12 +0000 Subject: [PATCH 042/117] Image-Bot - Image updates Changelog: None Signed-off-by: mender-test-bot --- scripts/test/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index 6bf52b1f50..288fe2a3fa 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -21,7 +21,7 @@ BBB_DEBIAN_SDCARD_IMAGE_URL="https://debian.beagleboard.org/images/bone-debian-1 # Not on official home page, but found via https://elinux.org/Beagleboard:BeagleBoneBlack_Debian: ## Auto-update -BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-09-01/buster-console/bone-debian-10.10-console-armhf-2021-09-01-1gb.img.xz" +BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-09-17/buster-console/bone-debian-10.10-console-armhf-2021-09-17-1gb.img.xz" ## Auto-update RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2020-02-14/2020-02-13-raspbian-buster-lite.zip" From 833634bb7930b1957754f57a4f7005fc4398ccf2 Mon Sep 17 00:00:00 2001 From: Lluis Campos Date: Tue, 21 Sep 2021 15:45:33 +0200 Subject: [PATCH 043/117] MEN-5086: Create mender-monitor state directory Changelog: Create symlink from `/var/lib/mender-monitor` to `/data/mender-configure` Signed-off-by: Lluis Campos --- mender-convert-modify | 10 ++++++---- tests/mender-image-tests | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/mender-convert-modify b/mender-convert-modify index 8022fbeaba..9b6e8baca6 100755 --- a/mender-convert-modify +++ b/mender-convert-modify @@ -163,12 +163,14 @@ if [ "${MENDER_ADDON_CONFIGURE_INSTALL}" = "y" ]; then deb_extract_package "work/deb-packages/${deb_name}" "work/rootfs/" fi -# Do this unconditionally even if not installing mender-configure. The reason is -# that if the Debian package is installed later, this folder has to preexist. It -# is part of the rootfs setup, independently of the software. -log_info "Creating the Mender Configure state folder in the data partition" +# Do this unconditionally even if not installing add-ons. The reason is that if +# the Debian package is installed later, this folder has to preexist. It is part +# of the rootfs setup, independently of the software. +log_info "Creating state folder in the data partition for Mender add-ons" run_and_log_cmd "sudo mkdir -p work/rootfs/data/mender-configure" run_and_log_cmd "sudo ln -sf /data/mender-configure work/rootfs/var/lib/mender-configure" +run_and_log_cmd "sudo mkdir -p work/rootfs/data/mender-monitor" +run_and_log_cmd "sudo ln -sf /data/mender-monitor work/rootfs/var/lib/mender-monitor" if [ "${MENDER_GRUB_EFI_INTEGRATION}" == "y" ]; then # Check for known U-Boot problems in all files on the boot partition. diff --git a/tests/mender-image-tests b/tests/mender-image-tests index 49f2152130..47d645966f 160000 --- a/tests/mender-image-tests +++ b/tests/mender-image-tests @@ -1 +1 @@ -Subproject commit 49f21521304d059408e4416e8361dc3d4542292d +Subproject commit 47d645966fb0565bbc88e71125bc38e0865a9e6a From 8f2e198ce6e8d7bb7702dad9b28e3b6be1bf0a08 Mon Sep 17 00:00:00 2001 From: Andrey Basov Date: Sat, 18 Sep 2021 17:39:29 +0300 Subject: [PATCH 044/117] Closing curly brace added in bootstrap-rootfs-overlay-production-server.sh Changelog: Title Signed-off-by: Andrey Basov Signed-off-by: Kristian Amlie --- scripts/bootstrap-rootfs-overlay-production-server.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/bootstrap-rootfs-overlay-production-server.sh b/scripts/bootstrap-rootfs-overlay-production-server.sh index deb28d0a0d..a60bbb00ac 100755 --- a/scripts/bootstrap-rootfs-overlay-production-server.sh +++ b/scripts/bootstrap-rootfs-overlay-production-server.sh @@ -75,6 +75,10 @@ EOF cp -f "${server_cert}" ${output_dir}/etc/mender/server.crt fi +cat <<- EOF >> ${root_dir}/resources/mender.conf +} +EOF + sudo chown -R 0 ${output_dir} sudo chgrp -R 0 ${output_dir} From 00fb38b26915c52080393d0c7b40b0077439dcb6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 Sep 2021 14:01:08 +0000 Subject: [PATCH 045/117] Changelog:All: Bump tests/mender-image-tests from `47d6459` to `02b3578` Bumps [tests/mender-image-tests](https://github.com/mendersoftware/mender-image-tests) from `47d6459` to `02b3578`. - [Release notes](https://github.com/mendersoftware/mender-image-tests/releases) - [Commits](https://github.com/mendersoftware/mender-image-tests/compare/47d645966fb0565bbc88e71125bc38e0865a9e6a...02b357825d6f14b1088a116e83fd5e675fedb87a) --- updated-dependencies: - dependency-name: tests/mender-image-tests dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/mender-image-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mender-image-tests b/tests/mender-image-tests index 47d645966f..02b357825d 160000 --- a/tests/mender-image-tests +++ b/tests/mender-image-tests @@ -1 +1 @@ -Subproject commit 47d645966fb0565bbc88e71125bc38e0865a9e6a +Subproject commit 02b357825d6f14b1088a116e83fd5e675fedb87a From 6a245906d16894a9db6fa892774a1d30bf6fedca Mon Sep 17 00:00:00 2001 From: mender-test-bot Date: Mon, 27 Sep 2021 03:17:17 +0000 Subject: [PATCH 046/117] Image-Bot - Image updates Changelog: None Signed-off-by: mender-test-bot --- scripts/test/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index 288fe2a3fa..4e4a402c1b 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -21,7 +21,7 @@ BBB_DEBIAN_SDCARD_IMAGE_URL="https://debian.beagleboard.org/images/bone-debian-1 # Not on official home page, but found via https://elinux.org/Beagleboard:BeagleBoneBlack_Debian: ## Auto-update -BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-09-17/buster-console/bone-debian-10.10-console-armhf-2021-09-17-1gb.img.xz" +BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-09-20/buster-console/bone-debian-10.10-console-armhf-2021-09-20-1gb.img.xz" ## Auto-update RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2020-02-14/2020-02-13-raspbian-buster-lite.zip" From 6af01759177d44a91d6fec3b7b7d03cfc6cede14 Mon Sep 17 00:00:00 2001 From: mender-test-bot Date: Mon, 4 Oct 2021 03:17:39 +0000 Subject: [PATCH 047/117] Image-Bot - Image updates Changelog: None Signed-off-by: mender-test-bot --- scripts/test/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index 4e4a402c1b..50752077fb 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -21,7 +21,7 @@ BBB_DEBIAN_SDCARD_IMAGE_URL="https://debian.beagleboard.org/images/bone-debian-1 # Not on official home page, but found via https://elinux.org/Beagleboard:BeagleBoneBlack_Debian: ## Auto-update -BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-09-20/buster-console/bone-debian-10.10-console-armhf-2021-09-20-1gb.img.xz" +BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-10-01/buster-console/bone-debian-10.10-console-armhf-2021-10-01-1gb.img.xz" ## Auto-update RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2020-02-14/2020-02-13-raspbian-buster-lite.zip" From 7906c477ac274737aed36f0c63898c15db8460bd Mon Sep 17 00:00:00 2001 From: Lluis Campos Date: Wed, 29 Sep 2021 13:59:11 +0200 Subject: [PATCH 048/117] pipeline: Use mendertesting common function to save/load tmp artifacts Changelog: None Signed-off-by: Lluis Campos --- .gitlab-ci.yml | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 56f217c376..3e5cb3b328 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -101,7 +101,7 @@ build: - export AWS_ACCESS_KEY_ID=$TMP_STORAGE_AWS_ACCESS_KEY_ID - export AWS_SECRET_ACCESS_KEY=$TMP_STORAGE_AWS_SECRET_ACCESS_KEY - - apt update && apt install -yy bash wget unzip awscli docker.io + - apt update && apt install -yy bash wget unzip awscli docker.io curl - export IMAGE_NAME=$DOCKER_REPOSITORY:pr - docker load -i image.tar @@ -109,6 +109,8 @@ build: - wget -q ${RASPBIAN_URL} - unzip ${RASPBIAN_NAME}.zip + - eval "$(curl https://raw.githubusercontent.com/mendersoftware/mendertesting/master/mender-ci-common.sh)" + script: - env MENDER_ARTIFACT_NAME=${RASPBIAN_NAME}-mender ./docker-mender-convert --disk-image ${RASPBIAN_NAME}.img @@ -118,11 +120,8 @@ build: # Collect artifacts. - mv deploy ${RASPBERRYPI_PLATFORM} - tar czf ${RASPBERRYPI_PLATFORM}.tar.gz ${RASPBERRYPI_PLATFORM} - # Compute checksum - - mkdir checksums - - sha256sum ${RASPBERRYPI_PLATFORM}.tar.gz > checksums/${RASPBERRYPI_PLATFORM}.tar.gz.sha256 # Upload to temporary S3 bucket - - aws s3 cp ${RASPBERRYPI_PLATFORM}.tar.gz s3://mender-gitlab-tmp-storage/$CI_PROJECT_NAME/$CI_PIPELINE_ID/${RASPBERRYPI_PLATFORM}.tar.gz + - mender_ci_save_tmp_artifact ${RASPBERRYPI_PLATFORM}.tar.gz artifacts: paths: @@ -198,10 +197,9 @@ convert_raspbian_raspberrypi4: .template_test_acceptance_prebuilt_raspberrypi: &test_acceptance_prebuilt_raspberrypi <<: *test_acceptance script: + - eval "$(curl https://raw.githubusercontent.com/mendersoftware/mendertesting/master/mender-ci-common.sh)" # Fetch artifacts from temporary S3 bucket - - aws s3 cp s3://mender-gitlab-tmp-storage/$CI_PROJECT_NAME/$CI_PIPELINE_ID/${RASPBERRYPI_PLATFORM}.tar.gz ${RASPBERRYPI_PLATFORM}.tar.gz - # Check checksum - - sha256sum -c checksums/${RASPBERRYPI_PLATFORM}.tar.gz.sha256 + - mender_ci_load_tmp_artifact ${RASPBERRYPI_PLATFORM}.tar.gz - tar xzf ${RASPBERRYPI_PLATFORM}.tar.gz - mv ${RASPBERRYPI_PLATFORM} deploy # Extract converted Raspbian artifacts @@ -256,12 +254,11 @@ test_acceptance_ubuntu: - export PUBLISH_AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY - export AWS_ACCESS_KEY_ID=$TMP_STORAGE_AWS_ACCESS_KEY_ID - export AWS_SECRET_ACCESS_KEY=$TMP_STORAGE_AWS_SECRET_ACCESS_KEY - - apt update && apt install -yyq awscli + - apt update && apt install -yyq awscli curl + - eval "$(curl https://raw.githubusercontent.com/mendersoftware/mendertesting/master/mender-ci-common.sh)" # Fetch artifacts from temporary S3 bucket - for RASPBERRYPI_PLATFORM in raspberrypi3 raspberrypi4; do - - aws s3 cp s3://mender-gitlab-tmp-storage/$CI_PROJECT_NAME/$CI_PIPELINE_ID/${RASPBERRYPI_PLATFORM}.tar.gz ${RASPBERRYPI_PLATFORM}.tar.gz - # Check checksum - - sha256sum -c checksums/${RASPBERRYPI_PLATFORM}.tar.gz.sha256 + - mender_ci_load_tmp_artifact ${RASPBERRYPI_PLATFORM}.tar.gz - tar xzf ${RASPBERRYPI_PLATFORM}.tar.gz - done script: From 5490dc8faa25a243231bc27836f86ab2de22000e Mon Sep 17 00:00:00 2001 From: Lluis Campos Date: Tue, 5 Oct 2021 16:41:03 +0200 Subject: [PATCH 049/117] pipeline: Clean-up AWS keys manipulation for temp bucket This is now handled by the code in mendertesting, see: https://github.com/mendersoftware/mendertesting/pull/160 Changelog: None Signed-off-by: Lluis Campos --- .gitlab-ci.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3e5cb3b328..a20ac3e8f9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -98,9 +98,6 @@ build: tags: - mender-qa-slave before_script: - - export AWS_ACCESS_KEY_ID=$TMP_STORAGE_AWS_ACCESS_KEY_ID - - export AWS_SECRET_ACCESS_KEY=$TMP_STORAGE_AWS_SECRET_ACCESS_KEY - - apt update && apt install -yy bash wget unzip awscli docker.io curl - export IMAGE_NAME=$DOCKER_REPOSITORY:pr @@ -149,8 +146,6 @@ convert_raspbian_raspberrypi4: - mender-qa-slave timeout: 2h before_script: - - export AWS_ACCESS_KEY_ID=$TMP_STORAGE_AWS_ACCESS_KEY_ID - - export AWS_SECRET_ACCESS_KEY=$TMP_STORAGE_AWS_SECRET_ACCESS_KEY # Install dependencies - apt update - apt install -qyy bash wget git util-linux mtools python3 python3-pip @@ -250,10 +245,6 @@ test_acceptance_ubuntu: stage: publish image: debian:buster before_script: - - export PUBLISH_AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID - - export PUBLISH_AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY - - export AWS_ACCESS_KEY_ID=$TMP_STORAGE_AWS_ACCESS_KEY_ID - - export AWS_SECRET_ACCESS_KEY=$TMP_STORAGE_AWS_SECRET_ACCESS_KEY - apt update && apt install -yyq awscli curl - eval "$(curl https://raw.githubusercontent.com/mendersoftware/mendertesting/master/mender-ci-common.sh)" # Fetch artifacts from temporary S3 bucket @@ -262,10 +253,6 @@ test_acceptance_ubuntu: - tar xzf ${RASPBERRYPI_PLATFORM}.tar.gz - done script: - # Prepare high privilege S3 keys (the TMP_STORAGE keys are for the tmp storage only) - - export AWS_ACCESS_KEY_ID=$PUBLISH_AWS_ACCESS_KEY_ID - - export AWS_SECRET_ACCESS_KEY=$PUBLISH_AWS_SECRET_ACCESS_KEY - - IMAGE_VERSION=${MENDER_CONVERT_PUBLISH_VERSION:-${CI_COMMIT_REF_NAME}} - for RASPBERRYPI_PLATFORM in raspberrypi3 raspberrypi4; do From 3a031bffbbeb345804b6cf5b11914dd43f51ea97 Mon Sep 17 00:00:00 2001 From: Lluis Campos Date: Wed, 6 Oct 2021 09:21:37 +0200 Subject: [PATCH 050/117] Modify `ln` calls for data store links to support successive runs Fixes MEN-5078 According to `ln` man page, the correct way to create a link on a directory is with `ln [OPTION]... TARGET... DIRECTORY`. Create first `DIRECTORY`, and then use the above form for the link. This way a successive run of `ln ...` will not end up creating a `.../mender/mender` kind of link. Changelog: Title Signed-off-by: Lluis Campos --- mender-convert-modify | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mender-convert-modify b/mender-convert-modify index 9b6e8baca6..2d876cecd8 100755 --- a/mender-convert-modify +++ b/mender-convert-modify @@ -167,10 +167,11 @@ fi # the Debian package is installed later, this folder has to preexist. It is part # of the rootfs setup, independently of the software. log_info "Creating state folder in the data partition for Mender add-ons" +run_and_log_cmd "sudo mkdir -p work/rootfs/var/lib" run_and_log_cmd "sudo mkdir -p work/rootfs/data/mender-configure" -run_and_log_cmd "sudo ln -sf /data/mender-configure work/rootfs/var/lib/mender-configure" +run_and_log_cmd "sudo ln -sf /data/mender-configure work/rootfs/var/lib" run_and_log_cmd "sudo mkdir -p work/rootfs/data/mender-monitor" -run_and_log_cmd "sudo ln -sf /data/mender-monitor work/rootfs/var/lib/mender-monitor" +run_and_log_cmd "sudo ln -sf /data/mender-monitor work/rootfs/var/lib" if [ "${MENDER_GRUB_EFI_INTEGRATION}" == "y" ]; then # Check for known U-Boot problems in all files on the boot partition. @@ -277,7 +278,8 @@ if [ -d work/rootfs/boot/dtbs ]; then fi run_and_log_cmd "sudo mkdir -p work/rootfs/data/mender" -run_and_log_cmd "sudo ln -sf /data/mender work/rootfs/var/lib/mender" +run_and_log_cmd "sudo mkdir -p work/rootfs/var/lib" +run_and_log_cmd "sudo ln -sf /data/mender work/rootfs/var/lib" log_info "Using root device A in mender.conf: $root_part_a_device" log_info "Using root device B in mender.conf: $root_part_b_device" From 9cbaa2af3b03dea51de1a5842be08b9b0ac2d6dd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 07:05:04 +0000 Subject: [PATCH 051/117] Changelog:All: Bump jsdom from 17.0.0 to 18.0.0 in /scripts/linkbot Bumps [jsdom](https://github.com/jsdom/jsdom) from 17.0.0 to 18.0.0. - [Release notes](https://github.com/jsdom/jsdom/releases) - [Changelog](https://github.com/jsdom/jsdom/blob/master/Changelog.md) - [Commits](https://github.com/jsdom/jsdom/compare/17.0.0...18.0.0) --- updated-dependencies: - dependency-name: jsdom dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- scripts/linkbot/package-lock.json | 318 +++++++++++++++--------------- scripts/linkbot/package.json | 2 +- 2 files changed, 157 insertions(+), 163 deletions(-) diff --git a/scripts/linkbot/package-lock.json b/scripts/linkbot/package-lock.json index 6f52a220bf..c95b132221 100644 --- a/scripts/linkbot/package-lock.json +++ b/scripts/linkbot/package-lock.json @@ -6,15 +6,15 @@ "": { "dependencies": { "bent": "^7.3.12", - "jsdom": "^17.0.0" + "jsdom": "^18.0.0" } }, "node_modules/@tootallnate/once": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", - "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", + "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", "engines": { - "node": ">= 6" + "node": ">= 10" } }, "node_modules/abab": { @@ -124,13 +124,13 @@ "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==" }, "node_modules/data-urls": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.0.tgz", - "integrity": "sha512-4AefxbTTdFtxDUdh0BuMBs2qJVL25Mow2zlcuuePegQwgD6GEmQao42LLEeksOui8nL4RcNEugIpFP7eRd33xg==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.1.tgz", + "integrity": "sha512-Ds554NeT5Gennfoo9KN50Vh6tpgtvYEwraYjejXnyTpu1C7oXKxdFk75REooENHE8ndTVOJuv+BEs4/J/xcozw==", "dependencies": { "abab": "^2.0.3", - "whatwg-mimetype": "^2.3.0", - "whatwg-url": "^9.0.0" + "whatwg-mimetype": "^3.0.0", + "whatwg-url": "^10.0.0" }, "engines": { "node": ">=12" @@ -171,22 +171,14 @@ } }, "node_modules/domexception": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/domexception/-/domexception-2.0.1.tgz", - "integrity": "sha512-yxJ2mFy/sibVQlu5qHjOkf9J3K6zgmCxgJ94u2EdvDOV09H+32LtRswEcUsmUWN72pVLOEnTSRaIVVzVQgS0dg==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/domexception/-/domexception-4.0.0.tgz", + "integrity": "sha512-A2is4PLG+eeSfoTMA95/s4pvAoSo2mKtiM5jlHkAVewmiO8ISFTFKZjH7UAM1Atli/OT/7JHOrJRJiMKUZKYBw==", "dependencies": { - "webidl-conversions": "^5.0.0" + "webidl-conversions": "^7.0.0" }, "engines": { - "node": ">=8" - } - }, - "node_modules/domexception/node_modules/webidl-conversions": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-5.0.0.tgz", - "integrity": "sha512-VlZwKPCkYKxQgeSbH5EyngOmRp7Ww7I9rQLERETtf5ofd9pGeswWiOtogpEO850jziPRarreGxn5QIiTqpb2wA==", - "engines": { - "node": ">=8" + "node": ">=12" } }, "node_modules/escodegen": { @@ -257,22 +249,22 @@ } }, "node_modules/html-encoding-sniffer": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz", - "integrity": "sha512-D5JbOMBIR/TVZkubHT+OyT2705QvogUW4IBn6nHd756OwieSF9aDYFj4dv6HHEVGYbHaLETa3WggZYWWMyy3ZQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz", + "integrity": "sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==", "dependencies": { - "whatwg-encoding": "^1.0.5" + "whatwg-encoding": "^2.0.0" }, "engines": { - "node": ">=10" + "node": ">=12" } }, "node_modules/http-proxy-agent": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", - "integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", + "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", "dependencies": { - "@tootallnate/once": "1", + "@tootallnate/once": "2", "agent-base": "6", "debug": "4" }, @@ -293,11 +285,11 @@ } }, "node_modules/iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", "dependencies": { - "safer-buffer": ">= 2.1.2 < 3" + "safer-buffer": ">= 2.1.2 < 3.0.0" }, "engines": { "node": ">=0.10.0" @@ -317,22 +309,22 @@ } }, "node_modules/jsdom": { - "version": "17.0.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-17.0.0.tgz", - "integrity": "sha512-MUq4XdqwtNurZDVeKScENMPHnkgmdIvMzZ1r1NSwHkDuaqI6BouPjr+17COo4/19oLNnmdpFDPOHVpgIZmZ+VA==", + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-18.0.0.tgz", + "integrity": "sha512-HVLuBcFmwdWulStv5U+J59b1AyzXhM92KXlM8HQ3ecYtRM2OQEUCPMa4oNuDeCBmtRcC7tJvb0Xz5OeFXMOKTA==", "dependencies": { "abab": "^2.0.5", - "acorn": "^8.4.1", + "acorn": "^8.5.0", "acorn-globals": "^6.0.0", "cssom": "^0.5.0", "cssstyle": "^2.3.0", - "data-urls": "^3.0.0", + "data-urls": "^3.0.1", "decimal.js": "^10.3.1", - "domexception": "^2.0.1", + "domexception": "^4.0.0", "escodegen": "^2.0.0", "form-data": "^4.0.0", - "html-encoding-sniffer": "^2.0.1", - "http-proxy-agent": "^4.0.1", + "html-encoding-sniffer": "^3.0.0", + "http-proxy-agent": "^5.0.0", "https-proxy-agent": "^5.0.0", "is-potential-custom-element-name": "^1.0.1", "nwsapi": "^2.2.0", @@ -341,13 +333,13 @@ "symbol-tree": "^3.2.4", "tough-cookie": "^4.0.0", "w3c-hr-time": "^1.0.2", - "w3c-xmlserializer": "^2.0.0", - "webidl-conversions": "^6.1.0", - "whatwg-encoding": "^1.0.5", - "whatwg-mimetype": "^2.3.0", - "whatwg-url": "^9.0.0", - "ws": "^8.0.0", - "xml-name-validator": "^3.0.0" + "w3c-xmlserializer": "^3.0.0", + "webidl-conversions": "^7.0.0", + "whatwg-encoding": "^2.0.0", + "whatwg-mimetype": "^3.0.0", + "whatwg-url": "^10.0.0", + "ws": "^8.2.3", + "xml-name-validator": "^4.0.0" }, "engines": { "node": ">=12" @@ -362,9 +354,9 @@ } }, "node_modules/jsdom/node_modules/acorn": { - "version": "8.4.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.4.1.tgz", - "integrity": "sha512-asabaBSkEKosYKMITunzX177CXxQ4Q8BSSzMTKD+FefUhipQC70gfW5SiUDhYQ3vk8G+81HqQk7Fv9OXwwn9KA==", + "version": "8.5.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.5.0.tgz", + "integrity": "sha512-yXbYeFy+jUuYd3/CDcg2NkIYE991XYX/bje7LmjJigUciaeO1JR4XxXgCIV1/Zc/dRuFEyw1L0pbA+qynJkW5Q==", "bin": { "acorn": "bin/acorn" }, @@ -499,14 +491,14 @@ } }, "node_modules/tr46": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-2.1.0.tgz", - "integrity": "sha512-15Ih7phfcdP5YxqiB+iDtLoaTz4Nd35+IiAv0kQ5FNKHzXgdWqPoTIqEDDJmXceQt4JZk6lVPT8lnDlPpGDppw==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", + "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==", "dependencies": { "punycode": "^2.1.1" }, "engines": { - "node": ">=8" + "node": ">=12" } }, "node_modules/type-check": { @@ -537,44 +529,50 @@ } }, "node_modules/w3c-xmlserializer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-2.0.0.tgz", - "integrity": "sha512-4tzD0mF8iSiMiNs30BiLO3EpfGLZUT2MSX/G+o7ZywDzliWQ3OPtTZ0PTC3B3ca1UAf4cJMHB+2Bf56EriJuRA==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-3.0.0.tgz", + "integrity": "sha512-3WFqGEgSXIyGhOmAFtlicJNMjEps8b1MG31NCA0/vOF9+nKMUW1ckhi9cnNHmf88Rzw5V+dwIwsm2C7X8k9aQg==", "dependencies": { - "xml-name-validator": "^3.0.0" + "xml-name-validator": "^4.0.0" }, "engines": { - "node": ">=10" + "node": ">=12" } }, "node_modules/webidl-conversions": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-6.1.0.tgz", - "integrity": "sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", "engines": { - "node": ">=10.4" + "node": ">=12" } }, "node_modules/whatwg-encoding": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz", - "integrity": "sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz", + "integrity": "sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==", "dependencies": { - "iconv-lite": "0.4.24" + "iconv-lite": "0.6.3" + }, + "engines": { + "node": ">=12" } }, "node_modules/whatwg-mimetype": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz", - "integrity": "sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==" + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz", + "integrity": "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==", + "engines": { + "node": ">=12" + } }, "node_modules/whatwg-url": { - "version": "9.1.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-9.1.0.tgz", - "integrity": "sha512-CQ0UcrPHyomtlOCot1TL77WyMIm/bCwrJ2D6AOKGwEczU9EpyoqAokfqrf/MioU9kHcMsmJZcg1egXix2KYEsA==", + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-10.0.0.tgz", + "integrity": "sha512-CLxxCmdUby142H5FZzn4D8ikO1cmypvXVQktsgosNy4a4BHrDHeciBBGZhb0bNoR5/MltoCatso+vFjjGx8t0w==", "dependencies": { - "tr46": "^2.1.0", - "webidl-conversions": "^6.1.0" + "tr46": "^3.0.0", + "webidl-conversions": "^7.0.0" }, "engines": { "node": ">=12" @@ -589,9 +587,9 @@ } }, "node_modules/ws": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.1.0.tgz", - "integrity": "sha512-0UWlCD2s3RSclw8FN+D0zDTUyMO+1kHwJQQJzkgUh16S8d3NYON0AKCEQPffE0ez4JyRFu76QDA9KR5bOG/7jw==", + "version": "8.2.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.2.3.tgz", + "integrity": "sha512-wBuoj1BDpC6ZQ1B7DWQBYVLphPWkm8i9Y0/3YdHjHKHiohOJ1ws+3OccDWtH+PoC9DZD5WOTrJvNbWvjS6JWaA==", "engines": { "node": ">=10.0.0" }, @@ -609,9 +607,12 @@ } }, "node_modules/xml-name-validator": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz", - "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==" + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-4.0.0.tgz", + "integrity": "sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw==", + "engines": { + "node": ">=12" + } }, "node_modules/xmlchars": { "version": "2.2.0", @@ -621,9 +622,9 @@ }, "dependencies": { "@tootallnate/once": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", - "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", + "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==" }, "abab": { "version": "2.0.5", @@ -716,13 +717,13 @@ } }, "data-urls": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.0.tgz", - "integrity": "sha512-4AefxbTTdFtxDUdh0BuMBs2qJVL25Mow2zlcuuePegQwgD6GEmQao42LLEeksOui8nL4RcNEugIpFP7eRd33xg==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.1.tgz", + "integrity": "sha512-Ds554NeT5Gennfoo9KN50Vh6tpgtvYEwraYjejXnyTpu1C7oXKxdFk75REooENHE8ndTVOJuv+BEs4/J/xcozw==", "requires": { "abab": "^2.0.3", - "whatwg-mimetype": "^2.3.0", - "whatwg-url": "^9.0.0" + "whatwg-mimetype": "^3.0.0", + "whatwg-url": "^10.0.0" } }, "debug": { @@ -749,18 +750,11 @@ "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" }, "domexception": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/domexception/-/domexception-2.0.1.tgz", - "integrity": "sha512-yxJ2mFy/sibVQlu5qHjOkf9J3K6zgmCxgJ94u2EdvDOV09H+32LtRswEcUsmUWN72pVLOEnTSRaIVVzVQgS0dg==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/domexception/-/domexception-4.0.0.tgz", + "integrity": "sha512-A2is4PLG+eeSfoTMA95/s4pvAoSo2mKtiM5jlHkAVewmiO8ISFTFKZjH7UAM1Atli/OT/7JHOrJRJiMKUZKYBw==", "requires": { - "webidl-conversions": "^5.0.0" - }, - "dependencies": { - "webidl-conversions": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-5.0.0.tgz", - "integrity": "sha512-VlZwKPCkYKxQgeSbH5EyngOmRp7Ww7I9rQLERETtf5ofd9pGeswWiOtogpEO850jziPRarreGxn5QIiTqpb2wA==" - } + "webidl-conversions": "^7.0.0" } }, "escodegen": { @@ -806,19 +800,19 @@ } }, "html-encoding-sniffer": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz", - "integrity": "sha512-D5JbOMBIR/TVZkubHT+OyT2705QvogUW4IBn6nHd756OwieSF9aDYFj4dv6HHEVGYbHaLETa3WggZYWWMyy3ZQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz", + "integrity": "sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==", "requires": { - "whatwg-encoding": "^1.0.5" + "whatwg-encoding": "^2.0.0" } }, "http-proxy-agent": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", - "integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", + "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", "requires": { - "@tootallnate/once": "1", + "@tootallnate/once": "2", "agent-base": "6", "debug": "4" } @@ -833,11 +827,11 @@ } }, "iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", "requires": { - "safer-buffer": ">= 2.1.2 < 3" + "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "is-potential-custom-element-name": { @@ -851,22 +845,22 @@ "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==" }, "jsdom": { - "version": "17.0.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-17.0.0.tgz", - "integrity": "sha512-MUq4XdqwtNurZDVeKScENMPHnkgmdIvMzZ1r1NSwHkDuaqI6BouPjr+17COo4/19oLNnmdpFDPOHVpgIZmZ+VA==", + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-18.0.0.tgz", + "integrity": "sha512-HVLuBcFmwdWulStv5U+J59b1AyzXhM92KXlM8HQ3ecYtRM2OQEUCPMa4oNuDeCBmtRcC7tJvb0Xz5OeFXMOKTA==", "requires": { "abab": "^2.0.5", - "acorn": "^8.4.1", + "acorn": "^8.5.0", "acorn-globals": "^6.0.0", "cssom": "^0.5.0", "cssstyle": "^2.3.0", - "data-urls": "^3.0.0", + "data-urls": "^3.0.1", "decimal.js": "^10.3.1", - "domexception": "^2.0.1", + "domexception": "^4.0.0", "escodegen": "^2.0.0", "form-data": "^4.0.0", - "html-encoding-sniffer": "^2.0.1", - "http-proxy-agent": "^4.0.1", + "html-encoding-sniffer": "^3.0.0", + "http-proxy-agent": "^5.0.0", "https-proxy-agent": "^5.0.0", "is-potential-custom-element-name": "^1.0.1", "nwsapi": "^2.2.0", @@ -875,19 +869,19 @@ "symbol-tree": "^3.2.4", "tough-cookie": "^4.0.0", "w3c-hr-time": "^1.0.2", - "w3c-xmlserializer": "^2.0.0", - "webidl-conversions": "^6.1.0", - "whatwg-encoding": "^1.0.5", - "whatwg-mimetype": "^2.3.0", - "whatwg-url": "^9.0.0", - "ws": "^8.0.0", - "xml-name-validator": "^3.0.0" + "w3c-xmlserializer": "^3.0.0", + "webidl-conversions": "^7.0.0", + "whatwg-encoding": "^2.0.0", + "whatwg-mimetype": "^3.0.0", + "whatwg-url": "^10.0.0", + "ws": "^8.2.3", + "xml-name-validator": "^4.0.0" }, "dependencies": { "acorn": { - "version": "8.4.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.4.1.tgz", - "integrity": "sha512-asabaBSkEKosYKMITunzX177CXxQ4Q8BSSzMTKD+FefUhipQC70gfW5SiUDhYQ3vk8G+81HqQk7Fv9OXwwn9KA==" + "version": "8.5.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.5.0.tgz", + "integrity": "sha512-yXbYeFy+jUuYd3/CDcg2NkIYE991XYX/bje7LmjJigUciaeO1JR4XxXgCIV1/Zc/dRuFEyw1L0pbA+qynJkW5Q==" } } }, @@ -991,9 +985,9 @@ } }, "tr46": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-2.1.0.tgz", - "integrity": "sha512-15Ih7phfcdP5YxqiB+iDtLoaTz4Nd35+IiAv0kQ5FNKHzXgdWqPoTIqEDDJmXceQt4JZk6lVPT8lnDlPpGDppw==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", + "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==", "requires": { "punycode": "^2.1.1" } @@ -1020,38 +1014,38 @@ } }, "w3c-xmlserializer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-2.0.0.tgz", - "integrity": "sha512-4tzD0mF8iSiMiNs30BiLO3EpfGLZUT2MSX/G+o7ZywDzliWQ3OPtTZ0PTC3B3ca1UAf4cJMHB+2Bf56EriJuRA==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-3.0.0.tgz", + "integrity": "sha512-3WFqGEgSXIyGhOmAFtlicJNMjEps8b1MG31NCA0/vOF9+nKMUW1ckhi9cnNHmf88Rzw5V+dwIwsm2C7X8k9aQg==", "requires": { - "xml-name-validator": "^3.0.0" + "xml-name-validator": "^4.0.0" } }, "webidl-conversions": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-6.1.0.tgz", - "integrity": "sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w==" + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==" }, "whatwg-encoding": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz", - "integrity": "sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz", + "integrity": "sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==", "requires": { - "iconv-lite": "0.4.24" + "iconv-lite": "0.6.3" } }, "whatwg-mimetype": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz", - "integrity": "sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==" + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz", + "integrity": "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==" }, "whatwg-url": { - "version": "9.1.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-9.1.0.tgz", - "integrity": "sha512-CQ0UcrPHyomtlOCot1TL77WyMIm/bCwrJ2D6AOKGwEczU9EpyoqAokfqrf/MioU9kHcMsmJZcg1egXix2KYEsA==", + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-10.0.0.tgz", + "integrity": "sha512-CLxxCmdUby142H5FZzn4D8ikO1cmypvXVQktsgosNy4a4BHrDHeciBBGZhb0bNoR5/MltoCatso+vFjjGx8t0w==", "requires": { - "tr46": "^2.1.0", - "webidl-conversions": "^6.1.0" + "tr46": "^3.0.0", + "webidl-conversions": "^7.0.0" } }, "word-wrap": { @@ -1060,15 +1054,15 @@ "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==" }, "ws": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.1.0.tgz", - "integrity": "sha512-0UWlCD2s3RSclw8FN+D0zDTUyMO+1kHwJQQJzkgUh16S8d3NYON0AKCEQPffE0ez4JyRFu76QDA9KR5bOG/7jw==", + "version": "8.2.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.2.3.tgz", + "integrity": "sha512-wBuoj1BDpC6ZQ1B7DWQBYVLphPWkm8i9Y0/3YdHjHKHiohOJ1ws+3OccDWtH+PoC9DZD5WOTrJvNbWvjS6JWaA==", "requires": {} }, "xml-name-validator": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz", - "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==" + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-4.0.0.tgz", + "integrity": "sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw==" }, "xmlchars": { "version": "2.2.0", diff --git a/scripts/linkbot/package.json b/scripts/linkbot/package.json index 145a096192..9fb6638a2a 100644 --- a/scripts/linkbot/package.json +++ b/scripts/linkbot/package.json @@ -1,6 +1,6 @@ { "dependencies": { "bent": "^7.3.12", - "jsdom": "^17.0.0" + "jsdom": "^18.0.0" } } From e48602f1cd9e714c577bc19e080778d17325f65c Mon Sep 17 00:00:00 2001 From: Ole Petter Date: Wed, 27 Oct 2021 08:28:10 +0200 Subject: [PATCH 052/117] style: Fix lint errors Changelog: None Signed-off-by: Ole Petter --- scripts/linkbot/common.js | 16 ++++++------- scripts/linkbot/monitor-rpi.js | 44 +++++++++++++++++----------------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/scripts/linkbot/common.js b/scripts/linkbot/common.js index 402024ab8e..6603031f50 100644 --- a/scripts/linkbot/common.js +++ b/scripts/linkbot/common.js @@ -1,18 +1,18 @@ -const fs = require('fs') +const fs = require('fs'); const updateURLLink = (newLine, target) => { try { const data = fs.readFileSync('../test/run-tests.sh', 'utf8') - .replace(RegExp(`## Auto-update\n${target}=.*`), `## Auto-update\n${newLine}`) + .replace(RegExp(`## Auto-update\n${target}=.*`), `## Auto-update\n${newLine}`); fs.writeFile('../test/run-tests.sh', data, (err, data) => { if (err) { - console.error(err) + console.error(err); } - }) + }); } catch (err) { - console.error(err) - process.exit(1) + console.error(err); + process.exit(1); } -} +}; module.exports = { updateURLLink -} +}; diff --git a/scripts/linkbot/monitor-rpi.js b/scripts/linkbot/monitor-rpi.js index 3deef387b6..c9760973d8 100644 --- a/scripts/linkbot/monitor-rpi.js +++ b/scripts/linkbot/monitor-rpi.js @@ -1,40 +1,40 @@ const jsdom = require("jsdom"); const { JSDOM } = jsdom; -const fs = require('fs') +const fs = require('fs'); const { updateURLLink } = require('./common'); -const target = "RASPBIAN_IMAGE_URL" +const target = "RASPBIAN_IMAGE_URL"; // Read the input file, and parse the variable input try { const data = fs.readFileSync('../test/run-tests.sh', 'utf8') .split('\n') - .filter(line => line.match(`${target}=.*`)) - var line = data[0] - console.log(line) - var reg = "raspbian_lite-(?[0-9]{4}-[0-9]{2}-[0-9]{1,2})/(?[0-9]{4}-[0-9]{2}-[0-9]{1,2}).*$" - var m = line.match(".*=\"(?[a-zA-Z-://\._]*)(?raspbian_lite-[0-9]{4}-[0-9]{2}-[0-9]{1,2})/(?[0-9]{4}-[0-9]{2}-[0-9]{1,2}).*$") - console.log(m) - var url = m.groups.url - var imageName = m.groups.imageName - var updated = m.groups.updated + .filter(line => line.match(`${target}=.*`)); + var line = data[0]; + console.log(line); + var reg = "raspbian_lite-(?[0-9]{4}-[0-9]{2}-[0-9]{1,2})/(?[0-9]{4}-[0-9]{2}-[0-9]{1,2}).*$"; + var m = line.match(".*=\"(?[a-zA-Z-://\._]*)(?raspbian_lite-[0-9]{4}-[0-9]{2}-[0-9]{1,2})/(?[0-9]{4}-[0-9]{2}-[0-9]{1,2}).*$"); + console.log(m); + var url = m.groups.url; + var imageName = m.groups.imageName; + var updated = m.groups.updated; } catch (err) { - console.error(err) - process.exit(1) + console.error(err); + process.exit(1); } function getNewRaspbian(url) { - console.log("getNewRaspbian") - console.log(`${url}`) + console.log("getNewRaspbian"); + console.log(`${url}`); return JSDOM.fromURL(url, {}).then(dom => { var document = dom.window.document; var refs = document.getElementsByTagName("a"); var test = Array.from(refs) .filter(ref => ref.textContent.match(("raspbian-.*-lite.*\.zip$"))) .reduce((acc, element) => { - acc.push(element.textContent.match("raspbian-.*-lite.*\.zip$").input) - return acc - }, [])[0] + acc.push(element.textContent.match("raspbian-.*-lite.*\.zip$").input); + return acc; + }, [])[0]; return `${target}=\"${url}/${test}\"`; }); } @@ -63,15 +63,15 @@ JSDOM.fromURL(url, {}).then(dom => { } return bl - al; }); - var matchOn = matches[0].input.split("/")[0] + var matchOn = matches[0].input.split("/")[0]; if (matchOn !== imageName) { // We also need to extract the new image name from the index folder, as // dated folders contain images with different dates in them o_O console.error("We've got a new release! \\o/"); var newVar = getNewRaspbian(`${url}${matches[0].input.split(" ")[0].split("/").slice(0, -1)}`) .then(res => { - console.log(`New release: ${res}`) - updateURLLink(res, target) - }) + console.log(`New release: ${res}`); + updateURLLink(res, target); + }); } }); From 1472635f5f6dc6d40d6008762bca30025ab13d43 Mon Sep 17 00:00:00 2001 From: Ole Petter Date: Wed, 27 Oct 2021 09:15:46 +0200 Subject: [PATCH 053/117] QA-334: Make the Raspbian URL auto-updateable Changelog: None Signed-off-by: Ole Petter --- .../automatically-update-test-links.yml | 1 + .gitlab-ci.yml | 3 +- scripts/linkbot/monitor-raspbian-os.js | 114 ++++++++++++++++++ 3 files changed, 116 insertions(+), 2 deletions(-) create mode 100644 scripts/linkbot/monitor-raspbian-os.js diff --git a/.github/workflows/automatically-update-test-links.yml b/.github/workflows/automatically-update-test-links.yml index d30630b560..81ece20967 100644 --- a/.github/workflows/automatically-update-test-links.yml +++ b/.github/workflows/automatically-update-test-links.yml @@ -17,6 +17,7 @@ jobs: node monitor-bb-sd.js node monitor-rpi.js node monitor-ub-server.js + node monitor-raspbian-os.js working-directory: scripts/linkbot - name: Create Pull Request diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a20ac3e8f9..a45c12cabd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,8 +10,7 @@ variables: MENDER_ADDON_CONNECT_VERSION: latest MENDER_ADDON_CONFIGURE_VERSION: latest - # Make sure to update the link in mender-docs to the new one when changing - # this. + ## Auto-update RASPBIAN_URL: https://downloads.raspberrypi.org/raspios_lite_armhf/images/raspios_lite_armhf-2021-01-12/2021-01-11-raspios-buster-armhf-lite.zip RASPBIAN_NAME: 2021-01-11-raspios-buster-armhf-lite diff --git a/scripts/linkbot/monitor-raspbian-os.js b/scripts/linkbot/monitor-raspbian-os.js new file mode 100644 index 0000000000..2655533665 --- /dev/null +++ b/scripts/linkbot/monitor-raspbian-os.js @@ -0,0 +1,114 @@ +const jsdom = require("jsdom"); +const { JSDOM } = jsdom; +const fs = require("fs"); +const { updateURLLink } = require("./common"); + +const target = "RASPBIAN_URL"; + +// Read the input file, and parse the variable input +try { + const data = fs + .readFileSync("../../.gitlab-ci.yml", "utf8") + .split("\n") + .filter((line) => line.match(`${target}: .*`)); + var line = data[0]; + console.log(line); + var reg = + "raspios_lite_armhf-(?[0-9]{4}-[0-9]{2}-[0-9]{1,2})/(?[0-9]{4}-[0-9]{2}-[0-9]{1,2}).*$"; + var m = line.match( + ".*: (?[a-zA-Z-://._]*)(?raspios_lite_armhf-[0-9]{4}-[0-9]{2}-[0-9]{1,2})/(?[0-9]{4}-[0-9]{2}-[0-9]{1,2}).*$" + ); + console.log(m); + var url = m.groups.url; + var imageName = m.groups.imageName; + var updated = m.groups.updated; +} catch (err) { + console.error(err); + process.exit(1); +} + + +function getNewRaspbian(target, url, separator) { + console.log("getNewRaspbian"); + console.log(`${url}`); + return JSDOM.fromURL(url, {}).then((dom) => { + var document = dom.window.document; + var refs = document.getElementsByTagName("a"); + var test = Array.from(refs) + .filter((ref) => ref.textContent.match("raspios-.*-lite.*.zip$")) + .reduce((acc, element) => { + acc.push(element.textContent.match("raspios-.*-lite.*.zip$").input); + return acc; + }, [])[0]; + console.log(`Test var: ${test}`); + if (separator) { + return { + replacementLine: `${target}${separator} \"${url}/${test}\"`, + imageName: test, + }; + } + return { replacementLine: `${target}=\"${url}/${test}\"`, imageName: test }; + }); +} + + +function updateGitlabCILink(newLine, newURL, newName) { + try { + const data = fs + .readFileSync("../../.gitlab-ci.yml", "utf8") + .replace( + RegExp(` *## Auto-update\n *${newURL}:.*\n *RASPBIAN_NAME: .*`), + ` ## Auto-update\n ${newLine}\n RASPBIAN_NAME: ${newName}` + ); + fs.writeFile("../../.gitlab-ci.yml", data, (err, data) => { + if (err) { + console.error(err); + } + }); + } catch (err) { + console.error(err); + process.exit(1); + } +} + +JSDOM.fromURL(url, {}).then((dom) => { + var document = dom.window.document; + var table = document.getElementsByTagName("table"); + var rows = table[0].rows; + var matches = []; + for (var i = 0; i < rows.length; i++) { + var rowText = rows[i].textContent; + var regMatch = rowText.match(reg); + if (regMatch) { + matches.push(regMatch); + } + } + // Sort the accumulated matches + matches.sort(function (a, b) { + let al = Date.parse(a.groups.date); + let bl = Date.parse(b.groups.date); + if (al == bl) { + let ad = Date.parse(a.groups.updated); + let bd = Date.parse(b.groups.updated); + return bd - ad; + } + return bl - al; + }); + var matchOn = matches[0].input.split("/")[0]; + if (matchOn !== imageName) { + // We also need to extract the new image name from the index folder, as + // dated folders contain images with different dates in them o_O + console.error("We've got a new release! \\o/"); + + // Update the GitlabCI link unconditionally + console.log("Updating the GitlabCI Link"); + var newVar = getNewRaspbian( + "RASPBIAN_URL", + `${url}${matches[0].input.split(" ")[0].split("/").slice(0, -1)}`, + (separator = ":") + ).then(({ replacementLine, imageName }) => { + console.log(`New release: ${replacementLine}`); + updateGitlabCILink(replacementLine, "RASPBIAN_URL", imageName); + }); + } +}); From a9740723a2de954ac435083f8aa7e5ea6f70b99e Mon Sep 17 00:00:00 2001 From: Ole Petter Date: Mon, 1 Nov 2021 10:00:04 +0100 Subject: [PATCH 054/117] linkbot: Remove the image name suffix from the image name Changelog: None Signed-off-by: Ole Petter --- scripts/linkbot/monitor-raspbian-os.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/linkbot/monitor-raspbian-os.js b/scripts/linkbot/monitor-raspbian-os.js index 2655533665..024dd35c3b 100644 --- a/scripts/linkbot/monitor-raspbian-os.js +++ b/scripts/linkbot/monitor-raspbian-os.js @@ -41,13 +41,14 @@ function getNewRaspbian(target, url, separator) { return acc; }, [])[0]; console.log(`Test var: ${test}`); + const imageName = test.substring(0, test.length - 4); if (separator) { return { replacementLine: `${target}${separator} \"${url}/${test}\"`, - imageName: test, + imageName: imageName, }; } - return { replacementLine: `${target}=\"${url}/${test}\"`, imageName: test }; + return { replacementLine: `${target}=\"${url}/${test}\"`, imageName: imageName }; }); } From a5238bac000b53177d406fe8f8c6877317588cec Mon Sep 17 00:00:00 2001 From: mender-test-bot Date: Mon, 8 Nov 2021 03:18:09 +0000 Subject: [PATCH 055/117] Image-Bot - Image updates Changelog: None Signed-off-by: mender-test-bot --- .gitlab-ci.yml | 4 ++-- scripts/test/run-tests.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a45c12cabd..7a63fa47dc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,8 +11,8 @@ variables: MENDER_ADDON_CONFIGURE_VERSION: latest ## Auto-update - RASPBIAN_URL: https://downloads.raspberrypi.org/raspios_lite_armhf/images/raspios_lite_armhf-2021-01-12/2021-01-11-raspios-buster-armhf-lite.zip - RASPBIAN_NAME: 2021-01-11-raspios-buster-armhf-lite + RASPBIAN_URL: "https://downloads.raspberrypi.org/raspios_lite_armhf/images/raspios_lite_armhf-2021-05-28/2021-05-07-raspios-buster-armhf-lite.zip" + RASPBIAN_NAME: 2021-05-07-raspios-buster-armhf-lite # Which version of mender-convert to use in published image name. Normally # empty, in which case the branch or tag name will be used, but this variable diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index 50752077fb..227be33604 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -21,7 +21,7 @@ BBB_DEBIAN_SDCARD_IMAGE_URL="https://debian.beagleboard.org/images/bone-debian-1 # Not on official home page, but found via https://elinux.org/Beagleboard:BeagleBoneBlack_Debian: ## Auto-update -BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-10-01/buster-console/bone-debian-10.10-console-armhf-2021-10-01-1gb.img.xz" +BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-11-01/buster-console/bone-debian-10.11-console-armhf-2021-11-01-1gb.img.xz" ## Auto-update RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2020-02-14/2020-02-13-raspbian-buster-lite.zip" From 5533be91b1937e5237fceee24c9205d35a0e0be7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Nov 2021 07:06:27 +0000 Subject: [PATCH 056/117] Changelog:All: Bump jsdom from 18.0.0 to 18.0.1 in /scripts/linkbot Bumps [jsdom](https://github.com/jsdom/jsdom) from 18.0.0 to 18.0.1. - [Release notes](https://github.com/jsdom/jsdom/releases) - [Changelog](https://github.com/jsdom/jsdom/blob/master/Changelog.md) - [Commits](https://github.com/jsdom/jsdom/compare/18.0.0...18.0.1) --- updated-dependencies: - dependency-name: jsdom dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- scripts/linkbot/package-lock.json | 14 +++++++------- scripts/linkbot/package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/linkbot/package-lock.json b/scripts/linkbot/package-lock.json index c95b132221..da16b78eb7 100644 --- a/scripts/linkbot/package-lock.json +++ b/scripts/linkbot/package-lock.json @@ -6,7 +6,7 @@ "": { "dependencies": { "bent": "^7.3.12", - "jsdom": "^18.0.0" + "jsdom": "^18.0.1" } }, "node_modules/@tootallnate/once": { @@ -309,9 +309,9 @@ } }, "node_modules/jsdom": { - "version": "18.0.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-18.0.0.tgz", - "integrity": "sha512-HVLuBcFmwdWulStv5U+J59b1AyzXhM92KXlM8HQ3ecYtRM2OQEUCPMa4oNuDeCBmtRcC7tJvb0Xz5OeFXMOKTA==", + "version": "18.0.1", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-18.0.1.tgz", + "integrity": "sha512-mgVzrYP4IJiJKVqXkAdBn+jg+nQgPusBxTJulz3m1Y/1RIrkk8aDoNaQE5BNbHwe72WwiwE7k3Av2THXDpvzPQ==", "dependencies": { "abab": "^2.0.5", "acorn": "^8.5.0", @@ -845,9 +845,9 @@ "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==" }, "jsdom": { - "version": "18.0.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-18.0.0.tgz", - "integrity": "sha512-HVLuBcFmwdWulStv5U+J59b1AyzXhM92KXlM8HQ3ecYtRM2OQEUCPMa4oNuDeCBmtRcC7tJvb0Xz5OeFXMOKTA==", + "version": "18.0.1", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-18.0.1.tgz", + "integrity": "sha512-mgVzrYP4IJiJKVqXkAdBn+jg+nQgPusBxTJulz3m1Y/1RIrkk8aDoNaQE5BNbHwe72WwiwE7k3Av2THXDpvzPQ==", "requires": { "abab": "^2.0.5", "acorn": "^8.5.0", diff --git a/scripts/linkbot/package.json b/scripts/linkbot/package.json index 9fb6638a2a..409bdd1ed1 100644 --- a/scripts/linkbot/package.json +++ b/scripts/linkbot/package.json @@ -1,6 +1,6 @@ { "dependencies": { "bent": "^7.3.12", - "jsdom": "^18.0.0" + "jsdom": "^18.0.1" } } From 7830760785cde6ec43e71a27423775cc4c68c27f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Nov 2021 07:06:38 +0000 Subject: [PATCH 057/117] Changelog:All: Bump jsdom from 18.0.1 to 18.1.0 in /scripts/linkbot Bumps [jsdom](https://github.com/jsdom/jsdom) from 18.0.1 to 18.1.0. - [Release notes](https://github.com/jsdom/jsdom/releases) - [Changelog](https://github.com/jsdom/jsdom/blob/master/Changelog.md) - [Commits](https://github.com/jsdom/jsdom/compare/18.0.1...18.1.0) --- updated-dependencies: - dependency-name: jsdom dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- scripts/linkbot/package-lock.json | 14 +++++++------- scripts/linkbot/package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/linkbot/package-lock.json b/scripts/linkbot/package-lock.json index da16b78eb7..e2174289b5 100644 --- a/scripts/linkbot/package-lock.json +++ b/scripts/linkbot/package-lock.json @@ -6,7 +6,7 @@ "": { "dependencies": { "bent": "^7.3.12", - "jsdom": "^18.0.1" + "jsdom": "^18.1.0" } }, "node_modules/@tootallnate/once": { @@ -309,9 +309,9 @@ } }, "node_modules/jsdom": { - "version": "18.0.1", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-18.0.1.tgz", - "integrity": "sha512-mgVzrYP4IJiJKVqXkAdBn+jg+nQgPusBxTJulz3m1Y/1RIrkk8aDoNaQE5BNbHwe72WwiwE7k3Av2THXDpvzPQ==", + "version": "18.1.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-18.1.0.tgz", + "integrity": "sha512-q6QFAfSGLEUqRJ+GCV6vn6ItZCMARWh1d33wiJZPxc+wMNw7HK71JPmQ4C2lIZAsBH8TiJu4uplach/UcrC6bQ==", "dependencies": { "abab": "^2.0.5", "acorn": "^8.5.0", @@ -845,9 +845,9 @@ "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==" }, "jsdom": { - "version": "18.0.1", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-18.0.1.tgz", - "integrity": "sha512-mgVzrYP4IJiJKVqXkAdBn+jg+nQgPusBxTJulz3m1Y/1RIrkk8aDoNaQE5BNbHwe72WwiwE7k3Av2THXDpvzPQ==", + "version": "18.1.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-18.1.0.tgz", + "integrity": "sha512-q6QFAfSGLEUqRJ+GCV6vn6ItZCMARWh1d33wiJZPxc+wMNw7HK71JPmQ4C2lIZAsBH8TiJu4uplach/UcrC6bQ==", "requires": { "abab": "^2.0.5", "acorn": "^8.5.0", diff --git a/scripts/linkbot/package.json b/scripts/linkbot/package.json index 409bdd1ed1..c6b51086f9 100644 --- a/scripts/linkbot/package.json +++ b/scripts/linkbot/package.json @@ -1,6 +1,6 @@ { "dependencies": { "bent": "^7.3.12", - "jsdom": "^18.0.1" + "jsdom": "^18.1.0" } } From 6070d7eb5a7f09b58ab02a661a0b8ba0c351a476 Mon Sep 17 00:00:00 2001 From: Ole Petter Date: Mon, 15 Nov 2021 09:50:42 +0100 Subject: [PATCH 058/117] linkbot: Handle the double quote in the raspi-os image link Changelog: None Signed-off-by: Ole Petter --- scripts/linkbot/monitor-raspbian-os.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/linkbot/monitor-raspbian-os.js b/scripts/linkbot/monitor-raspbian-os.js index 024dd35c3b..d9cc1a4327 100644 --- a/scripts/linkbot/monitor-raspbian-os.js +++ b/scripts/linkbot/monitor-raspbian-os.js @@ -16,7 +16,7 @@ try { var reg = "raspios_lite_armhf-(?[0-9]{4}-[0-9]{2}-[0-9]{1,2})/(?[0-9]{4}-[0-9]{2}-[0-9]{1,2}).*$"; var m = line.match( - ".*: (?[a-zA-Z-://._]*)(?raspios_lite_armhf-[0-9]{4}-[0-9]{2}-[0-9]{1,2})/(?[0-9]{4}-[0-9]{2}-[0-9]{1,2}).*$" + ".*: \"(?[a-zA-Z-://._]*)(?raspios_lite_armhf-[0-9]{4}-[0-9]{2}-[0-9]{1,2})/(?[0-9]{4}-[0-9]{2}-[0-9]{1,2}).*\"$" ); console.log(m); var url = m.groups.url; From ff6e308c01d8e62b4cc2d9a670bc63c1602410eb Mon Sep 17 00:00:00 2001 From: mender-test-bot Date: Mon, 22 Nov 2021 03:17:10 +0000 Subject: [PATCH 059/117] Image-Bot - Image updates Changelog: None Signed-off-by: mender-test-bot --- .gitlab-ci.yml | 4 ++-- scripts/test/run-tests.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7a63fa47dc..c37ab70b81 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,8 +11,8 @@ variables: MENDER_ADDON_CONFIGURE_VERSION: latest ## Auto-update - RASPBIAN_URL: "https://downloads.raspberrypi.org/raspios_lite_armhf/images/raspios_lite_armhf-2021-05-28/2021-05-07-raspios-buster-armhf-lite.zip" - RASPBIAN_NAME: 2021-05-07-raspios-buster-armhf-lite + RASPBIAN_URL: "https://downloads.raspberrypi.org/raspios_lite_armhf/images/raspios_lite_armhf-2021-11-08/2021-10-30-raspios-bullseye-armhf-lite.zip" + RASPBIAN_NAME: 2021-10-30-raspios-bullseye-armhf-lite # Which version of mender-convert to use in published image name. Normally # empty, in which case the branch or tag name will be used, but this variable diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index 227be33604..688f30628f 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -21,7 +21,7 @@ BBB_DEBIAN_SDCARD_IMAGE_URL="https://debian.beagleboard.org/images/bone-debian-1 # Not on official home page, but found via https://elinux.org/Beagleboard:BeagleBoneBlack_Debian: ## Auto-update -BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-11-01/buster-console/bone-debian-10.11-console-armhf-2021-11-01-1gb.img.xz" +BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-11-14/buster-console/bone-debian-10.11-console-armhf-2021-11-14-1gb.img.xz" ## Auto-update RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2020-02-14/2020-02-13-raspbian-buster-lite.zip" From d84ef5b7e33f0fa4a1ecbeb799438e18bb9a8b88 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Nov 2021 07:05:43 +0000 Subject: [PATCH 060/117] Changelog:All: Bump jsdom from 18.1.0 to 18.1.1 in /scripts/linkbot Bumps [jsdom](https://github.com/jsdom/jsdom) from 18.1.0 to 18.1.1. - [Release notes](https://github.com/jsdom/jsdom/releases) - [Changelog](https://github.com/jsdom/jsdom/blob/master/Changelog.md) - [Commits](https://github.com/jsdom/jsdom/compare/18.1.0...18.1.1) --- updated-dependencies: - dependency-name: jsdom dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- scripts/linkbot/package-lock.json | 14 +++++++------- scripts/linkbot/package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/linkbot/package-lock.json b/scripts/linkbot/package-lock.json index e2174289b5..14933362a9 100644 --- a/scripts/linkbot/package-lock.json +++ b/scripts/linkbot/package-lock.json @@ -6,7 +6,7 @@ "": { "dependencies": { "bent": "^7.3.12", - "jsdom": "^18.1.0" + "jsdom": "^18.1.1" } }, "node_modules/@tootallnate/once": { @@ -309,9 +309,9 @@ } }, "node_modules/jsdom": { - "version": "18.1.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-18.1.0.tgz", - "integrity": "sha512-q6QFAfSGLEUqRJ+GCV6vn6ItZCMARWh1d33wiJZPxc+wMNw7HK71JPmQ4C2lIZAsBH8TiJu4uplach/UcrC6bQ==", + "version": "18.1.1", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-18.1.1.tgz", + "integrity": "sha512-NmJQbjQ/gpS/1at/ce3nCx89HbXL/f5OcenBe8wU1Eik0ROhyUc3LtmG3567dEHAGXkN8rmILW/qtCOPxPHQJw==", "dependencies": { "abab": "^2.0.5", "acorn": "^8.5.0", @@ -845,9 +845,9 @@ "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==" }, "jsdom": { - "version": "18.1.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-18.1.0.tgz", - "integrity": "sha512-q6QFAfSGLEUqRJ+GCV6vn6ItZCMARWh1d33wiJZPxc+wMNw7HK71JPmQ4C2lIZAsBH8TiJu4uplach/UcrC6bQ==", + "version": "18.1.1", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-18.1.1.tgz", + "integrity": "sha512-NmJQbjQ/gpS/1at/ce3nCx89HbXL/f5OcenBe8wU1Eik0ROhyUc3LtmG3567dEHAGXkN8rmILW/qtCOPxPHQJw==", "requires": { "abab": "^2.0.5", "acorn": "^8.5.0", diff --git a/scripts/linkbot/package.json b/scripts/linkbot/package.json index c6b51086f9..7099c99c2d 100644 --- a/scripts/linkbot/package.json +++ b/scripts/linkbot/package.json @@ -1,6 +1,6 @@ { "dependencies": { "bent": "^7.3.12", - "jsdom": "^18.1.0" + "jsdom": "^18.1.1" } } From fb0bd0982586ea2ca2139df3c09605e663a38958 Mon Sep 17 00:00:00 2001 From: mender-test-bot Date: Mon, 6 Dec 2021 03:18:01 +0000 Subject: [PATCH 061/117] Image-Bot - Image updates Changelog: None Signed-off-by: mender-test-bot --- scripts/test/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index 688f30628f..a7a687bfce 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -21,7 +21,7 @@ BBB_DEBIAN_SDCARD_IMAGE_URL="https://debian.beagleboard.org/images/bone-debian-1 # Not on official home page, but found via https://elinux.org/Beagleboard:BeagleBoneBlack_Debian: ## Auto-update -BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-11-14/buster-console/bone-debian-10.11-console-armhf-2021-11-14-1gb.img.xz" +BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-12-01/buster-console/bone-debian-10.11-console-armhf-2021-12-01-1gb.img.xz" ## Auto-update RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2020-02-14/2020-02-13-raspbian-buster-lite.zip" From fe00bc46cec22d7ac2a17a0a8ffefb25eecac057 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Dec 2021 07:06:49 +0000 Subject: [PATCH 062/117] Changelog:All: Bump jsdom from 18.1.1 to 19.0.0 in /scripts/linkbot Bumps [jsdom](https://github.com/jsdom/jsdom) from 18.1.1 to 19.0.0. - [Release notes](https://github.com/jsdom/jsdom/releases) - [Changelog](https://github.com/jsdom/jsdom/blob/master/Changelog.md) - [Commits](https://github.com/jsdom/jsdom/compare/18.1.1...19.0.0) --- updated-dependencies: - dependency-name: jsdom dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- scripts/linkbot/package-lock.json | 14 +++++++------- scripts/linkbot/package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/linkbot/package-lock.json b/scripts/linkbot/package-lock.json index 14933362a9..6f9c63c533 100644 --- a/scripts/linkbot/package-lock.json +++ b/scripts/linkbot/package-lock.json @@ -6,7 +6,7 @@ "": { "dependencies": { "bent": "^7.3.12", - "jsdom": "^18.1.1" + "jsdom": "^19.0.0" } }, "node_modules/@tootallnate/once": { @@ -309,9 +309,9 @@ } }, "node_modules/jsdom": { - "version": "18.1.1", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-18.1.1.tgz", - "integrity": "sha512-NmJQbjQ/gpS/1at/ce3nCx89HbXL/f5OcenBe8wU1Eik0ROhyUc3LtmG3567dEHAGXkN8rmILW/qtCOPxPHQJw==", + "version": "19.0.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-19.0.0.tgz", + "integrity": "sha512-RYAyjCbxy/vri/CfnjUWJQQtZ3LKlLnDqj+9XLNnJPgEGeirZs3hllKR20re8LUZ6o1b1X4Jat+Qd26zmP41+A==", "dependencies": { "abab": "^2.0.5", "acorn": "^8.5.0", @@ -845,9 +845,9 @@ "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==" }, "jsdom": { - "version": "18.1.1", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-18.1.1.tgz", - "integrity": "sha512-NmJQbjQ/gpS/1at/ce3nCx89HbXL/f5OcenBe8wU1Eik0ROhyUc3LtmG3567dEHAGXkN8rmILW/qtCOPxPHQJw==", + "version": "19.0.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-19.0.0.tgz", + "integrity": "sha512-RYAyjCbxy/vri/CfnjUWJQQtZ3LKlLnDqj+9XLNnJPgEGeirZs3hllKR20re8LUZ6o1b1X4Jat+Qd26zmP41+A==", "requires": { "abab": "^2.0.5", "acorn": "^8.5.0", diff --git a/scripts/linkbot/package.json b/scripts/linkbot/package.json index 7099c99c2d..060bf24868 100644 --- a/scripts/linkbot/package.json +++ b/scripts/linkbot/package.json @@ -1,6 +1,6 @@ { "dependencies": { "bent": "^7.3.12", - "jsdom": "^18.1.1" + "jsdom": "^19.0.0" } } From ecf0631564b09b561d74c7d7f557c188d5b67495 Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Thu, 9 Dec 2021 08:39:42 +0100 Subject: [PATCH 063/117] Fix "unbound variable" error when using `raspberrypi4_ubuntu_config`. Changelog: Title Signed-off-by: Kristian Amlie --- configs/raspberrypi4_ubuntu_config | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/configs/raspberrypi4_ubuntu_config b/configs/raspberrypi4_ubuntu_config index 2d0b7a52ee..9708f5e793 100644 --- a/configs/raspberrypi4_ubuntu_config +++ b/configs/raspberrypi4_ubuntu_config @@ -1,7 +1,4 @@ -# Binaries generated with the following script: -# -# https://github.com/drewmoseley/mender-convert-integration-scripts/blob/master/build-uboot-rpi.shraspberrypi-integration-scripts.tar.gz -RASPBERRYPI_BINARIES="raspberrypi4-integration-2019.01.tar.gz" +RASPBERRYPI_CONFIG="raspberrypi4" RASPBERRYPI_KERNEL_IMAGE="vmlinuz" MENDER_KERNEL_IMAGETYPE="zImage" MENDER_DEVICE_TYPE="raspberrypi4" From c095619dc1593982759d2e09f21264a5c1f51afa Mon Sep 17 00:00:00 2001 From: mender-test-bot Date: Mon, 27 Dec 2021 03:21:58 +0000 Subject: [PATCH 064/117] Image-Bot - Image updates Changelog: None Signed-off-by: mender-test-bot --- scripts/test/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index a7a687bfce..ea03170602 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -21,7 +21,7 @@ BBB_DEBIAN_SDCARD_IMAGE_URL="https://debian.beagleboard.org/images/bone-debian-1 # Not on official home page, but found via https://elinux.org/Beagleboard:BeagleBoneBlack_Debian: ## Auto-update -BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-12-01/buster-console/bone-debian-10.11-console-armhf-2021-12-01-1gb.img.xz" +BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-12-20/buster-console/bone-debian-10.11-console-armhf-2021-12-20-1gb.img.xz" ## Auto-update RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2020-02-14/2020-02-13-raspbian-buster-lite.zip" From e3edfc0cb9ed0c5f6b9e309929fb9007f16af770 Mon Sep 17 00:00:00 2001 From: Ole Petter Date: Thu, 20 Jan 2022 12:39:33 +0100 Subject: [PATCH 065/117] Bump license year to 2022 Changelog: None Signed-off-by: Ole Petter (cherry picked from commit 9fdb558426236b4390503e9c2fc656dfb764e57c) --- LICENSE | 2 +- LIC_FILES_CHKSUM.sha256 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index 9f2ba3e1aa..c58aee758a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2021 Northern.tech AS +Copyright 2022 Northern.tech AS All content in this project is licensed under the Apache License v2, unless indicated otherwise. diff --git a/LIC_FILES_CHKSUM.sha256 b/LIC_FILES_CHKSUM.sha256 index ef08ffac52..0c4ef152b3 100644 --- a/LIC_FILES_CHKSUM.sha256 +++ b/LIC_FILES_CHKSUM.sha256 @@ -1 +1 @@ -b4acfcfa2a0ba1a8c82ec3965fbcee886cff8394ca4214e0ddac0a36beb1e05a LICENSE +1033348db7606a7e61b6484f293847cf8d7a35766efebb97e304d4bd5d7f3f6b LICENSE From 6d0237a94d9de8cc8b47149fb59f2d761b556e4d Mon Sep 17 00:00:00 2001 From: mender-test-bot Date: Mon, 3 Jan 2022 03:18:33 +0000 Subject: [PATCH 066/117] Image-Bot - Image updates Changelog: None Signed-off-by: mender-test-bot --- scripts/test/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index ea03170602..3afecd96a2 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -21,7 +21,7 @@ BBB_DEBIAN_SDCARD_IMAGE_URL="https://debian.beagleboard.org/images/bone-debian-1 # Not on official home page, but found via https://elinux.org/Beagleboard:BeagleBoneBlack_Debian: ## Auto-update -BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-12-20/buster-console/bone-debian-10.11-console-armhf-2021-12-20-1gb.img.xz" +BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2022-01-01/buster-console/bone-debian-10.11-console-armhf-2022-01-01-1gb.img.xz" ## Auto-update RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2020-02-14/2020-02-13-raspbian-buster-lite.zip" From 144622acb4a116baa50913a262a0e4d5a1b6ee0a Mon Sep 17 00:00:00 2001 From: mender-test-bot Date: Mon, 31 Jan 2022 03:16:37 +0000 Subject: [PATCH 067/117] Image-Bot - Image updates Changelog: None Signed-off-by: mender-test-bot --- .gitlab-ci.yml | 4 ++-- scripts/test/run-tests.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c37ab70b81..6b974bb42f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,8 +11,8 @@ variables: MENDER_ADDON_CONFIGURE_VERSION: latest ## Auto-update - RASPBIAN_URL: "https://downloads.raspberrypi.org/raspios_lite_armhf/images/raspios_lite_armhf-2021-11-08/2021-10-30-raspios-bullseye-armhf-lite.zip" - RASPBIAN_NAME: 2021-10-30-raspios-bullseye-armhf-lite + RASPBIAN_URL: "https://downloads.raspberrypi.org/raspios_lite_armhf/images/raspios_lite_armhf-2022-01-28/2022-01-28-raspios-bullseye-armhf-lite.zip" + RASPBIAN_NAME: 2022-01-28-raspios-bullseye-armhf-lite # Which version of mender-convert to use in published image name. Normally # empty, in which case the branch or tag name will be used, but this variable diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index 3afecd96a2..30bef3b393 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -21,7 +21,7 @@ BBB_DEBIAN_SDCARD_IMAGE_URL="https://debian.beagleboard.org/images/bone-debian-1 # Not on official home page, but found via https://elinux.org/Beagleboard:BeagleBoneBlack_Debian: ## Auto-update -BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2022-01-01/buster-console/bone-debian-10.11-console-armhf-2022-01-01-1gb.img.xz" +BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2022-01-17/buster-console/bone-debian-10.11-console-armhf-2022-01-17-1gb.img.xz" ## Auto-update RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2020-02-14/2020-02-13-raspbian-buster-lite.zip" From b19680413f59ca403006c077f4bea10249b22c40 Mon Sep 17 00:00:00 2001 From: Lluis Campos Date: Mon, 31 Jan 2022 14:30:53 +0100 Subject: [PATCH 068/117] MEN-5410: Modify Debian packages installation for new repositories Changelog: Download and install Debian packages taking into account the target OS. Now downloads.mender.io serves four distributions: the two latests releases for Debian and Ubuntu. Probe /etc/os-release to figure out the correct package to install, and fallback to Debian Buster packages which was the previous default. Signed-off-by: Lluis Campos --- configs/mender_convert_config | 6 +++++- mender-convert-modify | 24 +++++++++++++++--------- modules/probe.sh | 23 +++++++++++++++++++++++ 3 files changed, 43 insertions(+), 10 deletions(-) diff --git a/configs/mender_convert_config b/configs/mender_convert_config index 4357caf7ba..31cb651a5e 100644 --- a/configs/mender_convert_config +++ b/configs/mender_convert_config @@ -146,10 +146,14 @@ MENDER_ADDON_CONFIGURE_VERSION="latest" # what you are doing. MENDER_STORAGE_URL="https://downloads.mender.io" -# Mender APT repo, containing binary files, do not modify this unless you know +# Mender APT repo url, containing binary files, do not modify this unless you know # what you are doing. MENDER_APT_REPO_URL="${MENDER_STORAGE_URL}/repos/debian" +# Mender APT repo available distributions, do not modify this unless you know +# what you are doing. +MENDER_APT_REPO_DISTS="debian/buster debian/bullseye ubuntu/bionic ubuntu/focal" + # Mender GitHub organization URL prefix MENDER_GITHUB_ORG="https://github.com/mendersoftware" diff --git a/mender-convert-modify b/mender-convert-modify index 2d876cecd8..085105a677 100755 --- a/mender-convert-modify +++ b/mender-convert-modify @@ -102,17 +102,23 @@ mkdir -p work/deb-packages log_info "Installing Mender client and related files" deb_arch=$(probe_debian_arch_name) +deb_distro=$(probe_debian_distro_name) +deb_codename=$(probe_debian_distro_codename) +if ! [[ "$MENDER_APT_REPO_DISTS" == *"${deb_distro}/${deb_codename}"* ]]; then + deb_distro="debian" + deb_codename="buster" +fi if [ "${MENDER_CLIENT_INSTALL}" = "y" ]; then log_info "Installing Mender client version ${MENDER_CLIENT_VERSION}" if [ "${MENDER_CLIENT_VERSION}" = "latest" ]; then - deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "stable" "mender-client") + deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "${deb_distro}/${deb_codename}/stable" "mender-client") elif [ "${MENDER_CLIENT_VERSION}" = "master" ]; then - deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "experimental" "mender-client") + deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "${deb_distro}/${deb_codename}/experimental" "mender-client") else - DEBIAN_REVISION="-1" + DEBIAN_REVISION="-1+${deb_distro}+${deb_codename}" deb_name=$(deb_from_repo_pool_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "mender-client" "${MENDER_CLIENT_VERSION}${DEBIAN_REVISION}") fi @@ -133,11 +139,11 @@ if [ "${MENDER_ADDON_CONNECT_INSTALL}" = "y" ]; then log_info "Installing Mender Connect addon" if [ "${MENDER_ADDON_CONNECT_VERSION}" = "latest" ]; then - deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "stable" "mender-connect") + deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "${deb_distro}/${deb_codename}/stable" "mender-connect") elif [ "${MENDER_ADDON_CONNECT_VERSION}" = "master" ]; then - deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "experimental" "mender-connect") + deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "${deb_distro}/${deb_codename}/experimental" "mender-connect") else - DEBIAN_REVISION="-1" + DEBIAN_REVISION="-1+${deb_distro}+${deb_codename}" deb_name=$(deb_from_repo_pool_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "mender-connect" "${MENDER_ADDON_CONNECT_VERSION}${DEBIAN_REVISION}") fi @@ -152,11 +158,11 @@ if [ "${MENDER_ADDON_CONFIGURE_INSTALL}" = "y" ]; then log_info "Installing Mender Configure addon" if [ "${MENDER_ADDON_CONFIGURE_VERSION}" = "latest" ]; then - deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "stable" "mender-configure") + deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "${deb_distro}/${deb_codename}/stable" "mender-configure") elif [ "${MENDER_ADDON_CONFIGURE_VERSION}" = "master" ]; then - deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "experimental" "mender-configure") + deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "${deb_distro}/${deb_codename}/experimental" "mender-configure") else - DEBIAN_REVISION="-1" + DEBIAN_REVISION="-1+${deb_distro}+${deb_codename}" deb_name=$(deb_from_repo_pool_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "mender-configure" "${MENDER_ADDON_CONFIGURE_VERSION}${DEBIAN_REVISION}") fi diff --git a/modules/probe.sh b/modules/probe.sh index 4a71aabc38..e90902a92e 100644 --- a/modules/probe.sh +++ b/modules/probe.sh @@ -97,6 +97,29 @@ probe_debian_arch_name() { echo "${deb_arch}" } +# Prints Debian distro name based on ID from /etc/os-release +# +# Special handling for raspbian, where ID_LIKE is used instead +# +# No input parameters and these work on the assumption that boot and root parts +# are mounted at work/boot and work/rootfs +probe_debian_distro_name() { + distro_name="$(. work/rootfs/etc/os-release && echo "$ID")" + if [[ "$distro_name" == "raspbian" ]]; then + distro_name="$(. work/rootfs/etc/os-release && echo "$ID_LIKE")" + fi + echo "${distro_name}" +} + +# Prints Debian distro codename based on VERSION_CODENAME from /etc/os-release +# +# No input parameters and these work on the assumption that boot and root parts +# are mounted at work/boot and work/rootfs +probe_debian_distro_codename() { + distro_codename="$(. work/rootfs/etc/os-release && echo "$VERSION_CODENAME")" + echo "${distro_codename}" +} + # Prints GRUB EFI target name depending on target architecture # # This is what the file name should be when put on target boot part. From 423686a44b761d105026f9a1243b2b51b12e52a7 Mon Sep 17 00:00:00 2001 From: Lluis Campos Date: Mon, 31 Jan 2022 14:40:01 +0100 Subject: [PATCH 069/117] mender_convert_config: Refactor deb packages install into a function Do not repeat yourself... Changelog: None Signed-off-by: Lluis Campos --- mender-convert-modify | 52 ++++--------------------------------------- modules/deb.sh | 39 +++++++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 49 deletions(-) diff --git a/mender-convert-modify b/mender-convert-modify index 085105a677..f5f2fb4b62 100755 --- a/mender-convert-modify +++ b/mender-convert-modify @@ -97,36 +97,14 @@ mkdir -p work/rootfs sudo mount ${boot_part} work/boot sudo mount ${root_part} work/rootfs -mkdir -p work/deb-packages - log_info "Installing Mender client and related files" -deb_arch=$(probe_debian_arch_name) -deb_distro=$(probe_debian_distro_name) -deb_codename=$(probe_debian_distro_codename) -if ! [[ "$MENDER_APT_REPO_DISTS" == *"${deb_distro}/${deb_codename}"* ]]; then - deb_distro="debian" - deb_codename="buster" -fi - if [ "${MENDER_CLIENT_INSTALL}" = "y" ]; then - log_info "Installing Mender client version ${MENDER_CLIENT_VERSION}" - - if [ "${MENDER_CLIENT_VERSION}" = "latest" ]; then - deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "${deb_distro}/${deb_codename}/stable" "mender-client") - elif [ "${MENDER_CLIENT_VERSION}" = "master" ]; then - deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "${deb_distro}/${deb_codename}/experimental" "mender-client") - else - DEBIAN_REVISION="-1+${deb_distro}+${deb_codename}" - deb_name=$(deb_from_repo_pool_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "mender-client" "${MENDER_CLIENT_VERSION}${DEBIAN_REVISION}") - fi - - deb_extract_package "work/deb-packages/${deb_name}" "work/rootfs/" + deb_get_and_install_pacakge mender-client "${MENDER_CLIENT_VERSION}" # Save installed client version for tests in Yocto variable format - testscfg_add "PREFERRED_VERSION_mender-client" "$(echo ${deb_name} | sed -r 's/.*_([0-9]+\.[0-9]+\.[0-9]+).*/\1/')" - + testscfg_add "PREFERRED_VERSION_mender-client" "$(echo ${DEB_NAME} | sed -r 's/.*_([0-9]+\.[0-9]+\.[0-9]+).*/\1/')" fi if [ "${MENDER_ENABLE_SYSTEMD}" == "y" ]; then @@ -135,38 +113,16 @@ if [ "${MENDER_ENABLE_SYSTEMD}" == "y" ]; then fi if [ "${MENDER_ADDON_CONNECT_INSTALL}" = "y" ]; then - log_info "Installing Mender Connect addon" - - if [ "${MENDER_ADDON_CONNECT_VERSION}" = "latest" ]; then - deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "${deb_distro}/${deb_codename}/stable" "mender-connect") - elif [ "${MENDER_ADDON_CONNECT_VERSION}" = "master" ]; then - deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "${deb_distro}/${deb_codename}/experimental" "mender-connect") - else - DEBIAN_REVISION="-1+${deb_distro}+${deb_codename}" - deb_name=$(deb_from_repo_pool_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "mender-connect" "${MENDER_ADDON_CONNECT_VERSION}${DEBIAN_REVISION}") - fi - - deb_extract_package "work/deb-packages/${deb_name}" "work/rootfs/" + deb_get_and_install_pacakge mender-connect "${MENDER_ADDON_CONNECT_VERSION}" run_and_log_cmd "sudo ln -sf /lib/systemd/system/mender-connect.service \ work/rootfs/etc/systemd/system/multi-user.target.wants/mender-connect.service" fi if [ "${MENDER_ADDON_CONFIGURE_INSTALL}" = "y" ]; then - log_info "Installing Mender Configure addon" - - if [ "${MENDER_ADDON_CONFIGURE_VERSION}" = "latest" ]; then - deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "${deb_distro}/${deb_codename}/stable" "mender-configure") - elif [ "${MENDER_ADDON_CONFIGURE_VERSION}" = "master" ]; then - deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "${deb_distro}/${deb_codename}/experimental" "mender-configure") - else - DEBIAN_REVISION="-1+${deb_distro}+${deb_codename}" - deb_name=$(deb_from_repo_pool_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "mender-configure" "${MENDER_ADDON_CONFIGURE_VERSION}${DEBIAN_REVISION}") - fi - - deb_extract_package "work/deb-packages/${deb_name}" "work/rootfs/" + deb_get_and_install_pacakge mender-configure "${MENDER_ADDON_CONFIGURE_VERSION}" fi # Do this unconditionally even if not installing add-ons. The reason is that if diff --git a/modules/deb.sh b/modules/deb.sh index 57299629fa..c25075388b 100644 --- a/modules/deb.sh +++ b/modules/deb.sh @@ -1,5 +1,5 @@ # -# Copyright 2020 Northern.tech AS +# Copyright 2022 Northern.tech AS # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,6 +14,7 @@ # limitations under the License. source modules/log.sh +source modules/probe.sh . # Download of latest deb package for the given distribution of an APT repository # @@ -111,3 +112,39 @@ function deb_extract_package() { log_info "Successfully installed $(basename ${deb_package}) into ${dest_dir}" } + +# Download and install the binary files of a deb package into work/deb-packages +# This is the main entry point of deb.sh +# Defines variable DEB_NAME with the actual filename installed +# +# $1 - Package name +# $2 - Package version +# +function deb_get_and_install_pacakge() { + if [[ $# -ne 2 ]]; then + log_fatal "deb_get_and_install_pacakge() requires 2 arguments" + fi + local package="$1" + local version="$2" + + mkdir -p work/deb-packages + + local deb_arch=$(probe_debian_arch_name) + local deb_distro=$(probe_debian_distro_name) + local deb_codename=$(probe_debian_distro_codename) + if ! [[ "$MENDER_APT_REPO_DISTS" == *"${deb_distro}/${deb_codename}"* ]]; then + deb_distro="debian" + deb_codename="buster" + fi + + DEB_NAME="" + if [ "${version}" = "latest" ]; then + DEB_NAME=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "${deb_distro}/${deb_codename}/stable" "${package}") + elif [ "${version}" = "master" ]; then + DEB_NAME=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "${deb_distro}/${deb_codename}/experimental" "${package}") + else + local debian_version="-1+${deb_distro}+${deb_codename}" + DEB_NAME=$(deb_from_repo_pool_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "${package}" "${version}${debian_version}") + fi + deb_extract_package "work/deb-packages/${DEB_NAME}" "work/rootfs/" +} From 766f2a9c2bcf0f98bf55d69134b50bd937cf8190 Mon Sep 17 00:00:00 2001 From: Lluis Campos Date: Mon, 31 Jan 2022 20:40:36 +0100 Subject: [PATCH 070/117] MEN-5410: Fallback to old version strings when package is missing Otherwise won't be possible to install old versions published with the old schema. Amends commit b196804 Changelog: None Signed-off-by: Lluis Campos --- modules/deb.sh | 19 ++++++++++++++++--- modules/run.sh | 20 ++++++++++++++++++-- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/modules/deb.sh b/modules/deb.sh index c25075388b..28572f6d0f 100644 --- a/modules/deb.sh +++ b/modules/deb.sh @@ -82,11 +82,17 @@ function deb_from_repo_pool_get() { local -r deb_package_path="pool/${component}/${initial}/${package}/${package}_${version}_${architecture}.deb" local -r filename=$(basename $deb_package_path) - run_and_log_cmd "wget -Nq ${repo_url}/${deb_package_path} -P ${download_dir}" + run_and_log_cmd_noexit "wget -Nq ${repo_url}/${deb_package_path} -P ${download_dir}" + local exit_code=$? rm -f /tmp/Packages - log_info "Successfully downloaded ${filename}" - echo ${filename} + if [[ ${exit_code} -ne 0 ]]; then + log_warn "Could not download ${filename}" + echo "" + else + log_info "Successfully downloaded ${filename}" + echo ${filename} + fi } # Extract the binary files of a deb package into a directory @@ -145,6 +151,13 @@ function deb_get_and_install_pacakge() { else local debian_version="-1+${deb_distro}+${deb_codename}" DEB_NAME=$(deb_from_repo_pool_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "${package}" "${version}${debian_version}") + if [[ -z "${DEB_NAME}" ]]; then + local debian_version_fallback="-1" + DEB_NAME=$(deb_from_repo_pool_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "${package}" "${version}${debian_version_fallback}") + if [[ -z "${DEB_NAME}" ]]; then + log_fatal "Specified version for ${package} cannot be found, tried ${version}${debian_version} and ${version}${debian_version_fallback}" + fi + fi fi deb_extract_package "work/deb-packages/${DEB_NAME}" "work/rootfs/" } diff --git a/modules/run.sh b/modules/run.sh index a52efd3ea1..93f9159da6 100644 --- a/modules/run.sh +++ b/modules/run.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright 2019 Northern.tech AS +# Copyright 2022 Northern.tech AS # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Run a command, capture output and log it +# Run a command, capture and log output, and exit on non-zero return code # # $1 - command to run function run_and_log_cmd() { @@ -31,3 +31,19 @@ function run_and_log_cmd() { exit ${exit_code} fi } + +# Run a command, capture and log output, and return the command's return code +# +# $1 - command to run +function run_and_log_cmd_noexit() { + local -r cmd="${1}" + local -r position="(${BASH_SOURCE[1]}:${BASH_LINENO[0]}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }" + local exit_code=0 + output="$({ eval ${cmd}; } 2>&1)" || exit_code=$? + local log_msg="Running: ${position} \n\r\n\r\t${cmd}" + if [[ "${output}" != "" ]]; then + log_msg="${log_msg}\n\t${output}\n" + fi + log_debug "${log_msg}" + return ${exit_code} +} From 6a72ff4cae337278dadceb7d18e7ef1c0d43ff18 Mon Sep 17 00:00:00 2001 From: Lluis Campos Date: Mon, 31 Jan 2022 20:59:53 +0100 Subject: [PATCH 071/117] MEN-5410: URL encode '+' also on pool downloads Now the packages from tags will also contain this character. Same fix as in ed5da386. Changelog: None Signed-off-by: Lluis Campos --- modules/deb.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/deb.sh b/modules/deb.sh index 28572f6d0f..229eee3b32 100644 --- a/modules/deb.sh +++ b/modules/deb.sh @@ -82,7 +82,8 @@ function deb_from_repo_pool_get() { local -r deb_package_path="pool/${component}/${initial}/${package}/${package}_${version}_${architecture}.deb" local -r filename=$(basename $deb_package_path) - run_and_log_cmd_noexit "wget -Nq ${repo_url}/${deb_package_path} -P ${download_dir}" + local -r deb_package_url=$(echo ${repo_url}/${deb_package_path} | sed 's/+/%2B/g') + run_and_log_cmd_noexit "wget -Nq ${deb_package_url} -P ${download_dir}" local exit_code=$? rm -f /tmp/Packages From d460de3c80e120792bb1c0987f591ef9e89b49b1 Mon Sep 17 00:00:00 2001 From: Lluis Campos Date: Tue, 1 Feb 2022 10:09:47 +0100 Subject: [PATCH 072/117] Fix installation of user specified versions for mender-configure add-on It has been broken since the introduction of mender-configure. This add-on is architecture independent, so when downloading the .deb directly from the pool we need to look for "all" suffix instead of the target architecture. This worked fine for "latest" and "master" versions because the filename for these were resolved via repository's Packages. Changelog: title Signed-off-by: Lluis Campos --- mender-convert-modify | 2 +- modules/deb.sh | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/mender-convert-modify b/mender-convert-modify index f5f2fb4b62..f7d39f1c65 100755 --- a/mender-convert-modify +++ b/mender-convert-modify @@ -122,7 +122,7 @@ fi if [ "${MENDER_ADDON_CONFIGURE_INSTALL}" = "y" ]; then log_info "Installing Mender Configure addon" - deb_get_and_install_pacakge mender-configure "${MENDER_ADDON_CONFIGURE_VERSION}" + deb_get_and_install_pacakge mender-configure "${MENDER_ADDON_CONFIGURE_VERSION}" "true" fi # Do this unconditionally even if not installing add-ons. The reason is that if diff --git a/modules/deb.sh b/modules/deb.sh index 229eee3b32..346de7e326 100644 --- a/modules/deb.sh +++ b/modules/deb.sh @@ -126,13 +126,15 @@ function deb_extract_package() { # # $1 - Package name # $2 - Package version +# $3 - Arch independent (optional, default "false") # function deb_get_and_install_pacakge() { - if [[ $# -ne 2 ]]; then - log_fatal "deb_get_and_install_pacakge() requires 2 arguments" + if ! [[ $# -eq 2 || $# -eq 3 ]]; then + log_fatal "deb_get_and_install_pacakge() requires 2 or 3 arguments" fi local package="$1" local version="$2" + local arch_indep="${3:-false}" mkdir -p work/deb-packages @@ -150,11 +152,19 @@ function deb_get_and_install_pacakge() { elif [ "${version}" = "master" ]; then DEB_NAME=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "${deb_distro}/${deb_codename}/experimental" "${package}") else + # On direct downloads, the architecture suffix will be "all" for arch independent packages + local pool_arch="" + if [[ "$arch_indep" == "true" ]]; then + pool_arch="all" + else + pool_arch="$deb_arch" + fi + local debian_version="-1+${deb_distro}+${deb_codename}" - DEB_NAME=$(deb_from_repo_pool_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "${package}" "${version}${debian_version}") + DEB_NAME=$(deb_from_repo_pool_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${pool_arch} "${package}" "${version}${debian_version}") if [[ -z "${DEB_NAME}" ]]; then local debian_version_fallback="-1" - DEB_NAME=$(deb_from_repo_pool_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "${package}" "${version}${debian_version_fallback}") + DEB_NAME=$(deb_from_repo_pool_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${pool_arch} "${package}" "${version}${debian_version_fallback}") if [[ -z "${DEB_NAME}" ]]; then log_fatal "Specified version for ${package} cannot be found, tried ${version}${debian_version} and ${version}${debian_version_fallback}" fi From 8a8d34f51b136d75d9cdf691aa43030b9154540d Mon Sep 17 00:00:00 2001 From: Peter Grzybowski Date: Wed, 2 Feb 2022 19:10:42 +0100 Subject: [PATCH 073/117] MEN-5257 Debian 11 tests support. ChangeLog:title Signed-off-by: Peter Grzybowski --- .gitlab-ci.yml | 5 +++++ configs/debian-qemux86-64_config | 22 ++++++++++++++++++++++ scripts/test/run-tests.sh | 14 ++++++++++++++ 3 files changed, 41 insertions(+) create mode 100644 configs/debian-qemux86-64_config diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c37ab70b81..9d029e8ee1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -225,6 +225,11 @@ test_acceptance_qemux86_64: script: - ./scripts/test/run-tests.sh --config versions_override_config --only qemux86_64 +test_acceptance_debian_qemux86_64: + <<: *test_acceptance + script: + - ./scripts/test/run-tests.sh --config versions_override_config --only debian-qemux86_64 + test_acceptance_raspberrypi: <<: *test_acceptance script: diff --git a/configs/debian-qemux86-64_config b/configs/debian-qemux86-64_config new file mode 100644 index 0000000000..d26fb19cd9 --- /dev/null +++ b/configs/debian-qemux86-64_config @@ -0,0 +1,22 @@ +# This configuration can be used to run on a qemux86-64 machine. +# +# The image is generated with the following command: +# +# mkosi --root-size=2G --distribution=debian --release=bullseye --format=gpt_ext4 --bootable --checksum +# --password password --package=openssh-server,dhcpcd5 --output=Debian-11-x86-64.img build +# +# Then manually uploaded to Mender's AWS S3 bucket for CI testing +# +# Locally, converted with the following command: +# +# MENDER_ARTIFACT_NAME=release-1 ./docker-mender-convert --disk-image input/Debian-11-x86-64.img --overlay rootfs_overlay_demo --config configs/qemux86-64_config +# +# and qemu is executed with the following command: +# +# qemu-system-x86_64 -enable-kvm -m 512 -smp 2 -bios /usr/share/OVMF/OVMF_CODE.fd -drive format=raw,file=deploy/Debian-11-x86-64-qemux86_64-mender.img + +MENDER_STORAGE_DEVICE_BASE=/dev/sda +MENDER_DEVICE_TYPE="qemux86_64" + +# Nothing to copy +MENDER_COPY_BOOT_GAP="n" diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index 3afecd96a2..0a2890f611 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -28,6 +28,8 @@ RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbi UBUNTU_IMAGE_URL="https://downloads.mender.io/mender-convert/images/Ubuntu-Focal-x86-64.img.gz" +DEBIAN_IMAGE_URL="https://downloads.mender.io/mender-convert/images/Debian-11-x86-64.img.gz" + ## Auto-update UBUNTU_SERVER_RPI_IMAGE_URL="http://cdimage.ubuntu.com/ubuntu/releases/20.04/release/ubuntu-20.04.3-preinstalled-server-armhf+raspi.img.xz" @@ -160,5 +162,17 @@ else || test_result=$? fi + if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "debian-qemux86_64" ]; then + wget --progress=dot:giga -N ${DEBIAN_IMAGE_URL} -P input/ + convert_and_test "qemux86_64" \ + "release-1" \ + "input/Debian-11-x86-64.img.gz" \ + "--overlay tests/ssh-public-key-overlay" \ + "--config configs/debian-qemux86-64_config $EXTRA_CONFIG" \ + || test_result=$? + fi + + + exit $test_result fi From f497dcb48ac8e318e9f41ca5a5a63868c62ade89 Mon Sep 17 00:00:00 2001 From: mender-test-bot Date: Mon, 7 Feb 2022 03:16:56 +0000 Subject: [PATCH 074/117] Image-Bot - Image updates Changelog: None Signed-off-by: mender-test-bot --- scripts/test/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index 9d6d179b37..416b06d1e9 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -21,7 +21,7 @@ BBB_DEBIAN_SDCARD_IMAGE_URL="https://debian.beagleboard.org/images/bone-debian-1 # Not on official home page, but found via https://elinux.org/Beagleboard:BeagleBoneBlack_Debian: ## Auto-update -BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2022-01-17/buster-console/bone-debian-10.11-console-armhf-2022-01-17-1gb.img.xz" +BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2022-02-03/buster-console/bone-debian-10.11-console-armhf-2022-02-03-1gb.img.xz" ## Auto-update RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2020-02-14/2020-02-13-raspbian-buster-lite.zip" From 30d1ee1eb33159d4b630e26c4e4ab448840184cb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Feb 2022 07:02:58 +0000 Subject: [PATCH 075/117] Changelog:All: Bump tests/mender-image-tests from `02b3578` to `ee1266a` Bumps [tests/mender-image-tests](https://github.com/mendersoftware/mender-image-tests) from `02b3578` to `ee1266a`. - [Release notes](https://github.com/mendersoftware/mender-image-tests/releases) - [Commits](https://github.com/mendersoftware/mender-image-tests/compare/02b357825d6f14b1088a116e83fd5e675fedb87a...ee1266afe8ec58efce9b4223e2489ac023b0582f) --- updated-dependencies: - dependency-name: tests/mender-image-tests dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/mender-image-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mender-image-tests b/tests/mender-image-tests index 02b357825d..ee1266afe8 160000 --- a/tests/mender-image-tests +++ b/tests/mender-image-tests @@ -1 +1 @@ -Subproject commit 02b357825d6f14b1088a116e83fd5e675fedb87a +Subproject commit ee1266afe8ec58efce9b4223e2489ac023b0582f From afb1b23bdd834b7d35135c00b35c17f868d6568a Mon Sep 17 00:00:00 2001 From: Simon Ensslen Date: Mon, 31 Jan 2022 15:56:16 +0100 Subject: [PATCH 076/117] modify fstab instead of replacing in order to keep modifications made Changelog: The fstab file from the image being converted is now preserved across convertions, with the Mender specific additions merged with the existing fstab file, as opposed to replacing it completely. Which was the previous approach. Signed-off-by: Simon Ensslen --- configs/mender_convert_config | 3 +++ mender-convert-modify | 13 ++++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/configs/mender_convert_config b/configs/mender_convert_config index 4357caf7ba..f5242580be 100644 --- a/configs/mender_convert_config +++ b/configs/mender_convert_config @@ -32,6 +32,9 @@ MENDER_ARTIFACT_COMPRESSION="gzip" # this. MENDER_ENABLE_SYSTEMD=y +# Set the fstab options for mounting the boot partition +MENDER_BOOT_PART_FSTAB_OPTS="defaults,sync" + # Custom mkfs options for creating the rootfs partition MENDER_ROOT_PART_MKFS_OPTS="" diff --git a/mender-convert-modify b/mender-convert-modify index 2d876cecd8..1a1f7a4da3 100755 --- a/mender-convert-modify +++ b/mender-convert-modify @@ -356,14 +356,17 @@ fi log_info "Using boot partition device in fstab: $boot_part_device" log_info "Using data partition device in fstab: $data_part_device" -sudo bash -c "cat <<- EOF > work/rootfs/etc/fstab -# stock fstab - you probably want to override this with a machine specific one +fstab=$(sed '/\/boot/d' work/rootfs/etc/fstab) +fstab=$(sed '/.*\s\/\s.*/d' <<< "$fstab") +cat <<- EOF > work/rootfs/etc/fstab +${boot_part_device} ${boot_part_mountpoint} auto ${MENDER_BOOT_PART_FSTAB_OPTS} 0 0 /dev/root / auto ${MENDER_ROOT_PART_FSTAB_OPTS} 1 ${MENDER_ROOT_PART_FS_PASSNO} -proc /proc proc defaults 0 0 -${boot_part_device} ${boot_part_mountpoint} auto defaults,sync 0 0 ${data_part_device} /data auto ${MENDER_DATA_PART_FSTAB_OPTS} 0 0 -EOF" + +# entries kept from original fstab +$fstab +EOF # # Make sure to re-label rootfs when selinux is in enforcing mode From c67b5ea1deacddeef7a6d54172e6a655ce8ceaaa Mon Sep 17 00:00:00 2001 From: Josef Holzmayr Date: Fri, 11 Feb 2022 15:07:50 +0100 Subject: [PATCH 077/117] docker-mender-convert: fix mktemp invocation for macOS/*BSD Changelog: fix compatibility of docker-mender-convert with macOS/*BSD The macOS/*BSD versions of mktemp do not support the -p flag for passing in a path prefix. Fixing this by making $(pwd) part of the template (-t). Based on a suggestion at [1]. [1]: https://hub.mender.io/t/bug-convert-a-raw-disk-image-in-macos-fails-mktemp-illegal-option-p/4428/3?u=theyoctojester Signed-off-by: Josef Holzmayr --- docker-mender-convert | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-mender-convert b/docker-mender-convert index a2dad60ddf..a4cb80d6f5 100755 --- a/docker-mender-convert +++ b/docker-mender-convert @@ -26,7 +26,7 @@ MENDER_CONVERT_DIR="$(pwd)" MENDER_CONVERT_VERSION=$(git_mender_convert_version) # Create a unique work directory so we can run multiple containers in parallel -WORK_DIR=$(mktemp -d -p $(pwd) -t work.XXXX) +WORK_DIR=$(mktemp -d $(pwd)/work.XXXX) WORK_SUFFIX=$(echo $WORK_DIR | awk -F. '{print $NF}') LOG_FILE="convert.log.${WORK_SUFFIX}" From aed2dbd4e74dd35d1771cd5f678f4450c36ec74c Mon Sep 17 00:00:00 2001 From: Ole Petter Date: Mon, 14 Feb 2022 14:05:14 +0100 Subject: [PATCH 078/117] MEN-5221: Keep the UUID of the original first partition This simply writes the GPT UUID of the boot partition to be the same as the UUID of the first partition in the original disk image. Changelog: Keep the UUID of the original first partition on the boot partition of the converted image when the partition table is GPT. Signed-off-by: Ole Petter --- mender-convert-package | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mender-convert-package b/mender-convert-package index 7d30bb1584..30b8e80133 100755 --- a/mender-convert-package +++ b/mender-convert-package @@ -256,6 +256,10 @@ if [ "${partition_scheme}" == "gpt" ]; then log_info "Writing GPT partition table" run_and_log_cmd "${PARTED} -s ${img_path} mklabel gpt" run_and_log_cmd "${PARTED} -s ${img_path} unit s mkpart ESP fat32 ${boot_part_start} ${boot_part_end}" + # Keep the uuid of the original gpt table for the boot partition + boot_partuuid=$(disk_get_part_value "${disk_image}" 1 UUID) + log_info "Updating GPT partition uuid of the boot partition in image: ${img_path} to ${boot_partuuid}" + run_and_log_cmd "sgdisk -u ${MENDER_BOOT_PART_NUMBER}:${boot_partuuid} '${img_path}'" else log_info "Writing DOS (MBR) partition table" run_and_log_cmd "${PARTED} -s ${img_path} mklabel msdos" From 2b805e29dcfbca1bcf9ea6339ef7755025737e56 Mon Sep 17 00:00:00 2001 From: Ole Petter Date: Mon, 14 Feb 2022 16:51:15 +0100 Subject: [PATCH 079/117] MEN-5221: Keep Grub if EFI shim is found in the boot partition In the case a boot shim is found on the boot partition, we do keep the boot partition pristine, and only install the generated mender `grub.cfg` file we need. In all other cases, we fall back to the old functionality of installing mender-grub and nuking the existing bootloader. Changelog: commit Signed-off-by: Ole Petter --- mender-convert-modify | 101 ++--------------------------- modules/grub.sh | 144 ++++++++++++++++++++++++++++++++++++++++++ modules/probe.sh | 13 +++- 3 files changed, 161 insertions(+), 97 deletions(-) create mode 100644 modules/grub.sh diff --git a/mender-convert-modify b/mender-convert-modify index f7d39f1c65..8f0a2ae800 100755 --- a/mender-convert-modify +++ b/mender-convert-modify @@ -51,6 +51,7 @@ source modules/disk.sh source modules/probe.sh source modules/deb.sh source modules/testscfg.sh +source modules/grub.sh # The mender_convert_config is always used and provides all the defaults declare -a configs=("configs/mender_convert_config") @@ -139,104 +140,14 @@ if [ "${MENDER_GRUB_EFI_INTEGRATION}" == "y" ]; then # Check for known U-Boot problems in all files on the boot partition. check_for_broken_uboot_uefi_support work/boot - run_and_log_cmd "wget -Nq '${MENDER_GRUBENV_URL}' -P work/" - run_and_log_cmd "tar xzvf work/${MENDER_GRUBENV_VERSION}.tar.gz -C work/" - - if [ -z "${MENDER_GRUB_KERNEL_IMAGETYPE}" ]; then - kernel_imagetype=$(probe_kernel_in_boot_and_root) - else - kernel_imagetype="${MENDER_GRUB_KERNEL_IMAGETYPE}" - fi - - if [ -z "${MENDER_GRUB_INITRD_IMAGETYPE}" ]; then - initrd_imagetype=$(probe_initrd_in_boot_and_root) - else - initrd_imagetype="${MENDER_GRUB_INITRD_IMAGETYPE}" - fi - - cat <<- EOF > work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION}/mender_grubenv_defines -mender_rootfsa_part=${MENDER_ROOTFS_PART_A_NUMBER} -mender_rootfsb_part=${MENDER_ROOTFS_PART_B_NUMBER} -mender_grub_storage_device=${MENDER_GRUB_STORAGE_DEVICE} -kernel_imagetype=kernel -initrd_imagetype=initrd -EOF - - run_and_log_cmd "sudo ln -s ${kernel_imagetype} work/rootfs/boot/kernel" - if [ "${initrd_imagetype}" != "" ]; then - run_and_log_cmd "sudo ln -s ${initrd_imagetype} work/rootfs/boot/initrd" - fi - - # For partuuid support grub.cfg expects dedicated variables to be added - if [ "${MENDER_ENABLE_PARTUUID}" == "y" ]; then - rootfsa_partuuid=$(disk_get_partuuid_from_device "${root_part_a_device}") - rootfsb_partuuid=$(disk_get_partuuid_from_device "${root_part_b_device}") - log_info "Using root partition A partuuid in grubenv: $rootfsa_partuuid" - log_info "Using root partition B partuuid in grubenv: $rootfsb_partuuid" - cat <<- EOF >> work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION}/mender_grubenv_defines -mender_rootfsa_uuid=${rootfsa_partuuid} -mender_rootfsb_uuid=${rootfsb_partuuid} -EOF + if has_secureboot_shim "work/boot"; then + # No need to install Grub, use the one already present, and only install + # our grub.cfg + grub_install_with_shim_present else - cat <<- EOF >> work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION}/mender_grubenv_defines -mender_kernel_root_base=${MENDER_STORAGE_DEVICE_BASE} -EOF + grub_install_mender_grub fi - if [ -n "${MENDER_GRUB_KERNEL_BOOT_ARGS}" ]; then - cat <<- EOF > work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION}/11_bootargs_grub.cfg -set bootargs="${MENDER_GRUB_KERNEL_BOOT_ARGS}" -EOF - fi - - cd work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION} - run_and_log_cmd "make 2>&1" - run_and_log_cmd "sudo make DESTDIR=../ BOOT_DIR=boot install-boot-files" - run_and_log_cmd "sudo make DESTDIR=../rootfs install-tools" - cd - > /dev/null 2>&1 - - # Remove conflicting boot files. These files do not necessarily effect the - # functionality, but lets get rid of them to avoid confusion. - # - # There is no Mender integration for EFI boot or systemd-boot. - sudo rm -rf work/boot/loader work/rootfs/boot/loader - sudo rm -rf work/boot/EFI/Linux - sudo rm -rf work/boot/EFI/systemd - sudo rm -rf work/boot/NvVars - for empty_dir in $( - cd work/boot - find . -maxdepth 1 -type d -empty - ); do - sudo rmdir work/boot/$empty_dir - done - - log_info "Installing GRUB" - - arch=$(probe_arch) - efi_name=$(probe_grub_efi_name) - efi_target_name=$(probe_grub_efi_target_name) - - log_info "GRUB EFI: ${efi_target_name}" - - run_and_log_cmd "wget -Nq ${MENDER_GRUB_BINARY_STORAGE_URL}/${arch}/${efi_name} -P work/" - run_and_log_cmd "wget -Nq ${MENDER_GRUB_BINARY_STORAGE_URL}/${arch}/grub-editenv -P work/" - - run_and_log_cmd "sudo install -m 751 work/grub-editenv work/rootfs/usr/bin/" - - run_and_log_cmd "sudo mkdir -p work/boot/EFI/BOOT" - run_and_log_cmd "sudo cp work/${efi_name} -P work/boot/EFI/BOOT/${efi_target_name}" -fi - -# Copy dtb directory to the boot partition for use by the bootloader. -if [ -d work/rootfs/boot/dtbs ]; then - # Look for the first directory that has dtb files. First check the base - # folder, then any subfolders in versioned order. - for candidate in work/rootfs/boot/dtbs $(find work/rootfs/boot/dtbs/ -maxdepth 1 -type d | sort -V -r); do - if [ $(find $candidate -maxdepth 1 -name '*.dtb' | wc -l) -gt 0 ]; then - run_and_log_cmd "sudo cp -r $candidate work/boot/dtb" - break - fi - done fi run_and_log_cmd "sudo mkdir -p work/rootfs/data/mender" diff --git a/modules/grub.sh b/modules/grub.sh new file mode 100644 index 0000000000..73bbb60235 --- /dev/null +++ b/modules/grub.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# +# Copyright 2022 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# grub_create_grub_config +# +# +function grub_create_grub_config() { + + run_and_log_cmd "wget -Nq '${MENDER_GRUBENV_URL}' -P work/" + run_and_log_cmd "tar xzvf work/${MENDER_GRUBENV_VERSION}.tar.gz -C work/" + + cat <<- EOF > work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION}/mender_grubenv_defines +mender_rootfsa_part=${MENDER_ROOTFS_PART_A_NUMBER} +mender_rootfsb_part=${MENDER_ROOTFS_PART_B_NUMBER} +mender_grub_storage_device=${MENDER_GRUB_STORAGE_DEVICE} +kernel_imagetype=kernel +initrd_imagetype=initrd +EOF + + # For partuuid support grub.cfg expects dedicated variables to be added + if [ "${MENDER_ENABLE_PARTUUID}" == "y" ]; then + rootfsa_partuuid=$(disk_get_partuuid_from_device "${root_part_a_device}") + rootfsb_partuuid=$(disk_get_partuuid_from_device "${root_part_b_device}") + log_info "Using root partition A partuuid in grubenv: $rootfsa_partuuid" + log_info "Using root partition B partuuid in grubenv: $rootfsb_partuuid" + cat <<- EOF >> work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION}/mender_grubenv_defines +mender_rootfsa_uuid=${rootfsa_partuuid} +mender_rootfsb_uuid=${rootfsb_partuuid} +EOF + else + cat <<- EOF >> work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION}/mender_grubenv_defines +mender_kernel_root_base=${MENDER_STORAGE_DEVICE_BASE} +EOF + fi + + if [ -n "${MENDER_GRUB_KERNEL_BOOT_ARGS}" ]; then + cat <<- EOF > work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION}/11_bootargs_grub.cfg +set bootargs="${MENDER_GRUB_KERNEL_BOOT_ARGS}" +EOF + fi + + ( + cd work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION} + run_and_log_cmd "make 2>&1" + run_and_log_cmd "sudo make DESTDIR=../ BOOT_DIR=boot install-boot-files" + run_and_log_cmd "sudo make DESTDIR=../rootfs install-tools" + ) + +} + +# grub_install_grub_editenv_binary +# +# Install the editenv binary +function grub_install_grub_editenv_binary() { + log_info "Installing the GRUB editenv binary" + + arch=$(probe_arch) + + run_and_log_cmd "wget -Nq ${MENDER_GRUB_BINARY_STORAGE_URL}/${arch}/grub-editenv -P work/" + run_and_log_cmd "sudo install -m 751 work/grub-editenv work/rootfs/usr/bin/" + +} + +# grub_install_with_shim_present +# +# Keep the existing boot shim, and bootloader, and only install the mender-grub +# config +function grub_install_with_shim_present() { + + grub_create_grub_config + + grub_install_grub_editenv_binary + +} + +# grub_install_mender_grub +# +# Install mender-grub on the converted boot partition +function grub_install_mender_grub() { + kernel_imagetype=${MENDER_GRUB_KERNEL_IMAGETYPE:-$(probe_kernel_in_boot_and_root)} + initrd_imagetype=${MENDER_GRUB_INITRD_IMAGETYPE:-$(probe_initrd_in_boot_and_root)} + + run_and_log_cmd "sudo ln -s ${kernel_imagetype} work/rootfs/boot/kernel" + if [ "${initrd_imagetype}" != "" ]; then + run_and_log_cmd "sudo ln -s ${initrd_imagetype} work/rootfs/boot/initrd" + fi + + log_info "Generating the mender-grub config..." + grub_create_grub_config + + # Remove conflicting boot files. These files do not necessarily effect the + # functionality, but lets get rid of them to avoid confusion. + # + # There is no Mender integration for EFI boot or systemd-boot. + sudo rm -rf work/boot/loader work/rootfs/boot/loader + sudo rm -rf work/boot/EFI/Linux + sudo rm -rf work/boot/EFI/systemd + sudo rm -rf work/boot/NvVars + for empty_dir in $( + cd work/boot && find . -maxdepth 1 -type d -empty + ); do + sudo rmdir work/boot/$empty_dir + done + + log_info "Installing GRUB..." + + log_info "Installing mender-grub-editenv" + grub_install_grub_editenv_binary + + local -r arch=$(probe_arch) + local -r efi_name=$(probe_grub_efi_name) + local -r efi_target_name=$(probe_grub_efi_target_name) + + log_info "GRUB EFI: ${efi_target_name}" + + run_and_log_cmd "wget -Nq ${MENDER_GRUB_BINARY_STORAGE_URL}/${arch}/${efi_name} -P work/" + run_and_log_cmd "sudo mkdir -p work/boot/EFI/BOOT" + run_and_log_cmd "sudo cp work/${efi_name} -P work/boot/EFI/BOOT/${efi_target_name}" + + # Copy dtb directory to the boot partition for use by the bootloader. + if [ -d work/rootfs/boot/dtbs ]; then + # Look for the first directory that has dtb files. First check the base + # folder, then any subfolders in versioned order. + for candidate in work/rootfs/boot/dtbs $(find work/rootfs/boot/dtbs/ -maxdepth 1 -type d | sort -V -r); do + if [ $(find $candidate -maxdepth 1 -name '*.dtb' | wc -l) -gt 0 ]; then + run_and_log_cmd "sudo cp -r $candidate work/boot/dtb" + break + fi + done + fi +} diff --git a/modules/probe.sh b/modules/probe.sh index e90902a92e..ef16b919b2 100644 --- a/modules/probe.sh +++ b/modules/probe.sh @@ -54,8 +54,8 @@ probe_arch() { # No input parameters and these work on the assumption that boot and root parts # are mounted at work/boot and work/rootfs probe_grub_efi_name() { - efi_name="" - arch=$(probe_arch) + local efi_name="" + local -r arch=$(probe_arch) case "${arch}" in "x86-64") efi_name="grub-efi-bootx64.efi" @@ -344,3 +344,12 @@ is_efi_compatible_kernel() { esac return 0 } + +# has_secureboot_shim +# +# $1 - the boot partition to search for a secureboot shim +# +# Checks the EFI/* filesystem for the presence of a signed boot shim +has_secureboot_shim() { + find "${1}" -type f -name 'shim*.efi' -print0 | grep -qz shim +} From 687d09e9b115b99046b861c2a5d24804beec67e8 Mon Sep 17 00:00:00 2001 From: Ole Petter Date: Fri, 18 Feb 2022 14:39:12 +0100 Subject: [PATCH 080/117] refac: Use local for function local variables Changelog: None Signed-off-by: Ole Petter --- modules/probe.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/probe.sh b/modules/probe.sh index ef16b919b2..1885fc5055 100644 --- a/modules/probe.sh +++ b/modules/probe.sh @@ -104,7 +104,7 @@ probe_debian_arch_name() { # No input parameters and these work on the assumption that boot and root parts # are mounted at work/boot and work/rootfs probe_debian_distro_name() { - distro_name="$(. work/rootfs/etc/os-release && echo "$ID")" + local distro_name="$(. work/rootfs/etc/os-release && echo "$ID")" if [[ "$distro_name" == "raspbian" ]]; then distro_name="$(. work/rootfs/etc/os-release && echo "$ID_LIKE")" fi @@ -116,7 +116,7 @@ probe_debian_distro_name() { # No input parameters and these work on the assumption that boot and root parts # are mounted at work/boot and work/rootfs probe_debian_distro_codename() { - distro_codename="$(. work/rootfs/etc/os-release && echo "$VERSION_CODENAME")" + local -r distro_codename="$(. work/rootfs/etc/os-release && echo "$VERSION_CODENAME")" echo "${distro_codename}" } @@ -127,8 +127,8 @@ probe_debian_distro_codename() { # No input parameters and these work on the assumption that boot and root parts # are mounted at work/boot and work/rootfs probe_grub_efi_target_name() { - efi_target_name="" - arch=$(probe_arch) + local efi_target_name="" + local -r arch=$(probe_arch) case "$arch" in "x86-64") efi_target_name="bootx64.efi" @@ -151,7 +151,7 @@ probe_grub_efi_target_name() { # $1 - directory in which the search is performed # probe_kernel_image() { - kernel_image_path="" + local kernel_image_path="" for image in vmlinuz zImage bzImage; do # Linux kernel image type and naming varies between different platforms. # @@ -188,7 +188,7 @@ probe_kernel_image() { # $1 - directory in which the search is performed # probe_initrd_image() { - initrd_image_path="" + local initrd_image_path="" for image in initramfs initrd; do # initrd/initramfs naming varies between different platforms. # From fdb9d60f64adadb6a6caf47d05770576b479afba Mon Sep 17 00:00:00 2001 From: Josef Holzmayr Date: Fri, 25 Feb 2022 13:14:10 +0100 Subject: [PATCH 081/117] README: add section "Requisites" Changelog: clarify supported development platforms Defining Ubuntu 18.04 x86_64 as the officially supported host platform and clarifying requirements for local storage Signed-off-by: Josef Holzmayr --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index e06241b507..8f7bcea77c 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,19 @@ on your device. ![Mender logo](https://github.com/mendersoftware/mender/raw/master/mender_logo.png) +## Requisites + +`mender-convert` is supported on the following development platform(s): + + - Ubuntu 18.04 x86, 64bit + +Other platforms may work, but are not under active testing. Patches to add additional +platforms or fix compatibility issues are welcome. + +The storage used during the process, specifically the work directories (usually below the clone of +`mender-convert`) and the output directory should be located on local, Unix-style filesystem. +Using network storage or emulated filesystems might cause permission or owner issues. + ## Getting started From a8df0abd02681b1dfae11af098b83b0920d7cfd0 Mon Sep 17 00:00:00 2001 From: Josef Holzmayr Date: Fri, 25 Feb 2022 14:13:47 +0100 Subject: [PATCH 082/117] README: note 32bit-compatibility and Raspbian Changelog: clarify Raspbian compatibility Signed-off-by: Josef Holzmayr --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 8f7bcea77c..b824f2c299 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,11 @@ For more detailed information about `mender-convert` please visit the ### Prepare image and configuration +*Only 32bit-Images are supported at the moment!* + +The following steps give a quick start using Raspbian. For a more detailed guide, +especially concerning version compatibilities, please visit [the the corresponding thread](https://hub.mender.io/t/raspberry-pi-3-model-b-b-raspbian/140) on the Mender Hub. + Download the raw Raspberry Pi disk image into a subdirectory input: ```bash From 5c65235b14ef197a656efd81d7869ce458282f6a Mon Sep 17 00:00:00 2001 From: mender-test-bot Date: Mon, 28 Feb 2022 03:04:39 +0000 Subject: [PATCH 083/117] Image-Bot - Image updates Changelog: None Signed-off-by: mender-test-bot --- scripts/test/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index 416b06d1e9..4453f284dc 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -31,7 +31,7 @@ UBUNTU_IMAGE_URL="https://downloads.mender.io/mender-convert/images/Ubuntu-Focal DEBIAN_IMAGE_URL="https://downloads.mender.io/mender-convert/images/Debian-11-x86-64.img.gz" ## Auto-update -UBUNTU_SERVER_RPI_IMAGE_URL="http://cdimage.ubuntu.com/ubuntu/releases/20.04/release/ubuntu-20.04.3-preinstalled-server-armhf+raspi.img.xz" +UBUNTU_SERVER_RPI_IMAGE_URL="http://cdimage.ubuntu.com/ubuntu/releases/20.04/release/ubuntu-20.04.4-preinstalled-server-armhf+raspi.img.xz" # Keep common function declarations in separate utils script UTILS_PATH=${0/$(basename $0)/test-utils.sh} From 86c47bd633561b27a1bd407a970ab30ee6025d97 Mon Sep 17 00:00:00 2001 From: Josef Holzmayr Date: Mon, 28 Feb 2022 16:09:54 +0100 Subject: [PATCH 084/117] README: add compatibility tables for configurations Changelog: title The configuration files are only meant to work on specific combinations of board+OS. These are added verbatim in order to avoid confusion. Signed-off-by: Josef Holzmayr --- README.md | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b824f2c299..0a967a6ea3 100644 --- a/README.md +++ b/README.md @@ -44,10 +44,35 @@ For more detailed information about `mender-convert` please visit the [Debian family](https://docs.mender.io/artifacts/debian-family) section in [the Mender documentation](https://docs.mender.io/). +## Included configurations -### Prepare image and configuration +### Core configurations + +These configurations are officially supported. + +| Configuration | Supported OS / hardware | +| :------------ | :---------------------- | +| generic_x86-64_config | Generic x86, 64bit distribtion *use this as a starting point only!* | +| raspberrypi0w_config | RaspberryPi 0w, Raspbian 32bit | +| raspberrypi3_config | RaspberryPi 3, Raspbian 32bit | +| raspberrypi4_config | RaspberryPi 4, Raspbian 32bit | +| raspberrypi4_ubuntu_config | RaspberryPi 4, Ubuntu 32bit | + +### Contributed configurations -*Only 32bit-Images are supported at the moment!* +These configurations have been submitted by community contributors. + +| Configuration | Supported OS / hardware | +| :------------ | :---------------------- | +| beaglebone_black_debian_emmc_config | BeagleBone Black 4GB, Debian 32bit on internal eMMC storage | +| beaglebone_black_debian_sdcard_config | BeagleBone Black, Debian 32bit on external SD card storage | +| comfile_pi_config | Comfile Pi, Raspbian 32bit | +| rockpro64_emmc_config | RockPro64, Debian 32bit on internal eMMC storage | +| rockpro64_sd_config | BeagleBone Black, Debian 32bit on external SD card storage | + +## Example usage: Raspberry Pi 3, Raspbian 32bit + +### Prepare image and configuration The following steps give a quick start using Raspbian. For a more detailed guide, especially concerning version compatibilities, please visit [the the corresponding thread](https://hub.mender.io/t/raspberry-pi-3-model-b-b-raspbian/140) on the Mender Hub. From 5fc630e935be89370dc08c63955dd8f42f8aa6f1 Mon Sep 17 00:00:00 2001 From: Simon Ensslen Date: Mon, 14 Feb 2022 16:17:33 +0100 Subject: [PATCH 085/117] Make docker container independent from mender-convert repository Changelog: The mender-convert docker image does now contain the mender-convert application and does no longer need companion checkouts. Changelog: All input files passed to docker-mender-convert must be located within the input directory as opposed to anywhere in the mender-convert checkout directory Signed-off-by: Simon Ensslen --- .gitlab-ci.yml | 53 +++++++++++++++++++++++++++++++++------ Dockerfile | 14 +++++++++-- README.md | 15 ++++++++--- docker-build | 4 +-- docker-entrypoint.sh | 6 ++--- docker-mender-convert | 35 ++++++++++++-------------- scripts/test/run-tests.sh | 40 ++++++++++++++++------------- 7 files changed, 112 insertions(+), 55 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 10237baaa9..659532b98a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -102,14 +102,17 @@ build: - export IMAGE_NAME=$DOCKER_REPOSITORY:pr - docker load -i image.tar + - mkdir -p input + - cd input - wget -q ${RASPBIAN_URL} - unzip ${RASPBIAN_NAME}.zip + - cd .. - eval "$(curl https://raw.githubusercontent.com/mendersoftware/mendertesting/master/mender-ci-common.sh)" script: - - env MENDER_ARTIFACT_NAME=${RASPBIAN_NAME}-mender - ./docker-mender-convert --disk-image ${RASPBIAN_NAME}.img + - env MENDER_ARTIFACT_NAME=${RASPBIAN_NAME}-mender + ./docker-mender-convert --disk-image input/${RASPBIAN_NAME}.img --config configs/${RASPBERRYPI_PLATFORM}_config --config configs/images/raspberrypi_raspbian_config @@ -223,27 +226,37 @@ test_acceptance_prebuilt_raspberrypi4: test_acceptance_qemux86_64: <<: *test_acceptance script: - - ./scripts/test/run-tests.sh --config versions_override_config --only qemux86_64 + - mkdir -p input/config + - cp versions_override_config input/config/versions_override_config + - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only qemux86_64 test_acceptance_debian_qemux86_64: <<: *test_acceptance script: - - ./scripts/test/run-tests.sh --config versions_override_config --only debian-qemux86_64 + - mkdir -p input/config + - cp versions_override_config input/config/versions_override_config + - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only debian-qemux86_64 test_acceptance_raspberrypi: <<: *test_acceptance script: - - ./scripts/test/run-tests.sh --config versions_override_config --only raspberrypi3 + - mkdir -p input/config + - cp versions_override_config input/config/versions_override_config + - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only raspberrypi3 test_acceptance_beaglebone: <<: *test_acceptance script: - - ./scripts/test/run-tests.sh --config versions_override_config --only beaglebone + - mkdir -p input/config + - cp versions_override_config input/config/versions_override_config + - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only beaglebone test_acceptance_ubuntu: <<: *test_acceptance script: - - ./scripts/test/run-tests.sh --config versions_override_config --only ubuntu + - mkdir -p input/config + - cp versions_override_config input/config/versions_override_config + - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only ubuntu .template:publish:s3: stage: publish @@ -275,6 +288,23 @@ test_acceptance_ubuntu: --key ${RASPBIAN_NAME}/arm/${PUBLISH_NAME} - done +.template:publish:docker-image: + stage: publish + tags: + - docker + image: docker + services: + - name: docker:20.10.8-dind + alias: docker + command: ["--mtu=1440"] # https://gitlab.com/gitlab-com/gl-infra/production/-/issues/5590 + script: + - docker tag $DOCKER_REPOSITORY:pr $DOCKER_REPOSITORY:$MENDER_CONVERT_PUBLISH_VERSION + - docker push $DOCKER_REPOSITORY:$MENDER_CONVERT_PUBLISH_VERSION + - echo "PUBLISH_IMAGE_DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' $DOCKER_REPOSITORY:$MENDER_CONVERT_PUBLISH_VERSION)" >> publish.env + artifacts: + reports: + dotenv: publish.env + publish:s3:manual: when: manual extends: .template:publish:s3 @@ -283,3 +313,12 @@ publish:s3:automatic: rules: - if: '$PUBLISH_MENDER_CONVERT_AUTOMATIC == "true"' extends: .template:publish:s3 + +publish:docker-image:manual: + when: manual + extends: .template:publish:docker-image + +publish:docker-image:automatic: + rules: + - if: '$PUBLISH_MENDER_CONVERT_AUTOMATIC == "true"' + extends: .template:publish:docker-image diff --git a/Dockerfile b/Dockerfile index b54cc9b760..0807eb93d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -72,5 +72,15 @@ RUN wget -q -O /usr/bin/mender-artifact https://downloads.mender.io/mender-artif WORKDIR / -COPY docker-entrypoint.sh /usr/local/bin/ -ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] +COPY . /mender-convert + +RUN mkdir -p /mender-convert/work +RUN mkdir -p /mender-convert/input +RUN mkdir -p /mender-convert/deploy +RUN mkdir -p /mender-convert/logs + +VOLUME ["/mender-convert/input"] +VOLUME ["/mender-convert/deploy"] +VOLUME ["/mender-convert/logs"] + +ENTRYPOINT ["/mender-convert/docker-entrypoint.sh"] diff --git a/README.md b/README.md index 0a967a6ea3..27d8cf0141 100644 --- a/README.md +++ b/README.md @@ -152,15 +152,22 @@ necessary dependencies. Run mender-convert from inside the container with your desired options, e.g. ```bash +mkdir -p input/image +cp $PATH_TO_DISK_IMAGE/$INPUT_DISK_IMAGE input/image + +mkdir -p input/config +cp $PATH_TO_MY_OWN_CONFIG/$CUSTOM_CONFIG input/config + MENDER_ARTIFACT_NAME=release-1 ./docker-mender-convert \ - --disk-image input/$INPUT_DISK_IMAGE \ + --disk-image input/image/$INPUT_DISK_IMAGE \ --config configs/raspberrypi3_config \ + --config input/config/$CUSTOM_CONFIG \ --overlay rootfs_overlay_demo/ ``` Conversion will take 10-30 minutes, depending on image size and resources -available. You can watch `WORKDIR/convert.log` for progress and diagnostics -information. +available. You can watch `log/convert.log.XXXXX` for progress and diagnostics +information. The exact log file path is printed before the conversion starts. After it finishes, you can find your images in the `deploy` directory on your host machine! @@ -188,7 +195,7 @@ MENDER_ARTIFACT_NAME=release-1 ./mender-convert \ **NOTE!** You will be prompted to enter `sudo` password during the conversion process. This is required to be able to loopback mount images and for modifying them. Our recommendation is to use the provided Docker container, to run the -tool in a isolated environment. +tool in an isolated environment. ------------------------------------------------------------------------------- diff --git a/docker-build b/docker-build index 8aa5f25728..e05629817a 100755 --- a/docker-build +++ b/docker-build @@ -16,6 +16,6 @@ set -e -IMAGE_NAME=${IMAGE_NAME:-mender-convert} +IMAGE_NAME=${IMAGE_NAME:-"mendersoftware/mender-convert"} -eval docker build . -t ${IMAGE_NAME} "$@" +eval docker build . -t "${IMAGE_NAME}" "$*" diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 092334ddec..f4bea4e24a 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -20,9 +20,9 @@ set -e cd /mender-convert -echo "Running mender-convert "$@"" +echo "Running mender-convert $*" ./mender-convert "$@" -# Set owner and group to same as launch directory. -[ -d deploy ] && chown -R --reference=. deploy +# Set owner and group to same as input directory. +[ -d deploy ] && chown -R --reference=input deploy diff --git a/docker-mender-convert b/docker-mender-convert index a4cb80d6f5..d956ab1deb 100755 --- a/docker-mender-convert +++ b/docker-mender-convert @@ -16,38 +16,35 @@ set -e -. modules/git.sh +IMAGE_NAME=${IMAGE_NAME:-"mendersoftware/mender-convert"} +DEPLOY_DIRECTORY=${DEPLOY_DIRECTORY:-"$(pwd)/deploy"} +INPUT_DIRECTORY=${INPUT_DIRECTORY:-"$(pwd)/input"} +LOGS_DIRECTORY=${LOGS_DIRECTORY:-"$(pwd)/logs"} -IMAGE_NAME=${IMAGE_NAME:-mender-convert} +mkdir -p "$DEPLOY_DIRECTORY" +mkdir -p "$LOGS_DIRECTORY" -MENDER_CONVERT_DIR="$(pwd)" +# Create a unique log file so we can run multiple containers in parallel +LOG_FILE="convert.log.$(date +%s)-$$" -#Pass in version in-case we are added as a git submodule -MENDER_CONVERT_VERSION=$(git_mender_convert_version) - -# Create a unique work directory so we can run multiple containers in parallel -WORK_DIR=$(mktemp -d $(pwd)/work.XXXX) -WORK_SUFFIX=$(echo $WORK_DIR | awk -F. '{print $NF}') -LOG_FILE="convert.log.${WORK_SUFFIX}" +echo "using log file at: ${LOGS_DIRECTORY}/${LOG_FILE}" set +e docker run \ --rm \ - -v $MENDER_CONVERT_DIR:/mender-convert \ - -v $WORK_DIR:/mender-convert/work \ + -v "$INPUT_DIRECTORY":/mender-convert/input \ + -v "$LOGS_DIRECTORY":/mender-convert/logs \ + -v "$DEPLOY_DIRECTORY":/mender-convert/deploy \ --privileged=true \ --cap-add=SYS_MODULE \ -v /dev:/dev \ -v /lib/modules:/lib/modules:ro \ - --env MENDER_ARTIFACT_NAME=${MENDER_ARTIFACT_NAME} \ - --env MENDER_CONVERT_VERSION=${MENDER_CONVERT_VERSION} \ - --env MENDER_CONVERT_LOG_FILE=logs/${LOG_FILE} \ - $IMAGE_NAME "$@" + --env MENDER_ARTIFACT_NAME="${MENDER_ARTIFACT_NAME}" \ + --env MENDER_CONVERT_LOG_FILE=logs/"${LOG_FILE}" \ + "$IMAGE_NAME" "$@" exit_code=$? -[ ${exit_code} -eq 0 ] && rmdir ${WORK_DIR} - -echo "Log file available at: logs/${LOG_FILE}" +echo "Log file available at: ${LOGS_DIRECTORY}/${LOG_FILE}" exit ${exit_code} diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index 4453f284dc..dda3e8e52e 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -89,11 +89,13 @@ if [ -n "$PREBUILT_IMAGE" ]; then else if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "qemux86_64" ]; then - wget --progress=dot:giga -N ${UBUNTU_IMAGE_URL} -P input/ + wget --progress=dot:giga -N ${UBUNTU_IMAGE_URL} -P input/image/ + mkdir -p input/tests + cp -r "tests/ssh-public-key-overlay" "input/tests/ssh-public-key-overlay" convert_and_test "qemux86_64" \ "release-1" \ - "input/Ubuntu-Focal-x86-64.img.gz" \ - "--overlay tests/ssh-public-key-overlay" \ + "input/image/Ubuntu-Focal-x86-64.img.gz" \ + "--overlay input/tests/ssh-public-key-overlay" \ "--config configs/qemux86-64_config $EXTRA_CONFIG" \ || test_result=$? @@ -101,9 +103,9 @@ else echo >&2 "Running the uncompressed test" echo >&2 "----------------------------------------" rm -rf deploy - gunzip --force "input/Ubuntu-Focal-x86-64.img.gz" + gunzip --force "input/image/Ubuntu-Focal-x86-64.img.gz" run_convert "release-2" \ - "input/Ubuntu-Focal-x86-64.img" \ + "input/image/Ubuntu-Focal-x86-64.img" \ "--config configs/qemux86-64_config $EXTRA_CONFIG" || test_result=$? ret=0 test -f deploy/Ubuntu-Focal-x86-64-qemux86_64-mender.img || ret=$? @@ -111,11 +113,11 @@ else fi if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "raspberrypi3" ]; then - wget --progress=dot:giga -N ${RASPBIAN_IMAGE_URL} -P input/ + wget --progress=dot:giga -N ${RASPBIAN_IMAGE_URL} -P input/image/ RASPBIAN_IMAGE="${RASPBIAN_IMAGE_URL##*/}" convert_and_test "raspberrypi3" \ "release-1" \ - "input/${RASPBIAN_IMAGE}" \ + "input/image/${RASPBIAN_IMAGE}" \ "--config configs/raspberrypi3_config $EXTRA_CONFIG" \ -- \ "-k" "'not test_update'" \ @@ -123,27 +125,27 @@ else fi if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "beaglebone" ]; then - wget --progress=dot:giga -N ${BBB_DEBIAN_SDCARD_IMAGE_URL} -P input/ + wget --progress=dot:giga -N ${BBB_DEBIAN_SDCARD_IMAGE_URL} -P input/image/ BBB_DEBIAN_SDCARD_IMAGE_COMPRESSED="${BBB_DEBIAN_SDCARD_IMAGE_URL##*/}" BBB_DEBIAN_SDCARD_IMAGE_UNCOMPRESSED="${BBB_DEBIAN_SDCARD_IMAGE_COMPRESSED%.xz}" # Convert uncompressed images to speed up this job - unxz --force "input/${BBB_DEBIAN_SDCARD_IMAGE_COMPRESSED}" + unxz --force "input/image/${BBB_DEBIAN_SDCARD_IMAGE_COMPRESSED}" convert_and_test "beaglebone-sdcard" \ "release-1" \ - "input/${BBB_DEBIAN_SDCARD_IMAGE_UNCOMPRESSED}" \ + "input/image/${BBB_DEBIAN_SDCARD_IMAGE_UNCOMPRESSED}" \ "--config configs/beaglebone_black_debian_sdcard_config $EXTRA_CONFIG" \ -- \ "-k" "'not test_update'" \ || test_result=$? rm -rf deploy - wget --progress=dot:giga -N ${BBB_DEBIAN_EMMC_IMAGE_URL} -P input/ + wget --progress=dot:giga -N ${BBB_DEBIAN_EMMC_IMAGE_URL} -P input/image/ BBB_DEBIAN_EMMC_IMAGE_COMPRESSED="${BBB_DEBIAN_EMMC_IMAGE_URL##*/}" BBB_DEBIAN_EMMC_IMAGE_UNCOMPRESSED="${BBB_DEBIAN_EMMC_IMAGE_COMPRESSED%.xz}" - unxz --force "input/${BBB_DEBIAN_EMMC_IMAGE_COMPRESSED}" + unxz --force "input/image/${BBB_DEBIAN_EMMC_IMAGE_COMPRESSED}" convert_and_test "beaglebone-emmc" \ "release-1" \ - "input/${BBB_DEBIAN_EMMC_IMAGE_UNCOMPRESSED}" \ + "input/image/${BBB_DEBIAN_EMMC_IMAGE_UNCOMPRESSED}" \ "--config configs/beaglebone_black_debian_emmc_config $EXTRA_CONFIG" \ -- \ "-k" "'not test_update'" \ @@ -151,11 +153,11 @@ else fi if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "ubuntu" ]; then - wget --progress=dot:giga -N ${UBUNTU_SERVER_RPI_IMAGE_URL} -P input/ + wget --progress=dot:giga -N ${UBUNTU_SERVER_RPI_IMAGE_URL} -P input/image/ UBUNTU_SERVER_RPI_IMAGE_COMPRESSED="${UBUNTU_SERVER_RPI_IMAGE_URL##*/}" convert_and_test "raspberrypi3" \ "release-1" \ - "input/${UBUNTU_SERVER_RPI_IMAGE_COMPRESSED}" \ + "input/image/${UBUNTU_SERVER_RPI_IMAGE_COMPRESSED}" \ "--config configs/raspberrypi3_config $EXTRA_CONFIG" \ -- \ "-k" "'not test_update'" \ @@ -163,11 +165,13 @@ else fi if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "debian-qemux86_64" ]; then - wget --progress=dot:giga -N ${DEBIAN_IMAGE_URL} -P input/ + wget --progress=dot:giga -N ${DEBIAN_IMAGE_URL} -P input/image/ + mkdir -p input/tests + cp -r "tests/ssh-public-key-overlay" "input/tests/ssh-public-key-overlay" convert_and_test "qemux86_64" \ "release-1" \ - "input/Debian-11-x86-64.img.gz" \ - "--overlay tests/ssh-public-key-overlay" \ + "input/image/Debian-11-x86-64.img.gz" \ + "--overlay input/tests/ssh-public-key-overlay" \ "--config configs/debian-qemux86-64_config $EXTRA_CONFIG" \ || test_result=$? fi From 3553cff6a9be82d982c1c66ae7f29669fd19de55 Mon Sep 17 00:00:00 2001 From: mender-test-bot Date: Mon, 7 Mar 2022 03:04:48 +0000 Subject: [PATCH 086/117] Image-Bot - Image updates Changelog: None Signed-off-by: mender-test-bot --- scripts/test/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index 4453f284dc..b178fce0e2 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -21,7 +21,7 @@ BBB_DEBIAN_SDCARD_IMAGE_URL="https://debian.beagleboard.org/images/bone-debian-1 # Not on official home page, but found via https://elinux.org/Beagleboard:BeagleBoneBlack_Debian: ## Auto-update -BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2022-02-03/buster-console/bone-debian-10.11-console-armhf-2022-02-03-1gb.img.xz" +BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2022-03-02/buster-console/bone-debian-10.11-console-armhf-2022-03-02-1gb.img.xz" ## Auto-update RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2020-02-14/2020-02-13-raspbian-buster-lite.zip" From 9b8b8fb7a116a293814ed64cab9aa9daddb53301 Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Tue, 22 Feb 2022 14:08:39 +0100 Subject: [PATCH 087/117] Some cleanups to naming of test labels. * Standardize on "qemux86-64", not "qemux86_64", since the former is what is used everywhere else. * Rename qemux86_64 to ubuntu-qemux86-64, since we now have more than one qemu configuration. Changelog: None Signed-off-by: Kristian Amlie --- .gitlab-ci.yml | 4 ++-- configs/debian-qemux86-64_config | 4 ++-- ...emux86-64_config => ubuntu-qemux86-64_config} | 4 ++-- scripts/test/run-tests.sh | 16 ++++++++-------- 4 files changed, 14 insertions(+), 14 deletions(-) rename configs/{qemux86-64_config => ubuntu-qemux86-64_config} (93%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 10237baaa9..25a2349d3f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -223,12 +223,12 @@ test_acceptance_prebuilt_raspberrypi4: test_acceptance_qemux86_64: <<: *test_acceptance script: - - ./scripts/test/run-tests.sh --config versions_override_config --only qemux86_64 + - ./scripts/test/run-tests.sh --config versions_override_config --only ubuntu-qemux86-64 test_acceptance_debian_qemux86_64: <<: *test_acceptance script: - - ./scripts/test/run-tests.sh --config versions_override_config --only debian-qemux86_64 + - ./scripts/test/run-tests.sh --config versions_override_config --only debian-qemux86-64 test_acceptance_raspberrypi: <<: *test_acceptance diff --git a/configs/debian-qemux86-64_config b/configs/debian-qemux86-64_config index d26fb19cd9..5a31285a61 100644 --- a/configs/debian-qemux86-64_config +++ b/configs/debian-qemux86-64_config @@ -13,10 +13,10 @@ # # and qemu is executed with the following command: # -# qemu-system-x86_64 -enable-kvm -m 512 -smp 2 -bios /usr/share/OVMF/OVMF_CODE.fd -drive format=raw,file=deploy/Debian-11-x86-64-qemux86_64-mender.img +# qemu-system-x86_64 -enable-kvm -m 512 -smp 2 -bios /usr/share/OVMF/OVMF_CODE.fd -drive format=raw,file=deploy/Debian-11-x86-64-qemux86-64-mender.img MENDER_STORAGE_DEVICE_BASE=/dev/sda -MENDER_DEVICE_TYPE="qemux86_64" +MENDER_DEVICE_TYPE="qemux86-64" # Nothing to copy MENDER_COPY_BOOT_GAP="n" diff --git a/configs/qemux86-64_config b/configs/ubuntu-qemux86-64_config similarity index 93% rename from configs/qemux86-64_config rename to configs/ubuntu-qemux86-64_config index 2f21b14659..f7ace3a212 100644 --- a/configs/qemux86-64_config +++ b/configs/ubuntu-qemux86-64_config @@ -13,10 +13,10 @@ # # and qemu is executed with the following command: # -# qemu-system-x86_64 -enable-kvm -m 512 -smp 2 -bios /usr/share/OVMF/OVMF_CODE.fd -drive format=raw,file=deploy/Ubuntu-Focal-x86-64-qemux86_64-mender.img +# qemu-system-x86_64 -enable-kvm -m 512 -smp 2 -bios /usr/share/OVMF/OVMF_CODE.fd -drive format=raw,file=deploy/Ubuntu-Focal-x86-64-qemux86-64-mender.img MENDER_STORAGE_DEVICE_BASE=/dev/sda -MENDER_DEVICE_TYPE="qemux86_64" +MENDER_DEVICE_TYPE="qemux86-64" # Nothing to copy MENDER_COPY_BOOT_GAP="n" diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index 416b06d1e9..7a83ec392d 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -88,13 +88,13 @@ if [ -n "$PREBUILT_IMAGE" ]; then exit $test_result else - if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "qemux86_64" ]; then + if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "ubuntu-qemux86-64" ]; then wget --progress=dot:giga -N ${UBUNTU_IMAGE_URL} -P input/ - convert_and_test "qemux86_64" \ + convert_and_test "qemux86-64" \ "release-1" \ "input/Ubuntu-Focal-x86-64.img.gz" \ "--overlay tests/ssh-public-key-overlay" \ - "--config configs/qemux86-64_config $EXTRA_CONFIG" \ + "--config configs/ubuntu-qemux86-64_config $EXTRA_CONFIG" \ || test_result=$? echo >&2 "----------------------------------------" @@ -104,10 +104,10 @@ else gunzip --force "input/Ubuntu-Focal-x86-64.img.gz" run_convert "release-2" \ "input/Ubuntu-Focal-x86-64.img" \ - "--config configs/qemux86-64_config $EXTRA_CONFIG" || test_result=$? + "--config configs/ubuntu-qemux86-64_config $EXTRA_CONFIG" || test_result=$? ret=0 - test -f deploy/Ubuntu-Focal-x86-64-qemux86_64-mender.img || ret=$? - assert "${ret}" "0" "Expected uncompressed file deploy/Ubuntu-Focal-x86-64-qemux86_64-mender.img" + test -f deploy/Ubuntu-Focal-x86-64-qemux86-64-mender.img || ret=$? + assert "${ret}" "0" "Expected uncompressed file deploy/Ubuntu-Focal-x86-64-qemux86-64-mender.img" fi if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "raspberrypi3" ]; then @@ -162,9 +162,9 @@ else || test_result=$? fi - if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "debian-qemux86_64" ]; then + if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "debian-qemux86-64" ]; then wget --progress=dot:giga -N ${DEBIAN_IMAGE_URL} -P input/ - convert_and_test "qemux86_64" \ + convert_and_test "qemux86-64" \ "release-1" \ "input/Debian-11-x86-64.img.gz" \ "--overlay tests/ssh-public-key-overlay" \ From f0d96f777de5f4b3c6004db9ad70a1ca84c83ba9 Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Fri, 25 Feb 2022 15:42:41 +0100 Subject: [PATCH 088/117] Switch to detect `grub*.efi` instead of `shim*.efi`. The presence of the shim depends on whether Secure Boot is enabled or not, plus the configuration of the distro. GRUB itself however, will always be present if the distro uses GRUB with UEFI at all. So check for that instead. Of course, without the shim, Secure Boot will not work out of the box, but this is a misconfigured platform, not a problem with Mender. Cancel-Changelog: 2b805e29dcfbca1bcf9ea6339ef7755025737e56 Changelog: If `grub*.efi` preexists on the EFI partition, keep it instead of installing our own. In all other cases, we fall back to the old functionality of installing mender-grub and nuking the existing bootloader. Signed-off-by: Kristian Amlie --- mender-convert-modify | 4 ++-- modules/grub.sh | 2 +- modules/probe.sh | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/mender-convert-modify b/mender-convert-modify index 9403e8e8f6..f083d3c0e1 100755 --- a/mender-convert-modify +++ b/mender-convert-modify @@ -140,10 +140,10 @@ if [ "${MENDER_GRUB_EFI_INTEGRATION}" == "y" ]; then # Check for known U-Boot problems in all files on the boot partition. check_for_broken_uboot_uefi_support work/boot - if has_secureboot_shim "work/boot"; then + if has_grub_efi "work/boot"; then # No need to install Grub, use the one already present, and only install # our grub.cfg - grub_install_with_shim_present + grub_install_with_grub_efi_present else grub_install_mender_grub fi diff --git a/modules/grub.sh b/modules/grub.sh index 73bbb60235..f662445ad9 100644 --- a/modules/grub.sh +++ b/modules/grub.sh @@ -78,7 +78,7 @@ function grub_install_grub_editenv_binary() { # # Keep the existing boot shim, and bootloader, and only install the mender-grub # config -function grub_install_with_shim_present() { +function grub_install_with_grub_efi_present() { grub_create_grub_config diff --git a/modules/probe.sh b/modules/probe.sh index 1885fc5055..b951f73a73 100644 --- a/modules/probe.sh +++ b/modules/probe.sh @@ -345,11 +345,11 @@ is_efi_compatible_kernel() { return 0 } -# has_secureboot_shim +# has_grub_efi # -# $1 - the boot partition to search for a secureboot shim +# $1 - the boot partition to search for a grub*.efi # -# Checks the EFI/* filesystem for the presence of a signed boot shim -has_secureboot_shim() { - find "${1}" -type f -name 'shim*.efi' -print0 | grep -qz shim +# Checks the EFI/* filesystem for the presence of a GRUB bootloader +has_grub_efi() { + find "${1}" -type f -name 'grub*.efi' -print0 | grep -qz grub } From afe5056d2b25c2a89cabfc240fba9d12ea939eb1 Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Mon, 28 Feb 2022 13:39:36 +0100 Subject: [PATCH 089/117] Switch probe_arch to return x86_64 to match uname. Changelog: None Signed-off-by: Kristian Amlie --- modules/probe.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/probe.sh b/modules/probe.sh index b951f73a73..5e3517fd05 100644 --- a/modules/probe.sh +++ b/modules/probe.sh @@ -38,7 +38,7 @@ probe_arch() { target_arch="unknown" if grep -q x86-64 <<< "${file_info}"; then - target_arch="x86-64" + target_arch="x86_64" elif grep -Eq "ELF 32-bit.*ARM" <<< "${file_info}"; then target_arch="arm" elif grep -Eq "ELF 64-bit.*aarch64" <<< "${file_info}"; then @@ -57,7 +57,7 @@ probe_grub_efi_name() { local efi_name="" local -r arch=$(probe_arch) case "${arch}" in - "x86-64") + "x86_64") efi_name="grub-efi-bootx64.efi" ;; "arm") @@ -81,7 +81,7 @@ probe_debian_arch_name() { deb_arch="" arch=$(probe_arch) case "${arch}" in - "x86-64") + "x86_64") deb_arch="amd64" ;; "arm") @@ -130,7 +130,7 @@ probe_grub_efi_target_name() { local efi_target_name="" local -r arch=$(probe_arch) case "$arch" in - "x86-64") + "x86_64") efi_target_name="bootx64.efi" ;; "arm") From d680319fb084871a6ea65656ed5f36668478d452 Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Mon, 14 Mar 2022 14:28:16 +0100 Subject: [PATCH 090/117] Add new script for generating test images. Changelog: None Signed-off-by: Kristian Amlie --- configs/debian-qemux86-64_config | 3 +- configs/ubuntu-qemux86-64_config | 3 +- scripts/test/generate-image.sh | 169 +++++++++++++++++++++++++++++++ 3 files changed, 171 insertions(+), 4 deletions(-) create mode 100755 scripts/test/generate-image.sh diff --git a/configs/debian-qemux86-64_config b/configs/debian-qemux86-64_config index 5a31285a61..48c6a40323 100644 --- a/configs/debian-qemux86-64_config +++ b/configs/debian-qemux86-64_config @@ -2,8 +2,7 @@ # # The image is generated with the following command: # -# mkosi --root-size=2G --distribution=debian --release=bullseye --format=gpt_ext4 --bootable --checksum -# --password password --package=openssh-server,dhcpcd5 --output=Debian-11-x86-64.img build +# mender-convert/scripts/test/generate-image.sh debian # # Then manually uploaded to Mender's AWS S3 bucket for CI testing # diff --git a/configs/ubuntu-qemux86-64_config b/configs/ubuntu-qemux86-64_config index f7ace3a212..9d62ef05a1 100644 --- a/configs/ubuntu-qemux86-64_config +++ b/configs/ubuntu-qemux86-64_config @@ -2,8 +2,7 @@ # # The image is generated with the following command: # -# mkosi --root-size=2G --distribution=ubuntu --release=focal --format=gpt_ext4 --bootable --checksum -# --password password --package=openssh-server,dhcpcd5 --output=Ubuntu-Focal-x86-64.img build +# mender-convert/scripts/test/generate-image.sh ubuntu # # Then manually uploaded to Mender's AWS S3 bucket for CI testing # diff --git a/scripts/test/generate-image.sh b/scripts/test/generate-image.sh new file mode 100755 index 0000000000..13d4e56ac3 --- /dev/null +++ b/scripts/test/generate-image.sh @@ -0,0 +1,169 @@ +#!/bin/bash + +set -e + +usage() { + cat 1>&2 <&2 + exec sudo "$0" "$@" +fi + +while [ -n "$1" ]; do + case "$1" in + "ubuntu") + GENERATE_VARIANT=generate_ubuntu + ;; + "debian") + GENERATE_VARIANT=generate_debian + ;; + *) + usage + exit 1 + ;; + esac + shift +done + +cleanup_losetup() { + set +e + for dev in ${LO_DEVICE}p*; do + umount $dev + done + losetup -d $LO_DEVICE + rmdir tmp-p1 + rmdir tmp-p2 +} + +generate_debian() { + local -r image="Debian-11-x86-64.img" + + mkosi --root-size=2G --distribution=debian --release=bullseye --format=gpt_ext4 --bootable --checksum \ + --password password --package=openssh-server,dhcpcd5 --package grub-efi-amd64-signed \ + --package shim-signed --package lsb-release --output="$image" build + + post_process_image "$image" + + echo "Image successfully generated!" 1>&2 +} + +generate_ubuntu() { + local -r image="Ubuntu-Focal-x86-64.img" + + mkosi --root-size=2G --distribution=ubuntu --release=focal --format=gpt_ext4 --bootable --checksum \ + --password password --package=openssh-server,dhcpcd5 --package grub-efi-amd64-signed \ + --package shim-signed --package lsb-release --output="$image" build + + post_process_image "$image" + + echo "Image successfully generated!" 1>&2 +} + +post_process_image() { + local -r image="$1" + + mkdir -p tmp-p1 + mkdir -p tmp-p2 + + LO_DEVICE=$(losetup --find --show --partscan "$image") + trap cleanup_losetup EXIT + mount ${LO_DEVICE}p1 tmp-p1 + mount ${LO_DEVICE}p2 tmp-p2 + + pre_tweaks tmp-p1 tmp-p2 + + create_grub_regeneration_service tmp-p1 tmp-p2 + umount tmp-p1 + umount tmp-p2 + regenerate_grub_live "$image" + mount ${LO_DEVICE}p1 tmp-p1 + mount ${LO_DEVICE}p2 tmp-p2 + + post_tweaks tmp-p1 tmp-p2 +} + +pre_tweaks() { + local -r boot="$1" + local -r root="$2" + + # Fstab is missing for some reason. I'm not exactly sure why systemd-boot + # works without this, and GRUB doesn't. + cat > "$root/etc/fstab" < /dev/null`' > $root/etc/default/grub + fi + + sed -E -i -e 's/^#? *PermitRootLogin .*/PermitRootLogin yes/' $root/etc/ssh/sshd_config +} + +post_tweaks() { + local -r boot="$1" + local -r root="$2" + + # Delete systemd-boot, which isn't normally present in images that were + # installed with OS installers, at least not at the time of writing. + rm -rf "$boot/EFI/systemd" + + # Also replace bootx64.efi, which is the default bootloader. Mkosi installs + # systemd-bootx86.efi, but we want the shim. + rm -f "$boot/EFI/BOOT/*" + mkdir -p "$boot/EFI/BOOT" + cp "$root/usr/lib/shim/shimx64.efi.signed" "$boot/EFI/BOOT/BOOTX64.EFI" +} + +# Unfortunately installing grub scripts is something which is not really +# possible when offline. This is something which is easier with systemd-boot, so +# longterm GRUB will probably follow, or systemd-boot will take over. Anyway, +# let's do it by using a systemd service to perform the job, and then shut down. +create_grub_regeneration_service() { + local -r boot="$1" + local -r root="$2" + + cat > "$root/etc/systemd/system/mender-regenerate-grub-and-shutdown.service" <&2 + qemu-system-x86_64 \ + $maybe_kvm \ + -drive file="$image",if=ide,format=raw \ + -drive file=/usr/share/OVMF/OVMF_CODE.fd,if=pflash,format=raw,unit=0,readonly=on \ + -drive file="$nvvars",if=pflash,format=raw,unit=1 \ + -display vnc=:23 \ + -m 512 \ + || ret=$? + [ $ret -eq 0 ] && break + done + + return $ret +} + +$GENERATE_VARIANT From 47c1b9c529362c57b7cedd0a2b4074a1f6d5424b Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Mon, 14 Mar 2022 14:29:17 +0100 Subject: [PATCH 091/117] Use sudo consistently. Changelog: None Signed-off-by: Kristian Amlie --- scripts/test/test-utils.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/test/test-utils.sh b/scripts/test/test-utils.sh index 1ef608b3c8..5e83c26688 100644 --- a/scripts/test/test-utils.sh +++ b/scripts/test/test-utils.sh @@ -163,15 +163,15 @@ prepare_ssh_keys() { sudo chown -R root:root tests/ssh-public-key-overlay/root fi if [ "$(stat -c %a tests/ssh-public-key-overlay/root)" != "755" ]; then - chmod 755 tests/ssh-public-key-overlay/root + sudo chmod 755 tests/ssh-public-key-overlay/root fi if [ "$(stat -c %a tests/ssh-public-key-overlay/root/.ssh)" != "755" ]; then - chmod 700 tests/ssh-public-key-overlay/root/.ssh + sudo chmod 700 tests/ssh-public-key-overlay/root/.ssh fi if [ "$(stat -c %a tests/ssh-public-key-overlay/root/.ssh/authorized_keys)" != "755" ]; then - chmod 600 tests/ssh-public-key-overlay/root/.ssh/authorized_keys + sudo chmod 600 tests/ssh-public-key-overlay/root/.ssh/authorized_keys fi if [ "$(stat -c %a tests/ssh-priv-key/key)" != "600" ]; then - chmod 600 tests/ssh-priv-key/key + sudo chmod 600 tests/ssh-priv-key/key fi } From 245930608be356e219e5a70bff4684e52eb9f058 Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Mon, 14 Mar 2022 14:32:19 +0100 Subject: [PATCH 092/117] MEN-5219: Integrate with `grub.d` framework. This means that `grub-install` and `update-grub` no longer risk bricking the device, but will produce boot scripts with Mender support integrated. It also means that the standard GRUB menu will be available. It is supported on x86_64 platforms where `grub.d` is available, and can be turned on and off with `MENDER_GRUB_D_INTEGRATION`. The default is to use it if available. Devices that did not previously use `grub.d` integration won't be upgraded correctly with it turned on, so it is advised to set `MENDER_GRUB_D_INTEGRATION=n` if you are upgrading existing devices. Changelog: Commit Signed-off-by: Kristian Amlie --- Dockerfile | 4 +- configs/mender_grub_config | 13 +- mender-convert-modify | 16 ++- modules/grub.sh | 97 ++++++++++++--- modules/probe.sh | 11 ++ scripts/test/mender-convert-qemu | 2 +- scripts/test/test-utils.sh | 1 - tests/conftest.py | 4 +- tests/mender-image-tests | 2 +- tests/test_grub_integration.py | 199 +++++++++++++++++++++++++++++++ 10 files changed, 323 insertions(+), 26 deletions(-) create mode 100644 tests/test_grub_integration.py diff --git a/Dockerfile b/Dockerfile index b54cc9b760..6e77706403 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,7 +53,9 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install -y \ # manipulate binary and hex xxd \ # JSON power tool - jq + jq \ +# GRUB command line tools, primarily grub-probe + grub-common COPY --from=build /root/pxz/pxz /usr/bin/pxz diff --git a/configs/mender_grub_config b/configs/mender_grub_config index 69382ca11b..c14c513e54 100644 --- a/configs/mender_grub_config +++ b/configs/mender_grub_config @@ -1,8 +1,17 @@ -# Use U-boot -> GRUB -> EFI boot-loader integration +# Use UEFI -> GRUB boot-loader integration +# +# On ARM, U-boot is used as UEFI (EFI) provider # # Only disable this if you know what you are doing MENDER_GRUB_EFI_INTEGRATION=y +# Integrate with /etc/grub.d boot scripts. +# +# This has no effect unless MENDER_GRUB_EFI_INTEGRATION is enabled. Default is +# auto, in which case it will be detected. This is only supported for x86_64 +# platforms at the moment. +MENDER_GRUB_D_INTEGRATION=auto + # Version of GRUB to use. Note that there needs to be a precompiled version # available at the MENDER_STORAGE_URL download source. GRUB_VERSION=2.04 @@ -17,7 +26,7 @@ GRUB_VERSION=2.04 MENDER_GRUB_KERNEL_BOOT_ARGS="" # grub-mender-grubenv is the Mender integration for the GRUB bootloader -MENDER_GRUBENV_VERSION="1a7db967495bbe8be53b7a69dcb42822f39d9a74" +MENDER_GRUBENV_VERSION="b06a8e2cf13776b5cfc896fa8068006dd9992ebb" MENDER_GRUBENV_URL="${MENDER_GITHUB_ORG}/grub-mender-grubenv/archive/${MENDER_GRUBENV_VERSION}.tar.gz" # Name of the storage device containing root filesystem partitions in GRUB diff --git a/mender-convert-modify b/mender-convert-modify index f083d3c0e1..f6ad9b1008 100755 --- a/mender-convert-modify +++ b/mender-convert-modify @@ -142,12 +142,24 @@ if [ "${MENDER_GRUB_EFI_INTEGRATION}" == "y" ]; then if has_grub_efi "work/boot"; then # No need to install Grub, use the one already present, and only install - # our grub.cfg - grub_install_with_grub_efi_present + # our tools. + log_info "GRUB EFI bootloader already present, not installing one." + grub_install_grub_editenv_binary else + log_info "Installing GRUB EFI bootloader..." grub_install_mender_grub fi + if [ "$MENDER_GRUB_D_INTEGRATION" = y ] || ( [ "$MENDER_GRUB_D_INTEGRATION" = auto ] && supports_grub_d "work/rootfs" ); then + log_info "Generating grub config using update-grub..." + grub_create_grub_config + grub_install_grub_d_config + else + log_info "Generating the mender-grub config..." + grub_create_grub_config + grub_install_standalone_grub_config + fi + fi run_and_log_cmd "sudo mkdir -p work/rootfs/data/mender" diff --git a/modules/grub.sh b/modules/grub.sh index f662445ad9..a19b659361 100644 --- a/modules/grub.sh +++ b/modules/grub.sh @@ -45,7 +45,12 @@ EOF mender_kernel_root_base=${MENDER_STORAGE_DEVICE_BASE} EOF fi +} +# grub_install_standalone_grub_config +# +# +function grub_install_standalone_grub_config() { if [ -n "${MENDER_GRUB_KERNEL_BOOT_ARGS}" ]; then cat <<- EOF > work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION}/11_bootargs_grub.cfg set bootargs="${MENDER_GRUB_KERNEL_BOOT_ARGS}" @@ -55,10 +60,83 @@ EOF ( cd work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION} run_and_log_cmd "make 2>&1" - run_and_log_cmd "sudo make DESTDIR=../ BOOT_DIR=boot install-boot-files" - run_and_log_cmd "sudo make DESTDIR=../rootfs install-tools" + run_and_log_cmd "sudo make DESTDIR=$PWD/../ BOOT_DIR=boot install-standalone-boot-files" + run_and_log_cmd "sudo make DESTDIR=$PWD/../rootfs install-tools" + ) + +} + +# grub_install_grub_d_config +# +# +function grub_install_grub_d_config() { + if [ -n "${MENDER_GRUB_KERNEL_BOOT_ARGS}" ]; then + log_warn "MENDER_GRUB_KERNEL_BOOT_ARGS is ignored when MENDER_GRUB_D_INTEGRATION is enabled. Set it in the GRUB configuration instead." + fi + + # When using grub.d integration, /boot/efi must point to the boot partition, + # and /boot/grub must point to grub-mender-grubenv on the boot partition. + if [ ! -d work/rootfs/boot/efi ]; then + run_and_log_cmd "sudo mkdir work/rootfs/boot/efi" + fi + run_and_log_cmd "sudo mkdir work/boot/grub-mender-grubenv" + run_and_log_cmd "sudo mv work/rootfs/boot/grub/* work/boot/grub-mender-grubenv/" + run_and_log_cmd "sudo rmdir work/rootfs/boot/grub" + run_and_log_cmd "sudo ln -s efi/grub-mender-grubenv work/rootfs/boot/grub" + + ( + cd work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION} + run_and_log_cmd "make 2>&1" + run_and_log_cmd "sudo make DESTDIR=$PWD/../ BOOT_DIR=boot install-boot-env" + run_and_log_cmd "sudo make DESTDIR=$PWD/../rootfs install-grub.d-boot-scripts" + run_and_log_cmd "sudo make DESTDIR=$PWD/../rootfs install-tools" + # We need this for running the scripts once. + run_and_log_cmd "sudo make DESTDIR=$PWD/../rootfs install-offline-files" ) + # Mender-convert usually runs in a container. It's difficult to launch + # additional containers from within an existing one, but we need to run + # `update-grub` on a simulated device using some sort of container. Use good + # old `chroot`, which doesn't provide perfect containment, but it is good + # enough for our purposes, and doesn't require special containment + # capabilities. This will not work for foreign architectures, but we could + # probably use something like qemu-aarch64-static to get around that. + run_and_log_cmd "sudo mount work/boot work/rootfs/boot/efi -o bind" + run_and_log_cmd "sudo mount /dev work/rootfs/dev -o bind,ro" + run_and_log_cmd "sudo mount /proc work/rootfs/proc -o bind,ro" + run_and_log_cmd "sudo mount /sys work/rootfs/sys -o bind,ro" + + local ret=0 + + # Use `--no-nvram`, since we cannot update firmware memory in an offline + # build. Instead, use `--removable`, which creates entries that automate + # booting if you put the image into a new device, which you almost certainly + # will after using mender-convert. + run_and_log_cmd_noexit "sudo chroot work/rootfs grub-install --removable --no-nvram" || ret=$? + if [ $ret -eq 0 ]; then + run_and_log_cmd_noexit "sudo chroot work/rootfs grub-install --no-nvram" || ret=$? + fi + + if [ $ret -eq 0 ]; then + run_and_log_cmd_noexit "sudo chroot work/rootfs update-grub" || ret=$? + fi + + # Very important that these are unmounted, otherwise Docker may start to + # remove files inside them while tearing down the container. You can guess + # how I found that out... We run without the logger because otherwise the + # message from the previous command, which is the important one, is lost. + sudo umount -l work/rootfs/boot/efi || true + sudo umount -l work/rootfs/dev || true + sudo umount -l work/rootfs/proc || true + sudo umount -l work/rootfs/sys || true + + [ $ret -ne 0 ] && exit $ret + + ( + cd work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION} + # Should be removed after running. + run_and_log_cmd "sudo make DESTDIR=$PWD/../rootfs uninstall-offline-files" + ) } # grub_install_grub_editenv_binary @@ -74,18 +152,6 @@ function grub_install_grub_editenv_binary() { } -# grub_install_with_shim_present -# -# Keep the existing boot shim, and bootloader, and only install the mender-grub -# config -function grub_install_with_grub_efi_present() { - - grub_create_grub_config - - grub_install_grub_editenv_binary - -} - # grub_install_mender_grub # # Install mender-grub on the converted boot partition @@ -98,9 +164,6 @@ function grub_install_mender_grub() { run_and_log_cmd "sudo ln -s ${initrd_imagetype} work/rootfs/boot/initrd" fi - log_info "Generating the mender-grub config..." - grub_create_grub_config - # Remove conflicting boot files. These files do not necessarily effect the # functionality, but lets get rid of them to avoid confusion. # diff --git a/modules/probe.sh b/modules/probe.sh index 5e3517fd05..0491fad365 100644 --- a/modules/probe.sh +++ b/modules/probe.sh @@ -353,3 +353,14 @@ is_efi_compatible_kernel() { has_grub_efi() { find "${1}" -type f -name 'grub*.efi' -print0 | grep -qz grub } + +supports_grub_d() { + test -d "$1"/etc/grub.d || return 1 + + # Because we are executing programs inside a chroot in the image, we cannot + # currently convert non-native architectures to use grub.d integration. See + # relevant section on chroot inside grub_install_grub_d_config. + [ "$(probe_arch)" = "$(uname -m)" ] || return 1 + + return 0 +} diff --git a/scripts/test/mender-convert-qemu b/scripts/test/mender-convert-qemu index 80a9b08acc..7e8cd6c97b 100755 --- a/scripts/test/mender-convert-qemu +++ b/scripts/test/mender-convert-qemu @@ -17,7 +17,7 @@ fi qemu-system-x86_64 \ -enable-kvm \ -nographic \ - -m 256 \ + -m 512 \ -net user,hostfwd=tcp::8822-:22 \ -net nic,macaddr=52:54:00$(od -txC -An -N3 /dev/urandom|tr \ :) \ -bios ${ovmf_file} \ diff --git a/scripts/test/test-utils.sh b/scripts/test/test-utils.sh index 5e83c26688..8e23583962 100644 --- a/scripts/test/test-utils.sh +++ b/scripts/test/test-utils.sh @@ -140,7 +140,6 @@ run_tests() { --sdimg-location="${MENDER_CONVERT_DIR}/deploy" \ --ssh-priv-key="./ssh-priv-key/key" \ --qemu-wrapper="../scripts/test/mender-convert-qemu" \ - mender-image-tests \ ${pytest_extra_args} exitcode=$? diff --git a/tests/conftest.py b/tests/conftest.py index 5344b09056..c00bfff55f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,4 @@ -# Copyright 2021 Northern.tech AS +# Copyright 2022 Northern.tech AS # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,3 +22,5 @@ # Load the parser for our custom option flags pytest_plugins = "utils.parseropts.parseropts" + +from utils.fixtures import * diff --git a/tests/mender-image-tests b/tests/mender-image-tests index ee1266afe8..8af131b9ef 160000 --- a/tests/mender-image-tests +++ b/tests/mender-image-tests @@ -1 +1 @@ -Subproject commit ee1266afe8ec58efce9b4223e2489ac023b0582f +Subproject commit 8af131b9eface1555d2141d513c0dd6860c7815f diff --git a/tests/test_grub_integration.py b/tests/test_grub_integration.py new file mode 100644 index 0000000000..4eaaa56f9f --- /dev/null +++ b/tests/test_grub_integration.py @@ -0,0 +1,199 @@ +#!/usr/bin/python +# Copyright 2022 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import re +import os +import subprocess + +from utils.common import ( + extract_partition, + get_no_sftp, +) + + +@pytest.fixture(scope="function") +def cleanup_boot_scripts(request, connection): + """Take a backup of the various grub.cfg files and restore them after the + test. This is recommended for tests that call `grub-install` and/or + `update-grub`, so that other tests can also run them from a pristine + state.""" + + connection.run( + "cp $(find /boot/efi/EFI/ -name grub.cfg -not -path '*/EFI/BOOT/*') /data/grub-efi.cfg" + ) + connection.run("cp /boot/grub/grub.cfg /data/grub-main.cfg") + connection.run("cp /boot/grub-mender-grubenv.cfg /data/grub-mender-grubenv.cfg") + + def cleanup(): + connection.run( + "mv /data/grub-efi.cfg $(find /boot/efi/EFI/ -name grub.cfg -not -path '*/EFI/BOOT/*')" + ) + connection.run("mv /data/grub-main.cfg /boot/grub/grub.cfg") + connection.run("mv /data/grub-mender-grubenv.cfg /boot/grub-mender-grubenv.cfg") + + request.addfinalizer(cleanup) + + +def check_all_root_occurrences_valid(grub_cfg): + found_expected = False + inside_10_header = False + # One of the functions we define and use. + expected = "mender_check_and_restore_env" + with open(grub_cfg) as fd: + lineno = 0 + for line in fd.readlines(): + lineno += 1 + if re.match(r'^\s*root="\$\{mender_grub_storage_device\}', line): + continue + + if line.strip() == "### BEGIN /etc/grub.d/00_header ###": + # We allow root references inside the 00_header, because they + # are overriden by Mender later. + inside_10_header = True + elif line.strip() == "### END /etc/grub.d/00_header ###": + inside_10_header = False + if not inside_10_header and re.match(r"^\s*(set +)?root=", line): + pytest.fail( + "Found unexpected occurrence of `root=` in grub boot script\n" + "%d:%s" % (lineno, line) + ) + + if line.find(expected) >= 0: + found_expected = True + + assert found_expected, "Expected content (%s) not found" % expected + + +@pytest.mark.usefixtures("setup_board", "cleanup_boot_scripts") +class TestGrubIntegration: + @pytest.mark.min_mender_version("1.0.0") + def test_no_root_occurrences(self, connection, latest_part_image): + """Test that the generated grub scripts do not contain any occurrences of + `root=` except for known instances that we control. This is + important because Mender needs to keep tight control of when this + variable is set, in order to boot from, and mount, the correct root + partition.""" + + # First, check that the offline generated scripts don't have any. + extract_partition(latest_part_image, 1) + try: + subprocess.check_call( + ["mcopy", "-i", "img1.fs", "::/grub-mender-grubenv/grub.cfg", "."] + ) + check_all_root_occurrences_valid("grub.cfg") + finally: + os.remove("img1.fs") + os.remove("grub.cfg") + + extract_partition(latest_part_image, 2) + try: + subprocess.check_call( + [ + "debugfs", + "-R", + "dump -p /boot/grub-mender-grubenv.cfg grub-mender-grubenv.cfg", + "img2.fs", + ] + ) + check_all_root_occurrences_valid("grub-mender-grubenv.cfg") + finally: + os.remove("img2.fs") + os.remove("grub-mender-grubenv.cfg") + + # Then, check that the runtime generated scripts don't have any. + get_no_sftp("/boot/grub/grub.cfg", connection) + try: + check_all_root_occurrences_valid("grub.cfg") + finally: + os.remove("grub.cfg") + + get_no_sftp("/boot/grub-mender-grubenv.cfg", connection) + try: + check_all_root_occurrences_valid("grub-mender-grubenv.cfg") + finally: + os.remove("grub-mender-grubenv.cfg") + + # Check again after running `update-grub`. + connection.run("grub-install && update-grub") + get_no_sftp("/boot/grub/grub.cfg", connection) + try: + check_all_root_occurrences_valid("grub.cfg") + finally: + os.remove("grub.cfg") + + get_no_sftp("/boot/grub-mender-grubenv.cfg", connection) + try: + check_all_root_occurrences_valid("grub-mender-grubenv.cfg") + finally: + os.remove("grub-mender-grubenv.cfg") + + @pytest.mark.min_mender_version("1.0.0") + def test_offline_and_runtime_boot_scripts_identical(self, connection): + # Update scripts at runtime. + connection.run("grub-install && update-grub") + + # Take advantage of the copies already made by cleanup_boot_scripts + # fixture above, and use the copies in /data. + + # Take into account some known, but harmless differences. The "hd0,gpt1" + # style location is missing from the offline generated grub-efi.cfg + # file, but it is harmless because the filesystem UUID is being used + # instead. + connection.run( + r"sed -Ee 's/ *hd[0-9]+,gpt[0-9]+//' " + "$(find /boot/efi/EFI/ -name grub.cfg -not -path '*/EFI/BOOT/*') " + "> /data/new-grub-efi-modified.cfg" + ) + try: + connection.run("diff -u /data/grub-efi.cfg /data/new-grub-efi-modified.cfg") + finally: + connection.run("rm -f /data/new-grub-efi-modified.cfg") + + # Another few differences we work around in the main grub files: + # * `--hint` parameters are not generated in offline copy. + # * `root` variable is not set in offline copy. + # * `fwsetup` is added somewhat randomly depending on availability both + # on build host and device. + try: + connection.run("cp /data/grub-main.cfg /data/old-grub-modified.cfg") + connection.run("cp /boot/grub/grub.cfg /data/new-grub-modified.cfg") + connection.run( + r"sed -i -En -e '/\bsearch\b/{s/ --hint[^ ]*//g;}' " + "-e \"/^set root='hd0,gpt1'$/d\" " + r"-e '\,### BEGIN /etc/grub.d/30_uefi-firmware ###,{p; n; :loop; \,### END /etc/grub.d/30_uefi-firmware ###,b end; n; b loop; :end;}' " + "-e p " + "/data/old-grub-modified.cfg /data/new-grub-modified.cfg" + ) + connection.run("diff -u /data/old-grub-modified.cfg /data/new-grub-modified.cfg") + finally: + connection.run("rm -f /data/old-grub-modified.cfg /data/new-grub-modified.cfg") + + # Same differences as in previous check. + try: + connection.run("cp /data/grub-mender-grubenv.cfg /data/old-grub-mender-grubenv-modified.cfg") + connection.run("cp /boot/grub-mender-grubenv.cfg /data/new-grub-mender-grubenv-modified.cfg") + connection.run( + r"sed -i -En -e '/\bsearch\b/{s/ --hint[^ ]*//g;}' " + "-e \"/^set root='hd0,gpt1'$/d\" " + r"-e '\,### BEGIN /etc/grub.d/30_uefi-firmware ###,{p; n; :loop; \,### END /etc/grub.d/30_uefi-firmware ###,b end; n; b loop; :end;}' " + "-e p " + "/data/old-grub-mender-grubenv-modified.cfg /data/new-grub-mender-grubenv-modified.cfg" + ) + connection.run( + "diff -u /data/old-grub-mender-grubenv-modified.cfg /data/new-grub-mender-grubenv-modified.cfg" + ) + finally: + connection.run("rm -f /data/old-grub-mender-grubenv-modified.cfg /data/new-grub-mender-grubenv-modified.cfg") From edb4e115ce8207a3928543ecec77e92812facc1c Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Tue, 15 Mar 2022 08:56:44 +0100 Subject: [PATCH 093/117] Do not remove the boot directory, if it happens to end up empty. Changelog: None Signed-off-by: Kristian Amlie --- modules/grub.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/grub.sh b/modules/grub.sh index a19b659361..904f96f93d 100644 --- a/modules/grub.sh +++ b/modules/grub.sh @@ -173,7 +173,7 @@ function grub_install_mender_grub() { sudo rm -rf work/boot/EFI/systemd sudo rm -rf work/boot/NvVars for empty_dir in $( - cd work/boot && find . -maxdepth 1 -type d -empty + cd work/boot && find . -maxdepth 1 -type d -empty -not -name . ); do sudo rmdir work/boot/$empty_dir done From bb8ec8044f5181e946a7fb8977fb6d0f4a80641d Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Tue, 15 Mar 2022 09:15:53 +0100 Subject: [PATCH 094/117] Automatically skip non-qemu board tests for mender-convert. Changelog: None Signed-off-by: Kristian Amlie --- scripts/test/run-tests.sh | 9 --------- tests/mender-image-tests | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index 7a83ec392d..377c9c4aa4 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -83,7 +83,6 @@ test_result=0 if [ -n "$PREBUILT_IMAGE" ]; then run_tests $PREBUILT_IMAGE \ - "-k" "'not test_update'" \ || test_result=$? exit $test_result @@ -117,8 +116,6 @@ else "release-1" \ "input/${RASPBIAN_IMAGE}" \ "--config configs/raspberrypi3_config $EXTRA_CONFIG" \ - -- \ - "-k" "'not test_update'" \ || test_result=$? fi @@ -132,8 +129,6 @@ else "release-1" \ "input/${BBB_DEBIAN_SDCARD_IMAGE_UNCOMPRESSED}" \ "--config configs/beaglebone_black_debian_sdcard_config $EXTRA_CONFIG" \ - -- \ - "-k" "'not test_update'" \ || test_result=$? rm -rf deploy @@ -145,8 +140,6 @@ else "release-1" \ "input/${BBB_DEBIAN_EMMC_IMAGE_UNCOMPRESSED}" \ "--config configs/beaglebone_black_debian_emmc_config $EXTRA_CONFIG" \ - -- \ - "-k" "'not test_update'" \ || test_result=$? fi @@ -157,8 +150,6 @@ else "release-1" \ "input/${UBUNTU_SERVER_RPI_IMAGE_COMPRESSED}" \ "--config configs/raspberrypi3_config $EXTRA_CONFIG" \ - -- \ - "-k" "'not test_update'" \ || test_result=$? fi diff --git a/tests/mender-image-tests b/tests/mender-image-tests index 8af131b9ef..626c9f43f6 160000 --- a/tests/mender-image-tests +++ b/tests/mender-image-tests @@ -1 +1 @@ -Subproject commit 8af131b9eface1555d2141d513c0dd6860c7815f +Subproject commit 626c9f43f615531d4ec45c2e388b8829d91e087c From 35a25ed2dcc97ef34f79624422308e8c744dead5 Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Tue, 15 Mar 2022 09:59:56 +0100 Subject: [PATCH 095/117] Upgrade Ubuntu Raspberry Pi test image. Changelog: None Signed-off-by: Kristian Amlie --- scripts/test/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index 377c9c4aa4..42d164327b 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -31,7 +31,7 @@ UBUNTU_IMAGE_URL="https://downloads.mender.io/mender-convert/images/Ubuntu-Focal DEBIAN_IMAGE_URL="https://downloads.mender.io/mender-convert/images/Debian-11-x86-64.img.gz" ## Auto-update -UBUNTU_SERVER_RPI_IMAGE_URL="http://cdimage.ubuntu.com/ubuntu/releases/20.04/release/ubuntu-20.04.3-preinstalled-server-armhf+raspi.img.xz" +UBUNTU_SERVER_RPI_IMAGE_URL="http://cdimage.ubuntu.com/ubuntu/releases/20.04/release/ubuntu-20.04.4-preinstalled-server-armhf+raspi.img.xz" # Keep common function declarations in separate utils script UTILS_PATH=${0/$(basename $0)/test-utils.sh} From 1b3ab7c5e0692eaa8e192a7dabe80c5a26c482f0 Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Tue, 15 Mar 2022 13:04:15 +0100 Subject: [PATCH 096/117] MEN-5255: Enable and start testing Secure Boot. This is using pre-signed binaries by Microsoft, Canonical and Debian, no customized signing. Changelog: None Signed-off-by: Kristian Amlie --- scripts/test/mender-convert-qemu | 3 +- tests/mender-image-tests | 2 +- .../MicCorThiParMarRoo_2010-10-05.crt | Bin 0 -> 1539 bytes tests/uefi-nvram/OVMF_VARS.fd | Bin 0 -> 131072 bytes tests/uefi-nvram/README.md | 71 ++++++++++++++++++ 5 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 tests/uefi-nvram/MicCorThiParMarRoo_2010-10-05.crt create mode 100644 tests/uefi-nvram/OVMF_VARS.fd create mode 100644 tests/uefi-nvram/README.md diff --git a/scripts/test/mender-convert-qemu b/scripts/test/mender-convert-qemu index 7e8cd6c97b..e9956fbd34 100755 --- a/scripts/test/mender-convert-qemu +++ b/scripts/test/mender-convert-qemu @@ -20,7 +20,8 @@ qemu-system-x86_64 \ -m 512 \ -net user,hostfwd=tcp::8822-:22 \ -net nic,macaddr=52:54:00$(od -txC -An -N3 /dev/urandom|tr \ :) \ - -bios ${ovmf_file} \ + -drive file=${ovmf_file},if=pflash,format=raw,unit=0,readonly=on \ + -drive file=./uefi-nvram/OVMF_VARS.fd,if=pflash,format=raw,unit=1,readonly=on \ -drive format=raw,file=${DISK_IMG} & qemu_pid=$! diff --git a/tests/mender-image-tests b/tests/mender-image-tests index 626c9f43f6..7c081c042f 160000 --- a/tests/mender-image-tests +++ b/tests/mender-image-tests @@ -1 +1 @@ -Subproject commit 626c9f43f615531d4ec45c2e388b8829d91e087c +Subproject commit 7c081c042f0024e87e9e15144b18d991fb378bcd diff --git a/tests/uefi-nvram/MicCorThiParMarRoo_2010-10-05.crt b/tests/uefi-nvram/MicCorThiParMarRoo_2010-10-05.crt new file mode 100644 index 0000000000000000000000000000000000000000..d6a50b817ab7263155aa5f49f293d87a4e696fa2 GIT binary patch literal 1539 zcmXqLV*PK>#Qb~#GZP~dlYsHmIg$Y$g%h1D?iXL)GRxM0myJ`a&7Zl;k>SJIxF!cy-3*n;)AzVuw4CwSX)U{0XGL$Ta@p1IRjO;i(O zcAPCH@@~eCXz$A(^>5c1{Ha`)A>`FoZu91;@!|Wg8bc-5a>S)xzPEmFT#2w+^PF=L zVIB{CiI^zw`0h3#>06?w&PETqrbTSxWJgAHiXtWFPLnd2Z4p_2}9o zVRK%G|KXkYWKmV*%)QD-QgbHca%_rgm&#jy@xv=tqt*(x0MBVNIyJ(wc{M9d(|07z zUwr<3Yx0kMVt3w7VX4VYLts3%+ijEIxmhFP}!q zsx_Scd>s4#Mea{~_vmw=jCh}&U39%l#bTK+rw(0{{QKhdTmLCb<4umtc@(nCqq#f9 z<+78tXx1^V3;`7_rF!oTmDlb)eJn0HVFsJk>&V7seqW|3+b`-pQ+7hqZ*NAZ&5ghP zEiRUwz57znU&YO^u2 zFgBTsGJ;HCKu&bP{0vNVj12md!e>_9J-H;|k=asrMtykLIw*A6idm9&gbKITMwf)hJebOd>tqKI6 zulT*|XR*sEE8l&Oi&pVTE;HP1QdfVdruK4P^KGu#hrK8F--~{4knfgec1^Q+MbwT2 zo+mbZn_LQHYGSkvOA0OGcb#1!^UJkslKjO~fugA|{0%hrJo@IISZgh3rmDp&=5}&@ zw7`Od^#xO(F~@&la@us}|Ejk8Y9}n(b{zPh>!Wn^vB{G(&I?gbqIFg$pLkm+W^&s{ z=2ww}w2^sZn$q!=(f0SBGE6xl_)WT-b3^k9=}y;u^VhT0xBvB=#cuHN%c17EyH2vi zHCqNpGricg>AG@h{L0F28WXRrFixJa(m1cd>Dwt&iJ%QysmW?RFS`9RCV1U`(&84K z(6WE;o%vJcYCNRdo30nfKD6zRP}&sqM225guHdXy$dqL>PR!iTBg}a9hO6sDPgAd$ zO|dig*?a$1`Etu-?RH1s13xxMc(Lkv|M+!&qm#rc!AgHVukS}Rrs#3?l*~0^nPs}W pF+=^=!e8yV8*eV>d4KM}x;(Qt2l_wzDg0gb_*-tGv8T#8JpkSSbU*+A literal 0 HcmV?d00001 diff --git a/tests/uefi-nvram/OVMF_VARS.fd b/tests/uefi-nvram/OVMF_VARS.fd new file mode 100644 index 0000000000000000000000000000000000000000..fbb72b7206521efb08fc387c00ec2869e8fdd699 GIT binary patch literal 131072 zcmeI&2V7J~9tZH*0@8~hiXb6aKqO|VDxiXbQlyB03N}JlSa3ljMxzuViWRYd1xsus zis96F8quh^pfQ%1SYo0E(RhmOG{l%7_n%$KsoBNN$C}G;_`KbHZ)fMtd}rRwym>J8 zXqeUIR^!6)g#$KBv+ds^IwX`ap*+IKevy8S_r=VQng2ougx3715r0vH2@bc*-zhHb zyK-t*)8Kt?7i~YsSk!&RcjXbDOw<%n#k(0D7tp3xH-3*@dP(0Eva#2P`4pD*W#d=| z%VcRRl?Bmj9E;~EG?qv7X14rOGr@qq6VoQWSatflai{&aoZRa2x?$|ZSV}`iX~wa) z9>i#*^gLPnX42wcwWW>c@qBH@+`3WHz!K2_8_!OLo{^NElu}EhR0HXA7UejYjpwoD z`D{U-nhRR=CC{g^SNc_<>2c%PZ?hE#l4Tx2ES&YF(AQCrERIO>*-*C%h6kC!c!rLY)EDM4PNscZo4Nm(MrjF%FdT5dWNo7d^% zX+NA+qMCKk)b1`%`Gs=)UUTZ5i(iZWv_0;jI9JGnnp1T)fd)Un z$iRfZF}9aq5sYaG1R@<%HmLun=c|7C%D7aI&9&|QX^=cq>Rd0p{9|ZSo+q`+FA%E~ zdOmqp<)~-{FUOJRsYs>cv&~bp;JjJmYnhCX}|v`=F*Y6{7|3(Rp5NXidhK^`ly_rq>j@x96)f zS@eynyKOPu<}=3UlffTqWgC|b4Vq?9(>BauXr+2%0xvU9LRJ1AU566TnvQzo7%z8t zGwCYI?L;w}E4MY(Y;$GjzkRy0^YuLS%Pvy8&PO+suA=PD6ocpA>Y+LxNVOkC`_{~d zMbh>L3!|05Ale^DuT3)xXWIXx3N5e*wB#u&6H({>nYRVLc=TsH6l+RPUJvB$A*GqL z6=idw7^+SDXtgqoRD9_A7(}bLkE%E}qe96(WjoTFZFq_pF&6wGMAue|D!nPihoKw}`0!)u8@J{XS+BokLZ+t)i#b>CQ^5+&(TJcVywmFC;(SC~j z>R0uYA$N^F>`8_%nlK`P#Xn&$FjAv}xnx0T1nI<+dkhf|7U%HKv#oFvgbMKO0+da5n@dfVZ8i)r^Lj@TZg5kWlF4C>pMyuB@$OBCyA3& z_o&wT&aMx3dpJ8sy^?v(5!yUVOHUz0IYL7w$r0)ZbA$qRbK{5{FO4^nIt=~dtj^`U zLw^7CXt%jAtfN6o)BQ%R-1Dlk&yTyb!{Bbrg)!|8FS9yVX6NkoZLU-G%9YoK2Gq

o3EtuFiFKOopr}9L-YC`ll`|pV!Gu)S6yF^b`JZ+_>^yzAsKz#W^8U!BR|o&5zr1$U#Gxg-+V77~nVYIrIijGg zbltI=S2UcaP0$SWU$}U>UE~=3&RMRB+utf%edNZp*t=C`-(4#fPfCq3Tr$DMDrdm= z_Lg%}r*8XxqQ+}^*G@&8G`?Cezjaw@kbzxh`G?ve23ouC58a(`z2^2n%ht1d^@^U- zVZv(5I|uikFmJeg^;$^rnvpL1m)3;u@XIZX^Euwfqt)nt=_DC-u(z2KxH;>@*^3ui zo6lXW>3(%+_S)b(3)}ZzHS5s$YV+V-NfDl>8s_Br^q3xG6tkf1<)w!zZUuGUlw%od zxjX5K?<%)#?~N8ajQJwr>(9Nl^*7$$yk&9AjAYx{qb_OIU7wLAxj6hxEf05_(d#8fn%Z471!6%?w^jn40(-d8(e+G?PQczQ8eB5* z`vdQfs&QM>-@B!uSKdlLysoZR zXOvggj)HBwO@5k{y`tWGXWWdBYfe>JxHPz@8~?ce*B!rP_#AW(s=6R6H!xr8xYOm` zDSIbPJ}%8YqqAgR;QTpfqi;yk`X#uX=$yMgYWpa?T2F&YpLEMf!@4+Tj_p2j$Cv9Z z@A=LsdhJ-ek!-=`5Q*Ivy0c+Gxe7iu47SL?iDA`&1rkEYCmJ zsk(dq_C1ZM18hFK;8L5QeKe{z+TnxP>T6@oT+R%zyeIRraO#$wVDtHg=-%fpvf};5 z^%jNNn{%rzru$ZvRcKBrXz(v>A-Q#DZ|<@k2gDGds!l-pYFgsUMwQr+4G zx;=Al2U|C+y-=STwpKbH;@pIkPH?=AveM|N; zBkcq3K7(l7j-@BgGJ0>CH~P{z9z*{Rl0wJmSMs^&vmJkVo?p Date: Fri, 18 Mar 2022 11:10:07 +0100 Subject: [PATCH 097/117] Fix empty variable when pushing Docker container. Changelog: None Signed-off-by: Kristian Amlie --- .gitlab-ci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f51a319786..9c2b5c7933 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -298,9 +298,11 @@ test_acceptance_ubuntu: alias: docker command: ["--mtu=1440"] # https://gitlab.com/gitlab-com/gl-infra/production/-/issues/5590 script: - - docker tag $DOCKER_REPOSITORY:pr $DOCKER_REPOSITORY:$MENDER_CONVERT_PUBLISH_VERSION - - docker push $DOCKER_REPOSITORY:$MENDER_CONVERT_PUBLISH_VERSION - - echo "PUBLISH_IMAGE_DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' $DOCKER_REPOSITORY:$MENDER_CONVERT_PUBLISH_VERSION)" >> publish.env + - IMAGE_VERSION=${MENDER_CONVERT_PUBLISH_VERSION:-${CI_COMMIT_REF_NAME}} + + - docker tag $DOCKER_REPOSITORY:pr $DOCKER_REPOSITORY:$IMAGE_VERSION + - docker push $DOCKER_REPOSITORY:$IMAGE_VERSION + - echo "PUBLISH_IMAGE_DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' $DOCKER_REPOSITORY:$IMAGE_VERSION)" >> publish.env artifacts: reports: dotenv: publish.env From 6abc1187837660b9c0268bf2a910b6eca32d62a1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Mar 2022 07:03:17 +0000 Subject: [PATCH 098/117] Changelog:All: Bump tests/mender-image-tests from `7c081c0` to `6d3e3ad` Bumps [tests/mender-image-tests](https://github.com/mendersoftware/mender-image-tests) from `7c081c0` to `6d3e3ad`. - [Release notes](https://github.com/mendersoftware/mender-image-tests/releases) - [Commits](https://github.com/mendersoftware/mender-image-tests/compare/7c081c042f0024e87e9e15144b18d991fb378bcd...6d3e3ad681b573d74cf07b8ab13f91d0f587833d) --- updated-dependencies: - dependency-name: tests/mender-image-tests dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/mender-image-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mender-image-tests b/tests/mender-image-tests index 7c081c042f..6d3e3ad681 160000 --- a/tests/mender-image-tests +++ b/tests/mender-image-tests @@ -1 +1 @@ -Subproject commit 7c081c042f0024e87e9e15144b18d991fb378bcd +Subproject commit 6d3e3ad681b573d74cf07b8ab13f91d0f587833d From d4c7522a85bebf5966baac4bc8c62afe4b148ad2 Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Mon, 21 Mar 2022 11:10:13 +0100 Subject: [PATCH 099/117] Fix missing Docker login when pushing Docker image. Changelog: None Signed-off-by: Kristian Amlie --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9c2b5c7933..52a06b7c83 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -297,6 +297,9 @@ test_acceptance_ubuntu: - name: docker:20.10.8-dind alias: docker command: ["--mtu=1440"] # https://gitlab.com/gitlab-com/gl-infra/production/-/issues/5590 + before_script: + - docker login -u $DOCKER_HUB_USERNAME -p $DOCKER_HUB_PASSWORD + - docker load -i image.tar script: - IMAGE_VERSION=${MENDER_CONVERT_PUBLISH_VERSION:-${CI_COMMIT_REF_NAME}} From 52d51fb36fe37bedd4a75834c09ec9ee766ff79e Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Mon, 21 Mar 2022 11:10:35 +0100 Subject: [PATCH 100/117] Only publish Docker image on fresh tags or branch heads. This is to make sure we do not accidentally re-publish already existing tags. They can still be published manually by using the manual publish job, if necessary. Changelog: None Signed-off-by: Kristian Amlie --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 52a06b7c83..a8ce22834a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -325,5 +325,5 @@ publish:docker-image:manual: publish:docker-image:automatic: rules: - - if: '$PUBLISH_MENDER_CONVERT_AUTOMATIC == "true"' + - if: '$CI_COMMIT_BRANCH =~ /^(master|staging|production|feature-.+|[0-9]+\.[0-9]+\.([0-9]+|x))$/' extends: .template:publish:docker-image From 9b2436aff9ee1996b29ae335ca93b1a8aecafee6 Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Mon, 21 Mar 2022 15:47:20 +0100 Subject: [PATCH 101/117] Make sure that Docker image matches the checked out Git branch/tag. Changelog: None Signed-off-by: Kristian Amlie --- docker-build | 9 ++++++++- docker-mender-convert | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/docker-build b/docker-build index e05629817a..fd7cf116f4 100755 --- a/docker-build +++ b/docker-build @@ -16,6 +16,13 @@ set -e -IMAGE_NAME=${IMAGE_NAME:-"mendersoftware/mender-convert"} +GIT_PROVIDED_TAG_NAME="$(git describe --exact HEAD 2>/dev/null || \ + git for-each-ref "refs/heads/*" --format '%(refname:short)' --points-at HEAD 2>/dev/null)" + +if [ -z "$IMAGE_NAME" -a -z "$GIT_PROVIDED_TAG_NAME" ]; then + echo "Could not deduce mendersoftware/mender-convert container version from currently checked out commit. Using latest." 1>&2 +fi + +IMAGE_NAME=${IMAGE_NAME:-"mendersoftware/mender-convert${GIT_PROVIDED_TAG_NAME:+:$GIT_PROVIDED_TAG_NAME}"} eval docker build . -t "${IMAGE_NAME}" "$*" diff --git a/docker-mender-convert b/docker-mender-convert index d956ab1deb..119655356b 100755 --- a/docker-mender-convert +++ b/docker-mender-convert @@ -16,7 +16,14 @@ set -e -IMAGE_NAME=${IMAGE_NAME:-"mendersoftware/mender-convert"} +GIT_PROVIDED_TAG_NAME="$(git describe --exact HEAD 2>/dev/null || \ + git for-each-ref "refs/heads/*" --format '%(refname:short)' --points-at HEAD 2>/dev/null)" + +if [ -z "$IMAGE_NAME" -a -z "$GIT_PROVIDED_TAG_NAME" ]; then + echo "Could not deduce mendersoftware/mender-convert container version from currently checked out commit. Using latest." 1>&2 +fi + +IMAGE_NAME=${IMAGE_NAME:-"mendersoftware/mender-convert${GIT_PROVIDED_TAG_NAME:+:$GIT_PROVIDED_TAG_NAME}"} DEPLOY_DIRECTORY=${DEPLOY_DIRECTORY:-"$(pwd)/deploy"} INPUT_DIRECTORY=${INPUT_DIRECTORY:-"$(pwd)/input"} LOGS_DIRECTORY=${LOGS_DIRECTORY:-"$(pwd)/logs"} From 7cd6be9bce25d0951e5df3191ac03423386cae52 Mon Sep 17 00:00:00 2001 From: Ole Petter Date: Mon, 4 Apr 2022 09:31:28 +0200 Subject: [PATCH 102/117] chore(linkbot): fix lint errors Changelog: None Signed-off-by: Ole Petter --- scripts/linkbot/monitor-ub-server.js | 42 ++++++++++++++-------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/scripts/linkbot/monitor-ub-server.js b/scripts/linkbot/monitor-ub-server.js index 28bfb6e468..b9fa4df9c5 100644 --- a/scripts/linkbot/monitor-ub-server.js +++ b/scripts/linkbot/monitor-ub-server.js @@ -1,24 +1,24 @@ const jsdom = require("jsdom"); const { JSDOM } = jsdom; -const fs = require('fs') +const fs = require('fs'); const { updateURLLink } = require('./common'); -const target = "UBUNTU_SERVER_RPI_IMAGE_URL" -const url = "http://cdimage.ubuntu.com/ubuntu/releases/" -const reg = ".*(?[0-9][02468])\.04\.?(?[0-9]{1})?.*" +const target = "UBUNTU_SERVER_RPI_IMAGE_URL"; +const url = "http://cdimage.ubuntu.com/ubuntu/releases/"; +const reg = ".*(?[0-9][02468])\.04\.?(?[0-9]{1})?.*"; // Read the input file, and parse the variable input try { const data = fs.readFileSync('../test/run-tests.sh', 'utf8') .split('\n') - .filter(line => line.match(`${target}=.*`)) - var line = data[0] - var m = line.match(`.*=\"${reg}\"`) - var imageName = m.groups.release - var minor = m.groups.minor || 0 + .filter(line => line.match(`${target}=.*`)); + var line = data[0]; + var m = line.match(`.*=\"${reg}\"`); + var imageName = m.groups.release; + var minor = m.groups.minor || 0; } catch (err) { - console.error(err) - process.exit(1) + console.error(err); + process.exit(1); } JSDOM.fromURL(url, {}).then(dom => { @@ -28,27 +28,27 @@ JSDOM.fromURL(url, {}).then(dom => { .filter(ref => ref.textContent.match(reg)) .filter(ref => !ref.textContent.match(reg).groups.minor) // Ignore the minor number .reduce((acc, ref) => { - acc.push(ref.textContent.match(reg)) - return acc + acc.push(ref.textContent.match(reg)); + return acc; }, []) .sort((a,b) => { - return parseInt(b.groups.release) - parseInt(a.groups.release) - }) - var matchOn = matches[0].input + return parseInt(b.groups.release) - parseInt(a.groups.release); + }); + var matchOn = matches[0].input; return matchOn; // Get the release image url from the releases (sub)-page // const url = "http://cdimage.ubuntu.com/ubuntu/releases/" }).then(releaseVersion => { - var releaseVersion = releaseVersion.replace(/\s/g, "").replace(/\//g, "") - JSDOM.fromURL(`${url}${releaseVersion}/release/`, {}).then(dom => { + var releasedVersion = releaseVersion.replace(/\s/g, "").replace(/\//g, ""); + JSDOM.fromURL(`${url}${releasedVersion}/release/`, {}).then(dom => { var document = dom.window.document; var refs = document.getElementsByTagName("a"); - const match = Array.from(refs).find(ref => ref.href.match(`.*ubuntu-${releaseVersion}\.?[0-9]+-preinstalled-server-armhf.*\.img\.xz$`)) + const match = Array.from(refs).find(ref => ref.href.match(`.*ubuntu-${releasedVersion}\.?[0-9]+-preinstalled-server-armhf.*\.img\.xz$`)); if (match) { - console.log(`Ubuntu server image has a new release: ${match}`) - updateURLLink(`${target}=\"${match}\"`, target) + console.log(`Ubuntu server image has a new release: ${match}`); + updateURLLink(`${target}=\"${match}\"`, target); } }) }); From 64a2413442a59ecae740c6bef73e8bad02c5aeb1 Mon Sep 17 00:00:00 2001 From: Ole Petter Date: Mon, 4 Apr 2022 09:37:10 +0200 Subject: [PATCH 103/117] chore(linkbot): Log errors when HTTP fetching Changelog: None Signed-off-by: Ole Petter --- scripts/linkbot/monitor-ub-server.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/linkbot/monitor-ub-server.js b/scripts/linkbot/monitor-ub-server.js index b9fa4df9c5..19dcb57f82 100644 --- a/scripts/linkbot/monitor-ub-server.js +++ b/scripts/linkbot/monitor-ub-server.js @@ -51,4 +51,13 @@ JSDOM.fromURL(url, {}).then(dom => { updateURLLink(`${target}=\"${match}\"`, target); } }) -}); + .catch(err => { + console.log('Failed to get the update URL'); + console.log(`${url}${releaseVersion}/release/`); + }); +}) + .catch(err => { + console.log('Failed to get the URL'); + console.log(`${url}`); + console.log(err); + }); From 065a4db99696c02b0c65a24b91f9ec4e647ec904 Mon Sep 17 00:00:00 2001 From: Ole Petter Date: Mon, 4 Apr 2022 09:40:14 +0200 Subject: [PATCH 104/117] chore(linkbot): handle Ubuntu beta releases Changelog: None Signed-off-by: Ole Petter --- scripts/linkbot/monitor-ub-server.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/linkbot/monitor-ub-server.js b/scripts/linkbot/monitor-ub-server.js index 19dcb57f82..dc40016b62 100644 --- a/scripts/linkbot/monitor-ub-server.js +++ b/scripts/linkbot/monitor-ub-server.js @@ -53,11 +53,16 @@ JSDOM.fromURL(url, {}).then(dom => { }) .catch(err => { console.log('Failed to get the update URL'); - console.log(`${url}${releaseVersion}/release/`); + console.log(`${url}${releasedVersion}/release/`); + JSDOM.fromURL(`${url}${releasedVersion}/beta/`, {}) + .then(dom => { + console.log('Only the beta is out still'); + }); }); }) .catch(err => { console.log('Failed to get the URL'); - console.log(`${url}`); + console.log(url); console.log(err); + throw err; }); From 991297b6619865d37c29641b8d33f7daa5d63390 Mon Sep 17 00:00:00 2001 From: Ole Petter Date: Wed, 20 Apr 2022 13:47:40 +0200 Subject: [PATCH 105/117] Handle xz images in monitor-raspbian They've changed their image extensions again Changelog: None Signed-off-by: Ole Petter --- scripts/linkbot/monitor-raspbian-os.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/linkbot/monitor-raspbian-os.js b/scripts/linkbot/monitor-raspbian-os.js index d9cc1a4327..c684b329bc 100644 --- a/scripts/linkbot/monitor-raspbian-os.js +++ b/scripts/linkbot/monitor-raspbian-os.js @@ -35,13 +35,13 @@ function getNewRaspbian(target, url, separator) { var document = dom.window.document; var refs = document.getElementsByTagName("a"); var test = Array.from(refs) - .filter((ref) => ref.textContent.match("raspios-.*-lite.*.zip$")) + .filter((ref) => ref.textContent.match("raspios-.*-lite.*\.img\.(zip|xz)$")) .reduce((acc, element) => { - acc.push(element.textContent.match("raspios-.*-lite.*.zip$").input); + acc.push(element.textContent.match("raspios-.*-lite.*\.img\.(zip|xz)$").input); return acc; }, [])[0]; console.log(`Test var: ${test}`); - const imageName = test.substring(0, test.length - 4); + const imageName = test.substring(0, test.lastIndexOf('.img')); if (separator) { return { replacementLine: `${target}${separator} \"${url}/${test}\"`, From 1396aeeeadcff6bcf0aca0314829b94cac5d60d8 Mon Sep 17 00:00:00 2001 From: Ole Petter Date: Mon, 17 Jan 2022 19:41:54 +0100 Subject: [PATCH 106/117] QA-365: Update the PR template to go with conventional commits Changelog: None Signed-off-by: Ole Petter --- .github/PULL_REQUEST_TEMPLATE.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index dd8c5eb7f3..0885ffda8a 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -3,7 +3,19 @@ 🚨 Please review the [guidelines for contributing](https://github.com/mendersoftware/mender/blob/master/CONTRIBUTING.md) to this repository. -- [ ] Make sure that all commits have a [`Changelog`](https://github.com/mendersoftware/mender/blob/master/CONTRIBUTING.md#changelog-tags) tag. If nothing should be added to the Changelog, add a `Changelog: None` tag. If there is a change, add `Changelog: Commit`, or `Changelog: Title`, depending on what should be included in the changelog. +- [ ] Make sure that all commits follow the conventional commit [specification](https://www.github.com/mendersoftware/mendertesting/commitlint/grammar.md) for the Mender project. + +The majority of our contributions are fixes, which means your commit should have +the form below: + +``` +fix: + + + +Changelog: or +Ticket: or +``` - [ ] Make sure that all commits are signed with [`git --signoff`](https://git-scm.com/book/en/v2/Git-Tools-Signing-Your-Work). Also note that the signoff author must match the author of the commit. From 914c9f97941f32f36078293513662e9b76f93a44 Mon Sep 17 00:00:00 2001 From: Ole Petter Date: Tue, 16 Nov 2021 09:47:46 +0100 Subject: [PATCH 107/117] ci: Run a converted image on the hardware test farm Ticket: QA-294 Signed-off-by: Ole Petter --- .gitlab-ci.yml | 75 +++++++++++++++++++++++++++++++- configs/testing/hardware-testing | 24 ++++++++++ tests/mender-image-tests | 2 +- 3 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 configs/testing/hardware-testing diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a8ce22834a..8060515cef 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,6 +50,7 @@ stages: - test - build - convert + - flash - test_acceptance - publish @@ -152,7 +153,7 @@ convert_raspbian_raspberrypi4: - apt update - apt install -qyy bash wget git util-linux mtools python3 python3-pip gcc python3-dev libffi-dev liblzo2-dev libc-dev libssl-dev make sudo - awscli unzip qemu-system-x86 ovmf curl docker.io + awscli unzip qemu-system-x86 ovmf curl docker.io s3cmd # Python3 dependencies - python3 -m pip install -U pip - pip3 install --ignore-installed -r https://raw.githubusercontent.com/mendersoftware/meta-mender/master/tests/acceptance/requirements_py3.txt @@ -327,3 +328,75 @@ publish:docker-image:automatic: rules: - if: '$CI_COMMIT_BRANCH =~ /^(master|staging|production|feature-.+|[0-9]+\.[0-9]+\.([0-9]+|x))$/' extends: .template:publish:docker-image + +#################### +# Hardware testing # +#################### + +test:hardware:convert: + <<: *convert_raspbian + variables: + RASPBERRYPI_PLATFORM: raspberrypi4 + when: manual + stage: convert + script: + # Convert the image for the hardware tester + - env MENDER_ARTIFACT_NAME=${RASPBIAN_NAME}-mender + ./docker-mender-convert --disk-image input/${RASPBIAN_NAME}.img + --config configs/${RASPBERRYPI_PLATFORM}_config + --config configs/images/raspberrypi_raspbian_config + --config configs/testing/hardware-testing + --overlay tests/ssh-public-key-overlay + # Collect artifacts. + - mv deploy ${RASPBERRYPI_PLATFORM} + - tar czf ${RASPBERRYPI_PLATFORM}.tar.gz ${RASPBERRYPI_PLATFORM} + # Upload to temporary S3 bucket + - mender_ci_save_tmp_artifact ${RASPBERRYPI_PLATFORM}.tar.gz + artifacts: + paths: + - checksums + +test:hardware:flash: + needs: + - job: test:hardware:convert + artifacts: true + variables: + RASPBERRYPI_PLATFORM: raspberrypi4 + tags: + - hw + stage: flash + script: + - eval "$(curl https://raw.githubusercontent.com/mendersoftware/mendertesting/master/mender-ci-common.sh)" + # Fetch artifacts from temporary S3 bucket + - mender_ci_load_tmp_artifact ${RASPBERRYPI_PLATFORM}.tar.gz + # Flash and boot the device under test + - slave-control mode flash + - tar -xvzf ${RASPBERRYPI_PLATFORM}.tar.gz --wildcards --no-anchored '*.img' --to-stdout | dd of=/dev/sdmux bs=8M + - slave-control mode test + +test:hardware:acceptance: + <<: *test_acceptance + variables: + RASPBERRYPI_PLATFORM: raspberrypi4 + stage: test_acceptance + needs: [] + needs: + - job: build + artifacts: true + - job: test:hardware:convert + artifacts: true + - job: test:hardware:flash + script: + # Fetch artifacts from temporary S3 bucket + - eval "$(curl https://raw.githubusercontent.com/mendersoftware/mendertesting/master/mender-ci-common.sh)" + - mender_ci_load_tmp_artifact ${RASPBERRYPI_PLATFORM}.tar.gz + # Extract the configuration + - tar xzf ${RASPBERRYPI_PLATFORM}.tar.gz + - mv ${RASPBERRYPI_PLATFORM} deploy + # Run the tests + - ./scripts/test/run-tests.sh + --prebuilt-image + raspberrypi4 + "${RASPBIAN_NAME}-${RASPBERRYPI_PLATFORM}-mender + --hardware-testing + --host ${SSH_JUMP_HOST_IP}:${SSH_JUMP_HOST_PORT} mender-image-tests -k 'not test_network_based_image_update and not test_image_update_broken_kernel'" diff --git a/configs/testing/hardware-testing b/configs/testing/hardware-testing new file mode 100644 index 0000000000..38d55df899 --- /dev/null +++ b/configs/testing/hardware-testing @@ -0,0 +1,24 @@ +MENDER_COMPRESS_DISK_IMAGE="none" +MENDER_ARTIFACT_COMPRESSION="none" +# Install the mender-client +MENDER_CLIENT_INSTALL="y" +# Enable sshd +function enable_sshd() { + log_warn "Enabling sshd by default" + touch work/boot/ssh +} +PLATFORM_MODIFY_HOOKS+=(enable_sshd) +# Allow root ssh logins +function allow_root_ssh_logins() { + log_warn "Allowing ssh root logins" + echo "PermitRootLogin yes" >> work/rootfs/etc/ssh/sshd_config +} +PLATFORM_MODIFY_HOOKS+=(allow_root_ssh_logins) +# Configure the ssh key permissions in the overlay +function set_root_ssh_permissions() { + sudo chown -R root:root work/rootfs/root + sudo chmod 755 work/rootfs/root + sudo chmod 700 work/rootfs/root/.ssh + sudo chmod 600 work/rootfs/root/.ssh/authorized_keys +} +OVERLAY_MODIFY_HOOKS+=(set_root_ssh_permissions) diff --git a/tests/mender-image-tests b/tests/mender-image-tests index 6d3e3ad681..1d76120611 160000 --- a/tests/mender-image-tests +++ b/tests/mender-image-tests @@ -1 +1 @@ -Subproject commit 6d3e3ad681b573d74cf07b8ab13f91d0f587833d +Subproject commit 1d7612061138d9102ffb6a86ef362565a0e18937 From 4753f5ea51b29432d4a83b4115eb41c0ebf7d749 Mon Sep 17 00:00:00 2001 From: Mikael Torp-Holte Date: Thu, 28 Apr 2022 10:10:48 +0200 Subject: [PATCH 108/117] chore: Change dependabot prefix Ticket: QA-412 Signed-off-by: Mikael Torp-Holte --- .github/dependabot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4694569188..59190f7cf3 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,13 +1,13 @@ version: 2 updates: - commit-message: - prefix: "Changelog:All" + prefix: "chore" directory: / package-ecosystem: gitsubmodule schedule: interval: weekly - commit-message: - prefix: 'Changelog:All' + prefix: 'chore' directory: /scripts/linkbot/ package-ecosystem: npm schedule: From 7f86759dcfe91adb4d105457994e19e7cc2bd6fc Mon Sep 17 00:00:00 2001 From: Ole Petter Date: Mon, 2 May 2022 08:58:48 +0200 Subject: [PATCH 109/117] ci: make linkbot respect our conventional commit format Signed-off-by: Ole Petter --- .github/workflows/automatically-update-test-links.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/automatically-update-test-links.yml b/.github/workflows/automatically-update-test-links.yml index 81ece20967..88a4f57f27 100644 --- a/.github/workflows/automatically-update-test-links.yml +++ b/.github/workflows/automatically-update-test-links.yml @@ -28,9 +28,9 @@ jobs: body: | New images are available for update commit-message: | - Image-Bot - Image updates + chore: Image-Bot image updates - Changelog: None + Base image updates for our provided images. committer: mender-test-bot author: mender-test-bot signoff: true From 45ac1056697379becbf29b43b29fc915ad98a99a Mon Sep 17 00:00:00 2001 From: Ole Petter Date: Thu, 19 May 2022 10:55:05 +0200 Subject: [PATCH 110/117] chore(mender-artifact): bump mender-artifact version to latest Signed-off-by: Ole Petter --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0822d3c958..167d14e552 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ RUN cd /root/pxz && make FROM ubuntu:20.04 -ARG MENDER_ARTIFACT_VERSION=master +ARG MENDER_ARTIFACT_VERSION=3.7.x RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install -y \ # For 'ar' command to unpack .deb From e2e70faa0e1526f757bfc62282e4f7d2eb174703 Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Wed, 8 Jun 2022 10:18:52 +0200 Subject: [PATCH 111/117] fix: `resource` folder needs to be inside `input` after Dockerization. No changelog needed, since nothing is released yet. Changelog: None Ticket: None Signed-off-by: Kristian Amlie --- mender-convert-modify | 6 +++--- scripts/bootstrap-rootfs-overlay-demo-server.sh | 6 +++--- scripts/bootstrap-rootfs-overlay-hosted-server.sh | 8 ++++---- scripts/bootstrap-rootfs-overlay-production-server.sh | 10 +++++----- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/mender-convert-modify b/mender-convert-modify index f6ad9b1008..59712d8f1b 100755 --- a/mender-convert-modify +++ b/mender-convert-modify @@ -179,13 +179,13 @@ EOF run_and_log_cmd "sudo cp work/mender.conf.data work/rootfs/data/mender/mender.conf" run_and_log_cmd "sudo chmod 600 work/rootfs/data/mender/mender.conf" -if [ -f resources/mender.conf ]; then +if [ -f input/resources/mender.conf ]; then log_info "Installing the local mender.conf file" run_and_log_cmd "mkdir -p work/rootfs/etc/mender" - run_and_log_cmd "cp resources/mender.conf work/rootfs/etc/mender" + run_and_log_cmd "cp input/resources/mender.conf work/rootfs/etc/mender" run_and_log_cmd "sudo chmod 600 work/rootfs/etc/mender/mender.conf" else - log_warn "No mender.conf file found in resources. Have you remembered to run the bootstrap script?" + log_warn "No mender.conf file found in input/resources. Have you remembered to run the bootstrap script?" fi if [ -z "${MENDER_DEVICE_TYPE}" ]; then diff --git a/scripts/bootstrap-rootfs-overlay-demo-server.sh b/scripts/bootstrap-rootfs-overlay-demo-server.sh index 5b3e4eca13..d2d22a8bac 100755 --- a/scripts/bootstrap-rootfs-overlay-demo-server.sh +++ b/scripts/bootstrap-rootfs-overlay-demo-server.sh @@ -58,8 +58,8 @@ if [ -e ${output_dir} ]; then fi mkdir -p ${output_dir}/etc/mender -mkdir -p ${root_dir}/resources -cat <<- EOF > ${root_dir}/resources/mender.conf +mkdir -p ${root_dir}/input/resources +cat <<- EOF > ${root_dir}/input/resources/mender.conf { "ServerURL": "https://docker.mender.io", "ServerCertificate": "/etc/mender/server.crt" @@ -81,4 +81,4 @@ wget -q "https://raw.githubusercontent.com/mendersoftware/mender/master/support/ sudo chown -R 0 ${output_dir} sudo chgrp -R 0 ${output_dir} -echo "Configuration file for using Demo Mender Server written to: ${root_dir}/resources/mender.conf" +echo "Configuration file for using Demo Mender Server written to: ${root_dir}/input/resources/mender.conf" diff --git a/scripts/bootstrap-rootfs-overlay-hosted-server.sh b/scripts/bootstrap-rootfs-overlay-hosted-server.sh index 01b5b91393..897768d982 100755 --- a/scripts/bootstrap-rootfs-overlay-hosted-server.sh +++ b/scripts/bootstrap-rootfs-overlay-hosted-server.sh @@ -37,7 +37,7 @@ while (("$#")); do ;; *) echo "Sorry but the provided option is not supported: $1" - echo "Usage: $(basename $0) --output-dir ./rootfs_overlay_demo --tenant-token " + echo "Usage: $(basename $0) --output-dir ./input/rootfs_overlay_demo --tenant-token " exit 1 ;; esac @@ -57,8 +57,8 @@ if [ -e ${output_dir} ]; then sudo chown -R $(id -u) ${output_dir} sudo chgrp -R $(id -g) ${output_dir} fi -mkdir -p ${root_dir}/resources -cat <<- EOF > ${root_dir}/resources/mender.conf +mkdir -p ${root_dir}/input/resources +cat <<- EOF > ${root_dir}/input/resources/mender.conf { "ServerURL": "https://hosted.mender.io/", "TenantToken": "${tenant_token}" @@ -68,4 +68,4 @@ EOF sudo chown -R 0 ${output_dir} sudo chgrp -R 0 ${output_dir} -echo "Configuration file for using Hosted Mender written to: ${root_dir}/resources/mender.conf" +echo "Configuration file for using Hosted Mender written to: ${root_dir}/input/resources/mender.conf" diff --git a/scripts/bootstrap-rootfs-overlay-production-server.sh b/scripts/bootstrap-rootfs-overlay-production-server.sh index a60bbb00ac..dd21786207 100755 --- a/scripts/bootstrap-rootfs-overlay-production-server.sh +++ b/scripts/bootstrap-rootfs-overlay-production-server.sh @@ -62,24 +62,24 @@ if [ -e ${output_dir} ]; then sudo chgrp -R $(id -g) ${output_dir} fi -mkdir -p ${root_dir}/resources -cat <<- EOF > ${root_dir}/resources/mender.conf +mkdir -p ${root_dir}/input/resources +cat <<- EOF > ${root_dir}/input/resources/mender.conf { "ServerURL": "${server_url}", EOF if [ -n "${server_cert}" ]; then - cat <<- EOF >> ${root_dir}/resources/mender.conf + cat <<- EOF >> ${root_dir}/input/resources/mender.conf "ServerCertificate": "/etc/mender/server.crt" EOF cp -f "${server_cert}" ${output_dir}/etc/mender/server.crt fi -cat <<- EOF >> ${root_dir}/resources/mender.conf +cat <<- EOF >> ${root_dir}/input/resources/mender.conf } EOF sudo chown -R 0 ${output_dir} sudo chgrp -R 0 ${output_dir} -echo "Configuration file for using Production Mender Server written to: ${root_dir}/resources/mender.conf" +echo "Configuration file for using Production Mender Server written to: ${root_dir}/input/resources/mender.conf" From effe995d813c227182559ad078d517989c8b4847 Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Wed, 8 Jun 2022 10:21:03 +0200 Subject: [PATCH 112/117] fix: Move `rootfs_overlay_demo` into `input`. This is needed after Dockerization. Changelog: None Ticket: None Signed-off-by: Kristian Amlie --- {rootfs_overlay_demo => input/rootfs_overlay_demo}/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {rootfs_overlay_demo => input/rootfs_overlay_demo}/.gitkeep (100%) diff --git a/rootfs_overlay_demo/.gitkeep b/input/rootfs_overlay_demo/.gitkeep similarity index 100% rename from rootfs_overlay_demo/.gitkeep rename to input/rootfs_overlay_demo/.gitkeep From 6e5d906ebbee661ce74f6214fd1706032e62ff25 Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Wed, 8 Jun 2022 15:25:55 +0200 Subject: [PATCH 113/117] fix: Upgrade to latest version of grub-mender-grubenv. This fixes a boot problem which was introduced in the standalone grub boot scripts, during the grub.d integration work. Changelog: None Ticket: None Signed-off-by: Kristian Amlie --- configs/mender_grub_config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/mender_grub_config b/configs/mender_grub_config index c14c513e54..275eee8672 100644 --- a/configs/mender_grub_config +++ b/configs/mender_grub_config @@ -26,7 +26,7 @@ GRUB_VERSION=2.04 MENDER_GRUB_KERNEL_BOOT_ARGS="" # grub-mender-grubenv is the Mender integration for the GRUB bootloader -MENDER_GRUBENV_VERSION="b06a8e2cf13776b5cfc896fa8068006dd9992ebb" +MENDER_GRUBENV_VERSION="2ac898f5924d5870f8394ad8ecd3ef1ab1422e3b" MENDER_GRUBENV_URL="${MENDER_GITHUB_ORG}/grub-mender-grubenv/archive/${MENDER_GRUBENV_VERSION}.tar.gz" # Name of the storage device containing root filesystem partitions in GRUB From 2f8c0331e5f5f18e57d5bba5015e8229ac1bcbd0 Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Fri, 10 Jun 2022 11:36:10 +0200 Subject: [PATCH 114/117] test: Make launching script independent of current folder. Signed-off-by: Kristian Amlie --- scripts/test/mender-convert-qemu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/mender-convert-qemu b/scripts/test/mender-convert-qemu index e9956fbd34..809ab25e79 100755 --- a/scripts/test/mender-convert-qemu +++ b/scripts/test/mender-convert-qemu @@ -21,7 +21,7 @@ qemu-system-x86_64 \ -net user,hostfwd=tcp::8822-:22 \ -net nic,macaddr=52:54:00$(od -txC -An -N3 /dev/urandom|tr \ :) \ -drive file=${ovmf_file},if=pflash,format=raw,unit=0,readonly=on \ - -drive file=./uefi-nvram/OVMF_VARS.fd,if=pflash,format=raw,unit=1,readonly=on \ + -drive file="$(dirname "$0")/../../tests/uefi-nvram/OVMF_VARS.fd",if=pflash,format=raw,unit=1,readonly=on \ -drive format=raw,file=${DISK_IMG} & qemu_pid=$! From 69da7596094456a6f2b88aef65de56aedac24e8c Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Fri, 10 Jun 2022 11:40:46 +0200 Subject: [PATCH 115/117] test(grub): Fix incorrectly generated test images. Because the shim has hardcoded paths for each distribution, it is better to use the `grub-install --removable` argument to install into `/EFI/BOOT` than to do it ourselves. This makes sure that all files are present and loaded from correct locations. The symptom before this change was that the shim could not find the grubx64.efi file, which is because we were not installing it correctly. So in fact the image would not boot, even though it would boot after a Mender conversion. Signed-off-by: Kristian Amlie --- scripts/test/generate-image.sh | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/scripts/test/generate-image.sh b/scripts/test/generate-image.sh index 13d4e56ac3..92cf96ced5 100755 --- a/scripts/test/generate-image.sh +++ b/scripts/test/generate-image.sh @@ -82,8 +82,6 @@ post_process_image() { regenerate_grub_live "$image" mount ${LO_DEVICE}p1 tmp-p1 mount ${LO_DEVICE}p2 tmp-p2 - - post_tweaks tmp-p1 tmp-p2 } pre_tweaks() { @@ -107,21 +105,6 @@ EOF sed -E -i -e 's/^#? *PermitRootLogin .*/PermitRootLogin yes/' $root/etc/ssh/sshd_config } -post_tweaks() { - local -r boot="$1" - local -r root="$2" - - # Delete systemd-boot, which isn't normally present in images that were - # installed with OS installers, at least not at the time of writing. - rm -rf "$boot/EFI/systemd" - - # Also replace bootx64.efi, which is the default bootloader. Mkosi installs - # systemd-bootx86.efi, but we want the shim. - rm -f "$boot/EFI/BOOT/*" - mkdir -p "$boot/EFI/BOOT" - cp "$root/usr/lib/shim/shimx64.efi.signed" "$boot/EFI/BOOT/BOOTX64.EFI" -} - # Unfortunately installing grub scripts is something which is not really # possible when offline. This is something which is easier with systemd-boot, so # longterm GRUB will probably follow, or systemd-boot will take over. Anyway, @@ -136,7 +119,7 @@ Description=Regenerate grub scripts, disable itself and then shut down. [Service] Type=oneshot -ExecStart=sh -c "grub-install && update-grub && systemctl disable mender-regenerate-grub-and-shutdown.service && poweroff" +ExecStart=sh -c "grub-install && grub-install --removable && update-grub && systemctl disable mender-regenerate-grub-and-shutdown.service && poweroff" EOF ln -sf "$root/etc/systemd/system/mender-regenerate-grub-and-shutdown.service" "$root/etc/systemd/system/multi-user.target.wants/" From 1de050c78f1d0f0054f29bc8322d95d9a40934a3 Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Fri, 10 Jun 2022 12:45:45 +0200 Subject: [PATCH 116/117] fix(grub): Fix failure to boot when using standalone mode. When introducing grub.d integration, we switched to using the already- installed grub binary if it existed. But this is not a good idea, because it may come with hardcoded paths which may fail to load the script that we put in `/EFI/BOOT`. So revert to the old behavior, install our own pre-compiled grub binary if grub.d integration is off. No changelog, since we never released this regression. Changelog: None Ticket: None Signed-off-by: Kristian Amlie --- mender-convert-modify | 14 +++++++------- modules/probe.sh | 9 --------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/mender-convert-modify b/mender-convert-modify index 59712d8f1b..7168716267 100755 --- a/mender-convert-modify +++ b/mender-convert-modify @@ -140,21 +140,21 @@ if [ "${MENDER_GRUB_EFI_INTEGRATION}" == "y" ]; then # Check for known U-Boot problems in all files on the boot partition. check_for_broken_uboot_uefi_support work/boot - if has_grub_efi "work/boot"; then + if [ "$MENDER_GRUB_D_INTEGRATION" = y ] || ( [ "$MENDER_GRUB_D_INTEGRATION" = auto ] && supports_grub_d "work/rootfs" ); then # No need to install Grub, use the one already present, and only install # our tools. - log_info "GRUB EFI bootloader already present, not installing one." + log_info "Not installing GRUB EFI bootloader, relying on platform provided one." + + log_info "Installing Mender GRUB tools..." grub_install_grub_editenv_binary - else - log_info "Installing GRUB EFI bootloader..." - grub_install_mender_grub - fi - if [ "$MENDER_GRUB_D_INTEGRATION" = y ] || ( [ "$MENDER_GRUB_D_INTEGRATION" = auto ] && supports_grub_d "work/rootfs" ); then log_info "Generating grub config using update-grub..." grub_create_grub_config grub_install_grub_d_config else + log_info "Installing GRUB EFI bootloader..." + grub_install_mender_grub + log_info "Generating the mender-grub config..." grub_create_grub_config grub_install_standalone_grub_config diff --git a/modules/probe.sh b/modules/probe.sh index 0491fad365..92233828de 100644 --- a/modules/probe.sh +++ b/modules/probe.sh @@ -345,15 +345,6 @@ is_efi_compatible_kernel() { return 0 } -# has_grub_efi -# -# $1 - the boot partition to search for a grub*.efi -# -# Checks the EFI/* filesystem for the presence of a GRUB bootloader -has_grub_efi() { - find "${1}" -type f -name 'grub*.efi' -print0 | grep -qz grub -} - supports_grub_d() { test -d "$1"/etc/grub.d || return 1 From 335d22374f6dc15ea435846eb0f0de5aa1dccf7b Mon Sep 17 00:00:00 2001 From: Mark Schulte Date: Tue, 23 Aug 2022 16:07:37 -0700 Subject: [PATCH 117/117] Use sudo when necessary --- mender-convert-modify | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mender-convert-modify b/mender-convert-modify index 7168716267..4623a085a8 100755 --- a/mender-convert-modify +++ b/mender-convert-modify @@ -243,7 +243,7 @@ log_info "Using data partition device in fstab: $data_part_device" fstab=$(sed '/\/boot/d' work/rootfs/etc/fstab) fstab=$(sed '/.*\s\/\s.*/d' <<< "$fstab") -cat <<- EOF > work/rootfs/etc/fstab +sudo bash -c 'cat <<- EOF > work/rootfs/etc/fstab ${boot_part_device} ${boot_part_mountpoint} auto ${MENDER_BOOT_PART_FSTAB_OPTS} 0 0 /dev/root / auto ${MENDER_ROOT_PART_FSTAB_OPTS} 1 ${MENDER_ROOT_PART_FS_PASSNO} @@ -251,7 +251,7 @@ ${data_part_device} /data auto ${MENDER_DATA_PART_FSTAB_OPTS} # entries kept from original fstab $fstab -EOF +EOF' # # Make sure to re-label rootfs when selinux is in enforcing mode