diff --git a/pkg/steps/release/promote.go b/pkg/steps/release/promote.go index 5c6508e5f2..19f094f9b7 100644 --- a/pkg/steps/release/promote.go +++ b/pkg/steps/release/promote.go @@ -338,8 +338,8 @@ const ( ) // getMirrorRetryShell mirrors images with retries and fails the promotion pod if all attempts fail. -func getMirrorRetryShell(registryConfig string, images []string, loglevel int) string { - mirrorCmd := getMirrorCommand(registryConfig, images, loglevel) +func getMirrorRetryShell(registryConfig string, images []string) string { + mirrorCmd := getMirrorCommand(registryConfig, images, 2) n := quayPromotionMirrorAttempts return fmt.Sprintf(`for r in {1..%d}; do echo Mirror attempt $r @@ -381,8 +381,64 @@ done const ( retryLoopTemplate = "for r in {1..%d}; do echo %s; %s && break; %s; done" retryLoopWithBackoff = "backoff=$(($RANDOM % 120))s; echo Sleeping randomized $backoff before retry; sleep $backoff" + + quayImageIncomingSuffix = "_incoming" + quayImagePreSuffix = "__pre" + quayImagePost1Suffix = "__post1" ) +type quayFloatPromotion struct { + floatTag string + newSrc string + pruneTag string +} + +// getStagedQuayFloatPromotionShell returns shell that repoints floatTag from its incoming staging tag. +// If floatTag already exists on the registry, the current image is copied to pruneTag (when set) and +// floatTag+quayImagePreSuffix before the repoint; floatTag+quayImagePost1Suffix is added afterward. +func getStagedQuayFloatPromotionShell(registryConfig, floatTag, pruneTag string) string { + incomingTag := floatTag + quayImageIncomingSuffix + preTag := floatTag + quayImagePreSuffix + post1Tag := floatTag + quayImagePost1Suffix + + checkOld := fmt.Sprintf(`_OLD_EXISTS=false +if oc image info --registry-config=%s %s >/dev/null 2>&1; then _OLD_EXISTS=true; fi`, registryConfig, floatTag) + + var backupOld string + if pruneTag != "" { + backupOld = fmt.Sprintf(`if [ "${_OLD_EXISTS}" = "true" ]; then +%s +%s +fi`, + indentShell(getMirrorRetryShell(registryConfig, []string{fmt.Sprintf("%s=%s", floatTag, pruneTag)})), + indentShell(getMirrorRetryShell(registryConfig, []string{fmt.Sprintf("%s=%s", floatTag, preTag)})), + ) + } else { + backupOld = fmt.Sprintf(`if [ "${_OLD_EXISTS}" = "true" ]; then +%s +fi`, indentShell(getMirrorRetryShell(registryConfig, []string{fmt.Sprintf("%s=%s", floatTag, preTag)}))) + } + + latch := getMirrorRetryShell(registryConfig, []string{fmt.Sprintf("%s=%s", incomingTag, floatTag)}) + + postLatch := fmt.Sprintf(`if [ "${_OLD_EXISTS}" = "true" ]; then +%s +fi`, indentShell(getMirrorRetryShell(registryConfig, []string{fmt.Sprintf("%s=%s", floatTag, post1Tag)}))) + + return strings.Join([]string{checkOld, backupOld, latch, postLatch}, "\n") +} + +func indentShell(script string) string { + const prefix = " " + lines := strings.Split(script, "\n") + for i, line := range lines { + if line != "" { + lines[i] = prefix + line + } + } + return strings.Join(lines, "\n") +} + func getPromotionPod(imageMirrorTarget map[string]string, timeStr string, namespace string, name string, cliImage string, nodeArchitectures []string) *coreapi.Pod { keys := make([]string, 0, len(imageMirrorTarget)) for k := range imageMirrorTarget { @@ -391,7 +447,10 @@ func getPromotionPod(imageMirrorTarget map[string]string, timeStr string, namesp sort.Strings(keys) var images []string + var incomingImages []string var pruneImages []string + var quayFloatPromotions []quayFloatPromotion + pruneTagForFloat := map[string]string{} var tags []string // resolveAndTagPairs holds [quayProxyTag, isTag] for official ocp IS targets (consolidated // ocp/4.x:tag and legacy *-quay). Resolved post-mirror via oc image info + oc tag. @@ -401,12 +460,25 @@ func getPromotionPod(imageMirrorTarget map[string]string, timeStr string, namesp for _, k := range keys { if strings.Contains(k, fmt.Sprintf("%s_prune_", timeStr)) { - pruneImages = append(pruneImages, fmt.Sprintf("%s=%s", imageMirrorTarget[k], k)) + floatTag := imageMirrorTarget[k] + if isQuayStep { + pruneTagForFloat[floatTag] = k + } else { + pruneImages = append(pruneImages, fmt.Sprintf("%s=%s", floatTag, k)) + } } else { src := imageMirrorTarget[k] if strings.HasPrefix(k, api.QuayOpenShiftCIRepo+":") { // Images promoted into quay.io/openshift/ci always use oc image mirror (tag or digest sources). - images = append(images, fmt.Sprintf("%s=%s", src, k)) + if isQuayStep { + incomingImages = append(incomingImages, fmt.Sprintf("%s=%s", src, k+quayImageIncomingSuffix)) + quayFloatPromotions = append(quayFloatPromotions, quayFloatPromotion{ + floatTag: k, + newSrc: src, + }) + } else { + images = append(images, fmt.Sprintf("%s=%s", src, k)) + } } else if isQuayStep && !strings.Contains(k, api.ComponentFormatReplacement) { // Concrete quay IS target: resolve the QCI digest after mirroring instead of // using a pre-computed (potentially tag-only) source so spec.from is always digest-based. @@ -429,6 +501,10 @@ func getPromotionPod(imageMirrorTarget map[string]string, timeStr string, namesp } } + for i := range quayFloatPromotions { + quayFloatPromotions[i].pruneTag = pruneTagForFloat[quayFloatPromotions[i].floatTag] + } + registryConfig := filepath.Join(api.RegistryPushCredentialsCICentralSecretMountPath, coreapi.DockerConfigJsonKey) command := []string{"/bin/sh", "-c"} @@ -436,7 +512,7 @@ func getPromotionPod(imageMirrorTarget map[string]string, timeStr string, namesp // Generate mirror commands if there are images to mirror if len(images) > 0 { - commands = append(commands, getMirrorRetryShell(registryConfig, images, 2)) + commands = append(commands, getMirrorRetryShell(registryConfig, images)) } if isQuayStep { @@ -466,6 +542,17 @@ func getPromotionPod(imageMirrorTarget map[string]string, timeStr string, namesp } var args []string + if isQuayStep && len(incomingImages) > 0 { + args = append(args, getMirrorRetryShell(registryConfig, incomingImages)) + } + if isQuayStep && len(quayFloatPromotions) > 0 { + sort.Slice(quayFloatPromotions, func(i, j int) bool { + return quayFloatPromotions[i].floatTag < quayFloatPromotions[j].floatTag + }) + for _, promotion := range quayFloatPromotions { + args = append(args, getStagedQuayFloatPromotionShell(registryConfig, promotion.floatTag, promotion.pruneTag)) + } + } if len(pruneImages) > 0 { // See https://github.com/openshift/release/blob/2080ec4a49337c27577a4b2ff08a538e96436e65/hack/qci_registry_pruner.py for details. // Note that we don't retry here and we ignore failures because (a) it may be the first time an image tag is diff --git a/pkg/steps/release/promote_test.go b/pkg/steps/release/promote_test.go index eb1514e6ed..bc928e980b 100644 --- a/pkg/steps/release/promote_test.go +++ b/pkg/steps/release/promote_test.go @@ -1112,7 +1112,7 @@ func TestGetPublicImageReference(t *testing.T) { func TestGetMirrorRetryShell(t *testing.T) { regcfg := "/etc/push-secret/.dockerconfigjson" - got := getMirrorRetryShell(regcfg, []string{"src=dst"}, 2) + got := getMirrorRetryShell(regcfg, []string{"src=dst"}) for _, sub := range []string{ "for r in {1..5}", "Mirror attempt $r", diff --git a/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay.yaml b/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay.yaml index 0b2dabdb99..52857f7a3c 100644 --- a/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay.yaml +++ b/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay.yaml @@ -8,10 +8,9 @@ spec: containers: - args: - |- - oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ci_a_latest=quay.io/openshift/ci:20240603235401_prune_ci_a_latest quay.io/openshift/ci:ci_c_latest=quay.io/openshift/ci:20240603235401_prune_ci_c_latest || true for r in {1..5}; do echo Mirror attempt $r - if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:bbb=quay.io/openshift/ci:ci_a_latest registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:ddd=quay.io/openshift/ci:ci_c_latest; then break; fi + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:bbb=quay.io/openshift/ci:ci_a_latest_incoming registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:ddd=quay.io/openshift/ci:ci_c_latest_incoming; then break; fi if [ "${r}" -eq 5 ]; then exit 1 fi @@ -19,6 +18,98 @@ spec: echo Sleeping randomized $backoff before retry sleep $backoff done + _OLD_EXISTS=false + if oc image info --registry-config=/etc/push-secret/.dockerconfigjson quay.io/openshift/ci:ci_a_latest >/dev/null 2>&1; then _OLD_EXISTS=true; fi + if [ "${_OLD_EXISTS}" = "true" ]; then + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ci_a_latest=quay.io/openshift/ci:20240603235401_prune_ci_a_latest; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ci_a_latest=quay.io/openshift/ci:ci_a_latest__pre; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + fi + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ci_a_latest_incoming=quay.io/openshift/ci:ci_a_latest; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + if [ "${_OLD_EXISTS}" = "true" ]; then + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ci_a_latest=quay.io/openshift/ci:ci_a_latest__post1; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + fi + _OLD_EXISTS=false + if oc image info --registry-config=/etc/push-secret/.dockerconfigjson quay.io/openshift/ci:ci_c_latest >/dev/null 2>&1; then _OLD_EXISTS=true; fi + if [ "${_OLD_EXISTS}" = "true" ]; then + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ci_c_latest=quay.io/openshift/ci:20240603235401_prune_ci_c_latest; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ci_c_latest=quay.io/openshift/ci:ci_c_latest__pre; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + fi + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ci_c_latest_incoming=quay.io/openshift/ci:ci_c_latest; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + if [ "${_OLD_EXISTS}" = "true" ]; then + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ci_c_latest=quay.io/openshift/ci:ci_c_latest__post1; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + fi set +e for r in {1..2}; do echo "Tag attempt $r (all together)"; oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@sha256:ddd ci/${component}-quay:c quay-proxy.ci.openshift.org/openshift/ci@sha256:bbb ci/ci-quay:${component} && break; :; done for r in {1..3}; do echo "Tag attempt $r (individual)"; oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@sha256:ddd ci/${component}-quay:c && break; backoff=$(($RANDOM % 120))s; echo Sleeping randomized $backoff before retry; sleep $backoff; done diff --git a/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay_4.12.yaml b/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay_4.12.yaml index 877b909f8a..82da2b6129 100644 --- a/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay_4.12.yaml +++ b/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay_4.12.yaml @@ -8,10 +8,9 @@ spec: containers: - args: - | - oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.12_ovn-kubernetes=quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes quay.io/openshift/ci:ocp_4.12_ovn-kubernetes-base=quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes-base || true for r in {1..5}; do echo Mirror attempt $r - if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:aaa=quay.io/openshift/ci:ocp_4.12_ovn-kubernetes registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:bbb=quay.io/openshift/ci:ocp_4.12_ovn-kubernetes-base; then break; fi + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:aaa=quay.io/openshift/ci:ocp_4.12_ovn-kubernetes_incoming registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:bbb=quay.io/openshift/ci:ocp_4.12_ovn-kubernetes-base_incoming; then break; fi if [ "${r}" -eq 5 ]; then exit 1 fi @@ -19,6 +18,98 @@ spec: echo Sleeping randomized $backoff before retry sleep $backoff done + _OLD_EXISTS=false + if oc image info --registry-config=/etc/push-secret/.dockerconfigjson quay.io/openshift/ci:ocp_4.12_ovn-kubernetes >/dev/null 2>&1; then _OLD_EXISTS=true; fi + if [ "${_OLD_EXISTS}" = "true" ]; then + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.12_ovn-kubernetes=quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.12_ovn-kubernetes=quay.io/openshift/ci:ocp_4.12_ovn-kubernetes__pre; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + fi + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.12_ovn-kubernetes_incoming=quay.io/openshift/ci:ocp_4.12_ovn-kubernetes; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + if [ "${_OLD_EXISTS}" = "true" ]; then + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.12_ovn-kubernetes=quay.io/openshift/ci:ocp_4.12_ovn-kubernetes__post1; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + fi + _OLD_EXISTS=false + if oc image info --registry-config=/etc/push-secret/.dockerconfigjson quay.io/openshift/ci:ocp_4.12_ovn-kubernetes-base >/dev/null 2>&1; then _OLD_EXISTS=true; fi + if [ "${_OLD_EXISTS}" = "true" ]; then + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.12_ovn-kubernetes-base=quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes-base; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.12_ovn-kubernetes-base=quay.io/openshift/ci:ocp_4.12_ovn-kubernetes-base__pre; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + fi + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.12_ovn-kubernetes-base_incoming=quay.io/openshift/ci:ocp_4.12_ovn-kubernetes-base; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + if [ "${_OLD_EXISTS}" = "true" ]; then + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.12_ovn-kubernetes-base=quay.io/openshift/ci:ocp_4.12_ovn-kubernetes-base__post1; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + fi for r in {1..5}; do _digest=$(oc image info --registry-config=/etc/push-secret/.dockerconfigjson --filter-by-os=linux/amd64 quay.io/openshift/ci:ocp_4.12_ovn-kubernetes | sed -n '/^Digest:[[:space:]]/s/^Digest:[[:space:]]*//p' | head -n1) if [ -n "${_digest}" ] && oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@${_digest} ocp/4.12:ovn-kubernetes; then diff --git a/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay_multiple_tags.yaml b/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay_multiple_tags.yaml index 93b51a90a5..e02a5f5368 100644 --- a/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay_multiple_tags.yaml +++ b/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay_multiple_tags.yaml @@ -8,10 +8,9 @@ spec: containers: - args: - | - oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.21_ovn-kubernetes=quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-base=quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes-base quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-microshift=quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes-microshift || true for r in {1..5}; do echo Mirror attempt $r - if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:aaa=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:bbb=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-base registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:ccc=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-microshift; then break; fi + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:aaa=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes_incoming registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:bbb=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-base_incoming registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:ccc=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-microshift_incoming; then break; fi if [ "${r}" -eq 5 ]; then exit 1 fi @@ -19,6 +18,144 @@ spec: echo Sleeping randomized $backoff before retry sleep $backoff done + _OLD_EXISTS=false + if oc image info --registry-config=/etc/push-secret/.dockerconfigjson quay.io/openshift/ci:ocp_4.21_ovn-kubernetes >/dev/null 2>&1; then _OLD_EXISTS=true; fi + if [ "${_OLD_EXISTS}" = "true" ]; then + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.21_ovn-kubernetes=quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.21_ovn-kubernetes=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes__pre; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + fi + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.21_ovn-kubernetes_incoming=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + if [ "${_OLD_EXISTS}" = "true" ]; then + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.21_ovn-kubernetes=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes__post1; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + fi + _OLD_EXISTS=false + if oc image info --registry-config=/etc/push-secret/.dockerconfigjson quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-base >/dev/null 2>&1; then _OLD_EXISTS=true; fi + if [ "${_OLD_EXISTS}" = "true" ]; then + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-base=quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes-base; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-base=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-base__pre; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + fi + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-base_incoming=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-base; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + if [ "${_OLD_EXISTS}" = "true" ]; then + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-base=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-base__post1; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + fi + _OLD_EXISTS=false + if oc image info --registry-config=/etc/push-secret/.dockerconfigjson quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-microshift >/dev/null 2>&1; then _OLD_EXISTS=true; fi + if [ "${_OLD_EXISTS}" = "true" ]; then + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-microshift=quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes-microshift; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-microshift=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-microshift__pre; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + fi + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-microshift_incoming=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-microshift; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + if [ "${_OLD_EXISTS}" = "true" ]; then + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-microshift=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-microshift__post1; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + fi for r in {1..5}; do _digest=$(oc image info --registry-config=/etc/push-secret/.dockerconfigjson --filter-by-os=linux/amd64 quay.io/openshift/ci:ocp_4.21_ovn-kubernetes | sed -n '/^Digest:[[:space:]]/s/^Digest:[[:space:]]*//p' | head -n1) if [ -n "${_digest}" ] && oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@${_digest} ocp/4.21:ovn-kubernetes; then diff --git a/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay_non_release_namespace.yaml b/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay_non_release_namespace.yaml index 0e6945e479..f5968c4767 100644 --- a/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay_non_release_namespace.yaml +++ b/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay_non_release_namespace.yaml @@ -10,7 +10,7 @@ spec: - |- for r in {1..5}; do echo Mirror attempt $r - if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:bbb=quay.io/openshift/ci:ci_ci_sanitize-prow-jobs registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:aaa=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes; then break; fi + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:bbb=quay.io/openshift/ci:ci_ci_sanitize-prow-jobs_incoming registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:aaa=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes_incoming; then break; fi if [ "${r}" -eq 5 ]; then exit 1 fi @@ -18,6 +18,78 @@ spec: echo Sleeping randomized $backoff before retry sleep $backoff done + _OLD_EXISTS=false + if oc image info --registry-config=/etc/push-secret/.dockerconfigjson quay.io/openshift/ci:ci_ci_sanitize-prow-jobs >/dev/null 2>&1; then _OLD_EXISTS=true; fi + if [ "${_OLD_EXISTS}" = "true" ]; then + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ci_ci_sanitize-prow-jobs=quay.io/openshift/ci:ci_ci_sanitize-prow-jobs__pre; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + fi + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ci_ci_sanitize-prow-jobs_incoming=quay.io/openshift/ci:ci_ci_sanitize-prow-jobs; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + if [ "${_OLD_EXISTS}" = "true" ]; then + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ci_ci_sanitize-prow-jobs=quay.io/openshift/ci:ci_ci_sanitize-prow-jobs__post1; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + fi + _OLD_EXISTS=false + if oc image info --registry-config=/etc/push-secret/.dockerconfigjson quay.io/openshift/ci:ocp_4.21_ovn-kubernetes >/dev/null 2>&1; then _OLD_EXISTS=true; fi + if [ "${_OLD_EXISTS}" = "true" ]; then + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.21_ovn-kubernetes=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes__pre; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + fi + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.21_ovn-kubernetes_incoming=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + if [ "${_OLD_EXISTS}" = "true" ]; then + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.21_ovn-kubernetes=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes__post1; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + fi for r in {1..5}; do _digest=$(oc image info --registry-config=/etc/push-secret/.dockerconfigjson --filter-by-os=linux/amd64 quay.io/openshift/ci:ocp_4.21_ovn-kubernetes | sed -n '/^Digest:[[:space:]]/s/^Digest:[[:space:]]*//p' | head -n1) if [ -n "${_digest}" ] && oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@${_digest} ocp/4.21:ovn-kubernetes; then