diff --git a/.buildkite/pipeline.backport-dispatch.yml b/.buildkite/pipeline.backport-dispatch.yml new file mode 100644 index 00000000000..6a82c3bd3b3 --- /dev/null +++ b/.buildkite/pipeline.backport-dispatch.yml @@ -0,0 +1,43 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json + +env: + SETUP_GVM_VERSION: "v0.6.0" + LINUX_AGENT_IMAGE: "golang:${GO_VERSION}" + YQ_VERSION: 'v4.35.2' + GH_CLI_VERSION: "2.29.0" + NOTIFY_TO: "ecosystem-team@elastic.co" + +steps: + - label: ":ballot_box_with_check: Backports inventory validation" + key: "check-backports-inventory" + command: ".buildkite/scripts/check_backports_inventory.sh" + agents: + image: "${LINUX_AGENT_IMAGE}" + if_changed: + - ".backports.yml" + - ".buildkite/scripts/run_dev_scripts_tests.sh" + - "dev/scripts/**.sh" + - "dev/backports/**" + if: | + build.env('BUILDKITE_PULL_REQUEST') == "false" && build.branch == "main" + + - label: ":git: Create backport branches for new entries" + key: "trigger-backport-create" + command: ".buildkite/scripts/trigger_backport.sh" + agents: + image: "${LINUX_AGENT_IMAGE}" + plugins: + - elastic/vault-github-token#v0.1.0: + depends_on: + - step: "check-backports-inventory" + allow_failure: false + if_changed: + - ".backports.yml" + if: | + build.env('BUILDKITE_PULL_REQUEST') == "false" && + build.branch == "main" && + build.env('BUILDKITE_PIPELINE_SLUG') == "integrations-backport-dispatch" + +notify: + - email: "$NOTIFY_TO" + if: "build.state == 'failed' && build.env('BUILDKITE_PULL_REQUEST') == 'false'" diff --git a/.buildkite/pipeline.backport.yml b/.buildkite/pipeline.backport.yml index 2538e32d651..f3562a2d969 100644 --- a/.buildkite/pipeline.backport.yml +++ b/.buildkite/pipeline.backport.yml @@ -3,21 +3,33 @@ name: "integrations-backport" env: + SETUP_GVM_VERSION: "v0.6.0" YQ_VERSION: 'v4.35.2' + GH_CLI_VERSION: "2.29.0" # Agent images used in pipeline steps LINUX_AGENT_IMAGE: "golang:${GO_VERSION}" + NOTIFY_TO: "ecosystem-team@elastic.co" steps: - label: "Check that it runs from UI" key: "check-ui" command: - - "buildkite-agent annotate \"The $BUILDKITE_PIPELINE_SLUG is used only for running from UI!\" --style 'warning'" + - "buildkite-agent annotate \"The $${BUILDKITE_PIPELINE_SLUG} pipeline can only be triggered from the UI by members of the 'ecosystem' team, or via a trigger step from the 'integrations' or 'integrations-backport-dispatch' pipeline. For the recommended workflow see the [documentation](https://www.elastic.co/guide/en/integrations-developer/current/developer-workflow-support-old-package.html).\" --style 'warning'" - "exit 1" - if: "build.source != 'ui'" + if: | + !( + (build.source == 'ui' && build.creator.teams includes "ecosystem") || + (build.source == 'trigger_job' && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == 'integrations') || + (build.source == 'trigger_job' && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == 'integrations-backport-dispatch') + ) + + # Ensure that the check-ui step runs before any other step + - wait: ~ - input: "Input values for the variables" key: "input-variables" + if: "build.source != 'trigger_job'" fields: - select: "DRY_RUN" key: "DRY_RUN" @@ -35,10 +47,6 @@ steps: key: "PACKAGE_NAME" required: true default: "" - - text: "Enter name of the folder for the package (in most cases coincides with PACKAGE_NAME)" - key: "PACKAGE_FOLDER_NAME" - required: true - default: "" - text: "Enter package version (examples: 1.5.7, 1.0.0-beta1)" key: "PACKAGE_VERSION" required: true @@ -50,7 +58,15 @@ steps: value: "true" - label: "False" value: "false" - default: "false" + default: "true" + - text: "Backport branch name (optional, defaults to backport--.)" + key: "BACKPORT_BRANCH_NAME" + required: false + default: "" + - text: "PR number to notify on completion (optional)" + key: "PR_NUMBER" + required: false + default: "" - label: "Creating the backport branch" key: "create-backport-branch" @@ -65,3 +81,30 @@ steps: depends_on: - step: "input-variables" allow_failure: false + + - label: ":github: Notify PR" + key: "notify-pr" + command: | + if ! outcome=$$(buildkite-agent step get "outcome" --step "create-backport-branch"); then + echo "Failed to retrieve step outcome, skipping PR notification" + exit 0 + fi + if [[ "$${outcome}" == "passed" ]]; then + NOTIFY_STATUS=success .buildkite/scripts/notify_backport_pr.sh + else + NOTIFY_STATUS=failure .buildkite/scripts/notify_backport_pr.sh + fi + agents: + image: "${LINUX_AGENT_IMAGE}" + plugins: + - elastic/vault-github-token#v0.1.0: + depends_on: + - step: "create-backport-branch" + allow_failure: true + +notify: + # Only notify for automated backport creation failures (triggered by integrations-backport-dispatch). + # Dry-run failures from PR builds (triggered by integrations) are already surfaced as a failing + # CI status on the PR itself and do not warrant a team email. + - email: "$NOTIFY_TO" + if: "build.state == 'failed' && build.source == 'trigger_job' && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == 'integrations-backport-dispatch'" diff --git a/.buildkite/pipeline.publish.yml b/.buildkite/pipeline.publish.yml index ba0d922b38f..64f5d60c7dc 100644 --- a/.buildkite/pipeline.publish.yml +++ b/.buildkite/pipeline.publish.yml @@ -38,6 +38,9 @@ steps: ARTIFACTS_FOLDER: "artifacts-to-sign" # by default it will publish packages DRY_RUN: "${DRY_RUN:-false}" + # Comma-separated list of package zip filenames to skip (e.g. "foo-1.0.0.zip,bar-2.0.0.zip") + # kubeletstats_input_otel-0.1.0 was deleted but it is kept in the repository for now, in case it is needed to release a new version if needed + SKIP_PACKAGES: "kubeletstats_input_otel-0.1.0.zip" depends_on: - step: "check" allow_failure: false diff --git a/.buildkite/pipeline.schedule-daily.yml b/.buildkite/pipeline.schedule-daily.yml index de122a940b2..aff5682d0ff 100644 --- a/.buildkite/pipeline.schedule-daily.yml +++ b/.buildkite/pipeline.schedule-daily.yml @@ -34,7 +34,7 @@ steps: env: SERVERLESS: "false" FORCE_CHECK_ALL: "true" - STACK_VERSION: 8.19.11-SNAPSHOT + STACK_VERSION: 8.19.20-SNAPSHOT PUBLISH_COVERAGE_REPORTS: "true" depends_on: - step: "check" @@ -48,7 +48,7 @@ steps: env: SERVERLESS: "false" FORCE_CHECK_ALL: "true" - STACK_VERSION: 8.19.11-SNAPSHOT + STACK_VERSION: 8.19.20-SNAPSHOT STACK_LOGSDB_ENABLED: "true" PUBLISH_COVERAGE_REPORTS: "false" depends_on: @@ -86,13 +86,13 @@ steps: if: | build.env('TEST_PACKAGES_BASIC_SUBSCRIPTION') == "true" - - label: "Check integrations local stacks - Stack Version v9.4" + - label: "Check integrations local stacks - Stack Version v9.6" trigger: "integrations" build: env: SERVERLESS: "false" FORCE_CHECK_ALL: "true" - STACK_VERSION: 9.4.0-SNAPSHOT + STACK_VERSION: 9.6.0-SNAPSHOT PUBLISH_COVERAGE_REPORTS: "false" depends_on: - step: "check" diff --git a/.buildkite/pipeline.schedule-weekly.yml b/.buildkite/pipeline.schedule-weekly.yml index ca21a491579..a2a0666b312 100644 --- a/.buildkite/pipeline.schedule-weekly.yml +++ b/.buildkite/pipeline.schedule-weekly.yml @@ -21,20 +21,20 @@ steps: env: SERVERLESS: "false" FORCE_CHECK_ALL: "true" - STACK_VERSION: 8.19.11-SNAPSHOT + STACK_VERSION: 8.19.20-SNAPSHOT PUBLISH_COVERAGE_REPORTS: "false" ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "true" depends_on: - step: "check" allow_failure: false - - label: "Check integrations local stacks and non-wolfi images for Elastic Agent - Stack Version v9.4" + - label: "Check integrations local stacks and non-wolfi images for Elastic Agent - Stack Version v9.6" trigger: "integrations" build: env: SERVERLESS: "false" FORCE_CHECK_ALL: "true" - STACK_VERSION: 9.4.0-SNAPSHOT + STACK_VERSION: 9.6.0-SNAPSHOT PUBLISH_COVERAGE_REPORTS: "false" ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "true" depends_on: diff --git a/.buildkite/pipeline.serverless.yml b/.buildkite/pipeline.serverless.yml index 195bacac447..9a405774a12 100644 --- a/.buildkite/pipeline.serverless.yml +++ b/.buildkite/pipeline.serverless.yml @@ -5,10 +5,10 @@ env: LINUX_AGENT_IMAGE: "golang:${GO_VERSION}" DOCKER_COMPOSE_VERSION: "v2.24.1" DOCKER_VERSION: "false" # not required to set since system tests are not running yet - KIND_VERSION: 'v0.27.0' - K8S_VERSION: 'v1.33.0' + KIND_VERSION: 'v0.32.0' + K8S_VERSION: 'v1.36.1' YQ_VERSION: 'v4.35.2' - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" + IMAGE_UBUNTU_X86_64: "family/platform-obs-integrations-ubuntu-2204" GH_CLI_VERSION: "2.29.0" # This pipeline is intended to test packages with Elastic Serverless SERVERLESS: true @@ -71,14 +71,13 @@ steps: # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/integrations/01-aws-buildkite-oidc.tf # This plugin creates the environment variables required by the service deployer (AWS_SECRET_ACCESS_KEY and AWS_SECRET_KEY_ID) - elastic/oblt-aws-auth#v0.1.0: - duration: 10800 # seconds + duration: 21600 # seconds # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/integrations/01-gcp-buildkite-oidc.tf # This plugin authenticates to CI Google Cloud using the OIDC token. - elastic/oblt-google-auth#v1.3.0: - lifetime: 10800 # seconds + lifetime: 21600 # seconds project-id: "elastic-observability-ci" project-number: "911195782929" - lifetime: 10800 # seconds - avaly/gcp-secret-manager#v1.2.0: env: EC_API_KEY: elastic-cloud-observability-team-qa-api-key diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 57e83fad413..ab11f026725 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -8,10 +8,11 @@ env: YQ_VERSION: 'v4.35.2' JQ_VERSION: '1.7' GH_CLI_VERSION: "2.29.0" + CHANGELOG_SKIP_LABEL: "changelog-link-check:skip" # Agent images used in pipeline steps LINUX_AGENT_IMAGE: "golang:${GO_VERSION}" - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" + IMAGE_UBUNTU_X86_64: "family/platform-obs-integrations-ubuntu-2204" # Elastic package settings # Manage docker output/logs @@ -60,6 +61,47 @@ steps: artifact_paths: - tests-report.xml + - label: ":bash: :snake: Buildkite scripts unit tests" + key: "check-buildkite-scripts" + command: ".buildkite/scripts/run_buildkite_scripts_tests.sh" + agents: + # GCP VM used to match the same Python 3 version available in the steps that test packages + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + + - label: ":bash: Dev scripts unit tests" + key: "check-dev-scripts" + command: ".buildkite/scripts/run_dev_scripts_tests.sh" + + - label: ":ballot_box_with_check: Backports inventory validation" + key: "check-backports-inventory" + command: ".buildkite/scripts/check_backports_inventory.sh" + agents: + image: "${LINUX_AGENT_IMAGE}" + if_changed: + - ".backports.yml" + - ".buildkite/scripts/run_dev_scripts_tests.sh" + - "dev/scripts/**.sh" + - "dev/backports/**" + if: | + build.env('BUILDKITE_PULL_REQUEST') != "false" && + build.env('BUILDKITE_PULL_REQUEST_BASE_BRANCH') == "main" + + - label: ":git: Trigger backport dry-runs" + key: "trigger-backport-dryrun" + command: ".buildkite/scripts/trigger_backport.sh" + agents: + image: "${LINUX_AGENT_IMAGE}" + depends_on: + - step: "check-backports-inventory" + allow_failure: false + if_changed: + - ".backports.yml" + if: | + build.env('BUILDKITE_PULL_REQUEST') != "false" && + build.env('BUILDKITE_PIPELINE_SLUG') == "integrations" && + build.env('BUILDKITE_PULL_REQUEST_BASE_BRANCH') == "main" + - label: ":junit: Sources Junit annotate" agents: # requires at least "bash", "curl" and "git" @@ -76,6 +118,62 @@ steps: run-in-docker: false context: junit-sources + - label: ":scroll: Check changelog versions not in main" + key: "check-changelog-versions-in-main" + command: ".buildkite/scripts/check_changelog_versions_in_main.sh" + agents: + image: "${LINUX_AGENT_IMAGE}" + cpu: "2" + memory: "1G" + if: | + build.env('BUILDKITE_PULL_REQUEST') != "false" && + build.env('BUILDKITE_PULL_REQUEST_BASE_BRANCH') =~ /^backport-/ + if_changed: + - "**/changelog.yml" + + - label: ":scroll: Check changelog PR links" + key: "check-changelog-pr-links" + command: ".buildkite/scripts/check_changelog_entries.sh" + soft_fail: + - exit_status: 2 + agents: + image: "${LINUX_AGENT_IMAGE}" + cpu: "2" + memory: "1G" + plugins: + - elastic/vault-github-token#v0.1.0: + depends_on: + - step: "check-buildkite-scripts" + allow_failure: false + # CHANGELOG_SKIP_LABEL is substituted into the regex at pipeline upload time. + # This works because the variable is defined in the global env section above; + # step-level env vars are not available at upload time and cannot be used here. + if: | + build.env('BUILDKITE_PULL_REQUEST') != "false" && + !(build.env("GITHUB_PR_LABELS") =~ /${CHANGELOG_SKIP_LABEL}/) + if_changed: + - "**/changelog.yml" + + - label: ":busts_in_silhouette: Check backport package owners" + key: "check-backport-owners" + command: ".buildkite/scripts/check_backport_owners.sh" + soft_fail: true + agents: + image: "${LINUX_AGENT_IMAGE}" + cpu: "2" + memory: "1G" + plugins: + - elastic/vault-github-token#v0.1.0: + depends_on: + - step: "check-buildkite-scripts" + allow_failure: false + if: | + build.env('BUILDKITE_PULL_REQUEST') != "false" && + build.env('BUILDKITE_PULL_REQUEST_BASE_BRANCH') =~ /^backport-/ + if_changed: + - "packages/**" + - ".github/CODEOWNERS" + - label: "Trigger integrations" key: "test-integrations" command: ".buildkite/scripts/trigger_integrations_in_parallel.sh" @@ -84,6 +182,8 @@ steps: depends_on: - step: "check" allow_failure: false + - step: "check-buildkite-scripts" + allow_failure: false - wait: ~ continue_on_failure: true diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index cfa177cc02a..e69e992bfbb 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -14,16 +14,29 @@ "skip_ci_labels": [], "skip_target_branches": [], "skip_ci_on_only_changed": [ - "^.github/workflows/", + "^.agents/skills/", + "^.buildkite/pipeline.backport.yml$", + "^.buildkite/pipeline.backport-dispatch.yml$", + "^.buildkite/pipeline.publish.yml$", + "^.buildkite/pipeline.serverless.yml$", + "^.buildkite/pipeline.schedule-daily.yml$", + "^.buildkite/pipeline.schedule-weekly.yml$", + "^.buildkite/pull-requests.json$", + "^.buildkite/scripts/backport_branch.sh$", + "^.buildkite/scripts/build_packages.sh$", + "^.buildkite/scripts/notify_backport_pr.sh$", "^.github/dependabot.yml$", + "^.github/workflows/", + "^.github/stale.yml$", "^.github/ISSUE_TEMPLATE/", - "^docs/", + "^.github/PULL_REQUEST_TEMPLATE.md$", + "^.gitignore", + "^.mergify.yml$", "^catalog-info.yaml$", + "^docs/", "^CODE_OF_CONDUCT.md$", - "^.buildkite/pipeline.schedule-daily.yml$", - "^.buildkite/pipeline.schedule-weekly.yml$", - "^.buildkite/pipeline.backport.yml$", - "^.buildkite/pull-requests.json$" + "^CONTRIBUTING.md$", + "^README.md$" ], "always_require_ci_on_changed": [] }, @@ -75,6 +88,22 @@ "skip_ci_on_only_changed": [], "always_require_ci_on_changed": [] }, + { + "enabled": false, + "pipelineSlug": "integrations-backport-dispatch", + "allow_org_users": true, + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [], + "set_commit_status": false, + "build_on_commit": false, + "build_on_comment": false, + "trigger_comment_regex": "", + "always_trigger_comment_regex": "", + "skip_ci_labels": [], + "skip_target_branches": [], + "skip_ci_on_only_changed": [], + "always_require_ci_on_changed": [] + }, { "enabled": false, "pipelineSlug": "integrations-publish", diff --git a/.buildkite/scripts/backport_branch.sh b/.buildkite/scripts/backport_branch.sh index 0e1dc51007a..38073c3cef7 100755 --- a/.buildkite/scripts/backport_branch.sh +++ b/.buildkite/scripts/backport_branch.sh @@ -1,27 +1,50 @@ #!/bin/bash source .buildkite/scripts/common.sh +source "$(dirname "${BASH_SOURCE[0]}")/backport_branch_lib.sh" set -euo pipefail cleanup_gh() { - pushd $WORKSPACE > /dev/null + pushd "$WORKSPACE" > /dev/null git config remote.origin.url "https://github.com/elastic/integrations.git" popd > /dev/null } -trap cleanup_gh EXIT +cleanup() { + local exit_code=$? + cleanup_gh + exit "${exit_code}" +} +trap cleanup EXIT + +# annotate_and_echo posts a Buildkite annotation and echoes the same message +# to the build log so it is visible in both the annotation panel and the raw output. +# Usage: annotate_and_echo