diff --git a/.adms/python/gitlab.yaml b/.adms/python/gitlab.yaml new file mode 100644 index 000000000..d2b572bfb --- /dev/null +++ b/.adms/python/gitlab.yaml @@ -0,0 +1,9 @@ +# File generated and managed by #dependency-management. +# Changes are subject to overwriting. +# DO NOT EDIT + +variables: + PIP_INDEX_URL: "https://depot-read-api-python.us1.ddbuild.io/magicmirror/magicmirror/@current/simple" + PIP_EXTRA_INDEX_URL: "https://depot-read-api-python.us1.ddbuild.io/magicmirror/testing/@current/simple" + UV_INDEX: "https://depot-read-api-python.us1.ddbuild.io/magicmirror/magicmirror/@current/simple https://depot-read-api-python.us1.ddbuild.io/magicmirror/testing/@current/simple" + UV_DEFAULT_INDEX: "https://depot-read-api-python.us1.ddbuild.io/magicmirror/magicmirror/@current/simple" diff --git a/.github/chainguard/async-profiler-build.ci.sts.yaml b/.github/chainguard/async-profiler-build.ci.sts.yaml index e5b58da5a..f999a5949 100644 --- a/.github/chainguard/async-profiler-build.ci.sts.yaml +++ b/.github/chainguard/async-profiler-build.ci.sts.yaml @@ -1,10 +1,10 @@ # Allow async-profiler-build CI to publish to gh-pages issuer: https://gitlab.ddbuild.io -subject_pattern: "project_path:DataDog/apm-reliability/async-profiler-build:ref_type:branch:ref:.*" +subject_pattern: "project_path:DataDog/java-profiler:ref_type:branch:ref:.*" claim_pattern: - project_path: "DataDog/apm-reliability/async-profiler-build" + project_path: "DataDog/java-profiler" ref_type: "branch" ref: ".*" diff --git a/.github/chainguard/gh-pages.sts.yaml b/.github/chainguard/gh-pages.sts.yaml new file mode 100644 index 000000000..fe170dd12 --- /dev/null +++ b/.github/chainguard/gh-pages.sts.yaml @@ -0,0 +1,26 @@ +# Octo-STS Trust Policy for GitHub Pages Publishing +# This policy allows GitLab CI to push integration test reports to gh-pages branch +# +# Trust Policy Location: .github/chainguard/gh-pages.sts.yaml +# Referenced by: scripts/get-github-token-via-octo-sts.sh (OCTO_STS_POLICY=gh-pages) +# +# How it works: +# 1. GitLab CI generates OIDC token with issuer: https://gitlab.ddbuild.io +# 2. Token includes claims: project_path, ref, namespace_path, etc. +# 3. Octo-STS validates token against this policy +# 4. If valid, Octo-STS returns short-lived GitHub token with specified permissions + +# GitLab OIDC issuer +issuer: https://gitlab.ddbuild.io + +# Match GitLab CI jobs from any branch (needed for PR comments) +# GitLab token includes: project_path=DataDog/java-profiler, ref= +subject_pattern: project_path:DataDog/java-profiler:ref_type:branch:ref:.* + +# GitHub API permissions for the returned token +# contents:write - Required to push to gh-pages branch +permissions: + contents: write + +# Token lifetime (default: 1 hour) +# Short-lived tokens reduce security risk diff --git a/.github/chainguard/update-images.sts.yaml b/.github/chainguard/update-images.sts.yaml new file mode 100644 index 000000000..c09e441b2 --- /dev/null +++ b/.github/chainguard/update-images.sts.yaml @@ -0,0 +1,17 @@ +# Octo-STS Trust Policy for Image Update PRs +# +# Allows the GitLab CI check-image-updates and rebuild-images-pr jobs to push +# branches and create pull requests for CI image reference updates. +# +# Referenced by: scripts/create-image-update-pr.sh (OCTO_STS_POLICY=update-images) + +# GitLab OIDC issuer +issuer: https://gitlab.ddbuild.io + +# Match GitLab CI jobs from the async-profiler-build project on any branch +subject_pattern: project_path:DataDog/java-profiler:ref_type:branch:ref:.* + +# GitHub API permissions +permissions: + contents: write + pull_requests: write diff --git a/.gitignore b/.gitignore index 765412a2e..152d4dfa2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ **/build_*/ **/build-*/ !build-logic/ +!.gitlab/build-deploy/ /nbproject/ /out/ /.idea/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 025512c76..39512a4d7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,24 +1,159 @@ -# Triggers a build within the Datadog infrastructure in the ddprof-build repository -trigger_internal_build: +image: alpine + +variables: + REGISTRY: registry.ddbuild.io + PREPARE_IMAGE: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:latest + # Image with dd-octo-sts for GitHub token exchange (check-image-updates, rebuild-images-pr) + DD_OCTO_STS_IMAGE: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 + FORCE_BUILD: + value: "" + description: "Force build even if no new commits (any non-empty value)" + +default: + tags: ["arch:amd64"] + interruptible: true + before_script: + - '[ "${CANCELLED:-}" != "true" ] || { echo "No PR for this branch — skipping job"; exit 0; }' + +stages: + - images + - generate-signing-key + - prepare + - build + - stresstest + - deploy + - integration-test + - reliability + - benchmarks + - notify + +# Detects newer images in registry and creates GitHub PR with updates +check-image-updates: + stage: images rules: - - if: $CI_COMMIT_BRANCH =~ /release\/.*/ + - if: '$CI_PIPELINE_SOURCE == "schedule" && $CHECK_IMAGE_UPDATES == "true"' + when: always + - if: '$CI_PIPELINE_SOURCE == "web"' + when: manual + allow_failure: true + extends: .bootstrap-gh-tools + tags: ["arch:arm64"] + image: ${DD_OCTO_STS_IMAGE} + id_tokens: + DDOCTOSTS_ID_TOKEN: + aud: dd-octo-sts + script: + - set -euo pipefail + - echo "Checking for image updates..." + - .gitlab/scripts/check-image-updates.sh > updates.json + - | + update_count=$(jq 'length' updates.json) + echo "Found ${update_count} update(s)" + if [ "$update_count" -gt 0 ]; then + echo "Updates available:" + jq . updates.json + .gitlab/scripts/create-image-update-pr.sh updates.json + else + echo "All images are up to date" + fi + artifacts: + when: always + paths: + - updates.json + expire_in: 7 days + +rebuild-images: + stage: images + rules: + - if: '$CI_COMMIT_TAG' when: never - - when: always - allow_failure: false + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: never + - when: manual + allow_failure: true + tags: ["arch:amd64"] + variables: + REBUILD_IMAGES: "" # comma/space-separated short names, or empty = all + image: ${DOCKER_IMAGE} + id_tokens: + DDSIGN_ID_TOKEN: + aud: image-integrity + script: + - set -euo pipefail + - .gitlab/scripts/rebuild-images.sh + artifacts: + when: always + paths: + - updates.json + expire_in: 1 day + +rebuild-images-pr: + stage: images + rules: + - if: '$CI_COMMIT_TAG' + when: never + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: never + - when: on_success + needs: + - job: rebuild-images + artifacts: true + extends: .bootstrap-gh-tools + tags: ["arch:arm64"] + image: ${DD_OCTO_STS_IMAGE} + id_tokens: + DDOCTOSTS_ID_TOKEN: + aud: dd-octo-sts + script: + - set -euo pipefail + - .gitlab/scripts/create-image-update-pr.sh updates.json + +create_key: + stage: generate-signing-key + when: manual + needs: [] + tags: ["arch:amd64"] variables: - DOWNSTREAM_BRANCH: "main" - UPSTREAM_PROJECT: ${CI_PROJECT_PATH} - UPSTREAM_PROJECT_NAME: ${CI_PROJECT_NAME} - UPSTREAM_BRANCH: ${CI_COMMIT_BRANCH} - UPSTREAM_COMMIT_SHA: ${CI_COMMIT_SHA} - DDPROF_DEFAULT_BRANCH: "main" - DDPROF_COMMIT_BRANCH: ${CI_COMMIT_BRANCH} - DDROF_COMMIT_SHA: ${CI_COMMIT_SHA} - DPROF_SHORT_COMMIT_SHA: ${CI_COMMIT_SHORT_SHA} - DDPROF_COMMIT_TAG: ${CI_COMMIT_TAG} + PROJECT_NAME: "java-profiler" + EXPORT_TO_KEYSERVER: "true" + KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: java-profiler + image: $REGISTRY/ci/agent-key-management-tools/gpg:1 + script: + - /create.sh + artifacts: + expire_in: 13 mos + paths: + - pubkeys + +# Shared version detection used by benchmarks and reliability pipelines +get-versions: + extends: .get-versions + needs: + - job: prepare:start + artifacts: false + +# Triggered externally from async-profiler-build with JDK build parameters; +# kept as a child pipeline because it is mutually exclusive with the main build +jdk-integration-test: + stage: build + rules: + - if: '$JDK_VERSION == null || $DEBUG_LEVEL == null || $HASH == null || $DOWNSTREAM == null' + when: never + - if: '$CI_PIPELINE_SOURCE == "trigger" || $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "web"' + when: always + allow_failure: false + - when: always trigger: - project: DataDog/apm-reliability/async-profiler-build + include: .gitlab/jdk-integration/.gitlab-ci.yml strategy: depend - branch: $DOWNSTREAM_BRANCH forward: pipeline_variables: true + +include: + - local: .gitlab/common.yml + - local: .adms/python/gitlab.yaml + - local: .gitlab/benchmarks/images.yml + - local: .gitlab/build-deploy/images.yml + - local: .gitlab/build-deploy/.gitlab-ci.yml + - local: .gitlab/benchmarks/.gitlab-ci.yml + - local: .gitlab/reliability/.gitlab-ci.yml diff --git a/.gitlab/Dockerfile.datadog-ci b/.gitlab/Dockerfile.datadog-ci new file mode 100644 index 000000000..7a1873dfe --- /dev/null +++ b/.gitlab/Dockerfile.datadog-ci @@ -0,0 +1,60 @@ +ARG BASEIMAGE=registry.ddbuild.io/images/base/gbi-ubuntu_2404:release +FROM ${BASEIMAGE} + +USER root + +# Create non-root user for security +RUN useradd --create-home --shell /bin/bash --uid 1001 ci-user + +# Install Node.js 20 and npm +# Default seems to be 14 which does not work with datadog-ci +RUN set -x \ + && apt-get update && apt-get -y install --no-install-recommends curl xz-utils\ + && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \ + && apt-get install -y nodejs \ + pipx=1.4.3-1 \ + binutils \ + jq \ + && npm install -g @datadog/datadog-ci@3.16.0 \ + && apt-get -y clean \ + && rm -rf /var/lib/apt/lists/* + +# Install GitHub CLI +RUN set -x \ + && curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg \ + -o /usr/share/keyrings/githubcli-archive-keyring.gpg \ + && chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" \ + > /etc/apt/sources.list.d/github-cli.list \ + && apt-get update \ + && apt-get install -y gh \ + && apt-get -y clean \ + && rm -rf /var/lib/apt/lists/* + +# awscli is not available in Ubuntu 2404 for some inexplicable reason so lets install in via other means +RUN PIPX_HOME=/opt/pipx PIPX_BIN_DIR=/usr/local/bin pipx install awscli + +# Install Go 1.22.3 +RUN set -x \ + && curl -LO https://golang.org/dl/go1.22.3.linux-amd64.tar.gz \ + && tar -C /usr/local -xzf go1.22.3.linux-amd64.tar.gz \ + && rm go1.22.3.linux-amd64.tar.gz + +# Set up Go environment for root and install Crane +ENV PATH="/usr/local/go/bin:${PATH}" +ENV GOPATH="/root/go" +ENV GOBIN="/usr/local/bin" + +# Install Crane version 0.19.1 directly to /usr/local/bin so it's available for all users +RUN set -x \ + && go install github.com/google/go-containerregistry/cmd/crane@v0.19.1 + +# Switch to non-root user +USER ci-user +WORKDIR /home/ci-user + +# Set PATH for the ci-user (crane is now in /usr/local/bin) +ENV PATH="/usr/local/go/bin:/usr/local/bin:${PATH}" + +# Verify installation (as non-root user) +RUN node -v && npm -v && go version && crane version && datadog-ci --help && jq --version && gh --version diff --git a/.gitlab/base/Dockerfile b/.gitlab/base/Dockerfile new file mode 100644 index 000000000..ef9a8a004 --- /dev/null +++ b/.gitlab/base/Dockerfile @@ -0,0 +1,10 @@ +ARG BASE_IMAGE=openjdk:11-slim-buster +FROM ${BASE_IMAGE} as base +ARG CI_JOB_TOKEN +WORKDIR /root + +RUN mkdir -p /usr/share/man/man1 # https://github.com/debuerreotype/docker-debian-artifacts/issues/24 +RUN (apt update && DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends curl git moreutils awscli amazon-ecr-credential-helper gnupg2 npm build-essential wget bsdmainutils clang libclang-rt-dev jq zip unzip maven) || true +RUN (apk update && apk add curl git moreutils aws-cli docker-credential-ecr-login gnupg alpine-sdk build-base wget npm hexdump linux-headers clang compiler-rt bash jq gradle zip unzip) || true +RUN npm install -g --save-dev @datadog/datadog-ci +RUN rm -rf "/var/lib/apt/lists/*" \ No newline at end of file diff --git a/.gitlab/base/centos7/Dockerfile b/.gitlab/base/centos7/Dockerfile new file mode 100644 index 000000000..004a00128 --- /dev/null +++ b/.gitlab/base/centos7/Dockerfile @@ -0,0 +1,43 @@ +ARG BASE_IMAGE=openjdk:11-slim-buster +FROM ${BASE_IMAGE} as base +ARG CI_JOB_TOKEN +WORKDIR /root + +# 1. Replace dead mirrorlist entries with HTTPS vault URLs +RUN set -eux; \ + sed -i -e 's/^mirrorlist/#mirrorlist/' \ + -e 's|^#baseurl=http://mirror.centos.org|baseurl=https://vault.centos.org|' \ + /etc/yum.repos.d/CentOS-*.repo + +# 2. Add a vault mirror that still contains Software Collections +RUN cat > /etc/yum.repos.d/CentOS-SCLo-Vault.repo <<'EOF' +[centos-sclo-rh] +name=CentOS-7 - SCLo rh (Rocky Vault) +baseurl=https://dl.rockylinux.org/vault/centos/7.9.2009/sclo/$basearch/rh/ +gpgcheck=0 +enabled=1 + +[centos-sclo-sclo] +name=CentOS-7 - SCLo sclo (Rocky Vault) +baseurl=https://dl.rockylinux.org/vault/centos/7.9.2009/sclo/$basearch/sclo/ +gpgcheck=0 +enabled=1 +EOF + +# 3. Expose devtoolset-11 binaries & libs by default (they are installed a bit later) +ENV PATH="/opt/rh/devtoolset-11/root/usr/bin:${PATH}" \ + LD_LIBRARY_PATH="/opt/rh/devtoolset-11/root/usr/lib64:${LD_LIBRARY_PATH}" + +RUN yum -y clean all +RUN yum -y update && yum -y install scl-utils devtoolset-11 devtoolset-11-toolchain curl zip unzip git libstdc++-static make which wget cmake binutils +RUN yum -y clean all +RUN (curl -s "https://get.sdkman.io" | bash) +RUN (source ~/.sdkman/bin/sdkman-init.sh && sdk install java 21.0.3-tem) +RUN (curl -sL https://rpm.nodesource.com/setup_16.x | bash -) +# installing JQ requires two steps - adding the repo and then installing the tool +RUN yum install -y epel-release +RUN yum install -y jq +# now install nodejs and datadog CI support +RUN yum -y install nodejs +RUN npm install -g --save-dev @datadog/datadog-ci +RUN rm -rf "/var/lib/apt/lists/*" \ No newline at end of file diff --git a/.gitlab/benchmarks/.gitlab-ci.yml b/.gitlab/benchmarks/.gitlab-ci.yml new file mode 100644 index 000000000..335118d91 --- /dev/null +++ b/.gitlab/benchmarks/.gitlab-ci.yml @@ -0,0 +1,101 @@ +variables: + PREPARE_IMAGE: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:latest + DD_OCTO_STS_IMAGE: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 + +.benchmark_job: + extends: .deploy-sa + stage: benchmarks + timeout: 6h + variables: + ITERATIONS: "${BENCHMARK_ITERATIONS:-1}" + MODES: "${BENCHMARK_MODES:-cpu,wall,alloc,memleak}" + needs: + - job: get-versions + artifacts: true + - job: deploy-artifact + artifacts: false + rules: + - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null' + when: never + - if: '$CI_PIPELINE_SOURCE == "trigger" || $CI_PIPELINE_SOURCE == "pipeline"' + when: on_success + - if: '$CI_PIPELINE_SOURCE == "web"' + when: manual + allow_failure: true + - if: '$CI_PIPELINE_SOURCE == "push"' + when: manual + allow_failure: true + script: | + # setup the env + export ARTIFACTS_DIR="$(pwd)/reports" && (mkdir "${ARTIFACTS_DIR}" || :) + export CANDIDATE_VERSION=${CURRENT_VERSION} + export BASELINE_VERSION=${PREVIOUS_VERSION} + export PLATFORM_DIR=".benchmarks/platform" + + # check for missing candidate version + if [ -z "${CANDIDATE_VERSION}" ]; then echo "Missing candidate version. Skipping."; exit 0; fi + + # fetch the common platform scripts + git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/" + git clone --branch dd-trace-go https://github.com/DataDog/benchmarking-platform ${PLATFORM_DIR} + + # apply the specific step scripts + cp -r .gitlab/benchmarks/steps/* ${PLATFORM_DIR}/steps/ + chmod a+x ${PLATFORM_DIR}/steps/* + + # check for mode validity + ${PLATFORM_DIR}/steps/check_modes.sh + if [ "$(cat .job_status)" == "SKIP" ]; then exit 0; fi + + # run benchmarks + ${PLATFORM_DIR}/steps/capture-hardware-software-info.sh + ${PLATFORM_DIR}/steps/run-benchmarks.sh + ${PLATFORM_DIR}/steps/analyze-results.sh + ${PLATFORM_DIR}/steps/upload-results-to-s3.sh + ${PLATFORM_DIR}/steps/post-pr-comment.sh + parallel: + matrix: + - RUN_MODE: ["cpu", "wall", "alloc", "memleak", "cpu,wall", "memleak,alloc", "cpu,wall,alloc,memleak"] + artifacts: + when: always + name: "reports" + paths: + - reports/ + expire_in: 3 months + +benchmarks-candidate-amd64: + extends: .benchmark_job + tags: ["arch:amd64"] + image: $BENCHMARK_IMAGE_AMD64 + +benchmarks-candidate-aarch64: + extends: .benchmark_job + tags: ["arch:arm64"] + image: $BENCHMARK_IMAGE_ARM64 + variables: + KUBERNETES_MEMORY_REQUEST: 200Gi + KUBERNETES_MEMORY_LIMIT: 200Gi + +publish-benchmark-gh-pages: + stage: benchmarks + tags: ["arch:arm64"] + image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 + id_tokens: + DDOCTOSTS_ID_TOKEN: + aud: dd-octo-sts + needs: + - job: benchmarks-candidate-amd64 + artifacts: true + - job: benchmarks-candidate-aarch64 + artifacts: true + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "main"' + when: always + timeout: 10m + script: + - ./.gitlab/benchmarks/publish-gh-pages.sh + allow_failure: true + +include: + - local: .gitlab/common.yml + - local: .gitlab/benchmarks/images.yml diff --git a/.gitlab/benchmarks/docker/Dockerfile b/.gitlab/benchmarks/docker/Dockerfile new file mode 100644 index 000000000..cb3833a71 --- /dev/null +++ b/.gitlab/benchmarks/docker/Dockerfile @@ -0,0 +1,5 @@ +ARG BASE_IMAGE=ubuntu:22.04 +FROM ${BASE_IMAGE} + +COPY ./setup.sh /tmp/setup.sh +RUN /tmp/setup.sh \ No newline at end of file diff --git a/.gitlab/benchmarks/docker/setup.sh b/.gitlab/benchmarks/docker/setup.sh new file mode 100755 index 000000000..4b23f0f85 --- /dev/null +++ b/.gitlab/benchmarks/docker/setup.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e +set -x + +apt update && apt install -y wget curl zip unzip time maven jq git libjemalloc-dev libtcmalloc-minimal4 + +pip install numpy matplotlib + +# debug output +#echo $JAVA_HOME +#mvn -version + +# retrieve the standard benchmark apps +mkdir -p /var/lib/benchmarks +wget -q -O /var/lib/benchmarks/renaissance.jar https://github.com/renaissance-benchmarks/renaissance/releases/download/v0.15.0/renaissance-mit-0.15.0.jar +wget -q -O /var/lib/dacapo.jar https://netix.dl.sourceforge.net/project/dacapobench/9.12-bach-MR1/dacapo-9.12-MR1-bach.jar + diff --git a/.gitlab/benchmarks/generate-run-json.sh b/.gitlab/benchmarks/generate-run-json.sh new file mode 100755 index 000000000..541be33cd --- /dev/null +++ b/.gitlab/benchmarks/generate-run-json.sh @@ -0,0 +1,151 @@ +#!/bin/bash + +# generate-run-json.sh - Generate run JSON for benchmark tests +# +# Usage: generate-run-json.sh [reports-dir] +# +# Parses benchmark report files and outputs a JSON object +# suitable for update-history.sh. Reads CI environment variables for metadata. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="${SCRIPT_DIR}/../.." +REPORTS_DIR="${1:-${PROJECT_ROOT}/reports}" + +# Read metadata from environment or defaults +TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +PIPELINE_ID="${CI_PIPELINE_ID:-0}" +PIPELINE_URL="${CI_PIPELINE_URL:-#}" +DDPROF_BRANCH="${DDPROF_COMMIT_BRANCH:-${CI_COMMIT_BRANCH:-main}}" +DDPROF_SHA="${DDPROF_COMMIT_SHA:-${CI_COMMIT_SHA:-unknown}}" + +# Read version from environment or version.txt +LIB_VERSION="${CURRENT_VERSION:-unknown}" +if [ "${LIB_VERSION}" = "unknown" ] && [ -f "${PROJECT_ROOT}/version.txt" ]; then + LIB_VERSION=$(awk -F: '{print $NF}' "${PROJECT_ROOT}/version.txt" | tr -d ' ') +fi + +# Lookup PR for branch +PR_JSON="{}" +if [ -x "${SCRIPT_DIR}/../common/lookup-pr.sh" ]; then + PR_JSON=$("${SCRIPT_DIR}/../common/lookup-pr.sh" "${DDPROF_BRANCH}" 2>/dev/null) || PR_JSON="{}" +fi + +# Parse benchmark results +python3 <= 5: + break + except Exception: + pass + + results["architectures"] = sorted(architectures) + results["modes_tested"] = sorted(modes) + + return results + +# Analyze results +summary = analyze_benchmarks(reports_dir) + +# Determine overall status +if summary["regression_detected"]: + status = "failed" +elif summary["total_benchmarks"] > 0: + status = "passed" +else: + status = "unknown" + +# Build run JSON +run = { + "id": pipeline_id, + "timestamp": timestamp, + "ddprof_branch": ddprof_branch, + "ddprof_sha": ddprof_sha, + "ddprof_pr": pr_info if pr_info.get("number") else None, + "pipeline": { + "id": pipeline_id, + "url": pipeline_url + }, + "lib_version": lib_version, + "status": status, + "summary": summary +} + +# Output JSON +print(json.dumps(run, indent=2)) +EOF diff --git a/.gitlab/benchmarks/images.yml b/.gitlab/benchmarks/images.yml new file mode 100644 index 000000000..007a97f99 --- /dev/null +++ b/.gitlab/benchmarks/images.yml @@ -0,0 +1,11 @@ +stages: + - images + +variables: + BASE_BENCHMARK_IMAGE_NAME: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:latest + BASE_CI_IMAGE_NAME: registry.ddbuild.io/ci/async-profiler-build + + # Generated by https://gitlab.ddbuild.io/DataDog/apm-reliability/async-profiler-build/-/jobs/1476153903 + BENCHMARK_IMAGE_AMD64: registry.ddbuild.io/ci/async-profiler-build-amd64:v100389018-amd64-benchmarks@sha256:de8bfba2340dcf77879d5d6d1a656500aa007d0711c9297f1b7144213ea12894 + # Generated by https://gitlab.ddbuild.io/DataDog/apm-reliability/async-profiler-build/-/jobs/1476153903 + BENCHMARK_IMAGE_ARM64: registry.ddbuild.io/ci/async-profiler-build-arm64:v100389018-arm64-benchmarks@sha256:55c438254df680b94ece4b34610a0c0bf499273e2ec73d4ed103985ab117b8db diff --git a/.gitlab/benchmarks/publish-gh-pages.sh b/.gitlab/benchmarks/publish-gh-pages.sh new file mode 100755 index 000000000..03b65460e --- /dev/null +++ b/.gitlab/benchmarks/publish-gh-pages.sh @@ -0,0 +1,170 @@ +#!/bin/bash + +# publish-gh-pages.sh - Publish benchmark reports to GitHub Pages +# +# Usage: publish-gh-pages.sh [reports-dir] +# +# Updates benchmark history and regenerates GitHub Pages site. +# Reports are available at: https://datadog.github.io/async-profiler-build/benchmarks/ +# +# In CI: Uses Octo-STS for secure, short-lived GitHub tokens (no secrets needed) +# Locally: Use 'devflow gitlab auth' for Octo-STS, or set GITHUB_TOKEN env var + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="${SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPORTS_DIR="${1:-${PROJECT_ROOT}/reports}" +export MAX_HISTORY=10 + +# GitHub repo for Pages +GITHUB_REPO="DataDog/java-profiler" +PAGES_URL="https://datadog.github.io/java-profiler" + +# Colors +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +NC='\033[0m' + +log_info() { echo -e "${GREEN}[INFO]${NC} $*"; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } +log_error() { echo -e "${RED}[ERROR]${NC} $*"; } + +# Obtain GitHub token +obtain_github_token() { + # Try dd-octo-sts CLI (works in CI with DDOCTOSTS_ID_TOKEN) + if command -v dd-octo-sts >/dev/null 2>&1 && [ -n "${DDOCTOSTS_ID_TOKEN:-}" ]; then + log_info "Obtaining GitHub token via dd-octo-sts CLI..." + # Policy name matches the .sts.yaml filename (without extension) + + # Run dd-octo-sts and capture only stdout (don't capture stderr to avoid error messages in token) + local TOKEN_OUTPUT + local TOKEN_EXIT_CODE + TOKEN_OUTPUT=$(dd-octo-sts token --scope DataDog/java-profiler --policy async-profiler-build.ci 2>/tmp/dd-octo-sts-error.log) + TOKEN_EXIT_CODE=$? + + if [ $TOKEN_EXIT_CODE -eq 0 ] && [ -n "${TOKEN_OUTPUT}" ]; then + # Validate token format (GitHub tokens start with ghs_, ghp_, or look like JWT) + if [[ "${TOKEN_OUTPUT}" =~ ^(ghs_|ghp_|v1\.|[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.) ]]; then + GITHUB_TOKEN="${TOKEN_OUTPUT}" + log_info "GitHub token obtained via dd-octo-sts (expires in 1 hour)" + return 0 + else + log_warn "dd-octo-sts returned invalid token format (first 50 chars): ${TOKEN_OUTPUT:0:50}" + fi + else + log_warn "dd-octo-sts token exchange failed (exit code: ${TOKEN_EXIT_CODE})" + if [ -s /tmp/dd-octo-sts-error.log ]; then + log_warn "dd-octo-sts error output:" + cat /tmp/dd-octo-sts-error.log | head -10 >&2 + fi + fi + fi + + # Fall back to GITHUB_TOKEN environment variable + if [ -n "${GITHUB_TOKEN:-}" ]; then + log_info "Using GITHUB_TOKEN from environment" + return 0 + fi + + return 1 +} + +if ! obtain_github_token; then + log_error "Failed to obtain GitHub token" + log_error "Options:" + log_error " 1. Run in GitLab CI with dd-octo-sts-ci-base image and DDOCTOSTS_ID_TOKEN" + log_error " 2. Set GITHUB_TOKEN env var (PAT with 'repo' scope)" + exit 1 +fi + +# Create temporary directory for gh-pages content +WORK_DIR=$(mktemp -d) +RUN_JSON_FILE=$(mktemp) +# shellcheck disable=SC2064 # Intentional: capture values at setup time +trap "rm -rf ${WORK_DIR} ${RUN_JSON_FILE}" EXIT + +log_info "Preparing gh-pages content in: ${WORK_DIR}" + +# Clone gh-pages branch (or create if doesn't exist) +log_info "Cloning gh-pages branch..." +cd "${WORK_DIR}" + +if git clone --depth 1 --branch gh-pages "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPO}.git" pages 2>/dev/null; then + cd pages + log_info "Cloned existing gh-pages branch" +else + log_info "Creating new gh-pages branch..." + mkdir pages && cd pages + git init + git remote add origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPO}.git" + git checkout -b gh-pages +fi + +# Create Jekyll config if not exists +if [ ! -f "_config.yml" ]; then + cat > "_config.yml" </dev/null 2>&1 && pwd )" + +source ${DIR}/config-benchmark-analyzer.sh + +BASELINE_JSON=$(find $ARTIFACTS_DIR -name 'baseline*.json' 2>/dev/null) +CANDIDATE_JSON=$(find $ARTIFACTS_DIR -name 'candidate*.json' 2>/dev/null) + +# Append $SUFFIX for unique names to match the configurations +ARCH=`uname -m` +SUFFIX=$(echo "$RUN_MODE" | tr ',' '_') +SUFFIX=${ARCH}_${SUFFIX} + +benchmark_analyzer analyze \ + --format=html \ + --outpath="$ARTIFACTS_DIR/baseline-analysis_${SUFFIX}.html" \ + "${BASELINE_JSON}" + +benchmark_analyzer analyze \ + --format=html \ + --outpath="$ARTIFACTS_DIR/candidate-analysis_${SUFFIX}.html" \ + "${CANDIDATE_JSON}" + +benchmark_analyzer compare pairwise \ + --baseline='{"config":"baseline"}' \ + --candidate='{"config":"candidate"}' \ + --format=html \ + --outpath="$ARTIFACTS_DIR/comparison-baseline-vs-candidate_${SUFFIX}.html" \ + "${BASELINE_JSON}" "${CANDIDATE_JSON}" + +benchmark_analyzer compare pairwise \ + --baseline='{"config":"baseline"}' \ + --candidate='{"config":"candidate"}' \ + --format=md \ + --outpath="$ARTIFACTS_DIR/comparison-baseline-vs-candidate_${SUFFIX}.md" \ + "${BASELINE_JSON}" "${CANDIDATE_JSON}" diff --git a/.gitlab/benchmarks/steps/check_modes.sh b/.gitlab/benchmarks/steps/check_modes.sh new file mode 100755 index 000000000..1ca131de6 --- /dev/null +++ b/.gitlab/benchmarks/steps/check_modes.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +echo "Checking the requested modes [${MODES:=cpu,wall,alloc,memleak}] in [$RUN_MODE] ..." +RESULT="SKIP" +for MODE in ${MODES//,/$IFS}; do + echo ":: $MODE -< $RUN_MODE" + if [[ $RUN_MODE =~ .*?$MODE.* ]]; then + RESULT="CONTINUE" + break + fi +done +echo $RESULT > .job_status +if [ "$RESULT" == "SKIP" ]; then + echo "Skipping run for mode set: [$RUN_MODE]. It does not include any mode from ${MODES}." +fi \ No newline at end of file diff --git a/.gitlab/benchmarks/steps/config-benchmark-analyzer.sh b/.gitlab/benchmarks/steps/config-benchmark-analyzer.sh new file mode 100755 index 000000000..963472bc7 --- /dev/null +++ b/.gitlab/benchmarks/steps/config-benchmark-analyzer.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +export UNCONFIDENCE_THRESHOLD=1.5 +export DISABLE_NHST=1 +export CI_TYPE=theoretical +export MD_REPORT_ONLY_CHANGES=1 diff --git a/.gitlab/benchmarks/steps/mem_watch.sh b/.gitlab/benchmarks/steps/mem_watch.sh new file mode 100755 index 000000000..6b4e7e66d --- /dev/null +++ b/.gitlab/benchmarks/steps/mem_watch.sh @@ -0,0 +1,14 @@ +#! /bin/bash +set +x +ctl_file=$1 +out=$2 +while [ -f $ctl_file ]; do + pid=$(ps ax | grep 'java' | grep "${ctl_file}" | grep -v 'grep' | grep -v 'time' | sed -e 's/^[[:space:]]*//' | cut -f1 -d' ') + if [ -n "${pid}" ]; then + rss="$(cat /proc/${pid}/smaps_rollup | grep 'Rss:' | cut -f2 -d':' | sed -e 's/^[[:space:]]*//' | cut -f1 -d' ' 2>/dev/null)" + if [ -n "$rss" ]; then + echo "mem: $rss" | tee -a $out + fi + fi + sleep 5 +done \ No newline at end of file diff --git a/.gitlab/benchmarks/steps/post-pr-comment.sh b/.gitlab/benchmarks/steps/post-pr-comment.sh new file mode 100755 index 000000000..39134991a --- /dev/null +++ b/.gitlab/benchmarks/steps/post-pr-comment.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +if [ -z "${UPSTREAM_PROJECT_NAME}" ]; then + echo "No upstream project defined. Skipping." + exit 0 +fi + +# Append $SUFFIX for unique names to match the configurations +ARCH=`uname -m` +SUFFIX=$(echo "$RUN_MODE" | tr ',' '_') +SUFFIX=${ARCH}_${SUFFIX} + +cat "$ARTIFACTS_DIR/comparison-baseline-vs-candidate_${SUFFIX}.md" | pr-commenter --for-repo="$UPSTREAM_PROJECT_NAME" --for-pr="$UPSTREAM_BRANCH" --header="Benchmarks [$ARCH $RUN_MODE]" --on-duplicate=replace diff --git a/.gitlab/benchmarks/steps/run-benchmarks.sh b/.gitlab/benchmarks/steps/run-benchmarks.sh new file mode 100755 index 000000000..0e0f08d5e --- /dev/null +++ b/.gitlab/benchmarks/steps/run-benchmarks.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +set -euo pipefail +HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +mkdir -p "${ARTIFACTS_DIR}" || : + +LATEST_VERSION="" # initially empty +retries=0 +if [ -z "$CANDIDATE_VERSION" ] || [ -z "$BASELINE_VERSION" ]; then + echo "Retrieving the latest released version ..." + while [ -z "$LATEST_VERSION" ] && [ $retries -lt 5 ]; do + # default to the latest released version + LATEST_VERSION=$(curl -s https://api.github.com/repos/DataDog/java-profiler/releases/latest | grep "tag_name" | cut -f2 -d: | sed -e "s#\"v_##g" | sed -e "s#\",##g" | sed -e "s# ##g") + if [ -z "$LATEST_VERSION" ]; then + echo "Can not retrieve the baseline version from Github. Waiting 10s ..." + sleep 10 + fi + retries=$((retries+1)) + done + if [ -z "$LATEST_VERSION" ]; then + echo "Unable to resolve the latest released version!" + exit 1 + fi +fi +CANDIDATE_VERSION=${CANDIDATE_VERSION:=$LATEST_VERSION} +BASELINE_VERSION=${BASELINE_VERSION:=$LATEST_VERSION} +RUN_MODE=${RUN_MODE:=cpu,wall} +MODES=${RUN_MODE} + +SUFFIX=$(echo "$MODES" | tr ',' '_') + +echo "Setting baseline to: $BASELINE_VERSION" +echo "Setting candidate version to: $CANDIDATE_VERSION" + +"${HERE}"/run_benchmark.sh --iterations ${BENCHMARK_ITERATIONS:-5} --version $CANDIDATE_VERSION --output $ARTIFACTS_DIR/candidate-$SUFFIX.json --modes "${RUN_MODE}" --config candidate + +if [ "$BASELINE_VERSION" != "$CANDIDATE_VERSION" ]; then + # run comparison against the baseline version, same profiling modes + "${HERE}"/run_benchmark.sh --iterations ${BENCHMARK_ITERATIONS:-5} --version $BASELINE_VERSION --output $ARTIFACTS_DIR/baseline-$SUFFIX.json --modes "${RUN_MODE}" --config baseline +else + # run comparison against the same version but with profiling disabled + "${HERE}"/run_benchmark.sh --iterations ${BENCHMARK_ITERATIONS:-5} --version $CANDIDATE_VERSION --output $ARTIFACTS_DIR/baseline-$SUFFIX.json --modes "" --config baseline +fi + diff --git a/.gitlab/benchmarks/steps/run_benchmark.sh b/.gitlab/benchmarks/steps/run_benchmark.sh new file mode 100755 index 000000000..5f8960d2c --- /dev/null +++ b/.gitlab/benchmarks/steps/run_benchmark.sh @@ -0,0 +1,120 @@ +#!/bin/bash + +# Install SDKMAN and JDK +export SDKMAN_DIR="/usr/local/sdkman" && curl -s "https://get.sdkman.io" | bash +source /usr/local/sdkman/bin/sdkman-init.sh +sdk install java 11.0.28-tem + +export JAVA_HOME=`sdk home java 11.0.28-tem` + +function usage() { + echo "./run_benchmark.sh --iterations (default 10) --version --output --modes (default 'cpu,wall') --config [candidate|baseline]" + echo "Modes are encoded as comma separated list of modes. Supported modes are: cpu, wall and alloc" +} + +HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +ITERATIONS=5 +MODES="cpu,wall" +while [ $# -gt 0 ]; do + case "$1" in + "--iterations") ITERATIONS=$2; shift; shift;; + "--version") VERSION=$2; shift; shift;; + "--output") JSON_OUTPUT=$2; shift; shift;; + "--modes") MODES=$2; shift; shift;; + "--config") CONFIG=$2; shift; shift;; + "--help") + usage + exit 0 + ;; + *) usage; exit 1;; + esac +done + +if [ -z "$VERSION" ] || [ -z "$JSON_OUTPUT" ] || [ -z "$CONFIG" ]; then + usage + exit 1 +fi + +JFR_DIR="${ARTIFACTS_DIR}/jfr" +TXT_OUTPUT="$JSON_OUTPUT.txt" + +mkdir -p $JFR_DIR + +CPU="off" +WALL="off" +ALLOC="off" +MEMLEAK="off" +for MODE in ${MODES//,/$IFS}; do + if [ "$MODE" == "cpu" ]; then + CPU_ARG="cpu=10m" + CPU="on" + elif [ "$MODE" == "wall" ]; then + WALL_ARG="wall=10m" + WALL="on" + elif [ "$MODE" == "alloc" ]; then + ALLOC_ARG="alloc=262144" + ALLOC="on" + elif [ "$MODE" == "memleak" ]; then + MEMLEAK_ARG="memleak=262144" + MEMLEAK="on" + fi +done + +mvn org.apache.maven.plugins:maven-dependency-plugin:2.1:get \ + -DrepoUrl=https://central.sonatype.com/repository/maven-snapshots/ \ + -Dartifact=com.datadoghq:ddprof:$VERSION + +mkdir -p /var/lib/datadog/$VERSION +rm -rf /var/lib/datadog/$VERSION/* + + +# The directory contains platform specific profiling library +ARCH=`uname -m` +ARCH_DIR="" +if [ "$ARCH" == "x86_64" ]; then + ARCH_DIR=linux-x64 +elif [ "$ARCH" == "aarch64" ]; then + ARCH_DIR=linux-arm64 +fi + +unzip -q -d /var/lib/datadog/$VERSION /root/.m2/repository/com/datadoghq/ddprof/$VERSION/ddprof-$VERSION.jar +AGENT_LIB=$(find /var/lib/datadog/$VERSION -name 'libjavaProfiler.so' | fgrep ${ARCH_DIR}/) + +ls -la /var/lib/datadog/$VERSION/* +echo $AGENT_LIB + +JAVA_VERSION=$(cat $JAVA_HOME/release | fgrep 'JAVA_VERSION=' | cut -f2 -d'=') +if [ -z "$JAVA_VERSION" ]; then + JAVA_VERSION=$(cat $JAVA_HOME/release | fgrep 'JVM_VERSION=' | cut -f2 -d'=') +fi + +for test_item in 'dotty|-r 10' 'finagle-http|-r 4' 'finagle-chirper|-r 4' 'page-rank|-r 3' 'future-genetic|-r 5' 'akka-uct|-r 2' 'movie-lens|-r 3' 'scala-doku|-r 4' 'chi-square|-r 6' 'fj-kmeans|-r 15' 'dec-tree|-r 15' 'naive-bayes|-r 15' 'als|-r 10' 'par-mnemonics|-r 5' 'scala-kmeans|-r 60' 'philosophers|-r 2' 'log-regression|-r 30' 'gauss-mix|-r 8' 'mnemonics|-r 4' ; do + test=$(echo $test_item | cut -f1 -d'|') + param=$(echo $test_item | cut -f2 -d'|') + + echo "==== renaissance:$test (cpu=$CPU, wall=$WALL, alloc=$ALLOC, memleak=$MEMLEAK)" | tee -a $TXT_OUTPUT + echo "== config: $CONFIG" | tee -a $TXT_OUTPUT + echo "== ddprof: $VERSION" | tee -a $TXT_OUTPUT + echo "== java: $JAVA_VERSION" | tee -a $TXT_OUTPUT + + chmod a+x "${HERE}"/mem_watch.sh + for i in $( seq 1 $ITERATIONS ); do + if [ ! -z "$MODES" ]; then + AGENT_ARG="-agentpath:${AGENT_LIB}=start,jfr=7,file=${JFR_DIR}/${CONFIG}_${test}_${i}.jfr,jstackdepth=512,cstack=dwarf,safemode=0,$CPU_ARG,$WALL_ARG,$ALLOC_ARG,$MEMLEAK_ARG" + fi + + CONTROL_FILE=".running_${CONFIG}_${test}_${i}" + touch $CONTROL_FILE + "${HERE}"/mem_watch.sh $CONTROL_FILE $TXT_OUTPUT & + TIME=$(/usr/bin/time -f "# %e" java -Xms1g -Xmx1g -XX:+AlwaysPreTouch -Dctl=$CONTROL_FILE $AGENT_ARG -jar /var/lib/benchmarks/renaissance.jar $test $param 2>&1 | fgrep '#' | cut -f2 -d' ') + rm -f $CONTROL_FILE + + echo "time: $TIME" | tee -a $TXT_OUTPUT + done +done + +benchmark_analyzer convert \ + --framework=JavaProfilerRenaissance \ + --extra_params="{\"iterations\": \"${ITERATIONS}\", \"modes\": \"${MODES}\"}" \ + --outpath="$JSON_OUTPUT" \ + $TXT_OUTPUT diff --git a/.gitlab/build-deploy/.gitlab-ci.yml b/.gitlab/build-deploy/.gitlab-ci.yml new file mode 100644 index 000000000..3e23e1ebc --- /dev/null +++ b/.gitlab/build-deploy/.gitlab-ci.yml @@ -0,0 +1,326 @@ +image: alpine + +variables: + # Generated by https://gitlab.ddbuild.io/DataDog/apm-reliability/async-profiler-build/-/jobs/1476153903 + BUILD_IMAGE_X64: registry.ddbuild.io/ci/async-profiler-build:v100389018-x64-base@sha256:564e47c48f3b7621d20e01195b4fe17ef0e627db3e4cf10827522843ccf50db0 + # Generated by https://gitlab.ddbuild.io/DataDog/apm-reliability/async-profiler-build/-/jobs/1476153903 + BUILD_IMAGE_ARM64: registry.ddbuild.io/ci/async-profiler-build:v100389018-arm64-base@sha256:01f907a050878eefdff772646d914f5865c5a4e00c81a06808973593bd7abecc + # Generated by https://gitlab.ddbuild.io/DataDog/apm-reliability/async-profiler-build/-/jobs/1476153903 + BUILD_IMAGE_X64_2_17: registry.ddbuild.io/ci/async-profiler-build:v100389018-x64-2.17-base@sha256:c2f3a2bac4e0c6a019ea9f5160aae45d41ae4765d7fc9eb0073c24d69df63c37 + # Generated by https://gitlab.ddbuild.io/DataDog/apm-reliability/async-profiler-build/-/jobs/1476153903 + BUILD_IMAGE_X64_MUSL: registry.ddbuild.io/ci/async-profiler-build:v100389018-x64-musl-base@sha256:73074a34d65e31d2e8a41b0f24d48f76f940ce19553b5d28baceba89a646adc5 + # Generated by https://gitlab.ddbuild.io/DataDog/apm-reliability/async-profiler-build/-/jobs/1476153903 + BUILD_IMAGE_ARM64_MUSL: registry.ddbuild.io/ci/async-profiler-build:v100389018-arm64-musl-base@sha256:4a6ea75cecf637167e1569144e3e598a4e956e80ad4448f025ff82ac0b3dc302 + # Generated by https://gitlab.ddbuild.io/DataDog/apm-reliability/async-profiler-build/-/jobs/1476153903 + DATADOG_CI_IMAGE: registry.ddbuild.io/ci/async-profiler-build:v100389018-datadog-ci@sha256:fed27b6ca36d1411c951bf77f8a3f69c5f5b962b9797526aaa95e123b5b9d3e8 + + LAST_COMMIT_FILE: .last.commit + CACHE_FALLBACK_KEY: async-profiler + FORCE_BUILD: $FORCE_BUILD + + REGISTRY: registry.ddbuild.io + SONATYPE_USERNAME: robot-sonatype-apm-java + +.build_job: + extends: + - .retry-config + - .cache-config + stage: build + needs: + - job: prepare:start + artifacts: true + when: on_success + tags: [ "arch:amd64" ] + image: ${BUILD_IMAGE} + script: + - .gitlab/scripts/build.sh + artifacts: + when: always + paths: + - libs/$TARGET/libjavaProfiler.so + - libs/$TARGET/libjavaProfiler.so.debug + - test/$TARGET/reports + - test/$TARGET/logs + - "**/output.txt" + - "**/options.txt" + - "ddprof-test/${TARGET}/reports/tests/test/**/*" + expire_in: 1 day + +.stresstest_job: + extends: + - .retry-config + - .cache-config-pull + stage: stresstest + needs: + - job: prepare:start + artifacts: true + when: on_success + tags: [ "arch:amd64" ] + image: ${BUILD_IMAGE} + script: + - .gitlab/scripts/stresstests.sh + artifacts: + when: always + paths: + - stresstest/$TARGET/logs + - stresstest/$TARGET/results + expire_in: 2 weeks + +prepare:start: + extends: .retry-config + stage: prepare + rules: + - if: '$DOWNSTREAM != null' + when: never + - if: '$CI_PIPELINE_SOURCE == "pipeline"' + when: always + - when: always + interruptible: true + tags: [ "arch:arm64" ] + image: ${PREPARE_IMAGE} + + script: + - .gitlab/scripts/prepare.sh + + artifacts: + paths: + - version.txt + reports: + dotenv: build.env + expire_in: 1 day + +build:x64: + extends: .build_job + variables: + # default to the libc 2.17 linked version + BUILD_IMAGE: ${BUILD_IMAGE_X64_2_17} + TARGET: linux-x64 + +build:x64-musl: + extends: .build_job + variables: + BUILD_IMAGE: ${BUILD_IMAGE_X64_MUSL} + TARGET: linux-x64-musl + +build:arm64: + extends: .build_job + tags: [ "arch:arm64" ] + variables: + BUILD_IMAGE: ${BUILD_IMAGE_ARM64} + TARGET: linux-arm64 + +build:arm64-musl: + extends: .build_job + timeout: 3h + tags: [ "arch:arm64" ] + variables: + BUILD_IMAGE: ${BUILD_IMAGE_ARM64_MUSL} + TARGET: linux-arm64-musl + +stresstest:x64: + extends: .stresstest_job + needs: + - job: prepare:start + artifacts: true + - job: build:x64 + artifacts: true + variables: + # default to the libc 2.17 linked version + BUILD_IMAGE: ${BUILD_IMAGE_X64_2_17} + TARGET: linux-x64 + +stresstest:x64-musl: + extends: .stresstest_job + needs: + - job: prepare:start + artifacts: true + - job: build:x64-musl + artifacts: true + variables: + BUILD_IMAGE: ${BUILD_IMAGE_X64_MUSL} + TARGET: linux-x64-musl + +stresstest:arm64: + extends: .stresstest_job + needs: + - job: prepare:start + artifacts: true + - job: build:arm64 + artifacts: true + tags: [ "arch:arm64" ] + variables: + BUILD_IMAGE: ${BUILD_IMAGE_ARM64} + TARGET: linux-arm64 + +stresstest:arm64-musl: + extends: .stresstest_job + needs: + - job: prepare:start + artifacts: true + - job: build:arm64-musl + artifacts: true + timeout: 3h + tags: [ "arch:arm64" ] + variables: + BUILD_IMAGE: ${BUILD_IMAGE_ARM64_MUSL} + TARGET: linux-arm64-musl + +build-artifact: + extends: .cache-config-pull + stage: deploy + needs: + - job: prepare:start + artifacts: true + - job: build:x64 + artifacts: true + - job: build:x64-musl + artifacts: true + - job: build:arm64 + artifacts: true + - job: build:arm64-musl + artifacts: true + when: on_success + tags: [ "arch:amd64" ] + image: ${BUILD_IMAGE_X64} + + script: + - .gitlab/scripts/deploy.sh assemble + + artifacts: + name: java-profiler.zip + paths: + - ddprof-lib/build/libs/ddprof-*.jar + - ddprof-lib/build/classes/java/main/META-INF/native-libs/* + - version.txt + expire_in: 28 days + +deploy-artifact: + extends: + - .cache-config-pull + - .deploy-sa + stage: deploy + needs: + - job: prepare:start + artifacts: true + - job: build-artifact + artifacts: true + - job: build:x64 + artifacts: true + - job: build:x64-musl + artifacts: true + - job: build:arm64 + artifacts: true + - job: build:arm64-musl + artifacts: true + when: on_success + tags: [ "arch:amd64" ] + image: ${BUILD_IMAGE_X64} + + script: + - .gitlab/scripts/deploy.sh publish + + artifacts: + name: java-profiler-published.zip + paths: + - version.txt + expire_in: 7 days + + +upload-s3: + extends: .deploy-sa + stage: deploy + needs: + - job: prepare:start + artifacts: true + optional: true + - job: build:x64 + artifacts: true + - job: build:x64-musl + artifacts: true + - job: build:arm64 + artifacts: true + - job: build:arm64-musl + artifacts: true + tags: [ "arch:amd64" ] + image: ${DATADOG_CI_IMAGE} + allow_failure: true + interruptible: true + script: + - source .gitlab/config.env + - aws --version + - datadog-ci --version + - | + if [ ! -f "version.txt" ]; then + echo "WARNING: version.txt not found. This might be a manual run without prepare:start artifacts." + export LIB_VERSION="manual-${CI_COMMIT_SHORT_SHA:-unknown}" + else + export LIB_VERSION=$(cat version.txt | awk -F ':' '{print $3}') + fi + export S3_PREFIX_RELEASE="${S3_PREFIX}/release/${LIB_VERSION}" + echo "Using version: ${LIB_VERSION}" + - echo "=== Available library files ===" + - find libs/ -name "*.so*" -type f + - | + for lib_file in libs/*/libjavaProfiler.so; do + if [ -f "$lib_file" ]; then + platform=$(basename $(dirname "$lib_file")) + echo "Uploading library: $lib_file -> libjavaProfiler-${platform}.so" + .gitlab/scripts/upload.sh -p "${S3_PREFIX_RELEASE}" -f "$lib_file" -n "libjavaProfiler-${platform}.so" + + debug_file="${lib_file}.debug" + if [ -f "$debug_file" ]; then + echo "Uploading debug file: $debug_file -> libjavaProfiler-${platform}.debug" + .gitlab/scripts/upload.sh -p "${S3_PREFIX_RELEASE}" -f "$debug_file" -n "libjavaProfiler-${platform}.debug" + fi + fi + done + - echo "=== Uploading ELF symbols to Datadog ===" + - set +x + - export DATADOG_API_KEY_PROD=$(aws ssm get-parameter --region ${AWS_REGION} --name ${SSM_PREFIX}.api_key_public_symbols_prod_us1 --with-decryption --query "Parameter.Value" --out text) + - export DATADOG_API_KEY_STAGING=$(aws ssm get-parameter --region ${AWS_REGION} --name ${SSM_PREFIX}.api_key_public_symbols_staging --with-decryption --query "Parameter.Value" --out text) + - | + if [ -n "${DATADOG_API_KEY_STAGING:-}" ]; then + DATADOG_API_KEY=$DATADOG_API_KEY_STAGING DATADOG_SITE=datad0g.com DD_BETA_COMMANDS_ENABLED=1 datadog-ci elf-symbols upload --disable-git ./libs + fi + if [ -n "${DATADOG_API_KEY_PROD:-}" ]; then + DATADOG_API_KEY=$DATADOG_API_KEY_PROD DATADOG_SITE=datadoghq.com DD_BETA_COMMANDS_ENABLED=1 datadog-ci elf-symbols upload --disable-git ./libs + fi + - set -x + rules: + - if: '($CI_PIPELINE_SOURCE == "trigger" || $CI_PIPELINE_SOURCE == "pipeline") && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + when: on_success + - if: '$CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + when: on_success + - when: manual + +notify-slack-on-success: + extends: .deploy-sa + stage: notify + needs: + - job: prepare:start + artifacts: true + - job: deploy-artifact + artifacts: false + when: on_success + image: registry.ddbuild.io/slack-notifier:latest + tags: ["arch:amd64"] + script: + - .gitlab/build-deploy/notify_channel.sh success $(cat version.txt) + +notify-slack-on-failure: + extends: .deploy-sa + stage: notify + needs: + - job: prepare:start + artifacts: true + - job: deploy-artifact + artifacts: true + when: on_failure + image: registry.ddbuild.io/slack-notifier:latest + tags: ["arch:amd64"] + script: + - .gitlab/build-deploy/notify_channel.sh alert $(cat version.txt) + +include: + - local: .gitlab/common.yml + - local: .gitlab/dd-trace-integration/.gitlab-ci.yml diff --git a/.gitlab/build-deploy/images.yml b/.gitlab/build-deploy/images.yml new file mode 100644 index 000000000..75a283262 --- /dev/null +++ b/.gitlab/build-deploy/images.yml @@ -0,0 +1,11 @@ +stages: + - images +variables: + # Base images for the build docker images + OPENJDK_BASE_IMAGE: bellsoft/liberica-openjdk-debian:21 + OPENJDK_BASE_IMAGE_ARM64: bellsoft/liberica-openjdk-debian:21-aarch64 + OPENJDK_BASE_IMAGE_MUSL: bellsoft/liberica-openjdk-alpine-musl:21 + OPENJDK_BASE_IMAGE_ARM64_MUSL: bellsoft/liberica-openjdk-alpine-musl:21 + BASE_IMAGE_LIBC_2_17: centos:7 + + DOCKER_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/images/docker:24.0.4-gbi-focal diff --git a/.gitlab/build-deploy/notify_channel.sh b/.gitlab/build-deploy/notify_channel.sh new file mode 100755 index 000000000..f1e8da051 --- /dev/null +++ b/.gitlab/build-deploy/notify_channel.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set -euxo pipefail + +if [ ! -z "${CANCELLED:-}" ]; then + exit 0 +fi + +# Source centralized configuration +HERE=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${HERE}/../../.gitlab/config.env" + +COMMIT_SHA=${CI_COMMIT_SHA} +PIPELINE_URL="$CI_PROJECT_URL/pipelines/$CI_PIPELINE_ID" +PIPELINE_LINK="<$PIPELINE_URL|pipeline #$CI_PIPELINE_ID>" + +COMMIT_URL="https://github.com/DataDog/java-profiler/commit/$COMMIT_SHA" +COMMIT_LINK="<$COMMIT_URL|${COMMIT_SHA:0:8}>" + +BRANCH="${CI_COMMIT_BRANCH:-${CI_COMMIT_REF_NAME:-unknown}}" +BRANCH_URL="https://github.com/DataDog/java-profiler/tree/$BRANCH" +BRANCH_LINK="<$BRANCH_URL|$BRANCH>" + +# get status from argument +STATUS=$1 +VERSION=$2 + +if [[ $STATUS == "success" ]]; then + if [[ ! $VERSION =~ .*?-SNAPSHOT ]]; then + MESSAGE_TEXT=":tada: Release succeeded for ${VERSION} (branch=$BRANCH_LINK, commit=$COMMIT_LINK, pipeline=$PIPELINE_LINK) + +:point_up: Now you can publish the artifacts from " + else + MESSAGE_TEXT=":done: Build succeeded for ${VERSION} (branch=$BRANCH_LINK, commit=$COMMIT_LINK, pipeline=$PIPELINE_LINK)" + fi +else + MESSAGE_TEXT=":better-siren: Build failed for ${VERSION} (branch=$BRANCH_LINK, commit=$COMMIT_LINK, pipeline=$PIPELINE_LINK)" +fi + +postmessage "$SLACK_CHANNEL" "$MESSAGE_TEXT" "$STATUS" diff --git a/.gitlab/common.yml b/.gitlab/common.yml new file mode 100644 index 000000000..3b5b6c2c7 --- /dev/null +++ b/.gitlab/common.yml @@ -0,0 +1,78 @@ +# Common GitLab CI templates and configurations +# Include this file in pipeline configurations to reuse common patterns + +# Retry configuration for transient failures +.retry-config: + retry: + max: 2 + when: + - unmet_prerequisites + - runner_system_failure + - data_integrity_failure + - api_failure + - scheduler_failure + - archived_failure + - stale_schedule + - unknown_failure + +# Gradle/Maven cache configuration (push+pull) +.cache-config: + cache: + key: + files: + - gradle/wrapper/gradle-wrapper.properties + prefix: build-${CI_COMMIT_REF_SLUG} + fallback_keys: + - build-${CI_DEFAULT_BRANCH} + - build-main + paths: + - .gradle/caches/ + - .gradle/wrapper/ + - .m2/repository/ + +# Read-only variant — extends base config and overrides policy +.cache-config-pull: + extends: .cache-config + cache: + policy: pull + +# Service account for jobs that publish artifacts or read SSM secrets +.deploy-sa: + variables: + KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: java-profiler + +# Install gh and crane when not already present in the image. +# Extend this in before_script for jobs that need GitHub CLI or crane. +.bootstrap-gh-tools: + before_script: + - | + mkdir -p /tmp/bootstrap-bin + if ! command -v crane >/dev/null 2>&1; then + curl -fsSL "https://github.com/google/go-containerregistry/releases/download/v0.19.1/go-containerregistry_Linux_x86_64.tar.gz" \ + | tar -xz -C /tmp/bootstrap-bin crane + fi + if ! command -v gh >/dev/null 2>&1; then + curl -fsSL "https://github.com/cli/cli/releases/download/v2.45.0/gh_2.45.0_linux_amd64.tar.gz" \ + | tar -xz -C /tmp/bootstrap-bin --strip-components=2 'gh_2.45.0_linux_amd64/bin/gh' + fi + export PATH="/tmp/bootstrap-bin:$PATH" + +# Common job to determine versions (runs from the CI checkout) +.get-versions: + stage: prepare + rules: + - if: '$DOWNSTREAM == null' + when: always + - when: never + interruptible: true + tags: [ "arch:amd64" ] + image: ${PREPARE_IMAGE} + script: + - set -x + - source .gitlab/scripts/includes.sh + - echo "CURRENT_VERSION=$(get_current_version)" > build.env + - echo "PREVIOUS_VERSION=$(get_previous_version)" >> build.env + artifacts: + reports: + dotenv: build.env + expire_in: 1 day diff --git a/.gitlab/common/generate-dashboard.sh b/.gitlab/common/generate-dashboard.sh new file mode 100755 index 000000000..1affc0af6 --- /dev/null +++ b/.gitlab/common/generate-dashboard.sh @@ -0,0 +1,184 @@ +#!/bin/bash + +# generate-dashboard.sh - Generate main index.md dashboard from history data +# +# Usage: generate-dashboard.sh [work-dir] +# +# Reads _data/{integration,benchmarks,reliability}.json and generates index.md +# with quick status table and recent runs across all test types. +# +# Pure bash/jq implementation - no Python required + +set -euo pipefail + +WORK_DIR="${1:-.}" +TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M UTC") + +# Check if jq is available +if ! command -v jq >/dev/null 2>&1; then + echo "Error: jq is required for JSON parsing" >&2 + exit 1 +fi + +# Helper functions +status_emoji() { + case "$1" in + passed) echo "✅" ;; + failed) echo "❌" ;; + partial) echo "⚠️" ;; + *) echo "❓" ;; + esac +} + +format_pr_link() { + local pr_json="$1" + if [ "$pr_json" != "null" ] && [ -n "$pr_json" ]; then + local number=$(echo "$pr_json" | jq -r '.number // empty') + local url=$(echo "$pr_json" | jq -r '.url // empty') + if [ -n "$number" ]; then + echo "[#${number}](${url})" + return + fi + fi + echo "-" +} + +format_pipeline_link() { + local pipeline_json="$1" + if [ "$pipeline_json" != "null" ]; then + local id=$(echo "$pipeline_json" | jq -r '.id // empty') + local url=$(echo "$pipeline_json" | jq -r '.url // "#"') + if [ -n "$id" ]; then + echo "[#${id}](${url})" + return + fi + fi + echo "-" +} + +# Generate the dashboard markdown +{ +cat < **Last Updated:** ${TIMESTAMP} + +## Quick Status + +| Test Type | Latest | Status | Branch | PR | +|-----------|--------|--------|--------|-----| +EOF_HEADER + +# Quick status for each test type +for test_type in integration benchmarks reliability; do + # Capitalize first letter + display_name="$(echo "${test_type:0:1}" | tr '[:lower:]' '[:upper:]')${test_type:1}" + history_file="${WORK_DIR}/_data/${test_type}.json" + + if [ -f "$history_file" ] && [ -s "$history_file" ]; then + # Get latest run (first in array) + latest=$(jq -r '.runs[0] // empty' "$history_file" 2>/dev/null) + + if [ -n "$latest" ]; then + status=$(echo "$latest" | jq -r '.status // "unknown"') + status_symbol=$(status_emoji "$status") + branch=$(echo "$latest" | jq -r '.ddprof_branch // "unknown"') + pr_info=$(echo "$latest" | jq -c '.ddprof_pr // null') + pipeline=$(echo "$latest" | jq -c '.pipeline // null') + + pr_link=$(format_pr_link "$pr_info") + pipeline_link=$(format_pipeline_link "$pipeline") + + echo "| [$display_name]($test_type/) | $pipeline_link | $status_symbol | $branch | $pr_link |" + else + echo "| [$display_name]($test_type/) | - | - | - | - |" + fi + else + echo "| [$display_name]($test_type/) | - | - | - | - |" + fi +done + +cat <<'EOF_TEST_TYPES' + +--- + +## Test Types + +### Integration Tests +dd-trace-java compatibility tests verifying profiler works correctly with the Datadog tracer. +Tests run on every main branch build across multiple JDK versions and platforms. + +### Benchmarks +Performance regression testing using Renaissance benchmark suite. +Compares profiler overhead against baseline (no profiling). + +### Reliability Tests +Long-running stability tests checking for memory leaks and crashes. +Tests multiple allocator configurations (gmalloc, tcmalloc, jemalloc). + +--- + +## Recent Runs (All Types) + +| Date | Type | Pipeline | Branch | PR | Status | +|------|------|----------|--------|-----|--------| +EOF_TEST_TYPES + +# Collect all runs from all types (last 5 from each) into a single JSON array +tmpfile=$(mktemp) +trap "rm -f $tmpfile" EXIT + +# Build array by merging runs from all test types +all_runs="[]" +for test_type in integration benchmarks reliability; do + history_file="${WORK_DIR}/_data/${test_type}.json" + + if [ -f "$history_file" ] && [ -s "$history_file" ]; then + # Get last 5 runs, add type field, merge into all_runs + runs=$(jq --arg type "$test_type" \ + '.runs[:5] | map(. + {_type: $type})' \ + "$history_file" 2>/dev/null || echo "[]") + + all_runs=$(echo "$all_runs" "$runs" | jq -s 'add') + fi +done + +echo "$all_runs" > "$tmpfile" + +# Sort by timestamp descending, take 15 most recent +if [ "$(jq 'length' "$tmpfile" 2>/dev/null || echo 0)" -gt 0 ]; then + jq -c 'sort_by(.timestamp) | reverse | .[:15] | .[]' "$tmpfile" 2>/dev/null | \ + while IFS= read -r run; do + date=$(echo "$run" | jq -r '(.timestamp // "")[:10]') + type_val=$(echo "$run" | jq -r '._type // "unknown"') + type_name="$(echo "${type_val:0:1}" | tr '[:lower:]' '[:upper:]')${type_val:1}" + status=$(echo "$run" | jq -r '.status // "unknown"') + status_symbol=$(status_emoji "$status") + branch=$(echo "$run" | jq -r '.ddprof_branch // "unknown"') + pr_info=$(echo "$run" | jq -c '.ddprof_pr // null') + pipeline=$(echo "$run" | jq -c '.pipeline // null') + + pr_link=$(format_pr_link "$pr_info") + pipeline_link=$(format_pipeline_link "$pipeline") + + echo "| $date | $type_name | $pipeline_link | $branch | $pr_link | $status_symbol |" + done +else + echo "| - | - | - | - | - | - |" +fi + +cat <<'EOF_FOOTER' + +--- + +[Repository](https://github.com/DataDog/java-profiler) | [java-profiler](https://github.com/DataDog/java-profiler) | [View history](https://github.com/DataDog/java-profiler/commits/gh-pages) +EOF_FOOTER + +} > "${WORK_DIR}/index.md" + +echo "Generated ${WORK_DIR}/index.md" diff --git a/.gitlab/common/generate-index.sh b/.gitlab/common/generate-index.sh new file mode 100755 index 000000000..64e8496e8 --- /dev/null +++ b/.gitlab/common/generate-index.sh @@ -0,0 +1,243 @@ +#!/bin/bash + +# generate-index.sh - Generate test type index page with expandable details +# +# Usage: generate-index.sh [work-dir] +# +# test-type: integration|benchmarks|reliability +# Reads _data/{test-type}.json and generates {test-type}/index.md +# +# Pure bash/jq implementation - no Python required + +set -euo pipefail + +TEST_TYPE="${1:-}" +WORK_DIR="${2:-.}" + +if [ -z "${TEST_TYPE}" ]; then + echo "Usage: generate-index.sh [work-dir]" >&2 + exit 1 +fi + +# Check if jq is available +if ! command -v jq >/dev/null 2>&1; then + echo "Error: jq is required for JSON parsing" >&2 + exit 1 +fi + +# Ensure output directory exists +mkdir -p "${WORK_DIR}/${TEST_TYPE}" + +# Helper functions +status_emoji() { + case "$1" in + passed) echo "✅" ;; + failed) echo "❌" ;; + partial) echo "⚠️" ;; + *) echo "❓" ;; + esac +} + +format_pr_link() { + local pr_json="$1" + if [ "$pr_json" != "null" ] && [ -n "$pr_json" ]; then + local number=$(echo "$pr_json" | jq -r '.number // empty') + local url=$(echo "$pr_json" | jq -r '.url // empty') + if [ -n "$number" ]; then + echo "[#${number}](${url})" + return + fi + fi + echo "-" +} + +format_pipeline_link() { + local pipeline_json="$1" + if [ "$pipeline_json" != "null" ]; then + local id=$(echo "$pipeline_json" | jq -r '.id // empty') + local url=$(echo "$pipeline_json" | jq -r '.url // "#"') + if [ -n "$id" ]; then + echo "[#${id}](${url})" + return + fi + fi + echo "-" +} + +# Test type metadata +case "$TEST_TYPE" in + integration) + TITLE="DD-Trace Integration Test History" + DESCRIPTION="Tests dd-trace-java compatibility with ddprof across multiple JDK versions and platforms." + ;; + benchmarks) + TITLE="Benchmark Test History" + DESCRIPTION="Performance regression testing using Renaissance benchmark suite." + ;; + reliability) + TITLE="Reliability Test History" + DESCRIPTION="Long-running stability tests for memory leaks and crashes." + ;; + *) + # Capitalize first letter + cap_type="$(echo "${TEST_TYPE:0:1}" | tr '[:lower:]' '[:upper:]')${TEST_TYPE:1}" + TITLE="${cap_type} Test History" + DESCRIPTION="" + ;; +esac + +# Generate the index page +{ +cat </dev/null || echo "") + + if [ -z "$runs" ]; then + echo "*No test runs recorded yet.*" + else + echo "$runs" | while IFS= read -r run; do + timestamp=$(echo "$run" | jq -r '.timestamp // ""') + date_str="${timestamp:0:16}" + date_str="${date_str/T/ }" + [ -z "$date_str" ] && date_str="Unknown" + + status=$(echo "$run" | jq -r '.status // "unknown"') + status_symbol=$(status_emoji "$status") + branch=$(echo "$run" | jq -r '.ddprof_branch // "unknown"') + version=$(echo "$run" | jq -r '.lib_version // "unknown"') + sha=$(echo "$run" | jq -r '.ddprof_sha // "unknown"') + sha="${sha:0:8}" + + pr_info=$(echo "$run" | jq -c '.ddprof_pr // null') + pipeline=$(echo "$run" | jq -c '.pipeline // null') + + pr_link=$(format_pr_link "$pr_info") + pipeline_link=$(format_pipeline_link "$pipeline") + + # PR text for summary line + pr_text="" + if [ "$pr_info" != "null" ]; then + pr_num=$(echo "$pr_info" | jq -r '.number // empty') + if [ -n "$pr_num" ]; then + pr_text=" | PR $pr_link" + fi + fi + + # Print expandable details + cat < + +${date_str} | ${status_symbol} | ${branch}${pr_text} | Pipeline ${pipeline_link} + + +**Version:** ${version} +**Commit:** ${sha} + +EOF_RUN + + # Summary table based on test type + summary=$(echo "$run" | jq -c '.summary // {}') + + case "$TEST_TYPE" in + integration) + total_jobs=$(echo "$summary" | jq -r '.total_jobs // "N/A"') + passed_jobs=$(echo "$summary" | jq -r '.passed_jobs // "N/A"') + failed_jobs=$(echo "$summary" | jq -r '.failed_jobs // "N/A"') + + echo "| Metric | Value |" + echo "|--------|-------|" + echo "| Jobs | $total_jobs |" + echo "| Passed | $passed_jobs |" + echo "| Failed | $failed_jobs |" + + # Failed configs + failed_configs=$(echo "$summary" | jq -r '.failed_configs // [] | join(", ")') + if [ -n "$failed_configs" ] && [ "$failed_configs" != "" ]; then + echo "" + echo "**Failed Configs:** $failed_configs" + fi + ;; + + benchmarks) + architectures=$(echo "$summary" | jq -r '.architectures // "N/A"') + modes_tested=$(echo "$summary" | jq -r '.modes_tested // "N/A"') + regression=$(echo "$summary" | jq -r '.regression_detected // false') + regression_text=$([ "$regression" = "true" ] && echo "Yes" || echo "No") + + echo "| Metric | Value |" + echo "|--------|-------|" + echo "| Architectures | $architectures |" + echo "| Modes | $modes_tested |" + echo "| Regression | $regression_text |" + + # Regression details + regression_details=$(echo "$summary" | jq -r '.regression_details // [] | .[:5] | .[]' 2>/dev/null) + if [ -n "$regression_details" ]; then + echo "" + echo "**Regressions:**" + echo "$regression_details" | while read -r detail; do + echo "- $detail" + done + fi + ;; + + reliability) + total_configs=$(echo "$summary" | jq -r '.total_configs // "N/A"') + passed=$(echo "$summary" | jq -r '.passed // "N/A"') + failed=$(echo "$summary" | jq -r '.failed // "N/A"') + + echo "| Metric | Value |" + echo "|--------|-------|" + echo "| Configs | $total_configs |" + echo "| Passed | $passed |" + echo "| Failed | $failed |" + + # Failures + failures=$(echo "$summary" | jq -r '.failures // [] | .[:5] | .[]' 2>/dev/null) + if [ -n "$failures" ]; then + echo "" + echo "**Failures:**" + echo "$failures" | while read -r failure; do + echo "- $failure" + done + fi + ;; + esac + + echo "" + echo "" + echo "" + done + fi +else + echo "*No test runs recorded yet.*" +fi + +cat <<'EOF_FOOTER' + +--- + +[← Back to Dashboard](../) | [View git history](https://github.com/DataDog/java-profiler/commits/gh-pages) +EOF_FOOTER + +} > "${WORK_DIR}/${TEST_TYPE}/index.md" + +echo "Generated ${WORK_DIR}/${TEST_TYPE}/index.md" diff --git a/.gitlab/common/lookup-pr.sh b/.gitlab/common/lookup-pr.sh new file mode 100755 index 000000000..92e04a722 --- /dev/null +++ b/.gitlab/common/lookup-pr.sh @@ -0,0 +1,128 @@ +#!/bin/bash + +# lookup-pr.sh - Find GitHub PR for a java-profiler branch +# +# Usage: lookup-pr.sh +# Output: JSON with PR info {"number": N, "url": "...", "title": "..."} or {} +# +# Authentication: +# - In CI: Uses Octo-STS with java-profiler-build-read policy +# - Fallback: GITHUB_TOKEN env var or unauthenticated (public repo) +# +# Falls back gracefully if API fails or no PR found. + +set -euo pipefail + +BRANCH="${1:-}" +REPO="DataDog/java-profiler" + +# Debug logging to stderr (doesn't affect JSON output on stdout) +debug() { echo "[DEBUG] $*" >&2; } + +debug "Looking for PR in ${REPO} with head branch: ${BRANCH}" + +# Skip lookup for main/master branches (never have PRs) +if [ -z "${BRANCH}" ] || [ "${BRANCH}" = "main" ] || [ "${BRANCH}" = "master" ]; then + debug "Skipping lookup for main/master branch" + echo "{}" + exit 0 +fi + +# Obtain GitHub token via dd-octo-sts if available +GITHUB_TOKEN="" +if command -v dd-octo-sts >/dev/null 2>&1 && [ -n "${DDOCTOSTS_ID_TOKEN:-}" ]; then + debug "Attempting to get token via Octo-STS..." + TOKEN_OUTPUT=$(dd-octo-sts token --scope DataDog/java-profiler --policy async-profiler-build.ci 2>/tmp/dd-octo-sts-lookup-error.log) + TOKEN_EXIT_CODE=$? + + if [ $TOKEN_EXIT_CODE -eq 0 ] && [ -n "${TOKEN_OUTPUT}" ]; then + GITHUB_TOKEN="${TOKEN_OUTPUT}" + debug "Got GitHub token via Octo-STS" + else + debug "Failed to get token via Octo-STS (exit code: ${TOKEN_EXIT_CODE})" + if [ -s /tmp/dd-octo-sts-lookup-error.log ]; then + debug "dd-octo-sts error: $(cat /tmp/dd-octo-sts-lookup-error.log | head -5)" + fi + fi +else + debug "Octo-STS not available (dd-octo-sts: $(command -v dd-octo-sts 2>/dev/null || echo 'not found'), DDOCTOSTS_ID_TOKEN: ${DDOCTOSTS_ID_TOKEN:+set})" +fi + +# URL-encode the branch name (/ -> %2F, etc.) +# Use jq if available, otherwise use sed for common cases +url_encode() { + local string="$1" + if command -v jq >/dev/null 2>&1; then + printf '%s' "$string" | jq -sRr @uri + else + # Fallback: encode common special characters with sed + printf '%s' "$string" | sed 's|/|%2F|g; s| |%20|g; s|#|%23|g' + fi +} + +ENCODED_BRANCH=$(url_encode "${BRANCH}") +API_URL="https://api.github.com/repos/${REPO}/pulls?head=DataDog:${ENCODED_BRANCH}&state=all&per_page=1" +debug "API URL: ${API_URL}" + +if [ -n "${GITHUB_TOKEN}" ]; then + debug "Using authenticated request" + response=$(curl -s --max-time 10 \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + "${API_URL}" 2>/dev/null) || { + debug "curl failed" + echo "{}" + exit 0 + } +else + # Anonymous access for public repos (60 req/hour limit) + debug "Using anonymous request (may be rate limited)" + response=$(curl -s --max-time 10 \ + -H "Accept: application/vnd.github+json" \ + "${API_URL}" 2>/dev/null) || { + debug "curl failed" + echo "{}" + exit 0 + } +fi + +debug "API response length: ${#response} chars" +debug "API response preview: ${response:0:200}" + +# Parse JSON response - use jq if available +if command -v jq >/dev/null 2>&1; then + # Check if response is a non-empty array + if ! echo "${response}" | jq -e 'if type == "array" and length > 0 then true else false end' >/dev/null 2>&1; then + debug "Response is not a valid JSON array with items (might be error response or empty)" + echo "{}" + exit 0 + fi + + debug "Found valid PR response" + + # Extract PR info + echo "${response}" | jq -c '.[0] | {number: .number, url: .html_url, title: .title}' +else + # Fallback: use grep/sed for basic JSON extraction (less reliable but works without jq) + debug "jq not available, using fallback JSON parsing" + + # Check if it looks like a non-empty array + if ! echo "${response}" | grep -q '^\[{'; then + debug "Response doesn't look like a JSON array with items" + echo "{}" + exit 0 + fi + + debug "Found valid PR response (basic check)" + + # Extract fields using grep/sed (fragile but works for simple cases) + pr_number=$(echo "${response}" | grep -o '"number":[0-9]*' | head -1 | sed 's/"number"://') + pr_url=$(echo "${response}" | grep -o '"html_url":"[^"]*"' | head -1 | sed 's/"html_url":"//; s/"$//') + pr_title=$(echo "${response}" | grep -o '"title":"[^"]*"' | head -1 | sed 's/"title":"//; s/"$//') + + if [ -n "${pr_number}" ]; then + echo "{\"number\":${pr_number},\"url\":\"${pr_url}\",\"title\":\"${pr_title}\"}" + else + echo "{}" + fi +fi diff --git a/.gitlab/common/setup-publish-env.sh b/.gitlab/common/setup-publish-env.sh new file mode 100755 index 000000000..256863b10 --- /dev/null +++ b/.gitlab/common/setup-publish-env.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +# setup-publish-env.sh - Install dependencies for GitHub Pages publishing +# +# This script auto-detects the package manager and installs required tools + +set -euo pipefail + +echo "=== Setting up publishing environment ===" +echo "Current user: $(whoami), UID: $(id -u)" +echo "Image: ${CI_JOB_IMAGE:-unknown}" + +# Check if Python3 is already available +if command -v python3 >/dev/null 2>&1; then + echo "Python3 already available: $(python3 --version)" + echo "Skipping installation" + exit 0 +fi + +echo "Python3 not found, attempting installation..." + +# Check if we need sudo +SUDO="" +if [ "$(id -u)" -ne 0 ]; then + if command -v sudo >/dev/null 2>&1; then + echo "Running as non-root user, will use sudo" + SUDO="sudo" + else + echo "ERROR: Running as non-root but sudo not available" + echo "Current user: $(whoami), UID: $(id -u)" + exit 1 + fi +fi + +# Detect and use appropriate package manager +if command -v apt-get >/dev/null 2>&1; then + echo "Detected: apt-get (Debian/Ubuntu)" + $SUDO apt-get update -qq + $SUDO apt-get install -y python3 git curl jq +elif command -v apk >/dev/null 2>&1; then + echo "Detected: apk (Alpine)" + $SUDO apk add --no-cache python3 git curl jq +elif command -v yum >/dev/null 2>&1; then + echo "Detected: yum (RHEL/CentOS)" + $SUDO yum install -y python3 git curl jq +elif command -v dnf >/dev/null 2>&1; then + echo "Detected: dnf (Fedora/RHEL 8+)" + $SUDO dnf install -y python3 git curl jq +else + echo "ERROR: No supported package manager found (tried apt-get, apk, yum, dnf)" + echo "Available commands:" + command -v apt-get apk yum dnf 2>&1 || echo "None found" + exit 1 +fi + +echo "" +echo "=== Verifying installations ===" +echo -n "Python3: " +python3 --version || (echo "FAILED" && exit 1) + +echo -n "Git: " +git --version || echo "WARNING: git not found" + +echo -n "curl: " +curl --version | head -1 || echo "WARNING: curl not found" + +echo -n "jq: " +jq --version || echo "WARNING: jq not found" + +echo -n "dd-octo-sts: " +dd-octo-sts version || echo "WARNING: dd-octo-sts not available or failed to run" + +echo "" +echo "=== Environment ready for publishing ===" diff --git a/.gitlab/common/update-history.sh b/.gitlab/common/update-history.sh new file mode 100755 index 000000000..0b5b9d4ef --- /dev/null +++ b/.gitlab/common/update-history.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +# update-history.sh - Update JSON history file with new run, keeping last N entries +# +# Usage: update-history.sh [work-dir] +# +# test-type: integration|benchmarks|reliability +# new-run-json-file: Path to file containing new run JSON +# work-dir: Directory containing _data/ (default: current directory) +# +# Updates _data/{test-type}.json, prepending new run and keeping last MAX_HISTORY entries. +# +# Pure bash/jq implementation - no Python required + +set -euo pipefail + +TEST_TYPE="${1:-}" +NEW_RUN_FILE="${2:-}" +WORK_DIR="${3:-.}" +MAX_HISTORY="${MAX_HISTORY:-10}" + +if [ -z "${TEST_TYPE}" ] || [ -z "${NEW_RUN_FILE}" ]; then + echo "Usage: update-history.sh [work-dir]" >&2 + exit 1 +fi + +if [ ! -f "${NEW_RUN_FILE}" ]; then + echo "Error: New run JSON file not found: ${NEW_RUN_FILE}" >&2 + exit 1 +fi + +HISTORY_FILE="${WORK_DIR}/_data/${TEST_TYPE}.json" + +# Ensure _data directory exists +mkdir -p "${WORK_DIR}/_data" + +# Initialize history file if it doesn't exist +if [ ! -f "${HISTORY_FILE}" ]; then + echo '{"runs":[]}' > "${HISTORY_FILE}" +fi + +# Check if jq is available +if ! command -v jq >/dev/null 2>&1; then + echo "Error: jq is required for JSON manipulation" >&2 + echo "Install with: apt-get install jq (Debian/Ubuntu) or apk add jq (Alpine)" >&2 + exit 1 +fi + +# Read new run JSON +NEW_RUN_JSON=$(cat "${NEW_RUN_FILE}") + +# Validate JSON +if ! echo "$NEW_RUN_JSON" | jq empty 2>/dev/null; then + echo "Error: Invalid JSON in ${NEW_RUN_FILE}" >&2 + exit 1 +fi + +# Update history using jq: +# 1. Read existing history (or start with empty {"runs":[]}) +# 2. Parse new run from variable +# 3. Prepend new run to .runs array +# 4. Keep only last MAX_HISTORY entries +jq --argjson newrun "$NEW_RUN_JSON" \ + --argjson maxhistory "$MAX_HISTORY" \ + '.runs = ([$newrun] + .runs) | .runs = .runs[:$maxhistory]' \ + "${HISTORY_FILE}" > "${HISTORY_FILE}.tmp" + +# Atomic move +mv "${HISTORY_FILE}.tmp" "${HISTORY_FILE}" + +# Report success +RUN_COUNT=$(jq '.runs | length' "${HISTORY_FILE}") +echo "Updated ${HISTORY_FILE}: now has ${RUN_COUNT} run(s)" diff --git a/.gitlab/config.env b/.gitlab/config.env new file mode 100644 index 000000000..f4a4ba024 --- /dev/null +++ b/.gitlab/config.env @@ -0,0 +1,37 @@ +# CI Configuration - Centralized configuration for all build scripts +# This file is sourced by scripts to provide consistent configuration across the pipeline + +# Java Versions +# Build version used for compiling the library +JAVA_BUILD_VERSION=21 +# Test version used for reliability tests +JAVA_TEST_VERSION=21.0.3-tem +# Benchmark version used for performance testing +JAVA_BENCHMARK_VERSION=21 + +# Docker Base Images +OPENJDK_BASE_IMAGE_AMD64=bellsoft/liberica-openjdk-debian:21 +OPENJDK_BASE_IMAGE_ARM64=bellsoft/liberica-openjdk-debian:21-aarch64 +OPENJDK_BASE_IMAGE_AMD64_MUSL=bellsoft/liberica-openjdk-alpine-musl:21 +OPENJDK_BASE_IMAGE_ARM64_MUSL=bellsoft/liberica-openjdk-alpine-musl:21 + +# AWS Configuration +AWS_REGION=us-east-1 +SSM_PREFIX=ci.java-profiler +S3_BUCKET=binaries.ddbuild.io +S3_PREFIX=async-profiler-build + +# Slack Configuration +SLACK_CHANNEL="#java-profiler-lib" + +# Build Configuration +DEFAULT_BENCHMARK_ITERATIONS=1 +DEFAULT_BENCHMARK_MODES="cpu,wall,alloc,memleak" + +# Git Configuration +# JAVA_PROFILER_REPO is no longer used; CI runs inside the repo + +# dd-trace-java Integration Test Configuration +DD_TRACE_JAVA_REPO=https://github.com/DataDog/dd-trace-java.git +DD_TRACE_JAVA_BRANCH=master +INTEGRATION_TEST_TIMEOUT=30 diff --git a/.gitlab/dd-trace-integration/.gitlab-ci.yml b/.gitlab/dd-trace-integration/.gitlab-ci.yml new file mode 100644 index 000000000..ad9276024 --- /dev/null +++ b/.gitlab/dd-trace-integration/.gitlab-ci.yml @@ -0,0 +1,307 @@ +# dd-trace-java Integration Tests +# Tests the java-profiler ddprof-lib artifact with patched dd-java-agent + +variables: + DD_TRACE_VERSION: + value: "" + description: "dd-java-agent snapshot version to download (empty = auto-detect latest)" + +prepare-patched-agent: + extends: .retry-config + stage: integration-test + tags: [ "arch:amd64" ] + image: ${BUILD_IMAGE_X64} + needs: + - job: prepare:start + artifacts: true + - job: build-artifact + artifacts: true + rules: + - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null' + when: never + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: never + - when: on_success + interruptible: true + timeout: 10m + script: + - | + echo "=== Preparing Patched dd-java-agent ===" + + echo "Using ddprof.jar from build-artifact job" + echo "Available artifacts:" + ls -lh ddprof-lib/build/libs/ || echo "Directory not found" + + LIB_VERSION=$(awk -F ':' '{print $3}' version.txt) + DDPROF_JAR="ddprof-lib/build/libs/ddprof-lib-${LIB_VERSION}.jar" + if [ ! -f "${DDPROF_JAR}" ]; then + echo "ERROR: ddprof JAR not found at ${DDPROF_JAR} (version=${LIB_VERSION})" + ls -lh ddprof-lib/build/libs/ || true + exit 1 + fi + + echo "Found: ${DDPROF_JAR}" + cp "${DDPROF_JAR}" ddprof.jar + ls -lh ddprof.jar + + if [ -n "${DD_TRACE_VERSION}" ]; then + .gitlab/dd-trace-integration/download-snapshot-artifacts.sh \ + --dd-trace-version "${DD_TRACE_VERSION}" \ + --skip-ddprof + else + .gitlab/dd-trace-integration/download-snapshot-artifacts.sh \ + --skip-ddprof + fi + + .gitlab/dd-trace-integration/patch-dd-java-agent.sh + + DD_AGENT_JAR=dd-java-agent-original.jar \ + DDPROF_JAR=ddprof.jar \ + PATCHED_JAR=dd-java-agent-patched.jar \ + .gitlab/dd-trace-integration/verify-patch-compatibility.sh | tee compatibility-check.log + + DDPROF_SHA=$(git rev-parse HEAD) + echo "${DDPROF_SHA}" > ddprof-commit-sha.txt + echo "Captured ddprof SHA: ${DDPROF_SHA}" + + echo "" + echo "=== Patched agent prepared successfully ===" + artifacts: + name: patched-agent + paths: + - dd-java-agent-patched.jar + - dd-java-agent-original.jar + - ddprof.jar + - ddprof-commit-sha.txt + - compatibility-check.log + expire_in: 1 day + +.integration_test_base: + extends: .retry-config + stage: integration-test + needs: + - job: prepare:start + artifacts: true + - job: build-artifact + artifacts: true + - job: prepare-patched-agent + artifacts: true + rules: + - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null' + when: never + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: never + - when: on_success + interruptible: true + timeout: 10m + allow_failure: true + script: + - | + echo "=== Integration Test ===" + echo "Platform: ${ARCH}-${LIBC_VARIANT}" + echo "JVM: ${JVM_TYPE} JDK${JAVA_VERSION}" + echo "" + + if command -v apt-get &> /dev/null; then + apt-get update -qq && apt-get install -y -qq curl wget unzip bc ca-certificates + elif command -v apk &> /dev/null; then + apk add --no-cache curl wget unzip bc bash ca-certificates + fi + + JDK_INSTALLED=false + if command -v apt-get &> /dev/null; then + if apt-get install -y -qq openjdk-${JAVA_VERSION}-jdk 2>/dev/null || apt-get install -y -qq openjdk-${JAVA_VERSION}-jre-headless 2>/dev/null; then + JDK_INSTALLED=true + fi + elif command -v apk &> /dev/null; then + JDK_INSTALLED=false + fi + + if [ "${JDK_INSTALLED}" = false ]; then + ARCH_NAME=$(uname -m) + case "${ARCH_NAME}" in + x86_64) JDK_ARCH="amd64" ;; + aarch64) JDK_ARCH="aarch64" ;; + *) echo "ERROR: Unsupported architecture: ${ARCH_NAME}"; exit 1 ;; + esac + + if command -v apk &> /dev/null; then + JDK_LIBC="musl" + else + JDK_LIBC="glibc" + fi + + JDK_DIR="/opt/java/jdk-${JAVA_VERSION}" + mkdir -p /opt/java + + if [ "${JDK_LIBC}" = "musl" ]; then + case "${JAVA_VERSION}" in + 8) LIBERICA_VERSION="8u482+10" ;; + 11) LIBERICA_VERSION="11.0.30+9" ;; + 17) LIBERICA_VERSION="17.0.18+10" ;; + 21) LIBERICA_VERSION="21.0.10+10" ;; + 25) LIBERICA_VERSION="25.0.2+12" ;; + *) echo "ERROR: Unsupported JDK version: ${JAVA_VERSION}"; exit 1 ;; + esac + case "${JDK_ARCH}" in + amd64) LIBERICA_ARCH="x64" ;; + *) LIBERICA_ARCH="${JDK_ARCH}" ;; + esac + DOWNLOAD_URL="https://download.bell-sw.com/java/${LIBERICA_VERSION}/bellsoft-jdk${LIBERICA_VERSION}-linux-${LIBERICA_ARCH}-musl-lite.tar.gz" + else + case "${JDK_ARCH}" in + amd64) ADOPTIUM_ARCH="x64" ;; + *) ADOPTIUM_ARCH="${JDK_ARCH}" ;; + esac + DOWNLOAD_URL="https://api.adoptium.net/v3/binary/latest/${JAVA_VERSION}/ga/linux/${ADOPTIUM_ARCH}/jdk/hotspot/normal/eclipse" + fi + + MAX_RETRIES=3 + for attempt in $(seq 1 $MAX_RETRIES); do + if curl -Lk --retry 3 --retry-delay 5 "${DOWNLOAD_URL}" | tar xzf - -C /opt/java; then + break + fi + [ $attempt -lt $MAX_RETRIES ] && sleep 10 || exit 1 + done + + EXTRACTED_DIR=$(find /opt/java -maxdepth 1 -type d \( -name "jdk-*" -o -name "jdk8u*" \) ! -name "jdk-${JAVA_VERSION}" | head -1) + [ -z "${EXTRACTED_DIR}" ] && exit 1 + ln -sf "${EXTRACTED_DIR}" "${JDK_DIR}" + export JAVA_HOME="${JDK_DIR}" + else + export JAVA_HOME=$(find /usr/lib/jvm -name "java-${JAVA_VERSION}-*" -type d | head -1) + [ -z "${JAVA_HOME}" ] && export JAVA_HOME=$(find /usr/lib/jvm -name "openjdk-${JAVA_VERSION}" -type d | head -1) + [ -z "${JAVA_HOME}" ] && export JAVA_HOME=$(find /usr/lib/jvm -name "java-1.${JAVA_VERSION}*" -type d | head -1) + [ -z "${JAVA_HOME}" ] && export JAVA_HOME=$(find /usr/lib/jvm -maxdepth 1 -type d | grep -E "(jdk|java|openjdk)" | head -1) + [ -z "${JAVA_HOME}" ] && exit 1 + fi + + echo "Using JAVA_HOME: ${JAVA_HOME}" + ${JAVA_HOME}/bin/java -version + + .gitlab/dd-trace-integration/install-prerequisites.sh + .gitlab/dd-trace-integration/run-integration-test.sh + + echo "" + echo "=== All tests completed successfully ===" + artifacts: + when: always + name: "integration-test-${LIBC_VARIANT}-${ARCH}-${JVM_TYPE}-jdk${JAVA_VERSION}" + paths: + - integration-test-results/${LIBC_VARIANT}-${ARCH}-${JVM_TYPE}-jdk${JAVA_VERSION}/ + expire_in: 7 days + +integration-test-x64-glibc: + extends: .integration_test_base + tags: [ "arch:amd64" ] + image: ${BUILD_IMAGE_X64} + parallel: + matrix: + - JVM_TYPE: hotspot + JAVA_VERSION: [8, 11, 17, 21, 25] + - JVM_TYPE: openj9 + JAVA_VERSION: [8, 11, 17, 21, 25] + variables: + LIBC_VARIANT: glibc + ARCH: x64 + +integration-test-x64-musl: + extends: .integration_test_base + tags: [ "arch:amd64" ] + image: ${BUILD_IMAGE_X64_MUSL} + parallel: + matrix: + - JVM_TYPE: hotspot + JAVA_VERSION: [8, 11, 17, 21, 25] + - JVM_TYPE: openj9 + JAVA_VERSION: [8, 11, 17, 21, 25] + variables: + LIBC_VARIANT: musl + ARCH: x64 + +integration-test-arm64-glibc: + extends: .integration_test_base + tags: [ "arch:arm64" ] + image: ${BUILD_IMAGE_ARM64} + parallel: + matrix: + - JVM_TYPE: hotspot + JAVA_VERSION: [8, 11, 17, 21, 25] + - JVM_TYPE: openj9 + JAVA_VERSION: [8, 11, 17, 21, 25] + variables: + LIBC_VARIANT: glibc + ARCH: arm64 + +integration-test-arm64-musl: + extends: .integration_test_base + tags: [ "arch:arm64" ] + image: ${BUILD_IMAGE_ARM64_MUSL} + parallel: + matrix: + - JVM_TYPE: hotspot + JAVA_VERSION: [8, 11, 17, 21, 25] + - JVM_TYPE: openj9 + JAVA_VERSION: [8, 11, 17, 21, 25] + variables: + LIBC_VARIANT: musl + ARCH: arm64 + +report-dd-trace-results: + extends: .retry-config + stage: integration-test + tags: [ "arch:arm64" ] + image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 + id_tokens: + DDOCTOSTS_ID_TOKEN: + aud: dd-octo-sts + needs: + - job: prepare-patched-agent + artifacts: true + - job: integration-test-x64-glibc + artifacts: true + - job: integration-test-x64-musl + artifacts: true + - job: integration-test-arm64-glibc + artifacts: true + - job: integration-test-arm64-musl + artifacts: true + rules: + - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null' + when: never + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: never + - when: on_success + interruptible: true + timeout: 10m + script: + - ./.gitlab/dd-trace-integration/publish-gh-pages.sh + allow_failure: true + +post-pr-comment: + extends: .retry-config + stage: integration-test + tags: [ "arch:arm64" ] + image: registry.ddbuild.io/ci/async-profiler-build-amd64:v49110984-amd64-benchmarks@sha256:b952a00db56a22a83392fca37b93c7664bc86d8e81e29e3e2ef2e33042024d2f + needs: + - job: prepare-patched-agent + artifacts: true + - job: integration-test-x64-glibc + artifacts: true + - job: integration-test-x64-musl + artifacts: true + - job: integration-test-arm64-glibc + artifacts: true + - job: integration-test-arm64-musl + artifacts: true + rules: + - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null' + when: never + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: never + - when: on_success + interruptible: true + timeout: 5m + script: + - .gitlab/dd-trace-integration/post-pr-comment.sh integration-test-results + allow_failure: true diff --git a/.gitlab/dd-trace-integration/download-snapshot-artifacts.sh b/.gitlab/dd-trace-integration/download-snapshot-artifacts.sh new file mode 100755 index 000000000..81f2c69e1 --- /dev/null +++ b/.gitlab/dd-trace-integration/download-snapshot-artifacts.sh @@ -0,0 +1,337 @@ +#!/bin/bash + +set -euo pipefail + +# Download dd-java-agent and ddprof snapshot artifacts from Maven + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +# Maven snapshot repository +SNAPSHOT_REPO="https://central.sonatype.com/repository/maven-snapshots/" + +# Default versions (can be overridden via environment) +DEFAULT_DD_TRACE_VERSION="1.50.0-SNAPSHOT" + +# Output directory for downloaded artifacts +OUTPUT_DIR="${OUTPUT_DIR:-${PROJECT_ROOT}}" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +function log_info() { + echo -e "${GREEN}[INFO]${NC} $*" +} + +function log_warn() { + echo -e "${YELLOW}[WARN]${NC} $*" +} + +function log_error() { + echo -e "${RED}[ERROR]${NC} $*" +} + +function detect_latest_snapshot_version() { + local group_id=$1 + local artifact_id=$2 + local repo_url=$3 + + # Convert group_id to path (com.datadoghq -> com/datadoghq) + local group_path=$(echo "${group_id}" | tr '.' '/') + + # Construct metadata URL + local metadata_url="${repo_url}${group_path}/${artifact_id}/maven-metadata.xml" + + # Log to stderr so it doesn't interfere with return value + echo -e "${GREEN}[INFO]${NC} Querying for latest ${artifact_id} version from metadata" >&2 + + # Fetch metadata + local metadata=$(curl -fsSL "${metadata_url}" 2>/dev/null || echo "") + + if [ -z "${metadata}" ]; then + echo -e "${RED}[ERROR]${NC} Could not fetch metadata from ${metadata_url}" >&2 + return 1 + fi + + # Extract latest version using sed (portable across macOS and Linux) + local latest_version=$(echo "${metadata}" | sed -n 's/.*\(.*\)<\/latest>.*/\1/p') + + # Validate that latest version is vanilla (no branch name) + # Vanilla patterns: X.Y.Z-SNAPSHOT or X.Y.Z-DD-SNAPSHOT + if [ -n "${latest_version}" ]; then + if ! echo "${latest_version}" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+(-DD)?-SNAPSHOT$'; then + echo -e "${YELLOW}[WARN]${NC} Latest version '${latest_version}' contains branch name, searching for vanilla snapshot..." >&2 + latest_version="" + fi + fi + + if [ -z "${latest_version}" ]; then + # Fallback: try to find latest from versions list + # Extract all versions, filter for vanilla SNAPSHOT (no branch names) + # Vanilla patterns: X.Y.Z-SNAPSHOT or X.Y.Z-DD-SNAPSHOT + # Reject patterns with branch names: X.Y.Z-branch_name-SNAPSHOT + latest_version=$(echo "${metadata}" | \ + sed -n 's/.*\(.*\)<\/version>.*/\1/p' | \ + grep -E '^[0-9]+\.[0-9]+\.[0-9]+(-DD)?-SNAPSHOT$' | \ + sort -V | \ + tail -1 || echo "") + fi + + if [ -z "${latest_version}" ]; then + echo -e "${RED}[ERROR]${NC} Could not detect latest version for ${artifact_id}" >&2 + return 1 + fi + + echo -e "${GREEN}[INFO]${NC} Detected latest version: ${latest_version}" >&2 + echo "${latest_version}" +} + +function usage() { + cat << EOF +Usage: $0 [OPTIONS] + +Download dd-java-agent and ddprof snapshot artifacts from Maven. + +OPTIONS: + --dd-trace-version dd-java-agent version (default: auto-detect from Maven) + --ddprof-version ddprof version (auto-detected from build.env, CURRENT_VERSION, or Maven) + --output-dir Output directory (default: ${OUTPUT_DIR}) + --skip-ddprof Skip ddprof download (use when ddprof.jar already available) + --auto-detect Force auto-detection even if versions are set + --help Show this help message + +ENVIRONMENT VARIABLES: + DD_TRACE_VERSION Override dd-java-agent version + DDPROF_VERSION Override ddprof version + CURRENT_VERSION Auto-detected ddprof version from CI (build.env) + OUTPUT_DIR Output directory for artifacts + +OUTPUTS: + dd-java-agent-original.jar Downloaded dd-java-agent artifact + ddprof.jar Downloaded ddprof artifact + +NOTES: + - If no version is specified, the script will attempt to auto-detect + the latest SNAPSHOT version from the Maven repository + - Auto-detection queries maven-metadata.xml from OSSRH + - For CI/production use, explicit versions are recommended + +EXAMPLES: + # Download with auto-detected versions + $0 + + # Force auto-detection (ignore environment variables) + $0 --auto-detect + + # Download specific versions + $0 --dd-trace-version 1.49.0-SNAPSHOT --ddprof-version 1.35.0-DD-SNAPSHOT + + # Download to specific directory + $0 --output-dir /tmp/artifacts +EOF +} + +# Parse command line arguments +AUTO_DETECT=false +SKIP_DDPROF=false +DD_TRACE_VERSION_ARG="" +DDPROF_VERSION_ARG="" +while [ $# -gt 0 ]; do + case "$1" in + --dd-trace-version) + DD_TRACE_VERSION_ARG="$2" + shift 2 + ;; + --ddprof-version) + DDPROF_VERSION_ARG="$2" + shift 2 + ;; + --output-dir) + OUTPUT_DIR="$2" + shift 2 + ;; + --skip-ddprof) + SKIP_DDPROF=true + shift + ;; + --auto-detect) + AUTO_DETECT=true + shift + ;; + --help) + usage + exit 0 + ;; + *) + log_error "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +# Determine dd-trace-java version +if [ "${AUTO_DETECT}" = "true" ]; then + log_info "Auto-detect flag set, detecting latest dd-java-agent version..." + if ! DD_TRACE_VERSION=$(detect_latest_snapshot_version "com.datadoghq" "dd-java-agent" "${SNAPSHOT_REPO}"); then + log_warn "Auto-detection failed, using default: ${DEFAULT_DD_TRACE_VERSION}" + DD_TRACE_VERSION="${DEFAULT_DD_TRACE_VERSION}" + fi +elif [ -n "${DD_TRACE_VERSION_ARG}" ]; then + DD_TRACE_VERSION="${DD_TRACE_VERSION_ARG}" +elif [ -n "${DD_TRACE_VERSION:-}" ]; then + DD_TRACE_VERSION="${DD_TRACE_VERSION}" +else + # Auto-detect latest + log_info "No dd-java-agent version specified, detecting latest..." + if ! DD_TRACE_VERSION=$(detect_latest_snapshot_version "com.datadoghq" "dd-java-agent" "${SNAPSHOT_REPO}"); then + log_warn "Auto-detection failed, using default: ${DEFAULT_DD_TRACE_VERSION}" + DD_TRACE_VERSION="${DEFAULT_DD_TRACE_VERSION}" + fi +fi + +# Determine ddprof version (skip if --skip-ddprof flag is set) +if [ "${SKIP_DDPROF}" = "false" ]; then + if [ "${AUTO_DETECT}" = "true" ]; then + log_info "Auto-detect flag set, detecting latest ddprof version..." + if ! DDPROF_VERSION=$(detect_latest_snapshot_version "com.datadoghq" "ddprof" "${SNAPSHOT_REPO}"); then + log_error "Could not determine ddprof version (auto-detection failed)" + exit 1 + fi + elif [ -n "${DDPROF_VERSION_ARG}" ]; then + DDPROF_VERSION="${DDPROF_VERSION_ARG}" + elif [ -n "${DDPROF_VERSION:-}" ]; then + DDPROF_VERSION="${DDPROF_VERSION}" + elif [ -n "${CURRENT_VERSION:-}" ]; then + DDPROF_VERSION="${CURRENT_VERSION}" + elif [ -f "${PROJECT_ROOT}/build.env" ]; then + # Try to source build.env to get CURRENT_VERSION + log_info "Detecting ddprof version from build.env" + # shellcheck disable=SC1091 + source "${PROJECT_ROOT}/build.env" 2>/dev/null || true + DDPROF_VERSION="${CURRENT_VERSION:-}" + fi + + if [ -z "${DDPROF_VERSION:-}" ]; then + # Auto-detect latest + log_info "No ddprof version specified, detecting latest..." + if ! DDPROF_VERSION=$(detect_latest_snapshot_version "com.datadoghq" "ddprof" "${SNAPSHOT_REPO}"); then + log_error "Could not determine ddprof version (auto-detection failed and no fallback available)" + exit 1 + fi + fi +else + log_info "Skipping ddprof download (--skip-ddprof flag set)" +fi + +# Validate Maven is available +if ! command -v mvn &> /dev/null; then + log_error "Maven (mvn) is not installed or not in PATH" + log_error "Please install Maven to download artifacts" + exit 1 +fi + +# Create output directory +mkdir -p "${OUTPUT_DIR}" + +log_info "Downloading artifacts to: ${OUTPUT_DIR}" +log_info "dd-java-agent version: ${DD_TRACE_VERSION}" +if [ "${SKIP_DDPROF}" = "false" ]; then + log_info "ddprof version: ${DDPROF_VERSION}" +else + log_info "ddprof: skipping (using existing artifact)" +fi + +# Run Maven from a temp dir to avoid picking up the Gradle project's pom.xml (or a stale empty one) +MVN_WORK_DIR=$(mktemp -d) +trap "rm -rf ${MVN_WORK_DIR}" EXIT + +# Download dd-java-agent +log_info "Downloading dd-java-agent:${DD_TRACE_VERSION}..." +if (cd "${MVN_WORK_DIR}" && mvn org.apache.maven.plugins:maven-dependency-plugin:2.1:get \ + -DrepoUrl="${SNAPSHOT_REPO}" \ + -Dartifact="com.datadoghq:dd-java-agent:${DD_TRACE_VERSION}" \ + -q > /tmp/mvn-dd-trace.log 2>&1); then + log_info "Successfully downloaded dd-java-agent" +else + log_error "Failed to download dd-java-agent" + log_error "Maven output:" + cat /tmp/mvn-dd-trace.log + exit 1 +fi + +# Download ddprof (skip if --skip-ddprof flag is set) +if [ "${SKIP_DDPROF}" = "false" ]; then + log_info "Downloading ddprof:${DDPROF_VERSION}..." + if (cd "${MVN_WORK_DIR}" && mvn org.apache.maven.plugins:maven-dependency-plugin:2.1:get \ + -DrepoUrl="${SNAPSHOT_REPO}" \ + -Dartifact="com.datadoghq:ddprof:${DDPROF_VERSION}" \ + -q > /tmp/mvn-ddprof.log 2>&1); then + log_info "Successfully downloaded ddprof" + else + log_error "Failed to download ddprof" + log_error "Maven output:" + cat /tmp/mvn-ddprof.log + exit 1 + fi +fi + +# Determine Maven local repository path (default: ~/.m2/repository) +MVN_REPO="${HOME}/.m2/repository" +if [ -n "${MAVEN_CONFIG:-}" ]; then + # Check if custom Maven settings specify different local repo + log_warn "Custom MAVEN_CONFIG detected, using default ~/.m2/repository" +fi + +# Copy artifacts to output directory +DD_AGENT_JAR="${MVN_REPO}/com/datadoghq/dd-java-agent/${DD_TRACE_VERSION}/dd-java-agent-${DD_TRACE_VERSION}.jar" + +if [ ! -f "${DD_AGENT_JAR}" ]; then + log_error "dd-java-agent JAR not found at: ${DD_AGENT_JAR}" + exit 1 +fi + +log_info "Copying dd-java-agent to ${OUTPUT_DIR}/dd-java-agent-original.jar" +cp "${DD_AGENT_JAR}" "${OUTPUT_DIR}/dd-java-agent-original.jar" + +if [ "${SKIP_DDPROF}" = "false" ]; then + DDPROF_JAR="${MVN_REPO}/com/datadoghq/ddprof/${DDPROF_VERSION}/ddprof-${DDPROF_VERSION}.jar" + + if [ ! -f "${DDPROF_JAR}" ]; then + log_error "ddprof JAR not found at: ${DDPROF_JAR}" + exit 1 + fi + + log_info "Copying ddprof to ${OUTPUT_DIR}/ddprof.jar" + cp "${DDPROF_JAR}" "${OUTPUT_DIR}/ddprof.jar" +fi + +# Validate JAR integrity +log_info "Validating JAR integrity..." + +if unzip -t "${OUTPUT_DIR}/dd-java-agent-original.jar" > /dev/null 2>&1; then + log_info "✓ dd-java-agent-original.jar is valid" +else + log_error "✗ dd-java-agent-original.jar is corrupted" + exit 1 +fi + +if [ "${SKIP_DDPROF}" = "false" ]; then + if unzip -t "${OUTPUT_DIR}/ddprof.jar" > /dev/null 2>&1; then + log_info "✓ ddprof.jar is valid" + else + log_error "✗ ddprof.jar is corrupted" + exit 1 + fi +fi + +# Print summary +log_info "Download complete!" +log_info "Artifacts:" +log_info " dd-java-agent: $(du -h "${OUTPUT_DIR}/dd-java-agent-original.jar" | cut -f1)" +if [ "${SKIP_DDPROF}" = "false" ]; then + log_info " ddprof: $(du -h "${OUTPUT_DIR}/ddprof.jar" | cut -f1)" +fi diff --git a/.gitlab/dd-trace-integration/generate-report.sh b/.gitlab/dd-trace-integration/generate-report.sh new file mode 100755 index 000000000..816f3670b --- /dev/null +++ b/.gitlab/dd-trace-integration/generate-report.sh @@ -0,0 +1,199 @@ +#!/bin/bash + +# generate-report.sh - Generate full integration test report in Markdown +# +# Usage: generate-report.sh [output-file] +# +# Generates a comprehensive Markdown report including: +# - Test configuration +# - Both scenario results (profiler-only, tracer+profiler) +# - System diagnostics (CPU, throttling) +# - Health scores and sample rates + +set -euo pipefail + +RESULTS_DIR="${1:-}" +OUTPUT_FILE="${2:-}" + +if [ -z "${RESULTS_DIR}" ]; then + echo "Usage: $0 [output-file]" >&2 + exit 1 +fi + +if [ ! -d "${RESULTS_DIR}" ]; then + echo "Error: Results directory not found: ${RESULTS_DIR}" >&2 + exit 1 +fi + +# Extract config from directory name (e.g., glibc-x64-hotspot-jdk17) +CONFIG_NAME=$(basename "${RESULTS_DIR}") +IFS='-' read -r LIBC ARCH JVM_TYPE JDK_VERSION <<< "${CONFIG_NAME}" + +# Get timestamp +TIMESTAMP=$(date -Iseconds 2>/dev/null || date +%Y-%m-%dT%H:%M:%S%z) +DATE_HUMAN=$(date "+%Y-%m-%d %H:%M:%S %Z" 2>/dev/null || date) + +# Parse diagnostic data +DIAGNOSTICS_DIR="${RESULTS_DIR}/diagnostics" +CPU_START="N/A" +CPU_END="N/A" +THROTTLE_PCT="N/A" +CONTAINER="N/A" + +if [ -d "${DIAGNOSTICS_DIR}" ]; then + if [ -f "${DIAGNOSTICS_DIR}/system-metrics-start.json" ]; then + CPU_START=$(grep '"cpu_count"' "${DIAGNOSTICS_DIR}/system-metrics-start.json" | awk -F': ' '{print $2}' | tr -d ',' || echo "N/A") + CONTAINER=$(grep '"container"' "${DIAGNOSTICS_DIR}/system-metrics-start.json" | awk -F': ' '{print $2}' | tr -d ',' || echo "N/A") + fi + if [ -f "${DIAGNOSTICS_DIR}/system-metrics-end.json" ]; then + CPU_END=$(grep '"cpu_count"' "${DIAGNOSTICS_DIR}/system-metrics-end.json" | awk -F': ' '{print $2}' | tr -d ',' || echo "N/A") + THROTTLE_PCT=$(grep '"percentage"' "${DIAGNOSTICS_DIR}/system-metrics-end.json" | awk -F': ' '{print $2}' | tr -d ',' || echo "N/A") + elif [ -f "${DIAGNOSTICS_DIR}/system-metrics-mid.json" ]; then + CPU_END=$(grep '"cpu_count"' "${DIAGNOSTICS_DIR}/system-metrics-mid.json" | awk -F': ' '{print $2}' | tr -d ',' || echo "N/A") + THROTTLE_PCT=$(grep '"percentage"' "${DIAGNOSTICS_DIR}/system-metrics-mid.json" | awk -F': ' '{print $2}' | tr -d ',' || echo "N/A") + fi +fi + +# Parse scenario 1 (profiler-only) results +S1_STATUS="N/A" +S1_SAMPLES="N/A" +S1_THREADS="N/A" +S1_ALLOC="N/A" +S1_LOG="${RESULTS_DIR}/profiler-only-${CONFIG_NAME}.log" + +if [ -f "${S1_LOG}" ]; then + if grep -q "SUCCESS: All validations passed" "${S1_LOG}"; then + S1_STATUS="PASS" + elif grep -q "VALIDATION_FAILED" "${S1_LOG}"; then + S1_STATUS="FAIL" + fi + S1_SAMPLES=$(grep "ExecutionSample:" "${S1_LOG}" | grep -oE '[0-9]+ events' | awk '{print $1}' | head -1 || echo "N/A") + S1_THREADS=$(grep "Thread diversity:" "${S1_LOG}" | grep -oE '[0-9]+ threads' | awk '{print $1}' | head -1 || echo "N/A") + S1_ALLOC=$(grep "Allocation samples:" "${S1_LOG}" | grep -oE '[0-9]+ events' | awk '{print $1}' | head -1 || echo "N/A") +fi + +# Parse scenario 2 (tracer+profiler) results +S2_STATUS="N/A" +S2_SAMPLES="N/A" +S2_THREADS="N/A" +S2_ALLOC="N/A" +S2_LOG="${RESULTS_DIR}/tracer-profiler-${CONFIG_NAME}.log" + +if [ -f "${S2_LOG}" ]; then + if grep -q "SUCCESS: All validations passed" "${S2_LOG}"; then + S2_STATUS="PASS" + elif grep -q "VALIDATION_FAILED" "${S2_LOG}"; then + S2_STATUS="FAIL" + fi + S2_SAMPLES=$(grep "ExecutionSample:" "${S2_LOG}" | grep -oE '[0-9]+ events' | awk '{print $1}' | head -1 || echo "N/A") + S2_THREADS=$(grep "Thread diversity:" "${S2_LOG}" | grep -oE '[0-9]+ threads' | awk '{print $1}' | head -1 || echo "N/A") + S2_ALLOC=$(grep "Allocation samples:" "${S2_LOG}" | grep -oE '[0-9]+ events' | awk '{print $1}' | head -1 || echo "N/A") +fi + +# Calculate overall status +OVERALL_STATUS="PASS" +if [ "${S1_STATUS}" = "FAIL" ] || [ "${S2_STATUS}" = "FAIL" ]; then + OVERALL_STATUS="FAIL" +fi + +# Calculate health scores (samples/sec compared to expected 1.6/sec baseline) +TEST_DURATION=60 # Default test duration (see run-integration-test.sh) +S1_RATE="N/A" +S1_HEALTH="N/A" +S2_RATE="N/A" +S2_HEALTH="N/A" + +if [ "${S1_SAMPLES}" != "N/A" ] && [ -n "${S1_SAMPLES}" ]; then + S1_RATE=$(awk "BEGIN {printf \"%.2f\", ${S1_SAMPLES} / ${TEST_DURATION}}") + S1_HEALTH=$(awk "BEGIN {printf \"%.0f\", (${S1_RATE} / 1.6) * 100}") +fi + +if [ "${S2_SAMPLES}" != "N/A" ] && [ -n "${S2_SAMPLES}" ]; then + S2_RATE=$(awk "BEGIN {printf \"%.2f\", ${S2_SAMPLES} / ${TEST_DURATION}}") + S2_HEALTH=$(awk "BEGIN {printf \"%.0f\", (${S2_RATE} / 1.6) * 100}") +fi + +# Status emoji +status_emoji() { + case "$1" in + PASS) echo "✅" ;; + FAIL) echo "❌" ;; + *) echo "⚠️" ;; + esac +} + +# Generate Markdown report +generate_report() { + cat < +CPU Timeline (${cpu_values} unique values: ${cpu_min}-${cpu_max} cores) + +\`\`\` +$(head -20 "${DIAGNOSTICS_DIR}/cpu-timeline.log") +\`\`\` + + +EOF + fi + + echo "---" + echo "" +} + +# Output report +if [ -n "${OUTPUT_FILE}" ]; then + generate_report > "${OUTPUT_FILE}" + echo "Report generated: ${OUTPUT_FILE}" +else + generate_report +fi diff --git a/.gitlab/dd-trace-integration/generate-run-json.sh b/.gitlab/dd-trace-integration/generate-run-json.sh new file mode 100755 index 000000000..616d709e4 --- /dev/null +++ b/.gitlab/dd-trace-integration/generate-run-json.sh @@ -0,0 +1,223 @@ +#!/bin/bash + +# generate-run-json.sh - Generate run JSON for integration tests +# +# Usage: generate-run-json.sh [results-dir] [--verbose] +# +# Parses all test configuration directories and outputs a JSON object +# suitable for update-history.sh. Reads CI environment variables for metadata. +# +# Pure bash implementation - no Python required + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="${SCRIPT_DIR}/../.." + +# Parse arguments +VERBOSE=false +RESULTS_BASE="" +for arg in "$@"; do + case "$arg" in + --verbose) + VERBOSE=true + ;; + *) + RESULTS_BASE="$arg" + ;; + esac +done + +# Default results base if not provided +RESULTS_BASE="${RESULTS_BASE:-${PROJECT_ROOT}/integration-test-results}" + +# Read metadata from environment or defaults +TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +PIPELINE_ID="${CI_PIPELINE_ID:-0}" +PIPELINE_URL="${CI_PIPELINE_URL:-#}" +DDPROF_BRANCH="${DDPROF_COMMIT_BRANCH:-${CI_COMMIT_BRANCH:-main}}" +DDPROF_SHA="${DDPROF_COMMIT_SHA:-$(cat "${PROJECT_ROOT}/ddprof-commit-sha.txt" 2>/dev/null || echo unknown)}" + +# Read version from version.txt if available +LIB_VERSION="unknown" +if [ -f "${PROJECT_ROOT}/version.txt" ]; then + LIB_VERSION=$(awk -F: '{print $NF}' "${PROJECT_ROOT}/version.txt" | tr -d ' ') +fi + +# Lookup PR for branch +PR_JSON="{}" +if [ -x "${SCRIPT_DIR}/../common/lookup-pr.sh" ]; then + PR_JSON=$("${SCRIPT_DIR}/../common/lookup-pr.sh" "${DDPROF_BRANCH}" 2>/dev/null) || PR_JSON="{}" +fi + +# Parse validation log for status +parse_validation_log() { + local log_path="$1" + + if [ ! -f "$log_path" ]; then + [ "$VERBOSE" = "true" ] && echo "[DEBUG] Log not found: $log_path" >&2 + echo "missing" + return + fi + + if grep -q "SUCCESS: All validations passed" "$log_path" 2>/dev/null; then + [ "$VERBOSE" = "true" ] && echo "[DEBUG] ✓ PASSED: $log_path" >&2 + echo "passed" + elif grep -q "VALIDATION_FAILED" "$log_path" 2>/dev/null; then + [ "$VERBOSE" = "true" ] && echo "[DEBUG] ✗ FAILED: $log_path" >&2 + echo "failed" + else + # Log exists but has no status markers - this is the problem! + echo "[WARN] Log missing markers: $log_path" >&2 + if [ "$VERBOSE" = "true" ]; then + echo "[DEBUG] Log preview (first 10 lines):" >&2 + head -10 "$log_path" >&2 + echo "[DEBUG] Log preview (last 10 lines):" >&2 + tail -10 "$log_path" >&2 + fi + echo "unknown" + fi +} + +# Initialize counters +total_jobs=0 +passed_jobs=0 +failed_jobs=0 +total_scenarios=0 +passed_scenarios=0 +failed_scenarios=0 +unknown_scenarios=0 +failed_configs="" + +# Parse test results if directory exists +if [ -d "${RESULTS_BASE}" ]; then + for config_dir in "${RESULTS_BASE}"/*; do + [ -d "$config_dir" ] || continue + + config_name=$(basename "$config_dir") + [ "$VERBOSE" = "true" ] && echo "[DEBUG] Processing config: $config_name" >&2 + + # Skip history directory + [ "$config_name" = "history" ] && continue + + # Skip if no log files + [ -z "$(ls "$config_dir"/*.log 2>/dev/null)" ] && continue + + total_jobs=$((total_jobs + 1)) + + # Check profiler-only scenario + s1_log="${config_dir}/profiler-only-${config_name}.log" + s1_status=$(parse_validation_log "$s1_log") + + # Check tracer+profiler scenario + s2_log="${config_dir}/tracer-profiler-${config_name}.log" + s2_status=$(parse_validation_log "$s2_log") + + # Count scenarios + for status in "$s1_status" "$s2_status"; do + if [ "$status" != "missing" ]; then + total_scenarios=$((total_scenarios + 1)) + if [ "$status" = "passed" ]; then + passed_scenarios=$((passed_scenarios + 1)) + elif [ "$status" = "failed" ]; then + failed_scenarios=$((failed_scenarios + 1)) + elif [ "$status" = "unknown" ]; then + unknown_scenarios=$((unknown_scenarios + 1)) + fi + fi + done + + # Determine job status + if [ "$s1_status" = "passed" ] && [ "$s2_status" = "passed" ]; then + passed_jobs=$((passed_jobs + 1)) + else + failed_jobs=$((failed_jobs + 1)) + failed_configs="${failed_configs}${failed_configs:+, }\"${config_name}\"" + fi + done +fi + +# Report diagnostic summary to stderr +if [ $unknown_scenarios -gt 0 ]; then + echo "" >&2 + echo "[ERROR] ================================================" >&2 + echo "[ERROR] Found $unknown_scenarios scenario(s) with logs but NO status markers!" >&2 + echo "[ERROR] ================================================" >&2 + echo "[ERROR]" >&2 + echo "[ERROR] This means validation logs exist but don't contain:" >&2 + echo "[ERROR] - 'SUCCESS: All validations passed'" >&2 + echo "[ERROR] - 'VALIDATION_FAILED'" >&2 + echo "[ERROR]" >&2 + echo "[ERROR] Likely causes:" >&2 + echo "[ERROR] 1. Validation script crashed before completion" >&2 + echo "[ERROR] 2. jbang/jfr-shell failed to run" >&2 + echo "[ERROR] 3. Process killed/timeout before markers written" >&2 + echo "[ERROR]" >&2 + echo "[ERROR] Check the logs above for '[WARN] Log missing markers:'" >&2 + echo "[ERROR] ================================================" >&2 + echo "" >&2 +fi + +if [ "$VERBOSE" = "true" ]; then + echo "[DEBUG] Counters: total_jobs=$total_jobs, passed_jobs=$passed_jobs, failed_jobs=$failed_jobs" >&2 + echo "[DEBUG] Scenarios: total=$total_scenarios, passed=$passed_scenarios, failed=$failed_scenarios, unknown=$unknown_scenarios" >&2 + echo "[INFO] Parsing summary:" >&2 + echo "[INFO] Total configs found: $total_jobs" >&2 + echo "[INFO] Total scenarios: $total_scenarios" >&2 + echo "[INFO] Passed: $passed_scenarios" >&2 + echo "[INFO] Failed: $failed_scenarios" >&2 + echo "[INFO] Unknown: $unknown_scenarios" >&2 + echo "" >&2 +fi + +# Determine overall status +if [ $failed_jobs -eq 0 ] && [ $total_jobs -gt 0 ]; then + status="passed" +elif [ $passed_jobs -eq 0 ] && [ $total_jobs -gt 0 ]; then + status="failed" +elif [ $total_jobs -eq 0 ]; then + status="unknown" +else + status="partial" +fi + +# Extract PR number if available +pr_field="null" +if command -v jq >/dev/null 2>&1; then + pr_number=$(echo "$PR_JSON" | jq -r '.number // empty' 2>/dev/null || echo "") + if [ -n "$pr_number" ]; then + pr_field="$PR_JSON" + fi +else + # Fallback: simple grep for number field + if echo "$PR_JSON" | grep -q '"number"'; then + pr_field="$PR_JSON" + fi +fi + +# Generate JSON (without jq dependency for maximum compatibility) +cat < /dev/null; then + log_info "Installing jbang..." + + # Download and install jbang with retry logic + for attempt in $(seq 1 $MAX_RETRIES); do + log_info "jbang installation attempt $attempt of $MAX_RETRIES..." + if curl -Ls https://sh.jbang.dev | bash -s - app setup; then + break + fi + if [ "$attempt" -lt $MAX_RETRIES ]; then + log_warn "jbang installation failed, retrying in ${RETRY_DELAY}s..." + sleep $RETRY_DELAY + RETRY_DELAY=$((RETRY_DELAY * 2)) + else + log_warn "jbang installation failed after $MAX_RETRIES attempts" + fi + done + + # Add to PATH for current session + export PATH="$HOME/.jbang/bin:$PATH" + + # Verify installation + if command -v jbang &> /dev/null; then + JBANG_VERSION=$(jbang version 2>&1 | head -1) + log_info "jbang installed successfully: ${JBANG_VERSION}" + else + log_warn "jbang installation completed but not found in PATH" + log_warn "Please ensure ~/.jbang/bin is in your PATH" + fi +else + JBANG_VERSION=$(jbang version 2>&1 | head -1) + log_info "jbang already installed: ${JBANG_VERSION}" +fi + +# Add jbang trust for jfr-shell from btraceio +log_info "Adding jbang trust for btraceio catalog..." +jbang trust add https://github.com/btraceio/ 2>/dev/null || true + +# Pre-install JDK 25 for jbang (required by jfr-shell) +# First check if JDK 25 is already available (e.g., Java 25 test jobs) +log_info "Checking if JDK 25 is available for jbang..." +if jbang jdk list 2>&1 | grep -q "25"; then + log_info "JDK 25 already available for jbang" + JDK25_INSTALLED=true +else + log_info "Pre-installing JDK 25 for jbang (required by jfr-shell)..." + JDK25_INSTALLED=false + + # Try jbang's built-in install first + for attempt in $(seq 1 $MAX_RETRIES); do + log_info "JDK 25 installation attempt $attempt of $MAX_RETRIES (via jbang)..." + if jbang jdk install 25 2>&1; then + JDK25_INSTALLED=true + log_info "JDK 25 installed for jbang" + break + fi + if [ "$attempt" -lt $MAX_RETRIES ]; then + log_warn "JDK 25 installation failed, retrying in ${RETRY_DELAY}s..." + sleep $RETRY_DELAY + fi + done + + # Fallback: manually download from Adoptium if jbang failed (Foojay API down) + if [ "$JDK25_INSTALLED" = "false" ]; then + log_warn "jbang install failed (Foojay API may be down), trying direct Adoptium download..." + + # Detect architecture + ARCH=$(uname -m) + case "$ARCH" in + x86_64|amd64) ADOPTIUM_ARCH="x64" ;; + aarch64|arm64) ADOPTIUM_ARCH="aarch64" ;; + *) log_warn "Unknown arch: $ARCH"; ADOPTIUM_ARCH="x64" ;; + esac + + # Detect OS and libc + if [ -f /etc/alpine-release ]; then + ADOPTIUM_OS="alpine-linux" + else + ADOPTIUM_OS="linux" + fi + + JBANG_JDK_DIR="$HOME/.jbang/cache/jdks/25" + ADOPTIUM_URL="https://api.adoptium.net/v3/binary/latest/25/ga/${ADOPTIUM_OS}/${ADOPTIUM_ARCH}/jdk/hotspot/normal/eclipse" + + log_info "Downloading JDK 25 from Adoptium: $ADOPTIUM_URL" + if curl -fsSL -o /tmp/jdk25.tar.gz "$ADOPTIUM_URL"; then + mkdir -p "$JBANG_JDK_DIR" + tar -xzf /tmp/jdk25.tar.gz -C "$JBANG_JDK_DIR" --strip-components=1 + rm -f /tmp/jdk25.tar.gz + + if [ -x "$JBANG_JDK_DIR/bin/java" ]; then + JDK25_INSTALLED=true + log_info "JDK 25 installed manually from Adoptium" + "$JBANG_JDK_DIR/bin/java" -version 2>&1 | head -1 + else + log_warn "JDK 25 extraction failed" + fi + else + log_warn "Failed to download JDK 25 from Adoptium" + fi + fi +fi + +if [ "$JDK25_INSTALLED" = "false" ]; then + log_warn "Failed to install JDK 25 for jbang" + log_warn "JFR validation will be skipped" + # Create marker file to skip JFR validation + touch /tmp/skip-jfr-validation +fi + +# ======================================== +# 2. Verify Java is available +# ======================================== +if [ -z "${JAVA_HOME:-}" ]; then + if command -v java &> /dev/null; then + log_info "Java found in PATH" + java -version 2>&1 | head -3 + else + echo "ERROR: Java not found. Please set JAVA_HOME or ensure java is in PATH" + exit 1 + fi +else + if [ ! -x "${JAVA_HOME}/bin/java" ]; then + echo "ERROR: Java not found at: ${JAVA_HOME}/bin/java" + exit 1 + fi + + log_info "Java found at JAVA_HOME: ${JAVA_HOME}" + "${JAVA_HOME}/bin/java" -version 2>&1 | head -3 +fi + +# ======================================== +# 3. Install basic tools (if needed) +# ======================================== +# Check for javac (needed to compile test app) +if [ -n "${JAVA_HOME:-}" ]; then + if [ ! -x "${JAVA_HOME}/bin/javac" ]; then + log_warn "javac not found at ${JAVA_HOME}/bin/javac" + log_warn "This may cause test app compilation to fail" + fi +elif ! command -v javac &> /dev/null; then + log_warn "javac not found in PATH" + log_warn "This may cause test app compilation to fail" +fi + +# ======================================== +# 4. Create output directories +# ======================================== +log_info "Creating output directories..." + +mkdir -p integration-test-results +mkdir -p /tmp/jfr-validation + +log_info "✓ Prerequisites installation complete" +log_info "" +log_info "Installed tools:" +log_info " - jbang: $(command -v jbang || echo 'not in PATH')" +log_info " - jbang JDKs: $(jbang jdk list 2>&1 | grep -v '^$' | tr '\n' ' ')" +log_info " - java: $(command -v java || echo 'not in PATH')" +log_info " - javac: $(command -v javac || echo 'not in PATH')" +log_info "" diff --git a/.gitlab/dd-trace-integration/notify_channel.sh b/.gitlab/dd-trace-integration/notify_channel.sh new file mode 100755 index 000000000..12c7c9926 --- /dev/null +++ b/.gitlab/dd-trace-integration/notify_channel.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -euxo pipefail + +if [ ! -z "${CANCELLED:-}" ]; then + exit 0 +fi + +# Source centralized configuration +HERE=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${HERE}/../../.gitlab/config.env" + +VERSION=${1:-"unknown"} +PIPELINE_URL="$CI_PROJECT_URL/pipelines/$CI_PIPELINE_ID" +PIPELINE_LINK="<$PIPELINE_URL|pipeline #$CI_PIPELINE_ID>" + +MESSAGE_TEXT=":better-siren: dd-trace-java integration tests failed for ${VERSION} (pipeline=$PIPELINE_LINK) + +Some integration tests failed across the test matrix. +Please review the pipeline artifacts for details." + +postmessage "$SLACK_CHANNEL" "$MESSAGE_TEXT" "alert" diff --git a/.gitlab/dd-trace-integration/patch-dd-java-agent.sh b/.gitlab/dd-trace-integration/patch-dd-java-agent.sh new file mode 100755 index 000000000..768e319b4 --- /dev/null +++ b/.gitlab/dd-trace-integration/patch-dd-java-agent.sh @@ -0,0 +1,385 @@ +#!/bin/bash + +set -euo pipefail + +# Patch dd-java-agent.jar with ddprof contents + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +# Input JARs +DD_AGENT_JAR="${DD_AGENT_JAR:-${PROJECT_ROOT}/dd-java-agent-original.jar}" +DDPROF_JAR="${DDPROF_JAR:-${PROJECT_ROOT}/ddprof.jar}" + +# Output JAR +OUTPUT_JAR="${OUTPUT_JAR:-${PROJECT_ROOT}/dd-java-agent-patched.jar}" + +# Working directory for extraction +# Use mktemp for guaranteed unique directory, fall back to PID-based if mktemp unavailable +WORK_DIR="${WORK_DIR:-$(mktemp -d 2>/dev/null || echo "/tmp/jar-patch-$$")}" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +function log_info() { + echo -e "${GREEN}[INFO]${NC} $*" +} + +function log_warn() { + echo -e "${YELLOW}[WARN]${NC} $*" +} + +function log_error() { + echo -e "${RED}[ERROR]${NC} $*" +} + +function log_debug() { + if [ "${DEBUG:-false}" = "true" ]; then + echo -e "${BLUE}[DEBUG]${NC} $*" + fi +} + +function cleanup() { + if [ -d "${WORK_DIR}" ]; then + log_debug "Cleaning up work directory: ${WORK_DIR}" + rm -rf "${WORK_DIR}" + fi +} + +trap cleanup EXIT + +# Validate required tools are available +for tool in unzip zip dirname basename; do + if ! command -v "$tool" &> /dev/null; then + log_error "Required tool not found: $tool" + log_error "Please install $tool to continue" + exit 1 + fi +done + +function usage() { + cat << EOF +Usage: $0 [OPTIONS] + +Patch dd-java-agent.jar with ddprof contents. + +Mapping rules: + - Native libraries: ddprof.jar:META-INF/native-libs/** → dd-java-agent.jar:shared/META-INF/native-libs/** + - Class files: ddprof.jar:**/*.class → dd-java-agent.jar:shared/**/*.classdata (same package structure) + +OPTIONS: + --dd-agent-jar Path to dd-java-agent-original.jar (default: ${DD_AGENT_JAR}) + --ddprof-jar Path to ddprof.jar (default: ${DDPROF_JAR}) + --output-jar Path to output patched jar (default: ${OUTPUT_JAR}) + --work-dir Working directory for extraction (default: /tmp/jar-patch-\$\$) + --debug Enable debug output + --help Show this help message + +ENVIRONMENT VARIABLES: + DD_AGENT_JAR Path to dd-java-agent-original.jar + DDPROF_JAR Path to ddprof.jar + OUTPUT_JAR Path to output patched jar + WORK_DIR Working directory for extraction + DEBUG Enable debug output (true/false) + +EXAMPLES: + # Patch with default paths + $0 + + # Patch with custom paths + $0 --dd-agent-jar /path/to/agent.jar --ddprof-jar /path/to/ddprof.jar + + # Enable debug output + DEBUG=true $0 +EOF +} + +# Parse command line arguments +while [ $# -gt 0 ]; do + case "$1" in + --dd-agent-jar) + DD_AGENT_JAR="$2" + shift 2 + ;; + --ddprof-jar) + DDPROF_JAR="$2" + shift 2 + ;; + --output-jar) + OUTPUT_JAR="$2" + shift 2 + ;; + --work-dir) + WORK_DIR="$2" + shift 2 + ;; + --debug) + DEBUG=true + shift + ;; + --help) + usage + exit 0 + ;; + *) + log_error "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +# Validate input JARs exist +if [ ! -f "${DD_AGENT_JAR}" ]; then + log_error "dd-java-agent JAR not found: ${DD_AGENT_JAR}" + exit 1 +fi + +if [ ! -f "${DDPROF_JAR}" ]; then + log_error "ddprof JAR not found: ${DDPROF_JAR}" + exit 1 +fi + +log_info "Starting JAR patching process" +log_info " dd-java-agent: ${DD_AGENT_JAR}" +log_info " ddprof: ${DDPROF_JAR}" +log_info " output: ${OUTPUT_JAR}" +log_info " work dir: ${WORK_DIR}" + +# Create working directories +if ! mkdir -p "${WORK_DIR}/agent" "${WORK_DIR}/ddprof"; then + log_error "Failed to create working directories in: ${WORK_DIR}" + log_error "Check disk space and permissions" + exit 1 +fi + +# Extract dd-java-agent +log_info "Extracting dd-java-agent..." +if ! unzip -q "${DD_AGENT_JAR}" -d "${WORK_DIR}/agent/"; then + log_error "Failed to extract dd-java-agent: ${DD_AGENT_JAR}" + log_error "Check if file is corrupted or disk is full" + exit 1 +fi +log_info "✓ Extracted dd-java-agent" + +# Extract ddprof +log_info "Extracting ddprof..." +if ! unzip -q "${DDPROF_JAR}" -d "${WORK_DIR}/ddprof/"; then + log_error "Failed to extract ddprof: ${DDPROF_JAR}" + log_error "Check if file is corrupted or disk is full" + exit 1 +fi +log_info "✓ Extracted ddprof" + +# Create shared directory structure in agent +mkdir -p "${WORK_DIR}/agent/shared/META-INF" + +# Copy native libraries +log_info "Copying native libraries..." +if [ -d "${WORK_DIR}/ddprof/META-INF/native-libs" ]; then + cp -r "${WORK_DIR}/ddprof/META-INF/native-libs" "${WORK_DIR}/agent/shared/META-INF/" + + # Count native libraries + NATIVE_COUNT=$(find "${WORK_DIR}/agent/shared/META-INF/native-libs" -type f -name "*.so" | wc -l | tr -d ' ') + log_info "✓ Copied ${NATIVE_COUNT} native libraries" + + # List platforms + if [ "${DEBUG:-false}" = "true" ]; then + log_debug "Native library platforms:" + find "${WORK_DIR}/agent/shared/META-INF/native-libs" -type d -mindepth 1 -maxdepth 1 -exec basename {} \; | while read -r platform; do + log_debug " - ${platform}" + done + fi +else + log_warn "No native libraries found in ddprof (META-INF/native-libs missing)" +fi + +# Copy and rename class files +log_info "Copying and renaming class files..." +CLASS_COUNT=0 +SKIPPED_COUNT=0 + +# Count total class files first for progress tracking +TOTAL_CLASSES=$(find "${WORK_DIR}/ddprof" -name "*.class" -type f | wc -l | tr -d ' ') + +# Validate TOTAL_CLASSES is a number +if ! [[ "${TOTAL_CLASSES}" =~ ^[0-9]+$ ]]; then + log_error "Failed to count class files, got: '${TOTAL_CLASSES}'" + exit 1 +fi + +log_info "Found ${TOTAL_CLASSES} class files to process" + +# Verify ddprof extraction directory exists and has content +if [ ! -d "${WORK_DIR}/ddprof" ]; then + log_error "ddprof extraction directory not found: ${WORK_DIR}/ddprof" + exit 1 +fi + +# Exit early if no files to process +if [ "${TOTAL_CLASSES}" -eq 0 ]; then + log_warn "No class files found to process" + # Skip loop but don't fail - might be expected for some builds +fi + +log_info "Starting class file copy loop..." + +# Use find to locate all .class files, excluding META-INF +while IFS= read -r -d '' classfile; do + log_debug "Processing: ${classfile}" + + # Get relative path from ddprof root + relpath="${classfile#"${WORK_DIR}"/ddprof/}" + + # Skip META-INF directory + if [[ "$relpath" == META-INF/* ]]; then + log_debug "Skipping META-INF class: ${relpath}" + SKIPPED_COUNT=$((SKIPPED_COUNT + 1)) + continue + fi + + # Convert path: foo/bar/Baz.class → shared/foo/bar/Baz.classdata + targetpath="${WORK_DIR}/agent/shared/${relpath%.class}.classdata" + targetdir=$(dirname "$targetpath") + + # Create target directory if needed + if ! mkdir -p "$targetdir"; then + log_error "Failed to create directory: $targetdir" + log_error "Check disk space and permissions" + df -h "${WORK_DIR}" >&2 + exit 1 + fi + + # Copy file + if ! cp "$classfile" "$targetpath"; then + log_error "Failed to copy class file: $classfile" + log_error "Target: $targetpath" + log_error "Check disk space and permissions" + df -h "${WORK_DIR}" >&2 + exit 1 + fi + CLASS_COUNT=$((CLASS_COUNT + 1)) + + # Show progress every 100 files + if [ $((CLASS_COUNT % 100)) -eq 0 ]; then + log_debug "Progress: ${CLASS_COUNT}/${TOTAL_CLASSES} files copied" + fi + + log_debug "Copied: ${relpath} → shared/${relpath%.class}.classdata" +done < <(find "${WORK_DIR}/ddprof" -name "*.class" -type f -print0) + +# Verify loop completed successfully +if [ "${CLASS_COUNT}" -eq 0 ] && [ "${TOTAL_CLASSES}" -gt 0 ]; then + log_error "Class file loop failed - no files were copied despite ${TOTAL_CLASSES} files found" + exit 1 +fi + +log_info "✓ Copied and renamed ${CLASS_COUNT} class files (${SKIPPED_COUNT} skipped from META-INF)" + +# List some sample class files for verification +if [ "${DEBUG:-false}" = "true" ]; then + log_debug "Sample classdata files:" + # Use process substitution to avoid SIGPIPE from head in pipeline + count=0 + while IFS= read -r f && [ "$count" -lt 5 ]; do + relpath="${f#"${WORK_DIR}"/agent/}" + log_debug " - ${relpath}" + count=$((count + 1)) + done < <(find "${WORK_DIR}/agent/shared" -name "*.classdata" -type f) +fi + +# Repackage JAR +log_info "Repackaging JAR..." + +# Save original directory and change to agent directory +ORIG_DIR=$(pwd) +if ! cd "${WORK_DIR}/agent"; then + log_error "Failed to change directory to ${WORK_DIR}/agent" + log_error "Check if directory exists and is accessible" + exit 1 +fi + +if ! zip -r -q "${OUTPUT_JAR}" .; then + log_error "Failed to repackage JAR" + log_error "Check disk space and write permissions for: ${OUTPUT_JAR}" + cd "$ORIG_DIR" || true + exit 1 +fi + +# Restore original directory +cd "$ORIG_DIR" || log_warn "Failed to return to original directory" + +log_info "✓ Created patched JAR: ${OUTPUT_JAR}" + +# Validate patched JAR +log_info "Validating patched JAR..." + +if ! unzip -t "${OUTPUT_JAR}" > /dev/null 2>&1; then + log_error "Patched JAR is corrupted" + exit 1 +fi +log_info "✓ JAR integrity check passed" + +# Verify shared directory structure +log_info "Verifying structure..." +SHARED_NATIVE_COUNT=$(unzip -l "${OUTPUT_JAR}" | grep -c "shared/META-INF/native-libs/.*\.so$" || echo "0") +TOTAL_CLASSDATA_COUNT=$(unzip -l "${OUTPUT_JAR}" | grep -c "shared/.*\.classdata$" || echo "0") + +if [ "${SHARED_NATIVE_COUNT}" -eq 0 ] && [ "${NATIVE_COUNT}" -gt 0 ]; then + log_error "Native libraries missing in patched JAR" + exit 1 +fi + +# Diagnostic: Check for unexpected classes in ddprof-owned package namespaces +# Only check com/datadoghq/profiler - this is the actual ddprof package +log_debug "Checking for unexpected classes in com/datadoghq/profiler package..." +DDPROF_CLASSES=$(unzip -l "${DDPROF_JAR}" '*.class' 2>/dev/null | \ + awk '{print $NF}' | \ + grep "^com/datadoghq/profiler/" | \ + grep "\.class$" | \ + sed 's/\.class$//' | \ + sort || true) + +# Get classes from patched JAR in the ddprof package +PATCHED_CLASSES=$(unzip -l "${OUTPUT_JAR}" 2>/dev/null | \ + grep "shared/com/datadoghq/profiler/.*\.classdata$" | \ + awk '{print $NF}' | \ + sed 's|^shared/||' | \ + sed 's/\.classdata$//' | \ + sort || true) + +# Find classes in patched JAR that weren't in original ddprof.jar +UNEXPECTED_CLASSES=$(comm -13 <(echo "${DDPROF_CLASSES}") <(echo "${PATCHED_CLASSES}") || true) + +if [ -n "${UNEXPECTED_CLASSES}" ]; then + UNEXPECTED_COUNT=$(echo "${UNEXPECTED_CLASSES}" | grep -c . || echo "0") + log_warn "Found ${UNEXPECTED_COUNT} unexpected classes in com/datadoghq/profiler:" + log_warn "These classes exist in dd-java-agent but not in ddprof.jar:" + echo "${UNEXPECTED_CLASSES}" | while IFS= read -r cls; do + log_debug " - ${cls}" + done | head -10 + if [ "${UNEXPECTED_COUNT}" -gt 10 ]; then + log_debug " ... and $((UNEXPECTED_COUNT - 10)) more" + fi + log_warn "This may indicate:" + log_warn " - Package namespace overlap between dd-java-agent and ddprof" + log_warn " - Classes added to ddprof.jar (valid scenario)" +fi + +log_info "✓ Structure verification passed" +log_info " - ${SHARED_NATIVE_COUNT} native libraries in shared/META-INF/native-libs/" +log_info " - ${CLASS_COUNT} ddprof classes added to patched JAR" +log_info " - ${TOTAL_CLASSDATA_COUNT} total classdata files in patched JAR" + +# Print summary +JAR_SIZE=$(du -h "${OUTPUT_JAR}" | cut -f1) +log_info "" +log_info "Patching complete!" +log_info " Output: ${OUTPUT_JAR} (${JAR_SIZE})" +log_info "" +log_info "To inspect the patched JAR structure:" +log_info " unzip -l ${OUTPUT_JAR} | grep shared/" diff --git a/.gitlab/dd-trace-integration/post-pr-comment.sh b/.gitlab/dd-trace-integration/post-pr-comment.sh new file mode 100755 index 000000000..a0d834915 --- /dev/null +++ b/.gitlab/dd-trace-integration/post-pr-comment.sh @@ -0,0 +1,237 @@ +#!/bin/bash + +# post-pr-comment.sh - Post integration test results as PR comment +# +# Usage: post-pr-comment.sh +# +# Posts a formatted comment to the java-profiler PR with: +# - Pass/fail summary with badges +# - Test matrix results +# - Link to full dashboard +# - Failure details if any +# +# Requires: +# - DDPROF_COMMIT_BRANCH: Branch name to find PR +# - CI_PIPELINE_URL: Link to pipeline +# - pr-commenter tool (available in CI images) + +set -euo pipefail + +# Colors for logging +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +NC='\033[0m' + +log_info() { echo -e "${GREEN}[INFO]${NC} $*" >&2; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $*" >&2; } +log_error() { echo -e "${RED}[ERROR]${NC} $*" >&2; } + +RESULTS_DIR="${1:-integration-test-results}" +REPO="DataDog/java-profiler" + +# Dashboard URL (GitHub Pages) +DASHBOARD_URL="https://datadog.github.io/java-profiler/integration/" + +# Check required tools - try to get pr-commenter from benchmarking-platform if not available +PR_COMMENTER_AVAILABLE=false +if command -v pr-commenter >/dev/null 2>&1; then + PR_COMMENTER_AVAILABLE=true +elif [ -n "${CI_JOB_TOKEN:-}" ]; then + # In CI, clone benchmarking-platform to get pr-commenter + log_info "pr-commenter not found, cloning benchmarking-platform..." + PLATFORM_DIR=$(mktemp -d) + trap "rm -rf ${PLATFORM_DIR}" EXIT + git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/" + if git clone --depth 1 --branch dd-trace-go https://github.com/DataDog/benchmarking-platform "${PLATFORM_DIR}" 2>/dev/null; then + if [ -x "${PLATFORM_DIR}/tools/pr-commenter" ]; then + export PATH="${PLATFORM_DIR}/tools:${PATH}" + PR_COMMENTER_AVAILABLE=true + log_info "pr-commenter available from benchmarking-platform" + elif [ -f "${PLATFORM_DIR}/tools/pr-commenter.py" ]; then + # Try Python version + alias pr-commenter="python3 ${PLATFORM_DIR}/tools/pr-commenter.py" + PR_COMMENTER_AVAILABLE=true + log_info "pr-commenter.py available from benchmarking-platform" + else + log_warn "pr-commenter not found in benchmarking-platform" + ls -la "${PLATFORM_DIR}/tools/" 2>/dev/null || log_warn "No tools directory" + fi + else + log_warn "Failed to clone benchmarking-platform" + fi +else + log_warn "pr-commenter not found and not in CI - will print comment instead" +fi + +# Check required environment +if [ -z "${DDPROF_COMMIT_BRANCH:-}" ]; then + log_warn "DDPROF_COMMIT_BRANCH not set - skipping comment" + exit 0 +fi + +# Skip for main/master branches (no PR) +if [ "${DDPROF_COMMIT_BRANCH}" = "main" ] || [ "${DDPROF_COMMIT_BRANCH}" = "master" ]; then + log_info "Skipping PR comment for ${DDPROF_COMMIT_BRANCH} branch" + exit 0 +fi + +log_info "Posting comment for branch: ${DDPROF_COMMIT_BRANCH}" + +# Collect test results +log_info "Collecting test results from ${RESULTS_DIR}..." + +declare -A RESULTS +TOTAL_PASS=0 +TOTAL_FAIL=0 +FAILURES="" + +for config_dir in "${RESULTS_DIR}"/*; do + [ -d "${config_dir}" ] || continue + config_name=$(basename "${config_dir}") + + # Check validation logs for pass/fail + s1_status="unknown" + s2_status="unknown" + + # Scenario 1: profiler-only + s1_log="${config_dir}/profiler-only-${config_name}.log" + if [ -f "${s1_log}" ]; then + if grep -q "SUCCESS:" "${s1_log}" 2>/dev/null; then + s1_status="pass" + elif grep -q "VALIDATION_FAILED" "${s1_log}" 2>/dev/null; then + s1_status="fail" + fi + fi + + # Scenario 2: tracer+profiler + s2_log="${config_dir}/tracer-profiler-${config_name}.log" + if [ -f "${s2_log}" ]; then + if grep -q "SUCCESS:" "${s2_log}" 2>/dev/null; then + s2_status="pass" + elif grep -q "VALIDATION_FAILED" "${s2_log}" 2>/dev/null; then + s2_status="fail" + fi + fi + + # Determine overall status for this config + if [ "${s1_status}" = "pass" ] && [ "${s2_status}" = "pass" ]; then + RESULTS["${config_name}"]="pass" + TOTAL_PASS=$((TOTAL_PASS + 1)) + elif [ "${s1_status}" = "fail" ] || [ "${s2_status}" = "fail" ]; then + RESULTS["${config_name}"]="fail" + TOTAL_FAIL=$((TOTAL_FAIL + 1)) + # Collect failure details + FAILURES="${FAILURES}\n
${config_name}\n\n" + if [ "${s1_status}" = "fail" ] && [ -f "${s1_log}" ]; then + FAILURES="${FAILURES}**Profiler-only:**\n\`\`\`\n$(tail -20 "${s1_log}")\n\`\`\`\n" + fi + if [ "${s2_status}" = "fail" ] && [ -f "${s2_log}" ]; then + FAILURES="${FAILURES}**Tracer+profiler:**\n\`\`\`\n$(tail -20 "${s2_log}")\n\`\`\`\n" + fi + FAILURES="${FAILURES}
\n" + else + RESULTS["${config_name}"]="unknown" + fi +done + +TOTAL=$((TOTAL_PASS + TOTAL_FAIL)) + +# Determine overall status +if [ "${TOTAL_FAIL}" -gt 0 ]; then + OVERALL_STATUS="failure" + STATUS_EMOJI=":x:" + STATUS_TEXT="FAILED" +elif [ "${TOTAL_PASS}" -gt 0 ]; then + OVERALL_STATUS="success" + STATUS_EMOJI=":white_check_mark:" + STATUS_TEXT="PASSED" +else + OVERALL_STATUS="neutral" + STATUS_EMOJI=":grey_question:" + STATUS_TEXT="NO RESULTS" +fi + +log_info "Results: ${TOTAL_PASS} passed, ${TOTAL_FAIL} failed out of ${TOTAL} configurations" + +# Build the comment body +DDPROF_SHA="${DDPROF_COMMIT_SHA:-$(cat ddprof-commit-sha.txt 2>/dev/null || echo unknown)}" + +if [ "${OVERALL_STATUS}" = "success" ]; then + # All tests passed - keep it short + COMMENT_BODY=":white_check_mark: **All ${TOTAL} integration tests passed** + +:bar_chart: [Dashboard](${DASHBOARD_URL}) · :construction_worker: [Pipeline](${CI_PIPELINE_URL:-}) · :package: \`${DDPROF_SHA:0:8}\`" +else + # Some failures or unknowns - show full matrix + COMMENT_BODY="${STATUS_EMOJI} **${TOTAL_PASS}** passed, **${TOTAL_FAIL}** failed out of **${TOTAL}** configurations + +### Test Matrix + +| Platform | JDK 8 | JDK 11 | JDK 17 | JDK 21 | JDK 25 | +|----------|-------|--------|--------|--------|--------|" + + # Build matrix rows + for platform in "glibc-x64-hotspot" "glibc-x64-openj9" "glibc-arm64-hotspot" "glibc-arm64-openj9" \ + "musl-x64-hotspot" "musl-x64-openj9" "musl-arm64-hotspot" "musl-arm64-openj9"; do + row="| ${platform} |" + for jdk in 8 11 17 21 25; do + config="${platform}-jdk${jdk}" + status="${RESULTS[${config}]:-unknown}" + case "${status}" in + pass) row="${row} :white_check_mark: |" ;; + fail) row="${row} :x: |" ;; + *) row="${row} :grey_question: |" ;; + esac + done + COMMENT_BODY="${COMMENT_BODY} +${row}" + done + + # Add failure details if any + if [ -n "${FAILURES}" ]; then + COMMENT_BODY="${COMMENT_BODY} + +### Failure Details +$(echo -e "${FAILURES}")" + fi + + # Add links + COMMENT_BODY="${COMMENT_BODY} + +### Links +- :bar_chart: [Full Dashboard](${DASHBOARD_URL}) +- :construction_worker: [Pipeline](${CI_PIPELINE_URL:-}) +- :package: Commit: \`${DDPROF_SHA}\`" +fi + +# Post comment using pr-commenter +if [ "${PR_COMMENTER_AVAILABLE}" = "true" ]; then + log_info "Posting comment via pr-commenter..." + + if echo "${COMMENT_BODY}" | pr-commenter \ + --for-repo="${REPO}" \ + --for-pr="${DDPROF_COMMIT_BRANCH}" \ + --header="Integration Tests" \ + --on-duplicate=replace; then + log_info "Successfully posted comment" + else + log_error "Failed to post comment via pr-commenter" + log_info "Comment that would be posted:" + echo "${COMMENT_BODY}" + exit 1 + fi +else + log_info "Comment that would be posted to PR:" + echo "" + echo "${COMMENT_BODY}" + echo "" +fi + +# Exit with failure if tests failed (makes pipeline fail) +if [ "${OVERALL_STATUS}" = "failure" ]; then + log_error "Integration tests failed - marking pipeline as failed" + exit 1 +fi + +exit 0 diff --git a/.gitlab/dd-trace-integration/publish-gh-pages.sh b/.gitlab/dd-trace-integration/publish-gh-pages.sh new file mode 100755 index 000000000..78c86d04f --- /dev/null +++ b/.gitlab/dd-trace-integration/publish-gh-pages.sh @@ -0,0 +1,254 @@ +#!/bin/bash + +# publish-gh-pages.sh - Publish integration test reports to GitHub Pages +# +# Usage: publish-gh-pages.sh [results-dir] +# +# Generates reports for all test configurations and publishes to gh-pages branch. +# Reports are available at: https://datadog.github.io/async-profiler-build/ +# +# In CI: Uses Octo-STS for secure, short-lived GitHub tokens (no secrets needed) +# Locally: Use 'devflow gitlab auth' for Octo-STS, or set GITHUB_TOKEN env var + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="${SCRIPT_DIR}/../.." +RESULTS_BASE="${1:-${PROJECT_ROOT}/integration-test-results}" +MAX_HISTORY=10 + +# GitHub repo for Pages +GITHUB_REPO="DataDog/java-profiler" +PAGES_URL="https://datadog.github.io/java-profiler" + +# Colors +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +NC='\033[0m' + +log_info() { echo -e "${GREEN}[INFO]${NC} $*"; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } +log_error() { echo -e "${RED}[ERROR]${NC} $*"; } + +# Obtain GitHub token +obtain_github_token() { + # Try dd-octo-sts CLI (works in CI with DDOCTOSTS_ID_TOKEN) + if command -v dd-octo-sts >/dev/null 2>&1 && [ -n "${DDOCTOSTS_ID_TOKEN:-}" ]; then + log_info "Obtaining GitHub token via dd-octo-sts CLI..." + # Policy name matches the .sts.yaml filename (without extension) + + # Run dd-octo-sts and capture only stdout (don't capture stderr to avoid error messages in token) + local TOKEN_OUTPUT + local TOKEN_EXIT_CODE + TOKEN_OUTPUT=$(dd-octo-sts token --scope DataDog/java-profiler --policy async-profiler-build.ci 2>/tmp/dd-octo-sts-error.log) + TOKEN_EXIT_CODE=$? + + if [ $TOKEN_EXIT_CODE -eq 0 ] && [ -n "${TOKEN_OUTPUT}" ]; then + # Validate token format (GitHub tokens start with ghs_, ghp_, or look like JWT) + if [[ "${TOKEN_OUTPUT}" =~ ^(ghs_|ghp_|v1\.|[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.) ]]; then + GITHUB_TOKEN="${TOKEN_OUTPUT}" + log_info "GitHub token obtained via dd-octo-sts (expires in 1 hour)" + return 0 + else + log_warn "dd-octo-sts returned invalid token format (first 50 chars): ${TOKEN_OUTPUT:0:50}" + fi + else + log_warn "dd-octo-sts token exchange failed (exit code: ${TOKEN_EXIT_CODE})" + if [ -s /tmp/dd-octo-sts-error.log ]; then + log_warn "dd-octo-sts error output:" + cat /tmp/dd-octo-sts-error.log | head -10 >&2 + fi + fi + fi + + # Fall back to GITHUB_TOKEN environment variable + if [ -n "${GITHUB_TOKEN:-}" ]; then + log_info "Using GITHUB_TOKEN from environment" + return 0 + fi + + return 1 +} + +if ! obtain_github_token; then + log_error "Failed to obtain GitHub token" + log_error "Options:" + log_error " 1. Run in GitLab CI with dd-octo-sts-ci-base image and DDOCTOSTS_ID_TOKEN" + log_error " 2. Set GITHUB_TOKEN env var (PAT with 'repo' scope)" + exit 1 +fi + +# Create temporary directory for gh-pages content +WORK_DIR=$(mktemp -d) +trap "rm -rf ${WORK_DIR}" EXIT + +log_info "Preparing gh-pages content in: ${WORK_DIR}" + +# Clone gh-pages branch (or create if doesn't exist) +log_info "Cloning gh-pages branch..." +cd "${WORK_DIR}" + +if git clone --depth 1 --branch gh-pages "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPO}.git" pages 2>/dev/null; then + cd pages + log_info "Cloned existing gh-pages branch" +else + log_info "Creating new gh-pages branch..." + mkdir pages && cd pages + git init + git remote add origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPO}.git" + git checkout -b gh-pages +fi + +# Create/update Jekyll config with baseurl for subpath deployment +cat > "_config.yml" </dev/null | wc -l)" +else + log_warn "Results directory does not exist: ${RESULTS_BASE}" +fi + +# Generate run JSON for this pipeline +RUN_JSON_FILE=$(mktemp) +trap "rm -rf ${WORK_DIR} ${RUN_JSON_FILE}" EXIT + +log_info "Running generate-run-json.sh..." +if "${SCRIPT_DIR}/generate-run-json.sh" "${RESULTS_BASE}" > "${RUN_JSON_FILE}"; then + log_info "Generated run JSON successfully" + log_info "Run JSON size: $(wc -c < "${RUN_JSON_FILE}") bytes" + log_info "Run JSON preview:" + head -20 "${RUN_JSON_FILE}" || true + + # Update history (prepend new run, keep last MAX_HISTORY) + log_info "Running update-history.sh..." + if "${SCRIPT_DIR}/../common/update-history.sh" integration "${RUN_JSON_FILE}" "."; then + log_info "Updated integration history" + else + log_warn "Failed to update history, error output above" + fi +else + log_warn "Failed to generate run JSON, error output:" + log_warn "Showing last 100 lines of output:" + tail -100 "${RUN_JSON_FILE}" || true + + log_warn "" + log_warn "Checking first config directory for debugging:" + first_config=$(find "${RESULTS_BASE}" -maxdepth 1 -type d ! -path "${RESULTS_BASE}" | head -1) + if [ -n "$first_config" ]; then + log_warn "Contents of $(basename "$first_config"):" + ls -la "$first_config" || true + log_warn "" + log_warn "Sample log file content (if exists):" + find "$first_config" -name "*.log" -type f | head -1 | xargs head -20 2>/dev/null || log_warn "No log files found" + fi +fi + +# Generate dashboard and index pages +log_info "Generating dashboard..." +if "${SCRIPT_DIR}/../common/generate-dashboard.sh" "." 2>&1; then + log_info "Generated dashboard index.md" + if [ -f "index.md" ]; then + log_info "Dashboard size: $(wc -c < "index.md") bytes" + fi +else + log_warn "Failed to generate dashboard, error output above" +fi + +log_info "Generating integration index..." +if "${SCRIPT_DIR}/../common/generate-index.sh" integration "." 2>&1; then + log_info "Generated integration/index.md" + if [ -f "integration/index.md" ]; then + log_info "Integration index size: $(wc -c < "integration/index.md") bytes" + fi +else + log_warn "Failed to generate integration index, error output above" +fi + +# ============================================ +# DETAILED REPORTS (per-config reports for current run) +# ============================================ +log_info "Generating detailed reports..." + +TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M UTC") + +# Find all test result directories +RESULT_DIRS=() +while IFS= read -r -d '' dir; do + if [[ "${dir}" == *"/history"* ]]; then + continue + fi + if ls "${dir}"/*.log &>/dev/null 2>&1; then + RESULT_DIRS+=("${dir}") + fi +done < <(find "${RESULTS_BASE}" -mindepth 1 -maxdepth 1 -type d -print0 2>/dev/null | sort -z) + +if [ ${#RESULT_DIRS[@]} -eq 0 ]; then + log_warn "No test results found in ${RESULTS_BASE}" +fi + +log_info "Found ${#RESULT_DIRS[@]} test configuration(s)" + +# Create reports directory for detailed per-config reports +mkdir -p reports + +# Process each configuration +for dir in "${RESULT_DIRS[@]}"; do + config_name=$(basename "${dir}") + log_info "Processing: ${config_name}" + + # Generate detailed report + report_file="reports/${config_name}.md" + if "${SCRIPT_DIR}/generate-report.sh" "${dir}" "${WORK_DIR}/pages/${report_file}" 2>/dev/null; then + # Add front matter for Jekyll + tmp_file=$(mktemp) + cat > "${tmp_file}" <> "${tmp_file}" + mv "${tmp_file}" "${WORK_DIR}/pages/${report_file}" + else + log_warn "Failed to generate report for ${config_name}" + fi +done + +# Commit and push +log_info "Committing changes..." +git add -A +if git diff --staged --quiet; then + log_info "No changes to commit" +else + git config user.email "ci@datadoghq.com" + git config user.name "CI Bot" + git commit -m "Update integration test reports - ${TIMESTAMP}" + + log_info "Pushing to gh-pages..." + git push origin gh-pages --force + + log_info "✅ Reports published successfully!" + log_info "View at: ${PAGES_URL}" +fi + +echo "" +echo "PAGES_URL=${PAGES_URL}" diff --git a/.gitlab/dd-trace-integration/run-integration-test.sh b/.gitlab/dd-trace-integration/run-integration-test.sh new file mode 100755 index 000000000..555fd0dd6 --- /dev/null +++ b/.gitlab/dd-trace-integration/run-integration-test.sh @@ -0,0 +1,628 @@ +#!/bin/bash + +set -euo pipefail + +# run-integration-test.sh - Simplified integration tests for profiler +# +# This script runs self-contained integration tests without building dd-trace-java. +# It tests the patched dd-java-agent with both profiler-only and tracer+profiler scenarios. +# +# Expected environment variables: +# - LIBC_VARIANT: glibc or musl +# - ARCH: x64 or arm64 +# - JVM_TYPE: hotspot or openj9 +# - JAVA_VERSION: 8, 11, 17, 21, 25 +# +# Optional: +# - CI_PROJECT_DIR: Project root (auto-detected if not set) +# - JAVA_HOME: Java installation (must be set) +# - TEST_DURATION: Test duration in seconds (default: 30) + +# ======================================== +# Configuration +# ======================================== + +# Use CI_PROJECT_DIR if available, otherwise calculate from script location +if [ -n "${CI_PROJECT_DIR:-}" ]; then + PROJECT_ROOT="${CI_PROJECT_DIR}" +else + HERE=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) + PROJECT_ROOT="${HERE}/../.." +fi + +# Test configuration +TEST_DURATION="${TEST_DURATION:-60}" +TEST_THREADS=4 +CPU_ITERATIONS=10000 +ALLOC_RATE=1000 + +# Colors for output +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +NC='\033[0m' # No Color + +function log_info() { + echo -e "${GREEN}[INFO]${NC} $*" +} + +function log_warn() { + echo -e "${YELLOW}[WARN]${NC} $*" +} + +function log_error() { + echo -e "${RED}[ERROR]${NC} $*" +} + +# ======================================== +# Validate Environment +# ======================================== +echo "=== Integration Test Configuration ===" +echo "LIBC_VARIANT=${LIBC_VARIANT:-}" +echo "ARCH=${ARCH:-}" +echo "JVM_TYPE=${JVM_TYPE:-}" +echo "JAVA_VERSION=${JAVA_VERSION:-}" +echo "JAVA_HOME=${JAVA_HOME:-}" +echo "TEST_DURATION=${TEST_DURATION}" +echo "" + +# Validate required variables +if [ -z "${LIBC_VARIANT:-}" ] || [ -z "${ARCH:-}" ] || [ -z "${JVM_TYPE:-}" ] || [ -z "${JAVA_VERSION:-}" ]; then + log_error "Missing required environment variables" + log_error "Required: LIBC_VARIANT, ARCH, JVM_TYPE, JAVA_VERSION" + exit 1 +fi + +# Validate JAVA_HOME +if [ -z "${JAVA_HOME:-}" ]; then + log_error "JAVA_HOME not set" + exit 1 +fi + +if [ ! -x "${JAVA_HOME}/bin/java" ]; then + log_error "Java not found at: ${JAVA_HOME}/bin/java" + exit 1 +fi + +if [ ! -x "${JAVA_HOME}/bin/javac" ]; then + log_error "javac not found at: ${JAVA_HOME}/bin/javac" + exit 1 +fi + +# ======================================== +# System Diagnostics Functions +# ======================================== + +collect_system_metrics() { + local phase="$1" # start|mid|end + local output="${RESULTS_DIR}/diagnostics/system-metrics-${phase}.json" + + mkdir -p "${RESULTS_DIR}/diagnostics" + + # Collect metrics + local cpu_count=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo "1") + local cpu_quota=$(cat /sys/fs/cgroup/cpu/cpu.cfs_quota_us 2>/dev/null || echo "-1") + local cpu_period=$(cat /sys/fs/cgroup/cpu/cpu.cfs_period_us 2>/dev/null || echo "-1") + local load_avg=$(uptime | awk -F'load average:' '{print $2}' | xargs) + local container=$(test -f /.dockerenv && echo "true" || echo "false") + + # Parse throttling stats + local throttle_stats=$(cat /sys/fs/cgroup/cpu/cpu.stat 2>/dev/null || echo "") + local nr_periods=$(echo "$throttle_stats" | grep nr_periods | awk '{print $2}') + local nr_throttled=$(echo "$throttle_stats" | grep nr_throttled | awk '{print $2}') + local throttled_time=$(echo "$throttle_stats" | grep throttled_time | awk '{print $2}') + + # Calculate throttle percentage + local throttle_pct=0 + if [ -n "$nr_periods" ] && [ "$nr_periods" -gt 0 ] && [ "$cpu_period" -gt 0 ]; then + throttle_pct=$(awk "BEGIN {printf \"%.2f\", ($throttled_time / ($nr_periods * $cpu_period)) * 100}") + fi + + # Write JSON + cat > "$output" </dev/null || date +%Y-%m-%dT%H:%M:%S%z)", + "phase": "$phase", + "cpu_count": $cpu_count, + "cpu_quota": $cpu_quota, + "cpu_period": $cpu_period, + "load_average": "$load_avg", + "container": $container, + "throttling": { + "nr_periods": ${nr_periods:-0}, + "nr_throttled": ${nr_throttled:-0}, + "throttled_time_ns": ${throttled_time:-0}, + "percentage": $throttle_pct + } +} +EOF + + log_info "System metrics collected ($phase): CPU=$cpu_count, Throttled=${throttle_pct}%" +} + +start_cpu_monitor() { + # Background process sampling CPU count every 5s + local monitor_file="${RESULTS_DIR}/diagnostics/cpu-timeline.log" + mkdir -p "${RESULTS_DIR}/diagnostics" + + ( + while true; do + local cpu_now=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo "1") + echo "$(date +%s) $cpu_now" >> "$monitor_file" + sleep 5 + done + ) & + + echo $! > "${RESULTS_DIR}/diagnostics/monitor.pid" + log_info "CPU monitor started (PID: $!)" +} + +stop_cpu_monitor() { + local pid_file="${RESULTS_DIR}/diagnostics/monitor.pid" + if [ -f "$pid_file" ]; then + local monitor_pid=$(cat "$pid_file") + kill "$monitor_pid" 2>/dev/null || true + rm "$pid_file" + log_info "CPU monitor stopped" + fi +} + +generate_diagnostics_summary() { + local output="${RESULTS_DIR}/diagnostics/summary.txt" + + # Analyze CPU timeline for changes + local cpu_changes=$(awk '{print $2}' "${RESULTS_DIR}/diagnostics/cpu-timeline.log" | sort -u | wc -l) + local cpu_min=$(awk '{print $2}' "${RESULTS_DIR}/diagnostics/cpu-timeline.log" | sort -n | head -1) + local cpu_max=$(awk '{print $2}' "${RESULTS_DIR}/diagnostics/cpu-timeline.log" | sort -n | tail -1) + + # Extract throttling from end metrics + local throttle_pct=$(grep '"percentage"' "${RESULTS_DIR}/diagnostics/system-metrics-end.json" | awk -F': ' '{print $2}' | tr -d ',') + + cat > "$output" <&1 | head -3 + +# ======================================== +# Install Prerequisites +# ======================================== +echo "" +log_info "Installing prerequisites (jbang for JFR validation)..." + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [ -f "${SCRIPT_DIR}/install-prerequisites.sh" ]; then + source "${SCRIPT_DIR}/install-prerequisites.sh" +else + log_warn "install-prerequisites.sh not found, skipping" +fi + +# Ensure jbang is in PATH +export PATH="$HOME/.jbang/bin:$PATH" + +# Verify jbang is available +if ! command -v jbang &> /dev/null; then + log_error "jbang not found after installation" + log_error "JFR validation will not work" + exit 1 +fi + +log_info "jbang version: $(jbang version 2>&1 | head -1)" + +# ======================================== +# Artifact Collection on Exit +# ======================================== +function collect_artifacts() { + log_info "Collecting artifacts..." + + # Copy JFR recordings from dump directories + find /tmp -type d -name 'jfr-*' -exec sh -c 'cp "$0"/*.jfr "${1}/" 2>/dev/null || true' {} "${RESULTS_DIR}" \; 2>/dev/null || true + + # Copy agent logs + find /tmp -maxdepth 1 -name '*-agent.log' -exec cp {} "${RESULTS_DIR}/" \; 2>/dev/null || true + + # Copy HotSpot JVM crash dumps + find /tmp -maxdepth 1 -name 'hs_err*' -exec cp {} "${RESULTS_DIR}/" \; 2>/dev/null || true + find . -maxdepth 1 -name 'hs_err*' -exec cp {} "${RESULTS_DIR}/" \; 2>/dev/null || true + + # Copy OpenJ9 crash dumps + find /tmp -maxdepth 1 -name 'javacore*.txt' -exec cp {} "${RESULTS_DIR}/" \; 2>/dev/null || true + find /tmp -maxdepth 1 -name 'Snap*.trc' -exec cp {} "${RESULTS_DIR}/" \; 2>/dev/null || true + find /tmp -maxdepth 1 -name 'jitdump*' -exec cp {} "${RESULTS_DIR}/" \; 2>/dev/null || true + find /tmp -maxdepth 1 -name 'core.*' -exec cp {} "${RESULTS_DIR}/" \; 2>/dev/null || true + find . -maxdepth 1 -name 'javacore*.txt' -exec cp {} "${RESULTS_DIR}/" \; 2>/dev/null || true + find . -maxdepth 1 -name 'Snap*.trc' -exec cp {} "${RESULTS_DIR}/" \; 2>/dev/null || true + + # Copy validation reports + find /tmp/jfr-validation -name '*.log' -exec cp {} "${RESULTS_DIR}/" \; 2>/dev/null || true + + # Copy any test output logs + find /tmp -maxdepth 1 -name 'test-*.log' -exec cp {} "${RESULTS_DIR}/" \; 2>/dev/null || true + + log_info "Artifacts collected to: ${RESULTS_DIR}" +} + +trap collect_artifacts EXIT + +# ======================================== +# Calculate Threshold Multiplier +# ======================================== +# Base multiplier is 1.0, adjusted for platform and JVM type + +THRESHOLD_MULTIPLIER=1.0 + +# Platform adjustments +case "${ARCH}-${LIBC_VARIANT}" in + x64-glibc) + PLATFORM_MULT=1.0 + ;; + x64-musl) + PLATFORM_MULT=1.0 + ;; + arm64-glibc) + PLATFORM_MULT=0.8 + ;; + arm64-musl) + PLATFORM_MULT=0.8 + ;; + *) + log_warn "Unknown platform: ${ARCH}-${LIBC_VARIANT}, using default multiplier" + PLATFORM_MULT=1.0 + ;; +esac + +# JVM type adjustments +case "${JVM_TYPE}" in + hotspot) + JVM_MULT=1.0 + ;; + openj9) + JVM_MULT=0.5 # OpenJ9 typically produces fewer samples + ;; + *) + log_warn "Unknown JVM type: ${JVM_TYPE}, using default multiplier" + JVM_MULT=1.0 + ;; +esac + +# Extra adjustment for musl libc (Docker-on-runner with significant overhead) +LIBC_MULT=1.0 +if [ "${LIBC_VARIANT}" = "musl" ]; then + LIBC_MULT=0.25 +fi + +# Calculate final multiplier +THRESHOLD_MULTIPLIER=$(awk "BEGIN {print ${PLATFORM_MULT} * ${JVM_MULT} * ${LIBC_MULT}}") + +log_info "Threshold multiplier: ${THRESHOLD_MULTIPLIER} (platform=${PLATFORM_MULT}, jvm=${JVM_MULT}, libc=${LIBC_MULT})" + +# ======================================== +# Find Patched Agent +# ======================================== +log_info "Locating patched dd-java-agent..." + +PATCHED_AGENT="${PROJECT_ROOT}/dd-java-agent-patched.jar" + +if [ ! -f "${PATCHED_AGENT}" ]; then + log_error "Patched agent not found: ${PATCHED_AGENT}" + log_error "Expected artifact from prepare-patched-agent job" + exit 1 +fi + +log_info "Patched agent: ${PATCHED_AGENT}" +log_info "Agent size: $(du -h "${PATCHED_AGENT}" | cut -f1)" + +# Verify ddprof is actually in the patched agent +log_info "Verifying ddprof presence in patched agent..." +NATIVE_LIBS=$(unzip -l "${PATCHED_AGENT}" | grep -c "shared/META-INF/native-libs/.*\.so$" || echo "0") +CLASSDATA=$(unzip -l "${PATCHED_AGENT}" | grep -c "shared/.*\.classdata$" || echo "0") +log_info " - Native libraries: ${NATIVE_LIBS}" +log_info " - Classdata files: ${CLASSDATA}" + +if [ "${NATIVE_LIBS}" -eq 0 ]; then + log_error "No ddprof native libraries found in patched agent!" + log_error "Patching may have failed" + exit 1 +fi + +# ======================================== +# Compile Test Application +# ======================================== +log_info "Compiling test application..." + +TEST_APP_SRC="${PROJECT_ROOT}/.gitlab/test-apps/ProfilerTestApp.java" +TEST_APP_DIR="/tmp/test-app-$$" + +if [ ! -f "${TEST_APP_SRC}" ]; then + log_error "Test app not found: ${TEST_APP_SRC}" + exit 1 +fi + +mkdir -p "${TEST_APP_DIR}" +cp "${TEST_APP_SRC}" "${TEST_APP_DIR}/" + +cd "${TEST_APP_DIR}" + +if ! "${JAVA_HOME}/bin/javac" ProfilerTestApp.java 2>&1; then + log_error "Test app compilation failed" + exit 1 +fi + +log_info "✓ Test application compiled successfully" + +# ======================================== +# Run Scenario 1: Profiler-Only +# ======================================== +echo "" +echo "========================================" +echo " Scenario 1: Profiler-Only" +echo "========================================" +echo "" + +SCENARIO1_JFR_DIR="/tmp/jfr-profiler-only-${LIBC_VARIANT}-${ARCH}-${JVM_TYPE}-jdk${JAVA_VERSION}" +SCENARIO1_LOG="/tmp/profiler-only-agent.log" + +mkdir -p "${SCENARIO1_JFR_DIR}" + +# Collect system metrics at test start +collect_system_metrics "start" +start_cpu_monitor + +log_info "Running profiler-only test (${TEST_DURATION}s)..." +log_info "JFR dump directory: ${SCENARIO1_JFR_DIR}" +log_info "Log output: ${SCENARIO1_LOG}" + +# Log Java version details for debugging allocation profiling +log_info "Java version details:" +"${JAVA_HOME}/bin/java" -XshowSettings:properties -version 2>&1 | grep -E "java\.(version|vendor|runtime|vm)" || true +echo "" + +# Run test with profiler-only configuration +"${JAVA_HOME}/bin/java" \ + -javaagent:"${PATCHED_AGENT}" \ + -Ddd.profiling.enabled=true \ + -Ddd.profiling.ddprof.enabled=true \ + -Ddd.trace.enabled=false \ + -Ddd.profiling.start-delay=0 \ + -Ddd.profiling.start-force-first=true \ + -Ddd.profiling.upload.period=10 \ + -Ddd.profiling.debug.dump_path="${SCENARIO1_JFR_DIR}" \ + -Ddd.service.name=profiler-integration-test \ + -Ddd.trace.startup.logs=true \ + -Ddatadog.slf4j.simpleLogger.defaultLogLevel=debug \ + -cp . ProfilerTestApp \ + --duration "${TEST_DURATION}" \ + --threads "${TEST_THREADS}" \ + --cpu-iterations "${CPU_ITERATIONS}" \ + --alloc-rate "${ALLOC_RATE}" \ + > "${SCENARIO1_LOG}" 2>&1 + +# Show agent startup logs +log_info "Agent startup log (first 100 lines):" +head -100 "${SCENARIO1_LOG}" + +log_info "✓ Profiler-only test completed" + +# Collect mid-test system metrics +collect_system_metrics "mid" + +# Find the JFR recording (profiler writes continuously) +log_info "Locating JFR recording in ${SCENARIO1_JFR_DIR}..." + +SCENARIO1_JFR=$(find "${SCENARIO1_JFR_DIR}" -name "*.jfr" -print -quit) + +if [ -z "${SCENARIO1_JFR}" ] || [ ! -f "${SCENARIO1_JFR}" ]; then + log_error "JFR recording not found in: ${SCENARIO1_JFR_DIR}" + ls -la "${SCENARIO1_JFR_DIR}" || true + exit 1 +fi + +log_info "Found JFR recording: ${SCENARIO1_JFR}" + +JFR_SIZE=$(stat -f%z "${SCENARIO1_JFR}" 2>/dev/null || stat -c%s "${SCENARIO1_JFR}" 2>/dev/null) +log_info "JFR recording size: ${JFR_SIZE} bytes" + +if [ "${JFR_SIZE}" -lt 1024 ]; then + log_error "JFR recording is too small (< 1KB)" + exit 1 +fi + +# Run JFR validation with conformance checking +VALIDATION_LOG="/tmp/jfr-validation/profiler-only-${LIBC_VARIANT}-${ARCH}-${JVM_TYPE}-jdk${JAVA_VERSION}.log" +mkdir -p /tmp/jfr-validation + +log_info "Running conformance-based JFR validation..." + +# Use conformance wrapper with explicit configuration flags +# Since we're using the patched agent with ddprof, ddprof is enabled by default +set +e +"${PROJECT_ROOT}/test-validation/validate-jfr-conformance.sh" \ + "${SCENARIO1_JFR}" \ + --ddprof-enabled=true \ + --tracer-enabled=false \ + --test-duration="${TEST_DURATION}" \ + --arch="${ARCH}" \ + --libc="${LIBC_VARIANT}" \ + --jvm-type="${JVM_TYPE}" \ + --output="${VALIDATION_LOG}" + +VALIDATION_EXIT=$? +set -e + +# Show validation output and check exit code +if [ ${VALIDATION_EXIT} -ne 0 ]; then + echo "" + log_error "==========================================" + log_error " PROFILER-ONLY VALIDATION FAILED" + log_error " Exit code: ${VALIDATION_EXIT}" + log_error "==========================================" + log_error "" + + if [ -f "${VALIDATION_LOG}" ]; then + log_error "Validation output:" + cat "${VALIDATION_LOG}" + else + log_error "Validation log file not found: ${VALIDATION_LOG}" + fi + + exit 1 +fi + +log_info "✓ Profiler-only validation PASSED" + +# Copy validation log to results directory for artifact collection +CONFIG_NAME="${LIBC_VARIANT}-${ARCH}-${JVM_TYPE}-jdk${JAVA_VERSION}" +if [ -f "${VALIDATION_LOG}" ]; then + cp "${VALIDATION_LOG}" "${RESULTS_DIR}/profiler-only-${CONFIG_NAME}.log" + log_info "Copied validation log to ${RESULTS_DIR}/profiler-only-${CONFIG_NAME}.log" +fi + +# ======================================== +# Run Scenario 2: Tracer+Profiler +# ======================================== +echo "" +echo "========================================" +echo " Scenario 2: Tracer+Profiler" +echo "========================================" +echo "" + +SCENARIO2_JFR_DIR="/tmp/jfr-tracer-profiler-${LIBC_VARIANT}-${ARCH}-${JVM_TYPE}-jdk${JAVA_VERSION}" +SCENARIO2_LOG="/tmp/tracer-profiler-agent.log" + +mkdir -p "${SCENARIO2_JFR_DIR}" + +log_info "Running tracer+profiler test (${TEST_DURATION}s)..." +log_info "JFR dump directory: ${SCENARIO2_JFR_DIR}" +log_info "Log output: ${SCENARIO2_LOG}" + +# Run test with tracer+profiler configuration +"${JAVA_HOME}/bin/java" \ + -javaagent:"${PATCHED_AGENT}" \ + -Ddd.profiling.enabled=true \ + -Ddd.profiling.ddprof.enabled=true \ + -Ddd.trace.enabled=true \ + -Ddd.profiling.start-delay=0 \ + -Ddd.profiling.start-force-first=true \ + -Ddd.profiling.upload.period=10 \ + -Ddd.profiling.debug.dump_path="${SCENARIO2_JFR_DIR}" \ + -Ddd.service.name=profiler-integration-test \ + -Ddd.trace.startup.logs=true \ + -Ddatadog.slf4j.simpleLogger.defaultLogLevel=debug \ + -cp . ProfilerTestApp \ + --duration "${TEST_DURATION}" \ + --threads "${TEST_THREADS}" \ + --cpu-iterations "${CPU_ITERATIONS}" \ + --alloc-rate "${ALLOC_RATE}" \ + > "${SCENARIO2_LOG}" 2>&1 + +log_info "✓ Tracer+profiler test completed" + +# Find the JFR recording (profiler writes continuously) +log_info "Locating JFR recording in ${SCENARIO2_JFR_DIR}..." + +SCENARIO2_JFR=$(find "${SCENARIO2_JFR_DIR}" -name "*.jfr" -print -quit) + +if [ -z "${SCENARIO2_JFR}" ] || [ ! -f "${SCENARIO2_JFR}" ]; then + log_error "JFR recording not found in: ${SCENARIO2_JFR_DIR}" + ls -la "${SCENARIO2_JFR_DIR}" || true + exit 1 +fi + +log_info "Found JFR recording: ${SCENARIO2_JFR}" + +JFR_SIZE=$(stat -f%z "${SCENARIO2_JFR}" 2>/dev/null || stat -c%s "${SCENARIO2_JFR}" 2>/dev/null) +log_info "JFR recording size: ${JFR_SIZE} bytes" + +if [ "${JFR_SIZE}" -lt 1024 ]; then + log_error "JFR recording is too small (< 1KB)" + exit 1 +fi + +# Run JFR validation with conformance checking +VALIDATION_LOG="/tmp/jfr-validation/tracer-profiler-${LIBC_VARIANT}-${ARCH}-${JVM_TYPE}-jdk${JAVA_VERSION}.log" +mkdir -p /tmp/jfr-validation + +log_info "Running conformance-based JFR validation..." + +# Use conformance wrapper with tracer enabled +set +e +"${PROJECT_ROOT}/test-validation/validate-jfr-conformance.sh" \ + "${SCENARIO2_JFR}" \ + --ddprof-enabled=true \ + --tracer-enabled=true \ + --test-duration="${TEST_DURATION}" \ + --arch="${ARCH}" \ + --libc="${LIBC_VARIANT}" \ + --jvm-type="${JVM_TYPE}" \ + --output="${VALIDATION_LOG}" + +VALIDATION_EXIT=$? +set -e + +# Show validation output and check exit code +if [ ${VALIDATION_EXIT} -ne 0 ]; then + echo "" + log_error "==========================================" + log_error " TRACER+PROFILER VALIDATION FAILED" + log_error " Exit code: ${VALIDATION_EXIT}" + log_error "==========================================" + log_error "" + + if [ -f "${VALIDATION_LOG}" ]; then + log_error "Validation output:" + cat "${VALIDATION_LOG}" + else + log_error "Validation log file not found: ${VALIDATION_LOG}" + fi + + exit 1 +fi + +log_info "✓ Tracer+profiler validation PASSED" + +# Copy validation log to results directory for artifact collection +if [ -f "${VALIDATION_LOG}" ]; then + cp "${VALIDATION_LOG}" "${RESULTS_DIR}/tracer-profiler-${CONFIG_NAME}.log" + log_info "Copied validation log to ${RESULTS_DIR}/tracer-profiler-${CONFIG_NAME}.log" +fi + +# Collect final system metrics and generate summary +collect_system_metrics "end" +stop_cpu_monitor +generate_diagnostics_summary + +# ======================================== +# Cleanup +# ======================================== +cd "${PROJECT_ROOT}" +rm -rf "${TEST_APP_DIR}" + +# ======================================== +# Summary +# ======================================== +echo "" +echo "========================================" +echo " Integration Tests Summary" +echo "========================================" +echo "" +log_info "Platform: ${LIBC_VARIANT}-${ARCH}" +log_info "JVM: ${JVM_TYPE} JDK${JAVA_VERSION}" +log_info "✓ Scenario 1: Profiler-Only - PASSED" +log_info "✓ Scenario 2: Tracer+Profiler - PASSED" +log_info "" +log_info "All integration tests PASSED" +log_info "Results saved to: ${RESULTS_DIR}" +echo "" diff --git a/.gitlab/dd-trace-integration/verify-patch-compatibility.sh b/.gitlab/dd-trace-integration/verify-patch-compatibility.sh new file mode 100755 index 000000000..cbae6b7d4 --- /dev/null +++ b/.gitlab/dd-trace-integration/verify-patch-compatibility.sh @@ -0,0 +1,397 @@ +#!/bin/bash + +set -euo pipefail + +# Verify compatibility between ddprof classes and patched dd-java-agent + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +# Input JARs +DDPROF_JAR="${DDPROF_JAR:-${PROJECT_ROOT}/ddprof.jar}" +DD_AGENT_JAR="${DD_AGENT_JAR:-${PROJECT_ROOT}/dd-java-agent-original.jar}" +PATCHED_JAR="${PATCHED_JAR:-${PROJECT_ROOT}/dd-java-agent-patched.jar}" + +# Working directory +WORK_DIR="${WORK_DIR:-/tmp/jar-verify-$$}" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +function log_info() { + echo -e "${GREEN}[INFO]${NC} $*" +} + +function log_warn() { + echo -e "${YELLOW}[WARN]${NC} $*" +} + +function log_error() { + echo -e "${RED}[ERROR]${NC} $*" +} + +function log_debug() { + if [ "${DEBUG:-false}" = "true" ]; then + echo -e "${BLUE}[DEBUG]${NC} $*" + fi +} + +function cleanup() { + if [ -d "${WORK_DIR}" ]; then + log_debug "Cleaning up work directory: ${WORK_DIR}" + rm -rf "${WORK_DIR}" + fi +} + +trap cleanup EXIT + +function usage() { + cat << EOF +Usage: $0 [OPTIONS] + +Verify compatibility between ddprof and patched dd-java-agent. + +This script checks for breaking changes in public API by comparing +class signatures using javap. + +OPTIONS: + --ddprof-jar Path to ddprof.jar (default: ${DDPROF_JAR}) + --dd-agent-jar Path to original dd-java-agent.jar (default: ${DD_AGENT_JAR}) + --patched-jar Path to patched dd-java-agent.jar (default: ${PATCHED_JAR}) + --work-dir Working directory (default: /tmp/jar-verify-\$\$) + --debug Enable debug output + --help Show this help message + +ENVIRONMENT VARIABLES: + DDPROF_JAR Path to ddprof.jar + DD_AGENT_JAR Path to original dd-java-agent.jar + PATCHED_JAR Path to patched dd-java-agent.jar + WORK_DIR Working directory + DEBUG Enable debug output (true/false) + +EXIT CODES: + 0 - Compatible, safe to proceed + 1 - Incompatible, breaking changes detected or validation error + +EXAMPLES: + # Verify with default paths + $0 + + # Verify with custom paths + $0 --ddprof-jar /path/to/ddprof.jar --patched-jar /path/to/patched.jar + + # Enable debug output + DEBUG=true $0 +EOF +} + +# Parse command line arguments +while [ $# -gt 0 ]; do + case "$1" in + --ddprof-jar) + DDPROF_JAR="$2" + shift 2 + ;; + --dd-agent-jar) + DD_AGENT_JAR="$2" + shift 2 + ;; + --patched-jar) + PATCHED_JAR="$2" + shift 2 + ;; + --work-dir) + WORK_DIR="$2" + shift 2 + ;; + --debug) + DEBUG=true + shift + ;; + --help) + usage + exit 0 + ;; + *) + log_error "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +# Validate input JARs exist +if [ ! -f "${DDPROF_JAR}" ]; then + log_error "ddprof JAR not found: ${DDPROF_JAR}" + exit 1 +fi + +if [ ! -f "${DD_AGENT_JAR}" ]; then + log_error "dd-java-agent JAR not found: ${DD_AGENT_JAR}" + exit 1 +fi + +if [ ! -f "${PATCHED_JAR}" ]; then + log_error "patched JAR not found: ${PATCHED_JAR}" + exit 1 +fi + +# Check for javap +if ! command -v javap &> /dev/null; then + log_error "javap not found in PATH" + log_error "javap is part of the JDK. Please ensure JAVA_HOME is set and JDK bin is in PATH" + exit 1 +fi + +log_info "Starting compatibility verification" +log_info " ddprof: ${DDPROF_JAR}" +log_info " dd-agent-orig: ${DD_AGENT_JAR}" +log_info " patched: ${PATCHED_JAR}" + +# Create working directories +mkdir -p "${WORK_DIR}/original" "${WORK_DIR}/ddprof" + +# Extract JARs once +log_info "Extracting dd-java-agent-original..." +if ! unzip -q "${DD_AGENT_JAR}" -d "${WORK_DIR}/original/"; then + log_error "Failed to extract dd-java-agent" + exit 1 +fi + +log_info "Extracting ddprof..." +if ! unzip -q "${DDPROF_JAR}" -d "${WORK_DIR}/ddprof/"; then + log_error "Failed to extract ddprof" + exit 1 +fi + +# Create symlinks for .classdata files so javap can read them +# NOTE: These symlinks are temporary and only for validation - they exist +# in WORK_DIR and are cleaned up by trap. They do NOT affect the patched JAR. +# Only symlink profiler classes (com/datadoghq/profiler) since those are the only ones ddprof provides +log_info "Creating .class symlinks for profiler .classdata files..." + +cd "${WORK_DIR}/original" +if [ -d "shared/com/datadoghq/profiler" ]; then + CLASSDATA_COUNT=$(find shared/com/datadoghq/profiler -name "*.classdata" -type f | wc -l | tr -d ' ') + log_debug "Found ${CLASSDATA_COUNT} profiler .classdata files" + + find shared/com/datadoghq/profiler -name "*.classdata" -type f | while IFS= read -r classdata; do + # URL-decode: %24 -> $ + classfile="${classdata%.classdata}.class" + classfile="${classfile//%24/$}" + + # Create directory if needed (for inner classes) + mkdir -p "$(dirname "${classfile}")" + + log_debug " Symlinking: ${classfile} -> $(basename "${classdata}")" + ln -sf "$(basename "${classdata}")" "${classfile}" + done +else + log_warn "No shared/com/datadoghq/profiler directory found in original" +fi +cd - > /dev/null + +log_info "✓ Created symlinks for javap compatibility" + +# Find all classes in ddprof (excluding META-INF) +log_info "Finding classes to verify..." +CLASSES=$(cd "${WORK_DIR}/ddprof" && find . -name "*.class" -type f | grep -v "^\\./META-INF/" | sed 's|^\./||' || true) +CLASS_COUNT=$(echo "${CLASSES}" | grep -c . || echo 0) + +if [ "${CLASS_COUNT}" -eq 0 ]; then + log_warn "No classes found in ddprof JAR (this is unusual but not necessarily an error)" + log_info "Verification passed (no classes to verify)" + exit 0 +fi + +log_info "Found ${CLASS_COUNT} classes to verify" + +# Extract and compare signatures for each class +CHECKED_COUNT=0 +SKIPPED_COUNT=0 +BREAKING_CHANGES=0 + +function extract_signature() { + local classfile="$1" + local output="$2" + + log_debug " extract_signature: classfile=${classfile}" + + if [ ! -f "${classfile}" ] && [ ! -L "${classfile}" ]; then + log_debug " extract_signature: File does not exist" + return 1 + fi + + # Run javap with -protected flag to get clean method/field signatures + # Skip first line (Compiled from) and class declaration line + if javap -protected "${classfile}" 2>/dev/null | \ + tail -n +2 | \ + grep -E "^\s+(public|protected)" | \ + sed 's/^[[:space:]]*//' | \ + sort > "${output}"; then + log_debug " extract_signature: SUCCESS - extracted signatures" + return 0 + else + log_debug " extract_signature: FAILED - no public/protected members found" + return 1 + fi +} + +function normalize_signature() { + # Normalize method signatures for comparison + # Remove whitespace variations, package names from return types, etc. + sed 's/\s\+/ /g' | \ + sed 's/^ //g' | \ + sed 's/ $//g' | \ + sort -u +} + +log_info "Comparing public/protected API signatures..." + +while IFS= read -r classfile; do + if [ -z "${classfile}" ]; then + continue + fi + + # Convert path to fully qualified class name + classname=$(echo "${classfile}" | sed 's#/#.#g' | sed 's#\.class$##') + + log_debug "Checking class: ${classname}" + + # Check if class exists in original (as .class symlink we created) + ORIGINAL_FILE="${WORK_DIR}/original/shared/${classfile}" + + if [ ! -f "${ORIGINAL_FILE}" ] && [ ! -L "${ORIGINAL_FILE}" ]; then + log_debug "Class ${classname} is new in ddprof (not in original dd-agent)" + SKIPPED_COUNT=$((SKIPPED_COUNT + 1)) + continue + fi + + log_debug " Found in original: shared/${classfile}" + + # Extract signatures from both versions + ORIGINAL_SIG="${WORK_DIR}/original-${classname}.txt" + DDPROF_SIG="${WORK_DIR}/ddprof-${classname}.txt" + + DDPROF_FILE="${WORK_DIR}/ddprof/${classfile}" + + if ! extract_signature "${ORIGINAL_FILE}" "${ORIGINAL_SIG}"; then + log_debug "Could not extract ${classname} from dd-agent (might be non-public or package-private)" + SKIPPED_COUNT=$((SKIPPED_COUNT + 1)) + continue + fi + + if ! extract_signature "${DDPROF_FILE}" "${DDPROF_SIG}"; then + log_warn "⚠ Could not extract ${classname} from ddprof" + SKIPPED_COUNT=$((SKIPPED_COUNT + 1)) + continue + fi + + # Normalize both signatures for comparison + cat "${ORIGINAL_SIG}" | normalize_signature > "${ORIGINAL_SIG}.norm" + cat "${DDPROF_SIG}" | normalize_signature > "${DDPROF_SIG}.norm" + + # Check if all public/protected methods from original are present in ddprof + MISSING_METHODS="${WORK_DIR}/missing-${classname}.txt" + comm -23 "${ORIGINAL_SIG}.norm" "${DDPROF_SIG}.norm" > "${MISSING_METHODS}" + + if [ -s "${MISSING_METHODS}" ]; then + log_warn "⚠ API changes detected in ${classname}:" + log_warn " Methods in dd-java-agent-original but missing in ddprof:" + while IFS= read -r method; do + log_warn " ${method}" + done < "${MISSING_METHODS}" + + # Also check for methods in ddprof that are not in original (additions) + ADDED_METHODS="${WORK_DIR}/added-${classname}.txt" + comm -13 "${ORIGINAL_SIG}.norm" "${DDPROF_SIG}.norm" > "${ADDED_METHODS}" + if [ -s "${ADDED_METHODS}" ]; then + log_info " Methods added in ddprof (not in original):" + while IFS= read -r method; do + log_info " ${method}" + done < "${ADDED_METHODS}" + fi + + BREAKING_CHANGES=$((BREAKING_CHANGES + 1)) + else + log_debug "✓ ${classname} is API-compatible" + fi + + CHECKED_COUNT=$((CHECKED_COUNT + 1)) + +done <<< "${CLASSES}" + +log_info "API compatibility check complete:" +log_info " - ${CHECKED_COUNT} classes verified for API compatibility" +log_info " - ${SKIPPED_COUNT} classes skipped (new or non-public)" +log_info " - ${BREAKING_CHANGES} classes with breaking changes" + +# Verify native libraries are present +log_info "Verifying native libraries..." +NATIVE_IN_DDPROF=$(unzip -l "${DDPROF_JAR}" | grep -c "META-INF/native-libs/.*\.so$" || true) +if [ -z "${NATIVE_IN_DDPROF}" ] || ! [[ "${NATIVE_IN_DDPROF}" =~ ^[0-9]+$ ]]; then + NATIVE_IN_DDPROF=0 +fi + +NATIVE_IN_PATCHED=$(unzip -l "${PATCHED_JAR}" | grep -c "shared/META-INF/native-libs/.*\.so$" || true) +if [ -z "${NATIVE_IN_PATCHED}" ] || ! [[ "${NATIVE_IN_PATCHED}" =~ ^[0-9]+$ ]]; then + NATIVE_IN_PATCHED=0 +fi + +log_debug "Native libraries in ddprof: ${NATIVE_IN_DDPROF}" +log_debug "Native libraries in patched: ${NATIVE_IN_PATCHED}" + +# Note: ddprof.jar may only contain 1 lib (e.g., from debug build on one arch) +# The patched JAR gets all platform libs from libs/$TARGET/ directories, not from ddprof.jar +if [ "${NATIVE_IN_PATCHED}" -eq 0 ]; then + log_warn "No native libraries found in patched JAR" + log_warn "This may be expected if building without native libraries" +else + log_info "✓ Found ${NATIVE_IN_PATCHED} native libraries in patched JAR" +fi + +# Verify all expected platforms are present +log_info "Verifying platform coverage..." +EXPECTED_PLATFORMS=("linux-x64" "linux-x64-musl" "linux-arm64" "linux-arm64-musl") +MISSING_PLATFORMS=() + +for platform in "${EXPECTED_PLATFORMS[@]}"; do + if unzip -l "${PATCHED_JAR}" "shared/META-INF/native-libs/${platform}/libjavaProfiler.so" > /dev/null 2>&1; then + log_debug "✓ Found platform: ${platform}" + else + log_debug "✗ Missing platform: ${platform}" + MISSING_PLATFORMS+=("${platform}") + fi +done + +if [ ${#MISSING_PLATFORMS[@]} -gt 0 ]; then + log_warn "Some platforms are missing: ${MISSING_PLATFORMS[*]}" + log_warn "This may be expected if not all platforms were built" +else + log_info "✓ All expected platforms present" +fi + +# Final verdict +log_info "" +log_info "============================================" +log_info "VERIFICATION COMPLETED" +log_info "============================================" +log_info "" +log_info "Summary:" +log_info " - ${CHECKED_COUNT} classes verified for API compatibility" +log_info " - ${SKIPPED_COUNT} classes skipped (new or non-public)" +log_info " - ${BREAKING_CHANGES} classes with API changes (warnings)" +log_info " - ${NATIVE_IN_PATCHED} native libraries verified" +log_info " - ${#MISSING_PLATFORMS[@]} platforms missing (may be expected)" + +if [ "${BREAKING_CHANGES}" -gt 0 ]; then + log_warn "" + log_warn "API changes detected but not blocking (see warnings above)" + log_warn "Review warnings for methods removed from dd-java-agent" +fi + +exit 0 diff --git a/.gitlab/ghtools/Dockerfile b/.gitlab/ghtools/Dockerfile new file mode 100644 index 000000000..6170903fc --- /dev/null +++ b/.gitlab/ghtools/Dockerfile @@ -0,0 +1,5 @@ +FROM --platform=linux/amd64 registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:latest +ARG CI_JOB_TOKEN + +COPY ./setup.sh /tmp/setup.sh +RUN /tmp/setup.sh $CI_JOB_TOKEN \ No newline at end of file diff --git a/.gitlab/ghtools/setup.sh b/.gitlab/ghtools/setup.sh new file mode 100755 index 000000000..86b41a28d --- /dev/null +++ b/.gitlab/ghtools/setup.sh @@ -0,0 +1,27 @@ +#! /bin/bash + +set -eou pipefail + +CI_JOB_TOKEN=$1 + +if [ -z "$CI_JOB_TOKEN" ]; then + echo "Skip installation of Github tools." + exit 0 +fi + +#apt update && apt install -y hwinfo procps git curl software-properties-common build-essential libnss3-dev zlib1g-dev libgdbm-dev libncurses5-dev libssl-dev libffi-dev libreadline-dev libsqlite3-dev libbz2-dev openjdk-11-jdk +#git clone -q --depth 1 https://github.com/pyenv/pyenv.git --branch "v2.0.4" --single-branch /pyenv +#export PYENV_ROOT="/pyenv" +#export PATH="/pyenv/shims:/pyenv/bin:$PATH" +#eval "$(pyenv init -)" +#pyenv install 3.9.6 && pyenv global 3.9.6 +# +#pip3 install awscli virtualenv setuptools + +git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform-tools.git benchmarking-tools + +cd benchmarking-tools/github-tools +./install.sh + +echo "GITHUB_TOOLS_HOME=$(pwd)" >> ~/.bashrc +echo 'PATH=${GITHUB_TOOLS_HOME}:$PATH' >> ~/.bashrc \ No newline at end of file diff --git a/.gitlab/jdk-integration/.gitlab-ci.yml b/.gitlab/jdk-integration/.gitlab-ci.yml new file mode 100644 index 000000000..4fe94b90f --- /dev/null +++ b/.gitlab/jdk-integration/.gitlab-ci.yml @@ -0,0 +1,53 @@ +stages: [trigger, notify] + +benchmark-with-jdk: + stage: trigger + timeout: 6h + variables: + DOWNSTREAM: "${DOWNSTREAM}" + BENCHMARK_JDK: "${BENCHMARK_JDK}" + ITERATIONS: "${BENCHMARK_ITERATIONS:-1}" + MODES: "${BENCHMARK_MODES:-cpu,wall,alloc,memleak}" + KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: java-profiler + rules: + - if: '$BENCHMARK_JDK == null || $DOWNSTREAM == null' + when: never + - when: always + tags: [ "arch:amd64" ] + image: $BENCHMARK_IMAGE_AMD64 + +test-with-jdk: + stage: trigger + variables: + JDK_VERSION: "${JDK_VERSION}" + HASH: "${HASH}" + DEBUG_LEVEL: "${DEBUG_LEVEL}" + rules: + - if: '$JDK_VERSION == null || $DEBUG_LEVEL == null || $HASH == null || $DOWNSTREAM == null' + when: never + - when: always + image: "registry.ddbuild.io/ci/openjdk-build:${JDK_VERSION}-${DEBUG_LEVEL}-${HASH}" + tags: [ "arch:amd64" ] + script: + - apt update && DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends curl git moreutils awscli amazon-ecr-credential-helper gnupg2 build-essential g++ zip unzip cmake + - curl -s "https://get.sdkman.io" | bash + - source "$HOME/.sdkman/bin/sdkman-init.sh" + - sdk install java 11.0.18-tem + - export TEST_JAVA_HOME=/usr/lib/jvm + - export JAVA_HOME=/root/.sdkman/candidates/java/11.0.18-tem + - export JDK_TOOL_OPTIONS="-XX:ErrorFile=/tmp/hs_err_pid_%p.log" + - ./gradlew :ddprof-test:testDebug --max-workers=1 --no-build-cache --stacktrace --info --no-watch-fs --no-daemon + artifacts: + when: always + name: "test-with-jdk-${JDK_VERSION}-${DEBUG_LEVEL}-${HASH}.zip" + paths: + - ddprof-test/build/reports + - /tmp/hs_err*.log + +notify-slack: + stage: notify + when: on_failure + image: registry.ddbuild.io/slack-notifier:latest + tags: ["arch:amd64"] + script: + - .gitlab/jdk-integration/notify_channel.sh "${JDK_VERSION}" "${DEBUG_LEVEL}" "${HASH}" diff --git a/.gitlab/jdk-integration/notify_channel.sh b/.gitlab/jdk-integration/notify_channel.sh new file mode 100755 index 000000000..77dedb6a4 --- /dev/null +++ b/.gitlab/jdk-integration/notify_channel.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -euxo pipefail + +if [ ! -z "${CANCELLED:-}" ]; then + exit 0 +fi + +# Source centralized configuration +HERE=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${HERE}/../../.gitlab/config.env" + +PIPELINE_URL="$CI_PROJECT_URL/pipelines/$CI_PIPELINE_ID" +PIPELINE_LINK="<$PIPELINE_URL|pipeline #$CI_PIPELINE_ID>" + +# get the JDK build args +JDK_VERSION=$1 +DEBUG_LEVEL=$2 +HASH=$3 + +JDK_IMAGE="registry.ddbuild.io/ci/openjdk-build:${JDK_VERSION}-${DEBUG_LEVEL}-${HASH}" + +MESSAGE_TEXT=":nuke: JDK Integration tests failed (image=\"${JDK_IMAGE}\", pipeline=$PIPELINE_LINK)" + +postmessage "$SLACK_CHANNEL" "$MESSAGE_TEXT" "alert" diff --git a/.gitlab/reliability/.gitlab-ci.yml b/.gitlab/reliability/.gitlab-ci.yml new file mode 100644 index 000000000..27c59a3f1 --- /dev/null +++ b/.gitlab/reliability/.gitlab-ci.yml @@ -0,0 +1,99 @@ +variables: + PREPARE_IMAGE: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:latest + DD_OCTO_STS_IMAGE: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 + +# Shared template — concrete jobs supply image, tags, and ARCH variable +.reliability_job: + stage: reliability + timeout: 6h + variables: + RUNTIME: "${RUNTIME}" + needs: + - get-versions + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: always + - when: never + parallel: + matrix: + - CONFIG: ["profiler", "profiler+tracer"] + VARIANT: ["jit", "memory"] + ALLOCATOR: ["gmalloc", "jemalloc", "tcmalloc"] + script: + - set +e + - echo "runtime=${RUNTIME}, config=${CONFIG}, variant=${VARIANT}, allocator=${ALLOCATOR}, arch=${ARCH}" + - .gitlab/reliability/run.sh "$RUNTIME" "$CONFIG" "$VARIANT" "$ALLOCATOR" "$ARCH" 2>err.log 1>out.log + - REASON=$(cat err.log | grep "FAIL:" | cut -f2 -d':') || true + - if [ -n "${REASON}" ]; then echo "REASON_${CONFIG}_${ALLOCATOR}_${ARCH}X${VARIANT}=${REASON}" | tr '+' '_' >> build.env; exit 1; fi + after_script: + - | + if [[ "$CI_JOB_STATUS" == "failed" ]]; then + echo "REASON_${CONFIG}_${ALLOCATOR}_${ARCH}X${VARIANT}=Unknown failure, perhaps timeout" | tr '+' '_' >> build.env + fi + artifacts: + name: "results-${ARCH}" + when: always + paths: + - memwatch.log + - memwatch-trend.png + - hs_err.log + - err.log + - out.log + reports: + dotenv: build.env + expire_in: 1 day + +reliability-amd64: + extends: .reliability_job + tags: [ "arch:amd64" ] + image: $BENCHMARK_IMAGE_AMD64 + variables: + ARCH: amd64 + +reliability-aarch64: + extends: .reliability_job + tags: [ "arch:arm64" ] + image: $BENCHMARK_IMAGE_ARM64 + variables: + ARCH: aarch64 + KUBERNETES_MEMORY_REQUEST: 200Gi + KUBERNETES_MEMORY_LIMIT: 200Gi + +notify-slack: + stage: notify + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: on_failure + - when: never + image: registry.ddbuild.io/slack-notifier:latest + tags: [ "arch:amd64" ] + needs: + - get-versions + - reliability-amd64 + - reliability-aarch64 + script: + - .gitlab/reliability/notify_channel.sh "${CURRENT_VERSION}" + +publish-reliability-gh-pages: + stage: notify + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule" && ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "main")' + when: always + image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 + tags: [ "arch:arm64" ] + id_tokens: + DDOCTOSTS_ID_TOKEN: + aud: dd-octo-sts + needs: + - job: reliability-amd64 + artifacts: true + - job: reliability-aarch64 + artifacts: true + timeout: 10m + script: + - ./.gitlab/reliability/publish-gh-pages.sh + allow_failure: true + +include: + - local: .gitlab/common.yml + - local: .gitlab/benchmarks/images.yml diff --git a/.gitlab/reliability/generate-run-json.sh b/.gitlab/reliability/generate-run-json.sh new file mode 100755 index 000000000..befd29932 --- /dev/null +++ b/.gitlab/reliability/generate-run-json.sh @@ -0,0 +1,152 @@ +#!/bin/bash + +# generate-run-json.sh - Generate run JSON for reliability tests +# +# Usage: generate-run-json.sh [artifacts-dir] +# +# Parses build.env files for failure reasons and outputs a JSON object +# suitable for update-history.sh. Reads CI environment variables for metadata. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="${SCRIPT_DIR}/../.." +ARTIFACTS_DIR="${1:-${PROJECT_ROOT}}" + +# Read metadata from environment or defaults +TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +PIPELINE_ID="${CI_PIPELINE_ID:-0}" +PIPELINE_URL="${CI_PIPELINE_URL:-#}" +DDPROF_BRANCH="${DDPROF_COMMIT_BRANCH:-${CI_COMMIT_BRANCH:-main}}" +DDPROF_SHA="${DDPROF_COMMIT_SHA:-${CI_COMMIT_SHA:-unknown}}" + +# Read version from environment or version.txt +LIB_VERSION="${CURRENT_VERSION:-unknown}" +if [ "${LIB_VERSION}" = "unknown" ] && [ -f "${PROJECT_ROOT}/version.txt" ]; then + LIB_VERSION=$(awk -F: '{print $NF}' "${PROJECT_ROOT}/version.txt" | tr -d ' ') +fi + +# Lookup PR for branch +PR_JSON="{}" +if [ -x "${SCRIPT_DIR}/../common/lookup-pr.sh" ]; then + PR_JSON=$("${SCRIPT_DIR}/../common/lookup-pr.sh" "${DDPROF_BRANCH}" 2>/dev/null) || PR_JSON="{}" +fi + +# Expected configs: 2 configs x 2 variants x 3 allocators x 2 architectures = 24 +EXPECTED_CONFIGS=24 + +# Parse reliability results +python3 </dev/null 2>&1 && pwd )" + +RUNTIME=${1} +CONFIG=${2:-profiler} +ALLOCATOR=${3:-gmalloc} + +echo "JIT Stability Check with runtime: ${RUNTIME} seconds, config=${CONFIG}" + +curl -s "https://get.sdkman.io" | bash +source "/root/.sdkman/bin/sdkman-init.sh" 1>/dev/null 2>/dev/null + +sdk install java 21.0.3-tem 1>/dev/null 2>/dev/null + +mvn org.apache.maven.plugins:maven-dependency-plugin:2.1:get \ + -DrepoUrl=https://central.sonatype.com/repository/maven-snapshots/ \ + -Dartifact=com.datadoghq:ddprof:${CURRENT_VERSION} 1>/dev/null 2>/dev/null + +mkdir -p /var/lib/datadog/${CURRENT_VERSION} +rm -rf /var/lib/datadog/${CURRENT_VERSION}/* + +unzip -q -d /var/lib/datadog/${CURRENT_VERSION} /root/.m2/repository/com/datadoghq/ddprof/${CURRENT_VERSION}/ddprof-${CURRENT_VERSION}.jar +AGENT_LIB=$(find /var/lib/datadog/${CURRENT_VERSION} -name 'libjavaProfiler.so' | fgrep '/linux-x64/') + +echo "Agent lib: ${AGENT_LIB}" +uname -a +echo "Run duration: ${RUNTIME} seconds" + +wget -q -O /var/lib/datadog/dd-java-agent.jar 'https://dtdg.co/latest-java-tracer' + +case $CONFIG in + profiler) + echo "Running with profiler" + ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=false" + ;; + profiler+tracer) + echo "Running with profiler and tracer" + ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=true" + ;; + *) + echo "Unknown configuration: $CONFIG" + exit 1 + ;; +esac + +case $ALLOCATOR in + gmalloc) + echo "Running with gmalloc" + ;; + tcmalloc) + echo "Running with tcmalloc" + export LD_PRELOAD=$(find /usr/lib/ -name 'libtcmalloc_minimal.so.4') + ;; + jemalloc) + echo "Running with jemalloc" + export LD_PRELOAD=$(find /usr/lib/ -name 'libjemalloc.so') + ;; + *) + echo "Unknown allocator: $ALLOCATOR" + echo "Valid values are: gmalloc, tcmalloc, jemalloc" + exit 1 + ;; +esac + +echo "LD_PRELOAD=$LD_PRELOAD" + +start_time=$(date +%s) + +while true; do + # Get the current time + current_time=$(date +%s) + + # Calculate the elapsed time + elapsed_time=$((current_time - start_time)) + + # Break the loop if the elapsed time is greater than or equal to the duration + if ((elapsed_time >= ${RUNTIME})); then + break + fi + + java -javaagent:/var/lib/datadog/dd-java-agent.jar \ + $ENABLEMENT \ + -Ddd.profiling.upload.period=1 \ + -Ddd.profiling.start-force-first=true \ + -Ddd.profiling.ddprof.liveheap.enabled=true \ + -Ddd.profiling.ddprof.alloc.enabled=true \ + -Ddd.profiling.dprof.wall.enabled=true \ + -Ddd.integration.renaissance.enabled=true \ + -Ddd.env=java-profiler-stability \ + -Ddd.service=java-profiler-jit-stability \ + -Ddd.profiling.ddprof.debug.lib="${AGENT_LIB}" \ + -Xmx800m -Xms800m \ + -Dctl=$CONTROL_FILE \ + -XX:ErrorFile=${HERE}/../../hs_err.log \ + -XX:OnError="${HERE}/../../dd_crash_uploader.sh %p" \ + -jar /var/lib/benchmarks/renaissance.jar akka-uct -t 30 + + RC=$? + echo "RC=$RC" + + if [ $RC -ne 0 ]; then + echo "FAIL:Benchmark crashed" >&2 + exit 1 + fi +done diff --git a/.gitlab/reliability/memory_trend_check.py b/.gitlab/reliability/memory_trend_check.py new file mode 100644 index 000000000..d56281f75 --- /dev/null +++ b/.gitlab/reliability/memory_trend_check.py @@ -0,0 +1,70 @@ +import argparse +import numpy as np +import matplotlib.pyplot as plt + +def detect_memory_growth(memory_log, window_size=5, threshold=0.01): + growth_trend = [] + for i in range(len(memory_log) - window_size + 1): + window = memory_log[i:i + window_size] + growth = np.diff(window) + avg_growth = np.mean(growth) + growth_trend.append(avg_growth > threshold * window[0]) + + persistent_growth = sum(growth_trend) / len(growth_trend) > 0.5 # More than 50% of the windows show growth + return persistent_growth, growth_trend + +def read_memory_log(file_path): + with open(file_path, 'r') as file: + lines = file.readlines() + memory_log = [int(line.split(":")[1].strip()) for line in lines if line.startswith("mem:")] + return memory_log + +def plot_memory_log(memory_log, growth_trend, window_size, output=None, show=False): + print("Plotting memory log to " + output) + + plt.figure(figsize=(10, 5)) + plt.plot(range(0, len(memory_log)), memory_log, label='Memory Usage') + plt.title('Memory Usage Over Time') + plt.xlabel('Sample') + plt.ylabel('Memory (units)') + + # Highlight the windows where growth is detected + for i in range(len(growth_trend)): + if growth_trend[i]: + plt.axvspan(i, i + window_size, color='red', alpha=0.3) + + plt.legend() + if (output): + plt.savefig(output) + plt.close() + if (show): + plt.show() + +def main(): + # Set up the argument parser + parser = argparse.ArgumentParser(description="Calculate moving average and check if the trend is increasing.") + parser.add_argument("filename", type=str, help="The path to the data file.") + parser.add_argument("--plot", action="store_true", help="Plot the memory usage over time.") + parser.add_argument("--output", type=str, help="Output file for the plot.") + + print("Arguments: ", parser.parse_args()) + + # Parse the command line arguments + args = parser.parse_args() + + memory_log = read_memory_log(args.filename ) + + window_size = 5 + threshold = 0.01 + persistent_growth, growth_trend = detect_memory_growth(memory_log, window_size, threshold) + + if persistent_growth: + print("Persistent and significant memory growth detected.") + print("TREND:up") + else: + print("No persistent and significant memory growth detected.") + + plot_memory_log(memory_log, growth_trend, window_size, args.output, args.plot) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/.gitlab/reliability/memory_trend_check.sh b/.gitlab/reliability/memory_trend_check.sh new file mode 100755 index 000000000..38a64e430 --- /dev/null +++ b/.gitlab/reliability/memory_trend_check.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash + +set +e # Disable exit on error + +HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +RUNTIME=${1} +CONFIG=${2:-profiler} +ALLOCATOR=${3:-gmalloc} + +# Function to count CPUs from ranges and individual numbers +count_cpus() { + local cpus=$(cat /sys/fs/cgroup/cpuset/cpuset.cpus) + local cpu_count=0 + local IFS=',' # Use comma as delimiter to split ranges and numbers + + # Iterate over ranges and numbers + for range in $cpus; do + if [[ "$range" =~ - ]]; then + # It's a range, calculate the number of CPUs in the range + local start_cpu=$(echo $range | cut -d '-' -f 1) + local end_cpu=$(echo $range | cut -d '-' -f 2) + local num_cpus=$((end_cpu - start_cpu + 1)) + cpu_count=$((cpu_count + num_cpus)) + else + # It's a single CPU number + cpu_count=$((cpu_count + 1)) + fi + done + + echo $cpu_count +} + +curl -s "https://get.sdkman.io" | bash +source "/root/.sdkman/bin/sdkman-init.sh" 1>/dev/null 2>/dev/null + +sdk install java 21.0.3-tem 1>/dev/null 2>/dev/null + +mvn org.apache.maven.plugins:maven-dependency-plugin:2.1:get \ + -DrepoUrl=https://central.sonatype.com/repository/maven-snapshots/ \ + -Dartifact=com.datadoghq:ddprof:${CURRENT_VERSION} 1>/dev/null 2>/dev/null + +mkdir -p /var/lib/datadog/${CURRENT_VERSION} +rm -rf /var/lib/datadog/${CURRENT_VERSION}/* + +unzip -q -d /var/lib/datadog/${CURRENT_VERSION} /root/.m2/repository/com/datadoghq/ddprof/${CURRENT_VERSION}/ddprof-${CURRENT_VERSION}.jar +AGENT_LIB=$(find /var/lib/datadog/${CURRENT_VERSION} -name 'libjavaProfiler.so' | fgrep '/linux-x64/') + +echo "Agent lib: ${AGENT_LIB}" +uname -a +echo "CPU Cores: $(count_cpus)" +echo "Run duration: ${RUNTIME} seconds" + +wget -q -O /var/lib/datadog/dd-java-agent.jar 'https://dtdg.co/latest-java-tracer' + +CONTROL_FILE=".running" +touch $CONTROL_FILE +sh ./benchmarks/steps/mem_watch.sh $CONTROL_FILE ${HERE}/../../memwatch.log & + +case $CONFIG in + profiler) + echo "Running with profiler" + ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=false" + ;; + profiler+tracer) + echo "Running with profiler and tracer" + ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=true" + ;; + *) + echo "Unknown configuration: $CONFIG" + exit 1 + ;; +esac + +case $ALLOCATOR in + gmalloc) + echo "Running with gmalloc" + ;; + tcmalloc) + echo "Running with tcmalloc" + export LD_PRELOAD=$(find /usr/lib/ -name 'libtcmalloc_minimal.so.4') + ;; + jemalloc) + echo "Running with jemalloc" + export LD_PRELOAD=$(find /usr/lib/ -name 'libjemalloc.so') + ;; + *) + echo "Unknown allocator: $ALLOCATOR" + echo "Valid values are: gmalloc, tcmalloc, jemalloc" + exit 1 + ;; +esac + +echo "LD_PRELOAD=$LD_PRELOAD" + +java -javaagent:/var/lib/datadog/dd-java-agent.jar \ + $ENABLEMENT \ + -Ddd.profiling.upload.period=5 \ + -Ddd.profiling.start-force-first=true \ + -Ddd.profiling.ddprof.liveheap.enabled=true \ + -Ddd.profiling.ddprof.alloc.enabled=true \ + -Ddd.profiling.dprof.wall.enabled=true \ + -Ddd.integration.renaissance.enabled=true \ + -Ddd.env=java-profiler-stability \ + -Ddd.service=java-profiler-memory-trend \ + -Ddd.profiling.ddprof.debug.lib="${AGENT_LIB}" \ + -Dddprof.debug.malloc_arena_max=2 \ + -Xmx800m -Xms800m \ + -Dctl=$CONTROL_FILE \ + -XX:ErrorFile=${HERE}/../../hs_err.log \ + -jar /var/lib/benchmarks/renaissance.jar akka-uct -t ${RUNTIME} + +RC=$? +echo "RC=$RC" +rm -f $CONTROL_FILE > /dev/null + +if [ $RC -ne 0 ]; then + echo "FAIL:Benchmark crashed" >&2 +fi + +LOG=$(python3 ${HERE}/memory_trend_check.py --output ${HERE}/../../memwatch-trend.png ${HERE}/../../memwatch.log) +echo "$LOG" + +TREND=$(echo "$LOG" | grep "TREND:" | cut -f2 -d':') + +if [ "$TREND" == "up" ]; then + echo "FAIL:Memory usage is trending up" >&2 +fi \ No newline at end of file diff --git a/.gitlab/reliability/notify_channel.sh b/.gitlab/reliability/notify_channel.sh new file mode 100755 index 000000000..80675d5d9 --- /dev/null +++ b/.gitlab/reliability/notify_channel.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -euxo pipefail + +if [ ! -z "${CANCELLED:-}" ]; then + exit 0 +fi + +# Source centralized configuration +HERE=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${HERE}/../../.gitlab/config.env" + +PIPELINE_URL="$CI_PROJECT_URL/pipelines/$CI_PIPELINE_ID" +PIPELINE_LINK="<$PIPELINE_URL|pipeline #$CI_PIPELINE_ID>" + +VERSION=$1 + +FOUND= +env | while IFS='=' read -r name value; do + # Check if the variable name starts with 'REASON_' + if [[ $name == REASON_* ]]; then + STRIPPED="${name#REASON_}" + CONFIG="${STRIPPED%%X*}" + VARIANT="${STRIPPED#*X}" + REASON=$value + MESSAGE_TEXT=":bomb: Reliability test failed for ${VERSION} (pipeline=$PIPELINE_LINK, reason=\"$REASON\", config=\"$CONFIG\", variant=\"$VARIANT\")" + postmessage "$SLACK_CHANNEL" "$MESSAGE_TEXT" "alert" + found="true" + fi +done diff --git a/.gitlab/reliability/publish-gh-pages.sh b/.gitlab/reliability/publish-gh-pages.sh new file mode 100755 index 000000000..9f79e5d22 --- /dev/null +++ b/.gitlab/reliability/publish-gh-pages.sh @@ -0,0 +1,170 @@ +#!/bin/bash + +# publish-gh-pages.sh - Publish reliability test reports to GitHub Pages +# +# Usage: publish-gh-pages.sh [artifacts-dir] +# +# Updates reliability history and regenerates GitHub Pages site. +# Reports are available at: https://datadog.github.io/async-profiler-build/reliability/ +# +# In CI: Uses Octo-STS for secure, short-lived GitHub tokens (no secrets needed) +# Locally: Use 'devflow gitlab auth' for Octo-STS, or set GITHUB_TOKEN env var + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="${SCRIPT_DIR}/../.." +ARTIFACTS_DIR="${1:-${PROJECT_ROOT}}" +export MAX_HISTORY=10 + +# GitHub repo for Pages +GITHUB_REPO="DataDog/java-profiler" +PAGES_URL="https://datadog.github.io/java-profiler" + +# Colors +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +NC='\033[0m' + +log_info() { echo -e "${GREEN}[INFO]${NC} $*"; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } +log_error() { echo -e "${RED}[ERROR]${NC} $*"; } + +# Obtain GitHub token +obtain_github_token() { + # Try dd-octo-sts CLI (works in CI with DDOCTOSTS_ID_TOKEN) + if command -v dd-octo-sts >/dev/null 2>&1 && [ -n "${DDOCTOSTS_ID_TOKEN:-}" ]; then + log_info "Obtaining GitHub token via dd-octo-sts CLI..." + # Policy name matches the .sts.yaml filename (without extension) + + # Run dd-octo-sts and capture only stdout (don't capture stderr to avoid error messages in token) + local TOKEN_OUTPUT + local TOKEN_EXIT_CODE + TOKEN_OUTPUT=$(dd-octo-sts token --scope DataDog/java-profiler --policy async-profiler-build.ci 2>/tmp/dd-octo-sts-error.log) + TOKEN_EXIT_CODE=$? + + if [ $TOKEN_EXIT_CODE -eq 0 ] && [ -n "${TOKEN_OUTPUT}" ]; then + # Validate token format (GitHub tokens start with ghs_, ghp_, or look like JWT) + if [[ "${TOKEN_OUTPUT}" =~ ^(ghs_|ghp_|v1\.|[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.) ]]; then + GITHUB_TOKEN="${TOKEN_OUTPUT}" + log_info "GitHub token obtained via dd-octo-sts (expires in 1 hour)" + return 0 + else + log_warn "dd-octo-sts returned invalid token format (first 50 chars): ${TOKEN_OUTPUT:0:50}" + fi + else + log_warn "dd-octo-sts token exchange failed (exit code: ${TOKEN_EXIT_CODE})" + if [ -s /tmp/dd-octo-sts-error.log ]; then + log_warn "dd-octo-sts error output:" + cat /tmp/dd-octo-sts-error.log | head -10 >&2 + fi + fi + fi + + # Fall back to GITHUB_TOKEN environment variable + if [ -n "${GITHUB_TOKEN:-}" ]; then + log_info "Using GITHUB_TOKEN from environment" + return 0 + fi + + return 1 +} + +if ! obtain_github_token; then + log_error "Failed to obtain GitHub token" + log_error "Options:" + log_error " 1. Run in GitLab CI with dd-octo-sts-ci-base image and DDOCTOSTS_ID_TOKEN" + log_error " 2. Set GITHUB_TOKEN env var (PAT with 'repo' scope)" + exit 1 +fi + +# Create temporary directory for gh-pages content +WORK_DIR=$(mktemp -d) +RUN_JSON_FILE=$(mktemp) +# shellcheck disable=SC2064 # Intentional: capture values at setup time +trap "rm -rf ${WORK_DIR} ${RUN_JSON_FILE}" EXIT + +log_info "Preparing gh-pages content in: ${WORK_DIR}" + +# Clone gh-pages branch (or create if doesn't exist) +log_info "Cloning gh-pages branch..." +cd "${WORK_DIR}" + +if git clone --depth 1 --branch gh-pages "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPO}.git" pages 2>/dev/null; then + cd pages + log_info "Cloned existing gh-pages branch" +else + log_info "Creating new gh-pages branch..." + mkdir pages && cd pages + git init + git remote add origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPO}.git" + git checkout -b gh-pages +fi + +# Create Jekyll config if not exists +if [ ! -f "_config.yml" ]; then + cat > "_config.yml" < "${RUN_JSON_FILE}" 2>/dev/null; then + log_info "Generated run JSON" + + # Update history (prepend new run, keep last MAX_HISTORY) + if "${SCRIPT_DIR}/../common/update-history.sh" reliability "${RUN_JSON_FILE}" "." 2>/dev/null; then + log_info "Updated reliability history" + else + log_warn "Failed to update history" + fi +else + log_warn "Failed to generate run JSON" +fi + +# Generate dashboard and index pages +log_info "Generating dashboard..." +if "${SCRIPT_DIR}/../common/generate-dashboard.sh" "." 2>&1; then + log_info "Generated dashboard index.md" +else + log_warn "Failed to generate dashboard" +fi + +log_info "Generating reliability index..." +if "${SCRIPT_DIR}/../common/generate-index.sh" reliability "." 2>/dev/null; then + log_info "Generated reliability/index.md" +else + log_warn "Failed to generate reliability index" +fi + +# Commit and push +TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M UTC") +log_info "Committing changes..." +git add -A +if git diff --staged --quiet; then + log_info "No changes to commit" +else + git config user.email "ci@datadoghq.com" + git config user.name "CI Bot" + git commit -m "Update reliability reports - ${TIMESTAMP}" + + log_info "Pushing to gh-pages..." + git push origin gh-pages --force + + log_info "Reports published successfully!" + log_info "View at: ${PAGES_URL}/reliability/" +fi + +echo "" +echo "PAGES_URL=${PAGES_URL}/reliability/" diff --git a/.gitlab/reliability/run.sh b/.gitlab/reliability/run.sh new file mode 100755 index 000000000..e598600d2 --- /dev/null +++ b/.gitlab/reliability/run.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -e + +HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +RUNTIME=${1} +CONFIG=${2:-profiler} +VARIANT=${3:-jit} +ALLOCATOR=${4:-gmalloc} +ARCH=${5:-AMD64} + +echo "Running with runtime: ${RUNTIME} seconds, config=${CONFIG}, variant=${VARIANT}, allocator=${ALLOCATOR}, arch=${ARCH}" + +case $VARIANT in + jit) + # Short variant + ${HERE}/jit_stability_check.sh $RUNTIME $CONFIG $ALLOCATOR + ;; + memory) + # Long variant + ${HERE}/memory_trend_check.sh $RUNTIME $CONFIG $ALLOCATOR + ;; + *) + echo "Unknown variant: $VARIANT" + exit 1 + ;; +esac \ No newline at end of file diff --git a/.gitlab/scripts/build.sh b/.gitlab/scripts/build.sh new file mode 100755 index 000000000..ba6a209f2 --- /dev/null +++ b/.gitlab/scripts/build.sh @@ -0,0 +1,43 @@ +#! /bin/bash + +set -eo pipefail # exit on any failure, including mid-pipeline +set -x + +if [ ! -z "${CANCELLED:-}" ]; then + exit 0 +fi + +if [ -z "$TARGET" ]; then + echo "Expecting the TARGET variable to be set" + exit 1 +fi + +HERE=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +REPO_ROOT=$( cd "${HERE}/../.." && pwd ) + +if [ -z "${JAVA_HOME}" ]; then + # workaround for CI when JAVA_HOME is not properly defined + export JAVA_HOME=~/.sdkman/candidates/java/current +fi + +echo "Using Java @ ${JAVA_HOME}" + +source .gitlab/scripts/includes.sh + +function onexit { + mkdir -p "${REPO_ROOT}/test/${TARGET}/reports" + mkdir -p "${REPO_ROOT}/test/${TARGET}/logs" + mv "${REPO_ROOT}/ddprof-test/build/reports" "${REPO_ROOT}/test/${TARGET}/" 2>/dev/null || true + mv /tmp/*.jfr "${REPO_ROOT}/test/${TARGET}/logs" 2>/dev/null || true + mv /tmp/*.json "${REPO_ROOT}/test/${TARGET}/logs" 2>/dev/null || true + mv /tmp/*.txt "${REPO_ROOT}/test/${TARGET}/logs" 2>/dev/null || true + find . -name 'hs_err*' | xargs -I {} cp {} "${REPO_ROOT}/test/${TARGET}/logs" 2>/dev/null || true +} + +trap onexit EXIT + +set -x +./gradlew -Pddprof_version="$(get_version)" -Pskip-cpp-tests :ddprof-lib:assembleReleaseJar --build-cache --stacktrace --info --no-watch-fs --no-daemon + +mkdir -p "${REPO_ROOT}/libs" +cp -r "${REPO_ROOT}/ddprof-lib/build/native/release/META-INF/native-libs/"* "${REPO_ROOT}/libs/" diff --git a/.gitlab/scripts/check-image-updates.sh b/.gitlab/scripts/check-image-updates.sh new file mode 100755 index 000000000..1255d3adc --- /dev/null +++ b/.gitlab/scripts/check-image-updates.sh @@ -0,0 +1,282 @@ +#!/bin/bash +# check-image-updates.sh - Detect available image updates in registry +# +# Compares current image references in YAML files against the registry +# and outputs a JSON array of needed updates. +# +# Usage: ./scripts/check-image-updates.sh +# Output: JSON array to stdout +# +# Required environment: +# CI_JOB_TOKEN - GitLab CI job token (for API access) +# CI_PROJECT_ID - GitLab project ID (auto-set in CI) +# +# Optional environment: +# GITLAB_URL - GitLab instance URL (default: https://gitlab.ddbuild.io) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +# Configuration +REGISTRY="registry.ddbuild.io" +GITLAB_URL="${GITLAB_URL:-https://gitlab.ddbuild.io}" +GITLAB_PROJECT_PATH="DataDog/java-profiler" + +# Colors for stderr output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +log_info() { echo -e "${GREEN}[INFO]${NC} $*" >&2; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $*" >&2; } +log_error() { echo -e "${RED}[ERROR]${NC} $*" >&2; } + +# Image definitions: var_name|yaml_file|tag_suffix|job_name|registry_path +# registry_path is relative to REGISTRY/ci/ +IMAGE_DEFS=( + "BUILD_IMAGE_X64|.gitlab/build-deploy/.gitlab-ci.yml|x64-base|image-base-build-x64|async-profiler-build" + "BUILD_IMAGE_X64_2_17|.gitlab/build-deploy/.gitlab-ci.yml|x64-2.17-base|image-base-build-x64-2.17|async-profiler-build" + "BUILD_IMAGE_X64_MUSL|.gitlab/build-deploy/.gitlab-ci.yml|x64-musl-base|image-base-build-x64-musl|async-profiler-build" + "BUILD_IMAGE_ARM64|.gitlab/build-deploy/.gitlab-ci.yml|arm64-base|image-base-build-arm64|async-profiler-build" + "BUILD_IMAGE_ARM64_MUSL|.gitlab/build-deploy/.gitlab-ci.yml|arm64-musl-base|image-base-build-arm64-musl|async-profiler-build" + "DATADOG_CI_IMAGE|.gitlab/build-deploy/.gitlab-ci.yml|datadog-ci|image-datadog-ci|async-profiler-build" + "BENCHMARK_IMAGE_AMD64|.gitlab/benchmarks/images.yml|amd64-benchmarks|image-benchmarks-amd64|async-profiler-build-amd64" + "BENCHMARK_IMAGE_ARM64|.gitlab/benchmarks/images.yml|arm64-benchmarks|image-benchmarks-arm64|async-profiler-build-arm64" +) + +# Extract current image reference from YAML file +# Returns: full image reference (e.g., registry.../v12345-x64-base@sha256:abc...) +get_current_ref() { + local var_name="$1" + local yaml_file="$2" + local full_path="${PROJECT_ROOT}/${yaml_file}" + + if [[ ! -f "$full_path" ]]; then + log_error "YAML file not found: $full_path" + return 1 + fi + + # Match lines like: BUILD_IMAGE_X64: registry.../... + # Handle both with and without quotes + # Use awk to split on first colon after variable name + grep -E "^\s*${var_name}:" "$full_path" | \ + sed "s/^[[:space:]]*${var_name}:[[:space:]]*//" | \ + tr -d ' "'"'" | \ + head -1 +} + +# Extract pipeline ID from image tag +# Input: v95799584-x64-base or registry.../v95799584-x64-base@sha256:... +# Output: 95799584 +extract_pipeline_id() { + local ref="$1" + echo "$ref" | grep -oE 'v[0-9]+' | head -1 | sed 's/^v//' +} + +# Extract tag from full image reference +# Input: registry.../v95799584-x64-base@sha256:abc... +# Output: v95799584-x64-base +extract_tag() { + local ref="$1" + echo "$ref" | sed 's/@sha256:.*//' | rev | cut -d':' -f1 | rev +} + +# Query registry for latest tag matching pattern +# Returns the tag with highest pipeline ID +get_latest_tag() { + local registry_path="$1" + local tag_suffix="$2" + local full_registry="${REGISTRY}/ci/${registry_path}" + + log_info "Querying registry: ${full_registry} for *-${tag_suffix}" + + # List all tags, filter by suffix, sort by pipeline ID (numeric), get latest + local tags + tags=$(crane ls "${full_registry}" 2>/dev/null || echo "") + + if [[ -z "$tags" ]]; then + log_warn "No tags found in ${full_registry}" + return 1 + fi + + # Filter tags matching pattern v- + local matching_tags + matching_tags=$(echo "$tags" | grep -E "^v[0-9]+-${tag_suffix}$" || true) + + if [[ -z "$matching_tags" ]]; then + log_warn "No tags matching pattern *-${tag_suffix}" + return 1 + fi + + # Sort by pipeline ID (extract number after 'v', sort numerically) + echo "$matching_tags" | \ + awk -F'-' '{print $1}' | \ + sed 's/^v//' | \ + sort -n | \ + tail -1 | \ + xargs -I{} echo "v{}-${tag_suffix}" +} + +# Get SHA256 digest for an image tag +get_digest() { + local registry_path="$1" + local tag="$2" + local full_image="${REGISTRY}/ci/${registry_path}:${tag}" + + crane digest "${full_image}" 2>/dev/null +} + +# Query GitLab API to find job URL within a pipeline +get_job_url() { + local pipeline_id="$1" + local job_name="$2" + + if [[ -z "${CI_JOB_TOKEN:-}" ]]; then + log_warn "CI_JOB_TOKEN not set, cannot query GitLab API for job URL" + echo "" + return 0 + fi + + # URL-encode the project path + local encoded_project + encoded_project=$(echo -n "${GITLAB_PROJECT_PATH}" | jq -sRr @uri) + + local api_url="${GITLAB_URL}/api/v4/projects/${encoded_project}/pipelines/${pipeline_id}/jobs" + + log_info "Querying GitLab API for job '${job_name}' in pipeline ${pipeline_id}" + + local response + response=$(curl -s --fail \ + -H "JOB-TOKEN: ${CI_JOB_TOKEN}" \ + "${api_url}" 2>/dev/null || echo "[]") + + # Find the job with matching name and extract web_url + local job_url + job_url=$(echo "$response" | jq -r ".[] | select(.name == \"${job_name}\") | .web_url" 2>/dev/null | head -1) + + if [[ -n "$job_url" && "$job_url" != "null" ]]; then + echo "$job_url" + else + log_warn "Job '${job_name}' not found in pipeline ${pipeline_id}" + # Fallback to pipeline URL + echo "${GITLAB_URL}/${GITLAB_PROJECT_PATH}/-/pipelines/${pipeline_id}" + fi +} + +# Main detection logic +main() { + log_info "Starting image update detection..." + + # Verify crane is available + if ! command -v crane &>/dev/null; then + log_error "crane command not found. Please install google/go-containerregistry" + exit 1 + fi + + # Verify jq is available + if ! command -v jq &>/dev/null; then + log_error "jq command not found. Please install jq" + exit 1 + fi + + cd "$PROJECT_ROOT" + + local updates="[]" + local checked=0 + local found=0 + + for def in "${IMAGE_DEFS[@]}"; do + IFS='|' read -r var_name yaml_file tag_suffix job_name registry_path <<< "$def" + ((checked++)) + + log_info "Checking ${var_name}..." + + # Get current reference from YAML + local current_ref + current_ref=$(get_current_ref "$var_name" "$yaml_file" || echo "") + if [[ -z "$current_ref" ]]; then + log_warn "Could not find ${var_name} in ${yaml_file}, skipping" + continue + fi + + local current_tag + current_tag=$(extract_tag "$current_ref") + local current_pipeline_id + current_pipeline_id=$(extract_pipeline_id "$current_ref") + + log_info " Current: ${current_tag} (pipeline ${current_pipeline_id})" + + # Get latest tag from registry + local latest_tag + latest_tag=$(get_latest_tag "$registry_path" "$tag_suffix" || echo "") + if [[ -z "$latest_tag" ]]; then + log_warn " Could not determine latest tag, skipping" + continue + fi + + local latest_pipeline_id + latest_pipeline_id=$(extract_pipeline_id "$latest_tag") + + log_info " Latest: ${latest_tag} (pipeline ${latest_pipeline_id})" + + # Compare pipeline IDs + if [[ "$latest_pipeline_id" -gt "$current_pipeline_id" ]]; then + log_info " UPDATE AVAILABLE: ${current_tag} -> ${latest_tag}" + ((found++)) + + # Get digest for new image + local new_digest + new_digest=$(get_digest "$registry_path" "$latest_tag" || echo "") + if [[ -z "$new_digest" ]]; then + log_warn " Could not get digest for ${latest_tag}, skipping" + continue + fi + + # Get job URL from GitLab API + local job_url + job_url=$(get_job_url "$latest_pipeline_id" "$job_name") + + # Build full image reference + local new_full_ref="${REGISTRY}/ci/${registry_path}:${latest_tag}@${new_digest}" + + # Extract current digest for comparison + local current_digest + current_digest=$(echo "$current_ref" | grep -oE 'sha256:[a-f0-9]+' || echo "") + + # Add to updates array + updates=$(echo "$updates" | jq \ + --arg var_name "$var_name" \ + --arg yaml_file "$yaml_file" \ + --arg current_tag "$current_tag" \ + --arg current_digest "$current_digest" \ + --arg new_tag "$latest_tag" \ + --arg new_digest "$new_digest" \ + --arg new_full_ref "$new_full_ref" \ + --arg job_url "$job_url" \ + --arg job_name "$job_name" \ + '. + [{ + var_name: $var_name, + yaml_file: $yaml_file, + current_tag: $current_tag, + current_digest: $current_digest, + new_tag: $new_tag, + new_digest: $new_digest, + new_full_ref: $new_full_ref, + job_url: $job_url, + job_name: $job_name + }]') + else + log_info " Up to date" + fi + done + + log_info "Detection complete: checked ${checked} images, found ${found} updates" + + # Output JSON to stdout + echo "$updates" | jq . +} + +main "$@" diff --git a/.gitlab/scripts/create-image-update-pr.sh b/.gitlab/scripts/create-image-update-pr.sh new file mode 100755 index 000000000..032644658 --- /dev/null +++ b/.gitlab/scripts/create-image-update-pr.sh @@ -0,0 +1,394 @@ +#!/bin/bash +# create-image-update-pr.sh - Create GitHub PR for image updates +# +# Takes JSON output from check-image-updates.sh and creates a PR on GitHub +# with the updated image references. +# +# Usage: ./scripts/create-image-update-pr.sh [updates.json] +# cat updates.json | ./scripts/create-image-update-pr.sh +# +# Required environment: +# DDOCTOSTS_ID_TOKEN - GitLab OIDC token for Octo-STS (CI provides this) +# OR +# GITHUB_TOKEN - GitHub token with repo write access (for bootstrap) +# +# Optional environment: +# DRY_RUN - Set to 'true' to skip PR creation + +set -euo pipefail + +# Configuration +GITHUB_REPO="DataDog/java-profiler" +OCTO_STS_SCOPE="DataDog/java-profiler" +OCTO_STS_POLICY="update-images" + +# Colors for stderr output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +log_info() { echo -e "${GREEN}[INFO]${NC} $*" >&2; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $*" >&2; } +log_error() { echo -e "${RED}[ERROR]${NC} $*" >&2; } + +# Obtain GitHub token +obtain_github_token() { + # Try Octo-STS first (preferred for CI) + if command -v dd-octo-sts &>/dev/null && [[ -n "${DDOCTOSTS_ID_TOKEN:-}" ]]; then + log_info "Obtaining GitHub token via Octo-STS..." + local token + token=$(dd-octo-sts token --scope "${OCTO_STS_SCOPE}" --policy "${OCTO_STS_POLICY}" 2>/dev/null || echo "") + if [[ -n "$token" ]]; then + log_info "Successfully obtained GitHub token via Octo-STS" + echo "$token" + return 0 + fi + log_warn "Octo-STS token exchange failed (policy may not exist yet)" + fi + + # Fall back to GITHUB_TOKEN environment variable + if [[ -n "${GITHUB_TOKEN:-}" ]]; then + log_info "Using GITHUB_TOKEN from environment" + echo "${GITHUB_TOKEN}" + return 0 + fi + + log_error "No GitHub authentication available" + log_error "Either:" + log_error " 1. Run in GitLab CI with DDOCTOSTS_ID_TOKEN configured, OR" + log_error " 2. Set GITHUB_TOKEN environment variable (for bootstrap)" + return 1 +} + +# Check if STS policy exists in the cloned repo +check_sts_policy_exists() { + local repo_dir="$1" + [[ -f "${repo_dir}/.github/chainguard/update-images.sts.yaml" ]] +} + +# Create STS policy file content +create_sts_policy_content() { + cat <<'EOF' +# Octo-STS Trust Policy for Image Update PRs +# +# This policy allows the GitLab CI check-image-updates job to: +# - Push branches to the repository +# - Create pull requests for image updates +# +# Trust Policy Location: .github/chainguard/update-images.sts.yaml +# Documentation: https://edu.chainguard.dev/open-source/octo-sts/ + +# GitLab OIDC issuer +issuer: https://gitlab.ddbuild.io + +# Match GitLab CI jobs from the async-profiler-build project +# The scheduled job runs on main, but we allow any branch for manual triggers +subject_pattern: project_path:DataDog/java-profiler:ref_type:branch:ref:.* + +# GitHub API permissions +permissions: + # Required to push branches + contents: write + # Required to create PRs + pull_requests: write +EOF +} + +# Update a YAML file with new image reference +update_yaml_file() { + local yaml_file="$1" + local var_name="$2" + local new_full_ref="$3" + local job_url="$4" + + log_info "Updating ${var_name} in ${yaml_file}" + + # Comment format: # Generated by + # For .gitlab-ci.yml (GHTOOLS_IMAGE), format is just: # + local comment_prefix="# Generated by" + if [[ "$yaml_file" == ".gitlab-ci.yml" ]]; then + comment_prefix="#" + fi + + # First, update or add the comment line before the variable + # Look for existing comment and variable pattern + if grep -q "^\s*${comment_prefix}.*${var_name}" "$yaml_file" 2>/dev/null || \ + grep -B1 "^\s*${var_name}:" "$yaml_file" | grep -q "^#"; then + # Update existing comment (line before the variable) + # Use sed with pattern matching: find the line before VAR_NAME and update it + sed -i.bak -E "/^\s*${var_name}:/{ + x + s|^.*$| ${comment_prefix} ${job_url}| + x + }" "$yaml_file" + rm -f "${yaml_file}.bak" + fi + + # Update the variable value + # Match: " VAR_NAME: old_value" and replace with " VAR_NAME: new_value" + sed -i.bak -E "s|^(\s*${var_name}:).*$|\1 ${new_full_ref}|" "$yaml_file" + rm -f "${yaml_file}.bak" +} + +# More robust YAML update using line-by-line processing +update_yaml_robust() { + local yaml_file="$1" + local var_name="$2" + local new_full_ref="$3" + local job_url="$4" + + log_info "Updating ${var_name} in ${yaml_file}" + + local temp_file + temp_file=$(mktemp) + + # Determine comment format + local comment_line + if [[ "$yaml_file" == ".gitlab-ci.yml" ]]; then + comment_line=" # ${job_url}" + else + comment_line=" # Generated by ${job_url}" + fi + + local prev_was_comment=false + + while IFS= read -r line || [[ -n "$line" ]]; do + # Check if this line is the target variable + if [[ "$line" =~ ^[[:space:]]*${var_name}: ]]; then + # Check if previous line was a comment we should replace + if $prev_was_comment; then + # Remove the last line from temp_file (the old comment) + head -n -1 "$temp_file" > "${temp_file}.tmp" + mv "${temp_file}.tmp" "$temp_file" + fi + # Write new comment and updated variable + echo "$comment_line" >> "$temp_file" + # Extract indentation from original line + local indent + indent=$(echo "$line" | sed 's/^\([[:space:]]*\).*/\1/') + echo "${indent}${var_name}: ${new_full_ref}" >> "$temp_file" + prev_was_comment=false + continue + fi + + # Track if this line is a comment (for next iteration) + if [[ "$line" =~ ^[[:space:]]*# ]]; then + prev_was_comment=true + else + prev_was_comment=false + fi + + echo "$line" >> "$temp_file" + done < "$yaml_file" + + mv "$temp_file" "$yaml_file" +} + +# Build PR description +build_pr_description() { + local updates_json="$1" + local include_sts_policy="$2" + + cat </dev/null || echo "") + + if [[ -n "$existing_pr" ]]; then + echo "$existing_pr" + fi +} + +main() { + local updates_file="${1:-}" + + # Read updates JSON + local updates_json + if [[ -n "$updates_file" ]]; then + if [[ ! -f "$updates_file" ]]; then + log_error "Updates file not found: $updates_file" + exit 1 + fi + updates_json=$(cat "$updates_file") + else + log_info "Reading updates from stdin..." + updates_json=$(cat) + fi + + # Validate JSON + if ! echo "$updates_json" | jq empty 2>/dev/null; then + log_error "Invalid JSON input" + exit 1 + fi + + # Reject any entry missing a valid sha256 digest in new_full_ref + local bad_refs + bad_refs=$(echo "$updates_json" | jq -r '.[] | select(.new_full_ref | test("@sha256:[a-f0-9]{64}$") | not) | .var_name') + if [[ -n "$bad_refs" ]]; then + log_error "Refusing to create PR: the following entries have missing or invalid digest in new_full_ref:" + echo "$bad_refs" | while read -r v; do log_error " $v"; done + exit 1 + fi + + # Check if there are any updates + local update_count + update_count=$(echo "$updates_json" | jq 'length') + if [[ "$update_count" -eq 0 ]]; then + log_info "No updates to apply" + exit 0 + fi + + log_info "Processing ${update_count} image update(s)..." + + # Build PR title and description early (for dry run) + local pr_title + pr_title="ci: Update CI images ($(date +%Y-%m-%d))" + local branch_name + branch_name="ci/update-images-$(date +%Y%m%d-%H%M%S)" + + # For dry run, show what would happen without cloning + if [[ "${DRY_RUN:-}" == "true" ]]; then + # Assume STS policy doesn't exist for dry run display + local include_sts_policy="true" + local pr_body + pr_body=$(build_pr_description "$updates_json" "$include_sts_policy") + + log_info "DRY RUN: Would create PR with:" + log_info " Title: ${pr_title}" + log_info " Branch: ${branch_name}" + log_info " Files that would be changed:" + echo "$updates_json" | jq -r '.[] | " - \(.yaml_file): \(.var_name)"' + log_info " - .github/chainguard/update-images.sts.yaml (bootstrap, if not exists)" + log_info "" + log_info " PR Description:" + echo "$pr_body" + exit 0 + fi + + # Obtain GitHub token + local gh_token + gh_token=$(obtain_github_token) || exit 1 + + # Check for existing PR + local existing_pr + existing_pr=$(check_existing_pr "$gh_token") + if [[ -n "$existing_pr" ]]; then + log_warn "An open PR already exists: ${existing_pr}" + log_warn "Please close or merge it before creating a new one" + exit 0 + fi + + # Create temp directory for clone + local work_dir + work_dir=$(mktemp -d) + # shellcheck disable=SC2064 + trap "rm -rf '$work_dir'" EXIT + + log_info "Cloning ${GITHUB_REPO}..." + cd "$work_dir" + git clone --depth 1 "https://x-access-token:${gh_token}@github.com/${GITHUB_REPO}.git" repo + cd repo + + # Check if STS policy exists + local include_sts_policy="false" + if ! check_sts_policy_exists "."; then + log_info "STS policy not found, will include in PR (bootstrap)" + include_sts_policy="true" + mkdir -p .github/chainguard + create_sts_policy_content > .github/chainguard/update-images.sts.yaml + fi + + # Create branch + log_info "Creating branch: ${branch_name}" + git checkout -b "$branch_name" + + # Apply updates to YAML files + echo "$updates_json" | jq -c '.[]' | while read -r update; do + local var_name yaml_file new_full_ref job_url + var_name=$(echo "$update" | jq -r '.var_name') + yaml_file=$(echo "$update" | jq -r '.yaml_file') + new_full_ref=$(echo "$update" | jq -r '.new_full_ref') + job_url=$(echo "$update" | jq -r '.job_url // ""') + + update_yaml_robust "$yaml_file" "$var_name" "$new_full_ref" "$job_url" + done + + # Build final PR description + local pr_body + pr_body=$(build_pr_description "$updates_json" "$include_sts_policy") + + # Commit changes + git config user.email "ci@datadoghq.com" + git config user.name "CI Bot" + git add -A + + # Check if there are changes to commit + if git diff --cached --quiet; then + log_warn "No changes to commit" + exit 0 + fi + + git commit -m "$pr_title" + + # Push branch + log_info "Pushing branch to origin..." + git push -u origin "$branch_name" + + # Create PR + log_info "Creating pull request..." + local pr_url + pr_url=$(GH_TOKEN="$gh_token" gh pr create \ + --repo "${GITHUB_REPO}" \ + --title "$pr_title" \ + --body "$pr_body" \ + --base main \ + --head "$branch_name" \ + --draft) + + log_info "PR created: ${pr_url}" + + # Try to add label (may fail if label doesn't exist) + GH_TOKEN="$gh_token" gh pr edit "$pr_url" --add-label "CI" 2>/dev/null || true + GH_TOKEN="$gh_token" gh pr edit "$pr_url" --add-label "AI" 2>/dev/null || true + + echo "$pr_url" +} + +main "$@" diff --git a/.gitlab/scripts/deploy.sh b/.gitlab/scripts/deploy.sh new file mode 100755 index 000000000..c0e74f676 --- /dev/null +++ b/.gitlab/scripts/deploy.sh @@ -0,0 +1,51 @@ +#! /bin/bash + +set -eo pipefail # exit on any failure, including mid-pipeline +set +x + +if [ ! -z "${CANCELLED:-}" ]; then + exit 0 +fi + +# NEW: Mode parameter +MODE="${1:-all}" # Options: all, assemble, publish + +HERE=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +# Load centralized configuration +source "${HERE}/../../.gitlab/config.env" + +# debug the CI env +echo "CI_COMMIT_TAG=${CI_COMMIT_TAG}" +echo "CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH}" +echo "CI_DEFAULT_BRANCH=${CI_DEFAULT_BRANCH}" +echo "MODE=${MODE}" + +# Only fetch AWS SSM secrets when publishing +if [ "$MODE" = "publish" ] || [ "$MODE" = "all" ]; then + export SONATYPE_USERNAME=$(aws ssm get-parameter --region ${AWS_REGION} --name ${SSM_PREFIX}.sonatype_token_user --with-decryption --query "Parameter.Value" --out text) + export SONATYPE_PASSWORD=$(aws ssm get-parameter --region ${AWS_REGION} --name ${SSM_PREFIX}.sonatype_token --with-decryption --query "Parameter.Value" --out text) + export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region ${AWS_REGION} --name ${SSM_PREFIX}.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text) + export GPG_PASSWORD=$(aws ssm get-parameter --region ${AWS_REGION} --name ${SSM_PREFIX}.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text) +fi + +source .gitlab/scripts/includes.sh + +LIB_VERSION=$(get_version) +echo "com.datadoghq:ddprof:${LIB_VERSION}" > version.txt + +# Assemble task (always needed for artifact creation) +if [ "$MODE" = "assemble" ] || [ "$MODE" = "all" ]; then + echo "=== Assembling artifact ===" + ./gradlew -Pskip-native -Pskip-tests -Pddprof_version="${LIB_VERSION}" -PbuildInfo.build.number=$CI_JOB_ID -Pwith-libs="$(pwd)/libs" :ddprof-lib:jar assembleAll --exclude-task compileFuzzer --exclude-task sign --max-workers=1 --no-build-cache --stacktrace --info --no-watch-fs --no-daemon +fi + +# Publish task (only when publishing to Maven Central) +if [ "$MODE" = "publish" ] || [ "$MODE" = "all" ]; then + echo "=== Publishing to Sonatype ===" + if [ -z "${GPG_PRIVATE_KEY:-}" ]; then + echo "ERROR: GPG_PRIVATE_KEY is not set — run the create_key CI job first to provision the signing key in SSM (ci.java-profiler.signing.gpg_private_key)" + exit 1 + fi + ./gradlew -Pskip-native -Pskip-tests -Pddprof_version="${LIB_VERSION}" -PbuildInfo.build.number=$CI_JOB_ID -Pwith-libs="$(pwd)/libs" publishToSonatype closeAndReleaseSonatypeStagingRepository --exclude-task compileFuzzer --max-workers=1 --no-build-cache --stacktrace --info --no-watch-fs --no-daemon +fi diff --git a/.gitlab/scripts/get-github-token-via-octo-sts.sh b/.gitlab/scripts/get-github-token-via-octo-sts.sh new file mode 100755 index 000000000..0bb85011f --- /dev/null +++ b/.gitlab/scripts/get-github-token-via-octo-sts.sh @@ -0,0 +1,101 @@ +#!/bin/bash + +# Exchange GitLab CI OIDC Token for GitHub Token via Datadog Octo-STS +# This script uses GitLab's OIDC provider to obtain a short-lived GitHub token +# without requiring any stored secrets in the GitLab project. + +set -euo pipefail + +# Configuration +OCTO_STS_DOMAIN="${OCTO_STS_DOMAIN:-octo-sts.chainguard.dev}" +OCTO_STS_AUDIENCE="${OCTO_STS_AUDIENCE:-dd-octo-sts}" +OCTO_STS_SCOPE="${OCTO_STS_SCOPE:-DataDog/java-profiler}" +OCTO_STS_POLICY="${OCTO_STS_POLICY:-gist-update}" + +# Colors for output +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +NC='\033[0m' # No Color + +function log_info() { + echo -e "${GREEN}[INFO]${NC} $*" >&2 +} + +function log_warn() { + echo -e "${YELLOW}[WARN]${NC} $*" >&2 +} + +function log_error() { + echo -e "${RED}[ERROR]${NC} $*" >&2 +} + +# Validate GitLab CI environment +if [ -z "${CI:-}" ]; then + log_error "This script must run in GitLab CI environment" + log_error "CI environment variable not set" + exit 1 +fi + +# Check for GitLab OIDC token +if [ -z "${CI_JOB_JWT_V2:-}" ]; then + log_error "GitLab OIDC token (CI_JOB_JWT_V2) not available" + log_error "Ensure the CI job has 'id_tokens:' configured" + exit 1 +fi + +log_info "Exchanging GitLab OIDC token for GitHub token via Octo-STS..." +log_info "Scope: ${OCTO_STS_SCOPE}" +log_info "Policy: ${OCTO_STS_POLICY}" + +# Build Octo-STS exchange URL +EXCHANGE_URL="https://${OCTO_STS_DOMAIN}/sts/exchange?scope=${OCTO_STS_SCOPE}&identity=${OCTO_STS_POLICY}" + +log_info "Exchange URL: ${EXCHANGE_URL}" + +# Exchange OIDC token for GitHub token +response=$(curl -s -w "\n%{http_code}" \ + -X POST \ + -H "Authorization: Bearer ${CI_JOB_JWT_V2}" \ + -H "Accept: application/json" \ + "${EXCHANGE_URL}") + +# Split response into body and status code +http_code=$(echo "${response}" | tail -n1) +body=$(echo "${response}" | sed '$d') + +log_info "HTTP status: ${http_code}" +log_info "Response body: ${body}" + +# Check HTTP status code +if [ "${http_code}" -ne 200 ]; then + log_error "Octo-STS token exchange failed (HTTP ${http_code})" + log_error "Response: ${body}" + + if [ "${http_code}" -eq 401 ]; then + log_error "Authentication failed - OIDC token was rejected" + log_error "Possible causes:" + log_error " 1. Trust policy not configured for this repository" + log_error " 2. Trust policy doesn't match GitLab CI claims" + log_error " 3. Octo-STS configuration issue" + elif [ "${http_code}" -eq 404 ]; then + log_error "Trust policy not found" + log_error "Expected location: https://github.com/${OCTO_STS_SCOPE}/.github/chainguard/${OCTO_STS_POLICY}.sts.yaml" + fi + + exit 1 +fi + +# Extract token from response (expecting JSON: {"token": "ghs_..."}) +github_token=$(echo "${body}" | grep -o '"token":"[^"]*"' | cut -d'"' -f4) + +if [ -z "${github_token}" ]; then + log_error "Failed to extract GitHub token from response" + log_error "Response: ${body}" + exit 1 +fi + +log_info "✅ Successfully obtained GitHub token (expires in 1 hour)" + +# Output token to stdout (caller can capture with TOKEN=$(./get-github-token-via-octo-sts.sh)) +echo "${github_token}" diff --git a/.gitlab/scripts/includes.sh b/.gitlab/scripts/includes.sh new file mode 100755 index 000000000..bfb0fbccb --- /dev/null +++ b/.gitlab/scripts/includes.sh @@ -0,0 +1,75 @@ +function get_version() { + rm -f .version + + if [[ "${CI_COMMIT_TAG}" =~ ^v_[0-9.]+(-SNAPSHOT)?$ ]]; then + echo "${CI_COMMIT_TAG//v_/}" + return + fi + + local gradlecmd=$GRADLE_CMD + if [ -z "$gradlecmd" ]; then + gradlecmd="./gradlew" + fi + ${gradlecmd} printVersion --max-workers=1 --build-cache --stacktrace --info --no-watch-fs --no-daemon | grep 'Version:' | cut -f2 -d' ' > .version + local version=$(cat .version) + if [ -z "$version" ]; then + echo "ERROR: Failed to determine version from Gradle printVersion task" >&2 + return 1 + fi + + local branch="${CI_COMMIT_BRANCH:-${CI_COMMIT_REF_NAME:-}}" + local default_branch="${CI_DEFAULT_BRANCH:-main}" + if [ -n "${branch}" ] && [ "${branch}" != "${default_branch}" ] && [ "${branch}" != "main" ]; then + local suffix=$(echo "$branch" | tr '/' '_') + version=$(echo "$version" | sed "s#-SNAPSHOT#-${suffix}-SNAPSHOT#g") + fi + echo "${version}" +} + +function get_current_version() { + get_version +} + +function get_previous_version() { + CURRENT=$(get_current_version) + LOOKBACK=1 + if [[ ! $CURRENT =~ ^.*?-SNAPSHOT$ ]]; then + # current version is not a snapshot; need to look at the previous tag + LOOKBACK=2 + fi + git tag | grep v_ | sort -t_ -k2,2V | tail -n ${LOOKBACK} | head -n 1 | sed -e "s#v_##g" +} + +function setup_java_home() { + if [ -z "${JAVA_HOME}" ]; then + export JAVA_HOME=~/.sdkman/candidates/java/current + fi + + if [ ! -d "$JAVA_HOME" ]; then + echo "ERROR: JAVA_HOME does not exist: $JAVA_HOME" + exit 1 + fi + + echo "Using Java @ ${JAVA_HOME}" +} + +function collect_artifacts() { + local target=$1 + local artifact_type=$2 # "test" or "stresstest" + local source_dir=$3 + local base_dir=${4:-${HERE:-$(pwd)}} + + mkdir -p "${base_dir}/${artifact_type}/${target}/reports" + mkdir -p "${base_dir}/${artifact_type}/${target}/logs" + + # Collect reports + if [ -d "${source_dir}/build/reports" ]; then + cp -r "${source_dir}/build/reports" "${base_dir}/${artifact_type}/${target}/" || echo "WARNING: No reports found" + fi + + # Collect logs from /tmp + find /tmp -maxdepth 1 \( -name "*.jfr" -o -name "*.json" -o -name "*.txt" \) -exec cp {} "${base_dir}/${artifact_type}/${target}/logs/" \; 2>/dev/null || true + + # Collect crash logs (limit search depth to avoid long searches) + find . -maxdepth 2 -name 'hs_err*' -exec cp {} "${base_dir}/${artifact_type}/${target}/logs/" \; 2>/dev/null || true +} diff --git a/.gitlab/scripts/prepare.sh b/.gitlab/scripts/prepare.sh new file mode 100755 index 000000000..c9b567ef9 --- /dev/null +++ b/.gitlab/scripts/prepare.sh @@ -0,0 +1,33 @@ +#! /bin/bash + +set -eo pipefail # exit on any failure, including mid-pipeline + +# Normalize CI_COMMIT_BRANCH — it may be empty for trigger/pipeline sources; +# fall back to CI_COMMIT_REF_NAME which is always populated by GitLab. +CI_COMMIT_BRANCH="${CI_COMMIT_BRANCH:-${CI_COMMIT_REF_NAME:-}}" +export CI_COMMIT_BRANCH + +# Check if we should skip non-PR branch builds +# Allow: default branch pushes, scheduled runs, and web (manual) triggers +# Gate: push/trigger/pipeline sources on non-default branches must have an open GitHub PR +if [ "${CI_PIPELINE_SOURCE}" == "push" ] || [ "${CI_PIPELINE_SOURCE}" == "trigger" ] || [ "${CI_PIPELINE_SOURCE}" == "pipeline" ]; then + if [ -n "${CI_COMMIT_BRANCH}" ] && [ "${CI_COMMIT_BRANCH}" != "${CI_DEFAULT_BRANCH:-main}" ] && [[ ! ${CI_COMMIT_TAG} =~ ^v_[0-9]+(-SNAPSHOT)?$ ]]; then + # Check if the branch has an open PR in DataDog/java-profiler + API_RESPONSE=$(curl -sf "https://api.github.com/repos/DataDog/java-profiler/pulls?head=DataDog:${CI_COMMIT_BRANCH}&state=open&per_page=1" 2>/dev/null || echo "") + if [ -n "${API_RESPONSE}" ] && ! echo "${API_RESPONSE}" | grep -q '"number"'; then + echo "No open PR for branch ${CI_COMMIT_BRANCH}, skipping pipeline" + echo "CANCELLED=true" >> build.env + exit 0 + fi + fi +fi + +apt-get update -qq && apt-get install -y -qq openjdk-21-jdk-headless + +source .gitlab/scripts/includes.sh + +LIB_VERSION=$(get_version) +echo "com.datadoghq:ddprof:${LIB_VERSION}" > version.txt + +# Export CI_COMMIT_BRANCH so downstream jobs inherit the resolved value +echo "CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH}" >> build.env diff --git a/.gitlab/scripts/rebuild-images.sh b/.gitlab/scripts/rebuild-images.sh new file mode 100755 index 000000000..091eab6d9 --- /dev/null +++ b/.gitlab/scripts/rebuild-images.sh @@ -0,0 +1,276 @@ +#!/bin/bash +# rebuild-images.sh - Build and push CI images, then create a GitHub PR with updated references +# +# Usage: REBUILD_IMAGES="x64,arm64" ./scripts/rebuild-images.sh +# REBUILD_IMAGES="" ./scripts/rebuild-images.sh # builds all images +# +# Required environment (auto-set in GitLab CI): +# CI_PIPELINE_ID - Pipeline ID used as image tag prefix +# CI_JOB_URL - URL of the CI job (included in PR comment) +# +# Optional environment: +# REBUILD_IMAGES - Comma/space-separated short names; empty = all +# DRY_RUN - Set to 'true' to print plan and exit without building +# REGISTRY - Override registry (default: registry.ddbuild.io) +# +# Base image variables come from .gitlab/build-deploy/images.yml which is +# included in the root pipeline and available as CI variables: +# OPENJDK_BASE_IMAGE, OPENJDK_BASE_IMAGE_ARM64, OPENJDK_BASE_IMAGE_MUSL, +# OPENJDK_BASE_IMAGE_ARM64_MUSL, BASE_IMAGE_LIBC_2_17, BASE_BENCHMARK_IMAGE_NAME, +# DOCKER_IMAGE + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +REGISTRY="${REGISTRY:-registry.ddbuild.io}" + +# Colors for stderr +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +log_info() { echo -e "${GREEN}[INFO]${NC} $*" >&2; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $*" >&2; } +log_error() { echo -e "${RED}[ERROR]${NC} $*" >&2; } + +usage() { + echo "Usage: REBUILD_IMAGES=\"\" $0" >&2 + echo "" >&2 + echo "Valid short names:" >&2 + echo " x64 glibc x86_64 build image" >&2 + echo " x64-2.17 glibc 2.17 (centos7) x86_64 build image" >&2 + echo " x64-musl musl x86_64 build image" >&2 + echo " arm64 glibc arm64 build image" >&2 + echo " arm64-musl musl arm64 build image" >&2 + echo " datadog-ci datadog-ci utility image" >&2 + echo " benchmarks-amd64 benchmark runner image for amd64" >&2 + echo " benchmarks-arm64 benchmark runner image for arm64" >&2 + echo "" >&2 + echo "Leave REBUILD_IMAGES empty to rebuild all images." >&2 +} + +# IMAGE_DEFS format (pipe-delimited): +# short_name | VAR_NAME | yaml_file | tag_suffix | dockerfile | platform | registry_path | base_image_var +# +# base_image_var is the name of the env var holding the base image (empty if none needed). +IMAGE_DEFS=( + "x64|BUILD_IMAGE_X64|.gitlab/build-deploy/.gitlab-ci.yml|x64-base|.gitlab/base/Dockerfile|linux/amd64|async-profiler-build|OPENJDK_BASE_IMAGE" + "x64-2.17|BUILD_IMAGE_X64_2_17|.gitlab/build-deploy/.gitlab-ci.yml|x64-2.17-base|.gitlab/base/centos7/Dockerfile|linux/amd64|async-profiler-build|BASE_IMAGE_LIBC_2_17" + "x64-musl|BUILD_IMAGE_X64_MUSL|.gitlab/build-deploy/.gitlab-ci.yml|x64-musl-base|.gitlab/base/Dockerfile|linux/amd64|async-profiler-build|OPENJDK_BASE_IMAGE_MUSL" + "arm64|BUILD_IMAGE_ARM64|.gitlab/build-deploy/.gitlab-ci.yml|arm64-base|.gitlab/base/Dockerfile|linux/arm64|async-profiler-build|OPENJDK_BASE_IMAGE_ARM64" + "arm64-musl|BUILD_IMAGE_ARM64_MUSL|.gitlab/build-deploy/.gitlab-ci.yml|arm64-musl-base|.gitlab/base/Dockerfile|linux/arm64|async-profiler-build|OPENJDK_BASE_IMAGE_ARM64_MUSL" + "datadog-ci|DATADOG_CI_IMAGE|.gitlab/build-deploy/.gitlab-ci.yml|datadog-ci|.gitlab/Dockerfile.datadog-ci|linux/amd64|async-profiler-build|" + "benchmarks-amd64|BENCHMARK_IMAGE_AMD64|.gitlab/benchmarks/images.yml|amd64-benchmarks|.gitlab/benchmarks/docker/Dockerfile|linux/amd64|async-profiler-build-amd64|BASE_BENCHMARK_IMAGE_NAME" + "benchmarks-arm64|BENCHMARK_IMAGE_ARM64|.gitlab/benchmarks/images.yml|arm64-benchmarks|.gitlab/benchmarks/docker/Dockerfile|linux/arm64|async-profiler-build-arm64|BASE_BENCHMARK_IMAGE_NAME" +) + +# Extract current image reference from YAML file (reused from check-image-updates.sh) +get_current_ref() { + local var_name="$1" + local yaml_file="$2" + local full_path="${PROJECT_ROOT}/${yaml_file}" + + if [[ ! -f "$full_path" ]]; then + log_error "YAML file not found: $full_path" + return 1 + fi + + grep -E "^\s*${var_name}:" "$full_path" | \ + sed "s/^[[:space:]]*${var_name}:[[:space:]]*//" | \ + tr -d ' "'"'" | \ + head -1 +} + +# Extract tag from full image reference (reused from check-image-updates.sh) +extract_tag() { + local ref="$1" + echo "$ref" | sed 's/@sha256:.*//' | rev | cut -d':' -f1 | rev +} + +# Build one image and return its digest. +# Arguments: short_name tag dockerfile platform registry_path base_image +# Prints the sha256 digest to stdout on success. +build_image() { + local short_name="$1" + local tag="$2" + local dockerfile="$3" + local platform="$4" + local registry_path="$5" + local base_image="$6" + + local full_tag="${REGISTRY}/ci/${registry_path}:${tag}" + local meta_file + meta_file=$(mktemp) + + local build_args=() + [[ -n "$base_image" ]] && build_args+=(--build-arg "BASE_IMAGE=${base_image}") + [[ -n "${CI_JOB_TOKEN:-}" ]] && build_args+=(--build-arg "CI_JOB_TOKEN=${CI_JOB_TOKEN}") + + # benchmarks images change into the docker sub-directory before building + local build_context="." + local dockerfile_flag="-f ${PROJECT_ROOT}/${dockerfile}" + if [[ "$short_name" == benchmarks-* ]]; then + build_context="${PROJECT_ROOT}/.gitlab/benchmarks/docker" + dockerfile_flag="-f ${PROJECT_ROOT}/${dockerfile}" + elif [[ "$dockerfile" == */ ]]; then + # dockerfile is a directory (build context) + build_context="${PROJECT_ROOT}/${dockerfile}" + dockerfile_flag="" + fi + + log_info " Running: docker buildx build --platform=${platform} --tag=${full_tag} ... --push" + + # shellcheck disable=SC2086 + docker buildx build \ + --platform "${platform}" \ + --tag "${full_tag}" \ + "${build_args[@]}" \ + --push \ + --metadata-file "${meta_file}" \ + ${dockerfile_flag} \ + "${build_context}" + + ddsign sign "${full_tag}" --docker-metadata-file "${meta_file}" >&2 + + # Get manifest digest from registry: more reliable than --metadata-file + # which ddsign corrupts for some image types. + docker buildx imagetools inspect "${full_tag}" 2>/dev/null \ + | awk '/Digest:/{print $NF; exit}' +} + +find_def() { + local target_name="$1" + for def in "${IMAGE_DEFS[@]}"; do + local short_name + short_name=$(cut -d'|' -f1 <<< "$def") + if [[ "$short_name" == "$target_name" ]]; then + echo "$def" + return 0 + fi + done + return 1 +} + +main() { + cd "$PROJECT_ROOT" + + # Build list of all valid short names + local all_names=() + for def in "${IMAGE_DEFS[@]}"; do + all_names+=("$(cut -d'|' -f1 <<< "$def")") + done + + # Parse REBUILD_IMAGES (split on comma and/or whitespace) + local selected=() + if [[ -n "${REBUILD_IMAGES:-}" ]]; then + IFS=', ' read -r -a selected <<< "${REBUILD_IMAGES}" + # Validate + for name in "${selected[@]}"; do + if ! find_def "$name" > /dev/null 2>&1; then + log_error "Unknown image name: '${name}'" + usage + exit 1 + fi + done + else + selected=("${all_names[@]}") + fi + + log_info "Images to build: ${selected[*]}" + + if [[ "${DRY_RUN:-}" == "true" ]]; then + log_info "DRY RUN: would build the following images:" + for name in "${selected[@]}"; do + local def + def=$(find_def "$name") + IFS='|' read -r s_name var_name yaml_file tag_suffix dockerfile platform registry_path base_image_var <<< "$def" + local tag="v${CI_PIPELINE_ID:-DRY_RUN}-${tag_suffix}" + log_info " ${name}: ${REGISTRY}/ci/${registry_path}:${tag} (${platform})" + done + exit 0 + fi + + local updates="[]" + local failed=0 + + for name in "${selected[@]}"; do + local def + def=$(find_def "$name") + IFS='|' read -r s_name var_name yaml_file tag_suffix dockerfile platform registry_path base_image_var <<< "$def" + + local base_image="" + if [[ -n "$base_image_var" ]]; then + base_image="${!base_image_var:-}" + if [[ -z "$base_image" ]]; then + log_warn "Base image variable ${base_image_var} is not set for ${name}, building without BASE_IMAGE arg" + fi + fi + + local new_tag="v${CI_PIPELINE_ID}-${tag_suffix}" + log_info "Building ${name} (${new_tag})..." + + local digest + if ! digest=$(build_image "$name" "$new_tag" "$dockerfile" "$platform" "$registry_path" "$base_image"); then + log_error "Build failed for: ${name}" + (( failed++ )) || true + continue + fi + + if [[ -z "$digest" || "$digest" == "null" ]]; then + log_error "Build succeeded but digest is empty for ${name} (ddsign may have corrupted metadata)" + (( failed++ )) || true + continue + fi + + local new_full_ref="${REGISTRY}/ci/${registry_path}:${new_tag}@${digest}" + local current_ref + current_ref=$(get_current_ref "$var_name" "$yaml_file" || echo "") + local current_tag="" + [[ -n "$current_ref" ]] && current_tag=$(extract_tag "$current_ref") + local current_digest + current_digest=$(echo "$current_ref" | grep -oE 'sha256:[a-f0-9]+' || echo "") + local job_url="${CI_JOB_URL:-}" + + updates=$(echo "$updates" | jq \ + --arg var_name "$var_name" \ + --arg yaml_file "$yaml_file" \ + --arg current_tag "$current_tag" \ + --arg current_digest "$current_digest" \ + --arg new_tag "$new_tag" \ + --arg new_digest "$digest" \ + --arg new_full_ref "$new_full_ref" \ + --arg job_url "$job_url" \ + --arg job_name "rebuild-images" \ + '. + [{ + var_name: $var_name, + yaml_file: $yaml_file, + current_tag: $current_tag, + current_digest: $current_digest, + new_tag: $new_tag, + new_digest: $new_digest, + new_full_ref: $new_full_ref, + job_url: $job_url, + job_name: $job_name + }]') + + done + + local update_count + update_count=$(echo "$updates" | jq 'length') + + # Always write updates.json before exiting so the PR job always has valid JSON + echo "$updates" | jq . > "${PROJECT_ROOT}/updates.json" + log_info "Wrote ${update_count} update(s) to updates.json" + + if [[ "$update_count" -eq 0 ]]; then + log_error "No successful builds; nothing to create a PR for" + exit 1 + fi + + exit $failed +} + +main "$@" diff --git a/.gitlab/scripts/stresstests.sh b/.gitlab/scripts/stresstests.sh new file mode 100755 index 000000000..7efb1e6db --- /dev/null +++ b/.gitlab/scripts/stresstests.sh @@ -0,0 +1,37 @@ +#! /bin/bash + +set -eo pipefail # exit on any failure, including mid-pipeline +set -x + +if [ ! -z "${CANCELLED:-}" ]; then + exit 0 +fi + +if [ -z "$TARGET" ]; then + echo "Expecting the TARGET variable to be set" + exit 1 +fi + +HERE=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +REPO_ROOT=$( cd "${HERE}/../.." && pwd ) + +if [ -z "${JAVA_HOME}" ]; then + # workaround for CI when JAVA_HOME is not properly defined + export JAVA_HOME=~/.sdkman/candidates/java/current +fi + +echo "Using Java @ ${JAVA_HOME}" + +source .gitlab/scripts/includes.sh + +function onexit { + mkdir -p "${REPO_ROOT}/stresstest/${TARGET}/logs" + mkdir -p "${REPO_ROOT}/stresstest/${TARGET}/results" + mv "${REPO_ROOT}/ddprof-stresstest/jmh-result.html" "${REPO_ROOT}/stresstest/${TARGET}/results" 2>/dev/null || true + find . -name 'hs_err*' | xargs -I {} cp {} "${REPO_ROOT}/stresstest/${TARGET}/logs" 2>/dev/null || true +} + +trap onexit EXIT + +./gradlew -Pddprof_version="$(get_version)" -Pskip-native -Pskip-tests -Pwith-libs="$(pwd)/libs" assembleDebugJar --max-workers=1 --build-cache --stacktrace --info --no-watch-fs --no-daemon +./gradlew -Pddprof_version="$(get_version)" -Pskip-native -Pwith-libs="$(pwd)/libs" -x gtestDebug runStressTests --max-workers=1 --build-cache --stacktrace --info --no-watch-fs --no-daemon diff --git a/.gitlab/scripts/upload.sh b/.gitlab/scripts/upload.sh new file mode 100755 index 000000000..267abe891 --- /dev/null +++ b/.gitlab/scripts/upload.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present Datadog, Inc. + +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' + +# Load centralized configuration +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +if [ -f "${SCRIPT_DIR}/../../.gitlab/config.env" ]; then + source "${SCRIPT_DIR}/../../.gitlab/config.env" +fi + +# Helper functions +print_help() { + echo "binary artifact upload tool + + -f + path to local file + -n + what to name the object in S3 + -p + prefix for key (e.g., jplib/) + -b + S3 bucket (default: ${S3_BUCKET:-binaries.ddbuild.io}) + -h + print this help and exit +" +} + +# Parameters +BUCKET="${S3_BUCKET:-binaries.ddbuild.io}" +PRE="" +NAME="" +FILE="" +DRY="no" +CMD="aws s3 cp --region ${AWS_REGION:-us-east-1} --sse AES256 --acl bucket-owner-full-control" + +if [ $# -eq 0 ]; then print_help && exit 0; fi +while getopts ":f:n:b:p:dh" arg; do + case $arg in + f) + FILE=${OPTARG} + ;; + n) + NAME=${OPTARG} + ;; + b) + BUCKET=${OPTARG} + ;; + p) + PRE=${OPTARG} + ;; + d) + DRY="yes" + ;; + h) + print_help + exit 0 + ;; + esac +done + +if [ -z "$NAME" ]; then + echo "No name (-n) given, error" + exit -1 +fi + +if [ -z "$FILE" ]; then + echo "No file (-f) given, error" + exit -1 +fi + +if [ ! -f "$FILE" ]; then + echo "File ($FILE) does not exist, error" + exit -1 +fi + +SHA_FILE="/tmp/$(basename ${FILE}).sha" +sha256sum ${FILE} > ${SHA_FILE} + +if [ "yes" == ${DRY} ]; then + echo ${CMD} ${FILE} s3://${BUCKET}/${PRE}/${NAME} + echo ${CMD} ${SHA_FILE} s3://${BUCKET}/${PRE}/${NAME}.sha +else + eval ${CMD} ${FILE} s3://${BUCKET}/${PRE}/${NAME} + eval ${CMD} ${SHA_FILE} s3://${BUCKET}/${PRE}/${NAME}.sha +fi \ No newline at end of file diff --git a/.gitlab/test-apps/ProfilerTestApp.java b/.gitlab/test-apps/ProfilerTestApp.java new file mode 100644 index 000000000..05925f1bf --- /dev/null +++ b/.gitlab/test-apps/ProfilerTestApp.java @@ -0,0 +1,353 @@ +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Profiler Test Application + * + * Generates workloads to validate profiler functionality: + * - CPU activity (ExecutionSample events) + * - Memory allocations (ObjectAllocationSample events) + * - Thread activity (multiple threads for sampling) + * + * Compatible with JDK 8-25. Compiles with plain javac. + * + * Expected JFR Events: + * - jdk.ExecutionSample: CPU profiling samples + * - jdk.ObjectAllocationSample: Allocation events + * - jdk.ThreadAllocationStatistics: Per-thread allocation stats + * + * Usage: + * javac ProfilerTestApp.java + * java ProfilerTestApp [options] + * + * Options: + * --duration Duration to run (default: 30) + * --threads Number of worker threads (default: 4) + * --cpu-iterations CPU work iterations (default: 10000) + * --alloc-rate Allocations per second (default: 1000) + */ +public class ProfilerTestApp { + + // Configuration + private int durationSeconds = 30; + private int threadCount = 4; + private int cpuIterations = 10000; + private int allocationsPerSecond = 1000; + + // Runtime state + private final AtomicBoolean running = new AtomicBoolean(true); + private final AtomicLong totalIterations = new AtomicLong(0); + private final AtomicLong totalAllocations = new AtomicLong(0); + private final List threads = new ArrayList(); + + /** + * Metrics task - monitors system resources and detects CPU changes + */ + private class MetricsTask implements Runnable { + private int lastCpuCount; + + public MetricsTask() { + this.lastCpuCount = Runtime.getRuntime().availableProcessors(); + } + + public void run() { + while (running.get()) { + try { + Thread.sleep(5000); + + int cpus = Runtime.getRuntime().availableProcessors(); + long freeMemory = Runtime.getRuntime().freeMemory(); + long totalMemory = Runtime.getRuntime().totalMemory(); + + // Structured logging for parsing + System.out.printf("[METRICS] timestamp=%d cpus=%d free_mb=%d total_mb=%d%n", + System.currentTimeMillis() / 1000, + cpus, + freeMemory / 1024 / 1024, + totalMemory / 1024 / 1024); + + // Detect CPU changes + if (cpus != lastCpuCount) { + System.err.printf("[WARN] CPU count changed: %d -> %d%n", lastCpuCount, cpus); + lastCpuCount = cpus; + } + } catch (InterruptedException e) { + break; + } + } + } + } + + public static void main(String[] args) throws Exception { + ProfilerTestApp app = new ProfilerTestApp(); + app.parseArgs(args); + app.run(); + } + + private void parseArgs(String[] args) { + for (int i = 0; i < args.length; i++) { + String arg = args[i]; + + if (arg.equals("--duration") && i + 1 < args.length) { + durationSeconds = Integer.parseInt(args[++i]); + } else if (arg.equals("--threads") && i + 1 < args.length) { + threadCount = Integer.parseInt(args[++i]); + } else if (arg.equals("--cpu-iterations") && i + 1 < args.length) { + cpuIterations = Integer.parseInt(args[++i]); + } else if (arg.equals("--alloc-rate") && i + 1 < args.length) { + allocationsPerSecond = Integer.parseInt(args[++i]); + } else if (arg.equals("--help") || arg.equals("-h")) { + printUsage(); + System.exit(0); + } else { + System.err.println("Unknown argument: " + arg); + printUsage(); + System.exit(1); + } + } + } + + private void printUsage() { + System.out.println("Usage: java ProfilerTestApp [options]"); + System.out.println(); + System.out.println("Options:"); + System.out.println(" --duration Duration to run (default: 30)"); + System.out.println(" --threads Number of worker threads (default: 4)"); + System.out.println(" --cpu-iterations CPU work iterations (default: 10000)"); + System.out.println(" --alloc-rate Allocations per second (default: 1000)"); + System.out.println(" --help, -h Show this help message"); + } + + private void run() throws Exception { + System.out.println("=== Profiler Test Application ==="); + System.out.println("Configuration:"); + System.out.println(" Duration: " + durationSeconds + " seconds"); + System.out.println(" Threads: " + threadCount); + System.out.println(" CPU iterations: " + cpuIterations); + System.out.println(" Allocation rate: " + allocationsPerSecond + " per second"); + System.out.println(); + + // Set up shutdown handler + Runtime.getRuntime().addShutdownHook(new Thread() { + public void run() { + shutdown(); + } + }); + + // Start timer thread + Thread timerThread = new Thread(new TimerTask(), "timer-thread"); + timerThread.setDaemon(false); + timerThread.start(); + + // Start metrics thread + Thread metricsThread = new Thread(new MetricsTask(), "metrics-thread"); + metricsThread.setDaemon(true); + metricsThread.start(); + + // Create barrier for synchronized start + final CyclicBarrier startBarrier = new CyclicBarrier(threadCount); + + // Start worker threads + for (int i = 0; i < threadCount; i++) { + Thread workerThread = new Thread(new WorkerTask(i, startBarrier), "worker-" + i); + workerThread.setDaemon(false); + threads.add(workerThread); + workerThread.start(); + } + + System.out.println("Started " + threadCount + " worker threads"); + System.out.println("Test running..."); + System.out.println(); + + // Wait for timer to expire + timerThread.join(); + + // Stop workers + running.set(false); + + // Wait for all workers to finish + for (Thread thread : threads) { + thread.join(5000); // 5 second timeout per thread + } + + // Print summary + System.out.println(); + System.out.println("=== Test Complete ==="); + System.out.println("Total iterations: " + totalIterations.get()); + System.out.println("Total allocations: " + totalAllocations.get()); + System.out.println(); + } + + private void shutdown() { + running.set(false); + } + + /** + * Timer task - runs for specified duration + */ + private class TimerTask implements Runnable { + public void run() { + try { + Thread.sleep(durationSeconds * 1000L); + } catch (InterruptedException e) { + // Expected + } + } + } + + /** + * Worker task - generates CPU and allocation workload + */ + private class WorkerTask implements Runnable { + private final int workerId; + private final CyclicBarrier startBarrier; + private final Random random = new Random(); + + WorkerTask(int workerId, CyclicBarrier startBarrier) { + this.workerId = workerId; + this.startBarrier = startBarrier; + } + + public void run() { + try { + // Wait for all threads to be ready + startBarrier.await(); + + long iterations = 0; + long allocations = 0; + long lastReportTime = System.currentTimeMillis(); + + while (running.get()) { + // CPU-intensive work + performCPUWork(); + iterations++; + + // Memory allocations + performAllocations(); + // Note: Division by 100 assumes ~100 iterations/second (10ms sleep + work time) + // This is an approximation based on typical execution timing + allocations += allocationsPerSecond / 100; // Per iteration + + // Small sleep to avoid spinning + Thread.sleep(10); + + // Periodic reporting + long now = System.currentTimeMillis(); + if (now - lastReportTime >= 5000) { + System.out.println("Worker " + workerId + ": " + iterations + " iterations, " + allocations + " allocations"); + lastReportTime = now; + } + } + + totalIterations.addAndGet(iterations); + totalAllocations.addAndGet(allocations); + + } catch (Exception e) { + System.err.println("Worker " + workerId + " failed: " + e.getMessage()); + e.printStackTrace(); + } + } + + /** + * Perform CPU-intensive work to generate ExecutionSample events + */ + private void performCPUWork() { + // Mix of different CPU operations + + // 1. Math operations + double result = 0; + for (int i = 0; i < cpuIterations; i++) { + result += Math.sqrt(i); + result *= Math.sin(i * 0.001); + } + + // 2. Prime number calculation + int primeCount = 0; + for (int num = 2; num < cpuIterations && num < 1000; num++) { + if (isPrime(num)) { + primeCount++; + } + } + + // 3. String operations + String text = "profiler-test-" + result + "-" + primeCount; + int hash = text.hashCode(); + + // Prevent optimization + if (hash == Integer.MAX_VALUE) { + System.out.println("Unlikely: " + hash); + } + } + + /** + * Check if number is prime (CPU-intensive) + */ + private boolean isPrime(int n) { + if (n <= 1) return false; + if (n <= 3) return true; + if (n % 2 == 0 || n % 3 == 0) return false; + + for (int i = 5; i * i <= n; i += 6) { + if (n % i == 0 || n % (i + 2) == 0) { + return false; + } + } + return true; + } + + /** + * Perform memory allocations to generate ObjectAllocationSample events + */ + private void performAllocations() { + int allocsPerCall = allocationsPerSecond / 100; // Called ~100 times per second + + // 1. String allocations + List strings = new ArrayList(allocsPerCall); + for (int i = 0; i < allocsPerCall / 3; i++) { + strings.add("allocation-worker-" + workerId + "-iteration-" + i + "-" + random.nextInt()); + } + + // 2. Array allocations + List arrays = new ArrayList(allocsPerCall / 3); + for (int i = 0; i < allocsPerCall / 3; i++) { + arrays.add(new byte[1024]); // 1KB allocations + } + + // 3. Object allocations + List objects = new ArrayList(allocsPerCall / 3); + for (int i = 0; i < allocsPerCall / 3; i++) { + objects.add(new WorkerData(workerId, i, System.nanoTime())); + } + + // Prevent optimization - occasionally use the data + if (random.nextInt(1000) == 0) { + System.out.println("Allocated: " + strings.size() + " strings, " + arrays.size() + " arrays, " + objects.size() + " objects"); + } + } + } + + /** + * Data class for allocation testing + */ + private static class WorkerData { + private final int workerId; + private final int iteration; + private final long timestamp; + private final String description; + + WorkerData(int workerId, int iteration, long timestamp) { + this.workerId = workerId; + this.iteration = iteration; + this.timestamp = timestamp; + this.description = "Worker " + workerId + " iteration " + iteration; + } + + public String getDescription() { + return description + " at " + timestamp; + } + } +} diff --git a/.vdiff/annotations.yaml b/.vdiff/annotations.yaml new file mode 100644 index 000000000..acbc8beba --- /dev/null +++ b/.vdiff/annotations.yaml @@ -0,0 +1,29 @@ +- id: 27fc7941-fac1-447c-9668-30f25bab5ad9 + file: build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt + base_ref: v_1.39.0 + head_ref: main + line_start: 255 + line_end: 258 + side: head + context: '' + text: my annotation + author: human + created: 2026-03-21T17:00:35.360904Z + resolved: false + replies: + - author: human + text: yeah + created: 2026-03-21T18:03:34.224545Z +- id: f1e5e0a2-d4f5-44d1-94cd-5e429f450ee2 + file: build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt + base_ref: v_1.39.0 + head_ref: main + line_start: 260 + line_end: 260 + side: head + context: '' + text: interesting stuff + author: human + created: 2026-03-21T17:00:51.447846Z + resolved: false + replies: [] diff --git a/doc/octo-sts-pr-comment-policy.yaml b/doc/octo-sts-pr-comment-policy.yaml new file mode 100644 index 000000000..b0dd15e5e --- /dev/null +++ b/doc/octo-sts-pr-comment-policy.yaml @@ -0,0 +1,19 @@ +# Octo-STS Policy for PR Comments +# +# This file should be placed in the java-profiler repository at: +# .github/chainguard/async-profiler-build.pr-comment.sts.yaml +# +# It allows the java-profiler-build CI to post comments on PRs +# with integration test results. +# +# Reference: https://github.com/chainguard-dev/octo-sts + +issuer: https://gitlab.ddbuild.io + +subject_pattern: project_path:DataDog/apm-reliability/async-profiler-build:* + +permissions: + # Required for posting PR comments (uses Issues API) + issues: write + # Required for looking up PRs by branch + pull_requests: read diff --git a/doc/octo-sts-update-images-policy.yaml b/doc/octo-sts-update-images-policy.yaml new file mode 100644 index 000000000..458479704 --- /dev/null +++ b/doc/octo-sts-update-images-policy.yaml @@ -0,0 +1,37 @@ +# Octo-STS Trust Policy for Image Update PRs +# +# This is a DOCUMENTATION COPY of the policy that should be created in the +# GitHub repository: DataDog/async-profiler-build +# +# Location in GitHub repo: .github/chainguard/update-images.sts.yaml +# +# PURPOSE: +# This policy allows the GitLab CI check-image-updates job to: +# - Push branches to the repository +# - Create pull requests for image updates +# +# BOOTSTRAP: +# On first run, the create-image-update-pr.sh script will automatically +# include this policy file in the PR if it doesn't exist. After merging +# that first PR, subsequent runs will use Octo-STS for authentication. +# +# For the bootstrap run, set GITHUB_TOKEN as a CI variable with a +# personal access token that has 'repo' scope. +# +# DOCUMENTATION: +# - Octo-STS: https://edu.chainguard.dev/open-source/octo-sts/ +# - GitLab OIDC: https://docs.gitlab.com/ee/ci/secrets/id_token_authentication.html + +# GitLab OIDC issuer +issuer: https://gitlab.ddbuild.io + +# Match GitLab CI jobs from the async-profiler-build project +# The scheduled job runs on main, but we allow any branch for manual triggers +subject_pattern: project_path:DataDog/apm-reliability/async-profiler-build:ref_type:branch:ref:.* + +# GitHub API permissions +permissions: + # Required to push branches + contents: write + # Required to create PRs + pull_requests: write diff --git a/repository.datadog.yml b/repository.datadog.yml new file mode 100644 index 000000000..b4ce7a233 --- /dev/null +++ b/repository.datadog.yml @@ -0,0 +1,3 @@ +schema-version: v1 +kind: adms +auto-version-updates-enabled: true diff --git a/utils/README.md b/utils/README.md index 594415d5b..d61b6e722 100644 --- a/utils/README.md +++ b/utils/README.md @@ -2,7 +2,9 @@ This directory contains utility scripts for managing the java-profiler project. -## Release Script +--- + +## Release ### `release.sh` @@ -28,88 +30,22 @@ Triggers the Validated Release workflow using GitHub CLI to create a new release - `--commit `: Specify commit SHA to release (default: interactive selection) - `--help`: Show help message -**Examples:** - -1. **Interactive commit selection** (recommended first step): - ```bash - ./utils/release.sh minor - ``` - This will show a nice arrow-key navigable menu of the last 10 commits. - -2. **Perform an actual minor release** from main branch: - ```bash - ./utils/release.sh minor --no-dry-run - ``` - -3. **Release a specific commit**: - ```bash - ./utils/release.sh patch --commit abc123def456 --no-dry-run - ``` - -4. **Perform a patch release** from a release branch: - ```bash - # Ensure you're on a release/X.Y._ branch - git checkout release/1.35._ - ./utils/release.sh patch --no-dry-run - ``` - -5. **Emergency patch** without tests (use with caution): - ```bash - ./utils/release.sh patch --no-dry-run --skip-tests - ``` - -**Branch Rules:** -- **Major/Minor releases**: Must be run from the `main` branch -- **Patch releases**: Must be run from a `release/X.Y._` branch - -**Interactive Features:** -- **Commit Selection**: If no `--commit` is specified, the script shows an interactive menu of the last 10 commits -- Use ↑/↓ arrow keys to navigate -- Press Enter to select a commit -- Press 'q' to quit -- **Comprehensive Summary**: After execution, displays a detailed summary of all actions performed, including any errors or warnings - -**Release Flow:** -1. Script validates inputs and branch rules -2. Interactive commit selection (or use specified commit) -3. Triggers GitHub Actions "Validated Release" workflow on selected commit -4. Workflow runs pre-release tests (testDebug + testAsan) -5. Workflow creates annotated git tag -6. Tag push triggers GitLab build pipeline -7. GitLab builds multi-platform artifacts -8. GitLab publishes to Maven Central -9. GitHub workflows create release with assets -10. Script displays comprehensive execution summary - -**Monitoring:** - -After triggering the release, monitor progress: - -```bash -# Watch the workflow run in real-time -gh run watch - -# List recent workflow runs -gh run list --workflow=release-validated.yml --limit 5 - -# View in browser -gh workflow view release-validated.yml --web -``` - -**Troubleshooting:** +**Branch rules:** +- **Major/Minor releases**: must be run from `main` +- **Patch releases**: must be run from a `release/X.Y._` branch -If the release fails: +**Release flow:** +1. Validates inputs and branch rules +2. Interactive commit selection (or use `--commit`) +3. Triggers GitHub Actions "Validated Release" workflow +4. Workflow runs pre-release tests, creates annotated git tag +5. Tag push triggers GitLab build pipeline +6. GitLab builds multi-platform artifacts and publishes to Maven Central +7. GitHub workflows create release with assets -1. **Tests fail**: Fix the issues and re-run -2. **Tag already exists**: Delete the tag and retry: - ```bash - git tag -d v_X.Y.Z - git push origin :v_X.Y.Z - ``` -3. **GitLab build fails**: Check GitLab pipeline and retry -4. **Authentication issues**: Run `gh auth login` +--- -## Backport Script +## Backport ### `backport-pr.sh` @@ -118,7 +54,7 @@ Cherry-picks a merged PR onto a release branch, pushes the backport branch, and **Prerequisites:** - [GitHub CLI](https://cli.github.com/) installed and authenticated - [jq](https://jqlang.github.io/jq/) installed -- Clean working tree (no uncommitted changes) +- Clean working tree **Usage:** ```bash @@ -126,58 +62,87 @@ Cherry-picks a merged PR onto a release branch, pushes the backport branch, and ``` **Arguments:** -- ``: Target release branch suffix, e.g. `1.9._` (maps to `release/1.9._`). If omitted, the script presents an interactive picker of the 10 most recent release branches. -- ``: PR number (`420`) or full GitHub URL (`https://github.com/DataDog/java-profiler/pull/420`). -- `--dry-run`: Show what would happen without making any changes. +- ``: Target release branch suffix, e.g. `1.9._` (maps to `release/1.9._`). If omitted, an interactive picker is shown. +- ``: PR number (`420`) or full GitHub URL. +- `--dry-run`: Preview without making changes. **Examples:** ```bash -# Backport PR #420 to release/1.9._ ./utils/backport-pr.sh 1.9._ 420 +./utils/backport-pr.sh 420 # interactive branch selection +./utils/backport-pr.sh --dry-run 1.9._ 420 +``` -# Same, using a URL -./utils/backport-pr.sh 1.9._ https://github.com/DataDog/java-profiler/pull/420 +--- -# Interactive branch selection -./utils/backport-pr.sh 420 +## Testing -# Preview without making changes -./utils/backport-pr.sh --dry-run 1.9._ 420 -``` +### `run-docker-tests.sh` -**Features:** -- Interactive release branch picker when no branch is specified -- Accepts both PR numbers and full GitHub URLs -- Single GitHub API call for all PR metadata -- Warns if the PR is not merged and asks for confirmation -- Handles squashed/garbage-collected commits by falling back to the merge commit -- Detects and cleans up existing backport branches from previous attempts -- Guided recovery on cherry-pick conflicts (does not leave you stranded) -- Comments on the original PR with a link to the backport for traceability -- Colored terminal output (degrades gracefully in non-TTY contexts) -- Restores the original branch on completion or failure +Runs tests in Docker across various OS/libc/JDK combinations, mirroring the CI matrix locally. -## Patch dd-java-agent Script +**Usage:** +```bash +./utils/run-docker-tests.sh [options] + --libc=glibc|musl (default: glibc) + --jdk=8|11|17|21|25|8-j9|... (default: 21) + --arch=x64|aarch64 (default: auto-detect) + --config=debug|release|asan|tsan (default: debug) + --tests="TestPattern" (optional) + --gtest (enable C++ gtests) + --shell (drop to shell instead of running tests) + --mount (mount local repo instead of cloning) + --rebuild (force rebuild of Docker images) +``` ### `patch-dd-java-agent.sh` -Patches a `dd-java-agent.jar` with locally-built ddprof library contents for quick local testing without a full dd-trace-java rebuild. +Patches a `dd-java-agent.jar` with a locally-built ddprof library for quick local testing without a full dd-trace-java rebuild. **Usage:** ```bash -DD_AGENT_JAR=path/to/dd-java-agent.jar DDPROF_JAR=path/to/ddprof.jar ./utils/patch-dd-java-agent.sh +DD_AGENT_JAR=path/to/dd-java-agent.jar DDPROF_JAR=path/to/ddprof.jar \ + ./utils/patch-dd-java-agent.sh ``` -## Cherry-Pick Scripts +--- + +## Upstream Tracking + +See [README_UPSTREAM_TRACKER.md](README_UPSTREAM_TRACKER.md) for full documentation. + +### `check_upstream_changes.sh` + +Wrapper to compare local files against a given upstream async-profiler commit and produce a change report. + +### `track_upstream_changes.sh` + +Core change detection and report generation logic. + +### `generate_tracked_files.sh` -### `cherry.sh` +Identifies which local files should be tracked against upstream (based on async-profiler copyright headers). -Helper script for cherry-picking commits from upstream async-profiler. +### `check_contribution_candidates.sh` -### `init_cherypick_repo.sh` +Identifies divergences from upstream async-profiler that could be contributed back. -Initializes the repository for cherry-picking from upstream. +### `find_contribution_candidates.sh` + +Core diff analysis and report generation for contribution candidate detection. --- -For more information about the release process, see `.github/workflows/release-validated.yml`. +## CI / Ops + +### `update-sonatype-credentials.sh` + +Updates the Sonatype (Maven Central) OSSRH credentials stored in AWS SSM, used by the CI publish pipeline. + +**Prerequisites:** +- AWS CLI authenticated with `ssm:PutParameter` permission + +**Usage:** +```bash +./utils/update-sonatype-credentials.sh +``` diff --git a/utils/update-sonatype-credentials.sh b/utils/update-sonatype-credentials.sh new file mode 100755 index 000000000..15a509ba5 --- /dev/null +++ b/utils/update-sonatype-credentials.sh @@ -0,0 +1,52 @@ +#!/bin/bash +set -euo pipefail + +# Update Sonatype (Maven Central) credentials in AWS SSM for java-profiler CI. +# +# Usage: update-sonatype-credentials.sh + +AWS_REGION=us-east-1 +SSM_PREFIX=ci.java-profiler +AWS_VAULT_PROFILE=sso-build-stable-developer + +usage() { + echo "Usage: $0 " + echo "" + echo "Updates Sonatype OSSRH credentials in AWS SSM:" + echo " ${SSM_PREFIX}.sonatype_token_user" + echo " ${SSM_PREFIX}.sonatype_token" + exit 1 +} + +if [ $# -ne 2 ]; then + usage +fi + +USERNAME="$1" +TOKEN="$2" + +aws-vault login sso-build-stable-developer + +# Verify AWS authentication +if ! aws-vault exec "${AWS_VAULT_PROFILE}" -- aws sts get-caller-identity --query "Arn" --output text 2>/dev/null; then + echo "ERROR: Not authenticated with AWS. Run 'aws-vault login ${AWS_VAULT_PROFILE}' and retry." + exit 1 +fi + +echo "Updating ${SSM_PREFIX}.sonatype_token_user ..." +aws-vault exec "${AWS_VAULT_PROFILE}" -- aws ssm put-parameter \ + --region "${AWS_REGION}" \ + --name "${SSM_PREFIX}.sonatype_token_user" \ + --value "${USERNAME}" \ + --type SecureString \ + --overwrite + +echo "Updating ${SSM_PREFIX}.sonatype_token ..." +aws-vault exec "${AWS_VAULT_PROFILE}" -- aws ssm put-parameter \ + --region "${AWS_REGION}" \ + --name "${SSM_PREFIX}.sonatype_token" \ + --value "${TOKEN}" \ + --type SecureString \ + --overwrite + +echo "Done."