Skip to content

Commit a08f964

Browse files
chrajeshbabuchrajeshbabu
committed
HBASE-27949 [JDK17] Add JDK17 compilation and unit test support to ni… (#5689)
HBASE-27949 [JDK17] Add JDK17 compilation and unit test support to nightly job HBASE-26372 [JDK17] Jenkins build support
1 parent 102b1a5 commit a08f964

3 files changed

Lines changed: 274 additions & 0 deletions

File tree

‎dev-support/Jenkinsfile‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ pipeline {
3737
OUTPUT_DIR_RELATIVE_JDK8_HADOOP2 = 'output-jdk8-hadoop2'
3838
OUTPUT_DIR_RELATIVE_JDK8_HADOOP3 = 'output-jdk8-hadoop3'
3939
OUTPUT_DIR_RELATIVE_JDK11_HADOOP3 = 'output-jdk11-hadoop3'
40+
OUTPUT_DIR_RELATIVE_JDK17_HADOOP3 = 'output-jdk17-hadoop3'
4041

4142
PROJECT = 'hbase'
4243
PROJECT_PERSONALITY = 'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
@@ -188,6 +189,7 @@ pipeline {
188189
stash name: 'jdk8-hadoop2-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK8_HADOOP2}/doesn't-match"
189190
stash name: 'jdk8-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK8_HADOOP3}/doesn't-match"
190191
stash name: 'jdk11-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}/doesn't-match"
192+
stash name: 'jdk17-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK17_HADOOP3}/doesn't-match"
191193
stash name: 'srctarball-result', allowEmpty: true, includes: "output-srctarball/doesn't-match"
192194
}
193195
}
@@ -628,6 +630,123 @@ pipeline {
628630
}
629631
}
630632
}
633+
634+
stage ('yetus jdk17 hadoop3 checks') {
635+
when {
636+
anyOf {
637+
branch 'master';branch 'branch-3'
638+
}
639+
}
640+
agent {
641+
node {
642+
label 'hbase'
643+
}
644+
}
645+
environment {
646+
BASEDIR = "${env.WORKSPACE}/component"
647+
TESTS = "${env.DEEP_CHECKS}"
648+
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_JDK17_HADOOP3}"
649+
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_JDK17_HADOOP3}"
650+
SET_JAVA_HOME = "/usr/lib/jvm/java-17"
651+
// Activates hadoop 3.0 profile in maven runs.
652+
HADOOP_PROFILE = '3.0'
653+
SKIP_ERRORPRONE = true
654+
}
655+
steps {
656+
// Must do prior to anything else, since if one of them timesout we'll stash the commentfile
657+
sh '''#!/usr/bin/env bash
658+
set -e
659+
rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
660+
echo '(x) {color:red}-1 jdk17 hadoop3 checks{color}' >"${OUTPUT_DIR}/commentfile"
661+
echo "-- Something went wrong running this stage, please [check relevant console output|${BUILD_URL}/console]." >> "${OUTPUT_DIR}/commentfile"
662+
'''
663+
unstash 'yetus'
664+
dir('component') {
665+
checkout scm
666+
}
667+
sh '''#!/usr/bin/env bash
668+
set -e
669+
rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
670+
"${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
671+
echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'"
672+
ls -lh "${OUTPUT_DIR_RELATIVE}/machine"
673+
'''
674+
script {
675+
def ret = sh(
676+
returnStatus: true,
677+
script: '''#!/usr/bin/env bash
678+
set -e
679+
declare -i status=0
680+
if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then
681+
echo '(/) {color:green}+1 jdk17 hadoop3 checks{color}' > "${OUTPUT_DIR}/commentfile"
682+
else
683+
echo '(x) {color:red}-1 jdk17 hadoop3 checks{color}' > "${OUTPUT_DIR}/commentfile"
684+
status=1
685+
fi
686+
echo "-- For more information [see jdk17 report|${BUILD_URL}JDK17_20Nightly_20Build_20Report_20_28Hadoop3_29/]" >> "${OUTPUT_DIR}/commentfile"
687+
exit "${status}"
688+
'''
689+
)
690+
if (ret != 0) {
691+
// mark the build as UNSTABLE instead of FAILURE, to avoid skipping the later publish of
692+
// test output. See HBASE-26339 for more details.
693+
currentBuild.result = 'UNSTABLE'
694+
}
695+
}
696+
}
697+
post {
698+
always {
699+
stash name: 'jdk17-hadoop3-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile"
700+
junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults: true
701+
// zip surefire reports.
702+
sh '''#!/bin/bash -e
703+
if [ -d "${OUTPUT_DIR}/archiver" ]; then
704+
count=$(find "${OUTPUT_DIR}/archiver" -type f | wc -l)
705+
if [[ 0 -ne ${count} ]]; then
706+
echo "zipping ${count} archived files"
707+
zip -q -m -r "${OUTPUT_DIR}/test_logs.zip" "${OUTPUT_DIR}/archiver"
708+
else
709+
echo "No archived files, skipping compressing."
710+
fi
711+
else
712+
echo "No archiver directory, skipping compressing."
713+
fi
714+
'''
715+
sshPublisher(publishers: [
716+
sshPublisherDesc(configName: 'Nightlies',
717+
transfers: [
718+
sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
719+
sourceFiles: "${env.OUTPUT_DIR_RELATIVE}/test_logs.zip"
720+
)
721+
]
722+
)
723+
])
724+
// remove the big test logs zip file, store the nightlies url in test_logs.html
725+
sh '''#!/bin/bash -e
726+
if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
727+
echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
728+
rm -rf "${OUTPUT_DIR}/test_logs.zip"
729+
python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
730+
else
731+
echo "No test_logs.zip, skipping"
732+
fi
733+
'''
734+
// Has to be relative to WORKSPACE.
735+
archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/*"
736+
archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*"
737+
publishHTML target: [
738+
allowMissing : true,
739+
keepAll : true,
740+
alwaysLinkToLastBuild: true,
741+
// Has to be relative to WORKSPACE.
742+
reportDir : "${env.OUTPUT_DIR_RELATIVE}",
743+
reportFiles : 'console-report.html',
744+
reportName : 'JDK17 Nightly Build Report (Hadoop3)'
745+
]
746+
}
747+
}
748+
}
749+
631750
// This is meant to mimic what a release manager will do to create RCs.
632751
// See http://hbase.apache.org/book.html#maven.release
633752
// TODO (HBASE-23870): replace this with invocation of the release tool

‎dev-support/Jenkinsfile_GitHub‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ pipeline {
5555
WORKDIR_REL_GENERAL_CHECK = 'yetus-general-check'
5656
WORKDIR_REL_JDK8_HADOOP3_CHECK = 'yetus-jdk8-hadoop3-check'
5757
WORKDIR_REL_JDK11_HADOOP3_CHECK = 'yetus-jdk11-hadoop3-check'
58+
WORKDIR_REL_JDK17_HADOOP3_CHECK = 'yetus-jdk17-hadoop3-check'
5859
ASF_NIGHTLIES = 'https://nightlies.apache.org'
5960
ASF_NIGHTLIES_BASE_ORI = "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}"
6061
ASF_NIGHTLIES_BASE = "${ASF_NIGHTLIES_BASE_ORI.replaceAll(' ', '%20')}"
@@ -470,6 +471,145 @@ pipeline {
470471
}
471472
}
472473
}
474+
stage ('yetus jdk17 hadoop3 checks') {
475+
agent {
476+
node {
477+
label 'hbase'
478+
}
479+
}
480+
environment {
481+
// customized per parallel stage
482+
PLUGINS = "${JDK_SPECIFIC_PLUGINS}"
483+
SET_JAVA_HOME = '/usr/lib/jvm/java-17'
484+
WORKDIR_REL = "${WORKDIR_REL_JDK17_HADOOP3_CHECK}"
485+
// identical for all parallel stages
486+
WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
487+
YETUSDIR = "${WORKDIR}/${YETUS_REL}"
488+
SOURCEDIR = "${WORKDIR}/${SRC_REL}"
489+
PATCHDIR = "${WORKDIR}/${PATCH_REL}"
490+
BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
491+
DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
492+
YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
493+
SKIP_ERRORPRONE = true
494+
}
495+
when {
496+
allOf {
497+
// this will return true if the pipeline is building a change request, such as a GitHub pull request.
498+
changeRequest()
499+
expression { env.CHANGE_TARGET in ['master', 'branch-3'] }
500+
}
501+
}
502+
steps {
503+
dir("${SOURCEDIR}") {
504+
checkout scm
505+
}
506+
dir("${YETUSDIR}") {
507+
checkout([
508+
$class : 'GitSCM',
509+
branches : [[name: "${YETUS_VERSION}"]],
510+
userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
511+
)
512+
}
513+
dir("${WORKDIR}") {
514+
withCredentials([
515+
usernamePassword(
516+
credentialsId: 'apache-hbase-at-github.com',
517+
passwordVariable: 'GITHUB_PASSWORD',
518+
usernameVariable: 'GITHUB_USER'
519+
)]) {
520+
script {
521+
def ret = sh(
522+
label: 'test-patch',
523+
returnStatus: true,
524+
script: '''#!/bin/bash -e
525+
hostname -a ; pwd ; ls -la
526+
printenv 2>&1 | sort
527+
echo "[INFO] Launching Yetus via ${YETUS_DRIVER}"
528+
"${YETUS_DRIVER}"
529+
'''
530+
)
531+
if (ret != 0) {
532+
// mark the build as UNSTABLE instead of FAILURE, to avoid skipping the later publish of
533+
// test output. See HBASE-26339 for more details.
534+
currentBuild.result = 'UNSTABLE'
535+
}
536+
}
537+
}
538+
}
539+
}
540+
post {
541+
always {
542+
junit testResults: "${WORKDIR_REL}/${SRC_REL}/**/target/**/TEST-*.xml",
543+
allowEmptyResults: true, skipPublishingChecks: true
544+
sh label: 'zip surefire reports', script: '''#!/bin/bash -e
545+
if [ -d "${PATCHDIR}/archiver" ]; then
546+
count=$(find "${PATCHDIR}/archiver" -type f | wc -l)
547+
if [[ 0 -ne ${count} ]]; then
548+
echo "zipping ${count} archived files"
549+
zip -q -m -r "${PATCHDIR}/test_logs.zip" "${PATCHDIR}/archiver"
550+
else
551+
echo "No archived files, skipping compressing."
552+
fi
553+
else
554+
echo "No archiver directory, skipping compressing."
555+
fi
556+
'''
557+
sshPublisher(publishers: [
558+
sshPublisherDesc(configName: 'Nightlies',
559+
transfers: [
560+
sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
561+
sourceFiles: "${env.WORKDIR_REL}/${env.PATCH_REL}/test_logs.zip"
562+
)
563+
]
564+
)
565+
])
566+
// remove the big test logs zip file, store the nightlies url in test_logs.txt
567+
sh '''#!/bin/bash -e
568+
if [ -f "${PATCHDIR}/test_logs.zip" ]; then
569+
echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
570+
rm -rf "${PATCHDIR}/test_logs.zip"
571+
python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
572+
else
573+
echo "No test_logs.zip, skipping"
574+
fi
575+
'''
576+
// Has to be relative to WORKSPACE.
577+
archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit"
578+
archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
579+
publishHTML target: [
580+
allowMissing: true,
581+
keepAll: true,
582+
alwaysLinkToLastBuild: true,
583+
// Has to be relative to WORKSPACE
584+
reportDir: "${WORKDIR_REL}/${PATCH_REL}",
585+
reportFiles: 'report.html',
586+
reportName: 'PR JDK17 Hadoop3 Check Report'
587+
]
588+
}
589+
// Jenkins pipeline jobs fill slaves on PRs without this :(
590+
cleanup() {
591+
script {
592+
sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
593+
# See YETUS-764
594+
if [ -f "${PATCHDIR}/pidfile.txt" ]; then
595+
echo "test-patch process appears to still be running: killing"
596+
kill `cat "${PATCHDIR}/pidfile.txt"` || true
597+
sleep 10
598+
fi
599+
if [ -f "${PATCHDIR}/cidfile.txt" ]; then
600+
echo "test-patch container appears to still be running: killing"
601+
docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
602+
fi
603+
# See HADOOP-13951
604+
chmod -R u+rxw "${WORKSPACE}"
605+
'''
606+
dir ("${WORKDIR}") {
607+
deleteDir()
608+
}
609+
}
610+
}
611+
}
612+
}
473613
}
474614
}
475615
}

‎dev-support/docker/Dockerfile‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,13 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
110110
RUN curl --location --fail --silent --show-error --output /tmp/adoptopenjdk11.tar.gz "${OPENJDK11_URL}" && \
111111
echo "${OPENJDK11_SHA256} */tmp/adoptopenjdk11.tar.gz" | sha256sum -c -
112112

113+
FROM base_image AS openjdk17_download_image
114+
ENV OPENJDK17_URL 'https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.10%2B7/OpenJDK17U-jdk_x64_linux_hotspot_17.0.10_7.tar.gz'
115+
ENV OPENJDK17_SHA256 'a8fd07e1e97352e97e330beb20f1c6b351ba064ca7878e974c7d68b8a5c1b378'
116+
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
117+
RUN curl --location --fail --silent --show-error --output /tmp/adoptopenjdk17.tar.gz "${OPENJDK17_URL}" && \
118+
echo "${OPENJDK17_SHA256} */tmp/adoptopenjdk17.tar.gz" | sha256sum -c -
119+
113120
##
114121
# build the final image
115122
#
@@ -158,6 +165,14 @@ RUN mkdir -p /usr/lib/jvm && \
158165
ln -s /usr/lib/jvm/java-11-adoptopenjdk /usr/lib/jvm/java-11 && \
159166
rm /tmp/adoptopenjdk11.tar.gz
160167

168+
# hadolint ignore=DL3010
169+
COPY --from=openjdk17_download_image /tmp/adoptopenjdk17.tar.gz /tmp/adoptopenjdk17.tar.gz
170+
RUN mkdir -p /usr/lib/jvm && \
171+
tar xzf /tmp/adoptopenjdk17.tar.gz -C /usr/lib/jvm && \
172+
ln -s "/usr/lib/jvm/$(basename "$(tar -tf /tmp/adoptopenjdk17.tar.gz | head -n1)")" /usr/lib/jvm/java-17-adoptopenjdk && \
173+
ln -s /usr/lib/jvm/java-17-adoptopenjdk /usr/lib/jvm/java-17 && \
174+
rm /tmp/adoptopenjdk17.tar.gz
175+
161176
# configure default environment for Yetus. Yetus in dockermode seems to require
162177
# these values to be specified here; the various --foo-path flags do not
163178
# propigate as expected, while these are honored.

0 commit comments

Comments
 (0)