From f3802e52a48de6251f3afc17b10976dd5d08187a Mon Sep 17 00:00:00 2001 From: Andrew Hughes Date: Wed, 20 Mar 2024 15:49:40 +0000 Subject: [PATCH] Make source tarballs reproducible and support construction in a temporary directory. - generate_source_tarball.sh: Add WITH_TEMP environment variable - generate_source_tarball.sh: Multithread xz on all available cores - generate_source_tarball.sh: Add OPENJDK_LATEST environment variable - generate_source_tarball.sh: Update comment about tarball naming - generate_source_tarball.sh: Reformat comment header - generate_source_tarball.sh: Reformat and update help output - generate_source_tarball.sh: Do a shallow clone, for speed - generate_source_tarball.sh: Append -ea designator when required - generate_source_tarball.sh: Eliminate some removal prompting - generate_source_tarball.sh: Make tarball reproducible - generate_source_tarball.sh: Prefix temporary directory with temp- - generate_source_tarball.sh: Remove temporary directory exit conditions - generate_source_tarball.sh: Fix -ea logic to add dash Related: RHEL-30946 --- generate_source_tarball.sh | 157 +++++++++++++++++++++++++++---------- java-21-openjdk.spec | 13 +++ sources | 2 +- 3 files changed, 128 insertions(+), 44 deletions(-) diff --git a/generate_source_tarball.sh b/generate_source_tarball.sh index 7f22f4e..691f635 100755 --- a/generate_source_tarball.sh +++ b/generate_source_tarball.sh @@ -1,51 +1,99 @@ #!/bin/bash # Generates the 'source tarball' for JDK projects. # -# Example: -# When used from local repo set REPO_ROOT pointing to file:// with your repo -# If your local repo follows upstream forests conventions, it may be enough to set OPENJDK_URL +# Example 1: +# When used from local repo set REPO_ROOT pointing to file:// with your repo. +# If your local repo follows upstream forests conventions, it may be enough to +# set OPENJDK_URL. +# +# Example 2: +# This will read the OpenJDK feature version from the spec file, then create a +# tarball from the most recent tag for that version in the upstream Git +# repository. +# +# $ OPENJDK_LATEST=1 ./generate_source_tarball.sh +# [...] +# Tarball is: temp-generated-source-tarball-ujD/openjdk-17.0.10+6-ea.tar.xz +# +# Unless you use OPENJDK_LATEST, you have to set PROJECT_NAME, REPO_NAME and +# VERSION, e.g.: # -# In any case you have to set PROJECT_NAME REPO_NAME and VERSION. eg: # PROJECT_NAME=openjdk # REPO_NAME=jdk17u # VERSION=jdk-17.0.3+5 -# or to eg prepare systemtap: -# icedtea7's jstack and other tapsets +# +# or to e.g., prepare systemtap, icedtea7's jstack and other tapsets: +# # VERSION=6327cf1cea9e # REPO_NAME=icedtea7-2.6 # PROJECT_NAME=release # OPENJDK_URL=http://icedtea.classpath.org/hg/ # TO_COMPRESS="*/tapset" # -# They are used to create correct name and are used in construction of sources url (unless REPO_ROOT is set) - -# This script creates a single source tarball out of the repository -# based on the given tag and removes code not allowed in fedora/rhel. For -# consistency, the source tarball will always contain 'openjdk' as the top -# level folder, name is created, based on parameter +# They are used to create correct name and are used in construction of sources +# URL (unless REPO_ROOT is set). # +# This script creates a single source tarball out of the repository based on the +# given tag and removes code not allowed in Fedora/RHEL. set -e OPENJDK_URL_DEFAULT=https://github.com COMPRESSION_DEFAULT=xz -if [ "x$1" = "xhelp" ] ; then - echo -e "Behaviour may be specified by setting the following variables:\n" - echo "VERSION - the version of the specified OpenJDK project" - echo "PROJECT_NAME -- the name of the OpenJDK project being archived (optional; only needed by defaults)" - echo "REPO_NAME - the name of the OpenJDK repository (optional; only needed by defaults)" - echo "OPENJDK_URL - the URL to retrieve code from (optional; defaults to ${OPENJDK_URL_DEFAULT})" - echo "COMPRESSION - the compression type to use (optional; defaults to ${COMPRESSION_DEFAULT})" - echo "FILE_NAME_ROOT - name of the archive, minus extensions (optional; defaults to PROJECT_NAME-REPO_NAME-VERSION)" - echo "REPO_ROOT - the location of the Git repository to archive (optional; defaults to OPENJDK_URL/PROJECT_NAME/REPO_NAME.git)" - echo "TO_COMPRESS - what part of clone to pack (default is ${VERSION})" - echo "BOOT_JDK - the bootstrap JDK to satisfy the configure run" +if [ "$1" = "help" ] ; then + echo "Behaviour may be specified by setting the following variables:" + echo + echo "VERSION - the version of the specified OpenJDK project" + echo " (required unless OPENJDK_LATEST is set)" + echo "PROJECT_NAME - the name of the OpenJDK project being archived" + echo " (needed to compute REPO_ROOT and/or" + echo " FILE_NAME_ROOT automatically;" + echo " optional if they are set explicitly)" + echo "REPO_NAME - the name of the OpenJDK repository" + echo " (needed to compute REPO_ROOT automatically;" + echo " optional if REPO_ROOT is set explicitly)" + echo "OPENJDK_URL - the URL to retrieve code from" + echo " (defaults to ${OPENJDK_URL_DEFAULT})" + echo "COMPRESSION - the compression type to use" + echo " (defaults to ${COMPRESSION_DEFAULT})" + echo "FILE_NAME_ROOT - name of the archive, minus extensions" + echo " (defaults to PROJECT_NAME-VERSION)" + echo "REPO_ROOT - the location of the Git repository to archive" + echo " (defaults to OPENJDK_URL/PROJECT_NAME/REPO_NAME.git)" + echo "TO_COMPRESS - what part of clone to pack" + echo " (defaults to ${VERSION})" + echo "BOOT_JDK - the bootstrap JDK to satisfy the configure run" + echo " (defaults to packaged JDK version)" + echo "WITH_TEMP - run in a temporary directory" + echo " (defaults to disabled)" + echo "OPENJDK_LATEST - deduce VERSION from most recent upstream tag" + echo " (implies WITH_TEMP, computes everything else" + echo " automatically; Note: accesses network to read" + echo " tag list from remote Git repository)" exit 1; fi +if [ "$OPENJDK_LATEST" != "" ] ; then + FEATURE_VERSION=$(echo '%featurever' \ + | rpmspec --shell ./*.spec 2>/dev/null \ + | grep --after-context 1 featurever \ + | tail --lines 1) + PROJECT_NAME=openjdk + REPO_NAME=jdk"${FEATURE_VERSION}"u + VERSION=$(git ls-remote --tags --refs --sort=-version:refname \ + "${OPENJDK_URL_DEFAULT}/${PROJECT_NAME}/${REPO_NAME}.git" \ + "jdk-${FEATURE_VERSION}*" \ + | head --lines 1 | cut --characters 52-) + FILE_NAME_ROOT=open${VERSION} + WITH_TEMP=1 +fi -if [ "x$VERSION" = "x" ] ; then +if [ "$WITH_TEMP" != "" ] ; then + pushd "$(mktemp --directory temp-generated-source-tarball-XXX)" +fi + +if [ "$VERSION" = "" ] ; then echo "No VERSION specified" exit 2 fi @@ -131,53 +179,76 @@ echo -e "\tREPO_ROOT: ${REPO_ROOT}" echo -e "\tTO_COMPRESS: ${TO_COMPRESS}" echo -e "\tBOOT_JDK: ${BOOT_JDK}" -if [ -d ${FILE_NAME_ROOT} ] ; then +if [ -d "${FILE_NAME_ROOT}" ] ; then echo "exists exists exists exists exists exists exists " echo "reusing reusing reusing reusing reusing reusing " - echo ${FILE_NAME_ROOT} + echo "${FILE_NAME_ROOT}" + STAT_TIME="$(stat --format=%Y "${FILE_NAME_ROOT}")" + TAR_TIME="$(date --date=@"${STAT_TIME}" --iso-8601=seconds)" else mkdir "${FILE_NAME_ROOT}" pushd "${FILE_NAME_ROOT}" echo "Cloning ${VERSION} root repository from ${REPO_ROOT}" - git clone -b ${VERSION} ${REPO_ROOT} ${VERSION} + git clone --depth=1 -b "${VERSION}" "${REPO_ROOT}" "${VERSION}" + pushd "${VERSION}" + TAR_TIME="$(git log --max-count 1 --format=%cI)" + popd popd fi pushd "${FILE_NAME_ROOT}" - - # Generate .src-rev so build has knowledge of the revision the tarball was created from + # Generate .src-rev so build has knowledge of the revision the tarball was + # created from mkdir build pushd build - sh ${PWD}/../${VERSION}/configure --with-boot-jdk=${BOOT_JDK} + sh "${PWD}"/../"${VERSION}"/configure --with-boot-jdk="${BOOT_JDK}" make store-source-revision popd rm -rf build # Remove commit checks - echo "Removing $(find ${VERSION} -name '.jcheck' -print)" - find ${VERSION} -name '.jcheck' -print0 | xargs -0 rm -r + echo "Removing $(find "${VERSION}" -name '.jcheck' -print)" + find "${VERSION}" -name '.jcheck' -print0 | xargs -0 rm -r # Remove history and GHA echo "find ${VERSION} -name '.hgtags'" - find ${VERSION} -name '.hgtags' -exec rm -v '{}' '+' + find "${VERSION}" -name '.hgtags' -exec rm -v '{}' '+' echo "find ${VERSION} -name '.hgignore'" - find ${VERSION} -name '.hgignore' -exec rm -v '{}' '+' + find "${VERSION}" -name '.hgignore' -exec rm -v '{}' '+' echo "find ${VERSION} -name '.gitattributes'" - find ${VERSION} -name '.gitattributes' -exec rm -v '{}' '+' + find "${VERSION}" -name '.gitattributes' -exec rm -v '{}' '+' echo "find ${VERSION} -name '.gitignore'" - find ${VERSION} -name '.gitignore' -exec rm -v '{}' '+' + find "${VERSION}" -name '.gitignore' -exec rm -v '{}' '+' + # Work around some Git objects not having write permissions. + echo "chmod --recursive u+w ${VERSION}/.git" + chmod --recursive u+w "${VERSION}"/.git echo "find ${VERSION} -name '.git'" - find ${VERSION} -name '.git' -exec rm -rv '{}' '+' + find "${VERSION}" -name '.git' -exec rm -rv '{}' '+' echo "find ${VERSION} -name '.github'" - find ${VERSION} -name '.github' -exec rm -rv '{}' '+' + find "${VERSION}" -name '.github' -exec rm -rv '{}' '+' echo "Compressing remaining forest" - if [ "X$COMPRESSION" = "Xxz" ] ; then + if [ "$COMPRESSION" = "xz" ] ; then SWITCH=cJf else SWITCH=czf fi - TARBALL_NAME=${FILE_NAME_ROOT}.tar.${COMPRESSION} - tar --exclude-vcs -$SWITCH ${TARBALL_NAME} $TO_COMPRESS - mv ${TARBALL_NAME} .. + EA_PART="$(awk -F= \ + '/^DEFAULT_PROMOTED_VERSION_PRE/ { if ($2) print "-"$2 }' \ + "${VERSION}"/make/conf/version-numbers.conf)" + TARBALL_NAME=${FILE_NAME_ROOT}${EA_PART}.tar.${COMPRESSION} + XZ_OPT=${XZ_OPT-"-T0"} \ + tar --mtime="${TAR_TIME}" --owner=root --group=root --sort=name \ + --exclude-vcs -$SWITCH "${TARBALL_NAME}" "${TO_COMPRESS}" + mv "${TARBALL_NAME}" .. popd -echo "Done. You may want to remove the uncompressed version - $FILE_NAME_ROOT." +if [ "$WITH_TEMP" != "" ] ; then + echo "Tarball is: $(realpath --relative-to=.. .)/${TARBALL_NAME}" + popd +else + echo -n "Done. You may want to remove the uncompressed version" + echo " - $FILE_NAME_ROOT." +fi + +# Local Variables: +# fill-column: 80 +# End: diff --git a/java-21-openjdk.spec b/java-21-openjdk.spec index 8f00192..d483347 100644 --- a/java-21-openjdk.spec +++ b/java-21-openjdk.spec @@ -2492,6 +2492,19 @@ cjc.mainProgram(args) %changelog * Sun Apr 14 2024 Thomas Fitzsimmons - 1:21.0.3.0.1-0.2.ea - Invoke xz in multi-threaded mode +- generate_source_tarball.sh: Add WITH_TEMP environment variable +- generate_source_tarball.sh: Multithread xz on all available cores +- generate_source_tarball.sh: Add OPENJDK_LATEST environment variable +- generate_source_tarball.sh: Update comment about tarball naming +- generate_source_tarball.sh: Reformat comment header +- generate_source_tarball.sh: Reformat and update help output +- generate_source_tarball.sh: Do a shallow clone, for speed +- generate_source_tarball.sh: Append -ea designator when required +- generate_source_tarball.sh: Eliminate some removal prompting +- generate_source_tarball.sh: Make tarball reproducible +- generate_source_tarball.sh: Prefix temporary directory with temp- +- generate_source_tarball.sh: Remove temporary directory exit conditions +- generate_source_tarball.sh: Fix -ea logic to add dash - Related: RHEL-30946 * Sun Apr 14 2024 Andrew Hughes - 1:21.0.3.0.1-0.2.ea diff --git a/sources b/sources index 710e80b..1d840be 100644 --- a/sources +++ b/sources @@ -1,2 +1,2 @@ SHA512 (tapsets-icedtea-6.0.0pre00-c848b93a8598.tar.xz) = 97d026212363b3c83f6a04100ad7f6fdde833d16579717f8756e2b8c2eb70e144a41a330cb9ccde9c3badd37a2d54fdf4650a950ec21d8b686d545ecb2a64d30 -SHA512 (openjdk-21.0.3+1-ea.tar.xz) = 5925387ebca2b05fd053cf6633e5469585e68eda5bcb0e9d8f51c46233b73f58db2641838b7aae79a81b6257ecafb1b89a223c0e8619b12414df83b35e15ccff +SHA512 (openjdk-21.0.3+1-ea.tar.xz) = 5b1eb2fc253787a22857c4a5b644f2eeec4774d860c9d90d76f8daf4a40299941fd2865baee4811a16e7c80b4d7dadd1413514fb68d218431f94445a89ea1b75