Make source tarballs reproducible and support construction in a temporary directory.

- generate_source_tarball.sh: Add WITH_TEMP environment variable
- generate_source_tarball.sh: Multithread xz on all available cores
- generate_source_tarball.sh: Add OPENJDK_LATEST environment variable
- generate_source_tarball.sh: Update comment about tarball naming
- generate_source_tarball.sh: Reformat comment header
- generate_source_tarball.sh: Reformat and update help output
- generate_source_tarball.sh: Do a shallow clone, for speed
- generate_source_tarball.sh: Append -ea designator when required
- generate_source_tarball.sh: Eliminate some removal prompting
- generate_source_tarball.sh: Make tarball reproducible
- generate_source_tarball.sh: Prefix temporary directory with temp-
- generate_source_tarball.sh: Remove temporary directory exit conditions
- generate_source_tarball.sh: Fix -ea logic to add dash

Related: RHEL-52723

generate_source_tarball.sh

@@ -1,51 +1,99 @@
 #!/bin/bash
 # Generates the 'source tarball' for JDK projects.
 #
-# Example:
+# Example 1:
-# When used from local repo set REPO_ROOT pointing to file:// with your repo
+# When used from local repo set REPO_ROOT pointing to file:// with your repo.
-# If your local repo follows upstream forests conventions, it may be enough to set OPENJDK_URL
+# If your local repo follows upstream forests conventions, it may be enough to
+# set OPENJDK_URL.
+#
+# Example 2:
+# This will read the OpenJDK feature version from the spec file, then create a
+# tarball from the most recent tag for that version in the upstream Git
+# repository.
+#
+# $ OPENJDK_LATEST=1 ./generate_source_tarball.sh
+# [...]
+# Tarball is: temp-generated-source-tarball-ujD/openjdk-17.0.10+6-ea.tar.xz
+#
+# Unless you use OPENJDK_LATEST, you have to set PROJECT_NAME, REPO_NAME and
+# VERSION, e.g.:
 #
-# In any case you have to set PROJECT_NAME REPO_NAME and VERSION. eg:
 # PROJECT_NAME=openjdk
 # REPO_NAME=jdk17u
 # VERSION=jdk-17.0.3+5
-# or to eg prepare systemtap:
+#
-# icedtea7's jstack and other tapsets
+# or to e.g., prepare systemtap, icedtea7's jstack and other tapsets:
+#
 # VERSION=6327cf1cea9e
 # REPO_NAME=icedtea7-2.6
 # PROJECT_NAME=release
 # OPENJDK_URL=http://icedtea.classpath.org/hg/
 # TO_COMPRESS="*/tapset"
 #
-# They are used to create correct name and are used in construction of sources url (unless REPO_ROOT is set)
+# They are used to create correct name and are used in construction of sources
-
+# URL (unless REPO_ROOT is set).
-# This script creates a single source tarball out of the repository
-# based on the given tag and removes code not allowed in fedora/rhel. For
-# consistency, the source tarball will always contain 'openjdk' as the top
-# level folder, name is created, based on parameter
 #
+# This script creates a single source tarball out of the repository based on the
+# given tag and removes code not allowed in Fedora/RHEL.
 
 set -e
 
 OPENJDK_URL_DEFAULT=https://github.com
 COMPRESSION_DEFAULT=xz
 
-if [ "x$1" = "xhelp" ] ; then
+if [ "$1" = "help" ] ; then
-    echo -e "Behaviour may be specified by setting the following variables:\n"
+    echo "Behaviour may be specified by setting the following variables:"
+    echo
     echo "VERSION        - the version of the specified OpenJDK project"
-    echo "PROJECT_NAME -- the name of the OpenJDK project being archived (optional; only needed by defaults)"
+    echo "                 (required unless OPENJDK_LATEST is set)"
-    echo "REPO_NAME - the name of the OpenJDK repository (optional; only needed by defaults)"
+    echo "PROJECT_NAME   - the name of the OpenJDK project being archived"
-    echo "OPENJDK_URL - the URL to retrieve code from (optional; defaults to ${OPENJDK_URL_DEFAULT})"
+    echo "                 (needed to compute REPO_ROOT and/or"
-    echo "COMPRESSION - the compression type to use (optional; defaults to ${COMPRESSION_DEFAULT})"
+    echo "                  FILE_NAME_ROOT automatically;"
-    echo "FILE_NAME_ROOT - name of the archive, minus extensions (optional; defaults to PROJECT_NAME-REPO_NAME-VERSION)"
+    echo "                  optional if they are set explicitly)"
-    echo "REPO_ROOT - the location of the Git repository to archive (optional; defaults to OPENJDK_URL/PROJECT_NAME/REPO_NAME.git)"
+    echo "REPO_NAME      - the name of the OpenJDK repository"
-    echo "TO_COMPRESS - what part of clone to pack (default is ${VERSION})"
+    echo "                 (needed to compute REPO_ROOT automatically;"
+    echo "                  optional if REPO_ROOT is set explicitly)"
+    echo "OPENJDK_URL    - the URL to retrieve code from"
+    echo "                 (defaults to ${OPENJDK_URL_DEFAULT})"
+    echo "COMPRESSION    - the compression type to use"
+    echo "                 (defaults to ${COMPRESSION_DEFAULT})"
+    echo "FILE_NAME_ROOT - name of the archive, minus extensions"
+    echo "                 (defaults to PROJECT_NAME-VERSION)"
+    echo "REPO_ROOT      - the location of the Git repository to archive"
+    echo "                 (defaults to OPENJDK_URL/PROJECT_NAME/REPO_NAME.git)"
+    echo "TO_COMPRESS    - what part of clone to pack"
+    echo "                 (defaults to ${VERSION})"
     echo "BOOT_JDK       - the bootstrap JDK to satisfy the configure run"
+    echo "                 (defaults to packaged JDK version)"
+    echo "WITH_TEMP      - run in a temporary directory"
+    echo "                 (defaults to disabled)"
+    echo "OPENJDK_LATEST - deduce VERSION from most recent upstream tag"
+    echo "                 (implies WITH_TEMP, computes everything else"
+    echo "                  automatically; Note: accesses network to read"
+    echo "                  tag list from remote Git repository)"
     exit 1;
 fi
 
+if [ "$OPENJDK_LATEST" != "" ] ; then
+    FEATURE_VERSION=$(echo '%featurever' \
+                          | rpmspec --shell ./*.spec 2>/dev/null \
+                          | grep --after-context 1 featurever \
+                          | tail --lines 1)
+    PROJECT_NAME=openjdk
+    REPO_NAME=jdk"${FEATURE_VERSION}"u
+    VERSION=$(git ls-remote --tags --refs --sort=-version:refname \
+                  "${OPENJDK_URL_DEFAULT}/${PROJECT_NAME}/${REPO_NAME}.git" \
+                  "jdk-${FEATURE_VERSION}*" \
+                  | head --lines 1 | cut --characters 52-)
+    FILE_NAME_ROOT=open${VERSION}
+    WITH_TEMP=1
+fi
 
-if [ "x$VERSION" = "x" ] ; then
+if [ "$WITH_TEMP" != "" ] ; then
+    pushd "$(mktemp --directory temp-generated-source-tarball-XXX)"
+fi
+
+if [ "$VERSION" = "" ] ; then
     echo "No VERSION specified"
     exit 2
 fi
@@ -131,53 +179,76 @@ echo -e "\tREPO_ROOT: ${REPO_ROOT}"
 echo -e "\tTO_COMPRESS: ${TO_COMPRESS}"
 echo -e "\tBOOT_JDK: ${BOOT_JDK}"
 
-if [ -d ${FILE_NAME_ROOT} ] ; then
+if [ -d "${FILE_NAME_ROOT}" ] ; then
   echo "exists exists exists exists exists exists exists "
   echo "reusing reusing reusing reusing reusing reusing "
-  echo ${FILE_NAME_ROOT}
+  echo "${FILE_NAME_ROOT}"
+  STAT_TIME="$(stat --format=%Y "${FILE_NAME_ROOT}")"
+  TAR_TIME="$(date --date=@"${STAT_TIME}" --iso-8601=seconds)"
 else
   mkdir "${FILE_NAME_ROOT}"
   pushd "${FILE_NAME_ROOT}"
     echo "Cloning ${VERSION} root repository from ${REPO_ROOT}"
-    git clone -b ${VERSION} ${REPO_ROOT} ${VERSION}
+    git clone --depth=1 -b "${VERSION}" "${REPO_ROOT}" "${VERSION}"
+    pushd "${VERSION}"
+      TAR_TIME="$(git log --max-count 1 --format=%cI)"
+    popd
   popd
 fi
 pushd "${FILE_NAME_ROOT}"
-
+    # Generate .src-rev so build has knowledge of the revision the tarball was
-    # Generate .src-rev so build has knowledge of the revision the tarball was created from
+    # created from
     mkdir build
     pushd build
-    sh ${PWD}/../${VERSION}/configure --with-boot-jdk=${BOOT_JDK}
+    sh "${PWD}"/../"${VERSION}"/configure --with-boot-jdk="${BOOT_JDK}"
     make store-source-revision
     popd
     rm -rf build
 
     # Remove commit checks
-    echo "Removing $(find ${VERSION} -name '.jcheck' -print)"
+    echo "Removing $(find "${VERSION}" -name '.jcheck' -print)"
-    find ${VERSION} -name '.jcheck' -print0 | xargs -0 rm -r
+    find "${VERSION}" -name '.jcheck' -print0 | xargs -0 rm -r
 
     # Remove history and GHA
     echo "find ${VERSION} -name '.hgtags'"
-    find ${VERSION} -name '.hgtags' -exec rm -v '{}' '+'
+    find "${VERSION}" -name '.hgtags' -exec rm -v '{}' '+'
     echo "find ${VERSION} -name '.hgignore'"
-    find ${VERSION} -name '.hgignore' -exec rm -v '{}' '+'
+    find "${VERSION}" -name '.hgignore' -exec rm -v '{}' '+'
     echo "find ${VERSION} -name '.gitattributes'"
-    find ${VERSION} -name '.gitattributes' -exec rm -v '{}' '+'
+    find "${VERSION}" -name '.gitattributes' -exec rm -v '{}' '+'
     echo "find ${VERSION} -name '.gitignore'"
-    find ${VERSION} -name '.gitignore' -exec rm -v '{}' '+'
+    find "${VERSION}" -name '.gitignore' -exec rm -v '{}' '+'
+    # Work around some Git objects not having write permissions.
+    echo "chmod --recursive u+w ${VERSION}/.git"
+    chmod --recursive u+w "${VERSION}"/.git
     echo "find ${VERSION} -name '.git'"
-    find ${VERSION} -name '.git' -exec rm -rv '{}' '+'
+    find "${VERSION}" -name '.git' -exec rm -rv '{}' '+'
     echo "find ${VERSION} -name '.github'"
-    find ${VERSION} -name '.github' -exec rm -rv '{}' '+'
+    find "${VERSION}" -name '.github' -exec rm -rv '{}' '+'
 
     echo "Compressing remaining forest"
-    if [ "X$COMPRESSION" = "Xxz" ] ; then
+    if [ "$COMPRESSION" = "xz" ] ; then
         SWITCH=cJf
     else
         SWITCH=czf
     fi
-    TARBALL_NAME=${FILE_NAME_ROOT}.tar.${COMPRESSION}
+    EA_PART="$(awk -F= \
-    tar --exclude-vcs -$SWITCH ${TARBALL_NAME} $TO_COMPRESS
+               '/^DEFAULT_PROMOTED_VERSION_PRE/ { if ($2) print "-"$2 }' \
-    mv ${TARBALL_NAME} ..
+               "${VERSION}"/make/conf/version-numbers.conf)"
+    TARBALL_NAME=${FILE_NAME_ROOT}${EA_PART}.tar.${COMPRESSION}
+    XZ_OPT=${XZ_OPT-"-T0"} \
+          tar --mtime="${TAR_TIME}" --owner=root --group=root --sort=name \
+          --exclude-vcs -$SWITCH "${TARBALL_NAME}" "${TO_COMPRESS}"
+    mv "${TARBALL_NAME}" ..
 popd
-echo "Done. You may want to remove the uncompressed version - $FILE_NAME_ROOT."
+if [ "$WITH_TEMP" != "" ] ; then
+    echo "Tarball is: $(realpath --relative-to=.. .)/${TARBALL_NAME}"
+    popd
+else
+    echo -n "Done. You may want to remove the uncompressed version"
+    echo " - $FILE_NAME_ROOT."
+fi
+
+# Local Variables:
+# fill-column: 80
+# End:

java-21-openjdk.spec

@@ -2502,6 +2502,19 @@ cjc.mainProgram(args)
 %changelog
 * Sat Aug 03 2024 Thomas Fitzsimmons <fitzsim@redhat.com> - 1:21.0.3.0.1-0.1.ea
 - Invoke xz in multi-threaded mode
+- generate_source_tarball.sh: Add WITH_TEMP environment variable
+- generate_source_tarball.sh: Multithread xz on all available cores
+- generate_source_tarball.sh: Add OPENJDK_LATEST environment variable
+- generate_source_tarball.sh: Update comment about tarball naming
+- generate_source_tarball.sh: Reformat comment header
+- generate_source_tarball.sh: Reformat and update help output
+- generate_source_tarball.sh: Do a shallow clone, for speed
+- generate_source_tarball.sh: Append -ea designator when required
+- generate_source_tarball.sh: Eliminate some removal prompting
+- generate_source_tarball.sh: Make tarball reproducible
+- generate_source_tarball.sh: Prefix temporary directory with temp-
+- generate_source_tarball.sh: Remove temporary directory exit conditions
+- generate_source_tarball.sh: Fix -ea logic to add dash
 - Related: RHEL-52723
 
 * Sat Aug 03 2024 Andrew Hughes <gnu.andrew@redhat.com> - 1:21.0.3.0.1-0.1.ea

sources

@@ -1,2 +1,2 @@
 SHA512 (tapsets-icedtea-6.0.0pre00-c848b93a8598.tar.xz) = 97d026212363b3c83f6a04100ad7f6fdde833d16579717f8756e2b8c2eb70e144a41a330cb9ccde9c3badd37a2d54fdf4650a950ec21d8b686d545ecb2a64d30
-SHA512 (openjdk-21.0.3+1-ea.tar.xz) = 5925387ebca2b05fd053cf6633e5469585e68eda5bcb0e9d8f51c46233b73f58db2641838b7aae79a81b6257ecafb1b89a223c0e8619b12414df83b35e15ccff
+SHA512 (openjdk-21.0.3+1-ea.tar.xz) = 5b1eb2fc253787a22857c4a5b644f2eeec4774d860c9d90d76f8daf4a40299941fd2865baee4811a16e7c80b4d7dadd1413514fb68d218431f94445a89ea1b75