From 2bf928dd97411ab1ed47c4ae715bba91d15b5150 Mon Sep 17 00:00:00 2001 From: Don Zickus Date: Mon, 6 Nov 2017 17:00:42 -0500 Subject: [PATCH] configs: Update scripts and spec file with layout changes With the configs in the new place, update the scripts to find them and utilize the base-generic and generic heirarchy to apply configs and overrides. Implement new process_configs.sh script that post-process the config changes in a simple script and remove those commands from the spec file. Add option flags to preserve functionality. config_generation is a simple rename of baseconfig -> generic and debugconfig -> debug and arm64 -> aarach64. build_configs.sh is modified to find configs in generic and base-generic and then apply base-generic first and if any generic files, apply those next. The generic directory is used as an overrides and is expected to be empty for Fedora initially. kernel.spec is modified to use process_configs.sh instead of all the commands in the spec file. Enabled spec options are translated to script options. The config manipulation is moved to be grouped with all config manipulation commands. This makes 'cd configs/' simpler. Now all config scripts and executiion are done in configs/ directory. v1 -> v2: * the scripts were not working with SUBARCH correctly * checkoptions was using wrong comparison file * passing wrong kernel version to process_configs in spec file v2 -> v3: (incorporate Laura A's feedback) * update README.txt * fix build_configs.sh warnings * Output info message on listnewconfig failure --- README.txt | 16 ++--- configs/build_configs.sh | 60 ++++++++++++++-- configs/config_generation | 36 +++++----- configs/process_configs.sh | 136 +++++++++++++++++++++++++++++++++++++ kernel.spec | 62 ++++++++--------- process_configs.sh | 1 + 6 files changed, 243 insertions(+), 68 deletions(-) create mode 100755 configs/process_configs.sh create mode 120000 process_configs.sh diff --git a/README.txt b/README.txt index 516119838..f0a42351c 100644 --- a/README.txt +++ b/README.txt @@ -33,9 +33,9 @@ config heirarchy. Instead of having to maintain a config file for every arch variant we build on, the kernel spec uses a nested system of configs. Each option CONFIG_FOO is represented by a single file named CONFIG_FOO which contains the state (=y, =m, -=n). These options are collected in the folder baseconfig. Architecture specifi -options are set in nested folders. An option set in a nested folder will -override the same option set in one of the higher levels. +=n). These options are collected in the folder base-generic. Architecture +specific options are set in nested folders. An option set in a nested folder +will override the same option set in one of the higher levels. The individual CONFIG_FOO files only exist in the pkg-git repository. The RPM contains kernel-foo.config files which are the result of combining all the @@ -45,10 +45,10 @@ script _must_ be run each time one of the options is changed. Example flow: # Enable the option CONFIG_ABC123 as a module for all arches -echo "CONFIG_ABC123=m" > baseconfig/CONFIG_ABC1234 +echo "CONFIG_ABC123=m" > configs/base-generic/CONFIG_ABC1234 # enable the option CONFIG_XYZ321 for only x86 -echo "# CONFIG_XYZ321 is not set" > baseconfig/CONFIG_XYZ321 -echo "CONFIG_XYZ321=m" > baseconfig/x86/CONFIG_XYZ321 +echo "# CONFIG_XYZ321 is not set" > configs/base-generic/CONFIG_XYZ321 +echo "CONFIG_XYZ321=m" > configs/base-generic/x86/CONFIG_XYZ321 # regenerate the combined config files ./build_configs.sh @@ -72,7 +72,7 @@ the status quo to: This is done to increase coverage testing, as not many people actually run kernel-debug. -The debug options are managed in a separate heierarchy under debugconfig. This -works in a similar manner to baseconfig. More deeply nested folders, again, +The debug options are managed in a separate heierarchy under base-debug. This +works in a similar manner to base-generic. More deeply nested folders, again, override options. The file config_generation gives a listing of what folders go into each config file generated. diff --git a/configs/build_configs.sh b/configs/build_configs.sh index 140511f19..f345e2af7 100755 --- a/configs/build_configs.sh +++ b/configs/build_configs.sh @@ -4,13 +4,33 @@ # and debug to form the necessary $PACKAGE_NAME--.config # files for building RHEL kernels, based on the contents of a control file -PACKAGE_NAME=kernel # defines the package name used +PACKAGE_NAME="${1:-kernel}" # defines the package name used +KVERREL="${2:-}" +SUBARCH="${3:-}" # defines a specific arch for use with rh-configs-arch-prep target +SCRIPT="$(readlink -f $0)" +OUTPUT_DIR="$PWD" +SCRIPT_DIR="$(dirname $SCRIPT)" + +# to handle this script being a symlink +cd $SCRIPT_DIR set errexit set nounset control_file="config_generation" +cleanup() +{ + rm -f config-* +} + +die() +{ + echo "$1" + cleanup + exit 1 +} + function combine_config_layer() { dir=$1 @@ -29,14 +49,27 @@ function merge_configs() archvar=$1 arch=$(echo "$archvar" | cut -f1 -d"-") configs=$2 - name=$PACKAGE_NAME-$archvar.config + name=$OUTPUT_DIR/$PACKAGE_NAME-$archvar.config echo -n "Building $name ... " touch config-merging config-merged + + # apply base first for config in $(echo $configs | sed -e 's/:/ /g') do + perl merge.pl config-base-$config config-merging > config-merged + if [ ! $? -eq 0 ]; then + die "Failed to merge base" + fi + mv config-merged config-merging + done + for config in $(echo $configs | sed -e 's/:/ /g') + do + # not all override files exist + test -e config-$config || continue + perl merge.pl config-$config config-merging > config-merged if [ ! $? -eq 0 ]; then - exit + die "Failed to merge configs" fi mv config-merged config-merging done @@ -60,10 +93,12 @@ function merge_configs() echo "done" } -glist=$(find baseconfig -type d) -dlist=$(find debugconfig -type d) +glist=$(find base-generic -type d) +dlist=$(find base-debug -type d) +gllist=$(test -d generic && find generic -type d) +dllist=$(test -d debug && find debug -type d) -for d in $glist $dlist +for d in $glist $dlist $gllist $dllist do combine_config_layer $d done @@ -86,4 +121,15 @@ do fi done < $control_file -rm -f config-* +# A passed in kernel version implies copy to final location +# otherwise defer to another script +if test -n "$KVERREL" +then + for i in kernel-*.config + do + NEW="$(echo $i | sed "s/$PACKAGE_NAME-$SUBARCH/$PACKAGE_NAME-$KVERREL-$SUBARCH/")" + mv $i $NEW + done +fi + +cleanup diff --git a/configs/config_generation b/configs/config_generation index 64ae444f4..b88a5ea94 100644 --- a/configs/config_generation +++ b/configs/config_generation @@ -4,33 +4,33 @@ # config options, lowest priority to highest # x86_64 -x86_64=baseconfig:baseconfig-x86:baseconfig-x86-x86_64 -x86_64-debug=baseconfig:baseconfig-x86:baseconfig-x86-x86_64:debugconfig:debugconfig-x86:debugconfig-x86-x86_64 +x86_64=generic:generic-x86:generic-x86-x86_64 +x86_64-debug=generic:generic-x86:generic-x86-x86_64:debug:debug-x86:debug-x86-x86_64 # i686 -i686=baseconfig:baseconfig-x86:baseconfig-x86-i686 -i686-debug=baseconfig:baseconfig-x86:baseconfig-x86-i686:debugconfig:debugconfig-x86 -i686-PAE=baseconfig:baseconfig-x86:baseconfig-x86-i686PAE -i686-PAEdebug=baseconfig:baseconfig-x86:baseconfig-x86-i686PAE:debugconfig:debugconfig-x86 +i686=generic:generic-x86:generic-x86-i686 +i686-debug=generic:generic-x86:generic-x86-i686:debug:debug-x86 +i686-PAE=generic:generic-x86:generic-x86-i686PAE +i686-PAEdebug=generic:generic-x86:generic-x86-i686PAE:debug:debug-x86 # ppc64 -ppc64=baseconfig:baseconfig-powerpc:baseconfig-powerpc-powerpc64 -ppc64-debug=baseconfig:baseconfig-powerpc:baseconfig-powerpc-powerpc64:debugconfig +ppc64=generic:generic-powerpc:generic-powerpc-powerpc64 +ppc64-debug=generic:generic-powerpc:generic-powerpc-powerpc64:debug # ppc64le -ppc64le=baseconfig:baseconfig-powerpc:baseconfig-powerpc-powerpc64le -ppc64le-debug=baseconfig:baseconfig-powerpc:baseconfig-powerpc-powerpc64le:debugconfig +ppc64le=generic:generic-powerpc:generic-powerpc-powerpc64le +ppc64le-debug=generic:generic-powerpc:generic-powerpc-powerpc64le:debug # s390x -s390x=baseconfig:baseconfig-s390x -s390x-debug=baseconfig:baseconfig-s390x:debugconfig +s390x=generic:generic-s390x +s390x-debug=generic:generic-s390x:debug # aarch64 -aarch64=baseconfig:baseconfig-arm:baseconfig-arm-arm64 -aarch64-debug=baseconfig:baseconfig-arm:baseconfig-arm-arm64:debugconfig:debugconfig-arm +aarch64=generic:generic-arm:generic-arm-aarch64 +aarch64-debug=generic:generic-arm:generic-arm-aarch64:debug:debug-arm # arm -armv7hl=baseconfig:baseconfig-arm:baseconfig-arm-armv7:baseconfig-arm-armv7-armv7 -armv7hl-debug=baseconfig:baseconfig-arm:baseconfig-arm-armv7:baseconfig-arm-armv7-armv7:debugconfig:debugconfig-arm -armv7hl-lpae=baseconfig:baseconfig-arm:baseconfig-arm-armv7:baseconfig-arm-armv7-lpae -armv7hl-lpae-debug=baseconfig:baseconfig-arm:baseconfig-arm-armv7:baseconfig-arm-armv7-lpae:debugconfig:debugconfig-arm +armv7hl=generic:generic-arm:generic-arm-armv7:generic-arm-armv7-armv7 +armv7hl-debug=generic:generic-arm:generic-arm-armv7:generic-arm-armv7-armv7:debug:debug-arm +armv7hl-lpae=generic:generic-arm:generic-arm-armv7:generic-arm-armv7-lpae +armv7hl-lpae-debug=generic:generic-arm:generic-arm-armv7:generic-arm-armv7-lpae:debug:debug-arm diff --git a/configs/process_configs.sh b/configs/process_configs.sh new file mode 100755 index 000000000..4de45d65a --- /dev/null +++ b/configs/process_configs.sh @@ -0,0 +1,136 @@ +#!/bin/bash +# +# This script takes the merged config files and processes them through oldconfig +# and listnewconfig + + +die() +{ + echo "$1" + exit 1 +} + +# stupid function to find top of tree to do kernel make configs +switch_to_toplevel() +{ + path="$(pwd)" + while test -n "$path" + do + test -d $path/firmware && \ + test -e $path/MAINTAINERS && \ + test -d $path/drivers && \ + break + + path="$(dirname $path)" + done + + test -n "$path" || die "Can't find toplevel" + echo "$path" +} + +checkoptions() +{ + /usr/bin/awk ' + + /is not set/ { + split ($0, a, "#"); + split(a[2], b); + if (NR==FNR) { + configs[b[1]]="is not set"; + } else { + if (configs[b[1]] != "" && configs[b[1]] != "is not set") + print "Found # "b[1] " is not set, after generation, had " b[1] " " configs[b[1]] " in Source tree"; + } + } + + /=/ { + split ($0, a, "="); + if (NR==FNR) { + configs[a[1]]=a[2]; + } else { + if (configs[a[1]] != "" && configs[a[1]] != a[2]) + print "Found "a[1]"="configs[a[1]]" after generation, had " a[1]"="a[2]" in Source tree"; + } + } + ' $1 $2 > .mismatches + + if test -s .mismatches + then + echo "Error: Mismatches found in configuration files" + cat .mismatches + exit 1 + fi +} + +function process_configs() +{ + # assume we are in $source_tree/configs, need to get to top level + pushd $(switch_to_toplevel) + + for cfg in $SCRIPT_DIR/${PACKAGE_NAME}${KVERREL}${SUBARCH}*.config + do + arch=$(head -1 $cfg | cut -b 3-) + cfgtmp="${cfg}.tmp" + cfgorig="${cfg}.orig" + cat $cfg > $cfgorig + + echo -n "Processing $cfg ... " + + # an empty grep is good but leaves a return value, so use # 'true' to bypass + make ARCH=$arch KCONFIG_CONFIG=$cfg listnewconfig | grep -E 'CONFIG_' > .newoptions || true + if test -n "$NEWOPTIONS" && test -s .newoptions + then + echo "Found unset config items, please set them to an appropriate value" + cat .newoptions + rm .newoptions + exit 1 + fi + rm .newoptions + + make ARCH=$arch KCONFIG_CONFIG=$cfg oldnoconfig > /dev/null || exit 1 + echo "# $arch" > ${cfgtmp} + cat "${cfg}" >> ${cfgtmp} + if test -n "$CHECKOPTIONS" + then + checkoptions $cfgtmp $cfgorig + fi + mv ${cfgtmp} ${cfg} + rm ${cfgorig} + echo "done" + done + rm "$SCRIPT_DIR"/*.config.old + popd > /dev/null + + echo "Processed config files are in $SCRIPT_DIR" +} + +NEWOPTIONS="" +CHECKOPTIONS="" + +while [[ $# -gt 0 ]] +do + key="$1" + case $key in + -n) + NEWOPTIONS="x" + ;; + -c) + CHECKOPTIONS="x" + ;; + *) + break;; + esac + shift +done + +PACKAGE_NAME="${1:-kernel}" # defines the package name used +KVERREL="$(test -n "$2" && echo "-$2" || echo "")" +SUBARCH="$(test -n "$3" && echo "-$3" || echo "")" +SCRIPT="$(readlink -f $0)" +OUTPUT_DIR="$PWD" +SCRIPT_DIR="$(dirname $SCRIPT)" + +# to handle this script being a symlink +cd $SCRIPT_DIR + +process_configs diff --git a/kernel.spec b/kernel.spec index 7c19ebbe0..7d2b5c590 100644 --- a/kernel.spec +++ b/kernel.spec @@ -478,7 +478,7 @@ Source39: kernel-x86_64-debug.config Source40: generate_all_configs.sh Source41: generate_debug_configs.sh -Source42: check_configs.awk +Source42: process_configs.sh # This file is intentionally left empty in the stock kernel. Its a nicety # added for those wanting to do custom rebuilds with altered config opts. @@ -1225,6 +1225,24 @@ xzcat %{SOURCE5000} | patch -p1 -F1 -s git commit -a -m "Stable update" %endif +# Note: Even in the "nopatches" path some patches (build tweaks and compile +# fixes) will always get applied; see patch defition above for details + +git am %{patches} + +# END OF PATCH APPLICATIONS + +# Any further pre-build tree manipulations happen here. + +chmod +x scripts/checkpatch.pl + +# This Prevents scripts/setlocalversion from mucking with our version numbers. +touch .scmversion + +# Deal with configs stuff +mkdir configs +cd configs + # Drop some necessary files from the source dir into the buildroot cp $RPM_SOURCE_DIR/kernel-*.config . cp %{SOURCE1000} . @@ -1255,25 +1273,9 @@ do done %endif -# Note: Even in the "nopatches" path some patches (build tweaks and compile -# fixes) will always get applied; see patch defition above for details - -git am %{patches} - -# END OF PATCH APPLICATIONS - -# Any further pre-build tree manipulations happen here. - -chmod +x scripts/checkpatch.pl - -# This Prevents scripts/setlocalversion from mucking with our version numbers. -touch .scmversion - # only deal with configs if we are going to build for the arch %ifnarch %nobuildarches -mkdir configs - %if !%{debugbuildsenabled} rm -f kernel-%{version}-*debug.config %endif @@ -1291,31 +1293,21 @@ CheckConfigs() { } cp %{SOURCE42} . -# now run oldconfig over all the config files -for i in *.config -do - cat $i > temp-$i - mv $i .config - Arch=`head -1 .config | cut -b 3-` - make ARCH=$Arch listnewconfig | grep -E '^CONFIG_' >.newoptions || true +OPTS="" %if %{listnewconfig_fail} - if [ -s .newoptions ]; then - cat .newoptions - exit 1 - fi + OPTS="$OPTS -n" %endif - rm -f .newoptions - make ARCH=$Arch oldnoconfig - echo "# $Arch" > configs/$i - cat .config >> configs/$i %if %{configmismatch_fail} - CheckConfigs configs/$i temp-$i + OPTS="$OPTS -c" %endif - rm temp-$i -done +./process_configs.sh $OPTS kernel %{rpmversion} + # end of kernel config %endif +cd .. +# End of Configs stuff + # get rid of unwanted files resulting from patch fuzz find . \( -name "*.orig" -o -name "*~" \) -exec rm -f {} \; >/dev/null diff --git a/process_configs.sh b/process_configs.sh new file mode 120000 index 000000000..134a69099 --- /dev/null +++ b/process_configs.sh @@ -0,0 +1 @@ +configs/process_configs.sh \ No newline at end of file