#!/usr/bin/env python3 """ kmap.py - Kernel Source-to-Binary Mapping Tool DESCRIPTION This tool parses Linux kernel build artifacts (.cmd files, archives) to create a JSON mapping of which source files contribute to which kernel binaries (modules and vmlinux). It supports merging data from multiple kernel build variants (e.g., stock, debug, rt) into a single output file using variant indices to track per-variant mappings. Supported RHEL versions: 7, 8, 9, upstream FEATURES - Parses .cmd files generated by kbuild to extract compilation commands - Supports C (gcc/clang) and Rust (rustc) source files - Handles built-in objects via built-in.o.cmd (RHEL 7), built-in.a (RHEL 8/9), or vmlinux.a (upstream) - Merges multiple kernel variants into single output with variant indices - Maps modules to their containing RPM packages OUTPUT FORMAT The output is a JSON file with the following structure: { "variants": ["stock", "rt", ...], "source-map": { "obj-src": {: {: []}}, "src-obj": {: {: []}} }, "module-map": { "module-rpm": {: {: []}}, "rpm-modules": {: {: []}} } } PARSING THE OUTPUT import json with open('kernel-map.json') as f: data = json.load(f) variants = data['variants'] obj_src = data['source-map']['obj-src'] src_obj = data['source-map']['src-obj'] module_rpm = data['module-map']['module-rpm'] rpm_modules = data['module-map']['rpm-modules'] # Get sources for vmlinux in variant "stock" (index 0) vmlinux_sources = [src for src, indices in obj_src['vmlinux'].items() if 0 in indices] # Find which object a source contributes to fork_objects = [obj for obj, indices in src_obj['kernel/fork.c'].items() if 0 in indices] # Get RPM for a module def get_rpm(module, variant_idx): for rpm, indices in module_rpm.get(module, {}).items(): if variant_idx in indices: return rpm return None USAGE # Single variant ./kmap.py -d /build/stock -r upstream -v stock -o /output # Merge multiple variants ./kmap.py -d /build/stock -r upstream -v stock -o /output ./kmap.py -d /build/debug -r upstream -v debug -o /output -i kernel-map.json ./kmap.py -d /build/rt -r upstream -v rt -o /output -i kernel-map.json # With module-to-RPM mapping ./kmap.py -d /build -r upstream -v stock -o /output \\ --vmlinux-rpm kernel-core-5.14.0.rpm \\ --module-list kernel-modules-5.14.0.rpm:modules.list CREDITS Inspired by gen_compile_commands.py and Joe Lawrence. Time measurement option added for Jan Stancek. """ import re import os import pathlib import json import argparse import subprocess import time # ============================================================================= # Constants # ============================================================================= VMLINUX = 'vmlinux' KERNEL_PREFIX = 'kernel/' # Supported compilers and linkers COMPILERS = ('gcc', 'clang') LINKER = 'ld' RUST_COMPILER = 'rustc' # ============================================================================= # Regex Patterns # # These patterns parse the kernel build .cmd files to extract: # - What compiler/linker was used # - What source files were compiled # - What object files were linked # ============================================================================= # Pattern for .ko.cmd files (module linking) # Matches: cmd_path/to/module.ko := ld -o # Note: upstream kernels use 'savedcmd_' prefix instead of 'cmd_' _KO_CMD_PATTERN = ( r'^(saved)?cmd_[^ ]*\.ko\s*:=\s*ld\s+' r'(?P.*) -o (?P\S+) (?P[^;]+?)\s*(?:;|$)' ) # Pattern for .o.cmd files (object compilation) # Matches multiple formats: # - C/asm: cmd_*.o := gcc/clang -o # - Rust: cmd_*.o := rustc --emit=obj= # - Rust source mapping: source_*.o := _O_CMD_PATTERN = ( r'^(saved)?(?:cmd_|source_)[^ ]*\.o\s+:=\s+' r'(?:' r'(?P(?:\S+=\S+\s+)*)?' r'(?Pgcc|clang|ld|rustc)\s+' r'(?P.*?)' r'(?:' r'\s+-o\s+(?P\S+)\s+(?P[^;]+?)' r'|' r'\s+--emit=obj=(?P\S+).*?\s+(?P\S+\.rs)' r')' r'|' r'(?P\S+\.rs)' r')\s*(?:;|$)' ) ko_matcher = re.compile(_KO_CMD_PATTERN) o_matcher = re.compile(_O_CMD_PATTERN) # ============================================================================= # Utility Functions # ============================================================================= def parse_args(): """ Parse command line arguments """ parser = argparse.ArgumentParser( prog='Kernel Build Mapper', description='Scrape Linux Kernel .cmd files to map contribution ' 'of individual source files to kernel binaries.', ) parser.add_argument('-d', '--directory', help='Directory with kernel build artifacts.', default='', type=str) parser.add_argument('-o', '--outputdir', help='Directory where to put generated files.', default='', type=str) parser.add_argument('-p', '--prefix', help='Name prefix to the generated files.', default='', type=str) parser.add_argument('-r', '--rhel', help='Specify to what version of RHEL artifacts belong (upstream for RHEL 10+/kernel-ark)', choices=['7', '8', '9', 'upstream'], required=True) parser.add_argument('-m', '--no-modules', help='*SKIP* collecting information about modules', action='store_true') parser.add_argument('-b', '--no-builtin', help='*SKIP* collecting information about built-in', action='store_true') parser.add_argument('-t', '--time', help='Print elapsed time at the end', action='store_true') parser.add_argument('-v', '--variant', help='Name of this kernel variant (e.g., stock, debug, rt)', required=True, type=str) parser.add_argument('-i', '--input', help='Existing JSON file to merge with (from previous variant runs)', default='', type=str) parser.add_argument('--module-list', help='Module list in format : (can be specified multiple times)', action='append', default=[], metavar='RPM:PATH') parser.add_argument('--vmlinux-rpm', help='RPM package name for vmlinux (e.g., "kernel-5.14.0.rpm")', default='', type=str) return parser.parse_args() def load_file(filename, build_dir=''): """ Load a text file and return non-empty lines as a list """ full_path = os.path.join(build_dir, filename) try: with open(full_path, 'rt', encoding='utf-8') as file: return [line.rstrip() for line in file if line.rstrip()] except FileNotFoundError: print(f'File "{full_path}" not found.') return [] def name_to_cmd(filename, suffix): """ Convert object filename to corresponding .cmd script filename Example: kernel/fork.o -> kernel/.fork.o.cmd """ path, _ = os.path.splitext(filename) dirname = os.path.dirname(path) basename = os.path.basename(path) return os.path.join(dirname, '.' + basename + suffix) def strip_path_prefixes(path, build_dir): """ Normalize path by removing build directory prefix and other artifacts """ path = pathlib.Path(path) if build_dir: try: return str(path.resolve().relative_to(pathlib.Path(build_dir).resolve())) except ValueError: pass if path.is_absolute(): path = pathlib.Path(*path.parts[1:]) while path.parts and path.parts[0] == 'repos': path = pathlib.Path(*path.parts[1:]) return str(path) def is_external_source(path): """ Check if path should be excluded from source mapping Filters out: - Rust toolchain sources (not part of kernel source) - Generated assembly files (e.g., initramfs_data.S) """ if 'rustlib' in path: return True if path.endswith('_data.S'): return True return False def is_mod_file_reference(ld_input): """ Check if linker input is a .mod file reference (RHEL 9+ module linking) In RHEL 9+, module linking uses @path/to/module.mod files that contain the list of object files to link. """ files = ld_input.split() return len(files) == 1 and files[0].startswith('@') and files[0].endswith('.mod') # ============================================================================= # SourceMap Class # ============================================================================= class SourceMap: """ Collects and manages kernel source-to-binary mappings. This class parses kernel build artifacts (.cmd files, archives) to determine which source files contribute to which kernel binaries (modules and vmlinux). Data structures: obj_to_src: {object_name: {source_file: [variant_indices]}} src_to_obj: {source_file: {object_name: [variant_indices]}} module_rpm: {module_name: {rpm_name: [variant_indices]}} rpm_to_modules: {rpm_name: {module_name: [variant_indices]}} Variant indices correspond to positions in the 'variants' list. """ # Dispatch table for RHEL-specific builtin collection methods BUILTIN_COLLECTORS = { '7': '_collect_builtin_rhel7', '8': '_collect_builtin_archive', '9': '_collect_builtin_archive', 'upstream': '_collect_builtin_vmlinux_archive', } def __init__(self, build_dir='', output_dir='', outprefix='', rhel='7', variant=''): # Source mappings self.obj_to_src = {} self.src_to_obj = {} # Module-to-RPM mappings self.module_rpm = {} self.rpm_to_modules = {} # Configuration self.build_dir = build_dir self.output_dir = output_dir self.outprefix = outprefix self.rhel = rhel # Module list self.modlist = set() # Variant tracking for multi-variant merging self.variants = [] self.variant = variant self.variant_idx = 0 # ------------------------------------------------------------------------- # Variant Management # ------------------------------------------------------------------------- def load_existing(self, filename): """ Load existing merged data from previous variant runs """ if not filename: return full_path = os.path.join(self.output_dir, filename) if self.output_dir else filename if not os.path.exists(full_path): return with open(full_path, 'rt', encoding='utf-8') as f: data = json.load(f) self.variants = data.get('variants', []) module_map = data.get('module-map', {}) self.module_rpm = module_map.get('module-rpm', {}) self.rpm_to_modules = module_map.get('rpm-modules', {}) source_map = data.get('source-map', {}) self.obj_to_src = source_map.get('obj-src', {}) self.src_to_obj = source_map.get('src-obj', {}) def add_variant(self): """ Register current variant and store its index """ if self.variant in self.variants: self.variant_idx = self.variants.index(self.variant) else: self.variant_idx = len(self.variants) self.variants.append(self.variant) # ------------------------------------------------------------------------- # Source Merging Helpers # ------------------------------------------------------------------------- def _merge_sources(self, target, source): """ Merge source dict into target dict by combining variant index lists """ for src, indices in source.items(): if src not in target: target[src] = [] for idx in indices: if idx not in target[src]: target[src].append(idx) def _add_source(self, sources, source_path): """ Add a source file to the sources dict with current variant index Returns True if source was added, False if filtered out """ source_path = os.path.normpath(source_path) source_path = strip_path_prefixes(source_path, self.build_dir) if is_external_source(source_path): return False sources[source_path] = [self.variant_idx] return True # ------------------------------------------------------------------------- # .cmd File Parsing # ------------------------------------------------------------------------- def _parse_ko_cmd(self, ko_name): """ Parse .ko.cmd file to find the primary object file linked into the module """ ko_script = name_to_cmd(ko_name, '.ko.cmd') lines = load_file(ko_script, build_dir=self.build_dir) for line in lines: result = ko_matcher.match(line) if result: input_files = result.group('input_files').split() return input_files[0] return None def _parse_o_cmd(self, o_name, translate=True): """ Parse .o.cmd file to extract source files that contribute to the object Handles: - C/assembly compilation (gcc, clang) - Linking (ld) - recursively parses linked objects - Rust compilation (rustc) - Rust source file mappings Args: o_name: Object file name or .cmd file path translate: If True, convert object name to .cmd path (e.g., kernel/fork.o becomes kernel/.fork.o.cmd). If False, use o_name as-is. This parameter exists to handle different kbuild behaviors across RHEL versions - some code paths may already have .cmd paths while others have object names that need translation. """ sources = {} o_script = name_to_cmd(o_name, '.o.cmd') if translate else o_name lines = load_file(o_script, build_dir=self.build_dir) for line in lines: result = o_matcher.match(line) if not result: continue command = result.group('command') if command in COMPILERS: self._handle_compiler(result, sources) break elif command == LINKER: self._handle_linker(result, sources) break elif command == RUST_COMPILER: self._handle_rustc(result, sources) break else: self._handle_rust_source_mapping(result, sources) break return sources def _handle_compiler(self, match_result, sources): """ Handle gcc/clang compilation - extract the input source file """ obj_input = match_result.group('input_files') if obj_input: self._add_source(sources, obj_input) def _handle_linker(self, match_result, sources): """ Handle ld linking - recursively parse linked objects """ obj_input = match_result.group('input_files') if not obj_input: return if is_mod_file_reference(obj_input): self._merge_sources(sources, self._parse_mod_cmd(obj_input)) else: for obj_file in obj_input.split(): self._merge_sources(sources, self._parse_o_cmd(obj_file)) def _handle_rustc(self, match_result, sources): """ Handle rustc compilation - extract the Rust source file """ rust_input = match_result.group('rust_input_file') if rust_input: self._add_source(sources, rust_input) def _handle_rust_source_mapping(self, match_result, sources): """ Handle Rust source file mapping (source_*.o := *.rs) """ source_file = match_result.group('source_file') if source_file and source_file.endswith('.rs'): self._add_source(sources, source_file) def _parse_mod_cmd(self, mod_name): """ Parse .mod file used by RHEL 9+ for linking modules These files contain a list of object files to link into the module. """ sources = {} mod_name = mod_name.strip('@') lines = load_file(mod_name, build_dir=self.build_dir) for line in lines: self._merge_sources(sources, self._parse_o_cmd(line)) return sources def _parse_a_cmd(self, a_name): """ Parse archive .cmd file (RHEL 7 built-in.o.cmd files) """ sources = {} lines = load_file(a_name, build_dir=self.build_dir) for line in lines: result = o_matcher.match(line) if not result: continue ar_input = result.group('input_files') for filename in ar_input.rsplit(): if filename.endswith('.o'): self._merge_sources(sources, self._parse_o_cmd(filename)) elif filename.endswith('.a'): self._merge_sources(sources, self._parse_a_cmd(filename)) return sources # ------------------------------------------------------------------------- # Module Collection # ------------------------------------------------------------------------- def _load_module_list(self, modorder='modules.order'): """ Load modules.order file and normalize paths RHEL 7, 8: modules.order has 'kernel/' prefix that needs to be stripped RHEL 9, upstream: modules.order paths are used as-is """ lines = load_file(modorder, build_dir=self.build_dir) if not lines: print(f'Warning: modules.order is empty or not found') self.modlist = set() return if self.rhel in ['9', 'upstream']: self.modlist = set(lines) else: # RHEL 7, 8: strip fake 'kernel/' prefix added by kbuild self.modlist = { path[len(KERNEL_PREFIX):] if path.startswith(KERNEL_PREFIX) else path for path in lines } def _parse_module_objects(self): """ Parse all module .cmd files and extract source mappings. Note on RHEL version differences: - RHEL 7/8/9: modules.order contains .ko paths (e.g., fs/exfat/exfat.ko) - upstream: modules.order contains .o paths (e.g., fs/exfat/exfat.o) The normalization below ensures consistent .ko keys in output regardless of the modules.order format. """ for kobj in self.modlist: # Normalize to .ko - upstream has .o paths in modules.order if self.rhel == 'upstream' and kobj.endswith('.o'): module_name = kobj[:-2] + '.ko' else: module_name = kobj obj_name = self._parse_ko_cmd(module_name) if obj_name is None: print(f'Warning: Could not parse .ko.cmd for "{module_name}", skipping.') continue obj_source = self._parse_o_cmd(obj_name) if module_name not in self.obj_to_src: self.obj_to_src[module_name] = {} self._merge_sources(self.obj_to_src[module_name], obj_source) def collect_modules(self, no_modules=False, modorder='modules.order'): """ Collect source files for kernel modules """ if no_modules: return self._load_module_list(modorder) self._parse_module_objects() # ------------------------------------------------------------------------- # Builtin Collection (RHEL version-specific) # ------------------------------------------------------------------------- def _collect_builtin_rhel7(self): """ Collect built-in objects for RHEL 7 using *built-in.o.cmd files """ pattern = '*built-in.o.cmd' cmd_list = { strip_path_prefixes(path, self.build_dir) for path in pathlib.Path(self.build_dir).rglob(pattern) } for entry in cmd_list: sources = self._parse_a_cmd(entry) self._merge_sources(self.obj_to_src[VMLINUX], sources) def _collect_builtin_archive(self): """ Collect built-in objects for RHEL 8/9 using *built-in.a archives """ pattern = '*built-in.a' archive_list = { strip_path_prefixes(path, self.build_dir) for path in pathlib.Path(self.build_dir).rglob(pattern) } for entry in archive_list: path = os.path.join(self.build_dir, entry) objects = subprocess.check_output(['ar', 't', path]).decode().split() for obj in objects: obj_path = strip_path_prefixes(obj, build_dir=self.build_dir) sources = self._parse_o_cmd(obj_path) self._merge_sources(self.obj_to_src[VMLINUX], sources) def _collect_builtin_vmlinux_archive(self): """ Collect built-in objects for upstream kernels using single vmlinux.a archive """ path = os.path.join(self.build_dir, 'vmlinux.a') objects = subprocess.check_output(['ar', 't', path]).decode().split() for obj in objects: obj_path = strip_path_prefixes(obj, build_dir=self.build_dir) sources = self._parse_o_cmd(obj_path) self._merge_sources(self.obj_to_src[VMLINUX], sources) def collect_builtin(self, no_builtin=False): """ Collect source files for vmlinux built-in objects Uses RHEL version-specific collection method via dispatch table. """ if VMLINUX not in self.obj_to_src: self.obj_to_src[VMLINUX] = {} if no_builtin: return collector_name = self.BUILTIN_COLLECTORS.get(self.rhel, '_collect_builtin_archive') collector = getattr(self, collector_name) collector() # ------------------------------------------------------------------------- # Module-to-RPM Mapping # ------------------------------------------------------------------------- def _parse_module_list_file(self, filename, rpm_name): """ Parse a module list file and add entries to module_rpm mapping Handles various formats: - Absolute paths: /lib/modules/5.14.0/kernel/drivers/net/e1000.ko.xz - Relative paths: kernel/drivers/net/e1000.ko - Skips RPM spec directives (%dir, %defattr, etc.) Args: filename: Path to the module list file rpm_name: Literal RPM package name (e.g., "kernel-modules-5.14.0.rpm") """ if not filename or not os.path.exists(filename): return with open(filename, 'rt', encoding='utf-8') as f: for line in f: line = line.strip() # Skip empty lines, comments, and RPM spec directives if not line or line.startswith('#') or line.startswith('%'): continue # Skip non-module files if '.ko' not in line: continue # Extract module name from path module = os.path.basename(line) # Remove .ko and any compression suffix module = re.sub(r'\.ko(\.[^.]+)?$', '', module) if not module: continue if module not in self.module_rpm: self.module_rpm[module] = {} if rpm_name not in self.module_rpm[module]: self.module_rpm[module][rpm_name] = [] if self.variant_idx not in self.module_rpm[module][rpm_name]: self.module_rpm[module][rpm_name].append(self.variant_idx) def collect_module_rpm(self, module_lists, vmlinux_rpm=''): """ Collect module-to-RPM name mappings from module list files Args: module_lists: List of "rpm_name:path" strings vmlinux_rpm: RPM package name for vmlinux """ if not module_lists and not vmlinux_rpm: return # Add vmlinux to specified RPM if vmlinux_rpm: if VMLINUX not in self.module_rpm: self.module_rpm[VMLINUX] = {} if vmlinux_rpm not in self.module_rpm[VMLINUX]: self.module_rpm[VMLINUX][vmlinux_rpm] = [] if self.variant_idx not in self.module_rpm[VMLINUX][vmlinux_rpm]: self.module_rpm[VMLINUX][vmlinux_rpm].append(self.variant_idx) # Process module list files for entry in module_lists: if ':' not in entry: print(f'Warning: Invalid --module-list format "{entry}", expected RPM:PATH') continue rpm_name, path = entry.split(':', 1) self._parse_module_list_file(path, rpm_name) # ------------------------------------------------------------------------- # Output # ------------------------------------------------------------------------- def _build_reverse_map(self): """ Create reverse mapping (source -> {obj: [indices]}) from obj_to_src """ for obj, sources in self.obj_to_src.items(): for source, indices in sources.items(): if source not in self.src_to_obj: self.src_to_obj[source] = {} if obj not in self.src_to_obj[source]: self.src_to_obj[source][obj] = [] for idx in indices: if idx not in self.src_to_obj[source][obj]: self.src_to_obj[source][obj].append(idx) def _build_rpm_reverse_map(self): """ Create reverse mapping (rpm -> {module: [indices]}) from module_rpm """ for module, rpms in self.module_rpm.items(): for rpm, indices in rpms.items(): if rpm not in self.rpm_to_modules: self.rpm_to_modules[rpm] = {} if module not in self.rpm_to_modules[rpm]: self.rpm_to_modules[rpm][module] = [] for idx in indices: if idx not in self.rpm_to_modules[rpm][module]: self.rpm_to_modules[rpm][module].append(idx) def save_data(self): """ Save collected data to JSON file Output structure: - variants: list of variant names - source-map: source file mappings - obj-src: object/module -> {source: [variant_indices]} - src-obj: source -> {object/module: [variant_indices]} - module-map: module to RPM mappings - module-rpm: module name -> {rpm-name: [variant_indices]} - rpm-modules: rpm-name -> {module name: [variant_indices]} Variant indices correspond to positions in the 'variants' list. """ output = { 'variants': self.variants, 'source-map': { 'obj-src': self.obj_to_src, 'src-obj': self.src_to_obj }, 'module-map': { 'module-rpm': self.module_rpm, 'rpm-modules': self.rpm_to_modules } } filename = self.outprefix + 'kernel-map.json' with open(os.path.join(self.output_dir, filename), 'wt', encoding='utf-8') as f: json.dump(output, f, indent=2, sort_keys=True) # ------------------------------------------------------------------------- # Main Entry Point # ------------------------------------------------------------------------- def collect_data(self, no_modules=False, no_builtin=False, modorder='modules.order', input_file='', module_lists=None, vmlinux_rpm=''): """ Main entry point - collect all data and save to JSON """ self.load_existing(input_file) self.add_variant() self.collect_modules(no_modules, modorder) self.collect_builtin(no_builtin) self.collect_module_rpm(module_lists or [], vmlinux_rpm) self._build_reverse_map() self._build_rpm_reverse_map() self.save_data() # ============================================================================= # Main # ============================================================================= def main(): """ Main entry point """ args = parse_args() start = time.monotonic() source_map = SourceMap( build_dir=args.directory, output_dir=args.outputdir, outprefix=args.prefix, rhel=args.rhel, variant=args.variant) source_map.collect_data( no_modules=args.no_modules, no_builtin=args.no_builtin, input_file=args.input, module_lists=args.module_list, vmlinux_rpm=args.vmlinux_rpm) if args.time: elapsed = time.monotonic() - start minutes, seconds = divmod(int(elapsed), 60) print(f'Elapsed time: {minutes:02d}:{seconds:02d}') if __name__ == '__main__': main()