Added modules and their dependencies to initrd.img

etc/templates/includes/initrd/initrd.common

@@ -2,6 +2,9 @@
 
 # create required directories
 makedir @initrd@/modules
+makedir @initrd@/firmware
+makedir @initrd@/lib/modules
+makedir @initrd@/lib/firmware
 makedir @initrd@/sbin
 makedir @initrd@/dev
 makedir @initrd@/etc

src/pylorax/_backup/utils.py

@@ -8,60 +8,6 @@ import logging
 logging.basicConfig(level=logging.DEBUG)
 logger = logging.getLogger('pylorax')
 
-
-def genmodinfo(path, output):
-    mods = {}
-    for root, dirs, files in os.walk(path):
-        for file in files:
-            mods[file] = os.path.join(root, file)
-
-    modules = { 'scsi_hostadapter': ['block'], 'eth': ['networking'] }
-    blacklist = ('floppy', 'scsi_mod', 'libiscsi')
-
-    list = {}
-    for modtype in modules:
-        list[modtype] = {}
-        for file in modules[modtype]:
-            try:
-                filename = os.path.join(path, 'modules.%s' % file)
-                f = open(filename, 'r')
-            except IOError:
-                logger.error('cannot open file %s', filename)
-                continue
-            else:
-                lines = f.readlines()
-                f.close()
-
-            for line in lines:
-                line = line.strip()
-                if line in mods:
-                    modname, ext = os.path.splitext(line)
-                    if modname in blacklist:
-                        logger.info('skipping %s', modname)
-                        continue
-
-                    outtext = commands.getoutput('modinfo -F description %s' % mods[line])
-                    desc = outtext.split('\n')[0]
-                    desc = desc.strip()
-
-                    # XXX why we need to do this?
-                    desc = desc[:65]
-
-                    if not desc:
-                        desc = '%s driver' % modname
-                        modinfo = '%s\n\t%s\n\t"%s"\n' % (modname, modtype, desc)
-                        list[modtype][modname] = modinfo
-
-    f = open(output, 'a')
-    f.write('Version 0\n')
-    for type in list:
-        modlist = list[type].keys()
-        modlist.sort()
-        for m in modlist:
-            f.write('%s\n' %list[type][m])
-    f.close()
-
-
 # XXX is the input a path to a file?
 def filtermoddeps(input):
     try:

src/pylorax/images.py

@@ -5,6 +5,7 @@ import os
 import commands
 import re
 import datetime
+import fnmatch
 
 import actions
 import actions.base
@@ -12,6 +13,7 @@ from config import Template
 
 from utils.fileutils import cp, mv, rm, touch, edit, replace
 from utils.ldd import LDD
+from utils.genmodinfo import genmodinfo
 
 
 class InitRD(object):
@@ -81,6 +83,106 @@ class InitRD(object):
         for action in self.template.actions + self._actions:
             action.execute()
 
+    def getModules(self):
+        modlist = os.path.join(self.conf.treedir, 'usr', 'lib', 'anaconda-runtime', 'modlist')
+        modinfo = os.path.join(self.conf.tempdir, 'modinfo')
+        genmodinfo(os.path.join(self.conf.treedir, 'lib', 'modules', self.conf.kernelver), modinfo)
+
+        modfiles = []
+        modfiles.append(os.path.join(self.conf.confdir, 'modules', 'modules'))
+        modfiles.append(os.path.join(self.conf.confdir, 'modules', self.conf.buildarch, 'modules'))
+
+        modules = set()
+        for file in modfiles:
+            if os.path.isfile(file):
+                f = open(file, 'r')
+                lines = f.readlines()
+                f.close()
+
+                for line in lines:
+                    line = line.strip()
+                    line, sep, comment = line.partition('#')
+                    if line.startswith('-'):
+                        modules.discard(line[1:])
+                    elif line.startswith('='):
+                        cmd = '%s --modinfo-file %s %s' % (modlist, modinfo, line[1:])
+                        output = commands.getoutput(cmd)
+                        for module in output.splitlines():
+                            modules.add(module)
+                    else:
+                        modules.add(line)
+
+        # resolve deps
+        depfile = os.path.join(self.conf.treedir, 'lib', 'modules', self.conf.kernelver, 'modules.dep')
+        f = open(depfile, 'r')
+        lines = f.readlines()
+        f.close()
+
+        changed = True
+        while changed:
+            for line in lines:
+                changed = False
+                line = line.strip()
+                m = re.match(r'^.*/(?P<name>.*)\.ko:(?P<deps>.*)$', line)
+                modname = m.group('name')
+                if modname in modules:
+                    for dep in m.group('deps').split():
+                        m = re.match(r'^.*/(?P<name>.*)\.ko$', dep)
+                        if m.group('name') not in modules:
+                            changed = True
+                            print('Adding module dependency %s' % m.group('name'))
+                            modules.add(m.group('name'))
+
+        srcdir = os.path.join(self.conf.treedir, 'lib', 'modules')
+        dstdir = os.path.join(self.conf.initrddir, 'lib', 'modules')
+        cp(os.path.join(srcdir, '*'), dstdir)
+
+        for root, dirs, files in os.walk(dstdir):
+            for file in files:
+                name, ext = os.path.splitext(file)
+                if ext == '.ko' and name not in modules:
+                    print('Removing %s module' % name)
+                    rm(os.path.join(root, file))
+
+        # copy firmware
+        srcdir = os.path.join(self.conf.treedir, 'lib', 'firmware')
+        dstdir = os.path.join(self.conf.initrddir, 'lib', 'firmware')
+        
+        fw = ( ('ipw2100', 'ipw2100*'),
+               ('ipw2200', 'ipw2200*'),
+               ('iwl3945', 'iwlwifi-3945*'),
+               ('iwl4965', 'iwlwifi-4965*'),
+               ('atmel', 'atmel_*.bin'),
+               ('zd1211rw', 'zd1211'),
+               ('qla2xxx', 'ql*') )
+
+        for module, file in fw:
+            if module in modules:
+                print('Copying %s firmware' % module)
+                cp(os.path.join(srcdir, file), dstdir)
+
+        # create modinfo
+        dst = os.path.join(self.conf.initrddir, 'lib', 'modules', 'module-info')
+        cmd = '%s --modinfo-file %s --ignore-missing --modinfo %s > %s' % \
+              (modlist, modinfo, ' '.join(list(modules)), dst)
+        commands.getoutput(cmd)
+
+        # compress modules
+        cmd = 'find -H %s -type f -name *.ko -exec gzip -9 {} \\;' % \
+              os.path.join(self.conf.initrddir, 'lib', 'modules')
+        commands.getoutput(cmd)
+
+        # run depmod
+        cmd = '/sbin/depmod -a -F %s -b %s %s' % \
+              (os.path.join(self.conf.treedir, 'boot', 'System.map-%s' % self.conf.kernelver),
+               self.conf.initrddir, self.conf.kernelver)
+        commands.getoutput(cmd)
+
+        # remove leftovers
+        rm(os.path.join(self.conf.initrddir, 'lib', 'modules', self.conf.kernelver, 'modules.*map'))
+        rm(os.path.join(self.conf.initrddir, 'lib', 'modules', self.conf.kernelver, 'source'))
+        rm(os.path.join(self.conf.initrddir, 'lib', 'modules', self.conf.kernelver, 'build'))
+
     def create(self, dst):
         # create the productfile
         text = '%s\n' % self.conf.imageuuid
@@ -89,6 +191,9 @@ class InitRD(object):
         text = text + '%s\n' % self.conf.bugurl
         edit(os.path.join(self.conf.initrddir, '.buildstamp'), text)
 
+        # get modules
+        self.getModules()
+
         # create the initrd
         err, output = commands.getstatusoutput('find %s | cpio --quiet -c -o | gzip -9 > %s' %
                                                (self.conf.initrddir, dst))
@@ -142,7 +247,7 @@ class Images(object):
 
     def installPackages(self):
         # required packages
-        self.yum.addPackages(['anaconda', 'anaconda-runtime', 'kernel', 'syslinux'])
+        self.yum.addPackages(['anaconda', 'anaconda-runtime', 'kernel', '*firmware*', 'syslinux'])
 
         # optional packages from confdir
         packages_files = []
@@ -231,6 +336,15 @@ class Images(object):
             # copy the grub.conf
             cp(os.path.join(bootdiskdir, 'grub.conf'), self.isodir)
 
+            # copy the vmlinuz
+            self.conf.addAttr('kernelver')
+            for file in os.listdir(os.path.join(self.conf.treedir, 'boot')):
+                if fnmatch.fnmatch(file, 'vmlinuz-*'):
+                    vmlinuz = os.path.join(self.conf.treedir, 'boot', file)
+                    m = re.match(r'.*vmlinuz-(?P<ver>.*)', vmlinuz)
+                    self.conf.set(kernelver=m.group('ver'))
+                    cp(vmlinuz, os.path.join(self.isodir, 'vmlinuz'))
+            
             # XXX do we want this here?
             # create the initrd.img
             print('Creating the initrd.img')
@@ -239,10 +353,6 @@ class Images(object):
             self.initrd.create(os.path.join(self.isodir, 'initrd.img'))
             #self.initrd.cleanUp()
 
-            # copy the vmlinuz
-            vmlinuz = os.path.join(self.conf.treedir, 'boot', 'vmlinuz-*')
-            cp(vmlinuz, os.path.join(self.isodir, 'vmlinuz'))
-
             # copy the splash files
             vesasplash = os.path.join(anacondadir, 'syslinux-vesa-splash.jpg')
             if os.path.isfile(vesasplash):

src/pylorax/utils/fileutils.py

@@ -33,14 +33,15 @@ def mv(src, dst, mode=None, verbose=False):
     return errors
 
 def rm(target, verbose=False):
-    if os.path.isdir(target):
+    for name in glob.iglob(target):
-        if verbose:
+        if os.path.isdir(name):
-            print('removing directory "%s"' % target)
+            if verbose:
-        shutil.rmtree(target, ignore_errors=True)
+                print('removing directory "%s"' % name)
-    else:
+            shutil.rmtree(name, ignore_errors=True)
-        if verbose:
+        else:
-            print('removing file "%s"' % target)
+            if verbose:
-        os.unlink(target)
+                print('removing file "%s"' % name)
+            os.unlink(name)
 
     return True
 

src/pylorax/utils/genmodinfo.py

@@ -0,0 +1,60 @@
+import sys
+import os
+import commands
+
+import logging
+logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger('pylorax')
+
+
+def genmodinfo(path, output):
+    mods = {}
+    for root, dirs, files in os.walk(path):
+        for file in files:
+            mods[file] = os.path.join(root, file)
+
+    modules = { 'scsi_hostadapter': ['block'], 'eth': ['networking'] }
+    blacklist = ('floppy', 'scsi_mod', 'libiscsi')
+
+    list = {}
+    for modtype in modules:
+        list[modtype] = {}
+        for file in modules[modtype]:
+            try:
+                filename = os.path.join(path, 'modules.%s' % file)
+                f = open(filename, 'r')
+            except IOError:
+                logger.error('cannot open file %s', filename)
+                continue
+            else:
+                lines = f.readlines()
+                f.close()
+
+            for line in lines:
+                line = line.strip()
+                if line in mods:
+                    modname, ext = os.path.splitext(line)
+                    if modname in blacklist:
+                        logger.info('skipping %s', modname)
+                        continue
+
+                    outtext = commands.getoutput('modinfo -F description %s' % mods[line])
+                    desc = outtext.split('\n')[0]
+                    desc = desc.strip()
+
+                    # XXX why we need to do this?
+                    desc = desc[:65]
+
+                    if not desc:
+                        desc = '%s driver' % modname
+                        modinfo = '%s\n\t%s\n\t"%s"\n' % (modname, modtype, desc)
+                        list[modtype][modname] = modinfo
+
+    f = open(output, 'a')
+    f.write('Version 0\n')
+    for type in list:
+        modlist = list[type].keys()
+        modlist.sort()
+        for m in modlist:
+            f.write('%s\n' %list[type][m])
+    f.close()