- Get group files out of configured repos and create a mashup

of the comps.  Filter it and make use of it when creating repos.
- Move _doRunCommand into pypungi/__init__.py so that it can be
  easily used by all modules.  Also pass in a logger for correct
  logging
- Quiet down creatrepo calls

Changelog

@@ -4,6 +4,12 @@
 - make "product_name" "name" instead.  KISS
 - hard set product_path to 'Packages'
 - Use group metadata from repos instead of our own comps file
+- Get group files out of configured repos and create a mashup
+  of the comps.  Filter it and make use of it when creating repos.
+- Move _doRunCommand into pypungi/__init__.py so that it can be
+  easily used by all modules.  Also pass in a logger for correct
+  logging
+- Quiet down creatrepo calls
 
 * Thu Aug 23 2007 Jesse Keating <jkeating@redhat.com>
 - Add a source config for the Fedora spin

MANIFEST.in

@@ -8,3 +8,4 @@ include TESTING
 include ToDo
 include pungi.spec
 include config/*
+include share/*

pungi

@@ -94,6 +94,7 @@ def main():
             mygather = pypungi.gather.Gather(config, pkglist)
             mygather.getPackageObjects()
             mygather.downloadPackages()
+            mygather.makeCompsFile()
             if not opts.nosource:
                 mygather.getSRPMList()
                 mygather.downloadSRPMs()

pungi.spec

@@ -43,6 +43,7 @@ rm -rf $RPM_BUILD_ROOT
 # For noarch packages: sitelib
 %{python_sitelib}/pypungi
 %{_bindir}/pungi
+%{_datadir}/pungi
 
 
 %changelog

pypungi/__init__.py

@@ -14,6 +14,7 @@
 
 import logging
 import os
+import subprocess
 
 class PungiBase():
     """The base Pungi class.  Set up config items and logging here"""
@@ -48,3 +49,21 @@ class PungiBase():
                             format='%(name)s.%(levelname)s: %(message)s',
                             filename=logfile)
 
+
+def _doRunCommand(command, logger, rundir='/tmp', output=subprocess.PIPE, error=subprocess.PIPE):
+    """Run a command and log the output.  Error out if we get something on stderr"""
+
+
+    logger.info("Running %s" % ' '.join(command))
+
+    p1 = subprocess.Popen(command, cwd=rundir, stdout=output, stderr=error, universal_newlines=True)
+    (out, err) = p1.communicate()
+
+    if out:
+        logger.debug(out)
+
+    if p1.returncode != 0:
+        logger.error("Got an error from %s" % command[0])
+        logger.error(err)
+        raise OSError, "Got an error from %s: %s" % (command[0], err)
+

pypungi/gather.py

@@ -338,6 +338,54 @@ class Gather(pypungi.PungiBase):
 
         self.logger.info('Finished downloading packages.')
 
+    def makeCompsFile(self):
+        """Gather any comps files we can from repos and merge them into one."""
+
+        # get our list of repos
+        repos = self.ayum.repos.repos.values()
+
+        compsstub = '<?xml version="1.0" encoding="UTF-8"?>\n<!DOCTYPE comps PUBLIC "-//Red Hat, Inc.//DTD Comps info//EN" "comps.dtd">\n<comps>\n'
+
+        closestub = '\n</comps>\n'
+
+        ourcompspath = os.path.join(self.workdir, '%s-%s-comps.xml' % (self.config.get('default', 'name'), self.config.get('default', 'version')))
+
+        ourcomps = open(ourcompspath, 'w')
+
+        ourcomps.write(compsstub)
+
+        # iterate through the list and get what comps we can.
+        # Strip the first three lines and the last line of substance off
+        # once done, write it to our comps file
+        for repo in repos:
+            try:
+                groupfile = repo.getGroups()
+            except yum.Errors.RepoMDError, e:
+                self.logger.warn("No group data found for %s" % repo.id)
+                pass
+            else:
+                compslines = open(groupfile, 'r').readlines()
+                for line in compslines:
+                    if line.startswith('</comps>'):
+                        end = compslines.index(line)
+
+                for line in compslines:
+                    if line.startswith('<comps>'):
+                        start = compslines.index(line) + 1
+                
+                ourcomps.writelines(compslines[start:end])
+
+        ourcomps.write(closestub)
+        ourcomps.close()
+
+        # Run the xslt filter over our comps file
+        compsfilter = ['/usr/bin/xsltproc', '--novalid']
+        compsfilter.append('-o')
+        compsfilter.append(ourcompspath)
+        compsfilter.append('/usr/share/pungi/comps-cleanup.xsl')
+        compsfilter.append(ourcompspath)
+
+        pypungi._doRunCommand(compsfilter, self.logger)
 
     def downloadSRPMs(self):
         """Cycle through the list of srpms and

pypungi/pungi.py

@@ -67,38 +67,24 @@ class Pungi(pypungi.PungiBase):
         if subfile.startswith(self.destdir):
             return subfile.replace(self.destdir + os.path.sep, '')
 
-    def _doRunCommand(self, command, rundir='/tmp', output=subprocess.PIPE, error=subprocess.PIPE):
-        """Run a command and log the output.  Error out if we get something on stderr"""
-
-
-        self.logger.info("Running %s" % ' '.join(command))
-
-        p1 = subprocess.Popen(command, cwd=rundir, stdout=output, stderr=error, universal_newlines=True)
-        (out, err) = p1.communicate()
-
-        if out:
-            self.logger.debug(out)
-
-        if p1.returncode != 0:
-            self.logger.error("Got an error from %s" % command[0])
-            self.logger.error(err)
-            raise OSError, "Got an error from %s: %s" % (command[0], err)
-
     def doCreaterepo(self):
         """Run createrepo to generate repodata in the tree."""
 
 
+        compsfile = os.path.join(self.workdir, '%s-%s-comps.xml' % (self.config.get('default', 'name'), self.config.get('default', 'version')))
+
         # setup the createrepo call
         createrepo = ['/usr/bin/createrepo']
+        createrepo.append('--quiet')
         createrepo.append('--database')
 
         createrepo.append('--groupfile')
-        createrepo.append(self.config.get('default', 'comps'))
+        createrepo.append(compsfile)
 
         createrepo.append(self.topdir)
 
         # run the command
-        self._doRunCommand(createrepo, rundir=self.topdir)
+        pypungi._doRunCommand(createrepo, self.logger, rundir=self.topdir)
 
     def doBuildinstall(self):
         """Run anaconda-runtime's buildinstall on the tree."""
@@ -131,7 +117,7 @@ class Pungi(pypungi.PungiBase):
         buildinstall.append(self.topdir)
 
         # run the command
-        self._doRunCommand(buildinstall)
+        pypungi._doRunCommand(buildinstall, self.logger)
 
         # write out the tree data for snake
         self.writeinfo('tree: %s' % self.mkrelative(self.topdir))
@@ -153,7 +139,7 @@ class Pungi(pypungi.PungiBase):
         pkgorder.append(self.config.get('default', 'product_path'))
 
         # run the command
-        self._doRunCommand(pkgorder, output=pkgorderfile)
+        pypungi._doRunCommand(pkgorder, self.logger, output=pkgorderfile)
         pkgorderfile.close()
 
     def doGetRelnotes(self):
@@ -293,12 +279,15 @@ class Pungi(pypungi.PungiBase):
             discinfo = open(os.path.join(self.topdir, '.discinfo'), 'r').readlines()
         mediaid = discinfo[0].rstrip('\n')
 
+        compsfile = os.path.join(self.workdir, '%s-%s-comps.xml' % (self.config.get('default', 'name'), self.config.get('default', 'version')))
+
         # set up the process
         createrepo = ['/usr/bin/createrepo']
+        createrepo.append('--quiet')
         createrepo.append('--database')
 
         createrepo.append('--groupfile')
-        createrepo.append(self.config.get('default', 'comps'))
+        createrepo.append(compsfile)
 
         createrepo.append('--baseurl')
         createrepo.append('media://%s' % mediaid)
@@ -322,7 +311,7 @@ class Pungi(pypungi.PungiBase):
                 createrepo.append('%s-disc%s' % (self.topdir, disc))
 
         # run the command
-        self._doRunCommand(createrepo)
+        pypungi._doRunCommand(createrepo, self.logger)
 
     def doCreateIsos(self):
         """Create isos from the various split directories."""
@@ -384,15 +373,15 @@ class Pungi(pypungi.PungiBase):
                 extraargs.append(os.path.join('%s-disc%s' % (self.topdir, disc)))
 
                 # run the command
-                self._doRunCommand(mkisofs + extraargs)
+                pypungi._doRunCommand(mkisofs + extraargs, self.logger)
 
                 # implant md5 for mediacheck on all but source arches
                 if not self.config.get('default', 'arch') == 'source':
-                    self._doRunCommand(['/usr/lib/anaconda-runtime/implantisomd5', isofile])
+                    pypungi._doRunCommand(['/usr/lib/anaconda-runtime/implantisomd5', isofile], self.logger)
 
                 # shove the sha1sum into a file
                 sha1file = open(os.path.join(self.isodir, 'SHA1SUM'), 'a')
-                self._doRunCommand(['/usr/bin/sha1sum', isoname], rundir=self.isodir, output=sha1file)
+                pypungi._doRunCommand(['/usr/bin/sha1sum', isoname], self.logger, rundir=self.isodir, output=sha1file)
                 sha1file.close()
 
                 # keep track of the CD images we've written
@@ -450,15 +439,15 @@ class Pungi(pypungi.PungiBase):
                 extraargs.append(os.path.join(self.archdir, 'SRPMS'))
 
             # run the command
-            self._doRunCommand(mkisofs + extraargs)
+            pypungi._doRunCommand(mkisofs + extraargs, self.logger)
 
             # implant md5 for mediacheck on all but source arches
             if not self.config.get('default', 'arch') == 'source':
-                self._doRunCommand(['/usr/lib/anaconda-runtime/implantisomd5', isofile])
+                pypungi._doRunCommand(['/usr/lib/anaconda-runtime/implantisomd5', isofile], self.logger)
 
             # shove the sha1sum into a file
             sha1file = open(os.path.join(self.isodir, 'SHA1SUM'), 'a')
-            self._doRunCommand(['/usr/bin/sha1sum', isoname], rundir=self.isodir, output=sha1file)
+            pypungi._doRunCommand(['/usr/bin/sha1sum', isoname], self.logger, rundir=self.isodir, output=sha1file)
             sha1file.close()
 
             # return the .discinfo file
@@ -490,7 +479,7 @@ class Pungi(pypungi.PungiBase):
             rescue.append(self.config.get('default', 'product_path'))
 
             # run the command
-            self._doRunCommand(rescue)
+            pypungi._doRunCommand(rescue, self.logger)
 
             # write the iso
             extraargs = []
@@ -515,11 +504,11 @@ class Pungi(pypungi.PungiBase):
             extraargs.append(os.path.join(self.workdir, "%s-rescueimage" % self.config.get('default', 'arch')))
 
             # run the command
-            self._doRunCommand(mkisofs + extraargs)
+            pypungi._doRunCommand(mkisofs + extraargs, self.logger)
 
             # shove the sha1sum into a file
             sha1file = open(os.path.join(self.isodir, 'SHA1SUM'), 'a')
-            self._doRunCommand(['/usr/bin/sha1sum', isoname], rundir=self.isodir, output=sha1file)
+            pypungi._doRunCommand(['/usr/bin/sha1sum', isoname], self.logger, self.logger, rundir=self.isodir, output=sha1file)
             sha1file.close()
 
         # Do some clean up

setup.py

@@ -10,6 +10,6 @@ setup(name='pungi',
       license='GPL',
       packages = ['pypungi'],
       scripts = ['pungi'],
-      data_files=[('/etc/pungi', glob.glob('config/*'))]
+      data_files=[('/etc/pungi', glob.glob('config/*')), ('/usr/share/pungi', glob.glob('share/*'))]
       )