diff --git a/Changelog b/Changelog index ff5be54e..ab83bfd8 100644 --- a/Changelog +++ b/Changelog @@ -4,6 +4,12 @@ - make "product_name" "name" instead. KISS - hard set product_path to 'Packages' - Use group metadata from repos instead of our own comps file +- Get group files out of configured repos and create a mashup + of the comps. Filter it and make use of it when creating repos. +- Move _doRunCommand into pypungi/__init__.py so that it can be + easily used by all modules. Also pass in a logger for correct + logging +- Quiet down creatrepo calls * Thu Aug 23 2007 Jesse Keating - Add a source config for the Fedora spin diff --git a/MANIFEST.in b/MANIFEST.in index 422e3f93..dc102432 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,3 +8,4 @@ include TESTING include ToDo include pungi.spec include config/* +include share/* diff --git a/pungi b/pungi index 289927f5..08ff86ec 100755 --- a/pungi +++ b/pungi @@ -94,6 +94,7 @@ def main(): mygather = pypungi.gather.Gather(config, pkglist) mygather.getPackageObjects() mygather.downloadPackages() + mygather.makeCompsFile() if not opts.nosource: mygather.getSRPMList() mygather.downloadSRPMs() diff --git a/pungi.spec b/pungi.spec index 474593f3..4fcb6737 100644 --- a/pungi.spec +++ b/pungi.spec @@ -43,6 +43,7 @@ rm -rf $RPM_BUILD_ROOT # For noarch packages: sitelib %{python_sitelib}/pypungi %{_bindir}/pungi +%{_datadir}/pungi %changelog diff --git a/pypungi/__init__.py b/pypungi/__init__.py index 32fdcd95..0fab6fd8 100644 --- a/pypungi/__init__.py +++ b/pypungi/__init__.py @@ -14,6 +14,7 @@ import logging import os +import subprocess class PungiBase(): """The base Pungi class. Set up config items and logging here""" @@ -48,3 +49,21 @@ class PungiBase(): format='%(name)s.%(levelname)s: %(message)s', filename=logfile) + +def _doRunCommand(command, logger, rundir='/tmp', output=subprocess.PIPE, error=subprocess.PIPE): + """Run a command and log the output. Error out if we get something on stderr""" + + + logger.info("Running %s" % ' '.join(command)) + + p1 = subprocess.Popen(command, cwd=rundir, stdout=output, stderr=error, universal_newlines=True) + (out, err) = p1.communicate() + + if out: + logger.debug(out) + + if p1.returncode != 0: + logger.error("Got an error from %s" % command[0]) + logger.error(err) + raise OSError, "Got an error from %s: %s" % (command[0], err) + diff --git a/pypungi/gather.py b/pypungi/gather.py index 8036c15d..ca7a0efd 100755 --- a/pypungi/gather.py +++ b/pypungi/gather.py @@ -338,6 +338,54 @@ class Gather(pypungi.PungiBase): self.logger.info('Finished downloading packages.') + def makeCompsFile(self): + """Gather any comps files we can from repos and merge them into one.""" + + # get our list of repos + repos = self.ayum.repos.repos.values() + + compsstub = '\n\n\n' + + closestub = '\n\n' + + ourcompspath = os.path.join(self.workdir, '%s-%s-comps.xml' % (self.config.get('default', 'name'), self.config.get('default', 'version'))) + + ourcomps = open(ourcompspath, 'w') + + ourcomps.write(compsstub) + + # iterate through the list and get what comps we can. + # Strip the first three lines and the last line of substance off + # once done, write it to our comps file + for repo in repos: + try: + groupfile = repo.getGroups() + except yum.Errors.RepoMDError, e: + self.logger.warn("No group data found for %s" % repo.id) + pass + else: + compslines = open(groupfile, 'r').readlines() + for line in compslines: + if line.startswith(''): + end = compslines.index(line) + + for line in compslines: + if line.startswith(''): + start = compslines.index(line) + 1 + + ourcomps.writelines(compslines[start:end]) + + ourcomps.write(closestub) + ourcomps.close() + + # Run the xslt filter over our comps file + compsfilter = ['/usr/bin/xsltproc', '--novalid'] + compsfilter.append('-o') + compsfilter.append(ourcompspath) + compsfilter.append('/usr/share/pungi/comps-cleanup.xsl') + compsfilter.append(ourcompspath) + + pypungi._doRunCommand(compsfilter, self.logger) def downloadSRPMs(self): """Cycle through the list of srpms and diff --git a/pypungi/pungi.py b/pypungi/pungi.py index 6f4d5881..2b12cbf0 100755 --- a/pypungi/pungi.py +++ b/pypungi/pungi.py @@ -67,38 +67,24 @@ class Pungi(pypungi.PungiBase): if subfile.startswith(self.destdir): return subfile.replace(self.destdir + os.path.sep, '') - def _doRunCommand(self, command, rundir='/tmp', output=subprocess.PIPE, error=subprocess.PIPE): - """Run a command and log the output. Error out if we get something on stderr""" - - - self.logger.info("Running %s" % ' '.join(command)) - - p1 = subprocess.Popen(command, cwd=rundir, stdout=output, stderr=error, universal_newlines=True) - (out, err) = p1.communicate() - - if out: - self.logger.debug(out) - - if p1.returncode != 0: - self.logger.error("Got an error from %s" % command[0]) - self.logger.error(err) - raise OSError, "Got an error from %s: %s" % (command[0], err) - def doCreaterepo(self): """Run createrepo to generate repodata in the tree.""" + compsfile = os.path.join(self.workdir, '%s-%s-comps.xml' % (self.config.get('default', 'name'), self.config.get('default', 'version'))) + # setup the createrepo call createrepo = ['/usr/bin/createrepo'] + createrepo.append('--quiet') createrepo.append('--database') createrepo.append('--groupfile') - createrepo.append(self.config.get('default', 'comps')) + createrepo.append(compsfile) createrepo.append(self.topdir) # run the command - self._doRunCommand(createrepo, rundir=self.topdir) + pypungi._doRunCommand(createrepo, self.logger, rundir=self.topdir) def doBuildinstall(self): """Run anaconda-runtime's buildinstall on the tree.""" @@ -131,7 +117,7 @@ class Pungi(pypungi.PungiBase): buildinstall.append(self.topdir) # run the command - self._doRunCommand(buildinstall) + pypungi._doRunCommand(buildinstall, self.logger) # write out the tree data for snake self.writeinfo('tree: %s' % self.mkrelative(self.topdir)) @@ -153,7 +139,7 @@ class Pungi(pypungi.PungiBase): pkgorder.append(self.config.get('default', 'product_path')) # run the command - self._doRunCommand(pkgorder, output=pkgorderfile) + pypungi._doRunCommand(pkgorder, self.logger, output=pkgorderfile) pkgorderfile.close() def doGetRelnotes(self): @@ -293,12 +279,15 @@ class Pungi(pypungi.PungiBase): discinfo = open(os.path.join(self.topdir, '.discinfo'), 'r').readlines() mediaid = discinfo[0].rstrip('\n') + compsfile = os.path.join(self.workdir, '%s-%s-comps.xml' % (self.config.get('default', 'name'), self.config.get('default', 'version'))) + # set up the process createrepo = ['/usr/bin/createrepo'] + createrepo.append('--quiet') createrepo.append('--database') createrepo.append('--groupfile') - createrepo.append(self.config.get('default', 'comps')) + createrepo.append(compsfile) createrepo.append('--baseurl') createrepo.append('media://%s' % mediaid) @@ -322,7 +311,7 @@ class Pungi(pypungi.PungiBase): createrepo.append('%s-disc%s' % (self.topdir, disc)) # run the command - self._doRunCommand(createrepo) + pypungi._doRunCommand(createrepo, self.logger) def doCreateIsos(self): """Create isos from the various split directories.""" @@ -384,15 +373,15 @@ class Pungi(pypungi.PungiBase): extraargs.append(os.path.join('%s-disc%s' % (self.topdir, disc))) # run the command - self._doRunCommand(mkisofs + extraargs) + pypungi._doRunCommand(mkisofs + extraargs, self.logger) # implant md5 for mediacheck on all but source arches if not self.config.get('default', 'arch') == 'source': - self._doRunCommand(['/usr/lib/anaconda-runtime/implantisomd5', isofile]) + pypungi._doRunCommand(['/usr/lib/anaconda-runtime/implantisomd5', isofile], self.logger) # shove the sha1sum into a file sha1file = open(os.path.join(self.isodir, 'SHA1SUM'), 'a') - self._doRunCommand(['/usr/bin/sha1sum', isoname], rundir=self.isodir, output=sha1file) + pypungi._doRunCommand(['/usr/bin/sha1sum', isoname], self.logger, rundir=self.isodir, output=sha1file) sha1file.close() # keep track of the CD images we've written @@ -450,15 +439,15 @@ class Pungi(pypungi.PungiBase): extraargs.append(os.path.join(self.archdir, 'SRPMS')) # run the command - self._doRunCommand(mkisofs + extraargs) + pypungi._doRunCommand(mkisofs + extraargs, self.logger) # implant md5 for mediacheck on all but source arches if not self.config.get('default', 'arch') == 'source': - self._doRunCommand(['/usr/lib/anaconda-runtime/implantisomd5', isofile]) + pypungi._doRunCommand(['/usr/lib/anaconda-runtime/implantisomd5', isofile], self.logger) # shove the sha1sum into a file sha1file = open(os.path.join(self.isodir, 'SHA1SUM'), 'a') - self._doRunCommand(['/usr/bin/sha1sum', isoname], rundir=self.isodir, output=sha1file) + pypungi._doRunCommand(['/usr/bin/sha1sum', isoname], self.logger, rundir=self.isodir, output=sha1file) sha1file.close() # return the .discinfo file @@ -490,7 +479,7 @@ class Pungi(pypungi.PungiBase): rescue.append(self.config.get('default', 'product_path')) # run the command - self._doRunCommand(rescue) + pypungi._doRunCommand(rescue, self.logger) # write the iso extraargs = [] @@ -515,11 +504,11 @@ class Pungi(pypungi.PungiBase): extraargs.append(os.path.join(self.workdir, "%s-rescueimage" % self.config.get('default', 'arch'))) # run the command - self._doRunCommand(mkisofs + extraargs) + pypungi._doRunCommand(mkisofs + extraargs, self.logger) # shove the sha1sum into a file sha1file = open(os.path.join(self.isodir, 'SHA1SUM'), 'a') - self._doRunCommand(['/usr/bin/sha1sum', isoname], rundir=self.isodir, output=sha1file) + pypungi._doRunCommand(['/usr/bin/sha1sum', isoname], self.logger, self.logger, rundir=self.isodir, output=sha1file) sha1file.close() # Do some clean up diff --git a/setup.py b/setup.py index 7f3c3eb3..c566cb76 100644 --- a/setup.py +++ b/setup.py @@ -10,6 +10,6 @@ setup(name='pungi', license='GPL', packages = ['pypungi'], scripts = ['pungi'], - data_files=[('/etc/pungi', glob.glob('config/*'))] + data_files=[('/etc/pungi', glob.glob('config/*')), ('/usr/share/pungi', glob.glob('share/*'))] )