Merge #456 Add tests for dependency solving

bin/pungi

@@ -12,11 +12,19 @@
 # along with this program; if not, see <https://gnu.org/licenses/>.
 
 import os
+import selinux
+import sys
+
+here = sys.path[0]
+if here != '/usr/bin':
+    # Git checkout
+    sys.path.insert(1, os.path.dirname(here))
+
 from pungi import get_full_version
 import pungi.gather
 import pungi.config
 import pungi.ks
-import selinux
+
 
 def main():
 

pungi/config.py

@@ -24,7 +24,7 @@ from ConfigParser import SafeConfigParser
 
 # In development, `here` will point to the bin/ directory with scripts.
 here = sys.path[0]
-MULTILIBCONF = (os.path.join(os.path.dirname(here), 'share', 'multilib')
+MULTILIBCONF = (os.path.join(os.path.dirname(__file__), '..', 'share', 'multilib')
                 if here != '/usr/bin'
                 else '/usr/share/pungi/multilib')
 

pungi/gather.py

@@ -150,7 +150,17 @@ class PungiBase(object):
 class CallBack(urlgrabber.progress.TextMeter):
     """A call back function used with yum."""
 
-    def progressbar(self, current, total, name=None):
+    def __init__(self, logger):
+        self.logger = logger
+
+    def start(self, filename=None, url=None, basename=None, size=None, now=None, text=None):
+        self.logger.info('Downloading %s (%sB)'
+                         % (text, urlgrabber.progress.format_number(size)))
+
+    def update(self, amount_read, name=None):
+        return
+
+    def end(self, amount_read, now=None):
         return
 
 
@@ -204,12 +214,13 @@ class Pungi(PungiBase):
         lock = lockfile.LockFile(filename)
         self.yumlock = ReentrantYumLock(lock, self.logger)
 
-        # Create the stdout/err streams and only send INFO+ stuff there
+        if not self.logger.handlers:
-        formatter = logging.Formatter('%(name)s:%(levelname)s: %(message)s')
+            # Create the stdout/err streams and only send INFO+ stuff there
-        console = logging.StreamHandler()
+            formatter = logging.Formatter('%(name)s:%(levelname)s: %(message)s')
-        console.setFormatter(formatter)
+            console = logging.StreamHandler()
-        console.setLevel(logging.INFO)
+            console.setFormatter(formatter)
-        self.logger.addHandler(console)
+            console.setLevel(logging.INFO)
+            self.logger.addHandler(console)
 
         self.destdir = self.config.get('pungi', 'destdir')
         self.archdir = os.path.join(self.destdir,
@@ -325,8 +336,8 @@ class Pungi(PungiBase):
         self.ayum.repos.enableRepo(thisrepo.id)
         self.ayum._getRepos(thisrepo=thisrepo.id, doSetup=True)
         # Set the repo callback.
-        self.ayum.repos.setProgressBar(CallBack())
+        self.ayum.repos.setProgressBar(CallBack(logger=self.logger))
-        self.ayum.repos.callback = CallBack()
+        self.ayum.repos.callback = CallBack(logger=self.logger)
         thisrepo.metadata_expire = 0
         thisrepo.mirrorlist_expire = 0
         if os.path.exists(os.path.join(thisrepo.cachedir, 'repomd.xml')):
@@ -864,6 +875,7 @@ class Pungi(PungiBase):
         for name in searchlist:
             pattern = name
             multilib = False
+            orig_name = name
             if name.endswith(".+"):
                 name = name[:-2]
                 multilib = True
@@ -898,7 +910,7 @@ class Pungi(PungiBase):
                     # works for both "none" and "build" greedy methods
                     packages = [self.ayum._bestPackageFromList(packages)]
 
-                if name in input_packages:
+                if orig_name in input_packages:
                     self.input_packages.update(packages)
                 if name in comps_package_names:
                     self.comps_packages.update(packages)
@@ -1104,7 +1116,7 @@ class Pungi(PungiBase):
                     elif po.arch in self.valid_native_arches:
                         has_native = True
                     continue
-                if po.arch in self.valid_multilib_arches and self.greedy_method == "all":
+                if po.arch in self.valid_multilib_arches and (po in self.input_packages or self.greedy_method == "all"):
                     include_multilib = True
                 elif po.arch in self.valid_native_arches:
                     include_native = True

pungi/wrappers/pungi.py

@@ -197,3 +197,58 @@ class PungiWrapper(object):
                 result.setdefault(match.group(2), set()).add(match.group(1))
 
         return result
+
+    def run_pungi(self, ks_file, destdir, name, selfhosting=False, fulltree=False,
+                  greedy='', cache_dir=None, arch='', multilib_methods=[],
+                  nodeps=False, lookaside_repos=[]):
+        """
+        This is a replacement for get_pungi_cmd that runs it in-process. Not
+        all arguments are supported.
+        """
+        from .. import ks, gather, config
+        ksparser = ks.get_ksparser(ks_path=ks_file)
+        cfg = config.Config()
+        cfg.set('pungi', 'destdir', destdir)
+        cfg.set('pungi', 'family', name)
+        cfg.set('pungi', 'iso_basename', name)
+        cfg.set('pungi', 'fulltree', str(fulltree))
+        cfg.set('pungi', 'selfhosting', str(selfhosting))
+        cfg.set('pungi', 'cachedir', cache_dir)
+        cfg.set('pungi', 'full_archlist', "True")
+        cfg.set('pungi', 'workdirbase', "%s/work" % destdir)
+        cfg.set('pungi', 'greedy', greedy)
+        cfg.set('pungi', 'nosource', 'False')
+        cfg.set('pungi', 'nodebuginfo', 'False')
+        cfg.set('pungi', 'force', 'False')
+        cfg.set('pungi', 'resolve_deps', str(not nodeps))
+        if arch:
+            cfg.set('pungi', 'arch', arch)
+        if multilib_methods:
+            cfg.set('pungi', 'multilib', " ".join(multilib_methods))
+        if lookaside_repos:
+            cfg.set('pungi', 'lookaside_repos', " ".join(lookaside_repos))
+
+        mypungi = gather.Pungi(cfg, ksparser)
+
+        with open(os.path.join(destdir, 'out'), 'w') as f:
+            with mypungi.yumlock:
+                mypungi._inityum()
+                mypungi.gather()
+
+                for line in mypungi.list_packages():
+                    flags_str = ",".join(line["flags"])
+                    if flags_str:
+                        flags_str = "(%s)" % flags_str
+                    f.write("RPM%s: %s\n" % (flags_str, line["path"]))
+                mypungi.makeCompsFile()
+                mypungi.getDebuginfoList()
+                for line in mypungi.list_debuginfo():
+                    flags_str = ",".join(line["flags"])
+                    if flags_str:
+                        flags_str = "(%s)" % flags_str
+                    f.write("DEBUGINFO%s: %s\n" % (flags_str, line["path"]))
+                for line in mypungi.list_srpms():
+                    flags_str = ",".join(line["flags"])
+                    if flags_str:
+                        flags_str = "(%s)" % flags_str
+                    f.write("SRPM%s: %s\n" % (flags_str, line["path"]))

tests/data/dummy-comps.xml

@@ -159,5 +159,6 @@
     <match install="openoffice.org-langpack-%s" name="openoffice.org-core"/>
     <match install="tesseract-langpack-%s" name="tesseract"/>
     <match install="tkgate-%s" name="tkgate"/>
+    <match install="dummy-release-notes-%s" name="dummy-release-notes" />
   </langpacks>
 </comps>

tests/fixtures/repos/repo-krb5-lookaside/repodata/repomd.xml

@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">
+  <revision>1473660739</revision>
+  <data type="primary">
+    <checksum type="sha256">e2007d0883d3c7e7b1409b578d6134b55e42a81a2b8568483f6a177dca33a5ab</checksum>
+    <open-checksum type="sha256">46587058c56b96e825286b8554a5b86283520d83ec58a104614d5844bcfd3cdb</open-checksum>
+    <location href="repodata/e2007d0883d3c7e7b1409b578d6134b55e42a81a2b8568483f6a177dca33a5ab-primary.xml.gz"/>
+    <timestamp>1473660739</timestamp>
+    <size>2801</size>
+    <open-size>33779</open-size>
+  </data>
+  <data type="filelists">
+    <checksum type="sha256">d10e4fa40128e91767cbe4f9fa8d647da70b24e252ea5690248c4c4dbeb8c55b</checksum>
+    <open-checksum type="sha256">9f4376455b1ea41d6e006959067acde4605a9b5de50e01f1b651bf7d8dfda033</open-checksum>
+    <location href="repodata/d10e4fa40128e91767cbe4f9fa8d647da70b24e252ea5690248c4c4dbeb8c55b-filelists.xml.gz"/>
+    <timestamp>1473660739</timestamp>
+    <size>1632</size>
+    <open-size>5515</open-size>
+  </data>
+  <data type="other">
+    <checksum type="sha256">b9dcd75dda89076e1fdbc4a194d97ce3efc6dcd5715191fbd459d569e83596fc</checksum>
+    <open-checksum type="sha256">1879f419d0383daf019f80a86a7222f5350984c77f12eced1c2b4a5e01c2d9a1</open-checksum>
+    <location href="repodata/b9dcd75dda89076e1fdbc4a194d97ce3efc6dcd5715191fbd459d569e83596fc-other.xml.gz"/>
+    <timestamp>1473660739</timestamp>
+    <size>1709</size>
+    <open-size>9014</open-size>
+  </data>
+  <data type="primary_db">
+    <checksum type="sha256">aeb3a17a03cc002051da0120cc3095d105fc74bb764a4354cf199c917c074698</checksum>
+    <open-checksum type="sha256">4cc6ddfb9385c77a10e97dac90f09d5a44b0acf600800a3926fd5e28992b4a19</open-checksum>
+    <location href="repodata/aeb3a17a03cc002051da0120cc3095d105fc74bb764a4354cf199c917c074698-primary.sqlite.bz2"/>
+    <timestamp>1473660739</timestamp>
+    <size>6090</size>
+    <open-size>118784</open-size>
+    <database_version>10</database_version>
+  </data>
+  <data type="filelists_db">
+    <checksum type="sha256">4fbc657d01000f79cec7b74ada066dfa66fc745a2745cb2f1e5b36e103078637</checksum>
+    <open-checksum type="sha256">06c7919d67cd29cacd4976a0fcd7f42a03f2787b3fdc8947899ef1742a8803bf</open-checksum>
+    <location href="repodata/4fbc657d01000f79cec7b74ada066dfa66fc745a2745cb2f1e5b36e103078637-filelists.sqlite.bz2"/>
+    <timestamp>1473660739</timestamp>
+    <size>2598</size>
+    <open-size>28672</open-size>
+    <database_version>10</database_version>
+  </data>
+  <data type="other_db">
+    <checksum type="sha256">feada0072a73fd3d224f4ddfb0447c5cfaafa8caab7105920d60244000e17446</checksum>
+    <open-checksum type="sha256">f7e63aca97e99e2553e113ad85943fbfd1e10aaff7f2a449046de2adc6bc376d</open-checksum>
+    <location href="repodata/feada0072a73fd3d224f4ddfb0447c5cfaafa8caab7105920d60244000e17446-other.sqlite.bz2"/>
+    <timestamp>1473660739</timestamp>
+    <size>3055</size>
+    <open-size>24576</open-size>
+    <database_version>10</database_version>
+  </data>
+</repomd>

tests/fixtures/repos/repo/repodata/cd2e022a3f1163bed5dd38328a2c0bd7a8db1589a9e67b25c6341941914e1077-dummy-comps.xml

@@ -0,0 +1,164 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE comps PUBLIC "-//Red Hat, Inc.//DTD Comps info//EN" "comps.dtd">
+<comps>
+
+  <!-- GROUPS -->
+
+  <group>
+    <id>core</id>
+    <name>Core</name>
+    <description>Smallest possible installation</description>
+    <default>true</default>
+    <uservisible>false</uservisible>
+    <packagelist>
+      <packagereq type="mandatory">dummy-bash</packagereq>
+    </packagelist>
+  </group>
+
+  <group>
+    <id>standard</id>
+    <name>Standard</name>
+    <description>Common set of utilities that extend the minimal installation.</description>
+    <default>false</default>
+    <uservisible>true</uservisible>
+    <packagelist>
+      <packagereq>dummy-lvm2</packagereq>
+    </packagelist>
+  </group>
+
+  <group>
+    <id>text-internet</id>
+    <name>Text-based Internet</name>
+    <description>This group includes text-based email, Web, and chat clients.  These applications do not require the X Window System.</description>
+    <default>false</default>
+    <uservisible>true</uservisible>
+    <packagelist>
+      <packagereq type="optional">dummy-elinks</packagereq>
+      <packagereq type="optional">dummy-tftp</packagereq>
+    </packagelist>
+  </group>
+
+  <group>
+    <id>firefox</id>
+    <name>Firefox Web Browser</name>
+    <description>The Firefox web browser</description>
+    <default>false</default>
+    <uservisible>false</uservisible>
+    <packagelist>
+      <packagereq>Dummy-firefox</packagereq>
+      <packagereq>dummy-icedtea-web</packagereq>
+    </packagelist>
+  </group>
+
+  <group arch="i386 x86_64">
+    <id>skype</id>
+    <name>Skype</name>
+    <description>Free internet telephony</description>
+    <default>false</default>
+    <uservisible>true</uservisible>
+    <packagelist>
+      <packagereq>dummy-skype</packagereq>
+    </packagelist>
+  </group>
+
+  <group arch="x86_64">
+    <id>resilient-storage</id>
+    <name>Resilient Storage</name>
+    <description>Clustered storage, including the GFS2 filesystem.</description>
+    <default>false</default>
+    <uservisible>true</uservisible>
+    <packagelist>
+      <packagereq type="mandatory">dummy-gfs2-utils</packagereq>
+      <packagereq type="mandatory">dummy-lvm2-cluster</packagereq>
+      <packagereq type="mandatory">dummy-pacemaker</packagereq>
+      <packagereq type="mandatory">dummy-resource-agents</packagereq>
+    </packagelist>
+  </group>
+
+  <group>
+    <id>gluster</id>
+    <name>Gluster</name>
+    <description>GlusterFS support packages</description>
+    <default>false</default>
+    <uservisible>true</uservisible>
+    <packagelist>
+      <packagereq type="mandatory">dummy-glusterfs-resource-agents</packagereq>
+    </packagelist>
+  </group>
+
+  <group>
+    <id>basic-desktop</id>
+    <name>Desktop</name>
+    <description>Basic Desktop packages</description>
+    <default>true</default>
+    <uservisible>true</uservisible>
+    <packagelist>
+      <packagereq type="conditional" requires="dummy-imsettings">dummy-imsettings-gnome</packagereq>
+    </packagelist>
+  </group>
+
+  <!-- ENVIRONMENTS -->
+
+  <environment>
+    <id>minimal</id>
+    <name>Minimal install</name>
+    <description>Basic functionality.</description>
+    <display_order>99</display_order>
+    <grouplist>
+      <groupid>core</groupid>
+    </grouplist>
+    <optionlist>
+    </optionlist>
+  </environment>
+
+  <environment>
+    <id>desktop</id>
+    <name>Desktop</name>
+    <description>Desktop.</description>
+    <display_order>10</display_order>
+    <grouplist>
+      <groupid>core</groupid>
+      <groupid>standard</groupid>
+      <groupid>basic-desktop</groupid>
+    </grouplist>
+    <optionlist>
+    </optionlist>
+  </environment>
+
+  <environment>
+    <id>empty</id>
+    <name>Empty</name>
+    <description>Should not appear in the repos.</description>
+    <display_order>10</display_order>
+    <grouplist>
+      <groupid>does-not-exist</groupid>
+    </grouplist>
+  </environment>
+
+  <!-- LANGPACKS -->
+
+  <langpacks>
+    <match install="LabPlot-doc-%s" name="LabPlot-doc"/>
+    <match install="aspell-%s" name="aspell"/>
+    <match install="autocorr-%s" name="autocorr-en"/>
+    <match install="calligra-l10n-%s" name="calligra-core"/>
+    <match install="childsplay-alphabet_sounds_%s" name="childsplay"/>
+    <match install="eclipse-nls-%s" name="eclipse-platform"/>
+    <match install="firefox-langpack-%s" name="firefox"/>
+    <match install="gcompris-sound-%s" name="gcompris"/>
+    <match install="gimp-help-%s" name="gimp-help"/>
+    <match install="hunspell-%s" name="hunspell"/>
+    <match install="hyphen-%s" name="hyphen"/>
+    <match install="kde-l10n-%s" name="kdelibs"/>
+    <match install="kde-i18n-%s" name="kdelibs3"/>
+    <match install="libreoffice-langpack-%s" name="libreoffice-core"/>
+    <match install="man-pages-%s" name="man-pages"/>
+    <match install="moodle-%s" name="moodle"/>
+    <match install="mythes-%s" name="mythes"/>
+    <match install="nqc-doc-%s" name="nqc-doc"/>
+    <match install="openoffice.org-langpack-%s" name="openoffice.org-core"/>
+    <match install="tesseract-langpack-%s" name="tesseract"/>
+    <match install="tkgate-%s" name="tkgate"/>
+    <match install="dummy-release-notes-%s" name="dummy-release-notes" />
+  </langpacks>
+</comps>

tests/fixtures/repos/repo/repodata/repomd.xml

@@ -0,0 +1,69 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">
+  <revision>1478525452</revision>
+  <data type="primary">
+    <checksum type="sha256">443dade43e632f2bbb768b1768cb090175429b5e70823525c3a3f4d45b7984ed</checksum>
+    <open-checksum type="sha256">e60486cc17a922a5cfa3b0eeb6b02176d046bb81d4477806ccf76e49f2f619dd</open-checksum>
+    <location href="repodata/443dade43e632f2bbb768b1768cb090175429b5e70823525c3a3f4d45b7984ed-primary.xml.gz"/>
+    <timestamp>1478525452</timestamp>
+    <size>32219</size>
+    <open-size>442634</open-size>
+  </data>
+  <data type="filelists">
+    <checksum type="sha256">05ecf05b96e9d055dc9e3df028b1af2cf6661af7bb082278d96eebe4bf0b9012</checksum>
+    <open-checksum type="sha256">2df95e0dc72d04e62064d15d3b95d3aac7c4f1733ebb67c16beab89345808ceb</open-checksum>
+    <location href="repodata/05ecf05b96e9d055dc9e3df028b1af2cf6661af7bb082278d96eebe4bf0b9012-filelists.xml.gz"/>
+    <timestamp>1478525452</timestamp>
+    <size>19701</size>
+    <open-size>73844</open-size>
+  </data>
+  <data type="other">
+    <checksum type="sha256">74a554f4a2c7392d9ef0cf6a07652d7a198a104a202584e71f57bc247a32f41b</checksum>
+    <open-checksum type="sha256">56bb0c634647446b741246930d55542b6ca50084008678ab441d5f4960d1eaea</open-checksum>
+    <location href="repodata/74a554f4a2c7392d9ef0cf6a07652d7a198a104a202584e71f57bc247a32f41b-other.xml.gz"/>
+    <timestamp>1478525452</timestamp>
+    <size>19712</size>
+    <open-size>117697</open-size>
+  </data>
+  <data type="primary_db">
+    <checksum type="sha256">9249118941570779bf043a7a393f0308f891ac1d7fdd2dfe006d3caf69d26a6f</checksum>
+    <open-checksum type="sha256">c0ecdec1aeac2c4e1f32f97c04c703a438e139f4d9d9631a95e3fca5fec61e5c</open-checksum>
+    <location href="repodata/9249118941570779bf043a7a393f0308f891ac1d7fdd2dfe006d3caf69d26a6f-primary.sqlite.bz2"/>
+    <timestamp>1478525452</timestamp>
+    <size>57571</size>
+    <open-size>368640</open-size>
+    <database_version>10</database_version>
+  </data>
+  <data type="filelists_db">
+    <checksum type="sha256">a5891b48313e9cd503e94e48c93a08b5160ce04f4fde2d1e32f21ef69f550141</checksum>
+    <open-checksum type="sha256">b8398ea076836772a553717e9abfcd6fec1cb7d69ad32c09094b4abf1345aa88</open-checksum>
+    <location href="repodata/a5891b48313e9cd503e94e48c93a08b5160ce04f4fde2d1e32f21ef69f550141-filelists.sqlite.bz2"/>
+    <timestamp>1478525452</timestamp>
+    <size>25706</size>
+    <open-size>94208</open-size>
+    <database_version>10</database_version>
+  </data>
+  <data type="other_db">
+    <checksum type="sha256">129bcb220b766abd0d38b33f919af26b017c6ae086cf087ca00d183d97d57a06</checksum>
+    <open-checksum type="sha256">41a274ac1928cbd1b9126972b154826842af8781a6037fade0ca1bc57443d5d2</open-checksum>
+    <location href="repodata/129bcb220b766abd0d38b33f919af26b017c6ae086cf087ca00d183d97d57a06-other.sqlite.bz2"/>
+    <timestamp>1478525452</timestamp>
+    <size>29618</size>
+    <open-size>122880</open-size>
+    <database_version>10</database_version>
+  </data>
+  <data type="group">
+    <checksum type="sha256">cd2e022a3f1163bed5dd38328a2c0bd7a8db1589a9e67b25c6341941914e1077</checksum>
+    <location href="repodata/cd2e022a3f1163bed5dd38328a2c0bd7a8db1589a9e67b25c6341941914e1077-dummy-comps.xml"/>
+    <timestamp>1478525452</timestamp>
+    <size>5178</size>
+  </data>
+  <data type="group_gz">
+    <checksum type="sha256">2f8cc4b5cf852eca9838393db012ad4e494ccea437ffbbbf95e110a388e15f35</checksum>
+    <open-checksum type="sha256">cd2e022a3f1163bed5dd38328a2c0bd7a8db1589a9e67b25c6341941914e1077</open-checksum>
+    <location href="repodata/2f8cc4b5cf852eca9838393db012ad4e494ccea437ffbbbf95e110a388e15f35-dummy-comps.xml.gz"/>
+    <timestamp>1478525452</timestamp>
+    <size>1244</size>
+    <open-size>5178</open-size>
+  </data>
+</repomd>