Merge #75 Start of development guide

.gitignore

@@ -1,5 +1,6 @@
 *.py[co]
 *~
+*.egg-info
 MANIFEST
 build/*
 dist/*
@@ -7,3 +8,5 @@ doc/_build
 noarch/*
 tests/data/repo
 tests/data/repo-krb5-lookaside
+tests/_composes
+htmlcov/

bin/comps_filter

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 

bin/pungi

@@ -1,4 +1,4 @@
-#!/usr/bin/python -tt
+#!/usr/bin/env python
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; version 2 of the License.

bin/pungi-koji

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 

doc/about.rst

@@ -2,7 +2,7 @@
  About Pungi
 =============
 
-Pungi is a distribution compose tool.
+*Pungi* is a distribution compose tool.
 
 Composes are release snapshots that contain release deliverables such as:
 
@@ -19,6 +19,24 @@ Composes are release snapshots that contain release deliverables such as:
   - images for PXE boot
 
 
+Tool overview
+=============
+
+*Pungi* consists of multiple separate executables backed by a common library.
+
+The main entry-point is the ``pungi-koji`` script. It loads the compose
+configuration and kicks off the process. Composing itself is done in phases.
+Each phase is responsible for generating some artifacts on disk and updating
+the ``compose`` object that is threaded through all the phases.
+
+*Pungi* itself does not actually do that much. Most of the actual work is
+delegated to separate executables. *Pungi* just makes sure that all the
+commands are invoked in the appropriate order and with correct arguments. It
+also moves the artifacts to correct locations.
+
+
 Links
 =====
 - Upstream GIT: https://pagure.io/pungi/
+- Issue tracker: https://pagure.io/pungi/issues
+- Questions can be asked on *#fedora-releng* IRC channel on FreeNode

doc/contributing.rst

@@ -3,6 +3,63 @@ Contributing to Pungi
 =====================
 
 
+Set up development environment
+==============================
+
+In order to work on *Pungi*, you should install *Fedora 23*. These packages
+will have to installed:
+
+ * createrepo
+ * createrepo_c
+ * cvs
+ * genisoimage
+ * gettext
+ * git
+ * isomd5sum
+ * jigdo
+ * kobo
+ * kobo-rpmlib
+ * koji
+ * libselinux-python
+ * lorax
+ * python-kickstart
+ * python-lockfile
+ * python-lxml
+ * python-productmd
+ * repoview
+ * syslinux
+ * yum
+ * yum-utils
+
+For running unit tests, these packages are recommended as well:
+
+ * python-mock
+ * python-nose
+ * python-nose-cov
+
+Technically, it is possible to work on *Fedora 22*, but setting it up is a
+major headache. Packaged version of *productmd* is too old, and *Pungi* does
+not easily work with *virtualenv*, mostly because many of its Python
+dependencies are not available on PyPI. To get it working, you have to install
+*kobo* and *productmd* directly from Git and provide symlinks to other packages
+and files in ``$(virtualenvwrapper_get_site_packages_dir)``. You will still
+need to install all of the non-Python packages above as they are used by
+calling an executable. ::
+
+    $ ln -vs "$(deactivate && python -c 'import os, koji; print os.path.dirname(koji.__file__)')" "$(virtualenvwrapper_get_site_packages_dir)"
+    $ ln -vs "$(deactivate && python -c 'import os, rpm; print os.path.dirname(rpm.__file__)')" "$(virtualenvwrapper_get_site_packages_dir)"
+    $ ln -vs "$(deactivate && python -c 'import os, rpmUtils; print os.path.dirname(rpmUtils.__file__)')" "$(virtualenvwrapper_get_site_packages_dir)"
+    $ ln -vs "$(deactivate && python -c 'import os, yum; print os.path.dirname(yum.__file__)')" "$(virtualenvwrapper_get_site_packages_dir)"
+    $ ln -vs "$(deactivate && python -c 'import os, urlgrabber; print os.path.dirname(urlgrabber.__file__)')" "$(virtualenvwrapper_get_site_packages_dir)"
+    $ ln -vs "$(deactivate && python -c 'import os, krbV; print krbV.__file__')" "$(virtualenvwrapper_get_site_packages_dir)"
+    $ ln -vs "$(deactivate && python -c 'import os, sqlitecachec; print sqlitecachec.__file__')" "$(virtualenvwrapper_get_site_packages_dir)"
+    $ ln -vs "$(deactivate && python -c 'import os, _sqlitecache; print _sqlitecache.__file__')" "$(virtualenvwrapper_get_site_packages_dir)"
+    $ PYCURL_SSL_LIBRARY=nss pip install pycurl
+    $ pip install https://git.fedorahosted.org/cgit/kobo.git/snapshot/kobo-0.4.3.tar.gz#egg=kobo
+    $ pip install https://github.com/release-engineering/productmd/archive/master.tar.gz#egg=productmd
+    $ pip install lxml pyopenssl mock sphinx setuptools nose nose-cov
+
+
 Developing
 ==========
 
@@ -18,9 +75,10 @@ Currently the development workflow for Pungi is on master branch:
     cd pungi
     git remote add upstream git@pagure.io:pungi.git
 
-    # NOTE: This workflow assumes that you never 'git commit' directly to
-    # the master branch of your fork. This will make more sense when we
-    # cover rebasing below.
+  .. note::
+      This workflow assumes that you never ``git commit`` directly to the master
+      branch of your fork. This will make more sense when we cover rebasing
+      below.
 
 - create a topic branch based on master::
 
@@ -36,7 +94,7 @@ Currently the development workflow for Pungi is on master branch:
 
     # make changes to setup.py
     git add setup.py
-    git commit -m "added awesome feature to setup.py"
+    git commit -s -m "added awesome feature to setup.py"
 
     # now we rebase
     git checkout master
@@ -50,10 +108,14 @@ Currently the development workflow for Pungi is on master branch:
     # your topic branch
     git push origin my_topic_branch
 
+  .. note::
+      In order to for your commit to be merged, you must sign-off on it. Use
+      ``-s`` option when running ``git commit``.
+
 - Create pull request in the pagure.io web UI
 
 - For convenience, here is a bash shell function that can be placed in your
-  ~/.bashrc and called such as 'pullupstream pungi-4-devel' that will
+  ~/.bashrc and called such as ``pullupstream pungi-4-devel`` that will
   automate a large portion of the rebase steps from above::
 
     pullupstream () {
@@ -73,8 +135,31 @@ Currently the development workflow for Pungi is on master branch:
 Testing
 =======
 
-You must write unit tests for any code but trivial changes.
-Any code without sufficient test coverage may not be merged.
+You must write unit tests for any code but trivial changes. Any code without
+sufficient test coverage may not be merged.
+
+To run all existing tests, suggested method is to use *nosetests*. With
+additional options, it can generate code coverage. To make sure even tests from
+executable files are run, don't forget to use the ``--exe`` option. ::
+
+    $ nosetests --exe
+    $ nosetests --exe --with-cov --cov pungi --cov-report html
+
+    # Running single test file
+    $ nosetests --exe test_arch
+
+In the ``tests/`` directory there is a shell script ``test_compose.sh`` that
+you can use to try and create a miniature compose on dummy data. The actual
+data will be created by running ``make test-data`` in project root.
+
+This testing compose does not actually use all phases that are available, and
+there is no checking that the result is correct. It only tells you whether it
+crashed or not.
+
+.. note::
+   Even when it finishes successfully, it may print errors about
+   ``repoclosure`` on *Server-Gluster.x86_64* in *test* phase. This is not a
+   bug.
 
 
 Documenting
@@ -82,3 +167,5 @@ Documenting
 
 You must write documentation for any new features and functional changes.
 Any code without sufficient documentation may not be merged.
+
+To generate the documentation, run ``make doc`` in project root.

git-changelog

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 #
 # git-changelog - Output a rpm changelog
 #

pungi/config.py

@@ -16,12 +16,20 @@
 
 
 import os
+import sys
 import time
 import yum
 
 from ConfigParser import SafeConfigParser
 
 
+# In development, `here` will point to the bin/ directory with scripts.
+here = sys.path[0]
+MULTILIBCONF = (os.path.join(os.path.dirname(here), 'share', 'multilib')
+                if here != '/usr/bin'
+                else '/usr/share/pungi/multilib')
+
+
 class Config(SafeConfigParser):
     def __init__(self):
         SafeConfigParser.__init__(self)
@@ -33,7 +41,7 @@ class Config(SafeConfigParser):
         self.set('pungi', 'sourcedir', 'source')
         self.set('pungi', 'debugdir', 'debug')
         self.set('pungi', 'isodir', 'iso')
-        self.set('pungi', 'multilibconf', '/usr/share/pungi/multilib/')
+        self.set('pungi', 'multilibconf', MULTILIBCONF)
         self.set('pungi', 'relnotefilere', 'LICENSE README-BURNING-ISOS-en_US.txt ^RPM-GPG')
         self.set('pungi', 'relnotedirre', '')
         self.set('pungi', 'relnotepkgs', 'fedora-repos fedora-release fedora-release-notes')

pungi/multilib.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 

pungi/wrappers/variants.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 

setup.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 

tests/test_arch.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 

tests/test_imagechecksumphase.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 

tests/test_notifier.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 import json

tests/test_pathmatch.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-