diff --git a/.gitignore b/.gitignore index 33ebd792..042fa4e8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.py[co] *~ +*.egg-info MANIFEST build/* dist/* @@ -7,3 +8,5 @@ doc/_build noarch/* tests/data/repo tests/data/repo-krb5-lookaside +tests/_composes +htmlcov/ diff --git a/bin/comps_filter b/bin/comps_filter index 8b00283c..a023fb82 100755 --- a/bin/comps_filter +++ b/bin/comps_filter @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # -*- coding: utf-8 -*- diff --git a/bin/pungi b/bin/pungi index edcbcb87..9008b16f 100755 --- a/bin/pungi +++ b/bin/pungi @@ -1,4 +1,4 @@ -#!/usr/bin/python -tt +#!/usr/bin/env python # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. diff --git a/bin/pungi-koji b/bin/pungi-koji index 4298f149..3f1441d3 100755 --- a/bin/pungi-koji +++ b/bin/pungi-koji @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # -*- coding: utf-8 -*- diff --git a/doc/about.rst b/doc/about.rst index 4901ef3b..bb4963cf 100644 --- a/doc/about.rst +++ b/doc/about.rst @@ -2,7 +2,7 @@ About Pungi ============= -Pungi is a distribution compose tool. +*Pungi* is a distribution compose tool. Composes are release snapshots that contain release deliverables such as: @@ -19,6 +19,24 @@ Composes are release snapshots that contain release deliverables such as: - images for PXE boot +Tool overview +============= + +*Pungi* consists of multiple separate executables backed by a common library. + +The main entry-point is the ``pungi-koji`` script. It loads the compose +configuration and kicks off the process. Composing itself is done in phases. +Each phase is responsible for generating some artifacts on disk and updating +the ``compose`` object that is threaded through all the phases. + +*Pungi* itself does not actually do that much. Most of the actual work is +delegated to separate executables. *Pungi* just makes sure that all the +commands are invoked in the appropriate order and with correct arguments. It +also moves the artifacts to correct locations. + + Links ===== - Upstream GIT: https://pagure.io/pungi/ +- Issue tracker: https://pagure.io/pungi/issues +- Questions can be asked on *#fedora-releng* IRC channel on FreeNode diff --git a/doc/contributing.rst b/doc/contributing.rst index eea20c77..ba42ca85 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -3,6 +3,63 @@ Contributing to Pungi ===================== +Set up development environment +============================== + +In order to work on *Pungi*, you should install *Fedora 23*. These packages +will have to installed: + + * createrepo + * createrepo_c + * cvs + * genisoimage + * gettext + * git + * isomd5sum + * jigdo + * kobo + * kobo-rpmlib + * koji + * libselinux-python + * lorax + * python-kickstart + * python-lockfile + * python-lxml + * python-productmd + * repoview + * syslinux + * yum + * yum-utils + +For running unit tests, these packages are recommended as well: + + * python-mock + * python-nose + * python-nose-cov + +Technically, it is possible to work on *Fedora 22*, but setting it up is a +major headache. Packaged version of *productmd* is too old, and *Pungi* does +not easily work with *virtualenv*, mostly because many of its Python +dependencies are not available on PyPI. To get it working, you have to install +*kobo* and *productmd* directly from Git and provide symlinks to other packages +and files in ``$(virtualenvwrapper_get_site_packages_dir)``. You will still +need to install all of the non-Python packages above as they are used by +calling an executable. :: + + $ ln -vs "$(deactivate && python -c 'import os, koji; print os.path.dirname(koji.__file__)')" "$(virtualenvwrapper_get_site_packages_dir)" + $ ln -vs "$(deactivate && python -c 'import os, rpm; print os.path.dirname(rpm.__file__)')" "$(virtualenvwrapper_get_site_packages_dir)" + $ ln -vs "$(deactivate && python -c 'import os, rpmUtils; print os.path.dirname(rpmUtils.__file__)')" "$(virtualenvwrapper_get_site_packages_dir)" + $ ln -vs "$(deactivate && python -c 'import os, yum; print os.path.dirname(yum.__file__)')" "$(virtualenvwrapper_get_site_packages_dir)" + $ ln -vs "$(deactivate && python -c 'import os, urlgrabber; print os.path.dirname(urlgrabber.__file__)')" "$(virtualenvwrapper_get_site_packages_dir)" + $ ln -vs "$(deactivate && python -c 'import os, krbV; print krbV.__file__')" "$(virtualenvwrapper_get_site_packages_dir)" + $ ln -vs "$(deactivate && python -c 'import os, sqlitecachec; print sqlitecachec.__file__')" "$(virtualenvwrapper_get_site_packages_dir)" + $ ln -vs "$(deactivate && python -c 'import os, _sqlitecache; print _sqlitecache.__file__')" "$(virtualenvwrapper_get_site_packages_dir)" + $ PYCURL_SSL_LIBRARY=nss pip install pycurl + $ pip install https://git.fedorahosted.org/cgit/kobo.git/snapshot/kobo-0.4.3.tar.gz#egg=kobo + $ pip install https://github.com/release-engineering/productmd/archive/master.tar.gz#egg=productmd + $ pip install lxml pyopenssl mock sphinx setuptools nose nose-cov + + Developing ========== @@ -18,9 +75,10 @@ Currently the development workflow for Pungi is on master branch: cd pungi git remote add upstream git@pagure.io:pungi.git - # NOTE: This workflow assumes that you never 'git commit' directly to - # the master branch of your fork. This will make more sense when we - # cover rebasing below. + .. note:: + This workflow assumes that you never ``git commit`` directly to the master + branch of your fork. This will make more sense when we cover rebasing + below. - create a topic branch based on master:: @@ -36,7 +94,7 @@ Currently the development workflow for Pungi is on master branch: # make changes to setup.py git add setup.py - git commit -m "added awesome feature to setup.py" + git commit -s -m "added awesome feature to setup.py" # now we rebase git checkout master @@ -50,10 +108,14 @@ Currently the development workflow for Pungi is on master branch: # your topic branch git push origin my_topic_branch + .. note:: + In order to for your commit to be merged, you must sign-off on it. Use + ``-s`` option when running ``git commit``. + - Create pull request in the pagure.io web UI - For convenience, here is a bash shell function that can be placed in your - ~/.bashrc and called such as 'pullupstream pungi-4-devel' that will + ~/.bashrc and called such as ``pullupstream pungi-4-devel`` that will automate a large portion of the rebase steps from above:: pullupstream () { @@ -73,8 +135,31 @@ Currently the development workflow for Pungi is on master branch: Testing ======= -You must write unit tests for any code but trivial changes. -Any code without sufficient test coverage may not be merged. +You must write unit tests for any code but trivial changes. Any code without +sufficient test coverage may not be merged. + +To run all existing tests, suggested method is to use *nosetests*. With +additional options, it can generate code coverage. To make sure even tests from +executable files are run, don't forget to use the ``--exe`` option. :: + + $ nosetests --exe + $ nosetests --exe --with-cov --cov pungi --cov-report html + + # Running single test file + $ nosetests --exe test_arch + +In the ``tests/`` directory there is a shell script ``test_compose.sh`` that +you can use to try and create a miniature compose on dummy data. The actual +data will be created by running ``make test-data`` in project root. + +This testing compose does not actually use all phases that are available, and +there is no checking that the result is correct. It only tells you whether it +crashed or not. + +.. note:: + Even when it finishes successfully, it may print errors about + ``repoclosure`` on *Server-Gluster.x86_64* in *test* phase. This is not a + bug. Documenting @@ -82,3 +167,5 @@ Documenting You must write documentation for any new features and functional changes. Any code without sufficient documentation may not be merged. + +To generate the documentation, run ``make doc`` in project root. diff --git a/git-changelog b/git-changelog index 88df9d60..093d1ed6 100755 --- a/git-changelog +++ b/git-changelog @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # # git-changelog - Output a rpm changelog # diff --git a/pungi/config.py b/pungi/config.py index 7403c0a7..417e4a37 100644 --- a/pungi/config.py +++ b/pungi/config.py @@ -16,12 +16,20 @@ import os +import sys import time import yum from ConfigParser import SafeConfigParser +# In development, `here` will point to the bin/ directory with scripts. +here = sys.path[0] +MULTILIBCONF = (os.path.join(os.path.dirname(here), 'share', 'multilib') + if here != '/usr/bin' + else '/usr/share/pungi/multilib') + + class Config(SafeConfigParser): def __init__(self): SafeConfigParser.__init__(self) @@ -33,7 +41,7 @@ class Config(SafeConfigParser): self.set('pungi', 'sourcedir', 'source') self.set('pungi', 'debugdir', 'debug') self.set('pungi', 'isodir', 'iso') - self.set('pungi', 'multilibconf', '/usr/share/pungi/multilib/') + self.set('pungi', 'multilibconf', MULTILIBCONF) self.set('pungi', 'relnotefilere', 'LICENSE README-BURNING-ISOS-en_US.txt ^RPM-GPG') self.set('pungi', 'relnotedirre', '') self.set('pungi', 'relnotepkgs', 'fedora-repos fedora-release fedora-release-notes') diff --git a/pungi/multilib.py b/pungi/multilib.py index 7d072a48..687bc335 100755 --- a/pungi/multilib.py +++ b/pungi/multilib.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # -*- coding: utf-8 -*- diff --git a/pungi/wrappers/variants.py b/pungi/wrappers/variants.py index ba1e0382..14e66647 100755 --- a/pungi/wrappers/variants.py +++ b/pungi/wrappers/variants.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # -*- coding: utf-8 -*- diff --git a/setup.py b/setup.py index df1d5a52..5c31b9e1 100755 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # -*- coding: utf-8 -*- diff --git a/tests/test_arch.py b/tests/test_arch.py index c38d6de9..f30a148f 100755 --- a/tests/test_arch.py +++ b/tests/test_arch.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # -*- coding: utf-8 -*- diff --git a/tests/test_imagechecksumphase.py b/tests/test_imagechecksumphase.py index a4482c24..628508a7 100755 --- a/tests/test_imagechecksumphase.py +++ b/tests/test_imagechecksumphase.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # -*- coding: utf-8 -*- diff --git a/tests/test_notifier.py b/tests/test_notifier.py index b93ca352..d579892b 100755 --- a/tests/test_notifier.py +++ b/tests/test_notifier.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # -*- coding: utf-8 -*- import json diff --git a/tests/test_pathmatch.py b/tests/test_pathmatch.py index 3013e965..12f182d3 100755 --- a/tests/test_pathmatch.py +++ b/tests/test_pathmatch.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # -*- coding: utf-8 -*-