From 94ad7603b8312ad81e207ca819e7a22631f92360 Mon Sep 17 00:00:00 2001 From: Stepan Oksanichenko Date: Fri, 15 Jan 2021 10:53:01 +0200 Subject: [PATCH] LNX-104: Create gather_prepopulate file generator for Pungi - It's added the tool which can generate json like as `centos-packages.json` using repodata from completed repos. @BS-LINKED-5ffda6156f44affc6c5ea239 # pungi & dependencies @BS-TARGET-CL8 Change-Id: Ib0466a1d8e06feb855e81fb7160fe170e2e82e04 --- pungi.spec | 2 + pungi/scripts/create_packages_json.py | 339 ++++++++++++++++++ setup.py | 1 + .../test_repo/repodata/filelists.xml.gz | Bin 0 -> 3970 bytes .../test_repo/repodata/other.xml.gz | Bin 0 -> 2191 bytes .../test_repo/repodata/primary.xml.gz | Bin 0 -> 3094 bytes .../test_repo/repodata/repomd.xml | 28 ++ tests/test_create_packages_json.py | 79 ++++ 8 files changed, 449 insertions(+) create mode 100644 pungi/scripts/create_packages_json.py create mode 100755 tests/data/test_create_packages_json/test_repo/repodata/filelists.xml.gz create mode 100755 tests/data/test_create_packages_json/test_repo/repodata/other.xml.gz create mode 100755 tests/data/test_create_packages_json/test_repo/repodata/primary.xml.gz create mode 100644 tests/data/test_create_packages_json/test_repo/repodata/repomd.xml create mode 100644 tests/test_create_packages_json.py diff --git a/pungi.spec b/pungi.spec index 7f58cc1a..1ca706dd 100644 --- a/pungi.spec +++ b/pungi.spec @@ -41,6 +41,7 @@ BuildRequires: python3-sphinx Requires: python3-kobo-rpmlib >= 0.18.0 Requires: python3-kickstart +Requires: python3-requests Requires: createrepo_c Requires: koji >= 1.10.1-13 Requires: python3-koji-cli-plugins @@ -122,6 +123,7 @@ rm %{buildroot}%{_bindir}/%{name}-fedmsg-notification %{_bindir}/%{name}-gather %{_bindir}/%{name}-gather-rpms %{_bindir}/%{name}-gather-modules +%{_bindir}/%{name}-generate-packages-json %{_bindir}/comps_filter %{_bindir}/%{name}-make-ostree %{_mandir}/man1/pungi.1.gz diff --git a/pungi/scripts/create_packages_json.py b/pungi/scripts/create_packages_json.py new file mode 100644 index 00000000..150fbd8e --- /dev/null +++ b/pungi/scripts/create_packages_json.py @@ -0,0 +1,339 @@ +# coding=utf-8 +""" +The tool allow to generate package.json. This file is used by pungi +# as parameter `gather_prepopulate` +Sample of using repodata files taken from +https://github.com/rpm-software-management/createrepo_c/blob/master/examples/python/repodata_parsing.py +""" + +import argparse +import json +import os +import tempfile +from collections import defaultdict +from typing import AnyStr, Dict, List + +import createrepo_c as cr +import dnf.subject +import hawkey +import requests +from dataclasses import dataclass + + +@dataclass +class RepoInfo: + # path to a directory with repo directories. E.g. '/var/repos' contains + # 'appstream', 'baseos', etc. + # Or 'http://koji.cloudlinux.com/mirrors/rhel_mirror' if you are + # using remote repo + path: AnyStr + # name of folder with a repodata folder. E.g. 'baseos', 'appstream', etc + folder: AnyStr + # name of repo. E.g. 'BaseOS', 'AppStream', etc + name: AnyStr + # architecture of repo. E.g. 'x86_64', 'i686', etc + arch: AnyStr + # Is a repo remote or local + is_remote: bool + + +class PackagesGenerator: + def __init__( + self, + repos: List[RepoInfo], + excluded_packages: List[AnyStr], + ): + self.repos = repos + self.excluded_packages = excluded_packages + + @staticmethod + def _warning_callback(warning_type, message): + """ + Warning callback for createrepo_c parsing functions + """ + print(f'Warning message: "{message}"; warning type: "{warning_type}"') + return True + + @staticmethod + def _get_remote_file_content(file_url: AnyStr) -> AnyStr: + """ + Get content from a remote file and write it to a temp file + :param file_url: url of a remote file + :return: path to a temp file + """ + + file_request = requests.get( + url=file_url, + ) + file_request.raise_for_status() + with tempfile.NamedTemporaryFile(delete=False) as file_stream: + file_stream.write(file_request.content) + return file_stream.name + + @staticmethod + def _parse_repomd(repomd_file_path: AnyStr) -> cr.Repomd: + """ + Parse file repomd.xml and create object Repomd + :param repomd_file_path: path to local repomd.xml + """ + return cr.Repomd(repomd_file_path) + + def _parse_primary_file( + self, + primary_file_path: AnyStr, + packages: Dict[AnyStr, cr.Package], + ) -> None: + """ + Parse primary.xml.gz, take from it info about packages and put it to + dict packages + :param primary_file_path: path to local primary.xml.gz + :param packages: dictionary which will be contain info about packages + from repository + """ + cr.xml_parse_primary( + path=primary_file_path, + pkgcb=lambda pkg: packages.update({ + pkg.pkgId: pkg, + }), + do_files=False, + warningcb=self._warning_callback, + ) + + def _parse_filelists_file( + self, + filelists_file_path: AnyStr, + packages: Dict[AnyStr, cr.Package], + ) -> None: + """ + Parse filelists.xml.gz, take from it info about packages and put it to + dict packages + :param filelists_file_path: path to local filelists.xml.gz + :param packages: dictionary which will be contain info about packages + from repository + """ + cr.xml_parse_filelists( + path=filelists_file_path, + newpkgcb=lambda pkg_id, name, arch: packages.get( + pkg_id, + None, + ), + warningcb=self._warning_callback, + ) + + def _parse_other_file( + self, + other_file_path: AnyStr, + packages: Dict[AnyStr, cr.Package], + ) -> None: + """ + Parse other.xml.gz, take from it info about packages and put it to + dict packages + :param other_file_path: path to local other.xml.gz + :param packages: dictionary which will be contain info about packages + from repository + """ + cr.xml_parse_other( + path=other_file_path, + newpkgcb=lambda pkg_id, name, arch: packages.get( + pkg_id, + None, + ), + warningcb=self._warning_callback, + ) + + def _get_repomd_records( + self, + repo_info: RepoInfo, + ) -> List[cr.RepomdRecord]: + """ + Get, parse file repomd.xml and extract from it repomd records + :param repo_info: structure which contains info about a current repo + :return: list with repomd records + """ + repomd_file_path = os.path.join( + repo_info.path, + repo_info.folder, + 'repodata', + 'repomd.xml', + ) + if repo_info.is_remote: + repomd_file_path = self._get_remote_file_content(repomd_file_path) + else: + repomd_file_path = repomd_file_path + repomd_object = self._parse_repomd(repomd_file_path) + if repo_info.is_remote: + os.remove(repomd_file_path) + return repomd_object.records + + def _parse_repomd_records( + self, + repo_info: RepoInfo, + repomd_records: List[cr.RepomdRecord], + packages: Dict[AnyStr, cr.Package], + ) -> None: + """ + Parse repomd records and extract from repodata file info about packages + :param repo_info: structure which contains info about a current repo + :param repomd_records: list with repomd records + :param packages: dictionary which will be contain info about packages + from repository + """ + for repomd_record in repomd_records: + if repomd_record.type not in ( + 'primary', + 'filelists', + 'other', + ): + continue + repomd_record_file_path = os.path.join( + repo_info.path, + repo_info.folder, + repomd_record.location_href, + ) + if repo_info.is_remote: + repomd_record_file_path = self._get_remote_file_content( + repomd_record_file_path, + ) + parse_file_method = getattr( + self, + f'_parse_{repomd_record.type}_file' + ) + parse_file_method( + repomd_record_file_path, + packages, + ) + if repo_info.is_remote: + os.remove(repomd_record_file_path) + + def generate_packages_json( + self + ) -> Dict[AnyStr, Dict[AnyStr, Dict[AnyStr, List[AnyStr]]]]: + """ + Generate packages.json + """ + packages_json = defaultdict( + lambda: defaultdict( + lambda: defaultdict( + list, + ) + ) + ) + for repo_info in self.repos: + packages = {} + repomd_records = self._get_repomd_records( + repo_info=repo_info, + ) + self._parse_repomd_records( + repo_info=repo_info, + repomd_records=repomd_records, + packages=packages, + ) + for package in packages.values(): + package_name = package.name + package_arch = package.arch + if 'module' in package.release: + continue + if package_name in self.excluded_packages: + continue + src_package_name = dnf.subject.Subject( + package.rpm_sourcerpm, + ).get_nevra_possibilities( + forms=hawkey.FORM_NEVRA, + ) + if len(src_package_name) > 1: + # We should stop utility if we can't get exact name of srpm + raise ValueError( + 'We can\'t get exact name of srpm ' + f'by its NEVRA "{package.rpm_sourcerpm}"' + ) + else: + src_package_name = src_package_name[0].name + pkgs_list = packages_json[repo_info.name][ + repo_info.arch][src_package_name] + added_pkg = f'{package_name}.{package_arch}' + if added_pkg not in pkgs_list: + pkgs_list.append(added_pkg) + return packages_json + + +def create_parser(): + parser = argparse.ArgumentParser() + parser.add_argument( + '--repo-path', + action='append', + help='Path to a folder with repofolders. E.g. "/var/repos" or ' + '"http://koji.cloudlinux.com/mirrors/rhel_mirror"', + required=True, + ) + parser.add_argument( + '--repo-folder', + action='append', + help='A folder which contains folder repodata . E.g. "baseos-stream"', + required=True, + ) + parser.add_argument( + '--repo-arch', + action='append', + help='What architecture packages a repository contains. E.g. "x86_64"', + required=True, + ) + parser.add_argument( + '--repo-name', + action='append', + help='Name of a repository. E.g. "AppStream"', + required=True, + ) + parser.add_argument( + '--is-remote', + action='append', + type=str, + help='A repository is remote or local', + choices=['yes', 'no'], + required=True, + ) + parser.add_argument( + '--excluded-packages', + nargs='+', + type=str, + default=[], + help='A list of globally excluded packages from generated json.' + 'All of list elements should be separated by space', + required=False, + ) + + return parser + + +def cli_main(): + args = create_parser().parse_args() + repos = [] + for repo_path, repo_folder, repo_name, repo_arch, is_remote in zip( + args.repo_path, + args.repo_folder, + args.repo_name, + args.repo_arch, + args.is_remote, + ): + repos.append(RepoInfo( + path=repo_path, + folder=repo_folder, + name=repo_name, + arch=repo_arch, + is_remote=True if is_remote == 'yes' else False, + )) + pg = PackagesGenerator( + repos=repos, + excluded_packages=args.excluded_packages, + ) + result = pg.generate_packages_json() + with open('packages.json', 'w') as packages_file: + json.dump( + result, + packages_file, + indent=4, + sort_keys=True, + ) + + +if __name__ == '__main__': + cli_main() diff --git a/setup.py b/setup.py index d740c1dd..13a679cd 100755 --- a/setup.py +++ b/setup.py @@ -49,6 +49,7 @@ setup( "pungi-config-validate = pungi.scripts.config_validate:cli_main", "pungi-gather-modules = pungi.scripts.gather_modules:cli_main", "pungi-gather-rpms = pungi.scripts.gather_rpms:cli_main", + "pungi-generate-packages-json = pungi.scripts.create_packages_json:cli_main", # noqa: E501 ] }, scripts=["contrib/yum-dnf-compare/pungi-compare-depsolving"], diff --git a/tests/data/test_create_packages_json/test_repo/repodata/filelists.xml.gz b/tests/data/test_create_packages_json/test_repo/repodata/filelists.xml.gz new file mode 100755 index 0000000000000000000000000000000000000000..920e81535eae997bb364e60e9281eb14bfe1cabe GIT binary patch literal 3970 zcmV-|4}I_-iwFP!000001Fc+HbK}Mle$TH^>D5JI?jyBp=kUhms`XKE@{qi^dM*hT zNH74XV}E@b6jv*eAUOlO%37I3f79rhK4+%&?wiL|b9EoPzHZm=uj!IpU&Xa=L%qIv zfBo-&{bOOSzxn!4?^4~wrtXLS3hCGV`|I0b*!;PwntHu`T!!skT*k1iR&js;15_{c z*H;_xcW@ICLnUYC8b{lIb+x&>sRMq`B$1KZ7$q1VRkTt7_uzs6%czo?QYWmUE_p3L z5^uRdOpHz{F4gtb8dmZBH8*w<`h|l&USC1yZ{J@(8uhCZ$n)yz-CoM8xM{~nC@Fq3 z$}E;-$?#(r8~jL?t2S(#_*HBSJ|JIlV%1laP-Ge8b+sF7l<#VI+920Zch_H6aqt!X zU*Fxo+X;V(s`BkR)i(#k6BLz?-LQhqa{QP<-Z!ub3*R;k3@B}%iyPuJ)qv_>%KMOhnyBI; zl0>PN5r5kxbYjXzorovi1;95-#vj_u1>Q$CIZae~MHETkl=c#B%A>|p<%o|Fq7qJ7 z5TsOYbsTA?h$<~4`VLOLP|gI@wm>P<5Q0VBqt*+ji7nJ~<#Tvr*preF2$4mG+>?w3 z(6S>({ljm)Oaa%SRPLPhR^xhmt{t@rjPfKgskvkcbv*j0RZ7VuRcd4EI9^W?q1@h~ zMn@wEG7Om2rDR1w-=b||_+yo#hhtsft#e|DmX$(XRv~Gg(2^BLV-`e6rWq$%M|3*T z235G;+UD@Km7yLjI0i?JRw7~4!ia`S0D=YvlsVKzcBng-c<1Q3222kfaE-1dNGDKc z7YqQy_`*%H8sAANaEEeS;+?bJI#mWVxEKQ==*%H_njDM33DE&m3;|Pm#tY>(IkCjh z3P3W+(Q#v8z*TY@m24~{C`K5Aq1`f#;k1x>FFY(Tqb4#nF7>_JP z8o&y|gCds1pwPiFi_`B61tki7i#^P9rim&U5+*1z5=%Ltr3o}}Z=B~i+ngwLcUU+U z=BYL(XR??!87e4I3=2}>6{f4>8gPS56?#)lB7$M0biH%@xs4UUfN3yfqnI9T-K&5u z9i^o}ZIt+*HX;@JRLEIxpcM{}Y&;`WaA^ny^fW|?F>er9tXqa2wKPYDLmxa%R9TiV zR40pxFy|UW76b^&d!jr8iRA&R11YQxASZ7DRYl=QB2q9+mbg!zfFprmWr7)_31f7z zL>Kxs$jKGK;j{-dao|E=Z9pw`3F9e%4~%P%wUM)imBtd6lOsD;8O%%ExV!|=r5i0U zsGvu27){Z^X-ynsM>+2W-beZ|O@w-m2@2+^SdvJ}g;Wkx35$u7!Z1RDQksY(o4Le0 zN7uZlBDxf}E4?C?CRhVnErJ$I5&=?TvdBy#g)!9Y9NxOpMrxn}gBPZ!9xaNXx1nA+ zK=)!(LPsB5;hcvYxr3;JqTUDyC^Ms24dc>NAQ}`}d*GQ_ubmtgM zgH?xl=Ox}b`C5buI7TsG1i7TKCJTfTZ$UA%>|_jTgJP^u*J4iI5>v%c&8=lFaST{k zff%c42obYHtO_*qB1#51dB`}H+Zly$j^Scq%B6!7#Cy#Z#S}qmX`LvPBjxCA5|yJ0 z6=h7dAbD!Bs<)oTaS5YSf}jfxAr<-&d(;gC4zr{`0=1Beph954EJ<5YWzr__a zP3xC^yFBzs=ecZ}?M=O&-iw|h9KQtbhfUkm{%P_2EwU{hVB3SMn|hc(Ml%-~!g?-r z7`nxFUH@+z=Z>%)+ErY+*v%XO1;1M~!L5pc_3%*lbKl$FLcOlnH^I#v;PVO5Z2WqN z>!E*I``IJ(Gm#;7G0nx+{m|8OXDzH@CUQJ5UHGQPIh#2`Q?Ku07*&4myscmjH<)9s zaR%b-L3Y;C&Yk}atQH&S`{B0R&K~0T!QEV=+q8ZCxOl)%^R<5S5c;9*a2;k2fT|z6 zdbSpghmf`ZJ(l0ap=)P*`>t(ft6tZ(1I7vrSFxWv#`9Tp7n>Hs+#$x}Q*^-V*=vaU zGr;3a@V1}3a`$z7m^*I|w=Jydli&+k&C_t(W~64%bX5X;957D{{i!4MDOl6qbnzxW zp25V&_;M>D^uO*?&tIXx`*rjx4{e7Z{Pg*HKPHY2?yw#|{TwT4Bsu#W#fpz-DJE}7 zydtIx9BS}ZuT6~R^v6OY4(Ry6(wEr<2jY9`{~;;(>O(*oe=7M z8LVE%VTCqUhPsR082Afc@vd|7F8DIgx^yD`S1t}}NWCp&gX<)WR%M_bRt{!Bcp)aP zL+LF1)yER@Z-4n^{4RVvQ3@X;K5V;o6JcG#WyJK`lP4^N_8Faqnjq zxb2F<&uiQk^>P-rhko7y?{fEf$el}C1UCmgg*l%;)DyRMr--V(#`Md4MDM#ezSQbe z0(gwT#oBKhl%TeoP{e1KaZU(weyUbjP+cB>q{5sv+`=vyjUKD;r{(G>l%40b8RJl1VpA-?T$)%-Jtju>uiA&mKUwQ6(S^=FhEQ(nvT449BC zH#Itdi9(nf6J^wVT(Q1XLQ0OemJ0`Xwb@vRT*b{yKZ%sT z!96Tm_Z!+{A=GT@rY!_I8 z+0J>@hnO23_b5fa99|V8(Y{h{cKU*fVLQj16ZZvpX)9iY{6e-^268v&*MCJIdDI zWs@GZF2Aexz3#_d?@R^5jT>+C6Jv?z;F2K5c}7TdK1rSuje;bqloTFBA$popBIi=MU=*wABPvD` ziH4!Sia6WaQXibqPNUWZYVkEZ&P3sQL(YYENR{F=P{Ywd8i#7l72#G}=}1KNQ1)8fATEM8zit-SIx5l7ib31lNS-`pW^dJDv4t@F{^!t(@o)Yh0Cl!QpR22Fh--p#OXNw*Cmt~|J~&zWW45o`tOb$QJ(COI-<VhSFwCx+x%D1t zS-D~h!1X~)!_32zhObDwS_qEchp`_6%>iPTm5@LlM~s%N;1SEiAc@l;D55+Gig}qv zEcVNsX1E8Sh!Ywo9%)z&M;^`c3D=W?Y48YBu>LC`z*|^L_8lguLzQcAXJo z%_zJoNK5TG9~FXv3-s6yXwdvu{Yn3aCgcmC@v@FiTQ# z)?V1>3o~+FOz{t%SAsG%Yss963O6pHA*Wh*c<}pAWXUu^e`<#Jx$uu36eT$x^sduJ z*0@i>zdO(daYZKAM7GAdH;KV*2koQ7;QI`(K zjU;USE7`wTySqPrUz%D-qnoWTgY#=d=%wr`ZWDu`uGin%>LA~TcAWKD6}&zm=Nhrl z2i}&(<8Jg^MfUZz-XC!WuFU>;rjwlZag)tlEH+%RWS6k6OhkLjw03`em5LK)friYw z=)Q;gED7wfHt{#xda`tdZmV3BjIf0QS30CXXIFatgT41~wqe)5lxxF#TpK*dwc&_^ zQI=Di|JHFWom8T*oY8g_RvZVcGLEfNfOcFb&=^F2bxjK74rD&PE8pmrZ!yhj@<<<{ zpJ-)rz9jV-~D>(i2MBX)p#w1A)b8NSTgAe zfeT$$Qg;flCs~f}wB?82p*TO?6||x)XeEhH!$+VMJ%FsaxFTA13bSUI!hluPuFQ_N z%uYieh1J7m$N%f@vMa}vEypv*1P?nt-d67jE-cD!ZnL)*U=mBHk$l}NM$Sls*yX&@ z6J4mYq2p_*>p4fuBCc z;DP^uy9%s-WDq`9BBJ)OutY9*NZ(Wg;vH@ap2OKN`QLvB5GFjMSrNwR+|P2qq^y`n zepZxdEEMHDrt^6Kkv|U-tdNrNya>u7D*ZU0`#cKQ?*P7i6DTB$NoTP9{)@K&{xqP| zBpY4=WK$gezdPz5S8#=W8=LuCxssRi+NFXO{looC^k=8r!+wxPSu$|WdCP~H*1`Zo z|L&hBlh6;cNkX&Waa4Q&E)~Osx&7qCLJmM#tXaG4xy|C2xm(pL`3I_CU^csV)aI#&F*qQW7I1W$f)d| zQNg%T>sWeL*5EAq%_D_qm=bs15xl@|5=K$_R4G9H5+Er9#!Ci}7GYGPXvtBxpc(~; zW0se+D0o@qC`(cVeHevh8Yd~oJ`e_Xq`5~Tp zAPL936#ymMUw<}8N|tPi7sYk^ke5gz*TQ9EV`Bl>&dGo7(?pw7E?AzOj%>pkX*9z* zVcGO_^yTIkJskb#?BFD&5)mYk2JlRrj%HF8$EKOEth_f!c}ERON^^-aah|4mHmcFb zyhuA4fEj58!gpv&p?uqSop1ydH;**=P|(wnLY*CG+DV2|dgg|aPJ|vKK~K!;iF$=N zo}Jx?!ENB3nAH#UwCSQoiyY5RN6<}pqB`sumf^rpP80ZP8EH<+gdWi(gcsJ4Yem74 zZP}sY6Mtk@W#JjcccM(2Mu{16`~X^iH)Z5>^ylUE=*)I}%cpKcsgFWT0~+}raxsdD zha$%g{IPAvUKH5z1dl!BTR3u|3X9U%4_rSuF_)E8HNt0#_=D=P_MUNyWzHY8Ys#S$ z+MzPQyAyK>QUpQ;=B$v=yQ&WvBv6o}4MB0P13~Fp8W@ zFolux>4~XcDv)F;)g~;V%H{mPjzY`!Mp|65gfx#FD|CWpux5z_Q?(@;o*Nlpoo)Z!b$oa$H$Jn{hfh9er**-gM11Z`PM?sdqS}QTw@}=0#mT zm^?*NeT5(J_&8yhW`dr5_XHTd(%}D;t_BFP7^8}spye!VHAB3c-cA3o? z&$ELp6XU$RL=~WO^rAgYol71@Wgm3MdT&N6uzD9 zWi7$6GF4svQ&~^t&nP04>l|g^@i3-b!uSPNsDdI4t}}MWQ8SVaqu}|RfiKsOnulNw zm}x2^osQNTI?^TynnHBD{O{;B0#R3ehsxA}_wG&V#d$Vi)5Epu-=^M#u!)n9I9=8D z;qpaoR$BoqMi|Q?hgtEU7fCr~nIZDScZY!&Gx@HYS_%}akhc#z8O|Xj8%~mv>t)9N zD(P@;Df5)3F-(l1ROsNHt_EiSg~4HP1!$FhytP;m0s0YaJeUeDOF7%aA2DCYsDxNoJAToDN`~}ziCzwcWBC1ubgqPlW7vux^sH@)@vRtQ-8I68i;G*g zuW1=B)nQlFHmhe{z+rZn4ds-05aZ6TJSJBpw=ajs{t$ekb`Xv+V0HqBpyGBfmhwr( z>1%;>h{t3CVyu$7WOeue!L#?2$wa}uA#gEZJ}%L}U0;J|+Ri?#y+}Rd=X$e;59`;#1J@aL?|MII=)U>hU&f! zx0T#G&TnP+_S+rop1!CQi8Cbd2{Oq2mLjhahHeKsLB*g^6CxQ9O_t~;n)Ef%2CL_& zY?8*kheHy4bTlM-uQM*bem(#BBh1;J(QBPsdCu3b{iE0pvSHgj`E!o*I%f19?dqI< zn1ifSpq1BuUf*1PdVlL!QI7&3Qn#JN!ZP8tvvry0t!gqaXr@;9=9sAqD%Ad=eP5fL}6oWeD4AaeRE9Jv{yFWM5;HAPyyfJnIDCu#~Fi1E&)! z*i>>S2?#=i^Nj9VA%lJhKYn?ttYxXqT`4ce|Y>N^b; zGO8U~v!HZt*P3-lSCgy{jKyvky~#NPmkBxi@Y|Q$U(c_9Z5tXv>wSG~f%br8LjZ1C z=0t5})^V}NiO~|-zC~YK2b69aZ$I1GJjkjFQde!zExG$}Yr-jI1=8!w_f_|=ZeDa> z4NRND*fiwkHRhJ$lr|4HJMgTvRSkJ8@`TEP*QQt7rh;quM%#+gyC7wWXtU|hnt7rU z1)8J}!uaUSq!OF(P4M}Nsa|&D%=(`61$+y}#FWbv6^8oRi%@AneRZ8B`ZK{@)p;b< zS;pj!ChqD{eSFxC`7Fa@XBHP~QQ!vBaD3TUXu^=-fW-SP{ z83LW8ZW4B&AYKLDaxfL;f3)EFwn1E-RmbSgA#odm#14Qu5G*6~z5}zvpVlBS$Ju1p zfQZ`EY|_!*Ce;8HZJ(8*kCUb71cu(FtPL}PQ28d}IaqXDw^?O$4+I+cko)(E*_W z%A{Tk`ege#0SVnOcKq?g3L^`XpASmJDJ$jvnF;B_#Q?iiftTw*z%)^ zMLcImw%?bHL^_ps*K}1=SB`+#o%46|60B1GSe32-0F)#T@M1cX3#$J8QDaieClfGE zRZ)}BqRIS#BEbMm??GN;sa08^6P~A4+MHPCxe+K^Lz)213_%xau3j73d9k{~6$cPI zwYbbkLK`x)C*Nc>#Z_5_^>wOhPij;Ni3}Dgs&dV$@(!RwYFQ^*oe)KSeBJ-N>y(BP=i@vmYHf@wKX*k z4sK?EU1qClDhmX$4A@Ch5uP$2=hqQgi_Zr2vGad5L+?dsRGd=Zc zVh+f9yi#e8&XET5=M#Lh4g1wHAlE(HtapLn#(lFNqi@Q7TpDN7EZ>^mXh)&ztQ7Fe zi}fw)dSjH@wtze1Yj?4I(imHc-#8qhUpv2uknrJ%|wci-M9N z%-$R5W(2;`_WgFgFHTRl?tC{TYZZ_6l%_4R7BB$n1nBYB{n#^5pQ`kf?Gi%@^!Nzj zqV+upv)1?bIqxb$LGsqUM}^s_X>ZN-Ty3!+4E}DdWsQj&qmWvzO>8?Rwi`Jv4J>C& z?J@O9Y&&Ee(8%_Y6VeD1+Yci;8AkyMXb|tvT31h6XeF3Ho9q-nONAr8Yp{PrgZ=AO zljC{rOV#9TRZWp^kDU$G8+YJg@Kxu-__D<43Fhk;{%U&y~N z + + 1610968727 + + 2826d3f5dd3b03cfb5d2c079123f7add3a7d068e8dfd210873eb27eb32586a8e + 78efcf6b74f4c56aaab183336eab44fcbcc9cb6c25045fe5980ab83a85e48db7 + + 1610968715 + 3094 + 16878 + + + e41805c927fc4ad1b9bde52509afb37e47acc153283b23da17560d4e250b3a3e + 5f659e8c05b7d056748bf809bec8aa9fa5f791c2b0546d6c49b02a7ebfb26ce2 + + 1610968715 + 3970 + 19897 + + + db6d0d88abcaf06dc8ef09207fdbb9ba5e3ffb505a7dd2bf94fdbc953a6de11e + 3ae1b186b4c3037805e2cf28a78b2204c37b4dc04acbd8bef98a7b24ab5b52a8 + + 1610968715 + 2191 + 8337 + + diff --git a/tests/test_create_packages_json.py b/tests/test_create_packages_json.py new file mode 100644 index 00000000..ac964e18 --- /dev/null +++ b/tests/test_create_packages_json.py @@ -0,0 +1,79 @@ +# coding=utf-8 + +import os +from collections import defaultdict +from unittest import TestCase, mock, main + +from pungi.scripts.create_packages_json import PackagesGenerator, RepoInfo + +FOLDER_WITH_TEST_DATA = os.path.join( + os.path.dirname( + os.path.abspath(__file__) + ), + 'data/test_create_packages_json/', +) + +test_repo_info = RepoInfo( + path=FOLDER_WITH_TEST_DATA, + folder='test_repo', + name='TestRepo', + arch='x86_64', + is_remote=False, +) + + +class TestPackagesJson(TestCase): + def test_01__get_remote_file_content(self): + """ + Test the getting of content from a remote file + """ + request_object = mock.Mock() + request_object.raise_for_status = lambda: True + request_object.content = b'TestContent' + with mock.patch( + 'pungi.scripts.create_packages_json.requests.get', + return_value=request_object, + ) as mock_requests_get, mock.patch( + 'pungi.scripts.create_packages_json.tempfile.NamedTemporaryFile', + ) as mock_tempfile: + mock_tempfile.return_value.__enter__.return_value.name = 'tmpfile' + file_name = PackagesGenerator._get_remote_file_content( + file_url='fakeurl' + ) + mock_requests_get.assert_called_once_with(url='fakeurl') + mock_tempfile.assert_called_once_with(delete=False) + mock_tempfile.return_value.__enter__().\ + write.assert_called_once_with(b'TestContent') + self.assertEqual( + file_name, + 'tmpfile', + ) + + def test_02_generate_additional_packages(self): + pg = PackagesGenerator( + repos=[ + test_repo_info, + ], + excluded_packages=['zziplib-utils'], + ) + test_packages = defaultdict( + lambda: defaultdict( + lambda: defaultdict( + list, + ) + ) + ) + test_packages['TestRepo']['x86_64']['zziplib'] = \ + [ + 'zziplib.i686', + 'zziplib.x86_64', + ] + result = pg.generate_packages_json() + self.assertEqual( + test_packages, + result, + ) + + +if __name__ == '__main__': + main()