almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

ALBS-987: Generate i686 and dev repositories with pungi on building new distr. version automatically #15

Merged
soksanichenko merged 8 commits from ALBS-987 into al_master 2023-03-31 09:03:40 +00:00
Conversation 0 Commits 8 Files Changed 9 +68 -111
2 changed files with 68 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 6d58bc2ed8 - Show all commits

141
pungi/scripts/create_packages_json.py
View File

@ -9,12 +9,22 @@ https://github.com/rpm-software-management/createrepo_c/blob/master/examples/pyt
import argparse
import gzip
import json
import logging
import lzma
import os
import re
import tempfile
from collections import defaultdict
from typing import AnyStr, Dict, List, Any, Iterator
from pathlib import Path
from typing import (
AnyStr,
Dict,
List,
Any,
Iterator,
Optional,
Tuple,
)
import binascii
import createrepo_c as cr
@ -26,6 +36,8 @@ import yaml
from createrepo_c import Package, PackageIterator
from dataclasses import dataclass
logging.basicConfig(level=logging.INFO)
def _is_compressed_file(first_two_bytes: bytes, initial_bytes: bytes):
return binascii.hexlify(first_two_bytes) == initial_bytes
@ -330,7 +342,7 @@ class PackagesGenerator:
for variant_name, variant_arch in package_dict['variants']:
package_arch = package_dict['arch']
package = package_dict['package']
package_name = package.name
package_name = f'{package.name}.{package_arch}'
if any(re.search(excluded_package, package_name)
for excluded_package in self.excluded_packages):
continue
@ -370,73 +382,15 @@ class PackagesGenerator:
def create_parser():
anfimovdm marked this conversation as resolved Outdated
anfimovdm commented 2023-03-28 12:05:56 +00:00
Outdated
Review
Copy Link

those if statements can be united

those `if` statements can be united
soksanichenko commented 2023-03-29 08:34:39 +00:00
Outdated
Review
Copy Link

It's splitted for more readable code.

It's splitted for more readable code.
parser = argparse.ArgumentParser()
parser.add_argument(
'--repo-path',
action='append',
help='Path to a folder with repofolders. E.g. "/var/repos" or '
'"http://koji.cloudlinux.com/mirrors/rhel_mirror"',
required=True,
)
parser.add_argument(
'--repo-folder',
action='append',
help='A folder which contains folder repodata . E.g. "baseos-stream"',
required=True,
)
parser.add_argument(
'--repo-arch',
action='append',
help='What architecture packages a repository contains. E.g. "x86_64"',
required=True,
)
parser.add_argument(
'--repo-name',
action='append',
help='Name of a repository. E.g. "AppStream"',
required=True,
)
parser.add_argument(
'--is-remote',
action='append',
type=str,
help='A repository is remote or local',
choices=['yes', 'no'],
required=True,
)
parser.add_argument(
'--is-reference',
action='append',
type=str,
help='A repository is used as reference for packages layout',
choices=['yes', 'no'],
required=True,
)
parser.add_argument(
'--repo-type',
action='append',
type=str,
help='Packages from repository will be removed or added to variant',
choices=['present', 'absent'],
required=True,
)
parser.add_argument(
'--excluded-packages',
nargs='+',
type=str,
default=[],
help='A list of globally excluded packages from generated json.'
'All of list elements should be separated by space',
required=False,
)
parser.add_argument(
'--included-packages',
nargs='+',
type=str,
default=[],
help='A list of globally included packages from generated json.'
'All of list elements should be separated by space',
'-c',
'--config',
type=Path,
default=Path('config.yaml'),
required=False,
help='Path to a config',
)
parser.add_argument(
'-o',
'--json-output-path',
type=str,
help='Full path to output json file',
@ -446,32 +400,43 @@ def create_parser():
return parser
def read_config(config_path: Path) -> Optional[Dict]:
if not config_path.exists():
logging.error('A config by path "%s" does not exist', config_path)
exit(1)
with config_path.open('r') as config_fd:
return yaml.safe_load(config_fd)
anfimovdm commented 2023-03-28 12:07:32 +00:00
Review
Copy Link

wrong indents

wrong indents
soksanichenko commented 2023-03-29 08:39:35 +00:00
Review
Copy Link

Pycharm didn't agree with you)

Pycharm didn't agree with you)
anfimovdm commented 2023-03-29 09:23:24 +00:00
Review
Copy Link

btw, even if linter doesn't show any warnings about indents, it looks wierd, because indents for function arguments supposed to be like this

self.compare_pkgs_version(
    package_1=package,
    package_2=package_info['package'],
):

I prefer to use black for code formatting, here is an example of black formatting for those lines

if "is_reference" not in package_info:
    package_info["variants"].append(variant_info.name)
    package_info["is_reference"] = is_reference
    package_info["package"] = package
elif (
    not package_info["is_reference"]
    or package_info["is_reference"] == is_reference
    and self.compare_pkgs_version(
        package_1=package,
        package_2=package_info["package"],
    )
    > 0
):
    package_info["variants"] = [variant_info.name]
    package_info["is_reference"] = is_reference
    package_info["package"] = package
elif (
    self.compare_pkgs_version(
        package_1=package,
        package_2=package_info["package"],
    )
    == 0
    and repo_info.repo_type != "absent"
):
    package_info["variants"].append(variant_info.name)
btw, even if linter doesn't show any warnings about indents, it looks wierd, because indents for function arguments supposed to be like this ```python3 self.compare_pkgs_version( package_1=package, package_2=package_info['package'], ): ``` I prefer to use `black` for code formatting, here is an example of black formatting for those lines ```python3 if "is_reference" not in package_info: package_info["variants"].append(variant_info.name) package_info["is_reference"] = is_reference package_info["package"] = package elif ( not package_info["is_reference"] or package_info["is_reference"] == is_reference and self.compare_pkgs_version( package_1=package, package_2=package_info["package"], ) > 0 ): package_info["variants"] = [variant_info.name] package_info["is_reference"] = is_reference package_info["package"] = package elif ( self.compare_pkgs_version( package_1=package, package_2=package_info["package"], ) == 0 and repo_info.repo_type != "absent" ): package_info["variants"].append(variant_info.name) ```
def process_config(config_data: Dict) -> Tuple[
List[RepoInfo],
List[str],
List[str],
]:
excluded_packages = config_data.get('excluded_packages', [])
included_packages = config_data.get('included_packages', [])
repos = [RepoInfo(
path=variant_repo['path'],
folder=variant_repo['folder'],
name=variant_name,
arch=variant_repo['arch'],
is_remote=variant_repo['remote'],
is_reference=variant_repo['reference'],
repo_type=variant_repo.get('repo_type', 'present'),
) for variant_name, variant_repos in config_data['variants'].items()
for variant_repo in variant_repos]
return repos, excluded_packages, included_packages
def cli_main():
args = create_parser().parse_args()
repos = []
for repo_path, repo_folder, repo_name, \
repo_arch, is_remote, is_reference, repo_type in zip(
args.repo_path,
args.repo_folder,
args.repo_name,
args.repo_arch,
args.is_remote,
args.is_reference,
args.repo_type,
):
repos.append(RepoInfo(
path=repo_path,
folder=repo_folder,
name=repo_name,
arch=repo_arch,
is_remote=True if is_remote == 'yes' else False,
is_reference=True if is_reference == 'yes' else False,
repo_type=repo_type,
))
repos, excluded_packages, included_packages = process_config(
config_data=read_config(args.config)
)
pg = PackagesGenerator(
repos=repos,
excluded_packages=args.excluded_packages,
included_packages=args.included_packages,
excluded_packages=excluded_packages,
included_packages=included_packages,
)
result = pg.generate_packages_json()
with open(args.json_output_path, 'w') as packages_file:

38
pungi/scripts/gather_rpms.py
View File

@ -2,38 +2,32 @@ from argparse import ArgumentParser
import os
from typing import List
from pathlib import Path
from attr import dataclass
from dataclasses import dataclass
from productmd.common import parse_nvra
@dataclass
class Package:
nvra: str
path: str
path: Path
def search_rpms(top_dir) -> List[Package]:
def search_rpms(top_dir: Path) -> List[Package]:
"""
Search for all *.rpm files recursively
in given top directory
Returns:
list: list of paths
"""
rpms = []
for root, dirs, files in os.walk(top_dir):
path = root.split(os.sep)
for file in files:
if not file.endswith('.rpm'):
continue
nvra, _ = os.path.splitext(file)
rpms.append(
Package(nvra=nvra, path=os.path.join('/', *path, file))
)
return rpms
return [Package(
nvra=path.stem,
path=path,
) for path in top_dir.rglob('*.rpm')]
def copy_rpms(packages: List[Package], target_top_dir: str):
def copy_rpms(packages: List[Package], target_top_dir: Path):
"""
Search synced repos for rpms and prepare
koji-like structure for pungi
@ -46,24 +40,22 @@ def copy_rpms(packages: List[Package], target_top_dir: str):
"""
for package in packages:
info = parse_nvra(package.nvra)
target_arch_dir = os.path.join(target_top_dir, info['arch'])
target_arch_dir = target_top_dir.joinpath(info['arch'])
target_file = target_arch_dir.joinpath(package.path.name)
os.makedirs(target_arch_dir, exist_ok=True)
target_file = os.path.join(target_arch_dir, os.path.basename(package.path))
if not os.path.exists(target_file):
if not target_file.exists():
try:
os.link(package.path, target_file)
except OSError:
# hardlink failed, try symlinking
os.symlink(package.path, target_file)
package.path.symlink_to(target_file)
def cli_main():
parser = ArgumentParser()
parser.add_argument('-p', '--path', required=True)
parser.add_argument('-t', '--target', required=True)
parser.add_argument('-p', '--path', required=True, type=Path)
parser.add_argument('-t', '--target', required=True, type=Path)
namespace = parser.parse_args()