|
|
|
@ -5,35 +5,43 @@ import os
|
|
|
|
|
import subprocess
|
|
|
|
|
import tempfile
|
|
|
|
|
from shutil import rmtree
|
|
|
|
|
from typing import AnyStr, List, Dict, Optional
|
|
|
|
|
from typing import (
|
|
|
|
|
AnyStr,
|
|
|
|
|
List,
|
|
|
|
|
Dict,
|
|
|
|
|
Optional,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
import createrepo_c as cr
|
|
|
|
|
import requests
|
|
|
|
|
import yaml
|
|
|
|
|
from dataclasses import dataclass, field
|
|
|
|
|
|
|
|
|
|
from .create_packages_json import PackagesGenerator, RepoInfo
|
|
|
|
|
from .create_packages_json import (
|
|
|
|
|
PackagesGenerator,
|
|
|
|
|
RepoInfo,
|
|
|
|
|
VariantInfo,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@dataclass
|
|
|
|
|
class ExtraRepoInfo(RepoInfo):
|
|
|
|
|
class ExtraVariantInfo(VariantInfo):
|
|
|
|
|
|
|
|
|
|
modules: List[AnyStr] = field(default_factory=list)
|
|
|
|
|
packages: List[AnyStr] = field(default_factory=list)
|
|
|
|
|
is_remote: bool = True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class CreateExtraRepo(PackagesGenerator):
|
|
|
|
|
|
|
|
|
|
def __init__(
|
|
|
|
|
self,
|
|
|
|
|
repos: List[ExtraRepoInfo],
|
|
|
|
|
variants: List[ExtraVariantInfo],
|
|
|
|
|
bs_auth_token: AnyStr,
|
|
|
|
|
local_repository_path: AnyStr,
|
|
|
|
|
clear_target_repo: bool = True,
|
|
|
|
|
):
|
|
|
|
|
self.repos = [] # type: List[ExtraRepoInfo]
|
|
|
|
|
super().__init__(repos, [], [])
|
|
|
|
|
self.variants = [] # type: List[ExtraVariantInfo]
|
|
|
|
|
super().__init__(variants, [], [])
|
|
|
|
|
self.auth_headers = {
|
|
|
|
|
'Authorization': f'Bearer {bs_auth_token}',
|
|
|
|
|
}
|
|
|
|
@ -92,7 +100,7 @@ class CreateExtraRepo(PackagesGenerator):
|
|
|
|
|
arch: AnyStr,
|
|
|
|
|
packages: Optional[List[AnyStr]] = None,
|
|
|
|
|
modules: Optional[List[AnyStr]] = None,
|
|
|
|
|
) -> List[ExtraRepoInfo]:
|
|
|
|
|
) -> List[ExtraVariantInfo]:
|
|
|
|
|
"""
|
|
|
|
|
Get info about a BS repo and save it to
|
|
|
|
|
an object of class ExtraRepoInfo
|
|
|
|
@ -110,7 +118,7 @@ class CreateExtraRepo(PackagesGenerator):
|
|
|
|
|
api_uri = 'api/v1'
|
|
|
|
|
bs_repo_suffix = 'build_repos'
|
|
|
|
|
|
|
|
|
|
repos_info = []
|
|
|
|
|
variants_info = []
|
|
|
|
|
|
|
|
|
|
# get the full info about a BS repo
|
|
|
|
|
repo_request = requests.get(
|
|
|
|
@ -132,22 +140,26 @@ class CreateExtraRepo(PackagesGenerator):
|
|
|
|
|
# skip repo with unsuitable architecture
|
|
|
|
|
if architecture != arch:
|
|
|
|
|
continue
|
|
|
|
|
repo_info = ExtraRepoInfo(
|
|
|
|
|
path=os.path.join(
|
|
|
|
|
bs_url,
|
|
|
|
|
bs_repo_suffix,
|
|
|
|
|
build_id,
|
|
|
|
|
platform_name,
|
|
|
|
|
),
|
|
|
|
|
folder=architecture,
|
|
|
|
|
variant_info = ExtraVariantInfo(
|
|
|
|
|
name=f'{build_id}-{platform_name}-{architecture}',
|
|
|
|
|
arch=architecture,
|
|
|
|
|
is_remote=True,
|
|
|
|
|
packages=packages,
|
|
|
|
|
modules=modules,
|
|
|
|
|
repos=[
|
|
|
|
|
RepoInfo(
|
|
|
|
|
path=os.path.join(
|
|
|
|
|
bs_url,
|
|
|
|
|
bs_repo_suffix,
|
|
|
|
|
build_id,
|
|
|
|
|
platform_name,
|
|
|
|
|
),
|
|
|
|
|
folder=architecture,
|
|
|
|
|
is_remote=True,
|
|
|
|
|
)
|
|
|
|
|
]
|
|
|
|
|
)
|
|
|
|
|
repos_info.append(repo_info)
|
|
|
|
|
return repos_info
|
|
|
|
|
variants_info.append(variant_info)
|
|
|
|
|
return variants_info
|
|
|
|
|
|
|
|
|
|
def _create_local_extra_repo(self):
|
|
|
|
|
"""
|
|
|
|
@ -184,7 +196,7 @@ class CreateExtraRepo(PackagesGenerator):
|
|
|
|
|
def _download_rpm_to_local_repo(
|
|
|
|
|
self,
|
|
|
|
|
package_location: AnyStr,
|
|
|
|
|
repo_info: ExtraRepoInfo,
|
|
|
|
|
repo_info: RepoInfo,
|
|
|
|
|
) -> None:
|
|
|
|
|
"""
|
|
|
|
|
Download a rpm package from a remote repo and save it to a local repo
|
|
|
|
@ -212,37 +224,38 @@ class CreateExtraRepo(PackagesGenerator):
|
|
|
|
|
def _download_packages(
|
|
|
|
|
self,
|
|
|
|
|
packages: Dict[AnyStr, cr.Package],
|
|
|
|
|
repo_info: ExtraRepoInfo
|
|
|
|
|
variant_info: ExtraVariantInfo
|
|
|
|
|
):
|
|
|
|
|
"""
|
|
|
|
|
Download all defined packages from a remote repo
|
|
|
|
|
:param packages: information about all packages (including
|
|
|
|
|
modularity) in a remote repo
|
|
|
|
|
:param repo_info: information about a remote repo
|
|
|
|
|
:param variant_info: information about a remote variant
|
|
|
|
|
"""
|
|
|
|
|
for package in packages.values():
|
|
|
|
|
package_name = package.name
|
|
|
|
|
# Skip a current package from a remote repo if we defined
|
|
|
|
|
# the list packages and a current package doesn't belong to it
|
|
|
|
|
if repo_info.packages and \
|
|
|
|
|
package_name not in repo_info.packages:
|
|
|
|
|
if variant_info.packages and \
|
|
|
|
|
package_name not in variant_info.packages:
|
|
|
|
|
continue
|
|
|
|
|
self._download_rpm_to_local_repo(
|
|
|
|
|
package_location=package.location_href,
|
|
|
|
|
repo_info=repo_info,
|
|
|
|
|
)
|
|
|
|
|
for repo_info in variant_info.repos:
|
|
|
|
|
self._download_rpm_to_local_repo(
|
|
|
|
|
package_location=package.location_href,
|
|
|
|
|
repo_info=repo_info,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
def _download_modules(
|
|
|
|
|
self,
|
|
|
|
|
modules_data: List[Dict],
|
|
|
|
|
repo_info: ExtraRepoInfo,
|
|
|
|
|
variant_info: ExtraVariantInfo,
|
|
|
|
|
packages: Dict[AnyStr, cr.Package]
|
|
|
|
|
):
|
|
|
|
|
"""
|
|
|
|
|
Download all defined modularity packages and their data from
|
|
|
|
|
a remote repo
|
|
|
|
|
:param modules_data: information about all modules in a remote repo
|
|
|
|
|
:param repo_info: information about a remote repo
|
|
|
|
|
:param variant_info: information about a remote variant
|
|
|
|
|
:param packages: information about all packages (including
|
|
|
|
|
modularity) in a remote repo
|
|
|
|
|
"""
|
|
|
|
@ -250,8 +263,8 @@ class CreateExtraRepo(PackagesGenerator):
|
|
|
|
|
module_data = module['data']
|
|
|
|
|
# Skip a current module from a remote repo if we defined
|
|
|
|
|
# the list modules and a current module doesn't belong to it
|
|
|
|
|
if repo_info.modules and \
|
|
|
|
|
module_data['name'] not in repo_info.modules:
|
|
|
|
|
if variant_info.modules and \
|
|
|
|
|
module_data['name'] not in variant_info.modules:
|
|
|
|
|
continue
|
|
|
|
|
# we should add info about a module if the local repodata
|
|
|
|
|
# doesn't have it
|
|
|
|
@ -266,15 +279,16 @@ class CreateExtraRepo(PackagesGenerator):
|
|
|
|
|
# Empty repo_info.packages means that we will download
|
|
|
|
|
# all packages from repo including
|
|
|
|
|
# the modularity packages
|
|
|
|
|
if not repo_info.packages:
|
|
|
|
|
if not variant_info.packages:
|
|
|
|
|
break
|
|
|
|
|
# skip a rpm if it doesn't belong to a processed repo
|
|
|
|
|
if rpm not in packages:
|
|
|
|
|
continue
|
|
|
|
|
self._download_rpm_to_local_repo(
|
|
|
|
|
package_location=packages[rpm].location_href,
|
|
|
|
|
repo_info=repo_info,
|
|
|
|
|
)
|
|
|
|
|
for repo_info in variant_info.repos:
|
|
|
|
|
self._download_rpm_to_local_repo(
|
|
|
|
|
package_location=packages[rpm].location_href,
|
|
|
|
|
repo_info=repo_info,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
def create_extra_repo(self):
|
|
|
|
|
"""
|
|
|
|
@ -284,45 +298,34 @@ class CreateExtraRepo(PackagesGenerator):
|
|
|
|
|
3. Call `createrepo_c` which creates a local repo
|
|
|
|
|
with the right repodata
|
|
|
|
|
"""
|
|
|
|
|
for repo_info in self.repos:
|
|
|
|
|
packages = {} # type: Dict[AnyStr, cr.Package]
|
|
|
|
|
repomd_records = self._get_repomd_records(
|
|
|
|
|
repo_info=repo_info,
|
|
|
|
|
)
|
|
|
|
|
repomd_records_dict = {} # type: Dict[str, str]
|
|
|
|
|
self._download_repomd_records(
|
|
|
|
|
repo_info=repo_info,
|
|
|
|
|
repomd_records=repomd_records,
|
|
|
|
|
repomd_records_dict=repomd_records_dict,
|
|
|
|
|
)
|
|
|
|
|
packages_iterator = cr.PackageIterator(
|
|
|
|
|
primary_path=repomd_records_dict['primary'],
|
|
|
|
|
filelists_path=repomd_records_dict['filelists'],
|
|
|
|
|
other_path=repomd_records_dict['other'],
|
|
|
|
|
warningcb=self._warning_callback,
|
|
|
|
|
)
|
|
|
|
|
# parse the repodata (including modules.yaml.gz)
|
|
|
|
|
modules_data = self._parse_module_repomd_record(
|
|
|
|
|
repo_info=repo_info,
|
|
|
|
|
repomd_records=repomd_records,
|
|
|
|
|
)
|
|
|
|
|
# convert the packages dict to more usable form
|
|
|
|
|
# for future checking that a rpm from the module's artifacts
|
|
|
|
|
# belongs to a processed repository
|
|
|
|
|
packages = {
|
|
|
|
|
f'{package.name}-{package.epoch}:{package.version}-'
|
|
|
|
|
f'{package.release}.{package.arch}':
|
|
|
|
|
package for package in packages_iterator
|
|
|
|
|
}
|
|
|
|
|
self._download_modules(
|
|
|
|
|
modules_data=modules_data,
|
|
|
|
|
repo_info=repo_info,
|
|
|
|
|
packages=packages,
|
|
|
|
|
)
|
|
|
|
|
self._download_packages(
|
|
|
|
|
packages=packages,
|
|
|
|
|
repo_info=repo_info,
|
|
|
|
|
)
|
|
|
|
|
for variant_info in self.variants:
|
|
|
|
|
for repo_info in variant_info.repos:
|
|
|
|
|
repomd_records = self._get_repomd_records(
|
|
|
|
|
repo_info=repo_info,
|
|
|
|
|
)
|
|
|
|
|
packages_iterator = self.get_packages_iterator(repo_info)
|
|
|
|
|
# parse the repodata (including modules.yaml.gz)
|
|
|
|
|
modules_data = self._parse_module_repomd_record(
|
|
|
|
|
repo_info=repo_info,
|
|
|
|
|
repomd_records=repomd_records,
|
|
|
|
|
)
|
|
|
|
|
# convert the packages dict to more usable form
|
|
|
|
|
# for future checking that a rpm from the module's artifacts
|
|
|
|
|
# belongs to a processed repository
|
|
|
|
|
packages = {
|
|
|
|
|
f'{package.name}-{package.epoch}:{package.version}-'
|
|
|
|
|
f'{package.release}.{package.arch}':
|
|
|
|
|
package for package in packages_iterator
|
|
|
|
|
}
|
|
|
|
|
self._download_modules(
|
|
|
|
|
modules_data=modules_data,
|
|
|
|
|
variant_info=variant_info,
|
|
|
|
|
packages=packages,
|
|
|
|
|
)
|
|
|
|
|
self._download_packages(
|
|
|
|
|
packages=packages,
|
|
|
|
|
variant_info=variant_info,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
self._dump_local_modules_yaml()
|
|
|
|
|
self._create_local_extra_repo()
|
|
|
|
@ -333,7 +336,6 @@ def create_parser():
|
|
|
|
|
parser.add_argument(
|
|
|
|
|
'--bs-auth-token',
|
|
|
|
|
help='Auth token for Build System',
|
|
|
|
|
required=True,
|
|
|
|
|
)
|
|
|
|
|
parser.add_argument(
|
|
|
|
|
'--local-repo-path',
|
|
|
|
@ -402,11 +404,16 @@ def cli_main():
|
|
|
|
|
packages = packages.split()
|
|
|
|
|
if repo.startswith('http://'):
|
|
|
|
|
repos_info.append(
|
|
|
|
|
ExtraRepoInfo(
|
|
|
|
|
path=repo,
|
|
|
|
|
folder=repo_folder,
|
|
|
|
|
ExtraVariantInfo(
|
|
|
|
|
name=repo_folder,
|
|
|
|
|
arch=repo_arch,
|
|
|
|
|
repos=[
|
|
|
|
|
RepoInfo(
|
|
|
|
|
path=repo,
|
|
|
|
|
folder=repo_folder,
|
|
|
|
|
is_remote=True,
|
|
|
|
|
)
|
|
|
|
|
],
|
|
|
|
|
modules=modules,
|
|
|
|
|
packages=packages,
|
|
|
|
|
)
|
|
|
|
@ -422,7 +429,7 @@ def cli_main():
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
cer = CreateExtraRepo(
|
|
|
|
|
repos=repos_info,
|
|
|
|
|
variants=repos_info,
|
|
|
|
|
bs_auth_token=args.bs_auth_token,
|
|
|
|
|
local_repository_path=args.local_repo_path,
|
|
|
|
|
clear_target_repo=args.clear_local_repo,
|
|
|
|
|