From 3395ddc9e31956c36e64f10f29ffea76b8bfb301 Mon Sep 17 00:00:00 2001 From: Daniil Anfimov Date: Fri, 24 Jun 2022 10:40:24 +0200 Subject: [PATCH 1/9] ALBS-444: Fixes for integraion in build node --- .gitignore | 4 ++++ cas_wrapper.py | 9 +++++++-- setup.py | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e928de1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class \ No newline at end of file diff --git a/cas_wrapper.py b/cas_wrapper.py index 8533dc9..3e1cffa 100644 --- a/cas_wrapper.py +++ b/cas_wrapper.py @@ -64,6 +64,7 @@ class CasWrapper: def authenticate( self, local_path: str, + return_json: bool = False, ): """ Wrapper around `cas authenticate` @@ -71,7 +72,8 @@ class CasWrapper: (should be started from `git://`) or to a single local file :return: true if a commit is trusted, vice versa - false - :rtype: bool + or dict with result if return_json param is True + :rtype: bool or dict """ command = self._cas[ 'authenticate', @@ -92,4 +94,7 @@ class CasWrapper: ): # in case if commit is untrusted result_of_execution = command(retcode=1) - return not bool(json.loads(result_of_execution)['status']) + json_result = json.loads(result_of_execution) + if return_json: + return json_result + return not bool(json_result['status']) diff --git a/setup.py b/setup.py index d7c772d..f19dd49 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup setup( name="cas_wrapper", - version="0.0.1", + version="0.0.2", author="Stepan Oksanichenko", author_email="soksanichenko@almalinux.org", description="The python wrapper around binary cas from " -- 2.40.1 From 69f0a5454c6eb1471ef9d9b88b7f91b8f6072dff Mon Sep 17 00:00:00 2001 From: Daniil Anfimov Date: Fri, 24 Jun 2022 14:50:14 +0200 Subject: [PATCH 2/9] Added context manager --- cas_wrapper.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/cas_wrapper.py b/cas_wrapper.py index 3e1cffa..e4bd8af 100644 --- a/cas_wrapper.py +++ b/cas_wrapper.py @@ -30,6 +30,18 @@ class CasWrapper: self._cas = local['cas'] self._cas['login']() + def __enter__(self): + with local.env( + CAS_API_KEY=self._cas_api_key, + SIGNER_ID=self._cas_signer_id, + ): + self._cas = local['cas'] + self._cas['login']() + return self + + def __exit__(self, exc_type, value, traceback): + self._cas['logout']() + def notarize( self, local_path: str, -- 2.40.1 From 4d05f388a288173c9db7ad89de56ca35d7b8482a Mon Sep 17 00:00:00 2001 From: Daniil Anfimov Date: Sun, 26 Jun 2022 19:01:57 +0200 Subject: [PATCH 3/9] Removed logout from __exit__ method --- cas_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cas_wrapper.py b/cas_wrapper.py index e4bd8af..8835270 100644 --- a/cas_wrapper.py +++ b/cas_wrapper.py @@ -40,7 +40,7 @@ class CasWrapper: return self def __exit__(self, exc_type, value, traceback): - self._cas['logout']() + pass def notarize( self, -- 2.40.1 From 0a94dc6f8f0c2cc59276bf481d8fd62ebe800dc6 Mon Sep 17 00:00:00 2001 From: Daniil Anfimov Date: Tue, 28 Jun 2022 21:05:51 +0200 Subject: [PATCH 4/9] Code refactoring, added functions for build and sign node integration --- cas_wrapper.py | 80 ++++++++++++++++++++++++++++++++++++++++++++------ setup.py | 1 + 2 files changed, 72 insertions(+), 9 deletions(-) diff --git a/cas_wrapper.py b/cas_wrapper.py index 8835270..7968685 100644 --- a/cas_wrapper.py +++ b/cas_wrapper.py @@ -1,7 +1,15 @@ +from concurrent.futures import ThreadPoolExecutor, as_completed import json -from typing import Dict +import logging +import typing from plumbum import local, ProcessExecutionError +from pydantic import BaseModel + + +class CasArtifact(BaseModel): + path: str + cas_hash: typing.Optional[str] class CasWrapper: @@ -16,6 +24,7 @@ class CasWrapper: self, cas_api_key: str, cas_signer_id: str, + logger: logging.Logger = None, ): if self.binary_name not in local: raise FileNotFoundError( @@ -23,19 +32,16 @@ class CasWrapper: ) self._cas_api_key = cas_api_key self._cas_signer_id = cas_signer_id - with local.env( - CAS_API_KEY=self._cas_api_key, - SIGNER_ID=self._cas_signer_id - ): - self._cas = local['cas'] - self._cas['login']() + self._cas = local['cas'] + self._logger = logger + if self._logger is None: + self._logger = logging.getLogger() def __enter__(self): with local.env( CAS_API_KEY=self._cas_api_key, SIGNER_ID=self._cas_signer_id, ): - self._cas = local['cas'] self._cas['login']() return self @@ -45,7 +51,7 @@ class CasWrapper: def notarize( self, local_path: str, - metadata: Dict = None, + metadata: typing.Dict = None, ) -> str: """ Wrapper around `cas notarize` @@ -110,3 +116,59 @@ class CasWrapper: if return_json: return json_result return not bool(json_result['status']) + + def authenticate_source( + self, + local_path: str, + ) -> typing.Tuple[bool, typing.Optional[str]]: + is_authenticated = False + commit_cas_hash = None + with self as cas: + try: + result_json = cas.authenticate(local_path, return_json=True) + # it should return 0 for authenticated and trusted commits + is_authenticated = not bool( + result_json.get('status', 1)) + commit_cas_hash = result_json.get('hash') + # we can fall with ProcessExecutionError, + # because source can be not notarized + except ProcessExecutionError: + self._logger.exception('Cannot authenticate %s:', local_path) + return is_authenticated, commit_cas_hash + + def authenticate_artifact( + self, + local_path: str, + ) -> bool: + is_authenticated = False + with self as cas: + try: + is_authenticated = cas.authenticate(local_path) + # we can fall with ProcessExecutionError, + # because source can be not notarized + except ProcessExecutionError: + self._logger.exception('Cannot authenticate %s:', local_path) + return is_authenticated + + def notarize_artifacts( + self, + artifacts: typing.List[CasArtifact], + metadata: typing.Dict[str, typing.Any], + ) -> bool: + all_artifacts_is_notarized = True + with self as cas, ThreadPoolExecutor(max_workers=4) as executor: + futures = { + executor.submit(cas.notarize, artifact.path, metadata): artifact + for artifact in artifacts + if not artifact.cas_hash + } + for future in as_completed(futures): + artifact = futures[future] + try: + cas_artifact_hash = future.result() + except Exception: + self._logger.exception('Cannot notarize artifact:') + all_artifacts_is_notarized = False + continue + artifact.cas_hash = cas_artifact_hash + return all_artifacts_is_notarized diff --git a/setup.py b/setup.py index f19dd49..5da82c7 100644 --- a/setup.py +++ b/setup.py @@ -21,6 +21,7 @@ setup( scripts=['cas_wrapper.py'], install_requires=[ 'plumbum>=1.7.2', + 'pydantic>=1.8.1', ], python_requires=">=3.6", ) -- 2.40.1 From 036d5c589ad9730aa732f6a68146d9ead76a0244 Mon Sep 17 00:00:00 2001 From: Daniil Anfimov Date: Wed, 29 Jun 2022 12:16:47 +0200 Subject: [PATCH 5/9] Added ability to authenticate artifact by hash --- cas_wrapper.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/cas_wrapper.py b/cas_wrapper.py index 7968685..0284389 100644 --- a/cas_wrapper.py +++ b/cas_wrapper.py @@ -83,22 +83,24 @@ class CasWrapper: self, local_path: str, return_json: bool = False, + use_hash: bool = False, ): """ Wrapper around `cas authenticate` :param local_path: path to a local Git repo (should be started from `git://`) - or to a single local file + or to a single local file or hash + :param return_json: flag for return json response + :param use_hash: flag for authenticate by hash :return: true if a commit is trusted, vice versa - false or dict with result if return_json param is True :rtype: bool or dict """ - command = self._cas[ - 'authenticate', - local_path, - '-o', - 'json', - ] + command_args = ['authenticate', local_path] + if use_hash: + command_args = ['authenticate', '--hash', local_path] + command_args.extend(('-o', 'json')) + command = self._cas[command_args] try: with local.env( CAS_API_KEY=self._cas_api_key, @@ -139,11 +141,13 @@ class CasWrapper: def authenticate_artifact( self, local_path: str, + use_hash: bool = False, ) -> bool: is_authenticated = False with self as cas: try: - is_authenticated = cas.authenticate(local_path) + is_authenticated = cas.authenticate(local_path, + use_hash=use_hash) # we can fall with ProcessExecutionError, # because source can be not notarized except ProcessExecutionError: -- 2.40.1 From b6dfd39655258ab91588e8685b7f81a09d1c5487 Mon Sep 17 00:00:00 2001 From: Vyacheslav Potoropin Date: Wed, 29 Jun 2022 18:50:51 +0200 Subject: [PATCH 6/9] Add return_json option to notarize_artifact --- cas_wrapper.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cas_wrapper.py b/cas_wrapper.py index 0284389..fd3c39e 100644 --- a/cas_wrapper.py +++ b/cas_wrapper.py @@ -142,12 +142,14 @@ class CasWrapper: self, local_path: str, use_hash: bool = False, + return_json: bool = False, ) -> bool: is_authenticated = False with self as cas: try: is_authenticated = cas.authenticate(local_path, - use_hash=use_hash) + use_hash=use_hash, + return_json=return_json) # we can fall with ProcessExecutionError, # because source can be not notarized except ProcessExecutionError: -- 2.40.1 From e5b832a30ad78c8c3357d9743cc04a508e81a1d2 Mon Sep 17 00:00:00 2001 From: Daniil Anfimov Date: Thu, 30 Jun 2022 12:05:57 +0200 Subject: [PATCH 7/9] Fix review comments --- cas_wrapper.py | 73 +++++++++++++++++++++++--------------------------- setup.py | 1 - 2 files changed, 33 insertions(+), 41 deletions(-) diff --git a/cas_wrapper.py b/cas_wrapper.py index fd3c39e..5f95ae4 100644 --- a/cas_wrapper.py +++ b/cas_wrapper.py @@ -4,12 +4,6 @@ import logging import typing from plumbum import local, ProcessExecutionError -from pydantic import BaseModel - - -class CasArtifact(BaseModel): - path: str - cas_hash: typing.Optional[str] class CasWrapper: @@ -37,16 +31,12 @@ class CasWrapper: if self._logger is None: self._logger = logging.getLogger() - def __enter__(self): + def ensure_login(self): with local.env( CAS_API_KEY=self._cas_api_key, SIGNER_ID=self._cas_signer_id, ): self._cas['login']() - return self - - def __exit__(self, exc_type, value, traceback): - pass def notarize( self, @@ -125,17 +115,17 @@ class CasWrapper: ) -> typing.Tuple[bool, typing.Optional[str]]: is_authenticated = False commit_cas_hash = None - with self as cas: - try: - result_json = cas.authenticate(local_path, return_json=True) - # it should return 0 for authenticated and trusted commits - is_authenticated = not bool( - result_json.get('status', 1)) - commit_cas_hash = result_json.get('hash') - # we can fall with ProcessExecutionError, - # because source can be not notarized - except ProcessExecutionError: - self._logger.exception('Cannot authenticate %s:', local_path) + self.ensure_login() + try: + result_json = self.authenticate(local_path, return_json=True) + # it should return 0 for authenticated and trusted commits + is_authenticated = not bool( + result_json.get('status', 1)) + commit_cas_hash = result_json.get('hash') + # we can fall with ProcessExecutionError, + # because source can be not notarized + except ProcessExecutionError: + self._logger.exception('Cannot authenticate %s:', local_path) return is_authenticated, commit_cas_hash def authenticate_artifact( @@ -145,36 +135,39 @@ class CasWrapper: return_json: bool = False, ) -> bool: is_authenticated = False - with self as cas: - try: - is_authenticated = cas.authenticate(local_path, - use_hash=use_hash, - return_json=return_json) - # we can fall with ProcessExecutionError, - # because source can be not notarized - except ProcessExecutionError: - self._logger.exception('Cannot authenticate %s:', local_path) + self.ensure_login() + try: + is_authenticated = self.authenticate( + local_path, + use_hash=use_hash, + return_json=return_json, + ) + # we can fall with ProcessExecutionError, + # because source can be not notarized + except ProcessExecutionError: + self._logger.exception('Cannot authenticate %s:', local_path) return is_authenticated def notarize_artifacts( self, - artifacts: typing.List[CasArtifact], + artifact_paths: typing.List[str], metadata: typing.Dict[str, typing.Any], - ) -> bool: + ) -> typing.Tuple[bool, typing.Dict[str, str]]: all_artifacts_is_notarized = True - with self as cas, ThreadPoolExecutor(max_workers=4) as executor: + notarized_artifacts = {} + self.ensure_login() + with ThreadPoolExecutor(max_workers=4) as executor: futures = { - executor.submit(cas.notarize, artifact.path, metadata): artifact - for artifact in artifacts - if not artifact.cas_hash + executor.submit(self.notarize, artifact_path, metadata): artifact_path + for artifact_path in artifact_paths } for future in as_completed(futures): - artifact = futures[future] + artifact_path = futures[future] try: cas_artifact_hash = future.result() except Exception: self._logger.exception('Cannot notarize artifact:') all_artifacts_is_notarized = False continue - artifact.cas_hash = cas_artifact_hash - return all_artifacts_is_notarized + notarized_artifacts[artifact_path] = cas_artifact_hash + return all_artifacts_is_notarized, notarized_artifacts diff --git a/setup.py b/setup.py index 5da82c7..f19dd49 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,6 @@ setup( scripts=['cas_wrapper.py'], install_requires=[ 'plumbum>=1.7.2', - 'pydantic>=1.8.1', ], python_requires=">=3.6", ) -- 2.40.1 From 07becc78df490acbdf09b7cbeb573d2625ab7494 Mon Sep 17 00:00:00 2001 From: Daniil Anfimov Date: Thu, 30 Jun 2022 15:05:12 +0200 Subject: [PATCH 8/9] Fix review comments --- cas_wrapper.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/cas_wrapper.py b/cas_wrapper.py index 5f95ae4..301e07e 100644 --- a/cas_wrapper.py +++ b/cas_wrapper.py @@ -113,19 +113,22 @@ class CasWrapper: self, local_path: str, ) -> typing.Tuple[bool, typing.Optional[str]]: + """ + Authenticates source by git path. + Returns authenticate result and source commit hash. + """ is_authenticated = False commit_cas_hash = None self.ensure_login() try: result_json = self.authenticate(local_path, return_json=True) # it should return 0 for authenticated and trusted commits - is_authenticated = not bool( - result_json.get('status', 1)) - commit_cas_hash = result_json.get('hash') + is_authenticated = not bool(result_json['status']) + commit_cas_hash = result_json['hash'] # we can fall with ProcessExecutionError, # because source can be not notarized except ProcessExecutionError: - self._logger.exception('Cannot authenticate %s:', local_path) + self._logger.exception('Cannot authenticate: %s', local_path) return is_authenticated, commit_cas_hash def authenticate_artifact( @@ -134,6 +137,10 @@ class CasWrapper: use_hash: bool = False, return_json: bool = False, ) -> bool: + """ + Authenticates artifact by artifact path or hash if `use_hash` is True. + Returns authenticate result. + """ is_authenticated = False self.ensure_login() try: @@ -145,7 +152,7 @@ class CasWrapper: # we can fall with ProcessExecutionError, # because source can be not notarized except ProcessExecutionError: - self._logger.exception('Cannot authenticate %s:', local_path) + self._logger.exception('Cannot authenticate: %s', local_path) return is_authenticated def notarize_artifacts( @@ -153,6 +160,11 @@ class CasWrapper: artifact_paths: typing.List[str], metadata: typing.Dict[str, typing.Any], ) -> typing.Tuple[bool, typing.Dict[str, str]]: + """ + Notarize artifacts by their paths. + Returns `True` if all artifacts was succesful notarizated + and dict with CAS hashes. + """ all_artifacts_is_notarized = True notarized_artifacts = {} self.ensure_login() @@ -166,7 +178,8 @@ class CasWrapper: try: cas_artifact_hash = future.result() except Exception: - self._logger.exception('Cannot notarize artifact:') + self._logger.exception('Cannot notarize artifact: %s', + artifact_path) all_artifacts_is_notarized = False continue notarized_artifacts[artifact_path] = cas_artifact_hash -- 2.40.1 From fa160270620d6fb326b9cf564901814d7c0a04cb Mon Sep 17 00:00:00 2001 From: Daniil Anfimov Date: Thu, 30 Jun 2022 15:24:56 +0200 Subject: [PATCH 9/9] Code refactoring --- cas_wrapper.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/cas_wrapper.py b/cas_wrapper.py index 301e07e..512a923 100644 --- a/cas_wrapper.py +++ b/cas_wrapper.py @@ -122,8 +122,7 @@ class CasWrapper: self.ensure_login() try: result_json = self.authenticate(local_path, return_json=True) - # it should return 0 for authenticated and trusted commits - is_authenticated = not bool(result_json['status']) + is_authenticated = result_json['verified'] commit_cas_hash = result_json['hash'] # we can fall with ProcessExecutionError, # because source can be not notarized @@ -135,7 +134,6 @@ class CasWrapper: self, local_path: str, use_hash: bool = False, - return_json: bool = False, ) -> bool: """ Authenticates artifact by artifact path or hash if `use_hash` is True. @@ -147,10 +145,10 @@ class CasWrapper: is_authenticated = self.authenticate( local_path, use_hash=use_hash, - return_json=return_json, - ) + return_json=True, + )['verified'] # we can fall with ProcessExecutionError, - # because source can be not notarized + # because artifact can be not notarized except ProcessExecutionError: self._logger.exception('Cannot authenticate: %s', local_path) return is_authenticated -- 2.40.1