From cec97aac1c9fad9b5bc18d1166b63edb13ca2bcc Mon Sep 17 00:00:00 2001 From: "Owen W. Taylor" Date: Tue, 17 Dec 2024 15:35:21 +0100 Subject: [PATCH 01/09] tests/oci-registry-client.py: Drop python2 compatibility --- tests/oci-registry-client.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/oci-registry-client.py b/tests/oci-registry-client.py index 7f1eeb54..b9ca63d7 100644 --- a/tests/oci-registry-client.py +++ b/tests/oci-registry-client.py @@ -2,12 +2,9 @@ import sys -if sys.version_info[0] >= 3: - import http.client as http_client - import urllib.parse as urllib_parse -else: - import http.client as http_client - import urllib as urllib_parse +import http.client +import urllib.parse + if sys.argv[2] == 'add': detach_icons = '--detach-icons' in sys.argv @@ -16,8 +13,8 @@ if sys.argv[2] == 'add': params = {'d': sys.argv[5]} if detach_icons: params['detach-icons'] = 1 - query = urllib_parse.urlencode(params) - conn = http_client.HTTPConnection(sys.argv[1]) + query = urllib.parse.urlencode(params) + conn = http.client.HTTPConnection(sys.argv[1]) path = "/testing/{repo}/{tag}?{query}".format(repo=sys.argv[3], tag=sys.argv[4], query=query) @@ -28,7 +25,7 @@ if sys.argv[2] == 'add': print("Failed: status={}".format(response.status), file=sys.stderr) sys.exit(1) elif sys.argv[2] == 'delete': - conn = http_client.HTTPConnection(sys.argv[1]) + conn = http.client.HTTPConnection(sys.argv[1]) path = "/testing/{repo}/{ref}".format(repo=sys.argv[3], ref=sys.argv[4]) conn.request("DELETE", path) -- 2.47.1 From d8ce35c9d1c0b1c83127b07abe9b3479170cc8f6 Mon Sep 17 00:00:00 2001 From: "Owen W. Taylor" Date: Tue, 17 Dec 2024 15:41:53 +0100 Subject: [PATCH 02/09] tests/oci-registry-client.py: Parse URL parameter --- tests/oci-registry-client.py | 9 +++++++-- tests/test-oci-registry.sh | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/oci-registry-client.py b/tests/oci-registry-client.py index b9ca63d7..cd835609 100644 --- a/tests/oci-registry-client.py +++ b/tests/oci-registry-client.py @@ -6,6 +6,11 @@ import http.client import urllib.parse +def get_conn(url): + parsed = urllib.parse.urlparse(url) + return http.client.HTTPConnection(host=parsed.hostname, port=parsed.port) + + if sys.argv[2] == 'add': detach_icons = '--detach-icons' in sys.argv if detach_icons: @@ -14,7 +19,7 @@ if sys.argv[2] == 'add': if detach_icons: params['detach-icons'] = 1 query = urllib.parse.urlencode(params) - conn = http.client.HTTPConnection(sys.argv[1]) + conn = get_conn(sys.argv[1]) path = "/testing/{repo}/{tag}?{query}".format(repo=sys.argv[3], tag=sys.argv[4], query=query) @@ -25,7 +30,7 @@ if sys.argv[2] == 'add': print("Failed: status={}".format(response.status), file=sys.stderr) sys.exit(1) elif sys.argv[2] == 'delete': - conn = http.client.HTTPConnection(sys.argv[1]) + conn = get_conn(sys.argv[1]) path = "/testing/{repo}/{ref}".format(repo=sys.argv[3], ref=sys.argv[4]) conn.request("DELETE", path) diff --git a/tests/test-oci-registry.sh b/tests/test-oci-registry.sh index 8eb154f5..51a6142f 100755 --- a/tests/test-oci-registry.sh +++ b/tests/test-oci-registry.sh @@ -29,7 +29,7 @@ echo "1..14" httpd oci-registry-server.py . port=$(cat httpd-port) -client="python3 $test_srcdir/oci-registry-client.py 127.0.0.1:$port" +client="python3 $test_srcdir/oci-registry-client.py http://127.0.0.1:$port" setup_repo_no_add oci -- 2.47.1 From 0757171aa07f3b8d390881e0765de09ed77d4825 Mon Sep 17 00:00:00 2001 From: "Owen W. Taylor" Date: Tue, 17 Dec 2024 15:47:07 +0100 Subject: [PATCH 03/09] tests/oci-registry-client.py: Convert to argparse --- tests/oci-registry-client.py | 57 +++++++++++++++++++++++------------- tests/test-oci-registry.sh | 2 +- 2 files changed, 38 insertions(+), 21 deletions(-) mode change 100644 => 100755 tests/oci-registry-client.py diff --git a/tests/oci-registry-client.py b/tests/oci-registry-client.py old mode 100644 new mode 100755 index cd835609..5654062b --- a/tests/oci-registry-client.py +++ b/tests/oci-registry-client.py @@ -1,45 +1,62 @@ #!/usr/bin/python3 +import argparse import sys import http.client import urllib.parse -def get_conn(url): - parsed = urllib.parse.urlparse(url) +def get_conn(args): + parsed = urllib.parse.urlparse(args.url) return http.client.HTTPConnection(host=parsed.hostname, port=parsed.port) -if sys.argv[2] == 'add': - detach_icons = '--detach-icons' in sys.argv - if detach_icons: - sys.argv.remove('--detach-icons') - params = {'d': sys.argv[5]} - if detach_icons: - params['detach-icons'] = 1 +def run_add(args): + params = {"d": args.oci_dir} + if args.detach_icons: + params["detach-icons"] = "1" query = urllib.parse.urlencode(params) - conn = get_conn(sys.argv[1]) - path = "/testing/{repo}/{tag}?{query}".format(repo=sys.argv[3], - tag=sys.argv[4], - query=query) + conn = get_conn(args) + path = "/testing/{repo}/{tag}?{query}".format( + repo=args.repo, tag=args.tag, query=query + ) conn.request("POST", path) response = conn.getresponse() if response.status != 200: print(response.read(), file=sys.stderr) print("Failed: status={}".format(response.status), file=sys.stderr) sys.exit(1) -elif sys.argv[2] == 'delete': - conn = get_conn(sys.argv[1]) - path = "/testing/{repo}/{ref}".format(repo=sys.argv[3], - ref=sys.argv[4]) + + +def run_delete(args): + conn = get_conn(args) + path = "/testing/{repo}/{ref}".format(repo=args.repo, ref=args.ref) conn.request("DELETE", path) response = conn.getresponse() if response.status != 200: print(response.read(), file=sys.stderr) print("Failed: status={}".format(response.status), file=sys.stderr) sys.exit(1) -else: - print("Usage: oci-registry-client.py [add|remove] ARGS", file=sys.stderr) - sys.exit(1) + +parser = argparse.ArgumentParser() +parser.add_argument("--url", required=True) + +subparsers = parser.add_subparsers() +subparsers.required = True + +add_parser = subparsers.add_parser("add") +add_parser.add_argument("repo") +add_parser.add_argument("tag") +add_parser.add_argument("oci_dir") +add_parser.add_argument("--detach-icons", action="store_true", default=False) +add_parser.set_defaults(func=run_add) + +delete_parser = subparsers.add_parser("delete") +delete_parser.add_argument("repo") +delete_parser.add_argument("ref") +delete_parser.set_defaults(func=run_delete) + +args = parser.parse_args() +args.func(args) diff --git a/tests/test-oci-registry.sh b/tests/test-oci-registry.sh index 51a6142f..bc2f138b 100755 --- a/tests/test-oci-registry.sh +++ b/tests/test-oci-registry.sh @@ -29,7 +29,7 @@ echo "1..14" httpd oci-registry-server.py . port=$(cat httpd-port) -client="python3 $test_srcdir/oci-registry-client.py http://127.0.0.1:$port" +client="python3 $test_srcdir/oci-registry-client.py --url=http://127.0.0.1:$port" setup_repo_no_add oci -- 2.47.1 From f38197c03dc77b9192c6dbc59e175b3cb640614a Mon Sep 17 00:00:00 2001 From: Sebastian Wick Date: Thu, 22 Aug 2024 02:43:13 -0400 Subject: [PATCH 04/09] tests/oci-registry-server.py: Clean up Python style --- .flake8 | 2 + tests/oci-registry-server.py | 170 ++++++++++++++++++----------------- 2 files changed, 88 insertions(+), 84 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..7da1f960 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 100 diff --git a/tests/oci-registry-server.py b/tests/oci-registry-server.py index 33c3b646..3050a883 100755 --- a/tests/oci-registry-server.py +++ b/tests/oci-registry-server.py @@ -13,65 +13,61 @@ import http.server as http_server repositories = {} icons = {} + def get_index(): results = [] for repo_name in sorted(repositories.keys()): repo = repositories[repo_name] - results.append({ - 'Name': repo_name, - 'Images': repo['images'], - 'Lists': [], - }) + results.append( + { + "Name": repo_name, + "Images": repo["images"], + "Lists": [], + } + ) + + return json.dumps({"Registry": "/", "Results": results}, indent=4) - return json.dumps({ - 'Registry': '/', - 'Results': results - }, indent=4) def cache_icon(data_uri): - prefix = 'data:image/png;base64,' + prefix = "data:image/png;base64," assert data_uri.startswith(prefix) - data = base64.b64decode(data_uri[len(prefix):]) + data = base64.b64decode(data_uri[len(prefix) :]) h = hashlib.sha256() h.update(data) digest = h.hexdigest() - filename = digest + '.png' + filename = digest + ".png" icons[filename] = data - return '/icons/' + filename + return "/icons/" + filename + serial = 0 server_start_time = int(time.time()) + def get_etag(): - return str(server_start_time) + '-' + str(serial) + return str(server_start_time) + "-" + str(serial) + def modified(): global serial serial += 1 -def parse_http_date(date): - parsed = parsedate(date) - if parsed is not None: - return timegm(parsed) - else: - return None class RequestHandler(http_server.BaseHTTPRequestHandler): def check_route(self, route): - parts = self.path.split('?', 1) - path = parts[0].split('/') - - result = [] + parts = self.path.split("?", 1) + path = parts[0].split("/") - route_path = route.split('/') + route_path = route.split("/") print((route_path, path)) if len(route_path) != len(path): return False matches = {} for i in range(1, len(route_path)): - if route_path[i][0] == '@': + if route_path[i][0] == "@": matches[route_path[i][1:]] = path[i] elif route_path[i] != path[i]: return False @@ -92,24 +88,25 @@ class RequestHandler(http_server.BaseHTTPRequestHandler): add_headers = {} - if self.check_route('/v2/@repo_name/blobs/@digest'): - repo_name = self.matches['repo_name'] - digest = self.matches['digest'] - response_file = repositories[repo_name]['blobs'][digest] - elif self.check_route('/v2/@repo_name/manifests/@ref'): - repo_name = self.matches['repo_name'] - ref = self.matches['ref'] - response_file = repositories[repo_name]['manifests'][ref] - elif self.check_route('/index/static') or self.check_route('/index/dynamic'): + if self.check_route("/v2/@repo_name/blobs/@digest"): + repo_name = self.matches["repo_name"] + digest = self.matches["digest"] + response_file = repositories[repo_name]["blobs"][digest] + elif self.check_route("/v2/@repo_name/manifests/@ref"): + repo_name = self.matches["repo_name"] + ref = self.matches["ref"] + response_file = repositories[repo_name]["manifests"][ref] + elif self.check_route("/index/static") or self.check_route("/index/dynamic"): etag = get_etag() if self.headers.get("If-None-Match") == etag: response = 304 else: response_string = get_index() - add_headers['Etag'] = etag - elif self.check_route('/icons/@filename') : - response_string = icons[self.matches['filename']] - response_content_type = 'image/png' + add_headers["Etag"] = etag + elif self.check_route("/icons/@filename"): + response_string = icons[self.matches["filename"]] + assert isinstance(response_string, bytes) + response_content_type = "image/png" else: response = 404 @@ -121,86 +118,89 @@ class RequestHandler(http_server.BaseHTTPRequestHandler): self.send_header("Content-Type", response_content_type) if response == 200 or response == 304: - self.send_header('Cache-Control', 'no-cache') + self.send_header("Cache-Control", "no-cache") self.end_headers() if response == 200: if response_file: - with open(response_file, 'rb') as f: + with open(response_file, "rb") as f: response_string = f.read() if isinstance(response_string, bytes): self.wfile.write(response_string) else: - self.wfile.write(response_string.encode('utf-8')) + assert isinstance(response_string, str) + self.wfile.write(response_string.encode("utf-8")) def do_HEAD(self): return self.do_GET() def do_POST(self): - if self.check_route('/testing/@repo_name/@tag'): - repo_name = self.matches['repo_name'] - tag = self.matches['tag'] - d = self.query['d'][0] - detach_icons = 'detach-icons' in self.query + if self.check_route("/testing/@repo_name/@tag"): + repo_name = self.matches["repo_name"] + tag = self.matches["tag"] + d = self.query["d"][0] + detach_icons = "detach-icons" in self.query repo = repositories.setdefault(repo_name, {}) - blobs = repo.setdefault('blobs', {}) - manifests = repo.setdefault('manifests', {}) - images = repo.setdefault('images', []) + blobs = repo.setdefault("blobs", {}) + manifests = repo.setdefault("manifests", {}) + images = repo.setdefault("images", []) - with open(os.path.join(d, 'index.json')) as f: + with open(os.path.join(d, "index.json")) as f: index = json.load(f) - manifest_digest = index['manifests'][0]['digest'] - manifest_path = os.path.join(d, 'blobs', *manifest_digest.split(':')) + manifest_digest = index["manifests"][0]["digest"] + manifest_path = os.path.join(d, "blobs", *manifest_digest.split(":")) manifests[manifest_digest] = manifest_path manifests[tag] = manifest_path with open(manifest_path) as f: manifest = json.load(f) - config_digest = manifest['config']['digest'] - config_path = os.path.join(d, 'blobs', *config_digest.split(':')) + config_digest = manifest["config"]["digest"] + config_path = os.path.join(d, "blobs", *config_digest.split(":")) with open(config_path) as f: config = json.load(f) - for dig in os.listdir(os.path.join(d, 'blobs', 'sha256')): - digest = 'sha256:' + dig - path = os.path.join(d, 'blobs', 'sha256', dig) + for dig in os.listdir(os.path.join(d, "blobs", "sha256")): + digest = "sha256:" + dig + path = os.path.join(d, "blobs", "sha256", dig) if digest != manifest_digest: blobs[digest] = path if detach_icons: for size in (64, 128): - annotation = 'org.freedesktop.appstream.icon-{}'.format(size) - icon = manifest.get('annotations', {}).get(annotation) + annotation = "org.freedesktop.appstream.icon-{}".format(size) + icon = manifest.get("annotations", {}).get(annotation) if icon: path = cache_icon(icon) - manifest['annotations'][annotation] = path + manifest["annotations"][annotation] = path else: - icon = config.get('config', {}).get('Labels', {}).get(annotation) + icon = ( + config.get("config", {}).get("Labels", {}).get(annotation) + ) if icon: path = cache_icon(icon) - config['config']['Labels'][annotation] = path + config["config"]["Labels"][annotation] = path image = { "Tags": [tag], "Digest": manifest_digest, "MediaType": "application/vnd.oci.image.manifest.v1+json", - "OS": config['os'], - "Architecture": config['architecture'], - "Annotations": manifest.get('annotations', {}), - "Labels": config.get('config', {}).get('Labels', {}), + "OS": config["os"], + "Architecture": config["architecture"], + "Annotations": manifest.get("annotations", {}), + "Labels": config.get("config", {}).get("Labels", {}), } # Delete old versions for i in images: - if tag in i['Tags']: + if tag in i["Tags"]: images.remove(i) - del manifests[i['Digest']] + del manifests[i["Digest"]] images.append(image) @@ -214,26 +214,26 @@ class RequestHandler(http_server.BaseHTTPRequestHandler): return def do_DELETE(self): - if self.check_route('/testing/@repo_name/@ref'): - repo_name = self.matches['repo_name'] - ref = self.matches['ref'] + if self.check_route("/testing/@repo_name/@ref"): + repo_name = self.matches["repo_name"] + ref = self.matches["ref"] repo = repositories.setdefault(repo_name, {}) - blobs = repo.setdefault('blobs', {}) - manifests = repo.setdefault('manifests', {}) - images = repo.setdefault('images', []) + repo.setdefault("blobs", {}) + manifests = repo.setdefault("manifests", {}) + images = repo.setdefault("images", []) image = None for i in images: - if i['Digest'] == ref or ref in i['Tags']: + if i["Digest"] == ref or ref in i["Tags"]: image = i break assert image images.remove(image) - del manifests[image['Digest']] - for t in image['Tags']: + del manifests[image["Digest"]] + for t in image["Tags"]: del manifests[t] modified() @@ -245,22 +245,24 @@ class RequestHandler(http_server.BaseHTTPRequestHandler): self.end_headers() return + def run(dir): RequestHandler.protocol_version = "HTTP/1.0" - httpd = http_server.HTTPServer( ("127.0.0.1", 0), RequestHandler) + httpd = http_server.HTTPServer(("127.0.0.1", 0), RequestHandler) host, port = httpd.socket.getsockname()[:2] - with open("httpd-port", 'w') as file: + with open("httpd-port", "w") as file: file.write("%d" % port) try: - os.write(3, bytes("Started\n", 'utf-8')); - except: + os.write(3, bytes("Started\n", "utf-8")) + except OSError: pass - print("Serving HTTP on port %d" % port); + print("Serving HTTP on port %d" % port) if dir: os.chdir(dir) httpd.serve_forever() -if __name__ == '__main__': + +if __name__ == "__main__": dir = None if len(sys.argv) >= 2 and len(sys.argv[1]) > 0: dir = sys.argv[1] -- 2.47.1 From 47ff75860097afe0b07758b2457a52f0e6b94bd7 Mon Sep 17 00:00:00 2001 From: "Owen W. Taylor" Date: Thu, 22 Aug 2024 02:47:19 -0400 Subject: [PATCH 05/09] tests/oci-registry-server.py: Convert to argparse --- tests/oci-registry-server.py | 15 ++++++++------- tests/test-oci-registry.sh | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/oci-registry-server.py b/tests/oci-registry-server.py index 3050a883..65421f34 100755 --- a/tests/oci-registry-server.py +++ b/tests/oci-registry-server.py @@ -1,5 +1,6 @@ #!/usr/bin/python3 +import argparse import base64 import hashlib import json @@ -246,7 +247,7 @@ class RequestHandler(http_server.BaseHTTPRequestHandler): return -def run(dir): +def run(args): RequestHandler.protocol_version = "HTTP/1.0" httpd = http_server.HTTPServer(("127.0.0.1", 0), RequestHandler) host, port = httpd.socket.getsockname()[:2] @@ -257,14 +258,14 @@ def run(dir): except OSError: pass print("Serving HTTP on port %d" % port) - if dir: - os.chdir(dir) + if args.dir: + os.chdir(args.dir) httpd.serve_forever() if __name__ == "__main__": - dir = None - if len(sys.argv) >= 2 and len(sys.argv[1]) > 0: - dir = sys.argv[1] + parser = argparse.ArgumentParser() + parser.add_argument("--dir") + args = parser.parse_args() - run(dir) + run(args) diff --git a/tests/test-oci-registry.sh b/tests/test-oci-registry.sh index bc2f138b..12036358 100755 --- a/tests/test-oci-registry.sh +++ b/tests/test-oci-registry.sh @@ -27,7 +27,7 @@ echo "1..14" # Start the fake registry server -httpd oci-registry-server.py . +httpd oci-registry-server.py --dir=. port=$(cat httpd-port) client="python3 $test_srcdir/oci-registry-client.py --url=http://127.0.0.1:$port" -- 2.47.1 From c2c2f3679608a32339dadb233dc11633f22b0793 Mon Sep 17 00:00:00 2001 From: Sebastian Wick Date: Tue, 17 Dec 2024 16:39:16 +0100 Subject: [PATCH 06/09] tests/oci-registry-server.py: Always get bytes for the response And sent the Content-Length header. --- tests/oci-registry-server.py | 46 +++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/tests/oci-registry-server.py b/tests/oci-registry-server.py index 65421f34..13bf50b3 100755 --- a/tests/oci-registry-server.py +++ b/tests/oci-registry-server.py @@ -5,7 +5,6 @@ import base64 import hashlib import json import os -import sys import time from urllib.parse import parse_qs @@ -27,7 +26,7 @@ def get_index(): } ) - return json.dumps({"Registry": "/", "Results": results}, indent=4) + return json.dumps({"Registry": "/", "Results": results}, indent=4).encode("UTF-8") def cache_icon(data_uri): @@ -62,7 +61,6 @@ class RequestHandler(http_server.BaseHTTPRequestHandler): path = parts[0].split("/") route_path = route.split("/") - print((route_path, path)) if len(route_path) != len(path): return False @@ -82,34 +80,47 @@ class RequestHandler(http_server.BaseHTTPRequestHandler): return True def do_GET(self): - response = 200 - response_string = None + response = 404 + response_string = b"" response_content_type = "application/octet-stream" - response_file = None add_headers = {} + def get_file_contents(repo_name, type, ref): + try: + path = repositories[repo_name][type][ref] + with open(path, "rb") as f: + return 200, f.read() + except KeyError: + return 404, b"" + if self.check_route("/v2/@repo_name/blobs/@digest"): repo_name = self.matches["repo_name"] digest = self.matches["digest"] - response_file = repositories[repo_name]["blobs"][digest] + response, response_string = get_file_contents(repo_name, "blobs", digest) elif self.check_route("/v2/@repo_name/manifests/@ref"): repo_name = self.matches["repo_name"] ref = self.matches["ref"] - response_file = repositories[repo_name]["manifests"][ref] + response, response_string = get_file_contents(repo_name, "manifests", ref) elif self.check_route("/index/static") or self.check_route("/index/dynamic"): etag = get_etag() + add_headers["Etag"] = etag if self.headers.get("If-None-Match") == etag: response = 304 + response_string = b"" else: + response = 200 response_string = get_index() - add_headers["Etag"] = etag elif self.check_route("/icons/@filename"): + response = 200 response_string = icons[self.matches["filename"]] - assert isinstance(response_string, bytes) response_content_type = "image/png" - else: - response = 404 + + assert isinstance(response, int) + assert isinstance(response_string, bytes) + assert isinstance(response_content_type, str) + + add_headers["Content-Length"] = len(response_string) self.send_response(response) for k, v in list(add_headers.items()): @@ -123,16 +134,7 @@ class RequestHandler(http_server.BaseHTTPRequestHandler): self.end_headers() - if response == 200: - if response_file: - with open(response_file, "rb") as f: - response_string = f.read() - - if isinstance(response_string, bytes): - self.wfile.write(response_string) - else: - assert isinstance(response_string, str) - self.wfile.write(response_string.encode("utf-8")) + self.wfile.write(response_string) def do_HEAD(self): return self.do_GET() -- 2.47.1 From 85379a0fe6b79d216a29497980ce2c8e0ab46997 Mon Sep 17 00:00:00 2001 From: "Owen W. Taylor" Date: Tue, 17 Dec 2024 16:41:38 +0100 Subject: [PATCH 07/09] tests/oci-registry: Add support for SSL to client and server --- tests/oci-registry-client.py | 19 ++++++++++++++++++- tests/oci-registry-server.py | 22 +++++++++++++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/tests/oci-registry-client.py b/tests/oci-registry-client.py index 5654062b..c4707c07 100755 --- a/tests/oci-registry-client.py +++ b/tests/oci-registry-client.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 import argparse +import ssl import sys import http.client @@ -9,7 +10,20 @@ import urllib.parse def get_conn(args): parsed = urllib.parse.urlparse(args.url) - return http.client.HTTPConnection(host=parsed.hostname, port=parsed.port) + if parsed.scheme == "http": + return http.client.HTTPConnection(host=parsed.hostname, port=parsed.port) + elif parsed.scheme == "https": + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.options |= ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 + if args.cert: + context.load_cert_chain(certfile=args.cert, keyfile=args.key) + if args.cacert: + context.load_verify_locations(cafile=args.cacert) + return http.client.HTTPSConnection( + host=parsed.hostname, port=parsed.port, context=context + ) + else: + assert False, "Bad scheme: " + parsed.scheme def run_add(args): @@ -42,6 +56,9 @@ def run_delete(args): parser = argparse.ArgumentParser() parser.add_argument("--url", required=True) +parser.add_argument("--cacert") +parser.add_argument("--cert") +parser.add_argument("--key") subparsers = parser.add_subparsers() subparsers.required = True diff --git a/tests/oci-registry-server.py b/tests/oci-registry-server.py index 13bf50b3..2bbe8c6e 100755 --- a/tests/oci-registry-server.py +++ b/tests/oci-registry-server.py @@ -5,6 +5,7 @@ import base64 import hashlib import json import os +import ssl import time from urllib.parse import parse_qs @@ -252,6 +253,19 @@ class RequestHandler(http_server.BaseHTTPRequestHandler): def run(args): RequestHandler.protocol_version = "HTTP/1.0" httpd = http_server.HTTPServer(("127.0.0.1", 0), RequestHandler) + + if args.cert: + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.options |= ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 + context.load_cert_chain(certfile=args.cert, keyfile=args.key) + + if args.mtls_cacert: + context.load_verify_locations(cafile=args.mtls_cacert) + # In a real application, we'd need to check the CN against authorized users + context.verify_mode = ssl.CERT_REQUIRED + + httpd.socket = context.wrap_socket(httpd.socket, server_side=True) + host, port = httpd.socket.getsockname()[:2] with open("httpd-port", "w") as file: file.write("%d" % port) @@ -259,7 +273,10 @@ def run(args): os.write(3, bytes("Started\n", "utf-8")) except OSError: pass - print("Serving HTTP on port %d" % port) + if args.cert: + print("Serving HTTPS on port %d" % port) + else: + print("Serving HTTP on port %d" % port) if args.dir: os.chdir(args.dir) httpd.serve_forever() @@ -268,6 +285,9 @@ def run(args): if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--dir") + parser.add_argument("--cert") + parser.add_argument("--key") + parser.add_argument("--mtls-cacert") args = parser.parse_args() run(args) -- 2.47.1 From 68b3fdcc0b00ee1080a1a3cd15dff33f1de6c0bf Mon Sep 17 00:00:00 2001 From: "Owen W. Taylor" Date: Fri, 23 Aug 2024 09:48:26 -0400 Subject: [PATCH 08/09] common: Implement /etc/containers/certs.d for OCI registries Docker and podman can be configured to use mutual TLS authentication to the registry by dropping files into system-wide and user directories. Implement this in a largely compatible way. (Because of the limitations of our underlying libraries, we can't support multiple certificates within the same host config, but I don't expect anybody actually needs that.) The certs.d handling is extended so that certificates are separately looked up when downloading the look-aside index. This is mostly to simplify our tests, so we can use one web server for both - in actual operation, we expect the indexes to be unauthenticated. Also for testing purposes, FLATPAK_CONTAINER_CERTS_D is supported to override the standard search path. Co-authored-by: Sebastian Wick --- .github/workflows/check.yml | 2 +- common/flatpak-oci-registry.c | 86 +++++-- common/flatpak-utils-http-private.h | 12 + common/flatpak-utils-http.c | 368 +++++++++++++++++++++++++++- doc/flatpak-remote.xml | 6 + tests/httpcache.c | 2 +- 6 files changed, 440 insertions(+), 36 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index dbf44c1d..6cce2a6f 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -52,7 +52,7 @@ jobs: libjson-glib-dev shared-mime-info desktop-file-utils libpolkit-agent-1-dev libpolkit-gobject-1-dev \ libseccomp-dev libsoup2.4-dev libcurl4-openssl-dev libsystemd-dev libxml2-utils libgpgme11-dev gobject-introspection \ libgirepository1.0-dev libappstream-dev libdconf-dev clang socat meson libdbus-1-dev e2fslibs-dev bubblewrap xdg-dbus-proxy \ - python3-pip meson ninja-build libyaml-dev libstemmer-dev gperf itstool libmalcontent-0-dev + python3-pip meson ninja-build libyaml-dev libstemmer-dev gperf itstool libmalcontent-0-dev openssl # One of the tests wants this sudo mkdir /tmp/flatpak-com.example.App-OwnedByRoot - name: Check out flatpak diff --git a/common/flatpak-oci-registry.c b/common/flatpak-oci-registry.c index ad595e53..6d36de2a 100644 --- a/common/flatpak-oci-registry.c +++ b/common/flatpak-oci-registry.c @@ -76,7 +76,8 @@ struct FlatpakOciRegistry /* Remote repos */ FlatpakHttpSession *http_session; - GUri *base_uri; + GUri *base_uri; + FlatpakCertificates *certificates; }; typedef struct @@ -353,31 +354,29 @@ choose_alt_uri (GUri *base_uri, } static GBytes * -remote_load_file (FlatpakHttpSession *http_session, - GUri *base, - const char *subpath, - const char **alt_uris, - const char *token, - char **out_content_type, - GCancellable *cancellable, - GError **error) +remote_load_file (FlatpakOciRegistry *self, + const char *subpath, + const char **alt_uris, + char **out_content_type, + GCancellable *cancellable, + GError **error) { g_autoptr(GBytes) bytes = NULL; g_autofree char *uri_s = NULL; - uri_s = choose_alt_uri (base, alt_uris); + uri_s = choose_alt_uri (self->base_uri, alt_uris); if (uri_s == NULL) { - uri_s = parse_relative_uri (base, subpath, error); + uri_s = parse_relative_uri (self->base_uri, subpath, error); if (uri_s == NULL) return NULL; } - bytes = flatpak_load_uri (http_session, - uri_s, FLATPAK_HTTP_FLAGS_ACCEPT_OCI, - token, - NULL, NULL, out_content_type, - cancellable, error); + bytes = flatpak_load_uri_full (self->http_session, + uri_s, self->certificates, FLATPAK_HTTP_FLAGS_ACCEPT_OCI, + NULL, self->token, + NULL, NULL, NULL, out_content_type, NULL, + cancellable, error); if (bytes == NULL) return NULL; @@ -395,7 +394,7 @@ flatpak_oci_registry_load_file (FlatpakOciRegistry *self, if (self->dfd != -1) return local_load_file (self->dfd, subpath, cancellable, error); else - return remote_load_file (self->http_session, self->base_uri, subpath, alt_uris, self->token, out_content_type, cancellable, error); + return remote_load_file (self, subpath, alt_uris, out_content_type, cancellable, error); } static JsonNode * @@ -548,6 +547,7 @@ flatpak_oci_registry_ensure_remote (FlatpakOciRegistry *self, GError **error) { g_autoptr(GUri) baseuri = NULL; + g_autoptr(GError) local_error = NULL; if (for_write) { @@ -568,6 +568,13 @@ flatpak_oci_registry_ensure_remote (FlatpakOciRegistry *self, self->is_docker = TRUE; self->base_uri = g_steal_pointer (&baseuri); + self->certificates = flatpak_get_certificates_for_uri (self->uri, &local_error); + if (local_error) + { + g_propagate_error (error, g_steal_pointer (&local_error)); + return FALSE; + } + return TRUE; } @@ -834,6 +841,7 @@ flatpak_oci_registry_download_blob (FlatpakOciRegistry *self, return -1; if (!flatpak_download_http_uri (self->http_session, uri_s, + self->certificates, FLATPAK_HTTP_FLAGS_ACCEPT_OCI, out_stream, self->token, @@ -925,7 +933,8 @@ flatpak_oci_registry_mirror_blob (FlatpakOciRegistry *self, out_stream = g_unix_output_stream_new (tmpf.fd, FALSE); - if (!flatpak_download_http_uri (source_registry->http_session, uri_s, + if (!flatpak_download_http_uri (source_registry->http_session, + uri_s, source_registry->certificates, FLATPAK_HTTP_FLAGS_ACCEPT_OCI, out_stream, self->token, progress_cb, user_data, @@ -1055,6 +1064,7 @@ get_token_for_www_auth (FlatpakOciRegistry *self, body = flatpak_load_uri_full (self->http_session, auth_uri_s, + self->certificates, FLATPAK_HTTP_FLAGS_NOCHECK_STATUS, auth, NULL, NULL, NULL, @@ -1146,7 +1156,7 @@ flatpak_oci_registry_get_token (FlatpakOciRegistry *self, if (uri_s == NULL) return NULL; - body = flatpak_load_uri_full (self->http_session, uri_s, + body = flatpak_load_uri_full (self->http_session, uri_s, self->certificates, FLATPAK_HTTP_FLAGS_ACCEPT_OCI | FLATPAK_HTTP_FLAGS_HEAD | FLATPAK_HTTP_FLAGS_NOCHECK_STATUS, NULL, NULL, NULL, NULL, @@ -2080,11 +2090,11 @@ flatpak_oci_registry_find_delta_manifest (FlatpakOciRegistry *registry, g_autofree char *uri_s = parse_relative_uri (registry->base_uri, delta_manifest_url, NULL); if (uri_s != NULL) - bytes = flatpak_load_uri (registry->http_session, - uri_s, FLATPAK_HTTP_FLAGS_ACCEPT_OCI, - registry->token, - NULL, NULL, NULL, - cancellable, NULL); + bytes = flatpak_load_uri_full (registry->http_session, + uri_s, registry->certificates, FLATPAK_HTTP_FLAGS_ACCEPT_OCI, + NULL, registry->token, + NULL, NULL, NULL, NULL, NULL, + cancellable, NULL); if (bytes != NULL) { g_autoptr(FlatpakOciVersioned) versioned = @@ -2760,6 +2770,7 @@ flatpak_oci_index_ensure_cached (FlatpakHttpSession *http_session, g_autofree char *tag = NULL; const char *oci_arch = NULL; gboolean success = FALSE; + g_autoptr(FlatpakCertificates) certificates = NULL; g_autoptr(GError) local_error = NULL; GUri *tmp_uri; @@ -2844,8 +2855,16 @@ flatpak_oci_index_ensure_cached (FlatpakHttpSession *http_session, query_uri_s = g_uri_to_string_partial (query_uri, G_URI_HIDE_PASSWORD); + certificates = flatpak_get_certificates_for_uri (query_uri_s, &local_error); + if (local_error) + { + g_propagate_error (error, g_steal_pointer (&local_error)); + return FALSE; + } + success = flatpak_cache_http_uri (http_session, query_uri_s, + certificates, FLATPAK_HTTP_FLAGS_STORE_COMPRESSED, AT_FDCWD, index_path, NULL, NULL, @@ -3082,6 +3101,7 @@ flatpak_oci_index_make_summary (GFile *index, static gboolean add_icon_image (FlatpakHttpSession *http_session, const char *index_uri, + FlatpakCertificates *certificates, int icons_dfd, GHashTable *used_icons, const char *subdir, @@ -3132,7 +3152,7 @@ add_icon_image (FlatpakHttpSession *http_session, if (icon_uri_s == NULL) return FALSE; - if (!flatpak_cache_http_uri (http_session, icon_uri_s, + if (!flatpak_cache_http_uri (http_session, icon_uri_s, certificates, 0 /* flags */, icons_dfd, icon_path, NULL, NULL, @@ -3152,6 +3172,7 @@ add_icon_image (FlatpakHttpSession *http_session, static void add_image_to_appstream (FlatpakHttpSession *http_session, const char *index_uri, + FlatpakCertificates *certificates, FlatpakXml *appstream_root, int icons_dfd, GHashTable *used_icons, @@ -3243,6 +3264,7 @@ add_image_to_appstream (FlatpakHttpSession *http_session, { if (!add_icon_image (http_session, index_uri, + certificates, icons_dfd, used_icons, icon_sizes[i].subdir, id, icon_data, @@ -3338,6 +3360,8 @@ flatpak_oci_index_make_appstream (FlatpakHttpSession *http_session, g_autoptr(FlatpakXml) appstream_root = NULL; g_autoptr(GBytes) bytes = NULL; g_autoptr(GHashTable) used_icons = NULL; + g_autoptr(FlatpakCertificates) certificates = NULL; + g_autoptr(GError) local_error = NULL; int i; const char *oci_arch = flatpak_arch_to_oci_arch (arch); @@ -3351,6 +3375,14 @@ flatpak_oci_index_make_appstream (FlatpakHttpSession *http_session, appstream_root = flatpak_appstream_xml_new (); + certificates = flatpak_get_certificates_for_uri (index_uri, &local_error); + if (local_error) + { + g_print ("Failed to load certificates for %s: %s", + index_uri, local_error->message); + g_clear_error (&local_error); + } + for (i = 0; response->results != NULL && response->results[i] != NULL; i++) { FlatpakOciIndexRepository *r = response->results[i]; @@ -3361,7 +3393,7 @@ flatpak_oci_index_make_appstream (FlatpakHttpSession *http_session, FlatpakOciIndexImage *image = r->images[j]; if (g_strcmp0 (image->architecture, oci_arch) == 0) add_image_to_appstream (http_session, - index_uri, + index_uri, certificates, appstream_root, icons_dfd, used_icons, r, image, cancellable); @@ -3377,7 +3409,7 @@ flatpak_oci_index_make_appstream (FlatpakHttpSession *http_session, FlatpakOciIndexImage *image = list->images[k]; if (g_strcmp0 (image->architecture, oci_arch) == 0) add_image_to_appstream (http_session, - index_uri, + index_uri, certificates, appstream_root, icons_dfd, used_icons, r, image, cancellable); diff --git a/common/flatpak-utils-http-private.h b/common/flatpak-utils-http-private.h index 2c89ba40..a930ee79 100644 --- a/common/flatpak-utils-http-private.h +++ b/common/flatpak-utils-http-private.h @@ -39,6 +39,15 @@ void flatpak_http_session_free (FlatpakHttpSession* http_session); G_DEFINE_AUTOPTR_CLEANUP_FUNC(FlatpakHttpSession, flatpak_http_session_free) +typedef struct FlatpakCertificates FlatpakCertificates; + +FlatpakCertificates* flatpak_get_certificates_for_uri (const char *uri, + GError **error); +FlatpakCertificates * flatpak_certificates_copy (FlatpakCertificates *other); +void flatpak_certificates_free (FlatpakCertificates *certificates); + +G_DEFINE_AUTOPTR_CLEANUP_FUNC(FlatpakCertificates, flatpak_certificates_free) + typedef enum { FLATPAK_HTTP_FLAGS_NONE = 0, FLATPAK_HTTP_FLAGS_ACCEPT_OCI = 1 << 0, @@ -52,6 +61,7 @@ typedef void (*FlatpakLoadUriProgress) (guint64 downloaded_bytes, GBytes * flatpak_load_uri_full (FlatpakHttpSession *http_session, const char *uri, + FlatpakCertificates *certificates, FlatpakHTTPFlags flags, const char *auth, const char *token, @@ -73,6 +83,7 @@ GBytes * flatpak_load_uri (FlatpakHttpSession *http_session, GError **error); gboolean flatpak_download_http_uri (FlatpakHttpSession *http_session, const char *uri, + FlatpakCertificates *certificates, FlatpakHTTPFlags flags, GOutputStream *out, const char *token, @@ -82,6 +93,7 @@ gboolean flatpak_download_http_uri (FlatpakHttpSession *http_session, GError **error); gboolean flatpak_cache_http_uri (FlatpakHttpSession *http_session, const char *uri, + FlatpakCertificates *certificates, FlatpakHTTPFlags flags, int dest_dfd, const char *dest_subpath, diff --git a/common/flatpak-utils-http.c b/common/flatpak-utils-http.c index 27de9c7a..e8d5d724 100644 --- a/common/flatpak-utils-http.c +++ b/common/flatpak-utils-http.c @@ -73,6 +73,17 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC (SoupURI, soup_uri_free) G_DEFINE_QUARK (flatpak_http_error, flatpak_http_error) +/* Holds information about CA and client certificates found in + * system-wide and per-user certificate directories as documented + * in container-certs.d(5). + */ +struct FlatpakCertificates +{ + char *ca_cert_file; + char *client_cert_file; + char *client_key_file; +}; + /* Information about the cache status of a file. Encoded in an xattr on the cached file, or a file on the side if xattrs don't work. */ @@ -95,6 +106,7 @@ typedef struct FlatpakHTTPFlags flags; const char *auth; const char *token; + FlatpakCertificates *certificates; FlatpakLoadUriProgress progress; GCancellable *cancellable; gpointer user_data; @@ -231,6 +243,157 @@ check_http_status (guint status_code, return FALSE; } +FlatpakCertificates* +flatpak_get_certificates_for_uri (const char *uri, + GError **error) +{ + g_autoptr(FlatpakCertificates) certificates = NULL; + g_autoptr(GUri) parsed_uri = NULL; + g_autofree char *hostport = NULL; + const char *system_certs_d = NULL; + g_autofree char *certs_path_str = NULL; + g_auto(GStrv) certs_path = NULL; + + certificates = g_new0 (FlatpakCertificates, 1); + + parsed_uri = g_uri_parse (uri, G_URI_FLAGS_PARSE_RELAXED, error); + if (!parsed_uri) + return NULL; + + if (!g_uri_get_host (parsed_uri)) + return NULL; + + if (g_uri_get_port (parsed_uri) != -1) + hostport = g_strdup_printf ("%s:%d", g_uri_get_host (parsed_uri), g_uri_get_port (parsed_uri)); + else + hostport = g_strdup (g_uri_get_host (parsed_uri)); + + system_certs_d = g_getenv ("FLATPAK_SYSTEM_CERTS_D"); + if (system_certs_d == NULL || system_certs_d[0] == '\0') + system_certs_d = "/etc/containers/certs.d:/etc/docker/certs.d"; + + /* containers/image hardcodes ~/.config and doesn't honor XDG_CONFIG_HOME */ + certs_path_str = g_strconcat (g_get_user_config_dir(), "/containers/certs.d:", + system_certs_d, NULL); + certs_path = g_strsplit (certs_path_str, ":", -1); + + for (int i = 0; certs_path[i]; i++) + { + g_autoptr(GFile) certs_dir = g_file_new_for_path (certs_path[i]); + g_autoptr(GFile) host_dir = g_file_get_child (certs_dir, hostport); + g_autoptr(GFileEnumerator) enumerator; + g_autoptr(GError) local_error = NULL; + + enumerator = g_file_enumerate_children (host_dir, G_FILE_ATTRIBUTE_STANDARD_NAME, + G_FILE_QUERY_INFO_NONE, + NULL, &local_error); + if (enumerator == NULL) + { + /* This matches libpod - missing certificate directory or a permission + * error causes the directory to be skipped; any other error is fatal + */ + if (g_error_matches(local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND) || + g_error_matches(local_error, G_IO_ERROR, G_IO_ERROR_PERMISSION_DENIED)) + { + g_clear_error (&local_error); + continue; + } + else + { + g_propagate_error (error, g_steal_pointer (&local_error)); + return NULL; + } + } + + while (TRUE) + { + GFile *child; + g_autofree char *basename = NULL; + + if (!g_file_enumerator_iterate (enumerator, NULL, &child, NULL, error)) + return NULL; + + if (child == NULL) + break; + + basename = g_file_get_basename (child); + + /* In libpod, all CA certificates are added to the CA certificate + * database. We just use the first in readdir order. + */ + if (g_str_has_suffix (basename, ".crt") && certificates->ca_cert_file == NULL) + certificates->ca_cert_file = g_file_get_path (child); + + if (g_str_has_suffix (basename, ".cert")) + { + g_autofree char *nosuffix = g_strndup (basename, strlen (basename) - 5); + g_autofree char *key_basename = g_strconcat (nosuffix, ".key", NULL); + g_autoptr(GFile) key_file = g_file_get_child (host_dir, key_basename); + + if (!g_file_query_exists (key_file, NULL)) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "missing key %s for client cert %s. " + "Note that CA certificates should use the extension .crt", + g_file_peek_path (key_file), + g_file_peek_path (child)); + return NULL; + } + + /* In libpod, all client certificates are added, and then the go TLS + * code selects the best based on TLS negotation. We just pick the first + * in readdir order + * */ + if (certificates->client_cert_file == NULL) + { + certificates->client_cert_file = g_file_get_path (child); + certificates->client_key_file = g_file_get_path (key_file); + } + } + + if (g_str_has_suffix (basename, ".key")) + { + g_autofree char *nosuffix = g_strndup (basename, strlen (basename) - 4); + g_autofree char *cert_basename = g_strconcat (nosuffix, ".cert", NULL); + g_autoptr(GFile) cert_file = g_file_get_child (host_dir, cert_basename); + + if (!g_file_query_exists (cert_file, NULL)) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "missing client certificate %s for key %s", + g_file_peek_path (cert_file), + g_file_peek_path (child)); + return NULL; + } + } + } + } + + return g_steal_pointer (&certificates); +} + +FlatpakCertificates * +flatpak_certificates_copy (FlatpakCertificates *other) +{ + FlatpakCertificates *certificates = g_new0 (FlatpakCertificates, 1); + + certificates->ca_cert_file = g_strdup (other->ca_cert_file); + certificates->client_cert_file = g_strdup (other->client_cert_file); + certificates->client_key_file = g_strdup (other->client_key_file); + + return certificates; +} + +void +flatpak_certificates_free (FlatpakCertificates *certificates) +{ + g_clear_pointer (&certificates->ca_cert_file, g_free); + g_clear_pointer (&certificates->client_cert_file, g_free); + g_clear_pointer (&certificates->client_key_file, g_free); + + g_free (certificates); +} + #if defined(HAVE_CURL) /************************************************************************ @@ -477,6 +640,18 @@ flatpak_download_http_uri_once (FlatpakHttpSession *session, curl_easy_setopt (curl, CURLOPT_WRITEDATA, (void *)data); curl_easy_setopt (curl, CURLOPT_HEADERDATA, (void *)data); + if (data->certificates) + { + if (data->certificates->ca_cert_file) + curl_easy_setopt (curl, CURLOPT_CAINFO, data->certificates->ca_cert_file); + + if (data->certificates->client_cert_file) + { + curl_easy_setopt (curl, CURLOPT_SSLCERT, data->certificates->client_cert_file); + curl_easy_setopt (curl, CURLOPT_SSLKEY, data->certificates->client_key_file); + } + } + if (data->flags & FLATPAK_HTTP_FLAGS_HEAD) curl_easy_setopt (curl, CURLOPT_NOBODY, 1L); else @@ -576,6 +751,73 @@ flatpak_download_http_uri_once (FlatpakHttpSession *session, * Soup implementation * ***********************************************************************/ +/* + * The implementation of /etc/containers/certs.d for Soup is made tricky + * because the CA certificate database in Soup is global to the session, + * but we share a single sesssion between different hosts that might + * need different custom CA certs based on what's configured in certs.d. + * + * So what we do is make the FlatpakSoupSession multiplex multiple + * SoupSessions, depending on the certificates we use. The most common + * case, is of course, a single session with no custom certificates. + */ + +typedef struct +{ + char *user_agent; + GHashTable *soup_sessions; +} FlatpakSoupSession; + +static guint +certificates_hash(const FlatpakCertificates *certificates) +{ + guint hash = 0; + + if (certificates && certificates->ca_cert_file) + hash |= 13 * g_str_hash (certificates->ca_cert_file); + if (certificates && certificates->client_cert_file) + hash |= 17 * g_str_hash (certificates->client_cert_file); + if (certificates && certificates->client_key_file) + hash |= 23 * g_str_hash (certificates->client_key_file); + + return hash; +} + +static gboolean +certificates_equal (const FlatpakCertificates *a, + const FlatpakCertificates *b) +{ + if (a && b) + { + return (g_strcmp0(a->ca_cert_file, b->ca_cert_file) == 0 && + g_strcmp0(a->client_cert_file, b->client_cert_file) == 0 && + g_strcmp0(a->client_key_file, b->client_key_file) == 0); + } + else + return a == b; +} + +static void +certificates_free (FlatpakCertificates *certificates) +{ + if (certificates) + flatpak_certificates_free (certificates); +} + +static FlatpakSoupSession * +flatpak_create_soup_session (const char *user_agent) +{ + FlatpakSoupSession *session = g_new0 (FlatpakSoupSession, 1); + + session->user_agent = g_strdup (user_agent); + session->soup_sessions = g_hash_table_new_full ((GHashFunc)certificates_hash, + (GEqualFunc)certificates_equal, + (GDestroyNotify)certificates_free, + (GDestroyNotify)g_object_unref); + + return session; +} + static gboolean check_soup_transfer_error (SoupMessage *msg, GError **error) { @@ -768,18 +1010,114 @@ load_uri_callback (GObject *source_object, load_uri_read_cb, data); } +/* Inline class for providing a pre-configured client certificate; from + * libsoup/examples/get.c. By Colin Walters. + */ +struct _FlatpakTlsInteraction +{ + GTlsInteraction parent_instance; + + GTlsCertificate *cert; +}; + +struct _FlatpakTlsInteractionClass +{ + GTlsInteractionClass parent_class; +}; + +G_DECLARE_FINAL_TYPE (FlatpakTlsInteraction, + flatpak_tls_interaction, + FLATPAK, TLS_INTERACTION, + GTlsInteraction) + +G_DEFINE_TYPE (FlatpakTlsInteraction, + flatpak_tls_interaction, + G_TYPE_TLS_INTERACTION) + +static GTlsInteractionResult +flatpak_tls_interaction_request_certificate (GTlsInteraction *interaction, + GTlsConnection *connection, + GTlsCertificateRequestFlags flags, + GCancellable *cancellable, + GError **error) +{ + FlatpakTlsInteraction *self = FLATPAK_TLS_INTERACTION (interaction); + + g_tls_connection_set_certificate (connection, self->cert); + + return G_TLS_INTERACTION_HANDLED; +} + +static void +flatpak_tls_interaction_finalize (GObject *object) +{ + FlatpakTlsInteraction *self = FLATPAK_TLS_INTERACTION (object); + + g_clear_object (&self->cert); + + G_OBJECT_CLASS (flatpak_tls_interaction_parent_class)->finalize (object); +} + +static void +flatpak_tls_interaction_init (FlatpakTlsInteraction *interaction) +{ +} + +static void +flatpak_tls_interaction_class_init (FlatpakTlsInteractionClass *klass) +{ + GObjectClass *object_class = G_OBJECT_CLASS (klass); + GTlsInteractionClass *interaction_class = G_TLS_INTERACTION_CLASS (klass); + + object_class->finalize = flatpak_tls_interaction_finalize; + + interaction_class->request_certificate = flatpak_tls_interaction_request_certificate; +} + +static FlatpakTlsInteraction * +flatpak_tls_interaction_new (GTlsCertificate *cert) +{ + FlatpakTlsInteraction *self = g_object_new (flatpak_tls_interaction_get_type (), NULL); + + self->cert = g_object_ref (cert); + + return self; +} + static SoupSession * -flatpak_create_soup_session (const char *user_agent) +get_soup_session (FlatpakSoupSession *session, FlatpakCertificates *certificates, GError **error) { SoupSession *soup_session; const char *http_proxy; - soup_session = soup_session_new_with_options (SOUP_SESSION_USER_AGENT, user_agent, - SOUP_SESSION_SSL_USE_SYSTEM_CA_FILE, TRUE, + soup_session = g_hash_table_lookup (session->soup_sessions, certificates); + if (soup_session) + return soup_session; + + soup_session = soup_session_new_with_options (SOUP_SESSION_USER_AGENT, session->user_agent, SOUP_SESSION_USE_THREAD_CONTEXT, TRUE, SOUP_SESSION_TIMEOUT, FLATPAK_HTTP_TIMEOUT_SECS, SOUP_SESSION_IDLE_TIMEOUT, FLATPAK_HTTP_TIMEOUT_SECS, NULL); + if (certificates && certificates->ca_cert_file) + g_object_set (soup_session, SOUP_SESSION_SSL_CA_FILE, certificates->ca_cert_file, NULL); + else + g_object_set (soup_session, SOUP_SESSION_SSL_USE_SYSTEM_CA_FILE, TRUE, NULL); + + if (certificates && certificates->client_cert_file) + { + g_autoptr(GTlsCertificate) client_cert = NULL; + g_autoptr(GTlsInteraction) interaction = NULL; + + client_cert = g_tls_certificate_new_from_files (certificates->client_cert_file, + certificates->client_key_file, error); + if (!client_cert) + return NULL; + + interaction = G_TLS_INTERACTION (flatpak_tls_interaction_new (client_cert)); + g_object_set (soup_session, SOUP_SESSION_TLS_INTERACTION, interaction, NULL); + } + http_proxy = g_getenv ("http_proxy"); if (http_proxy) { @@ -793,6 +1131,10 @@ flatpak_create_soup_session (const char *user_agent) if (g_getenv ("OSTREE_DEBUG_HTTP")) soup_session_add_feature (soup_session, (SoupSessionFeature *) soup_logger_new (SOUP_LOGGER_LOG_BODY, 500)); + g_hash_table_replace (session->soup_sessions, + certificates ? flatpak_certificates_copy (certificates) : NULL, + soup_session); + return soup_session; } @@ -805,9 +1147,11 @@ flatpak_create_http_session (const char *user_agent) void flatpak_http_session_free (FlatpakHttpSession* http_session) { - SoupSession *soup_session = (SoupSession *)http_session; + FlatpakSoupSession *session = (FlatpakSoupSession *)http_session; - g_object_unref (soup_session); + g_hash_table_destroy (session->soup_sessions); + g_free (session->user_agent); + g_free (session); } static gboolean @@ -816,12 +1160,16 @@ flatpak_download_http_uri_once (FlatpakHttpSession *http_session, const char *uri, GError **error) { - SoupSession *soup_session = (SoupSession *)http_session; + SoupSession *soup_session; g_autoptr(SoupRequestHTTP) request = NULL; SoupMessage *m; g_info ("Loading %s using libsoup", uri); + soup_session = get_soup_session ((FlatpakSoupSession *)http_session, data->certificates, error); + if (!soup_session) + return FALSE; + request = soup_session_request_http (soup_session, (data->flags & FLATPAK_HTTP_FLAGS_HEAD) != 0 ? "HEAD" : "GET", uri, error); @@ -927,6 +1275,7 @@ flatpak_http_should_retry_request (const GError *error, GBytes * flatpak_load_uri_full (FlatpakHttpSession *http_session, const char *uri, + FlatpakCertificates *certificates, FlatpakHTTPFlags flags, const char *auth, const char *token, @@ -965,6 +1314,7 @@ flatpak_load_uri_full (FlatpakHttpSession *http_session, data.last_progress_time = g_get_monotonic_time (); data.cancellable = cancellable; data.flags = flags; + data.certificates = certificates; data.auth = auth; data.token = token; @@ -1018,7 +1368,7 @@ flatpak_load_uri (FlatpakHttpSession *http_session, GCancellable *cancellable, GError **error) { - return flatpak_load_uri_full (http_session, uri, flags, NULL, token, + return flatpak_load_uri_full (http_session, uri, NULL, flags, NULL, token, progress, user_data, NULL, out_content_type, NULL, cancellable, error); } @@ -1026,6 +1376,7 @@ flatpak_load_uri (FlatpakHttpSession *http_session, gboolean flatpak_download_http_uri (FlatpakHttpSession *http_session, const char *uri, + FlatpakCertificates *certificates, FlatpakHTTPFlags flags, GOutputStream *out, const char *token, @@ -1047,6 +1398,7 @@ flatpak_download_http_uri (FlatpakHttpSession *http_session, data.user_data = user_data; data.last_progress_time = g_get_monotonic_time (); data.cancellable = cancellable; + data.certificates = certificates; data.flags = flags; data.token = token; @@ -1384,6 +1736,7 @@ set_cache_http_data_from_headers (CacheHttpData *cache_data, gboolean flatpak_cache_http_uri (FlatpakHttpSession *http_session, const char *uri, + FlatpakCertificates *certificates, FlatpakHTTPFlags flags, int dest_dfd, const char *dest_subpath, @@ -1444,6 +1797,7 @@ flatpak_cache_http_uri (FlatpakHttpSession *http_session, data.last_progress_time = g_get_monotonic_time (); data.cancellable = cancellable; data.flags = flags; + data.certificates = certificates; data.cache_data = cache_data; diff --git a/doc/flatpak-remote.xml b/doc/flatpak-remote.xml index 798f5c39..47a5a42a 100644 --- a/doc/flatpak-remote.xml +++ b/doc/flatpak-remote.xml @@ -80,6 +80,12 @@ is a Flatpak extension that indicates that the remote is not an ostree repository, but is rather an URL to an index of OCI images that are stored within a container image registry. + + + For OCI remotes, client and CA certificates are read from + /etc/containers/certs.d and + ~/.config/containers/certs.d as documented in + containers-certs.d5. diff --git a/tests/httpcache.c b/tests/httpcache.c index a4550fb0..f6f9de64 100644 --- a/tests/httpcache.c +++ b/tests/httpcache.c @@ -32,7 +32,7 @@ main (int argc, char *argv[]) if (!flatpak_cache_http_uri (session, - url, + url, NULL, flags, AT_FDCWD, dest, NULL, NULL, NULL, &error)) -- 2.47.1 From cf555f02fcc1cf410fdad7607ff83a6764864a14 Mon Sep 17 00:00:00 2001 From: "Owen W. Taylor" Date: Tue, 17 Dec 2024 17:47:35 +0100 Subject: [PATCH 09/09] tests: Add tests for https OCI remotes --- tests/libtest.sh | 17 +++++-- tests/test-matrix/meson.build | 6 ++- tests/test-oci-registry.sh | 90 ++++++++++++++++++++++++++++++----- tests/test-wrapper.sh | 6 +++ tests/update-test-matrix | 2 +- 5 files changed, 101 insertions(+), 20 deletions(-) diff --git a/tests/libtest.sh b/tests/libtest.sh index d63810e1..7dad594f 100644 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -332,12 +332,16 @@ make_runtime () { } httpd () { - COMMAND=${1:-web-server.py} - DIR=${2:-repos} + if [ $# -eq 0 ] ; then + set web-server.py repos + fi + + COMMAND=$1 + shift rm -f httpd-pipe mkfifo httpd-pipe - PYTHONUNBUFFERED=1 $(dirname $0)/$COMMAND "$DIR" 3> httpd-pipe 2>&1 | tee -a httpd-log >&2 & + PYTHONUNBUFFERED=1 $(dirname $0)/$COMMAND "$@" 3> httpd-pipe 2>&1 | tee -a httpd-log >&2 & read < httpd-pipe } @@ -589,10 +593,15 @@ skip_without_libsystemd () { fi } +FLATPAK_SYSTEM_CERTS_D=$(pwd)/certs.d +export FLATPAK_SYSTEM_CERTS_D + sed s#@testdir@#${test_builddir}# ${test_srcdir}/session.conf.in > session.conf dbus-daemon --fork --config-file=session.conf --print-address=3 --print-pid=4 \ 3> dbus-session-bus-address 4> dbus-session-bus-pid -export DBUS_SESSION_BUS_ADDRESS="$(cat dbus-session-bus-address)" + +DBUS_SESSION_BUS_ADDRESS="$(cat dbus-session-bus-address)" +export DBUS_SESSION_BUS_ADDRESS DBUS_SESSION_BUS_PID="$(cat dbus-session-bus-pid)" if ! /bin/kill -0 "$DBUS_SESSION_BUS_PID"; then diff --git a/tests/test-matrix/meson.build b/tests/test-matrix/meson.build index 15176048..fd0b5034 100644 --- a/tests/test-matrix/meson.build +++ b/tests/test-matrix/meson.build @@ -17,8 +17,10 @@ wrapped_tests += {'name' : 'test-sideload@system.wrap', 'script' : 'test-sideloa wrapped_tests += {'name' : 'test-bundle@user.wrap', 'script' : 'test-bundle.sh'} wrapped_tests += {'name' : 'test-bundle@system.wrap', 'script' : 'test-bundle.sh'} wrapped_tests += {'name' : 'test-bundle@system-norevokefs.wrap', 'script' : 'test-bundle.sh'} -wrapped_tests += {'name' : 'test-oci-registry@user.wrap', 'script' : 'test-oci-registry.sh'} -wrapped_tests += {'name' : 'test-oci-registry@system.wrap', 'script' : 'test-oci-registry.sh'} +wrapped_tests += {'name' : 'test-oci-registry@user,http.wrap', 'script' : 'test-oci-registry.sh'} +wrapped_tests += {'name' : 'test-oci-registry@user,https.wrap', 'script' : 'test-oci-registry.sh'} +wrapped_tests += {'name' : 'test-oci-registry@system,http.wrap', 'script' : 'test-oci-registry.sh'} +wrapped_tests += {'name' : 'test-oci-registry@system,https.wrap', 'script' : 'test-oci-registry.sh'} wrapped_tests += {'name' : 'test-update-remote-configuration@newsummary.wrap', 'script' : 'test-update-remote-configuration.sh'} wrapped_tests += {'name' : 'test-update-remote-configuration@oldsummary.wrap', 'script' : 'test-update-remote-configuration.sh'} wrapped_tests += {'name' : 'test-update-portal@user.wrap', 'script' : 'test-update-portal.sh'} diff --git a/tests/test-oci-registry.sh b/tests/test-oci-registry.sh index 12036358..da234ded 100755 --- a/tests/test-oci-registry.sh +++ b/tests/test-oci-registry.sh @@ -27,9 +27,73 @@ echo "1..14" # Start the fake registry server -httpd oci-registry-server.py --dir=. +if [ x${USE_HTTPS} = xyes ] ; then + cat > openssl.config <&2 +${FLATPAK} remote-add ${U} oci-registry "oci+${scheme}://127.0.0.1:${port}" >&2 # Check that the images we expect are listed @@ -144,7 +208,7 @@ fi assert_has_file $base/oci/oci-registry.index.gz assert_has_file $base/oci/oci-registry.summary assert_has_dir $base/appstream/oci-registry -${FLATPAK} remote-modify ${U} --url=http://127.0.0.1:${port} oci-registry >&2 +${FLATPAK} remote-modify ${U} --url=${scheme}://127.0.0.1:${port} oci-registry >&2 assert_not_has_file $base/oci/oci-registry.index.gz assert_not_has_file $base/oci/oci-registry.summary assert_not_has_dir $base/appstream/oci-registry @@ -153,7 +217,7 @@ ok "change remote to non-OCI" # Change it back and refetch -${FLATPAK} remote-modify ${U} --url=oci+http://127.0.0.1:${port} oci-registry >&2 +${FLATPAK} remote-modify ${U} --url=oci+${scheme}://127.0.0.1:${port} oci-registry >&2 ${FLATPAK} update ${U} --appstream oci-registry >&2 # Delete the remote, check that everything was removed @@ -177,7 +241,7 @@ ok "delete remote" cat << EOF > runtime-repo.flatpakrepo [Flatpak Repo] Version=1 -Url=oci+http://localhost:${port} +Url=oci+${scheme}://localhost:${port} Title=The OCI Title EOF @@ -186,7 +250,7 @@ cat << EOF > org.test.Platform.flatpakref Title=Test Platform Name=org.test.Platform Branch=master -Url=oci+http://127.0.0.1:${port} +Url=oci+${scheme}://127.0.0.1:${port} IsRuntime=true RuntimeRepo=file://$(pwd)/runtime-repo.flatpakrepo EOF @@ -214,12 +278,12 @@ ok "prune origin remote" # Install from a (non-OCI) bundle, check that the repo-url is respected -${FLATPAK} build-bundle --runtime --repo-url "oci+http://127.0.0.1:${port}" $FL_GPGARGS repos/oci org.test.Platform.flatpak org.test.Platform >&2 +${FLATPAK} build-bundle --runtime --repo-url "oci+${scheme}://127.0.0.1:${port}" $FL_GPGARGS repos/oci org.test.Platform.flatpak org.test.Platform >&2 ${FLATPAK} ${U} install -y --bundle org.test.Platform.flatpak >&2 ${FLATPAK} remotes -d > remotes-list -assert_file_has_content remotes-list "^platform-origin.*[ ]oci+http://127\.0\.0\.1:${port}" +assert_file_has_content remotes-list "^platform-origin.*[ ]oci+${scheme}://127\.0\.0\.1:${port}" assert_has_file $base/oci/platform-origin.index.gz @@ -227,12 +291,12 @@ ok "install via bundle" # Install an app from a bundle -${FLATPAK} build-bundle --repo-url "oci+http://127.0.0.1:${port}" $FL_GPGARGS repos/oci org.test.Hello.flatpak org.test.Hello >&2 +${FLATPAK} build-bundle --repo-url "oci+${scheme}://127.0.0.1:${port}" $FL_GPGARGS repos/oci org.test.Hello.flatpak org.test.Hello >&2 ${FLATPAK} ${U} install -y --bundle org.test.Hello.flatpak >&2 ${FLATPAK} remotes -d > remotes-list -assert_file_has_content remotes-list "^hello-origin.*[ ]oci+http://127\.0\.0\.1:${port}" +assert_file_has_content remotes-list "^hello-origin.*[ ]oci+${scheme}://127\.0\.0\.1:${port}" assert_has_file $base/oci/hello-origin.index.gz @@ -241,12 +305,12 @@ ok "app install via bundle" # Install an updated app bundle with a different origin make_updated_app oci -${FLATPAK} build-bundle --repo-url "http://127.0.0.1:${port}" $FL_GPGARGS repos/oci org.test.Hello.flatpak org.test.Hello >&2 +${FLATPAK} build-bundle --repo-url "${scheme}://127.0.0.1:${port}" $FL_GPGARGS repos/oci org.test.Hello.flatpak org.test.Hello >&2 ${FLATPAK} ${U} install -y --bundle org.test.Hello.flatpak >&2 ${FLATPAK} remotes -d > remotes-list -assert_file_has_content remotes-list "^hello-origin.*[ ]http://127\.0\.0\.1:${port}" +assert_file_has_content remotes-list "^hello-origin.*[ ]${scheme}://127\.0\.0\.1:${port}" assert_not_has_file $base/oci/hello-origin.index.gz diff --git a/tests/test-wrapper.sh b/tests/test-wrapper.sh index be624256..2dacc1bc 100755 --- a/tests/test-wrapper.sh +++ b/tests/test-wrapper.sh @@ -30,6 +30,12 @@ for feature in $(echo $1 | sed "s/^.*@\(.*\).wrap/\1/" | tr "," "\n"); do annotations) export USE_OCI_ANNOTATIONS=yes ;; + https) + export USE_HTTPS=yes + ;; + http) + export USE_HTTPS=no + ;; *) echo unsupported test feature $feature exit 1 diff --git a/tests/update-test-matrix b/tests/update-test-matrix index 2aff6f00..3a51d0ba 100755 --- a/tests/update-test-matrix +++ b/tests/update-test-matrix @@ -23,7 +23,7 @@ TEST_MATRIX_SOURCE=( 'tests/test-extensions.sh' \ 'tests/test-bundle.sh{user+system+system-norevokefs}' \ 'tests/test-oci.sh' \ - 'tests/test-oci-registry.sh{user+system}' \ + 'tests/test-oci-registry.sh{{user+system},{http+https}}' \ 'tests/test-update-remote-configuration.sh{newsummary+oldsummary}' \ 'tests/test-override.sh' \ 'tests/test-update-portal.sh{user+system}' \ -- 2.47.1