diff --git a/pungi/__init__.py b/pungi/__init__.py index d4a48dca..8b672a72 100644 --- a/pungi/__init__.py +++ b/pungi/__init__.py @@ -16,7 +16,8 @@ def get_full_version(): proc = subprocess.Popen( ["git", "--git-dir=%s/.git" % location, "describe", "--tags"], stdout=subprocess.PIPE, - universal_newlines=True, + text=True, + errors="replace", ) output, _ = proc.communicate() return re.sub(r"-1.fc\d\d?", "", output.strip().replace("pungi-", "")) @@ -24,7 +25,7 @@ def get_full_version(): import subprocess proc = subprocess.Popen( - ["rpm", "-q", "pungi"], stdout=subprocess.PIPE, universal_newlines=True + ["rpm", "-q", "pungi"], stdout=subprocess.PIPE, text=True, errors="replace" ) (output, err) = proc.communicate() if not err: diff --git a/pungi/notifier.py b/pungi/notifier.py index bef2ae63..a42c6184 100644 --- a/pungi/notifier.py +++ b/pungi/notifier.py @@ -104,7 +104,8 @@ class PungiNotifier(object): workdir=workdir, return_stdout=False, show_cmd=True, - universal_newlines=True, + text=True, + errors="replace", logfile=logfile, ) if ret != 0: diff --git a/pungi/ostree/tree.py b/pungi/ostree/tree.py index 1ba138b3..5da6fd7a 100644 --- a/pungi/ostree/tree.py +++ b/pungi/ostree/tree.py @@ -64,7 +64,8 @@ class Tree(OSTree): show_cmd=True, stdout=True, logfile=log_file, - universal_newlines=True, + text=True, + errors="replace", ) finally: os.umask(oldumask) @@ -77,7 +78,8 @@ class Tree(OSTree): show_cmd=True, stdout=True, logfile=log_file, - universal_newlines=True, + text=True, + errors="replace", ) def _update_ref(self): diff --git a/pungi/phases/ostree_container.py b/pungi/phases/ostree_container.py index c0179620..b752d16c 100644 --- a/pungi/phases/ostree_container.py +++ b/pungi/phases/ostree_container.py @@ -139,7 +139,7 @@ class OSTreeContainerThread(WorkerThread): "--version=%s" % version, ] - _, runroot_script = shortcuts.run(cmd, universal_newlines=True) + _, runroot_script = shortcuts.run(cmd, text=True, errors="replace") default_packages = ["ostree", "rpm-ostree", "selinux-policy-targeted"] additional_packages = config.get("runroot_packages", []) diff --git a/pungi/util.py b/pungi/util.py index 1e83fac5..62614439 100644 --- a/pungi/util.py +++ b/pungi/util.py @@ -649,7 +649,11 @@ def run_unmount_cmd(cmd, max_retries=10, path=None, logger=None): """ for i in range(max_retries): proc = subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + errors="replace", ) out, err = proc.communicate() if proc.returncode == 0: @@ -671,7 +675,8 @@ def run_unmount_cmd(cmd, max_retries=10, path=None, logger=None): c, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - universal_newlines=True, + text=True, + errors="replace", ) out, _ = proc.communicate() logger.debug( @@ -876,7 +881,7 @@ def git_ls_remote(baseurl, ref, credential_helper=None): if credential_helper: cmd.extend(["-c", "credential.useHttpPath=true"]) cmd.extend(["-c", "credential.helper=%s" % credential_helper]) - return run(cmd + ["ls-remote", baseurl, ref], universal_newlines=True) + return run(cmd + ["ls-remote", baseurl, ref], text=True, errors="replace") def get_tz_offset(): diff --git a/pungi/wrappers/iso.py b/pungi/wrappers/iso.py index 5a2533f8..2ca7a8db 100644 --- a/pungi/wrappers/iso.py +++ b/pungi/wrappers/iso.py @@ -227,7 +227,7 @@ def get_checkisomd5_cmd(iso_path, just_print=False): def get_checkisomd5_data(iso_path, logger=None): cmd = get_checkisomd5_cmd(iso_path, just_print=True) - retcode, output = run(cmd, universal_newlines=True) + retcode, output = run(cmd, text=True, errors="replace") items = [line.strip().rsplit(":", 1) for line in output.splitlines()] items = dict([(k, v.strip()) for k, v in items]) md5 = items.get(iso_path, "") @@ -283,13 +283,13 @@ def get_manifest_cmd(iso_name, xorriso=False, output_file=None): def get_volume_id(path, xorriso=False): if xorriso: cmd = ["xorriso", "-indev", path] - retcode, output = run(cmd, universal_newlines=True) + retcode, output = run(cmd, text=True, errors="replace") for line in output.splitlines(): if line.startswith("Volume id"): return line.split("'")[1] else: cmd = ["isoinfo", "-d", "-i", path] - retcode, output = run(cmd, universal_newlines=True) + retcode, output = run(cmd, text=True, errors="replace") for line in output.splitlines(): line = line.strip() @@ -500,7 +500,7 @@ def mount(image, logger=None, use_guestmount=True): else: env = {} cmd = ["mount", "-o", "loop", image, mount_dir] - ret, out = run(cmd, env=env, can_fail=True, universal_newlines=True) + ret, out = run(cmd, env=env, can_fail=True, text=True, errors="replace") if ret != 0: # The mount command failed, something is wrong. # Log the output and raise an exception. diff --git a/pungi/wrappers/kojiwrapper.py b/pungi/wrappers/kojiwrapper.py index 8419cbed..1da7bbd7 100644 --- a/pungi/wrappers/kojiwrapper.py +++ b/pungi/wrappers/kojiwrapper.py @@ -295,7 +295,8 @@ class KojiWrapper(object): show_cmd=True, env=env, buffer_size=-1, - universal_newlines=True, + text=True, + errors="replace", ) # Look for first line that contains only a number. This is the ID of @@ -431,7 +432,7 @@ class KojiWrapper(object): while True: retcode, output = run( - cmd, can_fail=True, logfile=logfile, universal_newlines=True + cmd, can_fail=True, logfile=logfile, text=True, errors="replace" ) if retcode == 0 or not ( @@ -464,7 +465,8 @@ class KojiWrapper(object): logfile=log_file, env=env, buffer_size=-1, - universal_newlines=True, + text=True, + errors="replace", ) match = re.search(r"Created task: (\d+)", output) @@ -848,7 +850,8 @@ def get_buildroot_rpms(compose, task_id): # local retcode, output = run( "rpm -qa --qf='%{name}-%{version}-%{release}.%{arch}\n'", - universal_newlines=True, + text=True, + errors="replace", ) for i in output.splitlines(): if not i: diff --git a/pungi/wrappers/scm.py b/pungi/wrappers/scm.py index 9b4bc994..cfdd2907 100644 --- a/pungi/wrappers/scm.py +++ b/pungi/wrappers/scm.py @@ -56,7 +56,8 @@ class ScmBase(kobo.log.LoggingBase): workdir=cwd, can_fail=True, stdin_data="", - universal_newlines=True, + text=True, + errors="replace", ) if retcode != 0: self.log_error("Output was: %r" % output) diff --git a/pungi_utils/patch_iso.py b/pungi_utils/patch_iso.py index 469736dd..13dddf58 100644 --- a/pungi_utils/patch_iso.py +++ b/pungi_utils/patch_iso.py @@ -25,7 +25,7 @@ from pungi.wrappers import iso def sh(log, cmd, *args, **kwargs): log.info("Running: %s", " ".join(shlex.quote(x) for x in cmd)) - ret, out = shortcuts.run(cmd, *args, universal_newlines=True, **kwargs) + ret, out = shortcuts.run(cmd, *args, text=True, errors="replace", **kwargs) if out: log.debug("%s", out) return ret, out @@ -35,7 +35,8 @@ def get_lorax_dir(default="/usr/share/lorax"): try: _, out = shortcuts.run( ["python3", "-c" "import pylorax; print(pylorax.find_templates())"], - universal_newlines=True, + text=True, + errors="replace", ) return out.strip() except Exception: diff --git a/pungi_utils/unified_isos.py b/pungi_utils/unified_isos.py index 517921b7..a692e3b0 100644 --- a/pungi_utils/unified_isos.py +++ b/pungi_utils/unified_isos.py @@ -394,7 +394,8 @@ class UnifiedISO(object): iso.get_mkisofs_cmd( iso_path, [source_dir], volid=volid, exclude=["./lost+found"] ), - universal_newlines=True, + text=True, + errors="replace", ) # implant MD5 diff --git a/tests/test_iso_wrapper.py b/tests/test_iso_wrapper.py index 28ea976e..c7770413 100644 --- a/tests/test_iso_wrapper.py +++ b/tests/test_iso_wrapper.py @@ -63,7 +63,8 @@ class TestIsoUtils(unittest.TestCase): [ mock.call( ["/usr/bin/checkisomd5", "--md5sumonly", "dummy.iso"], - universal_newlines=True, + text=True, + errors="replace", ) ], ) @@ -79,7 +80,8 @@ class TestIsoUtils(unittest.TestCase): [ mock.call( ["/usr/bin/checkisomd5", "--md5sumonly", "dummy.iso"], - universal_newlines=True, + text=True, + errors="replace", ) ], ) diff --git a/tests/test_koji_wrapper.py b/tests/test_koji_wrapper.py index 9e0a5c17..e06d19c5 100644 --- a/tests/test_koji_wrapper.py +++ b/tests/test_koji_wrapper.py @@ -529,7 +529,8 @@ class RunrootKojiWrapperTest(KojiWrapperBaseTestCase): buffer_size=-1, logfile=None, show_cmd=True, - universal_newlines=True, + text=True, + errors="replace", ) ], ) @@ -558,7 +559,8 @@ class RunrootKojiWrapperTest(KojiWrapperBaseTestCase): buffer_size=-1, logfile=None, show_cmd=True, - universal_newlines=True, + text=True, + errors="replace", ) ], ) @@ -582,7 +584,8 @@ class RunrootKojiWrapperTest(KojiWrapperBaseTestCase): buffer_size=-1, logfile=None, show_cmd=True, - universal_newlines=True, + text=True, + errors="replace", ) ], ) @@ -615,7 +618,8 @@ class RunrootKojiWrapperTest(KojiWrapperBaseTestCase): buffer_size=-1, logfile=None, show_cmd=True, - universal_newlines=True, + text=True, + errors="replace", ) ], ) @@ -641,7 +645,8 @@ class RunBlockingCmdTest(KojiWrapperBaseTestCase): show_cmd=True, env={"FOO": "BAR", "PYTHONUNBUFFERED": "1"}, buffer_size=-1, - universal_newlines=True, + text=True, + errors="replace", ) ], ) @@ -671,7 +676,8 @@ class RunBlockingCmdTest(KojiWrapperBaseTestCase): "PYTHONUNBUFFERED": "1", }, buffer_size=-1, - universal_newlines=True, + text=True, + errors="replace", ) ], ) @@ -694,7 +700,8 @@ class RunBlockingCmdTest(KojiWrapperBaseTestCase): logfile="logfile", env={"FOO": "BAR", "PYTHONUNBUFFERED": "1"}, buffer_size=-1, - universal_newlines=True, + text=True, + errors="replace", ) ], ) @@ -717,7 +724,8 @@ class RunBlockingCmdTest(KojiWrapperBaseTestCase): logfile=None, env={"FOO": "BAR", "PYTHONUNBUFFERED": "1"}, buffer_size=-1, - universal_newlines=True, + text=True, + errors="replace", ) ], ) @@ -740,7 +748,8 @@ class RunBlockingCmdTest(KojiWrapperBaseTestCase): logfile=None, env={"FOO": "BAR", "PYTHONUNBUFFERED": "1"}, buffer_size=-1, - universal_newlines=True, + text=True, + errors="replace", ) ], ) @@ -765,13 +774,15 @@ class RunBlockingCmdTest(KojiWrapperBaseTestCase): logfile=None, env={"FOO": "BAR", "PYTHONUNBUFFERED": "1"}, buffer_size=-1, - universal_newlines=True, + text=True, + errors="replace", ), mock.call( ["koji", "--profile=custom-koji", "watch-task", "1234"], can_fail=True, logfile=None, - universal_newlines=True, + text=True, + errors="replace", ), ], ) @@ -795,13 +806,15 @@ class RunBlockingCmdTest(KojiWrapperBaseTestCase): logfile=None, env={"FOO": "BAR", "PYTHONUNBUFFERED": "1"}, buffer_size=-1, - universal_newlines=True, + text=True, + errors="replace", ), mock.call( ["koji", "--profile=custom-koji", "watch-task", "1234"], can_fail=True, logfile=None, - universal_newlines=True, + text=True, + errors="replace", ), ], ) @@ -826,25 +839,29 @@ class RunBlockingCmdTest(KojiWrapperBaseTestCase): logfile=None, env={"FOO": "BAR", "PYTHONUNBUFFERED": "1"}, buffer_size=-1, - universal_newlines=True, + text=True, + errors="replace", ), mock.call( ["koji", "--profile=custom-koji", "watch-task", "1234"], can_fail=True, logfile=None, - universal_newlines=True, + text=True, + errors="replace", ), mock.call( ["koji", "--profile=custom-koji", "watch-task", "1234"], can_fail=True, logfile=None, - universal_newlines=True, + text=True, + errors="replace", ), mock.call( ["koji", "--profile=custom-koji", "watch-task", "1234"], can_fail=True, logfile=None, - universal_newlines=True, + text=True, + errors="replace", ), ], ) @@ -870,19 +887,22 @@ class RunBlockingCmdTest(KojiWrapperBaseTestCase): logfile=None, env={"FOO": "BAR", "PYTHONUNBUFFERED": "1"}, buffer_size=-1, - universal_newlines=True, + text=True, + errors="replace", ), mock.call( ["koji", "--profile=custom-koji", "watch-task", "1234"], can_fail=True, logfile=None, - universal_newlines=True, + text=True, + errors="replace", ), mock.call( ["koji", "--profile=custom-koji", "watch-task", "1234"], can_fail=True, logfile=None, - universal_newlines=True, + text=True, + errors="replace", ), ], ) @@ -907,13 +927,15 @@ class RunBlockingCmdTest(KojiWrapperBaseTestCase): logfile=None, env={"FOO": "BAR", "PYTHONUNBUFFERED": "1"}, buffer_size=-1, - universal_newlines=True, + text=True, + errors="replace", ), mock.call( ["koji", "--profile=custom-koji", "watch-task", "1234"], can_fail=True, logfile=None, - universal_newlines=True, + text=True, + errors="replace", ), ], ) diff --git a/tests/test_notifier.py b/tests/test_notifier.py index e5584cc6..53ea4c94 100644 --- a/tests/test_notifier.py +++ b/tests/test_notifier.py @@ -70,7 +70,8 @@ class TestNotifier(unittest.TestCase): can_fail=True, return_stdout=False, workdir=None, - universal_newlines=True, + text=True, + errors="replace", show_cmd=True, logfile=self.logfile, ) diff --git a/tests/test_ostree_script.py b/tests/test_ostree_script.py index 6276f729..b2f37a37 100644 --- a/tests/test_ostree_script.py +++ b/tests/test_ostree_script.py @@ -67,7 +67,8 @@ class OstreeTreeScriptTest(helpers.PungiTestCase): logfile=self.topdir + "/logs/Atomic/create-ostree-repo.log", show_cmd=True, stdout=True, - universal_newlines=True, + text=True, + errors="replace", ) ] + extra_calls, @@ -136,7 +137,8 @@ class OstreeTreeScriptTest(helpers.PungiTestCase): logfile=self.topdir + "/logs/Atomic/ostree-summary.log", show_cmd=True, stdout=True, - universal_newlines=True, + text=True, + errors="replace", ) ], ) diff --git a/tests/test_patch_iso.py b/tests/test_patch_iso.py index 87e052d0..4ba10206 100644 --- a/tests/test_patch_iso.py +++ b/tests/test_patch_iso.py @@ -34,7 +34,7 @@ class TestSh(unittest.TestCase): patch_iso.sh(log, ["ls"], foo="bar") self.assertEqual( mock_run.call_args_list, - [mock.call(["ls"], foo="bar", universal_newlines=True)], + [mock.call(["ls"], foo="bar", text=True, errors="replace")], ) self.assertEqual(log.info.call_args_list, [mock.call("Running: %s", "ls")]) self.assertEqual(log.debug.call_args_list, [mock.call("%s", "ok")]) diff --git a/tests/test_unified_isos.py b/tests/test_unified_isos.py index bf095d9c..134306e1 100755 --- a/tests/test_unified_isos.py +++ b/tests/test_unified_isos.py @@ -610,7 +610,7 @@ class TestCreateiso(PungiTestCase): self.assertEqual( run.mock_calls, [ - mock.call(self.mkisofs_cmd, universal_newlines=True), + mock.call(self.mkisofs_cmd, text=True, errors="replace"), mock.call(iso.get_implantisomd5_cmd.return_value), mock.call(iso.get_manifest_cmd.return_value), ] diff --git a/tests/test_util.py b/tests/test_util.py index 9040c122..b7b99091 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -25,7 +25,8 @@ class TestGitRefResolver(unittest.TestCase): self.assertEqual(url, "https://git.example.com/repo.git?somedir#CAFEBABE") run.assert_called_once_with( ["git", "ls-remote", "https://git.example.com/repo.git", "HEAD"], - universal_newlines=True, + text=True, + errors="replace", ) @mock.patch("pungi.util.run") @@ -39,7 +40,8 @@ class TestGitRefResolver(unittest.TestCase): self.assertEqual(url, "https://git.example.com/repo.git?somedir#CAFEBABE") run.assert_called_once_with( GIT_WITH_CREDS + ["ls-remote", "https://git.example.com/repo.git", "HEAD"], - universal_newlines=True, + text=True, + errors="replace", ) @mock.patch("pungi.util.run") @@ -53,7 +55,8 @@ class TestGitRefResolver(unittest.TestCase): self.assertEqual(url, "https://git.example.com/repo.git?somedir#CAFEBABE") run.assert_called_once_with( ["git", "ls-remote", "https://git.example.com/repo.git", "refs/heads/f24"], - universal_newlines=True, + text=True, + errors="replace", ) def test_resolve_ref_with_commit_id(self): @@ -72,7 +75,8 @@ class TestGitRefResolver(unittest.TestCase): self.assertEqual(ref, "CAFEBABE") run.assert_called_once_with( ["git", "ls-remote", "https://git.example.com/repo.git", "master"], - universal_newlines=True, + text=True, + errors="replace", ) @mock.patch("pungi.util.run") @@ -84,7 +88,8 @@ class TestGitRefResolver(unittest.TestCase): self.assertEqual(ref, "CAFEBABE") run.assert_called_once_with( ["git", "ls-remote", "https://git.example.com/repo.git", "HEAD"], - universal_newlines=True, + text=True, + errors="replace", ) @mock.patch("pungi.util.run") @@ -110,7 +115,8 @@ class TestGitRefResolver(unittest.TestCase): run.assert_called_once_with( ["git", "ls-remote", "https://git.example.com/repo.git", "HEAD"], - universal_newlines=True, + text=True, + errors="replace", ) @mock.patch("pungi.util.run") @@ -121,7 +127,8 @@ class TestGitRefResolver(unittest.TestCase): run.assert_called_once_with( ["git", "ls-remote", "https://git.example.com/repo.git", "HEAD"], - universal_newlines=True, + text=True, + errors="replace", ) self.assertEqual(url, "https://git.example.com/repo.git?#CAFEBABE") @@ -133,7 +140,8 @@ class TestGitRefResolver(unittest.TestCase): run.assert_called_once_with( ["git", "ls-remote", "https://git.example.com/repo.git", "HEAD"], - universal_newlines=True, + text=True, + errors="replace", ) self.assertEqual(url, "git+https://git.example.com/repo.git#CAFEBABE") @@ -153,7 +161,8 @@ class TestGitRefResolver(unittest.TestCase): "https://git.example.com/repo.git", "refs/heads/my-branch", ], - universal_newlines=True, + text=True, + errors="replace", ) self.assertIn("ref does not exist in remote repo", str(ctx.exception)) @@ -171,7 +180,8 @@ class TestGitRefResolver(unittest.TestCase): [ mock.call( ["git", "ls-remote", "https://git.example.com/repo.git", "HEAD"], - universal_newlines=True, + text=True, + errors="replace", ) ] * 2, @@ -600,7 +610,8 @@ class TestUnmountCmd(unittest.TestCase): cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, - universal_newlines=True, + text=True, + errors="replace", ) ], ) @@ -621,7 +632,8 @@ class TestUnmountCmd(unittest.TestCase): cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, - universal_newlines=True, + text=True, + errors="replace", ) ], ) @@ -643,7 +655,8 @@ class TestUnmountCmd(unittest.TestCase): cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, - universal_newlines=True, + text=True, + errors="replace", ) ] * 3, @@ -668,7 +681,8 @@ class TestUnmountCmd(unittest.TestCase): cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, - universal_newlines=True, + text=True, + errors="replace", ) ] * 3, @@ -707,37 +721,43 @@ class TestUnmountCmd(unittest.TestCase): cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, - universal_newlines=True, + text=True, + errors="replace", ), mock.call( cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, - universal_newlines=True, + text=True, + errors="replace", ), mock.call( cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, - universal_newlines=True, + text=True, + errors="replace", ), mock.call( ["ls", "-lA", "/path"], stderr=subprocess.STDOUT, stdout=subprocess.PIPE, - universal_newlines=True, + text=True, + errors="replace", ), mock.call( ["fuser", "-vm", "/path"], stderr=subprocess.STDOUT, stdout=subprocess.PIPE, - universal_newlines=True, + text=True, + errors="replace", ), mock.call( ["lsof", "+D", "/path"], stderr=subprocess.STDOUT, stdout=subprocess.PIPE, - universal_newlines=True, + text=True, + errors="replace", ), ] self.assertEqual(mockPopen.call_args_list, expected)