From 41113fc34e21ec22c3d042345d391ee181f5fb2d Mon Sep 17 00:00:00 2001 From: CentOS Sources Date: Tue, 5 Apr 2022 06:21:29 -0400 Subject: [PATCH] import createrepo_c-0.17.7-4.el9_0 --- ...-main-metadata-together-RhBug2062299.patch | 2640 +++++++++++++++++ SPECS/createrepo_c.spec | 9 +- 2 files changed, 2648 insertions(+), 1 deletion(-) create mode 100644 SOURCES/0002-Revert-added-API-for-parsing-main-metadata-together-RhBug2062299.patch diff --git a/SOURCES/0002-Revert-added-API-for-parsing-main-metadata-together-RhBug2062299.patch b/SOURCES/0002-Revert-added-API-for-parsing-main-metadata-together-RhBug2062299.patch new file mode 100644 index 0000000..35858b5 --- /dev/null +++ b/SOURCES/0002-Revert-added-API-for-parsing-main-metadata-together-RhBug2062299.patch @@ -0,0 +1,2640 @@ +From 9f0762b90cc1d85900064c24fad8f7d5a8b73afe Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= +Date: Wed, 9 Mar 2022 15:23:29 +0100 +Subject: [PATCH 1/5] Revert "Add xml_parse_main_metadata_together to python + parser examples" + +This reverts commit 0633e31e491179f0153ef8267874812b4691957b. +--- + examples/python/repodata_parsing.py | 46 ----------------------------- + 1 file changed, 46 deletions(-) + +diff --git a/examples/python/repodata_parsing.py b/examples/python/repodata_parsing.py +index 4519518..ea517a1 100755 +--- a/examples/python/repodata_parsing.py ++++ b/examples/python/repodata_parsing.py +@@ -211,48 +211,6 @@ def second_method(): + for pkg in packages.values(): + print_package_info(pkg) + +-def third_method(): +- """Parsing main metadata types (primary, filelists, other) at the same time. +- This approach significantly reduces memory footprint because we don't need +- to keep all the packages in memory, user can handle them one by one. +- +- The API reflects xml_parse_primary/filelists/other except that it handles +- all of them at the same time. +- +- """ +- def warningcb(warning_type, message): +- print("PARSER WARNING: %s" % message) +- return True +- +- repomd = cr.Repomd() +- cr.xml_parse_repomd(os.path.join(REPO_PATH, "repodata/repomd.xml"), repomd, warningcb) +- +- primary_xml_path = None +- filelists_xml_path = None +- other_xml_path = None +- for record in repomd.records: +- if record.type == "primary": +- primary_xml_path = os.path.join(REPO_PATH, record.location_href) +- elif record.type == "filelists": +- filelists_xml_path = os.path.join(REPO_PATH, record.location_href) +- elif record.type == "other": +- other_xml_path = os.path.join(REPO_PATH, record.location_href) +- +- # +- # Main XML metadata parsing (primary, filelists, other) +- # +- +- def pkgcb(pkg): +- # Called when whole package entry from all 3 metadata xml files is parsed +- print_package_info(pkg) +- +- cr.xml_parse_main_metadata_together(primary_xml_path, +- filelists_xml_path, +- other_xml_path, +- None, +- pkgcb, +- warningcb, +- False) + + if __name__ == "__main__": + print('"All in one shot" method:') +@@ -263,7 +221,3 @@ if __name__ == "__main__": + print("Callback based method:") + second_method() + +- print() +- +- print("Streaming callback based method:") +- third_method() +-- +2.35.1 + + +From 01493aea9bc53e46a0e1020656e60ecb6c543bca Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= +Date: Wed, 9 Mar 2022 15:23:40 +0100 +Subject: [PATCH 2/5] Revert "Add python tests for parsing metadata together" + +This reverts commit 0c050128f7e24222011cdfcc67ba58da744affa9. +--- + tests/python/tests/test_xml_parser.py | 206 -------------------------- + 1 file changed, 206 deletions(-) + +diff --git a/tests/python/tests/test_xml_parser.py b/tests/python/tests/test_xml_parser.py +index ed13cac..ddc089f 100644 +--- a/tests/python/tests/test_xml_parser.py ++++ b/tests/python/tests/test_xml_parser.py +@@ -950,209 +950,3 @@ class TestCaseXmlParserRepomd(unittest.TestCase): + self.assertEqual(repomd.content_tags, []) + self.assertEqual(len(repomd.records), 3) + +-class TestCaseXmlParserMainMetadataTogether(unittest.TestCase): +- def test_xml_parser_main_metadata_together_repo01(self): +- userdata = { +- "pkgs": [], +- "pkgcb_calls": 0, +- "warnings": [] +- } +- +- def newpkgcb(pkgId, name, arch): +- pkg = cr.Package() +- userdata["pkgs"].append(pkg) +- return pkg +- +- def pkgcb(pkg): +- userdata["pkgcb_calls"] += 1 +- +- def warningcb(warn_type, msg): +- userdata["warnings"].append((warn_type, msg)) +- +- cr.xml_parse_main_metadata_together(primary=REPO_01_PRIXML, filelists=REPO_01_FILXML, other=REPO_01_OTHXML, +- newpkgcb=newpkgcb, pkgcb=pkgcb, warningcb=warningcb) +- +- self.assertEqual([pkg.name for pkg in userdata["pkgs"]], ['super_kernel']) +- self.assertEqual(userdata["pkgcb_calls"], 1) +- self.assertEqual(userdata["warnings"], []) +- +- pkg = userdata["pkgs"][0] +- self.assertEqual(pkg.pkgId, "152824bff2aa6d54f429d43e87a3ff3a0286505c6d93ec87692b5e3a9e3b97bf") +- self.assertEqual(pkg.name, "super_kernel") +- self.assertEqual(pkg.arch, "x86_64") +- self.assertEqual(pkg.version, "6.0.1") +- self.assertEqual(pkg.epoch, "0") +- self.assertEqual(pkg.release, "2") +- self.assertEqual(pkg.summary, "Test package") +- self.assertEqual(pkg.description, "This package has provides, requires, obsoletes, conflicts options.") +- self.assertEqual(pkg.url, "http://so_super_kernel.com/it_is_awesome/yep_it_really_is") +- self.assertEqual(pkg.time_file, 1334667003) +- self.assertEqual(pkg.time_build, 1334667003) +- self.assertEqual(pkg.rpm_license, "LGPLv2") +- self.assertEqual(pkg.rpm_vendor, None) +- self.assertEqual(pkg.rpm_group, "Applications/System") +- self.assertEqual(pkg.rpm_buildhost, "localhost.localdomain") +- self.assertEqual(pkg.rpm_sourcerpm, "super_kernel-6.0.1-2.src.rpm") +- self.assertEqual(pkg.rpm_header_start, 280) +- self.assertEqual(pkg.rpm_header_end, 2637) +- self.assertEqual(pkg.rpm_packager, None) +- self.assertEqual(pkg.size_package, 2845) +- self.assertEqual(pkg.size_installed, 0) +- self.assertEqual(pkg.size_archive, 404) +- self.assertEqual(pkg.location_href, "super_kernel-6.0.1-2.x86_64.rpm") +- self.assertEqual(pkg.location_base, None) +- self.assertEqual(pkg.checksum_type, "sha256") +- self.assertEqual(pkg.requires, +- [('bzip2', 'GE', '0', '1.0.0', None, True), +- ('expat', None, None, None, None, True), +- ('glib', 'GE', '0', '2.26.0', None, False), +- ('zlib', None, None, None, None, False)]) +- self.assertEqual(pkg.provides, +- [('not_so_super_kernel', 'LT', '0', '5.8.0', None, False), +- ('super_kernel', 'EQ', '0', '6.0.0', None, False), +- ('super_kernel', 'EQ', '0', '6.0.1', '2', False), +- ('super_kernel(x86-64)', 'EQ', '0', '6.0.1', '2', False)]) +- self.assertEqual(pkg.conflicts, +- [('kernel', None, None, None, None, False), +- ('super_kernel', 'EQ', '0', '5.0.0', None, False), +- ('super_kernel', 'LT', '0', '4.0.0', None, False)]) +- self.assertEqual(pkg.obsoletes, +- [('kernel', None, None, None, None, False), +- ('super_kernel', 'EQ', '0', '5.9.0', None, False)]) +- self.assertEqual(pkg.files, +- [(None, '/usr/bin/', 'super_kernel'), +- (None, '/usr/share/man/', 'super_kernel.8.gz')]) +- self.assertEqual(pkg.changelogs, +- [('Tomas Mlcoch - 6.0.1-1', +- 1334664000, +- '- First release'), +- ('Tomas Mlcoch - 6.0.1-2', +- 1334664001, +- '- Second release')]) +- +- def test_xml_parser_main_metadata_together_repo02(self): +- +- userdata = { +- "pkgs": [], +- "pkgcb_calls": 0, +- "warnings": [] +- } +- +- def newpkgcb(pkgId, name, arch): +- pkg = cr.Package() +- userdata["pkgs"].append(pkg) +- return pkg +- +- def pkgcb(pkg): +- userdata["pkgcb_calls"] += 1 +- +- def warningcb(warn_type, msg): +- userdata["warnings"].append((warn_type, msg)) +- +- cr.xml_parse_main_metadata_together(REPO_02_PRIXML, REPO_02_FILXML, REPO_02_OTHXML, newpkgcb, pkgcb, warningcb, +- allow_out_of_order=True) +- +- self.assertEqual([pkg.name for pkg in userdata["pkgs"]], +- ['fake_bash', 'super_kernel']) +- self.assertEqual(userdata["pkgcb_calls"], 2) +- self.assertEqual(userdata["warnings"], []) +- +- def test_xml_parser_main_metadata_together_repo02_only_pkgcb(self): +- +- pkgs = [] +- +- def pkgcb(pkg): +- pkgs.append(pkg) +- +- cr.xml_parse_main_metadata_together(REPO_02_PRIXML, REPO_02_FILXML, REPO_02_OTHXML, +- None, pkgcb, None) +- +- self.assertEqual([pkg.name for pkg in pkgs], +- ['fake_bash', 'super_kernel']) +- +- def test_xml_parser_main_metadata_together_repo02_no_cbs(self): +- self.assertRaises(ValueError, +- cr.xml_parse_main_metadata_together, +- REPO_02_PRIXML, REPO_02_FILXML, REPO_02_OTHXML, +- None, None, None) +- +- def test_xml_parser_main_metadata_together_warnings(self): +- userdata = { +- "pkgs": [], +- "warnings": [] +- } +- +- def newpkgcb(pkgId, name, arch): +- pkg = cr.Package() +- userdata["pkgs"].append(pkg) +- return pkg +- +- def warningcb(warn_type, msg): +- userdata["warnings"].append((warn_type, msg)) +- +- cr.xml_parse_main_metadata_together(PRIMARY_MULTI_WARN_00_PATH, +- FILELISTS_MULTI_WARN_00_PATH, +- OTHER_MULTI_WARN_00_PATH, +- newpkgcb, None, warningcb) +- +- self.assertEqual([pkg.name for pkg in userdata["pkgs"]], +- ['fake_bash', 'super_kernel']) +- self.assertEqual(userdata["warnings"], +- [(0, 'Unknown element "fooelement"'), +- (1, 'Missing attribute "type" of a package element'), +- (0, 'Unknown element "foo"'), +- (3, 'Conversion of "foobar" to integer failed'), +- (0, 'Unknown element "bar"'), +- (1, 'Missing attribute "arch" of a package element'), +- (2, 'Unknown file type "xxx"'), +- (0, 'Unknown element "bar"'), +- (1, 'Missing attribute "name" of a package element'), +- (0, 'Unknown element "bar"'), +- (3, 'Conversion of "xxx" to integer failed')]) +- +- +- def test_xml_parser_main_metadata_together_error(self): +- userdata = { "pkgs": [] } +- +- def newpkgcb(pkgId, name, arch): +- pkg = cr.Package() +- userdata["pkgs"].append(pkg) +- return pkg +- +- self.assertRaises(cr.CreaterepoCError, cr.xml_parse_main_metadata_together, +- PRIMARY_ERROR_00_PATH, FILELISTS_ERROR_00_PATH, +- OTHER_ERROR_00_PATH, newpkgcb, None, None) +- +- # unlike parsing just primary, filelists or other separately when parsing together primary is parsed first fully +- # before newpkgcb is called so the error is detected before any user package is created +- self.assertEqual(len(userdata["pkgs"]), 0) +- +- def test_xml_parser_main_metadata_together_newpkgcb_abort(self): +- def newpkgcb(pkgId, name, arch): +- raise Error("Foo error") +- self.assertRaises(cr.CreaterepoCError, +- cr.xml_parse_main_metadata_together, +- REPO_02_PRIXML, REPO_02_FILXML, REPO_02_OTHXML, +- newpkgcb, None, None) +- +- def test_xml_parser_main_metadata_together_pkgcb_abort(self): +- def newpkgcb(pkgId, name, arch): +- return cr.Package() +- def pkgcb(pkg): +- raise Error("Foo error") +- self.assertRaises(cr.CreaterepoCError, +- cr.xml_parse_main_metadata_together, +- REPO_02_PRIXML, REPO_02_FILXML, REPO_02_OTHXML, +- newpkgcb, pkgcb, None) +- +- def test_xml_parser_main_metadata_together_warningcb_abort(self): +- def newpkgcb(pkgId, name, arch): +- return cr.Package() +- def warningcb(type, msg): +- raise Error("Foo error") +- self.assertRaises(cr.CreaterepoCError, +- cr.xml_parse_main_metadata_together, +- PRIMARY_MULTI_WARN_00_PATH, +- FILELISTS_MULTI_WARN_00_PATH, +- OTHER_MULTI_WARN_00_PATH, +- newpkgcb, None, warningcb) +-- +2.35.1 + + +From 9210f5d725a71f130bf5fb880f386c2e8e5f66fa Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= +Date: Wed, 9 Mar 2022 15:23:47 +0100 +Subject: [PATCH 3/5] Revert "Wrap c api to python for parsing metadata + together" + +This reverts commit 670006b9f37d8298fe2ec1d6cfc5cdf175a3128b. +--- + src/python/createrepo_cmodule.c | 2 - + src/python/xml_parser-py.c | 156 +++++++------------------------- + src/python/xml_parser-py.h | 8 -- + 3 files changed, 32 insertions(+), 134 deletions(-) + +diff --git a/src/python/createrepo_cmodule.c b/src/python/createrepo_cmodule.c +index c0b9200..6937246 100644 +--- a/src/python/createrepo_cmodule.c ++++ b/src/python/createrepo_cmodule.c +@@ -79,8 +79,6 @@ static struct PyMethodDef createrepo_c_methods[] = { + METH_VARARGS, xml_parse_repomd__doc__}, + {"xml_parse_updateinfo", (PyCFunction)py_xml_parse_updateinfo, + METH_VARARGS, xml_parse_updateinfo__doc__}, +- {"xml_parse_main_metadata_together",(PyCFunctionWithKeywords)py_xml_parse_main_metadata_together, +- METH_VARARGS | METH_KEYWORDS, xml_parse_main_metadata_together__doc__}, + {"checksum_name_str", (PyCFunction)py_checksum_name_str, + METH_VARARGS, checksum_name_str__doc__}, + {"checksum_type", (PyCFunction)py_checksum_type, +diff --git a/src/python/xml_parser-py.c b/src/python/xml_parser-py.c +index a1f92f8..0879128 100644 +--- a/src/python/xml_parser-py.c ++++ b/src/python/xml_parser-py.c +@@ -34,7 +34,7 @@ typedef struct { + PyObject *py_newpkgcb; + PyObject *py_pkgcb; + PyObject *py_warningcb; +- PyObject *py_pkgs; /*!< Current processed package */ ++ PyObject *py_pkg; /*!< Current processed package */ + } CbData; + + static int +@@ -48,6 +48,12 @@ c_newpkgcb(cr_Package **pkg, + PyObject *arglist, *result; + CbData *data = cbdata; + ++ if (data->py_pkg) { ++ // Decref ref count on previous processed package ++ Py_DECREF(data->py_pkg); ++ data->py_pkg = NULL; ++ } ++ + arglist = Py_BuildValue("(sss)", pkgId, name, arch); + result = PyObject_CallObject(data->py_newpkgcb, arglist); + Py_DECREF(arglist); +@@ -67,18 +73,13 @@ c_newpkgcb(cr_Package **pkg, + + if (result == Py_None) { + *pkg = NULL; ++ data->py_pkg = NULL; ++ Py_DECREF(result); + } else { + *pkg = Package_FromPyObject(result); +- if (data->py_pkgcb != Py_None) { +- // Store reference to the python pkg (result) only if we will need it later +- PyObject *keyFromPtr = PyLong_FromVoidPtr(*pkg); +- PyDict_SetItem(data->py_pkgs, keyFromPtr, result); +- Py_DECREF(keyFromPtr); +- } ++ data->py_pkg = result; // Store reference to current package + } + +- Py_DECREF(result); +- + return CR_CB_RET_OK; + } + +@@ -90,23 +91,16 @@ c_pkgcb(cr_Package *pkg, + PyObject *arglist, *result, *py_pkg; + CbData *data = cbdata; + +- PyObject *keyFromPtr = PyLong_FromVoidPtr(pkg); +- py_pkg = PyDict_GetItem(data->py_pkgs, keyFromPtr); +- if (py_pkg) { +- arglist = Py_BuildValue("(O)", py_pkg); +- result = PyObject_CallObject(data->py_pkgcb, arglist); +- PyDict_DelItem(data->py_pkgs, keyFromPtr); +- } else { +- // The package was not provided by user in c_newpkgcb, +- // create new python package object +- PyObject *new_py_pkg = Object_FromPackage(pkg, 1); +- arglist = Py_BuildValue("(O)", new_py_pkg); +- result = PyObject_CallObject(data->py_pkgcb, arglist); +- Py_DECREF(new_py_pkg); +- } ++ if (data->py_pkg) ++ py_pkg = data->py_pkg; ++ else ++ py_pkg = Object_FromPackage(pkg, 1); + ++ arglist = Py_BuildValue("(O)", py_pkg); ++ result = PyObject_CallObject(data->py_pkgcb, arglist); + Py_DECREF(arglist); +- Py_DECREF(keyFromPtr); ++ Py_DECREF(py_pkg); ++ data->py_pkg = NULL; + + if (result == NULL) { + // Exception raised +@@ -197,7 +191,7 @@ py_xml_parse_primary(G_GNUC_UNUSED PyObject *self, PyObject *args) + cbdata.py_newpkgcb = py_newpkgcb; + cbdata.py_pkgcb = py_pkgcb; + cbdata.py_warningcb = py_warningcb; +- cbdata.py_pkgs = PyDict_New(); ++ cbdata.py_pkg = NULL; + + cr_xml_parse_primary(filename, + ptr_c_newpkgcb, +@@ -212,8 +206,7 @@ py_xml_parse_primary(G_GNUC_UNUSED PyObject *self, PyObject *args) + Py_XDECREF(py_newpkgcb); + Py_XDECREF(py_pkgcb); + Py_XDECREF(py_warningcb); +- +- Py_XDECREF(cbdata.py_pkgs); ++ Py_XDECREF(cbdata.py_pkg); + + if (tmp_err) { + nice_exception(&tmp_err, NULL); +@@ -279,7 +272,7 @@ py_xml_parse_primary_snippet(G_GNUC_UNUSED PyObject *self, PyObject *args) + cbdata.py_newpkgcb = py_newpkgcb; + cbdata.py_pkgcb = py_pkgcb; + cbdata.py_warningcb = py_warningcb; +- cbdata.py_pkgs = PyDict_New(); ++ cbdata.py_pkg = NULL; + + cr_xml_parse_primary_snippet(target, ptr_c_newpkgcb, &cbdata, ptr_c_pkgcb, &cbdata, + ptr_c_warningcb, &cbdata, do_files, &tmp_err); +@@ -287,7 +280,7 @@ py_xml_parse_primary_snippet(G_GNUC_UNUSED PyObject *self, PyObject *args) + Py_XDECREF(py_newpkgcb); + Py_XDECREF(py_pkgcb); + Py_XDECREF(py_warningcb); +- Py_XDECREF(cbdata.py_pkgs); ++ Py_XDECREF(cbdata.py_pkg); + + if (tmp_err) { + nice_exception(&tmp_err, NULL); +@@ -351,7 +344,7 @@ py_xml_parse_filelists(G_GNUC_UNUSED PyObject *self, PyObject *args) + cbdata.py_newpkgcb = py_newpkgcb; + cbdata.py_pkgcb = py_pkgcb; + cbdata.py_warningcb = py_warningcb; +- cbdata.py_pkgs = PyDict_New(); ++ cbdata.py_pkg = NULL; + + cr_xml_parse_filelists(filename, + ptr_c_newpkgcb, +@@ -365,7 +358,7 @@ py_xml_parse_filelists(G_GNUC_UNUSED PyObject *self, PyObject *args) + Py_XDECREF(py_newpkgcb); + Py_XDECREF(py_pkgcb); + Py_XDECREF(py_warningcb); +- Py_XDECREF(cbdata.py_pkgs); ++ Py_XDECREF(cbdata.py_pkg); + + if (tmp_err) { + nice_exception(&tmp_err, NULL); +@@ -429,7 +422,7 @@ py_xml_parse_filelists_snippet(G_GNUC_UNUSED PyObject *self, PyObject *args) + cbdata.py_newpkgcb = py_newpkgcb; + cbdata.py_pkgcb = py_pkgcb; + cbdata.py_warningcb = py_warningcb; +- cbdata.py_pkgs = PyDict_New(); ++ cbdata.py_pkg = NULL; + + cr_xml_parse_filelists_snippet(target, ptr_c_newpkgcb, &cbdata, ptr_c_pkgcb, + &cbdata, ptr_c_warningcb, &cbdata, &tmp_err); +@@ -437,7 +430,7 @@ py_xml_parse_filelists_snippet(G_GNUC_UNUSED PyObject *self, PyObject *args) + Py_XDECREF(py_newpkgcb); + Py_XDECREF(py_pkgcb); + Py_XDECREF(py_warningcb); +- Py_XDECREF(cbdata.py_pkgs); ++ Py_XDECREF(cbdata.py_pkg); + + if (tmp_err) { + nice_exception(&tmp_err, NULL); +@@ -501,7 +494,7 @@ py_xml_parse_other(G_GNUC_UNUSED PyObject *self, PyObject *args) + cbdata.py_newpkgcb = py_newpkgcb; + cbdata.py_pkgcb = py_pkgcb; + cbdata.py_warningcb = py_warningcb; +- cbdata.py_pkgs = PyDict_New(); ++ cbdata.py_pkg = NULL; + + cr_xml_parse_other(filename, + ptr_c_newpkgcb, +@@ -515,7 +508,7 @@ py_xml_parse_other(G_GNUC_UNUSED PyObject *self, PyObject *args) + Py_XDECREF(py_newpkgcb); + Py_XDECREF(py_pkgcb); + Py_XDECREF(py_warningcb); +- Py_XDECREF(cbdata.py_pkgs); ++ Py_XDECREF(cbdata.py_pkg); + + if (tmp_err) { + nice_exception(&tmp_err, NULL); +@@ -579,7 +572,7 @@ py_xml_parse_other_snippet(G_GNUC_UNUSED PyObject *self, PyObject *args) + cbdata.py_newpkgcb = py_newpkgcb; + cbdata.py_pkgcb = py_pkgcb; + cbdata.py_warningcb = py_warningcb; +- cbdata.py_pkgs = PyDict_New(); ++ cbdata.py_pkg = NULL; + + cr_xml_parse_other_snippet(target, ptr_c_newpkgcb, &cbdata, ptr_c_pkgcb, &cbdata, + ptr_c_warningcb, &cbdata, &tmp_err); +@@ -587,7 +580,7 @@ py_xml_parse_other_snippet(G_GNUC_UNUSED PyObject *self, PyObject *args) + Py_XDECREF(py_newpkgcb); + Py_XDECREF(py_pkgcb); + Py_XDECREF(py_warningcb); +- Py_XDECREF(cbdata.py_pkgs); ++ Py_XDECREF(cbdata.py_pkg); + + if (tmp_err) { + nice_exception(&tmp_err, NULL); +@@ -630,7 +623,7 @@ py_xml_parse_repomd(G_GNUC_UNUSED PyObject *self, PyObject *args) + cbdata.py_newpkgcb = NULL; + cbdata.py_pkgcb = NULL; + cbdata.py_warningcb = py_warningcb; +- cbdata.py_pkgs = NULL; ++ cbdata.py_pkg = NULL; + + repomd = Repomd_FromPyObject(py_repomd); + +@@ -684,7 +677,7 @@ py_xml_parse_updateinfo(G_GNUC_UNUSED PyObject *self, PyObject *args) + cbdata.py_newpkgcb = NULL; + cbdata.py_pkgcb = NULL; + cbdata.py_warningcb = py_warningcb; +- cbdata.py_pkgs = NULL; ++ cbdata.py_pkg = NULL; + + updateinfo = UpdateInfo_FromPyObject(py_updateinfo); + +@@ -704,88 +697,3 @@ py_xml_parse_updateinfo(G_GNUC_UNUSED PyObject *self, PyObject *args) + + Py_RETURN_NONE; + } +- +- +-PyObject * +-py_xml_parse_main_metadata_together(G_GNUC_UNUSED PyObject *self, PyObject *args, PyObject *kwargs) +-{ +- char *primary_filename; +- char *filelists_filename; +- char *other_filename; +- int allow_out_of_order = 1; +- PyObject *py_newpkgcb, *py_pkgcb, *py_warningcb; +- CbData cbdata; +- GError *tmp_err = NULL; +- static char *kwlist[] = { "primary", "filelists", "other", "newpkgcb", "pkgcb", +- "warningcb", "allow_out_of_order", NULL }; +- +- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "sssOOO|p:py_xml_parse_main_metadata_together", kwlist, +- &primary_filename, &filelists_filename, &other_filename, &py_newpkgcb, +- &py_pkgcb, &py_warningcb, &allow_out_of_order)) { +- return NULL; +- } +- +- if (!PyCallable_Check(py_newpkgcb) && py_newpkgcb != Py_None) { +- PyErr_SetString(PyExc_TypeError, "newpkgcb must be callable or None"); +- return NULL; +- } +- +- if (!PyCallable_Check(py_pkgcb) && py_pkgcb != Py_None) { +- PyErr_SetString(PyExc_TypeError, "pkgcb must be callable or None"); +- return NULL; +- } +- +- if (!PyCallable_Check(py_warningcb) && py_warningcb != Py_None) { +- PyErr_SetString(PyExc_TypeError, "warningcb must be callable or None"); +- return NULL; +- } +- +- if (py_newpkgcb == Py_None && py_pkgcb == Py_None) { +- PyErr_SetString(PyExc_ValueError, "both pkgcb and newpkgcb cannot be None"); +- return NULL; +- } +- +- Py_XINCREF(py_newpkgcb); +- Py_XINCREF(py_pkgcb); +- Py_XINCREF(py_warningcb); +- +- cr_XmlParserNewPkgCb ptr_c_newpkgcb = NULL; +- cr_XmlParserPkgCb ptr_c_pkgcb = NULL; +- cr_XmlParserWarningCb ptr_c_warningcb = NULL; +- +- if (py_newpkgcb != Py_None) +- ptr_c_newpkgcb = c_newpkgcb; +- if (py_pkgcb != Py_None) +- ptr_c_pkgcb = c_pkgcb; +- if (py_warningcb != Py_None) +- ptr_c_warningcb = c_warningcb; +- +- cbdata.py_newpkgcb = py_newpkgcb; +- cbdata.py_pkgcb = py_pkgcb; +- cbdata.py_warningcb = py_warningcb; +- cbdata.py_pkgs = PyDict_New(); +- +- cr_xml_parse_main_metadata_together(primary_filename, +- filelists_filename, +- other_filename, +- ptr_c_newpkgcb, +- &cbdata, +- ptr_c_pkgcb, +- &cbdata, +- ptr_c_warningcb, +- &cbdata, +- allow_out_of_order, +- &tmp_err); +- +- Py_XDECREF(py_newpkgcb); +- Py_XDECREF(py_pkgcb); +- Py_XDECREF(py_warningcb); +- Py_XDECREF(cbdata.py_pkgs); +- +- if (tmp_err) { +- nice_exception(&tmp_err, NULL); +- return NULL; +- } +- +- Py_RETURN_NONE; +-} +diff --git a/src/python/xml_parser-py.h b/src/python/xml_parser-py.h +index 3260005..e5eea9f 100644 +--- a/src/python/xml_parser-py.h ++++ b/src/python/xml_parser-py.h +@@ -64,12 +64,4 @@ PyDoc_STRVAR(xml_parse_updateinfo__doc__, + + PyObject *py_xml_parse_updateinfo(PyObject *self, PyObject *args); + +-PyDoc_STRVAR(xml_parse_main_metadata_together__doc__, +-"xml_parse_main_metadata_together(primary_filename, filelists_filename, other_filename, newpkgcb, pkgcb, warningcb) -> None\n\n" +-"Parse primary.xml, filelists.xml and other.xml together at the same time." +-"- It can handle if packages are not in the same order in all 3 files but memory requirements grow." +-"- It is not guaranteed that newpkgcb is always followed by pkgcb for the given package, it is possible newpkgcb will be called several times for different packages and only after that pkgcbs will be called."); +- +-PyObject *py_xml_parse_main_metadata_together(PyObject *self, PyObject *args, PyObject *kwargs); +- + #endif +-- +2.35.1 + + +From ac6f56e87f8a4b1f0a35d9dedc02a8846c9a6390 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= +Date: Wed, 9 Mar 2022 15:23:58 +0100 +Subject: [PATCH 4/5] Revert "Add c tests for parsing metadata together" + +This reverts commit 3ef3d4d77e283ee916fc7977cdc3855b912463e0. +--- + tests/CMakeLists.txt | 4 - + tests/fixtures.h | 2 - + .../test_xml_parser_main_metadata_together.c | 298 ----- + .../modified_repo_files/long_primary.xml | 1060 ----------------- + .../repo_02_different_order_filelists.xml | 16 - + 5 files changed, 1380 deletions(-) + delete mode 100644 tests/test_xml_parser_main_metadata_together.c + delete mode 100644 tests/testdata/modified_repo_files/long_primary.xml + delete mode 100644 tests/testdata/modified_repo_files/repo_02_different_order_filelists.xml + +diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt +index 4ffe837..3936176 100644 +--- a/tests/CMakeLists.txt ++++ b/tests/CMakeLists.txt +@@ -38,10 +38,6 @@ ADD_EXECUTABLE(test_xml_parser_updateinfo test_xml_parser_updateinfo.c) + TARGET_LINK_LIBRARIES(test_xml_parser_updateinfo libcreaterepo_c ${GLIB2_LIBRARIES}) + ADD_DEPENDENCIES(tests test_xml_parser_updateinfo) + +-ADD_EXECUTABLE(test_xml_parser_main_metadata_together test_xml_parser_main_metadata_together.c) +-TARGET_LINK_LIBRARIES(test_xml_parser_main_metadata_together libcreaterepo_c ${GLIB2_LIBRARIES}) +-ADD_DEPENDENCIES(tests test_xml_parser_main_metadata_together) +- + ADD_EXECUTABLE(test_xml_dump test_xml_dump.c) + TARGET_LINK_LIBRARIES(test_xml_dump libcreaterepo_c ${GLIB2_LIBRARIES}) + ADD_DEPENDENCIES(tests test_xml_dump) +diff --git a/tests/fixtures.h b/tests/fixtures.h +index 676518c..a47dd66 100644 +--- a/tests/fixtures.h ++++ b/tests/fixtures.h +@@ -85,8 +85,6 @@ + #define TEST_MRF_UE_OTH_00 TEST_MODIFIED_REPO_FILES_PATH"unknown_element_00-other.xml" + #define TEST_MRF_UE_OTH_01 TEST_MODIFIED_REPO_FILES_PATH"unknown_element_01-other.xml" + #define TEST_MRF_UE_OTH_02 TEST_MODIFIED_REPO_FILES_PATH"unknown_element_02-other.xml" +-#define TEST_LONG_PRIMARY TEST_MODIFIED_REPO_FILES_PATH"long_primary.xml" +-#define TEST_DIFF_ORDER_FILELISTS TEST_MODIFIED_REPO_FILES_PATH"repo_02_different_order_filelists.xml" + + // Test files + +diff --git a/tests/test_xml_parser_main_metadata_together.c b/tests/test_xml_parser_main_metadata_together.c +deleted file mode 100644 +index a4e4ad7..0000000 +--- a/tests/test_xml_parser_main_metadata_together.c ++++ /dev/null +@@ -1,298 +0,0 @@ +-/* +- * Copyright (C) 2021 Red Hat, Inc. +- * +- * Licensed under the GNU Lesser General Public License Version 2.1 +- * +- * This library is free software; you can redistribute it and/or +- * modify it under the terms of the GNU Lesser General Public +- * License as published by the Free Software Foundation; either +- * version 2.1 of the License, or (at your option) any later version. +- * +- * This library is distributed in the hope that it will be useful, +- * but WITHOUT ANY WARRANTY; without even the implied warranty of +- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +- * Lesser General Public License for more details. +- * +- * You should have received a copy of the GNU Lesser General Public +- * License along with this library; if not, write to the Free Software +- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +- */ +- +-#include +-#include +-#include +-#include "fixtures.h" +-#include "createrepo/error.h" +-#include "createrepo/package.h" +-#include "createrepo/misc.h" +-#include "createrepo/xml_parser.h" +-#include "createrepo/xml_parser_internal.h" +- +-// Callbacks +- +-static int +-pkgcb(cr_Package *pkg, void *cbdata, GError **err) +-{ +- g_assert(pkg); +- g_assert(!err || *err == NULL); +- if (cbdata) *((int *)cbdata) += 1; +- cr_package_free(pkg); +- return CR_CB_RET_OK; +-} +- +-static int +-pkgcb_interrupt(cr_Package *pkg, void *cbdata, GError **err) +-{ +- g_assert(pkg); +- g_assert(!err || *err == NULL); +- if (cbdata) *((int *)cbdata) += 1; +- cr_package_free(pkg); +- return CR_CB_RET_ERR; +-} +- +-static int +-newpkgcb_skip_fake_bash(cr_Package **pkg, +- G_GNUC_UNUSED const char *pkgId, +- const char *name, +- G_GNUC_UNUSED const char *arch, +- G_GNUC_UNUSED void *cbdata, +- GError **err) +-{ +- g_assert(pkg != NULL); +- g_assert(*pkg == NULL); +- g_assert(pkgId != NULL); +- g_assert(!err || *err == NULL); +- +- +- if (!g_strcmp0(name, "fake_bash")) { +- return CRE_OK; +- } +- +- *pkg = cr_package_new(); +- return CR_CB_RET_OK; +-} +- +-static int +-newpkgcb_interrupt(cr_Package **pkg, +- G_GNUC_UNUSED const char *pkgId, +- G_GNUC_UNUSED const char *name, +- G_GNUC_UNUSED const char *arch, +- G_GNUC_UNUSED void *cbdata, +- GError **err) +-{ +- g_assert(pkg != NULL); +- g_assert(*pkg == NULL); +- g_assert(pkgId != NULL); +- g_assert(!err || *err == NULL); +- +- if (cbdata) *((int *)cbdata) += 1; +- +- return CR_CB_RET_ERR; +-} +- +-static int +-warningcb(G_GNUC_UNUSED cr_XmlParserWarningType type, +- G_GNUC_UNUSED char *msg, +- void *cbdata, +- G_GNUC_UNUSED GError **err) +-{ +- g_assert(type < CR_XML_WARNING_SENTINEL); +- g_assert(!err || *err == NULL); +- +- g_string_append((GString *) cbdata, msg); +- g_string_append((GString *) cbdata, ";"); +- +- return CR_CB_RET_OK; +-} +- +-static int +-warningcb_interrupt(G_GNUC_UNUSED cr_XmlParserWarningType type, +- G_GNUC_UNUSED char *msg, +- G_GNUC_UNUSED void *cbdata, +- G_GNUC_UNUSED GError **err) +-{ +- g_assert(type < CR_XML_WARNING_SENTINEL); +- g_assert(!err || *err == NULL); +- +- if (cbdata) *((int *)cbdata) += 1; +- +- return CR_CB_RET_ERR; +-} +- +-// Tests +- +-static void +-test_cr_xml_parse_main_metadata_together_00(void) +-{ +- int parsed = 0; +- GError *tmp_err = NULL; +- int ret = cr_xml_parse_main_metadata_together(TEST_REPO_02_PRIMARY, TEST_REPO_02_FILELISTS, TEST_REPO_02_OTHER, +- NULL, NULL, pkgcb, &parsed, NULL, NULL, TRUE, &tmp_err); +- g_assert(tmp_err == NULL); +- g_assert_cmpint(ret, ==, CRE_OK); +- g_assert_cmpint(parsed, ==, 2); +-} +- +-static void +-test_cr_xml_parse_main_metadata_together_01_out_of_order_pkgs(void) +-{ +- int parsed = 0; +- GError *tmp_err = NULL; +- int ret = cr_xml_parse_main_metadata_together(TEST_REPO_02_PRIMARY, +- TEST_DIFF_ORDER_FILELISTS, +- TEST_REPO_02_OTHER, +- NULL, NULL, pkgcb, &parsed, NULL, NULL, FALSE, &tmp_err); +- g_assert(tmp_err != NULL); +- g_assert_cmpint(ret, ==, CRE_XMLPARSER); +- +- g_clear_error(&tmp_err); +- ret = cr_xml_parse_main_metadata_together(TEST_REPO_02_PRIMARY, TEST_DIFF_ORDER_FILELISTS, TEST_REPO_02_OTHER, +- NULL, NULL, pkgcb, &parsed, NULL, NULL, TRUE, &tmp_err); +- g_assert(tmp_err == NULL); +- g_assert_cmpint(ret, ==, CRE_OK); +- g_assert_cmpint(parsed, ==, 2); +-} +- +-static void +-test_cr_xml_parse_main_metadata_together_02_invalid_path(void) +-{ +- int parsed = 0; +- GError *tmp_err = NULL; +- int ret = cr_xml_parse_main_metadata_together("/non/existent/file", TEST_REPO_02_FILELISTS, TEST_REPO_02_OTHER, +- NULL, NULL, pkgcb, &parsed, NULL, NULL, TRUE, &tmp_err); +- g_assert(tmp_err != NULL); +- g_assert_cmpint(ret, ==, CRE_NOFILE); +-} +- +-static void +-test_cr_xml_parse_main_metadata_together_03_newpkgcb_returns_null(void) +-{ +- int parsed = 0; +- GError *tmp_err = NULL; +- int ret = cr_xml_parse_main_metadata_together(TEST_REPO_02_PRIMARY, TEST_REPO_02_FILELISTS, TEST_REPO_02_OTHER, +- newpkgcb_skip_fake_bash, NULL, pkgcb, &parsed, NULL, NULL, TRUE, +- &tmp_err); +- g_assert(tmp_err == NULL); +- g_assert_cmpint(ret, ==, CRE_OK); +- g_assert_cmpint(parsed, ==, 1); +- +- parsed = 0; +- ret = cr_xml_parse_main_metadata_together(TEST_REPO_02_PRIMARY, TEST_DIFF_ORDER_FILELISTS, TEST_REPO_02_OTHER, +- newpkgcb_skip_fake_bash, NULL, pkgcb, &parsed, NULL, NULL, TRUE, +- &tmp_err); +- g_assert(tmp_err == NULL); +- g_assert_cmpint(ret, ==, CRE_OK); +- g_assert_cmpint(parsed, ==, 1); +- +- parsed = 0; +- ret = cr_xml_parse_main_metadata_together(TEST_REPO_02_PRIMARY, TEST_DIFF_ORDER_FILELISTS, TEST_REPO_02_OTHER, +- newpkgcb_skip_fake_bash, NULL, pkgcb, &parsed, NULL, NULL, FALSE, +- &tmp_err); +- g_assert(tmp_err == NULL); +- g_assert_cmpint(ret, ==, CRE_OK); +- g_assert_cmpint(parsed, ==, 1); +-} +- +-static void +-test_cr_xml_parse_main_metadata_together_04_newpkgcb_interrupt(void) +-{ +- int parsed = 0; +- GError *tmp_err = NULL; +- int ret = cr_xml_parse_main_metadata_together(TEST_REPO_02_PRIMARY, TEST_REPO_02_FILELISTS, TEST_REPO_02_OTHER, +- newpkgcb_interrupt, &parsed, NULL, NULL, NULL, NULL, TRUE, &tmp_err); +- g_assert(tmp_err != NULL); +- g_error_free(tmp_err); +- g_assert_cmpint(ret, ==, CRE_CBINTERRUPTED); +- g_assert_cmpint(parsed, ==, 1); +-} +- +-static void +-test_cr_xml_parse_main_metadata_together_05_pkgcb_interrupt(void) +-{ +- int parsed = 0; +- GError *tmp_err = NULL; +- int ret = cr_xml_parse_main_metadata_together(TEST_REPO_02_PRIMARY, TEST_REPO_02_FILELISTS, TEST_REPO_02_OTHER, +- NULL, NULL, pkgcb_interrupt, &parsed, NULL, NULL, TRUE, &tmp_err); +- g_assert(tmp_err != NULL); +- g_error_free(tmp_err); +- g_assert_cmpint(ret, ==, CRE_CBINTERRUPTED); +- g_assert_cmpint(parsed, ==, 1); +-} +- +-static void +-test_cr_xml_parse_main_metadata_together_06_warnings_bad_file_type(void) +-{ +- int parsed = 0; +- char *warnmsgs; +- GError *tmp_err = NULL; +- GString *warn_strings = g_string_new(0); +- int ret = cr_xml_parse_main_metadata_together(TEST_REPO_02_PRIMARY, TEST_MRF_BAD_TYPE_FIL, TEST_REPO_02_OTHER, +- NULL, NULL, pkgcb, &parsed, warningcb, warn_strings, TRUE, &tmp_err); +- g_assert(tmp_err == NULL); +- g_assert_cmpint(ret, ==, CRE_OK); +- g_assert_cmpint(parsed, ==, 2); +- warnmsgs = g_string_free(warn_strings, FALSE); +- g_assert_cmpstr(warnmsgs, ==, "Unknown file type \"foo\";"); +- g_free(warnmsgs); +-} +- +-static void +-test_cr_xml_parse_main_metadata_together_07_warningcb_interrupt(void) +-{ +- int numofwarnings = 0; +- GError *tmp_err = NULL; +- int ret = cr_xml_parse_main_metadata_together(TEST_REPO_02_PRIMARY, TEST_MRF_BAD_TYPE_FIL, TEST_REPO_02_OTHER, +- NULL, NULL, pkgcb, NULL, warningcb_interrupt, &numofwarnings, +- TRUE, &tmp_err); +- g_assert(tmp_err != NULL); +- g_error_free(tmp_err); +- g_assert_cmpint(ret, ==, CRE_CBINTERRUPTED); +- g_assert_cmpint(numofwarnings, ==, 1); +-} +- +-static void +-test_cr_xml_parse_main_metadata_together_08_long_primary(void) +-{ +- int parsed = 0; +- GError *tmp_err = NULL; +- int ret = cr_xml_parse_main_metadata_together(TEST_LONG_PRIMARY, TEST_REPO_02_FILELISTS, TEST_REPO_02_OTHER, +- NULL, NULL, pkgcb, &parsed, NULL, NULL, TRUE, &tmp_err); +- g_assert(tmp_err == NULL); +- g_assert_cmpint(ret, ==, CRE_OK); +- g_assert_cmpint(parsed, ==, 2); +-} +- +-int +-main(int argc, char *argv[]) +-{ +- g_test_init(&argc, &argv, NULL); +- +- g_test_add_func("/xml_parser_filelists/test_cr_xml_parse_main_metadata_together_00", +- test_cr_xml_parse_main_metadata_together_00); +- +- g_test_add_func("/xml_parser_filelists/test_cr_xml_parse_main_metadata_together_01_out_of_order_pkgs", +- test_cr_xml_parse_main_metadata_together_01_out_of_order_pkgs); +- +- g_test_add_func("/xml_parser_filelists/test_cr_xml_parse_main_metadata_together_02_invalid_path", +- test_cr_xml_parse_main_metadata_together_02_invalid_path); +- +- g_test_add_func("/xml_parser_filelists/test_cr_xml_parse_main_metadata_together_03_newpkgcb_returns_null", +- test_cr_xml_parse_main_metadata_together_03_newpkgcb_returns_null); +- +- g_test_add_func("/xml_parser_filelists/test_cr_xml_parse_main_metadata_together_04_newpkgcb_interrupt", +- test_cr_xml_parse_main_metadata_together_04_newpkgcb_interrupt); +- +- g_test_add_func("/xml_parser_filelists/test_cr_xml_parse_main_metadata_together_05_pkgcb_interrupt", +- test_cr_xml_parse_main_metadata_together_05_pkgcb_interrupt); +- +- g_test_add_func("/xml_parser_filelists/test_cr_xml_parse_main_metadata_together_06_warnings_bad_file_type", +- test_cr_xml_parse_main_metadata_together_06_warnings_bad_file_type); +- +- g_test_add_func("/xml_parser_filelists/test_cr_xml_parse_main_metadata_together_07_warningcb_interrupt", +- test_cr_xml_parse_main_metadata_together_07_warningcb_interrupt); +- +- g_test_add_func("/xml_parser_filelists/test_cr_xml_parse_main_metadata_together_08_long_primary", +- test_cr_xml_parse_main_metadata_together_08_long_primary); +- +- return g_test_run(); +-} +diff --git a/tests/testdata/modified_repo_files/long_primary.xml b/tests/testdata/modified_repo_files/long_primary.xml +deleted file mode 100644 +index 41d8a89..0000000 +--- a/tests/testdata/modified_repo_files/long_primary.xml ++++ /dev/null +@@ -1,1060 +0,0 @@ +- +- +- +- fake_bash +- x86_64 +- +- 90f61e546938a11449b710160ad294618a5bd3062e46f8cf851fd0088af184b7 +- Fake bash +- Fake bash package +- +- http://fake_bash_shell.com/ +- +- +- super_kernel +- x86_64 +- +- 6d43a638af70ef899933b1fd86a866f18f65b0e0e17dcbf2e42bfd0cdd7c63c3 +- Test package +- This package has provides, requires, obsoletes, conflicts options. +- +- http://so_super_kernel.com/it_is_awesome/yep_it_really_is +- +- +diff --git a/tests/testdata/modified_repo_files/repo_02_different_order_filelists.xml b/tests/testdata/modified_repo_files/repo_02_different_order_filelists.xml +deleted file mode 100644 +index 4953c95..0000000 +--- a/tests/testdata/modified_repo_files/repo_02_different_order_filelists.xml ++++ /dev/null +@@ -1,16 +0,0 @@ +- +- +- +- +- +- /usr/bin/super_kernel +- /usr/share/man/super_kernel.8.gz +- +- +- +- +- +- /usr/bin/fake_bash +- +- +- +-- +2.35.1 + + +From 16f3c1f1abe995fc1c0023f9109b7f5d520ff4b4 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= +Date: Wed, 9 Mar 2022 15:24:05 +0100 +Subject: [PATCH 5/5] Revert "Add c API for parsing metadata together" + +This reverts commit ff0912d2a89722700f001ef462abd730dacaa8e7. +--- + src/CMakeLists.txt | 1 - + src/xml_parser.h | 39 -- + src/xml_parser_main_metadata_together.c | 468 ------------------------ + 3 files changed, 508 deletions(-) + delete mode 100644 src/xml_parser_main_metadata_together.c + +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index b3ca9e6..64de052 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -31,7 +31,6 @@ SET (createrepo_c_SRCS + xml_parser_primary.c + xml_parser_repomd.c + xml_parser_updateinfo.c +- xml_parser_main_metadata_together.c + koji.c) + + SET(headers +diff --git a/src/xml_parser.h b/src/xml_parser.h +index 5ce7c0e..a31718d 100644 +--- a/src/xml_parser.h ++++ b/src/xml_parser.h +@@ -278,45 +278,6 @@ cr_xml_parse_updateinfo(const char *path, + void *warningcb_data, + GError **err); + +-/** Parse all 3 main metadata types (primary, filelists and other) at the same time. +- * Once a package is fully parsed pkgcb is called which transfers ownership of the package +- * to the user, cr_xml_parse_main_metadata_together no longer needs it and it can be freed. +- * This means we don't have store all the packages in memory at the same time, which +- * significantly reduces the memory footprint. +- * Input metadata files can be compressed. +- * @param primary_path Path to a primary xml file. +- * @param filelists_path Path to a filelists xml file. +- * @param other_path Path to an other xml file. +- * @param newpkgcb Callback for a new package. Called when the new package +- * xml chunk is found and a package object to store +- * the data is needed. +- * @param newpkgcb_data User data for the newpkgcb. +- * @param pkgcb Package callback. Called when a package is completely +- * parsed containing information from all 3 main metadata +- * files. Could be NULL if newpkgcb is not NULL. +- * @param pkgcb_data User data for the pkgcb. +- * @param warningcb Callback for warning messages. +- * @param warningcb_data User data for the warningcb. +- * @param allow_out_of_order Whether we should allow different order of packages +- * among the main metadata files. If allowed, the more +- * the order varies the more memory we will need to +- * store all the started but unfinished packages. +- * @param err GError ** +- * @return cr_Error code. +- */ +-int +-cr_xml_parse_main_metadata_together(const char *primary_path, +- const char *filelists_path, +- const char *other_path, +- cr_XmlParserNewPkgCb newpkgcb, +- void *newpkgcb_data, +- cr_XmlParserPkgCb pkgcb, +- void *pkgcb_data, +- cr_XmlParserWarningCb warningcb, +- void *warningcb_data, +- gboolean allow_out_of_order, +- GError **err); +- + /** @} */ + + #ifdef __cplusplus +diff --git a/src/xml_parser_main_metadata_together.c b/src/xml_parser_main_metadata_together.c +deleted file mode 100644 +index 9347c79..0000000 +--- a/src/xml_parser_main_metadata_together.c ++++ /dev/null +@@ -1,468 +0,0 @@ +-/* +- * Copyright (C) 2021 Red Hat, Inc. +- * +- * Licensed under the GNU Lesser General Public License Version 2.1 +- * +- * This library is free software; you can redistribute it and/or +- * modify it under the terms of the GNU Lesser General Public +- * License as published by the Free Software Foundation; either +- * version 2.1 of the License, or (at your option) any later version. +- * +- * This library is distributed in the hope that it will be useful, +- * but WITHOUT ANY WARRANTY; without even the implied warranty of +- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +- * Lesser General Public License for more details. +- * +- * You should have received a copy of the GNU Lesser General Public +- * License along with this library; if not, write to the Free Software +- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +- */ +- +-#include +-#include +-#include +-#include +-#include "error.h" +-#include "xml_parser.h" +-#include "xml_parser_internal.h" +-#include "package_internal.h" +-#include "misc.h" +- +-#define ERR_DOMAIN CREATEREPO_C_ERROR +- +-typedef struct { +- GHashTable *in_progress_pkgs_hash; //used only when allowing out of order pkgs +- GSList *in_progress_pkgs_list; // used only when not allowing out of order pkgs +- int in_progress_count_primary; +- int in_progress_count_filelists; +- int in_progress_count_other; +- cr_XmlParserNewPkgCb newpkgcb; // newpkgcb passed in from user +- void *newpkgcb_data;// newpkgcb data passed in from user +- cr_XmlParserPkgCb pkgcb; // pkgcb passed in from user +- void *pkgcb_data; // pkgcb data passed in from user +-} cr_CbData; +- +-static int +-call_user_callback_if_package_finished(cr_Package *pkg, cr_CbData *cb_data, GError **err) +-{ +- if (pkg && (pkg->loadingflags & CR_PACKAGE_LOADED_PRI) && (pkg->loadingflags & CR_PACKAGE_LOADED_OTH) && +- (pkg->loadingflags & CR_PACKAGE_LOADED_FIL)) +- { +- if (cb_data->in_progress_pkgs_hash) { +- g_hash_table_remove(cb_data->in_progress_pkgs_hash, pkg->pkgId); +- } else { +- //remove first element in the list +- cb_data->in_progress_pkgs_list = cb_data->in_progress_pkgs_list->next; +- } +- +- // One package was fully finished +- cb_data->in_progress_count_primary--; +- cb_data->in_progress_count_filelists--; +- cb_data->in_progress_count_other--; +- +- // call user package callback +- GError *tmp_err = NULL; +- if (cb_data->pkgcb && cb_data->pkgcb(pkg, cb_data->pkgcb_data, &tmp_err)) { +- if (tmp_err) +- g_propagate_prefixed_error(err, tmp_err, "Parsing interrupted: "); +- else +- g_set_error(err, ERR_DOMAIN, CRE_CBINTERRUPTED, "Parsing interrupted"); +- return CR_CB_RET_ERR; +- } else { +- // If callback return CRE_OK but it simultaneously set +- // the tmp_err then it's a programming error. +- assert(tmp_err == NULL); +- }; +- } +- return CR_CB_RET_OK; +-} +- +-static cr_Package* +-find_in_progress_pkg(cr_CbData *cb_data, const char *pkgId, int in_progress_pkg_index, GError **err) +-{ +- gpointer pval = NULL; +- if (cb_data->in_progress_pkgs_hash) { +- if (!g_hash_table_lookup_extended(cb_data->in_progress_pkgs_hash, pkgId, NULL, &pval)) { +- pval = NULL; +- } +- } else { +- // This is checking out of order pkgs because if we don't have in_progress_pkgs_hash -> we enforce +- // order by using a list +- pval = g_slist_nth_data(cb_data->in_progress_pkgs_list, in_progress_pkg_index); +- if (pval && g_strcmp0(((cr_Package *) pval)->pkgId, pkgId)) { +- g_set_error(err, ERR_DOMAIN, CRE_XMLPARSER, +- "Out of order metadata: %s vs %s.", ((cr_Package *) pval)->pkgId, pkgId); +- pval = NULL; +- } +- } +- +- return pval; +-} +- +-static void +-store_in_progress_pkg(cr_CbData *cb_data, cr_Package *pkg, const char *pkgId) +-{ +- if (!pkg) { +- return; +- } +- if (cb_data->in_progress_pkgs_hash) { +- g_hash_table_insert(cb_data->in_progress_pkgs_hash, g_strdup(pkgId), pkg); +- } else { +- cb_data->in_progress_pkgs_list = g_slist_append(cb_data->in_progress_pkgs_list, pkg); +- } +-} +- +-static int +-newpkgcb_primary(cr_Package **pkg, +- G_GNUC_UNUSED const char *pkgId, +- G_GNUC_UNUSED const char *name, +- G_GNUC_UNUSED const char *arch, +- G_GNUC_UNUSED void *cbdata, +- GError **err) +-{ +- assert(pkg && *pkg == NULL); +- assert(!err || *err == NULL); +- +- // This callback is called when parsing of the opening element of a package +- // is done. However because the opening element doesn't contain pkgId +- // (instead it looks like: ) we cannot check if we +- // already have this package. +- // The only option is to create a new package and after its fully +- // parsed (in pkgcb_primary) either use this package or copy its data +- // into an already existing one. +- // Filelists and other have pkgId present in the opening element so we can +- // avoid this overhead. +- *pkg = cr_package_new(); +- +- return CR_CB_RET_OK; +-} +- +-static int +-newpkg_general(cr_Package **pkg, +- const char *pkgId, +- const char *name, +- const char *arch, +- void *cbdata, +- int in_progress_count, +- GError **err) +-{ +- assert(pkg && *pkg == NULL); +- assert(!err || *err == NULL); +- +- cr_CbData *cb_data = cbdata; +- +- GError *out_of_order_err = NULL; +- *pkg = find_in_progress_pkg(cb_data, pkgId, in_progress_count, &out_of_order_err); +- +- if (!*pkg) { +- // we are handling never before seen package +- +- if (cb_data->newpkgcb) { +- // user specified their own new function: call it +- if (cb_data->newpkgcb(pkg, pkgId, name, arch, cb_data->newpkgcb_data, err)) { +- return CR_CB_RET_ERR; +- } +- if (!*pkg) { +- // when the user callback doesn't return a pkg we should skip it, +- // this means out of order error doesn't apply +- g_clear_error(&out_of_order_err); +- } +- } else { +- *pkg = cr_package_new(); +- } +- +- store_in_progress_pkg(cb_data, *pkg, pkgId); +- } +- +- if (*err) { +- return CR_CB_RET_ERR; +- } +- +- if (out_of_order_err) { +- g_propagate_error(err, out_of_order_err); +- return CR_CB_RET_ERR; +- } +- +- return CR_CB_RET_OK; +-} +- +-static int +-newpkgcb_filelists(cr_Package **pkg, +- const char *pkgId, +- G_GNUC_UNUSED const char *name, +- G_GNUC_UNUSED const char *arch, +- void *cbdata, +- GError **err) +-{ +- cr_CbData *cb_data = cbdata; +- return newpkg_general(pkg, pkgId, name, arch, cbdata, cb_data->in_progress_count_filelists, err); +-} +- +-static int +-newpkgcb_other(cr_Package **pkg, +- const char *pkgId, +- G_GNUC_UNUSED const char *name, +- G_GNUC_UNUSED const char *arch, +- void *cbdata, +- GError **err) +-{ +- cr_CbData *cb_data = cbdata; +- return newpkg_general(pkg, pkgId, name, arch, cbdata, cb_data->in_progress_count_other, err); +-} +- +-static int +-pkgcb_filelists(cr_Package *pkg, void *cbdata, G_GNUC_UNUSED GError **err) +-{ +- cr_CbData *cb_data = cbdata; +- cb_data->in_progress_count_filelists++; +- pkg->loadingflags |= CR_PACKAGE_LOADED_FIL; +- return call_user_callback_if_package_finished(pkg, cb_data, err); +-} +- +-static int +-pkgcb_other(cr_Package *pkg, void *cbdata, G_GNUC_UNUSED GError **err) +-{ +- cr_CbData *cb_data = cbdata; +- cb_data->in_progress_count_other++; +- pkg->loadingflags |= CR_PACKAGE_LOADED_OTH; +- return call_user_callback_if_package_finished(pkg, cb_data, err); +-} +- +-static int +-pkgcb_primary(cr_Package *pkg, void *cbdata, G_GNUC_UNUSED GError **err) +-{ +- cr_CbData *cb_data = cbdata; +- +- GError *out_of_order_err = NULL; +- cr_Package *in_progress_pkg = find_in_progress_pkg(cb_data, pkg->pkgId, cb_data->in_progress_count_primary, +- &out_of_order_err); +- if (in_progress_pkg) { +- // package was already encountered in some other metadata type +- +- cr_package_copy_into(pkg, in_progress_pkg); +- cr_package_free(pkg); +- pkg = in_progress_pkg; +- } else { +- // we are handling never before seen package +- +- if (cb_data->newpkgcb) { +- // user specified their own new function: call it and copy package data into user_created_pkg +- cr_Package *user_created_pkg = NULL; +- if (cb_data->newpkgcb(&user_created_pkg, pkg->pkgId, pkg->name, pkg->arch, cb_data->newpkgcb_data, err)) { +- return CR_CB_RET_ERR; +- } else { +- if (user_created_pkg) { +- cr_package_copy_into(pkg, user_created_pkg); +- } +- // user_created_pkg can be NULL if newpkgcb returns OK but +- // not an allocated pkg -> this means we should skip it +- store_in_progress_pkg(cb_data, user_created_pkg, pkg->pkgId); +- cr_package_free(pkg); +- pkg = user_created_pkg; +- } +- if (!pkg) { +- // when the user callback doesn't return a pkg we should skip it, +- // this means out of order error doesn't apply +- g_clear_error(&out_of_order_err); +- } +- } else { +- store_in_progress_pkg(cb_data, pkg, pkg->pkgId); +- } +- +- } +- +- if (*err) { +- return CR_CB_RET_ERR; +- } +- +- if (out_of_order_err) { +- g_propagate_error(err, out_of_order_err); +- return CR_CB_RET_ERR; +- } +- +- +- if (pkg) { +- cb_data->in_progress_count_primary++; +- pkg->loadingflags |= CR_PACKAGE_LOADED_PRI; +- pkg->loadingflags |= CR_PACKAGE_FROM_XML; +- } +- +- return call_user_callback_if_package_finished(pkg, cb_data, err); +-} +- +-static gboolean +-parse_next_section(CR_FILE *target_file, const char *path, cr_ParserData *pd, GError **err) +-{ +- char buf[XML_BUFFER_SIZE]; +- GError *tmp_err = NULL; +- int parsed_len = cr_read(target_file, buf, XML_BUFFER_SIZE, &tmp_err); +- if (tmp_err) { +- g_critical("%s: Error while reading xml '%s': %s", __func__, path, tmp_err->message); +- g_propagate_prefixed_error(err, tmp_err, "Read error: "); +- return FALSE; +- } +- int done = parsed_len == 0; +- if (xmlParseChunk(pd->parser, buf, parsed_len, done)) { +- xmlErrorPtr xml_err = xmlCtxtGetLastError(pd->parser); +- g_critical("%s: parsing error '%s': %s", __func__, path, +- (xml_err) ? xml_err->message : "UNKNOWN_ERROR"); +- g_set_error(err, ERR_DOMAIN, CRE_XMLPARSER, +- "Parse error '%s' at line: %d (%s)", +- path, +- (xml_err) ? (int) xml_err->line : 0, +- (xml_err) ? (char *) xml_err->message : "UNKNOWN_ERROR"); +- return FALSE; +- } +- +- if (pd->err) { +- g_propagate_error(err, pd->err); +- return FALSE; +- } +- +- return done; +-} +- +-//TODO(amatej): there is quite some overlap with this and cr_load_xml_files, +-// we could use this api and just wrap it in cr_loax_xml_files? +-int cr_xml_parse_main_metadata_together(const char *primary_path, +- const char *filelists_path, +- const char *other_path, +- cr_XmlParserNewPkgCb newpkgcb, +- void *newpkgcb_data, +- cr_XmlParserPkgCb pkgcb, +- void *pkgcb_data, +- cr_XmlParserWarningCb warningcb, +- void *warningcb_data, +- gboolean allow_out_of_order, +- GError **err) +-{ +- int ret = CRE_OK; +- CR_FILE *primary_f = NULL; +- CR_FILE *filelists_f = NULL; +- CR_FILE *other_f = NULL; +- GError *tmp_err = NULL; +- +- cr_CbData cbdata; +- cbdata.in_progress_pkgs_list = NULL; +- cbdata.in_progress_pkgs_hash = NULL; +- cbdata.newpkgcb = newpkgcb; +- cbdata.newpkgcb_data = newpkgcb_data; +- cbdata.pkgcb = pkgcb; +- cbdata.pkgcb_data = pkgcb_data; +- +- if (allow_out_of_order) { +- cbdata.in_progress_pkgs_hash = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); +- } +- +- assert(primary_path); +- assert(filelists_path); +- assert(other_path); +- assert(newpkgcb || pkgcb); +- assert(!err || *err == NULL); +- +- cr_ParserData *primary_pd = NULL; +- cr_ParserData *filelists_pd = NULL; +- cr_ParserData *other_pd = NULL; +- +- primary_f = cr_open(primary_path, CR_CW_MODE_READ, CR_CW_AUTO_DETECT_COMPRESSION, &tmp_err); +- if (tmp_err) { +- ret = tmp_err->code; +- g_propagate_prefixed_error(err, tmp_err, "Cannot open %s: ", primary_path); +- g_clear_error(&tmp_err); +- goto out; +- } +- filelists_f = cr_open(filelists_path, CR_CW_MODE_READ, CR_CW_AUTO_DETECT_COMPRESSION, &tmp_err); +- if (tmp_err) { +- ret = tmp_err->code; +- g_propagate_prefixed_error(err, tmp_err, "Cannot open %s: ", filelists_path); +- g_clear_error(&tmp_err); +- goto out; +- } +- other_f = cr_open(other_path, CR_CW_MODE_READ, CR_CW_AUTO_DETECT_COMPRESSION, &tmp_err); +- if (tmp_err) { +- ret = tmp_err->code; +- g_propagate_prefixed_error(err, tmp_err, "Cannot open %s: ", other_path); +- g_clear_error(&tmp_err); +- goto out; +- } +- +- //TODO(amatej): In the future we could make filelists/other optional if there is a need for it. That would mean we +- // should replace the last 0 in primary_parser_data_new depending on whether we have filelists or not. +- primary_pd = primary_parser_data_new(newpkgcb_primary, &cbdata, pkgcb_primary, &cbdata, warningcb, warningcb_data, 0); +- filelists_pd = filelists_parser_data_new(newpkgcb_filelists, &cbdata, pkgcb_filelists, &cbdata, warningcb, warningcb_data); +- other_pd = other_parser_data_new(newpkgcb_other, &cbdata, pkgcb_other, &cbdata, warningcb, warningcb_data); +- +- gboolean primary_is_done = 0; +- gboolean filelists_is_done = 0; +- gboolean other_is_done = 0; +- cbdata.in_progress_count_primary = 0; +- cbdata.in_progress_count_filelists = 0; +- cbdata.in_progress_count_other = 0; +- while (!primary_is_done || !filelists_is_done || !other_is_done) { +- while ((cbdata.in_progress_count_primary <= cbdata.in_progress_count_filelists || +- cbdata.in_progress_count_primary <= cbdata.in_progress_count_other) && +- !primary_is_done) +- { +- primary_is_done = parse_next_section(primary_f, primary_path, primary_pd, err); +- if (*err) { +- ret = (*err)->code; +- goto out; +- } +- +- } +- +- while ((cbdata.in_progress_count_filelists <= cbdata.in_progress_count_primary || +- cbdata.in_progress_count_filelists <= cbdata.in_progress_count_other) && +- !filelists_is_done) +- { +- filelists_is_done = parse_next_section(filelists_f, filelists_path, filelists_pd, err); +- if (*err) { +- ret = (*err)->code; +- goto out; +- } +- } +- +- while ((cbdata.in_progress_count_other <= cbdata.in_progress_count_filelists || +- cbdata.in_progress_count_other <= cbdata.in_progress_count_primary) && +- !other_is_done) +- { +- other_is_done = parse_next_section(other_f, other_path, other_pd, err); +- if (*err) { +- ret = (*err)->code; +- goto out; +- } +- } +- } +- +-out: +- if (ret != CRE_OK) { +- // An error already encountered +- // just close the file without error checking +- cr_close(primary_f, NULL); +- cr_close(filelists_f, NULL); +- cr_close(other_f, NULL); +- } else { +- // No error encountered yet +- cr_close(primary_f, &tmp_err); +- if (!tmp_err) +- cr_close(filelists_f, &tmp_err); +- if (!tmp_err) +- cr_close(other_f, &tmp_err); +- if (tmp_err) { +- ret = tmp_err->code; +- g_propagate_prefixed_error(err, tmp_err, "Error while closing: "); +- } +- } +- +- cr_xml_parser_data_free(primary_pd); +- cr_xml_parser_data_free(filelists_pd); +- cr_xml_parser_data_free(other_pd); +- +- if (allow_out_of_order) { +- g_hash_table_destroy(cbdata.in_progress_pkgs_hash); +- } else { +- cr_slist_free_full(cbdata.in_progress_pkgs_list, (GDestroyNotify) cr_package_free); +- } +- +- return ret; +-} +-- +2.35.1 + diff --git a/SPECS/createrepo_c.spec b/SPECS/createrepo_c.spec index 7aa15ea..c08df64 100644 --- a/SPECS/createrepo_c.spec +++ b/SPECS/createrepo_c.spec @@ -25,11 +25,12 @@ Summary: Creates a common metadata repository Name: createrepo_c Version: 0.17.7 -Release: 2%{?dist} +Release: 4%{?dist} License: GPLv2+ URL: https://github.com/rpm-software-management/createrepo_c Source0: %{url}/archive/%{version}/%{name}-%{version}.tar.gz Patch1: 0001-Default---keep-all-metadata-to-TRUE-and-add---discard-additional-metadata.patch +Patch2: 0002-Revert-added-API-for-parsing-main-metadata-together-RhBug2062299.patch BuildRequires: cmake BuildRequires: gcc @@ -176,6 +177,12 @@ ln -sr %{buildroot}%{_bindir}/modifyrepo_c %{buildroot}%{_bindir}/modifyrepo %{python3_sitearch}/%{name}-%{version}-py%{python3_version}.egg-info %changelog +* Tue Mar 15 2022 Pavla Kratochvilova - 0.17.7-4 +- Reference correct bug (RhBug:2062301) + +* Fri Mar 11 2022 Pavla Kratochvilova - 0.17.7-3 +- Revert addition of new API for parsing main metadata together (RhBug:2062299) + * Wed Feb 16 2022 Pavla Kratochvilova - 0.17.7-2 - Switch default of --keep-all-metadata to TRUE and add --discard-additional-metadata (RhBug:2055032)