From 298e37032ddec9a07e6cb396df3327200134230c Mon Sep 17 00:00:00 2001 From: Seppo Yli-Olli Date: Sat, 8 Oct 2022 22:36:49 +0300 Subject: [PATCH] Use tomllib in gi-docgen with fallback to tomli on older Python --- .gitlab-ci.yml | 11 ++++++++++- gidocgen/config.py | 15 ++++++++++----- meson.build | 10 +++++++++- pyproject.toml | 2 +- setup.cfg | 2 +- 5 files changed, 31 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 780d46f..b55f150 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,7 +33,16 @@ pytest: needs: [] image: fedora:latest script: - - dnf install -y pytest python3-markdown python3-jinja2 python3-pygments python3-toml python3-typogrify + - dnf install -y pytest python3-markdown python3-jinja2 python3-pygments python3-typogrify + - pytest --verbose + +pytest-tomli: + # Validates that tomli configuration works correctly + stage: check + needs: [] + image: fedora:36 + script: + - dnf install -y pytest python3-markdown python3-jinja2 python3-pygments python3-tomli python3-typogrify - pytest --verbose flake8: diff --git a/gidocgen/config.py b/gidocgen/config.py index 0047568..ca79428 100644 --- a/gidocgen/config.py +++ b/gidocgen/config.py @@ -3,7 +3,10 @@ import os import re -import toml +try: + import tomllib +except ImportError: + import tomli as tomllib from urllib.parse import urljoin @@ -19,8 +22,9 @@ class GIDocConfig: if self._config_file is not None: try: log.debug(f"Reading configuration file: {self._config_file}") - self._config = toml.load(self._config_file) - except toml.TomlDecodeError as err: + with open(self._config_file, "rb") as f: + self._config = tomllib.load(f) + except tomllib.TOMLDecodeError as err: log.error(f"Invalid configuration file: {self._config_file}: {err}") @property @@ -243,8 +247,9 @@ class GITemplateConfig: self._config = {} try: log.debug(f"Reading template configuration file: {self._config_file}") - self._config = toml.load(self._config_file) - except toml.TomlDecodeError as err: + with open(self._config_file, "rb") as f: + self._config = tomllib.load(f) + except tomllib.TOMLDecodeError as err: log.error(f"Invalid template configuration file: {self._config_file}: {err}") @property diff --git a/meson.build b/meson.build index bee1558..d967fd3 100644 --- a/meson.build +++ b/meson.build @@ -13,7 +13,6 @@ py = import('python').find_installation('python3', 'markdown', 'markupsafe', 'pygments', - 'toml', 'typogrify', ], ) @@ -27,6 +26,15 @@ if not markdown_version.version_compare('>=3.2') error('gi-docgen requires at least markdown >= 3.2') endif +if py.language_version().version_compare('<3.11') + tomli_version = run_command( + py, '-c', 'import tomli; print(tomli.__version__)', + ).stdout().strip() + if not tomli_version.version_compare('>=1.0') + error('tomli 1.0 or newer required on Python 3.10 and older') + endif +endif + configure_file( input: 'gi-docgen.py', output: 'gi-docgen', diff --git a/pyproject.toml b/pyproject.toml index 2b88735..b0b18e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ Markdown = "^3.2" MarkupSafe = "^1" Pygments = "^2" Jinja2 = "^2" -toml = "^0" +tomli = { version = ">=1,<3" markers = "python_version < '3.11'" } typogrify = "^2" [tool.poetry.dev-dependencies] diff --git a/setup.cfg b/setup.cfg index e7afae1..141d5e1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -56,7 +56,7 @@ install_requires = MarkupSafe Pygments jinja2 - toml + tomli; python_version < '3.11' typogrify [options.entry_points] -- GitLab From 247818203cd948763349b42b9723d76ec637f91c Mon Sep 17 00:00:00 2001 From: Emmanuele Bassi Date: Tue, 29 Nov 2022 12:11:49 +0000 Subject: [PATCH] Add a fallback for the old toml module Even if it's deprecated, to ease the porting away from it inside CI pipelines. --- gidocgen/config.py | 49 +++++++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/gidocgen/config.py b/gidocgen/config.py index ca79428..8863ac1 100644 --- a/gidocgen/config.py +++ b/gidocgen/config.py @@ -3,16 +3,45 @@ import os import re + +toml_module = None try: - import tomllib + import tomllib as toml_lib + toml_module = 'tomlib' except ImportError: - import tomli as tomllib + try: + import tomli as toml_lib + toml_module = 'tomli' + except ImportError: + import toml as toml_lib + toml_module = 'toml' from urllib.parse import urljoin from . import core, log, utils +class TomlConfig: + """Wrapper class for TOML loading""" + + @staticmethod + def load(toml): + log.debug(f"Using TOML module: {toml_module}") + if toml_module is None: + log.error("No toml module found") + elif toml_module in ['tomlib', 'tomli']: + try: + with open(toml, "rb") as f: + return toml_lib.load(f) + except toml_lib.TOMLDecodeError as err: + log.error(f"Invalid configuration file: {toml}: {err}") + elif toml_module in ['toml']: + try: + return toml_lib.load(toml) + except toml_lib.TomlDecodeError as err: + log.error(f"Invalid configuration file: {toml}: {err}") + + class GIDocConfig: """Load and represent the configuration for gidocgen""" def __init__(self, config_file=None): @@ -20,12 +49,8 @@ class GIDocConfig: self._config = {} if self._config_file is not None: - try: - log.debug(f"Reading configuration file: {self._config_file}") - with open(self._config_file, "rb") as f: - self._config = tomllib.load(f) - except tomllib.TOMLDecodeError as err: - log.error(f"Invalid configuration file: {self._config_file}: {err}") + log.debug(f"Reading configuration file: {self._config_file}") + self._config = TomlConfig.load(self._config_file) @property def library(self): @@ -245,12 +270,8 @@ class GITemplateConfig: self._config_file = os.path.join(templates_dir, template_name, f"{template_name}.toml") self._config = {} - try: - log.debug(f"Reading template configuration file: {self._config_file}") - with open(self._config_file, "rb") as f: - self._config = tomllib.load(f) - except tomllib.TOMLDecodeError as err: - log.error(f"Invalid template configuration file: {self._config_file}: {err}") + log.debug(f"Reading template configuration file: {self._config_file}") + self._config = TomlConfig.load(self._config_file) @property def name(self): -- GitLab From add62bee3df2418fc13543023c3c6eca3e025d65 Mon Sep 17 00:00:00 2001 From: Emmanuele Bassi Date: Tue, 29 Nov 2022 12:43:09 +0000 Subject: [PATCH 1/2] build: Check for toml fallback module Emit a message when using tomli, and a warning when using toml. --- meson.build | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/meson.build b/meson.build index d967fd3..f48f19e 100644 --- a/meson.build +++ b/meson.build @@ -29,9 +29,20 @@ endif if py.language_version().version_compare('<3.11') tomli_version = run_command( py, '-c', 'import tomli; print(tomli.__version__)', + check: true, ).stdout().strip() - if not tomli_version.version_compare('>=1.0') - error('tomli 1.0 or newer required on Python 3.10 and older') + if tomli_version.version_compare('>=1.0') + message('Falling back to toml on Python <3.11') + else + toml_version = run_command( + py, '-c', 'import toml; print(toml.__version__)', + check: true, + ).stdout().strip() + if toml_version.version_compare('>=0.10.2') + warning('Falling back to deprecated toml module; please update to tomli') + else + error('tomli 1.0 or newer required on Python 3.10 and older') + endif endif endif -- GitLab From 40212aaab05ef4d211ced0b54b4fe358626392bc Mon Sep 17 00:00:00 2001 From: Emmanuele Bassi Date: Tue, 29 Nov 2022 12:48:54 +0000 Subject: [PATCH 2/2] ci: Add Meson build job to the pipeline --- .gitlab-ci.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b55f150..1538424 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,14 +7,15 @@ stages: - check - deploy -#meson-build: -# stage: build -# needs: [] -# script: -# - dnf install -y meson ninja-build pytest python3-flake8 python3-mypy python3-markdown python3-jinja2 python3-toml python3-typogrify -# - meson _build . -# - meson test -C _build -# +meson-build: + image: fedora:latest + stage: build + needs: [] + script: + - dnf install -y meson ninja-build pytest python3-markdown python3-jinja2 python3-pygments python3-typogrify python3-flake8 python3-mypy + - meson _build . + - meson test -C _build + #pip-build: # stage: build # needs: [] -- GitLab From de9d716d8919f1f8434539b5974fc0b86b5aa5c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 30 Dec 2022 17:48:30 +0100 Subject: [PATCH] Make pyproject.toml valid TOML This avoids: TOMLDecodeError: Unclosed inline table (at line 42, column 30) When parsing the file with tomllib. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b0b18e4..4bf0712 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ Markdown = "^3.2" MarkupSafe = "^1" Pygments = "^2" Jinja2 = "^2" -tomli = { version = ">=1,<3" markers = "python_version < '3.11'" } +tomli = { version = ">=1,<3", markers = "python_version < '3.11'" } typogrify = "^2" [tool.poetry.dev-dependencies] -- GitLab