✨ Add gradients command to cpac

.github/workflows/test_cpac.yml

@@ -22,7 +22,7 @@ jobs:
         platform: [docker]
         tag: [latest, nightly]
         go: [1.14]
-        python: [3.7, 3.8, 3.9, '3.10', 3.11]
+        python: [3.8, 3.9, '3.10', 3.11, 3.12]
         singularity: [3.6.4]
 
     steps:

.pre-commit-config.yaml

@@ -14,8 +14,9 @@ repos:
   - id: mypy
     additional_dependencies:
     - types-tabulate
+    - types-toml
     - types-PyYAML
-    args: [--ignore-missing-imports]
+    args: [--python-version=3.8, --ignore-missing-imports]
 - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
   rev: v2.11.0
   hooks:
@@ -31,7 +32,7 @@ repos:
     - --no-sort
 
 - repo: https://github.com/python-poetry/poetry
-  rev: 1.5.1
+  rev: 1.6.0
   hooks:
   - id: poetry-check
   - id: poetry-lock

CHANGELOG.rst

@@ -11,6 +11,8 @@ Changelog
 * Replaces ``setup.cfg`` with ``pyproject.toml``
 * Replaces ``setuptools`` with ``poetry``
 * Adds pre-commit hooks for linting and formatting
+* Require Python ≥ 3.8
+* Wrap ``cmi-dair/ba_timeseries_gradients`` as ``gradients`` command
 
 `Version 0.5.0: Parse Resources <https://github.com/FCP-INDI/cpac/releases/tag/v0.5.0>`_
 ========================================================================================

pyproject.toml

@@ -31,20 +31,23 @@ repository = "https://github.com/FCP-INDI/cpac"
 version = "1.8.5.dev1"
 
 [tool.poetry.dependencies]
-python = ">=3.7"
+python = ">=3.8"
 docker = ">=4.2.1"
 dockerpty = "*"
 docker-pycreds = "*"
 importlib_metadata = {version = "*", markers = "python_version < '3.8'"}
 pandas = ">=0.23.4"
-spython = ">=0.0.81"
+poetry = "*"
 pyyaml = "*"
 rich = "*"
 semver = "*"
+spython = ">=0.0.81"
 tabulate = ">=0.8.6"
+toml = {version = "*", markers = "python_version < '3.11'"}
 tornado = "*"
 websocket-client = "*"
 alabaster = {version = "*", optional = true}
+ba_timeseries_gradients = {git = "https://github.com/cmi-dair/ba-timeseries-gradients.git", optional = true, markers = "python_version >= '3.11' and python_version < '3.12'"}
 coveralls = {version = "*", optional = true}
 imagesize = {version = "*", optional = true}
 pytest = {version = "*", optional = true}
@@ -55,6 +58,7 @@ sphinx = {version = "*", optional = true}
 
 [tool.poetry.extras]
 dev = ["coveralls", "pytest", "pytest-remotedata", "pytest-runner", "sphinx"]
+gradients = ["ba_timeseries_gradients"]
 testing = ["alabaster", "imagesize", "pytest", "pytest-cov", "pytest-runner", "sphinx"]
 
 [tool.poetry.scripts]
@@ -75,7 +79,7 @@ testpaths = ["tests"]
 [tool.ruff]
 extend-exclude = ["docs/conf.py"]
 extend-select = ["A", "C4", "D", "G", "I", "ICN", "NPY", "PL", "RET", "RSE", "RUF", "Q", "W"]
-target-version = "py37"
+target-version = "py38"
 
 [tool.ruff.lint.per-file-ignores]
 "CPAC/func_preproc/func_preproc.py" = ["E402"]

src/cpac/__init__.py

@@ -3,7 +3,7 @@
 try:
     from importlib.metadata import PackageNotFoundError, distribution
 except ModuleNotFoundError:
-    from importlib_metadata import PackageNotFoundError, distribution
+    from importlib_metadata import PackageNotFoundError, distribution  # type: ignore
 
 DIST_NAME = __name__
 try:

src/cpac/__main__.py

@@ -13,6 +13,7 @@
 from cpac import __version__
 from cpac.backends import Backends
 from cpac.helpers import TODOs, cpac_parse_resources as parse_resources
+from cpac.utils.bare_wrap import WRAPPED, add_bare_wrapper, call
 
 _logger = logging.getLogger(__name__)
 
@@ -194,6 +195,8 @@ def _parser():
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
 
+    add_bare_wrapper(subparsers, "gradients")
+
     subparsers.add_parser(
         "pull",
         add_help=True,
@@ -408,6 +411,9 @@ def run():
     if command is None:
         parser.print_help()
         parser.exit()
+    if command in WRAPPED:
+        # directly call external package and exit on completion or failure
+        call(command, args)
     reordered_args = []
     option_value_setting = False
     for i, arg in enumerate(args.copy()):

src/cpac/utils/__init__.py

@@ -1,8 +1,9 @@
 from .checks import check_version_at_least
-from .utils import LocalsToBind, PermissionMode, Volume, Volumes
+from .utils import LocalsToBind, PermissionMode, Volume, Volumes, get_project_root
 
 __all__ = [
     "check_version_at_least",
+    "get_project_root",
     "LocalsToBind",
     "PermissionMode",
     "Volume",

src/cpac/utils/bare_wrap.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""Wrap another Python package without any modifications."""
+from argparse import REMAINDER, ArgumentParser, _SubParsersAction
+from dataclasses import dataclass
+from logging import ERROR, WARNING, log
+from shutil import which
+from subprocess import CalledProcessError, call as sub_call
+from sys import exit as sys_exit, version_info
+from typing import Any, Literal, Optional, TypedDict
+
+if version_info.minor < 9:  # noqa: PLR2004
+    from typing import Dict
+else:
+    Dict = dict  # type: ignore
+
+from packaging.markers import Marker
+
+try:
+    import tomllib
+except (ImportError, ModuleNotFoundError):
+    import toml
+
+try:
+    from ba_timeseries_gradients.parser import get_parser as gradients_parser
+except (ImportError, ModuleNotFoundError):
+    gradients_parser = None
+
+from cpac.utils import get_project_root
+
+WRAPPED = {}
+"""memoization of wrapped packages"""
+
+
+class ScriptInfo(TypedDict):
+    """Info needed for bare package wrapping."""
+
+    command: str
+    helpstring: Optional[str]
+    url: str
+
+
+@dataclass
+class WrappedBare:
+    """Info needed for bare package wrapping."""
+
+    name: str
+    command: str
+    _helpstring: Optional[str]
+    url: str
+
+    @property
+    def helpstring(self):
+        """Get the helpstring."""
+        if self._helpstring is None:
+            return f"Extra script {self.command} not found. See {self.url} for more information, or run `pip install cpac[{self.name}]` to install it."
+        return self._helpstring
+
+
+def add_bare_wrapper(parser: _SubParsersAction, command: str) -> None:
+    """Add a bare wrapper to the parser.
+
+    Parameters
+    ----------
+    parser : _SubParsersAction
+        The subparsers to add the wrapper to
+
+    command : str
+        The command to wrap
+    """
+    from cpac.__main__ import ExtendAction
+
+    bare_parser: ArgumentParser = parser.add_parser(
+        command, usage=get_wrapped(command).helpstring
+    )
+    bare_parser.add_argument("args", nargs=REMAINDER)
+    bare_parser.register("action", "extend", ExtendAction)
+
+
+def call(name: str, command: list) -> None:
+    """Call a bare-wrapped command.
+
+    Parameters
+    ----------
+    name : str
+        The name of the package to call
+
+    command : list
+        The command to run
+    """
+    if version_info.minor < 11:  # noqa: PLR2004
+        pyproject = toml.load(get_project_root() / "pyproject.toml")
+    else:
+        with open(get_project_root() / "pyproject.toml", "rb") as _f:
+            pyproject = tomllib.load(_f)
+    script = get_wrapped(name)
+    try:
+        package_info = get_nested(
+            pyproject, ["tool", "poetry", "dependencies", script.command]
+        )
+    except KeyError as ke:
+        raise KeyError(f"Package {name} not defined in dependencies") from ke
+    marker = Marker(package_info["marker"]) if "marker" in package_info else None
+    if marker and marker.evaluate() is False:
+        raise EnvironmentError(
+            f"Current environment does not meet requirements ({marker}) for package {name}"
+        )
+    if not check_for_package(script.command):
+        sub_call(["poetry", "install", "--extras", name], cwd=get_project_root())
+    try:
+        sub_call([script.command, *command])
+    except CalledProcessError as cpe:
+        log(ERROR, str(cpe))
+        sys_exit(cpe.returncode)
+    sys_exit(0)
+
+
+def check_for_package(package_name: str) -> bool:
+    """Check if a package is installed."""
+    return which(package_name) is not None
+
+
+def get_nested(
+    dct: dict, keys: list, error: Literal["raise", "warn", None] = "raise"
+) -> Any:
+    """Get a nested dictionary value.
+
+    Parameters
+    ----------
+    dct : dict
+        The dictionary to search
+
+    keys : Iterable
+        The keys to search for
+
+    error : Literal['raise', 'warn', None]
+        How to handle errors:
+        'raise' : raise an error
+        'warn' : log a warning and return None
+        None : return None without a warning
+
+    Returns
+    -------
+    Any
+        The value of the nested key or None
+
+    Examples
+    --------
+    >>> dct = {'a': {'b': {'c': 1}}}
+    >>> get_nested(dct, ['a', 'b', 'c'])
+    1
+    >>> get_nested(dct, ['a', 'b', 'd'])
+    KeyError: 'd'
+    >>> get_nested(dct, ['a', 'b', 'd'], error=None)
+    None
+    """
+    key = keys.pop(0)
+    if key in dct:
+        if len(keys) == 0:
+            return dct[key]
+        return get_nested(dct[key], keys, error)
+    if error == "raise":
+        return dct[key]
+    if error == "warn":
+        log(WARNING, "Key %s not found in %s", key, dct)
+    return None
+
+
+def get_wrapped(name: str) -> WrappedBare:
+    """Get the name of the script to run.
+
+    Parameters
+    ----------
+    name : str
+        The name of the package to run
+
+    Returns
+    -------
+    str
+        The name of the script to run
+    """
+    if name not in WRAPPED:
+        scripts: Dict[str, ScriptInfo] = {
+            "gradients": {
+                "command": "ba_timeseries_gradients",
+                "helpstring": None,
+                "url": "https://cmi-dair.github.io/ba-timeseries-gradients/ba_timeseries_gradients.html",
+            }
+        }
+        if gradients_parser is not None:
+            scripts["gradients"]["helpstring"] = (
+                gradients_parser()
+                .format_help()
+                .replace("usage: ba_timeseries_gradients", "cpac gradients")
+            )
+        if name in scripts:
+            WRAPPED[name] = WrappedBare(
+                name,
+                scripts[name]["command"],
+                scripts[name]["helpstring"],
+                scripts[name]["url"],
+            )
+        else:
+            raise KeyError(f"Package {name} not defined in scripts")
+    return WRAPPED[name]
+
+
+__all__ = ["add_bare_wrapper", "call", "WRAPPED"]

src/cpac/utils/utils.py

@@ -2,6 +2,7 @@
 
 from itertools import permutations
 import os
+from pathlib import Path
 from typing import ClassVar, Iterator, Optional, Set, Union
 from warnings import warn
 
@@ -10,6 +11,11 @@
 from cpac import DIST_NAME
 
 
+def get_project_root() -> Path:
+    """Get project root directory."""
+    return Path(__file__).parents[3]
+
+
 class LocalsToBind:
     """Class to collect local directories to bind to containers."""