feat: hierachical, multi-source settings manager

DRAFT

datasalad/settings/__init__.py

@@ -0,0 +1,44 @@
+"""Hierarchical, multi-source settings management
+
+Validation of configuration item values
+
+
+There are two ways to do validation and type conversion.  on-access, or
+on-load. Doing it on-load would allow to reject invalid configuration
+immediately. But it might spend time on items that never get accessed.
+On-access might waste cycles on repeated checks, and possible complain later
+than useful. Here we nevertheless run a validator on-access in the default
+implementation. Particular sources may want to override this, or ensure that
+the stored value that is passed to a validator is already in the best possible
+form to make re-validation the cheapest.
+
+.. currentmodule:: datasalad.settings
+.. autosummary::
+   :toctree: generated
+
+   Settings
+   Setting
+   Source
+   CachingSource
+   Environment
+   Defaults
+
+"""
+
+from .defaults import Defaults
+from .env import Environment
+from .setting import Setting
+from .settings import Settings
+from .source import (
+    CachingSource,
+    Source,
+)
+
+__all__ = [
+    'CachingSource',
+    'Defaults',
+    'Environment',
+    'Setting',
+    'Settings',
+    'Source',
+]

datasalad/settings/defaults.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
+
+from datasalad.settings.source import InMemorySettings
+
+if TYPE_CHECKING:
+    from datasalad.settings.setting import Setting
+
+lgr = logging.getLogger('datasalad.settings')
+
+
+class Defaults(InMemorySettings):
+    """
+    Defaults are not loaded from any source. Clients have to set any
+    items they want to see a default be known for. There would typically be
+    only one instance of this class, and it is then the true source of the
+    information by itself.
+    """
+
+    def __setitem__(self, key: str, value: Setting) -> None:
+        if key in self:
+            # resetting is something that is an unusual event.
+            # __setitem__ does not allow for a dedicated "force" flag,
+            # so we leave a message at least
+            lgr.debug('Resetting %r default', key)
+        super().__setitem__(key, value)
+
+    def __str__(self):
+        return 'Defaults'

datasalad/settings/env.py

@@ -0,0 +1,143 @@
+from __future__ import annotations
+
+import logging
+from os import (
+    environ,
+)
+from os import (
+    name as os_name,
+)
+from typing import (
+    TYPE_CHECKING,
+    Any,
+)
+
+from datasalad.settings.setting import Setting
+from datasalad.settings.source import Source
+
+if TYPE_CHECKING:
+    from collections.abc import Collection
+
+lgr = logging.getLogger('datasalad.settings')
+
+
+class Environment(Source):
+    """Process environment settings source
+
+    This is a stateless source implementation that gets and sets items directly
+    in the process environment.
+
+    Environment variables can be filtered by declaring a name prefix. More
+    complex filter rules can be implemented by replacing the
+    :meth:`include_var()` method in a subclass.
+
+    It is possible to transform environment variable name to setting keys (and
+    vice versa), by implementing the methods :meth:`get_key_from_varname()` and
+    :meth:`get_varname_from_key()`.
+    """
+
+    is_writable = True
+
+    def __init__(
+        self,
+        *,
+        var_prefix: str | None = None,
+    ):
+        super().__init__()
+        self._var_prefix = var_prefix
+
+    def reinit(self):
+        """Does nothing"""
+
+    def load(self) -> None:
+        """Does nothing"""
+
+    def __getitem__(self, key: str) -> Setting:
+        matching = {
+            k: v
+            for k, v in environ.items()
+            # search for any var that match the key when transformed
+            if self.include_var(name=k, value=v) and self.get_key_from_varname(k) == key
+        }
+        if not matching:
+            raise KeyError
+        if len(matching) > 1:
+            lgr.warning(
+                'Ambiguous key %r matching multiple ENV vars: %r',
+                key,
+                list(matching.keys()),
+            )
+        k, v = matching.popitem()
+        return Setting(value=v)
+
+    def __setitem__(self, key: str, value: Setting) -> None:
+        name = self.get_varname_from_key(key)
+        environ[name] = str(value.value)
+
+    def get(self, key, default: Any = None) -> Setting:
+        try:
+            return self[key]
+        except KeyError:
+            if isinstance(default, Setting):
+                return default
+            return Setting(value=default)
+
+    def keys(self) -> Collection:
+        """Returns all keys that can be determined from the environment"""
+        return {
+            self.get_key_from_varname(k)
+            for k, v in environ.items()
+            if self.include_var(name=k, value=v)
+        }
+
+    def __str__(self):
+        return f'Environment[{self._var_prefix}]' if self._var_prefix else 'Environment'
+
+    def __contains__(self, key: str) -> bool:
+        # we only need to reimplement this due to Python's behavior to
+        # forece-modify environment variable names on Windows. Only
+        # talking directly for environ accounts for that
+        return self.get_varname_from_key(key) in environ
+
+    def __repr__(self):
+        # TODO: list keys?
+        return 'Environment()'
+
+    def include_var(
+        self,
+        name: str,
+        value: str,  # noqa: ARG002 (default implementation does not need it)
+    ) -> bool:
+        """Determine whether to source a setting from an environment variable
+
+        This default implementation tests whether the name of the variable
+        starts with the ``var_prefix`` given to the constructor.
+
+        Reimplement this method to perform custom tests.
+        """
+        return name.startswith(self._var_prefix or '')
+
+    def get_key_from_varname(self, name: str) -> str:
+        """Transform an environment variable name to a setting key
+
+        This default implementation performs returns the unchanged
+        name as a key.
+
+        Reimplement this method and ``get_varname_from_key()`` to perform
+        custom transformations.
+        """
+        return name
+
+    def get_varname_from_key(self, key: str) -> str:
+        """Transform a setting key to an environment variable name
+
+        This default implementation on checks for illegal names and
+        raises a ``ValueError``. Otherwise it returns the unchanged key.
+        """
+        if '=' in key or '\0' in key:
+            msg = "illegal environment variable name (contains '=' or NUL)"
+            raise ValueError(msg)
+        if os_name in ('os2', 'nt'):
+            # https://stackoverflow.com/questions/19023238/why-python-uppercases-all-environment-variables-in-windows
+            return key.upper()
+        return key

datasalad/settings/setting.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+from typing import (
+    Any,
+    Callable,
+)
+
+
+class UnsetValue:
+    pass
+
+
+class Setting:
+    def __init__(
+        self,
+        value: Any | UnsetValue = UnsetValue,
+        *,
+        coercer: Callable | None = None,
+        lazy: bool = False,
+    ):
+        if lazy and not callable(value):
+            msg = 'callable required for lazy evaluation'
+            raise ValueError(msg)
+        self._value = value
+        self._coercer = coercer
+        self._lazy = lazy
+
+    @property
+    def value(self) -> Any:
+        # we ignore the type error here
+        # "error: "UnsetValue" not callable"
+        # because we rule this out in the constructor
+        val = self._value() if self._lazy else self._value  # type: ignore [operator]
+        if self._coercer:
+            return self._coercer(val)
+        return val
+
+    @property
+    def coercer(self) -> Callable | None:
+        return self._coercer
+
+    @property
+    def is_lazy(self) -> bool:
+        return self._lazy
+
+    def update(self, item: Setting) -> None:
+        if item._value is not UnsetValue:  # noqa: SLF001
+            self._value = item._value  # noqa: SLF001
+            # we also need to syncronize the lazy eval flag
+            # so we can do the right thing (TM) with the
+            # new value
+            self._lazy = item._lazy  # noqa: SLF001
+
+        if item._coercer:  # noqa: SLF001
+            self._coercer = item._coercer  # noqa: SLF001
+
+    def __str__(self) -> str:
+        # wrap the value in the classname to make clear that
+        # the actual object type is different from the value
+        return f'{self.__class__.__name__}({self._value})'
+
+    def __repr__(self) -> str:
+        # wrap the value in the classname to make clear that
+        # the actual object type is different from the value
+        # TODO: report other props
+        return f'{self.__class__.__name__}({self.value!r})'
+
+    def __eq__(self, item: object) -> bool:
+        if not isinstance(item, type(self)):
+            return False
+        return self._lazy == item._lazy \
+            and self._value == item._value \
+            and self._coercer == item._coercer