From 675bb66e524ddc920ff78c773ecdd1d33712d0c2 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 19 Aug 2024 01:08:10 +0100 Subject: [PATCH] some notes on rethinking mixning in the user config --- doc/experiments_with_config/run | 12 +++ doc/experiments_with_config/src/pkg/config.py | 74 +++++++++++++++++++ .../src/pkg/current_impl.py | 29 ++++++++ doc/experiments_with_config/src/pkg/py.typed | 0 .../src/pkg/via_dataclass.py | 38 ++++++++++ .../src/pkg/via_properties.py | 38 ++++++++++ my/whatsapp/android.py | 26 +++++-- 7 files changed, 211 insertions(+), 6 deletions(-) create mode 100755 doc/experiments_with_config/run create mode 100644 doc/experiments_with_config/src/pkg/config.py create mode 100644 doc/experiments_with_config/src/pkg/current_impl.py create mode 100644 doc/experiments_with_config/src/pkg/py.typed create mode 100644 doc/experiments_with_config/src/pkg/via_dataclass.py create mode 100644 doc/experiments_with_config/src/pkg/via_properties.py diff --git a/doc/experiments_with_config/run b/doc/experiments_with_config/run new file mode 100755 index 0000000..8e77e8a --- /dev/null +++ b/doc/experiments_with_config/run @@ -0,0 +1,12 @@ +#!/bin/bash +set -eu +cd "$(dirname "0")" + +WHAT="$1" + +export PYTHONPATH=src + +ERROR=0 +python3 -m mypy -p "pkg.$WHAT" || ERROR=1 +python3 -c "import pkg.$WHAT as M; M.run()" || ERROR=1 +exit "$ERROR" diff --git a/doc/experiments_with_config/src/pkg/config.py b/doc/experiments_with_config/src/pkg/config.py new file mode 100644 index 0000000..8bc30e2 --- /dev/null +++ b/doc/experiments_with_config/src/pkg/config.py @@ -0,0 +1,74 @@ +from dataclasses import dataclass + + +# 'bare' config, no typing annotations even +# current_impl : works both mypy and runtime +# if we comment out export_path, mypy DOES NOT fail (bad!) +# via_dataclass : FAILS both mypy and runtime +# via_properties: works both mypy and runtime +# if we comment out export_path, mypy fails (good!) +# src/pkg/via_properties.py:32:12:32:28: error: Cannot instantiate abstract class "combined_config" with abstract attribute "export_path" [abstract] +# return combined_config() +class module_config_1: + custom_setting = 'adhoc setting' + export_path = '/path/to/data' + + +# config defined with @dataclass annotation +# current_impl : works in runtime +# mypy DOES NOT pass +# seems like it doesn't like that non-default attributes (export_path: str) in module config +# are following default attributes (export_path in this config) +# via_dataclass : works both mypy and runtime +# if we comment out export_path, mypy fails (good!) +# src/pkg/via_dataclass.py:56:12:56:28: error: Missing positional argument "export_path" in call to "combined_config" [call-arg] +# return combined_config() +# via_properties: works both mypy and runtime +# if we comment out export_path, mypy fails (good!) +# same error as above + +@dataclass +class module_config_2: + custom_setting: str = 'adhoc setting' + export_path: str = '/path/to/data' + + +# NOTE: ok, if a config attrubute happened to be a classproperty, then it fails mypy +# but it still works in runtime, which is good, easy to migrate if necessary + + +# mixed style config, some attributes are defined via property +# this is quite useful if you want to defer some computations from config import time +# current_impl : works both mypy and runtime +# if we comment out export_path, mypy DOES NOT fail (bad!) +# via_dataclass : FAILS both mypy and runtime +# via_properties: works both mypy and runtime +# if we comment out export_path, mypy fails (good!) +# same error as above +class module_config_3: + custom_setting: str = 'adhoc setting' + + @property + def export_path(self) -> str: + return '/path/to/data' + + +# same mixed style as above, but also a @dataclass annotation +# via_dataclass: FAILS both mypy and runtime +# src/pkg/via_dataclass.py: note: In function "make_config": +# src/pkg/via_dataclass.py:53:5:54:12: error: Definition of "export_path" in base class "module_config" is incompatible with definition in base class "config" [misc] +# class combined_config(user_config, config): +# ^ +# src/pkg/via_dataclass.py:56:12:56:28: error: Missing positional argument "export_path" in call to "combined_config" [call-arg] +# return combined_config() +# ^~~~~~~~~~~~~~~~~ +# via_properties: works both mypy and runtime +# if we comment out export_path, mypy fails (good!) +# same error as above +@dataclass +class module_config_4: + custom_setting: str = 'adhoc setting' + + @classproperty + def export_path(self) -> str: + return '/path/to/data' diff --git a/doc/experiments_with_config/src/pkg/current_impl.py b/doc/experiments_with_config/src/pkg/current_impl.py new file mode 100644 index 0000000..dbb4756 --- /dev/null +++ b/doc/experiments_with_config/src/pkg/current_impl.py @@ -0,0 +1,29 @@ +""" +Currently 'preferred' way of defining configs as of 20240818 +""" +from dataclasses import dataclass + +from pkg.config import module_config as user_config + + +@dataclass +class config(user_config): + export_path: str + + cache_path: str | None = None + + +def run() -> None: + print('hello from', __name__) + + cfg = config + + # check a required attribute + print(f'{cfg.export_path=}') + + # check a non-required attribute with default value + print(f'{cfg.cache_path=}') + + # check a 'dynamically' defined attribute in user config + print(f'{cfg.custom_setting=}') + diff --git a/doc/experiments_with_config/src/pkg/py.typed b/doc/experiments_with_config/src/pkg/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/doc/experiments_with_config/src/pkg/via_dataclass.py b/doc/experiments_with_config/src/pkg/via_dataclass.py new file mode 100644 index 0000000..b2135d8 --- /dev/null +++ b/doc/experiments_with_config/src/pkg/via_dataclass.py @@ -0,0 +1,38 @@ +from dataclasses import dataclass + + +@dataclass +class config: + export_path: str + cache_path: str | None = None + + +def make_config() -> config: + from pkg.config import module_config as user_config + + # NOTE: order is important -- attributes would be added in reverse order + # e.g. first from config, then from user_config -- just what we want + # NOTE: in theory, this works without @dataclass annotation on combined_config + # however, having @dataclass adds extra type checks about missing required attributes + # when we instantiate combined_config + @dataclass + class combined_config(user_config, config): ... + + return combined_config() + + +def run() -> None: + print('hello from', __name__) + + cfg = make_config() + + # check a required attribute + print(f'{cfg.export_path=}') + + # check a non-required attribute with default value + print(f'{cfg.cache_path=}') + + # check a 'dynamically' defined attribute in user config + # NOTE: mypy fails as it has no static knowledge of the attribute + # but kinda expected, not much we can do + print(f'{cfg.custom_setting=}') # type: ignore[attr-defined] diff --git a/doc/experiments_with_config/src/pkg/via_properties.py b/doc/experiments_with_config/src/pkg/via_properties.py new file mode 100644 index 0000000..b40b498 --- /dev/null +++ b/doc/experiments_with_config/src/pkg/via_properties.py @@ -0,0 +1,38 @@ +from abc import abstractmethod + +class config: + @property + @abstractmethod + def export_path(self) -> str: + raise NotImplementedError + + @property + def cache_path(self) -> str | None: + return None + + +def make_config() -> config: + from pkg.config import module_config as user_config + + # NOTE: order is important -- attributes would be added in reverse order + # e.g. first from config, then from user_config -- just what we want + class combined_config(user_config, config): ... + + return combined_config() + + +def run() -> None: + print('hello from', __name__) + + cfg = make_config() + + # check a required attribute + print(f'{cfg.export_path=}') + + # check a non-required attribute with default value + print(f'{cfg.cache_path=}') + + # check a 'dynamically' defined attribute in user config + # NOTE: mypy fails as it has no static knowledge of the attribute + # but kinda expected, not much we can do + print(f'{cfg.custom_setting=}') # type: ignore[attr-defined] diff --git a/my/whatsapp/android.py b/my/whatsapp/android.py index 3dfed3e..8fc798d 100644 --- a/my/whatsapp/android.py +++ b/my/whatsapp/android.py @@ -3,13 +3,14 @@ Whatsapp data from Android app database (in =/data/data/com.whatsapp/databases/m """ from __future__ import annotations +from abc import abstractmethod from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path import sqlite3 from typing import Union, Sequence, Iterator, Optional -from my.core import get_files, Paths, datetime_aware, Res, make_logger, make_config +from my.core import get_files, Paths, datetime_aware, Res, make_logger from my.core.common import unique_everseen from my.core.error import echain, notnone from my.core.sqlite import sqlite_connection @@ -19,17 +20,28 @@ import my.config logger = make_logger(__name__) -@dataclass -class Config(my.config.whatsapp.android): +class Config: # paths[s]/glob to the exported sqlite databases - export_path: Paths - my_user_id: Optional[str] = None + @property + @abstractmethod + def export_path(self) -> Paths: + raise NotImplementedError + + @property + def my_user_id(self) -> Optional[str]: + return None -config = make_config(Config) +def make_config() -> Config: + import my.config as user_config + + class combined_config(user_config.whatsapp.android, Config): ... + + return combined_config() def inputs() -> Sequence[Path]: + config = make_config() return get_files(config.export_path) @@ -62,6 +74,8 @@ Entity = Union[Chat, Sender, Message] def _process_db(db: sqlite3.Connection) -> Iterator[Entity]: # TODO later, split out Chat/Sender objects separately to safe on object creation, similar to other android data sources + config = make_config() + chats = {} for r in db.execute( '''