core/cachew: rely on ~/.cache for default cache path

- rely on appdirs for default cache path instead of hardcoded /var/tmp/cachew
  technically backwards incompatible, but no action needed
  you might want to clean /var/tmp/cachew after updating

- use default cache path (e.g. ~/.cache) by default
  see https://github.com/ActiveState/appdirs#some-example-output for more info
  *warning*: things will be cached by default now (used to be uncached before)

- treat cache_dir = None in the config
  *warning*: kind of backwards incompatible.. but again nothing disasterous

CHANGELOG.md

@@ -7,12 +7,10 @@ General/my.core changes:
 
     cli: add `hpi module install` and `hpi module requires`
 
-    relevant:
-  - https://github.com/karlicoss/HPI/issues/12
-  - https://github.com/karlicoss/HPI/issues/79
+    relevant: https://github.com/karlicoss/HPI/issues/12, https://github.com/karlicoss/HPI/issues/79
 
-- 97650adf3b48c653651b31c78cefe24ecae5ed4f add discovery_pure module to get modules and their dependencies via ast module
-- f90599d7e4463e936c8d95196ff767c730207202 make module discovery rely on =ast= module
+- 97650adf3b48c653651b31c78cefe24ecae5ed4f add discovery_pure module to get modules and their dependencies via `ast` module
+- f90599d7e4463e936c8d95196ff767c730207202 make module discovery rely on `ast` module
   Hopefully it will make it more robust & much faster.
 - 07f901e1e5fb2bd3009561c84cc4efd311c94733 helpers for **automatic dataframes** from sequences of NamedTuple/dataclass
 - 4012f9b7c2a429170df8600591ec8d1e1407b162 more generic functions to jsonify data
@@ -49,7 +47,7 @@ Misc:
 - d562f00dca720fd4f6736377a41168e9a796c122
 
     tests: run all tests, but exclude tests specific to my computer from CI
-    controllable via `HPI_TESTS_KARLICOSS=true``
+    controllable via `HPI_TESTS_KARLICOSS=true`
 
 - improved mypy coverage
 

my/core/cachew.py

@@ -1,8 +1,12 @@
 from contextlib import contextmanager
 from pathlib import Path
+from typing import Optional
 
+# can lead to some unexpected issues if you 'import cachew' which being in my/core directory.. so let's protect against it
+# NOTE: if we use overlay, name can be smth like my.origg.my.core.cachew ...
+assert 'my.core' in __name__, f'Expected module __name__ ({__name__}) to start with my.core'
 
-def disable_cachew():
+def disable_cachew() -> None:
     try:
         import cachew
     except ImportError:
@@ -13,8 +17,9 @@ def disable_cachew():
     settings.ENABLE = False
 
 
+from typing import Iterator
 @contextmanager
-def disabled_cachew():
+def disabled_cachew() -> Iterator[None]:
     try:
         import cachew
     except ImportError:
@@ -26,18 +31,13 @@ def disabled_cachew():
         yield
 
 
-def cache_dir() -> Path:
-    '''
-    Base directory for cachew.
-    To override, add to your config file:
-    class config:
-        cache_dir = '/your/custom/cache/path'
-    '''
-    from .core_config import config
-    cdir = config.cache_dir
-    if cdir is None:
-        # TODO handle this in core_config.py
-        # TODO fallback to default cachew dir instead? or appdirs cache
-        return Path('/var/tmp/cachew')
-    else:
-        return Path(cdir)
+def _appdirs_cache_dir() -> Path:
+    import appdirs # type: ignore
+    cd = Path(appdirs.user_cache_dir('my'))
+    cd.mkdir(exist_ok=True, parents=True)
+    return cd
+
+
+def cache_dir() -> Optional[Path]:
+    from . import core_config as CC
+    return CC.config.get_cache_dir()

my/core/common.py

@@ -214,7 +214,8 @@ if TYPE_CHECKING:
 
     mcachew: McachewType
 
-# TODO set default cache dir here instead?
+
+# TODO I don't really like 'mcachew', just 'cache' would be better... maybe?
 # todo ugh. I think it needs doublewrap, otherwise @mcachew without args doesn't work
 def mcachew(*args, **kwargs): # type: ignore[no-redef]
     """

my/core/core_config.py

@@ -4,8 +4,7 @@ Bindings for the 'core' HPI configuration
 import re
 from typing import Sequence, Optional
 
-from .common import PathIsh
-from . import warnings
+from . import warnings, PathIsh, Path
 
 try:
     from my.config import core as user_config # type: ignore[attr-defined]
@@ -20,22 +19,50 @@ except Exception as e:
         user_config = object # type: ignore[assignment, misc]
 
 
+_HPI_CACHE_DIR_DEFAULT = ''
+
 from dataclasses import dataclass
 @dataclass
 class Config(user_config):
-    # TODO if attr is set _and_ it's none, disable cache?
-    # todo or empty string?
-    # I guess flip the switch at some point when I'm confident in cachew
-    cache_dir: Optional[PathIsh] = None # FIXME use appdirs cache dir or something
+    '''
+    Config for the HPI itself.
+    To override, add to your config file something like
+
+    class config:
+        cache_dir = '/your/custom/cache/path'
+    '''
+
+    cache_dir: Optional[PathIsh] = _HPI_CACHE_DIR_DEFAULT
+    '''
+    Base directory for cachew.
+    - if None             , means cache is disabled
+    - if '' (empty string), use user cache dir (see https://github.com/ActiveState/appdirs for more info). This is the default.
+    - otherwise           , use the specified directory as base cache directory
+
+    NOTE: you shouldn't use this attribute in HPI modules directly, use Config.get_cache_dir()/cachew.cache_dir() instead
+    '''
 
-    # list of regexes/globs
-    # None means 'rely on disabled_modules'
     enabled_modules : Optional[Sequence[str]] = None
+    '''
+    list of regexes/globs
+    - None means 'rely on disabled_modules'
+    '''
 
-    # list of regexes/globs
-    # None means 'rely on enabled_modules'
     disabled_modules: Optional[Sequence[str]] = None
+    '''
+    list of regexes/globs
+    - None means 'rely on enabled_modules'
+    '''
 
+    def get_cache_dir(self) -> Optional[Path]:
+        cdir = self.cache_dir
+        if cdir is None:
+            return None
+        if cdir == _HPI_CACHE_DIR_DEFAULT:
+            from .cachew import _appdirs_cache_dir
+            return _appdirs_cache_dir()
+        else:
+            return Path(cdir)
 
     def _is_module_active(self, module: str) -> Optional[bool]:
         # None means the config doesn't specify anything
@@ -81,6 +108,7 @@ def _reset_config() -> Iterator[Config]:
     with override_config(config) as cc:
         cc.enabled_modules  = None
         cc.disabled_modules = None
+        cc.cache_dir        = None
         yield cc
 
 

tests/misc.py

@@ -4,9 +4,11 @@ import gzip
 import lzma
 import io
 import zipfile
+from typing import List
 
 from my.core.kompress import kopen, kexists, CPath
 
+
 def test_kopen(tmp_path: Path) -> None:
     "Plaintext handled transparently"
     assert kopen(tmp_path / 'file'   ).read() == 'just plaintext'
@@ -95,3 +97,26 @@ def test_warn_iterable() -> None:
 
         assert list(x2) == [1, 2, 3]
         assert len(w) == 0
+
+
+def test_cachew() -> None:
+    from cachew import settings
+    settings.ENABLE = True # by default it's off in tests (see conftest.py)
+
+    from my.core.cachew import cache_dir
+    from my.core.common import mcachew
+
+    called = 0
+    @mcachew
+    def cf() -> List[int]:
+        nonlocal called
+        called += 1
+        return [1, 2, 3]
+
+    list(cf())
+    cc = called
+    # todo ugh. how to clean cache?
+    # assert called == 1 # precondition, to avoid turdes from previous tests
+
+    assert list(cf()) == [1, 2, 3]
+    assert called == cc

tox.ini

@@ -22,6 +22,7 @@ commands =
 setenv = MY_CONFIG = nonexistent
 commands =
     pip install -e .[testing]
+    pip install cachew
 
     hpi module install my.location.google
     pip install ijson # optional dependency