core: add __NOT_HPI_MODULE__ flag to mark utility files etc

(more of an intermediate solution perhaps)

my/books/kobo.py

@@ -2,4 +2,6 @@ from ..core import warnings
 
 warnings.high('my.books.kobo is deprecated! Please use my.kobo instead!')
 
+from ..core.util import __NOT_HPI_MODULE__
+
 from ..kobo import *

my/cfg.py

@@ -29,3 +29,5 @@ def set_repo(name: str, repo: Union[Path, str]) -> None:
 # TODO set_repo is still useful, but perhaps move this thing away to core?
 
 # TODO ok, I need to get rid of this, better to rely on regular imports
+
+from .core import __NOT_HPI_MODULE__

my/common.py

@@ -1,2 +1,6 @@
-# will be deprecated. please add stuff to my.core
+from .core.warnings import high
+high("DEPRECATED! Please use my.core.common instead.")
+
+from .core import __NOT_HPI_MODULE__
+
 from .core.common import *

my/core/__init__.py

@@ -6,3 +6,4 @@ from .common import warn_if_empty
 from .common import stat
 
 from .cfg import make_config
+from .util import __NOT_HPI_MODULE__

my/core/util.py

@@ -27,28 +27,6 @@ def ignored(m: str) -> bool:
         'kython.*',
         'mycfg_stub',
         ##
-
-        ## these are just deprecated
-        'common',
-        'error',
-        'cfg',
-        ##
-
-        ## TODO vvv these should be moved away from here
-        'jawbone.plots',
-        'emfit.plot',
-        # 'google.takeout.paths',
-        'bluemaestro.check',
-        'location.__main__',
-        'photos.utils',
-        'books.kobo',
-        'coding',
-        'media',
-        'reading',
-        '_rss',
-        'twitter.common',
-        'rss.common',
-        'lastfm.fill_influxdb',
     ]
     exs = '|'.join(excluded)
     return re.match(f'^my.({exs})$', m) is not None
@@ -64,8 +42,27 @@ def get_stats(module: str):
     return getattr(mod, 'stats', None)
 
 
-__NOT_A_MODULE__ = 'Import this to mark a python file as a helper, not an actual module'
+__NOT_HPI_MODULE__ = 'Import this to mark a python file as a helper, not an actual HPI module'
 
+def has_not_module_flag(module: str) -> bool:
+    # if module == 'my.books.kobo':
+    #     breakpoint()
+    #     pass
+    try:
+        mod = import_module(module)
+    except Exception as e:
+        return False
+
+    return any(x is __NOT_HPI_MODULE__ for x in vars(mod).values())
+
+def is_not_hpi_module(module: str) -> Optional[str]:
+    # None if a module, otherwise returns reason
+    if has_not_module_flag(module):
+        return "marked explicitly (via __NOT_HPI_MODULE__)"
+    stats = get_stats(module)
+    if stats is None:
+        return "has no 'stats()' function"
+    return None
 
 # todo reuse in readme/blog post
 # borrowed from https://github.com/sanitizers/octomachinery/blob/24288774d6dcf977c5033ae11311dbff89394c89/tests/circular_imports_test.py#L22-L55
@@ -138,10 +135,10 @@ def _walk_packages(path=None, prefix='', onerror=None) -> Iterable[HPIModule]:
             skip_reason = 'suppressed in the user config'
         elif active is None:
             # unspecified by the user, rely on other means
-            # stats detection is the last resort (because it actually tries to import)
+            is_not_module = is_not_hpi_module(mname)
-            stats = get_stats(mname)
+            if is_not_module is not None:
-            if stats is None:
+                skip_reason = is_not_module
-                skip_reason = "has no 'stats()' function"
+
         else: # active is True
             # nothing to do, enabled explicitly
             pass

my/error.py

@@ -1,2 +1,6 @@
-# will be deprecated. please add stuff to my.core
+from .core.warnings import high
+high("DEPRECATED! Please use my.core.error instead.")
+
+from .core import __NOT_HPI_MODULE__
+
 from .core.error import *

my/google/takeout/html.py

@@ -139,3 +139,5 @@ def read_html(tpath: Path, file: str) -> Iterable[Parsed]:
         data = fo.read()
         parser.feed(data)
     return results
+
+from ...core import __NOT_HPI_MODULE__

my/google/takeout/paths.py

@@ -3,7 +3,8 @@ Module for locating and accessing [[https://takeout.google.com][Google Takeout]]
 '''
 
 from dataclasses import dataclass
-from ...core.common import Paths
+from ...core.common import Paths, get_files
+from ...core.util import __NOT_HPI_MODULE__
 
 from my.config import google as user_config
 @dataclass
@@ -19,7 +20,6 @@ config = make_config(google)
 from pathlib import Path
 from typing import Optional, Iterable
 
-from ...common import get_files
 from ...kython.kompress import kopen, kexists
 
 

my/location/__main__.py

@@ -1,24 +0,0 @@
-import sys
-
-from .takeout import logger, get_locations, iter_locations, get_groups
-
-
-# TODO remove this?
-def main():
-    if len(sys.argv) > 1:
-        cmd = sys.argv[1]
-        # TODO ok, update cache makes sense just to refresh in case of code changes...
-        # TODO don't even need it anymore? cachew handles this..
-        if cmd == "update_cache":
-            from .takeout import update_cache, get_locations
-            update_cache()
-        else:
-            raise RuntimeError(f"Unknown command {cmd}")
-    else:
-        for p in get_groups():
-            print(p)
-            # shit. ok, 4 gigs of ram is def too much for glumov...
-            # TODO need datetime!
-
-if __name__ == '__main__':
-    main()

my/photos/utils.py

@@ -84,3 +84,5 @@ def dt_from_path(p: Path) -> Optional[datetime]:
         return None
     dates = mm.group(1) + mm.group(2)
     return datetime.strptime(dates, "%Y%m%d%H%M%S")
+
+from ..core import __NOT_HPI_MODULE__

my/rss/common.py

@@ -47,3 +47,5 @@ def compute_subscriptions(*sources: Iterable[SubscriptionState]) -> List[Subscri
         present = u in last_urls
         res.append(x._replace(subscribed=present))
     return res
+
+from ..core import __NOT_HPI_MODULE__

my/twitter/common.py

@@ -2,7 +2,7 @@ from itertools import chain
 
 from more_itertools import unique_everseen
 
-from ..core import warn_if_empty
+from ..core import warn_if_empty, __NOT_HPI_MODULE__
 
 @warn_if_empty
 def merge_tweets(*sources):