my.core.logging: compatibility with HPI_LOGS

re-adds a removed check for HPI_LOGS, add some docs

fix the checks for browserexport/takeout logs to
use the computed level from my.core.logging

doc/MODULE_DESIGN.org

@@ -233,3 +233,17 @@ The main goals are:
 It could be argued that namespace packages and editable installs are a bit complex for a new user to get the hang of, and this is true. But fortunately ~import_source~ means any user just using HPI only needs to follow the instructions when a warning is printed, or peruse the docs here a bit -- there's no need to clone or create your own override to just use the ~all.py~ file.
 
 There's no requirement to use this for individual modules, it just seems to be the best solution we've arrived at so far
+
+* Logging
+
+The ~my.core~ module exports a ~make_logger~ function which works nicely with
+~cachew~ and gives you colored logs. You can use it like this:
+
+#+begin_src python
+    from my.core import make_logger
+
+    logger = make_logger(__name__)
+
+    # or to set a custom level
+    logger = make_logger(__name__, level='warning')
+#+end_src

doc/SETUP.org

@@ -192,7 +192,11 @@ HPI comes with a command line tool that can help you detect potential issues. Ru
 
 If you only have a few modules set up, lots of them will error for you, which is expected, so check the ones you expect to work.
 
-If you're having issues with ~cachew~ or want to show logs to troubleshoot what may be happening, you can pass the debug flag (e.g., ~hpi --debug doctor my.module_name~) or set the ~HPI_LOGS~ environment variable (e.g., ~HPI_LOGS=debug hpi query my.module_name~) to print all logs, including the ~cachew~ dependencies. ~HPI_LOGS~ could also be used to silence ~info~ logs, like ~HPI_LOGS=warning hpi ...~
+If you're having issues with ~cachew~ or want to show logs to troubleshoot what may be happening, you can pass the debug flag (e.g., ~hpi --debug doctor my.module_name~) or set the ~LOGGING_LEVEL_HPI~ environment variable (e.g., ~LOGGING_LEVEL_HPI=debug hpi query my.module_name~) to print all logs, including the ~cachew~ dependencies. ~LOGGING_LEVEL_HPI~ could also be used to silence ~info~ logs, like ~LOGGING_LEVEL_HPI=warning hpi ...~
+
+If you want to enable logs for a particular module, you can use the
+~LOGGING_LEVEL_~ prefix and then the module name with underscores, like
+~LOGGING_LEVEL_my_hypothesis=debug hpi query my.hypothesis~
 
 If you want ~HPI~ to autocomplete the module names for you, this comes with shell completion, see [[../misc/completion/][misc/completion]]
 

my/browser/active_browser.py

@@ -26,8 +26,7 @@ from browserexport.merge import read_visits, Visit
 from sqlite_backup import sqlite_backup
 
 from .common import _patch_browserexport_logs
-
+_patch_browserexport_logs(__name__)
-_patch_browserexport_logs()
 
 
 def inputs() -> Sequence[Path]:

my/browser/common.py

@@ -1,11 +1,12 @@
-import os
+from my.core import make_logger
 from my.core.util import __NOT_HPI_MODULE__
 
 
-def _patch_browserexport_logs():
+def _patch_browserexport_logs(module_name: str):
-    # patch browserexport logs if HPI_LOGS is present
+    # get the logger for the module this is being called from
-    if "HPI_LOGS" in os.environ:
+    module_logger = make_logger(module_name)
-        from browserexport.log import setup as setup_browserexport_logger
-        from my.core.logging import mklevel
 
-        setup_browserexport_logger(mklevel(os.environ["HPI_LOGS"]))
+    # grab the computed level (respects LOGGING_LEVEL_ prefixes) and set it on the browserexport logger
+    from browserexport.log import setup as setup_browserexport_logger
+
+    setup_browserexport_logger(module_logger.level)

my/browser/export.py

@@ -26,8 +26,7 @@ from .common import _patch_browserexport_logs
 
 
 logger = LazyLogger(__name__, level="warning")
-
+_patch_browserexport_logs(__name__)
-_patch_browserexport_logs()
 
 
 # all of my backed up databases

my/core/__main__.py

@@ -596,9 +596,9 @@ def main(debug: bool) -> None:
     Tool for HPI
     Work in progress, will be used for config management, troubleshooting & introspection
     '''
-    # should overwrite anything else in HPI_LOGS
+    # should overwrite anything else in LOGGING_LEVEL_HPI
     if debug:
-        os.environ["HPI_LOGS"] = "debug"
+        os.environ['LOGGING_LEVEL_HPI'] = 'debug'
 
     # for potential future reference, if shared state needs to be added to groups
     # https://click.palletsprojects.com/en/7.x/commands/#group-invocation-without-command

my/core/logging.py

@@ -81,6 +81,19 @@ def get_env_level(name: str) -> Level | None:
     lvl = os.environ.get(PREFIX + name, None) or os.environ.get(PREFIX + name.replace('.', '_'), None)
     if lvl is not None:
         return mklevel(lvl)
+    # if LOGGING_LEVEL_HPI is set, use that. This should override anything the module may set as its default
+    # this is also set when the user passes the --debug flag in the CLI
+    #
+    # check after LOGGING_LEVEL_ prefix since that is more specific
+    if 'LOGGING_LEVEL_HPI' in os.environ:
+        return mklevel(os.environ['LOGGING_LEVEL_HPI'])
+    # legacy name, for backwards compatibility
+    if 'HPI_LOGS' in os.environ:
+        from my.core.warnings import medium
+
+        medium('The HPI_LOGS environment variable is deprecated, use LOGGING_LEVEL_HPI instead')
+
+        return mklevel(os.environ['HPI_LOGS'])
     return None
 
 

my/google/takeout/parser.py

@@ -52,12 +52,9 @@ config = make_config(google)
 
 logger = LazyLogger(__name__, level="warning")
 
-# patch TAKEOUT_LOGS to match HPI_LOGS
+# patch the takeout parser logger to match the computed loglevel
-if "HPI_LOGS" in os.environ:
+from google_takeout_parser.log import setup as setup_takeout_logger
-    from google_takeout_parser.log import setup as setup_takeout_logger
+setup_takeout_logger(logger.level)
-    from my.core.logging import mklevel
-
-    setup_takeout_logger(mklevel(os.environ["HPI_LOGS"]))
 
 
 DISABLE_TAKEOUT_CACHE = "DISABLE_TAKEOUT_CACHE" in os.environ