core: helpers for automatic dataframes from sequences of NamedTuple/dataclass

also use in my.rescuetime

my/core/common.py

@@ -458,3 +458,16 @@ def guess_datetime(x: Any) -> Optional[datetime]:
         if isinstance(v, datetime):
             return v
     return None
+
+
+def asdict(thing) -> Json:
+    # todo primitive?
+    # todo exception?
+    if isinstance(thing, dict):
+        return thing
+    import dataclasses as D
+    if D.is_dataclass(thing):
+        return D.asdict(thing)
+    # must be a NT otherwise?
+    # todo add a proper check.. ()
+    return thing._asdict()

my/core/logging.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
 '''
-Default logger is a bit, see 'test'/run this file for a demo
+Default logger is a bit meh, see 'test'/run this file for a demo
+TODO name 'klogging' to avoid possible conflict with default 'logging' module
+TODO shit. too late already? maybe use fallback & deprecate
 '''
 
 def test() -> None:

my/core/pandas.py

@@ -6,7 +6,7 @@ Various pandas helpers and convenience functions
 from datetime import datetime
 from pprint import pformat
 from typing import Optional, TYPE_CHECKING, Any, Iterable
-from . import warnings
+from . import warnings, Res
 from .common import LazyLogger
 
 logger = LazyLogger(__name__)
@@ -109,3 +109,19 @@ def error_to_row(e: Exception, *, dt_col: str='dt', tz=None) -> Dict[str, Any]:
         'error': estr,
         dt_col : edt,
     }
+
+
+# todo add proper types
+@check_dataframe
+def as_dataframe(it: Iterable[Res[Any]]) -> DataFrameT:
+    # ok nice supports dataframe/NT natively
+    # https://github.com/pandas-dev/pandas/pull/27999
+    #    but it dispatches dataclass based on the first entry...
+    #    https://github.com/pandas-dev/pandas/blob/fc9fdba6592bdb5d0d1147ce4d65639acd897565/pandas/core/frame.py#L562
+    # same for NamedTuple -- seems that it takes whatever schema the first NT has
+    # so we need to convert each individually... sigh
+    from .common import asdict
+    ie = (error_to_row(r) if isinstance(r, Exception) else asdict(r) for r in it)
+    # TODO just add tests for it?
+    import pandas as pd
+    return pd.DataFrame(ie)