query: add --warn-exceptions, dateparser, docs (#290)
* query: add --warn-exceptions, dateparser, docs
added --warn-exceptions (like --raise-exceptions/--drop-exceptions, but
lets you pass a warn_func if you want to customize how the exceptions are
handled. By default this creates a logger in main and logs the exception
added dateparser as a fallback if its installed (it's not a strong dependency, but
I mentioned in the docs that it's useful for parsing dates/times)
added docs for query, and a few examples
--output gpx respects the --{drop,warn,raise}--exceptions flags, have
an example of that in the docs as well
This commit is contained in:
seanbreckenridge
GitHub
parent
82bc51d9fc
commit
7a32302d66
7 changed files with 407 additions and 24 deletions
|
|
@ -485,6 +485,13 @@ def _locate_functions_or_prompt(qualified_names: List[str], prompt: bool = True)
|
|||
yield data_providers[chosen_index]
|
||||
|
||||
|
||||
def _warn_exceptions(exc: Exception) -> None:
|
||||
from my.core.common import LazyLogger
|
||||
logger = LazyLogger('CLI', level='warning')
|
||||
|
||||
logger.exception(f'hpi query: {exc}')
|
||||
|
||||
|
||||
# handle the 'hpi query' call
|
||||
# can raise a QueryException, caught in the click command
|
||||
def query_hpi_functions(
|
||||
|
|
@ -501,10 +508,12 @@ def query_hpi_functions(
|
|||
limit: Optional[int],
|
||||
drop_unsorted: bool,
|
||||
wrap_unsorted: bool,
|
||||
warn_exceptions: bool,
|
||||
raise_exceptions: bool,
|
||||
drop_exceptions: bool,
|
||||
) -> None:
|
||||
from .query_range import select_range, RangeTuple
|
||||
import my.core.error as err
|
||||
|
||||
# chain list of functions from user, in the order they wrote them on the CLI
|
||||
input_src = chain(*(f() for f in _locate_functions_or_prompt(qualified_names)))
|
||||
|
|
@ -518,6 +527,8 @@ def query_hpi_functions(
|
|||
limit=limit,
|
||||
drop_unsorted=drop_unsorted,
|
||||
wrap_unsorted=wrap_unsorted,
|
||||
warn_exceptions=warn_exceptions,
|
||||
warn_func=_warn_exceptions,
|
||||
raise_exceptions=raise_exceptions,
|
||||
drop_exceptions=drop_exceptions)
|
||||
|
||||
|
|
@ -545,10 +556,21 @@ def query_hpi_functions(
|
|||
elif output == 'gpx':
|
||||
from my.location.common import locations_to_gpx
|
||||
|
||||
# if user didn't specify to ignore exceptions, warn if locations_to_gpx
|
||||
# cannot process the output of the command. This can be silenced by
|
||||
# passing --drop-exceptions
|
||||
if not raise_exceptions and not drop_exceptions:
|
||||
warn_exceptions = True
|
||||
|
||||
# can ignore the mypy warning here, locations_to_gpx yields any errors
|
||||
# if you didnt pass it something that matches the LocationProtocol
|
||||
for exc in locations_to_gpx(res, sys.stdout): # type: ignore[arg-type]
|
||||
click.echo(str(exc), err=True)
|
||||
if warn_exceptions:
|
||||
_warn_exceptions(exc)
|
||||
elif raise_exceptions:
|
||||
raise exc
|
||||
elif drop_exceptions:
|
||||
pass
|
||||
sys.stdout.flush()
|
||||
else:
|
||||
res = list(res) # type: ignore[assignment]
|
||||
|
|
@ -742,6 +764,10 @@ def module_install_cmd(user: bool, parallel: bool, modules: Sequence[str]) -> No
|
|||
default=False,
|
||||
is_flag=True,
|
||||
help="if the order of an item can't be determined while ordering, wrap them into an 'Unsortable' object")
|
||||
@click.option('--warn-exceptions',
|
||||
default=False,
|
||||
is_flag=True,
|
||||
help="if any errors are returned, print them as errors on STDERR")
|
||||
@click.option('--raise-exceptions',
|
||||
default=False,
|
||||
is_flag=True,
|
||||
|
|
@ -765,6 +791,7 @@ def query_cmd(
|
|||
limit: Optional[int],
|
||||
drop_unsorted: bool,
|
||||
wrap_unsorted: bool,
|
||||
warn_exceptions: bool,
|
||||
raise_exceptions: bool,
|
||||
drop_exceptions: bool,
|
||||
) -> None:
|
||||
|
|
@ -792,7 +819,7 @@ def query_cmd(
|
|||
|
||||
\b
|
||||
Can also query within a range. To filter comments between 2016 and 2018:
|
||||
hpi query --order-type datetime --after '2016-01-01 00:00:00' --before '2019-01-01 00:00:00' my.reddit.all.comments
|
||||
hpi query --order-type datetime --after '2016-01-01' --before '2019-01-01' my.reddit.all.comments
|
||||
'''
|
||||
|
||||
from datetime import datetime, date
|
||||
|
|
@ -831,6 +858,7 @@ def query_cmd(
|
|||
limit=limit,
|
||||
drop_unsorted=drop_unsorted,
|
||||
wrap_unsorted=wrap_unsorted,
|
||||
warn_exceptions=warn_exceptions,
|
||||
raise_exceptions=raise_exceptions,
|
||||
drop_exceptions=drop_exceptions)
|
||||
except QueryException as qe:
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ See https://beepb00p.xyz/mypy-error-handling.html#kiss for more detail
|
|||
"""
|
||||
|
||||
from itertools import tee
|
||||
from typing import Union, TypeVar, Iterable, List, Tuple, Type, Optional, Callable, Any, cast
|
||||
from typing import Union, TypeVar, Iterable, List, Tuple, Type, Optional, Callable, Any, cast, Iterator
|
||||
|
||||
from .compat import Literal
|
||||
|
||||
|
|
@ -29,6 +29,37 @@ def unwrap(res: Res[T]) -> T:
|
|||
else:
|
||||
return res
|
||||
|
||||
def drop_exceptions(itr: Iterator[Res[T]]) -> Iterator[T]:
|
||||
"""Return non-errors from the iterable"""
|
||||
for o in itr:
|
||||
if isinstance(o, Exception):
|
||||
continue
|
||||
yield o
|
||||
|
||||
|
||||
def raise_exceptions(itr: Iterable[Res[T]]) -> Iterator[T]:
|
||||
"""Raise errors from the iterable, stops the select function"""
|
||||
for o in itr:
|
||||
if isinstance(o, Exception):
|
||||
raise o
|
||||
yield o
|
||||
|
||||
|
||||
def warn_exceptions(itr: Iterable[Res[T]], warn_func: Optional[Callable[[Exception], None]] = None) -> Iterator[T]:
|
||||
# if not provided, use the 'warnings' module
|
||||
if warn_func is None:
|
||||
from my.core.warnings import medium
|
||||
def _warn_func(e: Exception) -> None:
|
||||
# TODO: print traceback? but user could always --raise-exceptions as well
|
||||
medium(str(e))
|
||||
warn_func = _warn_func
|
||||
|
||||
for o in itr:
|
||||
if isinstance(o, Exception):
|
||||
warn_func(o)
|
||||
continue
|
||||
yield o
|
||||
|
||||
|
||||
def echain(ex: E, cause: Exception) -> E:
|
||||
ex.__cause__ = cause
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from typing import TypeVar, Tuple, Optional, Union, Callable, Iterable, Iterator
|
|||
|
||||
import more_itertools
|
||||
|
||||
import my.core.error as err
|
||||
from .common import is_namedtuple
|
||||
from .error import Res, unwrap
|
||||
from .warnings import low
|
||||
|
|
@ -205,20 +206,6 @@ pass 'drop_exceptions' to ignore exceptions""")
|
|||
return None # couldn't compute a OrderFunc for this class/instance
|
||||
|
||||
|
||||
def _drop_exceptions(itr: Iterator[ET]) -> Iterator[T]:
|
||||
"""Return non-errors from the iterable"""
|
||||
for o in itr:
|
||||
if isinstance(o, Exception):
|
||||
continue
|
||||
yield o
|
||||
|
||||
|
||||
def _raise_exceptions(itr: Iterable[ET]) -> Iterator[T]:
|
||||
"""Raise errors from the iterable, stops the select function"""
|
||||
for o in itr:
|
||||
if isinstance(o, Exception):
|
||||
raise o
|
||||
yield o
|
||||
|
||||
|
||||
# currently using the 'key set' as a proxy for 'this is the same type of thing'
|
||||
|
|
@ -365,6 +352,8 @@ def select(
|
|||
limit: Optional[int] = None,
|
||||
drop_unsorted: bool = False,
|
||||
wrap_unsorted: bool = True,
|
||||
warn_exceptions: bool = False,
|
||||
warn_func: Optional[Callable[[Exception], None]] = None,
|
||||
drop_exceptions: bool = False,
|
||||
raise_exceptions: bool = False,
|
||||
) -> Iterator[ET]:
|
||||
|
|
@ -408,7 +397,9 @@ def select(
|
|||
to copy the iterator in memory (using itertools.tee) to determine how to order it
|
||||
in memory
|
||||
|
||||
The 'drop_exceptions' and 'raise_exceptions' let you ignore or raise when the src contains exceptions
|
||||
The 'drop_exceptions', 'raise_exceptions', 'warn_exceptions' let you ignore or raise
|
||||
when the src contains exceptions. The 'warn_func' lets you provide a custom function
|
||||
to call when an exception is encountered instead of using the 'warnings' module
|
||||
|
||||
src: an iterable of mixed types, or a function to be called,
|
||||
as the input to this function
|
||||
|
|
@ -469,10 +460,13 @@ Will attempt to call iter() on the value""")
|
|||
# if both drop_exceptions and drop_exceptions are provided for some reason,
|
||||
# should raise exceptions before dropping them
|
||||
if raise_exceptions:
|
||||
itr = _raise_exceptions(itr)
|
||||
itr = err.raise_exceptions(itr)
|
||||
|
||||
if drop_exceptions:
|
||||
itr = _drop_exceptions(itr)
|
||||
itr = err.drop_exceptions(itr)
|
||||
|
||||
if warn_exceptions:
|
||||
itr = err.warn_exceptions(itr, warn_func=warn_func)
|
||||
|
||||
if where is not None:
|
||||
itr = filter(where, itr)
|
||||
|
|
|
|||
|
|
@ -73,13 +73,28 @@ def parse_datetime_float(date_str: str) -> float:
|
|||
return ds_float
|
||||
try:
|
||||
# isoformat - default format when you call str() on datetime
|
||||
# this also parses dates like '2020-01-01'
|
||||
return datetime.fromisoformat(ds).timestamp()
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
return isoparse(ds).timestamp()
|
||||
except (AssertionError, ValueError):
|
||||
raise QueryException(f"Was not able to parse {ds} into a datetime")
|
||||
pass
|
||||
|
||||
try:
|
||||
import dateparser # type: ignore[import]
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
# dateparser is a bit more lenient than the above, lets you type
|
||||
# all sorts of dates as inputs
|
||||
# https://github.com/scrapinghub/dateparser#how-to-use
|
||||
res: Optional[datetime] = dateparser.parse(ds, settings={"DATE_ORDER": "YMD"})
|
||||
if res is not None:
|
||||
return res.timestamp()
|
||||
|
||||
raise QueryException(f"Was not able to parse {ds} into a datetime")
|
||||
|
||||
|
||||
# probably DateLike input? but a user could specify an order_key
|
||||
|
|
@ -267,6 +282,8 @@ def select_range(
|
|||
limit: Optional[int] = None,
|
||||
drop_unsorted: bool = False,
|
||||
wrap_unsorted: bool = False,
|
||||
warn_exceptions: bool = False,
|
||||
warn_func: Optional[Callable[[Exception], None]] = None,
|
||||
drop_exceptions: bool = False,
|
||||
raise_exceptions: bool = False,
|
||||
) -> Iterator[ET]:
|
||||
|
|
@ -293,9 +310,15 @@ def select_range(
|
|||
unparsed_range = None
|
||||
|
||||
# some operations to do before ordering/filtering
|
||||
if drop_exceptions or raise_exceptions or where is not None:
|
||||
if drop_exceptions or raise_exceptions or where is not None or warn_exceptions:
|
||||
# doesn't wrap unsortable items, because we pass no order related kwargs
|
||||
itr = select(itr, where=where, drop_exceptions=drop_exceptions, raise_exceptions=raise_exceptions)
|
||||
itr = select(
|
||||
itr,
|
||||
where=where,
|
||||
drop_exceptions=drop_exceptions,
|
||||
raise_exceptions=raise_exceptions,
|
||||
warn_exceptions=warn_exceptions,
|
||||
warn_func=warn_func)
|
||||
|
||||
order_by_chosen: Optional[OrderFunc] = None
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue