From 702d41fe908ee250f40d418394048159c87e57ce Mon Sep 17 00:00:00 2001 From: Sean Breckenridge Date: Mon, 5 Apr 2021 13:29:34 -0700 Subject: [PATCH] add click function to expose select to cli --- my/core/__main__.py | 203 ++++++++++++++++++++++++++++++++++++++++- my/core/query.py | 2 - my/core/query_range.py | 8 +- setup.py | 2 +- tests/core.py | 1 + 5 files changed, 211 insertions(+), 5 deletions(-) diff --git a/my/core/__main__.py b/my/core/__main__.py index 86049e3..bb0a8c0 100644 --- a/my/core/__main__.py +++ b/my/core/__main__.py @@ -3,7 +3,8 @@ import importlib import os import sys import traceback -from typing import Optional, Sequence, Iterable, List +from datetime import datetime +from typing import Optional, Sequence, Iterable, List, Type, Any from pathlib import Path from subprocess import check_call, run, PIPE, CompletedProcess @@ -329,6 +330,69 @@ def module_install(*, user: bool, module: str) -> None: check_call(cmd) +# handle the 'hpi query' call +# can raise a QueryException, caught in the click command +def query_hpi_functions( + *, + output: str = 'json', + qualified_names: List[str], + order_key: Optional[str], + order_by_value_type: Optional[Type], + after: Any, + before: Any, + within: Any, + reverse: bool = False, + limit: Optional[int], + drop_unsorted: bool, + wrap_unsorted: bool, + raise_exceptions: bool, + drop_exceptions: bool, +) -> None: + + from itertools import chain + + from .query import locate_qualified_function + from .query_range import select_range, RangeTuple + + # chain list of functions from user, in the order they wrote them on the CLI + input_src = chain(*(locate_qualified_function(f)() for f in qualified_names)) + + res = list(select_range( + input_src, + where=None, + order_key=order_key, + order_value=None, + order_by_value_type=order_by_value_type, + unparsed_range=RangeTuple(after=after, before=before, within=within), + reverse=reverse, + limit=limit, + drop_unsorted=drop_unsorted, + wrap_unsorted=wrap_unsorted, + raise_exceptions=raise_exceptions, + drop_exceptions=drop_exceptions)) + + if output == 'json': + from .serialize import dumps + + click.echo(dumps(res)) + elif output == 'pprint': + from pprint import pprint + + pprint(res) + else: + # output == 'repl' + try: + import IPython + except ModuleNotFoundError: + eprint("'repl' requires ipython, install it with 'python3 -m pip install ipython'") + sys.exit(1) + else: + eprint(f"\nInteract with the results by using the {click.style('res', fg='green')} variable\n") + IPython.embed() + + + + @click.group() def main() -> None: ''' @@ -434,6 +498,143 @@ def module_install_cmd(user: bool, module: str) -> None: module_install(user=user, module=module) +@main.command(name='query', short_help='query the results of a HPI function') +@click.option('-o', + '--output', + default='json', + type=click.Choice(['json', 'pprint', 'repl']), + help='what to do with the result [default: json]') +@click.option('-k', + '--order-key', + default=None, + type=click.STRING, + help='order by an object attribute or dict key on the individual objects returned by the HPI function') +@click.option('-t', + '--order-type', + default='none', + type=click.Choice(['datetime', 'int', 'float', 'none']), + help='order by searching for some type on the iterable') +@click.option('--after', + default=None, + type=click.STRING, + help='while ordering, filter items for the key or type larger than or equal to this') +@click.option('--before', + default=None, + type=click.STRING, + help='while ordering, filter items for the key or type smaller than this') +@click.option('--within', + default=None, + type=click.STRING, + help="a range 'after' or 'before' to filter items by. see above for further explanation") +@click.option('--recent', + default=None, + type=click.STRING, + help="a shorthand for '--order-type datetime --reverse --before now --within'. e.g. --recent 5d") +@click.option('--reverse/--no-reverse', + default=False, + help='reverse the results returned from the functions') +@click.option('--limit', + default=None, + type=click.INT, + help='limit the number of items returned from the (functions)') +@click.option('--drop-unsorted', + default=False, + is_flag=True, + help="If the order an item can't be determined while ordering, drop those items from the results") +@click.option('--wrap-unsorted', + default=False, + is_flag=True, + help="If the order an item can't be determined while ordering, drop those items from the results") +@click.option('--raise-exceptions', + default=False, + is_flag=True, + help="If any errors are returned (as objects, not raised) from the functions, raise them") +@click.option('--drop-exceptions', + default=False, + is_flag=True, + help='Ignore any errors returned as objects from the functions') +@click.argument('FUNCTION_NAME', nargs=-1, required=True) +def query_cmd( + function_name: Sequence[str], + output: str, + order_key: Optional[str] = None, + order_type: Optional[str] = None, + after: Optional[str] = None, + before: Optional[str] = None, + within: Optional[str] = None, + recent: Optional[str] = None, + reverse: bool = False, + limit: Optional[int] = None, + drop_unsorted: bool = False, + wrap_unsorted: bool = False, + raise_exceptions: bool = False, + drop_exceptions: bool = False, +) -> None: + ''' + This allows you to query the results from one or more functions in HPI + + By default this runs with '-o json', converting the results + to JSON and printing them to STDOUT + + You can specify '-o pprint' to just print the objects using their + repr, or '-o repl' to drop into a ipython shell with access to the results + + While filtering using --order-key datetime, the --after, --before and --within + flags parse the input to their datetime and timedelta equivalents. datetimes can + be epoch time, the string 'now', or an date formatted in the ISO format. timedelta + (durations) are parsed from a similar format to the GNU 'sleep' command, e.g. + 1w2d8h5m20s -> 1 week, 2 days, 8 hours, 5 minutes, 20 seconds + + As an example, to query reddit comments I've made in the last month + + \b + hpi query --order-type datetime --before now --within 4w my.reddit.comments + or... + hpi query --recent 4w my.reddit.comments + + \b + Can also query within a range. To filter comments between 2016 and 2018: + hpi query --order-type datetime --after '2016-01-01 00:00:00' --before '2019-01-01 00:00:00' my.reddit.comments + ''' + + chosen_order_type: Optional[Type] + if order_type == "datetime": + chosen_order_type = datetime + elif order_type == "int": + chosen_order_type = int + elif order_type == "float": + chosen_order_type = float + else: + chosen_order_type = None + + if recent is not None: + before = "now" + chosen_order_type = datetime + within = recent + reverse = not reverse + + from .query import QueryException + + try: + query_hpi_functions( + output=output, + qualified_names=list(function_name), + order_key=order_key, + order_by_value_type=chosen_order_type, + after=after, + before=before, + within=within, + reverse=reverse, + limit=limit, + drop_unsorted=drop_unsorted, + wrap_unsorted=wrap_unsorted, + raise_exceptions=raise_exceptions, + drop_exceptions=drop_exceptions) + except QueryException as qe: + eprint(str(qe)) + sys.exit(1) + + # todo: add more tests? # its standard click practice to have the function click calls be a separate # function from the decorated function, as it allows the application-specific code to be diff --git a/my/core/query.py b/my/core/query.py index bbc0623..570f059 100644 --- a/my/core/query.py +++ b/my/core/query.py @@ -531,8 +531,6 @@ def test_basic_orders() -> None: import random - import pytest - def basic_iter() -> Iterator[_Int]: for v in range(1, 6): yield _Int(v) diff --git a/my/core/query_range.py b/my/core/query_range.py index ee8ff00..1ffeb1d 100644 --- a/my/core/query_range.py +++ b/my/core/query_range.py @@ -123,6 +123,8 @@ class RangeTuple(NamedTuple): of the timeframe -- 'before' - before and after - anything after 'after' and before 'before', acts as a time range """ + # technically doesn't need to be Optional[Any], + # just to make it more clear these can be None after: Optional[Any] before: Optional[Any] within: Optional[Any] @@ -267,7 +269,7 @@ def select_range( unparsed_range: Optional[RangeTuple] = None, reverse: bool = False, limit: Optional[int] = None, - drop_unsorted: bool = True, + drop_unsorted: bool = False, wrap_unsorted: bool = False, drop_exceptions: bool = False, raise_exceptions: bool = False, @@ -290,6 +292,10 @@ def select_range( If you specify a range, drop_unsorted is forced to be True """ + # if the user specified a range with no data, set the unparsed_range to None + if unparsed_range == RangeTuple(None, None, None): + unparsed_range = None + # some operations to do before ordering/filtering if drop_exceptions or raise_exceptions or where is not None: # doesnt wrap unsortable items, because we pass no order related kwargs diff --git a/setup.py b/setup.py index 308467f..28371ce 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ def main(): 'optional': [ # todo document these? 'logzero', - 'orjson', + 'orjson', # for my.core.serialize 'cachew>=0.8.0', 'mypy', # used for config checks ], diff --git a/tests/core.py b/tests/core.py index 0fab518..96d1e07 100644 --- a/tests/core.py +++ b/tests/core.py @@ -19,5 +19,6 @@ from my.core.discovery_pure import * from my.core.freezer import * from my.core.stats import * from my.core.query import * +from my.core.query_range import * from my.core.serialize import test_serialize_fallback from my.core.__main__ import *