fz0x1/HPI
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

doc: some performance notes for query_range (#409)

Browse source 
* doc: some performance notes for query_range
* add ruff_cache to gitignore
This commit is contained in:
purarue 2024-11-26 13:53:10 -08:00 • committed by GitHub
parent a7f05c2cad
commit 95a16b956f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 27 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.gitignore vendored
View file

@ -155,6 +155,9 @@ celerybeat-schedule
.dmypy.json .dmypy.json
dmypy.json dmypy.json
# linters
.ruff_cache/
# Pyre type checker # Pyre type checker
.pyre/ .pyre/

33
my/core/__init__.py
View file

@ -29,22 +29,25 @@ if not TYPE_CHECKING:
__all__ = [ __all__ = [
'get_files', 'PathIsh', 'Paths',
'Json',
'make_logger',
'LazyLogger', # legacy import
'warn_if_empty',
'stat', 'Stats',
'datetime_aware', 'datetime_naive',
'assert_never', # TODO maybe deprecate from use in my.core? will be in stdlib soon
'make_config',
'__NOT_HPI_MODULE__', '__NOT_HPI_MODULE__',
'Json',
'Res', 'unwrap', 'notnone', 'LazyLogger', # legacy import
'Path',
'dataclass', 'Path', 'PathIsh',
'Paths',
'Res',
'Stats',
'assert_never', # TODO maybe deprecate from use in my.core? will be in stdlib soon
'dataclass',
'datetime_aware',
'datetime_naive',
'get_files',
'make_config',
'make_logger',
'notnone',
'stat',
'unwrap',
'warn_if_empty',
] ]

3
my/core/__main__.py
View file

@ -538,6 +538,9 @@ def query_hpi_functions(
# chain list of functions from user, in the order they wrote them on the CLI # chain list of functions from user, in the order they wrote them on the CLI
input_src = chain(*(f() for f in _locate_functions_or_prompt(qualified_names))) input_src = chain(*(f() for f in _locate_functions_or_prompt(qualified_names)))
# NOTE: if passing just one function to this which returns a single namedtuple/dataclass,
# using both --order-key and --order-type will often be faster as it does not need to
# duplicate the iterator in memory, or try to find the --order-type type on each object before sorting
res = select_range( res = select_range(
input_src, input_src,
order_key=order_key, order_key=order_key,

4
my/core/query_range.py
View file

@ -337,6 +337,8 @@ def select_range(
# if the user supplied a order_key, and/or we've generated an order_value, create # if the user supplied a order_key, and/or we've generated an order_value, create
# the function that accesses that type on each value in the iterator # the function that accesses that type on each value in the iterator
if order_key is not None or order_value is not None: if order_key is not None or order_value is not None:
# _generate_order_value_func internally here creates a copy of the iterator, which has to
# be consumed in-case we're sorting by mixed types
order_by_chosen, itr = _handle_generate_order_by(itr, order_key=order_key, order_value=order_value) order_by_chosen, itr = _handle_generate_order_by(itr, order_key=order_key, order_value=order_value)
# signifies that itr is empty -- can early return here # signifies that itr is empty -- can early return here
if order_by_chosen is None: if order_by_chosen is None:
@ -398,7 +400,7 @@ Specify a type or a key to order the value by""")
return itr return itr
# re-use items from query for testing # reuse items from query for testing
from .query import _A, _B, _Float, _mixed_iter_errors from .query import _A, _B, _Float, _mixed_iter_errors