general: migrate modules to use 3.9 features

This commit is contained in:
Dima Gerasimov 2024-10-19 22:10:40 +01:00 committed by karlicoss
parent d3f9a8e8b6
commit 8496d131e7
125 changed files with 889 additions and 739 deletions

View file

@ -3,14 +3,14 @@ Hackernews data via Dogsheep [[hacker-news-to-sqlite][https://github.com/dogshee
"""
from __future__ import annotations
from collections.abc import Iterator, Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Iterator, Sequence, Optional
from my.core import get_files, Paths, Res, datetime_aware
from my.core.sqlite import sqlite_connection
import my.config
from my.core import Paths, Res, datetime_aware, get_files
from my.core.sqlite import sqlite_connection
from .common import hackernews_link
@ -33,9 +33,9 @@ class Item:
id: str
type: str
created: datetime_aware # checked and it's utc
title: Optional[str] # only present for Story
text_html: Optional[str] # should be present for Comment and might for Story
url: Optional[str] # might be present for Story
title: str | None # only present for Story
text_html: str | None # should be present for Comment and might for Story
url: str | None # might be present for Story
# todo process 'deleted'? fields?
# todo process 'parent'?

View file

@ -1,17 +1,22 @@
"""
[[https://play.google.com/store/apps/details?id=com.simon.harmonichackernews][Harmonic]] app for Hackernews
"""
from __future__ import annotations
REQUIRES = ['lxml', 'orjson']
from collections.abc import Iterator, Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
import orjson
from pathlib import Path
from typing import Any, Dict, Iterator, List, Optional, Sequence, TypedDict, cast
from typing import Any, TypedDict, cast
import orjson
from lxml import etree
from more_itertools import one
import my.config
from my.core import (
Paths,
Res,
@ -22,8 +27,10 @@ from my.core import (
stat,
)
from my.core.common import unique_everseen
import my.config
from .common import hackernews_link, SavedBase
from .common import SavedBase, hackernews_link
import my.config # isort: skip
logger = make_logger(__name__)
@ -43,7 +50,7 @@ class Cached(TypedDict):
created_at_i: int
id: str
points: int
test: Optional[str]
test: str | None
title: str
type: str # TODO Literal['story', 'comment']? comments are only in 'children' field tho
url: str
@ -94,16 +101,16 @@ def _saved() -> Iterator[Res[Saved]]:
# TODO defensive for each item!
tr = etree.parse(path)
res = one(cast(List[Any], tr.xpath(f'//*[@name="{_PREFIX}_CACHED_STORIES_STRINGS"]')))
res = one(cast(list[Any], tr.xpath(f'//*[@name="{_PREFIX}_CACHED_STORIES_STRINGS"]')))
cached_ids = [x.text.split('-')[0] for x in res]
cached: Dict[str, Cached] = {}
cached: dict[str, Cached] = {}
for sid in cached_ids:
res = one(cast(List[Any], tr.xpath(f'//*[@name="{_PREFIX}_CACHED_STORY{sid}"]')))
res = one(cast(list[Any], tr.xpath(f'//*[@name="{_PREFIX}_CACHED_STORY{sid}"]')))
j = orjson.loads(res.text)
cached[sid] = j
res = one(cast(List[Any], tr.xpath(f'//*[@name="{_PREFIX}_BOOKMARKS"]')))
res = one(cast(list[Any], tr.xpath(f'//*[@name="{_PREFIX}_BOOKMARKS"]')))
for x in res.text.split('-'):
ids, item_timestamp = x.split('q')
# not sure if timestamp is any useful?

View file

@ -1,19 +1,20 @@
"""
[[https://play.google.com/store/apps/details?id=io.github.hidroh.materialistic][Materialistic]] app for Hackernews
"""
from collections.abc import Iterator, Sequence
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, Iterator, NamedTuple, Sequence
from typing import Any, NamedTuple
from more_itertools import unique_everseen
from my.core import get_files, datetime_aware, make_logger
from my.core import datetime_aware, get_files, make_logger
from my.core.sqlite import sqlite_connection
from my.config import materialistic as config # todo migrate config to my.hackernews.materialistic
from .common import hackernews_link
# todo migrate config to my.hackernews.materialistic
from my.config import materialistic as config # isort: skip
logger = make_logger(__name__)
@ -22,7 +23,7 @@ def inputs() -> Sequence[Path]:
return get_files(config.export_path)
Row = Dict[str, Any]
Row = dict[str, Any]
class Saved(NamedTuple):