f175acc848
BREAKING CHANGE! Data parsing was switched to pockexport. This would help to keep it consistent across different apps in the future. When you update, you'll need to: - clone pockexport (latest version) - set pockexport repository in your config (see doc/MODULES.org)
229 lines
8.1 KiB
Org Mode
229 lines
8.1 KiB
Org Mode
This file is an overview of *documented* modules (which I'm progressively expanding).
|
|
|
|
There are many more, see:
|
|
|
|
- [[file:../README.org::#whats-inside]["What's inside"]] for the full list of modules.
|
|
- you can also run =hpi modules= to list what's available on your system
|
|
- source code is always the primary source of truth
|
|
|
|
If you have some issues with the setup, see [[file:SETUP.org::#troubleshooting]["Troubleshooting"]].
|
|
|
|
* TOC
|
|
:PROPERTIES:
|
|
:TOC: :include all
|
|
:END:
|
|
:CONTENTS:
|
|
- [[#toc][TOC]]
|
|
- [[#intro][Intro]]
|
|
- [[#configs][Configs]]
|
|
- [[#mygoogletakeoutpaths][my.google.takeout.paths]]
|
|
- [[#myhypothesis][my.hypothesis]]
|
|
- [[#myreddit][my.reddit]]
|
|
- [[#mypocket][my.pocket]]
|
|
- [[#mytwittertwint][my.twitter.twint]]
|
|
- [[#mytwitterarchive][my.twitter.archive]]
|
|
- [[#mylastfm][my.lastfm]]
|
|
- [[#myreadingpolar][my.reading.polar]]
|
|
- [[#myinstapaper][my.instapaper]]
|
|
:END:
|
|
|
|
* Intro
|
|
|
|
See [[file:SETUP.org][SETUP]] to find out how to set up your own config.
|
|
|
|
Some explanations:
|
|
|
|
- =MY_CONFIG= is the path where you are keeping your private configuration (usually =~/.config/my/=)
|
|
- [[https://docs.python.org/3/library/pathlib.html#pathlib.Path][Path]] is a standard Python object to represent paths
|
|
- [[https://github.com/karlicoss/HPI/blob/5f4acfddeeeba18237e8b039c8f62bcaa62a4ac2/my/core/common.py#L9][PathIsh]] is a helper type to allow using either =str=, or a =Path=
|
|
- [[https://github.com/karlicoss/HPI/blob/5f4acfddeeeba18237e8b039c8f62bcaa62a4ac2/my/core/common.py#L108][Paths]] is another helper type for paths.
|
|
|
|
It's 'smart', allows you to be flexible about your config:
|
|
|
|
- simple =str= or a =Path=
|
|
- =/a/path/to/directory/=, so the module will consume all files from this directory
|
|
- a list of files/directories (it will be flattened)
|
|
- a [[https://docs.python.org/3/library/glob.html?highlight=glob#glob.glob][glob]] string, so you can be flexible about the format of your data on disk (e.g. if you want to keep it compressed)
|
|
- empty sequence (e.g. ~export_path = ()~), this is useful for modules that merge multiple data sources (for example, =my.twitter=)
|
|
|
|
Typically, such variable will be passed to =get_files= to actually extract the list of real files to use. You can see usage examples [[https://github.com/karlicoss/HPI/blob/master/tests/get_files.py][here]].
|
|
|
|
- if the field has a default value, you can omit it from your private config altogether
|
|
|
|
* Configs
|
|
|
|
The config snippets below are meant to be modified accordingly and *pasted into your private configuration*, e.g =$MY_CONFIG/my/config.py=.
|
|
|
|
You don't have to set them up all at once, it's recommended to do it gradually.
|
|
|
|
# TODO hmm. drawer raw means it can output outlines, but then have to manually erase the generated results. ugh.
|
|
|
|
#+begin_src python :dir .. :results output drawer raw :exports result
|
|
# TODO ugh, pkgutil.walk_packages doesn't recurse and find packages like my.twitter.archive??
|
|
# yep.. https://stackoverflow.com/q/41203765/706389
|
|
import importlib
|
|
# from lint import all_modules # meh
|
|
# TODO figure out how to discover configs automatically...
|
|
modules = [
|
|
('google' , 'my.google.takeout.paths'),
|
|
('hypothesis' , 'my.hypothesis' ),
|
|
('reddit' , 'my.reddit' ),
|
|
('pocket' , 'my.pocket' ),
|
|
('twint' , 'my.twitter.twint' ),
|
|
('twitter_archive', 'my.twitter.archive' ),
|
|
('lastfm' , 'my.lastfm' ),
|
|
('polar' , 'my.reading.polar' ),
|
|
('instapaper' , 'my.instapaper' ),
|
|
]
|
|
|
|
def indent(s, spaces=4):
|
|
return ''.join(' ' * spaces + l for l in s.splitlines(keepends=True))
|
|
|
|
from pathlib import Path
|
|
import inspect
|
|
from dataclasses import fields
|
|
import re
|
|
print('\n') # ugh. hack for org-ruby drawers bug
|
|
for cls, p in modules:
|
|
m = importlib.import_module(p)
|
|
C = getattr(m, cls)
|
|
src = inspect.getsource(C)
|
|
i = src.find('@property')
|
|
if i != -1:
|
|
src = src[:i]
|
|
src = src.strip()
|
|
src = re.sub(r'(class \w+)\(.*', r'\1:', src)
|
|
mpath = p.replace('.', '/')
|
|
for x in ['.py', '__init__.py']:
|
|
if Path(mpath + x).exists():
|
|
mpath = mpath + x
|
|
print(f'** [[file:../{mpath}][{p}]]')
|
|
mdoc = m.__doc__
|
|
if mdoc is not None:
|
|
print(indent(mdoc))
|
|
print(f' #+begin_src python')
|
|
print(indent(src))
|
|
print(f' #+end_src')
|
|
#+end_src
|
|
|
|
#+RESULTS:
|
|
|
|
|
|
** [[file:../my/google/takeout/paths.py][my.google.takeout.paths]]
|
|
|
|
Module for locating and accessing [[https://takeout.google.com][Google Takeout]] data
|
|
|
|
#+begin_src python
|
|
class google:
|
|
takeout_path: Paths # path/paths/glob for the takeout zips
|
|
#+end_src
|
|
** [[file:../my/hypothesis.py][my.hypothesis]]
|
|
|
|
[[https://hypothes.is][Hypothes.is]] highlights and annotations
|
|
|
|
#+begin_src python
|
|
class hypothesis:
|
|
'''
|
|
Uses [[https://github.com/karlicoss/hypexport][hypexport]] outputs
|
|
'''
|
|
|
|
# paths[s]/glob to the exported JSON data
|
|
export_path: Paths
|
|
|
|
# path to a local clone of hypexport
|
|
# alternatively, you can put the repository (or a symlink) in $MY_CONFIG/my/config/repos/hypexport
|
|
hypexport : Optional[PathIsh] = None
|
|
#+end_src
|
|
** [[file:../my/reddit.py][my.reddit]]
|
|
|
|
Reddit data: saved items/comments/upvotes/etc.
|
|
|
|
#+begin_src python
|
|
class reddit:
|
|
'''
|
|
Uses [[https://github.com/karlicoss/rexport][rexport]] output.
|
|
'''
|
|
|
|
# path[s]/glob to the exported JSON data
|
|
export_path: Paths
|
|
|
|
# path to a local clone of rexport
|
|
# alternatively, you can put the repository (or a symlink) in $MY_CONFIG/my/config/repos/rexport
|
|
rexport : Optional[PathIsh] = None
|
|
#+end_src
|
|
** [[file:../my/pocket.py][my.pocket]]
|
|
|
|
[[https://getpocket.com][Pocket]] bookmarks and highlights
|
|
|
|
#+begin_src python
|
|
class pocket:
|
|
'''
|
|
Uses [[https://github.com/karlicoss/pockexport][pockexport]] outputs
|
|
'''
|
|
|
|
# paths[s]/glob to the exported JSON data
|
|
export_path: Paths
|
|
|
|
# path to a local clone of pockexport
|
|
# alternatively, you can put the repository (or a symlink) in $MY_CONFIG/my/config/repos/pockexport
|
|
pockexport : Optional[PathIsh] = None
|
|
#+end_src
|
|
** [[file:../my/twitter/twint.py][my.twitter.twint]]
|
|
|
|
Twitter data (tweets and favorites).
|
|
|
|
Uses [[https://github.com/twintproject/twint][Twint]] data export.
|
|
|
|
Requirements: =pip3 install --user dataset=
|
|
|
|
#+begin_src python
|
|
class twint:
|
|
export_path: Paths # path[s]/glob to the twint Sqlite database
|
|
#+end_src
|
|
** [[file:../my/twitter/archive.py][my.twitter.archive]]
|
|
|
|
Twitter data (uses [[https://help.twitter.com/en/managing-your-account/how-to-download-your-twitter-archive][official twitter archive export]])
|
|
|
|
#+begin_src python
|
|
class twitter_archive:
|
|
export_path: Paths # path[s]/glob to the twitter archive takeout
|
|
#+end_src
|
|
** [[file:../my/lastfm][my.lastfm]]
|
|
|
|
Last.fm scrobbles
|
|
|
|
#+begin_src python
|
|
class lastfm:
|
|
"""
|
|
Uses [[https://github.com/karlicoss/lastfm-backup][lastfm-backup]] outputs
|
|
"""
|
|
export_path: Paths
|
|
#+end_src
|
|
** [[file:../my/reading/polar.py][my.reading.polar]]
|
|
|
|
[[https://github.com/burtonator/polar-bookshelf][Polar]] articles and highlights
|
|
|
|
#+begin_src python
|
|
class polar:
|
|
'''
|
|
Polar config is optional, you only need it if you want to specify custom 'polar_dir'
|
|
'''
|
|
polar_dir: PathIsh = Path('~/.polar').expanduser()
|
|
defensive: bool = True # pass False if you want it to fail faster on errors (useful for debugging)
|
|
#+end_src
|
|
** [[file:../my/instapaper.py][my.instapaper]]
|
|
|
|
[[https://www.instapaper.com][Instapaper]] bookmarks, highlights and annotations
|
|
|
|
#+begin_src python
|
|
class instapaper:
|
|
'''
|
|
Uses [[https://github.com/karlicoss/instapexport][instapexport]] outputs.
|
|
'''
|
|
# path[s]/glob to the exported JSON data
|
|
export_path : Paths
|
|
|
|
# path to a local clone of instapexport
|
|
# alternatively, you can put the repository (or a symlink) in $MY_CONFIG/my/config/repos/instapexport
|
|
instapexport: Optional[PathIsh] = None
|
|
#+end_src
|