From bc93e6684e1acf91e1d532c05db08d24433a5b0c Mon Sep 17 00:00:00 2001 From: fz0x1 Date: Thu, 16 Jan 2025 16:57:07 +0100 Subject: [PATCH] 20250116.1737043027 --- global/.zshrc | 1 + global/.zshrc-secrets | Bin 1300 -> 1300 bytes global/scripts/bin/memos2jrnl.py | 295 +++++++++++++++++++++++++++++++ 3 files changed, 296 insertions(+) create mode 100755 global/scripts/bin/memos2jrnl.py diff --git a/global/.zshrc b/global/.zshrc index 33f3c1a..f6a359a 100644 --- a/global/.zshrc +++ b/global/.zshrc @@ -117,6 +117,7 @@ alias em="emacsclient -c" alias relmacs="doom sync" ## diary alias di="diary.sh" +alias m2jrnl="memos2jrnl.py export default diaryf" # bindkeys ## autosuggest diff --git a/global/.zshrc-secrets b/global/.zshrc-secrets index 3ed72879b24c95ff4994206269cc91c82d12f307..a9040ccd8b24742ccf83da2cbc382e559331ab82 100644 GIT binary patch literal 1300 zcmV+v1?&0%M@dveQdv+`0OfwV811Wl@7R!w-KxR$SB&i4T-yUe+$kNEcCf2wbxq%1 z1>B4h!ZBv~+Xd_o)M z5FO#r1iG{Q!>=8b0ef85L6E*J`0s|jr+oJUNh>EhK;Bj?U1bm;h2RtP09wxc9OrI@ zn@Wd@f!>n5bDtrK(wK1NiJt&sz>Ex%{;9^tQjWwcHh<8Wlnm)h&9?2)&qhN zt|TbnpqU;0GliY{#dBhsQD{}L^x2?hAsxdP?X|sTvku(E zz~^EHD;-){nZ=$lq(BxcJo{{x!iK%KZNBhFHosh46c@j#Xrl_!hvucy)AHD=xzE8e z2X|Hi&H12WgD(wbpS4aiCG(HRu1A}4nPju?H?Q}xY-Xs{m?*SJ`2g_1-DLR~7;o8U}k{5^0yMZ4(uv1 zC1semY=LVKwNcUQv}K4CW2y7YE`O;qXF^5}`x9h!vQ~vw_67E0w|LDUbpf5n)d(I3X>1IIB=xeVOIA{CUDVbGpy85Fu$cnS@g2-D#ss)nsBKm<&4wlg2tfP|2kM z;&AZCWQg+gOAl8*Qo&>msYZ~|2+T?~%w#c;5yzW;02O688vG_*0f#VoMQyVdIEqU? zpfCP+%JiyYq$5)#Q`LS^NT|;gb$A797(-c=LMlGqN67DRT%&@7|GL_fZ>E&=eZcU7 z?rjLouXq2|+`7BF$T>1(4SvH~(fX)|-yoR0mzgy?E;`8rYHa-0NvvdU88hbi#b@@C zLDAn#CERR!bBvniaLi^|0S-5+*vIAt*E9BsPJzFTO3gLSTg6;S5B3GqoK==}C)80f ze7Zkv0Cv+!JtBK}!UyM!IDyNl0~0~R;i4lWTf9;E0W^#&>e9gYxUvzE#f(KwCdsmp z#2QAs_NtC3u5|n(MX!maKb=fqC;OmWFiE<=#jumLfr3)guedZtP$;W+**FC1FOuID?W2mlp7Y&-<~J|y`1Kv8$;EFq{}zu$ zC2MUNt`{*?xIEVnoS`QKJe9{raFkjV2LIg#_So`}BOi7r2fAB6DvR z$QomUZ@;aYGNiyLN@UyvtLV^8z%o8e`7_^ztRQ+qO@`fl(|tP68_$C2QA5wD2I94C zpeog+B2p-x7aiE+d>9cQ9!g#Mb{j2P;oQAmw$Ou5iY4x3s_MMbu`khtRG?nGeeME> zIN8-`K-b6ZeIl z)(*It8FGtP+%reO7VzL@ycKx&_OJ*PbfH zJPRa2KKcfCWux(MT|#|()hYj;QP(t=v(s9SynleV2|%6&EfnE4g(LEhUJ|SZ=G=Kb z0;;Z?V2W3&c9NO7CJ;S>fZ6>(qt7Cse}P5Jp1wqNUGY=Em zb%p7Z7$Vz#MAQY*X=JK_;Rmugy_(jo*4!KXY348#60BMI4o9fKc8TC!Dz0Pm(F#^l zXux*Vr0dM*P4x*I$psYC@3OiWuF>16?G^1h8CA;X1o-|lw$f*%ni}f}fm|1%MnhkP zM!>C=?*G~x48ktI8X;j1oW)}?=n-8*iHqNF+qX+{{x64_9yOqfhc7m5*fzp@f~lJ^ z=V619sX_I6a4E{S3xw-_lQ{i_;lEu8oI+r19#9cor)IaVMP}6bV1;Uc0bOHdMr)#N z!wTgPV1LZwg2Lfb7b+)3X64Qj@JdC`M`O<}K^})Zd`{ey*YKrJHg#^|Fbj1SK^vgw zsetlF%=Gag!q{Bs->u;Rt`^)e5}jK@!H;iJ?mx~@2GABKdGxKHA@h<;Jp*BIa$Kre zR|G#VNxXXD33De5A1MbT$u_mp1GbO0fO4o_6e2<#hDNAMr~xkMrjRpoSznyL39;nF zjLGWkbgAz7I|qEhW>J;)o>%I;0T4^Lw)3+X=PwBApIAmubmkJN#*XF`l*CSZf!qDS~IDDRJ~|I?k>1ikGxlam=A6hj&=7Bvoy8^|88FmaLk`)@Zn46HgSlv_Vh!%TR-g9LkedsU`5^;XQs&&sf z24`zBdNzBj62k3YE3njn`X(y-ySLXxqRU553=VXtc*{=ulLEm7_01kKBm2m@JmdU6 z-kJfOu|bNQEEZxOvu$gdn;KdZD^7vse72juQ~T&{eEn+tV8X_B%eaATZ6T{oU;St> zj}cDF8^I*V-$#}P0DgKT9q;leyqV^*t4xPShn$\s+(.+)" + matches = dict(re.findall(pattern, result.stdout.strip())) + diary_path = matches.get(name) + + if not diary_path or not Path(diary_path).exists(): + sys.exit(f"Diary '{name}' not found or path does not exist.") + return diary_path + + +def find_closest_entry(data, target_time): + """Find the entry closest to the target timestamp.""" + target_timestamp = int( + ( + datetime.strptime(target_time, "%Y-%m-%dT%H:%M:%SZ") + timedelta(hours=TZ) + ).timestamp() + ) + return min( + (entry for entry in data if "tst" in entry), + key=lambda e: abs(target_timestamp - e["tst"]), + default=None, + ), target_timestamp + + +def convert_diary_date(date_str): + """Convert date string to formatted diary date.""" + try: + dt = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%SZ") + timedelta(hours=TZ) + return dt.strftime("%d %b %Y at %H:%M:%S:") + except ValueError: + return None + + +def fetch_data(url, headers, data=None): + """Fetch data from the specified URL.""" + data = data or {} + method = "POST" if data else "GET" + encoded_data = urllib.parse.urlencode(data).encode("utf-8") if data else None + + req = urllib.request.Request( + url=url, headers=headers, data=encoded_data, method=method + ) + req.add_header("Content-Type", "application/x-www-form-urlencoded") + + with urllib.request.urlopen(req) as response: + if response.status != 200: + sys.exit(f"Error: HTTP {response.status}") + return json.loads(response.read().decode("utf-8")) + + +def delete_entity(url, headers): + """Delete an entity using the DELETE method.""" + req = urllib.request.Request(url=url, headers=headers, method="DELETE") + with urllib.request.urlopen(req) as response: + if response.status != 200: + sys.exit(f"Error while deleting entity: HTTP {response.status}") + + +def db_connection(diary_path: Path): + """Establish a database connection and enable foreign keys.""" + conn = sqlite3.connect(diary_path / DB_NAME) + conn.execute("PRAGMA foreign_keys = ON;") + return conn + + +def initialize_db(conn: sqlite3.Connection): + """Initialize the database with necessary tables.""" + with conn: + conn.executescript( + """ + CREATE TABLE IF NOT EXISTS metadata ( + id INTEGER PRIMARY KEY, + unixtime INTEGER NOT NULL, + type INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS weather ( + id INTEGER PRIMARY KEY, + temp INTEGER NOT NULL, + temp_like INTEGER NOT NULL, + sunrise INTEGER NOT NULL, + sunset INTEGER NOT NULL, + icon TEXT NOT NULL DEFAULT 'none', + metadata_id INTEGER NOT NULL, + FOREIGN KEY (metadata_id) REFERENCES metadata (id) ON DELETE CASCADE + ); + CREATE TABLE IF NOT EXISTS location ( + id INTEGER PRIMARY KEY, + city TEXT NOT NULL, + lon TEXT NOT NULL, + lat TEXT NOT NULL, + tz TEXT NOT NULL, + metadata_id INTEGER NOT NULL, + FOREIGN KEY (metadata_id) REFERENCES metadata (id) ON DELETE CASCADE + ); + """ + ) + + +def insert_metadata( + conn: sqlite3.Connection, unixtime: int, metadata_type: MetadataType +): + """Insert metadata and return its ID.""" + try: + cursor = conn.cursor() + cursor.execute( + "INSERT INTO metadata(unixtime, type) VALUES(?, ?)", + [unixtime, metadata_type], + ) + conn.commit() + return cursor.lastrowid + except sqlite3.DatabaseError as e: + print(f"Error inserting metadata: {e}") + conn.rollback() + raise + + +def insert_weather(weather: dict, conn: sqlite3.Connection, metadata_id: int): + """Insert weather data into the database.""" + cursor = conn.cursor() + weather = weather[0] + cursor.execute( + """ + INSERT INTO weather(temp, temp_like, sunrise, sunset, icon, metadata_id) + VALUES(?, ?, ?, ?, ?, ?) + """, + [ + weather["temp"], + weather["feels_like"], + weather["sunrise"], + weather["sunset"], + weather["weather"][0]["icon"], + metadata_id, + ], + ) + conn.commit() + + +def insert_location(location: dict, conn: sqlite3.Connection, metadata_id: int): + """Insert location data into the database.""" + cursor = conn.cursor() + cursor.execute( + """ + INSERT INTO location(city, lon, lat, tz, metadata_id) + VALUES(?, ?, ?, ?, ?) + """, + [ + location["locality"], + location["lon"], + location["lat"], + location["tzname"], + metadata_id, + ], + ) + conn.commit() + + +def export(): + """Main export function.""" + memo_token = os.getenv("MEMOS_TOKEN") + memo_url = os.getenv("MEMOS_URL") + openweathermap_api_key = os.getenv("OPENWEATHER_APIKEY") + owntracks_creds = os.getenv("OWNTRACKS_CREDS").encode() + owntracks_url = os.getenv("OWNTRACKS_URL") + geo_user, geo_device = os.getenv("OWNTRACKS_PARAMS").split(",") + + if not memo_token or not memo_url: + sys.exit("Missing MEMOS_TOKEN or MEMOS_URL environment variables.") + + if len(sys.argv) < 4 or sys.argv[1] != "export": + sys.exit("Usage: script.py export ") + + diary_name = sys.argv[2] + tag = sys.argv[3] + diary_full_path = Path(get_diary_path_by_name(diary_name)) + diary_path = diary_full_path.parent + + conn = None + try: + conn = db_connection(diary_path) + initialize_db(conn) + + headers = {"Cookie": f"memos.access-token={memo_token}"} + query_string = urllib.parse.urlencode( + {"filter": f"creator=='users/1'&&tag_search==['{tag}']"} + ) + data = fetch_data(f"{memo_url}api/v1/memos?{query_string}", headers) + + memos = data.get("memos", []) + if not memos: + sys.exit("No memos found.") + + if ( + input(f"There are {len(memos)} memos. Export them all? (Y/N): ") + .strip() + .upper() + != "Y" + ): + sys.exit("Export canceled.") + + for memo in memos: + create_time = memo["createTime"] + + # Check if entry exists + result = subprocess.run( + ["jrnl", "-on", create_time, "--format", "json"], + capture_output=True, + text=True, + ) + if result.stdout.strip(): + print(f"Skipping existing memo: {memo['name']}") + continue + + content = shlex.quote(memo["content"].replace(f"#{tag}", "").strip()) + result = subprocess.run( + f'printf "%s %s" "{convert_diary_date(memo["createTime"])}" {content} | jrnl', + shell=True, + capture_output=True, + text=True, + ) + if result.stderr: + print(f"There are some errors: {result.stderr}") + sys.exit(1) + + geo_url = f"https://{owntracks_url}/api/0/locations" + geo_headers = { + "Authorization": f"Basic {base64.b64encode(owntracks_creds).decode()}" + } + geo_response = fetch_data( + geo_url, + geo_headers, + data={ + "from": "1970-01-01", + "limit": 20, + "device": geo_device, + "user": geo_user, + }, + ) + closest_entry, create_time_unix = find_closest_entry( + geo_response.get("data", []), create_time + ) + print(f"Closest geo entry: {closest_entry}") + + metadata_id = insert_metadata(conn, create_time_unix, MetadataType.LOCATION) + insert_location(closest_entry, conn, metadata_id) + + weather_response = fetch_data( + f"https://api.openweathermap.org/data/3.0/onecall/timemachine?lat={closest_entry['lat']}&lon={closest_entry['lon']}&dt={create_time_unix}&appid={openweathermap_api_key}&units=metric", + headers={}, + ) + print(f"Weather: {create_time_unix} - {weather_response}") + metadata_id = insert_metadata(conn, create_time_unix, MetadataType.WEATHER) + insert_weather(weather_response["data"], conn, metadata_id) + + delete_entity(f"{memo_url}/api/v1/{memo['name']}", headers) + + except Exception as e: + print(f"An error occurred: {e}") + finally: + if conn: + conn.close() + print("Database connection closed.") + + +if __name__ == "__main__": + export()