From 810fe218393491749121fe29f4dc798f7ca9a117 Mon Sep 17 00:00:00 2001
From: Dima Gerasimov <karlicoss@gmail.com>
Date: Fri, 24 Apr 2020 16:35:20 +0100
Subject: [PATCH] attempt to use xmllint to speed up takeout parsing

---
 tests/takeout.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/tests/takeout.py b/tests/takeout.py
index bbe6271..6f7c8d8 100644
--- a/tests/takeout.py
+++ b/tests/takeout.py
@@ -37,3 +37,24 @@ def test_parser():
         dd = fo.read().decode('utf8')
         parser.feed(dd)
     print(len(results))
+
+
+def parse_takeout_xmllint(data: str):
+    # without xmllint (splitting by '<div class="content-cell' -- 0.68 secs)
+    # with xmllint -- 2 seconds
+    # using html.parser -- 4 seconds (+ all the parsing etc)
+    # not *that* much opportunity to speedup I guess
+    # the only downside is that html.parser isn't iterative.. might be able to hack with some iternal hacks?
+    # wonder what's the bottleneck..
+    #
+    from subprocess import Popen, PIPE, run
+    from more_itertools import split_before
+    res = run(
+        ['xmllint', '--html', '--xpath', '//div[contains(@class, "content-cell")]', '-'],
+        input=data.encode('utf8'),
+        check=True,
+        stdout=PIPE,
+    )
+    out = res.stdout.decode('utf8')
+    # out = data
+    return out.split('<div class="content-cell')