import json
from html.parser import HTMLParser
from pathlib import Path


ROOT = Path(__file__).resolve().parents[1]


class TextParser(HTMLParser):
    def __init__(self):
        super().__init__()
        self.depth = 0
        self.parts = []

    def handle_starttag(self, tag, attrs):
        if tag in ("script", "style"):
            self.depth += 1
        if tag in ("p", "div", "tr", "h1", "h2", "h3", "li"):
            self.parts.append("\n")

    def handle_endtag(self, tag):
        if tag in ("script", "style"):
            self.depth = max(0, self.depth - 1)
        if tag in ("p", "div", "tr", "li"):
            self.parts.append("\n")
        if tag in ("td", "th"):
            self.parts.append(" | ")

    def handle_data(self, data):
        if not self.depth:
            self.parts.append(data)


sources = json.loads((ROOT / "research" / "public_downloads.json").read_text(encoding="utf-8"))
for source in sources:
    if source["kind"] == "html" and source.get("file"):
        parser = TextParser()
        parser.feed((ROOT / source["file"]).read_text(encoding="utf-8", errors="replace"))
        text = "\n".join(line.strip() for line in "".join(parser.parts).splitlines() if line.strip())
        (ROOT / "research" / "extracted" / f"{source['id']}.txt").write_text(text, encoding="utf-8")
        print(source["id"], len(text))