#!/usr/bin/env python3

from bs4 import BeautifulSoup, Comment
import requests
import re
import csv


def lookup(word):
    r = requests.get("https://www.oxforddictionaries.com/definition/english/" +
                     word.replace(" ", "-"))
    soup = BeautifulSoup(r.text, "html.parser")
    entry_count = 0

    entry = "<meta charset=\"utf-8\">"

    for res in soup.find_all("section", class_="senseGroup"):
        parts = res.find_all("span", class_="partOfSpeech")
        if not len(parts):
            continue
        entry += parts[0].get_text()
        entry_count += 1

        # remove comments
        for el in res(text=lambda text: isinstance(text, Comment)):
            el.extract()

        # remove annyoing link
        for el in res.find_all("a", class_="moreInformationSynonyms") + \
                res.find_all("a", class_="moreInformationExemples"):
            el.extract()

        for sense in res.find_all("div", class_="msDict"):
            # transform div structure into 'examples'-like list
            for el in sense.find_all(class_="entrySynList"):
                el.name = "ul"
                for syno in el.find_all("div"):
                    syno.name = "li"

            # simplify structure
            for el in sense.find_all("span") + \
                    sense.find_all("div"):
                el.unwrap()

            # remove unused classes
            for el in sense.find_all():
                del el["class"]
                del el["href"]
            del sense["class"]

            # hide examples
            for el in sense.find_all("ul"):
                el["style"] = "font-size: 75%; list-style-type: none;"

            entry += str(sense)
        entry += "<br/>"

    # ugly hack. Why is this necessary:
    # - for el in sense.find_all("a", class_="moreInformationExemples"):
    # - AttributeError: 'NoneType' object has no attribute 'next_element'
    soup = BeautifulSoup(entry, "html.parser")
    for el in soup.find_all("li"):
        if "Get more examples" == el.get_text() or \
           "View synonyms" in el.get_text() or \
                el.get_text() == "":
            el.extract()

    text = str(soup)
    # another hack, I'm out of time
    text = re.sub(r"(\d)([A-Za-z])", r"\1 \2", text)
    return text, entry_count


def main():
    with open("words.txt", "r") as f:
        with open("errors.txt", "a+") as err:
            issues = []
            words = f.read().split("\n")
            words = list(set(words))  # remove dupes
            w = csv.writer(open("cards.csv", "a+"))
            for word in words:
                if len(word) <= 1:
                    continue
                entry, number = lookup(word)
                if entry and number > 0:
                    print("found " + word)
                    w.writerow([word, entry])
                else:
                    print("skipping " + word)
                    issues.append(word)

            err.write("\n".join(issues))

if __name__ == "__main__":
    main()