write to csvHEADmaster

1 files changed, 40 insertions, 17 deletions

diff --git a/ankivoc.py b/ankivoc.py
index 00ea9b9..bca077f 100644
--- a/ankivoc.py
+++ b/ankivoc.py
@@ -2,18 +2,24 @@
 
 from bs4 import BeautifulSoup, Comment
 import requests
-import sys
 import re
+import csv
+
 
 def lookup(word):
     r = requests.get("https://www.oxforddictionaries.com/definition/english/" +
-                     word)
+                     word.replace(" ", "-"))
     soup = BeautifulSoup(r.text, "html.parser")
+    entry_count = 0
 
     entry = "<meta charset=\"utf-8\">"
 
     for res in soup.find_all("section", class_="senseGroup"):
-        entry += res.find_all("span", class_="partOfSpeech")[0].get_text()
+        parts = res.find_all("span", class_="partOfSpeech")
+        if not len(parts):
+            continue
+        entry += parts[0].get_text()
+        entry_count += 1
 
         # remove comments
         for el in res(text=lambda text: isinstance(text, Comment)):
@@ -31,31 +37,27 @@ def lookup(word):
                 for syno in el.find_all("div"):
                     syno.name = "li"
 
-                #el.insert_before(soup.new_tag("br"))
-
             # simplify structure
             for el in sense.find_all("span") + \
                     sense.find_all("div"):
-                # skip "more" tags
-                if "onclick" not in el.attrs:
-                    el.unwrap()
+                el.unwrap()
 
             # remove unused classes
             for el in sense.find_all():
                 del el["class"]
+                del el["href"]
             del sense["class"]
 
             # hide examples
-            for el in sense.find_all("li"):
-                el["style"] = "font-size: 75%;"
+            for el in sense.find_all("ul"):
+                el["style"] = "font-size: 75%; list-style-type: none;"
 
-            entry += str(sense) #.prettify(formatter="html")
-            #entry += "<br/>"
+            entry += str(sense)
         entry += "<br/>"
 
     # ugly hack. Why is this necessary:
-    #- for el in sense.find_all("a", class_="moreInformationExemples"):
-    #- AttributeError: 'NoneType' object has no attribute 'next_element'
+    # - for el in sense.find_all("a", class_="moreInformationExemples"):
+    # - AttributeError: 'NoneType' object has no attribute 'next_element'
     soup = BeautifulSoup(entry, "html.parser")
     for el in soup.find_all("li"):
         if "Get more examples" == el.get_text() or \
@@ -66,7 +68,28 @@ def lookup(word):
     text = str(soup)
     # another hack, I'm out of time
     text = re.sub(r"(\d)([A-Za-z])", r"\1 \2", text)
-    return text
+    return text, entry_count
+
+
+def main():
+    with open("words.txt", "r") as f:
+        with open("errors.txt", "a+") as err:
+            issues = []
+            words = f.read().split("\n")
+            words = list(set(words))  # remove dupes
+            w = csv.writer(open("cards.csv", "a+"))
+            for word in words:
+                if len(word) <= 1:
+                    continue
+                entry, number = lookup(word)
+                if entry and number > 0:
+                    print("found " + word)
+                    w.writerow([word, entry])
+                else:
+                    print("skipping " + word)
+                    issues.append(word)
+
+            err.write("\n".join(issues))
 
-#lookup("clerk")
-print(lookup("smudge"))
+if __name__ == "__main__":
+    main()