1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
|
#!/usr/bin/env python3
from bs4 import BeautifulSoup, Comment
import requests
import re
import csv
def lookup(word):
r = requests.get("https://www.oxforddictionaries.com/definition/english/" +
word.replace(" ", "-"))
soup = BeautifulSoup(r.text, "html.parser")
entry_count = 0
entry = "<meta charset=\"utf-8\">"
for res in soup.find_all("section", class_="senseGroup"):
parts = res.find_all("span", class_="partOfSpeech")
if not len(parts):
continue
entry += parts[0].get_text()
entry_count += 1
# remove comments
for el in res(text=lambda text: isinstance(text, Comment)):
el.extract()
# remove annyoing link
for el in res.find_all("a", class_="moreInformationSynonyms") + \
res.find_all("a", class_="moreInformationExemples"):
el.extract()
for sense in res.find_all("div", class_="msDict"):
# transform div structure into 'examples'-like list
for el in sense.find_all(class_="entrySynList"):
el.name = "ul"
for syno in el.find_all("div"):
syno.name = "li"
# simplify structure
for el in sense.find_all("span") + \
sense.find_all("div"):
el.unwrap()
# remove unused classes
for el in sense.find_all():
del el["class"]
del el["href"]
del sense["class"]
# hide examples
for el in sense.find_all("ul"):
el["style"] = "font-size: 75%; list-style-type: none;"
entry += str(sense)
entry += "<br/>"
# ugly hack. Why is this necessary:
# - for el in sense.find_all("a", class_="moreInformationExemples"):
# - AttributeError: 'NoneType' object has no attribute 'next_element'
soup = BeautifulSoup(entry, "html.parser")
for el in soup.find_all("li"):
if "Get more examples" == el.get_text() or \
"View synonyms" in el.get_text() or \
el.get_text() == "":
el.extract()
text = str(soup)
# another hack, I'm out of time
text = re.sub(r"(\d)([A-Za-z])", r"\1 \2", text)
return text, entry_count
def main():
with open("words.txt", "r") as f:
with open("errors.txt", "a+") as err:
issues = []
words = f.read().split("\n")
words = list(set(words)) # remove dupes
w = csv.writer(open("cards.csv", "a+"))
for word in words:
if len(word) <= 1:
continue
entry, number = lookup(word)
if entry and number > 0:
print("found " + word)
w.writerow([word, entry])
else:
print("skipping " + word)
issues.append(word)
err.write("\n".join(issues))
if __name__ == "__main__":
main()
|
