From f64d11bcb5fee53743cab9add2790c3c2de231d8 Mon Sep 17 00:00:00 2001 From: schneefux Date: Mon, 26 May 2014 14:21:46 -0400 Subject: fixed vocabcompiler after init update --- boot/vocabcompiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'boot') diff --git a/boot/vocabcompiler.py b/boot/vocabcompiler.py index d5455d3..70c1ea3 100644 --- a/boot/vocabcompiler.py +++ b/boot/vocabcompiler.py @@ -17,7 +17,7 @@ def compile(): Gets the words and creates the dictionary """ - m = dir(modules) + m = [name for _, name, _ in pkgutil.iter_modules(['modules'])] words = [] for module_name in m: -- cgit v1.3.1 From a24021d3ea2d2cab18cbb2bd1060c66d0a4ccc03 Mon Sep 17 00:00:00 2001 From: schneefux Date: Mon, 26 May 2014 14:40:18 -0400 Subject: complete vocabcompiler fix --- boot/vocabcompiler.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'boot') diff --git a/boot/vocabcompiler.py b/boot/vocabcompiler.py index 70c1ea3..72691b0 100644 --- a/boot/vocabcompiler.py +++ b/boot/vocabcompiler.py @@ -5,9 +5,14 @@ import os import sys import pkgutil +import glob lib_path = os.path.abspath('../client') +mod_path = os.path.abspath('../client/modules/') + sys.path.append(lib_path) +sys.path.append(mod_path) + import modules import g2p @@ -17,12 +22,13 @@ def compile(): Gets the words and creates the dictionary """ - m = [name for _, name, _ in pkgutil.iter_modules(['modules'])] + m = [os.path.basename(f)[:-3] for f in glob.glob(os.path.dirname("../client/modules/")+"/*.py")] words = [] for module_name in m: try: - eval("words.extend(modules.%s.WORDS)" % module_name) + exec("import %s" % module_name) + eval("words.extend(%s.WORDS)" % module_name) except: pass # module probably doesn't have the property -- cgit v1.3.1 From cb090a02a9c168abfb9d5b4c01111893bc5a7304 Mon Sep 17 00:00:00 2001 From: schneefux Date: Mon, 26 May 2014 14:40:48 -0400 Subject: removed pkgutil --- boot/vocabcompiler.py | 1 - 1 file changed, 1 deletion(-) (limited to 'boot') diff --git a/boot/vocabcompiler.py b/boot/vocabcompiler.py index 72691b0..d8c2295 100644 --- a/boot/vocabcompiler.py +++ b/boot/vocabcompiler.py @@ -4,7 +4,6 @@ import os import sys -import pkgutil import glob lib_path = os.path.abspath('../client') -- cgit v1.3.1 From 64de5f735cf0dbdf1a17903b535a1da915d39a6f Mon Sep 17 00:00:00 2001 From: schneefux Date: Mon, 26 May 2014 14:15:35 -0700 Subject: removed unused imports --- boot/boot.py | 1 - boot/broadcast.txt | 2 +- boot/connect.txt | 4 ++-- boot/vocabcompiler.py | 1 - boot/wifi.py | 1 - 5 files changed, 3 insertions(+), 6 deletions(-) (limited to 'boot') diff --git a/boot/boot.py b/boot/boot.py index 658748a..56bcc7e 100755 --- a/boot/boot.py +++ b/boot/boot.py @@ -2,7 +2,6 @@ import os, json import urllib2 -import subprocess import yaml from wifi import * diff --git a/boot/broadcast.txt b/boot/broadcast.txt index 36942c4..ac01451 100755 --- a/boot/broadcast.txt +++ b/boot/broadcast.txt @@ -1,7 +1,7 @@ auto lo iface lo inet loopback iface eth0 inet dhcp - + auto wlan0 iface wlan0 inet static address 192.168.1.1 diff --git a/boot/connect.txt b/boot/connect.txt index b30d831..91d2b0c 100644 --- a/boot/connect.txt +++ b/boot/connect.txt @@ -2,8 +2,8 @@ auto lo iface lo inet loopback iface eth0 inet dhcp - -allow-hotplug wlan0 + +allow-hotplug wlan0 auto wlan0 iface wlan0 inet dhcp wpa-ssid "{{ SSID }}" diff --git a/boot/vocabcompiler.py b/boot/vocabcompiler.py index d8c2295..4daaa26 100644 --- a/boot/vocabcompiler.py +++ b/boot/vocabcompiler.py @@ -12,7 +12,6 @@ mod_path = os.path.abspath('../client/modules/') sys.path.append(lib_path) sys.path.append(mod_path) -import modules import g2p diff --git a/boot/wifi.py b/boot/wifi.py index 7e2fa43..3170ecf 100755 --- a/boot/wifi.py +++ b/boot/wifi.py @@ -15,7 +15,6 @@ class Wifi: if "Address" in line: - address = line[29:-1] name = lines[index + 1].split("\"")[1] self.access_points.append(name) -- cgit v1.3.1 From 2edb48484aa0736e78cb96fdb87a0f39719eb7d6 Mon Sep 17 00:00:00 2001 From: schneefux Date: Mon, 26 May 2014 14:20:10 -0700 Subject: modularized lm compilation --- boot/vocabcompiler.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) (limited to 'boot') diff --git a/boot/vocabcompiler.py b/boot/vocabcompiler.py index 4daaa26..d9feba1 100644 --- a/boot/vocabcompiler.py +++ b/boot/vocabcompiler.py @@ -15,12 +15,29 @@ sys.path.append(mod_path) import g2p +def text2lm(in_filename, out_filename): + """Wrapper around the language model compilation tools""" + def text2idngram(in_filename, out_filename): + cmd = "text2idngram -vocab %s < %s -idngram temp.idngram" % (out_filename, + in_filename) + os.system(cmd) + + def idngram2lm(in_filename, out_filename): + cmd = "idngram2lm -idngram temp.idngram -vocab ../client/sentences.txt -arpa %s" % ( + in_filename, out_filename) + os.system(cmd) + + text2idngram(in_filename, in_filename) + idngram2lm(in_filename, out_filename) + + def compile(): """ Gets the words and creates the dictionary """ - m = [os.path.basename(f)[:-3] for f in glob.glob(os.path.dirname("../client/modules/")+"/*.py")] + m = [os.path.basename(f)[:-3] + for f in glob.glob(os.path.dirname("../client/modules/") + "/*.py")] words = [] for module_name in m: @@ -33,7 +50,7 @@ def compile(): words = list(set(words)) # for spotify module - words.extend(["MUSIC","SPOTIFY"]) + words.extend(["MUSIC", "SPOTIFY"]) # create the dictionary pronounced = g2p.translateWords(words) @@ -50,7 +67,4 @@ def compile(): f.close() # make language model - os.system( - "text2idngram -vocab ../client/sentences.txt < ../client/sentences.txt -idngram temp.idngram") - os.system( - "idngram2lm -idngram temp.idngram -vocab ../client/sentences.txt -arpa ../client/languagemodel.lm") + text2lm("../client/sentences.txt", "../client/languagemodel.lm") -- cgit v1.3.1 From b0c595cc234f490bf41748f80b9b4523b88323be Mon Sep 17 00:00:00 2001 From: schneefux Date: Mon, 26 May 2014 14:20:43 -0700 Subject: edit docstring --- boot/vocabcompiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'boot') diff --git a/boot/vocabcompiler.py b/boot/vocabcompiler.py index d9feba1..5c00675 100644 --- a/boot/vocabcompiler.py +++ b/boot/vocabcompiler.py @@ -1,5 +1,5 @@ """ - This iterates over all the WORDS variables in the modules and creates a dictionary that the client program will use + Iterates over all the WORDS variables in the modules and creates a dictionary for the client. """ import os -- cgit v1.3.1 From 79b5d466202ecfeaae3d9472ed89d4f94ab66450 Mon Sep 17 00:00:00 2001 From: schneefux Date: Mon, 26 May 2014 14:49:45 -0700 Subject: vocabcompiler now takes arguments (useful for testing) --- boot/boot.py | 4 ++-- boot/vocabcompiler.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'boot') diff --git a/boot/boot.py b/boot/boot.py index 56bcc7e..b33fd34 100755 --- a/boot/boot.py +++ b/boot/boot.py @@ -22,7 +22,7 @@ def configure(): print "CONNECTED TO INTERNET" print "COMPILING DICTIONARY" - vocabcompiler.compile() + vocabcompiler.compile("../client/sentences.txt", "../client/dictionary.dic", "../client/languagemodel.lm") print "STARTING CLIENT PROGRAM" @@ -49,7 +49,7 @@ def configure(): print "CONNECTED TO INTERNET" print "COMPILING DICTIONARY" - vocabcompiler.compile() + vocabcompiler.compile("../client/sentences.txt", "../client/dictionary.dic", "../client/languagemodel.lm") print "STARTING CLIENT PROGRAM" diff --git a/boot/vocabcompiler.py b/boot/vocabcompiler.py index 5c00675..a3aaf66 100644 --- a/boot/vocabcompiler.py +++ b/boot/vocabcompiler.py @@ -23,7 +23,7 @@ def text2lm(in_filename, out_filename): os.system(cmd) def idngram2lm(in_filename, out_filename): - cmd = "idngram2lm -idngram temp.idngram -vocab ../client/sentences.txt -arpa %s" % ( + cmd = "idngram2lm -idngram temp.idngram -vocab %s -arpa %s" % ( in_filename, out_filename) os.system(cmd) @@ -31,7 +31,7 @@ def text2lm(in_filename, out_filename): idngram2lm(in_filename, out_filename) -def compile(): +def compile(sentences, dictionary, languagemodel): """ Gets the words and creates the dictionary """ @@ -57,14 +57,14 @@ def compile(): zipped = zip(words, pronounced) lines = ["%s %s" % (x, y) for x, y in zipped] - with open("../client/dictionary.dic", "w") as f: + with open(dictionary, "w") as f: f.write("\n".join(lines) + "\n") # create the language model - with open("../client/sentences.txt", "w") as f: + with open(sentences, "w") as f: f.write("\n".join(words) + "\n") f.write(" \n \n") f.close() # make language model - text2lm("../client/sentences.txt", "../client/languagemodel.lm") + text2lm(sentences, languagemodel) -- cgit v1.3.1 From 546d050fcb1c114ad828593a4133f90f20966268 Mon Sep 17 00:00:00 2001 From: schneefux Date: Mon, 26 May 2014 23:43:33 -0700 Subject: basic unit test for vocabcompiler --- boot/test.py | 36 ++++++++++++++++++++++++++++++++++++ boot/vocabcompiler.py | 5 +++-- 2 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 boot/test.py (limited to 'boot') diff --git a/boot/test.py b/boot/test.py new file mode 100644 index 0000000..1c03a33 --- /dev/null +++ b/boot/test.py @@ -0,0 +1,36 @@ +import os +import sys +import unittest +from mock import patch +import vocabcompiler + +lib_path = os.path.abspath('../client') +mod_path = os.path.abspath('../client/modules/') + +sys.path.append(lib_path) +sys.path.append(mod_path) + +import g2p + +class TestVocabCompiler(unittest.TestCase): + + def testWordExtraction(self): + sentences = "temp_sentences.txt" + dictionary = "temp_dictionary.dic" + languagemodel = "temp_languagemodel.lm" + + words = ['HACKER', 'LIFE', 'FACEBOOK', 'THIRD', 'NO', 'JOKE', 'NOTIFICATION', 'MEANING', 'TIME', 'TODAY', 'SECOND', 'BIRTHDAY', 'KNOCK KNOCK', 'INBOX', 'OF', 'NEWS', 'YES', 'TOMORROW', 'EMAIL', 'WEATHER', 'FIRST', 'MUSIC', 'SPOTIFY'] + + with patch.object(g2p, 'translateWords') as translateWords: + with patch.object(vocabcompiler, 'text2lm') as text2lm: + vocabcompiler.compile(sentences, dictionary, languagemodel) + + # 'words' is appended with ['MUSIC', 'SPOTIFY'] + # so must be > 2 to have received WORDS from modules + translateWords.assert_called_once_with(words) + self.assertTrue(text2lm.called) + os.remove(sentences) + os.remove(dictionary) + +if __name__ == '__main__': + unittest.main() diff --git a/boot/vocabcompiler.py b/boot/vocabcompiler.py index a3aaf66..33fb0a8 100644 --- a/boot/vocabcompiler.py +++ b/boot/vocabcompiler.py @@ -48,6 +48,7 @@ def compile(sentences, dictionary, languagemodel): pass # module probably doesn't have the property words = list(set(words)) + print words # for spotify module words.extend(["MUSIC", "SPOTIFY"]) @@ -57,11 +58,11 @@ def compile(sentences, dictionary, languagemodel): zipped = zip(words, pronounced) lines = ["%s %s" % (x, y) for x, y in zipped] - with open(dictionary, "w") as f: + with open(dictionary, "w+") as f: f.write("\n".join(lines) + "\n") # create the language model - with open(sentences, "w") as f: + with open(sentences, "w+") as f: f.write("\n".join(words) + "\n") f.write(" \n \n") f.close() -- cgit v1.3.1 From 5deb1799cefe9351b90d3457898990d454d4a4dc Mon Sep 17 00:00:00 2001 From: schneefux Date: Mon, 26 May 2014 23:44:50 -0700 Subject: finalized test --- boot/test.py | 8 +++++++- boot/vocabcompiler.py | 1 - 2 files changed, 7 insertions(+), 2 deletions(-) (limited to 'boot') diff --git a/boot/test.py b/boot/test.py index 1c03a33..c6bd3ff 100644 --- a/boot/test.py +++ b/boot/test.py @@ -12,6 +12,7 @@ sys.path.append(mod_path) import g2p + class TestVocabCompiler(unittest.TestCase): def testWordExtraction(self): @@ -19,7 +20,12 @@ class TestVocabCompiler(unittest.TestCase): dictionary = "temp_dictionary.dic" languagemodel = "temp_languagemodel.lm" - words = ['HACKER', 'LIFE', 'FACEBOOK', 'THIRD', 'NO', 'JOKE', 'NOTIFICATION', 'MEANING', 'TIME', 'TODAY', 'SECOND', 'BIRTHDAY', 'KNOCK KNOCK', 'INBOX', 'OF', 'NEWS', 'YES', 'TOMORROW', 'EMAIL', 'WEATHER', 'FIRST', 'MUSIC', 'SPOTIFY'] + words = [ + 'HACKER', 'LIFE', 'FACEBOOK', 'THIRD', 'NO', 'JOKE', + 'NOTIFICATION', 'MEANING', 'TIME', 'TODAY', 'SECOND', + 'BIRTHDAY', 'KNOCK KNOCK', 'INBOX', 'OF', 'NEWS', 'YES', + 'TOMORROW', 'EMAIL', 'WEATHER', 'FIRST', 'MUSIC', 'SPOTIFY' + ] with patch.object(g2p, 'translateWords') as translateWords: with patch.object(vocabcompiler, 'text2lm') as text2lm: diff --git a/boot/vocabcompiler.py b/boot/vocabcompiler.py index 33fb0a8..1d2a51d 100644 --- a/boot/vocabcompiler.py +++ b/boot/vocabcompiler.py @@ -48,7 +48,6 @@ def compile(sentences, dictionary, languagemodel): pass # module probably doesn't have the property words = list(set(words)) - print words # for spotify module words.extend(["MUSIC", "SPOTIFY"]) -- cgit v1.3.1