diff options
Diffstat (limited to 'client')
| -rw-r--r-- | client/brain.py | 21 | ||||
| -rw-r--r-- | client/conversation.py | 37 | ||||
| -rw-r--r-- | client/mic.py | 13 | ||||
| -rw-r--r-- | client/music.py | 6 | ||||
| -rw-r--r-- | client/musicmode.py | 3 | ||||
| -rw-r--r-- | client/speaker.py | 63 | ||||
| -rw-r--r-- | client/stt.py | 20 | ||||
| -rw-r--r-- | client/test.py | 21 | ||||
| -rw-r--r-- | client/tts.py | 360 |
9 files changed, 431 insertions, 113 deletions
diff --git a/client/brain.py b/client/brain.py index 1ed6db5..007b7fe 100644 --- a/client/brain.py +++ b/client/brain.py @@ -51,7 +51,7 @@ class Brain(object): modules.sort(key=lambda mod: mod.PRIORITY if hasattr(mod, 'PRIORITY') else 0, reverse=True) return modules - def query(self, text): + def query(self, texts): """ Passes user input to the appropriate module, testing it against each candidate module's isValid function. @@ -60,13 +60,14 @@ class Brain(object): text -- user input, typically speech, to be parsed by a module """ for module in self.modules: - if module.isValid(text): + for text in texts: - try: - module.handle(text, self.mic, self.profile) - break - except: - self._logger.error('Failed to execute module', exc_info=True) - self.mic.say( - "I'm sorry. I had some trouble with that operation. Please try again later.") - break + if module.isValid(text): + try: + module.handle(text, self.mic, self.profile) + return + except: + self._logger.error('Failed to execute module', exc_info=True) + self.mic.say( + "I'm sorry. I had some trouble with that operation. Please try again later.") + return diff --git a/client/conversation.py b/client/conversation.py index be9b8e9..8ec912c 100644 --- a/client/conversation.py +++ b/client/conversation.py @@ -14,28 +14,29 @@ class Conversation(object): self.brain = Brain(mic, profile) self.notifier = Notifier(profile) - def delegateInput(self, text): + def delegateInput(self, texts): """A wrapper for querying brain.""" # check if input is meant to start the music module - if any(x in text.upper() for x in ["SPOTIFY", "MUSIC"]): - # check if mpd client is running - try: - client = MPDClient() - client.timeout = None - client.idletimeout = None - client.connect("localhost", 6600) - except: - self.mic.say( - "I'm sorry. It seems that Spotify is not enabled. Please read the documentation to learn how to configure Spotify.") - return + for text in texts: + if any(x in text.upper() for x in ["SPOTIFY", "MUSIC"]): + # check if mpd client is running + try: + client = MPDClient() + client.timeout = None + client.idletimeout = None + client.connect("localhost", 6600) + except: + self.mic.say( + "I'm sorry. It seems that Spotify is not enabled. Please read the documentation to learn how to configure Spotify.") + return - self.mic.say("Please give me a moment, I'm loading your Spotify playlists.") - music_mode = MusicMode(self.persona, self.mic) - music_mode.handleForever() - return + self.mic.say("Please give me a moment, I'm loading your Spotify playlists.") + music_mode = MusicMode(self.persona, self.mic) + music_mode.handleForever() + return - self.brain.query(text) + self.brain.query(texts) def handleForever(self): """Delegates user input to the handling function when activated.""" @@ -50,7 +51,7 @@ class Conversation(object): if not transcribed or not threshold: continue - input = self.mic.activeListen(threshold) + input = self.mic.activeListenToAllOptions(threshold) if input: self.delegateInput(input) else: diff --git a/client/mic.py b/client/mic.py index 2bb8c57..2e8d603 100644 --- a/client/mic.py +++ b/client/mic.py @@ -175,6 +175,19 @@ class Mic: def activeListen(self, THRESHOLD=None, LISTEN=True, MUSIC=False): """ Records until a second of silence or times out after 12 seconds + + Returns the first matching string or None + """ + + options = self.activeListenToAllOptions(THRESHOLD, LISTEN, MUSIC) + if options: + return options[0] + + def activeListenToAllOptions(self, THRESHOLD=None, LISTEN=True, MUSIC=False): + """ + Records until a second of silence or times out after 12 seconds + + Returns a list of the matching options or None """ RATE = 16000 diff --git a/client/music.py b/client/music.py index 2b6c055..ca9b776 100644 --- a/client/music.py +++ b/client/music.py @@ -17,14 +17,14 @@ def reconnect(func, *default_args, **default_kwargs): # sometimes not enough to just connect try: - func(self, *default_args, **default_kwargs) + return func(self, *default_args, **default_kwargs) except: self.client = MPDClient() self.client.timeout = None self.client.idletimeout = None self.client.connect("localhost", 6600) - func(self, *default_args, **default_kwargs) + return func(self, *default_args, **default_kwargs) return wrap @@ -103,7 +103,7 @@ class Music: self.client.play() - #@reconnect -- this makes the function return None for some reason! + @reconnect def current_song(self): item = self.client.playlistinfo(int(self.client.status()["song"]))[0] result = "%s by %s" % (item["title"], item["artist"]) diff --git a/client/musicmode.py b/client/musicmode.py index 334f94c..900c4f1 100644 --- a/client/musicmode.py +++ b/client/musicmode.py @@ -7,7 +7,6 @@ import os from mic import Mic import g2p from music import * -import speaker import stt @@ -42,7 +41,7 @@ class MusicMode: # create a new mic with the new music models self.mic = Mic( - speaker.newSpeaker(), + mic.speaker, stt.PocketSphinxSTT(lmd_music="languagemodel_spotify.lm", dictd_music="dictionary_spotify.dic"), stt.PocketSphinxSTT(lmd_music="languagemodel_spotify.lm", dictd_music="dictionary_spotify.dic") ) diff --git a/client/speaker.py b/client/speaker.py deleted file mode 100644 index 8344132..0000000 --- a/client/speaker.py +++ /dev/null @@ -1,63 +0,0 @@ -# -*- coding: utf-8-*- -""" -A Speaker handles audio output from Jasper to the user - -Speaker methods: - say - output 'phrase' as speech - play - play the audio in 'filename' - isAvailable - returns True if the platform supports this implementation -""" -import os -import json - - -class eSpeakSpeaker: - - """ - Uses the eSpeak speech synthesizer included in the Jasper disk image - """ - @classmethod - def isAvailable(cls): - return os.system("which espeak") == 0 - - def say(self, phrase, OPTIONS=" -vdefault+m3 -p 40 -s 160 --stdout > say.wav"): - os.system("espeak " + json.dumps(phrase) + OPTIONS) - self.play("say.wav") - - def play(self, filename): - os.system("aplay -D hw:1,0 " + filename) - - -class saySpeaker: - - """ - Uses the OS X built-in 'say' command - """ - - @classmethod - def isAvailable(cls): - return os.system("which say") == 0 - - def shellquote(self, s): - return "'" + s.replace("'", "'\\''") + "'" - - def say(self, phrase): - os.system("say " + self.shellquote(phrase)) - - def play(self, filename): - os.system("afplay " + filename) - - -def newSpeaker(): - """ - Returns: - A speaker implementation available on the current platform - - Raises: - ValueError if no speaker implementation is supported on this platform - """ - - for cls in [eSpeakSpeaker, saySpeaker]: - if cls.isAvailable(): - return cls() - raise ValueError("Platform is not supported") diff --git a/client/stt.py b/client/stt.py index cf32a74..899775b 100644 --- a/client/stt.py +++ b/client/stt.py @@ -144,7 +144,7 @@ class PocketSphinxSTT(AbstractSTTEngine): print "JASPER: " + result[0] print "===================" - return result[0] + return [result[0]] @classmethod def is_available(cls): @@ -225,14 +225,18 @@ class GoogleSTT(AbstractSTTEngine): response = self.http.post(url, data=data, headers=headers) response.encoding = 'utf-8' response_read = response.text - decoded = json.loads(response_read.split("\n")[1]) - text = decoded['result'][0]['alternative'][0]['transcript'] - if text: - print "===================" - print "JASPER: " + text - print "===================" - return text + response_parts = response_read.strip().split("\n") + decoded = json.loads(response_parts[-1]) + if decoded['result']: + texts = [alt['transcript'] for alt in decoded['result'][0]['alternative']] + if texts: + print "===================" + print "JASPER: " + ', '.join(texts) + print "===================" + return texts + else: + return [] except Exception: traceback.print_exc() diff --git a/client/test.py b/client/test.py index 0ad37a3..9c99688 100644 --- a/client/test.py +++ b/client/test.py @@ -12,6 +12,7 @@ import vocabcompiler import g2p import brain import jasperpath +import tts from diagnose import Diagnostics DEFAULT_PROFILE = { @@ -21,9 +22,6 @@ DEFAULT_PROFILE = { 'phone_number': '012344321' } -def activeInternet(): - return Diagnostics.check_network_connection() - class UnorderedList(list): def __eq__(self, other): @@ -145,7 +143,7 @@ class TestModules(unittest.TestCase): inputs = [] self.runConversation(query, inputs, Time) - @unittest.skipIf(not activeInternet(), "No internet connection") + @unittest.skipIf(not Diagnostics.check_network_connection(), "No internet connection") def testGmail(self): key = 'gmail_password' if not key in self.profile or not self.profile[key]: @@ -157,7 +155,7 @@ class TestModules(unittest.TestCase): inputs = [] self.runConversation(query, inputs, Gmail) - @unittest.skipIf(not activeInternet(), "No internet connection") + @unittest.skipIf(not Diagnostics.check_network_connection(), "No internet connection") def testHN(self): from modules import HN @@ -169,7 +167,7 @@ class TestModules(unittest.TestCase): outputs = self.runConversation(query, inputs, HN) self.assertTrue("front-page articles" in outputs[1]) - @unittest.skipIf(not activeInternet(), "No internet connection") + @unittest.skipIf(not Diagnostics.check_network_connection(), "No internet connection") def testNews(self): from modules import News @@ -181,7 +179,7 @@ class TestModules(unittest.TestCase): outputs = self.runConversation(query, inputs, News) self.assertTrue("top headlines" in outputs[1]) - @unittest.skipIf(not activeInternet(), "No internet connection") + @unittest.skipIf(not Diagnostics.check_network_connection(), "No internet connection") def testWeather(self): from modules import Weather @@ -192,6 +190,11 @@ class TestModules(unittest.TestCase): "can't see that far ahead" in outputs[0] or "Tomorrow" in outputs[0]) +class TestTTS(unittest.TestCase): + def testTTS(self): + tts_engine = tts.get_engine_by_slug('dummy-tts') + tts_instance = tts_engine() + tts_instance.say('This is a test.') class TestBrain(unittest.TestCase): @@ -225,7 +228,7 @@ class TestBrain(unittest.TestCase): hn = filter(lambda m: m.__name__ == hn_module, my_brain.modules)[0] with patch.object(hn, 'handle') as mocked_handle: - my_brain.query("hacker news") + my_brain.query(["hacker news"]) self.assertTrue(mocked_handle.called) @@ -239,7 +242,7 @@ if __name__ == '__main__': # Change CWD to jasperpath.LIB_PATH os.chdir(jasperpath.LIB_PATH) - test_cases = [TestBrain, TestModules, TestVocabCompiler] + test_cases = [TestBrain, TestModules, TestVocabCompiler, TestTTS] if not args.light: test_cases.append(TestG2P) test_cases.append(TestMic) diff --git a/client/tts.py b/client/tts.py new file mode 100644 index 0000000..8fef776 --- /dev/null +++ b/client/tts.py @@ -0,0 +1,360 @@ +# -*- coding: utf-8-*- +""" +A Speaker handles audio output from Jasper to the user + +Speaker methods: + say - output 'phrase' as speech + play - play the audio in 'filename' + is_available - returns True if the platform supports this implementation +""" +import os +import platform +import re +import sys +import tempfile +import subprocess +import pipes +import logging +from abc import ABCMeta, abstractmethod +from distutils.spawn import find_executable + +import yaml +import argparse + +import wave +try: + import mad + import gtts +except ImportError: + pass + +class AbstractTTSEngine(object): + """ + Generic parent class for all speakers + """ + __metaclass__ = ABCMeta + + @classmethod + @abstractmethod + def is_available(cls): + return (find_executable('aplay') is not None) + + def __init__(self, **kwargs): + self._logger = logging.getLogger(__name__) + + @abstractmethod + def say(self, phrase, *args): + pass + + def play(self, filename): + # FIXME: Use platform-independent audio-output here + # See issue jasperproject/jasper-client#188 + cmd = ['aplay', '-D', 'hw:1,0', str(filename)] + self._logger.debug('Executing %s', ' '.join([pipes.quote(arg) for arg in cmd])) + with tempfile.TemporaryFile() as f: + subprocess.call(cmd, stdout=f, stderr=f) + f.seek(0) + output = f.read() + if output: + self._logger.debug("Output was: '%s'", output) + +class AbstractMp3TTSEngine(AbstractTTSEngine): + """ + Generic class that implements the 'play' method for mp3 files + """ + @classmethod + def is_available(cls): + return (super(AbstractMp3TTSEngine, cls).is_available() and 'mad' in sys.modules.keys()) + + def play_mp3(self, filename): + mf = mad.MadFile(filename) + with tempfile.NamedTemporaryFile(suffix='.wav') as f: + wav = wave.open(f, mode='wb') + wav.setframerate(mf.samplerate()) + wav.setnchannels(1 if mf.mode() == mad.MODE_SINGLE_CHANNEL else 2) + wav.setsampwidth(4L) # width of 32 bit audio + frame = mf.read() + while frame is not None: + wav.writeframes(frame) + frame = mf.read() + wav.close() + self.play(f.name) + +class DummyTTS(AbstractTTSEngine): + """ + Dummy TTS engine that logs phrases with INFO level instead of synthesizing + speech. + """ + + SLUG = "dummy-tts" + + @classmethod + def is_available(cls): + return True + + def say(self, phrase): + self._logger.info(phrase) + + def play(self, filename): + self._logger.debug("Playback of file '%s' requested") + pass + +class EspeakTTS(AbstractTTSEngine): + """ + Uses the eSpeak speech synthesizer included in the Jasper disk image + Requires espeak to be available + """ + + SLUG = "espeak-tts" + + def __init__(self, voice='default+m3', pitch_adjustment=40, words_per_minute=160): + super(self.__class__, self).__init__() + self.voice = voice + self.pitch_adjustment = pitch_adjustment + self.words_per_minute = words_per_minute + + @classmethod + def is_available(cls): + return (super(cls, cls).is_available() and find_executable('espeak') is not None) + + def say(self, phrase): + self._logger.debug("Saying '%s' with '%s'", phrase, self.SLUG) + with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as f: + fname = f.name + cmd = ['espeak', '-v', self.voice, + '-p', self.pitch_adjustment, + '-s', self.words_per_minute, + '-w', fname, + phrase] + cmd = [str(x) for x in cmd] + self._logger.debug('Executing %s', ' '.join([pipes.quote(arg) for arg in cmd])) + with tempfile.TemporaryFile() as f: + subprocess.call(cmd, stdout=f, stderr=f) + f.seek(0) + output = f.read() + if output: + self._logger.debug("Output was: '%s'", output) + self.play(fname) + os.remove(fname) + +class FestivalTTS(AbstractTTSEngine): + """ + Uses the festival speech synthesizer + Requires festival (text2wave) to be available + """ + + SLUG = 'festival-tts' + + @classmethod + def is_available(cls): + if super(cls, cls).is_available() and find_executable('text2wave') is not None and find_executable('festival') is not None: + logger = logging.getLogger(__name__) + cmd = ['festival', '--pipe'] + with tempfile.SpooledTemporaryFile() as out_f: + with tempfile.SpooledTemporaryFile() as in_f: + logger.debug('Executing %s', ' '.join([pipes.quote(arg) for arg in cmd])) + subprocess.call(cmd, stdin=in_f, stdout=out_f, stderr=out_f) + out_f.seek(0) + output = out_f.read().strip() + if output: + logger.debug("Output was: '%s'", output) + return ('No default voice found' not in output) + return False + + def say(self, phrase): + self._logger.debug("Saying '%s' with '%s'", phrase, self.SLUG) + cmd = ['text2wave'] + with tempfile.NamedTemporaryFile(suffix='.wav') as out_f: + with tempfile.SpooledTemporaryFile() as in_f: + in_f.write(phrase) + in_f.seek(0) + with tempfile.SpooledTemporaryFile() as err_f: + self._logger.debug('Executing %s', ' '.join([pipes.quote(arg) for arg in cmd])) + subprocess.call(cmd, stdin=in_f, stdout=out_f, stderr=err_f) + err_f.seek(0) + output = err_f.read() + if output: + self._logger.debug("Output was: '%s'", output) + self.play(out_f.name) + +class MacOSXTTS(AbstractTTSEngine): + """ + Uses the OS X built-in 'say' command + """ + + SLUG = "osx-tts" + + @classmethod + def is_available(cls): + return (platform.system() == 'darwin' and find_executable('say') is not None and find_executable('afplay') is not None) + + def say(self, phrase): + self._logger.debug("Saying '%s' with '%s'", phrase, self.SLUG) + cmd = ['say', str(phrase)] + self._logger.debug('Executing %s', ' '.join([pipes.quote(arg) for arg in cmd])) + with tempfile.TemporaryFile() as f: + subprocess.call(cmd, stdout=f, stderr=f) + f.seek(0) + output = f.read() + if output: + self._logger.debug("Output was: '%s'", output) + + def play(self, filename): + cmd = ['afplay', str(filename)] + self._logger.debug('Executing %s', ' '.join([pipes.quote(arg) for arg in cmd])) + with tempfile.TemporaryFile() as f: + subprocess.call(cmd, stdout=f, stderr=f) + f.seek(0) + output = f.read() + if output: + self._logger.debug("Output was: '%s'", output) + +class PicoTTS(AbstractTTSEngine): + """ + Uses the svox-pico-tts speech synthesizer + Requires pico2wave to be available + """ + + SLUG = "pico-tts" + + def __init__(self, language="en-US"): + super(self.__class__, self).__init__() + self.language = language + + @classmethod + def is_available(cls): + return (super(cls, cls).is_available() and find_executable('pico2wave') is not None) + + @property + def languages(self): + cmd = ['pico2wave', '-l', 'NULL', + '-w', os.devnull, + 'NULL'] + with tempfile.SpooledTemporaryFile() as f: + subprocess.call(cmd, stderr=f) + f.seek(0) + output = f.read() + pattern = re.compile(r'Unknown language: NULL\nValid languages:\n((?:[a-z]{2}-[A-Z]{2}\n)+)') + matchobj = pattern.match(output) + if not matchobj: + raise RuntimeError("pico2wave: valid languages not detected") + langs = matchobj.group(1).split() + return langs + + def say(self, phrase): + self._logger.debug("Saying '%s' with '%s'", phrase, self.SLUG) + with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as f: + fname = f.name + cmd = ['pico2wave', '--wave', fname] + if self.language not in self.languages: + raise ValueError("Language '%s' not supported by '%s'", self.language, self.SLUG) + cmd.extend(['-l', self.language]) + cmd.append(phrase) + self._logger.debug('Executing %s', ' '.join([pipes.quote(arg) for arg in cmd])) + with tempfile.TemporaryFile() as f: + subprocess.call(cmd, stdout=f, stderr=f) + f.seek(0) + output = f.read() + if output: + self._logger.debug("Output was: '%s'", output) + self.play(fname) + os.remove(fname) + +class GoogleTTS(AbstractMp3TTSEngine): + """ + Uses the Google TTS online translator + Requires pymad and gTTS to be available + """ + + SLUG = "google-tts" + + def __init__(self, language='en'): + super(self.__class__, self).__init__() + self.language = language + + @classmethod + def is_available(cls): + return (super(cls, cls).is_available() and 'gtts' in sys.modules.keys()) + + @property + def languages(self): + langs = ['af', 'sq', 'ar', 'hy', 'ca', 'zh-CN', 'zh-TW', 'hr', 'cs', 'da', 'nl', 'en', 'eo', 'fi', 'fr', 'de', + 'el', 'ht', 'hi', 'hu', 'is', 'id', 'it', 'ja', 'ko', 'la', 'lv', 'mk', 'no', 'pl', 'pt', 'ro', 'ru', + 'sr', 'sk', 'es', 'sw', 'sv', 'ta', 'th', 'tr', 'vi', 'cy'] + return langs + + def say(self, phrase): + self._logger.debug("Saying '%s' with '%s'", phrase, self.SLUG) + if self.language not in self.languages: + raise ValueError("Language '%s' not supported by '%s'", self.language, self.SLUG) + tts = gtts.gTTS(text=phrase, lang=self.language) + with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as f: + tmpfile = f.name + tts.save(tmpfile) + self.play_mp3(tmpfile) + os.remove(tmpfile) + +def get_default_engine_slug(): + return 'osx-tts' if platform.system() == 'darwin' else 'espeak-tts' + +def get_engine_by_slug(slug=None): + """ + Returns: + A speaker implementation available on the current platform + + Raises: + ValueError if no speaker implementation is supported on this platform + """ + + if not slug or type(slug) is not str: + raise TypeError("Invalid slug '%s'", slug) + + selected_engines = filter(lambda engine: hasattr(engine, "SLUG") and engine.SLUG == slug, get_engines()) + if len(selected_engines) == 0: + raise ValueError("No TTS engine found for slug '%s'" % slug) + else: + if len(selected_engines) > 1: + print("WARNING: Multiple TTS engines found for slug '%s'. This is most certainly a bug." % slug) + engine = selected_engines[0] + if not engine.is_available(): + raise ValueError("TTS engine '%s' is not available (due to missing dependencies, missing dependencies, etc.)" % slug) + return engine + +def get_engines(): + def get_subclasses(cls): + subclasses = set() + for subclass in cls.__subclasses__(): + subclasses.add(subclass) + subclasses.update(get_subclasses(subclass)) + return subclasses + return [tts_engine for tts_engine in list(get_subclasses(AbstractTTSEngine)) if hasattr(tts_engine, 'SLUG') and tts_engine.SLUG] + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Jasper TTS module') + parser.add_argument('--debug', action='store_true', help='Show debug messages') + args = parser.parse_args() + + logging.basicConfig() + if args.debug: + logger = logging.getLogger(__name__) + logger.setLevel(logging.DEBUG) + + engines = get_engines() + available_engines = [] + for engine in get_engines(): + if engine.is_available(): + available_engines.append(engine) + print("Available TTS engines:") + for i, engine in enumerate(available_engines, start=1): + print("%d. %s" % (i, engine.SLUG)) + + print("") + print("Disabled TTS engines:") + for i, engine in enumerate(list(set(engines).difference(set(available_engines))), start=1): + print("%d. %s" % (i, engine.SLUG)) + + print("") + for i, engine in enumerate(available_engines, start=1): + print("%d. Testing engine '%s'..." % (i, engine.SLUG)) + engine().say("This is a test.") + print("Done.") |