Rewritten G2P code

2 files changed, 154 insertions, 49 deletions

diff --git a/client/g2p.py b/client/g2p.py
index 89f2282..8ce7997 100644
--- a/client/g2p.py
+++ b/client/g2p.py
@@ -1,69 +1,170 @@
 # -*- coding: utf-8-*-
 import os
-import tempfile
-import subprocess
+import sys
 import re
-import yaml
+import subprocess
+import tempfile
+import shutil
+import logging
+if sys.version_info < (3, 3):
+    import distutils.spawn
 
 import jasperpath
+import yaml
+
+
+class PhonetisaurusG2P(object):
+    PATTERN = re.compile(r'^(?P<word>.+)\t(?P<precision>\d+\.\d+)\t<s> ' +
+                         r'(?P<pronounciation>.*) </s>', re.MULTILINE)
 
-PHONE_MATCH = re.compile(r'<s> (.*) </s>')
+    @classmethod
+    def executable_found(cls):
+        if sys.version_info < (3, 3):
+            cmd_exists = distutils.spawn.find_executable
+        else:
+            cmd_exists = shutil.which
+        # Required binary for this class
+        cmd = 'phonetisaurus-g2p'
+        if not cmd_exists(cmd):
+            return False
+        return True
 
-FST_MODEL = None
+    @classmethod
+    def execute(cls, fst_model, input, is_file=False, nbest=None):
+        logger = logging.getLogger(__name__)
 
-# Try to get fst_model from config
-profile_path = os.path.join(os.path.dirname(__file__), 'profile.yml')
-if os.path.exists(profile_path):
-    with open(profile_path, 'r') as f:
-        profile = yaml.safe_load(f)
-        if ('pocketsphinx' in profile and
-           'fst_model' in profile['pocketsphinx']):
-            FST_MODEL = profile['pocketsphinx']['fst_model']
+        cmd = ['phonetisaurus-g2p',
+               '--model=%s' % fst_model,
+               '--input=%s' % input,
+               '--words']
 
-if not FST_MODEL:
-    FST_MODEL = os.path.join(jasperpath.APP_PATH, os.pardir, 'phonetisaurus',
-                             'g014b2b.fst')
+        if is_file:
+            cmd.append('--isfile')
 
+        if nbest is not None:
+            cmd.extend(['--nbest=%d' % nbest])
 
-def parseLine(line):
-    return PHONE_MATCH.search(line).group(1)
+        cmd = [str(x) for x in cmd]
+        with tempfile.SpooledTemporaryFile() as err_f:
+            try:
+                # FIXME: We can't just use subprocess.call and redirect stdout
+                # and stderr, because it looks like Phonetisaurus can't open
+                # an already opened file descriptor a second time. This is why
+                # we have to use this somehow hacky subprocess.Popen approach.
+                proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                                        stderr=subprocess.PIPE)
+                stdoutdata, stderrdata = proc.communicate()
+                returncode = proc.returncode
+                if returncode != 0:
+                    logger.warning("Command '%s' return with exit status %d",
+                                   ' '.join(cmd), returncode)
+            except OSError:
+                logger.error("Error occured while executing command '%s'",
+                             ' '.join(cmd), exc_info=True)
+                stdoutdata, stderrdata = None, None
+            if stderrdata is not None:
+                for line in stderrdata.splitlines():
+                    message = line.strip()
+                    if message:
+                        logger.debug(message)
 
+        result = {}
+        if stdoutdata is not None:
+            for word, precision, pronounc in cls.PATTERN.findall(stdoutdata):
+                if word not in result:
+                    result[word] = []
+                result[word].append(pronounc)
+        return result
 
-def parseOutput(output):
-    return PHONE_MATCH.findall(output)
+    @classmethod
+    def get_config(cls):
+        # FIXME: Replace this as soon as pull request
+        # jasperproject/jasper-client#128 has been merged
 
+        conf = {'fst_model': os.path.join(jasperpath.APP_PATH, os.pardir,
+                                          'phonetisaurus', 'g014b2b.fst')}
+        # Try to get fst_model from config
+        profile_path = os.path.join(os.path.dirname(__file__), 'profile.yml')
+        if os.path.exists(profile_path):
+            with open(profile_path, 'r') as f:
+                profile = yaml.safe_load(f)
+                if 'pocketsphinx' in profile:
+                    if 'fst_model' in profile['pocketsphinx']:
+                        conf['fst_model'] = \
+                            profile['pocketsphinx']['fst_model']
+                    if 'nbest' in profile['pocketsphinx']:
+                        conf['nbest'] = int(profile['pocketsphinx']['nbest'])
+        print conf
+        return conf
 
-def translateWord(word):
-    out = subprocess.check_output(
-        ['phonetisaurus-g2p', '--model=%s' % FST_MODEL, '--input=%s' % word])
-    return parseLine(out)
+    def __new__(cls, fst_model=None, *args, **kwargs):
+        if not cls.executable_found():
+            raise OSError("Can't find command 'phonetisaurus-g2p'! Please " +
+                          "check if Phonetisaurus is installed and in your " +
+                          "$PATH.")
+        if fst_model is None or not os.access(fst_model, os.R_OK):
+            raise OSError("FST model '%r' does not exist! Can't create " +
+                          "instance." % fst_model)
+        inst = object.__new__(cls, fst_model, *args, **kwargs)
+        return inst
 
+    def __init__(self, fst_model=None, nbest=None):
+        self._logger = logging.getLogger(__name__)
 
-def translateWords(words):
-    full_text = '\n'.join(words)
+        self.fst_model = os.path.abspath(fst_model)
+        self._logger.debug("Using FST model: '%s'", self.fst_model)
 
-    with tempfile.NamedTemporaryFile(suffix='.g2p', delete=False) as f:
-        temp_filename = f.name
-        f.write(full_text)
+        self.nbest = nbest
+        if self.nbest is not None:
+            self._logger.debug("Will use the %d best results.", self.nbest)
 
-    output = translateFile(temp_filename)
-    os.remove(temp_filename)
+    def _translate_word(self, word):
+        return self.execute(self.fst_model, word, nbest=self.nbest)
 
-    return output
+    def _translate_words(self, words):
+        with tempfile.NamedTemporaryFile(suffix='.g2p', delete=False) as f:
+            # The 'delete=False' kwarg is kind of a hack, but Phonetisaurus
+            # won't work if we remove it, because it seems that I can't open
+            # a file descriptor a second time.
+            for word in words:
+                f.write("%s\n" % word)
+            tmp_fname = f.name
+        output = self.execute(self.fst_model, tmp_fname, is_file=True,
+                              nbest=self.nbest)
+        os.remove(tmp_fname)
+        return output
 
+    def translate(self, words):
+        if type(words) is str or len(words) == 1:
+            self._logger.debug('Converting single word to phonemes')
+            output = self._translate_word(words if type(words) is str
+                                          else words[0])
+        else:
+            self._logger.debug('Converting %d words to phonemes', len(words))
+            output = self._translate_words(words)
+        self._logger.debug('G2P conversion returned phonemes for %d words',
+                           len(output))
+        return output
 
-def translateFile(input_filename, output_filename=None):
-    out = subprocess.check_output(
-        ['phonetisaurus-g2p', '--model=%s' % FST_MODEL,
-         '--input=%s' % input_filename, '--words', '--isfile'])
-    out = parseOutput(out)
+if __name__ == "__main__":
+    import pprint
+    import argparse
+    parser = argparse.ArgumentParser(description='Phonetisaurus G2P module')
+    parser.add_argument('fst_model', action='store',
+                        help='Path to the FST Model')
+    parser.add_argument('--debug', action='store_true',
+                        help='Show debug messages')
+    args = parser.parse_args()
 
-    if output_filename:
-        out = '\n'.join(out)
+    logging.basicConfig()
+    logger = logging.getLogger()
+    if args.debug:
+        logger.setLevel(logging.DEBUG)
 
-        with open(output_filename, "wb") as f:
-            f.write(out)
+    words = ['THIS', 'IS', 'A', 'TEST']
 
-        return None
+    g2pconv = PhonetisaurusG2P(args.fst_model, nbest=3)
+    output = g2pconv.translate(words)
 
-    return out
+    pp = pprint.PrettyPrinter(indent=2)
+    pp.pprint(output)
diff --git a/client/vocabcompiler.py b/client/vocabcompiler.py
index 8edee6f..1cfe15e 100644
--- a/client/vocabcompiler.py
+++ b/client/vocabcompiler.py
@@ -10,10 +10,10 @@ import logging
 import hashlib
 from abc import ABCMeta, abstractmethod, abstractproperty
 
-import g2p
 import brain
 import jasperpath
 
+from g2p import PhonetisaurusG2P
 try:
     import cmuclmtk
 except ImportError:
@@ -312,14 +312,18 @@ class PocketsphinxVocabulary(AbstractVocabulary):
         """
         # create the dictionary
         self._logger.debug("Getting phonemes for %d words...", len(words))
-        pronounced = g2p.translateWords(words)
-        zipped = zip(words, pronounced)
-        lines = ["%s %s" % (x, y) for x, y in zipped]
+        g2pconverter = PhonetisaurusG2P(**PhonetisaurusG2P.get_config())
+        phonemes = g2pconverter.translate(words)
 
         self._logger.debug("Creating dict file: '%s'", output_file)
         with open(output_file, "w") as f:
-            for line in lines:
-                f.write("%s\n" % line)
+            for word, pronounciations in phonemes.items():
+                for i, pronounciation in enumerate(pronounciations, start=1):
+                    if i == 1:
+                        line = "%s\t%s\n" % (word, pronounciation)
+                    else:
+                        line = "%s(%d)\t%s\n" % (word, i, pronounciation)
+                    f.write(line)                
 
 
 def get_phrases_from_module(module):