diff options
| -rw-r--r-- | client/g2p.py | 187 | ||||
| -rw-r--r-- | client/vocabcompiler.py | 16 |
2 files changed, 154 insertions, 49 deletions
diff --git a/client/g2p.py b/client/g2p.py index 89f2282..8ce7997 100644 --- a/client/g2p.py +++ b/client/g2p.py @@ -1,69 +1,170 @@ # -*- coding: utf-8-*- import os -import tempfile -import subprocess +import sys import re -import yaml +import subprocess +import tempfile +import shutil +import logging +if sys.version_info < (3, 3): + import distutils.spawn import jasperpath +import yaml + + +class PhonetisaurusG2P(object): + PATTERN = re.compile(r'^(?P<word>.+)\t(?P<precision>\d+\.\d+)\t<s> ' + + r'(?P<pronounciation>.*) </s>', re.MULTILINE) -PHONE_MATCH = re.compile(r'<s> (.*) </s>') + @classmethod + def executable_found(cls): + if sys.version_info < (3, 3): + cmd_exists = distutils.spawn.find_executable + else: + cmd_exists = shutil.which + # Required binary for this class + cmd = 'phonetisaurus-g2p' + if not cmd_exists(cmd): + return False + return True -FST_MODEL = None + @classmethod + def execute(cls, fst_model, input, is_file=False, nbest=None): + logger = logging.getLogger(__name__) -# Try to get fst_model from config -profile_path = os.path.join(os.path.dirname(__file__), 'profile.yml') -if os.path.exists(profile_path): - with open(profile_path, 'r') as f: - profile = yaml.safe_load(f) - if ('pocketsphinx' in profile and - 'fst_model' in profile['pocketsphinx']): - FST_MODEL = profile['pocketsphinx']['fst_model'] + cmd = ['phonetisaurus-g2p', + '--model=%s' % fst_model, + '--input=%s' % input, + '--words'] -if not FST_MODEL: - FST_MODEL = os.path.join(jasperpath.APP_PATH, os.pardir, 'phonetisaurus', - 'g014b2b.fst') + if is_file: + cmd.append('--isfile') + if nbest is not None: + cmd.extend(['--nbest=%d' % nbest]) -def parseLine(line): - return PHONE_MATCH.search(line).group(1) + cmd = [str(x) for x in cmd] + with tempfile.SpooledTemporaryFile() as err_f: + try: + # FIXME: We can't just use subprocess.call and redirect stdout + # and stderr, because it looks like Phonetisaurus can't open + # an already opened file descriptor a second time. This is why + # we have to use this somehow hacky subprocess.Popen approach. + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdoutdata, stderrdata = proc.communicate() + returncode = proc.returncode + if returncode != 0: + logger.warning("Command '%s' return with exit status %d", + ' '.join(cmd), returncode) + except OSError: + logger.error("Error occured while executing command '%s'", + ' '.join(cmd), exc_info=True) + stdoutdata, stderrdata = None, None + if stderrdata is not None: + for line in stderrdata.splitlines(): + message = line.strip() + if message: + logger.debug(message) + result = {} + if stdoutdata is not None: + for word, precision, pronounc in cls.PATTERN.findall(stdoutdata): + if word not in result: + result[word] = [] + result[word].append(pronounc) + return result -def parseOutput(output): - return PHONE_MATCH.findall(output) + @classmethod + def get_config(cls): + # FIXME: Replace this as soon as pull request + # jasperproject/jasper-client#128 has been merged + conf = {'fst_model': os.path.join(jasperpath.APP_PATH, os.pardir, + 'phonetisaurus', 'g014b2b.fst')} + # Try to get fst_model from config + profile_path = os.path.join(os.path.dirname(__file__), 'profile.yml') + if os.path.exists(profile_path): + with open(profile_path, 'r') as f: + profile = yaml.safe_load(f) + if 'pocketsphinx' in profile: + if 'fst_model' in profile['pocketsphinx']: + conf['fst_model'] = \ + profile['pocketsphinx']['fst_model'] + if 'nbest' in profile['pocketsphinx']: + conf['nbest'] = int(profile['pocketsphinx']['nbest']) + print conf + return conf -def translateWord(word): - out = subprocess.check_output( - ['phonetisaurus-g2p', '--model=%s' % FST_MODEL, '--input=%s' % word]) - return parseLine(out) + def __new__(cls, fst_model=None, *args, **kwargs): + if not cls.executable_found(): + raise OSError("Can't find command 'phonetisaurus-g2p'! Please " + + "check if Phonetisaurus is installed and in your " + + "$PATH.") + if fst_model is None or not os.access(fst_model, os.R_OK): + raise OSError("FST model '%r' does not exist! Can't create " + + "instance." % fst_model) + inst = object.__new__(cls, fst_model, *args, **kwargs) + return inst + def __init__(self, fst_model=None, nbest=None): + self._logger = logging.getLogger(__name__) -def translateWords(words): - full_text = '\n'.join(words) + self.fst_model = os.path.abspath(fst_model) + self._logger.debug("Using FST model: '%s'", self.fst_model) - with tempfile.NamedTemporaryFile(suffix='.g2p', delete=False) as f: - temp_filename = f.name - f.write(full_text) + self.nbest = nbest + if self.nbest is not None: + self._logger.debug("Will use the %d best results.", self.nbest) - output = translateFile(temp_filename) - os.remove(temp_filename) + def _translate_word(self, word): + return self.execute(self.fst_model, word, nbest=self.nbest) - return output + def _translate_words(self, words): + with tempfile.NamedTemporaryFile(suffix='.g2p', delete=False) as f: + # The 'delete=False' kwarg is kind of a hack, but Phonetisaurus + # won't work if we remove it, because it seems that I can't open + # a file descriptor a second time. + for word in words: + f.write("%s\n" % word) + tmp_fname = f.name + output = self.execute(self.fst_model, tmp_fname, is_file=True, + nbest=self.nbest) + os.remove(tmp_fname) + return output + def translate(self, words): + if type(words) is str or len(words) == 1: + self._logger.debug('Converting single word to phonemes') + output = self._translate_word(words if type(words) is str + else words[0]) + else: + self._logger.debug('Converting %d words to phonemes', len(words)) + output = self._translate_words(words) + self._logger.debug('G2P conversion returned phonemes for %d words', + len(output)) + return output -def translateFile(input_filename, output_filename=None): - out = subprocess.check_output( - ['phonetisaurus-g2p', '--model=%s' % FST_MODEL, - '--input=%s' % input_filename, '--words', '--isfile']) - out = parseOutput(out) +if __name__ == "__main__": + import pprint + import argparse + parser = argparse.ArgumentParser(description='Phonetisaurus G2P module') + parser.add_argument('fst_model', action='store', + help='Path to the FST Model') + parser.add_argument('--debug', action='store_true', + help='Show debug messages') + args = parser.parse_args() - if output_filename: - out = '\n'.join(out) + logging.basicConfig() + logger = logging.getLogger() + if args.debug: + logger.setLevel(logging.DEBUG) - with open(output_filename, "wb") as f: - f.write(out) + words = ['THIS', 'IS', 'A', 'TEST'] - return None + g2pconv = PhonetisaurusG2P(args.fst_model, nbest=3) + output = g2pconv.translate(words) - return out + pp = pprint.PrettyPrinter(indent=2) + pp.pprint(output) diff --git a/client/vocabcompiler.py b/client/vocabcompiler.py index 8edee6f..1cfe15e 100644 --- a/client/vocabcompiler.py +++ b/client/vocabcompiler.py @@ -10,10 +10,10 @@ import logging import hashlib from abc import ABCMeta, abstractmethod, abstractproperty -import g2p import brain import jasperpath +from g2p import PhonetisaurusG2P try: import cmuclmtk except ImportError: @@ -312,14 +312,18 @@ class PocketsphinxVocabulary(AbstractVocabulary): """ # create the dictionary self._logger.debug("Getting phonemes for %d words...", len(words)) - pronounced = g2p.translateWords(words) - zipped = zip(words, pronounced) - lines = ["%s %s" % (x, y) for x, y in zipped] + g2pconverter = PhonetisaurusG2P(**PhonetisaurusG2P.get_config()) + phonemes = g2pconverter.translate(words) self._logger.debug("Creating dict file: '%s'", output_file) with open(output_file, "w") as f: - for line in lines: - f.write("%s\n" % line) + for word, pronounciations in phonemes.items(): + for i, pronounciation in enumerate(pronounciations, start=1): + if i == 1: + line = "%s\t%s\n" % (word, pronounciation) + else: + line = "%s(%d)\t%s\n" % (word, i, pronounciation) + f.write(line) def get_phrases_from_module(module): |