From faa0df57224fd5c7374c831af67e01bedfd30e1b Mon Sep 17 00:00:00 2001 From: schneefux Date: Sun, 28 Sep 2014 19:18:18 +0200 Subject: Rewritten G2P code --- client/g2p.py | 225 ++++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 163 insertions(+), 62 deletions(-) (limited to 'client/g2p.py') diff --git a/client/g2p.py b/client/g2p.py index 89f2282..8ce7997 100644 --- a/client/g2p.py +++ b/client/g2p.py @@ -1,69 +1,170 @@ # -*- coding: utf-8-*- import os -import tempfile -import subprocess +import sys import re -import yaml +import subprocess +import tempfile +import shutil +import logging +if sys.version_info < (3, 3): + import distutils.spawn import jasperpath +import yaml -PHONE_MATCH = re.compile(r' (.*) ') - -FST_MODEL = None - -# Try to get fst_model from config -profile_path = os.path.join(os.path.dirname(__file__), 'profile.yml') -if os.path.exists(profile_path): - with open(profile_path, 'r') as f: - profile = yaml.safe_load(f) - if ('pocketsphinx' in profile and - 'fst_model' in profile['pocketsphinx']): - FST_MODEL = profile['pocketsphinx']['fst_model'] - -if not FST_MODEL: - FST_MODEL = os.path.join(jasperpath.APP_PATH, os.pardir, 'phonetisaurus', - 'g014b2b.fst') - - -def parseLine(line): - return PHONE_MATCH.search(line).group(1) - - -def parseOutput(output): - return PHONE_MATCH.findall(output) - - -def translateWord(word): - out = subprocess.check_output( - ['phonetisaurus-g2p', '--model=%s' % FST_MODEL, '--input=%s' % word]) - return parseLine(out) - - -def translateWords(words): - full_text = '\n'.join(words) - - with tempfile.NamedTemporaryFile(suffix='.g2p', delete=False) as f: - temp_filename = f.name - f.write(full_text) - - output = translateFile(temp_filename) - os.remove(temp_filename) - - return output - - -def translateFile(input_filename, output_filename=None): - out = subprocess.check_output( - ['phonetisaurus-g2p', '--model=%s' % FST_MODEL, - '--input=%s' % input_filename, '--words', '--isfile']) - out = parseOutput(out) - - if output_filename: - out = '\n'.join(out) - - with open(output_filename, "wb") as f: - f.write(out) - - return None - return out +class PhonetisaurusG2P(object): + PATTERN = re.compile(r'^(?P.+)\t(?P\d+\.\d+)\t ' + + r'(?P.*) ', re.MULTILINE) + + @classmethod + def executable_found(cls): + if sys.version_info < (3, 3): + cmd_exists = distutils.spawn.find_executable + else: + cmd_exists = shutil.which + # Required binary for this class + cmd = 'phonetisaurus-g2p' + if not cmd_exists(cmd): + return False + return True + + @classmethod + def execute(cls, fst_model, input, is_file=False, nbest=None): + logger = logging.getLogger(__name__) + + cmd = ['phonetisaurus-g2p', + '--model=%s' % fst_model, + '--input=%s' % input, + '--words'] + + if is_file: + cmd.append('--isfile') + + if nbest is not None: + cmd.extend(['--nbest=%d' % nbest]) + + cmd = [str(x) for x in cmd] + with tempfile.SpooledTemporaryFile() as err_f: + try: + # FIXME: We can't just use subprocess.call and redirect stdout + # and stderr, because it looks like Phonetisaurus can't open + # an already opened file descriptor a second time. This is why + # we have to use this somehow hacky subprocess.Popen approach. + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdoutdata, stderrdata = proc.communicate() + returncode = proc.returncode + if returncode != 0: + logger.warning("Command '%s' return with exit status %d", + ' '.join(cmd), returncode) + except OSError: + logger.error("Error occured while executing command '%s'", + ' '.join(cmd), exc_info=True) + stdoutdata, stderrdata = None, None + if stderrdata is not None: + for line in stderrdata.splitlines(): + message = line.strip() + if message: + logger.debug(message) + + result = {} + if stdoutdata is not None: + for word, precision, pronounc in cls.PATTERN.findall(stdoutdata): + if word not in result: + result[word] = [] + result[word].append(pronounc) + return result + + @classmethod + def get_config(cls): + # FIXME: Replace this as soon as pull request + # jasperproject/jasper-client#128 has been merged + + conf = {'fst_model': os.path.join(jasperpath.APP_PATH, os.pardir, + 'phonetisaurus', 'g014b2b.fst')} + # Try to get fst_model from config + profile_path = os.path.join(os.path.dirname(__file__), 'profile.yml') + if os.path.exists(profile_path): + with open(profile_path, 'r') as f: + profile = yaml.safe_load(f) + if 'pocketsphinx' in profile: + if 'fst_model' in profile['pocketsphinx']: + conf['fst_model'] = \ + profile['pocketsphinx']['fst_model'] + if 'nbest' in profile['pocketsphinx']: + conf['nbest'] = int(profile['pocketsphinx']['nbest']) + print conf + return conf + + def __new__(cls, fst_model=None, *args, **kwargs): + if not cls.executable_found(): + raise OSError("Can't find command 'phonetisaurus-g2p'! Please " + + "check if Phonetisaurus is installed and in your " + + "$PATH.") + if fst_model is None or not os.access(fst_model, os.R_OK): + raise OSError("FST model '%r' does not exist! Can't create " + + "instance." % fst_model) + inst = object.__new__(cls, fst_model, *args, **kwargs) + return inst + + def __init__(self, fst_model=None, nbest=None): + self._logger = logging.getLogger(__name__) + + self.fst_model = os.path.abspath(fst_model) + self._logger.debug("Using FST model: '%s'", self.fst_model) + + self.nbest = nbest + if self.nbest is not None: + self._logger.debug("Will use the %d best results.", self.nbest) + + def _translate_word(self, word): + return self.execute(self.fst_model, word, nbest=self.nbest) + + def _translate_words(self, words): + with tempfile.NamedTemporaryFile(suffix='.g2p', delete=False) as f: + # The 'delete=False' kwarg is kind of a hack, but Phonetisaurus + # won't work if we remove it, because it seems that I can't open + # a file descriptor a second time. + for word in words: + f.write("%s\n" % word) + tmp_fname = f.name + output = self.execute(self.fst_model, tmp_fname, is_file=True, + nbest=self.nbest) + os.remove(tmp_fname) + return output + + def translate(self, words): + if type(words) is str or len(words) == 1: + self._logger.debug('Converting single word to phonemes') + output = self._translate_word(words if type(words) is str + else words[0]) + else: + self._logger.debug('Converting %d words to phonemes', len(words)) + output = self._translate_words(words) + self._logger.debug('G2P conversion returned phonemes for %d words', + len(output)) + return output + +if __name__ == "__main__": + import pprint + import argparse + parser = argparse.ArgumentParser(description='Phonetisaurus G2P module') + parser.add_argument('fst_model', action='store', + help='Path to the FST Model') + parser.add_argument('--debug', action='store_true', + help='Show debug messages') + args = parser.parse_args() + + logging.basicConfig() + logger = logging.getLogger() + if args.debug: + logger.setLevel(logging.DEBUG) + + words = ['THIS', 'IS', 'A', 'TEST'] + + g2pconv = PhonetisaurusG2P(args.fst_model, nbest=3) + output = g2pconv.translate(words) + + pp = pprint.PrettyPrinter(indent=2) + pp.pprint(output) -- cgit v1.3.1 From 210327ebb4f58b84ff5f38541beda57a06ee5a69 Mon Sep 17 00:00:00 2001 From: schneefux Date: Wed, 1 Oct 2014 14:00:28 +0200 Subject: Minor style fix in phonetisaurus-g2p code --- client/g2p.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'client/g2p.py') diff --git a/client/g2p.py b/client/g2p.py index 8ce7997..1e63d74 100644 --- a/client/g2p.py +++ b/client/g2p.py @@ -25,9 +25,7 @@ class PhonetisaurusG2P(object): cmd_exists = shutil.which # Required binary for this class cmd = 'phonetisaurus-g2p' - if not cmd_exists(cmd): - return False - return True + return cmd_exists(cmd) @classmethod def execute(cls, fst_model, input, is_file=False, nbest=None): -- cgit v1.3.1 From 550a133b7cc497c55c8c36ff75bf252860dc93aa Mon Sep 17 00:00:00 2001 From: schneefux Date: Wed, 1 Oct 2014 14:12:30 +0200 Subject: Remove unneccessary print from g2p.py --- client/g2p.py | 1 - 1 file changed, 1 deletion(-) (limited to 'client/g2p.py') diff --git a/client/g2p.py b/client/g2p.py index 1e63d74..741aa3f 100644 --- a/client/g2p.py +++ b/client/g2p.py @@ -92,7 +92,6 @@ class PhonetisaurusG2P(object): profile['pocketsphinx']['fst_model'] if 'nbest' in profile['pocketsphinx']: conf['nbest'] = int(profile['pocketsphinx']['nbest']) - print conf return conf def __new__(cls, fst_model=None, *args, **kwargs): -- cgit v1.3.1 From 897812b6659fdd7a68bceec9ccb008a16182ef9f Mon Sep 17 00:00:00 2001 From: schneefux Date: Mon, 6 Oct 2014 18:10:03 +0200 Subject: Remove unused tempfile from client/g2p.py --- client/g2p.py | 43 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 22 deletions(-) (limited to 'client/g2p.py') diff --git a/client/g2p.py b/client/g2p.py index 741aa3f..b14ee98 100644 --- a/client/g2p.py +++ b/client/g2p.py @@ -43,28 +43,27 @@ class PhonetisaurusG2P(object): cmd.extend(['--nbest=%d' % nbest]) cmd = [str(x) for x in cmd] - with tempfile.SpooledTemporaryFile() as err_f: - try: - # FIXME: We can't just use subprocess.call and redirect stdout - # and stderr, because it looks like Phonetisaurus can't open - # an already opened file descriptor a second time. This is why - # we have to use this somehow hacky subprocess.Popen approach. - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdoutdata, stderrdata = proc.communicate() - returncode = proc.returncode - if returncode != 0: - logger.warning("Command '%s' return with exit status %d", - ' '.join(cmd), returncode) - except OSError: - logger.error("Error occured while executing command '%s'", - ' '.join(cmd), exc_info=True) - stdoutdata, stderrdata = None, None - if stderrdata is not None: - for line in stderrdata.splitlines(): - message = line.strip() - if message: - logger.debug(message) + try: + # FIXME: We can't just use subprocess.call and redirect stdout + # and stderr, because it looks like Phonetisaurus can't open + # an already opened file descriptor a second time. This is why + # we have to use this somehow hacky subprocess.Popen approach. + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdoutdata, stderrdata = proc.communicate() + returncode = proc.returncode + if returncode != 0: + logger.warning("Command '%s' return with exit status %d", + ' '.join(cmd), returncode) + except OSError: + logger.error("Error occured while executing command '%s'", + ' '.join(cmd), exc_info=True) + stdoutdata, stderrdata = None, None + if stderrdata is not None: + for line in stderrdata.splitlines(): + message = line.strip() + if message: + logger.debug(message) result = {} if stdoutdata is not None: -- cgit v1.3.1 From a8809862e909e4c703228e4bd583bb25fabc3ae9 Mon Sep 17 00:00:00 2001 From: schneefux Date: Wed, 8 Oct 2014 19:59:17 +0200 Subject: Use diagnose.check_executable for executable detection in g2p.py --- client/g2p.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) (limited to 'client/g2p.py') diff --git a/client/g2p.py b/client/g2p.py index b14ee98..a2b3982 100644 --- a/client/g2p.py +++ b/client/g2p.py @@ -1,14 +1,11 @@ # -*- coding: utf-8-*- import os -import sys import re import subprocess import tempfile -import shutil import logging -if sys.version_info < (3, 3): - import distutils.spawn +import diagnose import jasperpath import yaml @@ -17,16 +14,6 @@ class PhonetisaurusG2P(object): PATTERN = re.compile(r'^(?P.+)\t(?P\d+\.\d+)\t ' + r'(?P.*) ', re.MULTILINE) - @classmethod - def executable_found(cls): - if sys.version_info < (3, 3): - cmd_exists = distutils.spawn.find_executable - else: - cmd_exists = shutil.which - # Required binary for this class - cmd = 'phonetisaurus-g2p' - return cmd_exists(cmd) - @classmethod def execute(cls, fst_model, input, is_file=False, nbest=None): logger = logging.getLogger(__name__) @@ -94,7 +81,7 @@ class PhonetisaurusG2P(object): return conf def __new__(cls, fst_model=None, *args, **kwargs): - if not cls.executable_found(): + if not diagnose.check_executable('phonetisaurus-g2p'): raise OSError("Can't find command 'phonetisaurus-g2p'! Please " + "check if Phonetisaurus is installed and in your " + "$PATH.") -- cgit v1.3.1 From d8e79ba4db37e4fcd1fdddb5e5554a762f97ed71 Mon Sep 17 00:00:00 2001 From: schneefux Date: Wed, 8 Oct 2014 19:59:46 +0200 Subject: Reorder imports --- client/g2p.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'client/g2p.py') diff --git a/client/g2p.py b/client/g2p.py index a2b3982..ec0b025 100644 --- a/client/g2p.py +++ b/client/g2p.py @@ -5,9 +5,10 @@ import subprocess import tempfile import logging +import yaml + import diagnose import jasperpath -import yaml class PhonetisaurusG2P(object): -- cgit v1.3.1 From 12a73debc9366aff44e32d119234cd8898205991 Mon Sep 17 00:00:00 2001 From: schneefux Date: Wed, 8 Oct 2014 20:01:41 +0200 Subject: Use configfile from jasper config dir in g2p.py (i.e. use ) --- client/g2p.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'client/g2p.py') diff --git a/client/g2p.py b/client/g2p.py index ec0b025..9e715c9 100644 --- a/client/g2p.py +++ b/client/g2p.py @@ -69,7 +69,7 @@ class PhonetisaurusG2P(object): conf = {'fst_model': os.path.join(jasperpath.APP_PATH, os.pardir, 'phonetisaurus', 'g014b2b.fst')} # Try to get fst_model from config - profile_path = os.path.join(os.path.dirname(__file__), 'profile.yml') + profile_path = jasperpath.config('profile.yml') if os.path.exists(profile_path): with open(profile_path, 'r') as f: profile = yaml.safe_load(f) -- cgit v1.3.1 From c763fb9760fb7bef5fd1a78af2fe32ff7a2b50da Mon Sep 17 00:00:00 2001 From: schneefux Date: Wed, 8 Oct 2014 20:07:05 +0200 Subject: remove redundant variable assignment --- client/g2p.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'client/g2p.py') diff --git a/client/g2p.py b/client/g2p.py index 9e715c9..7b03386 100644 --- a/client/g2p.py +++ b/client/g2p.py @@ -39,10 +39,9 @@ class PhonetisaurusG2P(object): proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdoutdata, stderrdata = proc.communicate() - returncode = proc.returncode - if returncode != 0: + if proc.returncode != 0: logger.warning("Command '%s' return with exit status %d", - ' '.join(cmd), returncode) + ' '.join(cmd), proc.returncode) except OSError: logger.error("Error occured while executing command '%s'", ' '.join(cmd), exc_info=True) -- cgit v1.3.1