Merge pull request #210 from Holzhaus/diagnose-improvements

Diagnose improvements

5 files changed, 196 insertions, 126 deletions

diff --git a/client/diagnose.py b/client/diagnose.py
index fb19294..df12df7 100755..100644
--- a/client/diagnose.py
+++ b/client/diagnose.py
@@ -1,139 +1,195 @@
-#!/usr/bin/env python2
+# -*- coding: utf-8-*-
+import os
+import sys
 import time
-import re
 import socket
-import os
-import jasperpath
 import subprocess
+import pkgutil
 import logging
-import sys
-from distutils.spawn import find_executable
+if sys.version_info < (3, 3):
+    from distutils.spawn import find_executable
+else:
+    from shutil import which as find_executable
+
+import pip.req
 import pip.util
+import jasperpath
 
 logger = logging.getLogger(__name__)
 
 
-class Diagnostics:
+def check_network_connection(server="www.google.com"):
+    """
+    Checks if jasper can connect a network server.
+
+    Arguments:
+        server -- (optional) the server to connect with (Default:
+                  "www.google.com")
 
+    Returns:
+        True or False
     """
-    Set of diagnostics to be run for determining the health of the
-    host running Jasper
+    logger = logging.getLogger(__name__)
+    logger.debug("Checking network connection to server '%s'...", server)
+    try:
+        # see if we can resolve the host name -- tells us if there is
+        # a DNS listening
+        host = socket.gethostbyname(server)
+        # connect to the host -- tells us if the host is actually
+        # reachable
+        socket.create_connection((host, 80), 2)
+    except Exception:
+        logger.debug("Network connection not working")
+        return False
+    else:
+        logger.debug("Network connection working")
+        return True
 
-    To add new checks, add a boolean returning method with a name that starts
-    with `check_`
+
+def check_executable(executable):
     """
+    Checks if an executable exists in $PATH.
 
-    @classmethod
-    def check_network_connection(cls):
-        try:
-            # see if we can resolve the host name -- tells us if there is
-            # a DNS listening
-            host = socket.gethostbyname("www.google.com")
-            # connect to the host -- tells us if the host is actually
-            # reachable
-            socket.create_connection((host, 80), 2)
-        except Exception:
-            return False
-        else:
-            return True
+    Arguments:
+        executable -- the name of the executable (e.g. "echo")
 
-    @classmethod
-    def check_phonetisaurus_dictionary_file(cls):
-        return os.path.isfile(os.path.join(jasperpath.APP_PATH, "..",
-                                           "phonetisaurus/g014b2b.fst"))
+    Returns:
+        True or False
+    """
+    logger = logging.getLogger(__name__)
+    logger.debug("Checking executable '%s'...", executable)
+    executable_path = find_executable(executable)
+    found = executable_path is not None
+    if found:
+        logger.debug("Executable '%s' found: '%s'", executable,
+                     executable_path)
+    else:
+        logger.debug("Executable '%s' not found", executable)
+    return found
 
-    @classmethod
-    def check_phonetisaurus_program(cls):
-        return cls.do_check_program('phonetisaurus-g2p')
 
-    @classmethod
-    def check_espeak_program(cls):
-        return cls.do_check_program('espeak')
+def check_python_import(package_or_module):
+    """
+    Checks if a python package or module is importable.
 
-    @classmethod
-    def check_say_program(cls):
-        return cls.do_check_program('say')
+    Arguments:
+        package_or_module -- the package or module name to check
 
-    @classmethod
-    def do_check_program(cls, program):
-        return find_executable(program) is not None
+    Returns:
+        True or False
+    """
+    logger = logging.getLogger(__name__)
+    logger.debug("Checking python import '%s'...", package_or_module)
+    loader = pkgutil.get_loader(package_or_module)
+    found = loader is not None
+    if found:
+        logger.debug("Python %s '%s' found: %r",
+                     "package" if loader.is_package(package_or_module)
+                     else "module", package_or_module, loader.get_filename())
+    else:
+        logger.debug("Python import '%s' not found", package_or_module)
+    return found
 
-    @classmethod
-    def check_all_pip_requirements_installed(cls):
-        distributions = pip.util.get_installed_distributions()
-        requirements_lines = [line.strip() for line in
-                              open('requirements.txt').readlines()]
-        requirements = [name.split('==')[0] for name in
-                        list(filter(None, requirements_lines))]
-        installed_packages = [pkg.project_name for pkg in list(distributions)]
-        missing_packages = [pkg for pkg in requirements
-                            if pkg not in installed_packages]
-        if missing_packages:
-            logger.info("Missing packages: "+', '.join(missing_packages))
-            return False
-        else:
-            return True
 
-    @classmethod
-    def info_git_revision(cls):
-        return subprocess.check_output(['git', 'rev-parse', 'HEAD'])
+def get_pip_requirements(fname=os.path.join(jasperpath.LIB_PATH,
+                                            'requirements.txt')):
+    """
+    Gets the PIP requirements from a text file. If the files does not exists
+    or is not readable, it returns None
+
+    Arguments:
+        fname -- (optional) the requirement text file (Default:
+                 "client/requirements.txt")
 
+    Returns:
+        A list of pip requirement objects or None
+    """
+    logger = logging.getLogger(__name__)
+    if os.access(fname, os.R_OK):
+        reqs = list(pip.req.parse_requirements(fname))
+        logger.debug("Found %d PIP requirements in file '%s'", len(reqs),
+                     fname)
+        return reqs
+    else:
+        logger.debug("PIP requirements file '%s' not found or not readable",
+                     fname)
 
-class DiagnosticRunner:
 
+def get_git_revision():
     """
-    Performs a series of checks against the system, printing the results to the
-    console and also saving them to diagnostic.log
+    Gets the current git revision hash as hex string. If the git executable is
+    missing or git is unable to get the revision, None is returned
+
+    Returns:
+        A hex string or None
     """
+    logger = logging.getLogger(__name__)
+    if not check_executable('git'):
+        logger.warning("'git' command not found, git revision not detectable")
+        return None
+    output = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
+    if not output:
+        logger.warning("Couldn't detect git revision (not a git repository?)")
+        return None
+    return output
 
-    def __init__(self, diagnostics):
-        self.diagnostics = diagnostics
 
-    def run(self):
-        self.initialize_log()
-        self.perform_checks()
+def run():
+    """
+    Performs a series of checks against the system and writes the results to
+    the logging system.
 
-    def perform_checks(self):
-        self.failed_checks = 0
-        for check in self.select_methods('check'):
-            self.do_check(check)
-        for info in self.select_methods('info'):
-            self.get_info(info)
-        if self.failed_checks == 0:
-            logger.info("All checks passed")
-        else:
-            logger.info("%d checks failed" % self.failed_checks)
+    Returns:
+        The number of failed checks as integer
+    """
+    logger = logging.getLogger(__name__)
+
+    # Set loglevel of this module least to info
+    loglvl = logger.getEffectiveLevel()
+    if loglvl == logging.NOTSET or loglvl > logging.INFO:
+        logger.setLevel(logging.INFO)
+
+    logger.info("Starting jasper diagnostic at %s" % time.strftime("%c"))
+    logger.info("Git revision: %r", get_git_revision())
 
-    def select_methods(self, prefix):
-        def is_match(method_name):
-            return (callable(getattr(self.diagnostics, method_name)) and
-                    re.match(r"\A" + prefix + "_", method_name))
+    failed_checks = 0
 
-        return [method_name for method_name in dir(self.diagnostics)
-                if is_match(method_name)]
+    if not check_network_connection():
+        failed_checks += 1
 
-    def initialize_log(self):
-        logger.info("Starting jasper diagnostic at %s" % time.strftime("%c"))
+    for executable in ['phonetisaurus-g2p', 'espeak', 'say']:
+        if not check_executable(executable):
+            logger.warning("Executable '%s' is missing in $PATH", executable)
+            failed_checks += 1
 
-    def get_info(self, info_name):
-        message = info_name.replace("info_", "").replace("_", " ")
-        info_method = getattr(self.diagnostics, info_name)
-        info = info_method()
-        logger.info("%s: %s" % (message, info))
+    for req in get_pip_requirements():
+        logger.debug("Checking PIP package '%s'...", req.name)
+        if not req.check_if_exists():
+            logger.warning("PIP package '%s' is missing", req.name)
+            failed_checks += 1
+        else:
+            logger.debug("PIP package '%s' found", req.name)
 
-    def do_check(self, check_name):
-        message = check_name.replace("check_", "").replace("_", " ")
-        check = getattr(self.diagnostics, check_name)
-        if check():
-            result = "OK"
+    for fname in [os.path.join(jasperpath.APP_PATH, os.pardir, "phonetisaurus",
+                               "g014b2b.fst")]:
+        logger.debug("Checking file '%s'...", fname)
+        if not os.access(fname, os.R_OK):
+            logger.warning("File '%s' is missing", fname)
+            failed_checks += 1
         else:
-            self.failed_checks += 1
-            result = "FAILED"
+            logger.debug("File '%s' found", fname)
 
-        logger.info("Checking %s... %s" % (message, result))
+    if not failed_checks:
+        logger.info("All checks passed")
+    else:
+        logger.info("%d checks failed" % failed_checks)
 
+    return failed_checks
 
-if __name__ == '__main__':
-    logging.basicConfig(stream=sys.stdout, level=logging.INFO)
 
-    DiagnosticRunner(Diagnostics).run()
+if __name__ == '__main__':
+    logging.basicConfig(stream=sys.stdout)
+    logger = logging.getLogger()
+    if '--debug' in sys.argv:
+        logger.setLevel(logging.DEBUG)
+    run()
diff --git a/client/stt.py b/client/stt.py
index cfaced6..485f5d9 100644
--- a/client/stt.py
+++ b/client/stt.py
@@ -5,12 +5,12 @@ import traceback
 import wave
 import json
 import tempfile
-import pkgutil
 import logging
 from abc import ABCMeta, abstractmethod
 import requests
 import yaml
 import jasperpath
+import diagnose
 
 """
 The default Speech-to-Text implementation which relies on PocketSphinx.
@@ -171,7 +171,7 @@ class PocketSphinxSTT(AbstractSTTEngine):
 
     @classmethod
     def is_available(cls):
-        return (pkgutil.get_loader('pocketsphinx') is not None)
+        return diagnose.check_python_import('pocketsphinx')
 
 """
 Speech-To-Text implementation which relies on the Google Speech API.
@@ -274,7 +274,7 @@ class GoogleSTT(AbstractSTTEngine):
 
     @classmethod
     def is_available(cls):
-        return True
+        return diagnose.check_network_connection()
 
 """
 Returns a Speech-To-Text engine.
diff --git a/client/test.py b/client/test.py
index d9ca8f5..6858700 100644
--- a/client/test.py
+++ b/client/test.py
@@ -13,8 +13,8 @@ import g2p
 import brain
 import jasperpath
 import tts
+import diagnose
 from stt import TranscriptionMode
-from diagnose import Diagnostics
 
 DEFAULT_PROFILE = {
     'prefers_email': False,
@@ -107,6 +107,14 @@ class TestG2P(unittest.TestCase):
         self.assertEqual(g2p.translateWords(words), translations)
 
 
+class TestDiagnose(unittest.TestCase):
+    def testPythonImportCheck(self):
+        # This a python stdlib module that definitely exists
+        self.assertTrue(diagnose.check_python_import("os"))
+        # I sincerly hope nobody will ever create a package with that name
+        self.assertFalse(diagnose.check_python_import("nonexistant_package"))
+
+
 class TestModules(unittest.TestCase):
 
     def setUp(self):
@@ -154,7 +162,7 @@ class TestModules(unittest.TestCase):
         inputs = []
         self.runConversation(query, inputs, Time)
 
-    @unittest.skipIf(not Diagnostics.check_network_connection(),
+    @unittest.skipIf(not diagnose.check_network_connection(),
                      "No internet connection")
     def testGmail(self):
         key = 'gmail_password'
@@ -167,7 +175,7 @@ class TestModules(unittest.TestCase):
         inputs = []
         self.runConversation(query, inputs, Gmail)
 
-    @unittest.skipIf(not Diagnostics.check_network_connection(),
+    @unittest.skipIf(not diagnose.check_network_connection(),
                      "No internet connection")
     def testHN(self):
         from modules import HN
@@ -180,7 +188,7 @@ class TestModules(unittest.TestCase):
         outputs = self.runConversation(query, inputs, HN)
         self.assertTrue("front-page articles" in outputs[1])
 
-    @unittest.skipIf(not Diagnostics.check_network_connection(),
+    @unittest.skipIf(not diagnose.check_network_connection(),
                      "No internet connection")
     def testNews(self):
         from modules import News
@@ -193,7 +201,7 @@ class TestModules(unittest.TestCase):
         outputs = self.runConversation(query, inputs, News)
         self.assertTrue("top headlines" in outputs[1])
 
-    @unittest.skipIf(not Diagnostics.check_network_connection(),
+    @unittest.skipIf(not diagnose.check_network_connection(),
                      "No internet connection")
     def testWeather(self):
         from modules import Weather
@@ -267,7 +275,8 @@ if __name__ == '__main__':
     # Change CWD to jasperpath.LIB_PATH
     os.chdir(jasperpath.LIB_PATH)
 
-    test_cases = [TestBrain, TestModules, TestVocabCompiler, TestTTS]
+    test_cases = [TestBrain, TestModules, TestVocabCompiler, TestTTS,
+                  TestDiagnose]
     if not args.light:
         test_cases.append(TestG2P)
         test_cases.append(TestMic)
diff --git a/client/tts.py b/client/tts.py
index 244c0f4..55c33b0 100644
--- a/client/tts.py
+++ b/client/tts.py
@@ -10,23 +10,22 @@ Speaker methods:
 import os
 import platform
 import re
-import sys
 import tempfile
 import subprocess
 import pipes
 import logging
+import wave
 from abc import ABCMeta, abstractmethod
-from distutils.spawn import find_executable
 
 import argparse
-
-import wave
 try:
     import mad
     import gtts
 except ImportError:
     pass
 
+import diagnose
+
 
 class AbstractTTSEngine(object):
     """
@@ -37,7 +36,7 @@ class AbstractTTSEngine(object):
     @classmethod
     @abstractmethod
     def is_available(cls):
-        return (find_executable('aplay') is not None)
+        return diagnose.check_executable('aplay')
 
     def __init__(self, **kwargs):
         self._logger = logging.getLogger(__name__)
@@ -67,7 +66,7 @@ class AbstractMp3TTSEngine(AbstractTTSEngine):
     @classmethod
     def is_available(cls):
         return (super(AbstractMp3TTSEngine, cls).is_available() and
-                'mad' in sys.modules.keys())
+                diagnose.check_python_import('mad'))
 
     def play_mp3(self, filename):
         mf = mad.MadFile(filename)
@@ -123,7 +122,7 @@ class EspeakTTS(AbstractTTSEngine):
     @classmethod
     def is_available(cls):
         return (super(cls, cls).is_available() and
-                find_executable('espeak') is not None)
+                diagnose.check_executable('espeak'))
 
     def say(self, phrase):
         self._logger.debug("Saying '%s' with '%s'", phrase, self.SLUG)
@@ -158,8 +157,9 @@ class FestivalTTS(AbstractTTSEngine):
     @classmethod
     def is_available(cls):
         if (super(cls, cls).is_available() and
-           find_executable('text2wave') is not None and
-           find_executable('festival') is not None):
+           diagnose.check_executable('text2wave') and
+           diagnose.check_executable('festival')):
+
             logger = logging.getLogger(__name__)
             cmd = ['festival', '--pipe']
             with tempfile.SpooledTemporaryFile() as out_f:
@@ -205,8 +205,8 @@ class MacOSXTTS(AbstractTTSEngine):
     @classmethod
     def is_available(cls):
         return (platform.system() == 'darwin' and
-                find_executable('say') is not None and
-                find_executable('afplay') is not None)
+                diagnose.check_executable('say') and
+                diagnose.check_executable('afplay'))
 
     def say(self, phrase):
         self._logger.debug("Saying '%s' with '%s'", phrase, self.SLUG)
@@ -247,7 +247,7 @@ class PicoTTS(AbstractTTSEngine):
     @classmethod
     def is_available(cls):
         return (super(cls, cls).is_available() and
-                find_executable('pico2wave') is not None)
+                diagnose.check_executable('pico2wave'))
 
     @property
     def languages(self):
@@ -303,7 +303,8 @@ class GoogleTTS(AbstractMp3TTSEngine):
     @classmethod
     def is_available(cls):
         return (super(cls, cls).is_available() and
-                'gtts' in sys.modules.keys())
+                diagnose.check_python_import('gtts') and
+                diagnose.check_network_connection())
 
     @property
     def languages(self):
diff --git a/jasper.py b/jasper.py
index 27a8de8..d116ce9 100755
--- a/jasper.py
+++ b/jasper.py
@@ -8,8 +8,7 @@ import logging
 import yaml
 import argparse
 
-from client.diagnose import Diagnostics
-from client import vocabcompiler, tts, stt, jasperpath
+from client import vocabcompiler, tts, stt, jasperpath, diagnose
 
 # Add jasperpath.LIB_PATH to sys.path
 sys.path.append(jasperpath.LIB_PATH)
@@ -21,6 +20,8 @@ parser.add_argument('--local', action='store_true',
                     help='Use text input instead of a real microphone')
 parser.add_argument('--no-network-check', action='store_true',
                     help='Disable the network connection check')
+parser.add_argument('--diagnose', action='store_true',
+                    help='Run diagnose and exit')
 parser.add_argument('--debug', action='store_true', help='Show debug messages')
 args = parser.parse_args()
 
@@ -132,11 +133,14 @@ if __name__ == "__main__":
     if args.debug:
         logger.setLevel(logging.DEBUG)
 
-    if (not args.no_network_check and
-       not Diagnostics.check_network_connection()):
+    if not args.no_network_check and not diagnose.check_network_connection():
         logger.warning("Network not connected. This may prevent Jasper from " +
                        "running properly.")
 
+    if args.diagnose:
+        failed_checks = diagnose.run()
+        sys.exit(0 if not failed_checks else 1)
+
     try:
         app = Jasper()
     except Exception: