redis to word2vec: fix index out of range

1 files changed, 8 insertions, 0 deletions

diff --git a/processors/redis_to_word2vec.py b/processors/redis_to_word2vec.py
index 9e46bdb..752ee37 100644
--- a/processors/redis_to_word2vec.py
+++ b/processors/redis_to_word2vec.py
@@ -27,6 +27,9 @@ except Exception as e:
 def clean_words(words):
     clean_words = []
 
+    if len(words) == 0:
+        return []  # return early
+
     # skip popular bot commands
     bot_prefixes = ["?", "!", ".", "~", "+", ",", "$"]
     if (len(words[0]) > 0 and words[0][0] in bot_prefixes) or \
@@ -34,12 +37,17 @@ def clean_words(words):
         return []  # return early
 
     for word in words:
+        if len(word) == 0:
+            continue
+
         # skip discord mentions
         if (word.startswith("<") and word.endswith(">")) or word.startswith("http"):
             continue
         
         # replace markdown
         word = word.replace("**", "").replace("__", "").replace("`", "")
+        if len(word) == 0:
+            continue
 
         # turn to lowercase - except if 2/3rds of the text is uppercase
         lowercase_chars = sum(1 for c in word if c.islower())