From d6dc571f94632621ffc42dcac775cfc9e7e74acd Mon Sep 17 00:00:00 2001 From: schneefux Date: Tue, 3 Apr 2018 11:21:12 +0200 Subject: redis to word2vec: fix index out of range --- processors/redis_to_word2vec.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'processors') diff --git a/processors/redis_to_word2vec.py b/processors/redis_to_word2vec.py index 9e46bdb..752ee37 100644 --- a/processors/redis_to_word2vec.py +++ b/processors/redis_to_word2vec.py @@ -27,6 +27,9 @@ except Exception as e: def clean_words(words): clean_words = [] + if len(words) == 0: + return [] # return early + # skip popular bot commands bot_prefixes = ["?", "!", ".", "~", "+", ",", "$"] if (len(words[0]) > 0 and words[0][0] in bot_prefixes) or \ @@ -34,12 +37,17 @@ def clean_words(words): return [] # return early for word in words: + if len(word) == 0: + continue + # skip discord mentions if (word.startswith("<") and word.endswith(">")) or word.startswith("http"): continue # replace markdown word = word.replace("**", "").replace("__", "").replace("`", "") + if len(word) == 0: + continue # turn to lowercase - except if 2/3rds of the text is uppercase lowercase_chars = sum(1 for c in word if c.islower()) -- cgit v1.3.1