diff options
| -rw-r--r-- | processors/redis_to_word2vec.py | 8 |
1 files changed, 8 insertions, 0 deletions
diff --git a/processors/redis_to_word2vec.py b/processors/redis_to_word2vec.py index 9e46bdb..752ee37 100644 --- a/processors/redis_to_word2vec.py +++ b/processors/redis_to_word2vec.py @@ -27,6 +27,9 @@ except Exception as e: def clean_words(words): clean_words = [] + if len(words) == 0: + return [] # return early + # skip popular bot commands bot_prefixes = ["?", "!", ".", "~", "+", ",", "$"] if (len(words[0]) > 0 and words[0][0] in bot_prefixes) or \ @@ -34,12 +37,17 @@ def clean_words(words): return [] # return early for word in words: + if len(word) == 0: + continue + # skip discord mentions if (word.startswith("<") and word.endswith(">")) or word.startswith("http"): continue # replace markdown word = word.replace("**", "").replace("__", "").replace("`", "") + if len(word) == 0: + continue # turn to lowercase - except if 2/3rds of the text is uppercase lowercase_chars = sum(1 for c in word if c.islower()) |