From 0d53332a7ecaa3a2b5020c7c58d6146d72d7b05c Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Sun, 25 Nov 2018 12:00:24 +0000
Subject: [Project] Various unicode fixes in language detector

---
 src/libstat/tokenizers/tokenizers.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

(limited to 'src/libstat')

diff --git a/src/libstat/tokenizers/tokenizers.c b/src/libstat/tokenizers/tokenizers.c
index 9bbe899fb..d27d9bc58 100644
--- a/src/libstat/tokenizers/tokenizers.c
+++ b/src/libstat/tokenizers/tokenizers.c
@@ -271,9 +271,8 @@ rspamd_tokenize_text (const gchar *text, gsize len,
 	buf.original.begin = text;
 	buf.original.len = len;
 	buf.flags = 0;
-	token.original.begin = NULL;
-	token.original.len = 0;
-	token.flags = 0;
+
+	memset (&token, 0, sizeof (token));
 
 	if (cfg != NULL) {
 		min_len = cfg->min_word_len;
-- 
cgit v1.2.3