From 63ef123b048d5f1f2f6a5d172be6dc1a2629e2d7 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Sun, 25 Nov 2018 17:03:12 +0000 Subject: [Project] Rework parts conversion and serialization --- src/libstat/tokenizers/tokenizers.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) (limited to 'src/libstat') diff --git a/src/libstat/tokenizers/tokenizers.c b/src/libstat/tokenizers/tokenizers.c index 32d9ba0df..9ec0c4315 100644 --- a/src/libstat/tokenizers/tokenizers.c +++ b/src/libstat/tokenizers/tokenizers.c @@ -745,28 +745,25 @@ rspamd_stem_words (GArray *words, rspamd_mempool_t *pool, if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_UTF) { if (stem) { - const gchar *stemmed; + const gchar *stemmed = NULL; stemmed = sb_stemmer_stem (stem, tok->normalized.begin, tok->normalized.len); - dlen = strlen (stemmed); + dlen = stemmed ? strlen (stemmed) : 0; if (dlen > 0) { - dest = rspamd_mempool_alloc (pool, dlen); + dest = rspamd_mempool_alloc (pool, dlen + 1); memcpy (dest, stemmed, dlen); - rspamd_str_lc_utf8 (dest, dlen); + dest[dlen] = '\0'; tok->stemmed.len = dlen; tok->stemmed.begin = dest; tok->flags |= RSPAMD_STAT_TOKEN_FLAG_STEMMED; } else { /* Fallback */ - dest = rspamd_mempool_alloc (pool, tok->normalized.len); - memcpy (dest, tok->normalized.begin, tok->normalized.len); - rspamd_str_lc_utf8 (dest, tok->normalized.len); tok->stemmed.len = tok->normalized.len; - tok->stemmed.begin = dest; + tok->stemmed.begin = tok->normalized.begin; } } else { -- cgit v1.2.3