diff options
| author | Vsevolod Stakhov <vsevolod@highsecure.ru> | 2018-11-25 17:03:12 +0000 |
|---|---|---|
| committer | Vsevolod Stakhov <vsevolod@highsecure.ru> | 2018-11-25 17:03:12 +0000 |
| commit | 63ef123b048d5f1f2f6a5d172be6dc1a2629e2d7 (patch) | |
| tree | 0b265d289f1221301f1a1aa26331c43e009bece3 /src/libstat | |
| parent | 9249059ba342bf32c3655e9af9c0cefa42466d87 (diff) | |
| download | rspamd-63ef123b048d5f1f2f6a5d172be6dc1a2629e2d7.tar.gz rspamd-63ef123b048d5f1f2f6a5d172be6dc1a2629e2d7.zip | |
[Project] Rework parts conversion and serialization
Diffstat (limited to 'src/libstat')
| -rw-r--r-- | src/libstat/tokenizers/tokenizers.c | 13 |
1 files changed, 5 insertions, 8 deletions
diff --git a/src/libstat/tokenizers/tokenizers.c b/src/libstat/tokenizers/tokenizers.c index 32d9ba0df..9ec0c4315 100644 --- a/src/libstat/tokenizers/tokenizers.c +++ b/src/libstat/tokenizers/tokenizers.c @@ -745,28 +745,25 @@ rspamd_stem_words (GArray *words, rspamd_mempool_t *pool, if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_UTF) { if (stem) { - const gchar *stemmed; + const gchar *stemmed = NULL; stemmed = sb_stemmer_stem (stem, tok->normalized.begin, tok->normalized.len); - dlen = strlen (stemmed); + dlen = stemmed ? strlen (stemmed) : 0; if (dlen > 0) { - dest = rspamd_mempool_alloc (pool, dlen); + dest = rspamd_mempool_alloc (pool, dlen + 1); memcpy (dest, stemmed, dlen); - rspamd_str_lc_utf8 (dest, dlen); + dest[dlen] = '\0'; tok->stemmed.len = dlen; tok->stemmed.begin = dest; tok->flags |= RSPAMD_STAT_TOKEN_FLAG_STEMMED; } else { /* Fallback */ - dest = rspamd_mempool_alloc (pool, tok->normalized.len); - memcpy (dest, tok->normalized.begin, tok->normalized.len); - rspamd_str_lc_utf8 (dest, tok->normalized.len); tok->stemmed.len = tok->normalized.len; - tok->stemmed.begin = dest; + tok->stemmed.begin = tok->normalized.begin; } } else { |