[Fix] Fix normalization of non-alphabet based languages

diff --git a/src/libstat/backends/redis_backend.c b/src/libstat/backends/redis_backend.c
index d54767c129f4a166bb4191148384a42e39e9c1c1..baeb2308d8bea6a317e1ad8f3bf4c668943f7b99 100644 (file)
--- a/src/libstat/backends/redis_backend.c
+++ b/src/libstat/backends/redis_backend.c
@@ -527,7 +527,8 @@ rspamd_redis_tokens_to_query (struct rspamd_task *task,
                                                                "HSET %b_tokens %b %b",
                                                                prefix, (size_t) prefix_len,
                                                                n0, (size_t) l0,
-                                                               tok->t1->stemmed.begin, tok->t1->stemmed.len);
+                                                               tok->t1->stemmed.begin,
+                                                               tok->t1->stemmed.len);
                                        }
                                }
                                else {

diff --git a/src/libstat/tokenizers/tokenizers.c b/src/libstat/tokenizers/tokenizers.c
index f69378f9b99abb132c5b9923c523c3cc4b361a5a..ffa1af9db1f77e7135c48d4f34038bb05b9a56cc 100644 (file)
--- a/src/libstat/tokenizers/tokenizers.c
+++ b/src/libstat/tokenizers/tokenizers.c
@@ -627,14 +627,10 @@ rspamd_uchars_to_ucs32 (const UChar *src, gsize srclen,
                        }
 #endif
 
-                       if (cat == U_UPPERCASE_LETTER ||
-                                       cat == U_LOWERCASE_LETTER ||
-                                       cat == U_DECIMAL_DIGIT_NUMBER ||
+                       if ((cat >= U_UPPERCASE_LETTER && cat <= U_OTHER_NUMBER) ||
                                        cat == U_CONNECTOR_PUNCTUATION ||
                                        cat == U_MATH_SYMBOL ||
-                                       cat == U_CURRENCY_SYMBOL ||
-                                       cat == U_INITIAL_PUNCTUATION ||
-                                       cat == U_FINAL_PUNCTUATION) {
+                                       cat == U_CURRENCY_SYMBOL) {
                                *d++ = u_tolower (t);
                        }
                }