Add fast version of ascii lowercase.

diff --git a/src/libmime/message.c b/src/libmime/message.c
index dfef04ce80c5554757f6df14c62a5b39cfae91e1..3472c70c16e39e2a736c79a8e083c691508ef508 100644 (file)
--- a/src/libmime/message.c
+++ b/src/libmime/message.c
@@ -1209,12 +1209,13 @@ rspamd_normalize_text_part (struct rspamd_task *task,
                                memcpy (stw.begin, r, nlen);
                                stw.len = nlen;
                        }
-
-                       if (part->is_utf) {
-                               rspamd_str_lc_utf8 (stw.begin, stw.len);
-                       }
                        else {
-                               rspamd_str_lc (stw.begin, stw.len);
+                               if (part->is_utf) {
+                                       rspamd_str_lc_utf8 (stw.begin, stw.len);
+                               }
+                               else {
+                                       rspamd_str_lc (stw.begin, stw.len);
+                               }
                        }
                }
                part->normalized_words = tmp;

diff --git a/src/libutil/util.c b/src/libutil/util.c
index 645c3d2fe94e8cbd5d609dd8bc6afdd660b35387..f64fe24019fdebebd4b20b1bde7e08268190da2a 100644 (file)
--- a/src/libutil/util.c
+++ b/src/libutil/util.c
@@ -658,13 +658,72 @@ rspamd_pass_signal (GHashTable * workers, gint signo)
        g_hash_table_foreach (workers, pass_signal_cb, GINT_TO_POINTER (signo));
 }
 
+static const guchar lc_map[256] = {
+               0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+               0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+               0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+               0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
+               0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27,
+               0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f,
+               0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
+               0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f,
+               0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
+               0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f,
+               0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
+               0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f,
+               0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
+               0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f,
+               0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
+               0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f,
+               0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+               0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+               0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+               0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f,
+               0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
+               0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
+               0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7,
+               0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf,
+               0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7,
+               0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf,
+               0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7,
+               0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf,
+               0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7,
+               0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef,
+               0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7,
+               0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff
+};
+
 void
 rspamd_str_lc (gchar *str, guint size)
 {
-       while (size--) {
-               *str = g_ascii_tolower (*str);
-               str++;
+       guint leftover = size % 4;
+       guint fp, i;
+       const uint8_t* s = (const uint8_t*) str;
+       gchar *dest = str;
+       guchar c1, c2, c3, c4;
+
+       fp = size - leftover;
+
+       for (i = 0; i != fp; i += 4) {
+               c1 = s[i], c2 = s[i + 1], c3 = s[i + 2], c4 = s[i + 3];
+               dest[0] = lc_map[c1];
+               dest[1] = lc_map[c2];
+               dest[2] = lc_map[c3];
+               dest[3] = lc_map[c4];
+               dest += 4;
        }
+
+       switch (leftover) {
+       case 3:
+               *dest++ = lc_map[(guchar)str[i++]];
+       case 2:
+               *dest++ = lc_map[(guchar)str[i++]];
+       case 1:
+               *dest++ = lc_map[(guchar)str[i]];
+       case 0:
+               *dest = '\0';
+       }
+
 }
 
 /*

diff --git a/src/plugins/lua/spamassassin.lua b/src/plugins/lua/spamassassin.lua
index 19b5737fdc2e9597aed41ed5b5ce76e9c431f4f9..35e49dce1ed950d565de8d0dd3bc3e35cb35640e 100644 (file)
--- a/src/plugins/lua/spamassassin.lua
+++ b/src/plugins/lua/spamassassin.lua
@@ -489,7 +489,7 @@ local function process_atom(atom, task)
     end
     return res
   else
-    rspamd_logger.err('Cannot find atom ' .. atom)
+    --rspamd_logger.err('Cannot find atom ' .. atom)
   end
   return 0
 end

diff --git a/test/lua/unit/utf.lua b/test/lua/unit/utf.lua
index 2549c501c5e7cc4d328130f8a6a8c2d6fdda2257..27988f918e3e42d3c1f67e322179ebb23473c600 100644 (file)
--- a/test/lua/unit/utf.lua
+++ b/test/lua/unit/utf.lua
@@ -4,6 +4,7 @@ context("UTF8 check functions", function()
   local ffi = require("ffi")
   ffi.cdef[[
     void rspamd_str_lc_utf8 (char *str, unsigned int size);
+    void rspamd_str_lc (char *str, unsigned int size);
   ]]
 
   test("UTF lowercase", function()
@@ -20,4 +21,20 @@ context("UTF8 check functions", function()
       assert_equal(s, c[2])
     end
   end)
+  test("ASCII lowercase", function()
+    local cases = {
+      {"AbCdEf", "abcdef"},
+      {"A", "a"},
+      {"AaAa", "aaaa"},
+      {"AaAaAaAa", "aaaaaaaa"}
+    }
+    
+    for _,c in ipairs(cases) do
+      local buf = ffi.new("char[?]", #c[1])
+      ffi.copy(buf, c[1])
+      ffi.C.rspamd_str_lc(buf, #c[1])
+      local s = ffi.string(buf)
+      assert_equal(s, c[2])
+    end
+  end)
 end)
\ No newline at end of file