[Minor] Add rspamd_str_make_utf_valid routine + unit tests

diff --git a/src/libutil/str_util.c b/src/libutil/str_util.c
index d8b17e3c369f5eb66a704ed6ad40f11b96651028..1e43e7726cd59d8ab6891d3c0198133738ea22e1 100644 (file)
--- a/src/libutil/str_util.c
+++ b/src/libutil/str_util.c
@@ -2425,7 +2425,7 @@ rspamd_str_regexp_escape (const gchar *pattern, gsize slen,
 
        if (flags & RSPAMD_REGEXP_ESCAPE_UTF) {
                if (!g_utf8_validate (pattern, slen, NULL)) {
-                       tmp_utf = g_utf8_make_valid (pattern, slen);
+                       tmp_utf = rspamd_str_make_utf_valid (pattern, slen, NULL);
                }
        }
 
@@ -2517,3 +2517,61 @@ rspamd_str_regexp_escape (const gchar *pattern, gsize slen,
 
        return res;
 }
+
+
+gchar *
+rspamd_str_make_utf_valid (const gchar *src, gsize slen, gsize *dstlen)
+{
+       GString *dst;
+       const gchar *last;
+       gchar *dchar;
+       gsize i, valid, prev;
+       UChar32 uc;
+
+       if (src == NULL) {
+               return NULL;
+       }
+
+       if (slen == 0) {
+               slen = strlen (src);
+       }
+
+       dst = g_string_sized_new (slen);
+       i = 0;
+       last = src;
+       valid = 0;
+       prev = 0;
+
+       while (i < slen) {
+               U8_NEXT (src, i, slen, uc);
+
+               if (uc <= 0) {
+                       if (valid > 0) {
+                               g_string_append_len (dst, last, valid);
+                       }
+                       /* 0xFFFD in UTF8 */
+                       g_string_append_len (dst, "\357\277\275", 3);
+                       valid = 0;
+                       last = &src[i];
+               }
+               else {
+                       valid += i - prev;
+               }
+
+               prev = i;
+       }
+
+       if (valid > 0) {
+               g_string_append_len (dst, last, valid);
+       }
+
+       dchar = dst->str;
+
+       if (dstlen) {
+               *dstlen = dst->len;
+       }
+
+       g_string_free (dst, FALSE);
+
+       return dchar;
+}
\ No newline at end of file

diff --git a/src/libutil/str_util.h b/src/libutil/str_util.h
index 688034ec69e7ee64d32571ca06291295aaaf1260..139a85416395751d42e5083838392426b374a789 100644 (file)
--- a/src/libutil/str_util.h
+++ b/src/libutil/str_util.h
@@ -419,4 +419,14 @@ gchar *
 rspamd_str_regexp_escape (const gchar *pattern, gsize slen,
                gsize *dst_len, enum rspamd_regexp_escape_flags flags);
 
+/**
+ * Returns copy of src (zero terminated) where all unicode is made valid or replaced
+ * to FFFD characters. Caller must free string after usage
+ * @param src
+ * @param slen
+ * @param dstelen
+ * @return
+ */
+gchar * rspamd_str_make_utf_valid (const gchar *src, gsize slen, gsize *dstlen);
+
 #endif /* SRC_LIBUTIL_STR_UTIL_H_ */

diff --git a/test/lua/unit/utf.lua b/test/lua/unit/utf.lua
index e22eb2a2f53beed10dc056f9bb2a067cf72cc105..277d99e41e86bdd821b68eba04a9807d9344ff59 100644 (file)
--- a/test/lua/unit/utf.lua
+++ b/test/lua/unit/utf.lua
@@ -5,36 +5,68 @@ context("UTF8 check functions", function()
   ffi.cdef[[
     void rspamd_str_lc_utf8 (char *str, unsigned int size);
     void rspamd_str_lc (char *str, unsigned int size);
+    char * rspamd_str_make_utf_valid (const char *src, size_t slen, size_t *dstlen);
   ]]
 
-  test("UTF lowercase", function()
-    local cases = {
-      {"АбЫрвАлг", "абырвалг"},
-      {"АAБBвc", "аaбbвc"}
-    }
-    
-    for _,c in ipairs(cases) do
+  local cases = {
+    {"АбЫрвАлг", "абырвалг"},
+    {"АAБBвc", "аaбbвc"},
+    --{"STRASSE", "straße"}, XXX: NYI
+    {"KEÇİ", "keçi"},
+  }
+
+  for i,c in ipairs(cases) do
+    test("UTF lowercase " .. tostring(i), function()
       local buf = ffi.new("char[?]", #c[1] + 1)
       ffi.copy(buf, c[1])
       ffi.C.rspamd_str_lc_utf8(buf, #c[1])
       local s = ffi.string(buf)
       assert_equal(s, c[2])
-    end
-  end)
-  test("ASCII lowercase", function()
-    local cases = {
-      {"AbCdEf", "abcdef"},
-      {"A", "a"},
-      {"AaAa", "aaaa"},
-      {"AaAaAaAa", "aaaaaaaa"}
-    }
-    
-    for _,c in ipairs(cases) do
+    end)
+  end
+
+  cases = {
+    {"AbCdEf", "abcdef"},
+    {"A", "a"},
+    {"AaAa", "aaaa"},
+    {"AaAaAaAa", "aaaaaaaa"}
+  }
+
+  for i,c in ipairs(cases) do
+    test("ASCII lowercase " .. tostring(i), function()
       local buf = ffi.new("char[?]", #c[1] + 1)
       ffi.copy(buf, c[1])
       ffi.C.rspamd_str_lc(buf, #c[1])
       local s = ffi.string(buf)
       assert_equal(s, c[2])
-    end
-  end)
+    end)
+  end
+
+  cases = {
+    {'тест', 'тест'},
+    {'\200\213\202', '���'},
+    {'тест\200\213\202test', 'тест���test'},
+    {'\200\213\202test', '���test'},
+    {'\200\213\202test\200\213\202', '���test���'},
+    {'тест\200\213\202test\200\213\202', 'тест���test���'},
+    {'тест\200\213\202test\200\213\202тест', 'тест���test���тест'},
+  }
+
+  local NULL = ffi.new 'void*'
+  for i,c in ipairs(cases) do
+    test("Unicode make valid " .. tostring(i), function()
+      local buf = ffi.new("char[?]", #c[1] + 1)
+      ffi.copy(buf, c[1])
+
+      local s = ffi.string(ffi.C.rspamd_str_make_utf_valid(buf, #c[1], NULL))
+      local function to_hex(s)
+        return (s:gsub('.', function (c)
+          return string.format('%02X', string.byte(c))
+        end))
+      end
+      print(to_hex(s))
+      print(to_hex(c[2]))
+      assert_equal(s, c[2])
+    end)
+  end
 end)
\ No newline at end of file