aboutsummaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/CMakeLists.txt2
-rw-r--r--src/libmime/mime_encoding.c2
-rw-r--r--src/libmime/mime_parser.c2
-rw-r--r--src/libmime/mime_string.hxx8
-rw-r--r--src/libmime/scan_result.c2
-rw-r--r--src/libserver/cfg_utils.cxx10
-rw-r--r--src/libserver/maps/map_helpers.c4
-rw-r--r--src/libserver/protocol.c2
-rw-r--r--src/libserver/re_cache.c2
-rw-r--r--src/libutil/CMakeLists.txt1
-rw-r--r--src/libutil/cxx/rspamd-simdutf.cxx41
-rw-r--r--src/libutil/fstring.c2
-rw-r--r--src/libutil/regexp.c2
-rw-r--r--src/libutil/rspamd_simdutf.h34
-rw-r--r--src/libutil/str_util.c2
-rw-r--r--src/lua/lua_text.c2
-rw-r--r--src/lua/lua_util.c2
17 files changed, 94 insertions, 26 deletions
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index f7fdcef7b..92edb0b6a 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -201,7 +201,7 @@ IF(SYSTEM_ZSTD MATCHES "OFF")
ELSE()
TARGET_LINK_LIBRARIES(rspamd-server zstd)
ENDIF()
-TARGET_LINK_LIBRARIES(rspamd-server rspamd-fastutf8)
+TARGET_LINK_LIBRARIES(rspamd-server rspamd-simdutf)
IF (ENABLE_CLANG_PLUGIN MATCHES "ON")
ADD_DEPENDENCIES(rspamd-server rspamd-clang)
diff --git a/src/libmime/mime_encoding.c b/src/libmime/mime_encoding.c
index 8b4c45436..995706d10 100644
--- a/src/libmime/mime_encoding.c
+++ b/src/libmime/mime_encoding.c
@@ -22,7 +22,7 @@
#include "libserver/task.h"
#include "mime_encoding.h"
#include "message.h"
-#include "contrib/fastutf8/fastutf8.h"
+#include "rspamd_simdutf.h"
#include "contrib/google-ced/ced_c.h"
#include <unicode/ucnv.h>
#if U_ICU_VERSION_MAJOR_NUM >= 44
diff --git a/src/libmime/mime_parser.c b/src/libmime/mime_parser.c
index ac35cffe3..1fe8b86e3 100644
--- a/src/libmime/mime_parser.c
+++ b/src/libmime/mime_parser.c
@@ -25,7 +25,7 @@
#include "contrib/uthash/utlist.h"
#include <openssl/cms.h>
#include <openssl/pkcs7.h>
-#include "contrib/fastutf8/fastutf8.h"
+#include "rspamd_simdutf.h"
struct rspamd_mime_parser_lib_ctx {
struct rspamd_multipattern *mp_boundary;
diff --git a/src/libmime/mime_string.hxx b/src/libmime/mime_string.hxx
index 7476816c6..b181576d3 100644
--- a/src/libmime/mime_string.hxx
+++ b/src/libmime/mime_string.hxx
@@ -1,11 +1,11 @@
-/*-
- * Copyright 2021 Vsevolod Stakhov
+/*
+ * Copyright 2024 Vsevolod Stakhov
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -28,7 +28,7 @@
#include "libutil/mem_pool.h"
#include "function2/function2.hpp"
#include "unicode/utf8.h"
-#include "contrib/fastutf8/fastutf8.h"
+#include "rspamd_simdutf.h"
namespace rspamd::mime {
/*
diff --git a/src/libmime/scan_result.c b/src/libmime/scan_result.c
index 894ae4f9e..992a8ea49 100644
--- a/src/libmime/scan_result.c
+++ b/src/libmime/scan_result.c
@@ -21,7 +21,7 @@
#include "lua/lua_common.h"
#include "libserver/cfg_file_private.h"
#include "libmime/scan_result_private.h"
-#include "contrib/fastutf8/fastutf8.h"
+#include "rspamd_simdutf.h"
#include <math.h>
#include "contrib/uthash/utlist.h"
diff --git a/src/libserver/cfg_utils.cxx b/src/libserver/cfg_utils.cxx
index 38adf8390..9612cdae4 100644
--- a/src/libserver/cfg_utils.cxx
+++ b/src/libserver/cfg_utils.cxx
@@ -35,7 +35,7 @@
#include "cryptobox.h"
#include "ssl_util.h"
#include "contrib/libottery/ottery.h"
-#include "contrib/fastutf8/fastutf8.h"
+#include "rspamd_simdutf.h"
#ifdef SYS_ZSTD
#include "zstd.h"
@@ -2658,14 +2658,6 @@ rspamd_init_libs(void)
/* Configure utf8 library */
unsigned int utf8_flags = 0;
-
- if ((ctx->crypto_ctx->cpu_config & CPUID_SSE41)) {
- utf8_flags |= RSPAMD_FAST_UTF8_FLAG_SSE41;
- }
- if ((ctx->crypto_ctx->cpu_config & CPUID_AVX2)) {
- utf8_flags |= RSPAMD_FAST_UTF8_FLAG_AVX2;
- }
-
rspamd_fast_utf8_library_init(utf8_flags);
#ifdef HAVE_LOCALE_H
diff --git a/src/libserver/maps/map_helpers.c b/src/libserver/maps/map_helpers.c
index 505932563..6f14a797a 100644
--- a/src/libserver/maps/map_helpers.c
+++ b/src/libserver/maps/map_helpers.c
@@ -1,5 +1,5 @@
/*
- * Copyright 2023 Vsevolod Stakhov
+ * Copyright 2024 Vsevolod Stakhov
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -21,7 +21,7 @@
#include "rspamd.h"
#include "cryptobox.h"
#include "mempool_vars_internal.h"
-#include "contrib/fastutf8/fastutf8.h"
+#include "rspamd_simdutf.h"
#include "contrib/cdb/cdb.h"
#ifdef WITH_HYPERSCAN
diff --git a/src/libserver/protocol.c b/src/libserver/protocol.c
index 1196d2d14..b683547a1 100644
--- a/src/libserver/protocol.c
+++ b/src/libserver/protocol.c
@@ -26,7 +26,7 @@
#include "protocol_internal.h"
#include "libserver/mempool_vars_internal.h"
#include "libserver/worker_util.h"
-#include "contrib/fastutf8/fastutf8.h"
+#include "rspamd_simdutf.h"
#include "task.h"
#include "lua/lua_classnames.h"
#include <math.h>
diff --git a/src/libserver/re_cache.c b/src/libserver/re_cache.c
index 0644980da..06e9f3328 100644
--- a/src/libserver/re_cache.c
+++ b/src/libserver/re_cache.c
@@ -46,7 +46,7 @@
#include <pcre2.h>
#endif
-#include "contrib/fastutf8/fastutf8.h"
+#include "rspamd_simdutf.h"
#ifdef HAVE_SYS_WAIT_H
#include <sys/wait.h>
diff --git a/src/libutil/CMakeLists.txt b/src/libutil/CMakeLists.txt
index 67b7e948f..acf082708 100644
--- a/src/libutil/CMakeLists.txt
+++ b/src/libutil/CMakeLists.txt
@@ -18,6 +18,7 @@ SET(LIBRSPAMDUTILSRC
${CMAKE_CURRENT_SOURCE_DIR}/heap.c
${CMAKE_CURRENT_SOURCE_DIR}/multipattern.c
${CMAKE_CURRENT_SOURCE_DIR}/cxx/utf8_util.cxx
+ ${CMAKE_CURRENT_SOURCE_DIR}/cxx/rspamd-simdutf.cxx
${CMAKE_CURRENT_SOURCE_DIR}/cxx/util_tests.cxx
${CMAKE_CURRENT_SOURCE_DIR}/cxx/file_util.cxx)
# Rspamdutil
diff --git a/src/libutil/cxx/rspamd-simdutf.cxx b/src/libutil/cxx/rspamd-simdutf.cxx
new file mode 100644
index 000000000..67b585812
--- /dev/null
+++ b/src/libutil/cxx/rspamd-simdutf.cxx
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2024 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * A simple interface for simdutf library to allow old functions to work properly
+ */
+
+#include "config.h"
+#include "simdutf.h"
+
+extern "C" {
+
+void rspamd_fast_utf8_library_init(unsigned flags)
+{
+ // This library requires no initialisation
+}
+
+off_t rspamd_fast_utf8_validate(const unsigned char *data, size_t len)
+{
+ auto res = simdutf::validate_utf8_with_errors((const char *) data, len);
+
+ if (res.error == simdutf::error_code::SUCCESS) {
+ return 0;
+ }
+
+ return res.count + 1;// We need to return offset for the first invalid character
+}
+} \ No newline at end of file
diff --git a/src/libutil/fstring.c b/src/libutil/fstring.c
index 082620c27..8da6b0068 100644
--- a/src/libutil/fstring.c
+++ b/src/libutil/fstring.c
@@ -15,7 +15,7 @@
*/
#include "fstring.h"
#include "str_util.h"
-#include "contrib/fastutf8/fastutf8.h"
+#include "rspamd_simdutf.h"
#include "contrib/mumhash/mum.h"
diff --git a/src/libutil/regexp.c b/src/libutil/regexp.c
index 9e98699fe..0646285ae 100644
--- a/src/libutil/regexp.c
+++ b/src/libutil/regexp.c
@@ -19,7 +19,7 @@
#include "ref.h"
#include "util.h"
#include "rspamd.h"
-#include "contrib/fastutf8/fastutf8.h"
+#include "rspamd_simdutf.h"
#ifndef WITH_PCRE2
/* Normal pcre path */
diff --git a/src/libutil/rspamd_simdutf.h b/src/libutil/rspamd_simdutf.h
new file mode 100644
index 000000000..c1fa07892
--- /dev/null
+++ b/src/libutil/rspamd_simdutf.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2024 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_RSPAMD_SIMDUTF_H
+#define RSPAMD_RSPAMD_SIMDUTF_H
+
+#pragma once
+#include "config.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void rspamd_fast_utf8_library_init(unsigned flags);
+off_t rspamd_fast_utf8_validate(const unsigned char *data, size_t len);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif//RSPAMD_RSPAMD_SIMDUTF_H
diff --git a/src/libutil/str_util.c b/src/libutil/str_util.c
index f8fff0dca..b3e47b7d4 100644
--- a/src/libutil/str_util.c
+++ b/src/libutil/str_util.c
@@ -31,7 +31,7 @@
#include <immintrin.h>
#endif
-#include "contrib/fastutf8/fastutf8.h"
+#include "rspamd_simdutf.h"
const unsigned char lc_map[256] = {
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
diff --git a/src/lua/lua_text.c b/src/lua/lua_text.c
index 3342fc95c..7ce7440c7 100644
--- a/src/lua/lua_text.c
+++ b/src/lua/lua_text.c
@@ -16,7 +16,7 @@
#include "lua_common.h"
#include "libcryptobox/cryptobox.h"
-#include "contrib/fastutf8/fastutf8.h"
+#include "rspamd_simdutf.h"
#include "unix-std.h"
/***
diff --git a/src/lua/lua_util.c b/src/lua/lua_util.c
index e92e4977a..14994751c 100644
--- a/src/lua/lua_util.c
+++ b/src/lua/lua_util.c
@@ -32,7 +32,7 @@
#include "unicode/uspoof.h"
#include "unicode/uscript.h"
-#include "contrib/fastutf8/fastutf8.h"
+#include "rspamd_simdutf.h"
/***
* @module rspamd_util
generated by cgit v1.2.3 (git 2.39.1) at 2025-01-20 21:18:06 +0000