[Rework] Further project structure reorganisation

cmake/Toolset.cmake

@@ -88,15 +88,6 @@ else ()
     endif ()
 endif ()
 
-# Google performance tools
-option (ENABLE_GPERF_TOOLS  "Enable google perftools [default: OFF]"             OFF)
-if (ENABLE_GPERF_TOOLS MATCHES "ON")
-    ProcessPackage(GPERF LIBRARY profiler INCLUDE profiler.h INCLUDE_SUFFIXES include/google
-            ROOT ${GPERF_ROOT_DIR})
-    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-omit-frame-pointer")
-    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
-    set (WITH_GPERF_TOOLS 1)
-endif (ENABLE_GPERF_TOOLS MATCHES "ON")
 
 # Legacy options support
 option (ENABLE_COVERAGE     "Build rspamd with code coverage options [default: OFF]" OFF)

config.h.in

@@ -146,7 +146,6 @@
 #cmakedefine WITH_LUA            1
 #cmakedefine WITH_LUAJIT         1
 #cmakedefine WITH_PCRE2          1
-#cmakedefine WITH_PROFILER       1
 #cmakedefine WITH_SNOWBALL       1
 #cmakedefine WITH_SQLITE         1
 #cmakedefine WITH_SYSTEM_HIREDIS 1

src/client/rspamc.c

@@ -15,8 +15,8 @@
  */
 #include "config.h"
 #include "libutil/util.h"
-#include "libutil/http_connection.h"
-#include "libutil/http_private.h"
+#include "libserver/http/http_connection.h"
+#include "libserver/http/http_private.h"
 #include "rspamdclient.h"
 #include "utlist.h"
 #include "unix-std.h"

src/client/rspamdclient.c

@@ -15,8 +15,8 @@
  */
 #include "rspamdclient.h"
 #include "libutil/util.h"
-#include "libutil/http_connection.h"
-#include "libutil/http_private.h"
+#include "libserver/http/http_connection.h"
+#include "libserver/http/http_private.h"
 #include "libserver/protocol_internal.h"
 #include "unix-std.h"
 #include "contrib/zstd/zstd.h"

src/controller.c

@@ -17,11 +17,11 @@
 #include "libserver/dynamic_cfg.h"
 #include "libserver/cfg_file_private.h"
 #include "libutil/rrd.h"
-#include "libutil/map.h"
-#include "libutil/map_helpers.h"
-#include "libutil/map_private.h"
-#include "libutil/http_private.h"
-#include "libutil/http_router.h"
+#include "libserver/maps/map.h"
+#include "libserver/maps/map_helpers.h"
+#include "libserver/maps/map_private.h"
+#include "libserver/http/http_private.h"
+#include "libserver/http/http_router.h"
 #include "libstat/stat_api.h"
 #include "rspamd.h"
 #include "libserver/worker_util.h"

src/fuzzy_storage.c

@@ -21,8 +21,8 @@
 #include "libserver/fuzzy_wire.h"
 #include "util.h"
 #include "rspamd.h"
-#include "map.h"
-#include "map_helpers.h"
+#include "libserver/maps/map.h"
+#include "libserver/maps/map_helpers.h"
 #include "fuzzy_wire.h"
 #include "libserver/fuzzy_backend/fuzzy_backend.h"
 #include "ottery.h"
@@ -33,9 +33,8 @@
 #include "libcryptobox/cryptobox.h"
 #include "libcryptobox/keypairs_cache.h"
 #include "libcryptobox/keypair.h"
-#include "libserver/rspamd_control.h"
 #include "libutil/hash.h"
-#include "libutil/map_private.h"
+#include "libserver/maps/map_private.h"
 #include "contrib/uthash/utlist.h"
 #include "unix-std.h"
 

src/libmime/scan_result.c

@@ -510,8 +510,6 @@ rspamd_task_option_safe_copy (struct rspamd_task *task,
 							  gsize *outlen)
 {
 	const gchar *p, *end;
-	off_t r;
-	UChar32 uc;
 
 	p = val;
 	end = val + vlen;

src/libserver/CMakeLists.txt

@@ -18,6 +18,7 @@ SET(LIBRSPAMDSERVERSRC
 				${CMAKE_CURRENT_SOURCE_DIR}/redis_pool.c
 				${CMAKE_CURRENT_SOURCE_DIR}/roll_history.c
 				${CMAKE_CURRENT_SOURCE_DIR}/spf.c
+				${CMAKE_CURRENT_SOURCE_DIR}/ssl_util.c
 				${CMAKE_CURRENT_SOURCE_DIR}/rspamd_symcache.c
 				${CMAKE_CURRENT_SOURCE_DIR}/task.c
 				${CMAKE_CURRENT_SOURCE_DIR}/url.c
@@ -25,7 +26,14 @@ SET(LIBRSPAMDSERVERSRC
 				${CMAKE_CURRENT_SOURCE_DIR}/logger/logger.c
 				${CMAKE_CURRENT_SOURCE_DIR}/logger/logger_file.c
 				${CMAKE_CURRENT_SOURCE_DIR}/logger/logger_syslog.c
-				${CMAKE_CURRENT_SOURCE_DIR}/logger/logger_console.c)
+				${CMAKE_CURRENT_SOURCE_DIR}/logger/logger_console.c
+				${CMAKE_CURRENT_SOURCE_DIR}/http/http_util.c
+				${CMAKE_CURRENT_SOURCE_DIR}/http/http_message.c
+				${CMAKE_CURRENT_SOURCE_DIR}/http/http_connection.c
+				${CMAKE_CURRENT_SOURCE_DIR}/http/http_router.c
+				${CMAKE_CURRENT_SOURCE_DIR}/http/http_context.c
+				${CMAKE_CURRENT_SOURCE_DIR}/maps/map.c
+				${CMAKE_CURRENT_SOURCE_DIR}/maps/map_helpers.c)
 
 # Librspamd-server
 SET(RSPAMD_SERVER ${LIBRSPAMDSERVERSRC} PARENT_SCOPE)

src/libserver/cfg_file.h

@@ -821,6 +821,36 @@ struct rspamd_action *rspamd_config_get_action_by_type (struct rspamd_config *cf
 int rspamd_config_ev_backend_get (struct rspamd_config *cfg);
 const gchar * rspamd_config_ev_backend_to_string (int ev_backend, gboolean *effective);
 
+struct rspamd_external_libs_ctx;
+
+/**
+ * Initialize rspamd libraries
+ */
+struct rspamd_external_libs_ctx *rspamd_init_libs (void);
+
+/**
+ * Reset and initialize decompressor
+ * @param ctx
+ */
+gboolean rspamd_libs_reset_decompression (struct rspamd_external_libs_ctx *ctx);
+
+/**
+ * Reset and initialize compressor
+ * @param ctx
+ */
+gboolean rspamd_libs_reset_compression (struct rspamd_external_libs_ctx *ctx);
+
+/**
+ * Destroy external libraries context
+ */
+void rspamd_deinit_libs (struct rspamd_external_libs_ctx *ctx);
+
+/**
+ * Configure libraries
+ */
+gboolean rspamd_config_libs (struct rspamd_external_libs_ctx *ctx,
+							 struct rspamd_config *cfg);
+
 #define msg_err_config(...) rspamd_default_log_function (G_LOG_LEVEL_CRITICAL, \
         cfg->cfg_pool->tag.tagname, cfg->checksum, \
         G_STRFUNC, \

src/libserver/cfg_utils.c

@@ -21,9 +21,9 @@
 #include "scan_result.h"
 #include "lua/lua_common.h"
 #include "lua/lua_thread_pool.h"
-#include "map.h"
-#include "map_helpers.h"
-#include "map_private.h"
+#include "maps/map.h"
+#include "maps/map_helpers.h"
+#include "maps/map_private.h"
 #include "dynamic_cfg.h"
 #include "utlist.h"
 #include "stat_api.h"
@@ -31,6 +31,26 @@
 #include "libutil/multipattern.h"
 #include "monitored.h"
 #include "ref.h"
+#include "cryptobox.h"
+#include "ssl_util.h"
+#include "contrib/libottery/ottery.h"
+#include "contrib/fastutf8/fastutf8.h"
+
+#define ZSTD_STATIC_LINKING_ONLY
+#include "contrib/zstd/zstd.h"
+#include "contrib/zstd/zdict.h"
+
+#ifdef HAVE_OPENSSL
+#include <openssl/rand.h>
+#include <openssl/err.h>
+#include <openssl/evp.h>
+#include <openssl/ssl.h>
+#include <openssl/conf.h>
+#include <openssl/engine.h>
+#endif
+#ifdef HAVE_LOCALE_H
+#include <locale.h>
+#endif
 #include <math.h>
 
 #define DEFAULT_SCORE 10.0
@@ -2614,4 +2634,357 @@ rspamd_config_ev_backend_to_string (int ev_backend, gboolean *effective)
 	SET_EFFECTIVE (FALSE);
 	return "unknown";
 #undef SET_EFFECTIVE
+}
+
+static void
+rspamd_openssl_maybe_init (void)
+{
+	static gboolean openssl_initialized = FALSE;
+
+	if (!openssl_initialized) {
+		ERR_load_crypto_strings ();
+		SSL_load_error_strings ();
+
+		OpenSSL_add_all_algorithms ();
+		OpenSSL_add_all_digests ();
+		OpenSSL_add_all_ciphers ();
+
+#if OPENSSL_VERSION_NUMBER >= 0x1000104fL && !defined(LIBRESSL_VERSION_NUMBER)
+		ENGINE_load_builtin_engines ();
+#endif
+#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
+		SSL_library_init ();
+#else
+		OPENSSL_init_ssl (0, NULL);
+#endif
+
+#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
+		OPENSSL_config (NULL);
+#endif
+		if (RAND_status () == 0) {
+			guchar seed[128];
+
+			/* Try to use ottery to seed rand */
+			ottery_rand_bytes (seed, sizeof (seed));
+			RAND_seed (seed, sizeof (seed));
+			rspamd_explicit_memzero (seed, sizeof (seed));
+		}
+
+		openssl_initialized = TRUE;
+	}
+}
+
+struct rspamd_external_libs_ctx *
+rspamd_init_libs (void)
+{
+	struct rlimit rlim;
+	struct rspamd_external_libs_ctx *ctx;
+	struct ottery_config *ottery_cfg;
+
+	ctx = g_malloc0 (sizeof (*ctx));
+	ctx->crypto_ctx = rspamd_cryptobox_init ();
+	ottery_cfg = g_malloc0 (ottery_get_sizeof_config ());
+	ottery_config_init (ottery_cfg);
+	ctx->ottery_cfg = ottery_cfg;
+
+	rspamd_openssl_maybe_init ();
+
+	/* Check if we have rdrand */
+	if ((ctx->crypto_ctx->cpu_config & CPUID_RDRAND) == 0) {
+		ottery_config_disable_entropy_sources (ottery_cfg,
+				OTTERY_ENTROPY_SRC_RDRAND);
+#if OPENSSL_VERSION_NUMBER >= 0x1000104fL && !defined(LIBRESSL_VERSION_NUMBER)
+		RAND_set_rand_engine (NULL);
+#endif
+	}
+
+	/* Configure utf8 library */
+	guint utf8_flags = 0;
+
+	if ((ctx->crypto_ctx->cpu_config & CPUID_SSE41)) {
+		utf8_flags |= RSPAMD_FAST_UTF8_FLAG_SSE41;
+	}
+	if ((ctx->crypto_ctx->cpu_config & CPUID_AVX2)) {
+		utf8_flags |= RSPAMD_FAST_UTF8_FLAG_AVX2;
+	}
+
+	rspamd_fast_utf8_library_init (utf8_flags);
+
+	g_assert (ottery_init (ottery_cfg) == 0);
+
+#ifdef HAVE_LOCALE_H
+	if (getenv ("LANG") == NULL) {
+		setlocale (LC_ALL, "C");
+		setlocale (LC_CTYPE, "C");
+		setlocale (LC_MESSAGES, "C");
+		setlocale (LC_TIME, "C");
+	}
+	else {
+		/* Just set the default locale */
+		setlocale (LC_ALL, "");
+		/* But for some issues we still want C locale */
+		setlocale (LC_NUMERIC, "C");
+	}
+#endif
+
+	ctx->ssl_ctx = rspamd_init_ssl_ctx ();
+	ctx->ssl_ctx_noverify = rspamd_init_ssl_ctx_noverify ();
+	rspamd_random_seed_fast ();
+
+	/* Set stack size for pcre */
+	getrlimit (RLIMIT_STACK, &rlim);
+	rlim.rlim_cur = 100 * 1024 * 1024;
+	rlim.rlim_max = rlim.rlim_cur;
+	setrlimit (RLIMIT_STACK, &rlim);
+
+	ctx->local_addrs = rspamd_inet_library_init ();
+	REF_INIT_RETAIN (ctx, rspamd_deinit_libs);
+
+	return ctx;
+}
+
+static struct zstd_dictionary *
+rspamd_open_zstd_dictionary (const char *path)
+{
+	struct zstd_dictionary *dict;
+
+	dict = g_malloc0 (sizeof (*dict));
+	dict->dict = rspamd_file_xmap (path, PROT_READ, &dict->size, TRUE);
+
+	if (dict->dict == NULL) {
+		g_free (dict);
+
+		return NULL;
+	}
+
+	dict->id = ZDICT_getDictID (dict->dict, dict->size);
+
+	if (dict->id == 0) {
+		g_free (dict);
+
+		return NULL;
+	}
+
+	return dict;
+}
+
+static void
+rspamd_free_zstd_dictionary (struct zstd_dictionary *dict)
+{
+	if (dict) {
+		munmap (dict->dict, dict->size);
+		g_free (dict);
+	}
+}
+
+gboolean
+rspamd_config_libs (struct rspamd_external_libs_ctx *ctx,
+					struct rspamd_config *cfg)
+{
+	static const char secure_ciphers[] = "HIGH:!aNULL:!kRSA:!PSK:!SRP:!MD5:!RC4";
+	size_t r;
+	gboolean ret = TRUE;
+
+	g_assert (cfg != NULL);
+
+	if (ctx != NULL) {
+		if (cfg->local_addrs) {
+			rspamd_config_radix_from_ucl (cfg, cfg->local_addrs,
+					"Local addresses",
+					ctx->local_addrs,
+					NULL,
+					NULL);
+		}
+
+		rspamd_free_zstd_dictionary (ctx->in_dict);
+		rspamd_free_zstd_dictionary (ctx->out_dict);
+
+		if (ctx->out_zstream) {
+			ZSTD_freeCStream (ctx->out_zstream);
+			ctx->out_zstream = NULL;
+		}
+
+		if (ctx->in_zstream) {
+			ZSTD_freeDStream (ctx->in_zstream);
+			ctx->in_zstream = NULL;
+		}
+
+		if (cfg->zstd_input_dictionary) {
+			ctx->in_dict = rspamd_open_zstd_dictionary (
+					cfg->zstd_input_dictionary);
+
+			if (ctx->in_dict == NULL) {
+				msg_err_config ("cannot open zstd dictionary in %s",
+						cfg->zstd_input_dictionary);
+			}
+		}
+		if (cfg->zstd_output_dictionary) {
+			ctx->out_dict = rspamd_open_zstd_dictionary (
+					cfg->zstd_output_dictionary);
+
+			if (ctx->out_dict == NULL) {
+				msg_err_config ("cannot open zstd dictionary in %s",
+						cfg->zstd_output_dictionary);
+			}
+		}
+
+		if (cfg->fips_mode) {
+#ifdef HAVE_FIPS_MODE
+			int mode = FIPS_mode ();
+			unsigned long err = (unsigned long)-1;
+
+			/* Toggle FIPS mode */
+			if (mode == 0) {
+				if (FIPS_mode_set (1) != 1) {
+					err = ERR_get_error ();
+				}
+			}
+			else {
+				msg_info_config ("OpenSSL FIPS mode is already enabled");
+			}
+
+			if (err != (unsigned long)-1) {
+				msg_err_config ("FIPS_mode_set failed: %s",
+						ERR_error_string (err, NULL));
+				ret = FALSE;
+			}
+			else {
+				msg_info_config ("OpenSSL FIPS mode is enabled");
+			}
+#else
+			msg_warn_config ("SSL FIPS mode is enabled but not supported by OpenSSL library!");
+#endif
+		}
+
+		if (cfg->ssl_ca_path) {
+			if (SSL_CTX_load_verify_locations (ctx->ssl_ctx, cfg->ssl_ca_path,
+					NULL) != 1) {
+				msg_err_config ("cannot load CA certs from %s: %s",
+						cfg->ssl_ca_path,
+						ERR_error_string (ERR_get_error (), NULL));
+			}
+		}
+		else {
+			msg_debug_config ("ssl_ca_path is not set, using default CA path");
+			SSL_CTX_set_default_verify_paths (ctx->ssl_ctx);
+		}
+
+		if (cfg->ssl_ciphers) {
+			if (SSL_CTX_set_cipher_list (ctx->ssl_ctx, cfg->ssl_ciphers) != 1) {
+				msg_err_config (
+						"cannot set ciphers set to %s: %s; fallback to %s",
+						cfg->ssl_ciphers,
+						ERR_error_string (ERR_get_error (), NULL),
+						secure_ciphers);
+				/* Default settings */
+				SSL_CTX_set_cipher_list (ctx->ssl_ctx, secure_ciphers);
+			}
+		}
+
+		/* Init decompression */
+		ctx->in_zstream = ZSTD_createDStream ();
+		r = ZSTD_initDStream (ctx->in_zstream);
+
+		if (ZSTD_isError (r)) {
+			msg_err ("cannot init decompression stream: %s",
+					ZSTD_getErrorName (r));
+			ZSTD_freeDStream (ctx->in_zstream);
+			ctx->in_zstream = NULL;
+		}
+
+		/* Init compression */
+		ctx->out_zstream = ZSTD_createCStream ();
+		r = ZSTD_initCStream (ctx->out_zstream, 1);
+
+		if (ZSTD_isError (r)) {
+			msg_err ("cannot init compression stream: %s",
+					ZSTD_getErrorName (r));
+			ZSTD_freeCStream (ctx->out_zstream);
+			ctx->out_zstream = NULL;
+		}
+#ifdef HAVE_CBLAS
+		openblas_set_num_threads (cfg->max_blas_threads);
+#endif
+	}
+
+	return ret;
+}
+
+gboolean
+rspamd_libs_reset_decompression (struct rspamd_external_libs_ctx *ctx)
+{
+	gsize r;
+
+	if (ctx->in_zstream == NULL) {
+		return FALSE;
+	}
+	else {
+		r = ZSTD_resetDStream (ctx->in_zstream);
+
+		if (ZSTD_isError (r)) {
+			msg_err ("cannot init decompression stream: %s",
+					ZSTD_getErrorName (r));
+			ZSTD_freeDStream (ctx->in_zstream);
+			ctx->in_zstream = NULL;
+
+			return FALSE;
+		}
+	}
+
+	return TRUE;
+}
+
+gboolean
+rspamd_libs_reset_compression (struct rspamd_external_libs_ctx *ctx)
+{
+	gsize r;
+
+	if (ctx->out_zstream == NULL) {
+		return FALSE;
+	}
+	else {
+		/* Dictionary will be reused automatically if specified */
+		r = ZSTD_resetCStream (ctx->out_zstream, 0);
+
+		if (ZSTD_isError (r)) {
+			msg_err ("cannot init compression stream: %s",
+					ZSTD_getErrorName (r));
+			ZSTD_freeCStream (ctx->out_zstream);
+			ctx->out_zstream = NULL;
+
+			return FALSE;
+		}
+	}
+
+	return TRUE;
+}
+
+void
+rspamd_deinit_libs (struct rspamd_external_libs_ctx *ctx)
+{
+	if (ctx != NULL) {
+		g_free (ctx->ottery_cfg);
+
+#ifdef HAVE_OPENSSL
+		EVP_cleanup ();
+		ERR_free_strings ();
+		SSL_CTX_free (ctx->ssl_ctx);
+		SSL_CTX_free (ctx->ssl_ctx_noverify);
+#endif
+		rspamd_inet_library_destroy ();
+		rspamd_free_zstd_dictionary (ctx->in_dict);
+		rspamd_free_zstd_dictionary (ctx->out_dict);
+
+		if (ctx->out_zstream) {
+			ZSTD_freeCStream (ctx->out_zstream);
+		}
+
+		if (ctx->in_zstream) {
+			ZSTD_freeDStream (ctx->in_zstream);
+		}
+
+		rspamd_cryptobox_deinit (ctx->crypto_ctx);
+
+		g_free (ctx);
+	}
 }

src/libserver/dynamic_cfg.c

@@ -15,7 +15,7 @@
  */
 #include "config.h"
 #include "rspamd.h"
-#include "map.h"
+#include "libserver/maps/map.h"
 #include "scan_result.h"
 #include "dynamic_cfg.h"
 #include "unix-std.h"

src/libutil/http_connection.c → src/libserver/http/http_connection.c

@@ -26,7 +26,7 @@
 #include "keypair_private.h"
 #include "cryptobox.h"
 #include "libutil/libev_helper.h"
-#include "libutil/ssl_util.h"
+#include "libserver/ssl_util.h"
 #include "libserver/url.h"
 
 #include "contrib/mumhash/mum.h"

src/libutil/http_message.c → src/libserver/http/http_message.c

@@ -14,8 +14,8 @@
  * limitations under the License.
  */
 #include "http_message.h"
-#include "libutil/http_connection.h"
-#include "libutil/http_private.h"
+#include "http_connection.h"
+#include "http_private.h"
 #include "libutil/printf.h"
 #include "libserver/logger.h"
 #include "utlist.h"

src/libutil/http_router.c → src/libserver/http/http_router.c

@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-#include "libutil/http_router.h"
-#include "libutil/http_connection.h"
-#include "libutil/http_private.h"
+#include "http_router.h"
+#include "http_connection.h"
+#include "http_private.h"
 #include "libutil/regexp.h"
 #include "libutil/printf.h"
 #include "libserver/logger.h"

src/libutil/http_util.c → src/libserver/http/http_util.c

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-#include "libutil/http_util.h"
+#include "libserver/http/http_util.h"
 #include "libutil/printf.h"
 #include "libutil/util.h"
 

src/libserver/logger/logger.c

@@ -16,8 +16,8 @@
 #include "config.h"
 #include "logger.h"
 #include "rspamd.h"
-#include "map.h"
-#include "map_helpers.h"
+#include "libserver/maps/map.h"
+#include "libserver/maps/map_helpers.h"
 #include "ottery.h"
 #include "unix-std.h"
 #include "logger_private.h"

src/libserver/logger/logger_file.c

@@ -89,7 +89,6 @@ direct_write_log_line (rspamd_logger_t *rspamd_log,
 					   gboolean is_iov,
 					   gint level_flags)
 {
-	gchar errmsg[128];
 	struct iovec *iov;
 	const gchar *line;
 	glong r;

src/libutil/map.c → src/libserver/maps/map.c

@@ -20,8 +20,8 @@
 #include "config.h"
 #include "map.h"
 #include "map_private.h"
-#include "http_connection.h"
-#include "http_private.h"
+#include "libserver/http/http_connection.h"
+#include "libserver/http/http_private.h"
 #include "rspamd.h"
 #include "contrib/zstd/zstd.h"
 #include "contrib/libev/ev.h"

src/libserver/milter.c

@@ -22,8 +22,8 @@
 #include "unix-std.h"
 #include "logger.h"
 #include "ottery.h"
-#include "libutil/http_connection.h"
-#include "libutil/http_private.h"
+#include "libserver/http/http_connection.h"
+#include "libserver/http/http_private.h"
 #include "libserver/protocol_internal.h"
 #include "libserver/cfg_file_private.h"
 #include "libmime/scan_result.h"

src/libserver/protocol.c

@@ -17,7 +17,7 @@
 #include "rspamd.h"
 #include "message.h"
 #include "utlist.h"
-#include "http_private.h"
+#include "libserver/http/http_private.h"
 #include "worker_private.h"
 #include "libserver/cfg_file_private.h"
 #include "libmime/scan_result_private.h"

src/libserver/protocol.h

@@ -8,7 +8,7 @@
 
 #include "config.h"
 #include "scan_result.h"
-#include "http_connection.h"
+#include "libserver/http/http_connection.h"
 #include "task.h"
 
 #ifdef  __cplusplus

src/libserver/rspamd_control.c

@@ -17,8 +17,8 @@
 #include "rspamd.h"
 #include "rspamd_control.h"
 #include "worker_util.h"
-#include "libutil/http_connection.h"
-#include "libutil/http_private.h"
+#include "libserver/http/http_connection.h"
+#include "libserver/http/http_private.h"
 #include "libutil/libev_helper.h"
 #include "unix-std.h"
 #include "utlist.h"

src/libutil/ssl_util.c → src/libserver/ssl_util.c

@@ -922,3 +922,52 @@ rspamd_ssl_connection_free (struct rspamd_ssl_connection *conn)
 		}
 	}
 }
+
+gpointer
+rspamd_init_ssl_ctx (void)
+{
+	SSL_CTX *ssl_ctx;
+	gint ssl_options;
+
+	rspamd_openssl_maybe_init ();
+
+	ssl_ctx = SSL_CTX_new (SSLv23_method ());
+	SSL_CTX_set_verify (ssl_ctx, SSL_VERIFY_PEER, NULL);
+	SSL_CTX_set_verify_depth (ssl_ctx, 4);
+	ssl_options = SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3;
+
+#ifdef SSL_OP_NO_COMPRESSION
+	ssl_options |= SSL_OP_NO_COMPRESSION;
+#elif OPENSSL_VERSION_NUMBER >= 0x00908000L
+	sk_SSL_COMP_zero (SSL_COMP_get_compression_methods ());
+#endif
+
+	SSL_CTX_set_options (ssl_ctx, ssl_options);
+
+	return ssl_ctx;
+}
+
+gpointer rspamd_init_ssl_ctx_noverify (void)
+{
+	SSL_CTX *ssl_ctx_noverify;
+	gint ssl_options;
+
+	rspamd_openssl_maybe_init ();
+
+	ssl_options = SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3;
+
+#ifdef SSL_OP_NO_COMPRESSION
+	ssl_options |= SSL_OP_NO_COMPRESSION;
+#elif OPENSSL_VERSION_NUMBER >= 0x00908000L
+	sk_SSL_COMP_zero (SSL_COMP_get_compression_methods ());
+#endif
+
+	ssl_ctx_noverify = SSL_CTX_new (SSLv23_method ());
+	SSL_CTX_set_verify (ssl_ctx_noverify, SSL_VERIFY_NONE, NULL);
+	SSL_CTX_set_options (ssl_ctx_noverify, ssl_options);
+#ifdef SSL_SESS_CACHE_BOTH
+	SSL_CTX_set_session_cache_mode (ssl_ctx_noverify, SSL_SESS_CACHE_BOTH);
+#endif
+
+	return ssl_ctx_noverify;
+}

src/libutil/ssl_util.h → src/libserver/ssl_util.h

@@ -94,6 +94,9 @@ gssize rspamd_ssl_writev (struct rspamd_ssl_connection *conn, struct iovec *iov,
  */
 void rspamd_ssl_connection_free (struct rspamd_ssl_connection *conn);
 
+gpointer rspamd_init_ssl_ctx (void);
+gpointer rspamd_init_ssl_ctx_noverify (void);
+
 #ifdef  __cplusplus
 }
 #endif

src/libserver/task.h

@@ -17,7 +17,7 @@
 #define TASK_H_
 
 #include "config.h"
-#include "http_connection.h"
+#include "libserver/http/http_connection.h"
 #include "async_session.h"
 #include "util.h"
 #include "mem_pool.h"

src/libserver/worker_util.c

@@ -21,15 +21,12 @@
 #include "utlist.h"
 #include "ottery.h"
 #include "rspamd_control.h"
-#include "libutil/map.h"
-#include "libutil/map_private.h"
-#include "libutil/http_private.h"
-#include "libutil/http_router.h"
+#include "libserver/maps/map.h"
+#include "libserver/maps/map_private.h"
+#include "libserver/http/http_private.h"
+#include "libserver/http/http_router.h"
 #include "libutil/rrd.h"
 
-#ifdef WITH_GPERF_TOOLS
-#include <gperftools/profiler.h>
-#endif
 /* sys/resource.h */
 #ifdef HAVE_SYS_RESOURCE_H
 #include <sys/resource.h>
@@ -218,9 +215,6 @@ rspamd_worker_on_delayed_shutdown (EV_P_ ev_timer *w, int revents)
 	worker->state = rspamd_worker_wanna_die;
 	ev_timer_stop (EV_A_ w);
 	ev_break (loop, EVBREAK_ALL);
-#ifdef WITH_GPERF_TOOLS
-	ProfilerStop ();
-#endif
 }
 
 static void
@@ -481,13 +475,6 @@ rspamd_prepare_worker (struct rspamd_worker *worker, const char *name,
 	struct rspamd_worker_listen_socket *ls;
 	struct rspamd_worker_accept_event *accept_ev;
 
-#ifdef WITH_PROFILER
-	extern void _start (void), etext (void);
-	monstartup ((u_long) & _start, (u_long) & etext);
-#endif
-
-	gperf_profiler_init (worker->srv->cfg, name);
-
 	worker->signal_events = g_hash_table_new_full (g_direct_hash, g_direct_equal,
 			NULL, rspamd_sigh_free);
 

src/libserver/worker_util.h

@@ -18,7 +18,7 @@
 
 #include "config.h"
 #include "util.h"
-#include "http_connection.h"
+#include "libserver/http/http_connection.h"
 #include "rspamd.h"
 
 #ifdef  __cplusplus

src/libutil/CMakeLists.txt

@@ -5,13 +5,6 @@ SET(LIBRSPAMDUTILSRC
 				${CMAKE_CURRENT_SOURCE_DIR}/expression.c
 				${CMAKE_CURRENT_SOURCE_DIR}/fstring.c
 				${CMAKE_CURRENT_SOURCE_DIR}/hash.c
-				${CMAKE_CURRENT_SOURCE_DIR}/http_util.c
-				${CMAKE_CURRENT_SOURCE_DIR}/http_message.c
-				${CMAKE_CURRENT_SOURCE_DIR}/http_connection.c
-				${CMAKE_CURRENT_SOURCE_DIR}/http_router.c
-				${CMAKE_CURRENT_SOURCE_DIR}/http_context.c
-				${CMAKE_CURRENT_SOURCE_DIR}/map.c
-				${CMAKE_CURRENT_SOURCE_DIR}/map_helpers.c
 				${CMAKE_CURRENT_SOURCE_DIR}/mem_pool.c
 				${CMAKE_CURRENT_SOURCE_DIR}/printf.c
 				${CMAKE_CURRENT_SOURCE_DIR}/radix.c
@@ -23,7 +16,6 @@ SET(LIBRSPAMDUTILSRC
 				${CMAKE_CURRENT_SOURCE_DIR}/upstream.c
 				${CMAKE_CURRENT_SOURCE_DIR}/util.c
 				${CMAKE_CURRENT_SOURCE_DIR}/heap.c
-				${CMAKE_CURRENT_SOURCE_DIR}/multipattern.c
-				${CMAKE_CURRENT_SOURCE_DIR}/ssl_util.c)
+				${CMAKE_CURRENT_SOURCE_DIR}/multipattern.c)
 # Rspamdutil
 SET(RSPAMD_UTIL ${LIBRSPAMDUTILSRC} PARENT_SCOPE)

src/libutil/addr.c

@@ -16,7 +16,10 @@
 #include "config.h"
 #include "addr.h"
 #include "util.h"
-#include "map_helpers.h"
+/*
+ * TODO: fix this cross dependency!
+ */
+#include "libserver/maps/map_helpers.h"
 #include "logger.h"
 #include "cryptobox.h"
 #include "unix-std.h"

src/libutil/util.h

@@ -20,8 +20,6 @@ extern "C" {
 #endif
 
 struct rspamd_config;
-struct rspamd_main;
-struct workq;
 
 /**
  * Create generic socket
@@ -64,32 +62,12 @@ gint rspamd_socket_unix (const gchar *,
 gint rspamd_socket (const gchar *credits, guint16 port, gint type,
 					gboolean async, gboolean is_server, gboolean try_resolve);
 
-/**
- * Make a universal sockets
- * @param credits host, ip or path to unix socket (several items may be separated by ',')
- * @param port port (used for network sockets)
- * @param type type of socket (SO_STREAM or SO_DGRAM)
- * @param async make this socket asynced
- * @param is_server make this socket as server socket
- * @param try_resolve try name resolution for a socket (BLOCKING)
- */
-GList *rspamd_sockets_list (const gchar *credits,
-							guint16 port,
-							gint type,
-							gboolean async,
-							gboolean is_server,
-							gboolean try_resolve);
 
 /*
  * Create socketpair
  */
 gboolean rspamd_socketpair (gint pair[2], gboolean is_stream);
 
-/*
- * Write pid to file
- */
-gint rspamd_write_pid (struct rspamd_main *);
-
 /*
  * Make specified socket non-blocking
  */
@@ -118,17 +96,12 @@ void rspamd_signals_init (struct sigaction *sa, void (*sig_handler) (gint,
 void rspamd_signals_init (struct sigaction *sa, void (*sig_handler)(gint));
 #endif
 
-/*
- * Send specified signal to each worker
- */
-void rspamd_pass_signal (GHashTable *, gint);
-
 #ifndef HAVE_SETPROCTITLE
 
 /*
  * Process title utility functions
  */
-gint init_title (struct rspamd_main *, gint argc, gchar *argv[], gchar *envp[]);
+gint init_title (rspamd_mempool_t *pool, gint argc, gchar *argv[], gchar *envp[]);
 
 gint setproctitle (const gchar *fmt, ...);
 
@@ -187,13 +160,6 @@ gboolean rspamd_file_lock (gint fd, gboolean async);
 
 gboolean rspamd_file_unlock (gint fd, gboolean async);
 
-/*
- * Google perf-tools initialization function
- */
-void gperf_profiler_init (struct rspamd_config *cfg, const gchar *descr);
-
-void gperf_profiler_stop (void);
-
 /*
  * Workarounds for older versions of glib
  */
@@ -354,39 +320,6 @@ void rspamd_gerror_free_maybe (gpointer p);
  */
 void rspamd_gstring_free_soft (gpointer p);
 
-struct rspamd_external_libs_ctx;
-
-/**
- * Initialize rspamd libraries
- */
-struct rspamd_external_libs_ctx *rspamd_init_libs (void);
-
-gpointer rspamd_init_ssl_ctx (void);
-
-gpointer rspamd_init_ssl_ctx_noverify (void);
-
-/**
- * Configure libraries
- */
-gboolean rspamd_config_libs (struct rspamd_external_libs_ctx *ctx,
-							 struct rspamd_config *cfg);
-
-/**
- * Reset and initialize decompressor
- * @param ctx
- */
-gboolean rspamd_libs_reset_decompression (struct rspamd_external_libs_ctx *ctx);
-
-/**
- * Reset and initialize compressor
- * @param ctx
- */
-gboolean rspamd_libs_reset_compression (struct rspamd_external_libs_ctx *ctx);
-
-/**
- * Destroy external libraries context
- */
-void rspamd_deinit_libs (struct rspamd_external_libs_ctx *ctx);
 
 /**
  * Returns some statically initialized random hash seed
@@ -545,6 +478,25 @@ double rspamd_set_counter_ema (struct rspamd_counter_data *cd,
 double rspamd_set_counter (struct rspamd_counter_data *cd,
 						   gdouble value);
 
+enum rspamd_pbkdf_version_id {
+	RSPAMD_PBKDF_ID_V1 = 1,
+	RSPAMD_PBKDF_ID_V2 = 2,
+	RSPAMD_PBKDF_ID_MAX
+};
+
+struct rspamd_controller_pbkdf {
+	const char *name;
+	const char *alias;
+	const char *description;
+	int type; /* enum rspamd_cryptobox_pbkdf_type */
+	gint id;
+	guint complexity;
+	gsize salt_len;
+	gsize key_len;
+};
+
+extern const struct rspamd_controller_pbkdf pbkdf_list[];
+
 #ifdef  __cplusplus
 }
 #endif

src/lua/lua_http.c

@@ -15,7 +15,7 @@
  */
 #include "lua_common.h"
 #include "lua_thread_pool.h"
-#include "http_private.h"
+#include "libserver/http/http_private.h"
 #include "ref.h"
 #include "unix-std.h"
 #include "zlib.h"

src/lua/lua_logger.c

@@ -14,8 +14,8 @@
  * limitations under the License.
  */
 #include "lua_common.h"
-#include "libutil/map.h"
-#include "libutil/map_private.h"
+#include "libserver/maps/map.h"
+#include "libserver/maps/map_private.h"
 
 /***
  * @module rspamd_logger

src/lua/lua_map.c

@@ -15,9 +15,9 @@
  */
 
 #include "lua_common.h"
-#include "libutil/map.h"
-#include "libutil/map_helpers.h"
-#include "libutil/map_private.h"
+#include "libserver/maps/map.h"
+#include "libserver/maps/map_helpers.h"
+#include "libserver/maps/map_private.h"
 #include "contrib/libucl/lua_ucl.h"
 
 /***

src/lua/lua_task.c

@@ -26,7 +26,7 @@
 #include "libserver/cfg_file_private.h"
 #include "libmime/scan_result_private.h"
 #include "libstat/stat_api.h"
-#include "libutil/map_helpers.h"
+#include "libserver/maps/map_helpers.h"
 
 #include <math.h>
 

src/lua/lua_tcp.c

@@ -15,7 +15,7 @@
  */
 #include "lua_common.h"
 #include "lua_thread_pool.h"
-#include "libutil/ssl_util.h"
+#include "libserver/ssl_util.h"
 #include "utlist.h"
 #include "unix-std.h"
 #include <math.h>

src/plugins/dkim_check.c

@@ -34,8 +34,8 @@
 #include "libmime/message.h"
 #include "libserver/dkim.h"
 #include "libutil/hash.h"
-#include "libutil/map.h"
-#include "libutil/map_helpers.h"
+#include "libserver/maps/map.h"
+#include "libserver/maps/map_helpers.h"
 #include "rspamd.h"
 #include "utlist.h"
 #include "unix-std.h"

src/plugins/fuzzy_check.c

@@ -32,8 +32,8 @@
 
 #include "config.h"
 #include "libmime/message.h"
-#include "libutil/map.h"
-#include "libutil/map_helpers.h"
+#include "libserver/maps/map.h"
+#include "libserver/maps/map_helpers.h"
 #include "libmime/images.h"
 #include "libserver/worker_util.h"
 #include "libserver/mempool_vars_internal.h"
@@ -42,11 +42,10 @@
 #include "ottery.h"
 #include "lua/lua_common.h"
 #include "unix-std.h"
-#include "libutil/http_private.h"
-#include "libutil/http_router.h"
+#include "libserver/http/http_private.h"
+#include "libserver/http/http_router.h"
 #include "libstat/stat_api.h"
 #include <math.h>
-#include <src/libmime/message.h>
 #include "libutil/libev_helper.h"
 
 #define DEFAULT_SYMBOL "R_FUZZY_HASH"

src/plugins/regexp.c

@@ -22,7 +22,7 @@
 #include "libmime/message.h"
 #include "expression.h"
 #include "mime_expressions.h"
-#include "libutil/map.h"
+#include "libserver/maps/map.h"
 #include "lua/lua_common.h"
 
 static const guint64 rspamd_regexp_cb_magic = 0xca9d9649fc3e2659ULL;

src/rspamadm/control.c

@@ -17,8 +17,8 @@
 #include "rspamadm.h"
 #include "cryptobox.h"
 #include "printf.h"
-#include "libutil/http_connection.h"
-#include "libutil/http_private.h"
+#include "libserver/http/http_connection.h"
+#include "libserver/http/http_private.h"
 #include "addr.h"
 #include "unix-std.h"
 #include "contrib/libev/ev.h"

src/rspamadm/lua_repl.c

@@ -16,9 +16,9 @@
 
 #include "config.h"
 #include "rspamadm.h"
-#include "libutil/http_connection.h"
-#include "libutil/http_private.h"
-#include "libutil/http_router.h"
+#include "libserver/http/http_connection.h"
+#include "libserver/http/http_private.h"
+#include "libserver/http/http_router.h"
 #include "printf.h"
 #include "lua/lua_common.h"
 #include "lua/lua_thread_pool.h"

src/rspamadm/rspamadm.c

@@ -475,7 +475,6 @@ main (gint argc, gchar **argv, gchar **env)
 	cfg->compiled_modules = modules;
 	cfg->compiled_workers = workers;
 
-	gperf_profiler_init (cfg, "rspamadm");
 	setproctitle ("rspamdadm");
 
 	L = cfg->lua_state;

src/rspamd.c

@@ -15,7 +15,7 @@
  */
 #include "config.h"
 #include "rspamd.h"
-#include "libutil/map.h"
+#include "libserver/maps/map.h"
 #include "lua/lua_common.h"
 #include "libserver/worker_util.h"
 #include "libserver/rspamd_control.h"
@@ -45,9 +45,6 @@
 #ifdef HAVE_LIBUTIL_H
 #include <libutil.h>
 #endif
-#ifdef WITH_GPERF_TOOLS
-#include <gperftools/profiler.h>
-#endif
 #ifdef HAVE_STROPS_H
 #include <stropts.h>
 #endif
@@ -201,6 +198,35 @@ read_cmd_line (gint *argc, gchar ***argv, struct rspamd_config *cfg)
 	g_option_context_free (context);
 }
 
+static int
+rspamd_write_pid (struct rspamd_main *main)
+{
+	pid_t pid;
+
+	if (main->cfg->pid_file == NULL) {
+		return -1;
+	}
+	main->pfh = rspamd_pidfile_open (main->cfg->pid_file, 0644, &pid);
+
+	if (main->pfh == NULL) {
+		return -1;
+	}
+
+	if (main->is_privilleged) {
+		/* Force root user as owner of pid file */
+#ifdef HAVE_PIDFILE_FILENO
+		if (fchown (pidfile_fileno (main->pfh), 0, 0) == -1) {
+#else
+		if (fchown (main->pfh->pf_fd, 0, 0) == -1) {
+#endif
+		}
+	}
+
+	rspamd_pidfile_write (main->pfh);
+
+	return 0;
+}
+
 /* Detect privilleged mode */
 static void
 detect_priv (struct rspamd_main *rspamd_main)
@@ -491,6 +517,21 @@ systemd_get_socket (struct rspamd_main *rspamd_main, gint number)
 	return result;
 }
 
+static void
+pass_signal_cb (gpointer key, gpointer value, gpointer ud)
+{
+	struct rspamd_worker *cur = value;
+	gint signo = GPOINTER_TO_INT (ud);
+
+	kill (cur->pid, signo);
+}
+
+static void
+rspamd_pass_signal (GHashTable * workers, gint signo)
+{
+	g_hash_table_foreach (workers, pass_signal_cb, GINT_TO_POINTER (signo));
+}
+
 static inline uintptr_t
 make_listen_key (struct rspamd_worker_bind_conf *cf)
 {
@@ -1188,7 +1229,7 @@ main (gint argc, gchar **argv, gchar **env)
 	}
 
 #ifndef HAVE_SETPROCTITLE
-	init_title (rspamd_main, argc, argv, env);
+	init_title (rspamd_main->server_pool, argc, argv, env);
 #endif
 
 	rspamd_main->cfg->libs_ctx = rspamd_init_libs ();
@@ -1278,8 +1319,6 @@ main (gint argc, gchar **argv, gchar **env)
 	rspamd_main->history = rspamd_roll_history_new (rspamd_main->server_pool,
 			rspamd_main->cfg->history_rows, rspamd_main->cfg);
 
-	gperf_profiler_init (rspamd_main->cfg, "main");
-
 	msg_info_main ("rspamd "
 			RVERSION
 			" is starting, build id: "
@@ -1361,9 +1400,6 @@ main (gint argc, gchar **argv, gchar **env)
 			rspamd_main->cfg->history_file);
 	}
 
-#if defined(WITH_GPERF_TOOLS)
-	ProfilerStop ();
-#endif
 	/* Spawn workers */
 	rspamd_main->workers = g_hash_table_new (g_direct_hash, g_direct_equal);
 

src/rspamd.h

@@ -22,7 +22,7 @@
 #include "libutil/mem_pool.h"
 #include "libutil/util.h"
 #include "libserver/logger.h"
-#include "libutil/http_connection.h"
+#include "libserver/http/http_connection.h"
 #include "libutil/upstream.h"
 #include "libutil/radix.h"
 #include "libserver/cfg_file.h"
@@ -56,6 +56,8 @@
 extern "C" {
 #endif
 
+struct rspamd_main;
+
 enum rspamd_worker_flags {
 	RSPAMD_WORKER_HAS_SOCKET = (1 << 0),
 	RSPAMD_WORKER_UNIQUE = (1 << 1),
@@ -152,17 +154,6 @@ struct rspamd_worker_signal_handler {
 	struct rspamd_worker_signal_handler_elt *cb;
 };
 
-struct rspamd_controller_pbkdf {
-	const char *name;
-	const char *alias;
-	const char *description;
-	enum rspamd_cryptobox_pbkdf_type type;
-	gint id;
-	guint complexity;
-	gsize salt_len;
-	gsize key_len;
-};
-
 /**
  * Common structure representing C module context
  */
@@ -387,14 +378,6 @@ void register_custom_controller_command (const gchar *name,
 										 gboolean privilleged,
 										 gboolean require_message);
 
-enum rspamd_pbkdf_version_id {
-	RSPAMD_PBKDF_ID_V1 = 1,
-	RSPAMD_PBKDF_ID_V2 = 2,
-	RSPAMD_PBKDF_ID_MAX
-};
-
-extern const struct rspamd_controller_pbkdf pbkdf_list[];
-
 #ifdef  __cplusplus
 }
 #endif

src/rspamd_proxy.c

@@ -15,10 +15,10 @@
  */
 #include "config.h"
 #include "libutil/util.h"
-#include "libutil/map.h"
+#include "libserver/maps/map.h"
 #include "libutil/upstream.h"
-#include "libutil/http_connection.h"
-#include "libutil/http_private.h"
+#include "libserver/http/http_connection.h"
+#include "libserver/http/http_private.h"
 #include "libserver/protocol.h"
 #include "libserver/protocol_internal.h"
 #include "libserver/cfg_file.h"

src/worker.c

@@ -19,7 +19,7 @@
 
 #include "config.h"
 #include "libutil/util.h"
-#include "libutil/map.h"
+#include "libserver/maps/map.h"
 #include "libutil/upstream.h"
 #include "libserver/protocol.h"
 #include "libserver/cfg_file.h"
@@ -31,7 +31,7 @@
 #include "libserver/worker_util.h"
 #include "libserver/rspamd_control.h"
 #include "worker_private.h"
-#include "libutil/http_private.h"
+#include "libserver/http/http_private.h"
 #include "libserver/cfg_file_private.h"
 #include <math.h>
 #include "unix-std.h"

test/rspamd_http_test.c

@@ -229,9 +229,7 @@ rspamd_http_start_servers (pid_t *sfd, rspamd_inet_addr_t *addr,
 		g_assert (sfd[i] != -1);
 
 		if (sfd[i] == 0) {
-			gperf_profiler_init (NULL, "plain-http-server");
 			rspamd_http_server_func (fd, "/tmp/", addr, serv_key, c);
-			gperf_profiler_stop ();
 			exit (EXIT_SUCCESS);
 		}
 	}

utils/rspamd_http_server.c

@@ -209,9 +209,7 @@ rspamd_http_start_servers (pid_t *sfd, rspamd_inet_addr_t *addr)
 		g_assert (sfd[i] != -1);
 
 		if (sfd[i] == 0) {
-			gperf_profiler_init (NULL, "http-server");
 			rspamd_http_server_func (fd, addr);
-			gperf_profiler_stop ();
 			exit (EXIT_SUCCESS);
 		}
 	}