/* * Copyright 2024 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "config.h" #include "lua/lua_common.h" #include "lua/lua_thread_pool.h" #include "cfg_file.h" #include "rspamd.h" #include "cfg_file_private.h" #include "maps/map.h" #include "maps/map_helpers.h" #include "maps/map_private.h" #include "dynamic_cfg.h" #include "utlist.h" #include "stat_api.h" #include "unix-std.h" #include "libutil/multipattern.h" #include "monitored.h" #include "ref.h" #include "cryptobox.h" #include "ssl_util.h" #include "contrib/libottery/ottery.h" #include "contrib/fastutf8/fastutf8.h" #ifdef SYS_ZSTD #include "zstd.h" #else #define ZSTD_STATIC_LINKING_ONLY #include "contrib/zstd/zstd.h" #endif #ifdef HAVE_OPENSSL #include #include #include #include #include #endif #ifdef HAVE_LOCALE_H #include #endif #ifdef HAVE_SYS_RESOURCE_H #include #endif #include #include "libserver/composites/composites.h" #include "blas-config.h" #include #include #include #include "fmt/base.h" #include "cxx/util.hxx" #include "frozen/unordered_map.h" #include "frozen/string.h" #include "contrib/expected/expected.hpp" #include "contrib/ankerl/unordered_dense.h" #define DEFAULT_SCORE 10.0 #define DEFAULT_RLIMIT_NOFILE 2048 #define DEFAULT_RLIMIT_MAXCORE 0 #define DEFAULT_MAP_TIMEOUT 60.0 * 5 #define DEFAULT_MAP_FILE_WATCH_MULTIPLIER 1 #define DEFAULT_MIN_WORD 0 #define DEFAULT_MAX_WORD 40 #define DEFAULT_WORDS_DECAY 600 #define DEFAULT_MAX_MESSAGE (50 * 1024 * 1024) #define DEFAULT_MAX_PIC (1 * 1024 * 1024) #define DEFAULT_MAX_SHOTS 100 #define DEFAULT_MAX_SESSIONS 100 #define DEFAULT_MAX_WORKERS 4 #define DEFAULT_MAX_HTML_SIZE DEFAULT_MAX_MESSAGE / 5 /* 10 Mb */ /* Timeout for task processing */ #define DEFAULT_TASK_TIMEOUT 8.0 #define DEFAULT_LUA_GC_STEP 200 #define DEFAULT_LUA_GC_PAUSE 200 #define DEFAULT_GC_MAXITERS 0 struct rspamd_ucl_map_cbdata { struct rspamd_config *cfg; std::string buf; explicit rspamd_ucl_map_cbdata(struct rspamd_config *cfg) : cfg(cfg) { } }; static char *rspamd_ucl_read_cb(char *chunk, int len, struct map_cb_data *data, gboolean final); static void rspamd_ucl_fin_cb(struct map_cb_data *data, void **target); static void rspamd_ucl_dtor_cb(struct map_cb_data *data); unsigned int rspamd_config_log_id = (unsigned int) -1; RSPAMD_CONSTRUCTOR(rspamd_config_log_init) { rspamd_config_log_id = rspamd_logger_add_debug_module("config"); } struct rspamd_actions_list { using action_ptr = std::shared_ptr; std::vector actions; ankerl::unordered_dense::map actions_by_name; explicit rspamd_actions_list() { actions.reserve(METRIC_ACTION_MAX + 2); actions_by_name.reserve(METRIC_ACTION_MAX + 2); } void add_action(action_ptr action) { actions.push_back(action); actions_by_name[action->name] = action; sort(); } void sort() { std::sort(actions.begin(), actions.end(), [](const action_ptr &a1, const action_ptr &a2) -> bool { if (!std::isnan(a1->threshold) && !std::isnan(a2->threshold)) { return a1->threshold < a2->threshold; } if (std::isnan(a1->threshold) && std::isnan(a2->threshold)) { return false; } else if (std::isnan(a1->threshold)) { return true; } return false; }); } void clear() { actions.clear(); actions_by_name.clear(); } }; #define RSPAMD_CFG_ACTIONS(cfg) (reinterpret_cast((cfg)->actions)) gboolean rspamd_parse_bind_line(struct rspamd_config *cfg, struct rspamd_worker_conf *cf, const char *str) { struct rspamd_worker_bind_conf *cnf; const char *fdname; gboolean ret = TRUE; if (str == nullptr) { return FALSE; } cnf = rspamd_mempool_alloc0_type(cfg->cfg_pool, struct rspamd_worker_bind_conf); cnf->cnt = 1024; cnf->bind_line = rspamd_mempool_strdup(cfg->cfg_pool, str); auto bind_line = std::string_view{cnf->bind_line}; if (bind_line.starts_with("systemd:")) { /* The actual socket will be passed by systemd environment */ fdname = str + sizeof("systemd:") - 1; cnf->is_systemd = TRUE; cnf->addrs = g_ptr_array_new_full(1, nullptr); rspamd_mempool_add_destructor(cfg->cfg_pool, rspamd_ptr_array_free_hard, cnf->addrs); if (fdname[0]) { g_ptr_array_add(cnf->addrs, rspamd_mempool_strdup(cfg->cfg_pool, fdname)); cnf->cnt = cnf->addrs->len; cnf->name = rspamd_mempool_strdup(cfg->cfg_pool, str); LL_PREPEND(cf->bind_conf, cnf); } else { msg_err_config("cannot parse bind line: %s", str); ret = FALSE; } } else { if (rspamd_parse_host_port_priority(str, &cnf->addrs, nullptr, &cnf->name, DEFAULT_BIND_PORT, TRUE, cfg->cfg_pool) == RSPAMD_PARSE_ADDR_FAIL) { msg_err_config("cannot parse bind line: %s", str); ret = FALSE; } else { cnf->cnt = cnf->addrs->len; LL_PREPEND(cf->bind_conf, cnf); } } return ret; } struct rspamd_config * rspamd_config_new(enum rspamd_config_init_flags flags) { struct rspamd_config *cfg; rspamd_mempool_t *pool; pool = rspamd_mempool_new(8 * 1024 * 1024, "cfg", 0); cfg = rspamd_mempool_alloc0_type(pool, struct rspamd_config); /* Allocate larger pool for cfg */ cfg->cfg_pool = pool; cfg->dns_timeout = 1.0; cfg->dns_retransmits = 5; /* 16 sockets per DNS server */ cfg->dns_io_per_server = 16; cfg->unknown_weight = NAN; cfg->actions = (void *) new rspamd_actions_list(); /* Add all internal actions to keep compatibility */ for (int i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i++) { auto &&action = std::make_shared(); action->threshold = NAN; action->name = rspamd_mempool_strdup(cfg->cfg_pool, rspamd_action_to_str(static_cast(i))); action->action_type = static_cast(i); if (i == METRIC_ACTION_SOFT_REJECT) { action->flags |= RSPAMD_ACTION_NO_THRESHOLD | RSPAMD_ACTION_HAM; } else if (i == METRIC_ACTION_GREYLIST) { action->flags |= RSPAMD_ACTION_THRESHOLD_ONLY | RSPAMD_ACTION_HAM; } else if (i == METRIC_ACTION_NOACTION) { action->flags |= RSPAMD_ACTION_HAM; } RSPAMD_CFG_ACTIONS(cfg)->add_action(std::move(action)); } /* Disable timeout */ cfg->task_timeout = DEFAULT_TASK_TIMEOUT; rspamd_config_init_metric(cfg); cfg->composites_manager = rspamd_composites_manager_create(cfg); cfg->classifiers_symbols = g_hash_table_new(rspamd_str_hash, rspamd_str_equal); cfg->cfg_params = g_hash_table_new(rspamd_str_hash, rspamd_str_equal); cfg->debug_modules = g_hash_table_new(rspamd_str_hash, rspamd_str_equal); cfg->explicit_modules = g_hash_table_new(rspamd_str_hash, rspamd_str_equal); cfg->trusted_keys = g_hash_table_new(rspamd_str_hash, rspamd_str_equal); cfg->map_timeout = DEFAULT_MAP_TIMEOUT; cfg->map_file_watch_multiplier = DEFAULT_MAP_FILE_WATCH_MULTIPLIER; cfg->log_level = G_LOG_LEVEL_WARNING; cfg->log_flags = RSPAMD_LOG_FLAG_DEFAULT; cfg->check_text_attachements = TRUE; cfg->dns_max_requests = 64; cfg->history_rows = 200; cfg->log_error_elts = 10; cfg->log_error_elt_maxlen = 1000; cfg->log_task_max_elts = 7; cfg->cache_reload_time = 30.0; cfg->max_lua_urls = 1024; cfg->max_urls = cfg->max_lua_urls * 10; cfg->max_recipients = 1024; cfg->max_blas_threads = 1; cfg->max_opts_len = 4096; cfg->gtube_patterns_policy = RSPAMD_GTUBE_REJECT; /* Default log line */ cfg->log_format_str = rspamd_mempool_strdup(cfg->cfg_pool, "id: <$mid>,$if_qid{ qid: <$>,}$if_ip{ ip: $,}" "$if_user{ user: $,}$if_smtp_from{ from: <$>,} (default: $is_spam " "($action): [$scores] [$symbols_scores_params]), len: $len, time: $time_real, " "dns req: $dns_req, digest: <$digest>" "$if_smtp_rcpts{ rcpts: <$>, }$if_mime_rcpt{ mime_rcpt: <$>, }"); /* Allow non-mime input by default */ cfg->allow_raw_input = TRUE; /* Default maximum words processed */ cfg->words_decay = DEFAULT_WORDS_DECAY; cfg->min_word_len = DEFAULT_MIN_WORD; cfg->max_word_len = DEFAULT_MAX_WORD; cfg->max_html_len = DEFAULT_MAX_HTML_SIZE; /* GC limits */ cfg->lua_gc_pause = DEFAULT_LUA_GC_PAUSE; cfg->lua_gc_step = DEFAULT_LUA_GC_STEP; cfg->full_gc_iters = DEFAULT_GC_MAXITERS; /* Default hyperscan cache */ cfg->hs_cache_dir = rspamd_mempool_strdup(cfg->cfg_pool, RSPAMD_DBDIR "/"); if (!(flags & RSPAMD_CONFIG_INIT_SKIP_LUA)) { cfg->lua_state = (void *) rspamd_lua_init(flags & RSPAMD_CONFIG_INIT_WIPE_LUA_MEM); cfg->own_lua_state = TRUE; cfg->lua_thread_pool = (void *) lua_thread_pool_new(RSPAMD_LUA_CFG_STATE(cfg)); } cfg->cache = rspamd_symcache_new(cfg); cfg->ups_ctx = rspamd_upstreams_library_init(); cfg->re_cache = rspamd_re_cache_new(); cfg->doc_strings = ucl_object_typed_new(UCL_OBJECT); /* * Unless exim is fixed */ cfg->enable_shutdown_workaround = TRUE; cfg->ssl_ciphers = rspamd_mempool_strdup(cfg->cfg_pool, "HIGH:!aNULL:!kRSA:!PSK:!SRP:!MD5:!RC4"); cfg->max_message = DEFAULT_MAX_MESSAGE; cfg->max_pic_size = DEFAULT_MAX_PIC; cfg->images_cache_size = 256; cfg->monitored_ctx = rspamd_monitored_ctx_init(); cfg->neighbours = ucl_object_typed_new(UCL_OBJECT); cfg->redis_pool = rspamd_redis_pool_init(); cfg->default_max_shots = DEFAULT_MAX_SHOTS; cfg->max_sessions_cache = DEFAULT_MAX_SESSIONS; cfg->maps_cache_dir = rspamd_mempool_strdup(cfg->cfg_pool, RSPAMD_DBDIR); cfg->c_modules = g_ptr_array_new(); cfg->heartbeat_interval = 10.0; cfg->enable_css_parser = true; cfg->enable_mime_utf = false; cfg->script_modules = g_ptr_array_new(); REF_INIT_RETAIN(cfg, rspamd_config_free); return cfg; } void rspamd_config_free(struct rspamd_config *cfg) { struct rspamd_config_cfg_lua_script *sc, *sctmp; struct rspamd_config_settings_elt *set, *stmp; struct rspamd_worker_log_pipe *lp, *ltmp; rspamd_lua_run_config_unload(RSPAMD_LUA_CFG_STATE(cfg), cfg); /* Scripts part */ DL_FOREACH_SAFE(cfg->on_term_scripts, sc, sctmp) { luaL_unref(RSPAMD_LUA_CFG_STATE(cfg), LUA_REGISTRYINDEX, sc->cbref); } DL_FOREACH_SAFE(cfg->on_load_scripts, sc, sctmp) { luaL_unref(RSPAMD_LUA_CFG_STATE(cfg), LUA_REGISTRYINDEX, sc->cbref); } DL_FOREACH_SAFE(cfg->post_init_scripts, sc, sctmp) { luaL_unref(RSPAMD_LUA_CFG_STATE(cfg), LUA_REGISTRYINDEX, sc->cbref); } DL_FOREACH_SAFE(cfg->config_unload_scripts, sc, sctmp) { luaL_unref(RSPAMD_LUA_CFG_STATE(cfg), LUA_REGISTRYINDEX, sc->cbref); } DL_FOREACH_SAFE(cfg->setting_ids, set, stmp) { REF_RELEASE(set); } rspamd_map_remove_all(cfg); rspamd_mempool_destructors_enforce(cfg->cfg_pool); g_list_free(cfg->classifiers); g_list_free(cfg->workers); rspamd_symcache_destroy(cfg->cache); ucl_object_unref(cfg->cfg_ucl_obj); ucl_object_unref(cfg->config_comments); ucl_object_unref(cfg->doc_strings); ucl_object_unref(cfg->neighbours); g_hash_table_remove_all(cfg->cfg_params); g_hash_table_unref(cfg->cfg_params); g_hash_table_unref(cfg->classifiers_symbols); g_hash_table_unref(cfg->debug_modules); g_hash_table_unref(cfg->explicit_modules); g_hash_table_unref(cfg->trusted_keys); rspamd_re_cache_unref(cfg->re_cache); g_ptr_array_free(cfg->c_modules, TRUE); g_ptr_array_free(cfg->script_modules, TRUE); if (cfg->monitored_ctx) { rspamd_monitored_ctx_destroy(cfg->monitored_ctx); } if (RSPAMD_LUA_CFG_STATE(cfg) && cfg->own_lua_state) { lua_thread_pool_free((struct lua_thread_pool *) cfg->lua_thread_pool); rspamd_lua_close(RSPAMD_LUA_CFG_STATE(cfg)); } if (cfg->redis_pool) { rspamd_redis_pool_destroy(cfg->redis_pool); } rspamd_upstreams_library_unref(cfg->ups_ctx); delete RSPAMD_CFG_ACTIONS(cfg); rspamd_mempool_destructors_enforce(cfg->cfg_pool); if (cfg->checksum) { g_free(cfg->checksum); } REF_RELEASE(cfg->libs_ctx); DL_FOREACH_SAFE(cfg->log_pipes, lp, ltmp) { close(lp->fd); g_free(lp); } rspamd_mempool_delete(cfg->cfg_pool); } const ucl_object_t * rspamd_config_get_module_opt(struct rspamd_config *cfg, const char *module_name, const char *opt_name) { const ucl_object_t *res = nullptr, *sec; sec = ucl_obj_get_key(cfg->cfg_ucl_obj, module_name); if (sec != nullptr) { res = ucl_obj_get_key(sec, opt_name); } return res; } int rspamd_config_parse_flag(const char *str, unsigned int len) { int c; if (!str || !*str) { return -1; } if (len == 0) { len = strlen(str); } switch (len) { case 1: c = g_ascii_tolower(*str); if (c == 'y' || c == '1') { return 1; } else if (c == 'n' || c == '0') { return 0; } break; case 2: if (g_ascii_strncasecmp(str, "no", len) == 0) { return 0; } else if (g_ascii_strncasecmp(str, "on", len) == 0) { return 1; } break; case 3: if (g_ascii_strncasecmp(str, "yes", len) == 0) { return 1; } else if (g_ascii_strncasecmp(str, "off", len) == 0) { return 0; } break; case 4: if (g_ascii_strncasecmp(str, "true", len) == 0) { return 1; } break; case 5: if (g_ascii_strncasecmp(str, "false", len) == 0) { return 0; } break; } return -1; } // A mapping between names and log format types + flags constexpr const auto config_vars = frozen::make_unordered_map>({ {"mid", {RSPAMD_LOG_MID, 0}}, {"qid", {RSPAMD_LOG_QID, 0}}, {"user", {RSPAMD_LOG_USER, 0}}, {"ip", {RSPAMD_LOG_IP, 0}}, {"len", {RSPAMD_LOG_LEN, 0}}, {"dns_req", {RSPAMD_LOG_DNS_REQ, 0}}, {"smtp_from", {RSPAMD_LOG_SMTP_FROM, 0}}, {"mime_from", {RSPAMD_LOG_MIME_FROM, 0}}, {"smtp_rcpt", {RSPAMD_LOG_SMTP_RCPT, 0}}, {"mime_rcpt", {RSPAMD_LOG_MIME_RCPT, 0}}, {"smtp_rcpts", {RSPAMD_LOG_SMTP_RCPTS, 0}}, {"mime_rcpts", {RSPAMD_LOG_MIME_RCPTS, 0}}, {"time_real", {RSPAMD_LOG_TIME_REAL, 0}}, {"time_virtual", {RSPAMD_LOG_TIME_VIRTUAL, 0}}, {"lua", {RSPAMD_LOG_LUA, 0}}, {"digest", {RSPAMD_LOG_DIGEST, 0}}, {"checksum", {RSPAMD_LOG_DIGEST, 0}}, {"filename", {RSPAMD_LOG_FILENAME, 0}}, {"forced_action", {RSPAMD_LOG_FORCED_ACTION, 0}}, {"settings_id", {RSPAMD_LOG_SETTINGS_ID, 0}}, {"mempool_size", {RSPAMD_LOG_MEMPOOL_SIZE, 0}}, {"mempool_waste", {RSPAMD_LOG_MEMPOOL_WASTE, 0}}, {"action", {RSPAMD_LOG_ACTION, 0}}, {"scores", {RSPAMD_LOG_SCORES, 0}}, {"symbols", {RSPAMD_LOG_SYMBOLS, 0}}, {"symbols_scores", {RSPAMD_LOG_SYMBOLS, RSPAMD_LOG_FMT_FLAG_SYMBOLS_SCORES}}, {"symbols_params", {RSPAMD_LOG_SYMBOLS, RSPAMD_LOG_FMT_FLAG_SYMBOLS_PARAMS}}, {"symbols_scores_params", {RSPAMD_LOG_SYMBOLS, RSPAMD_LOG_FMT_FLAG_SYMBOLS_PARAMS | RSPAMD_LOG_FMT_FLAG_SYMBOLS_SCORES}}, {"groups", {RSPAMD_LOG_GROUPS, 0}}, {"public_groups", {RSPAMD_LOG_PUBLIC_GROUPS, 0}}, {"is_spam", {RSPAMD_LOG_ISSPAM, 0}}, }); static gboolean rspamd_config_process_var(struct rspamd_config *cfg, const rspamd_ftok_t *var, const rspamd_ftok_t *content) { g_assert(var != nullptr); auto flags = 0; auto lc_var = std::string{var->begin, var->len}; std::transform(lc_var.begin(), lc_var.end(), lc_var.begin(), g_ascii_tolower); auto tok = std::string_view{lc_var}; if (var->len > 3 && tok.starts_with("if_")) { flags |= RSPAMD_LOG_FMT_FLAG_CONDITION; tok = tok.substr(3); } auto maybe_fmt_var = rspamd::find_map(config_vars, tok); if (maybe_fmt_var) { auto &fmt_var = maybe_fmt_var.value().get(); auto *log_format = rspamd_mempool_alloc0_type(cfg->cfg_pool, rspamd_log_format); log_format->type = fmt_var.first; log_format->flags = fmt_var.second | flags; if (log_format->type != RSPAMD_LOG_LUA) { if (content && content->len > 0) { log_format->data = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(rspamd_ftok_t)); memcpy(log_format->data, content, sizeof(*content)); log_format->len = sizeof(*content); } } else { /* Load lua code and ensure that we have function ref returned */ if (!content || content->len == 0) { msg_err_config("lua variable needs content: %T", &tok); return FALSE; } if (luaL_loadbuffer(RSPAMD_LUA_CFG_STATE(cfg), content->begin, content->len, "lua log variable") != 0) { msg_err_config("error loading lua code: '%T': %s", content, lua_tostring(RSPAMD_LUA_CFG_STATE(cfg), -1)); return FALSE; } if (lua_pcall(RSPAMD_LUA_CFG_STATE(cfg), 0, 1, 0) != 0) { msg_err_config("error executing lua code: '%T': %s", content, lua_tostring(RSPAMD_LUA_CFG_STATE(cfg), -1)); lua_pop(RSPAMD_LUA_CFG_STATE(cfg), 1); return FALSE; } if (lua_type(RSPAMD_LUA_CFG_STATE(cfg), -1) != LUA_TFUNCTION) { msg_err_config("lua variable should return function: %T", content); lua_pop(RSPAMD_LUA_CFG_STATE(cfg), 1); return FALSE; } auto id = luaL_ref(RSPAMD_LUA_CFG_STATE(cfg), LUA_REGISTRYINDEX); log_format->data = GINT_TO_POINTER(id); log_format->len = 0; } DL_APPEND(cfg->log_format, log_format); } else { std::string known_formats; for (const auto &v: config_vars) { known_formats += std::string_view{v.first.data(), v.first.size()}; known_formats += ", "; } if (known_formats.size() > 2) { // Remove last comma known_formats.resize(known_formats.size() - 2); } msg_err_config("unknown log variable: %T, known vars are: \"%s\"", var, known_formats.c_str()); return FALSE; } return TRUE; } static gboolean rspamd_config_parse_log_format(struct rspamd_config *cfg) { const char *p, *c, *end, *s; char *d; struct rspamd_log_format *lf = nullptr; rspamd_ftok_t var, var_content; enum { parse_str, parse_dollar, parse_var_name, parse_var_content, } state = parse_str; int braces = 0; g_assert(cfg != nullptr); c = cfg->log_format_str; if (c == nullptr) { return FALSE; } p = c; end = p + strlen(p); while (p < end) { switch (state) { case parse_str: if (*p == '$') { state = parse_dollar; } else { p++; } break; case parse_dollar: if (p > c) { /* We have string element that we need to store */ lf = rspamd_mempool_alloc0_type(cfg->cfg_pool, struct rspamd_log_format); lf->type = RSPAMD_LOG_STRING; lf->data = rspamd_mempool_alloc(cfg->cfg_pool, p - c + 1); /* Filter \r\n from the destination */ s = c; d = (char *) lf->data; while (s < p) { if (*s != '\r' && *s != '\n') { *d++ = *s++; } else { *d++ = ' '; s++; } } *d = '\0'; lf->len = d - (char *) lf->data; DL_APPEND(cfg->log_format, lf); lf = nullptr; } p++; c = p; state = parse_var_name; break; case parse_var_name: if (*p == '{') { var.begin = c; var.len = p - c; p++; c = p; state = parse_var_content; braces = 1; } else if (*p != '_' && *p != '-' && !g_ascii_isalnum(*p)) { /* Variable with no content */ var.begin = c; var.len = p - c; c = p; if (!rspamd_config_process_var(cfg, &var, nullptr)) { return FALSE; } state = parse_str; } else { p++; } break; case parse_var_content: if (*p == '}' && --braces == 0) { var_content.begin = c; var_content.len = p - c; p++; c = p; if (!rspamd_config_process_var(cfg, &var, &var_content)) { return FALSE; } state = parse_str; } else if (*p == '{') { braces++; p++; } else { p++; } break; } } /* Last state */ switch (state) { case parse_str: if (p > c) { /* We have string element that we need to store */ lf = rspamd_mempool_alloc0_type(cfg->cfg_pool, struct rspamd_log_format); lf->type = RSPAMD_LOG_STRING; lf->data = rspamd_mempool_alloc(cfg->cfg_pool, p - c + 1); /* Filter \r\n from the destination */ s = c; d = (char *) lf->data; while (s < p) { if (*s != '\r' && *s != '\n') { *d++ = *s++; } else { *d++ = ' '; s++; } } *d = '\0'; lf->len = d - (char *) lf->data; DL_APPEND(cfg->log_format, lf); lf = nullptr; } break; case parse_var_name: var.begin = c; var.len = p - c; if (!rspamd_config_process_var(cfg, &var, nullptr)) { return FALSE; } break; case parse_dollar: case parse_var_content: msg_err_config("cannot parse log format %s: incomplete string", cfg->log_format_str); return FALSE; break; } return TRUE; } static void rspamd_urls_config_dtor(gpointer _unused) { rspamd_url_deinit(); } static void rspamd_adjust_clocks_resolution(struct rspamd_config *cfg) { #ifdef HAVE_CLOCK_GETTIME struct timespec ts; #endif #ifdef HAVE_CLOCK_GETTIME #ifdef HAVE_CLOCK_PROCESS_CPUTIME_ID clock_getres(CLOCK_PROCESS_CPUTIME_ID, &ts); #elif defined(HAVE_CLOCK_VIRTUAL) clock_getres(CLOCK_VIRTUAL, &ts); #else clock_getres(CLOCK_REALTIME, &ts); #endif cfg->clock_res = log10(1000000. / ts.tv_nsec); if (cfg->clock_res < 0) { cfg->clock_res = 0; } if (cfg->clock_res > 3) { cfg->clock_res = 3; } #else /* For gettimeofday */ cfg->clock_res = 1; #endif } /* * Perform post load actions */ gboolean rspamd_config_post_load(struct rspamd_config *cfg, enum rspamd_post_load_options opts) { auto ret = tl::expected{}; rspamd_adjust_clocks_resolution(cfg); rspamd_logger_configure_modules(cfg->debug_modules); if (cfg->one_shot_mode) { msg_info_config("enabling one shot mode (was %d max shots)", cfg->default_max_shots); cfg->default_max_shots = 1; } #if defined(WITH_HYPERSCAN) && !defined(__aarch64__) && !defined(__powerpc64__) if (!cfg->disable_hyperscan) { if (!(cfg->libs_ctx->crypto_ctx->cpu_config & CPUID_SSSE3)) { msg_warn_config("CPU doesn't have SSSE3 instructions set " "required for hyperscan, disable it"); cfg->disable_hyperscan = TRUE; } } #endif rspamd_regexp_library_init(cfg); rspamd_multipattern_library_init(cfg->hs_cache_dir); if (opts & RSPAMD_CONFIG_INIT_URL) { if (cfg->tld_file == nullptr) { /* Try to guess tld file */ auto fpath = fmt::format("{0}{1}{2}", RSPAMD_SHAREDIR, G_DIR_SEPARATOR, "effective_tld_names.dat"); if (access(fpath.c_str(), R_OK) != -1) { msg_debug_config("url_tld option is not specified but %s is available," " therefore this file is assumed as TLD file for URL" " extraction", fpath.c_str()); cfg->tld_file = rspamd_mempool_strdup(cfg->cfg_pool, fpath.c_str()); } else { if (opts & RSPAMD_CONFIG_INIT_VALIDATE) { msg_err_config("no url_tld option has been specified"); ret = tl::make_unexpected(std::string{"no url_tld option has been specified"}); } } } else { if (access(cfg->tld_file, R_OK) == -1) { if (opts & RSPAMD_CONFIG_INIT_VALIDATE) { ret = tl::make_unexpected(fmt::format("cannot access tld file {}: {}", cfg->tld_file, strerror(errno))); msg_err_config("cannot access tld file %s: %s", cfg->tld_file, strerror(errno)); } else { msg_debug_config("cannot access tld file %s: %s", cfg->tld_file, strerror(errno)); cfg->tld_file = nullptr; } } } if (opts & RSPAMD_CONFIG_INIT_NO_TLD) { rspamd_url_init(nullptr); } else { rspamd_url_init(cfg->tld_file); } rspamd_mempool_add_destructor(cfg->cfg_pool, rspamd_urls_config_dtor, nullptr); } init_dynamic_config(cfg); /* Insert classifiers symbols */ rspamd_config_insert_classify_symbols(cfg); /* Parse format string that we have */ if (!rspamd_config_parse_log_format(cfg)) { msg_err_config("cannot parse log format, task logging will not be available"); if (opts & RSPAMD_CONFIG_INIT_VALIDATE) { ret = tl::make_unexpected(std::string{"cannot parse log format"}); } } if (opts & RSPAMD_CONFIG_INIT_SYMCACHE) { /* Init config cache */ auto symcache_ret = rspamd_symcache_init(cfg->cache); if (!symcache_ret) { ret = tl::make_unexpected(std::string{"symcache init failed"}); } /* Init re cache */ rspamd_re_cache_init(cfg->re_cache, cfg); /* Try load Hypersan */ auto hs_ret = rspamd_re_cache_load_hyperscan(cfg->re_cache, cfg->hs_cache_dir ? cfg->hs_cache_dir : RSPAMD_DBDIR "/", true); if (hs_ret == RSPAMD_HYPERSCAN_LOAD_ERROR) { msg_debug_config("cannot load hyperscan database, disable it"); } } if (opts & RSPAMD_CONFIG_INIT_LIBS) { /* Config other libraries */ auto libs_ret = rspamd_config_libs(cfg->libs_ctx, cfg); if (!libs_ret) { msg_err_config("cannot configure libraries, fatal error"); return FALSE; } } /* Validate cache */ if (opts & RSPAMD_CONFIG_INIT_VALIDATE) { /* Check for actions sanity */ auto seen_controller = FALSE; auto *cur = cfg->workers; while (cur) { auto *wcf = (struct rspamd_worker_conf *) cur->data; if (wcf->type == g_quark_from_static_string("controller")) { seen_controller = TRUE; break; } cur = g_list_next(cur); } if (!seen_controller) { msg_warn_config("controller worker is unconfigured: learning," " periodic scripts, maps watching and many other" " Rspamd features will be broken"); } auto val_ret = rspamd_symcache_validate(cfg->cache, cfg, FALSE); if (!val_ret) { ret = tl::make_unexpected(std::string{"symcache validation failed"}); } } if (opts & RSPAMD_CONFIG_INIT_POST_LOAD_LUA) { rspamd_lua_run_config_post_init(RSPAMD_LUA_CFG_STATE(cfg), cfg); } if (opts & RSPAMD_CONFIG_INIT_PRELOAD_MAPS) { rspamd_map_preload(cfg); } if (ret) { return true; } else { msg_err_config("error on post-init stage: %s", ret.error().c_str()); return false; } } struct rspamd_classifier_config * rspamd_config_new_classifier(struct rspamd_config *cfg, struct rspamd_classifier_config *c) { if (c == nullptr) { c = rspamd_mempool_alloc0_type(cfg->cfg_pool, struct rspamd_classifier_config); c->min_prob_strength = 0.05; c->min_token_hits = 2; } if (c->labels == nullptr) { c->labels = g_hash_table_new_full(rspamd_str_hash, rspamd_str_equal, nullptr, (GDestroyNotify) g_list_free); rspamd_mempool_add_destructor(cfg->cfg_pool, (rspamd_mempool_destruct_t) g_hash_table_destroy, c->labels); } return c; } struct rspamd_statfile_config * rspamd_config_new_statfile(struct rspamd_config *cfg, struct rspamd_statfile_config *c) { if (c == nullptr) { c = rspamd_mempool_alloc0_type(cfg->cfg_pool, struct rspamd_statfile_config); } return c; } void rspamd_config_init_metric(struct rspamd_config *cfg) { cfg->grow_factor = 1.0; cfg->symbols = g_hash_table_new(rspamd_str_hash, rspamd_str_equal); cfg->groups = g_hash_table_new(rspamd_strcase_hash, rspamd_strcase_equal); cfg->subject = SPAM_SUBJECT; rspamd_mempool_add_destructor(cfg->cfg_pool, (rspamd_mempool_destruct_t) g_hash_table_unref, cfg->symbols); rspamd_mempool_add_destructor(cfg->cfg_pool, (rspamd_mempool_destruct_t) g_hash_table_unref, cfg->groups); } struct rspamd_symbols_group * rspamd_config_new_group(struct rspamd_config *cfg, const char *name) { struct rspamd_symbols_group *gr; gr = rspamd_mempool_alloc0_type(cfg->cfg_pool, struct rspamd_symbols_group); gr->symbols = g_hash_table_new(rspamd_strcase_hash, rspamd_strcase_equal); rspamd_mempool_add_destructor(cfg->cfg_pool, (rspamd_mempool_destruct_t) g_hash_table_unref, gr->symbols); gr->name = rspamd_mempool_strdup(cfg->cfg_pool, name); gr->max_score = NAN; gr->min_score = NAN; if (strcmp(gr->name, "ungrouped") == 0) { gr->flags |= RSPAMD_SYMBOL_GROUP_UNGROUPED; } g_hash_table_insert(cfg->groups, gr->name, gr); return gr; } static void rspamd_worker_conf_dtor(struct rspamd_worker_conf *wcf) { if (wcf) { ucl_object_unref(wcf->options); g_queue_free(wcf->active_workers); g_hash_table_unref(wcf->params); g_free(wcf); } } static void rspamd_worker_conf_cfg_fin(gpointer d) { auto *wcf = (struct rspamd_worker_conf *) d; REF_RELEASE(wcf); } struct rspamd_worker_conf * rspamd_config_new_worker(struct rspamd_config *cfg, struct rspamd_worker_conf *c) { if (c == nullptr) { c = g_new0(struct rspamd_worker_conf, 1); c->params = g_hash_table_new(rspamd_str_hash, rspamd_str_equal); c->active_workers = g_queue_new(); #ifdef HAVE_SC_NPROCESSORS_ONLN auto nproc = sysconf(_SC_NPROCESSORS_ONLN); c->count = MIN(DEFAULT_MAX_WORKERS, MAX(1, nproc - 2)); #else c->count = DEFAULT_MAX_WORKERS; #endif c->rlimit_nofile = 0; c->rlimit_maxcore = 0; c->enabled = TRUE; REF_INIT_RETAIN(c, rspamd_worker_conf_dtor); rspamd_mempool_add_destructor(cfg->cfg_pool, rspamd_worker_conf_cfg_fin, c); } return c; } static bool rspamd_include_map_handler(const unsigned char *data, gsize len, const ucl_object_t *args, void *ud) { auto *cfg = (struct rspamd_config *) ud; auto ftok = rspamd_ftok_t{.len = len + 1, .begin = (char *) data}; auto *map_line = rspamd_mempool_ftokdup(cfg->cfg_pool, &ftok); auto *cbdata = new rspamd_ucl_map_cbdata{cfg}; auto **pcbdata = new rspamd_ucl_map_cbdata *(cbdata); return rspamd_map_add(cfg, map_line, "ucl include", rspamd_ucl_read_cb, rspamd_ucl_fin_cb, rspamd_ucl_dtor_cb, (void **) pcbdata, nullptr, RSPAMD_MAP_DEFAULT) != nullptr; } /* * Variables: * $CONFDIR - configuration directory * $LOCAL_CONFDIR - local configuration directory * $RUNDIR - local states directory * $DBDIR - databases dir * $LOGDIR - logs dir * $PLUGINSDIR - plugins dir * $PREFIX - installation prefix * $VERSION - rspamd version */ #define RSPAMD_CONFDIR_MACRO "CONFDIR" #define RSPAMD_LOCAL_CONFDIR_MACRO "LOCAL_CONFDIR" #define RSPAMD_RUNDIR_MACRO "RUNDIR" #define RSPAMD_DBDIR_MACRO "DBDIR" #define RSPAMD_LOGDIR_MACRO "LOGDIR" #define RSPAMD_PLUGINSDIR_MACRO "PLUGINSDIR" #define RSPAMD_SHAREDIR_MACRO "SHAREDIR" #define RSPAMD_RULESDIR_MACRO "RULESDIR" #define RSPAMD_WWWDIR_MACRO "WWWDIR" #define RSPAMD_PREFIX_MACRO "PREFIX" #define RSPAMD_VERSION_MACRO "VERSION" #define RSPAMD_VERSION_MAJOR_MACRO "VERSION_MAJOR" #define RSPAMD_VERSION_MINOR_MACRO "VERSION_MINOR" #define RSPAMD_BRANCH_VERSION_MACRO "BRANCH_VERSION" #define RSPAMD_HOSTNAME_MACRO "HOSTNAME" void rspamd_ucl_add_conf_variables(struct ucl_parser *parser, GHashTable *vars) { GHashTableIter it; gpointer k, v; ucl_parser_register_variable(parser, RSPAMD_CONFDIR_MACRO, RSPAMD_CONFDIR); ucl_parser_register_variable(parser, RSPAMD_LOCAL_CONFDIR_MACRO, RSPAMD_LOCAL_CONFDIR); ucl_parser_register_variable(parser, RSPAMD_RUNDIR_MACRO, RSPAMD_RUNDIR); ucl_parser_register_variable(parser, RSPAMD_DBDIR_MACRO, RSPAMD_DBDIR); ucl_parser_register_variable(parser, RSPAMD_LOGDIR_MACRO, RSPAMD_LOGDIR); ucl_parser_register_variable(parser, RSPAMD_PLUGINSDIR_MACRO, RSPAMD_PLUGINSDIR); ucl_parser_register_variable(parser, RSPAMD_SHAREDIR_MACRO, RSPAMD_SHAREDIR); ucl_parser_register_variable(parser, RSPAMD_RULESDIR_MACRO, RSPAMD_RULESDIR); ucl_parser_register_variable(parser, RSPAMD_WWWDIR_MACRO, RSPAMD_WWWDIR); ucl_parser_register_variable(parser, RSPAMD_PREFIX_MACRO, RSPAMD_PREFIX); ucl_parser_register_variable(parser, RSPAMD_VERSION_MACRO, RVERSION); ucl_parser_register_variable(parser, RSPAMD_VERSION_MAJOR_MACRO, RSPAMD_VERSION_MAJOR); ucl_parser_register_variable(parser, RSPAMD_VERSION_MINOR_MACRO, RSPAMD_VERSION_MINOR); ucl_parser_register_variable(parser, RSPAMD_BRANCH_VERSION_MACRO, RSPAMD_VERSION_BRANCH); auto hostlen = sysconf(_SC_HOST_NAME_MAX); if (hostlen <= 0) { hostlen = 256; } else { hostlen++; } auto hostbuf = std::string{}; hostbuf.resize(hostlen); if (gethostname(hostbuf.data(), hostlen) != 0) { hostbuf = "unknown"; } /* UCL copies variables, so it is safe to pass an ephemeral buffer here */ ucl_parser_register_variable(parser, RSPAMD_HOSTNAME_MACRO, hostbuf.c_str()); if (vars != nullptr) { g_hash_table_iter_init(&it, vars); while (g_hash_table_iter_next(&it, &k, &v)) { ucl_parser_register_variable(parser, (const char *) k, (const char *) v); } } } void rspamd_ucl_add_conf_macros(struct ucl_parser *parser, struct rspamd_config *cfg) { ucl_parser_register_macro(parser, "include_map", rspamd_include_map_handler, cfg); } static void symbols_classifiers_callback(gpointer key, gpointer value, gpointer ud) { auto *cfg = (struct rspamd_config *) ud; /* Actually, statistics should act like any ordinary symbol */ rspamd_symcache_add_symbol(cfg->cache, (const char *) key, 0, nullptr, nullptr, SYMBOL_TYPE_CLASSIFIER | SYMBOL_TYPE_NOSTAT, -1); } void rspamd_config_insert_classify_symbols(struct rspamd_config *cfg) { g_hash_table_foreach(cfg->classifiers_symbols, symbols_classifiers_callback, cfg); } struct rspamd_classifier_config * rspamd_config_find_classifier(struct rspamd_config *cfg, const char *name) { if (name == nullptr) { return nullptr; } auto *cur = cfg->classifiers; while (cur) { auto *cf = (struct rspamd_classifier_config *) cur->data; if (g_ascii_strcasecmp(cf->name, name) == 0) { return cf; } cur = g_list_next(cur); } return nullptr; } gboolean rspamd_config_check_statfiles(struct rspamd_classifier_config *cf) { gboolean has_other = FALSE, res = FALSE, cur_class = FALSE; /* First check classes directly */ auto *cur = cf->statfiles; while (cur) { auto *st = (struct rspamd_statfile_config *) cur->data; if (!has_other) { cur_class = st->is_spam; has_other = TRUE; } else { if (cur_class != st->is_spam) { return TRUE; } } cur = g_list_next(cur); } if (!has_other) { /* We have only one statfile */ return FALSE; } /* We have not detected any statfile that has different class, so turn on heuristic based on symbol's name */ has_other = FALSE; cur = cf->statfiles; while (cur) { auto *st = (struct rspamd_statfile_config *) cur->data; if (rspamd_substring_search_caseless(st->symbol, strlen(st->symbol), "spam", 4) != -1) { st->is_spam = TRUE; } else if (rspamd_substring_search_caseless(st->symbol, strlen(st->symbol), "ham", 3) != -1) { st->is_spam = FALSE; } if (!has_other) { cur_class = st->is_spam; has_other = TRUE; } else { if (cur_class != st->is_spam) { res = TRUE; } } cur = g_list_next(cur); } return res; } static char * rspamd_ucl_read_cb(char *chunk, int len, struct map_cb_data *data, gboolean final) { auto *cbdata = (struct rspamd_ucl_map_cbdata *) data->cur_data; auto *prev = (struct rspamd_ucl_map_cbdata *) data->prev_data; if (cbdata == nullptr) { cbdata = new rspamd_ucl_map_cbdata{prev->cfg}; data->cur_data = cbdata; } cbdata->buf.append(chunk, len); /* Say not to copy any part of this buffer */ return nullptr; } static void rspamd_ucl_fin_cb(struct map_cb_data *data, void **target) { auto *cbdata = (struct rspamd_ucl_map_cbdata *) data->cur_data; auto *prev = (struct rspamd_ucl_map_cbdata *) data->prev_data; auto *cfg = data->map->cfg; if (cbdata == nullptr) { msg_err_config("map fin error: new data is nullptr"); return; } /* New data available */ auto *parser = ucl_parser_new(0); if (!ucl_parser_add_chunk(parser, (unsigned char *) cbdata->buf.data(), cbdata->buf.size())) { msg_err_config("cannot parse map %s: %s", data->map->name, ucl_parser_get_error(parser)); ucl_parser_free(parser); } else { auto *obj = ucl_parser_get_object(parser); ucl_object_iter_t it = nullptr; for (auto *cur = ucl_object_iterate(obj, &it, true); cur != nullptr; cur = ucl_object_iterate(obj, &it, true)) { ucl_object_replace_key(cbdata->cfg->cfg_ucl_obj, (ucl_object_t *) cur, cur->key, cur->keylen, false); } ucl_parser_free(parser); ucl_object_unref(obj); } if (target) { *target = data->cur_data; } delete prev; } static void rspamd_ucl_dtor_cb(struct map_cb_data *data) { auto *cbdata = (struct rspamd_ucl_map_cbdata *) data->cur_data; delete cbdata; } gboolean rspamd_check_module(struct rspamd_config *cfg, module_t *mod) { gboolean ret = TRUE; if (mod != nullptr) { if (mod->module_version != RSPAMD_CUR_MODULE_VERSION) { msg_err_config("module %s has incorrect version %xd (%xd expected)", mod->name, (int) mod->module_version, RSPAMD_CUR_MODULE_VERSION); ret = FALSE; } if (ret && mod->rspamd_version != RSPAMD_VERSION_NUM) { msg_err_config("module %s has incorrect rspamd version %xL (%xL expected)", mod->name, mod->rspamd_version, RSPAMD_VERSION_NUM); ret = FALSE; } if (ret && strcmp(mod->rspamd_features, RSPAMD_FEATURES) != 0) { msg_err_config("module %s has incorrect rspamd features '%s' ('%s' expected)", mod->name, mod->rspamd_features, RSPAMD_FEATURES); ret = FALSE; } } else { ret = FALSE; } return ret; } gboolean rspamd_check_worker(struct rspamd_config *cfg, worker_t *wrk) { gboolean ret = TRUE; if (wrk != nullptr) { if (wrk->worker_version != RSPAMD_CUR_WORKER_VERSION) { msg_err_config("worker %s has incorrect version %xd (%xd expected)", wrk->name, wrk->worker_version, RSPAMD_CUR_WORKER_VERSION); ret = FALSE; } if (ret && wrk->rspamd_version != RSPAMD_VERSION_NUM) { msg_err_config("worker %s has incorrect rspamd version %xL (%xL expected)", wrk->name, wrk->rspamd_version, RSPAMD_VERSION_NUM); ret = FALSE; } if (ret && strcmp(wrk->rspamd_features, RSPAMD_FEATURES) != 0) { msg_err_config("worker %s has incorrect rspamd features '%s' ('%s' expected)", wrk->name, wrk->rspamd_features, RSPAMD_FEATURES); ret = FALSE; } } else { ret = FALSE; } return ret; } gboolean rspamd_init_filters(struct rspamd_config *cfg, bool reconfig, bool strict) { GList *cur; module_t *mod, **pmod; unsigned int i = 0; struct module_ctx *mod_ctx, *cur_ctx; gboolean ret = TRUE; /* Init all compiled modules */ for (pmod = cfg->compiled_modules; pmod != nullptr && *pmod != nullptr; pmod++) { mod = *pmod; if (rspamd_check_module(cfg, mod)) { if (mod->module_init_func(cfg, &mod_ctx) == 0) { g_assert(mod_ctx != nullptr); g_ptr_array_add(cfg->c_modules, mod_ctx); mod_ctx->mod = mod; mod->ctx_offset = i++; } } } /* Now check what's enabled */ cur = g_list_first(cfg->filters); while (cur) { /* Perform modules configuring */ mod_ctx = nullptr; PTR_ARRAY_FOREACH(cfg->c_modules, i, cur_ctx) { if (g_ascii_strcasecmp(cur_ctx->mod->name, (const char *) cur->data) == 0) { mod_ctx = cur_ctx; break; } } if (mod_ctx) { mod = mod_ctx->mod; mod_ctx->enabled = rspamd_config_is_module_enabled(cfg, mod->name); if (reconfig) { if (!mod->module_reconfig_func(cfg)) { msg_err_config("reconfig of %s failed!", mod->name); } else { msg_info_config("reconfig of %s", mod->name); } } else { if (!mod->module_config_func(cfg, strict)) { msg_err_config("config of %s failed", mod->name); ret = FALSE; if (strict) { return FALSE; } } } } if (mod_ctx == nullptr) { msg_warn_config("requested unknown module %s", cur->data); } cur = g_list_next(cur); } ret = rspamd_init_lua_filters(cfg, 0, strict) && ret; return ret; } static void rspamd_config_new_symbol(struct rspamd_config *cfg, const char *symbol, double score, const char *description, const char *group, unsigned int flags, unsigned int priority, int nshots) { struct rspamd_symbols_group *sym_group; struct rspamd_symbol *sym_def; double *score_ptr; sym_def = rspamd_mempool_alloc0_type(cfg->cfg_pool, struct rspamd_symbol); score_ptr = rspamd_mempool_alloc_type(cfg->cfg_pool, double); if (std::isnan(score)) { /* In fact, it could be defined later */ msg_debug_config("score is not defined for symbol %s, set it to zero", symbol); score = 0.0; /* Also set priority to 0 to allow override by anything */ sym_def->priority = 0; flags |= RSPAMD_SYMBOL_FLAG_UNSCORED; } else { sym_def->priority = priority; } *score_ptr = score; sym_def->score = score; sym_def->weight_ptr = score_ptr; sym_def->name = rspamd_mempool_strdup(cfg->cfg_pool, symbol); sym_def->flags = flags; sym_def->nshots = nshots != 0 ? nshots : cfg->default_max_shots; sym_def->groups = g_ptr_array_sized_new(1); rspamd_mempool_add_destructor(cfg->cfg_pool, rspamd_ptr_array_free_hard, sym_def->groups); if (description) { sym_def->description = rspamd_mempool_strdup(cfg->cfg_pool, description); } msg_debug_config("registered symbol %s with weight %.2f in and group %s", sym_def->name, score, group); g_hash_table_insert(cfg->symbols, sym_def->name, sym_def); /* Search for symbol group */ if (group == nullptr) { group = "ungrouped"; sym_def->flags |= RSPAMD_SYMBOL_FLAG_UNGROUPED; } else { if (strcmp(group, "ungrouped") == 0) { sym_def->flags |= RSPAMD_SYMBOL_FLAG_UNGROUPED; } } sym_group = reinterpret_cast(g_hash_table_lookup(cfg->groups, group)); if (sym_group == nullptr) { /* Create new group */ sym_group = rspamd_config_new_group(cfg, group); } sym_def->gr = sym_group; g_hash_table_insert(sym_group->symbols, sym_def->name, sym_def); if (!(sym_def->flags & RSPAMD_SYMBOL_FLAG_UNGROUPED)) { g_ptr_array_add(sym_def->groups, sym_group); } } gboolean rspamd_config_add_symbol(struct rspamd_config *cfg, const char *symbol, double score, const char *description, const char *group, unsigned int flags, unsigned int priority, int nshots) { struct rspamd_symbol *sym_def; struct rspamd_symbols_group *sym_group; unsigned int i; g_assert(cfg != nullptr); g_assert(symbol != nullptr); sym_def = reinterpret_cast(g_hash_table_lookup(cfg->symbols, symbol)); if (sym_def != nullptr) { if (group != nullptr) { gboolean has_group = FALSE; PTR_ARRAY_FOREACH(sym_def->groups, i, sym_group) { if (g_ascii_strcasecmp(sym_group->name, group) == 0) { /* Group is already here */ has_group = TRUE; break; } } if (!has_group) { /* Non-empty group has a priority over non-grouped one */ sym_group = reinterpret_cast(g_hash_table_lookup(cfg->groups, group)); if (sym_group == nullptr) { /* Create new group */ sym_group = rspamd_config_new_group(cfg, group); } if ((!sym_def->gr) || (sym_def->flags & RSPAMD_SYMBOL_FLAG_UNGROUPED)) { sym_def->gr = sym_group; sym_def->flags &= ~RSPAMD_SYMBOL_FLAG_UNGROUPED; } g_hash_table_insert(sym_group->symbols, sym_def->name, sym_def); sym_def->flags &= ~(RSPAMD_SYMBOL_FLAG_UNGROUPED); g_ptr_array_add(sym_def->groups, sym_group); } } if (sym_def->priority > priority && (std::isnan(score) || !(sym_def->flags & RSPAMD_SYMBOL_FLAG_UNSCORED))) { msg_debug_config("symbol %s has been already registered with " "priority %ud, do not override (new priority: %ud)", symbol, sym_def->priority, priority); /* But we can still add description */ if (!sym_def->description && description) { sym_def->description = rspamd_mempool_strdup(cfg->cfg_pool, description); } /* Or nshots in case of non-default setting */ if (nshots != 0 && sym_def->nshots == cfg->default_max_shots) { sym_def->nshots = nshots; } return FALSE; } else { if (!std::isnan(score)) { msg_debug_config("symbol %s has been already registered with " "priority %ud, override it with new priority: %ud, " "old score: %.2f, new score: %.2f", symbol, sym_def->priority, priority, sym_def->score, score); *sym_def->weight_ptr = score; sym_def->score = score; sym_def->priority = priority; sym_def->flags &= ~RSPAMD_SYMBOL_FLAG_UNSCORED; } sym_def->flags = flags; if (nshots != 0) { sym_def->nshots = nshots; } else { /* Do not reset unless we have exactly lower priority */ if (sym_def->priority < priority) { sym_def->nshots = cfg->default_max_shots; } } if (description) { sym_def->description = rspamd_mempool_strdup(cfg->cfg_pool, description); } /* We also check group information in this case */ if (group != nullptr && sym_def->gr != nullptr && strcmp(group, sym_def->gr->name) != 0) { sym_group = reinterpret_cast(g_hash_table_lookup(cfg->groups, group)); if (sym_group == nullptr) { /* Create new group */ sym_group = rspamd_config_new_group(cfg, group); } if (!(sym_group->flags & RSPAMD_SYMBOL_GROUP_UNGROUPED)) { msg_debug_config("move symbol %s from group %s to %s", sym_def->name, sym_def->gr->name, group); g_hash_table_remove(sym_def->gr->symbols, sym_def->name); sym_def->gr = sym_group; g_hash_table_insert(sym_group->symbols, sym_def->name, sym_def); } } return TRUE; } } /* This is called merely when we have an undefined symbol */ rspamd_config_new_symbol(cfg, symbol, score, description, group, flags, priority, nshots); return TRUE; } gboolean rspamd_config_add_symbol_group(struct rspamd_config *cfg, const char *symbol, const char *group) { struct rspamd_symbol *sym_def; struct rspamd_symbols_group *sym_group; unsigned int i; g_assert(cfg != nullptr); g_assert(symbol != nullptr); g_assert(group != nullptr); sym_def = reinterpret_cast(g_hash_table_lookup(cfg->symbols, symbol)); if (sym_def != nullptr) { gboolean has_group = FALSE; PTR_ARRAY_FOREACH(sym_def->groups, i, sym_group) { if (g_ascii_strcasecmp(sym_group->name, group) == 0) { /* Group is already here */ has_group = TRUE; break; } } if (!has_group) { /* Non-empty group has a priority over non-grouped one */ sym_group = reinterpret_cast(g_hash_table_lookup(cfg->groups, group)); if (sym_group == nullptr) { /* Create new group */ sym_group = rspamd_config_new_group(cfg, group); } if (!sym_def->gr) { sym_def->gr = sym_group; } g_hash_table_insert(sym_group->symbols, sym_def->name, sym_def); sym_def->flags &= ~(RSPAMD_SYMBOL_FLAG_UNGROUPED); g_ptr_array_add(sym_def->groups, sym_group); return TRUE; } } return FALSE; } gboolean rspamd_config_is_enabled_from_ucl(rspamd_mempool_t *pool, const ucl_object_t *obj) { const ucl_object_t *enabled; enabled = ucl_object_lookup(obj, "enabled"); if (enabled) { if (ucl_object_type(enabled) == UCL_BOOLEAN) { return ucl_object_toboolean(enabled); } else if (ucl_object_type(enabled) == UCL_STRING) { int ret = rspamd_config_parse_flag(ucl_object_tostring(enabled), 0); if (ret == 0) { return FALSE; } else if (ret == -1) { msg_info_pool_check("wrong value for the `enabled` key"); return FALSE; } /* Default return is TRUE here */ } } const ucl_object_t *disabled; disabled = ucl_object_lookup(obj, "disabled"); if (disabled) { if (ucl_object_type(disabled) == UCL_BOOLEAN) { return !ucl_object_toboolean(disabled); } else if (ucl_object_type(disabled) == UCL_STRING) { int ret = rspamd_config_parse_flag(ucl_object_tostring(disabled), 0); if (ret == 0) { return TRUE; } else if (ret == -1) { msg_info_pool_check("wrong value for the `disabled` key"); return FALSE; } return FALSE; } } return TRUE; } gboolean rspamd_config_is_module_enabled(struct rspamd_config *cfg, const char *module_name) { gboolean is_c = FALSE, enabled; const ucl_object_t *conf; GList *cur; struct rspamd_symbols_group *gr; lua_State *L = RSPAMD_LUA_CFG_STATE(cfg); struct module_ctx *cur_ctx; unsigned int i; PTR_ARRAY_FOREACH(cfg->c_modules, i, cur_ctx) { if (g_ascii_strcasecmp(cur_ctx->mod->name, module_name) == 0) { is_c = TRUE; break; } } if (g_hash_table_lookup(cfg->explicit_modules, module_name) != nullptr) { /* Always load module */ rspamd_plugins_table_push_elt(L, "enabled", module_name); return TRUE; } if (is_c) { gboolean found = FALSE; cur = g_list_first(cfg->filters); while (cur) { if (strcmp((char *) cur->data, module_name) == 0) { found = TRUE; break; } cur = g_list_next(cur); } if (!found) { msg_info_config("internal module %s is disable in `filters` line", module_name); rspamd_plugins_table_push_elt(L, "disabled_explicitly", module_name); return FALSE; } } conf = ucl_object_lookup(cfg->cfg_ucl_obj, module_name); if (conf == nullptr) { rspamd_plugins_table_push_elt(L, "disabled_unconfigured", module_name); msg_info_config("%s module %s is enabled but has not been configured", is_c ? "internal" : "lua", module_name); if (!is_c) { msg_info_config("%s disabling unconfigured lua module", module_name); return FALSE; } } else { enabled = rspamd_config_is_enabled_from_ucl(cfg->cfg_pool, conf); if (!enabled) { rspamd_plugins_table_push_elt(L, "disabled_explicitly", module_name); msg_info_config( "%s module %s is disabled in the configuration", is_c ? "internal" : "lua", module_name); return FALSE; } } /* Now we check symbols group */ gr = reinterpret_cast(g_hash_table_lookup(cfg->groups, module_name)); if (gr) { if (gr->flags & RSPAMD_SYMBOL_GROUP_DISABLED) { rspamd_plugins_table_push_elt(L, "disabled_explicitly", module_name); msg_info_config("%s module %s is disabled in the configuration as " "its group has been disabled", is_c ? "internal" : "lua", module_name); return FALSE; } } rspamd_plugins_table_push_elt(L, "enabled", module_name); return TRUE; } static gboolean rspamd_config_action_from_ucl(struct rspamd_config *cfg, struct rspamd_action *act, const ucl_object_t *obj, unsigned int priority) { auto threshold = NAN; int flags = 0; auto obj_type = ucl_object_type(obj); if (obj_type == UCL_OBJECT) { obj_type = ucl_object_type(obj); const auto *elt = ucl_object_lookup_any(obj, "score", "threshold", nullptr); if (elt) { threshold = ucl_object_todouble(elt); } elt = ucl_object_lookup(obj, "flags"); if (elt && ucl_object_type(elt) == UCL_ARRAY) { const ucl_object_t *cur; ucl_object_iter_t it = nullptr; while ((cur = ucl_object_iterate(elt, &it, true)) != nullptr) { if (ucl_object_type(cur) == UCL_STRING) { const char *fl_str = ucl_object_tostring(cur); if (g_ascii_strcasecmp(fl_str, "no_threshold") == 0) { flags |= RSPAMD_ACTION_NO_THRESHOLD; } else if (g_ascii_strcasecmp(fl_str, "threshold_only") == 0) { flags |= RSPAMD_ACTION_THRESHOLD_ONLY; } else if (g_ascii_strcasecmp(fl_str, "ham") == 0) { flags |= RSPAMD_ACTION_HAM; } else { msg_warn_config("unknown action flag: %s", fl_str); } } } } elt = ucl_object_lookup(obj, "milter"); if (elt) { const char *milter_action = ucl_object_tostring(elt); if (strcmp(milter_action, "discard") == 0) { flags |= RSPAMD_ACTION_MILTER; act->action_type = METRIC_ACTION_DISCARD; } else if (strcmp(milter_action, "quarantine") == 0) { flags |= RSPAMD_ACTION_MILTER; act->action_type = METRIC_ACTION_QUARANTINE; } else { msg_warn_config("unknown milter action: %s", milter_action); } } } else if (obj_type == UCL_FLOAT || obj_type == UCL_INT) { threshold = ucl_object_todouble(obj); } /* TODO: add lua references support */ if (std::isnan(threshold) && !(flags & RSPAMD_ACTION_NO_THRESHOLD)) { msg_err_config("action %s has no threshold being set and it is not" " a no threshold action", act->name); return FALSE; } act->threshold = threshold; act->flags = flags; enum rspamd_action_type std_act; if (!(flags & RSPAMD_ACTION_MILTER)) { if (rspamd_action_from_str(act->name, &std_act)) { act->action_type = std_act; } else { act->action_type = METRIC_ACTION_CUSTOM; } } return TRUE; } gboolean rspamd_config_set_action_score(struct rspamd_config *cfg, const char *action_name, const ucl_object_t *obj) { enum rspamd_action_type std_act; const ucl_object_t *elt; unsigned int priority = ucl_object_get_priority(obj), obj_type; g_assert(cfg != nullptr); g_assert(action_name != nullptr); obj_type = ucl_object_type(obj); if (obj_type == UCL_OBJECT) { elt = ucl_object_lookup(obj, "priority"); if (elt) { priority = ucl_object_toint(elt); } } /* Here are dragons: * We have `canonical` name for actions, such as `soft reject` and * configuration names for actions (used to be more convenient), such * as `soft_reject`. Unfortunately, we must have heuristic for this * variance of names. */ if (rspamd_action_from_str(action_name, &std_act)) { action_name = rspamd_action_to_str(std_act); } auto actions = RSPAMD_CFG_ACTIONS(cfg); auto existing_act_it = actions->actions_by_name.find(action_name); if (existing_act_it != actions->actions_by_name.end()) { auto *act = existing_act_it->second.get(); /* Existing element */ if (act->priority <= priority) { /* We can replace data */ auto old_pri = act->priority; auto old_thr = act->threshold; if (rspamd_config_action_from_ucl(cfg, act, obj, priority)) { msg_info_config("action %s has been already registered with " "priority %ud, override it with new priority: %ud, " "old threshold: %.2f, new threshold: %.2f", action_name, old_pri, priority, old_thr, act->threshold); actions->sort(); } else { return FALSE; } } else { msg_info_config("action %s has been already registered with " "priority %ud, do not override (new priority: %ud)", action_name, act->priority, priority); } } else { /* Add new element */ auto act = std::make_shared(); act->name = rspamd_mempool_strdup(cfg->cfg_pool, action_name); if (rspamd_config_action_from_ucl(cfg, act.get(), obj, priority)) { actions->add_action(std::move(act)); } else { return FALSE; } } return TRUE; } gboolean rspamd_config_maybe_disable_action(struct rspamd_config *cfg, const char *action_name, unsigned int priority) { auto actions = RSPAMD_CFG_ACTIONS(cfg); auto maybe_act = rspamd::find_map(actions->actions_by_name, action_name); if (maybe_act) { auto *act = maybe_act.value().get().get(); if (priority >= act->priority) { msg_info_config("disable action %s; old priority: %ud, new priority: %ud", action_name, act->priority, priority); act->threshold = NAN; act->priority = priority; act->flags |= RSPAMD_ACTION_NO_THRESHOLD; return TRUE; } else { msg_info_config("action %s has been already registered with " "priority %ud, cannot disable it with new priority: %ud", action_name, act->priority, priority); } } return FALSE; } struct rspamd_action * rspamd_config_get_action(struct rspamd_config *cfg, const char *name) { auto actions = RSPAMD_CFG_ACTIONS(cfg); auto maybe_act = rspamd::find_map(actions->actions_by_name, name); if (maybe_act) { return maybe_act.value().get().get(); } return nullptr; } struct rspamd_action * rspamd_config_get_action_by_type(struct rspamd_config *cfg, enum rspamd_action_type type) { for (const auto &act: RSPAMD_CFG_ACTIONS(cfg)->actions) { if (act->action_type == type) { return act.get(); } } return nullptr; } void rspamd_config_actions_foreach(struct rspamd_config *cfg, void (*func)(struct rspamd_action *act, void *d), void *data) { for (const auto &act: RSPAMD_CFG_ACTIONS(cfg)->actions) { func(act.get(), data); } } void rspamd_config_actions_foreach_enumerate(struct rspamd_config *cfg, void (*func)(int idx, struct rspamd_action *act, void *d), void *data) { for (const auto &[idx, act]: rspamd::enumerate(RSPAMD_CFG_ACTIONS(cfg)->actions)) { func(idx, act.get(), data); } } gsize rspamd_config_actions_size(struct rspamd_config *cfg) { return RSPAMD_CFG_ACTIONS(cfg)->actions.size(); } gboolean rspamd_config_radix_from_ucl(struct rspamd_config *cfg, const ucl_object_t *obj, const char *description, struct rspamd_radix_map_helper **target, GError **err, struct rspamd_worker *worker, const char *map_name) { ucl_type_t type; ucl_object_iter_t it = nullptr; const ucl_object_t *cur, *cur_elt; const char *str; /* Cleanup */ *target = nullptr; LL_FOREACH(obj, cur_elt) { type = ucl_object_type(cur_elt); switch (type) { case UCL_STRING: /* Either map or a list of IPs */ str = ucl_object_tostring(cur_elt); if (rspamd_map_is_map(str)) { if (rspamd_map_add_from_ucl(cfg, cur_elt, description, rspamd_radix_read, rspamd_radix_fin, rspamd_radix_dtor, (void **) target, worker, RSPAMD_MAP_DEFAULT) == nullptr) { g_set_error(err, g_quark_from_static_string("rspamd-config"), EINVAL, "bad map definition %s for %s", str, ucl_object_key(obj)); return FALSE; } return TRUE; } else { /* Just a list */ if (!*target) { *target = rspamd_map_helper_new_radix( rspamd_map_add_fake(cfg, description, map_name)); } rspamd_map_helper_insert_radix_resolve(*target, str, ""); } break; case UCL_OBJECT: /* Should be a map description */ if (rspamd_map_add_from_ucl(cfg, cur_elt, description, rspamd_radix_read, rspamd_radix_fin, rspamd_radix_dtor, (void **) target, worker, RSPAMD_MAP_DEFAULT) == nullptr) { g_set_error(err, g_quark_from_static_string("rspamd-config"), EINVAL, "bad map object for %s", ucl_object_key(obj)); return FALSE; } return TRUE; break; case UCL_ARRAY: /* List of IP addresses */ it = ucl_object_iterate_new(cur_elt); while ((cur = ucl_object_iterate_safe(it, true)) != nullptr) { if (ucl_object_type(cur) == UCL_STRING) { str = ucl_object_tostring(cur); if (!*target) { *target = rspamd_map_helper_new_radix( rspamd_map_add_fake(cfg, description, map_name)); } rspamd_map_helper_insert_radix_resolve(*target, str, ""); } else { g_set_error(err, g_quark_from_static_string("rspamd-config"), EINVAL, "bad element inside array object for %s: expected string, got: %s", ucl_object_key(obj), ucl_object_type_to_string(ucl_object_type(cur))); ucl_object_iterate_free(it); return FALSE; } } ucl_object_iterate_free(it); break; default: g_set_error(err, g_quark_from_static_string("rspamd-config"), EINVAL, "bad map type %s for %s", ucl_object_type_to_string(type), ucl_object_key(obj)); return FALSE; } } /* Destroy on cfg cleanup */ rspamd_mempool_add_destructor(cfg->cfg_pool, (rspamd_mempool_destruct_t) rspamd_map_helper_destroy_radix, *target); return TRUE; } constexpr const auto action_types = frozen::make_unordered_map({ {"reject", METRIC_ACTION_REJECT}, {"greylist", METRIC_ACTION_GREYLIST}, {"add header", METRIC_ACTION_ADD_HEADER}, {"add_header", METRIC_ACTION_ADD_HEADER}, {"rewrite subject", METRIC_ACTION_REWRITE_SUBJECT}, {"rewrite_subject", METRIC_ACTION_REWRITE_SUBJECT}, {"soft reject", METRIC_ACTION_SOFT_REJECT}, {"soft_reject", METRIC_ACTION_SOFT_REJECT}, {"no action", METRIC_ACTION_NOACTION}, {"no_action", METRIC_ACTION_NOACTION}, {"accept", METRIC_ACTION_NOACTION}, {"quarantine", METRIC_ACTION_QUARANTINE}, {"discard", METRIC_ACTION_DISCARD}, }); gboolean rspamd_action_from_str(const char *data, enum rspamd_action_type *result) { auto maybe_action = rspamd::find_map(action_types, std::string_view{data}); if (maybe_action) { *result = maybe_action.value().get(); return true; } else { return false; } } const char * rspamd_action_to_str(enum rspamd_action_type action) { switch (action) { case METRIC_ACTION_REJECT: return "reject"; case METRIC_ACTION_SOFT_REJECT: return "soft reject"; case METRIC_ACTION_REWRITE_SUBJECT: return "rewrite subject"; case METRIC_ACTION_ADD_HEADER: return "add header"; case METRIC_ACTION_GREYLIST: return "greylist"; case METRIC_ACTION_NOACTION: return "no action"; case METRIC_ACTION_MAX: return "invalid max action"; case METRIC_ACTION_CUSTOM: return "custom"; case METRIC_ACTION_DISCARD: return "discard"; case METRIC_ACTION_QUARANTINE: return "quarantine"; } return "unknown action"; } const char * rspamd_action_to_str_alt(enum rspamd_action_type action) { switch (action) { case METRIC_ACTION_REJECT: return "reject"; case METRIC_ACTION_SOFT_REJECT: return "soft_reject"; case METRIC_ACTION_REWRITE_SUBJECT: return "rewrite_subject"; case METRIC_ACTION_ADD_HEADER: return "add_header"; case METRIC_ACTION_GREYLIST: return "greylist"; case METRIC_ACTION_NOACTION: return "no action"; case METRIC_ACTION_MAX: return "invalid max action"; case METRIC_ACTION_CUSTOM: return "custom"; case METRIC_ACTION_DISCARD: return "discard"; case METRIC_ACTION_QUARANTINE: return "quarantine"; } return "unknown action"; } static void rspamd_config_settings_elt_dtor(struct rspamd_config_settings_elt *e) { if (e->symbols_enabled) { ucl_object_unref(e->symbols_enabled); } if (e->symbols_disabled) { ucl_object_unref(e->symbols_disabled); } } uint32_t rspamd_config_name_to_id(const char *name, gsize namelen) { uint64_t h; h = rspamd_cryptobox_fast_hash_specific(RSPAMD_CRYPTOBOX_XXHASH64, name, namelen, 0x0); /* Take the lower part of hash as LE number */ return ((uint32_t) GUINT64_TO_LE(h)); } struct rspamd_config_settings_elt * rspamd_config_find_settings_id_ref(struct rspamd_config *cfg, uint32_t id) { struct rspamd_config_settings_elt *cur; DL_FOREACH(cfg->setting_ids, cur) { if (cur->id == id) { REF_RETAIN(cur); return cur; } } return nullptr; } struct rspamd_config_settings_elt *rspamd_config_find_settings_name_ref( struct rspamd_config *cfg, const char *name, gsize namelen) { uint32_t id; id = rspamd_config_name_to_id(name, namelen); return rspamd_config_find_settings_id_ref(cfg, id); } void rspamd_config_register_settings_id(struct rspamd_config *cfg, const char *name, ucl_object_t *symbols_enabled, ucl_object_t *symbols_disabled, enum rspamd_config_settings_policy policy) { struct rspamd_config_settings_elt *elt; uint32_t id; id = rspamd_config_name_to_id(name, strlen(name)); elt = rspamd_config_find_settings_id_ref(cfg, id); if (elt) { /* Need to replace */ struct rspamd_config_settings_elt *nelt; DL_DELETE(cfg->setting_ids, elt); nelt = rspamd_mempool_alloc0_type(cfg->cfg_pool, struct rspamd_config_settings_elt); nelt->id = id; nelt->name = rspamd_mempool_strdup(cfg->cfg_pool, name); if (symbols_enabled) { nelt->symbols_enabled = ucl_object_ref(symbols_enabled); } if (symbols_disabled) { nelt->symbols_disabled = ucl_object_ref(symbols_disabled); } nelt->policy = policy; REF_INIT_RETAIN(nelt, rspamd_config_settings_elt_dtor); msg_warn_config("replace settings id %ud (%s)", id, name); rspamd_symcache_process_settings_elt(cfg->cache, elt); DL_APPEND(cfg->setting_ids, nelt); /* * Need to unref old element twice as there are two reference holders: * 1. Config structure as we call REF_INIT_RETAIN * 2. rspamd_config_find_settings_id_ref also increases refcount */ REF_RELEASE(elt); REF_RELEASE(elt); } else { elt = rspamd_mempool_alloc0_type(cfg->cfg_pool, struct rspamd_config_settings_elt); elt->id = id; elt->name = rspamd_mempool_strdup(cfg->cfg_pool, name); if (symbols_enabled) { elt->symbols_enabled = ucl_object_ref(symbols_enabled); } if (symbols_disabled) { elt->symbols_disabled = ucl_object_ref(symbols_disabled); } elt->policy = policy; msg_info_config("register new settings id %ud (%s)", id, name); REF_INIT_RETAIN(elt, rspamd_config_settings_elt_dtor); rspamd_symcache_process_settings_elt(cfg->cache, elt); DL_APPEND(cfg->setting_ids, elt); } } int rspamd_config_ev_backend_get(struct rspamd_config *cfg) { #define AUTO_BACKEND (ev_supported_backends() & ~EVBACKEND_IOURING) if (cfg == nullptr || cfg->events_backend == nullptr) { return AUTO_BACKEND; } if (strcmp(cfg->events_backend, "auto") == 0) { return AUTO_BACKEND; } else if (strcmp(cfg->events_backend, "epoll") == 0) { if (ev_supported_backends() & EVBACKEND_EPOLL) { return EVBACKEND_EPOLL; } else { msg_warn_config("unsupported events_backend: %s; defaulting to auto", cfg->events_backend); return AUTO_BACKEND; } } else if (strcmp(cfg->events_backend, "iouring") == 0) { if (ev_supported_backends() & EVBACKEND_IOURING) { return EVBACKEND_IOURING; } else { msg_warn_config("unsupported events_backend: %s; defaulting to auto", cfg->events_backend); return AUTO_BACKEND; } } else if (strcmp(cfg->events_backend, "kqueue") == 0) { if (ev_supported_backends() & EVBACKEND_KQUEUE) { return EVBACKEND_KQUEUE; } else { msg_warn_config("unsupported events_backend: %s; defaulting to auto", cfg->events_backend); return AUTO_BACKEND; } } else if (strcmp(cfg->events_backend, "poll") == 0) { return EVBACKEND_POLL; } else if (strcmp(cfg->events_backend, "select") == 0) { return EVBACKEND_SELECT; } else { msg_warn_config("unknown events_backend: %s; defaulting to auto", cfg->events_backend); } return AUTO_BACKEND; } const char * rspamd_config_ev_backend_to_string(int ev_backend, gboolean *effective) { #define SET_EFFECTIVE(b) \ do { \ if ((effective) != nullptr) *(effective) = b; \ } while (0) if ((ev_backend & EVBACKEND_ALL) == EVBACKEND_ALL) { SET_EFFECTIVE(TRUE); return "auto"; } if (ev_backend & EVBACKEND_IOURING) { SET_EFFECTIVE(TRUE); return "epoll+io_uring"; } if (ev_backend & EVBACKEND_LINUXAIO) { SET_EFFECTIVE(TRUE); return "epoll+aio"; } if (ev_backend & EVBACKEND_IOURING) { SET_EFFECTIVE(TRUE); return "epoll+io_uring"; } if (ev_backend & EVBACKEND_LINUXAIO) { SET_EFFECTIVE(TRUE); return "epoll+aio"; } if (ev_backend & EVBACKEND_EPOLL) { SET_EFFECTIVE(TRUE); return "epoll"; } if (ev_backend & EVBACKEND_KQUEUE) { SET_EFFECTIVE(TRUE); return "kqueue"; } if (ev_backend & EVBACKEND_POLL) { SET_EFFECTIVE(FALSE); return "poll"; } if (ev_backend & EVBACKEND_SELECT) { SET_EFFECTIVE(FALSE); return "select"; } SET_EFFECTIVE(FALSE); return "unknown"; #undef SET_EFFECTIVE } struct rspamd_external_libs_ctx * rspamd_init_libs(void) { struct rlimit rlim; struct ottery_config *ottery_cfg; auto *ctx = g_new0(struct rspamd_external_libs_ctx, 1); ctx->crypto_ctx = rspamd_cryptobox_init(); ottery_cfg = (struct ottery_config *) g_malloc0(ottery_get_sizeof_config()); ottery_config_init(ottery_cfg); ctx->ottery_cfg = ottery_cfg; rspamd_openssl_maybe_init(); /* Check if we have rdrand */ if ((ctx->crypto_ctx->cpu_config & CPUID_RDRAND) == 0) { ottery_config_disable_entropy_sources(ottery_cfg, OTTERY_ENTROPY_SRC_RDRAND); } g_assert(ottery_init(ottery_cfg) == 0); #if OPENSSL_VERSION_NUMBER >= 0x1000104fL && OPENSSL_VERSION_NUMBER < 0x30000000L && !defined(LIBRESSL_VERSION_NUMBER) RAND_set_rand_engine(nullptr); #endif /* Configure utf8 library */ unsigned int utf8_flags = 0; if ((ctx->crypto_ctx->cpu_config & CPUID_SSE41)) { utf8_flags |= RSPAMD_FAST_UTF8_FLAG_SSE41; } if ((ctx->crypto_ctx->cpu_config & CPUID_AVX2)) { utf8_flags |= RSPAMD_FAST_UTF8_FLAG_AVX2; } rspamd_fast_utf8_library_init(utf8_flags); #ifdef HAVE_LOCALE_H if (getenv("LANG") == nullptr) { setlocale(LC_ALL, "C"); setlocale(LC_CTYPE, "C"); setlocale(LC_MESSAGES, "C"); setlocale(LC_TIME, "C"); } else { /* Just set the default locale */ setlocale(LC_ALL, ""); /* But for some issues we still want C locale */ setlocale(LC_NUMERIC, "C"); } #endif ctx->ssl_ctx = rspamd_init_ssl_ctx(); ctx->ssl_ctx_noverify = rspamd_init_ssl_ctx_noverify(); rspamd_random_seed_fast(); /* Set stack size for pcre */ getrlimit(RLIMIT_STACK, &rlim); rlim.rlim_cur = 100 * 1024 * 1024; rlim.rlim_max = rlim.rlim_cur; setrlimit(RLIMIT_STACK, &rlim); ctx->local_addrs = rspamd_inet_library_init(); REF_INIT_RETAIN(ctx, rspamd_deinit_libs); return ctx; } static struct zstd_dictionary * rspamd_open_zstd_dictionary(const char *path) { struct zstd_dictionary *dict; dict = g_new0(zstd_dictionary, 1); dict->dict = rspamd_file_xmap(path, PROT_READ, &dict->size, TRUE); if (dict->dict == nullptr) { g_free(dict); return nullptr; } dict->id = -1; if (dict->id == 0) { g_free(dict); return nullptr; } return dict; } static void rspamd_free_zstd_dictionary(struct zstd_dictionary *dict) { if (dict) { munmap(dict->dict, dict->size); g_free(dict); } } #ifdef HAVE_OPENBLAS_SET_NUM_THREADS extern "C" void openblas_set_num_threads(int num_threads); #endif #ifdef HAVE_BLI_THREAD_SET_NUM_THREADS extern "C" void bli_thread_set_num_threads(int num_threads); #endif gboolean rspamd_config_libs(struct rspamd_external_libs_ctx *ctx, struct rspamd_config *cfg) { size_t r; gboolean ret = TRUE; g_assert(cfg != nullptr); if (ctx != nullptr) { if (cfg->local_addrs) { GError *err = nullptr; ret = rspamd_config_radix_from_ucl(cfg, cfg->local_addrs, "Local addresses", (struct rspamd_radix_map_helper **) ctx->local_addrs, &err, nullptr, "local addresses"); if (!ret) { msg_err_config("cannot load local addresses: %e", err); g_error_free(err); return ret; } } rspamd_free_zstd_dictionary(ctx->in_dict); rspamd_free_zstd_dictionary(ctx->out_dict); if (ctx->out_zstream) { ZSTD_freeCStream((ZSTD_CCtx *) ctx->out_zstream); ctx->out_zstream = nullptr; } if (ctx->in_zstream) { ZSTD_freeDStream((ZSTD_DCtx *) ctx->in_zstream); ctx->in_zstream = nullptr; } if (cfg->zstd_input_dictionary) { ctx->in_dict = rspamd_open_zstd_dictionary( cfg->zstd_input_dictionary); if (ctx->in_dict == nullptr) { msg_err_config("cannot open zstd dictionary in %s", cfg->zstd_input_dictionary); } } if (cfg->zstd_output_dictionary) { ctx->out_dict = rspamd_open_zstd_dictionary( cfg->zstd_output_dictionary); if (ctx->out_dict == nullptr) { msg_err_config("cannot open zstd dictionary in %s", cfg->zstd_output_dictionary); } } if (cfg->fips_mode) { #ifdef HAVE_FIPS_MODE int mode = FIPS_mode(); unsigned long err = (unsigned long) -1; /* Toggle FIPS mode */ if (mode == 0) { #if defined(OPENSSL_VERSION_MAJOR) && (OPENSSL_VERSION_MAJOR >= 3) if (EVP_set_default_properties(nullptr, "fips=yes") != 1) { #else if (FIPS_mode_set(1) != 1) { #endif err = ERR_get_error(); } } else { msg_info_config("OpenSSL FIPS mode is already enabled"); } if (err != (unsigned long) -1) { #if defined(OPENSSL_VERSION_MAJOR) && (OPENSSL_VERSION_MAJOR >= 3) msg_err_config("EVP_set_default_properties failed: %s", #else msg_err_config("FIPS_mode_set failed: %s", #endif ERR_error_string(err, nullptr)); ret = FALSE; } else { msg_info_config("OpenSSL FIPS mode is enabled"); } #else msg_warn_config("SSL FIPS mode is enabled but not supported by OpenSSL library!"); #endif } rspamd_ssl_ctx_config(cfg, ctx->ssl_ctx); rspamd_ssl_ctx_config(cfg, ctx->ssl_ctx_noverify); /* Init decompression */ ctx->in_zstream = ZSTD_createDStream(); r = ZSTD_initDStream((ZSTD_DCtx *) ctx->in_zstream); if (ZSTD_isError(r)) { msg_err("cannot init decompression stream: %s", ZSTD_getErrorName(r)); ZSTD_freeDStream((ZSTD_DCtx *) ctx->in_zstream); ctx->in_zstream = nullptr; } /* Init compression */ ctx->out_zstream = ZSTD_createCStream(); r = ZSTD_initCStream((ZSTD_CCtx *) ctx->out_zstream, 1); if (ZSTD_isError(r)) { msg_err("cannot init compression stream: %s", ZSTD_getErrorName(r)); ZSTD_freeCStream((ZSTD_CCtx *) ctx->out_zstream); ctx->out_zstream = nullptr; } #ifdef HAVE_OPENBLAS_SET_NUM_THREADS openblas_set_num_threads(cfg->max_blas_threads); #endif #ifdef HAVE_BLI_THREAD_SET_NUM_THREADS bli_thread_set_num_threads(cfg->max_blas_threads); #endif } return ret; } gboolean rspamd_libs_reset_decompression(struct rspamd_external_libs_ctx *ctx) { gsize r; if (ctx->in_zstream == nullptr) { return FALSE; } else { r = ZSTD_DCtx_reset((ZSTD_DCtx *) ctx->in_zstream, ZSTD_reset_session_only); if (ZSTD_isError(r)) { msg_err("cannot init decompression stream: %s", ZSTD_getErrorName(r)); ZSTD_freeDStream((ZSTD_DCtx *) ctx->in_zstream); ctx->in_zstream = nullptr; return FALSE; } } return TRUE; } gboolean rspamd_libs_reset_compression(struct rspamd_external_libs_ctx *ctx) { gsize r; if (ctx->out_zstream == nullptr) { return FALSE; } else { /* Dictionary will be reused automatically if specified */ r = ZSTD_CCtx_reset((ZSTD_CCtx *) ctx->out_zstream, ZSTD_reset_session_only); if (!ZSTD_isError(r)) { r = ZSTD_CCtx_setPledgedSrcSize((ZSTD_CCtx *) ctx->out_zstream, ZSTD_CONTENTSIZE_UNKNOWN); } if (ZSTD_isError(r)) { msg_err("cannot init compression stream: %s", ZSTD_getErrorName(r)); ZSTD_freeCStream((ZSTD_CCtx *) ctx->out_zstream); ctx->out_zstream = nullptr; return FALSE; } } return TRUE; } void rspamd_deinit_libs(struct rspamd_external_libs_ctx *ctx) { if (ctx != nullptr) { g_free(ctx->ottery_cfg); #ifdef HAVE_OPENSSL EVP_cleanup(); ERR_free_strings(); rspamd_ssl_ctx_free(ctx->ssl_ctx); rspamd_ssl_ctx_free(ctx->ssl_ctx_noverify); #endif rspamd_inet_library_destroy(); rspamd_free_zstd_dictionary(ctx->in_dict); rspamd_free_zstd_dictionary(ctx->out_dict); if (ctx->out_zstream) { ZSTD_freeCStream((ZSTD_CCtx *) ctx->out_zstream); } if (ctx->in_zstream) { ZSTD_freeDStream((ZSTD_DCtx *) ctx->in_zstream); } rspamd_cryptobox_deinit(ctx->crypto_ctx); g_free(ctx); } } gboolean rspamd_ip_is_local_cfg(struct rspamd_config *cfg, const rspamd_inet_addr_t *addr) { struct rspamd_radix_map_helper *local_addrs = nullptr; if (cfg && cfg->libs_ctx) { local_addrs = *(struct rspamd_radix_map_helper **) cfg->libs_ctx->local_addrs; } if (rspamd_inet_address_is_local(addr)) { return TRUE; } if (local_addrs) { if (rspamd_match_radix_map_addr(local_addrs, addr) != nullptr) { return TRUE; } } return FALSE; }