/* * Copyright 2023 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "config.h" #include "lua/lua_common.h" #include "lua/lua_thread_pool.h" #include "cfg_file.h" #include "rspamd.h" #include "cfg_file_private.h" #include "maps/map.h" #include "maps/map_helpers.h" #include "maps/map_private.h" #include "dynamic_cfg.h" #include "utlist.h" #include "stat_api.h" #include "unix-std.h" #include "libutil/multipattern.h" #include "monitored.h" #include "ref.h" #include "cryptobox.h" #include "ssl_util.h" #include "contrib/libottery/ottery.h" #include "contrib/fastutf8/fastutf8.h" #ifdef SYS_ZSTD #include "zstd.h" #else #define ZSTD_STATIC_LINKING_ONLY #include "contrib/zstd/zstd.h" #endif #ifdef HAVE_OPENSSL #include #include #include #include #include #endif #ifdef HAVE_LOCALE_H #include #endif #ifdef HAVE_SYS_RESOURCE_H #include #endif #include #include "libserver/composites/composites.h" #include "blas-config.h" #include #include #include #include "fmt/core.h" #include "cxx/util.hxx" #include "frozen/unordered_map.h" #include "frozen/string.h" #include "contrib/ankerl/unordered_dense.h" #define DEFAULT_SCORE 10.0 #define DEFAULT_RLIMIT_NOFILE 2048 #define DEFAULT_RLIMIT_MAXCORE 0 #define DEFAULT_MAP_TIMEOUT 60.0 * 5 #define DEFAULT_MAP_FILE_WATCH_MULTIPLIER 1 #define DEFAULT_MIN_WORD 0 #define DEFAULT_MAX_WORD 40 #define DEFAULT_WORDS_DECAY 600 #define DEFAULT_MAX_MESSAGE (50 * 1024 * 1024) #define DEFAULT_MAX_PIC (1 * 1024 * 1024) #define DEFAULT_MAX_SHOTS 100 #define DEFAULT_MAX_SESSIONS 100 #define DEFAULT_MAX_WORKERS 4 #define DEFAULT_MAX_HTML_SIZE DEFAULT_MAX_MESSAGE / 5 /* 10 Mb */ /* Timeout for task processing */ #define DEFAULT_TASK_TIMEOUT 8.0 #define DEFAULT_LUA_GC_STEP 200 #define DEFAULT_LUA_GC_PAUSE 200 #define DEFAULT_GC_MAXITERS 0 struct rspamd_ucl_map_cbdata { struct rspamd_config *cfg; std::string buf; explicit rspamd_ucl_map_cbdata(struct rspamd_config *cfg) : cfg(cfg) { } }; static gchar *rspamd_ucl_read_cb(gchar *chunk, gint len, struct map_cb_data *data, gboolean final); static void rspamd_ucl_fin_cb(struct map_cb_data *data, void **target); static void rspamd_ucl_dtor_cb(struct map_cb_data *data); guint rspamd_config_log_id = (guint) -1; RSPAMD_CONSTRUCTOR(rspamd_config_log_init) { rspamd_config_log_id = rspamd_logger_add_debug_module("config"); } struct rspamd_actions_list { using action_ptr = std::shared_ptr; std::vector actions; ankerl::unordered_dense::map actions_by_name; explicit rspamd_actions_list() { actions.reserve(METRIC_ACTION_MAX + 2); actions_by_name.reserve(METRIC_ACTION_MAX + 2); } void add_action(action_ptr action) { actions.push_back(action); actions_by_name[action->name] = action; sort(); } void sort() { std::sort(actions.begin(), actions.end(), [](const action_ptr &a1, const action_ptr &a2) -> bool { if (!isnan(a1->threshold) && !isnan(a2->threshold)) { return a1->threshold < a2->threshold; } if (isnan(a1->threshold) && isnan(a2->threshold)) { return false; } else if (isnan(a1->threshold)) { return true; } return false; }); } void clear() { actions.clear(); actions_by_name.clear(); } }; #define RSPAMD_CFG_ACTIONS(cfg) (reinterpret_cast((cfg)->actions)) gboolean rspamd_parse_bind_line(struct rspamd_config *cfg, struct rspamd_worker_conf *cf, const gchar *str) { struct rspamd_worker_bind_conf *cnf; const gchar *fdname; gboolean ret = TRUE; if (str == nullptr) { return FALSE; } cnf = rspamd_mempool_alloc0_type(cfg->cfg_pool, struct rspamd_worker_bind_conf); cnf->cnt = 1024; cnf->bind_line = rspamd_mempool_strdup(cfg->cfg_pool, str); auto bind_line = std::string_view{cnf->bind_line}; if (bind_line.starts_with("systemd:")) { /* The actual socket will be passed by systemd environment */ fdname = str + sizeof("systemd:") - 1; cnf->is_systemd = TRUE; cnf->addrs = g_ptr_array_new_full(1, nullptr); rspamd_mempool_add_destructor(cfg->cfg_pool, rspamd_ptr_array_free_hard, cnf->addrs); if (fdname[0]) { g_ptr_array_add(cnf->addrs, rspamd_mempool_strdup(cfg->cfg_pool, fdname)); cnf->cnt = cnf->addrs->len; cnf->name = rspamd_mempool_strdup(cfg->cfg_pool, str); LL_PREPEND(cf->bind_conf, cnf); } else { msg_err_config("cannot parse bind line: %s", str); ret = FALSE; } } else { if (rspamd_parse_host_port_priority(str, &cnf->addrs, nullptr, &cnf->name, DEFAULT_BIND_PORT, TRUE, cfg->cfg_pool) == RSPAMD_PARSE_ADDR_FAIL) { msg_err_config("cannot parse bind line: %s", str); ret = FALSE; } else { cnf->cnt = cnf->addrs->len; LL_PREPEND(cf->bind_conf, cnf); } } return ret; } struct rspamd_config * rspamd_config_new(enum rspamd_config_init_flags flags) { struct rspamd_config *cfg; rspamd_mempool_t *pool; pool = rspamd_mempool_new(8 * 1024 * 1024, "cfg", 0); cfg = rspamd_mempool_alloc0_type(pool, struct rspamd_config); /* Allocate larger pool for cfg */ cfg->cfg_pool = pool; cfg->dns_timeout = 1.0; cfg->dns_retransmits = 5; /* 16 sockets per DNS server */ cfg->dns_io_per_server = 16; cfg->unknown_weight = NAN; cfg->actions = (void *) new rspamd_actions_list(); /* Add all internal actions to keep compatibility */ for (int i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i++) { auto &&action = std::make_shared(); action->threshold = NAN; action->name = rspamd_mempool_strdup(cfg->cfg_pool, rspamd_action_to_str(static_cast(i))); action->action_type = static_cast(i); if (i == METRIC_ACTION_SOFT_REJECT) { action->flags |= RSPAMD_ACTION_NO_THRESHOLD | RSPAMD_ACTION_HAM; } else if (i == METRIC_ACTION_GREYLIST) { action->flags |= RSPAMD_ACTION_THRESHOLD_ONLY | RSPAMD_ACTION_HAM; } else if (i == METRIC_ACTION_NOACTION) { action->flags |= RSPAMD_ACTION_HAM; } RSPAMD_CFG_ACTIONS(cfg)->add_action(std::move(action)); } /* Disable timeout */ cfg->task_timeout = DEFAULT_TASK_TIMEOUT; rspamd_config_init_metric(cfg); cfg->composites_manager = rspamd_composites_manager_create(cfg); cfg->classifiers_symbols = g_hash_table_new(rspamd_str_hash, rspamd_str_equal); cfg->cfg_params = g_hash_table_new(rspamd_str_hash, rspamd_str_equal); cfg->debug_modules = g_hash_table_new(rspamd_str_hash, rspamd_str_equal); cfg->explicit_modules = g_hash_table_new(rspamd_str_hash, rspamd_str_equal); cfg->trusted_keys = g_hash_table_new(rspamd_str_hash, rspamd_str_equal); cfg->map_timeout = DEFAULT_MAP_TIMEOUT; cfg->map_file_watch_multiplier = DEFAULT_MAP_FILE_WATCH_MULTIPLIER; cfg->log_level = G_LOG_LEVEL_WARNING; cfg->log_flags = RSPAMD_LOG_FLAG_DEFAULT; cfg->check_text_attachements = TRUE; cfg->dns_max_requests = 64; cfg->history_rows = 200; cfg->log_error_elts = 10; cfg->log_error_elt_maxlen = 1000; cfg->log_task_max_elts = 7; cfg->cache_reload_time = 30.0; cfg->max_lua_urls = 1024; cfg->max_urls = cfg->max_lua_urls * 10; cfg->max_recipients = 1024; cfg->max_blas_threads = 1; cfg->max_opts_len = 4096; cfg->gtube_patterns_policy = RSPAMD_GTUBE_REJECT; /* Default log line */ cfg->log_format_str = rspamd_mempool_strdup(cfg->cfg_pool, "id: <$mid>,$if_qid{ qid: <$>,}$if_ip{ ip: $,}" "$if_user{ user: $,}$if_smtp_from{ from: <$>,} (default: $is_spam " "($action): [$scores] [$symbols_scores_params]), len: $len, time: $time_real, " "dns req: $dns_req, digest: <$digest>" "$if_smtp_rcpts{ rcpts: <$>, }$if_mime_rcpt{ mime_rcpt: <$>, }"); /* Allow non-mime input by default */ cfg->allow_raw_input = TRUE; /* Default maximum words processed */ cfg->words_decay = DEFAULT_WORDS_DECAY; cfg->min_word_len = DEFAULT_MIN_WORD; cfg->max_word_len = DEFAULT_MAX_WORD; cfg->max_html_len = DEFAULT_MAX_HTML_SIZE; /* GC limits */ cfg->lua_gc_pause = DEFAULT_LUA_GC_PAUSE; cfg->lua_gc_step = DEFAULT_LUA_GC_STEP; cfg->full_gc_iters = DEFAULT_GC_MAXITERS; /* Default hyperscan cache */ cfg->hs_cache_dir = rspamd_mempool_strdup(cfg->cfg_pool, RSPAMD_DBDIR "/"); if (!(flags & RSPAMD_CONFIG_INIT_SKIP_LUA)) { cfg->lua_state = (void *) rspamd_lua_init(flags & RSPAMD_CONFIG_INIT_WIPE_LUA_MEM); cfg->own_lua_state = TRUE; cfg->lua_thread_pool = (void *) lua_thread_pool_new(RSPAMD_LUA_CFG_STATE(cfg)); } cfg->cache = rspamd_symcache_new(cfg); cfg->ups_ctx = rspamd_upstreams_library_init(); cfg->re_cache = rspamd_re_cache_new(); cfg->doc_strings = ucl_object_typed_new(UCL_OBJECT); /* * Unless exim is fixed */ cfg->enable_shutdown_workaround = TRUE; cfg->ssl_ciphers = rspamd_mempool_strdup(cfg->cfg_pool, "HIGH:!anullptr:!kRSA:!PSK:!SRP:!MD5:!RC4"); cfg->max_message = DEFAULT_MAX_MESSAGE; cfg->max_pic_size = DEFAULT_MAX_PIC; cfg->images_cache_size = 256; cfg->monitored_ctx = rspamd_monitored_ctx_init(); cfg->neighbours = ucl_object_typed_new(UCL_OBJECT); cfg->redis_pool = rspamd_redis_pool_init(); cfg->default_max_shots = DEFAULT_MAX_SHOTS; cfg->max_sessions_cache = DEFAULT_MAX_SESSIONS; cfg->maps_cache_dir = rspamd_mempool_strdup(cfg->cfg_pool, RSPAMD_DBDIR); cfg->c_modules = g_ptr_array_new(); cfg->heartbeat_interval = 10.0; cfg->enable_css_parser = true; cfg->script_modules = g_ptr_array_new(); REF_INIT_RETAIN(cfg, rspamd_config_free); return cfg; } void rspamd_config_free(struct rspamd_config *cfg) { struct rspamd_config_cfg_lua_script *sc, *sctmp; struct rspamd_config_settings_elt *set, *stmp; struct rspamd_worker_log_pipe *lp, *ltmp; rspamd_lua_run_config_unload(RSPAMD_LUA_CFG_STATE(cfg), cfg); /* Scripts part */ DL_FOREACH_SAFE(cfg->on_term_scripts, sc, sctmp) { luaL_unref(RSPAMD_LUA_CFG_STATE(cfg), LUA_REGISTRYINDEX, sc->cbref); } DL_FOREACH_SAFE(cfg->on_load_scripts, sc, sctmp) { luaL_unref(RSPAMD_LUA_CFG_STATE(cfg), LUA_REGISTRYINDEX, sc->cbref); } DL_FOREACH_SAFE(cfg->post_init_scripts, sc, sctmp) { luaL_unref(RSPAMD_LUA_CFG_STATE(cfg), LUA_REGISTRYINDEX, sc->cbref); } DL_FOREACH_SAFE(cfg->config_unload_scripts, sc, sctmp) { luaL_unref(RSPAMD_LUA_CFG_STATE(cfg), LUA_REGISTRYINDEX, sc->cbref); } DL_FOREACH_SAFE(cfg->setting_ids, set, stmp) { REF_RELEASE(set); } rspamd_map_remove_all(cfg); rspamd_mempool_destructors_enforce(cfg->cfg_pool); g_list_free(cfg->classifiers); g_list_free(cfg->workers); rspamd_symcache_destroy(cfg->cache); ucl_object_unref(cfg->cfg_ucl_obj); ucl_object_unref(cfg->config_comments); ucl_object_unref(cfg->doc_strings); ucl_object_unref(cfg->neighbours); g_hash_table_remove_all(cfg->cfg_params); g_hash_table_unref(cfg->cfg_params); g_hash_table_unref(cfg->classifiers_symbols); g_hash_table_unref(cfg->debug_modules); g_hash_table_unref(cfg->explicit_modules); g_hash_table_unref(cfg->trusted_keys); rspamd_re_cache_unref(cfg->re_cache); g_ptr_array_free(cfg->c_modules, TRUE); g_ptr_array_free(cfg->script_modules, TRUE); if (cfg->monitored_ctx) { rspamd_monitored_ctx_destroy(cfg->monitored_ctx); } if (RSPAMD_LUA_CFG_STATE(cfg) && cfg->own_lua_state) { lua_thread_pool_free((struct lua_thread_pool *) cfg->lua_thread_pool); rspamd_lua_close(RSPAMD_LUA_CFG_STATE(cfg)); } if (cfg->redis_pool) { rspamd_redis_pool_destroy(cfg->redis_pool); } rspamd_upstreams_library_unref(cfg->ups_ctx); delete RSPAMD_CFG_ACTIONS(cfg); rspamd_mempool_destructors_enforce(cfg->cfg_pool); if (cfg->checksum) { g_free(cfg->checksum); } REF_RELEASE(cfg->libs_ctx); DL_FOREACH_SAFE(cfg->log_pipes, lp, ltmp) { close(lp->fd); g_free(lp); } rspamd_mempool_delete(cfg->cfg_pool); } const ucl_object_t * rspamd_config_get_module_opt(struct rspamd_config *cfg, const gchar *module_name, const gchar *opt_name) { const ucl_object_t *res = nullptr, *sec; sec = ucl_obj_get_key(cfg->cfg_ucl_obj, module_name); if (sec != nullptr) { res = ucl_obj_get_key(sec, opt_name); } return res; } gint rspamd_config_parse_flag(const gchar *str, guint len) { gint c; if (!str || !*str) { return -1; } if (len == 0) { len = strlen(str); } switch (len) { case 1: c = g_ascii_tolower(*str); if (c == 'y' || c == '1') { return 1; } else if (c == 'n' || c == '0') { return 0; } break; case 2: if (g_ascii_strncasecmp(str, "no", len) == 0) { return 0; } else if (g_ascii_strncasecmp(str, "on", len) == 0) { return 1; } break; case 3: if (g_ascii_strncasecmp(str, "yes", len) == 0) { return 1; } else if (g_ascii_strncasecmp(str, "off", len) == 0) { return 0; } break; case 4: if (g_ascii_strncasecmp(str, "true", len) == 0) { return 1; } break; case 5: if (g_ascii_strncasecmp(str, "false", len) == 0) { return 0; } break; } return -1; } // A mapping between names and log format types + flags constexpr const auto config_vars = frozen::make_unordered_map>({ {"mid", {RSPAMD_LOG_MID, 0}}, {"qid", {RSPAMD_LOG_QID, 0}}, {"user", {RSPAMD_LOG_USER, 0}}, {"ip", {RSPAMD_LOG_IP, 0}}, {"len", {RSPAMD_LOG_LEN, 0}}, {"dns_req", {RSPAMD_LOG_DNS_REQ, 0}}, {"smtp_from", {RSPAMD_LOG_SMTP_FROM, 0}}, {"mime_from", {RSPAMD_LOG_MIME_FROM, 0}}, {"smtp_rcpt", {RSPAMD_LOG_SMTP_RCPT, 0}}, {"mime_rcpt", {RSPAMD_LOG_MIME_RCPT, 0}}, {"smtp_rcpts", {RSPAMD_LOG_SMTP_RCPTS, 0}}, {"mime_rcpts", {RSPAMD_LOG_MIME_RCPTS, 0}}, {"time_real", {RSPAMD_LOG_TIME_REAL, 0}}, {"time_virtual", {RSPAMD_LOG_TIME_VIRTUAL, 0}}, {"lua", {RSPAMD_LOG_LUA, 0}}, {"digest", {RSPAMD_LOG_DIGEST, 0}}, {"checksum", {RSPAMD_LOG_DIGEST, 0}}, {"filename", {RSPAMD_LOG_FILENAME, 0}}, {"forced_action", {RSPAMD_LOG_FORCED_ACTION, 0}}, {"settings_id", {RSPAMD_LOG_SETTINGS_ID, 0}}, {"mempool_size", {RSPAMD_LOG_MEMPOOL_SIZE, 0}}, {"mempool_waste", {RSPAMD_LOG_MEMPOOL_WASTE, 0}}, {"action", {RSPAMD_LOG_ACTION, 0}}, {"scores", {RSPAMD_LOG_SCORES, 0}}, {"symbols", {RSPAMD_LOG_SYMBOLS, 0}}, {"symbols_scores", {RSPAMD_LOG_SYMBOLS, RSPAMD_LOG_FMT_FLAG_SYMBOLS_SCORES}}, {"symbols_params", {RSPAMD_LOG_SYMBOLS, RSPAMD_LOG_FMT_FLAG_SYMBOLS_PARAMS}}, {"symbols_scores_params", {RSPAMD_LOG_SYMBOLS, RSPAMD_LOG_FMT_FLAG_SYMBOLS_PARAMS | RSPAMD_LOG_FMT_FLAG_SYMBOLS_SCORES}}, {"groups", {RSPAMD_LOG_GROUPS, 0}}, {"public_groups", {RSPAMD_LOG_PUBLIC_GROUPS, 0}}, {"is_spam", {RSPAMD_LOG_ISSPAM, 0}}, }); static gboolean rspamd_config_process_var(struct rspamd_config *cfg, const rspamd_ftok_t *var, const rspamd_ftok_t *content) { g_assert(var != nullptr); auto flags = 0; auto lc_var = std::string{var->begin, var->len}; std::transform(lc_var.begin(), lc_var.end(), lc_var.begin(), g_ascii_tolower); auto tok = std::string_view{lc_var}; if (var->len > 3 && tok.starts_with("if_")) { flags |= RSPAMD_LOG_FMT_FLAG_CONDITION; tok = tok.substr(3); } auto maybe_fmt_var = rspamd::find_map(config_vars, tok); if (maybe_fmt_var) { auto &fmt_var = maybe_fmt_var.value().get(); auto *log_format = rspamd_mempool_alloc0_type(cfg->cfg_pool, rspamd_log_format); log_format->type = fmt_var.first; log_format->flags = fmt_var.second | flags; if (log_format->type != RSPAMD_LOG_LUA) { if (content && content->len > 0) { log_format->data = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(rspamd_ftok_t)); memcpy(log_format->data, content, sizeof(*content)); log_format->len = sizeof(*content); } } else { /* Load lua code and ensure that we have function ref returned */ if (!content || content->len == 0) { msg_err_config("lua variable needs content: %T", &tok); return FALSE; } if (luaL_loadbuffer(RSPAMD_LUA_CFG_STATE(cfg), content->begin, content->len, "lua log variable") != 0) { msg_err_config("error loading lua code: '%T': %s", content, lua_tostring(RSPAMD_LUA_CFG_STATE(cfg), -1)); return FALSE; } if (lua_pcall(RSPAMD_LUA_CFG_STATE(cfg), 0, 1, 0) != 0) { msg_err_config("error executing lua code: '%T': %s", content, lua_tostring(RSPAMD_LUA_CFG_STATE(cfg), -1)); lua_pop(RSPAMD_LUA_CFG_STATE(cfg), 1); return FALSE; } if (lua_type(RSPAMD_LUA_CFG_STATE(cfg), -1) != LUA_TFUNCTION) { msg_err_config("lua variable should return function: %T", content); lua_pop(RSPAMD_LUA_CFG_STATE(cfg), 1); return FALSE; } auto id = luaL_ref(RSPAMD_LUA_CFG_STATE(cfg), LUA_REGISTRYINDEX); log_format->data = GINT_TO_POINTER(id); log_format->len = 0; } DL_APPEND(cfg->log_format, log_format); } else { std::string known_formats; for (const auto &v: config_vars) { known_formats += std::string_view{v.first.data(), v.first.size()}; known_formats += ", "; } if (known_formats.size() > 2) { // Remove last comma known_formats.resize(known_formats.size() - 2); } msg_err_config("unknown log variable: %T, known vars are: \"%s\"", var, known_formats.c_str()); return FALSE; } return TRUE; } static gboolean rspamd_config_parse_log_format(struct rspamd_config *cfg) { const gchar *p, *c, *end, *s; gchar *d; struct rspamd_log_format *lf = nullptr; rspamd_ftok_t var, var_content; enum { parse_str, parse_dollar, parse_var_name, parse_var_content, } state = parse_str; gint braces = 0; g_assert(cfg != nullptr); c = cfg->log_format_str; if (c == nullptr) { return FALSE; } p = c; end = p + strlen(p); while (p < end) { switch (state) { case parse_str: if (*p == '$') { state = parse_dollar; } else { p++; } break; case parse_dollar: if (p > c) { /* We have string element that we need to store */ lf = rspamd_mempool_alloc0_type(cfg->cfg_pool, struct rspamd_log_format); lf->type = RSPAMD_LOG_STRING; lf->data = rspamd_mempool_alloc(cfg->cfg_pool, p - c + 1); /* Filter \r\n from the destination */ s = c; d = (char *) lf->data; while (s < p) { if (*s != '\r' && *s != '\n') { *d++ = *s++; } else { *d++ = ' '; s++; } } *d = '\0'; lf->len = d - (char *) lf->data; DL_APPEND(cfg->log_format, lf); lf = nullptr; } p++; c = p; state = parse_var_name; break; case parse_var_name: if (*p == '{') { var.begin = c; var.len = p - c; p++; c = p; state = parse_var_content; braces = 1; } else if (*p != '_' && *p != '-' && !g_ascii_isalnum(*p)) { /* Variable with no content */ var.begin = c; var.len = p - c; c = p; if (!rspamd_config_process_var(cfg, &var, nullptr)) { return FALSE; } state = parse_str; } else { p++; } break; case parse_var_content: if (*p == '}' && --braces == 0) { var_content.begin = c; var_content.len = p - c; p++; c = p; if (!rspamd_config_process_var(cfg, &var, &var_content)) { return FALSE; } state = parse_str; } else if (*p == '{') { braces++; p++; } else { p++; } break; } } /* Last state */ switch (state) { case parse_str: if (p > c) { /* We have string element that we need to store */ lf = rspamd_mempool_alloc0_type(cfg->cfg_pool, struct rspamd_log_format); lf->type = RSPAMD_LOG_STRING; lf->data = rspamd_mempool_alloc(cfg->cfg_pool, p - c + 1); /* Filter \r\n from the destination */ s = c; d = (char *) lf->data; while (s < p) { if (*s != '\r' && *s != '\n') { *d++ = *s++; } else { *d++ = ' '; s++; } } *d = '\0'; lf->len = d - (char *) lf->data; DL_APPEND(cfg->log_format, lf); lf = nullptr; } break; case parse_var_name: var.begin = c; var.len = p - c; if (!rspamd_config_process_var(cfg, &var, nullptr)) { return FALSE; } break; case parse_dollar: case parse_var_content: msg_err_config("cannot parse log format %s: incomplete string", cfg->log_format_str); return FALSE; break; } return TRUE; } static void rspamd_urls_config_dtor(gpointer _unused) { rspamd_url_deinit(); } static void rspamd_adjust_clocks_resolution(struct rspamd_config *cfg) { #ifdef HAVE_CLOCK_GETTIME struct timespec ts; #endif #ifdef HAVE_CLOCK_GETTIME #ifdef HAVE_CLOCK_PROCESS_CPUTIME_ID clock_getres(CLOCK_PROCESS_CPUTIME_ID, &ts); #elif defined(HAVE_CLOCK_VIRTUAL) clock_getres(CLOCK_VIRTUAL, &ts); #else clock_getres(CLOCK_REALTIME, &ts); #endif cfg->clock_res = log10(1000000. / ts.tv_nsec); if (cfg->clock_res < 0) { cfg->clock_res = 0; } if (cfg->clock_res > 3) { cfg->clock_res = 3; } #else /* For gettimeofday */ cfg->clock_res = 1; #endif } /* * Perform post load actions */ gboolean rspamd_config_post_load(struct rspamd_config *cfg, enum rspamd_post_load_options opts) { auto ret = TRUE; rspamd_adjust_clocks_resolution(cfg); rspamd_logger_configure_modules(cfg->debug_modules); if (cfg->one_shot_mode) { msg_info_config("enabling one shot mode (was %d max shots)", cfg->default_max_shots); cfg->default_max_shots = 1; } #if defined(WITH_HYPERSCAN) && !defined(__aarch64__) && !defined(__powerpc64__) if (!cfg->disable_hyperscan) { if (!(cfg->libs_ctx->crypto_ctx->cpu_config & CPUID_SSSE3)) { msg_warn_config("CPU doesn't have SSSE3 instructions set " "required for hyperscan, disable it"); cfg->disable_hyperscan = TR
/*
 * Copyright (C) 2008, Robin Rosenberg <robin.rosenberg@dewire.com>
 * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org>
 * and other copyright owners as documented in the project's IP log.
 *
 * This program and the accompanying materials are made available
 * under the terms of the Eclipse Distribution License v1.0 which
 * accompanies this distribution, is reproduced below, and is
 * available at http://www.eclipse.org/org/documents/edl-v10.php
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or
 * without modification, are permitted provided that the following
 * conditions are met:
 *
 * - Redistributions of source code must retain the above copyright
 *   notice, this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above
 *   copyright notice, this list of conditions and the following
 *   disclaimer in the documentation and/or other materials provided
 *   with the distribution.
 *
 * - Neither the name of the Eclipse Foundation, Inc. nor the
 *   names of its contributors may be used to endorse or promote
 *   products derived from this software without specific prior
 *   written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
 * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package org.eclipse.jgit.transport;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;

import java.util.List;

import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.lib.Config;
import org.junit.Before;
import org.junit.Test;

public class RemoteConfigTest {
	private Config config;

	@Before
	public void setUp() throws Exception {
		config = new Config();
	}

	private void readConfig(final String dat) throws ConfigInvalidException {
		config = new Config();
		config.fromText(dat);
	}

	private void checkConfig(final String exp) {
		assertEquals(exp, config.toText());
	}

	@Test
	public void testSimple() throws Exception {
		readConfig("[remote \"spearce\"]\n"
				+ "url = http://www.spearce.org/egit.git\n"
				+ "fetch = +refs/heads/*:refs/remotes/spearce/*\n");

		final RemoteConfig rc = new RemoteConfig(config, "spearce");
		final List<URIish> allURIs = rc.getURIs();
		RefSpec spec;

		assertEquals("spearce", rc.getName());
		assertNotNull(allURIs);
		assertNotNull(rc.getFetchRefSpecs());
		assertNotNull(rc.getPushRefSpecs());
		assertNotNull(rc.getTagOpt());
		assertEquals(0, rc.getTimeout());
		assertSame(TagOpt.AUTO_FOLLOW, rc.getTagOpt());

		assertEquals(1, allURIs.size());
		assertEquals("http://www.spearce.org/egit.git", allURIs.get(0)
				.toString());

		assertEquals(1, rc.getFetchRefSpecs().size());
		spec = rc.getFetchRefSpecs().get(0);
		assertTrue(spec.isForceUpdate());
		assertTrue(spec.isWildcard());
		assertEquals("refs/heads/*", spec.getSource());
		assertEquals("refs/remotes/spearce/*", spec.getDestination());

		assertEquals(0, rc.getPushRefSpecs().size());
	}

	@Test
	public void testSimpleNoTags() throws Exception {
		readConfig("[remote \"spearce\"]\n"
				+ "url = http://www.spearce.org/egit.git\n"
				+ "fetch = +refs/heads/*:refs/remotes/spearce/*\n"
				+ "tagopt = --no-tags\n");
		final RemoteConfig rc = new RemoteConfig(config, "spearce");
		assertSame(TagOpt.NO_TAGS, rc.getTagOpt());
	}

	@Test
	public void testSimpleAlwaysTags() throws Exception {
		readConfig("[remote \"spearce\"]\n"
				+ "url = http://www.spearce.org/egit.git\n"
				+ "fetch = +refs/heads/*:refs/remotes/spearce/*\n"
				+ "tagopt = --tags\n");
		final RemoteConfig rc = new RemoteConfig(config, "spearce");
		assertSame(TagOpt.FETCH_TAGS, rc.getTagOpt());
	}

	@Test
	public void testMirror() throws Exception {
		readConfig("[remote \"spearce\"]\n"
				+ "url = http://www.spearce.org/egit.git\n"
				+ "fetch = +refs/heads/*:refs/heads/*\n"
				+ "fetch = refs/tags/*:refs/tags/*\n");

		final RemoteConfig rc = new RemoteConfig(config, "spearce");
		final List<URIish> allURIs = rc.getURIs();
		RefSpec spec;

		assertEquals("spearce", rc.getName());
		assertNotNull(allURIs);
		assertNotNull(rc.getFetchRefSpecs());
		assertNotNull(rc.getPushRefSpecs());

		assertEquals(1, allURIs.size());
		assertEquals("http://www.spearce.org/egit.git", allURIs.get(0)
				.toString());

		assertEquals(2, rc.getFetchRefSpecs().size());

		spec = rc.getFetchRefSpecs().get(0);
		assertTrue(spec.isForceUpdate());
		assertTrue(spec.isWildcard());
		assertEquals("refs/heads/*", spec.getSource());
		assertEquals("refs/heads/*", spec.getDestination());

		spec = rc.getFetchRefSpecs().get(1);
		assertFalse(spec.isForceUpdate());
		assertTrue(spec.isWildcard());
		assertEquals("refs/tags/*", spec.getSource());
		assertEquals("refs/tags/*", spec.getDestination());

		assertEquals(0, rc.getPushRefSpecs().size());
	}

	@Test
	public void testBackup() throws Exception {
		readConfig("[remote \"backup\"]\n"
				+ "url = http://www.spearce.org/egit.git\n"
				+ "url = user@repo.or.cz:/srv/git/egit.git\n"
				+ "push = +refs/heads/*:refs/heads/*\n"
				+ "push = refs/tags/*:refs/tags/*\n");

		final RemoteConfig rc = new RemoteConfig(config, "backup");
		final List<URIish> allURIs = rc.getURIs();
		RefSpec spec;

		assertEquals("backup", rc.getName());
		assertNotNull(allURIs);
		assertNotNull(rc.getFetchRefSpecs());
		assertNotNull(rc.getPushRefSpecs());

		assertEquals(2, allURIs.size());
		assertEquals("http://www.spearce.org/egit.git", allURIs.get(0)
				.toString());
		assertEquals("user@repo.or.cz:/srv/git/egit.git", allURIs.get(1)
				.toString());

		assertEquals(0, rc.getFetchRefSpecs().size());

		assertEquals(2, rc.getPushRefSpecs().size());
		spec = rc.getPushRefSpecs().get(0);
		assertTrue(spec.isForceUpdate());
		assertTrue(spec.isWildcard());
		assertEquals("refs/heads/*", spec.getSource());
		assertEquals("refs/heads/*", spec.getDestination());

		spec = rc.getPushRefSpecs().get(1);
		assertFalse(spec.isForceUpdate());
		assertTrue(spec.isWildcard());
		assertEquals("refs/tags/*", spec.getSource());
		assertEquals("refs/tags/*", spec.getDestination());
	}

	@Test
	public void testUploadPack() throws Exception {
		readConfig("[remote \"example\"]\n"
				+ "url = user@example.com:egit.git\n"
				+ "fetch = +refs/heads/*:refs/remotes/example/*\n"
				+ "uploadpack = /path/to/git/git-upload-pack\n"
				+ "receivepack = /path/to/git/git-receive-pack\n");

		final RemoteConfig rc = new RemoteConfig(config, "example");
		final List<URIish> allURIs = rc.getURIs();
		RefSpec spec;

		assertEquals("example", rc.getName());
		assertNotNull(allURIs);
		assertNotNull(rc.getFetchRefSpecs());
		assertNotNull(rc.getPushRefSpecs());

		assertEquals(1, allURIs.size());
		assertEquals("user@example.com:egit.git", allURIs.get(0).toString());

		assertEquals(1, rc.getFetchRefSpecs().size());
		spec = rc.getFetchRefSpecs().get(0);
		assertTrue(spec.isForceUpdate());
		assertTrue(spec.isWildcard());
		assertEquals("refs/heads/*", spec.getSource());
		assertEquals("refs/remotes/example/*", spec.getDestination());

		assertEquals(0, rc.getPushRefSpecs().size());

		assertEquals("/path/to/git/git-upload-pack", rc.getUploadPack());
		assertEquals("/path/to/git/git-receive-pack", rc.getReceivePack());
	}

	@Test
	public void testUnknown() throws Exception {
		readConfig("");

		final RemoteConfig rc = new RemoteConfig(config, "backup");
		assertEquals(0, rc.getURIs().size());
		assertEquals(0, rc.getFetchRefSpecs().size());
		assertEquals(0, rc.getPushRefSpecs().size());
		assertEquals("git-upload-pack", rc.getUploadPack());
		assertEquals("git-receive-pack", rc.getReceivePack());
	}

	@Test
	public void testAddURI() throws Exception {
		readConfig("");

		final URIish uri = new URIish("/some/dir");
		final RemoteConfig rc = new RemoteConfig(config, "backup");
		assertEquals(0, rc.getURIs().size());

		assertTrue(rc.addURI(uri));
		assertEquals(1, rc.getURIs().size());
		assertSame(uri, rc.getURIs().get(0));

		assertFalse(rc.addURI(new URIish(uri.toString())));
		assertEquals(1, rc.getURIs().size());
	}

	@Test
	public void testRemoveFirstURI() throws Exception {
		readConfig("");

		final URIish a = new URIish("/some/dir");
		final URIish b = new URIish("/another/dir");
		final URIish c = new URIish("/more/dirs");
		final RemoteConfig rc = new RemoteConfig(config, "backup");
		assertTrue(rc.addURI(a));
		assertTrue(rc.addURI(b));
		assertTrue(rc.addURI(c));

		assertEquals(3, rc.getURIs().size());
		assertSame(a, rc.getURIs().get(0));
		assertSame(b, rc.getURIs().get(1));
		assertSame(c, rc.getURIs().get(2));

		assertTrue(rc.removeURI(a));
		assertEquals(2, rc.getURIs().size());
		assertSame(b, rc.getURIs().get(0));
		assertSame(c, rc.getURIs().get(1));
	}

	@Test
	public void testRemoveMiddleURI() throws Exception {
		readConfig("");

		final URIish a = new URIish("/some/dir");
		final URIish b = new URIish("/another/dir");
		final URIish c = new URIish("/more/dirs");
		final RemoteConfig rc = new RemoteConfig(config, "backup");
		assertTrue(rc.addURI(a));
		assertTrue(rc.addURI(b));
		assertTrue(rc.addURI(c));

		assertEquals(3, rc.getURIs().size());
		assertSame(a, rc.getURIs().get(0));
		assertSame(b, rc.getURIs().get(1));
		assertSame(c, rc.getURIs().get(2));

		assertTrue(rc.removeURI(b));
		assertEquals(2, rc.getURIs().size());
		assertSame(a, rc.getURIs().get(0));
		assertSame(c, rc.getURIs().get(1));
	}

	@Test
	public void testRemoveLastURI() throws Exception {
		readConfig("");

		final URIish a = new URIish("/some/dir");
		final URIish b = new URIish("/another/dir");
		final URIish c = new URIish("/more/dirs");
		final RemoteConfig rc = new RemoteConfig(config, "backup");
		assertTrue(rc.addURI(a));
		assertTrue(rc.addURI(b));
		assertTrue(rc.addURI(c));

		assertEquals(3, rc.getURIs().size());
		assertSame(a, rc.getURIs().get(0));
		assertSame(b, rc.getURIs().get(1));
		assertSame(c, rc.getURIs().get(2));

		assertTrue(rc.removeURI(c));
		assertEquals(2, rc.getURIs().size());
		assertSame(a, rc.getURIs().get(0));
		assertSame(b, rc.getURIs().get(1));
	}

	@Test
	public void testRemoveOnlyURI() throws Exception {
		readConfig("");

		final URIish a = new URIish("/some/dir");
		final RemoteConfig rc = new RemoteConfig(config, "backup");
		assertTrue(rc.addURI(a));

		assertEquals(1, rc.getURIs().size());
		assertSame(a, rc.getURIs().get(0));

		assertTrue(rc.removeURI(a));
		assertEquals(0, rc.getURIs().size());
	}

	@Test
	public void testCreateOrigin() throws Exception {
		final RemoteConfig rc = new RemoteConfig(config, "origin");
		rc.addURI(new URIish("/some/dir"));
		rc.addFetchRefSpec(new RefSpec("+refs/heads/*:refs/remotes/"
				+ rc.getName() + "/*"));
		rc.update(config);
		checkConfig("[remote \"origin\"]\n" + "\turl = /some/dir\n"
				+ "\tfetch = +refs/heads/*:refs/remotes/origin/*\n");
	}

	@Test
	public void testSaveAddURI() throws Exception {
		readConfig("[remote \"spearce\"]\n"
				+ "url = http://www.spearce.org/egit.git\n"
				+ "fetch = +refs/heads/*:refs/remotes/spearce/*\n");

		final RemoteConfig rc = new RemoteConfig(config, "spearce");
		rc.addURI(new URIish("/some/dir"));
		assertEquals(2, rc.getURIs().size());
		rc.update(config);
		checkConfig("[remote \"spearce\"]\n"
				+ "\turl = http://www.spearce.org/egit.git\n"
				+ "\turl = /some/dir\n"
				+ "\tfetch = +refs/heads/*:refs/remotes/spearce/*\n");
	}

	@Test
	public void testSaveRemoveLastURI() throws Exception {
		readConfig("[remote \"spearce\"]\n"
				+ "url = http://www.spearce.org/egit.git\n"
				+ "url = /some/dir\n"
				+ "fetch = +refs/heads/*:refs/remotes/spearce/*\n");

		final RemoteConfig rc = new RemoteConfig(config, "spearce");
		assertEquals(2, rc.getURIs().size());
		rc.removeURI(new URIish("/some/dir"));
		assertEquals(1, rc.getURIs().size());
		rc.update(config);
		checkConfig("[remote \"spearce\"]\n"
				+ "\turl = http://www.spearce.org/egit.git\n"
				+ "\tfetch = +refs/heads/*:refs/remotes/spearce/*\n");
	}

	@Test
	public void testSaveRemoveFirstURI() throws Exception {
		readConfig("[remote \"spearce\"]\n"
				+ "url = http://www.spearce.org/egit.git\n"
				+ "url = /some/dir\n"
				+ "fetch = +refs/heads/*:refs/remotes/spearce/*\n");

		final RemoteConfig rc = new RemoteConfig(config, "spearce");
		assertEquals(2, rc.getURIs().size());
		rc.removeURI(new URIish("http://www.spearce.org/egit.git"));
		assertEquals(1, rc.getURIs().size());
		rc.update(config);
		checkConfig("[remote \"spearce\"]\n" + "\turl = /some/dir\n"
				+ "\tfetch = +refs/heads/*:refs/remotes/spearce/*\n");
	}

	@Test
	public void testSaveNoTags() throws Exception {
		final RemoteConfig rc = new RemoteConfig(config, "origin");
		rc.addURI(new URIish("/some/dir"));
		rc.addFetchRefSpec(new RefSpec("+refs/heads/*:refs/remotes/"
				+ rc.getName() + "/*"));
		rc.setTagOpt(TagOpt.NO_TAGS);
		rc.update(config);
		checkConfig("[remote \"origin\"]\n" + "\turl = /some/dir\n"
				+ "\tfetch = +refs/heads/*:refs/remotes/origin/*\n"
				+ "\ttagopt = --no-tags\n");
	}

	@Test
	public void testSaveAllTags() throws Exception {
		final RemoteConfig rc = new RemoteConfig(config, "origin");
		rc.addURI(new URIish("/some/dir"));
		rc.addFetchRefSpec(new RefSpec("+refs/heads/*:refs/remotes/"
				+ rc.getName() + "/*"));
		rc.setTagOpt(TagOpt.FETCH_TAGS);
		rc.update(config);
		checkConfig("[remote \"origin\"]\n" + "\turl = /some/dir\n"
				+ "\tfetch = +refs/heads/*:refs/remotes/origin/*\n"
				+ "\ttagopt = --tags\n");
	}

	@Test
	public void testSimpleTimeout() throws Exception {
		readConfig("[remote \"spearce\"]\n"
				+ "url = http://www.spearce.org/egit.git\n"
				+ "fetch = +refs/heads/*:refs/remotes/spearce/*\n"
				+ "timeout = 12\n");
		final RemoteConfig rc = new RemoteConfig(config, "spearce");
		assertEquals(12, rc.getTimeout());
	}

	@Test
	public void testSaveTimeout() throws Exception {
		final RemoteConfig rc = new RemoteConfig(config, "origin");
		rc.addURI(new URIish("/some/dir"));
		rc.addFetchRefSpec(new RefSpec("+refs/heads/*:refs/remotes/"
				+ rc.getName() + "/*"));
		rc.setTimeout(60);
		rc.update(config);
		checkConfig("[remote \"origin\"]\n" + "\turl = /some/dir\n"
				+ "\tfetch = +refs/heads/*:refs/remotes/origin/*\n"
				+ "\ttimeout = 60\n");
	}
}