diff options
author | heraklit256 <37872459+heraklit256@users.noreply.github.com> | 2019-03-05 16:05:03 +0000 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-03-05 16:05:03 +0000 |
commit | 46de8e9f6e51dc44623f253f6a5fe9ecad3dc554 (patch) | |
tree | ee759abdc966c3415a87ea4ad91c8f10f2c132f7 | |
parent | f40cc5500eecf1e0b26d6b4be3b7f3bbb78dede0 (diff) | |
parent | ff5125a975f3a3c143c84aa9cf2f6878f5930272 (diff) | |
download | rspamd-46de8e9f6e51dc44623f253f6a5fe9ecad3dc554.tar.gz rspamd-46de8e9f6e51dc44623f253f6a5fe9ecad3dc554.zip |
Merge pull request #11 from rspamd/master
merge upstream into local master
123 files changed, 5467 insertions, 3435 deletions
diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 19d88fd31..738507d9d 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -2,9 +2,7 @@ * [x] Crash/Hang/Data loss * [ ] WebUI/Usability -* [ ] Serious bug -* [ ] Ordinary bug -* [ ] Feature +* [ ] Unintended behaviour (bug) * [ ] Enhancement ### Reproducibility (Please choose *one* option): @@ -18,7 +16,7 @@ ### Rspamd version: -### Operation system, CPU: +### Operation system (the exact version), CPU (including architecture and processor family): ### Description (Please provide a descriptive summary of the issue): diff --git a/CMakeLists.txt b/CMakeLists.txt index 1e20c6f8d..9935c4460 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1225,12 +1225,6 @@ SET(WITH_HIREDIS 1) INCLUDE_DIRECTORIES(BEFORE "${CMAKE_SOURCE_DIR}/contrib/hiredis") LIST(APPEND RSPAMD_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}") -LIST(APPEND RSPAMD_REQUIRED_LIBRARIES ucl) -LIST(APPEND RSPAMD_REQUIRED_LIBRARIES rdns) -LIST(APPEND RSPAMD_REQUIRED_LIBRARIES ottery) -LIST(APPEND RSPAMD_REQUIRED_LIBRARIES event) -LIST(APPEND RSPAMD_REQUIRED_LIBRARIES xxhash) -LIST(APPEND RSPAMD_REQUIRED_LIBRARIES rspamd-actrie) IF(HAVE_FETCH_H) LIST(APPEND RSPAMD_REQUIRED_LIBRARIES fetch) ENDIF(HAVE_FETCH_H) @@ -1238,8 +1232,12 @@ IF(WITH_DB) LIST(APPEND RSPAMD_REQUIRED_LIBRARIES db) ENDIF(WITH_DB) - LIST(APPEND RSPAMD_REQUIRED_LIBRARIES "${LUA_LIBRARY}") +LIST(APPEND RSPAMD_REQUIRED_LIBRARIES ucl) +LIST(APPEND RSPAMD_REQUIRED_LIBRARIES rdns) +LIST(APPEND RSPAMD_REQUIRED_LIBRARIES ottery) +LIST(APPEND RSPAMD_REQUIRED_LIBRARIES event) +LIST(APPEND RSPAMD_REQUIRED_LIBRARIES xxhash) IF(GLIB_COMPAT) LIST(APPEND RSPAMD_REQUIRED_LIBRARIES glibadditions) @@ -1267,9 +1265,8 @@ IF (ENABLE_SNOWBALL MATCHES "ON") LIST(APPEND RSPAMD_REQUIRED_LIBRARIES stemmer) ENDIF() LIST(APPEND RSPAMD_REQUIRED_LIBRARIES rspamd-hiredis) -IF(ENABLE_HYPERSCAN MATCHES "OFF") - LIST(APPEND RSPAMD_REQUIRED_LIBRARIES rspamd-actrie) -ENDIF() + +LIST(APPEND RSPAMD_REQUIRED_LIBRARIES rspamd-actrie) LIST(APPEND RSPAMD_REQUIRED_LIBRARIES rspamd-t1ha) IF(ENABLE_CLANG_PLUGIN MATCHES "ON") diff --git a/conf/composites.conf b/conf/composites.conf index 976225db1..3436b44ae 100644 --- a/conf/composites.conf +++ b/conf/composites.conf @@ -87,7 +87,7 @@ composites { policy = "leave"; } RCVD_UNAUTH_PBL { - expression = "RECEIVED_PBL & -RCVD_VIA_SMTP_AUTH"; + expression = "RECEIVED_PBL & !RCVD_VIA_SMTP_AUTH"; description = "Relayed through ZEN PBL IP without sufficient authentication (possible indicating an open relay)"; score = 2.0; policy = "leave"; diff --git a/contrib/http-parser/http_parser.c b/contrib/http-parser/http_parser.c index 0e75d964b..c14ecc034 100644 --- a/contrib/http-parser/http_parser.c +++ b/contrib/http-parser/http_parser.c @@ -122,6 +122,7 @@ do { \ #define KEEP_ALIVE "keep-alive" #define CLOSE "close" +enum rspamd_http_message_type { HTTP_REQUEST, HTTP_RESPONSE, HTTP_BOTH }; static const char *method_strings[] = { @@ -1981,7 +1982,7 @@ http_method_str (enum http_method m) void -http_parser_init (http_parser *parser, enum http_parser_type t) +http_parser_init (http_parser *parser, int t) { void *data = parser->data; /* preserve application data */ memset(parser, 0, sizeof(*parser)); diff --git a/contrib/http-parser/http_parser.h b/contrib/http-parser/http_parser.h index e2a0b4985..7b4ac497c 100644 --- a/contrib/http-parser/http_parser.h +++ b/contrib/http-parser/http_parser.h @@ -124,8 +124,6 @@ enum http_method }; -enum http_parser_type { HTTP_REQUEST, HTTP_RESPONSE, HTTP_BOTH }; - /* Flag values for http_parser.flags field */ enum flags @@ -280,7 +278,7 @@ struct http_parser_url { */ unsigned long http_parser_version(void); -void http_parser_init(http_parser *parser, enum http_parser_type type); +void http_parser_init(http_parser *parser, int type); size_t http_parser_execute(http_parser *parser, diff --git a/contrib/librdns/util.c b/contrib/librdns/util.c index b793b9077..9f1bc6018 100644 --- a/contrib/librdns/util.c +++ b/contrib/librdns/util.c @@ -537,8 +537,9 @@ rdns_resolver_conf_process_line (struct rdns_resolver *resolver, const char *line, rdns_resolv_conf_cb cb, void *ud) { const char *p, *c, *end; - bool has_obrace = false; + bool has_obrace = false, ret; unsigned int port = dns_port; + char *cpy_buf; end = line + strlen (line); @@ -546,7 +547,7 @@ rdns_resolver_conf_process_line (struct rdns_resolver *resolver, strncmp (line, "nameserver", sizeof ("nameserver") - 1) == 0) { p = line + sizeof ("nameserver") - 1; /* Skip spaces */ - while (*p == ' ' || *p == '\t') { + while (isspace (*p)) { p ++; } @@ -578,14 +579,23 @@ rdns_resolver_conf_process_line (struct rdns_resolver *resolver, } } + cpy_buf = malloc (p - c + 1); + assert (cpy_buf != NULL); + memcpy (cpy_buf, c, p - c); + cpy_buf[p - c] = '\0'; + if (cb == NULL) { - return rdns_resolver_add_server (resolver, c, port, 0, + ret = rdns_resolver_add_server (resolver, cpy_buf, port, 0, default_io_cnt) != NULL; } else { - return cb (resolver, c, port, 0, + ret = cb (resolver, cpy_buf, port, 0, default_io_cnt, ud); } + + free (cpy_buf); + + return ret; } else { return false; diff --git a/lualib/lua_cfg_transform.lua b/lualib/lua_cfg_transform.lua index 8a61dcd29..ae05eed59 100644 --- a/lualib/lua_cfg_transform.lua +++ b/lualib/lua_cfg_transform.lua @@ -242,6 +242,12 @@ return function(cfg) ret = true end + if cfg.symbols then + for k, v in metric_pairs(cfg.symbols) do + symbol_transform(cfg, k, v) + end + end + check_statistics_sanity() if not cfg.actions then diff --git a/lualib/lua_clickhouse.lua b/lualib/lua_clickhouse.lua index dbb74e283..96ea59f02 100644 --- a/lualib/lua_clickhouse.lua +++ b/lualib/lua_clickhouse.lua @@ -227,7 +227,7 @@ exports.select = function (upstream, settings, params, query, ok_cb, fail_cb) connect_prefix = 'https://' end local ip_addr = upstream:get_addr():to_string(true) - local database = params.database or 'default' + local database = settings.database or 'default' http_params.url = string.format('%s%s/?database=%s&default_format=JSONEachRow', connect_prefix, ip_addr, escape_spaces(database)) end @@ -278,7 +278,7 @@ exports.select_sync = function (upstream, settings, params, query, ok_cb, fail_c connect_prefix = 'https://' end local ip_addr = upstream:get_addr():to_string(true) - local database = params.database or 'default' + local database = settings.database or 'default' http_params.url = string.format('%s%s/?database=%s&default_format=JSONEachRow', connect_prefix, ip_addr, escape_spaces(database)) end @@ -342,7 +342,7 @@ exports.insert = function (upstream, settings, params, query, rows, connect_prefix = 'https://' end local ip_addr = upstream:get_addr():to_string(true) - local database = params.database or 'default' + local database = settings.database or 'default' http_params.url = string.format('%s%s/?database=%s&query=%s%%20FORMAT%%20TabSeparated', connect_prefix, ip_addr, @@ -394,7 +394,7 @@ exports.generic = function (upstream, settings, params, query, connect_prefix = 'https://' end local ip_addr = upstream:get_addr():to_string(true) - local database = params.database or 'default' + local database = settings.database or 'default' http_params.url = string.format('%s%s/?database=%s&default_format=JSONEachRow', connect_prefix, ip_addr, escape_spaces(database)) end @@ -439,7 +439,7 @@ exports.generic_sync = function (upstream, settings, params, query) connect_prefix = 'https://' end local ip_addr = upstream:get_addr():to_string(true) - local database = params.database or 'default' + local database = settings.database or 'default' http_params.url = string.format('%s%s/?database=%s&default_format=JSONEachRow', connect_prefix, ip_addr, escape_spaces(database)) end diff --git a/lualib/lua_dkim_tools.lua b/lualib/lua_dkim_tools.lua index 4302d24ad..fc891ba71 100644 --- a/lualib/lua_dkim_tools.lua +++ b/lualib/lua_dkim_tools.lua @@ -120,7 +120,17 @@ local function prepare_dkim_signing(N, task, settings) local is_local, is_sign_networks if settings.use_http_headers then - return parse_dkim_http_headers(N, task, settings) + local res,tbl = parse_dkim_http_headers(N, task, settings) + + if not res then + if not settings.allow_headers_fallback then + return res,{} + else + lua_util.debugm(N, task, 'failed to read http headers, fallback to normal schema') + end + else + return res,tbl + end end local auser = task:get_user() @@ -174,6 +184,8 @@ local function prepare_dkim_signing(N, task, settings) return udom elseif settings[dtype] == 'recipient' then return tdom + else + return settings[dtype]:lower() end end @@ -202,6 +214,31 @@ local function prepare_dkim_signing(N, task, settings) dkim_domain = get_dkim_domain('use_domain_sign_inbound') lua_util.debugm(N, task, 'inbound: use domain(%s) for signature: %s', settings.use_domain_sign_inbound, dkim_domain) + elseif settings.use_domain_custom then + if type(settings.use_domain_custom) == 'string' then + -- Load custom function + local loadstring = loadstring or load + local ret, res_or_err = pcall(loadstring(settings.use_domain_custom)) + if ret then + if type(res_or_err) == 'function' then + settings.use_domain_custom = res_or_err + dkim_domain = settings.use_domain_custom(task) + lua_util.debugm(N, task, 'use custom domain for signing: %s', + dkim_domain) + else + logger.errx(task, 'cannot load dkim domain custom script: invalid type: %s, expected function', + type(res_or_err)) + settings.use_domain_custom = nil + end + else + logger.errx(task, 'cannot load dkim domain custom script: %s', res_or_err) + settings.use_domain_custom = nil + end + else + dkim_domain = settings.use_domain_custom(task) + lua_util.debugm(N, task, 'use custom domain for signing: %s', + dkim_domain) + end else dkim_domain = get_dkim_domain('use_domain') lua_util.debugm(N, task, 'use domain(%s) for signature: %s', diff --git a/lualib/lua_ffi/init.lua b/lualib/lua_ffi/init.lua index d2cbd95aa..b0254bdd8 100644 --- a/lualib/lua_ffi/init.lua +++ b/lualib/lua_ffi/init.lua @@ -46,6 +46,7 @@ else end end +pcall(ffi.load, "rspamd-server", true) exports.common = require "lua_ffi/common" exports.dkim = require "lua_ffi/dkim" diff --git a/lualib/lua_selectors.lua b/lualib/lua_selectors.lua index 0a02edca9..b678ef179 100644 --- a/lualib/lua_selectors.lua +++ b/lualib/lua_selectors.lua @@ -199,6 +199,27 @@ the second optional argument is optional hash type (`blake2`, `sha256`, `sha1`, end, ['description'] = 'Get all attachments files', }, + -- Get languages for text parts + ['languages'] = { + ['get_value'] = function(task) + local text_parts = task:get_text_parts() or E + local languages = {} + + for _,p in ipairs(text_parts) do + local lang = p:get_language() + if lang then + table.insert(languages, lang) + end + end + + if #languages > 0 then + return languages,'string_list' + end + + return nil + end, + ['description'] = 'Get languages for text parts', + }, -- Get helo value ['helo'] = { ['get_value'] = function(task) diff --git a/rules/html.lua b/rules/html.lua index da4ef1d13..fb32e9179 100644 --- a/rules/html.lua +++ b/rules/html.lua @@ -23,6 +23,20 @@ reconf['MIME_HTML_ONLY'] = { group = 'headers' } +local function has_anchor_parent(tag) + local parent = tag + repeat + parent = parent:get_parent() + if parent then + if parent:get_type() == 'a' then + return true + end + end + until not parent + + return false +end + local function check_html_image(task, min, max) local tp = task:get_text_parts() @@ -38,13 +52,10 @@ local function check_html_image(task, min, max) for _,i in ipairs(images) do local tag = i['tag'] if tag then - local parent = tag:get_parent() - if parent then - if parent:get_type() == 'a' then - -- do not trigger on small and unknown size images - if i['height'] + i['width'] >= 210 or not i['embedded'] then - return true - end + if has_anchor_parent(tag) then + -- do not trigger on small and unknown size images + if i['height'] + i['width'] >= 210 or not i['embedded'] then + return true end end end @@ -81,6 +92,7 @@ rspamd_config.HTML_SHORT_LINK_IMG_3 = { group = 'html', description = 'Short html part (1.5K..2K) with a link to an image' } + rspamd_config.R_EMPTY_IMAGE = { callback = function(task) local tp = task:get_text_parts() -- get text parts in a message @@ -98,11 +110,8 @@ rspamd_config.R_EMPTY_IMAGE = { if i['height'] + i['width'] >= 400 then -- if we have a large image local tag = i['tag'] if tag then - local parent = tag:get_parent() - if parent then - if parent:get_type() ~= 'a' then - return true - end + if not has_anchor_parent(tag) then + return true end end end @@ -136,14 +145,10 @@ rspamd_config.R_SUSPICIOUS_IMAGES = { local tag = i['tag'] if tag then - local parent = tag:get_parent() - if parent then - if parent:get_type() == 'a' then - -- do not trigger on small and large images - if dim > 100 and dim < 3000 then - -- We assume that a single picture 100x200 contains approx 3 words of text - pic_words = pic_words + dim / 100 - end + if has_anchor_parent(tag) then + if dim > 100 and dim < 3000 then + -- We assume that a single picture 100x200 contains approx 3 words of text + pic_words = pic_words + dim / 100 end end end diff --git a/rules/regexp/misc.lua b/rules/regexp/misc.lua index aaf8425bd..642a02e21 100644 --- a/rules/regexp/misc.lua +++ b/rules/regexp/misc.lua @@ -64,7 +64,7 @@ reconf['HAS_ONION_URI'] = { local my_victim = [[/(?:victim|prey)/{words}]] local your_webcam = [[/webcam/{words}]] local your_onan = [[/(?:mast[ur]{2}bati(?:on|ng)|onanism|solitary)/{words}]] -local password_in_words = [[/^pass(?:(?:word)|(?:phrase))/i{words}]] +local password_in_words = [[/^pass(?:(?:word)|(?:phrase))$/i{words}]] local btc_wallet_address = [[/^[13][0-9a-zA-Z]{25,34}$/{words}]] local wallet_word = [[/^wallet$/{words}]] local broken_unicode = [[has_flag(bad_unicode)]] @@ -93,4 +93,4 @@ reconf['LEAKED_PASSWORD_SCAM'] = { }, score = 7.0, group = 'scams' -}
\ No newline at end of file +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fc2560ce6..256feb522 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -206,17 +206,15 @@ IF (ENABLE_HYPERSCAN MATCHES "ON") ENDIF() TARGET_LINK_LIBRARIES(rspamd-server rspamd-linenoise) - -IF(USE_CXX_LINKER) - SET_TARGET_PROPERTIES(rspamd PROPERTIES LINKER_LANGUAGE CXX) - SET_TARGET_PROPERTIES(rspamd-server PROPERTIES LINKER_LANGUAGE CXX) -ENDIF() - TARGET_LINK_LIBRARIES(rspamd-server ${RSPAMD_REQUIRED_LIBRARIES}) ADD_EXECUTABLE(rspamd ${RSPAMDSRC} ${CMAKE_CURRENT_BINARY_DIR}/workers.c) SET_TARGET_PROPERTIES(rspamd PROPERTIES LINKER_LANGUAGE C) SET_TARGET_PROPERTIES(rspamd PROPERTIES COMPILE_FLAGS "-DRSPAMD_MAIN") +IF(USE_CXX_LINKER) + SET_TARGET_PROPERTIES(rspamd PROPERTIES LINKER_LANGUAGE CXX) + SET_TARGET_PROPERTIES(rspamd-server PROPERTIES LINKER_LANGUAGE CXX) +ENDIF() IF(NOT DEBIAN_BUILD) SET_TARGET_PROPERTIES(rspamd PROPERTIES VERSION ${RSPAMD_VERSION}) ENDIF(NOT DEBIAN_BUILD) diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 0a92019d8..4e1225ae3 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -7,7 +7,6 @@ SET(RSPAMCSRC rspamc.c) ADD_EXECUTABLE(rspamc ${RSPAMCSRC} ${LIBRSPAMDCLIENTSRC}) SET_TARGET_PROPERTIES(rspamc PROPERTIES COMPILE_FLAGS "-I${CMAKE_SOURCE_DIR}/lib") TARGET_LINK_LIBRARIES(rspamc rspamd-server) -TARGET_LINK_LIBRARIES(rspamc ${RSPAMD_REQUIRED_LIBRARIES}) IF(USE_CXX_LINKER) SET_TARGET_PROPERTIES(rspamc PROPERTIES LINKER_LANGUAGE CXX) ENDIF() diff --git a/src/client/rspamc.c b/src/client/rspamc.c index 3433ef7d6..0568692b4 100644 --- a/src/client/rspamc.c +++ b/src/client/rspamc.c @@ -15,7 +15,7 @@ */ #include "config.h" #include "libutil/util.h" -#include "libutil/http.h" +#include "libutil/http_connection.h" #include "libutil/http_private.h" #include "rspamdclient.h" #include "utlist.h" @@ -67,6 +67,7 @@ static gchar *key = NULL; static gchar *user_agent = "rspamc"; static GList *children; static GPatternSpec **exclude_compiled = NULL; +static struct rspamd_http_context *http_ctx; static gint retcode = EXIT_SUCCESS; @@ -553,10 +554,6 @@ add_options (GQueue *opts) GString *numbuf; gchar **hdr, **rcpt; - if (user_agent) { - ADD_CLIENT_HEADER (opts, "User-Agent", user_agent); - } - if (ip != NULL) { rspamd_inet_addr_t *addr = NULL; @@ -1668,7 +1665,7 @@ rspamc_process_input (struct event_base *ev_base, struct rspamc_command *cmd, } - conn = rspamd_client_init (ev_base, hostbuf, port, timeout, key); + conn = rspamd_client_init (http_ctx, ev_base, hostbuf, port, timeout, key); if (conn != NULL) { cbdata = g_malloc0 (sizeof (struct rspamc_callback_data)); @@ -1894,6 +1891,15 @@ main (gint argc, gchar **argv, gchar **env) rspamd_init_libs (); ev_base = event_base_new (); + struct rspamd_http_context_cfg http_config; + + memset (&http_config, 0, sizeof (http_config)); + http_config.kp_cache_size_client = 32; + http_config.kp_cache_size_server = 0; + http_config.user_agent = user_agent; + http_ctx = rspamd_http_context_create_config (&http_config, + ev_base); + /* Ignore sigpipe */ sigemptyset (&sigpipe_act.sa_mask); sigaddset (&sigpipe_act.sa_mask, SIGPIPE); diff --git a/src/client/rspamdclient.c b/src/client/rspamdclient.c index a4a1fb95e..a2ff85458 100644 --- a/src/client/rspamdclient.c +++ b/src/client/rspamdclient.c @@ -15,7 +15,7 @@ */ #include "rspamdclient.h" #include "libutil/util.h" -#include "libutil/http.h" +#include "libutil/http_connection.h" #include "libutil/http_private.h" #include "unix-std.h" #include "contrib/zstd/zstd.h" @@ -118,9 +118,7 @@ rspamd_client_finish_handler (struct rspamd_http_connection *conn, rspamd_http_connection_reset (c->http_conn); rspamd_http_connection_read_message (c->http_conn, c->req, - c->fd, - &c->timeout, - c->ev_base); + &c->timeout); return 0; } else { @@ -241,8 +239,9 @@ rspamd_client_finish_handler (struct rspamd_http_connection *conn, } struct rspamd_client_connection * -rspamd_client_init (struct event_base *ev_base, const gchar *name, - guint16 port, gdouble timeout, const gchar *key) +rspamd_client_init (struct rspamd_http_context *http_ctx, + struct event_base *ev_base, const gchar *name, + guint16 port, gdouble timeout, const gchar *key) { struct rspamd_client_connection *conn; gint fd; @@ -256,14 +255,13 @@ rspamd_client_init (struct event_base *ev_base, const gchar *name, conn->ev_base = ev_base; conn->fd = fd; conn->req_sent = FALSE; - conn->keys_cache = rspamd_keypair_cache_new (32); - conn->http_conn = rspamd_http_connection_new (rspamd_client_body_handler, + conn->http_conn = rspamd_http_connection_new (http_ctx, + fd, + rspamd_client_body_handler, rspamd_client_error_handler, rspamd_client_finish_handler, 0, - RSPAMD_HTTP_CLIENT, - conn->keys_cache, - NULL); + RSPAMD_HTTP_CLIENT); conn->server_name = g_string_new (name); if (port != 0) { @@ -444,12 +442,12 @@ rspamd_client_command (struct rspamd_client_connection *conn, if (compressed) { rspamd_http_connection_write_message (conn->http_conn, req->msg, NULL, - "application/x-compressed", req, conn->fd, - &conn->timeout, conn->ev_base); + "application/x-compressed", req, + &conn->timeout); } else { rspamd_http_connection_write_message (conn->http_conn, req->msg, NULL, - "text/plain", req, conn->fd, &conn->timeout, conn->ev_base); + "text/plain", req, &conn->timeout); } return TRUE; diff --git a/src/client/rspamdclient.h b/src/client/rspamdclient.h index 129ee39a1..c2a3c1886 100644 --- a/src/client/rspamdclient.h +++ b/src/client/rspamdclient.h @@ -47,6 +47,7 @@ typedef void (*rspamd_client_callback) ( gdouble send_time, GError *err); +struct rspamd_http_context; /** * Start rspamd worker or controller command * @param ev_base event base @@ -56,11 +57,12 @@ typedef void (*rspamd_client_callback) ( * @return */ struct rspamd_client_connection * rspamd_client_init ( - struct event_base *ev_base, - const gchar *name, - guint16 port, - gdouble timeout, - const gchar *key); + struct rspamd_http_context *http_ctx, + struct event_base *ev_base, + const gchar *name, + guint16 port, + gdouble timeout, + const gchar *key); /** * diff --git a/src/controller.c b/src/controller.c index 6a839b4df..ac1acae81 100644 --- a/src/controller.c +++ b/src/controller.c @@ -21,6 +21,7 @@ #include "libutil/map_helpers.h" #include "libutil/map_private.h" #include "libutil/http_private.h" +#include "libutil/http_router.h" #include "libstat/stat_api.h" #include "rspamd.h" #include "libserver/worker_util.h" @@ -149,6 +150,7 @@ struct rspamd_controller_worker_ctx { rspamd_ftok_t cached_password; rspamd_ftok_t cached_enable_password; /* HTTP server */ + struct rspamd_http_context *http_ctx; struct rspamd_http_connection_router *http; /* Server's start time */ time_t start_time; @@ -1076,8 +1078,8 @@ rspamd_controller_handle_get_map (struct rspamd_http_connection_entry *conn_ent, rspamd_http_connection_reset (conn_ent->conn); rspamd_http_router_insert_headers (conn_ent->rt, reply); rspamd_http_connection_write_message (conn_ent->conn, reply, NULL, - "text/plain", conn_ent, conn_ent->conn->fd, - conn_ent->rt->ptv, conn_ent->rt->ev_base); + "text/plain", conn_ent, + conn_ent->rt->ptv); conn_ent->is_reply = TRUE; return 0; @@ -1940,8 +1942,7 @@ rspamd_controller_scan_reply (struct rspamd_task *task) rspamd_http_connection_reset (conn_ent->conn); rspamd_http_router_insert_headers (conn_ent->rt, msg); rspamd_http_connection_write_message (conn_ent->conn, msg, NULL, - "application/json", conn_ent, conn_ent->conn->fd, conn_ent->rt->ptv, - conn_ent->rt->ev_base); + "application/json", conn_ent, conn_ent->rt->ptv); conn_ent->is_reply = TRUE; } @@ -2909,9 +2910,7 @@ rspamd_controller_handle_ping (struct rspamd_http_connection_entry *conn_ent, NULL, "text/plain", conn_ent, - conn_ent->conn->fd, - conn_ent->rt->ptv, - conn_ent->rt->ev_base); + conn_ent->rt->ptv); conn_ent->is_reply = TRUE; return 0; @@ -2945,9 +2944,7 @@ rspamd_controller_handle_unknown (struct rspamd_http_connection_entry *conn_ent, NULL, "text/plain", conn_ent, - conn_ent->conn->fd, - conn_ent->rt->ptv, - conn_ent->rt->ev_base); + conn_ent->rt->ptv); conn_ent->is_reply = TRUE; } else { @@ -2963,9 +2960,7 @@ rspamd_controller_handle_unknown (struct rspamd_http_connection_entry *conn_ent, NULL, "text/plain", conn_ent, - conn_ent->conn->fd, - conn_ent->rt->ptv, - conn_ent->rt->ev_base); + conn_ent->rt->ptv); conn_ent->is_reply = TRUE; } @@ -3706,7 +3701,6 @@ start_controller_worker (struct rspamd_worker *worker) GHashTableIter iter; gpointer key, value; guint i; - struct rspamd_keypair_cache *cache; struct timeval stv; const guint save_stats_interval = 60 * 1000; /* 1 minute */ gpointer m; @@ -3782,10 +3776,10 @@ start_controller_worker (struct rspamd_worker *worker) "password"); /* Accept event */ - cache = rspamd_keypair_cache_new (256); + ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base); ctx->http = rspamd_http_router_new (rspamd_controller_error_handler, - rspamd_controller_finish_handler, &ctx->io_tv, ctx->ev_base, - ctx->static_files_dir, cache); + rspamd_controller_finish_handler, &ctx->io_tv, + ctx->static_files_dir, ctx->http_ctx); /* Add callbacks for different methods */ rspamd_http_router_add_path (ctx->http, @@ -3948,7 +3942,10 @@ start_controller_worker (struct rspamd_worker *worker) g_hash_table_unref (ctx->plugins); g_hash_table_unref (ctx->custom_commands); + + struct rspamd_http_context *http_ctx = ctx->http_ctx; REF_RELEASE (ctx->cfg); + rspamd_http_context_free (http_ctx); rspamd_log_close (worker->srv->logger, TRUE); exit (EXIT_SUCCESS); diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c index 36b41113d..7fdce82ae 100644 --- a/src/fuzzy_storage.c +++ b/src/fuzzy_storage.c @@ -34,10 +34,10 @@ #include "libcryptobox/keypairs_cache.h" #include "libcryptobox/keypair.h" #include "libserver/rspamd_control.h" -#include "libutil/map_private.h" #include "libutil/hash.h" +#include "libutil/map_private.h" #include "libutil/http_private.h" -#include "libutil/hash.h" +#include "libutil/http_router.h" #include "unix-std.h" #include <math.h> @@ -173,6 +173,7 @@ struct rspamd_fuzzy_storage_ctx { struct rspamd_cryptobox_keypair *collection_keypair; struct rspamd_cryptobox_pubkey *collection_sign_key; gchar *collection_id_file; + struct rspamd_http_context *http_ctx; struct rspamd_keypair_cache *keypair_cache; rspamd_lru_hash_t *errors_ips; rspamd_lru_hash_t *ratelimit_buckets; @@ -530,8 +531,7 @@ fuzzy_mirror_updates_version_cb (guint64 rev64, void *ud) double_to_tv (ctx->sync_timeout, &tv); rspamd_http_connection_write_message (conn->http_conn, msg, NULL, NULL, conn, - conn->sock, - &tv, ctx->ev_base); + &tv); msg_info ("send update request to %s", m->name); g_array_free (cbdata->updates_pending, TRUE); @@ -568,7 +568,7 @@ fuzzy_mirror_error_handler (struct rspamd_http_connection *conn, GError *err) msg_info ("abnormally closing connection from backend: %s:%s, " "error: %e", bk_conn->mirror->name, - rspamd_inet_address_to_string (rspamd_upstream_addr (bk_conn->up)), + rspamd_inet_address_to_string (rspamd_upstream_addr_cur (bk_conn->up)), err); fuzzy_mirror_close_connection (bk_conn); @@ -604,7 +604,7 @@ rspamd_fuzzy_send_update_mirror (struct rspamd_fuzzy_storage_ctx *ctx, } conn->sock = rspamd_inet_address_connect ( - rspamd_upstream_addr (conn->up), + rspamd_upstream_addr_next (conn->up), SOCK_STREAM, TRUE); if (conn->sock == -1) { @@ -616,13 +616,14 @@ rspamd_fuzzy_send_update_mirror (struct rspamd_fuzzy_storage_ctx *ctx, msg = rspamd_http_new_message (HTTP_REQUEST); rspamd_printf_fstring (&msg->url, "/update_v1/%s", m->name); - conn->http_conn = rspamd_http_connection_new (NULL, + conn->http_conn = rspamd_http_connection_new ( + ctx->http_ctx, + conn->sock, + NULL, fuzzy_mirror_error_handler, fuzzy_mirror_finish_handler, RSPAMD_HTTP_CLIENT_SIMPLE, - RSPAMD_HTTP_CLIENT, - ctx->keypair_cache, - NULL); + RSPAMD_HTTP_CLIENT); rspamd_http_connection_set_key (conn->http_conn, ctx->sync_keypair); @@ -1569,8 +1570,7 @@ rspamd_fuzzy_mirror_send_reply (struct fuzzy_master_update_session *session, rspamd_http_connection_reset (session->conn); rspamd_http_connection_write_message (session->conn, msg, NULL, "text/plain", - session, session->sock, &session->ctx->master_io_tv, - session->ctx->ev_base); + session, &session->ctx->master_io_tv); } static void @@ -1711,9 +1711,7 @@ rspamd_fuzzy_collection_send_error (struct rspamd_http_connection_entry *entry, NULL, "text/plain", entry, - entry->conn->fd, - entry->rt->ptv, - entry->rt->ev_base); + entry->rt->ptv); entry->is_reply = TRUE; } @@ -1738,9 +1736,7 @@ rspamd_fuzzy_collection_send_fstring (struct rspamd_http_connection_entry *entry NULL, "application/octet-stream", entry, - entry->conn->fd, - entry->rt->ptv, - entry->rt->ev_base); + entry->rt->ptv); entry->is_reply = TRUE; } @@ -1994,13 +1990,14 @@ accept_fuzzy_mirror_socket (gint fd, short what, void *arg) session->name = rspamd_inet_address_to_string (addr); rspamd_random_hex (session->uid, sizeof (session->uid) - 1); session->uid[sizeof (session->uid) - 1] = '\0'; - http_conn = rspamd_http_connection_new (NULL, + http_conn = rspamd_http_connection_new ( + ctx->http_ctx, + nfd, + NULL, rspamd_fuzzy_mirror_error_handler, rspamd_fuzzy_mirror_finish_handler, 0, - RSPAMD_HTTP_SERVER, - ctx->keypair_cache, - NULL); + RSPAMD_HTTP_SERVER); rspamd_http_connection_set_key (http_conn, ctx->sync_keypair); session->ctx = ctx; @@ -2010,9 +2007,7 @@ accept_fuzzy_mirror_socket (gint fd, short what, void *arg) rspamd_http_connection_read_message (http_conn, session, - nfd, - &ctx->master_io_tv, - ctx->ev_base); + &ctx->master_io_tv); } /* @@ -3004,6 +2999,8 @@ start_fuzzy (struct rspamd_worker *worker) ctx->keypair_cache = rspamd_keypair_cache_new (ctx->keypair_cache_size); } + ctx->http_ctx = rspamd_http_context_create (cfg, ctx->ev_base); + if (!ctx->collection_mode) { /* * Open DB and perform VACUUM @@ -3058,8 +3055,8 @@ start_fuzzy (struct rspamd_worker *worker) rspamd_fuzzy_collection_error_handler, rspamd_fuzzy_collection_finish_handler, &ctx->stat_tv, - ctx->ev_base, - NULL, ctx->keypair_cache); + NULL, + ctx->http_ctx); if (ctx->collection_keypair) { rspamd_http_router_set_key (ctx->collection_rt, @@ -3202,8 +3199,6 @@ start_fuzzy (struct rspamd_worker *worker) else if (worker->index == 0) { gint fd; - /* Steal keypairs cache... */ - ctx->collection_rt->cache = NULL; rspamd_http_router_free (ctx->collection_rt); /* Try to save collection id */ @@ -3240,8 +3235,9 @@ start_fuzzy (struct rspamd_worker *worker) rspamd_keypair_cache_destroy (ctx->keypair_cache); } + struct rspamd_http_context *http_ctx = ctx->http_ctx; REF_RELEASE (ctx->cfg); - + rspamd_http_context_free (http_ctx); rspamd_log_close (worker->srv->logger, TRUE); exit (EXIT_SUCCESS); diff --git a/src/libmime/content_type.c b/src/libmime/content_type.c index ca371ce30..b67ec155e 100644 --- a/src/libmime/content_type.c +++ b/src/libmime/content_type.c @@ -305,8 +305,6 @@ rspamd_content_type_add_param (rspamd_mempool_t *pool, nparam->value.len = value_end - value_start; } - RSPAMD_FTOK_ASSIGN (&srch, "charset"); - srch.begin = nparam->name.begin; srch.len = nparam->name.len; @@ -713,36 +711,47 @@ rspamd_content_disposition_add_param (rspamd_mempool_t *pool, const gchar *value_start, const gchar *value_end) { rspamd_ftok_t srch; - gchar *decoded; + gchar *name_cpy, *value_cpy, *name_cpy_end, *value_cpy_end; struct rspamd_content_type_param *found = NULL, *nparam; g_assert (cd != NULL); - srch.begin = name_start; - srch.len = name_end - name_start; + name_cpy = rspamd_mempool_alloc (pool, name_end - name_start); + memcpy (name_cpy, name_start, name_end - name_start); + name_cpy_end = name_cpy + (name_end - name_start); + + value_cpy = rspamd_mempool_alloc (pool, value_end - value_start); + memcpy (value_cpy, value_start, value_end - value_start); + value_cpy_end = value_cpy + (value_end - value_start); + + nparam = rspamd_mempool_alloc0 (pool, sizeof (*nparam)); + rspamd_str_lc (name_cpy, name_cpy_end - name_cpy); + + if (!rspamd_param_maybe_rfc2231_process (pool, nparam, name_cpy, + name_cpy_end, value_cpy, value_cpy_end)) { + nparam->name.begin = name_cpy; + nparam->name.len = name_cpy_end - name_cpy; + nparam->value.begin = value_cpy; + nparam->value.len = value_cpy_end - value_cpy; + } + + srch.begin = nparam->name.begin; + srch.len = nparam->name.len; if (cd->attrs) { found = g_hash_table_lookup (cd->attrs, &srch); - } - else { + } else { cd->attrs = g_hash_table_new (rspamd_ftok_icase_hash, rspamd_ftok_icase_equal); - rspamd_mempool_add_destructor (pool, - (rspamd_mempool_destruct_t)g_hash_table_unref, cd->attrs); } - nparam = rspamd_mempool_alloc0 (pool, sizeof (*nparam)); - nparam->name.begin = name_start; - nparam->name.len = name_end - name_start; - decoded = rspamd_mime_header_decode (pool, value_start, - value_end - value_start, NULL); - RSPAMD_FTOK_FROM_STR (&nparam->value, decoded); - if (!found) { + DL_APPEND (found, nparam); g_hash_table_insert (cd->attrs, &nparam->name, nparam); } - - DL_APPEND (found, nparam); + else { + DL_APPEND (found, nparam); + } } struct rspamd_content_disposition * @@ -757,8 +766,13 @@ rspamd_content_disposition_parse (const gchar *in, res->lc_data = rspamd_mempool_alloc (pool, len + 1); rspamd_strlcpy (res->lc_data, in, len + 1); rspamd_str_lc (res->lc_data, len); - rspamd_postprocess_ct_attributes (pool, res->attrs, - rspamd_content_disposition_postprocess, res); + + if (res->attrs) { + rspamd_postprocess_ct_attributes (pool, res->attrs, + rspamd_content_disposition_postprocess, res); + rspamd_mempool_add_destructor (pool, + (rspamd_mempool_destruct_t)g_hash_table_unref, res->attrs); + } } else { msg_warn_pool ("cannot parse content disposition: %*s", diff --git a/src/libmime/lang_detection.c b/src/libmime/lang_detection.c index 72964a93a..7bfcf0164 100644 --- a/src/libmime/lang_detection.c +++ b/src/libmime/lang_detection.c @@ -459,9 +459,16 @@ rspamd_language_detector_read_file (struct rspamd_config *cfg, const char *word = ucl_object_tolstring (w, &wlen); const char *saved; +#ifdef WITH_HYPERSCAN + rspamd_multipattern_add_pattern_len (d->stop_words[cat].mp, + word, wlen, + RSPAMD_MULTIPATTERN_ICASE|RSPAMD_MULTIPATTERN_UTF8 + |RSPAMD_MULTIPATTERN_RE); +#else rspamd_multipattern_add_pattern_len (d->stop_words[cat].mp, word, wlen, RSPAMD_MULTIPATTERN_ICASE|RSPAMD_MULTIPATTERN_UTF8); +#endif nelt->stop_words ++; nstop ++; @@ -817,8 +824,15 @@ rspamd_language_detector_init (struct rspamd_config *cfg) /* Map from ngramm in ucs32 to GPtrArray of rspamd_language_elt */ for (i = 0; i < RSPAMD_LANGUAGE_MAX; i ++) { ret->trigramms[i] = kh_init (rspamd_trigram_hash); +#ifdef WITH_HYPERSCAN + ret->stop_words[i].mp = rspamd_multipattern_create ( + RSPAMD_MULTIPATTERN_ICASE|RSPAMD_MULTIPATTERN_UTF8| + RSPAMD_MULTIPATTERN_RE); +#else ret->stop_words[i].mp = rspamd_multipattern_create ( RSPAMD_MULTIPATTERN_ICASE|RSPAMD_MULTIPATTERN_UTF8); +#endif + ret->stop_words[i].ranges = g_array_new (FALSE, FALSE, sizeof (struct rspamd_stop_word_range)); } @@ -1457,7 +1471,10 @@ rspamd_language_detector_set_language (struct rspamd_task *task, r->prob = 1.0; r->lang = code; - part->languages = g_ptr_array_sized_new (1); + if (part->languages == NULL) { + part->languages = g_ptr_array_sized_new (1); + } + g_ptr_array_add (part->languages, r); part->language = code; } diff --git a/src/libmime/message.c b/src/libmime/message.c index da065195d..cca134f81 100644 --- a/src/libmime/message.c +++ b/src/libmime/message.c @@ -1150,7 +1150,7 @@ rspamd_message_parse (struct rspamd_task *task) need_recv_correction = TRUE; } else { - if (rspamd_inet_address_compare (raddr, task->from_addr) != 0) { + if (rspamd_inet_address_compare (raddr, task->from_addr, FALSE) != 0) { need_recv_correction = TRUE; } } diff --git a/src/libmime/mime_headers.c b/src/libmime/mime_headers.c index d0388a632..caf69cd16 100644 --- a/src/libmime/mime_headers.c +++ b/src/libmime/mime_headers.c @@ -639,68 +639,69 @@ rspamd_mime_header_decode (rspamd_mempool_t *pool, const gchar *in, if (qmarks < 3) { state = got_encoded_start; } - /* Finished encoded boundary */ - else if (rspamd_rfc2047_parser (c, p - c + 1, &encoding, - &cur_charset.begin, &cur_charset.len, - &tok_start, &tok_len)) { - /* We have a token, so we can decode it from `encoding` */ - if (token->len > 0) { - if (old_charset.len == 0) { - memcpy (&old_charset, &cur_charset, - sizeof (old_charset)); - } - - rspamd_mime_header_maybe_save_token (pool, out, - token, decoded, - &old_charset, &cur_charset); + else { + /* Finished encoded boundary */ + if (*c == '"') { + /* Quoted string, non-RFC conformant but used by retards */ + c ++; } + if (rspamd_rfc2047_parser (c, p - c + 1, &encoding, + &cur_charset.begin, &cur_charset.len, + &tok_start, &tok_len)) { + /* We have a token, so we can decode it from `encoding` */ + if (token->len > 0) { + if (old_charset.len == 0) { + memcpy (&old_charset, &cur_charset, + sizeof (old_charset)); + } - qmarks = 0; - pos = token->len; - g_byte_array_set_size (token, pos + tok_len); + rspamd_mime_header_maybe_save_token (pool, out, + token, decoded, + &old_charset, &cur_charset); + } - if (encoding == RSPAMD_RFC2047_QP) { - r = rspamd_decode_qp2047_buf (tok_start, tok_len, - token->data + pos, tok_len); + qmarks = 0; + pos = token->len; + g_byte_array_set_size (token, pos + tok_len); - if (r != -1) { - token->len = pos + r; - } - else { - /* Cannot decode qp */ - token->len -= tok_len; - } - } - else { - if (rspamd_cryptobox_base64_decode (tok_start, tok_len, - token->data + pos, &tok_len)) { - token->len = pos + tok_len; - } - else { - /* Cannot decode */ - token->len -= tok_len; - } - } + if (encoding == RSPAMD_RFC2047_QP) { + r = rspamd_decode_qp2047_buf (tok_start, tok_len, + token->data + pos, tok_len); - c = p + 1; - state = skip_spaces; - } - else { - /* Not encoded-word */ - old_charset.len = 0; + if (r != -1) { + token->len = pos + r; + } else { + /* Cannot decode qp */ + token->len -= tok_len; + } + } else { + if (rspamd_cryptobox_base64_decode (tok_start, tok_len, + token->data + pos, &tok_len)) { + token->len = pos + tok_len; + } else { + /* Cannot decode */ + token->len -= tok_len; + } + } - if (token->len > 0) { - rspamd_mime_header_maybe_save_token (pool, out, - token, decoded, - &old_charset, &cur_charset); - } + c = p + 1; + state = skip_spaces; + } else { + /* Not encoded-word */ + old_charset.len = 0; - g_string_append_len (out, c, p - c); - c = p; - state = parse_normal; - } + if (token->len > 0) { + rspamd_mime_header_maybe_save_token (pool, out, + token, decoded, + &old_charset, &cur_charset); + } - } + g_string_append_len (out, c, p - c); + c = p; + state = parse_normal; + } + } /* qmarks >= 3 */ + } /* p == '=' */ else { state = got_encoded_start; } diff --git a/src/libmime/mime_parser.c b/src/libmime/mime_parser.c index 7151140cb..21a81575d 100644 --- a/src/libmime/mime_parser.c +++ b/src/libmime/mime_parser.c @@ -535,6 +535,7 @@ rspamd_mime_process_multipart_node (struct rspamd_task *task, struct rspamd_mime_parser_ctx *st, struct rspamd_mime_part *multipart, const gchar *start, const gchar *end, + gboolean is_finished, GError **err) { struct rspamd_content_type *ct, *sel = NULL; @@ -558,6 +559,24 @@ rspamd_mime_process_multipart_node (struct rspamd_task *task, */ hdr_pos = 0; body_pos = 0; + + if (!is_finished) { + /* Ignore garbage */ + const gchar *p = start; + gboolean seen_something = FALSE; + + while (p < end) { + if (g_ascii_isalnum (*p)) { + seen_something = TRUE; + break; + } + p ++; + } + + if (!seen_something) { + return RSPAMD_MIME_PARSE_NO_PART; + } + } } else { hdr_pos = rspamd_string_find_eoh (&str, &body_pos); @@ -609,7 +628,7 @@ rspamd_mime_process_multipart_node (struct rspamd_task *task, for (i = 0; i < hdrs->len; i ++) { hdr = g_ptr_array_index (hdrs, i); - ct = rspamd_content_type_parse (hdr->value, strlen (hdr->value), + ct = rspamd_content_type_parse (hdr->decoded, strlen (hdr->decoded), task->task_pool); /* Here we prefer multipart content-type or any content-type */ @@ -681,8 +700,8 @@ rspamd_mime_parse_multipart_cb (struct rspamd_task *task, g_assert (cb->cur_boundary != NULL); if ((ret = rspamd_mime_process_multipart_node (task, cb->st, - cb->multipart, cb->part_start, pos, cb->err)) - != RSPAMD_MIME_PARSE_OK) { + cb->multipart, cb->part_start, pos, TRUE, cb->err)) + != RSPAMD_MIME_PARSE_OK) { return ret; } @@ -1240,7 +1259,7 @@ rspamd_mime_parse_message (struct rspamd_task *task, else { for (i = 0; i < hdrs->len; i ++) { hdr = g_ptr_array_index (hdrs, i); - ct = rspamd_content_type_parse (hdr->value, strlen (hdr->value), + ct = rspamd_content_type_parse (hdr->decoded, strlen (hdr->decoded), task->task_pool); /* Here we prefer multipart content-type or any content-type */ @@ -1338,7 +1357,16 @@ rspamd_mime_parse_message (struct rspamd_task *task, if (end > start && (ret = rspamd_mime_process_multipart_node (task, st, - NULL, start, end, err)) != RSPAMD_MIME_PARSE_OK) { + NULL, start, end, FALSE, err)) != RSPAMD_MIME_PARSE_OK) { + + if (nst != st) { + rspamd_mime_parse_stack_free (nst); + } + + if (ret == RSPAMD_MIME_PARSE_NO_PART) { + return RSPAMD_MIME_PARSE_OK; + } + return ret; } } diff --git a/src/libmime/mime_parser.h b/src/libmime/mime_parser.h index 5164ea4d5..987ec1395 100644 --- a/src/libmime/mime_parser.h +++ b/src/libmime/mime_parser.h @@ -25,6 +25,7 @@ enum rspamd_mime_parse_error { RSPAMD_MIME_PARSE_OK = 0, RSPAMD_MIME_PARSE_FATAL, RSPAMD_MIME_PARSE_NESTING, + RSPAMD_MIME_PARSE_NO_PART, }; enum rspamd_mime_parse_error rspamd_mime_parse_task (struct rspamd_task *task, diff --git a/src/libserver/cfg_file.h b/src/libserver/cfg_file.h index c7475c327..c102ef570 100644 --- a/src/libserver/cfg_file.h +++ b/src/libserver/cfg_file.h @@ -98,6 +98,13 @@ enum lua_var_type { LUA_VAR_UNKNOWN }; +enum rspamd_symbol_group_flags { + RSPAMD_SYMBOL_GROUP_NORMAL = 0, + RSPAMD_SYMBOL_GROUP_DISABLED = (1 << 0), + RSPAMD_SYMBOL_GROUP_ONE_SHOT = (1 << 1), + RSPAMD_SYMBOL_GROUP_UNGROUPED = (1 << 2), +}; + /** * Symbols group */ @@ -106,13 +113,15 @@ struct rspamd_symbols_group { gchar *name; GHashTable *symbols; gdouble max_score; - gboolean disabled; - gboolean one_shot; + enum rspamd_symbol_group_flags flags; }; -#define RSPAMD_SYMBOL_FLAG_IGNORE (1 << 1) -#define RSPAMD_SYMBOL_FLAG_ONEPARAM (1 << 2) -#define RSPAMD_SYMBOL_FLAG_UNGROUPPED (1 << 3) +enum rspamd_symbol_flags { + RSPAMD_SYMBOL_FLAG_NORMAL = 0, + RSPAMD_SYMBOL_FLAG_IGNORE = (1 << 1), + RSPAMD_SYMBOL_FLAG_ONEPARAM = (1 << 2), + RSPAMD_SYMBOL_FLAG_UNGROUPPED = (1 << 3), +}; /** * Symbol config definition @@ -125,7 +134,7 @@ struct rspamd_symbol { guint priority; struct rspamd_symbols_group *gr; /* Main group */ GPtrArray *groups; /* Other groups */ - guint flags; + enum rspamd_symbol_flags flags; gint nshots; }; diff --git a/src/libserver/cfg_rcl.c b/src/libserver/cfg_rcl.c index fa1c07f6e..7520b66d8 100644 --- a/src/libserver/cfg_rcl.c +++ b/src/libserver/cfg_rcl.c @@ -326,7 +326,7 @@ rspamd_rcl_group_handler (rspamd_mempool_t *pool, const ucl_object_t *obj, { struct rspamd_config *cfg = ud; struct rspamd_symbols_group *gr; - const ucl_object_t *val; + const ucl_object_t *val, *elt; struct rspamd_rcl_section *subsection; struct rspamd_rcl_symbol_data sd; @@ -343,6 +343,51 @@ rspamd_rcl_group_handler (rspamd_mempool_t *pool, const ucl_object_t *obj, return FALSE; } + if ((elt = ucl_object_lookup (obj, "one_shot")) != NULL) { + if (ucl_object_type (elt) != UCL_BOOLEAN) { + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "one_shot attribute is not boolean for symbol: '%s'", + key); + + return FALSE; + } + if (ucl_object_toboolean (elt)) { + gr->flags |= RSPAMD_SYMBOL_GROUP_ONE_SHOT; + } + } + + if ((elt = ucl_object_lookup (obj, "disabled")) != NULL) { + if (ucl_object_type (elt) != UCL_BOOLEAN) { + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "disabled attribute is not boolean for symbol: '%s'", + key); + + return FALSE; + } + if (ucl_object_toboolean (elt)) { + gr->flags |= RSPAMD_SYMBOL_GROUP_DISABLED; + } + } + + if ((elt = ucl_object_lookup (obj, "enabled")) != NULL) { + if (ucl_object_type (elt) != UCL_BOOLEAN) { + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "enabled attribute is not boolean for symbol: '%s'", + key); + + return FALSE; + } + if (!ucl_object_toboolean (elt)) { + gr->flags |= RSPAMD_SYMBOL_GROUP_DISABLED; + } + } + sd.gr = gr; sd.cfg = cfg; @@ -525,39 +570,32 @@ rspamd_rcl_actions_handler (rspamd_mempool_t *pool, const ucl_object_t *obj, const gchar *key, gpointer ud, struct rspamd_rcl_section *section, GError **err) { - gdouble action_score; struct rspamd_config *cfg = ud; - gint action_value; const ucl_object_t *cur; ucl_object_iter_t it; it = ucl_object_iterate_new (obj); while ((cur = ucl_object_iterate_safe (it, true)) != NULL) { - if (!rspamd_action_from_str (ucl_object_key (cur), &action_value)) { - continue; + gint type = ucl_object_type (cur); + + if (type == UCL_NULL) { + rspamd_config_maybe_disable_action (cfg, ucl_object_key (cur), + ucl_object_get_priority (cur)); } - else { - if (ucl_object_type (cur) == UCL_NULL) { - rspamd_config_maybe_disable_action (cfg, ucl_object_key (cur), - ucl_object_get_priority (cur)); - } - else { - if (!ucl_object_todouble_safe (cur, &action_score)) { - g_set_error (err, - CFG_RCL_ERROR, - EINVAL, - "invalid action definition: '%s'", - ucl_object_key (cur)); - ucl_object_iterate_free (it); + else if (type == UCL_OBJECT || type == UCL_FLOAT || type == UCL_INT) { + if (!rspamd_config_set_action_score (cfg, + ucl_object_key (cur), + cur)) { + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "invalid action definition for: '%s'", + ucl_object_key (cur)); + ucl_object_iterate_free (it); - return FALSE; - } + return FALSE; } - - rspamd_config_set_action_score (cfg, - ucl_object_key (cur), - cur); } } @@ -1314,7 +1352,9 @@ rspamd_rcl_composite_handler (rspamd_mempool_t *pool, } rspamd_config_add_symbol (cfg, composite_name, score, - description, group, FALSE, FALSE, + description, group, + 0, + ucl_object_get_priority (obj), /* No +1 as it is default... */ 1); elt = ucl_object_lookup (obj, "groups"); @@ -1355,7 +1395,7 @@ rspamd_rcl_composite_handler (rspamd_mempool_t *pool, if (new) { rspamd_symcache_add_symbol (cfg->cache, composite_name, 0, - NULL, NULL, SYMBOL_TYPE_COMPOSITE, -1); + NULL, composite, SYMBOL_TYPE_COMPOSITE, -1); } return TRUE; @@ -1772,12 +1812,6 @@ rspamd_rcl_config_init (struct rspamd_config *cfg, GHashTable *skip_sections) 0, "Treat text attachments as normal text parts"); rspamd_rcl_add_default_handler (sub, - "check_attachments", - rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_config, check_text_attachements), - 0, - "Treat text attachments as normal text parts"); - rspamd_rcl_add_default_handler (sub, "tempdir", rspamd_rcl_parse_struct_string, G_STRUCT_OFFSET (struct rspamd_config, temp_dir), @@ -2192,18 +2226,6 @@ rspamd_rcl_config_init (struct rspamd_config *cfg, GHashTable *skip_sections) /* Group part */ rspamd_rcl_add_default_handler (sub, - "disabled", - rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_symbols_group, disabled), - 0, - "Disable symbols group"); - rspamd_rcl_add_default_handler (sub, - "enabled", - rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_symbols_group, disabled), - RSPAMD_CL_FLAG_BOOLEAN_INVERSE, - "Enable or disable symbols group"); - rspamd_rcl_add_default_handler (sub, "max_score", rspamd_rcl_parse_struct_double, G_STRUCT_OFFSET (struct rspamd_symbols_group, max_score), diff --git a/src/libserver/cfg_utils.c b/src/libserver/cfg_utils.c index cc07c4007..af31db849 100644 --- a/src/libserver/cfg_utils.c +++ b/src/libserver/cfg_utils.c @@ -58,7 +58,7 @@ static gchar * rspamd_ucl_read_cb (gchar * chunk, gint len, struct map_cb_data *data, gboolean final); -static void rspamd_ucl_fin_cb (struct map_cb_data *data); +static void rspamd_ucl_fin_cb (struct map_cb_data *data, void **target); static void rspamd_ucl_dtor_cb (struct map_cb_data *data); guint rspamd_config_log_id = (guint)-1; @@ -1039,6 +1039,10 @@ rspamd_config_new_group (struct rspamd_config *cfg, const gchar *name) (rspamd_mempool_destruct_t)g_hash_table_unref, gr->symbols); gr->name = rspamd_mempool_strdup (cfg->cfg_pool, name); + if (strcmp (gr->name, "ungrouped") == 0) { + gr->flags |= RSPAMD_SYMBOL_GROUP_UNGROUPED; + } + g_hash_table_insert (cfg->groups, gr->name, gr); return gr; @@ -1354,7 +1358,7 @@ rspamd_ucl_read_cb (gchar * chunk, } static void -rspamd_ucl_fin_cb (struct map_cb_data *data) +rspamd_ucl_fin_cb (struct map_cb_data *data, void **target) { struct rspamd_ucl_map_cbdata *cbdata = data->cur_data, *prev = data->prev_data; @@ -1364,13 +1368,6 @@ rspamd_ucl_fin_cb (struct map_cb_data *data) const ucl_object_t *cur; struct rspamd_config *cfg = data->map->cfg; - if (prev != NULL) { - if (prev->buf != NULL) { - g_string_free (prev->buf, TRUE); - } - g_free (prev); - } - if (cbdata == NULL) { msg_err_config ("map fin error: new data is NULL"); return; @@ -1396,6 +1393,17 @@ rspamd_ucl_fin_cb (struct map_cb_data *data) } ucl_object_unref (obj); } + + if (target) { + *target = data->cur_data; + } + + if (prev != NULL) { + if (prev->buf != NULL) { + g_string_free (prev->buf, TRUE); + } + g_free (prev); + } } static void @@ -1684,10 +1692,7 @@ rspamd_config_add_symbol (struct rspamd_config *cfg, /* We also check group information in this case */ if (group != NULL && sym_def->gr != NULL && strcmp (group, sym_def->gr->name) != 0) { - msg_debug_config ("move symbol %s from group %s to %s", - sym_def->gr->name, group); - g_hash_table_remove (sym_def->gr->symbols, sym_def->name); sym_group = g_hash_table_lookup (cfg->groups, group); if (sym_group == NULL) { @@ -1695,8 +1700,13 @@ rspamd_config_add_symbol (struct rspamd_config *cfg, sym_group = rspamd_config_new_group (cfg, group); } - sym_def->gr = sym_group; - g_hash_table_insert (sym_group->symbols, sym_def->name, sym_def); + if (!(sym_group->flags & RSPAMD_SYMBOL_GROUP_UNGROUPED)) { + msg_debug_config ("move symbol %s from group %s to %s", + sym_def->gr->name, group); + g_hash_table_remove (sym_def->gr->symbols, sym_def->name); + sym_def->gr = sym_group; + g_hash_table_insert (sym_group->symbols, sym_def->name, sym_def); + } } return TRUE; @@ -1870,7 +1880,7 @@ rspamd_config_is_module_enabled (struct rspamd_config *cfg, gr = g_hash_table_lookup (cfg->groups, module_name); if (gr) { - if (gr->disabled) { + if (gr->flags & RSPAMD_SYMBOL_GROUP_DISABLED) { rspamd_plugins_table_push_elt (L, "disabled_explicitly", module_name); msg_info_config ("%s module %s is disabled in the configuration as " diff --git a/src/libserver/composites.c b/src/libserver/composites.c index f46f8276d..b1bb2f694 100644 --- a/src/libserver/composites.c +++ b/src/libserver/composites.c @@ -559,9 +559,10 @@ composites_metric_callback (struct rspamd_metric_result *metric_res, NBYTES (g_hash_table_size (task->cfg->composite_symbols) * 2)); /* Process hash table */ - g_hash_table_foreach (task->cfg->composite_symbols, - composites_foreach_callback, - cd); + rspamd_symcache_composites_foreach (task, + task->cfg->cache, + composites_foreach_callback, + cd); /* Remove symbols that are in composites */ g_hash_table_foreach (cd->symbols_to_remove, composites_remove_symbols, cd); diff --git a/src/libserver/dkim.c b/src/libserver/dkim.c index 2a66146ed..129d78c9c 100644 --- a/src/libserver/dkim.c +++ b/src/libserver/dkim.c @@ -2681,13 +2681,25 @@ rspamd_dkim_sign_key_load (const gchar *key, gsize len, key = tmp; } - if (type == RSPAMD_DKIM_KEY_RAW && len == 32) { - unsigned char pk[32]; - nkey->type = RSPAMD_DKIM_KEY_EDDSA; - nkey->key.key_eddsa = g_malloc ( - rspamd_cryptobox_sk_sig_bytes (RSPAMD_CRYPTOBOX_MODE_25519)); - ed25519_seed_keypair (pk, nkey->key.key_eddsa, (char *)key); - nkey->keylen = rspamd_cryptobox_sk_sig_bytes (RSPAMD_CRYPTOBOX_MODE_25519); + if (type == RSPAMD_DKIM_KEY_RAW && (len == 32 || + len == rspamd_cryptobox_sk_sig_bytes (RSPAMD_CRYPTOBOX_MODE_25519))) { + if (len == 32) { + /* Seeded key, need scalarmult */ + unsigned char pk[32]; + nkey->type = RSPAMD_DKIM_KEY_EDDSA; + nkey->key.key_eddsa = g_malloc ( + rspamd_cryptobox_sk_sig_bytes (RSPAMD_CRYPTOBOX_MODE_25519)); + ed25519_seed_keypair (pk, nkey->key.key_eddsa, (char *) key); + nkey->keylen = rspamd_cryptobox_sk_sig_bytes (RSPAMD_CRYPTOBOX_MODE_25519); + } + else { + /* Full ed25519 key */ + unsigned klen = rspamd_cryptobox_sk_sig_bytes (RSPAMD_CRYPTOBOX_MODE_25519); + nkey->type = RSPAMD_DKIM_KEY_EDDSA; + nkey->key.key_eddsa = g_malloc (klen); + memcpy (nkey->key.key_eddsa, key, klen); + nkey->keylen = klen; + } } else { nkey->key_bio = BIO_new_mem_buf (key, len); diff --git a/src/libserver/dns.c b/src/libserver/dns.c index 7ad266c2f..4fbf40728 100644 --- a/src/libserver/dns.c +++ b/src/libserver/dns.c @@ -256,7 +256,7 @@ rspamd_dns_server_init (struct upstream *up, guint idx, gpointer ud) void *serv; struct rdns_upstream_elt *elt; - addr = rspamd_upstream_addr (up); + addr = rspamd_upstream_addr_next (up); if (r->cfg) { serv = rdns_resolver_add_server (r->r, rspamd_inet_address_to_string (addr), diff --git a/src/libserver/dynamic_cfg.c b/src/libserver/dynamic_cfg.c index 1e970a17b..984a26697 100644 --- a/src/libserver/dynamic_cfg.c +++ b/src/libserver/dynamic_cfg.c @@ -175,22 +175,12 @@ json_config_read_cb (gchar * chunk, } static void -json_config_fin_cb (struct map_cb_data *data) +json_config_fin_cb (struct map_cb_data *data, void **target) { struct config_json_buf *jb; ucl_object_t *top; struct ucl_parser *parser; - if (data->cur_data && data->prev_data) { - jb = data->prev_data; - /* Clean prev data */ - if (jb->buf) { - g_string_free (jb->buf, TRUE); - } - - g_free (jb); - } - /* Now parse json */ if (data->cur_data) { jb = data->cur_data; @@ -201,6 +191,7 @@ json_config_fin_cb (struct map_cb_data *data) if (jb->buf == NULL) { msg_err ("no data read"); + return; } @@ -225,6 +216,20 @@ json_config_fin_cb (struct map_cb_data *data) ucl_object_unref (jb->cfg->current_dynamic_conf); apply_dynamic_conf (top, jb->cfg); jb->cfg->current_dynamic_conf = top; + + if (target) { + *target = data->cur_data; + } + + if (data->prev_data) { + jb = data->prev_data; + /* Clean prev data */ + if (jb->buf) { + g_string_free (jb->buf, TRUE); + } + + g_free (jb); + } } static void diff --git a/src/libserver/fuzzy_backend_redis.c b/src/libserver/fuzzy_backend_redis.c index fbccac5ab..956979d42 100644 --- a/src/libserver/fuzzy_backend_redis.c +++ b/src/libserver/fuzzy_backend_redis.c @@ -648,7 +648,7 @@ rspamd_fuzzy_backend_check_redis (struct rspamd_fuzzy_backend *bk, 0); session->up = up; - addr = rspamd_upstream_addr (up); + addr = rspamd_upstream_addr_next (up); g_assert (addr != NULL); session->ctx = rspamd_redis_pool_connect (backend->pool, backend->dbname, backend->password, @@ -774,7 +774,7 @@ rspamd_fuzzy_backend_count_redis (struct rspamd_fuzzy_backend *bk, 0); session->up = up; - addr = rspamd_upstream_addr (up); + addr = rspamd_upstream_addr_next (up); g_assert (addr != NULL); session->ctx = rspamd_redis_pool_connect (backend->pool, backend->dbname, backend->password, @@ -899,7 +899,7 @@ rspamd_fuzzy_backend_version_redis (struct rspamd_fuzzy_backend *bk, 0); session->up = up; - addr = rspamd_upstream_addr (up); + addr = rspamd_upstream_addr_next (up); g_assert (addr != NULL); session->ctx = rspamd_redis_pool_connect (backend->pool, backend->dbname, backend->password, @@ -1459,7 +1459,7 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk, 0); session->up = up; - addr = rspamd_upstream_addr (up); + addr = rspamd_upstream_addr_next (up); g_assert (addr != NULL); session->ctx = rspamd_redis_pool_connect (backend->pool, backend->dbname, backend->password, diff --git a/src/libserver/html.c b/src/libserver/html.c index c33aacf82..63638d28b 100644 --- a/src/libserver/html.c +++ b/src/libserver/html.c @@ -664,7 +664,8 @@ rspamd_html_url_is_phished (rspamd_mempool_t *pool, } } #endif - if (rspamd_ftok_casecmp (&disp_tok, &href_tok) != 0) { + if (rspamd_ftok_casecmp (&disp_tok, &href_tok) != 0 && + text_url->tldlen > 0 && href_url->tldlen > 0) { /* Apply the same logic for TLD */ disp_tok.len = text_url->tldlen; @@ -1301,7 +1302,7 @@ rspamd_html_process_url (rspamd_mempool_t *pool, const gchar *start, guint len, gchar *decoded; gint rc; gsize decoded_len; - const gchar *p, *s; + const gchar *p, *s, *prefix = "http://"; gchar *d; guint i, dlen; gboolean has_bad_chars = FALSE, no_prefix = FALSE; @@ -1346,25 +1347,55 @@ rspamd_html_process_url (rspamd_mempool_t *pool, const gchar *start, guint len, } } - if (memchr (s, ':', len) == NULL) { - /* We have no prefix */ - dlen += sizeof ("http://") - 1; - no_prefix = TRUE; + if (rspamd_substring_search (start, len, "://", 3) == -1) { + if (len >= sizeof ("mailto:") && + (memcmp (start, "mailto:", sizeof ("mailto:") - 1) == 0 || + memcmp (start, "tel:", sizeof ("tel:") - 1) == 0 || + memcmp (start, "callto:", sizeof ("callto:") - 1) == 0)) { + /* Exclusion, has valid but 'strange' prefix */ + } + else { + for (i = 0; i < len; i ++) { + if (!((s[i] & 0x80) || g_ascii_isalnum (s[i]))) { + if (i == 0 && len > 2 && s[i] == '/' && s[i + 1] == '/') { + prefix = "http:"; + dlen += sizeof ("http:") - 1; + no_prefix = TRUE; + } + else if (s[i] == '@') { + /* Likely email prefix */ + prefix = "mailto://"; + dlen += sizeof ("mailto://") - 1; + no_prefix = TRUE; + } + else if (s[i] == ':' && i != 0) { + /* Special case */ + no_prefix = FALSE; + } + else { + if (i == 0) { + /* No valid data */ + return NULL; + } + else { + no_prefix = TRUE; + dlen += strlen (prefix); + } + } + + break; + } + } + } } decoded = rspamd_mempool_alloc (pool, dlen + 1); d = decoded; if (no_prefix) { - if (s[0] == '/' && (len > 2 && s[1] == '/')) { - /* //bla case */ - memcpy (d, "http:", sizeof ("http:") - 1); - d += sizeof ("http:") - 1; - } - else { - memcpy (d, "http://", sizeof ("http://") - 1); - d += sizeof ("http://") - 1; - } + gsize plen = strlen (prefix); + memcpy (d, prefix, plen); + d += plen; } /* @@ -1410,7 +1441,9 @@ rspamd_html_process_url (rspamd_mempool_t *pool, const gchar *start, guint len, rc = rspamd_url_parse (url, decoded, dlen, pool, RSPAMD_URL_PARSE_HREF); - if (rc == URI_ERRNO_OK) { + /* Filter some completely damaged urls */ + if (rc == URI_ERRNO_OK && url->hostlen > 0 && + !((url->flags & RSPAMD_URL_FLAG_OBSCURED) && (url->protocol & PROTOCOL_UNKNOWN))) { url->flags |= saved_flags; if (has_bad_chars) { @@ -2664,10 +2697,33 @@ rspamd_html_process_part_full (rspamd_mempool_t *pool, struct html_content *hc, case comment_tag: if (t != '-') { hc->flags |= RSPAMD_HTML_FLAG_BAD_ELEMENTS; + state = tag_end; + } + else { + p++; + ebrace = 0; + /* + * https://www.w3.org/TR/2012/WD-html5-20120329/syntax.html#syntax-comments + * ... the text must not start with a single + * U+003E GREATER-THAN SIGN character (>), + * nor start with a "-" (U+002D) character followed by + * a U+003E GREATER-THAN SIGN (>) character, + * nor contain two consecutive U+002D HYPHEN-MINUS + * characters (--), nor end with a "-" (U+002D) character. + */ + if (p[0] == '-' && p + 1 < end && p[1] == '>') { + hc->flags |= RSPAMD_HTML_FLAG_BAD_ELEMENTS; + p ++; + state = tag_end; + } + else if (*p == '>') { + hc->flags |= RSPAMD_HTML_FLAG_BAD_ELEMENTS; + state = tag_end; + } + else { + state = comment_content; + } } - p ++; - ebrace = 0; - state = comment_content; break; case comment_content: diff --git a/src/libserver/milter.c b/src/libserver/milter.c index b3cd46226..236a4bf75 100644 --- a/src/libserver/milter.c +++ b/src/libserver/milter.c @@ -22,7 +22,7 @@ #include "unix-std.h" #include "logger.h" #include "ottery.h" -#include "libutil/http.h" +#include "libutil/http_connection.h" #include "libutil/http_private.h" #include "libserver/protocol_internal.h" #include "libserver/cfg_file_private.h" @@ -235,6 +235,7 @@ static void rspamd_milter_on_protocol_error (struct rspamd_milter_session *session, struct rspamd_milter_private *priv, GError *err) { + msg_debug_milter ("protocol error: %e", err); priv->state = RSPAMD_MILTER_WANNA_DIE; REF_RETAIN (session); priv->err_cb (priv->fd, session, priv->ud, err); @@ -1189,6 +1190,7 @@ rspamd_milter_send_action (struct rspamd_milter_session *session, case RSPAMD_MILTER_REJECT: case RSPAMD_MILTER_TEMPFAIL: /* No additional arguments */ + msg_debug_milter ("send %c command", cmd); SET_COMMAND (cmd, 0, reply, pos); break; case RSPAMD_MILTER_QUARANTINE: @@ -1199,6 +1201,7 @@ rspamd_milter_send_action (struct rspamd_milter_session *session, } len = strlen (reason); + msg_debug_milter ("send quarantine action %s", reason); SET_COMMAND (cmd, len + 1, reply, pos); memcpy (pos, reason, len + 1); break; @@ -1207,6 +1210,7 @@ rspamd_milter_send_action (struct rspamd_milter_session *session, value = va_arg (ap, GString *); /* Name and value must be zero terminated */ + msg_debug_milter ("add header command - \"%v\"=\"%v\"", name, value); SET_COMMAND (cmd, name->len + value->len + 2, reply, pos); memcpy (pos, name->str, name->len + 1); pos += name->len + 1; @@ -1214,13 +1218,16 @@ rspamd_milter_send_action (struct rspamd_milter_session *session, break; case RSPAMD_MILTER_CHGHEADER: case RSPAMD_MILTER_INSHEADER: - idx = htonl (va_arg (ap, guint32)); + idx = va_arg (ap, guint32); name = va_arg (ap, GString *); value = va_arg (ap, GString *); + msg_debug_milter ("change/insert header command pos = %d- \"%v\"=\"%v\"", + idx, name, value); /* Name and value must be zero terminated */ SET_COMMAND (cmd, name->len + value->len + 2 + sizeof (guint32), reply, pos); + idx = htonl (idx); memcpy (pos, &idx, sizeof (idx)); pos += sizeof (idx); memcpy (pos, name->str, name->len + 1); @@ -1233,14 +1240,20 @@ rspamd_milter_send_action (struct rspamd_milter_session *session, case RSPAMD_MILTER_CHGFROM: /* Single GString * argument */ value = va_arg (ap, GString *); + msg_debug_milter ("command %c; value=%v", cmd, value); SET_COMMAND (cmd, value->len + 1, reply, pos); memcpy (pos, value->str, value->len + 1); break; case RSPAMD_MILTER_OPTNEG: - ver = htonl (va_arg (ap, guint32)); - actions = htonl (va_arg (ap, guint32)); - protocol = htonl (va_arg (ap, guint32)); - + ver = va_arg (ap, guint32); + actions = va_arg (ap, guint32); + protocol = va_arg (ap, guint32); + + msg_debug_milter ("optneg reply: ver=%d, actions=%d, protocol=%d", + ver, actions, protocol); + ver = htonl (ver); + actions = htonl (actions); + protocol = htonl (protocol); SET_COMMAND (cmd, sizeof (guint32) * 3, reply, pos); memcpy (pos, &ver, sizeof (ver)); pos += sizeof (ver); @@ -1531,9 +1544,9 @@ rspamd_milter_remove_header_safe (struct rspamd_milter_session *session, RSPAMD_MILTER_CHGHEADER, nhdr, hname, hvalue); } - else if (nhdr == 0) { + else if (nhdr == 0 && ar->len > 0) { /* We need to clear all headers */ - for (i = 1; i <= ar->len; i ++) { + for (i = ar->len; i > 0; i --) { rspamd_milter_send_action (session, RSPAMD_MILTER_CHGHEADER, i, hname, hvalue); diff --git a/src/libserver/monitored.c b/src/libserver/monitored.c index 9adcab943..39590bfcc 100644 --- a/src/libserver/monitored.c +++ b/src/libserver/monitored.c @@ -51,7 +51,6 @@ struct rspamd_monitored_ctx { struct rspamd_monitored { gchar *url; - gdouble monitoring_interval; gdouble monitoring_mult; gdouble offline_time; gdouble total_offline_time; @@ -173,7 +172,7 @@ rspamd_monitored_periodic (gint fd, short what, gpointer ud) gdouble jittered; gboolean ret = FALSE; - jittered = rspamd_time_jitter (m->monitoring_interval * m->monitoring_mult, + jittered = rspamd_time_jitter (m->ctx->monitoring_interval * m->monitoring_mult, 0.0); double_to_tv (jittered, &tv); @@ -480,7 +479,6 @@ rspamd_monitored_create_ (struct rspamd_monitored_ctx *ctx, m->flags = flags; m->url = g_strdup (line); m->ctx = ctx; - m->monitoring_interval = ctx->monitoring_interval; m->monitoring_mult = 1.0; m->max_errors = ctx->max_errors; m->alive = TRUE; @@ -602,7 +600,7 @@ rspamd_monitored_start (struct rspamd_monitored *m) g_assert (m != NULL); msg_debug_mon ("started monitored object %s", m->url); - jittered = rspamd_time_jitter (m->monitoring_interval * m->monitoring_mult, + jittered = rspamd_time_jitter (m->ctx->monitoring_interval * m->monitoring_mult, 0.0); double_to_tv (jittered, &tv); diff --git a/src/libserver/protocol.c b/src/libserver/protocol.c index 5bcfbc37a..2a9cc605a 100644 --- a/src/libserver/protocol.c +++ b/src/libserver/protocol.c @@ -1737,7 +1737,7 @@ rspamd_protocol_write_reply (struct rspamd_task *task) rspamd_http_connection_reset (task->http_conn); rspamd_http_connection_write_message (task->http_conn, msg, NULL, - ctype, task, task->sock, &task->tv, task->ev_base); + ctype, task, &task->tv); task->processed_stages |= RSPAMD_TASK_STAGE_REPLIED; } diff --git a/src/libserver/protocol.h b/src/libserver/protocol.h index 254c5fcd1..08372d765 100644 --- a/src/libserver/protocol.h +++ b/src/libserver/protocol.h @@ -8,7 +8,7 @@ #include "config.h" #include "filter.h" -#include "http.h" +#include "http_connection.h" #include "task.h" #define RSPAMD_BASE_ERROR 500 diff --git a/src/libserver/rspamd_control.c b/src/libserver/rspamd_control.c index 84c53700e..149ad4245 100644 --- a/src/libserver/rspamd_control.c +++ b/src/libserver/rspamd_control.c @@ -16,7 +16,8 @@ #include "config.h" #include "rspamd.h" #include "rspamd_control.h" -#include "libutil/http.h" +#include "worker_util.h" +#include "libutil/http_connection.h" #include "libutil/http_private.h" #include "unix-std.h" #include "utlist.h" @@ -51,6 +52,7 @@ struct rspamd_control_session { struct rspamd_http_connection *conn; struct rspamd_control_command cmd; struct rspamd_control_reply_elt *replies; + rspamd_inet_addr_t *addr; guint replies_remain; gboolean is_reply; }; @@ -129,9 +131,7 @@ rspamd_control_send_error (struct rspamd_control_session *session, NULL, "application/json", session, - session->fd, - &io_timeout, - session->rspamd_main->ev_base); + &io_timeout); } static void @@ -154,21 +154,25 @@ rspamd_control_send_ucl (struct rspamd_control_session *session, NULL, "application/json", session, - session->fd, - &io_timeout, - session->rspamd_main->ev_base); + &io_timeout); } static void rspamd_control_connection_close (struct rspamd_control_session *session) { struct rspamd_control_reply_elt *elt, *telt; + struct rspamd_main *rspamd_main; + + rspamd_main = session->rspamd_main; + msg_info_main ("finished connection from %s", + rspamd_inet_address_to_string (session->addr)); DL_FOREACH_SAFE (session->replies, elt, telt) { event_del (&elt->io_ev); g_free (elt); } + rspamd_inet_address_free (session->addr); rspamd_http_connection_unref (session->conn); close (session->fd); g_free (session); @@ -365,9 +369,12 @@ static void rspamd_control_error_handler (struct rspamd_http_connection *conn, GError *err) { struct rspamd_control_session *session = conn->ud; + struct rspamd_main *rspamd_main; + + rspamd_main = session->rspamd_main; if (!session->is_reply) { - msg_info ("abnormally closing control connection: %e", err); + msg_info_main ("abnormally closing control connection: %e", err); session->is_reply = TRUE; rspamd_control_send_error (session, err->code, "%s", err->message); } @@ -437,7 +444,7 @@ rspamd_control_broadcast_cmd (struct rspamd_main *rspamd_main, DL_APPEND (res, rep_elt); } else { - msg_err ("cannot write command %d(%z) to the worker %P(%s), fd: %d: %s", + msg_err_main ("cannot write command %d(%z) to the worker %P(%s), fd: %d: %s", (int)cmd->type, iov.iov_len, wrk->pid, g_quark_to_string (wrk->type), @@ -504,23 +511,24 @@ rspamd_control_finish_handler (struct rspamd_http_connection *conn, void rspamd_control_process_client_socket (struct rspamd_main *rspamd_main, - gint fd) + gint fd, rspamd_inet_addr_t *addr) { struct rspamd_control_session *session; session = g_malloc0 (sizeof (*session)); session->fd = fd; - session->conn = rspamd_http_connection_new (NULL, + session->conn = rspamd_http_connection_new (rspamd_main->http_ctx, + fd, + NULL, rspamd_control_error_handler, rspamd_control_finish_handler, 0, - RSPAMD_HTTP_SERVER, - NULL, - NULL); + RSPAMD_HTTP_SERVER); session->rspamd_main = rspamd_main; - rspamd_http_connection_read_message (session->conn, session, session->fd, - &io_timeout, rspamd_main->ev_base); + session->addr = addr; + rspamd_http_connection_read_message (session->conn, session, + &io_timeout); } struct rspamd_worker_control_data { @@ -543,14 +551,16 @@ rspamd_control_default_cmd_handler (gint fd, gssize r; struct rusage rusg; struct rspamd_config *cfg; + struct rspamd_main *rspamd_main; memset (&rep, 0, sizeof (rep)); rep.type = cmd->type; + rspamd_main = cd->worker->srv; switch (cmd->type) { case RSPAMD_CONTROL_STAT: if (getrusage (RUSAGE_SELF, &rusg) == -1) { - msg_err ("cannot get rusage stats: %s", + msg_err_main ("cannot get rusage stats: %s", strerror (errno)); } else { @@ -594,7 +604,7 @@ rspamd_control_default_cmd_handler (gint fd, r = write (fd, &rep, sizeof (rep)); if (r != sizeof (rep)) { - msg_err ("cannot write reply to the control socket: %s", + msg_err_main ("cannot write reply to the control socket: %s", strerror (errno)); } diff --git a/src/libserver/rspamd_control.h b/src/libserver/rspamd_control.h index bc42b662e..ec3962c64 100644 --- a/src/libserver/rspamd_control.h +++ b/src/libserver/rspamd_control.h @@ -193,7 +193,7 @@ typedef void (*rspamd_srv_reply_handler) (struct rspamd_worker *worker, * Process client socket connection */ void rspamd_control_process_client_socket (struct rspamd_main *rspamd_main, - gint fd); + gint fd, rspamd_inet_addr_t *addr); /** * Register default handlers for a worker diff --git a/src/libserver/rspamd_symcache.c b/src/libserver/rspamd_symcache.c index d02638add..3cfb15408 100644 --- a/src/libserver/rspamd_symcache.c +++ b/src/libserver/rspamd_symcache.c @@ -23,6 +23,7 @@ #include "unix-std.h" #include "contrib/t1ha/t1ha.h" #include "libserver/worker_util.h" +#include "khash.h" #include <math.h> #if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L @@ -81,51 +82,6 @@ struct symcache_order { ref_entry_t ref; }; -struct rspamd_symcache { - /* Hash table for fast access */ - GHashTable *items_by_symbol; - GPtrArray *items_by_id; - struct symcache_order *items_by_order; - GPtrArray *filters; - GPtrArray *prefilters; - GPtrArray *postfilters; - GPtrArray *composites; - GPtrArray *idempotent; - GPtrArray *virtual; - GPtrArray *squeezed; - GList *delayed_deps; - GList *delayed_conditions; - rspamd_mempool_t *static_pool; - guint64 cksum; - gdouble total_weight; - guint used_items; - guint stats_symbols_count; - guint64 total_hits; - guint id; - struct rspamd_config *cfg; - gdouble reload_time; - gint peak_cb; -}; - -struct item_stat { - struct rspamd_counter_data time_counter; - gdouble avg_time; - gdouble weight; - guint hits; - guint64 total_hits; - struct rspamd_counter_data frequency_counter; - gdouble avg_frequency; - gdouble stddev_frequency; -}; - -struct rspamd_symcache_dynamic_item { - guint16 start_msec; /* Relative to task time */ - unsigned started:1; - unsigned finished:1; - /* unsigned pad:14; */ - guint32 async_events; -}; - struct rspamd_symcache_item { /* This block is likely shared */ struct item_stat *st; @@ -165,6 +121,53 @@ struct rspamd_symcache_item { GPtrArray *rdeps; }; +struct item_stat { + struct rspamd_counter_data time_counter; + gdouble avg_time; + gdouble weight; + guint hits; + guint64 total_hits; + struct rspamd_counter_data frequency_counter; + gdouble avg_frequency; + gdouble stddev_frequency; +}; + +struct rspamd_symcache { + /* Hash table for fast access */ + GHashTable *items_by_symbol; + GPtrArray *items_by_id; + struct symcache_order *items_by_order; + GPtrArray *filters; + GPtrArray *prefilters; + GPtrArray *postfilters; + GPtrArray *composites; + GPtrArray *idempotent; + GPtrArray *virtual; + GPtrArray *squeezed; + GList *delayed_deps; + GList *delayed_conditions; + rspamd_mempool_t *static_pool; + guint64 cksum; + gdouble total_weight; + guint used_items; + guint stats_symbols_count; + guint64 total_hits; + guint id; + struct rspamd_config *cfg; + gdouble reload_time; + gint peak_cb; +}; + +struct rspamd_symcache_dynamic_item { + guint16 start_msec; /* Relative to task time */ + unsigned started:1; + unsigned finished:1; + /* unsigned pad:14; */ + guint32 async_events; +}; + + + struct cache_dependency { struct rspamd_symcache_item *item; gchar *sym; @@ -529,24 +532,31 @@ rspamd_symcache_post_init (struct rspamd_symcache *cache) dit = rspamd_symcache_find_filter (cache, dep->sym); if (dit != NULL) { - if (dit->id == i) { - msg_err_cache ("cannot add dependency on self: %s -> %s " - "(resolved to %s)", - it->symbol, dep->sym, dit->symbol); + if (!dit->is_filter) { + msg_err_cache ("cannot depend on non filter symbol " + "(%s wants to add dependency on %s)", + dep->sym, dit->symbol); } else { - rdep = rspamd_mempool_alloc (cache->static_pool, - sizeof (*rdep)); - - rdep->sym = dep->sym; - rdep->item = it; - rdep->id = i; - g_ptr_array_add (dit->rdeps, rdep); - dep->item = dit; - dep->id = dit->id; - - msg_debug_cache ("add dependency from %d on %d", it->id, - dit->id); + if (dit->id == i) { + msg_err_cache ("cannot add dependency on self: %s -> %s " + "(resolved to %s)", + it->symbol, dep->sym, dit->symbol); + } else { + rdep = rspamd_mempool_alloc (cache->static_pool, + sizeof (*rdep)); + + rdep->sym = dep->sym; + rdep->item = it; + rdep->id = i; + g_assert (dit->rdeps != NULL); + g_ptr_array_add (dit->rdeps, rdep); + dep->item = dit; + dep->id = dit->id; + + msg_debug_cache ("add dependency from %d on %d", it->id, + dit->id); + } } } else { @@ -921,6 +931,8 @@ rspamd_symcache_add_symbol (struct rspamd_symcache *cache, */ if (item->type & SYMBOL_TYPE_COMPOSITE) { item->specific.normal.condition_cb = -1; + item->specific.normal.user_data = user_data; + g_assert (user_data != NULL); g_ptr_array_add (cache->composites, item); item->id = cache->items_by_id->len; @@ -2834,4 +2846,26 @@ rspamd_symcache_get_symbol_flags (struct rspamd_symcache *cache, } return 0; +} + +void +rspamd_symcache_composites_foreach (struct rspamd_task *task, + struct rspamd_symcache *cache, + GHFunc func, + gpointer fd) +{ + guint i; + struct rspamd_symcache_item *item; + struct rspamd_symcache_dynamic_item *dyn_item; + + PTR_ARRAY_FOREACH (cache->composites, i, item) { + dyn_item = rspamd_symcache_get_dynamic (task->checkpoint, item); + + if (!CHECK_START_BIT (task->checkpoint, dyn_item)) { + /* Cannot do it due to 2 passes */ + /* SET_START_BIT (task->checkpoint, dyn_item); */ + func (item->symbol, item->specific.normal.user_data, fd); + SET_FINISH_BIT (task->checkpoint, dyn_item); + } + } }
\ No newline at end of file diff --git a/src/libserver/rspamd_symcache.h b/src/libserver/rspamd_symcache.h index 965895221..a038d6a9d 100644 --- a/src/libserver/rspamd_symcache.h +++ b/src/libserver/rspamd_symcache.h @@ -50,6 +50,7 @@ enum rspamd_symbol_type { SYMBOL_TYPE_MIME_ONLY = (1 << 15), /* Symbol is mime only */ SYMBOL_TYPE_EXPLICIT_DISABLE = (1 << 16), /* Symbol should be disabled explicitly only */ SYMBOL_TYPE_IGNORE_PASSTHROUGH = (1 << 17), /* Symbol ignores passthrough result */ + SYMBOL_TYPE_USE_CORO = (1 << 18), /* Symbol uses lua coroutines */ }; /** @@ -402,4 +403,16 @@ gboolean rspamd_symcache_item_async_dec_check_full (struct rspamd_task *task, void rspamd_symcache_disable_all_symbols (struct rspamd_task *task, struct rspamd_symcache *cache, guint skip_mask); + +/** + * Iterates over the list of the enabled composites calling specified function + * @param task + * @param cache + * @param func + * @param fd + */ +void rspamd_symcache_composites_foreach (struct rspamd_task *task, + struct rspamd_symcache *cache, + GHFunc func, + gpointer fd); #endif diff --git a/src/libserver/task.h b/src/libserver/task.h index 93e0ae0e8..684f5c2c0 100644 --- a/src/libserver/task.h +++ b/src/libserver/task.h @@ -17,7 +17,7 @@ #define TASK_H_ #include "config.h" -#include "http.h" +#include "http_connection.h" #include "events.h" #include "util.h" #include "mem_pool.h" diff --git a/src/libserver/url.c b/src/libserver/url.c index 421c8a181..f860eec0c 100644 --- a/src/libserver/url.c +++ b/src/libserver/url.c @@ -46,6 +46,7 @@ #include "message.h" #include "multipattern.h" #include "contrib/uthash/utlist.h" +#include "contrib/http-parser/http_parser.h" #include <unicode/utf8.h> #include <unicode/uchar.h> @@ -103,6 +104,11 @@ static const struct { .len = 3 }, { + .proto = PROTOCOL_TELEPHONE, + .name = "callto", + .len = 3 + }, + { .proto = PROTOCOL_UNKNOWN, .name = NULL, .len = 0 @@ -192,7 +198,7 @@ struct url_matcher static_matchers[] = { 0, 0}, {"mailto:", "", url_email_start, url_email_end, 0, 0}, - {"callto://", "", url_web_start, url_web_end, + {"callto:", "", url_tel_start, url_tel_end, 0, 0}, {"h323:", "", url_web_start, url_web_end, 0, 0}, @@ -454,7 +460,7 @@ rspamd_url_parse_tld_file (const gchar *fname, m.flags = flags; rspamd_multipattern_add_pattern (url_scanner->search_trie, p, - RSPAMD_MULTIPATTERN_TLD | RSPAMD_MULTIPATTERN_ICASE); + RSPAMD_MULTIPATTERN_TLD|RSPAMD_MULTIPATTERN_ICASE|RSPAMD_MULTIPATTERN_UTF8); m.pattern = rspamd_multipattern_get_pattern (url_scanner->search_trie, rspamd_multipattern_get_npatterns (url_scanner->search_trie) - 1); m.patlen = strlen (m.pattern); @@ -517,13 +523,13 @@ rspamd_url_init (const gchar *tld_file) url_scanner->matchers = g_array_sized_new (FALSE, TRUE, sizeof (struct url_matcher), 13000); url_scanner->search_trie = rspamd_multipattern_create_sized (13000, - RSPAMD_MULTIPATTERN_TLD | RSPAMD_MULTIPATTERN_ICASE); + RSPAMD_MULTIPATTERN_TLD|RSPAMD_MULTIPATTERN_ICASE|RSPAMD_MULTIPATTERN_UTF8); } else { url_scanner->matchers = g_array_sized_new (FALSE, TRUE, sizeof (struct url_matcher), 128); url_scanner->search_trie = rspamd_multipattern_create_sized (128, - RSPAMD_MULTIPATTERN_TLD | RSPAMD_MULTIPATTERN_ICASE); + RSPAMD_MULTIPATTERN_TLD|RSPAMD_MULTIPATTERN_ICASE|RSPAMD_MULTIPATTERN_UTF8); } rspamd_url_add_static_matchers (url_scanner); @@ -538,8 +544,10 @@ rspamd_url_init (const gchar *tld_file) g_error_free (err); } - msg_debug ("initialized trie of %ud elements", - url_scanner->matchers->len); + if (tld_file != NULL) { + msg_info ("initialized %ud url tld suffixes from '%s'", + url_scanner->matchers->len, tld_file); + } } #define SET_U(u, field) do { \ @@ -736,6 +744,144 @@ rspamd_mailto_parse (struct http_parser_url *u, } static gint +rspamd_telephone_parse (struct http_parser_url *u, + const gchar *str, gsize len, + gchar const **end, + enum rspamd_url_parse_flags parse_flags, + guint *flags) +{ + enum { + parse_protocol, + parse_semicolon, + parse_slash, + parse_slash_slash, + parse_spaces, + parse_plus, + parse_phone_start, + parse_phone, + } st = parse_protocol; + + const gchar *p = str, *c = str, *last = str + len; + gchar t; + gint ret = 1, i; + UChar32 uc; + + if (u != NULL) { + memset (u, 0, sizeof (*u)); + } + + while (p < last) { + t = *p; + + switch (st) { + case parse_protocol: + if (t == ':') { + st = parse_semicolon; + SET_U (u, UF_SCHEMA); + } + p++; + break; + case parse_semicolon: + if (t == '/' || t == '\\') { + st = parse_slash; + p++; + } + else { + st = parse_slash_slash; + } + break; + case parse_slash: + if (t == '/' || t == '\\') { + st = parse_slash_slash; + } + else { + goto out; + } + p++; + break; + case parse_slash_slash: + if (g_ascii_isspace (t)) { + st = parse_spaces; + p++; + } + else if (t == '+') { + c = p; + st = parse_plus; + } + else if (t == '/') { + /* Skip multiple slashes */ + p++; + } + else { + st = parse_phone_start; + c = p; + } + break; + case parse_spaces: + if (t == '+') { + c = p; + st = parse_plus; + } + else if (!g_ascii_isspace (t)) { + st = parse_phone_start; + c = p; + } + else { + p ++; + } + break; + case parse_plus: + c = p; + p ++; + st = parse_phone_start; + break; + case parse_phone_start: + if (*p == '%' || *p == '(' || g_ascii_isdigit (*p)) { + st = parse_phone; + p ++; + } + else { + goto out; + } + break; + case parse_phone: + i = p - str; + U8_NEXT (str, i, len, uc); + p = str + i; + + if (u_isdigit (uc) || uc == '(' || uc == ')' || uc == '[' || uc == ']' + || u_isspace (uc) || uc == '%') { + p ++; + } + else if (uc <= 0 || is_url_end (uc)) { + ret = 0; + goto set; + } + break; + } + } + + set: + if (st == parse_phone) { + if (p - c != 0) { + SET_U (u, UF_HOST); + ret = 0; + } + } + + out: + if (end != NULL) { + *end = p; + } + + if ((parse_flags & RSPAMD_URL_PARSE_CHECK)) { + return 0; + } + + return ret; +} + +static gint rspamd_web_parse (struct http_parser_url *u, const gchar *str, gsize len, gchar const **end, enum rspamd_url_parse_flags parse_flags, @@ -1020,11 +1166,13 @@ rspamd_web_parse (struct http_parser_url *u, const gchar *str, gsize len, (*flags) |= RSPAMD_URL_FLAG_OBSCURED; } else { - if (!(parse_flags & RSPAMD_URL_PARSE_CHECK)) { - goto out; - } - else { - goto set; + if (!u_isgraph (uc)) { + if (!(parse_flags & RSPAMD_URL_PARSE_CHECK)) { + goto out; + } + else { + goto set; + } } } } @@ -1240,7 +1388,7 @@ rspamd_web_parse (struct http_parser_url *u, const gchar *str, gsize len, /* Parse remaining */ switch (st) { case parse_domain: - if (p - c == 0) { + if (p - c == 0 || !is_domain (*(p - 1)) || !is_domain (*c)) { goto out; } SET_U (u, UF_HOST); @@ -1633,6 +1781,40 @@ rspamd_url_shift (struct rspamd_url *uri, gsize nlen, } } +static void +rspamd_telephone_normalise_inplace (struct rspamd_url *uri) +{ + gchar *t, *h, *end; + gint i = 0, w, orig_len; + UChar32 uc; + + t = uri->host; + h = t; + end = uri->host + uri->hostlen; + orig_len = uri->hostlen; + + if (*h == '+') { + h ++; + t ++; + } + + while (h < end) { + i = 0; + U8_NEXT (h, i, end - h, uc); + + if (u_isdigit (uc)) { + w = 0; + U8_APPEND_UNSAFE (t, w, uc); + t += w; + } + + h += i; + } + + uri->hostlen = t - uri->host; + uri->urllen -= (orig_len - uri->hostlen); +} + enum uri_errno rspamd_url_parse (struct rspamd_url *uri, gchar *uristring, gsize len, @@ -1654,6 +1836,7 @@ rspamd_url_parse (struct rspamd_url *uri, } p = uristring; + uri->protocol = PROTOCOL_UNKNOWN; if (len > sizeof ("mailto:") - 1) { /* For mailto: urls we also need to add slashes to make it a valid URL */ @@ -1661,75 +1844,11 @@ rspamd_url_parse (struct rspamd_url *uri, ret = rspamd_mailto_parse (&u, uristring, len, &end, parse_flags, &flags); } - else if (g_ascii_strncasecmp (p, "tel:", sizeof ("tel:") - 1) == 0) { - /* Telephone url */ - gint nlen = 0; - gboolean has_plus = FALSE; - end = p + len; - gchar *t, *tend; - UChar32 uc; - - uri->raw = p; - uri->rawlen = len; - uri->string = rspamd_mempool_alloc (pool, len + 1); - t = uri->string; - tend = t + len; - i = 4; - - memcpy (t, "tel:", 4); - t += 4; - p += 4; - nlen = 4; - - if (*p == '+') { - has_plus = TRUE; - *t++ = *p++; - nlen ++; - i ++; - } - - while (t < tend && i < len) { - U8_NEXT (uristring, i, len, uc); - - if (u_isdigit (uc)) { - if (g_ascii_isdigit (uc)) { - *t++ = uc; - nlen ++; - } - else { - /* Obfuscated number */ - uri->flags |= RSPAMD_URL_FLAG_OBSCURED; - } - } - else if (IS_OBSCURED_CHAR (uc)) { - uri->flags |= RSPAMD_URL_FLAG_OBSCURED; - } - } - - *t = '\0'; - - if (rspamd_normalise_unicode_inplace (pool, uri->string, &nlen)) { - uri->flags |= RSPAMD_URL_FLAG_UNNORMALISED; - } - - uri->urllen = nlen; - + else if (g_ascii_strncasecmp (p, "tel:", sizeof ("tel:") - 1) == 0 || + g_ascii_strncasecmp (p, "callto:", sizeof ("callto:") - 1) == 0) { + ret = rspamd_telephone_parse (&u, uristring, len, &end, parse_flags, + &flags); uri->protocol = PROTOCOL_TELEPHONE; - uri->protocollen = 4; - - uri->host = uri->string + 4; - uri->hostlen = nlen - 4; - - if (has_plus) { - uri->tld = uri->string + 5; - uri->tldlen = nlen - 5; - } - else { - uri->tld = uri->string + 4; - uri->tldlen = nlen - 4; - } - - return URI_ERRNO_OK; } else { ret = rspamd_web_parse (&u, uristring, len, &end, parse_flags, @@ -1862,32 +1981,46 @@ rspamd_url_parse (struct rspamd_url *uri, rspamd_str_lc (uri->string, uri->protocollen); rspamd_str_lc_utf8 (uri->host, uri->hostlen); - uri->protocol = PROTOCOL_UNKNOWN; - - for (i = 0; i < G_N_ELEMENTS (rspamd_url_protocols); i++) { - if (uri->protocollen == rspamd_url_protocols[i].len) { - if (memcmp (uri->string, rspamd_url_protocols[i].name, uri->protocollen) == - 0) { - uri->protocol = i; - break; + if (uri->protocol == PROTOCOL_UNKNOWN) { + for (i = 0; i < G_N_ELEMENTS (rspamd_url_protocols); i++) { + if (uri->protocollen == rspamd_url_protocols[i].len) { + if (memcmp (uri->string, + rspamd_url_protocols[i].name, uri->protocollen) == 0) { + uri->protocol = rspamd_url_protocols[i].proto; + break; + } } } } - /* Find TLD part */ - rspamd_multipattern_lookup (url_scanner->search_trie, + if (uri->protocol & (PROTOCOL_HTTP|PROTOCOL_HTTPS|PROTOCOL_MAILTO|PROTOCOL_FTP)) { + /* Find TLD part */ + rspamd_multipattern_lookup (url_scanner->search_trie, uri->host, uri->hostlen, rspamd_tld_trie_callback, uri, NULL); - if (uri->tldlen == 0) { - if (!(parse_flags & RSPAMD_URL_PARSE_HREF)) { - /* Ignore URL's without TLD if it is not a numeric URL */ - if (!rspamd_url_is_ip (uri, pool)) { - return URI_ERRNO_TLD_MISSING; + if (uri->tldlen == 0) { + if (!(parse_flags & RSPAMD_URL_PARSE_HREF)) { + /* Ignore URL's without TLD if it is not a numeric URL */ + if (!rspamd_url_is_ip (uri, pool)) { + return URI_ERRNO_TLD_MISSING; + } + } else { + /* Assume tld equal to host */ + uri->tld = uri->host; + uri->tldlen = uri->hostlen; } } + } + else if (uri->protocol & PROTOCOL_TELEPHONE) { + /* We need to normalise phone number: remove all spaces and braces */ + rspamd_telephone_normalise_inplace (uri); + + if (uri->host[0] == '+') { + uri->tld = uri->host + 1; + uri->tldlen = uri->hostlen - 1; + } else { - /* Assume tld equal to host */ uri->tld = uri->host; uri->tldlen = uri->hostlen; } @@ -1899,7 +2032,7 @@ rspamd_url_parse (struct rspamd_url *uri, } else { /* Hack, hack, hack */ - uri->protocol = PROTOCOL_HTTP; + uri->protocol = PROTOCOL_UNKNOWN; } } @@ -2366,13 +2499,15 @@ url_tel_start (struct url_callback_data *cb, const gchar *pos, url_match_t *match) { - if (!(*pos == '+' || g_ascii_isdigit (*pos))) { - /* Urls cannot start with . */ - return FALSE; - } - match->m_begin = pos; + if (pos >= cb->begin + 1) { + match->st = *(pos - 1); + } + else { + match->st = '\0'; + } + return TRUE; } @@ -2381,29 +2516,26 @@ url_tel_end (struct url_callback_data *cb, const gchar *pos, url_match_t *match) { - UChar32 uc; - gint len = cb->end - pos, i = 0; + const gchar *last = NULL; + struct http_parser_url u; + gint len = cb->end - pos; + guint flags = 0; if (match->newline_pos && match->st != '<') { /* We should also limit our match end to the newline */ len = MIN (len, match->newline_pos - pos); } - while (i < len) { - U8_NEXT (pos, i, len, uc); - - if (uc < 0) { - break; - } + if (rspamd_telephone_parse (&u, pos, len, &last, + RSPAMD_URL_PARSE_CHECK, &flags) != 0) { + return FALSE; + } - if (!(u_isdigit (uc) || u_isspace (uc) || - IS_OBSCURED_CHAR (uc) || uc == '+' || - uc == '-' || uc == '.')) { - break; - } + if (!(u.field_set & (1 << UF_HOST))) { + return FALSE; } - match->m_len = i; + match->m_len = (last - pos); return TRUE; } @@ -3283,10 +3415,11 @@ rspamd_url_encode (struct rspamd_url *url, gsize *pdlen, d = dest; dend = d + dlen; - if (url->protocollen > 0 && - (url->protocol >= 0 && url->protocol < G_N_ELEMENTS (rspamd_url_protocols))) { + if (url->protocollen > 0) { d += rspamd_snprintf ((gchar *) d, dend - d, - "%*s://", url->protocollen, rspamd_url_protocols[url->protocol].name); + "%*s://", + url->protocollen, + rspamd_url_protocol_name (url->protocol)); } else { d += rspamd_snprintf ((gchar *) d, dend - d, "http://"); @@ -3329,3 +3462,61 @@ rspamd_url_is_domain (int c) { return is_domain ((guchar)c); } + +const gchar* +rspamd_url_protocol_name (enum rspamd_url_protocol proto) +{ + const gchar *ret = "unknown"; + + switch (proto) { + case PROTOCOL_HTTP: + ret = "http"; + break; + case PROTOCOL_HTTPS: + ret = "https"; + break; + case PROTOCOL_FTP: + ret = "ftp"; + break; + case PROTOCOL_FILE: + ret = "file"; + break; + case PROTOCOL_MAILTO: + ret = "mailto"; + break; + case PROTOCOL_TELEPHONE: + ret = "telephone"; + break; + default: + break; + } + + return ret; +} + +enum rspamd_url_protocol +rspamd_url_protocol_from_string (const gchar *str) +{ + enum rspamd_url_protocol ret = PROTOCOL_UNKNOWN; + + if (strcmp (str, "http") == 0) { + ret = PROTOCOL_HTTP; + } + else if (strcmp (str, "https") == 0) { + ret = PROTOCOL_HTTPS; + } + else if (strcmp (str, "mailto") == 0) { + ret = PROTOCOL_MAILTO; + } + else if (strcmp (str, "ftp") == 0) { + ret = PROTOCOL_FTP; + } + else if (strcmp (str, "file") == 0) { + ret = PROTOCOL_FILE; + } + else if (strcmp (str, "telephone") == 0) { + ret = PROTOCOL_TELEPHONE; + } + + return ret; +} diff --git a/src/libserver/url.h b/src/libserver/url.h index 4c34be9e7..3deeb8cf5 100644 --- a/src/libserver/url.h +++ b/src/libserver/url.h @@ -79,13 +79,13 @@ enum uri_errno { }; enum rspamd_url_protocol { - PROTOCOL_FILE = 0, - PROTOCOL_FTP, - PROTOCOL_HTTP, - PROTOCOL_HTTPS, - PROTOCOL_MAILTO, - PROTOCOL_TELEPHONE, - PROTOCOL_UNKNOWN + PROTOCOL_FILE = 1u << 0, + PROTOCOL_FTP = 1u << 1, + PROTOCOL_HTTP = 1u << 2, + PROTOCOL_HTTPS = 1u << 3, + PROTOCOL_MAILTO = 1u << 4, + PROTOCOL_TELEPHONE = 1u << 5, + PROTOCOL_UNKNOWN = 1u << 31, }; /** @@ -240,4 +240,18 @@ const gchar * rspamd_url_encode (struct rspamd_url *url, gsize *dlen, */ gboolean rspamd_url_is_domain (int c); +/** + * Returns symbolic name for protocol + * @param proto + * @return + */ +const gchar* rspamd_url_protocol_name (enum rspamd_url_protocol proto); + + +/** + * Converts string to a numeric protocol + * @param str + * @return + */ +enum rspamd_url_protocol rspamd_url_protocol_from_string (const gchar *str); #endif diff --git a/src/libserver/worker_util.c b/src/libserver/worker_util.c index a0e511929..06296bba2 100644 --- a/src/libserver/worker_util.c +++ b/src/libserver/worker_util.c @@ -24,6 +24,7 @@ #include "libutil/map.h" #include "libutil/map_private.h" #include "libutil/http_private.h" +#include "libutil/http_router.h" #ifdef WITH_GPERF_TOOLS #include <gperftools/profiler.h> @@ -434,9 +435,7 @@ rspamd_controller_send_error (struct rspamd_http_connection_entry *entry, NULL, "application/json", entry, - entry->conn->fd, - entry->rt->ptv, - entry->rt->ev_base); + entry->rt->ptv); entry->is_reply = TRUE; } @@ -468,9 +467,7 @@ rspamd_controller_send_string (struct rspamd_http_connection_entry *entry, NULL, "application/json", entry, - entry->conn->fd, - entry->rt->ptv, - entry->rt->ev_base); + entry->rt->ptv); entry->is_reply = TRUE; } @@ -496,9 +493,7 @@ rspamd_controller_send_ucl (struct rspamd_http_connection_entry *entry, NULL, "application/json", entry, - entry->conn->fd, - entry->rt->ptv, - entry->rt->ev_base); + entry->rt->ptv); entry->is_reply = TRUE; } @@ -512,12 +507,14 @@ rspamd_worker_drop_priv (struct rspamd_main *rspamd_main) strerror (errno)); exit (-errno); } + if (rspamd_main->cfg->rspamd_user && - initgroups (rspamd_main->cfg->rspamd_user, rspamd_main->workers_gid) == - -1) { + initgroups (rspamd_main->cfg->rspamd_user, + rspamd_main->workers_gid) == -1) { msg_err_main ("initgroups failed (%s), aborting", strerror (errno)); exit (-errno); } + if (setuid (rspamd_main->workers_uid) == -1) { msg_err_main ("cannot setuid to %d (%s), aborting", (gint) rspamd_main->workers_uid, diff --git a/src/libserver/worker_util.h b/src/libserver/worker_util.h index dbcc8f8a2..3186025b3 100644 --- a/src/libserver/worker_util.h +++ b/src/libserver/worker_util.h @@ -18,7 +18,7 @@ #include "config.h" #include "util.h" -#include "http.h" +#include "http_connection.h" #include "rspamd.h" #ifndef HAVE_SA_SIGINFO diff --git a/src/libstat/backends/redis_backend.c b/src/libstat/backends/redis_backend.c index d7402db98..d54767c12 100644 --- a/src/libstat/backends/redis_backend.c +++ b/src/libstat/backends/redis_backend.c @@ -975,7 +975,7 @@ rspamd_redis_async_stat_cb (struct rspamd_stat_async_elt *elt, gpointer d) 0); g_assert (cbdata->selected != NULL); - addr = rspamd_upstream_addr (cbdata->selected); + addr = rspamd_upstream_addr_next (cbdata->selected); g_assert (addr != NULL); if (rspamd_inet_address_get_af (addr) == AF_UNIX) { @@ -1522,7 +1522,7 @@ rspamd_redis_runtime (struct rspamd_task *task, rt->stcf = stcf; rt->redis_object_expanded = object_expanded; - addr = rspamd_upstream_addr (up); + addr = rspamd_upstream_addr_next (up); g_assert (addr != NULL); if (rspamd_inet_address_get_af (addr) == AF_UNIX) { @@ -1693,7 +1693,7 @@ rspamd_redis_learn_tokens (struct rspamd_task *task, GPtrArray *tokens, } } - addr = rspamd_upstream_addr (up); + addr = rspamd_upstream_addr_next (up); g_assert (addr != NULL); if (rspamd_inet_address_get_af (addr) == AF_UNIX) { diff --git a/src/libstat/learn_cache/redis_cache.c b/src/libstat/learn_cache/redis_cache.c index 6a0aa1da7..aea024e06 100644 --- a/src/libstat/learn_cache/redis_cache.c +++ b/src/libstat/learn_cache/redis_cache.c @@ -385,7 +385,7 @@ rspamd_stat_cache_redis_runtime (struct rspamd_task *task, rt->task = task; rt->ctx = ctx; - addr = rspamd_upstream_addr (up); + addr = rspamd_upstream_addr_next (up); g_assert (addr != NULL); if (rspamd_inet_address_get_af (addr) == AF_UNIX) { diff --git a/src/libstat/stat_api.h b/src/libstat/stat_api.h index 533c42948..f9d1aab5a 100644 --- a/src/libstat/stat_api.h +++ b/src/libstat/stat_api.h @@ -39,6 +39,7 @@ #define RSPAMD_STAT_TOKEN_FLAG_STOP_WORD (1u << 10) #define RSPAMD_STAT_TOKEN_FLAG_SKIPPED (1u << 11) #define RSPAMD_STAT_TOKEN_FLAG_INVISIBLE_SPACES (1u << 12) +#define RSPAMD_STAT_TOKEN_FLAG_EMOJI (1u << 13) typedef struct rspamd_stat_token_s { rspamd_ftok_t original; /* utf8 raw */ diff --git a/src/libstat/tokenizers/tokenizers.c b/src/libstat/tokenizers/tokenizers.c index acbbcf2f0..caa4a48a5 100644 --- a/src/libstat/tokenizers/tokenizers.c +++ b/src/libstat/tokenizers/tokenizers.c @@ -610,7 +610,25 @@ rspamd_uchars_to_ucs32 (const UChar *src, gsize srclen, U16_NEXT_UNSAFE (src, i, t); if (u_isgraph (t)) { - *d++ = u_tolower (t); + UCharCategory cat; + + cat = u_charType (t); +#if U_ICU_VERSION_MAJOR_NUM >= 57 + if (u_hasBinaryProperty (t, UCHAR_EMOJI)) { + tok->flags |= RSPAMD_STAT_TOKEN_FLAG_EMOJI; + } +#endif + + if (cat == U_UPPERCASE_LETTER || + cat == U_LOWERCASE_LETTER || + cat == U_DECIMAL_DIGIT_NUMBER || + cat == U_CONNECTOR_PUNCTUATION || + cat == U_MATH_SYMBOL || + cat == U_CURRENCY_SYMBOL || + cat == U_INITIAL_PUNCTUATION || + cat == U_FINAL_PUNCTUATION) { + *d++ = u_tolower (t); + } } else { /* Invisible spaces ! */ diff --git a/src/libutil/CMakeLists.txt b/src/libutil/CMakeLists.txt index aab754195..f86d650f0 100644 --- a/src/libutil/CMakeLists.txt +++ b/src/libutil/CMakeLists.txt @@ -1,27 +1,31 @@ # Librspamd-util -SET(LIBRSPAMDUTILSRC - ${CMAKE_CURRENT_SOURCE_DIR}/addr.c - ${CMAKE_CURRENT_SOURCE_DIR}/aio_event.c - ${CMAKE_CURRENT_SOURCE_DIR}/bloom.c - ${CMAKE_CURRENT_SOURCE_DIR}/expression.c - ${CMAKE_CURRENT_SOURCE_DIR}/fstring.c - ${CMAKE_CURRENT_SOURCE_DIR}/hash.c - ${CMAKE_CURRENT_SOURCE_DIR}/http.c - ${CMAKE_CURRENT_SOURCE_DIR}/logger.c - ${CMAKE_CURRENT_SOURCE_DIR}/map.c - ${CMAKE_CURRENT_SOURCE_DIR}/map_helpers.c - ${CMAKE_CURRENT_SOURCE_DIR}/mem_pool.c - ${CMAKE_CURRENT_SOURCE_DIR}/printf.c - ${CMAKE_CURRENT_SOURCE_DIR}/radix.c - ${CMAKE_CURRENT_SOURCE_DIR}/regexp.c - ${CMAKE_CURRENT_SOURCE_DIR}/rrd.c - ${CMAKE_CURRENT_SOURCE_DIR}/shingles.c - ${CMAKE_CURRENT_SOURCE_DIR}/sqlite_utils.c - ${CMAKE_CURRENT_SOURCE_DIR}/str_util.c - ${CMAKE_CURRENT_SOURCE_DIR}/upstream.c - ${CMAKE_CURRENT_SOURCE_DIR}/util.c - ${CMAKE_CURRENT_SOURCE_DIR}/heap.c - ${CMAKE_CURRENT_SOURCE_DIR}/multipattern.c - ${CMAKE_CURRENT_SOURCE_DIR}/ssl_util.c) +SET(LIBRSPAMDUTILSRC + ${CMAKE_CURRENT_SOURCE_DIR}/addr.c + ${CMAKE_CURRENT_SOURCE_DIR}/aio_event.c + ${CMAKE_CURRENT_SOURCE_DIR}/bloom.c + ${CMAKE_CURRENT_SOURCE_DIR}/expression.c + ${CMAKE_CURRENT_SOURCE_DIR}/fstring.c + ${CMAKE_CURRENT_SOURCE_DIR}/hash.c + ${CMAKE_CURRENT_SOURCE_DIR}/http_util.c + ${CMAKE_CURRENT_SOURCE_DIR}/http_message.c + ${CMAKE_CURRENT_SOURCE_DIR}/http_connection.c + ${CMAKE_CURRENT_SOURCE_DIR}/http_router.c + ${CMAKE_CURRENT_SOURCE_DIR}/http_context.c + ${CMAKE_CURRENT_SOURCE_DIR}/logger.c + ${CMAKE_CURRENT_SOURCE_DIR}/map.c + ${CMAKE_CURRENT_SOURCE_DIR}/map_helpers.c + ${CMAKE_CURRENT_SOURCE_DIR}/mem_pool.c + ${CMAKE_CURRENT_SOURCE_DIR}/printf.c + ${CMAKE_CURRENT_SOURCE_DIR}/radix.c + ${CMAKE_CURRENT_SOURCE_DIR}/regexp.c + ${CMAKE_CURRENT_SOURCE_DIR}/rrd.c + ${CMAKE_CURRENT_SOURCE_DIR}/shingles.c + ${CMAKE_CURRENT_SOURCE_DIR}/sqlite_utils.c + ${CMAKE_CURRENT_SOURCE_DIR}/str_util.c + ${CMAKE_CURRENT_SOURCE_DIR}/upstream.c + ${CMAKE_CURRENT_SOURCE_DIR}/util.c + ${CMAKE_CURRENT_SOURCE_DIR}/heap.c + ${CMAKE_CURRENT_SOURCE_DIR}/multipattern.c + ${CMAKE_CURRENT_SOURCE_DIR}/ssl_util.c) # Rspamdutil SET(RSPAMD_UTIL ${LIBRSPAMDUTILSRC} PARENT_SCOPE)
\ No newline at end of file diff --git a/src/libutil/addr.c b/src/libutil/addr.c index fd4621aef..7fa634f6f 100644 --- a/src/libutil/addr.c +++ b/src/libutil/addr.c @@ -1636,7 +1636,7 @@ rspamd_inet_address_af_order (const rspamd_inet_addr_t *addr) gint rspamd_inet_address_compare (const rspamd_inet_addr_t *a1, - const rspamd_inet_addr_t *a2) + const rspamd_inet_addr_t *a2, gboolean compare_ports) { g_assert (a1 != NULL); g_assert (a2 != NULL); @@ -1648,11 +1648,33 @@ rspamd_inet_address_compare (const rspamd_inet_addr_t *a1, else { switch (a1->af) { case AF_INET: - return memcmp (&a1->u.in.addr.s4.sin_addr, - &a2->u.in.addr.s4.sin_addr, sizeof (struct in_addr)); + if (!compare_ports) { + return memcmp (&a1->u.in.addr.s4.sin_addr, + &a2->u.in.addr.s4.sin_addr, sizeof (struct in_addr)); + } + else { + if (a1->u.in.addr.s4.sin_port == a2->u.in.addr.s4.sin_port) { + return memcmp (&a1->u.in.addr.s4.sin_addr, + &a2->u.in.addr.s4.sin_addr, sizeof (struct in_addr)); + } + else { + return a1->u.in.addr.s4.sin_port - a2->u.in.addr.s4.sin_port; + } + } case AF_INET6: - return memcmp (&a1->u.in.addr.s6.sin6_addr, - &a2->u.in.addr.s6.sin6_addr, sizeof (struct in6_addr)); + if (!compare_ports) { + return memcmp (&a1->u.in.addr.s6.sin6_addr, + &a2->u.in.addr.s6.sin6_addr, sizeof (struct in6_addr)); + } + else { + if (a1->u.in.addr.s6.sin6_port == a2->u.in.addr.s6.sin6_port) { + return memcmp (&a1->u.in.addr.s6.sin6_addr, + &a2->u.in.addr.s6.sin6_addr, sizeof (struct in6_addr)); + } + else { + return a1->u.in.addr.s6.sin6_port - a2->u.in.addr.s6.sin6_port; + } + } case AF_UNIX: return strncmp (a1->u.un->addr.sun_path, a2->u.un->addr.sun_path, sizeof (a1->u.un->addr.sun_path)); @@ -1671,7 +1693,7 @@ rspamd_inet_address_compare_ptr (gconstpointer a1, const rspamd_inet_addr_t **i1 = (const rspamd_inet_addr_t **)a1, **i2 = (const rspamd_inet_addr_t **)a2; - return rspamd_inet_address_compare (*i1, *i2); + return rspamd_inet_address_compare (*i1, *i2, FALSE); } rspamd_inet_addr_t * @@ -1724,28 +1746,85 @@ guint rspamd_inet_address_hash (gconstpointer a) { const rspamd_inet_addr_t *addr = a; - rspamd_cryptobox_fast_hash_state_t st; - - rspamd_cryptobox_fast_hash_init (&st, rspamd_hash_seed ()); - rspamd_cryptobox_fast_hash_update (&st, &addr->af, sizeof (addr->af)); + struct { + gchar buf[sizeof(struct in6_addr)]; /* 16 bytes */ + int af; + } layout; + gint32 k; if (addr->af == AF_UNIX && addr->u.un) { + rspamd_cryptobox_fast_hash_state_t st; + + rspamd_cryptobox_fast_hash_init (&st, rspamd_hash_seed ()); + rspamd_cryptobox_fast_hash_update (&st, &addr->af, sizeof (addr->af)); rspamd_cryptobox_fast_hash_update (&st, addr->u.un, sizeof (*addr->u.un)); + + return rspamd_cryptobox_fast_hash_final (&st); } else { + memset (&layout, 0, sizeof (layout)); + layout.af = addr->af; + /* We ignore port part here */ if (addr->af == AF_INET) { - rspamd_cryptobox_fast_hash_update (&st, &addr->u.in.addr.s4.sin_addr, + memcpy (layout.buf, &addr->u.in.addr.s4.sin_addr, + sizeof (addr->u.in.addr.s4.sin_addr)); + } + else { + memcpy (layout.buf, &addr->u.in.addr.s6.sin6_addr, + sizeof (addr->u.in.addr.s6.sin6_addr)); + } + + k = rspamd_cryptobox_fast_hash (&layout, sizeof (layout), + rspamd_hash_seed ()); + } + + return k; +} + +guint +rspamd_inet_address_port_hash (gconstpointer a) +{ + const rspamd_inet_addr_t *addr = a; + struct { + gchar buf[sizeof(struct in6_addr)]; /* 16 bytes */ + int port; + int af; + } layout; + + gint32 k; + + if (addr->af == AF_UNIX && addr->u.un) { + rspamd_cryptobox_fast_hash_state_t st; + + rspamd_cryptobox_fast_hash_init (&st, rspamd_hash_seed ()); + rspamd_cryptobox_fast_hash_update (&st, &addr->af, sizeof (addr->af)); + rspamd_cryptobox_fast_hash_update (&st, addr->u.un, sizeof (*addr->u.un)); + + return rspamd_cryptobox_fast_hash_final (&st); + } + else { + memset (&layout, 0, sizeof (layout)); + layout.af = addr->af; + + /* We consider port part here */ + if (addr->af == AF_INET) { + memcpy (layout.buf, &addr->u.in.addr.s4.sin_addr, sizeof (addr->u.in.addr.s4.sin_addr)); + layout.port = addr->u.in.addr.s4.sin_port; } else { - rspamd_cryptobox_fast_hash_update (&st, &addr->u.in.addr.s6.sin6_addr, + memcpy (layout.buf, &addr->u.in.addr.s6.sin6_addr, sizeof (addr->u.in.addr.s6.sin6_addr)); + layout.port = addr->u.in.addr.s6.sin6_port; } + + k = rspamd_cryptobox_fast_hash (&layout, sizeof (layout), + rspamd_hash_seed ()); } - return rspamd_cryptobox_fast_hash_final (&st); + return k; } gboolean @@ -1753,7 +1832,15 @@ rspamd_inet_address_equal (gconstpointer a, gconstpointer b) { const rspamd_inet_addr_t *a1 = a, *a2 = b; - return rspamd_inet_address_compare (a1, a2) == 0; + return rspamd_inet_address_compare (a1, a2, FALSE) == 0; +} + +gboolean +rspamd_inet_address_port_equal (gconstpointer a, gconstpointer b) +{ + const rspamd_inet_addr_t *a1 = a, *a2 = b; + + return rspamd_inet_address_compare (a1, a2, TRUE) == 0; } #ifndef IN6_IS_ADDR_LOOPBACK diff --git a/src/libutil/addr.h b/src/libutil/addr.h index 46b705a4b..bfe586ad1 100644 --- a/src/libutil/addr.h +++ b/src/libutil/addr.h @@ -263,7 +263,7 @@ void rspamd_inet_address_apply_mask (rspamd_inet_addr_t *addr, guint mask); * @return */ gint rspamd_inet_address_compare (const rspamd_inet_addr_t *a1, - const rspamd_inet_addr_t *a2); + const rspamd_inet_addr_t *a2, gboolean compare_ports); /** * Utility function to compare addresses by in g_ptr_array @@ -281,15 +281,16 @@ gint rspamd_inet_address_compare_ptr (gconstpointer a1, rspamd_inet_addr_t *rspamd_inet_address_copy (const rspamd_inet_addr_t *addr); /** - * Returns hash for inet address + * Returns hash for inet address (ignoring port) */ guint rspamd_inet_address_hash (gconstpointer a); - +guint rspamd_inet_address_port_hash (gconstpointer a); /** * Returns true if two address are equal */ gboolean rspamd_inet_address_equal (gconstpointer a, gconstpointer b); +gboolean rspamd_inet_address_port_equal (gconstpointer a, gconstpointer b); /** * Returns TRUE if an address belongs to some local address diff --git a/src/libutil/http.h b/src/libutil/http.h deleted file mode 100644 index df6f99756..000000000 --- a/src/libutil/http.h +++ /dev/null @@ -1,578 +0,0 @@ -/*- - * Copyright 2016 Vsevolod Stakhov - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef HTTP_H_ -#define HTTP_H_ - -/** - * @file http.h - * - * This is an interface for HTTP client and conn. This code uses HTTP parser written - * by Joyent Inc based on nginx code. - */ - -#include "config.h" -#include "http_parser.h" -#include "keypair.h" -#include "keypairs_cache.h" -#include "fstring.h" -#include "ref.h" - -enum rspamd_http_connection_type { - RSPAMD_HTTP_SERVER, - RSPAMD_HTTP_CLIENT -}; - -struct rspamd_http_header; -struct rspamd_http_message; -struct rspamd_http_connection_private; -struct rspamd_http_connection; -struct rspamd_http_connection_router; -struct rspamd_http_connection_entry; - -struct rspamd_storage_shmem { - gchar *shm_name; - ref_entry_t ref; -}; - -/** - * Legacy spamc protocol - */ -#define RSPAMD_HTTP_FLAG_SPAMC (1 << 0) -/** - * Store body of the message in a shared memory segment - */ -#define RSPAMD_HTTP_FLAG_SHMEM (1 << 2) -/** - * Store body of the message in an immutable shared memory segment - */ -#define RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE (1 << 3) -/** - * Use tls for this message - */ -#define RSPAMD_HTTP_FLAG_SSL (1 << 4) -/** - * Body has been set for a message - */ -#define RSPAMD_HTTP_FLAG_HAS_BODY (1 << 5) -/** - * Do not verify server's certificate - */ -#define RSPAMD_HTTP_FLAG_SSL_NOVERIFY (1 << 6) -/** - * Options for HTTP connection - */ -enum rspamd_http_options { - RSPAMD_HTTP_BODY_PARTIAL = 0x1, /**< Call body handler on all body data portions *///!< RSPAMD_HTTP_BODY_PARTIAL - RSPAMD_HTTP_CLIENT_SIMPLE = 0x1u << 1, /**< Read HTTP client reply automatically */ //!< RSPAMD_HTTP_CLIENT_SIMPLE - RSPAMD_HTTP_CLIENT_ENCRYPTED = 0x1u << 2, /**< Encrypt data for client */ //!< RSPAMD_HTTP_CLIENT_ENCRYPTED - RSPAMD_HTTP_CLIENT_SHARED = 0x1u << 3, /**< Store reply in shared memory */ //!< RSPAMD_HTTP_CLIENT_SHARED - RSPAMD_HTTP_REQUIRE_ENCRYPTION = 0x1u << 4 -}; - -typedef int (*rspamd_http_body_handler_t) (struct rspamd_http_connection *conn, - struct rspamd_http_message *msg, - const gchar *chunk, - gsize len); - -typedef void (*rspamd_http_error_handler_t) (struct rspamd_http_connection *conn, - GError *err); - -typedef int (*rspamd_http_finish_handler_t) (struct rspamd_http_connection *conn, - struct rspamd_http_message *msg); - -typedef int (*rspamd_http_router_handler_t) (struct rspamd_http_connection_entry - *conn_ent, - struct rspamd_http_message *msg); -typedef void (*rspamd_http_router_error_handler_t) (struct - rspamd_http_connection_entry *conn_ent, - GError *err); -typedef void (*rspamd_http_router_finish_handler_t) (struct - rspamd_http_connection_entry *conn_ent); - -/** - * HTTP connection structure - */ -struct rspamd_http_connection { - struct rspamd_http_connection_private *priv; - rspamd_http_body_handler_t body_handler; - rspamd_http_error_handler_t error_handler; - rspamd_http_finish_handler_t finish_handler; - struct rspamd_keypair_cache *cache; - gpointer ud; - gsize max_size; - unsigned opts; - enum rspamd_http_connection_type type; - gboolean finished; - gint fd; - gint ref; -}; - -struct rspamd_http_connection_entry { - struct rspamd_http_connection_router *rt; - struct rspamd_http_connection *conn; - gpointer ud; - gboolean is_reply; - gboolean support_gzip; - struct rspamd_http_connection_entry *prev, *next; -}; - -struct rspamd_http_connection_router { - struct rspamd_http_connection_entry *conns; - GHashTable *paths; - GHashTable *response_headers; - GPtrArray *regexps; - struct timeval tv; - struct timeval *ptv; - struct event_base *ev_base; - struct rspamd_keypair_cache *cache; - gchar *default_fs_path; - rspamd_http_router_handler_t unknown_method_handler; - struct rspamd_cryptobox_keypair *key; - rspamd_http_router_error_handler_t error_handler; - rspamd_http_router_finish_handler_t finish_handler; -}; - -/** - * Create new http connection - * @param handler_t handler_t for body - * @param opts options - * @return new connection structure - */ -struct rspamd_http_connection *rspamd_http_connection_new ( - rspamd_http_body_handler_t body_handler, - rspamd_http_error_handler_t error_handler, - rspamd_http_finish_handler_t finish_handler, - unsigned opts, - enum rspamd_http_connection_type type, - struct rspamd_keypair_cache *cache, - gpointer ssl_ctx); - - -/** - * Set key pointed by an opaque pointer - * @param conn connection structure - * @param key opaque key structure - */ -void rspamd_http_connection_set_key (struct rspamd_http_connection *conn, - struct rspamd_cryptobox_keypair *key); - -/** - * Get peer's public key - * @param conn connection structure - * @return pubkey structure or NULL - */ -const struct rspamd_cryptobox_pubkey* rspamd_http_connection_get_peer_key ( - struct rspamd_http_connection *conn); - -/** - * Returns TRUE if a connection is encrypted - * @param conn - * @return - */ -gboolean rspamd_http_connection_is_encrypted (struct rspamd_http_connection *conn); - -/** - * Handle a request using socket fd and user data ud - * @param conn connection structure - * @param ud opaque user data - * @param fd fd to read/write - */ -void rspamd_http_connection_read_message ( - struct rspamd_http_connection *conn, - gpointer ud, - gint fd, - struct timeval *timeout, - struct event_base *base); - -void rspamd_http_connection_read_message_shared ( - struct rspamd_http_connection *conn, - gpointer ud, - gint fd, - struct timeval *timeout, - struct event_base *base); - -/** - * Send reply using initialised connection - * @param conn connection structure - * @param msg HTTP message - * @param ud opaque user data - * @param fd fd to read/write - */ -void rspamd_http_connection_write_message ( - struct rspamd_http_connection *conn, - struct rspamd_http_message *msg, - const gchar *host, - const gchar *mime_type, - gpointer ud, - gint fd, - struct timeval *timeout, - struct event_base *base); - -void rspamd_http_connection_write_message_shared ( - struct rspamd_http_connection *conn, - struct rspamd_http_message *msg, - const gchar *host, - const gchar *mime_type, - gpointer ud, - gint fd, - struct timeval *timeout, - struct event_base *base); - -/** - * Free connection structure - * @param conn - */ -void rspamd_http_connection_free (struct rspamd_http_connection *conn); - -/** - * Increase refcount for a connection - * @param conn - * @return - */ -static inline struct rspamd_http_connection * -rspamd_http_connection_ref (struct rspamd_http_connection *conn) -{ - conn->ref++; - return conn; -} - -/** - * Decrease a refcount for a connection and free it if refcount is equal to zero - * @param conn - */ -static void -rspamd_http_connection_unref (struct rspamd_http_connection *conn) -{ - if (--conn->ref <= 0) { - rspamd_http_connection_free (conn); - } -} - -/** - * Reset connection for a new request - * @param conn - */ -void rspamd_http_connection_reset (struct rspamd_http_connection *conn); - -/** - * Extract the current message from a connection to deal with separately - * @param conn - * @return - */ -struct rspamd_http_message * rspamd_http_connection_steal_msg ( - struct rspamd_http_connection *conn); - -/** - * Copy the current message from a connection to deal with separately - * @param conn - * @return - */ -struct rspamd_http_message * rspamd_http_connection_copy_msg ( - struct rspamd_http_message *msg, GError **err); - -/** - * Create new HTTP message - * @param type request or response - * @return new http message - */ -struct rspamd_http_message * rspamd_http_new_message (enum http_parser_type type); - -/** - * Increase refcount number for an HTTP message - * @param msg message to use - * @return - */ -struct rspamd_http_message * rspamd_http_message_ref (struct rspamd_http_message *msg); -/** - * Decrease number of refcounts for http message - * @param msg - */ -void rspamd_http_message_unref (struct rspamd_http_message *msg); - -/** - * Sets a key for peer - * @param msg - * @param pk - */ -void rspamd_http_message_set_peer_key (struct rspamd_http_message *msg, - struct rspamd_cryptobox_pubkey *pk); -/** - * Create HTTP message from URL - * @param url - * @return new message or NULL - */ -struct rspamd_http_message* rspamd_http_message_from_url (const gchar *url); - -/** - * Returns body for a message - * @param msg - * @param blen pointer where to save body length - * @return pointer to body start - */ -const gchar *rspamd_http_message_get_body (struct rspamd_http_message *msg, - gsize *blen); - -/** - * Set message's body from the string - * @param msg - * @param data - * @param len - * @return TRUE if a message's body has been set - */ -gboolean rspamd_http_message_set_body (struct rspamd_http_message *msg, - const gchar *data, gsize len); - -/** - * Set message's method by name - * @param msg - * @param method - */ -void rspamd_http_message_set_method (struct rspamd_http_message *msg, - const gchar *method); -/** - * Maps fd as message's body - * @param msg - * @param fd - * @return TRUE if a message's body has been set - */ -gboolean rspamd_http_message_set_body_from_fd (struct rspamd_http_message *msg, - gint fd); - -/** - * Uses rspamd_fstring_t as message's body, string is consumed by this operation - * @param msg - * @param fstr - * @return TRUE if a message's body has been set - */ -gboolean rspamd_http_message_set_body_from_fstring_steal (struct rspamd_http_message *msg, - rspamd_fstring_t *fstr); - -/** - * Uses rspamd_fstring_t as message's body, string is copied by this operation - * @param msg - * @param fstr - * @return TRUE if a message's body has been set - */ -gboolean rspamd_http_message_set_body_from_fstring_copy (struct rspamd_http_message *msg, - const rspamd_fstring_t *fstr); - -/** - * Appends data to message's body - * @param msg - * @param data - * @param len - * @return TRUE if a message's body has been set - */ -gboolean rspamd_http_message_append_body (struct rspamd_http_message *msg, - const gchar *data, gsize len); - -/** - * Append a header to http message - * @param rep - * @param name - * @param value - */ -void rspamd_http_message_add_header (struct rspamd_http_message *msg, - const gchar *name, - const gchar *value); - -void rspamd_http_message_add_header_len (struct rspamd_http_message *msg, - const gchar *name, - const gchar *value, - gsize len); - -void rspamd_http_message_add_header_fstr (struct rspamd_http_message *msg, - const gchar *name, - rspamd_fstring_t *value); - -/** - * Search for a specified header in message - * @param msg message - * @param name name of header - */ -const rspamd_ftok_t * rspamd_http_message_find_header ( - struct rspamd_http_message *msg, - const gchar *name); - -/** - * Search for a header that has multiple values - * @param msg - * @param name - * @return list of rspamd_ftok_t * with values - */ -GPtrArray* rspamd_http_message_find_header_multiple ( - struct rspamd_http_message *msg, - const gchar *name); - -/** - * Remove specific header from a message - * @param msg - * @param name - * @return - */ -gboolean rspamd_http_message_remove_header (struct rspamd_http_message *msg, - const gchar *name); - -/** - * Free HTTP message - * @param msg - */ -void rspamd_http_message_free (struct rspamd_http_message *msg); - -/** - * Sets global maximum size for HTTP message being processed - * @param sz - */ -void rspamd_http_connection_set_max_size (struct rspamd_http_connection *conn, - gsize sz); - -void rspamd_http_connection_disable_encryption (struct rspamd_http_connection *conn); - -/** - * Increase refcount for shared file (if any) to prevent early memory unlinking - * @param msg - */ -struct rspamd_storage_shmem* rspamd_http_message_shmem_ref (struct rspamd_http_message *msg); -/** - * Decrease external ref for shmem segment associated with a message - * @param msg - */ -void rspamd_http_message_shmem_unref (struct rspamd_storage_shmem *p); - -/** - * Returns message's flags - * @param msg - * @return - */ -guint rspamd_http_message_get_flags (struct rspamd_http_message *msg); - -/** - * Parse HTTP date header and return it as time_t - * @param header HTTP date header - * @param len length of header - * @return time_t or (time_t)-1 in case of error - */ -time_t rspamd_http_parse_date (const gchar *header, gsize len); - -/** - * Create new http connection router and the associated HTTP connection - * @param eh error handler callback - * @param fh finish handler callback - * @param default_fs_path if not NULL try to serve static files from - * the specified directory - * @return - */ -struct rspamd_http_connection_router * rspamd_http_router_new ( - rspamd_http_router_error_handler_t eh, - rspamd_http_router_finish_handler_t fh, - struct timeval *timeout, - struct event_base *base, - const char *default_fs_path, - struct rspamd_keypair_cache *cache); - -/** - * Set encryption key for the HTTP router - * @param router router structure - * @param key opaque key structure - */ -void rspamd_http_router_set_key (struct rspamd_http_connection_router *router, - struct rspamd_cryptobox_keypair *key); - -/** - * Add new path to the router - */ -void rspamd_http_router_add_path (struct rspamd_http_connection_router *router, - const gchar *path, rspamd_http_router_handler_t handler); - -/** - * Add custom header to append to router replies - * @param router - * @param name - * @param value - */ -void rspamd_http_router_add_header (struct rspamd_http_connection_router *router, - const gchar *name, const gchar *value); - -/** - * Sets method to handle unknown request methods - * @param router - * @param handler - */ -void rspamd_http_router_set_unknown_handler (struct rspamd_http_connection_router *router, - rspamd_http_router_handler_t handler); - -/** - * Inserts router headers to the outbound message - * @param router - * @param msg - */ -void rspamd_http_router_insert_headers (struct rspamd_http_connection_router *router, - struct rspamd_http_message *msg); - -struct rspamd_regexp_s; -/** - * Adds new pattern to router, regexp object is refcounted by this function - * @param router - * @param re - * @param handler - */ -void rspamd_http_router_add_regexp (struct rspamd_http_connection_router *router, - struct rspamd_regexp_s *re, rspamd_http_router_handler_t handler); -/** - * Handle new accepted socket - * @param router router object - * @param fd server socket - * @param ud opaque userdata - */ -void rspamd_http_router_handle_socket ( - struct rspamd_http_connection_router *router, - gint fd, - gpointer ud); - -/** - * Free router and all connections associated - * @param router - */ -void rspamd_http_router_free (struct rspamd_http_connection_router *router); - -/** - * Extract arguments from a message's URI contained inside query string decoding - * them if needed - * @param msg HTTP request message - * @return new GHashTable which maps rspamd_ftok_t* to rspamd_ftok_t* - * (table must be freed by a caller) - */ -GHashTable* rspamd_http_message_parse_query (struct rspamd_http_message *msg); - -/** - * Prints HTTP date from `time` to `buf` using standard HTTP date format - * @param buf date buffer - * @param len length of buffer - * @param time time in unix seconds - * @return number of bytes written - */ -glong rspamd_http_date_format (gchar *buf, gsize len, time_t time); - -/** - * Normalize HTTP path removing dot sequences and repeating '/' symbols as - * per rfc3986#section-5.2 - * @param path - * @param len - * @param nlen - */ -void rspamd_http_normalize_path_inplace (gchar *path, guint len, guint *nlen); - -#endif /* HTTP_H_ */ diff --git a/src/libutil/http.c b/src/libutil/http_connection.c index a82fc24f7..9e43e21e2 100644 --- a/src/libutil/http.c +++ b/src/libutil/http_connection.c @@ -14,8 +14,9 @@ * limitations under the License. */ #include "config.h" -#include "../../contrib/mumhash/mum.h" +#include "http_connection.h" #include "http_private.h" +#include "http_message.h" #include "utlist.h" #include "util.h" #include "printf.h" @@ -24,11 +25,13 @@ #include "ottery.h" #include "keypair_private.h" #include "cryptobox.h" -#include "unix-std.h" #include "libutil/ssl_util.h" -#include "libutil/regexp.h" #include "libserver/url.h" +#include "contrib/mumhash/mum.h" +#include "contrib/http-parser/http_parser.h" +#include "unix-std.h" + #include <openssl/err.h> #define ENCRYPTED_VERSION " HTTP/1.0" @@ -52,9 +55,10 @@ enum rspamd_http_priv_flags { #define IS_CONN_RESETED(c) ((c)->flags & RSPAMD_HTTP_CONN_FLAG_RESETED) struct rspamd_http_connection_private { - gpointer ssl_ctx; + struct rspamd_http_context *ctx; struct rspamd_ssl_connection *ssl; struct _rspamd_http_privbuf *buf; + struct rspamd_keypair_cache *cache; struct rspamd_cryptobox_pubkey *peer_key; struct rspamd_cryptobox_keypair *local_key; struct rspamd_http_header *header; @@ -71,30 +75,6 @@ struct rspamd_http_connection_private { gsize wr_total; }; -enum http_magic_type { - HTTP_MAGIC_PLAIN = 0, - HTTP_MAGIC_HTML, - HTTP_MAGIC_CSS, - HTTP_MAGIC_JS, - HTTP_MAGIC_PNG, - HTTP_MAGIC_JPG -}; - -static const struct _rspamd_http_magic { - const gchar *ext; - const gchar *ct; -} http_file_types[] = { - [HTTP_MAGIC_PLAIN] = { "txt", "text/plain" }, - [HTTP_MAGIC_HTML] = { "html", "text/html" }, - [HTTP_MAGIC_CSS] = { "css", "text/css" }, - [HTTP_MAGIC_JS] = { "js", "application/javascript" }, - [HTTP_MAGIC_PNG] = { "png", "image/png" }, - [HTTP_MAGIC_JPG] = { "jpg", "image/jpeg" }, -}; - -static const gchar *http_week[] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" }; -static const gchar *http_month[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", - "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; static const rspamd_ftok_t key_header = { .begin = "Key", .len = 3 @@ -108,9 +88,7 @@ static const rspamd_ftok_t last_modified_header = { .len = 13 }; -static void rspamd_http_message_storage_cleanup (struct rspamd_http_message *msg); -static gboolean rspamd_http_message_grow_body (struct rspamd_http_message *msg, - gsize len); + #define HTTP_ERROR http_error_quark () GQuark @@ -156,272 +134,6 @@ rspamd_http_code_to_str (gint code) return "Unknown error"; } -/* - * Obtained from nginx - * Copyright (C) Igor Sysoev - */ -static guint mday[] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; - -time_t -rspamd_http_parse_date (const gchar *header, gsize len) -{ - const gchar *p, *end; - gint month; - guint day, year, hour, min, sec; - guint64 time; - enum { - no = 0, rfc822, /* Tue, 10 Nov 2002 23:50:13 */ - rfc850, /* Tuesday, 10-Dec-02 23:50:13 */ - isoc /* Tue Dec 10 23:50:13 2002 */ - } fmt; - - fmt = 0; - if (len > 0) { - end = header + len; - } - else { - end = header + strlen (header); - } - - day = 32; - year = 2038; - - for (p = header; p < end; p++) { - if (*p == ',') { - break; - } - - if (*p == ' ') { - fmt = isoc; - break; - } - } - - for (p++; p < end; p++) - if (*p != ' ') { - break; - } - - if (end - p < 18) { - return (time_t)-1; - } - - if (fmt != isoc) { - if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') { - return (time_t)-1; - } - - day = (*p - '0') * 10 + *(p + 1) - '0'; - p += 2; - - if (*p == ' ') { - if (end - p < 18) { - return (time_t)-1; - } - fmt = rfc822; - - } - else if (*p == '-') { - fmt = rfc850; - - } - else { - return (time_t)-1; - } - - p++; - } - - switch (*p) { - - case 'J': - month = *(p + 1) == 'a' ? 0 : *(p + 2) == 'n' ? 5 : 6; - break; - - case 'F': - month = 1; - break; - - case 'M': - month = *(p + 2) == 'r' ? 2 : 4; - break; - - case 'A': - month = *(p + 1) == 'p' ? 3 : 7; - break; - - case 'S': - month = 8; - break; - - case 'O': - month = 9; - break; - - case 'N': - month = 10; - break; - - case 'D': - month = 11; - break; - - default: - return (time_t)-1; - } - - p += 3; - - if ((fmt == rfc822 && *p != ' ') || (fmt == rfc850 && *p != '-')) { - return (time_t)-1; - } - - p++; - - if (fmt == rfc822) { - if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9' - || *(p + 2) < '0' || *(p + 2) > '9' || *(p + 3) < '0' - || *(p + 3) > '9') { - return (time_t)-1; - } - - year = (*p - '0') * 1000 + (*(p + 1) - '0') * 100 - + (*(p + 2) - '0') * 10 + *(p + 3) - '0'; - p += 4; - - } - else if (fmt == rfc850) { - if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') { - return (time_t)-1; - } - - year = (*p - '0') * 10 + *(p + 1) - '0'; - year += (year < 70) ? 2000 : 1900; - p += 2; - } - - if (fmt == isoc) { - if (*p == ' ') { - p++; - } - - if (*p < '0' || *p > '9') { - return (time_t)-1; - } - - day = *p++ - '0'; - - if (*p != ' ') { - if (*p < '0' || *p > '9') { - return (time_t)-1; - } - - day = day * 10 + *p++ - '0'; - } - - if (end - p < 14) { - return (time_t)-1; - } - } - - if (*p++ != ' ') { - return (time_t)-1; - } - - if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') { - return (time_t)-1; - } - - hour = (*p - '0') * 10 + *(p + 1) - '0'; - p += 2; - - if (*p++ != ':') { - return (time_t)-1; - } - - if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') { - return (time_t)-1; - } - - min = (*p - '0') * 10 + *(p + 1) - '0'; - p += 2; - - if (*p++ != ':') { - return (time_t)-1; - } - - if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') { - return (time_t)-1; - } - - sec = (*p - '0') * 10 + *(p + 1) - '0'; - - if (fmt == isoc) { - p += 2; - - if (*p++ != ' ') { - return (time_t)-1; - } - - if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9' - || *(p + 2) < '0' || *(p + 2) > '9' || *(p + 3) < '0' - || *(p + 3) > '9') { - return (time_t)-1; - } - - year = (*p - '0') * 1000 + (*(p + 1) - '0') * 100 - + (*(p + 2) - '0') * 10 + *(p + 3) - '0'; - } - - if (hour > 23 || min > 59 || sec > 59) { - return (time_t)-1; - } - - if (day == 29 && month == 1) { - if ((year & 3) || ((year % 100 == 0) && (year % 400) != 0)) { - return (time_t)-1; - } - - } - else if (day > mday[month]) { - return (time_t)-1; - } - - /* - * shift new year to March 1 and start months from 1 (not 0), - * it is needed for Gauss' formula - */ - - if (--month <= 0) { - month += 12; - year -= 1; - } - - /* Gauss' formula for Gregorian days since March 1, 1 BC */ - - time = (guint64) ( - /* days in years including leap years since March 1, 1 BC */ - - 365 * year + year / 4 - year / 100 + year / 400 - - /* days before the month */ - - + 367 * month / 12 - 30 - - /* days before the day */ - - + day - 1 - - /* - * 719527 days were between March 1, 1 BC and March 1, 1970, - * 31 and 28 days were in January and February 1970 - */ - - - 719527 + 31 + 28) * 86400 + hour * 3600 + min * 60 + sec; - - return (time_t) time; -} - static void rspamd_http_parse_key (rspamd_ftok_t *data, struct rspamd_http_connection *conn, struct rspamd_http_connection_private *priv) @@ -453,9 +165,10 @@ rspamd_http_parse_key (rspamd_ftok_t *data, struct rspamd_http_connection *conn, RSPAMD_KEYPAIR_SHORT_ID_LEN) == 0) { priv->msg->peer_key = pk; - if (conn->cache && priv->msg->peer_key) { - rspamd_keypair_cache_process (conn->cache, - priv->local_key, priv->msg->peer_key); + if (priv->cache && priv->msg->peer_key) { + rspamd_keypair_cache_process (priv->cache, + priv->local_key, + priv->msg->peer_key); } } else { @@ -639,7 +352,15 @@ rspamd_http_on_headers_complete (http_parser * parser) msg->code = parser->status_code; rspamd_http_connection_ref (conn); ret = conn->finish_handler (conn, msg); - conn->finished = TRUE; + + if (conn->opts & RSPAMD_HTTP_CLIENT_KEEP_ALIVE) { + rspamd_http_context_push_keepalive (conn->priv->ctx, conn, + msg, conn->priv->ctx->ev_base); + } + else { + conn->finished = TRUE; + } + rspamd_http_connection_unref (conn); return ret; @@ -814,7 +535,15 @@ rspamd_http_on_headers_complete_decrypted (http_parser *parser) msg->code = parser->status_code; rspamd_http_connection_ref (conn); ret = conn->finish_handler (conn, msg); - conn->finished = TRUE; + + if (conn->opts & RSPAMD_HTTP_CLIENT_KEEP_ALIVE) { + rspamd_http_context_push_keepalive (conn->priv->ctx, conn, + msg, conn->priv->ctx->ev_base); + } + else { + conn->finished = TRUE; + } + rspamd_http_connection_unref (conn); return ret; @@ -964,7 +693,15 @@ rspamd_http_on_message_complete (http_parser * parser) rspamd_http_connection_ref (conn); ret = conn->finish_handler (conn, priv->msg); - conn->finished = TRUE; + + if (conn->opts & RSPAMD_HTTP_CLIENT_KEEP_ALIVE) { + rspamd_http_context_push_keepalive (conn->priv->ctx, conn, + priv->msg, conn->priv->ctx->ev_base); + } + else { + conn->finished = TRUE; + } + rspamd_http_connection_unref (conn); } @@ -974,30 +711,41 @@ rspamd_http_on_message_complete (http_parser * parser) static void rspamd_http_simple_client_helper (struct rspamd_http_connection *conn) { - struct event_base *base; struct rspamd_http_connection_private *priv; gpointer ssl; gint request_method; + rspamd_fstring_t *prev_host; priv = conn->priv; - base = conn->priv->ev.ev_base; ssl = priv->ssl; priv->ssl = NULL; request_method = priv->msg->method; + /* Preserve host for keepalive */ + prev_host = priv->msg->host; + priv->msg->host = NULL; rspamd_http_connection_reset (conn); priv->ssl = ssl; + /* Plan read message */ if (conn->opts & RSPAMD_HTTP_CLIENT_SHARED) { - rspamd_http_connection_read_message_shared (conn, conn->ud, conn->fd, - conn->priv->ptv, base); + rspamd_http_connection_read_message_shared (conn, conn->ud, + conn->priv->ptv); } else { - rspamd_http_connection_read_message (conn, conn->ud, conn->fd, - conn->priv->ptv, base); + rspamd_http_connection_read_message (conn, conn->ud, + conn->priv->ptv); } - priv->msg->method = request_method; + if (priv->msg) { + priv->msg->method = request_method; + priv->msg->host = prev_host; + } + else { + if (prev_host) { + rspamd_fstring_free (prev_host); + } + } } static void @@ -1322,13 +1070,13 @@ rspamd_http_parser_reset (struct rspamd_http_connection *conn) struct rspamd_http_connection * rspamd_http_connection_new ( + struct rspamd_http_context *ctx, + gint fd, rspamd_http_body_handler_t body_handler, rspamd_http_error_handler_t error_handler, rspamd_http_finish_handler_t finish_handler, unsigned opts, - enum rspamd_http_connection_type type, - struct rspamd_keypair_cache *cache, - gpointer ssl_ctx) + enum rspamd_http_connection_type type) { struct rspamd_http_connection *conn; struct rspamd_http_connection_private *priv; @@ -1343,15 +1091,28 @@ rspamd_http_connection_new ( conn->body_handler = body_handler; conn->error_handler = error_handler; conn->finish_handler = finish_handler; - conn->fd = -1; + conn->fd = fd; conn->ref = 1; conn->finished = FALSE; - conn->cache = cache; /* Init priv */ + if (ctx == NULL) { + ctx = rspamd_http_context_default (); + } + priv = g_malloc0 (sizeof (struct rspamd_http_connection_private)); conn->priv = priv; - priv->ssl_ctx = ssl_ctx; + priv->ctx = ctx; + + if (conn->type == RSPAMD_HTTP_CLIENT) { + priv->cache = ctx->client_kp_cache; + if (ctx->client_kp) { + priv->local_key = rspamd_keypair_ref (ctx->client_kp); + } + } + else { + priv->cache = ctx->server_kp_cache; + } rspamd_http_parser_reset (conn); priv->parser.data = conn; @@ -1359,6 +1120,47 @@ rspamd_http_connection_new ( return conn; } +struct rspamd_http_connection * +rspamd_http_connection_new_keepalive (struct rspamd_http_context *ctx, + rspamd_http_body_handler_t body_handler, + rspamd_http_error_handler_t error_handler, + rspamd_http_finish_handler_t finish_handler, + rspamd_inet_addr_t *addr, + const gchar *host) +{ + struct rspamd_http_connection *conn; + gint fd; + + if (error_handler == NULL || finish_handler == NULL) { + return NULL; + } + + conn = rspamd_http_context_check_keepalive (ctx, addr, host); + + if (conn) { + return conn; + } + + fd = rspamd_inet_address_connect (addr, SOCK_STREAM, TRUE); + + if (fd == -1) { + msg_info ("cannot connect to %s: %s", rspamd_inet_address_to_string (addr), + host); + return NULL; + } + + conn = rspamd_http_connection_new (ctx, fd, body_handler, error_handler, + finish_handler, + RSPAMD_HTTP_CLIENT_SIMPLE|RSPAMD_HTTP_CLIENT_KEEP_ALIVE, + RSPAMD_HTTP_CLIENT); + + if (conn) { + rspamd_http_context_prepare_keepalive (ctx, conn, addr, host); + } + + return conn; +} + void rspamd_http_connection_reset (struct rspamd_http_connection *conn) { @@ -1573,18 +1375,22 @@ rspamd_http_connection_free (struct rspamd_http_connection *conn) g_free (priv); } + if (conn->opts & RSPAMD_HTTP_CLIENT_KEEP_ALIVE) { + /* Fd is owned by a connection */ + close (conn->fd); + } + g_free (conn); } static void rspamd_http_connection_read_message_common (struct rspamd_http_connection *conn, - gpointer ud, gint fd, struct timeval *timeout, struct event_base *base, + gpointer ud, struct timeval *timeout, gint flags) { struct rspamd_http_connection_private *priv = conn->priv; struct rspamd_http_message *req; - conn->fd = fd; conn->ud = ud; req = rspamd_http_new_message ( conn->type == RSPAMD_HTTP_SERVER ? HTTP_REQUEST : HTTP_RESPONSE); @@ -1604,8 +1410,8 @@ rspamd_http_connection_read_message_common (struct rspamd_http_connection *conn, if (timeout == NULL) { priv->ptv = NULL; } - else if (&priv->tv != timeout) { - memcpy (&priv->tv, timeout, sizeof (struct timeval)); + else { + memmove (&priv->tv, timeout, sizeof (struct timeval)); priv->ptv = &priv->tv; } @@ -1616,13 +1422,12 @@ rspamd_http_connection_read_message_common (struct rspamd_http_connection *conn, priv->flags |= RSPAMD_HTTP_CONN_FLAG_NEW_HEADER; event_set (&priv->ev, - fd, + conn->fd, EV_READ | EV_PERSIST, rspamd_http_event_handler, conn); - if (base != NULL) { - event_base_set (base, &priv->ev); - } + + event_base_set (priv->ctx->ev_base, &priv->ev); priv->flags &= ~RSPAMD_HTTP_CONN_FLAG_RESETED; event_add (&priv->ev, priv->ptv); @@ -1630,16 +1435,16 @@ rspamd_http_connection_read_message_common (struct rspamd_http_connection *conn, void rspamd_http_connection_read_message (struct rspamd_http_connection *conn, - gpointer ud, gint fd, struct timeval *timeout, struct event_base *base) + gpointer ud, struct timeval *timeout) { - rspamd_http_connection_read_message_common (conn, ud, fd, timeout, base, 0); + rspamd_http_connection_read_message_common (conn, ud, timeout, 0); } void rspamd_http_connection_read_message_shared (struct rspamd_http_connection *conn, - gpointer ud, gint fd, struct timeval *timeout, struct event_base *base) + gpointer ud, struct timeval *timeout) { - rspamd_http_connection_read_message_common (conn, ud, fd, timeout, base, + rspamd_http_connection_read_message_common (conn, ud, timeout, RSPAMD_HTTP_FLAG_SHMEM); } @@ -1767,18 +1572,15 @@ rspamd_http_message_write_header (const gchar* mime_type, gboolean encrypted, { gchar datebuf[64]; gint meth_len = 0; - struct tm t; + const gchar *conn_type = "close"; if (conn->type == RSPAMD_HTTP_SERVER) { /* Format reply */ if (msg->method < HTTP_SYMBOLS) { rspamd_ftok_t status; - rspamd_gmtime (msg->date, &t); - rspamd_snprintf (datebuf, sizeof(datebuf), - "%s, %02d %s %4d %02d:%02d:%02d GMT", http_week[t.tm_wday], - t.tm_mday, http_month[t.tm_mon], t.tm_year + 1900, - t.tm_hour, t.tm_min, t.tm_sec); + rspamd_http_date_format (datebuf, sizeof (datebuf), msg->date); + if (mime_type == NULL) { mime_type = encrypted ? "application/octet-stream" : "text/plain"; @@ -1901,6 +1703,10 @@ rspamd_http_message_write_header (const gchar* mime_type, gboolean encrypted, } } else { + if (conn->opts & RSPAMD_HTTP_CLIENT_KEEP_ALIVE) { + conn_type = "keep-alive"; + } + /* Format request */ enclen += msg->url->len + strlen (http_method_str (msg->method)) + 1; @@ -1910,15 +1716,23 @@ rspamd_http_message_write_header (const gchar* mime_type, gboolean encrypted, rspamd_printf_fstring (buf, "%s %s HTTP/1.0\r\n" "Content-Length: %z\r\n" - "Content-Type: application/octet-stream\r\n", + "Content-Type: application/octet-stream\r\n" + "Connection: %s\r\n", "POST", - "/post", enclen); + "/post", + enclen, + conn_type); } else { rspamd_printf_fstring (buf, "%s %V HTTP/1.0\r\n" - "Content-Length: %z\r\n", - http_method_str (msg->method), msg->url, bodylen); + "Content-Length: %z\r\n" + "Connection: %s\r\n", + http_method_str (msg->method), + msg->url, + bodylen, + conn_type); + if (bodylen > 0) { if (mime_type == NULL) { mime_type = "text/plain"; @@ -1935,38 +1749,53 @@ rspamd_http_message_write_header (const gchar* mime_type, gboolean encrypted, if (host != NULL) { rspamd_printf_fstring (buf, "%s %s HTTP/1.1\r\n" - "Connection: close\r\n" + "Connection: %s\r\n" "Host: %s\r\n" "Content-Length: %z\r\n" "Content-Type: application/octet-stream\r\n", - "POST", "/post", host, enclen); + "POST", + "/post", + conn_type, + host, + enclen); } else { rspamd_printf_fstring (buf, "%s %s HTTP/1.1\r\n" - "Connection: close\r\n" + "Connection: %s\r\n" "Host: %V\r\n" "Content-Length: %z\r\n" "Content-Type: application/octet-stream\r\n", - "POST", "/post", msg->host, enclen); + "POST", + "/post", + conn_type, + msg->host, + enclen); } } else { if (host != NULL) { rspamd_printf_fstring (buf, - "%s %V HTTP/1.1\r\nConnection: close\r\n" + "%s %V HTTP/1.1\r\n" + "Connection: %s\r\n" "Host: %s\r\n" "Content-Length: %z\r\n", - http_method_str (msg->method), msg->url, host, + http_method_str (msg->method), + msg->url, + conn_type, + host, bodylen); } else { rspamd_printf_fstring (buf, "%s %V HTTP/1.1\r\n" - "Connection: close\r\n" + "Connection: %s\r\n" "Host: %V\r\n" "Content-Length: %z\r\n", - http_method_str (msg->method), msg->url, msg->host, + http_method_str (msg->method), + msg->url, + conn_type, + msg->host, bodylen); } @@ -1999,9 +1828,12 @@ rspamd_http_message_write_header (const gchar* mime_type, gboolean encrypted, static void rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn, - struct rspamd_http_message *msg, const gchar *host, const gchar *mime_type, - gpointer ud, gint fd, struct timeval *timeout, struct event_base *base, - gboolean allow_shared) + struct rspamd_http_message *msg, + const gchar *host, + const gchar *mime_type, + gpointer ud, + struct timeval *timeout, + gboolean allow_shared) { struct rspamd_http_connection_private *priv = conn->priv; struct rspamd_http_header *hdr, *htmp, *hcur; @@ -2016,7 +1848,6 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn enum rspamd_cryptobox_mode mode; GError *err; - conn->fd = fd; conn->ud = ud; priv->msg = msg; @@ -2049,8 +1880,8 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn encrypted = TRUE; - if (conn->cache) { - rspamd_keypair_cache_process (conn->cache, + if (priv->cache) { + rspamd_keypair_cache_process (priv->cache, priv->local_key, priv->msg->peer_key); } } @@ -2084,6 +1915,11 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn } } + if (priv->ctx->config.user_agent) { + rspamd_http_message_add_header (msg, "User-Agent", + priv->ctx->config.user_agent); + } + if (encrypted) { mode = rspamd_keypair_alg (priv->local_key); @@ -2314,10 +2150,12 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn } if (msg->flags & RSPAMD_HTTP_FLAG_SSL) { - if (base != NULL) { - event_base_set (base, &priv->ev); - } - if (!priv->ssl_ctx) { + gpointer ssl_ctx = (msg->flags & RSPAMD_HTTP_FLAG_SSL_NOVERIFY) ? + priv->ctx->ssl_ctx_noverify : priv->ctx->ssl_ctx; + + event_base_set (priv->ctx->ev_base, &priv->ev); + + if (!ssl_ctx) { err = g_error_new (HTTP_ERROR, errno, "ssl message requested " "with no ssl ctx"); rspamd_http_connection_ref (conn); @@ -2332,11 +2170,11 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn rspamd_ssl_connection_free (priv->ssl); } - priv->ssl = rspamd_ssl_connection_new (priv->ssl_ctx, base, + priv->ssl = rspamd_ssl_connection_new (ssl_ctx, priv->ctx->ev_base, !(msg->flags & RSPAMD_HTTP_FLAG_SSL_NOVERIFY)); g_assert (priv->ssl != NULL); - if (!rspamd_ssl_connect_fd (priv->ssl, fd, host, &priv->ev, + if (!rspamd_ssl_connect_fd (priv->ssl, conn->fd, host, &priv->ev, priv->ptv, rspamd_http_event_handler, rspamd_http_ssl_err_handler, conn)) { @@ -2353,11 +2191,8 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn } } else { - event_set (&priv->ev, fd, EV_WRITE, rspamd_http_event_handler, conn); - - if (base != NULL) { - event_base_set (base, &priv->ev); - } + event_set (&priv->ev, conn->fd, EV_WRITE, rspamd_http_event_handler, conn); + event_base_set (priv->ctx->ev_base, &priv->ev); event_add (&priv->ev, priv->ptv); } @@ -2365,465 +2200,28 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn void rspamd_http_connection_write_message (struct rspamd_http_connection *conn, - struct rspamd_http_message *msg, const gchar *host, const gchar *mime_type, - gpointer ud, gint fd, struct timeval *timeout, struct event_base *base) + struct rspamd_http_message *msg, + const gchar *host, + const gchar *mime_type, + gpointer ud, + struct timeval *timeout) { rspamd_http_connection_write_message_common (conn, msg, host, mime_type, - ud, fd, timeout, base, FALSE); + ud, timeout, FALSE); } void rspamd_http_connection_write_message_shared (struct rspamd_http_connection *conn, - struct rspamd_http_message *msg, const gchar *host, const gchar *mime_type, - gpointer ud, gint fd, struct timeval *timeout, struct event_base *base) + struct rspamd_http_message *msg, + const gchar *host, + const gchar *mime_type, + gpointer ud, + struct timeval *timeout) { rspamd_http_connection_write_message_common (conn, msg, host, mime_type, - ud, fd, timeout, base, TRUE); -} - -struct rspamd_http_message * -rspamd_http_new_message (enum http_parser_type type) -{ - struct rspamd_http_message *new; - - new = g_malloc0 (sizeof (struct rspamd_http_message)); - - if (type == HTTP_REQUEST) { - new->url = rspamd_fstring_new (); - } - else { - new->url = NULL; - new->code = 200; - } - - new->port = 80; - new->type = type; - new->method = HTTP_INVALID; - - REF_INIT_RETAIN (new, rspamd_http_message_free); - - return new; -} - -struct rspamd_http_message* -rspamd_http_message_from_url (const gchar *url) -{ - struct http_parser_url pu; - struct rspamd_http_message *msg; - const gchar *host, *path; - size_t pathlen, urllen; - guint flags = 0; - - if (url == NULL) { - return NULL; - } - - urllen = strlen (url); - memset (&pu, 0, sizeof (pu)); - - if (http_parser_parse_url (url, urllen, FALSE, &pu) != 0) { - msg_warn ("cannot parse URL: %s", url); - return NULL; - } - - if ((pu.field_set & (1 << UF_HOST)) == 0) { - msg_warn ("no host argument in URL: %s", url); - return NULL; - } - - if ((pu.field_set & (1 << UF_SCHEMA))) { - if (pu.field_data[UF_SCHEMA].len == sizeof ("https") - 1 && - memcmp (url + pu.field_data[UF_SCHEMA].off, "https", 5) == 0) { - flags |= RSPAMD_HTTP_FLAG_SSL; - } - } - - if ((pu.field_set & (1 << UF_PATH)) == 0) { - path = "/"; - pathlen = 1; - } - else { - path = url + pu.field_data[UF_PATH].off; - pathlen = urllen - pu.field_data[UF_PATH].off; - } - - msg = rspamd_http_new_message (HTTP_REQUEST); - host = url + pu.field_data[UF_HOST].off; - msg->flags = flags; - - if ((pu.field_set & (1 << UF_PORT)) != 0) { - msg->port = pu.port; - } - else { - /* XXX: magic constant */ - if (flags & RSPAMD_HTTP_FLAG_SSL) { - msg->port = 443; - } - else { - msg->port = 80; - } - } - - msg->host = rspamd_fstring_new_init (host, pu.field_data[UF_HOST].len); - msg->url = rspamd_fstring_append (msg->url, path, pathlen); - - REF_INIT_RETAIN (msg, rspamd_http_message_free); - - return msg; + ud, timeout, TRUE); } -const gchar * -rspamd_http_message_get_body (struct rspamd_http_message *msg, - gsize *blen) -{ - const gchar *ret = NULL; - - if (msg->body_buf.len > 0) { - ret = msg->body_buf.begin; - } - - if (blen) { - *blen = msg->body_buf.len; - } - - return ret; -} - -static void -rspamd_http_shname_dtor (void *p) -{ - struct rspamd_storage_shmem *n = p; - -#ifdef HAVE_SANE_SHMEM - shm_unlink (n->shm_name); -#else - unlink (n->shm_name); -#endif - g_free (n->shm_name); - g_free (n); -} - -struct rspamd_storage_shmem * -rspamd_http_message_shmem_ref (struct rspamd_http_message *msg) -{ - if ((msg->flags & RSPAMD_HTTP_FLAG_SHMEM) && msg->body_buf.c.shared.name) { - REF_RETAIN (msg->body_buf.c.shared.name); - return msg->body_buf.c.shared.name; - } - - return NULL; -} - -guint -rspamd_http_message_get_flags (struct rspamd_http_message *msg) -{ - return msg->flags; -} - -void -rspamd_http_message_shmem_unref (struct rspamd_storage_shmem *p) -{ - REF_RELEASE (p); -} - -gboolean -rspamd_http_message_set_body (struct rspamd_http_message *msg, - const gchar *data, gsize len) -{ - union _rspamd_storage_u *storage; - storage = &msg->body_buf.c; - - rspamd_http_message_storage_cleanup (msg); - - if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) { - storage->shared.name = g_malloc (sizeof (*storage->shared.name)); - REF_INIT_RETAIN (storage->shared.name, rspamd_http_shname_dtor); -#ifdef HAVE_SANE_SHMEM -#if defined(__DragonFly__) - // DragonFly uses regular files for shm. User rspamd is not allowed to create - // files in the root. - storage->shared.name->shm_name = g_strdup ("/tmp/rhm.XXXXXXXXXXXXXXXXXXXX"); -#else - storage->shared.name->shm_name = g_strdup ("/rhm.XXXXXXXXXXXXXXXXXXXX"); -#endif - storage->shared.shm_fd = rspamd_shmem_mkstemp (storage->shared.name->shm_name); -#else - /* XXX: assume that tempdir is /tmp */ - storage->shared.name->shm_name = g_strdup ("/tmp/rhm.XXXXXXXXXXXXXXXXXXXX"); - storage->shared.shm_fd = mkstemp (storage->shared.name->shm_name); -#endif - - if (storage->shared.shm_fd == -1) { - return FALSE; - } - - if (len != 0 && len != ULLONG_MAX) { - if (ftruncate (storage->shared.shm_fd, len) == -1) { - return FALSE; - } - - msg->body_buf.str = mmap (NULL, len, - PROT_WRITE|PROT_READ, MAP_SHARED, - storage->shared.shm_fd, 0); - - if (msg->body_buf.str == MAP_FAILED) { - return FALSE; - } - - msg->body_buf.begin = msg->body_buf.str; - msg->body_buf.allocated_len = len; - - if (data != NULL) { - memcpy (msg->body_buf.str, data, len); - msg->body_buf.len = len; - } - } - else { - msg->body_buf.len = 0; - msg->body_buf.begin = NULL; - msg->body_buf.str = NULL; - msg->body_buf.allocated_len = 0; - } - } - else { - if (len != 0 && len != ULLONG_MAX) { - if (data == NULL) { - storage->normal = rspamd_fstring_sized_new (len); - msg->body_buf.len = 0; - } - else { - storage->normal = rspamd_fstring_new_init (data, len); - msg->body_buf.len = len; - } - } - else { - storage->normal = rspamd_fstring_new (); - } - - msg->body_buf.begin = storage->normal->str; - msg->body_buf.str = storage->normal->str; - msg->body_buf.allocated_len = storage->normal->allocated; - } - - msg->flags |= RSPAMD_HTTP_FLAG_HAS_BODY; - - return TRUE; -} - -void -rspamd_http_message_set_method (struct rspamd_http_message *msg, - const gchar *method) -{ - gint i; - - /* Linear search: not very efficient method */ - for (i = 0; i < HTTP_METHOD_MAX; i ++) { - if (g_ascii_strcasecmp (method, http_method_str (i)) == 0) { - msg->method = i; - } - } -} - -gboolean -rspamd_http_message_set_body_from_fd (struct rspamd_http_message *msg, - gint fd) -{ - union _rspamd_storage_u *storage; - struct stat st; - - rspamd_http_message_storage_cleanup (msg); - - storage = &msg->body_buf.c; - msg->flags |= RSPAMD_HTTP_FLAG_SHMEM|RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE; - - storage->shared.shm_fd = dup (fd); - msg->body_buf.str = MAP_FAILED; - - if (storage->shared.shm_fd == -1) { - return FALSE; - } - - if (fstat (storage->shared.shm_fd, &st) == -1) { - return FALSE; - } - - msg->body_buf.str = mmap (NULL, st.st_size, - PROT_READ, MAP_SHARED, - storage->shared.shm_fd, 0); - - if (msg->body_buf.str == MAP_FAILED) { - return FALSE; - } - - msg->body_buf.begin = msg->body_buf.str; - msg->body_buf.len = st.st_size; - msg->body_buf.allocated_len = st.st_size; - - return TRUE; -} - -gboolean -rspamd_http_message_set_body_from_fstring_steal (struct rspamd_http_message *msg, - rspamd_fstring_t *fstr) -{ - union _rspamd_storage_u *storage; - - rspamd_http_message_storage_cleanup (msg); - - storage = &msg->body_buf.c; - msg->flags &= ~(RSPAMD_HTTP_FLAG_SHMEM|RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE); - - storage->normal = fstr; - msg->body_buf.str = fstr->str; - msg->body_buf.begin = msg->body_buf.str; - msg->body_buf.len = fstr->len; - msg->body_buf.allocated_len = fstr->allocated; - - return TRUE; -} - -gboolean -rspamd_http_message_set_body_from_fstring_copy (struct rspamd_http_message *msg, - const rspamd_fstring_t *fstr) -{ - union _rspamd_storage_u *storage; - - rspamd_http_message_storage_cleanup (msg); - - storage = &msg->body_buf.c; - msg->flags &= ~(RSPAMD_HTTP_FLAG_SHMEM|RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE); - - storage->normal = rspamd_fstring_new_init (fstr->str, fstr->len); - msg->body_buf.str = storage->normal->str; - msg->body_buf.begin = msg->body_buf.str; - msg->body_buf.len = storage->normal->len; - msg->body_buf.allocated_len = storage->normal->allocated; - - return TRUE; -} - - -static gboolean -rspamd_http_message_grow_body (struct rspamd_http_message *msg, gsize len) -{ - struct stat st; - union _rspamd_storage_u *storage; - gsize newlen; - - storage = &msg->body_buf.c; - - if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) { - if (storage->shared.shm_fd == -1) { - return FALSE; - } - - if (fstat (storage->shared.shm_fd, &st) == -1) { - return FALSE; - } - - /* Check if we need to grow */ - if ((gsize)st.st_size < msg->body_buf.len + len) { - /* Need to grow */ - newlen = rspamd_fstring_suggest_size (msg->body_buf.len, st.st_size, - len); - /* Unmap as we need another size of segment */ - if (msg->body_buf.str != MAP_FAILED) { - munmap (msg->body_buf.str, st.st_size); - } - - if (ftruncate (storage->shared.shm_fd, newlen) == -1) { - return FALSE; - } - - msg->body_buf.str = mmap (NULL, newlen, - PROT_WRITE|PROT_READ, MAP_SHARED, - storage->shared.shm_fd, 0); - if (msg->body_buf.str == MAP_FAILED) { - return FALSE; - } - - msg->body_buf.begin = msg->body_buf.str; - msg->body_buf.allocated_len = newlen; - } - } - else { - storage->normal = rspamd_fstring_grow (storage->normal, len); - - /* Append might cause realloc */ - msg->body_buf.begin = storage->normal->str; - msg->body_buf.len = storage->normal->len; - msg->body_buf.str = storage->normal->str; - msg->body_buf.allocated_len = storage->normal->allocated; - } - - return TRUE; -} - -gboolean -rspamd_http_message_append_body (struct rspamd_http_message *msg, - const gchar *data, gsize len) -{ - union _rspamd_storage_u *storage; - - storage = &msg->body_buf.c; - - if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) { - if (!rspamd_http_message_grow_body (msg, len)) { - return FALSE; - } - - memcpy (msg->body_buf.str + msg->body_buf.len, data, len); - msg->body_buf.len += len; - } - else { - storage->normal = rspamd_fstring_append (storage->normal, data, len); - - /* Append might cause realloc */ - msg->body_buf.begin = storage->normal->str; - msg->body_buf.len = storage->normal->len; - msg->body_buf.str = storage->normal->str; - msg->body_buf.allocated_len = storage->normal->allocated; - } - - return TRUE; -} - -static void -rspamd_http_message_storage_cleanup (struct rspamd_http_message *msg) -{ - union _rspamd_storage_u *storage; - struct stat st; - - if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) { - storage = &msg->body_buf.c; - - if (storage->shared.shm_fd > 0) { - g_assert (fstat (storage->shared.shm_fd, &st) != -1); - - if (msg->body_buf.str != MAP_FAILED) { - munmap (msg->body_buf.str, st.st_size); - } - - close (storage->shared.shm_fd); - } - - if (storage->shared.name != NULL) { - REF_RELEASE (storage->shared.name); - } - - storage->shared.shm_fd = -1; - msg->body_buf.str = MAP_FAILED; - } - else { - if (msg->body_buf.c.normal) { - rspamd_fstring_free (msg->body_buf.c.normal); - } - - msg->body_buf.c.normal = NULL; - } - - msg->body_buf.len = 0; -} void rspamd_http_connection_set_max_size (struct rspamd_http_connection *conn, @@ -2833,720 +2231,6 @@ rspamd_http_connection_set_max_size (struct rspamd_http_connection *conn, } void -rspamd_http_message_free (struct rspamd_http_message *msg) -{ - struct rspamd_http_header *hdr, *htmp, *hcur, *hcurtmp; - - - HASH_ITER (hh, msg->headers, hdr, htmp) { - HASH_DEL (msg->headers, hdr); - - DL_FOREACH_SAFE (hdr, hcur, hcurtmp) { - rspamd_fstring_free (hcur->combined); - g_free (hcur); - } - } - - - rspamd_http_message_storage_cleanup (msg); - - if (msg->url != NULL) { - rspamd_fstring_free (msg->url); - } - if (msg->status != NULL) { - rspamd_fstring_free (msg->status); - } - if (msg->host != NULL) { - rspamd_fstring_free (msg->host); - } - if (msg->peer_key != NULL) { - rspamd_pubkey_unref (msg->peer_key); - } - - g_free (msg); -} - -void -rspamd_http_message_set_peer_key (struct rspamd_http_message *msg, - struct rspamd_cryptobox_pubkey *pk) -{ - if (msg->peer_key != NULL) { - rspamd_pubkey_unref (msg->peer_key); - } - - if (pk) { - msg->peer_key = rspamd_pubkey_ref (pk); - } - else { - msg->peer_key = NULL; - } -} - -void -rspamd_http_message_add_header_len (struct rspamd_http_message *msg, - const gchar *name, - const gchar *value, - gsize len) -{ - struct rspamd_http_header *hdr, *found = NULL; - guint nlen, vlen; - - if (msg != NULL && name != NULL && value != NULL) { - hdr = g_malloc0 (sizeof (struct rspamd_http_header)); - nlen = strlen (name); - vlen = len; - hdr->combined = rspamd_fstring_sized_new (nlen + vlen + 4); - rspamd_printf_fstring (&hdr->combined, "%s: %*s\r\n", name, (gint)vlen, - value); - hdr->name.begin = hdr->combined->str; - hdr->name.len = nlen; - hdr->value.begin = hdr->combined->str + nlen + 2; - hdr->value.len = vlen; - - HASH_FIND (hh, msg->headers, hdr->name.begin, - hdr->name.len, found); - - if (found == NULL) { - HASH_ADD_KEYPTR (hh, msg->headers, hdr->name.begin, - hdr->name.len, hdr); - } - - DL_APPEND (found, hdr); - } -} - -void -rspamd_http_message_add_header (struct rspamd_http_message *msg, - const gchar *name, - const gchar *value) -{ - if (value) { - rspamd_http_message_add_header_len (msg, name, value, strlen (value)); - } -} - -void -rspamd_http_message_add_header_fstr (struct rspamd_http_message *msg, - const gchar *name, - rspamd_fstring_t *value) -{ - struct rspamd_http_header *hdr, *found = NULL; - guint nlen, vlen; - - if (msg != NULL && name != NULL && value != NULL) { - hdr = g_malloc0 (sizeof (struct rspamd_http_header)); - nlen = strlen (name); - vlen = value->len; - hdr->combined = rspamd_fstring_sized_new (nlen + vlen + 4); - rspamd_printf_fstring (&hdr->combined, "%s: %V\r\n", name, value); - hdr->name.begin = hdr->combined->str; - hdr->name.len = nlen; - hdr->value.begin = hdr->combined->str + nlen + 2; - hdr->value.len = vlen; - - HASH_FIND (hh, msg->headers, hdr->name.begin, - hdr->name.len, found); - - if (found == NULL) { - HASH_ADD_KEYPTR (hh, msg->headers, hdr->name.begin, - hdr->name.len, hdr); - } - - DL_APPEND (found, hdr); - } -} - -const rspamd_ftok_t * -rspamd_http_message_find_header (struct rspamd_http_message *msg, - const gchar *name) -{ - struct rspamd_http_header *hdr; - const rspamd_ftok_t *res = NULL; - guint slen = strlen (name); - - if (msg != NULL) { - HASH_FIND (hh, msg->headers, name, slen, hdr); - - if (hdr) { - res = &hdr->value; - } - } - - return res; -} - -GPtrArray* -rspamd_http_message_find_header_multiple ( - struct rspamd_http_message *msg, - const gchar *name) -{ - GPtrArray *res = NULL; - struct rspamd_http_header *hdr, *cur; - - guint slen = strlen (name); - - if (msg != NULL) { - HASH_FIND (hh, msg->headers, name, slen, hdr); - - if (hdr) { - res = g_ptr_array_sized_new (4); - - LL_FOREACH (hdr, cur) { - g_ptr_array_add (res, &cur->value); - } - } - } - - - return res; -} - - -gboolean -rspamd_http_message_remove_header (struct rspamd_http_message *msg, - const gchar *name) -{ - struct rspamd_http_header *hdr, *hcur, *hcurtmp; - gboolean res = FALSE; - guint slen = strlen (name); - - if (msg != NULL) { - HASH_FIND (hh, msg->headers, name, slen, hdr); - - if (hdr) { - HASH_DEL (msg->headers, hdr); - res = TRUE; - - DL_FOREACH_SAFE (hdr, hcur, hcurtmp) { - rspamd_fstring_free (hcur->combined); - g_free (hcur); - } - } - } - - return res; -} - -/* - * HTTP router functions - */ - -static void -rspamd_http_entry_free (struct rspamd_http_connection_entry *entry) -{ - if (entry != NULL) { - close (entry->conn->fd); - rspamd_http_connection_unref (entry->conn); - if (entry->rt->finish_handler) { - entry->rt->finish_handler (entry); - } - - DL_DELETE (entry->rt->conns, entry); - g_free (entry); - } -} - -static void -rspamd_http_router_error_handler (struct rspamd_http_connection *conn, - GError *err) -{ - struct rspamd_http_connection_entry *entry = conn->ud; - struct rspamd_http_message *msg; - - if (entry->is_reply) { - /* At this point we need to finish this session and close owned socket */ - if (entry->rt->error_handler != NULL) { - entry->rt->error_handler (entry, err); - } - rspamd_http_entry_free (entry); - } - else { - /* Here we can write a reply to a client */ - if (entry->rt->error_handler != NULL) { - entry->rt->error_handler (entry, err); - } - msg = rspamd_http_new_message (HTTP_RESPONSE); - msg->date = time (NULL); - msg->code = err->code; - rspamd_http_message_set_body (msg, err->message, strlen (err->message)); - rspamd_http_connection_reset (entry->conn); - rspamd_http_connection_write_message (entry->conn, - msg, - NULL, - "text/plain", - entry, - entry->conn->fd, - entry->rt->ptv, - entry->rt->ev_base); - entry->is_reply = TRUE; - } -} - -static const gchar * -rspamd_http_router_detect_ct (const gchar *path) -{ - const gchar *dot; - guint i; - - dot = strrchr (path, '.'); - if (dot == NULL) { - return http_file_types[HTTP_MAGIC_PLAIN].ct; - } - dot++; - - for (i = 0; i < G_N_ELEMENTS (http_file_types); i++) { - if (strcmp (http_file_types[i].ext, dot) == 0) { - return http_file_types[i].ct; - } - } - - return http_file_types[HTTP_MAGIC_PLAIN].ct; -} - -static gboolean -rspamd_http_router_is_subdir (const gchar *parent, const gchar *sub) -{ - if (parent == NULL || sub == NULL || *parent == '\0') { - return FALSE; - } - - while (*parent != '\0') { - if (*sub != *parent) { - return FALSE; - } - parent++; - sub++; - } - - parent--; - if (*parent == G_DIR_SEPARATOR) { - return TRUE; - } - - return (*sub == G_DIR_SEPARATOR || *sub == '\0'); -} - -static gboolean -rspamd_http_router_try_file (struct rspamd_http_connection_entry *entry, - rspamd_ftok_t *lookup, gboolean expand_path) -{ - struct stat st; - gint fd; - gchar filebuf[PATH_MAX], realbuf[PATH_MAX], *dir; - struct rspamd_http_message *reply_msg; - - rspamd_snprintf (filebuf, sizeof (filebuf), "%s%c%T", - entry->rt->default_fs_path, G_DIR_SEPARATOR, lookup); - - if (realpath (filebuf, realbuf) == NULL || - lstat (realbuf, &st) == -1) { - return FALSE; - } - - if (S_ISDIR (st.st_mode) && expand_path) { - /* Try to append 'index.html' to the url */ - rspamd_fstring_t *nlookup; - rspamd_ftok_t tok; - gboolean ret; - - nlookup = rspamd_fstring_sized_new (lookup->len + sizeof ("index.html")); - rspamd_printf_fstring (&nlookup, "%T%c%s", lookup, G_DIR_SEPARATOR, - "index.html"); - tok.begin = nlookup->str; - tok.len = nlookup->len; - ret = rspamd_http_router_try_file (entry, &tok, FALSE); - rspamd_fstring_free (nlookup); - - return ret; - } - else if (!S_ISREG (st.st_mode)) { - return FALSE; - } - - /* We also need to ensure that file is inside the defined dir */ - rspamd_strlcpy (filebuf, realbuf, sizeof (filebuf)); - dir = dirname (filebuf); - - if (dir == NULL || - !rspamd_http_router_is_subdir (entry->rt->default_fs_path, - dir)) { - return FALSE; - } - - fd = open (realbuf, O_RDONLY); - if (fd == -1) { - return FALSE; - } - - reply_msg = rspamd_http_new_message (HTTP_RESPONSE); - reply_msg->date = time (NULL); - reply_msg->code = 200; - rspamd_http_router_insert_headers (entry->rt, reply_msg); - - if (!rspamd_http_message_set_body_from_fd (reply_msg, fd)) { - close (fd); - return FALSE; - } - - close (fd); - - rspamd_http_connection_reset (entry->conn); - - msg_debug ("requested file %s", realbuf); - rspamd_http_connection_write_message (entry->conn, reply_msg, NULL, - rspamd_http_router_detect_ct (realbuf), entry, entry->conn->fd, - entry->rt->ptv, entry->rt->ev_base); - - return TRUE; -} - -static void -rspamd_http_router_send_error (GError *err, - struct rspamd_http_connection_entry *entry) -{ - struct rspamd_http_message *err_msg; - - err_msg = rspamd_http_new_message (HTTP_RESPONSE); - err_msg->date = time (NULL); - err_msg->code = err->code; - rspamd_http_message_set_body (err_msg, err->message, - strlen (err->message)); - entry->is_reply = TRUE; - err_msg->status = rspamd_fstring_new_init (err->message, strlen (err->message)); - rspamd_http_router_insert_headers (entry->rt, err_msg); - rspamd_http_connection_reset (entry->conn); - rspamd_http_connection_write_message (entry->conn, - err_msg, - NULL, - "text/plain", - entry, - entry->conn->fd, - entry->rt->ptv, - entry->rt->ev_base); -} - - -static int -rspamd_http_router_finish_handler (struct rspamd_http_connection *conn, - struct rspamd_http_message *msg) -{ - struct rspamd_http_connection_entry *entry = conn->ud; - rspamd_http_router_handler_t handler = NULL; - gpointer found; - - GError *err; - rspamd_ftok_t lookup; - const rspamd_ftok_t *encoding; - struct http_parser_url u; - guint i; - rspamd_regexp_t *re; - struct rspamd_http_connection_router *router; - - G_STATIC_ASSERT (sizeof (rspamd_http_router_handler_t) == - sizeof (gpointer)); - - memset (&lookup, 0, sizeof (lookup)); - router = entry->rt; - - if (entry->is_reply) { - /* Request is finished, it is safe to free a connection */ - rspamd_http_entry_free (entry); - } - else { - if (G_UNLIKELY (msg->method != HTTP_GET && msg->method != HTTP_POST)) { - if (router->unknown_method_handler) { - return router->unknown_method_handler (entry, msg); - } - else { - err = g_error_new (HTTP_ERROR, 500, - "Invalid method"); - if (entry->rt->error_handler != NULL) { - entry->rt->error_handler (entry, err); - } - - rspamd_http_router_send_error (err, entry); - g_error_free (err); - - return 0; - } - } - - /* Search for path */ - if (msg->url != NULL && msg->url->len != 0) { - - http_parser_parse_url (msg->url->str, msg->url->len, TRUE, &u); - - if (u.field_set & (1 << UF_PATH)) { - guint unnorm_len; - lookup.begin = msg->url->str + u.field_data[UF_PATH].off; - lookup.len = u.field_data[UF_PATH].len; - - rspamd_http_normalize_path_inplace ((gchar *)lookup.begin, - lookup.len, - &unnorm_len); - lookup.len = unnorm_len; - } - else { - lookup.begin = msg->url->str; - lookup.len = msg->url->len; - } - - found = g_hash_table_lookup (entry->rt->paths, &lookup); - memcpy (&handler, &found, sizeof (found)); - msg_debug ("requested known path: %T", &lookup); - } - else { - err = g_error_new (HTTP_ERROR, 404, - "Empty path requested"); - if (entry->rt->error_handler != NULL) { - entry->rt->error_handler (entry, err); - } - - rspamd_http_router_send_error (err, entry); - g_error_free (err); - - return 0; - } - - entry->is_reply = TRUE; - - encoding = rspamd_http_message_find_header (msg, "Accept-Encoding"); - - if (encoding && rspamd_substring_search (encoding->begin, encoding->len, - "gzip", 4) != -1) { - entry->support_gzip = TRUE; - } - - if (handler != NULL) { - return handler (entry, msg); - } - else { - /* Try regexps */ - for (i = 0; i < router->regexps->len; i ++) { - re = g_ptr_array_index (router->regexps, i); - if (rspamd_regexp_match (re, lookup.begin, lookup.len, - TRUE)) { - found = rspamd_regexp_get_ud (re); - memcpy (&handler, &found, sizeof (found)); - - return handler (entry, msg); - } - } - - /* Now try plain file */ - if (entry->rt->default_fs_path == NULL || lookup.len == 0 || - !rspamd_http_router_try_file (entry, &lookup, TRUE)) { - - err = g_error_new (HTTP_ERROR, 404, - "Not found"); - if (entry->rt->error_handler != NULL) { - entry->rt->error_handler (entry, err); - } - - msg_info ("path: %T not found", &lookup); - rspamd_http_router_send_error (err, entry); - g_error_free (err); - } - } - } - - return 0; -} - -struct rspamd_http_connection_router * -rspamd_http_router_new (rspamd_http_router_error_handler_t eh, - rspamd_http_router_finish_handler_t fh, - struct timeval *timeout, struct event_base *base, - const char *default_fs_path, - struct rspamd_keypair_cache *cache) -{ - struct rspamd_http_connection_router * new; - struct stat st; - - new = g_malloc0 (sizeof (struct rspamd_http_connection_router)); - new->paths = g_hash_table_new_full (rspamd_ftok_icase_hash, - rspamd_ftok_icase_equal, rspamd_fstring_mapped_ftok_free, NULL); - new->regexps = g_ptr_array_new (); - new->conns = NULL; - new->error_handler = eh; - new->finish_handler = fh; - new->ev_base = base; - new->response_headers = g_hash_table_new_full (rspamd_strcase_hash, - rspamd_strcase_equal, g_free, g_free); - - if (timeout) { - new->tv = *timeout; - new->ptv = &new->tv; - } - else { - new->ptv = NULL; - } - - new->default_fs_path = NULL; - - if (default_fs_path != NULL) { - if (stat (default_fs_path, &st) == -1) { - msg_err ("cannot stat %s", default_fs_path); - } - else { - if (!S_ISDIR (st.st_mode)) { - msg_err ("path %s is not a directory", default_fs_path); - } - else { - new->default_fs_path = realpath (default_fs_path, NULL); - } - } - } - - new->cache = cache; - - return new; -} - -void -rspamd_http_router_set_key (struct rspamd_http_connection_router *router, - struct rspamd_cryptobox_keypair *key) -{ - g_assert (key != NULL); - - router->key = rspamd_keypair_ref (key); -} - -void -rspamd_http_router_add_path (struct rspamd_http_connection_router *router, - const gchar *path, rspamd_http_router_handler_t handler) -{ - gpointer ptr; - rspamd_ftok_t *key; - rspamd_fstring_t *storage; - G_STATIC_ASSERT (sizeof (rspamd_http_router_handler_t) == - sizeof (gpointer)); - - if (path != NULL && handler != NULL && router != NULL) { - memcpy (&ptr, &handler, sizeof (ptr)); - storage = rspamd_fstring_new_init (path, strlen (path)); - key = g_malloc0 (sizeof (*key)); - key->begin = storage->str; - key->len = storage->len; - g_hash_table_insert (router->paths, key, ptr); - } -} - -void -rspamd_http_router_set_unknown_handler (struct rspamd_http_connection_router *router, - rspamd_http_router_handler_t handler) -{ - if (router != NULL) { - router->unknown_method_handler = handler; - } -} - -void -rspamd_http_router_add_header (struct rspamd_http_connection_router *router, - const gchar *name, const gchar *value) -{ - if (name != NULL && value != NULL && router != NULL) { - g_hash_table_replace (router->response_headers, g_strdup (name), - g_strdup (value)); - } -} - -void -rspamd_http_router_insert_headers (struct rspamd_http_connection_router *router, - struct rspamd_http_message *msg) -{ - GHashTableIter it; - gpointer k, v; - - if (router && msg) { - g_hash_table_iter_init (&it, router->response_headers); - - while (g_hash_table_iter_next (&it, &k, &v)) { - rspamd_http_message_add_header (msg, k, v); - } - } -} - -void -rspamd_http_router_add_regexp (struct rspamd_http_connection_router *router, - struct rspamd_regexp_s *re, rspamd_http_router_handler_t handler) -{ - gpointer ptr; - G_STATIC_ASSERT (sizeof (rspamd_http_router_handler_t) == - sizeof (gpointer)); - - if (re != NULL && handler != NULL && router != NULL) { - memcpy (&ptr, &handler, sizeof (ptr)); - rspamd_regexp_set_ud (re, ptr); - g_ptr_array_add (router->regexps, rspamd_regexp_ref (re)); - } -} - -void -rspamd_http_router_handle_socket (struct rspamd_http_connection_router *router, - gint fd, gpointer ud) -{ - struct rspamd_http_connection_entry *conn; - - conn = g_malloc0 (sizeof (struct rspamd_http_connection_entry)); - conn->rt = router; - conn->ud = ud; - conn->is_reply = FALSE; - - conn->conn = rspamd_http_connection_new (NULL, - rspamd_http_router_error_handler, - rspamd_http_router_finish_handler, - 0, - RSPAMD_HTTP_SERVER, - router->cache, - NULL); - - if (router->key) { - rspamd_http_connection_set_key (conn->conn, router->key); - } - - rspamd_http_connection_read_message (conn->conn, conn, fd, router->ptv, - router->ev_base); - DL_PREPEND (router->conns, conn); -} - -void -rspamd_http_router_free (struct rspamd_http_connection_router *router) -{ - struct rspamd_http_connection_entry *conn, *tmp; - rspamd_regexp_t *re; - guint i; - - if (router) { - DL_FOREACH_SAFE (router->conns, conn, tmp) { - rspamd_http_entry_free (conn); - } - - if (router->key) { - rspamd_keypair_unref (router->key); - } - - if (router->cache) { - rspamd_keypair_cache_destroy (router->cache); - } - - if (router->default_fs_path != NULL) { - g_free (router->default_fs_path); - } - - for (i = 0; i < router->regexps->len; i ++) { - re = g_ptr_array_index (router->regexps, i); - rspamd_regexp_unref (re); - } - - g_ptr_array_free (router->regexps, TRUE); - g_hash_table_unref (router->paths); - g_hash_table_unref (router->response_headers); - g_free (router); - } -} - -void rspamd_http_connection_set_key (struct rspamd_http_connection *conn, struct rspamd_cryptobox_keypair *key) { @@ -3710,19 +2394,6 @@ rspamd_http_message_parse_query (struct rspamd_http_message *msg) } -glong -rspamd_http_date_format (gchar *buf, gsize len, time_t time) -{ - struct tm tms; - - rspamd_gmtime (time, &tms); - - return rspamd_snprintf (buf, len, "%s, %02d %s %4d %02d:%02d:%02d GMT", - http_week[tms.tm_wday], tms.tm_mday, - http_month[tms.tm_mon], tms.tm_year + 1900, - tms.tm_hour, tms.tm_min, tms.tm_sec); -} - struct rspamd_http_message * rspamd_http_message_ref (struct rspamd_http_message *msg) { @@ -3737,218 +2408,6 @@ rspamd_http_message_unref (struct rspamd_http_message *msg) REF_RELEASE (msg); } - -void -rspamd_http_normalize_path_inplace (gchar *path, guint len, guint *nlen) -{ - const gchar *p, *end, *slash = NULL, *dot = NULL; - gchar *o; - enum { - st_normal = 0, - st_got_dot, - st_got_dot_dot, - st_got_slash, - st_got_slash_slash, - } state = st_normal; - - p = path; - end = path + len; - o = path; - - while (p < end) { - switch (state) { - case st_normal: - if (G_UNLIKELY (*p == '/')) { - state = st_got_slash; - slash = p; - } - else if (G_UNLIKELY (*p == '.')) { - state = st_got_dot; - dot = p; - } - else { - *o++ = *p; - } - p ++; - break; - case st_got_slash: - if (G_UNLIKELY (*p == '/')) { - /* Ignore double slash */ - *o++ = *p; - state = st_got_slash_slash; - } - else if (G_UNLIKELY (*p == '.')) { - dot = p; - state = st_got_dot; - } - else { - *o++ = '/'; - *o++ = *p; - slash = NULL; - dot = NULL; - state = st_normal; - } - p ++; - break; - case st_got_slash_slash: - if (G_LIKELY (*p != '/')) { - slash = p - 1; - dot = NULL; - state = st_normal; - continue; - } - p ++; - break; - case st_got_dot: - if (G_UNLIKELY (*p == '/')) { - /* Remove any /./ or ./ paths */ - if (((o > path && *(o - 1) != '/') || (o == path)) && slash) { - /* Preserve one slash */ - *o++ = '/'; - } - - slash = p; - dot = NULL; - /* Ignore last slash */ - state = st_normal; - } - else if (*p == '.') { - /* Double dot character */ - state = st_got_dot_dot; - } - else { - /* We have something like .some or /.some */ - if (dot && p > dot) { - if (slash == dot - 1 && (o > path && *(o - 1) != '/')) { - /* /.blah */ - memmove (o, slash, p - slash); - o += p - slash; - } - else { - memmove (o, dot, p - dot); - o += p - dot; - } - } - - slash = NULL; - dot = NULL; - state = st_normal; - continue; - } - - p ++; - break; - case st_got_dot_dot: - if (*p == '/') { - /* We have something like /../ or ../ */ - if (slash) { - /* We need to remove the last component from o if it is there */ - if (o > path + 2 && *(o - 1) == '/') { - slash = rspamd_memrchr (path, '/', o - path - 2); - } - else if (o > path + 1) { - slash = rspamd_memrchr (path, '/', o - path - 1); - } - else { - slash = NULL; - } - - if (slash) { - o = (gchar *)slash; - } - /* Otherwise we keep these dots */ - slash = p; - state = st_got_slash; - } - else { - /* We have something like bla../, so we need to copy it as is */ - if (o > path && dot && p > dot) { - memmove (o, dot, p - dot); - o += p - dot; - } - - slash = NULL; - dot = NULL; - state = st_normal; - continue; - } - } - else { - /* We have something like ..bla or ... */ - if (slash) { - *o ++ = '/'; - } - - if (dot && p > dot) { - memmove (o, dot, p - dot); - o += p - dot; - } - - slash = NULL; - dot = NULL; - state = st_normal; - continue; - } - - p ++; - break; - } - } - - /* Leftover */ - switch (state) { - case st_got_dot_dot: - /* Trailing .. */ - if (slash) { - /* We need to remove the last component from o if it is there */ - if (o > path + 2 && *(o - 1) == '/') { - slash = rspamd_memrchr (path, '/', o - path - 2); - } - else if (o > path + 1) { - slash = rspamd_memrchr (path, '/', o - path - 1); - } - else { - if (o == path) { - /* Corner case */ - *o++ = '/'; - } - - slash = NULL; - } - - if (slash) { - /* Remove last / */ - o = (gchar *)slash; - } - } - else { - /* Corner case */ - if (o == path) { - *o++ = '/'; - } - else { - if (dot && p > dot) { - memmove (o, dot, p - dot); - o += p - dot; - } - } - } - break; - case st_got_slash: - *o++ = '/'; - break; - default: - if (o > path + 1 && *(o - 1) == '/') { - o --; - } - break; - } - - if (nlen) { - *nlen = (o - path); - } -} - void rspamd_http_connection_disable_encryption (struct rspamd_http_connection *conn) { diff --git a/src/libutil/http_connection.h b/src/libutil/http_connection.h new file mode 100644 index 000000000..a327eec0d --- /dev/null +++ b/src/libutil/http_connection.h @@ -0,0 +1,251 @@ +/*- + * Copyright 2016 Vsevolod Stakhov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef HTTP_H_ +#define HTTP_H_ + +/** + * @file http.h + * + * This is an interface for HTTP client and conn. + * This code uses HTTP parser written by Joyent Inc based on nginx code. + */ + +#include "config.h" +#include "http_context.h" +#include "fstring.h" +#include "ref.h" +#include "http_message.h" +#include "http_util.h" +#include "addr.h" + +#include <event.h> + +enum rspamd_http_connection_type { + RSPAMD_HTTP_SERVER, + RSPAMD_HTTP_CLIENT +}; + +struct rspamd_http_header; +struct rspamd_http_message; +struct rspamd_http_connection_private; +struct rspamd_http_connection; +struct rspamd_http_connection_router; +struct rspamd_http_connection_entry; +struct rspamd_keepalive_hash_key; + +struct rspamd_storage_shmem { + gchar *shm_name; + ref_entry_t ref; +}; + +/** + * Legacy spamc protocol + */ +#define RSPAMD_HTTP_FLAG_SPAMC (1 << 0) +/** + * Store body of the message in a shared memory segment + */ +#define RSPAMD_HTTP_FLAG_SHMEM (1 << 2) +/** + * Store body of the message in an immutable shared memory segment + */ +#define RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE (1 << 3) +/** + * Use tls for this message + */ +#define RSPAMD_HTTP_FLAG_SSL (1 << 4) +/** + * Body has been set for a message + */ +#define RSPAMD_HTTP_FLAG_HAS_BODY (1 << 5) +/** + * Do not verify server's certificate + */ +#define RSPAMD_HTTP_FLAG_SSL_NOVERIFY (1 << 6) +/** + * Options for HTTP connection + */ +enum rspamd_http_options { + RSPAMD_HTTP_BODY_PARTIAL = 1, /**< Call body handler on all body data portions */ + RSPAMD_HTTP_CLIENT_SIMPLE = 1u << 1, /**< Read HTTP client reply automatically */ + RSPAMD_HTTP_CLIENT_ENCRYPTED = 1u << 2, /**< Encrypt data for client */ + RSPAMD_HTTP_CLIENT_SHARED = 1u << 3, /**< Store reply in shared memory */ + RSPAMD_HTTP_REQUIRE_ENCRYPTION = 1u << 4, + RSPAMD_HTTP_CLIENT_KEEP_ALIVE = 1u << 5, +}; + +typedef int (*rspamd_http_body_handler_t) (struct rspamd_http_connection *conn, + struct rspamd_http_message *msg, + const gchar *chunk, + gsize len); + +typedef void (*rspamd_http_error_handler_t) (struct rspamd_http_connection *conn, + GError *err); + +typedef int (*rspamd_http_finish_handler_t) (struct rspamd_http_connection *conn, + struct rspamd_http_message *msg); + +/** + * HTTP connection structure + */ +struct rspamd_http_connection { + struct rspamd_http_connection_private *priv; + rspamd_http_body_handler_t body_handler; + rspamd_http_error_handler_t error_handler; + rspamd_http_finish_handler_t finish_handler; + gpointer ud; + /* Used for keepalive */ + struct rspamd_keepalive_hash_key *keepalive_hash_key; + gsize max_size; + unsigned opts; + enum rspamd_http_connection_type type; + gboolean finished; + gint fd; + gint ref; +}; + +/** + * Create new http connection + * @param handler_t handler_t for body + * @param opts options + * @return new connection structure + */ +struct rspamd_http_connection *rspamd_http_connection_new ( + struct rspamd_http_context *ctx, + gint fd, + rspamd_http_body_handler_t body_handler, + rspamd_http_error_handler_t error_handler, + rspamd_http_finish_handler_t finish_handler, + unsigned opts, + enum rspamd_http_connection_type type); + +struct rspamd_http_connection *rspamd_http_connection_new_keepalive ( + struct rspamd_http_context *ctx, + rspamd_http_body_handler_t body_handler, + rspamd_http_error_handler_t error_handler, + rspamd_http_finish_handler_t finish_handler, + rspamd_inet_addr_t *addr, + const gchar *host); + + +/** + * Set key pointed by an opaque pointer + * @param conn connection structure + * @param key opaque key structure + */ +void rspamd_http_connection_set_key (struct rspamd_http_connection *conn, + struct rspamd_cryptobox_keypair *key); + +/** + * Get peer's public key + * @param conn connection structure + * @return pubkey structure or NULL + */ +const struct rspamd_cryptobox_pubkey* rspamd_http_connection_get_peer_key ( + struct rspamd_http_connection *conn); + +/** + * Returns TRUE if a connection is encrypted + * @param conn + * @return + */ +gboolean rspamd_http_connection_is_encrypted (struct rspamd_http_connection *conn); + +/** + * Handle a request using socket fd and user data ud + * @param conn connection structure + * @param ud opaque user data + * @param fd fd to read/write + */ +void rspamd_http_connection_read_message ( + struct rspamd_http_connection *conn, + gpointer ud, + struct timeval *timeout); + +void rspamd_http_connection_read_message_shared ( + struct rspamd_http_connection *conn, + gpointer ud, + struct timeval *timeout); + +/** + * Send reply using initialised connection + * @param conn connection structure + * @param msg HTTP message + * @param ud opaque user data + * @param fd fd to read/write + */ +void rspamd_http_connection_write_message ( + struct rspamd_http_connection *conn, + struct rspamd_http_message *msg, + const gchar *host, + const gchar *mime_type, + gpointer ud, + struct timeval *timeout); + +void rspamd_http_connection_write_message_shared ( + struct rspamd_http_connection *conn, + struct rspamd_http_message *msg, + const gchar *host, + const gchar *mime_type, + gpointer ud, + struct timeval *timeout); + +/** + * Free connection structure + * @param conn + */ +void rspamd_http_connection_free (struct rspamd_http_connection *conn); + +/** + * Increase refcount for a connection + * @param conn + * @return + */ +static inline struct rspamd_http_connection * +rspamd_http_connection_ref (struct rspamd_http_connection *conn) +{ + conn->ref++; + return conn; +} + +/** + * Decrease a refcount for a connection and free it if refcount is equal to zero + * @param conn + */ +static void +rspamd_http_connection_unref (struct rspamd_http_connection *conn) +{ + if (--conn->ref <= 0) { + rspamd_http_connection_free (conn); + } +} + +/** + * Reset connection for a new request + * @param conn + */ +void rspamd_http_connection_reset (struct rspamd_http_connection *conn); + +/** + * Sets global maximum size for HTTP message being processed + * @param sz + */ +void rspamd_http_connection_set_max_size (struct rspamd_http_connection *conn, + gsize sz); + +void rspamd_http_connection_disable_encryption (struct rspamd_http_connection *conn); + +#endif /* HTTP_H_ */ diff --git a/src/libutil/http_context.c b/src/libutil/http_context.c new file mode 100644 index 000000000..da2481640 --- /dev/null +++ b/src/libutil/http_context.c @@ -0,0 +1,433 @@ +/*- + * Copyright 2019 Vsevolod Stakhov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "http_context.h" +#include "http_private.h" +#include "keypair.h" +#include "keypairs_cache.h" +#include "cfg_file.h" +#include "contrib/libottery/ottery.h" +#include "rspamd.h" + +static struct rspamd_http_context *default_ctx = NULL; + +struct rspamd_http_keepalive_cbdata { + struct rspamd_http_connection *conn; + GQueue *queue; + GList *link; + struct event ev; +}; + +static void +rspamd_http_keepalive_queue_cleanup (GQueue *conns) +{ + GList *cur; + + cur = conns->head; + + while (cur) { + struct rspamd_http_keepalive_cbdata *cbd; + + cbd = (struct rspamd_http_keepalive_cbdata *)cur->data; + rspamd_http_connection_unref (cbd->conn); + event_del (&cbd->ev); + g_free (cbd); + + cur = cur->next; + } + + g_queue_clear (conns); +} + +static void +rspamd_http_context_client_rotate_ev (gint fd, short what, void *arg) +{ + struct timeval rot_tv; + struct rspamd_http_context *ctx = arg; + gpointer kp; + + double_to_tv (ctx->config.client_key_rotate_time, &rot_tv); + rot_tv.tv_sec += ottery_rand_range (rot_tv.tv_sec); + event_del (&ctx->client_rotate_ev); + event_add (&ctx->client_rotate_ev, &rot_tv); + + kp = ctx->client_kp; + ctx->client_kp = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX, + RSPAMD_CRYPTOBOX_MODE_25519); + rspamd_keypair_unref (kp); +} + +static struct rspamd_http_context* +rspamd_http_context_new_default (struct rspamd_config *cfg, + struct event_base *ev_base) +{ + struct rspamd_http_context *ctx; + + static const int default_kp_size = 1024; + static const gdouble default_rotate_time = 120; + static const gchar *default_user_agent = "rspamd-" RSPAMD_VERSION_FULL; + + ctx = g_malloc0 (sizeof (*ctx)); + ctx->config.kp_cache_size_client = default_kp_size; + ctx->config.kp_cache_size_server = default_kp_size; + ctx->config.client_key_rotate_time = default_rotate_time; + ctx->config.user_agent = default_user_agent; + + if (cfg) { + ctx->ssl_ctx = cfg->libs_ctx->ssl_ctx; + ctx->ssl_ctx_noverify = cfg->libs_ctx->ssl_ctx_noverify; + } + else { + ctx->ssl_ctx = rspamd_init_ssl_ctx (); + ctx->ssl_ctx_noverify = rspamd_init_ssl_ctx_noverify (); + } + + ctx->ev_base = ev_base; + + ctx->keep_alive_hash = kh_init (rspamd_keep_alive_hash); + + return ctx; +} + +static void +rspamd_http_context_init (struct rspamd_http_context *ctx) +{ + if (ctx->config.kp_cache_size_client > 0) { + ctx->client_kp_cache = rspamd_keypair_cache_new (ctx->config.kp_cache_size_client); + } + + if (ctx->config.kp_cache_size_client > 0) { + ctx->client_kp_cache = rspamd_keypair_cache_new (ctx->config.kp_cache_size_client); + } + + if (ctx->config.client_key_rotate_time > 0 && ctx->ev_base) { + struct timeval tv; + double jittered = rspamd_time_jitter (ctx->config.client_key_rotate_time, + 0); + + double_to_tv (jittered, &tv); + event_set (&ctx->client_rotate_ev, -1, EV_TIMEOUT, + rspamd_http_context_client_rotate_ev, ctx); + event_base_set (ctx->ev_base, &ctx->client_rotate_ev); + event_add (&ctx->client_rotate_ev, &tv); + } + + default_ctx = ctx; +} + +struct rspamd_http_context* +rspamd_http_context_create (struct rspamd_config *cfg, + struct event_base *ev_base) +{ + struct rspamd_http_context *ctx; + const ucl_object_t *http_obj; + + ctx = rspamd_http_context_new_default (cfg, ev_base); + http_obj = ucl_object_lookup (cfg->rcl_obj, "http"); + + if (http_obj) { + const ucl_object_t *server_obj, *client_obj; + + client_obj = ucl_object_lookup (http_obj, "client"); + + if (client_obj) { + const ucl_object_t *kp_size; + + kp_size = ucl_object_lookup (client_obj, "cache_size"); + + if (kp_size) { + ctx->config.kp_cache_size_client = ucl_object_toint (kp_size); + } + + const ucl_object_t *rotate_time; + + rotate_time = ucl_object_lookup (client_obj, "rotate_time"); + + if (rotate_time) { + ctx->config.client_key_rotate_time = ucl_object_todouble (rotate_time); + } + + const ucl_object_t *user_agent; + + user_agent = ucl_object_lookup (client_obj, "user_agent"); + + if (user_agent) { + ctx->config.user_agent = ucl_object_tostring (user_agent); + + if (ctx->config.user_agent && strlen (ctx->config.user_agent) == 0) { + ctx->config.user_agent = NULL; + } + } + } + + server_obj = ucl_object_lookup (http_obj, "server"); + + if (server_obj) { + const ucl_object_t *kp_size; + + kp_size = ucl_object_lookup (server_obj, "cache_size"); + + if (kp_size) { + ctx->config.kp_cache_size_server = ucl_object_toint (kp_size); + } + } + } + + rspamd_http_context_init (ctx); + + return ctx; +} + + +void +rspamd_http_context_free (struct rspamd_http_context *ctx) +{ + if (ctx == default_ctx) { + default_ctx = NULL; + } + + if (ctx->client_kp_cache) { + rspamd_keypair_cache_destroy (ctx->client_kp_cache); + } + + if (ctx->server_kp_cache) { + rspamd_keypair_cache_destroy (ctx->server_kp_cache); + } + + if (ctx->config.client_key_rotate_time > 0) { + /* Event is removed on base event loop termination */ + /* event_del (&ctx->client_rotate_ev); */ + + if (ctx->client_kp) { + rspamd_keypair_unref (ctx->client_kp); + } + } + + struct rspamd_keepalive_hash_key *hk; + + kh_foreach_key (ctx->keep_alive_hash, hk, { + if (hk->host) { + g_free (hk->host); + } + + rspamd_inet_address_free (hk->addr); + rspamd_http_keepalive_queue_cleanup (&hk->conns); + g_free (hk); + }); + + kh_destroy (rspamd_keep_alive_hash, ctx->keep_alive_hash); + + g_free (ctx); +} + +struct rspamd_http_context* +rspamd_http_context_create_config (struct rspamd_http_context_cfg *cfg, + struct event_base *ev_base) +{ + struct rspamd_http_context *ctx; + + ctx = rspamd_http_context_new_default (NULL, ev_base); + memcpy (&ctx->config, cfg, sizeof (*cfg)); + rspamd_http_context_init (ctx); + + return ctx; +} + +struct rspamd_http_context* +rspamd_http_context_default (void) +{ + g_assert (default_ctx != NULL); + + return default_ctx; +} + +gint32 +rspamd_keep_alive_key_hash (struct rspamd_keepalive_hash_key *k) +{ + gint32 h; + + h = rspamd_inet_address_port_hash (k->addr); + + if (k->host) { + h = rspamd_cryptobox_fast_hash (k->host, strlen (k->host), h); + } + + return h; +} + +bool +rspamd_keep_alive_key_equal (struct rspamd_keepalive_hash_key *k1, + struct rspamd_keepalive_hash_key *k2) +{ + if (k1->host && k2->host) { + if (rspamd_inet_address_port_equal (k1->addr, k2->addr)) { + return strcmp (k1->host, k2->host); + } + } + else if (!k1->host && !k2->host) { + return rspamd_inet_address_port_equal (k1->addr, k2->addr); + } + + /* One has host and another has no host */ + return false; +} + +struct rspamd_http_connection* +rspamd_http_context_check_keepalive (struct rspamd_http_context *ctx, + const rspamd_inet_addr_t *addr, + const gchar *host) +{ + struct rspamd_keepalive_hash_key hk, *phk; + khiter_t k; + + hk.addr = (rspamd_inet_addr_t *)addr; + hk.host = (gchar *)host; + + k = kh_get (rspamd_keep_alive_hash, ctx->keep_alive_hash, &hk); + + if (k != kh_end (ctx->keep_alive_hash)) { + phk = kh_key (ctx->keep_alive_hash, k); + GQueue *conns = &phk->conns; + + /* Use stack based approach */ + + if (g_queue_get_length (conns) > 0) { + struct rspamd_http_keepalive_cbdata *cbd; + struct rspamd_http_connection *conn; + + cbd = g_queue_pop_head (conns); + event_del (&cbd->ev); + conn = cbd->conn; + g_free (cbd); + + /* We transfer refcount here! */ + return conn; + } + } + + return NULL; +} + +void +rspamd_http_context_prepare_keepalive (struct rspamd_http_context *ctx, + struct rspamd_http_connection *conn, + const rspamd_inet_addr_t *addr, + const gchar *host) +{ + struct rspamd_keepalive_hash_key hk, *phk; + khiter_t k; + + hk.addr = (rspamd_inet_addr_t *)addr; + hk.host = (gchar *)host; + + k = kh_get (rspamd_keep_alive_hash, ctx->keep_alive_hash, &hk); + + if (k != kh_end (ctx->keep_alive_hash)) { + /* Reuse existing */ + conn->keepalive_hash_key = kh_key (ctx->keep_alive_hash, k); + } + else { + /* Create new one */ + GQueue empty_init = G_QUEUE_INIT; + gint r; + + phk = g_malloc (sizeof (*phk)); + phk->conns = empty_init; + phk->host = g_strdup (host); + phk->addr = rspamd_inet_address_copy (addr); + + kh_put (rspamd_keep_alive_hash, ctx->keep_alive_hash, phk, &r); + conn->keepalive_hash_key = phk; + } +} + +static void +rspamd_http_keepalive_handler (gint fd, short what, gpointer ud) +{ + struct rspamd_http_keepalive_cbdata *cbdata = + (struct rspamd_http_keepalive_cbdata *)ud; + /* + * We can get here if a remote side reported something or it has + * timed out. In both cases we just terminate keepalive connection. + */ + + g_queue_delete_link (cbdata->queue, cbdata->link); + rspamd_http_connection_unref (cbdata->conn); + g_free (cbdata); +} + +void +rspamd_http_context_push_keepalive (struct rspamd_http_context *ctx, + struct rspamd_http_connection *conn, + struct rspamd_http_message *msg, + struct event_base *ev_base) +{ + struct rspamd_http_keepalive_cbdata *cbdata; + struct timeval tv; + gdouble timeout = ctx->config.keepalive_interval; + + g_assert (conn->keepalive_hash_key != NULL); + + /* Move connection to the keepalive pool */ + cbdata = g_malloc0 (sizeof (*cbdata)); + + cbdata->conn = rspamd_http_connection_ref (conn); + g_queue_push_tail (&conn->keepalive_hash_key->conns, cbdata); + cbdata->link = conn->keepalive_hash_key->conns.tail; + cbdata->queue = &conn->keepalive_hash_key->conns; + conn->finished = FALSE; + + event_set (&cbdata->ev, conn->fd, EV_READ|EV_TIMEOUT, + rspamd_http_keepalive_handler, + &cbdata); + + if (msg) { + const rspamd_ftok_t *tok; + + tok = rspamd_http_message_find_header (msg, "Keep-Alive"); + + if (tok) { + goffset pos = rspamd_substring_search_caseless (tok->begin, + tok->len, "timeout=", sizeof ("timeout=") - 1); + + if (pos != -1) { + pos += sizeof ("timeout="); + + gchar *end_pos = memchr (tok->begin + pos, ',', tok->len - pos); + glong real_timeout; + + if (end_pos) { + if (rspamd_strtol (tok->begin + pos + 1, + (end_pos - tok->begin) - pos - 1, &real_timeout) && + real_timeout > 0) { + timeout = real_timeout; + } + } + else { + if (rspamd_strtol (tok->begin + pos + 1, + tok->len - pos - 1, &real_timeout) && + real_timeout > 0) { + timeout = real_timeout; + } + } + } + } + } + + double_to_tv (timeout, &tv); + event_base_set (ev_base, &cbdata->ev); + event_add (&cbdata->ev, &tv); +}
\ No newline at end of file diff --git a/src/libutil/http_context.h b/src/libutil/http_context.h new file mode 100644 index 000000000..74e5c69a6 --- /dev/null +++ b/src/libutil/http_context.h @@ -0,0 +1,95 @@ +/*- + * Copyright 2019 Vsevolod Stakhov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RSPAMD_HTTP_CONTEXT_H +#define RSPAMD_HTTP_CONTEXT_H + +#include "config.h" +#include "ucl.h" +#include "addr.h" + +#include <event.h> + +struct rspamd_http_context; +struct rspamd_config; +struct rspamd_http_message; + +struct rspamd_http_context_cfg { + guint kp_cache_size_client; + guint kp_cache_size_server; + guint ssl_cache_size; + gdouble keepalive_interval; + gdouble client_key_rotate_time; + const gchar *user_agent; +}; + +/** + * Creates and configures new HTTP context + * @param root_conf configuration object + * @param ev_base event base + * @return new context used for both client and server HTTP connections + */ +struct rspamd_http_context* rspamd_http_context_create (struct rspamd_config *cfg, + struct event_base *ev_base); + +struct rspamd_http_context* rspamd_http_context_create_config ( + struct rspamd_http_context_cfg *cfg, + struct event_base *ev_base); +/** + * Destroys context + * @param ctx + */ +void rspamd_http_context_free (struct rspamd_http_context *ctx); + +struct rspamd_http_context* rspamd_http_context_default (void); + +/** + * Returns preserved keepalive connection if it's available. + * Refcount is transferred to caller! + * @param ctx + * @param addr + * @param host + * @return + */ +struct rspamd_http_connection* rspamd_http_context_check_keepalive ( + struct rspamd_http_context *ctx, const rspamd_inet_addr_t *addr, + const gchar *host); + +/** + * Prepares keepalive key for a connection by creating a new entry or by reusing existent + * Bear in mind, that keepalive pool has currently no cleanup methods! + * @param ctx + * @param conn + * @param addr + * @param host + */ +void rspamd_http_context_prepare_keepalive (struct rspamd_http_context *ctx, + struct rspamd_http_connection *conn, + const rspamd_inet_addr_t *addr, + const gchar *host); +/** + * Pushes a connection to keepalive pool after client request is finished, + * keepalive key *must* be prepared before using of this function + * @param ctx + * @param conn + * @param msg + */ +void rspamd_http_context_push_keepalive (struct rspamd_http_context *ctx, + struct rspamd_http_connection *conn, + struct rspamd_http_message *msg, + struct event_base *ev_base); + +#endif diff --git a/src/libutil/http_message.c b/src/libutil/http_message.c new file mode 100644 index 000000000..0720dc416 --- /dev/null +++ b/src/libutil/http_message.c @@ -0,0 +1,660 @@ +/*- + * Copyright 2019 Vsevolod Stakhov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "http_message.h" +#include "libutil/http_connection.h" +#include "libutil/http_private.h" +#include "libutil/printf.h" +#include "libutil/logger.h" +#include "utlist.h" +#include "unix-std.h" + +struct rspamd_http_message * +rspamd_http_new_message (enum rspamd_http_message_type type) +{ + struct rspamd_http_message *new; + + new = g_malloc0 (sizeof (struct rspamd_http_message)); + + if (type == HTTP_REQUEST) { + new->url = rspamd_fstring_new (); + } + else { + new->url = NULL; + new->code = 200; + } + + new->port = 80; + new->type = type; + new->method = HTTP_INVALID; + + REF_INIT_RETAIN (new, rspamd_http_message_free); + + return new; +} + +struct rspamd_http_message* +rspamd_http_message_from_url (const gchar *url) +{ + struct http_parser_url pu; + struct rspamd_http_message *msg; + const gchar *host, *path; + size_t pathlen, urllen; + guint flags = 0; + + if (url == NULL) { + return NULL; + } + + urllen = strlen (url); + memset (&pu, 0, sizeof (pu)); + + if (http_parser_parse_url (url, urllen, FALSE, &pu) != 0) { + msg_warn ("cannot parse URL: %s", url); + return NULL; + } + + if ((pu.field_set & (1 << UF_HOST)) == 0) { + msg_warn ("no host argument in URL: %s", url); + return NULL; + } + + if ((pu.field_set & (1 << UF_SCHEMA))) { + if (pu.field_data[UF_SCHEMA].len == sizeof ("https") - 1 && + memcmp (url + pu.field_data[UF_SCHEMA].off, "https", 5) == 0) { + flags |= RSPAMD_HTTP_FLAG_SSL; + } + } + + if ((pu.field_set & (1 << UF_PATH)) == 0) { + path = "/"; + pathlen = 1; + } + else { + path = url + pu.field_data[UF_PATH].off; + pathlen = urllen - pu.field_data[UF_PATH].off; + } + + msg = rspamd_http_new_message (HTTP_REQUEST); + host = url + pu.field_data[UF_HOST].off; + msg->flags = flags; + + if ((pu.field_set & (1 << UF_PORT)) != 0) { + msg->port = pu.port; + } + else { + /* XXX: magic constant */ + if (flags & RSPAMD_HTTP_FLAG_SSL) { + msg->port = 443; + } + else { + msg->port = 80; + } + } + + msg->host = rspamd_fstring_new_init (host, pu.field_data[UF_HOST].len); + msg->url = rspamd_fstring_append (msg->url, path, pathlen); + + REF_INIT_RETAIN (msg, rspamd_http_message_free); + + return msg; +} + +const gchar * +rspamd_http_message_get_body (struct rspamd_http_message *msg, + gsize *blen) +{ + const gchar *ret = NULL; + + if (msg->body_buf.len > 0) { + ret = msg->body_buf.begin; + } + + if (blen) { + *blen = msg->body_buf.len; + } + + return ret; +} + +static void +rspamd_http_shname_dtor (void *p) +{ + struct rspamd_storage_shmem *n = p; + +#ifdef HAVE_SANE_SHMEM + shm_unlink (n->shm_name); +#else + unlink (n->shm_name); +#endif + g_free (n->shm_name); + g_free (n); +} + +struct rspamd_storage_shmem * +rspamd_http_message_shmem_ref (struct rspamd_http_message *msg) +{ + if ((msg->flags & RSPAMD_HTTP_FLAG_SHMEM) && msg->body_buf.c.shared.name) { + REF_RETAIN (msg->body_buf.c.shared.name); + return msg->body_buf.c.shared.name; + } + + return NULL; +} + +guint +rspamd_http_message_get_flags (struct rspamd_http_message *msg) +{ + return msg->flags; +} + +void +rspamd_http_message_shmem_unref (struct rspamd_storage_shmem *p) +{ + REF_RELEASE (p); +} + +gboolean +rspamd_http_message_set_body (struct rspamd_http_message *msg, + const gchar *data, gsize len) +{ + union _rspamd_storage_u *storage; + storage = &msg->body_buf.c; + + rspamd_http_message_storage_cleanup (msg); + + if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) { + storage->shared.name = g_malloc (sizeof (*storage->shared.name)); + REF_INIT_RETAIN (storage->shared.name, rspamd_http_shname_dtor); +#ifdef HAVE_SANE_SHMEM + #if defined(__DragonFly__) + // DragonFly uses regular files for shm. User rspamd is not allowed to create + // files in the root. + storage->shared.name->shm_name = g_strdup ("/tmp/rhm.XXXXXXXXXXXXXXXXXXXX"); +#else + storage->shared.name->shm_name = g_strdup ("/rhm.XXXXXXXXXXXXXXXXXXXX"); +#endif + storage->shared.shm_fd = rspamd_shmem_mkstemp (storage->shared.name->shm_name); +#else + /* XXX: assume that tempdir is /tmp */ + storage->shared.name->shm_name = g_strdup ("/tmp/rhm.XXXXXXXXXXXXXXXXXXXX"); + storage->shared.shm_fd = mkstemp (storage->shared.name->shm_name); +#endif + + if (storage->shared.shm_fd == -1) { + return FALSE; + } + + if (len != 0 && len != ULLONG_MAX) { + if (ftruncate (storage->shared.shm_fd, len) == -1) { + return FALSE; + } + + msg->body_buf.str = mmap (NULL, len, + PROT_WRITE|PROT_READ, MAP_SHARED, + storage->shared.shm_fd, 0); + + if (msg->body_buf.str == MAP_FAILED) { + return FALSE; + } + + msg->body_buf.begin = msg->body_buf.str; + msg->body_buf.allocated_len = len; + + if (data != NULL) { + memcpy (msg->body_buf.str, data, len); + msg->body_buf.len = len; + } + } + else { + msg->body_buf.len = 0; + msg->body_buf.begin = NULL; + msg->body_buf.str = NULL; + msg->body_buf.allocated_len = 0; + } + } + else { + if (len != 0 && len != ULLONG_MAX) { + if (data == NULL) { + storage->normal = rspamd_fstring_sized_new (len); + msg->body_buf.len = 0; + } + else { + storage->normal = rspamd_fstring_new_init (data, len); + msg->body_buf.len = len; + } + } + else { + storage->normal = rspamd_fstring_new (); + } + + msg->body_buf.begin = storage->normal->str; + msg->body_buf.str = storage->normal->str; + msg->body_buf.allocated_len = storage->normal->allocated; + } + + msg->flags |= RSPAMD_HTTP_FLAG_HAS_BODY; + + return TRUE; +} + +void +rspamd_http_message_set_method (struct rspamd_http_message *msg, + const gchar *method) +{ + gint i; + + /* Linear search: not very efficient method */ + for (i = 0; i < HTTP_METHOD_MAX; i ++) { + if (g_ascii_strcasecmp (method, http_method_str (i)) == 0) { + msg->method = i; + } + } +} + +gboolean +rspamd_http_message_set_body_from_fd (struct rspamd_http_message *msg, + gint fd) +{ + union _rspamd_storage_u *storage; + struct stat st; + + rspamd_http_message_storage_cleanup (msg); + + storage = &msg->body_buf.c; + msg->flags |= RSPAMD_HTTP_FLAG_SHMEM|RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE; + + storage->shared.shm_fd = dup (fd); + msg->body_buf.str = MAP_FAILED; + + if (storage->shared.shm_fd == -1) { + return FALSE; + } + + if (fstat (storage->shared.shm_fd, &st) == -1) { + return FALSE; + } + + msg->body_buf.str = mmap (NULL, st.st_size, + PROT_READ, MAP_SHARED, + storage->shared.shm_fd, 0); + + if (msg->body_buf.str == MAP_FAILED) { + return FALSE; + } + + msg->body_buf.begin = msg->body_buf.str; + msg->body_buf.len = st.st_size; + msg->body_buf.allocated_len = st.st_size; + + return TRUE; +} + +gboolean +rspamd_http_message_set_body_from_fstring_steal (struct rspamd_http_message *msg, + rspamd_fstring_t *fstr) +{ + union _rspamd_storage_u *storage; + + rspamd_http_message_storage_cleanup (msg); + + storage = &msg->body_buf.c; + msg->flags &= ~(RSPAMD_HTTP_FLAG_SHMEM|RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE); + + storage->normal = fstr; + msg->body_buf.str = fstr->str; + msg->body_buf.begin = msg->body_buf.str; + msg->body_buf.len = fstr->len; + msg->body_buf.allocated_len = fstr->allocated; + + return TRUE; +} + +gboolean +rspamd_http_message_set_body_from_fstring_copy (struct rspamd_http_message *msg, + const rspamd_fstring_t *fstr) +{ + union _rspamd_storage_u *storage; + + rspamd_http_message_storage_cleanup (msg); + + storage = &msg->body_buf.c; + msg->flags &= ~(RSPAMD_HTTP_FLAG_SHMEM|RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE); + + storage->normal = rspamd_fstring_new_init (fstr->str, fstr->len); + msg->body_buf.str = storage->normal->str; + msg->body_buf.begin = msg->body_buf.str; + msg->body_buf.len = storage->normal->len; + msg->body_buf.allocated_len = storage->normal->allocated; + + return TRUE; +} + + +gboolean +rspamd_http_message_grow_body (struct rspamd_http_message *msg, gsize len) +{ + struct stat st; + union _rspamd_storage_u *storage; + gsize newlen; + + storage = &msg->body_buf.c; + + if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) { + if (storage->shared.shm_fd == -1) { + return FALSE; + } + + if (fstat (storage->shared.shm_fd, &st) == -1) { + return FALSE; + } + + /* Check if we need to grow */ + if ((gsize)st.st_size < msg->body_buf.len + len) { + /* Need to grow */ + newlen = rspamd_fstring_suggest_size (msg->body_buf.len, st.st_size, + len); + /* Unmap as we need another size of segment */ + if (msg->body_buf.str != MAP_FAILED) { + munmap (msg->body_buf.str, st.st_size); + } + + if (ftruncate (storage->shared.shm_fd, newlen) == -1) { + return FALSE; + } + + msg->body_buf.str = mmap (NULL, newlen, + PROT_WRITE|PROT_READ, MAP_SHARED, + storage->shared.shm_fd, 0); + if (msg->body_buf.str == MAP_FAILED) { + return FALSE; + } + + msg->body_buf.begin = msg->body_buf.str; + msg->body_buf.allocated_len = newlen; + } + } + else { + storage->normal = rspamd_fstring_grow (storage->normal, len); + + /* Append might cause realloc */ + msg->body_buf.begin = storage->normal->str; + msg->body_buf.len = storage->normal->len; + msg->body_buf.str = storage->normal->str; + msg->body_buf.allocated_len = storage->normal->allocated; + } + + return TRUE; +} + +gboolean +rspamd_http_message_append_body (struct rspamd_http_message *msg, + const gchar *data, gsize len) +{ + union _rspamd_storage_u *storage; + + storage = &msg->body_buf.c; + + if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) { + if (!rspamd_http_message_grow_body (msg, len)) { + return FALSE; + } + + memcpy (msg->body_buf.str + msg->body_buf.len, data, len); + msg->body_buf.len += len; + } + else { + storage->normal = rspamd_fstring_append (storage->normal, data, len); + + /* Append might cause realloc */ + msg->body_buf.begin = storage->normal->str; + msg->body_buf.len = storage->normal->len; + msg->body_buf.str = storage->normal->str; + msg->body_buf.allocated_len = storage->normal->allocated; + } + + return TRUE; +} + +void +rspamd_http_message_storage_cleanup (struct rspamd_http_message *msg) +{ + union _rspamd_storage_u *storage; + struct stat st; + + if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) { + storage = &msg->body_buf.c; + + if (storage->shared.shm_fd > 0) { + g_assert (fstat (storage->shared.shm_fd, &st) != -1); + + if (msg->body_buf.str != MAP_FAILED) { + munmap (msg->body_buf.str, st.st_size); + } + + close (storage->shared.shm_fd); + } + + if (storage->shared.name != NULL) { + REF_RELEASE (storage->shared.name); + } + + storage->shared.shm_fd = -1; + msg->body_buf.str = MAP_FAILED; + } + else { + if (msg->body_buf.c.normal) { + rspamd_fstring_free (msg->body_buf.c.normal); + } + + msg->body_buf.c.normal = NULL; + } + + msg->body_buf.len = 0; +} + +void +rspamd_http_message_free (struct rspamd_http_message *msg) +{ + struct rspamd_http_header *hdr, *htmp, *hcur, *hcurtmp; + + + HASH_ITER (hh, msg->headers, hdr, htmp) { + HASH_DEL (msg->headers, hdr); + + DL_FOREACH_SAFE (hdr, hcur, hcurtmp) { + rspamd_fstring_free (hcur->combined); + g_free (hcur); + } + } + + rspamd_http_message_storage_cleanup (msg); + + if (msg->url != NULL) { + rspamd_fstring_free (msg->url); + } + if (msg->status != NULL) { + rspamd_fstring_free (msg->status); + } + if (msg->host != NULL) { + rspamd_fstring_free (msg->host); + } + if (msg->peer_key != NULL) { + rspamd_pubkey_unref (msg->peer_key); + } + + g_free (msg); +} + +void +rspamd_http_message_set_peer_key (struct rspamd_http_message *msg, + struct rspamd_cryptobox_pubkey *pk) +{ + if (msg->peer_key != NULL) { + rspamd_pubkey_unref (msg->peer_key); + } + + if (pk) { + msg->peer_key = rspamd_pubkey_ref (pk); + } + else { + msg->peer_key = NULL; + } +} + +void +rspamd_http_message_add_header_len (struct rspamd_http_message *msg, + const gchar *name, + const gchar *value, + gsize len) +{ + struct rspamd_http_header *hdr, *found = NULL; + guint nlen, vlen; + + if (msg != NULL && name != NULL && value != NULL) { + hdr = g_malloc0 (sizeof (struct rspamd_http_header)); + nlen = strlen (name); + vlen = len; + hdr->combined = rspamd_fstring_sized_new (nlen + vlen + 4); + rspamd_printf_fstring (&hdr->combined, "%s: %*s\r\n", name, (gint)vlen, + value); + hdr->name.begin = hdr->combined->str; + hdr->name.len = nlen; + hdr->value.begin = hdr->combined->str + nlen + 2; + hdr->value.len = vlen; + + HASH_FIND (hh, msg->headers, hdr->name.begin, + hdr->name.len, found); + + if (found == NULL) { + HASH_ADD_KEYPTR (hh, msg->headers, hdr->name.begin, + hdr->name.len, hdr); + } + + DL_APPEND (found, hdr); + } +} + +void +rspamd_http_message_add_header (struct rspamd_http_message *msg, + const gchar *name, + const gchar *value) +{ + if (value) { + rspamd_http_message_add_header_len (msg, name, value, strlen (value)); + } +} + +void +rspamd_http_message_add_header_fstr (struct rspamd_http_message *msg, + const gchar *name, + rspamd_fstring_t *value) +{ + struct rspamd_http_header *hdr, *found = NULL; + guint nlen, vlen; + + if (msg != NULL && name != NULL && value != NULL) { + hdr = g_malloc0 (sizeof (struct rspamd_http_header)); + nlen = strlen (name); + vlen = value->len; + hdr->combined = rspamd_fstring_sized_new (nlen + vlen + 4); + rspamd_printf_fstring (&hdr->combined, "%s: %V\r\n", name, value); + hdr->name.begin = hdr->combined->str; + hdr->name.len = nlen; + hdr->value.begin = hdr->combined->str + nlen + 2; + hdr->value.len = vlen; + + HASH_FIND (hh, msg->headers, hdr->name.begin, + hdr->name.len, found); + + if (found == NULL) { + HASH_ADD_KEYPTR (hh, msg->headers, hdr->name.begin, + hdr->name.len, hdr); + } + + DL_APPEND (found, hdr); + } +} + +const rspamd_ftok_t * +rspamd_http_message_find_header (struct rspamd_http_message *msg, + const gchar *name) +{ + struct rspamd_http_header *hdr; + const rspamd_ftok_t *res = NULL; + guint slen = strlen (name); + + if (msg != NULL) { + HASH_FIND (hh, msg->headers, name, slen, hdr); + + if (hdr) { + res = &hdr->value; + } + } + + return res; +} + +GPtrArray* +rspamd_http_message_find_header_multiple ( + struct rspamd_http_message *msg, + const gchar *name) +{ + GPtrArray *res = NULL; + struct rspamd_http_header *hdr, *cur; + + guint slen = strlen (name); + + if (msg != NULL) { + HASH_FIND (hh, msg->headers, name, slen, hdr); + + if (hdr) { + res = g_ptr_array_sized_new (4); + + LL_FOREACH (hdr, cur) { + g_ptr_array_add (res, &cur->value); + } + } + } + + + return res; +} + + +gboolean +rspamd_http_message_remove_header (struct rspamd_http_message *msg, + const gchar *name) +{ + struct rspamd_http_header *hdr, *hcur, *hcurtmp; + gboolean res = FALSE; + guint slen = strlen (name); + + if (msg != NULL) { + HASH_FIND (hh, msg->headers, name, slen, hdr); + + if (hdr) { + HASH_DEL (msg->headers, hdr); + res = TRUE; + + DL_FOREACH_SAFE (hdr, hcur, hcurtmp) { + rspamd_fstring_free (hcur->combined); + g_free (hcur); + } + } + } + + return res; +}
\ No newline at end of file diff --git a/src/libutil/http_message.h b/src/libutil/http_message.h new file mode 100644 index 000000000..c9e6abfce --- /dev/null +++ b/src/libutil/http_message.h @@ -0,0 +1,221 @@ +/*- + * Copyright 2019 Vsevolod Stakhov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RSPAMD_HTTP_MESSAGE_H +#define RSPAMD_HTTP_MESSAGE_H + +#include "config.h" +#include "keypair.h" +#include "keypairs_cache.h" +#include "fstring.h" +#include "ref.h" + +struct rspamd_http_connection; + +enum rspamd_http_message_type { HTTP_REQUEST = 0, HTTP_RESPONSE }; + +/** + * Extract the current message from a connection to deal with separately + * @param conn + * @return + */ +struct rspamd_http_message * rspamd_http_connection_steal_msg ( + struct rspamd_http_connection *conn); + +/** + * Copy the current message from a connection to deal with separately + * @param conn + * @return + */ +struct rspamd_http_message * rspamd_http_connection_copy_msg ( + struct rspamd_http_message *msg, GError **err); + +/** + * Create new HTTP message + * @param type request or response + * @return new http message + */ +struct rspamd_http_message * rspamd_http_new_message (enum rspamd_http_message_type type); + +/** + * Increase refcount number for an HTTP message + * @param msg message to use + * @return + */ +struct rspamd_http_message * rspamd_http_message_ref (struct rspamd_http_message *msg); +/** + * Decrease number of refcounts for http message + * @param msg + */ +void rspamd_http_message_unref (struct rspamd_http_message *msg); + +/** + * Sets a key for peer + * @param msg + * @param pk + */ +void rspamd_http_message_set_peer_key (struct rspamd_http_message *msg, + struct rspamd_cryptobox_pubkey *pk); +/** + * Create HTTP message from URL + * @param url + * @return new message or NULL + */ +struct rspamd_http_message* rspamd_http_message_from_url (const gchar *url); + +/** + * Returns body for a message + * @param msg + * @param blen pointer where to save body length + * @return pointer to body start + */ +const gchar *rspamd_http_message_get_body (struct rspamd_http_message *msg, + gsize *blen); + +/** + * Set message's body from the string + * @param msg + * @param data + * @param len + * @return TRUE if a message's body has been set + */ +gboolean rspamd_http_message_set_body (struct rspamd_http_message *msg, + const gchar *data, gsize len); + +/** + * Set message's method by name + * @param msg + * @param method + */ +void rspamd_http_message_set_method (struct rspamd_http_message *msg, + const gchar *method); +/** + * Maps fd as message's body + * @param msg + * @param fd + * @return TRUE if a message's body has been set + */ +gboolean rspamd_http_message_set_body_from_fd (struct rspamd_http_message *msg, + gint fd); + +/** + * Uses rspamd_fstring_t as message's body, string is consumed by this operation + * @param msg + * @param fstr + * @return TRUE if a message's body has been set + */ +gboolean rspamd_http_message_set_body_from_fstring_steal (struct rspamd_http_message *msg, + rspamd_fstring_t *fstr); + +/** + * Uses rspamd_fstring_t as message's body, string is copied by this operation + * @param msg + * @param fstr + * @return TRUE if a message's body has been set + */ +gboolean rspamd_http_message_set_body_from_fstring_copy (struct rspamd_http_message *msg, + const rspamd_fstring_t *fstr); + +/** + * Appends data to message's body + * @param msg + * @param data + * @param len + * @return TRUE if a message's body has been set + */ +gboolean rspamd_http_message_append_body (struct rspamd_http_message *msg, + const gchar *data, gsize len); + +/** + * Append a header to http message + * @param rep + * @param name + * @param value + */ +void rspamd_http_message_add_header (struct rspamd_http_message *msg, + const gchar *name, + const gchar *value); + +void rspamd_http_message_add_header_len (struct rspamd_http_message *msg, + const gchar *name, + const gchar *value, + gsize len); + +void rspamd_http_message_add_header_fstr (struct rspamd_http_message *msg, + const gchar *name, + rspamd_fstring_t *value); + +/** + * Search for a specified header in message + * @param msg message + * @param name name of header + */ +const rspamd_ftok_t * rspamd_http_message_find_header ( + struct rspamd_http_message *msg, + const gchar *name); + +/** + * Search for a header that has multiple values + * @param msg + * @param name + * @return list of rspamd_ftok_t * with values + */ +GPtrArray* rspamd_http_message_find_header_multiple ( + struct rspamd_http_message *msg, + const gchar *name); + +/** + * Remove specific header from a message + * @param msg + * @param name + * @return + */ +gboolean rspamd_http_message_remove_header (struct rspamd_http_message *msg, + const gchar *name); + +/** + * Free HTTP message + * @param msg + */ +void rspamd_http_message_free (struct rspamd_http_message *msg); + +/** + * Extract arguments from a message's URI contained inside query string decoding + * them if needed + * @param msg HTTP request message + * @return new GHashTable which maps rspamd_ftok_t* to rspamd_ftok_t* + * (table must be freed by a caller) + */ +GHashTable* rspamd_http_message_parse_query (struct rspamd_http_message *msg); + +/** + * Increase refcount for shared file (if any) to prevent early memory unlinking + * @param msg + */ +struct rspamd_storage_shmem* rspamd_http_message_shmem_ref (struct rspamd_http_message *msg); +/** + * Decrease external ref for shmem segment associated with a message + * @param msg + */ +void rspamd_http_message_shmem_unref (struct rspamd_storage_shmem *p); + +/** + * Returns message's flags + * @param msg + * @return + */ +guint rspamd_http_message_get_flags (struct rspamd_http_message *msg); + +#endif diff --git a/src/libutil/http_private.h b/src/libutil/http_private.h index 0e2658617..dd3d0c6a9 100644 --- a/src/libutil/http_private.h +++ b/src/libutil/http_private.h @@ -16,9 +16,13 @@ #ifndef SRC_LIBUTIL_HTTP_PRIVATE_H_ #define SRC_LIBUTIL_HTTP_PRIVATE_H_ -#include "http.h" +#include "http_connection.h" +#include "http_parser.h" #include "str_util.h" +#include "keypair.h" +#include "keypairs_cache.h" #include "ref.h" +#include "khash.h" #define HASH_CASELESS #include "uthash_strcase.h" @@ -66,12 +70,43 @@ struct rspamd_http_message { time_t date; time_t last_modified; unsigned port; - enum http_parser_type type; + int type; gint code; enum http_method method; gint flags; ref_entry_t ref; }; +struct rspamd_keepalive_hash_key { + rspamd_inet_addr_t *addr; + gchar *host; + GQueue conns; +}; + +gint32 rspamd_keep_alive_key_hash (struct rspamd_keepalive_hash_key* k); +bool rspamd_keep_alive_key_equal (struct rspamd_keepalive_hash_key* k1, + struct rspamd_keepalive_hash_key* k2); + +KHASH_INIT (rspamd_keep_alive_hash, struct rspamd_keepalive_hash_key *, + char, 0, rspamd_keep_alive_key_hash, rspamd_keep_alive_key_equal); + +struct rspamd_http_context { + struct rspamd_http_context_cfg config; + struct rspamd_keypair_cache *client_kp_cache; + struct rspamd_cryptobox_keypair *client_kp; + struct rspamd_keypair_cache *server_kp_cache; + gpointer ssl_ctx; + gpointer ssl_ctx_noverify; + struct event_base *ev_base; + struct event client_rotate_ev; + khash_t (rspamd_keep_alive_hash) *keep_alive_hash; +}; + +#define HTTP_ERROR http_error_quark () +GQuark http_error_quark (void); + +void rspamd_http_message_storage_cleanup (struct rspamd_http_message *msg); +gboolean rspamd_http_message_grow_body (struct rspamd_http_message *msg, + gsize len); #endif /* SRC_LIBUTIL_HTTP_PRIVATE_H_ */ diff --git a/src/libutil/http_router.c b/src/libutil/http_router.c new file mode 100644 index 000000000..570d3d5c6 --- /dev/null +++ b/src/libutil/http_router.c @@ -0,0 +1,555 @@ +/*- + * Copyright 2019 Vsevolod Stakhov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "libutil/http_router.h" +#include "libutil/http_connection.h" +#include "libutil/http_private.h" +#include "libutil/regexp.h" +#include "libutil/printf.h" +#include "libutil/logger.h" +#include "utlist.h" +#include "unix-std.h" + +enum http_magic_type { + HTTP_MAGIC_PLAIN = 0, + HTTP_MAGIC_HTML, + HTTP_MAGIC_CSS, + HTTP_MAGIC_JS, + HTTP_MAGIC_PNG, + HTTP_MAGIC_JPG +}; + +static const struct _rspamd_http_magic { + const gchar *ext; + const gchar *ct; +} http_file_types[] = { + [HTTP_MAGIC_PLAIN] = { "txt", "text/plain" }, + [HTTP_MAGIC_HTML] = { "html", "text/html" }, + [HTTP_MAGIC_CSS] = { "css", "text/css" }, + [HTTP_MAGIC_JS] = { "js", "application/javascript" }, + [HTTP_MAGIC_PNG] = { "png", "image/png" }, + [HTTP_MAGIC_JPG] = { "jpg", "image/jpeg" }, +}; + +/* + * HTTP router functions + */ + +static void +rspamd_http_entry_free (struct rspamd_http_connection_entry *entry) +{ + if (entry != NULL) { + close (entry->conn->fd); + rspamd_http_connection_unref (entry->conn); + if (entry->rt->finish_handler) { + entry->rt->finish_handler (entry); + } + + DL_DELETE (entry->rt->conns, entry); + g_free (entry); + } +} + +static void +rspamd_http_router_error_handler (struct rspamd_http_connection *conn, + GError *err) +{ + struct rspamd_http_connection_entry *entry = conn->ud; + struct rspamd_http_message *msg; + + if (entry->is_reply) { + /* At this point we need to finish this session and close owned socket */ + if (entry->rt->error_handler != NULL) { + entry->rt->error_handler (entry, err); + } + rspamd_http_entry_free (entry); + } + else { + /* Here we can write a reply to a client */ + if (entry->rt->error_handler != NULL) { + entry->rt->error_handler (entry, err); + } + msg = rspamd_http_new_message (HTTP_RESPONSE); + msg->date = time (NULL); + msg->code = err->code; + rspamd_http_message_set_body (msg, err->message, strlen (err->message)); + rspamd_http_connection_reset (entry->conn); + rspamd_http_connection_write_message (entry->conn, + msg, + NULL, + "text/plain", + entry, + entry->rt->ptv); + entry->is_reply = TRUE; + } +} + +static const gchar * +rspamd_http_router_detect_ct (const gchar *path) +{ + const gchar *dot; + guint i; + + dot = strrchr (path, '.'); + if (dot == NULL) { + return http_file_types[HTTP_MAGIC_PLAIN].ct; + } + dot++; + + for (i = 0; i < G_N_ELEMENTS (http_file_types); i++) { + if (strcmp (http_file_types[i].ext, dot) == 0) { + return http_file_types[i].ct; + } + } + + return http_file_types[HTTP_MAGIC_PLAIN].ct; +} + +static gboolean +rspamd_http_router_is_subdir (const gchar *parent, const gchar *sub) +{ + if (parent == NULL || sub == NULL || *parent == '\0') { + return FALSE; + } + + while (*parent != '\0') { + if (*sub != *parent) { + return FALSE; + } + parent++; + sub++; + } + + parent--; + if (*parent == G_DIR_SEPARATOR) { + return TRUE; + } + + return (*sub == G_DIR_SEPARATOR || *sub == '\0'); +} + +static gboolean +rspamd_http_router_try_file (struct rspamd_http_connection_entry *entry, + rspamd_ftok_t *lookup, gboolean expand_path) +{ + struct stat st; + gint fd; + gchar filebuf[PATH_MAX], realbuf[PATH_MAX], *dir; + struct rspamd_http_message *reply_msg; + + rspamd_snprintf (filebuf, sizeof (filebuf), "%s%c%T", + entry->rt->default_fs_path, G_DIR_SEPARATOR, lookup); + + if (realpath (filebuf, realbuf) == NULL || + lstat (realbuf, &st) == -1) { + return FALSE; + } + + if (S_ISDIR (st.st_mode) && expand_path) { + /* Try to append 'index.html' to the url */ + rspamd_fstring_t *nlookup; + rspamd_ftok_t tok; + gboolean ret; + + nlookup = rspamd_fstring_sized_new (lookup->len + sizeof ("index.html")); + rspamd_printf_fstring (&nlookup, "%T%c%s", lookup, G_DIR_SEPARATOR, + "index.html"); + tok.begin = nlookup->str; + tok.len = nlookup->len; + ret = rspamd_http_router_try_file (entry, &tok, FALSE); + rspamd_fstring_free (nlookup); + + return ret; + } + else if (!S_ISREG (st.st_mode)) { + return FALSE; + } + + /* We also need to ensure that file is inside the defined dir */ + rspamd_strlcpy (filebuf, realbuf, sizeof (filebuf)); + dir = dirname (filebuf); + + if (dir == NULL || + !rspamd_http_router_is_subdir (entry->rt->default_fs_path, + dir)) { + return FALSE; + } + + fd = open (realbuf, O_RDONLY); + if (fd == -1) { + return FALSE; + } + + reply_msg = rspamd_http_new_message (HTTP_RESPONSE); + reply_msg->date = time (NULL); + reply_msg->code = 200; + rspamd_http_router_insert_headers (entry->rt, reply_msg); + + if (!rspamd_http_message_set_body_from_fd (reply_msg, fd)) { + close (fd); + return FALSE; + } + + close (fd); + + rspamd_http_connection_reset (entry->conn); + + msg_debug ("requested file %s", realbuf); + rspamd_http_connection_write_message (entry->conn, reply_msg, NULL, + rspamd_http_router_detect_ct (realbuf), entry, + entry->rt->ptv); + + return TRUE; +} + +static void +rspamd_http_router_send_error (GError *err, + struct rspamd_http_connection_entry *entry) +{ + struct rspamd_http_message *err_msg; + + err_msg = rspamd_http_new_message (HTTP_RESPONSE); + err_msg->date = time (NULL); + err_msg->code = err->code; + rspamd_http_message_set_body (err_msg, err->message, + strlen (err->message)); + entry->is_reply = TRUE; + err_msg->status = rspamd_fstring_new_init (err->message, strlen (err->message)); + rspamd_http_router_insert_headers (entry->rt, err_msg); + rspamd_http_connection_reset (entry->conn); + rspamd_http_connection_write_message (entry->conn, + err_msg, + NULL, + "text/plain", + entry, + entry->rt->ptv); +} + + +static int +rspamd_http_router_finish_handler (struct rspamd_http_connection *conn, + struct rspamd_http_message *msg) +{ + struct rspamd_http_connection_entry *entry = conn->ud; + rspamd_http_router_handler_t handler = NULL; + gpointer found; + + GError *err; + rspamd_ftok_t lookup; + const rspamd_ftok_t *encoding; + struct http_parser_url u; + guint i; + rspamd_regexp_t *re; + struct rspamd_http_connection_router *router; + + G_STATIC_ASSERT (sizeof (rspamd_http_router_handler_t) == + sizeof (gpointer)); + + memset (&lookup, 0, sizeof (lookup)); + router = entry->rt; + + if (entry->is_reply) { + /* Request is finished, it is safe to free a connection */ + rspamd_http_entry_free (entry); + } + else { + if (G_UNLIKELY (msg->method != HTTP_GET && msg->method != HTTP_POST)) { + if (router->unknown_method_handler) { + return router->unknown_method_handler (entry, msg); + } + else { + err = g_error_new (HTTP_ERROR, 500, + "Invalid method"); + if (entry->rt->error_handler != NULL) { + entry->rt->error_handler (entry, err); + } + + rspamd_http_router_send_error (err, entry); + g_error_free (err); + + return 0; + } + } + + /* Search for path */ + if (msg->url != NULL && msg->url->len != 0) { + + http_parser_parse_url (msg->url->str, msg->url->len, TRUE, &u); + + if (u.field_set & (1 << UF_PATH)) { + guint unnorm_len; + lookup.begin = msg->url->str + u.field_data[UF_PATH].off; + lookup.len = u.field_data[UF_PATH].len; + + rspamd_http_normalize_path_inplace ((gchar *)lookup.begin, + lookup.len, + &unnorm_len); + lookup.len = unnorm_len; + } + else { + lookup.begin = msg->url->str; + lookup.len = msg->url->len; + } + + found = g_hash_table_lookup (entry->rt->paths, &lookup); + memcpy (&handler, &found, sizeof (found)); + msg_debug ("requested known path: %T", &lookup); + } + else { + err = g_error_new (HTTP_ERROR, 404, + "Empty path requested"); + if (entry->rt->error_handler != NULL) { + entry->rt->error_handler (entry, err); + } + + rspamd_http_router_send_error (err, entry); + g_error_free (err); + + return 0; + } + + entry->is_reply = TRUE; + + encoding = rspamd_http_message_find_header (msg, "Accept-Encoding"); + + if (encoding && rspamd_substring_search (encoding->begin, encoding->len, + "gzip", 4) != -1) { + entry->support_gzip = TRUE; + } + + if (handler != NULL) { + return handler (entry, msg); + } + else { + /* Try regexps */ + for (i = 0; i < router->regexps->len; i ++) { + re = g_ptr_array_index (router->regexps, i); + if (rspamd_regexp_match (re, lookup.begin, lookup.len, + TRUE)) { + found = rspamd_regexp_get_ud (re); + memcpy (&handler, &found, sizeof (found)); + + return handler (entry, msg); + } + } + + /* Now try plain file */ + if (entry->rt->default_fs_path == NULL || lookup.len == 0 || + !rspamd_http_router_try_file (entry, &lookup, TRUE)) { + + err = g_error_new (HTTP_ERROR, 404, + "Not found"); + if (entry->rt->error_handler != NULL) { + entry->rt->error_handler (entry, err); + } + + msg_info ("path: %T not found", &lookup); + rspamd_http_router_send_error (err, entry); + g_error_free (err); + } + } + } + + return 0; +} + +struct rspamd_http_connection_router * +rspamd_http_router_new (rspamd_http_router_error_handler_t eh, + rspamd_http_router_finish_handler_t fh, + struct timeval *timeout, + const char *default_fs_path, + struct rspamd_http_context *ctx) +{ + struct rspamd_http_connection_router * new; + struct stat st; + + new = g_malloc0 (sizeof (struct rspamd_http_connection_router)); + new->paths = g_hash_table_new_full (rspamd_ftok_icase_hash, + rspamd_ftok_icase_equal, rspamd_fstring_mapped_ftok_free, NULL); + new->regexps = g_ptr_array_new (); + new->conns = NULL; + new->error_handler = eh; + new->finish_handler = fh; + new->response_headers = g_hash_table_new_full (rspamd_strcase_hash, + rspamd_strcase_equal, g_free, g_free); + new->ev_base = ctx->ev_base; + + if (timeout) { + new->tv = *timeout; + new->ptv = &new->tv; + } + else { + new->ptv = NULL; + } + + new->default_fs_path = NULL; + + if (default_fs_path != NULL) { + if (stat (default_fs_path, &st) == -1) { + msg_err ("cannot stat %s", default_fs_path); + } + else { + if (!S_ISDIR (st.st_mode)) { + msg_err ("path %s is not a directory", default_fs_path); + } + else { + new->default_fs_path = realpath (default_fs_path, NULL); + } + } + } + + new->ctx = ctx; + + return new; +} + +void +rspamd_http_router_set_key (struct rspamd_http_connection_router *router, + struct rspamd_cryptobox_keypair *key) +{ + g_assert (key != NULL); + + router->key = rspamd_keypair_ref (key); +} + +void +rspamd_http_router_add_path (struct rspamd_http_connection_router *router, + const gchar *path, rspamd_http_router_handler_t handler) +{ + gpointer ptr; + rspamd_ftok_t *key; + rspamd_fstring_t *storage; + G_STATIC_ASSERT (sizeof (rspamd_http_router_handler_t) == + sizeof (gpointer)); + + if (path != NULL && handler != NULL && router != NULL) { + memcpy (&ptr, &handler, sizeof (ptr)); + storage = rspamd_fstring_new_init (path, strlen (path)); + key = g_malloc0 (sizeof (*key)); + key->begin = storage->str; + key->len = storage->len; + g_hash_table_insert (router->paths, key, ptr); + } +} + +void +rspamd_http_router_set_unknown_handler (struct rspamd_http_connection_router *router, + rspamd_http_router_handler_t handler) +{ + if (router != NULL) { + router->unknown_method_handler = handler; + } +} + +void +rspamd_http_router_add_header (struct rspamd_http_connection_router *router, + const gchar *name, const gchar *value) +{ + if (name != NULL && value != NULL && router != NULL) { + g_hash_table_replace (router->response_headers, g_strdup (name), + g_strdup (value)); + } +} + +void +rspamd_http_router_insert_headers (struct rspamd_http_connection_router *router, + struct rspamd_http_message *msg) +{ + GHashTableIter it; + gpointer k, v; + + if (router && msg) { + g_hash_table_iter_init (&it, router->response_headers); + + while (g_hash_table_iter_next (&it, &k, &v)) { + rspamd_http_message_add_header (msg, k, v); + } + } +} + +void +rspamd_http_router_add_regexp (struct rspamd_http_connection_router *router, + struct rspamd_regexp_s *re, rspamd_http_router_handler_t handler) +{ + gpointer ptr; + G_STATIC_ASSERT (sizeof (rspamd_http_router_handler_t) == + sizeof (gpointer)); + + if (re != NULL && handler != NULL && router != NULL) { + memcpy (&ptr, &handler, sizeof (ptr)); + rspamd_regexp_set_ud (re, ptr); + g_ptr_array_add (router->regexps, rspamd_regexp_ref (re)); + } +} + +void +rspamd_http_router_handle_socket (struct rspamd_http_connection_router *router, + gint fd, gpointer ud) +{ + struct rspamd_http_connection_entry *conn; + + conn = g_malloc0 (sizeof (struct rspamd_http_connection_entry)); + conn->rt = router; + conn->ud = ud; + conn->is_reply = FALSE; + + conn->conn = rspamd_http_connection_new (router->ctx, + fd, + NULL, + rspamd_http_router_error_handler, + rspamd_http_router_finish_handler, + 0, + RSPAMD_HTTP_SERVER); + + if (router->key) { + rspamd_http_connection_set_key (conn->conn, router->key); + } + + rspamd_http_connection_read_message (conn->conn, conn, router->ptv); + DL_PREPEND (router->conns, conn); +} + +void +rspamd_http_router_free (struct rspamd_http_connection_router *router) +{ + struct rspamd_http_connection_entry *conn, *tmp; + rspamd_regexp_t *re; + guint i; + + if (router) { + DL_FOREACH_SAFE (router->conns, conn, tmp) { + rspamd_http_entry_free (conn); + } + + if (router->key) { + rspamd_keypair_unref (router->key); + } + + if (router->default_fs_path != NULL) { + g_free (router->default_fs_path); + } + + for (i = 0; i < router->regexps->len; i ++) { + re = g_ptr_array_index (router->regexps, i); + rspamd_regexp_unref (re); + } + + g_ptr_array_free (router->regexps, TRUE); + g_hash_table_unref (router->paths); + g_hash_table_unref (router->response_headers); + g_free (router); + } +}
\ No newline at end of file diff --git a/src/libutil/http_router.h b/src/libutil/http_router.h new file mode 100644 index 000000000..8e8056240 --- /dev/null +++ b/src/libutil/http_router.h @@ -0,0 +1,138 @@ +/*- + * Copyright 2019 Vsevolod Stakhov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RSPAMD_HTTP_ROUTER_H +#define RSPAMD_HTTP_ROUTER_H + +#include "config.h" +#include "http_connection.h" + +struct rspamd_http_connection_router; +struct rspamd_http_connection_entry; + +typedef int (*rspamd_http_router_handler_t) (struct rspamd_http_connection_entry + *conn_ent, + struct rspamd_http_message *msg); +typedef void (*rspamd_http_router_error_handler_t) (struct rspamd_http_connection_entry *conn_ent, + GError *err); +typedef void (*rspamd_http_router_finish_handler_t) (struct rspamd_http_connection_entry *conn_ent); + + +struct rspamd_http_connection_entry { + struct rspamd_http_connection_router *rt; + struct rspamd_http_connection *conn; + gpointer ud; + gboolean is_reply; + gboolean support_gzip; + struct rspamd_http_connection_entry *prev, *next; +}; + +struct rspamd_http_connection_router { + struct rspamd_http_connection_entry *conns; + GHashTable *paths; + GHashTable *response_headers; + GPtrArray *regexps; + struct timeval tv; + struct timeval *ptv; + struct event_base *ev_base; + struct rspamd_http_context *ctx; + gchar *default_fs_path; + rspamd_http_router_handler_t unknown_method_handler; + struct rspamd_cryptobox_keypair *key; + rspamd_http_router_error_handler_t error_handler; + rspamd_http_router_finish_handler_t finish_handler; +}; + +/** + * Create new http connection router and the associated HTTP connection + * @param eh error handler callback + * @param fh finish handler callback + * @param default_fs_path if not NULL try to serve static files from + * the specified directory + * @return + */ +struct rspamd_http_connection_router * rspamd_http_router_new ( + rspamd_http_router_error_handler_t eh, + rspamd_http_router_finish_handler_t fh, + struct timeval *timeout, + const char *default_fs_path, + struct rspamd_http_context *ctx); + +/** + * Set encryption key for the HTTP router + * @param router router structure + * @param key opaque key structure + */ +void rspamd_http_router_set_key (struct rspamd_http_connection_router *router, + struct rspamd_cryptobox_keypair *key); + +/** + * Add new path to the router + */ +void rspamd_http_router_add_path (struct rspamd_http_connection_router *router, + const gchar *path, rspamd_http_router_handler_t handler); + +/** + * Add custom header to append to router replies + * @param router + * @param name + * @param value + */ +void rspamd_http_router_add_header (struct rspamd_http_connection_router *router, + const gchar *name, const gchar *value); + +/** + * Sets method to handle unknown request methods + * @param router + * @param handler + */ +void rspamd_http_router_set_unknown_handler (struct rspamd_http_connection_router *router, + rspamd_http_router_handler_t handler); + +/** + * Inserts router headers to the outbound message + * @param router + * @param msg + */ +void rspamd_http_router_insert_headers (struct rspamd_http_connection_router *router, + struct rspamd_http_message *msg); + +struct rspamd_regexp_s; +/** + * Adds new pattern to router, regexp object is refcounted by this function + * @param router + * @param re + * @param handler + */ +void rspamd_http_router_add_regexp (struct rspamd_http_connection_router *router, + struct rspamd_regexp_s *re, rspamd_http_router_handler_t handler); +/** + * Handle new accepted socket + * @param router router object + * @param fd server socket + * @param ud opaque userdata + */ +void rspamd_http_router_handle_socket ( + struct rspamd_http_connection_router *router, + gint fd, + gpointer ud); + +/** + * Free router and all connections associated + * @param router + */ +void rspamd_http_router_free (struct rspamd_http_connection_router *router); + +#endif diff --git a/src/libutil/http_util.c b/src/libutil/http_util.c new file mode 100644 index 000000000..8fb658e08 --- /dev/null +++ b/src/libutil/http_util.c @@ -0,0 +1,513 @@ +/*- + * Copyright 2019 Vsevolod Stakhov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "libutil/http_util.h" +#include "libutil/printf.h" +#include "libutil/util.h" + +static const gchar *http_week[] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" }; +static const gchar *http_month[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; + +/* + * Obtained from nginx + * Copyright (C) Igor Sysoev + */ +static guint mday[] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; + +time_t +rspamd_http_parse_date (const gchar *header, gsize len) +{ + const gchar *p, *end; + gint month; + guint day, year, hour, min, sec; + guint64 time; + enum { + no = 0, rfc822, /* Tue, 10 Nov 2002 23:50:13 */ + rfc850, /* Tuesday, 10-Dec-02 23:50:13 */ + isoc /* Tue Dec 10 23:50:13 2002 */ + } fmt; + + fmt = 0; + if (len > 0) { + end = header + len; + } + else { + end = header + strlen (header); + } + + day = 32; + year = 2038; + + for (p = header; p < end; p++) { + if (*p == ',') { + break; + } + + if (*p == ' ') { + fmt = isoc; + break; + } + } + + for (p++; p < end; p++) + if (*p != ' ') { + break; + } + + if (end - p < 18) { + return (time_t)-1; + } + + if (fmt != isoc) { + if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') { + return (time_t)-1; + } + + day = (*p - '0') * 10 + *(p + 1) - '0'; + p += 2; + + if (*p == ' ') { + if (end - p < 18) { + return (time_t)-1; + } + fmt = rfc822; + + } + else if (*p == '-') { + fmt = rfc850; + + } + else { + return (time_t)-1; + } + + p++; + } + + switch (*p) { + + case 'J': + month = *(p + 1) == 'a' ? 0 : *(p + 2) == 'n' ? 5 : 6; + break; + + case 'F': + month = 1; + break; + + case 'M': + month = *(p + 2) == 'r' ? 2 : 4; + break; + + case 'A': + month = *(p + 1) == 'p' ? 3 : 7; + break; + + case 'S': + month = 8; + break; + + case 'O': + month = 9; + break; + + case 'N': + month = 10; + break; + + case 'D': + month = 11; + break; + + default: + return (time_t)-1; + } + + p += 3; + + if ((fmt == rfc822 && *p != ' ') || (fmt == rfc850 && *p != '-')) { + return (time_t)-1; + } + + p++; + + if (fmt == rfc822) { + if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9' + || *(p + 2) < '0' || *(p + 2) > '9' || *(p + 3) < '0' + || *(p + 3) > '9') { + return (time_t)-1; + } + + year = (*p - '0') * 1000 + (*(p + 1) - '0') * 100 + + (*(p + 2) - '0') * 10 + *(p + 3) - '0'; + p += 4; + + } + else if (fmt == rfc850) { + if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') { + return (time_t)-1; + } + + year = (*p - '0') * 10 + *(p + 1) - '0'; + year += (year < 70) ? 2000 : 1900; + p += 2; + } + + if (fmt == isoc) { + if (*p == ' ') { + p++; + } + + if (*p < '0' || *p > '9') { + return (time_t)-1; + } + + day = *p++ - '0'; + + if (*p != ' ') { + if (*p < '0' || *p > '9') { + return (time_t)-1; + } + + day = day * 10 + *p++ - '0'; + } + + if (end - p < 14) { + return (time_t)-1; + } + } + + if (*p++ != ' ') { + return (time_t)-1; + } + + if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') { + return (time_t)-1; + } + + hour = (*p - '0') * 10 + *(p + 1) - '0'; + p += 2; + + if (*p++ != ':') { + return (time_t)-1; + } + + if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') { + return (time_t)-1; + } + + min = (*p - '0') * 10 + *(p + 1) - '0'; + p += 2; + + if (*p++ != ':') { + return (time_t)-1; + } + + if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') { + return (time_t)-1; + } + + sec = (*p - '0') * 10 + *(p + 1) - '0'; + + if (fmt == isoc) { + p += 2; + + if (*p++ != ' ') { + return (time_t)-1; + } + + if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9' + || *(p + 2) < '0' || *(p + 2) > '9' || *(p + 3) < '0' + || *(p + 3) > '9') { + return (time_t)-1; + } + + year = (*p - '0') * 1000 + (*(p + 1) - '0') * 100 + + (*(p + 2) - '0') * 10 + *(p + 3) - '0'; + } + + if (hour > 23 || min > 59 || sec > 59) { + return (time_t)-1; + } + + if (day == 29 && month == 1) { + if ((year & 3) || ((year % 100 == 0) && (year % 400) != 0)) { + return (time_t)-1; + } + + } + else if (day > mday[month]) { + return (time_t)-1; + } + + /* + * shift new year to March 1 and start months from 1 (not 0), + * it is needed for Gauss' formula + */ + + if (--month <= 0) { + month += 12; + year -= 1; + } + + /* Gauss' formula for Gregorian days since March 1, 1 BC */ + + time = (guint64) ( + /* days in years including leap years since March 1, 1 BC */ + + 365 * year + year / 4 - year / 100 + year / 400 + + /* days before the month */ + + + 367 * month / 12 - 30 + + /* days before the day */ + + + day - 1 + + /* + * 719527 days were between March 1, 1 BC and March 1, 1970, + * 31 and 28 days were in January and February 1970 + */ + + - 719527 + 31 + 28) * 86400 + hour * 3600 + min * 60 + sec; + + return (time_t) time; +} + +glong +rspamd_http_date_format (gchar *buf, gsize len, time_t time) +{ + struct tm tms; + + rspamd_gmtime (time, &tms); + + return rspamd_snprintf (buf, len, "%s, %02d %s %4d %02d:%02d:%02d GMT", + http_week[tms.tm_wday], tms.tm_mday, + http_month[tms.tm_mon], tms.tm_year + 1900, + tms.tm_hour, tms.tm_min, tms.tm_sec); +} + +void +rspamd_http_normalize_path_inplace (gchar *path, guint len, guint *nlen) +{ + const gchar *p, *end, *slash = NULL, *dot = NULL; + gchar *o; + enum { + st_normal = 0, + st_got_dot, + st_got_dot_dot, + st_got_slash, + st_got_slash_slash, + } state = st_normal; + + p = path; + end = path + len; + o = path; + + while (p < end) { + switch (state) { + case st_normal: + if (G_UNLIKELY (*p == '/')) { + state = st_got_slash; + slash = p; + } + else if (G_UNLIKELY (*p == '.')) { + state = st_got_dot; + dot = p; + } + else { + *o++ = *p; + } + p ++; + break; + case st_got_slash: + if (G_UNLIKELY (*p == '/')) { + /* Ignore double slash */ + *o++ = *p; + state = st_got_slash_slash; + } + else if (G_UNLIKELY (*p == '.')) { + dot = p; + state = st_got_dot; + } + else { + *o++ = '/'; + *o++ = *p; + slash = NULL; + dot = NULL; + state = st_normal; + } + p ++; + break; + case st_got_slash_slash: + if (G_LIKELY (*p != '/')) { + slash = p - 1; + dot = NULL; + state = st_normal; + continue; + } + p ++; + break; + case st_got_dot: + if (G_UNLIKELY (*p == '/')) { + /* Remove any /./ or ./ paths */ + if (((o > path && *(o - 1) != '/') || (o == path)) && slash) { + /* Preserve one slash */ + *o++ = '/'; + } + + slash = p; + dot = NULL; + /* Ignore last slash */ + state = st_normal; + } + else if (*p == '.') { + /* Double dot character */ + state = st_got_dot_dot; + } + else { + /* We have something like .some or /.some */ + if (dot && p > dot) { + if (slash == dot - 1 && (o > path && *(o - 1) != '/')) { + /* /.blah */ + memmove (o, slash, p - slash); + o += p - slash; + } + else { + memmove (o, dot, p - dot); + o += p - dot; + } + } + + slash = NULL; + dot = NULL; + state = st_normal; + continue; + } + + p ++; + break; + case st_got_dot_dot: + if (*p == '/') { + /* We have something like /../ or ../ */ + if (slash) { + /* We need to remove the last component from o if it is there */ + if (o > path + 2 && *(o - 1) == '/') { + slash = rspamd_memrchr (path, '/', o - path - 2); + } + else if (o > path + 1) { + slash = rspamd_memrchr (path, '/', o - path - 1); + } + else { + slash = NULL; + } + + if (slash) { + o = (gchar *)slash; + } + /* Otherwise we keep these dots */ + slash = p; + state = st_got_slash; + } + else { + /* We have something like bla../, so we need to copy it as is */ + if (o > path && dot && p > dot) { + memmove (o, dot, p - dot); + o += p - dot; + } + + slash = NULL; + dot = NULL; + state = st_normal; + continue; + } + } + else { + /* We have something like ..bla or ... */ + if (slash) { + *o ++ = '/'; + } + + if (dot && p > dot) { + memmove (o, dot, p - dot); + o += p - dot; + } + + slash = NULL; + dot = NULL; + state = st_normal; + continue; + } + + p ++; + break; + } + } + + /* Leftover */ + switch (state) { + case st_got_dot_dot: + /* Trailing .. */ + if (slash) { + /* We need to remove the last component from o if it is there */ + if (o > path + 2 && *(o - 1) == '/') { + slash = rspamd_memrchr (path, '/', o - path - 2); + } + else if (o > path + 1) { + slash = rspamd_memrchr (path, '/', o - path - 1); + } + else { + if (o == path) { + /* Corner case */ + *o++ = '/'; + } + + slash = NULL; + } + + if (slash) { + /* Remove last / */ + o = (gchar *)slash; + } + } + else { + /* Corner case */ + if (o == path) { + *o++ = '/'; + } + else { + if (dot && p > dot) { + memmove (o, dot, p - dot); + o += p - dot; + } + } + } + break; + case st_got_slash: + *o++ = '/'; + break; + default: + if (o > path + 1 && *(o - 1) == '/') { + o --; + } + break; + } + + if (nlen) { + *nlen = (o - path); + } +}
\ No newline at end of file diff --git a/src/libutil/http_util.h b/src/libutil/http_util.h new file mode 100644 index 000000000..30d806ae0 --- /dev/null +++ b/src/libutil/http_util.h @@ -0,0 +1,48 @@ +/*- + * Copyright 2019 Vsevolod Stakhov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RSPAMD_HTTP_UTIL_H +#define RSPAMD_HTTP_UTIL_H + +#include "config.h" + +/** + * Parse HTTP date header and return it as time_t + * @param header HTTP date header + * @param len length of header + * @return time_t or (time_t)-1 in case of error + */ +time_t rspamd_http_parse_date (const gchar *header, gsize len); + +/** + * Prints HTTP date from `time` to `buf` using standard HTTP date format + * @param buf date buffer + * @param len length of buffer + * @param time time in unix seconds + * @return number of bytes written + */ +glong rspamd_http_date_format (gchar *buf, gsize len, time_t time); + +/** + * Normalize HTTP path removing dot sequences and repeating '/' symbols as + * per rfc3986#section-5.2 + * @param path + * @param len + * @param nlen + */ +void rspamd_http_normalize_path_inplace (gchar *path, guint len, guint *nlen); + +#endif diff --git a/src/libutil/map.c b/src/libutil/map.c index f13f484d8..d4687e433 100644 --- a/src/libutil/map.c +++ b/src/libutil/map.c @@ -19,7 +19,7 @@ #include "config.h" #include "map.h" #include "map_private.h" -#include "http.h" +#include "http_connection.h" #include "http_private.h" #include "rspamd.h" #include "contrib/zstd/zstd.h" @@ -138,9 +138,7 @@ write_http_request (struct http_callback_data *cbd) cbd->data->host, NULL, cbd, - cbd->fd, - &cbd->tv, - cbd->ev_base); + &cbd->tv); } else { msg_err_map ("cannot connect to %s: %s", cbd->data->host, @@ -1155,11 +1153,7 @@ rspamd_map_periodic_dtor (struct map_periodic_cbdata *periodic) if (periodic->need_modify) { /* We are done */ - periodic->map->fin_callback (&periodic->cbdata); - - if (periodic->cbdata.cur_data) { - *periodic->map->user_data = periodic->cbdata.cur_data; - } + periodic->map->fin_callback (&periodic->cbdata, periodic->map->user_data); } else { /* Not modified */ @@ -1277,12 +1271,12 @@ rspamd_map_dns_callback (struct rdns_reply *reply, void *arg) if (cbd->fd != -1) { cbd->stage = map_load_file; cbd->conn = rspamd_http_connection_new (NULL, + cbd->fd, + NULL, http_map_error, http_map_finish, flags, - RSPAMD_HTTP_CLIENT, - NULL, - cbd->map->cfg->libs_ctx->ssl_ctx); + RSPAMD_HTTP_CLIENT); write_http_request (cbd); } @@ -1649,13 +1643,14 @@ check: if (cbd->fd != -1) { cbd->stage = map_load_file; - cbd->conn = rspamd_http_connection_new (NULL, + cbd->conn = rspamd_http_connection_new ( + NULL, + cbd->fd, + NULL, http_map_error, http_map_finish, flags, - RSPAMD_HTTP_CLIENT, - NULL, - cbd->map->cfg->libs_ctx->ssl_ctx); + RSPAMD_HTTP_CLIENT); write_http_request (cbd); MAP_RELEASE (cbd, "http_callback_data"); @@ -2039,11 +2034,7 @@ rspamd_map_preload (struct rspamd_config *cfg) } if (succeed) { - map->fin_callback (&fake_cbd.cbdata); - - if (fake_cbd.cbdata.cur_data) { - *map->user_data = fake_cbd.cbdata.cur_data; - } + map->fin_callback (&fake_cbd.cbdata, map->user_data); } else { msg_info_map ("preload of %s failed", map->name); diff --git a/src/libutil/map.h b/src/libutil/map.h index 80aa03825..acf6eea4e 100644 --- a/src/libutil/map.h +++ b/src/libutil/map.h @@ -21,7 +21,7 @@ struct map_cb_data; */ typedef gchar * (*map_cb_t)(gchar *chunk, gint len, struct map_cb_data *data, gboolean final); -typedef void (*map_fin_cb_t)(struct map_cb_data *data); +typedef void (*map_fin_cb_t)(struct map_cb_data *data, void **target); typedef void (*map_dtor_t)(struct map_cb_data *data); typedef gboolean (*rspamd_map_traverse_cb)(gconstpointer key, diff --git a/src/libutil/map_helpers.c b/src/libutil/map_helpers.c index ece3017ab..964e4d98c 100644 --- a/src/libutil/map_helpers.c +++ b/src/libutil/map_helpers.c @@ -504,17 +504,25 @@ rspamd_map_helper_insert_hash (gpointer st, gconstpointer key, gconstpointer val gsize vlen; gint r; - vlen = strlen (value); - val = rspamd_mempool_alloc0 (ht->pool, sizeof (*val) + - vlen + 1); - memcpy (val->value, value, vlen); - k = kh_get (rspamd_map_hash, ht->htb, key); + vlen = strlen (value); if (k == kh_end (ht->htb)) { nk = rspamd_mempool_strdup (ht->pool, key); k = kh_put (rspamd_map_hash, ht->htb, nk, &r); } + else { + val = kh_value (ht->htb, k); + + if (strcmp (value, val->value) == 0) { + /* Same element, skip */ + return; + } + } + + /* Null termination due to alloc0 */ + val = rspamd_mempool_alloc0 (ht->pool, sizeof (*val) + vlen + 1); + memcpy (val->value, value, vlen); nk = kh_key (ht->htb, k); val->key = nk; @@ -815,16 +823,11 @@ rspamd_kv_list_read ( } void -rspamd_kv_list_fin (struct map_cb_data *data) +rspamd_kv_list_fin (struct map_cb_data *data, void **target) { struct rspamd_map *map = data->map; struct rspamd_hash_map_helper *htb; - if (data->prev_data) { - htb = (struct rspamd_hash_map_helper *)data->prev_data; - rspamd_map_helper_destroy_hash (htb); - } - if (data->cur_data) { htb = (struct rspamd_hash_map_helper *)data->cur_data; msg_info_map ("read hash of %d elements", kh_size (htb->htb)); @@ -832,6 +835,15 @@ rspamd_kv_list_fin (struct map_cb_data *data) data->map->nelts = kh_size (htb->htb); data->map->digest = rspamd_cryptobox_fast_hash_final (&htb->hst); } + + if (target) { + *target = data->cur_data; + } + + if (data->prev_data) { + htb = (struct rspamd_hash_map_helper *)data->prev_data; + rspamd_map_helper_destroy_hash (htb); + } } void @@ -870,16 +882,11 @@ rspamd_radix_read ( } void -rspamd_radix_fin (struct map_cb_data *data) +rspamd_radix_fin (struct map_cb_data *data, void **target) { struct rspamd_map *map = data->map; struct rspamd_radix_map_helper *r; - if (data->prev_data) { - r = (struct rspamd_radix_map_helper *)data->prev_data; - rspamd_map_helper_destroy_radix (r); - } - if (data->cur_data) { r = (struct rspamd_radix_map_helper *)data->cur_data; msg_info_map ("read radix trie of %z elements: %s", @@ -888,6 +895,15 @@ rspamd_radix_fin (struct map_cb_data *data) data->map->nelts = kh_size (r->htb); data->map->digest = rspamd_cryptobox_fast_hash_final (&r->hst); } + + if (target) { + *target = data->cur_data; + } + + if (data->prev_data) { + r = (struct rspamd_radix_map_helper *)data->prev_data; + rspamd_map_helper_destroy_radix (r); + } } void @@ -1088,14 +1104,11 @@ rspamd_glob_list_read_multiple ( void -rspamd_regexp_list_fin (struct map_cb_data *data) +rspamd_regexp_list_fin (struct map_cb_data *data, void **target) { struct rspamd_regexp_map_helper *re_map; struct rspamd_map *map = data->map; - if (data->prev_data) { - rspamd_map_helper_destroy_regexp (data->prev_data); - } if (data->cur_data) { re_map = data->cur_data; rspamd_re_map_finalize (re_map); @@ -1105,6 +1118,14 @@ rspamd_regexp_list_fin (struct map_cb_data *data) data->map->nelts = kh_size (re_map->htb); data->map->digest = rspamd_cryptobox_fast_hash_final (&re_map->hst); } + + if (target) { + *target = data->cur_data; + } + + if (data->prev_data) { + rspamd_map_helper_destroy_regexp (data->prev_data); + } } void rspamd_regexp_list_dtor (struct map_cb_data *data) diff --git a/src/libutil/map_helpers.h b/src/libutil/map_helpers.h index f7c86436f..4645024e3 100644 --- a/src/libutil/map_helpers.h +++ b/src/libutil/map_helpers.h @@ -52,7 +52,7 @@ gchar * rspamd_radix_read ( gint len, struct map_cb_data *data, gboolean final); -void rspamd_radix_fin (struct map_cb_data *data); +void rspamd_radix_fin (struct map_cb_data *data, void **target); void rspamd_radix_dtor (struct map_cb_data *data); /** @@ -63,7 +63,7 @@ gchar * rspamd_kv_list_read ( gint len, struct map_cb_data *data, gboolean final); -void rspamd_kv_list_fin (struct map_cb_data *data); +void rspamd_kv_list_fin (struct map_cb_data *data, void **target); void rspamd_kv_list_dtor (struct map_cb_data *data); /** @@ -91,7 +91,7 @@ gchar * rspamd_glob_list_read_multiple ( gint len, struct map_cb_data *data, gboolean final); -void rspamd_regexp_list_fin (struct map_cb_data *data); +void rspamd_regexp_list_fin (struct map_cb_data *data, void **target); void rspamd_regexp_list_dtor (struct map_cb_data *data); /** diff --git a/src/libutil/multipattern.c b/src/libutil/multipattern.c index e4a39d5fe..268170512 100644 --- a/src/libutil/multipattern.c +++ b/src/libutil/multipattern.c @@ -99,8 +99,8 @@ rspamd_multipattern_escape_tld_hyperscan (const gchar *pattern, gsize slen, /* * We understand the following cases - * 1) blah -> \\.blah - * 2) *.blah -> \\..*\\.blah + * 1) blah -> .blah + * 2) *.blah -> ..*\\.blah * 3) ??? */ @@ -116,11 +116,11 @@ rspamd_multipattern_escape_tld_hyperscan (const gchar *pattern, gsize slen, p ++; } - prefix = "\\..*\\."; + prefix = ".*."; } else { - len = slen + 2; - prefix = "\\."; + len = slen + 1; + prefix = "."; p = pattern; } @@ -200,11 +200,18 @@ rspamd_multipattern_pattern_filter (const gchar *pattern, gsize len, } if (flags & RSPAMD_MULTIPATTERN_TLD) { - ret = rspamd_multipattern_escape_tld_hyperscan (pattern, len, dst_len); + gchar *tmp; + gsize tlen; + tmp = rspamd_multipattern_escape_tld_hyperscan (pattern, len, &tlen); + + ret = rspamd_str_regexp_escape (tmp, tlen, dst_len, + gl_flags|RSPAMD_REGEXP_ESCAPE_GLOB); + g_free (tmp); } else if (flags & RSPAMD_MULTIPATTERN_RE) { ret = malloc (len + 1); - *dst_len = rspamd_strlcpy (ret, pattern, len + 1); + ret = rspamd_str_regexp_escape (pattern, len, dst_len, gl_flags | + RSPAMD_REGEXP_ESCAPE_RE); } else if (flags & RSPAMD_MULTIPATTERN_GLOB) { ret = rspamd_str_regexp_escape (pattern, len, dst_len, diff --git a/src/libutil/str_util.c b/src/libutil/str_util.c index 06d7a6cc7..ac7471ada 100644 --- a/src/libutil/str_util.c +++ b/src/libutil/str_util.c @@ -734,18 +734,7 @@ rspamd_decode_base32 (const gchar *in, gsize inlen, gsize *outlen) } - -/** - * Decode string using base32 encoding - * @param in input - * @param inlen input length - * @param out output buf (may overlap with `in`) - * @param outlen output buf len - * @return TRUE if in is valid base32 and `outlen` is enough to encode `inlen` - */ - - -static gchar * +gchar * rspamd_encode_base64_common (const guchar *in, gsize inlen, gint str_len, gsize *outlen, gboolean fold, enum rspamd_newlines_type how) { @@ -2605,7 +2594,7 @@ rspamd_str_regexp_escape (const gchar *pattern, gsize slen, gsize *dst_len, enum rspamd_regexp_escape_flags flags) { const gchar *p, *end = pattern + slen; - gchar *res, *d, t, *tmp_utf = NULL; + gchar *res, *d, t, *tmp_utf = NULL, *dend; gsize len; static const gchar hexdigests[16] = "0123456789abcdef"; @@ -2634,15 +2623,22 @@ rspamd_str_regexp_escape (const gchar *pattern, gsize slen, case '$': case '|': case '#': - len ++; + if (!(flags & RSPAMD_REGEXP_ESCAPE_RE)) { + len++; + } break; default: if (g_ascii_isspace (t)) { len ++; } else { - if (!(flags & RSPAMD_REGEXP_ESCAPE_UTF)) { - if (!g_ascii_isprint (t)) { + if (!g_ascii_isprint (t) || (t & 0x80)) { + + if (flags & RSPAMD_REGEXP_ESCAPE_UTF) { + /* \x{code}, where code can be up to 5 digits */ + len += 4; + } + else { /* \\xHH -> 4 symbols */ len += 3; } @@ -2668,8 +2664,6 @@ rspamd_str_regexp_escape (const gchar *pattern, gsize slen, *dst_len = slen; } - - if (tmp_utf) { return tmp_utf; } @@ -2685,8 +2679,10 @@ rspamd_str_regexp_escape (const gchar *pattern, gsize slen, res = g_malloc (len + 1); p = pattern; d = res; + dend = d + len; while (p < end) { + g_assert (d < dend); t = *p ++; switch (t) { @@ -2704,7 +2700,9 @@ rspamd_str_regexp_escape (const gchar *pattern, gsize slen, case '$': case '|': case '#': - *d++ = '\\'; + if (!(flags & RSPAMD_REGEXP_ESCAPE_RE)) { + *d++ = '\\'; + } break; case '*': case '?': @@ -2714,19 +2712,40 @@ rspamd_str_regexp_escape (const gchar *pattern, gsize slen, *d++ = '.'; } else { - *d++ = '\\'; + if (!(flags & RSPAMD_REGEXP_ESCAPE_RE)) { + *d++ = '\\'; + } } break; default: if (g_ascii_isspace (t)) { - *d++ = '\\'; + if (!(flags & RSPAMD_REGEXP_ESCAPE_RE)) { + *d++ = '\\'; + } } - else if (!(flags & RSPAMD_REGEXP_ESCAPE_UTF) && !g_ascii_isgraph (t)) { - *d++ = '\\'; - *d++ = 'x'; - *d++ = hexdigests[((t >> 4) & 0xF)]; - *d++ = hexdigests[((t) & 0xF)]; - continue; /* To avoid *d++ = t; */ + else if (t & 0x80 || !g_ascii_isprint (t)) { + if (!(flags & RSPAMD_REGEXP_ESCAPE_UTF)) { + *d++ = '\\'; + *d++ = 'x'; + *d++ = hexdigests[((t >> 4) & 0xF)]; + *d++ = hexdigests[((t) & 0xF)]; + continue; /* To avoid *d++ = t; */ + } + else { + if (flags & (RSPAMD_REGEXP_ESCAPE_RE|RSPAMD_REGEXP_ESCAPE_GLOB)) { + UChar32 uc; + gint32 off = p - pattern - 1; + U8_NEXT (pattern, off, slen, uc); + + if (uc > 0) { + d += rspamd_snprintf (d, dend - d, + "\\x{%xd}", uc); + p = pattern + off; + } + + continue; /* To avoid *d++ = t; */ + } + } } break; } diff --git a/src/libutil/str_util.h b/src/libutil/str_util.h index 46b74001b..9145b97b4 100644 --- a/src/libutil/str_util.h +++ b/src/libutil/str_util.h @@ -184,6 +184,7 @@ gint rspamd_decode_base32_buf (const gchar *in, gsize inlen, gint rspamd_encode_hex_buf (const guchar *in, gsize inlen, gchar *out, gsize outlen); + /** * Decode string using hex encoding * @param in input @@ -196,6 +197,23 @@ gssize rspamd_decode_hex_buf (const gchar *in, gsize inlen, guchar *out, gsize outlen); /** + * Common version of base64 encoder + * @param in + * @param inlen + * @param str_len + * @param outlen + * @param fold + * @param how + * @return + */ +gchar * +rspamd_encode_base64_common (const guchar *in, + gsize inlen, + gint str_len, + gsize *outlen, + gboolean fold, + enum rspamd_newlines_type how); +/** * Encode string using base64 encoding * @param in input * @param inlen input length @@ -436,6 +454,7 @@ enum rspamd_regexp_escape_flags { RSPAMD_REGEXP_ESCAPE_ASCII = 0, RSPAMD_REGEXP_ESCAPE_UTF = 1u << 0, RSPAMD_REGEXP_ESCAPE_GLOB = 1u << 1, + RSPAMD_REGEXP_ESCAPE_RE = 1u << 2, }; /** * Escapes special characters when reading plain data to be processed in pcre diff --git a/src/libutil/upstream.c b/src/libutil/upstream.c index eb88e501a..64d5291fa 100644 --- a/src/libutil/upstream.c +++ b/src/libutil/upstream.c @@ -289,7 +289,7 @@ rspamd_upstream_update_addrs (struct upstream *up) rspamd_inet_address_set_port (cur->addr, port); PTR_ARRAY_FOREACH (up->addrs.addr, i, addr_elt) { - if (rspamd_inet_address_compare (addr_elt->addr, cur->addr) == 0) { + if (rspamd_inet_address_compare (addr_elt->addr, cur->addr, FALSE) == 0) { naddr = g_malloc0 (sizeof (*naddr)); naddr->addr = cur->addr; naddr->errors = reset_errors ? 0 : addr_elt->errors; @@ -630,7 +630,7 @@ rspamd_upstream_dtor (struct upstream *up) } rspamd_inet_addr_t* -rspamd_upstream_addr (struct upstream *up) +rspamd_upstream_addr_next (struct upstream *up) { guint idx, next_idx; struct upstream_addr_elt *e1, *e2; @@ -646,6 +646,16 @@ rspamd_upstream_addr (struct upstream *up) return e2->addr; } +rspamd_inet_addr_t* +rspamd_upstream_addr_cur (const struct upstream *up) +{ + struct upstream_addr_elt *elt; + + elt = g_ptr_array_index (up->addrs.addr, up->addrs.cur); + + return elt->addr; +} + const gchar* rspamd_upstream_name (struct upstream *up) { diff --git a/src/libutil/upstream.h b/src/libutil/upstream.h index 5c0c92afc..4db962765 100644 --- a/src/libutil/upstream.h +++ b/src/libutil/upstream.h @@ -208,11 +208,18 @@ void rspamd_upstreams_add_watch_callback (struct upstream_list *ups, gpointer ud); /** + * Returns the next IP address of the upstream (internal rotation) + * @param up + * @return + */ +rspamd_inet_addr_t* rspamd_upstream_addr_next (struct upstream *up); + +/** * Returns the current IP address of the upstream * @param up * @return */ -rspamd_inet_addr_t* rspamd_upstream_addr (struct upstream *up); +rspamd_inet_addr_t* rspamd_upstream_addr_cur (const struct upstream *up); /** * Add custom address for an upstream (ownership of addr is transferred to upstream) diff --git a/src/libutil/uthash_strcase.h b/src/libutil/uthash_strcase.h index fba97d9b1..5d6f2773c 100644 --- a/src/libutil/uthash_strcase.h +++ b/src/libutil/uthash_strcase.h @@ -20,10 +20,13 @@ #error Invalid include order: uthash is already included #endif +#include "libcryptobox/cryptobox.h" +#include "libutil/util.h" + /* Utils for uthash tuning */ #ifndef HASH_CASELESS #define HASH_FUNCTION(key,keylen,num_bkts,hashv,bkt) do {\ - hashv = mum(key, keylen, 0xdeadbabe); \ + hashv = rspamd_cryptobox_fast_hash(key, keylen, rspamd_hash_seed ()); \ bkt = (hashv) & (num_bkts-1); \ } while (0) @@ -31,6 +34,7 @@ #else #define HASH_FUNCTION(key,keylen,num_bkts,hashv,bkt) do {\ unsigned _len = keylen; \ + rspamd_cryptobox_fast_hash_state_t _hst; \ unsigned _leftover = keylen % 8; \ unsigned _fp, _i; \ const uint8_t* _s = (const uint8_t*)(key); \ @@ -40,9 +44,8 @@ } c; \ uint64_t pp; \ } _u; \ - uint64_t _r; \ _fp = _len - _leftover; \ - _r = 0xdeadbabe; \ + rspamd_cryptobox_fast_hash_init (&_hst, rspamd_hash_seed ()); \ for (_i = 0; _i != _fp; _i += 8) { \ _u.c.c1 = _s[_i], _u.c.c2 = _s[_i + 1], _u.c.c3 = _s[_i + 2], _u.c.c4 = _s[_i + 3]; \ _u.c.c5 = _s[_i + 4], _u.c.c6 = _s[_i + 5], _u.c.c7 = _s[_i + 6], _u.c.c8 = _s[_i + 7]; \ @@ -54,28 +57,28 @@ _u.c.c2 = lc_map[_u.c.c6]; \ _u.c.c3 = lc_map[_u.c.c7]; \ _u.c.c4 = lc_map[_u.c.c8]; \ - _r = mum_hash_step (_r, _u.pp); \ + rspamd_cryptobox_fast_hash_update (&_hst, &_u, sizeof (_u)); \ } \ _u.pp = 0; \ switch (_leftover) { \ case 7: \ - _u.c.c7 = lc_map[(unsigned char)_s[_i++]]; \ + /* fallthrough */ _u.c.c7 = lc_map[(unsigned char)_s[_i++]]; \ case 6: \ - _u.c.c6 = lc_map[(unsigned char)_s[_i++]]; \ + /* fallthrough */ _u.c.c6 = lc_map[(unsigned char)_s[_i++]]; \ case 5: \ - _u.c.c5 = lc_map[(unsigned char)_s[_i++]]; \ + /* fallthrough */ _u.c.c5 = lc_map[(unsigned char)_s[_i++]]; \ case 4: \ - _u.c.c4 = lc_map[(unsigned char)_s[_i++]]; \ + /* fallthrough */ _u.c.c4 = lc_map[(unsigned char)_s[_i++]]; \ case 3: \ - _u.c.c3 = lc_map[(unsigned char)_s[_i++]]; \ + /* fallthrough */ _u.c.c3 = lc_map[(unsigned char)_s[_i++]]; \ case 2: \ - _u.c.c2 = lc_map[(unsigned char)_s[_i++]]; \ + /* fallthrough */ _u.c.c2 = lc_map[(unsigned char)_s[_i++]]; \ case 1: \ - _u.c.c1 = lc_map[(unsigned char)_s[_i]]; \ - _r = mum_hash_step (_r, _u.pp); \ + /* fallthrough */ _u.c.c1 = lc_map[(unsigned char)_s[_i]]; \ + rspamd_cryptobox_fast_hash_update (&_hst, &_u, sizeof (_u)); \ break; \ } \ - hashv = mum_hash_finish (_r); \ + hashv = rspamd_cryptobox_fast_hash_final (&_hst); \ bkt = (hashv) & (num_bkts-1); \ } while (0) #define HASH_KEYCMP(a,b,len) rspamd_lc_cmp(a,b,len) diff --git a/src/libutil/util.c b/src/libutil/util.c index 68fbd5443..1f0c5fa70 100644 --- a/src/libutil/util.c +++ b/src/libutil/util.c @@ -2035,13 +2035,102 @@ void rspamd_gerror_free_maybe (gpointer p) } } + + +static void +rspamd_openssl_maybe_init (void) +{ + static gboolean openssl_initialized = FALSE; + + if (!openssl_initialized) { + ERR_load_crypto_strings (); + SSL_load_error_strings (); + + OpenSSL_add_all_algorithms (); + OpenSSL_add_all_digests (); + OpenSSL_add_all_ciphers (); + +#if OPENSSL_VERSION_NUMBER >= 0x1000104fL && !defined(LIBRESSL_VERSION_NUMBER) + ENGINE_load_builtin_engines (); +#endif +#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER) + SSL_library_init (); +#else + OPENSSL_init_ssl (0, NULL); +#endif + +#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER) + OPENSSL_config (NULL); +#endif + if (RAND_status () == 0) { + guchar seed[128]; + + /* Try to use ottery to seed rand */ + ottery_rand_bytes (seed, sizeof (seed)); + RAND_seed (seed, sizeof (seed)); + rspamd_explicit_memzero (seed, sizeof (seed)); + } + + openssl_initialized = TRUE; + } +} + +gpointer +rspamd_init_ssl_ctx (void) +{ + SSL_CTX *ssl_ctx; + gint ssl_options; + + rspamd_openssl_maybe_init (); + + ssl_ctx = SSL_CTX_new (SSLv23_method ()); + SSL_CTX_set_verify (ssl_ctx, SSL_VERIFY_PEER, NULL); + SSL_CTX_set_verify_depth (ssl_ctx, 4); + ssl_options = SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3; + +#ifdef SSL_OP_NO_COMPRESSION + ssl_options |= SSL_OP_NO_COMPRESSION; +#elif OPENSSL_VERSION_NUMBER >= 0x00908000L + sk_SSL_COMP_zero (SSL_COMP_get_compression_methods ()); +#endif + + SSL_CTX_set_options (ssl_ctx, ssl_options); + + return ssl_ctx; +} + +gpointer rspamd_init_ssl_ctx_noverify (void) +{ + SSL_CTX *ssl_ctx_noverify; + gint ssl_options; + + rspamd_openssl_maybe_init (); + + ssl_options = SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3; + +#ifdef SSL_OP_NO_COMPRESSION + ssl_options |= SSL_OP_NO_COMPRESSION; +#elif OPENSSL_VERSION_NUMBER >= 0x00908000L + sk_SSL_COMP_zero (SSL_COMP_get_compression_methods ()); +#endif + + ssl_ctx_noverify = SSL_CTX_new (SSLv23_method ()); + SSL_CTX_set_verify (ssl_ctx_noverify, SSL_VERIFY_NONE, NULL); + SSL_CTX_set_options (ssl_ctx_noverify, ssl_options); +#ifdef SSL_SESS_CACHE_BOTH + SSL_CTX_set_session_cache_mode (ssl_ctx_noverify, SSL_SESS_CACHE_BOTH); +#endif + + return ssl_ctx_noverify; +} + + struct rspamd_external_libs_ctx * rspamd_init_libs (void) { struct rlimit rlim; struct rspamd_external_libs_ctx *ctx; struct ottery_config *ottery_cfg; - gint ssl_options; ctx = g_malloc0 (sizeof (*ctx)); ctx->crypto_ctx = rspamd_cryptobox_init (); @@ -2049,10 +2138,15 @@ rspamd_init_libs (void) ottery_config_init (ottery_cfg); ctx->ottery_cfg = ottery_cfg; + rspamd_openssl_maybe_init (); + /* Check if we have rdrand */ if ((ctx->crypto_ctx->cpu_config & CPUID_RDRAND) == 0) { ottery_config_disable_entropy_sources (ottery_cfg, OTTERY_ENTROPY_SRC_RDRAND); +#if OPENSSL_VERSION_NUMBER >= 0x1000104fL && !defined(LIBRESSL_VERSION_NUMBER) + RAND_set_rand_engine (NULL); +#endif } g_assert (ottery_init (ottery_cfg) == 0); @@ -2072,60 +2166,8 @@ rspamd_init_libs (void) } #endif -#ifdef HAVE_OPENSSL - ERR_load_crypto_strings (); - SSL_load_error_strings (); - - OpenSSL_add_all_algorithms (); - OpenSSL_add_all_digests (); - OpenSSL_add_all_ciphers (); - -#if OPENSSL_VERSION_NUMBER >= 0x1000104fL && !defined(LIBRESSL_VERSION_NUMBER) - ENGINE_load_builtin_engines (); - - if ((ctx->crypto_ctx->cpu_config & CPUID_RDRAND) == 0) { - RAND_set_rand_engine (NULL); - } -#endif - -#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER) - SSL_library_init (); -#else - OPENSSL_init_ssl (0, NULL); -#endif - -#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER) - OPENSSL_config (NULL); -#endif - - if (RAND_poll () == 0) { - guchar seed[128]; - - /* Try to use ottery to seed rand */ - ottery_rand_bytes (seed, sizeof (seed)); - RAND_seed (seed, sizeof (seed)); - rspamd_explicit_memzero (seed, sizeof (seed)); - } - - ctx->ssl_ctx = SSL_CTX_new (SSLv23_method ()); - SSL_CTX_set_verify (ctx->ssl_ctx, SSL_VERIFY_PEER, NULL); - SSL_CTX_set_verify_depth (ctx->ssl_ctx, 4); - ssl_options = SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3; - -#ifdef SSL_OP_NO_COMPRESSION - ssl_options |= SSL_OP_NO_COMPRESSION; -#elif OPENSSL_VERSION_NUMBER >= 0x00908000L - sk_SSL_COMP_zero (SSL_COMP_get_compression_methods ()); -#endif - - SSL_CTX_set_options (ctx->ssl_ctx, ssl_options); - ctx->ssl_ctx_noverify = SSL_CTX_new (SSLv23_method ()); - SSL_CTX_set_verify (ctx->ssl_ctx_noverify, SSL_VERIFY_NONE, NULL); - SSL_CTX_set_options (ctx->ssl_ctx_noverify, ssl_options); -#endif -#ifdef SSL_SESS_CACHE_BOTH - SSL_CTX_set_session_cache_mode (ctx->ssl_ctx, SSL_SESS_CACHE_BOTH); -#endif + ctx->ssl_ctx = rspamd_init_ssl_ctx (); + ctx->ssl_ctx_noverify = rspamd_init_ssl_ctx_noverify (); rspamd_random_seed_fast (); /* Set stack size for pcre */ diff --git a/src/libutil/util.h b/src/libutil/util.h index e97593d21..e9c1fa1db 100644 --- a/src/libutil/util.h +++ b/src/libutil/util.h @@ -349,6 +349,9 @@ struct rspamd_external_libs_ctx; */ struct rspamd_external_libs_ctx* rspamd_init_libs (void); +gpointer rspamd_init_ssl_ctx (void); +gpointer rspamd_init_ssl_ctx_noverify (void); + /** * Configure libraries */ diff --git a/src/lua/lua_config.c b/src/lua/lua_config.c index a41ffa63f..2ea72f075 100644 --- a/src/lua/lua_config.c +++ b/src/lua/lua_config.c @@ -1113,6 +1113,119 @@ rspamd_compare_order_func (gconstpointer a, gconstpointer b) return cb2->order - cb1->order; } +static void +lua_metric_symbol_callback (struct rspamd_task *task, + struct rspamd_symcache_item *item, + gpointer ud) +{ + struct lua_callback_data *cd = ud; + struct rspamd_task **ptask; + gint level = lua_gettop (cd->L), nresults, err_idx, ret; + lua_State *L = cd->L; + GString *tb; + struct rspamd_symbol_result *s; + + cd->item = item; + rspamd_symcache_item_async_inc (task, item, "lua symbol"); + lua_pushcfunction (L, &rspamd_lua_traceback); + err_idx = lua_gettop (L); + + level ++; + + if (cd->cb_is_ref) { + lua_rawgeti (L, LUA_REGISTRYINDEX, cd->callback.ref); + } + else { + lua_getglobal (L, cd->callback.name); + } + + ptask = lua_newuserdata (L, sizeof (struct rspamd_task *)); + rspamd_lua_setclass (L, "rspamd{task}", -1); + *ptask = task; + + if ((ret = lua_pcall (L, 1, LUA_MULTRET, err_idx)) != 0) { + tb = lua_touserdata (L, -1); + msg_err_task ("call to (%s) failed (%d): %v", cd->symbol, ret, tb); + + if (tb) { + g_string_free (tb, TRUE); + lua_pop (L, 1); + } + } + else { + nresults = lua_gettop (L) - level; + + if (nresults >= 1) { + /* Function returned boolean, so maybe we need to insert result? */ + gint res = 0; + gint i; + gdouble flag = 1.0; + gint type; + + type = lua_type (cd->L, level + 1); + + if (type == LUA_TBOOLEAN) { + res = lua_toboolean (L, level + 1); + } + else if (type == LUA_TNUMBER) { + res = lua_tonumber (L, level + 1); + } + else if (type == LUA_TNIL) { + /* Can happen sometimes... */ + res = FALSE; + } + else { + g_assert_not_reached (); + } + + if (res) { + gint first_opt = 2; + + if (lua_type (L, level + 2) == LUA_TNUMBER) { + flag = lua_tonumber (L, level + 2); + /* Shift opt index */ + first_opt = 3; + } + else { + flag = res; + } + + s = rspamd_task_insert_result (task, cd->symbol, flag, NULL); + + if (s) { + guint last_pos = lua_gettop (L); + + for (i = level + first_opt; i <= last_pos; i++) { + if (lua_type (L, i) == LUA_TSTRING) { + const char *opt = lua_tostring (L, i); + + rspamd_task_add_result_option (task, s, opt); + } + else if (lua_type (L, i) == LUA_TTABLE) { + lua_pushvalue (L, i); + + for (lua_pushnil (L); lua_next (L, -2); lua_pop (L, 1)) { + const char *opt = lua_tostring (L, -1); + + rspamd_task_add_result_option (task, s, opt); + } + + lua_pop (L, 1); + } + } + } + + } + + lua_pop (L, nresults); + } + } + + lua_pop (L, 1); /* Error function */ + rspamd_symcache_item_async_dec_check (task, cd->item, "lua symbol"); + g_assert (lua_gettop (L) == level - 1); +} + static void lua_metric_symbol_callback_return (struct thread_entry *thread_entry, int ret); @@ -1121,15 +1234,15 @@ static void lua_metric_symbol_callback_error (struct thread_entry *thread_entry, const char *msg); static void -lua_metric_symbol_callback (struct rspamd_task *task, - struct rspamd_symcache_item *item, - gpointer ud) +lua_metric_symbol_callback_coro (struct rspamd_task *task, + struct rspamd_symcache_item *item, + gpointer ud) { struct lua_callback_data *cd = ud; struct rspamd_task **ptask; struct thread_entry *thread_entry; - rspamd_symcache_item_async_inc (task, item, "lua symbol"); + rspamd_symcache_item_async_inc (task, item, "lua coro symbol"); thread_entry = lua_thread_pool_get_for_task (task); g_assert(thread_entry->cd == NULL); @@ -1163,9 +1276,9 @@ lua_metric_symbol_callback_error (struct thread_entry *thread_entry, { struct lua_callback_data *cd = thread_entry->cd; struct rspamd_task *task = thread_entry->task; - msg_err_task ("call to (%s) failed (%d): %s", cd->symbol, ret, msg); + msg_err_task ("call to coroutine (%s) failed (%d): %s", cd->symbol, ret, msg); - rspamd_symcache_item_async_dec_check (task, cd->item, "lua symbol"); + rspamd_symcache_item_async_dec_check (task, cd->item, "lua coro symbol"); } static void @@ -1246,7 +1359,7 @@ lua_metric_symbol_callback_return (struct thread_entry *thread_entry, int ret) g_assert (lua_gettop (L) == cd->stack_level); /* we properly cleaned up the stack */ cd->stack_level = 0; - rspamd_symcache_item_async_dec_check (task, cd->item, "lua symbol"); + rspamd_symcache_item_async_dec_check (task, cd->item, "lua coro symbol"); } static gint @@ -1282,6 +1395,10 @@ rspamd_register_symbol_fromlua (lua_State *L, } if (ref != -1) { + if (type & SYMBOL_TYPE_USE_CORO) { + /* Coroutines are incompatible with squeezing */ + no_squeeze = TRUE; + } /* * We call for routine called lua_squeeze_rules.squeeze_rule if it exists */ @@ -1349,15 +1466,27 @@ rspamd_register_symbol_fromlua (lua_State *L, cd->symbol = rspamd_mempool_strdup (cfg->cfg_pool, name); } - ret = rspamd_symcache_add_symbol (cfg->cache, - name, - priority, - lua_metric_symbol_callback, - cd, - type, - parent); + if (type & SYMBOL_TYPE_USE_CORO) { + ret = rspamd_symcache_add_symbol (cfg->cache, + name, + priority, + lua_metric_symbol_callback_coro, + cd, + type, + parent); + } + else { + ret = rspamd_symcache_add_symbol (cfg->cache, + name, + priority, + lua_metric_symbol_callback, + cd, + type, + parent); + } + rspamd_mempool_add_destructor (cfg->cfg_pool, - (rspamd_mempool_destruct_t)lua_destroy_cfg_symbol, + (rspamd_mempool_destruct_t) lua_destroy_cfg_symbol, cd); } } @@ -1373,13 +1502,24 @@ rspamd_register_symbol_fromlua (lua_State *L, cd->symbol = rspamd_mempool_strdup (cfg->cfg_pool, name); } - ret = rspamd_symcache_add_symbol (cfg->cache, - name, - priority, - lua_metric_symbol_callback, - cd, - type, - parent); + if (type & SYMBOL_TYPE_USE_CORO) { + ret = rspamd_symcache_add_symbol (cfg->cache, + name, + priority, + lua_metric_symbol_callback_coro, + cd, + type, + parent); + } + else { + ret = rspamd_symcache_add_symbol (cfg->cache, + name, + priority, + lua_metric_symbol_callback, + cd, + type, + parent); + } rspamd_mempool_add_destructor (cfg->cfg_pool, (rspamd_mempool_destruct_t)lua_destroy_cfg_symbol, cd); @@ -1556,6 +1696,9 @@ lua_parse_symbol_flags (const gchar *str) if (strstr (str, "explicit_disable") != NULL) { ret |= SYMBOL_TYPE_EXPLICIT_DISABLE; } + if (strstr (str, "coro") != NULL) { + ret |= SYMBOL_TYPE_USE_CORO; + } } return ret; @@ -2431,7 +2574,7 @@ lua_config_add_composite (lua_State * L) if (new) { rspamd_symcache_add_symbol (cfg->cache, name, - 0, NULL, NULL, SYMBOL_TYPE_COMPOSITE, -1); + 0, NULL, composite, SYMBOL_TYPE_COMPOSITE, -1); } ret = TRUE; @@ -2450,7 +2593,7 @@ lua_config_newindex (lua_State *L) LUA_TRACE_POINT; struct rspamd_config *cfg = lua_check_config (L, 1); const gchar *name; - gint id, nshots; + gint id, nshots, flags = 0; gboolean optional = FALSE, no_squeeze = FALSE; name = luaL_checkstring (L, 2); @@ -2476,7 +2619,6 @@ lua_config_newindex (lua_State *L) gint type = SYMBOL_TYPE_NORMAL, priority = 0, idx; gdouble weight = 1.0, score = NAN; const char *type_str, *group = NULL, *description = NULL; - guint flags = 0; no_squeeze = cfg->disable_lua_squeeze; /* @@ -2485,6 +2627,7 @@ lua_config_newindex (lua_State *L) * "weight" - optional weight * "priority" - optional priority * "type" - optional type (normal, virtual, callback) + * "flags" - optional flags * -- Metric options * "score" - optional default score (overridden by metric) * "group" - optional default group @@ -2537,6 +2680,15 @@ lua_config_newindex (lua_State *L) } lua_pop (L, 1); + lua_pushstring (L, "flags"); + lua_gettable (L, -2); + + if (lua_type (L, -1) == LUA_TSTRING) { + type_str = lua_tostring (L, -1); + type |= lua_parse_symbol_flags (type_str); + } + lua_pop (L, 1); + lua_pushstring (L, "condition"); lua_gettable (L, -2); diff --git a/src/lua/lua_dns_resolver.c b/src/lua/lua_dns_resolver.c index 362a6b35e..b68e15397 100644 --- a/src/lua/lua_dns_resolver.c +++ b/src/lua/lua_dns_resolver.c @@ -301,6 +301,8 @@ lua_push_dns_reply (lua_State *L, const struct rdns_reply *reply) lua_rawseti (L, -2, ++i); break; + default: + continue; } } lua_pushnil (L); diff --git a/src/lua/lua_html.c b/src/lua/lua_html.c index 739776b01..63839c286 100644 --- a/src/lua/lua_html.c +++ b/src/lua/lua_html.c @@ -594,6 +594,9 @@ lua_html_tag_get_parent (lua_State *L) *ptag = node->data; rspamd_lua_setclass (L, "rspamd{html_tag}", -1); } + else { + lua_pushnil (L); + } } else { return luaL_error (L, "invalid arguments"); diff --git a/src/lua/lua_http.c b/src/lua/lua_http.c index 2ca01d8a5..a8616e82a 100644 --- a/src/lua/lua_http.c +++ b/src/lua/lua_http.c @@ -379,23 +379,24 @@ lua_http_make_connection (struct lua_http_cbdata *cbd) cbd->fd = fd; if (cbd->cfg) { - cbd->conn = rspamd_http_connection_new (NULL, + cbd->conn = rspamd_http_connection_new ( + NULL, + fd, + NULL, lua_http_error_handler, lua_http_finish_handler, RSPAMD_HTTP_CLIENT_SIMPLE, - RSPAMD_HTTP_CLIENT, - NULL, - (cbd->flags & RSPAMD_LUA_HTTP_FLAG_NOVERIFY) ? - cbd->cfg->libs_ctx->ssl_ctx_noverify : cbd->cfg->libs_ctx->ssl_ctx); + RSPAMD_HTTP_CLIENT); } else { - cbd->conn = rspamd_http_connection_new (NULL, + cbd->conn = rspamd_http_connection_new ( + NULL, + fd, + NULL, lua_http_error_handler, lua_http_finish_handler, RSPAMD_HTTP_CLIENT_SIMPLE, - RSPAMD_HTTP_CLIENT, - NULL, - NULL); + RSPAMD_HTTP_CLIENT); } if (cbd->conn) { @@ -437,8 +438,8 @@ lua_http_make_connection (struct lua_http_cbdata *cbd) cbd->msg = NULL; rspamd_http_connection_write_message (cbd->conn, msg, - cbd->host, cbd->mime_type, cbd, fd, - &cbd->tv, cbd->ev_base); + cbd->host, cbd->mime_type, cbd, + &cbd->tv); return TRUE; } diff --git a/src/lua/lua_ip.c b/src/lua/lua_ip.c index 800c9765c..7f24f7712 100644 --- a/src/lua/lua_ip.c +++ b/src/lua/lua_ip.c @@ -176,6 +176,9 @@ static const struct luaL_reg iplib_m[] = { LUA_INTERFACE_DEF (ip, apply_mask), LUA_INTERFACE_DEF (ip, copy), LUA_INTERFACE_DEF (ip, is_local), + {"tostring", lua_ip_to_string}, + {"totable", lua_ip_to_table}, + {"tonumber", lua_ip_to_number}, {"__tostring", lua_ip_to_string}, {"__eq", lua_ip_equal}, {"__gc", lua_ip_destroy}, @@ -487,7 +490,7 @@ lua_ip_equal (lua_State *L) gboolean res = FALSE; if (ip1 && ip2 && ip1->addr && ip2->addr) { - res = rspamd_inet_address_compare (ip1->addr, ip2->addr); + res = rspamd_inet_address_compare (ip1->addr, ip2->addr, TRUE); } lua_pushboolean (L, res); diff --git a/src/lua/lua_map.c b/src/lua/lua_map.c index 98b025ea2..3ba7d0ed1 100644 --- a/src/lua/lua_map.c +++ b/src/lua/lua_map.c @@ -416,7 +416,7 @@ lua_map_read (gchar *chunk, gint len, } static void -lua_map_fin (struct map_cb_data *data) +lua_map_fin (struct map_cb_data *data, void **target) { struct lua_map_callback_data *cbdata; struct rspamd_lua_map **pmap; @@ -432,10 +432,6 @@ lua_map_fin (struct map_cb_data *data) return; } - if (data->prev_data) { - data->prev_data = NULL; - } - if (cbdata->ref == -1) { msg_err_map ("map has no callback set"); } @@ -454,6 +450,14 @@ lua_map_fin (struct map_cb_data *data) } cbdata->data = rspamd_fstring_assign (cbdata->data, "", 0); + + if (target) { + *target = data->cur_data; + } + + if (data->prev_data) { + data->prev_data = NULL; + } } static void diff --git a/src/lua/lua_task.c b/src/lua/lua_task.c index b6c5c1fb7..8e1a851b4 100644 --- a/src/lua/lua_task.c +++ b/src/lua/lua_task.c @@ -174,13 +174,13 @@ end */ LUA_FUNCTION_DEF (task, append_message); /*** - * @method task:get_urls([need_emails]) - * Get all URLs found in a message. - * @param {boolean} need_emails if `true` then reutrn also email urls + * @method task:get_urls([need_emails|list_protos]) + * Get all URLs found in a message. Telephone urls and emails are not included unless explicitly asked in `list_protos` + * @param {boolean} need_emails if `true` then reutrn also email urls, this can be a comma separated string of protocols desired or a table (e.g. `mailto` or `telephone`) * @return {table rspamd_url} list of all urls found @example local function phishing_cb(task) - local urls = task:get_urls(); + local urls = task:get_urls({'https', 'http'}); if urls then for _,url in ipairs(urls) do @@ -1831,18 +1831,22 @@ lua_task_append_message (lua_State * L) struct lua_tree_cb_data { lua_State *L; int i; + gint mask; }; static void lua_tree_url_callback (gpointer key, gpointer value, gpointer ud) { - struct rspamd_lua_url *url; + struct rspamd_lua_url *lua_url; + struct rspamd_url *url = (struct rspamd_url *)value; struct lua_tree_cb_data *cb = ud; - url = lua_newuserdata (cb->L, sizeof (struct rspamd_lua_url)); - rspamd_lua_setclass (cb->L, "rspamd{url}", -1); - url->url = value; - lua_rawseti (cb->L, -2, cb->i++); + if (url->protocol & cb->mask) { + lua_url = lua_newuserdata (cb->L, sizeof (struct rspamd_lua_url)); + rspamd_lua_setclass (cb->L, "rspamd{url}", -1); + lua_url->url = url; + lua_rawseti (cb->L, -2, cb->i++); + } } static gint @@ -1851,38 +1855,104 @@ lua_task_get_urls (lua_State * L) LUA_TRACE_POINT; struct rspamd_task *task = lua_check_task (L, 1); struct lua_tree_cb_data cb; - gboolean need_emails = FALSE; + gint protocols_mask = 0; + static const gint default_mask = PROTOCOL_HTTP|PROTOCOL_HTTPS| + PROTOCOL_FILE|PROTOCOL_FTP; gsize sz; if (task) { if (lua_gettop (L) >= 2) { - need_emails = lua_toboolean (L, 2); + if (lua_type (L, 2) == LUA_TBOOLEAN) { + protocols_mask = default_mask; + if (lua_toboolean (L, 2)) { + protocols_mask |= PROTOCOL_MAILTO; + } + } + else if (lua_type (L, 2) == LUA_TTABLE) { + for (lua_pushnil (L); lua_next (L, 2); lua_pop (L, 1)) { + int nmask; + const gchar *pname = lua_tostring (L, -1); + + nmask = rspamd_url_protocol_from_string (pname); + + if (nmask != PROTOCOL_UNKNOWN) { + protocols_mask |= nmask; + } + else { + msg_info ("bad url protocol: %s", pname); + } + } + } + else if (lua_type (L, 2) == LUA_TSTRING) { + const gchar *plist = lua_tostring (L, 2); + gchar **strvec; + gchar * const *cvec; + + strvec = g_strsplit_set (plist, ",;", -1); + cvec = strvec; + + while (*cvec) { + int nmask; + + nmask = rspamd_url_protocol_from_string (*cvec); + + if (nmask != PROTOCOL_UNKNOWN) { + protocols_mask |= nmask; + } + else { + msg_info ("bad url protocol: %s", *cvec); + } + + cvec ++; + } + + g_strfreev (strvec); + } + else { + protocols_mask = default_mask; + } } + else { + protocols_mask = default_mask; + } + + cb.i = 1; + cb.L = L; + cb.mask = protocols_mask; - if (need_emails) { + if (protocols_mask & PROTOCOL_MAILTO) { sz = g_hash_table_size (task->urls) + g_hash_table_size (task->emails); - if (!lua_task_get_cached (L, task, "emails+urls", sz)) { + if (protocols_mask == (default_mask|PROTOCOL_MAILTO)) { + /* Can use cached version */ + if (!lua_task_get_cached (L, task, "emails+urls", sz)) { + lua_createtable (L, sz, 0); + g_hash_table_foreach (task->urls, lua_tree_url_callback, &cb); + g_hash_table_foreach (task->emails, lua_tree_url_callback, &cb); + + lua_task_set_cached (L, task, "emails+urls", -1, sz); + } + } + else { lua_createtable (L, sz, 0); - cb.i = 1; - cb.L = L; g_hash_table_foreach (task->urls, lua_tree_url_callback, &cb); g_hash_table_foreach (task->emails, lua_tree_url_callback, &cb); - - lua_task_set_cached (L, task, "emails+urls", -1, sz); } } else { sz = g_hash_table_size (task->urls); - if (!lua_task_get_cached (L, task, "urls", sz)) { + if (protocols_mask == (default_mask)) { + if (!lua_task_get_cached (L, task, "urls", sz)) { + lua_createtable (L, sz, 0); + g_hash_table_foreach (task->urls, lua_tree_url_callback, &cb); + lua_task_set_cached (L, task, "urls", -1, sz); + } + } + else { lua_createtable (L, sz, 0); - cb.i = 1; - cb.L = L; g_hash_table_foreach (task->urls, lua_tree_url_callback, &cb); - - lua_task_set_cached (L, task, "urls", -1, sz); } } } @@ -2005,6 +2075,7 @@ lua_task_get_emails (lua_State * L) lua_createtable (L, g_hash_table_size (task->emails), 0); cb.i = 1; cb.L = L; + cb.mask = PROTOCOL_MAILTO; g_hash_table_foreach (task->emails, lua_tree_url_callback, &cb); } else { diff --git a/src/lua/lua_upstream.c b/src/lua/lua_upstream.c index 1a4d6b128..37c541d9d 100644 --- a/src/lua/lua_upstream.c +++ b/src/lua/lua_upstream.c @@ -110,7 +110,7 @@ lua_upstream_get_addr (lua_State *L) struct upstream *up = lua_check_upstream (L); if (up) { - rspamd_lua_ip_push (L, rspamd_upstream_addr (up)); + rspamd_lua_ip_push (L, rspamd_upstream_addr_next (up)); } else { lua_pushnil (L); diff --git a/src/lua/lua_url.c b/src/lua/lua_url.c index 8bc0cf657..0a301e96d 100644 --- a/src/lua/lua_url.c +++ b/src/lua/lua_url.c @@ -51,6 +51,7 @@ LUA_FUNCTION_DEF (url, tostring); LUA_FUNCTION_DEF (url, get_raw); LUA_FUNCTION_DEF (url, get_tld); LUA_FUNCTION_DEF (url, get_flags); +LUA_FUNCTION_DEF (url, get_protocol); LUA_FUNCTION_DEF (url, to_table); LUA_FUNCTION_DEF (url, is_phished); LUA_FUNCTION_DEF (url, is_redirected); @@ -77,6 +78,7 @@ static const struct luaL_reg urllib_m[] = { LUA_INTERFACE_DEF (url, get_text), LUA_INTERFACE_DEF (url, get_tld), LUA_INTERFACE_DEF (url, get_raw), + LUA_INTERFACE_DEF (url, get_protocol), LUA_INTERFACE_DEF (url, to_table), LUA_INTERFACE_DEF (url, is_phished), LUA_INTERFACE_DEF (url, is_redirected), @@ -607,6 +609,27 @@ lua_url_get_tld (lua_State *L) } /*** + * @method url:get_protocol() + * Get protocol name + * @return {string} protocol as a string + */ +static gint +lua_url_get_protocol (lua_State *L) +{ + LUA_TRACE_POINT; + struct rspamd_lua_url *url = lua_check_url (L, 1); + + if (url != NULL && url->url->protocol != PROTOCOL_UNKNOWN) { + lua_pushstring (L, rspamd_url_protocol_name (url->url->protocol)); + } + else { + lua_pushnil (L); + } + + return 1; +} + +/*** * @method url:get_count() * Return number of occurrencies for this particular URL * @return {number} number of occurrencies @@ -697,28 +720,7 @@ lua_url_to_table (lua_State *L) lua_pushstring (L, "protocol"); - - switch (u->protocol) { - case PROTOCOL_FILE: - lua_pushstring (L, "file"); - break; - case PROTOCOL_FTP: - lua_pushstring (L, "ftp"); - break; - case PROTOCOL_HTTP: - lua_pushstring (L, "http"); - break; - case PROTOCOL_HTTPS: - lua_pushstring (L, "https"); - break; - case PROTOCOL_MAILTO: - lua_pushstring (L, "mailto"); - break; - case PROTOCOL_UNKNOWN: - default: - lua_pushstring (L, "unknown"); - break; - } + lua_pushstring (L, rspamd_url_protocol_name (u->protocol)); lua_settable (L, -3); } else { diff --git a/src/plugins/dkim_check.c b/src/plugins/dkim_check.c index bb66e5ccc..f7600552b 100644 --- a/src/plugins/dkim_check.c +++ b/src/plugins/dkim_check.c @@ -669,13 +669,18 @@ dkim_module_load_key_format (struct rspamd_task *task, * This fails for paths that are also valid base64. * Maybe the caller should have specified a format. */ - if (key_format == RSPAMD_DKIM_KEY_UNKNOWN && - (key[0] == '.' || key[0] == '/')) { - if (!rspamd_cryptobox_base64_is_valid (key, keylen)) { - key_format = RSPAMD_DKIM_KEY_FILE; + if (key_format == RSPAMD_DKIM_KEY_UNKNOWN) { + if (key[0] == '.' || key[0] == '/') { + if (!rspamd_cryptobox_base64_is_valid (key, keylen)) { + key_format = RSPAMD_DKIM_KEY_FILE; + } + } + else if (rspamd_cryptobox_base64_is_valid (key, keylen)) { + key_format = RSPAMD_DKIM_KEY_BASE64; } } + if (ret != NULL && key_format == RSPAMD_DKIM_KEY_FILE) { msg_debug_task("checking for stale file key"); @@ -770,7 +775,7 @@ lua_dkim_sign_handler (lua_State *L) dkim_key = dkim_module_load_key_format (task, dkim_module_ctx, key, keylen, RSPAMD_DKIM_KEY_UNKNOWN); } - else if(rawkey) { + else if (rawkey) { dkim_key = dkim_module_load_key_format (task, dkim_module_ctx, rawkey, rawlen, RSPAMD_DKIM_KEY_UNKNOWN); } @@ -1540,7 +1545,13 @@ dkim_module_lua_push_verify_result (struct rspamd_dkim_lua_verify_cbdata *cbd, ptask = lua_newuserdata (cbd->L, sizeof (*ptask)); *ptask = task; lua_pushboolean (cbd->L, success); - lua_pushstring (cbd->L, error_str); + + if (error_str) { + lua_pushstring (cbd->L, error_str); + } + else { + lua_pushnil (cbd->L); + } if (cbd->ctx) { if (res->domain) { diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c index 7edb0168d..467a67ed7 100644 --- a/src/plugins/fuzzy_check.c +++ b/src/plugins/fuzzy_check.c @@ -43,6 +43,7 @@ #include "lua/lua_common.h" #include "unix-std.h" #include "libutil/http_private.h" +#include "libutil/http_router.h" #include "libstat/stat_api.h" #include <math.h> #include <src/libmime/message.h> @@ -2208,15 +2209,16 @@ fuzzy_check_io_callback (gint fd, short what, void *arg) msg_err_task ("got error on IO with server %s(%s), on %s, %d, %s", rspamd_upstream_name (session->server), rspamd_inet_address_to_string_pretty ( - rspamd_upstream_addr (session->server)), + rspamd_upstream_addr_cur (session->server)), session->state == 1 ? "read" : "write", errno, strerror (errno)); - rspamd_upstream_fail (session->server, FALSE); + rspamd_upstream_fail (session->server, TRUE); if (session->item) { rspamd_symcache_item_async_dec_check (session->task, session->item, M); } + rspamd_session_remove_event (session->task->s, fuzzy_io_fin, session); } else { @@ -2254,9 +2256,10 @@ fuzzy_check_timer_callback (gint fd, short what, void *arg) msg_err_task ("got IO timeout with server %s(%s), after %d retransmits", rspamd_upstream_name (session->server), rspamd_inet_address_to_string_pretty ( - rspamd_upstream_addr (session->server)), + rspamd_upstream_addr_cur (session->server)), session->retransmits); - rspamd_upstream_fail (session->server, FALSE); + rspamd_upstream_fail (session->server, TRUE); + if (session->item) { rspamd_symcache_item_async_dec_check (session->task, session->item, M); } @@ -2463,7 +2466,7 @@ fuzzy_controller_io_callback (gint fd, short what, void *arg) msg_err_task ("got error in IO with server %s(%s), %d, %s", rspamd_upstream_name (session->server), rspamd_inet_address_to_string_pretty ( - rspamd_upstream_addr (session->server)), + rspamd_upstream_addr_cur (session->server)), errno, strerror (errno)); rspamd_upstream_fail (session->server, FALSE); } @@ -2562,12 +2565,12 @@ fuzzy_controller_timer_callback (gint fd, short what, void *arg) task = session->task; if (session->retransmits >= session->rule->ctx->retransmits) { - rspamd_upstream_fail (session->server, FALSE); + rspamd_upstream_fail (session->server, TRUE); msg_err_task_check ("got IO timeout with server %s(%s), " "after %d retransmits", rspamd_upstream_name (session->server), rspamd_inet_address_to_string_pretty ( - rspamd_upstream_addr (session->server)), + rspamd_upstream_addr_cur (session->server)), session->retransmits); if (session->session) { @@ -2724,7 +2727,7 @@ register_fuzzy_client_call (struct rspamd_task *task, selected = rspamd_upstream_get (rule->servers, RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0); if (selected) { - addr = rspamd_upstream_addr (selected); + addr = rspamd_upstream_addr_next (selected); if ((sock = rspamd_inet_address_connect (addr, SOCK_DGRAM, TRUE)) == -1) { msg_warn_task ("cannot connect to %s(%s), %d, %s", rspamd_upstream_name (selected), @@ -2852,7 +2855,7 @@ register_fuzzy_controller_call (struct rspamd_http_connection_entry *entry, while ((selected = rspamd_upstream_get (rule->servers, RSPAMD_UPSTREAM_SEQUENTIAL, NULL, 0))) { /* Create UDP socket */ - addr = rspamd_upstream_addr (selected); + addr = rspamd_upstream_addr_next (selected); if ((sock = rspamd_inet_address_connect (addr, SOCK_DGRAM, TRUE)) == -1) { @@ -3215,7 +3218,7 @@ fuzzy_check_send_lua_learn (struct fuzzy_rule *rule, while ((selected = rspamd_upstream_get (rule->servers, RSPAMD_UPSTREAM_SEQUENTIAL, NULL, 0))) { /* Create UDP socket */ - addr = rspamd_upstream_addr (selected); + addr = rspamd_upstream_addr_next (selected); if ((sock = rspamd_inet_address_connect (addr, SOCK_DGRAM, TRUE)) == -1) { diff --git a/src/plugins/lua/arc.lua b/src/plugins/lua/arc.lua index 302861755..59d97fcd4 100644 --- a/src/plugins/lua/arc.lua +++ b/src/plugins/lua/arc.lua @@ -91,6 +91,7 @@ local settings = { } local function parse_arc_header(hdr, target) + -- Split elements by ';' and trim spaces local arr = fun.totable(fun.map( function(val) return fun.totable(fun.map(lua_util.rspamd_str_trim, @@ -102,8 +103,9 @@ local function parse_arc_header(hdr, target) -- Now we have two tables in format: -- [sigs] -> [{sig1_elts}, {sig2_elts}...] for i,elts in ipairs(arr) do + if not target[i] then target[i] = {} end + -- Split by kv pair, like k=v fun.each(function(v) - if not target[i] then target[i] = {} end if v[1] and v[2] then target[i][v[1]] = v[2] end diff --git a/src/plugins/lua/dkim_signing.lua b/src/plugins/lua/dkim_signing.lua index 4bc002548..9b1bfef32 100644 --- a/src/plugins/lua/dkim_signing.lua +++ b/src/plugins/lua/dkim_signing.lua @@ -151,12 +151,15 @@ local function dkim_signing_cb(task) if #selectors > 0 then for _, k in ipairs(selectors) do -- templates - k.key = lua_util.template(k.key, { - domain = k.domain, - selector = k.selector - }) - lua_util.debugm(N, task, 'using key "%s", use selector "%s" for domain "%s"', - k.key, k.selector, k.domain) + if k.key then + k.key = lua_util.template(k.key, { + domain = k.domain, + selector = k.selector + }) + lua_util.debugm(N, task, 'using key "%s", use selector "%s" for domain "%s"', + k.key, k.selector, k.domain) + end + do_sign(k) end else diff --git a/src/plugins/lua/mime_types.lua b/src/plugins/lua/mime_types.lua index ab2484c34..98acd463f 100644 --- a/src/plugins/lua/mime_types.lua +++ b/src/plugins/lua/mime_types.lua @@ -142,7 +142,6 @@ local settings = { scf = 2, shs = 2, theme = 2, - tmp = 2, url = 2, vbp = 2, vsmacros = 2, @@ -335,7 +334,7 @@ local full_extensions_map = { {"etl", "application/etl"}, {"etx", "text/x-setext"}, {"evy", "application/envoy"}, - {"exe", "application/x-dosexec"}, + {"exe", {"application/x-dosexec", "application/x-msdownload"}}, {"exe.config", "text/xml"}, {"fdf", "application/vnd.fdf"}, {"fif", "application/fractals"}, diff --git a/src/plugins/lua/multimap.lua b/src/plugins/lua/multimap.lua index 1e348690f..aa3e4d04a 100644 --- a/src/plugins/lua/multimap.lua +++ b/src/plugins/lua/multimap.lua @@ -28,7 +28,6 @@ local regexp = require "rspamd_regexp" local rspamd_expression = require "rspamd_expression" local rspamd_ip = require "rspamd_ip" local lua_util = require "lua_util" -local rspamd_dns = require "rspamd_dns" local lua_selectors = require "lua_selectors" local redis_params local fun = require "fun" @@ -260,9 +259,15 @@ local function apply_addr_filter(task, filter, input, rule) if addr and addr[1] then return addr[1]['name'] end + elseif filter == 'ip_addr' then + local ip_addr = rspamd_ip.from_string(input) + + if ip_addr and ip_addr:is_valid() then + return ip_addr + end else -- regexp case - if not rule['re_filter'] then + if not rule['re_filter'] then local type,pat = string.match(filter, '(regexp:)(.+)') if type and pat then rule['re_filter'] = regexp.create(pat) @@ -397,16 +402,17 @@ local function multimap_callback(task, rule) if r['cdb'] then local srch = value - if r['type'] == 'ip' then - srch = value:to_string() + if type(value) == 'userdata' then + if value.class == 'rspamd{ip}' then + srch = value:tostring() + end end ret = r['cdb']:lookup(srch) elseif r['redis_key'] then local srch = {value} local cmd = 'HGET' - if r['type'] == 'ip' or (r['type'] == 'received' and - (r['filter'] == 'real_ip' or r['filter'] == 'from_ip' or not r['filter'])) then - srch = {value:to_string()} + if type(value) == 'userdata' and value.class == 'rspamd{ip}' then + srch = {value:tostring()} cmd = 'HMGET' local maxbits = 128 local minbits = 32 @@ -415,7 +421,7 @@ local function multimap_callback(task, rule) minbits = 8 end for i=maxbits,minbits,-1 do - local nip = value:apply_mask(i):to_string() .. "/" .. i + local nip = value:apply_mask(i):tostring() .. "/" .. i table.insert(srch, nip) end end @@ -434,6 +440,11 @@ local function multimap_callback(task, rule) elseif r['radix'] then ret = r['radix']:get_key(value) elseif r['hash'] then + if type(value) == 'userdata' then + if value.class == 'rspamd{ip}' then + value = value:tostring() + end + end ret = r['hash']:get_key(value) end @@ -721,26 +732,30 @@ local function multimap_callback(task, rule) local ip = task:get_from_ip() if ip:is_valid() then local to_resolve = ip_to_rbl(ip, rule['map']) + local function dns_cb(_, _, results, err) + lua_util.debugm(N, rspamd_config, + 'resolve() finished: results=%1, err=%2, to_resolve=%3', + results, err, to_resolve) + + if err and + (err ~= 'requested record is not found' and + err ~= 'no records with this name') then + rspamd_logger.errx(task, 'error looking up %s: %s', to_resolve, results) + elseif results then + task:insert_result(rule['symbol'], 1, rule['map']) + if pre_filter then + task:set_pre_result(rule['action'], + 'Matched map: ' .. rule['symbol'], N) + end + end + end - local is_ok, results = rspamd_dns.request({ - type = "a", - task = task, + task:get_resolver():resolve_a({ + task= task, name = to_resolve, + callback = dns_cb, + forced = true }) - - lua_util.debugm(N, rspamd_config, - 'resolve() finished: results=%1, is_ok=%2, to_resolve=%3', - results, is_ok, to_resolve) - - if not is_ok and - (results ~= 'requested record is not found' and results ~= 'no records with this name') then - rspamd_logger.errx(task, 'error looking up %s: %s', to_resolve, results) - elseif is_ok then - task:insert_result(rule['symbol'], 1, rule['map']) - if pre_filter then - task:set_pre_result(rule['action'], 'Matched map: ' .. rule['symbol'], N) - end - end end end, header = function() @@ -990,7 +1005,7 @@ local function add_multimap_rule(key, newrule) local map = urls[newrule['map']] if map and map['regexp'] == newrule['regexp'] and map['glob'] == newrule['glob'] then - if newrule['type'] == 'ip' then + if newrule['type'] == 'ip' or newrule['filter'] == 'ip_addr' then newrule['radix'] = map['map'] else newrule['hash'] = map['map'] @@ -1072,14 +1087,22 @@ local function add_multimap_rule(key, newrule) or newrule['type'] == 'mempool' or newrule['type'] == 'selector'then - multimap_load_hash(newrule) + if newrule.filter == 'ip_addr' then + newrule['radix'] = rspamd_config:add_map ({ + url = newrule['map'], + description = newrule['description'], + type = 'radix' + }) + else + multimap_load_hash(newrule) + end - if newrule['hash'] then + if newrule.hash or newrule.radix then ret = true if type(newrule['map']) == 'string' then urls[newrule['map']] = { type = newrule['type'], - map = newrule['hash'], + map = newrule.hash or newrule.radix, regexp = newrule['regexp'] } end diff --git a/src/plugins/lua/reputation.lua b/src/plugins/lua/reputation.lua index 374771c9b..e91c6ebb7 100644 --- a/src/plugins/lua/reputation.lua +++ b/src/plugins/lua/reputation.lua @@ -25,7 +25,6 @@ local N = 'reputation' local rspamd_logger = require "rspamd_logger" local rspamd_util = require "rspamd_util" -local rspamd_dns = require "rspamd_dns" local lua_util = require "lua_util" local lua_maps = require "lua_maps" local hash = require 'rspamd_cryptobox_hash' @@ -857,39 +856,42 @@ local function reputation_dns_get_token(task, rule, token, continuation_cb) local key = gen_token_key(token, rule) local dns_name = key .. '.' .. rule.backend.config.list - local is_ok, results = rspamd_dns.request({ - type = 'a', - task = task, - name = dns_name, - forced = true, - }) - - if not is_ok and (results ~= 'requested record is not found' and results ~= 'no records with this name') then - rspamd_logger.errx(task, 'error looking up %s: %s', dns_name, results) - end + local function dns_cb(_, _, results, err) + if err and (err ~= 'requested record is not found' and + err ~= 'no records with this name') then + rspamd_logger.errx(task, 'error looking up %s: %s', dns_name, err) + end - lua_util.debugm(N, task, 'DNS RESPONSE: label=%1 results=%2 is_ok=%3 list=%4', - dns_name, results, is_ok, rule.backend.config.list) + lua_util.debugm(N, task, 'DNS RESPONSE: label=%1 results=%2 err=%3 list=%4', + dns_name, results, err, rule.backend.config.list) - -- Now split tokens to list of values - if is_ok then - local values = {} - -- Format: key1=num1;key2=num2...keyn=numn - fun.each(function(e) - local vals = lua_util.rspamd_str_split(e, "=") - if vals and #vals == 2 then - local nv = tonumber(vals[2]) - if nv then - values[vals[1]] = nv + -- Now split tokens to list of values + if results then + local values = {} + -- Format: key1=num1;key2=num2...keyn=numn + fun.each(function(e) + local vals = lua_util.rspamd_str_split(e, "=") + if vals and #vals == 2 then + local nv = tonumber(vals[2]) + if nv then + values[vals[1]] = nv + end end - end - end, - lua_util.rspamd_str_split(results[1], ";")) + end, + lua_util.rspamd_str_split(results[1], ";")) - continuation_cb(nil, dns_name, values) - else - continuation_cb(results, dns_name, nil) + continuation_cb(nil, dns_name, values) + else + continuation_cb(results, dns_name, nil) + end end + + task:get_resolver():resolve_a({ + task = task, + name = dns_name, + callback = dns_cb, + forced = true, + }) end local function reputation_redis_init(rule, cfg, ev_base, worker) diff --git a/src/plugins/lua/rspamd_update.lua b/src/plugins/lua/rspamd_update.lua index 51cb5db02..d53d02112 100644 --- a/src/plugins/lua/rspamd_update.lua +++ b/src/plugins/lua/rspamd_update.lua @@ -124,21 +124,25 @@ end -- Configuration part local section = rspamd_config:get_all_opt("rspamd_update") -if section then +if section and section.rules then local trusted_key - fun.each(function(k, elt) - if k == 'key' then - trusted_key = elt + if section.key then + trusted_key = section.key + end + + if type(section.rules) ~= 'table' then + section.rules = {section.rules} + end + + fun.each(function(elt) + local map = rspamd_config:add_map(elt, "rspamd updates map", nil, "callback") + if not map then + rspamd_logger.errx(rspamd_config, 'cannot load updates from %1', elt) else - local map = rspamd_config:add_map(elt, "rspamd updates map", nil, "callback") - if not map then - rspamd_logger.errx(rspamd_config, 'cannot load updates from %1', elt) - else - map:set_callback(gen_callback(map)) - maps['elt'] = map - end + map:set_callback(gen_callback(map)) + maps['elt'] = map end - end, section) + end, section.rules) fun.each(function(k, map) -- Check sanity for maps diff --git a/src/plugins/lua/settings.lua b/src/plugins/lua/settings.lua index 8d2122868..4e7afbf84 100644 --- a/src/plugins/lua/settings.lua +++ b/src/plugins/lua/settings.lua @@ -78,6 +78,10 @@ local function apply_settings(task, to_apply) to_apply.symbols)) end end + + if to_apply.subject then + task:set_metric_subject(to_apply.subject) + end end -- Checks for overridden settings within query params and returns 'true' if diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c index 94d88334e..26af1210c 100644 --- a/src/plugins/surbl.c +++ b/src/plugins/surbl.c @@ -270,11 +270,15 @@ read_exceptions_list (gchar * chunk, } static void -fin_exceptions_list (struct map_cb_data *data) +fin_exceptions_list (struct map_cb_data *data, void **target) { GHashTable **t; gint i; + if (target) { + *target = data->cur_data; + } + if (data->prev_data) { t = data->prev_data; for (i = 0; i < MAX_LEVELS; i++) { @@ -385,11 +389,15 @@ read_redirectors_list (gchar * chunk, final); } -void -fin_redirectors_list (struct map_cb_data *data) +static void +fin_redirectors_list (struct map_cb_data *data, void **target) { GHashTable *tld_hash; + if (target) { + *target = data->cur_data; + } + if (data->prev_data) { tld_hash = data->prev_data; @@ -397,7 +405,7 @@ fin_redirectors_list (struct map_cb_data *data) } } -void +static void dtor_redirectors_list (struct map_cb_data *data) { GHashTable *tld_hash; @@ -1624,7 +1632,8 @@ surbl_redirector_error (struct rspamd_http_connection *conn, task = param->task; msg_err_surbl ("connection with http server %s terminated incorrectly: %e", - rspamd_inet_address_to_string (rspamd_upstream_addr (param->redirector)), + rspamd_inet_address_to_string ( + rspamd_upstream_addr_cur (param->redirector)), err); rspamd_upstream_fail (param->redirector, FALSE); rspamd_session_remove_event (param->task->s, free_redirector_session, @@ -1715,7 +1724,7 @@ register_redirector_call (struct rspamd_url *url, struct rspamd_task *task, RSPAMD_UPSTREAM_ROUND_ROBIN, url->host, url->hostlen); if (selected) { - s = rspamd_inet_address_connect (rspamd_upstream_addr (selected), + s = rspamd_inet_address_connect (rspamd_upstream_addr_next (selected), SOCK_STREAM, TRUE); } @@ -1733,12 +1742,12 @@ register_redirector_call (struct rspamd_url *url, struct rspamd_task *task, param->url = url; param->task = task; param->conn = rspamd_http_connection_new (NULL, + s, + NULL, surbl_redirector_error, surbl_redirector_finish, RSPAMD_HTTP_CLIENT_SIMPLE, - RSPAMD_HTTP_CLIENT, - NULL, - NULL); + RSPAMD_HTTP_CLIENT); param->ctx = surbl_module_ctx; msg = rspamd_http_new_message (HTTP_REQUEST); msg->url = rspamd_fstring_assign (msg->url, url->string, url->urllen); @@ -1757,7 +1766,7 @@ register_redirector_call (struct rspamd_url *url, struct rspamd_task *task, } rspamd_http_connection_write_message (param->conn, msg, NULL, - NULL, param, s, timeout, task->ev_base); + NULL, param, timeout); msg_info_surbl ( "<%s> registered redirector call for %*s to %s, according to rule: %s", diff --git a/src/rspamadm/CMakeLists.txt b/src/rspamadm/CMakeLists.txt index 8e3f09435..5be38aa28 100644 --- a/src/rspamadm/CMakeLists.txt +++ b/src/rspamadm/CMakeLists.txt @@ -25,7 +25,6 @@ IF (ENABLE_HYPERSCAN MATCHES "ON") ENDIF() ADD_EXECUTABLE(rspamadm ${RSPAMADMSRC}) TARGET_LINK_LIBRARIES(rspamadm rspamd-server) -TARGET_LINK_LIBRARIES(rspamadm ${RSPAMD_REQUIRED_LIBRARIES}) IF (NOT DEBIAN_BUILD) SET_TARGET_PROPERTIES(rspamadm PROPERTIES VERSION ${RSPAMD_VERSION}) diff --git a/src/rspamadm/control.c b/src/rspamadm/control.c index 0e39ed703..0a242e943 100644 --- a/src/rspamadm/control.c +++ b/src/rspamadm/control.c @@ -17,7 +17,7 @@ #include "rspamadm.h" #include "cryptobox.h" #include "printf.h" -#include "libutil/http.h" +#include "libutil/http_connection.h" #include "libutil/http_private.h" #include "addr.h" #include "unix-std.h" @@ -111,6 +111,7 @@ rspamd_control_finish_handler (struct rspamd_http_connection *conn, const gchar *body; gsize body_len; struct rspamadm_control_cbdata *cbdata = conn->ud; + struct timeval exit_tv; body = rspamd_http_message_get_body (msg, &body_len); parser = ucl_parser_new (0); @@ -141,7 +142,7 @@ rspamd_control_finish_handler (struct rspamd_http_connection *conn, rspamd_fstring_free (out); ucl_object_unref (obj); ucl_parser_free (parser); - return 0; + goto end; } else { rspamd_ucl_emit_fstring (obj, UCL_EMIT_CONFIG, &out); @@ -155,6 +156,11 @@ rspamd_control_finish_handler (struct rspamd_http_connection *conn, ucl_parser_free (parser); } +end: + exit_tv.tv_sec = 0; + exit_tv.tv_usec = 0; + event_base_loopexit (rspamd_main->ev_base, &exit_tv); + return 0; } @@ -163,7 +169,6 @@ rspamadm_control (gint argc, gchar **argv, const struct rspamadm_command *_cmd) { GOptionContext *context; GError *error = NULL; - struct event_base *ev_base; const gchar *cmd, *path = NULL; struct rspamd_http_connection *conn; struct rspamd_http_message *msg; @@ -225,7 +230,6 @@ rspamadm_control (gint argc, gchar **argv, const struct rspamadm_command *_cmd) exit (1); } - ev_base = event_init (); sock = rspamd_inet_address_connect (addr, SOCK_STREAM, TRUE); if (sock == -1) { @@ -234,13 +238,14 @@ rspamadm_control (gint argc, gchar **argv, const struct rspamadm_command *_cmd) exit (1); } - conn = rspamd_http_connection_new (NULL, + conn = rspamd_http_connection_new ( + rspamd_main->http_ctx, /* Default context */ + sock, + NULL, rspamd_control_error_handler, rspamd_control_finish_handler, RSPAMD_HTTP_CLIENT_SIMPLE, - RSPAMD_HTTP_CLIENT, - NULL, - NULL); + RSPAMD_HTTP_CLIENT); msg = rspamd_http_new_message (HTTP_REQUEST); msg->url = rspamd_fstring_new_init (path, strlen (path)); double_to_tv (timeout, &tv); @@ -249,10 +254,10 @@ rspamadm_control (gint argc, gchar **argv, const struct rspamadm_command *_cmd) cbdata.argv = argv; cbdata.path = path; - rspamd_http_connection_write_message (conn, msg, NULL, NULL, &cbdata, sock, - &tv, ev_base); + rspamd_http_connection_write_message (conn, msg, NULL, NULL, &cbdata, + &tv); - event_base_loop (ev_base, 0); + event_base_loop (rspamd_main->ev_base, 0); rspamd_http_connection_unref (conn); rspamd_inet_address_free (addr); diff --git a/src/rspamadm/dkim_keygen.c b/src/rspamadm/dkim_keygen.c index 86b228d01..62b47f557 100644 --- a/src/rspamadm/dkim_keygen.c +++ b/src/rspamadm/dkim_keygen.c @@ -17,6 +17,8 @@ #include "rspamadm.h" #include "printf.h" #include "str_util.h" +#include "libcryptobox/cryptobox.h" +#include "contrib/libottery/ottery.h" #include "lua/lua_common.h" #include <openssl/rsa.h> #include <openssl/bn.h> @@ -26,6 +28,7 @@ static gchar *privkey_file = NULL; static gchar *selector = NULL; static gchar *domain = NULL; static guint bits = 1024; +static gchar *type = "rsa"; static void rspamadm_dkim_keygen (gint argc, gchar **argv, const struct rspamadm_command *cmd); @@ -50,6 +53,8 @@ static GOptionEntry entries[] = { "Save private key in the specified file", NULL}, {"bits", 'b', 0, G_OPTION_ARG_INT, &bits, "Set key length to N bits (1024 by default)", NULL}, + {"type", 't', 0, G_OPTION_ARG_STRING, &type, + "Key type: rsa or ed25519 (rsa by default)", NULL}, {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL} }; @@ -76,8 +81,9 @@ rspamadm_dkim_keygen_help (gboolean full_help, const struct rspamadm_command *cm } static void -rspamadm_dkim_generate_keypair (const gchar *domain, const gchar *selector, - const gchar *priv_fname, const gchar *pub_fname, guint keylen) +rspamd_dkim_generate_rsa_keypair (const gchar *domain, const gchar *selector, + const gchar *priv_fname, const gchar *pub_fname, + guint keylen) { BIGNUM *e; RSA *r; @@ -109,8 +115,7 @@ rspamadm_dkim_generate_keypair (const gchar *domain, const gchar *selector, priv_fname, strerror (errno)); exit (EXIT_FAILURE); } - } - else { + } else { privout = BIO_new_fp (stdout, 0); } @@ -125,7 +130,7 @@ rspamadm_dkim_generate_keypair (const gchar *domain, const gchar *selector, BIO_free (privout); fflush (stdout); - pubout = BIO_new (BIO_s_mem()); + pubout = BIO_new (BIO_s_mem ()); rc = i2d_RSA_PUBKEY_bio (pubout, r); publen = BIO_get_mem_data (pubout, &pubdata); @@ -141,18 +146,16 @@ rspamadm_dkim_generate_keypair (const gchar *domain, const gchar *selector, pub_fname, strerror (errno)); exit (EXIT_FAILURE); } - } - else { + } else { pubfile = stdout; } if (b64_len < 255 - 2) { rspamd_fprintf (pubfile, "%s._domainkey IN TXT ( \"v=DKIM1; k=rsa; \"\n" - "\t\"p=%s\" ) ;\n", + "\t\"p=%s\" ) ;\n", selector ? selector : "selector", b64_data); - } - else { + } else { guint i; gint step = 253, remain = b64_len; @@ -162,8 +165,7 @@ rspamadm_dkim_generate_keypair (const gchar *domain, const gchar *selector, for (i = 0; i < b64_len; i += step, remain -= step) { if (i == 0) { rspamd_fprintf (pubfile, "\t\"p=%*s\"\n", MIN(step, remain), &b64_data[i]); - } - else { + } else { step = 255; rspamd_fprintf (pubfile, "\t\"%*s\"\n", MIN(step, remain), &b64_data[i]); } @@ -183,6 +185,120 @@ rspamadm_dkim_generate_keypair (const gchar *domain, const gchar *selector, BN_free (e); } +static void +rspamd_dkim_generate_ed25519_keypair (const gchar *domain, const gchar *selector, + const gchar *priv_fname, const gchar *pub_fname, + guint keylen, gboolean seeded) +{ + rspamd_sig_sk_t ed_sk; + rspamd_sig_pk_t ed_pk; + gchar *base64_pk, *base64_sk; + FILE *pubfile = NULL, *privfile = NULL; + + rspamd_cryptobox_keypair_sig (ed_pk, ed_sk, RSPAMD_CRYPTOBOX_MODE_25519); + if (seeded) { + /* Just encode seed, not the full sk */ + base64_sk = rspamd_encode_base64_common (ed_sk, 32, 0, NULL, FALSE, + RSPAMD_TASK_NEWLINES_LF); + } + else { + base64_sk = rspamd_encode_base64_common (ed_sk, + rspamd_cryptobox_sk_sig_bytes (RSPAMD_CRYPTOBOX_MODE_25519), + 0, NULL, FALSE, + RSPAMD_TASK_NEWLINES_LF); + } + base64_pk = rspamd_encode_base64_common (ed_pk, sizeof (ed_pk), 0, NULL, FALSE, + RSPAMD_TASK_NEWLINES_LF); + + /* Cleanup sensitive data */ + rspamd_explicit_memzero (ed_sk, sizeof (ed_sk)); + + if (priv_fname) { + privfile = fopen (priv_fname, "w"); + + if (privfile == NULL) { + rspamd_fprintf (stderr, "cannot open output file %s: %s\n", + priv_fname, strerror (errno)); + rspamd_explicit_memzero (base64_sk, strlen (base64_sk)); + g_free (base64_sk); + g_free (base64_pk); + exit (EXIT_FAILURE); + } + } + else { + privfile = stdout; + } + + if (rspamd_fprintf (privfile, "%s\n", base64_sk) == -1) { + rspamd_fprintf (stderr, "cannot write to output file %s: %s\n", + priv_fname, strerror (errno)); + rspamd_explicit_memzero (base64_sk, strlen (base64_sk)); + g_free (base64_sk); + g_free (base64_pk); + + if (privfile != stdout) { + fclose (privfile); + } + + exit (EXIT_FAILURE); + } + + if (privfile != stdout) { + fclose (privfile); + } + + if (pub_fname) { + pubfile = fopen (pub_fname, "w"); + + if (pubfile == NULL) { + rspamd_fprintf (stderr, "cannot open output file %s: %s\n", + pub_fname, strerror (errno)); + rspamd_explicit_memzero (base64_sk, strlen (base64_sk)); + g_free (base64_sk); + g_free (base64_pk); + exit (EXIT_FAILURE); + } + } + else { + pubfile = stdout; + } + + rspamd_fprintf (pubfile, "%s._domainkey IN TXT ( \"v=DKIM1; k=ed25519; \"\n" + "\t\"p=%s\" ) ;\n", + selector ? selector : "selector", + base64_pk); + + if (pubfile != stdout) { + fclose (pubfile); + } + + rspamd_explicit_memzero (base64_sk, strlen (base64_sk)); + g_free (base64_sk); + g_free (base64_pk); +} + +static void +rspamadm_dkim_generate_keypair (const gchar *domain, const gchar *selector, + const gchar *priv_fname, const gchar *pub_fname, guint keylen) +{ + if (strcmp (type, "rsa") == 0) { + rspamd_dkim_generate_rsa_keypair (domain, selector, priv_fname, + pub_fname, keylen); + } + else if (strcmp (type, "ed25519") == 0) { + rspamd_dkim_generate_ed25519_keypair (domain, selector, priv_fname, + pub_fname, keylen, FALSE); + } + else if (strcmp (type, "ed25519-seed") == 0) { + rspamd_dkim_generate_ed25519_keypair (domain, selector, priv_fname, + pub_fname, keylen, TRUE); + } + else { + fprintf (stderr, "invalid key type: %s\n", type); + exit (EXIT_FAILURE); + } +} + static gint rspamadm_dkim_keygen_lua_generate (lua_State *L) { diff --git a/src/rspamadm/lua_repl.c b/src/rspamadm/lua_repl.c index 6d1501100..e91bd34fa 100644 --- a/src/rspamadm/lua_repl.c +++ b/src/rspamadm/lua_repl.c @@ -16,8 +16,9 @@ #include "config.h" #include "rspamadm.h" -#include "libutil/http.h" +#include "libutil/http_connection.h" #include "libutil/http_private.h" +#include "libutil/http_router.h" #include "printf.h" #include "lua/lua_common.h" #include "lua/lua_thread_pool.h" @@ -807,8 +808,9 @@ rspamadm_lua (gint argc, gchar **argv, const struct rspamadm_command *cmd) ctx = g_malloc0 (sizeof (*ctx)); http = rspamd_http_router_new (rspamadm_lua_error_handler, rspamadm_lua_finish_handler, - NULL, ev_base, - NULL, NULL); + NULL, + NULL, + rspamd_main->http_ctx); ctx->L = L; ctx->rt = http; rspamd_http_router_add_path (http, diff --git a/src/rspamadm/rspamadm.c b/src/rspamadm/rspamadm.c index ef52af1e3..c0bb4bc72 100644 --- a/src/rspamadm/rspamadm.c +++ b/src/rspamadm/rspamadm.c @@ -436,6 +436,7 @@ main (gint argc, gchar **argv, gchar **env) (void) dns_resolver_init (rspamd_main->logger, rspamd_main->ev_base, cfg); + rspamd_main->http_ctx = rspamd_http_context_create (cfg, rspamd_main->ev_base); g_log_set_default_handler (rspamd_glib_log_function, rspamd_main->logger); g_set_printerr_handler (rspamd_glib_printerr_function); diff --git a/src/rspamd.c b/src/rspamd.c index 88b44d773..27ba5e032 100644 --- a/src/rspamd.c +++ b/src/rspamd.c @@ -178,6 +178,7 @@ read_cmd_line (gint *argc, gchar ***argv, struct rspamd_config *cfg) else { cfg->cfg_name = cfg_names[0]; } + for (i = 1; i < cfg_num; i++) { r = fork (); if (r == 0) { @@ -313,6 +314,9 @@ reread_config (struct rspamd_main *rspamd_main) REF_RELEASE (old_cfg); msg_info_main ("config has been reread successfully"); rspamd_map_preload (rspamd_main->cfg); + + rspamd_main->cfg->rspamd_user = rspamd_user; + rspamd_main->cfg->rspamd_group = rspamd_group; } } @@ -1159,7 +1163,7 @@ rspamd_control_handler (gint fd, short what, gpointer arg) msg_info_main ("accepted control connection from %s", rspamd_inet_address_to_string (addr)); - rspamd_control_process_client_socket (rspamd_main, nfd); + rspamd_control_process_client_socket (rspamd_main, nfd, addr); } static guint @@ -1485,6 +1489,9 @@ main (gint argc, gchar **argv, gchar **env) spawn_workers (rspamd_main, ev_base); rspamd_mempool_unlock_mutex (rspamd_main->start_mtx); + rspamd_main->http_ctx = rspamd_http_context_create (rspamd_main->cfg, + ev_base); + if (control_fd != -1) { msg_info_main ("listening for control commands on %s", rspamd_inet_address_to_string (control_addr)); diff --git a/src/rspamd.h b/src/rspamd.h index 74f08c2d3..10d3be9fb 100644 --- a/src/rspamd.h +++ b/src/rspamd.h @@ -22,7 +22,7 @@ #include "libutil/mem_pool.h" #include "libutil/util.h" #include "libutil/logger.h" -#include "libutil/http.h" +#include "libutil/http_connection.h" #include "libutil/upstream.h" #include "libutil/radix.h" #include "libserver/url.h" @@ -277,6 +277,7 @@ struct rspamd_main { gboolean cores_throttling; /**< turn off cores when limits are exceeded */ struct roll_history *history; /**< rolling history */ struct event_base *ev_base; + struct rspamd_http_context *http_ctx; }; enum rspamd_exception_type { diff --git a/src/rspamd_proxy.c b/src/rspamd_proxy.c index 9ea2b0d74..b6ede29b9 100644 --- a/src/rspamd_proxy.c +++ b/src/rspamd_proxy.c @@ -17,7 +17,7 @@ #include "libutil/util.h" #include "libutil/map.h" #include "libutil/upstream.h" -#include "libutil/http.h" +#include "libutil/http_connection.h" #include "libutil/http_private.h" #include "libserver/protocol.h" #include "libserver/protocol_internal.h" @@ -123,18 +123,14 @@ struct rspamd_proxy_ctx { struct timeval io_tv; /* Encryption key for clients */ struct rspamd_cryptobox_keypair *key; - /* Keys cache */ - struct rspamd_keypair_cache *keys_cache; + /* HTTP context */ + struct rspamd_http_context *http_ctx; /* Upstreams to use */ GHashTable *upstreams; /* Mirrors to send traffic to */ GPtrArray *mirrors; /* Default upstream */ struct rspamd_http_upstream *default_upstream; - /* Local rotating keypair for upstreams */ - struct rspamd_cryptobox_keypair *local_key; - struct event rotate_ev; - gdouble rotate_tm; lua_State *lua_state; /* Array of callback functions called on end of scan to compare results */ GArray *cmp_refs; @@ -746,7 +742,6 @@ init_rspamd_proxy (struct rspamd_config *cfg) ctx->mirrors = g_ptr_array_new (); rspamd_mempool_add_destructor (cfg->cfg_pool, (rspamd_mempool_destruct_t)rspamd_ptr_array_free_hard, ctx->mirrors); - ctx->rotate_tm = DEFAULT_ROTATION_TIME; ctx->cfg = cfg; ctx->lua_state = cfg->lua_state; ctx->cmp_refs = g_array_new (FALSE, FALSE, sizeof (gint)); @@ -765,15 +760,6 @@ init_rspamd_proxy (struct rspamd_config *cfg) "IO timeout"); rspamd_rcl_register_worker_option (cfg, type, - "rotate", - rspamd_rcl_parse_struct_time, - ctx, - G_STRUCT_OFFSET (struct rspamd_proxy_ctx, rotate_tm), - RSPAMD_CL_FLAG_TIME_FLOAT, - "Rotation keys time, default: " - G_STRINGIFY (DEFAULT_ROTATION_TIME) " seconds"); - rspamd_rcl_register_worker_option (cfg, - type, "keypair", rspamd_rcl_parse_struct_keypair, ctx, @@ -1309,7 +1295,8 @@ proxy_backend_mirror_error_handler (struct rspamd_http_connection *conn, GError msg_info_session ("abnormally closing connection from backend: %s:%s, " "error: %e", bk_conn->name, - rspamd_inet_address_to_string (rspamd_upstream_addr (bk_conn->up)), + rspamd_inet_address_to_string ( + rspamd_upstream_addr_cur (bk_conn->up)), err); if (err) { @@ -1337,7 +1324,8 @@ proxy_backend_mirror_finish_handler (struct rspamd_http_connection *conn, bk_conn->parser_from_ref, msg->body_buf.begin, msg->body_buf.len)) { msg_warn_session ("cannot parse results from the mirror backend %s:%s", bk_conn->name, - rspamd_inet_address_to_string (rspamd_upstream_addr (bk_conn->up))); + rspamd_inet_address_to_string ( + rspamd_upstream_addr_cur (bk_conn->up))); bk_conn->err = "cannot parse ucl"; } @@ -1387,7 +1375,7 @@ proxy_open_mirror_connections (struct rspamd_proxy_session *session) } bk_conn->backend_sock = rspamd_inet_address_connect ( - rspamd_upstream_addr (bk_conn->up), + rspamd_upstream_addr_next (bk_conn->up), SOCK_STREAM, TRUE); if (bk_conn->backend_sock == -1) { @@ -1416,23 +1404,22 @@ proxy_open_mirror_connections (struct rspamd_proxy_session *session) rspamd_http_message_add_header (msg, "Settings-ID", m->settings_id); } - bk_conn->backend_conn = rspamd_http_connection_new (NULL, + bk_conn->backend_conn = rspamd_http_connection_new ( + session->ctx->http_ctx, + bk_conn->backend_sock, + NULL, proxy_backend_mirror_error_handler, proxy_backend_mirror_finish_handler, RSPAMD_HTTP_CLIENT_SIMPLE, - RSPAMD_HTTP_CLIENT, - session->ctx->keys_cache, - NULL); + RSPAMD_HTTP_CLIENT); if (m->key) { - rspamd_http_connection_set_key (bk_conn->backend_conn, - session->ctx->local_key); msg->peer_key = rspamd_pubkey_ref (m->key); } if (m->local || rspamd_inet_address_is_local ( - rspamd_upstream_addr (bk_conn->up), FALSE)) { + rspamd_upstream_addr_cur (bk_conn->up), FALSE)) { if (session->fname) { rspamd_http_message_add_header (msg, "File", session->fname); @@ -1441,8 +1428,7 @@ proxy_open_mirror_connections (struct rspamd_proxy_session *session) msg->method = HTTP_GET; rspamd_http_connection_write_message_shared (bk_conn->backend_conn, msg, NULL, NULL, bk_conn, - bk_conn->backend_sock, - bk_conn->io_tv, session->ctx->ev_base); + bk_conn->io_tv); } else { if (session->fname) { @@ -1469,8 +1455,7 @@ proxy_open_mirror_connections (struct rspamd_proxy_session *session) rspamd_http_connection_write_message (bk_conn->backend_conn, msg, NULL, NULL, bk_conn, - bk_conn->backend_sock, - bk_conn->io_tv, session->ctx->ev_base); + bk_conn->io_tv); } g_ptr_array_add (session->mirror_conns, bk_conn); @@ -1495,8 +1480,8 @@ proxy_client_write_error (struct rspamd_proxy_session *session, gint code, reply->code = code; reply->status = rspamd_fstring_new_init (status, strlen (status)); rspamd_http_connection_write_message (session->client_conn, - reply, NULL, NULL, session, session->client_sock, - &session->ctx->io_tv, session->ctx->ev_base); + reply, NULL, NULL, session, + &session->ctx->io_tv); } } @@ -1509,7 +1494,8 @@ proxy_backend_master_error_handler (struct rspamd_http_connection *conn, GError session = bk_conn->s; msg_info_session ("abnormally closing connection from backend: %s, error: %e," " retries left: %d", - rspamd_inet_address_to_string (rspamd_upstream_addr (session->master_conn->up)), + rspamd_inet_address_to_string ( + rspamd_upstream_addr_cur (session->master_conn->up)), err, session->ctx->max_retries - session->retries); session->retries ++; @@ -1531,7 +1517,7 @@ proxy_backend_master_error_handler (struct rspamd_http_connection *conn, GError msg_info_session ("retry connection to: %s" " retries left: %d", rspamd_inet_address_to_string ( - rspamd_upstream_addr (session->master_conn->up)), + rspamd_upstream_addr_cur (session->master_conn->up)), session->ctx->max_retries - session->retries); } } @@ -1592,8 +1578,8 @@ proxy_backend_master_finish_handler (struct rspamd_http_connection *conn, } else { rspamd_http_connection_write_message (session->client_conn, - msg, NULL, NULL, session, session->client_sock, - bk_conn->io_tv, session->ctx->ev_base); + msg, NULL, NULL, session, + bk_conn->io_tv); } return 0; @@ -1652,9 +1638,7 @@ rspamd_proxy_scan_self_reply (struct rspamd_task *task) NULL, ctype, session, - session->client_sock, - NULL, - session->ctx->ev_base); + NULL); } } @@ -1821,14 +1805,15 @@ retry: } session->master_conn->backend_sock = rspamd_inet_address_connect ( - rspamd_upstream_addr (session->master_conn->up), + rspamd_upstream_addr_next (session->master_conn->up), SOCK_STREAM, TRUE); if (session->master_conn->backend_sock == -1) { msg_err_session ("cannot connect upstream: %s(%s)", host ? hostbuf : "default", - rspamd_inet_address_to_string (rspamd_upstream_addr ( - session->master_conn->up))); + rspamd_inet_address_to_string ( + rspamd_upstream_addr_cur ( + session->master_conn->up))); rspamd_upstream_fail (session->master_conn->up, TRUE); session->retries ++; goto retry; @@ -1847,21 +1832,19 @@ retry: } session->master_conn->backend_conn = rspamd_http_connection_new ( + session->ctx->http_ctx, + session->master_conn->backend_sock, NULL, proxy_backend_master_error_handler, proxy_backend_master_finish_handler, RSPAMD_HTTP_CLIENT_SIMPLE, - RSPAMD_HTTP_CLIENT, - session->ctx->keys_cache, - NULL); + RSPAMD_HTTP_CLIENT); session->master_conn->flags &= ~RSPAMD_BACKEND_CLOSED; session->master_conn->parser_from_ref = backend->parser_from_ref; session->master_conn->parser_to_ref = backend->parser_to_ref; if (backend->key) { msg->peer_key = rspamd_pubkey_ref (backend->key); - rspamd_http_connection_set_key (session->master_conn->backend_conn, - session->ctx->local_key); } if (backend->settings_id != NULL) { @@ -1872,7 +1855,8 @@ retry: if (backend->local || rspamd_inet_address_is_local ( - rspamd_upstream_addr (session->master_conn->up), FALSE)) { + rspamd_upstream_addr_cur ( + session->master_conn->up), FALSE)) { if (session->fname) { rspamd_http_message_add_header (msg, "File", session->fname); @@ -1883,8 +1867,7 @@ retry: rspamd_http_connection_write_message_shared ( session->master_conn->backend_conn, msg, NULL, NULL, session->master_conn, - session->master_conn->backend_sock, - session->master_conn->io_tv, session->ctx->ev_base); + session->master_conn->io_tv); } else { if (session->fname) { @@ -1912,8 +1895,7 @@ retry: rspamd_http_connection_write_message ( session->master_conn->backend_conn, msg, NULL, NULL, session->master_conn, - session->master_conn->backend_sock, - session->master_conn->io_tv, session->ctx->ev_base); + session->master_conn->io_tv); } } @@ -2100,13 +2082,14 @@ proxy_accept_socket (gint fd, short what, void *arg) } if (!ctx->milter) { - session->client_conn = rspamd_http_connection_new (NULL, + session->client_conn = rspamd_http_connection_new ( + ctx->http_ctx, + nfd, + NULL, proxy_client_error_handler, proxy_client_finish_handler, 0, - RSPAMD_HTTP_SERVER, - ctx->keys_cache, - NULL); + RSPAMD_HTTP_SERVER); if (ctx->key) { rspamd_http_connection_set_key (session->client_conn, ctx->key); @@ -2118,9 +2101,7 @@ proxy_accept_socket (gint fd, short what, void *arg) rspamd_http_connection_read_message_shared (session->client_conn, session, - nfd, - &ctx->io_tv, - ctx->ev_base); + &ctx->io_tv); } else { msg_info_session ("accepted milter connection from %s port %d", @@ -2154,24 +2135,6 @@ proxy_accept_socket (gint fd, short what, void *arg) } static void -proxy_rotate_key (gint fd, short what, void *arg) -{ - struct timeval rot_tv; - struct rspamd_proxy_ctx *ctx = arg; - gpointer kp; - - double_to_tv (ctx->rotate_tm, &rot_tv); - rot_tv.tv_sec += ottery_rand_range (rot_tv.tv_sec); - event_del (&ctx->rotate_ev); - event_add (&ctx->rotate_ev, &rot_tv); - - kp = ctx->local_key; - ctx->local_key = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX, - RSPAMD_CRYPTOBOX_MODE_25519); - rspamd_keypair_unref (kp); -} - -static void adjust_upstreams_limits (struct rspamd_proxy_ctx *ctx) { struct rspamd_http_upstream *backend; @@ -2200,9 +2163,9 @@ adjust_upstreams_limits (struct rspamd_proxy_ctx *ctx) } void -start_rspamd_proxy (struct rspamd_worker *worker) { +start_rspamd_proxy (struct rspamd_worker *worker) +{ struct rspamd_proxy_ctx *ctx = worker->ctx; - struct timeval rot_tv; ctx->cfg = worker->srv->cfg; ctx->ev_base = rspamd_prepare_worker (worker, "rspamd_proxy", @@ -2217,16 +2180,7 @@ start_rspamd_proxy (struct rspamd_worker *worker) { rspamd_upstreams_library_config (worker->srv->cfg, ctx->cfg->ups_ctx, ctx->ev_base, ctx->resolver->r); - /* XXX: stupid default */ - ctx->keys_cache = rspamd_keypair_cache_new (256); - ctx->local_key = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX, - RSPAMD_CRYPTOBOX_MODE_25519); - - double_to_tv (ctx->rotate_tm, &rot_tv); - rot_tv.tv_sec += ottery_rand_range (rot_tv.tv_sec); - event_set (&ctx->rotate_ev, -1, EV_TIMEOUT, proxy_rotate_key, ctx); - event_base_set (ctx->ev_base, &ctx->rotate_ev); - event_add (&ctx->rotate_ev, &rot_tv); + ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base); if (ctx->has_self_scan) { /* Additional initialisation needed */ @@ -2259,8 +2213,9 @@ start_rspamd_proxy (struct rspamd_worker *worker) { rspamd_stat_close (); } - rspamd_keypair_cache_destroy (ctx->keys_cache); + struct rspamd_http_context *http_ctx = ctx->http_ctx; REF_RELEASE (ctx->cfg); + rspamd_http_context_free (http_ctx); rspamd_log_close (worker->srv->logger, TRUE); exit (EXIT_SUCCESS); diff --git a/src/worker.c b/src/worker.c index 2f7176a8d..d81be54a1 100644 --- a/src/worker.c +++ b/src/worker.c @@ -331,9 +331,7 @@ rspamd_worker_error_handler (struct rspamd_http_connection *conn, GError *err) NULL, "application/json", task, - task->http_conn->fd, - &task->tv, - task->ev_base); + &task->tv); } } @@ -414,13 +412,14 @@ accept_socket (gint fd, short what, void *arg) http_opts = RSPAMD_HTTP_REQUIRE_ENCRYPTION; } - task->http_conn = rspamd_http_connection_new (rspamd_worker_body_handler, + task->http_conn = rspamd_http_connection_new ( + ctx->http_ctx, + nfd, + rspamd_worker_body_handler, rspamd_worker_error_handler, rspamd_worker_finish_handler, http_opts, - RSPAMD_HTTP_SERVER, - ctx->keys_cache, - NULL); + RSPAMD_HTTP_SERVER); rspamd_http_connection_set_max_size (task->http_conn, task->cfg->max_message); worker->nconns++; rspamd_mempool_add_destructor (task->task_pool, @@ -436,9 +435,7 @@ accept_socket (gint fd, short what, void *arg) rspamd_http_connection_read_message (task->http_conn, task, - nfd, - &ctx->io_tv, - ctx->ev_base); + &ctx->io_tv); } #ifdef WITH_HYPERSCAN @@ -696,8 +693,7 @@ start_worker (struct rspamd_worker *worker) rspamd_upstreams_library_config (worker->srv->cfg, ctx->cfg->ups_ctx, ctx->ev_base, ctx->resolver->r); - /* XXX: stupid default */ - ctx->keys_cache = rspamd_keypair_cache_new (256); + ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base); rspamd_worker_init_scanner (worker, ctx->ev_base, ctx->resolver, &ctx->lang_det); rspamd_lua_run_postloads (ctx->cfg->lua_state, ctx->cfg, ctx->ev_base, @@ -707,8 +703,9 @@ start_worker (struct rspamd_worker *worker) rspamd_worker_block_signals (); rspamd_stat_close (); - rspamd_keypair_cache_destroy (ctx->keys_cache); + struct rspamd_http_context *http_ctx = ctx->http_ctx; REF_RELEASE (ctx->cfg); + rspamd_http_context_free (http_ctx); rspamd_log_close (worker->srv->logger, TRUE); exit (EXIT_SUCCESS); diff --git a/src/worker_private.h b/src/worker_private.h index f07a95b41..398c5d23d 100644 --- a/src/worker_private.h +++ b/src/worker_private.h @@ -49,7 +49,7 @@ struct rspamd_worker_ctx { /* Encryption key */ struct rspamd_cryptobox_keypair *key; /* Keys cache */ - struct rspamd_keypair_cache *keys_cache; + struct rspamd_http_context *http_ctx; /* Language detector */ struct rspamd_lang_detector *lang_det; }; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ee6b074c3..10f0a19da 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -8,7 +8,6 @@ SET(TESTSRC rspamd_mem_pool_test.c rspamd_radix_test.c rspamd_shingles_test.c rspamd_upstream_test.c - rspamd_http_test.c rspamd_lua_pcall_vs_resume_test.c rspamd_lua_test.c rspamd_cryptobox_test.c @@ -19,11 +18,9 @@ ADD_EXECUTABLE(rspamd-test EXCLUDE_FROM_ALL ${TESTSRC}) SET_TARGET_PROPERTIES(rspamd-test PROPERTIES LINKER_LANGUAGE C) SET_TARGET_PROPERTIES(rspamd-test PROPERTIES COMPILE_FLAGS "-DRSPAMD_TEST") ADD_DEPENDENCIES(rspamd-test rspamd-server) -TARGET_LINK_LIBRARIES(rspamd-test ${RSPAMD_REQUIRED_LIBRARIES}) IF(USE_CXX_LINKER) SET_TARGET_PROPERTIES(rspamd-test PROPERTIES LINKER_LANGUAGE CXX) ENDIF() -TARGET_LINK_LIBRARIES(rspamd-test rspamd-actrie) TARGET_LINK_LIBRARIES(rspamd-test rspamd-server) IF(NOT "${CMAKE_CURRENT_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_BINARY_DIR}") diff --git a/test/functional/cases/106_mid.robot b/test/functional/cases/106_mid.robot new file mode 100644 index 000000000..7d8459dbd --- /dev/null +++ b/test/functional/cases/106_mid.robot @@ -0,0 +1,42 @@ +*** Settings *** +Suite Setup MID Setup +Suite Teardown Normal Teardown +Library ${TESTDIR}/lib/rspamd.py +Resource ${TESTDIR}/lib/rspamd.robot +Variables ${TESTDIR}/lib/vars.py + +*** Variables *** +${CONFIG} ${TESTDIR}/configs/plugins.conf +${RSPAMD_SCOPE} Suite +${URL_TLD} ${TESTDIR}/../lua/unit/test_tld.dat + +*** Test Cases *** +MID - invalid Message-ID + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/fws_fp.eml + Check Rspamc ${result} INVALID_MSGID (1.70) + Should Not Contain ${result.stdout} MISSING_MID + Should Not Contain ${result.stdout} INVALID_MSGID_ALLOWED + +MID - invalid Message-ID allowed + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/invalid_mid_allowed.eml + Check Rspamc ${result} INVALID_MSGID_ALLOWED (1.00) + Should Not Contain ${result.stdout} MISSING_MID + Should Not Contain ${result.stdout} INVALID_MSGID ( + +MID - missing Message-ID + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/freemail.eml + Check Rspamc ${result} MISSING_MID (2.50) + Should Not Contain ${result.stdout} MISSING_MID_ALLOWED + Should Not Contain ${result.stdout} INVALID_MSGID + +MID - missing Message-ID allowed + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/onsubdomain_pass_relaxed.eml + Check Rspamc ${result} MISSING_MID_ALLOWED (1.00) + Should Not Contain ${result.stdout} MISSING_MID ( + Should Not Contain ${result.stdout} INVALID_MSGID + +*** Keywords *** +MID Setup + ${PLUGIN_CONFIG} = Get File ${TESTDIR}/configs/mid.conf + Set Suite Variable ${PLUGIN_CONFIG} + Generic Setup PLUGIN_CONFIG diff --git a/test/functional/cases/131_dkim_signing/001_simple.robot b/test/functional/cases/131_dkim_signing/001_simple.robot index 96bdcc997..2bcd53f36 100644 --- a/test/functional/cases/131_dkim_signing/001_simple.robot +++ b/test/functional/cases/131_dkim_signing/001_simple.robot @@ -19,6 +19,11 @@ TEST SIGNED Check Rspamc ${result} DKIM-Signature: Should Contain ${result.stdout} DKIM_SIGNED +TEST SIGNED HTTP HEADERS + ${result} = Scan Message With Rspamc ${MESSAGE} -u bob@cacophony.za.org --header=PerformDkimSign:yes --header=DkimDomain:cacophony.za.org --header=DkimSelector:dkim --header=DkimPrivateKey:MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANe3EETkiI1Exyrb+VzbMSt90K8MXJA0GcyNs6MFCs9JPaTh90Zu2l7ki7m5LTUx6350AR/3hcvwjSHCZjD6fvQ8/zfjN8kaLZ6DAaqtqSlpawIM+8glkuTEkIkpBED/OtDrba4Rd29iLFVuwQZXDtTjAAZKZPmtTZ5TXLrcCU6VAgMBAAECgYEA1BFvmBsIN8Gu/+6kNupya2xUNVM0yLu/xT5lpNV3LBO325oejAq8+d87kkl/LTW3a2jGFlQ0ICuLw+2mo24QUWRyv8if3oeBMlnLqHE+6wNjFVqo5sOjKzjO363xSXwXNUrBT7jDhnZcDN8w3/FecYKjifGTVtUs1SLsYwhlc8ECQQDuCRymLZQ/imPn5eFVIydwUzg8ptZlvoA7bfIxUL9BQRX33s59kLCilA0tTed8Dd+GnxsT93XOj1ApIfBwmTSlAkEA5/63PDsN7fH+WInqVD8nU07M9S8LcGDlPbVVBr2S2I78/iwrSDAYtbkU2vEbhFK/JuKNML2j8OkzV3v1QulfMQJBALDzhx+l/HHr3+8RPhx7QKNIyiKUaAdEwbDsP8IXY8YPq1QThu9jM1v4sX7/TdkzuvoppwiFykbe1NlvCH279p0CQCmTg4Ee0DtBcCSr6rvYaZLLf329RZ6JLuwlMCy6ErQOxBZFEiiovfTrS2qFZToMnkc4uLbwdY36LQJTq7unGTECQCCok8LzBeZtAw+TJofpOM3F2Rlm2qXiBVBeubhRedsiljG0hpvvLJBMppnQ6r27p5Jk39SmaTRkxEKrxPWWLNM= + Check Rspamc ${result} DKIM-Signature: + Should Contain ${result.stdout} DKIM_SIGNED + TEST NOT SIGNED - USERNAME WRONG DOMAIN ${result} = Scan Message With Rspamc ${MESSAGE} -u bob@example.tk Check Rspamc ${result} DKIM-Signature: inverse=1 diff --git a/test/functional/cases/231_tcp_down.robot b/test/functional/cases/231_tcp_down.robot index 474faa3f5..f852724ad 100644 --- a/test/functional/cases/231_tcp_down.robot +++ b/test/functional/cases/231_tcp_down.robot @@ -16,7 +16,7 @@ ${RSPAMD_SCOPE} Test *** Test Cases *** Sync API TCP get request when server is down - [Documentation] We don't create HTTP server here, that's why + [Documentation] We don't create HTTP server here, that's why ... all requests fail with "connection refused" Check url /request get HTTP_ASYNC_RESPONSE (0.00)[Socket error detected: Connection refused] Check url /content-length HTTP_SYNC_WRITE_ERROR (0.00)[Socket error detected: Connection refused] diff --git a/test/functional/configs/dkim_signing/simple.conf b/test/functional/configs/dkim_signing/simple.conf index 2302a0c4f..14b8d5ec6 100644 --- a/test/functional/configs/dkim_signing/simple.conf +++ b/test/functional/configs/dkim_signing/simple.conf @@ -2,4 +2,6 @@ dkim_signing { path = "${TESTDIR}/configs/dkim.key"; check_pubkey = true; allow_pubkey_mismatch = false; + use_http_headers = true; + allow_headers_fallback = true; } diff --git a/test/functional/configs/maps/mid.list b/test/functional/configs/maps/mid.list new file mode 100644 index 000000000..b89c084f0 --- /dev/null +++ b/test/functional/configs/maps/mid.list @@ -0,0 +1,2 @@ +cacophony.za.org /^<[A-z0-9+]{18}>$/ +mom.za.org diff --git a/test/functional/configs/mid.conf b/test/functional/configs/mid.conf new file mode 100644 index 000000000..49f5cb568 --- /dev/null +++ b/test/functional/configs/mid.conf @@ -0,0 +1,9 @@ +mid = { + source = { + url = [ + "https://maps.rspamd.com/rspamd/mid.inc.zst", + "fallback+file://${TESTDIR}/../../../conf/mid.inc", + "file://${TESTDIR}/configs/maps/mid.list" + ]; + } +} diff --git a/test/functional/lua/dns.lua b/test/functional/lua/dns.lua index 62b7d21e6..702b98550 100644 --- a/test/functional/lua/dns.lua +++ b/test/functional/lua/dns.lua @@ -22,7 +22,8 @@ rspamd_config:register_symbol({ name = 'SIMPLE_DNS_SYNC', score = 1.0, callback = dns_sync_symbol, - no_squeeze = true + no_squeeze = true, + flags = 'coro', }) diff --git a/test/functional/lua/http.lua b/test/functional/lua/http.lua index 0263beb6f..1981c8734 100644 --- a/test/functional/lua/http.lua +++ b/test/functional/lua/http.lua @@ -109,7 +109,8 @@ rspamd_config:register_symbol({ name = 'SIMPLE_TEST', score = 1.0, callback = http_symbol, -no_squeeze = true +no_squeeze = true, +flags = 'coro' }) diff --git a/test/functional/lua/redis.lua b/test/functional/lua/redis.lua index e4512b5e4..1a1eaf2b7 100644 --- a/test/functional/lua/redis.lua +++ b/test/functional/lua/redis.lua @@ -23,12 +23,12 @@ local function redis_simple_async_symbol(task) end redis_lua.rspamd_redis_make_request( - task, + task, redis_params, "test_key", false, redis_cb, - 'GET', + 'GET', {'test_key'} ) end @@ -47,7 +47,7 @@ local function redis_simple_async_api201809(task) callback = redis_cb } local request = { - 'GET', + 'GET', 'test_key' } redis_lua.request(redis_params, attrs, request) @@ -108,6 +108,6 @@ rspamd_config:register_symbol({ name = 'REDIS_TEST', score = 1.0, callback = redis_symbol, - no_squeeze = true + flags = 'coro', }) -- ]] diff --git a/test/functional/lua/tcp.lua b/test/functional/lua/tcp.lua index 30285c4f5..d032a049f 100644 --- a/test/functional/lua/tcp.lua +++ b/test/functional/lua/tcp.lua @@ -61,7 +61,7 @@ local function http_simple_tcp_symbol(task) local data local got_content = '' repeat - is_ok, data = connection:read_once(); + is_ok, data = connection:read_once(); logger.errx(task, 'read_once: is_ok: %1, data: %2', is_ok, data) if not is_ok then task:insert_result('HTTP_SYNC_ERROR', 1.0, data) @@ -147,7 +147,7 @@ local function http_tcp_symbol(task) end local value - local header = header_line:gsub("([%w-]+): (.*)", + local header = header_line:gsub("([%w-]+): (.*)", function (h, v) value = v; return h:lower() end) logger.errx(task, 'parsed header: %1 -> "%2"', header, value) @@ -182,13 +182,15 @@ rspamd_config:register_symbol({ name = 'SIMPLE_TCP_TEST', score = 1.0, callback = http_simple_tcp_symbol, - no_squeeze = true + no_squeeze = true, + flags = 'coro', }) rspamd_config:register_symbol({ name = 'HTTP_TCP_TEST', score = 1.0, callback = http_tcp_symbol, - no_squeeze = true + no_squeeze = true, + flags = 'coro', }) -- ]] diff --git a/test/functional/messages/invalid_mid_allowed.eml b/test/functional/messages/invalid_mid_allowed.eml new file mode 100644 index 000000000..a7ee95081 --- /dev/null +++ b/test/functional/messages/invalid_mid_allowed.eml @@ -0,0 +1,10 @@ +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=cacophony.za.org; + s=dkim; t=1550213458; + h=from:sender:reply-to:subject:date:message-id:message-id:to:cc: + mime-version:content-type:content-transfer-encoding:in-reply-to: + references; bh=47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=; + b=vcs/v32V8AvN1x0qk8DihZq46R2C35z1A4h6xEg26dDq2JSVZQOd1+WYRtmHG5SL6JlQsb + KxBHbIAFIoah7JA+uKt49mrKOVcnq9dww/P4Os3lDjm7WMwjDb8eEX2kECINjVo5MDMp6C + cCMnjVLppV76SL8f5ExdHDsw6eZ9y9k= +Message-ID: <JE47oX6KasQDNv6BEw> + diff --git a/test/rspamd_http_test.c b/test/rspamd_http_test.c index e70c396aa..b81cc0b8b 100644 --- a/test/rspamd_http_test.c +++ b/test/rspamd_http_test.c @@ -16,7 +16,8 @@ #include "config.h" #include "rspamd.h" #include "util.h" -#include "libutil/http.h" +#include "libutil/http_connection.h" +#include "libutil/http_router.h" #include "libutil/http_private.h" #include "tests.h" #include "ottery.h" diff --git a/test/rspamd_test_suite.c b/test/rspamd_test_suite.c index 6ca8d6465..2abce72a1 100644 --- a/test/rspamd_test_suite.c +++ b/test/rspamd_test_suite.c @@ -74,13 +74,13 @@ main (int argc, char **argv) g_test_add_func ("/rspamd/rrd", rspamd_rrd_test_func); g_test_add_func ("/rspamd/upstream", rspamd_upstream_test_func); g_test_add_func ("/rspamd/shingles", rspamd_shingles_test_func); - g_test_add_func ("/rspamd/http", rspamd_http_test_func); g_test_add_func ("/rspamd/lua", rspamd_lua_test_func); g_test_add_func ("/rspamd/cryptobox", rspamd_cryptobox_test_func); g_test_add_func ("/rspamd/heap", rspamd_heap_test_func); g_test_add_func ("/rspamd/lua_pcall", rspamd_lua_lua_pcall_vs_resume_test_func); #if 0 + g_test_add_func ("/rspamd/http", rspamd_http_test_func); g_test_add_func ("/rspamd/url", rspamd_url_test_func); g_test_add_func ("/rspamd/statfile", rspamd_statfile_test_func); g_test_add_func ("/rspamd/aio", rspamd_async_test_func); diff --git a/test/rspamd_upstream_test.c b/test/rspamd_upstream_test.c index 4e4f1ae87..f9eb2bcfd 100644 --- a/test/rspamd_upstream_test.c +++ b/test/rspamd_upstream_test.c @@ -87,30 +87,30 @@ rspamd_upstream_test_func (void) rspamd_parse_inet_address (&paddr, "::1", 0); g_assert (rspamd_upstream_add_addr (up, paddr)); /* Rewind to start */ - addr = rspamd_upstream_addr (up); - addr = rspamd_upstream_addr (up); + addr = rspamd_upstream_addr_next (up); + addr = rspamd_upstream_addr_next (up); /* cur should be zero here */ - addr = rspamd_upstream_addr (up); - next_addr = rspamd_upstream_addr (up); + addr = rspamd_upstream_addr_next (up); + next_addr = rspamd_upstream_addr_next (up); g_assert (rspamd_inet_address_get_af (addr) == AF_INET); g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET); - next_addr = rspamd_upstream_addr (up); + next_addr = rspamd_upstream_addr_next (up); g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET6); - next_addr = rspamd_upstream_addr (up); + next_addr = rspamd_upstream_addr_next (up); g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET); - next_addr = rspamd_upstream_addr (up); + next_addr = rspamd_upstream_addr_next (up); g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET); - next_addr = rspamd_upstream_addr (up); + next_addr = rspamd_upstream_addr_next (up); g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET6); /* Test errors with IPv6 */ rspamd_upstream_fail (up, TRUE); /* Now we should have merely IPv4 addresses in rotation */ - addr = rspamd_upstream_addr (up); + addr = rspamd_upstream_addr_next (up); for (i = 0; i < 256; i++) { - next_addr = rspamd_upstream_addr (up); + next_addr = rspamd_upstream_addr_next (up); g_assert (rspamd_inet_address_get_af (addr) == AF_INET); g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET); - g_assert (rspamd_inet_address_compare (addr, next_addr) != 0); + g_assert (rspamd_inet_address_compare (addr, next_addr, FALSE) != 0); addr = next_addr; } rspamd_upstreams_destroy (nls); |