aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorheraklit256 <37872459+heraklit256@users.noreply.github.com>2019-03-05 16:05:03 +0000
committerGitHub <noreply@github.com>2019-03-05 16:05:03 +0000
commit46de8e9f6e51dc44623f253f6a5fe9ecad3dc554 (patch)
treeee759abdc966c3415a87ea4ad91c8f10f2c132f7
parentf40cc5500eecf1e0b26d6b4be3b7f3bbb78dede0 (diff)
parentff5125a975f3a3c143c84aa9cf2f6878f5930272 (diff)
downloadrspamd-46de8e9f6e51dc44623f253f6a5fe9ecad3dc554.tar.gz
rspamd-46de8e9f6e51dc44623f253f6a5fe9ecad3dc554.zip
Merge pull request #11 from rspamd/master
merge upstream into local master
-rw-r--r--.github/ISSUE_TEMPLATE.md6
-rw-r--r--CMakeLists.txt17
-rw-r--r--conf/composites.conf2
-rw-r--r--contrib/http-parser/http_parser.c3
-rw-r--r--contrib/http-parser/http_parser.h4
-rw-r--r--contrib/librdns/util.c18
-rw-r--r--lualib/lua_cfg_transform.lua6
-rw-r--r--lualib/lua_clickhouse.lua10
-rw-r--r--lualib/lua_dkim_tools.lua39
-rw-r--r--lualib/lua_ffi/init.lua1
-rw-r--r--lualib/lua_selectors.lua21
-rw-r--r--rules/html.lua45
-rw-r--r--rules/regexp/misc.lua4
-rw-r--r--src/CMakeLists.txt10
-rw-r--r--src/client/CMakeLists.txt1
-rw-r--r--src/client/rspamc.c18
-rw-r--r--src/client/rspamdclient.c26
-rw-r--r--src/client/rspamdclient.h12
-rw-r--r--src/controller.c31
-rw-r--r--src/fuzzy_storage.c56
-rw-r--r--src/libmime/content_type.c54
-rw-r--r--src/libmime/lang_detection.c19
-rw-r--r--src/libmime/message.c2
-rw-r--r--src/libmime/mime_headers.c109
-rw-r--r--src/libmime/mime_parser.c38
-rw-r--r--src/libmime/mime_parser.h1
-rw-r--r--src/libserver/cfg_file.h21
-rw-r--r--src/libserver/cfg_rcl.c110
-rw-r--r--src/libserver/cfg_utils.c40
-rw-r--r--src/libserver/composites.c7
-rw-r--r--src/libserver/dkim.c26
-rw-r--r--src/libserver/dns.c2
-rw-r--r--src/libserver/dynamic_cfg.c27
-rw-r--r--src/libserver/fuzzy_backend_redis.c8
-rw-r--r--src/libserver/html.c94
-rw-r--r--src/libserver/milter.c29
-rw-r--r--src/libserver/monitored.c6
-rw-r--r--src/libserver/protocol.c2
-rw-r--r--src/libserver/protocol.h2
-rw-r--r--src/libserver/rspamd_control.c46
-rw-r--r--src/libserver/rspamd_control.h2
-rw-r--r--src/libserver/rspamd_symcache.c156
-rw-r--r--src/libserver/rspamd_symcache.h13
-rw-r--r--src/libserver/task.h2
-rw-r--r--src/libserver/url.c429
-rw-r--r--src/libserver/url.h28
-rw-r--r--src/libserver/worker_util.c19
-rw-r--r--src/libserver/worker_util.h2
-rw-r--r--src/libstat/backends/redis_backend.c6
-rw-r--r--src/libstat/learn_cache/redis_cache.c2
-rw-r--r--src/libstat/stat_api.h1
-rw-r--r--src/libstat/tokenizers/tokenizers.c20
-rw-r--r--src/libutil/CMakeLists.txt52
-rw-r--r--src/libutil/addr.c115
-rw-r--r--src/libutil/addr.h7
-rw-r--r--src/libutil/http.h578
-rw-r--r--src/libutil/http_connection.c (renamed from src/libutil/http.c)1961
-rw-r--r--src/libutil/http_connection.h251
-rw-r--r--src/libutil/http_context.c433
-rw-r--r--src/libutil/http_context.h95
-rw-r--r--src/libutil/http_message.c660
-rw-r--r--src/libutil/http_message.h221
-rw-r--r--src/libutil/http_private.h39
-rw-r--r--src/libutil/http_router.c555
-rw-r--r--src/libutil/http_router.h138
-rw-r--r--src/libutil/http_util.c513
-rw-r--r--src/libutil/http_util.h48
-rw-r--r--src/libutil/map.c33
-rw-r--r--src/libutil/map.h2
-rw-r--r--src/libutil/map_helpers.c63
-rw-r--r--src/libutil/map_helpers.h6
-rw-r--r--src/libutil/multipattern.c21
-rw-r--r--src/libutil/str_util.c73
-rw-r--r--src/libutil/str_util.h19
-rw-r--r--src/libutil/upstream.c14
-rw-r--r--src/libutil/upstream.h9
-rw-r--r--src/libutil/uthash_strcase.h29
-rw-r--r--src/libutil/util.c152
-rw-r--r--src/libutil/util.h3
-rw-r--r--src/lua/lua_config.c202
-rw-r--r--src/lua/lua_dns_resolver.c2
-rw-r--r--src/lua/lua_html.c3
-rw-r--r--src/lua/lua_http.c23
-rw-r--r--src/lua/lua_ip.c5
-rw-r--r--src/lua/lua_map.c14
-rw-r--r--src/lua/lua_task.c115
-rw-r--r--src/lua/lua_upstream.c2
-rw-r--r--src/lua/lua_url.c46
-rw-r--r--src/plugins/dkim_check.c23
-rw-r--r--src/plugins/fuzzy_check.c23
-rw-r--r--src/plugins/lua/arc.lua4
-rw-r--r--src/plugins/lua/dkim_signing.lua15
-rw-r--r--src/plugins/lua/mime_types.lua3
-rw-r--r--src/plugins/lua/multimap.lua81
-rw-r--r--src/plugins/lua/reputation.lua60
-rw-r--r--src/plugins/lua/rspamd_update.lua28
-rw-r--r--src/plugins/lua/settings.lua4
-rw-r--r--src/plugins/surbl.c29
-rw-r--r--src/rspamadm/CMakeLists.txt1
-rw-r--r--src/rspamadm/control.c27
-rw-r--r--src/rspamadm/dkim_keygen.c140
-rw-r--r--src/rspamadm/lua_repl.c8
-rw-r--r--src/rspamadm/rspamadm.c1
-rw-r--r--src/rspamd.c9
-rw-r--r--src/rspamd.h3
-rw-r--r--src/rspamd_proxy.c137
-rw-r--r--src/worker.c23
-rw-r--r--src/worker_private.h2
-rw-r--r--test/CMakeLists.txt3
-rw-r--r--test/functional/cases/106_mid.robot42
-rw-r--r--test/functional/cases/131_dkim_signing/001_simple.robot5
-rw-r--r--test/functional/cases/231_tcp_down.robot2
-rw-r--r--test/functional/configs/dkim_signing/simple.conf2
-rw-r--r--test/functional/configs/maps/mid.list2
-rw-r--r--test/functional/configs/mid.conf9
-rw-r--r--test/functional/lua/dns.lua3
-rw-r--r--test/functional/lua/http.lua3
-rw-r--r--test/functional/lua/redis.lua8
-rw-r--r--test/functional/lua/tcp.lua10
-rw-r--r--test/functional/messages/invalid_mid_allowed.eml10
-rw-r--r--test/rspamd_http_test.c3
-rw-r--r--test/rspamd_test_suite.c2
-rw-r--r--test/rspamd_upstream_test.c22
123 files changed, 5467 insertions, 3435 deletions
diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 19d88fd31..738507d9d 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -2,9 +2,7 @@
* [x] Crash/Hang/Data loss
* [ ] WebUI/Usability
-* [ ] Serious bug
-* [ ] Ordinary bug
-* [ ] Feature
+* [ ] Unintended behaviour (bug)
* [ ] Enhancement
### Reproducibility (Please choose *one* option):
@@ -18,7 +16,7 @@
### Rspamd version:
-### Operation system, CPU:
+### Operation system (the exact version), CPU (including architecture and processor family):
### Description (Please provide a descriptive summary of the issue):
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1e20c6f8d..9935c4460 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1225,12 +1225,6 @@ SET(WITH_HIREDIS 1)
INCLUDE_DIRECTORIES(BEFORE "${CMAKE_SOURCE_DIR}/contrib/hiredis")
LIST(APPEND RSPAMD_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}")
-LIST(APPEND RSPAMD_REQUIRED_LIBRARIES ucl)
-LIST(APPEND RSPAMD_REQUIRED_LIBRARIES rdns)
-LIST(APPEND RSPAMD_REQUIRED_LIBRARIES ottery)
-LIST(APPEND RSPAMD_REQUIRED_LIBRARIES event)
-LIST(APPEND RSPAMD_REQUIRED_LIBRARIES xxhash)
-LIST(APPEND RSPAMD_REQUIRED_LIBRARIES rspamd-actrie)
IF(HAVE_FETCH_H)
LIST(APPEND RSPAMD_REQUIRED_LIBRARIES fetch)
ENDIF(HAVE_FETCH_H)
@@ -1238,8 +1232,12 @@ IF(WITH_DB)
LIST(APPEND RSPAMD_REQUIRED_LIBRARIES db)
ENDIF(WITH_DB)
-
LIST(APPEND RSPAMD_REQUIRED_LIBRARIES "${LUA_LIBRARY}")
+LIST(APPEND RSPAMD_REQUIRED_LIBRARIES ucl)
+LIST(APPEND RSPAMD_REQUIRED_LIBRARIES rdns)
+LIST(APPEND RSPAMD_REQUIRED_LIBRARIES ottery)
+LIST(APPEND RSPAMD_REQUIRED_LIBRARIES event)
+LIST(APPEND RSPAMD_REQUIRED_LIBRARIES xxhash)
IF(GLIB_COMPAT)
LIST(APPEND RSPAMD_REQUIRED_LIBRARIES glibadditions)
@@ -1267,9 +1265,8 @@ IF (ENABLE_SNOWBALL MATCHES "ON")
LIST(APPEND RSPAMD_REQUIRED_LIBRARIES stemmer)
ENDIF()
LIST(APPEND RSPAMD_REQUIRED_LIBRARIES rspamd-hiredis)
-IF(ENABLE_HYPERSCAN MATCHES "OFF")
- LIST(APPEND RSPAMD_REQUIRED_LIBRARIES rspamd-actrie)
-ENDIF()
+
+LIST(APPEND RSPAMD_REQUIRED_LIBRARIES rspamd-actrie)
LIST(APPEND RSPAMD_REQUIRED_LIBRARIES rspamd-t1ha)
IF(ENABLE_CLANG_PLUGIN MATCHES "ON")
diff --git a/conf/composites.conf b/conf/composites.conf
index 976225db1..3436b44ae 100644
--- a/conf/composites.conf
+++ b/conf/composites.conf
@@ -87,7 +87,7 @@ composites {
policy = "leave";
}
RCVD_UNAUTH_PBL {
- expression = "RECEIVED_PBL & -RCVD_VIA_SMTP_AUTH";
+ expression = "RECEIVED_PBL & !RCVD_VIA_SMTP_AUTH";
description = "Relayed through ZEN PBL IP without sufficient authentication (possible indicating an open relay)";
score = 2.0;
policy = "leave";
diff --git a/contrib/http-parser/http_parser.c b/contrib/http-parser/http_parser.c
index 0e75d964b..c14ecc034 100644
--- a/contrib/http-parser/http_parser.c
+++ b/contrib/http-parser/http_parser.c
@@ -122,6 +122,7 @@ do { \
#define KEEP_ALIVE "keep-alive"
#define CLOSE "close"
+enum rspamd_http_message_type { HTTP_REQUEST, HTTP_RESPONSE, HTTP_BOTH };
static const char *method_strings[] =
{
@@ -1981,7 +1982,7 @@ http_method_str (enum http_method m)
void
-http_parser_init (http_parser *parser, enum http_parser_type t)
+http_parser_init (http_parser *parser, int t)
{
void *data = parser->data; /* preserve application data */
memset(parser, 0, sizeof(*parser));
diff --git a/contrib/http-parser/http_parser.h b/contrib/http-parser/http_parser.h
index e2a0b4985..7b4ac497c 100644
--- a/contrib/http-parser/http_parser.h
+++ b/contrib/http-parser/http_parser.h
@@ -124,8 +124,6 @@ enum http_method
};
-enum http_parser_type { HTTP_REQUEST, HTTP_RESPONSE, HTTP_BOTH };
-
/* Flag values for http_parser.flags field */
enum flags
@@ -280,7 +278,7 @@ struct http_parser_url {
*/
unsigned long http_parser_version(void);
-void http_parser_init(http_parser *parser, enum http_parser_type type);
+void http_parser_init(http_parser *parser, int type);
size_t http_parser_execute(http_parser *parser,
diff --git a/contrib/librdns/util.c b/contrib/librdns/util.c
index b793b9077..9f1bc6018 100644
--- a/contrib/librdns/util.c
+++ b/contrib/librdns/util.c
@@ -537,8 +537,9 @@ rdns_resolver_conf_process_line (struct rdns_resolver *resolver,
const char *line, rdns_resolv_conf_cb cb, void *ud)
{
const char *p, *c, *end;
- bool has_obrace = false;
+ bool has_obrace = false, ret;
unsigned int port = dns_port;
+ char *cpy_buf;
end = line + strlen (line);
@@ -546,7 +547,7 @@ rdns_resolver_conf_process_line (struct rdns_resolver *resolver,
strncmp (line, "nameserver", sizeof ("nameserver") - 1) == 0) {
p = line + sizeof ("nameserver") - 1;
/* Skip spaces */
- while (*p == ' ' || *p == '\t') {
+ while (isspace (*p)) {
p ++;
}
@@ -578,14 +579,23 @@ rdns_resolver_conf_process_line (struct rdns_resolver *resolver,
}
}
+ cpy_buf = malloc (p - c + 1);
+ assert (cpy_buf != NULL);
+ memcpy (cpy_buf, c, p - c);
+ cpy_buf[p - c] = '\0';
+
if (cb == NULL) {
- return rdns_resolver_add_server (resolver, c, port, 0,
+ ret = rdns_resolver_add_server (resolver, cpy_buf, port, 0,
default_io_cnt) != NULL;
}
else {
- return cb (resolver, c, port, 0,
+ ret = cb (resolver, cpy_buf, port, 0,
default_io_cnt, ud);
}
+
+ free (cpy_buf);
+
+ return ret;
}
else {
return false;
diff --git a/lualib/lua_cfg_transform.lua b/lualib/lua_cfg_transform.lua
index 8a61dcd29..ae05eed59 100644
--- a/lualib/lua_cfg_transform.lua
+++ b/lualib/lua_cfg_transform.lua
@@ -242,6 +242,12 @@ return function(cfg)
ret = true
end
+ if cfg.symbols then
+ for k, v in metric_pairs(cfg.symbols) do
+ symbol_transform(cfg, k, v)
+ end
+ end
+
check_statistics_sanity()
if not cfg.actions then
diff --git a/lualib/lua_clickhouse.lua b/lualib/lua_clickhouse.lua
index dbb74e283..96ea59f02 100644
--- a/lualib/lua_clickhouse.lua
+++ b/lualib/lua_clickhouse.lua
@@ -227,7 +227,7 @@ exports.select = function (upstream, settings, params, query, ok_cb, fail_cb)
connect_prefix = 'https://'
end
local ip_addr = upstream:get_addr():to_string(true)
- local database = params.database or 'default'
+ local database = settings.database or 'default'
http_params.url = string.format('%s%s/?database=%s&default_format=JSONEachRow',
connect_prefix, ip_addr, escape_spaces(database))
end
@@ -278,7 +278,7 @@ exports.select_sync = function (upstream, settings, params, query, ok_cb, fail_c
connect_prefix = 'https://'
end
local ip_addr = upstream:get_addr():to_string(true)
- local database = params.database or 'default'
+ local database = settings.database or 'default'
http_params.url = string.format('%s%s/?database=%s&default_format=JSONEachRow',
connect_prefix, ip_addr, escape_spaces(database))
end
@@ -342,7 +342,7 @@ exports.insert = function (upstream, settings, params, query, rows,
connect_prefix = 'https://'
end
local ip_addr = upstream:get_addr():to_string(true)
- local database = params.database or 'default'
+ local database = settings.database or 'default'
http_params.url = string.format('%s%s/?database=%s&query=%s%%20FORMAT%%20TabSeparated',
connect_prefix,
ip_addr,
@@ -394,7 +394,7 @@ exports.generic = function (upstream, settings, params, query,
connect_prefix = 'https://'
end
local ip_addr = upstream:get_addr():to_string(true)
- local database = params.database or 'default'
+ local database = settings.database or 'default'
http_params.url = string.format('%s%s/?database=%s&default_format=JSONEachRow',
connect_prefix, ip_addr, escape_spaces(database))
end
@@ -439,7 +439,7 @@ exports.generic_sync = function (upstream, settings, params, query)
connect_prefix = 'https://'
end
local ip_addr = upstream:get_addr():to_string(true)
- local database = params.database or 'default'
+ local database = settings.database or 'default'
http_params.url = string.format('%s%s/?database=%s&default_format=JSONEachRow',
connect_prefix, ip_addr, escape_spaces(database))
end
diff --git a/lualib/lua_dkim_tools.lua b/lualib/lua_dkim_tools.lua
index 4302d24ad..fc891ba71 100644
--- a/lualib/lua_dkim_tools.lua
+++ b/lualib/lua_dkim_tools.lua
@@ -120,7 +120,17 @@ local function prepare_dkim_signing(N, task, settings)
local is_local, is_sign_networks
if settings.use_http_headers then
- return parse_dkim_http_headers(N, task, settings)
+ local res,tbl = parse_dkim_http_headers(N, task, settings)
+
+ if not res then
+ if not settings.allow_headers_fallback then
+ return res,{}
+ else
+ lua_util.debugm(N, task, 'failed to read http headers, fallback to normal schema')
+ end
+ else
+ return res,tbl
+ end
end
local auser = task:get_user()
@@ -174,6 +184,8 @@ local function prepare_dkim_signing(N, task, settings)
return udom
elseif settings[dtype] == 'recipient' then
return tdom
+ else
+ return settings[dtype]:lower()
end
end
@@ -202,6 +214,31 @@ local function prepare_dkim_signing(N, task, settings)
dkim_domain = get_dkim_domain('use_domain_sign_inbound')
lua_util.debugm(N, task, 'inbound: use domain(%s) for signature: %s',
settings.use_domain_sign_inbound, dkim_domain)
+ elseif settings.use_domain_custom then
+ if type(settings.use_domain_custom) == 'string' then
+ -- Load custom function
+ local loadstring = loadstring or load
+ local ret, res_or_err = pcall(loadstring(settings.use_domain_custom))
+ if ret then
+ if type(res_or_err) == 'function' then
+ settings.use_domain_custom = res_or_err
+ dkim_domain = settings.use_domain_custom(task)
+ lua_util.debugm(N, task, 'use custom domain for signing: %s',
+ dkim_domain)
+ else
+ logger.errx(task, 'cannot load dkim domain custom script: invalid type: %s, expected function',
+ type(res_or_err))
+ settings.use_domain_custom = nil
+ end
+ else
+ logger.errx(task, 'cannot load dkim domain custom script: %s', res_or_err)
+ settings.use_domain_custom = nil
+ end
+ else
+ dkim_domain = settings.use_domain_custom(task)
+ lua_util.debugm(N, task, 'use custom domain for signing: %s',
+ dkim_domain)
+ end
else
dkim_domain = get_dkim_domain('use_domain')
lua_util.debugm(N, task, 'use domain(%s) for signature: %s',
diff --git a/lualib/lua_ffi/init.lua b/lualib/lua_ffi/init.lua
index d2cbd95aa..b0254bdd8 100644
--- a/lualib/lua_ffi/init.lua
+++ b/lualib/lua_ffi/init.lua
@@ -46,6 +46,7 @@ else
end
end
+pcall(ffi.load, "rspamd-server", true)
exports.common = require "lua_ffi/common"
exports.dkim = require "lua_ffi/dkim"
diff --git a/lualib/lua_selectors.lua b/lualib/lua_selectors.lua
index 0a02edca9..b678ef179 100644
--- a/lualib/lua_selectors.lua
+++ b/lualib/lua_selectors.lua
@@ -199,6 +199,27 @@ the second optional argument is optional hash type (`blake2`, `sha256`, `sha1`,
end,
['description'] = 'Get all attachments files',
},
+ -- Get languages for text parts
+ ['languages'] = {
+ ['get_value'] = function(task)
+ local text_parts = task:get_text_parts() or E
+ local languages = {}
+
+ for _,p in ipairs(text_parts) do
+ local lang = p:get_language()
+ if lang then
+ table.insert(languages, lang)
+ end
+ end
+
+ if #languages > 0 then
+ return languages,'string_list'
+ end
+
+ return nil
+ end,
+ ['description'] = 'Get languages for text parts',
+ },
-- Get helo value
['helo'] = {
['get_value'] = function(task)
diff --git a/rules/html.lua b/rules/html.lua
index da4ef1d13..fb32e9179 100644
--- a/rules/html.lua
+++ b/rules/html.lua
@@ -23,6 +23,20 @@ reconf['MIME_HTML_ONLY'] = {
group = 'headers'
}
+local function has_anchor_parent(tag)
+ local parent = tag
+ repeat
+ parent = parent:get_parent()
+ if parent then
+ if parent:get_type() == 'a' then
+ return true
+ end
+ end
+ until not parent
+
+ return false
+end
+
local function check_html_image(task, min, max)
local tp = task:get_text_parts()
@@ -38,13 +52,10 @@ local function check_html_image(task, min, max)
for _,i in ipairs(images) do
local tag = i['tag']
if tag then
- local parent = tag:get_parent()
- if parent then
- if parent:get_type() == 'a' then
- -- do not trigger on small and unknown size images
- if i['height'] + i['width'] >= 210 or not i['embedded'] then
- return true
- end
+ if has_anchor_parent(tag) then
+ -- do not trigger on small and unknown size images
+ if i['height'] + i['width'] >= 210 or not i['embedded'] then
+ return true
end
end
end
@@ -81,6 +92,7 @@ rspamd_config.HTML_SHORT_LINK_IMG_3 = {
group = 'html',
description = 'Short html part (1.5K..2K) with a link to an image'
}
+
rspamd_config.R_EMPTY_IMAGE = {
callback = function(task)
local tp = task:get_text_parts() -- get text parts in a message
@@ -98,11 +110,8 @@ rspamd_config.R_EMPTY_IMAGE = {
if i['height'] + i['width'] >= 400 then -- if we have a large image
local tag = i['tag']
if tag then
- local parent = tag:get_parent()
- if parent then
- if parent:get_type() ~= 'a' then
- return true
- end
+ if not has_anchor_parent(tag) then
+ return true
end
end
end
@@ -136,14 +145,10 @@ rspamd_config.R_SUSPICIOUS_IMAGES = {
local tag = i['tag']
if tag then
- local parent = tag:get_parent()
- if parent then
- if parent:get_type() == 'a' then
- -- do not trigger on small and large images
- if dim > 100 and dim < 3000 then
- -- We assume that a single picture 100x200 contains approx 3 words of text
- pic_words = pic_words + dim / 100
- end
+ if has_anchor_parent(tag) then
+ if dim > 100 and dim < 3000 then
+ -- We assume that a single picture 100x200 contains approx 3 words of text
+ pic_words = pic_words + dim / 100
end
end
end
diff --git a/rules/regexp/misc.lua b/rules/regexp/misc.lua
index aaf8425bd..642a02e21 100644
--- a/rules/regexp/misc.lua
+++ b/rules/regexp/misc.lua
@@ -64,7 +64,7 @@ reconf['HAS_ONION_URI'] = {
local my_victim = [[/(?:victim|prey)/{words}]]
local your_webcam = [[/webcam/{words}]]
local your_onan = [[/(?:mast[ur]{2}bati(?:on|ng)|onanism|solitary)/{words}]]
-local password_in_words = [[/^pass(?:(?:word)|(?:phrase))/i{words}]]
+local password_in_words = [[/^pass(?:(?:word)|(?:phrase))$/i{words}]]
local btc_wallet_address = [[/^[13][0-9a-zA-Z]{25,34}$/{words}]]
local wallet_word = [[/^wallet$/{words}]]
local broken_unicode = [[has_flag(bad_unicode)]]
@@ -93,4 +93,4 @@ reconf['LEAKED_PASSWORD_SCAM'] = {
},
score = 7.0,
group = 'scams'
-} \ No newline at end of file
+}
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index fc2560ce6..256feb522 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -206,17 +206,15 @@ IF (ENABLE_HYPERSCAN MATCHES "ON")
ENDIF()
TARGET_LINK_LIBRARIES(rspamd-server rspamd-linenoise)
-
-IF(USE_CXX_LINKER)
- SET_TARGET_PROPERTIES(rspamd PROPERTIES LINKER_LANGUAGE CXX)
- SET_TARGET_PROPERTIES(rspamd-server PROPERTIES LINKER_LANGUAGE CXX)
-ENDIF()
-
TARGET_LINK_LIBRARIES(rspamd-server ${RSPAMD_REQUIRED_LIBRARIES})
ADD_EXECUTABLE(rspamd ${RSPAMDSRC} ${CMAKE_CURRENT_BINARY_DIR}/workers.c)
SET_TARGET_PROPERTIES(rspamd PROPERTIES LINKER_LANGUAGE C)
SET_TARGET_PROPERTIES(rspamd PROPERTIES COMPILE_FLAGS "-DRSPAMD_MAIN")
+IF(USE_CXX_LINKER)
+ SET_TARGET_PROPERTIES(rspamd PROPERTIES LINKER_LANGUAGE CXX)
+ SET_TARGET_PROPERTIES(rspamd-server PROPERTIES LINKER_LANGUAGE CXX)
+ENDIF()
IF(NOT DEBIAN_BUILD)
SET_TARGET_PROPERTIES(rspamd PROPERTIES VERSION ${RSPAMD_VERSION})
ENDIF(NOT DEBIAN_BUILD)
diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt
index 0a92019d8..4e1225ae3 100644
--- a/src/client/CMakeLists.txt
+++ b/src/client/CMakeLists.txt
@@ -7,7 +7,6 @@ SET(RSPAMCSRC rspamc.c)
ADD_EXECUTABLE(rspamc ${RSPAMCSRC} ${LIBRSPAMDCLIENTSRC})
SET_TARGET_PROPERTIES(rspamc PROPERTIES COMPILE_FLAGS "-I${CMAKE_SOURCE_DIR}/lib")
TARGET_LINK_LIBRARIES(rspamc rspamd-server)
-TARGET_LINK_LIBRARIES(rspamc ${RSPAMD_REQUIRED_LIBRARIES})
IF(USE_CXX_LINKER)
SET_TARGET_PROPERTIES(rspamc PROPERTIES LINKER_LANGUAGE CXX)
ENDIF()
diff --git a/src/client/rspamc.c b/src/client/rspamc.c
index 3433ef7d6..0568692b4 100644
--- a/src/client/rspamc.c
+++ b/src/client/rspamc.c
@@ -15,7 +15,7 @@
*/
#include "config.h"
#include "libutil/util.h"
-#include "libutil/http.h"
+#include "libutil/http_connection.h"
#include "libutil/http_private.h"
#include "rspamdclient.h"
#include "utlist.h"
@@ -67,6 +67,7 @@ static gchar *key = NULL;
static gchar *user_agent = "rspamc";
static GList *children;
static GPatternSpec **exclude_compiled = NULL;
+static struct rspamd_http_context *http_ctx;
static gint retcode = EXIT_SUCCESS;
@@ -553,10 +554,6 @@ add_options (GQueue *opts)
GString *numbuf;
gchar **hdr, **rcpt;
- if (user_agent) {
- ADD_CLIENT_HEADER (opts, "User-Agent", user_agent);
- }
-
if (ip != NULL) {
rspamd_inet_addr_t *addr = NULL;
@@ -1668,7 +1665,7 @@ rspamc_process_input (struct event_base *ev_base, struct rspamc_command *cmd,
}
- conn = rspamd_client_init (ev_base, hostbuf, port, timeout, key);
+ conn = rspamd_client_init (http_ctx, ev_base, hostbuf, port, timeout, key);
if (conn != NULL) {
cbdata = g_malloc0 (sizeof (struct rspamc_callback_data));
@@ -1894,6 +1891,15 @@ main (gint argc, gchar **argv, gchar **env)
rspamd_init_libs ();
ev_base = event_base_new ();
+ struct rspamd_http_context_cfg http_config;
+
+ memset (&http_config, 0, sizeof (http_config));
+ http_config.kp_cache_size_client = 32;
+ http_config.kp_cache_size_server = 0;
+ http_config.user_agent = user_agent;
+ http_ctx = rspamd_http_context_create_config (&http_config,
+ ev_base);
+
/* Ignore sigpipe */
sigemptyset (&sigpipe_act.sa_mask);
sigaddset (&sigpipe_act.sa_mask, SIGPIPE);
diff --git a/src/client/rspamdclient.c b/src/client/rspamdclient.c
index a4a1fb95e..a2ff85458 100644
--- a/src/client/rspamdclient.c
+++ b/src/client/rspamdclient.c
@@ -15,7 +15,7 @@
*/
#include "rspamdclient.h"
#include "libutil/util.h"
-#include "libutil/http.h"
+#include "libutil/http_connection.h"
#include "libutil/http_private.h"
#include "unix-std.h"
#include "contrib/zstd/zstd.h"
@@ -118,9 +118,7 @@ rspamd_client_finish_handler (struct rspamd_http_connection *conn,
rspamd_http_connection_reset (c->http_conn);
rspamd_http_connection_read_message (c->http_conn,
c->req,
- c->fd,
- &c->timeout,
- c->ev_base);
+ &c->timeout);
return 0;
}
else {
@@ -241,8 +239,9 @@ rspamd_client_finish_handler (struct rspamd_http_connection *conn,
}
struct rspamd_client_connection *
-rspamd_client_init (struct event_base *ev_base, const gchar *name,
- guint16 port, gdouble timeout, const gchar *key)
+rspamd_client_init (struct rspamd_http_context *http_ctx,
+ struct event_base *ev_base, const gchar *name,
+ guint16 port, gdouble timeout, const gchar *key)
{
struct rspamd_client_connection *conn;
gint fd;
@@ -256,14 +255,13 @@ rspamd_client_init (struct event_base *ev_base, const gchar *name,
conn->ev_base = ev_base;
conn->fd = fd;
conn->req_sent = FALSE;
- conn->keys_cache = rspamd_keypair_cache_new (32);
- conn->http_conn = rspamd_http_connection_new (rspamd_client_body_handler,
+ conn->http_conn = rspamd_http_connection_new (http_ctx,
+ fd,
+ rspamd_client_body_handler,
rspamd_client_error_handler,
rspamd_client_finish_handler,
0,
- RSPAMD_HTTP_CLIENT,
- conn->keys_cache,
- NULL);
+ RSPAMD_HTTP_CLIENT);
conn->server_name = g_string_new (name);
if (port != 0) {
@@ -444,12 +442,12 @@ rspamd_client_command (struct rspamd_client_connection *conn,
if (compressed) {
rspamd_http_connection_write_message (conn->http_conn, req->msg, NULL,
- "application/x-compressed", req, conn->fd,
- &conn->timeout, conn->ev_base);
+ "application/x-compressed", req,
+ &conn->timeout);
}
else {
rspamd_http_connection_write_message (conn->http_conn, req->msg, NULL,
- "text/plain", req, conn->fd, &conn->timeout, conn->ev_base);
+ "text/plain", req, &conn->timeout);
}
return TRUE;
diff --git a/src/client/rspamdclient.h b/src/client/rspamdclient.h
index 129ee39a1..c2a3c1886 100644
--- a/src/client/rspamdclient.h
+++ b/src/client/rspamdclient.h
@@ -47,6 +47,7 @@ typedef void (*rspamd_client_callback) (
gdouble send_time,
GError *err);
+struct rspamd_http_context;
/**
* Start rspamd worker or controller command
* @param ev_base event base
@@ -56,11 +57,12 @@ typedef void (*rspamd_client_callback) (
* @return
*/
struct rspamd_client_connection * rspamd_client_init (
- struct event_base *ev_base,
- const gchar *name,
- guint16 port,
- gdouble timeout,
- const gchar *key);
+ struct rspamd_http_context *http_ctx,
+ struct event_base *ev_base,
+ const gchar *name,
+ guint16 port,
+ gdouble timeout,
+ const gchar *key);
/**
*
diff --git a/src/controller.c b/src/controller.c
index 6a839b4df..ac1acae81 100644
--- a/src/controller.c
+++ b/src/controller.c
@@ -21,6 +21,7 @@
#include "libutil/map_helpers.h"
#include "libutil/map_private.h"
#include "libutil/http_private.h"
+#include "libutil/http_router.h"
#include "libstat/stat_api.h"
#include "rspamd.h"
#include "libserver/worker_util.h"
@@ -149,6 +150,7 @@ struct rspamd_controller_worker_ctx {
rspamd_ftok_t cached_password;
rspamd_ftok_t cached_enable_password;
/* HTTP server */
+ struct rspamd_http_context *http_ctx;
struct rspamd_http_connection_router *http;
/* Server's start time */
time_t start_time;
@@ -1076,8 +1078,8 @@ rspamd_controller_handle_get_map (struct rspamd_http_connection_entry *conn_ent,
rspamd_http_connection_reset (conn_ent->conn);
rspamd_http_router_insert_headers (conn_ent->rt, reply);
rspamd_http_connection_write_message (conn_ent->conn, reply, NULL,
- "text/plain", conn_ent, conn_ent->conn->fd,
- conn_ent->rt->ptv, conn_ent->rt->ev_base);
+ "text/plain", conn_ent,
+ conn_ent->rt->ptv);
conn_ent->is_reply = TRUE;
return 0;
@@ -1940,8 +1942,7 @@ rspamd_controller_scan_reply (struct rspamd_task *task)
rspamd_http_connection_reset (conn_ent->conn);
rspamd_http_router_insert_headers (conn_ent->rt, msg);
rspamd_http_connection_write_message (conn_ent->conn, msg, NULL,
- "application/json", conn_ent, conn_ent->conn->fd, conn_ent->rt->ptv,
- conn_ent->rt->ev_base);
+ "application/json", conn_ent, conn_ent->rt->ptv);
conn_ent->is_reply = TRUE;
}
@@ -2909,9 +2910,7 @@ rspamd_controller_handle_ping (struct rspamd_http_connection_entry *conn_ent,
NULL,
"text/plain",
conn_ent,
- conn_ent->conn->fd,
- conn_ent->rt->ptv,
- conn_ent->rt->ev_base);
+ conn_ent->rt->ptv);
conn_ent->is_reply = TRUE;
return 0;
@@ -2945,9 +2944,7 @@ rspamd_controller_handle_unknown (struct rspamd_http_connection_entry *conn_ent,
NULL,
"text/plain",
conn_ent,
- conn_ent->conn->fd,
- conn_ent->rt->ptv,
- conn_ent->rt->ev_base);
+ conn_ent->rt->ptv);
conn_ent->is_reply = TRUE;
}
else {
@@ -2963,9 +2960,7 @@ rspamd_controller_handle_unknown (struct rspamd_http_connection_entry *conn_ent,
NULL,
"text/plain",
conn_ent,
- conn_ent->conn->fd,
- conn_ent->rt->ptv,
- conn_ent->rt->ev_base);
+ conn_ent->rt->ptv);
conn_ent->is_reply = TRUE;
}
@@ -3706,7 +3701,6 @@ start_controller_worker (struct rspamd_worker *worker)
GHashTableIter iter;
gpointer key, value;
guint i;
- struct rspamd_keypair_cache *cache;
struct timeval stv;
const guint save_stats_interval = 60 * 1000; /* 1 minute */
gpointer m;
@@ -3782,10 +3776,10 @@ start_controller_worker (struct rspamd_worker *worker)
"password");
/* Accept event */
- cache = rspamd_keypair_cache_new (256);
+ ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base);
ctx->http = rspamd_http_router_new (rspamd_controller_error_handler,
- rspamd_controller_finish_handler, &ctx->io_tv, ctx->ev_base,
- ctx->static_files_dir, cache);
+ rspamd_controller_finish_handler, &ctx->io_tv,
+ ctx->static_files_dir, ctx->http_ctx);
/* Add callbacks for different methods */
rspamd_http_router_add_path (ctx->http,
@@ -3948,7 +3942,10 @@ start_controller_worker (struct rspamd_worker *worker)
g_hash_table_unref (ctx->plugins);
g_hash_table_unref (ctx->custom_commands);
+
+ struct rspamd_http_context *http_ctx = ctx->http_ctx;
REF_RELEASE (ctx->cfg);
+ rspamd_http_context_free (http_ctx);
rspamd_log_close (worker->srv->logger, TRUE);
exit (EXIT_SUCCESS);
diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c
index 36b41113d..7fdce82ae 100644
--- a/src/fuzzy_storage.c
+++ b/src/fuzzy_storage.c
@@ -34,10 +34,10 @@
#include "libcryptobox/keypairs_cache.h"
#include "libcryptobox/keypair.h"
#include "libserver/rspamd_control.h"
-#include "libutil/map_private.h"
#include "libutil/hash.h"
+#include "libutil/map_private.h"
#include "libutil/http_private.h"
-#include "libutil/hash.h"
+#include "libutil/http_router.h"
#include "unix-std.h"
#include <math.h>
@@ -173,6 +173,7 @@ struct rspamd_fuzzy_storage_ctx {
struct rspamd_cryptobox_keypair *collection_keypair;
struct rspamd_cryptobox_pubkey *collection_sign_key;
gchar *collection_id_file;
+ struct rspamd_http_context *http_ctx;
struct rspamd_keypair_cache *keypair_cache;
rspamd_lru_hash_t *errors_ips;
rspamd_lru_hash_t *ratelimit_buckets;
@@ -530,8 +531,7 @@ fuzzy_mirror_updates_version_cb (guint64 rev64, void *ud)
double_to_tv (ctx->sync_timeout, &tv);
rspamd_http_connection_write_message (conn->http_conn,
msg, NULL, NULL, conn,
- conn->sock,
- &tv, ctx->ev_base);
+ &tv);
msg_info ("send update request to %s", m->name);
g_array_free (cbdata->updates_pending, TRUE);
@@ -568,7 +568,7 @@ fuzzy_mirror_error_handler (struct rspamd_http_connection *conn, GError *err)
msg_info ("abnormally closing connection from backend: %s:%s, "
"error: %e",
bk_conn->mirror->name,
- rspamd_inet_address_to_string (rspamd_upstream_addr (bk_conn->up)),
+ rspamd_inet_address_to_string (rspamd_upstream_addr_cur (bk_conn->up)),
err);
fuzzy_mirror_close_connection (bk_conn);
@@ -604,7 +604,7 @@ rspamd_fuzzy_send_update_mirror (struct rspamd_fuzzy_storage_ctx *ctx,
}
conn->sock = rspamd_inet_address_connect (
- rspamd_upstream_addr (conn->up),
+ rspamd_upstream_addr_next (conn->up),
SOCK_STREAM, TRUE);
if (conn->sock == -1) {
@@ -616,13 +616,14 @@ rspamd_fuzzy_send_update_mirror (struct rspamd_fuzzy_storage_ctx *ctx,
msg = rspamd_http_new_message (HTTP_REQUEST);
rspamd_printf_fstring (&msg->url, "/update_v1/%s", m->name);
- conn->http_conn = rspamd_http_connection_new (NULL,
+ conn->http_conn = rspamd_http_connection_new (
+ ctx->http_ctx,
+ conn->sock,
+ NULL,
fuzzy_mirror_error_handler,
fuzzy_mirror_finish_handler,
RSPAMD_HTTP_CLIENT_SIMPLE,
- RSPAMD_HTTP_CLIENT,
- ctx->keypair_cache,
- NULL);
+ RSPAMD_HTTP_CLIENT);
rspamd_http_connection_set_key (conn->http_conn,
ctx->sync_keypair);
@@ -1569,8 +1570,7 @@ rspamd_fuzzy_mirror_send_reply (struct fuzzy_master_update_session *session,
rspamd_http_connection_reset (session->conn);
rspamd_http_connection_write_message (session->conn, msg, NULL, "text/plain",
- session, session->sock, &session->ctx->master_io_tv,
- session->ctx->ev_base);
+ session, &session->ctx->master_io_tv);
}
static void
@@ -1711,9 +1711,7 @@ rspamd_fuzzy_collection_send_error (struct rspamd_http_connection_entry *entry,
NULL,
"text/plain",
entry,
- entry->conn->fd,
- entry->rt->ptv,
- entry->rt->ev_base);
+ entry->rt->ptv);
entry->is_reply = TRUE;
}
@@ -1738,9 +1736,7 @@ rspamd_fuzzy_collection_send_fstring (struct rspamd_http_connection_entry *entry
NULL,
"application/octet-stream",
entry,
- entry->conn->fd,
- entry->rt->ptv,
- entry->rt->ev_base);
+ entry->rt->ptv);
entry->is_reply = TRUE;
}
@@ -1994,13 +1990,14 @@ accept_fuzzy_mirror_socket (gint fd, short what, void *arg)
session->name = rspamd_inet_address_to_string (addr);
rspamd_random_hex (session->uid, sizeof (session->uid) - 1);
session->uid[sizeof (session->uid) - 1] = '\0';
- http_conn = rspamd_http_connection_new (NULL,
+ http_conn = rspamd_http_connection_new (
+ ctx->http_ctx,
+ nfd,
+ NULL,
rspamd_fuzzy_mirror_error_handler,
rspamd_fuzzy_mirror_finish_handler,
0,
- RSPAMD_HTTP_SERVER,
- ctx->keypair_cache,
- NULL);
+ RSPAMD_HTTP_SERVER);
rspamd_http_connection_set_key (http_conn, ctx->sync_keypair);
session->ctx = ctx;
@@ -2010,9 +2007,7 @@ accept_fuzzy_mirror_socket (gint fd, short what, void *arg)
rspamd_http_connection_read_message (http_conn,
session,
- nfd,
- &ctx->master_io_tv,
- ctx->ev_base);
+ &ctx->master_io_tv);
}
/*
@@ -3004,6 +2999,8 @@ start_fuzzy (struct rspamd_worker *worker)
ctx->keypair_cache = rspamd_keypair_cache_new (ctx->keypair_cache_size);
}
+ ctx->http_ctx = rspamd_http_context_create (cfg, ctx->ev_base);
+
if (!ctx->collection_mode) {
/*
* Open DB and perform VACUUM
@@ -3058,8 +3055,8 @@ start_fuzzy (struct rspamd_worker *worker)
rspamd_fuzzy_collection_error_handler,
rspamd_fuzzy_collection_finish_handler,
&ctx->stat_tv,
- ctx->ev_base,
- NULL, ctx->keypair_cache);
+ NULL,
+ ctx->http_ctx);
if (ctx->collection_keypair) {
rspamd_http_router_set_key (ctx->collection_rt,
@@ -3202,8 +3199,6 @@ start_fuzzy (struct rspamd_worker *worker)
else if (worker->index == 0) {
gint fd;
- /* Steal keypairs cache... */
- ctx->collection_rt->cache = NULL;
rspamd_http_router_free (ctx->collection_rt);
/* Try to save collection id */
@@ -3240,8 +3235,9 @@ start_fuzzy (struct rspamd_worker *worker)
rspamd_keypair_cache_destroy (ctx->keypair_cache);
}
+ struct rspamd_http_context *http_ctx = ctx->http_ctx;
REF_RELEASE (ctx->cfg);
-
+ rspamd_http_context_free (http_ctx);
rspamd_log_close (worker->srv->logger, TRUE);
exit (EXIT_SUCCESS);
diff --git a/src/libmime/content_type.c b/src/libmime/content_type.c
index ca371ce30..b67ec155e 100644
--- a/src/libmime/content_type.c
+++ b/src/libmime/content_type.c
@@ -305,8 +305,6 @@ rspamd_content_type_add_param (rspamd_mempool_t *pool,
nparam->value.len = value_end - value_start;
}
- RSPAMD_FTOK_ASSIGN (&srch, "charset");
-
srch.begin = nparam->name.begin;
srch.len = nparam->name.len;
@@ -713,36 +711,47 @@ rspamd_content_disposition_add_param (rspamd_mempool_t *pool,
const gchar *value_start, const gchar *value_end)
{
rspamd_ftok_t srch;
- gchar *decoded;
+ gchar *name_cpy, *value_cpy, *name_cpy_end, *value_cpy_end;
struct rspamd_content_type_param *found = NULL, *nparam;
g_assert (cd != NULL);
- srch.begin = name_start;
- srch.len = name_end - name_start;
+ name_cpy = rspamd_mempool_alloc (pool, name_end - name_start);
+ memcpy (name_cpy, name_start, name_end - name_start);
+ name_cpy_end = name_cpy + (name_end - name_start);
+
+ value_cpy = rspamd_mempool_alloc (pool, value_end - value_start);
+ memcpy (value_cpy, value_start, value_end - value_start);
+ value_cpy_end = value_cpy + (value_end - value_start);
+
+ nparam = rspamd_mempool_alloc0 (pool, sizeof (*nparam));
+ rspamd_str_lc (name_cpy, name_cpy_end - name_cpy);
+
+ if (!rspamd_param_maybe_rfc2231_process (pool, nparam, name_cpy,
+ name_cpy_end, value_cpy, value_cpy_end)) {
+ nparam->name.begin = name_cpy;
+ nparam->name.len = name_cpy_end - name_cpy;
+ nparam->value.begin = value_cpy;
+ nparam->value.len = value_cpy_end - value_cpy;
+ }
+
+ srch.begin = nparam->name.begin;
+ srch.len = nparam->name.len;
if (cd->attrs) {
found = g_hash_table_lookup (cd->attrs, &srch);
- }
- else {
+ } else {
cd->attrs = g_hash_table_new (rspamd_ftok_icase_hash,
rspamd_ftok_icase_equal);
- rspamd_mempool_add_destructor (pool,
- (rspamd_mempool_destruct_t)g_hash_table_unref, cd->attrs);
}
- nparam = rspamd_mempool_alloc0 (pool, sizeof (*nparam));
- nparam->name.begin = name_start;
- nparam->name.len = name_end - name_start;
- decoded = rspamd_mime_header_decode (pool, value_start,
- value_end - value_start, NULL);
- RSPAMD_FTOK_FROM_STR (&nparam->value, decoded);
-
if (!found) {
+ DL_APPEND (found, nparam);
g_hash_table_insert (cd->attrs, &nparam->name, nparam);
}
-
- DL_APPEND (found, nparam);
+ else {
+ DL_APPEND (found, nparam);
+ }
}
struct rspamd_content_disposition *
@@ -757,8 +766,13 @@ rspamd_content_disposition_parse (const gchar *in,
res->lc_data = rspamd_mempool_alloc (pool, len + 1);
rspamd_strlcpy (res->lc_data, in, len + 1);
rspamd_str_lc (res->lc_data, len);
- rspamd_postprocess_ct_attributes (pool, res->attrs,
- rspamd_content_disposition_postprocess, res);
+
+ if (res->attrs) {
+ rspamd_postprocess_ct_attributes (pool, res->attrs,
+ rspamd_content_disposition_postprocess, res);
+ rspamd_mempool_add_destructor (pool,
+ (rspamd_mempool_destruct_t)g_hash_table_unref, res->attrs);
+ }
}
else {
msg_warn_pool ("cannot parse content disposition: %*s",
diff --git a/src/libmime/lang_detection.c b/src/libmime/lang_detection.c
index 72964a93a..7bfcf0164 100644
--- a/src/libmime/lang_detection.c
+++ b/src/libmime/lang_detection.c
@@ -459,9 +459,16 @@ rspamd_language_detector_read_file (struct rspamd_config *cfg,
const char *word = ucl_object_tolstring (w, &wlen);
const char *saved;
+#ifdef WITH_HYPERSCAN
+ rspamd_multipattern_add_pattern_len (d->stop_words[cat].mp,
+ word, wlen,
+ RSPAMD_MULTIPATTERN_ICASE|RSPAMD_MULTIPATTERN_UTF8
+ |RSPAMD_MULTIPATTERN_RE);
+#else
rspamd_multipattern_add_pattern_len (d->stop_words[cat].mp,
word, wlen,
RSPAMD_MULTIPATTERN_ICASE|RSPAMD_MULTIPATTERN_UTF8);
+#endif
nelt->stop_words ++;
nstop ++;
@@ -817,8 +824,15 @@ rspamd_language_detector_init (struct rspamd_config *cfg)
/* Map from ngramm in ucs32 to GPtrArray of rspamd_language_elt */
for (i = 0; i < RSPAMD_LANGUAGE_MAX; i ++) {
ret->trigramms[i] = kh_init (rspamd_trigram_hash);
+#ifdef WITH_HYPERSCAN
+ ret->stop_words[i].mp = rspamd_multipattern_create (
+ RSPAMD_MULTIPATTERN_ICASE|RSPAMD_MULTIPATTERN_UTF8|
+ RSPAMD_MULTIPATTERN_RE);
+#else
ret->stop_words[i].mp = rspamd_multipattern_create (
RSPAMD_MULTIPATTERN_ICASE|RSPAMD_MULTIPATTERN_UTF8);
+#endif
+
ret->stop_words[i].ranges = g_array_new (FALSE, FALSE,
sizeof (struct rspamd_stop_word_range));
}
@@ -1457,7 +1471,10 @@ rspamd_language_detector_set_language (struct rspamd_task *task,
r->prob = 1.0;
r->lang = code;
- part->languages = g_ptr_array_sized_new (1);
+ if (part->languages == NULL) {
+ part->languages = g_ptr_array_sized_new (1);
+ }
+
g_ptr_array_add (part->languages, r);
part->language = code;
}
diff --git a/src/libmime/message.c b/src/libmime/message.c
index da065195d..cca134f81 100644
--- a/src/libmime/message.c
+++ b/src/libmime/message.c
@@ -1150,7 +1150,7 @@ rspamd_message_parse (struct rspamd_task *task)
need_recv_correction = TRUE;
}
else {
- if (rspamd_inet_address_compare (raddr, task->from_addr) != 0) {
+ if (rspamd_inet_address_compare (raddr, task->from_addr, FALSE) != 0) {
need_recv_correction = TRUE;
}
}
diff --git a/src/libmime/mime_headers.c b/src/libmime/mime_headers.c
index d0388a632..caf69cd16 100644
--- a/src/libmime/mime_headers.c
+++ b/src/libmime/mime_headers.c
@@ -639,68 +639,69 @@ rspamd_mime_header_decode (rspamd_mempool_t *pool, const gchar *in,
if (qmarks < 3) {
state = got_encoded_start;
}
- /* Finished encoded boundary */
- else if (rspamd_rfc2047_parser (c, p - c + 1, &encoding,
- &cur_charset.begin, &cur_charset.len,
- &tok_start, &tok_len)) {
- /* We have a token, so we can decode it from `encoding` */
- if (token->len > 0) {
- if (old_charset.len == 0) {
- memcpy (&old_charset, &cur_charset,
- sizeof (old_charset));
- }
-
- rspamd_mime_header_maybe_save_token (pool, out,
- token, decoded,
- &old_charset, &cur_charset);
+ else {
+ /* Finished encoded boundary */
+ if (*c == '"') {
+ /* Quoted string, non-RFC conformant but used by retards */
+ c ++;
}
+ if (rspamd_rfc2047_parser (c, p - c + 1, &encoding,
+ &cur_charset.begin, &cur_charset.len,
+ &tok_start, &tok_len)) {
+ /* We have a token, so we can decode it from `encoding` */
+ if (token->len > 0) {
+ if (old_charset.len == 0) {
+ memcpy (&old_charset, &cur_charset,
+ sizeof (old_charset));
+ }
- qmarks = 0;
- pos = token->len;
- g_byte_array_set_size (token, pos + tok_len);
+ rspamd_mime_header_maybe_save_token (pool, out,
+ token, decoded,
+ &old_charset, &cur_charset);
+ }
- if (encoding == RSPAMD_RFC2047_QP) {
- r = rspamd_decode_qp2047_buf (tok_start, tok_len,
- token->data + pos, tok_len);
+ qmarks = 0;
+ pos = token->len;
+ g_byte_array_set_size (token, pos + tok_len);
- if (r != -1) {
- token->len = pos + r;
- }
- else {
- /* Cannot decode qp */
- token->len -= tok_len;
- }
- }
- else {
- if (rspamd_cryptobox_base64_decode (tok_start, tok_len,
- token->data + pos, &tok_len)) {
- token->len = pos + tok_len;
- }
- else {
- /* Cannot decode */
- token->len -= tok_len;
- }
- }
+ if (encoding == RSPAMD_RFC2047_QP) {
+ r = rspamd_decode_qp2047_buf (tok_start, tok_len,
+ token->data + pos, tok_len);
- c = p + 1;
- state = skip_spaces;
- }
- else {
- /* Not encoded-word */
- old_charset.len = 0;
+ if (r != -1) {
+ token->len = pos + r;
+ } else {
+ /* Cannot decode qp */
+ token->len -= tok_len;
+ }
+ } else {
+ if (rspamd_cryptobox_base64_decode (tok_start, tok_len,
+ token->data + pos, &tok_len)) {
+ token->len = pos + tok_len;
+ } else {
+ /* Cannot decode */
+ token->len -= tok_len;
+ }
+ }
- if (token->len > 0) {
- rspamd_mime_header_maybe_save_token (pool, out,
- token, decoded,
- &old_charset, &cur_charset);
- }
+ c = p + 1;
+ state = skip_spaces;
+ } else {
+ /* Not encoded-word */
+ old_charset.len = 0;
- g_string_append_len (out, c, p - c);
- c = p;
- state = parse_normal;
- }
+ if (token->len > 0) {
+ rspamd_mime_header_maybe_save_token (pool, out,
+ token, decoded,
+ &old_charset, &cur_charset);
+ }
- }
+ g_string_append_len (out, c, p - c);
+ c = p;
+ state = parse_normal;
+ }
+ } /* qmarks >= 3 */
+ } /* p == '=' */
else {
state = got_encoded_start;
}
diff --git a/src/libmime/mime_parser.c b/src/libmime/mime_parser.c
index 7151140cb..21a81575d 100644
--- a/src/libmime/mime_parser.c
+++ b/src/libmime/mime_parser.c
@@ -535,6 +535,7 @@ rspamd_mime_process_multipart_node (struct rspamd_task *task,
struct rspamd_mime_parser_ctx *st,
struct rspamd_mime_part *multipart,
const gchar *start, const gchar *end,
+ gboolean is_finished,
GError **err)
{
struct rspamd_content_type *ct, *sel = NULL;
@@ -558,6 +559,24 @@ rspamd_mime_process_multipart_node (struct rspamd_task *task,
*/
hdr_pos = 0;
body_pos = 0;
+
+ if (!is_finished) {
+ /* Ignore garbage */
+ const gchar *p = start;
+ gboolean seen_something = FALSE;
+
+ while (p < end) {
+ if (g_ascii_isalnum (*p)) {
+ seen_something = TRUE;
+ break;
+ }
+ p ++;
+ }
+
+ if (!seen_something) {
+ return RSPAMD_MIME_PARSE_NO_PART;
+ }
+ }
}
else {
hdr_pos = rspamd_string_find_eoh (&str, &body_pos);
@@ -609,7 +628,7 @@ rspamd_mime_process_multipart_node (struct rspamd_task *task,
for (i = 0; i < hdrs->len; i ++) {
hdr = g_ptr_array_index (hdrs, i);
- ct = rspamd_content_type_parse (hdr->value, strlen (hdr->value),
+ ct = rspamd_content_type_parse (hdr->decoded, strlen (hdr->decoded),
task->task_pool);
/* Here we prefer multipart content-type or any content-type */
@@ -681,8 +700,8 @@ rspamd_mime_parse_multipart_cb (struct rspamd_task *task,
g_assert (cb->cur_boundary != NULL);
if ((ret = rspamd_mime_process_multipart_node (task, cb->st,
- cb->multipart, cb->part_start, pos, cb->err))
- != RSPAMD_MIME_PARSE_OK) {
+ cb->multipart, cb->part_start, pos, TRUE, cb->err))
+ != RSPAMD_MIME_PARSE_OK) {
return ret;
}
@@ -1240,7 +1259,7 @@ rspamd_mime_parse_message (struct rspamd_task *task,
else {
for (i = 0; i < hdrs->len; i ++) {
hdr = g_ptr_array_index (hdrs, i);
- ct = rspamd_content_type_parse (hdr->value, strlen (hdr->value),
+ ct = rspamd_content_type_parse (hdr->decoded, strlen (hdr->decoded),
task->task_pool);
/* Here we prefer multipart content-type or any content-type */
@@ -1338,7 +1357,16 @@ rspamd_mime_parse_message (struct rspamd_task *task,
if (end > start &&
(ret = rspamd_mime_process_multipart_node (task, st,
- NULL, start, end, err)) != RSPAMD_MIME_PARSE_OK) {
+ NULL, start, end, FALSE, err)) != RSPAMD_MIME_PARSE_OK) {
+
+ if (nst != st) {
+ rspamd_mime_parse_stack_free (nst);
+ }
+
+ if (ret == RSPAMD_MIME_PARSE_NO_PART) {
+ return RSPAMD_MIME_PARSE_OK;
+ }
+
return ret;
}
}
diff --git a/src/libmime/mime_parser.h b/src/libmime/mime_parser.h
index 5164ea4d5..987ec1395 100644
--- a/src/libmime/mime_parser.h
+++ b/src/libmime/mime_parser.h
@@ -25,6 +25,7 @@ enum rspamd_mime_parse_error {
RSPAMD_MIME_PARSE_OK = 0,
RSPAMD_MIME_PARSE_FATAL,
RSPAMD_MIME_PARSE_NESTING,
+ RSPAMD_MIME_PARSE_NO_PART,
};
enum rspamd_mime_parse_error rspamd_mime_parse_task (struct rspamd_task *task,
diff --git a/src/libserver/cfg_file.h b/src/libserver/cfg_file.h
index c7475c327..c102ef570 100644
--- a/src/libserver/cfg_file.h
+++ b/src/libserver/cfg_file.h
@@ -98,6 +98,13 @@ enum lua_var_type {
LUA_VAR_UNKNOWN
};
+enum rspamd_symbol_group_flags {
+ RSPAMD_SYMBOL_GROUP_NORMAL = 0,
+ RSPAMD_SYMBOL_GROUP_DISABLED = (1 << 0),
+ RSPAMD_SYMBOL_GROUP_ONE_SHOT = (1 << 1),
+ RSPAMD_SYMBOL_GROUP_UNGROUPED = (1 << 2),
+};
+
/**
* Symbols group
*/
@@ -106,13 +113,15 @@ struct rspamd_symbols_group {
gchar *name;
GHashTable *symbols;
gdouble max_score;
- gboolean disabled;
- gboolean one_shot;
+ enum rspamd_symbol_group_flags flags;
};
-#define RSPAMD_SYMBOL_FLAG_IGNORE (1 << 1)
-#define RSPAMD_SYMBOL_FLAG_ONEPARAM (1 << 2)
-#define RSPAMD_SYMBOL_FLAG_UNGROUPPED (1 << 3)
+enum rspamd_symbol_flags {
+ RSPAMD_SYMBOL_FLAG_NORMAL = 0,
+ RSPAMD_SYMBOL_FLAG_IGNORE = (1 << 1),
+ RSPAMD_SYMBOL_FLAG_ONEPARAM = (1 << 2),
+ RSPAMD_SYMBOL_FLAG_UNGROUPPED = (1 << 3),
+};
/**
* Symbol config definition
@@ -125,7 +134,7 @@ struct rspamd_symbol {
guint priority;
struct rspamd_symbols_group *gr; /* Main group */
GPtrArray *groups; /* Other groups */
- guint flags;
+ enum rspamd_symbol_flags flags;
gint nshots;
};
diff --git a/src/libserver/cfg_rcl.c b/src/libserver/cfg_rcl.c
index fa1c07f6e..7520b66d8 100644
--- a/src/libserver/cfg_rcl.c
+++ b/src/libserver/cfg_rcl.c
@@ -326,7 +326,7 @@ rspamd_rcl_group_handler (rspamd_mempool_t *pool, const ucl_object_t *obj,
{
struct rspamd_config *cfg = ud;
struct rspamd_symbols_group *gr;
- const ucl_object_t *val;
+ const ucl_object_t *val, *elt;
struct rspamd_rcl_section *subsection;
struct rspamd_rcl_symbol_data sd;
@@ -343,6 +343,51 @@ rspamd_rcl_group_handler (rspamd_mempool_t *pool, const ucl_object_t *obj,
return FALSE;
}
+ if ((elt = ucl_object_lookup (obj, "one_shot")) != NULL) {
+ if (ucl_object_type (elt) != UCL_BOOLEAN) {
+ g_set_error (err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "one_shot attribute is not boolean for symbol: '%s'",
+ key);
+
+ return FALSE;
+ }
+ if (ucl_object_toboolean (elt)) {
+ gr->flags |= RSPAMD_SYMBOL_GROUP_ONE_SHOT;
+ }
+ }
+
+ if ((elt = ucl_object_lookup (obj, "disabled")) != NULL) {
+ if (ucl_object_type (elt) != UCL_BOOLEAN) {
+ g_set_error (err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "disabled attribute is not boolean for symbol: '%s'",
+ key);
+
+ return FALSE;
+ }
+ if (ucl_object_toboolean (elt)) {
+ gr->flags |= RSPAMD_SYMBOL_GROUP_DISABLED;
+ }
+ }
+
+ if ((elt = ucl_object_lookup (obj, "enabled")) != NULL) {
+ if (ucl_object_type (elt) != UCL_BOOLEAN) {
+ g_set_error (err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "enabled attribute is not boolean for symbol: '%s'",
+ key);
+
+ return FALSE;
+ }
+ if (!ucl_object_toboolean (elt)) {
+ gr->flags |= RSPAMD_SYMBOL_GROUP_DISABLED;
+ }
+ }
+
sd.gr = gr;
sd.cfg = cfg;
@@ -525,39 +570,32 @@ rspamd_rcl_actions_handler (rspamd_mempool_t *pool, const ucl_object_t *obj,
const gchar *key, gpointer ud,
struct rspamd_rcl_section *section, GError **err)
{
- gdouble action_score;
struct rspamd_config *cfg = ud;
- gint action_value;
const ucl_object_t *cur;
ucl_object_iter_t it;
it = ucl_object_iterate_new (obj);
while ((cur = ucl_object_iterate_safe (it, true)) != NULL) {
- if (!rspamd_action_from_str (ucl_object_key (cur), &action_value)) {
- continue;
+ gint type = ucl_object_type (cur);
+
+ if (type == UCL_NULL) {
+ rspamd_config_maybe_disable_action (cfg, ucl_object_key (cur),
+ ucl_object_get_priority (cur));
}
- else {
- if (ucl_object_type (cur) == UCL_NULL) {
- rspamd_config_maybe_disable_action (cfg, ucl_object_key (cur),
- ucl_object_get_priority (cur));
- }
- else {
- if (!ucl_object_todouble_safe (cur, &action_score)) {
- g_set_error (err,
- CFG_RCL_ERROR,
- EINVAL,
- "invalid action definition: '%s'",
- ucl_object_key (cur));
- ucl_object_iterate_free (it);
+ else if (type == UCL_OBJECT || type == UCL_FLOAT || type == UCL_INT) {
+ if (!rspamd_config_set_action_score (cfg,
+ ucl_object_key (cur),
+ cur)) {
+ g_set_error (err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "invalid action definition for: '%s'",
+ ucl_object_key (cur));
+ ucl_object_iterate_free (it);
- return FALSE;
- }
+ return FALSE;
}
-
- rspamd_config_set_action_score (cfg,
- ucl_object_key (cur),
- cur);
}
}
@@ -1314,7 +1352,9 @@ rspamd_rcl_composite_handler (rspamd_mempool_t *pool,
}
rspamd_config_add_symbol (cfg, composite_name, score,
- description, group, FALSE, FALSE,
+ description, group,
+ 0,
+ ucl_object_get_priority (obj), /* No +1 as it is default... */
1);
elt = ucl_object_lookup (obj, "groups");
@@ -1355,7 +1395,7 @@ rspamd_rcl_composite_handler (rspamd_mempool_t *pool,
if (new) {
rspamd_symcache_add_symbol (cfg->cache, composite_name, 0,
- NULL, NULL, SYMBOL_TYPE_COMPOSITE, -1);
+ NULL, composite, SYMBOL_TYPE_COMPOSITE, -1);
}
return TRUE;
@@ -1772,12 +1812,6 @@ rspamd_rcl_config_init (struct rspamd_config *cfg, GHashTable *skip_sections)
0,
"Treat text attachments as normal text parts");
rspamd_rcl_add_default_handler (sub,
- "check_attachments",
- rspamd_rcl_parse_struct_boolean,
- G_STRUCT_OFFSET (struct rspamd_config, check_text_attachements),
- 0,
- "Treat text attachments as normal text parts");
- rspamd_rcl_add_default_handler (sub,
"tempdir",
rspamd_rcl_parse_struct_string,
G_STRUCT_OFFSET (struct rspamd_config, temp_dir),
@@ -2192,18 +2226,6 @@ rspamd_rcl_config_init (struct rspamd_config *cfg, GHashTable *skip_sections)
/* Group part */
rspamd_rcl_add_default_handler (sub,
- "disabled",
- rspamd_rcl_parse_struct_boolean,
- G_STRUCT_OFFSET (struct rspamd_symbols_group, disabled),
- 0,
- "Disable symbols group");
- rspamd_rcl_add_default_handler (sub,
- "enabled",
- rspamd_rcl_parse_struct_boolean,
- G_STRUCT_OFFSET (struct rspamd_symbols_group, disabled),
- RSPAMD_CL_FLAG_BOOLEAN_INVERSE,
- "Enable or disable symbols group");
- rspamd_rcl_add_default_handler (sub,
"max_score",
rspamd_rcl_parse_struct_double,
G_STRUCT_OFFSET (struct rspamd_symbols_group, max_score),
diff --git a/src/libserver/cfg_utils.c b/src/libserver/cfg_utils.c
index cc07c4007..af31db849 100644
--- a/src/libserver/cfg_utils.c
+++ b/src/libserver/cfg_utils.c
@@ -58,7 +58,7 @@ static gchar * rspamd_ucl_read_cb (gchar * chunk,
gint len,
struct map_cb_data *data,
gboolean final);
-static void rspamd_ucl_fin_cb (struct map_cb_data *data);
+static void rspamd_ucl_fin_cb (struct map_cb_data *data, void **target);
static void rspamd_ucl_dtor_cb (struct map_cb_data *data);
guint rspamd_config_log_id = (guint)-1;
@@ -1039,6 +1039,10 @@ rspamd_config_new_group (struct rspamd_config *cfg, const gchar *name)
(rspamd_mempool_destruct_t)g_hash_table_unref, gr->symbols);
gr->name = rspamd_mempool_strdup (cfg->cfg_pool, name);
+ if (strcmp (gr->name, "ungrouped") == 0) {
+ gr->flags |= RSPAMD_SYMBOL_GROUP_UNGROUPED;
+ }
+
g_hash_table_insert (cfg->groups, gr->name, gr);
return gr;
@@ -1354,7 +1358,7 @@ rspamd_ucl_read_cb (gchar * chunk,
}
static void
-rspamd_ucl_fin_cb (struct map_cb_data *data)
+rspamd_ucl_fin_cb (struct map_cb_data *data, void **target)
{
struct rspamd_ucl_map_cbdata *cbdata = data->cur_data, *prev =
data->prev_data;
@@ -1364,13 +1368,6 @@ rspamd_ucl_fin_cb (struct map_cb_data *data)
const ucl_object_t *cur;
struct rspamd_config *cfg = data->map->cfg;
- if (prev != NULL) {
- if (prev->buf != NULL) {
- g_string_free (prev->buf, TRUE);
- }
- g_free (prev);
- }
-
if (cbdata == NULL) {
msg_err_config ("map fin error: new data is NULL");
return;
@@ -1396,6 +1393,17 @@ rspamd_ucl_fin_cb (struct map_cb_data *data)
}
ucl_object_unref (obj);
}
+
+ if (target) {
+ *target = data->cur_data;
+ }
+
+ if (prev != NULL) {
+ if (prev->buf != NULL) {
+ g_string_free (prev->buf, TRUE);
+ }
+ g_free (prev);
+ }
}
static void
@@ -1684,10 +1692,7 @@ rspamd_config_add_symbol (struct rspamd_config *cfg,
/* We also check group information in this case */
if (group != NULL && sym_def->gr != NULL &&
strcmp (group, sym_def->gr->name) != 0) {
- msg_debug_config ("move symbol %s from group %s to %s",
- sym_def->gr->name, group);
- g_hash_table_remove (sym_def->gr->symbols, sym_def->name);
sym_group = g_hash_table_lookup (cfg->groups, group);
if (sym_group == NULL) {
@@ -1695,8 +1700,13 @@ rspamd_config_add_symbol (struct rspamd_config *cfg,
sym_group = rspamd_config_new_group (cfg, group);
}
- sym_def->gr = sym_group;
- g_hash_table_insert (sym_group->symbols, sym_def->name, sym_def);
+ if (!(sym_group->flags & RSPAMD_SYMBOL_GROUP_UNGROUPED)) {
+ msg_debug_config ("move symbol %s from group %s to %s",
+ sym_def->gr->name, group);
+ g_hash_table_remove (sym_def->gr->symbols, sym_def->name);
+ sym_def->gr = sym_group;
+ g_hash_table_insert (sym_group->symbols, sym_def->name, sym_def);
+ }
}
return TRUE;
@@ -1870,7 +1880,7 @@ rspamd_config_is_module_enabled (struct rspamd_config *cfg,
gr = g_hash_table_lookup (cfg->groups, module_name);
if (gr) {
- if (gr->disabled) {
+ if (gr->flags & RSPAMD_SYMBOL_GROUP_DISABLED) {
rspamd_plugins_table_push_elt (L,
"disabled_explicitly", module_name);
msg_info_config ("%s module %s is disabled in the configuration as "
diff --git a/src/libserver/composites.c b/src/libserver/composites.c
index f46f8276d..b1bb2f694 100644
--- a/src/libserver/composites.c
+++ b/src/libserver/composites.c
@@ -559,9 +559,10 @@ composites_metric_callback (struct rspamd_metric_result *metric_res,
NBYTES (g_hash_table_size (task->cfg->composite_symbols) * 2));
/* Process hash table */
- g_hash_table_foreach (task->cfg->composite_symbols,
- composites_foreach_callback,
- cd);
+ rspamd_symcache_composites_foreach (task,
+ task->cfg->cache,
+ composites_foreach_callback,
+ cd);
/* Remove symbols that are in composites */
g_hash_table_foreach (cd->symbols_to_remove, composites_remove_symbols, cd);
diff --git a/src/libserver/dkim.c b/src/libserver/dkim.c
index 2a66146ed..129d78c9c 100644
--- a/src/libserver/dkim.c
+++ b/src/libserver/dkim.c
@@ -2681,13 +2681,25 @@ rspamd_dkim_sign_key_load (const gchar *key, gsize len,
key = tmp;
}
- if (type == RSPAMD_DKIM_KEY_RAW && len == 32) {
- unsigned char pk[32];
- nkey->type = RSPAMD_DKIM_KEY_EDDSA;
- nkey->key.key_eddsa = g_malloc (
- rspamd_cryptobox_sk_sig_bytes (RSPAMD_CRYPTOBOX_MODE_25519));
- ed25519_seed_keypair (pk, nkey->key.key_eddsa, (char *)key);
- nkey->keylen = rspamd_cryptobox_sk_sig_bytes (RSPAMD_CRYPTOBOX_MODE_25519);
+ if (type == RSPAMD_DKIM_KEY_RAW && (len == 32 ||
+ len == rspamd_cryptobox_sk_sig_bytes (RSPAMD_CRYPTOBOX_MODE_25519))) {
+ if (len == 32) {
+ /* Seeded key, need scalarmult */
+ unsigned char pk[32];
+ nkey->type = RSPAMD_DKIM_KEY_EDDSA;
+ nkey->key.key_eddsa = g_malloc (
+ rspamd_cryptobox_sk_sig_bytes (RSPAMD_CRYPTOBOX_MODE_25519));
+ ed25519_seed_keypair (pk, nkey->key.key_eddsa, (char *) key);
+ nkey->keylen = rspamd_cryptobox_sk_sig_bytes (RSPAMD_CRYPTOBOX_MODE_25519);
+ }
+ else {
+ /* Full ed25519 key */
+ unsigned klen = rspamd_cryptobox_sk_sig_bytes (RSPAMD_CRYPTOBOX_MODE_25519);
+ nkey->type = RSPAMD_DKIM_KEY_EDDSA;
+ nkey->key.key_eddsa = g_malloc (klen);
+ memcpy (nkey->key.key_eddsa, key, klen);
+ nkey->keylen = klen;
+ }
}
else {
nkey->key_bio = BIO_new_mem_buf (key, len);
diff --git a/src/libserver/dns.c b/src/libserver/dns.c
index 7ad266c2f..4fbf40728 100644
--- a/src/libserver/dns.c
+++ b/src/libserver/dns.c
@@ -256,7 +256,7 @@ rspamd_dns_server_init (struct upstream *up, guint idx, gpointer ud)
void *serv;
struct rdns_upstream_elt *elt;
- addr = rspamd_upstream_addr (up);
+ addr = rspamd_upstream_addr_next (up);
if (r->cfg) {
serv = rdns_resolver_add_server (r->r, rspamd_inet_address_to_string (addr),
diff --git a/src/libserver/dynamic_cfg.c b/src/libserver/dynamic_cfg.c
index 1e970a17b..984a26697 100644
--- a/src/libserver/dynamic_cfg.c
+++ b/src/libserver/dynamic_cfg.c
@@ -175,22 +175,12 @@ json_config_read_cb (gchar * chunk,
}
static void
-json_config_fin_cb (struct map_cb_data *data)
+json_config_fin_cb (struct map_cb_data *data, void **target)
{
struct config_json_buf *jb;
ucl_object_t *top;
struct ucl_parser *parser;
- if (data->cur_data && data->prev_data) {
- jb = data->prev_data;
- /* Clean prev data */
- if (jb->buf) {
- g_string_free (jb->buf, TRUE);
- }
-
- g_free (jb);
- }
-
/* Now parse json */
if (data->cur_data) {
jb = data->cur_data;
@@ -201,6 +191,7 @@ json_config_fin_cb (struct map_cb_data *data)
if (jb->buf == NULL) {
msg_err ("no data read");
+
return;
}
@@ -225,6 +216,20 @@ json_config_fin_cb (struct map_cb_data *data)
ucl_object_unref (jb->cfg->current_dynamic_conf);
apply_dynamic_conf (top, jb->cfg);
jb->cfg->current_dynamic_conf = top;
+
+ if (target) {
+ *target = data->cur_data;
+ }
+
+ if (data->prev_data) {
+ jb = data->prev_data;
+ /* Clean prev data */
+ if (jb->buf) {
+ g_string_free (jb->buf, TRUE);
+ }
+
+ g_free (jb);
+ }
}
static void
diff --git a/src/libserver/fuzzy_backend_redis.c b/src/libserver/fuzzy_backend_redis.c
index fbccac5ab..956979d42 100644
--- a/src/libserver/fuzzy_backend_redis.c
+++ b/src/libserver/fuzzy_backend_redis.c
@@ -648,7 +648,7 @@ rspamd_fuzzy_backend_check_redis (struct rspamd_fuzzy_backend *bk,
0);
session->up = up;
- addr = rspamd_upstream_addr (up);
+ addr = rspamd_upstream_addr_next (up);
g_assert (addr != NULL);
session->ctx = rspamd_redis_pool_connect (backend->pool,
backend->dbname, backend->password,
@@ -774,7 +774,7 @@ rspamd_fuzzy_backend_count_redis (struct rspamd_fuzzy_backend *bk,
0);
session->up = up;
- addr = rspamd_upstream_addr (up);
+ addr = rspamd_upstream_addr_next (up);
g_assert (addr != NULL);
session->ctx = rspamd_redis_pool_connect (backend->pool,
backend->dbname, backend->password,
@@ -899,7 +899,7 @@ rspamd_fuzzy_backend_version_redis (struct rspamd_fuzzy_backend *bk,
0);
session->up = up;
- addr = rspamd_upstream_addr (up);
+ addr = rspamd_upstream_addr_next (up);
g_assert (addr != NULL);
session->ctx = rspamd_redis_pool_connect (backend->pool,
backend->dbname, backend->password,
@@ -1459,7 +1459,7 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk,
0);
session->up = up;
- addr = rspamd_upstream_addr (up);
+ addr = rspamd_upstream_addr_next (up);
g_assert (addr != NULL);
session->ctx = rspamd_redis_pool_connect (backend->pool,
backend->dbname, backend->password,
diff --git a/src/libserver/html.c b/src/libserver/html.c
index c33aacf82..63638d28b 100644
--- a/src/libserver/html.c
+++ b/src/libserver/html.c
@@ -664,7 +664,8 @@ rspamd_html_url_is_phished (rspamd_mempool_t *pool,
}
}
#endif
- if (rspamd_ftok_casecmp (&disp_tok, &href_tok) != 0) {
+ if (rspamd_ftok_casecmp (&disp_tok, &href_tok) != 0 &&
+ text_url->tldlen > 0 && href_url->tldlen > 0) {
/* Apply the same logic for TLD */
disp_tok.len = text_url->tldlen;
@@ -1301,7 +1302,7 @@ rspamd_html_process_url (rspamd_mempool_t *pool, const gchar *start, guint len,
gchar *decoded;
gint rc;
gsize decoded_len;
- const gchar *p, *s;
+ const gchar *p, *s, *prefix = "http://";
gchar *d;
guint i, dlen;
gboolean has_bad_chars = FALSE, no_prefix = FALSE;
@@ -1346,25 +1347,55 @@ rspamd_html_process_url (rspamd_mempool_t *pool, const gchar *start, guint len,
}
}
- if (memchr (s, ':', len) == NULL) {
- /* We have no prefix */
- dlen += sizeof ("http://") - 1;
- no_prefix = TRUE;
+ if (rspamd_substring_search (start, len, "://", 3) == -1) {
+ if (len >= sizeof ("mailto:") &&
+ (memcmp (start, "mailto:", sizeof ("mailto:") - 1) == 0 ||
+ memcmp (start, "tel:", sizeof ("tel:") - 1) == 0 ||
+ memcmp (start, "callto:", sizeof ("callto:") - 1) == 0)) {
+ /* Exclusion, has valid but 'strange' prefix */
+ }
+ else {
+ for (i = 0; i < len; i ++) {
+ if (!((s[i] & 0x80) || g_ascii_isalnum (s[i]))) {
+ if (i == 0 && len > 2 && s[i] == '/' && s[i + 1] == '/') {
+ prefix = "http:";
+ dlen += sizeof ("http:") - 1;
+ no_prefix = TRUE;
+ }
+ else if (s[i] == '@') {
+ /* Likely email prefix */
+ prefix = "mailto://";
+ dlen += sizeof ("mailto://") - 1;
+ no_prefix = TRUE;
+ }
+ else if (s[i] == ':' && i != 0) {
+ /* Special case */
+ no_prefix = FALSE;
+ }
+ else {
+ if (i == 0) {
+ /* No valid data */
+ return NULL;
+ }
+ else {
+ no_prefix = TRUE;
+ dlen += strlen (prefix);
+ }
+ }
+
+ break;
+ }
+ }
+ }
}
decoded = rspamd_mempool_alloc (pool, dlen + 1);
d = decoded;
if (no_prefix) {
- if (s[0] == '/' && (len > 2 && s[1] == '/')) {
- /* //bla case */
- memcpy (d, "http:", sizeof ("http:") - 1);
- d += sizeof ("http:") - 1;
- }
- else {
- memcpy (d, "http://", sizeof ("http://") - 1);
- d += sizeof ("http://") - 1;
- }
+ gsize plen = strlen (prefix);
+ memcpy (d, prefix, plen);
+ d += plen;
}
/*
@@ -1410,7 +1441,9 @@ rspamd_html_process_url (rspamd_mempool_t *pool, const gchar *start, guint len,
rc = rspamd_url_parse (url, decoded, dlen, pool, RSPAMD_URL_PARSE_HREF);
- if (rc == URI_ERRNO_OK) {
+ /* Filter some completely damaged urls */
+ if (rc == URI_ERRNO_OK && url->hostlen > 0 &&
+ !((url->flags & RSPAMD_URL_FLAG_OBSCURED) && (url->protocol & PROTOCOL_UNKNOWN))) {
url->flags |= saved_flags;
if (has_bad_chars) {
@@ -2664,10 +2697,33 @@ rspamd_html_process_part_full (rspamd_mempool_t *pool, struct html_content *hc,
case comment_tag:
if (t != '-') {
hc->flags |= RSPAMD_HTML_FLAG_BAD_ELEMENTS;
+ state = tag_end;
+ }
+ else {
+ p++;
+ ebrace = 0;
+ /*
+ * https://www.w3.org/TR/2012/WD-html5-20120329/syntax.html#syntax-comments
+ * ... the text must not start with a single
+ * U+003E GREATER-THAN SIGN character (>),
+ * nor start with a "-" (U+002D) character followed by
+ * a U+003E GREATER-THAN SIGN (>) character,
+ * nor contain two consecutive U+002D HYPHEN-MINUS
+ * characters (--), nor end with a "-" (U+002D) character.
+ */
+ if (p[0] == '-' && p + 1 < end && p[1] == '>') {
+ hc->flags |= RSPAMD_HTML_FLAG_BAD_ELEMENTS;
+ p ++;
+ state = tag_end;
+ }
+ else if (*p == '>') {
+ hc->flags |= RSPAMD_HTML_FLAG_BAD_ELEMENTS;
+ state = tag_end;
+ }
+ else {
+ state = comment_content;
+ }
}
- p ++;
- ebrace = 0;
- state = comment_content;
break;
case comment_content:
diff --git a/src/libserver/milter.c b/src/libserver/milter.c
index b3cd46226..236a4bf75 100644
--- a/src/libserver/milter.c
+++ b/src/libserver/milter.c
@@ -22,7 +22,7 @@
#include "unix-std.h"
#include "logger.h"
#include "ottery.h"
-#include "libutil/http.h"
+#include "libutil/http_connection.h"
#include "libutil/http_private.h"
#include "libserver/protocol_internal.h"
#include "libserver/cfg_file_private.h"
@@ -235,6 +235,7 @@ static void
rspamd_milter_on_protocol_error (struct rspamd_milter_session *session,
struct rspamd_milter_private *priv, GError *err)
{
+ msg_debug_milter ("protocol error: %e", err);
priv->state = RSPAMD_MILTER_WANNA_DIE;
REF_RETAIN (session);
priv->err_cb (priv->fd, session, priv->ud, err);
@@ -1189,6 +1190,7 @@ rspamd_milter_send_action (struct rspamd_milter_session *session,
case RSPAMD_MILTER_REJECT:
case RSPAMD_MILTER_TEMPFAIL:
/* No additional arguments */
+ msg_debug_milter ("send %c command", cmd);
SET_COMMAND (cmd, 0, reply, pos);
break;
case RSPAMD_MILTER_QUARANTINE:
@@ -1199,6 +1201,7 @@ rspamd_milter_send_action (struct rspamd_milter_session *session,
}
len = strlen (reason);
+ msg_debug_milter ("send quarantine action %s", reason);
SET_COMMAND (cmd, len + 1, reply, pos);
memcpy (pos, reason, len + 1);
break;
@@ -1207,6 +1210,7 @@ rspamd_milter_send_action (struct rspamd_milter_session *session,
value = va_arg (ap, GString *);
/* Name and value must be zero terminated */
+ msg_debug_milter ("add header command - \"%v\"=\"%v\"", name, value);
SET_COMMAND (cmd, name->len + value->len + 2, reply, pos);
memcpy (pos, name->str, name->len + 1);
pos += name->len + 1;
@@ -1214,13 +1218,16 @@ rspamd_milter_send_action (struct rspamd_milter_session *session,
break;
case RSPAMD_MILTER_CHGHEADER:
case RSPAMD_MILTER_INSHEADER:
- idx = htonl (va_arg (ap, guint32));
+ idx = va_arg (ap, guint32);
name = va_arg (ap, GString *);
value = va_arg (ap, GString *);
+ msg_debug_milter ("change/insert header command pos = %d- \"%v\"=\"%v\"",
+ idx, name, value);
/* Name and value must be zero terminated */
SET_COMMAND (cmd, name->len + value->len + 2 + sizeof (guint32),
reply, pos);
+ idx = htonl (idx);
memcpy (pos, &idx, sizeof (idx));
pos += sizeof (idx);
memcpy (pos, name->str, name->len + 1);
@@ -1233,14 +1240,20 @@ rspamd_milter_send_action (struct rspamd_milter_session *session,
case RSPAMD_MILTER_CHGFROM:
/* Single GString * argument */
value = va_arg (ap, GString *);
+ msg_debug_milter ("command %c; value=%v", cmd, value);
SET_COMMAND (cmd, value->len + 1, reply, pos);
memcpy (pos, value->str, value->len + 1);
break;
case RSPAMD_MILTER_OPTNEG:
- ver = htonl (va_arg (ap, guint32));
- actions = htonl (va_arg (ap, guint32));
- protocol = htonl (va_arg (ap, guint32));
-
+ ver = va_arg (ap, guint32);
+ actions = va_arg (ap, guint32);
+ protocol = va_arg (ap, guint32);
+
+ msg_debug_milter ("optneg reply: ver=%d, actions=%d, protocol=%d",
+ ver, actions, protocol);
+ ver = htonl (ver);
+ actions = htonl (actions);
+ protocol = htonl (protocol);
SET_COMMAND (cmd, sizeof (guint32) * 3, reply, pos);
memcpy (pos, &ver, sizeof (ver));
pos += sizeof (ver);
@@ -1531,9 +1544,9 @@ rspamd_milter_remove_header_safe (struct rspamd_milter_session *session,
RSPAMD_MILTER_CHGHEADER,
nhdr, hname, hvalue);
}
- else if (nhdr == 0) {
+ else if (nhdr == 0 && ar->len > 0) {
/* We need to clear all headers */
- for (i = 1; i <= ar->len; i ++) {
+ for (i = ar->len; i > 0; i --) {
rspamd_milter_send_action (session,
RSPAMD_MILTER_CHGHEADER,
i, hname, hvalue);
diff --git a/src/libserver/monitored.c b/src/libserver/monitored.c
index 9adcab943..39590bfcc 100644
--- a/src/libserver/monitored.c
+++ b/src/libserver/monitored.c
@@ -51,7 +51,6 @@ struct rspamd_monitored_ctx {
struct rspamd_monitored {
gchar *url;
- gdouble monitoring_interval;
gdouble monitoring_mult;
gdouble offline_time;
gdouble total_offline_time;
@@ -173,7 +172,7 @@ rspamd_monitored_periodic (gint fd, short what, gpointer ud)
gdouble jittered;
gboolean ret = FALSE;
- jittered = rspamd_time_jitter (m->monitoring_interval * m->monitoring_mult,
+ jittered = rspamd_time_jitter (m->ctx->monitoring_interval * m->monitoring_mult,
0.0);
double_to_tv (jittered, &tv);
@@ -480,7 +479,6 @@ rspamd_monitored_create_ (struct rspamd_monitored_ctx *ctx,
m->flags = flags;
m->url = g_strdup (line);
m->ctx = ctx;
- m->monitoring_interval = ctx->monitoring_interval;
m->monitoring_mult = 1.0;
m->max_errors = ctx->max_errors;
m->alive = TRUE;
@@ -602,7 +600,7 @@ rspamd_monitored_start (struct rspamd_monitored *m)
g_assert (m != NULL);
msg_debug_mon ("started monitored object %s", m->url);
- jittered = rspamd_time_jitter (m->monitoring_interval * m->monitoring_mult,
+ jittered = rspamd_time_jitter (m->ctx->monitoring_interval * m->monitoring_mult,
0.0);
double_to_tv (jittered, &tv);
diff --git a/src/libserver/protocol.c b/src/libserver/protocol.c
index 5bcfbc37a..2a9cc605a 100644
--- a/src/libserver/protocol.c
+++ b/src/libserver/protocol.c
@@ -1737,7 +1737,7 @@ rspamd_protocol_write_reply (struct rspamd_task *task)
rspamd_http_connection_reset (task->http_conn);
rspamd_http_connection_write_message (task->http_conn, msg, NULL,
- ctype, task, task->sock, &task->tv, task->ev_base);
+ ctype, task, &task->tv);
task->processed_stages |= RSPAMD_TASK_STAGE_REPLIED;
}
diff --git a/src/libserver/protocol.h b/src/libserver/protocol.h
index 254c5fcd1..08372d765 100644
--- a/src/libserver/protocol.h
+++ b/src/libserver/protocol.h
@@ -8,7 +8,7 @@
#include "config.h"
#include "filter.h"
-#include "http.h"
+#include "http_connection.h"
#include "task.h"
#define RSPAMD_BASE_ERROR 500
diff --git a/src/libserver/rspamd_control.c b/src/libserver/rspamd_control.c
index 84c53700e..149ad4245 100644
--- a/src/libserver/rspamd_control.c
+++ b/src/libserver/rspamd_control.c
@@ -16,7 +16,8 @@
#include "config.h"
#include "rspamd.h"
#include "rspamd_control.h"
-#include "libutil/http.h"
+#include "worker_util.h"
+#include "libutil/http_connection.h"
#include "libutil/http_private.h"
#include "unix-std.h"
#include "utlist.h"
@@ -51,6 +52,7 @@ struct rspamd_control_session {
struct rspamd_http_connection *conn;
struct rspamd_control_command cmd;
struct rspamd_control_reply_elt *replies;
+ rspamd_inet_addr_t *addr;
guint replies_remain;
gboolean is_reply;
};
@@ -129,9 +131,7 @@ rspamd_control_send_error (struct rspamd_control_session *session,
NULL,
"application/json",
session,
- session->fd,
- &io_timeout,
- session->rspamd_main->ev_base);
+ &io_timeout);
}
static void
@@ -154,21 +154,25 @@ rspamd_control_send_ucl (struct rspamd_control_session *session,
NULL,
"application/json",
session,
- session->fd,
- &io_timeout,
- session->rspamd_main->ev_base);
+ &io_timeout);
}
static void
rspamd_control_connection_close (struct rspamd_control_session *session)
{
struct rspamd_control_reply_elt *elt, *telt;
+ struct rspamd_main *rspamd_main;
+
+ rspamd_main = session->rspamd_main;
+ msg_info_main ("finished connection from %s",
+ rspamd_inet_address_to_string (session->addr));
DL_FOREACH_SAFE (session->replies, elt, telt) {
event_del (&elt->io_ev);
g_free (elt);
}
+ rspamd_inet_address_free (session->addr);
rspamd_http_connection_unref (session->conn);
close (session->fd);
g_free (session);
@@ -365,9 +369,12 @@ static void
rspamd_control_error_handler (struct rspamd_http_connection *conn, GError *err)
{
struct rspamd_control_session *session = conn->ud;
+ struct rspamd_main *rspamd_main;
+
+ rspamd_main = session->rspamd_main;
if (!session->is_reply) {
- msg_info ("abnormally closing control connection: %e", err);
+ msg_info_main ("abnormally closing control connection: %e", err);
session->is_reply = TRUE;
rspamd_control_send_error (session, err->code, "%s", err->message);
}
@@ -437,7 +444,7 @@ rspamd_control_broadcast_cmd (struct rspamd_main *rspamd_main,
DL_APPEND (res, rep_elt);
}
else {
- msg_err ("cannot write command %d(%z) to the worker %P(%s), fd: %d: %s",
+ msg_err_main ("cannot write command %d(%z) to the worker %P(%s), fd: %d: %s",
(int)cmd->type, iov.iov_len,
wrk->pid,
g_quark_to_string (wrk->type),
@@ -504,23 +511,24 @@ rspamd_control_finish_handler (struct rspamd_http_connection *conn,
void
rspamd_control_process_client_socket (struct rspamd_main *rspamd_main,
- gint fd)
+ gint fd, rspamd_inet_addr_t *addr)
{
struct rspamd_control_session *session;
session = g_malloc0 (sizeof (*session));
session->fd = fd;
- session->conn = rspamd_http_connection_new (NULL,
+ session->conn = rspamd_http_connection_new (rspamd_main->http_ctx,
+ fd,
+ NULL,
rspamd_control_error_handler,
rspamd_control_finish_handler,
0,
- RSPAMD_HTTP_SERVER,
- NULL,
- NULL);
+ RSPAMD_HTTP_SERVER);
session->rspamd_main = rspamd_main;
- rspamd_http_connection_read_message (session->conn, session, session->fd,
- &io_timeout, rspamd_main->ev_base);
+ session->addr = addr;
+ rspamd_http_connection_read_message (session->conn, session,
+ &io_timeout);
}
struct rspamd_worker_control_data {
@@ -543,14 +551,16 @@ rspamd_control_default_cmd_handler (gint fd,
gssize r;
struct rusage rusg;
struct rspamd_config *cfg;
+ struct rspamd_main *rspamd_main;
memset (&rep, 0, sizeof (rep));
rep.type = cmd->type;
+ rspamd_main = cd->worker->srv;
switch (cmd->type) {
case RSPAMD_CONTROL_STAT:
if (getrusage (RUSAGE_SELF, &rusg) == -1) {
- msg_err ("cannot get rusage stats: %s",
+ msg_err_main ("cannot get rusage stats: %s",
strerror (errno));
}
else {
@@ -594,7 +604,7 @@ rspamd_control_default_cmd_handler (gint fd,
r = write (fd, &rep, sizeof (rep));
if (r != sizeof (rep)) {
- msg_err ("cannot write reply to the control socket: %s",
+ msg_err_main ("cannot write reply to the control socket: %s",
strerror (errno));
}
diff --git a/src/libserver/rspamd_control.h b/src/libserver/rspamd_control.h
index bc42b662e..ec3962c64 100644
--- a/src/libserver/rspamd_control.h
+++ b/src/libserver/rspamd_control.h
@@ -193,7 +193,7 @@ typedef void (*rspamd_srv_reply_handler) (struct rspamd_worker *worker,
* Process client socket connection
*/
void rspamd_control_process_client_socket (struct rspamd_main *rspamd_main,
- gint fd);
+ gint fd, rspamd_inet_addr_t *addr);
/**
* Register default handlers for a worker
diff --git a/src/libserver/rspamd_symcache.c b/src/libserver/rspamd_symcache.c
index d02638add..3cfb15408 100644
--- a/src/libserver/rspamd_symcache.c
+++ b/src/libserver/rspamd_symcache.c
@@ -23,6 +23,7 @@
#include "unix-std.h"
#include "contrib/t1ha/t1ha.h"
#include "libserver/worker_util.h"
+#include "khash.h"
#include <math.h>
#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
@@ -81,51 +82,6 @@ struct symcache_order {
ref_entry_t ref;
};
-struct rspamd_symcache {
- /* Hash table for fast access */
- GHashTable *items_by_symbol;
- GPtrArray *items_by_id;
- struct symcache_order *items_by_order;
- GPtrArray *filters;
- GPtrArray *prefilters;
- GPtrArray *postfilters;
- GPtrArray *composites;
- GPtrArray *idempotent;
- GPtrArray *virtual;
- GPtrArray *squeezed;
- GList *delayed_deps;
- GList *delayed_conditions;
- rspamd_mempool_t *static_pool;
- guint64 cksum;
- gdouble total_weight;
- guint used_items;
- guint stats_symbols_count;
- guint64 total_hits;
- guint id;
- struct rspamd_config *cfg;
- gdouble reload_time;
- gint peak_cb;
-};
-
-struct item_stat {
- struct rspamd_counter_data time_counter;
- gdouble avg_time;
- gdouble weight;
- guint hits;
- guint64 total_hits;
- struct rspamd_counter_data frequency_counter;
- gdouble avg_frequency;
- gdouble stddev_frequency;
-};
-
-struct rspamd_symcache_dynamic_item {
- guint16 start_msec; /* Relative to task time */
- unsigned started:1;
- unsigned finished:1;
- /* unsigned pad:14; */
- guint32 async_events;
-};
-
struct rspamd_symcache_item {
/* This block is likely shared */
struct item_stat *st;
@@ -165,6 +121,53 @@ struct rspamd_symcache_item {
GPtrArray *rdeps;
};
+struct item_stat {
+ struct rspamd_counter_data time_counter;
+ gdouble avg_time;
+ gdouble weight;
+ guint hits;
+ guint64 total_hits;
+ struct rspamd_counter_data frequency_counter;
+ gdouble avg_frequency;
+ gdouble stddev_frequency;
+};
+
+struct rspamd_symcache {
+ /* Hash table for fast access */
+ GHashTable *items_by_symbol;
+ GPtrArray *items_by_id;
+ struct symcache_order *items_by_order;
+ GPtrArray *filters;
+ GPtrArray *prefilters;
+ GPtrArray *postfilters;
+ GPtrArray *composites;
+ GPtrArray *idempotent;
+ GPtrArray *virtual;
+ GPtrArray *squeezed;
+ GList *delayed_deps;
+ GList *delayed_conditions;
+ rspamd_mempool_t *static_pool;
+ guint64 cksum;
+ gdouble total_weight;
+ guint used_items;
+ guint stats_symbols_count;
+ guint64 total_hits;
+ guint id;
+ struct rspamd_config *cfg;
+ gdouble reload_time;
+ gint peak_cb;
+};
+
+struct rspamd_symcache_dynamic_item {
+ guint16 start_msec; /* Relative to task time */
+ unsigned started:1;
+ unsigned finished:1;
+ /* unsigned pad:14; */
+ guint32 async_events;
+};
+
+
+
struct cache_dependency {
struct rspamd_symcache_item *item;
gchar *sym;
@@ -529,24 +532,31 @@ rspamd_symcache_post_init (struct rspamd_symcache *cache)
dit = rspamd_symcache_find_filter (cache, dep->sym);
if (dit != NULL) {
- if (dit->id == i) {
- msg_err_cache ("cannot add dependency on self: %s -> %s "
- "(resolved to %s)",
- it->symbol, dep->sym, dit->symbol);
+ if (!dit->is_filter) {
+ msg_err_cache ("cannot depend on non filter symbol "
+ "(%s wants to add dependency on %s)",
+ dep->sym, dit->symbol);
}
else {
- rdep = rspamd_mempool_alloc (cache->static_pool,
- sizeof (*rdep));
-
- rdep->sym = dep->sym;
- rdep->item = it;
- rdep->id = i;
- g_ptr_array_add (dit->rdeps, rdep);
- dep->item = dit;
- dep->id = dit->id;
-
- msg_debug_cache ("add dependency from %d on %d", it->id,
- dit->id);
+ if (dit->id == i) {
+ msg_err_cache ("cannot add dependency on self: %s -> %s "
+ "(resolved to %s)",
+ it->symbol, dep->sym, dit->symbol);
+ } else {
+ rdep = rspamd_mempool_alloc (cache->static_pool,
+ sizeof (*rdep));
+
+ rdep->sym = dep->sym;
+ rdep->item = it;
+ rdep->id = i;
+ g_assert (dit->rdeps != NULL);
+ g_ptr_array_add (dit->rdeps, rdep);
+ dep->item = dit;
+ dep->id = dit->id;
+
+ msg_debug_cache ("add dependency from %d on %d", it->id,
+ dit->id);
+ }
}
}
else {
@@ -921,6 +931,8 @@ rspamd_symcache_add_symbol (struct rspamd_symcache *cache,
*/
if (item->type & SYMBOL_TYPE_COMPOSITE) {
item->specific.normal.condition_cb = -1;
+ item->specific.normal.user_data = user_data;
+ g_assert (user_data != NULL);
g_ptr_array_add (cache->composites, item);
item->id = cache->items_by_id->len;
@@ -2834,4 +2846,26 @@ rspamd_symcache_get_symbol_flags (struct rspamd_symcache *cache,
}
return 0;
+}
+
+void
+rspamd_symcache_composites_foreach (struct rspamd_task *task,
+ struct rspamd_symcache *cache,
+ GHFunc func,
+ gpointer fd)
+{
+ guint i;
+ struct rspamd_symcache_item *item;
+ struct rspamd_symcache_dynamic_item *dyn_item;
+
+ PTR_ARRAY_FOREACH (cache->composites, i, item) {
+ dyn_item = rspamd_symcache_get_dynamic (task->checkpoint, item);
+
+ if (!CHECK_START_BIT (task->checkpoint, dyn_item)) {
+ /* Cannot do it due to 2 passes */
+ /* SET_START_BIT (task->checkpoint, dyn_item); */
+ func (item->symbol, item->specific.normal.user_data, fd);
+ SET_FINISH_BIT (task->checkpoint, dyn_item);
+ }
+ }
} \ No newline at end of file
diff --git a/src/libserver/rspamd_symcache.h b/src/libserver/rspamd_symcache.h
index 965895221..a038d6a9d 100644
--- a/src/libserver/rspamd_symcache.h
+++ b/src/libserver/rspamd_symcache.h
@@ -50,6 +50,7 @@ enum rspamd_symbol_type {
SYMBOL_TYPE_MIME_ONLY = (1 << 15), /* Symbol is mime only */
SYMBOL_TYPE_EXPLICIT_DISABLE = (1 << 16), /* Symbol should be disabled explicitly only */
SYMBOL_TYPE_IGNORE_PASSTHROUGH = (1 << 17), /* Symbol ignores passthrough result */
+ SYMBOL_TYPE_USE_CORO = (1 << 18), /* Symbol uses lua coroutines */
};
/**
@@ -402,4 +403,16 @@ gboolean rspamd_symcache_item_async_dec_check_full (struct rspamd_task *task,
void rspamd_symcache_disable_all_symbols (struct rspamd_task *task,
struct rspamd_symcache *cache,
guint skip_mask);
+
+/**
+ * Iterates over the list of the enabled composites calling specified function
+ * @param task
+ * @param cache
+ * @param func
+ * @param fd
+ */
+void rspamd_symcache_composites_foreach (struct rspamd_task *task,
+ struct rspamd_symcache *cache,
+ GHFunc func,
+ gpointer fd);
#endif
diff --git a/src/libserver/task.h b/src/libserver/task.h
index 93e0ae0e8..684f5c2c0 100644
--- a/src/libserver/task.h
+++ b/src/libserver/task.h
@@ -17,7 +17,7 @@
#define TASK_H_
#include "config.h"
-#include "http.h"
+#include "http_connection.h"
#include "events.h"
#include "util.h"
#include "mem_pool.h"
diff --git a/src/libserver/url.c b/src/libserver/url.c
index 421c8a181..f860eec0c 100644
--- a/src/libserver/url.c
+++ b/src/libserver/url.c
@@ -46,6 +46,7 @@
#include "message.h"
#include "multipattern.h"
#include "contrib/uthash/utlist.h"
+#include "contrib/http-parser/http_parser.h"
#include <unicode/utf8.h>
#include <unicode/uchar.h>
@@ -103,6 +104,11 @@ static const struct {
.len = 3
},
{
+ .proto = PROTOCOL_TELEPHONE,
+ .name = "callto",
+ .len = 3
+ },
+ {
.proto = PROTOCOL_UNKNOWN,
.name = NULL,
.len = 0
@@ -192,7 +198,7 @@ struct url_matcher static_matchers[] = {
0, 0},
{"mailto:", "", url_email_start, url_email_end,
0, 0},
- {"callto://", "", url_web_start, url_web_end,
+ {"callto:", "", url_tel_start, url_tel_end,
0, 0},
{"h323:", "", url_web_start, url_web_end,
0, 0},
@@ -454,7 +460,7 @@ rspamd_url_parse_tld_file (const gchar *fname,
m.flags = flags;
rspamd_multipattern_add_pattern (url_scanner->search_trie, p,
- RSPAMD_MULTIPATTERN_TLD | RSPAMD_MULTIPATTERN_ICASE);
+ RSPAMD_MULTIPATTERN_TLD|RSPAMD_MULTIPATTERN_ICASE|RSPAMD_MULTIPATTERN_UTF8);
m.pattern = rspamd_multipattern_get_pattern (url_scanner->search_trie,
rspamd_multipattern_get_npatterns (url_scanner->search_trie) - 1);
m.patlen = strlen (m.pattern);
@@ -517,13 +523,13 @@ rspamd_url_init (const gchar *tld_file)
url_scanner->matchers = g_array_sized_new (FALSE, TRUE,
sizeof (struct url_matcher), 13000);
url_scanner->search_trie = rspamd_multipattern_create_sized (13000,
- RSPAMD_MULTIPATTERN_TLD | RSPAMD_MULTIPATTERN_ICASE);
+ RSPAMD_MULTIPATTERN_TLD|RSPAMD_MULTIPATTERN_ICASE|RSPAMD_MULTIPATTERN_UTF8);
}
else {
url_scanner->matchers = g_array_sized_new (FALSE, TRUE,
sizeof (struct url_matcher), 128);
url_scanner->search_trie = rspamd_multipattern_create_sized (128,
- RSPAMD_MULTIPATTERN_TLD | RSPAMD_MULTIPATTERN_ICASE);
+ RSPAMD_MULTIPATTERN_TLD|RSPAMD_MULTIPATTERN_ICASE|RSPAMD_MULTIPATTERN_UTF8);
}
rspamd_url_add_static_matchers (url_scanner);
@@ -538,8 +544,10 @@ rspamd_url_init (const gchar *tld_file)
g_error_free (err);
}
- msg_debug ("initialized trie of %ud elements",
- url_scanner->matchers->len);
+ if (tld_file != NULL) {
+ msg_info ("initialized %ud url tld suffixes from '%s'",
+ url_scanner->matchers->len, tld_file);
+ }
}
#define SET_U(u, field) do { \
@@ -736,6 +744,144 @@ rspamd_mailto_parse (struct http_parser_url *u,
}
static gint
+rspamd_telephone_parse (struct http_parser_url *u,
+ const gchar *str, gsize len,
+ gchar const **end,
+ enum rspamd_url_parse_flags parse_flags,
+ guint *flags)
+{
+ enum {
+ parse_protocol,
+ parse_semicolon,
+ parse_slash,
+ parse_slash_slash,
+ parse_spaces,
+ parse_plus,
+ parse_phone_start,
+ parse_phone,
+ } st = parse_protocol;
+
+ const gchar *p = str, *c = str, *last = str + len;
+ gchar t;
+ gint ret = 1, i;
+ UChar32 uc;
+
+ if (u != NULL) {
+ memset (u, 0, sizeof (*u));
+ }
+
+ while (p < last) {
+ t = *p;
+
+ switch (st) {
+ case parse_protocol:
+ if (t == ':') {
+ st = parse_semicolon;
+ SET_U (u, UF_SCHEMA);
+ }
+ p++;
+ break;
+ case parse_semicolon:
+ if (t == '/' || t == '\\') {
+ st = parse_slash;
+ p++;
+ }
+ else {
+ st = parse_slash_slash;
+ }
+ break;
+ case parse_slash:
+ if (t == '/' || t == '\\') {
+ st = parse_slash_slash;
+ }
+ else {
+ goto out;
+ }
+ p++;
+ break;
+ case parse_slash_slash:
+ if (g_ascii_isspace (t)) {
+ st = parse_spaces;
+ p++;
+ }
+ else if (t == '+') {
+ c = p;
+ st = parse_plus;
+ }
+ else if (t == '/') {
+ /* Skip multiple slashes */
+ p++;
+ }
+ else {
+ st = parse_phone_start;
+ c = p;
+ }
+ break;
+ case parse_spaces:
+ if (t == '+') {
+ c = p;
+ st = parse_plus;
+ }
+ else if (!g_ascii_isspace (t)) {
+ st = parse_phone_start;
+ c = p;
+ }
+ else {
+ p ++;
+ }
+ break;
+ case parse_plus:
+ c = p;
+ p ++;
+ st = parse_phone_start;
+ break;
+ case parse_phone_start:
+ if (*p == '%' || *p == '(' || g_ascii_isdigit (*p)) {
+ st = parse_phone;
+ p ++;
+ }
+ else {
+ goto out;
+ }
+ break;
+ case parse_phone:
+ i = p - str;
+ U8_NEXT (str, i, len, uc);
+ p = str + i;
+
+ if (u_isdigit (uc) || uc == '(' || uc == ')' || uc == '[' || uc == ']'
+ || u_isspace (uc) || uc == '%') {
+ p ++;
+ }
+ else if (uc <= 0 || is_url_end (uc)) {
+ ret = 0;
+ goto set;
+ }
+ break;
+ }
+ }
+
+ set:
+ if (st == parse_phone) {
+ if (p - c != 0) {
+ SET_U (u, UF_HOST);
+ ret = 0;
+ }
+ }
+
+ out:
+ if (end != NULL) {
+ *end = p;
+ }
+
+ if ((parse_flags & RSPAMD_URL_PARSE_CHECK)) {
+ return 0;
+ }
+
+ return ret;
+}
+
+static gint
rspamd_web_parse (struct http_parser_url *u, const gchar *str, gsize len,
gchar const **end,
enum rspamd_url_parse_flags parse_flags,
@@ -1020,11 +1166,13 @@ rspamd_web_parse (struct http_parser_url *u, const gchar *str, gsize len,
(*flags) |= RSPAMD_URL_FLAG_OBSCURED;
}
else {
- if (!(parse_flags & RSPAMD_URL_PARSE_CHECK)) {
- goto out;
- }
- else {
- goto set;
+ if (!u_isgraph (uc)) {
+ if (!(parse_flags & RSPAMD_URL_PARSE_CHECK)) {
+ goto out;
+ }
+ else {
+ goto set;
+ }
}
}
}
@@ -1240,7 +1388,7 @@ rspamd_web_parse (struct http_parser_url *u, const gchar *str, gsize len,
/* Parse remaining */
switch (st) {
case parse_domain:
- if (p - c == 0) {
+ if (p - c == 0 || !is_domain (*(p - 1)) || !is_domain (*c)) {
goto out;
}
SET_U (u, UF_HOST);
@@ -1633,6 +1781,40 @@ rspamd_url_shift (struct rspamd_url *uri, gsize nlen,
}
}
+static void
+rspamd_telephone_normalise_inplace (struct rspamd_url *uri)
+{
+ gchar *t, *h, *end;
+ gint i = 0, w, orig_len;
+ UChar32 uc;
+
+ t = uri->host;
+ h = t;
+ end = uri->host + uri->hostlen;
+ orig_len = uri->hostlen;
+
+ if (*h == '+') {
+ h ++;
+ t ++;
+ }
+
+ while (h < end) {
+ i = 0;
+ U8_NEXT (h, i, end - h, uc);
+
+ if (u_isdigit (uc)) {
+ w = 0;
+ U8_APPEND_UNSAFE (t, w, uc);
+ t += w;
+ }
+
+ h += i;
+ }
+
+ uri->hostlen = t - uri->host;
+ uri->urllen -= (orig_len - uri->hostlen);
+}
+
enum uri_errno
rspamd_url_parse (struct rspamd_url *uri,
gchar *uristring, gsize len,
@@ -1654,6 +1836,7 @@ rspamd_url_parse (struct rspamd_url *uri,
}
p = uristring;
+ uri->protocol = PROTOCOL_UNKNOWN;
if (len > sizeof ("mailto:") - 1) {
/* For mailto: urls we also need to add slashes to make it a valid URL */
@@ -1661,75 +1844,11 @@ rspamd_url_parse (struct rspamd_url *uri,
ret = rspamd_mailto_parse (&u, uristring, len, &end, parse_flags,
&flags);
}
- else if (g_ascii_strncasecmp (p, "tel:", sizeof ("tel:") - 1) == 0) {
- /* Telephone url */
- gint nlen = 0;
- gboolean has_plus = FALSE;
- end = p + len;
- gchar *t, *tend;
- UChar32 uc;
-
- uri->raw = p;
- uri->rawlen = len;
- uri->string = rspamd_mempool_alloc (pool, len + 1);
- t = uri->string;
- tend = t + len;
- i = 4;
-
- memcpy (t, "tel:", 4);
- t += 4;
- p += 4;
- nlen = 4;
-
- if (*p == '+') {
- has_plus = TRUE;
- *t++ = *p++;
- nlen ++;
- i ++;
- }
-
- while (t < tend && i < len) {
- U8_NEXT (uristring, i, len, uc);
-
- if (u_isdigit (uc)) {
- if (g_ascii_isdigit (uc)) {
- *t++ = uc;
- nlen ++;
- }
- else {
- /* Obfuscated number */
- uri->flags |= RSPAMD_URL_FLAG_OBSCURED;
- }
- }
- else if (IS_OBSCURED_CHAR (uc)) {
- uri->flags |= RSPAMD_URL_FLAG_OBSCURED;
- }
- }
-
- *t = '\0';
-
- if (rspamd_normalise_unicode_inplace (pool, uri->string, &nlen)) {
- uri->flags |= RSPAMD_URL_FLAG_UNNORMALISED;
- }
-
- uri->urllen = nlen;
-
+ else if (g_ascii_strncasecmp (p, "tel:", sizeof ("tel:") - 1) == 0 ||
+ g_ascii_strncasecmp (p, "callto:", sizeof ("callto:") - 1) == 0) {
+ ret = rspamd_telephone_parse (&u, uristring, len, &end, parse_flags,
+ &flags);
uri->protocol = PROTOCOL_TELEPHONE;
- uri->protocollen = 4;
-
- uri->host = uri->string + 4;
- uri->hostlen = nlen - 4;
-
- if (has_plus) {
- uri->tld = uri->string + 5;
- uri->tldlen = nlen - 5;
- }
- else {
- uri->tld = uri->string + 4;
- uri->tldlen = nlen - 4;
- }
-
- return URI_ERRNO_OK;
}
else {
ret = rspamd_web_parse (&u, uristring, len, &end, parse_flags,
@@ -1862,32 +1981,46 @@ rspamd_url_parse (struct rspamd_url *uri,
rspamd_str_lc (uri->string, uri->protocollen);
rspamd_str_lc_utf8 (uri->host, uri->hostlen);
- uri->protocol = PROTOCOL_UNKNOWN;
-
- for (i = 0; i < G_N_ELEMENTS (rspamd_url_protocols); i++) {
- if (uri->protocollen == rspamd_url_protocols[i].len) {
- if (memcmp (uri->string, rspamd_url_protocols[i].name, uri->protocollen) ==
- 0) {
- uri->protocol = i;
- break;
+ if (uri->protocol == PROTOCOL_UNKNOWN) {
+ for (i = 0; i < G_N_ELEMENTS (rspamd_url_protocols); i++) {
+ if (uri->protocollen == rspamd_url_protocols[i].len) {
+ if (memcmp (uri->string,
+ rspamd_url_protocols[i].name, uri->protocollen) == 0) {
+ uri->protocol = rspamd_url_protocols[i].proto;
+ break;
+ }
}
}
}
- /* Find TLD part */
- rspamd_multipattern_lookup (url_scanner->search_trie,
+ if (uri->protocol & (PROTOCOL_HTTP|PROTOCOL_HTTPS|PROTOCOL_MAILTO|PROTOCOL_FTP)) {
+ /* Find TLD part */
+ rspamd_multipattern_lookup (url_scanner->search_trie,
uri->host, uri->hostlen,
rspamd_tld_trie_callback, uri, NULL);
- if (uri->tldlen == 0) {
- if (!(parse_flags & RSPAMD_URL_PARSE_HREF)) {
- /* Ignore URL's without TLD if it is not a numeric URL */
- if (!rspamd_url_is_ip (uri, pool)) {
- return URI_ERRNO_TLD_MISSING;
+ if (uri->tldlen == 0) {
+ if (!(parse_flags & RSPAMD_URL_PARSE_HREF)) {
+ /* Ignore URL's without TLD if it is not a numeric URL */
+ if (!rspamd_url_is_ip (uri, pool)) {
+ return URI_ERRNO_TLD_MISSING;
+ }
+ } else {
+ /* Assume tld equal to host */
+ uri->tld = uri->host;
+ uri->tldlen = uri->hostlen;
}
}
+ }
+ else if (uri->protocol & PROTOCOL_TELEPHONE) {
+ /* We need to normalise phone number: remove all spaces and braces */
+ rspamd_telephone_normalise_inplace (uri);
+
+ if (uri->host[0] == '+') {
+ uri->tld = uri->host + 1;
+ uri->tldlen = uri->hostlen - 1;
+ }
else {
- /* Assume tld equal to host */
uri->tld = uri->host;
uri->tldlen = uri->hostlen;
}
@@ -1899,7 +2032,7 @@ rspamd_url_parse (struct rspamd_url *uri,
}
else {
/* Hack, hack, hack */
- uri->protocol = PROTOCOL_HTTP;
+ uri->protocol = PROTOCOL_UNKNOWN;
}
}
@@ -2366,13 +2499,15 @@ url_tel_start (struct url_callback_data *cb,
const gchar *pos,
url_match_t *match)
{
- if (!(*pos == '+' || g_ascii_isdigit (*pos))) {
- /* Urls cannot start with . */
- return FALSE;
- }
-
match->m_begin = pos;
+ if (pos >= cb->begin + 1) {
+ match->st = *(pos - 1);
+ }
+ else {
+ match->st = '\0';
+ }
+
return TRUE;
}
@@ -2381,29 +2516,26 @@ url_tel_end (struct url_callback_data *cb,
const gchar *pos,
url_match_t *match)
{
- UChar32 uc;
- gint len = cb->end - pos, i = 0;
+ const gchar *last = NULL;
+ struct http_parser_url u;
+ gint len = cb->end - pos;
+ guint flags = 0;
if (match->newline_pos && match->st != '<') {
/* We should also limit our match end to the newline */
len = MIN (len, match->newline_pos - pos);
}
- while (i < len) {
- U8_NEXT (pos, i, len, uc);
-
- if (uc < 0) {
- break;
- }
+ if (rspamd_telephone_parse (&u, pos, len, &last,
+ RSPAMD_URL_PARSE_CHECK, &flags) != 0) {
+ return FALSE;
+ }
- if (!(u_isdigit (uc) || u_isspace (uc) ||
- IS_OBSCURED_CHAR (uc) || uc == '+' ||
- uc == '-' || uc == '.')) {
- break;
- }
+ if (!(u.field_set & (1 << UF_HOST))) {
+ return FALSE;
}
- match->m_len = i;
+ match->m_len = (last - pos);
return TRUE;
}
@@ -3283,10 +3415,11 @@ rspamd_url_encode (struct rspamd_url *url, gsize *pdlen,
d = dest;
dend = d + dlen;
- if (url->protocollen > 0 &&
- (url->protocol >= 0 && url->protocol < G_N_ELEMENTS (rspamd_url_protocols))) {
+ if (url->protocollen > 0) {
d += rspamd_snprintf ((gchar *) d, dend - d,
- "%*s://", url->protocollen, rspamd_url_protocols[url->protocol].name);
+ "%*s://",
+ url->protocollen,
+ rspamd_url_protocol_name (url->protocol));
}
else {
d += rspamd_snprintf ((gchar *) d, dend - d, "http://");
@@ -3329,3 +3462,61 @@ rspamd_url_is_domain (int c)
{
return is_domain ((guchar)c);
}
+
+const gchar*
+rspamd_url_protocol_name (enum rspamd_url_protocol proto)
+{
+ const gchar *ret = "unknown";
+
+ switch (proto) {
+ case PROTOCOL_HTTP:
+ ret = "http";
+ break;
+ case PROTOCOL_HTTPS:
+ ret = "https";
+ break;
+ case PROTOCOL_FTP:
+ ret = "ftp";
+ break;
+ case PROTOCOL_FILE:
+ ret = "file";
+ break;
+ case PROTOCOL_MAILTO:
+ ret = "mailto";
+ break;
+ case PROTOCOL_TELEPHONE:
+ ret = "telephone";
+ break;
+ default:
+ break;
+ }
+
+ return ret;
+}
+
+enum rspamd_url_protocol
+rspamd_url_protocol_from_string (const gchar *str)
+{
+ enum rspamd_url_protocol ret = PROTOCOL_UNKNOWN;
+
+ if (strcmp (str, "http") == 0) {
+ ret = PROTOCOL_HTTP;
+ }
+ else if (strcmp (str, "https") == 0) {
+ ret = PROTOCOL_HTTPS;
+ }
+ else if (strcmp (str, "mailto") == 0) {
+ ret = PROTOCOL_MAILTO;
+ }
+ else if (strcmp (str, "ftp") == 0) {
+ ret = PROTOCOL_FTP;
+ }
+ else if (strcmp (str, "file") == 0) {
+ ret = PROTOCOL_FILE;
+ }
+ else if (strcmp (str, "telephone") == 0) {
+ ret = PROTOCOL_TELEPHONE;
+ }
+
+ return ret;
+}
diff --git a/src/libserver/url.h b/src/libserver/url.h
index 4c34be9e7..3deeb8cf5 100644
--- a/src/libserver/url.h
+++ b/src/libserver/url.h
@@ -79,13 +79,13 @@ enum uri_errno {
};
enum rspamd_url_protocol {
- PROTOCOL_FILE = 0,
- PROTOCOL_FTP,
- PROTOCOL_HTTP,
- PROTOCOL_HTTPS,
- PROTOCOL_MAILTO,
- PROTOCOL_TELEPHONE,
- PROTOCOL_UNKNOWN
+ PROTOCOL_FILE = 1u << 0,
+ PROTOCOL_FTP = 1u << 1,
+ PROTOCOL_HTTP = 1u << 2,
+ PROTOCOL_HTTPS = 1u << 3,
+ PROTOCOL_MAILTO = 1u << 4,
+ PROTOCOL_TELEPHONE = 1u << 5,
+ PROTOCOL_UNKNOWN = 1u << 31,
};
/**
@@ -240,4 +240,18 @@ const gchar * rspamd_url_encode (struct rspamd_url *url, gsize *dlen,
*/
gboolean rspamd_url_is_domain (int c);
+/**
+ * Returns symbolic name for protocol
+ * @param proto
+ * @return
+ */
+const gchar* rspamd_url_protocol_name (enum rspamd_url_protocol proto);
+
+
+/**
+ * Converts string to a numeric protocol
+ * @param str
+ * @return
+ */
+enum rspamd_url_protocol rspamd_url_protocol_from_string (const gchar *str);
#endif
diff --git a/src/libserver/worker_util.c b/src/libserver/worker_util.c
index a0e511929..06296bba2 100644
--- a/src/libserver/worker_util.c
+++ b/src/libserver/worker_util.c
@@ -24,6 +24,7 @@
#include "libutil/map.h"
#include "libutil/map_private.h"
#include "libutil/http_private.h"
+#include "libutil/http_router.h"
#ifdef WITH_GPERF_TOOLS
#include <gperftools/profiler.h>
@@ -434,9 +435,7 @@ rspamd_controller_send_error (struct rspamd_http_connection_entry *entry,
NULL,
"application/json",
entry,
- entry->conn->fd,
- entry->rt->ptv,
- entry->rt->ev_base);
+ entry->rt->ptv);
entry->is_reply = TRUE;
}
@@ -468,9 +467,7 @@ rspamd_controller_send_string (struct rspamd_http_connection_entry *entry,
NULL,
"application/json",
entry,
- entry->conn->fd,
- entry->rt->ptv,
- entry->rt->ev_base);
+ entry->rt->ptv);
entry->is_reply = TRUE;
}
@@ -496,9 +493,7 @@ rspamd_controller_send_ucl (struct rspamd_http_connection_entry *entry,
NULL,
"application/json",
entry,
- entry->conn->fd,
- entry->rt->ptv,
- entry->rt->ev_base);
+ entry->rt->ptv);
entry->is_reply = TRUE;
}
@@ -512,12 +507,14 @@ rspamd_worker_drop_priv (struct rspamd_main *rspamd_main)
strerror (errno));
exit (-errno);
}
+
if (rspamd_main->cfg->rspamd_user &&
- initgroups (rspamd_main->cfg->rspamd_user, rspamd_main->workers_gid) ==
- -1) {
+ initgroups (rspamd_main->cfg->rspamd_user,
+ rspamd_main->workers_gid) == -1) {
msg_err_main ("initgroups failed (%s), aborting", strerror (errno));
exit (-errno);
}
+
if (setuid (rspamd_main->workers_uid) == -1) {
msg_err_main ("cannot setuid to %d (%s), aborting",
(gint) rspamd_main->workers_uid,
diff --git a/src/libserver/worker_util.h b/src/libserver/worker_util.h
index dbcc8f8a2..3186025b3 100644
--- a/src/libserver/worker_util.h
+++ b/src/libserver/worker_util.h
@@ -18,7 +18,7 @@
#include "config.h"
#include "util.h"
-#include "http.h"
+#include "http_connection.h"
#include "rspamd.h"
#ifndef HAVE_SA_SIGINFO
diff --git a/src/libstat/backends/redis_backend.c b/src/libstat/backends/redis_backend.c
index d7402db98..d54767c12 100644
--- a/src/libstat/backends/redis_backend.c
+++ b/src/libstat/backends/redis_backend.c
@@ -975,7 +975,7 @@ rspamd_redis_async_stat_cb (struct rspamd_stat_async_elt *elt, gpointer d)
0);
g_assert (cbdata->selected != NULL);
- addr = rspamd_upstream_addr (cbdata->selected);
+ addr = rspamd_upstream_addr_next (cbdata->selected);
g_assert (addr != NULL);
if (rspamd_inet_address_get_af (addr) == AF_UNIX) {
@@ -1522,7 +1522,7 @@ rspamd_redis_runtime (struct rspamd_task *task,
rt->stcf = stcf;
rt->redis_object_expanded = object_expanded;
- addr = rspamd_upstream_addr (up);
+ addr = rspamd_upstream_addr_next (up);
g_assert (addr != NULL);
if (rspamd_inet_address_get_af (addr) == AF_UNIX) {
@@ -1693,7 +1693,7 @@ rspamd_redis_learn_tokens (struct rspamd_task *task, GPtrArray *tokens,
}
}
- addr = rspamd_upstream_addr (up);
+ addr = rspamd_upstream_addr_next (up);
g_assert (addr != NULL);
if (rspamd_inet_address_get_af (addr) == AF_UNIX) {
diff --git a/src/libstat/learn_cache/redis_cache.c b/src/libstat/learn_cache/redis_cache.c
index 6a0aa1da7..aea024e06 100644
--- a/src/libstat/learn_cache/redis_cache.c
+++ b/src/libstat/learn_cache/redis_cache.c
@@ -385,7 +385,7 @@ rspamd_stat_cache_redis_runtime (struct rspamd_task *task,
rt->task = task;
rt->ctx = ctx;
- addr = rspamd_upstream_addr (up);
+ addr = rspamd_upstream_addr_next (up);
g_assert (addr != NULL);
if (rspamd_inet_address_get_af (addr) == AF_UNIX) {
diff --git a/src/libstat/stat_api.h b/src/libstat/stat_api.h
index 533c42948..f9d1aab5a 100644
--- a/src/libstat/stat_api.h
+++ b/src/libstat/stat_api.h
@@ -39,6 +39,7 @@
#define RSPAMD_STAT_TOKEN_FLAG_STOP_WORD (1u << 10)
#define RSPAMD_STAT_TOKEN_FLAG_SKIPPED (1u << 11)
#define RSPAMD_STAT_TOKEN_FLAG_INVISIBLE_SPACES (1u << 12)
+#define RSPAMD_STAT_TOKEN_FLAG_EMOJI (1u << 13)
typedef struct rspamd_stat_token_s {
rspamd_ftok_t original; /* utf8 raw */
diff --git a/src/libstat/tokenizers/tokenizers.c b/src/libstat/tokenizers/tokenizers.c
index acbbcf2f0..caa4a48a5 100644
--- a/src/libstat/tokenizers/tokenizers.c
+++ b/src/libstat/tokenizers/tokenizers.c
@@ -610,7 +610,25 @@ rspamd_uchars_to_ucs32 (const UChar *src, gsize srclen,
U16_NEXT_UNSAFE (src, i, t);
if (u_isgraph (t)) {
- *d++ = u_tolower (t);
+ UCharCategory cat;
+
+ cat = u_charType (t);
+#if U_ICU_VERSION_MAJOR_NUM >= 57
+ if (u_hasBinaryProperty (t, UCHAR_EMOJI)) {
+ tok->flags |= RSPAMD_STAT_TOKEN_FLAG_EMOJI;
+ }
+#endif
+
+ if (cat == U_UPPERCASE_LETTER ||
+ cat == U_LOWERCASE_LETTER ||
+ cat == U_DECIMAL_DIGIT_NUMBER ||
+ cat == U_CONNECTOR_PUNCTUATION ||
+ cat == U_MATH_SYMBOL ||
+ cat == U_CURRENCY_SYMBOL ||
+ cat == U_INITIAL_PUNCTUATION ||
+ cat == U_FINAL_PUNCTUATION) {
+ *d++ = u_tolower (t);
+ }
}
else {
/* Invisible spaces ! */
diff --git a/src/libutil/CMakeLists.txt b/src/libutil/CMakeLists.txt
index aab754195..f86d650f0 100644
--- a/src/libutil/CMakeLists.txt
+++ b/src/libutil/CMakeLists.txt
@@ -1,27 +1,31 @@
# Librspamd-util
-SET(LIBRSPAMDUTILSRC
- ${CMAKE_CURRENT_SOURCE_DIR}/addr.c
- ${CMAKE_CURRENT_SOURCE_DIR}/aio_event.c
- ${CMAKE_CURRENT_SOURCE_DIR}/bloom.c
- ${CMAKE_CURRENT_SOURCE_DIR}/expression.c
- ${CMAKE_CURRENT_SOURCE_DIR}/fstring.c
- ${CMAKE_CURRENT_SOURCE_DIR}/hash.c
- ${CMAKE_CURRENT_SOURCE_DIR}/http.c
- ${CMAKE_CURRENT_SOURCE_DIR}/logger.c
- ${CMAKE_CURRENT_SOURCE_DIR}/map.c
- ${CMAKE_CURRENT_SOURCE_DIR}/map_helpers.c
- ${CMAKE_CURRENT_SOURCE_DIR}/mem_pool.c
- ${CMAKE_CURRENT_SOURCE_DIR}/printf.c
- ${CMAKE_CURRENT_SOURCE_DIR}/radix.c
- ${CMAKE_CURRENT_SOURCE_DIR}/regexp.c
- ${CMAKE_CURRENT_SOURCE_DIR}/rrd.c
- ${CMAKE_CURRENT_SOURCE_DIR}/shingles.c
- ${CMAKE_CURRENT_SOURCE_DIR}/sqlite_utils.c
- ${CMAKE_CURRENT_SOURCE_DIR}/str_util.c
- ${CMAKE_CURRENT_SOURCE_DIR}/upstream.c
- ${CMAKE_CURRENT_SOURCE_DIR}/util.c
- ${CMAKE_CURRENT_SOURCE_DIR}/heap.c
- ${CMAKE_CURRENT_SOURCE_DIR}/multipattern.c
- ${CMAKE_CURRENT_SOURCE_DIR}/ssl_util.c)
+SET(LIBRSPAMDUTILSRC
+ ${CMAKE_CURRENT_SOURCE_DIR}/addr.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/aio_event.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/bloom.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/expression.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/fstring.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/hash.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/http_util.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/http_message.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/http_connection.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/http_router.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/http_context.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/logger.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/map.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/map_helpers.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/mem_pool.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/printf.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/radix.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/regexp.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/rrd.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/shingles.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/sqlite_utils.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/str_util.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/upstream.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/util.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/heap.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/multipattern.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/ssl_util.c)
# Rspamdutil
SET(RSPAMD_UTIL ${LIBRSPAMDUTILSRC} PARENT_SCOPE) \ No newline at end of file
diff --git a/src/libutil/addr.c b/src/libutil/addr.c
index fd4621aef..7fa634f6f 100644
--- a/src/libutil/addr.c
+++ b/src/libutil/addr.c
@@ -1636,7 +1636,7 @@ rspamd_inet_address_af_order (const rspamd_inet_addr_t *addr)
gint
rspamd_inet_address_compare (const rspamd_inet_addr_t *a1,
- const rspamd_inet_addr_t *a2)
+ const rspamd_inet_addr_t *a2, gboolean compare_ports)
{
g_assert (a1 != NULL);
g_assert (a2 != NULL);
@@ -1648,11 +1648,33 @@ rspamd_inet_address_compare (const rspamd_inet_addr_t *a1,
else {
switch (a1->af) {
case AF_INET:
- return memcmp (&a1->u.in.addr.s4.sin_addr,
- &a2->u.in.addr.s4.sin_addr, sizeof (struct in_addr));
+ if (!compare_ports) {
+ return memcmp (&a1->u.in.addr.s4.sin_addr,
+ &a2->u.in.addr.s4.sin_addr, sizeof (struct in_addr));
+ }
+ else {
+ if (a1->u.in.addr.s4.sin_port == a2->u.in.addr.s4.sin_port) {
+ return memcmp (&a1->u.in.addr.s4.sin_addr,
+ &a2->u.in.addr.s4.sin_addr, sizeof (struct in_addr));
+ }
+ else {
+ return a1->u.in.addr.s4.sin_port - a2->u.in.addr.s4.sin_port;
+ }
+ }
case AF_INET6:
- return memcmp (&a1->u.in.addr.s6.sin6_addr,
- &a2->u.in.addr.s6.sin6_addr, sizeof (struct in6_addr));
+ if (!compare_ports) {
+ return memcmp (&a1->u.in.addr.s6.sin6_addr,
+ &a2->u.in.addr.s6.sin6_addr, sizeof (struct in6_addr));
+ }
+ else {
+ if (a1->u.in.addr.s6.sin6_port == a2->u.in.addr.s6.sin6_port) {
+ return memcmp (&a1->u.in.addr.s6.sin6_addr,
+ &a2->u.in.addr.s6.sin6_addr, sizeof (struct in6_addr));
+ }
+ else {
+ return a1->u.in.addr.s6.sin6_port - a2->u.in.addr.s6.sin6_port;
+ }
+ }
case AF_UNIX:
return strncmp (a1->u.un->addr.sun_path,
a2->u.un->addr.sun_path, sizeof (a1->u.un->addr.sun_path));
@@ -1671,7 +1693,7 @@ rspamd_inet_address_compare_ptr (gconstpointer a1,
const rspamd_inet_addr_t **i1 = (const rspamd_inet_addr_t **)a1,
**i2 = (const rspamd_inet_addr_t **)a2;
- return rspamd_inet_address_compare (*i1, *i2);
+ return rspamd_inet_address_compare (*i1, *i2, FALSE);
}
rspamd_inet_addr_t *
@@ -1724,28 +1746,85 @@ guint
rspamd_inet_address_hash (gconstpointer a)
{
const rspamd_inet_addr_t *addr = a;
- rspamd_cryptobox_fast_hash_state_t st;
-
- rspamd_cryptobox_fast_hash_init (&st, rspamd_hash_seed ());
- rspamd_cryptobox_fast_hash_update (&st, &addr->af, sizeof (addr->af));
+ struct {
+ gchar buf[sizeof(struct in6_addr)]; /* 16 bytes */
+ int af;
+ } layout;
+ gint32 k;
if (addr->af == AF_UNIX && addr->u.un) {
+ rspamd_cryptobox_fast_hash_state_t st;
+
+ rspamd_cryptobox_fast_hash_init (&st, rspamd_hash_seed ());
+ rspamd_cryptobox_fast_hash_update (&st, &addr->af, sizeof (addr->af));
rspamd_cryptobox_fast_hash_update (&st, addr->u.un, sizeof (*addr->u.un));
+
+ return rspamd_cryptobox_fast_hash_final (&st);
}
else {
+ memset (&layout, 0, sizeof (layout));
+ layout.af = addr->af;
+
/* We ignore port part here */
if (addr->af == AF_INET) {
- rspamd_cryptobox_fast_hash_update (&st, &addr->u.in.addr.s4.sin_addr,
+ memcpy (layout.buf, &addr->u.in.addr.s4.sin_addr,
+ sizeof (addr->u.in.addr.s4.sin_addr));
+ }
+ else {
+ memcpy (layout.buf, &addr->u.in.addr.s6.sin6_addr,
+ sizeof (addr->u.in.addr.s6.sin6_addr));
+ }
+
+ k = rspamd_cryptobox_fast_hash (&layout, sizeof (layout),
+ rspamd_hash_seed ());
+ }
+
+ return k;
+}
+
+guint
+rspamd_inet_address_port_hash (gconstpointer a)
+{
+ const rspamd_inet_addr_t *addr = a;
+ struct {
+ gchar buf[sizeof(struct in6_addr)]; /* 16 bytes */
+ int port;
+ int af;
+ } layout;
+
+ gint32 k;
+
+ if (addr->af == AF_UNIX && addr->u.un) {
+ rspamd_cryptobox_fast_hash_state_t st;
+
+ rspamd_cryptobox_fast_hash_init (&st, rspamd_hash_seed ());
+ rspamd_cryptobox_fast_hash_update (&st, &addr->af, sizeof (addr->af));
+ rspamd_cryptobox_fast_hash_update (&st, addr->u.un, sizeof (*addr->u.un));
+
+ return rspamd_cryptobox_fast_hash_final (&st);
+ }
+ else {
+ memset (&layout, 0, sizeof (layout));
+ layout.af = addr->af;
+
+ /* We consider port part here */
+ if (addr->af == AF_INET) {
+ memcpy (layout.buf, &addr->u.in.addr.s4.sin_addr,
sizeof (addr->u.in.addr.s4.sin_addr));
+ layout.port = addr->u.in.addr.s4.sin_port;
}
else {
- rspamd_cryptobox_fast_hash_update (&st, &addr->u.in.addr.s6.sin6_addr,
+ memcpy (layout.buf, &addr->u.in.addr.s6.sin6_addr,
sizeof (addr->u.in.addr.s6.sin6_addr));
+ layout.port = addr->u.in.addr.s6.sin6_port;
}
+
+ k = rspamd_cryptobox_fast_hash (&layout, sizeof (layout),
+ rspamd_hash_seed ());
}
- return rspamd_cryptobox_fast_hash_final (&st);
+ return k;
}
gboolean
@@ -1753,7 +1832,15 @@ rspamd_inet_address_equal (gconstpointer a, gconstpointer b)
{
const rspamd_inet_addr_t *a1 = a, *a2 = b;
- return rspamd_inet_address_compare (a1, a2) == 0;
+ return rspamd_inet_address_compare (a1, a2, FALSE) == 0;
+}
+
+gboolean
+rspamd_inet_address_port_equal (gconstpointer a, gconstpointer b)
+{
+ const rspamd_inet_addr_t *a1 = a, *a2 = b;
+
+ return rspamd_inet_address_compare (a1, a2, TRUE) == 0;
}
#ifndef IN6_IS_ADDR_LOOPBACK
diff --git a/src/libutil/addr.h b/src/libutil/addr.h
index 46b705a4b..bfe586ad1 100644
--- a/src/libutil/addr.h
+++ b/src/libutil/addr.h
@@ -263,7 +263,7 @@ void rspamd_inet_address_apply_mask (rspamd_inet_addr_t *addr, guint mask);
* @return
*/
gint rspamd_inet_address_compare (const rspamd_inet_addr_t *a1,
- const rspamd_inet_addr_t *a2);
+ const rspamd_inet_addr_t *a2, gboolean compare_ports);
/**
* Utility function to compare addresses by in g_ptr_array
@@ -281,15 +281,16 @@ gint rspamd_inet_address_compare_ptr (gconstpointer a1,
rspamd_inet_addr_t *rspamd_inet_address_copy (const rspamd_inet_addr_t *addr);
/**
- * Returns hash for inet address
+ * Returns hash for inet address (ignoring port)
*/
guint rspamd_inet_address_hash (gconstpointer a);
-
+guint rspamd_inet_address_port_hash (gconstpointer a);
/**
* Returns true if two address are equal
*/
gboolean rspamd_inet_address_equal (gconstpointer a, gconstpointer b);
+gboolean rspamd_inet_address_port_equal (gconstpointer a, gconstpointer b);
/**
* Returns TRUE if an address belongs to some local address
diff --git a/src/libutil/http.h b/src/libutil/http.h
deleted file mode 100644
index df6f99756..000000000
--- a/src/libutil/http.h
+++ /dev/null
@@ -1,578 +0,0 @@
-/*-
- * Copyright 2016 Vsevolod Stakhov
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef HTTP_H_
-#define HTTP_H_
-
-/**
- * @file http.h
- *
- * This is an interface for HTTP client and conn. This code uses HTTP parser written
- * by Joyent Inc based on nginx code.
- */
-
-#include "config.h"
-#include "http_parser.h"
-#include "keypair.h"
-#include "keypairs_cache.h"
-#include "fstring.h"
-#include "ref.h"
-
-enum rspamd_http_connection_type {
- RSPAMD_HTTP_SERVER,
- RSPAMD_HTTP_CLIENT
-};
-
-struct rspamd_http_header;
-struct rspamd_http_message;
-struct rspamd_http_connection_private;
-struct rspamd_http_connection;
-struct rspamd_http_connection_router;
-struct rspamd_http_connection_entry;
-
-struct rspamd_storage_shmem {
- gchar *shm_name;
- ref_entry_t ref;
-};
-
-/**
- * Legacy spamc protocol
- */
-#define RSPAMD_HTTP_FLAG_SPAMC (1 << 0)
-/**
- * Store body of the message in a shared memory segment
- */
-#define RSPAMD_HTTP_FLAG_SHMEM (1 << 2)
-/**
- * Store body of the message in an immutable shared memory segment
- */
-#define RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE (1 << 3)
-/**
- * Use tls for this message
- */
-#define RSPAMD_HTTP_FLAG_SSL (1 << 4)
-/**
- * Body has been set for a message
- */
-#define RSPAMD_HTTP_FLAG_HAS_BODY (1 << 5)
-/**
- * Do not verify server's certificate
- */
-#define RSPAMD_HTTP_FLAG_SSL_NOVERIFY (1 << 6)
-/**
- * Options for HTTP connection
- */
-enum rspamd_http_options {
- RSPAMD_HTTP_BODY_PARTIAL = 0x1, /**< Call body handler on all body data portions *///!< RSPAMD_HTTP_BODY_PARTIAL
- RSPAMD_HTTP_CLIENT_SIMPLE = 0x1u << 1, /**< Read HTTP client reply automatically */ //!< RSPAMD_HTTP_CLIENT_SIMPLE
- RSPAMD_HTTP_CLIENT_ENCRYPTED = 0x1u << 2, /**< Encrypt data for client */ //!< RSPAMD_HTTP_CLIENT_ENCRYPTED
- RSPAMD_HTTP_CLIENT_SHARED = 0x1u << 3, /**< Store reply in shared memory */ //!< RSPAMD_HTTP_CLIENT_SHARED
- RSPAMD_HTTP_REQUIRE_ENCRYPTION = 0x1u << 4
-};
-
-typedef int (*rspamd_http_body_handler_t) (struct rspamd_http_connection *conn,
- struct rspamd_http_message *msg,
- const gchar *chunk,
- gsize len);
-
-typedef void (*rspamd_http_error_handler_t) (struct rspamd_http_connection *conn,
- GError *err);
-
-typedef int (*rspamd_http_finish_handler_t) (struct rspamd_http_connection *conn,
- struct rspamd_http_message *msg);
-
-typedef int (*rspamd_http_router_handler_t) (struct rspamd_http_connection_entry
- *conn_ent,
- struct rspamd_http_message *msg);
-typedef void (*rspamd_http_router_error_handler_t) (struct
- rspamd_http_connection_entry *conn_ent,
- GError *err);
-typedef void (*rspamd_http_router_finish_handler_t) (struct
- rspamd_http_connection_entry *conn_ent);
-
-/**
- * HTTP connection structure
- */
-struct rspamd_http_connection {
- struct rspamd_http_connection_private *priv;
- rspamd_http_body_handler_t body_handler;
- rspamd_http_error_handler_t error_handler;
- rspamd_http_finish_handler_t finish_handler;
- struct rspamd_keypair_cache *cache;
- gpointer ud;
- gsize max_size;
- unsigned opts;
- enum rspamd_http_connection_type type;
- gboolean finished;
- gint fd;
- gint ref;
-};
-
-struct rspamd_http_connection_entry {
- struct rspamd_http_connection_router *rt;
- struct rspamd_http_connection *conn;
- gpointer ud;
- gboolean is_reply;
- gboolean support_gzip;
- struct rspamd_http_connection_entry *prev, *next;
-};
-
-struct rspamd_http_connection_router {
- struct rspamd_http_connection_entry *conns;
- GHashTable *paths;
- GHashTable *response_headers;
- GPtrArray *regexps;
- struct timeval tv;
- struct timeval *ptv;
- struct event_base *ev_base;
- struct rspamd_keypair_cache *cache;
- gchar *default_fs_path;
- rspamd_http_router_handler_t unknown_method_handler;
- struct rspamd_cryptobox_keypair *key;
- rspamd_http_router_error_handler_t error_handler;
- rspamd_http_router_finish_handler_t finish_handler;
-};
-
-/**
- * Create new http connection
- * @param handler_t handler_t for body
- * @param opts options
- * @return new connection structure
- */
-struct rspamd_http_connection *rspamd_http_connection_new (
- rspamd_http_body_handler_t body_handler,
- rspamd_http_error_handler_t error_handler,
- rspamd_http_finish_handler_t finish_handler,
- unsigned opts,
- enum rspamd_http_connection_type type,
- struct rspamd_keypair_cache *cache,
- gpointer ssl_ctx);
-
-
-/**
- * Set key pointed by an opaque pointer
- * @param conn connection structure
- * @param key opaque key structure
- */
-void rspamd_http_connection_set_key (struct rspamd_http_connection *conn,
- struct rspamd_cryptobox_keypair *key);
-
-/**
- * Get peer's public key
- * @param conn connection structure
- * @return pubkey structure or NULL
- */
-const struct rspamd_cryptobox_pubkey* rspamd_http_connection_get_peer_key (
- struct rspamd_http_connection *conn);
-
-/**
- * Returns TRUE if a connection is encrypted
- * @param conn
- * @return
- */
-gboolean rspamd_http_connection_is_encrypted (struct rspamd_http_connection *conn);
-
-/**
- * Handle a request using socket fd and user data ud
- * @param conn connection structure
- * @param ud opaque user data
- * @param fd fd to read/write
- */
-void rspamd_http_connection_read_message (
- struct rspamd_http_connection *conn,
- gpointer ud,
- gint fd,
- struct timeval *timeout,
- struct event_base *base);
-
-void rspamd_http_connection_read_message_shared (
- struct rspamd_http_connection *conn,
- gpointer ud,
- gint fd,
- struct timeval *timeout,
- struct event_base *base);
-
-/**
- * Send reply using initialised connection
- * @param conn connection structure
- * @param msg HTTP message
- * @param ud opaque user data
- * @param fd fd to read/write
- */
-void rspamd_http_connection_write_message (
- struct rspamd_http_connection *conn,
- struct rspamd_http_message *msg,
- const gchar *host,
- const gchar *mime_type,
- gpointer ud,
- gint fd,
- struct timeval *timeout,
- struct event_base *base);
-
-void rspamd_http_connection_write_message_shared (
- struct rspamd_http_connection *conn,
- struct rspamd_http_message *msg,
- const gchar *host,
- const gchar *mime_type,
- gpointer ud,
- gint fd,
- struct timeval *timeout,
- struct event_base *base);
-
-/**
- * Free connection structure
- * @param conn
- */
-void rspamd_http_connection_free (struct rspamd_http_connection *conn);
-
-/**
- * Increase refcount for a connection
- * @param conn
- * @return
- */
-static inline struct rspamd_http_connection *
-rspamd_http_connection_ref (struct rspamd_http_connection *conn)
-{
- conn->ref++;
- return conn;
-}
-
-/**
- * Decrease a refcount for a connection and free it if refcount is equal to zero
- * @param conn
- */
-static void
-rspamd_http_connection_unref (struct rspamd_http_connection *conn)
-{
- if (--conn->ref <= 0) {
- rspamd_http_connection_free (conn);
- }
-}
-
-/**
- * Reset connection for a new request
- * @param conn
- */
-void rspamd_http_connection_reset (struct rspamd_http_connection *conn);
-
-/**
- * Extract the current message from a connection to deal with separately
- * @param conn
- * @return
- */
-struct rspamd_http_message * rspamd_http_connection_steal_msg (
- struct rspamd_http_connection *conn);
-
-/**
- * Copy the current message from a connection to deal with separately
- * @param conn
- * @return
- */
-struct rspamd_http_message * rspamd_http_connection_copy_msg (
- struct rspamd_http_message *msg, GError **err);
-
-/**
- * Create new HTTP message
- * @param type request or response
- * @return new http message
- */
-struct rspamd_http_message * rspamd_http_new_message (enum http_parser_type type);
-
-/**
- * Increase refcount number for an HTTP message
- * @param msg message to use
- * @return
- */
-struct rspamd_http_message * rspamd_http_message_ref (struct rspamd_http_message *msg);
-/**
- * Decrease number of refcounts for http message
- * @param msg
- */
-void rspamd_http_message_unref (struct rspamd_http_message *msg);
-
-/**
- * Sets a key for peer
- * @param msg
- * @param pk
- */
-void rspamd_http_message_set_peer_key (struct rspamd_http_message *msg,
- struct rspamd_cryptobox_pubkey *pk);
-/**
- * Create HTTP message from URL
- * @param url
- * @return new message or NULL
- */
-struct rspamd_http_message* rspamd_http_message_from_url (const gchar *url);
-
-/**
- * Returns body for a message
- * @param msg
- * @param blen pointer where to save body length
- * @return pointer to body start
- */
-const gchar *rspamd_http_message_get_body (struct rspamd_http_message *msg,
- gsize *blen);
-
-/**
- * Set message's body from the string
- * @param msg
- * @param data
- * @param len
- * @return TRUE if a message's body has been set
- */
-gboolean rspamd_http_message_set_body (struct rspamd_http_message *msg,
- const gchar *data, gsize len);
-
-/**
- * Set message's method by name
- * @param msg
- * @param method
- */
-void rspamd_http_message_set_method (struct rspamd_http_message *msg,
- const gchar *method);
-/**
- * Maps fd as message's body
- * @param msg
- * @param fd
- * @return TRUE if a message's body has been set
- */
-gboolean rspamd_http_message_set_body_from_fd (struct rspamd_http_message *msg,
- gint fd);
-
-/**
- * Uses rspamd_fstring_t as message's body, string is consumed by this operation
- * @param msg
- * @param fstr
- * @return TRUE if a message's body has been set
- */
-gboolean rspamd_http_message_set_body_from_fstring_steal (struct rspamd_http_message *msg,
- rspamd_fstring_t *fstr);
-
-/**
- * Uses rspamd_fstring_t as message's body, string is copied by this operation
- * @param msg
- * @param fstr
- * @return TRUE if a message's body has been set
- */
-gboolean rspamd_http_message_set_body_from_fstring_copy (struct rspamd_http_message *msg,
- const rspamd_fstring_t *fstr);
-
-/**
- * Appends data to message's body
- * @param msg
- * @param data
- * @param len
- * @return TRUE if a message's body has been set
- */
-gboolean rspamd_http_message_append_body (struct rspamd_http_message *msg,
- const gchar *data, gsize len);
-
-/**
- * Append a header to http message
- * @param rep
- * @param name
- * @param value
- */
-void rspamd_http_message_add_header (struct rspamd_http_message *msg,
- const gchar *name,
- const gchar *value);
-
-void rspamd_http_message_add_header_len (struct rspamd_http_message *msg,
- const gchar *name,
- const gchar *value,
- gsize len);
-
-void rspamd_http_message_add_header_fstr (struct rspamd_http_message *msg,
- const gchar *name,
- rspamd_fstring_t *value);
-
-/**
- * Search for a specified header in message
- * @param msg message
- * @param name name of header
- */
-const rspamd_ftok_t * rspamd_http_message_find_header (
- struct rspamd_http_message *msg,
- const gchar *name);
-
-/**
- * Search for a header that has multiple values
- * @param msg
- * @param name
- * @return list of rspamd_ftok_t * with values
- */
-GPtrArray* rspamd_http_message_find_header_multiple (
- struct rspamd_http_message *msg,
- const gchar *name);
-
-/**
- * Remove specific header from a message
- * @param msg
- * @param name
- * @return
- */
-gboolean rspamd_http_message_remove_header (struct rspamd_http_message *msg,
- const gchar *name);
-
-/**
- * Free HTTP message
- * @param msg
- */
-void rspamd_http_message_free (struct rspamd_http_message *msg);
-
-/**
- * Sets global maximum size for HTTP message being processed
- * @param sz
- */
-void rspamd_http_connection_set_max_size (struct rspamd_http_connection *conn,
- gsize sz);
-
-void rspamd_http_connection_disable_encryption (struct rspamd_http_connection *conn);
-
-/**
- * Increase refcount for shared file (if any) to prevent early memory unlinking
- * @param msg
- */
-struct rspamd_storage_shmem* rspamd_http_message_shmem_ref (struct rspamd_http_message *msg);
-/**
- * Decrease external ref for shmem segment associated with a message
- * @param msg
- */
-void rspamd_http_message_shmem_unref (struct rspamd_storage_shmem *p);
-
-/**
- * Returns message's flags
- * @param msg
- * @return
- */
-guint rspamd_http_message_get_flags (struct rspamd_http_message *msg);
-
-/**
- * Parse HTTP date header and return it as time_t
- * @param header HTTP date header
- * @param len length of header
- * @return time_t or (time_t)-1 in case of error
- */
-time_t rspamd_http_parse_date (const gchar *header, gsize len);
-
-/**
- * Create new http connection router and the associated HTTP connection
- * @param eh error handler callback
- * @param fh finish handler callback
- * @param default_fs_path if not NULL try to serve static files from
- * the specified directory
- * @return
- */
-struct rspamd_http_connection_router * rspamd_http_router_new (
- rspamd_http_router_error_handler_t eh,
- rspamd_http_router_finish_handler_t fh,
- struct timeval *timeout,
- struct event_base *base,
- const char *default_fs_path,
- struct rspamd_keypair_cache *cache);
-
-/**
- * Set encryption key for the HTTP router
- * @param router router structure
- * @param key opaque key structure
- */
-void rspamd_http_router_set_key (struct rspamd_http_connection_router *router,
- struct rspamd_cryptobox_keypair *key);
-
-/**
- * Add new path to the router
- */
-void rspamd_http_router_add_path (struct rspamd_http_connection_router *router,
- const gchar *path, rspamd_http_router_handler_t handler);
-
-/**
- * Add custom header to append to router replies
- * @param router
- * @param name
- * @param value
- */
-void rspamd_http_router_add_header (struct rspamd_http_connection_router *router,
- const gchar *name, const gchar *value);
-
-/**
- * Sets method to handle unknown request methods
- * @param router
- * @param handler
- */
-void rspamd_http_router_set_unknown_handler (struct rspamd_http_connection_router *router,
- rspamd_http_router_handler_t handler);
-
-/**
- * Inserts router headers to the outbound message
- * @param router
- * @param msg
- */
-void rspamd_http_router_insert_headers (struct rspamd_http_connection_router *router,
- struct rspamd_http_message *msg);
-
-struct rspamd_regexp_s;
-/**
- * Adds new pattern to router, regexp object is refcounted by this function
- * @param router
- * @param re
- * @param handler
- */
-void rspamd_http_router_add_regexp (struct rspamd_http_connection_router *router,
- struct rspamd_regexp_s *re, rspamd_http_router_handler_t handler);
-/**
- * Handle new accepted socket
- * @param router router object
- * @param fd server socket
- * @param ud opaque userdata
- */
-void rspamd_http_router_handle_socket (
- struct rspamd_http_connection_router *router,
- gint fd,
- gpointer ud);
-
-/**
- * Free router and all connections associated
- * @param router
- */
-void rspamd_http_router_free (struct rspamd_http_connection_router *router);
-
-/**
- * Extract arguments from a message's URI contained inside query string decoding
- * them if needed
- * @param msg HTTP request message
- * @return new GHashTable which maps rspamd_ftok_t* to rspamd_ftok_t*
- * (table must be freed by a caller)
- */
-GHashTable* rspamd_http_message_parse_query (struct rspamd_http_message *msg);
-
-/**
- * Prints HTTP date from `time` to `buf` using standard HTTP date format
- * @param buf date buffer
- * @param len length of buffer
- * @param time time in unix seconds
- * @return number of bytes written
- */
-glong rspamd_http_date_format (gchar *buf, gsize len, time_t time);
-
-/**
- * Normalize HTTP path removing dot sequences and repeating '/' symbols as
- * per rfc3986#section-5.2
- * @param path
- * @param len
- * @param nlen
- */
-void rspamd_http_normalize_path_inplace (gchar *path, guint len, guint *nlen);
-
-#endif /* HTTP_H_ */
diff --git a/src/libutil/http.c b/src/libutil/http_connection.c
index a82fc24f7..9e43e21e2 100644
--- a/src/libutil/http.c
+++ b/src/libutil/http_connection.c
@@ -14,8 +14,9 @@
* limitations under the License.
*/
#include "config.h"
-#include "../../contrib/mumhash/mum.h"
+#include "http_connection.h"
#include "http_private.h"
+#include "http_message.h"
#include "utlist.h"
#include "util.h"
#include "printf.h"
@@ -24,11 +25,13 @@
#include "ottery.h"
#include "keypair_private.h"
#include "cryptobox.h"
-#include "unix-std.h"
#include "libutil/ssl_util.h"
-#include "libutil/regexp.h"
#include "libserver/url.h"
+#include "contrib/mumhash/mum.h"
+#include "contrib/http-parser/http_parser.h"
+#include "unix-std.h"
+
#include <openssl/err.h>
#define ENCRYPTED_VERSION " HTTP/1.0"
@@ -52,9 +55,10 @@ enum rspamd_http_priv_flags {
#define IS_CONN_RESETED(c) ((c)->flags & RSPAMD_HTTP_CONN_FLAG_RESETED)
struct rspamd_http_connection_private {
- gpointer ssl_ctx;
+ struct rspamd_http_context *ctx;
struct rspamd_ssl_connection *ssl;
struct _rspamd_http_privbuf *buf;
+ struct rspamd_keypair_cache *cache;
struct rspamd_cryptobox_pubkey *peer_key;
struct rspamd_cryptobox_keypair *local_key;
struct rspamd_http_header *header;
@@ -71,30 +75,6 @@ struct rspamd_http_connection_private {
gsize wr_total;
};
-enum http_magic_type {
- HTTP_MAGIC_PLAIN = 0,
- HTTP_MAGIC_HTML,
- HTTP_MAGIC_CSS,
- HTTP_MAGIC_JS,
- HTTP_MAGIC_PNG,
- HTTP_MAGIC_JPG
-};
-
-static const struct _rspamd_http_magic {
- const gchar *ext;
- const gchar *ct;
-} http_file_types[] = {
- [HTTP_MAGIC_PLAIN] = { "txt", "text/plain" },
- [HTTP_MAGIC_HTML] = { "html", "text/html" },
- [HTTP_MAGIC_CSS] = { "css", "text/css" },
- [HTTP_MAGIC_JS] = { "js", "application/javascript" },
- [HTTP_MAGIC_PNG] = { "png", "image/png" },
- [HTTP_MAGIC_JPG] = { "jpg", "image/jpeg" },
-};
-
-static const gchar *http_week[] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" };
-static const gchar *http_month[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun",
- "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" };
static const rspamd_ftok_t key_header = {
.begin = "Key",
.len = 3
@@ -108,9 +88,7 @@ static const rspamd_ftok_t last_modified_header = {
.len = 13
};
-static void rspamd_http_message_storage_cleanup (struct rspamd_http_message *msg);
-static gboolean rspamd_http_message_grow_body (struct rspamd_http_message *msg,
- gsize len);
+
#define HTTP_ERROR http_error_quark ()
GQuark
@@ -156,272 +134,6 @@ rspamd_http_code_to_str (gint code)
return "Unknown error";
}
-/*
- * Obtained from nginx
- * Copyright (C) Igor Sysoev
- */
-static guint mday[] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 };
-
-time_t
-rspamd_http_parse_date (const gchar *header, gsize len)
-{
- const gchar *p, *end;
- gint month;
- guint day, year, hour, min, sec;
- guint64 time;
- enum {
- no = 0, rfc822, /* Tue, 10 Nov 2002 23:50:13 */
- rfc850, /* Tuesday, 10-Dec-02 23:50:13 */
- isoc /* Tue Dec 10 23:50:13 2002 */
- } fmt;
-
- fmt = 0;
- if (len > 0) {
- end = header + len;
- }
- else {
- end = header + strlen (header);
- }
-
- day = 32;
- year = 2038;
-
- for (p = header; p < end; p++) {
- if (*p == ',') {
- break;
- }
-
- if (*p == ' ') {
- fmt = isoc;
- break;
- }
- }
-
- for (p++; p < end; p++)
- if (*p != ' ') {
- break;
- }
-
- if (end - p < 18) {
- return (time_t)-1;
- }
-
- if (fmt != isoc) {
- if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') {
- return (time_t)-1;
- }
-
- day = (*p - '0') * 10 + *(p + 1) - '0';
- p += 2;
-
- if (*p == ' ') {
- if (end - p < 18) {
- return (time_t)-1;
- }
- fmt = rfc822;
-
- }
- else if (*p == '-') {
- fmt = rfc850;
-
- }
- else {
- return (time_t)-1;
- }
-
- p++;
- }
-
- switch (*p) {
-
- case 'J':
- month = *(p + 1) == 'a' ? 0 : *(p + 2) == 'n' ? 5 : 6;
- break;
-
- case 'F':
- month = 1;
- break;
-
- case 'M':
- month = *(p + 2) == 'r' ? 2 : 4;
- break;
-
- case 'A':
- month = *(p + 1) == 'p' ? 3 : 7;
- break;
-
- case 'S':
- month = 8;
- break;
-
- case 'O':
- month = 9;
- break;
-
- case 'N':
- month = 10;
- break;
-
- case 'D':
- month = 11;
- break;
-
- default:
- return (time_t)-1;
- }
-
- p += 3;
-
- if ((fmt == rfc822 && *p != ' ') || (fmt == rfc850 && *p != '-')) {
- return (time_t)-1;
- }
-
- p++;
-
- if (fmt == rfc822) {
- if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9'
- || *(p + 2) < '0' || *(p + 2) > '9' || *(p + 3) < '0'
- || *(p + 3) > '9') {
- return (time_t)-1;
- }
-
- year = (*p - '0') * 1000 + (*(p + 1) - '0') * 100
- + (*(p + 2) - '0') * 10 + *(p + 3) - '0';
- p += 4;
-
- }
- else if (fmt == rfc850) {
- if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') {
- return (time_t)-1;
- }
-
- year = (*p - '0') * 10 + *(p + 1) - '0';
- year += (year < 70) ? 2000 : 1900;
- p += 2;
- }
-
- if (fmt == isoc) {
- if (*p == ' ') {
- p++;
- }
-
- if (*p < '0' || *p > '9') {
- return (time_t)-1;
- }
-
- day = *p++ - '0';
-
- if (*p != ' ') {
- if (*p < '0' || *p > '9') {
- return (time_t)-1;
- }
-
- day = day * 10 + *p++ - '0';
- }
-
- if (end - p < 14) {
- return (time_t)-1;
- }
- }
-
- if (*p++ != ' ') {
- return (time_t)-1;
- }
-
- if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') {
- return (time_t)-1;
- }
-
- hour = (*p - '0') * 10 + *(p + 1) - '0';
- p += 2;
-
- if (*p++ != ':') {
- return (time_t)-1;
- }
-
- if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') {
- return (time_t)-1;
- }
-
- min = (*p - '0') * 10 + *(p + 1) - '0';
- p += 2;
-
- if (*p++ != ':') {
- return (time_t)-1;
- }
-
- if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') {
- return (time_t)-1;
- }
-
- sec = (*p - '0') * 10 + *(p + 1) - '0';
-
- if (fmt == isoc) {
- p += 2;
-
- if (*p++ != ' ') {
- return (time_t)-1;
- }
-
- if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9'
- || *(p + 2) < '0' || *(p + 2) > '9' || *(p + 3) < '0'
- || *(p + 3) > '9') {
- return (time_t)-1;
- }
-
- year = (*p - '0') * 1000 + (*(p + 1) - '0') * 100
- + (*(p + 2) - '0') * 10 + *(p + 3) - '0';
- }
-
- if (hour > 23 || min > 59 || sec > 59) {
- return (time_t)-1;
- }
-
- if (day == 29 && month == 1) {
- if ((year & 3) || ((year % 100 == 0) && (year % 400) != 0)) {
- return (time_t)-1;
- }
-
- }
- else if (day > mday[month]) {
- return (time_t)-1;
- }
-
- /*
- * shift new year to March 1 and start months from 1 (not 0),
- * it is needed for Gauss' formula
- */
-
- if (--month <= 0) {
- month += 12;
- year -= 1;
- }
-
- /* Gauss' formula for Gregorian days since March 1, 1 BC */
-
- time = (guint64) (
- /* days in years including leap years since March 1, 1 BC */
-
- 365 * year + year / 4 - year / 100 + year / 400
-
- /* days before the month */
-
- + 367 * month / 12 - 30
-
- /* days before the day */
-
- + day - 1
-
- /*
- * 719527 days were between March 1, 1 BC and March 1, 1970,
- * 31 and 28 days were in January and February 1970
- */
-
- - 719527 + 31 + 28) * 86400 + hour * 3600 + min * 60 + sec;
-
- return (time_t) time;
-}
-
static void
rspamd_http_parse_key (rspamd_ftok_t *data, struct rspamd_http_connection *conn,
struct rspamd_http_connection_private *priv)
@@ -453,9 +165,10 @@ rspamd_http_parse_key (rspamd_ftok_t *data, struct rspamd_http_connection *conn,
RSPAMD_KEYPAIR_SHORT_ID_LEN) == 0) {
priv->msg->peer_key = pk;
- if (conn->cache && priv->msg->peer_key) {
- rspamd_keypair_cache_process (conn->cache,
- priv->local_key, priv->msg->peer_key);
+ if (priv->cache && priv->msg->peer_key) {
+ rspamd_keypair_cache_process (priv->cache,
+ priv->local_key,
+ priv->msg->peer_key);
}
}
else {
@@ -639,7 +352,15 @@ rspamd_http_on_headers_complete (http_parser * parser)
msg->code = parser->status_code;
rspamd_http_connection_ref (conn);
ret = conn->finish_handler (conn, msg);
- conn->finished = TRUE;
+
+ if (conn->opts & RSPAMD_HTTP_CLIENT_KEEP_ALIVE) {
+ rspamd_http_context_push_keepalive (conn->priv->ctx, conn,
+ msg, conn->priv->ctx->ev_base);
+ }
+ else {
+ conn->finished = TRUE;
+ }
+
rspamd_http_connection_unref (conn);
return ret;
@@ -814,7 +535,15 @@ rspamd_http_on_headers_complete_decrypted (http_parser *parser)
msg->code = parser->status_code;
rspamd_http_connection_ref (conn);
ret = conn->finish_handler (conn, msg);
- conn->finished = TRUE;
+
+ if (conn->opts & RSPAMD_HTTP_CLIENT_KEEP_ALIVE) {
+ rspamd_http_context_push_keepalive (conn->priv->ctx, conn,
+ msg, conn->priv->ctx->ev_base);
+ }
+ else {
+ conn->finished = TRUE;
+ }
+
rspamd_http_connection_unref (conn);
return ret;
@@ -964,7 +693,15 @@ rspamd_http_on_message_complete (http_parser * parser)
rspamd_http_connection_ref (conn);
ret = conn->finish_handler (conn, priv->msg);
- conn->finished = TRUE;
+
+ if (conn->opts & RSPAMD_HTTP_CLIENT_KEEP_ALIVE) {
+ rspamd_http_context_push_keepalive (conn->priv->ctx, conn,
+ priv->msg, conn->priv->ctx->ev_base);
+ }
+ else {
+ conn->finished = TRUE;
+ }
+
rspamd_http_connection_unref (conn);
}
@@ -974,30 +711,41 @@ rspamd_http_on_message_complete (http_parser * parser)
static void
rspamd_http_simple_client_helper (struct rspamd_http_connection *conn)
{
- struct event_base *base;
struct rspamd_http_connection_private *priv;
gpointer ssl;
gint request_method;
+ rspamd_fstring_t *prev_host;
priv = conn->priv;
- base = conn->priv->ev.ev_base;
ssl = priv->ssl;
priv->ssl = NULL;
request_method = priv->msg->method;
+ /* Preserve host for keepalive */
+ prev_host = priv->msg->host;
+ priv->msg->host = NULL;
rspamd_http_connection_reset (conn);
priv->ssl = ssl;
+
/* Plan read message */
if (conn->opts & RSPAMD_HTTP_CLIENT_SHARED) {
- rspamd_http_connection_read_message_shared (conn, conn->ud, conn->fd,
- conn->priv->ptv, base);
+ rspamd_http_connection_read_message_shared (conn, conn->ud,
+ conn->priv->ptv);
}
else {
- rspamd_http_connection_read_message (conn, conn->ud, conn->fd,
- conn->priv->ptv, base);
+ rspamd_http_connection_read_message (conn, conn->ud,
+ conn->priv->ptv);
}
- priv->msg->method = request_method;
+ if (priv->msg) {
+ priv->msg->method = request_method;
+ priv->msg->host = prev_host;
+ }
+ else {
+ if (prev_host) {
+ rspamd_fstring_free (prev_host);
+ }
+ }
}
static void
@@ -1322,13 +1070,13 @@ rspamd_http_parser_reset (struct rspamd_http_connection *conn)
struct rspamd_http_connection *
rspamd_http_connection_new (
+ struct rspamd_http_context *ctx,
+ gint fd,
rspamd_http_body_handler_t body_handler,
rspamd_http_error_handler_t error_handler,
rspamd_http_finish_handler_t finish_handler,
unsigned opts,
- enum rspamd_http_connection_type type,
- struct rspamd_keypair_cache *cache,
- gpointer ssl_ctx)
+ enum rspamd_http_connection_type type)
{
struct rspamd_http_connection *conn;
struct rspamd_http_connection_private *priv;
@@ -1343,15 +1091,28 @@ rspamd_http_connection_new (
conn->body_handler = body_handler;
conn->error_handler = error_handler;
conn->finish_handler = finish_handler;
- conn->fd = -1;
+ conn->fd = fd;
conn->ref = 1;
conn->finished = FALSE;
- conn->cache = cache;
/* Init priv */
+ if (ctx == NULL) {
+ ctx = rspamd_http_context_default ();
+ }
+
priv = g_malloc0 (sizeof (struct rspamd_http_connection_private));
conn->priv = priv;
- priv->ssl_ctx = ssl_ctx;
+ priv->ctx = ctx;
+
+ if (conn->type == RSPAMD_HTTP_CLIENT) {
+ priv->cache = ctx->client_kp_cache;
+ if (ctx->client_kp) {
+ priv->local_key = rspamd_keypair_ref (ctx->client_kp);
+ }
+ }
+ else {
+ priv->cache = ctx->server_kp_cache;
+ }
rspamd_http_parser_reset (conn);
priv->parser.data = conn;
@@ -1359,6 +1120,47 @@ rspamd_http_connection_new (
return conn;
}
+struct rspamd_http_connection *
+rspamd_http_connection_new_keepalive (struct rspamd_http_context *ctx,
+ rspamd_http_body_handler_t body_handler,
+ rspamd_http_error_handler_t error_handler,
+ rspamd_http_finish_handler_t finish_handler,
+ rspamd_inet_addr_t *addr,
+ const gchar *host)
+{
+ struct rspamd_http_connection *conn;
+ gint fd;
+
+ if (error_handler == NULL || finish_handler == NULL) {
+ return NULL;
+ }
+
+ conn = rspamd_http_context_check_keepalive (ctx, addr, host);
+
+ if (conn) {
+ return conn;
+ }
+
+ fd = rspamd_inet_address_connect (addr, SOCK_STREAM, TRUE);
+
+ if (fd == -1) {
+ msg_info ("cannot connect to %s: %s", rspamd_inet_address_to_string (addr),
+ host);
+ return NULL;
+ }
+
+ conn = rspamd_http_connection_new (ctx, fd, body_handler, error_handler,
+ finish_handler,
+ RSPAMD_HTTP_CLIENT_SIMPLE|RSPAMD_HTTP_CLIENT_KEEP_ALIVE,
+ RSPAMD_HTTP_CLIENT);
+
+ if (conn) {
+ rspamd_http_context_prepare_keepalive (ctx, conn, addr, host);
+ }
+
+ return conn;
+}
+
void
rspamd_http_connection_reset (struct rspamd_http_connection *conn)
{
@@ -1573,18 +1375,22 @@ rspamd_http_connection_free (struct rspamd_http_connection *conn)
g_free (priv);
}
+ if (conn->opts & RSPAMD_HTTP_CLIENT_KEEP_ALIVE) {
+ /* Fd is owned by a connection */
+ close (conn->fd);
+ }
+
g_free (conn);
}
static void
rspamd_http_connection_read_message_common (struct rspamd_http_connection *conn,
- gpointer ud, gint fd, struct timeval *timeout, struct event_base *base,
+ gpointer ud, struct timeval *timeout,
gint flags)
{
struct rspamd_http_connection_private *priv = conn->priv;
struct rspamd_http_message *req;
- conn->fd = fd;
conn->ud = ud;
req = rspamd_http_new_message (
conn->type == RSPAMD_HTTP_SERVER ? HTTP_REQUEST : HTTP_RESPONSE);
@@ -1604,8 +1410,8 @@ rspamd_http_connection_read_message_common (struct rspamd_http_connection *conn,
if (timeout == NULL) {
priv->ptv = NULL;
}
- else if (&priv->tv != timeout) {
- memcpy (&priv->tv, timeout, sizeof (struct timeval));
+ else {
+ memmove (&priv->tv, timeout, sizeof (struct timeval));
priv->ptv = &priv->tv;
}
@@ -1616,13 +1422,12 @@ rspamd_http_connection_read_message_common (struct rspamd_http_connection *conn,
priv->flags |= RSPAMD_HTTP_CONN_FLAG_NEW_HEADER;
event_set (&priv->ev,
- fd,
+ conn->fd,
EV_READ | EV_PERSIST,
rspamd_http_event_handler,
conn);
- if (base != NULL) {
- event_base_set (base, &priv->ev);
- }
+
+ event_base_set (priv->ctx->ev_base, &priv->ev);
priv->flags &= ~RSPAMD_HTTP_CONN_FLAG_RESETED;
event_add (&priv->ev, priv->ptv);
@@ -1630,16 +1435,16 @@ rspamd_http_connection_read_message_common (struct rspamd_http_connection *conn,
void
rspamd_http_connection_read_message (struct rspamd_http_connection *conn,
- gpointer ud, gint fd, struct timeval *timeout, struct event_base *base)
+ gpointer ud, struct timeval *timeout)
{
- rspamd_http_connection_read_message_common (conn, ud, fd, timeout, base, 0);
+ rspamd_http_connection_read_message_common (conn, ud, timeout, 0);
}
void
rspamd_http_connection_read_message_shared (struct rspamd_http_connection *conn,
- gpointer ud, gint fd, struct timeval *timeout, struct event_base *base)
+ gpointer ud, struct timeval *timeout)
{
- rspamd_http_connection_read_message_common (conn, ud, fd, timeout, base,
+ rspamd_http_connection_read_message_common (conn, ud, timeout,
RSPAMD_HTTP_FLAG_SHMEM);
}
@@ -1767,18 +1572,15 @@ rspamd_http_message_write_header (const gchar* mime_type, gboolean encrypted,
{
gchar datebuf[64];
gint meth_len = 0;
- struct tm t;
+ const gchar *conn_type = "close";
if (conn->type == RSPAMD_HTTP_SERVER) {
/* Format reply */
if (msg->method < HTTP_SYMBOLS) {
rspamd_ftok_t status;
- rspamd_gmtime (msg->date, &t);
- rspamd_snprintf (datebuf, sizeof(datebuf),
- "%s, %02d %s %4d %02d:%02d:%02d GMT", http_week[t.tm_wday],
- t.tm_mday, http_month[t.tm_mon], t.tm_year + 1900,
- t.tm_hour, t.tm_min, t.tm_sec);
+ rspamd_http_date_format (datebuf, sizeof (datebuf), msg->date);
+
if (mime_type == NULL) {
mime_type =
encrypted ? "application/octet-stream" : "text/plain";
@@ -1901,6 +1703,10 @@ rspamd_http_message_write_header (const gchar* mime_type, gboolean encrypted,
}
}
else {
+ if (conn->opts & RSPAMD_HTTP_CLIENT_KEEP_ALIVE) {
+ conn_type = "keep-alive";
+ }
+
/* Format request */
enclen += msg->url->len + strlen (http_method_str (msg->method)) + 1;
@@ -1910,15 +1716,23 @@ rspamd_http_message_write_header (const gchar* mime_type, gboolean encrypted,
rspamd_printf_fstring (buf,
"%s %s HTTP/1.0\r\n"
"Content-Length: %z\r\n"
- "Content-Type: application/octet-stream\r\n",
+ "Content-Type: application/octet-stream\r\n"
+ "Connection: %s\r\n",
"POST",
- "/post", enclen);
+ "/post",
+ enclen,
+ conn_type);
}
else {
rspamd_printf_fstring (buf,
"%s %V HTTP/1.0\r\n"
- "Content-Length: %z\r\n",
- http_method_str (msg->method), msg->url, bodylen);
+ "Content-Length: %z\r\n"
+ "Connection: %s\r\n",
+ http_method_str (msg->method),
+ msg->url,
+ bodylen,
+ conn_type);
+
if (bodylen > 0) {
if (mime_type == NULL) {
mime_type = "text/plain";
@@ -1935,38 +1749,53 @@ rspamd_http_message_write_header (const gchar* mime_type, gboolean encrypted,
if (host != NULL) {
rspamd_printf_fstring (buf,
"%s %s HTTP/1.1\r\n"
- "Connection: close\r\n"
+ "Connection: %s\r\n"
"Host: %s\r\n"
"Content-Length: %z\r\n"
"Content-Type: application/octet-stream\r\n",
- "POST", "/post", host, enclen);
+ "POST",
+ "/post",
+ conn_type,
+ host,
+ enclen);
}
else {
rspamd_printf_fstring (buf,
"%s %s HTTP/1.1\r\n"
- "Connection: close\r\n"
+ "Connection: %s\r\n"
"Host: %V\r\n"
"Content-Length: %z\r\n"
"Content-Type: application/octet-stream\r\n",
- "POST", "/post", msg->host, enclen);
+ "POST",
+ "/post",
+ conn_type,
+ msg->host,
+ enclen);
}
}
else {
if (host != NULL) {
rspamd_printf_fstring (buf,
- "%s %V HTTP/1.1\r\nConnection: close\r\n"
+ "%s %V HTTP/1.1\r\n"
+ "Connection: %s\r\n"
"Host: %s\r\n"
"Content-Length: %z\r\n",
- http_method_str (msg->method), msg->url, host,
+ http_method_str (msg->method),
+ msg->url,
+ conn_type,
+ host,
bodylen);
}
else {
rspamd_printf_fstring (buf,
"%s %V HTTP/1.1\r\n"
- "Connection: close\r\n"
+ "Connection: %s\r\n"
"Host: %V\r\n"
"Content-Length: %z\r\n",
- http_method_str (msg->method), msg->url, msg->host,
+ http_method_str (msg->method),
+ msg->url,
+ conn_type,
+ msg->host,
bodylen);
}
@@ -1999,9 +1828,12 @@ rspamd_http_message_write_header (const gchar* mime_type, gboolean encrypted,
static void
rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn,
- struct rspamd_http_message *msg, const gchar *host, const gchar *mime_type,
- gpointer ud, gint fd, struct timeval *timeout, struct event_base *base,
- gboolean allow_shared)
+ struct rspamd_http_message *msg,
+ const gchar *host,
+ const gchar *mime_type,
+ gpointer ud,
+ struct timeval *timeout,
+ gboolean allow_shared)
{
struct rspamd_http_connection_private *priv = conn->priv;
struct rspamd_http_header *hdr, *htmp, *hcur;
@@ -2016,7 +1848,6 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn
enum rspamd_cryptobox_mode mode;
GError *err;
- conn->fd = fd;
conn->ud = ud;
priv->msg = msg;
@@ -2049,8 +1880,8 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn
encrypted = TRUE;
- if (conn->cache) {
- rspamd_keypair_cache_process (conn->cache,
+ if (priv->cache) {
+ rspamd_keypair_cache_process (priv->cache,
priv->local_key, priv->msg->peer_key);
}
}
@@ -2084,6 +1915,11 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn
}
}
+ if (priv->ctx->config.user_agent) {
+ rspamd_http_message_add_header (msg, "User-Agent",
+ priv->ctx->config.user_agent);
+ }
+
if (encrypted) {
mode = rspamd_keypair_alg (priv->local_key);
@@ -2314,10 +2150,12 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn
}
if (msg->flags & RSPAMD_HTTP_FLAG_SSL) {
- if (base != NULL) {
- event_base_set (base, &priv->ev);
- }
- if (!priv->ssl_ctx) {
+ gpointer ssl_ctx = (msg->flags & RSPAMD_HTTP_FLAG_SSL_NOVERIFY) ?
+ priv->ctx->ssl_ctx_noverify : priv->ctx->ssl_ctx;
+
+ event_base_set (priv->ctx->ev_base, &priv->ev);
+
+ if (!ssl_ctx) {
err = g_error_new (HTTP_ERROR, errno, "ssl message requested "
"with no ssl ctx");
rspamd_http_connection_ref (conn);
@@ -2332,11 +2170,11 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn
rspamd_ssl_connection_free (priv->ssl);
}
- priv->ssl = rspamd_ssl_connection_new (priv->ssl_ctx, base,
+ priv->ssl = rspamd_ssl_connection_new (ssl_ctx, priv->ctx->ev_base,
!(msg->flags & RSPAMD_HTTP_FLAG_SSL_NOVERIFY));
g_assert (priv->ssl != NULL);
- if (!rspamd_ssl_connect_fd (priv->ssl, fd, host, &priv->ev,
+ if (!rspamd_ssl_connect_fd (priv->ssl, conn->fd, host, &priv->ev,
priv->ptv, rspamd_http_event_handler,
rspamd_http_ssl_err_handler, conn)) {
@@ -2353,11 +2191,8 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn
}
}
else {
- event_set (&priv->ev, fd, EV_WRITE, rspamd_http_event_handler, conn);
-
- if (base != NULL) {
- event_base_set (base, &priv->ev);
- }
+ event_set (&priv->ev, conn->fd, EV_WRITE, rspamd_http_event_handler, conn);
+ event_base_set (priv->ctx->ev_base, &priv->ev);
event_add (&priv->ev, priv->ptv);
}
@@ -2365,465 +2200,28 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn
void
rspamd_http_connection_write_message (struct rspamd_http_connection *conn,
- struct rspamd_http_message *msg, const gchar *host, const gchar *mime_type,
- gpointer ud, gint fd, struct timeval *timeout, struct event_base *base)
+ struct rspamd_http_message *msg,
+ const gchar *host,
+ const gchar *mime_type,
+ gpointer ud,
+ struct timeval *timeout)
{
rspamd_http_connection_write_message_common (conn, msg, host, mime_type,
- ud, fd, timeout, base, FALSE);
+ ud, timeout, FALSE);
}
void
rspamd_http_connection_write_message_shared (struct rspamd_http_connection *conn,
- struct rspamd_http_message *msg, const gchar *host, const gchar *mime_type,
- gpointer ud, gint fd, struct timeval *timeout, struct event_base *base)
+ struct rspamd_http_message *msg,
+ const gchar *host,
+ const gchar *mime_type,
+ gpointer ud,
+ struct timeval *timeout)
{
rspamd_http_connection_write_message_common (conn, msg, host, mime_type,
- ud, fd, timeout, base, TRUE);
-}
-
-struct rspamd_http_message *
-rspamd_http_new_message (enum http_parser_type type)
-{
- struct rspamd_http_message *new;
-
- new = g_malloc0 (sizeof (struct rspamd_http_message));
-
- if (type == HTTP_REQUEST) {
- new->url = rspamd_fstring_new ();
- }
- else {
- new->url = NULL;
- new->code = 200;
- }
-
- new->port = 80;
- new->type = type;
- new->method = HTTP_INVALID;
-
- REF_INIT_RETAIN (new, rspamd_http_message_free);
-
- return new;
-}
-
-struct rspamd_http_message*
-rspamd_http_message_from_url (const gchar *url)
-{
- struct http_parser_url pu;
- struct rspamd_http_message *msg;
- const gchar *host, *path;
- size_t pathlen, urllen;
- guint flags = 0;
-
- if (url == NULL) {
- return NULL;
- }
-
- urllen = strlen (url);
- memset (&pu, 0, sizeof (pu));
-
- if (http_parser_parse_url (url, urllen, FALSE, &pu) != 0) {
- msg_warn ("cannot parse URL: %s", url);
- return NULL;
- }
-
- if ((pu.field_set & (1 << UF_HOST)) == 0) {
- msg_warn ("no host argument in URL: %s", url);
- return NULL;
- }
-
- if ((pu.field_set & (1 << UF_SCHEMA))) {
- if (pu.field_data[UF_SCHEMA].len == sizeof ("https") - 1 &&
- memcmp (url + pu.field_data[UF_SCHEMA].off, "https", 5) == 0) {
- flags |= RSPAMD_HTTP_FLAG_SSL;
- }
- }
-
- if ((pu.field_set & (1 << UF_PATH)) == 0) {
- path = "/";
- pathlen = 1;
- }
- else {
- path = url + pu.field_data[UF_PATH].off;
- pathlen = urllen - pu.field_data[UF_PATH].off;
- }
-
- msg = rspamd_http_new_message (HTTP_REQUEST);
- host = url + pu.field_data[UF_HOST].off;
- msg->flags = flags;
-
- if ((pu.field_set & (1 << UF_PORT)) != 0) {
- msg->port = pu.port;
- }
- else {
- /* XXX: magic constant */
- if (flags & RSPAMD_HTTP_FLAG_SSL) {
- msg->port = 443;
- }
- else {
- msg->port = 80;
- }
- }
-
- msg->host = rspamd_fstring_new_init (host, pu.field_data[UF_HOST].len);
- msg->url = rspamd_fstring_append (msg->url, path, pathlen);
-
- REF_INIT_RETAIN (msg, rspamd_http_message_free);
-
- return msg;
+ ud, timeout, TRUE);
}
-const gchar *
-rspamd_http_message_get_body (struct rspamd_http_message *msg,
- gsize *blen)
-{
- const gchar *ret = NULL;
-
- if (msg->body_buf.len > 0) {
- ret = msg->body_buf.begin;
- }
-
- if (blen) {
- *blen = msg->body_buf.len;
- }
-
- return ret;
-}
-
-static void
-rspamd_http_shname_dtor (void *p)
-{
- struct rspamd_storage_shmem *n = p;
-
-#ifdef HAVE_SANE_SHMEM
- shm_unlink (n->shm_name);
-#else
- unlink (n->shm_name);
-#endif
- g_free (n->shm_name);
- g_free (n);
-}
-
-struct rspamd_storage_shmem *
-rspamd_http_message_shmem_ref (struct rspamd_http_message *msg)
-{
- if ((msg->flags & RSPAMD_HTTP_FLAG_SHMEM) && msg->body_buf.c.shared.name) {
- REF_RETAIN (msg->body_buf.c.shared.name);
- return msg->body_buf.c.shared.name;
- }
-
- return NULL;
-}
-
-guint
-rspamd_http_message_get_flags (struct rspamd_http_message *msg)
-{
- return msg->flags;
-}
-
-void
-rspamd_http_message_shmem_unref (struct rspamd_storage_shmem *p)
-{
- REF_RELEASE (p);
-}
-
-gboolean
-rspamd_http_message_set_body (struct rspamd_http_message *msg,
- const gchar *data, gsize len)
-{
- union _rspamd_storage_u *storage;
- storage = &msg->body_buf.c;
-
- rspamd_http_message_storage_cleanup (msg);
-
- if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) {
- storage->shared.name = g_malloc (sizeof (*storage->shared.name));
- REF_INIT_RETAIN (storage->shared.name, rspamd_http_shname_dtor);
-#ifdef HAVE_SANE_SHMEM
-#if defined(__DragonFly__)
- // DragonFly uses regular files for shm. User rspamd is not allowed to create
- // files in the root.
- storage->shared.name->shm_name = g_strdup ("/tmp/rhm.XXXXXXXXXXXXXXXXXXXX");
-#else
- storage->shared.name->shm_name = g_strdup ("/rhm.XXXXXXXXXXXXXXXXXXXX");
-#endif
- storage->shared.shm_fd = rspamd_shmem_mkstemp (storage->shared.name->shm_name);
-#else
- /* XXX: assume that tempdir is /tmp */
- storage->shared.name->shm_name = g_strdup ("/tmp/rhm.XXXXXXXXXXXXXXXXXXXX");
- storage->shared.shm_fd = mkstemp (storage->shared.name->shm_name);
-#endif
-
- if (storage->shared.shm_fd == -1) {
- return FALSE;
- }
-
- if (len != 0 && len != ULLONG_MAX) {
- if (ftruncate (storage->shared.shm_fd, len) == -1) {
- return FALSE;
- }
-
- msg->body_buf.str = mmap (NULL, len,
- PROT_WRITE|PROT_READ, MAP_SHARED,
- storage->shared.shm_fd, 0);
-
- if (msg->body_buf.str == MAP_FAILED) {
- return FALSE;
- }
-
- msg->body_buf.begin = msg->body_buf.str;
- msg->body_buf.allocated_len = len;
-
- if (data != NULL) {
- memcpy (msg->body_buf.str, data, len);
- msg->body_buf.len = len;
- }
- }
- else {
- msg->body_buf.len = 0;
- msg->body_buf.begin = NULL;
- msg->body_buf.str = NULL;
- msg->body_buf.allocated_len = 0;
- }
- }
- else {
- if (len != 0 && len != ULLONG_MAX) {
- if (data == NULL) {
- storage->normal = rspamd_fstring_sized_new (len);
- msg->body_buf.len = 0;
- }
- else {
- storage->normal = rspamd_fstring_new_init (data, len);
- msg->body_buf.len = len;
- }
- }
- else {
- storage->normal = rspamd_fstring_new ();
- }
-
- msg->body_buf.begin = storage->normal->str;
- msg->body_buf.str = storage->normal->str;
- msg->body_buf.allocated_len = storage->normal->allocated;
- }
-
- msg->flags |= RSPAMD_HTTP_FLAG_HAS_BODY;
-
- return TRUE;
-}
-
-void
-rspamd_http_message_set_method (struct rspamd_http_message *msg,
- const gchar *method)
-{
- gint i;
-
- /* Linear search: not very efficient method */
- for (i = 0; i < HTTP_METHOD_MAX; i ++) {
- if (g_ascii_strcasecmp (method, http_method_str (i)) == 0) {
- msg->method = i;
- }
- }
-}
-
-gboolean
-rspamd_http_message_set_body_from_fd (struct rspamd_http_message *msg,
- gint fd)
-{
- union _rspamd_storage_u *storage;
- struct stat st;
-
- rspamd_http_message_storage_cleanup (msg);
-
- storage = &msg->body_buf.c;
- msg->flags |= RSPAMD_HTTP_FLAG_SHMEM|RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE;
-
- storage->shared.shm_fd = dup (fd);
- msg->body_buf.str = MAP_FAILED;
-
- if (storage->shared.shm_fd == -1) {
- return FALSE;
- }
-
- if (fstat (storage->shared.shm_fd, &st) == -1) {
- return FALSE;
- }
-
- msg->body_buf.str = mmap (NULL, st.st_size,
- PROT_READ, MAP_SHARED,
- storage->shared.shm_fd, 0);
-
- if (msg->body_buf.str == MAP_FAILED) {
- return FALSE;
- }
-
- msg->body_buf.begin = msg->body_buf.str;
- msg->body_buf.len = st.st_size;
- msg->body_buf.allocated_len = st.st_size;
-
- return TRUE;
-}
-
-gboolean
-rspamd_http_message_set_body_from_fstring_steal (struct rspamd_http_message *msg,
- rspamd_fstring_t *fstr)
-{
- union _rspamd_storage_u *storage;
-
- rspamd_http_message_storage_cleanup (msg);
-
- storage = &msg->body_buf.c;
- msg->flags &= ~(RSPAMD_HTTP_FLAG_SHMEM|RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE);
-
- storage->normal = fstr;
- msg->body_buf.str = fstr->str;
- msg->body_buf.begin = msg->body_buf.str;
- msg->body_buf.len = fstr->len;
- msg->body_buf.allocated_len = fstr->allocated;
-
- return TRUE;
-}
-
-gboolean
-rspamd_http_message_set_body_from_fstring_copy (struct rspamd_http_message *msg,
- const rspamd_fstring_t *fstr)
-{
- union _rspamd_storage_u *storage;
-
- rspamd_http_message_storage_cleanup (msg);
-
- storage = &msg->body_buf.c;
- msg->flags &= ~(RSPAMD_HTTP_FLAG_SHMEM|RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE);
-
- storage->normal = rspamd_fstring_new_init (fstr->str, fstr->len);
- msg->body_buf.str = storage->normal->str;
- msg->body_buf.begin = msg->body_buf.str;
- msg->body_buf.len = storage->normal->len;
- msg->body_buf.allocated_len = storage->normal->allocated;
-
- return TRUE;
-}
-
-
-static gboolean
-rspamd_http_message_grow_body (struct rspamd_http_message *msg, gsize len)
-{
- struct stat st;
- union _rspamd_storage_u *storage;
- gsize newlen;
-
- storage = &msg->body_buf.c;
-
- if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) {
- if (storage->shared.shm_fd == -1) {
- return FALSE;
- }
-
- if (fstat (storage->shared.shm_fd, &st) == -1) {
- return FALSE;
- }
-
- /* Check if we need to grow */
- if ((gsize)st.st_size < msg->body_buf.len + len) {
- /* Need to grow */
- newlen = rspamd_fstring_suggest_size (msg->body_buf.len, st.st_size,
- len);
- /* Unmap as we need another size of segment */
- if (msg->body_buf.str != MAP_FAILED) {
- munmap (msg->body_buf.str, st.st_size);
- }
-
- if (ftruncate (storage->shared.shm_fd, newlen) == -1) {
- return FALSE;
- }
-
- msg->body_buf.str = mmap (NULL, newlen,
- PROT_WRITE|PROT_READ, MAP_SHARED,
- storage->shared.shm_fd, 0);
- if (msg->body_buf.str == MAP_FAILED) {
- return FALSE;
- }
-
- msg->body_buf.begin = msg->body_buf.str;
- msg->body_buf.allocated_len = newlen;
- }
- }
- else {
- storage->normal = rspamd_fstring_grow (storage->normal, len);
-
- /* Append might cause realloc */
- msg->body_buf.begin = storage->normal->str;
- msg->body_buf.len = storage->normal->len;
- msg->body_buf.str = storage->normal->str;
- msg->body_buf.allocated_len = storage->normal->allocated;
- }
-
- return TRUE;
-}
-
-gboolean
-rspamd_http_message_append_body (struct rspamd_http_message *msg,
- const gchar *data, gsize len)
-{
- union _rspamd_storage_u *storage;
-
- storage = &msg->body_buf.c;
-
- if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) {
- if (!rspamd_http_message_grow_body (msg, len)) {
- return FALSE;
- }
-
- memcpy (msg->body_buf.str + msg->body_buf.len, data, len);
- msg->body_buf.len += len;
- }
- else {
- storage->normal = rspamd_fstring_append (storage->normal, data, len);
-
- /* Append might cause realloc */
- msg->body_buf.begin = storage->normal->str;
- msg->body_buf.len = storage->normal->len;
- msg->body_buf.str = storage->normal->str;
- msg->body_buf.allocated_len = storage->normal->allocated;
- }
-
- return TRUE;
-}
-
-static void
-rspamd_http_message_storage_cleanup (struct rspamd_http_message *msg)
-{
- union _rspamd_storage_u *storage;
- struct stat st;
-
- if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) {
- storage = &msg->body_buf.c;
-
- if (storage->shared.shm_fd > 0) {
- g_assert (fstat (storage->shared.shm_fd, &st) != -1);
-
- if (msg->body_buf.str != MAP_FAILED) {
- munmap (msg->body_buf.str, st.st_size);
- }
-
- close (storage->shared.shm_fd);
- }
-
- if (storage->shared.name != NULL) {
- REF_RELEASE (storage->shared.name);
- }
-
- storage->shared.shm_fd = -1;
- msg->body_buf.str = MAP_FAILED;
- }
- else {
- if (msg->body_buf.c.normal) {
- rspamd_fstring_free (msg->body_buf.c.normal);
- }
-
- msg->body_buf.c.normal = NULL;
- }
-
- msg->body_buf.len = 0;
-}
void
rspamd_http_connection_set_max_size (struct rspamd_http_connection *conn,
@@ -2833,720 +2231,6 @@ rspamd_http_connection_set_max_size (struct rspamd_http_connection *conn,
}
void
-rspamd_http_message_free (struct rspamd_http_message *msg)
-{
- struct rspamd_http_header *hdr, *htmp, *hcur, *hcurtmp;
-
-
- HASH_ITER (hh, msg->headers, hdr, htmp) {
- HASH_DEL (msg->headers, hdr);
-
- DL_FOREACH_SAFE (hdr, hcur, hcurtmp) {
- rspamd_fstring_free (hcur->combined);
- g_free (hcur);
- }
- }
-
-
- rspamd_http_message_storage_cleanup (msg);
-
- if (msg->url != NULL) {
- rspamd_fstring_free (msg->url);
- }
- if (msg->status != NULL) {
- rspamd_fstring_free (msg->status);
- }
- if (msg->host != NULL) {
- rspamd_fstring_free (msg->host);
- }
- if (msg->peer_key != NULL) {
- rspamd_pubkey_unref (msg->peer_key);
- }
-
- g_free (msg);
-}
-
-void
-rspamd_http_message_set_peer_key (struct rspamd_http_message *msg,
- struct rspamd_cryptobox_pubkey *pk)
-{
- if (msg->peer_key != NULL) {
- rspamd_pubkey_unref (msg->peer_key);
- }
-
- if (pk) {
- msg->peer_key = rspamd_pubkey_ref (pk);
- }
- else {
- msg->peer_key = NULL;
- }
-}
-
-void
-rspamd_http_message_add_header_len (struct rspamd_http_message *msg,
- const gchar *name,
- const gchar *value,
- gsize len)
-{
- struct rspamd_http_header *hdr, *found = NULL;
- guint nlen, vlen;
-
- if (msg != NULL && name != NULL && value != NULL) {
- hdr = g_malloc0 (sizeof (struct rspamd_http_header));
- nlen = strlen (name);
- vlen = len;
- hdr->combined = rspamd_fstring_sized_new (nlen + vlen + 4);
- rspamd_printf_fstring (&hdr->combined, "%s: %*s\r\n", name, (gint)vlen,
- value);
- hdr->name.begin = hdr->combined->str;
- hdr->name.len = nlen;
- hdr->value.begin = hdr->combined->str + nlen + 2;
- hdr->value.len = vlen;
-
- HASH_FIND (hh, msg->headers, hdr->name.begin,
- hdr->name.len, found);
-
- if (found == NULL) {
- HASH_ADD_KEYPTR (hh, msg->headers, hdr->name.begin,
- hdr->name.len, hdr);
- }
-
- DL_APPEND (found, hdr);
- }
-}
-
-void
-rspamd_http_message_add_header (struct rspamd_http_message *msg,
- const gchar *name,
- const gchar *value)
-{
- if (value) {
- rspamd_http_message_add_header_len (msg, name, value, strlen (value));
- }
-}
-
-void
-rspamd_http_message_add_header_fstr (struct rspamd_http_message *msg,
- const gchar *name,
- rspamd_fstring_t *value)
-{
- struct rspamd_http_header *hdr, *found = NULL;
- guint nlen, vlen;
-
- if (msg != NULL && name != NULL && value != NULL) {
- hdr = g_malloc0 (sizeof (struct rspamd_http_header));
- nlen = strlen (name);
- vlen = value->len;
- hdr->combined = rspamd_fstring_sized_new (nlen + vlen + 4);
- rspamd_printf_fstring (&hdr->combined, "%s: %V\r\n", name, value);
- hdr->name.begin = hdr->combined->str;
- hdr->name.len = nlen;
- hdr->value.begin = hdr->combined->str + nlen + 2;
- hdr->value.len = vlen;
-
- HASH_FIND (hh, msg->headers, hdr->name.begin,
- hdr->name.len, found);
-
- if (found == NULL) {
- HASH_ADD_KEYPTR (hh, msg->headers, hdr->name.begin,
- hdr->name.len, hdr);
- }
-
- DL_APPEND (found, hdr);
- }
-}
-
-const rspamd_ftok_t *
-rspamd_http_message_find_header (struct rspamd_http_message *msg,
- const gchar *name)
-{
- struct rspamd_http_header *hdr;
- const rspamd_ftok_t *res = NULL;
- guint slen = strlen (name);
-
- if (msg != NULL) {
- HASH_FIND (hh, msg->headers, name, slen, hdr);
-
- if (hdr) {
- res = &hdr->value;
- }
- }
-
- return res;
-}
-
-GPtrArray*
-rspamd_http_message_find_header_multiple (
- struct rspamd_http_message *msg,
- const gchar *name)
-{
- GPtrArray *res = NULL;
- struct rspamd_http_header *hdr, *cur;
-
- guint slen = strlen (name);
-
- if (msg != NULL) {
- HASH_FIND (hh, msg->headers, name, slen, hdr);
-
- if (hdr) {
- res = g_ptr_array_sized_new (4);
-
- LL_FOREACH (hdr, cur) {
- g_ptr_array_add (res, &cur->value);
- }
- }
- }
-
-
- return res;
-}
-
-
-gboolean
-rspamd_http_message_remove_header (struct rspamd_http_message *msg,
- const gchar *name)
-{
- struct rspamd_http_header *hdr, *hcur, *hcurtmp;
- gboolean res = FALSE;
- guint slen = strlen (name);
-
- if (msg != NULL) {
- HASH_FIND (hh, msg->headers, name, slen, hdr);
-
- if (hdr) {
- HASH_DEL (msg->headers, hdr);
- res = TRUE;
-
- DL_FOREACH_SAFE (hdr, hcur, hcurtmp) {
- rspamd_fstring_free (hcur->combined);
- g_free (hcur);
- }
- }
- }
-
- return res;
-}
-
-/*
- * HTTP router functions
- */
-
-static void
-rspamd_http_entry_free (struct rspamd_http_connection_entry *entry)
-{
- if (entry != NULL) {
- close (entry->conn->fd);
- rspamd_http_connection_unref (entry->conn);
- if (entry->rt->finish_handler) {
- entry->rt->finish_handler (entry);
- }
-
- DL_DELETE (entry->rt->conns, entry);
- g_free (entry);
- }
-}
-
-static void
-rspamd_http_router_error_handler (struct rspamd_http_connection *conn,
- GError *err)
-{
- struct rspamd_http_connection_entry *entry = conn->ud;
- struct rspamd_http_message *msg;
-
- if (entry->is_reply) {
- /* At this point we need to finish this session and close owned socket */
- if (entry->rt->error_handler != NULL) {
- entry->rt->error_handler (entry, err);
- }
- rspamd_http_entry_free (entry);
- }
- else {
- /* Here we can write a reply to a client */
- if (entry->rt->error_handler != NULL) {
- entry->rt->error_handler (entry, err);
- }
- msg = rspamd_http_new_message (HTTP_RESPONSE);
- msg->date = time (NULL);
- msg->code = err->code;
- rspamd_http_message_set_body (msg, err->message, strlen (err->message));
- rspamd_http_connection_reset (entry->conn);
- rspamd_http_connection_write_message (entry->conn,
- msg,
- NULL,
- "text/plain",
- entry,
- entry->conn->fd,
- entry->rt->ptv,
- entry->rt->ev_base);
- entry->is_reply = TRUE;
- }
-}
-
-static const gchar *
-rspamd_http_router_detect_ct (const gchar *path)
-{
- const gchar *dot;
- guint i;
-
- dot = strrchr (path, '.');
- if (dot == NULL) {
- return http_file_types[HTTP_MAGIC_PLAIN].ct;
- }
- dot++;
-
- for (i = 0; i < G_N_ELEMENTS (http_file_types); i++) {
- if (strcmp (http_file_types[i].ext, dot) == 0) {
- return http_file_types[i].ct;
- }
- }
-
- return http_file_types[HTTP_MAGIC_PLAIN].ct;
-}
-
-static gboolean
-rspamd_http_router_is_subdir (const gchar *parent, const gchar *sub)
-{
- if (parent == NULL || sub == NULL || *parent == '\0') {
- return FALSE;
- }
-
- while (*parent != '\0') {
- if (*sub != *parent) {
- return FALSE;
- }
- parent++;
- sub++;
- }
-
- parent--;
- if (*parent == G_DIR_SEPARATOR) {
- return TRUE;
- }
-
- return (*sub == G_DIR_SEPARATOR || *sub == '\0');
-}
-
-static gboolean
-rspamd_http_router_try_file (struct rspamd_http_connection_entry *entry,
- rspamd_ftok_t *lookup, gboolean expand_path)
-{
- struct stat st;
- gint fd;
- gchar filebuf[PATH_MAX], realbuf[PATH_MAX], *dir;
- struct rspamd_http_message *reply_msg;
-
- rspamd_snprintf (filebuf, sizeof (filebuf), "%s%c%T",
- entry->rt->default_fs_path, G_DIR_SEPARATOR, lookup);
-
- if (realpath (filebuf, realbuf) == NULL ||
- lstat (realbuf, &st) == -1) {
- return FALSE;
- }
-
- if (S_ISDIR (st.st_mode) && expand_path) {
- /* Try to append 'index.html' to the url */
- rspamd_fstring_t *nlookup;
- rspamd_ftok_t tok;
- gboolean ret;
-
- nlookup = rspamd_fstring_sized_new (lookup->len + sizeof ("index.html"));
- rspamd_printf_fstring (&nlookup, "%T%c%s", lookup, G_DIR_SEPARATOR,
- "index.html");
- tok.begin = nlookup->str;
- tok.len = nlookup->len;
- ret = rspamd_http_router_try_file (entry, &tok, FALSE);
- rspamd_fstring_free (nlookup);
-
- return ret;
- }
- else if (!S_ISREG (st.st_mode)) {
- return FALSE;
- }
-
- /* We also need to ensure that file is inside the defined dir */
- rspamd_strlcpy (filebuf, realbuf, sizeof (filebuf));
- dir = dirname (filebuf);
-
- if (dir == NULL ||
- !rspamd_http_router_is_subdir (entry->rt->default_fs_path,
- dir)) {
- return FALSE;
- }
-
- fd = open (realbuf, O_RDONLY);
- if (fd == -1) {
- return FALSE;
- }
-
- reply_msg = rspamd_http_new_message (HTTP_RESPONSE);
- reply_msg->date = time (NULL);
- reply_msg->code = 200;
- rspamd_http_router_insert_headers (entry->rt, reply_msg);
-
- if (!rspamd_http_message_set_body_from_fd (reply_msg, fd)) {
- close (fd);
- return FALSE;
- }
-
- close (fd);
-
- rspamd_http_connection_reset (entry->conn);
-
- msg_debug ("requested file %s", realbuf);
- rspamd_http_connection_write_message (entry->conn, reply_msg, NULL,
- rspamd_http_router_detect_ct (realbuf), entry, entry->conn->fd,
- entry->rt->ptv, entry->rt->ev_base);
-
- return TRUE;
-}
-
-static void
-rspamd_http_router_send_error (GError *err,
- struct rspamd_http_connection_entry *entry)
-{
- struct rspamd_http_message *err_msg;
-
- err_msg = rspamd_http_new_message (HTTP_RESPONSE);
- err_msg->date = time (NULL);
- err_msg->code = err->code;
- rspamd_http_message_set_body (err_msg, err->message,
- strlen (err->message));
- entry->is_reply = TRUE;
- err_msg->status = rspamd_fstring_new_init (err->message, strlen (err->message));
- rspamd_http_router_insert_headers (entry->rt, err_msg);
- rspamd_http_connection_reset (entry->conn);
- rspamd_http_connection_write_message (entry->conn,
- err_msg,
- NULL,
- "text/plain",
- entry,
- entry->conn->fd,
- entry->rt->ptv,
- entry->rt->ev_base);
-}
-
-
-static int
-rspamd_http_router_finish_handler (struct rspamd_http_connection *conn,
- struct rspamd_http_message *msg)
-{
- struct rspamd_http_connection_entry *entry = conn->ud;
- rspamd_http_router_handler_t handler = NULL;
- gpointer found;
-
- GError *err;
- rspamd_ftok_t lookup;
- const rspamd_ftok_t *encoding;
- struct http_parser_url u;
- guint i;
- rspamd_regexp_t *re;
- struct rspamd_http_connection_router *router;
-
- G_STATIC_ASSERT (sizeof (rspamd_http_router_handler_t) ==
- sizeof (gpointer));
-
- memset (&lookup, 0, sizeof (lookup));
- router = entry->rt;
-
- if (entry->is_reply) {
- /* Request is finished, it is safe to free a connection */
- rspamd_http_entry_free (entry);
- }
- else {
- if (G_UNLIKELY (msg->method != HTTP_GET && msg->method != HTTP_POST)) {
- if (router->unknown_method_handler) {
- return router->unknown_method_handler (entry, msg);
- }
- else {
- err = g_error_new (HTTP_ERROR, 500,
- "Invalid method");
- if (entry->rt->error_handler != NULL) {
- entry->rt->error_handler (entry, err);
- }
-
- rspamd_http_router_send_error (err, entry);
- g_error_free (err);
-
- return 0;
- }
- }
-
- /* Search for path */
- if (msg->url != NULL && msg->url->len != 0) {
-
- http_parser_parse_url (msg->url->str, msg->url->len, TRUE, &u);
-
- if (u.field_set & (1 << UF_PATH)) {
- guint unnorm_len;
- lookup.begin = msg->url->str + u.field_data[UF_PATH].off;
- lookup.len = u.field_data[UF_PATH].len;
-
- rspamd_http_normalize_path_inplace ((gchar *)lookup.begin,
- lookup.len,
- &unnorm_len);
- lookup.len = unnorm_len;
- }
- else {
- lookup.begin = msg->url->str;
- lookup.len = msg->url->len;
- }
-
- found = g_hash_table_lookup (entry->rt->paths, &lookup);
- memcpy (&handler, &found, sizeof (found));
- msg_debug ("requested known path: %T", &lookup);
- }
- else {
- err = g_error_new (HTTP_ERROR, 404,
- "Empty path requested");
- if (entry->rt->error_handler != NULL) {
- entry->rt->error_handler (entry, err);
- }
-
- rspamd_http_router_send_error (err, entry);
- g_error_free (err);
-
- return 0;
- }
-
- entry->is_reply = TRUE;
-
- encoding = rspamd_http_message_find_header (msg, "Accept-Encoding");
-
- if (encoding && rspamd_substring_search (encoding->begin, encoding->len,
- "gzip", 4) != -1) {
- entry->support_gzip = TRUE;
- }
-
- if (handler != NULL) {
- return handler (entry, msg);
- }
- else {
- /* Try regexps */
- for (i = 0; i < router->regexps->len; i ++) {
- re = g_ptr_array_index (router->regexps, i);
- if (rspamd_regexp_match (re, lookup.begin, lookup.len,
- TRUE)) {
- found = rspamd_regexp_get_ud (re);
- memcpy (&handler, &found, sizeof (found));
-
- return handler (entry, msg);
- }
- }
-
- /* Now try plain file */
- if (entry->rt->default_fs_path == NULL || lookup.len == 0 ||
- !rspamd_http_router_try_file (entry, &lookup, TRUE)) {
-
- err = g_error_new (HTTP_ERROR, 404,
- "Not found");
- if (entry->rt->error_handler != NULL) {
- entry->rt->error_handler (entry, err);
- }
-
- msg_info ("path: %T not found", &lookup);
- rspamd_http_router_send_error (err, entry);
- g_error_free (err);
- }
- }
- }
-
- return 0;
-}
-
-struct rspamd_http_connection_router *
-rspamd_http_router_new (rspamd_http_router_error_handler_t eh,
- rspamd_http_router_finish_handler_t fh,
- struct timeval *timeout, struct event_base *base,
- const char *default_fs_path,
- struct rspamd_keypair_cache *cache)
-{
- struct rspamd_http_connection_router * new;
- struct stat st;
-
- new = g_malloc0 (sizeof (struct rspamd_http_connection_router));
- new->paths = g_hash_table_new_full (rspamd_ftok_icase_hash,
- rspamd_ftok_icase_equal, rspamd_fstring_mapped_ftok_free, NULL);
- new->regexps = g_ptr_array_new ();
- new->conns = NULL;
- new->error_handler = eh;
- new->finish_handler = fh;
- new->ev_base = base;
- new->response_headers = g_hash_table_new_full (rspamd_strcase_hash,
- rspamd_strcase_equal, g_free, g_free);
-
- if (timeout) {
- new->tv = *timeout;
- new->ptv = &new->tv;
- }
- else {
- new->ptv = NULL;
- }
-
- new->default_fs_path = NULL;
-
- if (default_fs_path != NULL) {
- if (stat (default_fs_path, &st) == -1) {
- msg_err ("cannot stat %s", default_fs_path);
- }
- else {
- if (!S_ISDIR (st.st_mode)) {
- msg_err ("path %s is not a directory", default_fs_path);
- }
- else {
- new->default_fs_path = realpath (default_fs_path, NULL);
- }
- }
- }
-
- new->cache = cache;
-
- return new;
-}
-
-void
-rspamd_http_router_set_key (struct rspamd_http_connection_router *router,
- struct rspamd_cryptobox_keypair *key)
-{
- g_assert (key != NULL);
-
- router->key = rspamd_keypair_ref (key);
-}
-
-void
-rspamd_http_router_add_path (struct rspamd_http_connection_router *router,
- const gchar *path, rspamd_http_router_handler_t handler)
-{
- gpointer ptr;
- rspamd_ftok_t *key;
- rspamd_fstring_t *storage;
- G_STATIC_ASSERT (sizeof (rspamd_http_router_handler_t) ==
- sizeof (gpointer));
-
- if (path != NULL && handler != NULL && router != NULL) {
- memcpy (&ptr, &handler, sizeof (ptr));
- storage = rspamd_fstring_new_init (path, strlen (path));
- key = g_malloc0 (sizeof (*key));
- key->begin = storage->str;
- key->len = storage->len;
- g_hash_table_insert (router->paths, key, ptr);
- }
-}
-
-void
-rspamd_http_router_set_unknown_handler (struct rspamd_http_connection_router *router,
- rspamd_http_router_handler_t handler)
-{
- if (router != NULL) {
- router->unknown_method_handler = handler;
- }
-}
-
-void
-rspamd_http_router_add_header (struct rspamd_http_connection_router *router,
- const gchar *name, const gchar *value)
-{
- if (name != NULL && value != NULL && router != NULL) {
- g_hash_table_replace (router->response_headers, g_strdup (name),
- g_strdup (value));
- }
-}
-
-void
-rspamd_http_router_insert_headers (struct rspamd_http_connection_router *router,
- struct rspamd_http_message *msg)
-{
- GHashTableIter it;
- gpointer k, v;
-
- if (router && msg) {
- g_hash_table_iter_init (&it, router->response_headers);
-
- while (g_hash_table_iter_next (&it, &k, &v)) {
- rspamd_http_message_add_header (msg, k, v);
- }
- }
-}
-
-void
-rspamd_http_router_add_regexp (struct rspamd_http_connection_router *router,
- struct rspamd_regexp_s *re, rspamd_http_router_handler_t handler)
-{
- gpointer ptr;
- G_STATIC_ASSERT (sizeof (rspamd_http_router_handler_t) ==
- sizeof (gpointer));
-
- if (re != NULL && handler != NULL && router != NULL) {
- memcpy (&ptr, &handler, sizeof (ptr));
- rspamd_regexp_set_ud (re, ptr);
- g_ptr_array_add (router->regexps, rspamd_regexp_ref (re));
- }
-}
-
-void
-rspamd_http_router_handle_socket (struct rspamd_http_connection_router *router,
- gint fd, gpointer ud)
-{
- struct rspamd_http_connection_entry *conn;
-
- conn = g_malloc0 (sizeof (struct rspamd_http_connection_entry));
- conn->rt = router;
- conn->ud = ud;
- conn->is_reply = FALSE;
-
- conn->conn = rspamd_http_connection_new (NULL,
- rspamd_http_router_error_handler,
- rspamd_http_router_finish_handler,
- 0,
- RSPAMD_HTTP_SERVER,
- router->cache,
- NULL);
-
- if (router->key) {
- rspamd_http_connection_set_key (conn->conn, router->key);
- }
-
- rspamd_http_connection_read_message (conn->conn, conn, fd, router->ptv,
- router->ev_base);
- DL_PREPEND (router->conns, conn);
-}
-
-void
-rspamd_http_router_free (struct rspamd_http_connection_router *router)
-{
- struct rspamd_http_connection_entry *conn, *tmp;
- rspamd_regexp_t *re;
- guint i;
-
- if (router) {
- DL_FOREACH_SAFE (router->conns, conn, tmp) {
- rspamd_http_entry_free (conn);
- }
-
- if (router->key) {
- rspamd_keypair_unref (router->key);
- }
-
- if (router->cache) {
- rspamd_keypair_cache_destroy (router->cache);
- }
-
- if (router->default_fs_path != NULL) {
- g_free (router->default_fs_path);
- }
-
- for (i = 0; i < router->regexps->len; i ++) {
- re = g_ptr_array_index (router->regexps, i);
- rspamd_regexp_unref (re);
- }
-
- g_ptr_array_free (router->regexps, TRUE);
- g_hash_table_unref (router->paths);
- g_hash_table_unref (router->response_headers);
- g_free (router);
- }
-}
-
-void
rspamd_http_connection_set_key (struct rspamd_http_connection *conn,
struct rspamd_cryptobox_keypair *key)
{
@@ -3710,19 +2394,6 @@ rspamd_http_message_parse_query (struct rspamd_http_message *msg)
}
-glong
-rspamd_http_date_format (gchar *buf, gsize len, time_t time)
-{
- struct tm tms;
-
- rspamd_gmtime (time, &tms);
-
- return rspamd_snprintf (buf, len, "%s, %02d %s %4d %02d:%02d:%02d GMT",
- http_week[tms.tm_wday], tms.tm_mday,
- http_month[tms.tm_mon], tms.tm_year + 1900,
- tms.tm_hour, tms.tm_min, tms.tm_sec);
-}
-
struct rspamd_http_message *
rspamd_http_message_ref (struct rspamd_http_message *msg)
{
@@ -3737,218 +2408,6 @@ rspamd_http_message_unref (struct rspamd_http_message *msg)
REF_RELEASE (msg);
}
-
-void
-rspamd_http_normalize_path_inplace (gchar *path, guint len, guint *nlen)
-{
- const gchar *p, *end, *slash = NULL, *dot = NULL;
- gchar *o;
- enum {
- st_normal = 0,
- st_got_dot,
- st_got_dot_dot,
- st_got_slash,
- st_got_slash_slash,
- } state = st_normal;
-
- p = path;
- end = path + len;
- o = path;
-
- while (p < end) {
- switch (state) {
- case st_normal:
- if (G_UNLIKELY (*p == '/')) {
- state = st_got_slash;
- slash = p;
- }
- else if (G_UNLIKELY (*p == '.')) {
- state = st_got_dot;
- dot = p;
- }
- else {
- *o++ = *p;
- }
- p ++;
- break;
- case st_got_slash:
- if (G_UNLIKELY (*p == '/')) {
- /* Ignore double slash */
- *o++ = *p;
- state = st_got_slash_slash;
- }
- else if (G_UNLIKELY (*p == '.')) {
- dot = p;
- state = st_got_dot;
- }
- else {
- *o++ = '/';
- *o++ = *p;
- slash = NULL;
- dot = NULL;
- state = st_normal;
- }
- p ++;
- break;
- case st_got_slash_slash:
- if (G_LIKELY (*p != '/')) {
- slash = p - 1;
- dot = NULL;
- state = st_normal;
- continue;
- }
- p ++;
- break;
- case st_got_dot:
- if (G_UNLIKELY (*p == '/')) {
- /* Remove any /./ or ./ paths */
- if (((o > path && *(o - 1) != '/') || (o == path)) && slash) {
- /* Preserve one slash */
- *o++ = '/';
- }
-
- slash = p;
- dot = NULL;
- /* Ignore last slash */
- state = st_normal;
- }
- else if (*p == '.') {
- /* Double dot character */
- state = st_got_dot_dot;
- }
- else {
- /* We have something like .some or /.some */
- if (dot && p > dot) {
- if (slash == dot - 1 && (o > path && *(o - 1) != '/')) {
- /* /.blah */
- memmove (o, slash, p - slash);
- o += p - slash;
- }
- else {
- memmove (o, dot, p - dot);
- o += p - dot;
- }
- }
-
- slash = NULL;
- dot = NULL;
- state = st_normal;
- continue;
- }
-
- p ++;
- break;
- case st_got_dot_dot:
- if (*p == '/') {
- /* We have something like /../ or ../ */
- if (slash) {
- /* We need to remove the last component from o if it is there */
- if (o > path + 2 && *(o - 1) == '/') {
- slash = rspamd_memrchr (path, '/', o - path - 2);
- }
- else if (o > path + 1) {
- slash = rspamd_memrchr (path, '/', o - path - 1);
- }
- else {
- slash = NULL;
- }
-
- if (slash) {
- o = (gchar *)slash;
- }
- /* Otherwise we keep these dots */
- slash = p;
- state = st_got_slash;
- }
- else {
- /* We have something like bla../, so we need to copy it as is */
- if (o > path && dot && p > dot) {
- memmove (o, dot, p - dot);
- o += p - dot;
- }
-
- slash = NULL;
- dot = NULL;
- state = st_normal;
- continue;
- }
- }
- else {
- /* We have something like ..bla or ... */
- if (slash) {
- *o ++ = '/';
- }
-
- if (dot && p > dot) {
- memmove (o, dot, p - dot);
- o += p - dot;
- }
-
- slash = NULL;
- dot = NULL;
- state = st_normal;
- continue;
- }
-
- p ++;
- break;
- }
- }
-
- /* Leftover */
- switch (state) {
- case st_got_dot_dot:
- /* Trailing .. */
- if (slash) {
- /* We need to remove the last component from o if it is there */
- if (o > path + 2 && *(o - 1) == '/') {
- slash = rspamd_memrchr (path, '/', o - path - 2);
- }
- else if (o > path + 1) {
- slash = rspamd_memrchr (path, '/', o - path - 1);
- }
- else {
- if (o == path) {
- /* Corner case */
- *o++ = '/';
- }
-
- slash = NULL;
- }
-
- if (slash) {
- /* Remove last / */
- o = (gchar *)slash;
- }
- }
- else {
- /* Corner case */
- if (o == path) {
- *o++ = '/';
- }
- else {
- if (dot && p > dot) {
- memmove (o, dot, p - dot);
- o += p - dot;
- }
- }
- }
- break;
- case st_got_slash:
- *o++ = '/';
- break;
- default:
- if (o > path + 1 && *(o - 1) == '/') {
- o --;
- }
- break;
- }
-
- if (nlen) {
- *nlen = (o - path);
- }
-}
-
void
rspamd_http_connection_disable_encryption (struct rspamd_http_connection *conn)
{
diff --git a/src/libutil/http_connection.h b/src/libutil/http_connection.h
new file mode 100644
index 000000000..a327eec0d
--- /dev/null
+++ b/src/libutil/http_connection.h
@@ -0,0 +1,251 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef HTTP_H_
+#define HTTP_H_
+
+/**
+ * @file http.h
+ *
+ * This is an interface for HTTP client and conn.
+ * This code uses HTTP parser written by Joyent Inc based on nginx code.
+ */
+
+#include "config.h"
+#include "http_context.h"
+#include "fstring.h"
+#include "ref.h"
+#include "http_message.h"
+#include "http_util.h"
+#include "addr.h"
+
+#include <event.h>
+
+enum rspamd_http_connection_type {
+ RSPAMD_HTTP_SERVER,
+ RSPAMD_HTTP_CLIENT
+};
+
+struct rspamd_http_header;
+struct rspamd_http_message;
+struct rspamd_http_connection_private;
+struct rspamd_http_connection;
+struct rspamd_http_connection_router;
+struct rspamd_http_connection_entry;
+struct rspamd_keepalive_hash_key;
+
+struct rspamd_storage_shmem {
+ gchar *shm_name;
+ ref_entry_t ref;
+};
+
+/**
+ * Legacy spamc protocol
+ */
+#define RSPAMD_HTTP_FLAG_SPAMC (1 << 0)
+/**
+ * Store body of the message in a shared memory segment
+ */
+#define RSPAMD_HTTP_FLAG_SHMEM (1 << 2)
+/**
+ * Store body of the message in an immutable shared memory segment
+ */
+#define RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE (1 << 3)
+/**
+ * Use tls for this message
+ */
+#define RSPAMD_HTTP_FLAG_SSL (1 << 4)
+/**
+ * Body has been set for a message
+ */
+#define RSPAMD_HTTP_FLAG_HAS_BODY (1 << 5)
+/**
+ * Do not verify server's certificate
+ */
+#define RSPAMD_HTTP_FLAG_SSL_NOVERIFY (1 << 6)
+/**
+ * Options for HTTP connection
+ */
+enum rspamd_http_options {
+ RSPAMD_HTTP_BODY_PARTIAL = 1, /**< Call body handler on all body data portions */
+ RSPAMD_HTTP_CLIENT_SIMPLE = 1u << 1, /**< Read HTTP client reply automatically */
+ RSPAMD_HTTP_CLIENT_ENCRYPTED = 1u << 2, /**< Encrypt data for client */
+ RSPAMD_HTTP_CLIENT_SHARED = 1u << 3, /**< Store reply in shared memory */
+ RSPAMD_HTTP_REQUIRE_ENCRYPTION = 1u << 4,
+ RSPAMD_HTTP_CLIENT_KEEP_ALIVE = 1u << 5,
+};
+
+typedef int (*rspamd_http_body_handler_t) (struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg,
+ const gchar *chunk,
+ gsize len);
+
+typedef void (*rspamd_http_error_handler_t) (struct rspamd_http_connection *conn,
+ GError *err);
+
+typedef int (*rspamd_http_finish_handler_t) (struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg);
+
+/**
+ * HTTP connection structure
+ */
+struct rspamd_http_connection {
+ struct rspamd_http_connection_private *priv;
+ rspamd_http_body_handler_t body_handler;
+ rspamd_http_error_handler_t error_handler;
+ rspamd_http_finish_handler_t finish_handler;
+ gpointer ud;
+ /* Used for keepalive */
+ struct rspamd_keepalive_hash_key *keepalive_hash_key;
+ gsize max_size;
+ unsigned opts;
+ enum rspamd_http_connection_type type;
+ gboolean finished;
+ gint fd;
+ gint ref;
+};
+
+/**
+ * Create new http connection
+ * @param handler_t handler_t for body
+ * @param opts options
+ * @return new connection structure
+ */
+struct rspamd_http_connection *rspamd_http_connection_new (
+ struct rspamd_http_context *ctx,
+ gint fd,
+ rspamd_http_body_handler_t body_handler,
+ rspamd_http_error_handler_t error_handler,
+ rspamd_http_finish_handler_t finish_handler,
+ unsigned opts,
+ enum rspamd_http_connection_type type);
+
+struct rspamd_http_connection *rspamd_http_connection_new_keepalive (
+ struct rspamd_http_context *ctx,
+ rspamd_http_body_handler_t body_handler,
+ rspamd_http_error_handler_t error_handler,
+ rspamd_http_finish_handler_t finish_handler,
+ rspamd_inet_addr_t *addr,
+ const gchar *host);
+
+
+/**
+ * Set key pointed by an opaque pointer
+ * @param conn connection structure
+ * @param key opaque key structure
+ */
+void rspamd_http_connection_set_key (struct rspamd_http_connection *conn,
+ struct rspamd_cryptobox_keypair *key);
+
+/**
+ * Get peer's public key
+ * @param conn connection structure
+ * @return pubkey structure or NULL
+ */
+const struct rspamd_cryptobox_pubkey* rspamd_http_connection_get_peer_key (
+ struct rspamd_http_connection *conn);
+
+/**
+ * Returns TRUE if a connection is encrypted
+ * @param conn
+ * @return
+ */
+gboolean rspamd_http_connection_is_encrypted (struct rspamd_http_connection *conn);
+
+/**
+ * Handle a request using socket fd and user data ud
+ * @param conn connection structure
+ * @param ud opaque user data
+ * @param fd fd to read/write
+ */
+void rspamd_http_connection_read_message (
+ struct rspamd_http_connection *conn,
+ gpointer ud,
+ struct timeval *timeout);
+
+void rspamd_http_connection_read_message_shared (
+ struct rspamd_http_connection *conn,
+ gpointer ud,
+ struct timeval *timeout);
+
+/**
+ * Send reply using initialised connection
+ * @param conn connection structure
+ * @param msg HTTP message
+ * @param ud opaque user data
+ * @param fd fd to read/write
+ */
+void rspamd_http_connection_write_message (
+ struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg,
+ const gchar *host,
+ const gchar *mime_type,
+ gpointer ud,
+ struct timeval *timeout);
+
+void rspamd_http_connection_write_message_shared (
+ struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg,
+ const gchar *host,
+ const gchar *mime_type,
+ gpointer ud,
+ struct timeval *timeout);
+
+/**
+ * Free connection structure
+ * @param conn
+ */
+void rspamd_http_connection_free (struct rspamd_http_connection *conn);
+
+/**
+ * Increase refcount for a connection
+ * @param conn
+ * @return
+ */
+static inline struct rspamd_http_connection *
+rspamd_http_connection_ref (struct rspamd_http_connection *conn)
+{
+ conn->ref++;
+ return conn;
+}
+
+/**
+ * Decrease a refcount for a connection and free it if refcount is equal to zero
+ * @param conn
+ */
+static void
+rspamd_http_connection_unref (struct rspamd_http_connection *conn)
+{
+ if (--conn->ref <= 0) {
+ rspamd_http_connection_free (conn);
+ }
+}
+
+/**
+ * Reset connection for a new request
+ * @param conn
+ */
+void rspamd_http_connection_reset (struct rspamd_http_connection *conn);
+
+/**
+ * Sets global maximum size for HTTP message being processed
+ * @param sz
+ */
+void rspamd_http_connection_set_max_size (struct rspamd_http_connection *conn,
+ gsize sz);
+
+void rspamd_http_connection_disable_encryption (struct rspamd_http_connection *conn);
+
+#endif /* HTTP_H_ */
diff --git a/src/libutil/http_context.c b/src/libutil/http_context.c
new file mode 100644
index 000000000..da2481640
--- /dev/null
+++ b/src/libutil/http_context.c
@@ -0,0 +1,433 @@
+/*-
+ * Copyright 2019 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "http_context.h"
+#include "http_private.h"
+#include "keypair.h"
+#include "keypairs_cache.h"
+#include "cfg_file.h"
+#include "contrib/libottery/ottery.h"
+#include "rspamd.h"
+
+static struct rspamd_http_context *default_ctx = NULL;
+
+struct rspamd_http_keepalive_cbdata {
+ struct rspamd_http_connection *conn;
+ GQueue *queue;
+ GList *link;
+ struct event ev;
+};
+
+static void
+rspamd_http_keepalive_queue_cleanup (GQueue *conns)
+{
+ GList *cur;
+
+ cur = conns->head;
+
+ while (cur) {
+ struct rspamd_http_keepalive_cbdata *cbd;
+
+ cbd = (struct rspamd_http_keepalive_cbdata *)cur->data;
+ rspamd_http_connection_unref (cbd->conn);
+ event_del (&cbd->ev);
+ g_free (cbd);
+
+ cur = cur->next;
+ }
+
+ g_queue_clear (conns);
+}
+
+static void
+rspamd_http_context_client_rotate_ev (gint fd, short what, void *arg)
+{
+ struct timeval rot_tv;
+ struct rspamd_http_context *ctx = arg;
+ gpointer kp;
+
+ double_to_tv (ctx->config.client_key_rotate_time, &rot_tv);
+ rot_tv.tv_sec += ottery_rand_range (rot_tv.tv_sec);
+ event_del (&ctx->client_rotate_ev);
+ event_add (&ctx->client_rotate_ev, &rot_tv);
+
+ kp = ctx->client_kp;
+ ctx->client_kp = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX,
+ RSPAMD_CRYPTOBOX_MODE_25519);
+ rspamd_keypair_unref (kp);
+}
+
+static struct rspamd_http_context*
+rspamd_http_context_new_default (struct rspamd_config *cfg,
+ struct event_base *ev_base)
+{
+ struct rspamd_http_context *ctx;
+
+ static const int default_kp_size = 1024;
+ static const gdouble default_rotate_time = 120;
+ static const gchar *default_user_agent = "rspamd-" RSPAMD_VERSION_FULL;
+
+ ctx = g_malloc0 (sizeof (*ctx));
+ ctx->config.kp_cache_size_client = default_kp_size;
+ ctx->config.kp_cache_size_server = default_kp_size;
+ ctx->config.client_key_rotate_time = default_rotate_time;
+ ctx->config.user_agent = default_user_agent;
+
+ if (cfg) {
+ ctx->ssl_ctx = cfg->libs_ctx->ssl_ctx;
+ ctx->ssl_ctx_noverify = cfg->libs_ctx->ssl_ctx_noverify;
+ }
+ else {
+ ctx->ssl_ctx = rspamd_init_ssl_ctx ();
+ ctx->ssl_ctx_noverify = rspamd_init_ssl_ctx_noverify ();
+ }
+
+ ctx->ev_base = ev_base;
+
+ ctx->keep_alive_hash = kh_init (rspamd_keep_alive_hash);
+
+ return ctx;
+}
+
+static void
+rspamd_http_context_init (struct rspamd_http_context *ctx)
+{
+ if (ctx->config.kp_cache_size_client > 0) {
+ ctx->client_kp_cache = rspamd_keypair_cache_new (ctx->config.kp_cache_size_client);
+ }
+
+ if (ctx->config.kp_cache_size_client > 0) {
+ ctx->client_kp_cache = rspamd_keypair_cache_new (ctx->config.kp_cache_size_client);
+ }
+
+ if (ctx->config.client_key_rotate_time > 0 && ctx->ev_base) {
+ struct timeval tv;
+ double jittered = rspamd_time_jitter (ctx->config.client_key_rotate_time,
+ 0);
+
+ double_to_tv (jittered, &tv);
+ event_set (&ctx->client_rotate_ev, -1, EV_TIMEOUT,
+ rspamd_http_context_client_rotate_ev, ctx);
+ event_base_set (ctx->ev_base, &ctx->client_rotate_ev);
+ event_add (&ctx->client_rotate_ev, &tv);
+ }
+
+ default_ctx = ctx;
+}
+
+struct rspamd_http_context*
+rspamd_http_context_create (struct rspamd_config *cfg,
+ struct event_base *ev_base)
+{
+ struct rspamd_http_context *ctx;
+ const ucl_object_t *http_obj;
+
+ ctx = rspamd_http_context_new_default (cfg, ev_base);
+ http_obj = ucl_object_lookup (cfg->rcl_obj, "http");
+
+ if (http_obj) {
+ const ucl_object_t *server_obj, *client_obj;
+
+ client_obj = ucl_object_lookup (http_obj, "client");
+
+ if (client_obj) {
+ const ucl_object_t *kp_size;
+
+ kp_size = ucl_object_lookup (client_obj, "cache_size");
+
+ if (kp_size) {
+ ctx->config.kp_cache_size_client = ucl_object_toint (kp_size);
+ }
+
+ const ucl_object_t *rotate_time;
+
+ rotate_time = ucl_object_lookup (client_obj, "rotate_time");
+
+ if (rotate_time) {
+ ctx->config.client_key_rotate_time = ucl_object_todouble (rotate_time);
+ }
+
+ const ucl_object_t *user_agent;
+
+ user_agent = ucl_object_lookup (client_obj, "user_agent");
+
+ if (user_agent) {
+ ctx->config.user_agent = ucl_object_tostring (user_agent);
+
+ if (ctx->config.user_agent && strlen (ctx->config.user_agent) == 0) {
+ ctx->config.user_agent = NULL;
+ }
+ }
+ }
+
+ server_obj = ucl_object_lookup (http_obj, "server");
+
+ if (server_obj) {
+ const ucl_object_t *kp_size;
+
+ kp_size = ucl_object_lookup (server_obj, "cache_size");
+
+ if (kp_size) {
+ ctx->config.kp_cache_size_server = ucl_object_toint (kp_size);
+ }
+ }
+ }
+
+ rspamd_http_context_init (ctx);
+
+ return ctx;
+}
+
+
+void
+rspamd_http_context_free (struct rspamd_http_context *ctx)
+{
+ if (ctx == default_ctx) {
+ default_ctx = NULL;
+ }
+
+ if (ctx->client_kp_cache) {
+ rspamd_keypair_cache_destroy (ctx->client_kp_cache);
+ }
+
+ if (ctx->server_kp_cache) {
+ rspamd_keypair_cache_destroy (ctx->server_kp_cache);
+ }
+
+ if (ctx->config.client_key_rotate_time > 0) {
+ /* Event is removed on base event loop termination */
+ /* event_del (&ctx->client_rotate_ev); */
+
+ if (ctx->client_kp) {
+ rspamd_keypair_unref (ctx->client_kp);
+ }
+ }
+
+ struct rspamd_keepalive_hash_key *hk;
+
+ kh_foreach_key (ctx->keep_alive_hash, hk, {
+ if (hk->host) {
+ g_free (hk->host);
+ }
+
+ rspamd_inet_address_free (hk->addr);
+ rspamd_http_keepalive_queue_cleanup (&hk->conns);
+ g_free (hk);
+ });
+
+ kh_destroy (rspamd_keep_alive_hash, ctx->keep_alive_hash);
+
+ g_free (ctx);
+}
+
+struct rspamd_http_context*
+rspamd_http_context_create_config (struct rspamd_http_context_cfg *cfg,
+ struct event_base *ev_base)
+{
+ struct rspamd_http_context *ctx;
+
+ ctx = rspamd_http_context_new_default (NULL, ev_base);
+ memcpy (&ctx->config, cfg, sizeof (*cfg));
+ rspamd_http_context_init (ctx);
+
+ return ctx;
+}
+
+struct rspamd_http_context*
+rspamd_http_context_default (void)
+{
+ g_assert (default_ctx != NULL);
+
+ return default_ctx;
+}
+
+gint32
+rspamd_keep_alive_key_hash (struct rspamd_keepalive_hash_key *k)
+{
+ gint32 h;
+
+ h = rspamd_inet_address_port_hash (k->addr);
+
+ if (k->host) {
+ h = rspamd_cryptobox_fast_hash (k->host, strlen (k->host), h);
+ }
+
+ return h;
+}
+
+bool
+rspamd_keep_alive_key_equal (struct rspamd_keepalive_hash_key *k1,
+ struct rspamd_keepalive_hash_key *k2)
+{
+ if (k1->host && k2->host) {
+ if (rspamd_inet_address_port_equal (k1->addr, k2->addr)) {
+ return strcmp (k1->host, k2->host);
+ }
+ }
+ else if (!k1->host && !k2->host) {
+ return rspamd_inet_address_port_equal (k1->addr, k2->addr);
+ }
+
+ /* One has host and another has no host */
+ return false;
+}
+
+struct rspamd_http_connection*
+rspamd_http_context_check_keepalive (struct rspamd_http_context *ctx,
+ const rspamd_inet_addr_t *addr,
+ const gchar *host)
+{
+ struct rspamd_keepalive_hash_key hk, *phk;
+ khiter_t k;
+
+ hk.addr = (rspamd_inet_addr_t *)addr;
+ hk.host = (gchar *)host;
+
+ k = kh_get (rspamd_keep_alive_hash, ctx->keep_alive_hash, &hk);
+
+ if (k != kh_end (ctx->keep_alive_hash)) {
+ phk = kh_key (ctx->keep_alive_hash, k);
+ GQueue *conns = &phk->conns;
+
+ /* Use stack based approach */
+
+ if (g_queue_get_length (conns) > 0) {
+ struct rspamd_http_keepalive_cbdata *cbd;
+ struct rspamd_http_connection *conn;
+
+ cbd = g_queue_pop_head (conns);
+ event_del (&cbd->ev);
+ conn = cbd->conn;
+ g_free (cbd);
+
+ /* We transfer refcount here! */
+ return conn;
+ }
+ }
+
+ return NULL;
+}
+
+void
+rspamd_http_context_prepare_keepalive (struct rspamd_http_context *ctx,
+ struct rspamd_http_connection *conn,
+ const rspamd_inet_addr_t *addr,
+ const gchar *host)
+{
+ struct rspamd_keepalive_hash_key hk, *phk;
+ khiter_t k;
+
+ hk.addr = (rspamd_inet_addr_t *)addr;
+ hk.host = (gchar *)host;
+
+ k = kh_get (rspamd_keep_alive_hash, ctx->keep_alive_hash, &hk);
+
+ if (k != kh_end (ctx->keep_alive_hash)) {
+ /* Reuse existing */
+ conn->keepalive_hash_key = kh_key (ctx->keep_alive_hash, k);
+ }
+ else {
+ /* Create new one */
+ GQueue empty_init = G_QUEUE_INIT;
+ gint r;
+
+ phk = g_malloc (sizeof (*phk));
+ phk->conns = empty_init;
+ phk->host = g_strdup (host);
+ phk->addr = rspamd_inet_address_copy (addr);
+
+ kh_put (rspamd_keep_alive_hash, ctx->keep_alive_hash, phk, &r);
+ conn->keepalive_hash_key = phk;
+ }
+}
+
+static void
+rspamd_http_keepalive_handler (gint fd, short what, gpointer ud)
+{
+ struct rspamd_http_keepalive_cbdata *cbdata =
+ (struct rspamd_http_keepalive_cbdata *)ud;
+ /*
+ * We can get here if a remote side reported something or it has
+ * timed out. In both cases we just terminate keepalive connection.
+ */
+
+ g_queue_delete_link (cbdata->queue, cbdata->link);
+ rspamd_http_connection_unref (cbdata->conn);
+ g_free (cbdata);
+}
+
+void
+rspamd_http_context_push_keepalive (struct rspamd_http_context *ctx,
+ struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg,
+ struct event_base *ev_base)
+{
+ struct rspamd_http_keepalive_cbdata *cbdata;
+ struct timeval tv;
+ gdouble timeout = ctx->config.keepalive_interval;
+
+ g_assert (conn->keepalive_hash_key != NULL);
+
+ /* Move connection to the keepalive pool */
+ cbdata = g_malloc0 (sizeof (*cbdata));
+
+ cbdata->conn = rspamd_http_connection_ref (conn);
+ g_queue_push_tail (&conn->keepalive_hash_key->conns, cbdata);
+ cbdata->link = conn->keepalive_hash_key->conns.tail;
+ cbdata->queue = &conn->keepalive_hash_key->conns;
+ conn->finished = FALSE;
+
+ event_set (&cbdata->ev, conn->fd, EV_READ|EV_TIMEOUT,
+ rspamd_http_keepalive_handler,
+ &cbdata);
+
+ if (msg) {
+ const rspamd_ftok_t *tok;
+
+ tok = rspamd_http_message_find_header (msg, "Keep-Alive");
+
+ if (tok) {
+ goffset pos = rspamd_substring_search_caseless (tok->begin,
+ tok->len, "timeout=", sizeof ("timeout=") - 1);
+
+ if (pos != -1) {
+ pos += sizeof ("timeout=");
+
+ gchar *end_pos = memchr (tok->begin + pos, ',', tok->len - pos);
+ glong real_timeout;
+
+ if (end_pos) {
+ if (rspamd_strtol (tok->begin + pos + 1,
+ (end_pos - tok->begin) - pos - 1, &real_timeout) &&
+ real_timeout > 0) {
+ timeout = real_timeout;
+ }
+ }
+ else {
+ if (rspamd_strtol (tok->begin + pos + 1,
+ tok->len - pos - 1, &real_timeout) &&
+ real_timeout > 0) {
+ timeout = real_timeout;
+ }
+ }
+ }
+ }
+ }
+
+ double_to_tv (timeout, &tv);
+ event_base_set (ev_base, &cbdata->ev);
+ event_add (&cbdata->ev, &tv);
+} \ No newline at end of file
diff --git a/src/libutil/http_context.h b/src/libutil/http_context.h
new file mode 100644
index 000000000..74e5c69a6
--- /dev/null
+++ b/src/libutil/http_context.h
@@ -0,0 +1,95 @@
+/*-
+ * Copyright 2019 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_HTTP_CONTEXT_H
+#define RSPAMD_HTTP_CONTEXT_H
+
+#include "config.h"
+#include "ucl.h"
+#include "addr.h"
+
+#include <event.h>
+
+struct rspamd_http_context;
+struct rspamd_config;
+struct rspamd_http_message;
+
+struct rspamd_http_context_cfg {
+ guint kp_cache_size_client;
+ guint kp_cache_size_server;
+ guint ssl_cache_size;
+ gdouble keepalive_interval;
+ gdouble client_key_rotate_time;
+ const gchar *user_agent;
+};
+
+/**
+ * Creates and configures new HTTP context
+ * @param root_conf configuration object
+ * @param ev_base event base
+ * @return new context used for both client and server HTTP connections
+ */
+struct rspamd_http_context* rspamd_http_context_create (struct rspamd_config *cfg,
+ struct event_base *ev_base);
+
+struct rspamd_http_context* rspamd_http_context_create_config (
+ struct rspamd_http_context_cfg *cfg,
+ struct event_base *ev_base);
+/**
+ * Destroys context
+ * @param ctx
+ */
+void rspamd_http_context_free (struct rspamd_http_context *ctx);
+
+struct rspamd_http_context* rspamd_http_context_default (void);
+
+/**
+ * Returns preserved keepalive connection if it's available.
+ * Refcount is transferred to caller!
+ * @param ctx
+ * @param addr
+ * @param host
+ * @return
+ */
+struct rspamd_http_connection* rspamd_http_context_check_keepalive (
+ struct rspamd_http_context *ctx, const rspamd_inet_addr_t *addr,
+ const gchar *host);
+
+/**
+ * Prepares keepalive key for a connection by creating a new entry or by reusing existent
+ * Bear in mind, that keepalive pool has currently no cleanup methods!
+ * @param ctx
+ * @param conn
+ * @param addr
+ * @param host
+ */
+void rspamd_http_context_prepare_keepalive (struct rspamd_http_context *ctx,
+ struct rspamd_http_connection *conn,
+ const rspamd_inet_addr_t *addr,
+ const gchar *host);
+/**
+ * Pushes a connection to keepalive pool after client request is finished,
+ * keepalive key *must* be prepared before using of this function
+ * @param ctx
+ * @param conn
+ * @param msg
+ */
+void rspamd_http_context_push_keepalive (struct rspamd_http_context *ctx,
+ struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg,
+ struct event_base *ev_base);
+
+#endif
diff --git a/src/libutil/http_message.c b/src/libutil/http_message.c
new file mode 100644
index 000000000..0720dc416
--- /dev/null
+++ b/src/libutil/http_message.c
@@ -0,0 +1,660 @@
+/*-
+ * Copyright 2019 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "http_message.h"
+#include "libutil/http_connection.h"
+#include "libutil/http_private.h"
+#include "libutil/printf.h"
+#include "libutil/logger.h"
+#include "utlist.h"
+#include "unix-std.h"
+
+struct rspamd_http_message *
+rspamd_http_new_message (enum rspamd_http_message_type type)
+{
+ struct rspamd_http_message *new;
+
+ new = g_malloc0 (sizeof (struct rspamd_http_message));
+
+ if (type == HTTP_REQUEST) {
+ new->url = rspamd_fstring_new ();
+ }
+ else {
+ new->url = NULL;
+ new->code = 200;
+ }
+
+ new->port = 80;
+ new->type = type;
+ new->method = HTTP_INVALID;
+
+ REF_INIT_RETAIN (new, rspamd_http_message_free);
+
+ return new;
+}
+
+struct rspamd_http_message*
+rspamd_http_message_from_url (const gchar *url)
+{
+ struct http_parser_url pu;
+ struct rspamd_http_message *msg;
+ const gchar *host, *path;
+ size_t pathlen, urllen;
+ guint flags = 0;
+
+ if (url == NULL) {
+ return NULL;
+ }
+
+ urllen = strlen (url);
+ memset (&pu, 0, sizeof (pu));
+
+ if (http_parser_parse_url (url, urllen, FALSE, &pu) != 0) {
+ msg_warn ("cannot parse URL: %s", url);
+ return NULL;
+ }
+
+ if ((pu.field_set & (1 << UF_HOST)) == 0) {
+ msg_warn ("no host argument in URL: %s", url);
+ return NULL;
+ }
+
+ if ((pu.field_set & (1 << UF_SCHEMA))) {
+ if (pu.field_data[UF_SCHEMA].len == sizeof ("https") - 1 &&
+ memcmp (url + pu.field_data[UF_SCHEMA].off, "https", 5) == 0) {
+ flags |= RSPAMD_HTTP_FLAG_SSL;
+ }
+ }
+
+ if ((pu.field_set & (1 << UF_PATH)) == 0) {
+ path = "/";
+ pathlen = 1;
+ }
+ else {
+ path = url + pu.field_data[UF_PATH].off;
+ pathlen = urllen - pu.field_data[UF_PATH].off;
+ }
+
+ msg = rspamd_http_new_message (HTTP_REQUEST);
+ host = url + pu.field_data[UF_HOST].off;
+ msg->flags = flags;
+
+ if ((pu.field_set & (1 << UF_PORT)) != 0) {
+ msg->port = pu.port;
+ }
+ else {
+ /* XXX: magic constant */
+ if (flags & RSPAMD_HTTP_FLAG_SSL) {
+ msg->port = 443;
+ }
+ else {
+ msg->port = 80;
+ }
+ }
+
+ msg->host = rspamd_fstring_new_init (host, pu.field_data[UF_HOST].len);
+ msg->url = rspamd_fstring_append (msg->url, path, pathlen);
+
+ REF_INIT_RETAIN (msg, rspamd_http_message_free);
+
+ return msg;
+}
+
+const gchar *
+rspamd_http_message_get_body (struct rspamd_http_message *msg,
+ gsize *blen)
+{
+ const gchar *ret = NULL;
+
+ if (msg->body_buf.len > 0) {
+ ret = msg->body_buf.begin;
+ }
+
+ if (blen) {
+ *blen = msg->body_buf.len;
+ }
+
+ return ret;
+}
+
+static void
+rspamd_http_shname_dtor (void *p)
+{
+ struct rspamd_storage_shmem *n = p;
+
+#ifdef HAVE_SANE_SHMEM
+ shm_unlink (n->shm_name);
+#else
+ unlink (n->shm_name);
+#endif
+ g_free (n->shm_name);
+ g_free (n);
+}
+
+struct rspamd_storage_shmem *
+rspamd_http_message_shmem_ref (struct rspamd_http_message *msg)
+{
+ if ((msg->flags & RSPAMD_HTTP_FLAG_SHMEM) && msg->body_buf.c.shared.name) {
+ REF_RETAIN (msg->body_buf.c.shared.name);
+ return msg->body_buf.c.shared.name;
+ }
+
+ return NULL;
+}
+
+guint
+rspamd_http_message_get_flags (struct rspamd_http_message *msg)
+{
+ return msg->flags;
+}
+
+void
+rspamd_http_message_shmem_unref (struct rspamd_storage_shmem *p)
+{
+ REF_RELEASE (p);
+}
+
+gboolean
+rspamd_http_message_set_body (struct rspamd_http_message *msg,
+ const gchar *data, gsize len)
+{
+ union _rspamd_storage_u *storage;
+ storage = &msg->body_buf.c;
+
+ rspamd_http_message_storage_cleanup (msg);
+
+ if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) {
+ storage->shared.name = g_malloc (sizeof (*storage->shared.name));
+ REF_INIT_RETAIN (storage->shared.name, rspamd_http_shname_dtor);
+#ifdef HAVE_SANE_SHMEM
+ #if defined(__DragonFly__)
+ // DragonFly uses regular files for shm. User rspamd is not allowed to create
+ // files in the root.
+ storage->shared.name->shm_name = g_strdup ("/tmp/rhm.XXXXXXXXXXXXXXXXXXXX");
+#else
+ storage->shared.name->shm_name = g_strdup ("/rhm.XXXXXXXXXXXXXXXXXXXX");
+#endif
+ storage->shared.shm_fd = rspamd_shmem_mkstemp (storage->shared.name->shm_name);
+#else
+ /* XXX: assume that tempdir is /tmp */
+ storage->shared.name->shm_name = g_strdup ("/tmp/rhm.XXXXXXXXXXXXXXXXXXXX");
+ storage->shared.shm_fd = mkstemp (storage->shared.name->shm_name);
+#endif
+
+ if (storage->shared.shm_fd == -1) {
+ return FALSE;
+ }
+
+ if (len != 0 && len != ULLONG_MAX) {
+ if (ftruncate (storage->shared.shm_fd, len) == -1) {
+ return FALSE;
+ }
+
+ msg->body_buf.str = mmap (NULL, len,
+ PROT_WRITE|PROT_READ, MAP_SHARED,
+ storage->shared.shm_fd, 0);
+
+ if (msg->body_buf.str == MAP_FAILED) {
+ return FALSE;
+ }
+
+ msg->body_buf.begin = msg->body_buf.str;
+ msg->body_buf.allocated_len = len;
+
+ if (data != NULL) {
+ memcpy (msg->body_buf.str, data, len);
+ msg->body_buf.len = len;
+ }
+ }
+ else {
+ msg->body_buf.len = 0;
+ msg->body_buf.begin = NULL;
+ msg->body_buf.str = NULL;
+ msg->body_buf.allocated_len = 0;
+ }
+ }
+ else {
+ if (len != 0 && len != ULLONG_MAX) {
+ if (data == NULL) {
+ storage->normal = rspamd_fstring_sized_new (len);
+ msg->body_buf.len = 0;
+ }
+ else {
+ storage->normal = rspamd_fstring_new_init (data, len);
+ msg->body_buf.len = len;
+ }
+ }
+ else {
+ storage->normal = rspamd_fstring_new ();
+ }
+
+ msg->body_buf.begin = storage->normal->str;
+ msg->body_buf.str = storage->normal->str;
+ msg->body_buf.allocated_len = storage->normal->allocated;
+ }
+
+ msg->flags |= RSPAMD_HTTP_FLAG_HAS_BODY;
+
+ return TRUE;
+}
+
+void
+rspamd_http_message_set_method (struct rspamd_http_message *msg,
+ const gchar *method)
+{
+ gint i;
+
+ /* Linear search: not very efficient method */
+ for (i = 0; i < HTTP_METHOD_MAX; i ++) {
+ if (g_ascii_strcasecmp (method, http_method_str (i)) == 0) {
+ msg->method = i;
+ }
+ }
+}
+
+gboolean
+rspamd_http_message_set_body_from_fd (struct rspamd_http_message *msg,
+ gint fd)
+{
+ union _rspamd_storage_u *storage;
+ struct stat st;
+
+ rspamd_http_message_storage_cleanup (msg);
+
+ storage = &msg->body_buf.c;
+ msg->flags |= RSPAMD_HTTP_FLAG_SHMEM|RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE;
+
+ storage->shared.shm_fd = dup (fd);
+ msg->body_buf.str = MAP_FAILED;
+
+ if (storage->shared.shm_fd == -1) {
+ return FALSE;
+ }
+
+ if (fstat (storage->shared.shm_fd, &st) == -1) {
+ return FALSE;
+ }
+
+ msg->body_buf.str = mmap (NULL, st.st_size,
+ PROT_READ, MAP_SHARED,
+ storage->shared.shm_fd, 0);
+
+ if (msg->body_buf.str == MAP_FAILED) {
+ return FALSE;
+ }
+
+ msg->body_buf.begin = msg->body_buf.str;
+ msg->body_buf.len = st.st_size;
+ msg->body_buf.allocated_len = st.st_size;
+
+ return TRUE;
+}
+
+gboolean
+rspamd_http_message_set_body_from_fstring_steal (struct rspamd_http_message *msg,
+ rspamd_fstring_t *fstr)
+{
+ union _rspamd_storage_u *storage;
+
+ rspamd_http_message_storage_cleanup (msg);
+
+ storage = &msg->body_buf.c;
+ msg->flags &= ~(RSPAMD_HTTP_FLAG_SHMEM|RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE);
+
+ storage->normal = fstr;
+ msg->body_buf.str = fstr->str;
+ msg->body_buf.begin = msg->body_buf.str;
+ msg->body_buf.len = fstr->len;
+ msg->body_buf.allocated_len = fstr->allocated;
+
+ return TRUE;
+}
+
+gboolean
+rspamd_http_message_set_body_from_fstring_copy (struct rspamd_http_message *msg,
+ const rspamd_fstring_t *fstr)
+{
+ union _rspamd_storage_u *storage;
+
+ rspamd_http_message_storage_cleanup (msg);
+
+ storage = &msg->body_buf.c;
+ msg->flags &= ~(RSPAMD_HTTP_FLAG_SHMEM|RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE);
+
+ storage->normal = rspamd_fstring_new_init (fstr->str, fstr->len);
+ msg->body_buf.str = storage->normal->str;
+ msg->body_buf.begin = msg->body_buf.str;
+ msg->body_buf.len = storage->normal->len;
+ msg->body_buf.allocated_len = storage->normal->allocated;
+
+ return TRUE;
+}
+
+
+gboolean
+rspamd_http_message_grow_body (struct rspamd_http_message *msg, gsize len)
+{
+ struct stat st;
+ union _rspamd_storage_u *storage;
+ gsize newlen;
+
+ storage = &msg->body_buf.c;
+
+ if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) {
+ if (storage->shared.shm_fd == -1) {
+ return FALSE;
+ }
+
+ if (fstat (storage->shared.shm_fd, &st) == -1) {
+ return FALSE;
+ }
+
+ /* Check if we need to grow */
+ if ((gsize)st.st_size < msg->body_buf.len + len) {
+ /* Need to grow */
+ newlen = rspamd_fstring_suggest_size (msg->body_buf.len, st.st_size,
+ len);
+ /* Unmap as we need another size of segment */
+ if (msg->body_buf.str != MAP_FAILED) {
+ munmap (msg->body_buf.str, st.st_size);
+ }
+
+ if (ftruncate (storage->shared.shm_fd, newlen) == -1) {
+ return FALSE;
+ }
+
+ msg->body_buf.str = mmap (NULL, newlen,
+ PROT_WRITE|PROT_READ, MAP_SHARED,
+ storage->shared.shm_fd, 0);
+ if (msg->body_buf.str == MAP_FAILED) {
+ return FALSE;
+ }
+
+ msg->body_buf.begin = msg->body_buf.str;
+ msg->body_buf.allocated_len = newlen;
+ }
+ }
+ else {
+ storage->normal = rspamd_fstring_grow (storage->normal, len);
+
+ /* Append might cause realloc */
+ msg->body_buf.begin = storage->normal->str;
+ msg->body_buf.len = storage->normal->len;
+ msg->body_buf.str = storage->normal->str;
+ msg->body_buf.allocated_len = storage->normal->allocated;
+ }
+
+ return TRUE;
+}
+
+gboolean
+rspamd_http_message_append_body (struct rspamd_http_message *msg,
+ const gchar *data, gsize len)
+{
+ union _rspamd_storage_u *storage;
+
+ storage = &msg->body_buf.c;
+
+ if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) {
+ if (!rspamd_http_message_grow_body (msg, len)) {
+ return FALSE;
+ }
+
+ memcpy (msg->body_buf.str + msg->body_buf.len, data, len);
+ msg->body_buf.len += len;
+ }
+ else {
+ storage->normal = rspamd_fstring_append (storage->normal, data, len);
+
+ /* Append might cause realloc */
+ msg->body_buf.begin = storage->normal->str;
+ msg->body_buf.len = storage->normal->len;
+ msg->body_buf.str = storage->normal->str;
+ msg->body_buf.allocated_len = storage->normal->allocated;
+ }
+
+ return TRUE;
+}
+
+void
+rspamd_http_message_storage_cleanup (struct rspamd_http_message *msg)
+{
+ union _rspamd_storage_u *storage;
+ struct stat st;
+
+ if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) {
+ storage = &msg->body_buf.c;
+
+ if (storage->shared.shm_fd > 0) {
+ g_assert (fstat (storage->shared.shm_fd, &st) != -1);
+
+ if (msg->body_buf.str != MAP_FAILED) {
+ munmap (msg->body_buf.str, st.st_size);
+ }
+
+ close (storage->shared.shm_fd);
+ }
+
+ if (storage->shared.name != NULL) {
+ REF_RELEASE (storage->shared.name);
+ }
+
+ storage->shared.shm_fd = -1;
+ msg->body_buf.str = MAP_FAILED;
+ }
+ else {
+ if (msg->body_buf.c.normal) {
+ rspamd_fstring_free (msg->body_buf.c.normal);
+ }
+
+ msg->body_buf.c.normal = NULL;
+ }
+
+ msg->body_buf.len = 0;
+}
+
+void
+rspamd_http_message_free (struct rspamd_http_message *msg)
+{
+ struct rspamd_http_header *hdr, *htmp, *hcur, *hcurtmp;
+
+
+ HASH_ITER (hh, msg->headers, hdr, htmp) {
+ HASH_DEL (msg->headers, hdr);
+
+ DL_FOREACH_SAFE (hdr, hcur, hcurtmp) {
+ rspamd_fstring_free (hcur->combined);
+ g_free (hcur);
+ }
+ }
+
+ rspamd_http_message_storage_cleanup (msg);
+
+ if (msg->url != NULL) {
+ rspamd_fstring_free (msg->url);
+ }
+ if (msg->status != NULL) {
+ rspamd_fstring_free (msg->status);
+ }
+ if (msg->host != NULL) {
+ rspamd_fstring_free (msg->host);
+ }
+ if (msg->peer_key != NULL) {
+ rspamd_pubkey_unref (msg->peer_key);
+ }
+
+ g_free (msg);
+}
+
+void
+rspamd_http_message_set_peer_key (struct rspamd_http_message *msg,
+ struct rspamd_cryptobox_pubkey *pk)
+{
+ if (msg->peer_key != NULL) {
+ rspamd_pubkey_unref (msg->peer_key);
+ }
+
+ if (pk) {
+ msg->peer_key = rspamd_pubkey_ref (pk);
+ }
+ else {
+ msg->peer_key = NULL;
+ }
+}
+
+void
+rspamd_http_message_add_header_len (struct rspamd_http_message *msg,
+ const gchar *name,
+ const gchar *value,
+ gsize len)
+{
+ struct rspamd_http_header *hdr, *found = NULL;
+ guint nlen, vlen;
+
+ if (msg != NULL && name != NULL && value != NULL) {
+ hdr = g_malloc0 (sizeof (struct rspamd_http_header));
+ nlen = strlen (name);
+ vlen = len;
+ hdr->combined = rspamd_fstring_sized_new (nlen + vlen + 4);
+ rspamd_printf_fstring (&hdr->combined, "%s: %*s\r\n", name, (gint)vlen,
+ value);
+ hdr->name.begin = hdr->combined->str;
+ hdr->name.len = nlen;
+ hdr->value.begin = hdr->combined->str + nlen + 2;
+ hdr->value.len = vlen;
+
+ HASH_FIND (hh, msg->headers, hdr->name.begin,
+ hdr->name.len, found);
+
+ if (found == NULL) {
+ HASH_ADD_KEYPTR (hh, msg->headers, hdr->name.begin,
+ hdr->name.len, hdr);
+ }
+
+ DL_APPEND (found, hdr);
+ }
+}
+
+void
+rspamd_http_message_add_header (struct rspamd_http_message *msg,
+ const gchar *name,
+ const gchar *value)
+{
+ if (value) {
+ rspamd_http_message_add_header_len (msg, name, value, strlen (value));
+ }
+}
+
+void
+rspamd_http_message_add_header_fstr (struct rspamd_http_message *msg,
+ const gchar *name,
+ rspamd_fstring_t *value)
+{
+ struct rspamd_http_header *hdr, *found = NULL;
+ guint nlen, vlen;
+
+ if (msg != NULL && name != NULL && value != NULL) {
+ hdr = g_malloc0 (sizeof (struct rspamd_http_header));
+ nlen = strlen (name);
+ vlen = value->len;
+ hdr->combined = rspamd_fstring_sized_new (nlen + vlen + 4);
+ rspamd_printf_fstring (&hdr->combined, "%s: %V\r\n", name, value);
+ hdr->name.begin = hdr->combined->str;
+ hdr->name.len = nlen;
+ hdr->value.begin = hdr->combined->str + nlen + 2;
+ hdr->value.len = vlen;
+
+ HASH_FIND (hh, msg->headers, hdr->name.begin,
+ hdr->name.len, found);
+
+ if (found == NULL) {
+ HASH_ADD_KEYPTR (hh, msg->headers, hdr->name.begin,
+ hdr->name.len, hdr);
+ }
+
+ DL_APPEND (found, hdr);
+ }
+}
+
+const rspamd_ftok_t *
+rspamd_http_message_find_header (struct rspamd_http_message *msg,
+ const gchar *name)
+{
+ struct rspamd_http_header *hdr;
+ const rspamd_ftok_t *res = NULL;
+ guint slen = strlen (name);
+
+ if (msg != NULL) {
+ HASH_FIND (hh, msg->headers, name, slen, hdr);
+
+ if (hdr) {
+ res = &hdr->value;
+ }
+ }
+
+ return res;
+}
+
+GPtrArray*
+rspamd_http_message_find_header_multiple (
+ struct rspamd_http_message *msg,
+ const gchar *name)
+{
+ GPtrArray *res = NULL;
+ struct rspamd_http_header *hdr, *cur;
+
+ guint slen = strlen (name);
+
+ if (msg != NULL) {
+ HASH_FIND (hh, msg->headers, name, slen, hdr);
+
+ if (hdr) {
+ res = g_ptr_array_sized_new (4);
+
+ LL_FOREACH (hdr, cur) {
+ g_ptr_array_add (res, &cur->value);
+ }
+ }
+ }
+
+
+ return res;
+}
+
+
+gboolean
+rspamd_http_message_remove_header (struct rspamd_http_message *msg,
+ const gchar *name)
+{
+ struct rspamd_http_header *hdr, *hcur, *hcurtmp;
+ gboolean res = FALSE;
+ guint slen = strlen (name);
+
+ if (msg != NULL) {
+ HASH_FIND (hh, msg->headers, name, slen, hdr);
+
+ if (hdr) {
+ HASH_DEL (msg->headers, hdr);
+ res = TRUE;
+
+ DL_FOREACH_SAFE (hdr, hcur, hcurtmp) {
+ rspamd_fstring_free (hcur->combined);
+ g_free (hcur);
+ }
+ }
+ }
+
+ return res;
+} \ No newline at end of file
diff --git a/src/libutil/http_message.h b/src/libutil/http_message.h
new file mode 100644
index 000000000..c9e6abfce
--- /dev/null
+++ b/src/libutil/http_message.h
@@ -0,0 +1,221 @@
+/*-
+ * Copyright 2019 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_HTTP_MESSAGE_H
+#define RSPAMD_HTTP_MESSAGE_H
+
+#include "config.h"
+#include "keypair.h"
+#include "keypairs_cache.h"
+#include "fstring.h"
+#include "ref.h"
+
+struct rspamd_http_connection;
+
+enum rspamd_http_message_type { HTTP_REQUEST = 0, HTTP_RESPONSE };
+
+/**
+ * Extract the current message from a connection to deal with separately
+ * @param conn
+ * @return
+ */
+struct rspamd_http_message * rspamd_http_connection_steal_msg (
+ struct rspamd_http_connection *conn);
+
+/**
+ * Copy the current message from a connection to deal with separately
+ * @param conn
+ * @return
+ */
+struct rspamd_http_message * rspamd_http_connection_copy_msg (
+ struct rspamd_http_message *msg, GError **err);
+
+/**
+ * Create new HTTP message
+ * @param type request or response
+ * @return new http message
+ */
+struct rspamd_http_message * rspamd_http_new_message (enum rspamd_http_message_type type);
+
+/**
+ * Increase refcount number for an HTTP message
+ * @param msg message to use
+ * @return
+ */
+struct rspamd_http_message * rspamd_http_message_ref (struct rspamd_http_message *msg);
+/**
+ * Decrease number of refcounts for http message
+ * @param msg
+ */
+void rspamd_http_message_unref (struct rspamd_http_message *msg);
+
+/**
+ * Sets a key for peer
+ * @param msg
+ * @param pk
+ */
+void rspamd_http_message_set_peer_key (struct rspamd_http_message *msg,
+ struct rspamd_cryptobox_pubkey *pk);
+/**
+ * Create HTTP message from URL
+ * @param url
+ * @return new message or NULL
+ */
+struct rspamd_http_message* rspamd_http_message_from_url (const gchar *url);
+
+/**
+ * Returns body for a message
+ * @param msg
+ * @param blen pointer where to save body length
+ * @return pointer to body start
+ */
+const gchar *rspamd_http_message_get_body (struct rspamd_http_message *msg,
+ gsize *blen);
+
+/**
+ * Set message's body from the string
+ * @param msg
+ * @param data
+ * @param len
+ * @return TRUE if a message's body has been set
+ */
+gboolean rspamd_http_message_set_body (struct rspamd_http_message *msg,
+ const gchar *data, gsize len);
+
+/**
+ * Set message's method by name
+ * @param msg
+ * @param method
+ */
+void rspamd_http_message_set_method (struct rspamd_http_message *msg,
+ const gchar *method);
+/**
+ * Maps fd as message's body
+ * @param msg
+ * @param fd
+ * @return TRUE if a message's body has been set
+ */
+gboolean rspamd_http_message_set_body_from_fd (struct rspamd_http_message *msg,
+ gint fd);
+
+/**
+ * Uses rspamd_fstring_t as message's body, string is consumed by this operation
+ * @param msg
+ * @param fstr
+ * @return TRUE if a message's body has been set
+ */
+gboolean rspamd_http_message_set_body_from_fstring_steal (struct rspamd_http_message *msg,
+ rspamd_fstring_t *fstr);
+
+/**
+ * Uses rspamd_fstring_t as message's body, string is copied by this operation
+ * @param msg
+ * @param fstr
+ * @return TRUE if a message's body has been set
+ */
+gboolean rspamd_http_message_set_body_from_fstring_copy (struct rspamd_http_message *msg,
+ const rspamd_fstring_t *fstr);
+
+/**
+ * Appends data to message's body
+ * @param msg
+ * @param data
+ * @param len
+ * @return TRUE if a message's body has been set
+ */
+gboolean rspamd_http_message_append_body (struct rspamd_http_message *msg,
+ const gchar *data, gsize len);
+
+/**
+ * Append a header to http message
+ * @param rep
+ * @param name
+ * @param value
+ */
+void rspamd_http_message_add_header (struct rspamd_http_message *msg,
+ const gchar *name,
+ const gchar *value);
+
+void rspamd_http_message_add_header_len (struct rspamd_http_message *msg,
+ const gchar *name,
+ const gchar *value,
+ gsize len);
+
+void rspamd_http_message_add_header_fstr (struct rspamd_http_message *msg,
+ const gchar *name,
+ rspamd_fstring_t *value);
+
+/**
+ * Search for a specified header in message
+ * @param msg message
+ * @param name name of header
+ */
+const rspamd_ftok_t * rspamd_http_message_find_header (
+ struct rspamd_http_message *msg,
+ const gchar *name);
+
+/**
+ * Search for a header that has multiple values
+ * @param msg
+ * @param name
+ * @return list of rspamd_ftok_t * with values
+ */
+GPtrArray* rspamd_http_message_find_header_multiple (
+ struct rspamd_http_message *msg,
+ const gchar *name);
+
+/**
+ * Remove specific header from a message
+ * @param msg
+ * @param name
+ * @return
+ */
+gboolean rspamd_http_message_remove_header (struct rspamd_http_message *msg,
+ const gchar *name);
+
+/**
+ * Free HTTP message
+ * @param msg
+ */
+void rspamd_http_message_free (struct rspamd_http_message *msg);
+
+/**
+ * Extract arguments from a message's URI contained inside query string decoding
+ * them if needed
+ * @param msg HTTP request message
+ * @return new GHashTable which maps rspamd_ftok_t* to rspamd_ftok_t*
+ * (table must be freed by a caller)
+ */
+GHashTable* rspamd_http_message_parse_query (struct rspamd_http_message *msg);
+
+/**
+ * Increase refcount for shared file (if any) to prevent early memory unlinking
+ * @param msg
+ */
+struct rspamd_storage_shmem* rspamd_http_message_shmem_ref (struct rspamd_http_message *msg);
+/**
+ * Decrease external ref for shmem segment associated with a message
+ * @param msg
+ */
+void rspamd_http_message_shmem_unref (struct rspamd_storage_shmem *p);
+
+/**
+ * Returns message's flags
+ * @param msg
+ * @return
+ */
+guint rspamd_http_message_get_flags (struct rspamd_http_message *msg);
+
+#endif
diff --git a/src/libutil/http_private.h b/src/libutil/http_private.h
index 0e2658617..dd3d0c6a9 100644
--- a/src/libutil/http_private.h
+++ b/src/libutil/http_private.h
@@ -16,9 +16,13 @@
#ifndef SRC_LIBUTIL_HTTP_PRIVATE_H_
#define SRC_LIBUTIL_HTTP_PRIVATE_H_
-#include "http.h"
+#include "http_connection.h"
+#include "http_parser.h"
#include "str_util.h"
+#include "keypair.h"
+#include "keypairs_cache.h"
#include "ref.h"
+#include "khash.h"
#define HASH_CASELESS
#include "uthash_strcase.h"
@@ -66,12 +70,43 @@ struct rspamd_http_message {
time_t date;
time_t last_modified;
unsigned port;
- enum http_parser_type type;
+ int type;
gint code;
enum http_method method;
gint flags;
ref_entry_t ref;
};
+struct rspamd_keepalive_hash_key {
+ rspamd_inet_addr_t *addr;
+ gchar *host;
+ GQueue conns;
+};
+
+gint32 rspamd_keep_alive_key_hash (struct rspamd_keepalive_hash_key* k);
+bool rspamd_keep_alive_key_equal (struct rspamd_keepalive_hash_key* k1,
+ struct rspamd_keepalive_hash_key* k2);
+
+KHASH_INIT (rspamd_keep_alive_hash, struct rspamd_keepalive_hash_key *,
+ char, 0, rspamd_keep_alive_key_hash, rspamd_keep_alive_key_equal);
+
+struct rspamd_http_context {
+ struct rspamd_http_context_cfg config;
+ struct rspamd_keypair_cache *client_kp_cache;
+ struct rspamd_cryptobox_keypair *client_kp;
+ struct rspamd_keypair_cache *server_kp_cache;
+ gpointer ssl_ctx;
+ gpointer ssl_ctx_noverify;
+ struct event_base *ev_base;
+ struct event client_rotate_ev;
+ khash_t (rspamd_keep_alive_hash) *keep_alive_hash;
+};
+
+#define HTTP_ERROR http_error_quark ()
+GQuark http_error_quark (void);
+
+void rspamd_http_message_storage_cleanup (struct rspamd_http_message *msg);
+gboolean rspamd_http_message_grow_body (struct rspamd_http_message *msg,
+ gsize len);
#endif /* SRC_LIBUTIL_HTTP_PRIVATE_H_ */
diff --git a/src/libutil/http_router.c b/src/libutil/http_router.c
new file mode 100644
index 000000000..570d3d5c6
--- /dev/null
+++ b/src/libutil/http_router.c
@@ -0,0 +1,555 @@
+/*-
+ * Copyright 2019 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "libutil/http_router.h"
+#include "libutil/http_connection.h"
+#include "libutil/http_private.h"
+#include "libutil/regexp.h"
+#include "libutil/printf.h"
+#include "libutil/logger.h"
+#include "utlist.h"
+#include "unix-std.h"
+
+enum http_magic_type {
+ HTTP_MAGIC_PLAIN = 0,
+ HTTP_MAGIC_HTML,
+ HTTP_MAGIC_CSS,
+ HTTP_MAGIC_JS,
+ HTTP_MAGIC_PNG,
+ HTTP_MAGIC_JPG
+};
+
+static const struct _rspamd_http_magic {
+ const gchar *ext;
+ const gchar *ct;
+} http_file_types[] = {
+ [HTTP_MAGIC_PLAIN] = { "txt", "text/plain" },
+ [HTTP_MAGIC_HTML] = { "html", "text/html" },
+ [HTTP_MAGIC_CSS] = { "css", "text/css" },
+ [HTTP_MAGIC_JS] = { "js", "application/javascript" },
+ [HTTP_MAGIC_PNG] = { "png", "image/png" },
+ [HTTP_MAGIC_JPG] = { "jpg", "image/jpeg" },
+};
+
+/*
+ * HTTP router functions
+ */
+
+static void
+rspamd_http_entry_free (struct rspamd_http_connection_entry *entry)
+{
+ if (entry != NULL) {
+ close (entry->conn->fd);
+ rspamd_http_connection_unref (entry->conn);
+ if (entry->rt->finish_handler) {
+ entry->rt->finish_handler (entry);
+ }
+
+ DL_DELETE (entry->rt->conns, entry);
+ g_free (entry);
+ }
+}
+
+static void
+rspamd_http_router_error_handler (struct rspamd_http_connection *conn,
+ GError *err)
+{
+ struct rspamd_http_connection_entry *entry = conn->ud;
+ struct rspamd_http_message *msg;
+
+ if (entry->is_reply) {
+ /* At this point we need to finish this session and close owned socket */
+ if (entry->rt->error_handler != NULL) {
+ entry->rt->error_handler (entry, err);
+ }
+ rspamd_http_entry_free (entry);
+ }
+ else {
+ /* Here we can write a reply to a client */
+ if (entry->rt->error_handler != NULL) {
+ entry->rt->error_handler (entry, err);
+ }
+ msg = rspamd_http_new_message (HTTP_RESPONSE);
+ msg->date = time (NULL);
+ msg->code = err->code;
+ rspamd_http_message_set_body (msg, err->message, strlen (err->message));
+ rspamd_http_connection_reset (entry->conn);
+ rspamd_http_connection_write_message (entry->conn,
+ msg,
+ NULL,
+ "text/plain",
+ entry,
+ entry->rt->ptv);
+ entry->is_reply = TRUE;
+ }
+}
+
+static const gchar *
+rspamd_http_router_detect_ct (const gchar *path)
+{
+ const gchar *dot;
+ guint i;
+
+ dot = strrchr (path, '.');
+ if (dot == NULL) {
+ return http_file_types[HTTP_MAGIC_PLAIN].ct;
+ }
+ dot++;
+
+ for (i = 0; i < G_N_ELEMENTS (http_file_types); i++) {
+ if (strcmp (http_file_types[i].ext, dot) == 0) {
+ return http_file_types[i].ct;
+ }
+ }
+
+ return http_file_types[HTTP_MAGIC_PLAIN].ct;
+}
+
+static gboolean
+rspamd_http_router_is_subdir (const gchar *parent, const gchar *sub)
+{
+ if (parent == NULL || sub == NULL || *parent == '\0') {
+ return FALSE;
+ }
+
+ while (*parent != '\0') {
+ if (*sub != *parent) {
+ return FALSE;
+ }
+ parent++;
+ sub++;
+ }
+
+ parent--;
+ if (*parent == G_DIR_SEPARATOR) {
+ return TRUE;
+ }
+
+ return (*sub == G_DIR_SEPARATOR || *sub == '\0');
+}
+
+static gboolean
+rspamd_http_router_try_file (struct rspamd_http_connection_entry *entry,
+ rspamd_ftok_t *lookup, gboolean expand_path)
+{
+ struct stat st;
+ gint fd;
+ gchar filebuf[PATH_MAX], realbuf[PATH_MAX], *dir;
+ struct rspamd_http_message *reply_msg;
+
+ rspamd_snprintf (filebuf, sizeof (filebuf), "%s%c%T",
+ entry->rt->default_fs_path, G_DIR_SEPARATOR, lookup);
+
+ if (realpath (filebuf, realbuf) == NULL ||
+ lstat (realbuf, &st) == -1) {
+ return FALSE;
+ }
+
+ if (S_ISDIR (st.st_mode) && expand_path) {
+ /* Try to append 'index.html' to the url */
+ rspamd_fstring_t *nlookup;
+ rspamd_ftok_t tok;
+ gboolean ret;
+
+ nlookup = rspamd_fstring_sized_new (lookup->len + sizeof ("index.html"));
+ rspamd_printf_fstring (&nlookup, "%T%c%s", lookup, G_DIR_SEPARATOR,
+ "index.html");
+ tok.begin = nlookup->str;
+ tok.len = nlookup->len;
+ ret = rspamd_http_router_try_file (entry, &tok, FALSE);
+ rspamd_fstring_free (nlookup);
+
+ return ret;
+ }
+ else if (!S_ISREG (st.st_mode)) {
+ return FALSE;
+ }
+
+ /* We also need to ensure that file is inside the defined dir */
+ rspamd_strlcpy (filebuf, realbuf, sizeof (filebuf));
+ dir = dirname (filebuf);
+
+ if (dir == NULL ||
+ !rspamd_http_router_is_subdir (entry->rt->default_fs_path,
+ dir)) {
+ return FALSE;
+ }
+
+ fd = open (realbuf, O_RDONLY);
+ if (fd == -1) {
+ return FALSE;
+ }
+
+ reply_msg = rspamd_http_new_message (HTTP_RESPONSE);
+ reply_msg->date = time (NULL);
+ reply_msg->code = 200;
+ rspamd_http_router_insert_headers (entry->rt, reply_msg);
+
+ if (!rspamd_http_message_set_body_from_fd (reply_msg, fd)) {
+ close (fd);
+ return FALSE;
+ }
+
+ close (fd);
+
+ rspamd_http_connection_reset (entry->conn);
+
+ msg_debug ("requested file %s", realbuf);
+ rspamd_http_connection_write_message (entry->conn, reply_msg, NULL,
+ rspamd_http_router_detect_ct (realbuf), entry,
+ entry->rt->ptv);
+
+ return TRUE;
+}
+
+static void
+rspamd_http_router_send_error (GError *err,
+ struct rspamd_http_connection_entry *entry)
+{
+ struct rspamd_http_message *err_msg;
+
+ err_msg = rspamd_http_new_message (HTTP_RESPONSE);
+ err_msg->date = time (NULL);
+ err_msg->code = err->code;
+ rspamd_http_message_set_body (err_msg, err->message,
+ strlen (err->message));
+ entry->is_reply = TRUE;
+ err_msg->status = rspamd_fstring_new_init (err->message, strlen (err->message));
+ rspamd_http_router_insert_headers (entry->rt, err_msg);
+ rspamd_http_connection_reset (entry->conn);
+ rspamd_http_connection_write_message (entry->conn,
+ err_msg,
+ NULL,
+ "text/plain",
+ entry,
+ entry->rt->ptv);
+}
+
+
+static int
+rspamd_http_router_finish_handler (struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_http_connection_entry *entry = conn->ud;
+ rspamd_http_router_handler_t handler = NULL;
+ gpointer found;
+
+ GError *err;
+ rspamd_ftok_t lookup;
+ const rspamd_ftok_t *encoding;
+ struct http_parser_url u;
+ guint i;
+ rspamd_regexp_t *re;
+ struct rspamd_http_connection_router *router;
+
+ G_STATIC_ASSERT (sizeof (rspamd_http_router_handler_t) ==
+ sizeof (gpointer));
+
+ memset (&lookup, 0, sizeof (lookup));
+ router = entry->rt;
+
+ if (entry->is_reply) {
+ /* Request is finished, it is safe to free a connection */
+ rspamd_http_entry_free (entry);
+ }
+ else {
+ if (G_UNLIKELY (msg->method != HTTP_GET && msg->method != HTTP_POST)) {
+ if (router->unknown_method_handler) {
+ return router->unknown_method_handler (entry, msg);
+ }
+ else {
+ err = g_error_new (HTTP_ERROR, 500,
+ "Invalid method");
+ if (entry->rt->error_handler != NULL) {
+ entry->rt->error_handler (entry, err);
+ }
+
+ rspamd_http_router_send_error (err, entry);
+ g_error_free (err);
+
+ return 0;
+ }
+ }
+
+ /* Search for path */
+ if (msg->url != NULL && msg->url->len != 0) {
+
+ http_parser_parse_url (msg->url->str, msg->url->len, TRUE, &u);
+
+ if (u.field_set & (1 << UF_PATH)) {
+ guint unnorm_len;
+ lookup.begin = msg->url->str + u.field_data[UF_PATH].off;
+ lookup.len = u.field_data[UF_PATH].len;
+
+ rspamd_http_normalize_path_inplace ((gchar *)lookup.begin,
+ lookup.len,
+ &unnorm_len);
+ lookup.len = unnorm_len;
+ }
+ else {
+ lookup.begin = msg->url->str;
+ lookup.len = msg->url->len;
+ }
+
+ found = g_hash_table_lookup (entry->rt->paths, &lookup);
+ memcpy (&handler, &found, sizeof (found));
+ msg_debug ("requested known path: %T", &lookup);
+ }
+ else {
+ err = g_error_new (HTTP_ERROR, 404,
+ "Empty path requested");
+ if (entry->rt->error_handler != NULL) {
+ entry->rt->error_handler (entry, err);
+ }
+
+ rspamd_http_router_send_error (err, entry);
+ g_error_free (err);
+
+ return 0;
+ }
+
+ entry->is_reply = TRUE;
+
+ encoding = rspamd_http_message_find_header (msg, "Accept-Encoding");
+
+ if (encoding && rspamd_substring_search (encoding->begin, encoding->len,
+ "gzip", 4) != -1) {
+ entry->support_gzip = TRUE;
+ }
+
+ if (handler != NULL) {
+ return handler (entry, msg);
+ }
+ else {
+ /* Try regexps */
+ for (i = 0; i < router->regexps->len; i ++) {
+ re = g_ptr_array_index (router->regexps, i);
+ if (rspamd_regexp_match (re, lookup.begin, lookup.len,
+ TRUE)) {
+ found = rspamd_regexp_get_ud (re);
+ memcpy (&handler, &found, sizeof (found));
+
+ return handler (entry, msg);
+ }
+ }
+
+ /* Now try plain file */
+ if (entry->rt->default_fs_path == NULL || lookup.len == 0 ||
+ !rspamd_http_router_try_file (entry, &lookup, TRUE)) {
+
+ err = g_error_new (HTTP_ERROR, 404,
+ "Not found");
+ if (entry->rt->error_handler != NULL) {
+ entry->rt->error_handler (entry, err);
+ }
+
+ msg_info ("path: %T not found", &lookup);
+ rspamd_http_router_send_error (err, entry);
+ g_error_free (err);
+ }
+ }
+ }
+
+ return 0;
+}
+
+struct rspamd_http_connection_router *
+rspamd_http_router_new (rspamd_http_router_error_handler_t eh,
+ rspamd_http_router_finish_handler_t fh,
+ struct timeval *timeout,
+ const char *default_fs_path,
+ struct rspamd_http_context *ctx)
+{
+ struct rspamd_http_connection_router * new;
+ struct stat st;
+
+ new = g_malloc0 (sizeof (struct rspamd_http_connection_router));
+ new->paths = g_hash_table_new_full (rspamd_ftok_icase_hash,
+ rspamd_ftok_icase_equal, rspamd_fstring_mapped_ftok_free, NULL);
+ new->regexps = g_ptr_array_new ();
+ new->conns = NULL;
+ new->error_handler = eh;
+ new->finish_handler = fh;
+ new->response_headers = g_hash_table_new_full (rspamd_strcase_hash,
+ rspamd_strcase_equal, g_free, g_free);
+ new->ev_base = ctx->ev_base;
+
+ if (timeout) {
+ new->tv = *timeout;
+ new->ptv = &new->tv;
+ }
+ else {
+ new->ptv = NULL;
+ }
+
+ new->default_fs_path = NULL;
+
+ if (default_fs_path != NULL) {
+ if (stat (default_fs_path, &st) == -1) {
+ msg_err ("cannot stat %s", default_fs_path);
+ }
+ else {
+ if (!S_ISDIR (st.st_mode)) {
+ msg_err ("path %s is not a directory", default_fs_path);
+ }
+ else {
+ new->default_fs_path = realpath (default_fs_path, NULL);
+ }
+ }
+ }
+
+ new->ctx = ctx;
+
+ return new;
+}
+
+void
+rspamd_http_router_set_key (struct rspamd_http_connection_router *router,
+ struct rspamd_cryptobox_keypair *key)
+{
+ g_assert (key != NULL);
+
+ router->key = rspamd_keypair_ref (key);
+}
+
+void
+rspamd_http_router_add_path (struct rspamd_http_connection_router *router,
+ const gchar *path, rspamd_http_router_handler_t handler)
+{
+ gpointer ptr;
+ rspamd_ftok_t *key;
+ rspamd_fstring_t *storage;
+ G_STATIC_ASSERT (sizeof (rspamd_http_router_handler_t) ==
+ sizeof (gpointer));
+
+ if (path != NULL && handler != NULL && router != NULL) {
+ memcpy (&ptr, &handler, sizeof (ptr));
+ storage = rspamd_fstring_new_init (path, strlen (path));
+ key = g_malloc0 (sizeof (*key));
+ key->begin = storage->str;
+ key->len = storage->len;
+ g_hash_table_insert (router->paths, key, ptr);
+ }
+}
+
+void
+rspamd_http_router_set_unknown_handler (struct rspamd_http_connection_router *router,
+ rspamd_http_router_handler_t handler)
+{
+ if (router != NULL) {
+ router->unknown_method_handler = handler;
+ }
+}
+
+void
+rspamd_http_router_add_header (struct rspamd_http_connection_router *router,
+ const gchar *name, const gchar *value)
+{
+ if (name != NULL && value != NULL && router != NULL) {
+ g_hash_table_replace (router->response_headers, g_strdup (name),
+ g_strdup (value));
+ }
+}
+
+void
+rspamd_http_router_insert_headers (struct rspamd_http_connection_router *router,
+ struct rspamd_http_message *msg)
+{
+ GHashTableIter it;
+ gpointer k, v;
+
+ if (router && msg) {
+ g_hash_table_iter_init (&it, router->response_headers);
+
+ while (g_hash_table_iter_next (&it, &k, &v)) {
+ rspamd_http_message_add_header (msg, k, v);
+ }
+ }
+}
+
+void
+rspamd_http_router_add_regexp (struct rspamd_http_connection_router *router,
+ struct rspamd_regexp_s *re, rspamd_http_router_handler_t handler)
+{
+ gpointer ptr;
+ G_STATIC_ASSERT (sizeof (rspamd_http_router_handler_t) ==
+ sizeof (gpointer));
+
+ if (re != NULL && handler != NULL && router != NULL) {
+ memcpy (&ptr, &handler, sizeof (ptr));
+ rspamd_regexp_set_ud (re, ptr);
+ g_ptr_array_add (router->regexps, rspamd_regexp_ref (re));
+ }
+}
+
+void
+rspamd_http_router_handle_socket (struct rspamd_http_connection_router *router,
+ gint fd, gpointer ud)
+{
+ struct rspamd_http_connection_entry *conn;
+
+ conn = g_malloc0 (sizeof (struct rspamd_http_connection_entry));
+ conn->rt = router;
+ conn->ud = ud;
+ conn->is_reply = FALSE;
+
+ conn->conn = rspamd_http_connection_new (router->ctx,
+ fd,
+ NULL,
+ rspamd_http_router_error_handler,
+ rspamd_http_router_finish_handler,
+ 0,
+ RSPAMD_HTTP_SERVER);
+
+ if (router->key) {
+ rspamd_http_connection_set_key (conn->conn, router->key);
+ }
+
+ rspamd_http_connection_read_message (conn->conn, conn, router->ptv);
+ DL_PREPEND (router->conns, conn);
+}
+
+void
+rspamd_http_router_free (struct rspamd_http_connection_router *router)
+{
+ struct rspamd_http_connection_entry *conn, *tmp;
+ rspamd_regexp_t *re;
+ guint i;
+
+ if (router) {
+ DL_FOREACH_SAFE (router->conns, conn, tmp) {
+ rspamd_http_entry_free (conn);
+ }
+
+ if (router->key) {
+ rspamd_keypair_unref (router->key);
+ }
+
+ if (router->default_fs_path != NULL) {
+ g_free (router->default_fs_path);
+ }
+
+ for (i = 0; i < router->regexps->len; i ++) {
+ re = g_ptr_array_index (router->regexps, i);
+ rspamd_regexp_unref (re);
+ }
+
+ g_ptr_array_free (router->regexps, TRUE);
+ g_hash_table_unref (router->paths);
+ g_hash_table_unref (router->response_headers);
+ g_free (router);
+ }
+} \ No newline at end of file
diff --git a/src/libutil/http_router.h b/src/libutil/http_router.h
new file mode 100644
index 000000000..8e8056240
--- /dev/null
+++ b/src/libutil/http_router.h
@@ -0,0 +1,138 @@
+/*-
+ * Copyright 2019 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_HTTP_ROUTER_H
+#define RSPAMD_HTTP_ROUTER_H
+
+#include "config.h"
+#include "http_connection.h"
+
+struct rspamd_http_connection_router;
+struct rspamd_http_connection_entry;
+
+typedef int (*rspamd_http_router_handler_t) (struct rspamd_http_connection_entry
+ *conn_ent,
+ struct rspamd_http_message *msg);
+typedef void (*rspamd_http_router_error_handler_t) (struct rspamd_http_connection_entry *conn_ent,
+ GError *err);
+typedef void (*rspamd_http_router_finish_handler_t) (struct rspamd_http_connection_entry *conn_ent);
+
+
+struct rspamd_http_connection_entry {
+ struct rspamd_http_connection_router *rt;
+ struct rspamd_http_connection *conn;
+ gpointer ud;
+ gboolean is_reply;
+ gboolean support_gzip;
+ struct rspamd_http_connection_entry *prev, *next;
+};
+
+struct rspamd_http_connection_router {
+ struct rspamd_http_connection_entry *conns;
+ GHashTable *paths;
+ GHashTable *response_headers;
+ GPtrArray *regexps;
+ struct timeval tv;
+ struct timeval *ptv;
+ struct event_base *ev_base;
+ struct rspamd_http_context *ctx;
+ gchar *default_fs_path;
+ rspamd_http_router_handler_t unknown_method_handler;
+ struct rspamd_cryptobox_keypair *key;
+ rspamd_http_router_error_handler_t error_handler;
+ rspamd_http_router_finish_handler_t finish_handler;
+};
+
+/**
+ * Create new http connection router and the associated HTTP connection
+ * @param eh error handler callback
+ * @param fh finish handler callback
+ * @param default_fs_path if not NULL try to serve static files from
+ * the specified directory
+ * @return
+ */
+struct rspamd_http_connection_router * rspamd_http_router_new (
+ rspamd_http_router_error_handler_t eh,
+ rspamd_http_router_finish_handler_t fh,
+ struct timeval *timeout,
+ const char *default_fs_path,
+ struct rspamd_http_context *ctx);
+
+/**
+ * Set encryption key for the HTTP router
+ * @param router router structure
+ * @param key opaque key structure
+ */
+void rspamd_http_router_set_key (struct rspamd_http_connection_router *router,
+ struct rspamd_cryptobox_keypair *key);
+
+/**
+ * Add new path to the router
+ */
+void rspamd_http_router_add_path (struct rspamd_http_connection_router *router,
+ const gchar *path, rspamd_http_router_handler_t handler);
+
+/**
+ * Add custom header to append to router replies
+ * @param router
+ * @param name
+ * @param value
+ */
+void rspamd_http_router_add_header (struct rspamd_http_connection_router *router,
+ const gchar *name, const gchar *value);
+
+/**
+ * Sets method to handle unknown request methods
+ * @param router
+ * @param handler
+ */
+void rspamd_http_router_set_unknown_handler (struct rspamd_http_connection_router *router,
+ rspamd_http_router_handler_t handler);
+
+/**
+ * Inserts router headers to the outbound message
+ * @param router
+ * @param msg
+ */
+void rspamd_http_router_insert_headers (struct rspamd_http_connection_router *router,
+ struct rspamd_http_message *msg);
+
+struct rspamd_regexp_s;
+/**
+ * Adds new pattern to router, regexp object is refcounted by this function
+ * @param router
+ * @param re
+ * @param handler
+ */
+void rspamd_http_router_add_regexp (struct rspamd_http_connection_router *router,
+ struct rspamd_regexp_s *re, rspamd_http_router_handler_t handler);
+/**
+ * Handle new accepted socket
+ * @param router router object
+ * @param fd server socket
+ * @param ud opaque userdata
+ */
+void rspamd_http_router_handle_socket (
+ struct rspamd_http_connection_router *router,
+ gint fd,
+ gpointer ud);
+
+/**
+ * Free router and all connections associated
+ * @param router
+ */
+void rspamd_http_router_free (struct rspamd_http_connection_router *router);
+
+#endif
diff --git a/src/libutil/http_util.c b/src/libutil/http_util.c
new file mode 100644
index 000000000..8fb658e08
--- /dev/null
+++ b/src/libutil/http_util.c
@@ -0,0 +1,513 @@
+/*-
+ * Copyright 2019 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "libutil/http_util.h"
+#include "libutil/printf.h"
+#include "libutil/util.h"
+
+static const gchar *http_week[] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" };
+static const gchar *http_month[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun",
+ "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" };
+
+/*
+ * Obtained from nginx
+ * Copyright (C) Igor Sysoev
+ */
+static guint mday[] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 };
+
+time_t
+rspamd_http_parse_date (const gchar *header, gsize len)
+{
+ const gchar *p, *end;
+ gint month;
+ guint day, year, hour, min, sec;
+ guint64 time;
+ enum {
+ no = 0, rfc822, /* Tue, 10 Nov 2002 23:50:13 */
+ rfc850, /* Tuesday, 10-Dec-02 23:50:13 */
+ isoc /* Tue Dec 10 23:50:13 2002 */
+ } fmt;
+
+ fmt = 0;
+ if (len > 0) {
+ end = header + len;
+ }
+ else {
+ end = header + strlen (header);
+ }
+
+ day = 32;
+ year = 2038;
+
+ for (p = header; p < end; p++) {
+ if (*p == ',') {
+ break;
+ }
+
+ if (*p == ' ') {
+ fmt = isoc;
+ break;
+ }
+ }
+
+ for (p++; p < end; p++)
+ if (*p != ' ') {
+ break;
+ }
+
+ if (end - p < 18) {
+ return (time_t)-1;
+ }
+
+ if (fmt != isoc) {
+ if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') {
+ return (time_t)-1;
+ }
+
+ day = (*p - '0') * 10 + *(p + 1) - '0';
+ p += 2;
+
+ if (*p == ' ') {
+ if (end - p < 18) {
+ return (time_t)-1;
+ }
+ fmt = rfc822;
+
+ }
+ else if (*p == '-') {
+ fmt = rfc850;
+
+ }
+ else {
+ return (time_t)-1;
+ }
+
+ p++;
+ }
+
+ switch (*p) {
+
+ case 'J':
+ month = *(p + 1) == 'a' ? 0 : *(p + 2) == 'n' ? 5 : 6;
+ break;
+
+ case 'F':
+ month = 1;
+ break;
+
+ case 'M':
+ month = *(p + 2) == 'r' ? 2 : 4;
+ break;
+
+ case 'A':
+ month = *(p + 1) == 'p' ? 3 : 7;
+ break;
+
+ case 'S':
+ month = 8;
+ break;
+
+ case 'O':
+ month = 9;
+ break;
+
+ case 'N':
+ month = 10;
+ break;
+
+ case 'D':
+ month = 11;
+ break;
+
+ default:
+ return (time_t)-1;
+ }
+
+ p += 3;
+
+ if ((fmt == rfc822 && *p != ' ') || (fmt == rfc850 && *p != '-')) {
+ return (time_t)-1;
+ }
+
+ p++;
+
+ if (fmt == rfc822) {
+ if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9'
+ || *(p + 2) < '0' || *(p + 2) > '9' || *(p + 3) < '0'
+ || *(p + 3) > '9') {
+ return (time_t)-1;
+ }
+
+ year = (*p - '0') * 1000 + (*(p + 1) - '0') * 100
+ + (*(p + 2) - '0') * 10 + *(p + 3) - '0';
+ p += 4;
+
+ }
+ else if (fmt == rfc850) {
+ if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') {
+ return (time_t)-1;
+ }
+
+ year = (*p - '0') * 10 + *(p + 1) - '0';
+ year += (year < 70) ? 2000 : 1900;
+ p += 2;
+ }
+
+ if (fmt == isoc) {
+ if (*p == ' ') {
+ p++;
+ }
+
+ if (*p < '0' || *p > '9') {
+ return (time_t)-1;
+ }
+
+ day = *p++ - '0';
+
+ if (*p != ' ') {
+ if (*p < '0' || *p > '9') {
+ return (time_t)-1;
+ }
+
+ day = day * 10 + *p++ - '0';
+ }
+
+ if (end - p < 14) {
+ return (time_t)-1;
+ }
+ }
+
+ if (*p++ != ' ') {
+ return (time_t)-1;
+ }
+
+ if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') {
+ return (time_t)-1;
+ }
+
+ hour = (*p - '0') * 10 + *(p + 1) - '0';
+ p += 2;
+
+ if (*p++ != ':') {
+ return (time_t)-1;
+ }
+
+ if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') {
+ return (time_t)-1;
+ }
+
+ min = (*p - '0') * 10 + *(p + 1) - '0';
+ p += 2;
+
+ if (*p++ != ':') {
+ return (time_t)-1;
+ }
+
+ if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') {
+ return (time_t)-1;
+ }
+
+ sec = (*p - '0') * 10 + *(p + 1) - '0';
+
+ if (fmt == isoc) {
+ p += 2;
+
+ if (*p++ != ' ') {
+ return (time_t)-1;
+ }
+
+ if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9'
+ || *(p + 2) < '0' || *(p + 2) > '9' || *(p + 3) < '0'
+ || *(p + 3) > '9') {
+ return (time_t)-1;
+ }
+
+ year = (*p - '0') * 1000 + (*(p + 1) - '0') * 100
+ + (*(p + 2) - '0') * 10 + *(p + 3) - '0';
+ }
+
+ if (hour > 23 || min > 59 || sec > 59) {
+ return (time_t)-1;
+ }
+
+ if (day == 29 && month == 1) {
+ if ((year & 3) || ((year % 100 == 0) && (year % 400) != 0)) {
+ return (time_t)-1;
+ }
+
+ }
+ else if (day > mday[month]) {
+ return (time_t)-1;
+ }
+
+ /*
+ * shift new year to March 1 and start months from 1 (not 0),
+ * it is needed for Gauss' formula
+ */
+
+ if (--month <= 0) {
+ month += 12;
+ year -= 1;
+ }
+
+ /* Gauss' formula for Gregorian days since March 1, 1 BC */
+
+ time = (guint64) (
+ /* days in years including leap years since March 1, 1 BC */
+
+ 365 * year + year / 4 - year / 100 + year / 400
+
+ /* days before the month */
+
+ + 367 * month / 12 - 30
+
+ /* days before the day */
+
+ + day - 1
+
+ /*
+ * 719527 days were between March 1, 1 BC and March 1, 1970,
+ * 31 and 28 days were in January and February 1970
+ */
+
+ - 719527 + 31 + 28) * 86400 + hour * 3600 + min * 60 + sec;
+
+ return (time_t) time;
+}
+
+glong
+rspamd_http_date_format (gchar *buf, gsize len, time_t time)
+{
+ struct tm tms;
+
+ rspamd_gmtime (time, &tms);
+
+ return rspamd_snprintf (buf, len, "%s, %02d %s %4d %02d:%02d:%02d GMT",
+ http_week[tms.tm_wday], tms.tm_mday,
+ http_month[tms.tm_mon], tms.tm_year + 1900,
+ tms.tm_hour, tms.tm_min, tms.tm_sec);
+}
+
+void
+rspamd_http_normalize_path_inplace (gchar *path, guint len, guint *nlen)
+{
+ const gchar *p, *end, *slash = NULL, *dot = NULL;
+ gchar *o;
+ enum {
+ st_normal = 0,
+ st_got_dot,
+ st_got_dot_dot,
+ st_got_slash,
+ st_got_slash_slash,
+ } state = st_normal;
+
+ p = path;
+ end = path + len;
+ o = path;
+
+ while (p < end) {
+ switch (state) {
+ case st_normal:
+ if (G_UNLIKELY (*p == '/')) {
+ state = st_got_slash;
+ slash = p;
+ }
+ else if (G_UNLIKELY (*p == '.')) {
+ state = st_got_dot;
+ dot = p;
+ }
+ else {
+ *o++ = *p;
+ }
+ p ++;
+ break;
+ case st_got_slash:
+ if (G_UNLIKELY (*p == '/')) {
+ /* Ignore double slash */
+ *o++ = *p;
+ state = st_got_slash_slash;
+ }
+ else if (G_UNLIKELY (*p == '.')) {
+ dot = p;
+ state = st_got_dot;
+ }
+ else {
+ *o++ = '/';
+ *o++ = *p;
+ slash = NULL;
+ dot = NULL;
+ state = st_normal;
+ }
+ p ++;
+ break;
+ case st_got_slash_slash:
+ if (G_LIKELY (*p != '/')) {
+ slash = p - 1;
+ dot = NULL;
+ state = st_normal;
+ continue;
+ }
+ p ++;
+ break;
+ case st_got_dot:
+ if (G_UNLIKELY (*p == '/')) {
+ /* Remove any /./ or ./ paths */
+ if (((o > path && *(o - 1) != '/') || (o == path)) && slash) {
+ /* Preserve one slash */
+ *o++ = '/';
+ }
+
+ slash = p;
+ dot = NULL;
+ /* Ignore last slash */
+ state = st_normal;
+ }
+ else if (*p == '.') {
+ /* Double dot character */
+ state = st_got_dot_dot;
+ }
+ else {
+ /* We have something like .some or /.some */
+ if (dot && p > dot) {
+ if (slash == dot - 1 && (o > path && *(o - 1) != '/')) {
+ /* /.blah */
+ memmove (o, slash, p - slash);
+ o += p - slash;
+ }
+ else {
+ memmove (o, dot, p - dot);
+ o += p - dot;
+ }
+ }
+
+ slash = NULL;
+ dot = NULL;
+ state = st_normal;
+ continue;
+ }
+
+ p ++;
+ break;
+ case st_got_dot_dot:
+ if (*p == '/') {
+ /* We have something like /../ or ../ */
+ if (slash) {
+ /* We need to remove the last component from o if it is there */
+ if (o > path + 2 && *(o - 1) == '/') {
+ slash = rspamd_memrchr (path, '/', o - path - 2);
+ }
+ else if (o > path + 1) {
+ slash = rspamd_memrchr (path, '/', o - path - 1);
+ }
+ else {
+ slash = NULL;
+ }
+
+ if (slash) {
+ o = (gchar *)slash;
+ }
+ /* Otherwise we keep these dots */
+ slash = p;
+ state = st_got_slash;
+ }
+ else {
+ /* We have something like bla../, so we need to copy it as is */
+ if (o > path && dot && p > dot) {
+ memmove (o, dot, p - dot);
+ o += p - dot;
+ }
+
+ slash = NULL;
+ dot = NULL;
+ state = st_normal;
+ continue;
+ }
+ }
+ else {
+ /* We have something like ..bla or ... */
+ if (slash) {
+ *o ++ = '/';
+ }
+
+ if (dot && p > dot) {
+ memmove (o, dot, p - dot);
+ o += p - dot;
+ }
+
+ slash = NULL;
+ dot = NULL;
+ state = st_normal;
+ continue;
+ }
+
+ p ++;
+ break;
+ }
+ }
+
+ /* Leftover */
+ switch (state) {
+ case st_got_dot_dot:
+ /* Trailing .. */
+ if (slash) {
+ /* We need to remove the last component from o if it is there */
+ if (o > path + 2 && *(o - 1) == '/') {
+ slash = rspamd_memrchr (path, '/', o - path - 2);
+ }
+ else if (o > path + 1) {
+ slash = rspamd_memrchr (path, '/', o - path - 1);
+ }
+ else {
+ if (o == path) {
+ /* Corner case */
+ *o++ = '/';
+ }
+
+ slash = NULL;
+ }
+
+ if (slash) {
+ /* Remove last / */
+ o = (gchar *)slash;
+ }
+ }
+ else {
+ /* Corner case */
+ if (o == path) {
+ *o++ = '/';
+ }
+ else {
+ if (dot && p > dot) {
+ memmove (o, dot, p - dot);
+ o += p - dot;
+ }
+ }
+ }
+ break;
+ case st_got_slash:
+ *o++ = '/';
+ break;
+ default:
+ if (o > path + 1 && *(o - 1) == '/') {
+ o --;
+ }
+ break;
+ }
+
+ if (nlen) {
+ *nlen = (o - path);
+ }
+} \ No newline at end of file
diff --git a/src/libutil/http_util.h b/src/libutil/http_util.h
new file mode 100644
index 000000000..30d806ae0
--- /dev/null
+++ b/src/libutil/http_util.h
@@ -0,0 +1,48 @@
+/*-
+ * Copyright 2019 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_HTTP_UTIL_H
+#define RSPAMD_HTTP_UTIL_H
+
+#include "config.h"
+
+/**
+ * Parse HTTP date header and return it as time_t
+ * @param header HTTP date header
+ * @param len length of header
+ * @return time_t or (time_t)-1 in case of error
+ */
+time_t rspamd_http_parse_date (const gchar *header, gsize len);
+
+/**
+ * Prints HTTP date from `time` to `buf` using standard HTTP date format
+ * @param buf date buffer
+ * @param len length of buffer
+ * @param time time in unix seconds
+ * @return number of bytes written
+ */
+glong rspamd_http_date_format (gchar *buf, gsize len, time_t time);
+
+/**
+ * Normalize HTTP path removing dot sequences and repeating '/' symbols as
+ * per rfc3986#section-5.2
+ * @param path
+ * @param len
+ * @param nlen
+ */
+void rspamd_http_normalize_path_inplace (gchar *path, guint len, guint *nlen);
+
+#endif
diff --git a/src/libutil/map.c b/src/libutil/map.c
index f13f484d8..d4687e433 100644
--- a/src/libutil/map.c
+++ b/src/libutil/map.c
@@ -19,7 +19,7 @@
#include "config.h"
#include "map.h"
#include "map_private.h"
-#include "http.h"
+#include "http_connection.h"
#include "http_private.h"
#include "rspamd.h"
#include "contrib/zstd/zstd.h"
@@ -138,9 +138,7 @@ write_http_request (struct http_callback_data *cbd)
cbd->data->host,
NULL,
cbd,
- cbd->fd,
- &cbd->tv,
- cbd->ev_base);
+ &cbd->tv);
}
else {
msg_err_map ("cannot connect to %s: %s", cbd->data->host,
@@ -1155,11 +1153,7 @@ rspamd_map_periodic_dtor (struct map_periodic_cbdata *periodic)
if (periodic->need_modify) {
/* We are done */
- periodic->map->fin_callback (&periodic->cbdata);
-
- if (periodic->cbdata.cur_data) {
- *periodic->map->user_data = periodic->cbdata.cur_data;
- }
+ periodic->map->fin_callback (&periodic->cbdata, periodic->map->user_data);
}
else {
/* Not modified */
@@ -1277,12 +1271,12 @@ rspamd_map_dns_callback (struct rdns_reply *reply, void *arg)
if (cbd->fd != -1) {
cbd->stage = map_load_file;
cbd->conn = rspamd_http_connection_new (NULL,
+ cbd->fd,
+ NULL,
http_map_error,
http_map_finish,
flags,
- RSPAMD_HTTP_CLIENT,
- NULL,
- cbd->map->cfg->libs_ctx->ssl_ctx);
+ RSPAMD_HTTP_CLIENT);
write_http_request (cbd);
}
@@ -1649,13 +1643,14 @@ check:
if (cbd->fd != -1) {
cbd->stage = map_load_file;
- cbd->conn = rspamd_http_connection_new (NULL,
+ cbd->conn = rspamd_http_connection_new (
+ NULL,
+ cbd->fd,
+ NULL,
http_map_error,
http_map_finish,
flags,
- RSPAMD_HTTP_CLIENT,
- NULL,
- cbd->map->cfg->libs_ctx->ssl_ctx);
+ RSPAMD_HTTP_CLIENT);
write_http_request (cbd);
MAP_RELEASE (cbd, "http_callback_data");
@@ -2039,11 +2034,7 @@ rspamd_map_preload (struct rspamd_config *cfg)
}
if (succeed) {
- map->fin_callback (&fake_cbd.cbdata);
-
- if (fake_cbd.cbdata.cur_data) {
- *map->user_data = fake_cbd.cbdata.cur_data;
- }
+ map->fin_callback (&fake_cbd.cbdata, map->user_data);
}
else {
msg_info_map ("preload of %s failed", map->name);
diff --git a/src/libutil/map.h b/src/libutil/map.h
index 80aa03825..acf6eea4e 100644
--- a/src/libutil/map.h
+++ b/src/libutil/map.h
@@ -21,7 +21,7 @@ struct map_cb_data;
*/
typedef gchar * (*map_cb_t)(gchar *chunk, gint len,
struct map_cb_data *data, gboolean final);
-typedef void (*map_fin_cb_t)(struct map_cb_data *data);
+typedef void (*map_fin_cb_t)(struct map_cb_data *data, void **target);
typedef void (*map_dtor_t)(struct map_cb_data *data);
typedef gboolean (*rspamd_map_traverse_cb)(gconstpointer key,
diff --git a/src/libutil/map_helpers.c b/src/libutil/map_helpers.c
index ece3017ab..964e4d98c 100644
--- a/src/libutil/map_helpers.c
+++ b/src/libutil/map_helpers.c
@@ -504,17 +504,25 @@ rspamd_map_helper_insert_hash (gpointer st, gconstpointer key, gconstpointer val
gsize vlen;
gint r;
- vlen = strlen (value);
- val = rspamd_mempool_alloc0 (ht->pool, sizeof (*val) +
- vlen + 1);
- memcpy (val->value, value, vlen);
-
k = kh_get (rspamd_map_hash, ht->htb, key);
+ vlen = strlen (value);
if (k == kh_end (ht->htb)) {
nk = rspamd_mempool_strdup (ht->pool, key);
k = kh_put (rspamd_map_hash, ht->htb, nk, &r);
}
+ else {
+ val = kh_value (ht->htb, k);
+
+ if (strcmp (value, val->value) == 0) {
+ /* Same element, skip */
+ return;
+ }
+ }
+
+ /* Null termination due to alloc0 */
+ val = rspamd_mempool_alloc0 (ht->pool, sizeof (*val) + vlen + 1);
+ memcpy (val->value, value, vlen);
nk = kh_key (ht->htb, k);
val->key = nk;
@@ -815,16 +823,11 @@ rspamd_kv_list_read (
}
void
-rspamd_kv_list_fin (struct map_cb_data *data)
+rspamd_kv_list_fin (struct map_cb_data *data, void **target)
{
struct rspamd_map *map = data->map;
struct rspamd_hash_map_helper *htb;
- if (data->prev_data) {
- htb = (struct rspamd_hash_map_helper *)data->prev_data;
- rspamd_map_helper_destroy_hash (htb);
- }
-
if (data->cur_data) {
htb = (struct rspamd_hash_map_helper *)data->cur_data;
msg_info_map ("read hash of %d elements", kh_size (htb->htb));
@@ -832,6 +835,15 @@ rspamd_kv_list_fin (struct map_cb_data *data)
data->map->nelts = kh_size (htb->htb);
data->map->digest = rspamd_cryptobox_fast_hash_final (&htb->hst);
}
+
+ if (target) {
+ *target = data->cur_data;
+ }
+
+ if (data->prev_data) {
+ htb = (struct rspamd_hash_map_helper *)data->prev_data;
+ rspamd_map_helper_destroy_hash (htb);
+ }
}
void
@@ -870,16 +882,11 @@ rspamd_radix_read (
}
void
-rspamd_radix_fin (struct map_cb_data *data)
+rspamd_radix_fin (struct map_cb_data *data, void **target)
{
struct rspamd_map *map = data->map;
struct rspamd_radix_map_helper *r;
- if (data->prev_data) {
- r = (struct rspamd_radix_map_helper *)data->prev_data;
- rspamd_map_helper_destroy_radix (r);
- }
-
if (data->cur_data) {
r = (struct rspamd_radix_map_helper *)data->cur_data;
msg_info_map ("read radix trie of %z elements: %s",
@@ -888,6 +895,15 @@ rspamd_radix_fin (struct map_cb_data *data)
data->map->nelts = kh_size (r->htb);
data->map->digest = rspamd_cryptobox_fast_hash_final (&r->hst);
}
+
+ if (target) {
+ *target = data->cur_data;
+ }
+
+ if (data->prev_data) {
+ r = (struct rspamd_radix_map_helper *)data->prev_data;
+ rspamd_map_helper_destroy_radix (r);
+ }
}
void
@@ -1088,14 +1104,11 @@ rspamd_glob_list_read_multiple (
void
-rspamd_regexp_list_fin (struct map_cb_data *data)
+rspamd_regexp_list_fin (struct map_cb_data *data, void **target)
{
struct rspamd_regexp_map_helper *re_map;
struct rspamd_map *map = data->map;
- if (data->prev_data) {
- rspamd_map_helper_destroy_regexp (data->prev_data);
- }
if (data->cur_data) {
re_map = data->cur_data;
rspamd_re_map_finalize (re_map);
@@ -1105,6 +1118,14 @@ rspamd_regexp_list_fin (struct map_cb_data *data)
data->map->nelts = kh_size (re_map->htb);
data->map->digest = rspamd_cryptobox_fast_hash_final (&re_map->hst);
}
+
+ if (target) {
+ *target = data->cur_data;
+ }
+
+ if (data->prev_data) {
+ rspamd_map_helper_destroy_regexp (data->prev_data);
+ }
}
void
rspamd_regexp_list_dtor (struct map_cb_data *data)
diff --git a/src/libutil/map_helpers.h b/src/libutil/map_helpers.h
index f7c86436f..4645024e3 100644
--- a/src/libutil/map_helpers.h
+++ b/src/libutil/map_helpers.h
@@ -52,7 +52,7 @@ gchar * rspamd_radix_read (
gint len,
struct map_cb_data *data,
gboolean final);
-void rspamd_radix_fin (struct map_cb_data *data);
+void rspamd_radix_fin (struct map_cb_data *data, void **target);
void rspamd_radix_dtor (struct map_cb_data *data);
/**
@@ -63,7 +63,7 @@ gchar * rspamd_kv_list_read (
gint len,
struct map_cb_data *data,
gboolean final);
-void rspamd_kv_list_fin (struct map_cb_data *data);
+void rspamd_kv_list_fin (struct map_cb_data *data, void **target);
void rspamd_kv_list_dtor (struct map_cb_data *data);
/**
@@ -91,7 +91,7 @@ gchar * rspamd_glob_list_read_multiple (
gint len,
struct map_cb_data *data,
gboolean final);
-void rspamd_regexp_list_fin (struct map_cb_data *data);
+void rspamd_regexp_list_fin (struct map_cb_data *data, void **target);
void rspamd_regexp_list_dtor (struct map_cb_data *data);
/**
diff --git a/src/libutil/multipattern.c b/src/libutil/multipattern.c
index e4a39d5fe..268170512 100644
--- a/src/libutil/multipattern.c
+++ b/src/libutil/multipattern.c
@@ -99,8 +99,8 @@ rspamd_multipattern_escape_tld_hyperscan (const gchar *pattern, gsize slen,
/*
* We understand the following cases
- * 1) blah -> \\.blah
- * 2) *.blah -> \\..*\\.blah
+ * 1) blah -> .blah
+ * 2) *.blah -> ..*\\.blah
* 3) ???
*/
@@ -116,11 +116,11 @@ rspamd_multipattern_escape_tld_hyperscan (const gchar *pattern, gsize slen,
p ++;
}
- prefix = "\\..*\\.";
+ prefix = ".*.";
}
else {
- len = slen + 2;
- prefix = "\\.";
+ len = slen + 1;
+ prefix = ".";
p = pattern;
}
@@ -200,11 +200,18 @@ rspamd_multipattern_pattern_filter (const gchar *pattern, gsize len,
}
if (flags & RSPAMD_MULTIPATTERN_TLD) {
- ret = rspamd_multipattern_escape_tld_hyperscan (pattern, len, dst_len);
+ gchar *tmp;
+ gsize tlen;
+ tmp = rspamd_multipattern_escape_tld_hyperscan (pattern, len, &tlen);
+
+ ret = rspamd_str_regexp_escape (tmp, tlen, dst_len,
+ gl_flags|RSPAMD_REGEXP_ESCAPE_GLOB);
+ g_free (tmp);
}
else if (flags & RSPAMD_MULTIPATTERN_RE) {
ret = malloc (len + 1);
- *dst_len = rspamd_strlcpy (ret, pattern, len + 1);
+ ret = rspamd_str_regexp_escape (pattern, len, dst_len, gl_flags |
+ RSPAMD_REGEXP_ESCAPE_RE);
}
else if (flags & RSPAMD_MULTIPATTERN_GLOB) {
ret = rspamd_str_regexp_escape (pattern, len, dst_len,
diff --git a/src/libutil/str_util.c b/src/libutil/str_util.c
index 06d7a6cc7..ac7471ada 100644
--- a/src/libutil/str_util.c
+++ b/src/libutil/str_util.c
@@ -734,18 +734,7 @@ rspamd_decode_base32 (const gchar *in, gsize inlen, gsize *outlen)
}
-
-/**
- * Decode string using base32 encoding
- * @param in input
- * @param inlen input length
- * @param out output buf (may overlap with `in`)
- * @param outlen output buf len
- * @return TRUE if in is valid base32 and `outlen` is enough to encode `inlen`
- */
-
-
-static gchar *
+gchar *
rspamd_encode_base64_common (const guchar *in, gsize inlen, gint str_len,
gsize *outlen, gboolean fold, enum rspamd_newlines_type how)
{
@@ -2605,7 +2594,7 @@ rspamd_str_regexp_escape (const gchar *pattern, gsize slen,
gsize *dst_len, enum rspamd_regexp_escape_flags flags)
{
const gchar *p, *end = pattern + slen;
- gchar *res, *d, t, *tmp_utf = NULL;
+ gchar *res, *d, t, *tmp_utf = NULL, *dend;
gsize len;
static const gchar hexdigests[16] = "0123456789abcdef";
@@ -2634,15 +2623,22 @@ rspamd_str_regexp_escape (const gchar *pattern, gsize slen,
case '$':
case '|':
case '#':
- len ++;
+ if (!(flags & RSPAMD_REGEXP_ESCAPE_RE)) {
+ len++;
+ }
break;
default:
if (g_ascii_isspace (t)) {
len ++;
}
else {
- if (!(flags & RSPAMD_REGEXP_ESCAPE_UTF)) {
- if (!g_ascii_isprint (t)) {
+ if (!g_ascii_isprint (t) || (t & 0x80)) {
+
+ if (flags & RSPAMD_REGEXP_ESCAPE_UTF) {
+ /* \x{code}, where code can be up to 5 digits */
+ len += 4;
+ }
+ else {
/* \\xHH -> 4 symbols */
len += 3;
}
@@ -2668,8 +2664,6 @@ rspamd_str_regexp_escape (const gchar *pattern, gsize slen,
*dst_len = slen;
}
-
-
if (tmp_utf) {
return tmp_utf;
}
@@ -2685,8 +2679,10 @@ rspamd_str_regexp_escape (const gchar *pattern, gsize slen,
res = g_malloc (len + 1);
p = pattern;
d = res;
+ dend = d + len;
while (p < end) {
+ g_assert (d < dend);
t = *p ++;
switch (t) {
@@ -2704,7 +2700,9 @@ rspamd_str_regexp_escape (const gchar *pattern, gsize slen,
case '$':
case '|':
case '#':
- *d++ = '\\';
+ if (!(flags & RSPAMD_REGEXP_ESCAPE_RE)) {
+ *d++ = '\\';
+ }
break;
case '*':
case '?':
@@ -2714,19 +2712,40 @@ rspamd_str_regexp_escape (const gchar *pattern, gsize slen,
*d++ = '.';
}
else {
- *d++ = '\\';
+ if (!(flags & RSPAMD_REGEXP_ESCAPE_RE)) {
+ *d++ = '\\';
+ }
}
break;
default:
if (g_ascii_isspace (t)) {
- *d++ = '\\';
+ if (!(flags & RSPAMD_REGEXP_ESCAPE_RE)) {
+ *d++ = '\\';
+ }
}
- else if (!(flags & RSPAMD_REGEXP_ESCAPE_UTF) && !g_ascii_isgraph (t)) {
- *d++ = '\\';
- *d++ = 'x';
- *d++ = hexdigests[((t >> 4) & 0xF)];
- *d++ = hexdigests[((t) & 0xF)];
- continue; /* To avoid *d++ = t; */
+ else if (t & 0x80 || !g_ascii_isprint (t)) {
+ if (!(flags & RSPAMD_REGEXP_ESCAPE_UTF)) {
+ *d++ = '\\';
+ *d++ = 'x';
+ *d++ = hexdigests[((t >> 4) & 0xF)];
+ *d++ = hexdigests[((t) & 0xF)];
+ continue; /* To avoid *d++ = t; */
+ }
+ else {
+ if (flags & (RSPAMD_REGEXP_ESCAPE_RE|RSPAMD_REGEXP_ESCAPE_GLOB)) {
+ UChar32 uc;
+ gint32 off = p - pattern - 1;
+ U8_NEXT (pattern, off, slen, uc);
+
+ if (uc > 0) {
+ d += rspamd_snprintf (d, dend - d,
+ "\\x{%xd}", uc);
+ p = pattern + off;
+ }
+
+ continue; /* To avoid *d++ = t; */
+ }
+ }
}
break;
}
diff --git a/src/libutil/str_util.h b/src/libutil/str_util.h
index 46b74001b..9145b97b4 100644
--- a/src/libutil/str_util.h
+++ b/src/libutil/str_util.h
@@ -184,6 +184,7 @@ gint rspamd_decode_base32_buf (const gchar *in, gsize inlen,
gint rspamd_encode_hex_buf (const guchar *in, gsize inlen, gchar *out,
gsize outlen);
+
/**
* Decode string using hex encoding
* @param in input
@@ -196,6 +197,23 @@ gssize rspamd_decode_hex_buf (const gchar *in, gsize inlen,
guchar *out, gsize outlen);
/**
+ * Common version of base64 encoder
+ * @param in
+ * @param inlen
+ * @param str_len
+ * @param outlen
+ * @param fold
+ * @param how
+ * @return
+ */
+gchar *
+rspamd_encode_base64_common (const guchar *in,
+ gsize inlen,
+ gint str_len,
+ gsize *outlen,
+ gboolean fold,
+ enum rspamd_newlines_type how);
+/**
* Encode string using base64 encoding
* @param in input
* @param inlen input length
@@ -436,6 +454,7 @@ enum rspamd_regexp_escape_flags {
RSPAMD_REGEXP_ESCAPE_ASCII = 0,
RSPAMD_REGEXP_ESCAPE_UTF = 1u << 0,
RSPAMD_REGEXP_ESCAPE_GLOB = 1u << 1,
+ RSPAMD_REGEXP_ESCAPE_RE = 1u << 2,
};
/**
* Escapes special characters when reading plain data to be processed in pcre
diff --git a/src/libutil/upstream.c b/src/libutil/upstream.c
index eb88e501a..64d5291fa 100644
--- a/src/libutil/upstream.c
+++ b/src/libutil/upstream.c
@@ -289,7 +289,7 @@ rspamd_upstream_update_addrs (struct upstream *up)
rspamd_inet_address_set_port (cur->addr, port);
PTR_ARRAY_FOREACH (up->addrs.addr, i, addr_elt) {
- if (rspamd_inet_address_compare (addr_elt->addr, cur->addr) == 0) {
+ if (rspamd_inet_address_compare (addr_elt->addr, cur->addr, FALSE) == 0) {
naddr = g_malloc0 (sizeof (*naddr));
naddr->addr = cur->addr;
naddr->errors = reset_errors ? 0 : addr_elt->errors;
@@ -630,7 +630,7 @@ rspamd_upstream_dtor (struct upstream *up)
}
rspamd_inet_addr_t*
-rspamd_upstream_addr (struct upstream *up)
+rspamd_upstream_addr_next (struct upstream *up)
{
guint idx, next_idx;
struct upstream_addr_elt *e1, *e2;
@@ -646,6 +646,16 @@ rspamd_upstream_addr (struct upstream *up)
return e2->addr;
}
+rspamd_inet_addr_t*
+rspamd_upstream_addr_cur (const struct upstream *up)
+{
+ struct upstream_addr_elt *elt;
+
+ elt = g_ptr_array_index (up->addrs.addr, up->addrs.cur);
+
+ return elt->addr;
+}
+
const gchar*
rspamd_upstream_name (struct upstream *up)
{
diff --git a/src/libutil/upstream.h b/src/libutil/upstream.h
index 5c0c92afc..4db962765 100644
--- a/src/libutil/upstream.h
+++ b/src/libutil/upstream.h
@@ -208,11 +208,18 @@ void rspamd_upstreams_add_watch_callback (struct upstream_list *ups,
gpointer ud);
/**
+ * Returns the next IP address of the upstream (internal rotation)
+ * @param up
+ * @return
+ */
+rspamd_inet_addr_t* rspamd_upstream_addr_next (struct upstream *up);
+
+/**
* Returns the current IP address of the upstream
* @param up
* @return
*/
-rspamd_inet_addr_t* rspamd_upstream_addr (struct upstream *up);
+rspamd_inet_addr_t* rspamd_upstream_addr_cur (const struct upstream *up);
/**
* Add custom address for an upstream (ownership of addr is transferred to upstream)
diff --git a/src/libutil/uthash_strcase.h b/src/libutil/uthash_strcase.h
index fba97d9b1..5d6f2773c 100644
--- a/src/libutil/uthash_strcase.h
+++ b/src/libutil/uthash_strcase.h
@@ -20,10 +20,13 @@
#error Invalid include order: uthash is already included
#endif
+#include "libcryptobox/cryptobox.h"
+#include "libutil/util.h"
+
/* Utils for uthash tuning */
#ifndef HASH_CASELESS
#define HASH_FUNCTION(key,keylen,num_bkts,hashv,bkt) do {\
- hashv = mum(key, keylen, 0xdeadbabe); \
+ hashv = rspamd_cryptobox_fast_hash(key, keylen, rspamd_hash_seed ()); \
bkt = (hashv) & (num_bkts-1); \
} while (0)
@@ -31,6 +34,7 @@
#else
#define HASH_FUNCTION(key,keylen,num_bkts,hashv,bkt) do {\
unsigned _len = keylen; \
+ rspamd_cryptobox_fast_hash_state_t _hst; \
unsigned _leftover = keylen % 8; \
unsigned _fp, _i; \
const uint8_t* _s = (const uint8_t*)(key); \
@@ -40,9 +44,8 @@
} c; \
uint64_t pp; \
} _u; \
- uint64_t _r; \
_fp = _len - _leftover; \
- _r = 0xdeadbabe; \
+ rspamd_cryptobox_fast_hash_init (&_hst, rspamd_hash_seed ()); \
for (_i = 0; _i != _fp; _i += 8) { \
_u.c.c1 = _s[_i], _u.c.c2 = _s[_i + 1], _u.c.c3 = _s[_i + 2], _u.c.c4 = _s[_i + 3]; \
_u.c.c5 = _s[_i + 4], _u.c.c6 = _s[_i + 5], _u.c.c7 = _s[_i + 6], _u.c.c8 = _s[_i + 7]; \
@@ -54,28 +57,28 @@
_u.c.c2 = lc_map[_u.c.c6]; \
_u.c.c3 = lc_map[_u.c.c7]; \
_u.c.c4 = lc_map[_u.c.c8]; \
- _r = mum_hash_step (_r, _u.pp); \
+ rspamd_cryptobox_fast_hash_update (&_hst, &_u, sizeof (_u)); \
} \
_u.pp = 0; \
switch (_leftover) { \
case 7: \
- _u.c.c7 = lc_map[(unsigned char)_s[_i++]]; \
+ /* fallthrough */ _u.c.c7 = lc_map[(unsigned char)_s[_i++]]; \
case 6: \
- _u.c.c6 = lc_map[(unsigned char)_s[_i++]]; \
+ /* fallthrough */ _u.c.c6 = lc_map[(unsigned char)_s[_i++]]; \
case 5: \
- _u.c.c5 = lc_map[(unsigned char)_s[_i++]]; \
+ /* fallthrough */ _u.c.c5 = lc_map[(unsigned char)_s[_i++]]; \
case 4: \
- _u.c.c4 = lc_map[(unsigned char)_s[_i++]]; \
+ /* fallthrough */ _u.c.c4 = lc_map[(unsigned char)_s[_i++]]; \
case 3: \
- _u.c.c3 = lc_map[(unsigned char)_s[_i++]]; \
+ /* fallthrough */ _u.c.c3 = lc_map[(unsigned char)_s[_i++]]; \
case 2: \
- _u.c.c2 = lc_map[(unsigned char)_s[_i++]]; \
+ /* fallthrough */ _u.c.c2 = lc_map[(unsigned char)_s[_i++]]; \
case 1: \
- _u.c.c1 = lc_map[(unsigned char)_s[_i]]; \
- _r = mum_hash_step (_r, _u.pp); \
+ /* fallthrough */ _u.c.c1 = lc_map[(unsigned char)_s[_i]]; \
+ rspamd_cryptobox_fast_hash_update (&_hst, &_u, sizeof (_u)); \
break; \
} \
- hashv = mum_hash_finish (_r); \
+ hashv = rspamd_cryptobox_fast_hash_final (&_hst); \
bkt = (hashv) & (num_bkts-1); \
} while (0)
#define HASH_KEYCMP(a,b,len) rspamd_lc_cmp(a,b,len)
diff --git a/src/libutil/util.c b/src/libutil/util.c
index 68fbd5443..1f0c5fa70 100644
--- a/src/libutil/util.c
+++ b/src/libutil/util.c
@@ -2035,13 +2035,102 @@ void rspamd_gerror_free_maybe (gpointer p)
}
}
+
+
+static void
+rspamd_openssl_maybe_init (void)
+{
+ static gboolean openssl_initialized = FALSE;
+
+ if (!openssl_initialized) {
+ ERR_load_crypto_strings ();
+ SSL_load_error_strings ();
+
+ OpenSSL_add_all_algorithms ();
+ OpenSSL_add_all_digests ();
+ OpenSSL_add_all_ciphers ();
+
+#if OPENSSL_VERSION_NUMBER >= 0x1000104fL && !defined(LIBRESSL_VERSION_NUMBER)
+ ENGINE_load_builtin_engines ();
+#endif
+#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
+ SSL_library_init ();
+#else
+ OPENSSL_init_ssl (0, NULL);
+#endif
+
+#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
+ OPENSSL_config (NULL);
+#endif
+ if (RAND_status () == 0) {
+ guchar seed[128];
+
+ /* Try to use ottery to seed rand */
+ ottery_rand_bytes (seed, sizeof (seed));
+ RAND_seed (seed, sizeof (seed));
+ rspamd_explicit_memzero (seed, sizeof (seed));
+ }
+
+ openssl_initialized = TRUE;
+ }
+}
+
+gpointer
+rspamd_init_ssl_ctx (void)
+{
+ SSL_CTX *ssl_ctx;
+ gint ssl_options;
+
+ rspamd_openssl_maybe_init ();
+
+ ssl_ctx = SSL_CTX_new (SSLv23_method ());
+ SSL_CTX_set_verify (ssl_ctx, SSL_VERIFY_PEER, NULL);
+ SSL_CTX_set_verify_depth (ssl_ctx, 4);
+ ssl_options = SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3;
+
+#ifdef SSL_OP_NO_COMPRESSION
+ ssl_options |= SSL_OP_NO_COMPRESSION;
+#elif OPENSSL_VERSION_NUMBER >= 0x00908000L
+ sk_SSL_COMP_zero (SSL_COMP_get_compression_methods ());
+#endif
+
+ SSL_CTX_set_options (ssl_ctx, ssl_options);
+
+ return ssl_ctx;
+}
+
+gpointer rspamd_init_ssl_ctx_noverify (void)
+{
+ SSL_CTX *ssl_ctx_noverify;
+ gint ssl_options;
+
+ rspamd_openssl_maybe_init ();
+
+ ssl_options = SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3;
+
+#ifdef SSL_OP_NO_COMPRESSION
+ ssl_options |= SSL_OP_NO_COMPRESSION;
+#elif OPENSSL_VERSION_NUMBER >= 0x00908000L
+ sk_SSL_COMP_zero (SSL_COMP_get_compression_methods ());
+#endif
+
+ ssl_ctx_noverify = SSL_CTX_new (SSLv23_method ());
+ SSL_CTX_set_verify (ssl_ctx_noverify, SSL_VERIFY_NONE, NULL);
+ SSL_CTX_set_options (ssl_ctx_noverify, ssl_options);
+#ifdef SSL_SESS_CACHE_BOTH
+ SSL_CTX_set_session_cache_mode (ssl_ctx_noverify, SSL_SESS_CACHE_BOTH);
+#endif
+
+ return ssl_ctx_noverify;
+}
+
+
struct rspamd_external_libs_ctx *
rspamd_init_libs (void)
{
struct rlimit rlim;
struct rspamd_external_libs_ctx *ctx;
struct ottery_config *ottery_cfg;
- gint ssl_options;
ctx = g_malloc0 (sizeof (*ctx));
ctx->crypto_ctx = rspamd_cryptobox_init ();
@@ -2049,10 +2138,15 @@ rspamd_init_libs (void)
ottery_config_init (ottery_cfg);
ctx->ottery_cfg = ottery_cfg;
+ rspamd_openssl_maybe_init ();
+
/* Check if we have rdrand */
if ((ctx->crypto_ctx->cpu_config & CPUID_RDRAND) == 0) {
ottery_config_disable_entropy_sources (ottery_cfg,
OTTERY_ENTROPY_SRC_RDRAND);
+#if OPENSSL_VERSION_NUMBER >= 0x1000104fL && !defined(LIBRESSL_VERSION_NUMBER)
+ RAND_set_rand_engine (NULL);
+#endif
}
g_assert (ottery_init (ottery_cfg) == 0);
@@ -2072,60 +2166,8 @@ rspamd_init_libs (void)
}
#endif
-#ifdef HAVE_OPENSSL
- ERR_load_crypto_strings ();
- SSL_load_error_strings ();
-
- OpenSSL_add_all_algorithms ();
- OpenSSL_add_all_digests ();
- OpenSSL_add_all_ciphers ();
-
-#if OPENSSL_VERSION_NUMBER >= 0x1000104fL && !defined(LIBRESSL_VERSION_NUMBER)
- ENGINE_load_builtin_engines ();
-
- if ((ctx->crypto_ctx->cpu_config & CPUID_RDRAND) == 0) {
- RAND_set_rand_engine (NULL);
- }
-#endif
-
-#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
- SSL_library_init ();
-#else
- OPENSSL_init_ssl (0, NULL);
-#endif
-
-#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
- OPENSSL_config (NULL);
-#endif
-
- if (RAND_poll () == 0) {
- guchar seed[128];
-
- /* Try to use ottery to seed rand */
- ottery_rand_bytes (seed, sizeof (seed));
- RAND_seed (seed, sizeof (seed));
- rspamd_explicit_memzero (seed, sizeof (seed));
- }
-
- ctx->ssl_ctx = SSL_CTX_new (SSLv23_method ());
- SSL_CTX_set_verify (ctx->ssl_ctx, SSL_VERIFY_PEER, NULL);
- SSL_CTX_set_verify_depth (ctx->ssl_ctx, 4);
- ssl_options = SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3;
-
-#ifdef SSL_OP_NO_COMPRESSION
- ssl_options |= SSL_OP_NO_COMPRESSION;
-#elif OPENSSL_VERSION_NUMBER >= 0x00908000L
- sk_SSL_COMP_zero (SSL_COMP_get_compression_methods ());
-#endif
-
- SSL_CTX_set_options (ctx->ssl_ctx, ssl_options);
- ctx->ssl_ctx_noverify = SSL_CTX_new (SSLv23_method ());
- SSL_CTX_set_verify (ctx->ssl_ctx_noverify, SSL_VERIFY_NONE, NULL);
- SSL_CTX_set_options (ctx->ssl_ctx_noverify, ssl_options);
-#endif
-#ifdef SSL_SESS_CACHE_BOTH
- SSL_CTX_set_session_cache_mode (ctx->ssl_ctx, SSL_SESS_CACHE_BOTH);
-#endif
+ ctx->ssl_ctx = rspamd_init_ssl_ctx ();
+ ctx->ssl_ctx_noverify = rspamd_init_ssl_ctx_noverify ();
rspamd_random_seed_fast ();
/* Set stack size for pcre */
diff --git a/src/libutil/util.h b/src/libutil/util.h
index e97593d21..e9c1fa1db 100644
--- a/src/libutil/util.h
+++ b/src/libutil/util.h
@@ -349,6 +349,9 @@ struct rspamd_external_libs_ctx;
*/
struct rspamd_external_libs_ctx* rspamd_init_libs (void);
+gpointer rspamd_init_ssl_ctx (void);
+gpointer rspamd_init_ssl_ctx_noverify (void);
+
/**
* Configure libraries
*/
diff --git a/src/lua/lua_config.c b/src/lua/lua_config.c
index a41ffa63f..2ea72f075 100644
--- a/src/lua/lua_config.c
+++ b/src/lua/lua_config.c
@@ -1113,6 +1113,119 @@ rspamd_compare_order_func (gconstpointer a, gconstpointer b)
return cb2->order - cb1->order;
}
+static void
+lua_metric_symbol_callback (struct rspamd_task *task,
+ struct rspamd_symcache_item *item,
+ gpointer ud)
+{
+ struct lua_callback_data *cd = ud;
+ struct rspamd_task **ptask;
+ gint level = lua_gettop (cd->L), nresults, err_idx, ret;
+ lua_State *L = cd->L;
+ GString *tb;
+ struct rspamd_symbol_result *s;
+
+ cd->item = item;
+ rspamd_symcache_item_async_inc (task, item, "lua symbol");
+ lua_pushcfunction (L, &rspamd_lua_traceback);
+ err_idx = lua_gettop (L);
+
+ level ++;
+
+ if (cd->cb_is_ref) {
+ lua_rawgeti (L, LUA_REGISTRYINDEX, cd->callback.ref);
+ }
+ else {
+ lua_getglobal (L, cd->callback.name);
+ }
+
+ ptask = lua_newuserdata (L, sizeof (struct rspamd_task *));
+ rspamd_lua_setclass (L, "rspamd{task}", -1);
+ *ptask = task;
+
+ if ((ret = lua_pcall (L, 1, LUA_MULTRET, err_idx)) != 0) {
+ tb = lua_touserdata (L, -1);
+ msg_err_task ("call to (%s) failed (%d): %v", cd->symbol, ret, tb);
+
+ if (tb) {
+ g_string_free (tb, TRUE);
+ lua_pop (L, 1);
+ }
+ }
+ else {
+ nresults = lua_gettop (L) - level;
+
+ if (nresults >= 1) {
+ /* Function returned boolean, so maybe we need to insert result? */
+ gint res = 0;
+ gint i;
+ gdouble flag = 1.0;
+ gint type;
+
+ type = lua_type (cd->L, level + 1);
+
+ if (type == LUA_TBOOLEAN) {
+ res = lua_toboolean (L, level + 1);
+ }
+ else if (type == LUA_TNUMBER) {
+ res = lua_tonumber (L, level + 1);
+ }
+ else if (type == LUA_TNIL) {
+ /* Can happen sometimes... */
+ res = FALSE;
+ }
+ else {
+ g_assert_not_reached ();
+ }
+
+ if (res) {
+ gint first_opt = 2;
+
+ if (lua_type (L, level + 2) == LUA_TNUMBER) {
+ flag = lua_tonumber (L, level + 2);
+ /* Shift opt index */
+ first_opt = 3;
+ }
+ else {
+ flag = res;
+ }
+
+ s = rspamd_task_insert_result (task, cd->symbol, flag, NULL);
+
+ if (s) {
+ guint last_pos = lua_gettop (L);
+
+ for (i = level + first_opt; i <= last_pos; i++) {
+ if (lua_type (L, i) == LUA_TSTRING) {
+ const char *opt = lua_tostring (L, i);
+
+ rspamd_task_add_result_option (task, s, opt);
+ }
+ else if (lua_type (L, i) == LUA_TTABLE) {
+ lua_pushvalue (L, i);
+
+ for (lua_pushnil (L); lua_next (L, -2); lua_pop (L, 1)) {
+ const char *opt = lua_tostring (L, -1);
+
+ rspamd_task_add_result_option (task, s, opt);
+ }
+
+ lua_pop (L, 1);
+ }
+ }
+ }
+
+ }
+
+ lua_pop (L, nresults);
+ }
+ }
+
+ lua_pop (L, 1); /* Error function */
+ rspamd_symcache_item_async_dec_check (task, cd->item, "lua symbol");
+ g_assert (lua_gettop (L) == level - 1);
+}
+
static void lua_metric_symbol_callback_return (struct thread_entry *thread_entry,
int ret);
@@ -1121,15 +1234,15 @@ static void lua_metric_symbol_callback_error (struct thread_entry *thread_entry,
const char *msg);
static void
-lua_metric_symbol_callback (struct rspamd_task *task,
- struct rspamd_symcache_item *item,
- gpointer ud)
+lua_metric_symbol_callback_coro (struct rspamd_task *task,
+ struct rspamd_symcache_item *item,
+ gpointer ud)
{
struct lua_callback_data *cd = ud;
struct rspamd_task **ptask;
struct thread_entry *thread_entry;
- rspamd_symcache_item_async_inc (task, item, "lua symbol");
+ rspamd_symcache_item_async_inc (task, item, "lua coro symbol");
thread_entry = lua_thread_pool_get_for_task (task);
g_assert(thread_entry->cd == NULL);
@@ -1163,9 +1276,9 @@ lua_metric_symbol_callback_error (struct thread_entry *thread_entry,
{
struct lua_callback_data *cd = thread_entry->cd;
struct rspamd_task *task = thread_entry->task;
- msg_err_task ("call to (%s) failed (%d): %s", cd->symbol, ret, msg);
+ msg_err_task ("call to coroutine (%s) failed (%d): %s", cd->symbol, ret, msg);
- rspamd_symcache_item_async_dec_check (task, cd->item, "lua symbol");
+ rspamd_symcache_item_async_dec_check (task, cd->item, "lua coro symbol");
}
static void
@@ -1246,7 +1359,7 @@ lua_metric_symbol_callback_return (struct thread_entry *thread_entry, int ret)
g_assert (lua_gettop (L) == cd->stack_level); /* we properly cleaned up the stack */
cd->stack_level = 0;
- rspamd_symcache_item_async_dec_check (task, cd->item, "lua symbol");
+ rspamd_symcache_item_async_dec_check (task, cd->item, "lua coro symbol");
}
static gint
@@ -1282,6 +1395,10 @@ rspamd_register_symbol_fromlua (lua_State *L,
}
if (ref != -1) {
+ if (type & SYMBOL_TYPE_USE_CORO) {
+ /* Coroutines are incompatible with squeezing */
+ no_squeeze = TRUE;
+ }
/*
* We call for routine called lua_squeeze_rules.squeeze_rule if it exists
*/
@@ -1349,15 +1466,27 @@ rspamd_register_symbol_fromlua (lua_State *L,
cd->symbol = rspamd_mempool_strdup (cfg->cfg_pool, name);
}
- ret = rspamd_symcache_add_symbol (cfg->cache,
- name,
- priority,
- lua_metric_symbol_callback,
- cd,
- type,
- parent);
+ if (type & SYMBOL_TYPE_USE_CORO) {
+ ret = rspamd_symcache_add_symbol (cfg->cache,
+ name,
+ priority,
+ lua_metric_symbol_callback_coro,
+ cd,
+ type,
+ parent);
+ }
+ else {
+ ret = rspamd_symcache_add_symbol (cfg->cache,
+ name,
+ priority,
+ lua_metric_symbol_callback,
+ cd,
+ type,
+ parent);
+ }
+
rspamd_mempool_add_destructor (cfg->cfg_pool,
- (rspamd_mempool_destruct_t)lua_destroy_cfg_symbol,
+ (rspamd_mempool_destruct_t) lua_destroy_cfg_symbol,
cd);
}
}
@@ -1373,13 +1502,24 @@ rspamd_register_symbol_fromlua (lua_State *L,
cd->symbol = rspamd_mempool_strdup (cfg->cfg_pool, name);
}
- ret = rspamd_symcache_add_symbol (cfg->cache,
- name,
- priority,
- lua_metric_symbol_callback,
- cd,
- type,
- parent);
+ if (type & SYMBOL_TYPE_USE_CORO) {
+ ret = rspamd_symcache_add_symbol (cfg->cache,
+ name,
+ priority,
+ lua_metric_symbol_callback_coro,
+ cd,
+ type,
+ parent);
+ }
+ else {
+ ret = rspamd_symcache_add_symbol (cfg->cache,
+ name,
+ priority,
+ lua_metric_symbol_callback,
+ cd,
+ type,
+ parent);
+ }
rspamd_mempool_add_destructor (cfg->cfg_pool,
(rspamd_mempool_destruct_t)lua_destroy_cfg_symbol,
cd);
@@ -1556,6 +1696,9 @@ lua_parse_symbol_flags (const gchar *str)
if (strstr (str, "explicit_disable") != NULL) {
ret |= SYMBOL_TYPE_EXPLICIT_DISABLE;
}
+ if (strstr (str, "coro") != NULL) {
+ ret |= SYMBOL_TYPE_USE_CORO;
+ }
}
return ret;
@@ -2431,7 +2574,7 @@ lua_config_add_composite (lua_State * L)
if (new) {
rspamd_symcache_add_symbol (cfg->cache, name,
- 0, NULL, NULL, SYMBOL_TYPE_COMPOSITE, -1);
+ 0, NULL, composite, SYMBOL_TYPE_COMPOSITE, -1);
}
ret = TRUE;
@@ -2450,7 +2593,7 @@ lua_config_newindex (lua_State *L)
LUA_TRACE_POINT;
struct rspamd_config *cfg = lua_check_config (L, 1);
const gchar *name;
- gint id, nshots;
+ gint id, nshots, flags = 0;
gboolean optional = FALSE, no_squeeze = FALSE;
name = luaL_checkstring (L, 2);
@@ -2476,7 +2619,6 @@ lua_config_newindex (lua_State *L)
gint type = SYMBOL_TYPE_NORMAL, priority = 0, idx;
gdouble weight = 1.0, score = NAN;
const char *type_str, *group = NULL, *description = NULL;
- guint flags = 0;
no_squeeze = cfg->disable_lua_squeeze;
/*
@@ -2485,6 +2627,7 @@ lua_config_newindex (lua_State *L)
* "weight" - optional weight
* "priority" - optional priority
* "type" - optional type (normal, virtual, callback)
+ * "flags" - optional flags
* -- Metric options
* "score" - optional default score (overridden by metric)
* "group" - optional default group
@@ -2537,6 +2680,15 @@ lua_config_newindex (lua_State *L)
}
lua_pop (L, 1);
+ lua_pushstring (L, "flags");
+ lua_gettable (L, -2);
+
+ if (lua_type (L, -1) == LUA_TSTRING) {
+ type_str = lua_tostring (L, -1);
+ type |= lua_parse_symbol_flags (type_str);
+ }
+ lua_pop (L, 1);
+
lua_pushstring (L, "condition");
lua_gettable (L, -2);
diff --git a/src/lua/lua_dns_resolver.c b/src/lua/lua_dns_resolver.c
index 362a6b35e..b68e15397 100644
--- a/src/lua/lua_dns_resolver.c
+++ b/src/lua/lua_dns_resolver.c
@@ -301,6 +301,8 @@ lua_push_dns_reply (lua_State *L, const struct rdns_reply *reply)
lua_rawseti (L, -2, ++i);
break;
+ default:
+ continue;
}
}
lua_pushnil (L);
diff --git a/src/lua/lua_html.c b/src/lua/lua_html.c
index 739776b01..63839c286 100644
--- a/src/lua/lua_html.c
+++ b/src/lua/lua_html.c
@@ -594,6 +594,9 @@ lua_html_tag_get_parent (lua_State *L)
*ptag = node->data;
rspamd_lua_setclass (L, "rspamd{html_tag}", -1);
}
+ else {
+ lua_pushnil (L);
+ }
}
else {
return luaL_error (L, "invalid arguments");
diff --git a/src/lua/lua_http.c b/src/lua/lua_http.c
index 2ca01d8a5..a8616e82a 100644
--- a/src/lua/lua_http.c
+++ b/src/lua/lua_http.c
@@ -379,23 +379,24 @@ lua_http_make_connection (struct lua_http_cbdata *cbd)
cbd->fd = fd;
if (cbd->cfg) {
- cbd->conn = rspamd_http_connection_new (NULL,
+ cbd->conn = rspamd_http_connection_new (
+ NULL,
+ fd,
+ NULL,
lua_http_error_handler,
lua_http_finish_handler,
RSPAMD_HTTP_CLIENT_SIMPLE,
- RSPAMD_HTTP_CLIENT,
- NULL,
- (cbd->flags & RSPAMD_LUA_HTTP_FLAG_NOVERIFY) ?
- cbd->cfg->libs_ctx->ssl_ctx_noverify : cbd->cfg->libs_ctx->ssl_ctx);
+ RSPAMD_HTTP_CLIENT);
}
else {
- cbd->conn = rspamd_http_connection_new (NULL,
+ cbd->conn = rspamd_http_connection_new (
+ NULL,
+ fd,
+ NULL,
lua_http_error_handler,
lua_http_finish_handler,
RSPAMD_HTTP_CLIENT_SIMPLE,
- RSPAMD_HTTP_CLIENT,
- NULL,
- NULL);
+ RSPAMD_HTTP_CLIENT);
}
if (cbd->conn) {
@@ -437,8 +438,8 @@ lua_http_make_connection (struct lua_http_cbdata *cbd)
cbd->msg = NULL;
rspamd_http_connection_write_message (cbd->conn, msg,
- cbd->host, cbd->mime_type, cbd, fd,
- &cbd->tv, cbd->ev_base);
+ cbd->host, cbd->mime_type, cbd,
+ &cbd->tv);
return TRUE;
}
diff --git a/src/lua/lua_ip.c b/src/lua/lua_ip.c
index 800c9765c..7f24f7712 100644
--- a/src/lua/lua_ip.c
+++ b/src/lua/lua_ip.c
@@ -176,6 +176,9 @@ static const struct luaL_reg iplib_m[] = {
LUA_INTERFACE_DEF (ip, apply_mask),
LUA_INTERFACE_DEF (ip, copy),
LUA_INTERFACE_DEF (ip, is_local),
+ {"tostring", lua_ip_to_string},
+ {"totable", lua_ip_to_table},
+ {"tonumber", lua_ip_to_number},
{"__tostring", lua_ip_to_string},
{"__eq", lua_ip_equal},
{"__gc", lua_ip_destroy},
@@ -487,7 +490,7 @@ lua_ip_equal (lua_State *L)
gboolean res = FALSE;
if (ip1 && ip2 && ip1->addr && ip2->addr) {
- res = rspamd_inet_address_compare (ip1->addr, ip2->addr);
+ res = rspamd_inet_address_compare (ip1->addr, ip2->addr, TRUE);
}
lua_pushboolean (L, res);
diff --git a/src/lua/lua_map.c b/src/lua/lua_map.c
index 98b025ea2..3ba7d0ed1 100644
--- a/src/lua/lua_map.c
+++ b/src/lua/lua_map.c
@@ -416,7 +416,7 @@ lua_map_read (gchar *chunk, gint len,
}
static void
-lua_map_fin (struct map_cb_data *data)
+lua_map_fin (struct map_cb_data *data, void **target)
{
struct lua_map_callback_data *cbdata;
struct rspamd_lua_map **pmap;
@@ -432,10 +432,6 @@ lua_map_fin (struct map_cb_data *data)
return;
}
- if (data->prev_data) {
- data->prev_data = NULL;
- }
-
if (cbdata->ref == -1) {
msg_err_map ("map has no callback set");
}
@@ -454,6 +450,14 @@ lua_map_fin (struct map_cb_data *data)
}
cbdata->data = rspamd_fstring_assign (cbdata->data, "", 0);
+
+ if (target) {
+ *target = data->cur_data;
+ }
+
+ if (data->prev_data) {
+ data->prev_data = NULL;
+ }
}
static void
diff --git a/src/lua/lua_task.c b/src/lua/lua_task.c
index b6c5c1fb7..8e1a851b4 100644
--- a/src/lua/lua_task.c
+++ b/src/lua/lua_task.c
@@ -174,13 +174,13 @@ end
*/
LUA_FUNCTION_DEF (task, append_message);
/***
- * @method task:get_urls([need_emails])
- * Get all URLs found in a message.
- * @param {boolean} need_emails if `true` then reutrn also email urls
+ * @method task:get_urls([need_emails|list_protos])
+ * Get all URLs found in a message. Telephone urls and emails are not included unless explicitly asked in `list_protos`
+ * @param {boolean} need_emails if `true` then reutrn also email urls, this can be a comma separated string of protocols desired or a table (e.g. `mailto` or `telephone`)
* @return {table rspamd_url} list of all urls found
@example
local function phishing_cb(task)
- local urls = task:get_urls();
+ local urls = task:get_urls({'https', 'http'});
if urls then
for _,url in ipairs(urls) do
@@ -1831,18 +1831,22 @@ lua_task_append_message (lua_State * L)
struct lua_tree_cb_data {
lua_State *L;
int i;
+ gint mask;
};
static void
lua_tree_url_callback (gpointer key, gpointer value, gpointer ud)
{
- struct rspamd_lua_url *url;
+ struct rspamd_lua_url *lua_url;
+ struct rspamd_url *url = (struct rspamd_url *)value;
struct lua_tree_cb_data *cb = ud;
- url = lua_newuserdata (cb->L, sizeof (struct rspamd_lua_url));
- rspamd_lua_setclass (cb->L, "rspamd{url}", -1);
- url->url = value;
- lua_rawseti (cb->L, -2, cb->i++);
+ if (url->protocol & cb->mask) {
+ lua_url = lua_newuserdata (cb->L, sizeof (struct rspamd_lua_url));
+ rspamd_lua_setclass (cb->L, "rspamd{url}", -1);
+ lua_url->url = url;
+ lua_rawseti (cb->L, -2, cb->i++);
+ }
}
static gint
@@ -1851,38 +1855,104 @@ lua_task_get_urls (lua_State * L)
LUA_TRACE_POINT;
struct rspamd_task *task = lua_check_task (L, 1);
struct lua_tree_cb_data cb;
- gboolean need_emails = FALSE;
+ gint protocols_mask = 0;
+ static const gint default_mask = PROTOCOL_HTTP|PROTOCOL_HTTPS|
+ PROTOCOL_FILE|PROTOCOL_FTP;
gsize sz;
if (task) {
if (lua_gettop (L) >= 2) {
- need_emails = lua_toboolean (L, 2);
+ if (lua_type (L, 2) == LUA_TBOOLEAN) {
+ protocols_mask = default_mask;
+ if (lua_toboolean (L, 2)) {
+ protocols_mask |= PROTOCOL_MAILTO;
+ }
+ }
+ else if (lua_type (L, 2) == LUA_TTABLE) {
+ for (lua_pushnil (L); lua_next (L, 2); lua_pop (L, 1)) {
+ int nmask;
+ const gchar *pname = lua_tostring (L, -1);
+
+ nmask = rspamd_url_protocol_from_string (pname);
+
+ if (nmask != PROTOCOL_UNKNOWN) {
+ protocols_mask |= nmask;
+ }
+ else {
+ msg_info ("bad url protocol: %s", pname);
+ }
+ }
+ }
+ else if (lua_type (L, 2) == LUA_TSTRING) {
+ const gchar *plist = lua_tostring (L, 2);
+ gchar **strvec;
+ gchar * const *cvec;
+
+ strvec = g_strsplit_set (plist, ",;", -1);
+ cvec = strvec;
+
+ while (*cvec) {
+ int nmask;
+
+ nmask = rspamd_url_protocol_from_string (*cvec);
+
+ if (nmask != PROTOCOL_UNKNOWN) {
+ protocols_mask |= nmask;
+ }
+ else {
+ msg_info ("bad url protocol: %s", *cvec);
+ }
+
+ cvec ++;
+ }
+
+ g_strfreev (strvec);
+ }
+ else {
+ protocols_mask = default_mask;
+ }
}
+ else {
+ protocols_mask = default_mask;
+ }
+
+ cb.i = 1;
+ cb.L = L;
+ cb.mask = protocols_mask;
- if (need_emails) {
+ if (protocols_mask & PROTOCOL_MAILTO) {
sz = g_hash_table_size (task->urls) + g_hash_table_size (task->emails);
- if (!lua_task_get_cached (L, task, "emails+urls", sz)) {
+ if (protocols_mask == (default_mask|PROTOCOL_MAILTO)) {
+ /* Can use cached version */
+ if (!lua_task_get_cached (L, task, "emails+urls", sz)) {
+ lua_createtable (L, sz, 0);
+ g_hash_table_foreach (task->urls, lua_tree_url_callback, &cb);
+ g_hash_table_foreach (task->emails, lua_tree_url_callback, &cb);
+
+ lua_task_set_cached (L, task, "emails+urls", -1, sz);
+ }
+ }
+ else {
lua_createtable (L, sz, 0);
- cb.i = 1;
- cb.L = L;
g_hash_table_foreach (task->urls, lua_tree_url_callback, &cb);
g_hash_table_foreach (task->emails, lua_tree_url_callback, &cb);
-
- lua_task_set_cached (L, task, "emails+urls", -1, sz);
}
}
else {
sz = g_hash_table_size (task->urls);
- if (!lua_task_get_cached (L, task, "urls", sz)) {
+ if (protocols_mask == (default_mask)) {
+ if (!lua_task_get_cached (L, task, "urls", sz)) {
+ lua_createtable (L, sz, 0);
+ g_hash_table_foreach (task->urls, lua_tree_url_callback, &cb);
+ lua_task_set_cached (L, task, "urls", -1, sz);
+ }
+ }
+ else {
lua_createtable (L, sz, 0);
- cb.i = 1;
- cb.L = L;
g_hash_table_foreach (task->urls, lua_tree_url_callback, &cb);
-
- lua_task_set_cached (L, task, "urls", -1, sz);
}
}
}
@@ -2005,6 +2075,7 @@ lua_task_get_emails (lua_State * L)
lua_createtable (L, g_hash_table_size (task->emails), 0);
cb.i = 1;
cb.L = L;
+ cb.mask = PROTOCOL_MAILTO;
g_hash_table_foreach (task->emails, lua_tree_url_callback, &cb);
}
else {
diff --git a/src/lua/lua_upstream.c b/src/lua/lua_upstream.c
index 1a4d6b128..37c541d9d 100644
--- a/src/lua/lua_upstream.c
+++ b/src/lua/lua_upstream.c
@@ -110,7 +110,7 @@ lua_upstream_get_addr (lua_State *L)
struct upstream *up = lua_check_upstream (L);
if (up) {
- rspamd_lua_ip_push (L, rspamd_upstream_addr (up));
+ rspamd_lua_ip_push (L, rspamd_upstream_addr_next (up));
}
else {
lua_pushnil (L);
diff --git a/src/lua/lua_url.c b/src/lua/lua_url.c
index 8bc0cf657..0a301e96d 100644
--- a/src/lua/lua_url.c
+++ b/src/lua/lua_url.c
@@ -51,6 +51,7 @@ LUA_FUNCTION_DEF (url, tostring);
LUA_FUNCTION_DEF (url, get_raw);
LUA_FUNCTION_DEF (url, get_tld);
LUA_FUNCTION_DEF (url, get_flags);
+LUA_FUNCTION_DEF (url, get_protocol);
LUA_FUNCTION_DEF (url, to_table);
LUA_FUNCTION_DEF (url, is_phished);
LUA_FUNCTION_DEF (url, is_redirected);
@@ -77,6 +78,7 @@ static const struct luaL_reg urllib_m[] = {
LUA_INTERFACE_DEF (url, get_text),
LUA_INTERFACE_DEF (url, get_tld),
LUA_INTERFACE_DEF (url, get_raw),
+ LUA_INTERFACE_DEF (url, get_protocol),
LUA_INTERFACE_DEF (url, to_table),
LUA_INTERFACE_DEF (url, is_phished),
LUA_INTERFACE_DEF (url, is_redirected),
@@ -607,6 +609,27 @@ lua_url_get_tld (lua_State *L)
}
/***
+ * @method url:get_protocol()
+ * Get protocol name
+ * @return {string} protocol as a string
+ */
+static gint
+lua_url_get_protocol (lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url (L, 1);
+
+ if (url != NULL && url->url->protocol != PROTOCOL_UNKNOWN) {
+ lua_pushstring (L, rspamd_url_protocol_name (url->url->protocol));
+ }
+ else {
+ lua_pushnil (L);
+ }
+
+ return 1;
+}
+
+/***
* @method url:get_count()
* Return number of occurrencies for this particular URL
* @return {number} number of occurrencies
@@ -697,28 +720,7 @@ lua_url_to_table (lua_State *L)
lua_pushstring (L, "protocol");
-
- switch (u->protocol) {
- case PROTOCOL_FILE:
- lua_pushstring (L, "file");
- break;
- case PROTOCOL_FTP:
- lua_pushstring (L, "ftp");
- break;
- case PROTOCOL_HTTP:
- lua_pushstring (L, "http");
- break;
- case PROTOCOL_HTTPS:
- lua_pushstring (L, "https");
- break;
- case PROTOCOL_MAILTO:
- lua_pushstring (L, "mailto");
- break;
- case PROTOCOL_UNKNOWN:
- default:
- lua_pushstring (L, "unknown");
- break;
- }
+ lua_pushstring (L, rspamd_url_protocol_name (u->protocol));
lua_settable (L, -3);
}
else {
diff --git a/src/plugins/dkim_check.c b/src/plugins/dkim_check.c
index bb66e5ccc..f7600552b 100644
--- a/src/plugins/dkim_check.c
+++ b/src/plugins/dkim_check.c
@@ -669,13 +669,18 @@ dkim_module_load_key_format (struct rspamd_task *task,
* This fails for paths that are also valid base64.
* Maybe the caller should have specified a format.
*/
- if (key_format == RSPAMD_DKIM_KEY_UNKNOWN &&
- (key[0] == '.' || key[0] == '/')) {
- if (!rspamd_cryptobox_base64_is_valid (key, keylen)) {
- key_format = RSPAMD_DKIM_KEY_FILE;
+ if (key_format == RSPAMD_DKIM_KEY_UNKNOWN) {
+ if (key[0] == '.' || key[0] == '/') {
+ if (!rspamd_cryptobox_base64_is_valid (key, keylen)) {
+ key_format = RSPAMD_DKIM_KEY_FILE;
+ }
+ }
+ else if (rspamd_cryptobox_base64_is_valid (key, keylen)) {
+ key_format = RSPAMD_DKIM_KEY_BASE64;
}
}
+
if (ret != NULL && key_format == RSPAMD_DKIM_KEY_FILE) {
msg_debug_task("checking for stale file key");
@@ -770,7 +775,7 @@ lua_dkim_sign_handler (lua_State *L)
dkim_key = dkim_module_load_key_format (task, dkim_module_ctx, key,
keylen, RSPAMD_DKIM_KEY_UNKNOWN);
}
- else if(rawkey) {
+ else if (rawkey) {
dkim_key = dkim_module_load_key_format (task, dkim_module_ctx, rawkey,
rawlen, RSPAMD_DKIM_KEY_UNKNOWN);
}
@@ -1540,7 +1545,13 @@ dkim_module_lua_push_verify_result (struct rspamd_dkim_lua_verify_cbdata *cbd,
ptask = lua_newuserdata (cbd->L, sizeof (*ptask));
*ptask = task;
lua_pushboolean (cbd->L, success);
- lua_pushstring (cbd->L, error_str);
+
+ if (error_str) {
+ lua_pushstring (cbd->L, error_str);
+ }
+ else {
+ lua_pushnil (cbd->L);
+ }
if (cbd->ctx) {
if (res->domain) {
diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c
index 7edb0168d..467a67ed7 100644
--- a/src/plugins/fuzzy_check.c
+++ b/src/plugins/fuzzy_check.c
@@ -43,6 +43,7 @@
#include "lua/lua_common.h"
#include "unix-std.h"
#include "libutil/http_private.h"
+#include "libutil/http_router.h"
#include "libstat/stat_api.h"
#include <math.h>
#include <src/libmime/message.h>
@@ -2208,15 +2209,16 @@ fuzzy_check_io_callback (gint fd, short what, void *arg)
msg_err_task ("got error on IO with server %s(%s), on %s, %d, %s",
rspamd_upstream_name (session->server),
rspamd_inet_address_to_string_pretty (
- rspamd_upstream_addr (session->server)),
+ rspamd_upstream_addr_cur (session->server)),
session->state == 1 ? "read" : "write",
errno,
strerror (errno));
- rspamd_upstream_fail (session->server, FALSE);
+ rspamd_upstream_fail (session->server, TRUE);
if (session->item) {
rspamd_symcache_item_async_dec_check (session->task, session->item, M);
}
+
rspamd_session_remove_event (session->task->s, fuzzy_io_fin, session);
}
else {
@@ -2254,9 +2256,10 @@ fuzzy_check_timer_callback (gint fd, short what, void *arg)
msg_err_task ("got IO timeout with server %s(%s), after %d retransmits",
rspamd_upstream_name (session->server),
rspamd_inet_address_to_string_pretty (
- rspamd_upstream_addr (session->server)),
+ rspamd_upstream_addr_cur (session->server)),
session->retransmits);
- rspamd_upstream_fail (session->server, FALSE);
+ rspamd_upstream_fail (session->server, TRUE);
+
if (session->item) {
rspamd_symcache_item_async_dec_check (session->task, session->item, M);
}
@@ -2463,7 +2466,7 @@ fuzzy_controller_io_callback (gint fd, short what, void *arg)
msg_err_task ("got error in IO with server %s(%s), %d, %s",
rspamd_upstream_name (session->server),
rspamd_inet_address_to_string_pretty (
- rspamd_upstream_addr (session->server)),
+ rspamd_upstream_addr_cur (session->server)),
errno, strerror (errno));
rspamd_upstream_fail (session->server, FALSE);
}
@@ -2562,12 +2565,12 @@ fuzzy_controller_timer_callback (gint fd, short what, void *arg)
task = session->task;
if (session->retransmits >= session->rule->ctx->retransmits) {
- rspamd_upstream_fail (session->server, FALSE);
+ rspamd_upstream_fail (session->server, TRUE);
msg_err_task_check ("got IO timeout with server %s(%s), "
"after %d retransmits",
rspamd_upstream_name (session->server),
rspamd_inet_address_to_string_pretty (
- rspamd_upstream_addr (session->server)),
+ rspamd_upstream_addr_cur (session->server)),
session->retransmits);
if (session->session) {
@@ -2724,7 +2727,7 @@ register_fuzzy_client_call (struct rspamd_task *task,
selected = rspamd_upstream_get (rule->servers, RSPAMD_UPSTREAM_ROUND_ROBIN,
NULL, 0);
if (selected) {
- addr = rspamd_upstream_addr (selected);
+ addr = rspamd_upstream_addr_next (selected);
if ((sock = rspamd_inet_address_connect (addr, SOCK_DGRAM, TRUE)) == -1) {
msg_warn_task ("cannot connect to %s(%s), %d, %s",
rspamd_upstream_name (selected),
@@ -2852,7 +2855,7 @@ register_fuzzy_controller_call (struct rspamd_http_connection_entry *entry,
while ((selected = rspamd_upstream_get (rule->servers,
RSPAMD_UPSTREAM_SEQUENTIAL, NULL, 0))) {
/* Create UDP socket */
- addr = rspamd_upstream_addr (selected);
+ addr = rspamd_upstream_addr_next (selected);
if ((sock = rspamd_inet_address_connect (addr,
SOCK_DGRAM, TRUE)) == -1) {
@@ -3215,7 +3218,7 @@ fuzzy_check_send_lua_learn (struct fuzzy_rule *rule,
while ((selected = rspamd_upstream_get (rule->servers,
RSPAMD_UPSTREAM_SEQUENTIAL, NULL, 0))) {
/* Create UDP socket */
- addr = rspamd_upstream_addr (selected);
+ addr = rspamd_upstream_addr_next (selected);
if ((sock = rspamd_inet_address_connect (addr,
SOCK_DGRAM, TRUE)) == -1) {
diff --git a/src/plugins/lua/arc.lua b/src/plugins/lua/arc.lua
index 302861755..59d97fcd4 100644
--- a/src/plugins/lua/arc.lua
+++ b/src/plugins/lua/arc.lua
@@ -91,6 +91,7 @@ local settings = {
}
local function parse_arc_header(hdr, target)
+ -- Split elements by ';' and trim spaces
local arr = fun.totable(fun.map(
function(val)
return fun.totable(fun.map(lua_util.rspamd_str_trim,
@@ -102,8 +103,9 @@ local function parse_arc_header(hdr, target)
-- Now we have two tables in format:
-- [sigs] -> [{sig1_elts}, {sig2_elts}...]
for i,elts in ipairs(arr) do
+ if not target[i] then target[i] = {} end
+ -- Split by kv pair, like k=v
fun.each(function(v)
- if not target[i] then target[i] = {} end
if v[1] and v[2] then
target[i][v[1]] = v[2]
end
diff --git a/src/plugins/lua/dkim_signing.lua b/src/plugins/lua/dkim_signing.lua
index 4bc002548..9b1bfef32 100644
--- a/src/plugins/lua/dkim_signing.lua
+++ b/src/plugins/lua/dkim_signing.lua
@@ -151,12 +151,15 @@ local function dkim_signing_cb(task)
if #selectors > 0 then
for _, k in ipairs(selectors) do
-- templates
- k.key = lua_util.template(k.key, {
- domain = k.domain,
- selector = k.selector
- })
- lua_util.debugm(N, task, 'using key "%s", use selector "%s" for domain "%s"',
- k.key, k.selector, k.domain)
+ if k.key then
+ k.key = lua_util.template(k.key, {
+ domain = k.domain,
+ selector = k.selector
+ })
+ lua_util.debugm(N, task, 'using key "%s", use selector "%s" for domain "%s"',
+ k.key, k.selector, k.domain)
+ end
+
do_sign(k)
end
else
diff --git a/src/plugins/lua/mime_types.lua b/src/plugins/lua/mime_types.lua
index ab2484c34..98acd463f 100644
--- a/src/plugins/lua/mime_types.lua
+++ b/src/plugins/lua/mime_types.lua
@@ -142,7 +142,6 @@ local settings = {
scf = 2,
shs = 2,
theme = 2,
- tmp = 2,
url = 2,
vbp = 2,
vsmacros = 2,
@@ -335,7 +334,7 @@ local full_extensions_map = {
{"etl", "application/etl"},
{"etx", "text/x-setext"},
{"evy", "application/envoy"},
- {"exe", "application/x-dosexec"},
+ {"exe", {"application/x-dosexec", "application/x-msdownload"}},
{"exe.config", "text/xml"},
{"fdf", "application/vnd.fdf"},
{"fif", "application/fractals"},
diff --git a/src/plugins/lua/multimap.lua b/src/plugins/lua/multimap.lua
index 1e348690f..aa3e4d04a 100644
--- a/src/plugins/lua/multimap.lua
+++ b/src/plugins/lua/multimap.lua
@@ -28,7 +28,6 @@ local regexp = require "rspamd_regexp"
local rspamd_expression = require "rspamd_expression"
local rspamd_ip = require "rspamd_ip"
local lua_util = require "lua_util"
-local rspamd_dns = require "rspamd_dns"
local lua_selectors = require "lua_selectors"
local redis_params
local fun = require "fun"
@@ -260,9 +259,15 @@ local function apply_addr_filter(task, filter, input, rule)
if addr and addr[1] then
return addr[1]['name']
end
+ elseif filter == 'ip_addr' then
+ local ip_addr = rspamd_ip.from_string(input)
+
+ if ip_addr and ip_addr:is_valid() then
+ return ip_addr
+ end
else
-- regexp case
- if not rule['re_filter'] then
+ if not rule['re_filter'] then
local type,pat = string.match(filter, '(regexp:)(.+)')
if type and pat then
rule['re_filter'] = regexp.create(pat)
@@ -397,16 +402,17 @@ local function multimap_callback(task, rule)
if r['cdb'] then
local srch = value
- if r['type'] == 'ip' then
- srch = value:to_string()
+ if type(value) == 'userdata' then
+ if value.class == 'rspamd{ip}' then
+ srch = value:tostring()
+ end
end
ret = r['cdb']:lookup(srch)
elseif r['redis_key'] then
local srch = {value}
local cmd = 'HGET'
- if r['type'] == 'ip' or (r['type'] == 'received' and
- (r['filter'] == 'real_ip' or r['filter'] == 'from_ip' or not r['filter'])) then
- srch = {value:to_string()}
+ if type(value) == 'userdata' and value.class == 'rspamd{ip}' then
+ srch = {value:tostring()}
cmd = 'HMGET'
local maxbits = 128
local minbits = 32
@@ -415,7 +421,7 @@ local function multimap_callback(task, rule)
minbits = 8
end
for i=maxbits,minbits,-1 do
- local nip = value:apply_mask(i):to_string() .. "/" .. i
+ local nip = value:apply_mask(i):tostring() .. "/" .. i
table.insert(srch, nip)
end
end
@@ -434,6 +440,11 @@ local function multimap_callback(task, rule)
elseif r['radix'] then
ret = r['radix']:get_key(value)
elseif r['hash'] then
+ if type(value) == 'userdata' then
+ if value.class == 'rspamd{ip}' then
+ value = value:tostring()
+ end
+ end
ret = r['hash']:get_key(value)
end
@@ -721,26 +732,30 @@ local function multimap_callback(task, rule)
local ip = task:get_from_ip()
if ip:is_valid() then
local to_resolve = ip_to_rbl(ip, rule['map'])
+ local function dns_cb(_, _, results, err)
+ lua_util.debugm(N, rspamd_config,
+ 'resolve() finished: results=%1, err=%2, to_resolve=%3',
+ results, err, to_resolve)
+
+ if err and
+ (err ~= 'requested record is not found' and
+ err ~= 'no records with this name') then
+ rspamd_logger.errx(task, 'error looking up %s: %s', to_resolve, results)
+ elseif results then
+ task:insert_result(rule['symbol'], 1, rule['map'])
+ if pre_filter then
+ task:set_pre_result(rule['action'],
+ 'Matched map: ' .. rule['symbol'], N)
+ end
+ end
+ end
- local is_ok, results = rspamd_dns.request({
- type = "a",
- task = task,
+ task:get_resolver():resolve_a({
+ task= task,
name = to_resolve,
+ callback = dns_cb,
+ forced = true
})
-
- lua_util.debugm(N, rspamd_config,
- 'resolve() finished: results=%1, is_ok=%2, to_resolve=%3',
- results, is_ok, to_resolve)
-
- if not is_ok and
- (results ~= 'requested record is not found' and results ~= 'no records with this name') then
- rspamd_logger.errx(task, 'error looking up %s: %s', to_resolve, results)
- elseif is_ok then
- task:insert_result(rule['symbol'], 1, rule['map'])
- if pre_filter then
- task:set_pre_result(rule['action'], 'Matched map: ' .. rule['symbol'], N)
- end
- end
end
end,
header = function()
@@ -990,7 +1005,7 @@ local function add_multimap_rule(key, newrule)
local map = urls[newrule['map']]
if map and map['regexp'] == newrule['regexp'] and
map['glob'] == newrule['glob'] then
- if newrule['type'] == 'ip' then
+ if newrule['type'] == 'ip' or newrule['filter'] == 'ip_addr' then
newrule['radix'] = map['map']
else
newrule['hash'] = map['map']
@@ -1072,14 +1087,22 @@ local function add_multimap_rule(key, newrule)
or newrule['type'] == 'mempool'
or newrule['type'] == 'selector'then
- multimap_load_hash(newrule)
+ if newrule.filter == 'ip_addr' then
+ newrule['radix'] = rspamd_config:add_map ({
+ url = newrule['map'],
+ description = newrule['description'],
+ type = 'radix'
+ })
+ else
+ multimap_load_hash(newrule)
+ end
- if newrule['hash'] then
+ if newrule.hash or newrule.radix then
ret = true
if type(newrule['map']) == 'string' then
urls[newrule['map']] = {
type = newrule['type'],
- map = newrule['hash'],
+ map = newrule.hash or newrule.radix,
regexp = newrule['regexp']
}
end
diff --git a/src/plugins/lua/reputation.lua b/src/plugins/lua/reputation.lua
index 374771c9b..e91c6ebb7 100644
--- a/src/plugins/lua/reputation.lua
+++ b/src/plugins/lua/reputation.lua
@@ -25,7 +25,6 @@ local N = 'reputation'
local rspamd_logger = require "rspamd_logger"
local rspamd_util = require "rspamd_util"
-local rspamd_dns = require "rspamd_dns"
local lua_util = require "lua_util"
local lua_maps = require "lua_maps"
local hash = require 'rspamd_cryptobox_hash'
@@ -857,39 +856,42 @@ local function reputation_dns_get_token(task, rule, token, continuation_cb)
local key = gen_token_key(token, rule)
local dns_name = key .. '.' .. rule.backend.config.list
- local is_ok, results = rspamd_dns.request({
- type = 'a',
- task = task,
- name = dns_name,
- forced = true,
- })
-
- if not is_ok and (results ~= 'requested record is not found' and results ~= 'no records with this name') then
- rspamd_logger.errx(task, 'error looking up %s: %s', dns_name, results)
- end
+ local function dns_cb(_, _, results, err)
+ if err and (err ~= 'requested record is not found' and
+ err ~= 'no records with this name') then
+ rspamd_logger.errx(task, 'error looking up %s: %s', dns_name, err)
+ end
- lua_util.debugm(N, task, 'DNS RESPONSE: label=%1 results=%2 is_ok=%3 list=%4',
- dns_name, results, is_ok, rule.backend.config.list)
+ lua_util.debugm(N, task, 'DNS RESPONSE: label=%1 results=%2 err=%3 list=%4',
+ dns_name, results, err, rule.backend.config.list)
- -- Now split tokens to list of values
- if is_ok then
- local values = {}
- -- Format: key1=num1;key2=num2...keyn=numn
- fun.each(function(e)
- local vals = lua_util.rspamd_str_split(e, "=")
- if vals and #vals == 2 then
- local nv = tonumber(vals[2])
- if nv then
- values[vals[1]] = nv
+ -- Now split tokens to list of values
+ if results then
+ local values = {}
+ -- Format: key1=num1;key2=num2...keyn=numn
+ fun.each(function(e)
+ local vals = lua_util.rspamd_str_split(e, "=")
+ if vals and #vals == 2 then
+ local nv = tonumber(vals[2])
+ if nv then
+ values[vals[1]] = nv
+ end
end
- end
- end,
- lua_util.rspamd_str_split(results[1], ";"))
+ end,
+ lua_util.rspamd_str_split(results[1], ";"))
- continuation_cb(nil, dns_name, values)
- else
- continuation_cb(results, dns_name, nil)
+ continuation_cb(nil, dns_name, values)
+ else
+ continuation_cb(results, dns_name, nil)
+ end
end
+
+ task:get_resolver():resolve_a({
+ task = task,
+ name = dns_name,
+ callback = dns_cb,
+ forced = true,
+ })
end
local function reputation_redis_init(rule, cfg, ev_base, worker)
diff --git a/src/plugins/lua/rspamd_update.lua b/src/plugins/lua/rspamd_update.lua
index 51cb5db02..d53d02112 100644
--- a/src/plugins/lua/rspamd_update.lua
+++ b/src/plugins/lua/rspamd_update.lua
@@ -124,21 +124,25 @@ end
-- Configuration part
local section = rspamd_config:get_all_opt("rspamd_update")
-if section then
+if section and section.rules then
local trusted_key
- fun.each(function(k, elt)
- if k == 'key' then
- trusted_key = elt
+ if section.key then
+ trusted_key = section.key
+ end
+
+ if type(section.rules) ~= 'table' then
+ section.rules = {section.rules}
+ end
+
+ fun.each(function(elt)
+ local map = rspamd_config:add_map(elt, "rspamd updates map", nil, "callback")
+ if not map then
+ rspamd_logger.errx(rspamd_config, 'cannot load updates from %1', elt)
else
- local map = rspamd_config:add_map(elt, "rspamd updates map", nil, "callback")
- if not map then
- rspamd_logger.errx(rspamd_config, 'cannot load updates from %1', elt)
- else
- map:set_callback(gen_callback(map))
- maps['elt'] = map
- end
+ map:set_callback(gen_callback(map))
+ maps['elt'] = map
end
- end, section)
+ end, section.rules)
fun.each(function(k, map)
-- Check sanity for maps
diff --git a/src/plugins/lua/settings.lua b/src/plugins/lua/settings.lua
index 8d2122868..4e7afbf84 100644
--- a/src/plugins/lua/settings.lua
+++ b/src/plugins/lua/settings.lua
@@ -78,6 +78,10 @@ local function apply_settings(task, to_apply)
to_apply.symbols))
end
end
+
+ if to_apply.subject then
+ task:set_metric_subject(to_apply.subject)
+ end
end
-- Checks for overridden settings within query params and returns 'true' if
diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c
index 94d88334e..26af1210c 100644
--- a/src/plugins/surbl.c
+++ b/src/plugins/surbl.c
@@ -270,11 +270,15 @@ read_exceptions_list (gchar * chunk,
}
static void
-fin_exceptions_list (struct map_cb_data *data)
+fin_exceptions_list (struct map_cb_data *data, void **target)
{
GHashTable **t;
gint i;
+ if (target) {
+ *target = data->cur_data;
+ }
+
if (data->prev_data) {
t = data->prev_data;
for (i = 0; i < MAX_LEVELS; i++) {
@@ -385,11 +389,15 @@ read_redirectors_list (gchar * chunk,
final);
}
-void
-fin_redirectors_list (struct map_cb_data *data)
+static void
+fin_redirectors_list (struct map_cb_data *data, void **target)
{
GHashTable *tld_hash;
+ if (target) {
+ *target = data->cur_data;
+ }
+
if (data->prev_data) {
tld_hash = data->prev_data;
@@ -397,7 +405,7 @@ fin_redirectors_list (struct map_cb_data *data)
}
}
-void
+static void
dtor_redirectors_list (struct map_cb_data *data)
{
GHashTable *tld_hash;
@@ -1624,7 +1632,8 @@ surbl_redirector_error (struct rspamd_http_connection *conn,
task = param->task;
msg_err_surbl ("connection with http server %s terminated incorrectly: %e",
- rspamd_inet_address_to_string (rspamd_upstream_addr (param->redirector)),
+ rspamd_inet_address_to_string (
+ rspamd_upstream_addr_cur (param->redirector)),
err);
rspamd_upstream_fail (param->redirector, FALSE);
rspamd_session_remove_event (param->task->s, free_redirector_session,
@@ -1715,7 +1724,7 @@ register_redirector_call (struct rspamd_url *url, struct rspamd_task *task,
RSPAMD_UPSTREAM_ROUND_ROBIN, url->host, url->hostlen);
if (selected) {
- s = rspamd_inet_address_connect (rspamd_upstream_addr (selected),
+ s = rspamd_inet_address_connect (rspamd_upstream_addr_next (selected),
SOCK_STREAM, TRUE);
}
@@ -1733,12 +1742,12 @@ register_redirector_call (struct rspamd_url *url, struct rspamd_task *task,
param->url = url;
param->task = task;
param->conn = rspamd_http_connection_new (NULL,
+ s,
+ NULL,
surbl_redirector_error,
surbl_redirector_finish,
RSPAMD_HTTP_CLIENT_SIMPLE,
- RSPAMD_HTTP_CLIENT,
- NULL,
- NULL);
+ RSPAMD_HTTP_CLIENT);
param->ctx = surbl_module_ctx;
msg = rspamd_http_new_message (HTTP_REQUEST);
msg->url = rspamd_fstring_assign (msg->url, url->string, url->urllen);
@@ -1757,7 +1766,7 @@ register_redirector_call (struct rspamd_url *url, struct rspamd_task *task,
}
rspamd_http_connection_write_message (param->conn, msg, NULL,
- NULL, param, s, timeout, task->ev_base);
+ NULL, param, timeout);
msg_info_surbl (
"<%s> registered redirector call for %*s to %s, according to rule: %s",
diff --git a/src/rspamadm/CMakeLists.txt b/src/rspamadm/CMakeLists.txt
index 8e3f09435..5be38aa28 100644
--- a/src/rspamadm/CMakeLists.txt
+++ b/src/rspamadm/CMakeLists.txt
@@ -25,7 +25,6 @@ IF (ENABLE_HYPERSCAN MATCHES "ON")
ENDIF()
ADD_EXECUTABLE(rspamadm ${RSPAMADMSRC})
TARGET_LINK_LIBRARIES(rspamadm rspamd-server)
-TARGET_LINK_LIBRARIES(rspamadm ${RSPAMD_REQUIRED_LIBRARIES})
IF (NOT DEBIAN_BUILD)
SET_TARGET_PROPERTIES(rspamadm PROPERTIES VERSION ${RSPAMD_VERSION})
diff --git a/src/rspamadm/control.c b/src/rspamadm/control.c
index 0e39ed703..0a242e943 100644
--- a/src/rspamadm/control.c
+++ b/src/rspamadm/control.c
@@ -17,7 +17,7 @@
#include "rspamadm.h"
#include "cryptobox.h"
#include "printf.h"
-#include "libutil/http.h"
+#include "libutil/http_connection.h"
#include "libutil/http_private.h"
#include "addr.h"
#include "unix-std.h"
@@ -111,6 +111,7 @@ rspamd_control_finish_handler (struct rspamd_http_connection *conn,
const gchar *body;
gsize body_len;
struct rspamadm_control_cbdata *cbdata = conn->ud;
+ struct timeval exit_tv;
body = rspamd_http_message_get_body (msg, &body_len);
parser = ucl_parser_new (0);
@@ -141,7 +142,7 @@ rspamd_control_finish_handler (struct rspamd_http_connection *conn,
rspamd_fstring_free (out);
ucl_object_unref (obj);
ucl_parser_free (parser);
- return 0;
+ goto end;
}
else {
rspamd_ucl_emit_fstring (obj, UCL_EMIT_CONFIG, &out);
@@ -155,6 +156,11 @@ rspamd_control_finish_handler (struct rspamd_http_connection *conn,
ucl_parser_free (parser);
}
+end:
+ exit_tv.tv_sec = 0;
+ exit_tv.tv_usec = 0;
+ event_base_loopexit (rspamd_main->ev_base, &exit_tv);
+
return 0;
}
@@ -163,7 +169,6 @@ rspamadm_control (gint argc, gchar **argv, const struct rspamadm_command *_cmd)
{
GOptionContext *context;
GError *error = NULL;
- struct event_base *ev_base;
const gchar *cmd, *path = NULL;
struct rspamd_http_connection *conn;
struct rspamd_http_message *msg;
@@ -225,7 +230,6 @@ rspamadm_control (gint argc, gchar **argv, const struct rspamadm_command *_cmd)
exit (1);
}
- ev_base = event_init ();
sock = rspamd_inet_address_connect (addr, SOCK_STREAM, TRUE);
if (sock == -1) {
@@ -234,13 +238,14 @@ rspamadm_control (gint argc, gchar **argv, const struct rspamadm_command *_cmd)
exit (1);
}
- conn = rspamd_http_connection_new (NULL,
+ conn = rspamd_http_connection_new (
+ rspamd_main->http_ctx, /* Default context */
+ sock,
+ NULL,
rspamd_control_error_handler,
rspamd_control_finish_handler,
RSPAMD_HTTP_CLIENT_SIMPLE,
- RSPAMD_HTTP_CLIENT,
- NULL,
- NULL);
+ RSPAMD_HTTP_CLIENT);
msg = rspamd_http_new_message (HTTP_REQUEST);
msg->url = rspamd_fstring_new_init (path, strlen (path));
double_to_tv (timeout, &tv);
@@ -249,10 +254,10 @@ rspamadm_control (gint argc, gchar **argv, const struct rspamadm_command *_cmd)
cbdata.argv = argv;
cbdata.path = path;
- rspamd_http_connection_write_message (conn, msg, NULL, NULL, &cbdata, sock,
- &tv, ev_base);
+ rspamd_http_connection_write_message (conn, msg, NULL, NULL, &cbdata,
+ &tv);
- event_base_loop (ev_base, 0);
+ event_base_loop (rspamd_main->ev_base, 0);
rspamd_http_connection_unref (conn);
rspamd_inet_address_free (addr);
diff --git a/src/rspamadm/dkim_keygen.c b/src/rspamadm/dkim_keygen.c
index 86b228d01..62b47f557 100644
--- a/src/rspamadm/dkim_keygen.c
+++ b/src/rspamadm/dkim_keygen.c
@@ -17,6 +17,8 @@
#include "rspamadm.h"
#include "printf.h"
#include "str_util.h"
+#include "libcryptobox/cryptobox.h"
+#include "contrib/libottery/ottery.h"
#include "lua/lua_common.h"
#include <openssl/rsa.h>
#include <openssl/bn.h>
@@ -26,6 +28,7 @@ static gchar *privkey_file = NULL;
static gchar *selector = NULL;
static gchar *domain = NULL;
static guint bits = 1024;
+static gchar *type = "rsa";
static void rspamadm_dkim_keygen (gint argc, gchar **argv,
const struct rspamadm_command *cmd);
@@ -50,6 +53,8 @@ static GOptionEntry entries[] = {
"Save private key in the specified file", NULL},
{"bits", 'b', 0, G_OPTION_ARG_INT, &bits,
"Set key length to N bits (1024 by default)", NULL},
+ {"type", 't', 0, G_OPTION_ARG_STRING, &type,
+ "Key type: rsa or ed25519 (rsa by default)", NULL},
{NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL}
};
@@ -76,8 +81,9 @@ rspamadm_dkim_keygen_help (gboolean full_help, const struct rspamadm_command *cm
}
static void
-rspamadm_dkim_generate_keypair (const gchar *domain, const gchar *selector,
- const gchar *priv_fname, const gchar *pub_fname, guint keylen)
+rspamd_dkim_generate_rsa_keypair (const gchar *domain, const gchar *selector,
+ const gchar *priv_fname, const gchar *pub_fname,
+ guint keylen)
{
BIGNUM *e;
RSA *r;
@@ -109,8 +115,7 @@ rspamadm_dkim_generate_keypair (const gchar *domain, const gchar *selector,
priv_fname, strerror (errno));
exit (EXIT_FAILURE);
}
- }
- else {
+ } else {
privout = BIO_new_fp (stdout, 0);
}
@@ -125,7 +130,7 @@ rspamadm_dkim_generate_keypair (const gchar *domain, const gchar *selector,
BIO_free (privout);
fflush (stdout);
- pubout = BIO_new (BIO_s_mem());
+ pubout = BIO_new (BIO_s_mem ());
rc = i2d_RSA_PUBKEY_bio (pubout, r);
publen = BIO_get_mem_data (pubout, &pubdata);
@@ -141,18 +146,16 @@ rspamadm_dkim_generate_keypair (const gchar *domain, const gchar *selector,
pub_fname, strerror (errno));
exit (EXIT_FAILURE);
}
- }
- else {
+ } else {
pubfile = stdout;
}
if (b64_len < 255 - 2) {
rspamd_fprintf (pubfile, "%s._domainkey IN TXT ( \"v=DKIM1; k=rsa; \"\n"
- "\t\"p=%s\" ) ;\n",
+ "\t\"p=%s\" ) ;\n",
selector ? selector : "selector",
b64_data);
- }
- else {
+ } else {
guint i;
gint step = 253, remain = b64_len;
@@ -162,8 +165,7 @@ rspamadm_dkim_generate_keypair (const gchar *domain, const gchar *selector,
for (i = 0; i < b64_len; i += step, remain -= step) {
if (i == 0) {
rspamd_fprintf (pubfile, "\t\"p=%*s\"\n", MIN(step, remain), &b64_data[i]);
- }
- else {
+ } else {
step = 255;
rspamd_fprintf (pubfile, "\t\"%*s\"\n", MIN(step, remain), &b64_data[i]);
}
@@ -183,6 +185,120 @@ rspamadm_dkim_generate_keypair (const gchar *domain, const gchar *selector,
BN_free (e);
}
+static void
+rspamd_dkim_generate_ed25519_keypair (const gchar *domain, const gchar *selector,
+ const gchar *priv_fname, const gchar *pub_fname,
+ guint keylen, gboolean seeded)
+{
+ rspamd_sig_sk_t ed_sk;
+ rspamd_sig_pk_t ed_pk;
+ gchar *base64_pk, *base64_sk;
+ FILE *pubfile = NULL, *privfile = NULL;
+
+ rspamd_cryptobox_keypair_sig (ed_pk, ed_sk, RSPAMD_CRYPTOBOX_MODE_25519);
+ if (seeded) {
+ /* Just encode seed, not the full sk */
+ base64_sk = rspamd_encode_base64_common (ed_sk, 32, 0, NULL, FALSE,
+ RSPAMD_TASK_NEWLINES_LF);
+ }
+ else {
+ base64_sk = rspamd_encode_base64_common (ed_sk,
+ rspamd_cryptobox_sk_sig_bytes (RSPAMD_CRYPTOBOX_MODE_25519),
+ 0, NULL, FALSE,
+ RSPAMD_TASK_NEWLINES_LF);
+ }
+ base64_pk = rspamd_encode_base64_common (ed_pk, sizeof (ed_pk), 0, NULL, FALSE,
+ RSPAMD_TASK_NEWLINES_LF);
+
+ /* Cleanup sensitive data */
+ rspamd_explicit_memzero (ed_sk, sizeof (ed_sk));
+
+ if (priv_fname) {
+ privfile = fopen (priv_fname, "w");
+
+ if (privfile == NULL) {
+ rspamd_fprintf (stderr, "cannot open output file %s: %s\n",
+ priv_fname, strerror (errno));
+ rspamd_explicit_memzero (base64_sk, strlen (base64_sk));
+ g_free (base64_sk);
+ g_free (base64_pk);
+ exit (EXIT_FAILURE);
+ }
+ }
+ else {
+ privfile = stdout;
+ }
+
+ if (rspamd_fprintf (privfile, "%s\n", base64_sk) == -1) {
+ rspamd_fprintf (stderr, "cannot write to output file %s: %s\n",
+ priv_fname, strerror (errno));
+ rspamd_explicit_memzero (base64_sk, strlen (base64_sk));
+ g_free (base64_sk);
+ g_free (base64_pk);
+
+ if (privfile != stdout) {
+ fclose (privfile);
+ }
+
+ exit (EXIT_FAILURE);
+ }
+
+ if (privfile != stdout) {
+ fclose (privfile);
+ }
+
+ if (pub_fname) {
+ pubfile = fopen (pub_fname, "w");
+
+ if (pubfile == NULL) {
+ rspamd_fprintf (stderr, "cannot open output file %s: %s\n",
+ pub_fname, strerror (errno));
+ rspamd_explicit_memzero (base64_sk, strlen (base64_sk));
+ g_free (base64_sk);
+ g_free (base64_pk);
+ exit (EXIT_FAILURE);
+ }
+ }
+ else {
+ pubfile = stdout;
+ }
+
+ rspamd_fprintf (pubfile, "%s._domainkey IN TXT ( \"v=DKIM1; k=ed25519; \"\n"
+ "\t\"p=%s\" ) ;\n",
+ selector ? selector : "selector",
+ base64_pk);
+
+ if (pubfile != stdout) {
+ fclose (pubfile);
+ }
+
+ rspamd_explicit_memzero (base64_sk, strlen (base64_sk));
+ g_free (base64_sk);
+ g_free (base64_pk);
+}
+
+static void
+rspamadm_dkim_generate_keypair (const gchar *domain, const gchar *selector,
+ const gchar *priv_fname, const gchar *pub_fname, guint keylen)
+{
+ if (strcmp (type, "rsa") == 0) {
+ rspamd_dkim_generate_rsa_keypair (domain, selector, priv_fname,
+ pub_fname, keylen);
+ }
+ else if (strcmp (type, "ed25519") == 0) {
+ rspamd_dkim_generate_ed25519_keypair (domain, selector, priv_fname,
+ pub_fname, keylen, FALSE);
+ }
+ else if (strcmp (type, "ed25519-seed") == 0) {
+ rspamd_dkim_generate_ed25519_keypair (domain, selector, priv_fname,
+ pub_fname, keylen, TRUE);
+ }
+ else {
+ fprintf (stderr, "invalid key type: %s\n", type);
+ exit (EXIT_FAILURE);
+ }
+}
+
static gint
rspamadm_dkim_keygen_lua_generate (lua_State *L)
{
diff --git a/src/rspamadm/lua_repl.c b/src/rspamadm/lua_repl.c
index 6d1501100..e91bd34fa 100644
--- a/src/rspamadm/lua_repl.c
+++ b/src/rspamadm/lua_repl.c
@@ -16,8 +16,9 @@
#include "config.h"
#include "rspamadm.h"
-#include "libutil/http.h"
+#include "libutil/http_connection.h"
#include "libutil/http_private.h"
+#include "libutil/http_router.h"
#include "printf.h"
#include "lua/lua_common.h"
#include "lua/lua_thread_pool.h"
@@ -807,8 +808,9 @@ rspamadm_lua (gint argc, gchar **argv, const struct rspamadm_command *cmd)
ctx = g_malloc0 (sizeof (*ctx));
http = rspamd_http_router_new (rspamadm_lua_error_handler,
rspamadm_lua_finish_handler,
- NULL, ev_base,
- NULL, NULL);
+ NULL,
+ NULL,
+ rspamd_main->http_ctx);
ctx->L = L;
ctx->rt = http;
rspamd_http_router_add_path (http,
diff --git a/src/rspamadm/rspamadm.c b/src/rspamadm/rspamadm.c
index ef52af1e3..c0bb4bc72 100644
--- a/src/rspamadm/rspamadm.c
+++ b/src/rspamadm/rspamadm.c
@@ -436,6 +436,7 @@ main (gint argc, gchar **argv, gchar **env)
(void) dns_resolver_init (rspamd_main->logger,
rspamd_main->ev_base,
cfg);
+ rspamd_main->http_ctx = rspamd_http_context_create (cfg, rspamd_main->ev_base);
g_log_set_default_handler (rspamd_glib_log_function, rspamd_main->logger);
g_set_printerr_handler (rspamd_glib_printerr_function);
diff --git a/src/rspamd.c b/src/rspamd.c
index 88b44d773..27ba5e032 100644
--- a/src/rspamd.c
+++ b/src/rspamd.c
@@ -178,6 +178,7 @@ read_cmd_line (gint *argc, gchar ***argv, struct rspamd_config *cfg)
else {
cfg->cfg_name = cfg_names[0];
}
+
for (i = 1; i < cfg_num; i++) {
r = fork ();
if (r == 0) {
@@ -313,6 +314,9 @@ reread_config (struct rspamd_main *rspamd_main)
REF_RELEASE (old_cfg);
msg_info_main ("config has been reread successfully");
rspamd_map_preload (rspamd_main->cfg);
+
+ rspamd_main->cfg->rspamd_user = rspamd_user;
+ rspamd_main->cfg->rspamd_group = rspamd_group;
}
}
@@ -1159,7 +1163,7 @@ rspamd_control_handler (gint fd, short what, gpointer arg)
msg_info_main ("accepted control connection from %s",
rspamd_inet_address_to_string (addr));
- rspamd_control_process_client_socket (rspamd_main, nfd);
+ rspamd_control_process_client_socket (rspamd_main, nfd, addr);
}
static guint
@@ -1485,6 +1489,9 @@ main (gint argc, gchar **argv, gchar **env)
spawn_workers (rspamd_main, ev_base);
rspamd_mempool_unlock_mutex (rspamd_main->start_mtx);
+ rspamd_main->http_ctx = rspamd_http_context_create (rspamd_main->cfg,
+ ev_base);
+
if (control_fd != -1) {
msg_info_main ("listening for control commands on %s",
rspamd_inet_address_to_string (control_addr));
diff --git a/src/rspamd.h b/src/rspamd.h
index 74f08c2d3..10d3be9fb 100644
--- a/src/rspamd.h
+++ b/src/rspamd.h
@@ -22,7 +22,7 @@
#include "libutil/mem_pool.h"
#include "libutil/util.h"
#include "libutil/logger.h"
-#include "libutil/http.h"
+#include "libutil/http_connection.h"
#include "libutil/upstream.h"
#include "libutil/radix.h"
#include "libserver/url.h"
@@ -277,6 +277,7 @@ struct rspamd_main {
gboolean cores_throttling; /**< turn off cores when limits are exceeded */
struct roll_history *history; /**< rolling history */
struct event_base *ev_base;
+ struct rspamd_http_context *http_ctx;
};
enum rspamd_exception_type {
diff --git a/src/rspamd_proxy.c b/src/rspamd_proxy.c
index 9ea2b0d74..b6ede29b9 100644
--- a/src/rspamd_proxy.c
+++ b/src/rspamd_proxy.c
@@ -17,7 +17,7 @@
#include "libutil/util.h"
#include "libutil/map.h"
#include "libutil/upstream.h"
-#include "libutil/http.h"
+#include "libutil/http_connection.h"
#include "libutil/http_private.h"
#include "libserver/protocol.h"
#include "libserver/protocol_internal.h"
@@ -123,18 +123,14 @@ struct rspamd_proxy_ctx {
struct timeval io_tv;
/* Encryption key for clients */
struct rspamd_cryptobox_keypair *key;
- /* Keys cache */
- struct rspamd_keypair_cache *keys_cache;
+ /* HTTP context */
+ struct rspamd_http_context *http_ctx;
/* Upstreams to use */
GHashTable *upstreams;
/* Mirrors to send traffic to */
GPtrArray *mirrors;
/* Default upstream */
struct rspamd_http_upstream *default_upstream;
- /* Local rotating keypair for upstreams */
- struct rspamd_cryptobox_keypair *local_key;
- struct event rotate_ev;
- gdouble rotate_tm;
lua_State *lua_state;
/* Array of callback functions called on end of scan to compare results */
GArray *cmp_refs;
@@ -746,7 +742,6 @@ init_rspamd_proxy (struct rspamd_config *cfg)
ctx->mirrors = g_ptr_array_new ();
rspamd_mempool_add_destructor (cfg->cfg_pool,
(rspamd_mempool_destruct_t)rspamd_ptr_array_free_hard, ctx->mirrors);
- ctx->rotate_tm = DEFAULT_ROTATION_TIME;
ctx->cfg = cfg;
ctx->lua_state = cfg->lua_state;
ctx->cmp_refs = g_array_new (FALSE, FALSE, sizeof (gint));
@@ -765,15 +760,6 @@ init_rspamd_proxy (struct rspamd_config *cfg)
"IO timeout");
rspamd_rcl_register_worker_option (cfg,
type,
- "rotate",
- rspamd_rcl_parse_struct_time,
- ctx,
- G_STRUCT_OFFSET (struct rspamd_proxy_ctx, rotate_tm),
- RSPAMD_CL_FLAG_TIME_FLOAT,
- "Rotation keys time, default: "
- G_STRINGIFY (DEFAULT_ROTATION_TIME) " seconds");
- rspamd_rcl_register_worker_option (cfg,
- type,
"keypair",
rspamd_rcl_parse_struct_keypair,
ctx,
@@ -1309,7 +1295,8 @@ proxy_backend_mirror_error_handler (struct rspamd_http_connection *conn, GError
msg_info_session ("abnormally closing connection from backend: %s:%s, "
"error: %e",
bk_conn->name,
- rspamd_inet_address_to_string (rspamd_upstream_addr (bk_conn->up)),
+ rspamd_inet_address_to_string (
+ rspamd_upstream_addr_cur (bk_conn->up)),
err);
if (err) {
@@ -1337,7 +1324,8 @@ proxy_backend_mirror_finish_handler (struct rspamd_http_connection *conn,
bk_conn->parser_from_ref, msg->body_buf.begin, msg->body_buf.len)) {
msg_warn_session ("cannot parse results from the mirror backend %s:%s",
bk_conn->name,
- rspamd_inet_address_to_string (rspamd_upstream_addr (bk_conn->up)));
+ rspamd_inet_address_to_string (
+ rspamd_upstream_addr_cur (bk_conn->up)));
bk_conn->err = "cannot parse ucl";
}
@@ -1387,7 +1375,7 @@ proxy_open_mirror_connections (struct rspamd_proxy_session *session)
}
bk_conn->backend_sock = rspamd_inet_address_connect (
- rspamd_upstream_addr (bk_conn->up),
+ rspamd_upstream_addr_next (bk_conn->up),
SOCK_STREAM, TRUE);
if (bk_conn->backend_sock == -1) {
@@ -1416,23 +1404,22 @@ proxy_open_mirror_connections (struct rspamd_proxy_session *session)
rspamd_http_message_add_header (msg, "Settings-ID", m->settings_id);
}
- bk_conn->backend_conn = rspamd_http_connection_new (NULL,
+ bk_conn->backend_conn = rspamd_http_connection_new (
+ session->ctx->http_ctx,
+ bk_conn->backend_sock,
+ NULL,
proxy_backend_mirror_error_handler,
proxy_backend_mirror_finish_handler,
RSPAMD_HTTP_CLIENT_SIMPLE,
- RSPAMD_HTTP_CLIENT,
- session->ctx->keys_cache,
- NULL);
+ RSPAMD_HTTP_CLIENT);
if (m->key) {
- rspamd_http_connection_set_key (bk_conn->backend_conn,
- session->ctx->local_key);
msg->peer_key = rspamd_pubkey_ref (m->key);
}
if (m->local ||
rspamd_inet_address_is_local (
- rspamd_upstream_addr (bk_conn->up), FALSE)) {
+ rspamd_upstream_addr_cur (bk_conn->up), FALSE)) {
if (session->fname) {
rspamd_http_message_add_header (msg, "File", session->fname);
@@ -1441,8 +1428,7 @@ proxy_open_mirror_connections (struct rspamd_proxy_session *session)
msg->method = HTTP_GET;
rspamd_http_connection_write_message_shared (bk_conn->backend_conn,
msg, NULL, NULL, bk_conn,
- bk_conn->backend_sock,
- bk_conn->io_tv, session->ctx->ev_base);
+ bk_conn->io_tv);
}
else {
if (session->fname) {
@@ -1469,8 +1455,7 @@ proxy_open_mirror_connections (struct rspamd_proxy_session *session)
rspamd_http_connection_write_message (bk_conn->backend_conn,
msg, NULL, NULL, bk_conn,
- bk_conn->backend_sock,
- bk_conn->io_tv, session->ctx->ev_base);
+ bk_conn->io_tv);
}
g_ptr_array_add (session->mirror_conns, bk_conn);
@@ -1495,8 +1480,8 @@ proxy_client_write_error (struct rspamd_proxy_session *session, gint code,
reply->code = code;
reply->status = rspamd_fstring_new_init (status, strlen (status));
rspamd_http_connection_write_message (session->client_conn,
- reply, NULL, NULL, session, session->client_sock,
- &session->ctx->io_tv, session->ctx->ev_base);
+ reply, NULL, NULL, session,
+ &session->ctx->io_tv);
}
}
@@ -1509,7 +1494,8 @@ proxy_backend_master_error_handler (struct rspamd_http_connection *conn, GError
session = bk_conn->s;
msg_info_session ("abnormally closing connection from backend: %s, error: %e,"
" retries left: %d",
- rspamd_inet_address_to_string (rspamd_upstream_addr (session->master_conn->up)),
+ rspamd_inet_address_to_string (
+ rspamd_upstream_addr_cur (session->master_conn->up)),
err,
session->ctx->max_retries - session->retries);
session->retries ++;
@@ -1531,7 +1517,7 @@ proxy_backend_master_error_handler (struct rspamd_http_connection *conn, GError
msg_info_session ("retry connection to: %s"
" retries left: %d",
rspamd_inet_address_to_string (
- rspamd_upstream_addr (session->master_conn->up)),
+ rspamd_upstream_addr_cur (session->master_conn->up)),
session->ctx->max_retries - session->retries);
}
}
@@ -1592,8 +1578,8 @@ proxy_backend_master_finish_handler (struct rspamd_http_connection *conn,
}
else {
rspamd_http_connection_write_message (session->client_conn,
- msg, NULL, NULL, session, session->client_sock,
- bk_conn->io_tv, session->ctx->ev_base);
+ msg, NULL, NULL, session,
+ bk_conn->io_tv);
}
return 0;
@@ -1652,9 +1638,7 @@ rspamd_proxy_scan_self_reply (struct rspamd_task *task)
NULL,
ctype,
session,
- session->client_sock,
- NULL,
- session->ctx->ev_base);
+ NULL);
}
}
@@ -1821,14 +1805,15 @@ retry:
}
session->master_conn->backend_sock = rspamd_inet_address_connect (
- rspamd_upstream_addr (session->master_conn->up),
+ rspamd_upstream_addr_next (session->master_conn->up),
SOCK_STREAM, TRUE);
if (session->master_conn->backend_sock == -1) {
msg_err_session ("cannot connect upstream: %s(%s)",
host ? hostbuf : "default",
- rspamd_inet_address_to_string (rspamd_upstream_addr (
- session->master_conn->up)));
+ rspamd_inet_address_to_string (
+ rspamd_upstream_addr_cur (
+ session->master_conn->up)));
rspamd_upstream_fail (session->master_conn->up, TRUE);
session->retries ++;
goto retry;
@@ -1847,21 +1832,19 @@ retry:
}
session->master_conn->backend_conn = rspamd_http_connection_new (
+ session->ctx->http_ctx,
+ session->master_conn->backend_sock,
NULL,
proxy_backend_master_error_handler,
proxy_backend_master_finish_handler,
RSPAMD_HTTP_CLIENT_SIMPLE,
- RSPAMD_HTTP_CLIENT,
- session->ctx->keys_cache,
- NULL);
+ RSPAMD_HTTP_CLIENT);
session->master_conn->flags &= ~RSPAMD_BACKEND_CLOSED;
session->master_conn->parser_from_ref = backend->parser_from_ref;
session->master_conn->parser_to_ref = backend->parser_to_ref;
if (backend->key) {
msg->peer_key = rspamd_pubkey_ref (backend->key);
- rspamd_http_connection_set_key (session->master_conn->backend_conn,
- session->ctx->local_key);
}
if (backend->settings_id != NULL) {
@@ -1872,7 +1855,8 @@ retry:
if (backend->local ||
rspamd_inet_address_is_local (
- rspamd_upstream_addr (session->master_conn->up), FALSE)) {
+ rspamd_upstream_addr_cur (
+ session->master_conn->up), FALSE)) {
if (session->fname) {
rspamd_http_message_add_header (msg, "File", session->fname);
@@ -1883,8 +1867,7 @@ retry:
rspamd_http_connection_write_message_shared (
session->master_conn->backend_conn,
msg, NULL, NULL, session->master_conn,
- session->master_conn->backend_sock,
- session->master_conn->io_tv, session->ctx->ev_base);
+ session->master_conn->io_tv);
}
else {
if (session->fname) {
@@ -1912,8 +1895,7 @@ retry:
rspamd_http_connection_write_message (
session->master_conn->backend_conn,
msg, NULL, NULL, session->master_conn,
- session->master_conn->backend_sock,
- session->master_conn->io_tv, session->ctx->ev_base);
+ session->master_conn->io_tv);
}
}
@@ -2100,13 +2082,14 @@ proxy_accept_socket (gint fd, short what, void *arg)
}
if (!ctx->milter) {
- session->client_conn = rspamd_http_connection_new (NULL,
+ session->client_conn = rspamd_http_connection_new (
+ ctx->http_ctx,
+ nfd,
+ NULL,
proxy_client_error_handler,
proxy_client_finish_handler,
0,
- RSPAMD_HTTP_SERVER,
- ctx->keys_cache,
- NULL);
+ RSPAMD_HTTP_SERVER);
if (ctx->key) {
rspamd_http_connection_set_key (session->client_conn, ctx->key);
@@ -2118,9 +2101,7 @@ proxy_accept_socket (gint fd, short what, void *arg)
rspamd_http_connection_read_message_shared (session->client_conn,
session,
- nfd,
- &ctx->io_tv,
- ctx->ev_base);
+ &ctx->io_tv);
}
else {
msg_info_session ("accepted milter connection from %s port %d",
@@ -2154,24 +2135,6 @@ proxy_accept_socket (gint fd, short what, void *arg)
}
static void
-proxy_rotate_key (gint fd, short what, void *arg)
-{
- struct timeval rot_tv;
- struct rspamd_proxy_ctx *ctx = arg;
- gpointer kp;
-
- double_to_tv (ctx->rotate_tm, &rot_tv);
- rot_tv.tv_sec += ottery_rand_range (rot_tv.tv_sec);
- event_del (&ctx->rotate_ev);
- event_add (&ctx->rotate_ev, &rot_tv);
-
- kp = ctx->local_key;
- ctx->local_key = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX,
- RSPAMD_CRYPTOBOX_MODE_25519);
- rspamd_keypair_unref (kp);
-}
-
-static void
adjust_upstreams_limits (struct rspamd_proxy_ctx *ctx)
{
struct rspamd_http_upstream *backend;
@@ -2200,9 +2163,9 @@ adjust_upstreams_limits (struct rspamd_proxy_ctx *ctx)
}
void
-start_rspamd_proxy (struct rspamd_worker *worker) {
+start_rspamd_proxy (struct rspamd_worker *worker)
+{
struct rspamd_proxy_ctx *ctx = worker->ctx;
- struct timeval rot_tv;
ctx->cfg = worker->srv->cfg;
ctx->ev_base = rspamd_prepare_worker (worker, "rspamd_proxy",
@@ -2217,16 +2180,7 @@ start_rspamd_proxy (struct rspamd_worker *worker) {
rspamd_upstreams_library_config (worker->srv->cfg, ctx->cfg->ups_ctx,
ctx->ev_base, ctx->resolver->r);
- /* XXX: stupid default */
- ctx->keys_cache = rspamd_keypair_cache_new (256);
- ctx->local_key = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX,
- RSPAMD_CRYPTOBOX_MODE_25519);
-
- double_to_tv (ctx->rotate_tm, &rot_tv);
- rot_tv.tv_sec += ottery_rand_range (rot_tv.tv_sec);
- event_set (&ctx->rotate_ev, -1, EV_TIMEOUT, proxy_rotate_key, ctx);
- event_base_set (ctx->ev_base, &ctx->rotate_ev);
- event_add (&ctx->rotate_ev, &rot_tv);
+ ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base);
if (ctx->has_self_scan) {
/* Additional initialisation needed */
@@ -2259,8 +2213,9 @@ start_rspamd_proxy (struct rspamd_worker *worker) {
rspamd_stat_close ();
}
- rspamd_keypair_cache_destroy (ctx->keys_cache);
+ struct rspamd_http_context *http_ctx = ctx->http_ctx;
REF_RELEASE (ctx->cfg);
+ rspamd_http_context_free (http_ctx);
rspamd_log_close (worker->srv->logger, TRUE);
exit (EXIT_SUCCESS);
diff --git a/src/worker.c b/src/worker.c
index 2f7176a8d..d81be54a1 100644
--- a/src/worker.c
+++ b/src/worker.c
@@ -331,9 +331,7 @@ rspamd_worker_error_handler (struct rspamd_http_connection *conn, GError *err)
NULL,
"application/json",
task,
- task->http_conn->fd,
- &task->tv,
- task->ev_base);
+ &task->tv);
}
}
@@ -414,13 +412,14 @@ accept_socket (gint fd, short what, void *arg)
http_opts = RSPAMD_HTTP_REQUIRE_ENCRYPTION;
}
- task->http_conn = rspamd_http_connection_new (rspamd_worker_body_handler,
+ task->http_conn = rspamd_http_connection_new (
+ ctx->http_ctx,
+ nfd,
+ rspamd_worker_body_handler,
rspamd_worker_error_handler,
rspamd_worker_finish_handler,
http_opts,
- RSPAMD_HTTP_SERVER,
- ctx->keys_cache,
- NULL);
+ RSPAMD_HTTP_SERVER);
rspamd_http_connection_set_max_size (task->http_conn, task->cfg->max_message);
worker->nconns++;
rspamd_mempool_add_destructor (task->task_pool,
@@ -436,9 +435,7 @@ accept_socket (gint fd, short what, void *arg)
rspamd_http_connection_read_message (task->http_conn,
task,
- nfd,
- &ctx->io_tv,
- ctx->ev_base);
+ &ctx->io_tv);
}
#ifdef WITH_HYPERSCAN
@@ -696,8 +693,7 @@ start_worker (struct rspamd_worker *worker)
rspamd_upstreams_library_config (worker->srv->cfg, ctx->cfg->ups_ctx,
ctx->ev_base, ctx->resolver->r);
- /* XXX: stupid default */
- ctx->keys_cache = rspamd_keypair_cache_new (256);
+ ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base);
rspamd_worker_init_scanner (worker, ctx->ev_base, ctx->resolver,
&ctx->lang_det);
rspamd_lua_run_postloads (ctx->cfg->lua_state, ctx->cfg, ctx->ev_base,
@@ -707,8 +703,9 @@ start_worker (struct rspamd_worker *worker)
rspamd_worker_block_signals ();
rspamd_stat_close ();
- rspamd_keypair_cache_destroy (ctx->keys_cache);
+ struct rspamd_http_context *http_ctx = ctx->http_ctx;
REF_RELEASE (ctx->cfg);
+ rspamd_http_context_free (http_ctx);
rspamd_log_close (worker->srv->logger, TRUE);
exit (EXIT_SUCCESS);
diff --git a/src/worker_private.h b/src/worker_private.h
index f07a95b41..398c5d23d 100644
--- a/src/worker_private.h
+++ b/src/worker_private.h
@@ -49,7 +49,7 @@ struct rspamd_worker_ctx {
/* Encryption key */
struct rspamd_cryptobox_keypair *key;
/* Keys cache */
- struct rspamd_keypair_cache *keys_cache;
+ struct rspamd_http_context *http_ctx;
/* Language detector */
struct rspamd_lang_detector *lang_det;
};
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index ee6b074c3..10f0a19da 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -8,7 +8,6 @@ SET(TESTSRC rspamd_mem_pool_test.c
rspamd_radix_test.c
rspamd_shingles_test.c
rspamd_upstream_test.c
- rspamd_http_test.c
rspamd_lua_pcall_vs_resume_test.c
rspamd_lua_test.c
rspamd_cryptobox_test.c
@@ -19,11 +18,9 @@ ADD_EXECUTABLE(rspamd-test EXCLUDE_FROM_ALL ${TESTSRC})
SET_TARGET_PROPERTIES(rspamd-test PROPERTIES LINKER_LANGUAGE C)
SET_TARGET_PROPERTIES(rspamd-test PROPERTIES COMPILE_FLAGS "-DRSPAMD_TEST")
ADD_DEPENDENCIES(rspamd-test rspamd-server)
-TARGET_LINK_LIBRARIES(rspamd-test ${RSPAMD_REQUIRED_LIBRARIES})
IF(USE_CXX_LINKER)
SET_TARGET_PROPERTIES(rspamd-test PROPERTIES LINKER_LANGUAGE CXX)
ENDIF()
-TARGET_LINK_LIBRARIES(rspamd-test rspamd-actrie)
TARGET_LINK_LIBRARIES(rspamd-test rspamd-server)
IF(NOT "${CMAKE_CURRENT_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_BINARY_DIR}")
diff --git a/test/functional/cases/106_mid.robot b/test/functional/cases/106_mid.robot
new file mode 100644
index 000000000..7d8459dbd
--- /dev/null
+++ b/test/functional/cases/106_mid.robot
@@ -0,0 +1,42 @@
+*** Settings ***
+Suite Setup MID Setup
+Suite Teardown Normal Teardown
+Library ${TESTDIR}/lib/rspamd.py
+Resource ${TESTDIR}/lib/rspamd.robot
+Variables ${TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${TESTDIR}/configs/plugins.conf
+${RSPAMD_SCOPE} Suite
+${URL_TLD} ${TESTDIR}/../lua/unit/test_tld.dat
+
+*** Test Cases ***
+MID - invalid Message-ID
+ ${result} = Scan Message With Rspamc ${TESTDIR}/messages/fws_fp.eml
+ Check Rspamc ${result} INVALID_MSGID (1.70)
+ Should Not Contain ${result.stdout} MISSING_MID
+ Should Not Contain ${result.stdout} INVALID_MSGID_ALLOWED
+
+MID - invalid Message-ID allowed
+ ${result} = Scan Message With Rspamc ${TESTDIR}/messages/invalid_mid_allowed.eml
+ Check Rspamc ${result} INVALID_MSGID_ALLOWED (1.00)
+ Should Not Contain ${result.stdout} MISSING_MID
+ Should Not Contain ${result.stdout} INVALID_MSGID (
+
+MID - missing Message-ID
+ ${result} = Scan Message With Rspamc ${TESTDIR}/messages/freemail.eml
+ Check Rspamc ${result} MISSING_MID (2.50)
+ Should Not Contain ${result.stdout} MISSING_MID_ALLOWED
+ Should Not Contain ${result.stdout} INVALID_MSGID
+
+MID - missing Message-ID allowed
+ ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/onsubdomain_pass_relaxed.eml
+ Check Rspamc ${result} MISSING_MID_ALLOWED (1.00)
+ Should Not Contain ${result.stdout} MISSING_MID (
+ Should Not Contain ${result.stdout} INVALID_MSGID
+
+*** Keywords ***
+MID Setup
+ ${PLUGIN_CONFIG} = Get File ${TESTDIR}/configs/mid.conf
+ Set Suite Variable ${PLUGIN_CONFIG}
+ Generic Setup PLUGIN_CONFIG
diff --git a/test/functional/cases/131_dkim_signing/001_simple.robot b/test/functional/cases/131_dkim_signing/001_simple.robot
index 96bdcc997..2bcd53f36 100644
--- a/test/functional/cases/131_dkim_signing/001_simple.robot
+++ b/test/functional/cases/131_dkim_signing/001_simple.robot
@@ -19,6 +19,11 @@ TEST SIGNED
Check Rspamc ${result} DKIM-Signature:
Should Contain ${result.stdout} DKIM_SIGNED
+TEST SIGNED HTTP HEADERS
+ ${result} = Scan Message With Rspamc ${MESSAGE} -u bob@cacophony.za.org --header=PerformDkimSign:yes --header=DkimDomain:cacophony.za.org --header=DkimSelector:dkim --header=DkimPrivateKey:MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANe3EETkiI1Exyrb+VzbMSt90K8MXJA0GcyNs6MFCs9JPaTh90Zu2l7ki7m5LTUx6350AR/3hcvwjSHCZjD6fvQ8/zfjN8kaLZ6DAaqtqSlpawIM+8glkuTEkIkpBED/OtDrba4Rd29iLFVuwQZXDtTjAAZKZPmtTZ5TXLrcCU6VAgMBAAECgYEA1BFvmBsIN8Gu/+6kNupya2xUNVM0yLu/xT5lpNV3LBO325oejAq8+d87kkl/LTW3a2jGFlQ0ICuLw+2mo24QUWRyv8if3oeBMlnLqHE+6wNjFVqo5sOjKzjO363xSXwXNUrBT7jDhnZcDN8w3/FecYKjifGTVtUs1SLsYwhlc8ECQQDuCRymLZQ/imPn5eFVIydwUzg8ptZlvoA7bfIxUL9BQRX33s59kLCilA0tTed8Dd+GnxsT93XOj1ApIfBwmTSlAkEA5/63PDsN7fH+WInqVD8nU07M9S8LcGDlPbVVBr2S2I78/iwrSDAYtbkU2vEbhFK/JuKNML2j8OkzV3v1QulfMQJBALDzhx+l/HHr3+8RPhx7QKNIyiKUaAdEwbDsP8IXY8YPq1QThu9jM1v4sX7/TdkzuvoppwiFykbe1NlvCH279p0CQCmTg4Ee0DtBcCSr6rvYaZLLf329RZ6JLuwlMCy6ErQOxBZFEiiovfTrS2qFZToMnkc4uLbwdY36LQJTq7unGTECQCCok8LzBeZtAw+TJofpOM3F2Rlm2qXiBVBeubhRedsiljG0hpvvLJBMppnQ6r27p5Jk39SmaTRkxEKrxPWWLNM=
+ Check Rspamc ${result} DKIM-Signature:
+ Should Contain ${result.stdout} DKIM_SIGNED
+
TEST NOT SIGNED - USERNAME WRONG DOMAIN
${result} = Scan Message With Rspamc ${MESSAGE} -u bob@example.tk
Check Rspamc ${result} DKIM-Signature: inverse=1
diff --git a/test/functional/cases/231_tcp_down.robot b/test/functional/cases/231_tcp_down.robot
index 474faa3f5..f852724ad 100644
--- a/test/functional/cases/231_tcp_down.robot
+++ b/test/functional/cases/231_tcp_down.robot
@@ -16,7 +16,7 @@ ${RSPAMD_SCOPE} Test
*** Test Cases ***
Sync API TCP get request when server is down
- [Documentation] We don't create HTTP server here, that's why
+ [Documentation] We don't create HTTP server here, that's why
... all requests fail with "connection refused"
Check url /request get HTTP_ASYNC_RESPONSE (0.00)[Socket error detected: Connection refused]
Check url /content-length HTTP_SYNC_WRITE_ERROR (0.00)[Socket error detected: Connection refused]
diff --git a/test/functional/configs/dkim_signing/simple.conf b/test/functional/configs/dkim_signing/simple.conf
index 2302a0c4f..14b8d5ec6 100644
--- a/test/functional/configs/dkim_signing/simple.conf
+++ b/test/functional/configs/dkim_signing/simple.conf
@@ -2,4 +2,6 @@ dkim_signing {
path = "${TESTDIR}/configs/dkim.key";
check_pubkey = true;
allow_pubkey_mismatch = false;
+ use_http_headers = true;
+ allow_headers_fallback = true;
}
diff --git a/test/functional/configs/maps/mid.list b/test/functional/configs/maps/mid.list
new file mode 100644
index 000000000..b89c084f0
--- /dev/null
+++ b/test/functional/configs/maps/mid.list
@@ -0,0 +1,2 @@
+cacophony.za.org /^<[A-z0-9+]{18}>$/
+mom.za.org
diff --git a/test/functional/configs/mid.conf b/test/functional/configs/mid.conf
new file mode 100644
index 000000000..49f5cb568
--- /dev/null
+++ b/test/functional/configs/mid.conf
@@ -0,0 +1,9 @@
+mid = {
+ source = {
+ url = [
+ "https://maps.rspamd.com/rspamd/mid.inc.zst",
+ "fallback+file://${TESTDIR}/../../../conf/mid.inc",
+ "file://${TESTDIR}/configs/maps/mid.list"
+ ];
+ }
+}
diff --git a/test/functional/lua/dns.lua b/test/functional/lua/dns.lua
index 62b7d21e6..702b98550 100644
--- a/test/functional/lua/dns.lua
+++ b/test/functional/lua/dns.lua
@@ -22,7 +22,8 @@ rspamd_config:register_symbol({
name = 'SIMPLE_DNS_SYNC',
score = 1.0,
callback = dns_sync_symbol,
- no_squeeze = true
+ no_squeeze = true,
+ flags = 'coro',
})
diff --git a/test/functional/lua/http.lua b/test/functional/lua/http.lua
index 0263beb6f..1981c8734 100644
--- a/test/functional/lua/http.lua
+++ b/test/functional/lua/http.lua
@@ -109,7 +109,8 @@ rspamd_config:register_symbol({
name = 'SIMPLE_TEST',
score = 1.0,
callback = http_symbol,
-no_squeeze = true
+no_squeeze = true,
+flags = 'coro'
})
diff --git a/test/functional/lua/redis.lua b/test/functional/lua/redis.lua
index e4512b5e4..1a1eaf2b7 100644
--- a/test/functional/lua/redis.lua
+++ b/test/functional/lua/redis.lua
@@ -23,12 +23,12 @@ local function redis_simple_async_symbol(task)
end
redis_lua.rspamd_redis_make_request(
- task,
+ task,
redis_params,
"test_key",
false,
redis_cb,
- 'GET',
+ 'GET',
{'test_key'}
)
end
@@ -47,7 +47,7 @@ local function redis_simple_async_api201809(task)
callback = redis_cb
}
local request = {
- 'GET',
+ 'GET',
'test_key'
}
redis_lua.request(redis_params, attrs, request)
@@ -108,6 +108,6 @@ rspamd_config:register_symbol({
name = 'REDIS_TEST',
score = 1.0,
callback = redis_symbol,
- no_squeeze = true
+ flags = 'coro',
})
-- ]]
diff --git a/test/functional/lua/tcp.lua b/test/functional/lua/tcp.lua
index 30285c4f5..d032a049f 100644
--- a/test/functional/lua/tcp.lua
+++ b/test/functional/lua/tcp.lua
@@ -61,7 +61,7 @@ local function http_simple_tcp_symbol(task)
local data
local got_content = ''
repeat
- is_ok, data = connection:read_once();
+ is_ok, data = connection:read_once();
logger.errx(task, 'read_once: is_ok: %1, data: %2', is_ok, data)
if not is_ok then
task:insert_result('HTTP_SYNC_ERROR', 1.0, data)
@@ -147,7 +147,7 @@ local function http_tcp_symbol(task)
end
local value
- local header = header_line:gsub("([%w-]+): (.*)",
+ local header = header_line:gsub("([%w-]+): (.*)",
function (h, v) value = v; return h:lower() end)
logger.errx(task, 'parsed header: %1 -> "%2"', header, value)
@@ -182,13 +182,15 @@ rspamd_config:register_symbol({
name = 'SIMPLE_TCP_TEST',
score = 1.0,
callback = http_simple_tcp_symbol,
- no_squeeze = true
+ no_squeeze = true,
+ flags = 'coro',
})
rspamd_config:register_symbol({
name = 'HTTP_TCP_TEST',
score = 1.0,
callback = http_tcp_symbol,
- no_squeeze = true
+ no_squeeze = true,
+ flags = 'coro',
})
-- ]]
diff --git a/test/functional/messages/invalid_mid_allowed.eml b/test/functional/messages/invalid_mid_allowed.eml
new file mode 100644
index 000000000..a7ee95081
--- /dev/null
+++ b/test/functional/messages/invalid_mid_allowed.eml
@@ -0,0 +1,10 @@
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=cacophony.za.org;
+ s=dkim; t=1550213458;
+ h=from:sender:reply-to:subject:date:message-id:message-id:to:cc:
+ mime-version:content-type:content-transfer-encoding:in-reply-to:
+ references; bh=47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=;
+ b=vcs/v32V8AvN1x0qk8DihZq46R2C35z1A4h6xEg26dDq2JSVZQOd1+WYRtmHG5SL6JlQsb
+ KxBHbIAFIoah7JA+uKt49mrKOVcnq9dww/P4Os3lDjm7WMwjDb8eEX2kECINjVo5MDMp6C
+ cCMnjVLppV76SL8f5ExdHDsw6eZ9y9k=
+Message-ID: <JE47oX6KasQDNv6BEw>
+
diff --git a/test/rspamd_http_test.c b/test/rspamd_http_test.c
index e70c396aa..b81cc0b8b 100644
--- a/test/rspamd_http_test.c
+++ b/test/rspamd_http_test.c
@@ -16,7 +16,8 @@
#include "config.h"
#include "rspamd.h"
#include "util.h"
-#include "libutil/http.h"
+#include "libutil/http_connection.h"
+#include "libutil/http_router.h"
#include "libutil/http_private.h"
#include "tests.h"
#include "ottery.h"
diff --git a/test/rspamd_test_suite.c b/test/rspamd_test_suite.c
index 6ca8d6465..2abce72a1 100644
--- a/test/rspamd_test_suite.c
+++ b/test/rspamd_test_suite.c
@@ -74,13 +74,13 @@ main (int argc, char **argv)
g_test_add_func ("/rspamd/rrd", rspamd_rrd_test_func);
g_test_add_func ("/rspamd/upstream", rspamd_upstream_test_func);
g_test_add_func ("/rspamd/shingles", rspamd_shingles_test_func);
- g_test_add_func ("/rspamd/http", rspamd_http_test_func);
g_test_add_func ("/rspamd/lua", rspamd_lua_test_func);
g_test_add_func ("/rspamd/cryptobox", rspamd_cryptobox_test_func);
g_test_add_func ("/rspamd/heap", rspamd_heap_test_func);
g_test_add_func ("/rspamd/lua_pcall", rspamd_lua_lua_pcall_vs_resume_test_func);
#if 0
+ g_test_add_func ("/rspamd/http", rspamd_http_test_func);
g_test_add_func ("/rspamd/url", rspamd_url_test_func);
g_test_add_func ("/rspamd/statfile", rspamd_statfile_test_func);
g_test_add_func ("/rspamd/aio", rspamd_async_test_func);
diff --git a/test/rspamd_upstream_test.c b/test/rspamd_upstream_test.c
index 4e4f1ae87..f9eb2bcfd 100644
--- a/test/rspamd_upstream_test.c
+++ b/test/rspamd_upstream_test.c
@@ -87,30 +87,30 @@ rspamd_upstream_test_func (void)
rspamd_parse_inet_address (&paddr, "::1", 0);
g_assert (rspamd_upstream_add_addr (up, paddr));
/* Rewind to start */
- addr = rspamd_upstream_addr (up);
- addr = rspamd_upstream_addr (up);
+ addr = rspamd_upstream_addr_next (up);
+ addr = rspamd_upstream_addr_next (up);
/* cur should be zero here */
- addr = rspamd_upstream_addr (up);
- next_addr = rspamd_upstream_addr (up);
+ addr = rspamd_upstream_addr_next (up);
+ next_addr = rspamd_upstream_addr_next (up);
g_assert (rspamd_inet_address_get_af (addr) == AF_INET);
g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET);
- next_addr = rspamd_upstream_addr (up);
+ next_addr = rspamd_upstream_addr_next (up);
g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET6);
- next_addr = rspamd_upstream_addr (up);
+ next_addr = rspamd_upstream_addr_next (up);
g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET);
- next_addr = rspamd_upstream_addr (up);
+ next_addr = rspamd_upstream_addr_next (up);
g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET);
- next_addr = rspamd_upstream_addr (up);
+ next_addr = rspamd_upstream_addr_next (up);
g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET6);
/* Test errors with IPv6 */
rspamd_upstream_fail (up, TRUE);
/* Now we should have merely IPv4 addresses in rotation */
- addr = rspamd_upstream_addr (up);
+ addr = rspamd_upstream_addr_next (up);
for (i = 0; i < 256; i++) {
- next_addr = rspamd_upstream_addr (up);
+ next_addr = rspamd_upstream_addr_next (up);
g_assert (rspamd_inet_address_get_af (addr) == AF_INET);
g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET);
- g_assert (rspamd_inet_address_compare (addr, next_addr) != 0);
+ g_assert (rspamd_inet_address_compare (addr, next_addr, FALSE) != 0);
addr = next_addr;
}
rspamd_upstreams_destroy (nls);