diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/controller.c | 120 | ||||
| -rw-r--r-- | src/libutil/map.c | 9 | ||||
| -rw-r--r-- | src/libutil/rrd.c | 24 | ||||
| -rw-r--r-- | src/plugins/lua/phishing.lua | 163 |
4 files changed, 245 insertions, 71 deletions
diff --git a/src/controller.c b/src/controller.c index 085eda5d4..b45e5add5 100644 --- a/src/controller.c +++ b/src/controller.c @@ -1022,6 +1022,45 @@ rspamd_controller_handle_pie_chart ( return 0; } +void +rspamd_controller_graph_point (gulong t, gulong step, + struct rspamd_rrd_query_result* rrd_result, + gdouble *acc, + ucl_object_t **elt) +{ + guint nan_cnt; + gdouble sum = 0.0, yval; + ucl_object_t* data_elt; + guint i, j; + + for (i = 0; i < rrd_result->ds_count; i++) { + sum = 0.0; + nan_cnt = 0; + data_elt = ucl_object_typed_new (UCL_OBJECT); + ucl_object_insert_key (data_elt, ucl_object_fromint (t), "x", 1, false); + + for (j = 0; j < step; j++) { + yval = acc[i + j * rrd_result->ds_count]; + if (isnan(yval)) { + nan_cnt++; + } + else { + sum += yval; + } + } + if (nan_cnt == step) { + ucl_object_insert_key (data_elt, ucl_object_typed_new (UCL_NULL), + "y", 1, false); + } + else { + ucl_object_insert_key (data_elt, + ucl_object_fromdouble (sum / (gdouble) step), "y", 1, + false); + } + ucl_array_append (elt[i], data_elt); + } +} + /* * Graph command handler: * request: /graph?type=<hourly|daily|weekly|monthly> @@ -1042,8 +1081,9 @@ rspamd_controller_handle_graph ( struct rspamd_controller_worker_ctx *ctx; rspamd_ftok_t srch, *value; struct rspamd_rrd_query_result *rrd_result; - gulong i, j, ts, start_row, cnt, t; - ucl_object_t *res, *elt[4], *data_elt; + gulong i, j, k, start_row, cnt, t, ts, step; + gdouble *acc; + ucl_object_t *res, *elt[4]; enum { rra_hourly = 0, rra_daily, @@ -1051,6 +1091,8 @@ rspamd_controller_handle_graph ( rra_monthly, rra_invalid } rra_num = rra_invalid; + /* How many points are we going to send to display */ + static const guint desired_points = 500; ctx = session->ctx; @@ -1080,16 +1122,16 @@ rspamd_controller_handle_graph ( return 0; } - if (strncmp (value->begin, "hourly", value->len) == 0) { + if (value->len == 6 && rspamd_lc_cmp (value->begin, "hourly", value->len) == 0) { rra_num = rra_hourly; } - else if (strncmp (value->begin, "daily", value->len) == 0) { - rra_num = rra_hourly; + else if (value->len == 5 && rspamd_lc_cmp (value->begin, "daily", value->len) == 0) { + rra_num = rra_daily; } - else if (strncmp (value->begin, "weekly", value->len) == 0) { - rra_num = rra_hourly; + else if (value->len == 6 && rspamd_lc_cmp (value->begin, "weekly", value->len) == 0) { + rra_num = rra_weekly; } - else if (strncmp (value->begin, "monthly", value->len) == 0) { + else if (value->len == 7 && rspamd_lc_cmp (value->begin, "monthly", value->len) == 0) { rra_num = rra_monthly; } @@ -1124,43 +1166,49 @@ rspamd_controller_handle_graph ( start_row = rrd_result->cur_row == rrd_result->rra_rows - 1 ? 0 : rrd_result->cur_row; - for (i = start_row, cnt = 0; cnt < rrd_result->rra_rows; cnt ++) { - for (j = 0; j < rrd_result->ds_count; j++) { - gdouble yval; - - data_elt = ucl_object_typed_new (UCL_OBJECT); - t = ts * rrd_result->pdp_per_cdp; - ucl_object_insert_key (data_elt, - ucl_object_fromint (t), - "x", 1, - false); - yval = rrd_result->data[i * rrd_result->ds_count + j]; + /* Create window */ + step = (rrd_result->rra_rows / desired_points + 0.5); + acc = g_malloc0 (sizeof (double) * rrd_result->ds_count * step); - if (!isnan (yval)) { - ucl_object_insert_key (data_elt, - ucl_object_fromdouble (yval), - "y", 1, - false); + for (i = start_row, cnt = 0; cnt < rrd_result->rra_rows; + cnt ++) { + for (j = 0; j < rrd_result->ds_count; j++) { + if (k < step) { + /* Just update window */ + acc[k * rrd_result->ds_count + j] = + rrd_result->data[i * rrd_result->ds_count + j]; + k ++; } else { - ucl_object_insert_key (data_elt, - ucl_object_typed_new (UCL_NULL), - "y", 1, - false); + t = ts * rrd_result->pdp_per_cdp; + + /* Need a fresh point */ + rspamd_controller_graph_point (t, step, rrd_result, acc, elt); + k = 0; } - ucl_array_append (elt[j], data_elt); } - i = start_row == 0 ? i + 1 : (i + 1) % start_row; + if (i == rrd_result->rra_rows - 1) { + i = 0; + } + else { + i ++; + } + ts ++; } + if (k > 0) { + rspamd_controller_graph_point (t, k, rrd_result, acc, elt); + } + for (i = 0; i < rrd_result->ds_count; i++) { ucl_array_append (res, elt[i]); } rspamd_controller_send_ucl (conn_ent, res); ucl_object_unref (res); + g_free (acc); return 0; } @@ -2619,7 +2667,9 @@ start_controller_worker (struct rspamd_worker *worker) /* RRD collector */ if (ctx->cfg->rrd_file && worker->index == 0) { - ctx->rrd = rspamd_rrd_file_default (ctx->cfg->rrd_file, NULL); + GError *rrd_err = NULL; + + ctx->rrd = rspamd_rrd_file_default (ctx->cfg->rrd_file, &rrd_err); if (ctx->rrd) { ctx->rrd_event = g_slice_alloc0 (sizeof (*ctx->rrd_event)); @@ -2627,6 +2677,14 @@ start_controller_worker (struct rspamd_worker *worker) event_base_set (ctx->ev_base, ctx->rrd_event); event_add (ctx->rrd_event, &rrd_update_time); } + else if (rrd_err) { + msg_err ("cannot load rrd from %s: %e", ctx->cfg->rrd_file, + rrd_err); + g_error_free (rrd_err); + } + else { + msg_err ("cannot load rrd from %s: unknown error", ctx->cfg->rrd_file); + } } else { ctx->rrd = NULL; diff --git a/src/libutil/map.c b/src/libutil/map.c index a29939f26..354ac722e 100644 --- a/src/libutil/map.c +++ b/src/libutil/map.c @@ -599,9 +599,8 @@ rspamd_map_periodic_dtor (struct map_periodic_cbdata *periodic) /* Not modified */ } - rspamd_map_schedule_periodic (periodic->map, FALSE, FALSE, FALSE); - if (periodic->locked) { + rspamd_map_schedule_periodic (periodic->map, FALSE, FALSE, FALSE); g_atomic_int_set (periodic->map->locked, 0); } @@ -627,7 +626,7 @@ rspamd_map_schedule_periodic (struct rspamd_map *map, timeout = map->poll_timeout * error_mult; } else if (locked) { - timeout = map->poll_timeout * lock_mult; + timeout = lock_mult; } cbd = g_slice_alloc0 (sizeof (*cbd)); @@ -638,8 +637,6 @@ rspamd_map_schedule_periodic (struct rspamd_map *map, cbd->map = map; REF_INIT_RETAIN (cbd, rspamd_map_periodic_dtor); - msg_debug_map ("schedule new periodic event %p in %.2f seconds", cbd, timeout); - if (initial) { evtimer_set (&map->ev, rspamd_map_periodic_callback, cbd); event_base_set (map->ev_base, &map->ev); @@ -647,9 +644,11 @@ rspamd_map_schedule_periodic (struct rspamd_map *map, else { evtimer_del (&map->ev); evtimer_set (&map->ev, rspamd_map_periodic_callback, cbd); + event_base_set (map->ev_base, &map->ev); } jittered_sec = rspamd_time_jitter (timeout, 0); + msg_debug_map ("schedule new periodic event %p in %.2f seconds", cbd, jittered_sec); double_to_tv (jittered_sec, &map->tv); evtimer_add (&map->ev, &map->tv); diff --git a/src/libutil/rrd.c b/src/libutil/rrd.c index 470896efa..a230faf2b 100644 --- a/src/libutil/rrd.c +++ b/src/libutil/rrd.c @@ -217,20 +217,30 @@ rspamd_rrd_check_file (const gchar *filename, gboolean need_data, GError **err) return FALSE; } /* Check magic */ - if (memcmp (head.cookie, RRD_COOKIE, sizeof (head.cookie)) != 0 || - memcmp (head.version, RRD_VERSION, sizeof (head.version)) != 0 || - head.float_cookie != RRD_FLOAT_COOKIE) { + if (memcmp (head.version, RRD_VERSION, sizeof (head.version)) != 0) { g_set_error (err, - rrd_error_quark (), EINVAL, "rrd head cookies error: %s", - strerror (errno)); + rrd_error_quark (), EINVAL, "rrd head error: bad cookie"); + close (fd); + return FALSE; + } + if (memcmp (head.version, RRD_VERSION, sizeof (head.version)) != 0) { + g_set_error (err, + rrd_error_quark (), EINVAL, "rrd head error: invalid version"); + close (fd); + return FALSE; + } + if (head.float_cookie != RRD_FLOAT_COOKIE) { + g_set_error (err, + rrd_error_quark (), EINVAL, "rrd head error: another architecture " + "(file cookie %g != our cookie %g)", + head.float_cookie, RRD_FLOAT_COOKIE); close (fd); return FALSE; } /* Check for other params */ if (head.ds_cnt <= 0 || head.rra_cnt <= 0) { g_set_error (err, - rrd_error_quark (), EINVAL, "rrd head cookies error: %s", - strerror (errno)); + rrd_error_quark (), EINVAL, "rrd head cookies error: bad rra or ds count"); close (fd); return FALSE; } diff --git a/src/plugins/lua/phishing.lua b/src/plugins/lua/phishing.lua index 2e2b92244..ec9b9d2ad 100644 --- a/src/plugins/lua/phishing.lua +++ b/src/plugins/lua/phishing.lua @@ -30,42 +30,88 @@ local phishtank_enabled = false local openphish_premium = false local openphish_hash local phishtank_hash -local openphish_json = {} +local openphish_data = {} local phishtank_data = {} local rspamd_logger = require "rspamd_logger" local util = require "rspamd_util" local opts = rspamd_config:get_all_opt('phishing') local function phishing_cb(task) - local urls = task:get_urls() + local function check_phishing_map(map, url, symbol) + local host = url:get_host() + + if host then + local elt = map[host] + local found_path = false + local found_query = false + local data = nil + + if elt then + local path = url:get_path() + local query = url:get_query() + + if path then + for _,d in ipairs(elt) do + if d['path'] == path then + found_path = true + data = d['data'] + + if query and d['query'] and query == d['query'] then + found_query = true + elseif not d['query'] then + found_query = true + end + end + end + else + if not d['path'] then + found_path = true + found_query = true + end + end - if urls then - for _,url in ipairs(urls) do - if openphish_hash then - local t = url:get_text() - - if openphish_premium then - local elt = openphish_json[t] - if elt then - task:insert_result(openphish_symbol, 1.0, { - elt['tld'], - elt['sector'], - elt['brand'], - }) + if found_path then + local args = nil + + if type(data) == 'table' then + args = { + data['tld'], + data['sector'], + data['brand'], + } + elseif type(data) == 'string' then + args = data + else + args = host + end + + if found_query then + -- Query + path match + task:insert_result(symbol, 1.0, args) + else + -- Host + path match + task:insert_result(symbol, 0.3, args) end else - if openphish_hash:get_key(t) then - task:insert_result(openphish_symbol, 1.0, url:get_tld()) + if url:is_phished() then + -- Only host matches + task:insert_result(symbol, 0.1, host) end end end + end + end + + local urls = task:get_urls() + + if urls then + for _,url in ipairs(urls) do + if openphish_hash then + check_phishing_map(openphish_data, url, openphish_symbol) + end if phishtank_hash then - local t = url:get_text() - local elt = phishtank_data[t] - if elt then - task:insert_result(phishtank_symbol, 1.0, elt) - end + check_phishing_map(phishtank_data, url, phishtank_symbol) end if url:is_phished() and not url:is_redirected() then @@ -158,12 +204,42 @@ local function rspamd_str_split_fun(s, sep, func) return lpeg.match(p, s) end +local function insert_url_from_string(pool, tbl, str, data) + local rspamd_url = require "rspamd_url" + + local u = rspamd_url.create(pool, str) + + if u then + local host = u:get_host() + if host then + local elt = { + data = data, + path = u:get_path(), + query = u:get_query() + } + + if tbl[host] then + table.insert(tbl[host], elt) + else + tbl[host] = {elt} + end + + return true + end + end + + return false +end + local function openphish_json_cb(string) local ucl = require "ucl" + local rspamd_mempool = require "rspamd_mempool" local nelts = 0 local new_json_map = {} local valid = true + local pool = rspamd_mempool.create() + local function openphish_elt_parser(cap) if valid then local parser = ucl.parser() @@ -175,8 +251,9 @@ local function openphish_json_cb(string) local obj = parser:get_object() if obj['url'] then - new_json_map[obj['url']] = obj - nelts = nelts + 1 + if insert_url_from_string(pool, new_json_map, obj['url'], obj) then + nelts = nelts + 1 + end end end end @@ -185,10 +262,32 @@ local function openphish_json_cb(string) rspamd_str_split_fun(string, '\n', openphish_elt_parser) if valid then - openphish_json = new_json_map + openphish_data = new_json_map rspamd_logger.infox(openphish_hash, "parsed %s elements from openphish feed", nelts) end + + pool:destroy() +end + +local function openphish_plain_cb(string) + local nelts = 0 + local new_data = {} + local rspamd_mempool = require "rspamd_mempool" + local pool = rspamd_mempool.create() + + local function openphish_elt_parser(cap) + if insert_url_from_string(pool, new_data, cap, nil) then + nelts = nelts + 1 + end + end + + rspamd_str_split_fun(string, '\n', openphish_elt_parser) + + openphish_data = new_data + rspamd_logger.infox(openphish_hash, "parsed %s elements from openphish feed", + nelts) + pool:destroy() end local function phishtank_json_cb(string) @@ -198,6 +297,8 @@ local function phishtank_json_cb(string) local valid = true local parser = ucl.parser() local res,err = parser:parse_string(string) + local rspamd_mempool = require "rspamd_mempool" + local pool = rspamd_mempool.create() if not res then valid = false @@ -207,8 +308,10 @@ local function phishtank_json_cb(string) for _,elt in ipairs(obj) do if elt['url'] then - new_data[elt['url']] = elt['phish_detail_url'] - nelts = nelts + 1 + if insert_url_from_string(pool, new_data, elt['url'], + elt['phish_detail_url']) then + nelts = nelts + 1 + end end end end @@ -218,6 +321,9 @@ local function phishtank_json_cb(string) rspamd_logger.infox(phishtank_hash, "parsed %s elements from phishtank feed", nelts) end + + + pool:destroy() end if opts then @@ -243,8 +349,9 @@ if opts then if opts['openphish_enabled'] then if not openphish_premium then openphish_hash = rspamd_config:add_map({ - type = 'set', + type = 'callback', url = openphish_map, + callback = openphish_plain_cb, description = 'Open phishing feed map (see https://www.openphish.com for details)' }) else |