@@ -721,9 +721,6 @@ rspamd_html_url_is_phished (rspamd_mempool_t *pool, | |||
href_url->phished_url = text_url; | |||
phished_tld.begin = href_tok.begin; | |||
phished_tld.len = href_tok.len; | |||
rspamd_url_add_tag (text_url, "phishing", | |||
rspamd_mempool_ftokdup (pool, &phished_tld), | |||
pool); | |||
text_url->flags |= RSPAMD_URL_FLAG_HTML_DISPLAYED; | |||
} | |||
} |
@@ -3176,34 +3176,6 @@ rspamd_url_task_subject_callback (struct rspamd_url *url, gsize start_offset, | |||
} | |||
} | |||
void | |||
rspamd_url_add_tag (struct rspamd_url *url, const gchar *tag, | |||
const gchar *value, | |||
rspamd_mempool_t *pool) | |||
{ | |||
struct rspamd_url_tag *found, *ntag; | |||
g_assert (url != NULL && tag != NULL && value != NULL); | |||
if (url->tags == NULL) { | |||
url->tags = g_hash_table_new (rspamd_strcase_hash, rspamd_strcase_equal); | |||
rspamd_mempool_add_destructor (pool, | |||
(rspamd_mempool_destruct_t)g_hash_table_unref, url->tags); | |||
} | |||
found = g_hash_table_lookup (url->tags, tag); | |||
ntag = rspamd_mempool_alloc0 (pool, sizeof (*ntag)); | |||
ntag->data = rspamd_mempool_strdup (pool, value); | |||
if (found == NULL) { | |||
g_hash_table_insert (url->tags, rspamd_mempool_strdup (pool, tag), | |||
ntag); | |||
} | |||
DL_APPEND (found, ntag); | |||
} | |||
guint | |||
rspamd_url_hash (gconstpointer u) | |||
{ |
@@ -70,7 +70,6 @@ struct rspamd_url { | |||
enum rspamd_url_flags flags; | |||
guint count; | |||
GHashTable *tags; | |||
}; | |||
enum uri_errno { | |||
@@ -214,16 +213,6 @@ void rspamd_url_task_subject_callback (struct rspamd_url *url, | |||
gsize start_offset, | |||
gsize end_offset, gpointer ud); | |||
/** | |||
* Adds a tag for url | |||
* @param url | |||
* @param tag | |||
* @param pool | |||
*/ | |||
void rspamd_url_add_tag (struct rspamd_url *url, const gchar *tag, | |||
const gchar *value, | |||
rspamd_mempool_t *pool); | |||
guint rspamd_url_hash (gconstpointer u); | |||
guint rspamd_email_hash (gconstpointer u); |
@@ -59,10 +59,7 @@ LUA_FUNCTION_DEF (url, is_obscured); | |||
LUA_FUNCTION_DEF (url, is_html_displayed); | |||
LUA_FUNCTION_DEF (url, is_subject); | |||
LUA_FUNCTION_DEF (url, get_phished); | |||
LUA_FUNCTION_DEF (url, get_tag); | |||
LUA_FUNCTION_DEF (url, get_count); | |||
LUA_FUNCTION_DEF (url, get_tags); | |||
LUA_FUNCTION_DEF (url, add_tag); | |||
LUA_FUNCTION_DEF (url, get_visible); | |||
LUA_FUNCTION_DEF (url, create); | |||
LUA_FUNCTION_DEF (url, init); | |||
@@ -87,9 +84,7 @@ static const struct luaL_reg urllib_m[] = { | |||
LUA_INTERFACE_DEF (url, is_html_displayed), | |||
LUA_INTERFACE_DEF (url, is_subject), | |||
LUA_INTERFACE_DEF (url, get_phished), | |||
LUA_INTERFACE_DEF (url, get_tag), | |||
LUA_INTERFACE_DEF (url, get_tags), | |||
LUA_INTERFACE_DEF (url, add_tag), | |||
LUA_INTERFACE_DEF (url, get_visible), | |||
LUA_INTERFACE_DEF (url, get_count), | |||
LUA_INTERFACE_DEF (url, get_flags), | |||
@@ -438,129 +433,6 @@ lua_url_is_subject (lua_State *L) | |||
return 1; | |||
} | |||
/*** | |||
* @method url:get_tag(tag) | |||
* Returns list of string for a specific tagname for an url | |||
* @return {table/strings} list of tags for an url | |||
*/ | |||
static gint | |||
lua_url_get_tag (lua_State *L) | |||
{ | |||
LUA_TRACE_POINT; | |||
struct rspamd_lua_url *url = lua_check_url (L, 1); | |||
guint i; | |||
const gchar *tag = luaL_checkstring (L, 2); | |||
struct rspamd_url_tag *tval, *cur; | |||
if (url != NULL && tag != NULL) { | |||
if (url->url->tags == NULL) { | |||
lua_createtable (L, 0, 0); | |||
} | |||
else { | |||
tval = g_hash_table_lookup (url->url->tags, tag); | |||
if (tval) { | |||
lua_newtable (L); | |||
i = 1; | |||
DL_FOREACH (tval, cur) { | |||
lua_pushstring (L, cur->data); | |||
lua_rawseti (L, -2, i ++); | |||
} | |||
lua_settable (L, -3); | |||
} | |||
else { | |||
lua_createtable (L, 0, 0); | |||
} | |||
} | |||
} | |||
else { | |||
lua_pushnil (L); | |||
} | |||
return 1; | |||
} | |||
/*** | |||
* @method url:get_tags() | |||
* Returns list of string tags for an url | |||
* @return {table/strings} list of tags for an url | |||
*/ | |||
static gint | |||
lua_url_get_tags (lua_State *L) | |||
{ | |||
LUA_TRACE_POINT; | |||
struct rspamd_lua_url *url = lua_check_url (L, 1); | |||
guint i; | |||
GHashTableIter it; | |||
struct rspamd_url_tag *tval, *cur; | |||
gpointer k, v; | |||
if (url != NULL) { | |||
if (url->url->tags == NULL) { | |||
lua_createtable (L, 0, 0); | |||
} | |||
else { | |||
lua_createtable (L, 0, g_hash_table_size (url->url->tags)); | |||
g_hash_table_iter_init (&it, url->url->tags); | |||
while (g_hash_table_iter_next (&it, &k, &v)) { | |||
tval = v; | |||
lua_pushstring (L, (const gchar *)k); | |||
lua_newtable (L); | |||
i = 1; | |||
DL_FOREACH (tval, cur) { | |||
lua_pushstring (L, cur->data); | |||
lua_rawseti (L, -2, i ++); | |||
} | |||
lua_settable (L, -3); | |||
} | |||
} | |||
} | |||
else { | |||
lua_pushnil (L); | |||
} | |||
return 1; | |||
} | |||
/*** | |||
* @method url:add_tag(tag, mempool) | |||
* Adds a new tag for url | |||
* @param {string} tag new tag to add | |||
* @param {mempool} mempool memory pool (e.g. `task:get_pool()`) | |||
*/ | |||
static gint | |||
lua_url_add_tag (lua_State *L) | |||
{ | |||
LUA_TRACE_POINT; | |||
struct rspamd_lua_url *url = lua_check_url (L, 1); | |||
rspamd_mempool_t *mempool = rspamd_lua_check_mempool (L, 4); | |||
const gchar *tag = luaL_checkstring (L, 2); | |||
const gchar *value; | |||
if (lua_type (L, 3) == LUA_TSTRING) { | |||
value = lua_tostring (L, 3); | |||
} | |||
else { | |||
value = "1"; /* Some stupid placeholder */ | |||
} | |||
if (url != NULL && mempool != NULL && tag != NULL) { | |||
rspamd_url_add_tag (url->url, tag, value, mempool); | |||
} | |||
else { | |||
return luaL_error (L, "invalid arguments"); | |||
} | |||
return 0; | |||
} | |||
/*** | |||
* @method url:get_phished() | |||
* Get another URL that pretends to be this URL (e.g. used in phishing) |
@@ -1584,9 +1584,6 @@ process_dns_results (struct rspamd_task *task, | |||
bit->bit); | |||
rspamd_task_insert_result (task, bit->symbol, 1, resolved_name); | |||
if (surbl_module_ctx->use_tags) { | |||
rspamd_url_add_tag (uri, "surbl", bit->symbol, task->task_pool); | |||
} | |||
got_result = TRUE; | |||
} | |||
} | |||
@@ -1605,10 +1602,6 @@ process_dns_results (struct rspamd_task *task, | |||
resolved_name, suffix->suffix, | |||
bit->bit); | |||
rspamd_task_insert_result (task, bit->symbol, 1, resolved_name); | |||
if (surbl_module_ctx->use_tags) { | |||
rspamd_url_add_tag (uri, "surbl", bit->symbol, task->task_pool); | |||
} | |||
} | |||
} | |||
} | |||
@@ -1618,10 +1611,6 @@ process_dns_results (struct rspamd_task *task, | |||
msg_info_surbl ("domain [%s] is in surbl %s", | |||
resolved_name, suffix->suffix); | |||
rspamd_task_insert_result (task, suffix->symbol, 1, resolved_name); | |||
if (surbl_module_ctx->use_tags) { | |||
rspamd_url_add_tag (uri, "surbl", suffix->symbol, task->task_pool); | |||
} | |||
} | |||
else { | |||
ina.s_addr = addr; | |||
@@ -1795,11 +1784,6 @@ surbl_redirector_finish (struct rspamd_http_connection *conn, | |||
else { | |||
existing->count ++; | |||
} | |||
if (surbl_module_ctx->use_tags) { | |||
rspamd_url_add_tag (param->url, "redirector", urlstr, | |||
task->task_pool); | |||
} | |||
} | |||
else { | |||
msg_info_surbl ("cannot parse redirector reply: %s", urlstr); | |||
@@ -1879,38 +1863,6 @@ register_redirector_call (struct rspamd_url *url, struct rspamd_task *task, | |||
} | |||
} | |||
static gboolean | |||
surbl_test_tags (struct rspamd_task *task, struct redirector_param *param, | |||
struct rspamd_url *url) | |||
{ | |||
struct rspamd_url_tag *tag = NULL, *cur; | |||
gchar *ftld = NULL; | |||
rspamd_ftok_t tld; | |||
gboolean processed = FALSE; | |||
if (url->tags) { | |||
tag = g_hash_table_lookup (url->tags, "surbl"); | |||
} | |||
if (tag) { | |||
tld.begin = url->tld; | |||
tld.len = url->tldlen; | |||
ftld = rspamd_mempool_ftokdup (task->task_pool, &tld); | |||
/* We know results for this URL */ | |||
DL_FOREACH (tag, cur) { | |||
msg_info_surbl ("domain [%s] is in surbl %s (tags)", | |||
ftld, cur->data); | |||
rspamd_task_insert_result (task, cur->data, 1, ftld); | |||
} | |||
processed = TRUE; | |||
} | |||
return processed; | |||
} | |||
static void | |||
surbl_tree_redirector_callback (gpointer key, gpointer value, void *data) | |||
{ | |||
@@ -2026,15 +1978,6 @@ surbl_tree_url_callback (gpointer key, gpointer value, void *data) | |||
msg_debug_surbl ("check url %*s in %s", url->urllen, url->string, | |||
param->suffix->suffix); | |||
if (surbl_module_ctx->use_tags && surbl_test_tags (param->task, param, url)) { | |||
return; | |||
} | |||
if (url->tags && g_hash_table_lookup (url->tags, "redirector")) { | |||
/* URL is redirected, skip from checks */ | |||
return; | |||
} | |||
make_surbl_requests (url, param->task, param->item, param->suffix, FALSE, | |||
param->tree, surbl_module_ctx); | |||
} | |||
@@ -2324,11 +2267,6 @@ surbl_continue_process_handler (lua_State *L) | |||
redirected_url->phished_url = param->url; | |||
redirected_url->flags |= RSPAMD_URL_FLAG_REDIRECTED; | |||
} | |||
if (surbl_module_ctx->use_tags) { | |||
rspamd_url_add_tag (param->url, "redirector", urlstr, | |||
task->task_pool); | |||
} | |||
} | |||
else { | |||
msg_info_surbl ("could not resolve '%*s' on redirector", |
@@ -1,56 +0,0 @@ | |||
local rspamd_logger = require 'rspamd_logger' | |||
rspamd_config:register_symbol({ | |||
name = 'ADDED_TAGS', | |||
score = 1.0, | |||
callback = function(task) | |||
if not task:get_request_header('addtags') then | |||
return true, 'nope! not requested' | |||
end | |||
local urls = task:get_urls() | |||
if not (urls and urls[1]) then | |||
return true, 'nope! found no urls' | |||
end | |||
local mpool = task:get_mempool() | |||
for _, u in ipairs(urls) do | |||
u:add_tag('test1', 'meta1', mpool) | |||
u:add_tag('test1', 'meta2', mpool) | |||
u:add_tag('test2', 'http://www.example.com', mpool) | |||
end | |||
return true, 'no worry' | |||
end | |||
}) | |||
rspamd_config:register_symbol({ | |||
name = 'FOUND_TAGS', | |||
score = 1.0, | |||
callback = function(task) | |||
local urls = task:get_urls() | |||
if not (urls and urls[1]) then | |||
return true, 'nope! found no urls' | |||
end | |||
for _, u in ipairs(urls) do | |||
local tags = u:get_tags() | |||
rspamd_logger.debugx(task, 'tags: %1', tags) | |||
if not tags['test1'] then | |||
return true, 'no key - test1' | |||
end | |||
local found1, found2 = false, false | |||
for _, e in ipairs(tags['test1']) do | |||
if e == 'meta1' then found1 = true end | |||
if e == 'meta2' then found2 = true end | |||
end | |||
if not (found1 and found2) then | |||
return true, 'missing metatags in test1' | |||
end | |||
if not tags['test2'] then | |||
return true, 'no key - test2' | |||
end | |||
if not tags['test2'][1] == 'http://www.example.com' then | |||
return true, 'wrong value in test2 metatag: ' .. tags['test2'][1] | |||
end | |||
end | |||
return true, 'no worry' | |||
end | |||
}) |