+++ /dev/null
-/*-
- * Copyright 2016 Vsevolod Stakhov
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/***MODULE:surbl
- * rspamd module that implements SURBL url checking
- *
- * Allowed options:
- * - weight (integer): weight of symbol
- * Redirecotor options:
- * - redirector (string): address of http redirector utility in format "host:port"
- * - redirector_connect_timeout (seconds): redirector connect timeout (default: 1s)
- * - redirector_read_timeout (seconds): timeout for reading data (default: 5s)
- * - redirector_hosts_map (map string): map that contains domains to check with redirector
- * Surbl options:
- * - exceptions (map string): map of domains that should be checked via surbl using 3 (e.g. somehost.domain.com)
- * components of domain name instead of normal 2 (e.g. domain.com)
- * - whitelist (map string): map of domains that should be whitelisted for surbl checks
- * - max_urls (integer): maximum allowed number of urls in message to be checked
- * - suffix (string): surbl address (for example insecure-bl.rambler.ru), may contain %b if bits are used (read documentation about it)
- * - bit (string): describes a prefix for a single bit
- */
-
-#include "config.h"
-#include "libmime/message.h"
-#include "libutil/map.h"
-#include "libutil/map_helpers.h"
-#include "rspamd.h"
-#include "utlist.h"
-#include "multipattern.h"
-#include "monitored.h"
-#include "libserver/html.h"
-#include "libutil/http_private.h"
-#include "unix-std.h"
-#include "lua/lua_common.h"
-
-#define msg_err_surbl(...) rspamd_default_log_function (G_LOG_LEVEL_CRITICAL, \
- "surbl", task->task_pool->tag.uid, \
- G_STRFUNC, \
- __VA_ARGS__)
-#define msg_warn_surbl(...) rspamd_default_log_function (G_LOG_LEVEL_WARNING, \
- "surbl", task->task_pool->tag.uid, \
- G_STRFUNC, \
- __VA_ARGS__)
-#define msg_info_surbl(...) rspamd_default_log_function (G_LOG_LEVEL_INFO, \
- "surbl", task->task_pool->tag.uid, \
- G_STRFUNC, \
- __VA_ARGS__)
-#define msg_debug_surbl(...) rspamd_conditional_debug_fast (NULL, task->from_addr, \
- rspamd_surbl_log_id, "surbl", task->task_pool->tag.uid, \
- G_STRFUNC, \
- __VA_ARGS__)
-
-INIT_LOG_MODULE(surbl)
-
-static const gchar *M = "surbl";
-
-#define DEFAULT_SURBL_WEIGHT 10
-#define DEFAULT_REDIRECTOR_READ_TIMEOUT 5.0
-#define DEFAULT_SURBL_SYMBOL "SURBL_DNS"
-#define SURBL_OPTION_NOIP (1u << 0u)
-#define SURBL_OPTION_RESOLVEIP (1u << 1u)
-#define SURBL_OPTION_CHECKIMAGES (1u << 2u)
-#define SURBL_OPTION_CHECKDKIM (1u << 3u)
-#define SURBL_OPTION_FULLDOMAIN (1u << 4u)
-#define SURBL_OPTION_CHECKEMAILS (1u << 5u)
-#define MAX_LEVELS 10
-
-struct surbl_ctx {
- struct module_ctx ctx;
- guint16 weight;
- gdouble read_timeout;
- gboolean use_tags;
- GList *suffixes;
- const gchar *redirector_symbol;
- GHashTable **exceptions;
- struct rspamd_hash_map_helper *whitelist;
- GHashTable *redirector_tlds;
- guint use_redirector;
- guint max_redirected_urls;
- gint redirector_cbid;
- struct upstream_list *redirectors;
-};
-
-struct suffix_item {
- guint64 magic;
- const gchar *monitored_domain;
- const gchar *suffix;
- const gchar *symbol;
- GArray *bits;
- GHashTable *ips;
- struct rspamd_monitored *m;
- guint32 options;
- gboolean reported_offline;
- gint callback_id;
- gint url_process_cbref;
-};
-
-struct dns_param {
- struct rspamd_url *url;
- struct rspamd_task *task;
- gchar *host_resolve;
- gchar *host_orig; /* Name with no uribl suffix */
- struct suffix_item *suffix;
- struct rspamd_symcache_item *item;
- struct surbl_module_ctx *ctx;
-};
-
-struct redirector_param {
- struct rspamd_url *url;
- struct rspamd_task *task;
- struct upstream *redirector;
- struct surbl_ctx *ctx;
- struct rspamd_http_connection *conn;
- GHashTable *tree;
- struct suffix_item *suffix;
- struct rspamd_symcache_item *item;
- guint redirector_requests;
-};
-
-struct surbl_bit_item {
- guint32 bit;
- gchar *symbol;
-};
-
-#define SURBL_REDIRECTOR_CALLBACK "SURBL_REDIRECTOR_CALLBACK"
-
-static const guint64 rspamd_surbl_cb_magic = 0xe09b8536f80de0d1ULL;
-static const gchar *rspamd_surbl_default_monitored = "facebook.com";
-static const guint default_max_redirected_urls = 10;
-
-static void surbl_test_url (struct rspamd_task *task,
- struct rspamd_symcache_item *item,
- void *user_data);
-static void surbl_test_redirector (struct rspamd_task *task,
- struct rspamd_symcache_item *item,
- void *user_data);
-static void surbl_dns_callback (struct rdns_reply *reply, gpointer arg);
-static void surbl_dns_ip_callback (struct rdns_reply *reply, gpointer arg);
-static void process_dns_results (struct rspamd_task *task,
- struct suffix_item *suffix, gchar *resolved_name,
- guint32 addr, struct rspamd_url *url);
-static gint surbl_register_redirect_handler (lua_State *L);
-static gint surbl_continue_process_handler (lua_State *L);
-static gint surbl_is_redirector_handler (lua_State *L);
-
-#define NO_REGEXP (gpointer) - 1
-
-#define SURBL_ERROR surbl_error_quark ()
-#define WHITELIST_ERROR 0
-#define CONVERSION_ERROR 1
-#define DUPLICATE_ERROR 1
-
-GQuark
-surbl_error_quark (void)
-{
- return g_quark_from_static_string ("surbl-error-quark");
-}
-
-/* Initialization */
-gint surbl_module_init (struct rspamd_config *cfg, struct module_ctx **ctx);
-gint surbl_module_config (struct rspamd_config *cfg);
-gint surbl_module_reconfig (struct rspamd_config *cfg);
-
-module_t surbl_module = {
- "surbl",
- surbl_module_init,
- surbl_module_config,
- surbl_module_reconfig,
- NULL,
- RSPAMD_MODULE_VER,
- (guint)-1,
-};
-
-static inline struct surbl_ctx *
-surbl_get_context (struct rspamd_config *cfg)
-{
- return (struct surbl_ctx *)g_ptr_array_index (cfg->c_modules,
- surbl_module.ctx_offset);
-}
-
-static void
-exceptions_free_value (gpointer v)
-{
- rspamd_ftok_t *val = v;
-
- g_free ((gpointer)val->begin);
- g_free (val);
-}
-
-static void
-exception_insert (gpointer st, gconstpointer key, gconstpointer value)
-{
- GHashTable **t = st;
- gint level = 0;
- const gchar *p = key;
- rspamd_ftok_t *val;
-
- while (*p) {
- if (*p == '.') {
- level++;
- }
- p++;
- }
- if (level >= MAX_LEVELS) {
- msg_err ("invalid domain in exceptions list: %s, levels: %d",
- (gchar *)key,
- level);
- return;
- }
-
- val = g_malloc (sizeof (rspamd_ftok_t));
- val->begin = g_strdup (key);
- val->len = strlen (key);
-
- if (t[level] == NULL) {
- t[level] = g_hash_table_new_full (rspamd_ftok_icase_hash,
- rspamd_ftok_icase_equal,
- exceptions_free_value,
- g_free);
- }
-
- g_hash_table_replace (t[level], val, g_strdup (value));
-}
-
-static gchar *
-read_exceptions_list (gchar * chunk,
- gint len,
- struct map_cb_data *data,
- gboolean final)
-{
- GHashTable **t;
- guint i;
-
- if (data->cur_data == NULL) {
- t = data->prev_data;
-
- if (t) {
- for (i = 0; i < MAX_LEVELS; i++) {
- if (t[i] != NULL) {
- g_hash_table_destroy (t[i]);
- }
- t[i] = NULL;
- }
-
- g_free (t);
- }
-
- data->prev_data = NULL;
- data->cur_data = g_malloc0 (MAX_LEVELS * sizeof (GHashTable *));
- }
-
- return rspamd_parse_kv_list (
- chunk,
- len,
- data,
- exception_insert,
- "",
- final);
-}
-
-static void
-fin_exceptions_list (struct map_cb_data *data, void **target)
-{
- GHashTable **t;
- gint i;
-
- if (target) {
- *target = data->cur_data;
- }
-
- if (data->prev_data) {
- t = data->prev_data;
- for (i = 0; i < MAX_LEVELS; i++) {
- if (t[i] != NULL) {
- rspamd_default_log_function (G_LOG_LEVEL_DEBUG,
- "surbl", "",
- G_STRFUNC,
- "exceptions level %d: %d elements",
- i, g_hash_table_size (t[i]));
- }
- }
- }
-}
-
-static void
-dtor_exceptions_list (struct map_cb_data *data)
-{
- GHashTable **t;
- gint i;
-
- if (data->cur_data) {
- t = data->cur_data;
- for (i = 0; i < MAX_LEVELS; i++) {
- if (t[i] != NULL) {
- g_hash_table_destroy (t[i]);
- }
- t[i] = NULL;
- }
-
- g_free (t);
- }
-}
-
-static void
-redirector_insert (gpointer st, gconstpointer key, gconstpointer value)
-{
- GHashTable *tld_hash = st;
- const gchar *p = key, *begin = key;
- rspamd_fstring_t *pat;
- rspamd_ftok_t *tok;
- rspamd_regexp_t *re = NO_REGEXP;
- GError *err = NULL;
-
- while (*p && !g_ascii_isspace (*p)) {
- p++;
- }
-
- pat = rspamd_fstring_new_init (begin, p - begin);
- tok = g_malloc0 (sizeof (*tok));
- tok->begin = pat->str;
- tok->len = pat->len;
-
- if (g_ascii_isspace (*p)) {
- while (g_ascii_isspace (*p) && *p) {
- p++;
- }
- if (*p) {
- re = rspamd_regexp_new (p,
- "ir",
- &err);
- if (re == NULL) {
- msg_warn ("could not read regexp: %e while reading regexp %s",
- err,
- p);
- g_error_free (err);
- re = NO_REGEXP;
- }
- }
- }
-
- g_hash_table_replace (tld_hash, tok, re);
-}
-
-static void
-redirector_item_free (gpointer p)
-{
- rspamd_regexp_t *re;
-
- if (p != NULL && p != NO_REGEXP) {
- re = (rspamd_regexp_t *)p;
- rspamd_regexp_unref (re);
- }
-}
-
-static gchar *
-read_redirectors_list (gchar * chunk,
- gint len,
- struct map_cb_data *data,
- gboolean final)
-{
- GHashTable *tld_hash;
-
- if (data->cur_data == NULL) {
- tld_hash = g_hash_table_new_full (rspamd_ftok_icase_hash,
- rspamd_ftok_icase_equal,
- rspamd_fstring_mapped_ftok_free,
- redirector_item_free);
-
- data->cur_data = tld_hash;
- }
-
- return rspamd_parse_kv_list (
- chunk,
- len,
- data,
- redirector_insert,
- "",
- final);
-}
-
-static void
-fin_redirectors_list (struct map_cb_data *data, void **target)
-{
- GHashTable *tld_hash;
-
- if (target) {
- *target = data->cur_data;
- }
-
- if (data->prev_data) {
- tld_hash = data->prev_data;
-
- g_hash_table_unref (tld_hash);
- }
-}
-
-static void
-dtor_redirectors_list (struct map_cb_data *data)
-{
- GHashTable *tld_hash;
-
- if (data->cur_data) {
- tld_hash = data->cur_data;
-
- g_hash_table_unref (tld_hash);
- }
-}
-
-gint
-surbl_module_init (struct rspamd_config *cfg, struct module_ctx **ctx)
-{
- struct surbl_ctx *surbl_module_ctx;
-
- surbl_module_ctx = rspamd_mempool_alloc0 (cfg->cfg_pool,
- sizeof (struct surbl_ctx));
-
- surbl_module_ctx->use_redirector = 0;
- surbl_module_ctx->suffixes = NULL;
-
- surbl_module_ctx->redirectors = NULL;
- surbl_module_ctx->whitelist = NULL;
- surbl_module_ctx->exceptions = NULL;
- surbl_module_ctx->redirector_cbid = -1;
-
-
- *ctx = (struct module_ctx *)surbl_module_ctx;
-
- rspamd_rcl_add_doc_by_path (cfg,
- NULL,
- "URL blacklist plugin",
- "surbl",
- UCL_OBJECT,
- NULL,
- 0,
- NULL,
- 0);
- rspamd_rcl_add_doc_by_path (cfg,
- "surbl",
- "List of redirector servers",
- "redirector",
- UCL_STRING,
- NULL,
- 0,
- NULL,
- 0);
- rspamd_rcl_add_doc_by_path (cfg,
- "surbl",
- "Map of domains that should be checked with redirector",
- "redirector_hosts_map",
- UCL_STRING,
- NULL,
- 0,
- NULL,
- 0);
- rspamd_rcl_add_doc_by_path (cfg,
- "surbl",
- "Connect timeout for redirector",
- "redirector_connect_timeout",
- UCL_TIME,
- NULL,
- 0,
- NULL,
- 0);
- rspamd_rcl_add_doc_by_path (cfg,
- "surbl",
- "Read timeout for redirector",
- "redirector_read_timeout",
- UCL_TIME,
- NULL,
- 0,
- NULL,
- 0);
- rspamd_rcl_add_doc_by_path (cfg,
- "surbl",
- "Maximum number of URLs to process per message",
- "max_urls",
- UCL_INT,
- NULL,
- 0,
- NULL,
- 0);
- rspamd_rcl_add_doc_by_path (cfg,
- "surbl",
- "Rules for TLD composition",
- "exceptions",
- UCL_STRING,
- NULL,
- 0,
- NULL,
- 0);
- rspamd_rcl_add_doc_by_path (cfg,
- "surbl",
- "Map of whitelisted domains",
- "whitelist",
- UCL_STRING,
- NULL,
- 0,
- NULL,
- 0);
- rspamd_rcl_add_doc_by_path (cfg,
- "surbl",
- "URL blacklist rule",
- "rule",
- UCL_OBJECT,
- NULL,
- 0,
- NULL,
- 0);
- /* Rules doc strings */
- rspamd_rcl_add_doc_by_path (cfg,
- "surbl.rule",
- "Name of DNS black list (e.g. `multi.surbl.com`)",
- "suffix",
- UCL_STRING,
- NULL,
- 0,
- NULL,
- 0);
- rspamd_rcl_add_doc_by_path (cfg,
- "surbl.rule",
- "Symbol to insert (if no bits or suffixes are defined)",
- "symbol",
- UCL_STRING,
- NULL,
- 0,
- NULL,
- 0);
- rspamd_rcl_add_doc_by_path (cfg,
- "surbl.rule",
- "Whether the defined rule should be used",
- "enabled",
- UCL_BOOLEAN,
- NULL,
- 0,
- NULL,
- 0);
- rspamd_rcl_add_doc_by_path (cfg,
- "surbl.rule",
- "Do not try to check URLs with IP address instead of hostname",
- "no_ip",
- UCL_BOOLEAN,
- NULL,
- 0,
- NULL,
- 0);
- rspamd_rcl_add_doc_by_path (cfg,
- "surbl.rule",
- "Resolve URL host and then check against the specified suffix with reversed IP octets",
- "resolve_ip",
- UCL_BOOLEAN,
- NULL,
- 0,
- NULL,
- 0);
- rspamd_rcl_add_doc_by_path (cfg,
- "surbl.rule",
- "Check images URLs with this URL list",
- "images",
- UCL_BOOLEAN,
- NULL,
- 0,
- NULL,
- 0);
- rspamd_rcl_add_doc_by_path (cfg,
- "surbl.rule",
- "Parse IP bits in DNS reply, the content is 'symbol = <bit>'",
- "bits",
- UCL_OBJECT,
- NULL,
- 0,
- NULL,
- 0);
- rspamd_rcl_add_doc_by_path (cfg,
- "surbl.rule",
- "Parse IP addresses in DNS reply, the content is 'symbol = address'",
- "ips",
- UCL_OBJECT,
- NULL,
- 0,
- NULL,
- 0);
- rspamd_rcl_add_doc_by_path (cfg,
- "surbl.rule",
- "Check domains in valid DKIM signatures",
- "check_dkim",
- UCL_BOOLEAN,
- NULL,
- 0,
- NULL,
- 0);
- rspamd_rcl_add_doc_by_path (cfg,
- "surbl.rule",
- "Check full domain name instead of eSLD",
- "full_domain",
- UCL_BOOLEAN,
- NULL,
- 0,
- NULL,
- 0);
-
- return 0;
-}
-
-/*
- * Register virtual symbols for suffixes with bit wildcard
- */
-static void
-register_bit_symbols (struct rspamd_config *cfg, struct suffix_item *suffix,
- gint parent_id)
-{
- guint i;
- GHashTableIter it;
- struct surbl_bit_item *bit;
- gpointer k, v;
-
- if (suffix->ips != NULL) {
- g_hash_table_iter_init (&it, suffix->ips);
-
- while (g_hash_table_iter_next (&it, &k, &v)) {
- bit = v;
-
- /*
- * We can have multiple IPs mapped to a single symbol,
- * so skip symbol's registration to avoid duplicates
- */
- if (rspamd_symcache_find_symbol (cfg->cache, bit->symbol) == -1) {
- rspamd_symcache_add_symbol (cfg->cache, bit->symbol,
- 0, NULL, NULL,
- SYMBOL_TYPE_VIRTUAL, parent_id);
- }
- msg_debug_config ("bit: %d", bit->bit);
- }
- }
- else if (suffix->bits != NULL) {
- for (i = 0; i < suffix->bits->len; i++) {
- bit = &g_array_index (suffix->bits, struct surbl_bit_item, i);
- rspamd_symcache_add_symbol (cfg->cache, bit->symbol,
- 0, NULL, NULL,
- SYMBOL_TYPE_VIRTUAL, parent_id);
- }
- }
- else {
- rspamd_symcache_add_symbol (cfg->cache, suffix->symbol,
- 0, NULL, NULL,
- SYMBOL_TYPE_VIRTUAL, parent_id);
- }
-}
-
-static void
-surbl_module_add_ip (const ucl_object_t *ip, const gchar *symbol,
- struct suffix_item* suffix,
- struct rspamd_config* cfg)
-{
- gchar* p;
- guint32 bit;
- const gchar* ip_val;
- struct surbl_bit_item* new_bit;
-
- ip_val = ucl_obj_tostring (ip);
- new_bit = rspamd_mempool_alloc (
- cfg->cfg_pool,
- sizeof(struct surbl_bit_item));
- if (inet_pton (AF_INET, ip_val, &bit) != 1) {
- msg_err_config ("cannot parse ip %s: %s", ip_val,
- strerror (errno));
- return;
- }
-
- new_bit->bit = bit;
- new_bit->symbol = rspamd_mempool_strdup (
- cfg->cfg_pool,
- symbol);
- /* Convert to uppercase */
- p = new_bit->symbol;
- while (*p) {
- *p = g_ascii_toupper (*p);
- p++;
- }
- msg_debug_config ("add new IP suffix: %d with symbol: %s",
- (gint)new_bit->bit, new_bit->symbol);
- g_hash_table_insert (suffix->ips, &new_bit->bit,
- new_bit);
-}
-
-static gint
-surbl_module_parse_rule (const ucl_object_t* value, struct rspamd_config* cfg)
-{
- const ucl_object_t* cur_rule;
- const ucl_object_t* cur;
- gint cb_id;
- gint nrules = 0;
- struct suffix_item* new_suffix;
- const gchar *monitored_domain = NULL;
- struct surbl_bit_item* new_bit;
- ucl_object_t *ropts;
- struct surbl_ctx *surbl_module_ctx = surbl_get_context (cfg);
-
- LL_FOREACH(value, cur_rule) {
- monitored_domain = NULL;
-
- cur = ucl_object_lookup (cur_rule, "enabled");
- if (cur != NULL && cur->type == UCL_BOOLEAN) {
- if (!ucl_object_toboolean (cur)) {
- continue;
- }
- }
-
- cur = ucl_object_lookup (cur_rule, "suffix");
- if (cur == NULL) {
- msg_err_config("surbl rule must have explicit symbol "
- "definition");
- continue;
- }
-
- new_suffix = rspamd_mempool_alloc0 (cfg->cfg_pool,
- sizeof (struct suffix_item));
- new_suffix->magic = rspamd_surbl_cb_magic;
- new_suffix->suffix = rspamd_mempool_strdup (
- cfg->cfg_pool, ucl_obj_tostring (cur));
- new_suffix->options = 0;
- new_suffix->bits = g_array_new (FALSE, FALSE,
- sizeof (struct surbl_bit_item));
- rspamd_mempool_add_destructor (cfg->cfg_pool,
- (rspamd_mempool_destruct_t )rspamd_array_free_hard,
- new_suffix->bits);
-
- cur = ucl_object_lookup (cur_rule, "symbol");
- if (cur == NULL) {
- if (ucl_object_key (value)) {
- new_suffix->symbol = rspamd_mempool_strdup (
- cfg->cfg_pool,
- ucl_object_key (value));
- }
- else {
- msg_warn_config(
- "surbl rule for suffix %s lacks symbol, using %s as symbol",
- new_suffix->suffix, DEFAULT_SURBL_SYMBOL);
- new_suffix->symbol = rspamd_mempool_strdup (
- cfg->cfg_pool, DEFAULT_SURBL_SYMBOL);
- }
- }
- else {
- new_suffix->symbol = rspamd_mempool_strdup (
- cfg->cfg_pool, ucl_obj_tostring (cur));
- }
-
- cur = ucl_object_lookup (cur_rule, "options");
- if (cur != NULL && cur->type == UCL_STRING) {
- if (strstr(ucl_obj_tostring (cur), "noip") != NULL) {
- new_suffix->options |= SURBL_OPTION_NOIP;
- }
- }
-
- cur = ucl_object_lookup (cur_rule, "no_ip");
- if (cur != NULL && cur->type == UCL_BOOLEAN) {
- if (ucl_object_toboolean (cur)) {
- new_suffix->options |= SURBL_OPTION_NOIP;
- }
- }
-
- cur = ucl_object_lookup (cur_rule, "monitored_domain");
- if (cur != NULL && cur->type == UCL_STRING) {
- monitored_domain = ucl_object_tostring (cur);
- }
-
- cur = ucl_object_lookup (cur_rule, "resolve_ip");
- if (cur != NULL && cur->type == UCL_BOOLEAN) {
- if (ucl_object_toboolean (cur)) {
- new_suffix->options |= SURBL_OPTION_RESOLVEIP;
-
- if (!monitored_domain) {
- monitored_domain = "1.0.0.127";
- }
- }
- }
-
- if (!monitored_domain) {
- monitored_domain = rspamd_surbl_default_monitored;
- }
-
- ropts = ucl_object_typed_new (UCL_OBJECT);
- ucl_object_insert_key (ropts,
- ucl_object_fromstring (monitored_domain),
- "prefix", 0, false);
- ucl_object_insert_key (ropts,
- ucl_object_fromstring ("nxdomain"),
- "rcode", 0, false);
- rspamd_mempool_add_destructor (cfg->cfg_pool,
- (rspamd_mempool_destruct_t )ucl_object_unref,
- ropts);
-
- cur = ucl_object_lookup_any (cur_rule, "images", "check_images", NULL);
- if (cur != NULL && cur->type == UCL_BOOLEAN) {
- if (ucl_object_toboolean (cur)) {
- new_suffix->options |= SURBL_OPTION_CHECKIMAGES;
- }
- }
-
- cur = ucl_object_lookup_any (cur_rule, "emails", "check_emails", NULL);
- if (cur != NULL && cur->type == UCL_BOOLEAN) {
- if (ucl_object_toboolean (cur)) {
- new_suffix->options |= SURBL_OPTION_CHECKEMAILS;
- }
- }
-
- cur = ucl_object_lookup_any (cur_rule, "dkim", "check_dkim", NULL);
- if (cur != NULL && cur->type == UCL_BOOLEAN) {
- if (ucl_object_toboolean (cur)) {
- new_suffix->options |= SURBL_OPTION_CHECKDKIM;
- }
- }
-
- cur = ucl_object_lookup (cur_rule, "full_domain");
- if (cur != NULL && cur->type == UCL_BOOLEAN) {
- if (ucl_object_toboolean (cur)) {
- new_suffix->options |= SURBL_OPTION_FULLDOMAIN;
- }
- }
-
- if ((new_suffix->options & (SURBL_OPTION_RESOLVEIP | SURBL_OPTION_NOIP))
- == (SURBL_OPTION_NOIP | SURBL_OPTION_RESOLVEIP)) {
- /* Mutually exclusive options */
- msg_err_config ("options noip and resolve_ip are "
- "mutually exclusive for suffix %s", new_suffix->suffix);
-
- continue;
- }
-
- GString *sym = g_string_sized_new (127);
- gchar *p;
-
- rspamd_printf_gstring (sym, "SURBL_%s",
- new_suffix->suffix);
-
- p = sym->str;
-
- while (*p) {
- if (*p == '.') {
- *p = '_';
- }
- else {
- *p = g_ascii_toupper (*p);
- }
-
- p ++;
- }
-
- cb_id = rspamd_symcache_add_symbol (cfg->cache, sym->str,
- 0, surbl_test_url, new_suffix, SYMBOL_TYPE_CALLBACK, -1);
- rspamd_config_add_symbol (cfg,
- sym->str,
- 0.0,
- "SURBL rule check callback",
- "surbl",
- RSPAMD_SYMBOL_FLAG_IGNORE,
- 1,
- 1);
- rspamd_symcache_add_dependency (cfg->cache, cb_id,
- SURBL_REDIRECTOR_CALLBACK, -1);
- /* Failure symbol */
- g_string_append (sym, "_FAIL");
- rspamd_symcache_add_symbol (cfg->cache, sym->str,
- 0, NULL, NULL, SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_NOSTAT, cb_id);
- rspamd_config_add_symbol (cfg, sym->str, 0.0, "SURBL failure symbol",
- "surbl", 0, 0, 0);
- g_string_free (sym, TRUE);
-
- nrules++;
- new_suffix->callback_id = cb_id;
- cur = ucl_object_lookup (cur_rule, "bits");
-
- if (cur != NULL && cur->type == UCL_OBJECT) {
- ucl_object_iter_t it = NULL;
- const ucl_object_t* cur_bit;
- guint32 bit;
-
- while ((cur_bit = ucl_object_iterate (cur, &it, true)) != NULL) {
- if (ucl_object_key (cur_bit) != NULL
- && cur_bit->type == UCL_INT) {
- gchar* p;
- bit = ucl_obj_toint (cur_bit);
- new_bit = rspamd_mempool_alloc (
- cfg->cfg_pool,
- sizeof(struct surbl_bit_item));
- new_bit->bit = bit;
- new_bit->symbol = rspamd_mempool_strdup (
- cfg->cfg_pool,
- ucl_object_key (cur_bit));
- /* Convert to uppercase */
- p = new_bit->symbol;
-
- while (*p) {
- *p = g_ascii_toupper (*p);
- p++;
- }
-
- msg_debug_config("add new bit suffix: %d with symbol: %s",
- (gint)new_bit->bit, new_bit->symbol);
- g_array_append_val(new_suffix->bits, *new_bit);
- }
- }
- }
-
- cur = ucl_object_lookup(cur_rule, "ips");
- if (cur != NULL && cur->type == UCL_OBJECT) {
- ucl_object_iter_t it = NULL;
- const ucl_object_t* cur_bit;
-
- new_suffix->ips = g_hash_table_new (g_int_hash, g_int_equal);
- rspamd_mempool_add_destructor (cfg->cfg_pool,
- (rspamd_mempool_destruct_t )g_hash_table_unref,
- new_suffix->ips);
-
- while ((cur_bit = ucl_object_iterate (cur, &it, true)) != NULL) {
- if (ucl_object_key (cur_bit) != NULL) {
- if (ucl_object_type (cur_bit) == UCL_STRING) {
- /* Single IP */
- surbl_module_add_ip (cur_bit, ucl_object_key (cur_bit),
- new_suffix, cfg);
- }
- else if (ucl_object_type (cur_bit) == UCL_ARRAY) {
- ucl_object_iter_t ar_it = NULL;
- const ucl_object_t* cur_ar;
-
- /* Array of IPs */
- while ((cur_ar = ucl_object_iterate (cur_bit, &ar_it,
- true)) != NULL) {
- if (ucl_object_type (cur_ar) == UCL_STRING) {
- surbl_module_add_ip (cur_ar,
- ucl_object_key (cur_bit),
- new_suffix, cfg);
- }
- else {
- msg_err_config ("garbadge in ips element");
- }
- }
- }
- }
- }
- }
-
- cur = ucl_object_lookup (cur_rule, "process_script");
- if (cur != NULL && cur->type == UCL_STRING) {
- lua_State *L = cfg->lua_state;
- GString *tb;
- gint err_idx;
- const gchar *input = ucl_object_tostring (cur);
- gboolean loaded = FALSE;
-
- lua_pushcfunction (L, &rspamd_lua_traceback);
- err_idx = lua_gettop (L);
-
- /* First try return + input */
- tb = g_string_sized_new (strlen (input) + sizeof ("return "));
- rspamd_printf_gstring (tb, "return %s", input);
-
- if (luaL_loadstring (L, tb->str) != 0) {
- /* Reset stack */
- lua_settop (L, err_idx - 1);
- lua_pushcfunction (L, &rspamd_lua_traceback);
- err_idx = lua_gettop (L);
- /* Try with no return */
- if (luaL_loadstring (L, input) != 0) {
- msg_err_config ("cannot load string %s\n",
- input);
- }
- else {
- loaded = TRUE;
- }
- }
- else {
- loaded = TRUE;
- }
-
- g_string_free (tb, TRUE);
-
- if (loaded) {
- if (lua_pcall (L, 0, 1, err_idx) != 0) {
- msg_err_config ("call failed: %s\n", lua_tostring (L, -1));
- }
- else if (lua_isfunction (L, -1)) {
- new_suffix->url_process_cbref = luaL_ref (L,
- LUA_REGISTRYINDEX);
- }
- }
-
- lua_settop (L, err_idx - 1);
- }
-
- if (new_suffix->symbol) {
- /* Register just a symbol itself */
- rspamd_symcache_add_symbol (cfg->cache,
- new_suffix->symbol, 0,
- NULL, NULL, SYMBOL_TYPE_VIRTUAL, cb_id);
- nrules++;
- }
-
- new_suffix->m = rspamd_monitored_create (cfg->monitored_ctx,
- new_suffix->suffix, RSPAMD_MONITORED_DNS,
- RSPAMD_MONITORED_DEFAULT, ropts);
- surbl_module_ctx->suffixes = g_list_prepend (surbl_module_ctx->suffixes,
- new_suffix);
- }
-
- return nrules;
-}
-
-gint
-surbl_module_config (struct rspamd_config *cfg)
-{
- GList *cur_opt;
- struct suffix_item *cur_suffix = NULL;
- const ucl_object_t *value, *cur;
- const gchar *redir_val;
- gint nrules = 0;
- lua_State *L;
- struct surbl_ctx *surbl_module_ctx = surbl_get_context (cfg);
-
- if (!rspamd_config_is_module_enabled (cfg, "surbl")) {
- return TRUE;
- }
-
- /* Register global methods */
- L = cfg->lua_state;
- lua_getglobal (L, "rspamd_plugins");
-
- if (lua_type (L, -1) == LUA_TTABLE) {
- lua_pushstring (L, "surbl");
- lua_createtable (L, 0, 3);
- /* Set methods */
- lua_pushstring (L, "register_redirect");
- lua_pushcfunction (L, surbl_register_redirect_handler);
- lua_settable (L, -3);
- lua_pushstring (L, "continue_process");
- lua_pushcfunction (L, surbl_continue_process_handler);
- lua_settable (L, -3);
- lua_pushstring (L, "is_redirector");
- lua_pushcfunction (L, surbl_is_redirector_handler);
- lua_settable (L, -3);
- /* Finish surbl key */
- lua_settable (L, -3);
- }
-
- lua_pop (L, 1); /* Remove global function */
-
- (void) rspamd_symcache_add_symbol (cfg->cache, SURBL_REDIRECTOR_CALLBACK,
- 0, surbl_test_redirector, NULL,
- SYMBOL_TYPE_CALLBACK, -1);
- rspamd_config_add_symbol (cfg,
- SURBL_REDIRECTOR_CALLBACK,
- 0.0,
- "SURBL redirector check callback",
- "surbl",
- RSPAMD_SYMBOL_FLAG_IGNORE,
- 1,
- 1);
-
- if ((value =
- rspamd_config_get_module_opt (cfg, "surbl", "redirector")) != NULL) {
- surbl_module_ctx->redirectors = rspamd_upstreams_create (cfg->ups_ctx);
- rspamd_mempool_add_destructor (cfg->cfg_pool,
- (rspamd_mempool_destruct_t)rspamd_upstreams_destroy,
- surbl_module_ctx->redirectors);
- LL_FOREACH (value, cur)
- {
- redir_val = ucl_obj_tostring (cur);
- if (rspamd_upstreams_add_upstream (surbl_module_ctx->redirectors,
- redir_val, 80, RSPAMD_UPSTREAM_PARSE_DEFAULT,
- NULL)) {
- surbl_module_ctx->use_redirector = TRUE;
- }
- }
- }
- if ((value =
- rspamd_config_get_module_opt (cfg, "surbl",
- "redirector_symbol")) != NULL) {
- surbl_module_ctx->redirector_symbol = ucl_obj_tostring (value);
- rspamd_symcache_add_symbol (cfg->cache,
- surbl_module_ctx->redirector_symbol,
- 0, NULL, NULL, SYMBOL_TYPE_COMPOSITE, -1);
- }
- else {
- surbl_module_ctx->redirector_symbol = NULL;
- }
- if ((value =
- rspamd_config_get_module_opt (cfg, "surbl", "weight")) != NULL) {
- surbl_module_ctx->weight = ucl_obj_toint (value);
- }
- else {
- surbl_module_ctx->weight = DEFAULT_SURBL_WEIGHT;
- }
-
-
- if ((value =
- rspamd_config_get_module_opt (cfg, "surbl", "use_tags")) != NULL) {
- surbl_module_ctx->use_tags = ucl_obj_toboolean (value);
- }
- else {
- surbl_module_ctx->use_tags = FALSE;
- }
-
- if ((value =
- rspamd_config_get_module_opt (cfg, "surbl",
- "redirector_read_timeout")) != NULL) {
- surbl_module_ctx->read_timeout = ucl_obj_todouble (value);
- }
- else {
- surbl_module_ctx->read_timeout = DEFAULT_REDIRECTOR_READ_TIMEOUT;
- }
- if ((value =
- rspamd_config_get_module_opt (cfg, "surbl",
- "redirector_hosts_map")) != NULL) {
- if (!rspamd_map_add_from_ucl (cfg, value,
- "SURBL redirectors list",
- read_redirectors_list,
- fin_redirectors_list,
- dtor_redirectors_list,
- (void **)&surbl_module_ctx->redirector_tlds)) {
-
- msg_warn_config ("bad redirectors map definition: %s",
- ucl_obj_tostring (value));
- }
- }
-
- if ((value =
- rspamd_config_get_module_opt (cfg, "surbl", "exceptions")) != NULL) {
- rspamd_map_add_from_ucl (cfg, value,
- "SURBL exceptions list",
- read_exceptions_list,
- fin_exceptions_list,
- dtor_exceptions_list,
- (void **)&surbl_module_ctx->exceptions);
- }
- if ((value =
- rspamd_config_get_module_opt (cfg, "surbl", "whitelist")) != NULL) {
- rspamd_map_add_from_ucl (cfg, value,
- "SURBL whitelist",
- rspamd_kv_list_read,
- rspamd_kv_list_fin,
- rspamd_kv_list_dtor,
- (void **)&surbl_module_ctx->whitelist);
- }
-
- value = rspamd_config_get_module_opt (cfg, "surbl", "rule");
- if (value != NULL && value->type == UCL_OBJECT) {
- ucl_object_iter_t it = NULL;
- const ucl_object_t *cur_value;
-
- if (ucl_object_lookup (value, "symbol") != NULL) {
- /* Old style */
- nrules += surbl_module_parse_rule (value, cfg);
- }
- else {
- /* New style */
- while ((cur_value = ucl_object_iterate (value, &it, true)) != NULL) {
- nrules += surbl_module_parse_rule (cur_value, cfg);
- }
- }
- }
-
- value = rspamd_config_get_module_opt (cfg, "surbl", "rules");
- if (value != NULL && value->type == UCL_OBJECT) {
- ucl_object_iter_t it = NULL;
- const ucl_object_t *cur_value;
-
- /* New style only */
- while ((cur_value = ucl_object_iterate (value, &it, true)) != NULL) {
- nrules += surbl_module_parse_rule (cur_value, cfg);
- }
- }
-
- /* Add default suffix */
- if (surbl_module_ctx->suffixes == NULL) {
- msg_err_config ("surbl module loaded but no suffixes defined, skip "
- "checks");
- return TRUE;
- }
-
- if (surbl_module_ctx->suffixes != NULL) {
- rspamd_mempool_add_destructor (cfg->cfg_pool,
- (rspamd_mempool_destruct_t) g_list_free,
- surbl_module_ctx->suffixes);
- }
-
-
- cur_opt = surbl_module_ctx->suffixes;
- while (cur_opt) {
- cur_suffix = cur_opt->data;
-
- if (cur_suffix->bits != NULL || cur_suffix->ips != NULL) {
- register_bit_symbols (cfg, cur_suffix, cur_suffix->callback_id);
- }
-
- if (cur_suffix->options & SURBL_OPTION_CHECKDKIM) {
- rspamd_symcache_add_dependency (cfg->cache,
- cur_suffix->callback_id, "DKIM_TRACE", -1);
- }
-
- cur_opt = g_list_next (cur_opt);
- }
-
- surbl_module_ctx->max_redirected_urls = default_max_redirected_urls;
-
- if ((value =
- rspamd_config_get_module_opt (cfg, "surbl", "max_redirected_urls")) != NULL) {
- surbl_module_ctx->max_redirected_urls = ucl_obj_toint (value);
- }
-
- msg_info_config ("init internal surbls module, %d uribl rules loaded",
- nrules);
-
- return TRUE;
-}
-
-gint
-surbl_module_reconfig (struct rspamd_config *cfg)
-{
- struct surbl_ctx *surbl_module_ctx = surbl_get_context (cfg);
-
- /* Reinit module */
- surbl_module_ctx->use_redirector = 0;
- surbl_module_ctx->suffixes = NULL;
- surbl_module_ctx->redirectors = NULL;
- surbl_module_ctx->whitelist = NULL;
- /* Zero exceptions hashes */
- surbl_module_ctx->exceptions = NULL;
-
- rspamd_mempool_add_destructor (cfg->cfg_pool,
- (rspamd_mempool_destruct_t) g_list_free,
- surbl_module_ctx->suffixes);
-
- /* Perform configure */
- return surbl_module_config (cfg);
-}
-
-
-
-static gchar *
-format_surbl_request (rspamd_mempool_t * pool,
- rspamd_ftok_t * hostname,
- struct suffix_item *suffix,
- gboolean append_suffix,
- GError ** err,
- gboolean forced,
- GHashTable *tree,
- struct rspamd_url *url,
- lua_State *L,
- struct surbl_ctx *surbl_module_ctx)
-{
- GHashTable *t;
- gchar *result = NULL;
- const gchar *p, *dots[MAX_LEVELS];
- gint r, i, dots_num = 0, level = MAX_LEVELS;
- gsize slen, len;
- gboolean found_exception = FALSE;
- rspamd_ftok_t f;
-
-
- if (G_LIKELY (suffix != NULL)) {
- slen = strlen (suffix->suffix);
- }
- else if (!append_suffix) {
- slen = 0;
- }
- else {
- g_assert_not_reached ();
- }
-
- len = hostname->len + slen + 2;
-
- p = hostname->begin;
-
- while (p - hostname->begin < (gint)hostname->len && dots_num < MAX_LEVELS) {
- if (*p == '.') {
- dots[dots_num] = p;
- dots_num++;
- }
-
- p++;
- }
-
- /* Check for numeric expressions */
- if (url->flags & RSPAMD_URL_FLAG_NUMERIC) {
- /* This is ip address */
- if (suffix != NULL && (suffix->options & SURBL_OPTION_NOIP) != 0) {
- /* Ignore such requests */
- msg_info_pool ("ignore request of ip url for list %s",
- suffix->symbol);
- return NULL;
- }
-
- if (dots_num == 3) {
- /* IPv4 address */
- result = rspamd_mempool_alloc (pool, len);
- r = rspamd_snprintf (result, len, "%*s.%*s.%*s.%*s",
- (gint) (hostname->len - (dots[2] - hostname->begin + 1)),
- dots[2] + 1,
- (gint) (dots[2] - dots[1] - 1),
- dots[1] + 1,
- (gint) (dots[1] - dots[0] - 1),
- dots[0] + 1,
- (gint) (dots[0] - hostname->begin),
- hostname->begin);
- }
- else {
- /* Just pring ip as is */
- result = rspamd_mempool_alloc (pool, len);
- r = rspamd_snprintf (result, len, "%*s",
- (gint)hostname->len, hostname->begin);
- }
- }
- else {
- /* Not a numeric url */
- result = rspamd_mempool_alloc (pool, len);
-
- if (suffix->options & SURBL_OPTION_FULLDOMAIN) {
- /* Full domain case */
- r = rspamd_snprintf (result,
- len,
- "%*s",
- url->hostlen,
- url->host);
- }
- else {
- /* Now we should try to check for exceptions */
- if (!forced && surbl_module_ctx->exceptions) {
- for (i = MAX_LEVELS - 1; i >= 0; i--) {
- t = surbl_module_ctx->exceptions[i];
- if (t != NULL && dots_num >= i + 1) {
- f.begin = dots[dots_num - i - 1] + 1;
- f.len = hostname->len -
- (dots[dots_num - i - 1] - hostname->begin + 1);
- if (g_hash_table_lookup (t, &f) != NULL) {
- level = dots_num - i - 1;
- found_exception = TRUE;
- break;
- }
- }
- }
- }
-
- if (found_exception || url->tldlen == 0) {
- if (level != MAX_LEVELS) {
- if (level == 0) {
- r = rspamd_snprintf (result,
- len,
- "%T",
- hostname);
- }
- else {
- r = rspamd_snprintf (result, len, "%*s",
- (gint) (hostname->len -
- (dots[level - 1] - hostname->begin + 1)),
- dots[level - 1] + 1);
- }
- }
- else if (dots_num >= 2) {
- r = rspamd_snprintf (result, len, "%*s",
- (gint) (hostname->len -
- (dots[dots_num - 2] - hostname->begin + 1)),
- dots[dots_num - 2] + 1);
- }
- else {
- r = rspamd_snprintf (result,
- len,
- "%T",
- hostname);
- }
- }
- else {
- /* No exception */
- r = rspamd_snprintf (result,
- len,
- "%*s",
- url->tldlen,
- url->tld);
- }
- }
- }
-
- url->surbl = result;
- url->surbllen = r;
-
- if (!forced &&
- rspamd_match_hash_map (surbl_module_ctx->whitelist, result) != NULL) {
- msg_debug_pool ("url %s is whitelisted", result);
- g_set_error (err, SURBL_ERROR,
- WHITELIST_ERROR,
- "URL is whitelisted: %s",
- result);
- return NULL;
- }
-
- if (append_suffix) {
- if (suffix->url_process_cbref > 0) {
- lua_rawgeti (L, LUA_REGISTRYINDEX, suffix->url_process_cbref);
- lua_pushstring (L, result);
- lua_pushstring (L, suffix->suffix);
-
- if (lua_pcall (L, 2, 1, 0) != 0) {
- msg_err_pool ("cannot call url process script: %s",
- lua_tostring (L, -1));
- lua_pop (L, 1);
- rspamd_snprintf (result + r, len - r, ".%s", suffix->suffix);
- }
- else {
- result = rspamd_mempool_strdup (pool, lua_tostring (L, -1));
- lua_pop (L, 1);
- }
- }
- else {
- rspamd_snprintf (result + r, len - r, ".%s", suffix->suffix);
- }
- }
-
- if (tree != NULL) {
- if (g_hash_table_lookup (tree, result) != NULL) {
- msg_debug_pool ("url %s is already registered", result);
- g_set_error (err, SURBL_ERROR,
- DUPLICATE_ERROR,
- "URL is duplicated: %s",
- result);
- return NULL;
- }
- else {
- g_hash_table_insert (tree, result, url);
- }
- }
-
- msg_debug_pool ("request: %s, dots: %d, level: %d, orig: %*s",
- result,
- dots_num,
- level,
- (gint)hostname->len,
- hostname->begin);
-
- return result;
-}
-
-static void
-make_surbl_requests (struct rspamd_url *url, struct rspamd_task *task,
- struct rspamd_symcache_item *item,
- struct suffix_item *suffix,
- gboolean forced, GHashTable *tree,
- struct surbl_ctx *surbl_module_ctx)
-{
- gchar *surbl_req;
- rspamd_ftok_t f;
- GError *err = NULL;
- struct dns_param *param;
-
-
- f.begin = url->host;
- f.len = url->hostlen;
-
- if (suffix->options & SURBL_OPTION_RESOLVEIP) {
- /*
- * We need to get url real TLD, resolve it with no suffix and then
- * check against surbl using reverse octets printing
- */
- surbl_req = format_surbl_request (task->task_pool,
- &f,
- suffix,
- FALSE,
- &err,
- forced,
- tree,
- url,
- task->cfg->lua_state,
- surbl_module_ctx);
-
- if (surbl_req == NULL) {
- if (err != NULL) {
- if (err->code != WHITELIST_ERROR && err->code != DUPLICATE_ERROR) {
- msg_info_surbl ("cannot format url string for surbl %*s, %e",
- url->urllen, url->string,
- err);
- }
- g_error_free (err);
- return;
- }
- }
- else {
- /* XXX: We make merely A request here */
- param =
- rspamd_mempool_alloc (task->task_pool,
- sizeof (struct dns_param));
- param->url = url;
- param->task = task;
- param->suffix = suffix;
- param->host_resolve =
- rspamd_mempool_strdup (task->task_pool, surbl_req);
-
- rspamd_ftok_t ftmp;
- ftmp.begin = url->surbl;
- ftmp.len = url->surbllen;
- param->host_orig = rspamd_mempool_ftokdup (task->task_pool, &ftmp);
-
- msg_debug_surbl ("send surbl dns ip request %s to %s", surbl_req,
- suffix->suffix);
-
- if (rspamd_dns_resolver_request_task (task,
- surbl_dns_ip_callback,
- (void *) param, RDNS_REQUEST_A, surbl_req)) {
- param->item = item;
- rspamd_symcache_item_async_inc (task, item, M);
- }
- }
- }
- else if ((surbl_req = format_surbl_request (task->task_pool,
- &f,
- suffix,
- TRUE,
- &err,
- forced,
- tree,
- url,
- task->cfg->lua_state,
- surbl_module_ctx)) != NULL) {
- param =
- rspamd_mempool_alloc (task->task_pool, sizeof (struct dns_param));
- param->url = url;
- param->task = task;
- param->suffix = suffix;
- param->host_resolve =
- rspamd_mempool_strdup (task->task_pool, url->surbl);
-
- rspamd_ftok_t ftmp;
- ftmp.begin = url->surbl;
- ftmp.len = url->surbllen;
- param->host_orig = rspamd_mempool_ftokdup (task->task_pool, &ftmp);
-
- msg_debug_surbl ("send surbl dns request %s", surbl_req);
-
- if (rspamd_dns_resolver_request_task (task,
- surbl_dns_callback,
- (void *) param, RDNS_REQUEST_A, surbl_req)) {
- param->item = item;
- rspamd_symcache_item_async_inc (task, item, M);
- }
- }
- else if (err != NULL) {
- if (err->code != WHITELIST_ERROR && err->code != DUPLICATE_ERROR) {
- msg_info_surbl ("cannot format url string for surbl %*s, %e",
- url->urllen,
- url->string, err);
- }
- g_error_free (err);
- return;
- }
-}
-
-static void
-process_dns_results (struct rspamd_task *task,
- struct suffix_item *suffix,
- gchar *resolved_name,
- guint32 addr,
- struct rspamd_url *uri)
-{
- guint i;
- gboolean got_result = FALSE;
- struct surbl_bit_item *bit;
- struct in_addr ina;
- struct surbl_ctx *surbl_module_ctx = surbl_get_context (task->cfg);
-
- if (suffix->ips && g_hash_table_size (suffix->ips) > 0) {
-
- bit = g_hash_table_lookup (suffix->ips, &addr);
- if (bit != NULL) {
- msg_info_surbl ("domain [%s] is in surbl %s(%xd)",
- resolved_name, suffix->suffix,
- bit->bit);
- rspamd_task_insert_result (task, bit->symbol, 1, resolved_name);
-
- got_result = TRUE;
- }
- }
- else if (suffix->bits != NULL && suffix->bits->len > 0) {
- for (i = 0; i < suffix->bits->len; i ++) {
-
- bit = &g_array_index (suffix->bits, struct surbl_bit_item, i);
- msg_debug_surbl ("got result(%d) AND bit(%d): %d",
- (gint)addr,
- (gint)ntohl (bit->bit),
- (gint)bit->bit & (gint)ntohl (addr));
-
- if (((gint)bit->bit & (gint)ntohl (addr)) != 0) {
- got_result = TRUE;
- msg_info_surbl ("domain [%s] is in surbl %s(%xd)",
- resolved_name, suffix->suffix,
- bit->bit);
- rspamd_task_insert_result (task, bit->symbol, 1, resolved_name);
- }
- }
- }
- if (!got_result) {
- if ((suffix->bits == NULL || suffix->bits->len == 0) &&
- suffix->ips == NULL) {
- msg_info_surbl ("domain [%s] is in surbl %s",
- resolved_name, suffix->suffix);
- rspamd_task_insert_result (task, suffix->symbol, 1, resolved_name);
- }
- else {
- ina.s_addr = addr;
- msg_info_surbl ("domain [%s] is in surbl %s but at unknown result: %s",
- resolved_name, suffix->suffix,
- inet_ntoa (ina));
- }
- }
-}
-
-static void
-surbl_dns_callback (struct rdns_reply *reply, gpointer arg)
-{
- struct dns_param *param = (struct dns_param *)arg;
- struct rspamd_task *task;
- struct rdns_reply_entry *elt;
-
- task = param->task;
- if (reply->code == RDNS_RC_NOERROR && reply->entries) {
- msg_debug_surbl ("domain [%s] is in surbl %s",
- param->host_orig, param->suffix->suffix);
-
- DL_FOREACH (reply->entries, elt) {
- if (elt->type == RDNS_REQUEST_A) {
- process_dns_results (param->task, param->suffix,
- param->host_orig, (guint32) elt->content.a.addr.s_addr,
- param->url);
- }
- }
- }
- else {
- if (reply->code == RDNS_RC_NXDOMAIN || reply->code == RDNS_RC_NOREC) {
- msg_debug_surbl ("domain [%s] is not in surbl %s",
- param->host_orig,
- param->suffix->suffix);
- }
- else {
- /* Insert failure symbol */
- GString *sym = g_string_new (param->suffix->symbol);
-
- g_string_append (sym, "_FAIL");
- rspamd_task_insert_result (task, sym->str, 1.0,
- rdns_strerror (reply->code));
- g_string_free (sym, TRUE);
- }
- }
-
- rspamd_symcache_item_async_dec_check (param->task, param->item, M);
-}
-
-static void
-surbl_dns_ip_callback (struct rdns_reply *reply, gpointer arg)
-{
- struct dns_param *param = (struct dns_param *) arg;
- struct rspamd_task *task;
- struct rdns_reply_entry *elt;
- GString *to_resolve;
- guint32 ip_addr;
-
- task = param->task;
- /* If we have result from DNS server, this url exists in SURBL, so increase score */
- if (reply->code == RDNS_RC_NOERROR && reply->entries) {
-
- LL_FOREACH (reply->entries, elt) {
-
- if (elt->type == RDNS_REQUEST_A) {
- to_resolve = g_string_sized_new (
- strlen (param->suffix->suffix) +
- sizeof ("255.255.255.255."));
- ip_addr = elt->content.a.addr.s_addr;
-
- /* Big endian <4>.<3>.<2>.<1> */
- rspamd_printf_gstring (to_resolve, "%d.%d.%d.%d.%s",
- ip_addr >> 24 & 0xff,
- ip_addr >> 16 & 0xff,
- ip_addr >> 8 & 0xff,
- ip_addr & 0xff, param->suffix->suffix);
- msg_debug_surbl (
- "domain [%s] send %v request to surbl",
- param->host_orig,
- to_resolve);
-
- if (rspamd_dns_resolver_request_task (task,
- surbl_dns_callback,
- param, RDNS_REQUEST_A, to_resolve->str)) {
- rspamd_symcache_item_async_inc (param->task, param->item, M);
- }
-
- g_string_free (to_resolve, TRUE);
- }
- }
- }
- else {
- msg_debug_surbl ("domain [%s] cannot be resolved for SURBL check %s",
- param->host_resolve,
- param->suffix->suffix);
-
- }
-
- rspamd_symcache_item_async_dec_check (param->task, param->item, M);
-}
-
-static void
-free_redirector_session (void *ud)
-{
- struct redirector_param *param = (struct redirector_param *)ud;
-
- if (param->item) {
- rspamd_symcache_item_async_dec_check (param->task, param->item, M);
- }
-
- rspamd_http_connection_unref (param->conn);
-}
-
-static void
-surbl_redirector_error (struct rspamd_http_connection *conn,
- GError *err)
-{
- struct redirector_param *param = (struct redirector_param *)conn->ud;
- struct rspamd_task *task;
-
- task = param->task;
- msg_err_surbl ("connection with http server %s terminated incorrectly: %e",
- rspamd_inet_address_to_string (
- rspamd_upstream_addr_cur (param->redirector)),
- err);
- rspamd_upstream_fail (param->redirector, FALSE);
- rspamd_session_remove_event (param->task->s, free_redirector_session,
- param);
-}
-
-static int
-surbl_redirector_finish (struct rspamd_http_connection *conn,
- struct rspamd_http_message *msg)
-{
- struct redirector_param *param = (struct redirector_param *)conn->ud;
- struct rspamd_task *task;
- struct surbl_ctx *surbl_module_ctx;
- gint r, urllen;
- struct rspamd_url *redirected_url, *existing;
- const rspamd_ftok_t *hdr;
- gchar *urlstr;
-
- task = param->task;
- surbl_module_ctx = surbl_get_context (task->cfg);
-
- if (msg->code == 200) {
- hdr = rspamd_http_message_find_header (msg, "Uri");
-
- if (hdr != NULL) {
- msg_info_surbl ("got reply from redirector: '%*s' -> '%T'",
- param->url->urllen, param->url->string,
- hdr);
- urllen = hdr->len;
- urlstr = rspamd_mempool_alloc (task->task_pool,
- urllen + 1);
- redirected_url = rspamd_mempool_alloc0 (task->task_pool,
- sizeof (*redirected_url));
- rspamd_strlcpy (urlstr, hdr->begin, urllen + 1);
- r = rspamd_url_parse (redirected_url, urlstr, urllen,
- task->task_pool, RSPAMD_URL_PARSE_TEXT);
-
- if (r == URI_ERRNO_OK) {
- if ((existing = g_hash_table_lookup (MESSAGE_FIELD (task, urls),
- redirected_url)) == NULL) {
- g_hash_table_insert (MESSAGE_FIELD (task, urls), redirected_url,
- redirected_url);
- redirected_url->phished_url = param->url;
- redirected_url->flags |= RSPAMD_URL_FLAG_REDIRECTED;
- }
- else {
- existing->count ++;
- }
- }
- else {
- msg_info_surbl ("cannot parse redirector reply: %s", urlstr);
- }
- }
- }
- else {
- msg_info_surbl ("could not resolve '%*s' on redirector",
- param->url->urllen, param->url->string);
- }
-
- rspamd_upstream_ok (param->redirector);
- rspamd_session_remove_event (param->task->s, free_redirector_session,
- param);
-
- return 0;
-}
-
-
-static void
-register_redirector_call (struct rspamd_url *url, struct rspamd_task *task,
- const gchar *rule)
-{
- struct redirector_param *param;
- struct upstream *selected;
- struct rspamd_http_message *msg;
- struct surbl_ctx *surbl_module_ctx = surbl_get_context (task->cfg);
-
- if (!rspamd_session_blocked (task->s)) {
-
- selected = rspamd_upstream_get (surbl_module_ctx->redirectors,
- RSPAMD_UPSTREAM_ROUND_ROBIN, url->host, url->hostlen);
- param = rspamd_mempool_alloc0 (task->task_pool,
- sizeof (struct redirector_param));
-
- if (selected) {
- param->conn = rspamd_http_connection_new_client (NULL,
- NULL,
- surbl_redirector_error,
- surbl_redirector_finish,
- RSPAMD_HTTP_CLIENT_SIMPLE,
- rspamd_upstream_addr_next (selected));
- }
-
- if (param->conn == NULL) {
- msg_info_surbl ("cannot create tcp socket failed: %s",
- strerror (errno));
-
- return;
- }
-
-
- param->url = url;
- param->task = task;
- param->ctx = surbl_module_ctx;
- msg = rspamd_http_new_message (HTTP_REQUEST);
- msg->url = rspamd_fstring_assign (msg->url, url->string, url->urllen);
- param->redirector = selected;
-
- rspamd_session_add_event (task->s,
- free_redirector_session, param,
- M);
- param->item = rspamd_symcache_get_cur_item (task);
-
- if (param->item) {
- rspamd_symcache_item_async_inc (param->task, param->item, M);
- }
-
- rspamd_http_connection_write_message (param->conn, msg, NULL,
- NULL, param, surbl_module_ctx->read_timeout);
-
- msg_info_surbl (
- "registered redirector call for %*s to %s, according to rule: %s",
- url->urllen, url->string,
- rspamd_upstream_name (param->redirector),
- rule);
- }
-}
-
-static void
-surbl_tree_redirector_callback (gpointer key, gpointer value, void *data)
-{
- struct redirector_param *param = data, *nparam;
- struct rspamd_task *task, **ptask;
- struct rspamd_url *url = value, **purl;
- lua_State *L;
- rspamd_regexp_t *re;
- rspamd_ftok_t srch;
- gboolean found = FALSE;
- gchar *found_tld;
- struct surbl_ctx *surbl_module_ctx;
-
- task = param->task;
- surbl_module_ctx = param->ctx;
-
- msg_debug_surbl ("check url redirection %*s", url->urllen, url->string);
-
- if (url->hostlen <= 0) {
- return;
- }
-
- /* Search in trie */
- srch.begin = url->tld;
- srch.len = url->tldlen;
- re = g_hash_table_lookup (surbl_module_ctx->redirector_tlds, &srch);
-
- if (re) {
- if (re == NO_REGEXP) {
- found = TRUE;
- }
- else if (rspamd_regexp_search (re, url->string, 0,
- NULL, NULL, TRUE, NULL)) {
- found = TRUE;
- }
-
- if (found) {
- found_tld = rspamd_mempool_ftokdup (task->task_pool, &srch);
-
- if (surbl_module_ctx->redirector_symbol != NULL) {
- rspamd_task_insert_result (param->task,
- surbl_module_ctx->redirector_symbol,
- 1,
- found_tld);
- }
-
- if (param->redirector_requests >= surbl_module_ctx->max_redirected_urls) {
- msg_info_surbl ("cannot register redirector request for url domain: "
- "%s, max_redirected_urls is reached: %d",
- found_tld, surbl_module_ctx->max_redirected_urls);
-
- return;
- }
-
- param->redirector_requests ++;
-
- if (surbl_module_ctx->redirector_cbid != -1) {
- nparam = rspamd_mempool_alloc (task->task_pool,
- sizeof (*nparam));
- /* Copy to detach from the shared param */
- memcpy (nparam, param, sizeof (*param));
- nparam->url = url;
- L = task->cfg->lua_state;
- lua_rawgeti (L, LUA_REGISTRYINDEX,
- surbl_module_ctx->redirector_cbid);
- ptask = lua_newuserdata (L, sizeof (*ptask));
- *ptask = task;
- rspamd_lua_setclass (L, "rspamd{task}", -1);
- purl = lua_newuserdata (L, sizeof (*purl));
- *purl = url;
- rspamd_lua_setclass (L, "rspamd{url}", -1);
- lua_pushlightuserdata (L, nparam);
- rspamd_symcache_set_cur_item (task, param->item);
-
- if (lua_pcall (L, 3, 0, 0) != 0) {
- msg_err_task ("cannot call for redirector script: %s",
- lua_tostring (L, -1));
- lua_pop (L, 1);
- }
- else {
- nparam->item = param->item;
- }
- }
- else {
- register_redirector_call (url,
- param->task,
- found_tld);
- }
- }
- }
-}
-
-static void
-surbl_tree_url_callback (gpointer key, gpointer value, void *data)
-{
- struct redirector_param *param = data;
- struct rspamd_url *url = value;
- struct rspamd_task *task;
- struct surbl_ctx *surbl_module_ctx;
-
- if (url->hostlen <= 0) {
- return;
- }
-
- if (url->flags & RSPAMD_URL_FLAG_HTML_DISPLAYED) {
- /* Skip urls that are displayed only */
- return;
- }
-
- task = param->task;
- surbl_module_ctx = param->ctx;
-
- msg_debug_surbl ("check url %*s in %s", url->urllen, url->string,
- param->suffix->suffix);
-
- make_surbl_requests (url, param->task, param->item, param->suffix, FALSE,
- param->tree, surbl_module_ctx);
-}
-
-static void
-surbl_test_url (struct rspamd_task *task,
- struct rspamd_symcache_item *item,
- void *user_data)
-{
- struct redirector_param *param;
- struct suffix_item *suffix = user_data;
- guint i, j;
- struct rspamd_mime_text_part *part;
- struct html_image *img;
- struct rspamd_url *url;
- struct surbl_ctx *surbl_module_ctx = surbl_get_context (task->cfg);
-
- if (!rspamd_monitored_alive (suffix->m)) {
- if (!suffix->reported_offline) {
- msg_info_surbl ("disable surbl %s as it is reported to be offline",
- suffix->suffix);
- suffix->reported_offline = TRUE;
- }
- rspamd_symcache_finalize_item (task, item);
-
- return;
- }
-
- suffix->reported_offline = FALSE;
- param = rspamd_mempool_alloc0 (task->task_pool, sizeof (*param));
- param->task = task;
- param->suffix = suffix;
- param->tree = g_hash_table_new (rspamd_strcase_hash, rspamd_strcase_equal);
- param->ctx = surbl_module_ctx;
- param->item = item;
-
- rspamd_mempool_add_destructor (task->task_pool,
- (rspamd_mempool_destruct_t)g_hash_table_unref,
- param->tree);
- g_hash_table_foreach (MESSAGE_FIELD (task, urls),
- surbl_tree_url_callback, param);
-
- rspamd_symcache_item_async_inc (task, item, M);
-
- if (suffix->options & SURBL_OPTION_CHECKEMAILS) {
- g_hash_table_foreach (MESSAGE_FIELD (task, emails),
- surbl_tree_url_callback, param);
- }
-
- /* We also need to check and process img URLs */
- if (suffix->options & SURBL_OPTION_CHECKIMAGES) {
- PTR_ARRAY_FOREACH (MESSAGE_FIELD (task, text_parts), i, part) {
- if (part->html && part->html->images) {
- for (j = 0; j < part->html->images->len; j ++) {
- img = g_ptr_array_index (part->html->images, j);
-
- if ((img->flags & RSPAMD_HTML_FLAG_IMAGE_EXTERNAL)
- && img->url) {
- surbl_tree_url_callback (img->url, img->url, param);
- msg_debug_surbl ("checked image url %s over %s",
- img->src, suffix->suffix);
- }
- }
- }
- }
- }
-
- if (suffix->options & SURBL_OPTION_CHECKDKIM) {
- struct rspamd_symbol_result *s;
- struct rspamd_symbol_option *opt;
-
- s = rspamd_task_find_symbol_result (task, "DKIM_TRACE");
-
- if (s && s->opts_head) {
- DL_FOREACH (s->opts_head, opt) {
- gsize len = strlen (opt->option);
- gchar *p = opt->option + len - 1;
-
- if (*p == '+') {
- url = rspamd_html_process_url (task->task_pool,
- opt->option, len - 2, NULL);
-
- if (url) {
- surbl_tree_url_callback (url, url, param);
- msg_debug_surbl ("checked dkim url %s over %s",
- url->string, suffix->suffix);
- }
- }
- }
- }
- }
-
- rspamd_symcache_item_async_dec_check (task, item, M);
-}
-
-static void
-surbl_test_redirector (struct rspamd_task *task,
- struct rspamd_symcache_item *item,
- void *user_data)
-{
- struct redirector_param *param;
- guint i, j;
- struct rspamd_mime_text_part *part;
- struct html_image *img;
- struct rspamd_url *url;
- struct surbl_ctx *surbl_module_ctx = surbl_get_context (task->cfg);
-
- if (!surbl_module_ctx->use_redirector || !surbl_module_ctx->redirector_tlds) {
- rspamd_symcache_finalize_item (task, item);
-
- return;
- }
-
- rspamd_symcache_item_async_inc (task, item, M);
-
- param = rspamd_mempool_alloc0 (task->task_pool, sizeof (*param));
- param->task = task;
- param->suffix = NULL;
- param->redirector_requests = 0;
- param->ctx = surbl_module_ctx;
- param->item = item;
- g_hash_table_foreach (MESSAGE_FIELD (task, urls),
- surbl_tree_redirector_callback, param);
-
- /* We also need to check and process img URLs */
- PTR_ARRAY_FOREACH (MESSAGE_FIELD (task, text_parts), i, part) {
- if (part->html && part->html->images) {
- for (j = 0; j < part->html->images->len; j ++) {
- img = g_ptr_array_index (part->html->images, j);
-
- if ((img->flags & RSPAMD_HTML_FLAG_IMAGE_EXTERNAL)
- && img->src) {
- url = rspamd_html_process_url (task->task_pool,
- img->src, strlen (img->src), NULL);
-
- if (url) {
- surbl_tree_redirector_callback (url, url, param);
- msg_debug_surbl ("checked image url %s for redirectors",
- img->src);
- }
- }
- }
- }
- }
-
- rspamd_symcache_item_async_dec_check (task, item, M);
-}
-
-
-static gint
-surbl_register_redirect_handler (lua_State *L)
-{
- struct surbl_ctx *surbl_module_ctx;
- struct rspamd_config *cfg = lua_check_config (L, 1);
-
- if (!cfg) {
- return luaL_error (L, "config is now required as the first parameter");
- }
-
- surbl_module_ctx = surbl_get_context (cfg);
-
- if (surbl_module_ctx->redirector_cbid != -1) {
- luaL_unref (L, LUA_REGISTRYINDEX, surbl_module_ctx->redirector_cbid);
- }
-
- lua_pushvalue (L, 2);
-
- if (lua_type (L, -1) == LUA_TFUNCTION) {
- surbl_module_ctx->redirector_cbid = luaL_ref (L, LUA_REGISTRYINDEX);
- surbl_module_ctx->use_redirector = TRUE;
- }
- else {
- lua_pop (L, 1);
-
- return luaL_error (L, "argument must be a function");
- }
-
- return 0;
-}
-
-static gint
-surbl_is_redirector_handler (lua_State *L)
-{
- const gchar *url;
- struct rspamd_task *task;
- struct rspamd_url uri;
- gsize len;
- rspamd_regexp_t *re;
- rspamd_ftok_t srch;
- gboolean found = FALSE;
- gchar *found_tld, *url_cpy;
- struct surbl_ctx *surbl_module_ctx;
-
- task = lua_check_task (L, 1);
- url = luaL_checklstring (L, 2, &len);
-
- if (task && url) {
- surbl_module_ctx = surbl_get_context (task->cfg);
- url_cpy = rspamd_mempool_alloc (task->task_pool, len);
- memcpy (url_cpy, url, len);
-
- if (rspamd_url_parse (&uri, url_cpy, len, task->task_pool, RSPAMD_URL_PARSE_TEXT)) {
- msg_debug_surbl ("check url redirection %*s", uri.urllen,
- uri.string);
-
- if (uri.hostlen <= 0) {
- lua_pushboolean (L, false);
-
- return 1;
- }
-
- /* Search in trie */
- srch.begin = uri.tld;
- srch.len = uri.tldlen;
- re = g_hash_table_lookup (surbl_module_ctx->redirector_tlds, &srch);
-
- if (re) {
- if (re == NO_REGEXP) {
- found = TRUE;
- }
- else if (rspamd_regexp_search (re, uri.string, 0,
- NULL, NULL, TRUE, NULL)) {
- found = TRUE;
- }
-
- if (found) {
- found_tld = rspamd_mempool_ftokdup (task->task_pool, &srch);
- lua_pushboolean (L, true);
- lua_pushstring (L, found_tld);
-
- return 2;
- }
- }
- }
- }
- else {
- return luaL_error (L, "arguments must be: task, url");
- }
-
- lua_pushboolean (L, false);
-
- return 1;
-}
-
-/*
- * Accepts two arguments:
- * url: string with a redirected URL, if url is nil, then it couldn't be resolved
- * userdata: opaque pointer of `struct redirector_param *`
- */
-static gint
-surbl_continue_process_handler (lua_State *L)
-{
- struct redirector_param *param;
- struct rspamd_task *task;
- const gchar *nurl;
- gint r;
- gsize urllen;
- struct rspamd_url *redirected_url;
- gchar *urlstr;
- struct surbl_ctx *surbl_module_ctx;
-
- nurl = lua_tolstring (L, 1, &urllen);
- param = (struct redirector_param *)lua_topointer (L, 2);
-
- if (param != NULL) {
- task = param->task;
- surbl_module_ctx = surbl_get_context (task->cfg);
-
- if (nurl != NULL) {
- msg_info_surbl ("got reply from redirector: '%*s' -> '%*s'",
- param->url->urllen, param->url->string,
- (gint)urllen, nurl);
- urlstr = rspamd_mempool_alloc (task->task_pool,
- urllen + 1);
- redirected_url = rspamd_mempool_alloc0 (task->task_pool,
- sizeof (*redirected_url));
- rspamd_strlcpy (urlstr, nurl, urllen + 1);
- r = rspamd_url_parse (redirected_url, urlstr, urllen,
- task->task_pool, RSPAMD_URL_PARSE_TEXT);
-
- if (r == URI_ERRNO_OK) {
- if (!g_hash_table_lookup (MESSAGE_FIELD (task, urls),
- redirected_url)) {
- g_hash_table_insert (MESSAGE_FIELD (task, urls),
- redirected_url,
- redirected_url);
- redirected_url->phished_url = param->url;
- redirected_url->flags |= RSPAMD_URL_FLAG_REDIRECTED;
- }
- }
- else {
- msg_info_surbl ("could not resolve '%*s' on redirector",
- param->url->urllen, param->url->string);
- }
- }
- else {
- msg_info_surbl ("could not resolve '%*s' on redirector",
- param->url->urllen, param->url->string);
- }
- }
- else {
- return luaL_error (L, "invalid arguments");
- }
-
- return 0;
-}
projects / rspamd.git / commitdiff