* Fix freeing of module parameters (they should NOT be freed)

* Add minimum hash length to check with fuzzy module as well as whitelisting for fuzzy checks:
  min_length = "300"; <- messages with more than 300 characters would be checked with fuzzy check
  whitelist = "http://somehost/somefile"; <- ip addresses whitelisted
* Extend comments about module options for comfortable usage

diff --git a/src/plugins/chartable.c b/src/plugins/chartable.c
index ceedc13258ad480fa70d3e0cbef85e3f55987368..8486f66c593f26290852c38b0f815f8be86dfce9 100644 (file)
--- a/src/plugins/chartable.c
+++ b/src/plugins/chartable.c
@@ -24,6 +24,12 @@
 
 /***MODULE:chartable
  * rspamd module that make marks based on symbol chains
+ *
+ * Allowed options:
+ * - metric (string): metric to insert symbol (default: 'default')
+ * - symbol (string): symbol to insert (default: 'R_BAD_CHARSET')
+ * - threshold (double): value that would be used as threshold in expression characters_changed / total_characters
+ *   (e.g. if threshold is 0.1 than charset change should occure more often than in 10 symbols), default: 0.1
  */
 
 #include "../config.h"
@@ -75,14 +81,12 @@ chartable_module_config (struct config_file *cfg)
 
        if ((value = get_module_opt (cfg, "chartable", "metric")) != NULL) {
                chartable_module_ctx->metric = memory_pool_strdup (chartable_module_ctx->chartable_pool, value);
-               g_free (value);
        }
        else {
                chartable_module_ctx->metric = DEFAULT_METRIC;
        }
        if ((value = get_module_opt (cfg, "chartable", "symbol")) != NULL) {
                chartable_module_ctx->symbol = memory_pool_strdup (chartable_module_ctx->chartable_pool, value);
-               g_free (value);
        }
        else {
                chartable_module_ctx->symbol = DEFAULT_SYMBOL;

diff --git a/src/plugins/emails.c b/src/plugins/emails.c
index 88ea34b8b6f7f44999d422ebbdeb8436e9fe3f83..37d0b8d33d04a3864c3b1aef6394813319d5890f 100644 (file)
--- a/src/plugins/emails.c
+++ b/src/plugins/emails.c
@@ -24,6 +24,11 @@
 
 /***MODULE:email
  * rspamd module that extracts emails from messages and check them via blacklist
+ * 
+ * Allowed options:
+ * - metric (string): metric to insert symbol (default: 'default')
+ * - symbol (string): symbol to insert (default: 'R_BAD_EMAIL')
+ * - blacklist (map string): map that contains list of bad emails
  */
 
 #include "../config.h"
@@ -89,14 +94,12 @@ emails_module_config (struct config_file *cfg)
 
        if ((value = get_module_opt (cfg, "emails", "metric")) != NULL) {
                email_module_ctx->metric = memory_pool_strdup (email_module_ctx->email_pool, value);
-               g_free (value);
        }
        else {
                email_module_ctx->metric = DEFAULT_METRIC;
        }
        if ((value = get_module_opt (cfg, "emails", "symbol")) != NULL) {
                email_module_ctx->symbol = memory_pool_strdup (email_module_ctx->email_pool, value);
-               g_free (value);
        }
        else {
                email_module_ctx->symbol = DEFAULT_SYMBOL;

diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c
index e79c4036896d22d5b675603c87803b2379a12c4e..0a58464d69b375e0a9bfe6c4a44d142e36917a95 100644 (file)
--- a/src/plugins/fuzzy_check.c
+++ b/src/plugins/fuzzy_check.c
@@ -24,6 +24,15 @@
 
 /***MODULE:fuzzy
  * rspamd module that checks fuzzy checksums for messages
+ *
+ * Allowed options:
+ * - metric (string): metric to insert symbol (default: 'default')
+ * - symbol (string): symbol to insert (default: 'R_FUZZY')
+ * - max_score (double): maximum score to that weights of hashes would be normalized (default: 0 - no normalization)
+ * - min_length (integer): minimum length (in characters) for text part to be checked for fuzzy hash (default: 0 - no limit)
+ * - whitelist (map string): map of ip addresses that should not be checked with this module
+ * - servers (string): list of fuzzy servers in format "server1:port,server2:port" - these servers would be used for checking and storing
+ *   fuzzy hashes
  */
 
 #include "../config.h"
@@ -60,6 +69,8 @@ struct fuzzy_ctx {
        int                             servers_num;
        memory_pool_t                  *fuzzy_pool;
        double                          max_score;
+       uint32_t                        min_hash_len;
+       radix_tree_t                   *whitelist;
 };
 
 struct fuzzy_client_session {
@@ -197,26 +208,41 @@ fuzzy_check_module_config (struct config_file *cfg)
 
        if ((value = get_module_opt (cfg, "fuzzy_check", "metric")) != NULL) {
                fuzzy_module_ctx->metric = memory_pool_strdup (fuzzy_module_ctx->fuzzy_pool, value);
-               g_free (value);
        }
        else {
                fuzzy_module_ctx->metric = DEFAULT_METRIC;
        }
        if ((value = get_module_opt (cfg, "fuzzy_check", "symbol")) != NULL) {
                fuzzy_module_ctx->symbol = memory_pool_strdup (fuzzy_module_ctx->fuzzy_pool, value);
-               g_free (value);
        }
        else {
                fuzzy_module_ctx->symbol = DEFAULT_SYMBOL;
        }
        if ((value = get_module_opt (cfg, "fuzzy_check", "max_score")) != NULL) {
                fuzzy_module_ctx->max_score = strtod (value, NULL);
-               g_free (value);
        }
        else {
                fuzzy_module_ctx->max_score = 0.;
        }
-       
+
+       if ((value = get_module_opt (cfg, "fuzzy_check", "min_length")) != NULL) {
+               fuzzy_module_ctx->min_hash_len = strtoul (value, NULL, 10);
+       }
+       else {
+               fuzzy_module_ctx->min_hash_len = 0.;
+       }
+
+       if ((value = get_module_opt (cfg, "fuzzy_check", "whitelist")) != NULL) {
+               fuzzy_module_ctx->whitelist = radix_tree_create ();
+               if (!add_map (value, read_radix_list, fin_radix_list, (void **)&fuzzy_module_ctx->whitelist)) {
+                       msg_err ("cannot add whitelist '%s'", value);   
+               }
+       }
+       else {
+               fuzzy_module_ctx->whitelist = NULL;
+       }
+
+
        if ((value = get_module_opt (cfg, "fuzzy_check", "servers")) != NULL) {
                parse_servers_string (value);
        }
@@ -397,6 +423,14 @@ fuzzy_symbol_callback (struct worker_task *task, void *unused)
        GList                          *cur;
        int                             sock;
 
+       /* Check whitelist */
+       if (fuzzy_module_ctx->whitelist && task->from_addr.s_addr != 0) {
+               if (radix32tree_find (fuzzy_module_ctx->whitelist, ntohl ((uint32_t) task->from_addr.s_addr)) != RADIX_NO_VALUE) {
+                       msg_info ("address %s is whitelisted, skip fuzzy check", inet_ntoa (task->from_addr));
+                       return;
+               }
+       }
+
        cur = task->text_parts;
 
        while (cur) {
@@ -405,6 +439,15 @@ fuzzy_symbol_callback (struct worker_task *task, void *unused)
                        cur = g_list_next (cur);
                        continue;
                }
+
+               /* Check length of hash */
+               if (fuzzy_module_ctx->min_hash_len != 0 && 
+                               strlen (part->fuzzy->hash_pipe) * part->fuzzy->block_size < fuzzy_module_ctx->min_hash_len) {
+                       msg_info ("part hash is shorter than %d symbols, skip fuzzy check", fuzzy_module_ctx->min_hash_len);
+                       cur = g_list_next (cur);
+                       continue;
+               }
+
                /* Get upstream */
                selected = (struct storage_server *)get_upstream_by_hash (fuzzy_module_ctx->servers, fuzzy_module_ctx->servers_num,
                        sizeof (struct storage_server), task->ts.tv_sec,

diff --git a/src/plugins/regexp.c b/src/plugins/regexp.c
index 7be53e2529ad5f01ef38a0b9052c7d314b63874d..d44983785c9ea9137d6bb43828e203c9339ae55a 100644 (file)
--- a/src/plugins/regexp.c
+++ b/src/plugins/regexp.c
@@ -158,14 +158,12 @@ regexp_module_config (struct config_file *cfg)
 
        if ((value = get_module_opt (cfg, "regexp", "metric")) != NULL) {
                regexp_module_ctx->metric = memory_pool_strdup (regexp_module_ctx->regexp_pool, value);
-               g_free (value);
        }
        else {
                regexp_module_ctx->metric = DEFAULT_METRIC;
        }
        if ((value = get_module_opt (cfg, "regexp", "statfile_prefix")) != NULL) {
                regexp_module_ctx->statfile_prefix = memory_pool_strdup (regexp_module_ctx->regexp_pool, value);
-               g_free (value);
        }
        else {
                regexp_module_ctx->statfile_prefix = DEFAULT_STATFILE_PREFIX;

diff --git a/src/plugins/spf.c b/src/plugins/spf.c
index 57a6f4407134e1a8424b80c2d24230f1121c102e..8f1e8a07e3a7cad861bf9dde8a2d4f05f5b5a23c 100644 (file)
--- a/src/plugins/spf.c
+++ b/src/plugins/spf.c
@@ -24,6 +24,12 @@
 
 /***MODULE:spf
  * rspamd module that checks spf records of incoming email
+ *
+ * Allowed options:
+ * - metric (string): metric to insert symbol (default: 'default')
+ * - symbol_allow (string): symbol to insert (default: 'R_SPF_ALLOW')
+ * - symbol_fail (string): symbol to insert (default: 'R_SPF_FAIL')
+ * - symbol_softfail (string): symbol to insert (default: 'R_SPF_SOFTFAIL')
  */
 
 #include "../config.h"
@@ -78,28 +84,24 @@ spf_module_config (struct config_file *cfg)
        
        if ((value = get_module_opt (cfg, "spf", "metric")) != NULL) {
                spf_module_ctx->metric = memory_pool_strdup (spf_module_ctx->spf_pool, value);
-               g_free (value);
        }
        else {
                spf_module_ctx->metric = DEFAULT_METRIC;
        }
        if ((value = get_module_opt (cfg, "spf", "symbol_fail")) != NULL) {
                spf_module_ctx->symbol_fail = memory_pool_strdup (spf_module_ctx->spf_pool, value);
-               g_free (value);
        }
        else {
                spf_module_ctx->symbol_fail = DEFAULT_SYMBOL_FAIL;
        }
        if ((value = get_module_opt (cfg, "spf", "symbol_softfail")) != NULL) {
                spf_module_ctx->symbol_softfail = memory_pool_strdup (spf_module_ctx->spf_pool, value);
-               g_free (value);
        }
        else {
                spf_module_ctx->symbol_softfail = DEFAULT_SYMBOL_SOFTFAIL;
        }
        if ((value = get_module_opt (cfg, "spf", "symbol_allow")) != NULL) {
                spf_module_ctx->symbol_allow = memory_pool_strdup (spf_module_ctx->spf_pool, value);
-               g_free (value);
        }
        else {
                spf_module_ctx->symbol_allow = DEFAULT_SYMBOL_ALLOW;

diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c
index d93e48dd32be490521bf8736e0ea5d23356d90f6..1f84e69463f3c454185320e3976da816c47f9fb2 100644 (file)
--- a/src/plugins/surbl.c
+++ b/src/plugins/surbl.c
@@ -24,6 +24,22 @@
 
 /***MODULE:surbl
  * rspamd module that implements SURBL url checking
+ *
+ * Allowed options:
+ * - metric (string): metric to insert symbol (default: 'default')
+ * - weight (integer): weight of symbol
+ * Redirecotor options:
+ * - redirector (string): address of http redirector utility in format "host:port"
+ * - redirector_connect_timeout (seconds): redirector connect timeout (default: 1s)
+ * - redirector_read_timeout (seconds): timeout for reading data (default: 5s)
+ * - redirector_hosts_map (map string): map that contains domains to check with redirector
+ * Surbl options:
+ * - 2tld (map string): map of domains that should be checked via surbl using 3 (e.g. somehost.domain.com)
+ *   components of domain name instead of normal 2 (e.g. domain.com)
+ * - whitelist (map string): map of domains that should be whitelisted for surbl checks
+ * - max_urls (integer): maximum allowed number of urls in message to be checked
+ * - suffix (string): surbl address (for example insecure-bl.rambler.ru), may contain %b if bits are used (read documentation about it)
+ * - bit (string): describes a prefix for a single bit
  */
 
 #include "../config.h"