Merge branch 'master' into vstakhov-utf8-mime

5 files changed, 103 insertions, 158 deletions

diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c
index 91b77c702..ece9a91e0 100644
--- a/src/plugins/fuzzy_check.c
+++ b/src/plugins/fuzzy_check.c
@@ -49,6 +49,9 @@
 #include "libutil/libev_helper.h"
 
 #define DEFAULT_SYMBOL "R_FUZZY_HASH"
+#define RSPAMD_FUZZY_SYMBOL_FORBIDDEN "FUZZY_FORBIDDEN"
+#define RSPAMD_FUZZY_SYMBOL_RATELIMITED "FUZZY_RATELIMITED"
+#define RSPAMD_FUZZY_SYMBOL_ENCRYPTION_REQUIRED "FUZZY_ENCRYPTION_REQUIRED"
 
 #define DEFAULT_IO_TIMEOUT 1.0
 #define DEFAULT_RETRANSMITS 3
@@ -68,6 +71,12 @@ struct fuzzy_mapping {
 	double weight;
 };
 
+enum fuzzy_rule_mode {
+	fuzzy_rule_read_only,
+	fuzzy_rule_write_only,
+	fuzzy_rule_read_write
+};
+
 struct fuzzy_rule {
 	struct upstream_list *servers;
 	const char *symbol;
@@ -84,7 +93,7 @@ struct fuzzy_rule {
 	struct rspamd_cryptobox_pubkey *peer_key;
 	double max_score;
 	double weight_threshold;
-	gboolean read_only;
+	enum fuzzy_rule_mode mode;
 	gboolean skip_unknown;
 	gboolean no_share;
 	gboolean no_subject;
@@ -328,7 +337,7 @@ fuzzy_rule_new(const char *default_symbol, rspamd_mempool_t *pool)
 	rspamd_mempool_add_destructor(pool,
 								  (rspamd_mempool_destruct_t) g_hash_table_unref,
 								  rule->mappings);
-	rule->read_only = FALSE;
+	rule->mode = fuzzy_rule_read_write;
 	rule->weight_threshold = NAN;
 
 	return rule;
@@ -458,7 +467,26 @@ fuzzy_parse_rule(struct rspamd_config *cfg, const ucl_object_t *obj,
 
 
 	if ((value = ucl_object_lookup(obj, "read_only")) != NULL) {
-		rule->read_only = ucl_obj_toboolean(value);
+		rule->mode = ucl_obj_toboolean(value) ? fuzzy_rule_read_only : fuzzy_rule_read_write;
+	}
+
+	if ((value = ucl_object_lookup(obj, "mode")) != NULL) {
+		const char *mode_str = ucl_object_tostring(value);
+
+		if (g_ascii_strcasecmp(mode_str, "read_only") == 0) {
+			rule->mode = fuzzy_rule_read_only;
+		}
+		else if (g_ascii_strcasecmp(mode_str, "write_only") == 0) {
+			rule->mode = fuzzy_rule_write_only;
+		}
+		else if (g_ascii_strcasecmp(mode_str, "read_write") == 0) {
+			rule->mode = fuzzy_rule_read_write;
+		}
+		else {
+			msg_warn_config("unknown mode: %s, use read_write by default",
+							mode_str);
+			rule->mode = fuzzy_rule_read_write;
+		}
 	}
 
 	if ((value = ucl_object_lookup(obj, "skip_unknown")) != NULL) {
@@ -1153,6 +1181,44 @@ int fuzzy_check_module_config(struct rspamd_config *cfg, bool validate)
 								 1,
 								 1);
 
+		/* Register meta symbols (blocked, ratelimited, etc) */
+		rspamd_symcache_add_symbol(cfg->cache,
+								   RSPAMD_FUZZY_SYMBOL_FORBIDDEN, 0, NULL, NULL,
+								   SYMBOL_TYPE_VIRTUAL,
+								   cb_id);
+		rspamd_config_add_symbol(cfg,
+								 RSPAMD_FUZZY_SYMBOL_FORBIDDEN,
+								 0.0,
+								 "Fuzzy access denied",
+								 "fuzzy",
+								 0,
+								 1,
+								 1);
+		rspamd_symcache_add_symbol(cfg->cache,
+								   RSPAMD_FUZZY_SYMBOL_RATELIMITED, 0, NULL, NULL,
+								   SYMBOL_TYPE_VIRTUAL,
+								   cb_id);
+		rspamd_config_add_symbol(cfg,
+								 RSPAMD_FUZZY_SYMBOL_RATELIMITED,
+								 0.0,
+								 "Fuzzy rate limit is reached",
+								 "fuzzy",
+								 0,
+								 1,
+								 1);
+		rspamd_symcache_add_symbol(cfg->cache,
+								   RSPAMD_FUZZY_SYMBOL_ENCRYPTION_REQUIRED, 0, NULL, NULL,
+								   SYMBOL_TYPE_VIRTUAL,
+								   cb_id);
+		rspamd_config_add_symbol(cfg,
+								 RSPAMD_FUZZY_SYMBOL_ENCRYPTION_REQUIRED,
+								 0.0,
+								 "Fuzzy encryption is required by a server",
+								 "fuzzy",
+								 0,
+								 1,
+								 1);
+
 		/*
 		 * Here we can have 2 possibilities:
 		 *
@@ -2486,7 +2552,16 @@ fuzzy_check_try_read(struct fuzzy_client_session *session)
 				}
 			}
 			else if (rep->v1.value == 403) {
-				rspamd_task_insert_result(task, "FUZZY_BLOCKED", 0.0,
+				/* In fact, it should be 429, but we preserve compatibility */
+				rspamd_task_insert_result(task, RSPAMD_FUZZY_SYMBOL_RATELIMITED, 1.0,
+										  session->rule->name);
+			}
+			else if (rep->v1.value == 503) {
+				rspamd_task_insert_result(task, RSPAMD_FUZZY_SYMBOL_FORBIDDEN, 1.0,
+										  session->rule->name);
+			}
+			else if (rep->v1.value == 415) {
+				rspamd_task_insert_result(task, RSPAMD_FUZZY_SYMBOL_ENCRYPTION_REQUIRED, 1.0,
 										  session->rule->name);
 			}
 			else if (rep->v1.value == 401) {
@@ -3400,11 +3475,14 @@ fuzzy_symbol_callback(struct rspamd_task *task,
 
 	PTR_ARRAY_FOREACH(fuzzy_module_ctx->fuzzy_rules, i, rule)
 	{
-		commands = fuzzy_generate_commands(task, rule, FUZZY_CHECK, 0, 0, 0);
+		if (rule->mode != fuzzy_rule_write_only) {
+			commands = fuzzy_generate_commands(task, rule, FUZZY_CHECK, 0, 0, 0);
 
-		if (commands != NULL) {
-			register_fuzzy_client_call(task, rule, commands);
+			if (commands != NULL) {
+				register_fuzzy_client_call(task, rule, commands);
+			}
 		}
+		/* Skip write only rules from checks */
 	}
 
 	rspamd_symcache_item_async_dec_check(task, item, M);
@@ -3491,9 +3569,9 @@ register_fuzzy_controller_call(struct rspamd_http_connection_entry *entry,
 }
 
 static void
-fuzzy_process_handler(struct rspamd_http_connection_entry *conn_ent,
-					  struct rspamd_http_message *msg, int cmd, int value, int flag,
-					  struct fuzzy_ctx *ctx, gboolean is_hash, unsigned int flags)
+fuzzy_modify_handler(struct rspamd_http_connection_entry *conn_ent,
+					 struct rspamd_http_message *msg, int cmd, int value, int flag,
+					 struct fuzzy_ctx *ctx, gboolean is_hash, unsigned int flags)
 {
 	struct fuzzy_rule *rule;
 	struct rspamd_controller_session *session = conn_ent->ud;
@@ -3541,7 +3619,7 @@ fuzzy_process_handler(struct rspamd_http_connection_entry *conn_ent,
 
 	PTR_ARRAY_FOREACH(fuzzy_module_ctx->fuzzy_rules, i, rule)
 	{
-		if (rule->read_only) {
+		if (rule->mode == fuzzy_rule_read_only) {
 			continue;
 		}
 
@@ -3796,8 +3874,8 @@ fuzzy_controller_handler(struct rspamd_http_connection_entry *conn_ent,
 		send_flags |= FUZZY_CHECK_FLAG_NOTEXT;
 	}
 
-	fuzzy_process_handler(conn_ent, msg, cmd, value, flag,
-						  (struct fuzzy_ctx *) ctx, is_hash, send_flags);
+	fuzzy_modify_handler(conn_ent, msg, cmd, value, flag,
+						 (struct fuzzy_ctx *) ctx, is_hash, send_flags);
 
 	return 0;
 }
@@ -3879,7 +3957,7 @@ fuzzy_check_lua_process_learn(struct rspamd_task *task,
 		if (!res) {
 			break;
 		}
-		if (rule->read_only) {
+		if (rule->mode == fuzzy_rule_read_only) {
 			continue;
 		}
 
@@ -4181,7 +4259,7 @@ fuzzy_lua_gen_hashes_handler(lua_State *L)
 
 	PTR_ARRAY_FOREACH(fuzzy_module_ctx->fuzzy_rules, i, rule)
 	{
-		if (rule->read_only) {
+		if (rule->mode == fuzzy_rule_read_only) {
 			continue;
 		}
 
@@ -4409,7 +4487,7 @@ fuzzy_lua_list_storages(lua_State *L)
 	{
 		lua_newtable(L);
 
-		lua_pushboolean(L, rule->read_only);
+		lua_pushboolean(L, rule->mode == fuzzy_rule_read_only);
 		lua_setfield(L, -2, "read_only");
 
 		/* Push servers */
diff --git a/src/plugins/lua/aws_s3.lua b/src/plugins/lua/aws_s3.lua
index 30e88d2cd..ac344d86c 100644
--- a/src/plugins/lua/aws_s3.lua
+++ b/src/plugins/lua/aws_s3.lua
@@ -238,7 +238,7 @@ settings = lua_util.override_defaults(settings, opts)
 local res, err = settings_schema:transform(settings)
 
 if not res then
-  rspamd_logger.warnx(rspamd_config, 'plugin is misconfigured: %s', err)
+  rspamd_logger.warnx(rspamd_config, 'plugin %s is misconfigured: %s', N, err)
   lua_util.disable_module(N, "config")
   return
 end
diff --git a/src/plugins/lua/bimi.lua b/src/plugins/lua/bimi.lua
index 278359069..78949a5c0 100644
--- a/src/plugins/lua/bimi.lua
+++ b/src/plugins/lua/bimi.lua
@@ -265,7 +265,7 @@ local function check_bimi_vmc(task, domain, record)
         end
         if redis_params.username then
           if redis_params.password then
-            password = string.format( '%s:%s@', redis_params.username, redis_params.password)
+            password = string.format('%s:%s@', redis_params.username, redis_params.password)
           else
             rspamd_logger.warnx(task, "Redis requires a password when username is supplied")
           end
@@ -358,7 +358,7 @@ settings = lua_util.override_defaults(settings, opts)
 local res, err = settings_schema:transform(settings)
 
 if not res then
-  rspamd_logger.warnx(rspamd_config, 'plugin is misconfigured: %s', err)
+  rspamd_logger.warnx(rspamd_config, 'plugin %s is misconfigured: %s', N, err)
   local err_msg = string.format("schema error: %s", res)
   lua_util.config_utils.push_config_error(N, err_msg)
   lua_util.disable_module(N, "failed", err_msg)
diff --git a/src/plugins/lua/history_redis.lua b/src/plugins/lua/history_redis.lua
index fff9f46b3..a3fdb0ec4 100644
--- a/src/plugins/lua/history_redis.lua
+++ b/src/plugins/lua/history_redis.lua
@@ -281,7 +281,7 @@ if opts then
   local res, err = settings_schema:transform(settings)
 
   if not res then
-    rspamd_logger.warnx(rspamd_config, '%s: plugin is misconfigured: %s', N, err)
+    rspamd_logger.warnx(rspamd_config, 'plugin %s is misconfigured: %s', N, err)
     lua_util.disable_module(N, "config")
     return
   end
diff --git a/src/plugins/lua/ratelimit.lua b/src/plugins/lua/ratelimit.lua
index f3331e850..168d8d63a 100644
--- a/src/plugins/lua/ratelimit.lua
+++ b/src/plugins/lua/ratelimit.lua
@@ -29,8 +29,7 @@ local lua_util = require "lua_util"
 local lua_verdict = require "lua_verdict"
 local rspamd_hash = require "rspamd_cryptobox_hash"
 local lua_selectors = require "lua_selectors"
-local ts = require("tableshape").types
-
+local ratelimit_common = require "plugins/ratelimit"
 -- A plugin that implements ratelimits using redis
 
 local E = {}
@@ -76,138 +75,6 @@ local function load_scripts(_, _)
   bucket_cleanup_id = lua_redis.load_redis_script_from_file(bucket_cleanup_script, redis_params)
 end
 
-local limit_parser
-local function parse_string_limit(lim, no_error)
-  local function parse_time_suffix(s)
-    if s == 's' then
-      return 1
-    elseif s == 'm' then
-      return 60
-    elseif s == 'h' then
-      return 3600
-    elseif s == 'd' then
-      return 86400
-    end
-  end
-  local function parse_num_suffix(s)
-    if s == '' then
-      return 1
-    elseif s == 'k' then
-      return 1000
-    elseif s == 'm' then
-      return 1000000
-    elseif s == 'g' then
-      return 1000000000
-    end
-  end
-  local lpeg = require "lpeg"
-
-  if not limit_parser then
-    local digit = lpeg.R("09")
-    limit_parser = {}
-    limit_parser.integer = (lpeg.S("+-") ^ -1) *
-        (digit ^ 1)
-    limit_parser.fractional = (lpeg.P(".")) *
-        (digit ^ 1)
-    limit_parser.number = (limit_parser.integer *
-        (limit_parser.fractional ^ -1)) +
-        (lpeg.S("+-") * limit_parser.fractional)
-    limit_parser.time = lpeg.Cf(lpeg.Cc(1) *
-        (limit_parser.number / tonumber) *
-        ((lpeg.S("smhd") / parse_time_suffix) ^ -1),
-        function(acc, val)
-          return acc * val
-        end)
-    limit_parser.suffixed_number = lpeg.Cf(lpeg.Cc(1) *
-        (limit_parser.number / tonumber) *
-        ((lpeg.S("kmg") / parse_num_suffix) ^ -1),
-        function(acc, val)
-          return acc * val
-        end)
-    limit_parser.limit = lpeg.Ct(limit_parser.suffixed_number *
-        (lpeg.S(" ") ^ 0) * lpeg.S("/") * (lpeg.S(" ") ^ 0) *
-        limit_parser.time)
-  end
-  local t = lpeg.match(limit_parser.limit, lim)
-
-  if t and t[1] and t[2] and t[2] ~= 0 then
-    return t[2], t[1]
-  end
-
-  if not no_error then
-    rspamd_logger.errx(rspamd_config, 'bad limit: %s', lim)
-  end
-
-  return nil
-end
-
-local function str_to_rate(str)
-  local divider, divisor = parse_string_limit(str, false)
-
-  if not divisor then
-    rspamd_logger.errx(rspamd_config, 'bad rate string: %s', str)
-
-    return nil
-  end
-
-  return divisor / divider
-end
-
-local bucket_schema = ts.shape {
-  burst = ts.number + ts.string / lua_util.dehumanize_number,
-  rate = ts.number + ts.string / str_to_rate,
-  skip_recipients = ts.boolean:is_optional(),
-  symbol = ts.string:is_optional(),
-  message = ts.string:is_optional(),
-  skip_soft_reject = ts.boolean:is_optional(),
-}
-
-local function parse_limit(name, data)
-  if type(data) == 'table' then
-    -- 2 cases here:
-    --  * old limit in format [burst, rate]
-    --  * vector of strings in Andrew's string format (removed from 1.8.2)
-    --  * proper bucket table
-    if #data == 2 and tonumber(data[1]) and tonumber(data[2]) then
-      -- Old style ratelimit
-      rspamd_logger.warnx(rspamd_config, 'old style ratelimit for %s', name)
-      if tonumber(data[1]) > 0 and tonumber(data[2]) > 0 then
-        return {
-          burst = data[1],
-          rate = data[2]
-        }
-      elseif data[1] ~= 0 then
-        rspamd_logger.warnx(rspamd_config, 'invalid numbers for %s', name)
-      else
-        rspamd_logger.infox(rspamd_config, 'disable limit %s, burst is zero', name)
-      end
-
-      return nil
-    else
-      local parsed_bucket, err = bucket_schema:transform(data)
-
-      if not parsed_bucket or err then
-        rspamd_logger.errx(rspamd_config, 'cannot parse bucket for %s: %s; original value: %s',
-            name, err, data)
-      else
-        return parsed_bucket
-      end
-    end
-  elseif type(data) == 'string' then
-    local rep_rate, burst = parse_string_limit(data)
-    rspamd_logger.warnx(rspamd_config, 'old style rate bucket config detected for %s: %s',
-        name, data)
-    if rep_rate and burst then
-      return {
-        burst = burst,
-        rate = burst / rep_rate -- reciprocal
-      }
-    end
-  end
-
-  return nil
-end
-
 --- Check whether this addr is bounce
 local function check_bounce(from)
   return fun.any(function(b)
@@ -490,7 +357,7 @@ local function ratelimit_cb(task)
     local ret, redis_key, bd = pcall(hdl, task)
 
     if ret then
-      local bucket = parse_limit(k, bd)
+      local bucket = ratelimit_common.parse_limit(k, bd)
       if bucket then
         prefixes[redis_key] = make_prefix(redis_key, k, bucket)
       end
@@ -718,7 +585,7 @@ if opts then
 
         if lim.bucket[1] then
           for _, bucket in ipairs(lim.bucket) do
-            local b = parse_limit(t, bucket)
+            local b = ratelimit_common.parse_limit(t, bucket)
 
             if not b then
               rspamd_logger.errx(rspamd_config, 'bad ratelimit bucket for %s: "%s"',
@@ -729,7 +596,7 @@ if opts then
             table.insert(buckets, b)
           end
         else
-          local bucket = parse_limit(t, lim.bucket)
+          local bucket = ratelimit_common.parse_limit(t, lim.bucket)
 
           if not bucket then
             rspamd_logger.errx(rspamd_config, 'bad ratelimit bucket for %s: "%s"',
@@ -757,7 +624,7 @@ if opts then
         end
       else
         rspamd_logger.warnx(rspamd_config, 'old syntax for ratelimits: %s', lim)
-        buckets = parse_limit(t, lim)
+        buckets = ratelimit_common.parse_limit(t, lim)
         if buckets then
           settings.limits[t] = {
             buckets = { buckets }