diff options
-rw-r--r-- | lualib/lua_bayes_learn.lua | 7 | ||||
-rw-r--r-- | lualib/lua_redis.lua | 10 | ||||
-rw-r--r-- | lualib/plugins/dmarc.lua | 8 | ||||
-rw-r--r-- | lualib/plugins/ratelimit.lua | 155 | ||||
-rw-r--r-- | package-lock.json | 417 | ||||
-rw-r--r-- | package.json | 6 | ||||
-rw-r--r-- | src/fuzzy_storage.c | 328 | ||||
-rw-r--r-- | src/lua/lua_cryptobox.c | 20 | ||||
-rw-r--r-- | src/lua/lua_rsa.c | 18 | ||||
-rw-r--r-- | src/plugins/fuzzy_check.c | 110 | ||||
-rw-r--r-- | src/plugins/lua/aws_s3.lua | 2 | ||||
-rw-r--r-- | src/plugins/lua/bimi.lua | 4 | ||||
-rw-r--r-- | src/plugins/lua/history_redis.lua | 2 | ||||
-rw-r--r-- | src/plugins/lua/ratelimit.lua | 143 |
14 files changed, 736 insertions, 494 deletions
diff --git a/lualib/lua_bayes_learn.lua b/lualib/lua_bayes_learn.lua index 82f044d7d..89470edba 100644 --- a/lualib/lua_bayes_learn.lua +++ b/lualib/lua_bayes_learn.lua @@ -76,6 +76,13 @@ exports.autolearn = function(task, conf) mime_rcpts) end + if not task:get_queue_id() then + -- We should skip messages that come from `rspamc` or webui as they are usually + -- not intended for autolearn at all + lua_util.debugm(N, task, 'no need to autolearn - queue id is missing') + return + end + -- We have autolearn config so let's figure out what is requested local verdict, score = lua_verdict.get_specific_verdict("bayes", task) local learn_spam, learn_ham = false, false diff --git a/lualib/lua_redis.lua b/lualib/lua_redis.lua index 2c77c100a..43acfee65 100644 --- a/lualib/lua_redis.lua +++ b/lualib/lua_redis.lua @@ -24,11 +24,12 @@ local exports = {} local E = {} local N = "lua_redis" +local db_schema = (ts.number / tostring + ts.string):is_optional():describe("Database number") local common_schema = { timeout = (ts.number + ts.string / lutil.parse_time_interval):is_optional():describe("Connection timeout"), - db = ts.string:is_optional():describe("Database number"), - database = ts.string:is_optional():describe("Database number"), - dbname = ts.string:is_optional():describe("Database number"), + db = db_schema, + database = db_schema, + dbname = db_schema, prefix = ts.string:is_optional():describe("Key prefix"), username = ts.string:is_optional():describe("Username"), password = ts.string:is_optional():describe("Password"), @@ -64,7 +65,8 @@ local server_schema = lutil.table_merge({ local enrich_schema = function(external) return ts.one_of { - ts.shape(external), -- no specific redis parameters + ts.shape(lutil.table_merge(common_schema, + external)), -- no specific redis servers (e.g when global settings are used) ts.shape(lutil.table_merge(read_schema, external)), -- read_servers specified ts.shape(lutil.table_merge(write_schema, external)), -- write_servers specified ts.shape(lutil.table_merge(rw_schema, external)), -- both read and write servers defined diff --git a/lualib/plugins/dmarc.lua b/lualib/plugins/dmarc.lua index 38977ff58..720c76e16 100644 --- a/lualib/plugins/dmarc.lua +++ b/lualib/plugins/dmarc.lua @@ -239,12 +239,12 @@ local function gen_dmarc_grammar() local lpeg = require "lpeg" lpeg.locale(lpeg) local space = lpeg.space ^ 0 - local name = lpeg.C(lpeg.alpha ^ 1) * space - local sep = space * (lpeg.S("\\;") * space) + (lpeg.P(lpeg.graph - lpeg.P(',')) * lpeg.space ^ 1) - local value = lpeg.C(lpeg.P(lpeg.P(lpeg.graph + lpeg.space) - sep) ^ 1) + local name = lpeg.C(lpeg.alpha ^ 1) + local sep = space * lpeg.S("\\;") * space + local value = lpeg.C(((lpeg.space ^ 1 * lpeg.graph + lpeg.graph) - sep) ^ 1) local pair = lpeg.Cg(name * "=" * space * value) * sep ^ -1 local list = lpeg.Cf(lpeg.Ct("") * pair ^ 0, rawset) - local version = lpeg.P("v") * space * lpeg.P("=") * space * lpeg.P("DMARC1") + local version = "v" * space * "=" * space * "DMARC1" local record = version * sep * list return record diff --git a/lualib/plugins/ratelimit.lua b/lualib/plugins/ratelimit.lua new file mode 100644 index 000000000..24afed1f8 --- /dev/null +++ b/lualib/plugins/ratelimit.lua @@ -0,0 +1,155 @@ +--[[ +Copyright (c) 2024, Vsevolod Stakhov <vsevolod@rspamd.com> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +local rspamd_logger = require "rspamd_logger" +local lua_util = require "lua_util" +local ts = require("tableshape").types + +local exports = {} + +local limit_parser +local function parse_string_limit(lim, no_error) + local function parse_time_suffix(s) + if s == 's' then + return 1 + elseif s == 'm' then + return 60 + elseif s == 'h' then + return 3600 + elseif s == 'd' then + return 86400 + end + end + local function parse_num_suffix(s) + if s == '' then + return 1 + elseif s == 'k' then + return 1000 + elseif s == 'm' then + return 1000000 + elseif s == 'g' then + return 1000000000 + end + end + local lpeg = require "lpeg" + + if not limit_parser then + local digit = lpeg.R("09") + limit_parser = {} + limit_parser.integer = (lpeg.S("+-") ^ -1) * + (digit ^ 1) + limit_parser.fractional = (lpeg.P(".")) * + (digit ^ 1) + limit_parser.number = (limit_parser.integer * + (limit_parser.fractional ^ -1)) + + (lpeg.S("+-") * limit_parser.fractional) + limit_parser.time = lpeg.Cf(lpeg.Cc(1) * + (limit_parser.number / tonumber) * + ((lpeg.S("smhd") / parse_time_suffix) ^ -1), + function(acc, val) + return acc * val + end) + limit_parser.suffixed_number = lpeg.Cf(lpeg.Cc(1) * + (limit_parser.number / tonumber) * + ((lpeg.S("kmg") / parse_num_suffix) ^ -1), + function(acc, val) + return acc * val + end) + limit_parser.limit = lpeg.Ct(limit_parser.suffixed_number * + (lpeg.S(" ") ^ 0) * lpeg.S("/") * (lpeg.S(" ") ^ 0) * + limit_parser.time) + end + local t = lpeg.match(limit_parser.limit, lim) + + if t and t[1] and t[2] and t[2] ~= 0 then + return t[2], t[1] + end + + if not no_error then + rspamd_logger.errx(rspamd_config, 'bad limit: %s', lim) + end + + return nil +end + +local function str_to_rate(str) + local divider, divisor = parse_string_limit(str, false) + + if not divisor then + rspamd_logger.errx(rspamd_config, 'bad rate string: %s', str) + + return nil + end + + return divisor / divider +end + +local bucket_schema = ts.shape { + burst = ts.number + ts.string / lua_util.dehumanize_number, + rate = ts.number + ts.string / str_to_rate, + skip_recipients = ts.boolean:is_optional(), + symbol = ts.string:is_optional(), + message = ts.string:is_optional(), + skip_soft_reject = ts.boolean:is_optional(), +} + +exports.parse_limit = function(name, data) + if type(data) == 'table' then + -- 2 cases here: + -- * old limit in format [burst, rate] + -- * vector of strings in Andrew's string format (removed from 1.8.2) + -- * proper bucket table + if #data == 2 and tonumber(data[1]) and tonumber(data[2]) then + -- Old style ratelimit + rspamd_logger.warnx(rspamd_config, 'old style ratelimit for %s', name) + if tonumber(data[1]) > 0 and tonumber(data[2]) > 0 then + return { + burst = data[1], + rate = data[2] + } + elseif data[1] ~= 0 then + rspamd_logger.warnx(rspamd_config, 'invalid numbers for %s', name) + else + rspamd_logger.infox(rspamd_config, 'disable limit %s, burst is zero', name) + end + + return nil + else + local parsed_bucket, err = bucket_schema:transform(data) + + if not parsed_bucket or err then + rspamd_logger.errx(rspamd_config, 'cannot parse bucket for %s: %s; original value: %s', + name, err, data) + else + return parsed_bucket + end + end + elseif type(data) == 'string' then + local rep_rate, burst = parse_string_limit(data) + rspamd_logger.warnx(rspamd_config, 'old style rate bucket config detected for %s: %s', + name, data) + if rep_rate and burst then + return { + burst = burst, + rate = burst / rep_rate -- reciprocal + } + end + end + + return nil +end + +return exports
\ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 67504a812..3c2f6796d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,10 +5,10 @@ "packages": { "": { "devDependencies": { - "@stylistic/eslint-plugin": "*", - "eslint": "^9.7.0", + "@stylistic/eslint-plugin": "^2.8.0", + "eslint": "^9.10.0", "postcss-html": "*", - "stylelint": ">=13.6.0", + "stylelint": ">=16.9.0", "stylelint-config-standard": "*" } }, @@ -127,9 +127,9 @@ } }, "node_modules/@csstools/css-parser-algorithms": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-2.7.1.tgz", - "integrity": "sha512-2SJS42gxmACHgikc1WGesXLIT8d/q2l0UFM7TaEeIzdFCE/FPMtTiizcPGGJtlPo2xuQzY09OhrLTzRxqJqwGw==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-3.0.1.tgz", + "integrity": "sha512-lSquqZCHxDfuTg/Sk2hiS0mcSFCEBuj49JfzPHJogDBT0mGCyY5A1AQzBWngitrp7i1/HAZpIgzF/VjhOEIJIg==", "dev": true, "funding": [ { @@ -141,17 +141,18 @@ "url": "https://opencollective.com/csstools" } ], + "license": "MIT", "engines": { - "node": "^14 || ^16 || >=18" + "node": ">=18" }, "peerDependencies": { - "@csstools/css-tokenizer": "^2.4.1" + "@csstools/css-tokenizer": "^3.0.1" } }, "node_modules/@csstools/css-tokenizer": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-2.4.1.tgz", - "integrity": "sha512-eQ9DIktFJBhGjioABJRtUucoWR2mwllurfnM8LuNGAqX3ViZXaUchqk+1s7jjtkFiT9ySdACsFEA3etErkALUg==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-3.0.1.tgz", + "integrity": "sha512-UBqaiu7kU0lfvaP982/o3khfXccVlHPWp0/vwwiIgDF0GmqqqxoiXC/6FCjlS9u92f7CoEz6nXKQnrn1kIAkOw==", "dev": true, "funding": [ { @@ -163,14 +164,15 @@ "url": "https://opencollective.com/csstools" } ], + "license": "MIT", "engines": { - "node": "^14 || ^16 || >=18" + "node": ">=18" } }, "node_modules/@csstools/media-query-list-parser": { - "version": "2.1.13", - "resolved": "https://registry.npmjs.org/@csstools/media-query-list-parser/-/media-query-list-parser-2.1.13.tgz", - "integrity": "sha512-XaHr+16KRU9Gf8XLi3q8kDlI18d5vzKSKCY510Vrtc9iNR0NJzbY9hhTmwhzYZj/ZwGL4VmB3TA9hJW0Um2qFA==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@csstools/media-query-list-parser/-/media-query-list-parser-3.0.1.tgz", + "integrity": "sha512-HNo8gGD02kHmcbX6PvCoUuOQvn4szyB9ca63vZHKX5A81QytgDG4oxG4IaEfHTlEZSZ6MjPEMWIVU+zF2PZcgw==", "dev": true, "funding": [ { @@ -182,18 +184,19 @@ "url": "https://opencollective.com/csstools" } ], + "license": "MIT", "engines": { - "node": "^14 || ^16 || >=18" + "node": ">=18" }, "peerDependencies": { - "@csstools/css-parser-algorithms": "^2.7.1", - "@csstools/css-tokenizer": "^2.4.1" + "@csstools/css-parser-algorithms": "^3.0.1", + "@csstools/css-tokenizer": "^3.0.1" } }, "node_modules/@csstools/selector-specificity": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@csstools/selector-specificity/-/selector-specificity-3.1.1.tgz", - "integrity": "sha512-a7cxGcJ2wIlMFLlh8z2ONm+715QkPHiyJcxwQlKOz/03GPw1COpfhcmC9wm4xlZfp//jWHNNMwzjtqHXVWU9KA==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@csstools/selector-specificity/-/selector-specificity-4.0.0.tgz", + "integrity": "sha512-189nelqtPd8++phaHNwYovKZI0FOzH1vQEE3QhHHkNIGrg5fSs9CbYP3RvfEH5geztnIA9Jwq91wyOIwAW5JIQ==", "dev": true, "funding": [ { @@ -205,11 +208,12 @@ "url": "https://opencollective.com/csstools" } ], + "license": "MIT-0", "engines": { - "node": "^14 || ^16 || >=18" + "node": ">=18" }, "peerDependencies": { - "postcss-selector-parser": "^6.0.13" + "postcss-selector-parser": "^6.1.0" } }, "node_modules/@dual-bundle/import-meta-resolve": { @@ -259,10 +263,11 @@ } }, "node_modules/@eslint/config-array": { - "version": "0.17.0", - "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.17.0.tgz", - "integrity": "sha512-A68TBu6/1mHHuc5YJL0U0VVeGNiklLAL6rRmhTCP2B5XjWLMnrX+HkO+IAXyHvks5cyyY1jjK5ITPQ1HGS2EVA==", + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.18.0.tgz", + "integrity": "sha512-fTxvnS1sRMu3+JjXwJG0j/i4RT9u4qJ+lqS/yCGap4lH4zZGzQ7tu+xZqQmcMZq5OBZDL4QRxQzRjkWcGt8IVw==", "dev": true, + "license": "Apache-2.0", "dependencies": { "@eslint/object-schema": "^2.1.4", "debug": "^4.3.1", @@ -296,10 +301,11 @@ } }, "node_modules/@eslint/js": { - "version": "9.7.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.7.0.tgz", - "integrity": "sha512-ChuWDQenef8OSFnvuxv0TCVxEwmu3+hPNKvM9B34qpM0rDRbjL8t5QkQeHHeAfsKQjuH9wS82WeCi1J/owatng==", + "version": "9.10.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.10.0.tgz", + "integrity": "sha512-fuXtbiP5GWIn8Fz+LWoOMVf/Jxm+aajZYkhi6CuEm4SxymFM+eUWzbO9qXT+L0iCkL5+KGYMCSGxo686H19S1g==", "dev": true, + "license": "MIT", "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } @@ -309,6 +315,20 @@ "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-2.1.4.tgz", "integrity": "sha512-BsWiH1yFGjXXS2yvrf5LyuoSIIbPrGUWob917o+BTKuZ7qJdxX8aJLRxs1fS9n6r7vESrq1OUqb68dANcFXuQQ==", "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/plugin-kit": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.1.0.tgz", + "integrity": "sha512-autAXT203ixhqei9xt+qkYOvY8l6LAFIdT2UXc/RPNeUVfqRF1BV94GTJyVPFKT8nFM6MyVJhjLj9E8JWvf5zQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "levn": "^0.4.1" + }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } @@ -375,50 +395,15 @@ } }, "node_modules/@stylistic/eslint-plugin": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@stylistic/eslint-plugin/-/eslint-plugin-2.3.0.tgz", - "integrity": "sha512-rtiz6u5gRyyEZp36FcF1/gHJbsbT3qAgXZ1qkad6Nr/xJ9wrSJkiSFFQhpYVTIZ7FJNRJurEcumZDCwN9dEI4g==", + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@stylistic/eslint-plugin/-/eslint-plugin-2.8.0.tgz", + "integrity": "sha512-Ufvk7hP+bf+pD35R/QfunF793XlSRIC7USr3/EdgduK9j13i2JjmsM0LUz3/foS+jDYp2fzyWZA9N44CPur0Ow==", "dev": true, + "license": "MIT", "dependencies": { - "@stylistic/eslint-plugin-js": "2.3.0", - "@stylistic/eslint-plugin-jsx": "2.3.0", - "@stylistic/eslint-plugin-plus": "2.3.0", - "@stylistic/eslint-plugin-ts": "2.3.0", - "@types/eslint": "^8.56.10" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "peerDependencies": { - "eslint": ">=8.40.0" - } - }, - "node_modules/@stylistic/eslint-plugin-js": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@stylistic/eslint-plugin-js/-/eslint-plugin-js-2.3.0.tgz", - "integrity": "sha512-lQwoiYb0Fs6Yc5QS3uT8+T9CPKK2Eoxc3H8EnYJgM26v/DgtW+1lvy2WNgyBflU+ThShZaHm3a6CdD9QeKx23w==", - "dev": true, - "dependencies": { - "@types/eslint": "^8.56.10", - "acorn": "^8.11.3", + "@typescript-eslint/utils": "^8.4.0", "eslint-visitor-keys": "^4.0.0", - "espree": "^10.0.1" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "peerDependencies": { - "eslint": ">=8.40.0" - } - }, - "node_modules/@stylistic/eslint-plugin-jsx": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@stylistic/eslint-plugin-jsx/-/eslint-plugin-jsx-2.3.0.tgz", - "integrity": "sha512-tsQ0IEKB195H6X9A4iUSgLLLKBc8gUBWkBIU8tp1/3g2l8stu+PtMQVV/VmK1+3bem5FJCyvfcZIQ/WF1fsizA==", - "dev": true, - "dependencies": { - "@stylistic/eslint-plugin-js": "^2.3.0", - "@types/eslint": "^8.56.10", + "espree": "^10.1.0", "estraverse": "^5.3.0", "picomatch": "^4.0.2" }, @@ -429,113 +414,18 @@ "eslint": ">=8.40.0" } }, - "node_modules/@stylistic/eslint-plugin-plus": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@stylistic/eslint-plugin-plus/-/eslint-plugin-plus-2.3.0.tgz", - "integrity": "sha512-xboPWGUU5yaPlR+WR57GwXEuY4PSlPqA0C3IdNA/+1o2MuBi95XgDJcZiJ9N+aXsqBXAPIpFFb+WQ7QEHo4f7g==", - "dev": true, - "dependencies": { - "@types/eslint": "^8.56.10", - "@typescript-eslint/utils": "^7.12.0" - }, - "peerDependencies": { - "eslint": "*" - } - }, - "node_modules/@stylistic/eslint-plugin-plus/node_modules/@typescript-eslint/utils": { - "version": "7.16.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-7.16.1.tgz", - "integrity": "sha512-WrFM8nzCowV0he0RlkotGDujx78xudsxnGMBHI88l5J8wEhED6yBwaSLP99ygfrzAjsQvcYQ94quDwI0d7E1fA==", - "dev": true, - "dependencies": { - "@eslint-community/eslint-utils": "^4.4.0", - "@typescript-eslint/scope-manager": "7.16.1", - "@typescript-eslint/types": "7.16.1", - "@typescript-eslint/typescript-estree": "7.16.1" - }, - "engines": { - "node": "^18.18.0 || >=20.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^8.56.0" - } - }, - "node_modules/@stylistic/eslint-plugin-ts": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@stylistic/eslint-plugin-ts/-/eslint-plugin-ts-2.3.0.tgz", - "integrity": "sha512-wqOR38/uz/0XPnHX68ftp8sNMSAqnYGjovOTN7w00xnjS6Lxr3Sk7q6AaxWWqbMvOj7V2fQiMC5HWAbTruJsCg==", - "dev": true, - "dependencies": { - "@stylistic/eslint-plugin-js": "2.3.0", - "@types/eslint": "^8.56.10", - "@typescript-eslint/utils": "^7.12.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "peerDependencies": { - "eslint": ">=8.40.0" - } - }, - "node_modules/@stylistic/eslint-plugin-ts/node_modules/@typescript-eslint/utils": { - "version": "7.16.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-7.16.1.tgz", - "integrity": "sha512-WrFM8nzCowV0he0RlkotGDujx78xudsxnGMBHI88l5J8wEhED6yBwaSLP99ygfrzAjsQvcYQ94quDwI0d7E1fA==", - "dev": true, - "dependencies": { - "@eslint-community/eslint-utils": "^4.4.0", - "@typescript-eslint/scope-manager": "7.16.1", - "@typescript-eslint/types": "7.16.1", - "@typescript-eslint/typescript-estree": "7.16.1" - }, - "engines": { - "node": "^18.18.0 || >=20.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^8.56.0" - } - }, - "node_modules/@types/eslint": { - "version": "8.56.10", - "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.56.10.tgz", - "integrity": "sha512-Shavhk87gCtY2fhXDctcfS3e6FdxWkCx1iUZ9eEUbh7rTqlZT0/IzOkCOVt0fCjcFuZ9FPYfuezTBImfHCDBGQ==", - "dev": true, - "dependencies": { - "@types/estree": "*", - "@types/json-schema": "*" - } - }, - "node_modules/@types/estree": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz", - "integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==", - "dev": true - }, - "node_modules/@types/json-schema": { - "version": "7.0.15", - "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", - "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", - "dev": true - }, "node_modules/@typescript-eslint/scope-manager": { - "version": "7.16.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-7.16.1.tgz", - "integrity": "sha512-nYpyv6ALte18gbMz323RM+vpFpTjfNdyakbf3nsLvF43uF9KeNC289SUEW3QLZ1xPtyINJ1dIsZOuWuSRIWygw==", + "version": "8.5.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.5.0.tgz", + "integrity": "sha512-06JOQ9Qgj33yvBEx6tpC8ecP9o860rsR22hWMEd12WcTRrfaFgHr2RB/CA/B+7BMhHkXT4chg2MyboGdFGawYg==", "dev": true, + "license": "MIT", "dependencies": { - "@typescript-eslint/types": "7.16.1", - "@typescript-eslint/visitor-keys": "7.16.1" + "@typescript-eslint/types": "8.5.0", + "@typescript-eslint/visitor-keys": "8.5.0" }, "engines": { - "node": "^18.18.0 || >=20.0.0" + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "type": "opencollective", @@ -543,12 +433,13 @@ } }, "node_modules/@typescript-eslint/types": { - "version": "7.16.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-7.16.1.tgz", - "integrity": "sha512-AQn9XqCzUXd4bAVEsAXM/Izk11Wx2u4H3BAfQVhSfzfDOm/wAON9nP7J5rpkCxts7E5TELmN845xTUCQrD1xIQ==", + "version": "8.5.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.5.0.tgz", + "integrity": "sha512-qjkormnQS5wF9pjSi6q60bKUHH44j2APxfh9TQRXK8wbYVeDYYdYJGIROL87LGZZ2gz3Rbmjc736qyL8deVtdw==", "dev": true, + "license": "MIT", "engines": { - "node": "^18.18.0 || >=20.0.0" + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "type": "opencollective", @@ -556,22 +447,23 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "7.16.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-7.16.1.tgz", - "integrity": "sha512-0vFPk8tMjj6apaAZ1HlwM8w7jbghC8jc1aRNJG5vN8Ym5miyhTQGMqU++kuBFDNKe9NcPeZ6x0zfSzV8xC1UlQ==", + "version": "8.5.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.5.0.tgz", + "integrity": "sha512-vEG2Sf9P8BPQ+d0pxdfndw3xIXaoSjliG0/Ejk7UggByZPKXmJmw3GW5jV2gHNQNawBUyfahoSiCFVov0Ruf7Q==", "dev": true, + "license": "BSD-2-Clause", "dependencies": { - "@typescript-eslint/types": "7.16.1", - "@typescript-eslint/visitor-keys": "7.16.1", + "@typescript-eslint/types": "8.5.0", + "@typescript-eslint/visitor-keys": "8.5.0", "debug": "^4.3.4", - "globby": "^11.1.0", + "fast-glob": "^3.3.2", "is-glob": "^4.0.3", "minimatch": "^9.0.4", "semver": "^7.6.0", "ts-api-utils": "^1.3.0" }, "engines": { - "node": "^18.18.0 || >=20.0.0" + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "type": "opencollective", @@ -588,6 +480,7 @@ "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", "dev": true, + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" } @@ -597,6 +490,7 @@ "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", "dev": true, + "license": "ISC", "dependencies": { "brace-expansion": "^2.0.1" }, @@ -607,17 +501,41 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/@typescript-eslint/utils": { + "version": "8.5.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.5.0.tgz", + "integrity": "sha512-6yyGYVL0e+VzGYp60wvkBHiqDWOpT63pdMV2CVG4LVDd5uR6q1qQN/7LafBZtAtNIn/mqXjsSeS5ggv/P0iECw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.4.0", + "@typescript-eslint/scope-manager": "8.5.0", + "@typescript-eslint/types": "8.5.0", + "@typescript-eslint/typescript-estree": "8.5.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0" + } + }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "7.16.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-7.16.1.tgz", - "integrity": "sha512-Qlzzx4sE4u3FsHTPQAAQFJFNOuqtuY0LFrZHwQ8IHK705XxBiWOFkfKRWu6niB7hwfgnwIpO4jTC75ozW1PHWg==", + "version": "8.5.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.5.0.tgz", + "integrity": "sha512-yTPqMnbAZJNy2Xq2XU8AdtOW9tJIr+UQb64aXB9f3B1498Zx9JorVgFJcZpEc9UBuCCrdzKID2RGAMkYcDtZOw==", "dev": true, + "license": "MIT", "dependencies": { - "@typescript-eslint/types": "7.16.1", + "@typescript-eslint/types": "8.5.0", "eslint-visitor-keys": "^3.4.3" }, "engines": { - "node": "^18.18.0 || >=20.0.0" + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "type": "opencollective", @@ -629,6 +547,7 @@ "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", "dev": true, + "license": "Apache-2.0", "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, @@ -871,6 +790,7 @@ "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==", "dev": true, + "license": "MIT", "bin": { "cssesc": "bin/cssesc" }, @@ -879,12 +799,13 @@ } }, "node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", "dev": true, + "license": "MIT", "dependencies": { - "ms": "2.1.2" + "ms": "^2.1.3" }, "engines": { "node": ">=6.0" @@ -1017,16 +938,18 @@ } }, "node_modules/eslint": { - "version": "9.7.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.7.0.tgz", - "integrity": "sha512-FzJ9D/0nGiCGBf8UXO/IGLTgLVzIxze1zpfA8Ton2mjLovXdAPlYDv+MQDcqj3TmrhAGYfOpz9RfR+ent0AgAw==", + "version": "9.10.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.10.0.tgz", + "integrity": "sha512-Y4D0IgtBZfOcOUAIQTSXBKoNGfY0REGqHJG6+Q81vNippW5YlKjHFj4soMxamKK1NXHUWuBZTLdU3Km+L/pcHw==", "dev": true, + "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.11.0", - "@eslint/config-array": "^0.17.0", + "@eslint/config-array": "^0.18.0", "@eslint/eslintrc": "^3.1.0", - "@eslint/js": "9.7.0", + "@eslint/js": "9.10.0", + "@eslint/plugin-kit": "^0.1.0", "@humanwhocodes/module-importer": "^1.0.1", "@humanwhocodes/retry": "^0.3.0", "@nodelib/fs.walk": "^1.2.8", @@ -1049,7 +972,6 @@ "is-glob": "^4.0.0", "is-path-inside": "^3.0.3", "json-stable-stringify-without-jsonify": "^1.0.1", - "levn": "^0.4.1", "lodash.merge": "^4.6.2", "minimatch": "^3.1.2", "natural-compare": "^1.4.0", @@ -1065,6 +987,14 @@ }, "funding": { "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "jiti": "*" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + } } }, "node_modules/eslint-scope": { @@ -1412,10 +1342,11 @@ } }, "node_modules/ignore": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz", - "integrity": "sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==", + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", "dev": true, + "license": "MIT", "engines": { "node": ">= 4" } @@ -1670,10 +1601,11 @@ } }, "node_modules/micromatch": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz", - "integrity": "sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==", + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", "dev": true, + "license": "MIT", "dependencies": { "braces": "^3.0.3", "picomatch": "^2.3.1" @@ -1707,10 +1639,11 @@ } }, "node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" }, "node_modules/nanoid": { "version": "3.3.7", @@ -1860,6 +1793,7 @@ "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", "dev": true, + "license": "MIT", "engines": { "node": ">=12" }, @@ -1868,9 +1802,9 @@ } }, "node_modules/postcss": { - "version": "8.4.39", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.39.tgz", - "integrity": "sha512-0vzE+lAiG7hZl1/9I8yzKLx3aR9Xbof3fBHKunvMfOCYAtMhrsnccJY2iTURb9EZd5+pLuiNV9/c/GZJOHsgIw==", + "version": "8.4.45", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.45.tgz", + "integrity": "sha512-7KTLTdzdZZYscUc65XmjFiB73vBhBfbPztCYdUNvlaso9PrzjzcmjqBPR0lNGkcVlcO4BjiO5rK/qNz+XAen1Q==", "dev": true, "funding": [ { @@ -1886,6 +1820,7 @@ "url": "https://github.com/sponsors/ai" } ], + "license": "MIT", "dependencies": { "nanoid": "^3.3.7", "picocolors": "^1.0.1", @@ -1911,10 +1846,11 @@ } }, "node_modules/postcss-resolve-nested-selector": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/postcss-resolve-nested-selector/-/postcss-resolve-nested-selector-0.1.1.tgz", - "integrity": "sha512-HvExULSwLqHLgUy1rl3ANIqCsvMS0WHss2UOsXhXnQaZ9VCc2oBvIpXrl00IUFT5ZDITME0o6oiXeiHr2SAIfw==", - "dev": true + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/postcss-resolve-nested-selector/-/postcss-resolve-nested-selector-0.1.6.tgz", + "integrity": "sha512-0sglIs9Wmkzbr8lQwEyIzlDOOC9bGmfVKcJTaxv3vMmd3uo4o4DerC3En0bnmgceeql9BfC8hRkp7cg0fjdVqw==", + "dev": true, + "license": "MIT" }, "node_modules/postcss-safe-parser": { "version": "6.0.0", @@ -1933,10 +1869,11 @@ } }, "node_modules/postcss-selector-parser": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.1.tgz", - "integrity": "sha512-b4dlw/9V8A71rLIDsSwVmak9z2DuBUB7CA1/wSdelNEzqsjoSPeADTWNO09lpH49Diy3/JIZ2bSPB1dI3LJCHg==", + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz", + "integrity": "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==", "dev": true, + "license": "MIT", "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" @@ -2045,6 +1982,7 @@ "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", "dev": true, + "license": "ISC", "bin": { "semver": "bin/semver.js" }, @@ -2159,9 +2097,9 @@ } }, "node_modules/stylelint": { - "version": "16.7.0", - "resolved": "https://registry.npmjs.org/stylelint/-/stylelint-16.7.0.tgz", - "integrity": "sha512-Q1ATiXlz+wYr37a7TGsfvqYn2nSR3T/isw3IWlZQzFzCNoACHuGBb6xBplZXz56/uDRJHIygxjh7jbV/8isewA==", + "version": "16.9.0", + "resolved": "https://registry.npmjs.org/stylelint/-/stylelint-16.9.0.tgz", + "integrity": "sha512-31Nm3WjxGOBGpQqF43o3wO9L5AC36TPIe6030Lnm13H3vDMTcS21DrLh69bMX+DBilKqMMVLian4iG6ybBoNRQ==", "dev": true, "funding": [ { @@ -2173,18 +2111,19 @@ "url": "https://github.com/sponsors/stylelint" } ], + "license": "MIT", "dependencies": { - "@csstools/css-parser-algorithms": "^2.7.1", - "@csstools/css-tokenizer": "^2.4.1", - "@csstools/media-query-list-parser": "^2.1.13", - "@csstools/selector-specificity": "^3.1.1", + "@csstools/css-parser-algorithms": "^3.0.1", + "@csstools/css-tokenizer": "^3.0.1", + "@csstools/media-query-list-parser": "^3.0.1", + "@csstools/selector-specificity": "^4.0.0", "@dual-bundle/import-meta-resolve": "^4.1.0", "balanced-match": "^2.0.0", "colord": "^2.9.3", "cosmiconfig": "^9.0.0", "css-functions-list": "^3.2.2", "css-tree": "^2.3.1", - "debug": "^4.3.5", + "debug": "^4.3.6", "fast-glob": "^3.3.2", "fastest-levenshtein": "^1.0.16", "file-entry-cache": "^9.0.0", @@ -2192,24 +2131,24 @@ "globby": "^11.1.0", "globjoin": "^0.1.4", "html-tags": "^3.3.1", - "ignore": "^5.3.1", + "ignore": "^5.3.2", "imurmurhash": "^0.1.4", "is-plain-object": "^5.0.0", "known-css-properties": "^0.34.0", "mathml-tag-names": "^2.1.3", "meow": "^13.2.0", - "micromatch": "^4.0.7", + "micromatch": "^4.0.8", "normalize-path": "^3.0.0", "picocolors": "^1.0.1", - "postcss": "^8.4.39", - "postcss-resolve-nested-selector": "^0.1.1", + "postcss": "^8.4.41", + "postcss-resolve-nested-selector": "^0.1.6", "postcss-safe-parser": "^7.0.0", - "postcss-selector-parser": "^6.1.0", + "postcss-selector-parser": "^6.1.2", "postcss-value-parser": "^4.2.0", "resolve-from": "^5.0.0", "string-width": "^4.2.3", "strip-ansi": "^7.1.0", - "supports-hyperlinks": "^3.0.0", + "supports-hyperlinks": "^3.1.0", "svg-tags": "^1.0.0", "table": "^6.8.2", "write-file-atomic": "^5.0.1" @@ -2374,16 +2313,20 @@ } }, "node_modules/supports-hyperlinks": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/supports-hyperlinks/-/supports-hyperlinks-3.0.0.tgz", - "integrity": "sha512-QBDPHyPQDRTy9ku4URNGY5Lah8PAaXs6tAAwp55sL5WCsSW7GIfdf6W5ixfziW+t7wh3GVvHyHHyQ1ESsoRvaA==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/supports-hyperlinks/-/supports-hyperlinks-3.1.0.tgz", + "integrity": "sha512-2rn0BZ+/f7puLOHZm1HOJfwBggfaHXUpPUSSG/SWM4TWp5KCfmNYwnC3hruy2rZlMnmWZ+QAGpZfchu3f3695A==", "dev": true, + "license": "MIT", "dependencies": { "has-flag": "^4.0.0", "supports-color": "^7.0.0" }, "engines": { "node": ">=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/svg-tags": { @@ -2453,6 +2396,7 @@ "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.3.0.tgz", "integrity": "sha512-UQMIo7pb8WRomKR1/+MFVLTroIvDVtMX3K6OUir8ynLyzB8Jeriont2bTAtmNPa1ekAgN7YPDyf6V+ygrdU+eQ==", "dev": true, + "license": "MIT", "engines": { "node": ">=16" }, @@ -2499,7 +2443,8 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/which": { "version": "2.0.2", diff --git a/package.json b/package.json index 58fc7ba62..0d80376ab 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,9 @@ { "devDependencies": { - "@stylistic/eslint-plugin": "*", - "eslint": "^9.7.0", + "@stylistic/eslint-plugin": "^2.8.0", + "eslint": "^9.10.0", "postcss-html": "*", - "stylelint": ">=13.6.0", + "stylelint": ">=16.9.0", "stylelint-config-standard": "*" } } diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c index 5fd3303dc..841d040b2 100644 --- a/src/fuzzy_storage.c +++ b/src/fuzzy_storage.c @@ -128,11 +128,17 @@ KHASH_SET_INIT_INT(fuzzy_key_ids_set); KHASH_INIT(fuzzy_key_flag_stat, int, struct fuzzy_key_stat, 1, kh_int_hash_func, kh_int_hash_equal); struct fuzzy_key { + char *name; struct rspamd_cryptobox_keypair *key; struct rspamd_cryptobox_pubkey *pk; struct fuzzy_key_stat *stat; khash_t(fuzzy_key_flag_stat) * flags_stat; khash_t(fuzzy_key_ids_set) * forbidden_ids; + struct rspamd_leaky_bucket_elt *rl_bucket; + double burst; + double rate; + ev_tstamp expire; + bool expired; ref_entry_t ref; }; @@ -258,7 +264,8 @@ static gboolean rspamd_fuzzy_check_client(struct rspamd_fuzzy_storage_ctx *ctx, static void rspamd_fuzzy_maybe_call_blacklisted(struct rspamd_fuzzy_storage_ctx *ctx, rspamd_inet_addr_t *addr, const char *reason); -static struct fuzzy_key *fuzzy_add_keypair_from_ucl(const ucl_object_t *obj, +static struct fuzzy_key *fuzzy_add_keypair_from_ucl(struct rspamd_config *cfg, + const ucl_object_t *obj, khash_t(rspamd_fuzzy_keys_hash) * target); struct fuzzy_keymap_ucl_buf { @@ -366,7 +373,7 @@ ucl_keymap_fin_cb(struct map_cb_data *data, void **target) while ((cur = ucl_object_iterate(top, &it, true)) != NULL) { struct fuzzy_key *nk; - nk = fuzzy_add_keypair_from_ucl(cur, jb->ctx->dynamic_keys); + nk = fuzzy_add_keypair_from_ucl(cfg, cur, jb->ctx->dynamic_keys); if (nk == NULL) { msg_warn_config("cannot add dynamic keypair"); @@ -404,6 +411,78 @@ ucl_keymap_dtor_cb(struct map_cb_data *data) } } +enum rspamd_ratelimit_check_result { + ratelimit_pass, + ratelimit_new, + ratelimit_existing, +}; + +enum rspamd_ratelimit_check_policy { + ratelimit_policy_permanent, + ratelimit_policy_normal, +}; + +static enum rspamd_ratelimit_check_result +rspamd_fuzzy_check_ratelimit_bucket(struct fuzzy_session *session, struct rspamd_leaky_bucket_elt *elt, + enum rspamd_ratelimit_check_policy policy, double max_burst, double max_rate) +{ + gboolean ratelimited = FALSE, new_ratelimit = FALSE; + + if (isnan(elt->cur)) { + /* There is an issue with the previous logic: the TTL is updated each time + * we see that new bucket. Hence, we need to check the `last` and act accordingly + */ + if (elt->last < session->timestamp && session->timestamp - elt->last >= session->ctx->leaky_bucket_ttl) { + /* + * We reset bucket to it's 90% capacity to allow some requests + * This should cope with the issue when we block an IP network for some burst and never unblock it + */ + elt->cur = max_burst * 0.9; + elt->last = session->timestamp; + } + else { + ratelimited = TRUE; + } + } + else { + /* Update bucket: leak some elements */ + if (elt->last < session->timestamp) { + elt->cur -= max_rate * (session->timestamp - elt->last); + elt->last = session->timestamp; + + if (elt->cur < 0) { + elt->cur = 0; + } + } + else { + elt->last = session->timestamp; + } + + /* Check the bucket */ + if (elt->cur >= max_burst) { + + if (policy == ratelimit_policy_permanent) { + elt->cur = NAN; + } + new_ratelimit = TRUE; + ratelimited = TRUE; + } + else { + elt->cur++; /* Allow one more request */ + } + } + + if (ratelimited) { + rspamd_fuzzy_maybe_call_blacklisted(session->ctx, session->addr, "ratelimit"); + } + + if (new_ratelimit) { + return ratelimit_new; + } + + return ratelimited ? ratelimit_existing : ratelimit_pass; +} + static gboolean rspamd_fuzzy_check_ratelimit(struct fuzzy_session *session) { @@ -443,59 +522,17 @@ rspamd_fuzzy_check_ratelimit(struct fuzzy_session *session) (time_t) session->timestamp); if (elt) { - gboolean ratelimited = FALSE, new_ratelimit = FALSE; - - if (isnan(elt->cur)) { - /* There is an issue with the previous logic: the TTL is updated each time - * we see that new bucket. Hence, we need to check the `last` and act accordingly - */ - if (elt->last < session->timestamp && session->timestamp - elt->last >= session->ctx->leaky_bucket_ttl) { - /* - * We reset bucket to it's 90% capacity to allow some requests - * This should cope with the issue when we block an IP network for some burst and never unblock it - */ - elt->cur = session->ctx->leaky_bucket_burst * 0.9; - elt->last = session->timestamp; - } - else { - ratelimited = TRUE; - } - } - else { - /* Update bucket: leak some elements */ - if (elt->last < session->timestamp) { - elt->cur -= session->ctx->leaky_bucket_rate * (session->timestamp - elt->last); - elt->last = session->timestamp; - - if (elt->cur < 0) { - elt->cur = 0; - } - } - else { - elt->last = session->timestamp; - } - - /* Check the bucket */ - if (elt->cur >= session->ctx->leaky_bucket_burst) { - - msg_info("ratelimiting %s (%s), %.1f max elts", - rspamd_inet_address_to_string(session->addr), - rspamd_inet_address_to_string(masked), - session->ctx->leaky_bucket_burst); - elt->cur = NAN; - new_ratelimit = TRUE; - ratelimited = TRUE; - } - else { - elt->cur++; /* Allow one more request */ - } - } + enum rspamd_ratelimit_check_result res = rspamd_fuzzy_check_ratelimit_bucket(session, elt, + ratelimit_policy_permanent, + session->ctx->leaky_bucket_burst, + session->ctx->leaky_bucket_rate); - if (ratelimited) { - rspamd_fuzzy_maybe_call_blacklisted(session->ctx, session->addr, "ratelimit"); - } + if (res == ratelimit_new) { + msg_info("ratelimiting %s (%s), %.1f max elts", + rspamd_inet_address_to_string(session->addr), + rspamd_inet_address_to_string(masked), + session->ctx->leaky_bucket_burst); - if (new_ratelimit) { struct rspamd_srv_command srv_cmd; srv_cmd.type = RSPAMD_SRV_FUZZY_BLOCKED; @@ -514,11 +551,16 @@ rspamd_fuzzy_check_ratelimit(struct fuzzy_session *session) msg_err("bad address length: %d, expected to be %d", (int) slen, (int) sizeof(srv_cmd.cmd.fuzzy_blocked.addr)); } } + + rspamd_fuzzy_maybe_call_blacklisted(session->ctx, session->addr, "ratelimit"); + } + else if (res == ratelimit_existing) { + rspamd_fuzzy_maybe_call_blacklisted(session->ctx, session->addr, "ratelimit"); } rspamd_inet_address_free(masked); - return !ratelimited; + return res == ratelimit_pass; } else { /* New bucket */ @@ -659,6 +701,15 @@ fuzzy_key_dtor(gpointer p) kh_destroy(fuzzy_key_ids_set, key->forbidden_ids); } + if (key->rl_bucket) { + /* TODO: save bucket stats */ + g_free(key->rl_bucket); + } + + if (key->name) { + g_free(key->name); + } + g_free(key); } } @@ -1464,7 +1515,14 @@ rspamd_fuzzy_process_command(struct fuzzy_session *session) if (session->ctx->encrypted_only && !encrypted) { /* Do not accept unencrypted commands */ - result.v1.value = 403; + result.v1.value = 415; + result.v1.prob = 0.0f; + rspamd_fuzzy_make_reply(cmd, &result, session, send_flags); + return; + } + + if (!rspamd_fuzzy_check_client(session->ctx, session->addr)) { + result.v1.value = 503; result.v1.prob = 0.0f; rspamd_fuzzy_make_reply(cmd, &result, session, send_flags); return; @@ -1487,23 +1545,95 @@ rspamd_fuzzy_process_command(struct fuzzy_session *session) } if (cmd->cmd == FUZZY_CHECK) { - bool can_continue = true; + bool is_rate_allowed = true; if (session->ctx->ratelimit_buckets) { if (session->ctx->ratelimit_log_only) { (void) rspamd_fuzzy_check_ratelimit(session); /* Check but ignore */ } else { - can_continue = rspamd_fuzzy_check_ratelimit(session); + is_rate_allowed = rspamd_fuzzy_check_ratelimit(session); + } + } + + if (session->key && session->key->rl_bucket) { + /* Check per-key bucket */ + + enum rspamd_ratelimit_check_result res = rspamd_fuzzy_check_ratelimit_bucket(session, session->key->rl_bucket, + ratelimit_policy_normal, + session->key->burst, + session->key->rate); + + if (res == ratelimit_new) { + msg_info("ratelimiting key %s %.1f max elts", + session->key->name ? session->key->name : "unknown", + session->key->burst); + + struct rspamd_srv_command srv_cmd; + + srv_cmd.type = RSPAMD_SRV_FUZZY_BLOCKED; + srv_cmd.cmd.fuzzy_blocked.af = rspamd_inet_address_get_af(session->addr); + + if (srv_cmd.cmd.fuzzy_blocked.af == AF_INET || srv_cmd.cmd.fuzzy_blocked.af == AF_INET6) { + socklen_t slen; + struct sockaddr *sa = rspamd_inet_address_get_sa(session->addr, &slen); + + if (slen <= sizeof(srv_cmd.cmd.fuzzy_blocked.addr)) { + memcpy(&srv_cmd.cmd.fuzzy_blocked.addr, sa, slen); + msg_debug("propagating blocked address to other workers"); + rspamd_srv_send_command(session->worker, + session->ctx->event_loop, + &srv_cmd, -1, NULL, NULL); + } + else { + msg_err("bad address length: %d, expected to be %d", + (int) slen, (int) sizeof(srv_cmd.cmd.fuzzy_blocked.addr)); + } + } + + rspamd_fuzzy_maybe_call_blacklisted(session->ctx, session->addr, "ratelimit"); + is_rate_allowed = session->ctx->ratelimit_log_only ? true : false; + } + else if (res == ratelimit_existing) { + rspamd_fuzzy_maybe_call_blacklisted(session->ctx, session->addr, "ratelimit"); + is_rate_allowed = session->ctx->ratelimit_log_only ? true : false; + } + } + + if (session->key && !isnan(session->key->expire)) { + /* Check expire */ + static ev_tstamp today = NAN; + + /* + * Update `today` sometimes + */ + if (isnan(today)) { + today = ev_time(); + } + else if (rspamd_random_uint64_fast() > 0xFFFF000000000000ULL) { + today = ev_time(); + } + + if (today > session->key->expire) { + if (!session->key->expired) { + msg_info("key %s is expired", session->key->name); + session->key->expired = true; + } + + result.v1.value = 503; + result.v1.prob = 0.0f; + rspamd_fuzzy_make_reply(cmd, &result, session, send_flags); + return; } } - if (can_continue) { + if (is_rate_allowed) { REF_RETAIN(session); rspamd_fuzzy_backend_check(session->ctx->backend, cmd, rspamd_fuzzy_check_callback, session); } else { + /* Should be 429 but we keep compatibility */ result.v1.value = 403; result.v1.prob = 0.0f; result.v1.flag = 0; @@ -1574,7 +1704,7 @@ rspamd_fuzzy_process_command(struct fuzzy_session *session) result.v1.prob = 1.0f; } else { - result.v1.value = 403; + result.v1.value = 503; result.v1.prob = 0.0f; } reply: @@ -2041,11 +2171,6 @@ accept_fuzzy_socket(EV_P_ ev_io *w, int revents) if (MSG_FIELD(msg[i], msg_namelen) >= sizeof(struct sockaddr)) { client_addr = rspamd_inet_address_from_sa(MSG_FIELD(msg[i], msg_name), MSG_FIELD(msg[i], msg_namelen)); - if (!rspamd_fuzzy_check_client(worker->ctx, client_addr)) { - /* Disallow forbidden clients silently */ - rspamd_inet_address_free(client_addr); - continue; - } } else { client_addr = NULL; @@ -2761,7 +2886,8 @@ fuzzy_parse_ids(rspamd_mempool_t *pool, } static struct fuzzy_key * -fuzzy_add_keypair_from_ucl(const ucl_object_t *obj, khash_t(rspamd_fuzzy_keys_hash) * target) +fuzzy_add_keypair_from_ucl(struct rspamd_config *cfg, const ucl_object_t *obj, + khash_t(rspamd_fuzzy_keys_hash) * target) { struct rspamd_cryptobox_keypair *kp = rspamd_keypair_from_ucl(obj); @@ -2785,6 +2911,10 @@ fuzzy_add_keypair_from_ucl(const ucl_object_t *obj, khash_t(rspamd_fuzzy_keys_ha rspamd_inet_address_hash, rspamd_inet_address_equal); key->stat = keystat; key->flags_stat = kh_init(fuzzy_key_flag_stat); + key->burst = NAN; + key->rate = NAN; + key->expire = NAN; + key->rl_bucket = NULL; /* Preallocate some space for flags */ kh_resize(fuzzy_key_flag_stat, key->flags_stat, 8); const unsigned char *pk = rspamd_keypair_component(kp, RSPAMD_KEYPAIR_COMPONENT_PK, @@ -2816,6 +2946,7 @@ fuzzy_add_keypair_from_ucl(const ucl_object_t *obj, khash_t(rspamd_fuzzy_keys_ha const ucl_object_t *extensions = rspamd_keypair_get_extensions(kp); if (extensions) { + lua_State *L = RSPAMD_LUA_CFG_STATE(cfg); const ucl_object_t *forbidden_ids = ucl_object_lookup(extensions, "forbidden_ids"); if (forbidden_ids && ucl_object_type(forbidden_ids) == UCL_ARRAY) { @@ -2832,9 +2963,72 @@ fuzzy_add_keypair_from_ucl(const ucl_object_t *obj, khash_t(rspamd_fuzzy_keys_ha } } } + + const ucl_object_t *ratelimit = ucl_object_lookup(extensions, "ratelimit"); + + static int ratelimit_lua_id = -1; + + if (ratelimit_lua_id == -1) { + /* Load ratelimit parsing function */ + if (!rspamd_lua_require_function(L, "plugins/ratelimit", "parse_limit")) { + msg_err_config("cannot load ratelimit parser from ratelimit plugin"); + } + else { + ratelimit_lua_id = luaL_ref(L, LUA_REGISTRYINDEX); + } + } + + if (ratelimit && ratelimit_lua_id != -1) { + lua_rawgeti(L, LUA_REGISTRYINDEX, ratelimit_lua_id); + lua_pushstring(L, "fuzzy_key_ratelimit"); + ucl_object_push_lua(L, ratelimit, false); + + if (lua_pcall(L, 2, 1, 0) != 0) { + msg_err_config("cannot call ratelimit parser from ratelimit plugin"); + } + else { + if (lua_type(L, -1) == LUA_TTABLE) { + /* + * The returned table is in form { rate = xx, burst = yy } + */ + lua_getfield(L, -1, "rate"); + key->rate = lua_tonumber(L, -1); + lua_pop(L, 1); + + lua_getfield(L, -1, "burst"); + key->burst = lua_tonumber(L, -1); + lua_pop(L, 1); + + key->rl_bucket = g_malloc0(sizeof(*key->rl_bucket)); + } + } + + lua_settop(L, 0); + } + + const ucl_object_t *expire = ucl_object_lookup(extensions, "expire"); + if (expire && ucl_object_type(expire) == UCL_STRING) { + struct tm tm; + + /* DD-MM-YYYY */ + char *end = strptime(ucl_object_tostring(expire), "%d-%m-%Y", &tm); + + if (end != NULL && *end != '\0') { + msg_err_config("cannot parse expire date: %s", ucl_object_tostring(expire)); + } + else { + key->expire = mktime(&tm); + } + } + + const ucl_object_t *name = ucl_object_lookup(extensions, "name"); + if (name && ucl_object_type(name) == UCL_STRING) { + key->name = g_strdup(ucl_object_tostring(name)); + } } - msg_debug("loaded keypair %*bs", crypto_box_publickeybytes(), pk); + msg_debug("loaded keypair %*bs; expire=%f; rate=%f; burst=%f; name=%s", (int) crypto_box_publickeybytes(), pk, + key->expire, key->rate, key->burst, key->name); return key; } @@ -2867,7 +3061,7 @@ fuzzy_parse_keypair(rspamd_mempool_t *pool, return ret; } - key = fuzzy_add_keypair_from_ucl(obj, ctx->keys); + key = fuzzy_add_keypair_from_ucl(ctx->cfg, obj, ctx->keys); if (key == NULL) { return FALSE; diff --git a/src/lua/lua_cryptobox.c b/src/lua/lua_cryptobox.c index fbd44cecd..c9cac1562 100644 --- a/src/lua/lua_cryptobox.c +++ b/src/lua/lua_cryptobox.c @@ -998,25 +998,13 @@ rspamd_lua_ssl_hmac_create(struct rspamd_lua_cryptobox_hash *h, const EVP_MD *ht bool insecure) { h->type = LUA_CRYPTOBOX_HASH_HMAC; - OSSL_PROVIDER *dflt = OSSL_PROVIDER_load(NULL, "default"); - -#if OPENSSL_VERSION_NUMBER > 0x10100000L - if (insecure) { - /* Should never ever be used for crypto/security purposes! */ -#ifdef EVP_MD_CTX_FLAG_NON_FIPS_ALLOW -#if OPENSSL_VERSION_MAJOR >= 3 - OSSL_PROVIDER *fips = OSSL_PROVIDER_load(NULL, "fips"); -#endif - } -#endif -#endif #if OPENSSL_VERSION_NUMBER < 0x10100000L || \ (defined(LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER < 0x30500000) h->content.hmac_c = g_malloc0(sizeof(*h->content.hmac_c)); #else #if OPENSSL_VERSION_MAJOR >= 3 - EVP_MAC* mac = EVP_MAC_fetch(NULL, "HMAC", NULL); + EVP_MAC *mac = EVP_MAC_fetch(NULL, "HMAC", NULL); h->content.hmac_c = EVP_MAC_CTX_new(mac); EVP_MAC_free(mac); #else @@ -1038,7 +1026,7 @@ rspamd_lua_ssl_hmac_create(struct rspamd_lua_cryptobox_hash *h, const EVP_MD *ht h->out_len = EVP_MD_size(htype); #if OPENSSL_VERSION_MAJOR >= 3 OSSL_PARAM params[2]; - params[0] = OSSL_PARAM_construct_utf8_string("digest", EVP_MD_get0_name(htype), 0); + params[0] = OSSL_PARAM_construct_utf8_string("digest", (char *) EVP_MD_get0_name(htype), 0); params[1] = OSSL_PARAM_construct_end(); EVP_MAC_init(h->content.hmac_c, key, keylen, params); @@ -1500,7 +1488,7 @@ lua_cryptobox_hash_finish(struct rspamd_lua_cryptobox_hash *h) g_assert(ssl_outlen <= sizeof(h->out)); memcpy(h->out, out, ssl_outlen); break; - case LUA_CRYPTOBOX_HASH_HMAC: + case LUA_CRYPTOBOX_HASH_HMAC: { #if OPENSSL_VERSION_MAJOR >= 3 size_t ssl_outlen_size_t = ssl_outlen; EVP_MAC_final(h->content.hmac_c, out, &ssl_outlen_size_t, sizeof(out)); @@ -1512,6 +1500,7 @@ lua_cryptobox_hash_finish(struct rspamd_lua_cryptobox_hash *h) g_assert(ssl_outlen <= sizeof(h->out)); memcpy(h->out, out, ssl_outlen); break; + } case LUA_CRYPTOBOX_HASH_XXHASH64: case LUA_CRYPTOBOX_HASH_XXHASH32: case LUA_CRYPTOBOX_HASH_XXHASH3: @@ -2520,7 +2509,6 @@ lua_cryptobox_gen_dkim_keypair(lua_State *L) if (strcmp(alg_str, "rsa") == 0) { BIGNUM *e; - RSA *r; EVP_PKEY *pk; e = BN_new(); diff --git a/src/lua/lua_rsa.c b/src/lua/lua_rsa.c index 0c56b223b..b7be612b0 100644 --- a/src/lua/lua_rsa.c +++ b/src/lua/lua_rsa.c @@ -1,11 +1,11 @@ -/*- - * Copyright 2016 Vsevolod Stakhov +/* + * Copyright 2024 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -184,7 +184,14 @@ lua_rsa_privkey_save(lua_State *L) else { if (f != stdout) { /* Set secure permissions for the private key file */ - chmod(filename, S_IRUSR | S_IWUSR); + if (fchmod(fileno(f), S_IRUSR | S_IWUSR) == -1) { + msg_err("cannot set permissions for private key file: %s, %s", + filename, + strerror(errno)); + fclose(f); + lua_pushboolean(L, FALSE); + return 1; + } } if (strcmp(type, "der") == 0) { @@ -463,7 +470,6 @@ lua_rsa_privkey_load_base64(lua_State *L) rspamd_lua_setclass(L, rspamd_rsa_privkey_classname, -1); *ppkey = pkey; } - } else { msg_err("cannot open EVP private key from data, %s", @@ -706,7 +712,7 @@ lua_rsa_verify_memory(lua_State *L) if (pkey != NULL && signature != NULL && data != NULL) { EVP_PKEY_CTX *pctx = EVP_PKEY_CTX_new(pkey, NULL); - g_assert(pctx != NULL); + g_assert(pctx != NULL); g_assert(EVP_PKEY_verify_init(pctx) == 1); ret = EVP_PKEY_verify(pctx, signature->str, signature->len, data, sz); diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c index 91b77c702..ece9a91e0 100644 --- a/src/plugins/fuzzy_check.c +++ b/src/plugins/fuzzy_check.c @@ -49,6 +49,9 @@ #include "libutil/libev_helper.h" #define DEFAULT_SYMBOL "R_FUZZY_HASH" +#define RSPAMD_FUZZY_SYMBOL_FORBIDDEN "FUZZY_FORBIDDEN" +#define RSPAMD_FUZZY_SYMBOL_RATELIMITED "FUZZY_RATELIMITED" +#define RSPAMD_FUZZY_SYMBOL_ENCRYPTION_REQUIRED "FUZZY_ENCRYPTION_REQUIRED" #define DEFAULT_IO_TIMEOUT 1.0 #define DEFAULT_RETRANSMITS 3 @@ -68,6 +71,12 @@ struct fuzzy_mapping { double weight; }; +enum fuzzy_rule_mode { + fuzzy_rule_read_only, + fuzzy_rule_write_only, + fuzzy_rule_read_write +}; + struct fuzzy_rule { struct upstream_list *servers; const char *symbol; @@ -84,7 +93,7 @@ struct fuzzy_rule { struct rspamd_cryptobox_pubkey *peer_key; double max_score; double weight_threshold; - gboolean read_only; + enum fuzzy_rule_mode mode; gboolean skip_unknown; gboolean no_share; gboolean no_subject; @@ -328,7 +337,7 @@ fuzzy_rule_new(const char *default_symbol, rspamd_mempool_t *pool) rspamd_mempool_add_destructor(pool, (rspamd_mempool_destruct_t) g_hash_table_unref, rule->mappings); - rule->read_only = FALSE; + rule->mode = fuzzy_rule_read_write; rule->weight_threshold = NAN; return rule; @@ -458,7 +467,26 @@ fuzzy_parse_rule(struct rspamd_config *cfg, const ucl_object_t *obj, if ((value = ucl_object_lookup(obj, "read_only")) != NULL) { - rule->read_only = ucl_obj_toboolean(value); + rule->mode = ucl_obj_toboolean(value) ? fuzzy_rule_read_only : fuzzy_rule_read_write; + } + + if ((value = ucl_object_lookup(obj, "mode")) != NULL) { + const char *mode_str = ucl_object_tostring(value); + + if (g_ascii_strcasecmp(mode_str, "read_only") == 0) { + rule->mode = fuzzy_rule_read_only; + } + else if (g_ascii_strcasecmp(mode_str, "write_only") == 0) { + rule->mode = fuzzy_rule_write_only; + } + else if (g_ascii_strcasecmp(mode_str, "read_write") == 0) { + rule->mode = fuzzy_rule_read_write; + } + else { + msg_warn_config("unknown mode: %s, use read_write by default", + mode_str); + rule->mode = fuzzy_rule_read_write; + } } if ((value = ucl_object_lookup(obj, "skip_unknown")) != NULL) { @@ -1153,6 +1181,44 @@ int fuzzy_check_module_config(struct rspamd_config *cfg, bool validate) 1, 1); + /* Register meta symbols (blocked, ratelimited, etc) */ + rspamd_symcache_add_symbol(cfg->cache, + RSPAMD_FUZZY_SYMBOL_FORBIDDEN, 0, NULL, NULL, + SYMBOL_TYPE_VIRTUAL, + cb_id); + rspamd_config_add_symbol(cfg, + RSPAMD_FUZZY_SYMBOL_FORBIDDEN, + 0.0, + "Fuzzy access denied", + "fuzzy", + 0, + 1, + 1); + rspamd_symcache_add_symbol(cfg->cache, + RSPAMD_FUZZY_SYMBOL_RATELIMITED, 0, NULL, NULL, + SYMBOL_TYPE_VIRTUAL, + cb_id); + rspamd_config_add_symbol(cfg, + RSPAMD_FUZZY_SYMBOL_RATELIMITED, + 0.0, + "Fuzzy rate limit is reached", + "fuzzy", + 0, + 1, + 1); + rspamd_symcache_add_symbol(cfg->cache, + RSPAMD_FUZZY_SYMBOL_ENCRYPTION_REQUIRED, 0, NULL, NULL, + SYMBOL_TYPE_VIRTUAL, + cb_id); + rspamd_config_add_symbol(cfg, + RSPAMD_FUZZY_SYMBOL_ENCRYPTION_REQUIRED, + 0.0, + "Fuzzy encryption is required by a server", + "fuzzy", + 0, + 1, + 1); + /* * Here we can have 2 possibilities: * @@ -2486,7 +2552,16 @@ fuzzy_check_try_read(struct fuzzy_client_session *session) } } else if (rep->v1.value == 403) { - rspamd_task_insert_result(task, "FUZZY_BLOCKED", 0.0, + /* In fact, it should be 429, but we preserve compatibility */ + rspamd_task_insert_result(task, RSPAMD_FUZZY_SYMBOL_RATELIMITED, 1.0, + session->rule->name); + } + else if (rep->v1.value == 503) { + rspamd_task_insert_result(task, RSPAMD_FUZZY_SYMBOL_FORBIDDEN, 1.0, + session->rule->name); + } + else if (rep->v1.value == 415) { + rspamd_task_insert_result(task, RSPAMD_FUZZY_SYMBOL_ENCRYPTION_REQUIRED, 1.0, session->rule->name); } else if (rep->v1.value == 401) { @@ -3400,11 +3475,14 @@ fuzzy_symbol_callback(struct rspamd_task *task, PTR_ARRAY_FOREACH(fuzzy_module_ctx->fuzzy_rules, i, rule) { - commands = fuzzy_generate_commands(task, rule, FUZZY_CHECK, 0, 0, 0); + if (rule->mode != fuzzy_rule_write_only) { + commands = fuzzy_generate_commands(task, rule, FUZZY_CHECK, 0, 0, 0); - if (commands != NULL) { - register_fuzzy_client_call(task, rule, commands); + if (commands != NULL) { + register_fuzzy_client_call(task, rule, commands); + } } + /* Skip write only rules from checks */ } rspamd_symcache_item_async_dec_check(task, item, M); @@ -3491,9 +3569,9 @@ register_fuzzy_controller_call(struct rspamd_http_connection_entry *entry, } static void -fuzzy_process_handler(struct rspamd_http_connection_entry *conn_ent, - struct rspamd_http_message *msg, int cmd, int value, int flag, - struct fuzzy_ctx *ctx, gboolean is_hash, unsigned int flags) +fuzzy_modify_handler(struct rspamd_http_connection_entry *conn_ent, + struct rspamd_http_message *msg, int cmd, int value, int flag, + struct fuzzy_ctx *ctx, gboolean is_hash, unsigned int flags) { struct fuzzy_rule *rule; struct rspamd_controller_session *session = conn_ent->ud; @@ -3541,7 +3619,7 @@ fuzzy_process_handler(struct rspamd_http_connection_entry *conn_ent, PTR_ARRAY_FOREACH(fuzzy_module_ctx->fuzzy_rules, i, rule) { - if (rule->read_only) { + if (rule->mode == fuzzy_rule_read_only) { continue; } @@ -3796,8 +3874,8 @@ fuzzy_controller_handler(struct rspamd_http_connection_entry *conn_ent, send_flags |= FUZZY_CHECK_FLAG_NOTEXT; } - fuzzy_process_handler(conn_ent, msg, cmd, value, flag, - (struct fuzzy_ctx *) ctx, is_hash, send_flags); + fuzzy_modify_handler(conn_ent, msg, cmd, value, flag, + (struct fuzzy_ctx *) ctx, is_hash, send_flags); return 0; } @@ -3879,7 +3957,7 @@ fuzzy_check_lua_process_learn(struct rspamd_task *task, if (!res) { break; } - if (rule->read_only) { + if (rule->mode == fuzzy_rule_read_only) { continue; } @@ -4181,7 +4259,7 @@ fuzzy_lua_gen_hashes_handler(lua_State *L) PTR_ARRAY_FOREACH(fuzzy_module_ctx->fuzzy_rules, i, rule) { - if (rule->read_only) { + if (rule->mode == fuzzy_rule_read_only) { continue; } @@ -4409,7 +4487,7 @@ fuzzy_lua_list_storages(lua_State *L) { lua_newtable(L); - lua_pushboolean(L, rule->read_only); + lua_pushboolean(L, rule->mode == fuzzy_rule_read_only); lua_setfield(L, -2, "read_only"); /* Push servers */ diff --git a/src/plugins/lua/aws_s3.lua b/src/plugins/lua/aws_s3.lua index 30e88d2cd..ac344d86c 100644 --- a/src/plugins/lua/aws_s3.lua +++ b/src/plugins/lua/aws_s3.lua @@ -238,7 +238,7 @@ settings = lua_util.override_defaults(settings, opts) local res, err = settings_schema:transform(settings) if not res then - rspamd_logger.warnx(rspamd_config, 'plugin is misconfigured: %s', err) + rspamd_logger.warnx(rspamd_config, 'plugin %s is misconfigured: %s', N, err) lua_util.disable_module(N, "config") return end diff --git a/src/plugins/lua/bimi.lua b/src/plugins/lua/bimi.lua index 278359069..78949a5c0 100644 --- a/src/plugins/lua/bimi.lua +++ b/src/plugins/lua/bimi.lua @@ -265,7 +265,7 @@ local function check_bimi_vmc(task, domain, record) end if redis_params.username then if redis_params.password then - password = string.format( '%s:%s@', redis_params.username, redis_params.password) + password = string.format('%s:%s@', redis_params.username, redis_params.password) else rspamd_logger.warnx(task, "Redis requires a password when username is supplied") end @@ -358,7 +358,7 @@ settings = lua_util.override_defaults(settings, opts) local res, err = settings_schema:transform(settings) if not res then - rspamd_logger.warnx(rspamd_config, 'plugin is misconfigured: %s', err) + rspamd_logger.warnx(rspamd_config, 'plugin %s is misconfigured: %s', N, err) local err_msg = string.format("schema error: %s", res) lua_util.config_utils.push_config_error(N, err_msg) lua_util.disable_module(N, "failed", err_msg) diff --git a/src/plugins/lua/history_redis.lua b/src/plugins/lua/history_redis.lua index fff9f46b3..a3fdb0ec4 100644 --- a/src/plugins/lua/history_redis.lua +++ b/src/plugins/lua/history_redis.lua @@ -281,7 +281,7 @@ if opts then local res, err = settings_schema:transform(settings) if not res then - rspamd_logger.warnx(rspamd_config, '%s: plugin is misconfigured: %s', N, err) + rspamd_logger.warnx(rspamd_config, 'plugin %s is misconfigured: %s', N, err) lua_util.disable_module(N, "config") return end diff --git a/src/plugins/lua/ratelimit.lua b/src/plugins/lua/ratelimit.lua index f3331e850..168d8d63a 100644 --- a/src/plugins/lua/ratelimit.lua +++ b/src/plugins/lua/ratelimit.lua @@ -29,8 +29,7 @@ local lua_util = require "lua_util" local lua_verdict = require "lua_verdict" local rspamd_hash = require "rspamd_cryptobox_hash" local lua_selectors = require "lua_selectors" -local ts = require("tableshape").types - +local ratelimit_common = require "plugins/ratelimit" -- A plugin that implements ratelimits using redis local E = {} @@ -76,138 +75,6 @@ local function load_scripts(_, _) bucket_cleanup_id = lua_redis.load_redis_script_from_file(bucket_cleanup_script, redis_params) end -local limit_parser -local function parse_string_limit(lim, no_error) - local function parse_time_suffix(s) - if s == 's' then - return 1 - elseif s == 'm' then - return 60 - elseif s == 'h' then - return 3600 - elseif s == 'd' then - return 86400 - end - end - local function parse_num_suffix(s) - if s == '' then - return 1 - elseif s == 'k' then - return 1000 - elseif s == 'm' then - return 1000000 - elseif s == 'g' then - return 1000000000 - end - end - local lpeg = require "lpeg" - - if not limit_parser then - local digit = lpeg.R("09") - limit_parser = {} - limit_parser.integer = (lpeg.S("+-") ^ -1) * - (digit ^ 1) - limit_parser.fractional = (lpeg.P(".")) * - (digit ^ 1) - limit_parser.number = (limit_parser.integer * - (limit_parser.fractional ^ -1)) + - (lpeg.S("+-") * limit_parser.fractional) - limit_parser.time = lpeg.Cf(lpeg.Cc(1) * - (limit_parser.number / tonumber) * - ((lpeg.S("smhd") / parse_time_suffix) ^ -1), - function(acc, val) - return acc * val - end) - limit_parser.suffixed_number = lpeg.Cf(lpeg.Cc(1) * - (limit_parser.number / tonumber) * - ((lpeg.S("kmg") / parse_num_suffix) ^ -1), - function(acc, val) - return acc * val - end) - limit_parser.limit = lpeg.Ct(limit_parser.suffixed_number * - (lpeg.S(" ") ^ 0) * lpeg.S("/") * (lpeg.S(" ") ^ 0) * - limit_parser.time) - end - local t = lpeg.match(limit_parser.limit, lim) - - if t and t[1] and t[2] and t[2] ~= 0 then - return t[2], t[1] - end - - if not no_error then - rspamd_logger.errx(rspamd_config, 'bad limit: %s', lim) - end - - return nil -end - -local function str_to_rate(str) - local divider, divisor = parse_string_limit(str, false) - - if not divisor then - rspamd_logger.errx(rspamd_config, 'bad rate string: %s', str) - - return nil - end - - return divisor / divider -end - -local bucket_schema = ts.shape { - burst = ts.number + ts.string / lua_util.dehumanize_number, - rate = ts.number + ts.string / str_to_rate, - skip_recipients = ts.boolean:is_optional(), - symbol = ts.string:is_optional(), - message = ts.string:is_optional(), - skip_soft_reject = ts.boolean:is_optional(), -} - -local function parse_limit(name, data) - if type(data) == 'table' then - -- 2 cases here: - -- * old limit in format [burst, rate] - -- * vector of strings in Andrew's string format (removed from 1.8.2) - -- * proper bucket table - if #data == 2 and tonumber(data[1]) and tonumber(data[2]) then - -- Old style ratelimit - rspamd_logger.warnx(rspamd_config, 'old style ratelimit for %s', name) - if tonumber(data[1]) > 0 and tonumber(data[2]) > 0 then - return { - burst = data[1], - rate = data[2] - } - elseif data[1] ~= 0 then - rspamd_logger.warnx(rspamd_config, 'invalid numbers for %s', name) - else - rspamd_logger.infox(rspamd_config, 'disable limit %s, burst is zero', name) - end - - return nil - else - local parsed_bucket, err = bucket_schema:transform(data) - - if not parsed_bucket or err then - rspamd_logger.errx(rspamd_config, 'cannot parse bucket for %s: %s; original value: %s', - name, err, data) - else - return parsed_bucket - end - end - elseif type(data) == 'string' then - local rep_rate, burst = parse_string_limit(data) - rspamd_logger.warnx(rspamd_config, 'old style rate bucket config detected for %s: %s', - name, data) - if rep_rate and burst then - return { - burst = burst, - rate = burst / rep_rate -- reciprocal - } - end - end - - return nil -end - --- Check whether this addr is bounce local function check_bounce(from) return fun.any(function(b) @@ -490,7 +357,7 @@ local function ratelimit_cb(task) local ret, redis_key, bd = pcall(hdl, task) if ret then - local bucket = parse_limit(k, bd) + local bucket = ratelimit_common.parse_limit(k, bd) if bucket then prefixes[redis_key] = make_prefix(redis_key, k, bucket) end @@ -718,7 +585,7 @@ if opts then if lim.bucket[1] then for _, bucket in ipairs(lim.bucket) do - local b = parse_limit(t, bucket) + local b = ratelimit_common.parse_limit(t, bucket) if not b then rspamd_logger.errx(rspamd_config, 'bad ratelimit bucket for %s: "%s"', @@ -729,7 +596,7 @@ if opts then table.insert(buckets, b) end else - local bucket = parse_limit(t, lim.bucket) + local bucket = ratelimit_common.parse_limit(t, lim.bucket) if not bucket then rspamd_logger.errx(rspamd_config, 'bad ratelimit bucket for %s: "%s"', @@ -757,7 +624,7 @@ if opts then end else rspamd_logger.warnx(rspamd_config, 'old syntax for ratelimits: %s', lim) - buckets = parse_limit(t, lim) + buckets = ratelimit_common.parse_limit(t, lim) if buckets then settings.limits[t] = { buckets = { buckets } |