From 6800a28b755c11fa427e9875c5380b09a147fca8 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Sun, 5 Feb 2017 15:25:21 +0100 Subject: [PATCH] [Feature] Add the preliminary version of redirects resolver in Lua --- src/plugins/lua/url_redirector.lua | 164 +++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 src/plugins/lua/url_redirector.lua diff --git a/src/plugins/lua/url_redirector.lua b/src/plugins/lua/url_redirector.lua new file mode 100644 index 000000000..7465eb7aa --- /dev/null +++ b/src/plugins/lua/url_redirector.lua @@ -0,0 +1,164 @@ +--[[ +Copyright (c) 2017, Vsevolod Stakhov + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +local redis_params +local settings = { + expire = 86400, -- 1 day by default + timeout = 10, -- 10 seconds by default + nested_limit = 1, -- How many redirects to follow + --proxy = "http://example.com:3128", -- Send request through proxy + key_prefix = 'rdr:', -- default hash name + check_ssl = false, -- check ssl certificates +} + +local rspamd_logger = require "rspamd_logger" +local rspamd_util = require "rspamd_util" +local rspamd_http = require "rspamd_http" +local fun = require "fun" +local hash = require "rspamd_cryptobox_hash" + +local function cache_url(task, orig_url, url, key, param) + local function redis_set_cb(err, data) + if err then + rspamd_logger.errx(task, 'got error while setting redirect keys: %s', err) + end + rspamd_plugins.surbl.continue_process(url, param) + end + + local ret = rspamd_redis_make_request(task, + redis_params, -- connect params + key, -- hash key + true, -- is write + redis_set_cb, --callback + 'SETEX', -- command + {key, tostring(settings.expire), url} -- arguments + ) + if not ret then + rspamd_logger.errx(task, 'cannot make redis request to cache results') + end +end + +local function resolve_url(task, orig_url, url, key, param, ntries) + if ntries > settings.nested_limit then + -- We cannot resolve more, stop + rspamd_logger.infox(task, 'cannot get more requests to resolve %s, stop on %s after %s attempts', + orig_url, url, ntries) + cache_url(task, orig_url, url, key, param) + end + + local function http_callback(err, code, body, headers) + if err then + rspamd_logger.infox(task, 'found redirect error from %s to %s, err message: %s', + orig_url, url, err) + cache_url(task, orig_url, url, key, param) + else + if code == 200 then + rspamd_logger.infox(task, 'found redirect from %s to %s, err code 200', + orig_url, url) + cache_url(task, orig_url, url, key, param) + elseif code == 301 or code == 302 then + local loc = headers['Location'] + rspamd_logger.infox(task, 'found redirect from %s to %s, err code 200', + orig_url, loc) + if loc then + resolve_url(task, orig_url, loc, key, param, ntries + 1) + else + cache_url(task, orig_url, url, key, param) + end + else + rspamd_logger.infox(task, 'found redirect error from %s to %s, err code: %s', + orig_url, url, code) + cache_url(task, orig_url, url, key, param) + end + end + end + + rspamd_http.request{ + url = url, + task = task, + timeout = settings.timeout, + opaque_body = true, + no_ssl_verify = not settings.check_ssl, + callback = http_callback + } +end + +local function url_redirector_handler(task, url, param) + local url_str = tostring(url) + local key = settings.key_prefix .. hash.create(url_str):base32() + + local function redis_get_cb(err, data) + if not err then + if type(data) == 'string' then + if data == 'processing' then + -- We have already requested this url to be resolved, so just return + -- the original url + + else + -- Got cached result + rspamd_logger.infox(task, 'found cached redirect from %s to %s', + url, data) + rspamd_plugins.surbl.continue_process(data, param) + return + end + end + end + local function redis_reserve_cb(err, data) + if err then + rspamd_logger.errx(task, 'got error while setting redirect keys: %s', err) + elseif data == 1 then + resolve_url(task, url_str, url_str, key, param, 1) + end + end + + local ret = rspamd_redis_make_request(task, + redis_params, -- connect params + key, -- hash key + true, -- is write + redis_reserve_cb, --callback + 'SETNX', -- command + {key, 'processing'} -- arguments + ) + end + local ret = rspamd_redis_make_request(task, + redis_params, -- connect params + key, -- hash key + false, -- is write + redis_get_cb, --callback + 'GET', -- command + {key} -- arguments + ) + if not ret then + rspamd_logger.errx(task, 'cannot make redis request to check results') + end +end + +local opts = rspamd_config:get_all_opt('url_redirector') +if opts then + for k,v in pairs(opts) do + settings[k] = v + end + redis_params = rspamd_parse_redis_server('url_redirector') + if not redis_params then + rspamd_logger.infox(rspamd_config, 'no servers are specified, disabling module') + else + if rspamd_plugins.surbl then + rspamd_plugins.surbl.register_redirect(url_redirector_handler) + else + rspamd_logger.infox(rspamd_config, 'surbl module is not enabled, disabling module') + end + end +end -- 2.39.5