2017-09-09 16:40:15 +02:00
--[[
Copyright ( c ) 2017 , Vsevolod Stakhov < vsevolod @ highsecure.ru >
Licensed under the Apache License , Version 2.0 ( the " License " ) ;
you may not use this file except in compliance with the License .
You may obtain a copy of the License at
http : // www.apache . org / licenses / LICENSE - 2.0
Unless required by applicable law or agreed to in writing , software
distributed under the License is distributed on an " AS IS " BASIS ,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND , either express or implied .
See the License for the specific language governing permissions and
limitations under the License .
] ] --
2017-05-23 12:37:43 +02:00
local logger = require " rspamd_logger "
2017-09-19 17:52:26 +02:00
local lutil = require " lua_util "
2017-12-01 18:06:10 +01:00
local rspamd_util = require " rspamd_util "
2018-09-20 14:08:31 +02:00
local ts = require ( " tableshape " ) . types
2017-05-23 12:37:43 +02:00
local exports = { }
2017-09-19 17:52:26 +02:00
local E = { }
2018-10-25 15:21:42 +02:00
local N = " lua_redis "
2017-09-19 17:52:26 +02:00
2018-09-20 14:08:31 +02:00
local common_schema = ts.shape {
timeout = ( ts.number + ts.string / lutil.parse_time_interval ) : is_optional ( ) ,
db = ts.string : is_optional ( ) ,
database = ts.string : is_optional ( ) ,
dbname = ts.string : is_optional ( ) ,
prefix = ts.string : is_optional ( ) ,
password = ts.string : is_optional ( ) ,
expand_keys = ts.boolean : is_optional ( ) ,
2018-11-22 18:23:04 +01:00
sentinels = ( ts.string + ts.array_of ( ts.string ) ) : is_optional ( ) ,
sentinel_watch_time = ( ts.number + ts.string / lutil.parse_time_interval ) : is_optional ( ) ,
sentinel_masters_pattern = ts.string : is_optional ( ) ,
2018-12-05 15:41:01 +01:00
sentinel_master_maxerrors = ( ts.number + ts.string / tonumber ) : is_optional ( ) ,
2018-09-20 14:08:31 +02:00
}
local config_schema =
ts.shape ( {
read_servers = ts.string + ts.array_of ( ts.string ) ,
write_servers = ts.string + ts.array_of ( ts.string ) ,
} , { extra_opts = common_schema } ) +
ts.shape ( {
servers = ts.string + ts.array_of ( ts.string ) ,
} , { extra_opts = common_schema } ) +
ts.shape ( {
server = ts.string + ts.array_of ( ts.string ) ,
} , { extra_opts = common_schema } )
exports.config_schema = config_schema
2018-11-22 18:23:04 +01:00
local function redis_query_sentinel ( ev_base , params , initialised )
local function flatten_redis_table ( tbl )
local res = { }
for i = 1 , # tbl , 2 do
res [ tbl [ i ] ] = tbl [ i + 1 ]
end
return res
end
-- Coroutines syntax
local rspamd_redis = require " rspamd_redis "
local addr = params.sentinels : get_upstream_round_robin ( )
local is_ok , connection = rspamd_redis.connect_sync ( {
host = addr : get_addr ( ) ,
timeout = params.timeout ,
config = rspamd_config ,
ev_base = ev_base ,
2019-06-28 13:13:52 +02:00
no_pool = true ,
2018-11-22 18:23:04 +01:00
} )
if not is_ok then
logger.errx ( rspamd_config , ' cannot connect sentinel at address: %s ' ,
tostring ( addr : get_addr ( ) ) )
addr : fail ( )
return
end
-- Get masters list
connection : add_cmd ( ' SENTINEL ' , { ' masters ' } )
local ok , result = connection : exec ( )
if ok and result and type ( result ) == ' table ' then
local masters = { }
for _ , m in ipairs ( result ) do
local master = flatten_redis_table ( m )
2019-08-14 14:01:13 +02:00
-- Wrap IPv6-adresses in brackets
if ( master.ip : match ( " : " ) ) then
master.ip = " [ " .. master.ip .. " ] "
end
2018-11-22 18:23:04 +01:00
if params.sentinel_masters_pattern then
if master.name : match ( params.sentinel_masters_pattern ) then
lutil.debugm ( N , ' found master %s with ip %s and port %s ' ,
master.name , master.ip , master.port )
masters [ master.name ] = master
else
lutil.debugm ( N , ' skip master %s with ip %s and port %s, pattern %s ' ,
master.name , master.ip , master.port , params.sentinel_masters_pattern )
end
else
lutil.debugm ( N , ' found master %s with ip %s and port %s ' ,
master.name , master.ip , master.port )
masters [ master.name ] = master
end
end
-- For each master we need to get a list of slaves
for k , v in pairs ( masters ) do
v.slaves = { }
local slave_result
connection : add_cmd ( ' SENTINEL ' , { ' slaves ' , k } )
ok , slave_result = connection : exec ( )
if ok then
for _ , s in ipairs ( slave_result ) do
local slave = flatten_redis_table ( s )
lutil.debugm ( N , rspamd_config ,
2019-08-21 14:17:04 +02:00
' found slave for master %s with ip %s and port %s ' ,
2018-11-22 18:23:04 +01:00
v.name , slave.ip , slave.port )
2019-08-14 14:01:13 +02:00
-- Wrap IPv6-adresses in brackets
if ( slave.ip : match ( " : " ) ) then
slave.ip = " [ " .. slave.ip .. " ] "
end
2018-11-22 18:23:04 +01:00
v.slaves [ # v.slaves + 1 ] = slave
end
end
end
-- We now form new strings for masters and slaves
local read_servers_tbl , write_servers_tbl = { } , { }
for _ , master in pairs ( masters ) do
write_servers_tbl [ # write_servers_tbl + 1 ] = string.format (
' %s:%s ' , master.ip , master.port
)
read_servers_tbl [ # read_servers_tbl + 1 ] = string.format (
' %s:%s ' , master.ip , master.port
)
for _ , slave in ipairs ( master.slaves ) do
2018-12-05 15:33:48 +01:00
if slave [ ' master-link-status ' ] == ' ok ' then
read_servers_tbl [ # read_servers_tbl + 1 ] = string.format (
' %s:%s ' , slave.ip , slave.port
)
end
2018-11-22 18:23:04 +01:00
end
end
2018-12-04 17:23:18 +01:00
table.sort ( read_servers_tbl )
table.sort ( write_servers_tbl )
2018-11-22 18:23:04 +01:00
local read_servers_str = table.concat ( read_servers_tbl , ' , ' )
2018-12-04 17:20:39 +01:00
local write_servers_str = table.concat ( write_servers_tbl , ' , ' )
2018-11-22 18:23:04 +01:00
lutil.debugm ( N , rspamd_config ,
2018-12-04 17:20:39 +01:00
' new servers list: %s read; %s write ' ,
read_servers_str ,
write_servers_str )
2018-11-22 18:23:04 +01:00
if read_servers_str ~= params.read_servers_str then
local upstream_list = require " rspamd_upstream_list "
local read_upstreams = upstream_list.create ( rspamd_config ,
read_servers_str , 6379 )
if read_upstreams then
logger.infox ( rspamd_config , ' sentinel %s: replace read servers with new list: %s ' ,
addr : get_addr ( ) : to_string ( true ) , read_servers_str )
params.read_servers = read_upstreams
params.read_servers_str = read_servers_str
end
end
if write_servers_str ~= params.write_servers_str then
local upstream_list = require " rspamd_upstream_list "
local write_upstreams = upstream_list.create ( rspamd_config ,
write_servers_str , 6379 )
if write_upstreams then
logger.infox ( rspamd_config , ' sentinel %s: replace write servers with new list: %s ' ,
addr : get_addr ( ) : to_string ( true ) , write_servers_str )
params.write_servers = write_upstreams
params.write_servers_str = write_servers_str
2018-12-05 15:41:01 +01:00
local queried = false
local function monitor_failures ( up , _ , count )
if count > params.sentinel_master_maxerrors and not queried then
logger.infox ( rspamd_config , ' sentinel: master with address %s, caused %s failures, try to query sentinel ' ,
up : get_addr ( ) : to_string ( true ) , count )
queried = true -- Avoid multiple checks caused by this monitor
redis_query_sentinel ( ev_base , params , true )
end
end
write_upstreams : add_watcher ( ' failure ' , monitor_failures )
2018-11-22 18:23:04 +01:00
end
end
addr : ok ( )
else
logger.errx ( ' cannot get data from Redis Sentinel %s: %s ' ,
addr : get_addr ( ) : to_string ( true ) , result )
addr : fail ( )
end
end
local function add_redis_sentinels ( params )
local upstream_list = require " rspamd_upstream_list "
local upstreams_sentinels = upstream_list.create ( rspamd_config ,
params.sentinels , 5000 )
if not upstreams_sentinels then
logger.errx ( rspamd_config , ' cannot load redis sentinels string: %s ' ,
params.sentinels )
return
end
params.sentinels = upstreams_sentinels
if not params.sentinel_watch_time then
params.sentinel_watch_time = 60 -- Each minute
end
2018-12-05 15:41:01 +01:00
if not params.sentinel_master_maxerrors then
params.sentinel_master_maxerrors = 2 -- Maximum number of errors before rechecking
end
2019-06-26 14:24:03 +02:00
rspamd_config : add_on_load ( function ( _ , ev_base , worker )
2018-11-22 18:23:04 +01:00
local initialised = false
if worker : is_scanner ( ) then
rspamd_config : add_periodic ( ev_base , 0.0 , function ( )
redis_query_sentinel ( ev_base , params , initialised )
initialised = true
return params.sentinel_watch_time
end , false )
end
end )
end
local cached_results = { }
local function calculate_redis_hash ( params )
local cr = require " rspamd_cryptobox_hash "
local h = cr.create ( )
local function rec_hash ( k , v )
if type ( v ) == ' string ' then
h : update ( k )
h : update ( v )
elseif type ( v ) == ' number ' then
h : update ( k )
h : update ( tostring ( v ) )
elseif type ( v ) == ' table ' then
for kk , vv in pairs ( v ) do
rec_hash ( kk , vv )
end
end
end
2018-11-24 10:26:33 +01:00
rec_hash ( ' top ' , params )
2018-11-22 18:23:04 +01:00
return h : base32 ( )
end
2019-03-29 10:41:07 +01:00
local function process_redis_opts ( options , redis_params )
local default_timeout = 1.0
local default_expand_keys = false
if not redis_params [ ' timeout ' ] or redis_params [ ' timeout ' ] == default_timeout then
if options [ ' timeout ' ] then
redis_params [ ' timeout ' ] = tonumber ( options [ ' timeout ' ] )
else
redis_params [ ' timeout ' ] = default_timeout
end
end
if options [ ' prefix ' ] and not redis_params [ ' prefix ' ] then
redis_params [ ' prefix ' ] = options [ ' prefix ' ]
end
if type ( options [ ' expand_keys ' ] ) == ' boolean ' then
redis_params [ ' expand_keys ' ] = options [ ' expand_keys ' ]
else
redis_params [ ' expand_keys ' ] = default_expand_keys
end
if not redis_params [ ' db ' ] then
if options [ ' db ' ] then
redis_params [ ' db ' ] = tostring ( options [ ' db ' ] )
elseif options [ ' dbname ' ] then
redis_params [ ' db ' ] = tostring ( options [ ' dbname ' ] )
elseif options [ ' database ' ] then
redis_params [ ' db ' ] = tostring ( options [ ' database ' ] )
end
end
if options [ ' password ' ] and not redis_params [ ' password ' ] then
redis_params [ ' password ' ] = options [ ' password ' ]
end
2019-06-26 14:24:03 +02:00
if not redis_params.sentinels and options.sentinels then
redis_params.sentinels = options.sentinels
2019-03-29 10:41:07 +01:00
end
2019-08-22 12:52:25 +02:00
if options [ ' sentinel_masters_pattern ' ] and not redis_params [ ' sentinel_masters_pattern ' ] then
redis_params [ ' sentinel_masters_pattern ' ] = options [ ' sentinel_masters_pattern ' ]
end
2019-03-29 10:41:07 +01:00
end
local function enrich_defaults ( rspamd_config , module , redis_params )
2019-04-06 10:36:00 +02:00
if rspamd_config then
local opts = rspamd_config : get_all_opt ( ' redis ' )
2019-03-29 10:41:07 +01:00
2019-04-06 10:36:00 +02:00
if opts then
if module then
if opts [ module ] then
process_redis_opts ( opts [ module ] , redis_params )
end
2019-04-01 15:22:17 +02:00
end
2019-03-29 10:41:07 +01:00
2019-04-06 10:36:00 +02:00
process_redis_opts ( opts , redis_params )
end
2019-03-29 10:41:07 +01:00
end
end
local function maybe_return_cached ( redis_params )
local h = calculate_redis_hash ( redis_params )
if cached_results [ h ] then
lutil.debugm ( N , ' reused redis server: %s ' , redis_params )
return cached_results [ h ]
end
redis_params.hash = h
cached_results [ h ] = redis_params
if not redis_params.read_only and redis_params.sentinels then
add_redis_sentinels ( redis_params )
end
lutil.debugm ( N , ' loaded new redis server: %s ' , redis_params )
return redis_params
end
2018-02-23 14:32:17 +01:00
--[[[
-- @module lua_redis
-- This module contains helper functions for working with Redis
--]]
2019-04-01 15:22:17 +02:00
local function process_redis_options ( options , rspamd_config , result )
2017-05-23 12:37:43 +02:00
local default_port = 6379
local upstream_list = require " rspamd_upstream_list "
2018-04-19 14:42:23 +02:00
local read_only = true
2017-05-23 12:37:43 +02:00
2018-02-15 16:28:47 +01:00
-- Try to get read servers:
local upstreams_read , upstreams_write
2017-05-23 12:37:43 +02:00
2018-02-15 16:28:47 +01:00
if options [ ' read_servers ' ] then
if rspamd_config then
2017-05-23 12:37:43 +02:00
upstreams_read = upstream_list.create ( rspamd_config ,
options [ ' read_servers ' ] , default_port )
2018-02-15 16:28:47 +01:00
else
upstreams_read = upstream_list.create ( options [ ' read_servers ' ] ,
default_port )
end
2018-11-22 18:23:04 +01:00
result.read_servers_str = options [ ' read_servers ' ]
2018-02-15 16:28:47 +01:00
elseif options [ ' servers ' ] then
if rspamd_config then
2017-05-23 12:37:43 +02:00
upstreams_read = upstream_list.create ( rspamd_config ,
options [ ' servers ' ] , default_port )
2018-02-15 16:28:47 +01:00
else
upstreams_read = upstream_list.create ( options [ ' servers ' ] , default_port )
end
2018-11-22 18:23:04 +01:00
result.read_servers_str = options [ ' servers ' ]
2018-04-19 14:42:23 +02:00
read_only = false
2018-02-15 16:28:47 +01:00
elseif options [ ' server ' ] then
if rspamd_config then
2017-05-23 12:37:43 +02:00
upstreams_read = upstream_list.create ( rspamd_config ,
options [ ' server ' ] , default_port )
2018-02-15 16:28:47 +01:00
else
upstreams_read = upstream_list.create ( options [ ' server ' ] , default_port )
2017-05-23 12:37:43 +02:00
end
2018-11-22 18:23:04 +01:00
result.read_servers_str = options [ ' server ' ]
2018-04-19 14:42:23 +02:00
read_only = false
2018-02-15 16:28:47 +01:00
end
2017-05-23 12:37:43 +02:00
2018-02-15 16:28:47 +01:00
if upstreams_read then
if options [ ' write_servers ' ] then
if rspamd_config then
2017-05-23 12:37:43 +02:00
upstreams_write = upstream_list.create ( rspamd_config ,
2018-04-19 14:42:23 +02:00
options [ ' write_servers ' ] , default_port )
2017-05-23 12:37:43 +02:00
else
2018-02-15 16:28:47 +01:00
upstreams_write = upstream_list.create ( options [ ' write_servers ' ] ,
2018-04-19 14:42:23 +02:00
default_port )
2017-05-23 12:37:43 +02:00
end
2018-11-22 18:23:04 +01:00
result.write_servers_str = options [ ' write_servers ' ]
2018-04-19 14:42:23 +02:00
read_only = false
elseif not read_only then
2018-02-15 16:28:47 +01:00
upstreams_write = upstreams_read
2018-11-22 18:23:04 +01:00
result.write_servers_str = result.read_servers_str
2017-05-23 12:37:43 +02:00
end
2018-02-15 16:28:47 +01:00
end
2017-05-23 12:37:43 +02:00
2018-02-15 16:28:47 +01:00
-- Store options
2019-03-29 10:41:07 +01:00
process_redis_opts ( options , result )
2018-02-15 16:28:47 +01:00
2018-07-04 11:04:34 +02:00
if read_only and not upstreams_write then
2018-04-19 14:42:23 +02:00
result.read_only = true
2018-07-04 11:04:34 +02:00
elseif upstreams_write then
2018-04-19 14:42:23 +02:00
result.read_only = false
end
2018-04-17 18:34:37 +02:00
if upstreams_read then
2018-02-15 16:28:47 +01:00
result.read_servers = upstreams_read
2018-11-22 18:23:04 +01:00
2018-04-17 18:34:37 +02:00
if upstreams_write then
result.write_servers = upstreams_write
end
2018-10-25 15:21:42 +02:00
2018-02-15 16:28:47 +01:00
return true
2017-05-23 12:37:43 +02:00
end
2018-11-23 19:09:28 +01:00
lutil.debugm ( N , rspamd_config ,
' cannot load redis server from obj: %s, processed to %s ' ,
options , result )
2018-02-15 16:28:47 +01:00
return false
end
2019-04-01 15:22:17 +02:00
--[[[
@ function try_load_redis_servers ( options , rspamd_config , no_fallback )
Tries to load redis servers from the specified ` options ` object .
Returns ` redis_params ` table or nil in case of failure
--]]
exports.try_load_redis_servers = function ( options , rspamd_config , no_fallback , module_name )
local result = { }
if process_redis_options ( options , rspamd_config , result ) then
if not no_fallback then
enrich_defaults ( rspamd_config , module_name , result )
end
return maybe_return_cached ( result )
end
end
2018-02-15 16:28:47 +01:00
-- This function parses redis server definition using either
-- specific server string for this module or global
-- redis section
local function rspamd_parse_redis_server ( module_name , module_opts , no_fallback )
local result = { }
2017-05-23 12:37:43 +02:00
-- Try local options
2017-06-10 15:45:29 +02:00
local opts
2018-10-25 15:21:42 +02:00
lutil.debugm ( N , rspamd_config , ' try load redis config for: %s ' , module_name )
2017-06-10 15:45:29 +02:00
if not module_opts then
opts = rspamd_config : get_all_opt ( module_name )
else
opts = module_opts
end
2017-05-23 12:37:43 +02:00
if opts then
2017-11-26 19:05:23 +01:00
local ret
2017-11-12 13:52:02 +01:00
if opts.redis then
2019-04-01 15:22:17 +02:00
ret = process_redis_options ( opts.redis , rspamd_config , result )
2017-11-12 13:52:02 +01:00
if ret then
2019-03-29 10:41:07 +01:00
if not no_fallback then
enrich_defaults ( rspamd_config , module_name , result )
end
return maybe_return_cached ( result )
2017-11-12 13:52:02 +01:00
end
end
2019-04-01 15:22:17 +02:00
ret = process_redis_options ( opts , rspamd_config , result )
2017-05-23 12:37:43 +02:00
2017-11-12 13:52:02 +01:00
if ret then
2019-03-29 10:41:07 +01:00
if not no_fallback then
enrich_defaults ( rspamd_config , module_name , result )
end
return maybe_return_cached ( result )
2017-11-12 13:52:02 +01:00
end
2017-05-23 12:37:43 +02:00
end
2018-09-20 14:08:31 +02:00
if no_fallback then
2018-09-20 15:11:45 +02:00
logger.infox ( rspamd_config , " cannot find Redis definitions for %s and fallback is disabled " ,
module_name )
2018-09-20 14:08:31 +02:00
return nil
end
2017-06-10 15:45:29 +02:00
2017-05-23 12:37:43 +02:00
-- Try global options
opts = rspamd_config : get_all_opt ( ' redis ' )
if opts then
2017-11-26 19:05:23 +01:00
local ret
2017-05-23 12:37:43 +02:00
if opts [ module_name ] then
2019-04-01 15:22:17 +02:00
ret = process_redis_options ( opts [ module_name ] , rspamd_config , result )
2018-09-20 14:08:31 +02:00
2017-05-23 12:37:43 +02:00
if ret then
2019-03-29 10:41:07 +01:00
return maybe_return_cached ( result )
2017-05-23 12:37:43 +02:00
end
else
2019-04-01 15:22:17 +02:00
ret = process_redis_options ( opts , rspamd_config , result )
2017-05-23 12:37:43 +02:00
-- Exclude disabled
if opts [ ' disabled_modules ' ] then
for _ , v in ipairs ( opts [ ' disabled_modules ' ] ) do
if v == module_name then
logger.infox ( rspamd_config , " NOT using default redis server for module %s: it is disabled " ,
module_name )
return nil
end
end
end
if ret then
2018-09-20 15:11:45 +02:00
logger.infox ( rspamd_config , " use default Redis settings for %s " ,
module_name )
2019-03-29 10:41:07 +01:00
return maybe_return_cached ( result )
2017-05-23 12:37:43 +02:00
end
end
end
if result.read_servers then
2019-03-29 10:41:07 +01:00
return maybe_return_cached ( result )
2017-05-23 12:37:43 +02:00
end
2018-09-20 15:11:45 +02:00
return nil
2017-05-23 12:37:43 +02:00
end
2018-02-23 14:32:17 +01:00
--[[[
-- @function lua_redis.parse_redis_server(module_name, module_opts, no_fallback)
-- Extracts Redis server settings from configuration
-- @param {string} module_name name of module to get settings for
-- @param {table} module_opts settings for module or `nil` to fetch them from configuration
-- @param {boolean} no_fallback should be `true` if global settings must not be used
-- @return {table} redis server settings
-- @example
-- local rconfig = lua_redis.parse_redis_server('my_module')
-- -- rconfig contains upstream_list objects in ['write_servers'] and ['read_servers']
-- -- ['timeout'] contains timeout in seconds
-- -- ['expand_keys'] if true tells that redis key expansion is enabled
--]]
2017-05-23 12:37:43 +02:00
exports.rspamd_parse_redis_server = rspamd_parse_redis_server
exports.parse_redis_server = rspamd_parse_redis_server
2017-09-19 17:52:26 +02:00
local process_cmd = {
bitop = function ( args )
local idx_l = { }
for i = 2 , # args do
table.insert ( idx_l , i )
end
return idx_l
end ,
blpop = function ( args )
local idx_l = { }
for i = 1 , # args - 1 do
table.insert ( idx_l , i )
end
return idx_l
end ,
eval = function ( args )
local idx_l = { }
local numkeys = args [ 2 ]
2018-02-27 14:18:48 +01:00
if numkeys and tonumber ( numkeys ) >= 1 then
2017-09-19 17:52:26 +02:00
for i = 3 , numkeys + 2 do
table.insert ( idx_l , i )
end
end
return idx_l
end ,
set = function ( args )
return { 1 }
end ,
mget = function ( args )
local idx_l = { }
for i = 1 , # args do
table.insert ( idx_l , i )
end
return idx_l
end ,
mset = function ( args )
local idx_l = { }
for i = 1 , # args , 2 do
table.insert ( idx_l , i )
end
return idx_l
end ,
sdiffstore = function ( args )
local idx_l = { }
for i = 2 , # args do
table.insert ( idx_l , i )
end
return idx_l
end ,
smove = function ( args )
return { 1 , 2 }
end ,
script = function ( ) end
}
process_cmd.append = process_cmd.set
process_cmd.auth = process_cmd.script
process_cmd.bgrewriteaof = process_cmd.script
process_cmd.bgsave = process_cmd.script
process_cmd.bitcount = process_cmd.set
process_cmd.bitfield = process_cmd.set
process_cmd.bitpos = process_cmd.set
process_cmd.brpop = process_cmd.blpop
process_cmd.brpoplpush = process_cmd.blpop
process_cmd.client = process_cmd.script
process_cmd.cluster = process_cmd.script
process_cmd.command = process_cmd.script
process_cmd.config = process_cmd.script
process_cmd.dbsize = process_cmd.script
process_cmd.debug = process_cmd.script
process_cmd.decr = process_cmd.set
process_cmd.decrby = process_cmd.set
process_cmd.del = process_cmd.mget
process_cmd.discard = process_cmd.script
process_cmd.dump = process_cmd.set
process_cmd.echo = process_cmd.script
process_cmd.evalsha = process_cmd.eval
process_cmd.exec = process_cmd.script
process_cmd.exists = process_cmd.mget
process_cmd.expire = process_cmd.set
process_cmd.expireat = process_cmd.set
process_cmd.flushall = process_cmd.script
process_cmd.flushdb = process_cmd.script
process_cmd.geoadd = process_cmd.set
process_cmd.geohash = process_cmd.set
process_cmd.geopos = process_cmd.set
process_cmd.geodist = process_cmd.set
process_cmd.georadius = process_cmd.set
process_cmd.georadiusbymember = process_cmd.set
process_cmd.get = process_cmd.set
process_cmd.getbit = process_cmd.set
process_cmd.getrange = process_cmd.set
process_cmd.getset = process_cmd.set
process_cmd.hdel = process_cmd.set
process_cmd.hexists = process_cmd.set
process_cmd.hget = process_cmd.set
process_cmd.hgetall = process_cmd.set
process_cmd.hincrby = process_cmd.set
process_cmd.hincrbyfloat = process_cmd.set
process_cmd.hkeys = process_cmd.set
process_cmd.hlen = process_cmd.set
process_cmd.hmget = process_cmd.set
2018-01-29 14:06:19 +01:00
process_cmd.hmset = process_cmd.set
2017-09-19 17:52:26 +02:00
process_cmd.hscan = process_cmd.set
process_cmd.hset = process_cmd.set
process_cmd.hsetnx = process_cmd.set
process_cmd.hstrlen = process_cmd.set
process_cmd.hvals = process_cmd.set
process_cmd.incr = process_cmd.set
process_cmd.incrby = process_cmd.set
process_cmd.incrbyfloat = process_cmd.set
process_cmd.info = process_cmd.script
process_cmd.keys = process_cmd.script
process_cmd.lastsave = process_cmd.script
process_cmd.lindex = process_cmd.set
process_cmd.linsert = process_cmd.set
process_cmd.llen = process_cmd.set
process_cmd.lpop = process_cmd.set
process_cmd.lpush = process_cmd.set
process_cmd.lpushx = process_cmd.set
process_cmd.lrange = process_cmd.set
process_cmd.lrem = process_cmd.set
process_cmd.lset = process_cmd.set
process_cmd.ltrim = process_cmd.set
process_cmd.migrate = process_cmd.script
process_cmd.monitor = process_cmd.script
process_cmd.move = process_cmd.set
process_cmd.msetnx = process_cmd.mset
process_cmd.multi = process_cmd.script
process_cmd.object = process_cmd.script
process_cmd.persist = process_cmd.set
process_cmd.pexpire = process_cmd.set
process_cmd.pexpireat = process_cmd.set
process_cmd.pfadd = process_cmd.set
process_cmd.pfcount = process_cmd.set
process_cmd.pfmerge = process_cmd.mget
process_cmd.ping = process_cmd.script
process_cmd.psetex = process_cmd.set
process_cmd.psubscribe = process_cmd.script
process_cmd.pubsub = process_cmd.script
process_cmd.pttl = process_cmd.set
process_cmd.publish = process_cmd.script
process_cmd.punsubscribe = process_cmd.script
process_cmd.quit = process_cmd.script
process_cmd.randomkey = process_cmd.script
process_cmd.readonly = process_cmd.script
process_cmd.readwrite = process_cmd.script
process_cmd.rename = process_cmd.mget
process_cmd.renamenx = process_cmd.mget
process_cmd.restore = process_cmd.set
process_cmd.role = process_cmd.script
process_cmd.rpop = process_cmd.set
process_cmd.rpoplpush = process_cmd.mget
process_cmd.rpush = process_cmd.set
process_cmd.rpushx = process_cmd.set
process_cmd.sadd = process_cmd.set
process_cmd.save = process_cmd.script
process_cmd.scard = process_cmd.set
process_cmd.sdiff = process_cmd.mget
process_cmd.select = process_cmd.script
process_cmd.setbit = process_cmd.set
process_cmd.setex = process_cmd.set
process_cmd.setnx = process_cmd.set
process_cmd.sinterstore = process_cmd.sdiff
process_cmd.sismember = process_cmd.set
process_cmd.slaveof = process_cmd.script
process_cmd.slowlog = process_cmd.script
process_cmd.smembers = process_cmd.script
process_cmd.sort = process_cmd.set
process_cmd.spop = process_cmd.set
process_cmd.srandmember = process_cmd.set
process_cmd.srem = process_cmd.set
process_cmd.strlen = process_cmd.set
process_cmd.subscribe = process_cmd.script
process_cmd.sunion = process_cmd.mget
process_cmd.sunionstore = process_cmd.mget
process_cmd.swapdb = process_cmd.script
process_cmd.sync = process_cmd.script
process_cmd.time = process_cmd.script
process_cmd.touch = process_cmd.mget
process_cmd.ttl = process_cmd.set
process_cmd.type = process_cmd.set
process_cmd.unsubscribe = process_cmd.script
process_cmd.unlink = process_cmd.mget
process_cmd.unwatch = process_cmd.script
process_cmd.wait = process_cmd.script
process_cmd.watch = process_cmd.mget
process_cmd.zadd = process_cmd.set
process_cmd.zcard = process_cmd.set
process_cmd.zcount = process_cmd.set
process_cmd.zincrby = process_cmd.set
process_cmd.zinterstore = process_cmd.eval
process_cmd.zlexcount = process_cmd.set
process_cmd.zrange = process_cmd.set
process_cmd.zrangebylex = process_cmd.set
process_cmd.zrank = process_cmd.set
process_cmd.zrem = process_cmd.set
process_cmd.zrembylex = process_cmd.set
process_cmd.zrembyrank = process_cmd.set
process_cmd.zrembyscore = process_cmd.set
process_cmd.zrevrange = process_cmd.set
process_cmd.zrevrangebyscore = process_cmd.set
process_cmd.zrevrank = process_cmd.set
process_cmd.zscore = process_cmd.set
process_cmd.zunionstore = process_cmd.eval
process_cmd.scan = process_cmd.script
process_cmd.sscan = process_cmd.set
process_cmd.hscan = process_cmd.set
process_cmd.zscan = process_cmd.set
local function get_key_indexes ( cmd , args )
local idx_l = { }
cmd = string.lower ( cmd )
if process_cmd [ cmd ] then
idx_l = process_cmd [ cmd ] ( args )
else
logger.warnx ( rspamd_config , " Don't know how to extract keys for %s Redis command " , cmd )
end
return idx_l
end
2017-12-01 18:06:10 +01:00
local gen_meta = {
principal_recipient = function ( task )
return task : get_principal_recipient ( )
end ,
principal_recipient_domain = function ( task )
local p = task : get_principal_recipient ( )
if not p then return end
return string.match ( p , ' .*@(.*) ' )
end ,
ip = function ( task )
local i = task : get_ip ( )
if i and i : is_valid ( ) then return i : to_string ( ) end
end ,
from = function ( task )
return ( ( task : get_from ( ' smtp ' ) or E ) [ 1 ] or E ) [ ' addr ' ]
end ,
from_domain = function ( task )
return ( ( task : get_from ( ' smtp ' ) or E ) [ 1 ] or E ) [ ' domain ' ]
end ,
from_domain_or_helo_domain = function ( task )
local d = ( ( task : get_from ( ' smtp ' ) or E ) [ 1 ] or E ) [ ' domain ' ]
if d and # d > 0 then return d end
return task : get_helo ( )
end ,
mime_from = function ( task )
return ( ( task : get_from ( ' mime ' ) or E ) [ 1 ] or E ) [ ' addr ' ]
end ,
mime_from_domain = function ( task )
return ( ( task : get_from ( ' mime ' ) or E ) [ 1 ] or E ) [ ' domain ' ]
end ,
}
2017-09-19 17:52:26 +02:00
2017-12-01 18:06:10 +01:00
local function gen_get_esld ( f )
return function ( task )
local d = f ( task )
if not d then return end
return rspamd_util.get_tld ( d )
end
end
gen_meta.smtp_from = gen_meta.from
gen_meta.smtp_from_domain = gen_meta.from_domain
gen_meta.smtp_from_domain_or_helo_domain = gen_meta.from_domain_or_helo_domain
gen_meta.esld_principal_recipient_domain = gen_get_esld ( gen_meta.principal_recipient_domain )
gen_meta.esld_from_domain = gen_get_esld ( gen_meta.from_domain )
gen_meta.esld_smtp_from_domain = gen_meta.esld_from_domain
gen_meta.esld_mime_from_domain = gen_get_esld ( gen_meta.mime_from_domain )
gen_meta.esld_from_domain_or_helo_domain = gen_get_esld ( gen_meta.from_domain_or_helo_domain )
gen_meta.esld_smtp_from_domain_or_helo_domain = gen_meta.esld_from_domain_or_helo_domain
local function get_key_expansion_metadata ( task )
2017-09-19 17:52:26 +02:00
local md_mt = {
__index = function ( self , k )
k = string.lower ( k )
local v = rawget ( self , k )
if v then
return v
end
if gen_meta [ k ] then
2017-12-01 18:06:10 +01:00
v = gen_meta [ k ] ( task )
2017-09-19 17:52:26 +02:00
rawset ( self , k , v )
end
return v
end ,
}
local lazy_meta = { }
setmetatable ( lazy_meta , md_mt )
return lazy_meta
end
2017-05-23 12:37:43 +02:00
-- Performs async call to redis hiding all complexity inside function
-- task - rspamd_task
-- redis_params - valid params returned by rspamd_parse_redis_server
-- key - key to select upstream or nil to select round-robin/master-slave
-- is_write - true if need to write to redis server
-- callback - function to be called upon request is completed
-- command - redis command
-- args - table of arguments
2018-02-14 11:57:47 +01:00
-- extra_opts - table of optional request arguments
local function rspamd_redis_make_request ( task , redis_params , key , is_write ,
callback , command , args , extra_opts )
2017-05-23 12:37:43 +02:00
local addr
local function rspamd_redis_make_request_cb ( err , data )
if err then
addr : fail ( )
else
addr : ok ( )
end
2019-07-27 12:02:53 +02:00
if callback then
callback ( err , data , addr )
end
2017-05-23 12:37:43 +02:00
end
if not task or not redis_params or not callback or not command then
return false , nil , nil
end
local rspamd_redis = require " rspamd_redis "
if key then
if is_write then
addr = redis_params [ ' write_servers ' ] : get_upstream_by_hash ( key )
else
addr = redis_params [ ' read_servers ' ] : get_upstream_by_hash ( key )
end
else
if is_write then
addr = redis_params [ ' write_servers ' ] : get_upstream_master_slave ( key )
else
addr = redis_params [ ' read_servers ' ] : get_upstream_round_robin ( key )
end
end
if not addr then
logger.errx ( task , ' cannot select server to make redis request ' )
end
2017-09-19 17:52:26 +02:00
if redis_params [ ' expand_keys ' ] then
local m = get_key_expansion_metadata ( task )
local indexes = get_key_indexes ( command , args )
for _ , i in ipairs ( indexes ) do
args [ i ] = lutil.template ( args [ i ] , m )
end
end
2017-11-25 13:50:02 +01:00
local ip_addr = addr : get_addr ( )
2017-05-23 12:37:43 +02:00
local options = {
task = task ,
callback = rspamd_redis_make_request_cb ,
2017-11-25 13:50:02 +01:00
host = ip_addr ,
2017-05-23 12:37:43 +02:00
timeout = redis_params [ ' timeout ' ] ,
cmd = command ,
args = args
}
2018-02-14 11:57:47 +01:00
if extra_opts then
for k , v in pairs ( extra_opts ) do
options [ k ] = v
end
end
2017-05-23 12:37:43 +02:00
if redis_params [ ' password ' ] then
options [ ' password ' ] = redis_params [ ' password ' ]
end
if redis_params [ ' db ' ] then
options [ ' dbname ' ] = redis_params [ ' db ' ]
end
2018-10-25 15:21:42 +02:00
lutil.debugm ( N , task , ' perform request to redis server ' ..
2019-07-15 16:43:15 +02:00
' (host=%s, timeout=%s): cmd: %s ' , ip_addr ,
options.timeout , options.cmd )
2018-10-25 15:21:42 +02:00
2017-05-23 12:37:43 +02:00
local ret , conn = rspamd_redis.make_request ( options )
2017-11-25 13:50:02 +01:00
if not ret then
addr : fail ( )
logger.warnx ( task , " cannot make redis request to: %s " , tostring ( ip_addr ) )
end
2017-05-23 12:37:43 +02:00
return ret , conn , addr
end
2018-02-23 14:32:17 +01:00
--[[[
-- @function lua_redis.redis_make_request(task, redis_params, key, is_write, callback, command, args)
-- Sends a request to Redis
-- @param {rspamd_task} task task object
-- @param {table} redis_params redis configuration in format returned by lua_redis.parse_redis_server()
-- @param {string} key key to use for sharding
-- @param {boolean} is_write should be `true` if we are performing a write operating
-- @param {function} callback callback function (first parameter is error if applicable, second is a 2D array (table))
-- @param {string} command Redis command to run
-- @param {table} args Numerically indexed table containing arguments for command
--]]
2017-05-23 12:37:43 +02:00
exports.rspamd_redis_make_request = rspamd_redis_make_request
exports.redis_make_request = rspamd_redis_make_request
2018-02-14 11:57:47 +01:00
local function redis_make_request_taskless ( ev_base , cfg , redis_params , key ,
is_write , callback , command , args , extra_opts )
2017-05-23 12:37:43 +02:00
if not ev_base or not redis_params or not callback or not command then
return false , nil , nil
end
local addr
2017-11-26 22:48:16 +01:00
local function rspamd_redis_make_request_cb ( err , data )
if err then
addr : fail ( )
else
addr : ok ( )
end
2019-07-27 12:02:53 +02:00
if callback then
callback ( err , data , addr )
end
2017-11-26 22:48:16 +01:00
end
2017-05-23 12:37:43 +02:00
local rspamd_redis = require " rspamd_redis "
if key then
if is_write then
addr = redis_params [ ' write_servers ' ] : get_upstream_by_hash ( key )
else
addr = redis_params [ ' read_servers ' ] : get_upstream_by_hash ( key )
end
else
if is_write then
addr = redis_params [ ' write_servers ' ] : get_upstream_master_slave ( key )
else
addr = redis_params [ ' read_servers ' ] : get_upstream_round_robin ( key )
end
end
if not addr then
logger.errx ( cfg , ' cannot select server to make redis request ' )
end
local options = {
ev_base = ev_base ,
config = cfg ,
2017-11-26 22:48:16 +01:00
callback = rspamd_redis_make_request_cb ,
2017-05-23 12:37:43 +02:00
host = addr : get_addr ( ) ,
timeout = redis_params [ ' timeout ' ] ,
cmd = command ,
args = args
}
2018-02-14 11:57:47 +01:00
if extra_opts then
for k , v in pairs ( extra_opts ) do
options [ k ] = v
end
end
2017-05-23 12:37:43 +02:00
if redis_params [ ' password ' ] then
options [ ' password ' ] = redis_params [ ' password ' ]
end
if redis_params [ ' db ' ] then
options [ ' dbname ' ] = redis_params [ ' db ' ]
end
2018-10-26 16:27:47 +02:00
lutil.debugm ( N , cfg , ' perform taskless request to redis server ' ..
2019-07-15 16:43:15 +02:00
' (host=%s, timeout=%s): cmd: %s ' , options.host ,
options.timeout , options.cmd )
2017-05-23 12:37:43 +02:00
local ret , conn = rspamd_redis.make_request ( options )
if not ret then
logger.errx ( ' cannot execute redis request ' )
2017-11-25 13:50:02 +01:00
addr : fail ( )
2017-05-23 12:37:43 +02:00
end
2017-11-26 22:48:16 +01:00
2017-05-23 12:37:43 +02:00
return ret , conn , addr
end
2018-02-23 14:32:17 +01:00
--[[[
-- @function lua_redis.redis_make_request_taskless(ev_base, redis_params, key, is_write, callback, command, args)
-- Sends a request to Redis in context where `task` is not available for some specific use-cases
-- Identical to redis_make_request() except in that first parameter is an `event base` object
--]]
2017-05-23 12:37:43 +02:00
exports.rspamd_redis_make_request_taskless = redis_make_request_taskless
exports.redis_make_request_taskless = redis_make_request_taskless
2017-12-16 16:40:37 +01:00
local redis_scripts = {
}
2017-12-16 19:03:38 +01:00
local function script_set_loaded ( script )
if script.sha then
script.loaded = true
2017-12-16 16:40:37 +01:00
end
2017-12-16 19:03:38 +01:00
local wait_table = { }
for _ , s in ipairs ( script.waitq ) do
table.insert ( wait_table , s )
end
2017-12-16 16:40:37 +01:00
2017-12-16 19:03:38 +01:00
script.waitq = { }
2017-12-16 16:40:37 +01:00
2017-12-16 19:03:38 +01:00
for _ , s in ipairs ( wait_table ) do
s ( script.loaded )
end
end
2017-12-16 16:40:37 +01:00
2017-12-16 19:03:38 +01:00
local function prepare_redis_call ( script )
local function merge_tables ( t1 , t2 )
for k , v in pairs ( t2 ) do t1 [ k ] = v end
2017-12-16 16:40:37 +01:00
end
2017-12-16 19:03:38 +01:00
2017-12-16 16:40:37 +01:00
local servers = { }
2017-12-16 19:03:38 +01:00
local options = { }
2017-12-16 16:40:37 +01:00
if script.redis_params . read_servers then
merge_tables ( servers , script.redis_params . read_servers : all_upstreams ( ) )
end
if script.redis_params . write_servers then
merge_tables ( servers , script.redis_params . write_servers : all_upstreams ( ) )
end
-- Call load script on each server, set loaded flag
script.in_flight = # servers
for _ , s in ipairs ( servers ) do
2017-12-16 19:03:38 +01:00
local cur_opts = {
host = s : get_addr ( ) ,
timeout = script.redis_params [ ' timeout ' ] ,
cmd = ' SCRIPT ' ,
args = { ' LOAD ' , script.script } ,
upstream = s
}
if script.redis_params [ ' password ' ] then
cur_opts [ ' password ' ] = script.redis_params [ ' password ' ]
end
if script.redis_params [ ' db ' ] then
cur_opts [ ' dbname ' ] = script.redis_params [ ' db ' ]
end
table.insert ( options , cur_opts )
end
return options
end
local function load_script_task ( script , task )
local rspamd_redis = require " rspamd_redis "
local opts = prepare_redis_call ( script )
for _ , opt in ipairs ( opts ) do
opt.task = task
opt.callback = function ( err , data )
2017-12-16 16:40:37 +01:00
if err then
2019-07-07 10:44:45 +02:00
logger.errx ( task , ' cannot upload script to %s: %s; registered from: %s:%s ' ,
2019-07-19 17:51:53 +02:00
opt.upstream : get_addr ( ) : to_string ( true ) ,
err , script.caller . short_src , script.caller . currentline )
2017-12-16 19:03:38 +01:00
opt.upstream : fail ( )
2018-03-12 14:12:06 +01:00
script.fatal_error = err
2017-12-16 16:40:37 +01:00
else
2017-12-16 19:03:38 +01:00
opt.upstream : ok ( )
logger.infox ( task ,
2018-02-19 17:19:18 +01:00
" uploaded redis script to %s with id %s, sha: %s " ,
2019-07-19 17:51:53 +02:00
opt.upstream : get_addr ( ) : to_string ( true ) ,
script.id , data )
2017-12-16 16:40:37 +01:00
script.sha = data -- We assume that sha is the same on all servers
end
script.in_flight = script.in_flight - 1
if script.in_flight == 0 then
2017-12-16 19:03:38 +01:00
script_set_loaded ( script )
2017-12-16 16:40:37 +01:00
end
end
2017-12-16 19:03:38 +01:00
local ret = rspamd_redis.make_request ( opt )
2017-12-16 16:40:37 +01:00
2017-12-16 19:03:38 +01:00
if not ret then
2018-02-19 17:19:18 +01:00
logger.errx ( ' cannot execute redis request to load script on %s ' ,
opt.upstream : get_addr ( ) )
2017-12-16 19:03:38 +01:00
script.in_flight = script.in_flight - 1
opt.upstream : fail ( )
end
2017-12-16 16:40:37 +01:00
2017-12-16 19:03:38 +01:00
if script.in_flight == 0 then
script_set_loaded ( script )
2017-12-16 16:40:37 +01:00
end
2017-12-16 19:03:38 +01:00
end
end
2017-12-16 16:40:37 +01:00
2017-12-16 19:03:38 +01:00
local function load_script_taskless ( script , cfg , ev_base )
local rspamd_redis = require " rspamd_redis "
local opts = prepare_redis_call ( script )
for _ , opt in ipairs ( opts ) do
opt.config = cfg
opt.ev_base = ev_base
opt.callback = function ( err , data )
if err then
2019-07-07 10:44:45 +02:00
logger.errx ( cfg , ' cannot upload script to %s: %s; registered from: %s:%s ' ,
2019-07-19 17:51:53 +02:00
opt.upstream : get_addr ( ) : to_string ( true ) ,
err , script.caller . short_src , script.caller . currentline )
2017-12-16 19:03:38 +01:00
opt.upstream : fail ( )
2018-03-12 14:12:06 +01:00
script.fatal_error = err
2017-12-16 19:03:38 +01:00
else
opt.upstream : ok ( )
logger.infox ( cfg ,
2018-02-19 17:19:18 +01:00
" uploaded redis script to %s with id %s, sha: %s " ,
2019-07-19 17:51:53 +02:00
opt.upstream : get_addr ( ) : to_string ( true ) , script.id , data )
2017-12-16 19:03:38 +01:00
script.sha = data -- We assume that sha is the same on all servers
2018-04-20 16:39:11 +02:00
script.fatal_error = nil
2017-12-16 19:03:38 +01:00
end
script.in_flight = script.in_flight - 1
if script.in_flight == 0 then
script_set_loaded ( script )
end
2017-12-16 16:40:37 +01:00
end
2017-12-16 19:03:38 +01:00
local ret = rspamd_redis.make_request ( opt )
2017-12-16 16:40:37 +01:00
if not ret then
2018-02-19 17:19:18 +01:00
logger.errx ( ' cannot execute redis request to load script on %s ' ,
opt.upstream : get_addr ( ) )
2017-12-16 16:40:37 +01:00
script.in_flight = script.in_flight - 1
2017-12-16 19:03:38 +01:00
opt.upstream : fail ( )
2017-12-16 16:40:37 +01:00
end
2017-12-16 19:03:38 +01:00
if script.in_flight == 0 then
script_set_loaded ( script )
end
2017-12-16 16:40:37 +01:00
end
end
2017-12-16 19:03:38 +01:00
local function load_redis_script ( script , cfg , ev_base , _ )
2018-03-09 14:00:18 +01:00
if script.redis_params then
load_script_taskless ( script , cfg , ev_base )
end
2017-12-16 19:03:38 +01:00
end
2017-12-16 16:40:37 +01:00
local function add_redis_script ( script , redis_params )
2019-07-07 10:44:45 +02:00
local caller = debug.getinfo ( 2 )
2017-12-16 16:40:37 +01:00
local new_script = {
2019-07-07 10:44:45 +02:00
caller = caller ,
2017-12-16 16:40:37 +01:00
loaded = false ,
redis_params = redis_params ,
script = script ,
waitq = { } , -- callbacks pending for script being loaded
id = # redis_scripts + 1
}
-- Register on load function
rspamd_config : add_on_load ( function ( cfg , ev_base , worker )
2018-04-20 16:39:11 +02:00
local mult = 0.0
rspamd_config : add_periodic ( ev_base , 0.0 , function ( )
if not new_script.sha then
load_redis_script ( new_script , cfg , ev_base , worker )
mult = mult + 1
return 1.0 * mult -- Check one more time in one second
end
return false
end , false )
2017-12-16 16:40:37 +01:00
end )
table.insert ( redis_scripts , new_script )
return # redis_scripts
end
exports.add_redis_script = add_redis_script
2018-02-27 15:35:29 +01:00
local function exec_redis_script ( id , params , callback , keys , args )
2018-02-28 15:15:46 +01:00
local redis_args = { }
2017-12-16 19:03:38 +01:00
2017-12-16 16:40:37 +01:00
if not redis_scripts [ id ] then
2017-12-16 19:03:38 +01:00
logger.errx ( " cannot find registered script with id %s " , id )
2017-12-16 16:40:37 +01:00
return false
end
2018-03-12 14:12:06 +01:00
2017-12-16 16:40:37 +01:00
local script = redis_scripts [ id ]
2018-03-12 14:12:06 +01:00
if script.fatal_error then
callback ( script.fatal_error , nil )
return true
end
2018-03-09 14:00:18 +01:00
if not script.redis_params then
callback ( ' no redis servers defined ' , nil )
return true
end
2017-12-16 16:40:37 +01:00
2017-12-16 19:03:38 +01:00
local function do_call ( can_reload )
2017-12-16 16:40:37 +01:00
local function redis_cb ( err , data )
if not err then
callback ( err , data )
2017-12-16 19:03:38 +01:00
elseif string.match ( err , ' NOSCRIPT ' ) then
2017-12-16 16:40:37 +01:00
-- Schedule restart
2017-12-16 19:03:38 +01:00
script.sha = nil
if can_reload then
table.insert ( script.waitq , do_call )
if script.in_flight == 0 then
-- Reload scripts if this has not been initiated yet
if params.task then
load_script_task ( script , params.task )
else
load_script_taskless ( script , rspamd_config , params.ev_base )
end
2017-12-16 16:40:37 +01:00
end
2017-12-16 19:03:38 +01:00
else
callback ( err , data )
2017-12-16 16:40:37 +01:00
end
else
callback ( err , data )
end
end
2018-02-28 15:15:46 +01:00
if # redis_args == 0 then
table.insert ( redis_args , script.sha )
table.insert ( redis_args , tostring ( # keys ) )
for _ , k in ipairs ( keys ) do
table.insert ( redis_args , k )
end
2018-02-27 15:35:29 +01:00
if type ( args ) == ' table ' then
for _ , a in ipairs ( args ) do
2018-02-28 15:15:46 +01:00
table.insert ( redis_args , a )
2018-02-27 14:00:29 +01:00
end
end
2017-12-16 19:03:38 +01:00
end
2017-12-16 16:40:37 +01:00
if params.task then
if not rspamd_redis_make_request ( params.task , script.redis_params ,
2018-02-28 15:15:46 +01:00
params.key , params.is_write , redis_cb , ' EVALSHA ' , redis_args ) then
2017-12-16 16:40:37 +01:00
callback ( ' Cannot make redis request ' , nil )
end
else
if not redis_make_request_taskless ( params.ev_base , rspamd_config ,
script.redis_params ,
2018-02-28 15:15:46 +01:00
params.key , params.is_write , redis_cb , ' EVALSHA ' , redis_args ) then
2017-12-16 16:40:37 +01:00
callback ( ' Cannot make redis request ' , nil )
end
end
end
2017-12-16 19:03:38 +01:00
if script.loaded then
do_call ( true )
2017-12-16 16:40:37 +01:00
else
-- Delayed until scripts are loaded
2017-12-16 19:03:38 +01:00
if not params.task then
table.insert ( script.waitq , do_call )
else
-- TODO: fix taskfull requests
2019-07-19 19:36:47 +02:00
table.insert ( script.waitq , function ( )
if script.loaded then
do_call ( false )
else
callback ( ' NOSCRIPT ' , nil )
end
end )
load_script_task ( script , params.task )
2017-12-16 19:03:38 +01:00
end
2017-12-16 16:40:37 +01:00
end
return true
end
exports.exec_redis_script = exec_redis_script
2018-09-10 16:17:14 +02:00
local function redis_connect_sync ( redis_params , is_write , key , cfg , ev_base )
2018-02-12 13:15:55 +01:00
if not redis_params then
return false , nil
end
local rspamd_redis = require " rspamd_redis "
local addr
if key then
if is_write then
addr = redis_params [ ' write_servers ' ] : get_upstream_by_hash ( key )
else
addr = redis_params [ ' read_servers ' ] : get_upstream_by_hash ( key )
end
else
if is_write then
addr = redis_params [ ' write_servers ' ] : get_upstream_master_slave ( key )
else
addr = redis_params [ ' read_servers ' ] : get_upstream_round_robin ( key )
end
end
if not addr then
logger.errx ( cfg , ' cannot select server to make redis request ' )
end
local options = {
host = addr : get_addr ( ) ,
timeout = redis_params [ ' timeout ' ] ,
2018-10-01 11:41:51 +02:00
config = cfg or rspamd_config ,
ev_base = ev_base or rspamadm_ev_base ,
session = redis_params.session or rspamadm_session
2018-02-12 13:15:55 +01:00
}
2018-09-10 16:17:14 +02:00
for k , v in pairs ( redis_params ) do
options [ k ] = v
end
2018-02-12 13:15:55 +01:00
2018-10-01 11:41:51 +02:00
if not options.config then
logger.errx ( ' config is not set ' )
return false , nil , addr
end
if not options.ev_base then
logger.errx ( ' ev_base is not set ' )
return false , nil , addr
end
if not options.session then
logger.errx ( ' session is not set ' )
return false , nil , addr
end
2018-02-12 13:15:55 +01:00
local ret , conn = rspamd_redis.connect_sync ( options )
if not ret then
logger.errx ( ' cannot execute redis request: %s ' , conn )
addr : fail ( )
2018-02-21 16:45:12 +01:00
return false , nil , addr
2018-02-12 13:15:55 +01:00
end
if conn then
if redis_params [ ' password ' ] then
conn : add_cmd ( ' AUTH ' , { redis_params [ ' password ' ] } )
end
if redis_params [ ' db ' ] then
conn : add_cmd ( ' SELECT ' , { tostring ( redis_params [ ' db ' ] ) } )
elseif redis_params [ ' dbname ' ] then
conn : add_cmd ( ' SELECT ' , { tostring ( redis_params [ ' dbname ' ] ) } )
end
end
return ret , conn , addr
end
exports.redis_connect_sync = redis_connect_sync
2018-09-11 12:02:30 +02:00
--[[[
-- @function lua_redis.request(redis_params, attrs, req)
2018-09-11 17:24:22 +02:00
-- Sends a request to Redis synchronously with coroutines or asynchronously using
-- a callback (modern API)
2018-09-11 12:02:30 +02:00
-- @param redis_params a table of redis server parameters
-- @param attrs a table of redis request attributes (e.g. task, or ev_base + cfg + session)
-- @param req a table of request: a command + command options
2018-09-11 17:24:22 +02:00
-- @return {result,data/connection,address} boolean result, connection object in case of async request and results if using coroutines, redis server address
2018-09-11 12:02:30 +02:00
--]]
exports.request = function ( redis_params , attrs , req )
local lua_util = require " lua_util "
if not attrs or not redis_params or not req then
logger.errx ( ' invalid arguments for redis request ' )
return false , nil , nil
end
if not ( attrs.task or ( attrs.config and attrs.ev_base ) ) then
logger.errx ( ' invalid attributes for redis request ' )
return false , nil , nil
end
local opts = lua_util.shallowcopy ( attrs )
local log_obj = opts.task or opts.config
local addr
if opts.callback then
-- Wrap callback
local callback = opts.callback
local function rspamd_redis_make_request_cb ( err , data )
if err then
addr : fail ( )
else
addr : ok ( )
end
callback ( err , data , addr )
end
opts.callback = rspamd_redis_make_request_cb
end
local rspamd_redis = require " rspamd_redis "
local is_write = opts.is_write
if opts.key then
if is_write then
addr = redis_params [ ' write_servers ' ] : get_upstream_by_hash ( attrs.key )
else
addr = redis_params [ ' read_servers ' ] : get_upstream_by_hash ( attrs.key )
end
else
if is_write then
addr = redis_params [ ' write_servers ' ] : get_upstream_master_slave ( attrs.key )
else
addr = redis_params [ ' read_servers ' ] : get_upstream_round_robin ( attrs.key )
end
end
if not addr then
logger.errx ( log_obj , ' cannot select server to make redis request ' )
end
opts.host = addr : get_addr ( )
opts.timeout = redis_params.timeout
if type ( req ) == ' string ' then
opts.cmd = req
else
-- XXX: modifies the input table
opts.cmd = table.remove ( req , 1 ) ;
opts.args = req
end
if redis_params.password then
opts.password = redis_params.password
end
if redis_params.db then
opts.dbname = redis_params.db
end
2018-10-25 15:21:42 +02:00
lutil.debugm ( N , ' perform generic request to redis server ' ..
' (host=%s, timeout=%s): cmd: %s, arguments: %s ' , addr ,
opts.timeout , opts.cmd , opts.args )
2018-09-11 17:24:22 +02:00
if opts.callback then
local ret , conn = rspamd_redis.make_request ( opts )
if not ret then
logger.errx ( log_obj , ' cannot execute redis request ' )
addr : fail ( )
end
return ret , conn , addr
else
-- Coroutines version
local ret , conn = rspamd_redis.connect_sync ( opts )
if not ret then
logger.errx ( log_obj , ' cannot execute redis request ' )
addr : fail ( )
else
conn : add_cmd ( opts.cmd , opts.args )
return conn : exec ( )
end
return false , nil , addr
2018-09-11 12:02:30 +02:00
end
2018-09-11 17:24:22 +02:00
end
2018-09-11 12:02:30 +02:00
2018-09-11 17:24:22 +02:00
--[[[
-- @function lua_redis.connect(redis_params, attrs)
-- Connects to Redis synchronously with coroutines or asynchronously using a callback (modern API)
-- @param redis_params a table of redis server parameters
-- @param attrs a table of redis request attributes (e.g. task, or ev_base + cfg + session)
-- @return {result,connection,address} boolean result, connection object, redis server address
--]]
exports.connect = function ( redis_params , attrs )
local lua_util = require " lua_util "
if not attrs or not redis_params then
logger.errx ( ' invalid arguments for redis connect ' )
return false , nil , nil
end
if not ( attrs.task or ( attrs.config and attrs.ev_base ) ) then
logger.errx ( ' invalid attributes for redis connect ' )
return false , nil , nil
end
local opts = lua_util.shallowcopy ( attrs )
local log_obj = opts.task or opts.config
local addr
if opts.callback then
-- Wrap callback
local callback = opts.callback
local function rspamd_redis_make_request_cb ( err , data )
if err then
addr : fail ( )
else
addr : ok ( )
end
callback ( err , data , addr )
end
opts.callback = rspamd_redis_make_request_cb
end
local rspamd_redis = require " rspamd_redis "
local is_write = opts.is_write
if opts.key then
if is_write then
addr = redis_params [ ' write_servers ' ] : get_upstream_by_hash ( attrs.key )
else
addr = redis_params [ ' read_servers ' ] : get_upstream_by_hash ( attrs.key )
end
else
if is_write then
addr = redis_params [ ' write_servers ' ] : get_upstream_master_slave ( attrs.key )
else
addr = redis_params [ ' read_servers ' ] : get_upstream_round_robin ( attrs.key )
end
end
if not addr then
logger.errx ( log_obj , ' cannot select server to make redis connect ' )
end
opts.host = addr : get_addr ( )
opts.timeout = redis_params.timeout
if redis_params.password then
opts.password = redis_params.password
end
if redis_params.db then
opts.dbname = redis_params.db
end
if opts.callback then
local ret , conn = rspamd_redis.connect ( opts )
if not ret then
logger.errx ( log_obj , ' cannot execute redis connect ' )
addr : fail ( )
end
return ret , conn , addr
else
-- Coroutines version
local ret , conn = rspamd_redis.connect_sync ( opts )
if not ret then
logger.errx ( log_obj , ' cannot execute redis connect ' )
addr : fail ( )
else
return true , conn , addr
end
return false , nil , addr
end
2018-09-11 12:02:30 +02:00
end
2019-07-05 12:40:10 +02:00
local redis_prefixes = { }
--[[[
-- @function lua_redis.register_prefix(prefix, module, description[, optional])
-- Register new redis prefix for documentation purposes
-- @param {string} prefix string prefix
-- @param {string} module module name
-- @param {string} description prefix description
-- @param {table} optional optional kv pairs (e.g. pattern)
--]]
local function register_prefix ( prefix , module , description , optional )
local pr = {
module = module ,
description = description
}
if optional and type ( optional ) == ' table ' then
for k , v in pairs ( optional ) do
pr [ k ] = v
end
end
redis_prefixes [ prefix ] = pr
end
exports.register_prefix = register_prefix
--[[[
-- @function lua_redis.prefixes([mname])
-- Returns prefixes for specific module (or all prefixes). Returns a table prefix -> table
--]]
exports.prefixes = function ( mname )
if not mname then
return redis_prefixes
else
local fun = require " fun "
return fun.totable ( fun.filter ( function ( _ , data ) return data.module == mname end ,
redis_prefixes ) )
end
end
2018-09-11 17:24:22 +02:00
2017-05-23 12:37:43 +02:00
return exports