From: Vsevolod Stakhov Date: Tue, 15 Sep 2009 14:42:05 +0000 (+0400) Subject: * Add lua plugin for checking received headers X-Git-Tag: 0.2.7~15 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=6b57031f55ef73619b763c98eadb1dbf82494eac;p=rspamd.git * Add lua plugin for checking received headers * Some tunes to lua API * Fix bug with http maps * Optimize installation and make custom prefix for configs --- diff --git a/CMakeLists.txt b/CMakeLists.txt index 7be951bed..4db76fbfc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,9 +28,8 @@ OPTION(ENABLE_GPERF_TOOLS "Enable google perftools [default: OFF]" # Build optimized code for following CPU (default i386) #SET(CPU_TUNE "i686") -# ################################################################################ -# Check Includes -# +############################# CONFIG SECTION ############################################# + INCLUDE(CheckIncludeFiles) INCLUDE(CheckFunctionExists) INCLUDE(CheckSymbolExists) @@ -281,28 +280,28 @@ CHECK_C_COMPILER_FLAG(-Wno-pointer-sign SUPPORT_WPOINTER_SIGN) SET(CMAKE_C_WARN_FLAGS "-W") IF(SUPPORT_WALL) -SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -Wall") + SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -Wall") ENDIF(SUPPORT_WALL) IF(SUPPORT_WPOINTER) -SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -Wpointer-arith") + SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -Wpointer-arith") ENDIF(SUPPORT_WPOINTER) IF(SUPPORT_WPARAM) -SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -Wno-unused-parameter") + SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -Wno-unused-parameter") ENDIF(SUPPORT_WPARAM) IF(SUPPORT_WFUNCTION) -SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -Wno-unused-function") + SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -Wno-unused-function") ENDIF(SUPPORT_WFUNCTION) IF(SUPPORT_WSIGNCOMPARE) -SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -Wno-sign-compare ") + SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -Wno-sign-compare ") ENDIF(SUPPORT_WSIGNCOMPARE) IF(SUPPORT_WUNUSED_VAR) -SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -Wunused-variable") + SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -Wunused-variable") ENDIF(SUPPORT_WUNUSED_VAR) IF(SUPPORT_WDECL) -SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -Wno-declaration-after-statement") + SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -Wno-declaration-after-statement") ENDIF(SUPPORT_WDECL) IF(SUPPORT_WPOINTER_SIGN) -SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -Wno-pointer-sign") + SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -Wno-pointer-sign") ENDIF(SUPPORT_WPOINTER_SIGN) SET(CMAKE_C_FLAGS "${CMAKE_C_OPT_FLAGS} ${CMAKE_C_FLAGS} ${CMAKE_C_WARN_FLAGS}") @@ -315,11 +314,19 @@ IF(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") IF(ENABLE_REDIRECTOR MATCHES "ON") CONFIGURE_FILE(freebsd/redirector.sh.in freebsd/redirector.sh @ONLY) ENDIF(ENABLE_REDIRECTOR MATCHES "ON") + IF(NOT ETC_PREFIX) + SET(ETC_PREFIX /usr/local/etc) + SET(PREFIX /usr/local) + ENDIF(NOT ETC_PREFIX) ENDIF(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") IF(CMAKE_SYSTEM_NAME STREQUAL "Linux") SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_GNU_SOURCE -DLINUX") + IF(NOT ETC_PREFIX) + SET(ETC_PREFIX /etc) + SET(PREFIX /usr) + ENDIF(NOT ETC_PREFIX) ENDIF(CMAKE_SYSTEM_NAME STREQUAL "Linux") IF(ENABLE_REDIRECTOR MATCHES "ON") @@ -330,6 +337,8 @@ IF(DEBUG_MODE MATCHES "ON") SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g") ENDIF(DEBUG_MODE MATCHES "ON") +################################ SOURCES SECTION ########################### + SET(RSPAMDSRC src/modules.c src/hash.c src/worker.c @@ -422,6 +431,8 @@ SET(LEX_SRC src/cfg_file.l) STRING(REPLACE "_file.y" "_yacc.c" YACC_OUTPUT ${YACC_SRC}) STRING(REPLACE "_file.l" "_lex.c" LEX_OUTPUT ${LEX_SRC}) +############################ TARGETS SECTION ############################### + ADD_CUSTOM_COMMAND(OUTPUT ${LEX_OUTPUT} COMMAND ${LEX_EXECUTABLE} -o${LEX_OUTPUT} ${LEX_SRC}) ADD_CUSTOM_COMMAND(OUTPUT ${YACC_OUTPUT} @@ -446,6 +457,7 @@ CONFIGURE_FILE(config.h.in src/config.h) CONFIGURE_FILE(rspamc.pl.in rspamc.pl @ONLY) CONFIGURE_FILE(rspamd.conf.sample conf/rspamd.conf.sample @ONLY) +######################### LINK SECTION ############################### ADD_EXECUTABLE(rspamd ${RSPAMDSRC} ${CONTRIBSRC} ${TOKENIZERSSRC} ${CLASSIFIERSSRC} ${PLUGINSSRC} ${YACC_OUTPUT} ${LEX_OUTPUT}) @@ -510,20 +522,65 @@ ENDIF(LIBUTIL_LIBRARY) TARGET_LINK_LIBRARIES(utils/expression-parser ${GLIB2_LIBRARIES}) TARGET_LINK_LIBRARIES(utils/expression-parser ${GMIME2_LIBRARIES}) +##################### INSTALLATION ########################################## +# Binaries INSTALL(PROGRAMS rspamd-${RSPAMD_VERSION} DESTINATION bin RENAME rspamd) INSTALL(PROGRAMS rspamc.pl DESTINATION bin RENAME rspamc) -INSTALL(CODE "EXECUTE_PROCESS(COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_PREFIX}/etc/rspamd)") -INSTALL(DIRECTORY conf/ DESTINATION etc/rspamd) + +# Configs +INSTALL(CODE "EXECUTE_PROCESS(COMMAND ${CMAKE_COMMAND} -E make_directory ${ETC_PREFIX}/rspamd)") + +INSTALL(FILES conf/2tld.inc RENAME 2tld.inc.orig DESTINATION ${ETC_PREFIX}/rspamd) +IF(NOT EXISTS ${ETC_PREFIX}/rspamd/2tld.inc) +INSTALL(FILES conf/2tld.inc DESTINATION ${ETC_PREFIX}/rspamd) +ENDIF(NOT EXISTS ${ETC_PREFIX}/rspamd/2tld.inc) + +INSTALL(FILES conf/drugs.inc RENAME drugs.inc.orig DESTINATION ${ETC_PREFIX}/rspamd) +IF(NOT EXISTS ${ETC_PREFIX}/rspamd/drugs.inc) +INSTALL(FILES conf/drugs.inc DESTINATION ${ETC_PREFIX}/rspamd) +ENDIF(NOT EXISTS ${ETC_PREFIX}/rspamd/drugs.inc) + +INSTALL(FILES conf/fraud.inc RENAME fraud.inc.orig DESTINATION ${ETC_PREFIX}/rspamd) +IF(NOT EXISTS ${ETC_PREFIX}/rspamd/fraud.inc) +INSTALL(FILES conf/fraud.inc DESTINATION ${ETC_PREFIX}/rspamd) +ENDIF(NOT EXISTS ${ETC_PREFIX}/rspamd/fraud.inc) + +INSTALL(FILES conf/html.inc RENAME html.inc.orig DESTINATION ${ETC_PREFIX}/rspamd) +IF(NOT EXISTS ${ETC_PREFIX}/rspamd/html.inc) +INSTALL(FILES conf/html.inc DESTINATION ${ETC_PREFIX}/rspamd) +ENDIF(NOT EXISTS ${ETC_PREFIX}/rspamd/html.inc) + +INSTALL(FILES conf/headers.inc RENAME headers.inc.orig DESTINATION ${ETC_PREFIX}/rspamd) +IF(NOT EXISTS ${ETC_PREFIX}/rspamd/headers.inc) +INSTALL(FILES conf/headers.inc DESTINATION ${ETC_PREFIX}/rspamd) +ENDIF(NOT EXISTS ${ETC_PREFIX}/rspamd/headers.inc) + +INSTALL(FILES conf/lotto.inc RENAME lotto.inc.orig DESTINATION ${ETC_PREFIX}/rspamd) +IF(NOT EXISTS ${ETC_PREFIX}/rspamd/lotto.inc) +INSTALL(FILES conf/lotto.inc DESTINATION ${ETC_PREFIX}/rspamd) +ENDIF(NOT EXISTS ${ETC_PREFIX}/rspamd/lotto.inc) + +INSTALL(FILES conf/rspamd.conf.sample DESTINATION ${ETC_PREFIX}/) + +# Lua plugins +IF(ENABLE_LUA MATCHES "ON") + INSTALL(CODE "EXECUTE_PROCESS(COMMAND ${CMAKE_COMMAND} -E make_directory ${ETC_PREFIX}/rspamd/plugins)") + INSTALL(DIRECTORY src/plugins/lua DESTINATION ${ETC_PREFIX}/rspamd/plugins PATTERN "*.lua") +ENDIF(ENABLE_LUA MATCHES "ON") + +# Perl lib IF(ENABLE_PERL MATCHES "ON") INSTALL(CODE "EXECUTE_PROCESS(COMMAND make install WORKING_DIRECTORY perl)") ENDIF(ENABLE_PERL MATCHES "ON") +# Redirector IF(ENABLE_REDIRECTOR MATCHES "ON") INSTALL(PROGRAMS utils/redirector.pl DESTINATION bin RENAME rspamd-redirector) ENDIF(ENABLE_REDIRECTOR MATCHES "ON") +# Start scripts IF(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") INSTALL(PROGRAMS freebsd/rspamd.sh DESTINATION etc/rc.d) IF(ENABLE_REDIRECTOR MATCHES "ON") diff --git a/conf/rspamd.conf.sample b/conf/rspamd.conf.sample index f38285ee5..1bc8aa6fa 100644 --- a/conf/rspamd.conf.sample +++ b/conf/rspamd.conf.sample @@ -2,11 +2,11 @@ # $Id$ # -.include /usr/local/etc/rspamd/drugs.inc -.include /usr/local/etc/rspamd/lotto.inc -.include /usr/local/etc/rspamd/fraud.inc -.include /usr/local/etc/rspamd/headers.inc -.include /usr/local/etc/rspamd/html.inc +.include /etc/rspamd/drugs.inc +.include /etc/rspamd/lotto.inc +.include /etc/rspamd/fraud.inc +.include /etc/rspamd/headers.inc +.include /etc/rspamd/html.inc # pidfile - path to pid file diff --git a/config.h.in b/config.h.in index e57df9beb..5a7439788 100644 --- a/config.h.in +++ b/config.h.in @@ -289,6 +289,7 @@ #endif #define CMAKE_PREFIX "${CMAKE_INSTALL_PREFIX}" +#define ETC_PREFIX "${ETC_PREFIX}" /* Forwarded declaration */ struct module_ctx; diff --git a/rspamd.conf.sample b/rspamd.conf.sample index da67662a1..9e735f43f 100644 --- a/rspamd.conf.sample +++ b/rspamd.conf.sample @@ -2,11 +2,11 @@ # $Id$ # -.include @CMAKE_INSTALL_PREFIX@/etc/rspamd/drugs.inc -.include @CMAKE_INSTALL_PREFIX@/etc/rspamd/lotto.inc -.include @CMAKE_INSTALL_PREFIX@/etc/rspamd/fraud.inc -.include @CMAKE_INSTALL_PREFIX@/etc/rspamd/headers.inc -.include @CMAKE_INSTALL_PREFIX@/etc/rspamd/html.inc +.include @ETC_PREFIX@/rspamd/drugs.inc +.include @ETC_PREFIX@/rspamd/lotto.inc +.include @ETC_PREFIX@/rspamd/fraud.inc +.include @ETC_PREFIX@/rspamd/headers.inc +.include @ETC_PREFIX@/rspamd/html.inc # pidfile - path to pid file diff --git a/src/lua/lua_task.c b/src/lua/lua_task.c index 07edcb044..844229f9d 100644 --- a/src/lua/lua_task.c +++ b/src/lua/lua_task.c @@ -105,14 +105,26 @@ static int lua_task_insert_result (lua_State *L) { struct worker_task *task = lua_check_task (L); - const char *metric_name, *symbol_name; + const char *metric_name, *symbol_name, *param; double flag; + GList *params = NULL; + int i, top; if (task != NULL) { metric_name = luaL_checkstring (L, 2); symbol_name = luaL_checkstring (L, 3); flag = luaL_checknumber (L, 4); - insert_result (task, metric_name, symbol_name, flag, NULL); + top = lua_gettop (L); + /* Get additional options */ + for (i = 5; i <= top; i ++) { + param = luaL_checkstring (L, i); + params = g_list_prepend (params, memory_pool_strdup (task->task_pool, param)); + } + + insert_result (task, metric_name, symbol_name, flag, params); + if (params != NULL) { + memory_pool_add_destructor (task->task_pool, (pool_destruct_func)g_list_free, params); + } } return 1; } diff --git a/src/main.h b/src/main.h index 4c1ab8617..0590c8e5c 100644 --- a/src/main.h +++ b/src/main.h @@ -19,7 +19,7 @@ #include "util.h" /* Default values */ -#define FIXED_CONFIG_FILE CMAKE_PREFIX "/etc/rspamd.conf" +#define FIXED_CONFIG_FILE ETC_PREFIX "/rspamd.conf" /* Time in seconds to exit for old worker */ #define SOFT_SHUTDOWN_TIME 60 /* Default metric name */ diff --git a/src/map.c b/src/map.c index d250970b1..f878e59b1 100644 --- a/src/map.c +++ b/src/map.c @@ -189,7 +189,7 @@ static int read_chunk_header (u_char *buf, size_t len, struct http_map_data *data) { u_char chunkbuf[32], *p, *c; - int skip; + int skip = 0; p = chunkbuf; c = buf; @@ -280,6 +280,9 @@ read_http_common (struct rspamd_map *map, struct http_map_data *data, struct htt memmove (buf, remain, rlen); r = rlen; } + if (r <= 0) { + return TRUE; + } if (reply->parser_state == 6) { if (reply->code != 200 && reply->code != 304) { msg_err ("read_http: got error reply from server %s, %d", data->host, reply->code); diff --git a/src/plugins/lua/received_rbl.lua b/src/plugins/lua/received_rbl.lua new file mode 100644 index 000000000..bee027f50 --- /dev/null +++ b/src/plugins/lua/received_rbl.lua @@ -0,0 +1,53 @@ +-- This plugin is designed for testing received headers via rbl +-- Configuration: +-- .module 'received_rbl' { +-- rbl = "insecure-bl.rambler.ru"; +-- rbl = "xbl.spamhaus.org"; +-- metric = "default"; +-- symbol = "RECEIVED_RBL"; +-- }; + + +metric = 'default' +symbol = 'RECEIVED_RBL' +rbls = {} + +function dns_cb(task, to_resolve, results, err) + if results then + local _,_,rbl = string.find(to_resolve, '%d+\.%d+\.%d+\.%d+\.(.+)') + task:insert_result(metric, symbol, 1, rbl) + end +end + +function received_cb (task) + local recvh = task:get_received_headers() + for _,rh in ipairs(recvh) do + for k,v in pairs(rh) do + if k == 'real_ip' then + local _,_,o1,o2,o3,o4 = string.find(v, '(%d+)\.(%d+)\.(%d+)\.(%d+)') + for _,rbl in ipairs(rbls) do + rbl_str = o4 .. '.' .. o3 .. '.' .. o2 .. '.' .. o1 .. '.' .. rbl + task:resolve_dns_a(rbl_str, 'dns_cb') + end + end + end + end +end + +-- Configuration +local opts = rspamd_config:get_all_opt('received_rbl') +if opts then + for n,v in pairs(opts) do + if n == 'rbl' then + table.insert(rbls, v) + elseif n == 'metric' then + metric = v + elseif n == 'symbol' then + symbol = v + end + end +end + +-- Register symbol's callback +local m = rspamd_config:get_metric(metric) +m:register_symbol(symbol, 1.0, 'received_cb')