From: Vsevolod Stakhov Date: Sun, 6 Sep 2009 15:55:23 +0000 (+0400) Subject: * Add ability to call rspamd fucntions from lua api X-Git-Tag: 0.2.7~23 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=d1b63d4cff12936a5ea4380bccadcc77b7c3ed3f;p=rspamd.git * Add ability to call rspamd fucntions from lua api * Make logging adaptive based on log speed (buffered vs unbuffered IO) * Fix lua API docs * Now lua modules can be loaded with glob patterns --- diff --git a/CMakeLists.txt b/CMakeLists.txt index 0db2b84d5..50355c61e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -228,6 +228,7 @@ CHECK_INCLUDE_FILES(locale.h HAVE_LOCALE_H) CHECK_INCLUDE_FILES(libgen.h HAVE_LIBGEN_H) CHECK_INCLUDE_FILES(pwd.h HAVE_PWD_H) CHECK_INCLUDE_FILES(grp.h HAVE_GRP_H) +CHECK_INCLUDE_FILES(glob.h HAVE_GLOB_H) IF(HAVE_SYS_WAIT_H) LIST(APPEND CMAKE_REQUIRED_INCLUDES sys/wait.h) diff --git a/config.h.in b/config.h.in index 0db410bb5..e57df9beb 100644 --- a/config.h.in +++ b/config.h.in @@ -47,6 +47,8 @@ #cmakedefine HAVE_GRP_H 1 #cmakedefine HAVE_PWD_H 1 +#cmakedefine HAVE_GLOB_H 1 + #cmakedefine HAVE_ENDIAN_H 1 #cmakedefine HAVE_SYS_ENDIAN_H 1 #cmakedefine HAVE_MACHINE_ENDIAN_H 1 @@ -254,6 +256,10 @@ #include #endif +#ifdef HAVE_GLOB_H +#include +#endif + /* syslog */ #ifdef HAVE_SYSLOG_H #include diff --git a/src/cfg_file.l b/src/cfg_file.l index ce207dbf5..1fcf000f4 100644 --- a/src/cfg_file.l +++ b/src/cfg_file.l @@ -10,7 +10,7 @@ #include "cfg_file.h" #include "cfg_yacc.h" #ifdef WITH_LUA -#include "lua.h" +extern void add_luabuf (const char *line); #else #define add_luabuf(x) yyerror ("lua support diabled") #endif @@ -52,8 +52,8 @@ protocol return PROTOCOL; memcached return MEMCACHED; servers return SERVERS; -require return REQUIRE; -module return MODULE; +modules return MODULES; +module_path return MODULE_PATH; filters return FILTERS; factors return FACTORS; diff --git a/src/cfg_file.y b/src/cfg_file.y index 777a9367e..46b3bf110 100644 --- a/src/cfg_file.y +++ b/src/cfg_file.y @@ -47,7 +47,7 @@ struct rspamd_view *cur_view = NULL; %token MAXSIZE SIZELIMIT SECONDS BEANSTALK MYSQL USER PASSWORD DATABASE %token TEMPDIR PIDFILE SERVERS ERROR_TIME DEAD_TIME MAXERRORS CONNECT_TIMEOUT PROTOCOL RECONNECT_TIMEOUT %token READ_SERVERS WRITE_SERVER DIRECTORY_SERVERS MAILBOX_QUERY USERS_QUERY LASTLOGIN_QUERY -%token MEMCACHED WORKER TYPE REQUIRE MODULE +%token MEMCACHED WORKER TYPE MODULES MODULE_PATH %token MODULE_OPT PARAM VARIABLE %token FILTERS FACTORS METRIC NAME CACHE_FILE %token REQUIRED_SCORE FUNCTION FRACT COMPOSITES CONTROL PASSWORD @@ -83,7 +83,7 @@ command : | pidfile | memcached | worker - | require + | modules | filters | module_opt | variable @@ -447,31 +447,58 @@ factorparam: g_hash_table_insert (cfg->factors, $1, tmp); }; -require: - REQUIRE OBRACE requirebody EBRACE +modules: + MODULES OBRACE modulesbody EBRACE ; -requirebody: - requirecmd SEMICOLON - | requirebody requirecmd SEMICOLON +modulesbody: + modulescmd SEMICOLON + | modulesbody modulescmd SEMICOLON ; -requirecmd: - MODULE EQSIGN QUOTEDSTRING { +modulescmd: + MODULE_PATH EQSIGN QUOTEDSTRING { #if !defined(WITHOUT_PERL) || defined(WITH_LUA) struct stat st; struct script_module *cur; + glob_t globbuf; + char *pattern; + size_t len; + int i; + if (stat ($3, &st) == -1) { - yyerror ("yyparse: cannot stat file %s, %s", $3, strerror (errno)); - YYERROR; - } - cur = memory_pool_alloc (cfg->cfg_pool, sizeof (struct script_module)); - if (cur == NULL) { - yyerror ("yyparse: g_malloc: %s", strerror(errno)); + yyerror ("yyparse: cannot stat path %s, %s", $3, strerror (errno)); YYERROR; } - cur->path = $3; - cfg->script_modules = g_list_prepend (cfg->script_modules, cur); + + globbuf.gl_offs = 0; + #ifdef WITH_LUA + len = strlen ($3) + sizeof ("*.lua"); + pattern = g_malloc (len); + snprintf (pattern, len, "%s%s", $3, "*.lua"); + #else + len = strlen ($3) + sizeof ("*.pl") + pattern = g_malloc (len); + snprintf (pattern, len, "%s%s", $3, "*.pl"); + #endif + + if (glob(pattern, GLOB_DOOFFS, NULL, &globbuf) == 0) { + for (i = 0; i < globbuf.gl_pathc; i ++) { + cur = memory_pool_alloc (cfg->cfg_pool, sizeof (struct script_module)); + if (cur == NULL) { + yyerror ("yyparse: g_malloc: %s", strerror(errno)); + YYERROR; + } + cur->path = memory_pool_strdup (cfg->cfg_pool, globbuf.gl_pathv[i]); + cfg->script_modules = g_list_prepend (cfg->script_modules, cur); + } + globfree (&globbuf); + } + else { + yyerror ("yyparse: glob: %s", strerror (errno)); + YYERROR; + } + g_free (pattern); #else yyerror ("require command is not available when perl support is not compiled"); YYERROR; diff --git a/src/lua/lua_task.c b/src/lua/lua_task.c index c688c7c00..07edcb044 100644 --- a/src/lua/lua_task.c +++ b/src/lua/lua_task.c @@ -25,6 +25,7 @@ #include "lua_common.h" #include "../message.h" +#include "../expressions.h" #include /* Task methods */ @@ -36,6 +37,7 @@ LUA_FUNCTION_DEF(task, get_raw_headers); LUA_FUNCTION_DEF(task, get_received_headers); LUA_FUNCTION_DEF(task, resolve_dns_a); LUA_FUNCTION_DEF(task, resolve_dns_ptr); +LUA_FUNCTION_DEF(task, call_rspamd_function); static const struct luaL_reg tasklib_m[] = { LUA_INTERFACE_DEF(task, get_message), @@ -46,6 +48,7 @@ static const struct luaL_reg tasklib_m[] = { LUA_INTERFACE_DEF(task, get_received_headers), LUA_INTERFACE_DEF(task, resolve_dns_a), LUA_INTERFACE_DEF(task, resolve_dns_ptr), + LUA_INTERFACE_DEF(task, call_rspamd_function), {"__tostring", lua_class_tostring}, {NULL, NULL} }; @@ -320,6 +323,43 @@ lua_task_resolve_dns_ptr (lua_State *L) return 0; } +static int +lua_task_call_rspamd_function (lua_State *L) +{ + struct worker_task *task = lua_check_task (L); + struct expression_function f; + int i, top; + gboolean res; + char *arg; + + if (task) { + f.name = (char *)luaL_checkstring (L, 2); + if (f.name) { + f.args = NULL; + top = lua_gettop (L); + /* Get arguments after function name */ + for (i = 3; i <= top; i ++) { + arg = (char *)luaL_checkstring (L, i); + if (arg != NULL) { + f.args = g_list_prepend (f.args, arg); + } + } + res = call_expression_function (&f, task); + lua_pushboolean (L, res); + if (f.args) { + g_list_free (f.args); + } + + return 1; + } + } + + lua_pushnil (L); + + return 1; + +} + /**** Textpart implementation *****/ diff --git a/src/lua/rspamd.luadoc b/src/lua/rspamd.luadoc index b83832a29..28f3f12f3 100644 --- a/src/lua/rspamd.luadoc +++ b/src/lua/rspamd.luadoc @@ -7,9 +7,8 @@ -- textpart - a single textual part of message module Rspamd ---- Each lua module MUST have init function that is called after config file was read: --- it should be like --- function module:module_init(cfg) +--- Each lua module has global rspamd_config that can be used for getting config +-- options and registering callbacks (via metric interface) ------------------------------------- CONFIG ----------------------------------------- -- @@ -57,6 +56,31 @@ function task:get_urls () -- @return array of textpart objects function task:get_text_parts () +--- Get raw headers +-- @return string that contains raw headers +function task:get_raw_headers () + +--- Get array of received headers +-- @return array of received headers that are tables itself +function task:get_received_headers () + +--- Resolve A record using rspamd async resolver +-- @param host host to resolve +-- @param callback name of callback function +function task:resolve_dns_a (host, callback) + +--- Resolve PTR record using rspamd async resolver +-- @param host host to resolve +-- @param callback name of callback function +function task:resolve_dns_ptr (host, callback) + +--- Callback function for dns resolving +-- @param task task object +-- @param to_resolve ptr or a record that was resolved +-- @param results results of dns query (array or nil) +-- @param err resolver error or nil +function dns_cb(task, to_resolve, results, err) + ------------------------------------- TEXTPART --------------------------------------- --- Get part's content diff --git a/src/util.c b/src/util.c index 3cd24a6eb..61cf56725 100644 --- a/src/util.c +++ b/src/util.c @@ -28,6 +28,11 @@ #include "cfg_file.h" #include "main.h" +/* Check log messages intensity once per minute */ +#define CHECK_TIME 60 +/* More than 2 log messages per second */ +#define BUF_INTENSITY 2 + #ifdef RSPAMD_MAIN sig_atomic_t do_reopen_log = 0; extern rspamd_hash_t *counters; @@ -40,6 +45,12 @@ struct logger_params { static struct logger_params log_params; +/* Here would be put log messages intensity */ +static uint32_t log_written; +static time_t last_check; +static char *io_buf = NULL; +static gboolean log_buffered = FALSE; + int make_socket_nonblocking (int fd) { @@ -651,6 +662,8 @@ open_log (struct config_file *cfg) return -1; } cfg->logf = fdopen (cfg->log_fd, "w"); + /* Set line buffering */ + setvbuf (cfg->logf, (char *) NULL, _IOLBF, 0); return 0; } return -1; @@ -752,9 +765,32 @@ file_log_function (const gchar *log_domain, GLogLevelFlags log_level, const gcha if (log_level <= cfg->log_level) { now = time (NULL); tms = localtime (&now); + + if (last_check == 0) { + last_check = now; + } + else if (now - last_check > CHECK_TIME) { + if (log_written / (now - last_check) > BUF_INTENSITY && !log_buffered) { + /* Switch to buffered logging */ + if (io_buf == NULL) { + io_buf = g_malloc (BUFSIZ); + } + setvbuf (cfg->logf, io_buf, _IOFBF, BUFSIZ); + log_buffered = TRUE; + } + else if (log_buffered) { + /* Switch to line buffering */ + setvbuf (cfg->logf, NULL, _IOLBF, 0); + log_buffered = FALSE; + } + last_check = now; + log_written = 0; + } + strftime (timebuf, sizeof (timebuf), "%b %d %H:%M:%S", tms); snprintf (tmpbuf, sizeof (tmpbuf), "#%d: %s rspamd ", (int)getpid (), timebuf); fprintf (cfg->logf, "%s%s" CRLF, tmpbuf, message); + log_written ++; } }