* Make logging adaptive based on log speed (buffered vs unbuffered IO) * Fix lua API docs * Now lua modules can be loaded with glob patternstags/0.2.7
CHECK_INCLUDE_FILES(libgen.h HAVE_LIBGEN_H) | CHECK_INCLUDE_FILES(libgen.h HAVE_LIBGEN_H) | ||||
CHECK_INCLUDE_FILES(pwd.h HAVE_PWD_H) | CHECK_INCLUDE_FILES(pwd.h HAVE_PWD_H) | ||||
CHECK_INCLUDE_FILES(grp.h HAVE_GRP_H) | CHECK_INCLUDE_FILES(grp.h HAVE_GRP_H) | ||||
CHECK_INCLUDE_FILES(glob.h HAVE_GLOB_H) | |||||
IF(HAVE_SYS_WAIT_H) | IF(HAVE_SYS_WAIT_H) | ||||
LIST(APPEND CMAKE_REQUIRED_INCLUDES sys/wait.h) | LIST(APPEND CMAKE_REQUIRED_INCLUDES sys/wait.h) |
#cmakedefine HAVE_GRP_H 1 | #cmakedefine HAVE_GRP_H 1 | ||||
#cmakedefine HAVE_PWD_H 1 | #cmakedefine HAVE_PWD_H 1 | ||||
#cmakedefine HAVE_GLOB_H 1 | |||||
#cmakedefine HAVE_ENDIAN_H 1 | #cmakedefine HAVE_ENDIAN_H 1 | ||||
#cmakedefine HAVE_SYS_ENDIAN_H 1 | #cmakedefine HAVE_SYS_ENDIAN_H 1 | ||||
#cmakedefine HAVE_MACHINE_ENDIAN_H 1 | #cmakedefine HAVE_MACHINE_ENDIAN_H 1 | ||||
#include <grp.h> | #include <grp.h> | ||||
#endif | #endif | ||||
#ifdef HAVE_GLOB_H | |||||
#include <glob.h> | |||||
#endif | |||||
/* syslog */ | /* syslog */ | ||||
#ifdef HAVE_SYSLOG_H | #ifdef HAVE_SYSLOG_H | ||||
#include <syslog.h> | #include <syslog.h> |
#include "cfg_file.h" | #include "cfg_file.h" | ||||
#include "cfg_yacc.h" | #include "cfg_yacc.h" | ||||
#ifdef WITH_LUA | #ifdef WITH_LUA | ||||
#include "lua.h" | |||||
extern void add_luabuf (const char *line); | |||||
#else | #else | ||||
#define add_luabuf(x) yyerror ("lua support diabled") | #define add_luabuf(x) yyerror ("lua support diabled") | ||||
#endif | #endif | ||||
memcached return MEMCACHED; | memcached return MEMCACHED; | ||||
servers return SERVERS; | servers return SERVERS; | ||||
require return REQUIRE; | |||||
module return MODULE; | |||||
modules return MODULES; | |||||
module_path return MODULE_PATH; | |||||
filters return FILTERS; | filters return FILTERS; | ||||
factors return FACTORS; | factors return FACTORS; |
%token MAXSIZE SIZELIMIT SECONDS BEANSTALK MYSQL USER PASSWORD DATABASE | %token MAXSIZE SIZELIMIT SECONDS BEANSTALK MYSQL USER PASSWORD DATABASE | ||||
%token TEMPDIR PIDFILE SERVERS ERROR_TIME DEAD_TIME MAXERRORS CONNECT_TIMEOUT PROTOCOL RECONNECT_TIMEOUT | %token TEMPDIR PIDFILE SERVERS ERROR_TIME DEAD_TIME MAXERRORS CONNECT_TIMEOUT PROTOCOL RECONNECT_TIMEOUT | ||||
%token READ_SERVERS WRITE_SERVER DIRECTORY_SERVERS MAILBOX_QUERY USERS_QUERY LASTLOGIN_QUERY | %token READ_SERVERS WRITE_SERVER DIRECTORY_SERVERS MAILBOX_QUERY USERS_QUERY LASTLOGIN_QUERY | ||||
%token MEMCACHED WORKER TYPE REQUIRE MODULE | |||||
%token MEMCACHED WORKER TYPE MODULES MODULE_PATH | |||||
%token MODULE_OPT PARAM VARIABLE | %token MODULE_OPT PARAM VARIABLE | ||||
%token FILTERS FACTORS METRIC NAME CACHE_FILE | %token FILTERS FACTORS METRIC NAME CACHE_FILE | ||||
%token REQUIRED_SCORE FUNCTION FRACT COMPOSITES CONTROL PASSWORD | %token REQUIRED_SCORE FUNCTION FRACT COMPOSITES CONTROL PASSWORD | ||||
| pidfile | | pidfile | ||||
| memcached | | memcached | ||||
| worker | | worker | ||||
| require | |||||
| modules | |||||
| filters | | filters | ||||
| module_opt | | module_opt | ||||
| variable | | variable | ||||
g_hash_table_insert (cfg->factors, $1, tmp); | g_hash_table_insert (cfg->factors, $1, tmp); | ||||
}; | }; | ||||
require: | |||||
REQUIRE OBRACE requirebody EBRACE | |||||
modules: | |||||
MODULES OBRACE modulesbody EBRACE | |||||
; | ; | ||||
requirebody: | |||||
requirecmd SEMICOLON | |||||
| requirebody requirecmd SEMICOLON | |||||
modulesbody: | |||||
modulescmd SEMICOLON | |||||
| modulesbody modulescmd SEMICOLON | |||||
; | ; | ||||
requirecmd: | |||||
MODULE EQSIGN QUOTEDSTRING { | |||||
modulescmd: | |||||
MODULE_PATH EQSIGN QUOTEDSTRING { | |||||
#if !defined(WITHOUT_PERL) || defined(WITH_LUA) | #if !defined(WITHOUT_PERL) || defined(WITH_LUA) | ||||
struct stat st; | struct stat st; | ||||
struct script_module *cur; | struct script_module *cur; | ||||
glob_t globbuf; | |||||
char *pattern; | |||||
size_t len; | |||||
int i; | |||||
if (stat ($3, &st) == -1) { | if (stat ($3, &st) == -1) { | ||||
yyerror ("yyparse: cannot stat file %s, %s", $3, strerror (errno)); | |||||
YYERROR; | |||||
} | |||||
cur = memory_pool_alloc (cfg->cfg_pool, sizeof (struct script_module)); | |||||
if (cur == NULL) { | |||||
yyerror ("yyparse: g_malloc: %s", strerror(errno)); | |||||
yyerror ("yyparse: cannot stat path %s, %s", $3, strerror (errno)); | |||||
YYERROR; | YYERROR; | ||||
} | } | ||||
cur->path = $3; | |||||
cfg->script_modules = g_list_prepend (cfg->script_modules, cur); | |||||
globbuf.gl_offs = 0; | |||||
#ifdef WITH_LUA | |||||
len = strlen ($3) + sizeof ("*.lua"); | |||||
pattern = g_malloc (len); | |||||
snprintf (pattern, len, "%s%s", $3, "*.lua"); | |||||
#else | |||||
len = strlen ($3) + sizeof ("*.pl") | |||||
pattern = g_malloc (len); | |||||
snprintf (pattern, len, "%s%s", $3, "*.pl"); | |||||
#endif | |||||
if (glob(pattern, GLOB_DOOFFS, NULL, &globbuf) == 0) { | |||||
for (i = 0; i < globbuf.gl_pathc; i ++) { | |||||
cur = memory_pool_alloc (cfg->cfg_pool, sizeof (struct script_module)); | |||||
if (cur == NULL) { | |||||
yyerror ("yyparse: g_malloc: %s", strerror(errno)); | |||||
YYERROR; | |||||
} | |||||
cur->path = memory_pool_strdup (cfg->cfg_pool, globbuf.gl_pathv[i]); | |||||
cfg->script_modules = g_list_prepend (cfg->script_modules, cur); | |||||
} | |||||
globfree (&globbuf); | |||||
} | |||||
else { | |||||
yyerror ("yyparse: glob: %s", strerror (errno)); | |||||
YYERROR; | |||||
} | |||||
g_free (pattern); | |||||
#else | #else | ||||
yyerror ("require command is not available when perl support is not compiled"); | yyerror ("require command is not available when perl support is not compiled"); | ||||
YYERROR; | YYERROR; |
#include "lua_common.h" | #include "lua_common.h" | ||||
#include "../message.h" | #include "../message.h" | ||||
#include "../expressions.h" | |||||
#include <evdns.h> | #include <evdns.h> | ||||
/* Task methods */ | /* Task methods */ | ||||
LUA_FUNCTION_DEF(task, get_received_headers); | LUA_FUNCTION_DEF(task, get_received_headers); | ||||
LUA_FUNCTION_DEF(task, resolve_dns_a); | LUA_FUNCTION_DEF(task, resolve_dns_a); | ||||
LUA_FUNCTION_DEF(task, resolve_dns_ptr); | LUA_FUNCTION_DEF(task, resolve_dns_ptr); | ||||
LUA_FUNCTION_DEF(task, call_rspamd_function); | |||||
static const struct luaL_reg tasklib_m[] = { | static const struct luaL_reg tasklib_m[] = { | ||||
LUA_INTERFACE_DEF(task, get_message), | LUA_INTERFACE_DEF(task, get_message), | ||||
LUA_INTERFACE_DEF(task, get_received_headers), | LUA_INTERFACE_DEF(task, get_received_headers), | ||||
LUA_INTERFACE_DEF(task, resolve_dns_a), | LUA_INTERFACE_DEF(task, resolve_dns_a), | ||||
LUA_INTERFACE_DEF(task, resolve_dns_ptr), | LUA_INTERFACE_DEF(task, resolve_dns_ptr), | ||||
LUA_INTERFACE_DEF(task, call_rspamd_function), | |||||
{"__tostring", lua_class_tostring}, | {"__tostring", lua_class_tostring}, | ||||
{NULL, NULL} | {NULL, NULL} | ||||
}; | }; | ||||
return 0; | return 0; | ||||
} | } | ||||
static int | |||||
lua_task_call_rspamd_function (lua_State *L) | |||||
{ | |||||
struct worker_task *task = lua_check_task (L); | |||||
struct expression_function f; | |||||
int i, top; | |||||
gboolean res; | |||||
char *arg; | |||||
if (task) { | |||||
f.name = (char *)luaL_checkstring (L, 2); | |||||
if (f.name) { | |||||
f.args = NULL; | |||||
top = lua_gettop (L); | |||||
/* Get arguments after function name */ | |||||
for (i = 3; i <= top; i ++) { | |||||
arg = (char *)luaL_checkstring (L, i); | |||||
if (arg != NULL) { | |||||
f.args = g_list_prepend (f.args, arg); | |||||
} | |||||
} | |||||
res = call_expression_function (&f, task); | |||||
lua_pushboolean (L, res); | |||||
if (f.args) { | |||||
g_list_free (f.args); | |||||
} | |||||
return 1; | |||||
} | |||||
} | |||||
lua_pushnil (L); | |||||
return 1; | |||||
} | |||||
/**** Textpart implementation *****/ | /**** Textpart implementation *****/ | ||||
-- textpart - a single textual part of message | -- textpart - a single textual part of message | ||||
module Rspamd | module Rspamd | ||||
--- Each lua module MUST have init function that is called after config file was read: | |||||
--- Each lua module has global rspamd_config that can be used for getting config | |||||
-- options and registering callbacks (via metric interface) | |||||
------------------------------------- CONFIG ----------------------------------------- | ------------------------------------- CONFIG ----------------------------------------- | ||||
-- | -- | ||||
-- @return array of textpart objects | -- @return array of textpart objects | ||||
function task:get_text_parts () | function task:get_text_parts () | ||||
--- Get raw headers | |||||
-- @return string that contains raw headers | |||||
function task:get_raw_headers () | |||||
--- Get array of received headers | |||||
-- @return array of received headers that are tables itself | |||||
function task:get_received_headers () | |||||
--- Resolve A record using rspamd async resolver | |||||
-- @param host host to resolve | |||||
-- @param callback name of callback function | |||||
function task:resolve_dns_a (host, callback) | |||||
--- Resolve PTR record using rspamd async resolver | |||||
-- @param host host to resolve | |||||
-- @param callback name of callback function | |||||
function task:resolve_dns_ptr (host, callback) | |||||
--- Callback function for dns resolving | |||||
-- @param task task object | |||||
-- @param to_resolve ptr or a record that was resolved | |||||
-- @param results results of dns query (array or nil) | |||||
-- @param err resolver error or nil | |||||
function dns_cb(task, to_resolve, results, err) | |||||
------------------------------------- TEXTPART --------------------------------------- | ------------------------------------- TEXTPART --------------------------------------- | ||||
--- Get part's content | --- Get part's content |
#include "cfg_file.h" | #include "cfg_file.h" | ||||
#include "main.h" | #include "main.h" | ||||
/* Check log messages intensity once per minute */ | |||||
#define CHECK_TIME 60 | |||||
/* More than 2 log messages per second */ | |||||
#define BUF_INTENSITY 2 | |||||
#ifdef RSPAMD_MAIN | #ifdef RSPAMD_MAIN | ||||
sig_atomic_t do_reopen_log = 0; | sig_atomic_t do_reopen_log = 0; | ||||
extern rspamd_hash_t *counters; | extern rspamd_hash_t *counters; | ||||
static struct logger_params log_params; | static struct logger_params log_params; | ||||
/* Here would be put log messages intensity */ | |||||
static uint32_t log_written; | |||||
static time_t last_check; | |||||
static char *io_buf = NULL; | |||||
static gboolean log_buffered = FALSE; | |||||
int | int | ||||
make_socket_nonblocking (int fd) | make_socket_nonblocking (int fd) | ||||
{ | { | ||||
return -1; | return -1; | ||||
} | } | ||||
cfg->logf = fdopen (cfg->log_fd, "w"); | cfg->logf = fdopen (cfg->log_fd, "w"); | ||||
/* Set line buffering */ | |||||
setvbuf (cfg->logf, (char *) NULL, _IOLBF, 0); | |||||
return 0; | return 0; | ||||
} | } | ||||
return -1; | return -1; | ||||
if (log_level <= cfg->log_level) { | if (log_level <= cfg->log_level) { | ||||
now = time (NULL); | now = time (NULL); | ||||
tms = localtime (&now); | tms = localtime (&now); | ||||
if (last_check == 0) { | |||||
last_check = now; | |||||
} | |||||
else if (now - last_check > CHECK_TIME) { | |||||
if (log_written / (now - last_check) > BUF_INTENSITY && !log_buffered) { | |||||
/* Switch to buffered logging */ | |||||
if (io_buf == NULL) { | |||||
io_buf = g_malloc (BUFSIZ); | |||||
} | |||||
setvbuf (cfg->logf, io_buf, _IOFBF, BUFSIZ); | |||||
log_buffered = TRUE; | |||||
} | |||||
else if (log_buffered) { | |||||
/* Switch to line buffering */ | |||||
setvbuf (cfg->logf, NULL, _IOLBF, 0); | |||||
log_buffered = FALSE; | |||||
} | |||||
last_check = now; | |||||
log_written = 0; | |||||
} | |||||
strftime (timebuf, sizeof (timebuf), "%b %d %H:%M:%S", tms); | strftime (timebuf, sizeof (timebuf), "%b %d %H:%M:%S", tms); | ||||
snprintf (tmpbuf, sizeof (tmpbuf), "#%d: %s rspamd ", (int)getpid (), timebuf); | snprintf (tmpbuf, sizeof (tmpbuf), "#%d: %s rspamd ", (int)getpid (), timebuf); | ||||
fprintf (cfg->logf, "%s%s" CRLF, tmpbuf, message); | fprintf (cfg->logf, "%s%s" CRLF, tmpbuf, message); | ||||
log_written ++; | |||||
} | } | ||||
} | } | ||||