* Add ability to call rspamd fucntions from lua api

* Make logging adaptive based on log speed (buffered vs unbuffered IO)
* Fix lua API docs
* Now lua modules can be loaded with glob patterns

CMakeLists.txt

@@ -228,6 +228,7 @@ CHECK_INCLUDE_FILES(locale.h HAVE_LOCALE_H)
 CHECK_INCLUDE_FILES(libgen.h HAVE_LIBGEN_H)
 CHECK_INCLUDE_FILES(pwd.h HAVE_PWD_H)
 CHECK_INCLUDE_FILES(grp.h HAVE_GRP_H)
+CHECK_INCLUDE_FILES(glob.h HAVE_GLOB_H)
 
 IF(HAVE_SYS_WAIT_H)
 	LIST(APPEND CMAKE_REQUIRED_INCLUDES sys/wait.h)

config.h.in

@@ -47,6 +47,8 @@
 #cmakedefine HAVE_GRP_H          1
 #cmakedefine HAVE_PWD_H          1
 
+#cmakedefine HAVE_GLOB_H         1
+
 #cmakedefine HAVE_ENDIAN_H       1
 #cmakedefine HAVE_SYS_ENDIAN_H   1
 #cmakedefine HAVE_MACHINE_ENDIAN_H  1
@@ -254,6 +256,10 @@
 #include <grp.h>
 #endif
 
+#ifdef HAVE_GLOB_H
+#include <glob.h>
+#endif
+
 /* syslog */
 #ifdef HAVE_SYSLOG_H
 #include <syslog.h>

src/cfg_file.l

@@ -10,7 +10,7 @@
 #include "cfg_file.h"
 #include "cfg_yacc.h"
 #ifdef WITH_LUA
-#include "lua.h"
+extern void add_luabuf (const char *line);
 #else 
 #define add_luabuf(x) yyerror ("lua support diabled")
 #endif
@@ -52,8 +52,8 @@ protocol						return PROTOCOL;
 memcached						return MEMCACHED;
 servers							return SERVERS;
 
-require							return REQUIRE;
-module                          return MODULE;
+modules							return MODULES;
+module_path                     return MODULE_PATH;
 
 filters							return FILTERS;
 factors							return FACTORS;

src/cfg_file.y

@@ -47,7 +47,7 @@ struct rspamd_view *cur_view = NULL;
 %token  MAXSIZE SIZELIMIT SECONDS BEANSTALK MYSQL USER PASSWORD DATABASE
 %token  TEMPDIR PIDFILE SERVERS ERROR_TIME DEAD_TIME MAXERRORS CONNECT_TIMEOUT PROTOCOL RECONNECT_TIMEOUT
 %token  READ_SERVERS WRITE_SERVER DIRECTORY_SERVERS MAILBOX_QUERY USERS_QUERY LASTLOGIN_QUERY
-%token  MEMCACHED WORKER TYPE REQUIRE MODULE
+%token  MEMCACHED WORKER TYPE MODULES MODULE_PATH
 %token  MODULE_OPT PARAM VARIABLE
 %token  FILTERS FACTORS METRIC NAME CACHE_FILE
 %token  REQUIRED_SCORE FUNCTION FRACT COMPOSITES CONTROL PASSWORD
@@ -83,7 +83,7 @@ command	:
 	| pidfile
 	| memcached
 	| worker
-	| require
+	| modules
 	| filters
 	| module_opt
 	| variable
@@ -447,31 +447,58 @@ factorparam:
 		g_hash_table_insert (cfg->factors, $1, tmp);
 	};
 
-require:
-	REQUIRE OBRACE requirebody EBRACE
+modules:
+	MODULES OBRACE modulesbody EBRACE
 	;
 
-requirebody:
-	requirecmd SEMICOLON
-	| requirebody requirecmd SEMICOLON
+modulesbody:
+	modulescmd SEMICOLON
+	| modulesbody modulescmd SEMICOLON
 	;
 
-requirecmd:
-	MODULE EQSIGN QUOTEDSTRING {
+modulescmd:
+	MODULE_PATH EQSIGN QUOTEDSTRING {
 #if !defined(WITHOUT_PERL) || defined(WITH_LUA)
 		struct stat st;
 		struct script_module *cur;
+        glob_t globbuf;
+        char *pattern;
+        size_t len;
+        int i;
+
 		if (stat ($3, &st) == -1) {
-			yyerror ("yyparse: cannot stat file %s, %s", $3, strerror (errno));
-			YYERROR;
-		}
-		cur = memory_pool_alloc (cfg->cfg_pool, sizeof (struct script_module));
-		if (cur == NULL) {
-			yyerror ("yyparse: g_malloc: %s", strerror(errno));
+			yyerror ("yyparse: cannot stat path %s, %s", $3, strerror (errno));
 			YYERROR;
 		}
-		cur->path = $3;
-		cfg->script_modules = g_list_prepend (cfg->script_modules, cur);
+
+        globbuf.gl_offs = 0;
+        #ifdef WITH_LUA
+        len = strlen ($3) + sizeof ("*.lua");
+        pattern = g_malloc (len);
+        snprintf (pattern, len, "%s%s", $3, "*.lua");
+        #else
+        len = strlen ($3) + sizeof ("*.pl")
+        pattern = g_malloc (len);
+        snprintf (pattern, len, "%s%s", $3, "*.pl");
+        #endif
+
+        if (glob(pattern, GLOB_DOOFFS, NULL, &globbuf) == 0) {
+            for (i = 0; i < globbuf.gl_pathc; i ++) {
+                cur = memory_pool_alloc (cfg->cfg_pool, sizeof (struct script_module));
+                if (cur == NULL) {
+                    yyerror ("yyparse: g_malloc: %s", strerror(errno));
+                    YYERROR;
+                }
+                cur->path = memory_pool_strdup (cfg->cfg_pool, globbuf.gl_pathv[i]);
+                cfg->script_modules = g_list_prepend (cfg->script_modules, cur);
+            }
+            globfree (&globbuf);
+        }
+        else {
+            yyerror ("yyparse: glob: %s", strerror (errno));
+            YYERROR;
+        }
+        g_free (pattern);
 #else
 		yyerror ("require command is not available when perl support is not compiled");
 		YYERROR;

src/lua/lua_task.c

@@ -25,6 +25,7 @@
 
 #include "lua_common.h"
 #include "../message.h"
+#include "../expressions.h"
 #include <evdns.h>
 
 /* Task methods */
@@ -36,6 +37,7 @@ LUA_FUNCTION_DEF(task, get_raw_headers);
 LUA_FUNCTION_DEF(task, get_received_headers);
 LUA_FUNCTION_DEF(task, resolve_dns_a);
 LUA_FUNCTION_DEF(task, resolve_dns_ptr);
+LUA_FUNCTION_DEF(task, call_rspamd_function);
 
 static const struct luaL_reg tasklib_m[] = {
 	LUA_INTERFACE_DEF(task, get_message),
@@ -46,6 +48,7 @@ static const struct luaL_reg tasklib_m[] = {
     LUA_INTERFACE_DEF(task, get_received_headers),
 	LUA_INTERFACE_DEF(task, resolve_dns_a),
 	LUA_INTERFACE_DEF(task, resolve_dns_ptr),
+	LUA_INTERFACE_DEF(task, call_rspamd_function),
 	{"__tostring", lua_class_tostring},
 	{NULL, NULL}
 };
@@ -320,6 +323,43 @@ lua_task_resolve_dns_ptr (lua_State *L)
 	return 0;
 }
 
+static int
+lua_task_call_rspamd_function (lua_State *L)
+{
+	struct worker_task *task = lua_check_task (L);
+	struct expression_function f;
+	int i, top;
+	gboolean res;
+	char *arg;
+
+	if (task) {
+		f.name = (char *)luaL_checkstring (L, 2);
+		if (f.name) {
+			f.args = NULL;
+			top = lua_gettop (L);
+			/* Get arguments after function name */
+			for (i = 3; i <= top; i ++) {
+				arg = (char *)luaL_checkstring (L, i);
+				if (arg != NULL) {
+					f.args = g_list_prepend (f.args, arg);
+				}
+			}
+			res = call_expression_function (&f, task);
+			lua_pushboolean (L, res);
+			if (f.args) {
+				g_list_free (f.args);
+			}
+
+			return 1;
+		}
+	}
+
+	lua_pushnil (L);
+
+	return 1;
+
+}
+
 
 /**** Textpart implementation *****/
 

src/lua/rspamd.luadoc

@@ -7,9 +7,8 @@
 -- textpart - a single textual part of message
 module Rspamd
 
---- Each lua module MUST have init function that is called after config file was read:
+--- Each lua module has global rspamd_config that can be used for getting config
+-- options and registering callbacks (via metric interface)
 
 ------------------------------------- CONFIG -----------------------------------------
 --
@@ -57,6 +56,31 @@ function task:get_urls ()
 -- @return array of textpart objects
 function task:get_text_parts ()
 
+--- Get raw headers
+-- @return string that contains raw headers
+function task:get_raw_headers ()
+
+--- Get array of received headers
+-- @return array of received headers that are tables itself
+function task:get_received_headers ()
+
+--- Resolve A record using rspamd async resolver
+-- @param host host to resolve
+-- @param callback name of callback function
+function task:resolve_dns_a (host, callback)
+
+--- Resolve PTR record using rspamd async resolver
+-- @param host host to resolve
+-- @param callback name of callback function
+function task:resolve_dns_ptr (host, callback)
+
+--- Callback function for dns resolving
+-- @param task task object
+-- @param to_resolve ptr or a record that was resolved
+-- @param results results of dns query (array or nil)
+-- @param err resolver error or nil
+function dns_cb(task, to_resolve, results, err)
+
 ------------------------------------- TEXTPART ---------------------------------------
 
 --- Get part's content

src/util.c

@@ -28,6 +28,11 @@
 #include "cfg_file.h"
 #include "main.h"
 
+/* Check log messages intensity once per minute */
+#define CHECK_TIME 60
+/* More than 2 log messages per second */
+#define BUF_INTENSITY 2
+
 #ifdef RSPAMD_MAIN
 sig_atomic_t do_reopen_log = 0;
 extern rspamd_hash_t *counters;
@@ -40,6 +45,12 @@ struct logger_params {
 
 static struct logger_params log_params;
 
+/* Here would be put log messages intensity */
+static uint32_t log_written;
+static time_t last_check;
+static char *io_buf = NULL;
+static gboolean log_buffered = FALSE;
+
 int
 make_socket_nonblocking (int fd)
 {
@@ -651,6 +662,8 @@ open_log (struct config_file *cfg)
 				return -1;
 			}
 			cfg->logf = fdopen (cfg->log_fd, "w");
+			/* Set line buffering */
+			setvbuf (cfg->logf, (char *) NULL, _IOLBF, 0);
 			return 0;
 	}
 	return -1;
@@ -752,9 +765,32 @@ file_log_function (const gchar *log_domain, GLogLevelFlags log_level, const gcha
 	if (log_level <= cfg->log_level) {
 		now = time (NULL);
 		tms = localtime (&now);
+
+		if (last_check == 0) {
+			last_check = now;
+		}
+		else if (now - last_check > CHECK_TIME) {
+			if (log_written / (now - last_check) > BUF_INTENSITY && !log_buffered) {
+				/* Switch to buffered logging */
+				if (io_buf == NULL) {
+					io_buf = g_malloc (BUFSIZ);
+				}
+				setvbuf (cfg->logf, io_buf, _IOFBF, BUFSIZ);
+				log_buffered = TRUE;
+			}
+			else if (log_buffered) {
+				/* Switch to line buffering */
+				setvbuf (cfg->logf, NULL, _IOLBF, 0);
+				log_buffered = FALSE;
+			}
+			last_check = now;
+			log_written = 0;
+		}
+
 		strftime (timebuf, sizeof (timebuf), "%b %d %H:%M:%S", tms);
 		snprintf (tmpbuf, sizeof (tmpbuf), "#%d: %s rspamd ", (int)getpid (), timebuf);
 		fprintf (cfg->logf, "%s%s" CRLF, tmpbuf, message);
+		log_written ++;
 	}
 }