* Configure modules when worker starts for simplifying reconfig procedure

* Add two behaviour types of memory allocator: greedy and ungreedy
* Adopt surbl module for new configure style

cfg_utils.c

@@ -409,7 +409,7 @@ parse_regexp (memory_pool_t *pool, char *line)
 	struct rspamd_regexp *result;
 	int regexp_flags = 0;
 	enum rspamd_regexp_type type = REGEXP_NONE;
-	GError *err;
+	GError *err = NULL;
 	
 	/* Find begin */
 	while (*line != '/') {

config.h.in

@@ -7,6 +7,8 @@ struct config_file;
 typedef struct module_s {
 	const char *name;
 	int (*module_init_func)(struct config_file *cfg, struct module_ctx **ctx);
+	int (*module_config_func)(struct config_file *cfg);
+	int (*module_reconfig_func)(struct config_file *cfg);
 } module_t;
 
 extern module_t modules[];

configure

@@ -458,8 +458,10 @@ write_modules()
 	echo "module_t modules[] = {" >> modules.c;
 	modules_num=0
 	for m in $MODULES ; do
-		echo "{\"${m}\", ${m}_module_init}," >> modules.c
+		echo "{\"${m}\", ${m}_module_init, ${m}_module_config, ${m}_module_reconfig}," >> modules.c
 		echo "int ${m}_module_init(struct config_file *cfg, struct module_ctx **ctx);" >> modules.h
+		echo "int ${m}_module_config(struct config_file *cfg);" >> modules.h
+		echo "int ${m}_module_reconfig(struct config_file *cfg);" >> modules.h
 		modules_num=`expr $modules_num + 1`
 	done
 	echo "};" >> modules.c
@@ -658,6 +660,19 @@ do
 
 										 name=`echo $modline | sed 's/.*MODULE:\([^: ]*\).*/\1/'`
 										 MODULES="$MODULES $name"							;;
+		--disable-module)				 SOURCES=`echo $SOURCES | sed -e "s/${value}//g"`
+										 if [ ! -f $value ] ; then
+										 	echo "Cannot find $value, please check path"
+											exit 1
+										 fi
+										 modline=`grep '/***MODULE:' $value`
+										 if [ F"$modline" = "F" ] ; then
+										 	echo "Cannot find /***MODULE line in $value, please check syntax"
+											exit 1
+										 fi
+
+										 name=`echo $modline | sed 's/.*MODULE:\([^: ]*\).*/\1/'`
+										 MODULES=`echo $MODULES | sed -e "s/${name}//g"`		;;
 		*)
             echo "$0: error: invalid option \"$option\""
             exit 1
@@ -674,6 +689,7 @@ if [ "F$help" = "Fyes" ] ; then
   --enable-debug                     turn on extra debug messages
   --enable-opt                       turn on extra optimization
   --add-module						 add additional C module
+  --disable-module                   do not include C module
   --user=USER                        set user to use
   --group=GROUP                      set group to use
 END
@@ -740,6 +756,11 @@ if [ $? -eq 0 ] ; then
 	have_opt "SRANDOMDEV"
 fi
 
+check_function "getpagesize" "unistd.h"
+if [ $? -eq 0 ] ; then
+	have_opt "GETPAGESIZE"
+fi
+
 check_function "setproctitle" "unistd.h"
 if [ $? -eq 0 ] ; then
 	have_opt "SETPROCTITLE"

mem_pool.c

@@ -13,6 +13,16 @@ pthread_mutex_t stat_mtx = PTHREAD_MUTEX_INITIALIZER;
 #define STAT_UNLOCK() do {} while (0)
 #endif
 
+/* 
+ * This define specify whether we should check all pools for free space for new object
+ * or just begin scan from current (recently attached) pool
+ * If MEMORY_GREEDY is defined, then we scan all pools to find free space (more CPU usage, slower
+ * but requires less memory). If it is not defined check only current pool and if object is too large
+ * to place in it allocate new one (this may cause huge CPU usage in some cases too, but generally faster than
+ * greedy method)
+ */
+#undef MEMORY_GREEDY
+
 /* Internal statistic */
 static size_t bytes_allocated = 0;
 static size_t chunks_allocated = 0;
@@ -54,7 +64,11 @@ memory_pool_alloc (memory_pool_t *pool, size_t size)
 	struct _pool_chain *new, *cur;
 
 	if (pool) {
+#ifdef MEMORY_GREEDY
+		cur = pool->first_pool;
+#else
 		cur = pool->cur_pool;
+#endif
 		/* Find free space in pool chain */
 		while (memory_pool_free (cur) < size && cur->next) {
 			cur = cur->next;

plugins/surbl.c

@@ -49,7 +49,8 @@ struct surbl_ctx {
 	char *metric;
 	GHashTable *hosters;
 	GHashTable *whitelist;
-	unsigned use_redirector:1;
+	unsigned use_redirector;
+	memory_pool_t *surbl_pool;
 };
 
 struct redirector_param {
@@ -78,11 +79,8 @@ static int surbl_test_url (struct worker_task *task);
 int
 surbl_module_init (struct config_file *cfg, struct module_ctx **ctx)
 {
-	struct hostent *hent;
 	GError *err = NULL;
 
-	char *value, *cur_tok, *str;
-
 	surbl_module_ctx = g_malloc (sizeof (struct surbl_ctx));
 
 	surbl_module_ctx->header_filter = NULL;
@@ -90,9 +88,37 @@ surbl_module_init (struct config_file *cfg, struct module_ctx **ctx)
 	surbl_module_ctx->message_filter = NULL;
 	surbl_module_ctx->url_filter = surbl_test_url;
 	surbl_module_ctx->use_redirector = 0;
+	surbl_module_ctx->surbl_pool = memory_pool_new (1024);
+	
+	surbl_module_ctx->hosters = g_hash_table_new (g_str_hash, g_str_equal);
+	/* Register destructors */
+	memory_pool_add_destructor (surbl_module_ctx->surbl_pool, (pool_destruct_func)g_hash_table_remove_all, surbl_module_ctx->hosters);
+
+	surbl_module_ctx->whitelist = g_hash_table_new (g_str_hash, g_str_equal);
+	/* Register destructors */
+	memory_pool_add_destructor (surbl_module_ctx->surbl_pool, (pool_destruct_func)g_hash_table_remove_all, surbl_module_ctx->whitelist);
+		
+	/* Init matching regexps */
+	extract_hoster_regexp = g_regex_new ("([^.]+)\\.([^.]+)\\.([^.]+)$", G_REGEX_RAW, 0, &err);
+	extract_normal_regexp = g_regex_new ("([^.]+)\\.([^.]+)$", G_REGEX_RAW, 0, &err);
+	extract_numeric_regexp = g_regex_new ("(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})$", G_REGEX_RAW, 0, &err);
+
+	*ctx = (struct module_ctx *)surbl_module_ctx;
+
+	evdns_init ();
+
+	return 0;
+}
+
+int
+surbl_module_config (struct config_file *cfg)
+{
+	struct hostent *hent;
+
+	char *value, *cur_tok, *str;
 
 	if ((value = get_module_opt (cfg, "surbl", "redirector")) != NULL) {
-		str = strdup (value);
+		str = memory_pool_strdup (surbl_module_ctx->surbl_pool, value);
 		cur_tok = strsep (&str, ":");
 		if (!inet_aton (cur_tok, &surbl_module_ctx->redirector_addr)) {
 			/* Try to call gethostbyname */
@@ -142,26 +168,26 @@ surbl_module_init (struct config_file *cfg, struct module_ctx **ctx)
 		surbl_module_ctx->max_urls = DEFAULT_SURBL_MAX_URLS;
 	}
 	if ((value = get_module_opt (cfg, "surbl", "suffix")) != NULL) {
-		surbl_module_ctx->suffix = value;
+		surbl_module_ctx->suffix = memory_pool_strdup (surbl_module_ctx->surbl_pool, value);
+		g_free (value);
 	}
 	else {
 		surbl_module_ctx->suffix = DEFAULT_SURBL_SUFFIX;
 	}
 	if ((value = get_module_opt (cfg, "surbl", "symbol")) != NULL) {
-		surbl_module_ctx->symbol = value;
+		surbl_module_ctx->symbol = memory_pool_strdup (surbl_module_ctx->surbl_pool, value);
+		g_free (value);
 	}
 	else {
 		surbl_module_ctx->symbol = DEFAULT_SURBL_SYMBOL;
 	}
 	if ((value = get_module_opt (cfg, "surbl", "metric")) != NULL) {
-		surbl_module_ctx->metric = value;
+		surbl_module_ctx->metric = memory_pool_strdup (surbl_module_ctx->surbl_pool, value);
+		g_free (value);
 	}
 	else {
 		surbl_module_ctx->metric = DEFAULT_METRIC;
 	}
-	
-	surbl_module_ctx->hosters = g_hash_table_new (g_str_hash, g_str_equal);
-	surbl_module_ctx->whitelist = g_hash_table_new (g_str_hash, g_str_equal);
 	if ((value = get_module_opt (cfg, "surbl", "hostings")) != NULL) {
 		char comment_flag = 0;
 		str = value;
@@ -200,17 +226,15 @@ surbl_module_init (struct config_file *cfg, struct module_ctx **ctx)
 			}
 		}
 	}
-	
-	/* Init matching regexps */
-	extract_hoster_regexp = g_regex_new ("([^.]+)\\.([^.]+)\\.([^.]+)$", G_REGEX_RAW, 0, &err);
-	extract_normal_regexp = g_regex_new ("([^.]+)\\.([^.]+)$", G_REGEX_RAW, 0, &err);
-	extract_numeric_regexp = g_regex_new ("(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})$", G_REGEX_RAW, 0, &err);
-
-	*ctx = (struct module_ctx *)surbl_module_ctx;
+}
 
-	evdns_init ();
+int
+surbl_module_reconfig (struct config_file *cfg)
+{
+	memory_pool_delete (surbl_module_ctx->surbl_pool);
+	surbl_module_ctx->surbl_pool = memory_pool_new (1024);
 
-	return 0;
+	return surbl_module_config (cfg);
 }
 
 static char *

url.h

@@ -39,17 +39,17 @@ struct uri {
 
 	/* @protocollen should only be usable if @protocol is either
 	 * PROTOCOL_USER or an uri string should be composed. */
-	unsigned int protocollen:16;
-	unsigned int userlen:16;
-	unsigned int passwordlen:16;
-	unsigned int hostlen:16;
-	unsigned int portlen:8;
-	unsigned int datalen:16;
-	unsigned int fragmentlen:16;
+	unsigned int protocollen;
+	unsigned int userlen;
+	unsigned int passwordlen;
+	unsigned int hostlen;
+	unsigned int portlen;
+	unsigned int datalen;
+	unsigned int fragmentlen;
 
 	/* Flags */
-	unsigned int ipv6:1;	/* URI contains IPv6 host */
-	unsigned int form:1;	/* URI originated from form */
+	unsigned int ipv6;	/* URI contains IPv6 host */
+	unsigned int form;	/* URI originated from form */
 	
 	/* Link */
 	TAILQ_ENTRY(uri) next;

worker.c

@@ -27,15 +27,9 @@
 #include "upstream.h"
 #include "cfg_file.h"
 #include "url.h"
+#include "modules.h"
 
-#define CONTENT_LENGTH_HEADER "Content-Length: "
-#define HELO_HEADER "Helo: "
-#define FROM_HEADER "From: "
-#define IP_ADDR_HEADER "IP: "
-#define NRCPT_HEADER "Recipient-Number: "
-#define RCPT_HEADER "Rcpt: "
-
-#define TASK_POOL_SIZE 16384
+#define TASK_POOL_SIZE 4095
 
 const f_str_t CRLF = {
 	/* begin */"\r\n",
@@ -77,13 +71,9 @@ free_task (struct worker_task *task)
 	struct mime_part *part;
 
 	if (task) {
-		if (task->message) {
-			g_object_unref (task->message);
-		}
 		if (task->memc_ctx) {
 			memc_close_ctx (task->memc_ctx);
 		}
-
 		while (!TAILQ_EMPTY (&task->parts)) {
 			part = TAILQ_FIRST (&task->parts);
 			g_object_unref (part->type);
@@ -178,6 +168,7 @@ process_message (struct worker_task *task)
 	message = g_mime_parser_construct_message (parser);
 	
 	task->message = message;
+	memory_pool_add_destructor (task->task_pool, (pool_destruct_func)g_object_unref, task->message);
 
 	/* free the parser (and the stream) */
 	g_object_unref (parser);
@@ -299,7 +290,11 @@ accept_socket (int fd, short what, void *arg)
 	new_task->cfg = worker->srv->cfg;
 	TAILQ_INIT (&new_task->urls);
 	TAILQ_INIT (&new_task->parts);
+#ifdef HAVE_GETPAGESIZE
+	new_task->task_pool = memory_pool_new (getpagesize () - 1);
+#else
 	new_task->task_pool = memory_pool_new (TASK_POOL_SIZE);
+#endif
 	new_task->memc_ctx = memory_pool_alloc (new_task->task_pool, sizeof (memcached_ctx_t));
 	if (new_task->memc_ctx == NULL) {
 		msg_err ("accept_socket: cannot allocate memory for memcached ctx, %m");
@@ -319,6 +314,9 @@ void
 start_worker (struct rspamd_worker *worker, int listen_sock)
 {
 	struct sigaction signals;
+	int i;
+
+
 	worker->srv->pid = getpid ();
 	worker->srv->type = TYPE_WORKER;
 	event_init ();
@@ -335,6 +333,11 @@ start_worker (struct rspamd_worker *worker, int listen_sock)
 	event_set(&worker->bind_ev, listen_sock, EV_READ | EV_PERSIST, accept_socket, (void *)worker);
 	event_add(&worker->bind_ev, NULL);
 
+	/* Perform modules configuring */
+	for (i = 0; i < MODULES_NUM; i ++) {
+		modules[i].module_config_func (worker->srv->cfg);
+	}
+
 	/* Send SIGUSR2 to parent */
 	kill (getppid (), SIGUSR2);