* Fix url-extracter

* Fix soft-shutdown of dispacther
* Turn debug on by default in cmake
* Change logic of save points

8 files changed, 61 insertions, 45 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 275562b63..087dc2c7a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -14,8 +14,8 @@ SET(RSPAMD_MASTER_SITE_URL "http://cebka.pp.ru/hg/rspamd")
 
 CMAKE_MINIMUM_REQUIRED(VERSION 2.6.0 FATAL_ERROR)
 
-OPTION(DEBUG_MODE          "Enable debug output [default: OFF]"                 OFF)
-OPTION(ENABLE_OPTIMIZATION "Enable optimization [default: ON]"                  ON)
+OPTION(DEBUG_MODE          "Enable debug output [default: ON]"                  ON)
+OPTION(ENABLE_OPTIMIZATION "Enable optimization [default: OFF]"                 OFF)
 OPTION(SKIP_RELINK_RPATH   "Skip relinking and full RPATH for the install tree" OFF)
 
 # Build optimized code for following CPU (default i386)
@@ -194,13 +194,11 @@ ELSE (MD5_INCLUDE)
 	MESSAGE(STATUS "WARNING: Using internal MD5 support")
 ENDIF (MD5_INCLUDE)
 
-IF("${CMAKE_COMPILER_IS_GNUC}" MATCHES "1")
-	SET(CMAKE_C_WARN_FLAGS " -Wall -W -Wpointer-arith -Wno-unused-parameter -Wno-unused-function -Wunused-variable -Wno-sign-compare -Wunused-value")
+SET(CMAKE_C_WARN_FLAGS " -Wall -W -Wpointer-arith -Wno-unused-parameter -Wno-unused-function -Wunused-variable -Wno-sign-compare -Wunused-value")
 
-	IF(DEBUG_MODE MATCHES "ON")
-		SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ggdb ")
-	ENDIF(DEBUG_MODE MATCHES "ON")
-ENDIF("${CMAKE_COMPILER_IS_GNUC}" MATCHES "1")
+IF(DEBUG_MODE MATCHES "ON")
+	SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ggdb ")
+ENDIF(DEBUG_MODE MATCHES "ON")
 
 SET(RSPAMDSRC	src/modules.c
 				src/hash.c
@@ -251,8 +249,7 @@ SET(UTILSSRC	utils/url_extracter.c)
 SET(UTILSDEPENDS src/mem_pool.c
 				src/hash.c
 				src/url.c
-				src/util.c
-				src/memcached.c)
+				src/util.c)
 
 LIST(LENGTH PLUGINSSRC RSPAMD_MODULES_NUM)
 
@@ -311,7 +308,6 @@ SET_TARGET_PROPERTIES(utils/url-extracter PROPERTIES LINKER_LANGUAGE C)
 SET_TARGET_PROPERTIES(utils/url-extracter PROPERTIES COMPILE_FLAGS ${PERL_CFLAGS}
 										LINK_FLAGS ${PERL_LDFLAGS})
 TARGET_LINK_LIBRARIES(utils/url-extracter ${GLIB2_LIBRARIES})
-TARGET_LINK_LIBRARIES(utils/url-extracter event)
 TARGET_LINK_LIBRARIES(utils/url-extracter ${GMIME2_LIBRARIES})
 
 INSTALL(PROGRAMS rspamd-${RSPAMD_VERSION} DESTINATION bin RENAME rspamd)
diff --git a/rspamd.conf.sample b/rspamd.conf.sample
index b440a3325..63c8792fa 100644
--- a/rspamd.conf.sample
+++ b/rspamd.conf.sample
@@ -102,7 +102,7 @@ delivery {
 # SURBL module params, note that single quotes are mandatory here
 .module 'surbl' {
 	# Address to redirector in host:port format
-	redirector = "localhost:8080";
+	#redirector = "localhost:8080";
 	# Connect timeout for redirector
 	redirector_connect_timeout = "1s";
 	# IO timeout for redirector (may be usefull to set this value rather big)
diff --git a/src/controller.c b/src/controller.c
index 1b2e72634..503433d9b 100644
--- a/src/controller.c
+++ b/src/controller.c
@@ -356,7 +356,7 @@ process_command (struct controller_command *cmd, char **cmd_args, struct control
 }
 
 static void
-read_socket (f_str_t *in, void *arg)
+controller_read_socket (f_str_t *in, void *arg)
 {
 	struct controller_session *session = (struct controller_session *)arg;
 	struct classifier_ctx *cls_ctx;
@@ -436,7 +436,7 @@ read_socket (f_str_t *in, void *arg)
 }
 
 static void
-write_socket (void *arg)
+controller_write_socket (void *arg)
 {
 	struct controller_session *session = (struct controller_session *)arg;
 	
@@ -454,15 +454,15 @@ write_socket (void *arg)
 }
 
 static void
-err_socket (GError *err, void *arg)
+controller_err_socket (GError *err, void *arg)
 {
 	struct controller_session *session = (struct controller_session *)arg;
 
 	if (err->code == EOF) {
-		msg_info ("err_socket: client closed control connection");
+		msg_info ("controller_err_socket: client closed control connection");
 	}
 	else {
-		msg_info ("err_socket: abnormally closing control connection, error: %s", err->message);
+		msg_info ("controller_err_socket: abnormally closing control connection, error: %s", err->message);
 	}
 	/* Free buffers */
 	free_session (session);
@@ -498,8 +498,8 @@ accept_socket (int fd, short what, void *arg)
 	worker->srv->stat->control_connections_count ++;
 
 	/* Set up dispatcher */
-	new_session->dispatcher = rspamd_create_dispatcher (nfd, BUFFER_LINE, read_socket,
-														write_socket, err_socket, &io_tv,
+	new_session->dispatcher = rspamd_create_dispatcher (nfd, BUFFER_LINE, controller_read_socket,
+														controller_write_socket, controller_err_socket, &io_tv,
 														(void *)new_session);
 	rspamd_dispatcher_write (new_session->dispatcher, greetingbuf, strlen (greetingbuf), FALSE);
 }
diff --git a/src/filter.c b/src/filter.c
index a1d1e43fa..49f239f27 100644
--- a/src/filter.c
+++ b/src/filter.c
@@ -235,6 +235,8 @@ continue_process_filters (struct worker_task *task)
 			/* Process all metrics */
 			g_hash_table_foreach (task->results, metric_process_callback, task);
 			process_statfiles (task);
+			/* XXX: ugly direct call */
+			task->dispatcher->write_callback (task);
 			return 1;
 	}
 }
diff --git a/src/lmtp.c b/src/lmtp.c
index f2546e55a..fb7c8a8b0 100644
--- a/src/lmtp.c
+++ b/src/lmtp.c
@@ -35,7 +35,7 @@
 static char greetingbuf[1024];
 static struct timeval io_tv;
 
-static void write_socket (void *arg);
+static void lmtp_write_socket (void *arg);
 
 static 
 void sig_handler (int signo)
@@ -84,7 +84,7 @@ rcpt_destruct (void *pointer)
  * Free all structures of lmtp proto
  */
 static void
-free_task (struct rspamd_lmtp_proto *lmtp)
+free_task (struct rspamd_lmtp_proto *lmtp, gboolean is_soft)
 {
 	GList *part;
 	struct mime_part *p;
@@ -101,8 +101,13 @@ free_task (struct rspamd_lmtp_proto *lmtp)
 			g_list_free_1 (part);
 		}
 		memory_pool_delete (lmtp->task->task_pool);
-		/* Plan dispatcher shutdown */
-		lmtp->task->dispatcher->wanna_die = 1;
+		if (is_soft) {
+			/* Plan dispatcher shutdown */
+			lmtp->task->dispatcher->wanna_die = 1;
+		}
+		else {
+			rspamd_remove_dispatcher (lmtp->task->dispatcher);
+		}
 		close (lmtp->task->sock);
 		g_free (lmtp->task);
 		g_free (lmtp);
@@ -113,7 +118,7 @@ free_task (struct rspamd_lmtp_proto *lmtp)
  * Callback that is called when there is data to read in buffer
  */
 static void
-read_socket (f_str_t *in, void *arg)
+lmtp_read_socket (f_str_t *in, void *arg)
 {
 	struct rspamd_lmtp_proto *lmtp = (struct rspamd_lmtp_proto *)arg;
 	struct worker_task *task = lmtp->task;
@@ -128,7 +133,7 @@ read_socket (f_str_t *in, void *arg)
 			}
 			/* Task was read, recall read handler once more with new state to process message and write reply */
 			if (task->state == READ_MESSAGE) {
-				read_socket (in, arg);
+				lmtp_read_socket (in, arg);
 			}
 			break;
 		case READ_MESSAGE:
@@ -138,7 +143,7 @@ read_socket (f_str_t *in, void *arg)
 				task->last_error = "Filter processing error";
 				task->error_code = LMTP_FAILURE;
 				task->state = WRITE_ERROR;
-				write_socket (lmtp);
+				lmtp_write_socket (lmtp);
 			}
 			else if (r == 0) {
 				task->state = WAIT_FILTER;
@@ -147,7 +152,7 @@ read_socket (f_str_t *in, void *arg)
 			else {
 				process_statfiles (lmtp->task);
 				task->state = WRITE_REPLY;
-				write_socket (lmtp);
+				lmtp_write_socket (lmtp);
 			}
 			break;
 	}
@@ -157,7 +162,7 @@ read_socket (f_str_t *in, void *arg)
  * Callback for socket writing
  */
 static void
-write_socket (void *arg)
+lmtp_write_socket (void *arg)
 {
 	struct rspamd_lmtp_proto *lmtp = (struct rspamd_lmtp_proto *)arg;
 	
@@ -176,7 +181,7 @@ write_socket (void *arg)
 			break;
 		case CLOSING_CONNECTION:
 			msg_debug ("lmtp_write_socket: normally closing connection");
-			free_task (lmtp);
+			free_task (lmtp, TRUE);
 			break;
 	}
 }
@@ -185,12 +190,12 @@ write_socket (void *arg)
  * Called if something goes wrong
  */
 static void
-err_socket (GError *err, void *arg)
+lmtp_err_socket (GError *err, void *arg)
 {
 	struct rspamd_lmtp_proto *lmtp = (struct rspamd_lmtp_proto *)arg;
 	msg_info ("lmtp_err_socket: abnormally closing connection, error: %s", err->message);
 	/* Free buffers */
-	free_task (lmtp);
+	free_task (lmtp, FALSE);
 }
 
 /*
@@ -239,8 +244,8 @@ accept_socket (int fd, short what, void *arg)
 	lmtp->state = LMTP_READ_LHLO;
 
 	/* Set up dispatcher */
-	new_task->dispatcher = rspamd_create_dispatcher (nfd, BUFFER_LINE, read_socket,
-														write_socket, err_socket, &io_tv,
+	new_task->dispatcher = rspamd_create_dispatcher (nfd, BUFFER_LINE, lmtp_read_socket,
+														lmtp_write_socket, lmtp_err_socket, &io_tv,
 														(void *)lmtp);
 	rspamd_dispatcher_write (lmtp->task->dispatcher, greetingbuf, strlen (greetingbuf), FALSE);
 }
diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c
index 529fa5b09..e048ed132 100644
--- a/src/plugins/surbl.c
+++ b/src/plugins/surbl.c
@@ -145,8 +145,6 @@ surbl_module_config (struct config_file *cfg)
 				surbl_module_ctx->use_redirector = 1;
 			}
 		}
-		/* Free cur_tok as it is actually initial str after strsep */
-		free (cur_tok);
 	}
 	if ((value = get_module_opt (cfg, "surbl", "weight")) != NULL) {
 		surbl_module_ctx->weight = atoi (value);
@@ -180,21 +178,18 @@ surbl_module_config (struct config_file *cfg)
 	}
 	if ((value = get_module_opt (cfg, "surbl", "suffix")) != NULL) {
 		surbl_module_ctx->suffix = memory_pool_strdup (surbl_module_ctx->surbl_pool, value);
-		g_free (value);
 	}
 	else {
 		surbl_module_ctx->suffix = DEFAULT_SURBL_SUFFIX;
 	}
 	if ((value = get_module_opt (cfg, "surbl", "symbol")) != NULL) {
 		surbl_module_ctx->symbol = memory_pool_strdup (surbl_module_ctx->surbl_pool, value);
-		g_free (value);
 	}
 	else {
 		surbl_module_ctx->symbol = DEFAULT_SURBL_SYMBOL;
 	}
 	if ((value = get_module_opt (cfg, "surbl", "metric")) != NULL) {
 		surbl_module_ctx->metric = memory_pool_strdup (surbl_module_ctx->surbl_pool, value);
-		g_free (value);
 	}
 	else {
 		surbl_module_ctx->metric = DEFAULT_METRIC;
@@ -462,6 +457,8 @@ redirector_callback (int fd, short what, void *arg)
 	int r;
 	struct timeval timeout;
 	char *p, *c;
+	char *surbl_req;
+	f_str_t f;
 
 	switch (param->state) {
 		case STATE_CONNECT:
@@ -512,7 +509,16 @@ redirector_callback (int fd, short what, void *arg)
 					if (*p == '\0') {
 						msg_info ("redirector_callback: got reply from redirector: '%s' -> '%s'", struri (param->url), c);
 						parse_uri (param->url, c, param->task->task_pool);
-						register_memcached_call (param->url, param->task);
+						f.begin = param->url->host;
+						f.len = param->url->hostlen;
+						if ((surbl_req = format_surbl_request (param->task->task_pool, &f)) != NULL) {
+							msg_debug ("surbl_test_url: send surbl dns request %s", surbl_req);
+							evdns_resolve_ipv4 (surbl_req, DNS_QUERY_NO_SEARCH, dns_callback, (void *)param);
+						}
+						else {
+							msg_info ("surbl_test_url: cannot format url string for surbl %s", struri (param->url));
+							return;
+						}
 						param->task->save.saved ++;
 					}
 				}
diff --git a/src/worker.c b/src/worker.c
index 09dccb7ff..143cb54de 100644
--- a/src/worker.c
+++ b/src/worker.c
@@ -94,7 +94,7 @@ rcpt_destruct (void *pointer)
  * Free all structures of worker_task
  */
 static void
-free_task (struct worker_task *task)
+free_task (struct worker_task *task, gboolean is_soft)
 {
 	GList *part;
 	struct mime_part *p;
@@ -111,8 +111,13 @@ free_task (struct worker_task *task)
 			g_list_free_1 (part);
 		}
 		memory_pool_delete (task->task_pool);
-		/* Plan dispatcher shutdown */
-		task->dispatcher->wanna_die = 1;
+		if (is_soft) {
+			/* Plan dispatcher shutdown */
+			task->dispatcher->wanna_die = 1;
+		}
+		else {
+			rspamd_remove_dispatcher (task->dispatcher);
+		}
 		close (task->sock);
 		g_free (task);
 	}
@@ -178,11 +183,11 @@ write_socket (void *arg)
 			break;
 		case CLOSING_CONNECTION:
 			msg_debug ("write_socket: normally closing connection");
-			free_task (task);
+			free_task (task, TRUE);
 			break;
 		default:
 			msg_info ("write_socket: abnormally closing connection");
-			free_task (task);
+			free_task (task, TRUE);
 			break;
 	}
 }
@@ -196,7 +201,7 @@ err_socket (GError *err, void *arg)
 	struct worker_task *task = (struct worker_task *)arg;
 	msg_info ("err_socket: abnormally closing connection, error: %s", err->message);
 	/* Free buffers */
-	free_task (task);
+	free_task (task, FALSE);
 }
 
 /*
@@ -231,6 +236,7 @@ accept_socket (int fd, short what, void *arg)
 		msg_err ("accept_socket: cannot allocate memory for task, %m");
 		return;
 	}
+	msg_debug ("accept_socket: new task allocated: %p", new_task);
 	bzero (new_task, sizeof (struct worker_task));
 	new_task->worker = worker;
 	new_task->state = READ_COMMAND;
diff --git a/utils/url_extracter.c b/utils/url_extracter.c
index ede39ec27..9d5fa1241 100644
--- a/utils/url_extracter.c
+++ b/utils/url_extracter.c
@@ -134,6 +134,7 @@ main (int argc, char **argv)
 	message = g_mime_parser_construct_message (parser);
 	
 	task.message = message;
+	task.task_pool = memory_pool_new (memory_pool_get_size ());
 	TAILQ_INIT (&task.urls);
 
 	/* free the parser (and the stream) */