* Add ability for scanning non-mime (HTML) messages

5 files changed, 135 insertions, 49 deletions

diff --git a/src/main.h b/src/main.h
index 55a62c86c..00b2d22bc 100644
--- a/src/main.h
+++ b/src/main.h
@@ -173,6 +173,7 @@ struct worker_task {
 	enum rspamd_command cmd;									/**< command										*/
 	struct custom_command *custom_cmd;							/**< custom command if any							*/	
 	int sock;													/**< socket descriptor								*/
+    gboolean is_mime;                                           /**< if this task is mime task                      */
 	char *helo;													/**< helo header value								*/
 	char *from;													/**< from header value								*/
 	char *queue_id;												/**< queue id if specified							*/
@@ -182,6 +183,7 @@ struct worker_task {
 	struct in_addr from_addr;									/**< client addr in numeric form					*/
 	char *deliver_to;											/**< address to deliver								*/
 	char *user;													/**< user to deliver								*/
+	char *subject;												/**< subject (for non-mime)							*/
 	f_str_t *msg;												/**< message buffer									*/
 	rspamd_io_dispatcher_t *dispatcher;							/**< IO dispatcher object							*/
     struct rspamd_async_session* s;								/**< async session object							*/
diff --git a/src/message.c b/src/message.c
index 5cdd24768..597eb3a82 100644
--- a/src/message.c
+++ b/src/message.c
@@ -693,85 +693,141 @@ process_message (struct worker_task *task)
 	GMimeStream                    *stream;
 	GByteArray                     *tmp;
 	GList                          *first, *cur;
+	GMimePart                      *part;
+	GMimeDataWrapper               *wrapper;
 	struct received_header         *recv;
+	char                           *mid;
 
 	tmp = memory_pool_alloc (task->task_pool, sizeof (GByteArray));
 	tmp->data = task->msg->begin;
 	tmp->len = task->msg->len;
+
 	stream = g_mime_stream_mem_new_with_byte_array (tmp);
 	/* 
-	 * This causes g_mime_stream not to free memory by itself as it is memory allocated by
-	 * pool allocator
-	 */
+	* This causes g_mime_stream not to free memory by itself as it is memory allocated by
+	* pool allocator
+	*/
 	g_mime_stream_mem_set_owner (GMIME_STREAM_MEM (stream), FALSE);
 
-	msg_debug ("process_message: construct mime parser from string length %ld", (long int)task->msg->len);
-	/* create a new parser object to parse the stream */
-	parser = g_mime_parser_new_with_stream (stream);
-	g_object_unref (stream);
+	if (task->is_mime) {
 
-	/* parse the message from the stream */
-	message = g_mime_parser_construct_message (parser);
+		msg_debug ("process_message: construct mime parser from string length %ld", (long int)task->msg->len);
+		/* create a new parser object to parse the stream */
+		parser = g_mime_parser_new_with_stream (stream);
+		g_object_unref (stream);
 
-	if (message == NULL) {
-		msg_warn ("process_message: cannot construct mime from stream");
-		return -1;
-	}
+		/* parse the message from the stream */
+		message = g_mime_parser_construct_message (parser);
+
+		if (message == NULL) {
+			msg_warn ("process_message: cannot construct mime from stream");
+			return -1;
+		}
 
-	task->message = message;
-	memory_pool_add_destructor (task->task_pool, (pool_destruct_func) destroy_message, task->message);
+		task->message = message;
+		memory_pool_add_destructor (task->task_pool, (pool_destruct_func) destroy_message, task->message);
 
-	task->parser_recursion = 0;
+		task->parser_recursion = 0;
 #ifdef GMIME24
-	g_mime_message_foreach (message, mime_foreach_callback, task);
+		g_mime_message_foreach (message, mime_foreach_callback, task);
 #else
-	g_mime_message_foreach_part (message, mime_foreach_callback, task);
+		g_mime_message_foreach_part (message, mime_foreach_callback, task);
 #endif
 
-	msg_debug ("process_message: found %d parts in message", task->parts_count);
-	if (task->queue_id == NULL) {
-		task->queue_id = (char *)g_mime_message_get_message_id (task->message);
-	}
-	task->message_id = g_mime_message_get_message_id (task->message);
-	if (task->message_id == NULL) {
-		task->message_id = "undef";
-	}
+		msg_debug ("process_message: found %d parts in message", task->parts_count);
+		if (task->queue_id == NULL) {
+			task->queue_id = (char *)g_mime_message_get_message_id (task->message);
+		}
+		task->message_id = g_mime_message_get_message_id (task->message);
+		if (task->message_id == NULL) {
+			task->message_id = "undef";
+		}
 
 #ifdef GMIME24
-	task->raw_headers = g_mime_object_get_headers (GMIME_OBJECT (task->message));
+		task->raw_headers = g_mime_object_get_headers (GMIME_OBJECT (task->message));
 #else
-	task->raw_headers = g_mime_message_get_headers (task->message);
+		task->raw_headers = g_mime_message_get_headers (task->message);
 #endif
 
-	/* Parse received headers */
-	first = message_get_header (task->task_pool, message, "Received");
-	cur = first;
-	while (cur) {
-		recv = memory_pool_alloc0 (task->task_pool, sizeof (struct received_header));
-		parse_recv_header (task->task_pool, cur->data, recv);
-		task->received = g_list_prepend (task->received, recv);
-		cur = g_list_next (cur);
-	}
-	if (first) {
-		g_list_free (first);
-	}
+		/* Parse received headers */
+		first = message_get_header (task->task_pool, message, "Received");
+		cur = first;
+		while (cur) {
+			recv = memory_pool_alloc0 (task->task_pool, sizeof (struct received_header));
+			parse_recv_header (task->task_pool, cur->data, recv);
+			task->received = g_list_prepend (task->received, recv);
+			cur = g_list_next (cur);
+		}
+		if (first) {
+			g_list_free (first);
+		}
 
-	if (task->raw_headers) {
-		memory_pool_add_destructor (task->task_pool, (pool_destruct_func) g_free, task->raw_headers);
-	}
+		if (task->raw_headers) {
+			memory_pool_add_destructor (task->task_pool, (pool_destruct_func) g_free, task->raw_headers);
+		}
+
+		task->rcpts = g_mime_message_get_all_recipients (message);
+		if (task->rcpts) {
+			memory_pool_add_destructor (task->task_pool, (pool_destruct_func) internet_address_list_destroy, task->rcpts);
+		}
 
-	task->rcpts = g_mime_message_get_all_recipients (message);
-	if (task->rcpts) {
-		memory_pool_add_destructor (task->task_pool, (pool_destruct_func) internet_address_list_destroy, task->rcpts);
+
+		/* free the parser (and the stream) */
+		g_object_unref (parser);
+	}
+	else {
+		/* We got only message, no mime headers or anything like this */
+		/* Construct fake message for it */
+		task->message = g_mime_message_new (TRUE);
+		if (task->from) {
+			g_mime_message_set_sender (task->message, task->from);
+		}
+		/* Construct part for it */
+		part = g_mime_part_new_with_type ("text", "html");
+#ifdef GMIME24
+		wrapper = g_mime_data_wrapper_new_with_stream (stream, GMIME_CONTENT_ENCODING_8BIT);
+#else
+		wrapper = g_mime_data_wrapper_new_with_stream (stream, GMIME_PART_ENCODING_8BIT);
+#endif
+		g_mime_part_set_content_object (part, wrapper);
+		g_mime_message_set_mime_part (task->message, GMIME_OBJECT (part));
+		/* Register destructors */
+		memory_pool_add_destructor (task->task_pool, (pool_destruct_func) g_object_unref, wrapper);
+		memory_pool_add_destructor (task->task_pool, (pool_destruct_func) g_object_unref, part);
+		memory_pool_add_destructor (task->task_pool, (pool_destruct_func) destroy_message, task->message);
+		/* Now parse in a normal way */
+		task->parser_recursion = 0;
+#ifdef GMIME24
+		g_mime_message_foreach (task->message, mime_foreach_callback, task);
+#else
+		g_mime_message_foreach_part (task->message, mime_foreach_callback, task);
+#endif
+		/* Generate message ID */
+		mid = g_mime_utils_generate_message_id ("localhost.localdomain");
+		memory_pool_add_destructor (task->task_pool, (pool_destruct_func) g_free, mid);
+		g_mime_message_set_message_id (task->message, mid);
+		task->message_id = mid;
+		task->queue_id = mid;
+		/* Set headers for message */
+		if (task->subject) {
+			g_mime_message_set_subject (task->message, task->subject);
+		}
+		/* Add recipients */
+#ifndef GMIME24
+		if (task->rcpt) {
+			cur = task->rcpt;
+			while (cur) {
+				g_mime_message_add_recipient (task->message, GMIME_RECIPIENT_TYPE_TO, NULL, (char *)cur->data);
+				cur = g_list_next (cur);
+			}
+		}
+#endif
 	}
 
 	if (task->worker) {
 		task->worker->srv->stat->messages_scanned++;
 	}
 
-	/* free the parser (and the stream) */
-	g_object_unref (parser);
-
 	return 0;
 }
 
diff --git a/src/protocol.c b/src/protocol.c
index 6a8f9b8f0..cb6966632 100644
--- a/src/protocol.c
+++ b/src/protocol.c
@@ -79,6 +79,7 @@
 #define IP_ADDR_HEADER "IP"
 #define NRCPT_HEADER "Recipient-Number"
 #define RCPT_HEADER "Rcpt"
+#define SUBJECT_HEADER "Subject"
 #define QUEUE_ID_HEADER "Queue-ID"
 #define ERROR_HEADER "Error"
 #define USER_HEADER "User"
@@ -350,6 +351,15 @@ parse_header (struct worker_task *task, f_str_t * line)
 			return -1;
 		}
 		break;
+	case 's':
+	case 'S':
+		if (strncasecmp (headern, SUBJECT_HEADER, sizeof (SUBJECT_HEADER) - 1) == 0) {
+			task->subject = memory_pool_fstrdup (task->task_pool, line);
+		}
+		else {
+			return -1;
+		}
+		break;
 	case 'u':
 	case 'U':
 		if (strncasecmp (headern, USER_HEADER, sizeof (USER_HEADER) - 1) == 0) {
diff --git a/src/spf.c b/src/spf.c
index c72698bc1..97199009e 100644
--- a/src/spf.c
+++ b/src/spf.c
@@ -230,6 +230,11 @@ parse_spf_a (struct worker_task *task, const char *begin, struct spf_record *rec
 	int hostlen;
 	
 	CHECK_REC (rec);
+	
+	if (begin == NULL || *begin != ':') {
+		return FALSE;
+	}
+	begin ++;
 
 	bzero (mask_buf, sizeof (mask_buf));
 	p = strchr (begin, '/');
diff --git a/src/worker.c b/src/worker.c
index e246f3213..c5589a533 100644
--- a/src/worker.c
+++ b/src/worker.c
@@ -51,6 +51,8 @@ extern PerlInterpreter         *perl_interpreter;
 #endif
 
 static struct timeval           io_tv;
+/* Detect whether this worker is mime worker */
+static gboolean                 is_mime;
 
 static gboolean                 write_socket (void *arg);
 
@@ -296,6 +298,7 @@ construct_task (struct rspamd_worker *worker)
 	memory_pool_add_destructor (new_task->task_pool, (pool_destruct_func) g_hash_table_destroy, new_task->re_cache);
 	new_task->s = new_async_session (new_task->task_pool, free_task_hard, new_task);
 	new_task->sock = -1;
+	new_task->is_mime = TRUE;
 
 	return new_task;
 }
@@ -334,6 +337,7 @@ accept_socket (int fd, short what, void *arg)
 	new_task = construct_task (worker);
 
 	new_task->sock = nfd;
+	new_task->is_mime = is_mime;
 	worker->srv->stat->connections_count++;
 
 	/* Set up dispatcher */
@@ -348,6 +352,7 @@ void
 start_worker (struct rspamd_worker *worker)
 {
 	struct sigaction                signals;
+	char                           *is_mime_str;
 
 #ifdef WITH_PROFILER
 	extern void                     _start (void), etext (void);
@@ -374,6 +379,14 @@ start_worker (struct rspamd_worker *worker)
 
 	/* Maps events */
 	start_map_watch ();
+	/* Check whether we are mime worker */
+	is_mime_str = g_hash_table_lookup (worker->cf->params, "mime");
+	if (is_mime_str != NULL && (g_ascii_strcasecmp (is_mime_str, "no") == 0 || g_ascii_strcasecmp (is_mime_str, "false") == 0)) {
+		is_mime = FALSE;
+	}
+	else {
+		is_mime = TRUE;
+	}
 
 	/* Send SIGUSR2 to parent */
 	kill (getppid (), SIGUSR2);