[Project] Start mime structures refactoring

4 files changed, 113 insertions, 46 deletions

diff --git a/src/libmime/message.c b/src/libmime/message.c
index 482287769..1d9da26f2 100644
--- a/src/libmime/message.c
+++ b/src/libmime/message.c
@@ -1091,6 +1091,22 @@ rspamd_message_from_data (struct rspamd_task *task, const guchar *start,
 	task->queue_id = mid;
 }
 
+static void
+rspamd_message_dtor (struct rspamd_message *msg)
+{
+
+}
+
+struct rspamd_message*
+rspamd_message_new (struct rspamd_task *task)
+{
+	struct rspamd_message *msg;
+
+	msg = rspamd_mempool_alloc0 (sizeof (*msg));
+
+
+}
+
 gboolean
 rspamd_message_parse (struct rspamd_task *task)
 {
@@ -1593,3 +1609,14 @@ rspamd_message_get_mime_header_array (struct rspamd_task *task,
 
 	return ret;
 }
+
+struct rspamd_message *
+rspamd_message_ref (struct rspamd_message *msg)
+{
+	REF_RETAIN (msg);
+}
+
+void rspamd_message_unref (struct rspamd_message *msg)
+{
+	REF_RELEASE (msg);s
+}
diff --git a/src/libmime/message.h b/src/libmime/message.h
index 17c4ec5b9..7d58fa88f 100644
--- a/src/libmime/message.h
+++ b/src/libmime/message.h
@@ -12,6 +12,8 @@
 #include "cryptobox.h"
 #include "mime_headers.h"
 #include "content_type.h"
+#include "libutil/ref.h"
+#include "libutil/str_util.h"
 
 #include <unicode/uchar.h>
 #include <unicode/utext.h>
@@ -129,6 +131,36 @@ struct rspamd_mime_text_part {
 	guint unicode_scripts;
 };
 
+struct rspamd_message {
+	const gchar *message_id;
+	gchar *subject;
+
+	GPtrArray *parts;				/**< list of parsed parts							*/
+	GPtrArray *text_parts;			/**< list of text parts								*/
+	struct {
+		const gchar *begin;
+		gsize len;
+		const gchar *body_start;
+	} raw_headers_content;			/**< list of raw headers							*/
+	GPtrArray *received;			/**< list of received headers						*/
+	GHashTable *urls;				/**< list of parsed urls							*/
+	GHashTable *emails;				/**< list of parsed emails							*/
+	GHashTable *raw_headers;		/**< list of raw headers							*/
+	GQueue *headers_order;			/**< order of raw headers							*/
+	GPtrArray *rcpt_mime;
+	GPtrArray *from_mime;
+	enum rspamd_newlines_type nlines_type; /**< type of newlines (detected on most of headers 	*/
+	ref_entry_t ref;
+};
+
+#ifndef FULL_DEBUG
+#define MESSAGE_FIELD(task, field) ((task)->message->(field))
+#else
+#define MESSAGE_FIELD(task, field) do { \
+	if (!task->message) {msg_err_task("no message when getting field %s", #field); g_assert(0);} \
+	} while(0), ((task)->message->(field))
+#endif
+
 /**
  * Parse and pre-process mime message
  * @param task worker_task object
@@ -191,6 +223,12 @@ enum rspamd_cte rspamd_cte_from_string (const gchar *str);
  */
 const gchar *rspamd_cte_to_string (enum rspamd_cte ct);
 
+struct rspamd_message* rspamd_message_new (struct rspamd_task *task);
+
+struct rspamd_message *rspamd_message_ref (struct rspamd_message *msg);
+
+void rspamd_message_unref (struct rspamd_message *msg);
+
 #ifdef  __cplusplus
 }
 #endif
diff --git a/src/libmime/mime_headers.c b/src/libmime/mime_headers.c
index cf6d0f763..952a163b9 100644
--- a/src/libmime/mime_headers.c
+++ b/src/libmime/mime_headers.c
@@ -44,31 +44,31 @@ rspamd_mime_header_check_special (struct rspamd_task *task,
 			g_ptr_array_add (task->received, recv);
 		}
 
-		rh->type = RSPAMD_HEADER_RECEIVED;
+		rh->flags = RSPAMD_HEADER_RECEIVED;
 		break;
 	case 0x76F31A09F4352521ULL:	/* to */
 		task->rcpt_mime = rspamd_email_address_from_mime (task->task_pool,
 				rh->decoded, strlen (rh->decoded), task->rcpt_mime);
-		rh->type = RSPAMD_HEADER_TO|RSPAMD_HEADER_RCPT|RSPAMD_HEADER_UNIQUE;
+		rh->flags = RSPAMD_HEADER_TO|RSPAMD_HEADER_RCPT|RSPAMD_HEADER_UNIQUE;
 		break;
 	case 0x7EB117C1480B76ULL:	/* cc */
 		task->rcpt_mime = rspamd_email_address_from_mime (task->task_pool,
 				rh->decoded, strlen (rh->decoded), task->rcpt_mime);
-		rh->type = RSPAMD_HEADER_CC|RSPAMD_HEADER_RCPT|RSPAMD_HEADER_UNIQUE;
+		rh->flags = RSPAMD_HEADER_CC|RSPAMD_HEADER_RCPT|RSPAMD_HEADER_UNIQUE;
 		break;
 	case 0xE4923E11C4989C8DULL:	/* bcc */
 		task->rcpt_mime = rspamd_email_address_from_mime (task->task_pool,
 				rh->decoded, strlen (rh->decoded), task->rcpt_mime);
-		rh->type = RSPAMD_HEADER_BCC|RSPAMD_HEADER_RCPT|RSPAMD_HEADER_UNIQUE;
+		rh->flags = RSPAMD_HEADER_BCC|RSPAMD_HEADER_RCPT|RSPAMD_HEADER_UNIQUE;
 		break;
 	case 0x41E1985EDC1CBDE4ULL:	/* from */
 		task->from_mime = rspamd_email_address_from_mime (task->task_pool,
 				rh->decoded, strlen (rh->decoded), task->from_mime);
-		rh->type = RSPAMD_HEADER_FROM|RSPAMD_HEADER_SENDER|RSPAMD_HEADER_UNIQUE;
+		rh->flags = RSPAMD_HEADER_FROM|RSPAMD_HEADER_SENDER|RSPAMD_HEADER_UNIQUE;
 		break;
 	case 0x43A558FC7C240226ULL:	/* message-id */ {
 
-		rh->type = RSPAMD_HEADER_MESSAGE_ID|RSPAMD_HEADER_UNIQUE;
+		rh->flags = RSPAMD_HEADER_MESSAGE_ID|RSPAMD_HEADER_UNIQUE;
 		p = rh->decoded;
 		end = p + strlen (p);
 
@@ -107,20 +107,20 @@ rspamd_mime_header_check_special (struct rspamd_task *task,
 		if (task->subject == NULL) {
 			task->subject = rh->decoded;
 		}
-		rh->type = RSPAMD_HEADER_SUBJECT|RSPAMD_HEADER_UNIQUE;
+		rh->flags = RSPAMD_HEADER_SUBJECT|RSPAMD_HEADER_UNIQUE;
 		break;
 	case 0xEE4AA2EAAC61D6F4ULL:	/* return-path */
 		if (task->from_envelope == NULL) {
 			task->from_envelope = rspamd_email_address_from_smtp (rh->decoded,
 					strlen (rh->decoded));
 		}
-		rh->type = RSPAMD_HEADER_RETURN_PATH|RSPAMD_HEADER_UNIQUE;
+		rh->flags = RSPAMD_HEADER_RETURN_PATH|RSPAMD_HEADER_UNIQUE;
 		break;
 	case 0xB9EEFAD2E93C2161ULL:	/* delivered-to */
 		if (task->deliver_to == NULL) {
 			task->deliver_to = rh->decoded;
 		}
-		rh->type = RSPAMD_HEADER_DELIVERED_TO;
+		rh->flags = RSPAMD_HEADER_DELIVERED_TO;
 		break;
 	case 0x2EC3BFF3C393FC10ULL: /* date */
 	case 0xAC0DDB1A1D214CAULL: /* sender */
@@ -128,7 +128,7 @@ rspamd_mime_header_check_special (struct rspamd_task *task,
 	case 0x81CD9E9131AB6A9AULL: /* content-type */
 	case 0xC39BD9A75AA25B60ULL: /* content-transfer-encoding */
 	case 0xB3F6704CB3AD6589ULL: /* references */
-		rh->type = RSPAMD_HEADER_UNIQUE;
+		rh->flags = RSPAMD_HEADER_UNIQUE;
 		break;
 	}
 }
@@ -472,7 +472,7 @@ rspamd_mime_headers_process (struct rspamd_task *task, GHashTable *target,
 		while (cur) {
 			nh = cur->data;
 
-			if (nh->name && nh->type != RSPAMD_HEADER_RECEIVED) {
+			if (nh->name && nh->flags != RSPAMD_HEADER_RECEIVED) {
 				rspamd_cryptobox_hash_update (&hs, nh->name, strlen (nh->name));
 			}
 
diff --git a/src/libmime/mime_headers.h b/src/libmime/mime_headers.h
index 5cb300978..60fd7b697 100644
--- a/src/libmime/mime_headers.h
+++ b/src/libmime/mime_headers.h
@@ -31,20 +31,22 @@ enum rspamd_rfc2047_encoding {
 	RSPAMD_RFC2047_BASE64,
 };
 
-enum rspamd_mime_header_special_type {
-	RSPAMD_HEADER_GENERIC = 0,
-	RSPAMD_HEADER_RECEIVED = 1 << 0,
-	RSPAMD_HEADER_TO = 1 << 2,
-	RSPAMD_HEADER_CC = 1 << 3,
-	RSPAMD_HEADER_BCC = 1 << 4,
-	RSPAMD_HEADER_FROM = 1 << 5,
-	RSPAMD_HEADER_MESSAGE_ID = 1 << 6,
-	RSPAMD_HEADER_SUBJECT = 1 << 7,
-	RSPAMD_HEADER_RETURN_PATH = 1 << 8,
-	RSPAMD_HEADER_DELIVERED_TO = 1 << 9,
-	RSPAMD_HEADER_SENDER = 1 << 10,
-	RSPAMD_HEADER_RCPT = 1 << 11,
-	RSPAMD_HEADER_UNIQUE = 1 << 12
+enum rspamd_mime_header_flags {
+	RSPAMD_HEADER_GENERIC = 0u,
+	RSPAMD_HEADER_RECEIVED = 1u << 0u,
+	RSPAMD_HEADER_TO = 1u << 2u,
+	RSPAMD_HEADER_CC = 1u << 3u,
+	RSPAMD_HEADER_BCC = 1u << 4u,
+	RSPAMD_HEADER_FROM = 1u << 5u,
+	RSPAMD_HEADER_MESSAGE_ID = 1u << 6u,
+	RSPAMD_HEADER_SUBJECT = 1u << 7u,
+	RSPAMD_HEADER_RETURN_PATH = 1u << 8u,
+	RSPAMD_HEADER_DELIVERED_TO = 1u << 9u,
+	RSPAMD_HEADER_SENDER = 1u << 10u,
+	RSPAMD_HEADER_RCPT = 1u << 11u,
+	RSPAMD_HEADER_UNIQUE = 1u << 12u,
+	RSPAMD_HEADER_EMPTY_SEPARATOR = 1u << 13u,
+	RSPAMD_HEADER_TAB_SEPARATED = 1u << 14u,
 };
 
 struct rspamd_mime_header {
@@ -52,32 +54,31 @@ struct rspamd_mime_header {
 	gchar *value;
 	const gchar *raw_value; /* As it is in the message (unfolded and unparsed) */
 	gsize raw_len;
-	gboolean tab_separated;
-	gboolean empty_separator;
 	guint order;
-	enum rspamd_mime_header_special_type type;
+	int flags; /* see enum rspamd_mime_header_flags */
 	gchar *separator;
 	gchar *decoded;
+	struct rspamd_mime_header *prev, *next; /* Headers with the same name */
+	struct rspamd_mime_header *ord_prev, *ord_next; /* Overall order of headers */
 };
 
 enum rspamd_received_type {
 	RSPAMD_RECEIVED_SMTP = 0,
-	RSPAMD_RECEIVED_ESMTP,
-	RSPAMD_RECEIVED_ESMTPA,
-	RSPAMD_RECEIVED_ESMTPS,
-	RSPAMD_RECEIVED_ESMTPSA,
-	RSPAMD_RECEIVED_LMTP,
-	RSPAMD_RECEIVED_IMAP,
-	RSPAMD_RECEIVED_LOCAL,
-	RSPAMD_RECEIVED_HTTP,
-	RSPAMD_RECEIVED_MAPI,
-	RSPAMD_RECEIVED_UNKNOWN
+	RSPAMD_RECEIVED_ESMTP = 1u << 0u,
+	RSPAMD_RECEIVED_ESMTPA = 1u << 1u,
+	RSPAMD_RECEIVED_ESMTPS = 1u << 2u,
+	RSPAMD_RECEIVED_ESMTPSA = 1u << 3u,
+	RSPAMD_RECEIVED_LMTP = 1u << 4u,
+	RSPAMD_RECEIVED_IMAP = 1u << 5u,
+	RSPAMD_RECEIVED_LOCAL = 1u << 6u,
+	RSPAMD_RECEIVED_HTTP = 1u << 7u,
+	RSPAMD_RECEIVED_MAPI = 1u << 8u,
+	RSPAMD_RECEIVED_UNKNOWN = 1u << 9u,
+	RSPAMD_RECEIVED_FLAG_ARTIFICIAL =  (1u << 10u),
+	RSPAMD_RECEIVED_FLAG_SSL =  (1u << 11u),
+	RSPAMD_RECEIVED_FLAG_AUTHENTICATED =  (1u << 12u),
 };
 
-#define RSPAMD_RECEIVED_FLAG_ARTIFICIAL (1 << 0)
-#define RSPAMD_RECEIVED_FLAG_SSL (1 << 1)
-#define RSPAMD_RECEIVED_FLAG_AUTHENTICATED (1 << 2)
-
 struct received_header {
 	const gchar *from_hostname;
 	const gchar *from_ip;
@@ -88,8 +89,8 @@ struct received_header {
 	rspamd_inet_addr_t *addr;
 	struct rspamd_mime_header *hdr;
 	time_t timestamp;
-	enum rspamd_received_type type;
-	gint flags;
+	gint flags; /* See enum rspamd_received_type */
+	struct received_header *prev, *next;
 };
 
 /**
@@ -100,8 +101,9 @@ struct received_header {
  * @param len
  * @param check_newlines
  */
-void rspamd_mime_headers_process (struct rspamd_task *task, GHashTable *target,
-								  GQueue *order,
+void rspamd_mime_headers_process (struct rspamd_task *task,
+								  GHashTable *target,
+								  struct rspamd_mime_header **order_ptr,
 								  const gchar *in, gsize len,
 								  gboolean check_newlines);