* Add smart handling of message headers

CMakeLists.txt

@@ -272,6 +272,7 @@ SET(TESTDEPENDS	src/mem_pool.c
 				src/util.c
 				src/fuzzy.c
 				src/memcached.c
+				src/message.c
 				src/expressions.c
 				src/statfile.c)
 

perl/Rspamd.xs

@@ -126,262 +126,9 @@ call_sub_foreach(GMimeObject *mime_object, gpointer data)
 		perl_call_sv (svdata->svfunc, G_DISCARD);
 	}
 }
-/* known header field types */
-enum {
-	HEADER_FROM = 0,
-	HEADER_REPLY_TO,
-	HEADER_TO,
-	HEADER_CC,
-	HEADER_BCC,
-	HEADER_SUBJECT,
-	HEADER_DATE,
-	HEADER_MESSAGE_ID,
-	HEADER_UNKNOWN
-};
-
-static GList *
-local_message_get_header(GMimeMessage *message, const char *field)
-{
-	GList *	gret = NULL;
-#ifndef GMIME24
-	struct raw_header *h;
-
-	if (field == NULL) {
-		return NULL;
-	}
-	h = GMIME_OBJECT(message)->headers->headers;
-	while (h) {
-		if (h->value && !g_strncasecmp(field, h->name, strlen(field))) {
-			gret = g_list_prepend(gret, g_strdup(h->value));
-		}
-		h = h->next;
-	}
-	return gret;
-#else
-	GMimeHeaderList *ls;
-	GMimeHeaderIter *iter;
-	const char *name;
-
-	ls = GMIME_OBJECT(message)->headers;
-
-	if (g_mime_header_list_get_iter (ls, iter)) {
-		while (g_mime_header_iter_is_valid (iter)) {
-			name = g_mime_header_iter_get_name (iter);
-			if (!g_strncasecmp (field, name, strlen (name))) {
-				gret = g_list_prepend (gret, g_strdup (g_mime_header_iter_get_value (iter)));
-			}
-			if (!g_mime_header_iter_next (iter)) {
-				break;
-			}
-		}
-	}
-
-	return gret;
-#endif
-}
-
-/**
-* g_mime_message_set_date_from_string: Set the message sent-date
-* @message: MIME Message
-* @string: A string of date
-* 
-* Set the sent-date on a MIME Message.
-**/			 
-static void
-local_mime_message_set_date_from_string (GMimeMessage *message, const gchar *string) 
-{
-	time_t date;
-	int offset = 0;
-
-	date = g_mime_utils_header_decode_date (string, &offset);
-	g_mime_message_set_date (message, date, offset); 
-}
-
-#ifdef GMIME24
-
-#define ADD_RECIPIENT_TEMPLATE(type,def)														\
-static void																						\
-local_message_add_recipients_from_string_##type (GMimeMessage *message, const gchar *string, const gchar *value)	\
-{																								\
-	InternetAddressList *il, *new;																\
-																								\
-	il = g_mime_message_get_recipients (message, (def));										\
-	new = internet_address_list_parse_string (string);											\
-	internet_address_list_append (il, new);														\
-}																								\
-
-ADD_RECIPIENT_TEMPLATE(to, GMIME_RECIPIENT_TYPE_TO)
-ADD_RECIPIENT_TEMPLATE(cc, GMIME_RECIPIENT_TYPE_CC)
-ADD_RECIPIENT_TEMPLATE(bcc, GMIME_RECIPIENT_TYPE_BCC)
-
-#define GET_RECIPIENT_TEMPLATE(type,def)														\
-static InternetAddressList*																		\
-local_message_get_recipients_##type (GMimeMessage *message, const char *unused)					\
-{																								\
-	return g_mime_message_get_recipients (message, (def));										\
-}
-
-GET_RECIPIENT_TEMPLATE(to, GMIME_RECIPIENT_TYPE_TO)
-GET_RECIPIENT_TEMPLATE(cc, GMIME_RECIPIENT_TYPE_CC)
-GET_RECIPIENT_TEMPLATE(bcc, GMIME_RECIPIENT_TYPE_BCC)
-
-#endif
 
 
-/* different declarations for different types of set and get functions */
-typedef const char *(*GetFunc) (GMimeMessage *message);
-typedef InternetAddressList *(*GetRcptFunc) (GMimeMessage *message, const char *type );
-typedef GList *(*GetListFunc) (GMimeMessage *message, const char *type );
-typedef void	 (*SetFunc) (GMimeMessage *message, const char *value);
-typedef void	 (*SetListFunc) (GMimeMessage *message, const char *field, const char *value);
 
-/** different types of functions
-*
-* FUNC_CHARPTR
-*	- function with no arguments
-*	- get returns char*
-*
-* FUNC_IA (from Internet Address)
-*	- function with additional "field" argument from the fieldfunc table,
-*	- get returns Glist*
-*
-* FUNC_LIST
-*	- function with additional "field" argument (given arbitrary header field name)
-*	- get returns Glist*
-**/
-enum {
-	FUNC_CHARPTR = 0,
-	FUNC_CHARFREEPTR,
-	FUNC_IA,
-	FUNC_LIST
-};
-
-/**
-* fieldfunc struct: structure of MIME fields and corresponding get and set
-* functions.
-**/
-static struct {
-	char *	name;
-	GetFunc	func;
-	GetRcptFunc	rcptfunc;
-	GetListFunc	getlistfunc;
-	SetFunc	setfunc;
-	SetListFunc	setlfunc;
-	gint		functype;
-} fieldfunc[] = {
-	{ "From",		g_mime_message_get_sender,		NULL, NULL,				g_mime_message_set_sender,	NULL, FUNC_CHARPTR },
-	{ "Reply-To",	g_mime_message_get_reply_to,	NULL, NULL,				g_mime_message_set_reply_to,	NULL, FUNC_CHARPTR },
-#ifndef GMIME24
-	{ "To",	NULL,	g_mime_message_get_recipients,	NULL, NULL, 			g_mime_message_add_recipients_from_string, FUNC_IA },
-	{ "Cc",	NULL,	g_mime_message_get_recipients,	NULL, NULL, 			g_mime_message_add_recipients_from_string, FUNC_IA },
-	{ "Bcc",	NULL,	g_mime_message_get_recipients,	NULL, NULL, 		g_mime_message_add_recipients_from_string, FUNC_IA },
-	{ "Date",		g_mime_message_get_date_string, NULL, NULL,				local_mime_message_set_date_from_string,	NULL, FUNC_CHARFREEPTR },
-#else
-	{ "To",	NULL,	local_message_get_recipients_to,	NULL, NULL, 			local_message_add_recipients_from_string_to, FUNC_IA },
-	{ "Cc",	NULL,	local_message_get_recipients_cc,	NULL, NULL, 			local_message_add_recipients_from_string_cc, FUNC_IA },
-	{ "Bcc",	NULL,	local_message_get_recipients_bcc,	NULL, NULL, 		local_message_add_recipients_from_string_bcc, FUNC_IA },
-	{ "Date",		g_mime_message_get_date_as_string, NULL, NULL,				local_mime_message_set_date_from_string,	NULL, FUNC_CHARFREEPTR },
-#endif
-	{ "Subject",	g_mime_message_get_subject,		NULL, NULL,				g_mime_message_set_subject,	NULL, FUNC_CHARPTR },
-	{ "Message-Id",	g_mime_message_get_message_id,	NULL, NULL,				g_mime_message_set_message_id,	NULL, FUNC_CHARPTR },
-#ifndef GMIME24
-	{ NULL,	NULL,	NULL,	local_message_get_header,	  NULL,				g_mime_message_add_header, FUNC_LIST }
-#else
-	{ NULL,	NULL,	NULL,	local_message_get_header,	  NULL,				g_mime_object_append_header, FUNC_LIST }
-#endif
-};
-
-
-
-/**
-* message_set_header: set header of any type excluding special (Content- and MIME-Version:)
-**/
-static void
-message_set_header (GMimeMessage *message, const char *field, const char *value) 
-{
-	gint i;
-
-
-	if (!g_strcasecmp (field, "MIME-Version:") || !g_strncasecmp (field, "Content-", 8)) {
-		return;
-	}
-	for (i=0; i<=HEADER_UNKNOWN; ++i) {
-		if (!fieldfunc[i].name || !g_strncasecmp(field, fieldfunc[i].name, strlen(fieldfunc[i].name))) { 
-			switch (fieldfunc[i].functype) {
-				case FUNC_CHARPTR:
-					(*(fieldfunc[i].setfunc))(message, value);
-					break;
-				case FUNC_IA:
-					(*(fieldfunc[i].setlfunc))(message, fieldfunc[i].name, value);
-					break;
-				case FUNC_LIST:
-					(*(fieldfunc[i].setlfunc))(message, field, value);
-					break;
-			}
-			break;
-		}		 
-	}
-}
-
-
-/**
-* message_get_header: returns the list of 'any header' values
-* (except of unsupported yet Content- and MIME-Version special headers)
-*
-* You should free the GList list by yourself.
-**/
-static
-GList *
-message_get_header(GMimeMessage *message, const char *field) {
-	gint		i;
-	char *	ret = NULL, *ia_string;
-	GList *	gret = NULL;
-	InternetAddressList *ia_list = NULL, *ia;
-
-	for (i = 0; i <= HEADER_UNKNOWN; ++i) {
-		if (!fieldfunc[i].name || !g_strncasecmp(field, fieldfunc[i].name, strlen(fieldfunc[i].name))) { 
-			switch (fieldfunc[i].functype) {
-				case FUNC_CHARFREEPTR:
-					ret = (char *)(*(fieldfunc[i].func))(message);
-					break;
-				case FUNC_CHARPTR:
-					ret = (char *)(*(fieldfunc[i].func))(message);
-					break;
-				case FUNC_IA:
-					ia_list = (*(fieldfunc[i].rcptfunc))(message, field);
-					gret = g_list_alloc();
-					ia = ia_list;
-#ifndef GMIME24
-					while (ia && ia->address) {
-
-						ia_string = internet_address_to_string ((InternetAddress *)ia->address, FALSE);
-						gret = g_list_append (gret, ia_string);
-						ia = ia->next;
-					}
-#else
-					i = internet_address_list_length (ia);
-					while (i > 0) {
-						ia_string = internet_address_to_string (internet_address_list_get_address (ia, i), FALSE);
-						gret = g_list_append (gret, ia_string);
-						-- i;
-					}
-#endif
-					break;
-				case FUNC_LIST:
-					gret = (*(fieldfunc[i].getlistfunc))(message, field);
-					break;
-			}
-			break;
-		}		 
-	}
-	if (gret == NULL && ret != NULL) {
-		gret = g_list_prepend (gret, g_strdup (ret));
-	}
-	if (fieldfunc[i].functype == FUNC_CHARFREEPTR && ret) {
-		g_free (ret);
-	}
-	return gret;
-}
 
 MODULE = Mail::Rspamd   PACKAGE = Mail::Rspamd::Log PREFIX = rspamd_log_
 PROTOTYPES: DISABLE

src/expressions.c

@@ -585,6 +585,7 @@ gboolean
 rspamd_header_exists (struct worker_task *task, GList *args)
 {
 	struct expression_argument *arg;
+	GList *headerlist;
 
 	if (args == NULL || task == NULL) {
 		return FALSE;
@@ -595,11 +596,13 @@ rspamd_header_exists (struct worker_task *task, GList *args)
 		msg_warn ("rspamd_header_exists: invalid argument to function is passed");
 		return FALSE;
 	}
-#ifdef GMIME24
-	return (g_mime_object_get_header (GMIME_OBJECT (task->message), (char *)arg->data) != NULL);
-#else
-	return (g_mime_message_get_header (task->message, (char *)arg->data) != NULL);
-#endif
+
+	headerlist = message_get_header (task->message, (char *)arg->data);
+	if (headerlist) {
+		g_list_free (headerlist);
+		return TRUE;
+	}
+	return FALSE;
 }
 
 /*

src/message.c

@@ -552,3 +552,269 @@ get_next_text_part (memory_pool_t *pool, GList *parts, GList **cur)
 	
 	return NULL;
 }
+
+struct raw_header {
+    struct raw_header *next;
+    char *name;
+    char *value;
+};			
+
+typedef struct _GMimeHeader {
+	GHashTable *hash;
+	GHashTable *writers;
+	struct raw_header *headers;
+} local_GMimeHeader;
+
+
+/* known header field types */
+enum {
+	HEADER_FROM = 0,
+	HEADER_REPLY_TO,
+	HEADER_TO,
+	HEADER_CC,
+	HEADER_BCC,
+	HEADER_SUBJECT,
+	HEADER_DATE,
+	HEADER_MESSAGE_ID,
+	HEADER_UNKNOWN
+};
+
+static GList *
+local_message_get_header(GMimeMessage *message, const char *field)
+{
+	GList *	gret = NULL;
+#ifndef GMIME24
+	struct raw_header *h;
+
+	if (field == NULL) {
+		return NULL;
+	}
+	h = GMIME_OBJECT(message)->headers->headers;
+	while (h) {
+		if (h->value && !g_strncasecmp (field, h->name, strlen (field))) {
+			gret = g_list_prepend(gret, g_strdup (h->value));
+		}
+		h = h->next;
+	}
+	return gret;
+#else
+	GMimeHeaderList *ls;
+	GMimeHeaderIter *iter;
+	const char *name;
+
+	ls = GMIME_OBJECT(message)->headers;
+
+	if (g_mime_header_list_get_iter (ls, iter)) {
+		while (g_mime_header_iter_is_valid (iter)) {
+			name = g_mime_header_iter_get_name (iter);
+			if (!g_strncasecmp (field, name, strlen (name))) {
+				gret = g_list_prepend (gret, g_strdup (g_mime_header_iter_get_value (iter)));
+			}
+			if (!g_mime_header_iter_next (iter)) {
+				break;
+			}
+		}
+	}
+
+	return gret;
+#endif
+}
+
+/**
+* g_mime_message_set_date_from_string: Set the message sent-date
+* @message: MIME Message
+* @string: A string of date
+* 
+* Set the sent-date on a MIME Message.
+**/			 
+void
+local_mime_message_set_date_from_string (GMimeMessage *message, const gchar *string) 
+{
+	time_t date;
+	int offset = 0;
+
+	date = g_mime_utils_header_decode_date (string, &offset);
+	g_mime_message_set_date (message, date, offset); 
+}
+
+#ifdef GMIME24
+
+#define ADD_RECIPIENT_TEMPLATE(type,def)														\
+static void																						\
+local_message_add_recipients_from_string_##type (GMimeMessage *message, const gchar *string, const gchar *value)	\
+{																								\
+	InternetAddressList *il, *new;																\
+																								\
+	il = g_mime_message_get_recipients (message, (def));										\
+	new = internet_address_list_parse_string (string);											\
+	internet_address_list_append (il, new);														\
+}																								\
+
+ADD_RECIPIENT_TEMPLATE(to, GMIME_RECIPIENT_TYPE_TO)
+ADD_RECIPIENT_TEMPLATE(cc, GMIME_RECIPIENT_TYPE_CC)
+ADD_RECIPIENT_TEMPLATE(bcc, GMIME_RECIPIENT_TYPE_BCC)
+
+#define GET_RECIPIENT_TEMPLATE(type,def)														\
+static InternetAddressList*																		\
+local_message_get_recipients_##type (GMimeMessage *message, const char *unused)					\
+{																								\
+	return g_mime_message_get_recipients (message, (def));										\
+}
+
+GET_RECIPIENT_TEMPLATE(to, GMIME_RECIPIENT_TYPE_TO)
+GET_RECIPIENT_TEMPLATE(cc, GMIME_RECIPIENT_TYPE_CC)
+GET_RECIPIENT_TEMPLATE(bcc, GMIME_RECIPIENT_TYPE_BCC)
+
+#endif
+
+
+/* different declarations for different types of set and get functions */
+typedef const char *(*GetFunc) (GMimeMessage *message);
+typedef InternetAddressList *(*GetRcptFunc) (GMimeMessage *message, const char *type );
+typedef GList *(*GetListFunc) (GMimeMessage *message, const char *type );
+typedef void	 (*SetFunc) (GMimeMessage *message, const char *value);
+typedef void	 (*SetListFunc) (GMimeMessage *message, const char *field, const char *value);
+
+/** different types of functions
+*
+* FUNC_CHARPTR
+*	- function with no arguments
+*	- get returns char*
+*
+* FUNC_IA (from Internet Address)
+*	- function with additional "field" argument from the fieldfunc table,
+*	- get returns Glist*
+*
+* FUNC_LIST
+*	- function with additional "field" argument (given arbitrary header field name)
+*	- get returns Glist*
+**/
+enum {
+	FUNC_CHARPTR = 0,
+	FUNC_CHARFREEPTR,
+	FUNC_IA,
+	FUNC_LIST
+};
+
+/**
+* fieldfunc struct: structure of MIME fields and corresponding get and set
+* functions.
+**/
+static struct {
+	char *	name;
+	GetFunc	func;
+	GetRcptFunc	rcptfunc;
+	GetListFunc	getlistfunc;
+	SetFunc	setfunc;
+	SetListFunc	setlfunc;
+	gint		functype;
+} fieldfunc[] = {
+	{ "From",		g_mime_message_get_sender,		NULL, NULL,				g_mime_message_set_sender,	NULL, FUNC_CHARPTR },
+	{ "Reply-To",	g_mime_message_get_reply_to,	NULL, NULL,				g_mime_message_set_reply_to,	NULL, FUNC_CHARPTR },
+#ifndef GMIME24
+	{ "To",	NULL,	(GetRcptFunc)g_mime_message_get_recipients,	NULL, NULL, (SetListFunc)g_mime_message_add_recipients_from_string, FUNC_IA },
+	{ "Cc",	NULL,	(GetRcptFunc)g_mime_message_get_recipients,	NULL, NULL, (SetListFunc)g_mime_message_add_recipients_from_string, FUNC_IA },
+	{ "Bcc",NULL,	(GetRcptFunc)g_mime_message_get_recipients,	NULL, NULL, (SetListFunc)g_mime_message_add_recipients_from_string, FUNC_IA },
+	{ "Date", (GetFunc)g_mime_message_get_date_string, NULL, NULL,			local_mime_message_set_date_from_string,	NULL, FUNC_CHARFREEPTR },
+#else
+	{ "To",	NULL,	local_message_get_recipients_to,	NULL, NULL, 		local_message_add_recipients_from_string_to, FUNC_IA },
+	{ "Cc",	NULL,	local_message_get_recipients_cc,	NULL, NULL, 		local_message_add_recipients_from_string_cc, FUNC_IA },
+	{ "Bcc",	NULL,	local_message_get_recipients_bcc,	NULL, NULL, 	local_message_add_recipients_from_string_bcc, FUNC_IA },
+	{ "Date",		g_mime_message_get_date_as_string, NULL, NULL,			local_mime_message_set_date_from_string,	NULL, FUNC_CHARFREEPTR },
+#endif
+	{ "Subject",	g_mime_message_get_subject,		NULL, NULL,				g_mime_message_set_subject,	NULL, FUNC_CHARPTR },
+	{ "Message-Id",	g_mime_message_get_message_id,	NULL, NULL,				g_mime_message_set_message_id,	NULL, FUNC_CHARPTR },
+#ifndef GMIME24
+	{ NULL,	NULL,	NULL,	local_message_get_header,	  NULL,				g_mime_message_add_header, FUNC_LIST }
+#else
+	{ NULL,	NULL,	NULL,	local_message_get_header,	  NULL,				g_mime_object_append_header, FUNC_LIST }
+#endif
+};
+/**
+* message_set_header: set header of any type excluding special (Content- and MIME-Version:)
+**/
+void
+message_set_header (GMimeMessage *message, const char *field, const char *value) 
+{
+	gint i;
+
+	if (!g_strcasecmp (field, "MIME-Version:") || !g_strncasecmp (field, "Content-", 8)) {
+		return;
+	}
+	for (i=0; i<=HEADER_UNKNOWN; ++i) {
+		if (!fieldfunc[i].name || !g_strncasecmp(field, fieldfunc[i].name, strlen(fieldfunc[i].name))) { 
+			switch (fieldfunc[i].functype) {
+				case FUNC_CHARPTR:
+					(*(fieldfunc[i].setfunc))(message, value);
+					break;
+				case FUNC_IA:
+					(*(fieldfunc[i].setlfunc))(message, fieldfunc[i].name, value);
+					break;
+				case FUNC_LIST:
+					(*(fieldfunc[i].setlfunc))(message, field, value);
+					break;
+			}
+			break;
+		}		 
+	}
+}
+
+
+/**
+* message_get_header: returns the list of 'any header' values
+* (except of unsupported yet Content- and MIME-Version special headers)
+*
+* You should free the GList list by yourself.
+**/
+GList *
+message_get_header (GMimeMessage *message, const char *field) 
+{
+	gint		i;
+	char *	ret = NULL, *ia_string;
+	GList *	gret = NULL;
+	InternetAddressList *ia_list = NULL, *ia;
+
+	for (i = 0; i <= HEADER_UNKNOWN; ++i) {
+		if (!fieldfunc[i].name || !g_strncasecmp(field, fieldfunc[i].name, strlen(fieldfunc[i].name))) { 
+			switch (fieldfunc[i].functype) {
+				case FUNC_CHARFREEPTR:
+					ret = (char *)(*(fieldfunc[i].func))(message);
+					break;
+				case FUNC_CHARPTR:
+					ret = (char *)(*(fieldfunc[i].func))(message);
+					break;
+				case FUNC_IA:
+					ia_list = (*(fieldfunc[i].rcptfunc))(message, field);
+					gret = g_list_alloc();
+					ia = ia_list;
+#ifndef GMIME24
+					while (ia && ia->address) {
+
+						ia_string = internet_address_to_string ((InternetAddress *)ia->address, FALSE);
+						gret = g_list_append (gret, ia_string);
+						ia = ia->next;
+					}
+#else
+					i = internet_address_list_length (ia);
+					while (i > 0) {
+						ia_string = internet_address_to_string (internet_address_list_get_address (ia, i), FALSE);
+						gret = g_list_append (gret, ia_string);
+						-- i;
+					}
+#endif
+					break;
+				case FUNC_LIST:
+					gret = (*(fieldfunc[i].getlistfunc))(message, field);
+					break;
+			}
+			break;
+		}		 
+	}
+	if (gret == NULL && ret != NULL) {
+		gret = g_list_prepend (gret, g_strdup (ret));
+	}
+	if (fieldfunc[i].functype == FUNC_CHARFREEPTR && ret) {
+		g_free (ret);
+	}
+	return gret;
+}

src/message.h

@@ -44,4 +44,7 @@ int process_learn (struct controller_session *session);
  */
 GByteArray* get_next_text_part (memory_pool_t *pool, GList *parts, GList **cur);
 
+void message_set_header (GMimeMessage *message, const char *field, const char *value);
+GList* message_get_header (GMimeMessage *message, const char *field);
+
 #endif

src/plugins/regexp.c

@@ -154,7 +154,7 @@ process_regexp (struct rspamd_regexp *re, struct worker_task *task)
 {
 	char *headerv, *c, t;
 	struct mime_part *part;
-	GList *cur;
+	GList *cur, *headerlist;
 	struct uri *url;
 
 	if (re == NULL) {
@@ -171,25 +171,25 @@ process_regexp (struct rspamd_regexp *re, struct worker_task *task)
 				return 0;
 			}
 			msg_debug ("process_regexp: checking header regexp: %s = /%s/", re->header, re->regexp_text);
-#ifdef GMIME24
-			headerv = (char *)g_mime_object_get_header (GMIME_OBJECT (task->message), re->header);
-#else
-			headerv = (char *)g_mime_message_get_header (task->message, re->header);
-#endif
-			if (headerv == NULL) {
+			headerlist = message_get_header (task->message, re->header);
+			if (headerlist == NULL) {
 				return 0;
 			}
 			else {
 				if (re->regexp == NULL) {
 					msg_debug ("process_regexp: regexp contains only header and it is found %s", re->header);
+					g_list_free (headerlist);
 					return 1;
 				}
-				if (g_regex_match (re->regexp, headerv, 0, NULL) == TRUE) {
-					return 1;
-				}
-				else {
-					return 0;
+				cur = headerlist;
+				while (cur) {
+					if (g_regex_match (re->regexp, cur->data, 0, NULL) == TRUE) {
+						return 1;
+					}
+					cur = g_list_next (cur);
 				}
+				g_list_free (headerlist);
+				return 0;
 			}
 			break;
 		case REGEXP_MIME: