* Add ability to test regexp with 'T' flag

* Write more code for DNS resolver:
 - initial RR parser
 - name compression
 - replies handler

6 files changed, 317 insertions, 17 deletions

diff --git a/src/cfg_file.h b/src/cfg_file.h
index aad42d88c..fa73643f0 100644
--- a/src/cfg_file.h
+++ b/src/cfg_file.h
@@ -82,6 +82,7 @@ struct rspamd_regexp {
 	GRegex *regexp;									/**< glib regexp structure								*/
 	GRegex *raw_regexp;								/**< glib regexp structure for raw matching				*/
 	gchar *header;									/**< header name for header regexps						*/
+	gboolean is_test;								/**< true if this expression must be tested				*/
 };
 
 /**
diff --git a/src/dns.c b/src/dns.c
index fcdf1619e..c0282f66f 100644
--- a/src/dns.c
+++ b/src/dns.c
@@ -47,6 +47,8 @@
 #define DNS_RANDOM rand
 #endif
 
+#define UDP_PACKET_SIZE 512
+
 /*
  * P E R M U T A T I O N  G E N E R A T O R
  */
@@ -278,6 +280,40 @@ struct dns_request_key {
 	guint16 port;
 };
 
+/** Message compression */
+struct dns_name_table {
+	guint8 off;
+	guint8 *label;
+	guint8 len;
+};
+
+static gboolean
+try_compress_label (memory_pool_t *pool, guint8 *target, guint8 *start, guint8 len, guint8 *label, GList *table)
+{
+	GList *cur;
+	struct dns_name_table *tbl;
+
+	cur = table;
+	while (cur) {
+		tbl = cur->data;
+		if (tbl->len == len) {
+			if (memcmp (label, tbl->label, len) == 0) {
+				*target = tbl->off | 0xC0;
+				return TRUE;
+			}
+		}
+		cur = g_list_next (cur);
+	}
+
+	/* Insert label to list */
+	tbl = memory_pool_alloc (pool, sizeof (struct dns_name_table));
+	tbl->off = target - start;
+	tbl->label = label;
+	tbl->len = len;
+	table = g_list_prepend (table, tbl);
+
+	return FALSE;
+}
 
 /** Packet creating functions */
 static void
@@ -311,6 +347,7 @@ format_dns_name (struct rspamd_dns_request *req, const char *name, guint namelen
 {
 	guint8 *pos = req->packet + req->pos, *begin, *end;
 	guint remain = req->packet_len - req->pos - 5, label_len;
+	GList *table = NULL;
 
 	if (namelen == 0) {
 		namelen = strlen (name);
@@ -320,7 +357,6 @@ format_dns_name (struct rspamd_dns_request *req, const char *name, guint namelen
 	for (;;) {
 		end = strchr (begin, '.');
 		if (end) {
-			label_len = end - begin;
 			if (label_len > DNS_D_MAXLABEL) {
 				msg_err ("dns name component is longer than 63 bytes, should be stripped");
 				label_len = DNS_D_MAXLABEL;
@@ -329,11 +365,19 @@ format_dns_name (struct rspamd_dns_request *req, const char *name, guint namelen
 				label_len = remain - 1;
 				msg_err ("no buffer remain for constructing query, strip to %ud", label_len);
 			}
-			*pos++ = (guint8)label_len;
-			memcpy (pos, begin, label_len);
-			pos += label_len;
-			remain -= label_len + 1;
-			begin = end + 1;
+			/* First try to compress name */
+			if (! try_compress_label (req->pool, pos, req->packet, end - begin, begin, table)) {
+				label_len = end - begin;
+
+				*pos++ = (guint8)label_len;
+				memcpy (pos, begin, label_len);
+				pos += label_len;
+				remain -= label_len + 1;
+				begin = end + 1;
+			}
+			else {
+				pos ++;
+			}
 		}
 		else {
 			end = (guint8 *)name + namelen;
@@ -364,6 +408,9 @@ format_dns_name (struct rspamd_dns_request *req, const char *name, guint namelen
 	/* Termination label */
 	*(++pos) = '\0';
 	req->pos += pos - (req->packet + req->pos) + 1;
+	if (table != NULL) {
+		g_list_free (table);
+	}
 }
 
 static void
@@ -386,6 +433,7 @@ make_ptr_req (struct rspamd_dns_request *req, struct in_addr addr)
 	*p++ = htons (DNS_C_IN);
 	*p = htons (DNS_T_PTR);
 	req->pos += sizeof (guint16) * 2;
+	req->type = DNS_REQUEST_PTR;
 }
 
 static void
@@ -400,6 +448,7 @@ make_a_req (struct rspamd_dns_request *req, const char *name)
 	*p++ = htons (DNS_C_IN);
 	*p = htons (DNS_T_A);
 	req->pos += sizeof (guint16) * 2;
+	req->type = DNS_REQUEST_A;
 }
 
 static void
@@ -414,7 +463,7 @@ make_txt_req (struct rspamd_dns_request *req, const char *name)
 	*p++ = htons (DNS_C_IN);
 	*p = htons (DNS_T_A);
 	req->pos += sizeof (guint16) * 2;
-	
+	req->type = DNS_REQUEST_TXT;
 }
 
 static void
@@ -429,7 +478,7 @@ make_mx_req (struct rspamd_dns_request *req, const char *name)
 	*p++ = htons (DNS_C_IN);
 	*p = htons (DNS_T_A);
 	req->pos += sizeof (guint16) * 2;
-	
+	req->type = DNS_REQUEST_MX;
 }
 
 static int
@@ -469,6 +518,193 @@ dns_fin_cb (gpointer arg)
 	/* XXX: call callback if possible */
 }
 
+static guint8 *
+decompress_label (guint8 *begin, guint8 *len)
+{
+	guint8 offset;
+	offset = (*len) ^ 0xC0;
+
+	*len = *(begin + offset);
+	return begin + offset + 1;
+}
+
+static guint8 *
+dns_request_reply_cmp (struct rspamd_dns_request *req, guint8 *in, int len)
+{
+	guint8 *p, *c, *l1, *l2;
+	guint8 len1, len2;
+
+	/* QR format:
+	 * labels - len:octets
+	 * null label - 0
+	 * class - 2 octets
+	 * type - 2 octets
+	 */
+	
+	/* In p we would store current position in reply and in c - position in request */
+	p = in;
+	c = req->packet + sizeof (struct dns_header);
+
+	for (;;) {
+		/* Get current label */
+		len1 = *p;
+		len2 = *c;
+		if (p - in > len) {
+			msg_info ("invalid dns reply");
+			return NULL;
+		}
+		/* This may be compressed, so we need to decompress it */
+		if (len1 & 0xC0) {
+			l1 = decompress_label (in, &len1);
+			p ++;
+		}
+		else {
+			l1 = ++p;
+			p += len1;
+		}
+		if (len2 & 0xC0) {
+			l2 = decompress_label (req->packet, &len2);
+			c ++;
+		}
+		else {
+			l2 = ++c;
+			c += len2;
+		}
+		if (len1 != len2) {
+			return NULL;
+		}
+		if (len1 == 0) {
+			break;
+		}
+
+		if (memcmp (l1, l2, len1) != 0) {
+			return NULL;
+		}
+	}
+
+	/* p now points to the end of QR section */
+	/* Compare class and type */
+	if (memcmp (p, c, sizeof (guint16) * 2) == 0) {
+		return p + sizeof (guint16) * 2;
+	}
+	return NULL;
+}
+
+static gboolean
+dns_parse_rr (union rspamd_reply_element *elt, guint8 **pos, struct rspamd_dns_reply *rep, int *remain)
+{
+	guint8 *p = *pos;
+	guint16 type, datalen;
+
+	/* Skip the whole name */
+	while (p - *pos < *remain) {
+		if (*p & 0xC0) {
+			p ++;
+		}
+		else if (*p == 0) {
+			p ++;
+			break;
+		}
+		else {
+			p += *p + 1;
+		}
+	}
+	if (p - *pos >= *remain - sizeof (guint16) * 5) {
+		msg_info ("stripped dns reply");
+		return FALSE;
+	}
+	type = *((guint16 *)p);
+	/* Skip ttl and class */
+	p += sizeof (guint16) * 2 + sizeof (guint32);
+	datalen = *((guint16 *)p);
+	p += sizeof (guint16);
+	*remain -= p - *pos;
+	/* Now p points to RR data */
+	switch (type) {
+	case DNS_T_A:
+		if ((datalen & 0x3) && *remain >= datalen) {
+			elt->a.addr[0].s_addr = *((guint32 *)p);
+			p += sizeof (guint32);
+		}
+		else {
+			msg_info ("corrupted A record");
+			return FALSE;
+		}
+		break;
+	}
+	*remain -= datalen;
+	*pos = p;
+}
+
+static struct rspamd_dns_reply *
+dns_parse_reply (guint8 *in, int r, struct rspamd_dns_resolver *resolver)
+{
+	struct dns_header *header = (struct dns_header *)in;
+	struct rspamd_dns_request *req;
+	struct rspamd_dns_reply *rep;
+	union rspamd_reply_element *elt;
+	guint8 *pos;
+	int i;
+	
+	/* First check header fields */
+	if (header->qr == 0) {
+		msg_info ("got request while waiting for reply");
+		return NULL;
+	}
+
+	/* Now try to find corresponding request */
+	if ((req = g_hash_table_lookup (resolver->requests, GUINT_TO_POINTER (header->qid))) == NULL) {
+		/* No such requests found */
+		return NULL;
+	}
+	/* 
+	 * Now we have request and query data is now at the end of header, so compare
+	 * request QR section and reply QR section
+	 */
+	if ((pos = dns_request_reply_cmp (req, in + sizeof (struct dns_header), r - sizeof (struct dns_header))) == NULL) {
+		return NULL;
+	}
+	/*
+	 * Now pos is in answer section, so we should extract data and form reply
+	 */
+	rep = memory_pool_alloc (req->pool, sizeof (struct rspamd_dns_reply));
+	rep->request = req;
+	rep->type = req->type;
+	rep->elements = NULL;
+
+	r -= pos - in;
+	/* Extract RR records */
+	for (i = 0; i < header->ancount; i ++) {
+		elt = memory_pool_alloc (req->pool, sizeof (union rspamd_reply_element));
+		if (! dns_parse_rr (elt, &pos, rep, &r)) {
+			msg_info ("incomplete reply");
+			break;
+		}
+	}
+	
+	return rep;
+}
+
+static void
+dns_read_cb (int fd, short what, void *arg)
+{
+	struct rspamd_dns_resolver *resolver = arg;
+	int i, r;
+	struct rspamd_dns_server *serv;
+	struct rspamd_dns_reply *rep;
+	guint8 in[UDP_PACKET_SIZE];
+
+	/* This function is called each time when we have data on one of server's sockets */
+	
+	/* First read packet from socket */
+	r = read (fd, in, sizeof (in));
+	if (r > 96) {
+		if ((rep = dns_parse_reply (in, r, resolver)) != NULL) {
+		
+		}
+	}
+}
+
 static void
 dns_timer_cb (int fd, short what, void *arg)
 {
@@ -676,7 +912,7 @@ dns_resolver_init (struct config_file *cfg)
 	GList *cur;
 	struct rspamd_dns_resolver *new;
 	char *begin, *p;
-	int priority;
+	int priority, i;
 	struct rspamd_dns_server *serv;
 	
 	new = memory_pool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_dns_resolver));
@@ -730,6 +966,17 @@ dns_resolver_init (struct config_file *cfg)
 		}
 
 	}
+	/* Now init all servers */
+	for (i = 0; i < new->servers_num; i ++) {
+		serv = &new->servers[i];
+		serv->sock = make_udp_socket (&serv->addr, htons (53), FALSE, TRUE);
+		if (serv->sock == -1) {
+			msg_warn ("cannot create socket to server %s", serv->name);
+		}
+		else {
+			event_set (&serv->ev, serv->sock, EV_READ, dns_read_cb, new);
+		}
+	}
 
 	return new;
 }
diff --git a/src/dns.h b/src/dns.h
index 8f1f2c6cc..6573377cb 100644
--- a/src/dns.h
+++ b/src/dns.h
@@ -11,6 +11,8 @@
 #define DNS_D_MAXLABEL	63	/* + 1 '\0' */
 #define DNS_D_MAXNAME	255	/* + 1 '\0' */
 
+#define MAX_ADDRS 64
+
 struct rspamd_dns_reply;
 struct config_file;
 
@@ -23,6 +25,7 @@ struct rspamd_dns_server {
 	struct in_addr addr;				/**< address of DNS server					*/
 	char *name;							/**< name of DNS server						*/
 	int sock;							/**< persistent socket						*/
+	struct event ev;
 };
 
 #define DNS_K_TEA_KEY_SIZE	16
@@ -52,6 +55,13 @@ struct rspamd_dns_resolver {
 struct dns_header;
 struct dns_query;
 
+enum rspamd_request_type {
+	DNS_REQUEST_A = 0,
+	DNS_REQUEST_PTR,
+	DNS_REQUEST_MX,
+	DNS_REQUEST_TXT
+};
+
 struct rspamd_dns_request {
 	memory_pool_t *pool;				/**< pool associated with request			*/
 	struct rspamd_dns_resolver *resolver;
@@ -69,18 +79,15 @@ struct rspamd_dns_request {
 	off_t pos;
 	guint packet_len;
 	int sock;
+	enum rspamd_request_type type;
 };
 
-enum rspamd_request_type {
-	DNS_REQUEST_A = 0,
-	DNS_REQUEST_PTR,
-	DNS_REQUEST_MX,
-	DNS_REQUEST_TXT
-};
+
 
 union rspamd_reply_element {
 	struct {
-		struct in_addr addr;
+		struct in_addr addr[MAX_ADDRS];
+		guint16 addrcount;
 	} a;
 	struct {
 		char *name;
diff --git a/src/expressions.c b/src/expressions.c
index f19e67670..c4465742c 100644
--- a/src/expressions.c
+++ b/src/expressions.c
@@ -244,6 +244,7 @@ is_regexp_flag (char a)
 	case 'P':
 	case 'U':
 	case 'X':
+	case 'T':
 		return TRUE;
 	default:
 		return FALSE;
@@ -671,6 +672,10 @@ parse_regexp (memory_pool_t * pool, char *line, gboolean raw_mode)
 			}
 			p++;
 			break;
+		case 'T':
+			result->is_test = TRUE;
+			p ++;
+			break;
 			/* Stop flags parsing */
 		default:
 			p = NULL;
diff --git a/src/expressions.h b/src/expressions.h
index 14ae03f4f..6ffc7359d 100644
--- a/src/expressions.h
+++ b/src/expressions.h
@@ -9,6 +9,7 @@
 #include "config.h"
 
 struct worker_task;
+struct rspamd_regexp;
 
 /**
  * Rspamd expression function
diff --git a/src/plugins/regexp.c b/src/plugins/regexp.c
index c660f16c3..58e91d14b 100644
--- a/src/plugins/regexp.c
+++ b/src/plugins/regexp.c
@@ -562,7 +562,13 @@ tree_url_callback (gpointer key, gpointer value, void *data)
 	struct url_regexp_param        *param = data;
 	struct uri                     *url = value;
 
+	if (G_UNLIKELY (param->re->is_test)) {
+		msg_info ("process test regexp /%s/ for url %s", struri (url));
+	}
 	if (g_regex_match (param->regexp, struri (url), 0, NULL) == TRUE) {
+		if (G_UNLIKELY (param->re->is_test)) {
+			msg_info ("process test regexp /%s/ for url %s returned TRUE", struri (url));
+		}
 		task_cache_add (param->task, param->re, 1);
 		param->found = TRUE;
 		return TRUE;
@@ -599,7 +605,13 @@ process_regexp (struct rspamd_regexp *re, struct worker_task *task, const char *
 	
 	if (additional != NULL) {
 		/* We have additional parameter defined, so ignore type of regexp expression and use it for parsing */
+		if (G_UNLIKELY (re->is_test)) {
+			msg_info ("process test regexp /%s/ with test %s", re->regexp_text, additional);
+		}
 		if (g_regex_match_full (re->regexp, additional, strlen (additional), 0, 0, NULL, NULL) == TRUE) {
+			if (G_UNLIKELY (re->is_test)) {
+				msg_info ("result of regexp /%s/ is true", re->regexp_text);
+			}
 			task_cache_add (task, re, 1);
 			return 1;
 		}
@@ -620,6 +632,9 @@ process_regexp (struct rspamd_regexp *re, struct worker_task *task, const char *
 			return 0;
 		}
 		debug_task ("checking header regexp: %s = /%s/", re->header, re->regexp_text);
+		if (G_UNLIKELY (re->is_test)) {
+			msg_info ("process test regexp /%s/ for header %s", re->regexp_text, re->header);
+		}
 		headerlist = message_get_header (task->task_pool, task->message, re->header);
 		if (headerlist == NULL) {
 			task_cache_add (task, re, 0);
@@ -634,8 +649,14 @@ process_regexp (struct rspamd_regexp *re, struct worker_task *task, const char *
 			}
 			cur = headerlist;
 			while (cur) {
-				debug_task ("found header \"%s\" with value \"%s\"", re->header, (char *)cur->data);
+				debug_task ("found header \"%s\" with value \"%s\"", re->header, (const char *)cur->data);
+				if (G_UNLIKELY (re->is_test)) {
+					msg_info ("process test regexp /%s/ for header %s with value '%s'", re->regexp_text, re->header, (const char *)cur->data);
+				}
 				if (cur->data && g_regex_match (re->regexp, cur->data, 0, NULL) == TRUE) {
+					if (G_UNLIKELY (re->is_test)) {
+						msg_info ("process test regexp /%s/ for header %s with value '%s' returned TRUE", re->regexp_text, re->header, (const char *)cur->data);
+					}
 					task_cache_add (task, re, 1);
 					return 1;
 				}
@@ -661,7 +682,13 @@ process_regexp (struct rspamd_regexp *re, struct worker_task *task, const char *
 			else {
 				regexp = re->regexp;
 			}
+			if (G_UNLIKELY (re->is_test)) {
+				msg_info ("process test regexp /%s/ for mime part of length %d", re->regexp_text, (int)part->orig->len);
+			}
 			if (g_regex_match_full (regexp, part->orig->data, part->orig->len, 0, 0, NULL, NULL) == TRUE) {
+				if (G_UNLIKELY (re->is_test)) {
+					msg_info ("process test regexp /%s/ for mime part returned true", re->regexp_text);
+				}
 				task_cache_add (task, re, 1);
 				return 1;
 			}
@@ -671,7 +698,13 @@ process_regexp (struct rspamd_regexp *re, struct worker_task *task, const char *
 		return 0;
 	case REGEXP_MESSAGE:
 		debug_task ("checking message regexp: /%s/", re->regexp_text);
+		if (G_UNLIKELY (re->is_test)) {
+			msg_info ("process test regexp /%s/ for message of length %d", re->regexp_text, (int)task->msg->len);
+		}
 		if (g_regex_match_full (re->raw_regexp, task->msg->begin, task->msg->len, 0, 0, NULL, NULL) == TRUE) {
+			if (G_UNLIKELY (re->is_test)) {
+				msg_info ("process test regexp /%s/ for message of length %d returned TRUE", re->regexp_text, (int)task->msg->len);
+			}
 			task_cache_add (task, re, 1);
 			return 1;
 		}
@@ -745,7 +778,13 @@ process_regexp (struct rspamd_regexp *re, struct worker_task *task, const char *
 		t = *c;
 		*c = '\0';
 		debug_task ("found raw header \"%s\" with value \"%s\"", re->header, headerv);
+		if (G_UNLIKELY (re->is_test)) {
+			msg_info ("process test regexp /%s/ for header %s with value '%s'", re->regexp_text, re->header, headerv);
+		}
 		if (g_regex_match (re->raw_regexp, headerv, 0, NULL) == TRUE) {
+			if (G_UNLIKELY (re->is_test)) {
+				msg_info ("process test regexp /%s/ for header %s with value '%s' returned TRUE", re->regexp_text, re->header, headerv);
+			}
 			*c = t;
 			task_cache_add (task, re, 1);
 			return 1;