* Rework build process:

 - add librspamdserver
 - link this library to all daemons and utils of rspamd
 - use subdirectories more often
* Rework global variables logic - move them to the main process
* Fix logging to handle utf-8 correctly
* Add statshow utility and make it working
* Move printf functions to separate source file

1 files changed, 181 insertions, 540 deletions

diff --git a/src/util.c b/src/util.c
index a420493dc..89bc13336 100644
--- a/src/util.c
+++ b/src/util.c
@@ -28,6 +28,8 @@
 #include "cfg_file.h"
 #include "main.h"
 #include "statfile.h"
+#include "filter.h"
+#include "message.h"
 
 /* Check log messages intensity once per minute */
 #define CHECK_TIME 60
@@ -36,9 +38,7 @@
 /* Default connect timeout for sync sockets */
 #define CONNECT_TIMEOUT 3
 
-#ifdef RSPAMD_MAIN
-extern rspamd_hash_t           *counters;
-#endif
+rspamd_hash_t                      *counters = NULL;
 
 static gchar* rspamd_sprintf_num (gchar *buf, gchar *last, guint64 ui64, gchar zero, guint hexadecimal, guint width);
 
@@ -822,7 +822,6 @@ calculate_check_time (struct timeval *begin, gint resolution)
 double
 set_counter (const gchar *name, guint32 value)
 {
-#ifdef RSPAMD_MAIN
 	struct counter_data            *cd;
 	double                          alpha;
 	gchar                           *key;
@@ -847,9 +846,6 @@ set_counter (const gchar *name, guint32 value)
 	}
 
 	return cd->value;
-#else
-	return 0;
-#endif
 }
 
 #ifndef g_tolower
@@ -912,7 +908,7 @@ fstr_strcase_hash (gconstpointer key)
 void
 gperf_profiler_init (struct config_file *cfg, const gchar *descr)
 {
-#if defined(WITH_GPERF_TOOLS) && defined(RSPAMD_MAIN)
+#if defined(WITH_GPERF_TOOLS)
 	gchar                           prof_path[PATH_MAX];
 
 	if (getenv ("CPUPROFILE")) {
@@ -1031,7 +1027,6 @@ unlock_file (gint fd, gboolean async)
 }
 #endif /* HAVE_FLOCK */
 
-#ifdef RSPAMD_MAIN
 stat_file_t *
 get_statfile_by_symbol (statfile_pool_t *pool, struct classifier_config *ccf, 
         const gchar *symbol, struct statfile **st, gboolean try_create)
@@ -1076,539 +1071,8 @@ get_statfile_by_symbol (statfile_pool_t *pool, struct classifier_config *ccf,
     
     return res;
 }
-#endif /* RSPAMD_MAIN */
-
-
-gint
-rspamd_fprintf (FILE *f, const gchar *fmt, ...)
-{
-	gchar   *p;
-	va_list   args;
-    gchar buf[BUFSIZ];
-    gint r;
-
-	va_start (args, fmt);
-	p = rspamd_vsnprintf (buf, sizeof (buf), fmt, args);
-	va_end (args);
-
-    r = fprintf (f, "%s", buf);
-
-    return r;
-}
-
-gint
-rspamd_sprintf (gchar *buf, const gchar *fmt, ...)
-{
-	gchar   *p;
-	va_list   args;
-
-	va_start (args, fmt);
-	p = rspamd_vsnprintf (buf, /* STUB */ 65536, fmt, args);
-	va_end (args);
-
-	return p - buf;
-}
-
-
-gint
-rspamd_snprintf (gchar *buf, size_t max, const gchar *fmt, ...)
-{
-	gchar   *p;
-	va_list   args;
-
-	va_start (args, fmt);
-	p = rspamd_vsnprintf (buf, max - 1, fmt, args);
-	va_end (args);
-	*p = '\0';
-
-	return p - buf;
-}
-
-gchar *
-rspamd_escape_string (gchar *dst, const gchar *src, gsize len)
-{
-	gchar              *buf = dst, *last = dst + len;
-	guint8              c;
-	const gchar        *p = src;
-
-	while (*p && buf < last) {
-		c = *p++;
-		if (G_UNLIKELY ((c & 0x80))) {
-			c &= 0x7F;
-			if (last - buf >= 3) {
-				*buf++ = 'M';
-				*buf++ = '-';
-			}
-		}
-
-		if (G_UNLIKELY ( g_ascii_iscntrl (c))) {
-			if (c == '\n') {
-				*buf++ = ' ';
-			}
-			else if (c == '\t') {
-				*buf++ = '\t';
-			}
-			else {
-				*buf++ = '^';
-				if (buf != last) {
-					*buf++ = c ^ 0100;
-				}
-			}
-		}
-		else {
-			*buf++ = c;
-		}
-	}
-
-	*buf = '\0';
-
-	return buf;
-}
-
-gchar *
-rspamd_vsnprintf (gchar *buf, size_t max, const gchar *fmt, va_list args)
-{
-	gchar              *p, zero, *last;
-	gint                d;
-	long double         f, scale;
-	size_t              len, slen;
-	gint64              i64;
-	guint64             ui64;
-	guint               width, sign, hex, max_width, frac_width, i;
-	f_str_t			   *v;
-
-	if (max == 0) {
-		return buf;
-	}
-
-	last = buf + max;
-
-	while (*fmt && buf < last) {
-
-		/*
-		 * "buf < last" means that we could copy at least one character:
-		 * the plain character, "%%", "%c", and minus without the checking
-		 */
-
-		if (*fmt == '%') {
-
-			i64 = 0;
-			ui64 = 0;
-
-			zero = (gchar) ((*++fmt == '0') ? '0' : ' ');
-			width = 0;
-			sign = 1;
-			hex = 0;
-			max_width = 0;
-			frac_width = 0;
-			slen = (size_t) -1;
-
-			while (*fmt >= '0' && *fmt <= '9') {
-				width = width * 10 + *fmt++ - '0';
-			}
-
-
-			for ( ;; ) {
-				switch (*fmt) {
-
-				case 'u':
-					sign = 0;
-					fmt++;
-					continue;
-
-				case 'm':
-					max_width = 1;
-					fmt++;
-					continue;
-
-				case 'X':
-					hex = 2;
-					sign = 0;
-					fmt++;
-					continue;
-
-				case 'x':
-					hex = 1;
-					sign = 0;
-					fmt++;
-					continue;
-				case '.':
-					fmt++;
-
-					while (*fmt >= '0' && *fmt <= '9') {
-						frac_width = frac_width * 10 + *fmt++ - '0';
-					}
-
-					break;
-
-				case '*':
-					d = (gint)va_arg (args, gint);
-					if (G_UNLIKELY (d < 0)) {
-						msg_err ("crititcal error: size is less than 0");
-						g_assert (0);
-					}
-					slen = (size_t)d;
-					fmt++;
-					continue;
-
-				default:
-					break;
-				}
-
-				break;
-			}
-
-
-			switch (*fmt) {
-
-			case 'V':
-				v = va_arg (args, f_str_t *);
-
-				len = v->len;
-				len = (buf + len < last) ? len : (size_t) (last - buf);
-
-				buf = ((gchar *)memcpy (buf, v->begin, len)) + len;
-				fmt++;
-
-				continue;
-
-			case 's':
-				p = va_arg(args, gchar *);
-				if (p == NULL) {
-					p = "(NULL)";
-				}
-
-				if (slen == (size_t) -1) {
-					while (*p && buf < last) {
-						*buf++ = *p++;
-					}
-
-				} else {
-					len = (buf + slen < last) ? slen : (size_t) (last - buf);
-
-					buf = ((gchar *)memcpy (buf, p, len)) + len;
-				}
-
-				fmt++;
-
-				continue;
-
-			case 'S':
-				p = va_arg(args, gchar *);
-				if (p == NULL) {
-					p = "(NULL)";
-				}
-
-				if (slen == (size_t) -1) {
-					buf = rspamd_escape_string (buf, p, last - buf);
-
-				} else {
-					len = (buf + slen < last) ? slen : (size_t) (last - buf);
-
-					buf = rspamd_escape_string (buf, p, len);
-				}
-
-				fmt++;
-
-				continue;
-
-			case 'O':
-				i64 = (gint64) va_arg (args, off_t);
-				sign = 1;
-				break;
-
-			case 'P':
-				i64 = (gint64) va_arg (args, pid_t);
-				sign = 1;
-				break;
-
-			case 'T':
-				i64 = (gint64) va_arg (args, time_t);
-				sign = 1;
-				break;
-
-			case 'z':
-				if (sign) {
-					i64 = (gint64) va_arg (args, ssize_t);
-				} else {
-					ui64 = (guint64) va_arg (args, size_t);
-				}
-				break;
-
-			case 'd':
-				if (sign) {
-					i64 = (gint64) va_arg (args, gint);
-				} else {
-					ui64 = (guint64) va_arg (args, guint);
-				}
-				break;
-
-			case 'l':
-				if (sign) {
-					i64 = (gint64) va_arg(args, long);
-				} else {
-					ui64 = (guint64) va_arg(args, guint32);
-				}
-				break;
-
-			case 'D':
-				if (sign) {
-					i64 = (gint64) va_arg(args, gint32);
-				} else {
-					ui64 = (guint64) va_arg(args, guint32);
-				}
-				break;
-
-			case 'L':
-				if (sign) {
-					i64 = va_arg (args, gint64);
-				} else {
-					ui64 = va_arg (args, guint64);
-				}
-				break;
-
-
-			case 'f':
-				f = (double) va_arg (args, double);
-				if (f < 0) {
-					*buf++ = '-';
-					f = -f;
-				}
-				
-				ui64 = (gint64) f;
-
-				buf = rspamd_sprintf_num (buf, last, ui64, zero, 0, width);
-
-				if (frac_width) {
-
-					if (buf < last) {
-						*buf++ = '.';
-					}
-
-					scale = 1.0;
-
-					for (i = 0; i < frac_width; i++) {
-						scale *= 10.0;
-					}
-
-					/*
-					* (gint64) cast is required for msvc6:
-					* it can not convert guint64 to double
-					*/
-					ui64 = (guint64) ((f - (gint64) ui64) * scale);
-
-					buf = rspamd_sprintf_num (buf, last, ui64, '0', 0, frac_width);
-				}
-
-				fmt++;
-
-				continue;
-
-			case 'F':
-				f = (long double) va_arg (args, long double);
-
-				if (f < 0) {
-					*buf++ = '-';
-					f = -f;
-				}
-
-				ui64 = (gint64) f;
-
-				buf = rspamd_sprintf_num (buf, last, ui64, zero, 0, width);
-
-				if (frac_width) {
-
-					if (buf < last) {
-						*buf++ = '.';
-					}
-
-					scale = 1.0;
-
-					for (i = 0; i < frac_width; i++) {
-						scale *= 10.0;
-					}
-
-					/*
-					* (gint64) cast is required for msvc6:
-					* it can not convert guint64 to double
-					*/
-					ui64 = (guint64) ((f - (gint64) ui64) * scale);
-
-					buf = rspamd_sprintf_num (buf, last, ui64, '0', 0, frac_width);
-				}
 
-				fmt++;
 
-				continue;
-
-			case 'g':
-				f = (long double) va_arg (args, double);
-
-				if (f < 0) {
-					*buf++ = '-';
-					f = -f;
-				}
-				g_ascii_formatd (buf, last - buf, "%g", (double)f);
-				buf += strlen (buf);
-				fmt++;
-
-				continue;
-
-			case 'G':
-				f = (long double) va_arg (args, long double);
-
-				if (f < 0) {
-					*buf++ = '-';
-					f = -f;
-				}
-				g_ascii_formatd (buf, last - buf, "%g", (double)f);
-				buf += strlen (buf);
-				fmt++;
-
-				continue;
-
-			case 'p':
-				ui64 = (uintptr_t) va_arg (args, void *);
-				hex = 2;
-				sign = 0;
-				zero = '0';
-				width = sizeof (void *) * 2;
-				break;
-
-			case 'c':
-				d = va_arg (args, gint);
-				*buf++ = (gchar) (d & 0xff);
-				fmt++;
-
-				continue;
-
-			case 'Z':
-				*buf++ = '\0';
-				fmt++;
-
-				continue;
-
-			case 'N':
-				*buf++ = LF;
-				fmt++;
-
-				continue;
-
-			case '%':
-				*buf++ = '%';
-				fmt++;
-
-				continue;
-
-			default:
-				*buf++ = *fmt++;
-
-				continue;
-			}
-
-			if (sign) {
-				if (i64 < 0) {
-					*buf++ = '-';
-					ui64 = (guint64) -i64;
-
-				} else {
-					ui64 = (guint64) i64;
-				}
-			}
-
-			buf = rspamd_sprintf_num (buf, last, ui64, zero, hex, width);
-
-			fmt++;
-
-		} else {
-			*buf++ = *fmt++;
-		}
-	}
-
-	return buf;
-}
-
-
-static gchar *
-rspamd_sprintf_num (gchar *buf, gchar *last, guint64 ui64, gchar zero,
-	guint                           hexadecimal, guint width)
-{
-	gchar		   *p, temp[sizeof ("18446744073709551615")];
-	size_t		    len;
-	guint32                         ui32;
-	static gchar   hex[] = "0123456789abcdef";
-	static gchar   HEX[] = "0123456789ABCDEF";
-
-	p = temp + sizeof(temp);
-
-	if (hexadecimal == 0) {
-
-		if (ui64 <= G_MAXUINT32) {
-
-			/*
-			 * To divide 64-bit numbers and to find remainders
-			 * on the x86 platform gcc and icc call the libc functions
-			 * [u]divdi3() and [u]moddi3(), they call another function
-			 * in its turn.  On FreeBSD it is the qdivrem() function,
-			 * its source code is about 170 lines of the code.
-			 * The glibc counterpart is about 150 lines of the code.
-			 *
-			 * For 32-bit numbers and some divisors gcc and icc use
-			 * a inlined multiplication and shifts.  For example,
-			 * guint "i32 / 10" is compiled to
-			 *
-			 *	 (i32 * 0xCCCCCCCD) >> 35
-			 */
-
-			ui32 = (guint32) ui64;
-
-			do {
-				*--p = (gchar) (ui32 % 10 + '0');
-			} while (ui32 /= 10);
-
-		} else {
-			do {
-				*--p = (gchar) (ui64 % 10 + '0');
-			} while (ui64 /= 10);
-		}
-
-	} else if (hexadecimal == 1) {
-
-		do {
-
-			/* the "(guint32)" cast disables the BCC's warning */
-			*--p = hex[(guint32) (ui64 & 0xf)];
-
-		} while (ui64 >>= 4);
-
-	} else { /* hexadecimal == 2 */
-
-		do {
-
-			/* the "(guint32)" cast disables the BCC's warning */
-			*--p = HEX[(guint32) (ui64 & 0xf)];
-
-		} while (ui64 >>= 4);
-	}
-
-	/* zero or space padding */
-
-	len = (temp + sizeof (temp)) - p;
-
-	while (len++ < width && buf < last) {
-		*buf++ = zero;
-	}
-
-	/* number safe copy */
-
-	len = (temp + sizeof (temp)) - p;
-
-	if (buf + len > last) {
-		len = last - buf;
-	}
-
-	return ((gchar *)memcpy (buf, p, len)) + len;
-}
 
 #if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION < 22))
 void
@@ -1696,5 +1160,182 @@ str_to_process (const gchar *str)
 }
 
 /*
+ * Destructor for recipients list
+ */
+static void
+rcpt_destruct (void *pointer)
+{
+	struct worker_task             *task = (struct worker_task *) pointer;
+
+	if (task->rcpt) {
+		g_list_free (task->rcpt);
+	}
+}
+
+
+/* Compare two emails for building emails tree */
+static gint
+compare_email_func (gconstpointer a, gconstpointer b)
+{
+	const struct uri               *u1 = a, *u2 = b;
+	gint                            r;
+
+	if (u1->hostlen != u2->hostlen || u1->hostlen == 0) {
+		return u1->hostlen - u2->hostlen;
+	}
+	else {
+		if ((r = g_ascii_strncasecmp (u1->host, u2->host, u1->hostlen)) == 0){
+			if (u1->userlen != u2->userlen || u1->userlen == 0) {
+				return u1->userlen - u2->userlen;
+			}
+			else {
+				return g_ascii_strncasecmp (u1->user, u2->user, u1->userlen);
+			}
+		}
+		else {
+			return r;
+		}
+	}
+
+	return 0;
+}
+
+static gint
+compare_url_func (gconstpointer a, gconstpointer b)
+{
+	const struct uri               *u1 = a, *u2 = b;
+	int                             r;
+
+	if (u1->hostlen != u2->hostlen || u1->hostlen == 0) {
+		return u1->hostlen - u2->hostlen;
+	}
+	else {
+		r = g_ascii_strncasecmp (u1->host, u2->host, u1->hostlen);
+	}
+
+	return r;
+}
+
+/*
+ * Create new task
+ */
+struct worker_task             *
+construct_task (struct rspamd_worker *worker)
+{
+	struct worker_task             *new_task;
+
+	new_task = g_slice_alloc0 (sizeof (struct worker_task));
+
+	new_task->worker = worker;
+	new_task->state = READ_COMMAND;
+	new_task->cfg = worker->srv->cfg;
+	new_task->from_addr.s_addr = INADDR_NONE;
+	new_task->view_checked = FALSE;
+#ifdef HAVE_CLOCK_GETTIME
+# ifdef HAVE_CLOCK_PROCESS_CPUTIME_ID
+	clock_gettime (CLOCK_PROCESS_CPUTIME_ID, &new_task->ts);
+# elif defined(HAVE_CLOCK_VIRTUAL)
+	clock_gettime (CLOCK_VIRTUAL, &new_task->ts);
+# else
+	clock_gettime (CLOCK_REALTIME, &new_task->ts);
+# endif
+#endif
+	if (gettimeofday (&new_task->tv, NULL) == -1) {
+		msg_warn ("gettimeofday failed: %s", strerror (errno));
+	}
+
+	new_task->task_pool = memory_pool_new (memory_pool_get_size ());
+
+	/* Add destructor for recipients list (it would be better to use anonymous function here */
+	memory_pool_add_destructor (new_task->task_pool,
+			(pool_destruct_func) rcpt_destruct, new_task);
+	new_task->results = g_hash_table_new (g_str_hash, g_str_equal);
+	memory_pool_add_destructor (new_task->task_pool,
+			(pool_destruct_func) g_hash_table_destroy,
+			new_task->results);
+	new_task->re_cache = g_hash_table_new (g_str_hash, g_str_equal);
+	memory_pool_add_destructor (new_task->task_pool,
+			(pool_destruct_func) g_hash_table_destroy,
+			new_task->re_cache);
+	new_task->emails = g_tree_new (compare_email_func);
+	memory_pool_add_destructor (new_task->task_pool,
+				(pool_destruct_func) g_tree_destroy,
+				new_task->emails);
+	new_task->urls = g_tree_new (compare_url_func);
+	memory_pool_add_destructor (new_task->task_pool,
+					(pool_destruct_func) g_tree_destroy,
+					new_task->urls);
+	new_task->s =
+			new_async_session (new_task->task_pool, free_task_hard, new_task);
+	new_task->sock = -1;
+	new_task->is_mime = TRUE;
+
+	return new_task;
+}
+
+
+/*
+ * Free all structures of worker_task
+ */
+void
+free_task (struct worker_task *task, gboolean is_soft)
+{
+	GList                          *part;
+	struct mime_part               *p;
+
+	if (task) {
+		debug_task ("free pointer %p", task);
+		while ((part = g_list_first (task->parts))) {
+			task->parts = g_list_remove_link (task->parts, part);
+			p = (struct mime_part *) part->data;
+			g_byte_array_free (p->content, TRUE);
+			g_list_free_1 (part);
+		}
+		if (task->text_parts) {
+			g_list_free (task->text_parts);
+		}
+		if (task->images) {
+			g_list_free (task->images);
+		}
+		if (task->messages) {
+			g_list_free (task->messages);
+		}
+		if (task->received) {
+			g_list_free (task->received);
+		}
+		memory_pool_delete (task->task_pool);
+		if (task->dispatcher) {
+			if (is_soft) {
+				/* Plan dispatcher shutdown */
+				task->dispatcher->wanna_die = 1;
+			}
+			else {
+				rspamd_remove_dispatcher (task->dispatcher);
+			}
+		}
+		if (task->sock != -1) {
+			close (task->sock);
+		}
+		g_slice_free1 (sizeof (struct worker_task), task);
+	}
+}
+
+void
+free_task_hard (gpointer ud)
+{
+  struct worker_task             *task = ud;
+
+  free_task (task, FALSE);
+}
+
+void
+free_task_soft (gpointer ud)
+{
+  struct worker_task             *task = ud;
+
+  free_task (task, FALSE);
+}
+
+/*
  * vi:ts=4
  */