* Rewrite buffered input for line policy (again)

* Fix issue with links that are ip addresses in numeric form in surbl
* On Darwin use BSD style sendfile definition
* Reorganize platform specific knobs in CMakeLists
* Use gettimeofday on systems that have not clock_getres
* Use ftime for dns trans id generation on systems without clock_getres

12 files changed, 146 insertions, 51 deletions

diff --git a/src/buffer.c b/src/buffer.c
index 4351344fc..6be6ce7d0 100644
--- a/src/buffer.c
+++ b/src/buffer.c
@@ -43,7 +43,7 @@ sendfile_callback (rspamd_io_dispatcher_t *d)
 	GError                         *err;
 
 #ifdef HAVE_SENDFILE
-	#if defined(FREEBSD)
+	#if defined(FREEBSD) || defined(DARWIN)
 	off_t                           off = 0;
 	/* FreeBSD version */
 	if (sendfile (d->sendfile_fd, d->fd, d->offset, 0, 0, &off, 0) != 0) {
@@ -248,8 +248,8 @@ read_buffers (int fd, rspamd_io_dispatcher_t * d, gboolean skip_read)
 	GError                         *err;
 	f_str_t                         res;
 	char                           *c, *b;
-	char                          **pos;
-	size_t                         *len;
+	char                           *end;
+	size_t                          len;
 	enum io_policy                  saved_policy;
 
 	if (d->wanna_die) {
@@ -268,8 +268,8 @@ read_buffers (int fd, rspamd_io_dispatcher_t * d, gboolean skip_read)
 		d->in_buf->pos = d->in_buf->data->begin;
 	}
 
-	pos = &d->in_buf->pos;
-	len = &d->in_buf->data->len;
+	end = d->in_buf->pos;
+	len = d->in_buf->data->len;
 
 	if (BUFREMAIN (d->in_buf) == 0) {
 		/* Buffer is full, try to call callback with overflow error */
@@ -281,7 +281,7 @@ read_buffers (int fd, rspamd_io_dispatcher_t * d, gboolean skip_read)
 	}
 	else if (!skip_read) {
 		/* Try to read the whole buffer */
-		r = read (fd, *pos, BUFREMAIN (d->in_buf));
+		r = read (fd, end, BUFREMAIN (d->in_buf));
 		if (r == -1 && errno != EAGAIN) {
 			if (d->err_callback) {
 				err = g_error_new (G_DISPATCHER_ERROR, errno, "%s", strerror (errno));
@@ -302,8 +302,9 @@ read_buffers (int fd, rspamd_io_dispatcher_t * d, gboolean skip_read)
 			return;
 		}
 		else {
-			*pos += r;
-			*len += r;
+			/* Set current position in buffer */
+			d->in_buf->pos += r;
+			d->in_buf->data->len += r;
 		}
 		debug_ip (d->peer_addr, "read %z characters, policy is %s, watermark is: %z", r, 
 				d->policy == BUFFER_LINE ? "LINE" : "CHARACTER", d->nchars);
@@ -311,6 +312,8 @@ read_buffers (int fd, rspamd_io_dispatcher_t * d, gboolean skip_read)
 
 	saved_policy = d->policy;
 	c = d->in_buf->data->begin;
+	end = d->in_buf->pos;
+	len = d->in_buf->data->len;
 	b = c;
 	r = 0;
 	
@@ -323,7 +326,7 @@ read_buffers (int fd, rspamd_io_dispatcher_t * d, gboolean skip_read)
 		* c - pointer to current position (buffer->begin + r)
 		* res - result string 
 		*/
-		while (r < *len) {
+		while (r < len) {
 			if (*c == '\n') {
 				res.begin = b;
 				res.len = c - b;
@@ -344,19 +347,24 @@ read_buffers (int fd, rspamd_io_dispatcher_t * d, gboolean skip_read)
 					}
 					if (d->policy != saved_policy) {
 						/* Drain buffer as policy is changed */
-						len = &d->in_buf->data->len;
-						pos = &d->in_buf->pos;
-						if (c != *pos) {
-								memmove (d->in_buf->data->begin, c + 1, *len - r - 1);
-								*len = *len -r - 1;
-								*pos = d->in_buf->data->begin + *len;
+						/* Note that d->in_buffer is other pointer now, so we need to reinit all pointers */
+						/* First detect how much symbols do we have */
+						if (end == c) {
+							/* In fact we read the whole buffer and change input policy, so just set current pos to begin of buffer */
+							d->in_buf->pos = d->in_buf->data->begin;
+							d->in_buf->data->len = 0;
 						}
 						else {
-							*len = 0;
-							*pos = d->in_buf->data->begin;
+							/* Otherwise we need to move buffer */
+							/* Reinit pointers */
+							len = d->in_buf->data->len - r - 1;
+							end = d->in_buf->data->begin + r + 1;
+							memmove (d->in_buf->data->begin, end, len);
+							d->in_buf->data->len = len;
+							d->in_buf->pos = d->in_buf->data->begin + len;
+							/* Process remaining buffer */
+							read_buffers (fd, d, TRUE);
 						}
-						debug_ip (d->peer_addr, "policy changed during callback, restart buffer's processing");
-						read_buffers (fd, d, TRUE);
 						return;
 					}
 				}
@@ -366,16 +374,14 @@ read_buffers (int fd, rspamd_io_dispatcher_t * d, gboolean skip_read)
 			r++;
 			c++;
 		}
-		/* Now drain buffer */
-		len = &d->in_buf->data->len;
-		pos = &d->in_buf->pos;
+		/* Now drain remaining characters in buffer */
 		memmove (d->in_buf->data->begin, b, c - b);
-		*len = c - b;
-		*pos = d->in_buf->data->begin + *len;
+		d->in_buf->data->len = c - b;
+		d->in_buf->pos = d->in_buf->data->begin + (c - b);
 		break;
 	case BUFFER_CHARACTER:
 		r = d->nchars;
-		if (*len >= r) {
+		if (len >= r) {
 			res.begin = b;
 			res.len = r;
 			c = b + r;
@@ -384,14 +390,15 @@ read_buffers (int fd, rspamd_io_dispatcher_t * d, gboolean skip_read)
 					return;
 				}
 				/* Move remaining string to begin of buffer (draining) */
-				len = &d->in_buf->data->len;
-				pos = &d->in_buf->pos;
-				memmove (d->in_buf->data->begin, c, *len - r);
-				b = d->in_buf->data->begin;
-				c = b;
-				*len -= r;
-				*pos = b + *len;
-				if (d->policy != saved_policy) {
+				if (len > r) {
+					len -= r;
+					memmove (d->in_buf->data->begin, c, len);
+					d->in_buf->data->len = len;
+					d->in_buf->pos = d->in_buf->data->begin + len;
+					b = d->in_buf->data->begin;
+					c = b;
+				}
+				if (d->policy != saved_policy && len != r) {
 					debug_ip (d->peer_addr, "policy changed during callback, restart buffer's processing");
 					read_buffers (fd, d, TRUE);
 					return;
@@ -401,7 +408,7 @@ read_buffers (int fd, rspamd_io_dispatcher_t * d, gboolean skip_read)
 		break;
 	case BUFFER_ANY:
 		res.begin = d->in_buf->data->begin;
-		res.len = *len;
+		res.len = len;
 		if (d->read_callback) {
 			if (!d->read_callback (&res, d->user_data)) {
 				return;
diff --git a/src/cfg_utils.c b/src/cfg_utils.c
index 84115f4f7..26730305e 100644
--- a/src/cfg_utils.c
+++ b/src/cfg_utils.c
@@ -533,7 +533,9 @@ get_config_checksum (struct config_file *cfg)
 void
 post_load_config (struct config_file *cfg)
 {
+#ifdef HAVE_CLOCK_GETTIME
 	struct timespec                 ts;
+#endif
 	struct metric                  *def_metric;
 
 	g_hash_table_foreach (cfg->variables, substitute_all_variables, cfg);
@@ -541,13 +543,15 @@ post_load_config (struct config_file *cfg)
 	parse_filters_str (cfg, cfg->filters_str);
 	fill_cfg_params (cfg);
 
+#ifdef HAVE_CLOCK_GETTIME
 #ifdef HAVE_CLOCK_PROCESS_CPUTIME_ID
 	clock_getres (CLOCK_PROCESS_CPUTIME_ID, &ts);
-#elif defined(HAVE_CLOCK_VIRTUAL)
+# elif defined(HAVE_CLOCK_VIRTUAL)
 	clock_getres (CLOCK_VIRTUAL, &ts);
-#else
+# else
 	clock_getres (CLOCK_REALTIME, &ts);
-#endif
+# endif
+
 	cfg->clock_res = (int)log10 (1000000 / ts.tv_nsec);
 	if (cfg->clock_res < 0) {
 		cfg->clock_res = 0;
@@ -555,6 +559,10 @@ post_load_config (struct config_file *cfg)
 	if (cfg->clock_res > 3) {
 		cfg->clock_res = 3;
 	}
+#else 
+	/* For gettimeofday */
+	cfg->clock_res = 1;
+#endif
 
 	if ((def_metric = g_hash_table_lookup (cfg->metrics, DEFAULT_METRIC)) == NULL) {
 		def_metric = check_metric_conf (cfg, NULL);
diff --git a/src/evdns/evdns.c b/src/evdns/evdns.c
index e399a14a0..b19da2229 100644
--- a/src/evdns/evdns.c
+++ b/src/evdns/evdns.c
@@ -1126,6 +1126,7 @@ static u16
 default_transaction_id_fn(void)
 {
 	u16 trans_id;
+#ifdef HAVE_CLOCK_GETTIME
 	struct timespec ts;
 	static int clkid = -1;
 	if (clkid == -1) {
@@ -1135,6 +1136,13 @@ default_transaction_id_fn(void)
 	}
 	clock_gettime(clkid, &ts);
 	trans_id = ts.tv_nsec & 0xffff;
+#elif defined(HAVE_SYS_TIMEDB_H)
+	struct timeb tb;
+	ftime(&tb);
+	trans_id = tb.millitm & 0xffff;
+#else
+# error Cannot find way to generate dns transaction id
+#endif
 
 	return trans_id;
 }
diff --git a/src/main.h b/src/main.h
index 89169da10..4b407d705 100644
--- a/src/main.h
+++ b/src/main.h
@@ -221,7 +221,11 @@ struct worker_task {
 	char *last_error;											/**< last error										*/
 	int error_code;												/**< code of last error								*/
 	memory_pool_t *task_pool;									/**< memory pool for task							*/
+#ifdef HAVE_CLOCK_GETTIME
 	struct timespec ts;											/**< time of connection								*/
+#else
+	struct timeval tv;											/**< time of connection								*/
+#endif
 	struct rspamd_view *view;									/**< matching view									*/
 	gboolean view_checked;
 	gboolean pass_all_filters;									/**< pass task throught every rule					*/
diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c
index 3782e1194..e557c0a6d 100644
--- a/src/plugins/fuzzy_check.c
+++ b/src/plugins/fuzzy_check.c
@@ -506,9 +506,15 @@ fuzzy_symbol_callback (struct worker_task *task, void *unused)
 		}
 
 		/* Get upstream */
+#ifdef HAVE_CLOCK_GETTIME
 		selected = (struct storage_server *)get_upstream_by_hash (fuzzy_module_ctx->servers, fuzzy_module_ctx->servers_num,
 			sizeof (struct storage_server), task->ts.tv_sec,
 			DEFAULT_UPSTREAM_ERROR_TIME, DEFAULT_UPSTREAM_DEAD_TIME, DEFAULT_UPSTREAM_MAXERRORS, part->fuzzy->hash_pipe, sizeof (part->fuzzy->hash_pipe));
+#else
+		selected = (struct storage_server *)get_upstream_by_hash (fuzzy_module_ctx->servers, fuzzy_module_ctx->servers_num,
+			sizeof (struct storage_server), task->tv.tv_sec,
+			DEFAULT_UPSTREAM_ERROR_TIME, DEFAULT_UPSTREAM_DEAD_TIME, DEFAULT_UPSTREAM_MAXERRORS, part->fuzzy->hash_pipe, sizeof (part->fuzzy->hash_pipe));
+#endif
 		if (selected) {
 			if ((sock = make_udp_socket (&selected->addr, selected->port, FALSE, TRUE)) == -1) {
 				msg_warn ("cannot connect to %s, %d, %s", selected->name, errno, strerror (errno));
@@ -584,9 +590,15 @@ fuzzy_process_handler (struct controller_session *session, f_str_t * in)
 				continue;
 			}
 			/* Get upstream */
+#ifdef HAVE_CLOCK_GETTIME
 			selected = (struct storage_server *)get_upstream_by_hash (fuzzy_module_ctx->servers, fuzzy_module_ctx->servers_num,
 				sizeof (struct storage_server), task->ts.tv_sec,
 				DEFAULT_UPSTREAM_ERROR_TIME, DEFAULT_UPSTREAM_DEAD_TIME, DEFAULT_UPSTREAM_MAXERRORS, part->fuzzy->hash_pipe, sizeof (part->fuzzy->hash_pipe));
+#else
+			selected = (struct storage_server *)get_upstream_by_hash (fuzzy_module_ctx->servers, fuzzy_module_ctx->servers_num,
+				sizeof (struct storage_server), task->tv.tv_sec,
+				DEFAULT_UPSTREAM_ERROR_TIME, DEFAULT_UPSTREAM_DEAD_TIME, DEFAULT_UPSTREAM_MAXERRORS, part->fuzzy->hash_pipe, sizeof (part->fuzzy->hash_pipe));
+#endif
 			if (selected) {
 				/* Create UDP socket */
 				if ((sock = make_udp_socket (&selected->addr, selected->port, FALSE, TRUE)) == -1) {
diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c
index bee37e209..810b2fbe5 100644
--- a/src/plugins/surbl.c
+++ b/src/plugins/surbl.c
@@ -325,11 +325,11 @@ format_surbl_request (memory_pool_t * pool, f_str_t * hostname, struct suffix_it
 		r = snprintf (result, len, "%*s.%*s.%*s.%*s", 
 				(int)(hostname->len - (dots[2] - hostname->begin + 1)),
 				dots[2] + 1,
-				(int)(dots[2] - (dots[1] - hostname->begin + 1)),
+				(int)(dots[2] - dots[1] - 1),
 				dots[1],
-				(int)(dots[1] - (dots[0] - hostname->begin + 1)),
+				(int)(dots[1] - dots[0] - 1),
 				dots[0],
-				(int)(dots[0] - (hostname->begin + 1)),
+				(int)(dots[0] - hostname->begin),
 				hostname->begin);
 	}
 	else if (is_numeric && dots_num == 0) {
diff --git a/src/protocol.c b/src/protocol.c
index a4c679f96..a49682b2e 100644
--- a/src/protocol.c
+++ b/src/protocol.c
@@ -659,8 +659,13 @@ show_metric_result (gpointer metric_name, gpointer metric_value, void *user_data
 			show_metric_symbols (metric_res, cd);
 		}
 	}
+#ifdef HAVE_CLOCK_GETTIME
 	cd->log_offset += snprintf (cd->log_buf + cd->log_offset, cd->log_size - cd->log_offset, "]), len: %ld, time: %sms",
 		(long int)task->msg->len, calculate_check_time (&task->ts, task->cfg->clock_res));
+#else
+	cd->log_offset += snprintf (cd->log_buf + cd->log_offset, cd->log_size - cd->log_offset, "]), len: %ld, time: %sms",
+		(long int)task->msg->len, calculate_check_time (&task->tv, task->cfg->clock_res));
+#endif
 }
 
 static void
diff --git a/src/smtp.c b/src/smtp.c
index a6a1f901c..24e453c4f 100644
--- a/src/smtp.c
+++ b/src/smtp.c
@@ -201,7 +201,7 @@ read_smtp_command (struct smtp_session *session, f_str_t *line)
 		case SMTP_COMMAND_NOOP:
 			break;
 		case SMTP_COMMAND_MAIL:
-			if ((session->state == SMTP_STATE_GREETING || session->state == SMTP_STATE_HELO && !session->ctx->helo_required) 
+			if (((session->state == SMTP_STATE_GREETING || session->state == SMTP_STATE_HELO) && !session->ctx->helo_required) 
 					|| session->state == SMTP_STATE_FROM) {
 				if (parse_smtp_from (session, cmd)) {
 					session->state = SMTP_STATE_RCPT;
@@ -526,8 +526,13 @@ smtp_write_socket (void *arg)
 					cur = g_list_next (cur);
 				}
 				g_list_free (symbols);
+#ifdef HAVE_CLOCK_GETTIME
 				r += snprintf (logbuf + r, sizeof (logbuf) - r, "]), len: %ld, time: %sms",
 					(long int)session->task->msg->len, calculate_check_time (&session->task->ts, session->cfg->clock_res));
+#else
+				r += snprintf (logbuf + r, sizeof (logbuf) - r, "]), len: %ld, time: %sms",
+					(long int)session->task->msg->len, calculate_check_time (&session->task->tv, session->cfg->clock_res));
+#endif
 				msg_info ("%s", logbuf);
 
 				if (is_spam) {
diff --git a/src/symbols_cache.c b/src/symbols_cache.c
index 046d2f6d7..f24380f28 100644
--- a/src/symbols_cache.c
+++ b/src/symbols_cache.c
@@ -596,7 +596,11 @@ struct symbol_callback_data {
 gboolean
 call_symbol_callback (struct worker_task * task, struct symbols_cache * cache, gpointer *save)
 {
+#ifdef HAVE_CLOCK_GETTIME
 	struct timespec                 ts1, ts2;
+#else
+	struct timeval                  tv1, tv2;
+#endif
 	uint64_t                        diff;
 	struct cache_item              *item = NULL;
 	struct symbol_callback_data    *s = *save;
@@ -753,24 +757,40 @@ call_symbol_callback (struct worker_task * task, struct symbols_cache * cache, g
 		return FALSE;
 	}
 	if (check_view (task->cfg->views, item->s->symbol, task)) {
-#ifdef HAVE_CLOCK_PROCESS_CPUTIME_ID
+#ifdef HAVE_CLOCK_GETTIME
+# ifdef HAVE_CLOCK_PROCESS_CPUTIME_ID
 		clock_gettime (CLOCK_PROCESS_CPUTIME_ID, &ts1);
-#elif defined(HAVE_CLOCK_VIRTUAL)
+# elif defined(HAVE_CLOCK_VIRTUAL)
 		clock_gettime (CLOCK_VIRTUAL, &ts1);
-#else
+# else
 		clock_gettime (CLOCK_REALTIME, &ts1);
+# endif
+#else
+		if (gettimeofday (&tv1, NULL) == -1) {
+			msg_warn ("gettimeofday failed: %s", strerror (errno));
+		}
 #endif
 		item->func (task, item->user_data);
 
-#ifdef HAVE_CLOCK_PROCESS_CPUTIME_ID
+#ifdef HAVE_CLOCK_GETTIME
+# ifdef HAVE_CLOCK_PROCESS_CPUTIME_ID
 		clock_gettime (CLOCK_PROCESS_CPUTIME_ID, &ts2);
-#elif defined(HAVE_CLOCK_VIRTUAL)
+# elif defined(HAVE_CLOCK_VIRTUAL)
 		clock_gettime (CLOCK_VIRTUAL, &ts2);
-#else
+# else
 		clock_gettime (CLOCK_REALTIME, &ts2);
+# endif
+#else
+		if (gettimeofday (&tv2, NULL) == -1) {
+			msg_warn ("gettimeofday failed: %s", strerror (errno));
+		}
 #endif
 
+#ifdef HAVE_CLOCK_GETTIME
 		diff = (ts2.tv_sec - ts1.tv_sec) * 1000000 + (ts2.tv_nsec - ts1.tv_nsec) / 1000;
+#else
+		diff = (tv2.tv_sec - tv1.tv_sec) * 1000000 + (tv2.tv_usec - tv1.tv_usec);
+#endif
 		item->s->avg_time = set_counter (item->s->symbol, diff);
 	}
 
diff --git a/src/util.c b/src/util.c
index 9ab0820f0..7e9aeb463 100644
--- a/src/util.c
+++ b/src/util.c
@@ -769,13 +769,19 @@ resolve_stat_filename (memory_pool_t * pool, char *pattern, char *rcpt, char *fr
 	return new;
 }
 
+#ifdef HAVE_CLOCK_GETTIME
 const char                     *
 calculate_check_time (struct timespec *begin, int resolution)
+#else
+const char                     *
+calculate_check_time (struct timeval *begin, int resolution)
+#endif
 {
-	struct timespec                 ts;
 	double                          diff;
 	static char                     res[sizeof ("100000.000")];
 	static char                     fmt[sizeof ("%.10f")];
+#ifdef HAVE_CLOCK_GETTIME
+	struct timespec                 ts;
 
 #ifdef HAVE_CLOCK_PROCESS_CPUTIME_ID
 	clock_gettime (CLOCK_PROCESS_CPUTIME_ID, &ts);
@@ -787,6 +793,16 @@ calculate_check_time (struct timespec *begin, int resolution)
 
 	diff = (ts.tv_sec - begin->tv_sec) * 1000. +	/* Seconds */
 		(ts.tv_nsec - begin->tv_nsec) / 1000000.;	/* Nanoseconds */
+#else
+	struct timeval                  tv;
+
+	if (gettimeofday (&tv, NULL) == -1) {
+		msg_warn ("gettimeofday failed: %s", strerror (errno));
+	}
+	diff = (tv.tv_sec - begin->tv_sec) * 1000. +	/* Seconds */
+		(tv.tv_usec - begin->tv_usec) / 1000.;	/* Microseconds */
+#endif
+
 	sprintf (fmt, "%%.%df", resolution);
 	snprintf (res, sizeof (res), fmt, diff);
 
diff --git a/src/util.h b/src/util.h
index 990a31544..1e7df2378 100644
--- a/src/util.h
+++ b/src/util.h
@@ -64,7 +64,11 @@ int pidfile_remove(struct pidfh *pfh);
 
 /* Replace %r with rcpt value and %f with from value, new string is allocated in pool */
 char* resolve_stat_filename (memory_pool_t *pool, char *pattern, char *rcpt, char *from);
+#ifdef HAVE_CLOCK_GETTIME
 const char* calculate_check_time (struct timespec *begin, int resolution);
+#else
+const char* calculate_check_time (struct timeval *begin, int resolution);
+#endif
 
 double set_counter (const char *name, long int value);
 
diff --git a/src/worker.c b/src/worker.c
index 5ebdaf84b..bfc4e4331 100644
--- a/src/worker.c
+++ b/src/worker.c
@@ -412,12 +412,18 @@ construct_task (struct rspamd_worker *worker)
 	new_task->cfg = worker->srv->cfg;
 	new_task->from_addr.s_addr = INADDR_NONE;
 	new_task->view_checked = FALSE;
-#ifdef HAVE_CLOCK_PROCESS_CPUTIME_ID
+#ifdef HAVE_CLOCK_GETTIME
+# ifdef HAVE_CLOCK_PROCESS_CPUTIME_ID
 	clock_gettime (CLOCK_PROCESS_CPUTIME_ID, &new_task->ts);
-#elif defined(HAVE_CLOCK_VIRTUAL)
+# elif defined(HAVE_CLOCK_VIRTUAL)
 	clock_gettime (CLOCK_VIRTUAL, &new_task->ts);
-#else
+# else
 	clock_gettime (CLOCK_REALTIME, &new_task->ts);
+# endif
+#else
+	if (gettimeofday (&new_task->tv, NULL) == -1) {
+		msg_warn ("gettimeofday failed: %s", strerror (errno));
+	}
 #endif
 	io_tv.tv_sec = WORKER_IO_TIMEOUT;
 	io_tv.tv_usec = 0;