[Test] Surbl IDN + punycode

3 files changed, 53 insertions, 14 deletions

diff --git a/src/libutil/addr.c b/src/libutil/addr.c
index 138dc1318..15480a9ad 100644
--- a/src/libutil/addr.c
+++ b/src/libutil/addr.c
@@ -248,18 +248,19 @@ rspamd_accept_from_socket (gint sock, rspamd_inet_addr_t **target,
 				addr = rspamd_inet_addr_create (AF_INET, NULL);
 				memcpy (&addr->u.in.addr.s4.sin_addr, &p[12],
 						sizeof (struct in_addr));
+				addr->u.in.addr.s4.sin_port = su.s6.sin6_port;
 			}
 			else {
 				/* Something strange but not mapped v4 address */
 				addr = rspamd_inet_addr_create (AF_INET6, NULL);
-				memcpy (&addr->u.in.addr.s6.sin6_addr, &su.s6.sin6_addr,
-						sizeof (struct in6_addr));
+				memcpy (&addr->u.in.addr.s6, &su.s6,
+						sizeof (struct sockaddr_in6));
 			}
 		}
 		else {
 			addr = rspamd_inet_addr_create (AF_INET6, NULL);
-			memcpy (&addr->u.in.addr.s6.sin6_addr, &su.s6.sin6_addr,
-					sizeof (struct in6_addr));
+			memcpy (&addr->u.in.addr.s6, &su.s6,
+					sizeof (struct sockaddr_in6));
 		}
 
 	}
diff --git a/src/libutil/ssl_util.c b/src/libutil/ssl_util.c
index d145e4f5b..f2898d592 100644
--- a/src/libutil/ssl_util.c
+++ b/src/libutil/ssl_util.c
@@ -47,13 +47,13 @@ struct rspamd_ssl_connection {
 	gboolean verify_peer;
 	SSL *ssl;
 	gchar *hostname;
-	const gchar *log_tag;
 	struct rspamd_io_ev *ev;
 	struct rspamd_io_ev *shut_ev;
 	struct ev_loop *event_loop;
 	rspamd_ssl_handler_t handler;
 	rspamd_ssl_error_handler_t err_handler;
 	gpointer handler_data;
+	gchar log_tag[8];
 };
 
 #define msg_debug_ssl(...)  rspamd_conditional_debug_fast (NULL, NULL, \
@@ -621,7 +621,14 @@ rspamd_ssl_connection_new (gpointer ssl_ctx, struct ev_loop *ev_base,
 	c->ssl = SSL_new (ssl_ctx);
 	c->event_loop = ev_base;
 	c->verify_peer = verify_peer;
-	c->log_tag = log_tag;
+
+	if (log_tag) {
+		rspamd_strlcpy (c->log_tag, log_tag, sizeof (log_tag));
+	}
+	else {
+		rspamd_random_hex (c->log_tag, sizeof (log_tag) - 1);
+		c->log_tag[sizeof (log_tag) - 1] = '\0';
+	}
 
 	return c;
 }
diff --git a/src/plugins/lua/clickhouse.lua b/src/plugins/lua/clickhouse.lua
index 878a15064..c4e30f1db 100644
--- a/src/plugins/lua/clickhouse.lua
+++ b/src/plugins/lua/clickhouse.lua
@@ -32,6 +32,7 @@ local custom_rows = {}
 local nrows = 0
 local used_memory = 0
 local last_collection = 0
+local final_call = false -- If the final collection has been started
 local schema_version = 8 -- Current schema version
 
 local settings = {
@@ -375,7 +376,7 @@ local function clickhouse_check_symbol(task, settings_field_name, fields_table,
   return false
 end
 
-local function clickhouse_send_data(task, ev_base, why)
+local function clickhouse_send_data(task, ev_base, why, gen_rows, cust_rows)
   local log_object = task or rspamd_config
   local upstream = settings.upstream:get_upstream_round_robin()
   local ip_addr = upstream:get_addr():to_string(true)
@@ -429,11 +430,11 @@ local function clickhouse_send_data(task, ev_base, why)
     clickhouse_groups_row(fields)
   end
 
-  send_data('generic data', data_rows,
+  send_data('generic data', gen_rows,
       string.format('INSERT INTO rspamd (%s)',
           table.concat(fields, ',')))
 
-  for k,crows in pairs(custom_rows) do
+  for k,crows in pairs(cust_rows) do
     if #crows > 1 then
       send_data('custom data ('..k..')', crows,
           settings.custom_rules[k].first_row())
@@ -929,19 +930,27 @@ local function clickhouse_maybe_send_data_periodic(cfg, ev_base, now)
     return settings.check_timeout
   end
 
+  if final_call then
+    lua_util.debugm(N, cfg, "no need to send data, final call has been issued")
+    return 0
+  end
+
   if settings.limits.max_rows > 0 then
     if nrows > settings.limits.max_rows then
       need_collect = true
       reason = string.format('limit of rows has been reached: %d', nrows)
     end
   end
-  if settings.limits.max_interval > 0 then
+
+  if last_collection > 0 and settings.limits.max_interval > 0 then
     if now - last_collection > settings.limits.max_interval then
       need_collect = true
-      reason = string.format('limit of time since last collection has been reached: %d seconds passed',
-          (now - last_collection) - settings.limits.max_interval)
+      reason = string.format('limit of time since last collection has been reached: %d seconds passed ' ..
+          '(%d seconds trigger)',
+          (now - last_collection), settings.limits.max_interval)
     end
   end
+
   if settings.limits.max_memory > 0 then
     if used_memory >= settings.limits.max_memory then
       need_collect = true
@@ -950,14 +959,22 @@ local function clickhouse_maybe_send_data_periodic(cfg, ev_base, now)
     end
   end
 
+  if last_collection == 0 then
+    last_collection = now
+  end
+
   if need_collect then
-    clickhouse_send_data(nil, ev_base, reason)
+    -- Do it atomic
+    local saved_rows = data_rows
+    local saved_custom = custom_rows
     nrows = 0
     last_collection = now
     used_memory = 0
     data_rows = {}
     custom_rows = {}
 
+    clickhouse_send_data(nil, ev_base, reason, saved_rows, saved_custom)
+
     if settings.collect_garbadge then
       collectgarbage()
     end
@@ -1243,7 +1260,21 @@ if opts then
     })
     rspamd_config:register_finish_script(function(task)
       if nrows > 0 then
-        clickhouse_send_data(task, nil, 'final collection')
+        final_call = true
+        local saved_rows = data_rows
+        local saved_custom = custom_rows
+
+        nrows = 0
+        data_rows = {}
+        used_memory = 0
+        custom_rows = {}
+
+        clickhouse_send_data(task, nil, 'final collection',
+            saved_rows, saved_custom)
+
+        if settings.collect_garbadge then
+          collectgarbage()
+        end
       end
     end)
     -- Create tables on load