From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Thu, 10 Oct 2019 10:29:55 +0000 (+0100)
Subject: [Feature] Store Clickhouse data outside of lua alloc
X-Git-Tag: 2.0~27
X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=c92f84d76e32100d6f5e1767a468cf3fe49a5db7;p=rspamd.git

[Feature] Store Clickhouse data outside of lua alloc
---

diff --git a/lualib/lua_clickhouse.lua b/lualib/lua_clickhouse.lua
index 4a57afd3f..2a75326d4 100644
--- a/lualib/lua_clickhouse.lua
+++ b/lualib/lua_clickhouse.lua
@@ -23,6 +23,7 @@ limitations under the License.
 local rspamd_logger = require "rspamd_logger"
 local rspamd_http = require "rspamd_http"
 local lua_util = require "lua_util"
+local rspamd_text = require "rspamd_text"
 
 local exports = {}
 local N = 'clickhouse'
@@ -81,9 +82,11 @@ local function row_to_tsv(row)
     end
   end
 
-  return table.concat(row, '\t')
+  return rspamd_text.fromtable(row, '\t')
 end
 
+exports.row_to_tsv = row_to_tsv
+
 -- Parses JSONEachRow reply from CH
 local function parse_clickhouse_response_json_eachrow(params, data)
   local ucl = require "ucl"
@@ -373,9 +376,7 @@ exports.insert = function (upstream, settings, params, query, rows,
   http_params.user = settings.user
   http_params.password = settings.password
   http_params.method = 'POST'
-  http_params.body = {table.concat(fun.totable(fun.map(function(row)
-    return row_to_tsv(row)
-  end, rows)), '\n'), '\n'}
+  http_params.body = {rspamd_text.from_table(rows, '\n'), '\n'}
   http_params.log_obj = params.task or params.config
 
   if not http_params.url then
diff --git a/src/plugins/lua/clickhouse.lua b/src/plugins/lua/clickhouse.lua
index 1505b51aa..45c555460 100644
--- a/src/plugins/lua/clickhouse.lua
+++ b/src/plugins/lua/clickhouse.lua
@@ -421,12 +421,13 @@ local function clickhouse_send_data(task, ev_base)
   end
 
   send_data('generic data', data_rows,
-      string.format('INSERT INTO rspamd (%s)', table.concat(fields, ',')))
+      string.format('INSERT INTO rspamd (%s)',
+          table.concat(fields, ',')))
 
   for k,crows in pairs(custom_rows) do
     if #crows > 1 then
-      send_data('custom data ('..k..')', settings.custom_rules[k].first_row(),
-          crows)
+      send_data('custom data ('..k..')', crows,
+          settings.custom_rules[k].first_row())
     end
   end
 end
@@ -823,11 +824,11 @@ local function clickhouse_collect(task)
   -- Custom data
   for k,rule in pairs(settings.custom_rules) do
     if not custom_rows[k] then custom_rows[k] = {} end
-    table.insert(custom_rows[k], rule.get_row(task))
+    table.insert(custom_rows[k], lua_clickhouse.row_to_tsv(rule.get_row(task)))
   end
 
   nrows = nrows + 1
-  table.insert(data_rows, row)
+  data_rows[#data_rows + 1] = lua_clickhouse.row_to_tsv(row)
   lua_util.debugm(N, task, "add clickhouse row %s / %s", nrows, settings.limit)
 
   if nrows >= settings['limit'] then