[Test] Test URL tag persistence

test/functional/cases/106_url_tags.robot

@@ -0,0 +1,42 @@
+*** Settings ***
+Suite Setup     URL Tags Setup
+Suite Teardown  URL Tags Teardown
+Library         ${TESTDIR}/lib/rspamd.py
+Resource        ${TESTDIR}/lib/rspamd.robot
+Variables       ${TESTDIR}/lib/vars.py
+
+*** Variables ***
+${ADDITIONAL}   ${TESTDIR}/lua/url_tags.lua
+${CONFIG}       ${TESTDIR}/configs/pluginsplus.conf
+${MESSAGE}      ${TESTDIR}/messages/url1.eml
+${REDIS_SCOPE}  Suite
+${RSPAMD_SCOPE}  Suite
+${URL_TLD}      ${TESTDIR}/../lua/unit/test_tld.dat
+
+*** Test Cases ***
+URL TAGS PERSISTENCE
+  ${result} =  Scan Message With Rspamc  --header=addtags=1  ${MESSAGE}
+  Check Rspamc  ${result}  ADDED_TAGS (1.00)[no worry]
+  ${result} =  Scan Message With Rspamc  ${MESSAGE}
+  Check Rspamc  ${result}  FOUND_TAGS (1.00)[no worry]
+
+*** Keywords ***
+URL Tags Setup
+  ${TMPDIR} =  Make Temporary Directory
+  Set Suite Variable  ${TMPDIR}
+  Run Redis
+  ${LUA} =  Make Temporary File
+  ${goop} =  Get File  ${TESTDIR}/../../rules/global_functions.lua
+  ${goop2} =  Get File  ${ADDITIONAL}
+  ${goop_unesc} =  Catenate  ${goop}  ${goop2}
+  ${PLUGIN_CONFIG} =  Get File  ${TESTDIR}/configs/url_tags.conf
+  Set Suite Variable  ${LUA}
+  Set Suite Variable  ${PLUGIN_CONFIG}
+  Create File  ${LUA}  ${goop_unesc}
+  Generic Setup  TMPDIR=${TMPDIR}
+
+URL Tags Teardown
+  Normal Teardown
+  Remove File  ${LUA}
+  Shutdown Process With Children  ${REDIS_PID}
+  Wait For Port  ${SOCK_STREAM}  ${LOCAL_ADDR}  ${REDIS_PORT}

test/functional/configs/pluginsplus.conf

@@ -0,0 +1,34 @@
+options = {
+	filters = ["spf", "dkim", "regexp"]
+	url_tld = "${URL_TLD}"
+	pidfile = "${TMPDIR}/rspamd.pid"
+	lua_path = "${TESTDIR}/../../contrib/lua-fun/?.lua"
+	dns {
+		nameserver = ["8.8.8.8", "8.8.4.4"];
+		retransmits = 10;
+		timeout = 2s;
+	}
+}
+logging = {
+	type = "file",
+	level = "debug"
+	filename = "${TMPDIR}/rspamd.log"
+}
+metric = {
+	name = "default",
+	actions = {
+		reject = 100500,
+	}
+	unknown_weight = 1
+}
+worker {
+	type = normal
+	bind_socket = ${LOCAL_ADDR}:${PORT_NORMAL}
+	count = 1
+	task_timeout = 60s;
+}
+modules {
+    path = "${TESTDIR}/../../src/plugins/lua/"
+}
+lua = "${LUA}";
+${PLUGIN_CONFIG}

test/functional/configs/url_tags.conf

@@ -0,0 +1,5 @@
+url_tags {
+}
+redis {
+  servers = "${REDIS_ADDR}:${REDIS_PORT}";
+}

test/functional/lib/rspamd.robot

@@ -53,8 +53,8 @@ Follow Rspamd Log
   ...  ELSE  Fail  'RSPAMD_SCOPE must be Test or Suite'
 
 Generic Setup
-  [Arguments]  @{vargs}
-  &{d} =  Run Rspamd  @{vargs}
+  [Arguments]  @{vargs}  &{kwargs}
+  &{d} =  Run Rspamd  @{vargs}  &{kwargs}
   ${keys} =  Get Dictionary Keys  ${d}
   : FOR  ${i}  IN  @{keys}
   \  Run Keyword If  '${RSPAMD_SCOPE}' == 'Suite'  Set Suite Variable  ${${i}}  &{d}[${i}]
@@ -111,9 +111,11 @@ Run Rspamc
 Run Rspamd
   [Arguments]  @{vargs}  &{kwargs}
   ${has_CONFIG} =  Evaluate  'CONFIG' in $kwargs
-  ${CONFIG} =  Set Variable If  ${has_CONFIG} == True  &{kwargs}[CONFIG]  ${CONFIG}
+  ${has_TMPDIR} =  Evaluate  'TMPDIR' in $kwargs
+  ${CONFIG} =  Set Variable If  '${has_CONFIG}' == 'True'  &{kwargs}[CONFIG]  ${CONFIG}
   &{d} =  Create Dictionary
-  ${tmpdir} =  Make Temporary Directory
+  ${tmpdir} =  Run Keyword If  '${has_TMPDIR}' == 'True'  Set Variable  &{kwargs}[TMPDIR]
+  ...  ELSE  Make Temporary Directory
   Set Directory Ownership  ${tmpdir}  ${RSPAMD_USER}  ${RSPAMD_GROUP}
   ${template} =  Get File  ${CONFIG}
   : FOR  ${i}  IN  @{vargs}

test/functional/lua/url_tags.lua

@@ -0,0 +1,55 @@
+local rspamd_logger = require 'rspamd_logger'
+
+rspamd_config:register_symbol({
+  name = 'ADDED_TAGS',
+  score = 1.0,
+  callback = function(task)
+    if not task:get_request_header('addtags') then
+      return true, 'nope! not requested'
+    end
+    local urls = task:get_urls()
+    if not (urls and urls[1]) then
+      return true, 'nope! found no urls'
+    end
+    local mpool = task:get_mempool()
+    for _, u in ipairs(urls) do
+      u:add_tag('test1', 'meta1', mpool)
+      u:add_tag('test1', 'meta2', mpool)
+      u:add_tag('test2', 'http://www.example.com', mpool)
+    end
+    return true, 'no worry'
+  end
+})
+
+rspamd_config:register_symbol({
+  name = 'FOUND_TAGS',
+  score = 1.0,
+  callback = function(task)
+    local urls = task:get_urls()
+    if not (urls and urls[1]) then
+      return true, 'nope! found no urls'
+    end
+    for _, u in ipairs(urls) do
+      local tags = u:get_tags()
+      rspamd_logger.debugx(task, 'tags: %1', tags)
+      if not tags['test1'] then
+        return true, 'no key - test1'
+      end
+      local found1, found2 = false, false
+      for _, e in ipairs(tags['test1']) do
+        if e == 'meta1' then found1 = true end
+        if e == 'meta2' then found2 = true end
+      end
+      if not (found1 and found2) then
+        return true, 'missing metatags in test1'
+      end
+      if not tags['test2'] then
+        return true, 'no key - test2'
+      end
+      if not tags['test2'][1] == 'http://www.example.com' then
+        return true, 'wrong value in test2 metatag: ' .. tags['test2'][1]
+      end
+    end
+    return true, 'no worry'
+  end
+})