[Test] Test fuzzy replication

diff --git a/test/functional/cases/120_fuzzy/replication.robot b/test/functional/cases/120_fuzzy/replication.robot
new file mode 100644 (file)
index 0000000..cb7d425
--- /dev/null
+++ b/test/functional/cases/120_fuzzy/replication.robot
@@ -0,0 +1,84 @@
+*** Settings ***
+Suite Setup     Replication Setup
+Suite Teardown  Replication Teardown
+Resource        lib.robot
+Library         ${TESTDIR}/lib/rspamd.py
+Resource        ${TESTDIR}/lib/rspamd.robot
+Variables       ${TESTDIR}/lib/vars.py
+
+*** Variables ***
+${MESSAGE}      @{MESSAGES}[0]
+
+*** Test Cases ***
+Fuzzy Add And Check
+  Set Suite Variable  ${RSPAMD_FUZZY_ADD_${MESSAGE}}  0
+  ${result} =  Run Rspamc  -h  ${LOCAL_ADDR}:${PORT_CONTROLLER}  -w  10  -f
+  ...  ${FLAG1_NUMBER}  fuzzy_add  ${MESSAGE}
+  Custom Follow Rspamd Log  ${MASTER_TMPDIR}/rspamd.log  ${MASTER_LOGPOS}  MASTER_LOGPOS  Suite
+  Custom Follow Rspamd Log  ${SLAVE_TMPDIR}/rspamd.log  ${SLAVE_LOGPOS}  SLAVE_LOGPOS  Suite
+  Run Keyword If  ${result.rc} != 0  Log  ${result.stderr}
+  Should Contain  ${result.stdout}  success = true
+  Should Be Equal As Integers  ${result.rc}  0
+  Sync Fuzzy Storage  ${MASTER_TMPDIR}  ${MASTER_LOGPOS}  MASTER_LOGPOS  Suite
+  Sync Fuzzy Storage  ${SLAVE_TMPDIR}  ${SLAVE_LOGPOS}  SLAVE_LOGPOS  Suite
+  ${result} =  Scan Message With Rspamc  ${MESSAGE}
+  Custom Follow Rspamd Log  ${MASTER_TMPDIR}/rspamd.log  ${MASTER_LOGPOS}  MASTER_LOGPOS  Suite
+  Custom Follow Rspamd Log  ${SLAVE_TMPDIR}/rspamd.log  ${SLAVE_LOGPOS}  SLAVE_LOGPOS  Suite
+  Run Keyword If  ${result.rc} != 0  Log  ${result.stderr}
+  Should Contain  ${result.stdout}  ${FLAG1_SYMBOL}
+  Should Be Equal As Integers  ${result.rc}  0
+  Set Suite Variable  ${RSPAMD_FUZZY_ADD_${MESSAGE}}  1
+
+Fuzzy Check Slave
+  Run Keyword If  ${RSPAMD_FUZZY_ADD_${MESSAGE}} == 0  Fail  "Fuzzy Add was not run"
+  ${result} =  Run Rspamc  -h  ${LOCAL_ADDR}:${PORT_NORMAL_SLAVE}  ${MESSAGE}
+  Custom Follow Rspamd Log  ${SLAVE_TMPDIR}/rspamd.log  ${SLAVE_LOGPOS}  SLAVE_LOGPOS  Suite
+  Run Keyword If  ${result.rc} != 0  Log  ${result.stderr}
+  Should Contain  ${result.stdout}  ${FLAG1_SYMBOL}
+  Should Be Equal As Integers  ${result.rc}  0
+
+*** Keywords ***
+Replication Setup
+  ${tmp_fuzzy} =  Set Variable  ${PORT_FUZZY}
+  ${tmp_normal} =  Set Variable  ${PORT_NORMAL}
+  ${tmp_controller} =  Set Variable  ${PORT_CONTROLLER}
+  Set Suite Variable  ${PORT_FUZZY}  ${PORT_FUZZY_SLAVE}
+  Set Suite Variable  ${PORT_NORMAL}  ${PORT_NORMAL_SLAVE}
+  Set Suite Variable  ${PORT_CONTROLLER}  ${PORT_CONTROLLER_SLAVE}
+  ${algorithm} =  Set Variable  mumhash
+  ${worker_settings_tmpl} =  Get File  ${TESTDIR}/configs/fuzzy_slave_worker.conf
+  ${worker_settings} =  Replace Variables  ${worker_settings_tmpl}
+  ${tmp_include1} =  Make Temporary File
+  Set Suite Variable  ${TMP_INCLUDE1}  ${tmp_include1}
+  Create File  ${tmp_include1}  ${worker_settings}
+  ${worker_settings} =  Set Variable  .include ${tmp_include1}
+  ${check_settings} =  Set Variable  ${EMPTY}
+  Set Suite Variable  ${SETTINGS_FUZZY_WORKER}  ${worker_settings}
+  Set Suite Variable  ${SETTINGS_FUZZY_CHECK}  ${check_settings}
+  Set Suite Variable  ${ALGORITHM}  ${algorithm}
+  &{d} =  Run Rspamd  CONFIG=${TESTDIR}/configs/fuzzy.conf
+  Set Suite Variable  ${SLAVE_LOGPOS}  &{d}[RSPAMD_LOGPOS]
+  Set Suite Variable  ${SLAVE_PID}  &{d}[RSPAMD_PID]
+  Set Suite Variable  ${SLAVE_TMPDIR}  &{d}[TMPDIR]
+  Set Suite Variable  ${PORT_FUZZY}  ${tmp_fuzzy}
+  Set Suite Variable  ${PORT_NORMAL}  ${tmp_normal}
+  Set Suite Variable  ${PORT_CONTROLLER}  ${tmp_controller}
+  ${worker_settings_tmpl} =  Get File  ${TESTDIR}/configs/fuzzy_master_worker.conf
+  ${worker_settings} =  Replace Variables  ${worker_settings_tmpl}
+  ${tmp_include2} =  Make Temporary File
+  Set Suite Variable  ${TMP_INCLUDE2}  ${tmp_include2}
+  Create File  ${tmp_include2}  ${worker_settings}
+  ${worker_settings} =  Set Variable  .include ${tmp_include2}
+  Set Suite Variable  ${SETTINGS_FUZZY_WORKER}  ${worker_settings}
+  &{d} =  Run Rspamd  CONFIG=${TESTDIR}/configs/fuzzy.conf
+  Set Suite Variable  ${MASTER_LOGPOS}  &{d}[RSPAMD_LOGPOS]
+  Set Suite Variable  ${MASTER_PID}  &{d}[RSPAMD_PID]
+  Set Suite Variable  ${MASTER_TMPDIR}  &{d}[TMPDIR]
+
+Replication Teardown
+  Shutdown Process With Children  ${MASTER_PID}
+  Shutdown Process With Children  ${SLAVE_PID}
+  Cleanup Temporary Directory  ${MASTER_TMPDIR}
+  Cleanup Temporary Directory  ${SLAVE_TMPDIR}
+  Remove File  ${TMP_INCLUDE1}
+  Remove File  ${TMP_INCLUDE2}

diff --git a/test/functional/configs/fuzzy.conf b/test/functional/configs/fuzzy.conf
index deb518f4c26a1e39a3b4a1a40f28124e64530388..b3ab1225b7a4fe1816a98f29ce59b92f29113404 100644 (file)
--- a/test/functional/configs/fuzzy.conf
+++ b/test/functional/configs/fuzzy.conf
@@ -1,7 +1,8 @@
 options = {
        filters = "fuzzy_check";
-       pidfile = "${TMPDIR}/rspamd.pid"
+       pidfile = "${TMPDIR}/rspamd.pid";
        control_socket = "${TMPDIR}/rspamd.sock mode=0600";
+       url_tld = "${TESTDIR}/../lua/unit/test_tld.dat";
 }
 logging = {
        type = "file",
@@ -44,7 +45,7 @@ worker {
        type = "fuzzy";
        hashfile = "${TMPDIR}/fuzzy.db";
        allow_update = ["${LOCAL_ADDR}"];
-       ${SETTINGS_FUZZY_WORKER}
+${SETTINGS_FUZZY_WORKER}
 }
 
 fuzzy_check {
@@ -58,7 +59,7 @@ fuzzy_check {
                mime_types = ["application/*"];
                read_only = false;
                skip_unknown = true;
-               ${SETTINGS_FUZZY_CHECK}
+${SETTINGS_FUZZY_CHECK}
                fuzzy_map = {
                        ${FLAG1_SYMBOL} {
                                max_score = 10.0;

diff --git a/test/functional/configs/fuzzy_master_worker.conf b/test/functional/configs/fuzzy_master_worker.conf
new file mode 100644 (file)
index 0000000..0424afc
--- /dev/null
+++ b/test/functional/configs/fuzzy_master_worker.conf
@@ -0,0 +1,12 @@
+sync_keypair {
+       pubkey = "kmtnjffjzq9bq8qe14ii4gb4e1ihuq84ubfibe79ba85ob5nr3my";
+       privkey = "ea6etw8a47e66ypc38mmdnrwskx7nj86gqydwo7jrmfsxeryny1y";
+       id = "6jxoadfy8gr6dq1nyafoa5ym3391sioz3fbbq5qug7g9dchmgjr4w4kxjzqkkc9rzd7eftngyzwau9ooqe9g96bxiwq6c3xo8965gud";
+       encoding = "base32";
+       algorithm = "curve25519";
+}
+slave {
+       name = "slave1";
+        hosts = "${LOCAL_ADDR}:${PORT_FUZZY_SLAVE}";
+       key = "caen11q4eczdqgsb8u55zmmoh4z5qxd61y1rph7pz1fcubmfccdy";
+}

diff --git a/test/functional/configs/fuzzy_slave_worker.conf b/test/functional/configs/fuzzy_slave_worker.conf
new file mode 100644 (file)
index 0000000..44ed1de
--- /dev/null
+++ b/test/functional/configs/fuzzy_slave_worker.conf
@@ -0,0 +1,11 @@
+sync_keypair {
+       pubkey = "caen11q4eczdqgsb8u55zmmoh4z5qxd61y1rph7pz1fcubmfccdy";
+       privkey = "oi4k9g9prsqmk19xio7itaobmbz7yonktren1j5qwni1t88mwfty";
+       id = "qdwzxmoxpno4ge68ba1ai35n65q7mrqdp8dpp33h8haxy61omt881mep4944w1n6izuk64ktypqymrq55zkx3qob1ayfz9hxjtxbmxb";
+       encoding = "base32";
+       algorithm = "curve25519";
+       type = "kex";
+}
+
+masters = "${LOCAL_ADDR}";
+master_key = "kmtnjffjzq9bq8qe14ii4gb4e1ihuq84ubfibe79ba85ob5nr3my";

diff --git a/test/functional/lib/rspamd.py b/test/functional/lib/rspamd.py
index 31e79b095e59307776b3cd8d9c71512716abc7cc..89e79aec879af863d50eb17b764059835ceb6a2c 100644 (file)
--- a/test/functional/lib/rspamd.py
+++ b/test/functional/lib/rspamd.py
@@ -72,6 +72,9 @@ def HTTP(method, host, port, path, data=None, headers={}):
 def make_temporary_directory():
     return tempfile.mkdtemp()
 
+def make_temporary_file():
+    return tempfile.mktemp()
+
 def path_splitter(path):
     dirname = os.path.dirname(path)
     basename = os.path.basename(path)

diff --git a/test/functional/lib/rspamd.robot b/test/functional/lib/rspamd.robot
index 438aeb1bc732bd276c566bde25b7ce08239e9f8e..004df3f4b1c1f4c60b8df681029d20c74837c6c2 100644 (file)
--- a/test/functional/lib/rspamd.robot
+++ b/test/functional/lib/rspamd.robot
@@ -124,7 +124,11 @@ Scan Message With Rspamc
   [Return]  ${result}
 
 Sync Fuzzy Storage
-  ${result} =  Run Process  ${RSPAMADM}  control  -s  ${TMPDIR}/rspamd.sock  fuzzy_sync
+  [Arguments]  @{vargs}
+  ${len} =  Get Length  ${vargs}
+  ${result} =  Run Keyword If  $len == 0  Run Process  ${RSPAMADM}  control  -s  ${TMPDIR}/rspamd.sock  fuzzy_sync
+  ...  ELSE  Run Process  ${RSPAMADM}  control  -s  @{vargs}[0]/rspamd.sock  fuzzy_sync
   Log  ${result.stdout}
-  Follow Rspamd Log
+  Run Keyword If  $len == 0  Follow Rspamd Log
+  ...  ELSE  Custom Follow Rspamd Log  @{vargs}[0]/rspamd.log  @{vargs}[1]  @{vargs}[2]  @{vargs}[3] 
   Sleep  0.005s  Try give fuzzy storage time to sync

diff --git a/test/functional/lib/vars.py b/test/functional/lib/vars.py
index f260dc1b4dcadf892a5dcf6bc4985ecdec1920ee..917bde8424fb4eae0fcfd900ed840bdc54370a82 100644 (file)
--- a/test/functional/lib/vars.py
+++ b/test/functional/lib/vars.py
@@ -2,8 +2,11 @@ KEY_PVT1 = 'ekd3x36tfa5gd76t6pa8hqif3ott7n1siuux68exbkk7ukscte9y'
 KEY_PUB1 = 'm8kneubpcjsb8sbsoj7jy7azj9fdd3xmj63txni86a8ye9ncomny'
 LOCAL_ADDR = '127.0.0.1'
 PORT_CONTROLLER = 56790
+PORT_CONTROLLER_SLAVE = 56793
 PORT_FUZZY = 56791
+PORT_FUZZY_SLAVE = 56792
 PORT_NORMAL = 56789
+PORT_NORMAL_SLAVE = 56794
 PORT_PROXY = 56795
 REDIS_ADDR = '127.0.0.1'
 REDIS_PORT = 56379