Merge remote-tracking branch 'upstream/master'

22 files changed, 141 insertions, 278 deletions

diff --git a/test/functional/cases/100_general.robot b/test/functional/cases/100_general.robot
index 0e5a89c15..868c31d66 100644
--- a/test/functional/cases/100_general.robot
+++ b/test/functional/cases/100_general.robot
@@ -23,23 +23,19 @@ GTUBE - Encrypted
 
 GTUBE - Scan File feature
   ${result} =  Scan File  ${LOCAL_ADDR}  ${PORT_NORMAL}  ${GTUBE}
-  Follow Rspamd Log
   Should Contain  ${result}  GTUBE
 
 GTUBE - Scan File feature (encoded)
   ${encoded} =  Encode Filename  ${GTUBE}
   ${result} =  Scan File  ${LOCAL_ADDR}  ${PORT_NORMAL}  ${encoded}
-  Follow Rspamd Log
   Should Contain  ${result}  GTUBE
 
 GTUBE - SPAMC
   ${result} =  Spamc  ${LOCAL_ADDR}  ${PORT_NORMAL}  ${GTUBE}
-  Follow Rspamd Log
   Should Contain  ${result}  GTUBE
 
 GTUBE - RSPAMC
   ${result} =  Rspamc  ${LOCAL_ADDR}  ${PORT_NORMAL}  ${GTUBE}
-  Follow Rspamd Log
   Should Contain  ${result}  GTUBE
 
 # Broken
@@ -51,6 +47,5 @@ GTUBE - RSPAMC
 
 EMAILS DETECTION ZEROFONT
   ${result} =  Scan File  ${LOCAL_ADDR}  ${PORT_NORMAL}  ${TESTDIR}/messages/zerofont.eml
-  Follow Rspamd Log
   Should Contain  ${result}  MANY_INVISIBLE_PARTS
-  Should Contain  ${result}  ZERO_FONT
\ No newline at end of file
+  Should Contain  ${result}  ZERO_FONT
diff --git a/test/functional/cases/101_lua.robot b/test/functional/cases/101_lua.robot
index da5f8420b..8abb99b29 100644
--- a/test/functional/cases/101_lua.robot
+++ b/test/functional/cases/101_lua.robot
@@ -17,7 +17,6 @@ ${URL_TLD}      ${TESTDIR}/../lua/unit/test_tld.dat
 Flags
   [Setup]  Lua Setup  ${TESTDIR}/lua/flags.lua
   ${result} =  Scan Message With Rspamc  ${MESSAGE}
-  Follow Rspamd Log
   ${result} =  Run Rspamc  -h  ${LOCAL_ADDR}:${PORT_CONTROLLER}  stat
   Should Contain  ${result.stdout}  Messages scanned: 0
 
diff --git a/test/functional/cases/120_fuzzy/lib.robot b/test/functional/cases/120_fuzzy/lib.robot
index 319e5a900..54eaff0a8 100644
--- a/test/functional/cases/120_fuzzy/lib.robot
+++ b/test/functional/cases/120_fuzzy/lib.robot
@@ -21,7 +21,6 @@ ${SETTINGS_FUZZY_CHECK}  ${EMPTY}
 
 *** Keywords ***
 Fuzzy Skip Add Test Base
-  Create File  ${TMPDIR}/skip_hash.map
   [Arguments]  ${message}
   Set Suite Variable  ${RSPAMD_FUZZY_ADD_${message}}  0
   ${result} =  Run Rspamc  -h  ${LOCAL_ADDR}:${PORT_CONTROLLER}  -w  10  -f
@@ -31,7 +30,8 @@ Fuzzy Skip Add Test Base
   ${result} =  Scan Message With Rspamc  ${message}
   Create File  ${TMPDIR}/test.map
   Should Contain  ${result.stdout}  R_TEST_FUZZY_DENIED
-  Append To File  ${TMPDIR}/skip_hash.map  670cfcba72a87bab689958a8af5c22593dc17c907836c7c26a74d1bb49add25adfa45a5f172e3af82c9c638e8eb5fc860c22c7e966e61a459165ef0b9e1acc89
+  Append To File  ${TMPDIR}/skip_hash.map.tmp  670cfcba72a87bab689958a8af5c22593dc17c907836c7c26a74d1bb49add25adfa45a5f172e3af82c9c638e8eb5fc860c22c7e966e61a459165ef0b9e1acc89
+  Hard Link  ${TMPDIR}/skip_hash.map.tmp  ${TMPDIR}/skip_hash.map
   ${result} =  Scan Message With Rspamc  ${message}
   Check Rspamc  ${result}  R_TEST_FUZZY_DENIED  inverse=1
 
@@ -54,7 +54,6 @@ Fuzzy Delete Test
   Check Rspamc  ${result}
   Sync Fuzzy Storage
   ${result} =  Scan Message With Rspamc  ${message}
-  Follow Rspamd Log
   Should Not Contain  ${result.stdout}  ${FLAG1_SYMBOL}
   Should Be Equal As Integers  ${result.rc}  0
 
@@ -83,7 +82,6 @@ Fuzzy Overwrite Test
   END
   Sync Fuzzy Storage
   ${result} =  Scan Message With Rspamc  ${message}
-  Follow Rspamd Log
   Should Not Contain  ${result.stdout}  ${FLAG1_SYMBOL}
   Should Contain  ${result.stdout}  ${FLAG2_SYMBOL}
   Should Be Equal As Integers  ${result.rc}  0
diff --git a/test/functional/cases/131_dkim_signing/006_milter.robot b/test/functional/cases/131_dkim_signing/006_milter.robot
index fa6f532b4..cd48a62b7 100644
--- a/test/functional/cases/131_dkim_signing/006_milter.robot
+++ b/test/functional/cases/131_dkim_signing/006_milter.robot
@@ -25,7 +25,6 @@ Milter Test
   [Arguments]  ${mtlua}
   ${result} =  Run Process  miltertest  -Dport\=${PORT_PROXY}  -Dhost\=${LOCAL_ADDR}  -s  ${TESTDIR}/lua/miltertest/${mtlua}
   ...  cwd=${TESTDIR}/lua/miltertest
-  Follow Rspamd Log
   Should Match Regexp  ${result.stderr}  ^$
   Log  ${result.rc}
   Log  ${result.stdout}
diff --git a/test/functional/cases/140_proxy.robot b/test/functional/cases/140_proxy.robot
index 58b80461a..fabc5a4fd 100644
--- a/test/functional/cases/140_proxy.robot
+++ b/test/functional/cases/140_proxy.robot
@@ -13,32 +13,24 @@ ${URL_TLD}      ${TESTDIR}/../lua/unit/test_tld.dat
 *** Test Cases ***
 Rspamc Client
   ${result} =  Run Rspamc  -h  ${LOCAL_ADDR}:${PORT_PROXY}  -p  ${MESSAGE}
-  Custom Follow Rspamd Log  ${PROXY_TMPDIR}/rspamd.log  ${PROXY_LOGPOS}  PROXY_LOGPOS  Suite
-  Custom Follow Rspamd Log  ${SLAVE_TMPDIR}/rspamd.log  ${SLAVE_LOGPOS}  SLAVE_LOGPOS  Suite
   Run Keyword If  ${result.rc} != 0  Log  ${result.stderr}
   Should Contain  ${result.stdout}  SIMPLE_TEST
   Should Be Equal As Integers  ${result.rc}  0
 
 SPAMC
   ${result} =  Spamc  ${LOCAL_ADDR}  ${PORT_PROXY}  ${MESSAGE}
-  Custom Follow Rspamd Log  ${PROXY_TMPDIR}/rspamd.log  ${PROXY_LOGPOS}  PROXY_LOGPOS  Suite
-  Custom Follow Rspamd Log  ${SLAVE_TMPDIR}/rspamd.log  ${SLAVE_LOGPOS}  SLAVE_LOGPOS  Suite
   Should Contain  ${result}  SPAMD/1.1 0 EX_OK
 
 RSPAMC Legacy Protocol
   ${result} =  Rspamc  ${LOCAL_ADDR}  ${PORT_PROXY}  ${MESSAGE}
-  Custom Follow Rspamd Log  ${PROXY_TMPDIR}/rspamd.log  ${PROXY_LOGPOS}  PROXY_LOGPOS  Suite
-  Custom Follow Rspamd Log  ${SLAVE_TMPDIR}/rspamd.log  ${SLAVE_LOGPOS}  SLAVE_LOGPOS  Suite
   Should Contain  ${result}  RSPAMD/1.3 0 EX_OK
 
 *** Keywords ***
 Proxy Setup
   &{d} =  Run Rspamd  CONFIG=${TESTDIR}/configs/lua_test.conf
-  Set Suite Variable  ${SLAVE_LOGPOS}  ${d}[RSPAMD_LOGPOS]
   Set Suite Variable  ${SLAVE_PID}  ${d}[RSPAMD_PID]
   Set Suite Variable  ${SLAVE_TMPDIR}  ${d}[TMPDIR]
   &{d} =  Run Rspamd  CONFIG=${TESTDIR}/configs/proxy.conf
-  Set Suite Variable  ${PROXY_LOGPOS}  ${d}[RSPAMD_LOGPOS]
   Set Suite Variable  ${PROXY_PID}  ${d}[RSPAMD_PID]
   Set Suite Variable  ${PROXY_TMPDIR}  ${d}[TMPDIR]
 
diff --git a/test/functional/cases/180_milter.robot b/test/functional/cases/180_milter.robot
index 23a366814..ddbbf9799 100644
--- a/test/functional/cases/180_milter.robot
+++ b/test/functional/cases/180_milter.robot
@@ -34,7 +34,6 @@ Milter Test
   [Arguments]  ${mtlua}
   ${result} =  Run Process  miltertest  -Dport\=${PORT_PROXY}  -Dhost\=${LOCAL_ADDR}  -s  ${TESTDIR}/lua/miltertest/${mtlua}
   ...  cwd=${TESTDIR}/lua/miltertest
-  Follow Rspamd Log
   Should Match Regexp  ${result.stderr}  ^$
   Log  ${result.rc}
   Log  ${result.stdout}
diff --git a/test/functional/cases/220_http.robot b/test/functional/cases/220_http.robot
index c30fbba9c..86e882d5d 100644
--- a/test/functional/cases/220_http.robot
+++ b/test/functional/cases/220_http.robot
@@ -68,5 +68,6 @@ Run Dummy Https
 Check url
   [Arguments]  ${url}  ${method}  @{expect_results}
   ${result} =  Scan Message With Rspamc  --header=url:${url}  --header=method:${method}  ${MESSAGE}
-  : FOR  ${expect}  IN  @{expect_results}
-  \  Check Rspamc  ${result}  ${expect}
\ No newline at end of file
+  FOR  ${expect}  IN  @{expect_results}
+    Check Rspamc  ${result}  ${expect}
+  END
diff --git a/test/functional/cases/230_tcp.robot b/test/functional/cases/230_tcp.robot
index 51f590ea4..16d490826 100644
--- a/test/functional/cases/230_tcp.robot
+++ b/test/functional/cases/230_tcp.robot
@@ -75,5 +75,6 @@ Run Dummy Ssl
 Check url
   [Arguments]  ${url}  ${method}  @{expect_results}
   ${result} =  Scan Message With Rspamc  --header=url:${url}  --header=method:${method}  ${MESSAGE}
-  : FOR  ${expect}  IN  @{expect_results}
-  \  Check Rspamc  ${result}  ${expect}
+  FOR  ${expect}  IN  @{expect_results}
+    Check Rspamc  ${result}  ${expect}
+  END
diff --git a/test/functional/cases/231_tcp_down.robot b/test/functional/cases/231_tcp_down.robot
index 10043024f..e68e2053a 100644
--- a/test/functional/cases/231_tcp_down.robot
+++ b/test/functional/cases/231_tcp_down.robot
@@ -37,5 +37,6 @@ Http Teardown
 Check url
   [Arguments]  ${url}  ${method}  @{expect_results}
   ${result} =  Scan Message With Rspamc  --header=url:${url}  --header=method:${method}  ${MESSAGE}
-  : FOR  ${expect}  IN  @{expect_results}
-  \  Check Rspamc  ${result}  ${expect}
+  FOR  ${expect}  IN  @{expect_results}
+    Check Rspamc  ${result}  ${expect}
+  END
diff --git a/test/functional/cases/330_neural.robot b/test/functional/cases/330_neural.robot
index c64e9cc0a..89ba19f11 100644
--- a/test/functional/cases/330_neural.robot
+++ b/test/functional/cases/330_neural.robot
@@ -16,11 +16,12 @@ ${RSPAMD_SCOPE}  Suite
 *** Test Cases ***
 Train
   Sleep  2s  Wait for redis mess
-  : FOR    ${INDEX}    IN RANGE    0    10
-  \  ${result} =  Scan Message With Rspamc  ${MESSAGE}  --header  Settings={symbols_enabled = ["SPAM_SYMBOL"]}
-  \  Check Rspamc  ${result}  SPAM_SYMBOL (1.00)
-  \  ${result} =  Scan Message With Rspamc  ${MESSAGE}  --header  Settings={symbols_enabled = ["HAM_SYMBOL"]}
-  \  Check Rspamc  ${result}  HAM_SYMBOL (-1.00)
+  FOR    ${INDEX}    IN RANGE    0    10
+    ${result} =  Scan Message With Rspamc  ${MESSAGE}  --header  Settings={symbols_enabled = ["SPAM_SYMBOL"]}
+    Check Rspamc  ${result}  SPAM_SYMBOL (1.00)
+    ${result} =  Scan Message With Rspamc  ${MESSAGE}  --header  Settings={symbols_enabled = ["HAM_SYMBOL"]}
+    Check Rspamc  ${result}  HAM_SYMBOL (-1.00)
+  END
 
 Check Neural HAM
   Sleep  2s  Wait for neural to be loaded
@@ -35,11 +36,12 @@ Check Neural SPAM
 
 
 Train INVERSE
-  : FOR    ${INDEX}    IN RANGE    0    10
-  \  ${result} =  Scan Message With Rspamc  ${MESSAGE}  --header  Settings={symbols_enabled = ["SPAM_SYMBOL"]; SPAM_SYMBOL = -1}
-  \  Check Rspamc  ${result}  SPAM_SYMBOL (-1.00)
-  \  ${result} =  Scan Message With Rspamc  ${MESSAGE}  --header  Settings={symbols_enabled = ["HAM_SYMBOL"]; HAM_SYMBOL = 1;}
-  \  Check Rspamc  ${result}  HAM_SYMBOL (1.00)
+  FOR    ${INDEX}    IN RANGE    0    10
+    ${result} =  Scan Message With Rspamc  ${MESSAGE}  --header  Settings={symbols_enabled = ["SPAM_SYMBOL"]; SPAM_SYMBOL = -1}
+    Check Rspamc  ${result}  SPAM_SYMBOL (-1.00)
+    ${result} =  Scan Message With Rspamc  ${MESSAGE}  --header  Settings={symbols_enabled = ["HAM_SYMBOL"]; HAM_SYMBOL = 1;}
+    Check Rspamc  ${result}  HAM_SYMBOL (1.00)
+  END
 
 Check Neural HAM INVERSE
   Sleep  2s  Wait for neural to be loaded
diff --git a/test/functional/cases/350_magic.robot b/test/functional/cases/350_magic.robot
index b3a760086..787e4fe3c 100644
--- a/test/functional/cases/350_magic.robot
+++ b/test/functional/cases/350_magic.robot
@@ -14,7 +14,6 @@ ${URL_TLD}      ${TESTDIR}/../lua/unit/test_tld.dat
 *** Test Cases ***
 Magic detections bundle 1
   ${result} =  Scan Message With Rspamc  ${TESTDIR}/messages/gargantua.eml
-  Follow Rspamd Log
   Should Contain  ${result.stdout}  MAGIC_SYM_ZIP_2
   Should Contain  ${result.stdout}  MAGIC_SYM_RAR_3
   Should Contain  ${result.stdout}  MAGIC_SYM_EXE_4
diff --git a/test/functional/lib/rspamd.py b/test/functional/lib/rspamd.py
index 295484f9e..b8bc6ac03 100644
--- a/test/functional/lib/rspamd.py
+++ b/test/functional/lib/rspamd.py
@@ -1,32 +1,24 @@
+from urllib.request import urlopen
+import glob
 import grp
+import http.client
 import os
 import os.path
 import psutil
-import glob
 import pwd
 import shutil
 import signal
 import socket
+import stat
 import sys
 import tempfile
-import json
-import stat
-from robot.libraries.BuiltIn import BuiltIn
-from robot.api import logger
 
-if sys.version_info > (3,):
-    long = int
-try:
-    from urllib.request import urlopen
-except:
-    from urllib2 import urlopen
-try:
-    import http.client as httplib
-except:
-    import httplib
+from robot.api import logger
+from robot.libraries.BuiltIn import BuiltIn
+import demjson
 
 def Check_JSON(j):
-    d = json.loads(j, strict=True)
+    d = demjson.decode(j, strict=True)
     logger.debug('got json %s' % d)
     assert len(d) > 0
     assert 'error' not in d
@@ -99,7 +91,7 @@ def get_rspamadm():
     return dname + "/src/rspamadm/rspamadm"
 
 def HTTP(method, host, port, path, data=None, headers={}):
-    c = httplib.HTTPConnection("%s:%s" % (host, port))
+    c = http.client.HTTPConnection("%s:%s" % (host, port))
     c.request(method, path, data, headers)
     r = c.getresponse()
     t = r.read()
@@ -107,6 +99,9 @@ def HTTP(method, host, port, path, data=None, headers={}):
     c.close()
     return [s, t]
 
+def hard_link(src, dst):
+    os.link(src, dst)
+
 def make_temporary_directory():
     """Creates and returns a unique temporary directory
 
@@ -131,14 +126,6 @@ def path_splitter(path):
     basename = os.path.basename(path)
     return [dirname, basename]
 
-def read_log_from_position(filename, offset):
-    offset = long(offset)
-    with open(filename, 'rb') as f:
-        f.seek(offset)
-        goo = f.read()
-        size = len(goo)
-    return [goo, size+offset]
-
 def rspamc(addr, port, filename):
     mboxgoo = b"From MAILER-DAEMON Fri May 13 19:17:40 2016\r\n"
     goo = open(filename, 'rb').read()
@@ -251,13 +238,15 @@ def shutdown_process_with_children(pid):
             pass
 
 def write_to_stdin(process_handle, text):
+    if not isinstance(text, bytes):
+        text = bytes(text, 'utf-8')
     lib = BuiltIn().get_library_instance('Process')
     obj = lib.get_process_object()
-    obj.stdin.write(text + "\n")
+    obj.stdin.write(text + b"\n")
     obj.stdin.flush()
     obj.stdin.close()
     out = obj.stdout.read(4096)
-    return out
+    return out.decode('utf-8')
 
 def get_file_if_exists(file_path):
     if os.path.exists(file_path):
@@ -265,72 +254,6 @@ def get_file_if_exists(file_path):
             return myfile.read()
     return None
 
-# copy-paste from
-# https://hg.python.org/cpython/file/6860263c05b3/Lib/shutil.py#l1068
-# As soon as we move to Python 3, this should be removed in favor of shutil.which()
-def python3_which(cmd, mode=os.F_OK | os.X_OK, path=None):
-    """Given a command, mode, and a PATH string, return the path which
-    conforms to the given mode on the PATH, or None if there is no such
-    file.
-
-    `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result
-    of os.environ.get("PATH"), or can be overridden with a custom search
-    path.
-    """
-
-    # Check that a given file can be accessed with the correct mode.
-    # Additionally check that `file` is not a directory, as on Windows
-    # directories pass the os.access check.
-    def _access_check(fn, mode):
-        return (os.path.exists(fn) and os.access(fn, mode)
-                and not os.path.isdir(fn))
-
-    # If we're given a path with a directory part, look it up directly rather
-    # than referring to PATH directories. This includes checking relative to the
-    # current directory, e.g. ./script
-    if os.path.dirname(cmd):
-        if _access_check(cmd, mode):
-            return cmd
-        return None
-
-    if path is None:
-        path = os.environ.get("PATH", os.defpath)
-    if not path:
-        return None
-    path = path.split(os.pathsep)
-
-    if sys.platform == "win32":
-        # The current directory takes precedence on Windows.
-        if not os.curdir in path:
-            path.insert(0, os.curdir)
-
-        # PATHEXT is necessary to check on Windows.
-        pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
-        # See if the given file matches any of the expected path extensions.
-        # This will allow us to short circuit when given "python.exe".
-        # If it does match, only test that one, otherwise we have to try
-        # others.
-        if any(cmd.lower().endswith(ext.lower()) for ext in pathext):
-            files = [cmd]
-        else:
-            files = [cmd + ext for ext in pathext]
-    else:
-        # On other platforms you don't have things like PATHEXT to tell you
-        # what file suffixes are executable, so just pass on cmd as-is.
-        files = [cmd]
-
-    seen = set()
-    for dir in path:
-        normdir = os.path.normcase(dir)
-        if not normdir in seen:
-            seen.add(normdir)
-            for thefile in files:
-                name = os.path.join(dir, thefile)
-                if _access_check(name, mode):
-                    return name
-    return None
-
-
 def _merge_luacov_stats(statsfile, coverage):
     """
     Reads a coverage stats file written by luacov and merges coverage data to
@@ -339,7 +262,7 @@ def _merge_luacov_stats(statsfile, coverage):
     Format of the file defined in:
     https://github.com/keplerproject/luacov/blob/master/src/luacov/stats.lua
     """
-    with open(statsfile, 'rb') as fh:
+    with open(statsfile, 'r') as fh:
         while True:
             # max_line:filename
             line = fh.readline().rstrip()
@@ -369,7 +292,7 @@ def _dump_luacov_stats(statsfile, coverage):
     """
     src_files = sorted(coverage)
 
-    with open(statsfile, 'wb') as fh:
+    with open(statsfile, 'w') as fh:
         for src in src_files:
             stats = " ".join(str(n) for n in coverage[src])
             fh.write("%s:%s\n%s\n" % (len(coverage[src]), src, stats))
diff --git a/test/functional/lib/rspamd.robot b/test/functional/lib/rspamd.robot
index 03abc86c1..cd68b1d86 100644
--- a/test/functional/lib/rspamd.robot
+++ b/test/functional/lib/rspamd.robot
@@ -17,7 +17,6 @@ Check Pidfile
 
 Check Rspamc
   [Arguments]  ${result}  @{args}  &{kwargs}
-  Follow Rspamd Log
   Run Keyword If  ${result.rc} != 0  Log  ${result.stderr}
   ${has_rc} =  Evaluate  'rc' in $kwargs
   ${inverse} =  Evaluate  'inverse' in $kwargs
@@ -45,19 +44,6 @@ Check Rspamc Match String
   Run Keyword If  ${inverse} == False  Should Contain  ${subject}  ${str}
   ...  ELSE  Should Not Contain  ${subject}  ${str}
 
-Custom Follow Rspamd Log
-  [Arguments]  ${logfile}  ${logpos}  ${logpos_var}  ${scope}
-  ${logpos} =  Log Logs  ${logfile}  ${logpos}
-  Run Keyword If  '${scope}' == 'Test'  Set Test Variable  ${${logpos_var}}  ${logpos}
-  ...  ELSE IF  '${scope}' == 'Suite'  Set Suite Variable  ${${logpos_var}}  ${logpos}
-  ...  ELSE  Fail  'scope must be Test or Suite'
-
-Follow Rspamd Log
-  ${RSPAMD_LOGPOS} =  Log Logs  ${TMPDIR}/rspamd.log  ${RSPAMD_LOGPOS}
-  Run Keyword If  '${RSPAMD_SCOPE}' == 'Test'  Set Test Variable  ${RSPAMD_LOGPOS}
-  ...  ELSE IF  '${RSPAMD_SCOPE}' == 'Suite'  Set Suite Variable  ${RSPAMD_LOGPOS}
-  ...  ELSE  Fail  'RSPAMD_SCOPE must be Test or Suite'
-
 Generic Setup
   [Arguments]  @{vargs}  &{kwargs}
   &{d} =  Run Rspamd  @{vargs}  &{kwargs}
@@ -69,7 +55,6 @@ Generic Setup
   END
 
 Generic Teardown
-  [Arguments]  @{ports}
   Run Keyword If  '${CONTROLLER_ERRORS}' == 'True'  Check Controller Errors
   Shutdown Process With Children  ${RSPAMD_PID}
   Log does not contain segfault record
@@ -78,20 +63,11 @@ Generic Teardown
   Cleanup Temporary Directory  ${TMPDIR}
 
 Log does not contain segfault record
-  ${log} =  Get File  ${TMPDIR}/rspamd.log
+  ${log} =  Get File  ${TMPDIR}/rspamd.log  encoding_errors=ignore
   Should not contain  ${log}  Segmentation fault:  msg=Segmentation fault detected
 
-Log Logs
-  [Arguments]  ${logfile}  ${position}
-  ${the_log}  ${position} =  Read Log From Position  ${logfile}  ${position}
-  Log  ${the_log}
-  [Return]  ${position}
-
 Normal Teardown
-  ${port_normal} =  Create List  ${SOCK_STREAM}  ${LOCAL_ADDR}  ${PORT_NORMAL}
-  ${port_controller} =  Create List  ${SOCK_STREAM}  ${LOCAL_ADDR}  ${PORT_CONTROLLER}
-  ${ports} =  Create List  ${port_normal}  ${port_controller}
-  Generic Teardown  @{ports}
+  Generic Teardown
 
 Redis HSET
   [Arguments]  ${hash}  ${key}  ${value}
@@ -169,24 +145,21 @@ Run Rspamd
   ${result} =  Run Process  ${RSPAMD}  -u  ${RSPAMD_USER}  -g  ${RSPAMD_GROUP}
   ...  -c  ${tmpdir}/rspamd.conf  env:TMPDIR=${tmpdir}  env:DBDIR=${tmpdir}  env:LD_LIBRARY_PATH=${TESTDIR}/../../contrib/aho-corasick
   Run Keyword If  ${result.rc} != 0  Log  ${result.stderr}
-  ${rspamd_logpos} =  Log Logs  ${tmpdir}/rspamd.log  0
   Should Be Equal As Integers  ${result.rc}  0
   Wait Until Keyword Succeeds  10x  1 sec  Check Pidfile  ${tmpdir}/rspamd.pid  timeout=0.5s
   Wait Until Keyword Succeeds  5x  1 sec  Ping Rspamd  ${LOCAL_ADDR}  ${PORT_NORMAL}
   ${rspamd_pid} =  Get File  ${tmpdir}/rspamd.pid
-  Set To Dictionary  ${d}  RSPAMD_LOGPOS=${rspamd_logpos}  RSPAMD_PID=${rspamd_pid}  TMPDIR=${tmpdir}
+  Set To Dictionary  ${d}  RSPAMD_PID=${rspamd_pid}  TMPDIR=${tmpdir}
   [Return]  &{d}
 
+Simple Teardown
+  Generic Teardown
+
 Scan Message With Rspamc
   [Arguments]  ${msg_file}  @{vargs}
   ${result} =  Run Rspamc  -p  -h  ${LOCAL_ADDR}:${PORT_NORMAL}  @{vargs}  ${msg_file}
   [Return]  ${result}
 
-Simple Teardown
-  ${port_normal} =  Create List  ${SOCK_STREAM}  ${LOCAL_ADDR}  ${PORT_NORMAL}
-  ${ports} =  Create List  ${port_normal}
-  Generic Teardown  @{ports}
-
 Sync Fuzzy Storage
   [Arguments]  @{vargs}
   ${len} =  Get Length  ${vargs}
@@ -195,6 +168,4 @@ Sync Fuzzy Storage
   ...  ELSE  Run Process  ${RSPAMADM}  control  -s  ${vargs}[0]/rspamd.sock
   ...  fuzzy_sync
   Log  ${result.stdout}
-  Run Keyword If  $len == 0  Follow Rspamd Log
-  ...  ELSE  Custom Follow Rspamd Log  ${vargs}[0]/rspamd.log  ${vargs}[1]  ${vargs}[2]  ${vargs}[3]
   Sleep  0.1s  Try give fuzzy storage time to sync
diff --git a/test/functional/lib/vars.py b/test/functional/lib/vars.py
index d29c31c7b..3cec9767a 100644
--- a/test/functional/lib/vars.py
+++ b/test/functional/lib/vars.py
@@ -3,7 +3,7 @@ import socket
 CONTROLLER_ERRORS = True
 KEY_PVT1 = 'ekd3x36tfa5gd76t6pa8hqif3ott7n1siuux68exbkk7ukscte9y'
 KEY_PUB1 = 'm8kneubpcjsb8sbsoj7jy7azj9fdd3xmj63txni86a8ye9ncomny'
-LOCAL_ADDR = u'127.0.0.1'
+LOCAL_ADDR = '127.0.0.1'
 MAP_WATCH_INTERVAL = '1min'
 PORT_CONTROLLER = 56790
 PORT_CONTROLLER_SLAVE = 56793
@@ -17,9 +17,9 @@ PORT_FPROT = 56797
 PORT_FPROT2_DUPLICATE = 56798
 PORT_AVAST = 56799
 P0F_SOCKET = '/tmp/p0f.sock'
-REDIS_ADDR = u'127.0.0.1'
+REDIS_ADDR = '127.0.0.1'
 REDIS_PORT = 56379
-NGINX_ADDR = u'127.0.0.1'
+NGINX_ADDR = '127.0.0.1'
 NGINX_PORT = 56380
 RSPAMD_GROUP = 'nogroup'
 RSPAMD_USER = 'nobody'
diff --git a/test/functional/util/dummy_avast.py b/test/functional/util/dummy_avast.py
index 55945d8dc..a4a6c62a2 100755
--- a/test/functional/util/dummy_avast.py
+++ b/test/functional/util/dummy_avast.py
@@ -1,15 +1,13 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 PID = "/tmp/dummy_avast.pid"
 
 import os
-import sys
 import socket
+import socketserver
+import sys
+
 import dummy_killer
-try:
-    import SocketServer as socketserver
-except:
-    import socketserver
 
 class MyTCPHandler(socketserver.BaseRequestHandler):
 
@@ -50,6 +48,6 @@ if __name__ == "__main__":
     try:
         server.handle_request()
     except socket.error:
-        print "Socket closed"
+        print("Socket closed")
 
     server.server_close()
diff --git a/test/functional/util/dummy_clam.py b/test/functional/util/dummy_clam.py
index d25a63525..1b614f58b 100755
--- a/test/functional/util/dummy_clam.py
+++ b/test/functional/util/dummy_clam.py
@@ -1,15 +1,13 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 PID = "/tmp/dummy_clamav.pid"
 
 import os
-import sys
 import socket
+import socketserver
+import sys
+
 import dummy_killer
-try:
-    import SocketServer as socketserver
-except:
-    import socketserver
 
 class MyTCPHandler(socketserver.BaseRequestHandler):
 
@@ -47,6 +45,6 @@ if __name__ == "__main__":
     try:
         server.handle_request()
     except socket.error:
-        print "Socket closed"
+        print("Socket closed")
 
     server.server_close()
diff --git a/test/functional/util/dummy_fprot.py b/test/functional/util/dummy_fprot.py
index e6dc58bdd..31ae2c487 100755
--- a/test/functional/util/dummy_fprot.py
+++ b/test/functional/util/dummy_fprot.py
@@ -1,15 +1,12 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
+
 import os
-import sys
 import signal
 import socket
-import dummy_killer
-
+import socketserver
+import sys
 
-try:
-    import SocketServer as socketserver
-except:
-    import socketserver
+import dummy_killer
 
 PID = "/tmp/dummy_fprot.pid"
 
@@ -53,5 +50,5 @@ if __name__ == "__main__":
     try:
         server.handle_request()
     except socket.error:
-        print "Socket closed"
+        print("Socket closed")
     server.server_close()
diff --git a/test/functional/util/dummy_http.py b/test/functional/util/dummy_http.py
index 614fced29..c05ee2d02 100755
--- a/test/functional/util/dummy_http.py
+++ b/test/functional/util/dummy_http.py
@@ -1,14 +1,13 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
-import BaseHTTPServer
-import SocketServer
-import SimpleHTTPServer
-import dummy_killer
-
-import time
+import http.server
 import os
-import sys
 import socket
+import socketserver
+import sys
+import time
+
+import dummy_killer
 
 PORT = 18080
 HOST_NAME = '127.0.0.1'
@@ -16,10 +15,10 @@ HOST_NAME = '127.0.0.1'
 PID = "/tmp/dummy_http.pid"
 
 
-class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+class MyHandler(http.server.BaseHTTPRequestHandler):
 
     def setup(self):
-        BaseHTTPServer.BaseHTTPRequestHandler.setup(self)
+        http.server.BaseHTTPRequestHandler.setup(self)
         self.protocol_version = "HTTP/1.1" # allow connection: keep-alive
 
     def do_HEAD(self):
@@ -29,7 +28,7 @@ class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
         self.log_message("to be closed: " + self.close_connection)
 
     def do_GET(self):
-        response = "hello world"
+        response = b"hello world"
 
         """Respond to a GET request."""
         if self.path == "/empty":
@@ -60,7 +59,7 @@ class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
 
 
     def do_POST(self):
-        response = "hello post"
+        response = b"hello post"
         """Respond to a GET request."""
         if self.path == "/empty":
             self.finish()
@@ -82,12 +81,12 @@ class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
         self.wfile.write(response)
 
 
-class ThreadingSimpleServer(SocketServer.ThreadingMixIn,
-                   BaseHTTPServer.HTTPServer):
+class ThreadingSimpleServer(socketserver.ThreadingMixIn,
+                   http.server.HTTPServer):
     def __init__(self):
         self.allow_reuse_address = True
         self.timeout = 1
-        BaseHTTPServer.HTTPServer.__init__(self, (HOST_NAME, PORT), MyHandler)
+        http.server.HTTPServer.__init__(self, (HOST_NAME, PORT), MyHandler)
 
     def run(self):
         dummy_killer.write_pid(PID)
@@ -96,9 +95,9 @@ class ThreadingSimpleServer(SocketServer.ThreadingMixIn,
                 sys.stdout.flush()
                 server.handle_request()
         except KeyboardInterrupt:
-            print "Interrupt"
+            print("Interrupt")
         except socket.error:
-            print "Socket closed"
+            print("Socket closed")
 
     def stop(self):
         self.keep_running = False
diff --git a/test/functional/util/dummy_https.py b/test/functional/util/dummy_https.py
index 43c5d9122..1e6753423 100755
--- a/test/functional/util/dummy_https.py
+++ b/test/functional/util/dummy_https.py
@@ -1,15 +1,15 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
-import BaseHTTPServer
-import SocketServer
-import SimpleHTTPServer
-import dummy_killer
-import ssl
-
-import time
+import http.server
+import http.server
 import os
-import sys
 import socket
+import socketserver
+import ssl
+import sys
+import time
+
+import dummy_killer
 
 PORT = 18081
 HOST_NAME = '127.0.0.1'
@@ -17,10 +17,10 @@ HOST_NAME = '127.0.0.1'
 PID = "/tmp/dummy_https.pid"
 
 
-class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+class MyHandler(http.server.BaseHTTPRequestHandler):
 
     def setup(self):
-        BaseHTTPServer.BaseHTTPRequestHandler.setup(self)
+        http.server.BaseHTTPRequestHandler.setup(self)
         self.protocol_version = "HTTP/1.1" # allow connection: keep-alive
 
     def do_HEAD(self):
@@ -30,7 +30,7 @@ class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
         self.log_message("to be closed: " + self.close_connection)
 
     def do_GET(self):
-        response = "hello world"
+        response = b"hello world"
 
         """Respond to a GET request."""
         if self.path == "/empty":
@@ -64,11 +64,11 @@ class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
         """Respond to a POST request."""
 
         content_length = int(self.headers['Content-Length'])
-        response = 'hello post'
+        response = b'hello post'
 
         if content_length > 0:
             body = self.rfile.read(content_length)
-            response = "hello post: " + str(len(body))
+            response = b"hello post: " + bytes(len(body))
 
         if self.path == "/empty":
             self.finish()
@@ -90,13 +90,13 @@ class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
         self.wfile.write(response)
 
 
-class ThreadingSimpleServer(SocketServer.ThreadingMixIn,
-                   BaseHTTPServer.HTTPServer):
+class ThreadingSimpleServer(socketserver.ThreadingMixIn,
+                   http.server.HTTPServer):
     def __init__(self, certfile,
                  keyfile,):
         self.allow_reuse_address = True
         self.timeout = 10
-        BaseHTTPServer.HTTPServer.__init__(self, (HOST_NAME, PORT), MyHandler)
+        http.server.HTTPServer.__init__(self, (HOST_NAME, PORT), MyHandler)
         self.socket = ssl.wrap_socket (self.socket,
                          keyfile=keyfile,
                          certfile=certfile, server_side=True)
@@ -108,9 +108,9 @@ class ThreadingSimpleServer(SocketServer.ThreadingMixIn,
                 sys.stdout.flush()
                 server.handle_request()
         except KeyboardInterrupt:
-            print "Interrupt"
+            print("Interrupt")
         except socket.error:
-            print "Socket closed"
+            print("Socket closed")
 
     def stop(self):
         self.keep_running = False
diff --git a/test/functional/util/dummy_p0f.py b/test/functional/util/dummy_p0f.py
index d780bdb0d..2bcd183d4 100755
--- a/test/functional/util/dummy_p0f.py
+++ b/test/functional/util/dummy_p0f.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 PID = "/tmp/dummy_p0f.pid"
 
@@ -6,11 +6,9 @@ import os
 import sys
 import struct
 import socket
+import socketserver
+
 import dummy_killer
-try:
-    import SocketServer as socketserver
-except:
-    import socketserver
 
 class MyStreamHandler(socketserver.BaseRequestHandler):
 
@@ -22,8 +20,8 @@ class MyStreamHandler(socketserver.BaseRequestHandler):
         }
 
         OS = {
-            'windows' : ('Windows', '7 or 8'),
-            'linux'   : ('Linux', '3.11 and newer')
+            'windows' : (b'Windows', b'7 or 8'),
+            'linux'   : (b'Linux', b'3.11 and newer')
         }
 
         self.data = self.request.recv(21).strip()
@@ -47,10 +45,10 @@ class MyStreamHandler(socketserver.BaseRequestHandler):
                 0,                                # os_match_q
                 OS[self.server.p0f_os][0],        # os_name
                 OS[self.server.p0f_os][1],        # os_flavor
-                '',                               # http_name
-                '',                               # http_flavor
-                'Ethernet or modem',              # link_type
-                ''                                # language
+                b'',                              # http_name
+                b'',                              # http_flavor
+                b'Ethernet or modem',             # link_type
+                b''                               # language
             )
 
         self.request.sendall(response)
@@ -61,7 +59,7 @@ def cleanup(SOCK):
         try:
             os.unlink(SOCK)
         except OSError:
-            print "Could not unlink socket: " + SOCK
+            print("Could not unlink socket: " + SOCK)
 
 if __name__ == "__main__":
     SOCK = '/tmp/p0f.sock'
@@ -92,7 +90,7 @@ if __name__ == "__main__":
     try:
         server.handle_request()
     except socket.error:
-        print "Socket closed"
+        print("Socket closed")
 
     server.server_close()
     cleanup(SOCK)
diff --git a/test/functional/util/dummy_ssl.py b/test/functional/util/dummy_ssl.py
index 725f8052a..44b178293 100755
--- a/test/functional/util/dummy_ssl.py
+++ b/test/functional/util/dummy_ssl.py
@@ -1,64 +1,58 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
-import SocketServer
-import dummy_killer
-
-import time
 import os
-import sys
 import socket
 import ssl
+import sys
+import time
+
+import dummy_killer
+import socketserver
 
 PORT = 14433
 HOST_NAME = '127.0.0.1'
 
 PID = "/tmp/dummy_ssl.pid"
 
-class SSLTCPHandler(SocketServer.BaseRequestHandler):
+class SSLTCPHandler(socketserver.StreamRequestHandler):
     def handle(self):
         time.sleep(0.5)
         data = self.request.recv(6000000)
         while data:
-            print "{} wrote:".format(self.client_address[0])
-            print data
+            print("{} wrote:".format(self.client_address[0]))
+            print(data)
             time.sleep(0.1)
-            self.request.sendall('hello\n')
+            self.request.sendall(b'hello\n')
             time.sleep(0.1)
             data = self.request.recv(6000000)
 
-class SSL_TCP_Server(SocketServer.TCPServer):
+class SSL_TCP_Server(socketserver.ThreadingMixIn, socketserver.TCPServer):
     def __init__(self,
                  server_address,
                  RequestHandlerClass,
                  certfile,
                  keyfile,
-                 ssl_version=ssl.PROTOCOL_TLSv1,
                  bind_and_activate=True):
         self.allow_reuse_address = True
-        SocketServer.TCPServer.__init__(self, server_address,
-                                        RequestHandlerClass, bind_and_activate)
-        self.certfile = certfile
-        self.keyfile = keyfile
-        self.ssl_version = ssl_version
-        #self.timeout = 1
-
-    def get_request(self):
-        newsocket, fromaddr = self.socket.accept()
-        connstream = ssl.wrap_socket(newsocket,
-                                     server_side=True,
-                                     certfile = self.certfile,
-                                     keyfile = self.keyfile,
-                                     ssl_version = self.ssl_version)
-        return connstream, fromaddr
+        super().__init__(server_address, RequestHandlerClass, False)
+        self.timeout = 1
+        ctx = ssl.create_default_context()
+        ctx.load_cert_chain(certfile=certfile)
+        ctx.check_hostname = False
+        ctx.verify_mode = ssl.CERT_NONE
+        self.socket = ctx.wrap_socket(self.socket, server_side=True)
+        if (bind_and_activate):
+            self.server_bind()
+            self.server_activate()
 
     def run(self):
         dummy_killer.write_pid(PID)
         try:
             self.serve_forever()
         except KeyboardInterrupt:
-            print "Interrupt"
+            print("Interrupt")
         except socket.error as e:
-            print "Socket closed {}".format(e)
+            print("Socket closed {}".format(e))
         finally:
             self.server_close()
 
@@ -66,9 +60,7 @@ class SSL_TCP_Server(SocketServer.TCPServer):
         self.keep_running = False
         self.server_close()
 
-class SSL_ThreadingTCPServer(SocketServer.ThreadingMixIn, SSL_TCP_Server): pass
-
 if __name__ == '__main__':
-    server = SSL_ThreadingTCPServer((HOST_NAME, PORT), SSLTCPHandler, sys.argv[1], sys.argv[1])
+    server = SSL_TCP_Server((HOST_NAME, PORT), SSLTCPHandler, sys.argv[1], sys.argv[1])
     dummy_killer.setup_killer(server, server.stop)
     server.run()
diff --git a/test/functional/util/dummy_udp.py b/test/functional/util/dummy_udp.py
index 02ee6dce8..f05d6d680 100755
--- a/test/functional/util/dummy_udp.py
+++ b/test/functional/util/dummy_udp.py
@@ -1,7 +1,8 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import socket
 import sys
+
 import dummy_killer
 
 UDP_IP = "127.0.0.1"
@@ -21,5 +22,5 @@ if __name__ == "__main__":
 
     while True:
         data, addr = sock.recvfrom(1024) # buffer size is 1024 bytes
-        print "received message:", data
-        sock.sendto(data, addr)
\ No newline at end of file
+        print("received message:", data)
+        sock.sendto(data, addr)