diff options
| author | Andrew Lewis <nerf@judo.za.org> | 2020-06-25 21:34:48 +0200 |
|---|---|---|
| committer | Andrew Lewis <nerf@judo.za.org> | 2020-06-25 21:34:48 +0200 |
| commit | aa3d11920940f91b9677a802e4cca8775939376f (patch) | |
| tree | 9f53bfec6196bed234b6436d5967e2455d44bc6c | |
| parent | c31bf17fc483952e01468c7ff0f70b9ec0b8ad1d (diff) | |
| download | rspamd-aa3d11920940f91b9677a802e4cca8775939376f.tar.gz rspamd-aa3d11920940f91b9677a802e4cca8775939376f.zip | |
[Test] Use Python3 for functional tests
- Fix more deprecated Robot Framework syntax
- Remove log following
- Remove some artefacts in teardown
- Restore use of demjson
23 files changed, 142 insertions, 279 deletions
diff --git a/.drone.yml b/.drone.yml index 9e818c0e5..bbd71648e 100644 --- a/.drone.yml +++ b/.drone.yml @@ -12,7 +12,7 @@ steps: # any image with a root shell can be used here, but Ubuntu used as base # image for build and test images and we need to download it anyway - image: ubuntu:18.04 + image: ubuntu:20.04 pull: if-not-exists volumes: - name: rspamd diff --git a/test/functional/cases/100_general.robot b/test/functional/cases/100_general.robot index 0e5a89c15..868c31d66 100644 --- a/test/functional/cases/100_general.robot +++ b/test/functional/cases/100_general.robot @@ -23,23 +23,19 @@ GTUBE - Encrypted GTUBE - Scan File feature ${result} = Scan File ${LOCAL_ADDR} ${PORT_NORMAL} ${GTUBE} - Follow Rspamd Log Should Contain ${result} GTUBE GTUBE - Scan File feature (encoded) ${encoded} = Encode Filename ${GTUBE} ${result} = Scan File ${LOCAL_ADDR} ${PORT_NORMAL} ${encoded} - Follow Rspamd Log Should Contain ${result} GTUBE GTUBE - SPAMC ${result} = Spamc ${LOCAL_ADDR} ${PORT_NORMAL} ${GTUBE} - Follow Rspamd Log Should Contain ${result} GTUBE GTUBE - RSPAMC ${result} = Rspamc ${LOCAL_ADDR} ${PORT_NORMAL} ${GTUBE} - Follow Rspamd Log Should Contain ${result} GTUBE # Broken @@ -51,6 +47,5 @@ GTUBE - RSPAMC EMAILS DETECTION ZEROFONT ${result} = Scan File ${LOCAL_ADDR} ${PORT_NORMAL} ${TESTDIR}/messages/zerofont.eml - Follow Rspamd Log Should Contain ${result} MANY_INVISIBLE_PARTS - Should Contain ${result} ZERO_FONT
\ No newline at end of file + Should Contain ${result} ZERO_FONT diff --git a/test/functional/cases/101_lua.robot b/test/functional/cases/101_lua.robot index da5f8420b..8abb99b29 100644 --- a/test/functional/cases/101_lua.robot +++ b/test/functional/cases/101_lua.robot @@ -17,7 +17,6 @@ ${URL_TLD} ${TESTDIR}/../lua/unit/test_tld.dat Flags [Setup] Lua Setup ${TESTDIR}/lua/flags.lua ${result} = Scan Message With Rspamc ${MESSAGE} - Follow Rspamd Log ${result} = Run Rspamc -h ${LOCAL_ADDR}:${PORT_CONTROLLER} stat Should Contain ${result.stdout} Messages scanned: 0 diff --git a/test/functional/cases/120_fuzzy/lib.robot b/test/functional/cases/120_fuzzy/lib.robot index 319e5a900..54eaff0a8 100644 --- a/test/functional/cases/120_fuzzy/lib.robot +++ b/test/functional/cases/120_fuzzy/lib.robot @@ -21,7 +21,6 @@ ${SETTINGS_FUZZY_CHECK} ${EMPTY} *** Keywords *** Fuzzy Skip Add Test Base - Create File ${TMPDIR}/skip_hash.map [Arguments] ${message} Set Suite Variable ${RSPAMD_FUZZY_ADD_${message}} 0 ${result} = Run Rspamc -h ${LOCAL_ADDR}:${PORT_CONTROLLER} -w 10 -f @@ -31,7 +30,8 @@ Fuzzy Skip Add Test Base ${result} = Scan Message With Rspamc ${message} Create File ${TMPDIR}/test.map Should Contain ${result.stdout} R_TEST_FUZZY_DENIED - Append To File ${TMPDIR}/skip_hash.map 670cfcba72a87bab689958a8af5c22593dc17c907836c7c26a74d1bb49add25adfa45a5f172e3af82c9c638e8eb5fc860c22c7e966e61a459165ef0b9e1acc89 + Append To File ${TMPDIR}/skip_hash.map.tmp 670cfcba72a87bab689958a8af5c22593dc17c907836c7c26a74d1bb49add25adfa45a5f172e3af82c9c638e8eb5fc860c22c7e966e61a459165ef0b9e1acc89 + Hard Link ${TMPDIR}/skip_hash.map.tmp ${TMPDIR}/skip_hash.map ${result} = Scan Message With Rspamc ${message} Check Rspamc ${result} R_TEST_FUZZY_DENIED inverse=1 @@ -54,7 +54,6 @@ Fuzzy Delete Test Check Rspamc ${result} Sync Fuzzy Storage ${result} = Scan Message With Rspamc ${message} - Follow Rspamd Log Should Not Contain ${result.stdout} ${FLAG1_SYMBOL} Should Be Equal As Integers ${result.rc} 0 @@ -83,7 +82,6 @@ Fuzzy Overwrite Test END Sync Fuzzy Storage ${result} = Scan Message With Rspamc ${message} - Follow Rspamd Log Should Not Contain ${result.stdout} ${FLAG1_SYMBOL} Should Contain ${result.stdout} ${FLAG2_SYMBOL} Should Be Equal As Integers ${result.rc} 0 diff --git a/test/functional/cases/131_dkim_signing/006_milter.robot b/test/functional/cases/131_dkim_signing/006_milter.robot index fa6f532b4..cd48a62b7 100644 --- a/test/functional/cases/131_dkim_signing/006_milter.robot +++ b/test/functional/cases/131_dkim_signing/006_milter.robot @@ -25,7 +25,6 @@ Milter Test [Arguments] ${mtlua} ${result} = Run Process miltertest -Dport\=${PORT_PROXY} -Dhost\=${LOCAL_ADDR} -s ${TESTDIR}/lua/miltertest/${mtlua} ... cwd=${TESTDIR}/lua/miltertest - Follow Rspamd Log Should Match Regexp ${result.stderr} ^$ Log ${result.rc} Log ${result.stdout} diff --git a/test/functional/cases/140_proxy.robot b/test/functional/cases/140_proxy.robot index 58b80461a..fabc5a4fd 100644 --- a/test/functional/cases/140_proxy.robot +++ b/test/functional/cases/140_proxy.robot @@ -13,32 +13,24 @@ ${URL_TLD} ${TESTDIR}/../lua/unit/test_tld.dat *** Test Cases *** Rspamc Client ${result} = Run Rspamc -h ${LOCAL_ADDR}:${PORT_PROXY} -p ${MESSAGE} - Custom Follow Rspamd Log ${PROXY_TMPDIR}/rspamd.log ${PROXY_LOGPOS} PROXY_LOGPOS Suite - Custom Follow Rspamd Log ${SLAVE_TMPDIR}/rspamd.log ${SLAVE_LOGPOS} SLAVE_LOGPOS Suite Run Keyword If ${result.rc} != 0 Log ${result.stderr} Should Contain ${result.stdout} SIMPLE_TEST Should Be Equal As Integers ${result.rc} 0 SPAMC ${result} = Spamc ${LOCAL_ADDR} ${PORT_PROXY} ${MESSAGE} - Custom Follow Rspamd Log ${PROXY_TMPDIR}/rspamd.log ${PROXY_LOGPOS} PROXY_LOGPOS Suite - Custom Follow Rspamd Log ${SLAVE_TMPDIR}/rspamd.log ${SLAVE_LOGPOS} SLAVE_LOGPOS Suite Should Contain ${result} SPAMD/1.1 0 EX_OK RSPAMC Legacy Protocol ${result} = Rspamc ${LOCAL_ADDR} ${PORT_PROXY} ${MESSAGE} - Custom Follow Rspamd Log ${PROXY_TMPDIR}/rspamd.log ${PROXY_LOGPOS} PROXY_LOGPOS Suite - Custom Follow Rspamd Log ${SLAVE_TMPDIR}/rspamd.log ${SLAVE_LOGPOS} SLAVE_LOGPOS Suite Should Contain ${result} RSPAMD/1.3 0 EX_OK *** Keywords *** Proxy Setup &{d} = Run Rspamd CONFIG=${TESTDIR}/configs/lua_test.conf - Set Suite Variable ${SLAVE_LOGPOS} ${d}[RSPAMD_LOGPOS] Set Suite Variable ${SLAVE_PID} ${d}[RSPAMD_PID] Set Suite Variable ${SLAVE_TMPDIR} ${d}[TMPDIR] &{d} = Run Rspamd CONFIG=${TESTDIR}/configs/proxy.conf - Set Suite Variable ${PROXY_LOGPOS} ${d}[RSPAMD_LOGPOS] Set Suite Variable ${PROXY_PID} ${d}[RSPAMD_PID] Set Suite Variable ${PROXY_TMPDIR} ${d}[TMPDIR] diff --git a/test/functional/cases/180_milter.robot b/test/functional/cases/180_milter.robot index 23a366814..ddbbf9799 100644 --- a/test/functional/cases/180_milter.robot +++ b/test/functional/cases/180_milter.robot @@ -34,7 +34,6 @@ Milter Test [Arguments] ${mtlua} ${result} = Run Process miltertest -Dport\=${PORT_PROXY} -Dhost\=${LOCAL_ADDR} -s ${TESTDIR}/lua/miltertest/${mtlua} ... cwd=${TESTDIR}/lua/miltertest - Follow Rspamd Log Should Match Regexp ${result.stderr} ^$ Log ${result.rc} Log ${result.stdout} diff --git a/test/functional/cases/220_http.robot b/test/functional/cases/220_http.robot index c30fbba9c..86e882d5d 100644 --- a/test/functional/cases/220_http.robot +++ b/test/functional/cases/220_http.robot @@ -68,5 +68,6 @@ Run Dummy Https Check url [Arguments] ${url} ${method} @{expect_results} ${result} = Scan Message With Rspamc --header=url:${url} --header=method:${method} ${MESSAGE} - : FOR ${expect} IN @{expect_results} - \ Check Rspamc ${result} ${expect}
\ No newline at end of file + FOR ${expect} IN @{expect_results} + Check Rspamc ${result} ${expect} + END diff --git a/test/functional/cases/230_tcp.robot b/test/functional/cases/230_tcp.robot index 51f590ea4..16d490826 100644 --- a/test/functional/cases/230_tcp.robot +++ b/test/functional/cases/230_tcp.robot @@ -75,5 +75,6 @@ Run Dummy Ssl Check url [Arguments] ${url} ${method} @{expect_results} ${result} = Scan Message With Rspamc --header=url:${url} --header=method:${method} ${MESSAGE} - : FOR ${expect} IN @{expect_results} - \ Check Rspamc ${result} ${expect} + FOR ${expect} IN @{expect_results} + Check Rspamc ${result} ${expect} + END diff --git a/test/functional/cases/231_tcp_down.robot b/test/functional/cases/231_tcp_down.robot index 10043024f..e68e2053a 100644 --- a/test/functional/cases/231_tcp_down.robot +++ b/test/functional/cases/231_tcp_down.robot @@ -37,5 +37,6 @@ Http Teardown Check url [Arguments] ${url} ${method} @{expect_results} ${result} = Scan Message With Rspamc --header=url:${url} --header=method:${method} ${MESSAGE} - : FOR ${expect} IN @{expect_results} - \ Check Rspamc ${result} ${expect} + FOR ${expect} IN @{expect_results} + Check Rspamc ${result} ${expect} + END diff --git a/test/functional/cases/330_neural.robot b/test/functional/cases/330_neural.robot index c64e9cc0a..89ba19f11 100644 --- a/test/functional/cases/330_neural.robot +++ b/test/functional/cases/330_neural.robot @@ -16,11 +16,12 @@ ${RSPAMD_SCOPE} Suite *** Test Cases *** Train Sleep 2s Wait for redis mess - : FOR ${INDEX} IN RANGE 0 10 - \ ${result} = Scan Message With Rspamc ${MESSAGE} --header Settings={symbols_enabled = ["SPAM_SYMBOL"]} - \ Check Rspamc ${result} SPAM_SYMBOL (1.00) - \ ${result} = Scan Message With Rspamc ${MESSAGE} --header Settings={symbols_enabled = ["HAM_SYMBOL"]} - \ Check Rspamc ${result} HAM_SYMBOL (-1.00) + FOR ${INDEX} IN RANGE 0 10 + ${result} = Scan Message With Rspamc ${MESSAGE} --header Settings={symbols_enabled = ["SPAM_SYMBOL"]} + Check Rspamc ${result} SPAM_SYMBOL (1.00) + ${result} = Scan Message With Rspamc ${MESSAGE} --header Settings={symbols_enabled = ["HAM_SYMBOL"]} + Check Rspamc ${result} HAM_SYMBOL (-1.00) + END Check Neural HAM Sleep 2s Wait for neural to be loaded @@ -35,11 +36,12 @@ Check Neural SPAM Train INVERSE - : FOR ${INDEX} IN RANGE 0 10 - \ ${result} = Scan Message With Rspamc ${MESSAGE} --header Settings={symbols_enabled = ["SPAM_SYMBOL"]; SPAM_SYMBOL = -1} - \ Check Rspamc ${result} SPAM_SYMBOL (-1.00) - \ ${result} = Scan Message With Rspamc ${MESSAGE} --header Settings={symbols_enabled = ["HAM_SYMBOL"]; HAM_SYMBOL = 1;} - \ Check Rspamc ${result} HAM_SYMBOL (1.00) + FOR ${INDEX} IN RANGE 0 10 + ${result} = Scan Message With Rspamc ${MESSAGE} --header Settings={symbols_enabled = ["SPAM_SYMBOL"]; SPAM_SYMBOL = -1} + Check Rspamc ${result} SPAM_SYMBOL (-1.00) + ${result} = Scan Message With Rspamc ${MESSAGE} --header Settings={symbols_enabled = ["HAM_SYMBOL"]; HAM_SYMBOL = 1;} + Check Rspamc ${result} HAM_SYMBOL (1.00) + END Check Neural HAM INVERSE Sleep 2s Wait for neural to be loaded diff --git a/test/functional/cases/350_magic.robot b/test/functional/cases/350_magic.robot index b3a760086..787e4fe3c 100644 --- a/test/functional/cases/350_magic.robot +++ b/test/functional/cases/350_magic.robot @@ -14,7 +14,6 @@ ${URL_TLD} ${TESTDIR}/../lua/unit/test_tld.dat *** Test Cases *** Magic detections bundle 1 ${result} = Scan Message With Rspamc ${TESTDIR}/messages/gargantua.eml - Follow Rspamd Log Should Contain ${result.stdout} MAGIC_SYM_ZIP_2 Should Contain ${result.stdout} MAGIC_SYM_RAR_3 Should Contain ${result.stdout} MAGIC_SYM_EXE_4 diff --git a/test/functional/lib/rspamd.py b/test/functional/lib/rspamd.py index 295484f9e..b8bc6ac03 100644 --- a/test/functional/lib/rspamd.py +++ b/test/functional/lib/rspamd.py @@ -1,32 +1,24 @@ +from urllib.request import urlopen +import glob import grp +import http.client import os import os.path import psutil -import glob import pwd import shutil import signal import socket +import stat import sys import tempfile -import json -import stat -from robot.libraries.BuiltIn import BuiltIn -from robot.api import logger -if sys.version_info > (3,): - long = int -try: - from urllib.request import urlopen -except: - from urllib2 import urlopen -try: - import http.client as httplib -except: - import httplib +from robot.api import logger +from robot.libraries.BuiltIn import BuiltIn +import demjson def Check_JSON(j): - d = json.loads(j, strict=True) + d = demjson.decode(j, strict=True) logger.debug('got json %s' % d) assert len(d) > 0 assert 'error' not in d @@ -99,7 +91,7 @@ def get_rspamadm(): return dname + "/src/rspamadm/rspamadm" def HTTP(method, host, port, path, data=None, headers={}): - c = httplib.HTTPConnection("%s:%s" % (host, port)) + c = http.client.HTTPConnection("%s:%s" % (host, port)) c.request(method, path, data, headers) r = c.getresponse() t = r.read() @@ -107,6 +99,9 @@ def HTTP(method, host, port, path, data=None, headers={}): c.close() return [s, t] +def hard_link(src, dst): + os.link(src, dst) + def make_temporary_directory(): """Creates and returns a unique temporary directory @@ -131,14 +126,6 @@ def path_splitter(path): basename = os.path.basename(path) return [dirname, basename] -def read_log_from_position(filename, offset): - offset = long(offset) - with open(filename, 'rb') as f: - f.seek(offset) - goo = f.read() - size = len(goo) - return [goo, size+offset] - def rspamc(addr, port, filename): mboxgoo = b"From MAILER-DAEMON Fri May 13 19:17:40 2016\r\n" goo = open(filename, 'rb').read() @@ -251,13 +238,15 @@ def shutdown_process_with_children(pid): pass def write_to_stdin(process_handle, text): + if not isinstance(text, bytes): + text = bytes(text, 'utf-8') lib = BuiltIn().get_library_instance('Process') obj = lib.get_process_object() - obj.stdin.write(text + "\n") + obj.stdin.write(text + b"\n") obj.stdin.flush() obj.stdin.close() out = obj.stdout.read(4096) - return out + return out.decode('utf-8') def get_file_if_exists(file_path): if os.path.exists(file_path): @@ -265,72 +254,6 @@ def get_file_if_exists(file_path): return myfile.read() return None -# copy-paste from -# https://hg.python.org/cpython/file/6860263c05b3/Lib/shutil.py#l1068 -# As soon as we move to Python 3, this should be removed in favor of shutil.which() -def python3_which(cmd, mode=os.F_OK | os.X_OK, path=None): - """Given a command, mode, and a PATH string, return the path which - conforms to the given mode on the PATH, or None if there is no such - file. - - `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result - of os.environ.get("PATH"), or can be overridden with a custom search - path. - """ - - # Check that a given file can be accessed with the correct mode. - # Additionally check that `file` is not a directory, as on Windows - # directories pass the os.access check. - def _access_check(fn, mode): - return (os.path.exists(fn) and os.access(fn, mode) - and not os.path.isdir(fn)) - - # If we're given a path with a directory part, look it up directly rather - # than referring to PATH directories. This includes checking relative to the - # current directory, e.g. ./script - if os.path.dirname(cmd): - if _access_check(cmd, mode): - return cmd - return None - - if path is None: - path = os.environ.get("PATH", os.defpath) - if not path: - return None - path = path.split(os.pathsep) - - if sys.platform == "win32": - # The current directory takes precedence on Windows. - if not os.curdir in path: - path.insert(0, os.curdir) - - # PATHEXT is necessary to check on Windows. - pathext = os.environ.get("PATHEXT", "").split(os.pathsep) - # See if the given file matches any of the expected path extensions. - # This will allow us to short circuit when given "python.exe". - # If it does match, only test that one, otherwise we have to try - # others. - if any(cmd.lower().endswith(ext.lower()) for ext in pathext): - files = [cmd] - else: - files = [cmd + ext for ext in pathext] - else: - # On other platforms you don't have things like PATHEXT to tell you - # what file suffixes are executable, so just pass on cmd as-is. - files = [cmd] - - seen = set() - for dir in path: - normdir = os.path.normcase(dir) - if not normdir in seen: - seen.add(normdir) - for thefile in files: - name = os.path.join(dir, thefile) - if _access_check(name, mode): - return name - return None - - def _merge_luacov_stats(statsfile, coverage): """ Reads a coverage stats file written by luacov and merges coverage data to @@ -339,7 +262,7 @@ def _merge_luacov_stats(statsfile, coverage): Format of the file defined in: https://github.com/keplerproject/luacov/blob/master/src/luacov/stats.lua """ - with open(statsfile, 'rb') as fh: + with open(statsfile, 'r') as fh: while True: # max_line:filename line = fh.readline().rstrip() @@ -369,7 +292,7 @@ def _dump_luacov_stats(statsfile, coverage): """ src_files = sorted(coverage) - with open(statsfile, 'wb') as fh: + with open(statsfile, 'w') as fh: for src in src_files: stats = " ".join(str(n) for n in coverage[src]) fh.write("%s:%s\n%s\n" % (len(coverage[src]), src, stats)) diff --git a/test/functional/lib/rspamd.robot b/test/functional/lib/rspamd.robot index 03abc86c1..cd68b1d86 100644 --- a/test/functional/lib/rspamd.robot +++ b/test/functional/lib/rspamd.robot @@ -17,7 +17,6 @@ Check Pidfile Check Rspamc [Arguments] ${result} @{args} &{kwargs} - Follow Rspamd Log Run Keyword If ${result.rc} != 0 Log ${result.stderr} ${has_rc} = Evaluate 'rc' in $kwargs ${inverse} = Evaluate 'inverse' in $kwargs @@ -45,19 +44,6 @@ Check Rspamc Match String Run Keyword If ${inverse} == False Should Contain ${subject} ${str} ... ELSE Should Not Contain ${subject} ${str} -Custom Follow Rspamd Log - [Arguments] ${logfile} ${logpos} ${logpos_var} ${scope} - ${logpos} = Log Logs ${logfile} ${logpos} - Run Keyword If '${scope}' == 'Test' Set Test Variable ${${logpos_var}} ${logpos} - ... ELSE IF '${scope}' == 'Suite' Set Suite Variable ${${logpos_var}} ${logpos} - ... ELSE Fail 'scope must be Test or Suite' - -Follow Rspamd Log - ${RSPAMD_LOGPOS} = Log Logs ${TMPDIR}/rspamd.log ${RSPAMD_LOGPOS} - Run Keyword If '${RSPAMD_SCOPE}' == 'Test' Set Test Variable ${RSPAMD_LOGPOS} - ... ELSE IF '${RSPAMD_SCOPE}' == 'Suite' Set Suite Variable ${RSPAMD_LOGPOS} - ... ELSE Fail 'RSPAMD_SCOPE must be Test or Suite' - Generic Setup [Arguments] @{vargs} &{kwargs} &{d} = Run Rspamd @{vargs} &{kwargs} @@ -69,7 +55,6 @@ Generic Setup END Generic Teardown - [Arguments] @{ports} Run Keyword If '${CONTROLLER_ERRORS}' == 'True' Check Controller Errors Shutdown Process With Children ${RSPAMD_PID} Log does not contain segfault record @@ -78,20 +63,11 @@ Generic Teardown Cleanup Temporary Directory ${TMPDIR} Log does not contain segfault record - ${log} = Get File ${TMPDIR}/rspamd.log + ${log} = Get File ${TMPDIR}/rspamd.log encoding_errors=ignore Should not contain ${log} Segmentation fault: msg=Segmentation fault detected -Log Logs - [Arguments] ${logfile} ${position} - ${the_log} ${position} = Read Log From Position ${logfile} ${position} - Log ${the_log} - [Return] ${position} - Normal Teardown - ${port_normal} = Create List ${SOCK_STREAM} ${LOCAL_ADDR} ${PORT_NORMAL} - ${port_controller} = Create List ${SOCK_STREAM} ${LOCAL_ADDR} ${PORT_CONTROLLER} - ${ports} = Create List ${port_normal} ${port_controller} - Generic Teardown @{ports} + Generic Teardown Redis HSET [Arguments] ${hash} ${key} ${value} @@ -169,24 +145,21 @@ Run Rspamd ${result} = Run Process ${RSPAMD} -u ${RSPAMD_USER} -g ${RSPAMD_GROUP} ... -c ${tmpdir}/rspamd.conf env:TMPDIR=${tmpdir} env:DBDIR=${tmpdir} env:LD_LIBRARY_PATH=${TESTDIR}/../../contrib/aho-corasick Run Keyword If ${result.rc} != 0 Log ${result.stderr} - ${rspamd_logpos} = Log Logs ${tmpdir}/rspamd.log 0 Should Be Equal As Integers ${result.rc} 0 Wait Until Keyword Succeeds 10x 1 sec Check Pidfile ${tmpdir}/rspamd.pid timeout=0.5s Wait Until Keyword Succeeds 5x 1 sec Ping Rspamd ${LOCAL_ADDR} ${PORT_NORMAL} ${rspamd_pid} = Get File ${tmpdir}/rspamd.pid - Set To Dictionary ${d} RSPAMD_LOGPOS=${rspamd_logpos} RSPAMD_PID=${rspamd_pid} TMPDIR=${tmpdir} + Set To Dictionary ${d} RSPAMD_PID=${rspamd_pid} TMPDIR=${tmpdir} [Return] &{d} +Simple Teardown + Generic Teardown + Scan Message With Rspamc [Arguments] ${msg_file} @{vargs} ${result} = Run Rspamc -p -h ${LOCAL_ADDR}:${PORT_NORMAL} @{vargs} ${msg_file} [Return] ${result} -Simple Teardown - ${port_normal} = Create List ${SOCK_STREAM} ${LOCAL_ADDR} ${PORT_NORMAL} - ${ports} = Create List ${port_normal} - Generic Teardown @{ports} - Sync Fuzzy Storage [Arguments] @{vargs} ${len} = Get Length ${vargs} @@ -195,6 +168,4 @@ Sync Fuzzy Storage ... ELSE Run Process ${RSPAMADM} control -s ${vargs}[0]/rspamd.sock ... fuzzy_sync Log ${result.stdout} - Run Keyword If $len == 0 Follow Rspamd Log - ... ELSE Custom Follow Rspamd Log ${vargs}[0]/rspamd.log ${vargs}[1] ${vargs}[2] ${vargs}[3] Sleep 0.1s Try give fuzzy storage time to sync diff --git a/test/functional/lib/vars.py b/test/functional/lib/vars.py index d29c31c7b..3cec9767a 100644 --- a/test/functional/lib/vars.py +++ b/test/functional/lib/vars.py @@ -3,7 +3,7 @@ import socket CONTROLLER_ERRORS = True KEY_PVT1 = 'ekd3x36tfa5gd76t6pa8hqif3ott7n1siuux68exbkk7ukscte9y' KEY_PUB1 = 'm8kneubpcjsb8sbsoj7jy7azj9fdd3xmj63txni86a8ye9ncomny' -LOCAL_ADDR = u'127.0.0.1' +LOCAL_ADDR = '127.0.0.1' MAP_WATCH_INTERVAL = '1min' PORT_CONTROLLER = 56790 PORT_CONTROLLER_SLAVE = 56793 @@ -17,9 +17,9 @@ PORT_FPROT = 56797 PORT_FPROT2_DUPLICATE = 56798 PORT_AVAST = 56799 P0F_SOCKET = '/tmp/p0f.sock' -REDIS_ADDR = u'127.0.0.1' +REDIS_ADDR = '127.0.0.1' REDIS_PORT = 56379 -NGINX_ADDR = u'127.0.0.1' +NGINX_ADDR = '127.0.0.1' NGINX_PORT = 56380 RSPAMD_GROUP = 'nogroup' RSPAMD_USER = 'nobody' diff --git a/test/functional/util/dummy_avast.py b/test/functional/util/dummy_avast.py index 55945d8dc..a4a6c62a2 100755 --- a/test/functional/util/dummy_avast.py +++ b/test/functional/util/dummy_avast.py @@ -1,15 +1,13 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 PID = "/tmp/dummy_avast.pid" import os -import sys import socket +import socketserver +import sys + import dummy_killer -try: - import SocketServer as socketserver -except: - import socketserver class MyTCPHandler(socketserver.BaseRequestHandler): @@ -50,6 +48,6 @@ if __name__ == "__main__": try: server.handle_request() except socket.error: - print "Socket closed" + print("Socket closed") server.server_close() diff --git a/test/functional/util/dummy_clam.py b/test/functional/util/dummy_clam.py index d25a63525..1b614f58b 100755 --- a/test/functional/util/dummy_clam.py +++ b/test/functional/util/dummy_clam.py @@ -1,15 +1,13 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 PID = "/tmp/dummy_clamav.pid" import os -import sys import socket +import socketserver +import sys + import dummy_killer -try: - import SocketServer as socketserver -except: - import socketserver class MyTCPHandler(socketserver.BaseRequestHandler): @@ -47,6 +45,6 @@ if __name__ == "__main__": try: server.handle_request() except socket.error: - print "Socket closed" + print("Socket closed") server.server_close() diff --git a/test/functional/util/dummy_fprot.py b/test/functional/util/dummy_fprot.py index e6dc58bdd..31ae2c487 100755 --- a/test/functional/util/dummy_fprot.py +++ b/test/functional/util/dummy_fprot.py @@ -1,15 +1,12 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 + import os -import sys import signal import socket -import dummy_killer - +import socketserver +import sys -try: - import SocketServer as socketserver -except: - import socketserver +import dummy_killer PID = "/tmp/dummy_fprot.pid" @@ -53,5 +50,5 @@ if __name__ == "__main__": try: server.handle_request() except socket.error: - print "Socket closed" + print("Socket closed") server.server_close() diff --git a/test/functional/util/dummy_http.py b/test/functional/util/dummy_http.py index 614fced29..c05ee2d02 100755 --- a/test/functional/util/dummy_http.py +++ b/test/functional/util/dummy_http.py @@ -1,14 +1,13 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -import BaseHTTPServer -import SocketServer -import SimpleHTTPServer -import dummy_killer - -import time +import http.server import os -import sys import socket +import socketserver +import sys +import time + +import dummy_killer PORT = 18080 HOST_NAME = '127.0.0.1' @@ -16,10 +15,10 @@ HOST_NAME = '127.0.0.1' PID = "/tmp/dummy_http.pid" -class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler): +class MyHandler(http.server.BaseHTTPRequestHandler): def setup(self): - BaseHTTPServer.BaseHTTPRequestHandler.setup(self) + http.server.BaseHTTPRequestHandler.setup(self) self.protocol_version = "HTTP/1.1" # allow connection: keep-alive def do_HEAD(self): @@ -29,7 +28,7 @@ class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler): self.log_message("to be closed: " + self.close_connection) def do_GET(self): - response = "hello world" + response = b"hello world" """Respond to a GET request.""" if self.path == "/empty": @@ -60,7 +59,7 @@ class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler): def do_POST(self): - response = "hello post" + response = b"hello post" """Respond to a GET request.""" if self.path == "/empty": self.finish() @@ -82,12 +81,12 @@ class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler): self.wfile.write(response) -class ThreadingSimpleServer(SocketServer.ThreadingMixIn, - BaseHTTPServer.HTTPServer): +class ThreadingSimpleServer(socketserver.ThreadingMixIn, + http.server.HTTPServer): def __init__(self): self.allow_reuse_address = True self.timeout = 1 - BaseHTTPServer.HTTPServer.__init__(self, (HOST_NAME, PORT), MyHandler) + http.server.HTTPServer.__init__(self, (HOST_NAME, PORT), MyHandler) def run(self): dummy_killer.write_pid(PID) @@ -96,9 +95,9 @@ class ThreadingSimpleServer(SocketServer.ThreadingMixIn, sys.stdout.flush() server.handle_request() except KeyboardInterrupt: - print "Interrupt" + print("Interrupt") except socket.error: - print "Socket closed" + print("Socket closed") def stop(self): self.keep_running = False diff --git a/test/functional/util/dummy_https.py b/test/functional/util/dummy_https.py index 43c5d9122..1e6753423 100755 --- a/test/functional/util/dummy_https.py +++ b/test/functional/util/dummy_https.py @@ -1,15 +1,15 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -import BaseHTTPServer -import SocketServer -import SimpleHTTPServer -import dummy_killer -import ssl - -import time +import http.server +import http.server import os -import sys import socket +import socketserver +import ssl +import sys +import time + +import dummy_killer PORT = 18081 HOST_NAME = '127.0.0.1' @@ -17,10 +17,10 @@ HOST_NAME = '127.0.0.1' PID = "/tmp/dummy_https.pid" -class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler): +class MyHandler(http.server.BaseHTTPRequestHandler): def setup(self): - BaseHTTPServer.BaseHTTPRequestHandler.setup(self) + http.server.BaseHTTPRequestHandler.setup(self) self.protocol_version = "HTTP/1.1" # allow connection: keep-alive def do_HEAD(self): @@ -30,7 +30,7 @@ class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler): self.log_message("to be closed: " + self.close_connection) def do_GET(self): - response = "hello world" + response = b"hello world" """Respond to a GET request.""" if self.path == "/empty": @@ -64,11 +64,11 @@ class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler): """Respond to a POST request.""" content_length = int(self.headers['Content-Length']) - response = 'hello post' + response = b'hello post' if content_length > 0: body = self.rfile.read(content_length) - response = "hello post: " + str(len(body)) + response = b"hello post: " + bytes(len(body)) if self.path == "/empty": self.finish() @@ -90,13 +90,13 @@ class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler): self.wfile.write(response) -class ThreadingSimpleServer(SocketServer.ThreadingMixIn, - BaseHTTPServer.HTTPServer): +class ThreadingSimpleServer(socketserver.ThreadingMixIn, + http.server.HTTPServer): def __init__(self, certfile, keyfile,): self.allow_reuse_address = True self.timeout = 10 - BaseHTTPServer.HTTPServer.__init__(self, (HOST_NAME, PORT), MyHandler) + http.server.HTTPServer.__init__(self, (HOST_NAME, PORT), MyHandler) self.socket = ssl.wrap_socket (self.socket, keyfile=keyfile, certfile=certfile, server_side=True) @@ -108,9 +108,9 @@ class ThreadingSimpleServer(SocketServer.ThreadingMixIn, sys.stdout.flush() server.handle_request() except KeyboardInterrupt: - print "Interrupt" + print("Interrupt") except socket.error: - print "Socket closed" + print("Socket closed") def stop(self): self.keep_running = False diff --git a/test/functional/util/dummy_p0f.py b/test/functional/util/dummy_p0f.py index d780bdb0d..2bcd183d4 100755 --- a/test/functional/util/dummy_p0f.py +++ b/test/functional/util/dummy_p0f.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 PID = "/tmp/dummy_p0f.pid" @@ -6,11 +6,9 @@ import os import sys import struct import socket +import socketserver + import dummy_killer -try: - import SocketServer as socketserver -except: - import socketserver class MyStreamHandler(socketserver.BaseRequestHandler): @@ -22,8 +20,8 @@ class MyStreamHandler(socketserver.BaseRequestHandler): } OS = { - 'windows' : ('Windows', '7 or 8'), - 'linux' : ('Linux', '3.11 and newer') + 'windows' : (b'Windows', b'7 or 8'), + 'linux' : (b'Linux', b'3.11 and newer') } self.data = self.request.recv(21).strip() @@ -47,10 +45,10 @@ class MyStreamHandler(socketserver.BaseRequestHandler): 0, # os_match_q OS[self.server.p0f_os][0], # os_name OS[self.server.p0f_os][1], # os_flavor - '', # http_name - '', # http_flavor - 'Ethernet or modem', # link_type - '' # language + b'', # http_name + b'', # http_flavor + b'Ethernet or modem', # link_type + b'' # language ) self.request.sendall(response) @@ -61,7 +59,7 @@ def cleanup(SOCK): try: os.unlink(SOCK) except OSError: - print "Could not unlink socket: " + SOCK + print("Could not unlink socket: " + SOCK) if __name__ == "__main__": SOCK = '/tmp/p0f.sock' @@ -92,7 +90,7 @@ if __name__ == "__main__": try: server.handle_request() except socket.error: - print "Socket closed" + print("Socket closed") server.server_close() cleanup(SOCK) diff --git a/test/functional/util/dummy_ssl.py b/test/functional/util/dummy_ssl.py index 725f8052a..44b178293 100755 --- a/test/functional/util/dummy_ssl.py +++ b/test/functional/util/dummy_ssl.py @@ -1,64 +1,58 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -import SocketServer -import dummy_killer - -import time import os -import sys import socket import ssl +import sys +import time + +import dummy_killer +import socketserver PORT = 14433 HOST_NAME = '127.0.0.1' PID = "/tmp/dummy_ssl.pid" -class SSLTCPHandler(SocketServer.BaseRequestHandler): +class SSLTCPHandler(socketserver.StreamRequestHandler): def handle(self): time.sleep(0.5) data = self.request.recv(6000000) while data: - print "{} wrote:".format(self.client_address[0]) - print data + print("{} wrote:".format(self.client_address[0])) + print(data) time.sleep(0.1) - self.request.sendall('hello\n') + self.request.sendall(b'hello\n') time.sleep(0.1) data = self.request.recv(6000000) -class SSL_TCP_Server(SocketServer.TCPServer): +class SSL_TCP_Server(socketserver.ThreadingMixIn, socketserver.TCPServer): def __init__(self, server_address, RequestHandlerClass, certfile, keyfile, - ssl_version=ssl.PROTOCOL_TLSv1, bind_and_activate=True): self.allow_reuse_address = True - SocketServer.TCPServer.__init__(self, server_address, - RequestHandlerClass, bind_and_activate) - self.certfile = certfile - self.keyfile = keyfile - self.ssl_version = ssl_version - #self.timeout = 1 - - def get_request(self): - newsocket, fromaddr = self.socket.accept() - connstream = ssl.wrap_socket(newsocket, - server_side=True, - certfile = self.certfile, - keyfile = self.keyfile, - ssl_version = self.ssl_version) - return connstream, fromaddr + super().__init__(server_address, RequestHandlerClass, False) + self.timeout = 1 + ctx = ssl.create_default_context() + ctx.load_cert_chain(certfile=certfile) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + self.socket = ctx.wrap_socket(self.socket, server_side=True) + if (bind_and_activate): + self.server_bind() + self.server_activate() def run(self): dummy_killer.write_pid(PID) try: self.serve_forever() except KeyboardInterrupt: - print "Interrupt" + print("Interrupt") except socket.error as e: - print "Socket closed {}".format(e) + print("Socket closed {}".format(e)) finally: self.server_close() @@ -66,9 +60,7 @@ class SSL_TCP_Server(SocketServer.TCPServer): self.keep_running = False self.server_close() -class SSL_ThreadingTCPServer(SocketServer.ThreadingMixIn, SSL_TCP_Server): pass - if __name__ == '__main__': - server = SSL_ThreadingTCPServer((HOST_NAME, PORT), SSLTCPHandler, sys.argv[1], sys.argv[1]) + server = SSL_TCP_Server((HOST_NAME, PORT), SSLTCPHandler, sys.argv[1], sys.argv[1]) dummy_killer.setup_killer(server, server.stop) server.run() diff --git a/test/functional/util/dummy_udp.py b/test/functional/util/dummy_udp.py index 02ee6dce8..f05d6d680 100755 --- a/test/functional/util/dummy_udp.py +++ b/test/functional/util/dummy_udp.py @@ -1,7 +1,8 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import socket import sys + import dummy_killer UDP_IP = "127.0.0.1" @@ -21,5 +22,5 @@ if __name__ == "__main__": while True: data, addr = sock.recvfrom(1024) # buffer size is 1024 bytes - print "received message:", data - sock.sendto(data, addr)
\ No newline at end of file + print("received message:", data) + sock.sendto(data, addr) |