mirrors
/
rspamd
mirror of https://github.com/vstakhov/rspamd.git
Watch 1
Star 0
Fork 0
Code Issues 0 Releases 159 Wiki Activity
Browse Source

Merge pull request #3414 from fatalbanana/py3 

☠️ Use Ubuntu 20.04 & Python3 for functional tests ☠️
tags/2.6
Vsevolod Stakhov 3 years ago
parent
c31bf17fc4 aa3d119209
commit
898adeb2d2
No account linked to committer's email address
23 changed files with 142 additions and 279 deletions
Split View Show Diff Stats
  1. 1
    1
      .drone.yml
  2. 1
    6
      test/functional/cases/100_general.robot
  3. 0
    1
      test/functional/cases/101_lua.robot
  4. 2
    4
      test/functional/cases/120_fuzzy/lib.robot
  5. 0
    1
      test/functional/cases/131_dkim_signing/006_milter.robot
  6. 0
    8
      test/functional/cases/140_proxy.robot
  7. 0
    1
      test/functional/cases/180_milter.robot
  8. 3
    2
      test/functional/cases/220_http.robot
  9. 3
    2
      test/functional/cases/230_tcp.robot
  10. 3
    2
      test/functional/cases/231_tcp_down.robot
  11. 12
    10
      test/functional/cases/330_neural.robot
  12. 0
    1
      test/functional/cases/350_magic.robot
  13. 18
    95
      test/functional/lib/rspamd.py
  14. 6
    35
      test/functional/lib/rspamd.robot
  15. 3
    3
      test/functional/lib/vars.py
  16. 5
    7
      test/functional/util/dummy_avast.py
  17. 5
    7
      test/functional/util/dummy_clam.py
  18. 6
    9
      test/functional/util/dummy_fprot.py
  19. 16
    17
      test/functional/util/dummy_http.py
  20. 19
    19
      test/functional/util/dummy_https.py
  21. 11
    13
      test/functional/util/dummy_p0f.py
  22. 24
    32
      test/functional/util/dummy_ssl.py
  23. 4
    3
      test/functional/util/dummy_udp.py

+ 1
- 1
.drone.yml View File

@@ -12,7 +12,7 @@ steps:

# any image with a root shell can be used here, but Ubuntu used as base
# image for build and test images and we need to download it anyway
image: ubuntu:18.04
image: ubuntu:20.04
pull: if-not-exists
volumes:
- name: rspamd

+ 1
- 6
test/functional/cases/100_general.robot View File

@@ -23,23 +23,19 @@ GTUBE - Encrypted

GTUBE - Scan File feature
${result} = Scan File ${LOCAL_ADDR} ${PORT_NORMAL} ${GTUBE}
Follow Rspamd Log
Should Contain ${result} GTUBE

GTUBE - Scan File feature (encoded)
${encoded} = Encode Filename ${GTUBE}
${result} = Scan File ${LOCAL_ADDR} ${PORT_NORMAL} ${encoded}
Follow Rspamd Log
Should Contain ${result} GTUBE

GTUBE - SPAMC
${result} = Spamc ${LOCAL_ADDR} ${PORT_NORMAL} ${GTUBE}
Follow Rspamd Log
Should Contain ${result} GTUBE

GTUBE - RSPAMC
${result} = Rspamc ${LOCAL_ADDR} ${PORT_NORMAL} ${GTUBE}
Follow Rspamd Log
Should Contain ${result} GTUBE

# Broken
@@ -51,6 +47,5 @@ GTUBE - RSPAMC

EMAILS DETECTION ZEROFONT
${result} = Scan File ${LOCAL_ADDR} ${PORT_NORMAL} ${TESTDIR}/messages/zerofont.eml
Follow Rspamd Log
Should Contain ${result} MANY_INVISIBLE_PARTS
Should Contain ${result} ZERO_FONT
Should Contain ${result} ZERO_FONT

+ 0
- 1
test/functional/cases/101_lua.robot View File

@@ -17,7 +17,6 @@ ${URL_TLD} ${TESTDIR}/../lua/unit/test_tld.dat
Flags
[Setup] Lua Setup ${TESTDIR}/lua/flags.lua
${result} = Scan Message With Rspamc ${MESSAGE}
Follow Rspamd Log
${result} = Run Rspamc -h ${LOCAL_ADDR}:${PORT_CONTROLLER} stat
Should Contain ${result.stdout} Messages scanned: 0


+ 2
- 4
test/functional/cases/120_fuzzy/lib.robot View File

@@ -21,7 +21,6 @@ ${SETTINGS_FUZZY_CHECK} ${EMPTY}

*** Keywords ***
Fuzzy Skip Add Test Base
Create File ${TMPDIR}/skip_hash.map
[Arguments] ${message}
Set Suite Variable ${RSPAMD_FUZZY_ADD_${message}} 0
${result} = Run Rspamc -h ${LOCAL_ADDR}:${PORT_CONTROLLER} -w 10 -f
@@ -31,7 +30,8 @@ Fuzzy Skip Add Test Base
${result} = Scan Message With Rspamc ${message}
Create File ${TMPDIR}/test.map
Should Contain ${result.stdout} R_TEST_FUZZY_DENIED
Append To File ${TMPDIR}/skip_hash.map 670cfcba72a87bab689958a8af5c22593dc17c907836c7c26a74d1bb49add25adfa45a5f172e3af82c9c638e8eb5fc860c22c7e966e61a459165ef0b9e1acc89
Append To File ${TMPDIR}/skip_hash.map.tmp 670cfcba72a87bab689958a8af5c22593dc17c907836c7c26a74d1bb49add25adfa45a5f172e3af82c9c638e8eb5fc860c22c7e966e61a459165ef0b9e1acc89
Hard Link ${TMPDIR}/skip_hash.map.tmp ${TMPDIR}/skip_hash.map
${result} = Scan Message With Rspamc ${message}
Check Rspamc ${result} R_TEST_FUZZY_DENIED inverse=1

@@ -54,7 +54,6 @@ Fuzzy Delete Test
Check Rspamc ${result}
Sync Fuzzy Storage
${result} = Scan Message With Rspamc ${message}
Follow Rspamd Log
Should Not Contain ${result.stdout} ${FLAG1_SYMBOL}
Should Be Equal As Integers ${result.rc} 0

@@ -83,7 +82,6 @@ Fuzzy Overwrite Test
END
Sync Fuzzy Storage
${result} = Scan Message With Rspamc ${message}
Follow Rspamd Log
Should Not Contain ${result.stdout} ${FLAG1_SYMBOL}
Should Contain ${result.stdout} ${FLAG2_SYMBOL}
Should Be Equal As Integers ${result.rc} 0

+ 0
- 1
test/functional/cases/131_dkim_signing/006_milter.robot View File

@@ -25,7 +25,6 @@ Milter Test
[Arguments] ${mtlua}
${result} = Run Process miltertest -Dport\=${PORT_PROXY} -Dhost\=${LOCAL_ADDR} -s ${TESTDIR}/lua/miltertest/${mtlua}
... cwd=${TESTDIR}/lua/miltertest
Follow Rspamd Log
Should Match Regexp ${result.stderr} ^$
Log ${result.rc}
Log ${result.stdout}

+ 0
- 8
test/functional/cases/140_proxy.robot View File

@@ -13,32 +13,24 @@ ${URL_TLD} ${TESTDIR}/../lua/unit/test_tld.dat
*** Test Cases ***
Rspamc Client
${result} = Run Rspamc -h ${LOCAL_ADDR}:${PORT_PROXY} -p ${MESSAGE}
Custom Follow Rspamd Log ${PROXY_TMPDIR}/rspamd.log ${PROXY_LOGPOS} PROXY_LOGPOS Suite
Custom Follow Rspamd Log ${SLAVE_TMPDIR}/rspamd.log ${SLAVE_LOGPOS} SLAVE_LOGPOS Suite
Run Keyword If ${result.rc} != 0 Log ${result.stderr}
Should Contain ${result.stdout} SIMPLE_TEST
Should Be Equal As Integers ${result.rc} 0

SPAMC
${result} = Spamc ${LOCAL_ADDR} ${PORT_PROXY} ${MESSAGE}
Custom Follow Rspamd Log ${PROXY_TMPDIR}/rspamd.log ${PROXY_LOGPOS} PROXY_LOGPOS Suite
Custom Follow Rspamd Log ${SLAVE_TMPDIR}/rspamd.log ${SLAVE_LOGPOS} SLAVE_LOGPOS Suite
Should Contain ${result} SPAMD/1.1 0 EX_OK

RSPAMC Legacy Protocol
${result} = Rspamc ${LOCAL_ADDR} ${PORT_PROXY} ${MESSAGE}
Custom Follow Rspamd Log ${PROXY_TMPDIR}/rspamd.log ${PROXY_LOGPOS} PROXY_LOGPOS Suite
Custom Follow Rspamd Log ${SLAVE_TMPDIR}/rspamd.log ${SLAVE_LOGPOS} SLAVE_LOGPOS Suite
Should Contain ${result} RSPAMD/1.3 0 EX_OK

*** Keywords ***
Proxy Setup
&{d} = Run Rspamd CONFIG=${TESTDIR}/configs/lua_test.conf
Set Suite Variable ${SLAVE_LOGPOS} ${d}[RSPAMD_LOGPOS]
Set Suite Variable ${SLAVE_PID} ${d}[RSPAMD_PID]
Set Suite Variable ${SLAVE_TMPDIR} ${d}[TMPDIR]
&{d} = Run Rspamd CONFIG=${TESTDIR}/configs/proxy.conf
Set Suite Variable ${PROXY_LOGPOS} ${d}[RSPAMD_LOGPOS]
Set Suite Variable ${PROXY_PID} ${d}[RSPAMD_PID]
Set Suite Variable ${PROXY_TMPDIR} ${d}[TMPDIR]


+ 0
- 1
test/functional/cases/180_milter.robot View File

@@ -34,7 +34,6 @@ Milter Test
[Arguments] ${mtlua}
${result} = Run Process miltertest -Dport\=${PORT_PROXY} -Dhost\=${LOCAL_ADDR} -s ${TESTDIR}/lua/miltertest/${mtlua}
... cwd=${TESTDIR}/lua/miltertest
Follow Rspamd Log
Should Match Regexp ${result.stderr} ^$
Log ${result.rc}
Log ${result.stdout}

+ 3
- 2
test/functional/cases/220_http.robot View File

@@ -68,5 +68,6 @@ Run Dummy Https
Check url
[Arguments] ${url} ${method} @{expect_results}
${result} = Scan Message With Rspamc --header=url:${url} --header=method:${method} ${MESSAGE}
: FOR ${expect} IN @{expect_results}
\ Check Rspamc ${result} ${expect}
FOR ${expect} IN @{expect_results}
Check Rspamc ${result} ${expect}
END

+ 3
- 2
test/functional/cases/230_tcp.robot View File

@@ -75,5 +75,6 @@ Run Dummy Ssl
Check url
[Arguments] ${url} ${method} @{expect_results}
${result} = Scan Message With Rspamc --header=url:${url} --header=method:${method} ${MESSAGE}
: FOR ${expect} IN @{expect_results}
\ Check Rspamc ${result} ${expect}
FOR ${expect} IN @{expect_results}
Check Rspamc ${result} ${expect}
END

+ 3
- 2
test/functional/cases/231_tcp_down.robot View File

@@ -37,5 +37,6 @@ Http Teardown
Check url
[Arguments] ${url} ${method} @{expect_results}
${result} = Scan Message With Rspamc --header=url:${url} --header=method:${method} ${MESSAGE}
: FOR ${expect} IN @{expect_results}
\ Check Rspamc ${result} ${expect}
FOR ${expect} IN @{expect_results}
Check Rspamc ${result} ${expect}
END

+ 12
- 10
test/functional/cases/330_neural.robot View File

@@ -16,11 +16,12 @@ ${RSPAMD_SCOPE} Suite
*** Test Cases ***
Train
Sleep 2s Wait for redis mess
: FOR ${INDEX} IN RANGE 0 10
\ ${result} = Scan Message With Rspamc ${MESSAGE} --header Settings={symbols_enabled = ["SPAM_SYMBOL"]}
\ Check Rspamc ${result} SPAM_SYMBOL (1.00)
\ ${result} = Scan Message With Rspamc ${MESSAGE} --header Settings={symbols_enabled = ["HAM_SYMBOL"]}
\ Check Rspamc ${result} HAM_SYMBOL (-1.00)
FOR ${INDEX} IN RANGE 0 10
${result} = Scan Message With Rspamc ${MESSAGE} --header Settings={symbols_enabled = ["SPAM_SYMBOL"]}
Check Rspamc ${result} SPAM_SYMBOL (1.00)
${result} = Scan Message With Rspamc ${MESSAGE} --header Settings={symbols_enabled = ["HAM_SYMBOL"]}
Check Rspamc ${result} HAM_SYMBOL (-1.00)
END

Check Neural HAM
Sleep 2s Wait for neural to be loaded
@@ -35,11 +36,12 @@ Check Neural SPAM


Train INVERSE
: FOR ${INDEX} IN RANGE 0 10
\ ${result} = Scan Message With Rspamc ${MESSAGE} --header Settings={symbols_enabled = ["SPAM_SYMBOL"]; SPAM_SYMBOL = -1}
\ Check Rspamc ${result} SPAM_SYMBOL (-1.00)
\ ${result} = Scan Message With Rspamc ${MESSAGE} --header Settings={symbols_enabled = ["HAM_SYMBOL"]; HAM_SYMBOL = 1;}
\ Check Rspamc ${result} HAM_SYMBOL (1.00)
FOR ${INDEX} IN RANGE 0 10
${result} = Scan Message With Rspamc ${MESSAGE} --header Settings={symbols_enabled = ["SPAM_SYMBOL"]; SPAM_SYMBOL = -1}
Check Rspamc ${result} SPAM_SYMBOL (-1.00)
${result} = Scan Message With Rspamc ${MESSAGE} --header Settings={symbols_enabled = ["HAM_SYMBOL"]; HAM_SYMBOL = 1;}
Check Rspamc ${result} HAM_SYMBOL (1.00)
END

Check Neural HAM INVERSE
Sleep 2s Wait for neural to be loaded

+ 0
- 1
test/functional/cases/350_magic.robot View File

@@ -14,7 +14,6 @@ ${URL_TLD} ${TESTDIR}/../lua/unit/test_tld.dat
*** Test Cases ***
Magic detections bundle 1
${result} = Scan Message With Rspamc ${TESTDIR}/messages/gargantua.eml
Follow Rspamd Log
Should Contain ${result.stdout} MAGIC_SYM_ZIP_2
Should Contain ${result.stdout} MAGIC_SYM_RAR_3
Should Contain ${result.stdout} MAGIC_SYM_EXE_4

+ 18
- 95
test/functional/lib/rspamd.py View File

@@ -1,32 +1,24 @@
from urllib.request import urlopen
import glob
import grp
import http.client
import os
import os.path
import psutil
import glob
import pwd
import shutil
import signal
import socket
import stat
import sys
import tempfile
import json
import stat
from robot.libraries.BuiltIn import BuiltIn
from robot.api import logger

if sys.version_info > (3,):
long = int
try:
from urllib.request import urlopen
except:
from urllib2 import urlopen
try:
import http.client as httplib
except:
import httplib
from robot.api import logger
from robot.libraries.BuiltIn import BuiltIn
import demjson

def Check_JSON(j):
d = json.loads(j, strict=True)
d = demjson.decode(j, strict=True)
logger.debug('got json %s' % d)
assert len(d) > 0
assert 'error' not in d
@@ -99,7 +91,7 @@ def get_rspamadm():
return dname + "/src/rspamadm/rspamadm"

def HTTP(method, host, port, path, data=None, headers={}):
c = httplib.HTTPConnection("%s:%s" % (host, port))
c = http.client.HTTPConnection("%s:%s" % (host, port))
c.request(method, path, data, headers)
r = c.getresponse()
t = r.read()
@@ -107,6 +99,9 @@ def HTTP(method, host, port, path, data=None, headers={}):
c.close()
return [s, t]

def hard_link(src, dst):
os.link(src, dst)

def make_temporary_directory():
"""Creates and returns a unique temporary directory

@@ -131,14 +126,6 @@ def path_splitter(path):
basename = os.path.basename(path)
return [dirname, basename]

def read_log_from_position(filename, offset):
offset = long(offset)
with open(filename, 'rb') as f:
f.seek(offset)
goo = f.read()
size = len(goo)
return [goo, size+offset]

def rspamc(addr, port, filename):
mboxgoo = b"From MAILER-DAEMON Fri May 13 19:17:40 2016\r\n"
goo = open(filename, 'rb').read()
@@ -251,13 +238,15 @@ def shutdown_process_with_children(pid):
pass

def write_to_stdin(process_handle, text):
if not isinstance(text, bytes):
text = bytes(text, 'utf-8')
lib = BuiltIn().get_library_instance('Process')
obj = lib.get_process_object()
obj.stdin.write(text + "\n")
obj.stdin.write(text + b"\n")
obj.stdin.flush()
obj.stdin.close()
out = obj.stdout.read(4096)
return out
return out.decode('utf-8')

def get_file_if_exists(file_path):
if os.path.exists(file_path):
@@ -265,72 +254,6 @@ def get_file_if_exists(file_path):
return myfile.read()
return None

# copy-paste from
# https://hg.python.org/cpython/file/6860263c05b3/Lib/shutil.py#l1068
# As soon as we move to Python 3, this should be removed in favor of shutil.which()
def python3_which(cmd, mode=os.F_OK | os.X_OK, path=None):
"""Given a command, mode, and a PATH string, return the path which
conforms to the given mode on the PATH, or None if there is no such
file.

`mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result
of os.environ.get("PATH"), or can be overridden with a custom search
path.
"""

# Check that a given file can be accessed with the correct mode.
# Additionally check that `file` is not a directory, as on Windows
# directories pass the os.access check.
def _access_check(fn, mode):
return (os.path.exists(fn) and os.access(fn, mode)
and not os.path.isdir(fn))

# If we're given a path with a directory part, look it up directly rather
# than referring to PATH directories. This includes checking relative to the
# current directory, e.g. ./script
if os.path.dirname(cmd):
if _access_check(cmd, mode):
return cmd
return None

if path is None:
path = os.environ.get("PATH", os.defpath)
if not path:
return None
path = path.split(os.pathsep)

if sys.platform == "win32":
# The current directory takes precedence on Windows.
if not os.curdir in path:
path.insert(0, os.curdir)

# PATHEXT is necessary to check on Windows.
pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
# See if the given file matches any of the expected path extensions.
# This will allow us to short circuit when given "python.exe".
# If it does match, only test that one, otherwise we have to try
# others.
if any(cmd.lower().endswith(ext.lower()) for ext in pathext):
files = [cmd]
else:
files = [cmd + ext for ext in pathext]
else:
# On other platforms you don't have things like PATHEXT to tell you
# what file suffixes are executable, so just pass on cmd as-is.
files = [cmd]

seen = set()
for dir in path:
normdir = os.path.normcase(dir)
if not normdir in seen:
seen.add(normdir)
for thefile in files:
name = os.path.join(dir, thefile)
if _access_check(name, mode):
return name
return None


def _merge_luacov_stats(statsfile, coverage):
"""
Reads a coverage stats file written by luacov and merges coverage data to
@@ -339,7 +262,7 @@ def _merge_luacov_stats(statsfile, coverage):
Format of the file defined in:
https://github.com/keplerproject/luacov/blob/master/src/luacov/stats.lua
"""
with open(statsfile, 'rb') as fh:
with open(statsfile, 'r') as fh:
while True:
# max_line:filename
line = fh.readline().rstrip()
@@ -369,7 +292,7 @@ def _dump_luacov_stats(statsfile, coverage):
"""
src_files = sorted(coverage)

with open(statsfile, 'wb') as fh:
with open(statsfile, 'w') as fh:
for src in src_files:
stats = " ".join(str(n) for n in coverage[src])
fh.write("%s:%s\n%s\n" % (len(coverage[src]), src, stats))

+ 6
- 35
test/functional/lib/rspamd.robot View File

@@ -17,7 +17,6 @@ Check Pidfile

Check Rspamc
[Arguments] ${result} @{args} &{kwargs}
Follow Rspamd Log
Run Keyword If ${result.rc} != 0 Log ${result.stderr}
${has_rc} = Evaluate 'rc' in $kwargs
${inverse} = Evaluate 'inverse' in $kwargs
@@ -45,19 +44,6 @@ Check Rspamc Match String
Run Keyword If ${inverse} == False Should Contain ${subject} ${str}
... ELSE Should Not Contain ${subject} ${str}

Custom Follow Rspamd Log
[Arguments] ${logfile} ${logpos} ${logpos_var} ${scope}
${logpos} = Log Logs ${logfile} ${logpos}
Run Keyword If '${scope}' == 'Test' Set Test Variable ${${logpos_var}} ${logpos}
... ELSE IF '${scope}' == 'Suite' Set Suite Variable ${${logpos_var}} ${logpos}
... ELSE Fail 'scope must be Test or Suite'

Follow Rspamd Log
${RSPAMD_LOGPOS} = Log Logs ${TMPDIR}/rspamd.log ${RSPAMD_LOGPOS}
Run Keyword If '${RSPAMD_SCOPE}' == 'Test' Set Test Variable ${RSPAMD_LOGPOS}
... ELSE IF '${RSPAMD_SCOPE}' == 'Suite' Set Suite Variable ${RSPAMD_LOGPOS}
... ELSE Fail 'RSPAMD_SCOPE must be Test or Suite'

Generic Setup
[Arguments] @{vargs} &{kwargs}
&{d} = Run Rspamd @{vargs} &{kwargs}
@@ -69,7 +55,6 @@ Generic Setup
END

Generic Teardown
[Arguments] @{ports}
Run Keyword If '${CONTROLLER_ERRORS}' == 'True' Check Controller Errors
Shutdown Process With Children ${RSPAMD_PID}
Log does not contain segfault record
@@ -78,20 +63,11 @@ Generic Teardown
Cleanup Temporary Directory ${TMPDIR}

Log does not contain segfault record
${log} = Get File ${TMPDIR}/rspamd.log
${log} = Get File ${TMPDIR}/rspamd.log encoding_errors=ignore
Should not contain ${log} Segmentation fault: msg=Segmentation fault detected

Log Logs
[Arguments] ${logfile} ${position}
${the_log} ${position} = Read Log From Position ${logfile} ${position}
Log ${the_log}
[Return] ${position}

Normal Teardown
${port_normal} = Create List ${SOCK_STREAM} ${LOCAL_ADDR} ${PORT_NORMAL}
${port_controller} = Create List ${SOCK_STREAM} ${LOCAL_ADDR} ${PORT_CONTROLLER}
${ports} = Create List ${port_normal} ${port_controller}
Generic Teardown @{ports}
Generic Teardown

Redis HSET
[Arguments] ${hash} ${key} ${value}
@@ -169,24 +145,21 @@ Run Rspamd
${result} = Run Process ${RSPAMD} -u ${RSPAMD_USER} -g ${RSPAMD_GROUP}
... -c ${tmpdir}/rspamd.conf env:TMPDIR=${tmpdir} env:DBDIR=${tmpdir} env:LD_LIBRARY_PATH=${TESTDIR}/../../contrib/aho-corasick
Run Keyword If ${result.rc} != 0 Log ${result.stderr}
${rspamd_logpos} = Log Logs ${tmpdir}/rspamd.log 0
Should Be Equal As Integers ${result.rc} 0
Wait Until Keyword Succeeds 10x 1 sec Check Pidfile ${tmpdir}/rspamd.pid timeout=0.5s
Wait Until Keyword Succeeds 5x 1 sec Ping Rspamd ${LOCAL_ADDR} ${PORT_NORMAL}
${rspamd_pid} = Get File ${tmpdir}/rspamd.pid
Set To Dictionary ${d} RSPAMD_LOGPOS=${rspamd_logpos} RSPAMD_PID=${rspamd_pid} TMPDIR=${tmpdir}
Set To Dictionary ${d} RSPAMD_PID=${rspamd_pid} TMPDIR=${tmpdir}
[Return] &{d}

Simple Teardown
Generic Teardown

Scan Message With Rspamc
[Arguments] ${msg_file} @{vargs}
${result} = Run Rspamc -p -h ${LOCAL_ADDR}:${PORT_NORMAL} @{vargs} ${msg_file}
[Return] ${result}

Simple Teardown
${port_normal} = Create List ${SOCK_STREAM} ${LOCAL_ADDR} ${PORT_NORMAL}
${ports} = Create List ${port_normal}
Generic Teardown @{ports}

Sync Fuzzy Storage
[Arguments] @{vargs}
${len} = Get Length ${vargs}
@@ -195,6 +168,4 @@ Sync Fuzzy Storage
... ELSE Run Process ${RSPAMADM} control -s ${vargs}[0]/rspamd.sock
... fuzzy_sync
Log ${result.stdout}
Run Keyword If $len == 0 Follow Rspamd Log
... ELSE Custom Follow Rspamd Log ${vargs}[0]/rspamd.log ${vargs}[1] ${vargs}[2] ${vargs}[3]
Sleep 0.1s Try give fuzzy storage time to sync

+ 3
- 3
test/functional/lib/vars.py View File

@@ -3,7 +3,7 @@ import socket
CONTROLLER_ERRORS = True
KEY_PVT1 = 'ekd3x36tfa5gd76t6pa8hqif3ott7n1siuux68exbkk7ukscte9y'
KEY_PUB1 = 'm8kneubpcjsb8sbsoj7jy7azj9fdd3xmj63txni86a8ye9ncomny'
LOCAL_ADDR = u'127.0.0.1'
LOCAL_ADDR = '127.0.0.1'
MAP_WATCH_INTERVAL = '1min'
PORT_CONTROLLER = 56790
PORT_CONTROLLER_SLAVE = 56793
@@ -17,9 +17,9 @@ PORT_FPROT = 56797
PORT_FPROT2_DUPLICATE = 56798
PORT_AVAST = 56799
P0F_SOCKET = '/tmp/p0f.sock'
REDIS_ADDR = u'127.0.0.1'
REDIS_ADDR = '127.0.0.1'
REDIS_PORT = 56379
NGINX_ADDR = u'127.0.0.1'
NGINX_ADDR = '127.0.0.1'
NGINX_PORT = 56380
RSPAMD_GROUP = 'nogroup'
RSPAMD_USER = 'nobody'

+ 5
- 7
test/functional/util/dummy_avast.py View File

@@ -1,15 +1,13 @@
#!/usr/bin/env python
#!/usr/bin/env python3

PID = "/tmp/dummy_avast.pid"

import os
import sys
import socket
import socketserver
import sys

import dummy_killer
try:
import SocketServer as socketserver
except:
import socketserver

class MyTCPHandler(socketserver.BaseRequestHandler):

@@ -50,6 +48,6 @@ if __name__ == "__main__":
try:
server.handle_request()
except socket.error:
print "Socket closed"
print("Socket closed")

server.server_close()

+ 5
- 7
test/functional/util/dummy_clam.py View File

@@ -1,15 +1,13 @@
#!/usr/bin/env python
#!/usr/bin/env python3

PID = "/tmp/dummy_clamav.pid"

import os
import sys
import socket
import socketserver
import sys

import dummy_killer
try:
import SocketServer as socketserver
except:
import socketserver

class MyTCPHandler(socketserver.BaseRequestHandler):

@@ -47,6 +45,6 @@ if __name__ == "__main__":
try:
server.handle_request()
except socket.error:
print "Socket closed"
print("Socket closed")

server.server_close()

+ 6
- 9
test/functional/util/dummy_fprot.py View File

@@ -1,15 +1,12 @@
#!/usr/bin/env python
#!/usr/bin/env python3

import os
import sys
import signal
import socket
import dummy_killer
import socketserver
import sys

try:
import SocketServer as socketserver
except:
import socketserver
import dummy_killer

PID = "/tmp/dummy_fprot.pid"

@@ -53,5 +50,5 @@ if __name__ == "__main__":
try:
server.handle_request()
except socket.error:
print "Socket closed"
print("Socket closed")
server.server_close()

+ 16
- 17
test/functional/util/dummy_http.py View File

@@ -1,14 +1,13 @@
#!/usr/bin/env python
#!/usr/bin/env python3

import BaseHTTPServer
import SocketServer
import SimpleHTTPServer
import dummy_killer

import time
import http.server
import os
import sys
import socket
import socketserver
import sys
import time

import dummy_killer

PORT = 18080
HOST_NAME = '127.0.0.1'
@@ -16,10 +15,10 @@ HOST_NAME = '127.0.0.1'
PID = "/tmp/dummy_http.pid"


class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
class MyHandler(http.server.BaseHTTPRequestHandler):

def setup(self):
BaseHTTPServer.BaseHTTPRequestHandler.setup(self)
http.server.BaseHTTPRequestHandler.setup(self)
self.protocol_version = "HTTP/1.1" # allow connection: keep-alive

def do_HEAD(self):
@@ -29,7 +28,7 @@ class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
self.log_message("to be closed: " + self.close_connection)

def do_GET(self):
response = "hello world"
response = b"hello world"

"""Respond to a GET request."""
if self.path == "/empty":
@@ -60,7 +59,7 @@ class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler):


def do_POST(self):
response = "hello post"
response = b"hello post"
"""Respond to a GET request."""
if self.path == "/empty":
self.finish()
@@ -82,12 +81,12 @@ class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
self.wfile.write(response)


class ThreadingSimpleServer(SocketServer.ThreadingMixIn,
BaseHTTPServer.HTTPServer):
class ThreadingSimpleServer(socketserver.ThreadingMixIn,
http.server.HTTPServer):
def __init__(self):
self.allow_reuse_address = True
self.timeout = 1
BaseHTTPServer.HTTPServer.__init__(self, (HOST_NAME, PORT), MyHandler)
http.server.HTTPServer.__init__(self, (HOST_NAME, PORT), MyHandler)

def run(self):
dummy_killer.write_pid(PID)
@@ -96,9 +95,9 @@ class ThreadingSimpleServer(SocketServer.ThreadingMixIn,
sys.stdout.flush()
server.handle_request()
except KeyboardInterrupt:
print "Interrupt"
print("Interrupt")
except socket.error:
print "Socket closed"
print("Socket closed")

def stop(self):
self.keep_running = False

+ 19
- 19
test/functional/util/dummy_https.py View File

@@ -1,15 +1,15 @@
#!/usr/bin/env python
#!/usr/bin/env python3

import BaseHTTPServer
import SocketServer
import SimpleHTTPServer
import dummy_killer
import ssl

import time
import http.server
import http.server
import os
import sys
import socket
import socketserver
import ssl
import sys
import time

import dummy_killer

PORT = 18081
HOST_NAME = '127.0.0.1'
@@ -17,10 +17,10 @@ HOST_NAME = '127.0.0.1'
PID = "/tmp/dummy_https.pid"


class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
class MyHandler(http.server.BaseHTTPRequestHandler):

def setup(self):
BaseHTTPServer.BaseHTTPRequestHandler.setup(self)
http.server.BaseHTTPRequestHandler.setup(self)
self.protocol_version = "HTTP/1.1" # allow connection: keep-alive

def do_HEAD(self):
@@ -30,7 +30,7 @@ class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
self.log_message("to be closed: " + self.close_connection)

def do_GET(self):
response = "hello world"
response = b"hello world"

"""Respond to a GET request."""
if self.path == "/empty":
@@ -64,11 +64,11 @@ class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
"""Respond to a POST request."""

content_length = int(self.headers['Content-Length'])
response = 'hello post'
response = b'hello post'

if content_length > 0:
body = self.rfile.read(content_length)
response = "hello post: " + str(len(body))
response = b"hello post: " + bytes(len(body))

if self.path == "/empty":
self.finish()
@@ -90,13 +90,13 @@ class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
self.wfile.write(response)


class ThreadingSimpleServer(SocketServer.ThreadingMixIn,
BaseHTTPServer.HTTPServer):
class ThreadingSimpleServer(socketserver.ThreadingMixIn,
http.server.HTTPServer):
def __init__(self, certfile,
keyfile,):
self.allow_reuse_address = True
self.timeout = 10
BaseHTTPServer.HTTPServer.__init__(self, (HOST_NAME, PORT), MyHandler)
http.server.HTTPServer.__init__(self, (HOST_NAME, PORT), MyHandler)
self.socket = ssl.wrap_socket (self.socket,
keyfile=keyfile,
certfile=certfile, server_side=True)
@@ -108,9 +108,9 @@ class ThreadingSimpleServer(SocketServer.ThreadingMixIn,
sys.stdout.flush()
server.handle_request()
except KeyboardInterrupt:
print "Interrupt"
print("Interrupt")
except socket.error:
print "Socket closed"
print("Socket closed")

def stop(self):
self.keep_running = False

+ 11
- 13
test/functional/util/dummy_p0f.py View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3

PID = "/tmp/dummy_p0f.pid"

@@ -6,11 +6,9 @@ import os
import sys
import struct
import socket
import socketserver

import dummy_killer
try:
import SocketServer as socketserver
except:
import socketserver

class MyStreamHandler(socketserver.BaseRequestHandler):

@@ -22,8 +20,8 @@ class MyStreamHandler(socketserver.BaseRequestHandler):
}

OS = {
'windows' : ('Windows', '7 or 8'),
'linux' : ('Linux', '3.11 and newer')
'windows' : (b'Windows', b'7 or 8'),
'linux' : (b'Linux', b'3.11 and newer')
}

self.data = self.request.recv(21).strip()
@@ -47,10 +45,10 @@ class MyStreamHandler(socketserver.BaseRequestHandler):
0, # os_match_q
OS[self.server.p0f_os][0], # os_name
OS[self.server.p0f_os][1], # os_flavor
'', # http_name
'', # http_flavor
'Ethernet or modem', # link_type
'' # language
b'', # http_name
b'', # http_flavor
b'Ethernet or modem', # link_type
b'' # language
)

self.request.sendall(response)
@@ -61,7 +59,7 @@ def cleanup(SOCK):
try:
os.unlink(SOCK)
except OSError:
print "Could not unlink socket: " + SOCK
print("Could not unlink socket: " + SOCK)

if __name__ == "__main__":
SOCK = '/tmp/p0f.sock'
@@ -92,7 +90,7 @@ if __name__ == "__main__":
try:
server.handle_request()
except socket.error:
print "Socket closed"
print("Socket closed")

server.server_close()
cleanup(SOCK)

+ 24
- 32
test/functional/util/dummy_ssl.py View File

@@ -1,64 +1,58 @@
#!/usr/bin/env python
#!/usr/bin/env python3

import SocketServer
import dummy_killer

import time
import os
import sys
import socket
import ssl
import sys
import time

import dummy_killer
import socketserver

PORT = 14433
HOST_NAME = '127.0.0.1'

PID = "/tmp/dummy_ssl.pid"

class SSLTCPHandler(SocketServer.BaseRequestHandler):
class SSLTCPHandler(socketserver.StreamRequestHandler):
def handle(self):
time.sleep(0.5)
data = self.request.recv(6000000)
while data:
print "{} wrote:".format(self.client_address[0])
print data
print("{} wrote:".format(self.client_address[0]))
print(data)
time.sleep(0.1)
self.request.sendall('hello\n')
self.request.sendall(b'hello\n')
time.sleep(0.1)
data = self.request.recv(6000000)

class SSL_TCP_Server(SocketServer.TCPServer):
class SSL_TCP_Server(socketserver.ThreadingMixIn, socketserver.TCPServer):
def __init__(self,
server_address,
RequestHandlerClass,
certfile,
keyfile,
ssl_version=ssl.PROTOCOL_TLSv1,
bind_and_activate=True):
self.allow_reuse_address = True
SocketServer.TCPServer.__init__(self, server_address,
RequestHandlerClass, bind_and_activate)
self.certfile = certfile
self.keyfile = keyfile
self.ssl_version = ssl_version
#self.timeout = 1

def get_request(self):
newsocket, fromaddr = self.socket.accept()
connstream = ssl.wrap_socket(newsocket,
server_side=True,
certfile = self.certfile,
keyfile = self.keyfile,
ssl_version = self.ssl_version)
return connstream, fromaddr
super().__init__(server_address, RequestHandlerClass, False)
self.timeout = 1
ctx = ssl.create_default_context()
ctx.load_cert_chain(certfile=certfile)
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
self.socket = ctx.wrap_socket(self.socket, server_side=True)
if (bind_and_activate):
self.server_bind()
self.server_activate()

def run(self):
dummy_killer.write_pid(PID)
try:
self.serve_forever()
except KeyboardInterrupt:
print "Interrupt"
print("Interrupt")
except socket.error as e:
print "Socket closed {}".format(e)
print("Socket closed {}".format(e))
finally:
self.server_close()

@@ -66,9 +60,7 @@ class SSL_TCP_Server(SocketServer.TCPServer):
self.keep_running = False
self.server_close()

class SSL_ThreadingTCPServer(SocketServer.ThreadingMixIn, SSL_TCP_Server): pass

if __name__ == '__main__':
server = SSL_ThreadingTCPServer((HOST_NAME, PORT), SSLTCPHandler, sys.argv[1], sys.argv[1])
server = SSL_TCP_Server((HOST_NAME, PORT), SSLTCPHandler, sys.argv[1], sys.argv[1])
dummy_killer.setup_killer(server, server.stop)
server.run()

+ 4
- 3
test/functional/util/dummy_udp.py View File

@@ -1,7 +1,8 @@
#!/usr/bin/env python
#!/usr/bin/env python3

import socket
import sys

import dummy_killer

UDP_IP = "127.0.0.1"
@@ -21,5 +22,5 @@ if __name__ == "__main__":

while True:
data, addr = sock.recvfrom(1024) # buffer size is 1024 bytes
print "received message:", data
sock.sendto(data, addr)
print("received message:", data)
sock.sendto(data, addr)

Write Preview
Loading…
Cancel
Save