[Test] Wait for ports to be confirmed-free inbetween runs

circle.yml

@@ -3,7 +3,7 @@ dependencies:
       - sudo apt-get update -qq || true
       - sudo apt-get install -qq cmake gcc lcov libevent-dev libglib2.0-dev libgmime-2.6-dev libluajit-5.1-dev liblua5.1-0-dev libmagic-dev libpcre3-dev libsqlite3-dev libssl-dev make python-dev ragel redis-server
       - gem install coveralls-lcov
-      - sudo pip install demjson psutil robotframework
+      - sudo pip install demjson ipaddress psutil robotframework
       - wget http://download.redis.io/redis-stable.tar.gz
       - tar xvzf redis-stable.tar.gz
       - cd redis-stable && make && sudo cp src/redis-cli /usr/bin/

test/functional/cases/100_general.robot

@@ -1,6 +1,6 @@
 *** Settings ***
 Suite Setup     Generic Setup
-Suite Teardown  Generic Teardown
+Suite Teardown  Simple Teardown
 Library         ${TESTDIR}/lib/rspamd.py
 Resource        ${TESTDIR}/lib/rspamd.robot
 Variables       ${TESTDIR}/lib/vars.py

test/functional/cases/101_lua.robot

@@ -1,5 +1,5 @@
 *** Settings ***
-Test Teardown   Generic Teardown
+Test Teardown   Normal Teardown
 Library         ${TESTDIR}/lib/rspamd.py
 Resource        ${TESTDIR}/lib/rspamd.robot
 Variables       ${TESTDIR}/lib/vars.py

test/functional/cases/102_multimap.robot

@@ -246,5 +246,6 @@ Multimap Setup
   Run Redis
 
 Multimap Teardown
+  Normal Teardown
   Shutdown Process With Children  ${REDIS_PID}
-  Generic Teardown
+  Wait For Port  ${SOCK_STREAM}  ${LOCAL_ADDR}  ${REDIS_PORT}

test/functional/cases/103_password.robot

@@ -1,5 +1,5 @@
 *** Settings ***
-Test Teardown   Generic Teardown
+Test Teardown   Normal Teardown
 Library         ${TESTDIR}/lib/rspamd.py
 Resource        ${TESTDIR}/lib/rspamd.robot
 Variables       ${TESTDIR}/lib/vars.py

test/functional/cases/105_mimetypes.robot

@@ -1,6 +1,6 @@
 *** Settings ***
 Suite Setup     MIMETypes Setup
-Suite Teardown  Generic Teardown
+Suite Teardown  Simple Teardown
 Library         ${TESTDIR}/lib/rspamd.py
 Resource        ${TESTDIR}/lib/rspamd.robot
 Variables       ${TESTDIR}/lib/vars.py

test/functional/cases/110_statistics/lib.robot

@@ -47,11 +47,12 @@ Redis Statistics Setup
   Run Redis
 
 Redis Statistics Teardown
-  Generic Teardown
+  Normal Teardown
   Shutdown Process With Children  ${REDIS_PID}
+  Wait For Port  ${SOCK_STREAM}  ${LOCAL_ADDR}  ${REDIS_PORT}
 
 Statistics Setup
   Generic Setup  STATS_PATH_CACHE  STATS_PATH_HAM  STATS_PATH_SPAM
 
 Statistics Teardown
-  Generic Teardown
+  Normal Teardown

test/functional/cases/110_statistics/sqlite-broken-stats-dir.robot

@@ -1,6 +1,6 @@
 *** Settings ***
 Suite Setup     Generic Setup
-Suite Teardown  Generic Teardown
+Suite Teardown  Normal Teardown
 Resource        ${TESTDIR}/lib/rspamd.robot
 Resource        lib.robot
 

test/functional/cases/115_dmarc.robot

@@ -1,6 +1,6 @@
 *** Settings ***
 Suite Setup     DMARC Setup
-Suite Teardown  Generic Teardown
+Suite Teardown  Simple Teardown
 Library         ${TESTDIR}/lib/rspamd.py
 Resource        ${TESTDIR}/lib/rspamd.robot
 Variables       ${TESTDIR}/lib/vars.py

test/functional/cases/120_fuzzy/encrypted.robot

@@ -1,6 +1,6 @@
 *** Settings ***
 Suite Setup     Fuzzy Setup Encrypted Siphash
-Suite Teardown  Generic Teardown
+Suite Teardown  Fuzzy Teardown
 Resource        lib.robot
 
 *** Test Cases ***

test/functional/cases/120_fuzzy/fasthash-keyed.robot

@@ -1,6 +1,6 @@
 *** Settings ***
 Suite Setup     Fuzzy Setup Keyed Fasthash
-Suite Teardown  Generic Teardown
+Suite Teardown  Fuzzy Teardown
 Resource        lib.robot
 
 *** Test Cases ***

test/functional/cases/120_fuzzy/fasthash.robot

@@ -1,6 +1,6 @@
 *** Settings ***
 Suite Setup     Fuzzy Setup Plain Fasthash
-Suite Teardown  Generic Teardown
+Suite Teardown  Fuzzy Teardown
 Resource        lib.robot
 
 *** Test Cases ***

test/functional/cases/120_fuzzy/general.robot

@@ -1,6 +1,6 @@
 *** Settings ***
 Suite Setup     Fuzzy Setup Plain Siphash
-Suite Teardown  Generic Teardown
+Suite Teardown  Fuzzy Teardown
 Resource        lib.robot
 
 *** Test Cases ***

test/functional/cases/120_fuzzy/lib.robot

@@ -143,3 +143,10 @@ Fuzzy Multimessage Delete Test
 Fuzzy Multimessage Overwrite Test
   : FOR  ${i}  IN  @{MESSAGES}
   \  Fuzzy Overwrite Test  ${i}
+
+Fuzzy Teardown
+  ${port_normal} =  Create List  ${SOCK_STREAM}  ${LOCAL_ADDR}  ${PORT_NORMAL}
+  ${port_fuzzy} =  Create List  ${SOCK_DGRAM}  ${LOCAL_ADDR}  ${PORT_FUZZY}
+  ${port_controller} =  Create List  ${SOCK_STREAM}  ${LOCAL_ADDR}  ${PORT_CONTROLLER}
+  ${ports} =  Create List  ${port_normal}  ${port_fuzzy}  ${port_controller}
+  Generic Teardown  @{ports}

test/functional/cases/120_fuzzy/mumhash-keyed.robot

@@ -1,6 +1,6 @@
 *** Settings ***
 Suite Setup     Fuzzy Setup Keyed Mumhash
-Suite Teardown  Generic Teardown
+Suite Teardown  Fuzzy Teardown
 Resource        lib.robot
 
 *** Test Cases ***

test/functional/cases/120_fuzzy/mumhash.robot

@@ -1,6 +1,6 @@
 *** Settings ***
 Suite Setup     Fuzzy Setup Plain Mumhash
-Suite Teardown  Generic Teardown
+Suite Teardown  Fuzzy Teardown
 Resource        lib.robot
 
 *** Test Cases ***

test/functional/cases/120_fuzzy/replication.robot

@@ -82,3 +82,12 @@ Replication Teardown
   Cleanup Temporary Directory  ${SLAVE_TMPDIR}
   Remove File  ${TMP_INCLUDE1}
   Remove File  ${TMP_INCLUDE2}
+  ${port_normal} =  Create List  ${SOCK_STREAM}  ${LOCAL_ADDR}  ${PORT_NORMAL}
+  ${port_fuzzy} =  Create List  ${SOCK_DGRAM}  ${LOCAL_ADDR}  ${PORT_FUZZY}
+  ${port_controller} =  Create List  ${SOCK_STREAM}  ${LOCAL_ADDR}  ${PORT_CONTROLLER}
+  ${port_normal_slave} =  Create List  ${SOCK_STREAM}  ${LOCAL_ADDR}  ${PORT_NORMAL_SLAVE}
+  ${port_fuzzy_slave} =  Create List  ${SOCK_DGRAM}  ${LOCAL_ADDR}  ${PORT_FUZZY_SLAVE}
+  ${port_controller_slave} =  Create List  ${SOCK_STREAM}  ${LOCAL_ADDR}  ${PORT_CONTROLLER_SLAVE}
+  ${ports} =  Create List  ${port_normal}  ${port_fuzzy}  ${port_controller}  ${port_normal_slave}  ${port_fuzzy_slave}  ${port_controller_slave}
+  : FOR  ${i}  IN  @{ports}
+  \  Wait For Port  @{i}[0]  @{i}[1]  @{i}[2]

test/functional/cases/120_fuzzy/siphash-keyed.robot

@@ -1,6 +1,6 @@
 *** Settings ***
 Suite Setup     Fuzzy Setup Keyed Siphash
-Suite Teardown  Generic Teardown
+Suite Teardown  Fuzzy Teardown
 Resource        lib.robot
 
 *** Test Cases ***

test/functional/cases/120_fuzzy/siphash.robot

@@ -1,6 +1,6 @@
 *** Settings ***
 Suite Setup     Fuzzy Setup Plain Siphash
-Suite Teardown  Generic Teardown
+Suite Teardown  Fuzzy Teardown
 Resource        lib.robot
 
 *** Test Cases ***

test/functional/cases/120_fuzzy/xxhash-keyed.robot

@@ -1,6 +1,6 @@
 *** Settings ***
 Suite Setup     Fuzzy Setup Keyed Xxhash
-Suite Teardown  Generic Teardown
+Suite Teardown  Fuzzy Teardown
 Resource        lib.robot
 
 *** Test Cases ***

test/functional/cases/120_fuzzy/xxhash.robot

@@ -1,6 +1,6 @@
 *** Settings ***
 Suite Setup     Fuzzy Setup Plain Xxhash
-Suite Teardown  Generic Teardown
+Suite Teardown  Fuzzy Teardown
 Resource        lib.robot
 
 *** Test Cases ***

test/functional/cases/121_json/100_preresult.robot

@@ -1,6 +1,6 @@
 *** Settings ***
 Suite Setup     Generic Setup
-Suite Teardown  Generic Teardown
+Suite Teardown  Simple Teardown
 Library         ${TESTDIR}/lib/rspamd.py
 Resource        lib.robot
 Resource        ${TESTDIR}/lib/rspamd.robot

test/functional/cases/121_json/101_simple.robot

@@ -1,6 +1,6 @@
 *** Settings ***
 Suite Setup     Generic Setup
-Suite Teardown  Generic Teardown
+Suite Teardown  Simple Teardown
 Library         ${TESTDIR}/lib/rspamd.py
 Resource        lib.robot
 Resource        ${TESTDIR}/lib/rspamd.robot

test/functional/cases/130_dkim.robot

@@ -1,6 +1,6 @@
 *** Settings ***
 Suite Setup     Generic Setup
-Suite Teardown  Generic Teardown
+Suite Teardown  Simple Teardown
 Library         ${TESTDIR}/lib/rspamd.py
 Resource        ${TESTDIR}/lib/rspamd.robot
 Variables       ${TESTDIR}/lib/vars.py

test/functional/cases/135_spamassassin.robot

@@ -1,6 +1,6 @@
 *** Settings ***
 Suite Setup     SpamAssassin Setup
-Suite Teardown  Generic Teardown
+Suite Teardown  Simple Teardown
 Library         ${TESTDIR}/lib/rspamd.py
 Resource        ${TESTDIR}/lib/rspamd.robot
 Variables       ${TESTDIR}/lib/vars.py

test/functional/cases/140_proxy.robot

@@ -47,3 +47,8 @@ Proxy Teardown
   Shutdown Process With Children  ${SLAVE_PID}
   Cleanup Temporary Directory  ${PROXY_TMPDIR}
   Cleanup Temporary Directory  ${SLAVE_TMPDIR}
+  ${port_normal} =  Create List  ${SOCK_STREAM}  ${LOCAL_ADDR}  ${PORT_NORMAL}
+  ${port_proxy} =  Create List  ${SOCK_STREAM}  ${LOCAL_ADDR}  ${PORT_PROXY}
+  ${ports} =  Create List  ${port_normal}  ${port_proxy}
+  : FOR  ${i}  IN  @{ports}
+  \  Wait For Port  @{i}[0]  @{i}[1]  @{i}[2]

test/functional/lib/rspamd.py

@@ -1,5 +1,6 @@
 import demjson
 import grp
+import ipaddress
 import os
 import os.path
 import psutil
@@ -155,4 +156,26 @@ def shutdown_process_with_children(pid):
     children = psutil.Process(pid=pid).children(recursive=False)
     shutdown_process(pid)
     for child in children:
-        shutdown_process(child.pid)
+        try:
+            shutdown_process(child.pid)
+        except:
+            pass
+
+def wait_for_port(proto, addr, num):
+    i = ipaddress.ip_address(addr)
+    if (i.version == 6):
+        af = socket.AF_INET6
+    else:
+        af = socket.AF_INET
+    while True:
+        try:
+            s = socket.socket(af, proto)
+            s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            s.bind((addr, num))
+            if (proto == socket.SOCK_STREAM):
+                s.listen(0)
+            break
+        except:
+            pass
+            time.sleep(0.1)
+    s.close()

test/functional/lib/rspamd.robot

@@ -62,8 +62,11 @@ Generic Setup
   \  ...  ELSE  Fail  'RSPAMD_SCOPE must be Test or Suite'
 
 Generic Teardown
+  [Arguments]  @{ports}
   Shutdown Process With Children  ${RSPAMD_PID}
   Cleanup Temporary Directory  ${TMPDIR}
+  : FOR  ${i}  IN  @{ports}
+  \  Wait For Port  @{i}[0]  @{i}[1]  @{i}[2]
 
 Log Logs
   [Arguments]  ${logfile}  ${position}
@@ -71,6 +74,12 @@ Log Logs
   Log  ${the_log}
   [Return]  ${position}
 
+Normal Teardown
+  ${port_normal} =  Create List  ${SOCK_STREAM}  ${LOCAL_ADDR}  ${PORT_NORMAL}
+  ${port_controller} =  Create List  ${SOCK_STREAM}  ${LOCAL_ADDR}  ${PORT_CONTROLLER}
+  ${ports} =  Create List  ${port_normal}  ${port_controller}
+  Generic Teardown  @{ports}
+
 Redis HSET
   [Arguments]  ${hash}  ${key}  ${value}
   ${result} =  Run Process  redis-cli  -h  ${REDIS_ADDR}  -p  ${REDIS_PORT}
@@ -129,6 +138,11 @@ Scan Message With Rspamc
   ${result} =  Run Rspamc  -p  -h  ${LOCAL_ADDR}:${PORT_NORMAL}  @{vargs}  ${msg_file}
   [Return]  ${result}
 
+Simple Teardown
+  ${port_normal} =  Create List  ${SOCK_STREAM}  ${LOCAL_ADDR}  ${PORT_NORMAL}
+  ${ports} =  Create List  ${port_normal}
+  Generic Teardown  @{ports}
+
 Sync Fuzzy Storage
   [Arguments]  @{vargs}
   ${len} =  Get Length  ${vargs}

test/functional/lib/vars.py

@@ -1,6 +1,8 @@
+import socket
+
 KEY_PVT1 = 'ekd3x36tfa5gd76t6pa8hqif3ott7n1siuux68exbkk7ukscte9y'
 KEY_PUB1 = 'm8kneubpcjsb8sbsoj7jy7azj9fdd3xmj63txni86a8ye9ncomny'
-LOCAL_ADDR = '127.0.0.1'
+LOCAL_ADDR = u'127.0.0.1'
 PORT_CONTROLLER = 56790
 PORT_CONTROLLER_SLAVE = 56793
 PORT_FUZZY = 56791
@@ -8,7 +10,9 @@ PORT_FUZZY_SLAVE = 56792
 PORT_NORMAL = 56789
 PORT_NORMAL_SLAVE = 56794
 PORT_PROXY = 56795
-REDIS_ADDR = '127.0.0.1'
+REDIS_ADDR = u'127.0.0.1'
 REDIS_PORT = 56379
 RSPAMD_GROUP = 'nogroup'
 RSPAMD_USER = 'nobody'
+SOCK_DGRAM = socket.SOCK_DGRAM
+SOCK_STREAM = socket.SOCK_STREAM