[Test] Try harder to kill rspamd + child processes to avoid cascading failures

3 files changed, 32 insertions, 9 deletions

diff --git a/test/functional/cases/statistics/lib.robot b/test/functional/cases/statistics/lib.robot
index 1a6bc80b6..5f9e96038 100644
--- a/test/functional/cases/statistics/lib.robot
+++ b/test/functional/cases/statistics/lib.robot
@@ -48,7 +48,7 @@ Redis Statistics Setup
 
 Redis Statistics Teardown
   Generic Teardown
-  Shutdown Process  ${REDIS_PID}
+  Shutdown Process With Children  ${REDIS_PID}
 
 Statistics Setup
   Generic Setup  STATS_PATH_CACHE  STATS_PATH_HAM  STATS_PATH_SPAM
diff --git a/test/functional/lib/rspamd.py b/test/functional/lib/rspamd.py
index ef7c6e29c..a90b924bf 100644
--- a/test/functional/lib/rspamd.py
+++ b/test/functional/lib/rspamd.py
@@ -1,6 +1,7 @@
 import grp
 import os
 import os.path
+import psutil
 import pwd
 import shutil
 import signal
@@ -23,16 +24,24 @@ def cleanup_temporary_directory(directory):
 def encode_filename(filename):
     return "".join(['%%%0X' % ord(b) for b in filename])
 
+def get_process_children(pid):
+    children = []
+    for p in psutil.process_iter():
+        # ppid could be int or function depending on library version
+        if callable(p.ppid):
+            ppid = p.ppid()
+        else:
+            ppid = p.ppid
+        if ppid == pid:
+            children.append(p.pid)
+    return children
+
 def get_test_directory():
     return os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "../..")
 
 def make_temporary_directory():
     return tempfile.mkdtemp()
 
-def process_should_exist(pid):
-    pid = int(pid)
-    os.kill(pid, 0)
-
 def read_log_from_position(filename, offset):
     offset = long(offset)
     f = open(filename, 'rb')
@@ -83,14 +92,28 @@ def update_dictionary(a, b):
     return a
 
 def shutdown_process(pid):
-    pid = int(pid)
-    process_should_exist(pid)
     i = 0
     while i < 100:
         try:
             os.kill(pid, signal.SIGTERM)
         except OSError as e:
             assert e.errno == 3
-            break
+            return
         i += 1
         time.sleep(0.1)
+    while i < 200:
+        try:
+            os.kill(pid, signal.SIGKILL)
+        except OSError as e:
+            assert e.errno == 3
+            return
+        i += 1
+        time.sleep(0.1)
+    assert False, "Failed to shutdown process %s" % pid
+
+def shutdown_process_with_children(pid):
+    pid = int(pid)
+    children = get_process_children(pid)
+    shutdown_process(pid)
+    for child in children:
+        shutdown_process(child)
diff --git a/test/functional/lib/rspamd.robot b/test/functional/lib/rspamd.robot
index fe5ee83fb..51030c84f 100644
--- a/test/functional/lib/rspamd.robot
+++ b/test/functional/lib/rspamd.robot
@@ -62,7 +62,7 @@ Generic Setup
   ...  ELSE  Fail  'RSPAMD_SCOPE must be Test or Suite'
 
 Generic Teardown
-  Shutdown Process  ${RSPAMD_PID}
+  Shutdown Process With Children  ${RSPAMD_PID}
   Cleanup Temporary Directory  ${TMPDIR}
 
 Log Logs