From da9555606f6772434e5a565cb00bc0d1c2c2a4db Mon Sep 17 00:00:00 2001
From: Andrew Lewis <nerf@judo.za.org>
Date: Mon, 25 Jul 2016 17:06:49 +0100
Subject: [PATCH] [Test] Try harder to kill rspamd + child processes to avoid
 cascading failures

---
 circle.yml                                 |  7 +---
 test/functional/cases/statistics/lib.robot |  2 +-
 test/functional/lib/rspamd.py              | 37 ++++++++++++++++++----
 test/functional/lib/rspamd.robot           |  2 +-
 4 files changed, 33 insertions(+), 15 deletions(-)

diff --git a/circle.yml b/circle.yml
index 596a1469c..7fb9e4a91 100644
--- a/circle.yml
+++ b/circle.yml
@@ -1,13 +1,8 @@
 dependencies:
     pre:
       - sudo apt-get update -qq
-      - sudo apt-get install -qq cmake libevent-dev libglib2.0-dev libgmime-2.6-dev libluajit-5.1-dev liblua5.1-0-dev libmagic-dev libpcre3-dev libsqlite3-dev libssl-dev make ragel redis-server gcc
+      - sudo apt-get install -qq cmake gcc libevent-dev libglib2.0-dev libgmime-2.6-dev libluajit-5.1-dev liblua5.1-0-dev libmagic-dev libpcre3-dev libsqlite3-dev libssl-dev make python-psutil ragel redis-server
       - sudo pip install robotframework
-#  - echo $TRAVIS_OS_NAME
-#  - if [ "$TRAVIS_OS_NAME" == "linux" ]; then sudo apt-get update -qq ; fi
-#  - if [ "$TRAVIS_OS_NAME" == "linux" ]; then sudo apt-get install -qq cmake libevent-dev libglib2.0-dev libgmime-2.6-dev libluajit-5.1-dev libpcre3-dev libsqlite3-dev libhiredis-dev ; fi
-#  - if [ "$TRAVIS_OS_NAME" == "osx" ]; then brew update ; fi
-#  - if [ "$TRAVIS_OS_NAME" == "osx" ]; then brew install cmake gmime glib luajit pcre hiredis sqlite libevent ; fi
 
 test:
   override:
diff --git a/test/functional/cases/statistics/lib.robot b/test/functional/cases/statistics/lib.robot
index 1a6bc80b6..5f9e96038 100644
--- a/test/functional/cases/statistics/lib.robot
+++ b/test/functional/cases/statistics/lib.robot
@@ -48,7 +48,7 @@ Redis Statistics Setup
 
 Redis Statistics Teardown
   Generic Teardown
-  Shutdown Process  ${REDIS_PID}
+  Shutdown Process With Children  ${REDIS_PID}
 
 Statistics Setup
   Generic Setup  STATS_PATH_CACHE  STATS_PATH_HAM  STATS_PATH_SPAM
diff --git a/test/functional/lib/rspamd.py b/test/functional/lib/rspamd.py
index ef7c6e29c..a90b924bf 100644
--- a/test/functional/lib/rspamd.py
+++ b/test/functional/lib/rspamd.py
@@ -1,6 +1,7 @@
 import grp
 import os
 import os.path
+import psutil
 import pwd
 import shutil
 import signal
@@ -23,16 +24,24 @@ def cleanup_temporary_directory(directory):
 def encode_filename(filename):
     return "".join(['%%%0X' % ord(b) for b in filename])
 
+def get_process_children(pid):
+    children = []
+    for p in psutil.process_iter():
+        # ppid could be int or function depending on library version
+        if callable(p.ppid):
+            ppid = p.ppid()
+        else:
+            ppid = p.ppid
+        if ppid == pid:
+            children.append(p.pid)
+    return children
+
 def get_test_directory():
     return os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "../..")
 
 def make_temporary_directory():
     return tempfile.mkdtemp()
 
-def process_should_exist(pid):
-    pid = int(pid)
-    os.kill(pid, 0)
-
 def read_log_from_position(filename, offset):
     offset = long(offset)
     f = open(filename, 'rb')
@@ -83,14 +92,28 @@ def update_dictionary(a, b):
     return a
 
 def shutdown_process(pid):
-    pid = int(pid)
-    process_should_exist(pid)
     i = 0
     while i < 100:
         try:
             os.kill(pid, signal.SIGTERM)
         except OSError as e:
             assert e.errno == 3
-            break
+            return
         i += 1
         time.sleep(0.1)
+    while i < 200:
+        try:
+            os.kill(pid, signal.SIGKILL)
+        except OSError as e:
+            assert e.errno == 3
+            return
+        i += 1
+        time.sleep(0.1)
+    assert False, "Failed to shutdown process %s" % pid
+
+def shutdown_process_with_children(pid):
+    pid = int(pid)
+    children = get_process_children(pid)
+    shutdown_process(pid)
+    for child in children:
+        shutdown_process(child)
diff --git a/test/functional/lib/rspamd.robot b/test/functional/lib/rspamd.robot
index fe5ee83fb..51030c84f 100644
--- a/test/functional/lib/rspamd.robot
+++ b/test/functional/lib/rspamd.robot
@@ -62,7 +62,7 @@ Generic Setup
   ...  ELSE  Fail  'RSPAMD_SCOPE must be Test or Suite'
 
 Generic Teardown
-  Shutdown Process  ${RSPAMD_PID}
+  Shutdown Process With Children  ${RSPAMD_PID}
   Cleanup Temporary Directory  ${TMPDIR}
 
 Log Logs
-- 
2.39.5