Merge pull request #2574 from negram/store-and-merge-lua-coverage

Store and merge lua coverage

4 files changed, 279 insertions, 30 deletions

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 19a2d3057..039954f26 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -11,36 +11,30 @@ references:
       name: Capturing coverage data
       command: |
         set -e
-        sudo apt-get install -qq lcov
-        gem install coveralls-lcov
-        lcov --no-external -b ../project -d ../project -c --output-file coverage.${CIRCLE_JOB}.info
+        sudo apt-get install -qq python-pip
+        sudo pip install cpp-coveralls
 
-  - &capture_lua_coverage_data
-    run:
-      name: Capturing Lua coverage data
-      command: |
-        set -e
-        if [ ! -z $COVERALLS_REPO_TOKEN ]; then luacov-coveralls -t ${COVERALLS_REPO_TOKEN} || true; fi
-
-  - &restore_coverage_data
-    restore_cache:
-      keys:
-        - coverage-{{ .Environment.CIRCLE_WORKFLOW_ID }}
+        # further, these files will be saved in cache and used in "send-coverage" step
+        # see "merge_and_upload_coverage_data" and "save_cache" records
+        coveralls --dump coverage.${CIRCLE_JOB}.dump
 
   - &merge_and_upload_coverage_data
     run:
       name: Merging and uploading coverage data
       command: |
         set -e
-        if [ -f ~/project/coverage.rspamd-test.info ] && [ -f ~/project/coverage.functional.info ]; then
-            sudo apt-get install -qq lcov
-            lcov -a ~/project/coverage.rspamd-test.info -t rspamd-test -a ~/project/coverage.functional.info -t functional -o coverage.info
-            gem install coveralls-lcov
-            sudo pip install cpp-coveralls
-            sudo luarocks install luacov-coveralls
+        if [ -f ~/project/coverage.rspamd-test.dump ] && [ -f ~/project/coverage.functional.dump ]; then
+            sudo apt-get install -qq python-pip python-dev
+            sudo pip install --upgrade setuptools
+            sudo pip install --upgrade pyOpenSSL
+            sudo pip install cpp-coveralls requests cryptography
+
+            cd ~/project
             if [ ! -z $COVERALLS_REPO_TOKEN ]; then
-              coveralls --lcov-file coverage.info --dump coveralls.dump
-              luacov-coveralls -t ${COVERALLS_REPO_TOKEN} -j coveralls.dump --root=../project
+              # Merge Lua coverage (collected into lua_coverage_report.json) and with C-coverage
+              # (in coverage.rspamd-test.dump, coverage.functional.dump, see &capture_coverage_data)
+              # and finally upload it into coveralls.io
+              test/functional/util/merge_coveralls.py --input coverage.functional.dump coverage.rspamd-test.dump lua_coverage_report.json unit_test_lua.json --output out.josn --token=${COVERALLS_REPO_TOKEN}
             fi
         fi
 
@@ -80,20 +74,21 @@ jobs:
       - run: sudo apt-get install -qq cmake libevent-dev libglib2.0-dev libicu-dev libluajit-5.1-dev libmagic-dev libsqlite3-dev libssl-dev ragel libunwind-dev libunwind8 luarocks
       - run: sudo luarocks install luacheck
       - run: sudo luarocks install luacov
+      - run: sudo luarocks install luacov-coveralls
 
       - run: cd ../build
       - run: make rspamd-test -j`nproc`
       - run: set +e; test/rspamd-test -p /rspamd/lua; echo "export RETURN_CODE=$?" >> $BASH_ENV
+      - run: luacov-coveralls -o unit_test_lua.json --dryrun
 
       - *capture_coverage_data
-      - *capture_lua_coverage_data
 
       # Share coverage data between jobs
       - save_cache:
-          key: coverage-{{ .Environment.CIRCLE_WORKFLOW_ID }}
+          key: coverage-rspamd-test-{{ .Environment.CIRCLE_WORKFLOW_ID }}
           paths:
-            - coverage.rspamd-test.info
-            - luacov.stats.out
+            - coverage.rspamd-test.dump
+            - unit_test_lua.json
 
       - run: (exit $RETURN_CODE)
 
@@ -107,12 +102,13 @@ jobs:
       - run: sudo apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4    # optional, clickhouse key
 
       - run: sudo apt-get update -qq || true
-      - run: sudo apt-get install -qq libluajit-5.1-dev libpcre3-dev luarocks opendkim-tools python-pip redis-server libunwind8 libglib2.0-dev libicu-dev libevent-dev
+      - run: sudo apt-get install -qq libluajit-5.1-dev libpcre3-dev luarocks opendkim-tools python-pip redis-server libunwind8 libglib2.0-dev libicu-dev libevent-dev python-dev
       - run: sudo apt-get install clickhouse-server
 
       - run: sudo pip install demjson psutil robotframework requests http
       - run: sudo luarocks install luacheck
       - run: sudo luarocks install luacov
+      - run: sudo luarocks install luacov-coveralls
 
       - run: cd ../build
       # see coverage notice in "build" stage
@@ -122,9 +118,10 @@ jobs:
 
       # Share coverage data between jobs
       - save_cache:
-          key: coverage-{{ .Environment.CIRCLE_WORKFLOW_ID }}
+          key: coverage-functional-{{ .Environment.CIRCLE_WORKFLOW_ID }}
           paths:
-            - coverage.functional.info
+            - coverage.functional.dump
+            - lua_coverage_report.json
 
       - store_artifacts:
           path: output.xml
@@ -161,10 +158,16 @@ jobs:
     steps:
       - attach_workspace:
           at: *workspace_root
+      - restore_cache:
+          key: coverage-rspamd-test-{{ .Environment.CIRCLE_WORKFLOW_ID }}
+      - restore_cache:
+          key: coverage-functional-{{ .Environment.CIRCLE_WORKFLOW_ID }}
 
-      - *restore_coverage_data
       - *merge_and_upload_coverage_data
 
+      - store_artifacts:
+          path: out.josn
+
 notify:
   webhooks:
     - url: https://coveralls.io/webhook?repo_token={{ .Environment.COVERALLS_REPO_TOKEN }}
diff --git a/test/functional/lib/rspamd.py b/test/functional/lib/rspamd.py
index 313e8393e..e0454e347 100644
--- a/test/functional/lib/rspamd.py
+++ b/test/functional/lib/rspamd.py
@@ -3,6 +3,7 @@ import grp
 import os
 import os.path
 import psutil
+import glob
 import pwd
 import re
 import shutil
@@ -12,6 +13,7 @@ import errno
 import sys
 import tempfile
 import time
+import subprocess
 from robot.libraries.BuiltIn import BuiltIn
 from robot.api import logger
 
@@ -227,3 +229,107 @@ def get_file_if_exists(file_path):
             return myfile.read()
     return None
 
+# copy-paste from 
+# https://hg.python.org/cpython/file/6860263c05b3/Lib/shutil.py#l1068
+# As soon as we move to Python 3, this should be removed in favor of shutil.which()
+def python3_which(cmd, mode=os.F_OK | os.X_OK, path=None):
+    """Given a command, mode, and a PATH string, return the path which
+    conforms to the given mode on the PATH, or None if there is no such
+    file.
+
+    `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result
+    of os.environ.get("PATH"), or can be overridden with a custom search
+    path.
+    """
+
+    # Check that a given file can be accessed with the correct mode.
+    # Additionally check that `file` is not a directory, as on Windows
+    # directories pass the os.access check.
+    def _access_check(fn, mode):
+        return (os.path.exists(fn) and os.access(fn, mode)
+                and not os.path.isdir(fn))
+
+    # If we're given a path with a directory part, look it up directly rather
+    # than referring to PATH directories. This includes checking relative to the
+    # current directory, e.g. ./script
+    if os.path.dirname(cmd):
+        if _access_check(cmd, mode):
+            return cmd
+        return None
+
+    if path is None:
+        path = os.environ.get("PATH", os.defpath)
+    if not path:
+        return None
+    path = path.split(os.pathsep)
+
+    if sys.platform == "win32":
+        # The current directory takes precedence on Windows.
+        if not os.curdir in path:
+            path.insert(0, os.curdir)
+
+        # PATHEXT is necessary to check on Windows.
+        pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
+        # See if the given file matches any of the expected path extensions.
+        # This will allow us to short circuit when given "python.exe".
+        # If it does match, only test that one, otherwise we have to try
+        # others.
+        if any(cmd.lower().endswith(ext.lower()) for ext in pathext):
+            files = [cmd]
+        else:
+            files = [cmd + ext for ext in pathext]
+    else:
+        # On other platforms you don't have things like PATHEXT to tell you
+        # what file suffixes are executable, so just pass on cmd as-is.
+        files = [cmd]
+
+    seen = set()
+    for dir in path:
+        normdir = os.path.normcase(dir)
+        if not normdir in seen:
+            seen.add(normdir)
+            for thefile in files:
+                name = os.path.join(dir, thefile)
+                if _access_check(name, mode):
+                    return name
+    return None
+
+
+def collect_lua_coverage():
+    if python3_which("luacov-coveralls") is None:
+        logger.info("luacov-coveralls not found, will not collect Lua coverage")
+        return
+
+    # decided not to do optional coverage so far
+    #if not 'ENABLE_LUA_COVERAGE' in os.environ['HOME']:
+    #    logger.info("ENABLE_LUA_COVERAGE is not present in env, will not collect Lua coverage")
+    #    return
+
+    current_directory = os.getcwd()
+    report_file = current_directory + "/lua_coverage_report.json"
+    old_report = current_directory + "/lua_coverage_report.json.old"
+
+    tmp_dir = BuiltIn().get_variable_value("${TMPDIR}")
+    coverage_files = glob.glob('%s/*.luacov.stats.out' % (tmp_dir))
+
+    for stat_file in coverage_files:
+        shutil.move(stat_file, "luacov.stats.out")
+        # logger.console("statfile: " + stat_file)
+
+        if (os.path.isfile(report_file)):
+            shutil.move(report_file, old_report)
+            p = subprocess.Popen(["luacov-coveralls", "-o", report_file, "-j", old_report, "--merge", "--dryrun"], 
+                                 stdout = subprocess.PIPE, stderr= subprocess.PIPE)
+            output,error = p.communicate()
+
+            logger.info("luacov-coveralls stdout: " + output)
+            logger.info("luacov-coveralls stderr: " + error)
+            os.remove(old_report)
+        else:
+            p = subprocess.Popen(["luacov-coveralls", "-o", report_file, "--dryrun"], stdout = subprocess.PIPE, stderr= subprocess.PIPE)
+            output,error = p.communicate()
+
+            logger.info("luacov-coveralls stdout: " + output)
+            logger.info("luacov-coveralls stderr: " + error)
+        os.remove("luacov.stats.out")
+
diff --git a/test/functional/lib/rspamd.robot b/test/functional/lib/rspamd.robot
index 3dca630d7..6d3655f4d 100644
--- a/test/functional/lib/rspamd.robot
+++ b/test/functional/lib/rspamd.robot
@@ -72,6 +72,7 @@ Generic Teardown
   Shutdown Process With Children  ${RSPAMD_PID}
   Log does not contain segfault record
   Save Run Results  ${TMPDIR}  rspamd.log redis.log rspamd.conf clickhouse-server.log clickhouse-server.err.log clickhouse-config.xml
+  Collect Lua Coverage
   Cleanup Temporary Directory  ${TMPDIR}
 
 Log does not contain segfault record
diff --git a/test/functional/util/merge_coveralls.py b/test/functional/util/merge_coveralls.py
new file mode 100755
index 000000000..2e0369d15
--- /dev/null
+++ b/test/functional/util/merge_coveralls.py
@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+
+import argparse
+import json
+import os
+import requests
+
+
+# install path to repository mapping
+# if path mapped to None, it means that the file should be ignored (i.e. test file/helper)
+# first matched path counts.
+# terminating slash should be added for directories 
+path_mapping = [
+    ("${install-dir}/share/rspamd/lib/fun.lua", None),
+    ("${install-dir}/share/rspamd/lib/", "lualib/"),
+    ("${install-dir}/share/rspamd/rules/" , "rules/"),
+    ("${install-dir}/share/rspamd/lib/torch/" , None),
+    ("${build-dir}/CMakeFiles/", None),
+    ("${build-dir}/contrib/", None),
+    ("${build-dir}/test", None),
+    ("${project-root}/test/lua/", None),
+    ("${project-root}/test/", None),
+    ("${project-root}/clang-plugin/", None),
+    ("${project-root}/CMakeFiles/", None),
+    ("${project-root}/contrib/", None),
+    ("${project-root}/", ""),
+    ("contrib/", None),
+    ("CMakeFiles/", None),
+]
+
+parser = argparse.ArgumentParser(description='')
+parser.add_argument('--input', type=open, required=True, nargs='+', help='input files')
+parser.add_argument('--output', type=str, required=True, help='output file)')
+parser.add_argument('--root', type=str, required=False, default="/home/circleci/project", help='repository root)')
+parser.add_argument('--install-dir', type=str, required=False, default="/home/circleci/install", help='install root)')
+parser.add_argument('--build-dir', type=str, required=False, default="/home/circleci/build", help='build root)')
+parser.add_argument('--token', type=str, help='If present, the file will be uploaded to coveralls)')
+
+def merge_coverage_vectors(c1, c2):
+    assert(len(c1) == len(c2))
+    
+    for i in xrange(0, len(c1)):
+        if c1[i] is None and c2[i] is None:
+            pass
+        elif type(c1[i]) is int and c2[i] is None:
+            pass
+        elif c1[i] is None and type(c2[i]) is int:
+            c1[i] = c2[i]
+        elif type(c1[i]) is int and type(c2[i]) is int:
+            c1[i] += c2[i]
+        else:
+            raise RuntimeError("bad element types at %d: %s, %s", i, type(c1[i]), type(c1[i]))
+
+    return c1
+
+
+def normalize_name(name):
+    orig_name = name
+    name = os.path.normpath(name)
+    if not os.path.isabs(name):
+        name = os.path.abspath(repository_root + "/" + name)
+    for k in path_mapping:
+        if name.startswith(k[0]):
+            if k[1] is None:
+                return None
+            else:
+                name = k[1] + name[len(k[0]):]
+                break
+    return name
+
+def merge(files, j1):
+    for sf in j1['source_files']:
+        name = normalize_name(sf['name'])
+        if name is None:
+            continue
+        if name in files:
+            files[name]['coverage'] = merge_coverage_vectors(files[name]['coverage'], sf['coverage'])
+        else:
+            sf['name'] = name
+            files[name] = sf
+            if not ('source' in sf):
+                path = "%s/%s" % (repository_root, sf['name'])
+                if os.path.isfile(path):
+                    with open(path) as f:
+                        files[name]['source'] = f.read()
+
+    return files
+
+def prepare_path_mapping():
+    for i in xrange(0, len(path_mapping)):
+        new_key = path_mapping[i][0].replace("${install-dir}", install_dir)
+        new_key = new_key.replace("${project-root}", repository_root)
+        new_key = new_key.replace("${build-dir}", build_dir)
+                    
+        path_mapping[i] = (new_key, path_mapping[i][1])
+
+if __name__ == '__main__':
+    args = parser.parse_args()
+
+    repository_root = os.path.abspath(os.path.expanduser(args.root))
+    install_dir = os.path.normpath(os.path.expanduser(args.install_dir))
+    build_dir = os.path.normpath(os.path.expanduser(args.build_dir))
+
+    prepare_path_mapping()
+
+    j1 = json.loads(args.input[0].read())
+
+    files = merge({}, j1)
+    for i in xrange(1, len(args.input)):
+        j2 = json.loads(args.input[i].read())
+        files = merge(files, j2)
+        
+        if 'git' not in j1 and 'git' in j2:
+            j1['git'] = j2['git']
+        if 'service_name' not in j1 and 'service_name' in j2:
+            j1['service_name'] = j2['service_name']
+        if 'service_job_id' not in j1 and 'service_job_id' in j2:
+            j1['service_job_id'] = j2['service_job_id']
+        if not j1['service_job_id'] and 'CIRCLE_BUILD_NUM' in os.environ:
+            j1['service_job_id'] = os.environ['CIRCLE_BUILD_NUM']
+        if 'CIRCLECI' in os.environ and os.environ['CIRCLECI']:
+            j1['service_name'] = 'circleci'
+
+    j1['source_files'] = files.values()
+
+    with open(args.output, 'w') as f:
+        f.write(json.dumps(j1))
+
+    if not args.token is None:
+        j1['repo_token'] = args.token
+        print("sending data to coveralls...")
+        r = requests.post('https://coveralls.io/api/v1/jobs', files={"json_file": json.dumps(j1)})
+        response = json.loads(r.text)
+        print "uploaded %s\nmessage:%s" % (response['url'], response['message'])
+
+    # post https://coveralls.io/api/v1/jobs
+    # print args
+
+