Drone coverage fix (#2592)

* Build as nobody

This showld fix coverage files writing. At least some coverage files
(*.gcda, *.gcno) are written by rspamd running as nobody.

* Comment luacov-coveralls

* Run rspamd-test as nobody too

* Run coveralls from build root (/rspamd/build)

It need to find all `.gcda` files.

* Hack for coveralls was moved to Docker image

* Add test debug

* More debug

* Workaround to writing *.gcda from different users

*.gcda files written in parallel by root and by nobody

* Remove debug

* Use current build number instead of previous

* Remove unused variable

* Add more parameters

- If CI detected we don't need to check if service_job_id and other
fields in json already filled - this info anyway comes from environment

- coveralls.io API is not documented. Code from:
https://github.com/lemurheavy/coveralls-ruby/
and
https://github.com/coveralls-clients/coveralls-python/
was used as an example.

* In case of error there is no url in json respnse

Print message field anyway - it can contain useful error message.

* Add debug

* Simplify

* Fix: for pull request CI_BUILD_EVENT set to 'pull_request'

* Remove debug

diff --git a/.drone.yml b/.drone.yml
index d3ee6bafb16a27986b459db68f114c400ed51de4..78f585fab0148939fd948e4161d037f6b41e07e3 100644 (file)
--- a/.drone.yml
+++ b/.drone.yml
@@ -4,12 +4,23 @@ workspace:
 
 pipeline:
 
+  prepare:
+    # ubuntu used as base image for build and test images
+    # and we need to download it anyway
+    image: ubuntu:18.04
+    commands:
+      - install -d -o nobody -g nogroup /rspamd/build /rspamd/install
+      # lua-torch CMakeLists writes to src dir
+      - chown nobody $CI_WORKSPACE/contrib/lua-torch/nn
+
   build:
     # https://github.com/rspamd/rspamd-build-docker/blob/master/ubuntu-build/Dockerfile
     image: rspamd/ci-ubuntu-build
     group: build
     commands:
-      - mkdir /rspamd/build /rspamd/install
+      # build directories should be writable by nobody, for rspamd in functional tests
+      # works as nobody and writes coverage files there
+      - test "$(id -un)" = nobody
       - cd /rspamd/build
       - cmake $CI_WORKSPACE -DENABLE_COVERAGE=ON -DENABLE_LIBUNWIND=ON -DCMAKE_INSTALL_PREFIX=/rspamd/install -DCMAKE_RULE_MESSAGES=OFF
       - ncpu=$(getconf _NPROCESSORS_ONLN)
@@ -27,32 +38,36 @@ pipeline:
   rspamd-test:
     # https://github.com/rspamd/rspamd-build-docker/blob/master/ubuntu-test/Dockerfile
     image: rspamd/ci-ubuntu-test
+    pull: true
     group: tests
     commands:
+      - test "$(id -un)" = nobody
       # rspamd-test and functional test both use luacov.stats.out file and should be started from
       # different directories (if started in parallel)
       - cd /rspamd/build/test
       - set +e
       - ./rspamd-test -p /rspamd/lua; EXIT_CODE=$?
       - set -e
+      # luacov-coveralls reads luacov.stats.out written by rspamd-test using luacov module
+      # and writes json report for coveralls.io service
       - luacov-coveralls -o /rspamd/build/unit_test_lua.json --dryrun
-      # coveralls dosn't support layout when object files are located outside git repo root
-      # add hack to disable searching for git
-      - ln -s /bin/true /usr/local/bin/git
-      - coveralls --dump /rspamd/build/coverage.rspamd-test.dump
+      - cd /rspamd/build
+      - coveralls --dump coverage.rspamd-test.dump
       - exit $EXIT_CODE
 
   functional:
     # https://github.com/rspamd/rspamd-build-docker/blob/master/ubuntu-test-func/Dockerfile
     image: rspamd/ci-ubuntu-test-func
+    pull: true
     group: tests
     commands:
       - cd /rspamd/build
+      # some rspamd processes during this test work as root and some as nobody
+      # use umask to create world-writable files so nobody can write to *.gcda files created by root
+      - umask 0000
       - set +e
       - RSPAMD_INSTALLROOT=/rspamd/install robot --xunit xunit.xml --exclude isbroken $CI_WORKSPACE/test/functional/cases; EXIT_CODE=$?
       - set -e
-      # coveralls will not find git repo anyway, see above
-      - ln -s /bin/true /usr/local/bin/git
       - coveralls --dump coverage.functional.dump
       - exit $EXIT_CODE
 

diff --git a/test/functional/util/merge_coveralls.py b/test/functional/util/merge_coveralls.py
index e1498ac0139a32aa99f0a8f1469343cf7323cf94..b19aa383b5e1bbf44fd45dbf4800716310155e25 100755 (executable)
--- a/test/functional/util/merge_coveralls.py
+++ b/test/functional/util/merge_coveralls.py
@@ -63,7 +63,6 @@ def merge_coverage_vectors(c1, c2):
 
 
 def normalize_name(name):
-    orig_name = name
     name = os.path.normpath(name)
     if not os.path.isabs(name):
         name = os.path.abspath(repository_root + "/" + name)
@@ -130,13 +129,19 @@ if __name__ == '__main__':
 
         if not j1['service_job_id'] and 'CIRCLE_BUILD_NUM' in os.environ:
             j1['service_job_id'] = os.environ['CIRCLE_BUILD_NUM']
-        elif not j1['service_job_id'] and 'DRONE_PREV_BUILD_NUMBER' in os.environ:
-            j1['service_job_id'] = os.environ['DRONE_PREV_BUILD_NUMBER']
 
         if 'CIRCLECI' in os.environ and os.environ['CIRCLECI']:
             j1['service_name'] = 'circleci'
-        elif 'DRONE' in os.environ and os.environ['DRONE']:
+        elif os.getenv('CI') == 'drone':
             j1['service_name'] = 'drone'
+            j1['service_branch'] = os.getenv('CI_COMMIT_BRANCH')
+            j1['service_build_url'] = os.getenv('DRONE_BUILD_LINK')
+            j1['service_job_id'] = os.getenv('CI_JOB_NUMBER')
+            j1['service_number'] = os.getenv('CI_BUILD_NUMBER')
+            j1['commit_sha'] = os.getenv('CI_COMMIT_SHA')
+            if os.getenv('CI_BUILD_EVENT') == 'pull_request':
+                j1['service_pull_request'] = os.getenv('CI_PULL_REQUEST')
+
 
     j1['source_files'] = list(files.values())
 
@@ -147,8 +152,10 @@ if __name__ == '__main__':
         j1['repo_token'] = args.token
         print("sending data to coveralls...")
         r = requests.post('https://coveralls.io/api/v1/jobs', files={"json_file": json.dumps(j1)})
-        response = json.loads(r.text)
-        print("uploaded %s\nmessage:%s" % (response['url'], response['message']))
+        response = r.json()
+        print("[coveralls] %s" % response['message'])
+        if 'url' in response:
+            print("[coveralls] Uploaded to %s" % response['url'])
 
     # post https://coveralls.io/api/v1/jobs
     # print args