From ee0de908033cf7eb9b7cc1e4bee13793f61ffa24 Mon Sep 17 00:00:00 2001 From: Anton Yuzhaninov Date: Sun, 14 Oct 2018 04:38:57 -0400 Subject: [PATCH] Drone coverage fix (#2592) * Build as nobody This showld fix coverage files writing. At least some coverage files (*.gcda, *.gcno) are written by rspamd running as nobody. * Comment luacov-coveralls * Run rspamd-test as nobody too * Run coveralls from build root (/rspamd/build) It need to find all `.gcda` files. * Hack for coveralls was moved to Docker image * Add test debug * More debug * Workaround to writing *.gcda from different users *.gcda files written in parallel by root and by nobody * Remove debug * Use current build number instead of previous * Remove unused variable * Add more parameters - If CI detected we don't need to check if service_job_id and other fields in json already filled - this info anyway comes from environment - coveralls.io API is not documented. Code from: https://github.com/lemurheavy/coveralls-ruby/ and https://github.com/coveralls-clients/coveralls-python/ was used as an example. * In case of error there is no url in json respnse Print message field anyway - it can contain useful error message. * Add debug * Simplify * Fix: for pull request CI_BUILD_EVENT set to 'pull_request' * Remove debug --- .drone.yml | 29 +++++++++++++++++++------ test/functional/util/merge_coveralls.py | 19 +++++++++++----- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/.drone.yml b/.drone.yml index d3ee6bafb..78f585fab 100644 --- a/.drone.yml +++ b/.drone.yml @@ -4,12 +4,23 @@ workspace: pipeline: + prepare: + # ubuntu used as base image for build and test images + # and we need to download it anyway + image: ubuntu:18.04 + commands: + - install -d -o nobody -g nogroup /rspamd/build /rspamd/install + # lua-torch CMakeLists writes to src dir + - chown nobody $CI_WORKSPACE/contrib/lua-torch/nn + build: # https://github.com/rspamd/rspamd-build-docker/blob/master/ubuntu-build/Dockerfile image: rspamd/ci-ubuntu-build group: build commands: - - mkdir /rspamd/build /rspamd/install + # build directories should be writable by nobody, for rspamd in functional tests + # works as nobody and writes coverage files there + - test "$(id -un)" = nobody - cd /rspamd/build - cmake $CI_WORKSPACE -DENABLE_COVERAGE=ON -DENABLE_LIBUNWIND=ON -DCMAKE_INSTALL_PREFIX=/rspamd/install -DCMAKE_RULE_MESSAGES=OFF - ncpu=$(getconf _NPROCESSORS_ONLN) @@ -27,32 +38,36 @@ pipeline: rspamd-test: # https://github.com/rspamd/rspamd-build-docker/blob/master/ubuntu-test/Dockerfile image: rspamd/ci-ubuntu-test + pull: true group: tests commands: + - test "$(id -un)" = nobody # rspamd-test and functional test both use luacov.stats.out file and should be started from # different directories (if started in parallel) - cd /rspamd/build/test - set +e - ./rspamd-test -p /rspamd/lua; EXIT_CODE=$? - set -e + # luacov-coveralls reads luacov.stats.out written by rspamd-test using luacov module + # and writes json report for coveralls.io service - luacov-coveralls -o /rspamd/build/unit_test_lua.json --dryrun - # coveralls dosn't support layout when object files are located outside git repo root - # add hack to disable searching for git - - ln -s /bin/true /usr/local/bin/git - - coveralls --dump /rspamd/build/coverage.rspamd-test.dump + - cd /rspamd/build + - coveralls --dump coverage.rspamd-test.dump - exit $EXIT_CODE functional: # https://github.com/rspamd/rspamd-build-docker/blob/master/ubuntu-test-func/Dockerfile image: rspamd/ci-ubuntu-test-func + pull: true group: tests commands: - cd /rspamd/build + # some rspamd processes during this test work as root and some as nobody + # use umask to create world-writable files so nobody can write to *.gcda files created by root + - umask 0000 - set +e - RSPAMD_INSTALLROOT=/rspamd/install robot --xunit xunit.xml --exclude isbroken $CI_WORKSPACE/test/functional/cases; EXIT_CODE=$? - set -e - # coveralls will not find git repo anyway, see above - - ln -s /bin/true /usr/local/bin/git - coveralls --dump coverage.functional.dump - exit $EXIT_CODE diff --git a/test/functional/util/merge_coveralls.py b/test/functional/util/merge_coveralls.py index e1498ac01..b19aa383b 100755 --- a/test/functional/util/merge_coveralls.py +++ b/test/functional/util/merge_coveralls.py @@ -63,7 +63,6 @@ def merge_coverage_vectors(c1, c2): def normalize_name(name): - orig_name = name name = os.path.normpath(name) if not os.path.isabs(name): name = os.path.abspath(repository_root + "/" + name) @@ -130,13 +129,19 @@ if __name__ == '__main__': if not j1['service_job_id'] and 'CIRCLE_BUILD_NUM' in os.environ: j1['service_job_id'] = os.environ['CIRCLE_BUILD_NUM'] - elif not j1['service_job_id'] and 'DRONE_PREV_BUILD_NUMBER' in os.environ: - j1['service_job_id'] = os.environ['DRONE_PREV_BUILD_NUMBER'] if 'CIRCLECI' in os.environ and os.environ['CIRCLECI']: j1['service_name'] = 'circleci' - elif 'DRONE' in os.environ and os.environ['DRONE']: + elif os.getenv('CI') == 'drone': j1['service_name'] = 'drone' + j1['service_branch'] = os.getenv('CI_COMMIT_BRANCH') + j1['service_build_url'] = os.getenv('DRONE_BUILD_LINK') + j1['service_job_id'] = os.getenv('CI_JOB_NUMBER') + j1['service_number'] = os.getenv('CI_BUILD_NUMBER') + j1['commit_sha'] = os.getenv('CI_COMMIT_SHA') + if os.getenv('CI_BUILD_EVENT') == 'pull_request': + j1['service_pull_request'] = os.getenv('CI_PULL_REQUEST') + j1['source_files'] = list(files.values()) @@ -147,8 +152,10 @@ if __name__ == '__main__': j1['repo_token'] = args.token print("sending data to coveralls...") r = requests.post('https://coveralls.io/api/v1/jobs', files={"json_file": json.dumps(j1)}) - response = json.loads(r.text) - print("uploaded %s\nmessage:%s" % (response['url'], response['message'])) + response = r.json() + print("[coveralls] %s" % response['message']) + if 'url' in response: + print("[coveralls] Uploaded to %s" % response['url']) # post https://coveralls.io/api/v1/jobs # print args -- 2.39.5