Drone speedup (#2588)

* Start switchind test to custom Docker images

Using Docker images with pre-installed delendencies will speedup
testing.

* Debug build in drone.io

* Use variable instead hardcoded path to the source

* Debug variable substitution by drone

Also according to comments in code DRONE_WORKSPACE is deprecated and
CI_WORKSPACE is a new form.

* One more try to use variables.

It looks like drone uses same rules as docker-compose:
https://docs.docker.com/compose/compose-file/#variable-substitution

* Cleanup debug

* Less verbose make output

* Be consistent in variable escaping

It works without escaping only becuse ${VAR} currently expanded by
drone, but $VAR is not.

* Use alpine base image for nodejs

It is smaller.

* Return rspamd-test

* Revert variable escaping

Wihtout escaping it is more easy to copy-past command for testing.
Just don't use ${var} when var should be expanded by shell and not
drone.

* Add debug

* Fix yaml parsing error

* Fix exit code capture

It should be done on the same line, because drone insert 'echo + cmd'
between commands.

* Remove debug

* Readability

No functional changes.

* Return functional tests

* Be consistent

* Add git stub for coveralls

* Send coverage

* Fix yaml parsing error

* Add links to Dockerfiles

* Send test log in case of test failure

XXX if rspamd-test failed, but functional was successful robot log will be
attached (but not useful).

* Don't run rspamd-test and functional tests in parallel

It looks like there is a race condition in writing coverage files.

* Try to run tests in parallel

* Add debug

* Fix parallel test run

test/rspamd-test and functional test both use luacov.stats.out file
in current directory. Run ./rspamd-test from anther directory.

* Use drone condition for sending coveralls report

Also don't send report if empty tocken was provided.

* Add mininal python3 compatibility

.drone.yml

@@ -1,92 +1,76 @@
+---
 workspace:
   base: /rspamd
 
 pipeline:
 
   build:
-    image: gcc:5
+    # https://github.com/rspamd/rspamd-build-docker/blob/master/ubuntu-build/Dockerfile
+    image: rspamd/ci-ubuntu-build
     group: build
     commands:
-      - pwd
-      - set -e
-      - apt-get update -qq
-      - apt-get install -qq cmake libevent-dev libglib2.0-dev libicu-dev libluajit-5.1-dev libmagic-dev libsqlite3-dev libssl-dev ragel libunwind-dev libunwind8
-      - umask 0000
-      - mkdir /rspamd/build ; mkdir /rspamd/install ; cd /rspamd/build
-      - cmake /rspamd/src/github.com/rspamd/rspamd/ -DDBDIR=/nana -DENABLE_COVERAGE=ON -DENABLE_LIBUNWIND=ON -DCMAKE_INSTALL_PREFIX=/rspamd/install
-      - make install -j`nproc`
-      - make rspamd-test -j`nproc`
+      - mkdir /rspamd/build /rspamd/install
+      - cd /rspamd/build
+      - cmake $CI_WORKSPACE -DENABLE_COVERAGE=ON -DENABLE_LIBUNWIND=ON -DCMAKE_INSTALL_PREFIX=/rspamd/install -DCMAKE_RULE_MESSAGES=OFF
+      - ncpu=$(getconf _NPROCESSORS_ONLN)
+      - make -j $ncpu install
+      - make -j $ncpu rspamd-test
 
   eslint:
-    image: node:latest
+    image: node:10-alpine
     group: build
     commands:
       - npm install
-      - ./node_modules/.bin/eslint -v && ./node_modules/.bin/eslint ./
+      - ./node_modules/.bin/eslint -v
+      - ./node_modules/.bin/eslint ./
 
   rspamd-test:
-    image: gcc:5
+    # https://github.com/rspamd/rspamd-build-docker/blob/master/ubuntu-test/Dockerfile
+    image: rspamd/ci-ubuntu-test
     group: tests
     commands:
+      # rspamd-test and functional test both use luacov.stats.out file and should be started from
+      # different directories (if started in parallel)
+      - cd /rspamd/build/test
+      - set +e
+      - ./rspamd-test -p /rspamd/lua; EXIT_CODE=$?
       - set -e
-      - apt-get update -qq
-      - apt-get install -qq cmake libevent-dev libglib2.0-dev libicu-dev libluajit-5.1-dev libmagic-dev libsqlite3-dev libssl-dev ragel libunwind-dev libunwind8 luarocks
-      - apt-get install -qq python-pip python-dev
-      - pip install --upgrade setuptools
-      - pip install --upgrade pyOpenSSL
-      - pip install --upgrade certifi
-      - pip install --upgrade ipaddress
-      - pip install --upgrade urllib3
-      - pip install --upgrade enum34
-      - pip install --upgrade cpp-coveralls requests cryptography
-      - luarocks install luacov
-      - luarocks install luacov-coveralls
-
-      - cd /rspamd/build
-      - set +e; test/rspamd-test -p /rspamd/lua
-      - luacov-coveralls -o unit_test_lua.json --dryrun
-      - coveralls --dump coverage.rspamd-test.dump
+      - luacov-coveralls -o /rspamd/build/unit_test_lua.json --dryrun
+      # coveralls dosn't support layout when object files are located outside git repo root
+      # add hack to disable searching for git
+      - ln -s /bin/true /usr/local/bin/git
+      - coveralls --dump /rspamd/build/coverage.rspamd-test.dump
+      - exit $EXIT_CODE
 
   functional:
-    image: gcc:5
+    # https://github.com/rspamd/rspamd-build-docker/blob/master/ubuntu-test-func/Dockerfile
+    image: rspamd/ci-ubuntu-test-func
     group: tests
     commands:
-      - echo 'deb http://repo.yandex.ru/clickhouse/deb/stable/ main/' | tee /etc/apt/sources.list.d/clickhouse.list
-      - apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4    # optional, clickhouse key
-      - apt-get update -qq || true
-      - apt-get install -qq libluajit-5.1-dev libpcre3-dev luarocks opendkim-tools python-pip redis-server libunwind8 libglib2.0-dev libicu-dev libevent-dev python-dev
-      - apt-get install -qq clickhouse-server
-      - apt-get install -qq python-pip python-dev
-      - pip install demjson psutil robotframework http
-      - pip install --upgrade setuptools
-      - pip install --upgrade pyOpenSSL
-      - pip install --upgrade certifi
-      - pip install --upgrade ipaddress
-      - pip install --upgrade urllib3
-      - pip install --upgrade enum34
-      - pip install --upgrade cpp-coveralls requests cryptography
-
-      - luarocks install luacheck
-      - luarocks install luacov
-      - luarocks install luacov-coveralls
-
       - cd /rspamd/build
-      - umask 0000
-      - set +e; RSPAMD_INSTALLROOT=/rspamd/install robot -x xunit.xml --exclude isbroken /rspamd/src/github.com/rspamd/rspamd/test/functional/cases; export RETURN_CODE=$?
-      - cat log.html
+      - set +e
+      - RSPAMD_INSTALLROOT=/rspamd/install robot --xunit xunit.xml --exclude isbroken $CI_WORKSPACE/test/functional/cases; EXIT_CODE=$?
+      - set -e
+      # coveralls will not find git repo anyway, see above
+      - ln -s /bin/true /usr/local/bin/git
       - coveralls --dump coverage.functional.dump
-      - exit $RETURN_CODE
+      - exit $EXIT_CODE
 
   send-coverage:
-    image: gcc:5
+    image: rspamd/ci-ubuntu-test
     secrets: [ coveralls_repo_token ]
-    commands: |
-      [ ! $DRONE_COMMIT_BRANCH = 'master' ] && exit 0
-      apt-get update -qq || true
-      apt-get install -qq python-pip python-dev
-      pip install --upgrade setuptools
-      pip install --upgrade pyOpenSSL
-      pip install cpp-coveralls requests cryptography
-      cd /rspamd/build
-      [ ! -z $COVERALLS_REPO_TOKEN ] && /rspamd/src/github.com/rspamd/rspamd/test/functional/util/merge_coveralls.py --input coverage.functional.dump coverage.rspamd-test.dump unit_test_lua.json lua_coverage_report.json --output out.josn --token=$COVERALLS_REPO_TOKEN
+    commands:
+      - cd /rspamd/build
+      - $CI_WORKSPACE/test/functional/util/merge_coveralls.py --input coverage.functional.dump coverage.rspamd-test.dump unit_test_lua.json lua_coverage_report.json --output out.josn --token=$COVERALLS_REPO_TOKEN
+    when:
+      branch: master
+      # don't send coverage report for pull request
+      event: [push, tag]
 
+  send-test-log:
+    image: drillster/drone-email
+    from: noreply@rspamd.com
+    attachment: /rspamd/build/log.html
+    secrets: [email_host, email_username, email_password]
+    when:
+      status: failure

test/functional/util/merge_coveralls.py

@@ -1,10 +1,17 @@
 #!/usr/bin/env python
 
+from __future__ import print_function
+
 import argparse
 import json
 import os
+import sys
+
 import requests
 
+# Python 2/3 compatibility
+if sys.version_info.major > 2:
+    xrange = range
 
 # install path to repository mapping
 # if path mapped to None, it means that the file should be ignored (i.e. test file/helper)
@@ -132,12 +139,12 @@ if __name__ == '__main__':
     with open(args.output, 'w') as f:
         f.write(json.dumps(j1))
 
-    if not args.token is None:
+    if args.token:
         j1['repo_token'] = args.token
         print("sending data to coveralls...")
         r = requests.post('https://coveralls.io/api/v1/jobs', files={"json_file": json.dumps(j1)})
         response = json.loads(r.text)
-        print "uploaded %s\nmessage:%s" % (response['url'], response['message'])
+        print("uploaded %s\nmessage:%s" % (response['url'], response['message']))
 
     # post https://coveralls.io/api/v1/jobs
     # print args