1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
|
---
kind: pipeline
type: docker
name: default
platform:
os: linux
arch: amd64
steps:
- name: prepare
# any image with a root shell can be used here, but Ubuntu used as base
# image for build and test images and we need to download it anyway
image: ubuntu:18.04
pull: if-not-exists
volumes:
- name: rspamd
path: /rspamd
commands:
- install -d -o nobody -g nogroup /rspamd/build /rspamd/install /rspamd/fedora/build /rspamd/fedora/install
- name: build
# https://github.com/rspamd/rspamd-build-docker/blob/master/ubuntu-build/Dockerfile
image: rspamd/ci-ubuntu-build
pull: always
volumes:
- name: rspamd
path: /rspamd
depends_on: [ prepare ]
commands:
# build directories should be writable by nobody, for rspamd in functional tests
# works as nobody and writes coverage files there
- test "$(id -un)" = nobody
- cd /rspamd/build
- >
cmake
-DCMAKE_INSTALL_PREFIX=/rspamd/install
-DCMAKE_RULE_MESSAGES=OFF
-DCMAKE_VERBOSE_MAKEFILE=ON
-DENABLE_COVERAGE=ON
-DENABLE_LIBUNWIND=ON
$CI_WORKSPACE
- ncpu=$(getconf _NPROCESSORS_ONLN)
- make -j $ncpu install
- make -j $ncpu rspamd-test
- name: build-clang
# https://github.com/rspamd/rspamd-build-docker/blob/master/fedora-build/Dockerfile
image: rspamd/ci-fedora-build
pull: always
volumes:
- name: rspamd
path: /rspamd
depends_on: [ prepare ]
commands:
- test "$(id -un)" = nobody
- cd /rspamd/fedora/build
- export LDFLAGS='-fuse-ld=lld'
- export CFLAGS='-fsanitize=address,undefined,implicit-integer-truncation'
- export CXXFLAGS='-fsanitize=address,undefined,implicit-integer-truncation'
- export ASAN_OPTIONS=detect_leaks=0
- >
cmake
-DCMAKE_INSTALL_PREFIX=/rspamd/fedora/install
-DCMAKE_C_COMPILER=/usr/bin/clang -DCMAKE_CXX_COMPILER=/usr/bin/clang++
-DCMAKE_RULE_MESSAGES=OFF
-DCMAKE_VERBOSE_MAKEFILE=ON
-DENABLE_CLANG_PLUGIN=ON
-DENABLE_FULL_DEBUG=ON
$CI_WORKSPACE
- ncpu=$(getconf _NPROCESSORS_ONLN)
- make -j $ncpu install
- make -j $ncpu rspamd-test
# We run rspamd-test (unit test) and functional test (runned by robot) in
# parallel to save time. To avoid conflict in saving lua coverage we run them
# from different directories. For C code coverage counters is saved to .gcda
# files and binary contain absolute path to them, so rspamd-test and
# processes started by functional test are writing to the same files. On
# process exit new coverage data merged with existing content of .gcda file.
# Race is possible if rspamd-test and some rspamd process in functional test
# will try to write .gcda file simultaneous. But it is very unlikely and
# performance is more important then correct coverage data.
- name: rspamd-test
# https://github.com/rspamd/rspamd-build-docker/blob/master/ubuntu-test/Dockerfile
image: rspamd/ci-ubuntu-test
pull: always
volumes:
- name: rspamd
path: /rspamd
depends_on: [ build ]
commands:
- test "$(id -un)" = nobody
- ulimit -c unlimited
# rspamd-test and functional test both use luacov.stats.out file and should be started from
# different directories (if started in parallel)
- cd /rspamd/build/test
- set +e
- ./rspamd-test -p /rspamd/lua; EXIT_CODE=$?
- set -e
# shell sets exit status of a process terminated by a signal to '128 + signal-number'
# if rspamd-test was terminated by a signal it should be SIGSEGV or SIGABRT, try to examine core
- >
if [ $EXIT_CODE -gt 128 ]; then
gdb --batch -ex 'thread apply all bt full' -c /var/tmp/*.rspamd-test.core ./rspamd-test;
exit $EXIT_CODE;
fi
# luacov-coveralls reads luacov.stats.out written by rspamd-test using luacov module
# and writes json report for coveralls.io service
- luacov-coveralls -o /rspamd/build/unit_test_lua.json --dryrun
- exit $EXIT_CODE
- name: test-fedora-clang
# https://github.com/rspamd/rspamd-build-docker/blob/master/fedora-test/Dockerfile
image: rspamd/ci-fedora-test
pull: always
volumes:
- name: rspamd
path: /rspamd
depends_on: [ build-clang ]
commands:
- test "$(id -un)" = nobody
# Asan reserves 20Tb of virtual memory, limit core size to 2 Gb to avoid writing huge core
- ulimit -c 2097152
- ulimit -s unlimited
# disable leak sanitizer: too many leaks detected, most of them probably FP
- export ASAN_OPTIONS="detect_leaks=0:print_stacktrace=1:disable_coredump=0"
- export UBSAN_OPTIONS="print_stacktrace=1:print_summary=0:log_path=/tmp/ubsan"
- cd /rspamd/fedora/build/test
- set +e
- ./rspamd-test -p /rspamd/lua; EXIT_CODE=$?
- set -e
# shell sets exit status of a process terminated by a signal to '128 + signal-number'
# if rspamd-test was terminated by a signal it should be SIGSEGV or SIGABRT, try to examine core
- >
if [ $EXIT_CODE -gt 128 ]; then
gdb --batch -ex 'bt' -c /var/tmp/*.rspamd-test.core ./rspamd-test;
fi
- cat /tmp/ubsan.*
- exit $EXIT_CODE
- name: functional
# https://github.com/rspamd/rspamd-build-docker/blob/master/ubuntu-test-func/Dockerfile
image: rspamd/ci-ubuntu-test-func
pull: always
volumes:
- name: rspamd
path: /rspamd
depends_on: [ build ]
commands:
- cd /rspamd/build
- ulimit -c unlimited
- ulimit -s unlimited
# some rspamd processes during this test work as root and some as nobody
# use umask to create world-writable files so nobody can write to *.gcda files created by root
- umask 0000
- set +e
- RSPAMD_INSTALLROOT=/rspamd/install robot --removekeywords wuks --exclude isbroken $CI_WORKSPACE/test/functional/cases; EXIT_CODE=$?
- set -e
# upload test results to nginx frontent using WebDAV PUT
- >
if [ -n "$HTTP_PUT_AUTH" ]; then
$CI_WORKSPACE/test/tools/http_put.py log.html report.html https://$DRONE_SYSTEM_HOSTNAME/testlogs/$DRONE_REPO/$DRONE_BUILD_NUMBER/;
fi
# core_pattern=/var/tmp/%u.%e.core so one or two cores can be saved for each binary
- core_files=$(find /var/tmp/ -name '*.core')
# use 'info proc mappings' to find path to executable file for given core
# first mapping is usually program executable
- >
for core in $core_files;
do
exe=$(gdb --batch -ex 'info proc mappings' -c $core | awk 'h {print $5; exit}; /objfile/ {h=1}');
gdb --batch -ex 'bt' -c $core $exe; echo '---';
done
- exit $EXIT_CODE
environment:
HTTP_PUT_AUTH: { from_secret: http_put_auth }
- name: send-coverage
image: rspamd/ci-ubuntu-test
pull: if-not-exists
volumes:
- name: rspamd
path: /rspamd
depends_on: [ functional, rspamd-test ]
commands:
- cd /rspamd/build
# extract coverage data for C code from .gcda files and save it in a format suitable for coveralls.io
- $CI_WORKSPACE/test/tools/gcov_coveralls.py --exclude test --prefix /rspamd/build --prefix $CI_WORKSPACE --out coverage.c.json
# luacov-coveralls reads luacov.stats.out generated by functional tests
# (see collect_lua_coverage() in test/functional/lib/rspamd.py)
# and writes json report for coveralls.io
- luacov-coveralls -o coverage.functional.lua.json --dryrun
# * merge coverage for C and Lua code
# * remove prefixes from absolute paths (in luacov-coveralls files), filter test, contrib, e. t.c
# * upload report to coveralls.io
- $CI_WORKSPACE/test/tools/merge_coveralls.py --root $CI_WORKSPACE --input coverage.c.json unit_test_lua.json coverage.functional.lua.json --token=$COVERALLS_REPO_TOKEN
environment:
COVERALLS_REPO_TOKEN: { from_secret: coveralls_repo_token }
when:
branch: [ master ]
# don't send coverage report for pull request
event: [push, tag]
- name: eslint
image: node:10-alpine
pull: if-not-exists
commands:
- npm install
- ./node_modules/.bin/eslint -v
- ./node_modules/.bin/eslint ./
# Run stylelint checks
- ./node_modules/.bin/stylelint -v
- npm show stylelint-config-standard version
- ./node_modules/.bin/stylelint ./**/*.css ./**/*.html ./**/*.js
- name: perl-tidyall
# https://github.com/rspamd/rspamd-build-docker/blob/master/perl-tidyall/Dockerfile
image: rspamd/ci-perl-tidyall
pull: if-not-exists
commands:
- tidyall --version
- perltidy --version | head -1
# checks are configured in .tidyallrc at the top of rspamd repo
- tidyall --all --check-only --no-cache --data-dir /tmp/tidyall
- name: notify
image: drillster/drone-email
pull: if-not-exists
depends_on:
- rspamd-test
- test-fedora-clang
- functional
- send-coverage
- eslint
- perl-tidyall
settings:
from: noreply@rspamd.com
host: { from_secret: email_host }
username: { from_secret: email_username }
password: { from_secret: email_password }
when:
status: [ failure ]
volumes:
- name: rspamd
temp: {}
trigger:
event: [push, tag, pull_request]
---
kind: signature
hmac: 9997b0189f3cf3acce7575372f728ccc57bf57b9a72936d86e91b1cff852a48b
...
|
