1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
|
---
workspace:
base: /rspamd
pipeline:
prepare:
# ubuntu used as base image for build and test images
# and we need to download it anyway
image: ubuntu:18.04
commands:
- install -d -o nobody -g nogroup /rspamd/build /rspamd/install /rspamd/fedora/build /rspamd/fedora/install
# lua-torch CMakeLists writes to src dir
- chown nobody $CI_WORKSPACE/contrib/lua-torch/nn
# for debug
- echo $CI_COMMIT_AUTHOR
build:
# https://github.com/rspamd/rspamd-build-docker/blob/master/ubuntu-build/Dockerfile
image: rspamd/ci-ubuntu-build
group: build
commands:
# build directories should be writable by nobody, for rspamd in functional tests
# works as nobody and writes coverage files there
- test "$(id -un)" = nobody
- cd /rspamd/build
- cmake $CI_WORKSPACE -DENABLE_COVERAGE=ON -DENABLE_LIBUNWIND=ON -DCMAKE_INSTALL_PREFIX=/rspamd/install -DCMAKE_RULE_MESSAGES=OFF
- ncpu=$(getconf _NPROCESSORS_ONLN)
- make -j $ncpu install
- make -j $ncpu rspamd-test
build-clang:
# https://github.com/rspamd/rspamd-build-docker/blob/master/fedora-build/Dockerfile
image: rspamd/ci-fedora-build
pull: true
group: build
commands:
- test "$(id -un)" = nobody
- cd /rspamd/fedora/build
- export LDFLAGS='-fuse-ld=lld'
- export CFLAGS='-fsanitize=address,undefined,implicit-integer-truncation'
- export ASAN_OPTIONS=detect_leaks=0
- >
cmake
-DENABLE_CLANG_PLUGIN=ON
-DLLVM_CONFIG_BINARY=/usr/bin/llvm-config
-DENABLE_TORCH=OFF
-DCMAKE_C_COMPILER=/usr/bin/clang -DCMAKE_CXX_COMPILER=/usr/bin/clang++
-DCMAKE_INSTALL_PREFIX=/rspamd/fedora/install
-DENABLE_FULL_DEBUG=ON
-DCMAKE_RULE_MESSAGES=OFF
$CI_WORKSPACE
- ncpu=$(getconf _NPROCESSORS_ONLN)
- make -j $ncpu install
- make -j $ncpu rspamd-test
eslint:
image: node:10-alpine
group: build
commands:
- npm install
- ./node_modules/.bin/eslint -v
- ./node_modules/.bin/eslint ./
# Run stylelint checks
- ./node_modules/.bin/stylelint -v
- npm show stylelint-config-standard version
- ./node_modules/.bin/stylelint ./**/*.css ./**/*.html ./**/*.js
# Run checks on perl source using tidyall
perl-tidyall:
# https://github.com/rspamd/rspamd-build-docker/blob/master/perl-tidyall/Dockerfile
image: rspamd/ci-perl-tidyall
group: build
commands:
# checks are configured in .tidyallrc at the top of rspamd repo
- tidyall --all --check-only --no-cache --data-dir /tmp/tidyall
# We run rspamd-test (unit test) and functional test (runned by robot) in
# parallel to save time. To avoid conflict in saving lua coverage we run them
# from different directories. For C code coverage counters is saved to .gcda
# files and binary contain absolute path to them, so rspamd-test and
# processes started by functional test are writing to the same files. On
# process exit new coverage data merged with existing content of .gcda file.
# Race is possible if rspamd-test and some rspamd process in functional test
# will try to write .gcda file simultaneous. But it is very unlikely and
# performance is more important then correct coverage data.
rspamd-test:
# https://github.com/rspamd/rspamd-build-docker/blob/master/ubuntu-test/Dockerfile
image: rspamd/ci-ubuntu-test
pull: true
group: tests
commands:
- test "$(id -un)" = nobody
- ulimit -c unlimited
# rspamd-test and functional test both use luacov.stats.out file and should be started from
# different directories (if started in parallel)
- cd /rspamd/build/test
- set +e
- ./rspamd-test -p /rspamd/lua; EXIT_CODE=$?
- set -e
# shell sets exit status of a process terminated by a signal to '128 + signal-number'
# if rspamd-test was terminated by a signal it should be SIGSEGV or SIGABRT, try to examine core
- >
if [ $EXIT_CODE -gt 128 ]; then
gdb --batch -ex 'thread apply all bt full' -c /var/tmp/*.rspamd-test.core ./rspamd-test;
exit $EXIT_CODE;
fi
# luacov-coveralls reads luacov.stats.out written by rspamd-test using luacov module
# and writes json report for coveralls.io service
- luacov-coveralls -o /rspamd/build/unit_test_lua.json --dryrun
- exit $EXIT_CODE
test-fedora-clang:
# https://github.com/rspamd/rspamd-build-docker/blob/master/fedora-test/Dockerfile
image: rspamd/ci-fedora-test
pull: true
group: tests
commands:
- test "$(id -un)" = nobody
# Asan reserves 20Tb of virtual memory, limit core size to 2 Gb to avoid writing huge core
- ulimit -c 2097152
# disable leak sanitizer: too many leaks detected, most of them probably FP
- export ASAN_OPTIONS="detect_leaks=0:print_stacktrace=1:disable_coredump=0"
- export UBSAN_OPTIONS="print_stacktrace=1:print_summary=0:log_path=/tmp/ubsan"
- cd /rspamd/fedora/build/test
- set +e
- ./rspamd-test -p /rspamd/lua; EXIT_CODE=$?
- set -e
# shell sets exit status of a process terminated by a signal to '128 + signal-number'
# if rspamd-test was terminated by a signal it should be SIGSEGV or SIGABRT, try to examine core
- >
if [ $EXIT_CODE -gt 128 ]; then
gdb --batch -ex 'bt' -c /var/tmp/*.rspamd-test.core ./rspamd-test;
fi
- cat /tmp/ubsan.*
- exit $EXIT_CODE
functional:
# https://github.com/rspamd/rspamd-build-docker/blob/master/ubuntu-test-func/Dockerfile
image: rspamd/ci-ubuntu-test-func
pull: true
group: tests
commands:
- cd /rspamd/build
- ulimit -c unlimited
# some rspamd processes during this test work as root and some as nobody
# use umask to create world-writable files so nobody can write to *.gcda files created by root
- umask 0000
- set +e
- RSPAMD_INSTALLROOT=/rspamd/install robot --removekeywords wuks --exclude isbroken $CI_WORKSPACE/test/functional/cases; EXIT_CODE=$?
- set -e
# upload test results to nginx frontent using WebDAV PUT
- >
if [ -n "$HTTP_PUT_AUTH" ]; then
$CI_WORKSPACE/test/tools/http_put.py log.html report.html $CI_SYSTEM_LINK/testlogs/$CI_REPO_NAME/$CI_BUILD_NUMBER/;
fi
# core_pattern=/var/tmp/%u.%e.core so one or two cores can be saved for each binary
- core_files=$(find /var/tmp/ -name '*.core')
# use 'info proc mappings' to find path to executable file for given core
# first mapping is usually program executable
- >
for core in $core_files;
do
exe=$(gdb --batch -ex 'info proc mappings' -c $core | awk 'h {print $5; exit}; /objfile/ {h=1}');
gdb --batch -ex 'bt' -c $core $exe; echo '---';
done
- exit $EXIT_CODE
secrets: [http_put_auth]
send-coverage:
image: rspamd/ci-ubuntu-test
secrets: [ coveralls_repo_token ]
commands:
- cd /rspamd/build
# extract coverage data for C code from .gcda files and save it in a format suitable for coveralls.io
- $CI_WORKSPACE/test/tools/gcov_coveralls.py --exclude test --prefix /rspamd/build --prefix $CI_WORKSPACE --out coverage.c.json
# luacov-coveralls reads luacov.stats.out generated by functional tests
# (see collect_lua_coverage() in test/functional/lib/rspamd.py)
# and writes json report for coveralls.io
- luacov-coveralls -o coverage.functional.lua.json --dryrun
# * merge coverage for C and Lua code
# * remove prefixes from absolute paths (in luacov-coveralls files), filter test, contrib, e. t.c
# * upload report to coveralls.io
- $CI_WORKSPACE/test/tools/merge_coveralls.py --root $CI_WORKSPACE --input coverage.c.json unit_test_lua.json coverage.functional.lua.json --token=$COVERALLS_REPO_TOKEN
when:
branch: master
# don't send coverage report for pull request
event: [push, tag]
notify:
image: drillster/drone-email
from: noreply@rspamd.com
secrets: [email_host, email_username, email_password]
when:
status: failure
|
