diff options
34 files changed, 391 insertions, 1035 deletions
diff --git a/.drone.jsonnet b/.drone.jsonnet deleted file mode 100644 index 8d4119706..000000000 --- a/.drone.jsonnet +++ /dev/null @@ -1,310 +0,0 @@ -local docker_pipeline = { - kind: 'pipeline', - type: 'docker', -}; - -local default_trigger_events_ex_pr = [ - 'push', - 'tag', - 'custom', -]; - -local default_trigger = { - trigger: { - event: default_trigger_events_ex_pr + ['pull_request'], - }, -}; - -local platform(arch) = { - platform: { - os: 'linux', - arch: arch, - }, -}; - -local coveralls_attribs = { - branch: [ - 'master', - ], - event: [ - 'push', - 'tag', - ], -}; - -local close_coveralls_trigger = { - trigger: coveralls_attribs { status: ['success', 'failure'] }, -}; - -local coveralls_when = { - when: coveralls_attribs, -}; - -local notify_pipeline = { - name: 'notify', - depends_on: [ - 'default-amd64', - 'default-arm64', - 'default-noarch', - ], - steps: [ - { - name: 'notify', - image: 'drillster/drone-email', - pull: 'if-not-exists', - settings: { - from: 'noreply@rspamd.com', - host: { - from_secret: 'email_host', - }, - username: { - from_secret: 'email_username', - }, - password: { - from_secret: 'email_password', - }, - }, - }, - ], - trigger: { - event: default_trigger_events_ex_pr, - status: [ - 'failure', - ], - }, -} + docker_pipeline; - -local pipeline(arch) = { - local rspamd_volumes = { - volumes: [ - { - name: 'rspamd', - path: '/rspamd', - }, - ], - }, - local hyperscan_altroot = if (arch) == 'amd64' then '' else '-DHYPERSCAN_ROOT_DIR=/vectorscan', - depends_on: [ - ], - name: 'default-' + arch, - steps: [ - { - name: 'prepare', - image: 'ubuntu:22.04', - pull: 'if-not-exists', - commands: [ - 'install -d -o nobody -g nogroup /rspamd/build /rspamd/install /rspamd/fedora/build /rspamd/fedora/install', - ], - } + rspamd_volumes, - { - name: 'build', - image: 'rspamd/ci:ubuntu-build', - pull: 'always', - depends_on: [ - 'prepare', - ], - commands: [ - 'test "$(id -un)" = nobody', - 'cd /rspamd/build', - 'cmake -DCMAKE_INSTALL_PREFIX=/rspamd/install -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_COVERAGE=ON -DENABLE_LIBUNWIND=ON -DENABLE_HYPERSCAN=ON ' + hyperscan_altroot + ' -GNinja $DRONE_WORKSPACE\n', - 'ncpu=$(getconf _NPROCESSORS_ONLN)', - 'ninja -j $ncpu install', - 'ninja -j $ncpu rspamd-test', - 'ninja -j $ncpu rspamd-test-cxx', - ], - } + rspamd_volumes, - { - name: 'build-clang', - image: 'rspamd/ci:fedora-build', - pull: 'always', - depends_on: [ - 'prepare', - ], - commands: [ - 'test "$(id -un)" = nobody', - 'cd /rspamd/fedora/build', - "export LDFLAGS='-fuse-ld=lld'", - 'export ASAN_OPTIONS=detect_leaks=0', - 'cmake -DCMAKE_INSTALL_PREFIX=/rspamd/fedora/install -DCMAKE_C_COMPILER=/usr/bin/clang -DCMAKE_CXX_COMPILER=/usr/bin/clang++ -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_CLANG_PLUGIN=ON -DENABLE_FULL_DEBUG=ON -DENABLE_HYPERSCAN=ON ' + hyperscan_altroot + ' -DSANITIZE=address $DRONE_WORKSPACE\n', - 'ncpu=$(getconf _NPROCESSORS_ONLN)', - 'make -j $ncpu install', - 'make -j $ncpu rspamd-test', - 'make -j $ncpu rspamd-test-cxx', - ], - } + rspamd_volumes, - { - name: 'rspamd-test', - image: 'rspamd/ci:ubuntu-test', - pull: 'always', - depends_on: [ - 'build', - ], - commands: [ - 'test "$(id -un)" = nobody', - 'ulimit -c unlimited', - 'cd /rspamd/build/test', - 'set +e', - 'env RSPAMD_LUA_EXPENSIVE_TESTS=1 ./rspamd-test -p /rspamd/lua; EXIT_CODE=$?', - 'set -e', - "if [ $EXIT_CODE -gt 128 ]; then gdb --batch -ex 'thread apply all bt full' -c /var/tmp/*.rspamd-test.core ./rspamd-test; exit $EXIT_CODE; fi; if [ $EXIT_CODE -ne 0 ]; then exit $EXIT_CODE; fi\n", - 'luacov-coveralls -o /rspamd/build/unit_test_lua.json --dryrun', - 'set +e', - './rspamd-test-cxx -s; EXIT_CODE=$?', - 'set -e', - "if [ $EXIT_CODE -gt 128 ]; then gdb --batch -ex 'thread apply all bt full' -c /var/tmp/*.rspamd-test-cxx.core ./rspamd-test-cxx; exit $EXIT_CODE; fi\n", - 'exit $EXIT_CODE', - ], - } + rspamd_volumes, - { - name: 'test-fedora-clang', - image: 'rspamd/ci:fedora-test', - pull: 'always', - depends_on: [ - 'build-clang', - ], - commands: [ - 'test "$(id -un)" = nobody', - 'ulimit -c 2097152', - 'ulimit -s unlimited', - 'export ASAN_OPTIONS="detect_leaks=0:print_stacktrace=1:disable_coredump=0"', - 'export UBSAN_OPTIONS="print_stacktrace=1:print_summary=0:log_path=/tmp/ubsan"', - 'cd /rspamd/fedora/build/test', - 'set +e', - 'env RSPAMD_LUA_EXPENSIVE_TESTS=1 ./rspamd-test -p /rspamd/lua; EXIT_CODE=$?', - 'set -e', - "if [ $EXIT_CODE -gt 128 ]; then gdb --batch -ex 'bt' -c /var/tmp/*.rspamd-test.core ./rspamd-test; fi\n", - 'set +e', - './rspamd-test-cxx -s; EXIT_CODE=$?', - 'set -e', - "if [ $EXIT_CODE -gt 128 ]; then gdb --batch -ex 'thread apply all bt full' -c /var/tmp/*.rspamd-test-cxx.core ./rspamd-test-cxx; exit $EXIT_CODE; fi\n", - 'cat /tmp/ubsan.* || true', - 'exit $EXIT_CODE', - ], - } + rspamd_volumes, - { - name: 'functional', - image: 'rspamd/ci:ubuntu-test-func', - pull: 'always', - depends_on: [ - 'build', - ], - commands: [ - 'cd /rspamd/build', - 'ulimit -c unlimited', - 'ulimit -s unlimited', - 'set +e', - 'RSPAMD_INSTALLROOT=/rspamd/install robot --removekeywords wuks --exclude isbroken $DRONE_WORKSPACE/test/functional/cases; EXIT_CODE=$?', - 'set -e', - 'if [ -n "$HTTP_PUT_AUTH" ]; then $DRONE_WORKSPACE/test/tools/http_put.py log.html report.html https://$DRONE_SYSTEM_HOSTNAME/testlogs/$DRONE_REPO/${DRONE_BUILD_NUMBER}-' + arch + '/; fi\n', - "core_files=$(find /var/tmp/ -name '*.core')", - "for core in $core_files; do exe=$(gdb --batch -ex 'info proc mappings' -c $core | tail -1 | awk '{print $5}'); gdb --batch -ex 'bt' -c $core $exe; echo '---'; done\n", - 'exit $EXIT_CODE', - ], - environment: { - HTTP_PUT_AUTH: { - from_secret: 'http_put_auth', - }, - }, - } + rspamd_volumes, - { - name: 'send-coverage', - image: 'rspamd/ci:ubuntu-test', - pull: 'if-not-exists', - depends_on: [ - 'functional', - 'rspamd-test', - ], - commands: [ - 'cd /rspamd/build', - '$DRONE_WORKSPACE/test/tools/gcov_coveralls.py --exclude test --prefix /rspamd/build --prefix $DRONE_WORKSPACE --out coverage.c.json', - 'luacov-coveralls -o coverage.functional.lua.json --dryrun', - '$DRONE_WORKSPACE/test/tools/merge_coveralls.py --parallel --root $DRONE_WORKSPACE --input coverage.c.json unit_test_lua.json coverage.functional.lua.json --token=$COVERALLS_REPO_TOKEN', - ], - environment: { - COVERALLS_REPO_TOKEN: { - from_secret: 'coveralls_repo_token', - }, - }, - } + coveralls_when + rspamd_volumes, - ], - volumes: [ - { - name: 'rspamd', - temp: {}, - }, - ], -} + platform(arch) + default_trigger + docker_pipeline; - -local close_coveralls = { - name: 'close_coveralls', - depends_on: [ - 'default-amd64', - 'default-arm64', - ], - steps: [ - { - name: 'close_coveralls', - image: 'rspamd/ci:ubuntu-test-func', - pull: 'always', - commands: [ - '$DRONE_WORKSPACE/test/tools/merge_coveralls.py --parallel-close --token=$COVERALLS_REPO_TOKEN', - ], - environment: { - COVERALLS_REPO_TOKEN: { - from_secret: 'coveralls_repo_token', - }, - }, - }, - ], -} + close_coveralls_trigger + docker_pipeline; - -local noarch_pipeline = { - name: 'default-noarch', - steps: [ - { - name: 'perl-tidyall', - image: 'rspamd/ci:perl-tidyall', - pull: 'if-not-exists', - failure: 'ignore', - commands: [ - 'tidyall --version', - 'perltidy --version | head -1', - 'tidyall --all --check-only --no-cache --data-dir /tmp/tidyall', - ], - }, - { - name: 'eslint', - image: 'node:18-alpine', - pull: 'if-not-exists', - failure: 'ignore', - commands: [ - 'npm install', - 'npm ls', - './node_modules/.bin/eslint ./', - './node_modules/.bin/stylelint ./**/*.css ./**/*.html ./**/*.js', - ], - }, - { - name: 'luacheck', - image: 'pipelinecomponents/luacheck', - pull: 'if-not-exists', - commands: [ - 'luacheck -q --no-color .', - ], - }, - ], -} + default_trigger + docker_pipeline; - -local signature = { - kind: 'signature', - hmac: '0000000000000000000000000000000000000000000000000000000000000000', -}; - -[ - pipeline('amd64'), - pipeline('arm64'), - close_coveralls, - noarch_pipeline, - notify_pipeline, - signature, -] diff --git a/.drone.yml b/.drone.yml deleted file mode 100644 index cad368960..000000000 --- a/.drone.yml +++ /dev/null @@ -1,563 +0,0 @@ ---- -{ - "depends_on": [ ], - "kind": "pipeline", - "name": "default-amd64", - "platform": { - "arch": "amd64", - "os": "linux" - }, - "steps": [ - { - "commands": [ - "install -d -o nobody -g nogroup /rspamd/build /rspamd/install /rspamd/fedora/build /rspamd/fedora/install" - ], - "image": "ubuntu:22.04", - "name": "prepare", - "pull": "if-not-exists", - "volumes": [ - { - "name": "rspamd", - "path": "/rspamd" - } - ] - }, - { - "commands": [ - "test \"$(id -un)\" = nobody", - "cd /rspamd/build", - "cmake -DCMAKE_INSTALL_PREFIX=/rspamd/install -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_COVERAGE=ON -DENABLE_LIBUNWIND=ON -DENABLE_HYPERSCAN=ON -GNinja $DRONE_WORKSPACE\n", - "ncpu=$(getconf _NPROCESSORS_ONLN)", - "ninja -j $ncpu install", - "ninja -j $ncpu rspamd-test", - "ninja -j $ncpu rspamd-test-cxx" - ], - "depends_on": [ - "prepare" - ], - "image": "rspamd/ci:ubuntu-build", - "name": "build", - "pull": "always", - "volumes": [ - { - "name": "rspamd", - "path": "/rspamd" - } - ] - }, - { - "commands": [ - "test \"$(id -un)\" = nobody", - "cd /rspamd/fedora/build", - "export LDFLAGS='-fuse-ld=lld'", - "export ASAN_OPTIONS=detect_leaks=0", - "cmake -DCMAKE_INSTALL_PREFIX=/rspamd/fedora/install -DCMAKE_C_COMPILER=/usr/bin/clang -DCMAKE_CXX_COMPILER=/usr/bin/clang++ -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_CLANG_PLUGIN=ON -DENABLE_FULL_DEBUG=ON -DENABLE_HYPERSCAN=ON -DSANITIZE=address $DRONE_WORKSPACE\n", - "ncpu=$(getconf _NPROCESSORS_ONLN)", - "make -j $ncpu install", - "make -j $ncpu rspamd-test", - "make -j $ncpu rspamd-test-cxx" - ], - "depends_on": [ - "prepare" - ], - "image": "rspamd/ci:fedora-build", - "name": "build-clang", - "pull": "always", - "volumes": [ - { - "name": "rspamd", - "path": "/rspamd" - } - ] - }, - { - "commands": [ - "test \"$(id -un)\" = nobody", - "ulimit -c unlimited", - "cd /rspamd/build/test", - "set +e", - "env RSPAMD_LUA_EXPENSIVE_TESTS=1 ./rspamd-test -p /rspamd/lua; EXIT_CODE=$?", - "set -e", - "if [ $EXIT_CODE -gt 128 ]; then gdb --batch -ex 'thread apply all bt full' -c /var/tmp/*.rspamd-test.core ./rspamd-test; exit $EXIT_CODE; fi; if [ $EXIT_CODE -ne 0 ]; then exit $EXIT_CODE; fi\n", - "luacov-coveralls -o /rspamd/build/unit_test_lua.json --dryrun", - "set +e", - "./rspamd-test-cxx -s; EXIT_CODE=$?", - "set -e", - "if [ $EXIT_CODE -gt 128 ]; then gdb --batch -ex 'thread apply all bt full' -c /var/tmp/*.rspamd-test-cxx.core ./rspamd-test-cxx; exit $EXIT_CODE; fi\n", - "exit $EXIT_CODE" - ], - "depends_on": [ - "build" - ], - "image": "rspamd/ci:ubuntu-test", - "name": "rspamd-test", - "pull": "always", - "volumes": [ - { - "name": "rspamd", - "path": "/rspamd" - } - ] - }, - { - "commands": [ - "test \"$(id -un)\" = nobody", - "ulimit -c 2097152", - "ulimit -s unlimited", - "export ASAN_OPTIONS=\"detect_leaks=0:print_stacktrace=1:disable_coredump=0\"", - "export UBSAN_OPTIONS=\"print_stacktrace=1:print_summary=0:log_path=/tmp/ubsan\"", - "cd /rspamd/fedora/build/test", - "set +e", - "env RSPAMD_LUA_EXPENSIVE_TESTS=1 ./rspamd-test -p /rspamd/lua; EXIT_CODE=$?", - "set -e", - "if [ $EXIT_CODE -gt 128 ]; then gdb --batch -ex 'bt' -c /var/tmp/*.rspamd-test.core ./rspamd-test; fi\n", - "set +e", - "./rspamd-test-cxx -s; EXIT_CODE=$?", - "set -e", - "if [ $EXIT_CODE -gt 128 ]; then gdb --batch -ex 'thread apply all bt full' -c /var/tmp/*.rspamd-test-cxx.core ./rspamd-test-cxx; exit $EXIT_CODE; fi\n", - "cat /tmp/ubsan.* || true", - "exit $EXIT_CODE" - ], - "depends_on": [ - "build-clang" - ], - "image": "rspamd/ci:fedora-test", - "name": "test-fedora-clang", - "pull": "always", - "volumes": [ - { - "name": "rspamd", - "path": "/rspamd" - } - ] - }, - { - "commands": [ - "cd /rspamd/build", - "ulimit -c unlimited", - "ulimit -s unlimited", - "set +e", - "RSPAMD_INSTALLROOT=/rspamd/install robot --removekeywords wuks --exclude isbroken $DRONE_WORKSPACE/test/functional/cases; EXIT_CODE=$?", - "set -e", - "if [ -n \"$HTTP_PUT_AUTH\" ]; then $DRONE_WORKSPACE/test/tools/http_put.py log.html report.html https://$DRONE_SYSTEM_HOSTNAME/testlogs/$DRONE_REPO/${DRONE_BUILD_NUMBER}-amd64/; fi\n", - "core_files=$(find /var/tmp/ -name '*.core')", - "for core in $core_files; do exe=$(gdb --batch -ex 'info proc mappings' -c $core | tail -1 | awk '{print $5}'); gdb --batch -ex 'bt' -c $core $exe; echo '---'; done\n", - "exit $EXIT_CODE" - ], - "depends_on": [ - "build" - ], - "environment": { - "HTTP_PUT_AUTH": { - "from_secret": "http_put_auth" - } - }, - "image": "rspamd/ci:ubuntu-test-func", - "name": "functional", - "pull": "always", - "volumes": [ - { - "name": "rspamd", - "path": "/rspamd" - } - ] - }, - { - "commands": [ - "cd /rspamd/build", - "$DRONE_WORKSPACE/test/tools/gcov_coveralls.py --exclude test --prefix /rspamd/build --prefix $DRONE_WORKSPACE --out coverage.c.json", - "luacov-coveralls -o coverage.functional.lua.json --dryrun", - "$DRONE_WORKSPACE/test/tools/merge_coveralls.py --parallel --root $DRONE_WORKSPACE --input coverage.c.json unit_test_lua.json coverage.functional.lua.json --token=$COVERALLS_REPO_TOKEN" - ], - "depends_on": [ - "functional", - "rspamd-test" - ], - "environment": { - "COVERALLS_REPO_TOKEN": { - "from_secret": "coveralls_repo_token" - } - }, - "image": "rspamd/ci:ubuntu-test", - "name": "send-coverage", - "pull": "if-not-exists", - "volumes": [ - { - "name": "rspamd", - "path": "/rspamd" - } - ], - "when": { - "branch": [ - "master" - ], - "event": [ - "push", - "tag" - ] - } - } - ], - "trigger": { - "event": [ - "push", - "tag", - "custom", - "pull_request" - ] - }, - "type": "docker", - "volumes": [ - { - "name": "rspamd", - "temp": { } - } - ] -} ---- -{ - "depends_on": [ ], - "kind": "pipeline", - "name": "default-arm64", - "platform": { - "arch": "arm64", - "os": "linux" - }, - "steps": [ - { - "commands": [ - "install -d -o nobody -g nogroup /rspamd/build /rspamd/install /rspamd/fedora/build /rspamd/fedora/install" - ], - "image": "ubuntu:22.04", - "name": "prepare", - "pull": "if-not-exists", - "volumes": [ - { - "name": "rspamd", - "path": "/rspamd" - } - ] - }, - { - "commands": [ - "test \"$(id -un)\" = nobody", - "cd /rspamd/build", - "cmake -DCMAKE_INSTALL_PREFIX=/rspamd/install -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_COVERAGE=ON -DENABLE_LIBUNWIND=ON -DENABLE_HYPERSCAN=ON -DHYPERSCAN_ROOT_DIR=/vectorscan -GNinja $DRONE_WORKSPACE\n", - "ncpu=$(getconf _NPROCESSORS_ONLN)", - "ninja -j $ncpu install", - "ninja -j $ncpu rspamd-test", - "ninja -j $ncpu rspamd-test-cxx" - ], - "depends_on": [ - "prepare" - ], - "image": "rspamd/ci:ubuntu-build", - "name": "build", - "pull": "always", - "volumes": [ - { - "name": "rspamd", - "path": "/rspamd" - } - ] - }, - { - "commands": [ - "test \"$(id -un)\" = nobody", - "cd /rspamd/fedora/build", - "export LDFLAGS='-fuse-ld=lld'", - "export ASAN_OPTIONS=detect_leaks=0", - "cmake -DCMAKE_INSTALL_PREFIX=/rspamd/fedora/install -DCMAKE_C_COMPILER=/usr/bin/clang -DCMAKE_CXX_COMPILER=/usr/bin/clang++ -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_CLANG_PLUGIN=ON -DENABLE_FULL_DEBUG=ON -DENABLE_HYPERSCAN=ON -DHYPERSCAN_ROOT_DIR=/vectorscan -DSANITIZE=address $DRONE_WORKSPACE\n", - "ncpu=$(getconf _NPROCESSORS_ONLN)", - "make -j $ncpu install", - "make -j $ncpu rspamd-test", - "make -j $ncpu rspamd-test-cxx" - ], - "depends_on": [ - "prepare" - ], - "image": "rspamd/ci:fedora-build", - "name": "build-clang", - "pull": "always", - "volumes": [ - { - "name": "rspamd", - "path": "/rspamd" - } - ] - }, - { - "commands": [ - "test \"$(id -un)\" = nobody", - "ulimit -c unlimited", - "cd /rspamd/build/test", - "set +e", - "env RSPAMD_LUA_EXPENSIVE_TESTS=1 ./rspamd-test -p /rspamd/lua; EXIT_CODE=$?", - "set -e", - "if [ $EXIT_CODE -gt 128 ]; then gdb --batch -ex 'thread apply all bt full' -c /var/tmp/*.rspamd-test.core ./rspamd-test; exit $EXIT_CODE; fi; if [ $EXIT_CODE -ne 0 ]; then exit $EXIT_CODE; fi\n", - "luacov-coveralls -o /rspamd/build/unit_test_lua.json --dryrun", - "set +e", - "./rspamd-test-cxx -s; EXIT_CODE=$?", - "set -e", - "if [ $EXIT_CODE -gt 128 ]; then gdb --batch -ex 'thread apply all bt full' -c /var/tmp/*.rspamd-test-cxx.core ./rspamd-test-cxx; exit $EXIT_CODE; fi\n", - "exit $EXIT_CODE" - ], - "depends_on": [ - "build" - ], - "image": "rspamd/ci:ubuntu-test", - "name": "rspamd-test", - "pull": "always", - "volumes": [ - { - "name": "rspamd", - "path": "/rspamd" - } - ] - }, - { - "commands": [ - "test \"$(id -un)\" = nobody", - "ulimit -c 2097152", - "ulimit -s unlimited", - "export ASAN_OPTIONS=\"detect_leaks=0:print_stacktrace=1:disable_coredump=0\"", - "export UBSAN_OPTIONS=\"print_stacktrace=1:print_summary=0:log_path=/tmp/ubsan\"", - "cd /rspamd/fedora/build/test", - "set +e", - "env RSPAMD_LUA_EXPENSIVE_TESTS=1 ./rspamd-test -p /rspamd/lua; EXIT_CODE=$?", - "set -e", - "if [ $EXIT_CODE -gt 128 ]; then gdb --batch -ex 'bt' -c /var/tmp/*.rspamd-test.core ./rspamd-test; fi\n", - "set +e", - "./rspamd-test-cxx -s; EXIT_CODE=$?", - "set -e", - "if [ $EXIT_CODE -gt 128 ]; then gdb --batch -ex 'thread apply all bt full' -c /var/tmp/*.rspamd-test-cxx.core ./rspamd-test-cxx; exit $EXIT_CODE; fi\n", - "cat /tmp/ubsan.* || true", - "exit $EXIT_CODE" - ], - "depends_on": [ - "build-clang" - ], - "image": "rspamd/ci:fedora-test", - "name": "test-fedora-clang", - "pull": "always", - "volumes": [ - { - "name": "rspamd", - "path": "/rspamd" - } - ] - }, - { - "commands": [ - "cd /rspamd/build", - "ulimit -c unlimited", - "ulimit -s unlimited", - "set +e", - "RSPAMD_INSTALLROOT=/rspamd/install robot --removekeywords wuks --exclude isbroken $DRONE_WORKSPACE/test/functional/cases; EXIT_CODE=$?", - "set -e", - "if [ -n \"$HTTP_PUT_AUTH\" ]; then $DRONE_WORKSPACE/test/tools/http_put.py log.html report.html https://$DRONE_SYSTEM_HOSTNAME/testlogs/$DRONE_REPO/${DRONE_BUILD_NUMBER}-arm64/; fi\n", - "core_files=$(find /var/tmp/ -name '*.core')", - "for core in $core_files; do exe=$(gdb --batch -ex 'info proc mappings' -c $core | tail -1 | awk '{print $5}'); gdb --batch -ex 'bt' -c $core $exe; echo '---'; done\n", - "exit $EXIT_CODE" - ], - "depends_on": [ - "build" - ], - "environment": { - "HTTP_PUT_AUTH": { - "from_secret": "http_put_auth" - } - }, - "image": "rspamd/ci:ubuntu-test-func", - "name": "functional", - "pull": "always", - "volumes": [ - { - "name": "rspamd", - "path": "/rspamd" - } - ] - }, - { - "commands": [ - "cd /rspamd/build", - "$DRONE_WORKSPACE/test/tools/gcov_coveralls.py --exclude test --prefix /rspamd/build --prefix $DRONE_WORKSPACE --out coverage.c.json", - "luacov-coveralls -o coverage.functional.lua.json --dryrun", - "$DRONE_WORKSPACE/test/tools/merge_coveralls.py --parallel --root $DRONE_WORKSPACE --input coverage.c.json unit_test_lua.json coverage.functional.lua.json --token=$COVERALLS_REPO_TOKEN" - ], - "depends_on": [ - "functional", - "rspamd-test" - ], - "environment": { - "COVERALLS_REPO_TOKEN": { - "from_secret": "coveralls_repo_token" - } - }, - "image": "rspamd/ci:ubuntu-test", - "name": "send-coverage", - "pull": "if-not-exists", - "volumes": [ - { - "name": "rspamd", - "path": "/rspamd" - } - ], - "when": { - "branch": [ - "master" - ], - "event": [ - "push", - "tag" - ] - } - } - ], - "trigger": { - "event": [ - "push", - "tag", - "custom", - "pull_request" - ] - }, - "type": "docker", - "volumes": [ - { - "name": "rspamd", - "temp": { } - } - ] -} ---- -{ - "depends_on": [ - "default-amd64", - "default-arm64" - ], - "kind": "pipeline", - "name": "close_coveralls", - "steps": [ - { - "commands": [ - "$DRONE_WORKSPACE/test/tools/merge_coveralls.py --parallel-close --token=$COVERALLS_REPO_TOKEN" - ], - "environment": { - "COVERALLS_REPO_TOKEN": { - "from_secret": "coveralls_repo_token" - } - }, - "image": "rspamd/ci:ubuntu-test-func", - "name": "close_coveralls", - "pull": "always" - } - ], - "trigger": { - "branch": [ - "master" - ], - "event": [ - "push", - "tag" - ], - "status": [ - "success", - "failure" - ] - }, - "type": "docker" -} ---- -{ - "kind": "pipeline", - "name": "default-noarch", - "steps": [ - { - "commands": [ - "tidyall --version", - "perltidy --version | head -1", - "tidyall --all --check-only --no-cache --data-dir /tmp/tidyall" - ], - "failure": "ignore", - "image": "rspamd/ci:perl-tidyall", - "name": "perl-tidyall", - "pull": "if-not-exists" - }, - { - "commands": [ - "npm install", - "npm ls", - "./node_modules/.bin/eslint ./", - "./node_modules/.bin/stylelint ./**/*.css ./**/*.html ./**/*.js" - ], - "failure": "ignore", - "image": "node:18-alpine", - "name": "eslint", - "pull": "if-not-exists" - }, - { - "commands": [ - "luacheck -q --no-color ." - ], - "image": "pipelinecomponents/luacheck", - "name": "luacheck", - "pull": "if-not-exists" - } - ], - "trigger": { - "event": [ - "push", - "tag", - "custom", - "pull_request" - ] - }, - "type": "docker" -} ---- -{ - "depends_on": [ - "default-amd64", - "default-arm64", - "default-noarch" - ], - "kind": "pipeline", - "name": "notify", - "steps": [ - { - "image": "drillster/drone-email", - "name": "notify", - "pull": "if-not-exists", - "settings": { - "from": "noreply@rspamd.com", - "host": { - "from_secret": "email_host" - }, - "password": { - "from_secret": "email_password" - }, - "username": { - "from_secret": "email_username" - } - } - } - ], - "trigger": { - "event": [ - "push", - "tag", - "custom" - ], - "status": [ - "failure" - ] - }, - "type": "docker" -} ---- -{ - "hmac": "2351718d9a562ea71ff344fb39fcf4ad5dae5b9694219b933c1b63a8b87d2aa5", - "kind": "signature" -} -... diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..d92d5105e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,25 @@ +name: ci + +on: + pull_request: + branches: + - master + push: + branches: + - master + +jobs: + linters: + uses: ./.github/workflows/ci_linters.yml + + fedora: + uses: ./.github/workflows/ci_rspamd.yml + with: + image: ghcr.io/rspamd/rspamd-build-docker:fedora-ci + name: fedora-ci + + ubuntu: + uses: ./.github/workflows/ci_rspamd.yml + with: + image: ghcr.io/rspamd/rspamd-build-docker:ubuntu-ci + name: ubuntu-ci diff --git a/.github/workflows/ci_eslint.yml b/.github/workflows/ci_eslint.yml new file mode 100644 index 000000000..e7220d468 --- /dev/null +++ b/.github/workflows/ci_eslint.yml @@ -0,0 +1,25 @@ +name: ci_eslint + +on: + workflow_call: + +jobs: + eslint: + runs-on: ubuntu-latest + container: + image: node:18-alpine + steps: + - name: Check out source code + uses: actions/checkout@v4 + + - name: Install dependencies + run: npm install + + - name: Show installed packages + run: npm ls + + - name: Run eslint + run: ./node_modules/.bin/eslint ./ + + - name: Run stylelint + run: ./node_modules/.bin/stylelint ./**/*.css ./**/*.html ./**/*.js diff --git a/.github/workflows/ci_linters.yml b/.github/workflows/ci_linters.yml new file mode 100644 index 000000000..6df1a4480 --- /dev/null +++ b/.github/workflows/ci_linters.yml @@ -0,0 +1,14 @@ +name: ci_linters + +on: + workflow_call: + +jobs: + eslint: + uses: ./.github/workflows/ci_eslint.yml + + luacheck: + uses: ./.github/workflows/ci_luacheck.yml + + tidyall: + uses: ./.github/workflows/ci_tidyall.yml diff --git a/.github/workflows/ci_luacheck.yml b/.github/workflows/ci_luacheck.yml new file mode 100644 index 000000000..d930fdae5 --- /dev/null +++ b/.github/workflows/ci_luacheck.yml @@ -0,0 +1,16 @@ +name: ci_luacheck + +on: + workflow_call: + +jobs: + luacheck: + runs-on: ubuntu-latest + container: + image: pipelinecomponents/luacheck + steps: + - name: Check out source code + uses: actions/checkout@v4 + + - name: Run luacheck + run: luacheck -q --no-color . diff --git a/.github/workflows/ci_rspamd.yml b/.github/workflows/ci_rspamd.yml new file mode 100644 index 000000000..ebd4f2cf3 --- /dev/null +++ b/.github/workflows/ci_rspamd.yml @@ -0,0 +1,94 @@ +name: rspamd_test + +on: + workflow_call: + inputs: + image: + required: true + type: string + name: + required: true + type: string + +env: + CTEST_OUTPUT_ON_FAILURE: 1 + RSPAMD_LUA_EXPENSIVE_TESTS: 1 + +jobs: + test: + runs-on: [ "ubuntu-latest" ] + container: + image: ${{ inputs.image }} + options: --user root + steps: + - name: Create directories + run: | + sudo mkdir -p ${GITHUB_WORKSPACE} + sudo chown -R build:build ${GITHUB_WORKSPACE} + + - name: Check out source code + uses: actions/checkout@v4 + with: + path: src + + - name: Set variables on ARM64 + if: runner.arch == 'ARM64' + run: echo "HYPERSCAN_ALTROOT=-DHYPERSCAN_ROOT_DIR=/vectorscan" >> "$GITHUB_ENV" + + - name: Run cmake + run: | + mkdir ${GITHUB_WORKSPACE}/build + cd ${GITHUB_WORKSPACE}/build + cmake -DCMAKE_INSTALL_PREFIX=${GITHUB_WORKSPACE}/install -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_COVERAGE=ON -DENABLE_LIBUNWIND=ON -DENABLE_HYPERSCAN=ON ${{ env.HYPERSCAN_ALTROOT }} -GNinja ${GITHUB_WORKSPACE}/src + + - name: Build rspamd + run: | + cd ${GITHUB_WORKSPACE}/build + ncpu=$(getconf _NPROCESSORS_ONLN) + ninja -j $ncpu install + ninja -j $ncpu rspamd-test + ninja -j $ncpu rspamd-test-cxx + + - name: Run unit tests + if: "!(inputs.name == 'ubuntu-ci' && runner.arch == 'ARM64')" + run: | + cd ${GITHUB_WORKSPACE}/build + ninja test + + - name: Apply Fedora specifics + if: inputs.name == 'fedora-ci' + run: | + sudo mv /usr/bin/miltertest /usr/bin/miltertest.is.broken.on.fedora || true + + - name: Run functional tests + run: | + cd ${GITHUB_WORKSPACE}/build + ulimit -c unlimited + ulimit -s unlimited + set +e + RSPAMD_INSTALLROOT=${GITHUB_WORKSPACE}/install robot -v RSPAMD_USER:root -v RSPAMD_GROUP:root --removekeywords wuks --exclude isbroken ${GITHUB_WORKSPACE}/src/test/functional/cases; EXIT_CODE=$? + set -e + core_files=$(find /var/tmp/ -name '*.core') + for core in $core_files; do exe=$(gdb --batch -ex 'info proc mappings' -c $core | tail -1 | awk '{print $5}'); gdb --batch -ex 'bt' -c $core $exe; echo '---'; done + exit $EXIT_CODE + + - name: Save workspace directory + run: echo "CONTAINER_WORKSPACE=${GITHUB_WORKSPACE}" >> "$GITHUB_ENV" + + # FIXME: upload test logs even on ARM + - name: Upload robot logs + if: runner.arch == 'X64' && (success() || failure()) + uses: actions/upload-artifact@v4 + with: + name: robotlog-${{ inputs.name }} + path: | + ${{ env.CONTAINER_WORKSPACE }}/build/*.*ml + retention-days: 1 + + - name: Upload rspamd logs + if: runner.arch == 'X64' && (success() || failure()) + uses: actions/upload-artifact@v4 + with: + name: rspamdlog-${{ inputs.name }} + path: ${{ env.CONTAINER_WORKSPACE }}/build/robot-save + retention-days: 1 diff --git a/.github/workflows/ci_tidyall.yml b/.github/workflows/ci_tidyall.yml new file mode 100644 index 000000000..af22d8c3a --- /dev/null +++ b/.github/workflows/ci_tidyall.yml @@ -0,0 +1,23 @@ +name: ci_tidyall + +on: + workflow_call: + +jobs: + tidyall: + runs-on: ubuntu-latest + container: + image: rspamd/ci:perl-tidyall + options: --user root + steps: + - name: Check out source code + uses: actions/checkout@v4 + + - name: Show tidyall version + run: tidyall --version + + - name: Show perltidy version + run: perltidy --version | head -1 + + - name: Run tidyall + run: tidyall --all --check-only --no-cache --data-dir /tmp/tidyall @@ -1,6 +1,6 @@ # <a href="https://rspamd.com"><img src="https://rspamd.com/img/rspamd_logo_black.png" alt="Rspamd" width="220px"/></a> -[](https://ci.rspamd.com/rspamd/rspamd) +[](https://github.com/rspamd/rspamd/actions/workflows/ci.yml) ## Introduction diff --git a/contrib/libucl/ucl_parser.c b/contrib/libucl/ucl_parser.c index 354bfe857..b18fd06ce 100644 --- a/contrib/libucl/ucl_parser.c +++ b/contrib/libucl/ucl_parser.c @@ -2984,7 +2984,7 @@ ucl_parser_add_chunk_full (struct ucl_parser *parser, const unsigned char *data, if (parse_type == UCL_PARSE_AUTO && len > 0) { /* We need to detect parse type by the first symbol */ - if ((*data & 0x80) == 0x80 && (*data >= 0xdc && *data <= 0xdf)) { + if ((*data & 0x80) == 0x80) { parse_type = UCL_PARSE_MSGPACK; } else if (*data == '(') { diff --git a/lualib/lua_dkim_tools.lua b/lualib/lua_dkim_tools.lua index 165ea8f56..b7f520fae 100644 --- a/lualib/lua_dkim_tools.lua +++ b/lualib/lua_dkim_tools.lua @@ -461,21 +461,20 @@ local function prepare_dkim_signing(N, task, settings) if settings.use_vault then if settings.vault_domains then if settings.vault_domains:get_key(dkim_domain) then - return true, { + table.insert(p, { domain = dkim_domain, vault = true, - } + }) else lua_util.debugm(N, task, 'domain %s is not designated for vault', dkim_domain) - return false, {} end else -- TODO: try every domain in the vault - return true, { + table.insert(p, { domain = dkim_domain, vault = true, - } + }) end end @@ -546,7 +545,7 @@ local function prepare_dkim_signing(N, task, settings) insert_or_update_prop(N, task, p, 'domain', 'dkim_domain', dkim_domain) - return true, p + return #p > 0 and true or false, p end exports.prepare_dkim_signing = prepare_dkim_signing @@ -615,12 +614,12 @@ exports.sign_using_redis = function(N, task, settings, selectors, sign_func, err end end -exports.sign_using_vault = function(N, task, settings, selectors, sign_func, err_func) +exports.sign_using_vault = function(N, task, settings, selector, sign_func, err_func) local http = require "rspamd_http" local ucl = require "ucl" local full_url = string.format('%s/v1/%s/%s', - settings.vault_url, settings.vault_path or 'dkim', selectors.domain) + settings.vault_url, settings.vault_path or 'dkim', selector.domain) local upstream_list = lua_util.http_upstreams_by_url(rspamd_config:get_mempool(), settings.vault_url) local function vault_callback(err, code, body, _) @@ -641,22 +640,27 @@ exports.sign_using_vault = function(N, task, settings, selectors, sign_func, err full_url, body)) else local elts = obj.data.selectors or {} + local errs = {} + local nvalid = 0 -- Filter selectors by time/sanity local function is_selector_valid(p) if not p.key or not p.selector then + table.insert(errs, { "missing key/selector", p }) return false end if p.valid_start then -- Check start time if rspamd_util.get_time() < tonumber(p.valid_start) then + table.insert(errs, { "start time is in the future", p }) return false end end if p.valid_end then if rspamd_util.get_time() >= tonumber(p.valid_end) then + table.insert(errs, { "end time is in the past", p }) return false end end @@ -667,13 +671,22 @@ exports.sign_using_vault = function(N, task, settings, selectors, sign_func, err local dkim_sign_data = { rawkey = p.key, selector = p.selector, - domain = p.domain or selectors.domain, + domain = p.domain or selector.domain, alg = p.alg, } lua_util.debugm(N, task, 'found and parsed key for %s:%s in Vault', dkim_sign_data.domain, dkim_sign_data.selector) + nvalid = nvalid + 1 sign_func(task, dkim_sign_data) end, fun.filter(is_selector_valid, elts)) + for _, e in errs do + lua_util.debugm(N, task, 'error found during processing Vault selectors: %s:%s', + e[1], e[2]) + end + + if nvalid == 0 then + lua_util.debugm(N, task, 'no valid selectors have been returned from the Vault, skip signing') + end end end end @@ -694,7 +707,7 @@ exports.sign_using_vault = function(N, task, settings, selectors, sign_func, err if not ret then err_func(task, string.format("cannot make HTTP request to load DKIM data domain %s", - selectors.domain)) + selector.domain)) end end diff --git a/lualib/lua_maps.lua b/lualib/lua_maps.lua index d3573100a..a912039be 100644 --- a/lualib/lua_maps.lua +++ b/lualib/lua_maps.lua @@ -294,6 +294,11 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) end if opt[1] then + local function check_plain_map(line) + return lua_util.str_startswith(line, 'http') + or lua_util.str_startswith(line, 'file:') + or lua_util.str_startswith(line, '/') + end -- Adjust each element if needed local adjusted for i, source in ipairs(opt) do @@ -311,6 +316,7 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) if mtype == 'radix' then if string.find(opt[1], '^%d') then + -- List of numeric stuff (hope it's ipnets definitions) local map = rspamd_config:radix_from_ucl(opt) if map then @@ -338,7 +344,7 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) end end elseif mtype == 'regexp' or mtype == 'glob' then - if string.find(opt[1], '^/%a') or string.find(opt[1], '^http') then + if check_plain_map(opt[1]) then -- Plain table local map = rspamd_config:add_map { type = mtype, @@ -372,7 +378,8 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) end end else - if string.find(opt[1], '^/%a') or string.find(opt[1], '^http') then + -- Not regexp/glob + if check_plain_map(opt[1]) then -- Plain table local map = rspamd_config:add_map { type = mtype, diff --git a/src/client/rspamc.cxx b/src/client/rspamc.cxx index 50e3cbdee..d1b6a8ed0 100644 --- a/src/client/rspamc.cxx +++ b/src/client/rspamc.cxx @@ -1,5 +1,5 @@ /* - * Copyright 2023 Vsevolod Stakhov + * Copyright 2024 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -80,7 +80,7 @@ static gboolean ucl_reply = FALSE; static gboolean extended_urls = FALSE; static gboolean mime_output = FALSE; static gboolean empty_input = FALSE; -static gboolean compressed = FALSE; +static gboolean compressed = TRUE; static gboolean profile = FALSE; static gboolean skip_images = FALSE; static gboolean skip_attachments = FALSE; diff --git a/src/client/rspamdclient.c b/src/client/rspamdclient.c index 85f4749f1..e77f662eb 100644 --- a/src/client/rspamdclient.c +++ b/src/client/rspamdclient.c @@ -1,11 +1,11 @@ -/*- - * Copyright 2016 Vsevolod Stakhov +/* + * Copyright 2024 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -232,7 +232,9 @@ rspamd_client_finish_handler(struct rspamd_http_connection *conn, } parser = ucl_parser_new(0); - if (!ucl_parser_add_chunk(parser, start, len)) { + if (!ucl_parser_add_chunk_full(parser, start, len, + ucl_parser_get_default_priority(parser), + UCL_DUPLICATE_APPEND, UCL_PARSE_AUTO)) { err = g_error_new(RCLIENT_ERROR, msg->code, "Cannot parse UCL: %s", ucl_parser_get_error(parser)); ucl_parser_free(parser); @@ -454,6 +456,11 @@ rspamd_client_command(struct rspamd_client_connection *conn, rspamd_http_message_add_header(req->msg, "Filename", filename); } + /* + * Allow messagepack reply if supported + */ + rspamd_http_message_add_header(req->msg, "Accept", "application/msgpack"); + req->msg->url = rspamd_fstring_append(req->msg->url, "/", 1); req->msg->url = rspamd_fstring_append(req->msg->url, command, strlen(command)); diff --git a/src/controller.c b/src/controller.c index c9cda1d62..e68e10ced 100644 --- a/src/controller.c +++ b/src/controller.c @@ -1574,7 +1574,6 @@ rspamd_controller_handle_lua_history(lua_State *L, } task->http_conn = rspamd_http_connection_ref(conn_ent->conn); - ; task->sock = -1; session->task = task; @@ -1904,7 +1903,6 @@ rspamd_controller_handle_lua(struct rspamd_http_connection_entry *conn_ent, task); task->fin_arg = conn_ent; task->http_conn = rspamd_http_connection_ref(conn_ent->conn); - ; task->sock = -1; session->task = task; @@ -2005,16 +2003,25 @@ rspamd_controller_scan_reply(struct rspamd_task *task) { struct rspamd_http_message *msg; struct rspamd_http_connection_entry *conn_ent; + int out_type = UCL_EMIT_JSON_COMPACT; + const char *ctype = "application/json"; + const rspamd_ftok_t *accept_hdr = rspamd_task_get_request_header(task, "Accept"); + + if (accept_hdr && rspamd_substring_search(accept_hdr->begin, accept_hdr->len, + "application/msgpack", sizeof("application/msgpack") - 1) != -1) { + ctype = "application/msgpack"; + out_type = UCL_EMIT_MSGPACK; + } conn_ent = task->fin_arg; msg = rspamd_http_new_message(HTTP_RESPONSE); msg->date = time(NULL); msg->code = 200; - rspamd_protocol_http_reply(msg, task, NULL); + rspamd_protocol_http_reply(msg, task, NULL, out_type); rspamd_http_connection_reset(conn_ent->conn); rspamd_http_router_insert_headers(conn_ent->rt, msg); rspamd_http_connection_write_message(conn_ent->conn, msg, NULL, - "application/json", conn_ent, conn_ent->rt->timeout); + ctype, conn_ent, conn_ent->rt->timeout); conn_ent->is_reply = TRUE; } diff --git a/src/libmime/lang_detection.c b/src/libmime/lang_detection.c index bdd0aad74..c485de5ad 100644 --- a/src/libmime/lang_detection.c +++ b/src/libmime/lang_detection.c @@ -898,7 +898,7 @@ rspamd_language_detector_init(struct rspamd_config *cfg) rspamd_language_detector_process_chain(cfg, chain); }); - if (!rspamd_multipattern_compile(ret->stop_words[i].mp, &err)) { + if (!rspamd_multipattern_compile(ret->stop_words[i].mp, 0, &err)) { msg_err_config("cannot compile stop words for %z language group: %e", i, err); g_error_free(err); diff --git a/src/libmime/message.c b/src/libmime/message.c index 4fb118e60..f73c1ee35 100644 --- a/src/libmime/message.c +++ b/src/libmime/message.c @@ -674,7 +674,7 @@ rspamd_check_gtube(struct rspamd_task *task, struct rspamd_mime_text_part *part) RSPAMD_MULTIPATTERN_DEFAULT); GError *err = NULL; - rspamd_multipattern_compile(gtube_matcher, &err); + rspamd_multipattern_compile(gtube_matcher, RSPAMD_MULTIPATTERN_COMPILE_NO_FS, &err); if (err != NULL) { /* It will be expensive, but I don't care, still better than to abort */ diff --git a/src/libmime/mime_parser.c b/src/libmime/mime_parser.c index 217f0b87d..562a9d7de 100644 --- a/src/libmime/mime_parser.c +++ b/src/libmime/mime_parser.c @@ -1,11 +1,11 @@ -/*- - * Copyright 2016 Vsevolod Stakhov +/* + * Copyright 2024 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -169,7 +169,7 @@ rspamd_mime_parser_init_lib(void) rspamd_multipattern_add_pattern(lib_ctx->mp_boundary, "\n--", 0); GError *err = NULL; - if (!rspamd_multipattern_compile(lib_ctx->mp_boundary, &err)) { + if (!rspamd_multipattern_compile(lib_ctx->mp_boundary, RSPAMD_MULTIPATTERN_COMPILE_NO_FS, &err)) { msg_err("fatal error: cannot compile multipattern for mime parser boundaries: %e", err); g_error_free(err); g_abort(); diff --git a/src/libserver/protocol.c b/src/libserver/protocol.c index 37a41c111..a251dcd05 100644 --- a/src/libserver/protocol.c +++ b/src/libserver/protocol.c @@ -1636,7 +1636,7 @@ rspamd_protocol_write_ucl(struct rspamd_task *task, } void rspamd_protocol_http_reply(struct rspamd_http_message *msg, - struct rspamd_task *task, ucl_object_t **pobj) + struct rspamd_task *task, ucl_object_t **pobj, int how) { struct rspamd_scan_result *metric_res; const struct rspamd_re_cache_stat *restat; @@ -1695,7 +1695,7 @@ void rspamd_protocol_http_reply(struct rspamd_http_message *msg, if (msg->method < HTTP_SYMBOLS && !RSPAMD_TASK_IS_SPAMC(task)) { msg_debug_protocol("writing json reply"); - rspamd_ucl_emit_fstring(top, UCL_EMIT_JSON_COMPACT, &reply); + rspamd_ucl_emit_fstring(top, how, &reply); } else { if (RSPAMD_TASK_IS_SPAMC(task)) { @@ -2111,6 +2111,16 @@ void rspamd_protocol_write_reply(struct rspamd_task *task, ev_tstamp timeout) MESSAGE_FIELD_CHECK(task, message_id)); } + const rspamd_ftok_t *accept_hdr; + int out_type = UCL_EMIT_JSON_COMPACT; + accept_hdr = rspamd_task_get_request_header(task, "Accept"); + + if (accept_hdr && rspamd_substring_search(accept_hdr->begin, accept_hdr->len, + "application/msgpack", sizeof("application/msgpack") - 1) != -1) { + ctype = "application/msgpack"; + out_type = UCL_EMIT_MSGPACK; + } + /* Compatibility */ if (task->cmd == CMD_CHECK_RSPAMC) { msg->method = HTTP_SYMBOLS; @@ -2134,15 +2144,15 @@ void rspamd_protocol_write_reply(struct rspamd_task *task, ev_tstamp timeout) ucl_object_fromstring(g_quark_to_string(task->err->domain)), "error_domain", 0, false); reply = rspamd_fstring_sized_new(256); - rspamd_ucl_emit_fstring(top, UCL_EMIT_JSON_COMPACT, &reply); + rspamd_ucl_emit_fstring(top, out_type, &reply); ucl_object_unref(top); /* We also need to validate utf8 */ - if (rspamd_fast_utf8_validate(reply->str, reply->len) != 0) { + if (out_type != UCL_EMIT_MSGPACK && rspamd_fast_utf8_validate(reply->str, reply->len) != 0) { gsize valid_len; gchar *validated; - /* We copy reply several times here but it should be a rare case */ + /* We copy reply several times here, but it should be a rare case */ validated = rspamd_str_make_utf_valid(reply->str, reply->len, &valid_len, task->task_pool); rspamd_http_message_set_body(msg, validated, valid_len); @@ -2161,7 +2171,7 @@ void rspamd_protocol_write_reply(struct rspamd_task *task, ev_tstamp timeout) case CMD_CHECK_SPAMC: case CMD_SKIP: case CMD_CHECK_V2: - rspamd_protocol_http_reply(msg, task, NULL); + rspamd_protocol_http_reply(msg, task, NULL, out_type); rspamd_protocol_write_log_pipe(task); break; case CMD_PING: diff --git a/src/libserver/protocol.h b/src/libserver/protocol.h index 0e3c18744..38d9cef4b 100644 --- a/src/libserver/protocol.h +++ b/src/libserver/protocol.h @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Vsevolod Stakhov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /** * @file protocol.h * Rspamd protocol definition @@ -70,7 +86,8 @@ gboolean rspamd_protocol_handle_request(struct rspamd_task *task, * @param task */ void rspamd_protocol_http_reply(struct rspamd_http_message *msg, - struct rspamd_task *task, ucl_object_t **pobj); + struct rspamd_task *task, ucl_object_t **pobj, + int how); /** * Write data to log pipes diff --git a/src/libserver/url.c b/src/libserver/url.c index 0842a1ebd..ff75e3c76 100644 --- a/src/libserver/url.c +++ b/src/libserver/url.c @@ -1,5 +1,5 @@ /* - * Copyright 2023 Vsevolod Stakhov + * Copyright 2024 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -539,6 +539,7 @@ void rspamd_url_init(const gchar *tld_file) { GError *err = NULL; gboolean ret = TRUE; + int mp_compile_flags = 0; if (url_scanner != NULL) { rspamd_url_deinit(); @@ -564,6 +565,7 @@ void rspamd_url_init(const gchar *tld_file) url_scanner->matchers_full = NULL; url_scanner->search_trie_full = NULL; url_scanner->has_tld_file = false; + mp_compile_flags |= RSPAMD_MULTIPATTERN_COMPILE_NO_FS; } rspamd_url_add_static_matchers(url_scanner); @@ -577,13 +579,13 @@ void rspamd_url_init(const gchar *tld_file) url_scanner->matchers_full->len); } - if (!rspamd_multipattern_compile(url_scanner->search_trie_strict, &err)) { + if (!rspamd_multipattern_compile(url_scanner->search_trie_strict, mp_compile_flags, &err)) { msg_err("cannot compile url matcher static patterns, fatal error: %e", err); abort(); } if (url_scanner->search_trie_full) { - if (!rspamd_multipattern_compile(url_scanner->search_trie_full, &err)) { + if (!rspamd_multipattern_compile(url_scanner->search_trie_full, mp_compile_flags, &err)) { msg_err("cannot compile tld patterns, url matching will be " "incomplete: %e", err); diff --git a/src/libutil/multipattern.c b/src/libutil/multipattern.c index 630b1f921..bf3c7ad9a 100644 --- a/src/libutil/multipattern.c +++ b/src/libutil/multipattern.c @@ -1,11 +1,11 @@ -/*- - * Copyright 2016 Vsevolod Stakhov +/* + * Copyright 2024 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -466,7 +466,7 @@ rspamd_multipattern_try_save_hs(struct rspamd_multipattern *mp, #endif gboolean -rspamd_multipattern_compile(struct rspamd_multipattern *mp, GError **err) +rspamd_multipattern_compile(struct rspamd_multipattern *mp, int flags, GError **err) { g_assert(mp != NULL); g_assert(!mp->compiled); @@ -483,7 +483,7 @@ rspamd_multipattern_compile(struct rspamd_multipattern *mp, GError **err) rspamd_cryptobox_hash_update(&mp->hash_state, (void *) &plt, sizeof(plt)); rspamd_cryptobox_hash_final(&mp->hash_state, hash); - if (!rspamd_multipattern_try_load_hs(mp, hash)) { + if ((flags & RSPAMD_MULTIPATTERN_COMPILE_NO_FS) || !rspamd_multipattern_try_load_hs(mp, hash)) { hs_database_t *db = NULL; if (hs_compile_multi((const char *const *) mp->hs_pats->data, @@ -504,18 +504,23 @@ rspamd_multipattern_compile(struct rspamd_multipattern *mp, GError **err) return FALSE; } - if (hs_cache_dir != NULL) { - char fpath[PATH_MAX]; - rspamd_snprintf(fpath, sizeof(fpath), "%s/%*xs.hsmp", hs_cache_dir, - (gint) rspamd_cryptobox_HASHBYTES / 2, hash); - mp->hs_db = rspamd_hyperscan_from_raw_db(db, fpath); + if (!(flags & RSPAMD_MULTIPATTERN_COMPILE_NO_FS)) { + if (hs_cache_dir != NULL) { + char fpath[PATH_MAX]; + rspamd_snprintf(fpath, sizeof(fpath), "%s/%*xs.hsmp", hs_cache_dir, + (gint) rspamd_cryptobox_HASHBYTES / 2, hash); + mp->hs_db = rspamd_hyperscan_from_raw_db(db, fpath); + } + else { + /* Should not happen in the real life */ + mp->hs_db = rspamd_hyperscan_from_raw_db(db, NULL); + } + + rspamd_multipattern_try_save_hs(mp, hash); } else { - /* Should not happen in the real life */ mp->hs_db = rspamd_hyperscan_from_raw_db(db, NULL); } - - rspamd_multipattern_try_save_hs(mp, hash); } for (i = 0; i < MAX_SCRATCH; i++) { diff --git a/src/libutil/multipattern.h b/src/libutil/multipattern.h index 93027661d..15099aaca 100644 --- a/src/libutil/multipattern.h +++ b/src/libutil/multipattern.h @@ -1,11 +1,11 @@ -/*- - * Copyright 2016 Vsevolod Stakhov +/* + * Copyright 2024 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -117,12 +117,15 @@ void rspamd_multipattern_add_pattern(struct rspamd_multipattern *mp, void rspamd_multipattern_add_pattern_len(struct rspamd_multipattern *mp, const gchar *pattern, gsize patlen, gint flags); + +#define RSPAMD_MULTIPATTERN_COMPILE_NO_FS (0x1u << 0u) /** * Compiles multipattern structure * @param mp * @return */ gboolean rspamd_multipattern_compile(struct rspamd_multipattern *mp, + int flags, GError **err); /** diff --git a/src/lua/lua_common.c b/src/lua/lua_common.c index 92e26417a..cf4ca8023 100644 --- a/src/lua/lua_common.c +++ b/src/lua/lua_common.c @@ -1974,8 +1974,7 @@ rspamd_lua_check_udata_common(lua_State *L, gint pos, const gchar *classname, gboolean fatal) { void *p = lua_touserdata(L, pos); - guint i, top = lua_gettop(L); - khiter_t k; + gint i, top = lua_gettop(L); if (p == NULL) { goto err; diff --git a/src/lua/lua_trie.c b/src/lua/lua_trie.c index b92832928..70a685da2 100644 --- a/src/lua/lua_trie.c +++ b/src/lua/lua_trie.c @@ -1,11 +1,11 @@ -/*- - * Copyright 2016 Vsevolod Stakhov +/* + * Copyright 2024 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -142,7 +142,7 @@ lua_trie_create(lua_State *L) lua_pop(L, 1); /* table */ - if (!rspamd_multipattern_compile(trie, &err)) { + if (!rspamd_multipattern_compile(trie, 0, &err)) { msg_err("cannot compile multipattern: %e", err); g_error_free(err); rspamd_multipattern_destroy(trie); diff --git a/src/lua/lua_util.c b/src/lua/lua_util.c index d1d64b79c..612017d51 100644 --- a/src/lua/lua_util.c +++ b/src/lua/lua_util.c @@ -1,5 +1,5 @@ /* - * Copyright 2023 Vsevolod Stakhov + * Copyright 2024 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -2469,7 +2469,7 @@ lua_util_readline(lua_State *L) { LUA_TRACE_POINT; const gchar *prompt = ""; - gchar *input; + gchar *input = NULL; if (lua_type(L, 1) == LUA_TSTRING) { prompt = lua_tostring(L, 1); diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c index 62ddef504..15f556c40 100644 --- a/src/plugins/fuzzy_check.c +++ b/src/plugins/fuzzy_check.c @@ -3467,8 +3467,13 @@ fuzzy_process_handler(struct rspamd_http_connection_entry *conn_ent, if (!is_hash) { /* Allocate message from string */ /* XXX: what about encrypted messages ? */ - task->msg.begin = msg->body_buf.begin; - task->msg.len = msg->body_buf.len; + if (!rspamd_task_load_message(task, msg, msg->body_buf.begin, msg->body_buf.len)) { + msg_warn_task("cannot load message for fuzzy"); + rspamd_controller_send_error(conn_ent, 400, "Message load error"); + rspamd_task_free(task); + + return; + } r = rspamd_message_parse(task); diff --git a/src/plugins/lua/dkim_signing.lua b/src/plugins/lua/dkim_signing.lua index 6c05520ce..326ebab51 100644 --- a/src/plugins/lua/dkim_signing.lua +++ b/src/plugins/lua/dkim_signing.lua @@ -108,34 +108,31 @@ end local function dkim_signing_cb(task) local ret, selectors = dkim_sign_tools.prepare_dkim_signing(N, task, settings) - if not ret then + if not ret or #selectors == 0 then return end - if settings.use_redis then + -- Use only redis stuff here dkim_sign_tools.sign_using_redis(N, task, settings, selectors, do_sign, sign_error) - else - if selectors.vault then - dkim_sign_tools.sign_using_vault(N, task, settings, selectors, do_sign, sign_error) + return + end + + for _, k in ipairs(selectors) do + if k.vault then + dkim_sign_tools.sign_using_vault(N, task, settings, k, do_sign, sign_error) else - if #selectors > 0 then - for _, k in ipairs(selectors) do - -- templates - if k.key then - k.key = lua_util.template(k.key, { - domain = k.domain, - selector = k.selector - }) - lua_util.debugm(N, task, 'using key "%s", use selector "%s" for domain "%s"', - k.key, k.selector, k.domain) - end - - do_sign(task, k) - end - else - rspamd_logger.infox(task, 'key path or dkim selector unconfigured; no signing') - return false + -- templates + if k.key then + k.key = lua_util.template(k.key, { + domain = k.domain, + selector = k.selector + }) + lua_util.debugm(N, task, 'using key "%s", use selector "%s" for domain "%s"', + k.key, k.selector, k.domain) end + + do_sign(task, k) + end end end diff --git a/src/rspamadm/lua_repl.c b/src/rspamadm/lua_repl.c index d3f32c9db..a3d5541fd 100644 --- a/src/rspamadm/lua_repl.c +++ b/src/rspamadm/lua_repl.c @@ -604,7 +604,7 @@ lua_syntax_highlighter(const char *str, ReplxxColor *colours, int size, void *ud static void rspamadm_lua_run_repl(lua_State *L, bool is_batch) { - gchar *input; + gchar *input = NULL; #ifdef WITH_LUA_REPL gboolean is_multiline = FALSE; GString *tb = NULL; diff --git a/src/rspamd_proxy.c b/src/rspamd_proxy.c index 9139866b5..531a27be7 100644 --- a/src/rspamd_proxy.c +++ b/src/rspamd_proxy.c @@ -1713,7 +1713,15 @@ rspamd_proxy_scan_self_reply(struct rspamd_task *task) struct rspamd_http_message *msg; struct rspamd_proxy_session *session = task->fin_arg, *nsession; ucl_object_t *rep = NULL; + int out_type = UCL_EMIT_JSON_COMPACT; const char *ctype = "application/json"; + const rspamd_ftok_t *accept_hdr = rspamd_task_get_request_header(task, "Accept"); + + if (accept_hdr && rspamd_substring_search(accept_hdr->begin, accept_hdr->len, + "application/msgpack", sizeof("application/msgpack") - 1) != -1) { + ctype = "application/msgpack"; + out_type = UCL_EMIT_MSGPACK; + } msg = rspamd_http_new_message(HTTP_RESPONSE); msg->date = time(NULL); @@ -1726,7 +1734,7 @@ rspamd_proxy_scan_self_reply(struct rspamd_task *task) case CMD_CHECK_SPAMC: case CMD_CHECK_V2: rspamd_task_set_finish_time(task); - rspamd_protocol_http_reply(msg, task, &rep); + rspamd_protocol_http_reply(msg, task, &rep, out_type); rspamd_protocol_write_log_pipe(task); break; case CMD_PING: diff --git a/test/functional/cases/150_rspamadm.robot b/test/functional/cases/150_rspamadm.robot index ba9bef5bd..6bff14b2e 100644 --- a/test/functional/cases/150_rspamadm.robot +++ b/test/functional/cases/150_rspamadm.robot @@ -1,8 +1,8 @@ *** Settings *** Suite Setup Rspamadm Setup Suite Teardown Rspamadm Teardown -Library Process -Library ../lib/rspamd.py +Library ${RSPAMD_TESTDIR}/lib/rspamd.py +Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot *** Test Cases *** Config Test @@ -46,20 +46,3 @@ Verbose mode Should Match Regexp ${result.stderr} ^$ Should Match Regexp ${result.stdout} hello world\n Should Be Equal As Integers ${result.rc} 0 - -*** Keywords *** -Rspamadm Setup - ${RSPAMADM_TMPDIR} = Make Temporary Directory - Set Suite Variable ${RSPAMADM_TMPDIR} - -Rspamadm Teardown - Cleanup Temporary Directory ${RSPAMADM_TMPDIR} - -Rspamadm - [Arguments] @{args} - ${result} = Run Process ${RSPAMADM} - ... --var\=TMPDIR\=${RSPAMADM_TMPDIR} - ... --var\=DBDIR\=${RSPAMADM_TMPDIR} - ... --var\=LOCAL_CONFDIR\=/nonexistent - ... @{args} - [Return] ${result} diff --git a/test/functional/lib/rspamd.py b/test/functional/lib/rspamd.py index 57f179123..ea9c6204b 100644 --- a/test/functional/lib/rspamd.py +++ b/test/functional/lib/rspamd.py @@ -41,11 +41,11 @@ import tempfile from robot.api import logger from robot.libraries.BuiltIn import BuiltIn -import demjson +import json def Check_JSON(j): - d = demjson.decode(j, strict=True) + d = json.JSONDecoder(strict=True).decode(j.decode('utf-8')) logger.debug('got json %s' % d) assert len(d) > 0 assert 'error' not in d @@ -54,9 +54,9 @@ def Check_JSON(j): def check_json_log(fn): line_count = 0 - f = open(fn, 'r') + f = open(fn, 'r', encoding="utf-8") for l in f.readlines(): - d = demjson.decode(l, strict=True) + d = json.JSONDecoder(strict=True).decode(l) assert len(d) > 0 line_count = line_count + 1 assert line_count > 0 @@ -201,7 +201,7 @@ def Scan_File(filename, **headers): c.request("POST", "/checkv2", open(filename, "rb"), headers) r = c.getresponse() assert r.status == 200 - d = demjson.decode(r.read()) + d = json.JSONDecoder(strict=True).decode(r.read().decode('utf-8')) c.close() BuiltIn().set_test_variable("${SCAN_RESULT}", d) return diff --git a/test/functional/lib/rspamd.robot b/test/functional/lib/rspamd.robot index a5f897365..f9614c6fa 100644 --- a/test/functional/lib/rspamd.robot +++ b/test/functional/lib/rspamd.robot @@ -289,6 +289,7 @@ Run Rspamd ... --var\=DBDIR\=${RSPAMD_TMPDIR} ... --var\=LOCAL_CONFDIR\=/non-existent ... --var\=CONFDIR\=${RSPAMD_TESTDIR}/../../conf/ + ... --insecure ... env:RSPAMD_LOCAL_CONFDIR=/non-existent ... env:RSPAMD_TMPDIR=${RSPAMD_TMPDIR} ... env:RSPAMD_CONFDIR=${RSPAMD_TESTDIR}/../../conf/ @@ -302,6 +303,21 @@ Run Rspamd # Confirm worker is reachable Wait Until Keyword Succeeds 15x 1 sec Ping Rspamd ${RSPAMD_LOCAL_ADDR} ${check_port} +Rspamadm Setup + ${RSPAMADM_TMPDIR} = Make Temporary Directory + Set Suite Variable ${RSPAMADM_TMPDIR} + +Rspamadm Teardown + Cleanup Temporary Directory ${RSPAMADM_TMPDIR} + +Rspamadm + [Arguments] @{args} + ${result} = Run Process ${RSPAMADM} + ... --var\=TMPDIR\=${RSPAMADM_TMPDIR} + ... --var\=DBDIR\=${RSPAMADM_TMPDIR} + ... --var\=LOCAL_CONFDIR\=/nonexistent + ... @{args} + [Return] ${result} Run Nginx ${template} = Get File ${RSPAMD_TESTDIR}/configs/nginx.conf diff --git a/test/functional/lib/vars.py b/test/functional/lib/vars.py index a4bcbadda..bee2e92d4 100644 --- a/test/functional/lib/vars.py +++ b/test/functional/lib/vars.py @@ -11,54 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. + import shutil import socket |