diff options
| author | heraklit256 <37872459+heraklit256@users.noreply.github.com> | 2018-12-19 19:26:34 +0000 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2018-12-19 19:26:34 +0000 |
| commit | e36509986bcc059a99ca5c08d9a7d90c1f1217f5 (patch) | |
| tree | a715e2e95154a1f6aaf62bf7f73adb002c823fa7 | |
| parent | 5f4617948c64483dfb648b5bfe784f8c84dd87ea (diff) | |
| parent | 714faa2b804507f622d4780be646eaf8146a166d (diff) | |
| download | rspamd-e36509986bcc059a99ca5c08d9a7d90c1f1217f5.tar.gz rspamd-e36509986bcc059a99ca5c08d9a7d90c1f1217f5.zip | |
Merge pull request #8 from rspamd/master
merge upstream into local master
245 files changed, 16634 insertions, 8574 deletions
diff --git a/.circleci/config.yml b/.circleci/config.yml index 039954f26..1be12ac86 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,7 +34,7 @@ references: # Merge Lua coverage (collected into lua_coverage_report.json) and with C-coverage # (in coverage.rspamd-test.dump, coverage.functional.dump, see &capture_coverage_data) # and finally upload it into coveralls.io - test/functional/util/merge_coveralls.py --input coverage.functional.dump coverage.rspamd-test.dump lua_coverage_report.json unit_test_lua.json --output out.josn --token=${COVERALLS_REPO_TOKEN} + test/tools/merge_coveralls.py --input coverage.functional.dump coverage.rspamd-test.dump lua_coverage_report.json unit_test_lua.json --output out.josn --token=${COVERALLS_REPO_TOKEN} fi fi @@ -113,6 +113,10 @@ jobs: - run: cd ../build # see coverage notice in "build" stage - run: set +e; RSPAMD_INSTALLROOT=../install sudo -E bash -c "umask 0000; robot -x xunit.xml --exclude isbroken ../project/test/functional/cases"; echo "export RETURN_CODE=$?" >> $BASH_ENV + # luacov-coveralls reads luacov.stats.out generated by functional tests + # (see collect_lua_coverage() in test/functional/lib/rspamd.py) + # and writes json report for coveralls.io + - run: luacov-coveralls -o lua_coverage_report.json --dryrun - *capture_coverage_data @@ -153,6 +157,40 @@ jobs: key: v1-dependencies-{{ checksum "package.json" }} - run: ./node_modules/.bin/eslint -v && ./node_modules/.bin/eslint ./ + tidy: + docker: + - image: buildpack-deps:latest + steps: + - run: + name: Exporting env vars + command: | + cat >> $BASH_ENV <<EOF + export PATH=$PATH:$HOME/perl5/bin + export PERL_CPANM_OPT=--local-lib=$HOME/perl5 + export PERL5LIB=$HOME/perl5/lib/perl5:$PERL5LIB + EOF + - checkout + + - restore_cache: + key: v0-tidyall_dependencies + - run: + name: Installing cpanm + command: 'curl -L https://cpanmin.us | perl - App::cpanminus' + - run: + name: Installing CPAN dependencies + command: | + cpanm --quiet --notest \ + Code::TidyAll \ + Code::TidyAll::Plugin::Test::Vars \ + Perl::Critic \ + Perl::Tidy \ + Pod::Tidy + - save_cache: + key: v0-tidyall_dependencies + paths: + - ~/perl5 + - run: tidyall -a --check-only + send-coverage: <<: *defaults steps: @@ -178,6 +216,7 @@ workflows: jobs: - build - eslint + - tidy - rspamd-test: requires: - build diff --git a/.drone.yml b/.drone.yml index d3ee6bafb..2d01911ba 100644 --- a/.drone.yml +++ b/.drone.yml @@ -4,18 +4,56 @@ workspace: pipeline: + prepare: + # ubuntu used as base image for build and test images + # and we need to download it anyway + image: ubuntu:18.04 + commands: + - install -d -o nobody -g nogroup /rspamd/build /rspamd/install /rspamd/fedora/build /rspamd/fedora/install + # lua-torch CMakeLists writes to src dir + - chown nobody $CI_WORKSPACE/contrib/lua-torch/nn + # for debug + - echo $CI_COMMIT_AUTHOR + build: # https://github.com/rspamd/rspamd-build-docker/blob/master/ubuntu-build/Dockerfile image: rspamd/ci-ubuntu-build group: build commands: - - mkdir /rspamd/build /rspamd/install + # build directories should be writable by nobody, for rspamd in functional tests + # works as nobody and writes coverage files there + - test "$(id -un)" = nobody - cd /rspamd/build - cmake $CI_WORKSPACE -DENABLE_COVERAGE=ON -DENABLE_LIBUNWIND=ON -DCMAKE_INSTALL_PREFIX=/rspamd/install -DCMAKE_RULE_MESSAGES=OFF - ncpu=$(getconf _NPROCESSORS_ONLN) - make -j $ncpu install - make -j $ncpu rspamd-test + build-clang: + # https://github.com/rspamd/rspamd-build-docker/blob/master/fedora-build/Dockerfile + image: rspamd/ci-fedora-build + pull: true + group: build + commands: + - test "$(id -un)" = nobody + - cd /rspamd/fedora/build + - export LDFLAGS='-fuse-ld=lld' + - export CFLAGS='-fsanitize=address,undefined,implicit-integer-truncation' + - export ASAN_OPTIONS=detect_leaks=0 + - > + cmake + -DENABLE_CLANG_PLUGIN=ON + -DLLVM_CONFIG_BINARY=/usr/bin/llvm-config + -DENABLE_TORCH=OFF + -DCMAKE_C_COMPILER=/usr/bin/clang -DCMAKE_CXX_COMPILER=/usr/bin/clang++ + -DCMAKE_INSTALL_PREFIX=/rspamd/fedora/install + -DENABLE_FULL_DEBUG=ON + -DCMAKE_RULE_MESSAGES=OFF + $CI_WORKSPACE + - ncpu=$(getconf _NPROCESSORS_ONLN) + - make -j $ncpu install + - make -j $ncpu rspamd-test + eslint: image: node:10-alpine group: build @@ -24,53 +62,131 @@ pipeline: - ./node_modules/.bin/eslint -v - ./node_modules/.bin/eslint ./ + # Run checks on perl source using tidyall + perl-tidyall: + # https://github.com/rspamd/rspamd-build-docker/blob/master/perl-tidyall/Dockerfile + image: rspamd/ci-perl-tidyall + group: build + commands: + # checks are configured in .tidyallrc at the top of rspamd repo + - tidyall --all --check-only --no-cache --data-dir /tmp/tidyall + + # We run rspamd-test (unit test) and functional test (runned by robot) in + # parallel to save time. To avoid conflict in saving lua coverage we run them + # from different directories. For C code coverage counters is saved to .gcda + # files and binary contain absolute path to them, so rspamd-test and + # processes started by functional test are writing to the same files. On + # process exit new coverage data merged with existing content of .gcda file. + # Race is possible if rspamd-test and some rspamd process in functional test + # will try to write .gcda file simultaneous. But it is very unlikely and + # performance is more important then correct coverage data. + rspamd-test: # https://github.com/rspamd/rspamd-build-docker/blob/master/ubuntu-test/Dockerfile image: rspamd/ci-ubuntu-test + pull: true group: tests commands: + - test "$(id -un)" = nobody + - ulimit -c unlimited # rspamd-test and functional test both use luacov.stats.out file and should be started from # different directories (if started in parallel) - cd /rspamd/build/test - set +e - ./rspamd-test -p /rspamd/lua; EXIT_CODE=$? - set -e + # shell sets exit status of a process terminated by a signal to '128 + signal-number' + # if rspamd-test was terminated by a signal it should be SIGSEGV or SIGABRT, try to examine core + - > + if [ $EXIT_CODE -gt 128 ]; then + gdb --batch -ex 'thread apply all bt full' -c /var/tmp/*.rspamd-test.core ./rspamd-test; + exit $EXIT_CODE; + fi + # luacov-coveralls reads luacov.stats.out written by rspamd-test using luacov module + # and writes json report for coveralls.io service - luacov-coveralls -o /rspamd/build/unit_test_lua.json --dryrun - # coveralls dosn't support layout when object files are located outside git repo root - # add hack to disable searching for git - - ln -s /bin/true /usr/local/bin/git - - coveralls --dump /rspamd/build/coverage.rspamd-test.dump + - exit $EXIT_CODE + + test-fedora-clang: + # https://github.com/rspamd/rspamd-build-docker/blob/master/fedora-test/Dockerfile + image: rspamd/ci-fedora-test + pull: true + group: tests + commands: + - test "$(id -un)" = nobody + # Asan reserves 20Tb of virtual memory, limit core size to 2 Gb to avoid writing huge core + - ulimit -c 2097152 + # disable leak sanitizer: too many leaks detected, most of them probably FP + - export ASAN_OPTIONS="detect_leaks=0:print_stacktrace=1:disable_coredump=0" + - export UBSAN_OPTIONS="print_stacktrace=1:print_summary=0:log_path=/tmp/ubsan" + - cd /rspamd/fedora/build/test + - set +e + - ./rspamd-test -p /rspamd/lua; EXIT_CODE=$? + - set -e + # shell sets exit status of a process terminated by a signal to '128 + signal-number' + # if rspamd-test was terminated by a signal it should be SIGSEGV or SIGABRT, try to examine core + - > + if [ $EXIT_CODE -gt 128 ]; then + gdb --batch -ex 'bt' -c /var/tmp/*.rspamd-test.core ./rspamd-test; + fi + - cat /tmp/ubsan.* - exit $EXIT_CODE functional: # https://github.com/rspamd/rspamd-build-docker/blob/master/ubuntu-test-func/Dockerfile image: rspamd/ci-ubuntu-test-func + pull: true group: tests commands: - cd /rspamd/build + - ulimit -c unlimited + # some rspamd processes during this test work as root and some as nobody + # use umask to create world-writable files so nobody can write to *.gcda files created by root + - umask 0000 - set +e - - RSPAMD_INSTALLROOT=/rspamd/install robot --xunit xunit.xml --exclude isbroken $CI_WORKSPACE/test/functional/cases; EXIT_CODE=$? + - RSPAMD_INSTALLROOT=/rspamd/install robot --removekeywords wuks --exclude isbroken $CI_WORKSPACE/test/functional/cases; EXIT_CODE=$? - set -e - # coveralls will not find git repo anyway, see above - - ln -s /bin/true /usr/local/bin/git - - coveralls --dump coverage.functional.dump + # upload test results to nginx frontent using WebDAV PUT + - > + if [ -n "$HTTP_PUT_AUTH" ]; then + $CI_WORKSPACE/test/tools/http_put.py log.html report.html $CI_SYSTEM_LINK/testlogs/$CI_REPO_NAME/$CI_BUILD_NUMBER/; + fi + # core_pattern=/var/tmp/%u.%e.core so one or two cores can be saved for each binary + - core_files=$(find /var/tmp/ -name '*.core') + # use 'info proc mappings' to find path to executable file for given core + # first mapping is usually program executable + - > + for core in $core_files; + do + exe=$(gdb --batch -ex 'info proc mappings' -c $core | awk 'h {print $5; exit}; /objfile/ {h=1}'); + gdb --batch -ex 'bt' -c $core $exe; echo '---'; + done - exit $EXIT_CODE + secrets: [http_put_auth] send-coverage: image: rspamd/ci-ubuntu-test secrets: [ coveralls_repo_token ] commands: - cd /rspamd/build - - $CI_WORKSPACE/test/functional/util/merge_coveralls.py --input coverage.functional.dump coverage.rspamd-test.dump unit_test_lua.json lua_coverage_report.json --output out.josn --token=$COVERALLS_REPO_TOKEN + # extract coverage data for C code from .gcda files and save it in a format suitable for coveralls.io + - $CI_WORKSPACE/test/tools/gcov_coveralls.py --exclude test --prefix /rspamd/build --prefix $CI_WORKSPACE --out coverage.c.json + # luacov-coveralls reads luacov.stats.out generated by functional tests + # (see collect_lua_coverage() in test/functional/lib/rspamd.py) + # and writes json report for coveralls.io + - luacov-coveralls -o coverage.functional.lua.json --dryrun + # * merge coverage for C and Lua code + # * remove prefixes from absolute paths (in luacov-coveralls files), filter test, contrib, e. t.c + # * upload report to coveralls.io + - $CI_WORKSPACE/test/tools/merge_coveralls.py --root $CI_WORKSPACE --input coverage.c.json unit_test_lua.json coverage.functional.lua.json --token=$COVERALLS_REPO_TOKEN when: branch: master # don't send coverage report for pull request event: [push, tag] - send-test-log: + notify: image: drillster/drone-email from: noreply@rspamd.com - attachment: /rspamd/build/log.html secrets: [email_host, email_username, email_password] when: status: failure diff --git a/.eslintrc.json b/.eslintrc.json index 230365d86..b57c7129d 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -23,7 +23,7 @@ "singleLine": { "afterColon": false } }], "max-params": ["warn", 6], - "max-statements": ["warn", 31], + "max-statements": ["warn", 33], "max-statements-per-line": ["error", { "max": 2 }], "multiline-comment-style": "off", "multiline-ternary": ["error", "always-multiline"], diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..b8c677bfc --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +# Code::TidyAll +/.tidyall.d/ diff --git a/.luacheckrc b/.luacheckrc index 39b820b70..7aa54dcf6 100644 --- a/.luacheckrc +++ b/.luacheckrc @@ -39,6 +39,7 @@ ignore = { '212', -- unused argument '612', -- trailing whitespace '631', -- line is too long + '311', -- value assigned to variable X is unused } files['/**/src/plugins/lua/spamassassin.lua'].globals = { diff --git a/.tidyallrc b/.tidyallrc new file mode 100644 index 000000000..b90955938 --- /dev/null +++ b/.tidyallrc @@ -0,0 +1,25 @@ +; Run "tidyall -a" to process all files. +; Run "tidyall -g" to process all added or modified files in the current git working directory. + +; Ignore third-party code +ignore = contrib/**/* doc/doxydown/doxydown.pl + +;[PerlCritic] +;select = **/*.{pl,pl.in,pm,t} + +[PerlTidy] +select = **/*.{pl,pl.in,pm,t} +argv = -l=120 + +[PodChecker] +select = **/*.{pl,pl.in,pm,pod} + +;[PodSpell] +;select = **/*.{pl,pl.in,pm,pod} + +[PodTidy] +select = **/*.{pl,pl.in,pm,pod} +columns = 120 + +[Test::Vars] +select = **/*.{pl,pl.in,pm,t} diff --git a/AUTHORS.md b/AUTHORS.md index fdbce5c9e..ac0a68e7d 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -8,12 +8,12 @@ Rspamd was created by [Vsevolod Stakhov](https://github.com/vstakov) while worki ## Developers -* [Mikhail Galanin](https://github.com/negram) * [Alexander Moisseev](https://github.com/moisseev) ## Alumni Developers * [Andrew Lewis](https://github.com/fatalbanana) +* [Mikhail Galanin](https://github.com/negram) ## Significant contributors @@ -27,4 +27,4 @@ Rspamd was created by [Vsevolod Stakhov](https://github.com/vstakov) while worki * [Mimecast](https://mimecast.com) * [Rambler Mail](https://mail.rambler.ru/) -* [Locaweb](https://locaweb.com.br)
\ No newline at end of file +* [Locaweb](https://locaweb.com.br) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2d59bd0cc..9fc4fef74 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,7 @@ ENABLE_LANGUAGE(ASM) SET(RSPAMD_VERSION_MAJOR 1) SET(RSPAMD_VERSION_MINOR 8) -SET(RSPAMD_VERSION_PATCH 1) +SET(RSPAMD_VERSION_PATCH 4) # Keep two digits all the time SET(RSPAMD_VERSION_MAJOR_NUM ${RSPAMD_VERSION_MAJOR}0) @@ -506,10 +506,6 @@ INCLUDE_DIRECTORIES("${CMAKE_SOURCE_DIR}/" "${CMAKE_BINARY_DIR}/src" #Stored in the binary dir "${CMAKE_BINARY_DIR}/src/libcryptobox") -IF(CMAKE_INSTALL_PREFIX) - SET(PREFIX ${CMAKE_INSTALL_PREFIX}) -ENDIF(CMAKE_INSTALL_PREFIX) - LIST(APPEND CMAKE_REQUIRED_LIBRARIES m) SET(POE_LOOP "Loop::IO_Poll") @@ -564,7 +560,6 @@ IF(CMAKE_SYSTEM_NAME STREQUAL "Linux") LIST(APPEND CMAKE_REQUIRED_LIBRARIES rt) LIST(APPEND CMAKE_REQUIRED_LIBRARIES dl) LIST(APPEND CMAKE_REQUIRED_LIBRARIES resolv) - LIST(APPEND CMAKE_REQUIRED_LIBRARIES nsl) MESSAGE(STATUS "Configuring for Linux") IF(EXISTS "/etc/debian_version") SET(LINUX_START_SCRIPT "rspamd_debian.in") @@ -795,6 +790,7 @@ ENDIF() IF(SUPPORT_STD11_FLAG) SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -std=c11") ELSE(SUPPORT_STD11_FLAG) + MESSAGE("C11 compatible compiler is strongly recommended") IF(SUPPORT_STD99_FLAG) SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -std=c99") ENDIF(SUPPORT_STD99_FLAG) @@ -1,3 +1,134 @@ +1.8.3: 03 Dec 2018 + * [CritFix] Make flags mutually exclusive for mime parts + * [CritFix] Strictly deny unencoded bad utf8 sequences in headers + * [Feature] Add Kaspersky antivirus support + * [Feature] Add method to get dkim results + * [Feature] Add more words regexp classes + * [Feature] Allow to choose words format in `rspamadm mime` + * [Feature] Allow to get all types of words from Lua + * [Feature] Allow to get task flags in C expressions + * [Feature] Allow to require encryption when accepting connections + * [Feature] Ignore bogus whitespaces in the words + * [Feature] Implement more strict configuration tests + * [Feature] Improve SPF results in Authentication-Results + * [Feature] Support ClickHouse database + * [Fix] Add failsafety for utf8 regexps + * [Fix] Do not trigger BROKEN_CONTENT_TYPE on innocent text parts + * [Fix] Emit error if connection has been terminated with no stop pattern + * [Fix] Fix boundaries checks in embedded messages + * [Fix] Fix double free + * [Fix] Perform policy downgrade on sample out, add tests + * [Fix] Properly escape utf8 regexps in hyperscan mode + * [Fix] Selectors - attachments args condition + * [Fix] Some fixes for raw parts + * [Fix] Treat learning errors as non-fatal + * [Fix] Use tld when looking for DKIM domains + * [Project] Words unicode structure rework + * [Project] Add preliminary Redis Sentinel support + * [Project] Improve Authentication-Results header + * [Project] Rework DKIM checks results + * [Project] Use more generalised API to produce meta words + +1.8.2: 19 Nov 2018 + * [Conf] Add DWL support in the default configuration + * [Conf] Disable rspamd_update by default (again) + * [Conf] Fix configuration sample for ratelimit + * [CritFix] Disable broken url tags by default + * [CritFix] Fix \0 processing when doing RSA sign + * [CritFix] Fix adding symbols to their primary groups + * [Feature] Add `rspamadm cookie` utility + * [Feature] Add specialised functions for generating encrypted cookies + * [Feature] Add support of cookies in replies module + * [Feature] Add support of words regexps + * [Feature] Allow to add 3rd party clang plugins + * [Feature] Allow to create lua regexps from glob or plain patterns + * [Feature] Allow to set custom limits for upstream lists + * [Feature] Detect orphaned parts and attach them to message + * [Feature] Filter tokens in bayes + * [Feature] Fold b= value when doing arc sealing + * [Feature] Ignore cookies in the future and too old in the past + * [Feature] Skip stop words in statistics + * [Feature] Store stop words and allow to query them + * [Feature] Support query arguments in controller's custom commands + * [Feature] Tune upstream limits in Rspamd proxy + * [Feature] Use different callback symbols for different uribls + * [Feature] Write DKIM selector in dkim allow/reject symbols + * [Fix] Add obs_fws state support to eoh state machine + * [Fix] Add sanity check when applying mime boundaries heuristic + * [Fix] Antivirus - virus names with 0 were recognized as tables + * [Fix] Disable headernames in bayes temporarily + * [Fix] Do not allow syntax errors in include files... + * [Fix] Do not allow to merge an object with an array (or vice versa) + * [Fix] Don't perform forged recipients check for missing recipients + * [Fix] Fix DKIM based RBLs + * [Fix] Fix actrie implementation (sync from upstream), fixed OOB read + * [Fix] Fix explicit methods call in selectors + * [Fix] Fix extraction of additional parts + * [Fix] Fix finalization for internal plugins + * [Fix] Fix override_defaults function + * [Fix] Fix squeezed symbols when using settings + * [Fix] Fix urls insertion in Clickhouse module + * [Fix] Furhter fixes to ratelimits logic + * [Fix] Ignore signatures when looking for boundaries + * [Fix] Properly set learned count + * [Fix] Really fix ratelimits configuration and work + * [Fix] Remove ambigious format flag from printf + * [Fix] Restore URLs exporting in ClickHouse plugin + * [Fix] Rework bayes calculations... + * [Fix] Switch from chi-square to naive for large Fisher value + * [Fix] Treat normal password as enable password if there is no enable password + * [Fix] Use proper syntax for making DNS requests + * [Fix] Various fixes in embedded plugins + * [Project] Change fuzzy check selection logic to lua_fuzzy library + * [Project] Rework async events and symbols + * [Project] Move all metatokens in Bayes to lua_stat from C + * [WebUI] Add history rows per page control + +1.8.1: 16 Oct 2018 + * [CritFix] Fix options insertion + * [CritFix] Fix words decay one more time (affects long messages) + * [CritFix] Increase default words_decay + * [CritFix] Plug memory leak in redis pool + * [Feature] Add `check_violation` feature to DKIM/ARC signing + * [Feature] Add only unique elements to Clickhouse url arrays + * [Feature] Allow `g+:` and `g-:` composite atoms + * [Feature] Allow dkim domains check in surbl + * [Feature] Allow maps with HTTP auth + * [Feature] Allow to disable actions by users settings + * [Feature] Extend whitelisting options + * [Feature] Store url object in images + * [Feature] Use verdict instead of the plain action in plugins + * [Fix] Allow to call fstring append with NULL string + * [Fix] DCC - luacheck + * [Fix] Do not load torch on each rspamadm invocation + * [Fix] Fix boundaries detection and rework stop words algorithm + * [Fix] Fix dependencies for DNS_SIGNED symbol + * [Fix] Fix errors when dealing with dynamic rates/bursts in Ratelimit + * [Fix] Fix groups mess + * [Fix] Fix groups mess + * [Fix] Fix parsing address with comments + * [Fix] Fix resolving in DMARC reports + * [Fix] Fix various issues with parsing of the received headers + * [Fix] Fix watchers issue in lua_tcp when doing no resolving + * [Fix] Plug memory leak in language detector (affects reloads) + * [Fix] Remove one letter stop words + * [Fix] Slashing: backport chunk logic from libucl + * [Fix] Stop libevent from using cached time in rspamadm + * [Fix] Try to fix watchers chaining + * [Fix] Various fixes in redis sync interface + * [Fix] ip_score - respect check_authed and check_local settings from config + * [Project] Rework passthrough actions + * [Project] Clustering module + * [Rework] Always create result for a task + * [Rework] Completely rewrite DMARC checks logic + * [Rework] Rework and fix whitelist plugin + * [WebUI] Add symbols sorting buttons + * [WebUI] Change symbols order without updating history + * [WebUI] Colorize symbols + * [WebUI] Do not display password form when secure_ip is set + * [WebUI] Fix symbol description tooltips display + * [WebUI] History: add sorting by symbol score value + 1.8.0: 24 Sep 2018 * [Feature] Add arguments schemas to processors and extractors * [Feature] Add functional selectors library diff --git a/centos/rspamd.spec b/centos/rspamd.spec index d2b493085..112569ca8 100644 --- a/centos/rspamd.spec +++ b/centos/rspamd.spec @@ -73,6 +73,7 @@ lua. -DRUNDIR=%{_localstatedir}/run/rspamd \ %if 0%{?el6} -DWANT_SYSTEMD_UNITS=OFF \ + -DDISABLE_PTHREAD_MUTEX=1 \ %else -DWANT_SYSTEMD_UNITS=ON \ -DSYSTEMDDIR=%{_unitdir} \ diff --git a/clang-plugin/CMakeLists.txt b/clang-plugin/CMakeLists.txt index 8b2def17e..62a1d68f5 100644 --- a/clang-plugin/CMakeLists.txt +++ b/clang-plugin/CMakeLists.txt @@ -7,12 +7,54 @@ IF (ENABLE_CLANG_PLUGIN MATCHES "ON") LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}") ENABLE_LANGUAGE(CXX) FIND_PACKAGE(LLVM REQUIRED) + MESSAGE(STATUS "Trying to find libclang for llvm version ${LLVM_PACKAGE_VERSION}") + SET(libclang_llvm_header_search_paths + # LLVM Debian/Ubuntu nightly packages: http://llvm.org/apt/ + "/usr/lib/llvm-${LLVM_PACKAGE_VERSION}/include/" + # LLVM MacPorts + "/opt/local/libexec/llvm-${LLVM_PACKAGE_VERSION}/include" + # LLVM Homebrew + "/usr/local/Cellar/llvm/${LLVM_PACKAGE_VERSION}/include" + # LLVM Homebrew/versions + "/usr/local/lib/llvm-${LLVM_PACKAGE_VERSION}/include" + # FreeBSD ports versions + "/usr/local/llvm${LLVM_PACKAGE_VERSION}/include" + "${LLVM_INCLUDE_DIRS}" + ) + + SET(libclang_llvm_lib_search_paths + # LLVM Debian/Ubuntu nightly packages: http://llvm.org/apt/ + "/usr/lib/llvm-${LLVM_PACKAGE_VERSION}/lib/" + # LLVM MacPorts + "/opt/local/libexec/llvm-${LLVM_PACKAGE_VERSION}/lib" + # LLVM Homebrew + "/usr/local/Cellar/llvm/${LLVM_PACKAGE_VERSION}/lib" + # LLVM Homebrew/versions + "/usr/local/lib/llvm-${LLVM_PACKAGE_VERSION}/lib" + # FreeBSD ports versions + "/usr/local/llvm${LLVM_PACKAGE_VERSION}/lib" + "${LLVM_LIBRARY_DIRS}" + ) + + find_path(LIBCLANG_INCLUDE_DIR clang-c/Index.h + PATHS ${libclang_llvm_header_search_paths} + PATH_SUFFIXES LLVM/include #Windows package from http://llvm.org/releases/ + DOC "The path to the directory that contains clang-c/Index.h") + find_library(LIBCLANG_LIBRARY NAMES libclang.imp libclang clang + PATHS ${libclang_llvm_lib_search_paths} + PATH_SUFFIXES LLVM/lib #Windows package from http://llvm.org/releases/ + DOC "The file that corresponds to the libclang library.") + + get_filename_component(LIBCLANG_LIBRARY_DIR ${LIBCLANG_LIBRARY} PATH) + + set(LIBCLANG_LIBRARIES ${LIBCLANG_LIBRARY}) + set(LIBCLANG_INCLUDE_DIRS ${LIBCLANG_INCLUDE_DIR}) SET(CLANGPLUGINSRC plugin.cc printf_check.cc) ADD_LIBRARY(rspamd-clang SHARED ${CLANGPLUGINSRC}) SET_TARGET_PROPERTIES(rspamd-clang PROPERTIES - COMPILE_FLAGS "${LLVM_CXX_FLAGS} ${LLVM_CPP_FLAGS} ${LLVM_C_FLAGS}" + COMPILE_FLAGS "${LLVM_CPP_FLAGS} -O2 -Wall -Wextra -Wno-unused-parameter -Werror" INCLUDE_DIRECTORIES ${LIBCLANG_INCLUDE_DIR} LINKER_LANGUAGE CXX) TARGET_LINK_LIBRARIES(rspamd-clang ${LIBCLANG_LIBRARIES}) diff --git a/clang-plugin/FindLLVM.cmake b/clang-plugin/FindLLVM.cmake index fca9984ea..8da444d52 100644 --- a/clang-plugin/FindLLVM.cmake +++ b/clang-plugin/FindLLVM.cmake @@ -1,127 +1,251 @@ -# Copyright (c) 2007-2015 University of Illinois at Urbana-Champaign. -# Copyright (c) 2015, Vsevolod Stakhov -# - Find LLVM -# This module can be used to find LLVM. -# It requires that the llvm-config executable be available on the system path. -# Once found, llvm-config is used for everything else. +#===------------------------------------------------------------------------===# # -# Typical usage could be: -# find_package(LLVM QUIET REQUIRED COMPONENTS jit native interpreter) +# The KLEE Symbolic Virtual Machine # -# If the QUIET flag is not set, the specified components and LLVM version are -# outputted. +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. # -# If the COMPONENTS are not set, the default set of "all" is used. +#===------------------------------------------------------------------------===# # -# The following variables are set: +# This file provides multiple methods to detect LLVM. # -# LLVM_FOUND - Set to YES if LLVM is found. -# LLVM_VERSION - Set to the decimal version of the LLVM library. -# LLVM_C_FLAGS - All flags that should be passed to a C compiler. -# LLVM_CXX_FLAGS - All flags that should be passed to a C++ compiler. -# LLVM_CPP_FLAGS - All flags that should be passed to the C pre-processor. -# LLVM_LD_FLAGS - Additional flags to pass to the linker. -# LLVM_LIBRARY_DIRS - A list of directories where the LLVM libraries are located. -# LLVM_INCLUDE_DIRS - A list of directories where the LLVM headers are located. -# LLVM_LIBRARIES - A list of libraries which should be linked against. - -# A macro to run llvm config -macro(_llvm_config _var_name) - # Firstly, locate the LLVM config executable - find_program(_llvm_config_exe - NAMES llvm-config - DOC "llvm-config executable location" - ) - - # If no llvm-config executable was found, set the output variable to not - # found. - if (NOT _llvm_config_exe) - set(${_var_name} "${_var_name}-NOTFOUND") - else (NOT _llvm_config_exe) - # Otherwise, run llvm-config - execute_process( - COMMAND ${_llvm_config_exe} ${ARGN} - OUTPUT_VARIABLE ${_var_name} - RESULT_VARIABLE _llvm_config_retval +# * llvm-config executable. This method is portable across LLVM build systems +# (i.e. works if LLVM was built with autoconf/Makefile or with CMake). +# +# * find_package(LLVM CONFIG). This method only works if LLVM was built with +# CMake or with LLVM >= 3.5 when built with the autoconf/Makefile build system +# This method relies on the `LLVMConfig.cmake` file generated to be generated +# by LLVM's build system. +# +#===------------------------------------------------------------------------===# + +function(string_to_list s output_var) + string(REPLACE " " ";" _output "${s}") + set(${output_var} ${_output} PARENT_SCOPE) +endfunction() + +option(USE_CMAKE_FIND_PACKAGE_LLVM "Use find_package(LLVM CONFIG) to find LLVM" OFF) + +if (USE_CMAKE_FIND_PACKAGE_LLVM) + find_package(LLVM CONFIG REQUIRED) + + # Provide function to map LLVM components to libraries. + function(klee_get_llvm_libs output_var) + if (${LLVM_PACKAGE_VERSION} VERSION_LESS "3.5") + llvm_map_components_to_libraries(${output_var} ${ARGN}) + else() + llvm_map_components_to_libnames(${output_var} ${ARGN}) + endif() + set(${output_var} ${${output_var}} PARENT_SCOPE) + endfunction() + # HACK: This information is not exported so just pretend its OFF for now. + set(LLVM_ENABLE_VISIBILITY_INLINES_HIDDEN OFF) +else() + # Use the llvm-config binary to get the information needed. + # Try to detect it in the user's environment. The user can + # force a particular binary by passing `-DLLVM_CONFIG_BINARY=/path/to/llvm-config` + # to CMake. + find_program(LLVM_CONFIG_BINARY + NAMES llvm-config) + message(STATUS "LLVM_CONFIG_BINARY: ${LLVM_CONFIG_BINARY}") + + if (NOT LLVM_CONFIG_BINARY) + message(FATAL_ERROR + "Failed to find llvm-config.\n" + "Try passing -DLLVM_CONFIG_BINARY=/path/to/llvm-config to cmake") + endif() + + function(_run_llvm_config output_var) + set(_command "${LLVM_CONFIG_BINARY}" ${ARGN}) + execute_process(COMMAND ${_command} + RESULT_VARIABLE _exit_code + OUTPUT_VARIABLE ${output_var} OUTPUT_STRIP_TRAILING_WHITESPACE - ) - if (RESULT_VARIABLE) - message(SEND_ERROR - "Error running llvm-config with arguments: ${ARGN}") - endif (RESULT_VARIABLE) - endif (NOT _llvm_config_exe) -endmacro(_llvm_config) - -# The default set of components -set(_llvm_components all) - -# If components have been specified via find_package, use them -if (LLVM_FIND_COMPONENTS) - set(_llvm_components ${LLVM_FIND_COMPONENTS}) -endif (LLVM_FIND_COMPONENTS) - -if (NOT LLVM_FIND_QUIETLY) - message(STATUS "Looking for LLVM components: ${_llvm_components}") -endif (NOT LLVM_FIND_QUIETLY) - -_llvm_config(LLVM_VERSION --version) -_llvm_config(LLVM_C_FLAGS --cflags) -_llvm_config(LLVM_CXX_FLAGS --cxxflags) -_llvm_config(LLVM_CPP_FLAGS --cppflags) -_llvm_config(LLVM_LD_FLAGS --ldflags) -_llvm_config(LLVM_LIBRARY_DIRS --libdir) -_llvm_config(LLVM_INCLUDE_DIRS --includedir) -_llvm_config(LLVM_LIBRARIES --libs ${_llvm_components}) - -if (NOT LLVM_FIND_QUIETLY) - message(STATUS "Found LLVM version: ${LLVM_VERSION}") -endif (NOT LLVM_FIND_QUIETLY) - -SET(libclang_llvm_header_search_paths - # LLVM Debian/Ubuntu nightly packages: http://llvm.org/apt/ - "/usr/lib/llvm-${LLVM_VERSION}/include/" - # LLVM MacPorts - "/opt/local/libexec/llvm-${LLVM_VERSION}/include" - # LLVM Homebrew - "/usr/local/Cellar/llvm/${LLVM_VERSION}/include" - # LLVM Homebrew/versions - "/usr/local/lib/llvm-${LLVM_VERSION}/include" - # FreeBSD ports versions - "/usr/local/llvm${LLVM_VERSION}/include" -) - -SET(libclang_llvm_lib_search_paths - # LLVM Debian/Ubuntu nightly packages: http://llvm.org/apt/ - "/usr/lib/llvm-${LLVM_VERSION}/lib/" - # LLVM MacPorts - "/opt/local/libexec/llvm-${LLVM_VERSION}/lib" - # LLVM Homebrew - "/usr/local/Cellar/llvm/${LLVM_VERSION}/lib" - # LLVM Homebrew/versions - "/usr/local/lib/llvm-${LLVM_VERSION}/lib" - # FreeBSD ports versions - "/usr/local/llvm${LLVM_VERSION}/lib" -) - -find_path(LIBCLANG_INCLUDE_DIR clang-c/Index.h - PATHS ${libclang_llvm_header_search_paths} - PATH_SUFFIXES LLVM/include #Windows package from http://llvm.org/releases/ - DOC "The path to the directory that contains clang-c/Index.h") -find_library(LIBCLANG_LIBRARY NAMES libclang.imp libclang clang - PATHS ${libclang_llvm_lib_search_paths} - PATH_SUFFIXES LLVM/lib #Windows package from http://llvm.org/releases/ - DOC "The file that corresponds to the libclang library.") - -get_filename_component(LIBCLANG_LIBRARY_DIR ${LIBCLANG_LIBRARY} PATH) - -set(LIBCLANG_LIBRARIES ${LIBCLANG_LIBRARY}) -set(LIBCLANG_INCLUDE_DIRS ${LIBCLANG_INCLUDE_DIR}) - -# handle the QUIETLY and REQUIRED arguments and set LLVM_FOUND to TRUE if -# all listed variables are TRUE -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(LLVM - DEFAULT_MSG - LLVM_LIBRARIES - LLVM_INCLUDE_DIRS - LLVM_LIBRARY_DIRS) + ERROR_STRIP_TRAILING_WHITESPACE + ) + if (NOT ("${_exit_code}" EQUAL "0")) + message(FATAL_ERROR "Failed running ${_command}") + endif() + set(${output_var} ${${output_var}} PARENT_SCOPE) + endfunction() + + # Get LLVM version + _run_llvm_config(LLVM_PACKAGE_VERSION "--version") + # Try x.y.z patern + set(_llvm_version_regex "^([0-9]+)\\.([0-9]+)\\.([0-9]+)(svn)?$") + if ("${LLVM_PACKAGE_VERSION}" MATCHES "${_llvm_version_regex}") + string(REGEX REPLACE + "${_llvm_version_regex}" + "\\1" + LLVM_VERSION_MAJOR + "${LLVM_PACKAGE_VERSION}") + string(REGEX REPLACE + "${_llvm_version_regex}" + "\\2" + LLVM_VERSION_MINOR + "${LLVM_PACKAGE_VERSION}") + string(REGEX REPLACE + "${_llvm_version_regex}" + "\\3" + LLVM_VERSION_PATCH + "${LLVM_PACKAGE_VERSION}") + else() + # try x.y pattern + set(_llvm_version_regex "^([0-9]+)\\.([0-9]+)(svn)?$") + if ("${LLVM_PACKAGE_VERSION}" MATCHES "${_llvm_version_regex}") + string(REGEX REPLACE + "${_llvm_version_regex}" + "\\1" + LLVM_VERSION_MAJOR + "${LLVM_PACKAGE_VERSION}") + string(REGEX REPLACE + "${_llvm_version_regex}" + "\\2" + LLVM_VERSION_MINOR + "${LLVM_PACKAGE_VERSION}") + set(LLVM_VERSION_PATCH 0) + else() + message(FATAL_ERROR + "Failed to parse LLVM version from \"${LLVM_PACKAGE_VERSION}\"") + endif() + endif() + + set(LLVM_DEFINITIONS "") + _run_llvm_config(_llvm_cpp_flags "--cppflags") + string_to_list("${_llvm_cpp_flags}" _llvm_cpp_flags_list) + foreach (flag ${_llvm_cpp_flags_list}) + # Filter out -I flags by only looking for -D flags. + if ("${flag}" MATCHES "^-D" AND NOT ("${flag}" STREQUAL "-D_DEBUG")) + list(APPEND LLVM_DEFINITIONS "${flag}") + endif() + endforeach() + + set(LLVM_ENABLE_ASSERTIONS ON) + set(LLVM_ENABLE_EH ON) + set(LLVM_ENABLE_RTTI ON) + set(LLVM_ENABLE_VISIBILITY_INLINES_HIDDEN OFF) + _run_llvm_config(_llvm_cxx_flags "--cxxflags") + string_to_list("${_llvm_cxx_flags}" _llvm_cxx_flags_list) + foreach (flag ${_llvm_cxx_flags_list}) + if ("${flag}" STREQUAL "-DNDEBUG") + # Note we don't rely on `llvm-config --build-mode` because + # that seems broken when LLVM is built with CMake. + set(LLVM_ENABLE_ASSERTIONS OFF) + elseif ("${flag}" STREQUAL "-fno-exceptions") + set(LLVM_ENABLE_EH OFF) + elseif ("${flag}" STREQUAL "-fno-rtti") + set(LLVM_ENABLE_RTTI OFF) + elseif ("${flag}" STREQUAL "-fvisibility-inlines-hidden") + set(LLVM_ENABLE_VISIBILITY_INLINES_HIDDEN ON) + endif() + endforeach() + + set(LLVM_INCLUDE_DIRS "") + foreach (flag ${_llvm_cpp_flags_list}) + # Filter out -D flags by only looking for -I flags. + if ("${flag}" MATCHES "^-I") + string(REGEX REPLACE "^-I(.+)$" "\\1" _include_dir "${flag}") + list(APPEND LLVM_INCLUDE_DIRS "${_include_dir}") + endif() + endforeach() + + _run_llvm_config(LLVM_LIBRARY_DIRS "--libdir") + _run_llvm_config(LLVM_TOOLS_BINARY_DIR "--bindir") + _run_llvm_config(TARGET_TRIPLE "--host-target") + + # Provide function to map LLVM components to libraries. + function(klee_get_llvm_libs OUTPUT_VAR) + _run_llvm_config(_llvm_libs "--libfiles" ${ARGN}) + string_to_list("${_llvm_libs}" _llvm_libs_list) + + # Now find the system libs that are needed. + if (${LLVM_PACKAGE_VERSION} VERSION_LESS "3.5") + # For LLVM 3.4 and older system libraries + # appeared in the output of `--ldflags`. + _run_llvm_config(_system_libs "--ldflags") + # TODO: Filter out `-L<path>` flag. + else() + _run_llvm_config(_system_libs "--system-libs") + endif() + string_to_list("${_system_libs}" _system_libs_list) + + # Create an imported target for each LLVM library + # if it doesn't already exist. We need to do this + # so we can tell CMake that these libraries depend + # on the necessary libraries so that CMake + # can get the link order right. + set(targets_to_return "") + set(created_targets "") + foreach (llvm_lib ${_llvm_libs_list}) + # a bug in llvm-config from LLVM 3.9 + string(REGEX REPLACE "lib(libLLVM[-.a-zA-Z0-9]+\\.so)\\.so$" "\\1" llvm_lib "${llvm_lib}") + + get_filename_component(llvm_lib_file_name "${llvm_lib}" NAME) + + string(REGEX REPLACE "^(lib)?(LLVM[-.a-zA-Z0-9]+)\\..+$" "\\2" target_name "${llvm_lib_file_name}") + list(APPEND targets_to_return "${target_name}") + if (NOT TARGET "${target_name}") + # DEBUG: message(STATUS "Creating imported target \"${target_name}\"" " for \"${llvm_lib}\"") + list(APPEND created_targets "${target_name}") + + set(import_library_type "STATIC") + if ("${llvm_lib_file_name}" MATCHES "(so|dylib|dll)$") + set(import_library_type "SHARED") + endif() + # Create an imported target for the library + add_library("${target_name}" "${import_library_type}" IMPORTED GLOBAL) + set_property(TARGET "${target_name}" PROPERTY + IMPORTED_LOCATION "${llvm_lib}" + ) + endif() + endforeach() + + # Now state the dependencies of the created imported targets which we + # assume to be for each imported target the libraries which appear after + # the library in `{_llvm_libs_list}` and then finally the system libs. + # It is **essential** that we do this otherwise CMake will get the + # link order of the imported targets wrong. + list(LENGTH targets_to_return length_targets_to_return) + if ("${length_targets_to_return}" GREATER 0) + math(EXPR targets_to_return_last_index "${length_targets_to_return} -1") + foreach (llvm_target_lib ${created_targets}) + # DEBUG: message(STATUS "Adding deps for target ${llvm_target_lib}") + # Find position in `targets_to_return` + list(FIND targets_to_return "${llvm_target_lib}" position) + if ("${position}" EQUAL "-1") + message(FATAL_ERROR "couldn't find \"${llvm_target_lib}\" in list of targets") + endif() + if ("${position}" LESS "${targets_to_return_last_index}") + math(EXPR position_plus_one "${position} + 1") + foreach (index RANGE ${position_plus_one} ${targets_to_return_last_index}) + # Get the target for this index + list(GET targets_to_return ${index} target_for_index) + # DEBUG: message(STATUS "${llvm_target_libs} depends on ${target_for_index}") + set_property(TARGET "${llvm_target_lib}" APPEND PROPERTY + INTERFACE_LINK_LIBRARIES "${target_for_index}" + ) + endforeach() + endif() + # Now finally add the system library dependencies. These must be last. + set_property(TARGET "${target_name}" APPEND PROPERTY + INTERFACE_LINK_LIBRARIES "${_system_libs_list}" + ) + endforeach() + endif() + + set(${OUTPUT_VAR} ${targets_to_return} PARENT_SCOPE) + endfunction() +endif() + +# Filter out `-DNEBUG` from LLVM_DEFINITIONS. The caller can use +# `LLVM_ENABLE_ASSERTIONS` to decide how to set their defines. +set(_new_llvm_definitions "") +foreach (llvm_define ${LLVM_DEFINITIONS}) + if ("${llvm_define}" STREQUAL "-DNDEBUG") + # Skip + else() + list(APPEND _new_llvm_definitions "${llvm_define}") + endif() +endforeach() +set(LLVM_DEFINITIONS "${_new_llvm_definitions}") +unset(_new_llvm_definitions)
\ No newline at end of file diff --git a/clang-plugin/printf_check.cc b/clang-plugin/printf_check.cc index 9b4819d08..4bb611787 100644 --- a/clang-plugin/printf_check.cc +++ b/clang-plugin/printf_check.cc @@ -69,9 +69,6 @@ namespace rspamd { static bool int32_arg_handler (const Expr *arg, struct PrintfArgChecker *ctx); - static bool gboolean_arg_handler (const Expr *arg, - struct PrintfArgChecker *ctx); - static bool tok_arg_handler (const Expr *arg, struct PrintfArgChecker *ctx); @@ -187,9 +184,6 @@ namespace rspamd { case 'D': return llvm::make_unique<PrintfArgChecker> (int32_arg_handler, this->pcontext, this->ci); - case 'B': - return llvm::make_unique<PrintfArgChecker> (gboolean_arg_handler, - this->pcontext, this->ci); case 'T': return llvm::make_unique<PrintfArgChecker> (tok_arg_handler, this->pcontext, this->ci); @@ -369,7 +363,7 @@ namespace rspamd { format_specs = { 's', 'd', 'l', 'L', 'v', 'V', 'f', 'F', 'g', 'G', - 'T', 'z', 'D', 'c', 'p', 'P', 'e', 'B' + 'T', 'z', 'D', 'c', 'p', 'P', 'e' }; }; @@ -720,15 +714,6 @@ namespace rspamd { } static bool - gboolean_arg_handler (const Expr *arg, struct PrintfArgChecker *ctx) - { - return check_builtin_type (arg, - ctx, - {BuiltinType::Kind::Int}, // gboolean is int in fact - "%b"); - } - - static bool check_struct_type (const Expr *arg, struct PrintfArgChecker *ctx, const std::string &sname, const std::string &fmt) { diff --git a/conf/composites.conf b/conf/composites.conf index 431de669d..09ae5c156 100644 --- a/conf/composites.conf +++ b/conf/composites.conf @@ -24,6 +24,7 @@ composites { } FORGED_SENDER_FORWARDING { expression = "FORGED_SENDER & g:forwarding"; + description = "Forged sender, but message is forwarded"; policy = "remove_weight"; } SPF_FAIL_FORWARDING { @@ -42,17 +43,17 @@ composites { expression = "FORGED_SENDER & (ENVFROM_PRVS | ENVFROM_VERP)"; } FORGED_MUA_MAILLIST { - expression = "g:mua and -MAILLIST"; + expression = "g:mua & -MAILLIST"; } RBL_SPAMHAUS_XBL_ANY { expression = "RBL_SPAMHAUS_XBL & RECEIVED_SPAMHAUS_XBL"; description = "From and Received address are listed in Spamhaus XBL"; } AUTH_NA { - expression = "R_DKIM_NA & R_SPF_NA & DMARC_NA"; + expression = "R_DKIM_NA & R_SPF_NA & DMARC_NA & ARC_NA"; score = 1.0; policy = "remove_weight"; - description = "Authenticating message via SPF/DKIM/DMARC not possible"; + description = "Authenticating message via SPF/DKIM/DMARC/ARC not possible"; } DKIM_MIXED { expression = "-R_DKIM_ALLOW & (R_DKIM_DNSFAIL | R_DKIM_PERMFAIL | R_DKIM_REJECT)" @@ -66,6 +67,7 @@ composites { } MAILER_1C_8_BASE64 { expression = "MAILER_1C_8 & (FROM_EXCESS_BASE64 | MIME_BASE64_TEXT | SUBJ_EXCESS_BASE64 | TO_EXCESS_BASE64)"; + description = "Message was sent by '1C:Enterprise 8' and uses base64 encoded data"; } HACKED_WP_PHISHING { expression = "(HAS_X_POS | HAS_PHPMAILER_SIG) & HAS_WP_URI & (PHISHING | DBL_PHISH | PHISHED_OPENPHISH | PHISHED_PHISHTANK)"; @@ -106,20 +108,23 @@ composites { expression = "(HAS_X_POS | HAS_PHPMAILER_SIG | HAS_X_PHP_SCRIPT) & (SUBJECT_ENDS_QUESTION | SUBJECT_ENDS_EXCLAIM | MANY_INVISIBLE_PARTS)"; description = "Message was generated by PHP script and contains some spam indicators"; score = 1.0; + policy = "leave"; } PHISH_EMOTION { expression = "(PHISHING | DBL_PHISH | PHISHED_OPENPHISH | PHISHED_PHISHTANK) & (SUBJECT_ENDS_QUESTION | SUBJECT_ENDS_EXCLAIM)"; description = "Phish message with subject trying to address users emotion"; - score = 2.0; + score = 1.0; + policy = "leave"; } HAS_ANON_DOMAIN { - expression = "HAS_GUC_PROXY_URI | URIBL_RED | DBL_ABUSE_REDIR"; + expression = "HAS_GUC_PROXY_URI | URIBL_RED | DBL_ABUSE_REDIR | HAS_ONION_URI"; description = "Contains one or more domains trying to disguise owner/destination"; - score = 0.5; + score = 0.1; + policy = "leave"; } BAD_REP_POLICIES { - description = "Contains valid policies but are also marked by fuzzy/bayes"; - expression = "(~g-:policies) & (-g+:fuzzy | -g+:bayes)"; + description = "Contains valid policies but are also marked by fuzzy/bayes/surbl/rbl"; + expression = "(~g-:policies) & (-g+:fuzzy | -g+:bayes | -g+:surbl | -g+:rbl)"; score = 0.1; } diff --git a/conf/logging.inc b/conf/logging.inc index 6d0589418..2511d968f 100644 --- a/conf/logging.inc +++ b/conf/logging.inc @@ -13,7 +13,10 @@ EOD log_re_cache = true; # Can be used for console logging -color = false +color = false; + +# Log with microseconds resolution +log_usec = false; # Enable debug for specific modules (e.g. `debug_modules = ["dkim", "re_cache"];`) debug_modules = [] diff --git a/conf/mime_types.inc b/conf/mime_types.inc index e63dff09a..117770081 100644 --- a/conf/mime_types.inc +++ b/conf/mime_types.inc @@ -1352,6 +1352,7 @@ message/global-headers 0 message/http 0 message/imdn+xml 0 message/news 0 +message/rfc822 0 message/s-http 0 message/sip 0 message/sipfrag 0 diff --git a/conf/modules.d/clickhouse.conf b/conf/modules.d/clickhouse.conf index c35352a03..98ee47b82 100644 --- a/conf/modules.d/clickhouse.conf +++ b/conf/modules.d/clickhouse.conf @@ -31,14 +31,6 @@ clickhouse { # If a message has a domain in this map in From: header and DKIM signature, # record general metadata in a table named after the domain #from_tables = "/etc/rspamd/clickhouse_from.map"; - # These are tables used to store data in Clickhouse - # Table used to store ASN information (default unset: not collected) - #asn_table = "rspamd_asn"; # default unset - # The following table names are set by default - # Set these if you use want to use different table names - #table = "rspamd"; # general metadata - #attachments_table = "rspamd_attachments"; # attachment metadata - #urls_table = "rspamd_urls"; # url metadata # These are symbols of other checks in Rspamd # Set these if you use non-default symbol names (unlikely) #bayes_spam_symbols = ["BAYES_SPAM"]; diff --git a/conf/modules.d/ratelimit.conf b/conf/modules.d/ratelimit.conf index 521bdb1a8..c7f415583 100644 --- a/conf/modules.d/ratelimit.conf +++ b/conf/modules.d/ratelimit.conf @@ -15,20 +15,23 @@ ratelimit { #rates { - # Limit for all mail per recipient (rate 2 per minute) - #to = "2 / 1min"; - # Limit for all mail per one source ip (rate 1.5 per minute) - #to_ip = "3 / 2min"; - # Limit for all mail per one source ip and from address (rate 1 per minute) - #to_ip_from = "1 / 1min"; - # Limit for all bounce mail (rate 2 per hour) - #bounce_to = "2 / 1h"; - # Limit for bounce mail per one source ip (rate 1 per hour) - #bounce_to_ip = "1 / 1h"; - # Limit for all mail per authenticated user (rate 1 per minute) - #user = "1 / 1min"; + # Predefined ratelimit + #to = { + # bucket = { + # burst = 100; + # rate = 0.01666666666666666666; # leak 1 message per minute + # } + #} + # or define it with selector + #other_limit_alt = { + # selector = 'rcpts:addr.take_n(5)'; + # bucket = { + # burst = 100; + # rate = "1 / 1m"; # leak 1 message per minute + # } + #} #} - # If symbol is specified, then it is inserted instead of setting result + # If symbol is specified, then it is inserted *instead* of setting result to soft reject #symbol = "R_RATELIMIT"; # If info_symbol is specified, then it is inserted next to set the result diff --git a/conf/modules.d/rbl.conf b/conf/modules.d/rbl.conf index 27db68f71..57dc130e4 100644 --- a/conf/modules.d/rbl.conf +++ b/conf/modules.d/rbl.conf @@ -126,7 +126,7 @@ rbl { } } - nixspam { + nixspam { symbol = "RBL_NIXSPAM"; rbl = "ix.dnsbl.manitu.net"; ipv6 = true; @@ -146,6 +146,24 @@ rbl { from = false; ignore_whitelists = true; } + + dnswl_dwl { + symbol = "DWL_DNSWL"; + rbl = "dwl.dnswl.org"; + dkim = true; + dkim_domainonly = false; + dkim_match_from = true; + ignore_whitelist = true; + unknown = false; + + returncodes { + DWL_DNSWL_NONE = "127.0.%d+.0"; + DWL_DNSWL_LOW = "127.0.%d+.1"; + DWL_DNSWL_MED = "127.0.%d+.2"; + DWL_DNSWL_HI = "127.0.%d+.3"; + DWL_DNSWL_BLOCKED = "127.0.0.255"; + } + } } .include(try=true,priority=5) "${DBDIR}/dynamic/rbl.conf" diff --git a/conf/modules.d/rspamd_update.conf b/conf/modules.d/rspamd_update.conf index 0ed4a063c..8b0ccf8ba 100644 --- a/conf/modules.d/rspamd_update.conf +++ b/conf/modules.d/rspamd_update.conf @@ -16,6 +16,7 @@ rspamd_update { rules = "sign+https://updates.rspamd.com/rspamd-${BRANCH_VERSION}.ucl"; key = "qxuogdh5eghytji1utkkte1dn3n81c3y5twe61uzoddzwqzuxxyb"; + enabled = false; # Disable this module by default .include(try=true,priority=5) "${DBDIR}/dynamic/rspamd_update.conf" .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/rspamd_update.conf" diff --git a/conf/options.inc b/conf/options.inc index a75c6341d..f8abaa875 100644 --- a/conf/options.inc +++ b/conf/options.inc @@ -42,3 +42,9 @@ rrd = "${DBDIR}/rspamd.rrd"; # Local networks local_addrs = [192.168.0.0/16, 10.0.0.0/8, 172.16.0.0/12, fd00::/8, 169.254.0.0/16, fe80::/10]; hs_cache_dir = "${DBDIR}/"; + +# Timeout for messages processing (must be larger than any internal timeout used) +task_timeout = 8s; + +# Emit soft reject when timeout takes place +soft_reject_on_timeout = false; diff --git a/conf/scores.d/rbl_group.conf b/conf/scores.d/rbl_group.conf index bc9bb6b0e..9aea930d5 100644 --- a/conf/scores.d/rbl_group.conf +++ b/conf/scores.d/rbl_group.conf @@ -16,6 +16,7 @@ # See https://rspamd.com/doc/tutorials/writing_rules.html for details symbols = { + "DNSWL_BLOCKED" { weight = 0.0; description = "Resolver blocked due to excessive queries"; @@ -41,6 +42,31 @@ symbols = { description = "Sender listed at https://www.dnswl.org, high trust"; } + "DWL_DNSWL_BLOCKED" { + weight = 0.0; + description = "Resolver blocked due to excessive queries (dwl)"; + } + "DWL_DNSWL" { + weight = 0.0; + description = "Unrecognised result from https://www.dnswl.org (dwl)"; + } + "DWL_DNSWL_NONE" { + weight = 0.0; + description = "Message has a valid dkim signature originated from domain listed at https://www.dnswl.org, no trust"; + } + "DWL_DNSWL_LOW" { + weight = -1; + description = "Message has a valid dkim signature originated from domain listed at https://www.dnswl.org, low trust"; + } + "DWL_DNSWL_MED" { + weight = -2; + description = "Message has a valid dkim signature originated from domain listed at https://www.dnswl.org, medium trust"; + } + "DWL_DNSWL_HI" { + weight = -3.5; + description = "Message has a valid dkim signature originated from domain listed at https://www.dnswl.org, high trust"; + } + "RBL_SPAMHAUS" { weight = 0.0; description = "Unrecognised result from Spamhaus ZEN"; diff --git a/conf/statistic.conf b/conf/statistic.conf index 26e73c4d2..d9b9a1b72 100644 --- a/conf/statistic.conf +++ b/conf/statistic.conf @@ -43,22 +43,26 @@ classifier "bayes" { } learn_condition =<<EOD return function(task, is_spam, is_unlearn) - local prob = task:get_mempool():get_variable('bayes_prob', 'double') + local learn_type = task:get_request_header('Learn-Type') - if prob then - local in_class = false - local cl - if is_spam then - cl = 'spam' - in_class = prob >= 0.95 - else - cl = 'ham' - in_class = prob <= 0.05 - end + if not (learn_type and tostring(learn_type) == 'bulk') then + local prob = task:get_mempool():get_variable('bayes_prob', 'double') + + if prob then + local in_class = false + local cl + if is_spam then + cl = 'spam' + in_class = prob >= 0.95 + else + cl = 'ham' + in_class = prob <= 0.05 + end - if in_class then - return false,string.format('already in class %s; probability %.2f%%', - cl, math.abs((prob - 0.5) * 200.0)) + if in_class then + return false,string.format('already in class %s; probability %.2f%%', + cl, math.abs((prob - 0.5) * 200.0)) + end end end diff --git a/conf/worker-normal.inc b/conf/worker-normal.inc index 72aef0b84..532455d0a 100644 --- a/conf/worker-normal.inc +++ b/conf/worker-normal.inc @@ -1,4 +1,3 @@ # Included from top-level .conf file mime = true; -task_timeout = 8s; diff --git a/contrib/aho-corasick/_acism.h b/contrib/aho-corasick/_acism.h index 057bc31c3..3993594a0 100644 --- a/contrib/aho-corasick/_acism.h +++ b/contrib/aho-corasick/_acism.h @@ -2,7 +2,7 @@ ** Copyright (C) 2009-2014 Mischa Sandberg <mischasan@gmail.com> ** ** This program is free software; you can redistribute it and/or modify -** it under the terms of the GNU Lesser General Public License Version as +** it under the terms of the GNU Lesser General Public License Version 3 as ** published by the Free Software Foundation. You may not use, modify or ** distribute this program under any other version of the GNU Lesser General ** Public License. @@ -25,30 +25,50 @@ typedef int (*qsort_cmp)(const void *, const void *); +// "Width" specifier for different plats +#if __LONG_MAX__ == 9223372036854775807LL +# ifdef __APPLE__ +# define F64 "ll" +# else +# define F64 "l" +# endif +#elif __LONG_MAX__ == 2147483647L || defined(_LONG_LONG) || defined(__sun) // AIX 6.1 ... +# define F64 "ll" +#else +//XXX Assuming F64 is "ll" for VS +# define F64 "ll" +#endif + #ifndef ACISM_SIZE # define ACISM_SIZE 4 #endif #if ACISM_SIZE == 8 - typedef uint64_t TRAN, STATE, STRNO; +typedef uint64_t TRAN, STATE, STRNO; # define SYM_BITS 9U # define SYM_MASK 511 +# define FNO F64 #else - typedef uint32_t TRAN, STATE, STRNO; +typedef uint32_t TRAN, STATE, STRNO; # define SYM_BITS psp->sym_bits # define SYM_MASK psp->sym_mask +# define FNO #endif typedef uint16_t SYMBOL; typedef unsigned _SYMBOL; // An efficient stacklocal SYMBOL -#define IS_MATCH ((TRAN)1 << (8*sizeof(TRAN) - 1)) -#define IS_SUFFIX ((TRAN)1 << (8*sizeof(TRAN) - 2)) -#define T_FLAGS (IS_MATCH | IS_SUFFIX) +#define BACK ((SYMBOL)0) +#define ROOT ((STATE) 0) -typedef struct { STATE state; STRNO strno; } STRASH; +// MATCH and SUFFIX are the top 2 bits of a TRAN: +enum { + IS_MATCH = (TRAN)1 << (8*sizeof(TRAN) - 1), + IS_SUFFIX = (TRAN)1 << (8*sizeof(TRAN) - 2), + T_FLAGS = IS_MATCH | IS_SUFFIX +}; -typedef enum { BASE=2, USED=1 } USES; +typedef struct { STATE state; STRNO strno; } STRASH; struct acism { TRAN* tranv; @@ -70,34 +90,25 @@ struct acism { #include "acism.h" // p_size: size of tranv + hashv -static inline unsigned p_size(ac_trie_t const *psp) +static inline size_t p_size(ACISM const *psp) { return psp->hash_size * sizeof*psp->hashv - + psp->tran_size * sizeof*psp->tranv; } + + psp->tran_size * sizeof*psp->tranv; } -static inline unsigned p_hash(ac_trie_t const *psp, STATE s) - { return s * 107 % psp->hash_mod; } +static inline unsigned p_hash(ACISM const *psp, STATE s) +{ return s * 107 % psp->hash_mod; } -static inline void set_tranv(ac_trie_t *psp, void *mem) - { psp->hashv = (STRASH*)&(psp->tranv = mem)[psp->tran_size]; } +static inline void set_tranv(ACISM *psp, void *mem) +{ psp->hashv = (STRASH*)&(psp->tranv = (TRAN*)mem)[psp->tran_size]; } // TRAN accessors. For ACISM_SIZE=8, SYM_{BITS,MASK} do not use psp. -static inline TRAN p_tran(ac_trie_t const *psp, STATE s, _SYMBOL sym) - { return psp->tranv[s + sym] ^ sym; } - -static inline _SYMBOL t_sym(ac_trie_t const *psp, TRAN t) - { (void)psp; return t & SYM_MASK; } - -static inline STATE t_next(ac_trie_t const *psp, TRAN t) - { (void)psp; return (t & ~T_FLAGS) >> SYM_BITS; } - -static inline int t_isleaf(ac_trie_t const *psp, TRAN t) - { return t_next(psp, t) >= psp->tran_size; } - -static inline int t_strno(ac_trie_t const *psp, TRAN t) - { return t_next(psp, t) - psp->tran_size; } +static inline TRAN p_tran(ACISM const *psp, STATE s, _SYMBOL sym) +{ return psp->tranv[s + sym] ^ sym; } -static inline _SYMBOL t_valid(ac_trie_t const *psp, TRAN t) - { return !t_sym(psp, t); } +static inline _SYMBOL t_sym(ACISM const *psp, TRAN t) { (void)psp; return t & SYM_MASK; } +static inline STATE t_next(ACISM const *psp, TRAN t) { (void)psp; return (t & ~T_FLAGS) >> SYM_BITS; } +static inline int t_isleaf(ACISM const *psp, TRAN t) { return t_next(psp, t) >= psp->tran_size; } +static inline int t_strno(ACISM const *psp, TRAN t) { return t_next(psp, t) - psp->tran_size; } +static inline _SYMBOL t_valid(ACISM const *psp, TRAN t) { return !t_sym(psp, t); } -#endif//_ACISM_H +#endif//_ACISM_H
\ No newline at end of file diff --git a/contrib/aho-corasick/acism.h b/contrib/aho-corasick/acism.h index d942fd413..b5a3b1a5c 100644 --- a/contrib/aho-corasick/acism.h +++ b/contrib/aho-corasick/acism.h @@ -25,9 +25,11 @@ // "acism" uses MEMREF {ptr,len} bytevec structs for "string" args, // rather than NUL-terminated "C" strings. -typedef struct { char const *ptr; size_t len; } ac_trie_pat_t; +typedef struct ac_trie_pat_s { char const *ptr; size_t len; } ac_trie_pat_t; typedef struct acism ac_trie_t; +typedef struct acism ACISM; +typedef struct ac_trie_pat_s MEMREF; ac_trie_t* acism_create(ac_trie_pat_t const *strv, int nstrs); void acism_destroy(ac_trie_t*); diff --git a/contrib/aho-corasick/acism_create.c b/contrib/aho-corasick/acism_create.c index 15f9e801e..6b842cf3b 100644 --- a/contrib/aho-corasick/acism_create.c +++ b/contrib/aho-corasick/acism_create.c @@ -2,7 +2,7 @@ ** Copyright (C) 2009-2014 Mischa Sandberg <mischasan@gmail.com> ** ** This program is free software; you can redistribute it and/or modify -** it under the terms of the GNU Lesser General Public License Version as +** it under the terms of the GNU Lesser General Public License Version 3 as ** published by the Free Software Foundation. You may not use, modify or ** distribute this program under any other version of the GNU Lesser General ** Public License. @@ -18,11 +18,17 @@ */ #include "_acism.h" +typedef enum { BASE=2, USED=1 } USES; + typedef struct tnode { struct tnode *child, *next, *back; - union { unsigned nrefs; STATE state; } x; - STRNO match; - SYMBOL sym, is_suffix; + // nrefs was used in "prune_backlinks". + // It will be used again in "curtail". + unsigned nrefs; + STATE state; + STRNO match; + SYMBOL sym; + char is_suffix; // "bool" } TNODE; //--------------|--------------------------------------------- @@ -37,22 +43,23 @@ static inline int bitwid(unsigned u) if (u & 0x00000002) ret++; return ret; } -static void fill_symv(ac_trie_t*, ac_trie_pat_t const*, int ns); -static int create_tree(TNODE*, SYMBOL const*symv, ac_trie_pat_t const*strv, int nstrs); + +static void fill_symv(ACISM*, MEMREF const*, int ns); +static int create_tree(TNODE*, SYMBOL const*symv, MEMREF const*strv, int nstrs); static void add_backlinks(TNODE*, TNODE**, TNODE**); static int interleave(TNODE*, int nnodes, int nsyms, TNODE**, TNODE**); -static void fill_tranv(ac_trie_t*, TNODE const*); -static void fill_hashv(ac_trie_t*, TNODE const*, int nn); +static void fill_tranv(ACISM*, TNODE const*); +static void fill_hashv(ACISM*, TNODE const*, int nn); static TNODE* find_child(TNODE*, SYMBOL); // (ns) is either a STATE, or a (STRNO + tran_size) static inline void -set_tran(ac_trie_t *psp, STATE s, SYMBOL sym, int match, int suffix, TRAN ns) +set_tran(ACISM *psp, STATE s, SYMBOL sym, int match, int suffix, TRAN ns) { - psp->tranv[s + sym] = sym | (match ? IS_MATCH : 0) - | (suffix ? IS_SUFFIX : 0) - | (ns << SYM_BITS); + psp->tranv[s + sym] = sym | (match ? IS_MATCH : 0) + | (suffix ? IS_SUFFIX : 0) + | (ns << SYM_BITS); } // Track statistics for construction @@ -67,27 +74,26 @@ extern PSSTAT psstat[]; #endif //ACISM_STATS //--------------|--------------------------------------------- -ac_trie_t* -acism_create(ac_trie_pat_t const* strv, int nstrs) +ACISM* +acism_create(MEMREF const* strv, int nstrs) { TNODE **v1 = NULL, **v2 = NULL; - ac_trie_t *psp = g_malloc0(sizeof(*psp)); + ACISM *psp = g_malloc0(sizeof*psp); fill_symv(psp, strv, nstrs); - TNODE *troot = g_new0(TNODE, psp->nchars + 1); + TNODE *troot = g_malloc0((psp->nchars + 1) * sizeof(*troot)); int nnodes = create_tree(troot, psp->symv, strv, nstrs); NOTE(nnodes); // v1, v2: breadth-first work vectors for add_backlink and interleave. int i = (nstrs + 1) * sizeof(TNODE); - add_backlinks(troot, v1 = g_malloc(i), v2 = g_malloc(i)); + add_backlinks(troot, v1 = g_malloc0(i), v2 = g_malloc0(i)); + int nhash = 0; TNODE* tp = troot + nnodes; - - while (--tp > troot) { + while (--tp > troot) nhash += tp->match && tp->child; - } // Calculate each node's offset in tranv[]: psp->tran_size = interleave(troot, nnodes, psp->nsyms, v1, v2); @@ -102,7 +108,7 @@ acism_create(ac_trie_pat_t const* strv, int nstrs) psp->hash_size = psp->hash_mod + nhash; } - set_tranv(psp, g_malloc0(p_size(psp))); + set_tranv(psp, g_malloc0(p_size(psp) + sizeof(TRAN))); if (!psp->tranv) goto FAIL; fill_tranv(psp, troot); // The root state (0) must not look like a valid backref. @@ -119,14 +125,14 @@ acism_create(ac_trie_pat_t const* strv, int nstrs) set_tranv(psp, g_realloc(psp->tranv, p_size(psp))); } - // Diagnostics/statistics only: + // Diagnostics/statistics only: psp->nstrs = nstrs; for (i = psp->maxlen = 0; i < nstrs; ++i) if (psp->maxlen < strv[i].len) psp->maxlen = strv[i].len; goto DONE; -FAIL: acism_destroy(psp), psp = NULL; -DONE: g_free(troot), g_free(v1), g_free(v2); + FAIL: acism_destroy(psp), psp = NULL; + DONE: free(troot), free(v1), free(v2); return psp; } @@ -134,7 +140,7 @@ typedef struct { int freq, rank; } FRANK; static int frcmp(FRANK*a, FRANK*b) { return a->freq - b->freq; } static void -fill_symv(ac_trie_t *psp, ac_trie_pat_t const *strv, int nstrs) +fill_symv(ACISM *psp, MEMREF const *strv, int nstrs) { int i, j; FRANK frv[256]; @@ -149,14 +155,15 @@ fill_symv(ac_trie_t *psp, ac_trie_pat_t const *strv, int nstrs) for (i = 256; --i >= 0 && frv[i].freq;) psp->symv[frv[i].rank] = ++psp->nsyms; ++psp->nsyms; + #if ACISM_SIZE < 8 psp->sym_bits = bitwid(psp->nsyms); - psp->sym_mask = ~((~(uint32_t)0u) << psp->sym_bits); + psp->sym_mask = ~(-1 << psp->sym_bits); #endif } static int -create_tree(TNODE *Tree, SYMBOL const *symv, ac_trie_pat_t const *strv, int nstrs) +create_tree(TNODE *Tree, SYMBOL const *symv, MEMREF const *strv, int nstrs) { int i, j; TNODE *nextp = Tree + 1; @@ -209,30 +216,30 @@ add_backlinks(TNODE *troot, TNODE **v1, TNODE **v2) while (*v1) { TNODE **spp = v1, **dpp = v2, *srcp, *dstp; + while ((srcp = *spp++)) { for (dstp = srcp->child; dstp; dstp = dstp->next) { TNODE *bp = NULL; - if (dstp->child) *dpp++ = dstp; // Go through the parent (srcp) node's backlink chain, // looking for a useful backlink for the child (dstp). - // If the parent (srcp) has a backlink to (tp), and (tp) has a child (with children) - // matching the transition sym for (srcp -> dstp), - // then it is a useful backlink for the child (dstp). + // If the parent (srcp) has a backlink to (tp), + // and (tp) has a child matching the transition sym + // for (srcp -> dstp), then it is a useful backlink + // for the child (dstp). // Note that backlinks do not point at the suffix match; // they point at the PARENT of that match. for (tp = srcp->back; tp; tp = tp->back) - if ((bp = find_child(tp, dstp->sym)) && bp->child) + if ((bp = find_child(tp, dstp->sym))) break; - if (!bp) bp = troot; dstp->back = dstp->child ? bp : tp ? tp : troot; - dstp->back->x.nrefs++; + dstp->back->nrefs++; dstp->is_suffix = bp->match || bp->is_suffix; } } @@ -245,8 +252,8 @@ static int interleave(TNODE *troot, int nnodes, int nsyms, TNODE **v1, TNODE **v2) { unsigned usev_size = nnodes + nsyms; - char *usev = g_new0(char, usev_size); - STATE last_trans = 0, startv[nsyms][2]; + char *usev = g_malloc0(usev_size * sizeof(*usev)); + STATE last_trans = 0, startv[257][2]; TNODE *cp, **tmp; memset(startv, 0, nsyms * sizeof*startv); @@ -287,7 +294,7 @@ interleave(TNODE *troot, int nnodes, int nsyms, TNODE **v1, TNODE **v2) // No child needs an in-use slot? We're done. if (!cp) break; } - tp->x.state = pos; + tp->state = pos; // Mark node's base and children as used: usev[pos] |= need; @@ -302,15 +309,9 @@ interleave(TNODE *troot, int nnodes, int nsyms, TNODE **v1, TNODE **v2) if (last_trans < last) { last_trans = last; if (last + nsyms >= usev_size) { - char *tmp = g_realloc(usev, usev_size << 1); - if (tmp != NULL) { - usev = tmp; - memset(usev + usev_size, 0, usev_size); - usev_size <<= 1; - } else { - g_free(usev); - /* And handle error */ - } + usev = g_realloc(usev, usev_size << 1); + memset(usev + usev_size, 0, usev_size); + usev_size <<= 1; } } } @@ -318,19 +319,19 @@ interleave(TNODE *troot, int nnodes, int nsyms, TNODE **v1, TNODE **v2) *dstp = NULL; } - g_free(usev); + free(usev); return last_trans + 1; } static void -fill_hashv(ac_trie_t *psp, TNODE const treev[], int nnodes) +fill_hashv(ACISM *psp, TNODE const treev[], int nnodes) { - STRASH *sv = g_malloc(psp->hash_mod * sizeof*sv), *sp = sv; + STRASH *sv = g_malloc0(psp->hash_mod * sizeof*sv), *sp = sv; int i; // First pass: insert without resolving collisions. for (i = 0; i < nnodes; ++i) { - STATE base = treev[i].x.state; + STATE base = treev[i].state; TNODE const *tp; for (tp = treev[i].child; tp; tp = tp->next) { if (tp->match && tp->child) { @@ -348,21 +349,21 @@ fill_hashv(ac_trie_t *psp, TNODE const treev[], int nnodes) psp->hashv[i] = *sp; } - g_free(sv); + free(sv); } static void -fill_tranv(ac_trie_t *psp, TNODE const*tp) +fill_tranv(ACISM *psp, TNODE const*tp) { TNODE const *cp = tp->child; if (cp && tp->back) - set_tran(psp, tp->x.state, 0, 0, 0, tp->back->x.state); + set_tran(psp, tp->state, 0, 0, 0, tp->back->state); for (; cp; cp = cp->next) { //NOTE: cp->match is (strno+1) so that !cp->match means "no match". - set_tran(psp, tp->x.state, cp->sym, cp->match, cp->is_suffix, - cp->child ? cp->x.state : cp->match - 1 + psp->tran_size); + set_tran(psp, tp->state, cp->sym, cp->match, cp->is_suffix, + cp->child ? cp->state : cp->match - 1 + psp->tran_size); if (cp->child) fill_tranv(psp, cp); } @@ -378,4 +379,4 @@ find_child(TNODE *tp, SYMBOL sym) #ifdef ACISM_STATS PSSTAT psstat[__LINE__] = {{__LINE__,0}}; #endif//ACISM_STATS -//EOF +//EOF
\ No newline at end of file diff --git a/contrib/librdns/rdns.h b/contrib/librdns/rdns.h index 8aca1f9f6..016903ba6 100644 --- a/contrib/librdns/rdns.h +++ b/contrib/librdns/rdns.h @@ -105,7 +105,7 @@ union rdns_reply_element_un { struct rdns_reply_entry { union rdns_reply_element_un content; - uint16_t type; + enum rdns_request_type type; int32_t ttl; struct rdns_reply_entry *prev, *next; }; diff --git a/contrib/librdns/resolver.c b/contrib/librdns/resolver.c index b9b156c5e..b38e90514 100644 --- a/contrib/librdns/resolver.c +++ b/contrib/librdns/resolver.c @@ -984,7 +984,7 @@ void rdns_resolver_set_fake_reply (struct rdns_resolver *resolver, fake_rep->rcode = rcode; if (reply) { - DL_APPEND (fake_rep->result, reply); + DL_CONCAT (fake_rep->result, reply); } } else { @@ -999,7 +999,7 @@ void rdns_resolver_set_fake_reply (struct rdns_resolver *resolver, memcpy (&fake_rep->key, srch, sizeof (*srch) + len); if (reply) { - DL_APPEND (fake_rep->result, reply); + DL_CONCAT (fake_rep->result, reply); } HASH_ADD (hh, resolver->fake_elts, key, sizeof (*srch) + len, fake_rep); diff --git a/contrib/libucl/ucl_parser.c b/contrib/libucl/ucl_parser.c index c4b390407..574e240ab 100644 --- a/contrib/libucl/ucl_parser.c +++ b/contrib/libucl/ucl_parser.c @@ -89,6 +89,7 @@ ucl_set_err (struct ucl_parser *parser, int code, const char *str, UT_string **e } parser->err_code = code; + parser->state = UCL_STATE_ERROR; } static void @@ -633,14 +634,26 @@ ucl_parser_add_container (ucl_object_t *obj, struct ucl_parser *parser, bool is_array, uint32_t level, bool has_obrace) { struct ucl_stack *st; + bool need_free = false; if (!is_array) { if (obj == NULL) { obj = ucl_object_new_full (UCL_OBJECT, parser->chunks->priority); + need_free = true; } else { + if (obj->type == UCL_ARRAY) { + /* Bad combination for merge: array and object */ + ucl_set_err (parser, UCL_EMERGE, + "cannot merge an array with an object", + &parser->err); + + return NULL; + } + obj->type = UCL_OBJECT; } + if (obj->value.ov == NULL) { obj->value.ov = ucl_hash_create (parser->flags & UCL_PARSER_KEY_LOWERCASE); } @@ -649,8 +662,18 @@ ucl_parser_add_container (ucl_object_t *obj, struct ucl_parser *parser, else { if (obj == NULL) { obj = ucl_object_new_full (UCL_ARRAY, parser->chunks->priority); + need_free = true; } else { + if (obj->type == UCL_OBJECT) { + /* Bad combination for merge: array and object */ + ucl_set_err (parser, UCL_EMERGE, + "cannot merge an object with an array", + &parser->err); + + return NULL; + } + obj->type = UCL_ARRAY; } parser->state = UCL_STATE_VALUE; @@ -661,7 +684,10 @@ ucl_parser_add_container (ucl_object_t *obj, struct ucl_parser *parser, if (st == NULL) { ucl_set_err (parser, UCL_EINTERNAL, "cannot allocate memory for an object", &parser->err); - ucl_object_unref (obj); + if (need_free) { + ucl_object_unref (obj); + } + return NULL; } @@ -671,7 +697,10 @@ ucl_parser_add_container (ucl_object_t *obj, struct ucl_parser *parser, ucl_set_err (parser, UCL_ENESTED, "objects are nesting too deep (over 65535 limit)", &parser->err); - ucl_object_unref (obj); + if (need_free) { + ucl_object_unref (obj); + } + return NULL; } @@ -2404,6 +2433,7 @@ ucl_state_machine (struct ucl_parser *parser) parser->state = UCL_STATE_ERROR; return false; } + if (end_of_object) { p = chunk->pos; parser->state = UCL_STATE_AFTER_VALUE; @@ -2610,7 +2640,7 @@ ucl_state_machine (struct ucl_parser *parser) } } - if (parser->stack != NULL) { + if (parser->stack != NULL && parser->state != UCL_STATE_ERROR) { struct ucl_stack *st; bool has_error = false; diff --git a/contrib/libucl/ucl_util.c b/contrib/libucl/ucl_util.c index b7b471205..051ac2c27 100644 --- a/contrib/libucl/ucl_util.c +++ b/contrib/libucl/ucl_util.c @@ -1149,7 +1149,7 @@ ucl_include_file_single (const unsigned char *data, size_t len, } old_curfile = parser->cur_file; - parser->cur_file = strdup (realbuf); + parser->cur_file = NULL; /* Store old file vars */ DL_FOREACH_SAFE (parser->variables, cur_var, tmp_var) { @@ -1333,57 +1333,49 @@ ucl_include_file_single (const unsigned char *data, size_t len, res = ucl_parser_add_chunk_full (parser, buf, buflen, params->priority, params->strat, params->parse_type); - if (!res) { - if (!params->must_exist) { - /* Free error */ - utstring_free (parser->err); - parser->err = NULL; - res = true; + if (res) { + /* Stop nesting the include, take 1 level off the stack */ + if (params->prefix != NULL && nest_obj != NULL) { + parser->stack = st->next; + UCL_FREE (sizeof (struct ucl_stack), st); } - } - - /* Stop nesting the include, take 1 level off the stack */ - if (params->prefix != NULL && nest_obj != NULL) { - parser->stack = st->next; - UCL_FREE (sizeof (struct ucl_stack), st); - } - /* Remove chunk from the stack */ - chunk = parser->chunks; - if (chunk != NULL) { - parser->chunks = chunk->next; - ucl_chunk_free (chunk); - parser->recursion --; - } + /* Remove chunk from the stack */ + chunk = parser->chunks; + if (chunk != NULL) { + parser->chunks = chunk->next; + ucl_chunk_free (chunk); + parser->recursion--; + } - /* Restore old file vars */ - if (parser->cur_file) { - free (parser->cur_file); - } + /* Restore old file vars */ + if (parser->cur_file) { + free (parser->cur_file); + } - parser->cur_file = old_curfile; - DL_FOREACH_SAFE (parser->variables, cur_var, tmp_var) { - if (strcmp (cur_var->var, "CURDIR") == 0 && old_curdir) { - DL_DELETE (parser->variables, cur_var); - free (cur_var->var); - free (cur_var->value); - UCL_FREE (sizeof (struct ucl_variable), cur_var); + parser->cur_file = old_curfile; + DL_FOREACH_SAFE (parser->variables, cur_var, tmp_var) { + if (strcmp (cur_var->var, "CURDIR") == 0 && old_curdir) { + DL_DELETE (parser->variables, cur_var); + free (cur_var->var); + free (cur_var->value); + UCL_FREE (sizeof (struct ucl_variable), cur_var); + } else if (strcmp (cur_var->var, "FILENAME") == 0 && old_filename) { + DL_DELETE (parser->variables, cur_var); + free (cur_var->var); + free (cur_var->value); + UCL_FREE (sizeof (struct ucl_variable), cur_var); + } } - else if (strcmp (cur_var->var, "FILENAME") == 0 && old_filename) { - DL_DELETE (parser->variables, cur_var); - free (cur_var->var); - free (cur_var->value); - UCL_FREE (sizeof (struct ucl_variable), cur_var); + if (old_filename) { + DL_APPEND (parser->variables, old_filename); + } + if (old_curdir) { + DL_APPEND (parser->variables, old_curdir); } - } - if (old_filename) { - DL_APPEND (parser->variables, old_filename); - } - if (old_curdir) { - DL_APPEND (parser->variables, old_curdir); - } - parser->state = prev_state; + parser->state = prev_state; + } if (buflen > 0) { ucl_munmap (buf, buflen); @@ -1938,6 +1930,12 @@ ucl_parser_set_filevars (struct ucl_parser *parser, const char *filename, bool n ucl_strlcpy (realbuf, filename, sizeof (realbuf)); } + if (parser->cur_file) { + free (parser->cur_file); + } + + parser->cur_file = strdup (realbuf); + /* Define variables */ ucl_parser_register_variable (parser, "FILENAME", realbuf); curdir = dirname (realbuf); @@ -1974,10 +1972,6 @@ ucl_parser_add_file_full (struct ucl_parser *parser, const char *filename, return false; } - if (parser->cur_file) { - free (parser->cur_file); - } - parser->cur_file = strdup (realbuf); ucl_parser_set_filevars (parser, realbuf, false); ret = ucl_parser_add_chunk_full (parser, buf, len, priority, strat, parse_type); diff --git a/doc/Makefile b/doc/Makefile index 8e03fe71a..54f2fbdde 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -20,8 +20,8 @@ lua-dirs: lua-doc: lua-dirs rspamd_regexp rspamd_ip rspamd_config rspamd_task ucl rspamd_http rspamd_trie \ rspamd_resolver rspamd_redis rspamd_upstream_list rspamd_expression rspamd_mimepart rspamd_logger rspamd_url \ - rspamd_tcp rspamd_mempool rspamd_html rspamd_util rspamd_fann rspamd_sqlite3 rspamd_cryptobox \ - lua_redis lua_util lua_maps + rspamd_tcp rspamd_mempool rspamd_html rspamd_util rspamd_fann rspamd_sqlite3 rspamd_cryptobox rspamd_map \ + lua_redis lua_util lua_maps lua_clickhouse lua_selectors lua_redis: $(LLUADOC) < ../lualib/lua_redis.lua > markdown/lua/lua_redis.md @@ -32,6 +32,12 @@ lua_util: lua_maps: $(LLUADOC) < ../lualib/lua_maps.lua > markdown/lua/lua_maps.md +lua_clickhouse: + $(LLUADOC) < ../lualib/lua_clickhouse.lua > markdown/lua/lua_clickhouse.md + +lua_selectors: + $(LLUADOC) < ../lualib/lua_selectors.lua > markdown/lua/lua_selectors.md + rspamd_regexp: ../src/lua/lua_regexp.c $(LUADOC) < ../src/lua/lua_regexp.c > markdown/lua/rspamd_regexp.md rspamd_ip: ../src/lua/lua_ip.c @@ -74,3 +80,5 @@ rspamd_sqlite3: ../src/lua/lua_sqlite3.c $(LUADOC) < ../src/lua/lua_sqlite3.c > markdown/lua/rspamd_sqlite3.md rspamd_cryptobox: ../src/lua/lua_cryptobox.c $(LUADOC) < ../src/lua/lua_cryptobox.c > markdown/lua/rspamd_cryptobox.md +rspamd_map: ../src/lua/lua_map.c + $(LUADOC) < ../src/lua/lua_map.c > markdown/lua/rspamd_map.md
\ No newline at end of file diff --git a/doc/doxydown/doxydown.pl b/doc/doxydown/doxydown.pl index d58d4a906..993a8636e 100755 --- a/doc/doxydown/doxydown.pl +++ b/doc/doxydown/doxydown.pl @@ -68,9 +68,9 @@ EOD my $id = $f->{'id'}; if ($f->{'brief'}) { - print "> [`$name`](#$id): ". $f->{'brief'} . "\n\n"; + print "* [`$name`](#$id): ". $f->{'brief'} . "\n"; } else { - print "> [`$name`](#$id)\n\n"; + print "* [`$name`](#$id)\n"; } } @@ -339,15 +339,15 @@ sub parse_function { if ( /^\s*\@param\s*(?:\{([^}]+)\})?\s*(\S+)\s*(.+)?\s*$/ ) { my $p = { name => $2, - type => $1, - description => $3 + type => $1 || "no type", + description => $3 || "no description" }; push @{ $f->{'params'} }, $p; } elsif ( /^\s*\@return\s*(?:\{([^}]+)\})?\s*(.+)?\s*$/ ) { my $r = { type => $1, - description => $2 + description => $2 || "no description" }; push @{ $f->{'returns'} }, $r; @@ -379,6 +379,19 @@ sub parse_function { chomp $f->{'example'}; } + if ( !$f->{'brief'} && $f->{'data'} ) { + + + if ( $f->{'data'} =~ /^(.*?)(?:(?:[.:]\s|$)|\n).*/ ) { + $f->{'brief'} = "$1"; + chomp $f->{'brief'}; + + if ( $f->{'brief'} !~ /\.$/) { + $f->{'brief'} .= "."; + } + } + } + if ( $type eq "method" ) { push @{ $cur_module->{'methods'} }, $f; } elsif ( $type eq "function" || $type eq "fn") { diff --git a/interface/css/bootstrap.min.css b/interface/css/bootstrap.min.css index d65c66b1b..ed3905e0e 100644 --- a/interface/css/bootstrap.min.css +++ b/interface/css/bootstrap.min.css @@ -1,5 +1,6 @@ /*! - * Bootstrap v3.3.5 (http://getbootstrap.com) - * Copyright 2011-2015 Twitter, Inc. + * Bootstrap v3.3.7 (http://getbootstrap.com) + * Copyright 2011-2016 Twitter, Inc. * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) - *//*! normalize.css v3.0.3 | MIT License | github.com/necolas/normalize.css */html{font-family:sans-serif;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%}body{margin:0}article,aside,details,figcaption,figure,footer,header,hgroup,main,menu,nav,section,summary{display:block}audio,canvas,progress,video{display:inline-block;vertical-align:baseline}audio:not([controls]){display:none;height:0}[hidden],template{display:none}a{background-color:transparent}a:active,a:hover{outline:0}abbr[title]{border-bottom:1px dotted}b,strong{font-weight:700}dfn{font-style:italic}h1{margin:.67em 0;font-size:2em}mark{color:#000;background:#ff0}small{font-size:80%}sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline}sup{top:-.5em}sub{bottom:-.25em}img{border:0}svg:not(:root){overflow:hidden}figure{margin:1em 40px}hr{height:0;-webkit-box-sizing:content-box;-moz-box-sizing:content-box;box-sizing:content-box}pre{overflow:auto}code,kbd,pre,samp{font-family:monospace,monospace;font-size:1em}button,input,optgroup,select,textarea{margin:0;font:inherit;color:inherit}button{overflow:visible}button,select{text-transform:none}button,html input[type=button],input[type=reset],input[type=submit]{-webkit-appearance:button;cursor:pointer}button[disabled],html input[disabled]{cursor:default}button::-moz-focus-inner,input::-moz-focus-inner{padding:0;border:0}input{line-height:normal}input[type=checkbox],input[type=radio]{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;padding:0}input[type=number]::-webkit-inner-spin-button,input[type=number]::-webkit-outer-spin-button{height:auto}input[type=search]{-webkit-box-sizing:content-box;-moz-box-sizing:content-box;box-sizing:content-box;-webkit-appearance:textfield}input[type=search]::-webkit-search-cancel-button,input[type=search]::-webkit-search-decoration{-webkit-appearance:none}fieldset{padding:.35em .625em .75em;margin:0 2px;border:1px solid silver}legend{padding:0;border:0}textarea{overflow:auto}optgroup{font-weight:700}table{border-spacing:0;border-collapse:collapse}td,th{padding:0}/*! Source: https://github.com/h5bp/html5-boilerplate/blob/master/src/css/main.css */@media print{*,:after,:before{color:#000!important;text-shadow:none!important;background:0 0!important;-webkit-box-shadow:none!important;box-shadow:none!important}a,a:visited{text-decoration:underline}a[href]:after{content:" (" attr(href) ")"}abbr[title]:after{content:" (" attr(title) ")"}a[href^="javascript:"]:after,a[href^="#"]:after{content:""}blockquote,pre{border:1px solid #999;page-break-inside:avoid}thead{display:table-header-group}img,tr{page-break-inside:avoid}img{max-width:100%!important}h2,h3,p{orphans:3;widows:3}h2,h3{page-break-after:avoid}.navbar{display:none}.btn>.caret,.dropup>.btn>.caret{border-top-color:#000!important}.label{border:1px solid #000}.table{border-collapse:collapse!important}.table td,.table th{background-color:#fff!important}.table-bordered td,.table-bordered th{border:1px solid #ddd!important}}@font-face{font-family:'Glyphicons Halflings';src:url(../fonts/glyphicons-halflings-regular.eot);src:url(../fonts/glyphicons-halflings-regular.eot?#iefix) format('embedded-opentype'),url(../fonts/glyphicons-halflings-regular.woff2) format('woff2'),url(../fonts/glyphicons-halflings-regular.woff) format('woff'),url(../fonts/glyphicons-halflings-regular.ttf) format('truetype'),url(../fonts/glyphicons-halflings-regular.svg#glyphicons_halflingsregular) format('svg')}.glyphicon{position:relative;top:1px;display:inline-block;font-family:'Glyphicons Halflings';font-style:normal;font-weight:400;line-height:1;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}.glyphicon-asterisk:before{content:"\2a"}.glyphicon-plus:before{content:"\2b"}.glyphicon-eur:before,.glyphicon-euro:before{content:"\20ac"}.glyphicon-minus:before{content:"\2212"}.glyphicon-cloud:before{content:"\2601"}.glyphicon-envelope:before{content:"\2709"}.glyphicon-pencil:before{content:"\270f"}.glyphicon-glass:before{content:"\e001"}.glyphicon-music:before{content:"\e002"}.glyphicon-search:before{content:"\e003"}.glyphicon-heart:before{content:"\e005"}.glyphicon-star:before{content:"\e006"}.glyphicon-star-empty:before{content:"\e007"}.glyphicon-user:before{content:"\e008"}.glyphicon-film:before{content:"\e009"}.glyphicon-th-large:before{content:"\e010"}.glyphicon-th:before{content:"\e011"}.glyphicon-th-list:before{content:"\e012"}.glyphicon-ok:before{content:"\e013"}.glyphicon-remove:before{content:"\e014"}.glyphicon-zoom-in:before{content:"\e015"}.glyphicon-zoom-out:before{content:"\e016"}.glyphicon-off:before{content:"\e017"}.glyphicon-signal:before{content:"\e018"}.glyphicon-cog:before{content:"\e019"}.glyphicon-trash:before{content:"\e020"}.glyphicon-home:before{content:"\e021"}.glyphicon-file:before{content:"\e022"}.glyphicon-time:before{content:"\e023"}.glyphicon-road:before{content:"\e024"}.glyphicon-download-alt:before{content:"\e025"}.glyphicon-download:before{content:"\e026"}.glyphicon-upload:before{content:"\e027"}.glyphicon-inbox:before{content:"\e028"}.glyphicon-play-circle:before{content:"\e029"}.glyphicon-repeat:before{content:"\e030"}.glyphicon-refresh:before{content:"\e031"}.glyphicon-list-alt:before{content:"\e032"}.glyphicon-lock:before{content:"\e033"}.glyphicon-flag:before{content:"\e034"}.glyphicon-headphones:before{content:"\e035"}.glyphicon-volume-off:before{content:"\e036"}.glyphicon-volume-down:before{content:"\e037"}.glyphicon-volume-up:before{content:"\e038"}.glyphicon-qrcode:before{content:"\e039"}.glyphicon-barcode:before{content:"\e040"}.glyphicon-tag:before{content:"\e041"}.glyphicon-tags:before{content:"\e042"}.glyphicon-book:before{content:"\e043"}.glyphicon-bookmark:before{content:"\e044"}.glyphicon-print:before{content:"\e045"}.glyphicon-camera:before{content:"\e046"}.glyphicon-font:before{content:"\e047"}.glyphicon-bold:before{content:"\e048"}.glyphicon-italic:before{content:"\e049"}.glyphicon-text-height:before{content:"\e050"}.glyphicon-text-width:before{content:"\e051"}.glyphicon-align-left:before{content:"\e052"}.glyphicon-align-center:before{content:"\e053"}.glyphicon-align-right:before{content:"\e054"}.glyphicon-align-justify:before{content:"\e055"}.glyphicon-list:before{content:"\e056"}.glyphicon-indent-left:before{content:"\e057"}.glyphicon-indent-right:before{content:"\e058"}.glyphicon-facetime-video:before{content:"\e059"}.glyphicon-picture:before{content:"\e060"}.glyphicon-map-marker:before{content:"\e062"}.glyphicon-adjust:before{content:"\e063"}.glyphicon-tint:before{content:"\e064"}.glyphicon-edit:before{content:"\e065"}.glyphicon-share:before{content:"\e066"}.glyphicon-check:before{content:"\e067"}.glyphicon-move:before{content:"\e068"}.glyphicon-step-backward:before{content:"\e069"}.glyphicon-fast-backward:before{content:"\e070"}.glyphicon-backward:before{content:"\e071"}.glyphicon-play:before{content:"\e072"}.glyphicon-pause:before{content:"\e073"}.glyphicon-stop:before{content:"\e074"}.glyphicon-forward:before{content:"\e075"}.glyphicon-fast-forward:before{content:"\e076"}.glyphicon-step-forward:before{content:"\e077"}.glyphicon-eject:before{content:"\e078"}.glyphicon-chevron-left:before{content:"\e079"}.glyphicon-chevron-right:before{content:"\e080"}.glyphicon-plus-sign:before{content:"\e081"}.glyphicon-minus-sign:before{content:"\e082"}.glyphicon-remove-sign:before{content:"\e083"}.glyphicon-ok-sign:before{content:"\e084"}.glyphicon-question-sign:before{content:"\e085"}.glyphicon-info-sign:before{content:"\e086"}.glyphicon-screenshot:before{content:"\e087"}.glyphicon-remove-circle:before{content:"\e088"}.glyphicon-ok-circle:before{content:"\e089"}.glyphicon-ban-circle:before{content:"\e090"}.glyphicon-arrow-left:before{content:"\e091"}.glyphicon-arrow-right:before{content:"\e092"}.glyphicon-arrow-up:before{content:"\e093"}.glyphicon-arrow-down:before{content:"\e094"}.glyphicon-share-alt:before{content:"\e095"}.glyphicon-resize-full:before{content:"\e096"}.glyphicon-resize-small:before{content:"\e097"}.glyphicon-exclamation-sign:before{content:"\e101"}.glyphicon-gift:before{content:"\e102"}.glyphicon-leaf:before{content:"\e103"}.glyphicon-fire:before{content:"\e104"}.glyphicon-eye-open:before{content:"\e105"}.glyphicon-eye-close:before{content:"\e106"}.glyphicon-warning-sign:before{content:"\e107"}.glyphicon-plane:before{content:"\e108"}.glyphicon-calendar:before{content:"\e109"}.glyphicon-random:before{content:"\e110"}.glyphicon-comment:before{content:"\e111"}.glyphicon-magnet:before{content:"\e112"}.glyphicon-chevron-up:before{content:"\e113"}.glyphicon-chevron-down:before{content:"\e114"}.glyphicon-retweet:before{content:"\e115"}.glyphicon-shopping-cart:before{content:"\e116"}.glyphicon-folder-close:before{content:"\e117"}.glyphicon-folder-open:before{content:"\e118"}.glyphicon-resize-vertical:before{content:"\e119"}.glyphicon-resize-horizontal:before{content:"\e120"}.glyphicon-hdd:before{content:"\e121"}.glyphicon-bullhorn:before{content:"\e122"}.glyphicon-bell:before{content:"\e123"}.glyphicon-certificate:before{content:"\e124"}.glyphicon-thumbs-up:before{content:"\e125"}.glyphicon-thumbs-down:before{content:"\e126"}.glyphicon-hand-right:before{content:"\e127"}.glyphicon-hand-left:before{content:"\e128"}.glyphicon-hand-up:before{content:"\e129"}.glyphicon-hand-down:before{content:"\e130"}.glyphicon-circle-arrow-right:before{content:"\e131"}.glyphicon-circle-arrow-left:before{content:"\e132"}.glyphicon-circle-arrow-up:before{content:"\e133"}.glyphicon-circle-arrow-down:before{content:"\e134"}.glyphicon-globe:before{content:"\e135"}.glyphicon-wrench:before{content:"\e136"}.glyphicon-tasks:before{content:"\e137"}.glyphicon-filter:before{content:"\e138"}.glyphicon-briefcase:before{content:"\e139"}.glyphicon-fullscreen:before{content:"\e140"}.glyphicon-dashboard:before{content:"\e141"}.glyphicon-paperclip:before{content:"\e142"}.glyphicon-heart-empty:before{content:"\e143"}.glyphicon-link:before{content:"\e144"}.glyphicon-phone:before{content:"\e145"}.glyphicon-pushpin:before{content:"\e146"}.glyphicon-usd:before{content:"\e148"}.glyphicon-gbp:before{content:"\e149"}.glyphicon-sort:before{content:"\e150"}.glyphicon-sort-by-alphabet:before{content:"\e151"}.glyphicon-sort-by-alphabet-alt:before{content:"\e152"}.glyphicon-sort-by-order:before{content:"\e153"}.glyphicon-sort-by-order-alt:before{content:"\e154"}.glyphicon-sort-by-attributes:before{content:"\e155"}.glyphicon-sort-by-attributes-alt:before{content:"\e156"}.glyphicon-unchecked:before{content:"\e157"}.glyphicon-expand:before{content:"\e158"}.glyphicon-collapse-down:before{content:"\e159"}.glyphicon-collapse-up:before{content:"\e160"}.glyphicon-log-in:before{content:"\e161"}.glyphicon-flash:before{content:"\e162"}.glyphicon-log-out:before{content:"\e163"}.glyphicon-new-window:before{content:"\e164"}.glyphicon-record:before{content:"\e165"}.glyphicon-save:before{content:"\e166"}.glyphicon-open:before{content:"\e167"}.glyphicon-saved:before{content:"\e168"}.glyphicon-import:before{content:"\e169"}.glyphicon-export:before{content:"\e170"}.glyphicon-send:before{content:"\e171"}.glyphicon-floppy-disk:before{content:"\e172"}.glyphicon-floppy-saved:before{content:"\e173"}.glyphicon-floppy-remove:before{content:"\e174"}.glyphicon-floppy-save:before{content:"\e175"}.glyphicon-floppy-open:before{content:"\e176"}.glyphicon-credit-card:before{content:"\e177"}.glyphicon-transfer:before{content:"\e178"}.glyphicon-cutlery:before{content:"\e179"}.glyphicon-header:before{content:"\e180"}.glyphicon-compressed:before{content:"\e181"}.glyphicon-earphone:before{content:"\e182"}.glyphicon-phone-alt:before{content:"\e183"}.glyphicon-tower:before{content:"\e184"}.glyphicon-stats:before{content:"\e185"}.glyphicon-sd-video:before{content:"\e186"}.glyphicon-hd-video:before{content:"\e187"}.glyphicon-subtitles:before{content:"\e188"}.glyphicon-sound-stereo:before{content:"\e189"}.glyphicon-sound-dolby:before{content:"\e190"}.glyphicon-sound-5-1:before{content:"\e191"}.glyphicon-sound-6-1:before{content:"\e192"}.glyphicon-sound-7-1:before{content:"\e193"}.glyphicon-copyright-mark:before{content:"\e194"}.glyphicon-registration-mark:before{content:"\e195"}.glyphicon-cloud-download:before{content:"\e197"}.glyphicon-cloud-upload:before{content:"\e198"}.glyphicon-tree-conifer:before{content:"\e199"}.glyphicon-tree-deciduous:before{content:"\e200"}.glyphicon-cd:before{content:"\e201"}.glyphicon-save-file:before{content:"\e202"}.glyphicon-open-file:before{content:"\e203"}.glyphicon-level-up:before{content:"\e204"}.glyphicon-copy:before{content:"\e205"}.glyphicon-paste:before{content:"\e206"}.glyphicon-alert:before{content:"\e209"}.glyphicon-equalizer:before{content:"\e210"}.glyphicon-king:before{content:"\e211"}.glyphicon-queen:before{content:"\e212"}.glyphicon-pawn:before{content:"\e213"}.glyphicon-bishop:before{content:"\e214"}.glyphicon-knight:before{content:"\e215"}.glyphicon-baby-formula:before{content:"\e216"}.glyphicon-tent:before{content:"\26fa"}.glyphicon-blackboard:before{content:"\e218"}.glyphicon-bed:before{content:"\e219"}.glyphicon-apple:before{content:"\f8ff"}.glyphicon-erase:before{content:"\e221"}.glyphicon-hourglass:before{content:"\231b"}.glyphicon-lamp:before{content:"\e223"}.glyphicon-duplicate:before{content:"\e224"}.glyphicon-piggy-bank:before{content:"\e225"}.glyphicon-scissors:before{content:"\e226"}.glyphicon-bitcoin:before{content:"\e227"}.glyphicon-btc:before{content:"\e227"}.glyphicon-xbt:before{content:"\e227"}.glyphicon-yen:before{content:"\00a5"}.glyphicon-jpy:before{content:"\00a5"}.glyphicon-ruble:before{content:"\20bd"}.glyphicon-rub:before{content:"\20bd"}.glyphicon-scale:before{content:"\e230"}.glyphicon-ice-lolly:before{content:"\e231"}.glyphicon-ice-lolly-tasted:before{content:"\e232"}.glyphicon-education:before{content:"\e233"}.glyphicon-option-horizontal:before{content:"\e234"}.glyphicon-option-vertical:before{content:"\e235"}.glyphicon-menu-hamburger:before{content:"\e236"}.glyphicon-modal-window:before{content:"\e237"}.glyphicon-oil:before{content:"\e238"}.glyphicon-grain:before{content:"\e239"}.glyphicon-sunglasses:before{content:"\e240"}.glyphicon-text-size:before{content:"\e241"}.glyphicon-text-color:before{content:"\e242"}.glyphicon-text-background:before{content:"\e243"}.glyphicon-object-align-top:before{content:"\e244"}.glyphicon-object-align-bottom:before{content:"\e245"}.glyphicon-object-align-horizontal:before{content:"\e246"}.glyphicon-object-align-left:before{content:"\e247"}.glyphicon-object-align-vertical:before{content:"\e248"}.glyphicon-object-align-right:before{content:"\e249"}.glyphicon-triangle-right:before{content:"\e250"}.glyphicon-triangle-left:before{content:"\e251"}.glyphicon-triangle-bottom:before{content:"\e252"}.glyphicon-triangle-top:before{content:"\e253"}.glyphicon-console:before{content:"\e254"}.glyphicon-superscript:before{content:"\e255"}.glyphicon-subscript:before{content:"\e256"}.glyphicon-menu-left:before{content:"\e257"}.glyphicon-menu-right:before{content:"\e258"}.glyphicon-menu-down:before{content:"\e259"}.glyphicon-menu-up:before{content:"\e260"}*{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}:after,:before{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}html{font-size:10px;-webkit-tap-highlight-color:rgba(0,0,0,0)}body{font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:14px;line-height:1.42857143;color:#333;background-color:#fff}button,input,select,textarea{font-family:inherit;font-size:inherit;line-height:inherit}a{color:#337ab7;text-decoration:none}a:focus,a:hover{color:#23527c;text-decoration:underline}a:focus{outline:thin dotted;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}figure{margin:0}img{vertical-align:middle}.carousel-inner>.item>a>img,.carousel-inner>.item>img,.img-responsive,.thumbnail a>img,.thumbnail>img{display:block;max-width:100%;height:auto}.img-rounded{border-radius:6px}.img-thumbnail{display:inline-block;max-width:100%;height:auto;padding:4px;line-height:1.42857143;background-color:#fff;border:1px solid #ddd;border-radius:4px;-webkit-transition:all .2s ease-in-out;-o-transition:all .2s ease-in-out;transition:all .2s ease-in-out}.img-circle{border-radius:50%}hr{margin-top:20px;margin-bottom:20px;border:0;border-top:1px solid #eee}.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);border:0}.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;margin:0;overflow:visible;clip:auto}[role=button]{cursor:pointer}.h1,.h2,.h3,.h4,.h5,.h6,h1,h2,h3,h4,h5,h6{font-family:inherit;font-weight:500;line-height:1.1;color:inherit}.h1 .small,.h1 small,.h2 .small,.h2 small,.h3 .small,.h3 small,.h4 .small,.h4 small,.h5 .small,.h5 small,.h6 .small,.h6 small,h1 .small,h1 small,h2 .small,h2 small,h3 .small,h3 small,h4 .small,h4 small,h5 .small,h5 small,h6 .small,h6 small{font-weight:400;line-height:1;color:#777}.h1,.h2,.h3,h1,h2,h3{margin-top:20px;margin-bottom:10px}.h1 .small,.h1 small,.h2 .small,.h2 small,.h3 .small,.h3 small,h1 .small,h1 small,h2 .small,h2 small,h3 .small,h3 small{font-size:65%}.h4,.h5,.h6,h4,h5,h6{margin-top:10px;margin-bottom:10px}.h4 .small,.h4 small,.h5 .small,.h5 small,.h6 .small,.h6 small,h4 .small,h4 small,h5 .small,h5 small,h6 .small,h6 small{font-size:75%}.h1,h1{font-size:36px}.h2,h2{font-size:30px}.h3,h3{font-size:24px}.h4,h4{font-size:18px}.h5,h5{font-size:14px}.h6,h6{font-size:12px}p{margin:0 0 10px}.lead{margin-bottom:20px;font-size:16px;font-weight:300;line-height:1.4}@media (min-width:768px){.lead{font-size:21px}}.small,small{font-size:85%}.mark,mark{padding:.2em;background-color:#fcf8e3}.text-left{text-align:left}.text-right{text-align:right}.text-center{text-align:center}.text-justify{text-align:justify}.text-nowrap{white-space:nowrap}.text-lowercase{text-transform:lowercase}.text-uppercase{text-transform:uppercase}.text-capitalize{text-transform:capitalize}.text-muted{color:#777}.text-primary{color:#337ab7}a.text-primary:focus,a.text-primary:hover{color:#286090}.text-success{color:#3c763d}a.text-success:focus,a.text-success:hover{color:#2b542c}.text-info{color:#31708f}a.text-info:focus,a.text-info:hover{color:#245269}.text-warning{color:#8a6d3b}a.text-warning:focus,a.text-warning:hover{color:#66512c}.text-danger{color:#a94442}a.text-danger:focus,a.text-danger:hover{color:#843534}.bg-primary{color:#fff;background-color:#337ab7}a.bg-primary:focus,a.bg-primary:hover{background-color:#286090}.bg-success{background-color:#dff0d8}a.bg-success:focus,a.bg-success:hover{background-color:#c1e2b3}.bg-info{background-color:#d9edf7}a.bg-info:focus,a.bg-info:hover{background-color:#afd9ee}.bg-warning{background-color:#fcf8e3}a.bg-warning:focus,a.bg-warning:hover{background-color:#f7ecb5}.bg-danger{background-color:#f2dede}a.bg-danger:focus,a.bg-danger:hover{background-color:#e4b9b9}.page-header{padding-bottom:9px;margin:40px 0 20px;border-bottom:1px solid #eee}ol,ul{margin-top:0;margin-bottom:10px}ol ol,ol ul,ul ol,ul ul{margin-bottom:0}.list-unstyled{padding-left:0;list-style:none}.list-inline{padding-left:0;margin-left:-5px;list-style:none}.list-inline>li{display:inline-block;padding-right:5px;padding-left:5px}dl{margin-top:0;margin-bottom:20px}dd,dt{line-height:1.42857143}dt{font-weight:700}dd{margin-left:0}@media (min-width:768px){.dl-horizontal dt{float:left;width:160px;overflow:hidden;clear:left;text-align:right;text-overflow:ellipsis;white-space:nowrap}.dl-horizontal dd{margin-left:180px}}abbr[data-original-title],abbr[title]{cursor:help;border-bottom:1px dotted #777}.initialism{font-size:90%;text-transform:uppercase}blockquote{padding:10px 20px;margin:0 0 20px;font-size:17.5px;border-left:5px solid #eee}blockquote ol:last-child,blockquote p:last-child,blockquote ul:last-child{margin-bottom:0}blockquote .small,blockquote footer,blockquote small{display:block;font-size:80%;line-height:1.42857143;color:#777}blockquote .small:before,blockquote footer:before,blockquote small:before{content:'\2014 \00A0'}.blockquote-reverse,blockquote.pull-right{padding-right:15px;padding-left:0;text-align:right;border-right:5px solid #eee;border-left:0}.blockquote-reverse .small:before,.blockquote-reverse footer:before,.blockquote-reverse small:before,blockquote.pull-right .small:before,blockquote.pull-right footer:before,blockquote.pull-right small:before{content:''}.blockquote-reverse .small:after,.blockquote-reverse footer:after,.blockquote-reverse small:after,blockquote.pull-right .small:after,blockquote.pull-right footer:after,blockquote.pull-right small:after{content:'\00A0 \2014'}address{margin-bottom:20px;font-style:normal;line-height:1.42857143}code,kbd,pre,samp{font-family:Menlo,Monaco,Consolas,"Courier New",monospace}code{padding:2px 4px;font-size:90%;color:#c7254e;background-color:#f9f2f4;border-radius:4px}kbd{padding:2px 4px;font-size:90%;color:#fff;background-color:#333;border-radius:3px;-webkit-box-shadow:inset 0 -1px 0 rgba(0,0,0,.25);box-shadow:inset 0 -1px 0 rgba(0,0,0,.25)}kbd kbd{padding:0;font-size:100%;font-weight:700;-webkit-box-shadow:none;box-shadow:none}pre{display:block;padding:9.5px;margin:0 0 10px;font-size:13px;line-height:1.42857143;color:#333;word-break:break-all;word-wrap:break-word;background-color:#f5f5f5;border:1px solid #ccc;border-radius:4px}pre code{padding:0;font-size:inherit;color:inherit;white-space:pre-wrap;background-color:transparent;border-radius:0}.pre-scrollable{max-height:340px;overflow-y:scroll}.container{padding-right:15px;padding-left:15px;margin-right:auto;margin-left:auto}@media (min-width:768px){.container{width:750px}}@media (min-width:992px){.container{width:970px}}@media (min-width:1200px){.container{width:1170px}}.container-fluid{padding-right:15px;padding-left:15px;margin-right:auto;margin-left:auto}.row{margin-right:-15px;margin-left:-15px}.col-lg-1,.col-lg-10,.col-lg-11,.col-lg-12,.col-lg-2,.col-lg-3,.col-lg-4,.col-lg-5,.col-lg-6,.col-lg-7,.col-lg-8,.col-lg-9,.col-md-1,.col-md-10,.col-md-11,.col-md-12,.col-md-2,.col-md-3,.col-md-4,.col-md-5,.col-md-6,.col-md-7,.col-md-8,.col-md-9,.col-sm-1,.col-sm-10,.col-sm-11,.col-sm-12,.col-sm-2,.col-sm-3,.col-sm-4,.col-sm-5,.col-sm-6,.col-sm-7,.col-sm-8,.col-sm-9,.col-xs-1,.col-xs-10,.col-xs-11,.col-xs-12,.col-xs-2,.col-xs-3,.col-xs-4,.col-xs-5,.col-xs-6,.col-xs-7,.col-xs-8,.col-xs-9{position:relative;min-height:1px;padding-right:15px;padding-left:15px}.col-xs-1,.col-xs-10,.col-xs-11,.col-xs-12,.col-xs-2,.col-xs-3,.col-xs-4,.col-xs-5,.col-xs-6,.col-xs-7,.col-xs-8,.col-xs-9{float:left}.col-xs-12{width:100%}.col-xs-11{width:91.66666667%}.col-xs-10{width:83.33333333%}.col-xs-9{width:75%}.col-xs-8{width:66.66666667%}.col-xs-7{width:58.33333333%}.col-xs-6{width:50%}.col-xs-5{width:41.66666667%}.col-xs-4{width:33.33333333%}.col-xs-3{width:25%}.col-xs-2{width:16.66666667%}.col-xs-1{width:8.33333333%}.col-xs-pull-12{right:100%}.col-xs-pull-11{right:91.66666667%}.col-xs-pull-10{right:83.33333333%}.col-xs-pull-9{right:75%}.col-xs-pull-8{right:66.66666667%}.col-xs-pull-7{right:58.33333333%}.col-xs-pull-6{right:50%}.col-xs-pull-5{right:41.66666667%}.col-xs-pull-4{right:33.33333333%}.col-xs-pull-3{right:25%}.col-xs-pull-2{right:16.66666667%}.col-xs-pull-1{right:8.33333333%}.col-xs-pull-0{right:auto}.col-xs-push-12{left:100%}.col-xs-push-11{left:91.66666667%}.col-xs-push-10{left:83.33333333%}.col-xs-push-9{left:75%}.col-xs-push-8{left:66.66666667%}.col-xs-push-7{left:58.33333333%}.col-xs-push-6{left:50%}.col-xs-push-5{left:41.66666667%}.col-xs-push-4{left:33.33333333%}.col-xs-push-3{left:25%}.col-xs-push-2{left:16.66666667%}.col-xs-push-1{left:8.33333333%}.col-xs-push-0{left:auto}.col-xs-offset-12{margin-left:100%}.col-xs-offset-11{margin-left:91.66666667%}.col-xs-offset-10{margin-left:83.33333333%}.col-xs-offset-9{margin-left:75%}.col-xs-offset-8{margin-left:66.66666667%}.col-xs-offset-7{margin-left:58.33333333%}.col-xs-offset-6{margin-left:50%}.col-xs-offset-5{margin-left:41.66666667%}.col-xs-offset-4{margin-left:33.33333333%}.col-xs-offset-3{margin-left:25%}.col-xs-offset-2{margin-left:16.66666667%}.col-xs-offset-1{margin-left:8.33333333%}.col-xs-offset-0{margin-left:0}@media (min-width:768px){.col-sm-1,.col-sm-10,.col-sm-11,.col-sm-12,.col-sm-2,.col-sm-3,.col-sm-4,.col-sm-5,.col-sm-6,.col-sm-7,.col-sm-8,.col-sm-9{float:left}.col-sm-12{width:100%}.col-sm-11{width:91.66666667%}.col-sm-10{width:83.33333333%}.col-sm-9{width:75%}.col-sm-8{width:66.66666667%}.col-sm-7{width:58.33333333%}.col-sm-6{width:50%}.col-sm-5{width:41.66666667%}.col-sm-4{width:33.33333333%}.col-sm-3{width:25%}.col-sm-2{width:16.66666667%}.col-sm-1{width:8.33333333%}.col-sm-pull-12{right:100%}.col-sm-pull-11{right:91.66666667%}.col-sm-pull-10{right:83.33333333%}.col-sm-pull-9{right:75%}.col-sm-pull-8{right:66.66666667%}.col-sm-pull-7{right:58.33333333%}.col-sm-pull-6{right:50%}.col-sm-pull-5{right:41.66666667%}.col-sm-pull-4{right:33.33333333%}.col-sm-pull-3{right:25%}.col-sm-pull-2{right:16.66666667%}.col-sm-pull-1{right:8.33333333%}.col-sm-pull-0{right:auto}.col-sm-push-12{left:100%}.col-sm-push-11{left:91.66666667%}.col-sm-push-10{left:83.33333333%}.col-sm-push-9{left:75%}.col-sm-push-8{left:66.66666667%}.col-sm-push-7{left:58.33333333%}.col-sm-push-6{left:50%}.col-sm-push-5{left:41.66666667%}.col-sm-push-4{left:33.33333333%}.col-sm-push-3{left:25%}.col-sm-push-2{left:16.66666667%}.col-sm-push-1{left:8.33333333%}.col-sm-push-0{left:auto}.col-sm-offset-12{margin-left:100%}.col-sm-offset-11{margin-left:91.66666667%}.col-sm-offset-10{margin-left:83.33333333%}.col-sm-offset-9{margin-left:75%}.col-sm-offset-8{margin-left:66.66666667%}.col-sm-offset-7{margin-left:58.33333333%}.col-sm-offset-6{margin-left:50%}.col-sm-offset-5{margin-left:41.66666667%}.col-sm-offset-4{margin-left:33.33333333%}.col-sm-offset-3{margin-left:25%}.col-sm-offset-2{margin-left:16.66666667%}.col-sm-offset-1{margin-left:8.33333333%}.col-sm-offset-0{margin-left:0}}@media (min-width:992px){.col-md-1,.col-md-10,.col-md-11,.col-md-12,.col-md-2,.col-md-3,.col-md-4,.col-md-5,.col-md-6,.col-md-7,.col-md-8,.col-md-9{float:left}.col-md-12{width:100%}.col-md-11{width:91.66666667%}.col-md-10{width:83.33333333%}.col-md-9{width:75%}.col-md-8{width:66.66666667%}.col-md-7{width:58.33333333%}.col-md-6{width:50%}.col-md-5{width:41.66666667%}.col-md-4{width:33.33333333%}.col-md-3{width:25%}.col-md-2{width:16.66666667%}.col-md-1{width:8.33333333%}.col-md-pull-12{right:100%}.col-md-pull-11{right:91.66666667%}.col-md-pull-10{right:83.33333333%}.col-md-pull-9{right:75%}.col-md-pull-8{right:66.66666667%}.col-md-pull-7{right:58.33333333%}.col-md-pull-6{right:50%}.col-md-pull-5{right:41.66666667%}.col-md-pull-4{right:33.33333333%}.col-md-pull-3{right:25%}.col-md-pull-2{right:16.66666667%}.col-md-pull-1{right:8.33333333%}.col-md-pull-0{right:auto}.col-md-push-12{left:100%}.col-md-push-11{left:91.66666667%}.col-md-push-10{left:83.33333333%}.col-md-push-9{left:75%}.col-md-push-8{left:66.66666667%}.col-md-push-7{left:58.33333333%}.col-md-push-6{left:50%}.col-md-push-5{left:41.66666667%}.col-md-push-4{left:33.33333333%}.col-md-push-3{left:25%}.col-md-push-2{left:16.66666667%}.col-md-push-1{left:8.33333333%}.col-md-push-0{left:auto}.col-md-offset-12{margin-left:100%}.col-md-offset-11{margin-left:91.66666667%}.col-md-offset-10{margin-left:83.33333333%}.col-md-offset-9{margin-left:75%}.col-md-offset-8{margin-left:66.66666667%}.col-md-offset-7{margin-left:58.33333333%}.col-md-offset-6{margin-left:50%}.col-md-offset-5{margin-left:41.66666667%}.col-md-offset-4{margin-left:33.33333333%}.col-md-offset-3{margin-left:25%}.col-md-offset-2{margin-left:16.66666667%}.col-md-offset-1{margin-left:8.33333333%}.col-md-offset-0{margin-left:0}}@media (min-width:1200px){.col-lg-1,.col-lg-10,.col-lg-11,.col-lg-12,.col-lg-2,.col-lg-3,.col-lg-4,.col-lg-5,.col-lg-6,.col-lg-7,.col-lg-8,.col-lg-9{float:left}.col-lg-12{width:100%}.col-lg-11{width:91.66666667%}.col-lg-10{width:83.33333333%}.col-lg-9{width:75%}.col-lg-8{width:66.66666667%}.col-lg-7{width:58.33333333%}.col-lg-6{width:50%}.col-lg-5{width:41.66666667%}.col-lg-4{width:33.33333333%}.col-lg-3{width:25%}.col-lg-2{width:16.66666667%}.col-lg-1{width:8.33333333%}.col-lg-pull-12{right:100%}.col-lg-pull-11{right:91.66666667%}.col-lg-pull-10{right:83.33333333%}.col-lg-pull-9{right:75%}.col-lg-pull-8{right:66.66666667%}.col-lg-pull-7{right:58.33333333%}.col-lg-pull-6{right:50%}.col-lg-pull-5{right:41.66666667%}.col-lg-pull-4{right:33.33333333%}.col-lg-pull-3{right:25%}.col-lg-pull-2{right:16.66666667%}.col-lg-pull-1{right:8.33333333%}.col-lg-pull-0{right:auto}.col-lg-push-12{left:100%}.col-lg-push-11{left:91.66666667%}.col-lg-push-10{left:83.33333333%}.col-lg-push-9{left:75%}.col-lg-push-8{left:66.66666667%}.col-lg-push-7{left:58.33333333%}.col-lg-push-6{left:50%}.col-lg-push-5{left:41.66666667%}.col-lg-push-4{left:33.33333333%}.col-lg-push-3{left:25%}.col-lg-push-2{left:16.66666667%}.col-lg-push-1{left:8.33333333%}.col-lg-push-0{left:auto}.col-lg-offset-12{margin-left:100%}.col-lg-offset-11{margin-left:91.66666667%}.col-lg-offset-10{margin-left:83.33333333%}.col-lg-offset-9{margin-left:75%}.col-lg-offset-8{margin-left:66.66666667%}.col-lg-offset-7{margin-left:58.33333333%}.col-lg-offset-6{margin-left:50%}.col-lg-offset-5{margin-left:41.66666667%}.col-lg-offset-4{margin-left:33.33333333%}.col-lg-offset-3{margin-left:25%}.col-lg-offset-2{margin-left:16.66666667%}.col-lg-offset-1{margin-left:8.33333333%}.col-lg-offset-0{margin-left:0}}table{background-color:transparent}caption{padding-top:8px;padding-bottom:8px;color:#777;text-align:left}th{text-align:left}.table{width:100%;max-width:100%;margin-bottom:20px}.table>tbody>tr>td,.table>tbody>tr>th,.table>tfoot>tr>td,.table>tfoot>tr>th,.table>thead>tr>td,.table>thead>tr>th{padding:8px;line-height:1.42857143;vertical-align:top;border-top:1px solid #ddd}.table>thead>tr>th{vertical-align:bottom;border-bottom:2px solid #ddd}.table>caption+thead>tr:first-child>td,.table>caption+thead>tr:first-child>th,.table>colgroup+thead>tr:first-child>td,.table>colgroup+thead>tr:first-child>th,.table>thead:first-child>tr:first-child>td,.table>thead:first-child>tr:first-child>th{border-top:0}.table>tbody+tbody{border-top:2px solid #ddd}.table .table{background-color:#fff}.table-condensed>tbody>tr>td,.table-condensed>tbody>tr>th,.table-condensed>tfoot>tr>td,.table-condensed>tfoot>tr>th,.table-condensed>thead>tr>td,.table-condensed>thead>tr>th{padding:5px}.table-bordered{border:1px solid #ddd}.table-bordered>tbody>tr>td,.table-bordered>tbody>tr>th,.table-bordered>tfoot>tr>td,.table-bordered>tfoot>tr>th,.table-bordered>thead>tr>td,.table-bordered>thead>tr>th{border:1px solid #ddd}.table-bordered>thead>tr>td,.table-bordered>thead>tr>th{border-bottom-width:2px}.table-striped>tbody>tr:nth-of-type(odd){background-color:#f9f9f9}.table-hover>tbody>tr:hover{background-color:#f5f5f5}table col[class*=col-]{position:static;display:table-column;float:none}table td[class*=col-],table th[class*=col-]{position:static;display:table-cell;float:none}.table>tbody>tr.active>td,.table>tbody>tr.active>th,.table>tbody>tr>td.active,.table>tbody>tr>th.active,.table>tfoot>tr.active>td,.table>tfoot>tr.active>th,.table>tfoot>tr>td.active,.table>tfoot>tr>th.active,.table>thead>tr.active>td,.table>thead>tr.active>th,.table>thead>tr>td.active,.table>thead>tr>th.active{background-color:#f5f5f5}.table-hover>tbody>tr.active:hover>td,.table-hover>tbody>tr.active:hover>th,.table-hover>tbody>tr:hover>.active,.table-hover>tbody>tr>td.active:hover,.table-hover>tbody>tr>th.active:hover{background-color:#e8e8e8}.table>tbody>tr.success>td,.table>tbody>tr.success>th,.table>tbody>tr>td.success,.table>tbody>tr>th.success,.table>tfoot>tr.success>td,.table>tfoot>tr.success>th,.table>tfoot>tr>td.success,.table>tfoot>tr>th.success,.table>thead>tr.success>td,.table>thead>tr.success>th,.table>thead>tr>td.success,.table>thead>tr>th.success{background-color:#dff0d8}.table-hover>tbody>tr.success:hover>td,.table-hover>tbody>tr.success:hover>th,.table-hover>tbody>tr:hover>.success,.table-hover>tbody>tr>td.success:hover,.table-hover>tbody>tr>th.success:hover{background-color:#d0e9c6}.table>tbody>tr.info>td,.table>tbody>tr.info>th,.table>tbody>tr>td.info,.table>tbody>tr>th.info,.table>tfoot>tr.info>td,.table>tfoot>tr.info>th,.table>tfoot>tr>td.info,.table>tfoot>tr>th.info,.table>thead>tr.info>td,.table>thead>tr.info>th,.table>thead>tr>td.info,.table>thead>tr>th.info{background-color:#d9edf7}.table-hover>tbody>tr.info:hover>td,.table-hover>tbody>tr.info:hover>th,.table-hover>tbody>tr:hover>.info,.table-hover>tbody>tr>td.info:hover,.table-hover>tbody>tr>th.info:hover{background-color:#c4e3f3}.table>tbody>tr.warning>td,.table>tbody>tr.warning>th,.table>tbody>tr>td.warning,.table>tbody>tr>th.warning,.table>tfoot>tr.warning>td,.table>tfoot>tr.warning>th,.table>tfoot>tr>td.warning,.table>tfoot>tr>th.warning,.table>thead>tr.warning>td,.table>thead>tr.warning>th,.table>thead>tr>td.warning,.table>thead>tr>th.warning{background-color:#fcf8e3}.table-hover>tbody>tr.warning:hover>td,.table-hover>tbody>tr.warning:hover>th,.table-hover>tbody>tr:hover>.warning,.table-hover>tbody>tr>td.warning:hover,.table-hover>tbody>tr>th.warning:hover{background-color:#faf2cc}.table>tbody>tr.danger>td,.table>tbody>tr.danger>th,.table>tbody>tr>td.danger,.table>tbody>tr>th.danger,.table>tfoot>tr.danger>td,.table>tfoot>tr.danger>th,.table>tfoot>tr>td.danger,.table>tfoot>tr>th.danger,.table>thead>tr.danger>td,.table>thead>tr.danger>th,.table>thead>tr>td.danger,.table>thead>tr>th.danger{background-color:#f2dede}.table-hover>tbody>tr.danger:hover>td,.table-hover>tbody>tr.danger:hover>th,.table-hover>tbody>tr:hover>.danger,.table-hover>tbody>tr>td.danger:hover,.table-hover>tbody>tr>th.danger:hover{background-color:#ebcccc}.table-responsive{min-height:.01%;overflow-x:auto}@media screen and (max-width:767px){.table-responsive{width:100%;margin-bottom:15px;overflow-y:hidden;-ms-overflow-style:-ms-autohiding-scrollbar;border:1px solid #ddd}.table-responsive>.table{margin-bottom:0}.table-responsive>.table>tbody>tr>td,.table-responsive>.table>tbody>tr>th,.table-responsive>.table>tfoot>tr>td,.table-responsive>.table>tfoot>tr>th,.table-responsive>.table>thead>tr>td,.table-responsive>.table>thead>tr>th{white-space:nowrap}.table-responsive>.table-bordered{border:0}.table-responsive>.table-bordered>tbody>tr>td:first-child,.table-responsive>.table-bordered>tbody>tr>th:first-child,.table-responsive>.table-bordered>tfoot>tr>td:first-child,.table-responsive>.table-bordered>tfoot>tr>th:first-child,.table-responsive>.table-bordered>thead>tr>td:first-child,.table-responsive>.table-bordered>thead>tr>th:first-child{border-left:0}.table-responsive>.table-bordered>tbody>tr>td:last-child,.table-responsive>.table-bordered>tbody>tr>th:last-child,.table-responsive>.table-bordered>tfoot>tr>td:last-child,.table-responsive>.table-bordered>tfoot>tr>th:last-child,.table-responsive>.table-bordered>thead>tr>td:last-child,.table-responsive>.table-bordered>thead>tr>th:last-child{border-right:0}.table-responsive>.table-bordered>tbody>tr:last-child>td,.table-responsive>.table-bordered>tbody>tr:last-child>th,.table-responsive>.table-bordered>tfoot>tr:last-child>td,.table-responsive>.table-bordered>tfoot>tr:last-child>th{border-bottom:0}}fieldset{min-width:0;padding:0;margin:0;border:0}legend{display:block;width:100%;padding:0;margin-bottom:20px;font-size:21px;line-height:inherit;color:#333;border:0;border-bottom:1px solid #e5e5e5}label{display:inline-block;max-width:100%;margin-bottom:5px;font-weight:700}input[type=search]{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}input[type=checkbox],input[type=radio]{margin:4px 0 0;margin-top:1px\9;line-height:normal}input[type=file]{display:block}input[type=range]{display:block;width:100%}select[multiple],select[size]{height:auto}input[type=file]:focus,input[type=checkbox]:focus,input[type=radio]:focus{outline:thin dotted;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}output{display:block;padding-top:7px;font-size:14px;line-height:1.42857143;color:#555}.form-control{display:block;width:100%;height:34px;padding:6px 12px;font-size:14px;line-height:1.42857143;color:#555;background-color:#fff;background-image:none;border:1px solid #ccc;border-radius:4px;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075);box-shadow:inset 0 1px 1px rgba(0,0,0,.075);-webkit-transition:border-color ease-in-out .15s,-webkit-box-shadow ease-in-out .15s;-o-transition:border-color ease-in-out .15s,box-shadow ease-in-out .15s;transition:border-color ease-in-out .15s,box-shadow ease-in-out .15s}.form-control:focus{border-color:#66afe9;outline:0;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 8px rgba(102,175,233,.6);box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 8px rgba(102,175,233,.6)}.form-control::-moz-placeholder{color:#999;opacity:1}.form-control:-ms-input-placeholder{color:#999}.form-control::-webkit-input-placeholder{color:#999}.form-control[disabled],.form-control[readonly],fieldset[disabled] .form-control{background-color:#eee;opacity:1}.form-control[disabled],fieldset[disabled] .form-control{cursor:not-allowed}textarea.form-control{height:auto}input[type=search]{-webkit-appearance:none}@media screen and (-webkit-min-device-pixel-ratio:0){input[type=date].form-control,input[type=time].form-control,input[type=datetime-local].form-control,input[type=month].form-control{line-height:34px}.input-group-sm input[type=date],.input-group-sm input[type=time],.input-group-sm input[type=datetime-local],.input-group-sm input[type=month],input[type=date].input-sm,input[type=time].input-sm,input[type=datetime-local].input-sm,input[type=month].input-sm{line-height:30px}.input-group-lg input[type=date],.input-group-lg input[type=time],.input-group-lg input[type=datetime-local],.input-group-lg input[type=month],input[type=date].input-lg,input[type=time].input-lg,input[type=datetime-local].input-lg,input[type=month].input-lg{line-height:46px}}.form-group{margin-bottom:15px}.checkbox,.radio{position:relative;display:block;margin-top:10px;margin-bottom:10px}.checkbox label,.radio label{min-height:20px;padding-left:20px;margin-bottom:0;font-weight:400;cursor:pointer}.checkbox input[type=checkbox],.checkbox-inline input[type=checkbox],.radio input[type=radio],.radio-inline input[type=radio]{position:absolute;margin-top:4px\9;margin-left:-20px}.checkbox+.checkbox,.radio+.radio{margin-top:-5px}.checkbox-inline,.radio-inline{position:relative;display:inline-block;padding-left:20px;margin-bottom:0;font-weight:400;vertical-align:middle;cursor:pointer}.checkbox-inline+.checkbox-inline,.radio-inline+.radio-inline{margin-top:0;margin-left:10px}fieldset[disabled] input[type=checkbox],fieldset[disabled] input[type=radio],input[type=checkbox].disabled,input[type=checkbox][disabled],input[type=radio].disabled,input[type=radio][disabled]{cursor:not-allowed}.checkbox-inline.disabled,.radio-inline.disabled,fieldset[disabled] .checkbox-inline,fieldset[disabled] .radio-inline{cursor:not-allowed}.checkbox.disabled label,.radio.disabled label,fieldset[disabled] .checkbox label,fieldset[disabled] .radio label{cursor:not-allowed}.form-control-static{min-height:34px;padding-top:7px;padding-bottom:7px;margin-bottom:0}.form-control-static.input-lg,.form-control-static.input-sm{padding-right:0;padding-left:0}.input-sm{height:30px;padding:5px 10px;font-size:12px;line-height:1.5;border-radius:3px}select.input-sm{height:30px;line-height:30px}select[multiple].input-sm,textarea.input-sm{height:auto}.form-group-sm .form-control{height:30px;padding:5px 10px;font-size:12px;line-height:1.5;border-radius:3px}.form-group-sm select.form-control{height:30px;line-height:30px}.form-group-sm select[multiple].form-control,.form-group-sm textarea.form-control{height:auto}.form-group-sm .form-control-static{height:30px;min-height:32px;padding:6px 10px;font-size:12px;line-height:1.5}.input-lg{height:46px;padding:10px 16px;font-size:18px;line-height:1.3333333;border-radius:6px}select.input-lg{height:46px;line-height:46px}select[multiple].input-lg,textarea.input-lg{height:auto}.form-group-lg .form-control{height:46px;padding:10px 16px;font-size:18px;line-height:1.3333333;border-radius:6px}.form-group-lg select.form-control{height:46px;line-height:46px}.form-group-lg select[multiple].form-control,.form-group-lg textarea.form-control{height:auto}.form-group-lg .form-control-static{height:46px;min-height:38px;padding:11px 16px;font-size:18px;line-height:1.3333333}.has-feedback{position:relative}.has-feedback .form-control{padding-right:42.5px}.form-control-feedback{position:absolute;top:0;right:0;z-index:2;display:block;width:34px;height:34px;line-height:34px;text-align:center;pointer-events:none}.form-group-lg .form-control+.form-control-feedback,.input-group-lg+.form-control-feedback,.input-lg+.form-control-feedback{width:46px;height:46px;line-height:46px}.form-group-sm .form-control+.form-control-feedback,.input-group-sm+.form-control-feedback,.input-sm+.form-control-feedback{width:30px;height:30px;line-height:30px}.has-success .checkbox,.has-success .checkbox-inline,.has-success .control-label,.has-success .help-block,.has-success .radio,.has-success .radio-inline,.has-success.checkbox label,.has-success.checkbox-inline label,.has-success.radio label,.has-success.radio-inline label{color:#3c763d}.has-success .form-control{border-color:#3c763d;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075);box-shadow:inset 0 1px 1px rgba(0,0,0,.075)}.has-success .form-control:focus{border-color:#2b542c;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 6px #67b168;box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 6px #67b168}.has-success .input-group-addon{color:#3c763d;background-color:#dff0d8;border-color:#3c763d}.has-success .form-control-feedback{color:#3c763d}.has-warning .checkbox,.has-warning .checkbox-inline,.has-warning .control-label,.has-warning .help-block,.has-warning .radio,.has-warning .radio-inline,.has-warning.checkbox label,.has-warning.checkbox-inline label,.has-warning.radio label,.has-warning.radio-inline label{color:#8a6d3b}.has-warning .form-control{border-color:#8a6d3b;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075);box-shadow:inset 0 1px 1px rgba(0,0,0,.075)}.has-warning .form-control:focus{border-color:#66512c;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 6px #c0a16b;box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 6px #c0a16b}.has-warning .input-group-addon{color:#8a6d3b;background-color:#fcf8e3;border-color:#8a6d3b}.has-warning .form-control-feedback{color:#8a6d3b}.has-error .checkbox,.has-error .checkbox-inline,.has-error .control-label,.has-error .help-block,.has-error .radio,.has-error .radio-inline,.has-error.checkbox label,.has-error.checkbox-inline label,.has-error.radio label,.has-error.radio-inline label{color:#a94442}.has-error .form-control{border-color:#a94442;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075);box-shadow:inset 0 1px 1px rgba(0,0,0,.075)}.has-error .form-control:focus{border-color:#843534;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 6px #ce8483;box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 6px #ce8483}.has-error .input-group-addon{color:#a94442;background-color:#f2dede;border-color:#a94442}.has-error .form-control-feedback{color:#a94442}.has-feedback label~.form-control-feedback{top:25px}.has-feedback label.sr-only~.form-control-feedback{top:0}.help-block{display:block;margin-top:5px;margin-bottom:10px;color:#737373}@media (min-width:768px){.form-inline .form-group{display:inline-block;margin-bottom:0;vertical-align:middle}.form-inline .form-control{display:inline-block;width:auto;vertical-align:middle}.form-inline .form-control-static{display:inline-block}.form-inline .input-group{display:inline-table;vertical-align:middle}.form-inline .input-group .form-control,.form-inline .input-group .input-group-addon,.form-inline .input-group .input-group-btn{width:auto}.form-inline .input-group>.form-control{width:100%}.form-inline .control-label{margin-bottom:0;vertical-align:middle}.form-inline .checkbox,.form-inline .radio{display:inline-block;margin-top:0;margin-bottom:0;vertical-align:middle}.form-inline .checkbox label,.form-inline .radio label{padding-left:0}.form-inline .checkbox input[type=checkbox],.form-inline .radio input[type=radio]{position:relative;margin-left:0}.form-inline .has-feedback .form-control-feedback{top:0}}.form-horizontal .checkbox,.form-horizontal .checkbox-inline,.form-horizontal .radio,.form-horizontal .radio-inline{padding-top:7px;margin-top:0;margin-bottom:0}.form-horizontal .checkbox,.form-horizontal .radio{min-height:27px}.form-horizontal .form-group{margin-right:-15px;margin-left:-15px}@media (min-width:768px){.form-horizontal .control-label{padding-top:7px;margin-bottom:0;text-align:right}}.form-horizontal .has-feedback .form-control-feedback{right:15px}@media (min-width:768px){.form-horizontal .form-group-lg .control-label{padding-top:14.33px;font-size:18px}}@media (min-width:768px){.form-horizontal .form-group-sm .control-label{padding-top:6px;font-size:12px}}.btn{display:inline-block;padding:6px 12px;margin-bottom:0;font-size:14px;font-weight:400;line-height:1.42857143;text-align:center;white-space:nowrap;vertical-align:middle;-ms-touch-action:manipulation;touch-action:manipulation;cursor:pointer;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;background-image:none;border:1px solid transparent;border-radius:4px}.btn.active.focus,.btn.active:focus,.btn.focus,.btn:active.focus,.btn:active:focus,.btn:focus{outline:thin dotted;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}.btn.focus,.btn:focus,.btn:hover{color:#333;text-decoration:none}.btn.active,.btn:active{background-image:none;outline:0;-webkit-box-shadow:inset 0 3px 5px rgba(0,0,0,.125);box-shadow:inset 0 3px 5px rgba(0,0,0,.125)}.btn.disabled,.btn[disabled],fieldset[disabled] .btn{cursor:not-allowed;filter:alpha(opacity=65);-webkit-box-shadow:none;box-shadow:none;opacity:.65}a.btn.disabled,fieldset[disabled] a.btn{pointer-events:none}.btn-default{color:#333;background-color:#fff;border-color:#ccc}.btn-default.focus,.btn-default:focus{color:#333;background-color:#e6e6e6;border-color:#8c8c8c}.btn-default:hover{color:#333;background-color:#e6e6e6;border-color:#adadad}.btn-default.active,.btn-default:active,.open>.dropdown-toggle.btn-default{color:#333;background-color:#e6e6e6;border-color:#adadad}.btn-default.active.focus,.btn-default.active:focus,.btn-default.active:hover,.btn-default:active.focus,.btn-default:active:focus,.btn-default:active:hover,.open>.dropdown-toggle.btn-default.focus,.open>.dropdown-toggle.btn-default:focus,.open>.dropdown-toggle.btn-default:hover{color:#333;background-color:#d4d4d4;border-color:#8c8c8c}.btn-default.active,.btn-default:active,.open>.dropdown-toggle.btn-default{background-image:none}.btn-default.disabled,.btn-default.disabled.active,.btn-default.disabled.focus,.btn-default.disabled:active,.btn-default.disabled:focus,.btn-default.disabled:hover,.btn-default[disabled],.btn-default[disabled].active,.btn-default[disabled].focus,.btn-default[disabled]:active,.btn-default[disabled]:focus,.btn-default[disabled]:hover,fieldset[disabled] .btn-default,fieldset[disabled] .btn-default.active,fieldset[disabled] .btn-default.focus,fieldset[disabled] .btn-default:active,fieldset[disabled] .btn-default:focus,fieldset[disabled] .btn-default:hover{background-color:#fff;border-color:#ccc}.btn-default .badge{color:#fff;background-color:#333}.btn-primary{color:#fff;background-color:#337ab7;border-color:#2e6da4}.btn-primary.focus,.btn-primary:focus{color:#fff;background-color:#286090;border-color:#122b40}.btn-primary:hover{color:#fff;background-color:#286090;border-color:#204d74}.btn-primary.active,.btn-primary:active,.open>.dropdown-toggle.btn-primary{color:#fff;background-color:#286090;border-color:#204d74}.btn-primary.active.focus,.btn-primary.active:focus,.btn-primary.active:hover,.btn-primary:active.focus,.btn-primary:active:focus,.btn-primary:active:hover,.open>.dropdown-toggle.btn-primary.focus,.open>.dropdown-toggle.btn-primary:focus,.open>.dropdown-toggle.btn-primary:hover{color:#fff;background-color:#204d74;border-color:#122b40}.btn-primary.active,.btn-primary:active,.open>.dropdown-toggle.btn-primary{background-image:none}.btn-primary.disabled,.btn-primary.disabled.active,.btn-primary.disabled.focus,.btn-primary.disabled:active,.btn-primary.disabled:focus,.btn-primary.disabled:hover,.btn-primary[disabled],.btn-primary[disabled].active,.btn-primary[disabled].focus,.btn-primary[disabled]:active,.btn-primary[disabled]:focus,.btn-primary[disabled]:hover,fieldset[disabled] .btn-primary,fieldset[disabled] .btn-primary.active,fieldset[disabled] .btn-primary.focus,fieldset[disabled] .btn-primary:active,fieldset[disabled] .btn-primary:focus,fieldset[disabled] .btn-primary:hover{background-color:#337ab7;border-color:#2e6da4}.btn-primary .badge{color:#337ab7;background-color:#fff}.btn-success{color:#fff;background-color:#5cb85c;border-color:#4cae4c}.btn-success.focus,.btn-success:focus{color:#fff;background-color:#449d44;border-color:#255625}.btn-success:hover{color:#fff;background-color:#449d44;border-color:#398439}.btn-success.active,.btn-success:active,.open>.dropdown-toggle.btn-success{color:#fff;background-color:#449d44;border-color:#398439}.btn-success.active.focus,.btn-success.active:focus,.btn-success.active:hover,.btn-success:active.focus,.btn-success:active:focus,.btn-success:active:hover,.open>.dropdown-toggle.btn-success.focus,.open>.dropdown-toggle.btn-success:focus,.open>.dropdown-toggle.btn-success:hover{color:#fff;background-color:#398439;border-color:#255625}.btn-success.active,.btn-success:active,.open>.dropdown-toggle.btn-success{background-image:none}.btn-success.disabled,.btn-success.disabled.active,.btn-success.disabled.focus,.btn-success.disabled:active,.btn-success.disabled:focus,.btn-success.disabled:hover,.btn-success[disabled],.btn-success[disabled].active,.btn-success[disabled].focus,.btn-success[disabled]:active,.btn-success[disabled]:focus,.btn-success[disabled]:hover,fieldset[disabled] .btn-success,fieldset[disabled] .btn-success.active,fieldset[disabled] .btn-success.focus,fieldset[disabled] .btn-success:active,fieldset[disabled] .btn-success:focus,fieldset[disabled] .btn-success:hover{background-color:#5cb85c;border-color:#4cae4c}.btn-success .badge{color:#5cb85c;background-color:#fff}.btn-info{color:#fff;background-color:#5bc0de;border-color:#46b8da}.btn-info.focus,.btn-info:focus{color:#fff;background-color:#31b0d5;border-color:#1b6d85}.btn-info:hover{color:#fff;background-color:#31b0d5;border-color:#269abc}.btn-info.active,.btn-info:active,.open>.dropdown-toggle.btn-info{color:#fff;background-color:#31b0d5;border-color:#269abc}.btn-info.active.focus,.btn-info.active:focus,.btn-info.active:hover,.btn-info:active.focus,.btn-info:active:focus,.btn-info:active:hover,.open>.dropdown-toggle.btn-info.focus,.open>.dropdown-toggle.btn-info:focus,.open>.dropdown-toggle.btn-info:hover{color:#fff;background-color:#269abc;border-color:#1b6d85}.btn-info.active,.btn-info:active,.open>.dropdown-toggle.btn-info{background-image:none}.btn-info.disabled,.btn-info.disabled.active,.btn-info.disabled.focus,.btn-info.disabled:active,.btn-info.disabled:focus,.btn-info.disabled:hover,.btn-info[disabled],.btn-info[disabled].active,.btn-info[disabled].focus,.btn-info[disabled]:active,.btn-info[disabled]:focus,.btn-info[disabled]:hover,fieldset[disabled] .btn-info,fieldset[disabled] .btn-info.active,fieldset[disabled] .btn-info.focus,fieldset[disabled] .btn-info:active,fieldset[disabled] .btn-info:focus,fieldset[disabled] .btn-info:hover{background-color:#5bc0de;border-color:#46b8da}.btn-info .badge{color:#5bc0de;background-color:#fff}.btn-warning{color:#fff;background-color:#f0ad4e;border-color:#eea236}.btn-warning.focus,.btn-warning:focus{color:#fff;background-color:#ec971f;border-color:#985f0d}.btn-warning:hover{color:#fff;background-color:#ec971f;border-color:#d58512}.btn-warning.active,.btn-warning:active,.open>.dropdown-toggle.btn-warning{color:#fff;background-color:#ec971f;border-color:#d58512}.btn-warning.active.focus,.btn-warning.active:focus,.btn-warning.active:hover,.btn-warning:active.focus,.btn-warning:active:focus,.btn-warning:active:hover,.open>.dropdown-toggle.btn-warning.focus,.open>.dropdown-toggle.btn-warning:focus,.open>.dropdown-toggle.btn-warning:hover{color:#fff;background-color:#d58512;border-color:#985f0d}.btn-warning.active,.btn-warning:active,.open>.dropdown-toggle.btn-warning{background-image:none}.btn-warning.disabled,.btn-warning.disabled.active,.btn-warning.disabled.focus,.btn-warning.disabled:active,.btn-warning.disabled:focus,.btn-warning.disabled:hover,.btn-warning[disabled],.btn-warning[disabled].active,.btn-warning[disabled].focus,.btn-warning[disabled]:active,.btn-warning[disabled]:focus,.btn-warning[disabled]:hover,fieldset[disabled] .btn-warning,fieldset[disabled] .btn-warning.active,fieldset[disabled] .btn-warning.focus,fieldset[disabled] .btn-warning:active,fieldset[disabled] .btn-warning:focus,fieldset[disabled] .btn-warning:hover{background-color:#f0ad4e;border-color:#eea236}.btn-warning .badge{color:#f0ad4e;background-color:#fff}.btn-danger{color:#fff;background-color:#d9534f;border-color:#d43f3a}.btn-danger.focus,.btn-danger:focus{color:#fff;background-color:#c9302c;border-color:#761c19}.btn-danger:hover{color:#fff;background-color:#c9302c;border-color:#ac2925}.btn-danger.active,.btn-danger:active,.open>.dropdown-toggle.btn-danger{color:#fff;background-color:#c9302c;border-color:#ac2925}.btn-danger.active.focus,.btn-danger.active:focus,.btn-danger.active:hover,.btn-danger:active.focus,.btn-danger:active:focus,.btn-danger:active:hover,.open>.dropdown-toggle.btn-danger.focus,.open>.dropdown-toggle.btn-danger:focus,.open>.dropdown-toggle.btn-danger:hover{color:#fff;background-color:#ac2925;border-color:#761c19}.btn-danger.active,.btn-danger:active,.open>.dropdown-toggle.btn-danger{background-image:none}.btn-danger.disabled,.btn-danger.disabled.active,.btn-danger.disabled.focus,.btn-danger.disabled:active,.btn-danger.disabled:focus,.btn-danger.disabled:hover,.btn-danger[disabled],.btn-danger[disabled].active,.btn-danger[disabled].focus,.btn-danger[disabled]:active,.btn-danger[disabled]:focus,.btn-danger[disabled]:hover,fieldset[disabled] .btn-danger,fieldset[disabled] .btn-danger.active,fieldset[disabled] .btn-danger.focus,fieldset[disabled] .btn-danger:active,fieldset[disabled] .btn-danger:focus,fieldset[disabled] .btn-danger:hover{background-color:#d9534f;border-color:#d43f3a}.btn-danger .badge{color:#d9534f;background-color:#fff}.btn-link{font-weight:400;color:#337ab7;border-radius:0}.btn-link,.btn-link.active,.btn-link:active,.btn-link[disabled],fieldset[disabled] .btn-link{background-color:transparent;-webkit-box-shadow:none;box-shadow:none}.btn-link,.btn-link:active,.btn-link:focus,.btn-link:hover{border-color:transparent}.btn-link:focus,.btn-link:hover{color:#23527c;text-decoration:underline;background-color:transparent}.btn-link[disabled]:focus,.btn-link[disabled]:hover,fieldset[disabled] .btn-link:focus,fieldset[disabled] .btn-link:hover{color:#777;text-decoration:none}.btn-group-lg>.btn,.btn-lg{padding:10px 16px;font-size:18px;line-height:1.3333333;border-radius:6px}.btn-group-sm>.btn,.btn-sm{padding:5px 10px;font-size:12px;line-height:1.5;border-radius:3px}.btn-group-xs>.btn,.btn-xs{padding:1px 5px;font-size:12px;line-height:1.5;border-radius:3px}.btn-block{display:block;width:100%}.btn-block+.btn-block{margin-top:5px}input[type=button].btn-block,input[type=reset].btn-block,input[type=submit].btn-block{width:100%}.fade{opacity:0;-webkit-transition:opacity .15s linear;-o-transition:opacity .15s linear;transition:opacity .15s linear}.fade.in{opacity:1}.collapse{display:none}.collapse.in{display:block}tr.collapse.in{display:table-row}tbody.collapse.in{display:table-row-group}.collapsing{position:relative;height:0;overflow:hidden;-webkit-transition-timing-function:ease;-o-transition-timing-function:ease;transition-timing-function:ease;-webkit-transition-duration:.35s;-o-transition-duration:.35s;transition-duration:.35s;-webkit-transition-property:height,visibility;-o-transition-property:height,visibility;transition-property:height,visibility}.caret{display:inline-block;width:0;height:0;margin-left:2px;vertical-align:middle;border-top:4px dashed;border-top:4px solid\9;border-right:4px solid transparent;border-left:4px solid transparent}.dropdown,.dropup{position:relative}.dropdown-toggle:focus{outline:0}.dropdown-menu{position:absolute;top:100%;left:0;z-index:1000;display:none;float:left;min-width:160px;padding:5px 0;margin:2px 0 0;font-size:14px;text-align:left;list-style:none;background-color:#fff;-webkit-background-clip:padding-box;background-clip:padding-box;border:1px solid #ccc;border:1px solid rgba(0,0,0,.15);border-radius:4px;-webkit-box-shadow:0 6px 12px rgba(0,0,0,.175);box-shadow:0 6px 12px rgba(0,0,0,.175)}.dropdown-menu.pull-right{right:0;left:auto}.dropdown-menu .divider{height:1px;margin:9px 0;overflow:hidden;background-color:#e5e5e5}.dropdown-menu>li>a{display:block;padding:3px 20px;clear:both;font-weight:400;line-height:1.42857143;color:#333;white-space:nowrap}.dropdown-menu>li>a:focus,.dropdown-menu>li>a:hover{color:#262626;text-decoration:none;background-color:#f5f5f5}.dropdown-menu>.active>a,.dropdown-menu>.active>a:focus,.dropdown-menu>.active>a:hover{color:#fff;text-decoration:none;background-color:#337ab7;outline:0}.dropdown-menu>.disabled>a,.dropdown-menu>.disabled>a:focus,.dropdown-menu>.disabled>a:hover{color:#777}.dropdown-menu>.disabled>a:focus,.dropdown-menu>.disabled>a:hover{text-decoration:none;cursor:not-allowed;background-color:transparent;background-image:none;filter:progid:DXImageTransform.Microsoft.gradient(enabled=false)}.open>.dropdown-menu{display:block}.open>a{outline:0}.dropdown-menu-right{right:0;left:auto}.dropdown-menu-left{right:auto;left:0}.dropdown-header{display:block;padding:3px 20px;font-size:12px;line-height:1.42857143;color:#777;white-space:nowrap}.dropdown-backdrop{position:fixed;top:0;right:0;bottom:0;left:0;z-index:990}.pull-right>.dropdown-menu{right:0;left:auto}.dropup .caret,.navbar-fixed-bottom .dropdown .caret{content:"";border-top:0;border-bottom:4px dashed;border-bottom:4px solid\9}.dropup .dropdown-menu,.navbar-fixed-bottom .dropdown .dropdown-menu{top:auto;bottom:100%;margin-bottom:2px}@media (min-width:768px){.navbar-right .dropdown-menu{right:0;left:auto}.navbar-right .dropdown-menu-left{right:auto;left:0}}.btn-group,.btn-group-vertical{position:relative;display:inline-block;vertical-align:middle}.btn-group-vertical>.btn,.btn-group>.btn{position:relative;float:left}.btn-group-vertical>.btn.active,.btn-group-vertical>.btn:active,.btn-group-vertical>.btn:focus,.btn-group-vertical>.btn:hover,.btn-group>.btn.active,.btn-group>.btn:active,.btn-group>.btn:focus,.btn-group>.btn:hover{z-index:2}.btn-group .btn+.btn,.btn-group .btn+.btn-group,.btn-group .btn-group+.btn,.btn-group .btn-group+.btn-group{margin-left:-1px}.btn-toolbar{margin-left:-5px}.btn-toolbar .btn,.btn-toolbar .btn-group,.btn-toolbar .input-group{float:left}.btn-toolbar>.btn,.btn-toolbar>.btn-group,.btn-toolbar>.input-group{margin-left:5px}.btn-group>.btn:not(:first-child):not(:last-child):not(.dropdown-toggle){border-radius:0}.btn-group>.btn:first-child{margin-left:0}.btn-group>.btn:first-child:not(:last-child):not(.dropdown-toggle){border-top-right-radius:0;border-bottom-right-radius:0}.btn-group>.btn:last-child:not(:first-child),.btn-group>.dropdown-toggle:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.btn-group>.btn-group{float:left}.btn-group>.btn-group:not(:first-child):not(:last-child)>.btn{border-radius:0}.btn-group>.btn-group:first-child:not(:last-child)>.btn:last-child,.btn-group>.btn-group:first-child:not(:last-child)>.dropdown-toggle{border-top-right-radius:0;border-bottom-right-radius:0}.btn-group>.btn-group:last-child:not(:first-child)>.btn:first-child{border-top-left-radius:0;border-bottom-left-radius:0}.btn-group .dropdown-toggle:active,.btn-group.open .dropdown-toggle{outline:0}.btn-group>.btn+.dropdown-toggle{padding-right:8px;padding-left:8px}.btn-group>.btn-lg+.dropdown-toggle{padding-right:12px;padding-left:12px}.btn-group.open .dropdown-toggle{-webkit-box-shadow:inset 0 3px 5px rgba(0,0,0,.125);box-shadow:inset 0 3px 5px rgba(0,0,0,.125)}.btn-group.open .dropdown-toggle.btn-link{-webkit-box-shadow:none;box-shadow:none}.btn .caret{margin-left:0}.btn-lg .caret{border-width:5px 5px 0;border-bottom-width:0}.dropup .btn-lg .caret{border-width:0 5px 5px}.btn-group-vertical>.btn,.btn-group-vertical>.btn-group,.btn-group-vertical>.btn-group>.btn{display:block;float:none;width:100%;max-width:100%}.btn-group-vertical>.btn-group>.btn{float:none}.btn-group-vertical>.btn+.btn,.btn-group-vertical>.btn+.btn-group,.btn-group-vertical>.btn-group+.btn,.btn-group-vertical>.btn-group+.btn-group{margin-top:-1px;margin-left:0}.btn-group-vertical>.btn:not(:first-child):not(:last-child){border-radius:0}.btn-group-vertical>.btn:first-child:not(:last-child){border-top-right-radius:4px;border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn:last-child:not(:first-child){border-top-left-radius:0;border-top-right-radius:0;border-bottom-left-radius:4px}.btn-group-vertical>.btn-group:not(:first-child):not(:last-child)>.btn{border-radius:0}.btn-group-vertical>.btn-group:first-child:not(:last-child)>.btn:last-child,.btn-group-vertical>.btn-group:first-child:not(:last-child)>.dropdown-toggle{border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn-group:last-child:not(:first-child)>.btn:first-child{border-top-left-radius:0;border-top-right-radius:0}.btn-group-justified{display:table;width:100%;table-layout:fixed;border-collapse:separate}.btn-group-justified>.btn,.btn-group-justified>.btn-group{display:table-cell;float:none;width:1%}.btn-group-justified>.btn-group .btn{width:100%}.btn-group-justified>.btn-group .dropdown-menu{left:auto}[data-toggle=buttons]>.btn input[type=checkbox],[data-toggle=buttons]>.btn input[type=radio],[data-toggle=buttons]>.btn-group>.btn input[type=checkbox],[data-toggle=buttons]>.btn-group>.btn input[type=radio]{position:absolute;clip:rect(0,0,0,0);pointer-events:none}.input-group{position:relative;display:table;border-collapse:separate}.input-group[class*=col-]{float:none;padding-right:0;padding-left:0}.input-group .form-control{position:relative;z-index:2;float:left;width:100%;margin-bottom:0}.input-group-lg>.form-control,.input-group-lg>.input-group-addon,.input-group-lg>.input-group-btn>.btn{height:46px;padding:10px 16px;font-size:18px;line-height:1.3333333;border-radius:6px}select.input-group-lg>.form-control,select.input-group-lg>.input-group-addon,select.input-group-lg>.input-group-btn>.btn{height:46px;line-height:46px}select[multiple].input-group-lg>.form-control,select[multiple].input-group-lg>.input-group-addon,select[multiple].input-group-lg>.input-group-btn>.btn,textarea.input-group-lg>.form-control,textarea.input-group-lg>.input-group-addon,textarea.input-group-lg>.input-group-btn>.btn{height:auto}.input-group-sm>.form-control,.input-group-sm>.input-group-addon,.input-group-sm>.input-group-btn>.btn{height:30px;padding:5px 10px;font-size:12px;line-height:1.5;border-radius:3px}select.input-group-sm>.form-control,select.input-group-sm>.input-group-addon,select.input-group-sm>.input-group-btn>.btn{height:30px;line-height:30px}select[multiple].input-group-sm>.form-control,select[multiple].input-group-sm>.input-group-addon,select[multiple].input-group-sm>.input-group-btn>.btn,textarea.input-group-sm>.form-control,textarea.input-group-sm>.input-group-addon,textarea.input-group-sm>.input-group-btn>.btn{height:auto}.input-group .form-control,.input-group-addon,.input-group-btn{display:table-cell}.input-group .form-control:not(:first-child):not(:last-child),.input-group-addon:not(:first-child):not(:last-child),.input-group-btn:not(:first-child):not(:last-child){border-radius:0}.input-group-addon,.input-group-btn{width:1%;white-space:nowrap;vertical-align:middle}.input-group-addon{padding:6px 12px;font-size:14px;font-weight:400;line-height:1;color:#555;text-align:center;background-color:#eee;border:1px solid #ccc;border-radius:4px}.input-group-addon.input-sm{padding:5px 10px;font-size:12px;border-radius:3px}.input-group-addon.input-lg{padding:10px 16px;font-size:18px;border-radius:6px}.input-group-addon input[type=checkbox],.input-group-addon input[type=radio]{margin-top:0}.input-group .form-control:first-child,.input-group-addon:first-child,.input-group-btn:first-child>.btn,.input-group-btn:first-child>.btn-group>.btn,.input-group-btn:first-child>.dropdown-toggle,.input-group-btn:last-child>.btn-group:not(:last-child)>.btn,.input-group-btn:last-child>.btn:not(:last-child):not(.dropdown-toggle){border-top-right-radius:0;border-bottom-right-radius:0}.input-group-addon:first-child{border-right:0}.input-group .form-control:last-child,.input-group-addon:last-child,.input-group-btn:first-child>.btn-group:not(:first-child)>.btn,.input-group-btn:first-child>.btn:not(:first-child),.input-group-btn:last-child>.btn,.input-group-btn:last-child>.btn-group>.btn,.input-group-btn:last-child>.dropdown-toggle{border-top-left-radius:0;border-bottom-left-radius:0}.input-group-addon:last-child{border-left:0}.input-group-btn{position:relative;font-size:0;white-space:nowrap}.input-group-btn>.btn{position:relative}.input-group-btn>.btn+.btn{margin-left:-1px}.input-group-btn>.btn:active,.input-group-btn>.btn:focus,.input-group-btn>.btn:hover{z-index:2}.input-group-btn:first-child>.btn,.input-group-btn:first-child>.btn-group{margin-right:-1px}.input-group-btn:last-child>.btn,.input-group-btn:last-child>.btn-group{z-index:2;margin-left:-1px}.nav{padding-left:0;margin-bottom:0;list-style:none}.nav>li{position:relative;display:block}.nav>li>a{position:relative;display:block;padding:10px 15px}.nav>li>a:focus,.nav>li>a:hover{text-decoration:none;background-color:#eee}.nav>li.disabled>a{color:#777}.nav>li.disabled>a:focus,.nav>li.disabled>a:hover{color:#777;text-decoration:none;cursor:not-allowed;background-color:transparent}.nav .open>a,.nav .open>a:focus,.nav .open>a:hover{background-color:#eee;border-color:#337ab7}.nav .nav-divider{height:1px;margin:9px 0;overflow:hidden;background-color:#e5e5e5}.nav>li>a>img{max-width:none}.nav-tabs{border-bottom:1px solid #ddd}.nav-tabs>li{float:left;margin-bottom:-1px}.nav-tabs>li>a{margin-right:2px;line-height:1.42857143;border:1px solid transparent;border-radius:4px 4px 0 0}.nav-tabs>li>a:hover{border-color:#eee #eee #ddd}.nav-tabs>li.active>a,.nav-tabs>li.active>a:focus,.nav-tabs>li.active>a:hover{color:#555;cursor:default;background-color:#fff;border:1px solid #ddd;border-bottom-color:transparent}.nav-tabs.nav-justified{width:100%;border-bottom:0}.nav-tabs.nav-justified>li{float:none}.nav-tabs.nav-justified>li>a{margin-bottom:5px;text-align:center}.nav-tabs.nav-justified>.dropdown .dropdown-menu{top:auto;left:auto}@media (min-width:768px){.nav-tabs.nav-justified>li{display:table-cell;width:1%}.nav-tabs.nav-justified>li>a{margin-bottom:0}}.nav-tabs.nav-justified>li>a{margin-right:0;border-radius:4px}.nav-tabs.nav-justified>.active>a,.nav-tabs.nav-justified>.active>a:focus,.nav-tabs.nav-justified>.active>a:hover{border:1px solid #ddd}@media (min-width:768px){.nav-tabs.nav-justified>li>a{border-bottom:1px solid #ddd;border-radius:4px 4px 0 0}.nav-tabs.nav-justified>.active>a,.nav-tabs.nav-justified>.active>a:focus,.nav-tabs.nav-justified>.active>a:hover{border-bottom-color:#fff}}.nav-pills>li{float:left}.nav-pills>li>a{border-radius:4px}.nav-pills>li+li{margin-left:2px}.nav-pills>li.active>a,.nav-pills>li.active>a:focus,.nav-pills>li.active>a:hover{color:#fff;background-color:#337ab7}.nav-stacked>li{float:none}.nav-stacked>li+li{margin-top:2px;margin-left:0}.nav-justified{width:100%}.nav-justified>li{float:none}.nav-justified>li>a{margin-bottom:5px;text-align:center}.nav-justified>.dropdown .dropdown-menu{top:auto;left:auto}@media (min-width:768px){.nav-justified>li{display:table-cell;width:1%}.nav-justified>li>a{margin-bottom:0}}.nav-tabs-justified{border-bottom:0}.nav-tabs-justified>li>a{margin-right:0;border-radius:4px}.nav-tabs-justified>.active>a,.nav-tabs-justified>.active>a:focus,.nav-tabs-justified>.active>a:hover{border:1px solid #ddd}@media (min-width:768px){.nav-tabs-justified>li>a{border-bottom:1px solid #ddd;border-radius:4px 4px 0 0}.nav-tabs-justified>.active>a,.nav-tabs-justified>.active>a:focus,.nav-tabs-justified>.active>a:hover{border-bottom-color:#fff}}.tab-content>.tab-pane{display:none}.tab-content>.active{display:block}.nav-tabs .dropdown-menu{margin-top:-1px;border-top-left-radius:0;border-top-right-radius:0}.navbar{position:relative;min-height:50px;margin-bottom:20px;border:1px solid transparent}@media (min-width:768px){.navbar{border-radius:4px}}@media (min-width:768px){.navbar-header{float:left}}.navbar-collapse{padding-right:15px;padding-left:15px;overflow-x:visible;-webkit-overflow-scrolling:touch;border-top:1px solid transparent;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,.1);box-shadow:inset 0 1px 0 rgba(255,255,255,.1)}.navbar-collapse.in{overflow-y:auto}@media (min-width:768px){.navbar-collapse{width:auto;border-top:0;-webkit-box-shadow:none;box-shadow:none}.navbar-collapse.collapse{display:block!important;height:auto!important;padding-bottom:0;overflow:visible!important}.navbar-collapse.in{overflow-y:visible}.navbar-fixed-bottom .navbar-collapse,.navbar-fixed-top .navbar-collapse,.navbar-static-top .navbar-collapse{padding-right:0;padding-left:0}}.navbar-fixed-bottom .navbar-collapse,.navbar-fixed-top .navbar-collapse{max-height:340px}@media (max-device-width:480px) and (orientation:landscape){.navbar-fixed-bottom .navbar-collapse,.navbar-fixed-top .navbar-collapse{max-height:200px}}.container-fluid>.navbar-collapse,.container-fluid>.navbar-header,.container>.navbar-collapse,.container>.navbar-header{margin-right:-15px;margin-left:-15px}@media (min-width:768px){.container-fluid>.navbar-collapse,.container-fluid>.navbar-header,.container>.navbar-collapse,.container>.navbar-header{margin-right:0;margin-left:0}}.navbar-static-top{z-index:1000;border-width:0 0 1px}@media (min-width:768px){.navbar-static-top{border-radius:0}}.navbar-fixed-bottom,.navbar-fixed-top{position:fixed;right:0;left:0;z-index:1030}@media (min-width:768px){.navbar-fixed-bottom,.navbar-fixed-top{border-radius:0}}.navbar-fixed-top{top:0;border-width:0 0 1px}.navbar-fixed-bottom{bottom:0;margin-bottom:0;border-width:1px 0 0}.navbar-brand{float:left;height:50px;padding:15px 15px;font-size:18px;line-height:20px}.navbar-brand:focus,.navbar-brand:hover{text-decoration:none}.navbar-brand>img{display:block}@media (min-width:768px){.navbar>.container .navbar-brand,.navbar>.container-fluid .navbar-brand{margin-left:-15px}}.navbar-toggle{position:relative;float:right;padding:9px 10px;margin-top:8px;margin-right:15px;margin-bottom:8px;background-color:transparent;background-image:none;border:1px solid transparent;border-radius:4px}.navbar-toggle:focus{outline:0}.navbar-toggle .icon-bar{display:block;width:22px;height:2px;border-radius:1px}.navbar-toggle .icon-bar+.icon-bar{margin-top:4px}@media (min-width:768px){.navbar-toggle{display:none}}.navbar-nav{margin:7.5px -15px}.navbar-nav>li>a{padding-top:10px;padding-bottom:10px;line-height:20px}@media (max-width:767px){.navbar-nav .open .dropdown-menu{position:static;float:none;width:auto;margin-top:0;background-color:transparent;border:0;-webkit-box-shadow:none;box-shadow:none}.navbar-nav .open .dropdown-menu .dropdown-header,.navbar-nav .open .dropdown-menu>li>a{padding:5px 15px 5px 25px}.navbar-nav .open .dropdown-menu>li>a{line-height:20px}.navbar-nav .open .dropdown-menu>li>a:focus,.navbar-nav .open .dropdown-menu>li>a:hover{background-image:none}}@media (min-width:768px){.navbar-nav{float:left;margin:0}.navbar-nav>li{float:left}.navbar-nav>li>a{padding-top:15px;padding-bottom:15px}}.navbar-form{padding:10px 15px;margin-top:8px;margin-right:-15px;margin-bottom:8px;margin-left:-15px;border-top:1px solid transparent;border-bottom:1px solid transparent;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,.1),0 1px 0 rgba(255,255,255,.1);box-shadow:inset 0 1px 0 rgba(255,255,255,.1),0 1px 0 rgba(255,255,255,.1)}@media (min-width:768px){.navbar-form .form-group{display:inline-block;margin-bottom:0;vertical-align:middle}.navbar-form .form-control{display:inline-block;width:auto;vertical-align:middle}.navbar-form .form-control-static{display:inline-block}.navbar-form .input-group{display:inline-table;vertical-align:middle}.navbar-form .input-group .form-control,.navbar-form .input-group .input-group-addon,.navbar-form .input-group .input-group-btn{width:auto}.navbar-form .input-group>.form-control{width:100%}.navbar-form .control-label{margin-bottom:0;vertical-align:middle}.navbar-form .checkbox,.navbar-form .radio{display:inline-block;margin-top:0;margin-bottom:0;vertical-align:middle}.navbar-form .checkbox label,.navbar-form .radio label{padding-left:0}.navbar-form .checkbox input[type=checkbox],.navbar-form .radio input[type=radio]{position:relative;margin-left:0}.navbar-form .has-feedback .form-control-feedback{top:0}}@media (max-width:767px){.navbar-form .form-group{margin-bottom:5px}.navbar-form .form-group:last-child{margin-bottom:0}}@media (min-width:768px){.navbar-form{width:auto;padding-top:0;padding-bottom:0;margin-right:0;margin-left:0;border:0;-webkit-box-shadow:none;box-shadow:none}}.navbar-nav>li>.dropdown-menu{margin-top:0;border-top-left-radius:0;border-top-right-radius:0}.navbar-fixed-bottom .navbar-nav>li>.dropdown-menu{margin-bottom:0;border-top-left-radius:4px;border-top-right-radius:4px;border-bottom-right-radius:0;border-bottom-left-radius:0}.navbar-btn{margin-top:8px;margin-bottom:8px}.navbar-btn.btn-sm{margin-top:10px;margin-bottom:10px}.navbar-btn.btn-xs{margin-top:14px;margin-bottom:14px}.navbar-text{margin-top:15px;margin-bottom:15px}@media (min-width:768px){.navbar-text{float:left;margin-right:15px;margin-left:15px}}@media (min-width:768px){.navbar-left{float:left!important}.navbar-right{float:right!important;margin-right:-15px}.navbar-right~.navbar-right{margin-right:0}}.navbar-default{background-color:#f8f8f8;border-color:#e7e7e7}.navbar-default .navbar-brand{color:#777}.navbar-default .navbar-brand:focus,.navbar-default .navbar-brand:hover{color:#5e5e5e;background-color:transparent}.navbar-default .navbar-text{color:#777}.navbar-default .navbar-nav>li>a{color:#777}.navbar-default .navbar-nav>li>a:focus,.navbar-default .navbar-nav>li>a:hover{color:#333;background-color:transparent}.navbar-default .navbar-nav>.active>a,.navbar-default .navbar-nav>.active>a:focus,.navbar-default .navbar-nav>.active>a:hover{color:#555;background-color:#e7e7e7}.navbar-default .navbar-nav>.disabled>a,.navbar-default .navbar-nav>.disabled>a:focus,.navbar-default .navbar-nav>.disabled>a:hover{color:#ccc;background-color:transparent}.navbar-default .navbar-toggle{border-color:#ddd}.navbar-default .navbar-toggle:focus,.navbar-default .navbar-toggle:hover{background-color:#ddd}.navbar-default .navbar-toggle .icon-bar{background-color:#888}.navbar-default .navbar-collapse,.navbar-default .navbar-form{border-color:#e7e7e7}.navbar-default .navbar-nav>.open>a,.navbar-default .navbar-nav>.open>a:focus,.navbar-default .navbar-nav>.open>a:hover{color:#555;background-color:#e7e7e7}@media (max-width:767px){.navbar-default .navbar-nav .open .dropdown-menu>li>a{color:#777}.navbar-default .navbar-nav .open .dropdown-menu>li>a:focus,.navbar-default .navbar-nav .open .dropdown-menu>li>a:hover{color:#333;background-color:transparent}.navbar-default .navbar-nav .open .dropdown-menu>.active>a,.navbar-default .navbar-nav .open .dropdown-menu>.active>a:focus,.navbar-default .navbar-nav .open .dropdown-menu>.active>a:hover{color:#555;background-color:#e7e7e7}.navbar-default .navbar-nav .open .dropdown-menu>.disabled>a,.navbar-default .navbar-nav .open .dropdown-menu>.disabled>a:focus,.navbar-default .navbar-nav .open .dropdown-menu>.disabled>a:hover{color:#ccc;background-color:transparent}}.navbar-default .navbar-link{color:#777}.navbar-default .navbar-link:hover{color:#333}.navbar-default .btn-link{color:#777}.navbar-default .btn-link:focus,.navbar-default .btn-link:hover{color:#333}.navbar-default .btn-link[disabled]:focus,.navbar-default .btn-link[disabled]:hover,fieldset[disabled] .navbar-default .btn-link:focus,fieldset[disabled] .navbar-default .btn-link:hover{color:#ccc}.navbar-inverse{background-color:#222;border-color:#080808}.navbar-inverse .navbar-brand{color:#9d9d9d}.navbar-inverse .navbar-brand:focus,.navbar-inverse .navbar-brand:hover{color:#fff;background-color:transparent}.navbar-inverse .navbar-text{color:#9d9d9d}.navbar-inverse .navbar-nav>li>a{color:#9d9d9d}.navbar-inverse .navbar-nav>li>a:focus,.navbar-inverse .navbar-nav>li>a:hover{color:#fff;background-color:transparent}.navbar-inverse .navbar-nav>.active>a,.navbar-inverse .navbar-nav>.active>a:focus,.navbar-inverse .navbar-nav>.active>a:hover{color:#fff;background-color:#080808}.navbar-inverse .navbar-nav>.disabled>a,.navbar-inverse .navbar-nav>.disabled>a:focus,.navbar-inverse .navbar-nav>.disabled>a:hover{color:#444;background-color:transparent}.navbar-inverse .navbar-toggle{border-color:#333}.navbar-inverse .navbar-toggle:focus,.navbar-inverse .navbar-toggle:hover{background-color:#333}.navbar-inverse .navbar-toggle .icon-bar{background-color:#fff}.navbar-inverse .navbar-collapse,.navbar-inverse .navbar-form{border-color:#101010}.navbar-inverse .navbar-nav>.open>a,.navbar-inverse .navbar-nav>.open>a:focus,.navbar-inverse .navbar-nav>.open>a:hover{color:#fff;background-color:#080808}@media (max-width:767px){.navbar-inverse .navbar-nav .open .dropdown-menu>.dropdown-header{border-color:#080808}.navbar-inverse .navbar-nav .open .dropdown-menu .divider{background-color:#080808}.navbar-inverse .navbar-nav .open .dropdown-menu>li>a{color:#9d9d9d}.navbar-inverse .navbar-nav .open .dropdown-menu>li>a:focus,.navbar-inverse .navbar-nav .open .dropdown-menu>li>a:hover{color:#fff;background-color:transparent}.navbar-inverse .navbar-nav .open .dropdown-menu>.active>a,.navbar-inverse .navbar-nav .open .dropdown-menu>.active>a:focus,.navbar-inverse .navbar-nav .open .dropdown-menu>.active>a:hover{color:#fff;background-color:#080808}.navbar-inverse .navbar-nav .open .dropdown-menu>.disabled>a,.navbar-inverse .navbar-nav .open .dropdown-menu>.disabled>a:focus,.navbar-inverse .navbar-nav .open .dropdown-menu>.disabled>a:hover{color:#444;background-color:transparent}}.navbar-inverse .navbar-link{color:#9d9d9d}.navbar-inverse .navbar-link:hover{color:#fff}.navbar-inverse .btn-link{color:#9d9d9d}.navbar-inverse .btn-link:focus,.navbar-inverse .btn-link:hover{color:#fff}.navbar-inverse .btn-link[disabled]:focus,.navbar-inverse .btn-link[disabled]:hover,fieldset[disabled] .navbar-inverse .btn-link:focus,fieldset[disabled] .navbar-inverse .btn-link:hover{color:#444}.breadcrumb{padding:8px 15px;margin-bottom:20px;list-style:none;background-color:#f5f5f5;border-radius:4px}.breadcrumb>li{display:inline-block}.breadcrumb>li+li:before{padding:0 5px;color:#ccc;content:"/\00a0"}.breadcrumb>.active{color:#777}.pagination{display:inline-block;padding-left:0;margin:20px 0;border-radius:4px}.pagination>li{display:inline}.pagination>li>a,.pagination>li>span{position:relative;float:left;padding:6px 12px;margin-left:-1px;line-height:1.42857143;color:#337ab7;text-decoration:none;background-color:#fff;border:1px solid #ddd}.pagination>li:first-child>a,.pagination>li:first-child>span{margin-left:0;border-top-left-radius:4px;border-bottom-left-radius:4px}.pagination>li:last-child>a,.pagination>li:last-child>span{border-top-right-radius:4px;border-bottom-right-radius:4px}.pagination>li>a:focus,.pagination>li>a:hover,.pagination>li>span:focus,.pagination>li>span:hover{z-index:3;color:#23527c;background-color:#eee;border-color:#ddd}.pagination>.active>a,.pagination>.active>a:focus,.pagination>.active>a:hover,.pagination>.active>span,.pagination>.active>span:focus,.pagination>.active>span:hover{z-index:2;color:#fff;cursor:default;background-color:#337ab7;border-color:#337ab7}.pagination>.disabled>a,.pagination>.disabled>a:focus,.pagination>.disabled>a:hover,.pagination>.disabled>span,.pagination>.disabled>span:focus,.pagination>.disabled>span:hover{color:#777;cursor:not-allowed;background-color:#fff;border-color:#ddd}.pagination-lg>li>a,.pagination-lg>li>span{padding:10px 16px;font-size:18px;line-height:1.3333333}.pagination-lg>li:first-child>a,.pagination-lg>li:first-child>span{border-top-left-radius:6px;border-bottom-left-radius:6px}.pagination-lg>li:last-child>a,.pagination-lg>li:last-child>span{border-top-right-radius:6px;border-bottom-right-radius:6px}.pagination-sm>li>a,.pagination-sm>li>span{padding:5px 10px;font-size:12px;line-height:1.5}.pagination-sm>li:first-child>a,.pagination-sm>li:first-child>span{border-top-left-radius:3px;border-bottom-left-radius:3px}.pagination-sm>li:last-child>a,.pagination-sm>li:last-child>span{border-top-right-radius:3px;border-bottom-right-radius:3px}.pager{padding-left:0;margin:20px 0;text-align:center;list-style:none}.pager li{display:inline}.pager li>a,.pager li>span{display:inline-block;padding:5px 14px;background-color:#fff;border:1px solid #ddd;border-radius:15px}.pager li>a:focus,.pager li>a:hover{text-decoration:none;background-color:#eee}.pager .next>a,.pager .next>span{float:right}.pager .previous>a,.pager .previous>span{float:left}.pager .disabled>a,.pager .disabled>a:focus,.pager .disabled>a:hover,.pager .disabled>span{color:#777;cursor:not-allowed;background-color:#fff}.label{display:inline;padding:.2em .6em .3em;font-size:75%;font-weight:700;line-height:1;color:#fff;text-align:center;white-space:nowrap;vertical-align:baseline;border-radius:.25em}a.label:focus,a.label:hover{color:#fff;text-decoration:none;cursor:pointer}.label:empty{display:none}.btn .label{position:relative;top:-1px}.label-default{background-color:#777}.label-default[href]:focus,.label-default[href]:hover{background-color:#5e5e5e}.label-primary{background-color:#337ab7}.label-primary[href]:focus,.label-primary[href]:hover{background-color:#286090}.label-success{background-color:#5cb85c}.label-success[href]:focus,.label-success[href]:hover{background-color:#449d44}.label-info{background-color:#5bc0de}.label-info[href]:focus,.label-info[href]:hover{background-color:#31b0d5}.label-warning{background-color:#f0ad4e}.label-warning[href]:focus,.label-warning[href]:hover{background-color:#ec971f}.label-danger{background-color:#d9534f}.label-danger[href]:focus,.label-danger[href]:hover{background-color:#c9302c}.badge{display:inline-block;min-width:10px;padding:3px 7px;font-size:12px;font-weight:700;line-height:1;color:#fff;text-align:center;white-space:nowrap;vertical-align:middle;background-color:#777;border-radius:10px}.badge:empty{display:none}.btn .badge{position:relative;top:-1px}.btn-group-xs>.btn .badge,.btn-xs .badge{top:0;padding:1px 5px}a.badge:focus,a.badge:hover{color:#fff;text-decoration:none;cursor:pointer}.list-group-item.active>.badge,.nav-pills>.active>a>.badge{color:#337ab7;background-color:#fff}.list-group-item>.badge{float:right}.list-group-item>.badge+.badge{margin-right:5px}.nav-pills>li>a>.badge{margin-left:3px}.jumbotron{padding-top:30px;padding-bottom:30px;margin-bottom:30px;color:inherit;background-color:#eee}.jumbotron .h1,.jumbotron h1{color:inherit}.jumbotron p{margin-bottom:15px;font-size:21px;font-weight:200}.jumbotron>hr{border-top-color:#d5d5d5}.container .jumbotron,.container-fluid .jumbotron{border-radius:6px}.jumbotron .container{max-width:100%}@media screen and (min-width:768px){.jumbotron{padding-top:48px;padding-bottom:48px}.container .jumbotron,.container-fluid .jumbotron{padding-right:60px;padding-left:60px}.jumbotron .h1,.jumbotron h1{font-size:63px}}.thumbnail{display:block;padding:4px;margin-bottom:20px;line-height:1.42857143;background-color:#fff;border:1px solid #ddd;border-radius:4px;-webkit-transition:border .2s ease-in-out;-o-transition:border .2s ease-in-out;transition:border .2s ease-in-out}.thumbnail a>img,.thumbnail>img{margin-right:auto;margin-left:auto}a.thumbnail.active,a.thumbnail:focus,a.thumbnail:hover{border-color:#337ab7}.thumbnail .caption{padding:9px;color:#333}.alert{padding:15px;margin-bottom:20px;border:1px solid transparent;border-radius:4px}.alert h4{margin-top:0;color:inherit}.alert .alert-link{font-weight:700}.alert>p,.alert>ul{margin-bottom:0}.alert>p+p{margin-top:5px}.alert-dismissable,.alert-dismissible{padding-right:35px}.alert-dismissable .close,.alert-dismissible .close{position:relative;top:-2px;right:-21px;color:inherit}.alert-success{color:#3c763d;background-color:#dff0d8;border-color:#d6e9c6}.alert-success hr{border-top-color:#c9e2b3}.alert-success .alert-link{color:#2b542c}.alert-info{color:#31708f;background-color:#d9edf7;border-color:#bce8f1}.alert-info hr{border-top-color:#a6e1ec}.alert-info .alert-link{color:#245269}.alert-warning{color:#8a6d3b;background-color:#fcf8e3;border-color:#faebcc}.alert-warning hr{border-top-color:#f7e1b5}.alert-warning .alert-link{color:#66512c}.alert-danger{color:#a94442;background-color:#f2dede;border-color:#ebccd1}.alert-danger hr{border-top-color:#e4b9c0}.alert-danger .alert-link{color:#843534}@-webkit-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@-o-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}.progress{height:20px;margin-bottom:20px;overflow:hidden;background-color:#f5f5f5;border-radius:4px;-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,.1);box-shadow:inset 0 1px 2px rgba(0,0,0,.1)}.progress-bar{float:left;width:0;height:100%;font-size:12px;line-height:20px;color:#fff;text-align:center;background-color:#337ab7;-webkit-box-shadow:inset 0 -1px 0 rgba(0,0,0,.15);box-shadow:inset 0 -1px 0 rgba(0,0,0,.15);-webkit-transition:width .6s ease;-o-transition:width .6s ease;transition:width .6s ease}.progress-bar-striped,.progress-striped .progress-bar{background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);-webkit-background-size:40px 40px;background-size:40px 40px}.progress-bar.active,.progress.active .progress-bar{-webkit-animation:progress-bar-stripes 2s linear infinite;-o-animation:progress-bar-stripes 2s linear infinite;animation:progress-bar-stripes 2s linear infinite}.progress-bar-success{background-color:#5cb85c}.progress-striped .progress-bar-success{background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent)}.progress-bar-info{background-color:#5bc0de}.progress-striped .progress-bar-info{background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent)}.progress-bar-warning{background-color:#f0ad4e}.progress-striped .progress-bar-warning{background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent)}.progress-bar-danger{background-color:#d9534f}.progress-striped .progress-bar-danger{background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent)}.media{margin-top:15px}.media:first-child{margin-top:0}.media,.media-body{overflow:hidden;zoom:1}.media-body{width:10000px}.media-object{display:block}.media-object.img-thumbnail{max-width:none}.media-right,.media>.pull-right{padding-left:10px}.media-left,.media>.pull-left{padding-right:10px}.media-body,.media-left,.media-right{display:table-cell;vertical-align:top}.media-middle{vertical-align:middle}.media-bottom{vertical-align:bottom}.media-heading{margin-top:0;margin-bottom:5px}.media-list{padding-left:0;list-style:none}.list-group{padding-left:0;margin-bottom:20px}.list-group-item{position:relative;display:block;padding:10px 15px;margin-bottom:-1px;background-color:#fff;border:1px solid #ddd}.list-group-item:first-child{border-top-left-radius:4px;border-top-right-radius:4px}.list-group-item:last-child{margin-bottom:0;border-bottom-right-radius:4px;border-bottom-left-radius:4px}a.list-group-item,button.list-group-item{color:#555}a.list-group-item .list-group-item-heading,button.list-group-item .list-group-item-heading{color:#333}a.list-group-item:focus,a.list-group-item:hover,button.list-group-item:focus,button.list-group-item:hover{color:#555;text-decoration:none;background-color:#f5f5f5}button.list-group-item{width:100%;text-align:left}.list-group-item.disabled,.list-group-item.disabled:focus,.list-group-item.disabled:hover{color:#777;cursor:not-allowed;background-color:#eee}.list-group-item.disabled .list-group-item-heading,.list-group-item.disabled:focus .list-group-item-heading,.list-group-item.disabled:hover .list-group-item-heading{color:inherit}.list-group-item.disabled .list-group-item-text,.list-group-item.disabled:focus .list-group-item-text,.list-group-item.disabled:hover .list-group-item-text{color:#777}.list-group-item.active,.list-group-item.active:focus,.list-group-item.active:hover{z-index:2;color:#fff;background-color:#337ab7;border-color:#337ab7}.list-group-item.active .list-group-item-heading,.list-group-item.active .list-group-item-heading>.small,.list-group-item.active .list-group-item-heading>small,.list-group-item.active:focus .list-group-item-heading,.list-group-item.active:focus .list-group-item-heading>.small,.list-group-item.active:focus .list-group-item-heading>small,.list-group-item.active:hover .list-group-item-heading,.list-group-item.active:hover .list-group-item-heading>.small,.list-group-item.active:hover .list-group-item-heading>small{color:inherit}.list-group-item.active .list-group-item-text,.list-group-item.active:focus .list-group-item-text,.list-group-item.active:hover .list-group-item-text{color:#c7ddef}.list-group-item-success{color:#3c763d;background-color:#dff0d8}a.list-group-item-success,button.list-group-item-success{color:#3c763d}a.list-group-item-success .list-group-item-heading,button.list-group-item-success .list-group-item-heading{color:inherit}a.list-group-item-success:focus,a.list-group-item-success:hover,button.list-group-item-success:focus,button.list-group-item-success:hover{color:#3c763d;background-color:#d0e9c6}a.list-group-item-success.active,a.list-group-item-success.active:focus,a.list-group-item-success.active:hover,button.list-group-item-success.active,button.list-group-item-success.active:focus,button.list-group-item-success.active:hover{color:#fff;background-color:#3c763d;border-color:#3c763d}.list-group-item-info{color:#31708f;background-color:#d9edf7}a.list-group-item-info,button.list-group-item-info{color:#31708f}a.list-group-item-info .list-group-item-heading,button.list-group-item-info .list-group-item-heading{color:inherit}a.list-group-item-info:focus,a.list-group-item-info:hover,button.list-group-item-info:focus,button.list-group-item-info:hover{color:#31708f;background-color:#c4e3f3}a.list-group-item-info.active,a.list-group-item-info.active:focus,a.list-group-item-info.active:hover,button.list-group-item-info.active,button.list-group-item-info.active:focus,button.list-group-item-info.active:hover{color:#fff;background-color:#31708f;border-color:#31708f}.list-group-item-warning{color:#8a6d3b;background-color:#fcf8e3}a.list-group-item-warning,button.list-group-item-warning{color:#8a6d3b}a.list-group-item-warning .list-group-item-heading,button.list-group-item-warning .list-group-item-heading{color:inherit}a.list-group-item-warning:focus,a.list-group-item-warning:hover,button.list-group-item-warning:focus,button.list-group-item-warning:hover{color:#8a6d3b;background-color:#faf2cc}a.list-group-item-warning.active,a.list-group-item-warning.active:focus,a.list-group-item-warning.active:hover,button.list-group-item-warning.active,button.list-group-item-warning.active:focus,button.list-group-item-warning.active:hover{color:#fff;background-color:#8a6d3b;border-color:#8a6d3b}.list-group-item-danger{color:#a94442;background-color:#f2dede}a.list-group-item-danger,button.list-group-item-danger{color:#a94442}a.list-group-item-danger .list-group-item-heading,button.list-group-item-danger .list-group-item-heading{color:inherit}a.list-group-item-danger:focus,a.list-group-item-danger:hover,button.list-group-item-danger:focus,button.list-group-item-danger:hover{color:#a94442;background-color:#ebcccc}a.list-group-item-danger.active,a.list-group-item-danger.active:focus,a.list-group-item-danger.active:hover,button.list-group-item-danger.active,button.list-group-item-danger.active:focus,button.list-group-item-danger.active:hover{color:#fff;background-color:#a94442;border-color:#a94442}.list-group-item-heading{margin-top:0;margin-bottom:5px}.list-group-item-text{margin-bottom:0;line-height:1.3}.panel{margin-bottom:20px;background-color:#fff;border:1px solid transparent;border-radius:4px;-webkit-box-shadow:0 1px 1px rgba(0,0,0,.05);box-shadow:0 1px 1px rgba(0,0,0,.05)}.panel-body{padding:15px}.panel-heading{padding:10px 15px;border-bottom:1px solid transparent;border-top-left-radius:3px;border-top-right-radius:3px}.panel-heading>.dropdown .dropdown-toggle{color:inherit}.panel-title{margin-top:0;margin-bottom:0;font-size:16px;color:inherit}.panel-title>.small,.panel-title>.small>a,.panel-title>a,.panel-title>small,.panel-title>small>a{color:inherit}.panel-footer{padding:10px 15px;background-color:#f5f5f5;border-top:1px solid #ddd;border-bottom-right-radius:3px;border-bottom-left-radius:3px}.panel>.list-group,.panel>.panel-collapse>.list-group{margin-bottom:0}.panel>.list-group .list-group-item,.panel>.panel-collapse>.list-group .list-group-item{border-width:1px 0;border-radius:0}.panel>.list-group:first-child .list-group-item:first-child,.panel>.panel-collapse>.list-group:first-child .list-group-item:first-child{border-top:0;border-top-left-radius:3px;border-top-right-radius:3px}.panel>.list-group:last-child .list-group-item:last-child,.panel>.panel-collapse>.list-group:last-child .list-group-item:last-child{border-bottom:0;border-bottom-right-radius:3px;border-bottom-left-radius:3px}.panel>.panel-heading+.panel-collapse>.list-group .list-group-item:first-child{border-top-left-radius:0;border-top-right-radius:0}.panel-heading+.list-group .list-group-item:first-child{border-top-width:0}.list-group+.panel-footer{border-top-width:0}.panel>.panel-collapse>.table,.panel>.table,.panel>.table-responsive>.table{margin-bottom:0}.panel>.panel-collapse>.table caption,.panel>.table caption,.panel>.table-responsive>.table caption{padding-right:15px;padding-left:15px}.panel>.table-responsive:first-child>.table:first-child,.panel>.table:first-child{border-top-left-radius:3px;border-top-right-radius:3px}.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child,.panel>.table:first-child>tbody:first-child>tr:first-child,.panel>.table:first-child>thead:first-child>tr:first-child{border-top-left-radius:3px;border-top-right-radius:3px}.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child td:first-child,.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child th:first-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child td:first-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child th:first-child,.panel>.table:first-child>tbody:first-child>tr:first-child td:first-child,.panel>.table:first-child>tbody:first-child>tr:first-child th:first-child,.panel>.table:first-child>thead:first-child>tr:first-child td:first-child,.panel>.table:first-child>thead:first-child>tr:first-child th:first-child{border-top-left-radius:3px}.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child td:last-child,.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child th:last-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child td:last-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child th:last-child,.panel>.table:first-child>tbody:first-child>tr:first-child td:last-child,.panel>.table:first-child>tbody:first-child>tr:first-child th:last-child,.panel>.table:first-child>thead:first-child>tr:first-child td:last-child,.panel>.table:first-child>thead:first-child>tr:first-child th:last-child{border-top-right-radius:3px}.panel>.table-responsive:last-child>.table:last-child,.panel>.table:last-child{border-bottom-right-radius:3px;border-bottom-left-radius:3px}.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child,.panel>.table:last-child>tbody:last-child>tr:last-child,.panel>.table:last-child>tfoot:last-child>tr:last-child{border-bottom-right-radius:3px;border-bottom-left-radius:3px}.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child td:first-child,.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child th:first-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child td:first-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child th:first-child,.panel>.table:last-child>tbody:last-child>tr:last-child td:first-child,.panel>.table:last-child>tbody:last-child>tr:last-child th:first-child,.panel>.table:last-child>tfoot:last-child>tr:last-child td:first-child,.panel>.table:last-child>tfoot:last-child>tr:last-child th:first-child{border-bottom-left-radius:3px}.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child td:last-child,.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child th:last-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child td:last-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child th:last-child,.panel>.table:last-child>tbody:last-child>tr:last-child td:last-child,.panel>.table:last-child>tbody:last-child>tr:last-child th:last-child,.panel>.table:last-child>tfoot:last-child>tr:last-child td:last-child,.panel>.table:last-child>tfoot:last-child>tr:last-child th:last-child{border-bottom-right-radius:3px}.panel>.panel-body+.table,.panel>.panel-body+.table-responsive,.panel>.table+.panel-body,.panel>.table-responsive+.panel-body{border-top:1px solid #ddd}.panel>.table>tbody:first-child>tr:first-child td,.panel>.table>tbody:first-child>tr:first-child th{border-top:0}.panel>.table-bordered,.panel>.table-responsive>.table-bordered{border:0}.panel>.table-bordered>tbody>tr>td:first-child,.panel>.table-bordered>tbody>tr>th:first-child,.panel>.table-bordered>tfoot>tr>td:first-child,.panel>.table-bordered>tfoot>tr>th:first-child,.panel>.table-bordered>thead>tr>td:first-child,.panel>.table-bordered>thead>tr>th:first-child,.panel>.table-responsive>.table-bordered>tbody>tr>td:first-child,.panel>.table-responsive>.table-bordered>tbody>tr>th:first-child,.panel>.table-responsive>.table-bordered>tfoot>tr>td:first-child,.panel>.table-responsive>.table-bordered>tfoot>tr>th:first-child,.panel>.table-responsive>.table-bordered>thead>tr>td:first-child,.panel>.table-responsive>.table-bordered>thead>tr>th:first-child{border-left:0}.panel>.table-bordered>tbody>tr>td:last-child,.panel>.table-bordered>tbody>tr>th:last-child,.panel>.table-bordered>tfoot>tr>td:last-child,.panel>.table-bordered>tfoot>tr>th:last-child,.panel>.table-bordered>thead>tr>td:last-child,.panel>.table-bordered>thead>tr>th:last-child,.panel>.table-responsive>.table-bordered>tbody>tr>td:last-child,.panel>.table-responsive>.table-bordered>tbody>tr>th:last-child,.panel>.table-responsive>.table-bordered>tfoot>tr>td:last-child,.panel>.table-responsive>.table-bordered>tfoot>tr>th:last-child,.panel>.table-responsive>.table-bordered>thead>tr>td:last-child,.panel>.table-responsive>.table-bordered>thead>tr>th:last-child{border-right:0}.panel>.table-bordered>tbody>tr:first-child>td,.panel>.table-bordered>tbody>tr:first-child>th,.panel>.table-bordered>thead>tr:first-child>td,.panel>.table-bordered>thead>tr:first-child>th,.panel>.table-responsive>.table-bordered>tbody>tr:first-child>td,.panel>.table-responsive>.table-bordered>tbody>tr:first-child>th,.panel>.table-responsive>.table-bordered>thead>tr:first-child>td,.panel>.table-responsive>.table-bordered>thead>tr:first-child>th{border-bottom:0}.panel>.table-bordered>tbody>tr:last-child>td,.panel>.table-bordered>tbody>tr:last-child>th,.panel>.table-bordered>tfoot>tr:last-child>td,.panel>.table-bordered>tfoot>tr:last-child>th,.panel>.table-responsive>.table-bordered>tbody>tr:last-child>td,.panel>.table-responsive>.table-bordered>tbody>tr:last-child>th,.panel>.table-responsive>.table-bordered>tfoot>tr:last-child>td,.panel>.table-responsive>.table-bordered>tfoot>tr:last-child>th{border-bottom:0}.panel>.table-responsive{margin-bottom:0;border:0}.panel-group{margin-bottom:20px}.panel-group .panel{margin-bottom:0;border-radius:4px}.panel-group .panel+.panel{margin-top:5px}.panel-group .panel-heading{border-bottom:0}.panel-group .panel-heading+.panel-collapse>.list-group,.panel-group .panel-heading+.panel-collapse>.panel-body{border-top:1px solid #ddd}.panel-group .panel-footer{border-top:0}.panel-group .panel-footer+.panel-collapse .panel-body{border-bottom:1px solid #ddd}.panel-default{border-color:#ddd}.panel-default>.panel-heading{color:#333;background-color:#f5f5f5;border-color:#ddd}.panel-default>.panel-heading+.panel-collapse>.panel-body{border-top-color:#ddd}.panel-default>.panel-heading .badge{color:#f5f5f5;background-color:#333}.panel-default>.panel-footer+.panel-collapse>.panel-body{border-bottom-color:#ddd}.panel-primary{border-color:#337ab7}.panel-primary>.panel-heading{color:#fff;background-color:#337ab7;border-color:#337ab7}.panel-primary>.panel-heading+.panel-collapse>.panel-body{border-top-color:#337ab7}.panel-primary>.panel-heading .badge{color:#337ab7;background-color:#fff}.panel-primary>.panel-footer+.panel-collapse>.panel-body{border-bottom-color:#337ab7}.panel-success{border-color:#d6e9c6}.panel-success>.panel-heading{color:#3c763d;background-color:#dff0d8;border-color:#d6e9c6}.panel-success>.panel-heading+.panel-collapse>.panel-body{border-top-color:#d6e9c6}.panel-success>.panel-heading .badge{color:#dff0d8;background-color:#3c763d}.panel-success>.panel-footer+.panel-collapse>.panel-body{border-bottom-color:#d6e9c6}.panel-info{border-color:#bce8f1}.panel-info>.panel-heading{color:#31708f;background-color:#d9edf7;border-color:#bce8f1}.panel-info>.panel-heading+.panel-collapse>.panel-body{border-top-color:#bce8f1}.panel-info>.panel-heading .badge{color:#d9edf7;background-color:#31708f}.panel-info>.panel-footer+.panel-collapse>.panel-body{border-bottom-color:#bce8f1}.panel-warning{border-color:#faebcc}.panel-warning>.panel-heading{color:#8a6d3b;background-color:#fcf8e3;border-color:#faebcc}.panel-warning>.panel-heading+.panel-collapse>.panel-body{border-top-color:#faebcc}.panel-warning>.panel-heading .badge{color:#fcf8e3;background-color:#8a6d3b}.panel-warning>.panel-footer+.panel-collapse>.panel-body{border-bottom-color:#faebcc}.panel-danger{border-color:#ebccd1}.panel-danger>.panel-heading{color:#a94442;background-color:#f2dede;border-color:#ebccd1}.panel-danger>.panel-heading+.panel-collapse>.panel-body{border-top-color:#ebccd1}.panel-danger>.panel-heading .badge{color:#f2dede;background-color:#a94442}.panel-danger>.panel-footer+.panel-collapse>.panel-body{border-bottom-color:#ebccd1}.embed-responsive{position:relative;display:block;height:0;padding:0;overflow:hidden}.embed-responsive .embed-responsive-item,.embed-responsive embed,.embed-responsive iframe,.embed-responsive object,.embed-responsive video{position:absolute;top:0;bottom:0;left:0;width:100%;height:100%;border:0}.embed-responsive-16by9{padding-bottom:56.25%}.embed-responsive-4by3{padding-bottom:75%}.well{min-height:20px;padding:19px;margin-bottom:20px;background-color:#f5f5f5;border:1px solid #e3e3e3;border-radius:4px;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.05);box-shadow:inset 0 1px 1px rgba(0,0,0,.05)}.well blockquote{border-color:#ddd;border-color:rgba(0,0,0,.15)}.well-lg{padding:24px;border-radius:6px}.well-sm{padding:9px;border-radius:3px}.close{float:right;font-size:21px;font-weight:700;line-height:1;color:#000;text-shadow:0 1px 0 #fff;filter:alpha(opacity=20);opacity:.2}.close:focus,.close:hover{color:#000;text-decoration:none;cursor:pointer;filter:alpha(opacity=50);opacity:.5}button.close{-webkit-appearance:none;padding:0;cursor:pointer;background:0 0;border:0}.modal-open{overflow:hidden}.modal{position:fixed;top:0;right:0;bottom:0;left:0;z-index:1050;display:none;overflow:hidden;-webkit-overflow-scrolling:touch;outline:0}.modal.fade .modal-dialog{-webkit-transition:-webkit-transform .3s ease-out;-o-transition:-o-transform .3s ease-out;transition:transform .3s ease-out;-webkit-transform:translate(0,-25%);-ms-transform:translate(0,-25%);-o-transform:translate(0,-25%);transform:translate(0,-25%)}.modal.in .modal-dialog{-webkit-transform:translate(0,0);-ms-transform:translate(0,0);-o-transform:translate(0,0);transform:translate(0,0)}.modal-open .modal{overflow-x:hidden;overflow-y:auto}.modal-dialog{position:relative;width:auto;margin:10px}.modal-content{position:relative;background-color:#fff;-webkit-background-clip:padding-box;background-clip:padding-box;border:1px solid #999;border:1px solid rgba(0,0,0,.2);border-radius:6px;outline:0;-webkit-box-shadow:0 3px 9px rgba(0,0,0,.5);box-shadow:0 3px 9px rgba(0,0,0,.5)}.modal-backdrop{position:fixed;top:0;right:0;bottom:0;left:0;z-index:1040;background-color:#000}.modal-backdrop.fade{filter:alpha(opacity=0);opacity:0}.modal-backdrop.in{filter:alpha(opacity=50);opacity:.5}.modal-header{min-height:16.43px;padding:15px;border-bottom:1px solid #e5e5e5}.modal-header .close{margin-top:-2px}.modal-title{margin:0;line-height:1.42857143}.modal-body{position:relative;padding:15px}.modal-footer{padding:15px;text-align:right;border-top:1px solid #e5e5e5}.modal-footer .btn+.btn{margin-bottom:0;margin-left:5px}.modal-footer .btn-group .btn+.btn{margin-left:-1px}.modal-footer .btn-block+.btn-block{margin-left:0}.modal-scrollbar-measure{position:absolute;top:-9999px;width:50px;height:50px;overflow:scroll}@media (min-width:768px){.modal-dialog{width:600px;margin:30px auto}.modal-content{-webkit-box-shadow:0 5px 15px rgba(0,0,0,.5);box-shadow:0 5px 15px rgba(0,0,0,.5)}.modal-sm{width:300px}}@media (min-width:992px){.modal-lg{width:900px}}.tooltip{position:absolute;z-index:1070;display:block;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:12px;font-style:normal;font-weight:400;line-height:1.42857143;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;word-wrap:normal;white-space:normal;filter:alpha(opacity=0);opacity:0;line-break:auto}.tooltip.in{filter:alpha(opacity=90);opacity:.9}.tooltip.top{padding:5px 0;margin-top:-3px}.tooltip.right{padding:0 5px;margin-left:3px}.tooltip.bottom{padding:5px 0;margin-top:3px}.tooltip.left{padding:0 5px;margin-left:-3px}.tooltip-inner{max-width:200px;padding:3px 8px;color:#fff;text-align:center;background-color:#000;border-radius:4px}.tooltip-arrow{position:absolute;width:0;height:0;border-color:transparent;border-style:solid}.tooltip.top .tooltip-arrow{bottom:0;left:50%;margin-left:-5px;border-width:5px 5px 0;border-top-color:#000}.tooltip.top-left .tooltip-arrow{right:5px;bottom:0;margin-bottom:-5px;border-width:5px 5px 0;border-top-color:#000}.tooltip.top-right .tooltip-arrow{bottom:0;left:5px;margin-bottom:-5px;border-width:5px 5px 0;border-top-color:#000}.tooltip.right .tooltip-arrow{top:50%;left:0;margin-top:-5px;border-width:5px 5px 5px 0;border-right-color:#000}.tooltip.left .tooltip-arrow{top:50%;right:0;margin-top:-5px;border-width:5px 0 5px 5px;border-left-color:#000}.tooltip.bottom .tooltip-arrow{top:0;left:50%;margin-left:-5px;border-width:0 5px 5px;border-bottom-color:#000}.tooltip.bottom-left .tooltip-arrow{top:0;right:5px;margin-top:-5px;border-width:0 5px 5px;border-bottom-color:#000}.tooltip.bottom-right .tooltip-arrow{top:0;left:5px;margin-top:-5px;border-width:0 5px 5px;border-bottom-color:#000}.popover{position:absolute;top:0;left:0;z-index:1060;display:none;max-width:276px;padding:1px;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:14px;font-style:normal;font-weight:400;line-height:1.42857143;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;word-wrap:normal;white-space:normal;background-color:#fff;-webkit-background-clip:padding-box;background-clip:padding-box;border:1px solid #ccc;border:1px solid rgba(0,0,0,.2);border-radius:6px;-webkit-box-shadow:0 5px 10px rgba(0,0,0,.2);box-shadow:0 5px 10px rgba(0,0,0,.2);line-break:auto}.popover.top{margin-top:-10px}.popover.right{margin-left:10px}.popover.bottom{margin-top:10px}.popover.left{margin-left:-10px}.popover-title{padding:8px 14px;margin:0;font-size:14px;background-color:#f7f7f7;border-bottom:1px solid #ebebeb;border-radius:5px 5px 0 0}.popover-content{padding:9px 14px}.popover>.arrow,.popover>.arrow:after{position:absolute;display:block;width:0;height:0;border-color:transparent;border-style:solid}.popover>.arrow{border-width:11px}.popover>.arrow:after{content:"";border-width:10px}.popover.top>.arrow{bottom:-11px;left:50%;margin-left:-11px;border-top-color:#999;border-top-color:rgba(0,0,0,.25);border-bottom-width:0}.popover.top>.arrow:after{bottom:1px;margin-left:-10px;content:" ";border-top-color:#fff;border-bottom-width:0}.popover.right>.arrow{top:50%;left:-11px;margin-top:-11px;border-right-color:#999;border-right-color:rgba(0,0,0,.25);border-left-width:0}.popover.right>.arrow:after{bottom:-10px;left:1px;content:" ";border-right-color:#fff;border-left-width:0}.popover.bottom>.arrow{top:-11px;left:50%;margin-left:-11px;border-top-width:0;border-bottom-color:#999;border-bottom-color:rgba(0,0,0,.25)}.popover.bottom>.arrow:after{top:1px;margin-left:-10px;content:" ";border-top-width:0;border-bottom-color:#fff}.popover.left>.arrow{top:50%;right:-11px;margin-top:-11px;border-right-width:0;border-left-color:#999;border-left-color:rgba(0,0,0,.25)}.popover.left>.arrow:after{right:1px;bottom:-10px;content:" ";border-right-width:0;border-left-color:#fff}.carousel{position:relative}.carousel-inner{position:relative;width:100%;overflow:hidden}.carousel-inner>.item{position:relative;display:none;-webkit-transition:.6s ease-in-out left;-o-transition:.6s ease-in-out left;transition:.6s ease-in-out left}.carousel-inner>.item>a>img,.carousel-inner>.item>img{line-height:1}@media all and (transform-3d),(-webkit-transform-3d){.carousel-inner>.item{-webkit-transition:-webkit-transform .6s ease-in-out;-o-transition:-o-transform .6s ease-in-out;transition:transform .6s ease-in-out;-webkit-backface-visibility:hidden;backface-visibility:hidden;-webkit-perspective:1000px;perspective:1000px}.carousel-inner>.item.active.right,.carousel-inner>.item.next{left:0;-webkit-transform:translate3d(100%,0,0);transform:translate3d(100%,0,0)}.carousel-inner>.item.active.left,.carousel-inner>.item.prev{left:0;-webkit-transform:translate3d(-100%,0,0);transform:translate3d(-100%,0,0)}.carousel-inner>.item.active,.carousel-inner>.item.next.left,.carousel-inner>.item.prev.right{left:0;-webkit-transform:translate3d(0,0,0);transform:translate3d(0,0,0)}}.carousel-inner>.active,.carousel-inner>.next,.carousel-inner>.prev{display:block}.carousel-inner>.active{left:0}.carousel-inner>.next,.carousel-inner>.prev{position:absolute;top:0;width:100%}.carousel-inner>.next{left:100%}.carousel-inner>.prev{left:-100%}.carousel-inner>.next.left,.carousel-inner>.prev.right{left:0}.carousel-inner>.active.left{left:-100%}.carousel-inner>.active.right{left:100%}.carousel-control{position:absolute;top:0;bottom:0;left:0;width:15%;font-size:20px;color:#fff;text-align:center;text-shadow:0 1px 2px rgba(0,0,0,.6);filter:alpha(opacity=50);opacity:.5}.carousel-control.left{background-image:-webkit-linear-gradient(left,rgba(0,0,0,.5) 0,rgba(0,0,0,.0001) 100%);background-image:-o-linear-gradient(left,rgba(0,0,0,.5) 0,rgba(0,0,0,.0001) 100%);background-image:-webkit-gradient(linear,left top,right top,from(rgba(0,0,0,.5)),to(rgba(0,0,0,.0001)));background-image:linear-gradient(to right,rgba(0,0,0,.5) 0,rgba(0,0,0,.0001) 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#80000000', endColorstr='#00000000', GradientType=1);background-repeat:repeat-x}.carousel-control.right{right:0;left:auto;background-image:-webkit-linear-gradient(left,rgba(0,0,0,.0001) 0,rgba(0,0,0,.5) 100%);background-image:-o-linear-gradient(left,rgba(0,0,0,.0001) 0,rgba(0,0,0,.5) 100%);background-image:-webkit-gradient(linear,left top,right top,from(rgba(0,0,0,.0001)),to(rgba(0,0,0,.5)));background-image:linear-gradient(to right,rgba(0,0,0,.0001) 0,rgba(0,0,0,.5) 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#00000000', endColorstr='#80000000', GradientType=1);background-repeat:repeat-x}.carousel-control:focus,.carousel-control:hover{color:#fff;text-decoration:none;filter:alpha(opacity=90);outline:0;opacity:.9}.carousel-control .glyphicon-chevron-left,.carousel-control .glyphicon-chevron-right,.carousel-control .icon-next,.carousel-control .icon-prev{position:absolute;top:50%;z-index:5;display:inline-block;margin-top:-10px}.carousel-control .glyphicon-chevron-left,.carousel-control .icon-prev{left:50%;margin-left:-10px}.carousel-control .glyphicon-chevron-right,.carousel-control .icon-next{right:50%;margin-right:-10px}.carousel-control .icon-next,.carousel-control .icon-prev{width:20px;height:20px;font-family:serif;line-height:1}.carousel-control .icon-prev:before{content:'\2039'}.carousel-control .icon-next:before{content:'\203a'}.carousel-indicators{position:absolute;bottom:10px;left:50%;z-index:15;width:60%;padding-left:0;margin-left:-30%;text-align:center;list-style:none}.carousel-indicators li{display:inline-block;width:10px;height:10px;margin:1px;text-indent:-999px;cursor:pointer;background-color:#000\9;background-color:rgba(0,0,0,0);border:1px solid #fff;border-radius:10px}.carousel-indicators .active{width:12px;height:12px;margin:0;background-color:#fff}.carousel-caption{position:absolute;right:15%;bottom:20px;left:15%;z-index:10;padding-top:20px;padding-bottom:20px;color:#fff;text-align:center;text-shadow:0 1px 2px rgba(0,0,0,.6)}.carousel-caption .btn{text-shadow:none}@media screen and (min-width:768px){.carousel-control .glyphicon-chevron-left,.carousel-control .glyphicon-chevron-right,.carousel-control .icon-next,.carousel-control .icon-prev{width:30px;height:30px;margin-top:-15px;font-size:30px}.carousel-control .glyphicon-chevron-left,.carousel-control .icon-prev{margin-left:-15px}.carousel-control .glyphicon-chevron-right,.carousel-control .icon-next{margin-right:-15px}.carousel-caption{right:20%;left:20%;padding-bottom:30px}.carousel-indicators{bottom:20px}}.btn-group-vertical>.btn-group:after,.btn-group-vertical>.btn-group:before,.btn-toolbar:after,.btn-toolbar:before,.clearfix:after,.clearfix:before,.container-fluid:after,.container-fluid:before,.container:after,.container:before,.dl-horizontal dd:after,.dl-horizontal dd:before,.form-horizontal .form-group:after,.form-horizontal .form-group:before,.modal-footer:after,.modal-footer:before,.nav:after,.nav:before,.navbar-collapse:after,.navbar-collapse:before,.navbar-header:after,.navbar-header:before,.navbar:after,.navbar:before,.pager:after,.pager:before,.panel-body:after,.panel-body:before,.row:after,.row:before{display:table;content:" "}.btn-group-vertical>.btn-group:after,.btn-toolbar:after,.clearfix:after,.container-fluid:after,.container:after,.dl-horizontal dd:after,.form-horizontal .form-group:after,.modal-footer:after,.nav:after,.navbar-collapse:after,.navbar-header:after,.navbar:after,.pager:after,.panel-body:after,.row:after{clear:both}.center-block{display:block;margin-right:auto;margin-left:auto}.pull-right{float:right!important}.pull-left{float:left!important}.hide{display:none!important}.show{display:block!important}.invisible{visibility:hidden}.text-hide{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.hidden{display:none!important}.affix{position:fixed}@-ms-viewport{width:device-width}.visible-lg,.visible-md,.visible-sm,.visible-xs{display:none!important}.visible-lg-block,.visible-lg-inline,.visible-lg-inline-block,.visible-md-block,.visible-md-inline,.visible-md-inline-block,.visible-sm-block,.visible-sm-inline,.visible-sm-inline-block,.visible-xs-block,.visible-xs-inline,.visible-xs-inline-block{display:none!important}@media (max-width:767px){.visible-xs{display:block!important}table.visible-xs{display:table!important}tr.visible-xs{display:table-row!important}td.visible-xs,th.visible-xs{display:table-cell!important}}@media (max-width:767px){.visible-xs-block{display:block!important}}@media (max-width:767px){.visible-xs-inline{display:inline!important}}@media (max-width:767px){.visible-xs-inline-block{display:inline-block!important}}@media (min-width:768px) and (max-width:991px){.visible-sm{display:block!important}table.visible-sm{display:table!important}tr.visible-sm{display:table-row!important}td.visible-sm,th.visible-sm{display:table-cell!important}}@media (min-width:768px) and (max-width:991px){.visible-sm-block{display:block!important}}@media (min-width:768px) and (max-width:991px){.visible-sm-inline{display:inline!important}}@media (min-width:768px) and (max-width:991px){.visible-sm-inline-block{display:inline-block!important}}@media (min-width:992px) and (max-width:1199px){.visible-md{display:block!important}table.visible-md{display:table!important}tr.visible-md{display:table-row!important}td.visible-md,th.visible-md{display:table-cell!important}}@media (min-width:992px) and (max-width:1199px){.visible-md-block{display:block!important}}@media (min-width:992px) and (max-width:1199px){.visible-md-inline{display:inline!important}}@media (min-width:992px) and (max-width:1199px){.visible-md-inline-block{display:inline-block!important}}@media (min-width:1200px){.visible-lg{display:block!important}table.visible-lg{display:table!important}tr.visible-lg{display:table-row!important}td.visible-lg,th.visible-lg{display:table-cell!important}}@media (min-width:1200px){.visible-lg-block{display:block!important}}@media (min-width:1200px){.visible-lg-inline{display:inline!important}}@media (min-width:1200px){.visible-lg-inline-block{display:inline-block!important}}@media (max-width:767px){.hidden-xs{display:none!important}}@media (min-width:768px) and (max-width:991px){.hidden-sm{display:none!important}}@media (min-width:992px) and (max-width:1199px){.hidden-md{display:none!important}}@media (min-width:1200px){.hidden-lg{display:none!important}}.visible-print{display:none!important}@media print{.visible-print{display:block!important}table.visible-print{display:table!important}tr.visible-print{display:table-row!important}td.visible-print,th.visible-print{display:table-cell!important}}.visible-print-block{display:none!important}@media print{.visible-print-block{display:block!important}}.visible-print-inline{display:none!important}@media print{.visible-print-inline{display:inline!important}}.visible-print-inline-block{display:none!important}@media print{.visible-print-inline-block{display:inline-block!important}}@media print{.hidden-print{display:none!important}}
\ No newline at end of file + *//*! normalize.css v3.0.3 | MIT License | github.com/necolas/normalize.css */html{font-family:sans-serif;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%}body{margin:0}article,aside,details,figcaption,figure,footer,header,hgroup,main,menu,nav,section,summary{display:block}audio,canvas,progress,video{display:inline-block;vertical-align:baseline}audio:not([controls]){display:none;height:0}[hidden],template{display:none}a{background-color:transparent}a:active,a:hover{outline:0}abbr[title]{border-bottom:1px dotted}b,strong{font-weight:700}dfn{font-style:italic}h1{margin:.67em 0;font-size:2em}mark{color:#000;background:#ff0}small{font-size:80%}sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline}sup{top:-.5em}sub{bottom:-.25em}img{border:0}svg:not(:root){overflow:hidden}figure{margin:1em 40px}hr{height:0;-webkit-box-sizing:content-box;-moz-box-sizing:content-box;box-sizing:content-box}pre{overflow:auto}code,kbd,pre,samp{font-family:monospace,monospace;font-size:1em}button,input,optgroup,select,textarea{margin:0;font:inherit;color:inherit}button{overflow:visible}button,select{text-transform:none}button,html input[type=button],input[type=reset],input[type=submit]{-webkit-appearance:button;cursor:pointer}button[disabled],html input[disabled]{cursor:default}button::-moz-focus-inner,input::-moz-focus-inner{padding:0;border:0}input{line-height:normal}input[type=checkbox],input[type=radio]{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;padding:0}input[type=number]::-webkit-inner-spin-button,input[type=number]::-webkit-outer-spin-button{height:auto}input[type=search]{-webkit-box-sizing:content-box;-moz-box-sizing:content-box;box-sizing:content-box;-webkit-appearance:textfield}input[type=search]::-webkit-search-cancel-button,input[type=search]::-webkit-search-decoration{-webkit-appearance:none}fieldset{padding:.35em .625em .75em;margin:0 2px;border:1px solid silver}legend{padding:0;border:0}textarea{overflow:auto}optgroup{font-weight:700}table{border-spacing:0;border-collapse:collapse}td,th{padding:0}/*! Source: https://github.com/h5bp/html5-boilerplate/blob/master/src/css/main.css */@media print{*,:after,:before{color:#000!important;text-shadow:none!important;background:0 0!important;-webkit-box-shadow:none!important;box-shadow:none!important}a,a:visited{text-decoration:underline}a[href]:after{content:" (" attr(href) ")"}abbr[title]:after{content:" (" attr(title) ")"}a[href^="javascript:"]:after,a[href^="#"]:after{content:""}blockquote,pre{border:1px solid #999;page-break-inside:avoid}thead{display:table-header-group}img,tr{page-break-inside:avoid}img{max-width:100%!important}h2,h3,p{orphans:3;widows:3}h2,h3{page-break-after:avoid}.navbar{display:none}.btn>.caret,.dropup>.btn>.caret{border-top-color:#000!important}.label{border:1px solid #000}.table{border-collapse:collapse!important}.table td,.table th{background-color:#fff!important}.table-bordered td,.table-bordered th{border:1px solid #ddd!important}}@font-face{font-family:'Glyphicons Halflings';src:url(../fonts/glyphicons-halflings-regular.eot);src:url(../fonts/glyphicons-halflings-regular.eot?#iefix) format('embedded-opentype'),url(../fonts/glyphicons-halflings-regular.woff2) format('woff2'),url(../fonts/glyphicons-halflings-regular.woff) format('woff'),url(../fonts/glyphicons-halflings-regular.ttf) format('truetype'),url(../fonts/glyphicons-halflings-regular.svg#glyphicons_halflingsregular) format('svg')}.glyphicon{position:relative;top:1px;display:inline-block;font-family:'Glyphicons Halflings';font-style:normal;font-weight:400;line-height:1;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}.glyphicon-asterisk:before{content:"\002a"}.glyphicon-plus:before{content:"\002b"}.glyphicon-eur:before,.glyphicon-euro:before{content:"\20ac"}.glyphicon-minus:before{content:"\2212"}.glyphicon-cloud:before{content:"\2601"}.glyphicon-envelope:before{content:"\2709"}.glyphicon-pencil:before{content:"\270f"}.glyphicon-glass:before{content:"\e001"}.glyphicon-music:before{content:"\e002"}.glyphicon-search:before{content:"\e003"}.glyphicon-heart:before{content:"\e005"}.glyphicon-star:before{content:"\e006"}.glyphicon-star-empty:before{content:"\e007"}.glyphicon-user:before{content:"\e008"}.glyphicon-film:before{content:"\e009"}.glyphicon-th-large:before{content:"\e010"}.glyphicon-th:before{content:"\e011"}.glyphicon-th-list:before{content:"\e012"}.glyphicon-ok:before{content:"\e013"}.glyphicon-remove:before{content:"\e014"}.glyphicon-zoom-in:before{content:"\e015"}.glyphicon-zoom-out:before{content:"\e016"}.glyphicon-off:before{content:"\e017"}.glyphicon-signal:before{content:"\e018"}.glyphicon-cog:before{content:"\e019"}.glyphicon-trash:before{content:"\e020"}.glyphicon-home:before{content:"\e021"}.glyphicon-file:before{content:"\e022"}.glyphicon-time:before{content:"\e023"}.glyphicon-road:before{content:"\e024"}.glyphicon-download-alt:before{content:"\e025"}.glyphicon-download:before{content:"\e026"}.glyphicon-upload:before{content:"\e027"}.glyphicon-inbox:before{content:"\e028"}.glyphicon-play-circle:before{content:"\e029"}.glyphicon-repeat:before{content:"\e030"}.glyphicon-refresh:before{content:"\e031"}.glyphicon-list-alt:before{content:"\e032"}.glyphicon-lock:before{content:"\e033"}.glyphicon-flag:before{content:"\e034"}.glyphicon-headphones:before{content:"\e035"}.glyphicon-volume-off:before{content:"\e036"}.glyphicon-volume-down:before{content:"\e037"}.glyphicon-volume-up:before{content:"\e038"}.glyphicon-qrcode:before{content:"\e039"}.glyphicon-barcode:before{content:"\e040"}.glyphicon-tag:before{content:"\e041"}.glyphicon-tags:before{content:"\e042"}.glyphicon-book:before{content:"\e043"}.glyphicon-bookmark:before{content:"\e044"}.glyphicon-print:before{content:"\e045"}.glyphicon-camera:before{content:"\e046"}.glyphicon-font:before{content:"\e047"}.glyphicon-bold:before{content:"\e048"}.glyphicon-italic:before{content:"\e049"}.glyphicon-text-height:before{content:"\e050"}.glyphicon-text-width:before{content:"\e051"}.glyphicon-align-left:before{content:"\e052"}.glyphicon-align-center:before{content:"\e053"}.glyphicon-align-right:before{content:"\e054"}.glyphicon-align-justify:before{content:"\e055"}.glyphicon-list:before{content:"\e056"}.glyphicon-indent-left:before{content:"\e057"}.glyphicon-indent-right:before{content:"\e058"}.glyphicon-facetime-video:before{content:"\e059"}.glyphicon-picture:before{content:"\e060"}.glyphicon-map-marker:before{content:"\e062"}.glyphicon-adjust:before{content:"\e063"}.glyphicon-tint:before{content:"\e064"}.glyphicon-edit:before{content:"\e065"}.glyphicon-share:before{content:"\e066"}.glyphicon-check:before{content:"\e067"}.glyphicon-move:before{content:"\e068"}.glyphicon-step-backward:before{content:"\e069"}.glyphicon-fast-backward:before{content:"\e070"}.glyphicon-backward:before{content:"\e071"}.glyphicon-play:before{content:"\e072"}.glyphicon-pause:before{content:"\e073"}.glyphicon-stop:before{content:"\e074"}.glyphicon-forward:before{content:"\e075"}.glyphicon-fast-forward:before{content:"\e076"}.glyphicon-step-forward:before{content:"\e077"}.glyphicon-eject:before{content:"\e078"}.glyphicon-chevron-left:before{content:"\e079"}.glyphicon-chevron-right:before{content:"\e080"}.glyphicon-plus-sign:before{content:"\e081"}.glyphicon-minus-sign:before{content:"\e082"}.glyphicon-remove-sign:before{content:"\e083"}.glyphicon-ok-sign:before{content:"\e084"}.glyphicon-question-sign:before{content:"\e085"}.glyphicon-info-sign:before{content:"\e086"}.glyphicon-screenshot:before{content:"\e087"}.glyphicon-remove-circle:before{content:"\e088"}.glyphicon-ok-circle:before{content:"\e089"}.glyphicon-ban-circle:before{content:"\e090"}.glyphicon-arrow-left:before{content:"\e091"}.glyphicon-arrow-right:before{content:"\e092"}.glyphicon-arrow-up:before{content:"\e093"}.glyphicon-arrow-down:before{content:"\e094"}.glyphicon-share-alt:before{content:"\e095"}.glyphicon-resize-full:before{content:"\e096"}.glyphicon-resize-small:before{content:"\e097"}.glyphicon-exclamation-sign:before{content:"\e101"}.glyphicon-gift:before{content:"\e102"}.glyphicon-leaf:before{content:"\e103"}.glyphicon-fire:before{content:"\e104"}.glyphicon-eye-open:before{content:"\e105"}.glyphicon-eye-close:before{content:"\e106"}.glyphicon-warning-sign:before{content:"\e107"}.glyphicon-plane:before{content:"\e108"}.glyphicon-calendar:before{content:"\e109"}.glyphicon-random:before{content:"\e110"}.glyphicon-comment:before{content:"\e111"}.glyphicon-magnet:before{content:"\e112"}.glyphicon-chevron-up:before{content:"\e113"}.glyphicon-chevron-down:before{content:"\e114"}.glyphicon-retweet:before{content:"\e115"}.glyphicon-shopping-cart:before{content:"\e116"}.glyphicon-folder-close:before{content:"\e117"}.glyphicon-folder-open:before{content:"\e118"}.glyphicon-resize-vertical:before{content:"\e119"}.glyphicon-resize-horizontal:before{content:"\e120"}.glyphicon-hdd:before{content:"\e121"}.glyphicon-bullhorn:before{content:"\e122"}.glyphicon-bell:before{content:"\e123"}.glyphicon-certificate:before{content:"\e124"}.glyphicon-thumbs-up:before{content:"\e125"}.glyphicon-thumbs-down:before{content:"\e126"}.glyphicon-hand-right:before{content:"\e127"}.glyphicon-hand-left:before{content:"\e128"}.glyphicon-hand-up:before{content:"\e129"}.glyphicon-hand-down:before{content:"\e130"}.glyphicon-circle-arrow-right:before{content:"\e131"}.glyphicon-circle-arrow-left:before{content:"\e132"}.glyphicon-circle-arrow-up:before{content:"\e133"}.glyphicon-circle-arrow-down:before{content:"\e134"}.glyphicon-globe:before{content:"\e135"}.glyphicon-wrench:before{content:"\e136"}.glyphicon-tasks:before{content:"\e137"}.glyphicon-filter:before{content:"\e138"}.glyphicon-briefcase:before{content:"\e139"}.glyphicon-fullscreen:before{content:"\e140"}.glyphicon-dashboard:before{content:"\e141"}.glyphicon-paperclip:before{content:"\e142"}.glyphicon-heart-empty:before{content:"\e143"}.glyphicon-link:before{content:"\e144"}.glyphicon-phone:before{content:"\e145"}.glyphicon-pushpin:before{content:"\e146"}.glyphicon-usd:before{content:"\e148"}.glyphicon-gbp:before{content:"\e149"}.glyphicon-sort:before{content:"\e150"}.glyphicon-sort-by-alphabet:before{content:"\e151"}.glyphicon-sort-by-alphabet-alt:before{content:"\e152"}.glyphicon-sort-by-order:before{content:"\e153"}.glyphicon-sort-by-order-alt:before{content:"\e154"}.glyphicon-sort-by-attributes:before{content:"\e155"}.glyphicon-sort-by-attributes-alt:before{content:"\e156"}.glyphicon-unchecked:before{content:"\e157"}.glyphicon-expand:before{content:"\e158"}.glyphicon-collapse-down:before{content:"\e159"}.glyphicon-collapse-up:before{content:"\e160"}.glyphicon-log-in:before{content:"\e161"}.glyphicon-flash:before{content:"\e162"}.glyphicon-log-out:before{content:"\e163"}.glyphicon-new-window:before{content:"\e164"}.glyphicon-record:before{content:"\e165"}.glyphicon-save:before{content:"\e166"}.glyphicon-open:before{content:"\e167"}.glyphicon-saved:before{content:"\e168"}.glyphicon-import:before{content:"\e169"}.glyphicon-export:before{content:"\e170"}.glyphicon-send:before{content:"\e171"}.glyphicon-floppy-disk:before{content:"\e172"}.glyphicon-floppy-saved:before{content:"\e173"}.glyphicon-floppy-remove:before{content:"\e174"}.glyphicon-floppy-save:before{content:"\e175"}.glyphicon-floppy-open:before{content:"\e176"}.glyphicon-credit-card:before{content:"\e177"}.glyphicon-transfer:before{content:"\e178"}.glyphicon-cutlery:before{content:"\e179"}.glyphicon-header:before{content:"\e180"}.glyphicon-compressed:before{content:"\e181"}.glyphicon-earphone:before{content:"\e182"}.glyphicon-phone-alt:before{content:"\e183"}.glyphicon-tower:before{content:"\e184"}.glyphicon-stats:before{content:"\e185"}.glyphicon-sd-video:before{content:"\e186"}.glyphicon-hd-video:before{content:"\e187"}.glyphicon-subtitles:before{content:"\e188"}.glyphicon-sound-stereo:before{content:"\e189"}.glyphicon-sound-dolby:before{content:"\e190"}.glyphicon-sound-5-1:before{content:"\e191"}.glyphicon-sound-6-1:before{content:"\e192"}.glyphicon-sound-7-1:before{content:"\e193"}.glyphicon-copyright-mark:before{content:"\e194"}.glyphicon-registration-mark:before{content:"\e195"}.glyphicon-cloud-download:before{content:"\e197"}.glyphicon-cloud-upload:before{content:"\e198"}.glyphicon-tree-conifer:before{content:"\e199"}.glyphicon-tree-deciduous:before{content:"\e200"}.glyphicon-cd:before{content:"\e201"}.glyphicon-save-file:before{content:"\e202"}.glyphicon-open-file:before{content:"\e203"}.glyphicon-level-up:before{content:"\e204"}.glyphicon-copy:before{content:"\e205"}.glyphicon-paste:before{content:"\e206"}.glyphicon-alert:before{content:"\e209"}.glyphicon-equalizer:before{content:"\e210"}.glyphicon-king:before{content:"\e211"}.glyphicon-queen:before{content:"\e212"}.glyphicon-pawn:before{content:"\e213"}.glyphicon-bishop:before{content:"\e214"}.glyphicon-knight:before{content:"\e215"}.glyphicon-baby-formula:before{content:"\e216"}.glyphicon-tent:before{content:"\26fa"}.glyphicon-blackboard:before{content:"\e218"}.glyphicon-bed:before{content:"\e219"}.glyphicon-apple:before{content:"\f8ff"}.glyphicon-erase:before{content:"\e221"}.glyphicon-hourglass:before{content:"\231b"}.glyphicon-lamp:before{content:"\e223"}.glyphicon-duplicate:before{content:"\e224"}.glyphicon-piggy-bank:before{content:"\e225"}.glyphicon-scissors:before{content:"\e226"}.glyphicon-bitcoin:before{content:"\e227"}.glyphicon-btc:before{content:"\e227"}.glyphicon-xbt:before{content:"\e227"}.glyphicon-yen:before{content:"\00a5"}.glyphicon-jpy:before{content:"\00a5"}.glyphicon-ruble:before{content:"\20bd"}.glyphicon-rub:before{content:"\20bd"}.glyphicon-scale:before{content:"\e230"}.glyphicon-ice-lolly:before{content:"\e231"}.glyphicon-ice-lolly-tasted:before{content:"\e232"}.glyphicon-education:before{content:"\e233"}.glyphicon-option-horizontal:before{content:"\e234"}.glyphicon-option-vertical:before{content:"\e235"}.glyphicon-menu-hamburger:before{content:"\e236"}.glyphicon-modal-window:before{content:"\e237"}.glyphicon-oil:before{content:"\e238"}.glyphicon-grain:before{content:"\e239"}.glyphicon-sunglasses:before{content:"\e240"}.glyphicon-text-size:before{content:"\e241"}.glyphicon-text-color:before{content:"\e242"}.glyphicon-text-background:before{content:"\e243"}.glyphicon-object-align-top:before{content:"\e244"}.glyphicon-object-align-bottom:before{content:"\e245"}.glyphicon-object-align-horizontal:before{content:"\e246"}.glyphicon-object-align-left:before{content:"\e247"}.glyphicon-object-align-vertical:before{content:"\e248"}.glyphicon-object-align-right:before{content:"\e249"}.glyphicon-triangle-right:before{content:"\e250"}.glyphicon-triangle-left:before{content:"\e251"}.glyphicon-triangle-bottom:before{content:"\e252"}.glyphicon-triangle-top:before{content:"\e253"}.glyphicon-console:before{content:"\e254"}.glyphicon-superscript:before{content:"\e255"}.glyphicon-subscript:before{content:"\e256"}.glyphicon-menu-left:before{content:"\e257"}.glyphicon-menu-right:before{content:"\e258"}.glyphicon-menu-down:before{content:"\e259"}.glyphicon-menu-up:before{content:"\e260"}*{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}:after,:before{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}html{font-size:10px;-webkit-tap-highlight-color:rgba(0,0,0,0)}body{font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:14px;line-height:1.42857143;color:#333;background-color:#fff}button,input,select,textarea{font-family:inherit;font-size:inherit;line-height:inherit}a{color:#337ab7;text-decoration:none}a:focus,a:hover{color:#23527c;text-decoration:underline}a:focus{outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}figure{margin:0}img{vertical-align:middle}.carousel-inner>.item>a>img,.carousel-inner>.item>img,.img-responsive,.thumbnail a>img,.thumbnail>img{display:block;max-width:100%;height:auto}.img-rounded{border-radius:6px}.img-thumbnail{display:inline-block;max-width:100%;height:auto;padding:4px;line-height:1.42857143;background-color:#fff;border:1px solid #ddd;border-radius:4px;-webkit-transition:all .2s ease-in-out;-o-transition:all .2s ease-in-out;transition:all .2s ease-in-out}.img-circle{border-radius:50%}hr{margin-top:20px;margin-bottom:20px;border:0;border-top:1px solid #eee}.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);border:0}.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;margin:0;overflow:visible;clip:auto}[role=button]{cursor:pointer}.h1,.h2,.h3,.h4,.h5,.h6,h1,h2,h3,h4,h5,h6{font-family:inherit;font-weight:500;line-height:1.1;color:inherit}.h1 .small,.h1 small,.h2 .small,.h2 small,.h3 .small,.h3 small,.h4 .small,.h4 small,.h5 .small,.h5 small,.h6 .small,.h6 small,h1 .small,h1 small,h2 .small,h2 small,h3 .small,h3 small,h4 .small,h4 small,h5 .small,h5 small,h6 .small,h6 small{font-weight:400;line-height:1;color:#777}.h1,.h2,.h3,h1,h2,h3{margin-top:20px;margin-bottom:10px}.h1 .small,.h1 small,.h2 .small,.h2 small,.h3 .small,.h3 small,h1 .small,h1 small,h2 .small,h2 small,h3 .small,h3 small{font-size:65%}.h4,.h5,.h6,h4,h5,h6{margin-top:10px;margin-bottom:10px}.h4 .small,.h4 small,.h5 .small,.h5 small,.h6 .small,.h6 small,h4 .small,h4 small,h5 .small,h5 small,h6 .small,h6 small{font-size:75%}.h1,h1{font-size:36px}.h2,h2{font-size:30px}.h3,h3{font-size:24px}.h4,h4{font-size:18px}.h5,h5{font-size:14px}.h6,h6{font-size:12px}p{margin:0 0 10px}.lead{margin-bottom:20px;font-size:16px;font-weight:300;line-height:1.4}@media (min-width:768px){.lead{font-size:21px}}.small,small{font-size:85%}.mark,mark{padding:.2em;background-color:#fcf8e3}.text-left{text-align:left}.text-right{text-align:right}.text-center{text-align:center}.text-justify{text-align:justify}.text-nowrap{white-space:nowrap}.text-lowercase{text-transform:lowercase}.text-uppercase{text-transform:uppercase}.text-capitalize{text-transform:capitalize}.text-muted{color:#777}.text-primary{color:#337ab7}a.text-primary:focus,a.text-primary:hover{color:#286090}.text-success{color:#3c763d}a.text-success:focus,a.text-success:hover{color:#2b542c}.text-info{color:#31708f}a.text-info:focus,a.text-info:hover{color:#245269}.text-warning{color:#8a6d3b}a.text-warning:focus,a.text-warning:hover{color:#66512c}.text-danger{color:#a94442}a.text-danger:focus,a.text-danger:hover{color:#843534}.bg-primary{color:#fff;background-color:#337ab7}a.bg-primary:focus,a.bg-primary:hover{background-color:#286090}.bg-success{background-color:#dff0d8}a.bg-success:focus,a.bg-success:hover{background-color:#c1e2b3}.bg-info{background-color:#d9edf7}a.bg-info:focus,a.bg-info:hover{background-color:#afd9ee}.bg-warning{background-color:#fcf8e3}a.bg-warning:focus,a.bg-warning:hover{background-color:#f7ecb5}.bg-danger{background-color:#f2dede}a.bg-danger:focus,a.bg-danger:hover{background-color:#e4b9b9}.page-header{padding-bottom:9px;margin:40px 0 20px;border-bottom:1px solid #eee}ol,ul{margin-top:0;margin-bottom:10px}ol ol,ol ul,ul ol,ul ul{margin-bottom:0}.list-unstyled{padding-left:0;list-style:none}.list-inline{padding-left:0;margin-left:-5px;list-style:none}.list-inline>li{display:inline-block;padding-right:5px;padding-left:5px}dl{margin-top:0;margin-bottom:20px}dd,dt{line-height:1.42857143}dt{font-weight:700}dd{margin-left:0}@media (min-width:768px){.dl-horizontal dt{float:left;width:160px;overflow:hidden;clear:left;text-align:right;text-overflow:ellipsis;white-space:nowrap}.dl-horizontal dd{margin-left:180px}}abbr[data-original-title],abbr[title]{cursor:help;border-bottom:1px dotted #777}.initialism{font-size:90%;text-transform:uppercase}blockquote{padding:10px 20px;margin:0 0 20px;font-size:17.5px;border-left:5px solid #eee}blockquote ol:last-child,blockquote p:last-child,blockquote ul:last-child{margin-bottom:0}blockquote .small,blockquote footer,blockquote small{display:block;font-size:80%;line-height:1.42857143;color:#777}blockquote .small:before,blockquote footer:before,blockquote small:before{content:'\2014 \00A0'}.blockquote-reverse,blockquote.pull-right{padding-right:15px;padding-left:0;text-align:right;border-right:5px solid #eee;border-left:0}.blockquote-reverse .small:before,.blockquote-reverse footer:before,.blockquote-reverse small:before,blockquote.pull-right .small:before,blockquote.pull-right footer:before,blockquote.pull-right small:before{content:''}.blockquote-reverse .small:after,.blockquote-reverse footer:after,.blockquote-reverse small:after,blockquote.pull-right .small:after,blockquote.pull-right footer:after,blockquote.pull-right small:after{content:'\00A0 \2014'}address{margin-bottom:20px;font-style:normal;line-height:1.42857143}code,kbd,pre,samp{font-family:Menlo,Monaco,Consolas,"Courier New",monospace}code{padding:2px 4px;font-size:90%;color:#c7254e;background-color:#f9f2f4;border-radius:4px}kbd{padding:2px 4px;font-size:90%;color:#fff;background-color:#333;border-radius:3px;-webkit-box-shadow:inset 0 -1px 0 rgba(0,0,0,.25);box-shadow:inset 0 -1px 0 rgba(0,0,0,.25)}kbd kbd{padding:0;font-size:100%;font-weight:700;-webkit-box-shadow:none;box-shadow:none}pre{display:block;padding:9.5px;margin:0 0 10px;font-size:13px;line-height:1.42857143;color:#333;word-break:break-all;word-wrap:break-word;background-color:#f5f5f5;border:1px solid #ccc;border-radius:4px}pre code{padding:0;font-size:inherit;color:inherit;white-space:pre-wrap;background-color:transparent;border-radius:0}.pre-scrollable{max-height:340px;overflow-y:scroll}.container{padding-right:15px;padding-left:15px;margin-right:auto;margin-left:auto}@media (min-width:768px){.container{width:750px}}@media (min-width:992px){.container{width:970px}}@media (min-width:1200px){.container{width:1170px}}.container-fluid{padding-right:15px;padding-left:15px;margin-right:auto;margin-left:auto}.row{margin-right:-15px;margin-left:-15px}.col-lg-1,.col-lg-10,.col-lg-11,.col-lg-12,.col-lg-2,.col-lg-3,.col-lg-4,.col-lg-5,.col-lg-6,.col-lg-7,.col-lg-8,.col-lg-9,.col-md-1,.col-md-10,.col-md-11,.col-md-12,.col-md-2,.col-md-3,.col-md-4,.col-md-5,.col-md-6,.col-md-7,.col-md-8,.col-md-9,.col-sm-1,.col-sm-10,.col-sm-11,.col-sm-12,.col-sm-2,.col-sm-3,.col-sm-4,.col-sm-5,.col-sm-6,.col-sm-7,.col-sm-8,.col-sm-9,.col-xs-1,.col-xs-10,.col-xs-11,.col-xs-12,.col-xs-2,.col-xs-3,.col-xs-4,.col-xs-5,.col-xs-6,.col-xs-7,.col-xs-8,.col-xs-9{position:relative;min-height:1px;padding-right:15px;padding-left:15px}.col-xs-1,.col-xs-10,.col-xs-11,.col-xs-12,.col-xs-2,.col-xs-3,.col-xs-4,.col-xs-5,.col-xs-6,.col-xs-7,.col-xs-8,.col-xs-9{float:left}.col-xs-12{width:100%}.col-xs-11{width:91.66666667%}.col-xs-10{width:83.33333333%}.col-xs-9{width:75%}.col-xs-8{width:66.66666667%}.col-xs-7{width:58.33333333%}.col-xs-6{width:50%}.col-xs-5{width:41.66666667%}.col-xs-4{width:33.33333333%}.col-xs-3{width:25%}.col-xs-2{width:16.66666667%}.col-xs-1{width:8.33333333%}.col-xs-pull-12{right:100%}.col-xs-pull-11{right:91.66666667%}.col-xs-pull-10{right:83.33333333%}.col-xs-pull-9{right:75%}.col-xs-pull-8{right:66.66666667%}.col-xs-pull-7{right:58.33333333%}.col-xs-pull-6{right:50%}.col-xs-pull-5{right:41.66666667%}.col-xs-pull-4{right:33.33333333%}.col-xs-pull-3{right:25%}.col-xs-pull-2{right:16.66666667%}.col-xs-pull-1{right:8.33333333%}.col-xs-pull-0{right:auto}.col-xs-push-12{left:100%}.col-xs-push-11{left:91.66666667%}.col-xs-push-10{left:83.33333333%}.col-xs-push-9{left:75%}.col-xs-push-8{left:66.66666667%}.col-xs-push-7{left:58.33333333%}.col-xs-push-6{left:50%}.col-xs-push-5{left:41.66666667%}.col-xs-push-4{left:33.33333333%}.col-xs-push-3{left:25%}.col-xs-push-2{left:16.66666667%}.col-xs-push-1{left:8.33333333%}.col-xs-push-0{left:auto}.col-xs-offset-12{margin-left:100%}.col-xs-offset-11{margin-left:91.66666667%}.col-xs-offset-10{margin-left:83.33333333%}.col-xs-offset-9{margin-left:75%}.col-xs-offset-8{margin-left:66.66666667%}.col-xs-offset-7{margin-left:58.33333333%}.col-xs-offset-6{margin-left:50%}.col-xs-offset-5{margin-left:41.66666667%}.col-xs-offset-4{margin-left:33.33333333%}.col-xs-offset-3{margin-left:25%}.col-xs-offset-2{margin-left:16.66666667%}.col-xs-offset-1{margin-left:8.33333333%}.col-xs-offset-0{margin-left:0}@media (min-width:768px){.col-sm-1,.col-sm-10,.col-sm-11,.col-sm-12,.col-sm-2,.col-sm-3,.col-sm-4,.col-sm-5,.col-sm-6,.col-sm-7,.col-sm-8,.col-sm-9{float:left}.col-sm-12{width:100%}.col-sm-11{width:91.66666667%}.col-sm-10{width:83.33333333%}.col-sm-9{width:75%}.col-sm-8{width:66.66666667%}.col-sm-7{width:58.33333333%}.col-sm-6{width:50%}.col-sm-5{width:41.66666667%}.col-sm-4{width:33.33333333%}.col-sm-3{width:25%}.col-sm-2{width:16.66666667%}.col-sm-1{width:8.33333333%}.col-sm-pull-12{right:100%}.col-sm-pull-11{right:91.66666667%}.col-sm-pull-10{right:83.33333333%}.col-sm-pull-9{right:75%}.col-sm-pull-8{right:66.66666667%}.col-sm-pull-7{right:58.33333333%}.col-sm-pull-6{right:50%}.col-sm-pull-5{right:41.66666667%}.col-sm-pull-4{right:33.33333333%}.col-sm-pull-3{right:25%}.col-sm-pull-2{right:16.66666667%}.col-sm-pull-1{right:8.33333333%}.col-sm-pull-0{right:auto}.col-sm-push-12{left:100%}.col-sm-push-11{left:91.66666667%}.col-sm-push-10{left:83.33333333%}.col-sm-push-9{left:75%}.col-sm-push-8{left:66.66666667%}.col-sm-push-7{left:58.33333333%}.col-sm-push-6{left:50%}.col-sm-push-5{left:41.66666667%}.col-sm-push-4{left:33.33333333%}.col-sm-push-3{left:25%}.col-sm-push-2{left:16.66666667%}.col-sm-push-1{left:8.33333333%}.col-sm-push-0{left:auto}.col-sm-offset-12{margin-left:100%}.col-sm-offset-11{margin-left:91.66666667%}.col-sm-offset-10{margin-left:83.33333333%}.col-sm-offset-9{margin-left:75%}.col-sm-offset-8{margin-left:66.66666667%}.col-sm-offset-7{margin-left:58.33333333%}.col-sm-offset-6{margin-left:50%}.col-sm-offset-5{margin-left:41.66666667%}.col-sm-offset-4{margin-left:33.33333333%}.col-sm-offset-3{margin-left:25%}.col-sm-offset-2{margin-left:16.66666667%}.col-sm-offset-1{margin-left:8.33333333%}.col-sm-offset-0{margin-left:0}}@media (min-width:992px){.col-md-1,.col-md-10,.col-md-11,.col-md-12,.col-md-2,.col-md-3,.col-md-4,.col-md-5,.col-md-6,.col-md-7,.col-md-8,.col-md-9{float:left}.col-md-12{width:100%}.col-md-11{width:91.66666667%}.col-md-10{width:83.33333333%}.col-md-9{width:75%}.col-md-8{width:66.66666667%}.col-md-7{width:58.33333333%}.col-md-6{width:50%}.col-md-5{width:41.66666667%}.col-md-4{width:33.33333333%}.col-md-3{width:25%}.col-md-2{width:16.66666667%}.col-md-1{width:8.33333333%}.col-md-pull-12{right:100%}.col-md-pull-11{right:91.66666667%}.col-md-pull-10{right:83.33333333%}.col-md-pull-9{right:75%}.col-md-pull-8{right:66.66666667%}.col-md-pull-7{right:58.33333333%}.col-md-pull-6{right:50%}.col-md-pull-5{right:41.66666667%}.col-md-pull-4{right:33.33333333%}.col-md-pull-3{right:25%}.col-md-pull-2{right:16.66666667%}.col-md-pull-1{right:8.33333333%}.col-md-pull-0{right:auto}.col-md-push-12{left:100%}.col-md-push-11{left:91.66666667%}.col-md-push-10{left:83.33333333%}.col-md-push-9{left:75%}.col-md-push-8{left:66.66666667%}.col-md-push-7{left:58.33333333%}.col-md-push-6{left:50%}.col-md-push-5{left:41.66666667%}.col-md-push-4{left:33.33333333%}.col-md-push-3{left:25%}.col-md-push-2{left:16.66666667%}.col-md-push-1{left:8.33333333%}.col-md-push-0{left:auto}.col-md-offset-12{margin-left:100%}.col-md-offset-11{margin-left:91.66666667%}.col-md-offset-10{margin-left:83.33333333%}.col-md-offset-9{margin-left:75%}.col-md-offset-8{margin-left:66.66666667%}.col-md-offset-7{margin-left:58.33333333%}.col-md-offset-6{margin-left:50%}.col-md-offset-5{margin-left:41.66666667%}.col-md-offset-4{margin-left:33.33333333%}.col-md-offset-3{margin-left:25%}.col-md-offset-2{margin-left:16.66666667%}.col-md-offset-1{margin-left:8.33333333%}.col-md-offset-0{margin-left:0}}@media (min-width:1200px){.col-lg-1,.col-lg-10,.col-lg-11,.col-lg-12,.col-lg-2,.col-lg-3,.col-lg-4,.col-lg-5,.col-lg-6,.col-lg-7,.col-lg-8,.col-lg-9{float:left}.col-lg-12{width:100%}.col-lg-11{width:91.66666667%}.col-lg-10{width:83.33333333%}.col-lg-9{width:75%}.col-lg-8{width:66.66666667%}.col-lg-7{width:58.33333333%}.col-lg-6{width:50%}.col-lg-5{width:41.66666667%}.col-lg-4{width:33.33333333%}.col-lg-3{width:25%}.col-lg-2{width:16.66666667%}.col-lg-1{width:8.33333333%}.col-lg-pull-12{right:100%}.col-lg-pull-11{right:91.66666667%}.col-lg-pull-10{right:83.33333333%}.col-lg-pull-9{right:75%}.col-lg-pull-8{right:66.66666667%}.col-lg-pull-7{right:58.33333333%}.col-lg-pull-6{right:50%}.col-lg-pull-5{right:41.66666667%}.col-lg-pull-4{right:33.33333333%}.col-lg-pull-3{right:25%}.col-lg-pull-2{right:16.66666667%}.col-lg-pull-1{right:8.33333333%}.col-lg-pull-0{right:auto}.col-lg-push-12{left:100%}.col-lg-push-11{left:91.66666667%}.col-lg-push-10{left:83.33333333%}.col-lg-push-9{left:75%}.col-lg-push-8{left:66.66666667%}.col-lg-push-7{left:58.33333333%}.col-lg-push-6{left:50%}.col-lg-push-5{left:41.66666667%}.col-lg-push-4{left:33.33333333%}.col-lg-push-3{left:25%}.col-lg-push-2{left:16.66666667%}.col-lg-push-1{left:8.33333333%}.col-lg-push-0{left:auto}.col-lg-offset-12{margin-left:100%}.col-lg-offset-11{margin-left:91.66666667%}.col-lg-offset-10{margin-left:83.33333333%}.col-lg-offset-9{margin-left:75%}.col-lg-offset-8{margin-left:66.66666667%}.col-lg-offset-7{margin-left:58.33333333%}.col-lg-offset-6{margin-left:50%}.col-lg-offset-5{margin-left:41.66666667%}.col-lg-offset-4{margin-left:33.33333333%}.col-lg-offset-3{margin-left:25%}.col-lg-offset-2{margin-left:16.66666667%}.col-lg-offset-1{margin-left:8.33333333%}.col-lg-offset-0{margin-left:0}}table{background-color:transparent}caption{padding-top:8px;padding-bottom:8px;color:#777;text-align:left}th{text-align:left}.table{width:100%;max-width:100%;margin-bottom:20px}.table>tbody>tr>td,.table>tbody>tr>th,.table>tfoot>tr>td,.table>tfoot>tr>th,.table>thead>tr>td,.table>thead>tr>th{padding:8px;line-height:1.42857143;vertical-align:top;border-top:1px solid #ddd}.table>thead>tr>th{vertical-align:bottom;border-bottom:2px solid #ddd}.table>caption+thead>tr:first-child>td,.table>caption+thead>tr:first-child>th,.table>colgroup+thead>tr:first-child>td,.table>colgroup+thead>tr:first-child>th,.table>thead:first-child>tr:first-child>td,.table>thead:first-child>tr:first-child>th{border-top:0}.table>tbody+tbody{border-top:2px solid #ddd}.table .table{background-color:#fff}.table-condensed>tbody>tr>td,.table-condensed>tbody>tr>th,.table-condensed>tfoot>tr>td,.table-condensed>tfoot>tr>th,.table-condensed>thead>tr>td,.table-condensed>thead>tr>th{padding:5px}.table-bordered{border:1px solid #ddd}.table-bordered>tbody>tr>td,.table-bordered>tbody>tr>th,.table-bordered>tfoot>tr>td,.table-bordered>tfoot>tr>th,.table-bordered>thead>tr>td,.table-bordered>thead>tr>th{border:1px solid #ddd}.table-bordered>thead>tr>td,.table-bordered>thead>tr>th{border-bottom-width:2px}.table-striped>tbody>tr:nth-of-type(odd){background-color:#f9f9f9}.table-hover>tbody>tr:hover{background-color:#f5f5f5}table col[class*=col-]{position:static;display:table-column;float:none}table td[class*=col-],table th[class*=col-]{position:static;display:table-cell;float:none}.table>tbody>tr.active>td,.table>tbody>tr.active>th,.table>tbody>tr>td.active,.table>tbody>tr>th.active,.table>tfoot>tr.active>td,.table>tfoot>tr.active>th,.table>tfoot>tr>td.active,.table>tfoot>tr>th.active,.table>thead>tr.active>td,.table>thead>tr.active>th,.table>thead>tr>td.active,.table>thead>tr>th.active{background-color:#f5f5f5}.table-hover>tbody>tr.active:hover>td,.table-hover>tbody>tr.active:hover>th,.table-hover>tbody>tr:hover>.active,.table-hover>tbody>tr>td.active:hover,.table-hover>tbody>tr>th.active:hover{background-color:#e8e8e8}.table>tbody>tr.success>td,.table>tbody>tr.success>th,.table>tbody>tr>td.success,.table>tbody>tr>th.success,.table>tfoot>tr.success>td,.table>tfoot>tr.success>th,.table>tfoot>tr>td.success,.table>tfoot>tr>th.success,.table>thead>tr.success>td,.table>thead>tr.success>th,.table>thead>tr>td.success,.table>thead>tr>th.success{background-color:#dff0d8}.table-hover>tbody>tr.success:hover>td,.table-hover>tbody>tr.success:hover>th,.table-hover>tbody>tr:hover>.success,.table-hover>tbody>tr>td.success:hover,.table-hover>tbody>tr>th.success:hover{background-color:#d0e9c6}.table>tbody>tr.info>td,.table>tbody>tr.info>th,.table>tbody>tr>td.info,.table>tbody>tr>th.info,.table>tfoot>tr.info>td,.table>tfoot>tr.info>th,.table>tfoot>tr>td.info,.table>tfoot>tr>th.info,.table>thead>tr.info>td,.table>thead>tr.info>th,.table>thead>tr>td.info,.table>thead>tr>th.info{background-color:#d9edf7}.table-hover>tbody>tr.info:hover>td,.table-hover>tbody>tr.info:hover>th,.table-hover>tbody>tr:hover>.info,.table-hover>tbody>tr>td.info:hover,.table-hover>tbody>tr>th.info:hover{background-color:#c4e3f3}.table>tbody>tr.warning>td,.table>tbody>tr.warning>th,.table>tbody>tr>td.warning,.table>tbody>tr>th.warning,.table>tfoot>tr.warning>td,.table>tfoot>tr.warning>th,.table>tfoot>tr>td.warning,.table>tfoot>tr>th.warning,.table>thead>tr.warning>td,.table>thead>tr.warning>th,.table>thead>tr>td.warning,.table>thead>tr>th.warning{background-color:#fcf8e3}.table-hover>tbody>tr.warning:hover>td,.table-hover>tbody>tr.warning:hover>th,.table-hover>tbody>tr:hover>.warning,.table-hover>tbody>tr>td.warning:hover,.table-hover>tbody>tr>th.warning:hover{background-color:#faf2cc}.table>tbody>tr.danger>td,.table>tbody>tr.danger>th,.table>tbody>tr>td.danger,.table>tbody>tr>th.danger,.table>tfoot>tr.danger>td,.table>tfoot>tr.danger>th,.table>tfoot>tr>td.danger,.table>tfoot>tr>th.danger,.table>thead>tr.danger>td,.table>thead>tr.danger>th,.table>thead>tr>td.danger,.table>thead>tr>th.danger{background-color:#f2dede}.table-hover>tbody>tr.danger:hover>td,.table-hover>tbody>tr.danger:hover>th,.table-hover>tbody>tr:hover>.danger,.table-hover>tbody>tr>td.danger:hover,.table-hover>tbody>tr>th.danger:hover{background-color:#ebcccc}.table-responsive{min-height:.01%;overflow-x:auto}@media screen and (max-width:767px){.table-responsive{width:100%;margin-bottom:15px;overflow-y:hidden;-ms-overflow-style:-ms-autohiding-scrollbar;border:1px solid #ddd}.table-responsive>.table{margin-bottom:0}.table-responsive>.table>tbody>tr>td,.table-responsive>.table>tbody>tr>th,.table-responsive>.table>tfoot>tr>td,.table-responsive>.table>tfoot>tr>th,.table-responsive>.table>thead>tr>td,.table-responsive>.table>thead>tr>th{white-space:nowrap}.table-responsive>.table-bordered{border:0}.table-responsive>.table-bordered>tbody>tr>td:first-child,.table-responsive>.table-bordered>tbody>tr>th:first-child,.table-responsive>.table-bordered>tfoot>tr>td:first-child,.table-responsive>.table-bordered>tfoot>tr>th:first-child,.table-responsive>.table-bordered>thead>tr>td:first-child,.table-responsive>.table-bordered>thead>tr>th:first-child{border-left:0}.table-responsive>.table-bordered>tbody>tr>td:last-child,.table-responsive>.table-bordered>tbody>tr>th:last-child,.table-responsive>.table-bordered>tfoot>tr>td:last-child,.table-responsive>.table-bordered>tfoot>tr>th:last-child,.table-responsive>.table-bordered>thead>tr>td:last-child,.table-responsive>.table-bordered>thead>tr>th:last-child{border-right:0}.table-responsive>.table-bordered>tbody>tr:last-child>td,.table-responsive>.table-bordered>tbody>tr:last-child>th,.table-responsive>.table-bordered>tfoot>tr:last-child>td,.table-responsive>.table-bordered>tfoot>tr:last-child>th{border-bottom:0}}fieldset{min-width:0;padding:0;margin:0;border:0}legend{display:block;width:100%;padding:0;margin-bottom:20px;font-size:21px;line-height:inherit;color:#333;border:0;border-bottom:1px solid #e5e5e5}label{display:inline-block;max-width:100%;margin-bottom:5px;font-weight:700}input[type=search]{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}input[type=checkbox],input[type=radio]{margin:4px 0 0;margin-top:1px\9;line-height:normal}input[type=file]{display:block}input[type=range]{display:block;width:100%}select[multiple],select[size]{height:auto}input[type=file]:focus,input[type=checkbox]:focus,input[type=radio]:focus{outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}output{display:block;padding-top:7px;font-size:14px;line-height:1.42857143;color:#555}.form-control{display:block;width:100%;height:34px;padding:6px 12px;font-size:14px;line-height:1.42857143;color:#555;background-color:#fff;background-image:none;border:1px solid #ccc;border-radius:4px;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075);box-shadow:inset 0 1px 1px rgba(0,0,0,.075);-webkit-transition:border-color ease-in-out .15s,-webkit-box-shadow ease-in-out .15s;-o-transition:border-color ease-in-out .15s,box-shadow ease-in-out .15s;transition:border-color ease-in-out .15s,box-shadow ease-in-out .15s}.form-control:focus{border-color:#66afe9;outline:0;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 8px rgba(102,175,233,.6);box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 8px rgba(102,175,233,.6)}.form-control::-moz-placeholder{color:#999;opacity:1}.form-control:-ms-input-placeholder{color:#999}.form-control::-webkit-input-placeholder{color:#999}.form-control::-ms-expand{background-color:transparent;border:0}.form-control[disabled],.form-control[readonly],fieldset[disabled] .form-control{background-color:#eee;opacity:1}.form-control[disabled],fieldset[disabled] .form-control{cursor:not-allowed}textarea.form-control{height:auto}input[type=search]{-webkit-appearance:none}@media screen and (-webkit-min-device-pixel-ratio:0){input[type=date].form-control,input[type=time].form-control,input[type=datetime-local].form-control,input[type=month].form-control{line-height:34px}.input-group-sm input[type=date],.input-group-sm input[type=time],.input-group-sm input[type=datetime-local],.input-group-sm input[type=month],input[type=date].input-sm,input[type=time].input-sm,input[type=datetime-local].input-sm,input[type=month].input-sm{line-height:30px}.input-group-lg input[type=date],.input-group-lg input[type=time],.input-group-lg input[type=datetime-local],.input-group-lg input[type=month],input[type=date].input-lg,input[type=time].input-lg,input[type=datetime-local].input-lg,input[type=month].input-lg{line-height:46px}}.form-group{margin-bottom:15px}.checkbox,.radio{position:relative;display:block;margin-top:10px;margin-bottom:10px}.checkbox label,.radio label{min-height:20px;padding-left:20px;margin-bottom:0;font-weight:400;cursor:pointer}.checkbox input[type=checkbox],.checkbox-inline input[type=checkbox],.radio input[type=radio],.radio-inline input[type=radio]{position:absolute;margin-top:4px\9;margin-left:-20px}.checkbox+.checkbox,.radio+.radio{margin-top:-5px}.checkbox-inline,.radio-inline{position:relative;display:inline-block;padding-left:20px;margin-bottom:0;font-weight:400;vertical-align:middle;cursor:pointer}.checkbox-inline+.checkbox-inline,.radio-inline+.radio-inline{margin-top:0;margin-left:10px}fieldset[disabled] input[type=checkbox],fieldset[disabled] input[type=radio],input[type=checkbox].disabled,input[type=checkbox][disabled],input[type=radio].disabled,input[type=radio][disabled]{cursor:not-allowed}.checkbox-inline.disabled,.radio-inline.disabled,fieldset[disabled] .checkbox-inline,fieldset[disabled] .radio-inline{cursor:not-allowed}.checkbox.disabled label,.radio.disabled label,fieldset[disabled] .checkbox label,fieldset[disabled] .radio label{cursor:not-allowed}.form-control-static{min-height:34px;padding-top:7px;padding-bottom:7px;margin-bottom:0}.form-control-static.input-lg,.form-control-static.input-sm{padding-right:0;padding-left:0}.input-sm{height:30px;padding:5px 10px;font-size:12px;line-height:1.5;border-radius:3px}select.input-sm{height:30px;line-height:30px}select[multiple].input-sm,textarea.input-sm{height:auto}.form-group-sm .form-control{height:30px;padding:5px 10px;font-size:12px;line-height:1.5;border-radius:3px}.form-group-sm select.form-control{height:30px;line-height:30px}.form-group-sm select[multiple].form-control,.form-group-sm textarea.form-control{height:auto}.form-group-sm .form-control-static{height:30px;min-height:32px;padding:6px 10px;font-size:12px;line-height:1.5}.input-lg{height:46px;padding:10px 16px;font-size:18px;line-height:1.3333333;border-radius:6px}select.input-lg{height:46px;line-height:46px}select[multiple].input-lg,textarea.input-lg{height:auto}.form-group-lg .form-control{height:46px;padding:10px 16px;font-size:18px;line-height:1.3333333;border-radius:6px}.form-group-lg select.form-control{height:46px;line-height:46px}.form-group-lg select[multiple].form-control,.form-group-lg textarea.form-control{height:auto}.form-group-lg .form-control-static{height:46px;min-height:38px;padding:11px 16px;font-size:18px;line-height:1.3333333}.has-feedback{position:relative}.has-feedback .form-control{padding-right:42.5px}.form-control-feedback{position:absolute;top:0;right:0;z-index:2;display:block;width:34px;height:34px;line-height:34px;text-align:center;pointer-events:none}.form-group-lg .form-control+.form-control-feedback,.input-group-lg+.form-control-feedback,.input-lg+.form-control-feedback{width:46px;height:46px;line-height:46px}.form-group-sm .form-control+.form-control-feedback,.input-group-sm+.form-control-feedback,.input-sm+.form-control-feedback{width:30px;height:30px;line-height:30px}.has-success .checkbox,.has-success .checkbox-inline,.has-success .control-label,.has-success .help-block,.has-success .radio,.has-success .radio-inline,.has-success.checkbox label,.has-success.checkbox-inline label,.has-success.radio label,.has-success.radio-inline label{color:#3c763d}.has-success .form-control{border-color:#3c763d;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075);box-shadow:inset 0 1px 1px rgba(0,0,0,.075)}.has-success .form-control:focus{border-color:#2b542c;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 6px #67b168;box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 6px #67b168}.has-success .input-group-addon{color:#3c763d;background-color:#dff0d8;border-color:#3c763d}.has-success .form-control-feedback{color:#3c763d}.has-warning .checkbox,.has-warning .checkbox-inline,.has-warning .control-label,.has-warning .help-block,.has-warning .radio,.has-warning .radio-inline,.has-warning.checkbox label,.has-warning.checkbox-inline label,.has-warning.radio label,.has-warning.radio-inline label{color:#8a6d3b}.has-warning .form-control{border-color:#8a6d3b;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075);box-shadow:inset 0 1px 1px rgba(0,0,0,.075)}.has-warning .form-control:focus{border-color:#66512c;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 6px #c0a16b;box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 6px #c0a16b}.has-warning .input-group-addon{color:#8a6d3b;background-color:#fcf8e3;border-color:#8a6d3b}.has-warning .form-control-feedback{color:#8a6d3b}.has-error .checkbox,.has-error .checkbox-inline,.has-error .control-label,.has-error .help-block,.has-error .radio,.has-error .radio-inline,.has-error.checkbox label,.has-error.checkbox-inline label,.has-error.radio label,.has-error.radio-inline label{color:#a94442}.has-error .form-control{border-color:#a94442;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075);box-shadow:inset 0 1px 1px rgba(0,0,0,.075)}.has-error .form-control:focus{border-color:#843534;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 6px #ce8483;box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 6px #ce8483}.has-error .input-group-addon{color:#a94442;background-color:#f2dede;border-color:#a94442}.has-error .form-control-feedback{color:#a94442}.has-feedback label~.form-control-feedback{top:25px}.has-feedback label.sr-only~.form-control-feedback{top:0}.help-block{display:block;margin-top:5px;margin-bottom:10px;color:#737373}@media (min-width:768px){.form-inline .form-group{display:inline-block;margin-bottom:0;vertical-align:middle}.form-inline .form-control{display:inline-block;width:auto;vertical-align:middle}.form-inline .form-control-static{display:inline-block}.form-inline .input-group{display:inline-table;vertical-align:middle}.form-inline .input-group .form-control,.form-inline .input-group .input-group-addon,.form-inline .input-group .input-group-btn{width:auto}.form-inline .input-group>.form-control{width:100%}.form-inline .control-label{margin-bottom:0;vertical-align:middle}.form-inline .checkbox,.form-inline .radio{display:inline-block;margin-top:0;margin-bottom:0;vertical-align:middle}.form-inline .checkbox label,.form-inline .radio label{padding-left:0}.form-inline .checkbox input[type=checkbox],.form-inline .radio input[type=radio]{position:relative;margin-left:0}.form-inline .has-feedback .form-control-feedback{top:0}}.form-horizontal .checkbox,.form-horizontal .checkbox-inline,.form-horizontal .radio,.form-horizontal .radio-inline{padding-top:7px;margin-top:0;margin-bottom:0}.form-horizontal .checkbox,.form-horizontal .radio{min-height:27px}.form-horizontal .form-group{margin-right:-15px;margin-left:-15px}@media (min-width:768px){.form-horizontal .control-label{padding-top:7px;margin-bottom:0;text-align:right}}.form-horizontal .has-feedback .form-control-feedback{right:15px}@media (min-width:768px){.form-horizontal .form-group-lg .control-label{padding-top:11px;font-size:18px}}@media (min-width:768px){.form-horizontal .form-group-sm .control-label{padding-top:6px;font-size:12px}}.btn{display:inline-block;padding:6px 12px;margin-bottom:0;font-size:14px;font-weight:400;line-height:1.42857143;text-align:center;white-space:nowrap;vertical-align:middle;-ms-touch-action:manipulation;touch-action:manipulation;cursor:pointer;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;background-image:none;border:1px solid transparent;border-radius:4px}.btn.active.focus,.btn.active:focus,.btn.focus,.btn:active.focus,.btn:active:focus,.btn:focus{outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}.btn.focus,.btn:focus,.btn:hover{color:#333;text-decoration:none}.btn.active,.btn:active{background-image:none;outline:0;-webkit-box-shadow:inset 0 3px 5px rgba(0,0,0,.125);box-shadow:inset 0 3px 5px rgba(0,0,0,.125)}.btn.disabled,.btn[disabled],fieldset[disabled] .btn{cursor:not-allowed;filter:alpha(opacity=65);-webkit-box-shadow:none;box-shadow:none;opacity:.65}a.btn.disabled,fieldset[disabled] a.btn{pointer-events:none}.btn-default{color:#333;background-color:#fff;border-color:#ccc}.btn-default.focus,.btn-default:focus{color:#333;background-color:#e6e6e6;border-color:#8c8c8c}.btn-default:hover{color:#333;background-color:#e6e6e6;border-color:#adadad}.btn-default.active,.btn-default:active,.open>.dropdown-toggle.btn-default{color:#333;background-color:#e6e6e6;border-color:#adadad}.btn-default.active.focus,.btn-default.active:focus,.btn-default.active:hover,.btn-default:active.focus,.btn-default:active:focus,.btn-default:active:hover,.open>.dropdown-toggle.btn-default.focus,.open>.dropdown-toggle.btn-default:focus,.open>.dropdown-toggle.btn-default:hover{color:#333;background-color:#d4d4d4;border-color:#8c8c8c}.btn-default.active,.btn-default:active,.open>.dropdown-toggle.btn-default{background-image:none}.btn-default.disabled.focus,.btn-default.disabled:focus,.btn-default.disabled:hover,.btn-default[disabled].focus,.btn-default[disabled]:focus,.btn-default[disabled]:hover,fieldset[disabled] .btn-default.focus,fieldset[disabled] .btn-default:focus,fieldset[disabled] .btn-default:hover{background-color:#fff;border-color:#ccc}.btn-default .badge{color:#fff;background-color:#333}.btn-primary{color:#fff;background-color:#337ab7;border-color:#2e6da4}.btn-primary.focus,.btn-primary:focus{color:#fff;background-color:#286090;border-color:#122b40}.btn-primary:hover{color:#fff;background-color:#286090;border-color:#204d74}.btn-primary.active,.btn-primary:active,.open>.dropdown-toggle.btn-primary{color:#fff;background-color:#286090;border-color:#204d74}.btn-primary.active.focus,.btn-primary.active:focus,.btn-primary.active:hover,.btn-primary:active.focus,.btn-primary:active:focus,.btn-primary:active:hover,.open>.dropdown-toggle.btn-primary.focus,.open>.dropdown-toggle.btn-primary:focus,.open>.dropdown-toggle.btn-primary:hover{color:#fff;background-color:#204d74;border-color:#122b40}.btn-primary.active,.btn-primary:active,.open>.dropdown-toggle.btn-primary{background-image:none}.btn-primary.disabled.focus,.btn-primary.disabled:focus,.btn-primary.disabled:hover,.btn-primary[disabled].focus,.btn-primary[disabled]:focus,.btn-primary[disabled]:hover,fieldset[disabled] .btn-primary.focus,fieldset[disabled] .btn-primary:focus,fieldset[disabled] .btn-primary:hover{background-color:#337ab7;border-color:#2e6da4}.btn-primary .badge{color:#337ab7;background-color:#fff}.btn-success{color:#fff;background-color:#5cb85c;border-color:#4cae4c}.btn-success.focus,.btn-success:focus{color:#fff;background-color:#449d44;border-color:#255625}.btn-success:hover{color:#fff;background-color:#449d44;border-color:#398439}.btn-success.active,.btn-success:active,.open>.dropdown-toggle.btn-success{color:#fff;background-color:#449d44;border-color:#398439}.btn-success.active.focus,.btn-success.active:focus,.btn-success.active:hover,.btn-success:active.focus,.btn-success:active:focus,.btn-success:active:hover,.open>.dropdown-toggle.btn-success.focus,.open>.dropdown-toggle.btn-success:focus,.open>.dropdown-toggle.btn-success:hover{color:#fff;background-color:#398439;border-color:#255625}.btn-success.active,.btn-success:active,.open>.dropdown-toggle.btn-success{background-image:none}.btn-success.disabled.focus,.btn-success.disabled:focus,.btn-success.disabled:hover,.btn-success[disabled].focus,.btn-success[disabled]:focus,.btn-success[disabled]:hover,fieldset[disabled] .btn-success.focus,fieldset[disabled] .btn-success:focus,fieldset[disabled] .btn-success:hover{background-color:#5cb85c;border-color:#4cae4c}.btn-success .badge{color:#5cb85c;background-color:#fff}.btn-info{color:#fff;background-color:#5bc0de;border-color:#46b8da}.btn-info.focus,.btn-info:focus{color:#fff;background-color:#31b0d5;border-color:#1b6d85}.btn-info:hover{color:#fff;background-color:#31b0d5;border-color:#269abc}.btn-info.active,.btn-info:active,.open>.dropdown-toggle.btn-info{color:#fff;background-color:#31b0d5;border-color:#269abc}.btn-info.active.focus,.btn-info.active:focus,.btn-info.active:hover,.btn-info:active.focus,.btn-info:active:focus,.btn-info:active:hover,.open>.dropdown-toggle.btn-info.focus,.open>.dropdown-toggle.btn-info:focus,.open>.dropdown-toggle.btn-info:hover{color:#fff;background-color:#269abc;border-color:#1b6d85}.btn-info.active,.btn-info:active,.open>.dropdown-toggle.btn-info{background-image:none}.btn-info.disabled.focus,.btn-info.disabled:focus,.btn-info.disabled:hover,.btn-info[disabled].focus,.btn-info[disabled]:focus,.btn-info[disabled]:hover,fieldset[disabled] .btn-info.focus,fieldset[disabled] .btn-info:focus,fieldset[disabled] .btn-info:hover{background-color:#5bc0de;border-color:#46b8da}.btn-info .badge{color:#5bc0de;background-color:#fff}.btn-warning{color:#fff;background-color:#f0ad4e;border-color:#eea236}.btn-warning.focus,.btn-warning:focus{color:#fff;background-color:#ec971f;border-color:#985f0d}.btn-warning:hover{color:#fff;background-color:#ec971f;border-color:#d58512}.btn-warning.active,.btn-warning:active,.open>.dropdown-toggle.btn-warning{color:#fff;background-color:#ec971f;border-color:#d58512}.btn-warning.active.focus,.btn-warning.active:focus,.btn-warning.active:hover,.btn-warning:active.focus,.btn-warning:active:focus,.btn-warning:active:hover,.open>.dropdown-toggle.btn-warning.focus,.open>.dropdown-toggle.btn-warning:focus,.open>.dropdown-toggle.btn-warning:hover{color:#fff;background-color:#d58512;border-color:#985f0d}.btn-warning.active,.btn-warning:active,.open>.dropdown-toggle.btn-warning{background-image:none}.btn-warning.disabled.focus,.btn-warning.disabled:focus,.btn-warning.disabled:hover,.btn-warning[disabled].focus,.btn-warning[disabled]:focus,.btn-warning[disabled]:hover,fieldset[disabled] .btn-warning.focus,fieldset[disabled] .btn-warning:focus,fieldset[disabled] .btn-warning:hover{background-color:#f0ad4e;border-color:#eea236}.btn-warning .badge{color:#f0ad4e;background-color:#fff}.btn-danger{color:#fff;background-color:#d9534f;border-color:#d43f3a}.btn-danger.focus,.btn-danger:focus{color:#fff;background-color:#c9302c;border-color:#761c19}.btn-danger:hover{color:#fff;background-color:#c9302c;border-color:#ac2925}.btn-danger.active,.btn-danger:active,.open>.dropdown-toggle.btn-danger{color:#fff;background-color:#c9302c;border-color:#ac2925}.btn-danger.active.focus,.btn-danger.active:focus,.btn-danger.active:hover,.btn-danger:active.focus,.btn-danger:active:focus,.btn-danger:active:hover,.open>.dropdown-toggle.btn-danger.focus,.open>.dropdown-toggle.btn-danger:focus,.open>.dropdown-toggle.btn-danger:hover{color:#fff;background-color:#ac2925;border-color:#761c19}.btn-danger.active,.btn-danger:active,.open>.dropdown-toggle.btn-danger{background-image:none}.btn-danger.disabled.focus,.btn-danger.disabled:focus,.btn-danger.disabled:hover,.btn-danger[disabled].focus,.btn-danger[disabled]:focus,.btn-danger[disabled]:hover,fieldset[disabled] .btn-danger.focus,fieldset[disabled] .btn-danger:focus,fieldset[disabled] .btn-danger:hover{background-color:#d9534f;border-color:#d43f3a}.btn-danger .badge{color:#d9534f;background-color:#fff}.btn-link{font-weight:400;color:#337ab7;border-radius:0}.btn-link,.btn-link.active,.btn-link:active,.btn-link[disabled],fieldset[disabled] .btn-link{background-color:transparent;-webkit-box-shadow:none;box-shadow:none}.btn-link,.btn-link:active,.btn-link:focus,.btn-link:hover{border-color:transparent}.btn-link:focus,.btn-link:hover{color:#23527c;text-decoration:underline;background-color:transparent}.btn-link[disabled]:focus,.btn-link[disabled]:hover,fieldset[disabled] .btn-link:focus,fieldset[disabled] .btn-link:hover{color:#777;text-decoration:none}.btn-group-lg>.btn,.btn-lg{padding:10px 16px;font-size:18px;line-height:1.3333333;border-radius:6px}.btn-group-sm>.btn,.btn-sm{padding:5px 10px;font-size:12px;line-height:1.5;border-radius:3px}.btn-group-xs>.btn,.btn-xs{padding:1px 5px;font-size:12px;line-height:1.5;border-radius:3px}.btn-block{display:block;width:100%}.btn-block+.btn-block{margin-top:5px}input[type=button].btn-block,input[type=reset].btn-block,input[type=submit].btn-block{width:100%}.fade{opacity:0;-webkit-transition:opacity .15s linear;-o-transition:opacity .15s linear;transition:opacity .15s linear}.fade.in{opacity:1}.collapse{display:none}.collapse.in{display:block}tr.collapse.in{display:table-row}tbody.collapse.in{display:table-row-group}.collapsing{position:relative;height:0;overflow:hidden;-webkit-transition-timing-function:ease;-o-transition-timing-function:ease;transition-timing-function:ease;-webkit-transition-duration:.35s;-o-transition-duration:.35s;transition-duration:.35s;-webkit-transition-property:height,visibility;-o-transition-property:height,visibility;transition-property:height,visibility}.caret{display:inline-block;width:0;height:0;margin-left:2px;vertical-align:middle;border-top:4px dashed;border-top:4px solid\9;border-right:4px solid transparent;border-left:4px solid transparent}.dropdown,.dropup{position:relative}.dropdown-toggle:focus{outline:0}.dropdown-menu{position:absolute;top:100%;left:0;z-index:1000;display:none;float:left;min-width:160px;padding:5px 0;margin:2px 0 0;font-size:14px;text-align:left;list-style:none;background-color:#fff;-webkit-background-clip:padding-box;background-clip:padding-box;border:1px solid #ccc;border:1px solid rgba(0,0,0,.15);border-radius:4px;-webkit-box-shadow:0 6px 12px rgba(0,0,0,.175);box-shadow:0 6px 12px rgba(0,0,0,.175)}.dropdown-menu.pull-right{right:0;left:auto}.dropdown-menu .divider{height:1px;margin:9px 0;overflow:hidden;background-color:#e5e5e5}.dropdown-menu>li>a{display:block;padding:3px 20px;clear:both;font-weight:400;line-height:1.42857143;color:#333;white-space:nowrap}.dropdown-menu>li>a:focus,.dropdown-menu>li>a:hover{color:#262626;text-decoration:none;background-color:#f5f5f5}.dropdown-menu>.active>a,.dropdown-menu>.active>a:focus,.dropdown-menu>.active>a:hover{color:#fff;text-decoration:none;background-color:#337ab7;outline:0}.dropdown-menu>.disabled>a,.dropdown-menu>.disabled>a:focus,.dropdown-menu>.disabled>a:hover{color:#777}.dropdown-menu>.disabled>a:focus,.dropdown-menu>.disabled>a:hover{text-decoration:none;cursor:not-allowed;background-color:transparent;background-image:none;filter:progid:DXImageTransform.Microsoft.gradient(enabled=false)}.open>.dropdown-menu{display:block}.open>a{outline:0}.dropdown-menu-right{right:0;left:auto}.dropdown-menu-left{right:auto;left:0}.dropdown-header{display:block;padding:3px 20px;font-size:12px;line-height:1.42857143;color:#777;white-space:nowrap}.dropdown-backdrop{position:fixed;top:0;right:0;bottom:0;left:0;z-index:990}.pull-right>.dropdown-menu{right:0;left:auto}.dropup .caret,.navbar-fixed-bottom .dropdown .caret{content:"";border-top:0;border-bottom:4px dashed;border-bottom:4px solid\9}.dropup .dropdown-menu,.navbar-fixed-bottom .dropdown .dropdown-menu{top:auto;bottom:100%;margin-bottom:2px}@media (min-width:768px){.navbar-right .dropdown-menu{right:0;left:auto}.navbar-right .dropdown-menu-left{right:auto;left:0}}.btn-group,.btn-group-vertical{position:relative;display:inline-block;vertical-align:middle}.btn-group-vertical>.btn,.btn-group>.btn{position:relative;float:left}.btn-group-vertical>.btn.active,.btn-group-vertical>.btn:active,.btn-group-vertical>.btn:focus,.btn-group-vertical>.btn:hover,.btn-group>.btn.active,.btn-group>.btn:active,.btn-group>.btn:focus,.btn-group>.btn:hover{z-index:2}.btn-group .btn+.btn,.btn-group .btn+.btn-group,.btn-group .btn-group+.btn,.btn-group .btn-group+.btn-group{margin-left:-1px}.btn-toolbar{margin-left:-5px}.btn-toolbar .btn,.btn-toolbar .btn-group,.btn-toolbar .input-group{float:left}.btn-toolbar>.btn,.btn-toolbar>.btn-group,.btn-toolbar>.input-group{margin-left:5px}.btn-group>.btn:not(:first-child):not(:last-child):not(.dropdown-toggle){border-radius:0}.btn-group>.btn:first-child{margin-left:0}.btn-group>.btn:first-child:not(:last-child):not(.dropdown-toggle){border-top-right-radius:0;border-bottom-right-radius:0}.btn-group>.btn:last-child:not(:first-child),.btn-group>.dropdown-toggle:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.btn-group>.btn-group{float:left}.btn-group>.btn-group:not(:first-child):not(:last-child)>.btn{border-radius:0}.btn-group>.btn-group:first-child:not(:last-child)>.btn:last-child,.btn-group>.btn-group:first-child:not(:last-child)>.dropdown-toggle{border-top-right-radius:0;border-bottom-right-radius:0}.btn-group>.btn-group:last-child:not(:first-child)>.btn:first-child{border-top-left-radius:0;border-bottom-left-radius:0}.btn-group .dropdown-toggle:active,.btn-group.open .dropdown-toggle{outline:0}.btn-group>.btn+.dropdown-toggle{padding-right:8px;padding-left:8px}.btn-group>.btn-lg+.dropdown-toggle{padding-right:12px;padding-left:12px}.btn-group.open .dropdown-toggle{-webkit-box-shadow:inset 0 3px 5px rgba(0,0,0,.125);box-shadow:inset 0 3px 5px rgba(0,0,0,.125)}.btn-group.open .dropdown-toggle.btn-link{-webkit-box-shadow:none;box-shadow:none}.btn .caret{margin-left:0}.btn-lg .caret{border-width:5px 5px 0;border-bottom-width:0}.dropup .btn-lg .caret{border-width:0 5px 5px}.btn-group-vertical>.btn,.btn-group-vertical>.btn-group,.btn-group-vertical>.btn-group>.btn{display:block;float:none;width:100%;max-width:100%}.btn-group-vertical>.btn-group>.btn{float:none}.btn-group-vertical>.btn+.btn,.btn-group-vertical>.btn+.btn-group,.btn-group-vertical>.btn-group+.btn,.btn-group-vertical>.btn-group+.btn-group{margin-top:-1px;margin-left:0}.btn-group-vertical>.btn:not(:first-child):not(:last-child){border-radius:0}.btn-group-vertical>.btn:first-child:not(:last-child){border-top-left-radius:4px;border-top-right-radius:4px;border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn:last-child:not(:first-child){border-top-left-radius:0;border-top-right-radius:0;border-bottom-right-radius:4px;border-bottom-left-radius:4px}.btn-group-vertical>.btn-group:not(:first-child):not(:last-child)>.btn{border-radius:0}.btn-group-vertical>.btn-group:first-child:not(:last-child)>.btn:last-child,.btn-group-vertical>.btn-group:first-child:not(:last-child)>.dropdown-toggle{border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn-group:last-child:not(:first-child)>.btn:first-child{border-top-left-radius:0;border-top-right-radius:0}.btn-group-justified{display:table;width:100%;table-layout:fixed;border-collapse:separate}.btn-group-justified>.btn,.btn-group-justified>.btn-group{display:table-cell;float:none;width:1%}.btn-group-justified>.btn-group .btn{width:100%}.btn-group-justified>.btn-group .dropdown-menu{left:auto}[data-toggle=buttons]>.btn input[type=checkbox],[data-toggle=buttons]>.btn input[type=radio],[data-toggle=buttons]>.btn-group>.btn input[type=checkbox],[data-toggle=buttons]>.btn-group>.btn input[type=radio]{position:absolute;clip:rect(0,0,0,0);pointer-events:none}.input-group{position:relative;display:table;border-collapse:separate}.input-group[class*=col-]{float:none;padding-right:0;padding-left:0}.input-group .form-control{position:relative;z-index:2;float:left;width:100%;margin-bottom:0}.input-group .form-control:focus{z-index:3}.input-group-lg>.form-control,.input-group-lg>.input-group-addon,.input-group-lg>.input-group-btn>.btn{height:46px;padding:10px 16px;font-size:18px;line-height:1.3333333;border-radius:6px}select.input-group-lg>.form-control,select.input-group-lg>.input-group-addon,select.input-group-lg>.input-group-btn>.btn{height:46px;line-height:46px}select[multiple].input-group-lg>.form-control,select[multiple].input-group-lg>.input-group-addon,select[multiple].input-group-lg>.input-group-btn>.btn,textarea.input-group-lg>.form-control,textarea.input-group-lg>.input-group-addon,textarea.input-group-lg>.input-group-btn>.btn{height:auto}.input-group-sm>.form-control,.input-group-sm>.input-group-addon,.input-group-sm>.input-group-btn>.btn{height:30px;padding:5px 10px;font-size:12px;line-height:1.5;border-radius:3px}select.input-group-sm>.form-control,select.input-group-sm>.input-group-addon,select.input-group-sm>.input-group-btn>.btn{height:30px;line-height:30px}select[multiple].input-group-sm>.form-control,select[multiple].input-group-sm>.input-group-addon,select[multiple].input-group-sm>.input-group-btn>.btn,textarea.input-group-sm>.form-control,textarea.input-group-sm>.input-group-addon,textarea.input-group-sm>.input-group-btn>.btn{height:auto}.input-group .form-control,.input-group-addon,.input-group-btn{display:table-cell}.input-group .form-control:not(:first-child):not(:last-child),.input-group-addon:not(:first-child):not(:last-child),.input-group-btn:not(:first-child):not(:last-child){border-radius:0}.input-group-addon,.input-group-btn{width:1%;white-space:nowrap;vertical-align:middle}.input-group-addon{padding:6px 12px;font-size:14px;font-weight:400;line-height:1;color:#555;text-align:center;background-color:#eee;border:1px solid #ccc;border-radius:4px}.input-group-addon.input-sm{padding:5px 10px;font-size:12px;border-radius:3px}.input-group-addon.input-lg{padding:10px 16px;font-size:18px;border-radius:6px}.input-group-addon input[type=checkbox],.input-group-addon input[type=radio]{margin-top:0}.input-group .form-control:first-child,.input-group-addon:first-child,.input-group-btn:first-child>.btn,.input-group-btn:first-child>.btn-group>.btn,.input-group-btn:first-child>.dropdown-toggle,.input-group-btn:last-child>.btn-group:not(:last-child)>.btn,.input-group-btn:last-child>.btn:not(:last-child):not(.dropdown-toggle){border-top-right-radius:0;border-bottom-right-radius:0}.input-group-addon:first-child{border-right:0}.input-group .form-control:last-child,.input-group-addon:last-child,.input-group-btn:first-child>.btn-group:not(:first-child)>.btn,.input-group-btn:first-child>.btn:not(:first-child),.input-group-btn:last-child>.btn,.input-group-btn:last-child>.btn-group>.btn,.input-group-btn:last-child>.dropdown-toggle{border-top-left-radius:0;border-bottom-left-radius:0}.input-group-addon:last-child{border-left:0}.input-group-btn{position:relative;font-size:0;white-space:nowrap}.input-group-btn>.btn{position:relative}.input-group-btn>.btn+.btn{margin-left:-1px}.input-group-btn>.btn:active,.input-group-btn>.btn:focus,.input-group-btn>.btn:hover{z-index:2}.input-group-btn:first-child>.btn,.input-group-btn:first-child>.btn-group{margin-right:-1px}.input-group-btn:last-child>.btn,.input-group-btn:last-child>.btn-group{z-index:2;margin-left:-1px}.nav{padding-left:0;margin-bottom:0;list-style:none}.nav>li{position:relative;display:block}.nav>li>a{position:relative;display:block;padding:10px 15px}.nav>li>a:focus,.nav>li>a:hover{text-decoration:none;background-color:#eee}.nav>li.disabled>a{color:#777}.nav>li.disabled>a:focus,.nav>li.disabled>a:hover{color:#777;text-decoration:none;cursor:not-allowed;background-color:transparent}.nav .open>a,.nav .open>a:focus,.nav .open>a:hover{background-color:#eee;border-color:#337ab7}.nav .nav-divider{height:1px;margin:9px 0;overflow:hidden;background-color:#e5e5e5}.nav>li>a>img{max-width:none}.nav-tabs{border-bottom:1px solid #ddd}.nav-tabs>li{float:left;margin-bottom:-1px}.nav-tabs>li>a{margin-right:2px;line-height:1.42857143;border:1px solid transparent;border-radius:4px 4px 0 0}.nav-tabs>li>a:hover{border-color:#eee #eee #ddd}.nav-tabs>li.active>a,.nav-tabs>li.active>a:focus,.nav-tabs>li.active>a:hover{color:#555;cursor:default;background-color:#fff;border:1px solid #ddd;border-bottom-color:transparent}.nav-tabs.nav-justified{width:100%;border-bottom:0}.nav-tabs.nav-justified>li{float:none}.nav-tabs.nav-justified>li>a{margin-bottom:5px;text-align:center}.nav-tabs.nav-justified>.dropdown .dropdown-menu{top:auto;left:auto}@media (min-width:768px){.nav-tabs.nav-justified>li{display:table-cell;width:1%}.nav-tabs.nav-justified>li>a{margin-bottom:0}}.nav-tabs.nav-justified>li>a{margin-right:0;border-radius:4px}.nav-tabs.nav-justified>.active>a,.nav-tabs.nav-justified>.active>a:focus,.nav-tabs.nav-justified>.active>a:hover{border:1px solid #ddd}@media (min-width:768px){.nav-tabs.nav-justified>li>a{border-bottom:1px solid #ddd;border-radius:4px 4px 0 0}.nav-tabs.nav-justified>.active>a,.nav-tabs.nav-justified>.active>a:focus,.nav-tabs.nav-justified>.active>a:hover{border-bottom-color:#fff}}.nav-pills>li{float:left}.nav-pills>li>a{border-radius:4px}.nav-pills>li+li{margin-left:2px}.nav-pills>li.active>a,.nav-pills>li.active>a:focus,.nav-pills>li.active>a:hover{color:#fff;background-color:#337ab7}.nav-stacked>li{float:none}.nav-stacked>li+li{margin-top:2px;margin-left:0}.nav-justified{width:100%}.nav-justified>li{float:none}.nav-justified>li>a{margin-bottom:5px;text-align:center}.nav-justified>.dropdown .dropdown-menu{top:auto;left:auto}@media (min-width:768px){.nav-justified>li{display:table-cell;width:1%}.nav-justified>li>a{margin-bottom:0}}.nav-tabs-justified{border-bottom:0}.nav-tabs-justified>li>a{margin-right:0;border-radius:4px}.nav-tabs-justified>.active>a,.nav-tabs-justified>.active>a:focus,.nav-tabs-justified>.active>a:hover{border:1px solid #ddd}@media (min-width:768px){.nav-tabs-justified>li>a{border-bottom:1px solid #ddd;border-radius:4px 4px 0 0}.nav-tabs-justified>.active>a,.nav-tabs-justified>.active>a:focus,.nav-tabs-justified>.active>a:hover{border-bottom-color:#fff}}.tab-content>.tab-pane{display:none}.tab-content>.active{display:block}.nav-tabs .dropdown-menu{margin-top:-1px;border-top-left-radius:0;border-top-right-radius:0}.navbar{position:relative;min-height:50px;margin-bottom:20px;border:1px solid transparent}@media (min-width:768px){.navbar{border-radius:4px}}@media (min-width:768px){.navbar-header{float:left}}.navbar-collapse{padding-right:15px;padding-left:15px;overflow-x:visible;-webkit-overflow-scrolling:touch;border-top:1px solid transparent;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,.1);box-shadow:inset 0 1px 0 rgba(255,255,255,.1)}.navbar-collapse.in{overflow-y:auto}@media (min-width:768px){.navbar-collapse{width:auto;border-top:0;-webkit-box-shadow:none;box-shadow:none}.navbar-collapse.collapse{display:block!important;height:auto!important;padding-bottom:0;overflow:visible!important}.navbar-collapse.in{overflow-y:visible}.navbar-fixed-bottom .navbar-collapse,.navbar-fixed-top .navbar-collapse,.navbar-static-top .navbar-collapse{padding-right:0;padding-left:0}}.navbar-fixed-bottom .navbar-collapse,.navbar-fixed-top .navbar-collapse{max-height:340px}@media (max-device-width:480px) and (orientation:landscape){.navbar-fixed-bottom .navbar-collapse,.navbar-fixed-top .navbar-collapse{max-height:200px}}.container-fluid>.navbar-collapse,.container-fluid>.navbar-header,.container>.navbar-collapse,.container>.navbar-header{margin-right:-15px;margin-left:-15px}@media (min-width:768px){.container-fluid>.navbar-collapse,.container-fluid>.navbar-header,.container>.navbar-collapse,.container>.navbar-header{margin-right:0;margin-left:0}}.navbar-static-top{z-index:1000;border-width:0 0 1px}@media (min-width:768px){.navbar-static-top{border-radius:0}}.navbar-fixed-bottom,.navbar-fixed-top{position:fixed;right:0;left:0;z-index:1030}@media (min-width:768px){.navbar-fixed-bottom,.navbar-fixed-top{border-radius:0}}.navbar-fixed-top{top:0;border-width:0 0 1px}.navbar-fixed-bottom{bottom:0;margin-bottom:0;border-width:1px 0 0}.navbar-brand{float:left;height:50px;padding:15px 15px;font-size:18px;line-height:20px}.navbar-brand:focus,.navbar-brand:hover{text-decoration:none}.navbar-brand>img{display:block}@media (min-width:768px){.navbar>.container .navbar-brand,.navbar>.container-fluid .navbar-brand{margin-left:-15px}}.navbar-toggle{position:relative;float:right;padding:9px 10px;margin-top:8px;margin-right:15px;margin-bottom:8px;background-color:transparent;background-image:none;border:1px solid transparent;border-radius:4px}.navbar-toggle:focus{outline:0}.navbar-toggle .icon-bar{display:block;width:22px;height:2px;border-radius:1px}.navbar-toggle .icon-bar+.icon-bar{margin-top:4px}@media (min-width:768px){.navbar-toggle{display:none}}.navbar-nav{margin:7.5px -15px}.navbar-nav>li>a{padding-top:10px;padding-bottom:10px;line-height:20px}@media (max-width:767px){.navbar-nav .open .dropdown-menu{position:static;float:none;width:auto;margin-top:0;background-color:transparent;border:0;-webkit-box-shadow:none;box-shadow:none}.navbar-nav .open .dropdown-menu .dropdown-header,.navbar-nav .open .dropdown-menu>li>a{padding:5px 15px 5px 25px}.navbar-nav .open .dropdown-menu>li>a{line-height:20px}.navbar-nav .open .dropdown-menu>li>a:focus,.navbar-nav .open .dropdown-menu>li>a:hover{background-image:none}}@media (min-width:768px){.navbar-nav{float:left;margin:0}.navbar-nav>li{float:left}.navbar-nav>li>a{padding-top:15px;padding-bottom:15px}}.navbar-form{padding:10px 15px;margin-top:8px;margin-right:-15px;margin-bottom:8px;margin-left:-15px;border-top:1px solid transparent;border-bottom:1px solid transparent;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,.1),0 1px 0 rgba(255,255,255,.1);box-shadow:inset 0 1px 0 rgba(255,255,255,.1),0 1px 0 rgba(255,255,255,.1)}@media (min-width:768px){.navbar-form .form-group{display:inline-block;margin-bottom:0;vertical-align:middle}.navbar-form .form-control{display:inline-block;width:auto;vertical-align:middle}.navbar-form .form-control-static{display:inline-block}.navbar-form .input-group{display:inline-table;vertical-align:middle}.navbar-form .input-group .form-control,.navbar-form .input-group .input-group-addon,.navbar-form .input-group .input-group-btn{width:auto}.navbar-form .input-group>.form-control{width:100%}.navbar-form .control-label{margin-bottom:0;vertical-align:middle}.navbar-form .checkbox,.navbar-form .radio{display:inline-block;margin-top:0;margin-bottom:0;vertical-align:middle}.navbar-form .checkbox label,.navbar-form .radio label{padding-left:0}.navbar-form .checkbox input[type=checkbox],.navbar-form .radio input[type=radio]{position:relative;margin-left:0}.navbar-form .has-feedback .form-control-feedback{top:0}}@media (max-width:767px){.navbar-form .form-group{margin-bottom:5px}.navbar-form .form-group:last-child{margin-bottom:0}}@media (min-width:768px){.navbar-form{width:auto;padding-top:0;padding-bottom:0;margin-right:0;margin-left:0;border:0;-webkit-box-shadow:none;box-shadow:none}}.navbar-nav>li>.dropdown-menu{margin-top:0;border-top-left-radius:0;border-top-right-radius:0}.navbar-fixed-bottom .navbar-nav>li>.dropdown-menu{margin-bottom:0;border-top-left-radius:4px;border-top-right-radius:4px;border-bottom-right-radius:0;border-bottom-left-radius:0}.navbar-btn{margin-top:8px;margin-bottom:8px}.navbar-btn.btn-sm{margin-top:10px;margin-bottom:10px}.navbar-btn.btn-xs{margin-top:14px;margin-bottom:14px}.navbar-text{margin-top:15px;margin-bottom:15px}@media (min-width:768px){.navbar-text{float:left;margin-right:15px;margin-left:15px}}@media (min-width:768px){.navbar-left{float:left!important}.navbar-right{float:right!important;margin-right:-15px}.navbar-right~.navbar-right{margin-right:0}}.navbar-default{background-color:#f8f8f8;border-color:#e7e7e7}.navbar-default .navbar-brand{color:#777}.navbar-default .navbar-brand:focus,.navbar-default .navbar-brand:hover{color:#5e5e5e;background-color:transparent}.navbar-default .navbar-text{color:#777}.navbar-default .navbar-nav>li>a{color:#777}.navbar-default .navbar-nav>li>a:focus,.navbar-default .navbar-nav>li>a:hover{color:#333;background-color:transparent}.navbar-default .navbar-nav>.active>a,.navbar-default .navbar-nav>.active>a:focus,.navbar-default .navbar-nav>.active>a:hover{color:#555;background-color:#e7e7e7}.navbar-default .navbar-nav>.disabled>a,.navbar-default .navbar-nav>.disabled>a:focus,.navbar-default .navbar-nav>.disabled>a:hover{color:#ccc;background-color:transparent}.navbar-default .navbar-toggle{border-color:#ddd}.navbar-default .navbar-toggle:focus,.navbar-default .navbar-toggle:hover{background-color:#ddd}.navbar-default .navbar-toggle .icon-bar{background-color:#888}.navbar-default .navbar-collapse,.navbar-default .navbar-form{border-color:#e7e7e7}.navbar-default .navbar-nav>.open>a,.navbar-default .navbar-nav>.open>a:focus,.navbar-default .navbar-nav>.open>a:hover{color:#555;background-color:#e7e7e7}@media (max-width:767px){.navbar-default .navbar-nav .open .dropdown-menu>li>a{color:#777}.navbar-default .navbar-nav .open .dropdown-menu>li>a:focus,.navbar-default .navbar-nav .open .dropdown-menu>li>a:hover{color:#333;background-color:transparent}.navbar-default .navbar-nav .open .dropdown-menu>.active>a,.navbar-default .navbar-nav .open .dropdown-menu>.active>a:focus,.navbar-default .navbar-nav .open .dropdown-menu>.active>a:hover{color:#555;background-color:#e7e7e7}.navbar-default .navbar-nav .open .dropdown-menu>.disabled>a,.navbar-default .navbar-nav .open .dropdown-menu>.disabled>a:focus,.navbar-default .navbar-nav .open .dropdown-menu>.disabled>a:hover{color:#ccc;background-color:transparent}}.navbar-default .navbar-link{color:#777}.navbar-default .navbar-link:hover{color:#333}.navbar-default .btn-link{color:#777}.navbar-default .btn-link:focus,.navbar-default .btn-link:hover{color:#333}.navbar-default .btn-link[disabled]:focus,.navbar-default .btn-link[disabled]:hover,fieldset[disabled] .navbar-default .btn-link:focus,fieldset[disabled] .navbar-default .btn-link:hover{color:#ccc}.navbar-inverse{background-color:#222;border-color:#080808}.navbar-inverse .navbar-brand{color:#9d9d9d}.navbar-inverse .navbar-brand:focus,.navbar-inverse .navbar-brand:hover{color:#fff;background-color:transparent}.navbar-inverse .navbar-text{color:#9d9d9d}.navbar-inverse .navbar-nav>li>a{color:#9d9d9d}.navbar-inverse .navbar-nav>li>a:focus,.navbar-inverse .navbar-nav>li>a:hover{color:#fff;background-color:transparent}.navbar-inverse .navbar-nav>.active>a,.navbar-inverse .navbar-nav>.active>a:focus,.navbar-inverse .navbar-nav>.active>a:hover{color:#fff;background-color:#080808}.navbar-inverse .navbar-nav>.disabled>a,.navbar-inverse .navbar-nav>.disabled>a:focus,.navbar-inverse .navbar-nav>.disabled>a:hover{color:#444;background-color:transparent}.navbar-inverse .navbar-toggle{border-color:#333}.navbar-inverse .navbar-toggle:focus,.navbar-inverse .navbar-toggle:hover{background-color:#333}.navbar-inverse .navbar-toggle .icon-bar{background-color:#fff}.navbar-inverse .navbar-collapse,.navbar-inverse .navbar-form{border-color:#101010}.navbar-inverse .navbar-nav>.open>a,.navbar-inverse .navbar-nav>.open>a:focus,.navbar-inverse .navbar-nav>.open>a:hover{color:#fff;background-color:#080808}@media (max-width:767px){.navbar-inverse .navbar-nav .open .dropdown-menu>.dropdown-header{border-color:#080808}.navbar-inverse .navbar-nav .open .dropdown-menu .divider{background-color:#080808}.navbar-inverse .navbar-nav .open .dropdown-menu>li>a{color:#9d9d9d}.navbar-inverse .navbar-nav .open .dropdown-menu>li>a:focus,.navbar-inverse .navbar-nav .open .dropdown-menu>li>a:hover{color:#fff;background-color:transparent}.navbar-inverse .navbar-nav .open .dropdown-menu>.active>a,.navbar-inverse .navbar-nav .open .dropdown-menu>.active>a:focus,.navbar-inverse .navbar-nav .open .dropdown-menu>.active>a:hover{color:#fff;background-color:#080808}.navbar-inverse .navbar-nav .open .dropdown-menu>.disabled>a,.navbar-inverse .navbar-nav .open .dropdown-menu>.disabled>a:focus,.navbar-inverse .navbar-nav .open .dropdown-menu>.disabled>a:hover{color:#444;background-color:transparent}}.navbar-inverse .navbar-link{color:#9d9d9d}.navbar-inverse .navbar-link:hover{color:#fff}.navbar-inverse .btn-link{color:#9d9d9d}.navbar-inverse .btn-link:focus,.navbar-inverse .btn-link:hover{color:#fff}.navbar-inverse .btn-link[disabled]:focus,.navbar-inverse .btn-link[disabled]:hover,fieldset[disabled] .navbar-inverse .btn-link:focus,fieldset[disabled] .navbar-inverse .btn-link:hover{color:#444}.breadcrumb{padding:8px 15px;margin-bottom:20px;list-style:none;background-color:#f5f5f5;border-radius:4px}.breadcrumb>li{display:inline-block}.breadcrumb>li+li:before{padding:0 5px;color:#ccc;content:"/\00a0"}.breadcrumb>.active{color:#777}.pagination{display:inline-block;padding-left:0;margin:20px 0;border-radius:4px}.pagination>li{display:inline}.pagination>li>a,.pagination>li>span{position:relative;float:left;padding:6px 12px;margin-left:-1px;line-height:1.42857143;color:#337ab7;text-decoration:none;background-color:#fff;border:1px solid #ddd}.pagination>li:first-child>a,.pagination>li:first-child>span{margin-left:0;border-top-left-radius:4px;border-bottom-left-radius:4px}.pagination>li:last-child>a,.pagination>li:last-child>span{border-top-right-radius:4px;border-bottom-right-radius:4px}.pagination>li>a:focus,.pagination>li>a:hover,.pagination>li>span:focus,.pagination>li>span:hover{z-index:2;color:#23527c;background-color:#eee;border-color:#ddd}.pagination>.active>a,.pagination>.active>a:focus,.pagination>.active>a:hover,.pagination>.active>span,.pagination>.active>span:focus,.pagination>.active>span:hover{z-index:3;color:#fff;cursor:default;background-color:#337ab7;border-color:#337ab7}.pagination>.disabled>a,.pagination>.disabled>a:focus,.pagination>.disabled>a:hover,.pagination>.disabled>span,.pagination>.disabled>span:focus,.pagination>.disabled>span:hover{color:#777;cursor:not-allowed;background-color:#fff;border-color:#ddd}.pagination-lg>li>a,.pagination-lg>li>span{padding:10px 16px;font-size:18px;line-height:1.3333333}.pagination-lg>li:first-child>a,.pagination-lg>li:first-child>span{border-top-left-radius:6px;border-bottom-left-radius:6px}.pagination-lg>li:last-child>a,.pagination-lg>li:last-child>span{border-top-right-radius:6px;border-bottom-right-radius:6px}.pagination-sm>li>a,.pagination-sm>li>span{padding:5px 10px;font-size:12px;line-height:1.5}.pagination-sm>li:first-child>a,.pagination-sm>li:first-child>span{border-top-left-radius:3px;border-bottom-left-radius:3px}.pagination-sm>li:last-child>a,.pagination-sm>li:last-child>span{border-top-right-radius:3px;border-bottom-right-radius:3px}.pager{padding-left:0;margin:20px 0;text-align:center;list-style:none}.pager li{display:inline}.pager li>a,.pager li>span{display:inline-block;padding:5px 14px;background-color:#fff;border:1px solid #ddd;border-radius:15px}.pager li>a:focus,.pager li>a:hover{text-decoration:none;background-color:#eee}.pager .next>a,.pager .next>span{float:right}.pager .previous>a,.pager .previous>span{float:left}.pager .disabled>a,.pager .disabled>a:focus,.pager .disabled>a:hover,.pager .disabled>span{color:#777;cursor:not-allowed;background-color:#fff}.label{display:inline;padding:.2em .6em .3em;font-size:75%;font-weight:700;line-height:1;color:#fff;text-align:center;white-space:nowrap;vertical-align:baseline;border-radius:.25em}a.label:focus,a.label:hover{color:#fff;text-decoration:none;cursor:pointer}.label:empty{display:none}.btn .label{position:relative;top:-1px}.label-default{background-color:#777}.label-default[href]:focus,.label-default[href]:hover{background-color:#5e5e5e}.label-primary{background-color:#337ab7}.label-primary[href]:focus,.label-primary[href]:hover{background-color:#286090}.label-success{background-color:#5cb85c}.label-success[href]:focus,.label-success[href]:hover{background-color:#449d44}.label-info{background-color:#5bc0de}.label-info[href]:focus,.label-info[href]:hover{background-color:#31b0d5}.label-warning{background-color:#f0ad4e}.label-warning[href]:focus,.label-warning[href]:hover{background-color:#ec971f}.label-danger{background-color:#d9534f}.label-danger[href]:focus,.label-danger[href]:hover{background-color:#c9302c}.badge{display:inline-block;min-width:10px;padding:3px 7px;font-size:12px;font-weight:700;line-height:1;color:#fff;text-align:center;white-space:nowrap;vertical-align:middle;background-color:#777;border-radius:10px}.badge:empty{display:none}.btn .badge{position:relative;top:-1px}.btn-group-xs>.btn .badge,.btn-xs .badge{top:0;padding:1px 5px}a.badge:focus,a.badge:hover{color:#fff;text-decoration:none;cursor:pointer}.list-group-item.active>.badge,.nav-pills>.active>a>.badge{color:#337ab7;background-color:#fff}.list-group-item>.badge{float:right}.list-group-item>.badge+.badge{margin-right:5px}.nav-pills>li>a>.badge{margin-left:3px}.jumbotron{padding-top:30px;padding-bottom:30px;margin-bottom:30px;color:inherit;background-color:#eee}.jumbotron .h1,.jumbotron h1{color:inherit}.jumbotron p{margin-bottom:15px;font-size:21px;font-weight:200}.jumbotron>hr{border-top-color:#d5d5d5}.container .jumbotron,.container-fluid .jumbotron{padding-right:15px;padding-left:15px;border-radius:6px}.jumbotron .container{max-width:100%}@media screen and (min-width:768px){.jumbotron{padding-top:48px;padding-bottom:48px}.container .jumbotron,.container-fluid .jumbotron{padding-right:60px;padding-left:60px}.jumbotron .h1,.jumbotron h1{font-size:63px}}.thumbnail{display:block;padding:4px;margin-bottom:20px;line-height:1.42857143;background-color:#fff;border:1px solid #ddd;border-radius:4px;-webkit-transition:border .2s ease-in-out;-o-transition:border .2s ease-in-out;transition:border .2s ease-in-out}.thumbnail a>img,.thumbnail>img{margin-right:auto;margin-left:auto}a.thumbnail.active,a.thumbnail:focus,a.thumbnail:hover{border-color:#337ab7}.thumbnail .caption{padding:9px;color:#333}.alert{padding:15px;margin-bottom:20px;border:1px solid transparent;border-radius:4px}.alert h4{margin-top:0;color:inherit}.alert .alert-link{font-weight:700}.alert>p,.alert>ul{margin-bottom:0}.alert>p+p{margin-top:5px}.alert-dismissable,.alert-dismissible{padding-right:35px}.alert-dismissable .close,.alert-dismissible .close{position:relative;top:-2px;right:-21px;color:inherit}.alert-success{color:#3c763d;background-color:#dff0d8;border-color:#d6e9c6}.alert-success hr{border-top-color:#c9e2b3}.alert-success .alert-link{color:#2b542c}.alert-info{color:#31708f;background-color:#d9edf7;border-color:#bce8f1}.alert-info hr{border-top-color:#a6e1ec}.alert-info .alert-link{color:#245269}.alert-warning{color:#8a6d3b;background-color:#fcf8e3;border-color:#faebcc}.alert-warning hr{border-top-color:#f7e1b5}.alert-warning .alert-link{color:#66512c}.alert-danger{color:#a94442;background-color:#f2dede;border-color:#ebccd1}.alert-danger hr{border-top-color:#e4b9c0}.alert-danger .alert-link{color:#843534}@-webkit-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@-o-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}.progress{height:20px;margin-bottom:20px;overflow:hidden;background-color:#f5f5f5;border-radius:4px;-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,.1);box-shadow:inset 0 1px 2px rgba(0,0,0,.1)}.progress-bar{float:left;width:0;height:100%;font-size:12px;line-height:20px;color:#fff;text-align:center;background-color:#337ab7;-webkit-box-shadow:inset 0 -1px 0 rgba(0,0,0,.15);box-shadow:inset 0 -1px 0 rgba(0,0,0,.15);-webkit-transition:width .6s ease;-o-transition:width .6s ease;transition:width .6s ease}.progress-bar-striped,.progress-striped .progress-bar{background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);-webkit-background-size:40px 40px;background-size:40px 40px}.progress-bar.active,.progress.active .progress-bar{-webkit-animation:progress-bar-stripes 2s linear infinite;-o-animation:progress-bar-stripes 2s linear infinite;animation:progress-bar-stripes 2s linear infinite}.progress-bar-success{background-color:#5cb85c}.progress-striped .progress-bar-success{background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent)}.progress-bar-info{background-color:#5bc0de}.progress-striped .progress-bar-info{background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent)}.progress-bar-warning{background-color:#f0ad4e}.progress-striped .progress-bar-warning{background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent)}.progress-bar-danger{background-color:#d9534f}.progress-striped .progress-bar-danger{background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent)}.media{margin-top:15px}.media:first-child{margin-top:0}.media,.media-body{overflow:hidden;zoom:1}.media-body{width:10000px}.media-object{display:block}.media-object.img-thumbnail{max-width:none}.media-right,.media>.pull-right{padding-left:10px}.media-left,.media>.pull-left{padding-right:10px}.media-body,.media-left,.media-right{display:table-cell;vertical-align:top}.media-middle{vertical-align:middle}.media-bottom{vertical-align:bottom}.media-heading{margin-top:0;margin-bottom:5px}.media-list{padding-left:0;list-style:none}.list-group{padding-left:0;margin-bottom:20px}.list-group-item{position:relative;display:block;padding:10px 15px;margin-bottom:-1px;background-color:#fff;border:1px solid #ddd}.list-group-item:first-child{border-top-left-radius:4px;border-top-right-radius:4px}.list-group-item:last-child{margin-bottom:0;border-bottom-right-radius:4px;border-bottom-left-radius:4px}a.list-group-item,button.list-group-item{color:#555}a.list-group-item .list-group-item-heading,button.list-group-item .list-group-item-heading{color:#333}a.list-group-item:focus,a.list-group-item:hover,button.list-group-item:focus,button.list-group-item:hover{color:#555;text-decoration:none;background-color:#f5f5f5}button.list-group-item{width:100%;text-align:left}.list-group-item.disabled,.list-group-item.disabled:focus,.list-group-item.disabled:hover{color:#777;cursor:not-allowed;background-color:#eee}.list-group-item.disabled .list-group-item-heading,.list-group-item.disabled:focus .list-group-item-heading,.list-group-item.disabled:hover .list-group-item-heading{color:inherit}.list-group-item.disabled .list-group-item-text,.list-group-item.disabled:focus .list-group-item-text,.list-group-item.disabled:hover .list-group-item-text{color:#777}.list-group-item.active,.list-group-item.active:focus,.list-group-item.active:hover{z-index:2;color:#fff;background-color:#337ab7;border-color:#337ab7}.list-group-item.active .list-group-item-heading,.list-group-item.active .list-group-item-heading>.small,.list-group-item.active .list-group-item-heading>small,.list-group-item.active:focus .list-group-item-heading,.list-group-item.active:focus .list-group-item-heading>.small,.list-group-item.active:focus .list-group-item-heading>small,.list-group-item.active:hover .list-group-item-heading,.list-group-item.active:hover .list-group-item-heading>.small,.list-group-item.active:hover .list-group-item-heading>small{color:inherit}.list-group-item.active .list-group-item-text,.list-group-item.active:focus .list-group-item-text,.list-group-item.active:hover .list-group-item-text{color:#c7ddef}.list-group-item-success{color:#3c763d;background-color:#dff0d8}a.list-group-item-success,button.list-group-item-success{color:#3c763d}a.list-group-item-success .list-group-item-heading,button.list-group-item-success .list-group-item-heading{color:inherit}a.list-group-item-success:focus,a.list-group-item-success:hover,button.list-group-item-success:focus,button.list-group-item-success:hover{color:#3c763d;background-color:#d0e9c6}a.list-group-item-success.active,a.list-group-item-success.active:focus,a.list-group-item-success.active:hover,button.list-group-item-success.active,button.list-group-item-success.active:focus,button.list-group-item-success.active:hover{color:#fff;background-color:#3c763d;border-color:#3c763d}.list-group-item-info{color:#31708f;background-color:#d9edf7}a.list-group-item-info,button.list-group-item-info{color:#31708f}a.list-group-item-info .list-group-item-heading,button.list-group-item-info .list-group-item-heading{color:inherit}a.list-group-item-info:focus,a.list-group-item-info:hover,button.list-group-item-info:focus,button.list-group-item-info:hover{color:#31708f;background-color:#c4e3f3}a.list-group-item-info.active,a.list-group-item-info.active:focus,a.list-group-item-info.active:hover,button.list-group-item-info.active,button.list-group-item-info.active:focus,button.list-group-item-info.active:hover{color:#fff;background-color:#31708f;border-color:#31708f}.list-group-item-warning{color:#8a6d3b;background-color:#fcf8e3}a.list-group-item-warning,button.list-group-item-warning{color:#8a6d3b}a.list-group-item-warning .list-group-item-heading,button.list-group-item-warning .list-group-item-heading{color:inherit}a.list-group-item-warning:focus,a.list-group-item-warning:hover,button.list-group-item-warning:focus,button.list-group-item-warning:hover{color:#8a6d3b;background-color:#faf2cc}a.list-group-item-warning.active,a.list-group-item-warning.active:focus,a.list-group-item-warning.active:hover,button.list-group-item-warning.active,button.list-group-item-warning.active:focus,button.list-group-item-warning.active:hover{color:#fff;background-color:#8a6d3b;border-color:#8a6d3b}.list-group-item-danger{color:#a94442;background-color:#f2dede}a.list-group-item-danger,button.list-group-item-danger{color:#a94442}a.list-group-item-danger .list-group-item-heading,button.list-group-item-danger .list-group-item-heading{color:inherit}a.list-group-item-danger:focus,a.list-group-item-danger:hover,button.list-group-item-danger:focus,button.list-group-item-danger:hover{color:#a94442;background-color:#ebcccc}a.list-group-item-danger.active,a.list-group-item-danger.active:focus,a.list-group-item-danger.active:hover,button.list-group-item-danger.active,button.list-group-item-danger.active:focus,button.list-group-item-danger.active:hover{color:#fff;background-color:#a94442;border-color:#a94442}.list-group-item-heading{margin-top:0;margin-bottom:5px}.list-group-item-text{margin-bottom:0;line-height:1.3}.panel{margin-bottom:20px;background-color:#fff;border:1px solid transparent;border-radius:4px;-webkit-box-shadow:0 1px 1px rgba(0,0,0,.05);box-shadow:0 1px 1px rgba(0,0,0,.05)}.panel-body{padding:15px}.panel-heading{padding:10px 15px;border-bottom:1px solid transparent;border-top-left-radius:3px;border-top-right-radius:3px}.panel-heading>.dropdown .dropdown-toggle{color:inherit}.panel-title{margin-top:0;margin-bottom:0;font-size:16px;color:inherit}.panel-title>.small,.panel-title>.small>a,.panel-title>a,.panel-title>small,.panel-title>small>a{color:inherit}.panel-footer{padding:10px 15px;background-color:#f5f5f5;border-top:1px solid #ddd;border-bottom-right-radius:3px;border-bottom-left-radius:3px}.panel>.list-group,.panel>.panel-collapse>.list-group{margin-bottom:0}.panel>.list-group .list-group-item,.panel>.panel-collapse>.list-group .list-group-item{border-width:1px 0;border-radius:0}.panel>.list-group:first-child .list-group-item:first-child,.panel>.panel-collapse>.list-group:first-child .list-group-item:first-child{border-top:0;border-top-left-radius:3px;border-top-right-radius:3px}.panel>.list-group:last-child .list-group-item:last-child,.panel>.panel-collapse>.list-group:last-child .list-group-item:last-child{border-bottom:0;border-bottom-right-radius:3px;border-bottom-left-radius:3px}.panel>.panel-heading+.panel-collapse>.list-group .list-group-item:first-child{border-top-left-radius:0;border-top-right-radius:0}.panel-heading+.list-group .list-group-item:first-child{border-top-width:0}.list-group+.panel-footer{border-top-width:0}.panel>.panel-collapse>.table,.panel>.table,.panel>.table-responsive>.table{margin-bottom:0}.panel>.panel-collapse>.table caption,.panel>.table caption,.panel>.table-responsive>.table caption{padding-right:15px;padding-left:15px}.panel>.table-responsive:first-child>.table:first-child,.panel>.table:first-child{border-top-left-radius:3px;border-top-right-radius:3px}.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child,.panel>.table:first-child>tbody:first-child>tr:first-child,.panel>.table:first-child>thead:first-child>tr:first-child{border-top-left-radius:3px;border-top-right-radius:3px}.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child td:first-child,.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child th:first-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child td:first-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child th:first-child,.panel>.table:first-child>tbody:first-child>tr:first-child td:first-child,.panel>.table:first-child>tbody:first-child>tr:first-child th:first-child,.panel>.table:first-child>thead:first-child>tr:first-child td:first-child,.panel>.table:first-child>thead:first-child>tr:first-child th:first-child{border-top-left-radius:3px}.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child td:last-child,.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child th:last-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child td:last-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child th:last-child,.panel>.table:first-child>tbody:first-child>tr:first-child td:last-child,.panel>.table:first-child>tbody:first-child>tr:first-child th:last-child,.panel>.table:first-child>thead:first-child>tr:first-child td:last-child,.panel>.table:first-child>thead:first-child>tr:first-child th:last-child{border-top-right-radius:3px}.panel>.table-responsive:last-child>.table:last-child,.panel>.table:last-child{border-bottom-right-radius:3px;border-bottom-left-radius:3px}.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child,.panel>.table:last-child>tbody:last-child>tr:last-child,.panel>.table:last-child>tfoot:last-child>tr:last-child{border-bottom-right-radius:3px;border-bottom-left-radius:3px}.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child td:first-child,.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child th:first-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child td:first-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child th:first-child,.panel>.table:last-child>tbody:last-child>tr:last-child td:first-child,.panel>.table:last-child>tbody:last-child>tr:last-child th:first-child,.panel>.table:last-child>tfoot:last-child>tr:last-child td:first-child,.panel>.table:last-child>tfoot:last-child>tr:last-child th:first-child{border-bottom-left-radius:3px}.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child td:last-child,.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child th:last-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child td:last-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child th:last-child,.panel>.table:last-child>tbody:last-child>tr:last-child td:last-child,.panel>.table:last-child>tbody:last-child>tr:last-child th:last-child,.panel>.table:last-child>tfoot:last-child>tr:last-child td:last-child,.panel>.table:last-child>tfoot:last-child>tr:last-child th:last-child{border-bottom-right-radius:3px}.panel>.panel-body+.table,.panel>.panel-body+.table-responsive,.panel>.table+.panel-body,.panel>.table-responsive+.panel-body{border-top:1px solid #ddd}.panel>.table>tbody:first-child>tr:first-child td,.panel>.table>tbody:first-child>tr:first-child th{border-top:0}.panel>.table-bordered,.panel>.table-responsive>.table-bordered{border:0}.panel>.table-bordered>tbody>tr>td:first-child,.panel>.table-bordered>tbody>tr>th:first-child,.panel>.table-bordered>tfoot>tr>td:first-child,.panel>.table-bordered>tfoot>tr>th:first-child,.panel>.table-bordered>thead>tr>td:first-child,.panel>.table-bordered>thead>tr>th:first-child,.panel>.table-responsive>.table-bordered>tbody>tr>td:first-child,.panel>.table-responsive>.table-bordered>tbody>tr>th:first-child,.panel>.table-responsive>.table-bordered>tfoot>tr>td:first-child,.panel>.table-responsive>.table-bordered>tfoot>tr>th:first-child,.panel>.table-responsive>.table-bordered>thead>tr>td:first-child,.panel>.table-responsive>.table-bordered>thead>tr>th:first-child{border-left:0}.panel>.table-bordered>tbody>tr>td:last-child,.panel>.table-bordered>tbody>tr>th:last-child,.panel>.table-bordered>tfoot>tr>td:last-child,.panel>.table-bordered>tfoot>tr>th:last-child,.panel>.table-bordered>thead>tr>td:last-child,.panel>.table-bordered>thead>tr>th:last-child,.panel>.table-responsive>.table-bordered>tbody>tr>td:last-child,.panel>.table-responsive>.table-bordered>tbody>tr>th:last-child,.panel>.table-responsive>.table-bordered>tfoot>tr>td:last-child,.panel>.table-responsive>.table-bordered>tfoot>tr>th:last-child,.panel>.table-responsive>.table-bordered>thead>tr>td:last-child,.panel>.table-responsive>.table-bordered>thead>tr>th:last-child{border-right:0}.panel>.table-bordered>tbody>tr:first-child>td,.panel>.table-bordered>tbody>tr:first-child>th,.panel>.table-bordered>thead>tr:first-child>td,.panel>.table-bordered>thead>tr:first-child>th,.panel>.table-responsive>.table-bordered>tbody>tr:first-child>td,.panel>.table-responsive>.table-bordered>tbody>tr:first-child>th,.panel>.table-responsive>.table-bordered>thead>tr:first-child>td,.panel>.table-responsive>.table-bordered>thead>tr:first-child>th{border-bottom:0}.panel>.table-bordered>tbody>tr:last-child>td,.panel>.table-bordered>tbody>tr:last-child>th,.panel>.table-bordered>tfoot>tr:last-child>td,.panel>.table-bordered>tfoot>tr:last-child>th,.panel>.table-responsive>.table-bordered>tbody>tr:last-child>td,.panel>.table-responsive>.table-bordered>tbody>tr:last-child>th,.panel>.table-responsive>.table-bordered>tfoot>tr:last-child>td,.panel>.table-responsive>.table-bordered>tfoot>tr:last-child>th{border-bottom:0}.panel>.table-responsive{margin-bottom:0;border:0}.panel-group{margin-bottom:20px}.panel-group .panel{margin-bottom:0;border-radius:4px}.panel-group .panel+.panel{margin-top:5px}.panel-group .panel-heading{border-bottom:0}.panel-group .panel-heading+.panel-collapse>.list-group,.panel-group .panel-heading+.panel-collapse>.panel-body{border-top:1px solid #ddd}.panel-group .panel-footer{border-top:0}.panel-group .panel-footer+.panel-collapse .panel-body{border-bottom:1px solid #ddd}.panel-default{border-color:#ddd}.panel-default>.panel-heading{color:#333;background-color:#f5f5f5;border-color:#ddd}.panel-default>.panel-heading+.panel-collapse>.panel-body{border-top-color:#ddd}.panel-default>.panel-heading .badge{color:#f5f5f5;background-color:#333}.panel-default>.panel-footer+.panel-collapse>.panel-body{border-bottom-color:#ddd}.panel-primary{border-color:#337ab7}.panel-primary>.panel-heading{color:#fff;background-color:#337ab7;border-color:#337ab7}.panel-primary>.panel-heading+.panel-collapse>.panel-body{border-top-color:#337ab7}.panel-primary>.panel-heading .badge{color:#337ab7;background-color:#fff}.panel-primary>.panel-footer+.panel-collapse>.panel-body{border-bottom-color:#337ab7}.panel-success{border-color:#d6e9c6}.panel-success>.panel-heading{color:#3c763d;background-color:#dff0d8;border-color:#d6e9c6}.panel-success>.panel-heading+.panel-collapse>.panel-body{border-top-color:#d6e9c6}.panel-success>.panel-heading .badge{color:#dff0d8;background-color:#3c763d}.panel-success>.panel-footer+.panel-collapse>.panel-body{border-bottom-color:#d6e9c6}.panel-info{border-color:#bce8f1}.panel-info>.panel-heading{color:#31708f;background-color:#d9edf7;border-color:#bce8f1}.panel-info>.panel-heading+.panel-collapse>.panel-body{border-top-color:#bce8f1}.panel-info>.panel-heading .badge{color:#d9edf7;background-color:#31708f}.panel-info>.panel-footer+.panel-collapse>.panel-body{border-bottom-color:#bce8f1}.panel-warning{border-color:#faebcc}.panel-warning>.panel-heading{color:#8a6d3b;background-color:#fcf8e3;border-color:#faebcc}.panel-warning>.panel-heading+.panel-collapse>.panel-body{border-top-color:#faebcc}.panel-warning>.panel-heading .badge{color:#fcf8e3;background-color:#8a6d3b}.panel-warning>.panel-footer+.panel-collapse>.panel-body{border-bottom-color:#faebcc}.panel-danger{border-color:#ebccd1}.panel-danger>.panel-heading{color:#a94442;background-color:#f2dede;border-color:#ebccd1}.panel-danger>.panel-heading+.panel-collapse>.panel-body{border-top-color:#ebccd1}.panel-danger>.panel-heading .badge{color:#f2dede;background-color:#a94442}.panel-danger>.panel-footer+.panel-collapse>.panel-body{border-bottom-color:#ebccd1}.embed-responsive{position:relative;display:block;height:0;padding:0;overflow:hidden}.embed-responsive .embed-responsive-item,.embed-responsive embed,.embed-responsive iframe,.embed-responsive object,.embed-responsive video{position:absolute;top:0;bottom:0;left:0;width:100%;height:100%;border:0}.embed-responsive-16by9{padding-bottom:56.25%}.embed-responsive-4by3{padding-bottom:75%}.well{min-height:20px;padding:19px;margin-bottom:20px;background-color:#f5f5f5;border:1px solid #e3e3e3;border-radius:4px;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.05);box-shadow:inset 0 1px 1px rgba(0,0,0,.05)}.well blockquote{border-color:#ddd;border-color:rgba(0,0,0,.15)}.well-lg{padding:24px;border-radius:6px}.well-sm{padding:9px;border-radius:3px}.close{float:right;font-size:21px;font-weight:700;line-height:1;color:#000;text-shadow:0 1px 0 #fff;filter:alpha(opacity=20);opacity:.2}.close:focus,.close:hover{color:#000;text-decoration:none;cursor:pointer;filter:alpha(opacity=50);opacity:.5}button.close{-webkit-appearance:none;padding:0;cursor:pointer;background:0 0;border:0}.modal-open{overflow:hidden}.modal{position:fixed;top:0;right:0;bottom:0;left:0;z-index:1050;display:none;overflow:hidden;-webkit-overflow-scrolling:touch;outline:0}.modal.fade .modal-dialog{-webkit-transition:-webkit-transform .3s ease-out;-o-transition:-o-transform .3s ease-out;transition:transform .3s ease-out;-webkit-transform:translate(0,-25%);-ms-transform:translate(0,-25%);-o-transform:translate(0,-25%);transform:translate(0,-25%)}.modal.in .modal-dialog{-webkit-transform:translate(0,0);-ms-transform:translate(0,0);-o-transform:translate(0,0);transform:translate(0,0)}.modal-open .modal{overflow-x:hidden;overflow-y:auto}.modal-dialog{position:relative;width:auto;margin:10px}.modal-content{position:relative;background-color:#fff;-webkit-background-clip:padding-box;background-clip:padding-box;border:1px solid #999;border:1px solid rgba(0,0,0,.2);border-radius:6px;outline:0;-webkit-box-shadow:0 3px 9px rgba(0,0,0,.5);box-shadow:0 3px 9px rgba(0,0,0,.5)}.modal-backdrop{position:fixed;top:0;right:0;bottom:0;left:0;z-index:1040;background-color:#000}.modal-backdrop.fade{filter:alpha(opacity=0);opacity:0}.modal-backdrop.in{filter:alpha(opacity=50);opacity:.5}.modal-header{padding:15px;border-bottom:1px solid #e5e5e5}.modal-header .close{margin-top:-2px}.modal-title{margin:0;line-height:1.42857143}.modal-body{position:relative;padding:15px}.modal-footer{padding:15px;text-align:right;border-top:1px solid #e5e5e5}.modal-footer .btn+.btn{margin-bottom:0;margin-left:5px}.modal-footer .btn-group .btn+.btn{margin-left:-1px}.modal-footer .btn-block+.btn-block{margin-left:0}.modal-scrollbar-measure{position:absolute;top:-9999px;width:50px;height:50px;overflow:scroll}@media (min-width:768px){.modal-dialog{width:600px;margin:30px auto}.modal-content{-webkit-box-shadow:0 5px 15px rgba(0,0,0,.5);box-shadow:0 5px 15px rgba(0,0,0,.5)}.modal-sm{width:300px}}@media (min-width:992px){.modal-lg{width:900px}}.tooltip{position:absolute;z-index:1070;display:block;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:12px;font-style:normal;font-weight:400;line-height:1.42857143;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;word-wrap:normal;white-space:normal;filter:alpha(opacity=0);opacity:0;line-break:auto}.tooltip.in{filter:alpha(opacity=90);opacity:.9}.tooltip.top{padding:5px 0;margin-top:-3px}.tooltip.right{padding:0 5px;margin-left:3px}.tooltip.bottom{padding:5px 0;margin-top:3px}.tooltip.left{padding:0 5px;margin-left:-3px}.tooltip-inner{max-width:200px;padding:3px 8px;color:#fff;text-align:center;background-color:#000;border-radius:4px}.tooltip-arrow{position:absolute;width:0;height:0;border-color:transparent;border-style:solid}.tooltip.top .tooltip-arrow{bottom:0;left:50%;margin-left:-5px;border-width:5px 5px 0;border-top-color:#000}.tooltip.top-left .tooltip-arrow{right:5px;bottom:0;margin-bottom:-5px;border-width:5px 5px 0;border-top-color:#000}.tooltip.top-right .tooltip-arrow{bottom:0;left:5px;margin-bottom:-5px;border-width:5px 5px 0;border-top-color:#000}.tooltip.right .tooltip-arrow{top:50%;left:0;margin-top:-5px;border-width:5px 5px 5px 0;border-right-color:#000}.tooltip.left .tooltip-arrow{top:50%;right:0;margin-top:-5px;border-width:5px 0 5px 5px;border-left-color:#000}.tooltip.bottom .tooltip-arrow{top:0;left:50%;margin-left:-5px;border-width:0 5px 5px;border-bottom-color:#000}.tooltip.bottom-left .tooltip-arrow{top:0;right:5px;margin-top:-5px;border-width:0 5px 5px;border-bottom-color:#000}.tooltip.bottom-right .tooltip-arrow{top:0;left:5px;margin-top:-5px;border-width:0 5px 5px;border-bottom-color:#000}.popover{position:absolute;top:0;left:0;z-index:1060;display:none;max-width:276px;padding:1px;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:14px;font-style:normal;font-weight:400;line-height:1.42857143;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;word-wrap:normal;white-space:normal;background-color:#fff;-webkit-background-clip:padding-box;background-clip:padding-box;border:1px solid #ccc;border:1px solid rgba(0,0,0,.2);border-radius:6px;-webkit-box-shadow:0 5px 10px rgba(0,0,0,.2);box-shadow:0 5px 10px rgba(0,0,0,.2);line-break:auto}.popover.top{margin-top:-10px}.popover.right{margin-left:10px}.popover.bottom{margin-top:10px}.popover.left{margin-left:-10px}.popover-title{padding:8px 14px;margin:0;font-size:14px;background-color:#f7f7f7;border-bottom:1px solid #ebebeb;border-radius:5px 5px 0 0}.popover-content{padding:9px 14px}.popover>.arrow,.popover>.arrow:after{position:absolute;display:block;width:0;height:0;border-color:transparent;border-style:solid}.popover>.arrow{border-width:11px}.popover>.arrow:after{content:"";border-width:10px}.popover.top>.arrow{bottom:-11px;left:50%;margin-left:-11px;border-top-color:#999;border-top-color:rgba(0,0,0,.25);border-bottom-width:0}.popover.top>.arrow:after{bottom:1px;margin-left:-10px;content:" ";border-top-color:#fff;border-bottom-width:0}.popover.right>.arrow{top:50%;left:-11px;margin-top:-11px;border-right-color:#999;border-right-color:rgba(0,0,0,.25);border-left-width:0}.popover.right>.arrow:after{bottom:-10px;left:1px;content:" ";border-right-color:#fff;border-left-width:0}.popover.bottom>.arrow{top:-11px;left:50%;margin-left:-11px;border-top-width:0;border-bottom-color:#999;border-bottom-color:rgba(0,0,0,.25)}.popover.bottom>.arrow:after{top:1px;margin-left:-10px;content:" ";border-top-width:0;border-bottom-color:#fff}.popover.left>.arrow{top:50%;right:-11px;margin-top:-11px;border-right-width:0;border-left-color:#999;border-left-color:rgba(0,0,0,.25)}.popover.left>.arrow:after{right:1px;bottom:-10px;content:" ";border-right-width:0;border-left-color:#fff}.carousel{position:relative}.carousel-inner{position:relative;width:100%;overflow:hidden}.carousel-inner>.item{position:relative;display:none;-webkit-transition:.6s ease-in-out left;-o-transition:.6s ease-in-out left;transition:.6s ease-in-out left}.carousel-inner>.item>a>img,.carousel-inner>.item>img{line-height:1}@media all and (transform-3d),(-webkit-transform-3d){.carousel-inner>.item{-webkit-transition:-webkit-transform .6s ease-in-out;-o-transition:-o-transform .6s ease-in-out;transition:transform .6s ease-in-out;-webkit-backface-visibility:hidden;backface-visibility:hidden;-webkit-perspective:1000px;perspective:1000px}.carousel-inner>.item.active.right,.carousel-inner>.item.next{left:0;-webkit-transform:translate3d(100%,0,0);transform:translate3d(100%,0,0)}.carousel-inner>.item.active.left,.carousel-inner>.item.prev{left:0;-webkit-transform:translate3d(-100%,0,0);transform:translate3d(-100%,0,0)}.carousel-inner>.item.active,.carousel-inner>.item.next.left,.carousel-inner>.item.prev.right{left:0;-webkit-transform:translate3d(0,0,0);transform:translate3d(0,0,0)}}.carousel-inner>.active,.carousel-inner>.next,.carousel-inner>.prev{display:block}.carousel-inner>.active{left:0}.carousel-inner>.next,.carousel-inner>.prev{position:absolute;top:0;width:100%}.carousel-inner>.next{left:100%}.carousel-inner>.prev{left:-100%}.carousel-inner>.next.left,.carousel-inner>.prev.right{left:0}.carousel-inner>.active.left{left:-100%}.carousel-inner>.active.right{left:100%}.carousel-control{position:absolute;top:0;bottom:0;left:0;width:15%;font-size:20px;color:#fff;text-align:center;text-shadow:0 1px 2px rgba(0,0,0,.6);background-color:rgba(0,0,0,0);filter:alpha(opacity=50);opacity:.5}.carousel-control.left{background-image:-webkit-linear-gradient(left,rgba(0,0,0,.5) 0,rgba(0,0,0,.0001) 100%);background-image:-o-linear-gradient(left,rgba(0,0,0,.5) 0,rgba(0,0,0,.0001) 100%);background-image:-webkit-gradient(linear,left top,right top,from(rgba(0,0,0,.5)),to(rgba(0,0,0,.0001)));background-image:linear-gradient(to right,rgba(0,0,0,.5) 0,rgba(0,0,0,.0001) 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#80000000', endColorstr='#00000000', GradientType=1);background-repeat:repeat-x}.carousel-control.right{right:0;left:auto;background-image:-webkit-linear-gradient(left,rgba(0,0,0,.0001) 0,rgba(0,0,0,.5) 100%);background-image:-o-linear-gradient(left,rgba(0,0,0,.0001) 0,rgba(0,0,0,.5) 100%);background-image:-webkit-gradient(linear,left top,right top,from(rgba(0,0,0,.0001)),to(rgba(0,0,0,.5)));background-image:linear-gradient(to right,rgba(0,0,0,.0001) 0,rgba(0,0,0,.5) 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#00000000', endColorstr='#80000000', GradientType=1);background-repeat:repeat-x}.carousel-control:focus,.carousel-control:hover{color:#fff;text-decoration:none;filter:alpha(opacity=90);outline:0;opacity:.9}.carousel-control .glyphicon-chevron-left,.carousel-control .glyphicon-chevron-right,.carousel-control .icon-next,.carousel-control .icon-prev{position:absolute;top:50%;z-index:5;display:inline-block;margin-top:-10px}.carousel-control .glyphicon-chevron-left,.carousel-control .icon-prev{left:50%;margin-left:-10px}.carousel-control .glyphicon-chevron-right,.carousel-control .icon-next{right:50%;margin-right:-10px}.carousel-control .icon-next,.carousel-control .icon-prev{width:20px;height:20px;font-family:serif;line-height:1}.carousel-control .icon-prev:before{content:'\2039'}.carousel-control .icon-next:before{content:'\203a'}.carousel-indicators{position:absolute;bottom:10px;left:50%;z-index:15;width:60%;padding-left:0;margin-left:-30%;text-align:center;list-style:none}.carousel-indicators li{display:inline-block;width:10px;height:10px;margin:1px;text-indent:-999px;cursor:pointer;background-color:#000\9;background-color:rgba(0,0,0,0);border:1px solid #fff;border-radius:10px}.carousel-indicators .active{width:12px;height:12px;margin:0;background-color:#fff}.carousel-caption{position:absolute;right:15%;bottom:20px;left:15%;z-index:10;padding-top:20px;padding-bottom:20px;color:#fff;text-align:center;text-shadow:0 1px 2px rgba(0,0,0,.6)}.carousel-caption .btn{text-shadow:none}@media screen and (min-width:768px){.carousel-control .glyphicon-chevron-left,.carousel-control .glyphicon-chevron-right,.carousel-control .icon-next,.carousel-control .icon-prev{width:30px;height:30px;margin-top:-10px;font-size:30px}.carousel-control .glyphicon-chevron-left,.carousel-control .icon-prev{margin-left:-10px}.carousel-control .glyphicon-chevron-right,.carousel-control .icon-next{margin-right:-10px}.carousel-caption{right:20%;left:20%;padding-bottom:30px}.carousel-indicators{bottom:20px}}.btn-group-vertical>.btn-group:after,.btn-group-vertical>.btn-group:before,.btn-toolbar:after,.btn-toolbar:before,.clearfix:after,.clearfix:before,.container-fluid:after,.container-fluid:before,.container:after,.container:before,.dl-horizontal dd:after,.dl-horizontal dd:before,.form-horizontal .form-group:after,.form-horizontal .form-group:before,.modal-footer:after,.modal-footer:before,.modal-header:after,.modal-header:before,.nav:after,.nav:before,.navbar-collapse:after,.navbar-collapse:before,.navbar-header:after,.navbar-header:before,.navbar:after,.navbar:before,.pager:after,.pager:before,.panel-body:after,.panel-body:before,.row:after,.row:before{display:table;content:" "}.btn-group-vertical>.btn-group:after,.btn-toolbar:after,.clearfix:after,.container-fluid:after,.container:after,.dl-horizontal dd:after,.form-horizontal .form-group:after,.modal-footer:after,.modal-header:after,.nav:after,.navbar-collapse:after,.navbar-header:after,.navbar:after,.pager:after,.panel-body:after,.row:after{clear:both}.center-block{display:block;margin-right:auto;margin-left:auto}.pull-right{float:right!important}.pull-left{float:left!important}.hide{display:none!important}.show{display:block!important}.invisible{visibility:hidden}.text-hide{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.hidden{display:none!important}.affix{position:fixed}@-ms-viewport{width:device-width}.visible-lg,.visible-md,.visible-sm,.visible-xs{display:none!important}.visible-lg-block,.visible-lg-inline,.visible-lg-inline-block,.visible-md-block,.visible-md-inline,.visible-md-inline-block,.visible-sm-block,.visible-sm-inline,.visible-sm-inline-block,.visible-xs-block,.visible-xs-inline,.visible-xs-inline-block{display:none!important}@media (max-width:767px){.visible-xs{display:block!important}table.visible-xs{display:table!important}tr.visible-xs{display:table-row!important}td.visible-xs,th.visible-xs{display:table-cell!important}}@media (max-width:767px){.visible-xs-block{display:block!important}}@media (max-width:767px){.visible-xs-inline{display:inline!important}}@media (max-width:767px){.visible-xs-inline-block{display:inline-block!important}}@media (min-width:768px) and (max-width:991px){.visible-sm{display:block!important}table.visible-sm{display:table!important}tr.visible-sm{display:table-row!important}td.visible-sm,th.visible-sm{display:table-cell!important}}@media (min-width:768px) and (max-width:991px){.visible-sm-block{display:block!important}}@media (min-width:768px) and (max-width:991px){.visible-sm-inline{display:inline!important}}@media (min-width:768px) and (max-width:991px){.visible-sm-inline-block{display:inline-block!important}}@media (min-width:992px) and (max-width:1199px){.visible-md{display:block!important}table.visible-md{display:table!important}tr.visible-md{display:table-row!important}td.visible-md,th.visible-md{display:table-cell!important}}@media (min-width:992px) and (max-width:1199px){.visible-md-block{display:block!important}}@media (min-width:992px) and (max-width:1199px){.visible-md-inline{display:inline!important}}@media (min-width:992px) and (max-width:1199px){.visible-md-inline-block{display:inline-block!important}}@media (min-width:1200px){.visible-lg{display:block!important}table.visible-lg{display:table!important}tr.visible-lg{display:table-row!important}td.visible-lg,th.visible-lg{display:table-cell!important}}@media (min-width:1200px){.visible-lg-block{display:block!important}}@media (min-width:1200px){.visible-lg-inline{display:inline!important}}@media (min-width:1200px){.visible-lg-inline-block{display:inline-block!important}}@media (max-width:767px){.hidden-xs{display:none!important}}@media (min-width:768px) and (max-width:991px){.hidden-sm{display:none!important}}@media (min-width:992px) and (max-width:1199px){.hidden-md{display:none!important}}@media (min-width:1200px){.hidden-lg{display:none!important}}.visible-print{display:none!important}@media print{.visible-print{display:block!important}table.visible-print{display:table!important}tr.visible-print{display:table-row!important}td.visible-print,th.visible-print{display:table-cell!important}}.visible-print-block{display:none!important}@media print{.visible-print-block{display:block!important}}.visible-print-inline{display:none!important}@media print{.visible-print-inline{display:inline!important}}.visible-print-inline-block{display:none!important}@media print{.visible-print-inline-block{display:inline-block!important}}@media print{.hidden-print{display:none!important}} +/*# sourceMappingURL=bootstrap.min.css.map */
\ No newline at end of file diff --git a/interface/css/rspamd.css b/interface/css/rspamd.css index 14bb4e669..b99c3e2b3 100644 --- a/interface/css/rspamd.css +++ b/interface/css/rspamd.css @@ -30,12 +30,6 @@ textarea { font-family:"Courier New", Courier, monospace; } -.login { - width:480px; - margin-top:120px; - margin-left:-240px; - } - /* local overrides */ .disconnect { margin:9px 0 0; @@ -75,9 +69,6 @@ input.action-scores { padding:2px 0; text-align:center; } -.symbols-label { - font-size:11px !important; -} /* history table */ .table-log { @@ -164,13 +155,42 @@ input.action-scores { height: auto; } .widget-title-form label { + font-size: 12px; font-weight: normal; } +#history_page_size { + width:6em !important; + text-align: center; +} + +/* Symbols coloring */ +.symbol-default { + border-radius: 2px; + padding-left: 2px; + padding-right: 2px; +} +.symbol-default:hover { + background-color: #E6E6E6; +} +.symbol-negative { + background-color: #EEF9E7; +} +.symbol-positive { + background-color: #FBE9E5; +} +.symbol-special { + background-color: #E2E9FE; +} +.symbol-negative:hover { + background-color: #DCF9D3; +} +.symbol-positive:hover { + background-color: #FBD6D1; +} +.symbol-special:hover { + background-color: #CDDBFF; +} -.btn-upload-trigger { - position:relative; - z-index:1; - } .upload-textarea, .scan-textarea { width:100% !important; @@ -186,18 +206,6 @@ input.action-scores { margin-right: -10px !important; } -.row-bordered { - margin-bottom:13px; - border-bottom:1px solid #cdcdcd; - } -.symbol-description { - display:block; - margin:4px 0 0 6px; - font-size:10px; - font-weight:bold; - color:#666; - } - .list-textarea { width:100%; height:360px; @@ -375,44 +383,6 @@ td.maps-cell { .nopadding { padding:0 !important; } -.activity-list { - list-style:none outside none; - margin:0; - } - .activity-list li { - border-bottom:1px solid #EEEEEE; - display:block; - } - .activity-list li:last-child { - border-bottom:medium none; - } - .activity-list li a { - color:#888888; - display:block; - padding:7px 10px; - } - .activity-list li a:hover { - background-color:#FBFBFB; - } - .activity-list li a span { - color:#AAAAAA; - font-size:11px; - font-style:italic; - } - .activity-list li a i { - margin-right:10px; - opacity:0.6; - vertical-align:middle; - } -.recent-posts, .recent-comments, .recent-users { - margin:0; - padding:0; - } - .recent-posts li, .article-post li { - border-bottom:1px dotted #AEBDC8; - list-style:none outside none; - padding:10px; - } .modal-header { height:auto; padding:8px 15px 5px; @@ -542,12 +512,6 @@ td.maps-cell { #scanForm button { margin-top: 10px; } -#historyLog_wrapper div.row:first-child > div { - padding: 5px 20px 0 20px; - } -#historyLog_wrapper div.row:last-child > div { - padding: 5px 20px 0 20px; - } #throughput div.widget-content { text-align: center; @@ -673,21 +637,6 @@ input.radio { #clusterTable .col3 { width: 50%; } -table.dataTable thead .sorting { -background: url("../img/asc.png") no-repeat center right; -} -table.dataTable thead .sorting_asc { -background: url("../img/asc.png") no-repeat center right; -} -table.dataTable thead .sorting_desc { -background: url("../img/desc.png") no-repeat center right; -} -table.dataTable thead .sorting_asc_disabled { -background: url("../img/asc.png") no-repeat center right; -} -table.dataTable thead .sorting_desc_disabled { -background: url("../img/desc.png") no-repeat center right; -} #nprogress .bar { height: 1px; diff --git a/interface/index.html b/interface/index.html index 8d40da510..8b9b65c9d 100644 --- a/interface/index.html +++ b/interface/index.html @@ -287,22 +287,22 @@ <div class="widget-box"> <div class="widget-title"> - <form role="form" class="form-inline pull-right buttons"> - <div class="form-group widget-title-form"> + <div class="form-inline widget-title-form input-group-sm pull-right buttons"> <label for="selSymOrder">Symbols order:</label> <select id="selSymOrder" class="form-control"> <option value="magnitude" selected>Score magnitude</option> <option value="score">Score value</option> <option value="name">Name</option> </select> - </div> + <label for="history_page_size">Rows per page:</label> + <input id="history_page_size" class="form-control" value="25" min="1" type="number"> <button class="btn btn-danger btn-sm" id="resetHistory"> <i class="glyphicon glyphicon-remove-circle"></i> Reset </button> <button class="btn btn-info btn-sm" id="updateHistory"> <i class="glyphicon glyphicon-refresh"></i> Update </button> - </form> + </div> <span class="icon"><i class="glyphicon glyphicon-eye-open"></i></span> <h5>History</h5> </div> diff --git a/interface/js/app/history.js b/interface/js/app/history.js index 184be994c..d6e96f3e4 100644 --- a/interface/js/app/history.js +++ b/interface/js/app/history.js @@ -27,7 +27,22 @@ define(["jquery", "footable", "humanize"], function ($, _, Humanize) { "use strict"; - var rows_per_page = 25; + var page_size = { + errors: 25, + history: 25 + }; + + function set_page_size(n, callback) { + if (n !== page_size.history && n > 0) { + page_size.history = n; + if (callback) { + return callback(n); + } + } + return null; + } + + set_page_size($("#history_page_size").val()); var ui = {}; var prevVersion = null; @@ -177,24 +192,36 @@ define(["jquery", "footable", "humanize"], return {full:full, shrt:shrt}; } + function get_symbol_class(name, score) { + if (name.match(/^GREYLIST$/)) { + return "symbol-special"; + } + + if (score < 0) { + return "symbol-negative"; + } else if (score > 0) { + return "symbol-positive"; + } + return null; + } + preprocess_item(item); Object.keys(item.symbols).forEach(function (key) { - var str = null; var sym = item.symbols[key]; + sym.str = '<span class="symbol-default ' + get_symbol_class(sym.name, sym.score) + '"><strong>'; if (sym.description) { - str = "<strong><abbr data-sym-key=\"" + key + "\">" + sym.name + "</abbr></strong>(" + sym.score + ")"; - + sym.str += '<abbr data-sym-key="' + key + '">' + + sym.name + "</abbr></strong> (" + sym.score + ")</span>"; // Store description for tooltip symbolDescriptions[key] = sym.description; } else { - str = "<strong>" + sym.name + "</strong>(" + sym.score + ")"; + sym.str += sym.name + "</strong> (" + sym.score + ")</span>"; } if (sym.options) { - str += "[" + sym.options.join(",") + "]"; + sym.str += " [" + sym.options.join(",") + "]"; } - sym.str = str; }); unsorted_symbols.push(item.symbols); item.symbols = sort_symbols(item.symbols, compare_function); @@ -204,8 +231,7 @@ define(["jquery", "footable", "humanize"], sortValue: item.unix_time } }; - var scan_time = item.time_real.toFixed(3) + " / " + - item.time_virtual.toFixed(3); + var scan_time = item.time_real.toFixed(3) + " / " + item.time_virtual.toFixed(3); item.scan_time = { options: { sortValue: item.time_real @@ -370,7 +396,7 @@ define(["jquery", "footable", "humanize"], formatter: Humanize.compactInteger }, { name: "scan_time", - title: "Scan time", + title: '<span title="real / virtual">Scan time</span>', breakpoints: "xs sm md", style: { "font-size": "11px", @@ -458,7 +484,7 @@ define(["jquery", "footable", "humanize"], formatter: Humanize.compactInteger }, { name: "scan_time", - title: "Scan time", + title: '<span title="real / virtual">Scan time</span>', breakpoints: "xs sm", style: { "font-size": "11px", @@ -600,7 +626,7 @@ define(["jquery", "footable", "humanize"], paging: { enabled: true, limit: 5, - size: rows_per_page + size: page_size.history }, filtering: { enabled: true, @@ -641,7 +667,7 @@ define(["jquery", "footable", "humanize"], function waitForRowsDisplayed(rows_total, callback, iteration) { var i = (typeof iteration === "undefined") ? 10 : iteration; var num_rows = $("#historyTable > tbody > tr").length; - if (num_rows === rows_per_page || + if (num_rows === page_size.history || num_rows === rows_total) { return callback(); } else if (--i) { @@ -731,6 +757,9 @@ define(["jquery", "footable", "humanize"], var order = this.value; change_symbols_order(order); }); + $("#history_page_size").change(function () { + set_page_size(this.value, function (n) { tables.history.pageSize(n); }); + }); $(document).on("click", ".btn-sym-order button", function () { var order = this.value; $("#selSymOrder").val(order); @@ -771,7 +800,7 @@ define(["jquery", "footable", "humanize"], paging: { enabled: true, limit: 5, - size: rows_per_page + size: page_size.errors }, filtering: { enabled: true, diff --git a/interface/js/app/upload.js b/interface/js/app/upload.js index f5a5cf73f..466d5d25e 100644 --- a/interface/js/app/upload.js +++ b/interface/js/app/upload.js @@ -52,7 +52,7 @@ define(["jquery"], headers: headers, success: function (json) { cleanTextUpload(source); - if (json[0].data.success) { + if (json[0].status === true) { rspamd.alertMessage("alert-success", "Data successfully uploaded"); } } diff --git a/interface/js/lib/bootstrap.min.js b/interface/js/lib/bootstrap.min.js index 133aeecb9..9bcd2fcca 100644 --- a/interface/js/lib/bootstrap.min.js +++ b/interface/js/lib/bootstrap.min.js @@ -1,7 +1,7 @@ /*! - * Bootstrap v3.3.5 (http://getbootstrap.com) - * Copyright 2011-2015 Twitter, Inc. + * Bootstrap v3.3.7 (http://getbootstrap.com) + * Copyright 2011-2016 Twitter, Inc. * Licensed under the MIT license */ -if("undefined"==typeof jQuery)throw new Error("Bootstrap's JavaScript requires jQuery");+function(a){"use strict";var b=a.fn.jquery.split(" ")[0].split(".");if(b[0]<2&&b[1]<9||1==b[0]&&9==b[1]&&b[2]<1)throw new Error("Bootstrap's JavaScript requires jQuery version 1.9.1 or higher")}(jQuery),+function(a){"use strict";function b(){var a=document.createElement("bootstrap"),b={WebkitTransition:"webkitTransitionEnd",MozTransition:"transitionend",OTransition:"oTransitionEnd otransitionend",transition:"transitionend"};for(var c in b)if(void 0!==a.style[c])return{end:b[c]};return!1}a.fn.emulateTransitionEnd=function(b){var c=!1,d=this;a(this).one("bsTransitionEnd",function(){c=!0});var e=function(){c||a(d).trigger(a.support.transition.end)};return setTimeout(e,b),this},a(function(){a.support.transition=b(),a.support.transition&&(a.event.special.bsTransitionEnd={bindType:a.support.transition.end,delegateType:a.support.transition.end,handle:function(b){return a(b.target).is(this)?b.handleObj.handler.apply(this,arguments):void 0}})})}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var c=a(this),e=c.data("bs.alert");e||c.data("bs.alert",e=new d(this)),"string"==typeof b&&e[b].call(c)})}var c='[data-dismiss="alert"]',d=function(b){a(b).on("click",c,this.close)};d.VERSION="3.3.5",d.TRANSITION_DURATION=150,d.prototype.close=function(b){function c(){g.detach().trigger("closed.bs.alert").remove()}var e=a(this),f=e.attr("data-target");f||(f=e.attr("href"),f=f&&f.replace(/.*(?=#[^\s]*$)/,""));var g=a(f);b&&b.preventDefault(),g.length||(g=e.closest(".alert")),g.trigger(b=a.Event("close.bs.alert")),b.isDefaultPrevented()||(g.removeClass("in"),a.support.transition&&g.hasClass("fade")?g.one("bsTransitionEnd",c).emulateTransitionEnd(d.TRANSITION_DURATION):c())};var e=a.fn.alert;a.fn.alert=b,a.fn.alert.Constructor=d,a.fn.alert.noConflict=function(){return a.fn.alert=e,this},a(document).on("click.bs.alert.data-api",c,d.prototype.close)}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.button"),f="object"==typeof b&&b;e||d.data("bs.button",e=new c(this,f)),"toggle"==b?e.toggle():b&&e.setState(b)})}var c=function(b,d){this.$element=a(b),this.options=a.extend({},c.DEFAULTS,d),this.isLoading=!1};c.VERSION="3.3.5",c.DEFAULTS={loadingText:"loading..."},c.prototype.setState=function(b){var c="disabled",d=this.$element,e=d.is("input")?"val":"html",f=d.data();b+="Text",null==f.resetText&&d.data("resetText",d[e]()),setTimeout(a.proxy(function(){d[e](null==f[b]?this.options[b]:f[b]),"loadingText"==b?(this.isLoading=!0,d.addClass(c).attr(c,c)):this.isLoading&&(this.isLoading=!1,d.removeClass(c).removeAttr(c))},this),0)},c.prototype.toggle=function(){var a=!0,b=this.$element.closest('[data-toggle="buttons"]');if(b.length){var c=this.$element.find("input");"radio"==c.prop("type")?(c.prop("checked")&&(a=!1),b.find(".active").removeClass("active"),this.$element.addClass("active")):"checkbox"==c.prop("type")&&(c.prop("checked")!==this.$element.hasClass("active")&&(a=!1),this.$element.toggleClass("active")),c.prop("checked",this.$element.hasClass("active")),a&&c.trigger("change")}else this.$element.attr("aria-pressed",!this.$element.hasClass("active")),this.$element.toggleClass("active")};var d=a.fn.button;a.fn.button=b,a.fn.button.Constructor=c,a.fn.button.noConflict=function(){return a.fn.button=d,this},a(document).on("click.bs.button.data-api",'[data-toggle^="button"]',function(c){var d=a(c.target);d.hasClass("btn")||(d=d.closest(".btn")),b.call(d,"toggle"),a(c.target).is('input[type="radio"]')||a(c.target).is('input[type="checkbox"]')||c.preventDefault()}).on("focus.bs.button.data-api blur.bs.button.data-api",'[data-toggle^="button"]',function(b){a(b.target).closest(".btn").toggleClass("focus",/^focus(in)?$/.test(b.type))})}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.carousel"),f=a.extend({},c.DEFAULTS,d.data(),"object"==typeof b&&b),g="string"==typeof b?b:f.slide;e||d.data("bs.carousel",e=new c(this,f)),"number"==typeof b?e.to(b):g?e[g]():f.interval&&e.pause().cycle()})}var c=function(b,c){this.$element=a(b),this.$indicators=this.$element.find(".carousel-indicators"),this.options=c,this.paused=null,this.sliding=null,this.interval=null,this.$active=null,this.$items=null,this.options.keyboard&&this.$element.on("keydown.bs.carousel",a.proxy(this.keydown,this)),"hover"==this.options.pause&&!("ontouchstart"in document.documentElement)&&this.$element.on("mouseenter.bs.carousel",a.proxy(this.pause,this)).on("mouseleave.bs.carousel",a.proxy(this.cycle,this))};c.VERSION="3.3.5",c.TRANSITION_DURATION=600,c.DEFAULTS={interval:5e3,pause:"hover",wrap:!0,keyboard:!0},c.prototype.keydown=function(a){if(!/input|textarea/i.test(a.target.tagName)){switch(a.which){case 37:this.prev();break;case 39:this.next();break;default:return}a.preventDefault()}},c.prototype.cycle=function(b){return b||(this.paused=!1),this.interval&&clearInterval(this.interval),this.options.interval&&!this.paused&&(this.interval=setInterval(a.proxy(this.next,this),this.options.interval)),this},c.prototype.getItemIndex=function(a){return this.$items=a.parent().children(".item"),this.$items.index(a||this.$active)},c.prototype.getItemForDirection=function(a,b){var c=this.getItemIndex(b),d="prev"==a&&0===c||"next"==a&&c==this.$items.length-1;if(d&&!this.options.wrap)return b;var e="prev"==a?-1:1,f=(c+e)%this.$items.length;return this.$items.eq(f)},c.prototype.to=function(a){var b=this,c=this.getItemIndex(this.$active=this.$element.find(".item.active"));return a>this.$items.length-1||0>a?void 0:this.sliding?this.$element.one("slid.bs.carousel",function(){b.to(a)}):c==a?this.pause().cycle():this.slide(a>c?"next":"prev",this.$items.eq(a))},c.prototype.pause=function(b){return b||(this.paused=!0),this.$element.find(".next, .prev").length&&a.support.transition&&(this.$element.trigger(a.support.transition.end),this.cycle(!0)),this.interval=clearInterval(this.interval),this},c.prototype.next=function(){return this.sliding?void 0:this.slide("next")},c.prototype.prev=function(){return this.sliding?void 0:this.slide("prev")},c.prototype.slide=function(b,d){var e=this.$element.find(".item.active"),f=d||this.getItemForDirection(b,e),g=this.interval,h="next"==b?"left":"right",i=this;if(f.hasClass("active"))return this.sliding=!1;var j=f[0],k=a.Event("slide.bs.carousel",{relatedTarget:j,direction:h});if(this.$element.trigger(k),!k.isDefaultPrevented()){if(this.sliding=!0,g&&this.pause(),this.$indicators.length){this.$indicators.find(".active").removeClass("active");var l=a(this.$indicators.children()[this.getItemIndex(f)]);l&&l.addClass("active")}var m=a.Event("slid.bs.carousel",{relatedTarget:j,direction:h});return a.support.transition&&this.$element.hasClass("slide")?(f.addClass(b),f[0].offsetWidth,e.addClass(h),f.addClass(h),e.one("bsTransitionEnd",function(){f.removeClass([b,h].join(" ")).addClass("active"),e.removeClass(["active",h].join(" ")),i.sliding=!1,setTimeout(function(){i.$element.trigger(m)},0)}).emulateTransitionEnd(c.TRANSITION_DURATION)):(e.removeClass("active"),f.addClass("active"),this.sliding=!1,this.$element.trigger(m)),g&&this.cycle(),this}};var d=a.fn.carousel;a.fn.carousel=b,a.fn.carousel.Constructor=c,a.fn.carousel.noConflict=function(){return a.fn.carousel=d,this};var e=function(c){var d,e=a(this),f=a(e.attr("data-target")||(d=e.attr("href"))&&d.replace(/.*(?=#[^\s]+$)/,""));if(f.hasClass("carousel")){var g=a.extend({},f.data(),e.data()),h=e.attr("data-slide-to");h&&(g.interval=!1),b.call(f,g),h&&f.data("bs.carousel").to(h),c.preventDefault()}};a(document).on("click.bs.carousel.data-api","[data-slide]",e).on("click.bs.carousel.data-api","[data-slide-to]",e),a(window).on("load",function(){a('[data-ride="carousel"]').each(function(){var c=a(this);b.call(c,c.data())})})}(jQuery),+function(a){"use strict";function b(b){var c,d=b.attr("data-target")||(c=b.attr("href"))&&c.replace(/.*(?=#[^\s]+$)/,"");return a(d)}function c(b){return this.each(function(){var c=a(this),e=c.data("bs.collapse"),f=a.extend({},d.DEFAULTS,c.data(),"object"==typeof b&&b);!e&&f.toggle&&/show|hide/.test(b)&&(f.toggle=!1),e||c.data("bs.collapse",e=new d(this,f)),"string"==typeof b&&e[b]()})}var d=function(b,c){this.$element=a(b),this.options=a.extend({},d.DEFAULTS,c),this.$trigger=a('[data-toggle="collapse"][href="#'+b.id+'"],[data-toggle="collapse"][data-target="#'+b.id+'"]'),this.transitioning=null,this.options.parent?this.$parent=this.getParent():this.addAriaAndCollapsedClass(this.$element,this.$trigger),this.options.toggle&&this.toggle()};d.VERSION="3.3.5",d.TRANSITION_DURATION=350,d.DEFAULTS={toggle:!0},d.prototype.dimension=function(){var a=this.$element.hasClass("width");return a?"width":"height"},d.prototype.show=function(){if(!this.transitioning&&!this.$element.hasClass("in")){var b,e=this.$parent&&this.$parent.children(".panel").children(".in, .collapsing");if(!(e&&e.length&&(b=e.data("bs.collapse"),b&&b.transitioning))){var f=a.Event("show.bs.collapse");if(this.$element.trigger(f),!f.isDefaultPrevented()){e&&e.length&&(c.call(e,"hide"),b||e.data("bs.collapse",null));var g=this.dimension();this.$element.removeClass("collapse").addClass("collapsing")[g](0).attr("aria-expanded",!0),this.$trigger.removeClass("collapsed").attr("aria-expanded",!0),this.transitioning=1;var h=function(){this.$element.removeClass("collapsing").addClass("collapse in")[g](""),this.transitioning=0,this.$element.trigger("shown.bs.collapse")};if(!a.support.transition)return h.call(this);var i=a.camelCase(["scroll",g].join("-"));this.$element.one("bsTransitionEnd",a.proxy(h,this)).emulateTransitionEnd(d.TRANSITION_DURATION)[g](this.$element[0][i])}}}},d.prototype.hide=function(){if(!this.transitioning&&this.$element.hasClass("in")){var b=a.Event("hide.bs.collapse");if(this.$element.trigger(b),!b.isDefaultPrevented()){var c=this.dimension();this.$element[c](this.$element[c]())[0].offsetHeight,this.$element.addClass("collapsing").removeClass("collapse in").attr("aria-expanded",!1),this.$trigger.addClass("collapsed").attr("aria-expanded",!1),this.transitioning=1;var e=function(){this.transitioning=0,this.$element.removeClass("collapsing").addClass("collapse").trigger("hidden.bs.collapse")};return a.support.transition?void this.$element[c](0).one("bsTransitionEnd",a.proxy(e,this)).emulateTransitionEnd(d.TRANSITION_DURATION):e.call(this)}}},d.prototype.toggle=function(){this[this.$element.hasClass("in")?"hide":"show"]()},d.prototype.getParent=function(){return a(this.options.parent).find('[data-toggle="collapse"][data-parent="'+this.options.parent+'"]').each(a.proxy(function(c,d){var e=a(d);this.addAriaAndCollapsedClass(b(e),e)},this)).end()},d.prototype.addAriaAndCollapsedClass=function(a,b){var c=a.hasClass("in");a.attr("aria-expanded",c),b.toggleClass("collapsed",!c).attr("aria-expanded",c)};var e=a.fn.collapse;a.fn.collapse=c,a.fn.collapse.Constructor=d,a.fn.collapse.noConflict=function(){return a.fn.collapse=e,this},a(document).on("click.bs.collapse.data-api",'[data-toggle="collapse"]',function(d){var e=a(this);e.attr("data-target")||d.preventDefault();var f=b(e),g=f.data("bs.collapse"),h=g?"toggle":e.data();c.call(f,h)})}(jQuery),+function(a){"use strict";function b(b){var c=b.attr("data-target");c||(c=b.attr("href"),c=c&&/#[A-Za-z]/.test(c)&&c.replace(/.*(?=#[^\s]*$)/,""));var d=c&&a(c);return d&&d.length?d:b.parent()}function c(c){c&&3===c.which||(a(e).remove(),a(f).each(function(){var d=a(this),e=b(d),f={relatedTarget:this};e.hasClass("open")&&(c&&"click"==c.type&&/input|textarea/i.test(c.target.tagName)&&a.contains(e[0],c.target)||(e.trigger(c=a.Event("hide.bs.dropdown",f)),c.isDefaultPrevented()||(d.attr("aria-expanded","false"),e.removeClass("open").trigger("hidden.bs.dropdown",f))))}))}function d(b){return this.each(function(){var c=a(this),d=c.data("bs.dropdown");d||c.data("bs.dropdown",d=new g(this)),"string"==typeof b&&d[b].call(c)})}var e=".dropdown-backdrop",f='[data-toggle="dropdown"]',g=function(b){a(b).on("click.bs.dropdown",this.toggle)};g.VERSION="3.3.5",g.prototype.toggle=function(d){var e=a(this);if(!e.is(".disabled, :disabled")){var f=b(e),g=f.hasClass("open");if(c(),!g){"ontouchstart"in document.documentElement&&!f.closest(".navbar-nav").length&&a(document.createElement("div")).addClass("dropdown-backdrop").insertAfter(a(this)).on("click",c);var h={relatedTarget:this};if(f.trigger(d=a.Event("show.bs.dropdown",h)),d.isDefaultPrevented())return;e.trigger("focus").attr("aria-expanded","true"),f.toggleClass("open").trigger("shown.bs.dropdown",h)}return!1}},g.prototype.keydown=function(c){if(/(38|40|27|32)/.test(c.which)&&!/input|textarea/i.test(c.target.tagName)){var d=a(this);if(c.preventDefault(),c.stopPropagation(),!d.is(".disabled, :disabled")){var e=b(d),g=e.hasClass("open");if(!g&&27!=c.which||g&&27==c.which)return 27==c.which&&e.find(f).trigger("focus"),d.trigger("click");var h=" li:not(.disabled):visible a",i=e.find(".dropdown-menu"+h);if(i.length){var j=i.index(c.target);38==c.which&&j>0&&j--,40==c.which&&j<i.length-1&&j++,~j||(j=0),i.eq(j).trigger("focus")}}}};var h=a.fn.dropdown;a.fn.dropdown=d,a.fn.dropdown.Constructor=g,a.fn.dropdown.noConflict=function(){return a.fn.dropdown=h,this},a(document).on("click.bs.dropdown.data-api",c).on("click.bs.dropdown.data-api",".dropdown form",function(a){a.stopPropagation()}).on("click.bs.dropdown.data-api",f,g.prototype.toggle).on("keydown.bs.dropdown.data-api",f,g.prototype.keydown).on("keydown.bs.dropdown.data-api",".dropdown-menu",g.prototype.keydown)}(jQuery),+function(a){"use strict";function b(b,d){return this.each(function(){var e=a(this),f=e.data("bs.modal"),g=a.extend({},c.DEFAULTS,e.data(),"object"==typeof b&&b);f||e.data("bs.modal",f=new c(this,g)),"string"==typeof b?f[b](d):g.show&&f.show(d)})}var c=function(b,c){this.options=c,this.$body=a(document.body),this.$element=a(b),this.$dialog=this.$element.find(".modal-dialog"),this.$backdrop=null,this.isShown=null,this.originalBodyPad=null,this.scrollbarWidth=0,this.ignoreBackdropClick=!1,this.options.remote&&this.$element.find(".modal-content").load(this.options.remote,a.proxy(function(){this.$element.trigger("loaded.bs.modal")},this))};c.VERSION="3.3.5",c.TRANSITION_DURATION=300,c.BACKDROP_TRANSITION_DURATION=150,c.DEFAULTS={backdrop:!0,keyboard:!0,show:!0},c.prototype.toggle=function(a){return this.isShown?this.hide():this.show(a)},c.prototype.show=function(b){var d=this,e=a.Event("show.bs.modal",{relatedTarget:b});this.$element.trigger(e),this.isShown||e.isDefaultPrevented()||(this.isShown=!0,this.checkScrollbar(),this.setScrollbar(),this.$body.addClass("modal-open"),this.escape(),this.resize(),this.$element.on("click.dismiss.bs.modal",'[data-dismiss="modal"]',a.proxy(this.hide,this)),this.$dialog.on("mousedown.dismiss.bs.modal",function(){d.$element.one("mouseup.dismiss.bs.modal",function(b){a(b.target).is(d.$element)&&(d.ignoreBackdropClick=!0)})}),this.backdrop(function(){var e=a.support.transition&&d.$element.hasClass("fade");d.$element.parent().length||d.$element.appendTo(d.$body),d.$element.show().scrollTop(0),d.adjustDialog(),e&&d.$element[0].offsetWidth,d.$element.addClass("in"),d.enforceFocus();var f=a.Event("shown.bs.modal",{relatedTarget:b});e?d.$dialog.one("bsTransitionEnd",function(){d.$element.trigger("focus").trigger(f)}).emulateTransitionEnd(c.TRANSITION_DURATION):d.$element.trigger("focus").trigger(f)}))},c.prototype.hide=function(b){b&&b.preventDefault(),b=a.Event("hide.bs.modal"),this.$element.trigger(b),this.isShown&&!b.isDefaultPrevented()&&(this.isShown=!1,this.escape(),this.resize(),a(document).off("focusin.bs.modal"),this.$element.removeClass("in").off("click.dismiss.bs.modal").off("mouseup.dismiss.bs.modal"),this.$dialog.off("mousedown.dismiss.bs.modal"),a.support.transition&&this.$element.hasClass("fade")?this.$element.one("bsTransitionEnd",a.proxy(this.hideModal,this)).emulateTransitionEnd(c.TRANSITION_DURATION):this.hideModal())},c.prototype.enforceFocus=function(){a(document).off("focusin.bs.modal").on("focusin.bs.modal",a.proxy(function(a){this.$element[0]===a.target||this.$element.has(a.target).length||this.$element.trigger("focus")},this))},c.prototype.escape=function(){this.isShown&&this.options.keyboard?this.$element.on("keydown.dismiss.bs.modal",a.proxy(function(a){27==a.which&&this.hide()},this)):this.isShown||this.$element.off("keydown.dismiss.bs.modal")},c.prototype.resize=function(){this.isShown?a(window).on("resize.bs.modal",a.proxy(this.handleUpdate,this)):a(window).off("resize.bs.modal")},c.prototype.hideModal=function(){var a=this;this.$element.hide(),this.backdrop(function(){a.$body.removeClass("modal-open"),a.resetAdjustments(),a.resetScrollbar(),a.$element.trigger("hidden.bs.modal")})},c.prototype.removeBackdrop=function(){this.$backdrop&&this.$backdrop.remove(),this.$backdrop=null},c.prototype.backdrop=function(b){var d=this,e=this.$element.hasClass("fade")?"fade":"";if(this.isShown&&this.options.backdrop){var f=a.support.transition&&e;if(this.$backdrop=a(document.createElement("div")).addClass("modal-backdrop "+e).appendTo(this.$body),this.$element.on("click.dismiss.bs.modal",a.proxy(function(a){return this.ignoreBackdropClick?void(this.ignoreBackdropClick=!1):void(a.target===a.currentTarget&&("static"==this.options.backdrop?this.$element[0].focus():this.hide()))},this)),f&&this.$backdrop[0].offsetWidth,this.$backdrop.addClass("in"),!b)return;f?this.$backdrop.one("bsTransitionEnd",b).emulateTransitionEnd(c.BACKDROP_TRANSITION_DURATION):b()}else if(!this.isShown&&this.$backdrop){this.$backdrop.removeClass("in");var g=function(){d.removeBackdrop(),b&&b()};a.support.transition&&this.$element.hasClass("fade")?this.$backdrop.one("bsTransitionEnd",g).emulateTransitionEnd(c.BACKDROP_TRANSITION_DURATION):g()}else b&&b()},c.prototype.handleUpdate=function(){this.adjustDialog()},c.prototype.adjustDialog=function(){var a=this.$element[0].scrollHeight>document.documentElement.clientHeight;this.$element.css({paddingLeft:!this.bodyIsOverflowing&&a?this.scrollbarWidth:"",paddingRight:this.bodyIsOverflowing&&!a?this.scrollbarWidth:""})},c.prototype.resetAdjustments=function(){this.$element.css({paddingLeft:"",paddingRight:""})},c.prototype.checkScrollbar=function(){var a=window.innerWidth;if(!a){var b=document.documentElement.getBoundingClientRect();a=b.right-Math.abs(b.left)}this.bodyIsOverflowing=document.body.clientWidth<a,this.scrollbarWidth=this.measureScrollbar()},c.prototype.setScrollbar=function(){var a=parseInt(this.$body.css("padding-right")||0,10);this.originalBodyPad=document.body.style.paddingRight||"",this.bodyIsOverflowing&&this.$body.css("padding-right",a+this.scrollbarWidth)},c.prototype.resetScrollbar=function(){this.$body.css("padding-right",this.originalBodyPad)},c.prototype.measureScrollbar=function(){var a=document.createElement("div");a.className="modal-scrollbar-measure",this.$body.append(a);var b=a.offsetWidth-a.clientWidth;return this.$body[0].removeChild(a),b};var d=a.fn.modal;a.fn.modal=b,a.fn.modal.Constructor=c,a.fn.modal.noConflict=function(){return a.fn.modal=d,this},a(document).on("click.bs.modal.data-api",'[data-toggle="modal"]',function(c){var d=a(this),e=d.attr("href"),f=a(d.attr("data-target")||e&&e.replace(/.*(?=#[^\s]+$)/,"")),g=f.data("bs.modal")?"toggle":a.extend({remote:!/#/.test(e)&&e},f.data(),d.data());d.is("a")&&c.preventDefault(),f.one("show.bs.modal",function(a){a.isDefaultPrevented()||f.one("hidden.bs.modal",function(){d.is(":visible")&&d.trigger("focus")})}),b.call(f,g,this)})}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.tooltip"),f="object"==typeof b&&b;(e||!/destroy|hide/.test(b))&&(e||d.data("bs.tooltip",e=new c(this,f)),"string"==typeof b&&e[b]())})}var c=function(a,b){this.type=null,this.options=null,this.enabled=null,this.timeout=null,this.hoverState=null,this.$element=null,this.inState=null,this.init("tooltip",a,b)};c.VERSION="3.3.5",c.TRANSITION_DURATION=150,c.DEFAULTS={animation:!0,placement:"top",selector:!1,template:'<div class="tooltip" role="tooltip"><div class="tooltip-arrow"></div><div class="tooltip-inner"></div></div>',trigger:"hover focus",title:"",delay:0,html:!1,container:!1,viewport:{selector:"body",padding:0}},c.prototype.init=function(b,c,d){if(this.enabled=!0,this.type=b,this.$element=a(c),this.options=this.getOptions(d),this.$viewport=this.options.viewport&&a(a.isFunction(this.options.viewport)?this.options.viewport.call(this,this.$element):this.options.viewport.selector||this.options.viewport),this.inState={click:!1,hover:!1,focus:!1},this.$element[0]instanceof document.constructor&&!this.options.selector)throw new Error("`selector` option must be specified when initializing "+this.type+" on the window.document object!");for(var e=this.options.trigger.split(" "),f=e.length;f--;){var g=e[f];if("click"==g)this.$element.on("click."+this.type,this.options.selector,a.proxy(this.toggle,this));else if("manual"!=g){var h="hover"==g?"mouseenter":"focusin",i="hover"==g?"mouseleave":"focusout";this.$element.on(h+"."+this.type,this.options.selector,a.proxy(this.enter,this)),this.$element.on(i+"."+this.type,this.options.selector,a.proxy(this.leave,this))}}this.options.selector?this._options=a.extend({},this.options,{trigger:"manual",selector:""}):this.fixTitle()},c.prototype.getDefaults=function(){return c.DEFAULTS},c.prototype.getOptions=function(b){return b=a.extend({},this.getDefaults(),this.$element.data(),b),b.delay&&"number"==typeof b.delay&&(b.delay={show:b.delay,hide:b.delay}),b},c.prototype.getDelegateOptions=function(){var b={},c=this.getDefaults();return this._options&&a.each(this._options,function(a,d){c[a]!=d&&(b[a]=d)}),b},c.prototype.enter=function(b){var c=b instanceof this.constructor?b:a(b.currentTarget).data("bs."+this.type);return c||(c=new this.constructor(b.currentTarget,this.getDelegateOptions()),a(b.currentTarget).data("bs."+this.type,c)),b instanceof a.Event&&(c.inState["focusin"==b.type?"focus":"hover"]=!0),c.tip().hasClass("in")||"in"==c.hoverState?void(c.hoverState="in"):(clearTimeout(c.timeout),c.hoverState="in",c.options.delay&&c.options.delay.show?void(c.timeout=setTimeout(function(){"in"==c.hoverState&&c.show()},c.options.delay.show)):c.show())},c.prototype.isInStateTrue=function(){for(var a in this.inState)if(this.inState[a])return!0;return!1},c.prototype.leave=function(b){var c=b instanceof this.constructor?b:a(b.currentTarget).data("bs."+this.type);return c||(c=new this.constructor(b.currentTarget,this.getDelegateOptions()),a(b.currentTarget).data("bs."+this.type,c)),b instanceof a.Event&&(c.inState["focusout"==b.type?"focus":"hover"]=!1),c.isInStateTrue()?void 0:(clearTimeout(c.timeout),c.hoverState="out",c.options.delay&&c.options.delay.hide?void(c.timeout=setTimeout(function(){"out"==c.hoverState&&c.hide()},c.options.delay.hide)):c.hide())},c.prototype.show=function(){var b=a.Event("show.bs."+this.type);if(this.hasContent()&&this.enabled){this.$element.trigger(b);var d=a.contains(this.$element[0].ownerDocument.documentElement,this.$element[0]);if(b.isDefaultPrevented()||!d)return;var e=this,f=this.tip(),g=this.getUID(this.type);this.setContent(),f.attr("id",g),this.$element.attr("aria-describedby",g),this.options.animation&&f.addClass("fade");var h="function"==typeof this.options.placement?this.options.placement.call(this,f[0],this.$element[0]):this.options.placement,i=/\s?auto?\s?/i,j=i.test(h);j&&(h=h.replace(i,"")||"top"),f.detach().css({top:0,left:0,display:"block"}).addClass(h).data("bs."+this.type,this),this.options.container?f.appendTo(this.options.container):f.insertAfter(this.$element),this.$element.trigger("inserted.bs."+this.type);var k=this.getPosition(),l=f[0].offsetWidth,m=f[0].offsetHeight;if(j){var n=h,o=this.getPosition(this.$viewport);h="bottom"==h&&k.bottom+m>o.bottom?"top":"top"==h&&k.top-m<o.top?"bottom":"right"==h&&k.right+l>o.width?"left":"left"==h&&k.left-l<o.left?"right":h,f.removeClass(n).addClass(h)}var p=this.getCalculatedOffset(h,k,l,m);this.applyPlacement(p,h);var q=function(){var a=e.hoverState;e.$element.trigger("shown.bs."+e.type),e.hoverState=null,"out"==a&&e.leave(e)};a.support.transition&&this.$tip.hasClass("fade")?f.one("bsTransitionEnd",q).emulateTransitionEnd(c.TRANSITION_DURATION):q()}},c.prototype.applyPlacement=function(b,c){var d=this.tip(),e=d[0].offsetWidth,f=d[0].offsetHeight,g=parseInt(d.css("margin-top"),10),h=parseInt(d.css("margin-left"),10);isNaN(g)&&(g=0),isNaN(h)&&(h=0),b.top+=g,b.left+=h,a.offset.setOffset(d[0],a.extend({using:function(a){d.css({top:Math.round(a.top),left:Math.round(a.left)})}},b),0),d.addClass("in");var i=d[0].offsetWidth,j=d[0].offsetHeight;"top"==c&&j!=f&&(b.top=b.top+f-j);var k=this.getViewportAdjustedDelta(c,b,i,j);k.left?b.left+=k.left:b.top+=k.top;var l=/top|bottom/.test(c),m=l?2*k.left-e+i:2*k.top-f+j,n=l?"offsetWidth":"offsetHeight";d.offset(b),this.replaceArrow(m,d[0][n],l)},c.prototype.replaceArrow=function(a,b,c){this.arrow().css(c?"left":"top",50*(1-a/b)+"%").css(c?"top":"left","")},c.prototype.setContent=function(){var a=this.tip(),b=this.getTitle();a.find(".tooltip-inner")[this.options.html?"html":"text"](b),a.removeClass("fade in top bottom left right")},c.prototype.hide=function(b){function d(){"in"!=e.hoverState&&f.detach(),e.$element.removeAttr("aria-describedby").trigger("hidden.bs."+e.type),b&&b()}var e=this,f=a(this.$tip),g=a.Event("hide.bs."+this.type);return this.$element.trigger(g),g.isDefaultPrevented()?void 0:(f.removeClass("in"),a.support.transition&&f.hasClass("fade")?f.one("bsTransitionEnd",d).emulateTransitionEnd(c.TRANSITION_DURATION):d(),this.hoverState=null,this)},c.prototype.fixTitle=function(){var a=this.$element;(a.attr("title")||"string"!=typeof a.attr("data-original-title"))&&a.attr("data-original-title",a.attr("title")||"").attr("title","")},c.prototype.hasContent=function(){return this.getTitle()},c.prototype.getPosition=function(b){b=b||this.$element;var c=b[0],d="BODY"==c.tagName,e=c.getBoundingClientRect();null==e.width&&(e=a.extend({},e,{width:e.right-e.left,height:e.bottom-e.top}));var f=d?{top:0,left:0}:b.offset(),g={scroll:d?document.documentElement.scrollTop||document.body.scrollTop:b.scrollTop()},h=d?{width:a(window).width(),height:a(window).height()}:null;return a.extend({},e,g,h,f)},c.prototype.getCalculatedOffset=function(a,b,c,d){return"bottom"==a?{top:b.top+b.height,left:b.left+b.width/2-c/2}:"top"==a?{top:b.top-d,left:b.left+b.width/2-c/2}:"left"==a?{top:b.top+b.height/2-d/2,left:b.left-c}:{top:b.top+b.height/2-d/2,left:b.left+b.width}},c.prototype.getViewportAdjustedDelta=function(a,b,c,d){var e={top:0,left:0};if(!this.$viewport)return e;var f=this.options.viewport&&this.options.viewport.padding||0,g=this.getPosition(this.$viewport);if(/right|left/.test(a)){var h=b.top-f-g.scroll,i=b.top+f-g.scroll+d;h<g.top?e.top=g.top-h:i>g.top+g.height&&(e.top=g.top+g.height-i)}else{var j=b.left-f,k=b.left+f+c;j<g.left?e.left=g.left-j:k>g.right&&(e.left=g.left+g.width-k)}return e},c.prototype.getTitle=function(){var a,b=this.$element,c=this.options;return a=b.attr("data-original-title")||("function"==typeof c.title?c.title.call(b[0]):c.title)},c.prototype.getUID=function(a){do a+=~~(1e6*Math.random());while(document.getElementById(a));return a},c.prototype.tip=function(){if(!this.$tip&&(this.$tip=a(this.options.template),1!=this.$tip.length))throw new Error(this.type+" `template` option must consist of exactly 1 top-level element!");return this.$tip},c.prototype.arrow=function(){return this.$arrow=this.$arrow||this.tip().find(".tooltip-arrow")},c.prototype.enable=function(){this.enabled=!0},c.prototype.disable=function(){this.enabled=!1},c.prototype.toggleEnabled=function(){this.enabled=!this.enabled},c.prototype.toggle=function(b){var c=this;b&&(c=a(b.currentTarget).data("bs."+this.type),c||(c=new this.constructor(b.currentTarget,this.getDelegateOptions()),a(b.currentTarget).data("bs."+this.type,c))),b?(c.inState.click=!c.inState.click,c.isInStateTrue()?c.enter(c):c.leave(c)):c.tip().hasClass("in")?c.leave(c):c.enter(c)},c.prototype.destroy=function(){var a=this;clearTimeout(this.timeout),this.hide(function(){a.$element.off("."+a.type).removeData("bs."+a.type),a.$tip&&a.$tip.detach(),a.$tip=null,a.$arrow=null,a.$viewport=null})};var d=a.fn.tooltip;a.fn.tooltip=b,a.fn.tooltip.Constructor=c,a.fn.tooltip.noConflict=function(){return a.fn.tooltip=d,this}}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.popover"),f="object"==typeof b&&b;(e||!/destroy|hide/.test(b))&&(e||d.data("bs.popover",e=new c(this,f)),"string"==typeof b&&e[b]())})}var c=function(a,b){this.init("popover",a,b)};if(!a.fn.tooltip)throw new Error("Popover requires tooltip.js");c.VERSION="3.3.5",c.DEFAULTS=a.extend({},a.fn.tooltip.Constructor.DEFAULTS,{placement:"right",trigger:"click",content:"",template:'<div class="popover" role="tooltip"><div class="arrow"></div><h3 class="popover-title"></h3><div class="popover-content"></div></div>'}),c.prototype=a.extend({},a.fn.tooltip.Constructor.prototype),c.prototype.constructor=c,c.prototype.getDefaults=function(){return c.DEFAULTS},c.prototype.setContent=function(){var a=this.tip(),b=this.getTitle(),c=this.getContent();a.find(".popover-title")[this.options.html?"html":"text"](b),a.find(".popover-content").children().detach().end()[this.options.html?"string"==typeof c?"html":"append":"text"](c),a.removeClass("fade top bottom left right in"),a.find(".popover-title").html()||a.find(".popover-title").hide()},c.prototype.hasContent=function(){return this.getTitle()||this.getContent()},c.prototype.getContent=function(){var a=this.$element,b=this.options;return a.attr("data-content")||("function"==typeof b.content?b.content.call(a[0]):b.content)},c.prototype.arrow=function(){return this.$arrow=this.$arrow||this.tip().find(".arrow")};var d=a.fn.popover;a.fn.popover=b,a.fn.popover.Constructor=c,a.fn.popover.noConflict=function(){return a.fn.popover=d,this}}(jQuery),+function(a){"use strict";function b(c,d){this.$body=a(document.body),this.$scrollElement=a(a(c).is(document.body)?window:c),this.options=a.extend({},b.DEFAULTS,d),this.selector=(this.options.target||"")+" .nav li > a",this.offsets=[],this.targets=[],this.activeTarget=null,this.scrollHeight=0,this.$scrollElement.on("scroll.bs.scrollspy",a.proxy(this.process,this)),this.refresh(),this.process()}function c(c){return this.each(function(){var d=a(this),e=d.data("bs.scrollspy"),f="object"==typeof c&&c;e||d.data("bs.scrollspy",e=new b(this,f)),"string"==typeof c&&e[c]()})}b.VERSION="3.3.5",b.DEFAULTS={offset:10},b.prototype.getScrollHeight=function(){return this.$scrollElement[0].scrollHeight||Math.max(this.$body[0].scrollHeight,document.documentElement.scrollHeight)},b.prototype.refresh=function(){var b=this,c="offset",d=0;this.offsets=[],this.targets=[],this.scrollHeight=this.getScrollHeight(),a.isWindow(this.$scrollElement[0])||(c="position",d=this.$scrollElement.scrollTop()),this.$body.find(this.selector).map(function(){var b=a(this),e=b.data("target")||b.attr("href"),f=/^#./.test(e)&&a(e);return f&&f.length&&f.is(":visible")&&[[f[c]().top+d,e]]||null}).sort(function(a,b){return a[0]-b[0]}).each(function(){b.offsets.push(this[0]),b.targets.push(this[1])})},b.prototype.process=function(){var a,b=this.$scrollElement.scrollTop()+this.options.offset,c=this.getScrollHeight(),d=this.options.offset+c-this.$scrollElement.height(),e=this.offsets,f=this.targets,g=this.activeTarget;if(this.scrollHeight!=c&&this.refresh(),b>=d)return g!=(a=f[f.length-1])&&this.activate(a);if(g&&b<e[0])return this.activeTarget=null,this.clear();for(a=e.length;a--;)g!=f[a]&&b>=e[a]&&(void 0===e[a+1]||b<e[a+1])&&this.activate(f[a])},b.prototype.activate=function(b){this.activeTarget=b,this.clear();var c=this.selector+'[data-target="'+b+'"],'+this.selector+'[href="'+b+'"]',d=a(c).parents("li").addClass("active");d.parent(".dropdown-menu").length&&(d=d.closest("li.dropdown").addClass("active")), -d.trigger("activate.bs.scrollspy")},b.prototype.clear=function(){a(this.selector).parentsUntil(this.options.target,".active").removeClass("active")};var d=a.fn.scrollspy;a.fn.scrollspy=c,a.fn.scrollspy.Constructor=b,a.fn.scrollspy.noConflict=function(){return a.fn.scrollspy=d,this},a(window).on("load.bs.scrollspy.data-api",function(){a('[data-spy="scroll"]').each(function(){var b=a(this);c.call(b,b.data())})})}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.tab");e||d.data("bs.tab",e=new c(this)),"string"==typeof b&&e[b]()})}var c=function(b){this.element=a(b)};c.VERSION="3.3.5",c.TRANSITION_DURATION=150,c.prototype.show=function(){var b=this.element,c=b.closest("ul:not(.dropdown-menu)"),d=b.data("target");if(d||(d=b.attr("href"),d=d&&d.replace(/.*(?=#[^\s]*$)/,"")),!b.parent("li").hasClass("active")){var e=c.find(".active:last a"),f=a.Event("hide.bs.tab",{relatedTarget:b[0]}),g=a.Event("show.bs.tab",{relatedTarget:e[0]});if(e.trigger(f),b.trigger(g),!g.isDefaultPrevented()&&!f.isDefaultPrevented()){var h=a(d);this.activate(b.closest("li"),c),this.activate(h,h.parent(),function(){e.trigger({type:"hidden.bs.tab",relatedTarget:b[0]}),b.trigger({type:"shown.bs.tab",relatedTarget:e[0]})})}}},c.prototype.activate=function(b,d,e){function f(){g.removeClass("active").find("> .dropdown-menu > .active").removeClass("active").end().find('[data-toggle="tab"]').attr("aria-expanded",!1),b.addClass("active").find('[data-toggle="tab"]').attr("aria-expanded",!0),h?(b[0].offsetWidth,b.addClass("in")):b.removeClass("fade"),b.parent(".dropdown-menu").length&&b.closest("li.dropdown").addClass("active").end().find('[data-toggle="tab"]').attr("aria-expanded",!0),e&&e()}var g=d.find("> .active"),h=e&&a.support.transition&&(g.length&&g.hasClass("fade")||!!d.find("> .fade").length);g.length&&h?g.one("bsTransitionEnd",f).emulateTransitionEnd(c.TRANSITION_DURATION):f(),g.removeClass("in")};var d=a.fn.tab;a.fn.tab=b,a.fn.tab.Constructor=c,a.fn.tab.noConflict=function(){return a.fn.tab=d,this};var e=function(c){c.preventDefault(),b.call(a(this),"show")};a(document).on("click.bs.tab.data-api",'[data-toggle="tab"]',e).on("click.bs.tab.data-api",'[data-toggle="pill"]',e)}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.affix"),f="object"==typeof b&&b;e||d.data("bs.affix",e=new c(this,f)),"string"==typeof b&&e[b]()})}var c=function(b,d){this.options=a.extend({},c.DEFAULTS,d),this.$target=a(this.options.target).on("scroll.bs.affix.data-api",a.proxy(this.checkPosition,this)).on("click.bs.affix.data-api",a.proxy(this.checkPositionWithEventLoop,this)),this.$element=a(b),this.affixed=null,this.unpin=null,this.pinnedOffset=null,this.checkPosition()};c.VERSION="3.3.5",c.RESET="affix affix-top affix-bottom",c.DEFAULTS={offset:0,target:window},c.prototype.getState=function(a,b,c,d){var e=this.$target.scrollTop(),f=this.$element.offset(),g=this.$target.height();if(null!=c&&"top"==this.affixed)return c>e?"top":!1;if("bottom"==this.affixed)return null!=c?e+this.unpin<=f.top?!1:"bottom":a-d>=e+g?!1:"bottom";var h=null==this.affixed,i=h?e:f.top,j=h?g:b;return null!=c&&c>=e?"top":null!=d&&i+j>=a-d?"bottom":!1},c.prototype.getPinnedOffset=function(){if(this.pinnedOffset)return this.pinnedOffset;this.$element.removeClass(c.RESET).addClass("affix");var a=this.$target.scrollTop(),b=this.$element.offset();return this.pinnedOffset=b.top-a},c.prototype.checkPositionWithEventLoop=function(){setTimeout(a.proxy(this.checkPosition,this),1)},c.prototype.checkPosition=function(){if(this.$element.is(":visible")){var b=this.$element.height(),d=this.options.offset,e=d.top,f=d.bottom,g=Math.max(a(document).height(),a(document.body).height());"object"!=typeof d&&(f=e=d),"function"==typeof e&&(e=d.top(this.$element)),"function"==typeof f&&(f=d.bottom(this.$element));var h=this.getState(g,b,e,f);if(this.affixed!=h){null!=this.unpin&&this.$element.css("top","");var i="affix"+(h?"-"+h:""),j=a.Event(i+".bs.affix");if(this.$element.trigger(j),j.isDefaultPrevented())return;this.affixed=h,this.unpin="bottom"==h?this.getPinnedOffset():null,this.$element.removeClass(c.RESET).addClass(i).trigger(i.replace("affix","affixed")+".bs.affix")}"bottom"==h&&this.$element.offset({top:g-b-f})}};var d=a.fn.affix;a.fn.affix=b,a.fn.affix.Constructor=c,a.fn.affix.noConflict=function(){return a.fn.affix=d,this},a(window).on("load",function(){a('[data-spy="affix"]').each(function(){var c=a(this),d=c.data();d.offset=d.offset||{},null!=d.offsetBottom&&(d.offset.bottom=d.offsetBottom),null!=d.offsetTop&&(d.offset.top=d.offsetTop),b.call(c,d)})})}(jQuery);
\ No newline at end of file +if("undefined"==typeof jQuery)throw new Error("Bootstrap's JavaScript requires jQuery");+function(a){"use strict";var b=a.fn.jquery.split(" ")[0].split(".");if(b[0]<2&&b[1]<9||1==b[0]&&9==b[1]&&b[2]<1||b[0]>3)throw new Error("Bootstrap's JavaScript requires jQuery version 1.9.1 or higher, but lower than version 4")}(jQuery),+function(a){"use strict";function b(){var a=document.createElement("bootstrap"),b={WebkitTransition:"webkitTransitionEnd",MozTransition:"transitionend",OTransition:"oTransitionEnd otransitionend",transition:"transitionend"};for(var c in b)if(void 0!==a.style[c])return{end:b[c]};return!1}a.fn.emulateTransitionEnd=function(b){var c=!1,d=this;a(this).one("bsTransitionEnd",function(){c=!0});var e=function(){c||a(d).trigger(a.support.transition.end)};return setTimeout(e,b),this},a(function(){a.support.transition=b(),a.support.transition&&(a.event.special.bsTransitionEnd={bindType:a.support.transition.end,delegateType:a.support.transition.end,handle:function(b){if(a(b.target).is(this))return b.handleObj.handler.apply(this,arguments)}})})}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var c=a(this),e=c.data("bs.alert");e||c.data("bs.alert",e=new d(this)),"string"==typeof b&&e[b].call(c)})}var c='[data-dismiss="alert"]',d=function(b){a(b).on("click",c,this.close)};d.VERSION="3.3.7",d.TRANSITION_DURATION=150,d.prototype.close=function(b){function c(){g.detach().trigger("closed.bs.alert").remove()}var e=a(this),f=e.attr("data-target");f||(f=e.attr("href"),f=f&&f.replace(/.*(?=#[^\s]*$)/,""));var g=a("#"===f?[]:f);b&&b.preventDefault(),g.length||(g=e.closest(".alert")),g.trigger(b=a.Event("close.bs.alert")),b.isDefaultPrevented()||(g.removeClass("in"),a.support.transition&&g.hasClass("fade")?g.one("bsTransitionEnd",c).emulateTransitionEnd(d.TRANSITION_DURATION):c())};var e=a.fn.alert;a.fn.alert=b,a.fn.alert.Constructor=d,a.fn.alert.noConflict=function(){return a.fn.alert=e,this},a(document).on("click.bs.alert.data-api",c,d.prototype.close)}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.button"),f="object"==typeof b&&b;e||d.data("bs.button",e=new c(this,f)),"toggle"==b?e.toggle():b&&e.setState(b)})}var c=function(b,d){this.$element=a(b),this.options=a.extend({},c.DEFAULTS,d),this.isLoading=!1};c.VERSION="3.3.7",c.DEFAULTS={loadingText:"loading..."},c.prototype.setState=function(b){var c="disabled",d=this.$element,e=d.is("input")?"val":"html",f=d.data();b+="Text",null==f.resetText&&d.data("resetText",d[e]()),setTimeout(a.proxy(function(){d[e](null==f[b]?this.options[b]:f[b]),"loadingText"==b?(this.isLoading=!0,d.addClass(c).attr(c,c).prop(c,!0)):this.isLoading&&(this.isLoading=!1,d.removeClass(c).removeAttr(c).prop(c,!1))},this),0)},c.prototype.toggle=function(){var a=!0,b=this.$element.closest('[data-toggle="buttons"]');if(b.length){var c=this.$element.find("input");"radio"==c.prop("type")?(c.prop("checked")&&(a=!1),b.find(".active").removeClass("active"),this.$element.addClass("active")):"checkbox"==c.prop("type")&&(c.prop("checked")!==this.$element.hasClass("active")&&(a=!1),this.$element.toggleClass("active")),c.prop("checked",this.$element.hasClass("active")),a&&c.trigger("change")}else this.$element.attr("aria-pressed",!this.$element.hasClass("active")),this.$element.toggleClass("active")};var d=a.fn.button;a.fn.button=b,a.fn.button.Constructor=c,a.fn.button.noConflict=function(){return a.fn.button=d,this},a(document).on("click.bs.button.data-api",'[data-toggle^="button"]',function(c){var d=a(c.target).closest(".btn");b.call(d,"toggle"),a(c.target).is('input[type="radio"], input[type="checkbox"]')||(c.preventDefault(),d.is("input,button")?d.trigger("focus"):d.find("input:visible,button:visible").first().trigger("focus"))}).on("focus.bs.button.data-api blur.bs.button.data-api",'[data-toggle^="button"]',function(b){a(b.target).closest(".btn").toggleClass("focus",/^focus(in)?$/.test(b.type))})}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.carousel"),f=a.extend({},c.DEFAULTS,d.data(),"object"==typeof b&&b),g="string"==typeof b?b:f.slide;e||d.data("bs.carousel",e=new c(this,f)),"number"==typeof b?e.to(b):g?e[g]():f.interval&&e.pause().cycle()})}var c=function(b,c){this.$element=a(b),this.$indicators=this.$element.find(".carousel-indicators"),this.options=c,this.paused=null,this.sliding=null,this.interval=null,this.$active=null,this.$items=null,this.options.keyboard&&this.$element.on("keydown.bs.carousel",a.proxy(this.keydown,this)),"hover"==this.options.pause&&!("ontouchstart"in document.documentElement)&&this.$element.on("mouseenter.bs.carousel",a.proxy(this.pause,this)).on("mouseleave.bs.carousel",a.proxy(this.cycle,this))};c.VERSION="3.3.7",c.TRANSITION_DURATION=600,c.DEFAULTS={interval:5e3,pause:"hover",wrap:!0,keyboard:!0},c.prototype.keydown=function(a){if(!/input|textarea/i.test(a.target.tagName)){switch(a.which){case 37:this.prev();break;case 39:this.next();break;default:return}a.preventDefault()}},c.prototype.cycle=function(b){return b||(this.paused=!1),this.interval&&clearInterval(this.interval),this.options.interval&&!this.paused&&(this.interval=setInterval(a.proxy(this.next,this),this.options.interval)),this},c.prototype.getItemIndex=function(a){return this.$items=a.parent().children(".item"),this.$items.index(a||this.$active)},c.prototype.getItemForDirection=function(a,b){var c=this.getItemIndex(b),d="prev"==a&&0===c||"next"==a&&c==this.$items.length-1;if(d&&!this.options.wrap)return b;var e="prev"==a?-1:1,f=(c+e)%this.$items.length;return this.$items.eq(f)},c.prototype.to=function(a){var b=this,c=this.getItemIndex(this.$active=this.$element.find(".item.active"));if(!(a>this.$items.length-1||a<0))return this.sliding?this.$element.one("slid.bs.carousel",function(){b.to(a)}):c==a?this.pause().cycle():this.slide(a>c?"next":"prev",this.$items.eq(a))},c.prototype.pause=function(b){return b||(this.paused=!0),this.$element.find(".next, .prev").length&&a.support.transition&&(this.$element.trigger(a.support.transition.end),this.cycle(!0)),this.interval=clearInterval(this.interval),this},c.prototype.next=function(){if(!this.sliding)return this.slide("next")},c.prototype.prev=function(){if(!this.sliding)return this.slide("prev")},c.prototype.slide=function(b,d){var e=this.$element.find(".item.active"),f=d||this.getItemForDirection(b,e),g=this.interval,h="next"==b?"left":"right",i=this;if(f.hasClass("active"))return this.sliding=!1;var j=f[0],k=a.Event("slide.bs.carousel",{relatedTarget:j,direction:h});if(this.$element.trigger(k),!k.isDefaultPrevented()){if(this.sliding=!0,g&&this.pause(),this.$indicators.length){this.$indicators.find(".active").removeClass("active");var l=a(this.$indicators.children()[this.getItemIndex(f)]);l&&l.addClass("active")}var m=a.Event("slid.bs.carousel",{relatedTarget:j,direction:h});return a.support.transition&&this.$element.hasClass("slide")?(f.addClass(b),f[0].offsetWidth,e.addClass(h),f.addClass(h),e.one("bsTransitionEnd",function(){f.removeClass([b,h].join(" ")).addClass("active"),e.removeClass(["active",h].join(" ")),i.sliding=!1,setTimeout(function(){i.$element.trigger(m)},0)}).emulateTransitionEnd(c.TRANSITION_DURATION)):(e.removeClass("active"),f.addClass("active"),this.sliding=!1,this.$element.trigger(m)),g&&this.cycle(),this}};var d=a.fn.carousel;a.fn.carousel=b,a.fn.carousel.Constructor=c,a.fn.carousel.noConflict=function(){return a.fn.carousel=d,this};var e=function(c){var d,e=a(this),f=a(e.attr("data-target")||(d=e.attr("href"))&&d.replace(/.*(?=#[^\s]+$)/,""));if(f.hasClass("carousel")){var g=a.extend({},f.data(),e.data()),h=e.attr("data-slide-to");h&&(g.interval=!1),b.call(f,g),h&&f.data("bs.carousel").to(h),c.preventDefault()}};a(document).on("click.bs.carousel.data-api","[data-slide]",e).on("click.bs.carousel.data-api","[data-slide-to]",e),a(window).on("load",function(){a('[data-ride="carousel"]').each(function(){var c=a(this);b.call(c,c.data())})})}(jQuery),+function(a){"use strict";function b(b){var c,d=b.attr("data-target")||(c=b.attr("href"))&&c.replace(/.*(?=#[^\s]+$)/,"");return a(d)}function c(b){return this.each(function(){var c=a(this),e=c.data("bs.collapse"),f=a.extend({},d.DEFAULTS,c.data(),"object"==typeof b&&b);!e&&f.toggle&&/show|hide/.test(b)&&(f.toggle=!1),e||c.data("bs.collapse",e=new d(this,f)),"string"==typeof b&&e[b]()})}var d=function(b,c){this.$element=a(b),this.options=a.extend({},d.DEFAULTS,c),this.$trigger=a('[data-toggle="collapse"][href="#'+b.id+'"],[data-toggle="collapse"][data-target="#'+b.id+'"]'),this.transitioning=null,this.options.parent?this.$parent=this.getParent():this.addAriaAndCollapsedClass(this.$element,this.$trigger),this.options.toggle&&this.toggle()};d.VERSION="3.3.7",d.TRANSITION_DURATION=350,d.DEFAULTS={toggle:!0},d.prototype.dimension=function(){var a=this.$element.hasClass("width");return a?"width":"height"},d.prototype.show=function(){if(!this.transitioning&&!this.$element.hasClass("in")){var b,e=this.$parent&&this.$parent.children(".panel").children(".in, .collapsing");if(!(e&&e.length&&(b=e.data("bs.collapse"),b&&b.transitioning))){var f=a.Event("show.bs.collapse");if(this.$element.trigger(f),!f.isDefaultPrevented()){e&&e.length&&(c.call(e,"hide"),b||e.data("bs.collapse",null));var g=this.dimension();this.$element.removeClass("collapse").addClass("collapsing")[g](0).attr("aria-expanded",!0),this.$trigger.removeClass("collapsed").attr("aria-expanded",!0),this.transitioning=1;var h=function(){this.$element.removeClass("collapsing").addClass("collapse in")[g](""),this.transitioning=0,this.$element.trigger("shown.bs.collapse")};if(!a.support.transition)return h.call(this);var i=a.camelCase(["scroll",g].join("-"));this.$element.one("bsTransitionEnd",a.proxy(h,this)).emulateTransitionEnd(d.TRANSITION_DURATION)[g](this.$element[0][i])}}}},d.prototype.hide=function(){if(!this.transitioning&&this.$element.hasClass("in")){var b=a.Event("hide.bs.collapse");if(this.$element.trigger(b),!b.isDefaultPrevented()){var c=this.dimension();this.$element[c](this.$element[c]())[0].offsetHeight,this.$element.addClass("collapsing").removeClass("collapse in").attr("aria-expanded",!1),this.$trigger.addClass("collapsed").attr("aria-expanded",!1),this.transitioning=1;var e=function(){this.transitioning=0,this.$element.removeClass("collapsing").addClass("collapse").trigger("hidden.bs.collapse")};return a.support.transition?void this.$element[c](0).one("bsTransitionEnd",a.proxy(e,this)).emulateTransitionEnd(d.TRANSITION_DURATION):e.call(this)}}},d.prototype.toggle=function(){this[this.$element.hasClass("in")?"hide":"show"]()},d.prototype.getParent=function(){return a(this.options.parent).find('[data-toggle="collapse"][data-parent="'+this.options.parent+'"]').each(a.proxy(function(c,d){var e=a(d);this.addAriaAndCollapsedClass(b(e),e)},this)).end()},d.prototype.addAriaAndCollapsedClass=function(a,b){var c=a.hasClass("in");a.attr("aria-expanded",c),b.toggleClass("collapsed",!c).attr("aria-expanded",c)};var e=a.fn.collapse;a.fn.collapse=c,a.fn.collapse.Constructor=d,a.fn.collapse.noConflict=function(){return a.fn.collapse=e,this},a(document).on("click.bs.collapse.data-api",'[data-toggle="collapse"]',function(d){var e=a(this);e.attr("data-target")||d.preventDefault();var f=b(e),g=f.data("bs.collapse"),h=g?"toggle":e.data();c.call(f,h)})}(jQuery),+function(a){"use strict";function b(b){var c=b.attr("data-target");c||(c=b.attr("href"),c=c&&/#[A-Za-z]/.test(c)&&c.replace(/.*(?=#[^\s]*$)/,""));var d=c&&a(c);return d&&d.length?d:b.parent()}function c(c){c&&3===c.which||(a(e).remove(),a(f).each(function(){var d=a(this),e=b(d),f={relatedTarget:this};e.hasClass("open")&&(c&&"click"==c.type&&/input|textarea/i.test(c.target.tagName)&&a.contains(e[0],c.target)||(e.trigger(c=a.Event("hide.bs.dropdown",f)),c.isDefaultPrevented()||(d.attr("aria-expanded","false"),e.removeClass("open").trigger(a.Event("hidden.bs.dropdown",f)))))}))}function d(b){return this.each(function(){var c=a(this),d=c.data("bs.dropdown");d||c.data("bs.dropdown",d=new g(this)),"string"==typeof b&&d[b].call(c)})}var e=".dropdown-backdrop",f='[data-toggle="dropdown"]',g=function(b){a(b).on("click.bs.dropdown",this.toggle)};g.VERSION="3.3.7",g.prototype.toggle=function(d){var e=a(this);if(!e.is(".disabled, :disabled")){var f=b(e),g=f.hasClass("open");if(c(),!g){"ontouchstart"in document.documentElement&&!f.closest(".navbar-nav").length&&a(document.createElement("div")).addClass("dropdown-backdrop").insertAfter(a(this)).on("click",c);var h={relatedTarget:this};if(f.trigger(d=a.Event("show.bs.dropdown",h)),d.isDefaultPrevented())return;e.trigger("focus").attr("aria-expanded","true"),f.toggleClass("open").trigger(a.Event("shown.bs.dropdown",h))}return!1}},g.prototype.keydown=function(c){if(/(38|40|27|32)/.test(c.which)&&!/input|textarea/i.test(c.target.tagName)){var d=a(this);if(c.preventDefault(),c.stopPropagation(),!d.is(".disabled, :disabled")){var e=b(d),g=e.hasClass("open");if(!g&&27!=c.which||g&&27==c.which)return 27==c.which&&e.find(f).trigger("focus"),d.trigger("click");var h=" li:not(.disabled):visible a",i=e.find(".dropdown-menu"+h);if(i.length){var j=i.index(c.target);38==c.which&&j>0&&j--,40==c.which&&j<i.length-1&&j++,~j||(j=0),i.eq(j).trigger("focus")}}}};var h=a.fn.dropdown;a.fn.dropdown=d,a.fn.dropdown.Constructor=g,a.fn.dropdown.noConflict=function(){return a.fn.dropdown=h,this},a(document).on("click.bs.dropdown.data-api",c).on("click.bs.dropdown.data-api",".dropdown form",function(a){a.stopPropagation()}).on("click.bs.dropdown.data-api",f,g.prototype.toggle).on("keydown.bs.dropdown.data-api",f,g.prototype.keydown).on("keydown.bs.dropdown.data-api",".dropdown-menu",g.prototype.keydown)}(jQuery),+function(a){"use strict";function b(b,d){return this.each(function(){var e=a(this),f=e.data("bs.modal"),g=a.extend({},c.DEFAULTS,e.data(),"object"==typeof b&&b);f||e.data("bs.modal",f=new c(this,g)),"string"==typeof b?f[b](d):g.show&&f.show(d)})}var c=function(b,c){this.options=c,this.$body=a(document.body),this.$element=a(b),this.$dialog=this.$element.find(".modal-dialog"),this.$backdrop=null,this.isShown=null,this.originalBodyPad=null,this.scrollbarWidth=0,this.ignoreBackdropClick=!1,this.options.remote&&this.$element.find(".modal-content").load(this.options.remote,a.proxy(function(){this.$element.trigger("loaded.bs.modal")},this))};c.VERSION="3.3.7",c.TRANSITION_DURATION=300,c.BACKDROP_TRANSITION_DURATION=150,c.DEFAULTS={backdrop:!0,keyboard:!0,show:!0},c.prototype.toggle=function(a){return this.isShown?this.hide():this.show(a)},c.prototype.show=function(b){var d=this,e=a.Event("show.bs.modal",{relatedTarget:b});this.$element.trigger(e),this.isShown||e.isDefaultPrevented()||(this.isShown=!0,this.checkScrollbar(),this.setScrollbar(),this.$body.addClass("modal-open"),this.escape(),this.resize(),this.$element.on("click.dismiss.bs.modal",'[data-dismiss="modal"]',a.proxy(this.hide,this)),this.$dialog.on("mousedown.dismiss.bs.modal",function(){d.$element.one("mouseup.dismiss.bs.modal",function(b){a(b.target).is(d.$element)&&(d.ignoreBackdropClick=!0)})}),this.backdrop(function(){var e=a.support.transition&&d.$element.hasClass("fade");d.$element.parent().length||d.$element.appendTo(d.$body),d.$element.show().scrollTop(0),d.adjustDialog(),e&&d.$element[0].offsetWidth,d.$element.addClass("in"),d.enforceFocus();var f=a.Event("shown.bs.modal",{relatedTarget:b});e?d.$dialog.one("bsTransitionEnd",function(){d.$element.trigger("focus").trigger(f)}).emulateTransitionEnd(c.TRANSITION_DURATION):d.$element.trigger("focus").trigger(f)}))},c.prototype.hide=function(b){b&&b.preventDefault(),b=a.Event("hide.bs.modal"),this.$element.trigger(b),this.isShown&&!b.isDefaultPrevented()&&(this.isShown=!1,this.escape(),this.resize(),a(document).off("focusin.bs.modal"),this.$element.removeClass("in").off("click.dismiss.bs.modal").off("mouseup.dismiss.bs.modal"),this.$dialog.off("mousedown.dismiss.bs.modal"),a.support.transition&&this.$element.hasClass("fade")?this.$element.one("bsTransitionEnd",a.proxy(this.hideModal,this)).emulateTransitionEnd(c.TRANSITION_DURATION):this.hideModal())},c.prototype.enforceFocus=function(){a(document).off("focusin.bs.modal").on("focusin.bs.modal",a.proxy(function(a){document===a.target||this.$element[0]===a.target||this.$element.has(a.target).length||this.$element.trigger("focus")},this))},c.prototype.escape=function(){this.isShown&&this.options.keyboard?this.$element.on("keydown.dismiss.bs.modal",a.proxy(function(a){27==a.which&&this.hide()},this)):this.isShown||this.$element.off("keydown.dismiss.bs.modal")},c.prototype.resize=function(){this.isShown?a(window).on("resize.bs.modal",a.proxy(this.handleUpdate,this)):a(window).off("resize.bs.modal")},c.prototype.hideModal=function(){var a=this;this.$element.hide(),this.backdrop(function(){a.$body.removeClass("modal-open"),a.resetAdjustments(),a.resetScrollbar(),a.$element.trigger("hidden.bs.modal")})},c.prototype.removeBackdrop=function(){this.$backdrop&&this.$backdrop.remove(),this.$backdrop=null},c.prototype.backdrop=function(b){var d=this,e=this.$element.hasClass("fade")?"fade":"";if(this.isShown&&this.options.backdrop){var f=a.support.transition&&e;if(this.$backdrop=a(document.createElement("div")).addClass("modal-backdrop "+e).appendTo(this.$body),this.$element.on("click.dismiss.bs.modal",a.proxy(function(a){return this.ignoreBackdropClick?void(this.ignoreBackdropClick=!1):void(a.target===a.currentTarget&&("static"==this.options.backdrop?this.$element[0].focus():this.hide()))},this)),f&&this.$backdrop[0].offsetWidth,this.$backdrop.addClass("in"),!b)return;f?this.$backdrop.one("bsTransitionEnd",b).emulateTransitionEnd(c.BACKDROP_TRANSITION_DURATION):b()}else if(!this.isShown&&this.$backdrop){this.$backdrop.removeClass("in");var g=function(){d.removeBackdrop(),b&&b()};a.support.transition&&this.$element.hasClass("fade")?this.$backdrop.one("bsTransitionEnd",g).emulateTransitionEnd(c.BACKDROP_TRANSITION_DURATION):g()}else b&&b()},c.prototype.handleUpdate=function(){this.adjustDialog()},c.prototype.adjustDialog=function(){var a=this.$element[0].scrollHeight>document.documentElement.clientHeight;this.$element.css({paddingLeft:!this.bodyIsOverflowing&&a?this.scrollbarWidth:"",paddingRight:this.bodyIsOverflowing&&!a?this.scrollbarWidth:""})},c.prototype.resetAdjustments=function(){this.$element.css({paddingLeft:"",paddingRight:""})},c.prototype.checkScrollbar=function(){var a=window.innerWidth;if(!a){var b=document.documentElement.getBoundingClientRect();a=b.right-Math.abs(b.left)}this.bodyIsOverflowing=document.body.clientWidth<a,this.scrollbarWidth=this.measureScrollbar()},c.prototype.setScrollbar=function(){var a=parseInt(this.$body.css("padding-right")||0,10);this.originalBodyPad=document.body.style.paddingRight||"",this.bodyIsOverflowing&&this.$body.css("padding-right",a+this.scrollbarWidth)},c.prototype.resetScrollbar=function(){this.$body.css("padding-right",this.originalBodyPad)},c.prototype.measureScrollbar=function(){var a=document.createElement("div");a.className="modal-scrollbar-measure",this.$body.append(a);var b=a.offsetWidth-a.clientWidth;return this.$body[0].removeChild(a),b};var d=a.fn.modal;a.fn.modal=b,a.fn.modal.Constructor=c,a.fn.modal.noConflict=function(){return a.fn.modal=d,this},a(document).on("click.bs.modal.data-api",'[data-toggle="modal"]',function(c){var d=a(this),e=d.attr("href"),f=a(d.attr("data-target")||e&&e.replace(/.*(?=#[^\s]+$)/,"")),g=f.data("bs.modal")?"toggle":a.extend({remote:!/#/.test(e)&&e},f.data(),d.data());d.is("a")&&c.preventDefault(),f.one("show.bs.modal",function(a){a.isDefaultPrevented()||f.one("hidden.bs.modal",function(){d.is(":visible")&&d.trigger("focus")})}),b.call(f,g,this)})}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.tooltip"),f="object"==typeof b&&b;!e&&/destroy|hide/.test(b)||(e||d.data("bs.tooltip",e=new c(this,f)),"string"==typeof b&&e[b]())})}var c=function(a,b){this.type=null,this.options=null,this.enabled=null,this.timeout=null,this.hoverState=null,this.$element=null,this.inState=null,this.init("tooltip",a,b)};c.VERSION="3.3.7",c.TRANSITION_DURATION=150,c.DEFAULTS={animation:!0,placement:"top",selector:!1,template:'<div class="tooltip" role="tooltip"><div class="tooltip-arrow"></div><div class="tooltip-inner"></div></div>',trigger:"hover focus",title:"",delay:0,html:!1,container:!1,viewport:{selector:"body",padding:0}},c.prototype.init=function(b,c,d){if(this.enabled=!0,this.type=b,this.$element=a(c),this.options=this.getOptions(d),this.$viewport=this.options.viewport&&a(a.isFunction(this.options.viewport)?this.options.viewport.call(this,this.$element):this.options.viewport.selector||this.options.viewport),this.inState={click:!1,hover:!1,focus:!1},this.$element[0]instanceof document.constructor&&!this.options.selector)throw new Error("`selector` option must be specified when initializing "+this.type+" on the window.document object!");for(var e=this.options.trigger.split(" "),f=e.length;f--;){var g=e[f];if("click"==g)this.$element.on("click."+this.type,this.options.selector,a.proxy(this.toggle,this));else if("manual"!=g){var h="hover"==g?"mouseenter":"focusin",i="hover"==g?"mouseleave":"focusout";this.$element.on(h+"."+this.type,this.options.selector,a.proxy(this.enter,this)),this.$element.on(i+"."+this.type,this.options.selector,a.proxy(this.leave,this))}}this.options.selector?this._options=a.extend({},this.options,{trigger:"manual",selector:""}):this.fixTitle()},c.prototype.getDefaults=function(){return c.DEFAULTS},c.prototype.getOptions=function(b){return b=a.extend({},this.getDefaults(),this.$element.data(),b),b.delay&&"number"==typeof b.delay&&(b.delay={show:b.delay,hide:b.delay}),b},c.prototype.getDelegateOptions=function(){var b={},c=this.getDefaults();return this._options&&a.each(this._options,function(a,d){c[a]!=d&&(b[a]=d)}),b},c.prototype.enter=function(b){var c=b instanceof this.constructor?b:a(b.currentTarget).data("bs."+this.type);return c||(c=new this.constructor(b.currentTarget,this.getDelegateOptions()),a(b.currentTarget).data("bs."+this.type,c)),b instanceof a.Event&&(c.inState["focusin"==b.type?"focus":"hover"]=!0),c.tip().hasClass("in")||"in"==c.hoverState?void(c.hoverState="in"):(clearTimeout(c.timeout),c.hoverState="in",c.options.delay&&c.options.delay.show?void(c.timeout=setTimeout(function(){"in"==c.hoverState&&c.show()},c.options.delay.show)):c.show())},c.prototype.isInStateTrue=function(){for(var a in this.inState)if(this.inState[a])return!0;return!1},c.prototype.leave=function(b){var c=b instanceof this.constructor?b:a(b.currentTarget).data("bs."+this.type);if(c||(c=new this.constructor(b.currentTarget,this.getDelegateOptions()),a(b.currentTarget).data("bs."+this.type,c)),b instanceof a.Event&&(c.inState["focusout"==b.type?"focus":"hover"]=!1),!c.isInStateTrue())return clearTimeout(c.timeout),c.hoverState="out",c.options.delay&&c.options.delay.hide?void(c.timeout=setTimeout(function(){"out"==c.hoverState&&c.hide()},c.options.delay.hide)):c.hide()},c.prototype.show=function(){var b=a.Event("show.bs."+this.type);if(this.hasContent()&&this.enabled){this.$element.trigger(b);var d=a.contains(this.$element[0].ownerDocument.documentElement,this.$element[0]);if(b.isDefaultPrevented()||!d)return;var e=this,f=this.tip(),g=this.getUID(this.type);this.setContent(),f.attr("id",g),this.$element.attr("aria-describedby",g),this.options.animation&&f.addClass("fade");var h="function"==typeof this.options.placement?this.options.placement.call(this,f[0],this.$element[0]):this.options.placement,i=/\s?auto?\s?/i,j=i.test(h);j&&(h=h.replace(i,"")||"top"),f.detach().css({top:0,left:0,display:"block"}).addClass(h).data("bs."+this.type,this),this.options.container?f.appendTo(this.options.container):f.insertAfter(this.$element),this.$element.trigger("inserted.bs."+this.type);var k=this.getPosition(),l=f[0].offsetWidth,m=f[0].offsetHeight;if(j){var n=h,o=this.getPosition(this.$viewport);h="bottom"==h&&k.bottom+m>o.bottom?"top":"top"==h&&k.top-m<o.top?"bottom":"right"==h&&k.right+l>o.width?"left":"left"==h&&k.left-l<o.left?"right":h,f.removeClass(n).addClass(h)}var p=this.getCalculatedOffset(h,k,l,m);this.applyPlacement(p,h);var q=function(){var a=e.hoverState;e.$element.trigger("shown.bs."+e.type),e.hoverState=null,"out"==a&&e.leave(e)};a.support.transition&&this.$tip.hasClass("fade")?f.one("bsTransitionEnd",q).emulateTransitionEnd(c.TRANSITION_DURATION):q()}},c.prototype.applyPlacement=function(b,c){var d=this.tip(),e=d[0].offsetWidth,f=d[0].offsetHeight,g=parseInt(d.css("margin-top"),10),h=parseInt(d.css("margin-left"),10);isNaN(g)&&(g=0),isNaN(h)&&(h=0),b.top+=g,b.left+=h,a.offset.setOffset(d[0],a.extend({using:function(a){d.css({top:Math.round(a.top),left:Math.round(a.left)})}},b),0),d.addClass("in");var i=d[0].offsetWidth,j=d[0].offsetHeight;"top"==c&&j!=f&&(b.top=b.top+f-j);var k=this.getViewportAdjustedDelta(c,b,i,j);k.left?b.left+=k.left:b.top+=k.top;var l=/top|bottom/.test(c),m=l?2*k.left-e+i:2*k.top-f+j,n=l?"offsetWidth":"offsetHeight";d.offset(b),this.replaceArrow(m,d[0][n],l)},c.prototype.replaceArrow=function(a,b,c){this.arrow().css(c?"left":"top",50*(1-a/b)+"%").css(c?"top":"left","")},c.prototype.setContent=function(){var a=this.tip(),b=this.getTitle();a.find(".tooltip-inner")[this.options.html?"html":"text"](b),a.removeClass("fade in top bottom left right")},c.prototype.hide=function(b){function d(){"in"!=e.hoverState&&f.detach(),e.$element&&e.$element.removeAttr("aria-describedby").trigger("hidden.bs."+e.type),b&&b()}var e=this,f=a(this.$tip),g=a.Event("hide.bs."+this.type);if(this.$element.trigger(g),!g.isDefaultPrevented())return f.removeClass("in"),a.support.transition&&f.hasClass("fade")?f.one("bsTransitionEnd",d).emulateTransitionEnd(c.TRANSITION_DURATION):d(),this.hoverState=null,this},c.prototype.fixTitle=function(){var a=this.$element;(a.attr("title")||"string"!=typeof a.attr("data-original-title"))&&a.attr("data-original-title",a.attr("title")||"").attr("title","")},c.prototype.hasContent=function(){return this.getTitle()},c.prototype.getPosition=function(b){b=b||this.$element;var c=b[0],d="BODY"==c.tagName,e=c.getBoundingClientRect();null==e.width&&(e=a.extend({},e,{width:e.right-e.left,height:e.bottom-e.top}));var f=window.SVGElement&&c instanceof window.SVGElement,g=d?{top:0,left:0}:f?null:b.offset(),h={scroll:d?document.documentElement.scrollTop||document.body.scrollTop:b.scrollTop()},i=d?{width:a(window).width(),height:a(window).height()}:null;return a.extend({},e,h,i,g)},c.prototype.getCalculatedOffset=function(a,b,c,d){return"bottom"==a?{top:b.top+b.height,left:b.left+b.width/2-c/2}:"top"==a?{top:b.top-d,left:b.left+b.width/2-c/2}:"left"==a?{top:b.top+b.height/2-d/2,left:b.left-c}:{top:b.top+b.height/2-d/2,left:b.left+b.width}},c.prototype.getViewportAdjustedDelta=function(a,b,c,d){var e={top:0,left:0};if(!this.$viewport)return e;var f=this.options.viewport&&this.options.viewport.padding||0,g=this.getPosition(this.$viewport);if(/right|left/.test(a)){var h=b.top-f-g.scroll,i=b.top+f-g.scroll+d;h<g.top?e.top=g.top-h:i>g.top+g.height&&(e.top=g.top+g.height-i)}else{var j=b.left-f,k=b.left+f+c;j<g.left?e.left=g.left-j:k>g.right&&(e.left=g.left+g.width-k)}return e},c.prototype.getTitle=function(){var a,b=this.$element,c=this.options;return a=b.attr("data-original-title")||("function"==typeof c.title?c.title.call(b[0]):c.title)},c.prototype.getUID=function(a){do a+=~~(1e6*Math.random());while(document.getElementById(a));return a},c.prototype.tip=function(){if(!this.$tip&&(this.$tip=a(this.options.template),1!=this.$tip.length))throw new Error(this.type+" `template` option must consist of exactly 1 top-level element!");return this.$tip},c.prototype.arrow=function(){return this.$arrow=this.$arrow||this.tip().find(".tooltip-arrow")},c.prototype.enable=function(){this.enabled=!0},c.prototype.disable=function(){this.enabled=!1},c.prototype.toggleEnabled=function(){this.enabled=!this.enabled},c.prototype.toggle=function(b){var c=this;b&&(c=a(b.currentTarget).data("bs."+this.type),c||(c=new this.constructor(b.currentTarget,this.getDelegateOptions()),a(b.currentTarget).data("bs."+this.type,c))),b?(c.inState.click=!c.inState.click,c.isInStateTrue()?c.enter(c):c.leave(c)):c.tip().hasClass("in")?c.leave(c):c.enter(c)},c.prototype.destroy=function(){var a=this;clearTimeout(this.timeout),this.hide(function(){a.$element.off("."+a.type).removeData("bs."+a.type),a.$tip&&a.$tip.detach(),a.$tip=null,a.$arrow=null,a.$viewport=null,a.$element=null})};var d=a.fn.tooltip;a.fn.tooltip=b,a.fn.tooltip.Constructor=c,a.fn.tooltip.noConflict=function(){return a.fn.tooltip=d,this}}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.popover"),f="object"==typeof b&&b;!e&&/destroy|hide/.test(b)||(e||d.data("bs.popover",e=new c(this,f)),"string"==typeof b&&e[b]())})}var c=function(a,b){this.init("popover",a,b)};if(!a.fn.tooltip)throw new Error("Popover requires tooltip.js");c.VERSION="3.3.7",c.DEFAULTS=a.extend({},a.fn.tooltip.Constructor.DEFAULTS,{placement:"right",trigger:"click",content:"",template:'<div class="popover" role="tooltip"><div class="arrow"></div><h3 class="popover-title"></h3><div class="popover-content"></div></div>'}),c.prototype=a.extend({},a.fn.tooltip.Constructor.prototype),c.prototype.constructor=c,c.prototype.getDefaults=function(){return c.DEFAULTS},c.prototype.setContent=function(){var a=this.tip(),b=this.getTitle(),c=this.getContent();a.find(".popover-title")[this.options.html?"html":"text"](b),a.find(".popover-content").children().detach().end()[this.options.html?"string"==typeof c?"html":"append":"text"](c),a.removeClass("fade top bottom left right in"),a.find(".popover-title").html()||a.find(".popover-title").hide()},c.prototype.hasContent=function(){return this.getTitle()||this.getContent()},c.prototype.getContent=function(){var a=this.$element,b=this.options;return a.attr("data-content")||("function"==typeof b.content?b.content.call(a[0]):b.content)},c.prototype.arrow=function(){return this.$arrow=this.$arrow||this.tip().find(".arrow")};var d=a.fn.popover;a.fn.popover=b,a.fn.popover.Constructor=c,a.fn.popover.noConflict=function(){return a.fn.popover=d,this}}(jQuery),+function(a){"use strict";function b(c,d){this.$body=a(document.body),this.$scrollElement=a(a(c).is(document.body)?window:c),this.options=a.extend({},b.DEFAULTS,d),this.selector=(this.options.target||"")+" .nav li > a",this.offsets=[],this.targets=[],this.activeTarget=null,this.scrollHeight=0,this.$scrollElement.on("scroll.bs.scrollspy",a.proxy(this.process,this)),this.refresh(),this.process()}function c(c){return this.each(function(){var d=a(this),e=d.data("bs.scrollspy"),f="object"==typeof c&&c;e||d.data("bs.scrollspy",e=new b(this,f)),"string"==typeof c&&e[c]()})}b.VERSION="3.3.7",b.DEFAULTS={offset:10},b.prototype.getScrollHeight=function(){return this.$scrollElement[0].scrollHeight||Math.max(this.$body[0].scrollHeight,document.documentElement.scrollHeight)},b.prototype.refresh=function(){var b=this,c="offset",d=0;this.offsets=[],this.targets=[],this.scrollHeight=this.getScrollHeight(),a.isWindow(this.$scrollElement[0])||(c="position",d=this.$scrollElement.scrollTop()),this.$body.find(this.selector).map(function(){var b=a(this),e=b.data("target")||b.attr("href"),f=/^#./.test(e)&&a(e);return f&&f.length&&f.is(":visible")&&[[f[c]().top+d,e]]||null}).sort(function(a,b){return a[0]-b[0]}).each(function(){b.offsets.push(this[0]),b.targets.push(this[1])})},b.prototype.process=function(){var a,b=this.$scrollElement.scrollTop()+this.options.offset,c=this.getScrollHeight(),d=this.options.offset+c-this.$scrollElement.height(),e=this.offsets,f=this.targets,g=this.activeTarget;if(this.scrollHeight!=c&&this.refresh(),b>=d)return g!=(a=f[f.length-1])&&this.activate(a);if(g&&b<e[0])return this.activeTarget=null,this.clear();for(a=e.length;a--;)g!=f[a]&&b>=e[a]&&(void 0===e[a+1]||b<e[a+1])&&this.activate(f[a])},b.prototype.activate=function(b){ +this.activeTarget=b,this.clear();var c=this.selector+'[data-target="'+b+'"],'+this.selector+'[href="'+b+'"]',d=a(c).parents("li").addClass("active");d.parent(".dropdown-menu").length&&(d=d.closest("li.dropdown").addClass("active")),d.trigger("activate.bs.scrollspy")},b.prototype.clear=function(){a(this.selector).parentsUntil(this.options.target,".active").removeClass("active")};var d=a.fn.scrollspy;a.fn.scrollspy=c,a.fn.scrollspy.Constructor=b,a.fn.scrollspy.noConflict=function(){return a.fn.scrollspy=d,this},a(window).on("load.bs.scrollspy.data-api",function(){a('[data-spy="scroll"]').each(function(){var b=a(this);c.call(b,b.data())})})}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.tab");e||d.data("bs.tab",e=new c(this)),"string"==typeof b&&e[b]()})}var c=function(b){this.element=a(b)};c.VERSION="3.3.7",c.TRANSITION_DURATION=150,c.prototype.show=function(){var b=this.element,c=b.closest("ul:not(.dropdown-menu)"),d=b.data("target");if(d||(d=b.attr("href"),d=d&&d.replace(/.*(?=#[^\s]*$)/,"")),!b.parent("li").hasClass("active")){var e=c.find(".active:last a"),f=a.Event("hide.bs.tab",{relatedTarget:b[0]}),g=a.Event("show.bs.tab",{relatedTarget:e[0]});if(e.trigger(f),b.trigger(g),!g.isDefaultPrevented()&&!f.isDefaultPrevented()){var h=a(d);this.activate(b.closest("li"),c),this.activate(h,h.parent(),function(){e.trigger({type:"hidden.bs.tab",relatedTarget:b[0]}),b.trigger({type:"shown.bs.tab",relatedTarget:e[0]})})}}},c.prototype.activate=function(b,d,e){function f(){g.removeClass("active").find("> .dropdown-menu > .active").removeClass("active").end().find('[data-toggle="tab"]').attr("aria-expanded",!1),b.addClass("active").find('[data-toggle="tab"]').attr("aria-expanded",!0),h?(b[0].offsetWidth,b.addClass("in")):b.removeClass("fade"),b.parent(".dropdown-menu").length&&b.closest("li.dropdown").addClass("active").end().find('[data-toggle="tab"]').attr("aria-expanded",!0),e&&e()}var g=d.find("> .active"),h=e&&a.support.transition&&(g.length&&g.hasClass("fade")||!!d.find("> .fade").length);g.length&&h?g.one("bsTransitionEnd",f).emulateTransitionEnd(c.TRANSITION_DURATION):f(),g.removeClass("in")};var d=a.fn.tab;a.fn.tab=b,a.fn.tab.Constructor=c,a.fn.tab.noConflict=function(){return a.fn.tab=d,this};var e=function(c){c.preventDefault(),b.call(a(this),"show")};a(document).on("click.bs.tab.data-api",'[data-toggle="tab"]',e).on("click.bs.tab.data-api",'[data-toggle="pill"]',e)}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.affix"),f="object"==typeof b&&b;e||d.data("bs.affix",e=new c(this,f)),"string"==typeof b&&e[b]()})}var c=function(b,d){this.options=a.extend({},c.DEFAULTS,d),this.$target=a(this.options.target).on("scroll.bs.affix.data-api",a.proxy(this.checkPosition,this)).on("click.bs.affix.data-api",a.proxy(this.checkPositionWithEventLoop,this)),this.$element=a(b),this.affixed=null,this.unpin=null,this.pinnedOffset=null,this.checkPosition()};c.VERSION="3.3.7",c.RESET="affix affix-top affix-bottom",c.DEFAULTS={offset:0,target:window},c.prototype.getState=function(a,b,c,d){var e=this.$target.scrollTop(),f=this.$element.offset(),g=this.$target.height();if(null!=c&&"top"==this.affixed)return e<c&&"top";if("bottom"==this.affixed)return null!=c?!(e+this.unpin<=f.top)&&"bottom":!(e+g<=a-d)&&"bottom";var h=null==this.affixed,i=h?e:f.top,j=h?g:b;return null!=c&&e<=c?"top":null!=d&&i+j>=a-d&&"bottom"},c.prototype.getPinnedOffset=function(){if(this.pinnedOffset)return this.pinnedOffset;this.$element.removeClass(c.RESET).addClass("affix");var a=this.$target.scrollTop(),b=this.$element.offset();return this.pinnedOffset=b.top-a},c.prototype.checkPositionWithEventLoop=function(){setTimeout(a.proxy(this.checkPosition,this),1)},c.prototype.checkPosition=function(){if(this.$element.is(":visible")){var b=this.$element.height(),d=this.options.offset,e=d.top,f=d.bottom,g=Math.max(a(document).height(),a(document.body).height());"object"!=typeof d&&(f=e=d),"function"==typeof e&&(e=d.top(this.$element)),"function"==typeof f&&(f=d.bottom(this.$element));var h=this.getState(g,b,e,f);if(this.affixed!=h){null!=this.unpin&&this.$element.css("top","");var i="affix"+(h?"-"+h:""),j=a.Event(i+".bs.affix");if(this.$element.trigger(j),j.isDefaultPrevented())return;this.affixed=h,this.unpin="bottom"==h?this.getPinnedOffset():null,this.$element.removeClass(c.RESET).addClass(i).trigger(i.replace("affix","affixed")+".bs.affix")}"bottom"==h&&this.$element.offset({top:g-b-f})}};var d=a.fn.affix;a.fn.affix=b,a.fn.affix.Constructor=c,a.fn.affix.noConflict=function(){return a.fn.affix=d,this},a(window).on("load",function(){a('[data-spy="affix"]').each(function(){var c=a(this),d=c.data();d.offset=d.offset||{},null!=d.offsetBottom&&(d.offset.bottom=d.offsetBottom),null!=d.offsetTop&&(d.offset.top=d.offsetTop),b.call(c,d)})})}(jQuery);
\ No newline at end of file diff --git a/lualib/lua_antivirus.lua b/lualib/lua_antivirus.lua new file mode 100644 index 000000000..286ef64d0 --- /dev/null +++ b/lualib/lua_antivirus.lua @@ -0,0 +1,986 @@ +--[[ +Copyright (c) 2018, Vsevolod Stakhov <vsevolod@highsecure.ru> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +--[[[ +-- @module lua_antivirus +-- This module contains antivirus access functions +--]] + +local lua_util = require "lua_util" +local tcp = require "rspamd_tcp" +local upstream_list = require "rspamd_upstream_list" +local rspamd_util = require "rspamd_util" +local lua_redis = require "lua_redis" +local rspamd_logger = require "rspamd_logger" + +local N = "antivirus" + +local default_message = '${SCANNER}: virus found: "${VIRUS}"' + +local function match_patterns(default_sym, found, patterns) + if type(patterns) ~= 'table' then return default_sym end + if not patterns[1] then + for sym, pat in pairs(patterns) do + if pat:match(found) then + return sym + end + end + return default_sym + else + for _, p in ipairs(patterns) do + for sym, pat in pairs(p) do + if pat:match(found) then + return sym + end + end + end + return default_sym + end +end + +local function yield_result(task, rule, vname) + local all_whitelisted = true + if type(vname) == 'string' then + local symname = match_patterns(rule['symbol'], vname, rule['patterns']) + if rule['whitelist'] and rule['whitelist']:get_key(vname) then + rspamd_logger.infox(task, '%s: "%s" is in whitelist', rule['type'], vname) + return + end + task:insert_result(symname, 1.0, vname) + rspamd_logger.infox(task, '%s: virus found: "%s"', rule['type'], vname) + elseif type(vname) == 'table' then + for _, vn in ipairs(vname) do + local symname = match_patterns(rule['symbol'], vn, rule['patterns']) + if rule['whitelist'] and rule['whitelist']:get_key(vn) then + rspamd_logger.infox(task, '%s: "%s" is in whitelist', rule['type'], vn) + else + all_whitelisted = false + task:insert_result(symname, 1.0, vn) + rspamd_logger.infox(task, '%s: virus found: "%s"', rule['type'], vn) + end + end + end + if rule['action'] then + if type(vname) == 'table' then + if all_whitelisted then return end + vname = table.concat(vname, '; ') + end + task:set_pre_result(rule['action'], + lua_util.template(rule.message or 'Rejected', { + SCANNER = rule['type'], + VIRUS = vname, + }), N) + end +end + +local function clamav_config(opts) + local clamav_conf = { + scan_mime_parts = true; + scan_text_mime = false; + scan_image_mime = false; + default_port = 3310, + log_clean = false, + timeout = 15.0, -- FIXME: this will break task_timeout! + retransmits = 2, + cache_expire = 3600, -- expire redis in one hour + message = default_message, + } + + for k,v in pairs(opts) do + clamav_conf[k] = v + end + + if not clamav_conf.prefix then + clamav_conf.prefix = 'rs_cl' + end + + if not clamav_conf['servers'] then + rspamd_logger.errx(rspamd_config, 'no servers defined') + + return nil + end + + clamav_conf['upstreams'] = upstream_list.create(rspamd_config, + clamav_conf['servers'], + clamav_conf.default_port) + + if clamav_conf['upstreams'] then + return clamav_conf + end + + rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', + clamav_conf['servers']) + return nil +end + +local function fprot_config(opts) + local fprot_conf = { + scan_mime_parts = true; + scan_text_mime = false; + scan_image_mime = false; + default_port = 10200, + timeout = 15.0, -- FIXME: this will break task_timeout! + log_clean = false, + retransmits = 2, + cache_expire = 3600, -- expire redis in one hour + message = default_message, + } + + for k,v in pairs(opts) do + fprot_conf[k] = v + end + + if not fprot_conf.prefix then + fprot_conf.prefix = 'rs_fp' + end + + if not fprot_conf['servers'] then + rspamd_logger.errx(rspamd_config, 'no servers defined') + + return nil + end + + fprot_conf['upstreams'] = upstream_list.create(rspamd_config, + fprot_conf['servers'], + fprot_conf.default_port) + + if fprot_conf['upstreams'] then + return fprot_conf + end + + rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', + fprot_conf['servers']) + return nil +end + +local function sophos_config(opts) + local sophos_conf = { + scan_mime_parts = true; + scan_text_mime = false; + scan_image_mime = false; + default_port = 4010, + timeout = 15.0, + log_clean = false, + retransmits = 2, + cache_expire = 3600, -- expire redis in one hour + message = default_message, + savdi_report_encrypted = false, + savdi_report_oversize = false, + } + + for k,v in pairs(opts) do + sophos_conf[k] = v + end + + if not sophos_conf.prefix then + sophos_conf.prefix = 'rs_sp' + end + + if not sophos_conf['servers'] then + rspamd_logger.errx(rspamd_config, 'no servers defined') + + return nil + end + + sophos_conf['upstreams'] = upstream_list.create(rspamd_config, + sophos_conf['servers'], + sophos_conf.default_port) + + if sophos_conf['upstreams'] then + return sophos_conf + end + + rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', + sophos_conf['servers']) + return nil +end + +local function savapi_config(opts) + local savapi_conf = { + scan_mime_parts = true; + scan_text_mime = false; + scan_image_mime = false; + default_port = 4444, -- note: You must set ListenAddress in savapi.conf + product_id = 0, + log_clean = false, + timeout = 15.0, -- FIXME: this will break task_timeout! + retransmits = 1, -- FIXME: useless, for local files + cache_expire = 3600, -- expire redis in one hour + message = default_message, + tmpdir = '/tmp', + } + + for k,v in pairs(opts) do + savapi_conf[k] = v + end + + if not savapi_conf.prefix then + savapi_conf.prefix = 'rs_ap' + end + + if not savapi_conf['servers'] then + rspamd_logger.errx(rspamd_config, 'no servers defined') + + return nil + end + + savapi_conf['upstreams'] = upstream_list.create(rspamd_config, + savapi_conf['servers'], + savapi_conf.default_port) + + if savapi_conf['upstreams'] then + return savapi_conf + end + + rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', + savapi_conf['servers']) + return nil +end + +local function kaspersky_config(opts) + local kaspersky_conf = { + scan_mime_parts = true; + scan_text_mime = false; + scan_image_mime = false; + product_id = 0, + log_clean = false, + timeout = 5.0, + retransmits = 1, -- use local files, retransmits are useless + cache_expire = 3600, -- expire redis in one hour + message = default_message, + tmpdir = '/tmp', + prefix = 'rs_ak', + } + + kaspersky_conf = lua_util.override_defaults(kaspersky_conf, opts) + + if not kaspersky_conf['servers'] then + rspamd_logger.errx(rspamd_config, 'no servers defined') + + return nil + end + + kaspersky_conf['upstreams'] = upstream_list.create(rspamd_config, + kaspersky_conf['servers'], 0) + + if kaspersky_conf['upstreams'] then + return kaspersky_conf + end + + rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', + kaspersky_conf['servers']) + return nil +end + +local function message_not_too_large(task, content, rule) + local max_size = tonumber(rule['max_size']) + if not max_size then return true end + if #content > max_size then + rspamd_logger.infox("skip %s AV check as it is too large: %s (%s is allowed)", + rule.type, #content, max_size) + return false + end + return true +end + +local function need_av_check(task, content, rule) + return message_not_too_large(task, content, rule) +end + +local function check_av_cache(task, digest, rule, fn) + local key = digest + + local function redis_av_cb(err, data) + if data and type(data) == 'string' then + -- Cached + if data ~= 'OK' then + lua_util.debugm(N, task, 'got cached result for %s: %s', key, data) + data = rspamd_str_split(data, '\v') + yield_result(task, rule, data) + else + lua_util.debugm(N, task, 'got cached result for %s: %s', key, data) + end + else + if err then + rspamd_logger.errx(task, 'Got error checking cache: %1', err) + end + fn() + end + end + + if rule.redis_params then + + key = rule['prefix'] .. key + + if lua_redis.redis_make_request(task, + rule.redis_params, -- connect params + key, -- hash key + false, -- is write + redis_av_cb, --callback + 'GET', -- command + {key} -- arguments) + ) then + return true + end + end + + return false +end + +local function save_av_cache(task, digest, rule, to_save) + local key = digest + + local function redis_set_cb(err) + -- Do nothing + if err then + rspamd_logger.errx(task, 'failed to save virus cache for %s -> "%s": %s', + to_save, key, err) + else + lua_util.debugm(N, task, 'saved cached result for %s: %s', + key, to_save) + end + end + + if type(to_save) == 'table' then + to_save = table.concat(to_save, '\v') + end + + if rule.redis_params then + key = rule['prefix'] .. key + + lua_redis.redis_make_request(task, + rule.redis_params, -- connect params + key, -- hash key + true, -- is write + redis_set_cb, --callback + 'SETEX', -- command + { key, rule['cache_expire'], to_save } + ) + end + + return false +end + +local function fprot_check(task, content, digest, rule) + local function fprot_check_uncached () + local upstream = rule.upstreams:get_upstream_round_robin() + local addr = upstream:get_addr() + local retransmits = rule.retransmits + local scan_id = task:get_queue_id() + if not scan_id then scan_id = task:get_uid() end + local header = string.format('SCAN STREAM %s SIZE %d\n', scan_id, + #content) + local footer = '\n' + + local function fprot_callback(err, data) + if err then + -- set current upstream to fail because an error occurred + upstream:fail() + + -- retry with another upstream until retransmits exceeds + if retransmits > 0 then + + retransmits = retransmits - 1 + + -- Select a different upstream! + upstream = rule.upstreams:get_upstream_round_robin() + addr = upstream:get_addr() + + lua_util.debugm(N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr) + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + timeout = rule['timeout'], + callback = fprot_callback, + data = { header, content, footer }, + stop_pattern = '\n' + }) + else + rspamd_logger.errx(task, + '%s [%s]: failed to scan, maximum retransmits exceed', + rule['symbol'], rule['type']) + task:insert_result(rule['symbol_fail'], 0.0, + 'failed to scan and retransmits exceed') + end + else + upstream:ok() + data = tostring(data) + local cached + local clean = string.match(data, '^0 <clean>') + if clean then + cached = 'OK' + if rule['log_clean'] then + rspamd_logger.infox(task, + '%s [%s]: message or mime_part is clean', + rule['symbol'], rule['type']) + end + else + -- returncodes: 1: infected, 2: suspicious, 3: both, 4-255: some error occured + -- see http://www.f-prot.com/support/helpfiles/unix/appendix_c.html for more detail + local vname = string.match(data, '^[1-3] <[%w%s]-: (.-)>') + if not vname then + rspamd_logger.errx(task, 'Unhandled response: %s', data) + else + yield_result(task, rule, vname) + cached = vname + end + end + if cached then + save_av_cache(task, digest, rule, cached) + end + end + end + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + timeout = rule['timeout'], + callback = fprot_callback, + data = { header, content, footer }, + stop_pattern = '\n' + }) + end + + if need_av_check(task, content, rule) then + if check_av_cache(task, digest, rule, fprot_check_uncached) then + return + else + fprot_check_uncached() + end + end +end + +local function clamav_check(task, content, digest, rule) + local function clamav_check_uncached () + local upstream = rule.upstreams:get_upstream_round_robin() + local addr = upstream:get_addr() + local retransmits = rule.retransmits + local header = rspamd_util.pack("c9 c1 >I4", "zINSTREAM", "\0", + #content) + local footer = rspamd_util.pack(">I4", 0) + + local function clamav_callback(err, data) + if err then + + -- set current upstream to fail because an error occurred + upstream:fail() + + -- retry with another upstream until retransmits exceeds + if retransmits > 0 then + + retransmits = retransmits - 1 + + -- Select a different upstream! + upstream = rule.upstreams:get_upstream_round_robin() + addr = upstream:get_addr() + + lua_util.debugm(N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr) + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + timeout = rule['timeout'], + callback = clamav_callback, + data = { header, content, footer }, + stop_pattern = '\0' + }) + else + rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type']) + task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and retransmits exceed') + end + + else + upstream:ok() + data = tostring(data) + local cached + lua_util.debugm(N, task, '%s [%s]: got reply: %s', rule['symbol'], rule['type'], data) + if data == 'stream: OK' then + cached = 'OK' + if rule['log_clean'] then + rspamd_logger.infox(task, '%s [%s]: message or mime_part is clean', rule['symbol'], rule['type']) + else + lua_util.debugm(N, task, '%s [%s]: message or mime_part is clean', rule['symbol'], rule['type']) + end + else + local vname = string.match(data, 'stream: (.+) FOUND') + if vname then + yield_result(task, rule, vname) + cached = vname + else + rspamd_logger.errx(task, 'unhandled response: %s', data) + task:insert_result(rule['symbol_fail'], 0.0, 'unhandled response') + end + end + if cached then + save_av_cache(task, digest, rule, cached) + end + end + end + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + timeout = rule['timeout'], + callback = clamav_callback, + data = { header, content, footer }, + stop_pattern = '\0' + }) + end + + if need_av_check(task, content, rule) then + if check_av_cache(task, digest, rule, clamav_check_uncached) then + return + else + clamav_check_uncached() + end + end +end + +local function sophos_check(task, content, digest, rule) + local function sophos_check_uncached () + local upstream = rule.upstreams:get_upstream_round_robin() + local addr = upstream:get_addr() + local retransmits = rule.retransmits + local protocol = 'SSSP/1.0\n' + local streamsize = string.format('SCANDATA %d\n', #content) + local bye = 'BYE\n' + + local function sophos_callback(err, data, conn) + + if err then + -- set current upstream to fail because an error occurred + upstream:fail() + + -- retry with another upstream until retransmits exceeds + if retransmits > 0 then + + retransmits = retransmits - 1 + + -- Select a different upstream! + upstream = rule.upstreams:get_upstream_round_robin() + addr = upstream:get_addr() + + lua_util.debugm(N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr) + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + timeout = rule['timeout'], + callback = sophos_callback, + data = { protocol, streamsize, content, bye } + }) + else + rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type']) + task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and retransmits exceed') + end + else + upstream:ok() + data = tostring(data) + lua_util.debugm(N, task, '%s [%s]: got reply: %s', rule['symbol'], rule['type'], data) + local vname = string.match(data, 'VIRUS (%S+) ') + if vname then + yield_result(task, rule, vname) + save_av_cache(task, digest, rule, vname) + else + if string.find(data, 'DONE OK') then + if rule['log_clean'] then + rspamd_logger.infox(task, '%s [%s]: message or mime_part is clean', rule['symbol'], rule['type']) + else + lua_util.debugm(N, task, '%s [%s]: message or mime_part is clean', rule['symbol'], rule['type']) + end + save_av_cache(task, digest, rule, 'OK') + -- not finished - continue + elseif string.find(data, 'ACC') or string.find(data, 'OK SSSP') then + conn:add_read(sophos_callback) + -- set pseudo virus if configured, else do nothing since it's no fatal + elseif string.find(data, 'FAIL 0212') then + rspamd_logger.infox(task, 'Message is ENCRYPTED (0212 SOPHOS_SAVI_ERROR_FILE_ENCRYPTED): %s', data) + if rule['savdi_report_encrypted'] then + yield_result(task, rule, "SAVDI_FILE_ENCRYPTED") + save_av_cache(task, digest, rule, "SAVDI_FILE_ENCRYPTED") + end + -- set pseudo virus if configured, else set fail since part was not scanned + elseif string.find(data, 'REJ 4') then + if rule['savdi_report_oversize'] then + rspamd_logger.infox(task, 'SAVDI: Message is OVERSIZED (SSSP reject code 4): %s', data) + yield_result(task, rule, "SAVDI_FILE_OVERSIZED") + save_av_cache(task, digest, rule, "SAVDI_FILE_OVERSIZED") + else + rspamd_logger.errx(task, 'SAVDI: Message is OVERSIZED (SSSP reject code 4): %s', data) + task:insert_result(rule['symbol_fail'], 0.0, 'Message is OVERSIZED (SSSP reject code 4):' .. data) + end + -- excplicitly set REJ1 message when SAVDIreports a protocol error + elseif string.find(data, 'REJ 1') then + rspamd_logger.errx(task, 'SAVDI (Protocol error (REJ 1)): %s', data) + task:insert_result(rule['symbol_fail'], 0.0, 'SAVDI (Protocol error (REJ 1)):' .. data) + else + rspamd_logger.errx(task, 'unhandled response: %s', data) + task:insert_result(rule['symbol_fail'], 0.0, 'unhandled response') + end + + end + end + end + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + timeout = rule['timeout'], + callback = sophos_callback, + data = { protocol, streamsize, content, bye } + }) + end + + if need_av_check(task, content, rule) then + if check_av_cache(task, digest, rule, sophos_check_uncached) then + return + else + sophos_check_uncached() + end + end +end + +local function savapi_check(task, content, digest, rule) + local function savapi_check_uncached () + local upstream = rule.upstreams:get_upstream_round_robin() + local addr = upstream:get_addr() + local retransmits = rule.retransmits + local fname = string.format('%s/%s.tmp', + rule.tmpdir, rspamd_util.random_hex(32)) + local message_fd = rspamd_util.create_file(fname) + + if not message_fd then + rspamd_logger.errx('cannot store file for savapi scan: %s', fname) + return + end + + if type(content) == 'string' then + -- Create rspamd_text + local rspamd_text = require "rspamd_text" + content = rspamd_text.fromstring(content) + end + content:save_in_file(message_fd) + + -- Ensure cleanup + task:get_mempool():add_destructor(function() + os.remove(fname) + rspamd_util.close_file(message_fd) + end) + + local vnames = {} + + -- Forward declaration for recursive calls + local savapi_scan1_cb + + local function savapi_fin_cb(err, conn) + local vnames_reordered = {} + -- Swap table + for virus,_ in pairs(vnames) do + table.insert(vnames_reordered, virus) + end + lua_util.debugm(N, task, "%s: number of virus names found %s", rule['type'], #vnames_reordered) + if #vnames_reordered > 0 then + local vname = {} + for _,virus in ipairs(vnames_reordered) do + table.insert(vname, virus) + end + + yield_result(task, rule, vname) + save_av_cache(task, digest, rule, vname) + end + if conn then + conn:close() + end + end + + local function savapi_scan2_cb(err, data, conn) + local result = tostring(data) + lua_util.debugm(N, task, "%s: got reply: %s", + rule['type'], result) + + -- Terminal response - clean + if string.find(result, '200') or string.find(result, '210') then + if rule['log_clean'] then + rspamd_logger.infox(task, '%s: message or mime_part is clean', rule['type']) + end + save_av_cache(task, digest, rule, 'OK') + conn:add_write(savapi_fin_cb, 'QUIT\n') + + -- Terminal response - infected + elseif string.find(result, '319') then + conn:add_write(savapi_fin_cb, 'QUIT\n') + + -- Non-terminal response + elseif string.find(result, '310') then + local virus + virus = result:match "310.*<<<%s(.*)%s+;.*;.*" + if not virus then + virus = result:match "310%s(.*)%s+;.*;.*" + if not virus then + rspamd_logger.errx(task, "%s: virus result unparseable: %s", + rule['type'], result) + return + end + end + -- Store unique virus names + vnames[virus] = 1 + -- More content is expected + conn:add_write(savapi_scan1_cb, '\n') + end + end + + savapi_scan1_cb = function(err, conn) + conn:add_read(savapi_scan2_cb, '\n') + end + + -- 100 PRODUCT:xyz + local function savapi_greet2_cb(err, data, conn) + local result = tostring(data) + if string.find(result, '100 PRODUCT') then + lua_util.debugm(N, task, "%s: scanning file: %s", + rule['type'], fname) + conn:add_write(savapi_scan1_cb, {string.format('SCAN %s\n', + fname)}) + else + rspamd_logger.errx(task, '%s: invalid product id %s', rule['type'], + rule['product_id']) + conn:add_write(savapi_fin_cb, 'QUIT\n') + end + end + + local function savapi_greet1_cb(err, conn) + conn:add_read(savapi_greet2_cb, '\n') + end + + local function savapi_callback_init(err, data, conn) + if err then + + -- set current upstream to fail because an error occurred + upstream:fail() + + -- retry with another upstream until retransmits exceeds + if retransmits > 0 then + + retransmits = retransmits - 1 + + -- Select a different upstream! + upstream = rule.upstreams:get_upstream_round_robin() + addr = upstream:get_addr() + + lua_util.debugm(N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr) + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + timeout = rule['timeout'], + callback = savapi_callback_init, + stop_pattern = {'\n'}, + }) + else + rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type']) + task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and retransmits exceed') + end + else + upstream:ok() + local result = tostring(data) + + -- 100 SAVAPI:4.0 greeting + if string.find(result, '100') then + conn:add_write(savapi_greet1_cb, {string.format('SET PRODUCT %s\n', rule['product_id'])}) + end + end + end + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + timeout = rule['timeout'], + callback = savapi_callback_init, + stop_pattern = {'\n'}, + }) + end + + if need_av_check(task, content, rule) then + if check_av_cache(task, digest, rule, savapi_check_uncached) then + return + else + savapi_check_uncached() + end + end +end + +local function kaspersky_check(task, content, digest, rule) + local function kaspersky_check_uncached () + local upstream = rule.upstreams:get_upstream_round_robin() + local addr = upstream:get_addr() + local retransmits = rule.retransmits + local fname = string.format('%s/%s.tmp', + rule.tmpdir, rspamd_util.random_hex(32)) + local message_fd = rspamd_util.create_file(fname) + local clamav_compat_cmd = string.format("nSCAN %s\n", fname) + + if not message_fd then + rspamd_logger.errx('cannot store file for kaspersky scan: %s', fname) + return + end + + if type(content) == 'string' then + -- Create rspamd_text + local rspamd_text = require "rspamd_text" + content = rspamd_text.fromstring(content) + end + content:save_in_file(message_fd) + + -- Ensure file cleanup + task:get_mempool():add_destructor(function() + os.remove(fname) + rspamd_util.close_file(message_fd) + end) + + + local function kaspersky_callback(err, data) + if err then + -- set current upstream to fail because an error occurred + upstream:fail() + + -- retry with another upstream until retransmits exceeds + if retransmits > 0 then + + retransmits = retransmits - 1 + + -- Select a different upstream! + upstream = rule.upstreams:get_upstream_round_robin() + addr = upstream:get_addr() + + lua_util.debugm(N, task, + '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr) + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + timeout = rule['timeout'], + callback = kaspersky_callback, + data = { clamav_compat_cmd }, + stop_pattern = '\n' + }) + else + rspamd_logger.errx(task, + '%s [%s]: failed to scan, maximum retransmits exceed', + rule['symbol'], rule['type']) + task:insert_result(rule['symbol_fail'], 0.0, + 'failed to scan and retransmits exceed') + end + + else + upstream:ok() + data = tostring(data) + local cached + lua_util.debugm(N, task, '%s [%s]: got reply: %s', + rule['symbol'], rule['type'], data) + if data == 'stream: OK' then + cached = 'OK' + if rule['log_clean'] then + rspamd_logger.infox(task, '%s [%s]: message or mime_part is clean', + rule['symbol'], rule['type']) + else + lua_util.debugm(N, task, '%s [%s]: message or mime_part is clean', + rule['symbol'], rule['type']) + end + else + local vname = string.match(data, ': (.+) FOUND') + if vname then + yield_result(task, rule, vname) + cached = vname + else + rspamd_logger.errx(task, 'unhandled response: %s', data) + task:insert_result(rule['symbol_fail'], 0.0, 'unhandled response') + end + end + if cached then + save_av_cache(task, digest, rule, cached) + end + end + end + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + timeout = rule['timeout'], + callback = kaspersky_callback, + data = { clamav_compat_cmd }, + stop_pattern = '\n' + }) + end + + if need_av_check(task, content, rule) then + if check_av_cache(task, digest, rule, kaspersky_check_uncached) then + return + else + kaspersky_check_uncached() + end + end +end + +local exports = { + av_types = { + clamav = { + configure = clamav_config, + check = clamav_check + }, + fprot = { + configure = fprot_config, + check = fprot_check + }, + sophos = { + configure = sophos_config, + check = sophos_check + }, + savapi = { + configure = savapi_config, + check = savapi_check + }, + kaspersky = { + configure = kaspersky_config, + check = kaspersky_check + } + }, + -- Some utilities + match_patterns = match_patterns, + check_av_cache = check_av_cache, + save_av_cache = save_av_cache, +} + +exports.add_antivirus = function(name, conf_func, check_func) + assert(type(conf_func) == 'function' and type(check_func) == 'function', + 'bad arguments') + exports.av_types[name] = { + configure = conf_func, + check = check_func, + } +end + +return exports
\ No newline at end of file diff --git a/lualib/lua_auth_results.lua b/lualib/lua_auth_results.lua index 07957376d..b7c8bbc09 100644 --- a/lualib/lua_auth_results.lua +++ b/lualib/lua_auth_results.lua @@ -28,13 +28,6 @@ local default_settings = { none = 'R_SPF_NA', permerror = 'R_SPF_PERMFAIL', }, - dkim_symbols = { - pass = 'R_DKIM_ALLOW', - fail = 'R_DKIM_REJECT', - temperror = 'R_DKIM_TEMPFAIL', - none = 'R_DKIM_NA', - permerror = 'R_DKIM_PERMFAIL', - }, dmarc_symbols = { pass = 'DMARC_POLICY_ALLOW', permerror = 'DMARC_BAD_POLICY', @@ -78,11 +71,13 @@ local function gen_auth_results(task, settings) local mta_hostname = task:get_request_header('MTA-Name') or task:get_request_header('MTA-Tag') if mta_hostname then - table.insert(hdr_parts, tostring(mta_hostname)) + mta_hostname = tostring(mta_hostname) else - table.insert(hdr_parts, local_hostname) + mta_hostname = local_hostname end + table.insert(hdr_parts, mta_hostname) + for auth_type, symbols in pairs(auth_types) do for key, sym in pairs(symbols) do if not common.symbols.sym then @@ -105,6 +100,46 @@ local function gen_auth_results(task, settings) end end + local dkim_results = task:get_dkim_results() + -- For each signature we set authentication results + -- dkim=neutral (body hash did not verify) header.d=example.com header.s=sel header.b=fA8VVvJ8; + -- dkim=neutral (body hash did not verify) header.d=example.com header.s=sel header.b=f8pM8o90; + + for _,dres in ipairs(dkim_results) do + local ar_string = 'none' + + if dres.result == 'reject' then + ar_string = 'fail' -- imply failure, not neutral + elseif dres.result == 'allow' then + ar_string = 'pass' + elseif dres.result == 'bad record' or dres.result == 'permerror' then + ar_string = 'permerror' + elseif dres.result == 'tempfail' then + ar_string = 'temperror' + end + local hdr = {} + + hdr[1] = string.format('dkim=%s', ar_string) + + if dres.fail_reason then + hdr[#hdr + 1] = string.format('(%s)', dres.fail_reason) + end + + if dres.domain then + hdr[#hdr + 1] = string.format('header.d=%s', dres.domain) + end + + if dres.selector then + hdr[#hdr + 1] = string.format('header.s=%s', dres.selector) + end + + if dres.bhash then + hdr[#hdr + 1] = string.format('header.b=%s', dres.bhash) + end + + table.insert(hdr_parts, table.concat(hdr, ' ')) + end + for auth_type, keys in pairs(auth_results) do for _, key in ipairs(keys) do local hdr = '' @@ -134,15 +169,6 @@ local function gen_auth_results(task, settings) end end table.insert(hdr_parts, hdr) - elseif auth_type == 'dkim' and key ~= 'none' then - if common.symbols[auth_types['dkim'][key]][1] then - local dkim_parts = {} - local opts = common.symbols[auth_types['dkim'][key]][1]['options'] - for _, v in ipairs(opts) do - table.insert(dkim_parts, auth_type .. '=' .. key .. ' header.d=' .. v) - end - table.insert(hdr_parts, table.concat(dkim_parts, '; ')) - end elseif auth_type == 'arc' and key ~= 'none' then if common.symbols[auth_types['arc'][key]][1] then local opts = common.symbols[auth_types['arc'][key]][1]['options'] or {} @@ -152,16 +178,51 @@ local function gen_auth_results(task, settings) end end elseif auth_type == 'spf' and key ~= 'none' then - hdr = hdr .. auth_type .. '=' .. key + -- Main type + local sender + local sender_type local smtp_from = task:get_from('smtp') - if smtp_from and smtp_from[1] and smtp_from[1]['addr'] ~= '' and smtp_from[1]['addr'] ~= nil then - hdr = hdr .. ' smtp.mailfrom=' .. smtp_from[1]['addr'] + + if smtp_from and + smtp_from[1] and + smtp_from[1]['addr'] ~= '' and + smtp_from[1]['addr'] ~= nil then + sender = smtp_from[1]['addr'] + sender_type = 'smtp.mailfrom' else local helo = task:get_helo() if helo then - hdr = hdr .. ' smtp.helo=' .. task:get_helo() + sender = helo + sender_type = 'smtp.helo' end end + + if sender and sender_type then + -- Comment line + local comment = '' + if key == 'pass' then + comment = string.format('%s: domain of %s designates %s as permitted sender', + mta_hostname, sender, tostring(task:get_from_ip() or 'unknown')) + elseif key == 'fail' then + comment = string.format('%s: domain of %s does not designate %s as permitted sender', + mta_hostname, sender, tostring(task:get_from_ip() or 'unknown')) + elseif key == 'neutral' or key == 'softfail' then + comment = string.format('%s: %s is neither permitted nor denied by domain of %s', + mta_hostname, tostring(task:get_from_ip() or 'unknown'), sender) + elseif key == 'permerror' then + comment = string.format('%s: domain of %s uses mechanism not recognized by this client', + mta_hostname, sender) + elseif key == 'temperror' then + comment = string.format('%s: error in processing during lookup of %s: DNS error', + mta_hostname, sender) + end + hdr = string.format('%s=%s (%s) %s=%s', auth_type, key, + comment, sender_type, sender) + else + hdr = string.format('%s=%s', auth_type, key) + end + + table.insert(hdr_parts, hdr) end end diff --git a/lualib/lua_clickhouse.lua b/lualib/lua_clickhouse.lua index e14518ca6..dbb74e283 100644 --- a/lualib/lua_clickhouse.lua +++ b/lualib/lua_clickhouse.lua @@ -15,6 +15,11 @@ See the License for the specific language governing permissions and limitations under the License. ]]-- +--[[[ +-- @module lua_clickhouse +-- This module contains Clickhouse access functions +--]] + local rspamd_logger = require "rspamd_logger" local rspamd_http = require "rspamd_http" local lua_util = require "lua_util" @@ -222,7 +227,9 @@ exports.select = function (upstream, settings, params, query, ok_cb, fail_cb) connect_prefix = 'https://' end local ip_addr = upstream:get_addr():to_string(true) - http_params.url = connect_prefix .. ip_addr .. '/?default_format=JSONEachRow' + local database = params.database or 'default' + http_params.url = string.format('%s%s/?database=%s&default_format=JSONEachRow', + connect_prefix, ip_addr, escape_spaces(database)) end return rspamd_http.request(http_params) @@ -271,7 +278,9 @@ exports.select_sync = function (upstream, settings, params, query, ok_cb, fail_c connect_prefix = 'https://' end local ip_addr = upstream:get_addr():to_string(true) - http_params.url = connect_prefix .. ip_addr .. '/?default_format=JSONEachRow' + local database = params.database or 'default' + http_params.url = string.format('%s%s/?database=%s&default_format=JSONEachRow', + connect_prefix, ip_addr, escape_spaces(database)) end local err, response = rspamd_http.request(http_params) @@ -333,9 +342,11 @@ exports.insert = function (upstream, settings, params, query, rows, connect_prefix = 'https://' end local ip_addr = upstream:get_addr():to_string(true) - http_params.url = string.format('%s%s/?query=%s%%20FORMAT%%20TabSeparated', + local database = params.database or 'default' + http_params.url = string.format('%s%s/?database=%s&query=%s%%20FORMAT%%20TabSeparated', connect_prefix, ip_addr, + escape_spaces(database), escape_spaces(query)) end @@ -383,7 +394,9 @@ exports.generic = function (upstream, settings, params, query, connect_prefix = 'https://' end local ip_addr = upstream:get_addr():to_string(true) - http_params.url = connect_prefix .. ip_addr .. '/?default_format=JSONEachRow' + local database = params.database or 'default' + http_params.url = string.format('%s%s/?database=%s&default_format=JSONEachRow', + connect_prefix, ip_addr, escape_spaces(database)) end return rspamd_http.request(http_params) @@ -426,7 +439,9 @@ exports.generic_sync = function (upstream, settings, params, query) connect_prefix = 'https://' end local ip_addr = upstream:get_addr():to_string(true) - http_params.url = connect_prefix .. ip_addr .. '/?default_format=JSONEachRow' + local database = params.database or 'default' + http_params.url = string.format('%s%s/?database=%s&default_format=JSONEachRow', + connect_prefix, ip_addr, escape_spaces(database)) end return rspamd_http.request(http_params) diff --git a/lualib/lua_fuzzy.lua b/lualib/lua_fuzzy.lua new file mode 100644 index 000000000..ea74b4131 --- /dev/null +++ b/lualib/lua_fuzzy.lua @@ -0,0 +1,321 @@ +--[[ +Copyright (c) 2018, Vsevolod Stakhov <vsevolod@highsecure.ru> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +--[[[ +-- @module lua_fuzzy +-- This module contains helper functions for supporting fuzzy check module +--]] + + +local N = "lua_fuzzy" +local lua_util = require "lua_util" +local rspamd_regexp = require "rspamd_regexp" +local fun = require "fun" +local rspamd_logger = require "rspamd_logger" +local ts = require("tableshape").types + +-- Filled by C code, indexed by number in this table +local rules = {} + +-- Pre-defined rules options +local policies = { + recommended = { + min_bytes = 1024, + min_height = 500, + min_width = 500, + min_length = 32, + text_multiplier = 4.0, -- divide min_bytes by 4 for texts + mime_types = {"application/*"}, + scan_archives = true, + short_text_direct_hash = true, + text_shingles = true, + skip_images = false, + } +} + +local default_policy = policies.recommended + +local policy_schema = ts.shape{ + min_bytes = ts.number + ts.string / tonumber, + min_height = ts.number + ts.string / tonumber, + min_width = ts.number + ts.string / tonumber, + min_length = ts.number + ts.string / tonumber, + text_multiplier = ts.number, + mime_types = ts.array_of(ts.string), + scan_archives = ts.boolean, + short_text_direct_hash = ts.boolean, + text_shingles = ts.boolean, + skip_imagess = ts.boolean, +} + + +local exports = {} + + +--[[[ +-- @function lua_fuzzy.register_policy(name, policy) +-- Adds a new policy with name `name`. Must be valid, checked using policy_schema +--]] +exports.register_policy = function(name, policy) + if policies[name] then + rspamd_logger.warnx(rspamd_config, "overriding policy %s", name) + end + + local parsed_policy,err = policy_schema:transform(policy) + + if not parsed_policy then + rspamd_logger.errx(rspamd_config, 'invalid fuzzy rule policy %s: %s', + name, err) + + return + else + policies.name = parsed_policy + end +end + +--[[[ +-- @function lua_fuzzy.process_rule(rule) +-- Processes fuzzy rule (applying policies or defaults if needed). Returns policy id +--]] +exports.process_rule = function(rule) + local processed_rule = lua_util.shallowcopy(rule) + local policy = default_policy + + if processed_rule.policy then + policy = policies[processed_rule.policy] + end + + if policy then + processed_rule = lua_util.override_defaults(policy, processed_rule) + else + rspamd_logger.warnx(rspamd_config, "unknown policy %s", processed_rule.policy) + end + + if processed_rule.mime_types then + processed_rule.mime_types = fun.totable(fun.map(function(gl) + return rspamd_regexp.import_glob(gl, 'i') + end, processed_rule.mime_types)) + end + + table.insert(rules, processed_rule) + return #rules +end + +local function check_length(task, part, rule) + local bytes = part:get_length() + local length_ok = bytes > 0 + + local id = part:get_id() + lua_util.debugm(N, task, 'check size of part %s', id) + + if length_ok and rule.min_bytes > 0 then + + local adjusted_bytes = bytes + + if part:is_text() then + bytes = part:get_text():get_length() + if rule.text_multiplier then + adjusted_bytes = bytes * rule.text_multiplier + end + end + + if rule.min_bytes > adjusted_bytes then + lua_util.debugm(N, task, 'skip part of length %s (%s adjusted) ' .. + 'as it has less than %s bytes', + bytes, adjusted_bytes, rule.min_bytes) + length_ok = false + else + lua_util.debugm(N, task, 'allow part of length %s (%s adjusted)', + bytes, adjusted_bytes, rule.min_bytes) + end + else + lua_util.debugm(N, task, 'allow part %s, no length limits', id) + end + + return length_ok +end + +local function check_text_part(task, part, rule, text) + local allow_direct,allow_shingles = false,false + + local id = part:get_id() + lua_util.debugm(N, task, 'check text part %s', id) + local wcnt = text:get_words_count() + + if rule.text_shingles then + -- Check number of words + if rule.min_length > 0 and wcnt < rule.min_length then + lua_util.debugm(N, task, 'text has less than %s words: %s; disable shingles', + rule.min_length, wcnt) + allow_shingles = false + else + lua_util.debugm(N, task, 'allow shingles in text %s, %s words', + id, wcnt) + allow_shingles = wcnt > 0 + end + + if not rule.short_text_direct_hash and not allow_shingles then + allow_direct = false + else + if not allow_shingles then + lua_util.debugm(N, task, + 'allow direct hash for short text %s, %s words', + id, wcnt) + allow_direct = check_length(task, part, rule) + else + allow_direct = wcnt > 0 + end + end + else + lua_util.debugm(N, task, + 'disable shingles in text %s', id) + allow_direct = check_length(task, part, rule) + end + + return allow_direct,allow_shingles +end + +local function has_sane_text_parts(task) + local text_parts = task:get_text_parts() or {} + + return fun.any(function(tp) return tp:get_words_count() > 10 end, text_parts) +end + +local function check_image_part(task, part, rule, image) + if rule.skip_images then + lua_util.debugm(N, task, 'skip image part as images are disabled') + return false,false + end + + local id = part:get_id() + lua_util.debugm(N, task, 'check image part %s', id) + + if rule.min_width > 0 or rule.min_height > 0 then + -- Check dimensions + local min_width = rule.min_width or rule.min_height + local min_height = rule.min_height or rule.min_width + local height = image:get_height() + local width = image:get_width() + + if height and width then + if height < min_height or width < min_width then + + + if not has_sane_text_parts(task) then + lua_util.debugm(N, task, 'allow image part %s (%sx%s): no large enough text part found', + id, width, height) + return true, false + else + lua_util.debugm(N, task, 'skip image part %s as it does not meet minimum sizes: %sx%s < %sx%s', + id, width, height, min_width, min_height) + return false, false + end + + + else + lua_util.debugm(N, task, 'allow image part %s: %sx%s', + id, width, height) + end + end + end + + return check_length(task, part, rule),false +end + +local function mime_types_check(task, part, rule) + local t,st = part:get_type() + + if not t then return false, false end + + local ct = string.format('%s/%s', t, st) + t,st = part:get_detected_type() + local detected_ct = string.format('%s/%s', t, st) + local id = part:get_id() + lua_util.debugm(N, task, 'check binary part %s: %s', id, ct) + + -- For bad mime mime parts we implicitly enable fuzzy check + local mime_trace = (task:get_symbol('MIME_TRACE') or {})[1] + local opts = {} + + if mime_trace then + opts = mime_trace.options or opts + end + opts = fun.tomap(fun.map(function(opt) + local elts = lua_util.str_split(opt, ':') + return elts[1],elts[2] + end, opts)) + + if opts[id] and opts[id] == '-' then + lua_util.debugm(N, task, 'explicitly check binary part %s: bad mime type %s', id, ct) + return check_length(task, part, rule),false + end + + if rule.mime_types then + + if fun.any(function(gl_re) + if gl_re:match(ct) or (detected_ct and gl_re:match(detected_ct)) then + return true + else + return false + end + end, rule.mime_types) then + lua_util.debugm(N, task, 'found mime type match for part %s: %s (%s detected)', + id, ct, detected_ct) + return check_length(task, part, rule),false + end + + return false, false + end + + return false,false +end + +exports.check_mime_part = function(task, part, rule_id) + local rule = rules[rule_id] + + if not rule then + rspamd_logger.errx(task, 'cannot find rule with id %s', rule_id) + + return false,false + end + + if part:is_text() then + return check_text_part(task, part, rule, part:get_text()) + end + + if part:is_image() then + return check_image_part(task, part, rule, part:get_image()) + end + + if part:is_archive() and rule.scan_archives then + -- Always send archives + lua_util.debugm(N, task, 'check archive part %s', part:get_id()) + + return true,false + end + + if part:is_attachment() then + return mime_types_check(task, part, rule) + end + + return false,false +end + +exports.cleanup_rules = function() + rules = {} +end + +return exports
\ No newline at end of file diff --git a/lualib/lua_meta.lua b/lualib/lua_meta.lua index f2b008dd4..316c71d3d 100644 --- a/lualib/lua_meta.lua +++ b/lualib/lua_meta.lua @@ -103,7 +103,7 @@ local function meta_nparts_function(task) if parts then for _,p in ipairs(parts) do - if p:get_filename() then + if p:is_attachment() then nattachments = nattachments + 1 end totalparts = totalparts + 1 diff --git a/lualib/lua_redis.lua b/lualib/lua_redis.lua index e95981c82..a5879e130 100644 --- a/lualib/lua_redis.lua +++ b/lualib/lua_redis.lua @@ -22,6 +22,7 @@ local ts = require("tableshape").types local exports = {} local E = {} +local N = "lua_redis" local common_schema = ts.shape { timeout = (ts.number + ts.string / lutil.parse_time_interval):is_optional(), @@ -31,6 +32,10 @@ local common_schema = ts.shape { prefix = ts.string:is_optional(), password = ts.string:is_optional(), expand_keys = ts.boolean:is_optional(), + sentinels = (ts.string + ts.array_of(ts.string)):is_optional(), + sentinel_watch_time = (ts.number + ts.string / lutil.parse_time_interval):is_optional(), + sentinel_masters_pattern = ts.string:is_optional(), + sentinel_master_maxerrors = (ts.number + ts.string / tonumber):is_optional(), } local config_schema = @@ -47,11 +52,227 @@ local config_schema = exports.config_schema = config_schema + +local function redis_query_sentinel(ev_base, params, initialised) + local function flatten_redis_table(tbl) + local res = {} + for i=1,#tbl,2 do + res[tbl[i]] = tbl[i + 1] + end + + return res + end + -- Coroutines syntax + local rspamd_redis = require "rspamd_redis" + local addr = params.sentinels:get_upstream_round_robin() + + local is_ok, connection = rspamd_redis.connect_sync({ + host = addr:get_addr(), + timeout = params.timeout, + config = rspamd_config, + ev_base = ev_base, + }) + + if not is_ok then + logger.errx(rspamd_config, 'cannot connect sentinel at address: %s', + tostring(addr:get_addr())) + addr:fail() + + return + end + + -- Get masters list + connection:add_cmd('SENTINEL', {'masters'}) + + local ok,result = connection:exec() + + if ok and result and type(result) == 'table' then + local masters = {} + for _,m in ipairs(result) do + local master = flatten_redis_table(m) + + if params.sentinel_masters_pattern then + if master.name:match(params.sentinel_masters_pattern) then + lutil.debugm(N, 'found master %s with ip %s and port %s', + master.name, master.ip, master.port) + masters[master.name] = master + else + lutil.debugm(N, 'skip master %s with ip %s and port %s, pattern %s', + master.name, master.ip, master.port, params.sentinel_masters_pattern) + end + else + lutil.debugm(N, 'found master %s with ip %s and port %s', + master.name, master.ip, master.port) + masters[master.name] = master + end + end + + -- For each master we need to get a list of slaves + for k,v in pairs(masters) do + v.slaves = {} + local slave_result + + connection:add_cmd('SENTINEL', {'slaves', k}) + ok,slave_result = connection:exec() + + if ok then + for _,s in ipairs(slave_result) do + local slave = flatten_redis_table(s) + lutil.debugm(N, rspamd_config, + 'found slave form master %s with ip %s and port %s', + v.name, slave.ip, slave.port) + v.slaves[#v.slaves + 1] = slave + end + end + end + + -- We now form new strings for masters and slaves + local read_servers_tbl, write_servers_tbl = {}, {} + + for _,master in pairs(masters) do + write_servers_tbl[#write_servers_tbl + 1] = string.format( + '%s:%s', master.ip, master.port + ) + read_servers_tbl[#read_servers_tbl + 1] = string.format( + '%s:%s', master.ip, master.port + ) + + for _,slave in ipairs(master.slaves) do + if slave['master-link-status'] == 'ok' then + read_servers_tbl[#read_servers_tbl + 1] = string.format( + '%s:%s', slave.ip, slave.port + ) + end + end + end + + table.sort(read_servers_tbl) + table.sort(write_servers_tbl) + + local read_servers_str = table.concat(read_servers_tbl, ',') + local write_servers_str = table.concat(write_servers_tbl, ',') + + lutil.debugm(N, rspamd_config, + 'new servers list: %s read; %s write', + read_servers_str, + write_servers_str) + + if read_servers_str ~= params.read_servers_str then + local upstream_list = require "rspamd_upstream_list" + + local read_upstreams = upstream_list.create(rspamd_config, + read_servers_str, 6379) + + if read_upstreams then + logger.infox(rspamd_config, 'sentinel %s: replace read servers with new list: %s', + addr:get_addr():to_string(true), read_servers_str) + params.read_servers = read_upstreams + params.read_servers_str = read_servers_str + end + end + + if write_servers_str ~= params.write_servers_str then + local upstream_list = require "rspamd_upstream_list" + + local write_upstreams = upstream_list.create(rspamd_config, + write_servers_str, 6379) + + if write_upstreams then + logger.infox(rspamd_config, 'sentinel %s: replace write servers with new list: %s', + addr:get_addr():to_string(true), write_servers_str) + params.write_servers = write_upstreams + params.write_servers_str = write_servers_str + + local queried = false + + local function monitor_failures(up, _, count) + if count > params.sentinel_master_maxerrors and not queried then + logger.infox(rspamd_config, 'sentinel: master with address %s, caused %s failures, try to query sentinel', + up:get_addr():to_string(true), count) + queried = true -- Avoid multiple checks caused by this monitor + redis_query_sentinel(ev_base, params, true) + end + end + + write_upstreams:add_watcher('failure', monitor_failures) + end + end + + addr:ok() + else + logger.errx('cannot get data from Redis Sentinel %s: %s', + addr:get_addr():to_string(true), result) + addr:fail() + end + +end + +local function add_redis_sentinels(params) + local upstream_list = require "rspamd_upstream_list" + + local upstreams_sentinels = upstream_list.create(rspamd_config, + params.sentinels, 5000) + + if not upstreams_sentinels then + logger.errx(rspamd_config, 'cannot load redis sentinels string: %s', + params.sentinels) + + return + end + + params.sentinels = upstreams_sentinels + + if not params.sentinel_watch_time then + params.sentinel_watch_time = 60 -- Each minute + end + + if not params.sentinel_master_maxerrors then + params.sentinel_master_maxerrors = 2 -- Maximum number of errors before rechecking + end + + rspamd_config:add_on_load(function(cfg, ev_base, worker) + local initialised = false + if worker:is_scanner() then + rspamd_config:add_periodic(ev_base, 0.0, function() + redis_query_sentinel(ev_base, params, initialised) + initialised = true + + return params.sentinel_watch_time + end, false) + end + end) +end + +local cached_results = {} + +local function calculate_redis_hash(params) + local cr = require "rspamd_cryptobox_hash" + + local h = cr.create() + + local function rec_hash(k, v) + if type(v) == 'string' then + h:update(k) + h:update(v) + elseif type(v) == 'number' then + h:update(k) + h:update(tostring(v)) + elseif type(v) == 'table' then + for kk,vv in pairs(v) do + rec_hash(kk, vv) + end + end + end + + rec_hash('top', params) + + return h:base32() +end + --[[[ -- @module lua_redis -- This module contains helper functions for working with Redis --]] - local function try_load_redis_servers(options, rspamd_config, result) local default_port = 6379 local default_timeout = 1.0 @@ -70,6 +291,8 @@ local function try_load_redis_servers(options, rspamd_config, result) upstreams_read = upstream_list.create(options['read_servers'], default_port) end + + result.read_servers_str = options['read_servers'] elseif options['servers'] then if rspamd_config then upstreams_read = upstream_list.create(rspamd_config, @@ -77,6 +300,8 @@ local function try_load_redis_servers(options, rspamd_config, result) else upstreams_read = upstream_list.create(options['servers'], default_port) end + + result.read_servers_str = options['servers'] read_only = false elseif options['server'] then if rspamd_config then @@ -85,6 +310,8 @@ local function try_load_redis_servers(options, rspamd_config, result) else upstreams_read = upstream_list.create(options['server'], default_port) end + + result.read_servers_str = options['server'] read_only = false end @@ -97,9 +324,11 @@ local function try_load_redis_servers(options, rspamd_config, result) upstreams_write = upstream_list.create(options['write_servers'], default_port) end + result.write_servers_str = options['write_servers'] read_only = false elseif not read_only then upstreams_write = upstreams_read + result.write_servers_str = result.read_servers_str end end @@ -143,12 +372,37 @@ local function try_load_redis_servers(options, rspamd_config, result) if upstreams_read then result.read_servers = upstreams_read + if upstreams_write then result.write_servers = upstreams_write end + + local h = calculate_redis_hash(result) + + if cached_results[h] then + for k,v in pairs(cached_results[h]) do + result[k] = v + end + lutil.debugm(N, 'reused redis server: %s', result) + return true + end + + result.hash = h + cached_results[h] = result + + if not result.read_only and options.sentinels then + result.sentinels = options.sentinels + add_redis_sentinels(result) + end + + lutil.debugm(N, 'loaded redis server: %s', result) return true end + lutil.debugm(N, rspamd_config, + 'cannot load redis server from obj: %s, processed to %s', + options, result) + return false end @@ -162,6 +416,7 @@ local function rspamd_parse_redis_server(module_name, module_opts, no_fallback) -- Try local options local opts + lutil.debugm(N, rspamd_config, 'try load redis config for: %s', module_name) if not module_opts then opts = rspamd_config:get_all_opt(module_name) else @@ -621,6 +876,10 @@ local function rspamd_redis_make_request(task, redis_params, key, is_write, options['dbname'] = redis_params['db'] end + lutil.debugm(N, task, 'perform request to redis server' .. + ' (host=%s, timeout=%s): cmd: %s, arguments: %s', ip_addr, + options.timeout, options.cmd, args) + local ret,conn = rspamd_redis.make_request(options) if not ret then @@ -706,6 +965,9 @@ local function redis_make_request_taskless(ev_base, cfg, redis_params, key, options['dbname'] = redis_params['db'] end + lutil.debugm(N, cfg, 'perform taskless request to redis server' .. + ' (host=%s, timeout=%s): cmd: %s, arguments: %s', options.host, + options.timeout, options.cmd, args) local ret,conn = rspamd_redis.make_request(options) if not ret then logger.errx('cannot execute redis request') @@ -1154,6 +1416,10 @@ exports.request = function(redis_params, attrs, req) opts.dbname = redis_params.db end + lutil.debugm(N, 'perform generic request to redis server' .. + ' (host=%s, timeout=%s): cmd: %s, arguments: %s', addr, + opts.timeout, opts.cmd, opts.args) + if opts.callback then local ret,conn = rspamd_redis.make_request(opts) if not ret then diff --git a/lualib/lua_selectors.lua b/lualib/lua_selectors.lua index 065eac89b..760a15e8c 100644 --- a/lualib/lua_selectors.lua +++ b/lualib/lua_selectors.lua @@ -132,13 +132,35 @@ uses any type by default)]], }, -- Get list of all attachments digests ['attachments'] = { - ['get_value'] = function(task) + ['get_value'] = function(task, args) + + local s local parts = task:get_parts() or E local digests = {} - for _,p in ipairs(parts) do - if p:get_filename() then - table.insert(digests, p:get_digest()) + if #args > 0 then + local rspamd_cryptobox = require "rspamd_cryptobox_hash" + local encoding = args[1] or 'hex' + local ht = args[2] or 'blake2' + + for _,p in ipairs(parts) do + if p:get_filename() then + local h = rspamd_cryptobox.create_specific(ht, p:get_content('raw_parsed')) + if encoding == 'hex' then + s = h:hex() + elseif encoding == 'base32' then + s = h:base32() + elseif encoding == 'base64' then + s = h:base64() + end + table.insert(digests, s) + end + end + else + for _,p in ipairs(parts) do + if p:get_filename() then + table.insert(digests, p:get_digest()) + end end end @@ -148,7 +170,13 @@ uses any type by default)]], return nil end, - ['description'] = 'Get list of all attachments digests', + ['description'] = [[Get list of all attachments digests. +The first optional argument is encoding (`hex`, `base32`, `base64`), +the second optional argument is optional hash type (`blake2`, `sha256`, `sha1`, `sha512`, `md5`)]], + + ['args_schema'] = {ts.one_of{'hex', 'base32', 'base64'}:is_optional(), + ts.one_of{'blake2', 'sha256', 'sha1', 'sha512', 'md5'}:is_optional()} + }, -- Get all attachments files ['files'] = { @@ -415,7 +443,7 @@ the second argument is optional hash type (`blake2`, `sha256`, `sha1`, `sha512`, return inp:sub(start_pos, end_pos), 'string' end, - ['description'] = 'Extracts substring', + ['description'] = 'Extracts substring; the first argument is start, the second is the last (like in Lua)', ['args_schema'] = {(ts.number + ts.string / tonumber):is_optional(), (ts.number + ts.string / tonumber):is_optional()} }, @@ -469,6 +497,22 @@ the second argument is optional hash type (`blake2`, `sha256`, `sha1`, `sha512`, ['description'] = 'Drops input value and return values from function\'s arguments or an empty string', ['args_schema'] = (ts.string + ts.array_of(ts.string)):is_optional() }, + ['equal'] = { + ['types'] = { + ['string'] = true, + }, + ['map_type'] = 'string', + ['process'] = function(inp, _, args) + if inp == args[1] then + return inp,'string' + end + + return nil + end, + ['description'] = [[Boolean function equal. +Returns either nil or its argument if input is equal to argument]], + ['args_schema'] = {ts.string} + }, -- Boolean function in, returns either nil or its input if input is in args list ['in'] = { ['types'] = { @@ -496,8 +540,57 @@ Returns either nil or its input if input is in args list]], Returns either nil or its input if input is not in args list]], ['args_schema'] = ts.array_of(ts.string) }, + ['inverse'] = { + ['types'] = { + ['string'] = true, + }, + ['map_type'] = 'string', + ['process'] = function(inp, _, args) + if inp then + return nil + else + return (args[1] or 'true'),'string' + end + end, + ['description'] = [[Inverses input. +Empty string comes the first argument or 'true', non-empty string comes nil]], + ['args_schema'] = {ts.string:is_optional()} + }, + ['ipmask'] = { + ['types'] = { + ['string'] = true, + }, + ['map_type'] = 'string', + ['process'] = function(inp, _, args) + local rspamd_ip = require "rspamd_ip" + -- Non optimal: convert string to an IP address + local ip = rspamd_ip.from_string(inp) + + if not ip or not ip:is_valid() then + lua_util.debugm(M, "cannot convert %s to IP", inp) + return nil + end + + if ip:get_version() == 4 then + local mask = tonumber(args[1]) + + return ip:apply_mask(mask):to_string(),'string' + else + -- IPv6 takes the second argument or the first one... + local mask_str = args[2] or args[1] + local mask = tonumber(mask_str) + + return ip:apply_mask(mask):to_string(),'string' + end + end, + ['description'] = 'Applies mask to IP address. The first argument is the mask for IPv4 addresses, the second is the mask for IPv6 addresses.', + ['args_schema'] = {(ts.number + ts.string / tonumber), + (ts.number + ts.string / tonumber):is_optional()} + }, } +transform_function.match = transform_function.regexp + local function process_selector(task, sel) local function allowed_type(t) if t == 'string' or t == 'text' or t == 'string_list' or t == 'text_list' then @@ -770,10 +863,10 @@ exports.parse_selector = function(cfg, str) map_type = 'string', process = function(inp, t, args) if t == 'userdata' then - return inp[method_name](inp, args),string + return inp[method_name](inp, args),'string' else -- Table - return inp[method_name],string + return inp[method_name],'string' end end, } @@ -937,4 +1030,4 @@ exports.list_transforms = function() return display_selectors(transform_function) end -return exports
\ No newline at end of file +return exports diff --git a/lualib/lua_squeeze_rules.lua b/lualib/lua_squeeze_rules.lua index e85d7e012..d4298957b 100644 --- a/lualib/lua_squeeze_rules.lua +++ b/lualib/lua_squeeze_rules.lua @@ -16,6 +16,7 @@ limitations under the License. local exports = {} local logger = require 'rspamd_logger' +local lua_util = require 'lua_util' -- Squeezed rules part local squeezed_rules = {{}} -- plain vector of all rules squeezed @@ -37,7 +38,7 @@ local function gen_lua_squeeze_function(order) end -- Too expensive to call :( - --logger.debugm(SN, task, 'call for: %s', data[2]) + lua_util.debugm(SN, task, 'call for: %s', data[2]) local status, ret = pcall(real_call) if not status then @@ -90,7 +91,7 @@ local function gen_lua_squeeze_function(order) end end else - logger.debugm(SN, task, 'skip symbol due to settings: %s', data[2]) + lua_util.debugm(SN, task, 'skip symbol due to settings: %s', data[2]) end @@ -106,14 +107,14 @@ exports.squeeze_rule = function(s, func) order = 0, sym = s, } - logger.debugm(SN, rspamd_config, 'squeezed rule: %s', s) + lua_util.debugm(SN, rspamd_config, 'squeezed rule: %s', s) else logger.warnx(rspamd_config, 'duplicate symbol registered: %s, skip', s) end else -- Unconditionally add function to the squeezed rules local id = tostring(#squeezed_rules) - logger.debugm(SN, rspamd_config, 'squeezed unnamed rule: %s', id) + lua_util.debugm(SN, rspamd_config, 'squeezed unnamed rule: %s', id) table.insert(squeezed_rules[1], {func, 'unnamed: ' .. id}) end @@ -132,7 +133,7 @@ exports.squeeze_rule = function(s, func) end exports.squeeze_dependency = function(child, parent) - logger.debugm(SN, rspamd_config, 'squeeze dep %s->%s', child, parent) + lua_util.debugm(SN, rspamd_config, 'squeeze dep %s->%s', child, parent) if not squeezed_deps[parent] then squeezed_deps[parent] = {} @@ -176,7 +177,7 @@ local function register_topology_symbol(order) } local parent = get_ordered_symbol_name(order - 1) - logger.debugm(SN, rspamd_config, 'registered new order of deps: %s->%s', + lua_util.debugm(SN, rspamd_config, 'registered new order of deps: %s->%s', ord_sym, parent) rspamd_config:register_dependency(ord_sym, parent, true) end @@ -188,7 +189,7 @@ exports.squeeze_init = function() if order > node.order then node.order = order - logger.debugm(SN, rspamd_config, "symbol: %s, order: %s", node.sym, order) + lua_util.debugm(SN, rspamd_config, "symbol: %s, order: %s", node.sym, order) else return end @@ -215,12 +216,12 @@ exports.squeeze_init = function() if squeezed_symbols[s] then -- External dep depends on a squeezed symbol - logger.debugm(SN, rspamd_config, 'register external squeezed dependency on %s', + lua_util.debugm(SN, rspamd_config, 'register external squeezed dependency on %s', parent) rspamd_config:register_dependency(squeeze_sym, parent, true) else -- Generic rspamd symbols dependency - logger.debugm(SN, rspamd_config, 'register external dependency %s -> %s', + lua_util.debugm(SN, rspamd_config, 'register external dependency %s -> %s', s, parent) rspamd_config:register_dependency(s, parent, true) end @@ -232,7 +233,7 @@ exports.squeeze_init = function() for cld,_ in pairs(children) do if squeezed_symbols[cld] then -- Cross dependency - logger.debugm(SN, rspamd_config, 'cross dependency in squeezed symbols %s->%s', + lua_util.debugm(SN, rspamd_config, 'cross dependency in squeezed symbols %s->%s', cld, parent) local order = squeezed_symbols[cld].order if not squeeze_function_ids[order] then @@ -247,7 +248,7 @@ exports.squeeze_init = function() -- External symbol depends on a squeezed one local parent_symbol = get_ordered_symbol_name(ps.order) rspamd_config:register_dependency(cld, parent_symbol, true) - logger.debugm(SN, rspamd_config, 'register squeezed dependency for external symbol %s->%s', + lua_util.debugm(SN, rspamd_config, 'register squeezed dependency for external symbol %s->%s', cld, parent_symbol) end end @@ -258,7 +259,7 @@ exports.squeeze_init = function() -- and create squeezed rules for k,v in pairs(squeezed_symbols) do local parent_symbol = get_ordered_symbol_name(v.order) - logger.debugm(SN, rspamd_config, 'added squeezed rule: %s (%s): %s', + lua_util.debugm(SN, rspamd_config, 'added squeezed rule: %s (%s): %s', k, parent_symbol, v) rspamd_config:register_symbol{ type = 'virtual', @@ -277,7 +278,7 @@ exports.squeeze_init = function() if v.group then if not squeezed_groups[v.group] then - logger.debugm(SN, rspamd_config, 'added squeezed group: %s', v.group) + lua_util.debugm(SN, rspamd_config, 'added squeezed group: %s', v.group) squeezed_groups[v.group] = {} end @@ -287,7 +288,7 @@ exports.squeeze_init = function() if v.groups then for _,gr in ipairs(v.groups) do if not squeezed_groups[gr] then - logger.debugm(SN, rspamd_config, 'added squeezed group: %s', gr) + lua_util.debugm(SN, rspamd_config, 'added squeezed group: %s', gr) squeezed_groups[gr] = {} end @@ -295,7 +296,7 @@ exports.squeeze_init = function() end end else - logger.debugm(SN, rspamd_config, 'no metric symbol found for %s, maybe bug', k) + lua_util.debugm(SN, rspamd_config, 'no metric symbol found for %s, maybe bug', k) end if not squeezed_rules[v.order] then squeezed_rules[v.order] = {} @@ -324,7 +325,7 @@ exports.handle_settings = function(task, settings) found = true for _,s in ipairs(settings.symbols_enabled) do if squeezed_symbols[s] then - logger.debugm(SN, task, 'enable symbol %s as it is in `symbols_enabled`', s) + lua_util.debugm(SN, task, 'enable symbol %s as it is in `symbols_enabled`', s) symbols_enabled[s] = true symbols_disabled[s] = nil end @@ -337,7 +338,7 @@ exports.handle_settings = function(task, settings) for _,gr in ipairs(settings.groups_enabled) do if squeezed_groups[gr] then for _,sym in ipairs(squeezed_groups[gr]) do - logger.debugm(SN, task, 'enable symbol %s as it is in `groups_enabled`', sym) + lua_util.debugm(SN, task, 'enable symbol %s as it is in `groups_enabled`', sym) symbols_enabled[sym] = true symbols_disabled[sym] = nil end @@ -348,10 +349,10 @@ exports.handle_settings = function(task, settings) if settings.symbols_disabled then found = true for _,s in ipairs(settings.symbols_disabled) do - logger.debugm(SN, task, 'try disable symbol %s as it is in `symbols_disabled`', s) + lua_util.debugm(SN, task, 'try disable symbol %s as it is in `symbols_disabled`', s) if not symbols_enabled[s] then symbols_disabled[s] = true - logger.debugm(SN, task, 'disable symbol %s as it is in `symbols_disabled`', s) + lua_util.debugm(SN, task, 'disable symbol %s as it is in `symbols_disabled`', s) end end end @@ -359,11 +360,11 @@ exports.handle_settings = function(task, settings) if settings.groups_disabled then found = true for _,gr in ipairs(settings.groups_disabled) do - logger.debugm(SN, task, 'try disable group %s as it is in `groups_disabled`: %s', gr) + lua_util.debugm(SN, task, 'try disable group %s as it is in `groups_disabled`: %s', gr) if squeezed_groups[gr] then for _,sym in ipairs(squeezed_groups[gr]) do if not symbols_enabled[sym] then - logger.debugm(SN, task, 'disable symbol %s as it is in `groups_disabled`', sym) + lua_util.debugm(SN, task, 'disable symbol %s as it is in `groups_disabled`', sym) symbols_disabled[sym] = true end end diff --git a/lualib/lua_stat.lua b/lualib/lua_stat.lua index b5eaafcd3..2606a8d22 100644 --- a/lualib/lua_stat.lua +++ b/lualib/lua_stat.lua @@ -14,10 +14,16 @@ See the License for the specific language governing permissions and limitations under the License. ]]-- +--[[[ +-- @module lua_stat +-- This module contains helper functions for supporting statistics +--]] + local logger = require "rspamd_logger" local sqlite3 = require "rspamd_sqlite3" local util = require "rspamd_util" local lua_redis = require "lua_redis" +local lua_util = require "lua_util" local exports = {} local N = "stat_tools" -- luacheck: ignore (maybe unused) @@ -101,30 +107,30 @@ return nconverted local keys = redis.call('SMEMBERS', KEYS[1]..'_keys') for _,k in ipairs(keys) do - local learns = redis.call('HGET', k, 'learns') + local learns = redis.call('HGET', k, 'learns') or 0 local neutral_prefix = string.gsub(k, KEYS[1], 'RS') redis.call('HSET', neutral_prefix, KEYS[2], learns) redis.call('SADD', KEYS[1]..'_keys', neutral_prefix) redis.call('SREM', KEYS[1]..'_keys', k) - redis.call('DEL', k) - redis.call('SET', KEYS[1]..'_version', '2') + redis.call('DEL', KEYS[1]) + redis.call('SET', k ..'_version', '2') end ]] conn:add_cmd('EVAL', {lua_script, '2', symbol_spam, 'learns_spam'}) - ret = conn:exec() + ret,res = conn:exec() if not ret then - logger.errx('error converting metadata for symbol %s', symbol_spam) + logger.errx('error converting metadata for symbol %s: %s', symbol_spam, res) return false end conn:add_cmd('EVAL', {lua_script, '2', symbol_ham, 'learns_ham'}) - ret = conn:exec() + ret, res = conn:exec() if not ret then - logger.errx('error converting metadata for symbol %s', symbol_ham) + logger.errx('error converting metadata for symbol %s', symbol_ham, res) return false end @@ -511,4 +517,341 @@ end exports.redis_classifier_from_sqlite = redis_classifier_from_sqlite +-- Reads statistics config and return preprocessed table +local function process_stat_config(cfg) + local opts_section = cfg:get_all_opt('options') or {} + + -- Check if we have a dedicated section for statistics + if opts_section.statistics then + opts_section = opts_section.statistics + end + + -- Default + local res_config = { + classify_headers = { + "User-Agent", + "X-Mailer", + "Content-Type", + "X-MimeOLE", + "Organization", + "Organisation" + }, + classify_images = true, + classify_mime_info = true, + classify_urls = true, + classify_meta = true, + classify_max_tlds = 10, + } + + res_config = lua_util.override_defaults(res_config, opts_section) + + -- Postprocess classify_headers + local classify_headers_parsed = {} + + for _,v in ipairs(res_config.classify_headers) do + local s1, s2 = v:match("^([A-Z])[^%-]+%-([A-Z]).*$") + + local hname + if s1 and s2 then + hname = string.format('%s-%s', s1, s2) + else + s1 = v:match("^X%-([A-Z].*)$") + + if s1 then + hname = string.format('x%s', s1:sub(1, 3):lower()) + else + hname = string.format('%s', v:sub(1, 3):lower()) + end + end + + if classify_headers_parsed[hname] then + table.insert(classify_headers_parsed[hname], v) + else + classify_headers_parsed[hname] = {v} + end + end + + res_config.classify_headers_parsed = classify_headers_parsed + + return res_config +end + +local function get_mime_stat_tokens(task, res, i) + local parts = task:get_parts() or {} + local seen_multipart = false + local seen_plain = false + local seen_html = false + local empty_plain = false + local empty_html = false + local online_text = false + + for _,part in ipairs(parts) do + local fname = part:get_filename() + + local sz = part:get_length() + + if sz > 0 then + rawset(res, i, string.format("#ps:%d", + math.floor(math.log(sz)))) + lua_util.debugm("bayes", task, "part size: %s", + res[i]) + i = i + 1 + end + + if fname then + rawset(res, i, "#f:" .. fname) + i = i + 1 + + lua_util.debugm("bayes", task, "added attachment: #f:%s", + fname) + end + + if part:is_text() then + local tp = part:get_text() + + if tp:is_html() then + seen_html = true + + if tp:get_length() == 0 then + empty_html = true + end + else + seen_plain = true + + if tp:get_length() == 0 then + empty_plain = true + end + end + + if tp:get_lines_count() < 2 then + online_text = true + end + + rawset(res, i, "#lang:" .. (tp:get_language() or 'unk')) + lua_util.debugm("bayes", task, "added language: %s", + res[i]) + i = i + 1 + + rawset(res, i, "#cs:" .. (tp:get_charset() or 'unk')) + lua_util.debugm("bayes", task, "added charset: %s", + res[i]) + i = i + 1 + + elseif part:is_multipart() then + seen_multipart = true; + end + end + + -- Create a special token depending on parts structure + local st_tok = "#unk" + if seen_multipart and seen_html and seen_plain then + st_tok = '#mpth' + end + + if seen_html and not seen_plain then + st_tok = "#ho" + end + + if seen_plain and not seen_html then + st_tok = "#to" + end + + local spec_tok = "" + if online_text then + spec_tok = "#ot" + end + + if empty_plain then + spec_tok = spec_tok .. "#ep" + end + + if empty_html then + spec_tok = spec_tok .. "#eh" + end + + rawset(res, i, string.format("#m:%s%s", st_tok, spec_tok)) + lua_util.debugm("bayes", task, "added mime token: %s", + res[i]) + i = i + 1 + + return i +end + +local function get_headers_stat_tokens(task, cf, res, i) + local hdrs_cksum = task:get_mempool():get_variable("headers_hash") + + if hdrs_cksum then + rawset(res, i, string.format("#hh:%s", hdrs_cksum:sub(1, 7))) + lua_util.debugm("bayes", task, "added hdrs hash token: %s", + res[i]) + i = i + 1 + end + + for k,hdrs in pairs(cf.classify_headers_parsed) do + for _,hname in ipairs(hdrs) do + local value = task:get_header(hname) + + if value then + rawset(res, i, string.format("#h:%s:%s", k, value)) + lua_util.debugm("bayes", task, "added hdrs token: %s", + res[i]) + i = i + 1 + end + end + end + + local from = (task:get_from('mime') or {})[1] + + if from and from.name then + rawset(res, i, string.format("#F:%s", from.name)) + lua_util.debugm("bayes", task, "added from name token: %s", + res[i]) + i = i + 1 + end + + return i +end + +local function get_meta_stat_tokens(task, res, i) + local day_and_hour = os.date('%u:%H', + task:get_date{format = 'message', gmt = true}) + rawset(res, i, string.format("#dt:%s", day_and_hour)) + lua_util.debugm("bayes", task, "added day_of_week token: %s", + res[i]) + i = i + 1 + + local pol = {} + + -- Authentication results + if task:has_symbol('DKIM_TRACE') then + -- Autolearn or scan + if task:has_symbol('R_SPF_ALLOW') then + table.insert(pol, 's=pass') + end + + local trace = task:get_symbol('DKIM_TRACE') + local dkim_opts = trace[1]['options'] + if dkim_opts then + for _,o in ipairs(dkim_opts) do + local check_res = string.sub(o, -1) + local domain = string.sub(o, 1, -3) + + if check_res == '+' then + table.insert(pol, string.format('d=%s:%s', "pass", domain)) + end + end + end + else + -- Offline learn + local aur = task:get_header('Authentication-Results') + + if aur then + local spf = aur:match('spf=([a-z]+)') + local dkim,dkim_domain = aur:match('dkim=([a-z]+) header.d=([a-z.%-]+)') + + + if spf then + table.insert(pol, 's=' .. spf) + end + if dkim and dkim_domain then + table.insert(pol, string.format('d=%s:%s', dkim, dkim_domain)) + end + end + end + + if #pol > 0 then + rawset(res, i, string.format("#aur:%s", table.concat(pol, ','))) + lua_util.debugm("bayes", task, "added policies token: %s", + res[i]) + i = i + 1 + end + + local rh = task:get_received_headers() + + if rh and #rh > 0 then + local lim = math.min(5, #rh) + for j =1,lim do + local rcvd = rh[j] + local ip = rcvd.real_ip + if ip and ip:is_valid() and ip:get_version() == 4 then + local masked = ip:apply_mask(24) + + rawset(res, i, string.format("#rcv:%s:%s", tostring(masked), + rcvd.proto)) + lua_util.debugm("bayes", task, "added received token: %s", + res[i]) + i = i + 1 + end + end + end + + return i +end + +local function get_stat_tokens(task, cf) + local res = {} + local E = {} + local i = 1 + + if cf.classify_images then + local images = task:get_images() or E + + for _,img in ipairs(images) do + rawset(res, i, "image") + i = i + 1 + rawset(res, i, tostring(img:get_height())) + i = i + 1 + rawset(res, i, tostring(img:get_width())) + i = i + 1 + rawset(res, i, tostring(img:get_type())) + i = i + 1 + + local fname = img:get_filename() + + if fname then + rawset(res, i, tostring(img:get_filename())) + i = i + 1 + end + + lua_util.debugm("bayes", task, "added image: %s", + fname) + end + end + + if cf.classify_mime_info then + i = get_mime_stat_tokens(task, res, i) + end + + if cf.classify_headers and #cf.classify_headers > 0 then + i = get_headers_stat_tokens(task, cf, res, i) + end + + if cf.classify_urls then + local urls = lua_util.extract_specific_urls{task = task, limit = 5, esld_limit = 1} + + if urls then + for _,u in ipairs(urls) do + rawset(res, i, string.format("#u:%s", u:get_tld())) + lua_util.debugm("bayes", task, "added url token: %s", + res[i]) + i = i + 1 + end + end + end + + if cf.classify_meta then + i = get_meta_stat_tokens(task, res, i) + end + + return res +end + +exports.gen_stat_tokens = function(cfg) + local stat_config = process_stat_config(cfg) + + return function(task) + return get_stat_tokens(task, stat_config) + end +end + return exports diff --git a/lualib/lua_util.lua b/lualib/lua_util.lua index 3c1a431cf..0d56e689d 100644 --- a/lualib/lua_util.lua +++ b/lualib/lua_util.lua @@ -391,6 +391,55 @@ end exports.parse_time_interval = parse_time_interval --[[[ +-- @function lua_util.dehumanize_number(str) +-- Parses human readable number +-- Accepts 'k' for thousands, 'm' for millions, 'g' for billions, 'b' suffix for 1024 multiplier, +-- e.g. `10mb` equal to `10 * 1024 * 1024` +-- @param {string} str input string +-- @return {number|nil} parsed number +--]] +local function dehumanize_number(str) + local function parse_suffix(s) + if s == 'k' then + return 1000 + elseif s == 'm' then + return 1000000 + elseif s == 'g' then + return 1e9 + elseif s == 'kb' then + return 1024 + elseif s == 'mb' then + return 1024 * 1024 + elseif s == 'gb' then + return 1024 * 1024; + end + end + + local digit = lpeg.R("09") + local parser = {} + parser.integer = + (lpeg.S("+-") ^ -1) * + (digit ^ 1) + parser.fractional = + (lpeg.P(".") ) * + (digit ^ 1) + parser.number = + (parser.integer * + (parser.fractional ^ -1)) + + (lpeg.S("+-") * parser.fractional) + parser.humanized_number = lpeg.Cf(lpeg.Cc(1) * + (parser.number / tonumber) * + (((lpeg.S("kmg") * (lpeg.P("b") ^ -1)) / parse_suffix) ^ -1), + function (acc, val) return acc * val end) + + local t = lpeg.match(parser.humanized_number, str) + + return t +end + +exports.dehumanize_number = dehumanize_number + +--[[[ -- @function lua_util.table_cmp(t1, t2) -- Compare two tables deeply --]] @@ -471,7 +520,8 @@ local function override_defaults(def, override) end local res = {} - fun.each(function(k, v) + + for k,v in pairs(override) do if type(v) == 'table' then if def[k] and type(def[k]) == 'table' then -- Recursively override elements @@ -482,12 +532,13 @@ local function override_defaults(def, override) else res[k] = v end - end, override) - fun.each(function(k, v) - if not res[k] then + end + + for k,v in pairs(def) do + if type(res[k]) == 'nil' then res[k] = v end - end, def) + end return res end @@ -742,16 +793,60 @@ if type(rspamd_config) == 'userdata' then end end +--[[[ +-- @function lua_util.debugm(module, [log_object], format, ...) +-- Performs fast debug log for a specific module +--]] exports.debugm = function(mod, obj_or_fmt, fmt_or_something, ...) local logger = require "rspamd_logger" if unconditional_debug or debug_modules[mod] then if type(obj_or_fmt) == 'string' then - logger.logx(log_level, mod, 2, obj_or_fmt, fmt_or_something, ...) + logger.logx(log_level, mod, '', 2, obj_or_fmt, fmt_or_something, ...) else logger.logx(log_level, mod, obj_or_fmt, 2, fmt_or_something, ...) end end end +---[[[ +-- @function lua_util.get_task_verdict(task) +-- Returns verdict for a task, must be called from idempotent filters only +-- Returns string: +-- * `spam`: if message have over reject threshold and has more than one positive rule +-- * `junk`: if a message has between score between [add_header/rewrite subject] to reject thresholds and has more than two positive rules +-- * `passthrough`: if a message has been passed through some short-circuit rule +-- * `ham`: if a message has overall score below junk level **and** more than three negative rule, or negative total score +-- * `uncertain`: all other cases +--]] +exports.get_task_verdict = function(task) + local result = task:get_metric_result() + + if result then + + if result.passthrough then + return 'passthrough' + end + + local action = result.action + + if action == 'reject' and result.npositive > 1 then + return 'spam' + elseif action == 'no action' then + if result.score < 0 or result.nnegative > 3 then + return 'ham' + end + else + -- All colors of junk + if action == 'add header' or action == 'rewrite subject' then + if result.npositive > 2 then + return 'junk' + end + end + end + end + + return 'uncertain' +end + return exports diff --git a/lualib/rspamadm/configwizard.lua b/lualib/rspamadm/configwizard.lua index fb4caf20c..a4bccd7ab 100644 --- a/lualib/rspamadm/configwizard.lua +++ b/lualib/rspamadm/configwizard.lua @@ -395,6 +395,13 @@ local function check_redis_classifier(cls, changes) end local function get_version(conn) + conn:add_cmd("SMEMBERS", {"RS_keys"}) + + local ret,members = conn:exec() + + -- Empty db + if not ret or #members == 0 then return false,0 end + -- We still need to check versions local lua_script = [[ local ver = 0 @@ -469,8 +476,8 @@ return ttl local r,ver = get_version(conn) if not r then return false end if ver ~= 2 then - printf("You have configured new schema for %s/%s but your DB has old data", - symbol_spam, symbol_ham) + printf("You have configured new schema for %s/%s but your DB has old version: %s", + symbol_spam, symbol_ham, ver) try_convert(false) else printf("You have configured new schema for %s/%s and your DB already has new layout (v. %s). DB conversion is not needed.", diff --git a/lualib/rspamadm/cookie.lua b/lualib/rspamadm/cookie.lua new file mode 100644 index 000000000..b9995cdf9 --- /dev/null +++ b/lualib/rspamadm/cookie.lua @@ -0,0 +1,121 @@ +--[[ +Copyright (c) 2018, Vsevolod Stakhov <vsevolod@highsecure.ru> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +local argparse = require "argparse" + + +-- Define command line options +local parser = argparse() + :name "rspamadm cookie" + :description "Produces cookies or message ids" + :help_description_margin(30) + +parser:mutex( + parser:option "-k --key" + :description('Key to load') + :argname "<32hex>", + parser:flag "-K --new-key" + :description('Generates a new key') +) + +parser:option "-d --domain" + :description('Use specified domain and generate full message id') + :argname "<domain>" +parser:flag "-D --decrypt" + :description('Decrypt cookie instead of encrypting one') +parser:flag "-t --timestamp" + :description('Show cookie timestamp (valid for decrypting only)') +parser:argument "cookie":args "?" + :description('Use specified cookie') + +local function gen_cookie(args, key) + local cr = require "rspamd_cryptobox" + + if not args.cookie then return end + + local function encrypt() + if #args.cookie > 31 then + print('cookie too long (>31 characters), cannot encrypt') + os.exit(1) + end + + local enc_cookie = cr.encrypt_cookie(key, args.cookie) + if args.domain then + print(string.format('<%s@%s>', enc_cookie, args.domain)) + else + print(enc_cookie) + end + end + + local function decrypt() + local extracted_cookie = args.cookie:match('^%<?([^@]+)@.*$') + if not extracted_cookie then + -- Assume full message id as a cookie + extracted_cookie = args.cookie + end + + local dec_cookie,ts = cr.decrypt_cookie(key, extracted_cookie) + + if dec_cookie then + if args.timestamp then + print(string.format('%s %s', dec_cookie, ts)) + else + print(dec_cookie) + end + else + print('cannot decrypt cookie') + os.exit(1) + end + end + + if args.decrypt then + decrypt() + else + encrypt() + end +end + +local function handler(args) + local res = parser:parse(args) + + if not (res.key or res['new_key']) then + parser:error('--key or --new-key must be specified') + end + + if res.key then + local pattern = {'^'} + for i=1,32 do pattern[i + 1] = '[a-zA-Z0-9]' end + pattern[34] = '$' + + if not res.key:match(table.concat(pattern, '')) then + parser:error('invalid key: ' .. res.key) + end + + gen_cookie(res, res.key) + else + local util = require "rspamd_util" + local key = util.random_hex(32) + + print(key) + gen_cookie(res, res.key) + end +end + +return { + handler = handler, + description = parser._description, + name = 'cookie' +}
\ No newline at end of file diff --git a/lualib/rspamadm/mime.lua b/lualib/rspamadm/mime.lua index 974d98bcf..96bdfd72c 100644 --- a/lualib/rspamadm/mime.lua +++ b/lualib/rspamadm/mime.lua @@ -76,6 +76,16 @@ extract:flag "-p --part" :description "Show part info" extract:flag "-s --structure" :description "Show structure info (e.g. HTML tags)" +extract:option "-F --words-format" + :description "Words format ('stem', 'norm', 'raw', 'full')" + :argname("<type>") + :convert { + stem = "stem", + norm = "norm", + raw = "raw", + full = "full", + } + :default "stem" local stat = parser:command "stat st s" @@ -117,6 +127,32 @@ urls:flag "--count" urls:flag "-r --reverse" :description "Reverse sort order" +local modify = parser:command "modify mod m" + :description "Modifies MIME message" +modify:argument "file" + :description "File to process" + :argname "<file>" + :args "+" + +modify:option "-a --add-header" + :description "Adds specific header" + :argname "<header=value>" + :count "*" +modify:option "-r --remove-header" + :description "Removes specific header (all occurrences)" + :argname "<header>" + :count "*" +modify:option "-R --rewrite-header" + :description "Rewrites specific header, uses Lua string.format pattern" + :argname "<header=pattern>" + :count "*" +modify:option "-t --text-footer" + :description "Adds footer to text/plain parts from a specific file" + :argname "<file>" +modify:option "-H --html-footer" + :description "Adds footer to text/html parts from a specific file" + :argname "<file>" + local function load_config(opts) local _r,err = rspamd_config:load_ucl(opts['config']) @@ -245,6 +281,28 @@ local function extract_handler(opts) end end + local function print_words(words, full) + local fun = require "fun" + + if not full then + return table.concat(words, ' ') + else + return table.concat( + fun.totable( + fun.map(function(w) + -- [1] - stemmed word + -- [2] - normalised word + -- [3] - raw word + -- [4] - flags (table of strings) + return string.format('%s|%s|%s(%s)', + w[3], w[2], w[1], table.concat(w[4], ',')) + end, words) + ), + ' ' + ) + end + end + for _,fname in ipairs(opts.file) do local task = load_task(opts, fname) out_elts[fname] = {} @@ -254,6 +312,12 @@ local function extract_handler(opts) opts.html = true end + if opts.words then + local howw = opts['words_format'] or 'stem' + table.insert(out_elts[fname], 'meta_words: ' .. + print_words(task:get_meta_words(howw), howw == 'full')) + end + if opts.text or opts.html then local mp = task:get_parts() or {} @@ -265,14 +329,18 @@ local function extract_handler(opts) if part and opts.text and not part:is_html() then maybe_print_text_part_info(part, out_elts[fname]) if opts.words then - table.insert(out_elts[fname], table.concat(part:get_words(), ' ')) + local howw = opts['words_format'] or 'stem' + table.insert(out_elts[fname], print_words(part:get_words(howw), + howw == 'full')) else table.insert(out_elts[fname], tostring(part:get_content(how))) end elseif part and opts.html and part:is_html() then maybe_print_text_part_info(part, out_elts[fname]) if opts.words then - table.insert(out_elts[fname], table.concat(part:get_words(), ' ')) + local howw = opts['words_format'] or 'stem' + table.insert(out_elts[fname], print_words(part:get_words(howw), + howw == 'full')) else if opts.structure then local hc = part:get_html() @@ -515,6 +583,298 @@ local function urls_handler(opts) print_elts(out_elts, opts) end +local function modify_handler(opts) + local rspamd_util = require "rspamd_util" + load_config(opts) + rspamd_url.init(rspamd_config:get_tld_path()) + + local function newline(task) + local t = task:get_newlines_type() + + if t == 'cr' then + return '\r' + elseif t == 'lf' then + return '\n' + end + + return '\r\n' + end + + local function read_file(file) + local f = assert(io.open(file, "rb")) + local content = f:read("*all") + f:close() + return content + end + + local function do_append_footer(task, part, footer, is_multipart) + local newline_s = newline(task) + local tp = part:get_text() + local ct = 'text/plain' + local cte = 'quoted-printable' + + if tp:is_html() then + ct = 'text/html' + end + + if part:get_cte() == '7bit' then + cte = '7bit' + end + + if is_multipart then + io.write(string.format('Content-Type: %s; charset=utf-8%s'.. + 'Content-Transfer-Encoding: %s%s%s', + ct, newline_s, cte, newline_s, newline_s)) + io.flush() + end + + local content = tostring(tp:get_content('raw_utf') or '') + local double_nline = newline_s .. newline_s + local nlen = #double_nline + -- Hack, if part ends with 2 newline, then we append it after footer + if content:sub(-(nlen), nlen + 1) == double_nline then + content = string.format('%s%s', + content:sub(-(#newline_s), #newline_s + 1), -- content without last newline + footer) + rspamd_util.encode_qp(content, + 80, task:get_newlines_type()):save_in_file(1) + io.write(double_nline) + else + content = content .. footer + rspamd_util.encode_qp(content, + 80, task:get_newlines_type()):save_in_file(1) + io.write(double_nline) + end + + + end + + local text_footer, html_footer + + if opts['text_footer'] then + text_footer = read_file(opts['text_footer']) + end + + if opts['html_footer'] then + html_footer = read_file(opts['html_footer']) + end + + for _,fname in ipairs(opts.file) do + local task = load_task(opts, fname) + local newline_s = newline(task) + local need_rewrite_ct = false + local parsed_ct + local seen_cte = false + + local function process_headers_cb(name, hdr) + + for _,h in ipairs(opts['remove_header']) do + if name:match(h) then + return + end + end + + for _,h in ipairs(opts['rewrite_header']) do + local hname,hpattern = h:match('^([^=]+)=(.+)$') + if hname == name then + local new_value = string.format(hpattern, hdr.decoded) + new_value = string.format('%s:%s%s%s', + name, hdr.separator, + rspamd_util.fold_header(name, + rspamd_util.mime_header_encode(new_value), + task:get_newlines_type()), newline_s) + io.write(new_value) + return + end + end + + if need_rewrite_ct then + if name:lower() == 'content-type' then + local nct = string.format('%s: %s/%s; charset=utf-8%s', + 'Content-Type', parsed_ct.type, parsed_ct.subtype, newline_s) + io.write(nct) + return + elseif name:lower() == 'content-transfer-encoding' then + seen_cte = true + io.write(string.format('%s: %s%s', + 'Content-Transfer-Encoding', 'quoted-printable', newline_s)) + return + end + end + + io.write(hdr.raw) + end + + if html_footer or text_footer then + -- We need to take extra care about content-type and cte + local ct = task:get_header('Content-Type') + if ct then + ct = rspamd_util.parse_content_type(ct, task:get_mempool()) + end + + if ct then + if ct.type and ct.type == 'text' then + if ct.subtype then + if html_footer and (ct.subtype == 'html' or ct.subtype == 'htm') then + need_rewrite_ct = true + elseif text_footer and ct.subtype == 'plain' then + need_rewrite_ct = true + end + else + if text_footer then + need_rewrite_ct = true + end + end + + parsed_ct = ct + end + else + local text_parts = task:get_text_parts() + if text_parts then + + if #text_parts == 1 then + need_rewrite_ct = true + parsed_ct = { + type = 'text', + subtype = 'plain' + } + elseif #text_parts > 1 then + -- XXX: in fact, it cannot be + parsed_ct = { + type = 'multipart', + subtype = 'mixed' + } + end + end + end + end + + task:headers_foreach(process_headers_cb, {full = true}) + + for _,h in ipairs(opts['add_header']) do + local hname,hvalue = h:match('^([^=]+)=(.+)$') + + if hname and hvalue then + io.write(string.format('%s: %s%s', hname, + rspamd_util.fold_header(hname, hvalue, task:get_newlines_type()), + newline_s)) + end + end + + if not seen_cte and need_rewrite_ct then + io.write(string.format('%s: %s%s', + 'Content-Transfer-Encoding', 'quoted-printable', newline_s)) + end + + -- End of headers + io.write(newline_s) + + local boundaries = {} + local cur_boundary + + for _,part in ipairs(task:get_parts()) do + local boundary = part:get_boundary() + if part:is_multipart() then + if cur_boundary then + io.write(string.format('--%s%s', + boundaries[#boundaries], newline_s)) + end + + boundaries[#boundaries + 1] = boundary or '--XXX' + cur_boundary = boundary + io.flush () + + local rh = part:get_raw_headers() + if #rh > 0 then + rh:save_in_file(1) + io.write(newline_s) + io.flush() + end + elseif part:is_message() then + if boundary then + if cur_boundary and boundary ~= cur_boundary then + -- Need to close boundary + io.write(string.format('--%s--%s%s', + boundaries[#boundaries], newline_s, newline_s)) + table.remove(boundaries) + cur_boundary = nil + end + io.write(string.format('--%s%s', + boundary, newline_s)) + end + + io.flush() + part:get_raw_headers():save_in_file(1) + io.write(newline_s) + else + local append_footer = false + local skip_footer = part:is_attachment() + + local parent = part:get_parent() + if parent then + local t,st = parent:get_type() + + if t == 'multipart' and st == 'signed' then + -- Do not modify signed parts + skip_footer = true + end + end + if text_footer and part:is_text() then + local tp = part:get_text() + + if not tp:is_html() then + append_footer = text_footer + end + end + + if html_footer and part:is_text() then + local tp = part:get_text() + + if tp:is_html() then + append_footer = html_footer + end + end + + if boundary then + if cur_boundary and boundary ~= cur_boundary then + -- Need to close boundary + io.write(string.format('--%s--%s%s', + boundaries[#boundaries], newline_s, newline_s)) + table.remove(boundaries) + cur_boundary = boundary + end + io.write(string.format('--%s%s', + boundary, newline_s)) + end + + io.flush() + + if append_footer and not skip_footer then + do_append_footer(task, part, append_footer, + parent and parent:is_multipart()) + else + part:get_raw_headers():save_in_file(1) + io.write(newline_s) + io.flush() + part:get_raw_content():save_in_file(1) + end + end + end + + -- Close remaining + local b = table.remove(boundaries) + while b do + io.write(string.format('--%s--%s', b, newline_s)) + if #boundaries > 0 then + io.write(newline_s) + end + b = table.remove(boundaries) + end + + task:destroy() -- No automatic dtor + end +end + local function handler(args) local opts = parser:parse(args) @@ -532,6 +892,8 @@ local function handler(args) stat_handler(opts) elseif command == 'urls' then urls_handler(opts) + elseif command == 'modify' then + modify_handler(opts) else parser:error('command %s is not implemented', command) end diff --git a/package.json b/package.json index 14ee2bd7b..75e42eff7 100644 --- a/package.json +++ b/package.json @@ -2,5 +2,8 @@ "devDependencies": { "eslint": "*" }, - "eslintIgnore": ["*.min.js", "interface/js/lib/domReady.js"] + "eslintIgnore": [ + "*.min.js", + "interface/js/lib/domReady.js" + ] } diff --git a/rules/regexp/headers.lua b/rules/regexp/headers.lua index bd286b764..a8c6d5a85 100644 --- a/rules/regexp/headers.lua +++ b/rules/regexp/headers.lua @@ -490,7 +490,7 @@ reconf['FORGED_MUA_SEAMONKEY_MSGID'] = { group = 'mua' } reconf['FORGED_MUA_SEAMONKEY_MSGID_UNKNOWN'] = { - re = string.format('(%s) & !(%s) & !(%s) & !(%s)', user_agent_seamonkey, mozilla_msgid_common, mozilla_msgid, unusable_msgid), + re = string.format('(%s) & !((%s) | (%s)) & !(%s) & !(%s)', user_agent_seamonkey, mozilla_msgid_common, mozilla_msgid_common_sec, mozilla_msgid, unusable_msgid), score = 2.5, description = 'Forged mail pretending to be from Mozilla Seamonkey but has forged Message-ID', group = 'mua' diff --git a/rules/regexp/misc.lua b/rules/regexp/misc.lua index f63526a8e..3a78ec969 100644 --- a/rules/regexp/misc.lua +++ b/rules/regexp/misc.lua @@ -51,3 +51,25 @@ reconf['INTRODUCTION'] = { group = 'scams' } +-- Message contains a link to a .onion URI (Tor hidden service) +local onion_uri_v2 = '/[a-z0-9]{16}\\.onion?/{url}i' +local onion_uri_v3 = '/[a-z0-9]{56}\\.onion?/{url}i' +reconf['HAS_ONION_URI'] = { + re = string.format('(%s | %s)', onion_uri_v2, onion_uri_v3), + description = 'Contains .onion hidden service URI', + score = 0.0, + group = 'experimental' +} + +local password_in_words = [[/^password/i{words}]] +local btc_wallet_address = [[/^[13][0-9a-zA-Z]{25,34}$/{words}]] +local wallet_word = [[/^wallet$/i{words}]] +local broken_unicode = [[has_flag(bad_unicode)]] + +reconf['LEAKED_PASSWORD_SCAM'] = { + re = string.format('%s & %s & (%s | %s)', + password_in_words, btc_wallet_address, wallet_word, broken_unicode), + description = 'Contains password word and BTC wallet address', + score = 7.0, + group = 'scams' +}
\ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0f9cbd668..ff7198270 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -62,7 +62,20 @@ ENDMACRO(AddModules MLIST WLIST) # Rspamd core components IF (ENABLE_CLANG_PLUGIN MATCHES "ON") - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xclang -load -Xclang ${CMAKE_CURRENT_BINARY_DIR}/../clang-plugin/librspamd-clang.so -Xclang -add-plugin -Xclang rspamd-ast") + SET(CMAKE_C_FLAGS + "${CMAKE_C_FLAGS} -Xclang -load -Xclang ${CMAKE_CURRENT_BINARY_DIR}/../clang-plugin/librspamd-clang.so -Xclang -add-plugin -Xclang rspamd-ast") + IF(CLANG_EXTRA_PLUGINS_LIBS) + FOREACH(_lib ${CLANG_EXTRA_PLUGINS_LIBS}) + SET(CMAKE_C_FLAGS + "${CMAKE_C_FLAGS} -Xclang -load -Xclang ${_lib}") + ENDFOREACH() + ENDIF() + IF(CLANG_EXTRA_PLUGINS) + FOREACH(_plug ${CLANG_EXTRA_PLUGINS}) + SET(CMAKE_C_FLAGS + "${CMAKE_C_FLAGS} -Xclang -add-plugin -Xclang ${_plug}") + ENDFOREACH() + ENDIF() ENDIF () ADD_SUBDIRECTORY(lua) diff --git a/src/client/rspamc.c b/src/client/rspamc.c index 072aa583d..c52f615dc 100644 --- a/src/client/rspamc.c +++ b/src/client/rspamc.c @@ -68,6 +68,8 @@ static gchar *user_agent = "rspamc"; static GList *children; static GPatternSpec **exclude_compiled = NULL; +static gint retcode = EXIT_SUCCESS; + #define ADD_CLIENT_HEADER(o, n, v) do { \ struct rspamd_http_client_header *nh; \ nh = g_malloc (sizeof (*nh)); \ @@ -1365,7 +1367,7 @@ rspamc_mime_output (FILE *out, ucl_object_t *result, GString *input, symbuf->str, 0, nl_type, ","); rspamd_printf_gstring (added_headers, "X-Spam-Symbols: %v%s", - folded_symbuf, line_end, ","); + folded_symbuf, line_end); g_string_free (folded_symbuf, TRUE); g_string_free (symbuf, TRUE); @@ -1588,6 +1590,10 @@ rspamc_client_cb (struct rspamd_client_connection *conn, rspamd_client_destroy (conn); g_free (cbdata->filename); g_free (cbdata); + + if (err) { + retcode = EXIT_FAILURE; + } } static void @@ -2017,5 +2023,6 @@ main (gint argc, gchar **argv, gchar **env) g_pattern_spec_free (exclude_compiled[i]); } - return ret; + /* Mix retcode (return from Rspamd side) and ret (return from subprocess) */ + return ret | retcode; } diff --git a/src/client/rspamdclient.c b/src/client/rspamdclient.c index 490e5c369..a4a1fb95e 100644 --- a/src/client/rspamdclient.c +++ b/src/client/rspamdclient.c @@ -124,7 +124,7 @@ rspamd_client_finish_handler (struct rspamd_http_connection *conn, return 0; } else { - if (rspamd_http_message_get_body (msg, NULL) == NULL || msg->code != 200) { + if (rspamd_http_message_get_body (msg, NULL) == NULL || msg->code / 100 != 2) { err = g_error_new (RCLIENT_ERROR, msg->code, "HTTP error: %d, %.*s", msg->code, (gint)msg->status->len, msg->status->str); diff --git a/src/controller.c b/src/controller.c index 88d1d0ed6..b8eface74 100644 --- a/src/controller.c +++ b/src/controller.c @@ -23,6 +23,7 @@ #include "libstat/stat_api.h" #include "rspamd.h" #include "libserver/worker_util.h" +#include "worker_private.h" #include "lua/lua_common.h" #include "cryptobox.h" #include "ottery.h" @@ -182,6 +183,7 @@ struct rspamd_controller_worker_ctx { struct rspamd_rrd_file *rrd; struct event save_stats_event; struct rspamd_lang_detector *lang_det; + gdouble task_timeout; }; struct rspamd_controller_plugin_cbdata { @@ -666,6 +668,14 @@ rspamd_controller_check_password (struct rspamd_http_connection_entry *entry, } else { check_enable = FALSE; + + if (check_normal) { + /* + * If no enable password is specified use normal password as + * enable password + */ + session->is_enable = TRUE; + } } } } @@ -814,7 +824,7 @@ rspamd_controller_handle_symbols (struct rspamd_http_connection_entry *conn_ent, "description", 0, false); } - if (rspamd_symbols_cache_stat_symbol (session->ctx->cfg->cache, + if (rspamd_symcache_stat_symbol (session->ctx->cfg->cache, sym->name, &freq, &freq_dev, &tm, NULL)) { ucl_object_insert_key (sym_obj, ucl_object_fromdouble (freq), @@ -1514,10 +1524,9 @@ rspamd_controller_handle_lua_history (lua_State *L, if (lua_isfunction (L, -1)) { task = rspamd_task_new (session->ctx->worker, session->cfg, - session->pool, ctx->lang_det); + session->pool, ctx->lang_det, ctx->ev_base); task->resolver = ctx->resolver; - task->ev_base = ctx->ev_base; task->s = rspamd_session_create (session->pool, rspamd_controller_history_lua_fin_task, NULL, @@ -1815,11 +1824,9 @@ rspamd_controller_handle_lua (struct rspamd_http_connection_entry *conn_ent, } task = rspamd_task_new (session->ctx->worker, session->cfg, session->pool, - ctx->lang_det); + ctx->lang_det, ctx->ev_base); task->resolver = ctx->resolver; - task->ev_base = ctx->ev_base; - task->s = rspamd_session_create (session->pool, rspamd_controller_lua_fin_task, NULL, @@ -2000,12 +2007,9 @@ rspamd_controller_handle_learn_common ( } task = rspamd_task_new (session->ctx->worker, session->cfg, session->pool, - session->ctx->lang_det); + session->ctx->lang_det, ctx->ev_base); task->resolver = ctx->resolver; - task->ev_base = ctx->ev_base; - - task->s = rspamd_session_create (session->pool, rspamd_controller_learn_fin_task, NULL, @@ -2101,12 +2105,9 @@ rspamd_controller_handle_scan (struct rspamd_http_connection_entry *conn_ent, } task = rspamd_task_new (session->ctx->worker, session->cfg, session->pool, - ctx->lang_det); - task->ev_base = session->ctx->ev_base; + ctx->lang_det, ctx->ev_base); task->resolver = ctx->resolver; - task->ev_base = ctx->ev_base; - task->s = rspamd_session_create (session->pool, rspamd_controller_check_fin_task, NULL, @@ -2130,6 +2131,16 @@ rspamd_controller_handle_scan (struct rspamd_http_connection_entry *conn_ent, goto end; } + if (ctx->task_timeout > 0.0) { + struct timeval task_tv; + + event_set (&task->timeout_ev, -1, EV_TIMEOUT, rspamd_task_timeout, + task); + event_base_set (ctx->ev_base, &task->timeout_ev); + double_to_tv (ctx->task_timeout, &task_tv); + event_add (&task->timeout_ev, &task_tv); + } + end: session->task = task; rspamd_session_pending (task->s); @@ -2592,9 +2603,8 @@ rspamd_controller_handle_stat_common ( ctx = session->ctx; task = rspamd_task_new (session->ctx->worker, session->cfg, session->pool, - ctx->lang_det); + ctx->lang_det, ctx->ev_base); task->resolver = ctx->resolver; - task->ev_base = ctx->ev_base; cbdata = rspamd_mempool_alloc0 (session->pool, sizeof (*cbdata)); cbdata->conn_ent = conn_ent; cbdata->task = task; @@ -2735,7 +2745,7 @@ rspamd_controller_handle_counters ( { struct rspamd_controller_session *session = conn_ent->ud; ucl_object_t *top; - struct symbols_cache *cache; + struct rspamd_symcache *cache; if (!rspamd_controller_check_password (conn_ent, session, msg, FALSE)) { return 0; @@ -2744,7 +2754,7 @@ rspamd_controller_handle_counters ( cache = session->ctx->cfg->cache; if (cache != NULL) { - top = rspamd_symbols_cache_counters (cache); + top = rspamd_symcache_counters (cache); rspamd_controller_send_ucl (conn_ent, top); ucl_object_unref (top); } @@ -2762,13 +2772,32 @@ rspamd_controller_handle_custom (struct rspamd_http_connection_entry *conn_ent, struct rspamd_controller_session *session = conn_ent->ud; struct rspamd_custom_controller_command *cmd; gchar *url_str; + struct http_parser_url u; + rspamd_ftok_t lookup; - url_str = rspamd_fstring_cstr (msg->url); + http_parser_parse_url (msg->url->str, msg->url->len, TRUE, &u); + + if (u.field_set & (1 << UF_PATH)) { + guint unnorm_len; + lookup.begin = msg->url->str + u.field_data[UF_PATH].off; + lookup.len = u.field_data[UF_PATH].len; + + rspamd_http_normalize_path_inplace ((gchar *)lookup.begin, + lookup.len, + &unnorm_len); + lookup.len = unnorm_len; + } + else { + lookup.begin = msg->url->str; + lookup.len = msg->url->len; + } + + url_str = rspamd_ftok_cstr (&lookup); cmd = g_hash_table_lookup (session->ctx->custom_commands, url_str); g_free (url_str); if (cmd == NULL || cmd->handler == NULL) { - msg_err_session ("custom command %V has not been found", msg->url); + msg_err_session ("custom command %T has not been found", &lookup); rspamd_controller_send_error (conn_ent, 404, "No command associated"); return 0; } @@ -2785,6 +2814,33 @@ rspamd_controller_handle_custom (struct rspamd_http_connection_entry *conn_ent, return 0; } + /* Transfer query arguments to headers */ + if (u.field_set & (1u << UF_QUERY)) { + GHashTable *query_args; + GHashTableIter it; + gpointer k, v; + rspamd_ftok_t *key, *value; + + /* In case if we have a query, we need to store it somewhere */ + query_args = rspamd_http_message_parse_query (msg); + + /* Insert the rest of query params as HTTP headers */ + g_hash_table_iter_init (&it, query_args); + + while (g_hash_table_iter_next (&it, &k, &v)) { + key = k; + value = v; + /* Steal strings */ + g_hash_table_iter_steal (&it); + url_str = rspamd_ftok_cstr (key); + rspamd_http_message_add_header_len (msg, url_str, + value->begin, value->len); + g_free (url_str); + } + + g_hash_table_unref (query_args); + } + return cmd->handler (conn_ent, msg, cmd->ctx); } @@ -2955,11 +3011,9 @@ rspamd_controller_handle_lua_plugin (struct rspamd_http_connection_entry *conn_e } task = rspamd_task_new (session->ctx->worker, session->cfg, session->pool, - ctx->lang_det); + ctx->lang_det, ctx->ev_base); task->resolver = ctx->resolver; - task->ev_base = ctx->ev_base; - task->s = rspamd_session_create (session->pool, rspamd_controller_lua_fin_task, NULL, @@ -3280,6 +3334,7 @@ init_controller_worker (struct rspamd_config *cfg) ctx->magic = rspamd_controller_ctx_magic; ctx->timeout = DEFAULT_WORKER_IO_TIMEOUT; + ctx->task_timeout = NAN; rspamd_rcl_register_worker_option (cfg, type, @@ -3384,6 +3439,16 @@ init_controller_worker (struct rspamd_config *cfg) 0, "Directory where controller saves server's statistics between restarts"); + rspamd_rcl_register_worker_option (cfg, + type, + "task_timeout", + rspamd_rcl_parse_struct_time, + ctx, + G_STRUCT_OFFSET (struct rspamd_controller_worker_ctx, + task_timeout), + RSPAMD_CL_FLAG_TIME_FLOAT, + "Maximum task processing time, default: 8.0 seconds"); + return ctx; } @@ -3662,6 +3727,15 @@ start_controller_worker (struct rspamd_worker *worker) rspamd_strcase_equal, g_free, rspamd_plugin_cbdata_dtor); + if (isnan (ctx->task_timeout)) { + if (isnan (ctx->cfg->task_timeout)) { + ctx->task_timeout = 0; + } + else { + ctx->task_timeout = ctx->cfg->task_timeout; + } + } + if (ctx->secure_ip != NULL) { rspamd_config_radix_from_ucl (ctx->cfg, ctx->secure_ip, "Allow unauthenticated requests from these addresses", @@ -3829,7 +3903,7 @@ start_controller_worker (struct rspamd_worker *worker) rspamd_upstreams_library_config (worker->srv->cfg, worker->srv->cfg->ups_ctx, ctx->ev_base, ctx->resolver->r); - rspamd_symbols_cache_start_refresh (worker->srv->cfg->cache, ctx->ev_base, + rspamd_symcache_start_refresh (worker->srv->cfg->cache, ctx->ev_base, worker); rspamd_stat_init (worker->srv->cfg, ctx->ev_base); @@ -3840,6 +3914,14 @@ start_controller_worker (struct rspamd_worker *worker) rspamd_map_watch (worker->srv->cfg, ctx->ev_base, ctx->resolver, worker, TRUE); + + /* Schedule periodic stats saving, see #1823 */ + event_set (&ctx->save_stats_event, -1, EV_PERSIST, + rspamd_controller_stats_save_periodic, + ctx); + event_base_set (ctx->ev_base, &ctx->save_stats_event); + msec_to_tv (save_stats_interval, &stv); + evtimer_add (&ctx->save_stats_event, &stv); } else { rspamd_map_watch (worker->srv->cfg, ctx->ev_base, @@ -3848,13 +3930,6 @@ start_controller_worker (struct rspamd_worker *worker) rspamd_lua_run_postloads (ctx->cfg->lua_state, ctx->cfg, ctx->ev_base, worker); - /* Schedule periodic stats saving, see #1823 */ - evtimer_set (&ctx->save_stats_event, rspamd_controller_stats_save_periodic, - ctx); - event_base_set (ctx->ev_base, &ctx->save_stats_event); - msec_to_tv (save_stats_interval, &stv); - evtimer_add (&ctx->save_stats_event, &stv); - /* Start event loop */ event_base_loop (ctx->ev_base, 0); rspamd_worker_block_signals (); diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c index a3486635f..36b41113d 100644 --- a/src/fuzzy_storage.c +++ b/src/fuzzy_storage.c @@ -17,8 +17,8 @@ * Rspamd fuzzy storage server */ -#include <src/libserver/fuzzy_wire.h> #include "config.h" +#include "libserver/fuzzy_wire.h" #include "util.h" #include "rspamd.h" #include "map.h" @@ -37,14 +37,20 @@ #include "libutil/map_private.h" #include "libutil/hash.h" #include "libutil/http_private.h" +#include "libutil/hash.h" #include "unix-std.h" +#include <math.h> + /* Resync value in seconds */ #define DEFAULT_SYNC_TIMEOUT 60.0 #define DEFAULT_KEYPAIR_CACHE_SIZE 512 #define DEFAULT_MASTER_TIMEOUT 10.0 #define DEFAULT_UPDATES_MAXFAIL 3 #define COOKIE_SIZE 128 +#define DEFAULT_MAX_BUCKETS 2000 +#define DEFAULT_BUCKET_TTL 3600 +#define DEFAULT_BUCKET_MASK 24 static const gchar *local_db_name = "local"; @@ -115,6 +121,12 @@ struct rspamd_fuzzy_mirror { struct rspamd_cryptobox_pubkey *key; }; +struct rspamd_leaky_bucket_elt { + rspamd_inet_addr_t *addr; + gdouble last; + gdouble cur; +}; + static const guint64 rspamd_fuzzy_storage_magic = 0x291a3253eb1b3ea5ULL; struct rspamd_fuzzy_storage_ctx { @@ -132,6 +144,7 @@ struct rspamd_fuzzy_storage_ctx { struct rspamd_radix_map_helper *update_ips; struct rspamd_radix_map_helper *master_ips; struct rspamd_radix_map_helper *blocked_ips; + struct rspamd_radix_map_helper *ratelimit_whitelist; struct rspamd_cryptobox_keypair *sync_keypair; struct rspamd_cryptobox_pubkey *master_key; @@ -141,6 +154,7 @@ struct rspamd_fuzzy_storage_ctx { const ucl_object_t *update_map; const ucl_object_t *masters_map; const ucl_object_t *blocked_map; + const ucl_object_t *ratelimit_whitelist_map; GHashTable *master_flags; guint keypair_cache_size; @@ -148,6 +162,7 @@ struct rspamd_fuzzy_storage_ctx { struct event peer_ev; struct event stat_ev; struct timeval stat_tv; + /* Local keypair */ struct rspamd_cryptobox_keypair *default_keypair; /* Bad clash, need for parse keypair */ struct fuzzy_key *default_key; @@ -160,11 +175,21 @@ struct rspamd_fuzzy_storage_ctx { gchar *collection_id_file; struct rspamd_keypair_cache *keypair_cache; rspamd_lru_hash_t *errors_ips; + rspamd_lru_hash_t *ratelimit_buckets; struct rspamd_fuzzy_backend *backend; GArray *updates_pending; guint updates_failed; guint updates_maxfail; guint32 collection_id; + + /* Ratelimits */ + guint leaky_bucket_ttl; + guint leaky_bucket_mask; + guint max_buckets; + gboolean ratelimit_log_only; + gdouble leaky_bucket_burst; + gdouble leaky_bucket_rate; + struct rspamd_worker *worker; struct rspamd_http_connection_router *collection_rt; const ucl_object_t *skip_map; @@ -231,6 +256,105 @@ struct fuzzy_master_update_session { static void rspamd_fuzzy_write_reply (struct fuzzy_session *session); static gboolean +rspamd_fuzzy_check_ratelimit (struct fuzzy_session *session) +{ + rspamd_inet_addr_t *masked; + struct rspamd_leaky_bucket_elt *elt; + struct timeval tv; + gdouble now; + + if (session->ctx->ratelimit_whitelist != NULL) { + if (rspamd_match_radix_map_addr (session->ctx->ratelimit_whitelist, + session->addr) != NULL) { + return TRUE; + } + } + + /* + if (rspamd_inet_address_is_local (session->addr, TRUE)) { + return TRUE; + } + */ + + masked = rspamd_inet_address_copy (session->addr); + + if (rspamd_inet_address_get_af (masked) == AF_INET) { + rspamd_inet_address_apply_mask (masked, + MIN (session->ctx->leaky_bucket_mask, 32)); + } + else { + /* Must be at least /64 */ + rspamd_inet_address_apply_mask (masked, + MIN (MAX (session->ctx->leaky_bucket_mask * 4, 64), 128)); + } + +#ifdef HAVE_EVENT_NO_CACHE_TIME_FUNC + event_base_gettimeofday_cached (session->ctx->ev_base, &tv); +#else + gettimeofday (&tv, NULL); +#endif + + now = tv_to_double (&tv); + elt = rspamd_lru_hash_lookup (session->ctx->ratelimit_buckets, masked, + tv.tv_sec); + + if (elt) { + gboolean ratelimited = FALSE; + + if (isnan (elt->cur)) { + /* Ratelimit exceeded, preserve it for the whole ttl */ + ratelimited = TRUE; + } + else { + /* Update bucket */ + if (elt->last < now) { + elt->cur -= session->ctx->leaky_bucket_rate * (now - elt->last); + elt->last = now; + + if (elt->cur < 0) { + elt->cur = 0; + } + } + else { + elt->last = now; + } + + /* Check bucket */ + if (elt->cur >= session->ctx->leaky_bucket_burst) { + + msg_info ("ratelimiting %s (%s), %.1f max elts", + rspamd_inet_address_to_string (session->addr), + rspamd_inet_address_to_string (masked), + session->ctx->leaky_bucket_burst); + elt->cur = NAN; + } + else { + elt->cur ++; /* Allow one more request */ + } + } + + rspamd_inet_address_free (masked); + + return !ratelimited; + } + else { + /* New bucket */ + elt = g_malloc (sizeof (*elt)); + elt->addr = masked; /* transfer ownership */ + elt->cur = 1; + elt->last = now; + + rspamd_lru_hash_insert (session->ctx->ratelimit_buckets, + masked, + elt, + tv.tv_sec, + session->ctx->leaky_bucket_ttl); + } + + return TRUE; +} + +static gboolean rspamd_fuzzy_check_client (struct fuzzy_session *session, gboolean is_write) { if (session->ctx->blocked_ips != NULL) { @@ -259,6 +383,15 @@ rspamd_fuzzy_check_client (struct fuzzy_session *session, gboolean is_write) } /* Non write */ + if (session->ctx->ratelimit_buckets) { + if (session->ctx->ratelimit_log_only) { + (void)rspamd_fuzzy_check_ratelimit (session); /* Check but ignore */ + } + else { + return rspamd_fuzzy_check_ratelimit (session); + } + } + return TRUE; } @@ -300,6 +433,15 @@ struct fuzzy_slave_connection { }; static void +fuzzy_rl_bucket_free (gpointer p) +{ + struct rspamd_leaky_bucket_elt *elt = (struct rspamd_leaky_bucket_elt *)p; + + rspamd_inet_address_free (elt->addr); + g_free (elt); +} + +static void fuzzy_mirror_close_connection (struct fuzzy_slave_connection *conn) { if (conn) { @@ -974,6 +1116,8 @@ rspamd_fuzzy_process_command (struct fuzzy_session *session) else { result.v1.value = 403; result.v1.prob = 0.0; + result.v1.flag = 0; + rspamd_fuzzy_make_reply (cmd, &result, session, encrypted, is_shingle); } } else if (cmd->cmd == FUZZY_STAT) { @@ -2057,11 +2201,9 @@ rspamd_fuzzy_storage_stat_key (struct fuzzy_key_stat *key_stat) static ucl_object_t * rspamd_fuzzy_stat_to_ucl (struct rspamd_fuzzy_storage_ctx *ctx, gboolean ip_stat) { - GHashTableIter it, ip_it; - GHashTable *ip_hash; struct fuzzy_key_stat *key_stat; + GHashTableIter it; struct fuzzy_key *key; - rspamd_lru_element_t *lru_elt; ucl_object_t *obj, *keys_obj, *elt, *ip_elt, *ip_cur; gpointer k, v; gint i; @@ -2082,22 +2224,18 @@ rspamd_fuzzy_stat_to_ucl (struct rspamd_fuzzy_storage_ctx *ctx, gboolean ip_stat elt = rspamd_fuzzy_storage_stat_key (key_stat); if (key_stat->last_ips && ip_stat) { - ip_hash = rspamd_lru_hash_get_htable (key_stat->last_ips); - - if (ip_hash) { - g_hash_table_iter_init (&ip_it, ip_hash); - ip_elt = ucl_object_typed_new (UCL_OBJECT); - - while (g_hash_table_iter_next (&ip_it, &k, &v)) { - lru_elt = v; - ip_cur = rspamd_fuzzy_storage_stat_key ( - rspamd_lru_hash_element_data (lru_elt)); - ucl_object_insert_key (ip_elt, ip_cur, - rspamd_inet_address_to_string (k), 0, true); - } + i = 0; + + ip_elt = ucl_object_typed_new (UCL_OBJECT); - ucl_object_insert_key (elt, ip_elt, "ips", 0, false); + while ((i = rspamd_lru_hash_foreach (key_stat->last_ips, + i, &k, &v)) != -1) { + ip_cur = rspamd_fuzzy_storage_stat_key (v); + ucl_object_insert_key (ip_elt, ip_cur, + rspamd_inet_address_to_string (k), 0, true); } + + ucl_object_insert_key (elt, ip_elt, "ips", 0, false); } ucl_object_insert_key (keys_obj, elt, keyname, 0, true); @@ -2124,27 +2262,21 @@ rspamd_fuzzy_stat_to_ucl (struct rspamd_fuzzy_storage_ctx *ctx, gboolean ip_stat false); if (ctx->errors_ips && ip_stat) { - ip_hash = rspamd_lru_hash_get_htable (ctx->errors_ips); - - if (ip_hash) { - g_hash_table_iter_init (&ip_it, ip_hash); - ip_elt = ucl_object_typed_new (UCL_OBJECT); + i = 0; - while (g_hash_table_iter_next (&ip_it, &k, &v)) { - lru_elt = v; + ip_elt = ucl_object_typed_new (UCL_OBJECT); - ucl_object_insert_key (ip_elt, - ucl_object_fromint (*(guint64 *) - rspamd_lru_hash_element_data (lru_elt)), - rspamd_inet_address_to_string (k), 0, true); - } - - ucl_object_insert_key (obj, - ip_elt, - "errors_ips", - 0, - false); + while ((i = rspamd_lru_hash_foreach (ctx->errors_ips, i, &k, &v)) != -1) { + ucl_object_insert_key (ip_elt, + ucl_object_fromint (*(guint64 *)v), + rspamd_inet_address_to_string (k), 0, true); } + + ucl_object_insert_key (obj, + ip_elt, + "errors_ips", + 0, + false); } /* Checked by epoch */ @@ -2492,6 +2624,11 @@ init_fuzzy (struct rspamd_config *cfg) (rspamd_mempool_destruct_t)rspamd_ptr_array_free_hard, ctx->mirrors); ctx->updates_maxfail = DEFAULT_UPDATES_MAXFAIL; ctx->collection_id_file = RSPAMD_DBDIR "/fuzzy_collection.id"; + ctx->leaky_bucket_mask = DEFAULT_BUCKET_MASK; + ctx->leaky_bucket_ttl = DEFAULT_BUCKET_TTL; + ctx->max_buckets = DEFAULT_MAX_BUCKETS; + ctx->leaky_bucket_burst = NAN; + ctx->leaky_bucket_rate = NAN; rspamd_rcl_register_worker_option (cfg, type, @@ -2682,6 +2819,65 @@ init_fuzzy (struct rspamd_config *cfg) 0, "Skip specific hashes from the map"); + /* Ratelimits */ + rspamd_rcl_register_worker_option (cfg, + type, + "ratelimit_whitelist", + rspamd_rcl_parse_struct_ucl, + ctx, + G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, ratelimit_whitelist_map), + 0, + "Skip specific addresses from rate limiting"); + rspamd_rcl_register_worker_option (cfg, + type, + "ratelimit_max_buckets", + rspamd_rcl_parse_struct_integer, + ctx, + G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, max_buckets), + RSPAMD_CL_FLAG_UINT, + "Maximum number of leaky buckets (default: " G_STRINGIFY(DEFAULT_MAX_BUCKETS) ")"); + rspamd_rcl_register_worker_option (cfg, + type, + "ratelimit_network_mask", + rspamd_rcl_parse_struct_integer, + ctx, + G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, leaky_bucket_mask), + RSPAMD_CL_FLAG_UINT, + "Network mask to apply for IPv4 rate addresses (default: " G_STRINGIFY(DEFAULT_BUCKET_MASK) ")"); + rspamd_rcl_register_worker_option (cfg, + type, + "ratelimit_bucket_ttl", + rspamd_rcl_parse_struct_time, + ctx, + G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, leaky_bucket_ttl), + RSPAMD_CL_FLAG_TIME_INTEGER, + "Time to live for ratelimit element (default: " G_STRINGIFY(DEFAULT_BUCKET_TTL) ")"); + rspamd_rcl_register_worker_option (cfg, + type, + "ratelimit_rate", + rspamd_rcl_parse_struct_double, + ctx, + G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, leaky_bucket_rate), + 0, + "Leak rate in requests per second"); + rspamd_rcl_register_worker_option (cfg, + type, + "ratelimit_burst", + rspamd_rcl_parse_struct_double, + ctx, + G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, leaky_bucket_burst), + 0, + "Peak value for ratelimit bucket"); + rspamd_rcl_register_worker_option (cfg, + type, + "ratelimit_log_only", + rspamd_rcl_parse_struct_boolean, + ctx, + G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, ratelimit_log_only), + 0, + "Don't really ban on ratelimit reaching, just log"); + + return ctx; } @@ -2959,6 +3155,20 @@ start_fuzzy (struct rspamd_worker *worker) &ctx->blocked_ips, NULL); } + /* Create radix trees */ + if (ctx->ratelimit_whitelist_map != NULL) { + rspamd_config_radix_from_ucl (worker->srv->cfg, ctx->ratelimit_whitelist_map, + "Skip ratelimits from specific ip addresses/networks", + &ctx->ratelimit_whitelist, NULL); + } + + /* Ratelimits */ + if (!isnan (ctx->leaky_bucket_rate) && !isnan (ctx->leaky_bucket_burst)) { + ctx->ratelimit_buckets = rspamd_lru_hash_new_full (ctx->max_buckets, + NULL, fuzzy_rl_bucket_free, + rspamd_inet_address_hash, rspamd_inet_address_equal); + } + /* Maps events */ ctx->resolver = dns_resolver_init (worker->srv->logger, ctx->ev_base, diff --git a/src/libcryptobox/base64/ref.c b/src/libcryptobox/base64/ref.c index 797a91ce7..6d3c295f1 100644 --- a/src/libcryptobox/base64/ref.c +++ b/src/libcryptobox/base64/ref.c @@ -214,7 +214,7 @@ repeat: } if (!ret && inlen > 0) { - /* Skip to the next valid character in input */ + /* Skip to the next valid character in lua_dns_resolver_resolve_commoninput */ while (inlen > 0 && base64_table_dec[*c] >= 254) { c ++; inlen --; diff --git a/src/libcryptobox/blake2/blake2.h b/src/libcryptobox/blake2/blake2.h index 9966c66d2..3da1958ae 100644 --- a/src/libcryptobox/blake2/blake2.h +++ b/src/libcryptobox/blake2/blake2.h @@ -20,13 +20,13 @@ extern "C" { #endif -enum blake2b_constant { - BLAKE2B_BLOCKBYTES = 128, - BLAKE2B_OUTBYTES = 64, - BLAKE2B_KEYBYTES = 64, - BLAKE2B_SALTBYTES = 16, - BLAKE2B_PERSONALBYTES = 16 -}; + +#define BLAKE2B_BLOCKBYTES 128 +#define BLAKE2B_OUTBYTES 64 +#define BLAKE2B_KEYBYTES 64 +#define BLAKE2B_SALTBYTES 16 +#define BLAKE2B_PERSONALBYTES 16 + typedef struct blake2b_state_t { unsigned char opaque[256]; diff --git a/src/libcryptobox/chacha20/chacha.h b/src/libcryptobox/chacha20/chacha.h index f69a63db9..7f93a4517 100644 --- a/src/libcryptobox/chacha20/chacha.h +++ b/src/libcryptobox/chacha20/chacha.h @@ -26,9 +26,8 @@ #ifndef CHACHA_H_ #define CHACHA_H_ -enum chacha_constants { - CHACHA_BLOCKBYTES = 64, -}; + +#define CHACHA_BLOCKBYTES 64 typedef struct chacha_state_internal_t { unsigned char s[48]; diff --git a/src/libcryptobox/poly1305/ref-64.c b/src/libcryptobox/poly1305/ref-64.c index 48d5fbcc6..cceb1476d 100644 --- a/src/libcryptobox/poly1305/ref-64.c +++ b/src/libcryptobox/poly1305/ref-64.c @@ -7,9 +7,8 @@ #include "config.h" #include "poly1305.h" #include "poly1305_internal.h" -enum { - POLY1305_BLOCK_SIZE = 16 -}; + +#define POLY1305_BLOCK_SIZE 16 typedef struct poly1305_state_ref_t { uint64_t r[3]; diff --git a/src/libmime/archives.c b/src/libmime/archives.c index 6497f6b35..1f9a5c634 100644 --- a/src/libmime/archives.c +++ b/src/libmime/archives.c @@ -1176,7 +1176,7 @@ rspamd_7zip_read_archive_props (struct rspamd_task *task, struct rspamd_archive *arch) { guchar proptype; - uint64_t proplen; + guint64 proplen; /* * for (;;) @@ -1509,8 +1509,8 @@ rspamd_archive_cheat_detect (struct rspamd_mime_part *part, const gchar *str, } if (magic_start != NULL) { - if (part->parsed_data.len > magic_len && memcmp (part->parsed_data.begin, - magic_start, magic_len) == 0) { + if (part->parsed_data.len > magic_len && + memcmp (part->parsed_data.begin, magic_start, magic_len) == 0) { return TRUE; } } @@ -1531,20 +1531,26 @@ rspamd_archives_process (struct rspamd_task *task) for (i = 0; i < task->parts->len; i ++) { part = g_ptr_array_index (task->parts, i); - if (part->parsed_data.len > 0) { - if (rspamd_archive_cheat_detect (part, "zip", - zip_magic, sizeof (zip_magic))) { - rspamd_archive_process_zip (task, part); - } - else if (rspamd_archive_cheat_detect (part, "rar", - rar_magic, sizeof (rar_magic))) { - rspamd_archive_process_rar (task, part); - } - else if (rspamd_archive_cheat_detect (part, "7z", - sz_magic, sizeof (sz_magic))) { - rspamd_archive_process_7zip (task, part); - } + if (!(part->flags & (RSPAMD_MIME_PART_TEXT|RSPAMD_MIME_PART_IMAGE))) { + if (part->parsed_data.len > 0) { + if (rspamd_archive_cheat_detect (part, "zip", + zip_magic, sizeof (zip_magic))) { + rspamd_archive_process_zip (task, part); + } else if (rspamd_archive_cheat_detect (part, "rar", + rar_magic, sizeof (rar_magic))) { + rspamd_archive_process_rar (task, part); + } else if (rspamd_archive_cheat_detect (part, "7z", + sz_magic, sizeof (sz_magic))) { + rspamd_archive_process_7zip (task, part); + } + if (IS_CT_TEXT (part->ct) && + (part->flags & RSPAMD_MIME_PART_ARCHIVE)) { + msg_info_task ("found archive with incorrect content-type: %T/%T", + &part->ct->type, &part->ct->subtype); + part->ct->flags |= RSPAMD_CONTENT_TYPE_BROKEN; + } + } } } } diff --git a/src/libmime/content_type.c b/src/libmime/content_type.c index 524b6f636..6b99953f2 100644 --- a/src/libmime/content_type.c +++ b/src/libmime/content_type.c @@ -21,43 +21,28 @@ void rspamd_content_type_add_param (rspamd_mempool_t *pool, struct rspamd_content_type *ct, - const gchar *name_start, const gchar *name_end, - const gchar *value_start, const gchar *value_end) + gchar *name_start, gchar *name_end, + gchar *value_start, gchar *value_end) { rspamd_ftok_t srch; struct rspamd_content_type_param *found = NULL, *nparam; g_assert (ct != NULL); - srch.begin = name_start; - srch.len = name_end - name_start; - - if (ct->attrs) { - found = g_hash_table_lookup (ct->attrs, &srch); - } - else { - ct->attrs = g_hash_table_new (rspamd_ftok_icase_hash, - rspamd_ftok_icase_equal); - } nparam = rspamd_mempool_alloc (pool, sizeof (*nparam)); nparam->name.begin = name_start; nparam->name.len = name_end - name_start; + rspamd_str_lc (name_start, name_end - name_start); + nparam->value.begin = value_start; nparam->value.len = value_end - value_start; - if (!found) { - DL_APPEND (found, nparam); - g_hash_table_insert (ct->attrs, &nparam->name, nparam); - } - else { - DL_APPEND (found, nparam); - } - RSPAMD_FTOK_ASSIGN (&srch, "charset"); if (rspamd_ftok_cmp (&nparam->name, &srch) == 0) { /* Adjust charset */ + found = nparam; ct->charset.begin = nparam->value.begin; ct->charset.len = nparam->value.len; } @@ -65,17 +50,47 @@ rspamd_content_type_add_param (rspamd_mempool_t *pool, RSPAMD_FTOK_ASSIGN (&srch, "boundary"); if (rspamd_ftok_cmp (&nparam->name, &srch) == 0) { + found = nparam; + gchar *lc_boundary; /* Adjust boundary */ - ct->boundary.begin = nparam->value.begin; + lc_boundary = rspamd_mempool_alloc (pool, nparam->value.len); + memcpy (lc_boundary, nparam->value.begin, nparam->value.len); + rspamd_str_lc (lc_boundary, nparam->value.len); + ct->boundary.begin = lc_boundary; ct->boundary.len = nparam->value.len; + /* Preserve original (case sensitive) boundary */ + ct->orig_boundary.begin = nparam->value.begin; + ct->orig_boundary.len = nparam->value.len; + } + + if (!found) { + srch.begin = nparam->name.begin; + srch.len = nparam->name.len; + + rspamd_str_lc (value_start, value_end - value_start); + + if (ct->attrs) { + found = g_hash_table_lookup (ct->attrs, &srch); + } else { + ct->attrs = g_hash_table_new (rspamd_ftok_icase_hash, + rspamd_ftok_icase_equal); + } + + if (!found) { + DL_APPEND (found, nparam); + g_hash_table_insert (ct->attrs, &nparam->name, nparam); + } + else { + DL_APPEND (found, nparam); + } } } static struct rspamd_content_type * -rspamd_content_type_parser (const gchar *in, gsize len, rspamd_mempool_t *pool) +rspamd_content_type_parser (gchar *in, gsize len, rspamd_mempool_t *pool) { guint obraces = 0, ebraces = 0, qlen = 0; - const gchar *p, *c, *end, *pname_start = NULL, *pname_end = NULL; + gchar *p, *c, *end, *pname_start = NULL, *pname_end = NULL; struct rspamd_content_type *res = NULL, val; gboolean eqsign_seen = FALSE; enum { @@ -95,7 +110,7 @@ rspamd_content_type_parser (const gchar *in, gsize len, rspamd_mempool_t *pool) c = p; end = p + len; memset (&val, 0, sizeof (val)); - val.lc_data = (gchar *)in; + val.cpy = in; while (p < end) { switch (state) { @@ -346,6 +361,9 @@ rspamd_content_type_parser (const gchar *in, gsize len, rspamd_mempool_t *pool) if (val.type.len > 0) { res = rspamd_mempool_alloc (pool, sizeof (val)); memcpy (res, &val, sizeof (val)); + + /* Lowercase common thingies */ + } return res; @@ -359,9 +377,8 @@ rspamd_content_type_parse (const gchar *in, rspamd_ftok_t srch; gchar *lc_data; - lc_data = rspamd_mempool_alloc (pool, len); - memcpy (lc_data, in, len); - rspamd_str_lc (lc_data, len); + lc_data = rspamd_mempool_alloc (pool, len + 1); + rspamd_strlcpy (lc_data, in, len + 1); if ((res = rspamd_content_type_parser (lc_data, len, pool)) != NULL) { if (res->attrs) { @@ -477,7 +494,8 @@ rspamd_content_disposition_add_param (rspamd_mempool_t *pool, nparam = rspamd_mempool_alloc (pool, sizeof (*nparam)); nparam->name.begin = name_start; nparam->name.len = name_end - name_start; - decoded = rspamd_mime_header_decode (pool, value_start, value_end - value_start); + decoded = rspamd_mime_header_decode (pool, value_start, + value_end - value_start, NULL); RSPAMD_FTOK_FROM_STR (&nparam->value, decoded); if (!found) { diff --git a/src/libmime/content_type.h b/src/libmime/content_type.h index e71077911..554aad6a1 100644 --- a/src/libmime/content_type.h +++ b/src/libmime/content_type.h @@ -41,16 +41,17 @@ struct rspamd_content_type_param { }; struct rspamd_content_type { - gchar *lc_data; + gchar *cpy; rspamd_ftok_t type; rspamd_ftok_t subtype; rspamd_ftok_t charset; rspamd_ftok_t boundary; + rspamd_ftok_t orig_boundary; enum rspamd_content_type_flags flags; GHashTable *attrs; /* Can be empty */ }; -enum rspamd_contetn_disposition_type { +enum rspamd_content_disposition_type { RSPAMD_CT_UNKNOWN = 0, RSPAMD_CT_INLINE = 1, RSPAMD_CT_ATTACHMENT = 2, @@ -58,7 +59,7 @@ enum rspamd_contetn_disposition_type { struct rspamd_content_disposition { gchar *lc_data; - enum rspamd_contetn_disposition_type type; + enum rspamd_content_disposition_type type; rspamd_ftok_t filename; GHashTable *attrs; /* Can be empty */ }; @@ -66,16 +67,16 @@ struct rspamd_content_disposition { /** * Adds new parameter to content type structure * @param ct - * @param name_start + * @param name_start (can be modified) * @param name_end - * @param value_start + * @param value_start (can be modified) * @param value_end */ void rspamd_content_type_add_param (rspamd_mempool_t *pool, - struct rspamd_content_type *ct, - const gchar *name_start, const gchar *name_end, - const gchar *value_start, const gchar *value_end); + struct rspamd_content_type *ct, + gchar *name_start, gchar *name_end, + gchar *value_start, gchar *value_end); /** * Parse content type from the header (performs copy + lowercase) diff --git a/src/libmime/email_addr.c b/src/libmime/email_addr.c index 8376e8f78..9aa2d0618 100644 --- a/src/libmime/email_addr.c +++ b/src/libmime/email_addr.c @@ -137,7 +137,7 @@ rspamd_email_address_add (rspamd_mempool_t *pool, } if (name->len > 0) { - elt->name = rspamd_mime_header_decode (pool, name->str, name->len); + elt->name = rspamd_mime_header_decode (pool, name->str, name->len, NULL); } g_ptr_array_add (ar, elt); @@ -202,6 +202,7 @@ rspamd_email_address_check_and_add (const gchar *start, gsize len, struct rspamd_email_address addr; /* The whole email is likely address */ + memset (&addr, 0, sizeof (addr)); rspamd_smtp_addr_parse (start, len, &addr); if (addr.flags & RSPAMD_EMAIL_ADDR_VALID) { @@ -283,7 +284,7 @@ rspamd_email_address_from_mime (rspamd_mempool_t *pool, state = parse_addr; } else if (*p == ',') { - if (p > c) { + if (p > c && seen_at) { /* * Last token must be the address: * e.g. Some name name@domain.com @@ -485,3 +486,29 @@ rspamd_email_address_list_destroy (gpointer ptr) g_ptr_array_free (ar, TRUE); } + +void rspamd_smtp_maybe_process_smtp_comment (struct rspamd_task *task, + const char *data, size_t len, + struct received_header *rh) +{ + if (!rh->by_hostname) { + /* Heuristic to detect IP addresses like in Exim received: + * [xxx]:port or [xxx] + */ + + if (*data == '[' && len > 2) { + const gchar *p = data + 1; + gsize iplen = rspamd_memcspn (p, "]", len - 1); + + if (iplen > 0) { + guchar tbuf[sizeof(struct in6_addr) + sizeof(guint32)]; + + if (rspamd_parse_inet_address_ip4 (p, iplen, tbuf) || + rspamd_parse_inet_address_ip6 (p, iplen, tbuf)) { + rh->comment_ip = rspamd_mempool_alloc (task->task_pool, iplen + 1); + rspamd_strlcpy (rh->comment_ip, p, iplen + 1); + } + } + } + } +}
\ No newline at end of file diff --git a/src/libmime/email_addr.h b/src/libmime/email_addr.h index 8c9b54713..129d2ba44 100644 --- a/src/libmime/email_addr.h +++ b/src/libmime/email_addr.h @@ -51,6 +51,19 @@ struct rspamd_email_address { guchar flags; }; +struct received_header; +struct rspamd_task; +/** + * Try to parse SMTP comment to process stupid Exim received headers + * @param task + * @param data + * @param len + * @param rh + */ +void rspamd_smtp_maybe_process_smtp_comment (struct rspamd_task *task, + const char *data, size_t len, + struct received_header *rh); + /** * Create email address from a single rfc822 address (e.g. from mail from:) * @param str string to use diff --git a/src/libmime/filter.c b/src/libmime/filter.c index 7d7836d6f..94c9ac223 100644 --- a/src/libmime/filter.c +++ b/src/libmime/filter.c @@ -54,13 +54,10 @@ rspamd_create_metric_result (struct rspamd_task *task) return metric_res; } - metric_res = rspamd_mempool_alloc (task->task_pool, + metric_res = rspamd_mempool_alloc0 (task->task_pool, sizeof (struct rspamd_metric_result)); metric_res->symbols = kh_init (rspamd_symbols_hash); metric_res->sym_groups = kh_init (rspamd_symbols_group_hash); - metric_res->grow_factor = 0; - metric_res->score = 0; - metric_res->passthrough_result = NULL; /* Optimize allocation */ kh_resize (rspamd_symbols_group_hash, metric_res->sym_groups, 4); @@ -155,6 +152,10 @@ rspamd_check_group_score (struct rspamd_task *task, return w; } +#ifndef DBL_EPSILON +#define DBL_EPSILON 2.2204460492503131e-16 +#endif + static struct rspamd_symbol_result * insert_metric_result (struct rspamd_task *task, const gchar *symbol, @@ -364,9 +365,20 @@ insert_metric_result (struct rspamd_task *task, } if (!isnan (final_score)) { + const double epsilon = DBL_EPSILON; + metric_res->score += final_score; metric_res->grow_factor = next_gf; s->score = final_score; + + if (final_score > epsilon) { + metric_res->npositive ++; + metric_res->positive_score += final_score; + } + else if (final_score < -epsilon) { + metric_res->nnegative ++; + metric_res->negative_score += fabs (final_score); + } } else { s->score = 0; @@ -408,7 +420,7 @@ rspamd_task_insert_result_full (struct rspamd_task *task, /* Process cache item */ if (task->cfg->cache) { - rspamd_symbols_cache_inc_frequency (task->cfg->cache, symbol); + rspamd_symcache_inc_frequency (task->cfg->cache, symbol); } return s; diff --git a/src/libmime/filter.h b/src/libmime/filter.h index fdd0f8dc9..73f8269fa 100644 --- a/src/libmime/filter.h +++ b/src/libmime/filter.h @@ -7,7 +7,7 @@ #define RSPAMD_FILTER_H #include "config.h" -#include "symbols_cache.h" +#include "rspamd_symcache.h" #include "task.h" #include "khash.h" @@ -77,6 +77,10 @@ struct rspamd_metric_result { double score; /**< total score */ double grow_factor; /**< current grow factor */ struct rspamd_passthrough_result *passthrough_result; + guint npositive; + guint nnegative; + double positive_score; + double negative_score; khash_t(rspamd_symbols_hash) *symbols; /**< symbols of metric */ khash_t(rspamd_symbols_group_hash) *sym_groups; /**< groups of symbols */ gdouble actions_limits[METRIC_ACTION_MAX]; /**< set of actions for this metric */ diff --git a/src/libmime/images.c b/src/libmime/images.c index 194f6e4e8..c4d20b4ad 100644 --- a/src/libmime/images.c +++ b/src/libmime/images.c @@ -49,9 +49,12 @@ rspamd_images_process (struct rspamd_task *task) for (i = 0; i < task->parts->len; i ++) { part = g_ptr_array_index (task->parts, i); - if (rspamd_ftok_cmp (&part->ct->type, &srch) == 0 && + + if (!(part->flags & (RSPAMD_MIME_PART_TEXT|RSPAMD_MIME_PART_ARCHIVE))) { + if (rspamd_ftok_cmp (&part->ct->type, &srch) == 0 && part->parsed_data.len > 0) { - process_image (task, part); + process_image (task, part); + } } } diff --git a/src/libmime/lang_detection.c b/src/libmime/lang_detection.c index 7cf9ffec4..82e5fc2ff 100644 --- a/src/libmime/lang_detection.c +++ b/src/libmime/lang_detection.c @@ -20,8 +20,11 @@ #include "libutil/multipattern.h" #include "ucl.h" #include "khash.h" +#include "libstemmer.h" + #include <glob.h> #include <unicode/utf8.h> +#include <unicode/utf16.h> #include <unicode/ucnv.h> #include <unicode/uchar.h> #include <unicode/ustring.h> @@ -158,25 +161,30 @@ rspamd_language_search_str (const gchar *key, const gchar *elts[], size_t nelts) static guint rspamd_trigram_hash_func (gconstpointer key) { - return rspamd_cryptobox_fast_hash (key, 3 * sizeof (UChar), rspamd_hash_seed ()); + return rspamd_cryptobox_fast_hash (key, 3 * sizeof (UChar32), + rspamd_hash_seed ()); } static gboolean rspamd_trigram_equal_func (gconstpointer v, gconstpointer v2) { - return memcmp (v, v2, 3 * sizeof (UChar)) == 0; + return memcmp (v, v2, 3 * sizeof (UChar32)) == 0; } -KHASH_INIT (rspamd_trigram_hash, const UChar *, struct rspamd_ngramm_chain, true, +KHASH_INIT (rspamd_trigram_hash, const UChar32 *, struct rspamd_ngramm_chain, true, rspamd_trigram_hash_func, rspamd_trigram_equal_func); KHASH_INIT (rspamd_candidates_hash, const gchar *, struct rspamd_lang_detector_res *, true, rspamd_str_hash, rspamd_str_equal); +KHASH_INIT (rspamd_stopwords_hash, rspamd_ftok_t *, + char, false, + rspamd_ftok_hash, rspamd_ftok_equal); struct rspamd_lang_detector { GPtrArray *languages; khash_t(rspamd_trigram_hash) *trigramms[RSPAMD_LANGUAGE_MAX]; /* trigramms frequencies */ struct rspamd_stop_word_elt stop_words[RSPAMD_LANGUAGE_MAX]; + khash_t(rspamd_stopwords_hash) *stop_words_norm; UConverter *uchar_converter; gsize short_text_limit; gsize total_occurencies; /* number of all languages found */ @@ -184,7 +192,7 @@ struct rspamd_lang_detector { }; static void -rspamd_language_detector_ucs_lowercase (UChar *s, gsize len) +rspamd_language_detector_ucs_lowercase (UChar32 *s, gsize len) { gsize i; @@ -194,14 +202,13 @@ rspamd_language_detector_ucs_lowercase (UChar *s, gsize len) } static gboolean -rspamd_language_detector_ucs_is_latin (UChar *s, gsize len) +rspamd_language_detector_ucs_is_latin (const UChar32 *s, gsize len) { gsize i; gboolean ret = TRUE; for (i = 0; i < len; i ++) { - if (!((s[i] >= 'A' && s[i] <= 'Z') || (s[i] >= 'a' && s[i] <= 'z') - || s[i] == ' ')) { + if (s[i] >= 128 || !(g_ascii_isalnum (s[i]) || s[i] == ' ')) { ret = FALSE; break; } @@ -213,7 +220,7 @@ rspamd_language_detector_ucs_is_latin (UChar *s, gsize len) struct rspamd_language_ucs_elt { guint freq; const gchar *utf; - UChar s[0]; + UChar32 s[0]; }; static void @@ -439,17 +446,60 @@ rspamd_language_detector_read_file (struct rspamd_config *cfg, specific_stop_words = ucl_object_lookup (stop_words, nelt->name); if (specific_stop_words) { + struct sb_stemmer *stem = NULL; it = NULL; const ucl_object_t *w; guint start, stop; + stem = sb_stemmer_new (nelt->name, "UTF_8"); start = rspamd_multipattern_get_npatterns (d->stop_words[cat].mp); while ((w = ucl_object_iterate (specific_stop_words, &it, true)) != NULL) { - rspamd_multipattern_add_pattern (d->stop_words[cat].mp, - ucl_object_tostring (w), 0); + gsize wlen; + const char *word = ucl_object_tolstring (w, &wlen); + const char *saved; + + rspamd_multipattern_add_pattern_len (d->stop_words[cat].mp, + word, wlen, + RSPAMD_MULTIPATTERN_ICASE|RSPAMD_MULTIPATTERN_UTF8); nelt->stop_words ++; nstop ++; + + /* Also lemmatise and store normalised */ + if (stem) { + const char *nw = sb_stemmer_stem (stem, word, wlen); + + + if (nw) { + saved = nw; + wlen = strlen (nw); + } + else { + saved = word; + } + } + else { + saved = word; + } + + if (saved) { + gint rc; + rspamd_ftok_t *tok; + gchar *dst; + + tok = g_malloc (sizeof (*tok) + wlen + 1); + dst = ((gchar *)tok) + sizeof (*tok); + rspamd_strlcpy (dst, saved, wlen + 1); + tok->begin = dst; + tok->len = wlen; + + kh_put (rspamd_stopwords_hash, d->stop_words_norm, + tok, &rc); + } + } + + if (stem) { + sb_stemmer_delete (stem); } stop = rspamd_multipattern_get_npatterns (d->stop_words[cat].mp); @@ -502,22 +552,34 @@ rspamd_language_detector_read_file (struct rspamd_config *cfg, m2 += delta * delta2; if (key != NULL) { + UChar32 *cur_ucs; + const char *end = key + keylen, *cur_utf = key; + ucs_elt = rspamd_mempool_alloc (cfg->cfg_pool, - sizeof (*ucs_elt) + (keylen + 1) * sizeof (UChar)); + sizeof (*ucs_elt) + (keylen + 1) * sizeof (UChar32)); - nsym = ucnv_toUChars (d->uchar_converter, - ucs_elt->s, keylen + 1, - key, - keylen, &uc_err); - ucs_elt->utf = key; + cur_ucs = ucs_elt->s; + nsym = 0; + uc_err = U_ZERO_ERROR; + + while (cur_utf < end) { + *cur_ucs++ = ucnv_getNextUChar (d->uchar_converter, &cur_utf, + end, &uc_err); + if (!U_SUCCESS (uc_err)) { + break; + } - if (uc_err != U_ZERO_ERROR) { - msg_warn_config ("cannot convert key to unicode: %s", - u_errorName (uc_err)); + nsym ++; + } + + if (!U_SUCCESS (uc_err)) { + msg_warn_config ("cannot convert key %*s to unicode: %s", + (gint)keylen, key, u_errorName (uc_err)); continue; } + ucs_elt->utf = key; rspamd_language_detector_ucs_lowercase (ucs_elt->s, nsym); if (nsym == 3) { @@ -586,7 +648,7 @@ rspamd_language_detector_read_file (struct rspamd_config *cfg, nelt->mean = mean; nelt->std = std; - msg_info_config ("loaded %s language, %d trigramms, " + msg_debug_lang_det_cfg ("loaded %s language, %d trigramms, " "%d ngramms loaded; " "std=%.2f, mean=%.2f, skipped=%d, loaded=%d, stop_words=%d; " "(%s)", @@ -668,9 +730,7 @@ static void rspamd_language_detector_dtor (struct rspamd_lang_detector *d) { if (d) { - if (d->uchar_converter) { - ucnv_close (d->uchar_converter); - } + rspamd_ftok_t *tok; for (guint i = 0; i < RSPAMD_LANGUAGE_MAX; i ++) { kh_destroy (rspamd_trigram_hash, d->trigramms[i]); @@ -681,6 +741,10 @@ rspamd_language_detector_dtor (struct rspamd_lang_detector *d) if (d->languages) { g_ptr_array_free (d->languages, TRUE); } + + kh_foreach_key (d->stop_words_norm, tok, { + g_free (tok); /* String is embedded and freed automatically */ + }); } } @@ -746,8 +810,10 @@ rspamd_language_detector_init (struct rspamd_config *cfg) ret = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (*ret)); ret->languages = g_ptr_array_sized_new (gl.gl_pathc); - ret->uchar_converter = ucnv_open ("UTF-8", &uc_err); + ret->uchar_converter = rspamd_get_utf8_converter (); ret->short_text_limit = short_text_limit; + ret->stop_words_norm = kh_init (rspamd_stopwords_hash); + /* Map from ngramm in ucs32 to GPtrArray of rspamd_language_elt */ for (i = 0; i < RSPAMD_LANGUAGE_MAX; i ++) { ret->trigramms[i] = kh_init (rspamd_trigram_hash); @@ -784,7 +850,7 @@ rspamd_language_detector_init (struct rspamd_config *cfg) }); if (!rspamd_multipattern_compile (ret->stop_words[i].mp, &err)) { - msg_err_config ("cannot compile stop words for %d language group: %e", + msg_err_config ("cannot compile stop words for %z language group: %e", i, err); g_error_free (err); } @@ -816,31 +882,6 @@ end: return ret; } - -void -rspamd_language_detector_to_ucs (struct rspamd_lang_detector *d, - rspamd_mempool_t *pool, - rspamd_stat_token_t *utf_token, rspamd_stat_token_t *ucs_token) -{ - UChar *out; - int32_t nsym; - UErrorCode uc_err = U_ZERO_ERROR; - - ucs_token->flags = utf_token->flags; - out = rspamd_mempool_alloc (pool, sizeof (*out) * (utf_token->len + 1)); - nsym = ucnv_toUChars (d->uchar_converter, out, (utf_token->len + 1), - utf_token->begin, utf_token->len, &uc_err); - - if (nsym >= 0 && uc_err == U_ZERO_ERROR) { - rspamd_language_detector_ucs_lowercase (out, nsym); - ucs_token->begin = (const gchar *) out; - ucs_token->len = nsym; - } - else { - ucs_token->len = 0; - } -} - static void rspamd_language_detector_random_select (GArray *ucs_tokens, guint nwords, goffset *offsets_out) @@ -885,8 +926,10 @@ rspamd_language_detector_random_select (GArray *ucs_tokens, guint nwords, for (;;) { tok = &g_array_index (ucs_tokens, rspamd_stat_token_t, sel); /* Filter bad tokens */ - if (tok->len >= 2 && u_isalpha (*(UChar *)tok->begin) - && u_isalpha (*(((UChar *)tok->begin) + (tok->len - 1)))) { + + if (tok->unicode.len >= 2 && + u_isalpha (tok->unicode.begin[0]) && + u_isalpha (tok->unicode.begin[tok->unicode.len - 1])) { offsets_out[out_idx] = sel; break; } @@ -908,8 +951,6 @@ rspamd_language_detector_random_select (GArray *ucs_tokens, guint nwords, } } - - /* * Fisher-Yates algorithm: * for i from 0 to n−2 do @@ -931,7 +972,7 @@ rspamd_language_detector_random_select (GArray *ucs_tokens, guint nwords, } static goffset -rspamd_language_detector_next_ngramm (rspamd_stat_token_t *tok, UChar *window, +rspamd_language_detector_next_ngramm (rspamd_stat_token_t *tok, UChar32 *window, guint wlen, goffset cur_off) { guint i; @@ -940,36 +981,36 @@ rspamd_language_detector_next_ngramm (rspamd_stat_token_t *tok, UChar *window, /* Deal with spaces at the beginning and ending */ if (cur_off == 0) { - window[0] = (UChar)' '; + window[0] = (UChar32)' '; for (i = 0; i < wlen - 1; i ++) { - window[i + 1] = *(((UChar *)tok->begin) + i); + window[i + 1] = tok->unicode.begin[i]; } } - else if (cur_off + wlen == tok->len + 1) { + else if (cur_off + wlen == tok->unicode.len + 1) { /* Add trailing space */ for (i = 0; i < wlen - 1; i ++) { - window[i] = *(((UChar *)tok->begin) + cur_off + i); + window[i] = tok->unicode.begin[cur_off + i]; } - window[wlen - 1] = (UChar)' '; + window[wlen - 1] = (UChar32)' '; } - else if (cur_off + wlen > tok->len + 1) { + else if (cur_off + wlen > tok->unicode.len + 1) { /* No more fun */ return -1; } else { /* Normal case */ for (i = 0; i < wlen; i++) { - window[i] = *(((UChar *) tok->begin) + cur_off + i); + window[i] = tok->unicode.begin[cur_off + i]; } } } else { - if (tok->len <= cur_off) { + if (tok->normalized.len <= cur_off) { return -1; } - window[0] = *(((UChar *)tok->begin) + cur_off); + window[0] = tok->unicode.begin[cur_off]; } return cur_off + 1; @@ -981,7 +1022,7 @@ rspamd_language_detector_next_ngramm (rspamd_stat_token_t *tok, UChar *window, static void rspamd_language_detector_process_ngramm_full (struct rspamd_task *task, struct rspamd_lang_detector *d, - UChar *window, + UChar32 *window, khash_t(rspamd_candidates_hash) *candidates, khash_t(rspamd_trigram_hash) *trigramms) { @@ -1043,7 +1084,7 @@ rspamd_language_detector_detect_word (struct rspamd_task *task, khash_t(rspamd_trigram_hash) *trigramms) { const guint wlen = 3; - UChar window[3]; + UChar32 window[3]; goffset cur = 0; /* Split words */ @@ -1132,7 +1173,7 @@ rspamd_language_detector_detect_type (struct rspamd_task *task, { guint nparts = MIN (words->len, nwords); goffset *selected_words; - rspamd_stat_token_t *tok, ucs_w; + rspamd_stat_token_t *tok; guint i; selected_words = g_new0 (goffset, nparts); @@ -1142,11 +1183,11 @@ rspamd_language_detector_detect_type (struct rspamd_task *task, for (i = 0; i < nparts; i++) { tok = &g_array_index (words, rspamd_stat_token_t, selected_words[i]); - rspamd_language_detector_to_ucs (task->lang_det, - task->task_pool, - tok, &ucs_w); - rspamd_language_detector_detect_word (task, d, &ucs_w, candidates, - d->trigramms[cat]); + + if (tok->unicode.len >= 3) { + rspamd_language_detector_detect_word (task, d, tok, candidates, + d->trigramms[cat]); + } } /* Filter negligible candidates */ @@ -1624,11 +1665,11 @@ rspamd_language_detector_detect (struct rspamd_task *task, } if (!ret) { - if (part->utf_words->len < default_short_text_limit) { + if (part->nwords < default_short_text_limit) { r = rs_detect_none; msg_debug_lang_det ("text is too short for trigramms detection: " "%d words; at least %d words required", - (int)part->utf_words->len, + (int)part->nwords, (int)default_short_text_limit); rspamd_language_detector_set_language (task, part, "en"); candidates = kh_init (rspamd_candidates_hash); @@ -1687,7 +1728,7 @@ rspamd_language_detector_detect (struct rspamd_task *task, cbd.std = std; cbd.flags = RSPAMD_LANG_FLAG_DEFAULT; - if (part->utf_words->len < default_words / 2) { + if (part->nwords < default_words / 2) { cbd.flags |= RSPAMD_LANG_FLAG_SHORT; } } @@ -1749,4 +1790,23 @@ void rspamd_language_detector_unref (struct rspamd_lang_detector* d) { REF_RELEASE (d); +} + +gboolean +rspamd_language_detector_is_stop_word (struct rspamd_lang_detector *d, + const gchar *word, gsize wlen) +{ + khiter_t k; + rspamd_ftok_t search; + + search.begin = word; + search.len = wlen; + + k = kh_get (rspamd_stopwords_hash, d->stop_words_norm, &search); + + if (k != kh_end (d->stop_words_norm)) { + return TRUE; + } + + return FALSE; }
\ No newline at end of file diff --git a/src/libmime/lang_detection.h b/src/libmime/lang_detection.h index 50fe19b6e..517ab037e 100644 --- a/src/libmime/lang_detection.h +++ b/src/libmime/lang_detection.h @@ -63,17 +63,6 @@ struct rspamd_lang_detector* rspamd_language_detector_ref (struct rspamd_lang_de void rspamd_language_detector_unref (struct rspamd_lang_detector* d); /** - * Convert string from utf8 to ucs32 - * @param d - * @param utf_token - * @param ucs_token - */ -void rspamd_language_detector_to_ucs (struct rspamd_lang_detector *d, - rspamd_mempool_t *pool, - rspamd_stat_token_t *utf_token, - rspamd_stat_token_t *ucs_token); - -/** * Try to detect language of words * @param d * @param ucs_tokens @@ -84,4 +73,14 @@ gboolean rspamd_language_detector_detect (struct rspamd_task *task, struct rspamd_lang_detector *d, struct rspamd_mime_text_part *part); +/** + * Returns TRUE if the specified word is known to be a stop word + * @param d + * @param word + * @param wlen + * @return + */ +gboolean rspamd_language_detector_is_stop_word (struct rspamd_lang_detector *d, + const gchar *word, gsize wlen); + #endif diff --git a/src/libmime/message.c b/src/libmime/message.c index 43158d083..b75eb6b68 100644 --- a/src/libmime/message.c +++ b/src/libmime/message.c @@ -33,6 +33,7 @@ #endif #include <math.h> +#include <unicode/uchar.h> #define GTUBE_SYMBOL "GTUBE" @@ -60,108 +61,46 @@ static void rspamd_mime_part_extract_words (struct rspamd_task *task, struct rspamd_mime_text_part *part) { -#ifdef WITH_SNOWBALL - struct sb_stemmer *stem = NULL; -#endif rspamd_stat_token_t *w; - gchar *temp_word; - const guchar *r; - guint i, nlen, total_len = 0, short_len = 0; - gdouble avg_len = 0; + guint i, total_len = 0, short_len = 0; if (part->utf_words) { -#ifdef WITH_SNOWBALL - static GHashTable *stemmers = NULL; - - if (part->language && part->language[0] != '\0' && IS_PART_UTF (part)) { - - if (!stemmers) { - stemmers = g_hash_table_new (rspamd_strcase_hash, - rspamd_strcase_equal); - } - - stem = g_hash_table_lookup (stemmers, part->language); - - if (stem == NULL) { - - stem = sb_stemmer_new (part->language, "UTF_8"); - - if (stem == NULL) { - msg_debug_task ( - "<%s> cannot create lemmatizer for %s language", - task->message_id, part->language); - } else { - g_hash_table_insert (stemmers, g_strdup (part->language), - stem); - } - } - } -#endif - + rspamd_stem_words (part->utf_words, task->task_pool, part->language, + task->lang_det); for (i = 0; i < part->utf_words->len; i++) { guint64 h; w = &g_array_index (part->utf_words, rspamd_stat_token_t, i); - r = NULL; -#ifdef WITH_SNOWBALL - if (stem) { - r = sb_stemmer_stem (stem, w->begin, w->len); - } -#endif - - if (w->len > 0 && (w->flags & RSPAMD_STAT_TOKEN_FLAG_TEXT)) { - avg_len = avg_len + (w->len - avg_len) / (double) (i + 1); - - if (r != NULL) { - nlen = strlen (r); - nlen = MIN (nlen, w->len); - temp_word = rspamd_mempool_alloc (task->task_pool, nlen); - memcpy (temp_word, r, nlen); - - if (IS_PART_UTF (part)) { - rspamd_str_lc_utf8 (temp_word, nlen); - } - else { - rspamd_str_lc (temp_word, nlen); - } - - w->begin = temp_word; - w->len = nlen; - } - else { - temp_word = rspamd_mempool_alloc (task->task_pool, w->len); - memcpy (temp_word, w->begin, w->len); - - if (IS_PART_UTF (part)) { - rspamd_str_lc_utf8 (temp_word, w->len); - } - else { - rspamd_str_lc (temp_word, w->len); - } - w->begin = temp_word; - } - } - - if (w->len > 0) { + if (w->stemmed.len > 0) { /* * We use static hash seed if we would want to use that in shingles * computation in future */ h = rspamd_cryptobox_fast_hash_specific ( RSPAMD_CRYPTOBOX_HASHFAST_INDEPENDENT, - w->begin, w->len, words_hash_seed); + w->stemmed.begin, w->stemmed.len, words_hash_seed); g_array_append_val (part->normalized_hashes, h); - total_len += w->len; + total_len += w->stemmed.len; - if (w->len <= 3) { + if (w->stemmed.len <= 3) { short_len++; } } + + if (w->flags & RSPAMD_STAT_TOKEN_FLAG_TEXT) { + part->nwords ++; + } + + if (w->flags & (RSPAMD_STAT_TOKEN_FLAG_BROKEN_UNICODE| + RSPAMD_STAT_TOKEN_FLAG_NORMALISED| + RSPAMD_STAT_TOKEN_FLAG_INVISIBLE_SPACES)) { + task->flags |= RSPAMD_TASK_FLAG_BAD_UNICODE; + } } - if (part->utf_words && part->utf_words->len) { + if (part->utf_words->len) { gdouble *avg_len_p, *short_len_p; avg_len_p = rspamd_mempool_get_variable (task->task_pool, @@ -202,7 +141,37 @@ rspamd_mime_part_create_words (struct rspamd_task *task, enum rspamd_tokenize_type tok_type; if (IS_PART_UTF (part)) { + +#if U_ICU_VERSION_MAJOR_NUM < 50 + /* Hack to prevent hang with Thai in old libicu */ + const gchar *p = part->utf_stripped_content->data, *end; + guint i = 0; + end = p + part->utf_stripped_content->len; + gint32 uc, sc; + + tok_type = RSPAMD_TOKENIZE_UTF; + + while (p + i < end) { + U8_NEXT (p, i, part->utf_stripped_content->len, uc); + + if (((gint32) uc) < 0) { + tok_type = RSPAMD_TOKENIZE_RAW; + break; + } + + if (u_isalpha (uc)) { + sc = ublock_getCode (uc); + + if (sc == UBLOCK_THAI) { + msg_info_task ("enable workaround for Thai characters for old libicu"); + tok_type = RSPAMD_TOKENIZE_RAW; + break; + } + } + } +#else tok_type = RSPAMD_TOKENIZE_UTF; +#endif } else { tok_type = RSPAMD_TOKENIZE_RAW; @@ -214,12 +183,13 @@ rspamd_mime_part_create_words (struct rspamd_task *task, &part->utf_stripped_text, tok_type, task->cfg, part->exceptions, - NULL); + NULL, NULL); if (part->utf_words) { part->normalized_hashes = g_array_sized_new (FALSE, FALSE, sizeof (guint64), part->utf_words->len); + rspamd_normalize_words (part->utf_words, task->task_pool); } } @@ -680,7 +650,6 @@ rspamd_message_process_plain_text_part (struct rspamd_task *task, if (text_part->utf_raw_content != NULL) { /* Different from HTML, where we also parse HTML and strip tags */ text_part->utf_content = text_part->utf_raw_content; - text_part->unicode_content = text_part->unicode_raw_content; } else { /* @@ -727,22 +696,14 @@ rspamd_message_process_html_text_part (struct rspamd_task *task, text_part->flags |= RSPAMD_MIME_TEXT_PART_FLAG_EMPTY; } - /* Also add unicode content */ - text_part->unicode_content = g_array_sized_new (FALSE, FALSE, - sizeof (UChar), text_part->utf_content->len + 1); - rspamd_utf_to_unicode (text_part->utf_content, text_part->unicode_content); - rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t) free_byte_array_callback, text_part->utf_content); - rspamd_mempool_add_destructor (task->task_pool, - rspamd_array_free_hard, - text_part->unicode_content); return TRUE; } -static void +static gboolean rspamd_message_process_text_part_maybe (struct rspamd_task *task, struct rspamd_mime_part *mime_part) { @@ -780,7 +741,35 @@ rspamd_message_process_text_part_maybe (struct rspamd_task *task, found_html = TRUE; } else { - found_txt = TRUE; + /* We need to be extra careful with some stupid things here */ + + html_tok.begin = "plain"; + html_tok.len = 5; + + if (rspamd_ftok_cmp (&mime_part->ct->subtype, &html_tok) == 0) { + found_txt = TRUE; + } + else { + if (mime_part->cd && mime_part->cd->filename.len > 4) { + const gchar *pos = mime_part->cd->filename.begin + + mime_part->cd->filename.len - + sizeof (".txt") + 1; + if (rspamd_lc_cmp (pos, ".txt", sizeof ("txt") - 1) == 0) { + found_txt = TRUE; + } + else { + msg_debug_task ("found mime part with incorrect content-type: %T/%T, " + "filename: %T", + &mime_part->ct->type, + &mime_part->ct->subtype, + &mime_part->cd->filename); + } + } + else { + /* For something like Content-Type: text */ + found_txt = TRUE; + } + } } if (found_html) { @@ -823,11 +812,11 @@ rspamd_message_process_text_part_maybe (struct rspamd_task *task, mime_part->cd && mime_part->cd->type == RSPAMD_CT_ATTACHMENT && (task->cfg && !task->cfg->check_text_attachements)) { debug_task ("skip attachments for checking as text parts"); - return; + return TRUE; } else if (!(found_txt || found_html)) { /* Not a text part */ - return; + return FALSE; } text_part = rspamd_mempool_alloc0 (task->task_pool, @@ -841,12 +830,12 @@ rspamd_message_process_text_part_maybe (struct rspamd_task *task, if (found_html) { if (!rspamd_message_process_html_text_part (task, text_part)) { - return; + return FALSE; } } else { if (!rspamd_message_process_plain_text_part (task, text_part)) { - return; + return FALSE; } } @@ -877,7 +866,7 @@ rspamd_message_process_text_part_maybe (struct rspamd_task *task, rspamd_task_insert_result (task, GTUBE_SYMBOL, 0, NULL); - return; + return TRUE; } /* Post process part */ @@ -896,6 +885,8 @@ rspamd_message_process_text_part_maybe (struct rspamd_task *task, } rspamd_mime_part_create_words (task, text_part); + + return TRUE; } /* Creates message from various data using libmagic to detect type */ @@ -911,15 +902,18 @@ rspamd_message_from_data (struct rspamd_task *task, const guchar *start, g_assert (start != NULL); + part = rspamd_mempool_alloc0 (task->task_pool, sizeof (*part)); + tok = rspamd_task_get_request_header (task, "Content-Type"); if (tok) { /* We have Content-Type defined */ ct = rspamd_content_type_parse (tok->begin, tok->len, task->task_pool); + part->ct = ct; } - else if (task->cfg && task->cfg->libs_ctx) { - /* Try to predict it by content (slow) */ + + if (task->cfg && task->cfg->libs_ctx) { mb = magic_buffer (task->cfg->libs_ctx->libmagic, start, len); @@ -929,12 +923,16 @@ rspamd_message_from_data (struct rspamd_task *task, const guchar *start, srch.len = strlen (mb); ct = rspamd_content_type_parse (srch.begin, srch.len, task->task_pool); + msg_warn_task ("construct fake mime of type: %s", mb); + + if (!part->ct) { + part->ct = ct; + } + + part->detected_ct = ct; } } - msg_warn_task ("construct fake mime of type: %s", mb); - part = rspamd_mempool_alloc0 (task->task_pool, sizeof (*part)); - part->ct = ct; part->raw_data.begin = start; part->raw_data.len = len; part->parsed_data.begin = start; @@ -1200,12 +1198,47 @@ rspamd_message_process (struct rspamd_task *task) struct rspamd_mime_part *part; part = g_ptr_array_index (task->parts, i); - rspamd_message_process_text_part_maybe (task, part); + + + if (!rspamd_message_process_text_part_maybe (task, part) && + part->parsed_data.len > 0) { + const gchar *mb = magic_buffer (task->cfg->libs_ctx->libmagic, + part->parsed_data.begin, + part->parsed_data.len); + + if (mb) { + rspamd_ftok_t srch; + + srch.begin = mb; + srch.len = strlen (mb); + part->detected_ct = rspamd_content_type_parse (srch.begin, + srch.len, + task->task_pool); + } + + } } rspamd_images_process (task); rspamd_archives_process (task); + /* Calculate average words length and number of short words */ + struct rspamd_mime_text_part *text_part; + gdouble *var; + guint total_words = 0; + + PTR_ARRAY_FOREACH (task->text_parts, i, text_part) { + if (!text_part->language) { + rspamd_mime_part_detect_language (task, text_part); + } + + rspamd_mime_part_extract_words (task, text_part); + + if (text_part->utf_words) { + total_words += text_part->nwords; + } + } + /* Calculate distance for 2-parts messages */ if (task->text_parts->len == 2) { p1 = g_ptr_array_index (task->text_parts, 0); @@ -1235,7 +1268,7 @@ rspamd_message_process (struct rspamd_task *task) sel = p2; } else { - if (p1->unicode_content->len > p2->unicode_content->len) { + if (p1->utf_content->len > p2->utf_content->len) { sel = p1; } else { @@ -1243,8 +1276,6 @@ rspamd_message_process (struct rspamd_task *task) } } - rspamd_mime_part_detect_language (task, sel); - if (sel->language && sel->language[0]) { /* Propagate language */ if (sel == p1) { @@ -1295,23 +1326,6 @@ rspamd_message_process (struct rspamd_task *task) } } - /* Calculate average words length and number of short words */ - struct rspamd_mime_text_part *text_part; - gdouble *var; - guint total_words = 0; - - PTR_ARRAY_FOREACH (task->text_parts, i, text_part) { - if (!text_part->language) { - rspamd_mime_part_detect_language (task, text_part); - } - - rspamd_mime_part_extract_words (task, text_part); - - if (text_part->utf_words) { - total_words += text_part->utf_words->len; - } - } - if (total_words > 0) { var = rspamd_mempool_get_variable (task->task_pool, RSPAMD_MEMPOOL_AVG_WORDS_LEN); @@ -1327,6 +1341,8 @@ rspamd_message_process (struct rspamd_task *task) *var /= (double)total_words; } } + + rspamd_tokenize_meta_words (task); } diff --git a/src/libmime/message.h b/src/libmime/message.h index 205bf5bb2..19e8b40b5 100644 --- a/src/libmime/message.h +++ b/src/libmime/message.h @@ -43,28 +43,34 @@ struct rspamd_mime_text_part; struct rspamd_mime_multipart { GPtrArray *children; + rspamd_ftok_t boundary; }; struct rspamd_mime_part { struct rspamd_content_type *ct; + struct rspamd_content_type *detected_ct; struct rspamd_content_disposition *cd; rspamd_ftok_t raw_data; rspamd_ftok_t parsed_data; struct rspamd_mime_part *parent_part; - GHashTable *raw_headers; + GQueue *headers_order; + GHashTable *raw_headers; + gchar *raw_headers_str; gsize raw_headers_len; + enum rspamd_cte cte; + enum rspamd_mime_part_flags flags; + guint id; union { - struct rspamd_mime_multipart mp; + struct rspamd_mime_multipart *mp; struct rspamd_mime_text_part *txt; struct rspamd_image *img; struct rspamd_archive *arch; } specific; - enum rspamd_mime_part_flags flags; guchar digest[rspamd_cryptobox_HASHBYTES]; }; @@ -100,10 +106,6 @@ struct rspamd_mime_text_part { GArray *utf_words; UText utf_stripped_text; /* Used by libicu to represent the utf8 content */ - /* Unicode content, used by libicu */ - GArray *unicode_raw_content; /* unicode raw content (of UChar) */ - GArray *unicode_content; /* unicode processed content (of UChar) */ - GPtrArray *newlines; /**< positions of newlines in text, relative to content*/ struct html_content *html; GList *exceptions; /**< list of offsets of urls */ @@ -112,6 +114,7 @@ struct rspamd_mime_text_part { guint flags; guint nlines; guint spaces; + guint nwords; guint non_ascii_chars; guint ascii_chars; guint double_spaces; @@ -144,6 +147,7 @@ struct received_header { gchar *real_ip; gchar *by_hostname; gchar *for_mbox; + gchar *comment_ip; rspamd_inet_addr_t *addr; struct rspamd_mime_header *hdr; time_t timestamp; diff --git a/src/libmime/mime_encoding.c b/src/libmime/mime_encoding.c index 7fd75af9d..e3479c3e7 100644 --- a/src/libmime/mime_encoding.c +++ b/src/libmime/mime_encoding.c @@ -40,11 +40,6 @@ #define SET_PART_UTF(part) ((part)->flags |= RSPAMD_MIME_TEXT_PART_FLAG_UTF) static rspamd_regexp_t *utf_compatible_re = NULL; -UConverter *utf8_converter = NULL; - -#if U_ICU_VERSION_MAJOR_NUM >= 44 -static const UNormalizer2 *norm = NULL; -#endif struct rspamd_charset_substitution { const gchar *input; @@ -101,36 +96,6 @@ rspamd_mime_get_converter_cached (const gchar *enc, UErrorCode *err) return conv; } -static inline void -rspamd_mime_utf8_conv_init (void) -{ - if (utf8_converter == NULL) { - UErrorCode uc_err = U_ZERO_ERROR; - - utf8_converter = ucnv_open (UTF8_CHARSET, &uc_err); - - if (!U_SUCCESS (uc_err)) { - msg_err ("FATAL error: cannot open converter for utf8: %s", - u_errorName (uc_err)); - - g_assert_not_reached (); - } - - ucnv_setFromUCallBack (utf8_converter, - UCNV_FROM_U_CALLBACK_SUBSTITUTE, - NULL, - NULL, - NULL, - &uc_err); - ucnv_setToUCallBack (utf8_converter, - UCNV_TO_U_CALLBACK_SUBSTITUTE, - NULL, - NULL, - NULL, - &uc_err); - } -} - static void rspamd_mime_encoding_substitute_init (void) { @@ -224,10 +189,10 @@ rspamd_mime_text_to_utf8 (rspamd_mempool_t *pool, UChar *tmp_buf; UErrorCode uc_err = U_ZERO_ERROR; - UConverter *conv; + UConverter *conv, *utf8_converter; - rspamd_mime_utf8_conv_init (); conv = rspamd_mime_get_converter_cached (in_enc, &uc_err); + utf8_converter = rspamd_get_utf8_converter (); if (conv == NULL) { g_set_error (err, rspamd_iconv_error_quark (), EINVAL, @@ -276,117 +241,6 @@ rspamd_mime_text_to_utf8 (rspamd_mempool_t *pool, return d; } -static void -rspamd_mime_text_part_ucs_from_utf (struct rspamd_task *task, - struct rspamd_mime_text_part *text_part) -{ - GByteArray *utf; - UErrorCode uc_err = U_ZERO_ERROR; - - rspamd_mime_utf8_conv_init (); - utf = text_part->utf_raw_content; - text_part->unicode_raw_content = g_array_sized_new (FALSE, FALSE, - sizeof (UChar), utf->len + 1); - text_part->unicode_raw_content->len = ucnv_toUChars (utf8_converter, - (UChar *)text_part->unicode_raw_content->data, - utf->len + 1, - utf->data, - utf->len, - &uc_err); - - if (!U_SUCCESS (uc_err)) { - g_array_free (text_part->unicode_raw_content, TRUE); - text_part->unicode_raw_content = NULL; - } -} - -static void -rspamd_mime_text_part_normalise (struct rspamd_task *task, - struct rspamd_mime_text_part *text_part) -{ -#if U_ICU_VERSION_MAJOR_NUM >= 44 - UErrorCode uc_err = U_ZERO_ERROR; - gint32 nsym, end; - UChar *src = NULL, *dest = NULL; - - if (norm == NULL) { - norm = unorm2_getInstance (NULL, "nfkc", UNORM2_COMPOSE, &uc_err); - } - - if (!text_part->unicode_raw_content) { - return; - } - - src = (UChar *)text_part->unicode_raw_content->data; - nsym = text_part->unicode_raw_content->len; - - /* We can now check if we need to decompose */ - end = unorm2_spanQuickCheckYes (norm, src, nsym, &uc_err); - - if (!U_SUCCESS (uc_err)) { - msg_warn_task ("cannot normalise URL, cannot check normalisation: %s", - u_errorName (uc_err)); - return; - } - - if (end == nsym) { - /* Already normalised */ - return; - } - - text_part->flags |= RSPAMD_MIME_TEXT_PART_HAS_SUBNORMAL; - dest = g_malloc (nsym * sizeof (*dest)); - memcpy (dest, src, end * sizeof (*dest)); - nsym = unorm2_normalizeSecondAndAppend (norm, dest, end, nsym, - src + end, nsym - end, &uc_err); - - if (!U_SUCCESS (uc_err)) { - if (uc_err != U_BUFFER_OVERFLOW_ERROR) { - msg_warn_task ("cannot normalise URL: %s", - u_errorName (uc_err)); - } - } - else { - /* Copy normalised back */ - memcpy (text_part->unicode_raw_content->data, dest, nsym * sizeof (UChar)); - text_part->unicode_raw_content->len = nsym; - text_part->flags |= RSPAMD_MIME_TEXT_PART_NORMALISED; - } - - g_free (dest); -#endif -} - -/* - * Recode utf from normalised unichars if needed - */ -static void -rspamd_mime_text_part_maybe_renormalise (struct rspamd_task *task, - struct rspamd_mime_text_part *text_part) -{ - UErrorCode uc_err = U_ZERO_ERROR; - guint clen, dlen; - gint r; - - rspamd_mime_utf8_conv_init (); - - if ((text_part->flags & RSPAMD_MIME_TEXT_PART_NORMALISED) && - text_part->unicode_raw_content) { - clen = ucnv_getMaxCharSize (utf8_converter); - dlen = UCNV_GET_MAX_BYTES_FOR_STRING (text_part->unicode_raw_content->len, - clen); - g_byte_array_set_size (text_part->utf_raw_content, dlen); - r = ucnv_fromUChars (utf8_converter, - text_part->utf_raw_content->data, - dlen, - (UChar *)text_part->unicode_raw_content->data, - text_part->unicode_raw_content->len, - &uc_err); - text_part->utf_raw_content->len = r; - } -} - - static gboolean rspamd_mime_text_part_utf8_convert (struct rspamd_task *task, struct rspamd_mime_text_part *text_part, @@ -395,13 +249,13 @@ rspamd_mime_text_part_utf8_convert (struct rspamd_task *task, GError **err) { gchar *d; - gint32 r, clen, dlen; - + gint32 r, clen, dlen, uc_len; + UChar *tmp_buf; UErrorCode uc_err = U_ZERO_ERROR; - UConverter *conv; + UConverter *conv, *utf8_converter; - rspamd_mime_utf8_conv_init (); conv = rspamd_mime_get_converter_cached (charset, &uc_err); + utf8_converter = rspamd_get_utf8_converter (); if (conv == NULL) { g_set_error (err, rspamd_iconv_error_quark (), EINVAL, @@ -411,11 +265,10 @@ rspamd_mime_text_part_utf8_convert (struct rspamd_task *task, return FALSE; } - - text_part->unicode_raw_content = g_array_sized_new (FALSE, FALSE, - sizeof (UChar), input->len + 1); - r = ucnv_toUChars (conv, - (UChar *)text_part->unicode_raw_content->data, + tmp_buf = g_new (UChar, input->len + 1); + uc_err = U_ZERO_ERROR; + uc_len = ucnv_toUChars (conv, + tmp_buf, input->len + 1, input->data, input->len, @@ -425,33 +278,34 @@ rspamd_mime_text_part_utf8_convert (struct rspamd_task *task, g_set_error (err, rspamd_iconv_error_quark (), EINVAL, "cannot convert data to unicode from %s: %s", charset, u_errorName (uc_err)); + g_free (tmp_buf); + return FALSE; } - text_part->unicode_raw_content->len = r; - rspamd_mime_text_part_normalise (task, text_part); - /* Now, convert to utf8 */ clen = ucnv_getMaxCharSize (utf8_converter); - dlen = UCNV_GET_MAX_BYTES_FOR_STRING (r, clen); + dlen = UCNV_GET_MAX_BYTES_FOR_STRING (uc_len, clen); d = rspamd_mempool_alloc (task->task_pool, dlen); r = ucnv_fromUChars (utf8_converter, d, dlen, - (UChar *)text_part->unicode_raw_content->data, r, &uc_err); + tmp_buf, uc_len, &uc_err); if (!U_SUCCESS (uc_err)) { g_set_error (err, rspamd_iconv_error_quark (), EINVAL, "cannot convert data from unicode from %s: %s", charset, u_errorName (uc_err)); + g_free (tmp_buf); return FALSE; } - msg_info_task ("converted from %s to UTF-8 inlen: %z, outlen: %d", - charset, input->len, r); + msg_info_task ("converted from %s to UTF-8 inlen: %d, outlen: %d (%d UTF16 chars)", + charset, input->len, r, uc_len); text_part->utf_raw_content = rspamd_mempool_alloc (task->task_pool, sizeof (*text_part->utf_raw_content) + sizeof (gpointer) * 4); text_part->utf_raw_content->data = d; text_part->utf_raw_content->len = r; + g_free (tmp_buf); return TRUE; } @@ -464,7 +318,7 @@ rspamd_mime_to_utf8_byte_array (GByteArray *in, gint32 r, clen, dlen; UChar *tmp_buf; UErrorCode uc_err = U_ZERO_ERROR; - UConverter *conv; + UConverter *conv, *utf8_converter; rspamd_ftok_t charset_tok; RSPAMD_FTOK_FROM_STR (&charset_tok, enc); @@ -477,7 +331,7 @@ rspamd_mime_to_utf8_byte_array (GByteArray *in, return TRUE; } - rspamd_mime_utf8_conv_init (); + utf8_converter = rspamd_get_utf8_converter (); conv = rspamd_mime_get_converter_cached (enc, &uc_err); if (conv == NULL) { @@ -690,13 +544,12 @@ rspamd_mime_text_part_maybe_convert (struct rspamd_task *task, } checked = TRUE; + text_part->real_charset = charset; } else { SET_PART_UTF (text_part); text_part->utf_raw_content = part_content; - rspamd_mime_text_part_ucs_from_utf (task, text_part); - rspamd_mime_text_part_normalise (task, text_part); - rspamd_mime_text_part_maybe_renormalise (task, text_part); + text_part->real_charset = UTF8_CHARSET; return; } @@ -710,6 +563,7 @@ rspamd_mime_text_part_maybe_convert (struct rspamd_task *task, MIN (RSPAMD_CHARSET_MAX_CONTENT, part_content->len)); msg_info_task ("detected charset: %s", charset); checked = TRUE; + text_part->real_charset = charset; } } @@ -727,9 +581,7 @@ rspamd_mime_text_part_maybe_convert (struct rspamd_task *task, part_content->len, !checked)) { SET_PART_UTF (text_part); text_part->utf_raw_content = part_content; - rspamd_mime_text_part_ucs_from_utf (task, text_part); - rspamd_mime_text_part_normalise (task, text_part); - rspamd_mime_text_part_maybe_renormalise (task, text_part); + text_part->real_charset = UTF8_CHARSET; return; } @@ -748,21 +600,9 @@ rspamd_mime_text_part_maybe_convert (struct rspamd_task *task, text_part->utf_raw_content = part_content; return; } + + text_part->real_charset = charset; } SET_PART_UTF (text_part); } - -void -rspamd_utf_to_unicode (GByteArray *in, GArray *dest) -{ - UErrorCode uc_err = U_ZERO_ERROR; - - g_array_set_size (dest, in->len + 1); - dest->len = ucnv_toUChars (utf8_converter, - (UChar *)dest->data, - in->len + 1, - in->data, - in->len, - &uc_err); -} diff --git a/src/libmime/mime_encoding.h b/src/libmime/mime_encoding.h index 0754bb348..5f436d99d 100644 --- a/src/libmime/mime_encoding.h +++ b/src/libmime/mime_encoding.h @@ -18,6 +18,7 @@ #include "config.h" #include "mem_pool.h" +#include "fstring.h" struct rspamd_task; struct rspamd_mime_part; @@ -86,11 +87,5 @@ gboolean rspamd_mime_charset_utf_check (rspamd_ftok_t *charset, */ void rspamd_mime_charset_utf_enforce (gchar *in, gsize len); -/** - * Converts utf8 to libicu unichars - * @param in - * @param dest - */ -void rspamd_utf_to_unicode (GByteArray *in, GArray *dest); #endif /* SRC_LIBMIME_MIME_ENCODING_H_ */ diff --git a/src/libmime/mime_expressions.c b/src/libmime/mime_expressions.c index 596d959c6..3ab8cee32 100644 --- a/src/libmime/mime_expressions.c +++ b/src/libmime/mime_expressions.c @@ -24,62 +24,65 @@ #include "utlist.h" gboolean rspamd_compare_encoding (struct rspamd_task *task, - GArray * args, - void *unused); + GArray * args, + void *unused); gboolean rspamd_header_exists (struct rspamd_task *task, - GArray * args, - void *unused); + GArray * args, + void *unused); gboolean rspamd_parts_distance (struct rspamd_task *task, - GArray * args, - void *unused); + GArray * args, + void *unused); gboolean rspamd_recipients_distance (struct rspamd_task *task, - GArray * args, - void *unused); + GArray * args, + void *unused); gboolean rspamd_has_only_html_part (struct rspamd_task *task, - GArray * args, - void *unused); + GArray * args, + void *unused); gboolean rspamd_is_recipients_sorted (struct rspamd_task *task, - GArray * args, - void *unused); + GArray * args, + void *unused); gboolean rspamd_compare_transfer_encoding (struct rspamd_task *task, - GArray * args, - void *unused); + GArray * args, + void *unused); gboolean rspamd_is_html_balanced (struct rspamd_task *task, - GArray * args, - void *unused); + GArray * args, + void *unused); gboolean rspamd_has_html_tag (struct rspamd_task *task, - GArray * args, - void *unused); + GArray * args, + void *unused); gboolean rspamd_has_fake_html (struct rspamd_task *task, - GArray * args, - void *unused); + GArray * args, + void *unused); static gboolean rspamd_raw_header_exists (struct rspamd_task *task, - GArray * args, - void *unused); + GArray * args, + void *unused); static gboolean rspamd_check_smtp_data (struct rspamd_task *task, - GArray * args, - void *unused); + GArray * args, + void *unused); static gboolean rspamd_content_type_is_type (struct rspamd_task * task, - GArray * args, - void *unused); + GArray * args, + void *unused); static gboolean rspamd_content_type_is_subtype (struct rspamd_task *task, - GArray * args, - void *unused); + GArray * args, + void *unused); static gboolean rspamd_content_type_has_param (struct rspamd_task * task, - GArray * args, - void *unused); + GArray * args, + void *unused); static gboolean rspamd_content_type_compare_param (struct rspamd_task * task, - GArray * args, - void *unused); + GArray * args, + void *unused); static gboolean rspamd_has_content_part (struct rspamd_task *task, - GArray * args, - void *unused); + GArray * args, + void *unused); static gboolean rspamd_has_content_part_len (struct rspamd_task *task, - GArray * args, - void *unused); + GArray * args, + void *unused); static gboolean rspamd_is_empty_body (struct rspamd_task *task, - GArray * args, - void *unused); + GArray * args, + void *unused); +static gboolean rspamd_has_flag_expr (struct rspamd_task *task, + GArray * args, + void *unused); static rspamd_expression_atom_t * rspamd_mime_expr_parse (const gchar *line, gsize len, rspamd_mempool_t *pool, gpointer ud, GError **err); @@ -136,25 +139,26 @@ static struct _fl { rspamd_internal_func_t func; void *user_data; } rspamd_functions_list[] = { - {"check_smtp_data", rspamd_check_smtp_data, NULL}, - {"compare_encoding", rspamd_compare_encoding, NULL}, - {"compare_parts_distance", rspamd_parts_distance, NULL}, - {"compare_recipients_distance", rspamd_recipients_distance, NULL}, - {"compare_transfer_encoding", rspamd_compare_transfer_encoding, NULL}, - {"content_type_compare_param", rspamd_content_type_compare_param, NULL}, - {"content_type_has_param", rspamd_content_type_has_param, NULL}, - {"content_type_is_subtype", rspamd_content_type_is_subtype, NULL}, - {"content_type_is_type", rspamd_content_type_is_type, NULL}, - {"has_content_part", rspamd_has_content_part, NULL}, - {"has_content_part_len", rspamd_has_content_part_len, NULL}, - {"has_fake_html", rspamd_has_fake_html, NULL}, - {"has_html_tag", rspamd_has_html_tag, NULL}, - {"has_only_html_part", rspamd_has_only_html_part, NULL}, - {"header_exists", rspamd_header_exists, NULL}, - {"is_empty_body", rspamd_is_empty_body, NULL}, - {"is_html_balanced", rspamd_is_html_balanced, NULL}, - {"is_recipients_sorted", rspamd_is_recipients_sorted, NULL}, - {"raw_header_exists", rspamd_raw_header_exists, NULL} + {"check_smtp_data", rspamd_check_smtp_data, NULL}, + {"compare_encoding", rspamd_compare_encoding, NULL}, + {"compare_parts_distance", rspamd_parts_distance, NULL}, + {"compare_recipients_distance", rspamd_recipients_distance, NULL}, + {"compare_transfer_encoding", rspamd_compare_transfer_encoding, NULL}, + {"content_type_compare_param", rspamd_content_type_compare_param, NULL}, + {"content_type_has_param", rspamd_content_type_has_param, NULL}, + {"content_type_is_subtype", rspamd_content_type_is_subtype, NULL}, + {"content_type_is_type", rspamd_content_type_is_type, NULL}, + {"has_content_part", rspamd_has_content_part, NULL}, + {"has_content_part_len", rspamd_has_content_part_len, NULL}, + {"has_fake_html", rspamd_has_fake_html, NULL}, + {"has_flag", rspamd_has_flag_expr, NULL}, + {"has_html_tag", rspamd_has_html_tag, NULL}, + {"has_only_html_part", rspamd_has_only_html_part, NULL}, + {"header_exists", rspamd_header_exists, NULL}, + {"is_empty_body", rspamd_is_empty_body, NULL}, + {"is_html_balanced", rspamd_is_html_balanced, NULL}, + {"is_recipients_sorted", rspamd_is_recipients_sorted, NULL}, + {"raw_header_exists", rspamd_raw_header_exists, NULL}, }; const struct rspamd_atom_subr mime_expr_subr = { @@ -239,6 +243,18 @@ rspamd_parse_long_option (const gchar *start, gsize len, ret = TRUE; a->type = RSPAMD_RE_SARAWBODY; } + else if (TYPE_CHECK (start, "words", len)) { + ret = TRUE; + a->type = RSPAMD_RE_WORDS; + } + else if (TYPE_CHECK (start, "raw_words", len)) { + ret = TRUE; + a->type = RSPAMD_RE_RAWWORDS; + } + else if (TYPE_CHECK (start, "stem_words", len)) { + ret = TRUE; + a->type = RSPAMD_RE_STEMWORDS; + } else if (TYPE_CHECK (start, "selector", len)) { ret = TRUE; a->type = RSPAMD_RE_SELECTOR; @@ -1611,12 +1627,45 @@ rspamd_check_smtp_data (struct rspamd_task *task, GArray * args, void *unused) return FALSE; } +static inline gboolean +rspamd_check_ct_attr (const gchar *begin, gsize len, + struct expression_argument *arg_pattern) +{ + rspamd_regexp_t *re; + gboolean r = FALSE; + + if (arg_pattern->type == EXPRESSION_ARGUMENT_REGEXP) { + re = arg_pattern->data; + + if (len > 0) { + r = rspamd_regexp_search (re, + begin, len, + NULL, NULL, FALSE, NULL); + } + + if (r) { + return TRUE; + } + } + else { + /* Just do strcasecmp */ + gsize plen = strlen (arg_pattern->data); + + if (plen == len && + g_ascii_strncasecmp (arg_pattern->data, begin, len) == 0) { + return TRUE; + } + } + + return FALSE; +} + static gboolean rspamd_content_type_compare_param (struct rspamd_task * task, GArray * args, void *unused) { - rspamd_regexp_t *re; + struct expression_argument *arg, *arg1, *arg_pattern; gboolean recursive = FALSE; struct rspamd_mime_part *cur_part; @@ -1624,7 +1673,6 @@ rspamd_content_type_compare_param (struct rspamd_task * task, rspamd_ftok_t srch; struct rspamd_content_type_param *found = NULL, *cur; const gchar *param_name; - gboolean r = FALSE; if (args == NULL || args->len < 2) { msg_warn_task ("no parameters to function"); @@ -1656,32 +1704,33 @@ rspamd_content_type_compare_param (struct rspamd_task * task, } } - if (cur_part->ct->attrs) { - RSPAMD_FTOK_FROM_STR (&srch, param_name); + rspamd_ftok_t lit; + RSPAMD_FTOK_FROM_STR (&srch, param_name); + RSPAMD_FTOK_FROM_STR (&lit, "charset"); + + if (rspamd_ftok_equal (&srch, &lit)) { + if (rspamd_check_ct_attr (cur_part->ct->charset.begin, + cur_part->ct->charset.len, arg_pattern)) { + return TRUE; + } + } + + RSPAMD_FTOK_FROM_STR (&lit, "boundary"); + if (rspamd_ftok_equal (&srch, &lit)) { + if (rspamd_check_ct_attr (cur_part->ct->orig_boundary.begin, + cur_part->ct->orig_boundary.len, arg_pattern)) { + return TRUE; + } + } + if (cur_part->ct->attrs) { found = g_hash_table_lookup (cur_part->ct->attrs, &srch); if (found) { DL_FOREACH (found, cur) { - if (arg_pattern->type == EXPRESSION_ARGUMENT_REGEXP) { - re = arg_pattern->data; - - if (cur->value.len > 0) { - r = rspamd_regexp_search (re, - cur->value.begin, cur->value.len, - NULL, NULL, FALSE, NULL); - } - - if (r) { - return TRUE; - } - } - else { - /* Just do strcasecmp */ - RSPAMD_FTOK_FROM_STR (&srch, arg_pattern->data); - if (rspamd_ftok_casecmp (&srch, &cur->value) == 0) { - return TRUE; - } + if (rspamd_check_ct_attr (cur->value.begin, + cur->value.len, arg_pattern)) { + return TRUE; } } } @@ -1737,9 +1786,21 @@ rspamd_content_type_has_param (struct rspamd_task * task, } } - if (cur_part->ct->attrs) { - RSPAMD_FTOK_FROM_STR (&srch, param_name); + rspamd_ftok_t lit; + RSPAMD_FTOK_FROM_STR (&srch, param_name); + RSPAMD_FTOK_FROM_STR (&lit, "charset"); + + if (rspamd_ftok_equal (&srch, &lit)) { + return cur_part->ct->charset.len > 0; + } + + RSPAMD_FTOK_FROM_STR (&lit, "boundary"); + if (rspamd_ftok_equal (&srch, &lit)) { + return cur_part->ct->orig_boundary.len > 0; + } + + if (cur_part->ct->attrs) { found = g_hash_table_lookup (cur_part->ct->attrs, &srch); if (found) { @@ -2048,4 +2109,63 @@ rspamd_is_empty_body (struct rspamd_task *task, } return TRUE; +} + +#define TASK_FLAG_READ(flag) do { \ + result = !!(task->flags & (flag)); \ +} while(0) + +#define TASK_GET_FLAG(flag, strname, macro) do { \ + if (!found && strcmp ((flag), strname) == 0) { \ + TASK_FLAG_READ((macro)); \ + found = TRUE; \ + } \ +} while(0) + +static gboolean +rspamd_has_flag_expr (struct rspamd_task *task, + GArray * args, + void *unused) +{ + gboolean found = FALSE, result = FALSE; + struct expression_argument *flag_arg; + const gchar *flag_str; + + if (args == NULL) { + msg_warn_task ("no parameters to function"); + return FALSE; + } + + flag_arg = &g_array_index (args, struct expression_argument, 0); + + if (flag_arg->type != EXPRESSION_ARGUMENT_NORMAL) { + msg_warn_task ("invalid parameter to function"); + return FALSE; + } + + flag_str = (const gchar *)flag_arg->data; + + TASK_GET_FLAG (flag_str, "pass_all", RSPAMD_TASK_FLAG_PASS_ALL); + TASK_GET_FLAG (flag_str, "no_log", RSPAMD_TASK_FLAG_NO_LOG); + TASK_GET_FLAG (flag_str, "no_stat", RSPAMD_TASK_FLAG_NO_STAT); + TASK_GET_FLAG (flag_str, "skip", RSPAMD_TASK_FLAG_SKIP); + TASK_GET_FLAG (flag_str, "extended_urls", RSPAMD_TASK_FLAG_EXT_URLS); + TASK_GET_FLAG (flag_str, "learn_spam", RSPAMD_TASK_FLAG_LEARN_SPAM); + TASK_GET_FLAG (flag_str, "learn_ham", RSPAMD_TASK_FLAG_LEARN_HAM); + TASK_GET_FLAG (flag_str, "greylisted", RSPAMD_TASK_FLAG_GREYLISTED); + TASK_GET_FLAG (flag_str, "broken_headers", + RSPAMD_TASK_FLAG_BROKEN_HEADERS); + TASK_GET_FLAG (flag_str, "skip_process", + RSPAMD_TASK_FLAG_SKIP_PROCESS); + TASK_GET_FLAG (flag_str, "milter", + RSPAMD_TASK_FLAG_MILTER); + TASK_GET_FLAG (flag_str, "bad_unicode", + RSPAMD_TASK_FLAG_BAD_UNICODE); + + if (!found) { + msg_warn_task ("invalid flag name %s", flag_str); + return FALSE; + } + + return result; }
\ No newline at end of file diff --git a/src/libmime/mime_headers.c b/src/libmime/mime_headers.c index 9d0518c29..2769ae633 100644 --- a/src/libmime/mime_headers.c +++ b/src/libmime/mime_headers.c @@ -18,6 +18,7 @@ #include "smtp_parsers.h" #include "mime_encoding.h" #include "libserver/mempool_vars_internal.h" +#include <unicode/utf8.h> static void rspamd_mime_header_check_special (struct rspamd_task *task, @@ -52,22 +53,22 @@ rspamd_mime_header_check_special (struct rspamd_task *task, break; case 0x76F31A09F4352521ULL: /* to */ task->rcpt_mime = rspamd_email_address_from_mime (task->task_pool, - rh->value, strlen (rh->value), task->rcpt_mime); + rh->decoded, strlen (rh->decoded), task->rcpt_mime); rh->type = RSPAMD_HEADER_TO|RSPAMD_HEADER_RCPT|RSPAMD_HEADER_UNIQUE; break; case 0x7EB117C1480B76ULL: /* cc */ task->rcpt_mime = rspamd_email_address_from_mime (task->task_pool, - rh->value, strlen (rh->value), task->rcpt_mime); + rh->decoded, strlen (rh->decoded), task->rcpt_mime); rh->type = RSPAMD_HEADER_CC|RSPAMD_HEADER_RCPT|RSPAMD_HEADER_UNIQUE; break; case 0xE4923E11C4989C8DULL: /* bcc */ task->rcpt_mime = rspamd_email_address_from_mime (task->task_pool, - rh->value, strlen (rh->value), task->rcpt_mime); + rh->decoded, strlen (rh->decoded), task->rcpt_mime); rh->type = RSPAMD_HEADER_BCC|RSPAMD_HEADER_RCPT|RSPAMD_HEADER_UNIQUE; break; case 0x41E1985EDC1CBDE4ULL: /* from */ task->from_mime = rspamd_email_address_from_mime (task->task_pool, - rh->value, strlen (rh->value), task->from_mime); + rh->decoded, strlen (rh->decoded), task->from_mime); rh->type = RSPAMD_HEADER_FROM|RSPAMD_HEADER_SENDER|RSPAMD_HEADER_UNIQUE; break; case 0x43A558FC7C240226ULL: /* message-id */ { @@ -347,8 +348,15 @@ rspamd_mime_headers_process (struct rspamd_task *task, GHashTable *target, } nh->value = tmp; + + gboolean broken_utf = FALSE; + nh->decoded = rspamd_mime_header_decode (task->task_pool, - nh->value, strlen (tmp)); + nh->value, strlen (tmp), &broken_utf); + + if (broken_utf) { + task->flags |= RSPAMD_TASK_FLAG_BAD_UNICODE; + } if (nh->decoded == NULL) { nh->decoded = ""; @@ -530,10 +538,11 @@ rspamd_mime_header_sanity_check (GString *str) gchar * rspamd_mime_header_decode (rspamd_mempool_t *pool, const gchar *in, - gsize inlen) + gsize inlen, gboolean *invalid_utf) { GString *out; - const gchar *c, *p, *end, *tok_start = NULL; + const guchar *c, *p, *end; + const gchar *tok_start = NULL; gsize tok_len = 0, pos; GByteArray *token = NULL, *decoded; rspamd_ftok_t cur_charset = {0, NULL}, old_charset = {0, NULL}; @@ -566,6 +575,32 @@ rspamd_mime_header_decode (rspamd_mempool_t *pool, const gchar *in, c = p; state = got_eqsign; } + else if (*p >= 128) { + gint off = 0; + UChar32 uc; + /* Unencoded character */ + g_string_append_len (out, c, p - c); + /* Check if that's valid UTF8 */ + U8_NEXT (p, off, end - p, uc); + + if (uc <= 0) { + c = p + 1; + /* 0xFFFD in UTF8 */ + g_string_append_len (out, " ", 3); + off = 0; + U8_APPEND_UNSAFE (out->str + out->len - 3, + off, 0xfffd); + + if (invalid_utf) { + *invalid_utf = TRUE; + } + } + else { + c = p; + p = p + off; + continue; /* To avoid p ++ after this block */ + } + } p ++; break; case got_eqsign: diff --git a/src/libmime/mime_headers.h b/src/libmime/mime_headers.h index 03a7beae9..3c0c23a36 100644 --- a/src/libmime/mime_headers.h +++ b/src/libmime/mime_headers.h @@ -76,7 +76,7 @@ void rspamd_mime_headers_process (struct rspamd_task *task, GHashTable *target, * @return */ gchar * rspamd_mime_header_decode (rspamd_mempool_t *pool, const gchar *in, - gsize inlen); + gsize inlen, gboolean *invalid_utf); /** * Encode mime header if needed diff --git a/src/libmime/mime_parser.c b/src/libmime/mime_parser.c index 0365a02b2..242c656f0 100644 --- a/src/libmime/mime_parser.c +++ b/src/libmime/mime_parser.c @@ -510,6 +510,7 @@ rspamd_mime_parse_normal_part (struct rspamd_task *task, g_assert_not_reached (); } + part->id = task->parts->len; g_ptr_array_add (task->parts, part); msg_debug_mime ("parsed data part %T/%T of length %z (%z orig), %s cte", &part->ct->type, &part->ct->subtype, part->parsed_data.len, @@ -562,17 +563,20 @@ rspamd_mime_process_multipart_node (struct rspamd_task *task, hdr_pos = rspamd_string_find_eoh (&str, &body_pos); } - if (multipart->specific.mp.children == NULL) { - multipart->specific.mp.children = g_ptr_array_sized_new (2); - } - npart = rspamd_mempool_alloc0 (task->task_pool, sizeof (struct rspamd_mime_part)); npart->parent_part = multipart; npart->raw_headers = g_hash_table_new_full (rspamd_strcase_hash, rspamd_strcase_equal, NULL, rspamd_ptr_array_free_hard); npart->headers_order = g_queue_new (); - g_ptr_array_add (multipart->specific.mp.children, npart); + + if (multipart) { + if (multipart->specific.mp->children == NULL) { + multipart->specific.mp->children = g_ptr_array_sized_new (2); + } + + g_ptr_array_add (multipart->specific.mp->children, npart); + } if (hdr_pos > 0 && hdr_pos < str.len) { npart->raw_headers_str = str.str; @@ -631,6 +635,10 @@ rspamd_mime_process_multipart_node (struct rspamd_task *task, if (sel->flags & RSPAMD_CONTENT_TYPE_MULTIPART) { st->nesting ++; g_ptr_array_add (st->stack, npart); + npart->specific.mp = rspamd_mempool_alloc0 (task->task_pool, + sizeof (struct rspamd_mime_multipart)); + memcpy (&npart->specific.mp->boundary, &sel->orig_boundary, + sizeof (rspamd_ftok_t)); ret = rspamd_mime_parse_multipart_part (task, npart, st, err); } else if (sel->flags & RSPAMD_CONTENT_TYPE_MESSAGE) { @@ -812,6 +820,7 @@ rspamd_mime_parse_multipart_part (struct rspamd_task *task, return RSPAMD_MIME_PARSE_NESTING; } + part->id = task->parts->len; g_ptr_array_add (task->parts, part); st->nesting ++; rspamd_mime_part_get_cte (task, part->raw_headers, part, FALSE); @@ -866,10 +875,25 @@ rspamd_mime_preprocess_cb (struct rspamd_multipattern *mp, task = st->task; if (G_LIKELY (p < end)) { - blen = rspamd_memcspn (p, "\r\n", end - p); + gboolean seen_non_dash = FALSE; + + blen = 0; - if (blen > 0) { + while (p < end) { + if (*p == '\r' || *p == '\n') { + break; + } + else if (*p != '-') { + seen_non_dash = TRUE; + } + + blen ++; + p ++; + } + + if (blen > 0 && seen_non_dash) { /* We have found something like boundary */ + p = text + match_pos; bend = p + blen - 1; if (*bend == '-') { @@ -901,7 +925,7 @@ rspamd_mime_preprocess_cb (struct rspamd_multipattern *mp, bend ++; } - b.boundary = p - st->start - 3; + b.boundary = p - st->start - 2; b.start = bend - st->start; if (closing) { @@ -1192,8 +1216,8 @@ rspamd_mime_parse_message (struct rspamd_task *task, } } - pbegin = part->parsed_data.begin; - plen = part->parsed_data.len; + pbegin = part->parsed_data.begin + body_pos; + plen = part->parsed_data.len - body_pos; hdrs = rspamd_message_get_header_from_hash (npart->raw_headers, task->task_pool, @@ -1245,6 +1269,10 @@ rspamd_mime_parse_message (struct rspamd_task *task, if (sel->flags & RSPAMD_CONTENT_TYPE_MULTIPART) { g_ptr_array_add (nst->stack, npart); nst->nesting ++; + npart->specific.mp = rspamd_mempool_alloc0 (task->task_pool, + sizeof (struct rspamd_mime_multipart)); + memcpy (&npart->specific.mp->boundary, &sel->orig_boundary, + sizeof (rspamd_ftok_t)); ret = rspamd_mime_parse_multipart_part (task, npart, nst, err); } else if (sel->flags & RSPAMD_CONTENT_TYPE_MESSAGE) { @@ -1262,6 +1290,58 @@ rspamd_mime_parse_message (struct rspamd_task *task, st->nesting --; } + /* Process leftovers for boundaries */ + if (nst->boundaries) { + struct rspamd_mime_boundary *boundary, *start_boundary = NULL, + *end_boundary = NULL; + goffset cur_offset = nst->pos - nst->start, + end_offset = st->end - st->start; + guint sel_idx = 0; + + for (;;) { + start_boundary = NULL; + + for (i = sel_idx; i < nst->boundaries->len; i++) { + boundary = &g_array_index (nst->boundaries, + struct rspamd_mime_boundary, i); + + if (boundary->start > cur_offset && + boundary->boundary < end_offset && + !RSPAMD_BOUNDARY_IS_CLOSED (boundary)) { + start_boundary = boundary; + sel_idx = i; + break; + } + } + + if (start_boundary) { + const gchar *start, *end; + + if (nst->boundaries->len > sel_idx + 1) { + end_boundary = &g_array_index (nst->boundaries, + struct rspamd_mime_boundary, sel_idx + 1); + end = nst->start + end_boundary->boundary; + } + else { + end = nst->end; + } + + sel_idx ++; + + start = nst->start + start_boundary->start; + + if (end > start && + (ret = rspamd_mime_process_multipart_node (task, st, + NULL, start, end, err)) != RSPAMD_MIME_PARSE_OK) { + return ret; + } + } + else { + break; + } + } + } + if (nst != st) { rspamd_mime_parse_stack_free (nst); } diff --git a/src/libserver/CMakeLists.txt b/src/libserver/CMakeLists.txt index ac2c123dc..ccedbcdb3 100644 --- a/src/libserver/CMakeLists.txt +++ b/src/libserver/CMakeLists.txt @@ -16,7 +16,7 @@ SET(LIBRSPAMDSERVERSRC ${CMAKE_CURRENT_SOURCE_DIR}/re_cache.c ${CMAKE_CURRENT_SOURCE_DIR}/roll_history.c ${CMAKE_CURRENT_SOURCE_DIR}/spf.c - ${CMAKE_CURRENT_SOURCE_DIR}/symbols_cache.c + ${CMAKE_CURRENT_SOURCE_DIR}/rspamd_symcache.c ${CMAKE_CURRENT_SOURCE_DIR}/task.c ${CMAKE_CURRENT_SOURCE_DIR}/url.c ${CMAKE_CURRENT_SOURCE_DIR}/worker_util.c) diff --git a/src/libserver/cfg_file.h b/src/libserver/cfg_file.h index cb3581e97..6d6ed6f62 100644 --- a/src/libserver/cfg_file.h +++ b/src/libserver/cfg_file.h @@ -20,7 +20,7 @@ #include "config.h" #include "mem_pool.h" #include "upstream.h" -#include "symbols_cache.h" +#include "rspamd_symcache.h" #include "cfg_rcl.h" #include "ucl.h" #include "regexp.h" @@ -174,6 +174,8 @@ struct rspamd_classifier_config { gchar *name; /**< unique name of classifier */ guint32 min_tokens; /**< minimal number of tokens to process classifier */ guint32 max_tokens; /**< maximum number of tokens */ + guint min_token_hits; /**< minimum number of hits for a token to be considered */ + gdouble min_prob_strength; /**< use only tokens with probability in [0.5 - MPS, 0.5 + MPS] */ guint min_learns; /**< minimum number of learns for each statfile */ guint flags; }; @@ -315,14 +317,15 @@ struct rspamd_config { gboolean disable_pcre_jit; /**< Disable pcre JIT */ gboolean disable_lua_squeeze; /**< Disable lua rules squeezing */ gboolean own_lua_state; /**< True if we have created lua_state internally */ + gboolean soft_reject_on_timeout; /**< If true emit soft reject on task timeout (if not reject) */ - gsize max_diff; /**< maximum diff size for text parts */ gsize max_cores_size; /**< maximum size occupied by rspamd core files */ gsize max_cores_count; /**< maximum number of core files */ gchar *cores_dir; /**< directory for core files */ gsize max_message; /**< maximum size for messages */ gsize max_pic_size; /**< maximum size for a picture to process */ gsize images_cache_size; /**< size of LRU cache for DCT data from images */ + gdouble task_timeout; /**< maximum message processing time */ gint default_max_shots; /**< default maximum count of symbols hits permitted (-1 for unlimited) */ enum rspamd_log_type log_type; /**< log type */ @@ -334,7 +337,6 @@ struct rspamd_config { guint32 log_buf_size; /**< length of log buffer */ const ucl_object_t *debug_ip_map; /**< turn on debugging for specified ip addresses */ gboolean log_urls; /**< whether we should log URLs */ - GList *debug_symbols; /**< symbols to debug */ GHashTable *debug_modules; /**< logging modules to debug */ struct rspamd_cryptobox_pubkey *log_encryption_key; /**< encryption key for logs */ guint log_flags; /**< logging flags */ @@ -375,7 +377,7 @@ struct rspamd_config { gdouble monitored_interval; /**< interval between monitored checks */ gboolean disable_monitored; /**< disable monitoring completely */ - struct symbols_cache *cache; /**< symbols cache object */ + struct rspamd_symcache *cache; /**< symbols cache object */ gchar *cache_filename; /**< filename of cache file */ gdouble cache_reload_time; /**< how often cache reload should be performed */ gchar * checksum; /**< real checksum of config file */ diff --git a/src/libserver/cfg_rcl.c b/src/libserver/cfg_rcl.c index 3be0a655c..d6aadc03e 100644 --- a/src/libserver/cfg_rcl.c +++ b/src/libserver/cfg_rcl.c @@ -381,40 +381,108 @@ rspamd_rcl_symbol_handler (rspamd_mempool_t *pool, const ucl_object_t *obj, nshots = cfg->default_max_shots; if ((elt = ucl_object_lookup (obj, "one_shot")) != NULL) { + if (ucl_object_type (elt) != UCL_BOOLEAN) { + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "one_shot attribute is not boolean for symbol: '%s'", + key); + + return FALSE; + } if (ucl_object_toboolean (elt)) { nshots = 1; } } if ((elt = ucl_object_lookup (obj, "any_shot")) != NULL) { + if (ucl_object_type (elt) != UCL_BOOLEAN) { + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "any_shot attribute is not boolean for symbol: '%s'", + key); + + return FALSE; + } if (ucl_object_toboolean (elt)) { nshots = -1; } } if ((elt = ucl_object_lookup (obj, "one_param")) != NULL) { + if (ucl_object_type (elt) != UCL_BOOLEAN) { + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "one_param attribute is not boolean for symbol: '%s'", + key); + + return FALSE; + } + if (ucl_object_toboolean (elt)) { flags |= RSPAMD_SYMBOL_FLAG_ONEPARAM; } } if ((elt = ucl_object_lookup (obj, "ignore")) != NULL) { + if (ucl_object_type (elt) != UCL_BOOLEAN) { + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "ignore attribute is not boolean for symbol: '%s'", + key); + + return FALSE; + } + if (ucl_object_toboolean (elt)) { flags |= RSPAMD_SYMBOL_FLAG_IGNORE; } } if ((elt = ucl_object_lookup (obj, "nshots")) != NULL) { + if (ucl_object_type (elt) != UCL_FLOAT && ucl_object_type (elt) != UCL_INT) { + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "nshots attribute is not numeric for symbol: '%s'", + key); + + return FALSE; + } + nshots = ucl_object_toint (elt); } elt = ucl_object_lookup_any (obj, "score", "weight", NULL); if (elt) { + if (ucl_object_type (elt) != UCL_FLOAT && ucl_object_type (elt) != UCL_INT) { + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "score attribute is not numeric for symbol: '%s'", + key); + + return FALSE; + } + score = ucl_object_todouble (elt); } elt = ucl_object_lookup (obj, "priority"); if (elt) { + if (ucl_object_type (elt) != UCL_FLOAT && ucl_object_type (elt) != UCL_INT) { + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "priority attribute is not numeric for symbol: '%s'", + key); + + return FALSE; + } + priority = ucl_object_toint (elt); } else { @@ -1223,7 +1291,7 @@ rspamd_rcl_composite_handler (rspamd_mempool_t *pool, composite); if (new) { - rspamd_symbols_cache_add_symbol (cfg->cache, composite_name, 0, + rspamd_symcache_add_symbol (cfg->cache, composite_name, 0, NULL, NULL, SYMBOL_TYPE_COMPOSITE, -1); } @@ -1473,12 +1541,6 @@ rspamd_rcl_config_init (struct rspamd_config *cfg, GHashTable *skip_sections) 0, "Enable debugging log for the specified IP addresses"); rspamd_rcl_add_default_handler (sub, - "debug_symbols", - rspamd_rcl_parse_struct_string_list, - G_STRUCT_OFFSET (struct rspamd_config, debug_symbols), - 0, - "Enable debug for the specified symbols"); - rspamd_rcl_add_default_handler (sub, "debug_modules", rspamd_rcl_parse_struct_string_list, G_STRUCT_OFFSET (struct rspamd_config, debug_modules), @@ -1671,12 +1733,6 @@ rspamd_rcl_config_init (struct rspamd_config *cfg, GHashTable *skip_sections) 0, "List of internal filters enabled"); rspamd_rcl_add_default_handler (sub, - "max_diff", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, max_diff), - RSPAMD_CL_FLAG_INT_SIZE, - "Legacy option, do not use"); - rspamd_rcl_add_default_handler (sub, "map_watch_interval", rspamd_rcl_parse_struct_time, G_STRUCT_OFFSET (struct rspamd_config, map_timeout), @@ -1922,6 +1978,24 @@ rspamd_rcl_config_init (struct rspamd_config *cfg, GHashTable *skip_sections) G_STRUCT_OFFSET (struct rspamd_config, max_sessions_cache), 0, "Maximum number of sessions in cache before warning (default: 100)"); + rspamd_rcl_add_default_handler (sub, + "task_timeout", + rspamd_rcl_parse_struct_time, + G_STRUCT_OFFSET (struct rspamd_config, task_timeout), + RSPAMD_CL_FLAG_TIME_FLOAT, + "Maximum time for checking a message"); + rspamd_rcl_add_default_handler (sub, + "soft_reject_on_timeout", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_config, soft_reject_on_timeout), + 0, + "Emit soft reject if task timeout takes place"); + rspamd_rcl_add_default_handler (sub, + "check_timeout", + rspamd_rcl_parse_struct_time, + G_STRUCT_OFFSET (struct rspamd_config, task_timeout), + RSPAMD_CL_FLAG_TIME_FLOAT, + "Maximum time for checking a message (alias for task_timeout)"); /* Neighbours configuration */ rspamd_rcl_add_section_doc (&sub->subsections, "neighbours", "name", @@ -2148,6 +2222,18 @@ rspamd_rcl_config_init (struct rspamd_config *cfg, GHashTable *skip_sections) RSPAMD_CL_FLAG_INT_32, "Minimum count of tokens (words) to be considered for statistics"); rspamd_rcl_add_default_handler (sub, + "min_token_hits", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_classifier_config, min_token_hits), + RSPAMD_CL_FLAG_UINT, + "Minimum number of hits for a token to be considered"); + rspamd_rcl_add_default_handler (sub, + "min_prob_strength", + rspamd_rcl_parse_struct_double, + G_STRUCT_OFFSET (struct rspamd_classifier_config, min_token_hits), + 0, + "Use only tokens with probability in [0.5 - MPS, 0.5 + MPS]"); + rspamd_rcl_add_default_handler (sub, "max_tokens", rspamd_rcl_parse_struct_integer, G_STRUCT_OFFSET (struct rspamd_classifier_config, max_tokens), @@ -2509,7 +2595,8 @@ rspamd_rcl_parse_struct_string (rspamd_mempool_t *pool, break; case UCL_BOOLEAN: *target = rspamd_mempool_alloc (pool, num_str_len); - rspamd_snprintf (*target, num_str_len, "%B", (gboolean)obj->value.iv); + rspamd_snprintf (*target, num_str_len, "%s", + ((gboolean)obj->value.iv) ? "true" : "false"); break; default: g_set_error (err, @@ -2869,7 +2956,8 @@ rspamd_rcl_parse_struct_string_list (rspamd_mempool_t *pool, break; case UCL_BOOLEAN: val = rspamd_mempool_alloc (pool, num_str_len); - rspamd_snprintf (val, num_str_len, "%B", (gboolean)cur->value.iv); + rspamd_snprintf (val, num_str_len, "%s", + ((gboolean)cur->value.iv) ? "true" : "false"); break; default: g_set_error (err, @@ -3440,6 +3528,7 @@ rspamd_config_parse_ucl (struct rspamd_config *cfg, const gchar *filename, parser = ucl_parser_new (UCL_PARSER_SAVE_COMMENTS); rspamd_ucl_add_conf_variables (parser, vars); rspamd_ucl_add_conf_macros (parser, cfg); + ucl_parser_set_filevars (parser, filename, true); if (decrypt_keypair) { struct ucl_parser_special_handler *decrypt_handler; diff --git a/src/libserver/cfg_utils.c b/src/libserver/cfg_utils.c index ff1e77d15..2125a2854 100644 --- a/src/libserver/cfg_utils.c +++ b/src/libserver/cfg_utils.c @@ -47,6 +47,8 @@ #define DEFAULT_MAX_SHOTS 100 #define DEFAULT_MAX_SESSIONS 100 #define DEFAULT_MAX_WORKERS 4 +/* Timeout for task processing */ +#define DEFAULT_TASK_TIMEOUT 8.0 struct rspamd_ucl_map_cbdata { struct rspamd_config *cfg; @@ -131,8 +133,9 @@ rspamd_config_new (enum rspamd_config_init_flags flags) /* 16 sockets per DNS server */ cfg->dns_io_per_server = 16; - /* 20 Kb */ - cfg->max_diff = 20480; + /* Disable timeout */ + cfg->task_timeout = DEFAULT_TASK_TIMEOUT; + rspamd_config_init_metric (cfg); cfg->composite_symbols = @@ -179,7 +182,7 @@ rspamd_config_new (enum rspamd_config_init_flags flags) cfg->lua_thread_pool = lua_thread_pool_new (cfg->lua_state); } - cfg->cache = rspamd_symbols_cache_new (cfg); + cfg->cache = rspamd_symcache_new (cfg); cfg->ups_ctx = rspamd_upstreams_library_init (); cfg->re_cache = rspamd_re_cache_new (); cfg->doc_strings = ucl_object_typed_new (UCL_OBJECT); @@ -228,7 +231,7 @@ rspamd_config_free (struct rspamd_config *cfg) g_list_free (cfg->classifiers); g_list_free (cfg->workers); - rspamd_symbols_cache_destroy (cfg->cache); + rspamd_symcache_destroy (cfg->cache); ucl_object_unref (cfg->rcl_obj); ucl_object_unref (cfg->config_comments); ucl_object_unref (cfg->doc_strings); @@ -804,7 +807,7 @@ rspamd_config_post_load (struct rspamd_config *cfg, lua_settop (L, err_idx - 1); /* Init config cache */ - rspamd_symbols_cache_init (cfg->cache); + rspamd_symcache_init (cfg->cache); /* Init re cache */ rspamd_re_cache_init (cfg->re_cache, cfg); @@ -861,7 +864,7 @@ rspamd_config_post_load (struct rspamd_config *cfg, ret = FALSE; } - ret = rspamd_symbols_cache_validate (cfg->cache, cfg, FALSE) && ret; + ret = rspamd_symcache_validate (cfg->cache, cfg, FALSE) && ret; } if (opts & RSPAMD_CONFIG_INIT_PRELOAD_MAPS) { @@ -971,7 +974,10 @@ rspamd_config_new_classifier (struct rspamd_config *cfg, c = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_classifier_config)); + c->min_prob_strength = 0.05; + c->min_token_hits = 2; } + if (c->labels == NULL) { c->labels = g_hash_table_new_full (rspamd_str_hash, rspamd_str_equal, @@ -1229,8 +1235,8 @@ symbols_classifiers_callback (gpointer key, gpointer value, gpointer ud) struct rspamd_config *cfg = ud; /* Actually, statistics should act like any ordinary symbol */ - rspamd_symbols_cache_add_symbol (cfg->cache, key, 0, NULL, NULL, - SYMBOL_TYPE_CLASSIFIER|SYMBOL_TYPE_NOSTAT, -1); + rspamd_symcache_add_symbol (cfg->cache, key, 0, NULL, NULL, + SYMBOL_TYPE_CLASSIFIER | SYMBOL_TYPE_NOSTAT, -1); } void @@ -1467,6 +1473,7 @@ rspamd_init_filters (struct rspamd_config *cfg, bool reconfig) module_t *mod, **pmod; guint i = 0; struct module_ctx *mod_ctx, *cur_ctx; + gboolean ret = TRUE; /* Init all compiled modules */ @@ -1501,11 +1508,19 @@ rspamd_init_filters (struct rspamd_config *cfg, bool reconfig) mod_ctx->enabled = rspamd_config_is_module_enabled (cfg, mod->name); if (reconfig) { - (void)mod->module_reconfig_func (cfg); - msg_info_config ("reconfig of %s", mod->name); + if (!mod->module_reconfig_func (cfg)) { + msg_err_config ("reconfig of %s failed!", mod->name); + } + else { + msg_info_config ("reconfig of %s", mod->name); + } + } else { - (void)mod->module_config_func (cfg); + if (!mod->module_config_func (cfg)) { + msg_info_config ("config of %s failed!", mod->name); + ret = FALSE; + } } } @@ -1516,7 +1531,9 @@ rspamd_init_filters (struct rspamd_config *cfg, bool reconfig) cur = g_list_next (cur); } - return rspamd_init_lua_filters (cfg, 0); + ret = rspamd_init_lua_filters (cfg, 0) && ret; + + return ret; } static void @@ -1555,11 +1572,13 @@ rspamd_config_new_symbol (struct rspamd_config *cfg, const gchar *symbol, /* Search for symbol group */ if (group == NULL) { group = "ungrouped"; - } - - if (strcmp (group, "ungrouped") == 0) { sym_def->flags |= RSPAMD_SYMBOL_FLAG_UNGROUPPED; } + else { + if (strcmp (group, "ungrouped") == 0) { + sym_def->flags |= RSPAMD_SYMBOL_FLAG_UNGROUPPED; + } + } sym_group = g_hash_table_lookup (cfg->groups, group); if (sym_group == NULL) { @@ -1569,6 +1588,10 @@ rspamd_config_new_symbol (struct rspamd_config *cfg, const gchar *symbol, sym_def->gr = sym_group; g_hash_table_insert (sym_group->symbols, sym_def->name, sym_def); + + if (!(sym_def->flags & RSPAMD_SYMBOL_FLAG_UNGROUPPED)) { + g_ptr_array_add (sym_def->groups, sym_group); + } } diff --git a/src/libserver/composites.c b/src/libserver/composites.c index 308383605..f46f8276d 100644 --- a/src/libserver/composites.c +++ b/src/libserver/composites.c @@ -424,7 +424,7 @@ composites_foreach_callback (gpointer key, gpointer value, void *data) task = cd->task; if (!isset (cd->checked, cd->composite->id * 2)) { - if (rspamd_symbols_cache_is_checked (cd->task, cd->task->cfg->cache, + if (rspamd_symcache_is_checked (cd->task, cd->task->cfg->cache, key)) { msg_debug_composites ("composite %s is checked in symcache but not " "in composites bitfield", cd->composite->sym); diff --git a/src/libserver/dkim.c b/src/libserver/dkim.c index 6d3ace1df..c70b0bbc0 100644 --- a/src/libserver/dkim.c +++ b/src/libserver/dkim.c @@ -36,6 +36,8 @@ #define DKIM_CANON_DEFAULT DKIM_CANON_SIMPLE +#define RSPAMD_SHORT_BH_LEN 8 + /* Params */ enum rspamd_dkim_param_type { DKIM_PARAM_UNKNOWN = -1, @@ -111,6 +113,7 @@ enum rspamd_arc_seal_cv { RSPAMD_ARC_PASS }; + struct rspamd_dkim_context_s { struct rspamd_dkim_common_ctx common; rspamd_mempool_t *pool; @@ -123,6 +126,7 @@ struct rspamd_dkim_context_s { gchar *domain; gchar *selector; gint8 *b; + gchar *short_b; gint8 *bh; gchar *dns_key; enum rspamd_arc_seal_cv cv; @@ -264,6 +268,8 @@ rspamd_dkim_parse_signature (rspamd_dkim_context_t * ctx, GError **err) { ctx->b = rspamd_mempool_alloc0 (ctx->pool, len); + ctx->short_b = rspamd_mempool_alloc0 (ctx->pool, RSPAMD_SHORT_BH_LEN + 1); + rspamd_strlcpy (ctx->short_b, param, MIN (len, RSPAMD_SHORT_BH_LEN + 1)); (void)rspamd_cryptobox_base64_decode (param, len, ctx->b, &ctx->blen); return TRUE; @@ -2255,7 +2261,7 @@ rspamd_dkim_check_bh_cached (struct rspamd_dkim_common_ctx *ctx, * @param task task to check * @return */ -enum rspamd_dkim_check_result +struct rspamd_dkim_check_result * rspamd_dkim_check (rspamd_dkim_context_t *ctx, rspamd_dkim_key_t *key, struct rspamd_task *task) @@ -2265,21 +2271,30 @@ rspamd_dkim_check (rspamd_dkim_context_t *ctx, struct rspamd_dkim_cached_hash *cached_bh = NULL; EVP_MD_CTX *cpy_ctx = NULL; gsize dlen = 0; - enum rspamd_dkim_check_result res = DKIM_CONTINUE; + struct rspamd_dkim_check_result *res; guint i; struct rspamd_dkim_header *dh; gint nid; - g_return_val_if_fail (ctx != NULL, DKIM_ERROR); - g_return_val_if_fail (key != NULL, DKIM_ERROR); - g_return_val_if_fail (task->msg.len > 0, DKIM_ERROR); + g_return_val_if_fail (ctx != NULL, NULL); + g_return_val_if_fail (key != NULL, NULL); + g_return_val_if_fail (task->msg.len > 0, NULL); /* First of all find place of body */ body_end = task->msg.begin + task->msg.len; body_start = task->raw_headers_content.body_start; + res = rspamd_mempool_alloc0 (task->task_pool, sizeof (*res)); + res->ctx = ctx; + res->selector = ctx->selector; + res->domain = ctx->domain; + res->fail_reason = NULL; + res->short_b = ctx->short_b; + res->rcode = DKIM_CONTINUE; + if (!body_start) { - return DKIM_RECORD_ERROR; + res->rcode = DKIM_ERROR; + return res; } if (ctx->common.type != RSPAMD_DKIM_ARC_SEAL) { @@ -2291,7 +2306,8 @@ rspamd_dkim_check (rspamd_dkim_context_t *ctx, /* Start canonization of body part */ if (!rspamd_dkim_canonize_body (&ctx->common, body_start, body_end, FALSE)) { - return DKIM_RECORD_ERROR; + res->rcode = DKIM_RECORD_ERROR; + return res; } } } @@ -2380,8 +2396,11 @@ rspamd_dkim_check (rspamd_dkim_context_t *ctx, #else EVP_MD_CTX_reset (cpy_ctx); #endif + res->fail_reason = "body hash did not verify"; + res->rcode = DKIM_REJECT; EVP_MD_CTX_destroy (cpy_ctx); - return DKIM_REJECT; + + return res; } } } @@ -2398,11 +2417,18 @@ rspamd_dkim_check (rspamd_dkim_context_t *ctx, dlen, ctx->bh, dlen, cached_bh->digest_cr); - return DKIM_REJECT; + res->fail_reason = "body hash did not verify"; + res->rcode = DKIM_REJECT; + + return res; } } else { - return DKIM_REJECT; + + res->fail_reason = "body hash did not verify"; + res->rcode = DKIM_REJECT; + + return res; } } } @@ -2411,8 +2437,10 @@ rspamd_dkim_check (rspamd_dkim_context_t *ctx, "bh value mismatch: %*xs versus %*xs", dlen, ctx->bh, dlen, cached_bh->digest_normal); + res->fail_reason = "body hash did not verify"; + res->rcode = DKIM_REJECT; - return DKIM_REJECT; + return res; } } @@ -2452,21 +2480,24 @@ rspamd_dkim_check (rspamd_dkim_context_t *ctx, if (RSA_verify (nid, raw_digest, dlen, ctx->b, ctx->blen, key->key.key_rsa) != 1) { msg_debug_dkim ("rsa verify failed"); - res = DKIM_REJECT; + res->rcode = DKIM_REJECT; + res->fail_reason = "rsa verify failed"; } break; case RSPAMD_DKIM_KEY_ECDSA: if (ECDSA_verify (nid, raw_digest, dlen, ctx->b, ctx->blen, key->key.key_ecdsa) != 1) { msg_debug_dkim ("ecdsa verify failed"); - res = DKIM_REJECT; + res->rcode = DKIM_REJECT; + res->fail_reason = "ecdsa verify failed"; } break; case RSPAMD_DKIM_KEY_EDDSA: if (!rspamd_cryptobox_verify (ctx->b, ctx->blen, raw_digest, dlen, key->key.key_eddsa, RSPAMD_CRYPTOBOX_MODE_25519)) { msg_debug_dkim ("eddsa verify failed"); - res = DKIM_REJECT; + res->rcode = DKIM_REJECT; + res->fail_reason = "eddsa verify failed"; } break; } @@ -2476,11 +2507,13 @@ rspamd_dkim_check (rspamd_dkim_context_t *ctx, switch (ctx->cv) { case RSPAMD_ARC_INVALID: msg_info_dkim ("arc seal is invalid i=%d", ctx->common.idx); - res = DKIM_PERM_ERROR; + res->rcode = DKIM_PERM_ERROR; + res->fail_reason = "arc seal is invalid"; break; case RSPAMD_ARC_FAIL: msg_info_dkim ("arc seal failed i=%d", ctx->common.idx); - res = DKIM_REJECT; + res->rcode = DKIM_REJECT; + res->fail_reason = "arc seal failed"; break; default: break; @@ -2490,6 +2523,24 @@ rspamd_dkim_check (rspamd_dkim_context_t *ctx, return res; } +struct rspamd_dkim_check_result * +rspamd_dkim_create_result (rspamd_dkim_context_t *ctx, + enum rspamd_dkim_check_rcode rcode, + struct rspamd_task *task) +{ + struct rspamd_dkim_check_result *res; + + res = rspamd_mempool_alloc0 (task->task_pool, sizeof (*res)); + res->ctx = ctx; + res->selector = ctx->selector; + res->domain = ctx->domain; + res->fail_reason = NULL; + res->short_b = ctx->short_b; + res->rcode = rcode; + + return res; +} + rspamd_dkim_key_t * rspamd_dkim_key_ref (rspamd_dkim_key_t *k) { @@ -2528,6 +2579,16 @@ rspamd_dkim_get_domain (rspamd_dkim_context_t *ctx) return NULL; } +const gchar* +rspamd_dkim_get_selector (rspamd_dkim_context_t *ctx) +{ + if (ctx) { + return ctx->selector; + } + + return NULL; +} + guint rspamd_dkim_key_get_ttl (rspamd_dkim_key_t *k) { diff --git a/src/libserver/dkim.h b/src/libserver/dkim.h index 6ce099464..46953a21c 100644 --- a/src/libserver/dkim.h +++ b/src/libserver/dkim.h @@ -81,7 +81,7 @@ #define DKIM_SIGERROR_EMPTY_V 45 /* v= tag empty */ /* Check results */ -enum rspamd_dkim_check_result { +enum rspamd_dkim_check_rcode { DKIM_CONTINUE = 0, DKIM_REJECT, DKIM_TRYAGAIN, @@ -137,6 +137,16 @@ enum rspamd_dkim_key_type { RSPAMD_DKIM_KEY_EDDSA }; +struct rspamd_dkim_check_result { + enum rspamd_dkim_check_rcode rcode; + rspamd_dkim_context_t *ctx; + /* Processed parts */ + const gchar *selector; + const gchar *domain; + const gchar *short_b; + const gchar *fail_reason; +}; + /* Err MUST be freed if it is not NULL, key is allocated by slice allocator */ typedef void (*dkim_key_handler_f)(rspamd_dkim_key_t *key, gsize keylen, @@ -209,19 +219,30 @@ gboolean rspamd_get_dkim_key (rspamd_dkim_context_t *ctx, * @param task task to check * @return */ -enum rspamd_dkim_check_result rspamd_dkim_check (rspamd_dkim_context_t *ctx, - rspamd_dkim_key_t *key, - struct rspamd_task *task); - -GString *rspamd_dkim_sign (struct rspamd_task *task, const gchar *selector, - const gchar *domain, time_t expire, gsize len, guint idx, - const gchar *arc_cv, rspamd_dkim_sign_context_t *ctx); +struct rspamd_dkim_check_result * rspamd_dkim_check (rspamd_dkim_context_t *ctx, + rspamd_dkim_key_t *key, + struct rspamd_task *task); + +struct rspamd_dkim_check_result * +rspamd_dkim_create_result (rspamd_dkim_context_t *ctx, + enum rspamd_dkim_check_rcode rcode, + struct rspamd_task *task); + +GString *rspamd_dkim_sign (struct rspamd_task *task, + const gchar *selector, + const gchar *domain, + time_t expire, + gsize len, + guint idx, + const gchar *arc_cv, + rspamd_dkim_sign_context_t *ctx); rspamd_dkim_key_t * rspamd_dkim_key_ref (rspamd_dkim_key_t *k); void rspamd_dkim_key_unref (rspamd_dkim_key_t *k); rspamd_dkim_sign_key_t * rspamd_dkim_sign_key_ref (rspamd_dkim_sign_key_t *k); void rspamd_dkim_sign_key_unref (rspamd_dkim_sign_key_t *k); const gchar* rspamd_dkim_get_domain (rspamd_dkim_context_t *ctx); +const gchar* rspamd_dkim_get_selector (rspamd_dkim_context_t *ctx); const gchar* rspamd_dkim_get_dns_key (rspamd_dkim_context_t *ctx); guint rspamd_dkim_key_get_ttl (rspamd_dkim_key_t *k); diff --git a/src/libserver/dns.c b/src/libserver/dns.c index 4f8d70648..17b84884a 100644 --- a/src/libserver/dns.c +++ b/src/libserver/dns.c @@ -24,6 +24,8 @@ #include "rdns_event.h" #include "unix-std.h" +static const gchar *M = "rspamd dns"; + static struct rdns_upstream_elt* rspamd_dns_select_upstream (const char *name, size_t len, void *ups_data); static struct rdns_upstream_elt* rspamd_dns_select_upstream_retransmit ( @@ -49,6 +51,8 @@ struct rspamd_dns_request_ud { dns_callback_type cb; gpointer ud; rspamd_mempool_t *pool; + struct rspamd_task *task; + struct rspamd_symcache_item *item; struct rdns_request *req; struct rdns_reply *reply; }; @@ -58,6 +62,10 @@ rspamd_dns_fin_cb (gpointer arg) { struct rspamd_dns_request_ud *reqdata = (struct rspamd_dns_request_ud *)arg; + if (reqdata->item) { + rspamd_symcache_set_cur_item (reqdata->task, reqdata->item); + } + if (reqdata->reply) { reqdata->cb (reqdata->reply, reqdata->ud); } @@ -75,6 +83,11 @@ rspamd_dns_fin_cb (gpointer arg) rdns_request_release (reqdata->req); + if (reqdata->item) { + rspamd_symcache_item_async_dec_check (reqdata->task, + reqdata->item, M); + } + if (reqdata->pool == NULL) { g_free (reqdata); } @@ -104,7 +117,7 @@ rspamd_dns_callback (struct rdns_reply *reply, gpointer ud) } } -gboolean +struct rspamd_dns_request_ud * make_dns_request (struct rspamd_dns_resolver *resolver, struct rspamd_async_session *session, rspamd_mempool_t *pool, @@ -119,11 +132,11 @@ make_dns_request (struct rspamd_dns_resolver *resolver, g_assert (resolver != NULL); if (resolver->r == NULL) { - return FALSE; + return NULL; } if (session && rspamd_session_blocked (session)) { - return FALSE; + return NULL; } if (pool != NULL) { @@ -146,8 +159,10 @@ make_dns_request (struct rspamd_dns_resolver *resolver, if (session) { if (req != NULL) { - rspamd_session_add_event (session, NULL, (event_finalizer_t) rspamd_dns_fin_cb, reqdata, - g_quark_from_static_string ("dns resolver")); + rspamd_session_add_event (session, + (event_finalizer_t) rspamd_dns_fin_cb, + reqdata, + M); } } @@ -155,10 +170,11 @@ make_dns_request (struct rspamd_dns_resolver *resolver, if (pool == NULL) { g_free (reqdata); } - return FALSE; + + return NULL; } - return TRUE; + return reqdata; } static gboolean @@ -169,25 +185,35 @@ make_dns_request_task_common (struct rspamd_task *task, const char *name, gboolean forced) { - gboolean ret; + struct rspamd_dns_request_ud *reqdata; if (!forced && task->dns_requests >= task->cfg->dns_max_requests) { return FALSE; } - ret = make_dns_request (task->resolver, task->s, task->task_pool, cb, ud, + reqdata = make_dns_request (task->resolver, task->s, task->task_pool, cb, ud, type, name); - if (ret) { + if (reqdata) { task->dns_requests ++; + reqdata->task = task; + reqdata->item = rspamd_symcache_get_cur_item (task); + + if (reqdata->item) { + /* We are inside some session */ + rspamd_symcache_item_async_inc (task, reqdata->item, M); + } + if (!forced && task->dns_requests >= task->cfg->dns_max_requests) { msg_info_task ("<%s> stop resolving on reaching %ud requests", task->message_id, task->dns_requests); } + + return TRUE; } - return ret; + return FALSE; } gboolean @@ -473,7 +499,12 @@ rspamd_dns_resolver_config_ucl (struct rspamd_config *cfg, ucl_object_iterate_free (rep_it); if (replies) { - msg_info_config ("added fake record: %s(%s)", name, rdns_str_from_type (rtype)); + struct rdns_reply_entry *tmp_entry; + guint i = 0; + DL_COUNT (replies, tmp_entry, i); + + msg_info_config ("added fake record: %s(%s); %d replies", name, + rdns_str_from_type (rtype), i); rdns_resolver_set_fake_reply (dns_resolver->r, name, rtype, rcode, replies); } diff --git a/src/libserver/dns.h b/src/libserver/dns.h index 52325af91..c48e4b09c 100644 --- a/src/libserver/dns.h +++ b/src/libserver/dns.h @@ -43,6 +43,7 @@ struct rspamd_dns_resolver { struct rspamd_dns_resolver * dns_resolver_init (rspamd_logger_t *logger, struct event_base *ev_base, struct rspamd_config *cfg); +struct rspamd_dns_request_ud; /** * Make a DNS request * @param resolver resolver object @@ -54,7 +55,7 @@ struct rspamd_dns_resolver * dns_resolver_init (rspamd_logger_t *logger, * @param ... string or ip address based on a request type * @return TRUE if request was sent. */ -gboolean make_dns_request (struct rspamd_dns_resolver *resolver, +struct rspamd_dns_request_ud * make_dns_request (struct rspamd_dns_resolver *resolver, struct rspamd_async_session *session, rspamd_mempool_t *pool, dns_callback_type cb, diff --git a/src/libserver/events.c b/src/libserver/events.c index 325da4452..2fd0c55ff 100644 --- a/src/libserver/events.c +++ b/src/libserver/events.c @@ -19,11 +19,9 @@ #include "events.h" #include "cryptobox.h" -#define RSPAMD_SESSION_FLAG_WATCHING (1 << 0) #define RSPAMD_SESSION_FLAG_DESTROYING (1 << 1) #define RSPAMD_SESSION_FLAG_CLEANUP (1 << 2) -#define RSPAMD_SESSION_IS_WATCHING(s) ((s)->flags & RSPAMD_SESSION_FLAG_WATCHING) #define RSPAMD_SESSION_CAN_ADD_EVENT(s) (!((s)->flags & (RSPAMD_SESSION_FLAG_DESTROYING|RSPAMD_SESSION_FLAG_CLEANUP))) #define msg_err_session(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \ @@ -48,23 +46,12 @@ INIT_LOG_MODULE(events) /* Average symbols count to optimize hash allocation */ static struct rspamd_counter_data events_count; -struct rspamd_watch_stack { - event_watcher_t cb; - gpointer ud; - struct rspamd_watch_stack *next; -}; - -struct rspamd_async_watcher { - struct rspamd_watch_stack *st; - guint remain; - gint id; -}; struct rspamd_async_event { - GQuark subsystem; + const gchar *subsystem; + const gchar *loc; event_finalizer_t fin; void *user_data; - struct rspamd_async_watcher *w; }; static guint rspamd_event_hash (gconstpointer a); @@ -85,7 +72,6 @@ struct rspamd_async_session { khash_t(rspamd_events_hash) *events; void *user_data; rspamd_mempool_t *pool; - struct rspamd_async_watcher *cur_watcher; guint flags; }; @@ -160,11 +146,11 @@ rspamd_session_create (rspamd_mempool_t * pool, } struct rspamd_async_event * -rspamd_session_add_event (struct rspamd_async_session *session, - struct rspamd_async_watcher *w, - event_finalizer_t fin, - gpointer user_data, - GQuark subsystem) +rspamd_session_add_event_full (struct rspamd_async_session *session, + event_finalizer_t fin, + gpointer user_data, + const gchar *subsystem, + const gchar *loc) { struct rspamd_async_event *new_event; gint ret; @@ -177,7 +163,7 @@ rspamd_session_add_event (struct rspamd_async_session *session, if (!RSPAMD_SESSION_CAN_ADD_EVENT (session)) { msg_debug_session ("skip adding event subsystem: %s: " "session is destroying/cleaning", - g_quark_to_string (subsystem)); + subsystem); return NULL; } @@ -187,38 +173,14 @@ rspamd_session_add_event (struct rspamd_async_session *session, new_event->fin = fin; new_event->user_data = user_data; new_event->subsystem = subsystem; + new_event->loc = loc; - if (w == NULL) { - if (RSPAMD_SESSION_IS_WATCHING (session)) { - new_event->w = session->cur_watcher; - new_event->w->remain++; - msg_debug_session ("added event: %p, pending %d events, " - "subsystem: %s, watcher: %d (%d)", - user_data, - kh_size (session->events), - g_quark_to_string (subsystem), - new_event->w->id, - new_event->w->remain); - } else { - new_event->w = NULL; - msg_debug_session ("added event: %p, pending %d events, " - "subsystem: %s, no watcher!", - user_data, - kh_size (session->events), - g_quark_to_string (subsystem)); - } - } - else { - new_event->w = w; - new_event->w->remain++; - msg_debug_session ("added event: %p, pending %d events, " - "subsystem: %s, explicit watcher: %d (%d)", - user_data, - kh_size (session->events), - g_quark_to_string (subsystem), - new_event->w->id, - new_event->w->remain); - } + msg_debug_session ("added event: %p, pending %d (+1) events, " + "subsystem: %s (%s)", + user_data, + kh_size (session->events), + subsystem, + loc); kh_put (rspamd_events_hash, session->events, new_event, &ret); g_assert (ret > 0); @@ -226,23 +188,11 @@ rspamd_session_add_event (struct rspamd_async_session *session, return new_event; } -static inline void -rspamd_session_call_watcher_stack (struct rspamd_async_session *session, - struct rspamd_async_watcher *w) -{ - struct rspamd_watch_stack *st; - - LL_FOREACH (w->st, st) { - st->cb (session->user_data, st->ud); - } - - w->st = NULL; -} - void -rspamd_session_remove_event (struct rspamd_async_session *session, - event_finalizer_t fin, - void *ud) +rspamd_session_remove_event_full (struct rspamd_async_session *session, + event_finalizer_t fin, + void *ud, + const gchar *loc) { struct rspamd_async_event search_ev, *found_ev; khiter_t k; @@ -264,11 +214,13 @@ rspamd_session_remove_event (struct rspamd_async_session *session, if (k == kh_end (session->events)) { gchar t; - msg_err_session ("cannot find event: %p(%p)", fin, ud); + msg_err_session ("cannot find event: %p(%p) from %s", fin, ud, loc); kh_foreach (session->events, found_ev, t, { - msg_err_session ("existing event %s: %p(%p)", - g_quark_to_string (found_ev->subsystem), - found_ev->fin, found_ev->user_data); + msg_err_session ("existing event %s (%s): %p(%p)", + found_ev->subsystem, + found_ev->loc, + found_ev->fin, + found_ev->user_data); }); (void)t; @@ -277,32 +229,18 @@ rspamd_session_remove_event (struct rspamd_async_session *session, } found_ev = kh_key (session->events, k); + msg_debug_session ("removed event: %p, pending %d (-1) events, " + "subsystem: %s (%s), added at %s", + ud, + kh_size (session->events), + found_ev->subsystem, + loc, + found_ev->loc); kh_del (rspamd_events_hash, session->events, k); /* Remove event */ fin (ud); - /* Call watcher if needed */ - if (found_ev->w) { - msg_debug_session ("removed event: %p, subsystem: %s, " - "pending %d events, watcher: %d (%d pending)", ud, - g_quark_to_string (found_ev->subsystem), - kh_size (session->events), - found_ev->w->id, found_ev->w->remain - 1); - - if (found_ev->w->remain > 0) { - if (--found_ev->w->remain == 0) { - rspamd_session_call_watcher_stack (session, found_ev->w); - } - } - } - else { - msg_debug_session ("removed event: %p, subsystem: %s, " - "pending %d events, no watcher!", ud, - g_quark_to_string (found_ev->subsystem), - kh_size (session->events)); - } - rspamd_session_pending (session); } @@ -342,7 +280,7 @@ rspamd_session_cleanup (struct rspamd_async_session *session) /* Call event's finalizer */ msg_debug_session ("removed event on destroy: %p, subsystem: %s", ev->user_data, - g_quark_to_string (ev->subsystem)); + ev->subsystem); if (ev->fin != NULL) { ev->fin (ev->user_data); @@ -381,65 +319,6 @@ rspamd_session_pending (struct rspamd_async_session *session) return ret; } -void -rspamd_session_watch_start (struct rspamd_async_session *session, - gint id, - event_watcher_t cb, - gpointer ud) -{ - struct rspamd_watch_stack *st_elt; - - g_assert (session != NULL); - g_assert (!RSPAMD_SESSION_IS_WATCHING (session)); - - if (session->cur_watcher == NULL) { - session->cur_watcher = rspamd_mempool_alloc0 (session->pool, - sizeof (*session->cur_watcher)); - - st_elt = rspamd_mempool_alloc (session->pool, sizeof (*st_elt)); - st_elt->cb = cb; - st_elt->ud = ud; - LL_PREPEND (session->cur_watcher->st, st_elt); - } - else { - if (session->cur_watcher->st) { - /* Reuse the existing (empty) watcher */ - session->cur_watcher->st->cb = cb; - session->cur_watcher->st->ud = ud; - } - else { - st_elt = rspamd_mempool_alloc (session->pool, sizeof (*st_elt)); - st_elt->cb = cb; - st_elt->ud = ud; - LL_PREPEND (session->cur_watcher->st, st_elt); - } - } - - session->cur_watcher->id = id; - session->flags |= RSPAMD_SESSION_FLAG_WATCHING; -} - -guint -rspamd_session_watch_stop (struct rspamd_async_session *session) -{ - guint remain; - - g_assert (session != NULL); - g_assert (RSPAMD_SESSION_IS_WATCHING (session)); - - remain = session->cur_watcher->remain; - - if (remain > 0) { - /* Avoid reusing */ - session->cur_watcher = NULL; - } - - session->flags &= ~RSPAMD_SESSION_FLAG_WATCHING; - - return remain; -} - - guint rspamd_session_events_pending (struct rspamd_async_session *session) { @@ -450,123 +329,9 @@ rspamd_session_events_pending (struct rspamd_async_session *session) npending = kh_size (session->events); msg_debug_session ("pending %d events", npending); - if (RSPAMD_SESSION_IS_WATCHING (session)) { - npending += session->cur_watcher->remain; - msg_debug_session ("pending %d watchers, id: %d", - session->cur_watcher->remain, session->cur_watcher->id); - } - return npending; } -inline void -rspamd_session_watcher_push_callback (struct rspamd_async_session *session, - struct rspamd_async_watcher *w, - event_watcher_t cb, - gpointer ud) -{ - struct rspamd_watch_stack *st; - - g_assert (session != NULL); - - if (w == NULL) { - if (RSPAMD_SESSION_IS_WATCHING (session)) { - w = session->cur_watcher; - } - else { - return; - } - } - - if (w) { - w->remain ++; - msg_debug_session ("push session, watcher: %d, %d events", - w->id, - w->remain); - - if (cb) { - st = rspamd_mempool_alloc (session->pool, sizeof (*st)); - st->cb = cb; - st->ud = ud; - - LL_PREPEND (w->st, st); - } - } -} - -void -rspamd_session_watcher_push (struct rspamd_async_session *session) -{ - rspamd_session_watcher_push_callback (session, NULL, NULL, NULL); -} - -void -rspamd_session_watcher_push_specific (struct rspamd_async_session *session, - struct rspamd_async_watcher *w) -{ - rspamd_session_watcher_push_callback (session, w, NULL, NULL); -} - -void -rspamd_session_watcher_pop (struct rspamd_async_session *session, - struct rspamd_async_watcher *w) -{ - g_assert (session != NULL); - - if (w && w->remain > 0) { - msg_debug_session ("pop session, watcher: %d, %d events", w->id, - w->remain); - w->remain --; - - if (w->remain == 0) { - rspamd_session_call_watcher_stack (session, w); - } - } -} - -struct rspamd_async_watcher* -rspamd_session_get_watcher (struct rspamd_async_session *session) -{ - g_assert (session != NULL); - - if (RSPAMD_SESSION_IS_WATCHING (session)) { - return session->cur_watcher; - } - else { - return NULL; - } -} - -struct rspamd_async_watcher* -rspamd_session_replace_watcher (struct rspamd_async_session *s, - struct rspamd_async_watcher *w) -{ - struct rspamd_async_watcher *res = NULL; - - g_assert (s != NULL); - - if (s->cur_watcher) { - res = s->cur_watcher; - - if (!w) { - /* We remove watching, so clear watching flag as well */ - s->flags &= ~RSPAMD_SESSION_FLAG_WATCHING; - - } - - s->cur_watcher = w; - } - else { - if (w) { - s->flags |= RSPAMD_SESSION_FLAG_WATCHING; - } - - s->cur_watcher = w; - } - - return res; -} - rspamd_mempool_t * rspamd_session_mempool (struct rspamd_async_session *session) { diff --git a/src/libserver/events.h b/src/libserver/events.h index 68895d38d..7e0de8749 100644 --- a/src/libserver/events.h +++ b/src/libserver/events.h @@ -21,10 +21,8 @@ struct rspamd_async_event; struct rspamd_async_session; -struct rspamd_async_watcher; typedef void (*event_finalizer_t)(gpointer ud); -typedef void (*event_watcher_t)(gpointer session_data, gpointer ud); typedef gboolean (*session_finalizer_t)(gpointer user_data); /** @@ -48,11 +46,13 @@ struct rspamd_async_session * rspamd_session_create (rspamd_mempool_t *pool, * @param forced unused */ struct rspamd_async_event * -rspamd_session_add_event (struct rspamd_async_session *session, - struct rspamd_async_watcher *w, +rspamd_session_add_event_full (struct rspamd_async_session *session, event_finalizer_t fin, gpointer user_data, - GQuark subsystem); + const gchar *subsystem, + const gchar *loc); +#define rspamd_session_add_event(session, fin, user_data, subsystem) \ + rspamd_session_add_event_full(session, fin, user_data, subsystem, G_STRLOC) /** * Remove normal event @@ -60,9 +60,12 @@ rspamd_session_add_event (struct rspamd_async_session *session, * @param fin final callback * @param ud user data object */ -void rspamd_session_remove_event (struct rspamd_async_session *session, - event_finalizer_t fin, - gpointer ud); +void rspamd_session_remove_event_full (struct rspamd_async_session *session, + event_finalizer_t fin, + gpointer ud, + const gchar *loc); +#define rspamd_session_remove_event(session, fin, user_data) \ + rspamd_session_remove_event_full(session, fin, user_data, G_STRLOC) /** * Must be called at the end of session, it calls fin functions for all non-forced callbacks @@ -96,64 +99,6 @@ gboolean rspamd_session_pending (struct rspamd_async_session *session); */ guint rspamd_session_events_pending (struct rspamd_async_session *session); -/** - * Start watching for events in the session, so the specified watcher will be added - * to all subsequent events until `rspamd_session_watch_stop` is called - * @param s session object - * @param cb watcher callback that is called when all events watched are destroyed - * @param ud opaque data for the callback - */ -void rspamd_session_watch_start (struct rspamd_async_session *s, - gint id, - event_watcher_t cb, - gpointer ud); - -/** - * Stop watching mode, if no events are watched since the last `rspamd_session_watch_start`, - * then the watcher is silently ignored - * @param s session - * @return number of events watched - */ -guint rspamd_session_watch_stop (struct rspamd_async_session *s); - -/** - * Create a fake event just for event watcher - * @param s - */ -void rspamd_session_watcher_push (struct rspamd_async_session *s); - -/** - * Push callback to the watcher specified - */ -void rspamd_session_watcher_push_callback (struct rspamd_async_session *s, - struct rspamd_async_watcher *w, - event_watcher_t cb, - gpointer ud); - -/** - * Increase refcount for a specific watcher - */ -void rspamd_session_watcher_push_specific (struct rspamd_async_session *s, - struct rspamd_async_watcher *w); - -/** - * Remove a fake event from a watcher - * @param s - */ -void rspamd_session_watcher_pop (struct rspamd_async_session *s, - struct rspamd_async_watcher *w); - -/** - * Returns the current watcher for events session - * @param s - * @return - */ -struct rspamd_async_watcher* rspamd_session_get_watcher ( - struct rspamd_async_session *s); - -struct rspamd_async_watcher* rspamd_session_replace_watcher ( - struct rspamd_async_session *s, - struct rspamd_async_watcher *w); /** * Returns TRUE if an async session is currently destroying diff --git a/src/libserver/fuzzy_backend_redis.c b/src/libserver/fuzzy_backend_redis.c index 5081170b6..fbccac5ab 100644 --- a/src/libserver/fuzzy_backend_redis.c +++ b/src/libserver/fuzzy_backend_redis.c @@ -24,6 +24,7 @@ #include "upstream.h" #include "contrib/hiredis/hiredis.h" #include "contrib/hiredis/async.h" +#include "lua/lua_common.h" #define REDIS_DEFAULT_PORT 6379 #define REDIS_DEFAULT_OBJECT "fuzzy" @@ -49,14 +50,14 @@ INIT_LOG_MODULE(fuzzy_redis) struct rspamd_fuzzy_backend_redis { - struct upstream_list *read_servers; - struct upstream_list *write_servers; + lua_State *L; const gchar *redis_object; const gchar *password; const gchar *dbname; gchar *id; struct rspamd_redis_pool *pool; gdouble timeout; + gint conf_ref; ref_entry_t ref; }; @@ -98,6 +99,22 @@ struct rspamd_fuzzy_redis_session { guchar found_digest[rspamd_cryptobox_HASHBYTES]; }; +static inline struct upstream_list * +rspamd_redis_get_servers (struct rspamd_fuzzy_backend_redis *ctx, + const gchar *what) +{ + lua_State *L = ctx->L; + struct upstream_list *res; + + lua_rawgeti (L, LUA_REGISTRYINDEX, ctx->conf_ref); + lua_pushstring (L, what); + lua_gettable (L, -2); + res = *((struct upstream_list**)lua_touserdata (L, -1)); + lua_settop (L, 0); + + return res; +} + static inline void rspamd_fuzzy_redis_session_free_args (struct rspamd_fuzzy_redis_session *session) { @@ -136,97 +153,13 @@ rspamd_fuzzy_redis_session_dtor (struct rspamd_fuzzy_redis_session *session, g_free (session); } -static gboolean -rspamd_fuzzy_backend_redis_try_ucl (struct rspamd_fuzzy_backend_redis *backend, - const ucl_object_t *obj, - struct rspamd_config *cfg) -{ - const ucl_object_t *elt, *relt; - - elt = ucl_object_lookup_any (obj, "read_servers", "servers", NULL); - - if (elt == NULL) { - return FALSE; - } - - backend->read_servers = rspamd_upstreams_create (cfg->ups_ctx); - if (!rspamd_upstreams_from_ucl (backend->read_servers, elt, - REDIS_DEFAULT_PORT, NULL)) { - msg_err_config ("cannot get read servers configuration"); - return FALSE; - } - - relt = elt; - - elt = ucl_object_lookup (obj, "write_servers"); - if (elt == NULL) { - /* Use read servers as write ones */ - g_assert (relt != NULL); - backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx); - if (!rspamd_upstreams_from_ucl (backend->write_servers, relt, - REDIS_DEFAULT_PORT, NULL)) { - msg_err_config ("cannot get write servers configuration"); - return FALSE; - } - } - else { - backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx); - if (!rspamd_upstreams_from_ucl (backend->write_servers, elt, - REDIS_DEFAULT_PORT, NULL)) { - msg_err_config ("cannot get write servers configuration"); - rspamd_upstreams_destroy (backend->write_servers); - backend->write_servers = NULL; - } - } - - elt = ucl_object_lookup (obj, "prefix"); - if (elt == NULL || ucl_object_type (elt) != UCL_STRING) { - backend->redis_object = REDIS_DEFAULT_OBJECT; - } - else { - backend->redis_object = ucl_object_tostring (elt); - } - - elt = ucl_object_lookup (obj, "timeout"); - if (elt) { - backend->timeout = ucl_object_todouble (elt); - } - else { - backend->timeout = REDIS_DEFAULT_TIMEOUT; - } - - elt = ucl_object_lookup (obj, "password"); - if (elt) { - backend->password = ucl_object_tostring (elt); - } - else { - backend->password = NULL; - } - - elt = ucl_object_lookup_any (obj, "db", "database", "dbname", NULL); - if (elt) { - if (ucl_object_type (elt) == UCL_STRING) { - backend->dbname = ucl_object_tostring (elt); - } - else if (ucl_object_type (elt) == UCL_INT) { - backend->dbname = ucl_object_tostring_forced (elt); - } - } - else { - backend->dbname = NULL; - } - - return TRUE; -} - static void rspamd_fuzzy_backend_redis_dtor (struct rspamd_fuzzy_backend_redis *backend) { - if (backend->read_servers) { - rspamd_upstreams_destroy (backend->read_servers); - } - if (backend->write_servers) { - rspamd_upstreams_destroy (backend->write_servers); + lua_State *L = backend->L; + + if (backend->conf_ref) { + luaL_unref (L, LUA_REGISTRYINDEX, backend->conf_ref); } if (backend->id) { @@ -245,13 +178,16 @@ rspamd_fuzzy_backend_init_redis (struct rspamd_fuzzy_backend *bk, gboolean ret = FALSE; guchar id_hash[rspamd_cryptobox_HASHBYTES]; rspamd_cryptobox_hash_state_t st; + lua_State *L = (lua_State *)cfg->lua_state; + gint conf_ref = -1; backend = g_malloc0 (sizeof (*backend)); backend->timeout = REDIS_DEFAULT_TIMEOUT; backend->redis_object = REDIS_DEFAULT_OBJECT; + backend->L = L; - ret = rspamd_fuzzy_backend_redis_try_ucl (backend, obj, cfg); + ret = rspamd_lua_try_load_redis (L, obj, cfg, &conf_ref); /* Now try global redis settings */ if (!ret) { @@ -264,11 +200,10 @@ rspamd_fuzzy_backend_init_redis (struct rspamd_fuzzy_backend *bk, NULL); if (specific_obj) { - ret = rspamd_fuzzy_backend_redis_try_ucl (backend, specific_obj, - cfg); + ret = rspamd_lua_try_load_redis (L, specific_obj, cfg, &conf_ref); } else { - ret = rspamd_fuzzy_backend_redis_try_ucl (backend, elt, cfg); + ret = rspamd_lua_try_load_redis (L, elt, cfg, &conf_ref); } } } @@ -280,6 +215,44 @@ rspamd_fuzzy_backend_init_redis (struct rspamd_fuzzy_backend *bk, return NULL; } + elt = ucl_object_lookup (obj, "prefix"); + if (elt == NULL || ucl_object_type (elt) != UCL_STRING) { + backend->redis_object = REDIS_DEFAULT_OBJECT; + } + else { + backend->redis_object = ucl_object_tostring (elt); + } + + backend->conf_ref = conf_ref; + + /* Check some common table values */ + lua_rawgeti (L, LUA_REGISTRYINDEX, conf_ref); + + lua_pushstring (L, "timeout"); + lua_gettable (L, -2); + if (lua_type (L, -1) == LUA_TNUMBER) { + backend->timeout = lua_tonumber (L, -1); + } + lua_pop (L, 1); + + lua_pushstring (L, "db"); + lua_gettable (L, -2); + if (lua_type (L, -1) == LUA_TSTRING) { + backend->dbname = rspamd_mempool_strdup (cfg->cfg_pool, + lua_tostring (L, -1)); + } + lua_pop (L, 1); + + lua_pushstring (L, "password"); + lua_gettable (L, -2); + if (lua_type (L, -1) == LUA_TSTRING) { + backend->password = rspamd_mempool_strdup (cfg->cfg_pool, + lua_tostring (L, -1)); + } + lua_pop (L, 1); + + lua_settop (L, 0); + REF_INIT_RETAIN (backend, rspamd_fuzzy_backend_redis_dtor); backend->pool = cfg->redis_pool; rspamd_cryptobox_hash_init (&st, NULL, 0); @@ -628,6 +601,7 @@ rspamd_fuzzy_backend_check_redis (struct rspamd_fuzzy_backend *bk, struct rspamd_fuzzy_backend_redis *backend = subr_ud; struct rspamd_fuzzy_redis_session *session; struct upstream *up; + struct upstream_list *ups; struct timeval tv; rspamd_inet_addr_t *addr; struct rspamd_fuzzy_reply rep; @@ -667,7 +641,8 @@ rspamd_fuzzy_backend_check_redis (struct rspamd_fuzzy_backend *bk, session->argv_lens[4] = 1; g_string_free (key, FALSE); /* Do not free underlying array */ - up = rspamd_upstream_get (backend->read_servers, + ups = rspamd_redis_get_servers (backend, "read_servers"); + up = rspamd_upstream_get (ups, RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0); @@ -765,6 +740,7 @@ rspamd_fuzzy_backend_count_redis (struct rspamd_fuzzy_backend *bk, struct rspamd_fuzzy_backend_redis *backend = subr_ud; struct rspamd_fuzzy_redis_session *session; struct upstream *up; + struct upstream_list *ups; struct timeval tv; rspamd_inet_addr_t *addr; GString *key; @@ -791,7 +767,8 @@ rspamd_fuzzy_backend_count_redis (struct rspamd_fuzzy_backend *bk, session->argv_lens[1] = key->len; g_string_free (key, FALSE); /* Do not free underlying array */ - up = rspamd_upstream_get (backend->read_servers, + ups = rspamd_redis_get_servers (backend, "read_servers"); + up = rspamd_upstream_get (ups, RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0); @@ -888,6 +865,7 @@ rspamd_fuzzy_backend_version_redis (struct rspamd_fuzzy_backend *bk, struct rspamd_fuzzy_backend_redis *backend = subr_ud; struct rspamd_fuzzy_redis_session *session; struct upstream *up; + struct upstream_list *ups; struct timeval tv; rspamd_inet_addr_t *addr; GString *key; @@ -914,7 +892,8 @@ rspamd_fuzzy_backend_version_redis (struct rspamd_fuzzy_backend *bk, session->argv_lens[1] = key->len; g_string_free (key, FALSE); /* Do not free underlying array */ - up = rspamd_upstream_get (backend->read_servers, + ups = rspamd_redis_get_servers (backend, "read_servers"); + up = rspamd_upstream_get (ups, RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0); @@ -1376,6 +1355,7 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk, struct rspamd_fuzzy_backend_redis *backend = subr_ud; struct rspamd_fuzzy_redis_session *session; struct upstream *up; + struct upstream_list *ups; struct timeval tv; rspamd_inet_addr_t *addr; guint i; @@ -1472,7 +1452,8 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk, session->argv = g_malloc0 (sizeof (gchar *) * session->nargs); session->argv_lens = g_malloc0 (sizeof (gsize) * session->nargs); - up = rspamd_upstream_get (backend->write_servers, + ups = rspamd_redis_get_servers (backend, "write_servers"); + up = rspamd_upstream_get (ups, RSPAMD_UPSTREAM_MASTER_SLAVE, NULL, 0); diff --git a/src/libserver/html.c b/src/libserver/html.c index ff745f80d..2568d4c2a 100644 --- a/src/libserver/html.c +++ b/src/libserver/html.c @@ -2177,6 +2177,7 @@ rspamd_html_check_displayed_url (rspamd_mempool_t *pool, ex->pos = href_offset; ex->len = dest->len - href_offset; ex->type = RSPAMD_EXCEPTION_URL; + ex->ptr = url; *exceptions = g_list_prepend (*exceptions, ex); diff --git a/src/libserver/mempool_vars_internal.h b/src/libserver/mempool_vars_internal.h index a5195d325..ad2958ab4 100644 --- a/src/libserver/mempool_vars_internal.h +++ b/src/libserver/mempool_vars_internal.h @@ -32,6 +32,7 @@ #define RSPAMD_MEMPOOL_DKIM_SIGNATURE "dkim-signature" #define RSPAMD_MEMPOOL_DMARC_CHECKS "dmarc_checks" #define RSPAMD_MEMPOOL_DKIM_BH_CACHE "dkim_bh_cache" +#define RSPAMD_MEMPOOL_DKIM_CHECK_RESULTS "dkim_results" #define RSPAMD_MEMPOOL_DKIM_SIGN_KEY "dkim_key" #define RSPAMD_MEMPOOL_DKIM_SIGN_SELECTOR "dkim_selector" #define RSPAMD_MEMPOOL_ARC_SIGN_KEY "arc_key" diff --git a/src/libserver/protocol.c b/src/libserver/protocol.c index c83451058..76a41445b 100644 --- a/src/libserver/protocol.c +++ b/src/libserver/protocol.c @@ -1570,7 +1570,7 @@ rspamd_protocol_write_log_pipe (struct rspamd_task *task) i = 0; kh_foreach_value_ptr (mres->symbols, sym, { - id = rspamd_symbols_cache_find_symbol (task->cfg->cache, + id = rspamd_symcache_find_symbol (task->cfg->cache, sym->name); if (id >= 0) { diff --git a/src/libserver/re_cache.c b/src/libserver/re_cache.c index 64f53773d..f4f190ed5 100644 --- a/src/libserver/re_cache.c +++ b/src/libserver/re_cache.c @@ -23,6 +23,7 @@ #include "libutil/util.h" #include "libutil/regexp.h" #include "lua/lua_common.h" +#include "libstat/stat_api.h" #include "khash.h" @@ -77,11 +78,14 @@ static const guchar rspamd_hs_magic[] = {'r', 's', 'h', 's', 'r', 'e', '1', '1'} struct rspamd_re_class { guint64 id; enum rspamd_re_type type; + gboolean has_utf8; /* if there are any utf8 regexps */ gpointer type_data; gsize type_len; GHashTable *re; - gchar hash[rspamd_cryptobox_HASHBYTES + 1]; rspamd_cryptobox_hash_state_t *st; + + gchar hash[rspamd_cryptobox_HASHBYTES + 1]; + #ifdef WITH_HYPERSCAN hs_database_t *hs_db; hs_scratch_t *hs_scratch; @@ -297,6 +301,10 @@ rspamd_re_cache_add (struct rspamd_re_cache *cache, rspamd_regexp_t *re, g_hash_table_insert (re_class->re, rspamd_regexp_get_id (nre), nre); } + if (rspamd_regexp_get_flags (re) & RSPAMD_REGEXP_FLAG_UTF) { + re_class->has_utf8 = TRUE; + } + return nre; } @@ -693,7 +701,7 @@ rspamd_re_cache_process_regexp_data (struct rspamd_re_runtime *rt, re_class = rspamd_regexp_get_class (re); if (rt->cache->disable_hyperscan || elt->match_type == RSPAMD_RE_CACHE_PCRE || - !rt->has_hs) { + !rt->has_hs || (is_raw && re_class->has_utf8)) { for (i = 0; i < count; i++) { ret = rspamd_re_cache_process_pcre (rt, re, @@ -890,6 +898,60 @@ rspamd_re_cache_process_selector (struct rspamd_task *task, return result; } +static inline guint +rspamd_process_words_vector (GArray *words, + const guchar **scvec, + guint *lenvec, + struct rspamd_re_class *re_class, + guint cnt, + gboolean *raw) +{ + guint j; + rspamd_stat_token_t *tok; + + if (words) { + for (j = 0; j < words->len; j ++) { + tok = &g_array_index (words, rspamd_stat_token_t, j); + + if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_TEXT) { + if (!(tok->flags & RSPAMD_STAT_TOKEN_FLAG_UTF)) { + if (!re_class->has_utf8) { + *raw = TRUE; + } + else { + continue; /* Skip */ + } + } + } + else { + continue; /* Skip non text */ + } + + if (re_class->type == RSPAMD_RE_RAWWORDS) { + if (tok->original.len > 0) { + scvec[cnt] = tok->original.begin; + lenvec[cnt++] = tok->original.len; + } + } + else if (re_class->type == RSPAMD_RE_WORDS) { + if (tok->normalized.len > 0) { + scvec[cnt] = tok->normalized.begin; + lenvec[cnt++] = tok->normalized.len; + } + } + else { + /* Stemmed words */ + if (tok->stemmed.len > 0) { + scvec[cnt] = tok->stemmed.begin; + lenvec[cnt++] = tok->stemmed.len; + } + } + } + } + + return cnt; +} + /* * Calculates the specified regexp for the specified class if it's not calculated */ @@ -935,8 +997,11 @@ rspamd_re_cache_exec_re (struct rspamd_task *task, if (re_class->type == RSPAMD_RE_RAWHEADER) { in = rh->value; - raw = TRUE; lenvec[i] = strlen (rh->value); + + if (!g_utf8_validate (in, lenvec[i], NULL)) { + raw = TRUE; + } } else { in = rh->decoded; @@ -985,8 +1050,11 @@ rspamd_re_cache_exec_re (struct rspamd_task *task, if (re_class->type == RSPAMD_RE_RAWHEADER) { in = rh->value; - raw = TRUE; lenvec[i] = strlen (rh->value); + + if (!g_utf8_validate (in, lenvec[i], NULL)) { + raw = TRUE; + } } else { in = rh->decoded; @@ -996,6 +1064,7 @@ rspamd_re_cache_exec_re (struct rspamd_task *task, scvec[i] = (guchar *)""; continue; } + lenvec[i] = end - in; } @@ -1151,6 +1220,10 @@ rspamd_re_cache_exec_re (struct rspamd_task *task, if (part->utf_stripped_content) { scvec[i + 1] = (guchar *)part->utf_stripped_content->data; lenvec[i + 1] = part->utf_stripped_content->len; + + if (!IS_PART_UTF (part)) { + raw = TRUE; + } } else { scvec[i + 1] = (guchar *)""; @@ -1159,7 +1232,7 @@ rspamd_re_cache_exec_re (struct rspamd_task *task, } ret = rspamd_re_cache_process_regexp_data (rt, re, - task, scvec, lenvec, cnt, TRUE); + task, scvec, lenvec, cnt, raw); msg_debug_re_task ("checking sa body regexp: %s -> %d", rspamd_regexp_get_pattern (re), ret); g_free (scvec); @@ -1184,6 +1257,10 @@ rspamd_re_cache_exec_re (struct rspamd_task *task, if (part->parsed.len > 0) { scvec[i] = (guchar *)part->parsed.begin; lenvec[i] = part->parsed.len; + + if (!IS_PART_UTF (part)) { + raw = TRUE; + } } else { scvec[i] = (guchar *)""; @@ -1192,13 +1269,58 @@ rspamd_re_cache_exec_re (struct rspamd_task *task, } ret = rspamd_re_cache_process_regexp_data (rt, re, - task, scvec, lenvec, cnt, TRUE); + task, scvec, lenvec, cnt, raw); msg_debug_re_task ("checking sa rawbody regexp: %s -> %d", rspamd_regexp_get_pattern (re), ret); g_free (scvec); g_free (lenvec); } break; + case RSPAMD_RE_WORDS: + case RSPAMD_RE_STEMWORDS: + case RSPAMD_RE_RAWWORDS: + if (task->text_parts->len > 0) { + cnt = 0; + raw = FALSE; + + PTR_ARRAY_FOREACH (task->text_parts, i, part) { + if (part->utf_words) { + cnt += part->utf_words->len; + } + } + + if (task->meta_words && task->meta_words->len > 0) { + cnt += task->meta_words->len; + } + + if (cnt > 0) { + scvec = g_malloc (sizeof (*scvec) * cnt); + lenvec = g_malloc (sizeof (*lenvec) * cnt); + + cnt = 0; + + PTR_ARRAY_FOREACH (task->text_parts, i, part) { + if (part->utf_words) { + cnt = rspamd_process_words_vector (part->utf_words, + scvec, lenvec, re_class, cnt, &raw); + } + } + + if (task->meta_words) { + cnt = rspamd_process_words_vector (task->meta_words, + scvec, lenvec, re_class, cnt, &raw); + } + + ret = rspamd_re_cache_process_regexp_data (rt, re, + task, scvec, lenvec, cnt, raw); + + msg_debug_re_task ("checking sa words regexp: %s -> %d", + rspamd_regexp_get_pattern (re), ret); + g_free (scvec); + g_free (lenvec); + } + } + break; case RSPAMD_RE_SELECTOR: if (rspamd_re_cache_process_selector (task, rt, re_class->type_data, @@ -1206,7 +1328,7 @@ rspamd_re_cache_exec_re (struct rspamd_task *task, &lenvec, &cnt)) { ret = rspamd_re_cache_process_regexp_data (rt, re, - task, scvec, lenvec, cnt, TRUE); + task, scvec, lenvec, cnt, raw); msg_debug_re_task ("checking selector (%s) regexp: %s -> %d", re_class->type_data, rspamd_regexp_get_pattern (re), ret); @@ -1392,6 +1514,15 @@ rspamd_re_cache_type_to_string (enum rspamd_re_type type) case RSPAMD_RE_SELECTOR: ret = "selector"; break; + case RSPAMD_RE_WORDS: + ret = "words"; + break; + case RSPAMD_RE_RAWWORDS: + ret = "raw_words"; + break; + case RSPAMD_RE_STEMWORDS: + ret = "stem_words"; + break; case RSPAMD_RE_MAX: ret = "invalid class"; break; diff --git a/src/libserver/re_cache.h b/src/libserver/re_cache.h index c14b29ef0..15146c5dd 100644 --- a/src/libserver/re_cache.h +++ b/src/libserver/re_cache.h @@ -35,6 +35,9 @@ enum rspamd_re_type { RSPAMD_RE_BODY, /* full in SA */ RSPAMD_RE_SABODY, /* body in SA */ RSPAMD_RE_SARAWBODY, /* rawbody in SA */ + RSPAMD_RE_WORDS, /* normalized words */ + RSPAMD_RE_RAWWORDS, /* raw words */ + RSPAMD_RE_STEMWORDS, /* stemmed words */ RSPAMD_RE_SELECTOR, /* use lua selector to process regexp */ RSPAMD_RE_MAX }; diff --git a/src/libserver/symbols_cache.c b/src/libserver/rspamd_symcache.c index 9cd52458b..62340af3b 100644 --- a/src/libserver/symbols_cache.c +++ b/src/libserver/rspamd_symcache.c @@ -17,7 +17,7 @@ #include "util.h" #include "rspamd.h" #include "message.h" -#include "symbols_cache.h" +#include "rspamd_symcache.h" #include "cfg_file.h" #include "lua/lua_common.h" #include "unix-std.h" @@ -25,6 +25,10 @@ #include "libserver/worker_util.h" #include <math.h> +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +# include <stdalign.h> +#endif + #define msg_err_cache(...) rspamd_default_log_function (G_LOG_LEVEL_CRITICAL, \ cache->static_pool->tag.tagname, cache->cfg->checksum, \ G_STRFUNC, \ @@ -41,36 +45,54 @@ rspamd_symcache_log_id, "symcache", cache->cfg->checksum, \ G_STRFUNC, \ __VA_ARGS__) +#define msg_debug_cache_task(...) rspamd_conditional_debug_fast (NULL, NULL, \ + rspamd_symcache_log_id, "symcache", task->task_pool->tag.uid, \ + G_STRFUNC, \ + __VA_ARGS__) INIT_LOG_MODULE(symcache) -static const guchar rspamd_symbols_cache_magic[8] = {'r', 's', 'c', 2, 0, 0, 0, 0 }; +#define CHECK_START_BIT(checkpoint, dyn_item) \ + ((dyn_item)->started) +#define SET_START_BIT(checkpoint, dyn_item) \ + (dyn_item)->started = 1 +#define CLR_START_BIT(checkpoint, dyn_item) \ + (dyn_item)->started = 0 -static gint rspamd_symbols_cache_find_symbol_parent (struct symbols_cache *cache, - const gchar *name); +#define CHECK_FINISH_BIT(checkpoint, dyn_item) \ + ((dyn_item)->finished) +#define SET_FINISH_BIT(checkpoint, dyn_item) \ + (dyn_item)->finished = 1 +#define CLR_FINISH_BIT(checkpoint, dyn_item) \ + (dyn_item)->finished = 0 -struct rspamd_symbols_cache_header { +static const guchar rspamd_symcache_magic[8] = {'r', 's', 'c', 2, 0, 0, 0, 0 }; + +struct rspamd_symcache_header { guchar magic[8]; guint nitems; guchar checksum[64]; guchar unused[128]; }; -struct symbols_cache_order { +struct symcache_order { GPtrArray *d; guint id; ref_entry_t ref; }; -struct symbols_cache { +struct rspamd_symcache { /* Hash table for fast access */ GHashTable *items_by_symbol; - struct symbols_cache_order *items_by_order; GPtrArray *items_by_id; + struct symcache_order *items_by_order; + GPtrArray *filters; GPtrArray *prefilters; GPtrArray *postfilters; GPtrArray *composites; GPtrArray *idempotent; + GPtrArray *virtual; + GPtrArray *squeezed; GList *delayed_deps; GList *delayed_conditions; rspamd_mempool_t *static_pool; @@ -96,27 +118,41 @@ struct item_stat { gdouble stddev_frequency; }; -struct cache_item { +struct rspamd_symcache_dynamic_item { + guint16 start_msec; /* Relative to task time */ + unsigned started:1; + unsigned finished:1; + /* unsigned pad:14; */ + guint32 async_events; +}; + +struct rspamd_symcache_item { /* This block is likely shared */ struct item_stat *st; guint64 last_count; - - /* Per process counter */ struct rspamd_counter_data *cd; gchar *symbol; enum rspamd_symbol_type type; /* Callback data */ - symbol_func_t func; - gpointer user_data; + union { + struct { + symbol_func_t func; + gpointer user_data; + gint condition_cb; + } normal; + struct { + gint parent; + } virtual; + } specific; /* Condition of execution */ - gint condition_cb; gboolean enabled; + /* Used for async stuff checks */ + gboolean is_filter; + gboolean is_virtual; - /* Parent symbol id for virtual symbols */ - gint parent; /* Priority */ gint priority; /* Topological order */ @@ -130,7 +166,7 @@ struct cache_item { }; struct cache_dependency { - struct cache_item *item; + struct rspamd_symcache_item *item; gchar *sym; gint id; }; @@ -149,30 +185,30 @@ struct delayed_cache_condition { enum rspamd_cache_savepoint_stage { RSPAMD_CACHE_PASS_INIT = 0, RSPAMD_CACHE_PASS_PREFILTERS, - RSPAMD_CACHE_PASS_WAIT_PREFILTERS, RSPAMD_CACHE_PASS_FILTERS, - RSPAMD_CACHE_PASS_WAIT_FILTERS, RSPAMD_CACHE_PASS_POSTFILTERS, - RSPAMD_CACHE_PASS_WAIT_POSTFILTERS, RSPAMD_CACHE_PASS_IDEMPOTENT, RSPAMD_CACHE_PASS_WAIT_IDEMPOTENT, RSPAMD_CACHE_PASS_DONE, }; struct cache_savepoint { - guchar *processed_bits; enum rspamd_cache_savepoint_stage pass; guint version; + guint items_inflight; + struct rspamd_metric_result *rs; gdouble lim; - GPtrArray *waitq; - struct symbols_cache_order *order; + + struct rspamd_symcache_item *cur_item; + struct symcache_order *order; + struct rspamd_symcache_dynamic_item dynamic_items[]; }; struct rspamd_cache_refresh_cbdata { gdouble last_resort; struct event resort_ev; - struct symbols_cache *cache; + struct rspamd_symcache *cache; struct rspamd_worker *w; struct event_base *ev_base; }; @@ -185,66 +221,93 @@ struct rspamd_cache_refresh_cbdata { * ((f) > 0 ? (f) : FREQ_ALPHA) \ / (t > TIME_ALPHA ? t : TIME_ALPHA)) -static gboolean rspamd_symbols_cache_check_symbol (struct rspamd_task *task, - struct symbols_cache *cache, - struct cache_item *item, - struct cache_savepoint *checkpoint, - gdouble *total_diff); -static gboolean rspamd_symbols_cache_check_deps (struct rspamd_task *task, - struct symbols_cache *cache, - struct cache_item *item, +static gboolean rspamd_symcache_check_symbol (struct rspamd_task *task, + struct rspamd_symcache *cache, + struct rspamd_symcache_item *item, + struct cache_savepoint *checkpoint); +static gboolean rspamd_symcache_check_deps (struct rspamd_task *task, + struct rspamd_symcache *cache, + struct rspamd_symcache_item *item, struct cache_savepoint *checkpoint, guint recursion, gboolean check_only); -static void rspamd_symbols_cache_disable_symbol_checkpoint (struct rspamd_task *task, - struct symbols_cache *cache, const gchar *symbol); -static void rspamd_symbols_cache_enable_symbol_checkpoint (struct rspamd_task *task, - struct symbols_cache *cache, const gchar *symbol); -static void rspamd_symbols_cache_disable_all_symbols (struct rspamd_task *task, - struct symbols_cache *cache); - -static GQuark -rspamd_symbols_cache_quark (void) -{ - return g_quark_from_static_string ("symbols-cache"); -} +static void rspamd_symcache_disable_symbol_checkpoint (struct rspamd_task *task, + struct rspamd_symcache *cache, const gchar *symbol); +static void rspamd_symcache_enable_symbol_checkpoint (struct rspamd_task *task, + struct rspamd_symcache *cache, const gchar *symbol); +static void rspamd_symcache_disable_all_symbols (struct rspamd_task *task, + struct rspamd_symcache *cache); static void -rspamd_symbols_cache_order_dtor (gpointer p) +rspamd_symcache_order_dtor (gpointer p) { - struct symbols_cache_order *ord = p; + struct symcache_order *ord = p; g_ptr_array_free (ord->d, TRUE); g_free (ord); } static void -rspamd_symbols_cache_order_unref (gpointer p) +rspamd_symcache_order_unref (gpointer p) { - struct symbols_cache_order *ord = p; + struct symcache_order *ord = p; REF_RELEASE (ord); } -static struct symbols_cache_order * -rspamd_symbols_cache_order_new (struct symbols_cache *cache, +static struct symcache_order * +rspamd_symcache_order_new (struct rspamd_symcache *cache, gsize nelts) { - struct symbols_cache_order *ord; + struct symcache_order *ord; ord = g_malloc0 (sizeof (*ord)); ord->d = g_ptr_array_sized_new (nelts); ord->id = cache->id; - REF_INIT_RETAIN (ord, rspamd_symbols_cache_order_dtor); + REF_INIT_RETAIN (ord, rspamd_symcache_order_dtor); return ord; } +static inline struct rspamd_symcache_dynamic_item* +rspamd_symcache_get_dynamic (struct cache_savepoint *checkpoint, + struct rspamd_symcache_item *item) +{ + return &checkpoint->dynamic_items[item->id]; +} + +static inline struct rspamd_symcache_item * +rspamd_symcache_find_filter (struct rspamd_symcache *cache, + const gchar *name) +{ + struct rspamd_symcache_item *item; + + g_assert (cache != NULL); + + if (name == NULL) { + return NULL; + } + + item = g_hash_table_lookup (cache->items_by_symbol, name); + + if (item != NULL) { + + if (item->is_virtual) { + item = g_ptr_array_index (cache->items_by_id, + item->specific.virtual.parent); + } + + return item; + } + + return NULL; +} + static gint postfilters_cmp (const void *p1, const void *p2, gpointer ud) { - const struct cache_item *i1 = *(struct cache_item **)p1, - *i2 = *(struct cache_item **)p2; + const struct rspamd_symcache_item *i1 = *(struct rspamd_symcache_item **)p1, + *i2 = *(struct rspamd_symcache_item **)p2; double w1, w2; w1 = i1->priority; @@ -263,8 +326,8 @@ postfilters_cmp (const void *p1, const void *p2, gpointer ud) static gint prefilters_cmp (const void *p1, const void *p2, gpointer ud) { - const struct cache_item *i1 = *(struct cache_item **)p1, - *i2 = *(struct cache_item **)p2; + const struct rspamd_symcache_item *i1 = *(struct rspamd_symcache_item **)p1, + *i2 = *(struct rspamd_symcache_item **)p2; double w1, w2; w1 = i1->priority; @@ -289,9 +352,9 @@ prefilters_cmp (const void *p1, const void *p2, gpointer ud) static gint cache_logic_cmp (const void *p1, const void *p2, gpointer ud) { - const struct cache_item *i1 = *(struct cache_item **)p1, - *i2 = *(struct cache_item **)p2; - struct symbols_cache *cache = ud; + const struct rspamd_symcache_item *i1 = *(struct rspamd_symcache_item **)p1, + *i2 = *(struct rspamd_symcache_item **)p2; + struct rspamd_symcache *cache = ud; double w1, w2; double weight1, weight2; double f1 = 0, f2 = 0, t1, t2, avg_freq, avg_weight; @@ -333,8 +396,8 @@ cache_logic_cmp (const void *p1, const void *p2, gpointer ud) } static void -rspamd_symbols_cache_tsort_visit (struct symbols_cache *cache, - struct cache_item *it, +rspamd_symcache_tsort_visit (struct rspamd_symcache *cache, + struct rspamd_symcache_item *it, guint cur_order) { struct cache_dependency *dep; @@ -361,7 +424,7 @@ rspamd_symbols_cache_tsort_visit (struct symbols_cache *cache, PTR_ARRAY_FOREACH (it->deps, i, dep) { msg_debug_cache ("visiting dep: %s (%d)", dep->item->symbol, cur_order + 1); - rspamd_symbols_cache_tsort_visit (cache, dep->item, cur_order + 1); + rspamd_symcache_tsort_visit (cache, dep->item, cur_order + 1); } it->order = cur_order; @@ -370,27 +433,20 @@ rspamd_symbols_cache_tsort_visit (struct symbols_cache *cache, } static void -rspamd_symbols_cache_resort (struct symbols_cache *cache) +rspamd_symcache_resort (struct rspamd_symcache *cache) { - struct symbols_cache_order *ord; + struct symcache_order *ord; guint i; guint64 total_hits = 0; - struct cache_item *it; + struct rspamd_symcache_item *it; - ord = rspamd_symbols_cache_order_new (cache, cache->used_items); + ord = rspamd_symcache_order_new (cache, cache->filters->len); - for (i = 0; i < cache->used_items; i ++) { - it = g_ptr_array_index (cache->items_by_id, i); + for (i = 0; i < cache->filters->len; i ++) { + it = g_ptr_array_index (cache->filters, i); total_hits += it->st->total_hits; - - if (!(it->type & (SYMBOL_TYPE_PREFILTER| - SYMBOL_TYPE_POSTFILTER| - SYMBOL_TYPE_COMPOSITE))) { - if (it->parent == -1 && it->func) { - it->order = 0; - g_ptr_array_add (ord->d, it); - } - } + it->order = 0; + g_ptr_array_add (ord->d, it); } /* Topological sort, intended to be O(N) but my implementation @@ -402,7 +458,7 @@ rspamd_symbols_cache_resort (struct symbols_cache *cache) */ PTR_ARRAY_FOREACH (ord->d, i, it) { if (it->order == 0) { - rspamd_symbols_cache_tsort_visit (cache, it, 1); + rspamd_symcache_tsort_visit (cache, it, 1); } } @@ -422,27 +478,20 @@ rspamd_symbols_cache_resort (struct symbols_cache *cache) /* Sort items in logical order */ static void -rspamd_symbols_cache_post_init (struct symbols_cache *cache) +rspamd_symcache_post_init (struct rspamd_symcache *cache) { - struct cache_item *it, *dit; + struct rspamd_symcache_item *it, *dit; struct cache_dependency *dep, *rdep; struct delayed_cache_dependency *ddep; struct delayed_cache_condition *dcond; GList *cur; gint i, j; - gint id; cur = cache->delayed_deps; while (cur) { ddep = cur->data; - id = rspamd_symbols_cache_find_symbol_parent (cache, ddep->from); - if (id != -1) { - it = g_ptr_array_index (cache->items_by_id, id); - } - else { - it = NULL; - } + it = rspamd_symcache_find_filter (cache, ddep->from); if (it == NULL) { msg_err_cache ("cannot register delayed dependency between %s and %s, " @@ -451,7 +500,7 @@ rspamd_symbols_cache_post_init (struct symbols_cache *cache) else { msg_debug_cache ("delayed between %s(%d) -> %s", ddep->from, it->id, ddep->to); - rspamd_symbols_cache_add_dependency (cache, it->id, ddep->to); + rspamd_symcache_add_dependency (cache, it->id, ddep->to); } cur = g_list_next (cur); @@ -461,13 +510,7 @@ rspamd_symbols_cache_post_init (struct symbols_cache *cache) while (cur) { dcond = cur->data; - id = rspamd_symbols_cache_find_symbol_parent (cache, dcond->sym); - if (id != -1) { - it = g_ptr_array_index (cache->items_by_id, id); - } - else { - it = NULL; - } + it = rspamd_symcache_find_filter (cache, dcond->sym); if (it == NULL) { msg_err_cache ( @@ -476,25 +519,18 @@ rspamd_symbols_cache_post_init (struct symbols_cache *cache) luaL_unref (dcond->L, LUA_REGISTRYINDEX, dcond->cbref); } else { - rspamd_symbols_cache_add_condition (cache, it->id, dcond->L, - dcond->cbref); + it->specific.normal.condition_cb = dcond->cbref; } cur = g_list_next (cur); } - for (i = 0; i < cache->items_by_id->len; i ++) { - it = g_ptr_array_index (cache->items_by_id, i); + PTR_ARRAY_FOREACH (cache->items_by_id, i, it) { - for (j = 0; j < it->deps->len; j ++) { - dep = g_ptr_array_index (it->deps, j); - dit = g_hash_table_lookup (cache->items_by_symbol, dep->sym); + PTR_ARRAY_FOREACH (it->deps, j, dep) { + dit = rspamd_symcache_find_filter (cache, dep->sym); if (dit != NULL) { - if (dit->parent != -1) { - dit = g_ptr_array_index (cache->items_by_id, dit->parent); - } - if (dit->id == i) { msg_err_cache ("cannot add dependency on self: %s -> %s " "(resolved to %s)", @@ -503,6 +539,7 @@ rspamd_symbols_cache_post_init (struct symbols_cache *cache) else { rdep = rspamd_mempool_alloc (cache->static_pool, sizeof (*rdep)); + rdep->sym = dep->sym; rdep->item = it; rdep->id = i; @@ -519,13 +556,15 @@ rspamd_symbols_cache_post_init (struct symbols_cache *cache) } } - /* Reversed loop to make removal safe */ - for (j = it->deps->len - 1; j >= 0; j --) { - dep = g_ptr_array_index (it->deps, j); + if (it->deps) { + /* Reversed loop to make removal safe */ + for (j = it->deps->len - 1; j >= 0; j--) { + dep = g_ptr_array_index (it->deps, j); - if (dep->item == NULL) { - /* Remove useless dep */ - g_ptr_array_remove_index (it->deps, j); + if (dep->item == NULL) { + /* Remove useless dep */ + g_ptr_array_remove_index (it->deps, j); + } } } } @@ -534,19 +573,19 @@ rspamd_symbols_cache_post_init (struct symbols_cache *cache) g_ptr_array_sort_with_data (cache->postfilters, postfilters_cmp, cache); g_ptr_array_sort_with_data (cache->idempotent, postfilters_cmp, cache); - rspamd_symbols_cache_resort (cache); + rspamd_symcache_resort (cache); } static gboolean -rspamd_symbols_cache_load_items (struct symbols_cache *cache, const gchar *name) +rspamd_symcache_load_items (struct rspamd_symcache *cache, const gchar *name) { - struct rspamd_symbols_cache_header *hdr; + struct rspamd_symcache_header *hdr; struct stat st; struct ucl_parser *parser; ucl_object_t *top; const ucl_object_t *cur, *elt; ucl_object_iter_t it; - struct cache_item *item, *parent; + struct rspamd_symcache_item *item, *parent; const guchar *p; gint fd; gpointer map; @@ -590,8 +629,8 @@ rspamd_symbols_cache_load_items (struct symbols_cache *cache, const gchar *name) hdr = map; - if (memcmp (hdr->magic, rspamd_symbols_cache_magic, - sizeof (rspamd_symbols_cache_magic)) != 0) { + if (memcmp (hdr->magic, rspamd_symcache_magic, + sizeof (rspamd_symcache_magic)) != 0) { msg_info_cache ("cannot use file %s, bad magic", name); munmap (map, st.st_size); rspamd_file_unlock (fd, FALSE); @@ -674,10 +713,10 @@ rspamd_symbols_cache_load_items (struct symbols_cache *cache, const gchar *name) } } - if ((item->type & SYMBOL_TYPE_VIRTUAL) && - !(item->type & SYMBOL_TYPE_SQUEEZED) && item->parent != -1) { - g_assert (item->parent < (gint)cache->items_by_id->len); - parent = g_ptr_array_index (cache->items_by_id, item->parent); + if (item->is_virtual) { + g_assert (item->specific.virtual.parent < (gint)cache->items_by_id->len); + parent = g_ptr_array_index (cache->items_by_id, + item->specific.virtual.parent); if (parent->st->weight < item->st->weight) { parent->st->weight = item->st->weight; @@ -704,12 +743,12 @@ rspamd_symbols_cache_load_items (struct symbols_cache *cache, const gchar *name) #define ROUND_DOUBLE(x) (floor((x) * 100.0) / 100.0) static gboolean -rspamd_symbols_cache_save_items (struct symbols_cache *cache, const gchar *name) +rspamd_symcache_save_items (struct rspamd_symcache *cache, const gchar *name) { - struct rspamd_symbols_cache_header hdr; + struct rspamd_symcache_header hdr; ucl_object_t *top, *elt, *freq; GHashTableIter it; - struct cache_item *item; + struct rspamd_symcache_item *item; struct ucl_emitter_functions *efunc; gpointer k, v; gint fd; @@ -738,8 +777,8 @@ rspamd_symbols_cache_save_items (struct symbols_cache *cache, const gchar *name) rspamd_file_lock (fd, FALSE); memset (&hdr, 0, sizeof (hdr)); - memcpy (hdr.magic, rspamd_symbols_cache_magic, - sizeof (rspamd_symbols_cache_magic)); + memcpy (hdr.magic, rspamd_symcache_magic, + sizeof (rspamd_symcache_magic)); if (write (fd, &hdr, sizeof (hdr)) == -1) { msg_info_cache ("cannot write to file %s, error %d, %s", path, @@ -797,15 +836,15 @@ rspamd_symbols_cache_save_items (struct symbols_cache *cache, const gchar *name) #undef ROUND_DOUBLE gint -rspamd_symbols_cache_add_symbol (struct symbols_cache *cache, - const gchar *name, - gint priority, - symbol_func_t func, - gpointer user_data, - enum rspamd_symbol_type type, - gint parent) +rspamd_symcache_add_symbol (struct rspamd_symcache *cache, + const gchar *name, + gint priority, + symbol_func_t func, + gpointer user_data, + enum rspamd_symbol_type type, + gint parent) { - struct cache_item *item = NULL; + struct rspamd_symcache_item *item = NULL; g_assert (cache != NULL); @@ -831,10 +870,9 @@ rspamd_symbols_cache_add_symbol (struct symbols_cache *cache, } item = rspamd_mempool_alloc0 (cache->static_pool, - sizeof (struct cache_item)); + sizeof (struct rspamd_symcache_item)); item->st = rspamd_mempool_alloc0_shared (cache->static_pool, sizeof (*item->st)); - item->condition_cb = -1; item->enabled = TRUE; /* @@ -843,8 +881,6 @@ rspamd_symbols_cache_add_symbol (struct symbols_cache *cache, */ item->cd = rspamd_mempool_alloc0 (cache->static_pool, sizeof (struct rspamd_counter_data)); - item->func = func; - item->user_data = user_data; item->priority = priority; item->type = type; @@ -853,8 +889,71 @@ rspamd_symbols_cache_add_symbol (struct symbols_cache *cache, item->priority = 1; } - item->id = cache->used_items; - item->parent = parent; + if (func) { + /* Non-virtual symbol */ + g_assert (parent == -1); + + if (item->type & SYMBOL_TYPE_PREFILTER) { + g_ptr_array_add (cache->prefilters, item); + } + else if (item->type & SYMBOL_TYPE_IDEMPOTENT) { + g_ptr_array_add (cache->idempotent, item); + } + else if (item->type & SYMBOL_TYPE_POSTFILTER) { + g_ptr_array_add (cache->postfilters, item); + } + else { + item->is_filter = TRUE; + g_ptr_array_add (cache->filters, item); + } + + item->id = cache->items_by_id->len; + g_ptr_array_add (cache->items_by_id, item); + + item->specific.normal.func = func; + item->specific.normal.user_data = user_data; + item->specific.normal.condition_cb = -1; + } + else { + /* + * Three possibilities here when no function is specified: + * - virtual symbol + * - classifier symbol + * - composite symbol + */ + if (item->type & SYMBOL_TYPE_COMPOSITE) { + item->specific.normal.condition_cb = -1; + g_ptr_array_add (cache->composites, item); + + item->id = cache->items_by_id->len; + g_ptr_array_add (cache->items_by_id, item); + } + else if (item->type & SYMBOL_TYPE_CLASSIFIER) { + /* Treat it as normal symbol to allow enable/disable */ + item->id = cache->items_by_id->len; + g_ptr_array_add (cache->items_by_id, item); + + item->is_filter = TRUE; + item->specific.normal.func = NULL; + item->specific.normal.user_data = NULL; + item->specific.normal.condition_cb = -1; + } + else { + /* Require parent */ + g_assert (parent != -1); + + item->is_virtual = TRUE; + item->specific.virtual.parent = parent; + item->id = cache->virtual->len; + g_ptr_array_add (cache->virtual, item); + /* Not added to items_by_id, handled by parent */ + } + } + + if (item->type & SYMBOL_TYPE_SQUEEZED) { + g_ptr_array_add (cache->squeezed, item); + } + cache->used_items ++; cache->id ++; @@ -876,70 +975,31 @@ rspamd_symbols_cache_add_symbol (struct symbols_cache *cache, msg_debug_cache ("used items: %d, added symbol: %s, %d", cache->used_items, name, item->id); } else { + g_assert (func != NULL); msg_debug_cache ("used items: %d, added unnamed symbol: %d", cache->used_items, item->id); } - g_ptr_array_add (cache->items_by_id, item); - item->deps = g_ptr_array_new (); - item->rdeps = g_ptr_array_new (); - rspamd_mempool_add_destructor (cache->static_pool, - rspamd_ptr_array_free_hard, item->deps); - rspamd_mempool_add_destructor (cache->static_pool, - rspamd_ptr_array_free_hard, item->rdeps); + if (item->is_filter) { + /* Only plain filters can have deps and rdeps */ + item->deps = g_ptr_array_new (); + item->rdeps = g_ptr_array_new (); + rspamd_mempool_add_destructor (cache->static_pool, + rspamd_ptr_array_free_hard, item->deps); + rspamd_mempool_add_destructor (cache->static_pool, + rspamd_ptr_array_free_hard, item->rdeps); + } if (name != NULL) { g_hash_table_insert (cache->items_by_symbol, item->symbol, item); } - if (item->type & SYMBOL_TYPE_PREFILTER) { - g_ptr_array_add (cache->prefilters, item); - } - else if (item->type & SYMBOL_TYPE_IDEMPOTENT) { - g_ptr_array_add (cache->idempotent, item); - } - else if (item->type & SYMBOL_TYPE_POSTFILTER) { - g_ptr_array_add (cache->postfilters, item); - } - else if (item->type & SYMBOL_TYPE_COMPOSITE) { - g_ptr_array_add (cache->composites, item); - } - return item->id; } -gboolean -rspamd_symbols_cache_add_condition (struct symbols_cache *cache, gint id, - lua_State *L, gint cbref) -{ - struct cache_item *item; - - g_assert (cache != NULL); - - if (id < 0 || id >= (gint)cache->items_by_id->len) { - return FALSE; - } - - item = g_ptr_array_index (cache->items_by_id, id); - - if (item->condition_cb != -1) { - /* We already have a condition, so we need to remove old cbref first */ - msg_warn_cache ("rewriting condition for symbol %s", item->symbol); - luaL_unref (L, LUA_REGISTRYINDEX, item->condition_cb); - } - - item->condition_cb = cbref; - cache->id ++; - - msg_debug_cache ("adding condition at lua ref %d to %s (%d)", - cbref, item->symbol, item->id); - - return TRUE; -} - void -rspamd_symbols_cache_set_peak_callback (struct symbols_cache *cache, - gint cbref) +rspamd_symcache_set_peak_callback (struct rspamd_symcache *cache, + gint cbref) { g_assert (cache != NULL); @@ -953,22 +1013,14 @@ rspamd_symbols_cache_set_peak_callback (struct symbols_cache *cache, } gboolean -rspamd_symbols_cache_add_condition_delayed (struct symbols_cache *cache, - const gchar *sym, lua_State *L, gint cbref) +rspamd_symcache_add_condition_delayed (struct rspamd_symcache *cache, + const gchar *sym, lua_State *L, gint cbref) { - gint id; struct delayed_cache_condition *ncond; g_assert (cache != NULL); g_assert (sym != NULL); - id = rspamd_symbols_cache_find_symbol_parent (cache, sym); - - if (id != -1) { - /* We already know id, so just register a direct condition */ - return rspamd_symbols_cache_add_condition (cache, id, L, cbref); - } - ncond = g_malloc0 (sizeof (*ncond)); ncond->sym = g_strdup (sym); ncond->cbref = cbref; @@ -981,13 +1033,13 @@ rspamd_symbols_cache_add_condition_delayed (struct symbols_cache *cache, } void -rspamd_symbols_cache_save (struct symbols_cache *cache) +rspamd_symcache_save (struct rspamd_symcache *cache) { if (cache != NULL) { if (cache->cfg->cache_filename) { /* Try to sync values to the disk */ - if (!rspamd_symbols_cache_save_items (cache, + if (!rspamd_symcache_save_items (cache, cache->cfg->cache_filename)) { msg_err_cache ("cannot save cache data to %s", cache->cfg->cache_filename); @@ -997,14 +1049,14 @@ rspamd_symbols_cache_save (struct symbols_cache *cache) } void -rspamd_symbols_cache_destroy (struct symbols_cache *cache) +rspamd_symcache_destroy (struct rspamd_symcache *cache) { GList *cur; struct delayed_cache_dependency *ddep; struct delayed_cache_condition *dcond; if (cache != NULL) { - rspamd_symbols_cache_save (cache); + rspamd_symcache_save (cache); if (cache->delayed_deps) { cur = cache->delayed_deps; @@ -1034,12 +1086,14 @@ rspamd_symbols_cache_destroy (struct symbols_cache *cache) } g_hash_table_destroy (cache->items_by_symbol); - rspamd_mempool_delete (cache->static_pool); g_ptr_array_free (cache->items_by_id, TRUE); + rspamd_mempool_delete (cache->static_pool); + g_ptr_array_free (cache->filters, TRUE); g_ptr_array_free (cache->prefilters, TRUE); g_ptr_array_free (cache->postfilters, TRUE); g_ptr_array_free (cache->idempotent, TRUE); g_ptr_array_free (cache->composites, TRUE); + g_ptr_array_free (cache->squeezed, TRUE); REF_RELEASE (cache->items_by_order); if (cache->peak_cb != -1) { @@ -1050,21 +1104,24 @@ rspamd_symbols_cache_destroy (struct symbols_cache *cache) } } -struct symbols_cache* -rspamd_symbols_cache_new (struct rspamd_config *cfg) +struct rspamd_symcache* +rspamd_symcache_new (struct rspamd_config *cfg) { - struct symbols_cache *cache; + struct rspamd_symcache *cache; - cache = g_malloc0 (sizeof (struct symbols_cache)); + cache = g_malloc0 (sizeof (struct rspamd_symcache)); cache->static_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "symcache"); cache->items_by_symbol = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); cache->items_by_id = g_ptr_array_new (); + cache->filters = g_ptr_array_new (); cache->prefilters = g_ptr_array_new (); cache->postfilters = g_ptr_array_new (); cache->idempotent = g_ptr_array_new (); cache->composites = g_ptr_array_new (); + cache->virtual = g_ptr_array_new (); + cache->squeezed = g_ptr_array_new (); cache->reload_time = cfg->cache_reload_time; cache->total_hits = 1; cache->total_weight = 1.0; @@ -1077,7 +1134,7 @@ rspamd_symbols_cache_new (struct rspamd_config *cfg) } gboolean -rspamd_symbols_cache_init (struct symbols_cache* cache) +rspamd_symcache_init (struct rspamd_symcache *cache) { gboolean res; @@ -1087,24 +1144,24 @@ rspamd_symbols_cache_init (struct symbols_cache* cache) /* Just in-memory cache */ if (cache->cfg->cache_filename == NULL) { - rspamd_symbols_cache_post_init (cache); + rspamd_symcache_post_init (cache); return TRUE; } /* Copy saved cache entries */ - res = rspamd_symbols_cache_load_items (cache, cache->cfg->cache_filename); - rspamd_symbols_cache_post_init (cache); + res = rspamd_symcache_load_items (cache, cache->cfg->cache_filename); + rspamd_symcache_post_init (cache); return res; } static void -rspamd_symbols_cache_validate_cb (gpointer k, gpointer v, gpointer ud) +rspamd_symcache_validate_cb (gpointer k, gpointer v, gpointer ud) { - struct cache_item *item = v, *parent; + struct rspamd_symcache_item *item = v, *parent; struct rspamd_config *cfg; - struct symbols_cache *cache = (struct symbols_cache *)ud; + struct rspamd_symcache *cache = (struct rspamd_symcache *)ud; struct rspamd_symbol *s; gboolean skipped, ghost; gint p1, p2; @@ -1156,10 +1213,10 @@ rspamd_symbols_cache_validate_cb (gpointer k, gpointer v, gpointer ud) item->priority ++; } - if ((item->type & SYMBOL_TYPE_VIRTUAL) && - !(item->type & SYMBOL_TYPE_SQUEEZED) && item->parent != -1) { - g_assert (item->parent < (gint)cache->items_by_id->len); - parent = g_ptr_array_index (cache->items_by_id, item->parent); + if (item->is_virtual) { + g_assert (item->specific.virtual.parent < (gint)cache->items_by_id->len); + parent = g_ptr_array_index (cache->items_by_id, + item->specific.virtual.parent); if (fabs (parent->st->weight) < fabs (item->st->weight)) { parent->st->weight = item->st->weight; @@ -1178,13 +1235,13 @@ rspamd_symbols_cache_validate_cb (gpointer k, gpointer v, gpointer ud) } static void -rspamd_symbols_cache_metric_validate_cb (gpointer k, gpointer v, gpointer ud) +rspamd_symcache_metric_validate_cb (gpointer k, gpointer v, gpointer ud) { - struct symbols_cache *cache = (struct symbols_cache *)ud; + struct rspamd_symcache *cache = (struct rspamd_symcache *)ud; const gchar *sym = k; struct rspamd_symbol *s = (struct rspamd_symbol *)v; gdouble weight; - struct cache_item *item; + struct rspamd_symcache_item *item; weight = *s->weight_ptr; item = g_hash_table_lookup (cache->items_by_symbol, sym); @@ -1195,11 +1252,11 @@ rspamd_symbols_cache_metric_validate_cb (gpointer k, gpointer v, gpointer ud) } gboolean -rspamd_symbols_cache_validate (struct symbols_cache *cache, - struct rspamd_config *cfg, - gboolean strict) +rspamd_symcache_validate (struct rspamd_symcache *cache, + struct rspamd_config *cfg, + gboolean strict) { - struct cache_item *item; + struct rspamd_symcache_item *item; GHashTableIter it; gpointer k, v; struct rspamd_symbol *sym_def; @@ -1212,11 +1269,11 @@ rspamd_symbols_cache_validate (struct symbols_cache *cache, /* Now adjust symbol weights according to default metric */ g_hash_table_foreach (cfg->symbols, - rspamd_symbols_cache_metric_validate_cb, + rspamd_symcache_metric_validate_cb, cache); g_hash_table_foreach (cache->items_by_symbol, - rspamd_symbols_cache_validate_cb, + rspamd_symcache_validate_cb, cache); /* Now check each metric item and find corresponding symbol in a cache */ g_hash_table_iter_init (&it, cfg->symbols); @@ -1227,7 +1284,6 @@ rspamd_symbols_cache_validate (struct symbols_cache *cache, if (sym_def && (sym_def->flags & RSPAMD_SYMBOL_FLAG_IGNORE)) { ignore_symbol = TRUE; - break; } if (!ignore_symbol) { @@ -1250,7 +1306,7 @@ rspamd_symbols_cache_validate (struct symbols_cache *cache, /* Return true if metric has score that is more than spam score for it */ static gboolean -rspamd_symbols_cache_metric_limit (struct rspamd_task *task, +rspamd_symcache_metric_limit (struct rspamd_task *task, struct cache_savepoint *cp) { struct rspamd_metric_result *res; @@ -1287,175 +1343,112 @@ rspamd_symbols_cache_metric_limit (struct rspamd_task *task, return FALSE; } -static void -rspamd_symbols_cache_watcher_cb (gpointer sessiond, gpointer ud) -{ - struct rspamd_task *task = sessiond; - struct cache_item *item = ud, *it; - struct cache_savepoint *checkpoint; - struct symbols_cache *cache; - gint i, remain = 0; - - checkpoint = task->checkpoint; - cache = task->cfg->cache; - - /* Specify that we are done with this item */ - setbit (checkpoint->processed_bits, item->id * 2 + 1); - - if (checkpoint->pass > 0) { -#ifdef HAVE_EVENT_NO_CACHE_TIME_FUNC - event_base_update_cache_time (task->ev_base); -#endif - for (i = 0; i < (gint)checkpoint->waitq->len; i ++) { - it = g_ptr_array_index (checkpoint->waitq, i); - - if (!isset (checkpoint->processed_bits, it->id * 2)) { - if (!rspamd_symbols_cache_check_deps (task, cache, it, - checkpoint, 0, TRUE)) { - remain ++; - } - else { - msg_debug_task ("watcher for %d(%s), unblocked item %d(%s)", - item->id, - item->symbol, - it->id, - it->symbol); - rspamd_symbols_cache_check_symbol (task, cache, it, - checkpoint, - NULL); - } - } - } - } - - msg_debug_task ("finished watcher for %d(%s), %ud symbols waiting", - item->id, item->symbol, - remain); -} - static gboolean -rspamd_symbols_cache_check_symbol (struct rspamd_task *task, - struct symbols_cache *cache, - struct cache_item *item, - struct cache_savepoint *checkpoint, - gdouble *total_diff) +rspamd_symcache_check_symbol (struct rspamd_task *task, + struct rspamd_symcache *cache, + struct rspamd_symcache_item *item, + struct cache_savepoint *checkpoint) { - guint pending_before, pending_after; - double t1 = 0, t2 = 0; - gdouble diff; + double t1 = 0; struct rspamd_task **ptask; lua_State *L; gboolean check = TRUE; - const gdouble slow_diff_limit = 0.1; - - if (item->func) { - - g_assert (item->func != NULL); - /* Check has been started */ - setbit (checkpoint->processed_bits, item->id * 2); - - if (!item->enabled || - (RSPAMD_TASK_IS_EMPTY (task) && !(item->type & SYMBOL_TYPE_EMPTY))) { - check = FALSE; - } - else if (item->condition_cb != -1) { - /* We also executes condition callback to check if we need this symbol */ - L = task->cfg->lua_state; - lua_rawgeti (L, LUA_REGISTRYINDEX, item->condition_cb); - ptask = lua_newuserdata (L, sizeof (struct rspamd_task *)); - rspamd_lua_setclass (L, "rspamd{task}", -1); - *ptask = task; - - if (lua_pcall (L, 1, 1, 0) != 0) { - msg_info_task ("call to condition for %s failed: %s", - item->symbol, lua_tostring (L, -1)); - lua_pop (L, 1); - } - else { - check = lua_toboolean (L, -1); - lua_pop (L, 1); - } - } - - if (check) { - pending_before = rspamd_session_events_pending (task->s); - /* Watch for events appeared */ - rspamd_session_watch_start (task->s, - item->id, - rspamd_symbols_cache_watcher_cb, - item); - msg_debug_task ("execute %s, %d", item->symbol, item->id); -#ifdef HAVE_EVENT_NO_CACHE_TIME_FUNC - struct timeval tv; - - event_base_update_cache_time (task->ev_base); - event_base_gettimeofday_cached (task->ev_base, &tv); - t1 = tv_to_double (&tv); -#else - t1 = rspamd_get_ticks (FALSE); -#endif + struct rspamd_symcache_dynamic_item *dyn_item = + rspamd_symcache_get_dynamic (checkpoint, item); - item->func (task, item->user_data); - -#ifdef HAVE_EVENT_NO_CACHE_TIME_FUNC - event_base_update_cache_time (task->ev_base); - event_base_gettimeofday_cached (task->ev_base, &tv); - t2 = tv_to_double (&tv); -#else - t2 = rspamd_get_ticks (FALSE); -#endif + if (item->type & (SYMBOL_TYPE_CLASSIFIER|SYMBOL_TYPE_COMPOSITE)) { + /* Classifiers are special :( */ + return TRUE; + } - diff = (t2 - t1); + if (rspamd_session_blocked (task->s)) { + /* + * We cannot add new events as session is either destroyed or + * being cleaned up. + */ + return TRUE; + } - if (G_UNLIKELY (RSPAMD_TASK_IS_PROFILING (task))) { - rspamd_task_profile_set (task, item->symbol, diff); - } + g_assert (!item->is_virtual); + g_assert (item->specific.normal.func != NULL); + if (CHECK_START_BIT (checkpoint, dyn_item)) { + /* + * This can actually happen when deps span over different layers + */ + return CHECK_FINISH_BIT (checkpoint, dyn_item); + } - if (total_diff) { - *total_diff += diff; - } + /* Check has been started */ + SET_START_BIT (checkpoint, dyn_item); - if (diff > slow_diff_limit && !(item->type & SYMBOL_TYPE_SQUEEZED)) { - msg_info_task ("slow rule: %s: %.2f ms", item->symbol, - diff * 1000); - } + if (!item->enabled || + (RSPAMD_TASK_IS_EMPTY (task) && !(item->type & SYMBOL_TYPE_EMPTY))) { + check = FALSE; + } + else if (item->specific.normal.condition_cb != -1) { + /* We also executes condition callback to check if we need this symbol */ + L = task->cfg->lua_state; + lua_rawgeti (L, LUA_REGISTRYINDEX, item->specific.normal.condition_cb); + ptask = lua_newuserdata (L, sizeof (struct rspamd_task *)); + rspamd_lua_setclass (L, "rspamd{task}", -1); + *ptask = task; - if (rspamd_worker_is_scanner (task->worker)) { - rspamd_set_counter (item->cd, diff); - } + if (lua_pcall (L, 1, 1, 0) != 0) { + msg_info_task ("call to condition for %s failed: %s", + item->symbol, lua_tostring (L, -1)); + lua_pop (L, 1); + } + else { + check = lua_toboolean (L, -1); + lua_pop (L, 1); + } + } - pending_after = rspamd_session_events_pending (task->s); - rspamd_session_watch_stop (task->s); + if (check) { + msg_debug_cache_task ("execute %s, %d", item->symbol, item->id); +#ifdef HAVE_EVENT_NO_CACHE_TIME_FUNC + struct timeval tv; - if (pending_before == pending_after) { - /* No new events registered */ - setbit (checkpoint->processed_bits, item->id * 2 + 1); + event_base_update_cache_time (task->ev_base); + event_base_gettimeofday_cached (task->ev_base, &tv); + t1 = tv_to_double (&tv); +#else + t1 = rspamd_get_ticks (FALSE); +#endif + dyn_item->start_msec = (t1 - task->time_real) * 1e3; + dyn_item->async_events = 0; + checkpoint->cur_item = item; + checkpoint->items_inflight ++; + /* Callback now must finalize itself */ + item->specific.normal.func (task, item, item->specific.normal.user_data); + checkpoint->cur_item = NULL; - return TRUE; - } + if (checkpoint->items_inflight == 0) { - return FALSE; + return TRUE; } - else { - msg_debug_task ("skipping check of %s as its start condition is false", - item->symbol); - setbit (checkpoint->processed_bits, item->id * 2 + 1); - return TRUE; + if (dyn_item->async_events == 0 && !CHECK_FINISH_BIT (checkpoint, dyn_item)) { + msg_err_cache ("critical error: item %s has no async events pending, " + "but it is not finalised", item->symbol); + g_assert_not_reached (); } + + return FALSE; } else { - setbit (checkpoint->processed_bits, item->id * 2); - setbit (checkpoint->processed_bits, item->id * 2 + 1); - - return TRUE; + msg_debug_cache_task ("skipping check of %s as its start condition is false", + item->symbol); + SET_FINISH_BIT (checkpoint, dyn_item); } + + return TRUE; } static gboolean -rspamd_symbols_cache_check_deps (struct rspamd_task *task, - struct symbols_cache *cache, - struct cache_item *item, +rspamd_symcache_check_deps (struct rspamd_task *task, + struct rspamd_symcache *cache, + struct rspamd_symcache_item *item, struct cache_savepoint *checkpoint, guint recursion, gboolean check_only) @@ -1464,6 +1457,7 @@ rspamd_symbols_cache_check_deps (struct rspamd_task *task, guint i; gboolean ret = TRUE; static const guint max_recursion = 20; + struct rspamd_symcache_dynamic_item *dyn_item; if (recursion > max_recursion) { msg_err_task ("cyclic dependencies: maximum check level %ud exceed when " @@ -1478,59 +1472,46 @@ rspamd_symbols_cache_check_deps (struct rspamd_task *task, if (dep->item == NULL) { /* Assume invalid deps as done */ - msg_debug_task ("symbol %d(%s) has invalid dependencies on %d(%s)", + msg_debug_cache_task ("symbol %d(%s) has invalid dependencies on %d(%s)", item->id, item->symbol, dep->id, dep->sym); continue; } - if (!isset (checkpoint->processed_bits, dep->id * 2 + 1)) { - if (!isset (checkpoint->processed_bits, dep->id * 2)) { + dyn_item = rspamd_symcache_get_dynamic (checkpoint, dep->item); + + if (!CHECK_FINISH_BIT (checkpoint, dyn_item)) { + if (!CHECK_START_BIT (checkpoint, dyn_item)) { /* Not started */ if (!check_only) { - if (!rspamd_symbols_cache_check_deps (task, cache, + if (!rspamd_symcache_check_deps (task, cache, dep->item, checkpoint, recursion + 1, check_only)) { - gboolean found = FALSE; - guint j; - struct cache_item *tmp_it; - - PTR_ARRAY_FOREACH (checkpoint->waitq, j, tmp_it) { - if (item->id == tmp_it->id) { - found = TRUE; - break; - } - } - - if (!found) { - g_ptr_array_add (checkpoint->waitq, item); - } ret = FALSE; - msg_debug_task ("delayed dependency %d(%s) for " + msg_debug_cache_task ("delayed dependency %d(%s) for " "symbol %d(%s)", dep->id, dep->sym, item->id, item->symbol); } - else if (!rspamd_symbols_cache_check_symbol (task, cache, + else if (!rspamd_symcache_check_symbol (task, cache, dep->item, - checkpoint, - NULL)) { + checkpoint)) { /* Now started, but has events pending */ ret = FALSE; - msg_debug_task ("started check of %d(%s) symbol " + msg_debug_cache_task ("started check of %d(%s) symbol " "as dep for " "%d(%s)", dep->id, dep->sym, item->id, item->symbol); } else { - msg_debug_task ("dependency %d(%s) for symbol %d(%s) is " + msg_debug_cache_task ("dependency %d(%s) for symbol %d(%s) is " "already processed", dep->id, dep->sym, item->id, item->symbol); } } else { - msg_debug_task ("dependency %d(%s) for symbol %d(%s) " + msg_debug_cache_task ("dependency %d(%s) for symbol %d(%s) " "cannot be started now", dep->id, dep->sym, item->id, item->symbol); @@ -1539,7 +1520,7 @@ rspamd_symbols_cache_check_deps (struct rspamd_task *task, } else { /* Started but not finished */ - msg_debug_task ("dependency %d(%s) for symbol %d(%s) is " + msg_debug_cache_task ("dependency %d(%s) for symbol %d(%s) is " "still executing", dep->id, dep->sym, item->id, item->symbol); @@ -1547,7 +1528,7 @@ rspamd_symbols_cache_check_deps (struct rspamd_task *task, } } else { - msg_debug_task ("dependency %d(%s) for symbol %d(%s) is already " + msg_debug_cache_task ("dependency %d(%s) for symbol %d(%s) is already " "checked", dep->id, dep->sym, item->id, item->symbol); @@ -1558,26 +1539,9 @@ rspamd_symbols_cache_check_deps (struct rspamd_task *task, return ret; } -static void -rspamd_symbols_cache_continuation (void *data) -{ - struct rspamd_task *task = data; - - rspamd_task_process (task, RSPAMD_TASK_PROCESS_ALL); -} - -static void -rspamd_symbols_cache_tm (gint fd, short what, void *data) -{ - struct rspamd_task *task = data; - - rspamd_session_remove_event (task->s, rspamd_symbols_cache_continuation, - data); -} - static struct cache_savepoint * -rspamd_symbols_cache_make_checkpoint (struct rspamd_task *task, - struct symbols_cache *cache) +rspamd_symcache_make_checkpoint (struct rspamd_task *task, + struct rspamd_symcache *cache) { struct cache_savepoint *checkpoint; @@ -1588,22 +1552,20 @@ rspamd_symbols_cache_make_checkpoint (struct rspamd_task *task, msg_info_cache ("symbols cache has been modified since last check:" " old id: %ud, new id: %ud", cache->items_by_order->id, cache->id); - rspamd_symbols_cache_resort (cache); + rspamd_symcache_resort (cache); } - checkpoint = rspamd_mempool_alloc0 (task->task_pool, sizeof (*checkpoint)); - /* Bit 0: check started, Bit 1: check finished */ - checkpoint->processed_bits = rspamd_mempool_alloc0 (task->task_pool, - NBYTES (cache->used_items) * 2); - checkpoint->waitq = g_ptr_array_new (); + checkpoint = rspamd_mempool_alloc0 (task->task_pool, + sizeof (*checkpoint) + + sizeof (struct rspamd_symcache_dynamic_item) * cache->items_by_id->len); + g_assert (cache->items_by_order != NULL); checkpoint->version = cache->items_by_order->d->len; checkpoint->order = cache->items_by_order; REF_RETAIN (checkpoint->order); rspamd_mempool_add_destructor (task->task_pool, - rspamd_symbols_cache_order_unref, checkpoint->order); - rspamd_mempool_add_destructor (task->task_pool, - rspamd_ptr_array_free_hard, checkpoint->waitq); + rspamd_symcache_order_unref, checkpoint->order); + checkpoint->pass = RSPAMD_CACHE_PASS_INIT; task->checkpoint = checkpoint; @@ -1613,8 +1575,8 @@ rspamd_symbols_cache_make_checkpoint (struct rspamd_task *task, } gboolean -rspamd_symbols_cache_process_settings (struct rspamd_task *task, - struct symbols_cache *cache) +rspamd_symcache_process_settings (struct rspamd_task *task, + struct rspamd_symcache *cache) { const ucl_object_t *wl, *cur, *disabled, *enabled; struct rspamd_symbols_group *gr; @@ -1635,12 +1597,12 @@ rspamd_symbols_cache_process_settings (struct rspamd_task *task, if (enabled) { /* Disable all symbols but selected */ - rspamd_symbols_cache_disable_all_symbols (task, cache); + rspamd_symcache_disable_all_symbols (task, cache); already_disabled = TRUE; it = NULL; while ((cur = ucl_iterate_object (enabled, &it, true)) != NULL) { - rspamd_symbols_cache_enable_symbol_checkpoint (task, cache, + rspamd_symcache_enable_symbol_checkpoint (task, cache, ucl_object_tostring (cur)); } } @@ -1652,7 +1614,7 @@ rspamd_symbols_cache_process_settings (struct rspamd_task *task, it = NULL; if (!already_disabled) { - rspamd_symbols_cache_disable_all_symbols (task, cache); + rspamd_symcache_disable_all_symbols (task, cache); } while ((cur = ucl_iterate_object (enabled, &it, true)) != NULL) { @@ -1664,7 +1626,7 @@ rspamd_symbols_cache_process_settings (struct rspamd_task *task, g_hash_table_iter_init (&gr_it, gr->symbols); while (g_hash_table_iter_next (&gr_it, &k, &v)) { - rspamd_symbols_cache_enable_symbol_checkpoint (task, cache, k); + rspamd_symcache_enable_symbol_checkpoint (task, cache, k); } } } @@ -1677,7 +1639,7 @@ rspamd_symbols_cache_process_settings (struct rspamd_task *task, it = NULL; while ((cur = ucl_iterate_object (disabled, &it, true)) != NULL) { - rspamd_symbols_cache_disable_symbol_checkpoint (task, cache, + rspamd_symcache_disable_symbol_checkpoint (task, cache, ucl_object_tostring (cur)); } } @@ -1697,7 +1659,7 @@ rspamd_symbols_cache_process_settings (struct rspamd_task *task, g_hash_table_iter_init (&gr_it, gr->symbols); while (g_hash_table_iter_next (&gr_it, &k, &v)) { - rspamd_symbols_cache_disable_symbol_checkpoint (task, cache, k); + rspamd_symcache_disable_symbol_checkpoint (task, cache, k); } } } @@ -1708,22 +1670,21 @@ rspamd_symbols_cache_process_settings (struct rspamd_task *task, } gboolean -rspamd_symbols_cache_process_symbols (struct rspamd_task * task, - struct symbols_cache *cache, gint stage) +rspamd_symcache_process_symbols (struct rspamd_task *task, + struct rspamd_symcache *cache, gint stage) { - struct cache_item *item = NULL; + struct rspamd_symcache_item *item = NULL; + struct rspamd_symcache_dynamic_item *dyn_item; struct cache_savepoint *checkpoint; gint i; - gdouble total_ticks = 0; gboolean all_done; gint saved_priority; - const gdouble max_ticks = 3e8; guint start_events_pending; g_assert (cache != NULL); if (task->checkpoint == NULL) { - checkpoint = rspamd_symbols_cache_make_checkpoint (task, cache); + checkpoint = rspamd_symcache_make_checkpoint (task, cache); task->checkpoint = checkpoint; } else { @@ -1740,7 +1701,7 @@ rspamd_symbols_cache_process_symbols (struct rspamd_task * task, checkpoint->pass = RSPAMD_CACHE_PASS_IDEMPOTENT; } - msg_debug_task ("symbols processing stage at pass: %d", checkpoint->pass); + msg_debug_cache_task ("symbols processing stage at pass: %d", checkpoint->pass); start_events_pending = rspamd_session_events_pending (task->s); switch (checkpoint->pass) { @@ -1748,16 +1709,18 @@ rspamd_symbols_cache_process_symbols (struct rspamd_task * task, case RSPAMD_CACHE_PASS_PREFILTERS: /* Check for prefilters */ saved_priority = G_MININT; + all_done = TRUE; for (i = 0; i < (gint)cache->prefilters->len; i ++) { item = g_ptr_array_index (cache->prefilters, i); + dyn_item = rspamd_symcache_get_dynamic (checkpoint, item); if (RSPAMD_TASK_IS_SKIPPED (task)) { return TRUE; } - if (!isset (checkpoint->processed_bits, item->id * 2) && - !isset (checkpoint->processed_bits, item->id * 2 + 1)) { + if (!CHECK_START_BIT (checkpoint, dyn_item) && + !CHECK_FINISH_BIT (checkpoint, dyn_item)) { /* Check priorities */ if (saved_priority == G_MININT) { saved_priority = item->priority; @@ -1770,27 +1733,14 @@ rspamd_symbols_cache_process_symbols (struct rspamd_task * task, * priority filters to be processed */ checkpoint->pass = RSPAMD_CACHE_PASS_PREFILTERS; + return TRUE; } } - rspamd_symbols_cache_check_symbol (task, cache, item, - checkpoint, &total_ticks); - } - } - - checkpoint->pass = RSPAMD_CACHE_PASS_WAIT_PREFILTERS; - break; - - case RSPAMD_CACHE_PASS_WAIT_PREFILTERS: - all_done = TRUE; - - for (i = 0; i < (gint)cache->prefilters->len; i ++) { - item = g_ptr_array_index (cache->prefilters, i); - - if (!isset (checkpoint->processed_bits, item->id * 2 + 1)) { + rspamd_symcache_check_symbol (task, cache, item, + checkpoint); all_done = FALSE; - break; } } @@ -1799,131 +1749,67 @@ rspamd_symbols_cache_process_symbols (struct rspamd_task * task, } if (stage == RSPAMD_TASK_STAGE_FILTERS) { - return rspamd_symbols_cache_process_symbols (task, cache, stage); + return rspamd_symcache_process_symbols (task, cache, stage); } + break; + case RSPAMD_CACHE_PASS_FILTERS: /* * On the first pass we check symbols that do not have dependencies * If we figure out symbol that has no dependencies satisfied, then * we just save it for another pass */ + all_done = TRUE; + for (i = 0; i < (gint)checkpoint->version; i ++) { if (RSPAMD_TASK_IS_SKIPPED (task)) { return TRUE; } item = g_ptr_array_index (checkpoint->order->d, i); + dyn_item = rspamd_symcache_get_dynamic (checkpoint, item); if (item->type & SYMBOL_TYPE_CLASSIFIER) { continue; } - if (!(item->type & SYMBOL_TYPE_FINE) && - rspamd_session_events_pending (task->s) == 0) { - if (rspamd_symbols_cache_metric_limit (task, checkpoint)) { - msg_info_task ("<%s> has already scored more than %.2f, so do " - "not " - "plan more checks", task->message_id, - checkpoint->rs->score); - continue; - } - } + if (!CHECK_START_BIT (checkpoint, dyn_item)) { + all_done = FALSE; - if (!isset (checkpoint->processed_bits, item->id * 2)) { - if (!rspamd_symbols_cache_check_deps (task, cache, item, + if (!rspamd_symcache_check_deps (task, cache, item, checkpoint, 0, FALSE)) { - gboolean found = FALSE; - guint j; - struct cache_item *tmp_it; - msg_debug_task ("blocked execution of %d(%s) unless deps are " + msg_debug_cache_task ("blocked execution of %d(%s) unless deps are " "resolved", item->id, item->symbol); - PTR_ARRAY_FOREACH (checkpoint->waitq, j, tmp_it) { - if (item->id == tmp_it->id) { - found = TRUE; - break; - } - } - - if (!found) { - g_ptr_array_add (checkpoint->waitq, item); - } - continue; } - rspamd_symbols_cache_check_symbol (task, cache, item, - checkpoint, &total_ticks); - } - - if (total_ticks > max_ticks) { - /* Maybe we should stop and check pending events? */ - if (rspamd_session_events_pending (task->s) > start_events_pending) { - /* Add some timeout event to avoid too long waiting */ -#if 0 - struct event *ev; - struct timeval tv; - - rspamd_session_add_event (task->s, - rspamd_symbols_cache_continuation, task, - rspamd_symbols_cache_quark ()); - ev = rspamd_mempool_alloc (task->task_pool, sizeof (*ev)); - event_set (ev, -1, EV_TIMEOUT, rspamd_symbols_cache_tm, task); - event_base_set (task->ev_base, ev); - msec_to_tv (50, &tv); - event_add (ev, &tv); - rspamd_mempool_add_destructor (task->task_pool, - (rspamd_mempool_destruct_t)event_del, ev); -#endif - msg_info_task ("trying to check async events after spending " - "%.0f ticks processing symbols", - total_ticks); - - return TRUE; - } + rspamd_symcache_check_symbol (task, cache, item, + checkpoint); } - } - - checkpoint->pass = RSPAMD_CACHE_PASS_WAIT_FILTERS; - break; - - case RSPAMD_CACHE_PASS_WAIT_FILTERS: - /* We just go through the blocked symbols and check if they are ready */ - for (i = 0; i < (gint)checkpoint->waitq->len; i ++) { - item = g_ptr_array_index (checkpoint->waitq, i); - if (!isset (checkpoint->processed_bits, item->id * 2)) { - if (!rspamd_symbols_cache_check_deps (task, cache, item, - checkpoint, 0, FALSE)) { + if (!(item->type & SYMBOL_TYPE_FINE)) { + if (rspamd_symcache_metric_limit (task, checkpoint)) { + msg_info_task ("<%s> has already scored more than %.2f, so do " + "not " + "plan more checks", task->message_id, + checkpoint->rs->score); + all_done = TRUE; break; } - - rspamd_symbols_cache_check_symbol (task, cache, item, - checkpoint, &total_ticks); - } - - if (total_ticks > max_ticks) { - /* Maybe we should stop and check pending events? */ - if (rspamd_session_events_pending (task->s) > - start_events_pending) { - msg_debug_task ("trying to check async events after spending " - "%.0f microseconds processing symbols", - total_ticks); - return TRUE; - } } } - if (checkpoint->waitq->len == 0 || - stage == RSPAMD_TASK_STAGE_POST_FILTERS) { + if (all_done || stage == RSPAMD_TASK_STAGE_POST_FILTERS) { checkpoint->pass = RSPAMD_CACHE_PASS_POSTFILTERS; } if (stage == RSPAMD_TASK_STAGE_POST_FILTERS) { - return rspamd_symbols_cache_process_symbols (task, cache, stage); + + return rspamd_symcache_process_symbols (task, cache, stage); } break; @@ -1931,6 +1817,7 @@ rspamd_symbols_cache_process_symbols (struct rspamd_task * task, case RSPAMD_CACHE_PASS_POSTFILTERS: /* Check for postfilters */ saved_priority = G_MININT; + all_done = TRUE; for (i = 0; i < (gint)cache->postfilters->len; i ++) { if (RSPAMD_TASK_IS_SKIPPED (task)) { @@ -1938,10 +1825,13 @@ rspamd_symbols_cache_process_symbols (struct rspamd_task * task, } item = g_ptr_array_index (cache->postfilters, i); + dyn_item = rspamd_symcache_get_dynamic (checkpoint, item); - if (!isset (checkpoint->processed_bits, item->id * 2) && - !isset (checkpoint->processed_bits, item->id * 2 + 1)) { + if (!CHECK_START_BIT (checkpoint, dyn_item) && + !CHECK_FINISH_BIT (checkpoint, dyn_item)) { /* Check priorities */ + all_done = FALSE; + if (saved_priority == G_MININT) { saved_priority = item->priority; } @@ -1953,26 +1843,13 @@ rspamd_symbols_cache_process_symbols (struct rspamd_task * task, * priority filters to be processed */ checkpoint->pass = RSPAMD_CACHE_PASS_POSTFILTERS; + return TRUE; } } - rspamd_symbols_cache_check_symbol (task, cache, item, - checkpoint, &total_ticks); - } - } - checkpoint->pass = RSPAMD_CACHE_PASS_WAIT_POSTFILTERS; - break; - - case RSPAMD_CACHE_PASS_WAIT_POSTFILTERS: - all_done = TRUE; - - for (i = 0; i < (gint)cache->postfilters->len; i ++) { - item = g_ptr_array_index (cache->postfilters, i); - - if (!isset (checkpoint->processed_bits, item->id * 2 + 1)) { - all_done = FALSE; - break; + rspamd_symcache_check_symbol (task, cache, item, + checkpoint); } } @@ -1980,13 +1857,13 @@ rspamd_symbols_cache_process_symbols (struct rspamd_task * task, checkpoint->pass = RSPAMD_CACHE_PASS_IDEMPOTENT; } - if (checkpoint->waitq->len == 0 || + if (checkpoint->items_inflight == 0 || stage == RSPAMD_TASK_STAGE_IDEMPOTENT) { checkpoint->pass = RSPAMD_CACHE_PASS_IDEMPOTENT; } if (stage == RSPAMD_TASK_STAGE_IDEMPOTENT) { - return rspamd_symbols_cache_process_symbols (task, cache, stage); + return rspamd_symcache_process_symbols (task, cache, stage); } break; @@ -1997,9 +1874,10 @@ rspamd_symbols_cache_process_symbols (struct rspamd_task * task, for (i = 0; i < (gint)cache->idempotent->len; i ++) { item = g_ptr_array_index (cache->idempotent, i); + dyn_item = rspamd_symcache_get_dynamic (checkpoint, item); - if (!isset (checkpoint->processed_bits, item->id * 2) && - !isset (checkpoint->processed_bits, item->id * 2 + 1)) { + if (!CHECK_START_BIT (checkpoint, dyn_item) && + !CHECK_FINISH_BIT (checkpoint, dyn_item)) { /* Check priorities */ if (saved_priority == G_MININT) { saved_priority = item->priority; @@ -2012,11 +1890,12 @@ rspamd_symbols_cache_process_symbols (struct rspamd_task * task, * priority filters to be processed */ checkpoint->pass = RSPAMD_CACHE_PASS_IDEMPOTENT; + return TRUE; } } - rspamd_symbols_cache_check_symbol (task, cache, item, - checkpoint, &total_ticks); + rspamd_symcache_check_symbol (task, cache, item, + checkpoint); } } checkpoint->pass = RSPAMD_CACHE_PASS_WAIT_IDEMPOTENT; @@ -2027,8 +1906,9 @@ rspamd_symbols_cache_process_symbols (struct rspamd_task * task, for (i = 0; i < (gint)cache->idempotent->len; i ++) { item = g_ptr_array_index (cache->idempotent, i); + dyn_item = rspamd_symcache_get_dynamic (checkpoint, item); - if (!isset (checkpoint->processed_bits, item->id * 2 + 1)) { + if (!CHECK_FINISH_BIT (checkpoint, dyn_item)) { all_done = FALSE; break; } @@ -2051,66 +1931,63 @@ rspamd_symbols_cache_process_symbols (struct rspamd_task * task, struct counters_cbdata { ucl_object_t *top; - struct symbols_cache *cache; + struct rspamd_symcache *cache; }; #define ROUND_DOUBLE(x) (floor((x) * 100.0) / 100.0) static void -rspamd_symbols_cache_counters_cb (gpointer v, gpointer ud) +rspamd_symcache_counters_cb (gpointer k, gpointer v, gpointer ud) { struct counters_cbdata *cbd = ud; ucl_object_t *obj, *top; - struct cache_item *item = v, *parent; + struct rspamd_symcache_item *item = v, *parent; + const gchar *symbol = k; top = cbd->top; - if (!(item->type & SYMBOL_TYPE_CALLBACK)) { - obj = ucl_object_typed_new (UCL_OBJECT); - ucl_object_insert_key (obj, ucl_object_fromstring (item->symbol), - "symbol", 0, false); - - if ((item->type & SYMBOL_TYPE_VIRTUAL) && - !(item->type & SYMBOL_TYPE_SQUEEZED) && item->parent != -1) { - g_assert (item->parent < (gint)cbd->cache->items_by_id->len); - parent = g_ptr_array_index (cbd->cache->items_by_id, - item->parent); - ucl_object_insert_key (obj, - ucl_object_fromdouble (ROUND_DOUBLE (item->st->weight)), - "weight", 0, false); - ucl_object_insert_key (obj, - ucl_object_fromdouble (ROUND_DOUBLE (parent->st->avg_frequency)), - "frequency", 0, false); - ucl_object_insert_key (obj, - ucl_object_fromint (parent->st->total_hits), - "hits", 0, false); - ucl_object_insert_key (obj, - ucl_object_fromdouble (ROUND_DOUBLE (parent->st->avg_time)), - "time", 0, false); - } - else { - ucl_object_insert_key (obj, - ucl_object_fromdouble (ROUND_DOUBLE (item->st->weight)), - "weight", 0, false); - ucl_object_insert_key (obj, - ucl_object_fromdouble (ROUND_DOUBLE (item->st->avg_frequency)), - "frequency", 0, false); - ucl_object_insert_key (obj, - ucl_object_fromint (item->st->total_hits), - "hits", 0, false); - ucl_object_insert_key (obj, - ucl_object_fromdouble (ROUND_DOUBLE (item->st->avg_time)), - "time", 0, false); - } - - ucl_array_append (top, obj); + obj = ucl_object_typed_new (UCL_OBJECT); + ucl_object_insert_key (obj, ucl_object_fromstring (symbol ? symbol : "unknown"), + "symbol", 0, false); + + if (item->is_virtual) { + parent = g_ptr_array_index (cbd->cache->items_by_id, + item->specific.virtual.parent); + ucl_object_insert_key (obj, + ucl_object_fromdouble (ROUND_DOUBLE (item->st->weight)), + "weight", 0, false); + ucl_object_insert_key (obj, + ucl_object_fromdouble (ROUND_DOUBLE (parent->st->avg_frequency)), + "frequency", 0, false); + ucl_object_insert_key (obj, + ucl_object_fromint (parent->st->total_hits), + "hits", 0, false); + ucl_object_insert_key (obj, + ucl_object_fromdouble (ROUND_DOUBLE (parent->st->avg_time)), + "time", 0, false); } + else { + ucl_object_insert_key (obj, + ucl_object_fromdouble (ROUND_DOUBLE (item->st->weight)), + "weight", 0, false); + ucl_object_insert_key (obj, + ucl_object_fromdouble (ROUND_DOUBLE (item->st->avg_frequency)), + "frequency", 0, false); + ucl_object_insert_key (obj, + ucl_object_fromint (item->st->total_hits), + "hits", 0, false); + ucl_object_insert_key (obj, + ucl_object_fromdouble (ROUND_DOUBLE (item->st->avg_time)), + "time", 0, false); + } + + ucl_array_append (top, obj); } #undef ROUND_DOUBLE ucl_object_t * -rspamd_symbols_cache_counters (struct symbols_cache * cache) +rspamd_symcache_counters (struct rspamd_symcache *cache) { ucl_object_t *top; struct counters_cbdata cbd; @@ -2119,16 +1996,16 @@ rspamd_symbols_cache_counters (struct symbols_cache * cache) top = ucl_object_typed_new (UCL_ARRAY); cbd.top = top; cbd.cache = cache; - g_ptr_array_foreach (cache->items_by_id, - rspamd_symbols_cache_counters_cb, &cbd); + g_hash_table_foreach (cache->items_by_symbol, + rspamd_symcache_counters_cb, &cbd); return top; } static void -rspamd_symbols_cache_call_peak_cb (struct event_base *ev_base, - struct symbols_cache *cache, - struct cache_item *item, +rspamd_symcache_call_peak_cb (struct event_base *ev_base, + struct rspamd_symcache *cache, + struct rspamd_symcache_item *item, gdouble cur_value, gdouble cur_err) { @@ -2153,13 +2030,13 @@ rspamd_symbols_cache_call_peak_cb (struct event_base *ev_base, } static void -rspamd_symbols_cache_resort_cb (gint fd, short what, gpointer ud) +rspamd_symcache_resort_cb (gint fd, short what, gpointer ud) { struct timeval tv; gdouble tm; struct rspamd_cache_refresh_cbdata *cbdata = ud; - struct symbols_cache *cache; - struct cache_item *item, *parent; + struct rspamd_symcache *cache; + struct rspamd_symcache_item *item; guint i; gdouble cur_ticks; static const double decay_rate = 0.7; @@ -2170,15 +2047,15 @@ rspamd_symbols_cache_resort_cb (gint fd, short what, gpointer ud) cur_ticks = rspamd_get_ticks (FALSE); msg_debug_cache ("resort symbols cache, next reload in %.2f seconds", tm); g_assert (cache != NULL); - evtimer_set (&cbdata->resort_ev, rspamd_symbols_cache_resort_cb, cbdata); + evtimer_set (&cbdata->resort_ev, rspamd_symcache_resort_cb, cbdata); event_base_set (cbdata->ev_base, &cbdata->resort_ev); double_to_tv (tm, &tv); event_add (&cbdata->resort_ev, &tv); - if (rspamd_worker_is_primary_controller (cbdata->w)) + if (rspamd_worker_is_primary_controller (cbdata->w)) { /* Gather stats from shared execution times */ - for (i = 0; i < cache->items_by_id->len; i ++) { - item = g_ptr_array_index (cache->items_by_id, i); + for (i = 0; i < cache->filters->len; i ++) { + item = g_ptr_array_index (cache->filters, i); item->st->total_hits += item->st->hits; g_atomic_int_set (&item->st->hits, 0); @@ -2187,7 +2064,7 @@ rspamd_symbols_cache_resort_cb (gint fd, short what, gpointer ud) gdouble cur_err, cur_value; cur_value = (item->st->total_hits - item->last_count) / - (cur_ticks - cbdata->last_resort); + (cur_ticks - cbdata->last_resort); rspamd_set_counter_ema (&item->st->frequency_counter, cur_value, decay_rate); item->st->avg_frequency = item->st->frequency_counter.mean; @@ -2205,18 +2082,18 @@ rspamd_symbols_cache_resort_cb (gint fd, short what, gpointer ud) * TODO: replace magic number */ if (item->st->frequency_counter.number > 10 && - cur_err > sqrt (item->st->stddev_frequency) * 3) { + cur_err > sqrt (item->st->stddev_frequency) * 3) { item->frequency_peaks ++; msg_debug_cache ("peak found for %s is %.2f, avg: %.2f, " - "stddev: %.2f, error: %.2f, peaks: %d", - item->symbol, cur_value, - item->st->avg_frequency, - item->st->stddev_frequency, - cur_err, - item->frequency_peaks); + "stddev: %.2f, error: %.2f, peaks: %d", + item->symbol, cur_value, + item->st->avg_frequency, + item->st->stddev_frequency, + cur_err, + item->frequency_peaks); if (cache->peak_cb != -1) { - rspamd_symbols_cache_call_peak_cb (cbdata->ev_base, + rspamd_symcache_call_peak_cb (cbdata->ev_base, cache, item, cur_value, cur_err); } @@ -2236,26 +2113,15 @@ rspamd_symbols_cache_resort_cb (gint fd, short what, gpointer ud) } } - /* Sync virtual symbols */ - for (i = 0; i < cache->items_by_id->len; i ++) { - item = g_ptr_array_index (cache->items_by_id, i); - - if (item->parent != -1) { - parent = g_ptr_array_index (cache->items_by_id, item->parent); - if (parent) { - item->st->avg_time = parent->st->avg_time; - } - } + cbdata->last_resort = cur_ticks; + /* We don't do actual sorting due to topological guarantees */ } - - cbdata->last_resort = cur_ticks; - /* We don't do actual sorting due to topological guarantees */ } void -rspamd_symbols_cache_start_refresh (struct symbols_cache * cache, - struct event_base *ev_base, struct rspamd_worker *w) +rspamd_symcache_start_refresh (struct rspamd_symcache *cache, + struct event_base *ev_base, struct rspamd_worker *w) { struct timeval tv; gdouble tm; @@ -2269,7 +2135,7 @@ rspamd_symbols_cache_start_refresh (struct symbols_cache * cache, tm = rspamd_time_jitter (cache->reload_time, 0); msg_debug_cache ("next reload in %.2f seconds", tm); g_assert (cache != NULL); - evtimer_set (&cbdata->resort_ev, rspamd_symbols_cache_resort_cb, + evtimer_set (&cbdata->resort_ev, rspamd_symcache_resort_cb, cbdata); event_base_set (ev_base, &cbdata->resort_ev); double_to_tv (tm, &tv); @@ -2280,10 +2146,10 @@ rspamd_symbols_cache_start_refresh (struct symbols_cache * cache, } void -rspamd_symbols_cache_inc_frequency (struct symbols_cache *cache, - const gchar *symbol) +rspamd_symcache_inc_frequency (struct rspamd_symcache *cache, + const gchar *symbol) { - struct cache_item *item; + struct rspamd_symcache_item *item; g_assert (cache != NULL); @@ -2295,13 +2161,13 @@ rspamd_symbols_cache_inc_frequency (struct symbols_cache *cache, } void -rspamd_symbols_cache_add_dependency (struct symbols_cache *cache, - gint id_from, const gchar *to) +rspamd_symcache_add_dependency (struct rspamd_symcache *cache, + gint id_from, const gchar *to) { - struct cache_item *source; + struct rspamd_symcache_item *source; struct cache_dependency *dep; - g_assert (id_from < (gint)cache->items_by_id->len); + g_assert (id_from >= 0 && id_from < (gint)cache->items_by_id->len); source = g_ptr_array_index (cache->items_by_id, id_from); dep = rspamd_mempool_alloc (cache->static_pool, sizeof (*dep)); @@ -2313,8 +2179,8 @@ rspamd_symbols_cache_add_dependency (struct symbols_cache *cache, } void -rspamd_symbols_cache_add_delayed_dependency (struct symbols_cache *cache, - const gchar *from, const gchar *to) +rspamd_symcache_add_delayed_dependency (struct rspamd_symcache *cache, + const gchar *from, const gchar *to) { struct delayed_cache_dependency *ddep; @@ -2329,9 +2195,9 @@ rspamd_symbols_cache_add_delayed_dependency (struct symbols_cache *cache, } gint -rspamd_symbols_cache_find_symbol (struct symbols_cache *cache, const gchar *name) +rspamd_symcache_find_symbol (struct rspamd_symcache *cache, const gchar *name) { - struct cache_item *item; + struct rspamd_symcache_item *item; g_assert (cache != NULL); @@ -2349,11 +2215,14 @@ rspamd_symbols_cache_find_symbol (struct symbols_cache *cache, const gchar *name } gboolean -rspamd_symbols_cache_stat_symbol (struct symbols_cache *cache, - const gchar *name, gdouble *frequency, gdouble *freq_stddev, - gdouble *tm, guint *nhits) +rspamd_symcache_stat_symbol (struct rspamd_symcache *cache, + const gchar *name, + gdouble *frequency, + gdouble *freq_stddev, + gdouble *tm, + guint *nhits) { - struct cache_item *item; + struct rspamd_symcache_item *item; g_assert (cache != NULL); @@ -2378,37 +2247,11 @@ rspamd_symbols_cache_stat_symbol (struct symbols_cache *cache, return FALSE; } -static gint -rspamd_symbols_cache_find_symbol_parent (struct symbols_cache *cache, - const gchar *name) -{ - struct cache_item *item; - - g_assert (cache != NULL); - - if (name == NULL) { - return -1; - } - - item = g_hash_table_lookup (cache->items_by_symbol, name); - - if (item != NULL) { - - while (item != NULL && item->parent != -1) { - item = g_ptr_array_index (cache->items_by_id, item->parent); - } - - return item ? item->id : -1; - } - - return -1; -} - const gchar * -rspamd_symbols_cache_symbol_by_id (struct symbols_cache *cache, - gint id) +rspamd_symcache_symbol_by_id (struct rspamd_symcache *cache, + gint id) { - struct cache_item *item; + struct rspamd_symcache_item *item; g_assert (cache != NULL); @@ -2422,62 +2265,69 @@ rspamd_symbols_cache_symbol_by_id (struct symbols_cache *cache, } guint -rspamd_symbols_cache_stats_symbols_count (struct symbols_cache *cache) +rspamd_symcache_stats_symbols_count (struct rspamd_symcache *cache) { g_assert (cache != NULL); return cache->stats_symbols_count; } + static void -rspamd_symbols_cache_disable_all_symbols (struct rspamd_task *task, - struct symbols_cache *cache) +rspamd_symcache_disable_all_symbols (struct rspamd_task *task, + struct rspamd_symcache *cache) { struct cache_savepoint *checkpoint; + guint i; + struct rspamd_symcache_item *item; + struct rspamd_symcache_dynamic_item *dyn_item; if (task->checkpoint == NULL) { - checkpoint = rspamd_symbols_cache_make_checkpoint (task, cache); + checkpoint = rspamd_symcache_make_checkpoint (task, cache); task->checkpoint = checkpoint; } else { checkpoint = task->checkpoint; } - /* Set all symbols as started + finished to disable their execution */ - memset (checkpoint->processed_bits, 0xff, - NBYTES (cache->used_items) * 2); + /* Enable for squeezed symbols */ + PTR_ARRAY_FOREACH (cache->items_by_id, i, item) { + dyn_item = rspamd_symcache_get_dynamic (checkpoint, item); + + if (!(item->type & SYMBOL_TYPE_SQUEEZED)) { + SET_FINISH_BIT (checkpoint, dyn_item); + SET_START_BIT (checkpoint, dyn_item); + } + } } static void -rspamd_symbols_cache_disable_symbol_checkpoint (struct rspamd_task *task, - struct symbols_cache *cache, const gchar *symbol) +rspamd_symcache_disable_symbol_checkpoint (struct rspamd_task *task, + struct rspamd_symcache *cache, const gchar *symbol) { struct cache_savepoint *checkpoint; - struct cache_item *item; - gint id; + struct rspamd_symcache_item *item; + struct rspamd_symcache_dynamic_item *dyn_item; if (task->checkpoint == NULL) { - checkpoint = rspamd_symbols_cache_make_checkpoint (task, cache); + checkpoint = rspamd_symcache_make_checkpoint (task, cache); task->checkpoint = checkpoint; } else { checkpoint = task->checkpoint; } - id = rspamd_symbols_cache_find_symbol_parent (cache, symbol); - - if (id >= 0) { - /* Set executed and finished flags */ - item = g_ptr_array_index (cache->items_by_id, id); + item = rspamd_symcache_find_filter (cache, symbol); + if (item) { if (!(item->type & SYMBOL_TYPE_SQUEEZED)) { - setbit (checkpoint->processed_bits, item->id * 2); - setbit (checkpoint->processed_bits, item->id * 2 + 1); - - msg_debug_task ("disable execution of %s", symbol); + dyn_item = rspamd_symcache_get_dynamic (checkpoint, item); + SET_FINISH_BIT (checkpoint, dyn_item); + SET_START_BIT (checkpoint, dyn_item); + msg_debug_cache_task ("disable execution of %s", symbol); } else { - msg_debug_task ("skip squeezed symbol %s", symbol); + msg_debug_cache_task ("skip disabling squeezed symbol %s", symbol); } } else { @@ -2486,31 +2336,33 @@ rspamd_symbols_cache_disable_symbol_checkpoint (struct rspamd_task *task, } static void -rspamd_symbols_cache_enable_symbol_checkpoint (struct rspamd_task *task, - struct symbols_cache *cache, const gchar *symbol) +rspamd_symcache_enable_symbol_checkpoint (struct rspamd_task *task, + struct rspamd_symcache *cache, const gchar *symbol) { struct cache_savepoint *checkpoint; - struct cache_item *item; - gint id; + struct rspamd_symcache_item *item; + struct rspamd_symcache_dynamic_item *dyn_item; if (task->checkpoint == NULL) { - checkpoint = rspamd_symbols_cache_make_checkpoint (task, cache); + checkpoint = rspamd_symcache_make_checkpoint (task, cache); task->checkpoint = checkpoint; } else { checkpoint = task->checkpoint; } - id = rspamd_symbols_cache_find_symbol_parent (cache, symbol); - - if (id >= 0) { - /* Set executed and finished flags */ - item = g_ptr_array_index (cache->items_by_id, id); + item = rspamd_symcache_find_filter (cache, symbol); - clrbit (checkpoint->processed_bits, item->id * 2); - clrbit (checkpoint->processed_bits, item->id * 2 + 1); - - msg_debug_task ("enable execution of %s (%d)", symbol, id); + if (item) { + if (!(item->type & SYMBOL_TYPE_SQUEEZED)) { + dyn_item = rspamd_symcache_get_dynamic (checkpoint, item); + dyn_item->finished = 0; + dyn_item->started = 0; + msg_debug_cache_task ("enable execution of %s", symbol); + } + else { + msg_debug_cache_task ("skip enabling squeezed symbol %s", symbol); + } } else { msg_info_task ("cannot enable %s: not found", symbol); @@ -2518,111 +2370,86 @@ rspamd_symbols_cache_enable_symbol_checkpoint (struct rspamd_task *task, } struct rspamd_abstract_callback_data* -rspamd_symbols_cache_get_cbdata (struct symbols_cache *cache, - const gchar *symbol) -{ - gint id; - struct cache_item *item; - - g_assert (cache != NULL); - g_assert (symbol != NULL); - - id = rspamd_symbols_cache_find_symbol_parent (cache, symbol); - - if (id < 0) { - return NULL; - } - - item = g_ptr_array_index (cache->items_by_id, id); - - return item->user_data; -} - -gboolean -rspamd_symbols_cache_set_cbdata (struct symbols_cache *cache, - const gchar *symbol, struct rspamd_abstract_callback_data *cbdata) +rspamd_symcache_get_cbdata (struct rspamd_symcache *cache, + const gchar *symbol) { - gint id; - struct cache_item *item; + struct rspamd_symcache_item *item; g_assert (cache != NULL); g_assert (symbol != NULL); - id = rspamd_symbols_cache_find_symbol_parent (cache, symbol); + item = rspamd_symcache_find_filter (cache, symbol); - if (id < 0) { - return FALSE; + if (item) { + return item->specific.normal.user_data; } - item = g_ptr_array_index (cache->items_by_id, id); - item->user_data = cbdata; - - return TRUE; + return NULL; } gboolean -rspamd_symbols_cache_is_checked (struct rspamd_task *task, - struct symbols_cache *cache, const gchar *symbol) +rspamd_symcache_is_checked (struct rspamd_task *task, + struct rspamd_symcache *cache, const gchar *symbol) { - gint id; struct cache_savepoint *checkpoint; + struct rspamd_symcache_item *item; + struct rspamd_symcache_dynamic_item *dyn_item; g_assert (cache != NULL); g_assert (symbol != NULL); - id = rspamd_symbols_cache_find_symbol_parent (cache, symbol); - - if (id < 0) { - return FALSE; + if (task->checkpoint == NULL) { + checkpoint = rspamd_symcache_make_checkpoint (task, cache); + task->checkpoint = checkpoint; + } + else { + checkpoint = task->checkpoint; } - checkpoint = task->checkpoint; + item = rspamd_symcache_find_filter (cache, symbol); - if (checkpoint) { - return isset (checkpoint->processed_bits, id * 2); + if (item) { + dyn_item = rspamd_symcache_get_dynamic (checkpoint, item); + return dyn_item->started; } return FALSE; } void -rspamd_symbols_cache_disable_symbol (struct symbols_cache *cache, - const gchar *symbol) +rspamd_symcache_disable_symbol_perm (struct rspamd_symcache *cache, + const gchar *symbol) { - gint id; - struct cache_item *item; + struct rspamd_symcache_item *item; g_assert (cache != NULL); g_assert (symbol != NULL); - id = rspamd_symbols_cache_find_symbol_parent (cache, symbol); + item = g_hash_table_lookup (cache->items_by_symbol, symbol); - if (id >= 0) { - item = g_ptr_array_index (cache->items_by_id, id); + if (item) { item->enabled = FALSE; } } void -rspamd_symbols_cache_enable_symbol (struct symbols_cache *cache, - const gchar *symbol) +rspamd_symcache_enable_symbol_perm (struct rspamd_symcache *cache, + const gchar *symbol) { - gint id; - struct cache_item *item; + struct rspamd_symcache_item *item; g_assert (cache != NULL); g_assert (symbol != NULL); - id = rspamd_symbols_cache_find_symbol_parent (cache, symbol); + item = g_hash_table_lookup (cache->items_by_symbol, symbol); - if (id >= 0) { - item = g_ptr_array_index (cache->items_by_id, id); + if (item) { item->enabled = TRUE; } } guint64 -rspamd_symbols_cache_get_cksum (struct symbols_cache *cache) +rspamd_symcache_get_cksum (struct rspamd_symcache *cache) { g_assert (cache != NULL); @@ -2631,12 +2458,13 @@ rspamd_symbols_cache_get_cksum (struct symbols_cache *cache) gboolean -rspamd_symbols_cache_is_symbol_enabled (struct rspamd_task *task, - struct symbols_cache *cache, const gchar *symbol) +rspamd_symcache_is_symbol_enabled (struct rspamd_task *task, + struct rspamd_symcache *cache, + const gchar *symbol) { - gint id; struct cache_savepoint *checkpoint; - struct cache_item *item; + struct rspamd_symcache_item *item; + struct rspamd_symcache_dynamic_item *dyn_item; lua_State *L; struct rspamd_task **ptask; gboolean ret = TRUE; @@ -2644,36 +2472,111 @@ rspamd_symbols_cache_is_symbol_enabled (struct rspamd_task *task, g_assert (cache != NULL); g_assert (symbol != NULL); - id = rspamd_symbols_cache_find_symbol_parent (cache, symbol); + checkpoint = task->checkpoint; - if (id < 0) { - return FALSE; + + if (checkpoint) { + item = rspamd_symcache_find_filter (cache, symbol); + + if (item) { + dyn_item = rspamd_symcache_get_dynamic (checkpoint, item); + if (CHECK_START_BIT (checkpoint, dyn_item)) { + ret = FALSE; + } + else { + if (item->specific.normal.condition_cb != -1) { + /* We also executes condition callback to check if we need this symbol */ + L = task->cfg->lua_state; + lua_rawgeti (L, LUA_REGISTRYINDEX, + item->specific.normal.condition_cb); + ptask = lua_newuserdata (L, sizeof (struct rspamd_task *)); + rspamd_lua_setclass (L, "rspamd{task}", -1); + *ptask = task; + + if (lua_pcall (L, 1, 1, 0) != 0) { + msg_info_task ("call to condition for %s failed: %s", + item->symbol, lua_tostring (L, -1)); + lua_pop (L, 1); + } + else { + ret = lua_toboolean (L, -1); + lua_pop (L, 1); + } + } + } + } } + return ret; +} + + +gboolean +rspamd_symcache_enable_symbol (struct rspamd_task *task, + struct rspamd_symcache *cache, + const gchar *symbol) +{ + struct cache_savepoint *checkpoint; + struct rspamd_symcache_item *item; + struct rspamd_symcache_dynamic_item *dyn_item; + gboolean ret = FALSE; + + g_assert (cache != NULL); + g_assert (symbol != NULL); + checkpoint = task->checkpoint; - item = g_ptr_array_index (cache->items_by_id, id); if (checkpoint) { - if (isset (checkpoint->processed_bits, id * 2)) { - ret = FALSE; + item = rspamd_symcache_find_filter (cache, symbol); + + if (item) { + dyn_item = rspamd_symcache_get_dynamic (checkpoint, item); + + if (!CHECK_FINISH_BIT (checkpoint, dyn_item)) { + ret = TRUE; + CLR_START_BIT (checkpoint, dyn_item); + CLR_FINISH_BIT (checkpoint, dyn_item); + } + else { + msg_debug_task ("cannot enable symbol %s: already started", symbol); + } } - else { - if (item->condition_cb != -1) { - /* We also executes condition callback to check if we need this symbol */ - L = task->cfg->lua_state; - lua_rawgeti (L, LUA_REGISTRYINDEX, item->condition_cb); - ptask = lua_newuserdata (L, sizeof (struct rspamd_task *)); - rspamd_lua_setclass (L, "rspamd{task}", -1); - *ptask = task; - - if (lua_pcall (L, 1, 1, 0) != 0) { - msg_info_task ("call to condition for %s failed: %s", - item->symbol, lua_tostring (L, -1)); - lua_pop (L, 1); - } - else { - ret = lua_toboolean (L, -1); - lua_pop (L, 1); + } + + return ret; +} + + +gboolean +rspamd_symcache_disable_symbol (struct rspamd_task *task, + struct rspamd_symcache *cache, + const gchar *symbol) +{ + struct cache_savepoint *checkpoint; + struct rspamd_symcache_item *item; + struct rspamd_symcache_dynamic_item *dyn_item; + gboolean ret = FALSE; + + g_assert (cache != NULL); + g_assert (symbol != NULL); + + checkpoint = task->checkpoint; + + if (checkpoint) { + item = rspamd_symcache_find_filter (cache, symbol); + + if (item) { + dyn_item = rspamd_symcache_get_dynamic (checkpoint, item); + + if (!CHECK_START_BIT (checkpoint, dyn_item)) { + ret = TRUE; + SET_START_BIT (checkpoint, dyn_item); + SET_FINISH_BIT (checkpoint, dyn_item); + } + else { + if (!CHECK_FINISH_BIT (checkpoint, dyn_item)) { + msg_warn_task ("cannot disable symbol %s: already started", + symbol); } } } @@ -2683,14 +2586,190 @@ rspamd_symbols_cache_is_symbol_enabled (struct rspamd_task *task, } void -rspamd_symbols_cache_foreach (struct symbols_cache *cache, - void (*func)(gint , const gchar *, gint , gpointer ), - gpointer ud) +rspamd_symcache_foreach (struct rspamd_symcache *cache, + void (*func) (gint, const gchar *, gint, gpointer), + gpointer ud) { - guint i; - struct cache_item *item; + struct rspamd_symcache_item *item; + GHashTableIter it; + gpointer k, v; - PTR_ARRAY_FOREACH (cache->items_by_id, i, item) { + g_hash_table_iter_init (&it, cache->items_by_symbol); + + while (g_hash_table_iter_next (&it, &k, &v)) { + item = (struct rspamd_symcache_item *)v; func (item->id, item->symbol, item->type, ud); } } + +struct rspamd_symcache_item * +rspamd_symcache_get_cur_item (struct rspamd_task *task) +{ + struct cache_savepoint *checkpoint = task->checkpoint; + + if (checkpoint == NULL) { + return NULL; + } + + return checkpoint->cur_item; +} + +/** + * Replaces the current item being processed. + * Returns the current item being processed (if any) + * @param task + * @param item + * @return + */ +struct rspamd_symcache_item * +rspamd_symcache_set_cur_item (struct rspamd_task *task, + struct rspamd_symcache_item *item) +{ + struct cache_savepoint *checkpoint = task->checkpoint; + struct rspamd_symcache_item *ex; + + ex = checkpoint->cur_item; + checkpoint->cur_item = item; + + return ex; +} + + +/** + * Finalize the current async element potentially calling its deps + */ +void +rspamd_symcache_finalize_item (struct rspamd_task *task, + struct rspamd_symcache_item *item) +{ + struct cache_savepoint *checkpoint = task->checkpoint; + struct cache_dependency *rdep; + struct rspamd_symcache_dynamic_item *dyn_item; + gdouble t2, diff; + guint i; + struct timeval tv; + const gdouble slow_diff_limit = 300; + + /* Sanity checks */ + g_assert (checkpoint->items_inflight > 0); + dyn_item = rspamd_symcache_get_dynamic (checkpoint, item); + + if (dyn_item->async_events > 0) { + /* + * XXX: Race condition + * + * It is possible that some async event is still in flight, but we + * already know its result, however, it is the responsibility of that + * event to decrease async events count and call this function + * one more time + */ + msg_debug_cache_task ("postpone finalisation of %s(%d) as there are %d " + "async events pendning", + item->symbol, item->id, dyn_item->async_events); + + return; + } + + msg_debug_cache_task ("process finalize for item %s(%d)", item->symbol, item->id); + SET_FINISH_BIT (checkpoint, dyn_item); + checkpoint->items_inflight --; + checkpoint->cur_item = NULL; + +#ifdef HAVE_EVENT_NO_CACHE_TIME_FUNC + event_base_update_cache_time (task->ev_base); + event_base_gettimeofday_cached (task->ev_base, &tv); + t2 = tv_to_double (&tv); +#else + t2 = rspamd_get_ticks (FALSE); +#endif + + diff = ((t2 - task->time_real) * 1e3 - dyn_item->start_msec); + + if (G_UNLIKELY (RSPAMD_TASK_IS_PROFILING (task))) { + rspamd_task_profile_set (task, item->symbol, diff); + } + + if (!(item->type & SYMBOL_TYPE_SQUEEZED)) { + if (diff > slow_diff_limit) { + msg_info_task ("slow rule: %s(%d): %.2f ms", item->symbol, item->id, + diff); + } + + if (rspamd_worker_is_scanner (task->worker)) { + rspamd_set_counter (item->cd, diff); + } + } + + /* Process all reverse dependencies */ + PTR_ARRAY_FOREACH (item->rdeps, i, rdep) { + if (rdep->item) { + dyn_item = rspamd_symcache_get_dynamic (checkpoint, rdep->item); + if (!CHECK_START_BIT (checkpoint, dyn_item)) { + msg_debug_cache_task ("check item %d(%s) rdep of %s ", + rdep->item->id, rdep->item->symbol, item->symbol); + + if (!rspamd_symcache_check_deps (task, task->cfg->cache, + rdep->item, + checkpoint, 0, FALSE)) { + msg_debug_cache_task ("blocked execution of %d(%s) rdep of %s " + "unless deps are resolved", + rdep->item->id, rdep->item->symbol, item->symbol); + } + else { + rspamd_symcache_check_symbol (task, task->cfg->cache, + rdep->item, + checkpoint); + } + } + } + } +} + +guint +rspamd_symcache_item_async_inc_full (struct rspamd_task *task, + struct rspamd_symcache_item *item, + const gchar *subsystem, + const gchar *loc) +{ + struct rspamd_symcache_dynamic_item *dyn_item; + struct cache_savepoint *checkpoint = task->checkpoint; + + dyn_item = rspamd_symcache_get_dynamic (checkpoint, item); + msg_debug_cache_task ("increase async events counter for %s(%d) = %d + 1; " + "subsystem %s (%s)", + item->symbol, item->id, dyn_item->async_events, subsystem, loc); + return ++dyn_item->async_events; +} + +guint +rspamd_symcache_item_async_dec_full (struct rspamd_task *task, + struct rspamd_symcache_item *item, + const gchar *subsystem, + const gchar *loc) +{ + struct rspamd_symcache_dynamic_item *dyn_item; + struct cache_savepoint *checkpoint = task->checkpoint; + + dyn_item = rspamd_symcache_get_dynamic (checkpoint, item); + msg_debug_cache_task ("decrease async events counter for %s(%d) = %d - 1; " + "subsystem %s (%s)", + item->symbol, item->id, dyn_item->async_events, subsystem, loc); + g_assert (dyn_item->async_events > 0); + + return --dyn_item->async_events; +} + +gboolean +rspamd_symcache_item_async_dec_check_full (struct rspamd_task *task, + struct rspamd_symcache_item *item, + const gchar *subsystem, + const gchar *loc) +{ + if (rspamd_symcache_item_async_dec_full (task, item, subsystem, loc) == 0) { + rspamd_symcache_finalize_item (task, item); + + return TRUE; + } + + return FALSE; +}
\ No newline at end of file diff --git a/src/libserver/rspamd_symcache.h b/src/libserver/rspamd_symcache.h new file mode 100644 index 000000000..df495fc8e --- /dev/null +++ b/src/libserver/rspamd_symcache.h @@ -0,0 +1,376 @@ +/*- + * Copyright 2016 Vsevolod Stakhov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RSPAMD_SYMBOLS_CACHE_H +#define RSPAMD_SYMBOLS_CACHE_H + +#include "config.h" +#include "ucl.h" +#include <lua.h> +#include <event.h> + +struct rspamd_task; +struct rspamd_config; +struct rspamd_symcache; +struct rspamd_worker; +struct rspamd_symcache_item; + +typedef void (*symbol_func_t)(struct rspamd_task *task, + struct rspamd_symcache_item *item, + gpointer user_data); + +enum rspamd_symbol_type { + SYMBOL_TYPE_NORMAL = (1 << 0), + SYMBOL_TYPE_VIRTUAL = (1 << 1), + SYMBOL_TYPE_CALLBACK = (1 << 2), + SYMBOL_TYPE_GHOST = (1 << 3), + SYMBOL_TYPE_SKIPPED = (1 << 4), + SYMBOL_TYPE_COMPOSITE = (1 << 5), + SYMBOL_TYPE_CLASSIFIER = (1 << 6), + SYMBOL_TYPE_FINE = (1 << 7), + SYMBOL_TYPE_EMPTY = (1 << 8), /* Allow execution on empty tasks */ + SYMBOL_TYPE_PREFILTER = (1 << 9), + SYMBOL_TYPE_POSTFILTER = (1 << 10), + SYMBOL_TYPE_NOSTAT = (1 << 11), /* Skip as statistical symbol */ + SYMBOL_TYPE_IDEMPOTENT = (1 << 12), /* Symbol cannot change metric */ + SYMBOL_TYPE_SQUEEZED = (1 << 13), /* Symbol is squeezed inside Lua */ + SYMBOL_TYPE_TRIVIAL = (1 << 14), /* Symbol is trivial */ +}; + +/** + * Abstract structure for saving callback data for symbols + */ +struct rspamd_abstract_callback_data { + guint64 magic; + char data[]; +}; + +/** + * Creates new cache structure + * @return + */ +struct rspamd_symcache* rspamd_symcache_new (struct rspamd_config *cfg); + +/** + * Remove the cache structure syncing data if needed + * @param cache + */ +void rspamd_symcache_destroy (struct rspamd_symcache *cache); + +/** + * Saves symbols cache to disk if possible + * @param cache + */ +void rspamd_symcache_save (struct rspamd_symcache *cache); + +/** + * Load symbols cache from file, must be called _after_ init_symbols_cache + */ +gboolean rspamd_symcache_init (struct rspamd_symcache *cache); + +/** + * Generic function to register a symbol + * @param cache + * @param name + * @param weight + * @param priority + * @param func + * @param user_data + * @param type + * @param parent + */ +gint rspamd_symcache_add_symbol (struct rspamd_symcache *cache, + const gchar *name, + gint priority, + symbol_func_t func, + gpointer user_data, + enum rspamd_symbol_type type, + gint parent); + +/** + * Add callback to be executed whenever symbol has peak value + * @param cache + * @param cbref + */ +void rspamd_symcache_set_peak_callback (struct rspamd_symcache *cache, + gint cbref); +/** + * Add delayed condition to the specific symbol in cache. So symbol can be absent + * to the moment of addition + * @param cache + * @param id id of symbol + * @param L lua state pointer + * @param cbref callback reference (returned by luaL_ref) + * @return TRUE if condition has been added + */ +gboolean rspamd_symcache_add_condition_delayed (struct rspamd_symcache *cache, + const gchar *sym, + lua_State *L, gint cbref); + +/** + * Find symbol in cache by id and returns its id resolving virtual symbols if + * applicable + * @param cache + * @param name + * @return id of symbol or (-1) if a symbol has not been found + */ +gint rspamd_symcache_find_symbol (struct rspamd_symcache *cache, + const gchar *name); + +/** + * Get statistics for a specific symbol + * @param cache + * @param name + * @param frequency + * @param tm + * @return + */ +gboolean rspamd_symcache_stat_symbol (struct rspamd_symcache *cache, + const gchar *name, + gdouble *frequency, + gdouble *freq_stddev, + gdouble *tm, + guint *nhits); +/** + * Find symbol in cache by its id + * @param cache + * @param id + * @return symbol's name or NULL + */ +const gchar * rspamd_symcache_symbol_by_id (struct rspamd_symcache *cache, + gint id); + +/** + * Returns number of symbols registered in symbols cache + * @param cache + * @return number of symbols in the cache + */ +guint rspamd_symcache_stats_symbols_count (struct rspamd_symcache *cache); + +/** + * Call function for cached symbol using saved callback + * @param task task object + * @param cache symbols cache + * @param saved_item pointer to currently saved item + */ +gboolean rspamd_symcache_process_symbols (struct rspamd_task *task, + struct rspamd_symcache *cache, + gint stage); + +/** + * Validate cache items against theirs weights defined in metrics + * @param cache symbols cache + * @param cfg configuration + * @param strict do strict checks - symbols MUST be described in metrics + */ +gboolean rspamd_symcache_validate (struct rspamd_symcache *cache, + struct rspamd_config *cfg, + gboolean strict); + +/** + * Return statistics about the cache as ucl object (array of objects one per item) + * @param cache + * @return + */ +ucl_object_t *rspamd_symcache_counters (struct rspamd_symcache *cache); + +/** + * Start cache reloading + * @param cache + * @param ev_base + */ +void rspamd_symcache_start_refresh (struct rspamd_symcache *cache, + struct event_base *ev_base, + struct rspamd_worker *w); + +/** + * Increases counter for a specific symbol + * @param cache + * @param symbol + */ +void rspamd_symcache_inc_frequency (struct rspamd_symcache *cache, + const gchar *symbol); + +/** + * Add dependency relation between two symbols identified by id (source) and + * a symbolic name (destination). Destination could be virtual or real symbol. + * Callback destinations are not yet supported. + * @param id_from source symbol + * @param to destination name + */ +void rspamd_symcache_add_dependency (struct rspamd_symcache *cache, + gint id_from, const gchar *to); + +/** + * Add delayed dependency that is resolved on cache post-load routine + * @param cache + * @param from + * @param to + */ +void rspamd_symcache_add_delayed_dependency (struct rspamd_symcache *cache, + const gchar *from, const gchar *to); + +/** + * Disable specific symbol in the cache + * @param cache + * @param symbol + */ +void rspamd_symcache_disable_symbol_perm (struct rspamd_symcache *cache, + const gchar *symbol); + +/** + * Enable specific symbol in the cache + * @param cache + * @param symbol + */ +void rspamd_symcache_enable_symbol_perm (struct rspamd_symcache *cache, + const gchar *symbol); +/** + * Get abstract callback data for a symbol (or its parent symbol) + * @param cache cache object + * @param symbol symbol name + * @return abstract callback data or NULL if symbol is absent or has no data attached + */ +struct rspamd_abstract_callback_data* rspamd_symcache_get_cbdata ( + struct rspamd_symcache *cache, const gchar *symbol); + + +/** + * Process settings for task + * @param task + * @param cache + * @return + */ +gboolean rspamd_symcache_process_settings (struct rspamd_task *task, + struct rspamd_symcache *cache); + + +/** + * Checks if a symbol specified has been checked (or disabled) + * @param task + * @param cache + * @param symbol + * @return + */ +gboolean rspamd_symcache_is_checked (struct rspamd_task *task, + struct rspamd_symcache *cache, + const gchar *symbol); + +/** + * Returns checksum for all cache items + * @param cache + * @return + */ +guint64 rspamd_symcache_get_cksum (struct rspamd_symcache *cache); + +/** + * Checks if a symbols is enabled (not checked and conditions return true if present) + * @param task + * @param cache + * @param symbol + * @return + */ +gboolean rspamd_symcache_is_symbol_enabled (struct rspamd_task *task, + struct rspamd_symcache *cache, + const gchar *symbol); + +/** + * Enable this symbol for task + * @param task + * @param cache + * @param symbol + * @return TRUE if a symbol has been enabled (not executed before) + */ +gboolean rspamd_symcache_enable_symbol (struct rspamd_task *task, + struct rspamd_symcache *cache, + const gchar *symbol); + +/** + * Enable this symbol for task + * @param task + * @param cache + * @param symbol + * @return TRUE if a symbol has been disabled (not executed before) + */ +gboolean rspamd_symcache_disable_symbol (struct rspamd_task *task, + struct rspamd_symcache *cache, + const gchar *symbol); +/** + * Process specific function for each cache element (in order they are added) + * @param cache + * @param func + * @param ud + */ +void rspamd_symcache_foreach (struct rspamd_symcache *cache, + void (*func) (gint /* id */, const gchar * /* name */, + gint /* flags */, gpointer /* userdata */), + gpointer ud); + +/** + * Returns the current item being processed (if any) + * @param task + * @return + */ +struct rspamd_symcache_item *rspamd_symcache_get_cur_item (struct rspamd_task *task); + +/** + * Replaces the current item being processed. + * Returns the current item being processed (if any) + * @param task + * @param item + * @return + */ +struct rspamd_symcache_item *rspamd_symcache_set_cur_item (struct rspamd_task *task, + struct rspamd_symcache_item *item); + + +/** + * Finalize the current async element potentially calling its deps + */ +void rspamd_symcache_finalize_item (struct rspamd_task *task, + struct rspamd_symcache_item *item); + +/* + * Increase number of async events pending for an item + */ +guint rspamd_symcache_item_async_inc_full (struct rspamd_task *task, + struct rspamd_symcache_item *item, + const gchar *subsystem, + const gchar *loc); +#define rspamd_symcache_item_async_inc(task, item, subsystem) \ + rspamd_symcache_item_async_inc_full(task, item, subsystem, G_STRLOC) +/* + * Decrease number of async events pending for an item, asserts if no events pending + */ +guint rspamd_symcache_item_async_dec_full (struct rspamd_task *task, + struct rspamd_symcache_item *item, + const gchar *subsystem, + const gchar *loc); +#define rspamd_symcache_item_async_dec(task, item, subsystem) \ + rspamd_symcache_item_async_dec_full(task, item, subsystem, G_STRLOC) +/** + * Decrease number of async events pending for an item, asserts if no events pending + * If no events are left, this function calls `rspamd_symbols_cache_finalize_item` and returns TRUE + * @param task + * @param item + * @return + */ +gboolean rspamd_symcache_item_async_dec_check_full (struct rspamd_task *task, + struct rspamd_symcache_item *item, + const gchar *subsystem, + const gchar *loc); +#define rspamd_symcache_item_async_dec_check(task, item, subsystem) \ + rspamd_symcache_item_async_dec_check_full(task, item, subsystem, G_STRLOC) +#endif diff --git a/src/libserver/spf.c b/src/libserver/spf.c index 372b20189..6de2fa4b9 100644 --- a/src/libserver/spf.c +++ b/src/libserver/spf.c @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include <contrib/librdns/rdns.h> #include "config.h" #include "dns.h" #include "spf.h" @@ -611,6 +612,7 @@ spf_record_dns_callback (struct rdns_reply *reply, gpointer arg) struct rspamd_task *task; struct spf_addr *addr; struct spf_record *rec; + const struct rdns_request_name *req_name; rec = cb->rec; task = rec->task; @@ -619,6 +621,8 @@ spf_record_dns_callback (struct rdns_reply *reply, gpointer arg) addr = cb->addr; if (reply->code == RDNS_RC_NOERROR) { + req_name = rdns_request_get_name (reply->request, NULL); + LL_FOREACH (reply->entries, elt_data) { /* Adjust ttl if a resolved record has lower ttl than spf record itself */ if ((guint)elt_data->ttl < rec->ttl) { @@ -693,6 +697,12 @@ spf_record_dns_callback (struct rdns_reply *reply, gpointer arg) case SPF_RESOLVE_REDIRECT: if (elt_data->type == RDNS_REQUEST_TXT) { cb->addr->flags |= RSPAMD_SPF_FLAG_RESOLVED; + if (reply->entries) { + msg_debug_spf ("got redirection record for %s: '%s'", + req_name->name, + reply->entries[0].content.txt.data); + } + if (!spf_process_txt_record (rec, cb->resolved, reply)) { cb->addr->flags |= RSPAMD_SPF_FLAG_PERMFAIL; } @@ -702,6 +712,12 @@ spf_record_dns_callback (struct rdns_reply *reply, gpointer arg) break; case SPF_RESOLVE_INCLUDE: if (elt_data->type == RDNS_REQUEST_TXT) { + if (reply->entries) { + msg_debug_spf ("got include record for %s: '%s'", + req_name->name, + reply->entries[0].content.txt.data); + } + cb->addr->flags |= RSPAMD_SPF_FLAG_RESOLVED; spf_process_txt_record (rec, cb->resolved, reply); } diff --git a/src/libserver/symbols_cache.h b/src/libserver/symbols_cache.h deleted file mode 100644 index 2657b07cc..000000000 --- a/src/libserver/symbols_cache.h +++ /dev/null @@ -1,308 +0,0 @@ -/*- - * Copyright 2016 Vsevolod Stakhov - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef RSPAMD_SYMBOLS_CACHE_H -#define RSPAMD_SYMBOLS_CACHE_H - -#include "config.h" -#include "ucl.h" -#include <lua.h> -#include <event.h> - -struct rspamd_task; -struct rspamd_config; -struct symbols_cache; -struct rspamd_worker; - -typedef void (*symbol_func_t)(struct rspamd_task *task, gpointer user_data); - -enum rspamd_symbol_type { - SYMBOL_TYPE_NORMAL = (1 << 0), - SYMBOL_TYPE_VIRTUAL = (1 << 1), - SYMBOL_TYPE_CALLBACK = (1 << 2), - SYMBOL_TYPE_GHOST = (1 << 3), - SYMBOL_TYPE_SKIPPED = (1 << 4), - SYMBOL_TYPE_COMPOSITE = (1 << 5), - SYMBOL_TYPE_CLASSIFIER = (1 << 6), - SYMBOL_TYPE_FINE = (1 << 7), - SYMBOL_TYPE_EMPTY = (1 << 8), /* Allow execution on empty tasks */ - SYMBOL_TYPE_PREFILTER = (1 << 9), - SYMBOL_TYPE_POSTFILTER = (1 << 10), - SYMBOL_TYPE_NOSTAT = (1 << 11), /* Skip as statistical symbol */ - SYMBOL_TYPE_IDEMPOTENT = (1 << 12), /* Symbol cannot change metric */ - SYMBOL_TYPE_SQUEEZED = (1 << 13), /* Symbol is squeezed inside Lua */ - SYMBOL_TYPE_TRIVIAL = (1 << 14), /* Symbol is trivial */ -}; - -/** - * Abstract structure for saving callback data for symbols - */ -struct rspamd_abstract_callback_data { - guint64 magic; - char data[]; -}; - -/** - * Creates new cache structure - * @return - */ -struct symbols_cache* rspamd_symbols_cache_new (struct rspamd_config *cfg); - -/** - * Remove the cache structure syncing data if needed - * @param cache - */ -void rspamd_symbols_cache_destroy (struct symbols_cache *cache); - -/** - * Saves symbols cache to disk if possible - * @param cache - */ -void rspamd_symbols_cache_save (struct symbols_cache *cache); - -/** - * Load symbols cache from file, must be called _after_ init_symbols_cache - */ -gboolean rspamd_symbols_cache_init (struct symbols_cache* cache); - -/** - * Generic function to register a symbol - * @param cache - * @param name - * @param weight - * @param priority - * @param func - * @param user_data - * @param type - * @param parent - */ -gint rspamd_symbols_cache_add_symbol (struct symbols_cache *cache, - const gchar *name, - gint priority, - symbol_func_t func, - gpointer user_data, - enum rspamd_symbol_type type, - gint parent); - -/** - * Add condition to the specific symbol in cache - * @param cache - * @param id id of symbol - * @param L lua state pointer - * @param cbref callback reference (returned by luaL_ref) - * @return TRUE if condition has been added - */ -gboolean rspamd_symbols_cache_add_condition (struct symbols_cache *cache, - gint id, lua_State *L, gint cbref); - -/** - * Add callback to be executed whenever symbol has peak value - * @param cache - * @param cbref - */ -void rspamd_symbols_cache_set_peak_callback (struct symbols_cache *cache, - gint cbref); -/** - * Add delayed condition to the specific symbol in cache. So symbol can be absent - * to the moment of addition - * @param cache - * @param id id of symbol - * @param L lua state pointer - * @param cbref callback reference (returned by luaL_ref) - * @return TRUE if condition has been added - */ -gboolean rspamd_symbols_cache_add_condition_delayed (struct symbols_cache *cache, - const gchar *sym, lua_State *L, gint cbref); - -/** - * Find symbol in cache by id and returns its id resolving virtual symbols if - * applicable - * @param cache - * @param name - * @return id of symbol or (-1) if a symbol has not been found - */ -gint rspamd_symbols_cache_find_symbol (struct symbols_cache *cache, - const gchar *name); - -/** - * Get statistics for a specific symbol - * @param cache - * @param name - * @param frequency - * @param tm - * @return - */ -gboolean rspamd_symbols_cache_stat_symbol (struct symbols_cache *cache, - const gchar *name, gdouble *frequency, gdouble *freq_stddev, - gdouble *tm, guint *nhits); -/** - * Find symbol in cache by its id - * @param cache - * @param id - * @return symbol's name or NULL - */ -const gchar * rspamd_symbols_cache_symbol_by_id (struct symbols_cache *cache, - gint id); - -/** - * Returns number of symbols registered in symbols cache - * @param cache - * @return number of symbols in the cache - */ -guint rspamd_symbols_cache_stats_symbols_count (struct symbols_cache *cache); - -/** - * Call function for cached symbol using saved callback - * @param task task object - * @param cache symbols cache - * @param saved_item pointer to currently saved item - */ -gboolean rspamd_symbols_cache_process_symbols (struct rspamd_task *task, - struct symbols_cache *cache, gint stage); - -/** - * Validate cache items against theirs weights defined in metrics - * @param cache symbols cache - * @param cfg configuration - * @param strict do strict checks - symbols MUST be described in metrics - */ -gboolean rspamd_symbols_cache_validate (struct symbols_cache *cache, - struct rspamd_config *cfg, - gboolean strict); - -/** - * Return statistics about the cache as ucl object (array of objects one per item) - * @param cache - * @return - */ -ucl_object_t *rspamd_symbols_cache_counters (struct symbols_cache * cache); - -/** - * Start cache reloading - * @param cache - * @param ev_base - */ -void rspamd_symbols_cache_start_refresh (struct symbols_cache * cache, - struct event_base *ev_base, struct rspamd_worker *w); - -/** - * Increases counter for a specific symbol - * @param cache - * @param symbol - */ -void rspamd_symbols_cache_inc_frequency (struct symbols_cache *cache, - const gchar *symbol); - -/** - * Add dependency relation between two symbols identified by id (source) and - * a symbolic name (destination). Destination could be virtual or real symbol. - * Callback destinations are not yet supported. - * @param id_from source symbol - * @param to destination name - */ -void rspamd_symbols_cache_add_dependency (struct symbols_cache *cache, - gint id_from, const gchar *to); - -/** - * Add delayed dependency that is resolved on cache post-load routine - * @param cache - * @param from - * @param to - */ -void rspamd_symbols_cache_add_delayed_dependency (struct symbols_cache *cache, - const gchar *from, const gchar *to); - -/** - * Disable specific symbol in the cache - * @param cache - * @param symbol - */ -void rspamd_symbols_cache_disable_symbol (struct symbols_cache *cache, - const gchar *symbol); - -/** - * Enable specific symbol in the cache - * @param cache - * @param symbol - */ -void rspamd_symbols_cache_enable_symbol (struct symbols_cache *cache, - const gchar *symbol); -/** - * Get abstract callback data for a symbol (or its parent symbol) - * @param cache cache object - * @param symbol symbol name - * @return abstract callback data or NULL if symbol is absent or has no data attached - */ -struct rspamd_abstract_callback_data* rspamd_symbols_cache_get_cbdata ( - struct symbols_cache *cache, const gchar *symbol); - -/** - * Sets new callback data for a symbol in cache - * @param cache - * @param symbol - * @param cbdata - * @return - */ -gboolean rspamd_symbols_cache_set_cbdata (struct symbols_cache *cache, - const gchar *symbol, struct rspamd_abstract_callback_data *cbdata); - -/** - * Process settings for task - * @param task - * @param cache - * @return - */ -gboolean rspamd_symbols_cache_process_settings (struct rspamd_task *task, - struct symbols_cache *cache); - - -/** - * Checks if a symbol specified has been checked (or disabled) - * @param task - * @param cache - * @param symbol - * @return - */ -gboolean rspamd_symbols_cache_is_checked (struct rspamd_task *task, - struct symbols_cache *cache, const gchar *symbol); - -/** - * Returns checksum for all cache items - * @param cache - * @return - */ -guint64 rspamd_symbols_cache_get_cksum (struct symbols_cache *cache); - -/** - * Checks if a symbols is enabled (not checked and conditions return true if present) - * @param task - * @param cache - * @param symbol - * @return - */ -gboolean rspamd_symbols_cache_is_symbol_enabled (struct rspamd_task *task, - struct symbols_cache *cache, const gchar *symbol); -/** - * Process specific function for each cache element (in order they are added) - * @param cache - * @param func - * @param ud - */ -void rspamd_symbols_cache_foreach (struct symbols_cache *cache, - void (*func)(gint /* id */, const gchar * /* name */, - gint /* flags */, gpointer /* userdata */), - gpointer ud); - -#endif diff --git a/src/libserver/task.c b/src/libserver/task.c index f4a09fd30..eaa379361 100644 --- a/src/libserver/task.c +++ b/src/libserver/task.c @@ -62,8 +62,9 @@ rspamd_request_header_dtor (gpointer p) */ struct rspamd_task * rspamd_task_new (struct rspamd_worker *worker, struct rspamd_config *cfg, - rspamd_mempool_t *pool, - struct rspamd_lang_detector *lang_det) + rspamd_mempool_t *pool, + struct rspamd_lang_detector *lang_det, + struct event_base *ev_base) { struct rspamd_task *new_task; @@ -89,8 +90,23 @@ rspamd_task_new (struct rspamd_worker *worker, struct rspamd_config *cfg, } } + new_task->ev_base = ev_base; + +#ifdef HAVE_EVENT_NO_CACHE_TIME_FUNC + if (ev_base) { + event_base_update_cache_time (ev_base); + event_base_gettimeofday_cached (ev_base, &new_task->tv); + new_task->time_real = tv_to_double (&new_task->tv); + } + else { + gettimeofday (&new_task->tv, NULL); + new_task->time_real = tv_to_double (&new_task->tv); + } +#else gettimeofday (&new_task->tv, NULL); - new_task->time_real = rspamd_get_ticks (FALSE); + new_task->time_real = tv_to_double (&new_task->tv); +#endif + new_task->time_virtual = rspamd_get_virtual_ticks (); new_task->time_real_finish = NAN; new_task->time_virtual_finish = NAN; @@ -233,8 +249,8 @@ rspamd_task_free (struct rspamd_task *task) } if (IS_CT_MULTIPART (p->ct)) { - if (p->specific.mp.children) { - g_ptr_array_free (p->specific.mp.children, TRUE); + if (p->specific.mp->children) { + g_ptr_array_free (p->specific.mp->children, TRUE); } } } @@ -251,9 +267,6 @@ rspamd_task_free (struct rspamd_task *task) if (tp->languages) { g_ptr_array_unref (tp->languages); } - if (tp->unicode_raw_content) { - g_array_free (tp->unicode_raw_content, TRUE); - } } if (task->rcpt_envelope) { @@ -269,6 +282,10 @@ rspamd_task_free (struct rspamd_task *task) rspamd_email_address_free (task->from_envelope); } + if (task->meta_words) { + g_array_free (task->meta_words, TRUE); + } + ucl_object_unref (task->messages); if (task->re_rt) { @@ -744,7 +761,7 @@ rspamd_task_process (struct rspamd_task *task, guint stages) break; case RSPAMD_TASK_STAGE_PRE_FILTERS: - rspamd_symbols_cache_process_symbols (task, task->cfg->cache, + rspamd_symcache_process_symbols (task, task->cfg->cache, RSPAMD_TASK_STAGE_PRE_FILTERS); break; @@ -755,7 +772,7 @@ rspamd_task_process (struct rspamd_task *task, guint stages) break; case RSPAMD_TASK_STAGE_FILTERS: - rspamd_symbols_cache_process_symbols (task, task->cfg->cache, + rspamd_symcache_process_symbols (task, task->cfg->cache, RSPAMD_TASK_STAGE_FILTERS); break; @@ -776,7 +793,7 @@ rspamd_task_process (struct rspamd_task *task, guint stages) break; case RSPAMD_TASK_STAGE_POST_FILTERS: - rspamd_symbols_cache_process_symbols (task, task->cfg->cache, + rspamd_symcache_process_symbols (task, task->cfg->cache, RSPAMD_TASK_STAGE_POST_FILTERS); if ((task->flags & RSPAMD_TASK_FLAG_LEARN_AUTO) && @@ -827,7 +844,7 @@ rspamd_task_process (struct rspamd_task *task, guint stages) rspamd_make_composites (task); break; case RSPAMD_TASK_STAGE_IDEMPOTENT: - rspamd_symbols_cache_process_symbols (task, task->cfg->cache, + rspamd_symcache_process_symbols (task, task->cfg->cache, RSPAMD_TASK_STAGE_IDEMPOTENT); break; @@ -1645,8 +1662,24 @@ rspamd_task_profile_get (struct rspamd_task *task, const gchar *key) gboolean rspamd_task_set_finish_time (struct rspamd_task *task) { + struct timeval tv; + if (isnan (task->time_real_finish)) { - task->time_real_finish = rspamd_get_ticks (FALSE); + +#ifdef HAVE_EVENT_NO_CACHE_TIME_FUNC + if (task->ev_base) { + event_base_update_cache_time (task->ev_base); + event_base_gettimeofday_cached (task->ev_base, &tv); + task->time_real_finish = tv_to_double (&tv); + } + else { + gettimeofday (&tv, NULL); + task->time_real_finish = tv_to_double (&tv); + } +#else + gettimeofday (&tv, NULL); + task->time_real_finish = tv_to_double (&tv); +#endif task->time_virtual_finish = rspamd_get_virtual_ticks (); return TRUE; diff --git a/src/libserver/task.h b/src/libserver/task.h index c1eec96ed..34e160dc0 100644 --- a/src/libserver/task.h +++ b/src/libserver/task.h @@ -116,6 +116,7 @@ enum rspamd_task_stage { #define RSPAMD_TASK_FLAG_OWN_POOL (1 << 27) #define RSPAMD_TASK_FLAG_MILTER (1 << 28) #define RSPAMD_TASK_FLAG_SSL (1 << 29) +#define RSPAMD_TASK_FLAG_BAD_UNICODE (1 << 30) #define RSPAMD_TASK_IS_SKIPPED(task) (((task)->flags & RSPAMD_TASK_FLAG_SKIP)) #define RSPAMD_TASK_IS_JSON(task) (((task)->flags & RSPAMD_TASK_FLAG_JSON)) @@ -173,6 +174,8 @@ struct rspamd_task { struct rspamd_metric_result *result; /**< Metric result */ GHashTable *lua_cache; /**< cache of lua objects */ GPtrArray *tokens; /**< statistics tokens */ + GArray *meta_words; /**< rspamd_stat_token_t produced from meta headers + (e.g. Subject) */ GPtrArray *rcpt_mime; GPtrArray *rcpt_envelope; /**< array of rspamd_email_address */ @@ -212,9 +215,10 @@ struct rspamd_task { * Construct new task for worker */ struct rspamd_task *rspamd_task_new (struct rspamd_worker *worker, - struct rspamd_config *cfg, - rspamd_mempool_t *pool, - struct rspamd_lang_detector *lang_det); + struct rspamd_config *cfg, + rspamd_mempool_t *pool, + struct rspamd_lang_detector *lang_det, + struct event_base *ev_base); /** * Destroy task object and remove its IO dispatcher if it exists */ diff --git a/src/libserver/url.c b/src/libserver/url.c index 9e6ab72db..e27a2c39b 100644 --- a/src/libserver/url.c +++ b/src/libserver/url.c @@ -2546,6 +2546,7 @@ rspamd_url_text_part_callback (struct rspamd_url *url, gsize start_offset, ex->pos = start_offset; ex->len = end_offset - start_offset; ex->type = RSPAMD_EXCEPTION_URL; + ex->ptr = url; if (url->protocol == PROTOCOL_MAILTO) { if (url->userlen > 0) { diff --git a/src/libserver/worker_util.c b/src/libserver/worker_util.c index cdd98ded5..a0e511929 100644 --- a/src/libserver/worker_util.c +++ b/src/libserver/worker_util.c @@ -601,6 +601,7 @@ rspamd_fork_worker (struct rspamd_main *rspamd_main, wrk->finish_actions = g_ptr_array_new (); wrk->ppid = getpid (); wrk->pid = fork (); + wrk->cores_throttled = rspamd_main->cores_throttling; switch (wrk->pid) { case 0: diff --git a/src/libstat/backends/redis_backend.c b/src/libstat/backends/redis_backend.c index 74d8c3bf1..b003d5a27 100644 --- a/src/libstat/backends/redis_backend.c +++ b/src/libstat/backends/redis_backend.c @@ -42,9 +42,9 @@ INIT_LOG_MODULE(stat_redis) #define REDIS_STAT_TIMEOUT 30 struct redis_stat_ctx { + lua_State *L; struct rspamd_statfile_config *stcf; - struct upstream_list *read_servers; - struct upstream_list *write_servers; + gint conf_ref; struct rspamd_stat_async_elt *stat_elt; const gchar *redis_object; const gchar *password; @@ -104,12 +104,29 @@ struct rspamd_redis_stat_cbdata { #define GET_TASK_ELT(task, elt) (task == NULL ? NULL : (task)->elt) +static const gchar *M = "redis statistics"; + static GQuark rspamd_redis_stat_quark (void) { - return g_quark_from_static_string ("redis-statistics"); + return g_quark_from_static_string (M); } +static inline struct upstream_list * +rspamd_redis_get_servers (struct redis_stat_ctx *ctx, + const gchar *what) +{ + lua_State *L = ctx->L; + struct upstream_list *res; + + lua_rawgeti (L, LUA_REGISTRYINDEX, ctx->conf_ref); + lua_pushstring (L, what); + lua_gettable (L, -2); + res = *((struct upstream_list**)lua_touserdata (L, -1)); + lua_settop (L, 0); + + return res; +} /* * Non-static for lua unit testing @@ -134,6 +151,7 @@ rspamd_redis_expand_object (const gchar *pattern, GString *tb; const gchar *rcpt = NULL; gint err_idx; + gboolean expansion_errored = FALSE; g_assert (ctx != NULL); stcf = ctx->stcf; @@ -196,6 +214,9 @@ rspamd_redis_expand_object (const gchar *pattern, if (elt) { tlen += strlen (elt); } + else { + expansion_errored = TRUE; + } break; case 'r': @@ -209,11 +230,15 @@ rspamd_redis_expand_object (const gchar *pattern, if (elt) { tlen += strlen (elt); } + else { + expansion_errored = TRUE; + } break; case 'l': if (stcf->label) { tlen += strlen (stcf->label); } + /* Label miss is OK */ break; case 's': if (ctx->new_schema) { @@ -251,7 +276,8 @@ rspamd_redis_expand_object (const gchar *pattern, } } - if (target == NULL || task == NULL) { + + if (target == NULL || task == NULL || expansion_errored) { return tlen; } @@ -501,14 +527,14 @@ rspamd_redis_tokens_to_query (struct rspamd_task *task, "HSET %b_tokens %b %b:%b", prefix, (size_t) prefix_len, n0, (size_t) l0, - tok->t1->begin, tok->t1->len, - tok->t2->begin, tok->t2->len); + tok->t1->stemmed.begin, tok->t1->stemmed.len, + tok->t2->stemmed.begin, tok->t2->stemmed.len); } else if (tok->t1) { redisAsyncCommand (rt->redis, NULL, NULL, "HSET %b_tokens %b %b", prefix, (size_t) prefix_len, n0, (size_t) l0, - tok->t1->begin, tok->t1->len); + tok->t1->stemmed.begin, tok->t1->stemmed.len); } } else { @@ -522,14 +548,14 @@ rspamd_redis_tokens_to_query (struct rspamd_task *task, "HSET %b %s %b:%b", n0, (size_t) l0, "tokens", - tok->t1->begin, tok->t1->len, - tok->t2->begin, tok->t2->len); + tok->t1->stemmed.begin, tok->t1->stemmed.len, + tok->t2->stemmed.begin, tok->t2->stemmed.len); } else if (tok->t1) { redisAsyncCommand (rt->redis, NULL, NULL, "HSET %b %s %b", n0, (size_t) l0, "tokens", - tok->t1->begin, tok->t1->len); + tok->t1->stemmed.begin, tok->t1->stemmed.len); } } @@ -928,6 +954,7 @@ rspamd_redis_async_stat_cb (struct rspamd_stat_async_elt *elt, gpointer d) struct rspamd_redis_stat_elt *redis_elt = elt->ud; struct rspamd_redis_stat_cbdata *cbdata; rspamd_inet_addr_t *addr; + struct upstream_list *ups; g_assert (redis_elt != NULL); @@ -941,8 +968,15 @@ rspamd_redis_async_stat_cb (struct rspamd_stat_async_elt *elt, gpointer d) /* Disable further events unless needed */ elt->enabled = FALSE; + ups = rspamd_redis_get_servers (ctx, "read_servers"); + + if (!ups) { + return; + } + cbdata = g_malloc0 (sizeof (*cbdata)); - cbdata->selected = rspamd_upstream_get (ctx->read_servers, + + cbdata->selected = rspamd_upstream_get (ups, RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0); @@ -1231,78 +1265,6 @@ rspamd_redis_learned (redisAsyncContext *c, gpointer r, gpointer priv) rspamd_session_remove_event (task->s, rspamd_redis_fin_learn, rt); } } - -static gboolean -rspamd_redis_try_ucl (struct redis_stat_ctx *backend, - const ucl_object_t *obj, - struct rspamd_config *cfg, - const gchar *symbol) -{ - const ucl_object_t *elt, *relt; - - elt = ucl_object_lookup_any (obj, "read_servers", "servers", NULL); - - if (elt == NULL) { - return FALSE; - } - - backend->read_servers = rspamd_upstreams_create (cfg->ups_ctx); - if (!rspamd_upstreams_from_ucl (backend->read_servers, elt, - REDIS_DEFAULT_PORT, NULL)) { - msg_err ("statfile %s cannot get read servers configuration", - symbol); - return FALSE; - } - - relt = elt; - - elt = ucl_object_lookup (obj, "write_servers"); - if (elt == NULL) { - /* Use read servers as write ones */ - g_assert (relt != NULL); - backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx); - if (!rspamd_upstreams_from_ucl (backend->write_servers, relt, - REDIS_DEFAULT_PORT, NULL)) { - msg_err ("statfile %s cannot get write servers configuration", - symbol); - return FALSE; - } - } - else { - backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx); - if (!rspamd_upstreams_from_ucl (backend->write_servers, elt, - REDIS_DEFAULT_PORT, NULL)) { - msg_err ("statfile %s cannot get write servers configuration", - symbol); - rspamd_upstreams_destroy (backend->write_servers); - backend->write_servers = NULL; - } - } - - elt = ucl_object_lookup_any (obj, "db", "database", "dbname", NULL); - if (elt) { - if (ucl_object_type (elt) == UCL_STRING) { - backend->dbname = ucl_object_tostring (elt); - } - else if (ucl_object_type (elt) == UCL_INT) { - backend->dbname = ucl_object_tostring_forced (elt); - } - } - else { - backend->dbname = NULL; - } - - elt = ucl_object_lookup (obj, "password"); - if (elt) { - backend->password = ucl_object_tostring (elt); - } - else { - backend->password = NULL; - } - - return TRUE; -} - static void rspamd_redis_parse_classifier_opts (struct redis_stat_ctx *backend, const ucl_object_t *obj, @@ -1360,14 +1322,6 @@ rspamd_redis_parse_classifier_opts (struct redis_stat_ctx *backend, backend->redis_object = ucl_object_tostring (elt); } - elt = ucl_object_lookup (obj, "timeout"); - if (elt) { - backend->timeout = ucl_object_todouble (elt); - } - else { - backend->timeout = REDIS_DEFAULT_TIMEOUT; - } - elt = ucl_object_lookup (obj, "store_tokens"); if (elt) { backend->store_tokens = ucl_object_toboolean (elt); @@ -1414,24 +1368,27 @@ rspamd_redis_init (struct rspamd_stat_ctx *ctx, struct rspamd_redis_stat_elt *st_elt; const ucl_object_t *obj; gboolean ret = FALSE; + gint conf_ref = -1; + lua_State *L = (lua_State *)cfg->lua_state; backend = g_malloc0 (sizeof (*backend)); + backend->L = L; + backend->timeout = REDIS_DEFAULT_TIMEOUT; /* First search in backend configuration */ obj = ucl_object_lookup (st->classifier->cfg->opts, "backend"); if (obj != NULL && ucl_object_type (obj) == UCL_OBJECT) { - ret = rspamd_redis_try_ucl (backend, obj, cfg, stf->symbol); + ret = rspamd_lua_try_load_redis (L, obj, cfg, &conf_ref); } /* Now try statfiles config */ - if (!ret) { - ret = rspamd_redis_try_ucl (backend, stf->opts, cfg, stf->symbol); + if (!ret && stf->opts) { + ret = rspamd_lua_try_load_redis (L, stf->opts, cfg, &conf_ref); } /* Now try classifier config */ - if (!ret) { - ret = rspamd_redis_try_ucl (backend, st->classifier->cfg->opts, cfg, - stf->symbol); + if (!ret && st->classifier->cfg->opts) { + ret = rspamd_lua_try_load_redis (L, st->classifier->cfg->opts, cfg, &conf_ref); } /* Now try global redis settings */ @@ -1444,12 +1401,12 @@ rspamd_redis_init (struct rspamd_stat_ctx *ctx, specific_obj = ucl_object_lookup (obj, "statistics"); if (specific_obj) { - ret = rspamd_redis_try_ucl (backend, specific_obj, cfg, - stf->symbol); + ret = rspamd_lua_try_load_redis (L, + specific_obj, cfg, &conf_ref); } else { - ret = rspamd_redis_try_ucl (backend, obj, cfg, - stf->symbol); + ret = rspamd_lua_try_load_redis (L, + obj, cfg, &conf_ref); } } } @@ -1460,6 +1417,36 @@ rspamd_redis_init (struct rspamd_stat_ctx *ctx, return NULL; } + backend->conf_ref = conf_ref; + + /* Check some common table values */ + lua_rawgeti (L, LUA_REGISTRYINDEX, conf_ref); + + lua_pushstring (L, "timeout"); + lua_gettable (L, -2); + if (lua_type (L, -1) == LUA_TNUMBER) { + backend->timeout = lua_tonumber (L, -1); + } + lua_pop (L, 1); + + lua_pushstring (L, "db"); + lua_gettable (L, -2); + if (lua_type (L, -1) == LUA_TSTRING) { + backend->dbname = rspamd_mempool_strdup (cfg->cfg_pool, + lua_tostring (L, -1)); + } + lua_pop (L, 1); + + lua_pushstring (L, "password"); + lua_gettable (L, -2); + if (lua_type (L, -1) == LUA_TSTRING) { + backend->password = rspamd_mempool_strdup (cfg->cfg_pool, + lua_tostring (L, -1)); + } + lua_pop (L, 1); + + lua_settop (L, 0); + rspamd_redis_parse_classifier_opts (backend, st->classifier->cfg->opts, cfg); stf->clcf->flags |= RSPAMD_FLAG_CLASSIFIER_INCREMENTING_BACKEND; backend->stcf = stf; @@ -1485,24 +1472,35 @@ rspamd_redis_runtime (struct rspamd_task *task, struct redis_stat_ctx *ctx = REDIS_CTX (c); struct redis_stat_runtime *rt; struct upstream *up; + struct upstream_list *ups; + char *object_expanded = NULL; rspamd_inet_addr_t *addr; g_assert (ctx != NULL); g_assert (stcf != NULL); - if (learn && ctx->write_servers == NULL) { - msg_err_task ("no write servers defined for %s, cannot learn", stcf->symbol); - return NULL; - } - if (learn) { - up = rspamd_upstream_get (ctx->write_servers, + ups = rspamd_redis_get_servers (ctx, "write_servers"); + + if (!ups) { + msg_err_task ("no write servers defined for %s, cannot learn", + stcf->symbol); + return NULL; + } + up = rspamd_upstream_get (ups, RSPAMD_UPSTREAM_MASTER_SLAVE, NULL, 0); } else { - up = rspamd_upstream_get (ctx->read_servers, + ups = rspamd_redis_get_servers (ctx, "read_servers"); + + if (!ups) { + msg_err_task ("no read servers defined for %s, cannot stat", + stcf->symbol); + return NULL; + } + up = rspamd_upstream_get (ups, RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0); @@ -1513,15 +1511,22 @@ rspamd_redis_runtime (struct rspamd_task *task, return NULL; } + if (rspamd_redis_expand_object (ctx->redis_object, ctx, task, + &object_expanded) == 0) { + msg_err_task ("expansion for learning failed for symbol %s " + "(maybe learning per user classifier with no user or recipient)", + stcf->symbol); + return NULL; + } + rt = rspamd_mempool_alloc0 (task->task_pool, sizeof (*rt)); rspamd_mempool_add_destructor (task->task_pool, rspamd_gerror_free_maybe, &rt->err); - rspamd_redis_expand_object (ctx->redis_object, ctx, task, - &rt->redis_object_expanded); rt->selected = up; rt->task = task; rt->ctx = ctx; rt->stcf = stcf; + rt->redis_object_expanded = object_expanded; addr = rspamd_upstream_addr (up); g_assert (addr != NULL); @@ -1549,13 +1554,10 @@ void rspamd_redis_close (gpointer p) { struct redis_stat_ctx *ctx = REDIS_CTX (p); + lua_State *L = ctx->L; - if (ctx->read_servers) { - rspamd_upstreams_destroy (ctx->read_servers); - } - - if (ctx->write_servers) { - rspamd_upstreams_destroy (ctx->write_servers); + if (ctx->conf_ref) { + luaL_unref (L, LUA_REGISTRYINDEX, ctx->conf_ref); } g_free (ctx); @@ -1594,7 +1596,7 @@ rspamd_redis_process_tokens (struct rspamd_task *task, if (redisAsyncCommand (rt->redis, rspamd_redis_connected, rt, "HGET %s %s", rt->redis_object_expanded, learned_key) == REDIS_OK) { - rspamd_session_add_event (task->s, NULL, rspamd_redis_fin, rt, rspamd_redis_stat_quark ()); + rspamd_session_add_event (task->s, rspamd_redis_fin, rt, M); rt->has_event = TRUE; if (rspamd_event_pending (&rt->timeout_event, EV_TIMEOUT)) { @@ -1657,6 +1659,7 @@ rspamd_redis_learn_tokens (struct rspamd_task *task, GPtrArray *tokens, { struct redis_stat_runtime *rt = REDIS_RUNTIME (p); struct upstream *up; + struct upstream_list *ups; rspamd_inet_addr_t *addr; struct timeval tv; rspamd_fstring_t *query; @@ -1670,7 +1673,12 @@ rspamd_redis_learn_tokens (struct rspamd_task *task, GPtrArray *tokens, return FALSE; } - up = rspamd_upstream_get (rt->ctx->write_servers, + ups = rspamd_redis_get_servers (rt->ctx, "write_servers"); + + if (!ups) { + return FALSE; + } + up = rspamd_upstream_get (ups, RSPAMD_UPSTREAM_MASTER_SLAVE, NULL, 0); @@ -1798,7 +1806,7 @@ rspamd_redis_learn_tokens (struct rspamd_task *task, GPtrArray *tokens, "RSIG"); } - rspamd_session_add_event (task->s, NULL, rspamd_redis_fin_learn, rt, rspamd_redis_stat_quark ()); + rspamd_session_add_event (task->s, rspamd_redis_fin_learn, rt, M); rt->has_event = TRUE; /* Set timeout */ diff --git a/src/libstat/classifiers/bayes.c b/src/libstat/classifiers/bayes.c index 5b6b5a0fe..2b0cf21e8 100644 --- a/src/libstat/classifiers/bayes.c +++ b/src/libstat/classifiers/bayes.c @@ -38,7 +38,7 @@ G_STRFUNC, \ __VA_ARGS__) -INIT_LOG_MODULE(bayes) +INIT_LOG_MODULE_PUBLIC(bayes) static inline GQuark bayes_error_quark (void) @@ -80,6 +80,8 @@ inv_chi_square (struct rspamd_task *task, gdouble value, gint freedom_deg) sum = prob; + msg_debug_bayes ("m: %f, prob: %g", m, prob); + /* * m is our confidence in class * prob is e ^ x (small value since x is normally less than zero @@ -89,7 +91,7 @@ inv_chi_square (struct rspamd_task *task, gdouble value, gint freedom_deg) for (i = 1; i < freedom_deg; i++) { prob *= m / (gdouble)i; sum += prob; - msg_debug_bayes ("prob: %.6f, sum: %.6f", prob, sum); + msg_debug_bayes ("i=%d, prob: %g, sum: %g", i, prob, sum); } return MIN (1.0, sum); @@ -109,7 +111,7 @@ struct bayes_task_closure { * Mathematically we use pow(complexity, complexity), where complexity is the * window index */ -static const double feature_weight[] = { 0, 1, 4, 27, 256, 3125, 46656, 823543 }; +static const double feature_weight[] = { 0, 3125, 256, 27, 1, 0, 0, 0 }; #define PROB_COMBINE(prob, cnt, weight, assumed) (((weight) * (assumed) + (cnt) * (prob)) / ((weight) + (cnt))) /* @@ -121,12 +123,12 @@ bayes_classify_token (struct rspamd_classifier *ctx, { guint i; gint id; - guint64 spam_count = 0, ham_count = 0, total_count = 0; + guint spam_count = 0, ham_count = 0, total_count = 0; struct rspamd_statfile *st; struct rspamd_task *task; const gchar *token_type = "txt"; double spam_prob, spam_freq, ham_freq, bayes_spam_prob, bayes_ham_prob, - ham_prob, fw, w, norm_sum, norm_sub, val; + ham_prob, fw, w, val; task = cl->task; @@ -145,8 +147,8 @@ bayes_classify_token (struct rspamd_classifier *ctx, msg_debug_bayes ( "token(meta) %uL <%*s:%*s> probabilistically skipped", tok->data, - (int) tok->t1->len, tok->t1->begin, - (int) tok->t2->len, tok->t2->begin); + (int) tok->t1->original.len, tok->t1->original.begin, + (int) tok->t2->original.len, tok->t2->original.begin); } return; @@ -173,7 +175,7 @@ bayes_classify_token (struct rspamd_classifier *ctx, } /* Probability for this token */ - if (total_count > 0) { + if (total_count >= ctx->cfg->min_token_hits) { spam_freq = ((double)spam_count / MAX (1., (double) ctx->spam_learns)); ham_freq = ((double)ham_count / MAX (1., (double)ctx->ham_learns)); spam_prob = spam_freq / (spam_freq + ham_freq); @@ -187,19 +189,27 @@ bayes_classify_token (struct rspamd_classifier *ctx, G_N_ELEMENTS (feature_weight)]; } - norm_sum = (spam_freq + ham_freq) * (spam_freq + ham_freq); - norm_sub = (spam_freq - ham_freq) * (spam_freq - ham_freq); - w = (norm_sub) / (norm_sum) * - (fw * total_count) / (4.0 * (1.0 + fw * total_count)); + w = (fw * total_count) / (1.0 + fw * total_count); + bayes_spam_prob = PROB_COMBINE (spam_prob, total_count, w, 0.5); - norm_sub = (ham_freq - spam_freq) * (ham_freq - spam_freq); - w = (norm_sub) / (norm_sum) * - (fw * total_count) / (4.0 * (1.0 + fw * total_count)); + + if ((bayes_spam_prob > 0.5 && bayes_spam_prob < 0.5 + ctx->cfg->min_prob_strength) || + (bayes_spam_prob < 0.5 && bayes_spam_prob > 0.5 - ctx->cfg->min_prob_strength)) { + msg_debug_bayes ( + "token %uL <%*s:%*s> skipped, prob not in range: %f", + tok->data, + (int) tok->t1->stemmed.len, tok->t1->stemmed.begin, + (int) tok->t2->stemmed.len, tok->t2->stemmed.begin, + bayes_spam_prob); + + return; + } + bayes_ham_prob = PROB_COMBINE (ham_prob, total_count, w, 0.5); - cl->spam_prob += log2 (bayes_spam_prob); - cl->ham_prob += log2 (bayes_ham_prob); + cl->spam_prob += log (bayes_spam_prob); + cl->ham_prob += log (bayes_ham_prob); cl->processed_tokens ++; if (!(tok->flags & RSPAMD_STAT_TOKEN_FLAG_META)) { @@ -210,29 +220,31 @@ bayes_classify_token (struct rspamd_classifier *ctx, } if (tok->t1 && tok->t2) { - msg_debug_bayes ("token(%s) %uL <%*s:%*s>: weight: %f, total_count: %L, " - "spam_count: %L, ham_count: %L," + msg_debug_bayes ("token(%s) %uL <%*s:%*s>: weight: %f, cf: %f, " + "total_count: %ud, " + "spam_count: %ud, ham_count: %ud," "spam_prob: %.3f, ham_prob: %.3f, " "bayes_spam_prob: %.3f, bayes_ham_prob: %.3f, " "current spam prob: %.3f, current ham prob: %.3f", token_type, tok->data, - (int) tok->t1->len, tok->t1->begin, - (int) tok->t2->len, tok->t2->begin, - fw, total_count, spam_count, ham_count, + (int) tok->t1->stemmed.len, tok->t1->stemmed.begin, + (int) tok->t2->stemmed.len, tok->t2->stemmed.begin, + fw, w, total_count, spam_count, ham_count, spam_prob, ham_prob, bayes_spam_prob, bayes_ham_prob, cl->spam_prob, cl->ham_prob); } else { - msg_debug_bayes ("token(%s) %uL <?:?>: weight: %f, total_count: %L, " - "spam_count: %L, ham_count: %L," + msg_debug_bayes ("token(%s) %uL <?:?>: weight: %f, cf: %f, " + "total_count: %ud, " + "spam_count: %ud, ham_count: %ud," "spam_prob: %.3f, ham_prob: %.3f, " "bayes_spam_prob: %.3f, bayes_ham_prob: %.3f, " "current spam prob: %.3f, current ham prob: %.3f", token_type, tok->data, - fw, total_count, spam_count, ham_count, + fw, w, total_count, spam_count, ham_count, spam_prob, ham_prob, bayes_spam_prob, bayes_ham_prob, cl->spam_prob, cl->ham_prob); @@ -243,13 +255,20 @@ bayes_classify_token (struct rspamd_classifier *ctx, gboolean -bayes_init (rspamd_mempool_t *pool, struct rspamd_classifier *cl) +bayes_init (struct rspamd_config *cfg, + struct event_base *ev_base, + struct rspamd_classifier *cl) { cl->cfg->flags |= RSPAMD_FLAG_CLASSIFIER_INTEGER; return TRUE; } +void +bayes_fin (struct rspamd_classifier *cl) +{ +} + gboolean bayes_classify (struct rspamd_classifier * ctx, GPtrArray *tokens, @@ -318,14 +337,51 @@ bayes_classify (struct rspamd_classifier * ctx, bayes_classify_token (ctx, tok, &cl); } - h = 1 - inv_chi_square (task, cl.spam_prob, cl.processed_tokens); - s = 1 - inv_chi_square (task, cl.ham_prob, cl.processed_tokens); + if (cl.processed_tokens == 0) { + msg_info_bayes ("no tokens found in bayes database " + "(%ud total tokens, %ud text tokens), ignore stats", + tokens->len, text_tokens); + + return TRUE; + } + + if (ctx->cfg->min_tokens > 0 && + cl.text_tokens < (gint)(ctx->cfg->min_tokens * 0.1)) { + msg_info_bayes ("ignore bayes probability since we have " + "found too few text tokens: %uL (of %ud checked), " + "at least %d is required", + cl.text_tokens, + text_tokens, + (gint)(ctx->cfg->min_tokens * 0.1)); + + return TRUE; + } + + if (cl.spam_prob > -300 && cl.ham_prob > -300) { + /* Fisher value is low enough to apply inv_chi_square */ + h = 1 - inv_chi_square (task, cl.spam_prob, cl.processed_tokens); + s = 1 - inv_chi_square (task, cl.ham_prob, cl.processed_tokens); + } + else { + /* Use naive method */ + if (cl.spam_prob < cl.ham_prob) { + h = (1.0 - exp(cl.spam_prob - cl.ham_prob)) / + (1.0 + exp(cl.spam_prob - cl.ham_prob)); + s = 1.0 - h; + } + else { + s = (1.0 - exp(cl.ham_prob - cl.spam_prob)) / + (1.0 + exp(cl.ham_prob - cl.spam_prob)); + h = 1.0 - s; + } + } if (isfinite (s) && isfinite (h)) { final_prob = (s + 1.0 - h) / 2.; msg_debug_bayes ( "<%s> got ham prob %.2f -> %.2f and spam prob %.2f -> %.2f," - " %L tokens processed of %ud total tokens (%uL text tokens)", + " %L tokens processed of %ud total tokens;" + " %uL text tokens found of %ud text tokens)", task->message_id, cl.ham_prob, h, @@ -333,7 +389,8 @@ bayes_classify (struct rspamd_classifier * ctx, s, cl.processed_tokens, tokens->len, - cl.text_tokens); + cl.text_tokens, + text_tokens); } else { /* @@ -357,17 +414,6 @@ bayes_classify (struct rspamd_classifier * ctx, } } - if (ctx->cfg->min_tokens > 0 && - cl.text_tokens < (gint)(ctx->cfg->min_tokens * 0.1)) { - msg_info_bayes ("ignore bayes probability %.2f since we have " - "too few text tokens: %uL, at least %d is required", - final_prob, - cl.text_tokens, - (gint)(ctx->cfg->min_tokens * 0.1)); - - return TRUE; - } - pprob = rspamd_mempool_alloc (task->task_pool, sizeof (*pprob)); *pprob = final_prob; rspamd_mempool_set_variable (task->task_pool, "bayes_prob", pprob, NULL); @@ -399,6 +445,19 @@ bayes_classify (struct rspamd_classifier * ctx, (final_prob - 0.5) * 200.); final_prob = rspamd_normalize_probability (final_prob, 0.5); g_assert (st != NULL); + + if (final_prob > 1 || final_prob < 0) { + msg_err_bayes ("internal error: probability %f is outside of the " + "allowed range [0..1]", final_prob); + + if (final_prob > 1) { + final_prob = 1.0; + } + else { + final_prob = 0.0; + } + } + rspamd_task_insert_result (task, st->stcf->symbol, final_prob, @@ -483,8 +542,8 @@ bayes_learn_spam (struct rspamd_classifier * ctx, msg_debug_bayes ("token %uL <%*s:%*s>: window: %d, total_count: %d, " "spam_count: %d, ham_count: %d", tok->data, - (int) tok->t1->len, tok->t1->begin, - (int) tok->t2->len, tok->t2->begin, + (int) tok->t1->stemmed.len, tok->t1->stemmed.begin, + (int) tok->t2->stemmed.len, tok->t2->stemmed.begin, tok->window_idx, total_cnt, spam_cnt, ham_cnt); } else { diff --git a/src/libstat/classifiers/classifiers.h b/src/libstat/classifiers/classifiers.h index e30f2153a..fd6daf433 100644 --- a/src/libstat/classifiers/classifiers.h +++ b/src/libstat/classifiers/classifiers.h @@ -3,6 +3,7 @@ #include "config.h" #include "mem_pool.h" +#include <event.h> #define RSPAMD_DEFAULT_CLASSIFIER "bayes" /* Consider this value as 0 */ @@ -10,28 +11,32 @@ struct rspamd_classifier_config; struct rspamd_task; +struct rspamd_config; struct rspamd_classifier; struct token_node_s; struct rspamd_stat_classifier { char *name; - gboolean (*init_func)(rspamd_mempool_t *pool, - struct rspamd_classifier *cl); + gboolean (*init_func)(struct rspamd_config *cfg, + struct event_base *ev_base, + struct rspamd_classifier *cl); gboolean (*classify_func)(struct rspamd_classifier * ctx, - GPtrArray *tokens, - struct rspamd_task *task); + GPtrArray *tokens, + struct rspamd_task *task); gboolean (*learn_spam_func)(struct rspamd_classifier * ctx, - GPtrArray *input, - struct rspamd_task *task, - gboolean is_spam, - gboolean unlearn, - GError **err); + GPtrArray *input, + struct rspamd_task *task, + gboolean is_spam, + gboolean unlearn, + GError **err); + void (*fin_func)(struct rspamd_classifier *cl); }; /* Bayes algorithm */ -gboolean bayes_init (rspamd_mempool_t *pool, - struct rspamd_classifier *); +gboolean bayes_init (struct rspamd_config *cfg, + struct event_base *ev_base, + struct rspamd_classifier *); gboolean bayes_classify (struct rspamd_classifier *ctx, GPtrArray *tokens, struct rspamd_task *task); @@ -41,10 +46,12 @@ gboolean bayes_learn_spam (struct rspamd_classifier *ctx, gboolean is_spam, gboolean unlearn, GError **err); +void bayes_fin (struct rspamd_classifier *); /* Generic lua classifier */ -gboolean lua_classifier_init (rspamd_mempool_t *pool, - struct rspamd_classifier *); +gboolean lua_classifier_init (struct rspamd_config *cfg, + struct event_base *ev_base, + struct rspamd_classifier *); gboolean lua_classifier_classify (struct rspamd_classifier *ctx, GPtrArray *tokens, struct rspamd_task *task); @@ -55,6 +62,11 @@ gboolean lua_classifier_learn_spam (struct rspamd_classifier *ctx, gboolean unlearn, GError **err); +extern guint rspamd_bayes_log_id; +#define msg_debug_bayes(...) rspamd_conditional_debug_fast (NULL, task->from_addr, \ + rspamd_bayes_log_id, "bayes", task->task_pool->tag.uid, \ + G_STRFUNC, \ + __VA_ARGS__) #endif /* diff --git a/src/libstat/classifiers/lua_classifier.c b/src/libstat/classifiers/lua_classifier.c index 7b495b165..83ce7b0e1 100644 --- a/src/libstat/classifiers/lua_classifier.c +++ b/src/libstat/classifiers/lua_classifier.c @@ -47,8 +47,9 @@ static GHashTable *lua_classifiers = NULL; INIT_LOG_MODULE(luacl) gboolean -lua_classifier_init (rspamd_mempool_t *pool, - struct rspamd_classifier *cl) +lua_classifier_init (struct rspamd_config *cfg, + struct event_base *ev_base, + struct rspamd_classifier *cl) { struct rspamd_lua_classifier_ctx *ctx; lua_State *L = cl->ctx->cfg->lua_state; @@ -62,7 +63,7 @@ lua_classifier_init (rspamd_mempool_t *pool, ctx = g_hash_table_lookup (lua_classifiers, cl->subrs->name); if (ctx != NULL) { - msg_err_pool ("duplicate lua classifier definition: %s", + msg_err_config ("duplicate lua classifier definition: %s", cl->subrs->name); return FALSE; @@ -70,7 +71,7 @@ lua_classifier_init (rspamd_mempool_t *pool, lua_getglobal (L, "rspamd_classifiers"); if (lua_type (L, -1) != LUA_TTABLE) { - msg_err_pool ("cannot register classifier %s: no rspamd_classifier global", + msg_err_config ("cannot register classifier %s: no rspamd_classifier global", cl->subrs->name); lua_pop (L, 1); @@ -81,7 +82,7 @@ lua_classifier_init (rspamd_mempool_t *pool, lua_gettable (L, -2); if (lua_type (L, -1) != LUA_TTABLE) { - msg_err_pool ("cannot register classifier %s: bad lua type: %s", + msg_err_config ("cannot register classifier %s: bad lua type: %s", cl->subrs->name, lua_typename (L, lua_type (L, -1))); lua_pop (L, 2); @@ -92,7 +93,7 @@ lua_classifier_init (rspamd_mempool_t *pool, lua_gettable (L, -2); if (lua_type (L, -1) != LUA_TFUNCTION) { - msg_err_pool ("cannot register classifier %s: bad lua type for classify: %s", + msg_err_config ("cannot register classifier %s: bad lua type for classify: %s", cl->subrs->name, lua_typename (L, lua_type (L, -1))); lua_pop (L, 3); @@ -105,7 +106,7 @@ lua_classifier_init (rspamd_mempool_t *pool, lua_gettable (L, -2); if (lua_type (L, -1) != LUA_TFUNCTION) { - msg_err_pool ("cannot register classifier %s: bad lua type for learn: %s", + msg_err_config ("cannot register classifier %s: bad lua type for learn: %s", cl->subrs->name, lua_typename (L, lua_type (L, -1))); lua_pop (L, 3); diff --git a/src/libstat/learn_cache/redis_cache.c b/src/libstat/learn_cache/redis_cache.c index 11bc13aae..6a0aa1da7 100644 --- a/src/libstat/learn_cache/redis_cache.c +++ b/src/libstat/learn_cache/redis_cache.c @@ -22,20 +22,23 @@ #include "ucl.h" #include "hiredis.h" #include "adapters/libevent.h" +#include "lua/lua_common.h" #define REDIS_DEFAULT_TIMEOUT 0.5 #define REDIS_STAT_TIMEOUT 30 #define REDIS_DEFAULT_PORT 6379 #define DEFAULT_REDIS_KEY "learned_ids" +static const gchar *M = "redis learn cache"; + struct rspamd_redis_cache_ctx { + lua_State *L; struct rspamd_statfile_config *stcf; - struct upstream_list *read_servers; - struct upstream_list *write_servers; const gchar *password; const gchar *dbname; const gchar *redis_object; gdouble timeout; + gint conf_ref; }; struct rspamd_redis_cache_runtime { @@ -50,7 +53,23 @@ struct rspamd_redis_cache_runtime { static GQuark rspamd_stat_cache_redis_quark (void) { - return g_quark_from_static_string ("redis-statistics"); + return g_quark_from_static_string (M); +} + +static inline struct upstream_list * +rspamd_redis_get_servers (struct rspamd_redis_cache_ctx *ctx, + const gchar *what) +{ + lua_State *L = ctx->L; + struct upstream_list *res; + + lua_rawgeti (L, LUA_REGISTRYINDEX, ctx->conf_ref); + lua_pushstring (L, what); + lua_gettable (L, -2); + res = *((struct upstream_list**)lua_touserdata (L, -1)); + lua_settop (L, 0); + + return res; } static void @@ -208,94 +227,6 @@ rspamd_stat_cache_redis_generate_id (struct rspamd_task *task) rspamd_mempool_set_variable (task->task_pool, "words_hash", b32out, g_free); } -static gboolean -rspamd_redis_cache_try_ucl (struct rspamd_redis_cache_ctx *cache_ctx, - const ucl_object_t *obj, - struct rspamd_config *cfg, - const gchar *symbol) -{ - const ucl_object_t *elt, *relt; - - elt = ucl_object_lookup_any (obj, "read_servers", "servers", NULL); - - if (elt == NULL) { - return FALSE; - } - - cache_ctx->read_servers = rspamd_upstreams_create (cfg->ups_ctx); - if (!rspamd_upstreams_from_ucl (cache_ctx->read_servers, elt, - REDIS_DEFAULT_PORT, NULL)) { - msg_err ("statfile %s cannot get read servers configuration", - symbol); - return FALSE; - } - - relt = elt; - - elt = ucl_object_lookup (obj, "write_servers"); - if (elt == NULL) { - /* Use read servers as write ones */ - g_assert (relt != NULL); - cache_ctx->write_servers = rspamd_upstreams_create (cfg->ups_ctx); - if (!rspamd_upstreams_from_ucl (cache_ctx->write_servers, relt, - REDIS_DEFAULT_PORT, NULL)) { - msg_err ("statfile %s cannot get write servers configuration", - symbol); - return FALSE; - } - } - else { - cache_ctx->write_servers = rspamd_upstreams_create (cfg->ups_ctx); - if (!rspamd_upstreams_from_ucl (cache_ctx->write_servers, elt, - REDIS_DEFAULT_PORT, NULL)) { - msg_err ("statfile %s cannot get write servers configuration", - symbol); - rspamd_upstreams_destroy (cache_ctx->write_servers); - cache_ctx->write_servers = NULL; - } - } - - - elt = ucl_object_lookup (obj, "timeout"); - if (elt) { - cache_ctx->timeout = ucl_object_todouble (elt); - } - else { - cache_ctx->timeout = REDIS_DEFAULT_TIMEOUT; - } - - elt = ucl_object_lookup (obj, "password"); - if (elt) { - cache_ctx->password = ucl_object_tostring (elt); - } - else { - cache_ctx->password = NULL; - } - - elt = ucl_object_lookup_any (obj, "db", "database", "dbname", NULL); - if (elt) { - if (ucl_object_type (elt) == UCL_STRING) { - cache_ctx->dbname = ucl_object_tostring (elt); - } - else if (ucl_object_type (elt) == UCL_INT) { - cache_ctx->dbname = ucl_object_tostring_forced (elt); - } - } - else { - cache_ctx->dbname = NULL; - } - - elt = ucl_object_lookup_any (obj, "cache_key", "key", NULL); - if (elt == NULL || ucl_object_type (elt) != UCL_STRING) { - cache_ctx->redis_object = DEFAULT_REDIS_KEY; - } - else { - cache_ctx->redis_object = ucl_object_tostring (elt); - } - - return TRUE; -} - gpointer rspamd_stat_cache_redis_init (struct rspamd_stat_ctx *ctx, struct rspamd_config *cfg, @@ -306,24 +237,27 @@ rspamd_stat_cache_redis_init (struct rspamd_stat_ctx *ctx, struct rspamd_statfile_config *stf = st->stcf; const ucl_object_t *obj; gboolean ret = FALSE; + lua_State *L = (lua_State *)cfg->lua_state; + gint conf_ref = -1; cache_ctx = g_malloc0 (sizeof (*cache_ctx)); + cache_ctx->timeout = REDIS_DEFAULT_TIMEOUT; + cache_ctx->L = L; /* First search in backend configuration */ obj = ucl_object_lookup (st->classifier->cfg->opts, "backend"); if (obj != NULL && ucl_object_type (obj) == UCL_OBJECT) { - ret = rspamd_redis_cache_try_ucl (cache_ctx, obj, cfg, stf->symbol); + ret = rspamd_lua_try_load_redis (L, obj, cfg, &conf_ref); } /* Now try statfiles config */ - if (!ret) { - ret = rspamd_redis_cache_try_ucl (cache_ctx, stf->opts, cfg, stf->symbol); + if (!ret && stf->opts) { + ret = rspamd_lua_try_load_redis (L, stf->opts, cfg, &conf_ref); } /* Now try classifier config */ - if (!ret) { - ret = rspamd_redis_cache_try_ucl (cache_ctx, st->classifier->cfg->opts, cfg, - stf->symbol); + if (!ret && st->classifier->cfg->opts) { + ret = rspamd_lua_try_load_redis (L, st->classifier->cfg->opts, cfg, &conf_ref); } /* Now try global redis settings */ @@ -336,23 +270,61 @@ rspamd_stat_cache_redis_init (struct rspamd_stat_ctx *ctx, specific_obj = ucl_object_lookup (obj, "statistics"); if (specific_obj) { - ret = rspamd_redis_cache_try_ucl (cache_ctx, specific_obj, cfg, - stf->symbol); + ret = rspamd_lua_try_load_redis (L, + specific_obj, cfg, &conf_ref); } else { - ret = rspamd_redis_cache_try_ucl (cache_ctx, obj, cfg, - stf->symbol); + ret = rspamd_lua_try_load_redis (L, + obj, cfg, &conf_ref); } } } - if (!ret) { msg_err_config ("cannot init redis cache for %s", stf->symbol); g_free (cache_ctx); return NULL; } + obj = ucl_object_lookup (st->classifier->cfg->opts, "cache_key"); + + if (obj) { + cache_ctx->redis_object = ucl_object_tostring (obj); + } + else { + cache_ctx->redis_object = DEFAULT_REDIS_KEY; + } + + cache_ctx->conf_ref = conf_ref; + + /* Check some common table values */ + lua_rawgeti (L, LUA_REGISTRYINDEX, conf_ref); + + lua_pushstring (L, "timeout"); + lua_gettable (L, -2); + if (lua_type (L, -1) == LUA_TNUMBER) { + cache_ctx->timeout = lua_tonumber (L, -1); + } + lua_pop (L, 1); + + lua_pushstring (L, "db"); + lua_gettable (L, -2); + if (lua_type (L, -1) == LUA_TSTRING) { + cache_ctx->dbname = rspamd_mempool_strdup (cfg->cfg_pool, + lua_tostring (L, -1)); + } + lua_pop (L, 1); + + lua_pushstring (L, "password"); + lua_gettable (L, -2); + if (lua_type (L, -1) == LUA_TSTRING) { + cache_ctx->password = rspamd_mempool_strdup (cfg->cfg_pool, + lua_tostring (L, -1)); + } + lua_pop (L, 1); + + lua_settop (L, 0); + cache_ctx->stcf = stf; return (gpointer)cache_ctx; @@ -365,28 +337,39 @@ rspamd_stat_cache_redis_runtime (struct rspamd_task *task, struct rspamd_redis_cache_ctx *ctx = c; struct rspamd_redis_cache_runtime *rt; struct upstream *up; + struct upstream_list *ups; rspamd_inet_addr_t *addr; g_assert (ctx != NULL); - if (learn && ctx->write_servers == NULL) { - msg_err_task ("no write servers defined for %s, cannot learn", - ctx->stcf->symbol); - return NULL; - } - if (task->tokens == NULL || task->tokens->len == 0) { return NULL; } if (learn) { - up = rspamd_upstream_get (ctx->write_servers, + ups = rspamd_redis_get_servers (ctx, "write_servers"); + + if (!ups) { + msg_err_task ("no write servers defined for %s, cannot learn", + ctx->stcf->symbol); + return NULL; + } + + up = rspamd_upstream_get (ups, RSPAMD_UPSTREAM_MASTER_SLAVE, NULL, 0); } else { - up = rspamd_upstream_get (ctx->read_servers, + ups = rspamd_redis_get_servers (ctx, "read_servers"); + + if (!ups) { + msg_err_task ("no read servers defined for %s, cannot check", + ctx->stcf->symbol); + return NULL; + } + + up = rspamd_upstream_get (ups, RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0); @@ -453,7 +436,10 @@ rspamd_stat_cache_redis_check (struct rspamd_task *task, if (redisAsyncCommand (rt->redis, rspamd_stat_cache_redis_get, rt, "HGET %s %s", rt->ctx->redis_object, h) == REDIS_OK) { - rspamd_session_add_event (task->s, NULL, rspamd_redis_cache_fin, rt, rspamd_stat_cache_redis_quark ()); + rspamd_session_add_event (task->s, + rspamd_redis_cache_fin, + rt, + M); event_add (&rt->timeout_event, &tv); rt->has_event = TRUE; } @@ -485,7 +471,8 @@ rspamd_stat_cache_redis_learn (struct rspamd_task *task, if (redisAsyncCommand (rt->redis, rspamd_stat_cache_redis_set, rt, "HSET %s %s %d", rt->ctx->redis_object, h, flag) == REDIS_OK) { - rspamd_session_add_event (task->s, NULL, rspamd_redis_cache_fin, rt, rspamd_stat_cache_redis_quark ()); + rspamd_session_add_event (task->s, + rspamd_redis_cache_fin, rt, M); event_add (&rt->timeout_event, &tv); rt->has_event = TRUE; } @@ -497,5 +484,14 @@ rspamd_stat_cache_redis_learn (struct rspamd_task *task, void rspamd_stat_cache_redis_close (gpointer c) { + struct rspamd_redis_cache_ctx *ctx = (struct rspamd_redis_cache_ctx *)c; + lua_State *L; + + L = ctx->L; + + if (ctx->conf_ref) { + luaL_unref (L, LUA_REGISTRYINDEX, ctx->conf_ref); + } + g_free (ctx); } diff --git a/src/libstat/learn_cache/sqlite3_cache.c b/src/libstat/learn_cache/sqlite3_cache.c index 255c835bb..52921326d 100644 --- a/src/libstat/learn_cache/sqlite3_cache.c +++ b/src/libstat/learn_cache/sqlite3_cache.c @@ -221,6 +221,8 @@ rspamd_stat_cache_sqlite3_check (struct rspamd_task *task, /* We have some existing record in the table */ if (!!flag == !!is_spam) { /* Already learned */ + msg_warn_task ("already seen stat hash: %*bs", + rspamd_cryptobox_HASHBYTES, out); return RSPAMD_LEARN_INGORE; } else { diff --git a/src/libstat/stat_api.h b/src/libstat/stat_api.h index 84db8ee01..9dcd6f8e8 100644 --- a/src/libstat/stat_api.h +++ b/src/libstat/stat_api.h @@ -26,16 +26,25 @@ * High level statistics API */ -#define RSPAMD_STAT_TOKEN_FLAG_TEXT (1 << 0) -#define RSPAMD_STAT_TOKEN_FLAG_META (1 << 1) -#define RSPAMD_STAT_TOKEN_FLAG_LUA_META (1 << 2) -#define RSPAMD_STAT_TOKEN_FLAG_EXCEPTION (1 << 3) -#define RSPAMD_STAT_TOKEN_FLAG_SUBJECT (1 << 4) -#define RSPAMD_STAT_TOKEN_FLAG_UNIGRAM (1 << 5) +#define RSPAMD_STAT_TOKEN_FLAG_TEXT (1u << 0) +#define RSPAMD_STAT_TOKEN_FLAG_META (1u << 1) +#define RSPAMD_STAT_TOKEN_FLAG_LUA_META (1u << 2) +#define RSPAMD_STAT_TOKEN_FLAG_EXCEPTION (1u << 3) +#define RSPAMD_STAT_TOKEN_FLAG_HEADER (1u << 4) +#define RSPAMD_STAT_TOKEN_FLAG_UNIGRAM (1u << 5) +#define RSPAMD_STAT_TOKEN_FLAG_UTF (1u << 6) +#define RSPAMD_STAT_TOKEN_FLAG_NORMALISED (1u << 7) +#define RSPAMD_STAT_TOKEN_FLAG_STEMMED (1u << 8) +#define RSPAMD_STAT_TOKEN_FLAG_BROKEN_UNICODE (1u << 9) +#define RSPAMD_STAT_TOKEN_FLAG_STOP_WORD (1u << 9) +#define RSPAMD_STAT_TOKEN_FLAG_SKIPPED (1u << 10) +#define RSPAMD_STAT_TOKEN_FLAG_INVISIBLE_SPACES (1u << 11) typedef struct rspamd_stat_token_s { - const gchar *begin; - gsize len; + rspamd_ftok_t original; /* utf8 raw */ + rspamd_ftok_unicode_t unicode; /* array of unicode characters, normalized, lowercased */ + rspamd_ftok_t normalized; /* normalized and lowercased utf8 */ + rspamd_ftok_t stemmed; /* stemmed utf8 */ guint flags; } rspamd_stat_token_t; diff --git a/src/libstat/stat_config.c b/src/libstat/stat_config.c index 9d1e57f13..101db4fe6 100644 --- a/src/libstat/stat_config.c +++ b/src/libstat/stat_config.c @@ -28,6 +28,7 @@ static struct rspamd_stat_classifier lua_classifier = { .init_func = lua_classifier_init, .classify_func = lua_classifier_classify, .learn_spam_func = lua_classifier_learn_spam, + .fin_func = NULL, }; static struct rspamd_stat_classifier stat_classifiers[] = { @@ -36,6 +37,7 @@ static struct rspamd_stat_classifier stat_classifiers[] = { .init_func = bayes_init, .classify_func = bayes_classify, .learn_spam_func = bayes_learn_spam, + .fin_func = bayes_fin, } }; @@ -162,6 +164,67 @@ rspamd_stat_init (struct rspamd_config *cfg, struct event_base *ev_base) stat_ctx->classifiers = g_ptr_array_new (); stat_ctx->async_elts = g_queue_new (); stat_ctx->ev_base = ev_base; + stat_ctx->lua_stat_tokens_ref = -1; + + /* Interact with lua_stat */ + if (luaL_dostring (L, "return require \"lua_stat\"") != 0) { + msg_err_config ("cannot require lua_stat: %s", + lua_tostring (L, -1)); + } + else { + if (lua_type (L, -1) != LUA_TTABLE) { + msg_err_config ("lua stat must return " + "table and not %s", + lua_typename (L, lua_type (L, -1))); + } + else { + lua_pushstring (L, "gen_stat_tokens"); + lua_gettable (L, -2); + + if (lua_type (L, -1) != LUA_TFUNCTION) { + msg_err_config ("gen_stat_tokens must return " + "function and not %s", + lua_typename (L, lua_type (L, -1))); + } + else { + /* Call this function to obtain closure */ + gint err_idx, ret; + GString *tb; + struct rspamd_config **pcfg; + + lua_pushcfunction (L, &rspamd_lua_traceback); + err_idx = lua_gettop (L); + lua_pushvalue (L, err_idx - 1); + + pcfg = lua_newuserdata (L, sizeof (*pcfg)); + *pcfg = cfg; + rspamd_lua_setclass (L, "rspamd{config}", -1); + + if ((ret = lua_pcall (L, 1, 1, err_idx)) != 0) { + tb = lua_touserdata (L, -1); + msg_err_config ("call to gen_stat_tokens lua " + "script failed (%d): %v", ret, tb); + + if (tb) { + g_string_free (tb, TRUE); + } + } + else { + if (lua_type (L, -1) != LUA_TFUNCTION) { + msg_err_config ("gen_stat_tokens invocation must return " + "function and not %s", + lua_typename (L, lua_type (L, -1))); + } + else { + stat_ctx->lua_stat_tokens_ref = luaL_ref (L, LUA_REGISTRYINDEX); + } + } + } + } + } + + /* Cleanup mess */ + lua_settop (L, 0); /* Create statfiles from the classifiers */ cur = cfg->classifiers; @@ -182,7 +245,7 @@ rspamd_stat_init (struct rspamd_config *cfg, struct event_base *ev_base) continue; } - if (!cl->subrs->init_func (cfg->cfg_pool, cl)) { + if (!cl->subrs->init_func (cfg, ev_base, cl)) { g_free (cl); msg_err_config ("cannot init classifier type %s", clf->name); cur = g_list_next (cur); @@ -328,6 +391,11 @@ rspamd_stat_close (void) } g_array_free (cl->statfiles_ids, TRUE); + + if (cl->subrs->fin_func) { + cl->subrs->fin_func (cl); + } + g_free (cl); } @@ -342,6 +410,12 @@ rspamd_stat_close (void) g_queue_free (stat_ctx->async_elts); g_ptr_array_free (st_ctx->statfiles, TRUE); g_ptr_array_free (st_ctx->classifiers, TRUE); + + if (st_ctx->lua_stat_tokens_ref != -1) { + luaL_unref (st_ctx->cfg->lua_state, LUA_REGISTRYINDEX, + st_ctx->lua_stat_tokens_ref); + } + g_free (st_ctx); /* Set global var to NULL */ @@ -475,11 +549,11 @@ rspamd_stat_ctx_register_async (rspamd_stat_async_handler handler, g_assert (st_ctx != NULL); elt = g_malloc0 (sizeof (*elt)); - REF_INIT_RETAIN (elt, rspamd_async_elt_dtor); elt->handler = handler; elt->cleanup = cleanup; elt->ud = d; elt->timeout = timeout; + REF_INIT_RETAIN (elt, rspamd_async_elt_dtor); /* Enabled by default */ diff --git a/src/libstat/stat_internal.h b/src/libstat/stat_internal.h index 44f48ae5a..a547ca93a 100644 --- a/src/libstat/stat_internal.h +++ b/src/libstat/stat_internal.h @@ -41,6 +41,7 @@ struct rspamd_classifier { gulong ham_learns; struct rspamd_classifier_config *cfg; struct rspamd_stat_classifier *subrs; + gpointer specific; }; struct rspamd_statfile { @@ -85,6 +86,9 @@ struct rspamd_stat_ctx { GPtrArray *classifiers; /* struct rspamd_classifier */ GQueue *async_elts; /* struct rspamd_stat_async_elt */ struct rspamd_config *cfg; + + gint lua_stat_tokens_ref; + /* Global tokenizer */ struct rspamd_stat_tokenizer *tokenizer; gpointer tkcf; diff --git a/src/libstat/stat_process.c b/src/libstat/stat_process.c index ca51d7b02..d097e12e0 100644 --- a/src/libstat/stat_process.c +++ b/src/libstat/stat_process.c @@ -32,273 +32,85 @@ static const gdouble similarity_treshold = 80.0; static void -rspamd_stat_tokenize_header (struct rspamd_task *task, - const gchar *name, const gchar *prefix, GArray *ar) -{ - struct rspamd_mime_header *cur; - GPtrArray *hdrs; - guint i; - rspamd_stat_token_t str; - - hdrs = g_hash_table_lookup (task->raw_headers, name); - str.flags = RSPAMD_STAT_TOKEN_FLAG_META; - - if (hdrs != NULL) { - - PTR_ARRAY_FOREACH (hdrs, i, cur) { - if (cur->name != NULL) { - str.begin = cur->name; - str.len = strlen (cur->name); - g_array_append_val (ar, str); - } - if (cur->decoded != NULL) { - str.begin = cur->decoded; - str.len = strlen (cur->decoded); - g_array_append_val (ar, str); - } - else if (cur->value != NULL) { - str.begin = cur->value; - str.len = strlen (cur->value); - g_array_append_val (ar, str); - } - } - - msg_debug_task ("added stat tokens for header '%s'", name); - } -} - -static void rspamd_stat_tokenize_parts_metadata (struct rspamd_stat_ctx *st_ctx, struct rspamd_task *task) { - struct rspamd_image *img; - struct rspamd_mime_part *part; - struct rspamd_mime_text_part *tp; - GList *cur; GArray *ar; rspamd_stat_token_t elt; guint i; - gchar tmpbuf[128]; lua_State *L = task->cfg->lua_state; - const gchar *headers_hash; - struct rspamd_mime_header *hdr; ar = g_array_sized_new (FALSE, FALSE, sizeof (elt), 16); + memset (&elt, 0, sizeof (elt)); elt.flags = RSPAMD_STAT_TOKEN_FLAG_META; - /* Insert images */ - for (i = 0; i < task->parts->len; i ++) { - part = g_ptr_array_index (task->parts, i); - - if ((part->flags & RSPAMD_MIME_PART_IMAGE) && part->specific.img) { - img = part->specific.img; - - /* If an image has a linked HTML part, then we push its details to the stat */ - if (img->html_image) { - elt.begin = (gchar *)"image"; - elt.len = 5; - g_array_append_val (ar, elt); - elt.begin = (gchar *)&img->html_image->height; - elt.len = sizeof (img->html_image->height); - g_array_append_val (ar, elt); - elt.begin = (gchar *)&img->html_image->width; - elt.len = sizeof (img->html_image->width); - g_array_append_val (ar, elt); - elt.begin = (gchar *)&img->type; - elt.len = sizeof (img->type); - g_array_append_val (ar, elt); - - if (img->filename) { - elt.begin = (gchar *)img->filename; - elt.len = strlen (elt.begin); - g_array_append_val (ar, elt); - } + if (st_ctx->lua_stat_tokens_ref != -1) { + gint err_idx, ret; + GString *tb; + struct rspamd_task **ptask; - msg_debug_task ("added stat tokens for image '%s'", img->html_image->src); - } - } - else if (part->cd && part->cd->filename.len > 0) { - elt.begin = (gchar *)part->cd->filename.begin; - elt.len = part->cd->filename.len; - g_array_append_val (ar, elt); - } - } + lua_pushcfunction (L, &rspamd_lua_traceback); + err_idx = lua_gettop (L); + lua_rawgeti (L, LUA_REGISTRYINDEX, st_ctx->lua_stat_tokens_ref); - /* Process mime parts */ - for (i = 0; i < task->parts->len; i ++) { - part = g_ptr_array_index (task->parts, i); + ptask = lua_newuserdata (L, sizeof (*ptask)); + *ptask = task; + rspamd_lua_setclass (L, "rspamd{task}", -1); - if (IS_CT_MULTIPART (part->ct)) { - elt.begin = (gchar *)part->ct->boundary.begin; - elt.len = part->ct->boundary.len; + if ((ret = lua_pcall (L, 1, 1, err_idx)) != 0) { + tb = lua_touserdata (L, -1); + msg_err_task ("call to stat_tokens lua " + "script failed (%d): %v", ret, tb); - if (elt.len) { - msg_debug_task ("added stat tokens for mime boundary '%*s'", - (gint)elt.len, elt.begin); - g_array_append_val (ar, elt); + if (tb) { + g_string_free (tb, TRUE); } - - if (part->parsed_data.len > 1) { - rspamd_snprintf (tmpbuf, sizeof (tmpbuf), "mime%d:%dlog", - i, (gint)log2 (part->parsed_data.len)); - elt.begin = rspamd_mempool_strdup (task->task_pool, tmpbuf); - elt.len = strlen (elt.begin); - g_array_append_val (ar, elt); - } - } - } - - /* Process text parts metadata */ - for (i = 0; i < task->text_parts->len; i ++) { - tp = g_ptr_array_index (task->text_parts, i); - - if (tp->language != NULL && tp->language[0] != '\0') { - elt.begin = (gchar *)tp->language; - elt.len = strlen (elt.begin); - msg_debug_task ("added stat tokens for part language '%s'", elt.begin); - g_array_append_val (ar, elt); - } - if (tp->real_charset != NULL) { - elt.begin = (gchar *)tp->real_charset; - elt.len = strlen (elt.begin); - msg_debug_task ("added stat tokens for part charset '%s'", elt.begin); - g_array_append_val (ar, elt); - } - } - - cur = g_list_first (task->cfg->classify_headers); - - while (cur) { - rspamd_stat_tokenize_header (task, cur->data, "UA:", ar); - - cur = g_list_next (cur); - } - - /* Use headers order */ - headers_hash = rspamd_mempool_get_variable (task->task_pool, - RSPAMD_MEMPOOL_HEADERS_HASH); - - if (headers_hash) { - elt.begin = (gchar *)headers_hash; - elt.len = 16; - g_array_append_val (ar, elt); - } - - /* Use more precise headers order */ - cur = g_list_first (task->headers_order->head); - while (cur) { - hdr = cur->data; - - if (hdr->name && hdr->type != RSPAMD_HEADER_RECEIVED) { - elt.begin = hdr->name; - elt.len = strlen (hdr->name); - g_array_append_val (ar, elt); } + else { + if (lua_type (L, -1) != LUA_TTABLE) { + msg_err_task ("stat_tokens invocation must return " + "table and not %s", + lua_typename (L, lua_type (L, -1))); + } + else { + guint vlen; + rspamd_ftok_t tok; - cur = g_list_next (cur); - } - - /* Use metatokens plugin from Lua */ - lua_getglobal (L, "rspamd_plugins"); - - if (lua_type (L, -1) == LUA_TTABLE) { - lua_pushstring (L, "stat_metatokens"); - lua_gettable (L, -2); - - if (lua_type (L, -1) == LUA_TTABLE) { - gint old_top; + vlen = rspamd_lua_table_size (L, -1); - old_top = lua_gettop (L); - lua_pushstring (L, "callback"); - lua_gettable (L, -2); + for (i = 0; i < vlen; i ++) { + lua_rawgeti (L, -1, i + 1); + tok.begin = lua_tolstring (L, -1, &tok.len); - if (lua_type (L, -1) == LUA_TFUNCTION) { - struct rspamd_task **ptask; + if (tok.begin && tok.len > 0) { + elt.original.begin = + rspamd_mempool_ftokdup (task->task_pool, &tok); + elt.original.len = tok.len; + elt.stemmed.begin = elt.original.begin; + elt.stemmed.len = elt.original.len; + elt.normalized.begin = elt.original.begin; + elt.normalized.len = elt.original.len; - ptask = lua_newuserdata (L, sizeof (*ptask)); - rspamd_lua_setclass (L, "rspamd{task}", -1); - *ptask = task; + g_array_append_val (ar, elt); + } - if (lua_pcall (L, 1, LUA_MULTRET, 0) != 0) { - msg_err_task ("stat_metatokens failed: %s", - lua_tostring (L, -1)); lua_pop (L, 1); - } else { - if (lua_gettop (L) > old_top && - lua_istable (L, old_top + 1)) { - lua_pushvalue (L, old_top + 1); - /* Iterate over table of tables */ - for (lua_pushnil (L); lua_next (L, -2); - lua_pop (L, 1)) { - elt.flags = RSPAMD_STAT_TOKEN_FLAG_META| - RSPAMD_STAT_TOKEN_FLAG_LUA_META; - - if (lua_isnumber (L, -1)) { - gdouble num = lua_tonumber (L, -1); - guint8 *pnum = rspamd_mempool_alloc ( - task->task_pool, - sizeof (num)); - - msg_debug_task ("got metatoken number: %.2f", - num); - memcpy (pnum, &num, sizeof (num)); - elt.begin = (gchar *) pnum; - elt.len = sizeof (num); - g_array_append_val (ar, elt); - } else if (lua_isstring (L, -1)) { - const gchar *str; - gsize tlen; - - str = lua_tolstring (L, -1, &tlen); - guint8 *pstr = rspamd_mempool_alloc ( - task->task_pool, - tlen); - memcpy (pstr, str, tlen); - - msg_debug_task ("got metatoken string: %*s", - (gint) tlen, str); - elt.begin = (gchar *) pstr; - elt.len = tlen; - g_array_append_val (ar, elt); - } - else if (lua_istable (L, -1)) { - /* Treat that as unigramms */ - for (lua_pushnil (L); lua_next (L, -2); - lua_pop (L, 1)) { - if (lua_isstring (L, -1)) { - const gchar *str; - gsize tlen; - - str = lua_tolstring (L, -1, &tlen); - guint8 *pstr = rspamd_mempool_alloc ( - task->task_pool, - tlen); - memcpy (pstr, str, tlen); - - msg_debug_task ("got unigramm " - "metatoken string: %*s", - (gint) tlen, str); - elt.begin = (gchar *) pstr; - elt.len = tlen; - elt.flags |= RSPAMD_STAT_TOKEN_FLAG_UNIGRAM; - g_array_append_val (ar, elt); - } - } - } - } - } } } } + + lua_settop (L, 0); } - lua_settop (L, 0); - st_ctx->tokenizer->tokenize_func (st_ctx, - task->task_pool, - ar, - TRUE, - "META:", - task->tokens); + + if (ar->len > 0) { + st_ctx->tokenizer->tokenize_func (st_ctx, + task, + ar, + TRUE, + "M", + task->tokens); + } rspamd_mempool_add_destructor (task->task_pool, rspamd_array_free_hard, ar); @@ -313,10 +125,7 @@ rspamd_stat_process_tokenize (struct rspamd_stat_ctx *st_ctx, { struct rspamd_mime_text_part *part; rspamd_cryptobox_hash_state_t hst; - rspamd_stat_token_t *tok; rspamd_token_t *st_tok; - GArray *words; - gchar *sub = NULL; guint i, reserved_len = 0; gdouble *pdiff; guchar hout[rspamd_cryptobox_HASHBYTES]; @@ -347,55 +156,26 @@ rspamd_stat_process_tokenize (struct rspamd_stat_ctx *st_ctx, part = g_ptr_array_index (task->text_parts, i); if (!IS_PART_EMPTY (part) && part->utf_words != NULL) { - st_ctx->tokenizer->tokenize_func (st_ctx, task->task_pool, + st_ctx->tokenizer->tokenize_func (st_ctx, task, part->utf_words, IS_PART_UTF (part), NULL, task->tokens); } if (pdiff != NULL && (1.0 - *pdiff) * 100.0 > similarity_treshold) { - msg_debug_task ("message has two common parts (%.2f), so skip the last one", + msg_debug_bayes ("message has two common parts (%.2f), so skip the last one", *pdiff); break; } } - if (task->subject != NULL) { - sub = task->subject; - } - - if (sub != NULL) { - UText utxt = UTEXT_INITIALIZER; - UErrorCode uc_err = U_ZERO_ERROR; - gsize slen = strlen (sub); - - utext_openUTF8 (&utxt, - sub, - slen, - &uc_err); - - words = rspamd_tokenize_text (sub, slen, &utxt, RSPAMD_TOKENIZE_UTF, - NULL, NULL, NULL); - - if (words != NULL) { - - for (i = 0; i < words->len; i ++) { - tok = &g_array_index (words, rspamd_stat_token_t, i); - tok->flags |= RSPAMD_STAT_TOKEN_FLAG_SUBJECT; - } - - st_ctx->tokenizer->tokenize_func (st_ctx, - task->task_pool, - words, - TRUE, - "SUBJECT", - task->tokens); - - rspamd_mempool_add_destructor (task->task_pool, - rspamd_array_free_hard, words); - } - - utext_close (&utxt); + if (task->meta_words != NULL) { + st_ctx->tokenizer->tokenize_func (st_ctx, + task, + task->meta_words, + TRUE, + "SUBJECT", + task->tokens); } rspamd_stat_tokenize_parts_metadata (st_ctx, task); @@ -445,10 +225,10 @@ rspamd_stat_preprocess (struct rspamd_stat_ctx *st_ctx, continue; } - if (!rspamd_symbols_cache_is_symbol_enabled (task, task->cfg->cache, + if (!rspamd_symcache_is_symbol_enabled (task, task->cfg->cache, st->stcf->symbol)) { g_ptr_array_index (task->stat_runtimes, i) = NULL; - msg_debug_task ("symbol %s is disabled, skip classification", + msg_debug_bayes ("symbol %s is disabled, skip classification", st->stcf->symbol); continue; } @@ -550,6 +330,12 @@ rspamd_stat_classifiers_process (struct rspamd_stat_ctx *st_ctx, return; } + for (i = 0; i < st_ctx->classifiers->len; i++) { + cl = g_ptr_array_index (st_ctx->classifiers, i); + cl->spam_learns = 0; + cl->ham_learns = 0; + } + for (i = 0; i < st_ctx->statfiles->len; i++) { st = g_ptr_array_index (st_ctx->statfiles, i); cl = st->classifier; @@ -591,7 +377,7 @@ rspamd_stat_classifiers_process (struct rspamd_stat_ctx *st_ctx, if (bk_run == NULL) { skip = TRUE; - msg_debug_task ("disable classifier %s as statfile symbol %s is disabled", + msg_debug_bayes ("disable classifier %s as statfile symbol %s is disabled", cl->cfg->name, st->stcf->symbol); break; } @@ -600,7 +386,7 @@ rspamd_stat_classifiers_process (struct rspamd_stat_ctx *st_ctx, if (!skip) { if (cl->cfg->min_tokens > 0 && task->tokens->len < cl->cfg->min_tokens) { - msg_debug_task ( + msg_debug_bayes ( "<%s> contains less tokens than required for %s classifier: " "%ud < %ud", task->message_id, @@ -610,7 +396,7 @@ rspamd_stat_classifiers_process (struct rspamd_stat_ctx *st_ctx, continue; } else if (cl->cfg->max_tokens > 0 && task->tokens->len > cl->cfg->max_tokens) { - msg_debug_task ( + msg_debug_bayes ( "<%s> contains more tokens than allowed for %s classifier: " "%ud > %ud", task->message_id, @@ -740,7 +526,7 @@ rspamd_stat_classifiers_learn (struct rspamd_stat_ctx *st_ctx, if ((task->flags & RSPAMD_TASK_FLAG_ALREADY_LEARNED) && err != NULL && *err == NULL) { /* Do not learn twice */ - g_set_error (err, rspamd_stat_quark (), 404, "<%s> has been already " + g_set_error (err, rspamd_stat_quark (), 208, "<%s> has been already " "learned as %s, ignore it", task->message_id, spam ? "spam" : "ham"); @@ -849,7 +635,7 @@ rspamd_stat_classifiers_learn (struct rspamd_stat_ctx *st_ctx, if (!learned && err && *err == NULL) { if (too_large) { - g_set_error (err, rspamd_stat_quark (), 400, + g_set_error (err, rspamd_stat_quark (), 204, "<%s> contains more tokens than allowed for %s classifier: " "%d > %d", task->message_id, @@ -858,7 +644,7 @@ rspamd_stat_classifiers_learn (struct rspamd_stat_ctx *st_ctx, cl->cfg->max_tokens); } else if (too_small) { - g_set_error (err, rspamd_stat_quark (), 400, + g_set_error (err, rspamd_stat_quark (), 204, "<%s> contains less tokens than required for %s classifier: " "%d < %d", task->message_id, @@ -867,7 +653,7 @@ rspamd_stat_classifiers_learn (struct rspamd_stat_ctx *st_ctx, cl->cfg->min_tokens); } else if (conditionally_skipped) { - g_set_error (err, rspamd_stat_quark (), 410, + g_set_error (err, rspamd_stat_quark (), 204, "<%s> is skipped for %s classifier: " "%s", task->message_id, @@ -1107,7 +893,7 @@ rspamd_stat_has_classifier_symbols (struct rspamd_task *task, if (rspamd_task_find_symbol_result (task, st->stcf->symbol)) { if (is_spam == !!st->stcf->is_spam) { - msg_debug_task ("do not autolearn %s as symbol %s is already " + msg_debug_bayes ("do not autolearn %s as symbol %s is already " "added", is_spam ? "spam" : "ham", st->stcf->symbol); return TRUE; diff --git a/src/libstat/tokenizers/osb.c b/src/libstat/tokenizers/osb.c index f6f46c580..0b53f8af9 100644 --- a/src/libstat/tokenizers/osb.c +++ b/src/libstat/tokenizers/osb.c @@ -17,8 +17,10 @@ * OSB tokenizer */ + #include "tokenizers.h" #include "stat_internal.h" +#include "libmime/lang_detection.h" /* Size for features pipe */ #define DEFAULT_FEATURE_WINDOW_SIZE 5 @@ -259,11 +261,11 @@ struct token_pipe_entry { gint rspamd_tokenizer_osb (struct rspamd_stat_ctx *ctx, - rspamd_mempool_t *pool, - GArray *words, - gboolean is_utf, - const gchar *prefix, - GPtrArray *result) + struct rspamd_task *task, + GArray *words, + gboolean is_utf, + const gchar *prefix, + GPtrArray *result) { rspamd_token_t *new_tok = NULL; rspamd_stat_token_t *token; @@ -302,23 +304,40 @@ rspamd_tokenizer_osb (struct rspamd_stat_ctx *ctx, for (w = 0; w < words->len; w ++) { token = &g_array_index (words, rspamd_stat_token_t, w); token_flags = token->flags; + const gchar *begin; + gsize len; + + if (token->flags & + (RSPAMD_STAT_TOKEN_FLAG_STOP_WORD|RSPAMD_STAT_TOKEN_FLAG_SKIPPED)) { + /* Skip stop/skipped words */ + continue; + } + + if (token->flags & RSPAMD_STAT_TOKEN_FLAG_TEXT) { + begin = token->stemmed.begin; + len = token->stemmed.len; + } + else { + begin = token->original.begin; + len = token->original.len; + } if (osb_cf->ht == RSPAMD_OSB_HASH_COMPAT) { rspamd_ftok_t ftok; - ftok.begin = token->begin; - ftok.len = token->len; + ftok.begin = begin; + ftok.len = len; cur = rspamd_fstrhash_lc (&ftok, is_utf); } else { /* We know that the words are normalized */ if (osb_cf->ht == RSPAMD_OSB_HASH_XXHASH) { cur = rspamd_cryptobox_fast_hash_specific (RSPAMD_CRYPTOBOX_XXHASH64, - token->begin, token->len, osb_cf->seed); + begin, len, osb_cf->seed); } else { - rspamd_cryptobox_siphash ((guchar *)&cur, token->begin, - token->len, osb_cf->sk); + rspamd_cryptobox_siphash ((guchar *)&cur, begin, + len, osb_cf->sk); if (prefix) { cur ^= seed; @@ -327,7 +346,7 @@ rspamd_tokenizer_osb (struct rspamd_stat_ctx *ctx, } if (token_flags & RSPAMD_STAT_TOKEN_FLAG_UNIGRAM) { - new_tok = rspamd_mempool_alloc0 (pool, token_size); + new_tok = rspamd_mempool_alloc0 (task->task_pool, token_size); new_tok->flags = token_flags; new_tok->t1 = token; new_tok->t2 = token; @@ -339,7 +358,7 @@ rspamd_tokenizer_osb (struct rspamd_stat_ctx *ctx, } #define ADD_TOKEN do {\ - new_tok = rspamd_mempool_alloc0 (pool, token_size); \ + new_tok = rspamd_mempool_alloc0 (task->task_pool, token_size); \ new_tok->flags = token_flags; \ new_tok->t1 = hashpipe[0].t; \ new_tok->t2 = hashpipe[i].t; \ @@ -354,7 +373,7 @@ rspamd_tokenizer_osb (struct rspamd_stat_ctx *ctx, else { \ new_tok->data = hashpipe[0].h * primes[0] + hashpipe[i].h * primes[i << 1]; \ } \ - new_tok->window_idx = i + 1; \ + new_tok->window_idx = i; \ g_ptr_array_add (result, new_tok); \ } while(0) @@ -375,7 +394,9 @@ rspamd_tokenizer_osb (struct rspamd_stat_ctx *ctx, processed++; for (i = 1; i < window_size; i++) { - ADD_TOKEN; + if (!(hashpipe[i].t->flags & RSPAMD_STAT_TOKEN_FLAG_EXCEPTION)) { + ADD_TOKEN; + } } } } diff --git a/src/libstat/tokenizers/tokenizers.c b/src/libstat/tokenizers/tokenizers.c index c8e8e44df..acbbcf2f0 100644 --- a/src/libstat/tokenizers/tokenizers.c +++ b/src/libstat/tokenizers/tokenizers.c @@ -20,11 +20,19 @@ #include "rspamd.h" #include "tokenizers.h" #include "stat_internal.h" -#include "../../../contrib/mumhash/mum.h" +#include "contrib/mumhash/mum.h" +#include "libmime/lang_detection.h" +#include "libstemmer.h" + #include <unicode/utf8.h> #include <unicode/uchar.h> #include <unicode/uiter.h> #include <unicode/ubrk.h> +#include <unicode/ucnv.h> +#if U_ICU_VERSION_MAJOR_NUM >= 44 +#include <unicode/unorm2.h> +#endif + #include <math.h> typedef gboolean (*token_get_function) (rspamd_stat_token_t * buf, gchar const **pos, @@ -80,33 +88,33 @@ rspamd_tokenizer_get_word_raw (rspamd_stat_token_t * buf, ex = (*exceptions)->data; } - if (token->begin == NULL || *cur == NULL) { + if (token->original.begin == NULL || *cur == NULL) { if (ex != NULL) { if (ex->pos == 0) { - token->begin = buf->begin + ex->len; - token->len = ex->len; + token->original.begin = buf->original.begin + ex->len; + token->original.len = ex->len; token->flags = RSPAMD_STAT_TOKEN_FLAG_EXCEPTION; } else { - token->begin = buf->begin; - token->len = 0; + token->original.begin = buf->original.begin; + token->original.len = 0; } } else { - token->begin = buf->begin; - token->len = 0; + token->original.begin = buf->original.begin; + token->original.len = 0; } - *cur = token->begin; + *cur = token->original.begin; } - token->len = 0; + token->original.len = 0; - pos = *cur - buf->begin; - if (pos >= buf->len) { + pos = *cur - buf->original.begin; + if (pos >= buf->original.len) { return FALSE; } - remain = buf->len - pos; + remain = buf->original.len - pos; p = *cur; /* Skip non delimiters symbols */ @@ -122,7 +130,7 @@ rspamd_tokenizer_get_word_raw (rspamd_stat_token_t * buf, remain--; } while (remain > 0 && t_delimiters[(guchar)*p]); - token->begin = p; + token->original.begin = p; while (remain > 0 && !t_delimiters[(guchar)*p]) { if (ex != NULL && ex->pos == pos) { @@ -130,7 +138,7 @@ rspamd_tokenizer_get_word_raw (rspamd_stat_token_t * buf, *cur = p + ex->len; return TRUE; } - token->len++; + token->original.len++; pos++; remain--; p++; @@ -141,7 +149,7 @@ rspamd_tokenizer_get_word_raw (rspamd_stat_token_t * buf, } if (rl) { - *rl = token->len; + *rl = token->original.len; } token->flags = RSPAMD_STAT_TOKEN_FLAG_TEXT; @@ -164,12 +172,12 @@ rspamd_tokenize_check_limit (gboolean decay, static const gdouble avg_word_len = 6.0; if (!decay) { - if (token->len >= sizeof (guint64)) { + if (token->original.len >= sizeof (guint64)) { #ifdef _MUM_UNALIGNED_ACCESS - *hv = mum_hash_step (*hv, *(guint64 *)token->begin); + *hv = mum_hash_step (*hv, *(guint64 *)token->original.begin); #else guint64 tmp; - memcpy (&tmp, token->begin, sizeof (tmp)); + memcpy (&tmp, token->original.begin, sizeof (tmp)); *hv = mum_hash_step (*hv, tmp); #endif } @@ -221,7 +229,7 @@ rspamd_utf_word_valid (const gchar *text, const gchar *end, U8_NEXT (text, start, finish, c); - if (u_isalnum (c)) { + if (u_isJavaIDPart (c)) { return TRUE; } @@ -237,13 +245,51 @@ rspamd_utf_word_valid (const gchar *text, const gchar *end, } \ } while(0) +static inline void +rspamd_tokenize_exception (struct rspamd_process_exception *ex, GArray *res) +{ + rspamd_stat_token_t token; + + memset (&token, 0, sizeof (token)); + + if (ex->type == RSPAMD_EXCEPTION_GENERIC) { + token.original.begin = "!!EX!!"; + token.original.len = sizeof ("!!EX!!") - 1; + token.flags = RSPAMD_STAT_TOKEN_FLAG_EXCEPTION; + + g_array_append_val (res, token); + token.flags = 0; + } + else if (ex->type == RSPAMD_EXCEPTION_URL) { + struct rspamd_url *uri; + + uri = ex->ptr; + + if (uri && uri->tldlen > 0) { + token.original.begin = uri->tld; + token.original.len = uri->tldlen; + + } + else { + token.original.begin = "!!EX!!"; + token.original.len = sizeof ("!!EX!!") - 1; + } + + token.flags = RSPAMD_STAT_TOKEN_FLAG_EXCEPTION; + g_array_append_val (res, token); + token.flags = 0; + } +} + + GArray * rspamd_tokenize_text (const gchar *text, gsize len, const UText *utxt, enum rspamd_tokenize_type how, struct rspamd_config *cfg, GList *exceptions, - guint64 *hash) + guint64 *hash, + GArray *cur_words) { rspamd_stat_token_t token, buf; const gchar *pos = NULL; @@ -257,15 +303,14 @@ rspamd_tokenize_text (const gchar *text, gsize len, static UBreakIterator* bi = NULL; if (text == NULL) { - return NULL; + return cur_words; } - buf.begin = text; - buf.len = len; + buf.original.begin = text; + buf.original.len = len; buf.flags = 0; - token.begin = NULL; - token.len = 0; - token.flags = 0; + + memset (&token, 0, sizeof (token)); if (cfg != NULL) { min_len = cfg->min_word_len; @@ -274,31 +319,36 @@ rspamd_tokenize_text (const gchar *text, gsize len, initial_size = word_decay * 2; } - res = g_array_sized_new (FALSE, FALSE, sizeof (rspamd_stat_token_t), - initial_size); + if (!cur_words) { + res = g_array_sized_new (FALSE, FALSE, sizeof (rspamd_stat_token_t), + initial_size); + } + else { + res = cur_words; + } if (G_UNLIKELY (how == RSPAMD_TOKENIZE_RAW || utxt == NULL)) { while (rspamd_tokenizer_get_word_raw (&buf, &pos, &token, &cur, &l, FALSE)) { if (l == 0 || (min_len > 0 && l < min_len) || (max_len > 0 && l > max_len)) { - token.begin = pos; + token.original.begin = pos; continue; } - if (token.len > 0 && + if (token.original.len > 0 && rspamd_tokenize_check_limit (decay, word_decay, res->len, &hv, &prob, &token, pos - text, len)) { if (!decay) { decay = TRUE; } else { - token.begin = pos; + token.original.begin = pos; continue; } } g_array_append_val (res, token); - token.begin = pos; + token.original.begin = pos; } } else { @@ -323,7 +373,7 @@ rspamd_tokenize_text (const gchar *text, gsize len, while (p != UBRK_DONE) { start_over: - token.len = 0; + token.original.len = 0; if (p > last) { if (ex && cur) { @@ -334,15 +384,7 @@ start_over: while (cur && ex->pos <= last) { /* We have an exception at the beginning, skip those */ last += ex->len; - - if (ex->type == RSPAMD_EXCEPTION_URL) { - token.begin = "!!EX!!"; - token.len = sizeof ("!!EX!!") - 1; - token.flags = RSPAMD_STAT_TOKEN_FLAG_EXCEPTION; - - g_array_append_val (res, token); - token.flags = 0; - } + rspamd_tokenize_exception (ex, res); if (last > p) { /* Exception spread over the boundaries */ @@ -363,8 +405,8 @@ start_over: /* Append the first part */ if (rspamd_utf_word_valid (text, text + len, last, ex->pos)) { - token.begin = text + last; - token.len = ex->pos - last; + token.original.begin = text + last; + token.original.len = ex->pos - last; token.flags = 0; g_array_append_val (res, token); } @@ -372,13 +414,7 @@ start_over: /* Process the current exception */ last += ex->len + (ex->pos - last); - if (ex->type == RSPAMD_EXCEPTION_URL) { - token.begin = "!!EX!!"; - token.len = sizeof ("!!EX!!") - 1; - token.flags = RSPAMD_STAT_TOKEN_FLAG_EXCEPTION; - - g_array_append_val (res, token); - } + rspamd_tokenize_exception (ex, res); if (last > p) { /* Exception spread over the boundaries */ @@ -394,9 +430,10 @@ start_over: } else if (p > last) { if (rspamd_utf_word_valid (text, text + len, last, p)) { - token.begin = text + last; - token.len = p - last; - token.flags = RSPAMD_STAT_TOKEN_FLAG_TEXT; + token.original.begin = text + last; + token.original.len = p - last; + token.flags = RSPAMD_STAT_TOKEN_FLAG_TEXT | + RSPAMD_STAT_TOKEN_FLAG_UTF; } } } @@ -408,40 +445,43 @@ start_over: } if (rspamd_utf_word_valid (text, text + len, last, p)) { - token.begin = text + last; - token.len = p - last; - token.flags = RSPAMD_STAT_TOKEN_FLAG_TEXT; + token.original.begin = text + last; + token.original.len = p - last; + token.flags = RSPAMD_STAT_TOKEN_FLAG_TEXT | + RSPAMD_STAT_TOKEN_FLAG_UTF; } } else { /* No exceptions within boundary */ if (rspamd_utf_word_valid (text, text + len, last, p)) { - token.begin = text + last; - token.len = p - last; - token.flags = RSPAMD_STAT_TOKEN_FLAG_TEXT; + token.original.begin = text + last; + token.original.len = p - last; + token.flags = RSPAMD_STAT_TOKEN_FLAG_TEXT | + RSPAMD_STAT_TOKEN_FLAG_UTF; } } } else { if (rspamd_utf_word_valid (text, text + len, last, p)) { - token.begin = text + last; - token.len = p - last; - token.flags = RSPAMD_STAT_TOKEN_FLAG_TEXT; + token.original.begin = text + last; + token.original.len = p - last; + token.flags = RSPAMD_STAT_TOKEN_FLAG_TEXT | + RSPAMD_STAT_TOKEN_FLAG_UTF; } } - if (token.len > 0 && + if (token.original.len > 0 && rspamd_tokenize_check_limit (decay, word_decay, res->len, &hv, &prob, &token, p, len)) { if (!decay) { decay = TRUE; } else { - token.len = 0; + token.flags |= RSPAMD_STAT_TOKEN_FLAG_SKIPPED; } } } - if (token.len > 0) { + if (token.original.len > 0) { g_array_append_val (res, token); } @@ -463,6 +503,347 @@ start_over: #undef SHIFT_EX -/* - * vi:ts=4 - */ +static void +rspamd_add_metawords_from_str (const gchar *beg, gsize len, + struct rspamd_task *task) +{ + UText utxt = UTEXT_INITIALIZER; + UErrorCode uc_err = U_ZERO_ERROR; + guint i = 0; + UChar32 uc; + gboolean valid_utf = TRUE; + + while (i < len) { + U8_NEXT (beg, i, len, uc); + + if (((gint32) uc) < 0) { + valid_utf = FALSE; + break; + } + +#if U_ICU_VERSION_MAJOR_NUM < 50 + if (u_isalpha (uc)) { + gint32 sc = ublock_getCode (uc); + + if (sc == UBLOCK_THAI) { + valid_utf = FALSE; + msg_info_task ("enable workaround for Thai characters for old libicu"); + break; + } + } +#endif + } + + if (valid_utf) { + utext_openUTF8 (&utxt, + beg, + len, + &uc_err); + + task->meta_words = rspamd_tokenize_text (beg, len, + &utxt, RSPAMD_TOKENIZE_UTF, + task->cfg, NULL, NULL, task->meta_words); + + utext_close (&utxt); + } + else { + task->meta_words = rspamd_tokenize_text (beg, len, + NULL, RSPAMD_TOKENIZE_RAW, + task->cfg, NULL, NULL, task->meta_words); + } +} + +void +rspamd_tokenize_meta_words (struct rspamd_task *task) +{ + guint i = 0; + rspamd_stat_token_t *tok; + + if (task->subject) { + rspamd_add_metawords_from_str (task->subject, strlen (task->subject), task); + } + + if (task->from_mime && task->from_mime->len > 0) { + struct rspamd_email_address *addr; + + addr = g_ptr_array_index (task->from_mime, 0); + + if (addr->name) { + rspamd_add_metawords_from_str (addr->name, strlen (addr->name), task); + } + } + + if (task->meta_words != NULL) { + const gchar *language = NULL; + + if (task->text_parts && task->text_parts->len > 0) { + struct rspamd_mime_text_part *tp = g_ptr_array_index (task->text_parts, 0); + + if (tp->language) { + language = tp->language; + } + } + + rspamd_normalize_words (task->meta_words, task->task_pool); + rspamd_stem_words (task->meta_words, task->task_pool, language, + task->lang_det); + + for (i = 0; i < task->meta_words->len; i++) { + tok = &g_array_index (task->meta_words, rspamd_stat_token_t, i); + tok->flags |= RSPAMD_STAT_TOKEN_FLAG_HEADER; + } + } +} + +static inline void +rspamd_uchars_to_ucs32 (const UChar *src, gsize srclen, + rspamd_stat_token_t *tok, + rspamd_mempool_t *pool) +{ + UChar32 *dest, t, *d; + gint32 i = 0; + + dest = rspamd_mempool_alloc (pool, srclen * sizeof (UChar32)); + d = dest; + + while (i < srclen) { + U16_NEXT_UNSAFE (src, i, t); + + if (u_isgraph (t)) { + *d++ = u_tolower (t); + } + else { + /* Invisible spaces ! */ + tok->flags |= RSPAMD_STAT_TOKEN_FLAG_INVISIBLE_SPACES; + } + } + + tok->unicode.begin = dest; + tok->unicode.len = d - dest; +} + +static inline void +rspamd_ucs32_to_normalised (rspamd_stat_token_t *tok, + rspamd_mempool_t *pool) +{ + guint i, doff = 0; + gsize utflen = 0; + gchar *dest; + UChar32 t; + + for (i = 0; i < tok->unicode.len; i ++) { + utflen += U8_LENGTH (tok->unicode.begin[i]); + } + + dest = rspamd_mempool_alloc (pool, utflen + 1); + + for (i = 0; i < tok->unicode.len; i ++) { + t = tok->unicode.begin[i]; + U8_APPEND_UNSAFE (dest, doff, t); + } + + g_assert (doff <= utflen); + dest[doff] = '\0'; + + tok->normalized.len = doff; + tok->normalized.begin = dest; +} + +void +rspamd_normalize_single_word (rspamd_stat_token_t *tok, rspamd_mempool_t *pool) +{ + UErrorCode uc_err = U_ZERO_ERROR; + UConverter *utf8_converter; + UChar tmpbuf[1024]; /* Assume that we have no longer words... */ + gsize ulen; + + utf8_converter = rspamd_get_utf8_converter (); + + if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_UTF) { + ulen = ucnv_toUChars (utf8_converter, + tmpbuf, + G_N_ELEMENTS (tmpbuf), + tok->original.begin, + tok->original.len, + &uc_err); + + /* Now, we need to understand if we need to normalise the word */ + if (!U_SUCCESS (uc_err)) { + tok->flags |= RSPAMD_STAT_TOKEN_FLAG_BROKEN_UNICODE; + tok->unicode.begin = NULL; + tok->unicode.len = 0; + tok->normalized.begin = NULL; + tok->normalized.len = 0; + } + else { +#if U_ICU_VERSION_MAJOR_NUM >= 44 + const UNormalizer2 *norm = rspamd_get_unicode_normalizer (); + gint32 end; + + /* We can now check if we need to decompose */ + end = unorm2_spanQuickCheckYes (norm, tmpbuf, ulen, &uc_err); + + if (!U_SUCCESS (uc_err)) { + rspamd_uchars_to_ucs32 (tmpbuf, ulen, tok, pool); + tok->normalized.begin = NULL; + tok->normalized.len = 0; + tok->flags |= RSPAMD_STAT_TOKEN_FLAG_BROKEN_UNICODE; + } + else { + if (end == ulen) { + /* Already normalised, just lowercase */ + rspamd_uchars_to_ucs32 (tmpbuf, ulen, tok, pool); + rspamd_ucs32_to_normalised (tok, pool); + } + else { + /* Perform normalization */ + UChar normbuf[1024]; + + g_assert (end < G_N_ELEMENTS (normbuf)); + /* First part */ + memcpy (normbuf, tmpbuf, end * sizeof (UChar)); + /* Second part */ + ulen = unorm2_normalizeSecondAndAppend (norm, + normbuf, end, + G_N_ELEMENTS (normbuf), + tmpbuf + end, + ulen - end, + &uc_err); + + if (!U_SUCCESS (uc_err)) { + if (uc_err != U_BUFFER_OVERFLOW_ERROR) { + msg_warn_pool_check ("cannot normalise text '%*s': %s", + (gint)tok->original.len, tok->original.begin, + u_errorName (uc_err)); + rspamd_uchars_to_ucs32 (tmpbuf, ulen, tok, pool); + rspamd_ucs32_to_normalised (tok, pool); + tok->flags |= RSPAMD_STAT_TOKEN_FLAG_BROKEN_UNICODE; + } + } + else { + /* Copy normalised back */ + rspamd_uchars_to_ucs32 (normbuf, ulen, tok, pool); + tok->flags |= RSPAMD_STAT_TOKEN_FLAG_NORMALISED; + rspamd_ucs32_to_normalised (tok, pool); + } + } + } +#else + /* Legacy version with no unorm2 interface */ + rspamd_uchars_to_ucs32 (tmpbuf, ulen, tok, pool); + rspamd_ucs32_to_normalised (tok, pool); +#endif + } + } + else { + if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_TEXT) { + /* Simple lowercase */ + gchar *dest; + + dest = rspamd_mempool_alloc (pool, tok->original.len + 1); + rspamd_strlcpy (dest, tok->original.begin, tok->original.len + 1); + rspamd_str_lc (dest, tok->original.len); + tok->normalized.len = tok->original.len; + tok->normalized.begin = dest; + } + } +} + +void +rspamd_normalize_words (GArray *words, rspamd_mempool_t *pool) +{ + rspamd_stat_token_t *tok; + guint i; + + for (i = 0; i < words->len; i++) { + tok = &g_array_index (words, rspamd_stat_token_t, i); + rspamd_normalize_single_word (tok, pool); + } +} + +void +rspamd_stem_words (GArray *words, rspamd_mempool_t *pool, + const gchar *language, + struct rspamd_lang_detector *d) +{ + static GHashTable *stemmers = NULL; + struct sb_stemmer *stem = NULL; + guint i; + rspamd_stat_token_t *tok; + gchar *dest; + gsize dlen; + + if (!stemmers) { + stemmers = g_hash_table_new (rspamd_strcase_hash, + rspamd_strcase_equal); + } + + if (language && language[0] != '\0') { + stem = g_hash_table_lookup (stemmers, language); + + if (stem == NULL) { + + stem = sb_stemmer_new (language, "UTF_8"); + + if (stem == NULL) { + msg_debug_pool ( + "<%s> cannot create lemmatizer for %s language", + language); + g_hash_table_insert (stemmers, g_strdup (language), + GINT_TO_POINTER (-1)); + } + else { + g_hash_table_insert (stemmers, g_strdup (language), + stem); + } + } + else if (stem == GINT_TO_POINTER (-1)) { + /* Negative cache */ + stem = NULL; + } + } + for (i = 0; i < words->len; i++) { + tok = &g_array_index (words, rspamd_stat_token_t, i); + + if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_UTF) { + if (stem) { + const gchar *stemmed = NULL; + + stemmed = sb_stemmer_stem (stem, + tok->normalized.begin, tok->normalized.len); + + dlen = stemmed ? strlen (stemmed) : 0; + + if (dlen > 0) { + dest = rspamd_mempool_alloc (pool, dlen + 1); + memcpy (dest, stemmed, dlen); + dest[dlen] = '\0'; + tok->stemmed.len = dlen; + tok->stemmed.begin = dest; + tok->flags |= RSPAMD_STAT_TOKEN_FLAG_STEMMED; + } + else { + /* Fallback */ + tok->stemmed.len = tok->normalized.len; + tok->stemmed.begin = tok->normalized.begin; + } + } + else { + tok->stemmed.len = tok->normalized.len; + tok->stemmed.begin = tok->normalized.begin; + } + + if (tok->stemmed.len > 0 && d != NULL && + rspamd_language_detector_is_stop_word (d, tok->stemmed.begin, tok->stemmed.len)) { + tok->flags |= RSPAMD_STAT_TOKEN_FLAG_STOP_WORD; + } + } + else { + if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_TEXT) { + /* Raw text, lowercase */ + tok->stemmed.len = tok->normalized.len; + tok->stemmed.begin = tok->normalized.begin; + } + } + } +}
\ No newline at end of file diff --git a/src/libstat/tokenizers/tokenizers.h b/src/libstat/tokenizers/tokenizers.h index 6c538eafc..784426d31 100644 --- a/src/libstat/tokenizers/tokenizers.h +++ b/src/libstat/tokenizers/tokenizers.h @@ -18,13 +18,13 @@ struct rspamd_stat_ctx; struct rspamd_stat_tokenizer { gchar *name; gpointer (*get_config) (rspamd_mempool_t *pool, - struct rspamd_tokenizer_config *cf, gsize *len); + struct rspamd_tokenizer_config *cf, gsize *len); gint (*tokenize_func)(struct rspamd_stat_ctx *ctx, - rspamd_mempool_t *pool, - GArray *words, - gboolean is_utf, - const gchar *prefix, - GPtrArray *result); + struct rspamd_task *task, + GArray *words, + gboolean is_utf, + const gchar *prefix, + GPtrArray *result); }; enum rspamd_tokenize_type { @@ -43,20 +43,29 @@ GArray * rspamd_tokenize_text (const gchar *text, gsize len, enum rspamd_tokenize_type how, struct rspamd_config *cfg, GList *exceptions, - guint64 *hash); + guint64 *hash, + GArray *cur_words); /* OSB tokenize function */ gint rspamd_tokenizer_osb (struct rspamd_stat_ctx *ctx, - rspamd_mempool_t *pool, - GArray *words, - gboolean is_utf, - const gchar *prefix, - GPtrArray *result); + struct rspamd_task *task, + GArray *words, + gboolean is_utf, + const gchar *prefix, + GPtrArray *result); gpointer rspamd_tokenizer_osb_get_config (rspamd_mempool_t *pool, - struct rspamd_tokenizer_config *cf, - gsize *len); + struct rspamd_tokenizer_config *cf, + gsize *len); +struct rspamd_lang_detector; +void rspamd_normalize_single_word (rspamd_stat_token_t *tok, rspamd_mempool_t *pool); +void rspamd_normalize_words (GArray *words, rspamd_mempool_t *pool); +void rspamd_stem_words (GArray *words, rspamd_mempool_t *pool, + const gchar *language, + struct rspamd_lang_detector *d); + +void rspamd_tokenize_meta_words (struct rspamd_task *task); #endif /* * vi:ts=4 diff --git a/src/libutil/addr.c b/src/libutil/addr.c index a6f1adaf8..5cbc1c044 100644 --- a/src/libutil/addr.c +++ b/src/libutil/addr.c @@ -1055,7 +1055,7 @@ rspamd_inet_address_listen (const rspamd_inet_addr_t *addr, gint type, } } - if (type != SOCK_DGRAM) { + if (type != (int)SOCK_DGRAM) { if (addr->af == AF_UNIX) { path = addr->u.un->addr.sun_path; diff --git a/src/libutil/fstring.c b/src/libutil/fstring.c index fac3b364b..2cfbd7bf8 100644 --- a/src/libutil/fstring.c +++ b/src/libutil/fstring.c @@ -223,8 +223,6 @@ rspamd_fstring_erase (rspamd_fstring_t *str, gsize pos, gsize len) } } -char *rspamd_fstring_cstr (const rspamd_fstring_t *str); - /* Compat code */ static guint32 fstrhash_c (gchar c, guint32 hval) @@ -418,6 +416,22 @@ rspamd_fstring_cstr (const rspamd_fstring_t *s) return result; } +char * +rspamd_ftok_cstr (const rspamd_ftok_t *s) +{ + char *result; + + if (s == NULL) { + return NULL; + } + + result = g_malloc (s->len + 1); + memcpy (result, s->begin, s->len); + result[s->len] = '\0'; + + return result; +} + gboolean rspamd_ftok_cstr_equal (const rspamd_ftok_t *s, const gchar *pat, gboolean icase) diff --git a/src/libutil/fstring.h b/src/libutil/fstring.h index e5d34e022..96749052c 100644 --- a/src/libutil/fstring.h +++ b/src/libutil/fstring.h @@ -18,6 +18,7 @@ #include "config.h" #include "mem_pool.h" +#include <unicode/uchar.h> /** * Fixed strings library @@ -38,6 +39,11 @@ typedef struct f_str_tok { const gchar *begin; } rspamd_ftok_t; +typedef struct f_str_unicode_tok { + gsize len; /* in UChar32 */ + const UChar32 *begin; +} rspamd_ftok_unicode_t; + /** * Create new fixed length string */ @@ -87,12 +93,17 @@ void rspamd_fstring_erase (rspamd_fstring_t *str, gsize pos, gsize len); #define rspamd_fstring_clear(s) rspamd_fstring_erase(s, 0, s->len) /** - * Convert fixed string to a zero terminated string. This string should be + * Convert fixed string to a zero terminated string. This string must be * freed by a caller */ char * rspamd_fstring_cstr (const rspamd_fstring_t *str) G_GNUC_WARN_UNUSED_RESULT; - +/** + * Convert fixed string usign ftok_t to a zero terminated string. This string must be + * freed by a caller + */ +char * rspamd_ftok_cstr (const rspamd_ftok_t *str) + G_GNUC_WARN_UNUSED_RESULT; /* * Return fast hash value for fixed string converted to lowercase */ diff --git a/src/libutil/hash.c b/src/libutil/hash.c index 18b01f2ec..086eba8d1 100644 --- a/src/libutil/hash.c +++ b/src/libutil/hash.c @@ -16,6 +16,7 @@ #include "config.h" #include "hash.h" #include "util.h" +#include "khash.h" /** * LRU hashing @@ -25,42 +26,281 @@ static const guint log_base = 10; static const guint eviction_candidates = 16; static const gdouble lfu_base_value = 5.0; +struct rspamd_lru_volatile_element_s; + struct rspamd_lru_hash_s { guint maxsize; guint eviction_min_prio; guint eviction_used; + struct rspamd_lru_element_s **eviction_pool; + GDestroyNotify value_destroy; GDestroyNotify key_destroy; - struct rspamd_lru_element_s **eviction_pool; - GHashTable *tbl; + GHashFunc hfunc; + GEqualFunc eqfunc; + + khint_t n_buckets, size, n_occupied, upper_bound; + khint32_t *flags; + gpointer *keys; + struct rspamd_lru_volatile_element_s *vals; +}; + +enum rspamd_lru_element_flags { + RSPAMD_LRU_ELEMENT_NORMAL = 0, + RSPAMD_LRU_ELEMENT_VOLATILE = (1 << 0), }; struct rspamd_lru_element_s { - guint16 ttl; guint16 last; guint8 lg_usages; - guint eviction_pos; + guint8 eviction_pos; + guint8 flags; gpointer data; - gpointer key; - rspamd_lru_hash_t *hash; }; +struct rspamd_lru_volatile_element_s { + struct rspamd_lru_element_s e; + time_t creation_time; + time_t ttl; +}; +typedef struct rspamd_lru_volatile_element_s rspamd_lru_vol_element_t; + #define TIME_TO_TS(t) ((guint16)(((t) / 60) & 0xFFFFU)) -static void -rspamd_lru_destroy_node (gpointer value) +static rspamd_lru_vol_element_t * +rspamd_lru_hash_get (const rspamd_lru_hash_t *h, gconstpointer key) +{ + if (h->n_buckets) { + khint_t k, i, last, mask, step = 0; + mask = h->n_buckets - 1; + k = h->hfunc (key); + i = k & mask; + last = i; + + while (!__ac_isempty(h->flags, i) && + (__ac_isdel(h->flags, i) || !h->eqfunc(h->keys[i], key))) { + i = (i + (++step)) & mask; + if (i == last) { + return NULL; + } + } + + return __ac_iseither(h->flags, i) ? NULL : &h->vals[i]; + } + + return NULL; +} + +static int +rspamd_lru_hash_resize (rspamd_lru_hash_t *h, + khint_t new_n_buckets) { - rspamd_lru_element_t *elt = (rspamd_lru_element_t *)value; + /* This function uses 0.25*n_buckets bytes of working space instead of [sizeof(key_t+val_t)+.25]*n_buckets. */ + khint32_t *new_flags = 0; + khint_t j = 1; - if (elt) { - if (elt->hash && elt->hash->key_destroy) { - elt->hash->key_destroy (elt->key); + kroundup32(new_n_buckets); + if (new_n_buckets < 4) { + new_n_buckets = 4; + } + + if (h->size >= (khint_t) (new_n_buckets * __ac_HASH_UPPER + 0.5)) { + j = 0; + /* requested size is too small */ + } + else { + /* hash table size to be changed (shrink or expand); rehash */ + new_flags = (khint32_t *) g_malloc(__ac_fsize (new_n_buckets) * sizeof (khint32_t)); + + if (!new_flags) { + return -1; + } + + memset(new_flags, 0xaa, __ac_fsize (new_n_buckets) * sizeof (khint32_t)); + if (h->n_buckets < new_n_buckets) { + /* expand */ + gpointer *new_keys = (gpointer *) g_realloc((void *) h->keys, + new_n_buckets * sizeof (gpointer)); + + if (!new_keys) { + g_free(new_flags); + return -1; + } + + h->keys = new_keys; + rspamd_lru_vol_element_t *new_vals = + (rspamd_lru_vol_element_t *) g_realloc((void *) h->vals, + new_n_buckets * sizeof (rspamd_lru_vol_element_t)); + if (!new_vals) { + g_free(new_flags); + return -1; + } + + h->vals = new_vals; } - if (elt->hash && elt->hash->value_destroy) { - elt->hash->value_destroy (elt->data); + /* Shrink */ + } + + if (j) { + /* rehashing is needed */ + h->eviction_used = 0; + + for (j = 0; j != h->n_buckets; ++j) { + if (__ac_iseither(h->flags, j) == 0) { + gpointer key = h->keys[j]; + rspamd_lru_vol_element_t val; + khint_t new_mask; + new_mask = new_n_buckets - 1; + val = h->vals[j]; + val.e.eviction_pos = (guint8)-1; + __ac_set_isdel_true(h->flags, j); + + while (1) { /* kick-out process; sort of like in Cuckoo hashing */ + khint_t k, i, step = 0; + k = h->hfunc(key); + i = k & new_mask; + + while (!__ac_isempty(new_flags, i)) { + i = (i + (++step)) & new_mask; + } + + __ac_set_isempty_false(new_flags, i); + + if (i < h->n_buckets && __ac_iseither(h->flags, i) == 0) { + /* kick out the existing element */ + { + gpointer tmp = h->keys[i]; + h->keys[i] = key; + key = tmp; + } + { + rspamd_lru_vol_element_t tmp = h->vals[i]; + h->vals[i] = val; + val = tmp; + val.e.eviction_pos = (guint8)-1; + } + __ac_set_isdel_true(h->flags, i); + /* mark it as deleted in the old hash table */ + } else { /* write the element and jump out of the loop */ + h->keys[i] = key; + h->vals[i] = val; + break; + } + } + } } - g_free (elt); + if (h->n_buckets > new_n_buckets) { + /* shrink the hash table */ + h->keys = (gpointer *) g_realloc((void *) h->keys, + new_n_buckets * sizeof (gpointer)); + h->vals = (rspamd_lru_vol_element_t *) g_realloc((void *) h->vals, + new_n_buckets * sizeof (rspamd_lru_vol_element_t)); + } + + g_free(h->flags); /* free the working space */ + h->flags = new_flags; + h->n_buckets = new_n_buckets; + h->n_occupied = h->size; + h->upper_bound = (khint_t) (h->n_buckets * __ac_HASH_UPPER + 0.5); + } + + return 0; +} + +static rspamd_lru_vol_element_t * +rspamd_lru_hash_put (rspamd_lru_hash_t *h, gpointer key, int *ret) +{ + khint_t x; + + if (h->n_occupied >= h->upper_bound) { + /* update the hash table */ + if (h->n_buckets > (h->size << 1)) { + if (rspamd_lru_hash_resize (h, h->n_buckets - 1) < 0) { + /* clear "deleted" elements */ + *ret = -1; + return NULL; + } + } + else if (rspamd_lru_hash_resize (h, h->n_buckets + 1) < 0) { + /* expand the hash table */ + *ret = -1; + return NULL; + } + } + + khint_t k, i, site, last, mask = h->n_buckets - 1, step = 0; + x = site = h->n_buckets; + k = h->hfunc(key); + i = k & mask; + + if (__ac_isempty(h->flags, i)) { + x = i; /* for speed up */ + } + else { + last = i; + while (!__ac_isempty(h->flags, i) && + (__ac_isdel(h->flags, i) || + !h->eqfunc (h->keys[i], key))) { + if (__ac_isdel(h->flags, i)) { + site = i; + } + + i = (i + (++step)) & mask; + + if (i == last) { + x = site; + break; + } + } + + if (x == h->n_buckets) { + if (__ac_isempty(h->flags, i) && site != h->n_buckets) { + x = site; + } + else { + x = i; + } + } + } + + if (__ac_isempty(h->flags, x)) { /* not present at all */ + h->keys[x] = key; + __ac_set_isboth_false(h->flags, x); + ++h->size; + ++h->n_occupied; + *ret = 1; + } + else if (__ac_isdel(h->flags, x)) { /* deleted */ + h->keys[x] = key; + __ac_set_isboth_false(h->flags, x); + ++h->size; + *ret = 2; + } + else { + /* Don't touch h->keys[x] if present and not deleted */ + *ret = 0; + } + + return &h->vals[x]; +} + +static void +rspamd_lru_hash_del (rspamd_lru_hash_t *h, rspamd_lru_vol_element_t *elt) +{ + khint_t x = elt - h->vals; + + if (x != h->n_buckets && !__ac_iseither(h->flags, x)) { + __ac_set_isdel_true(h->flags, x); + --h->size; + + if (h->key_destroy) { + h->key_destroy (h->keys[x]); + } + + if (h->value_destroy) { + h->value_destroy (elt->e.data); + } } } @@ -141,7 +381,7 @@ rspamd_lru_hash_maybe_evict (rspamd_lru_hash_t *hash, guint i; rspamd_lru_element_t *cur; - if (elt->eviction_pos == -1) { + if (elt->eviction_pos == (guint8)-1) { if (hash->eviction_used < eviction_candidates) { /* There are free places in eviction pool */ hash->eviction_pool[hash->eviction_used] = elt; @@ -181,69 +421,14 @@ rspamd_lru_hash_maybe_evict (rspamd_lru_hash_t *hash, return FALSE; } -static rspamd_lru_element_t * -rspamd_lru_create_node (rspamd_lru_hash_t *hash, - gpointer key, - gpointer value, - time_t now, - guint ttl) -{ - rspamd_lru_element_t *node; - - node = g_malloc (sizeof (rspamd_lru_element_t)); - node->data = value; - node->key = key; - node->ttl = TIME_TO_TS (ttl); - - if (node->ttl == 0) { - node->ttl = 1; - } - - node->hash = hash; - node->lg_usages = lfu_base_value; - node->last = TIME_TO_TS (now); - node->eviction_pos = -1; - - return node; -} - static void rspamd_lru_hash_remove_node (rspamd_lru_hash_t *hash, rspamd_lru_element_t *elt) { - if (elt->eviction_pos != -1) { + if (elt->eviction_pos != (guint8)-1) { rspamd_lru_hash_remove_evicted (hash, elt); } - g_hash_table_remove (hash->tbl, elt->key); -} - -static rspamd_lru_element_t * -rspamd_lru_eviction_full_update (rspamd_lru_hash_t *hash, time_t now) -{ - GHashTableIter it; - gpointer k, v; - rspamd_lru_element_t *cur, *selected = NULL; - - g_hash_table_iter_init (&it, hash->tbl); - now = TIME_TO_TS (now); - - while (g_hash_table_iter_next (&it, &k, &v)) { - cur = v; - - rspamd_lru_hash_decrease_counter (cur, now); - - if (rspamd_lru_hash_maybe_evict (hash, cur)) { - - if (selected && cur->lg_usages < selected->lg_usages) { - selected = cur; - } - else if (selected == NULL) { - selected = cur; - } - } - } - - return selected; + rspamd_lru_hash_del (hash, (rspamd_lru_vol_element_t *)elt); } static void @@ -252,6 +437,7 @@ rspamd_lru_hash_evict (rspamd_lru_hash_t *hash, time_t now) double r; guint i; rspamd_lru_element_t *elt = NULL; + guint nexpired = 0; /* * We either evict one node from the eviction list @@ -262,9 +448,38 @@ rspamd_lru_hash_evict (rspamd_lru_hash_t *hash, time_t now) r = rspamd_random_double_fast (); if (r < ((double)eviction_candidates) / hash->maxsize) { - elt = rspamd_lru_eviction_full_update (hash, now); + /* Full hash scan */ + rspamd_lru_vol_element_t *cur; + rspamd_lru_element_t *selected = NULL; + + kh_foreach_value_ptr (hash, cur, { + rspamd_lru_element_t *node = &cur->e; + + if (node->flags & RSPAMD_LRU_ELEMENT_VOLATILE) { + /* If element is expired, just remove it */ + if (now - cur->creation_time > cur->ttl) { + rspamd_lru_hash_remove_node (hash, node); + + nexpired ++; + continue; + } + } + else { + rspamd_lru_hash_decrease_counter (node, now); + + if (rspamd_lru_hash_maybe_evict (hash, node)) { + if (selected && node->lg_usages < selected->lg_usages) { + selected = node; + } + else if (selected == NULL) { + selected = node; + } + } + } + }); } else { + /* Fast random eviction */ for (i = 0; i < hash->eviction_used; i ++) { elt = hash->eviction_pool[i]; @@ -274,41 +489,44 @@ rspamd_lru_hash_evict (rspamd_lru_hash_t *hash, time_t now) } } - g_assert (elt != NULL); - rspamd_lru_hash_remove_node (hash, elt); + if (elt && nexpired == 0) { + rspamd_lru_hash_remove_node (hash, elt); + } } rspamd_lru_hash_t * -rspamd_lru_hash_new_full ( - gint maxsize, - GDestroyNotify key_destroy, - GDestroyNotify value_destroy, - GHashFunc hf, - GEqualFunc cmpf) +rspamd_lru_hash_new_full (gint maxsize, + GDestroyNotify key_destroy, + GDestroyNotify value_destroy, + GHashFunc hf, + GEqualFunc cmpf) { - rspamd_lru_hash_t *new; + rspamd_lru_hash_t *h; if (maxsize < eviction_candidates * 2) { maxsize = eviction_candidates * 2; } - new = g_malloc0 (sizeof (rspamd_lru_hash_t)); - new->tbl = g_hash_table_new_full (hf, cmpf, NULL, rspamd_lru_destroy_node); - new->eviction_pool = g_malloc0 (sizeof (rspamd_lru_element_t *) * + h = g_malloc0 (sizeof (rspamd_lru_hash_t)); + h->hfunc = hf; + h->eqfunc = cmpf; + h->eviction_pool = g_malloc0 (sizeof (rspamd_lru_element_t *) * eviction_candidates); - new->maxsize = maxsize; - new->value_destroy = value_destroy; - new->key_destroy = key_destroy; - new->eviction_min_prio = G_MAXUINT; + h->maxsize = maxsize; + h->value_destroy = value_destroy; + h->key_destroy = key_destroy; + h->eviction_min_prio = G_MAXUINT; + + /* Preallocate some elements */ + rspamd_lru_hash_resize (h, MIN (h->maxsize, 128)); - return new; + return h; } rspamd_lru_hash_t * -rspamd_lru_hash_new ( - gint maxsize, - GDestroyNotify key_destroy, - GDestroyNotify value_destroy) +rspamd_lru_hash_new (gint maxsize, + GDestroyNotify key_destroy, + GDestroyNotify value_destroy) { return rspamd_lru_hash_new_full (maxsize, key_destroy, value_destroy, @@ -319,19 +537,23 @@ gpointer rspamd_lru_hash_lookup (rspamd_lru_hash_t *hash, gconstpointer key, time_t now) { rspamd_lru_element_t *res; + rspamd_lru_vol_element_t *vnode; - res = g_hash_table_lookup (hash->tbl, key); - if (res != NULL) { - now = TIME_TO_TS(now); + vnode = rspamd_lru_hash_get (hash, (gpointer)key); + if (vnode != NULL) { + res = &vnode->e; + + if (res->flags & RSPAMD_LRU_ELEMENT_VOLATILE) { + /* Check ttl */ - if (res->ttl != 0) { - if (now - res->last > res->ttl) { + if (now - vnode->creation_time > vnode->ttl) { rspamd_lru_hash_remove_node (hash, res); return NULL; } } + now = TIME_TO_TS(now); res->last = MAX (res->last, now); rspamd_lru_hash_update_counter (res); rspamd_lru_hash_maybe_evict (hash, res); @@ -346,12 +568,12 @@ gboolean rspamd_lru_hash_remove (rspamd_lru_hash_t *hash, gconstpointer key) { - rspamd_lru_element_t *res; + rspamd_lru_vol_element_t *res; - res = g_hash_table_lookup (hash->tbl, key); + res = rspamd_lru_hash_get (hash, key); if (res != NULL) { - rspamd_lru_hash_remove_node (hash, res); + rspamd_lru_hash_remove_node (hash, &res->e); return TRUE; } @@ -360,44 +582,113 @@ rspamd_lru_hash_remove (rspamd_lru_hash_t *hash, } void -rspamd_lru_hash_insert (rspamd_lru_hash_t *hash, gpointer key, gpointer value, - time_t now, guint ttl) +rspamd_lru_hash_insert (rspamd_lru_hash_t *hash, + gpointer key, + gpointer value, + time_t now, + guint ttl) { - rspamd_lru_element_t *res; + rspamd_lru_element_t *node; + rspamd_lru_vol_element_t *vnode; + gint ret; - res = g_hash_table_lookup (hash->tbl, key); + vnode = rspamd_lru_hash_put (hash, key, &ret); + node = &vnode->e; - if (res != NULL) { - rspamd_lru_hash_remove_node (hash, res); + if (ret == 0) { + /* Existing element, be carefull about destructors */ + if (hash->value_destroy) { + /* Remove old data */ + hash->value_destroy (vnode->e.data); + } + + if (hash->key_destroy) { + /* Here are dragons! */ + goffset off = vnode - hash->vals; + + hash->key_destroy (hash->keys[off]); + hash->keys[off] = key; + } + } + + + if (ttl == 0) { + node->flags = RSPAMD_LRU_ELEMENT_NORMAL; } else { - if (g_hash_table_size (hash->tbl) >= hash->maxsize) { + vnode->creation_time = now; + vnode->ttl = ttl; + node->flags = RSPAMD_LRU_ELEMENT_VOLATILE; + } + + node->data = value; + node->lg_usages = (guint8)lfu_base_value; + node->last = TIME_TO_TS (now); + node->eviction_pos = -1; + + if (ret != 0) { + /* Also need to check maxsize */ + if (kh_size (hash) >= hash->maxsize) { rspamd_lru_hash_evict (hash, now); } } - res = rspamd_lru_create_node (hash, key, value, now, ttl); - g_hash_table_insert (hash->tbl, key, res); - rspamd_lru_hash_maybe_evict (hash, res); + rspamd_lru_hash_maybe_evict (hash, node); } void rspamd_lru_hash_destroy (rspamd_lru_hash_t *hash) { - g_hash_table_unref (hash->tbl); - g_free (hash->eviction_pool); - g_free (hash); -} - + if (hash) { + if (hash->key_destroy || hash->value_destroy) { + gpointer k; + rspamd_lru_vol_element_t cur; + + kh_foreach (hash, k, cur, { + if (hash->key_destroy) { + hash->key_destroy (k); + } + if (hash->value_destroy) { + hash->value_destroy (cur.e.data); + } + }); + } -GHashTable * -rspamd_lru_hash_get_htable (rspamd_lru_hash_t *hash) -{ - return hash->tbl; + g_free (hash->keys); + g_free (hash->vals); + g_free (hash->flags); + g_free (hash->eviction_pool); + g_free (hash); + } } gpointer rspamd_lru_hash_element_data (rspamd_lru_element_t *elt) { return elt->data; +} + +int +rspamd_lru_hash_foreach (rspamd_lru_hash_t *h, int it, gpointer *k, + gpointer *v) +{ + gint i; + g_assert (it >= 0); + + for (i = it; i != kh_end (h); ++i) { + if (!kh_exist (h, i)) { + continue; + } + + *k = h->keys[i]; + *v = h->vals[i].e.data; + + break; + } + + if (i == kh_end (h)) { + return -1; + } + + return i; }
\ No newline at end of file diff --git a/src/libutil/hash.h b/src/libutil/hash.h index 7638d6397..f983a0be8 100644 --- a/src/libutil/hash.h +++ b/src/libutil/hash.h @@ -23,10 +23,9 @@ typedef struct rspamd_lru_element_s rspamd_lru_element_t; * @param key_equal_func pointer to function for comparing keys * @return new rspamd_hash object */ -rspamd_lru_hash_t * rspamd_lru_hash_new ( - gint maxsize, - GDestroyNotify key_destroy, - GDestroyNotify value_destroy); +rspamd_lru_hash_t * rspamd_lru_hash_new (gint maxsize, + GDestroyNotify key_destroy, + GDestroyNotify value_destroy); /** @@ -37,12 +36,11 @@ rspamd_lru_hash_t * rspamd_lru_hash_new ( * @param key_equal_func pointer to function for comparing keys * @return new rspamd_hash object */ -rspamd_lru_hash_t * rspamd_lru_hash_new_full ( - gint maxsize, - GDestroyNotify key_destroy, - GDestroyNotify value_destroy, - GHashFunc hfunc, - GEqualFunc eqfunc); +rspamd_lru_hash_t * rspamd_lru_hash_new_full (gint maxsize, + GDestroyNotify key_destroy, + GDestroyNotify value_destroy, + GHashFunc hfunc, + GEqualFunc eqfunc); /** * Lookup item from hash @@ -51,8 +49,8 @@ rspamd_lru_hash_t * rspamd_lru_hash_new_full ( * @return value of key or NULL if key is not found */ gpointer rspamd_lru_hash_lookup (rspamd_lru_hash_t *hash, - gconstpointer key, - time_t now); + gconstpointer key, + time_t now); /** * Removes key from LRU cache @@ -61,7 +59,7 @@ gpointer rspamd_lru_hash_lookup (rspamd_lru_hash_t *hash, * @return TRUE if key has been found and removed */ gboolean rspamd_lru_hash_remove (rspamd_lru_hash_t *hash, - gconstpointer key); + gconstpointer key); /** * Insert item in hash * @param hash hash object @@ -69,10 +67,10 @@ gboolean rspamd_lru_hash_remove (rspamd_lru_hash_t *hash, * @param value value of key */ void rspamd_lru_hash_insert (rspamd_lru_hash_t *hash, - gpointer key, - gpointer value, - time_t now, - guint ttl); + gpointer key, + gpointer value, + time_t now, + guint ttl); /** * Remove lru hash @@ -82,18 +80,13 @@ void rspamd_lru_hash_insert (rspamd_lru_hash_t *hash, void rspamd_lru_hash_destroy (rspamd_lru_hash_t *hash); /** - * Get hash table for this lru hash (use rspamd_lru_element_t as data) - */ -GHashTable *rspamd_lru_hash_get_htable (rspamd_lru_hash_t *hash); - -/** - * Get element's data - * @param elt - * @return + * Iterate over lru hash. Iterations must start from it=0 and are done when it==-1 + * @param hash + * @param it + * @param k + * @param v + * @return new it or -1 if iteration has been reached over */ -gpointer rspamd_lru_hash_element_data (rspamd_lru_element_t *elt); +int rspamd_lru_hash_foreach (rspamd_lru_hash_t *hash, int it, gpointer *k, + gpointer *v); #endif - -/* - * vi:ts=4 - */ diff --git a/src/libutil/http.c b/src/libutil/http.c index 637548dac..a82fc24f7 100644 --- a/src/libutil/http.c +++ b/src/libutil/http.c @@ -29,6 +29,8 @@ #include "libutil/regexp.h" #include "libserver/url.h" +#include <openssl/err.h> + #define ENCRYPTED_VERSION " HTTP/1.0" struct _rspamd_http_privbuf { @@ -43,6 +45,7 @@ enum rspamd_http_priv_flags { RSPAMD_HTTP_CONN_FLAG_NEW_HEADER = 1 << 1, RSPAMD_HTTP_CONN_FLAG_RESETED = 1 << 2, RSPAMD_HTTP_CONN_FLAG_TOO_LARGE = 1 << 3, + RSPAMD_HTTP_CONN_FLAG_ENCRYPTION_NEEDED = 1 << 4, }; #define IS_CONN_ENCRYPTED(c) ((c)->flags & RSPAMD_HTTP_CONN_FLAG_ENCRYPTED) @@ -430,7 +433,7 @@ rspamd_http_parse_key (rspamd_ftok_t *data, struct rspamd_http_connection *conn, if (priv->local_key == NULL) { /* In this case we cannot do anything, e.g. we cannot decrypt payload */ - priv->flags |= RSPAMD_HTTP_CONN_FLAG_ENCRYPTED; + priv->flags &= ~RSPAMD_HTTP_CONN_FLAG_ENCRYPTED; } else { /* Check sanity of what we have */ @@ -912,6 +915,12 @@ rspamd_http_on_message_complete (http_parser * parser) priv = conn->priv; + if ((conn->opts & RSPAMD_HTTP_REQUIRE_ENCRYPTION) && !IS_CONN_ENCRYPTED (priv)) { + priv->flags |= RSPAMD_HTTP_CONN_FLAG_ENCRYPTION_NEEDED; + msg_err ("unencrypted connection when encryption has been requested"); + return -1; + } + if ((conn->opts & RSPAMD_HTTP_BODY_PARTIAL) == 0 && IS_CONN_ENCRYPTED (priv)) { mode = rspamd_keypair_alg (priv->local_key); @@ -1178,8 +1187,12 @@ rspamd_http_event_handler (int fd, short what, gpointer ud) "Request entity too large: %zu", (size_t)priv->parser.content_length); } + else if (priv->flags & RSPAMD_HTTP_CONN_FLAG_ENCRYPTION_NEEDED) { + err = g_error_new (HTTP_ERROR, 400, + "Encryption required"); + } else { - err = g_error_new (HTTP_ERROR, priv->parser.http_errno, + err = g_error_new (HTTP_ERROR, 500 + priv->parser.http_errno, "HTTP parser error: %s", http_errno_description (priv->parser.http_errno)); } @@ -2184,7 +2197,7 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn } else { /* Invalid body for spamc method */ - g_assert (0); + abort (); } } @@ -2327,7 +2340,10 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn priv->ptv, rspamd_http_event_handler, rspamd_http_ssl_err_handler, conn)) { - err = g_error_new (HTTP_ERROR, errno, "ssl connection error"); + err = g_error_new (HTTP_ERROR, errno, + "ssl connection error: ssl error=%s, errno=%s", + ERR_error_string (ERR_get_error (), NULL), + strerror (errno)); rspamd_http_connection_ref (conn); conn->error_handler (conn, err); rspamd_http_connection_unref (conn); diff --git a/src/libutil/http.h b/src/libutil/http.h index c271caaa4..df6f99756 100644 --- a/src/libutil/http.h +++ b/src/libutil/http.h @@ -76,9 +76,10 @@ struct rspamd_storage_shmem { */ enum rspamd_http_options { RSPAMD_HTTP_BODY_PARTIAL = 0x1, /**< Call body handler on all body data portions *///!< RSPAMD_HTTP_BODY_PARTIAL - RSPAMD_HTTP_CLIENT_SIMPLE = 0x2, /**< Read HTTP client reply automatically */ //!< RSPAMD_HTTP_CLIENT_SIMPLE - RSPAMD_HTTP_CLIENT_ENCRYPTED = 0x4, /**< Encrypt data for client */ //!< RSPAMD_HTTP_CLIENT_ENCRYPTED - RSPAMD_HTTP_CLIENT_SHARED = 0x8, /**< Store reply in shared memory */ //!< RSPAMD_HTTP_CLIENT_SHARED + RSPAMD_HTTP_CLIENT_SIMPLE = 0x1u << 1, /**< Read HTTP client reply automatically */ //!< RSPAMD_HTTP_CLIENT_SIMPLE + RSPAMD_HTTP_CLIENT_ENCRYPTED = 0x1u << 2, /**< Encrypt data for client */ //!< RSPAMD_HTTP_CLIENT_ENCRYPTED + RSPAMD_HTTP_CLIENT_SHARED = 0x1u << 3, /**< Store reply in shared memory */ //!< RSPAMD_HTTP_CLIENT_SHARED + RSPAMD_HTTP_REQUIRE_ENCRYPTION = 0x1u << 4 }; typedef int (*rspamd_http_body_handler_t) (struct rspamd_http_connection *conn, diff --git a/src/libutil/logger.c b/src/libutil/logger.c index 610f1683d..45e99f8ae 100644 --- a/src/libutil/logger.c +++ b/src/libutil/logger.c @@ -571,7 +571,8 @@ rspamd_logger_need_log (rspamd_logger_t *rspamd_log, GLogLevelFlags log_level, { g_assert (rspamd_log != NULL); - if ((log_level & RSPAMD_LOG_FORCED) || log_level <= rspamd_log->log_level) { + if ((log_level & RSPAMD_LOG_FORCED) || + (log_level & (RSPAMD_LOG_LEVEL_MASK & G_LOG_LEVEL_MASK)) <= rspamd_log->log_level) { return TRUE; } diff --git a/src/libutil/logger.h b/src/libutil/logger.h index a969e66b0..7347e67b1 100644 --- a/src/libutil/logger.h +++ b/src/libutil/logger.h @@ -111,6 +111,12 @@ guint rspamd_logger_add_debug_module (const gchar *mod); rspamd_##mname##_log_id = rspamd_logger_add_debug_module(#mname); \ } +#define INIT_LOG_MODULE_PUBLIC(mname) \ + guint rspamd_##mname##_log_id = (guint)-1; \ + RSPAMD_CONSTRUCTOR(rspamd_##mname##_log_init) { \ + rspamd_##mname##_log_id = rspamd_logger_add_debug_module(#mname); \ +} + void rspamd_logger_configure_modules (GHashTable *mods_enabled); /** diff --git a/src/libutil/map_helpers.c b/src/libutil/map_helpers.c index e6b940f23..cc8002eac 100644 --- a/src/libutil/map_helpers.c +++ b/src/libutil/map_helpers.c @@ -426,6 +426,8 @@ rspamd_parse_kv_list ( } break; } + + data->state = map_skip_spaces_before_key; } return c; @@ -540,7 +542,7 @@ rspamd_map_helper_insert_re (gpointer st, gconstpointer key, gconstpointer value if (re_map->map_flags & RSPAMD_REGEXP_MAP_FLAG_GLOB) { escaped = rspamd_str_regexp_escape (key, strlen (key), &escaped_len, - TRUE); + RSPAMD_REGEXP_ESCAPE_GLOB|RSPAMD_REGEXP_ESCAPE_UTF); re = rspamd_regexp_new (escaped, NULL, &err); g_free (escaped); } diff --git a/src/libutil/mem_pool.c b/src/libutil/mem_pool.c index 322ebc409..9a5f316cc 100644 --- a/src/libutil/mem_pool.c +++ b/src/libutil/mem_pool.c @@ -163,6 +163,7 @@ rspamd_mempool_get_entry (const gchar *loc) return rspamd_mempool_entry_new (loc); } + static struct _pool_chain * rspamd_mempool_chain_new (gsize size, enum rspamd_mempool_chain_type pool_type) { @@ -171,7 +172,7 @@ rspamd_mempool_chain_new (gsize size, enum rspamd_mempool_chain_type pool_type) optimal_size = 0; gpointer map; - g_return_val_if_fail (size > 0, NULL); + g_assert (size > 0); if (pool_type == RSPAMD_MEMPOOL_SHARED) { #if defined(HAVE_MMAP_ANON) @@ -401,6 +402,11 @@ rspamd_mempool_new_ (gsize size, const gchar *tag, const gchar *loc) static void * memory_pool_alloc_common (rspamd_mempool_t * pool, gsize size, + enum rspamd_mempool_chain_type pool_type) +RSPAMD_ATTR_ALLOC_SIZE(2) RSPAMD_ATTR_ALLOC_ALIGN(MEM_ALIGNMENT) RSPAMD_ATTR_RETURNS_NONNUL; + +static void * +memory_pool_alloc_common (rspamd_mempool_t * pool, gsize size, enum rspamd_mempool_chain_type pool_type) { guint8 *tmp; @@ -464,7 +470,7 @@ memory_pool_alloc_common (rspamd_mempool_t * pool, gsize size, return tmp; } - return NULL; + abort (); } diff --git a/src/libutil/mem_pool.h b/src/libutil/mem_pool.h index c8dbf6042..33ada31dc 100644 --- a/src/libutil/mem_pool.h +++ b/src/libutil/mem_pool.h @@ -19,9 +19,34 @@ struct f_str_s; + + +#ifdef __has_attribute +# if __has_attribute(alloc_size) +# define RSPAMD_ATTR_ALLOC_SIZE(pos) __attribute__((alloc_size(pos))) +# else +# define RSPAMD_ATTR_ALLOC_SIZE(pos) +# endif + +# if __has_attribute(assume_aligned) +# define RSPAMD_ATTR_ALLOC_ALIGN(al) __attribute__((assume_aligned(al))) +# else +# define RSPAMD_ATTR_ALLOC_ALIGN(al) +# endif +# if __has_attribute(returns_nonnull) +# define RSPAMD_ATTR_RETURNS_NONNUL __attribute__((returns_nonnull)) +# else +# define RSPAMD_ATTR_RETURNS_NONNUL +# endif +#else +#define RSPAMD_ATTR_ALLOC_SIZE(pos) +#define RSPAMD_ATTR_ALLOC_ALIGN(al) +#define RSPAMD_ATTR_RETURNS_NONNUL +#endif + #define MEMPOOL_TAG_LEN 20 #define MEMPOOL_UID_LEN 20 -#define MEM_ALIGNMENT 16 +#define MEM_ALIGNMENT sizeof (guint64) #define align_ptr(p, a) \ (guint8 *) (((uintptr_t) (p) + ((uintptr_t) a - 1)) & ~((uintptr_t) a - 1)) @@ -131,7 +156,8 @@ rspamd_mempool_t * rspamd_mempool_new_ (gsize size, const gchar *tag, const gcha * @param size bytes to allocate * @return pointer to allocated object */ -void * rspamd_mempool_alloc (rspamd_mempool_t * pool, gsize size); +void * rspamd_mempool_alloc (rspamd_mempool_t * pool, gsize size) + RSPAMD_ATTR_ALLOC_SIZE(2) RSPAMD_ATTR_ALLOC_ALIGN(MEM_ALIGNMENT) RSPAMD_ATTR_RETURNS_NONNUL; /** * Get memory from temporary pool @@ -139,7 +165,7 @@ void * rspamd_mempool_alloc (rspamd_mempool_t * pool, gsize size); * @param size bytes to allocate * @return pointer to allocated object */ -void * rspamd_mempool_alloc_tmp (rspamd_mempool_t * pool, gsize size); +void * rspamd_mempool_alloc_tmp (rspamd_mempool_t * pool, gsize size) RSPAMD_ATTR_RETURNS_NONNUL; /** * Get memory and set it to zero @@ -147,7 +173,8 @@ void * rspamd_mempool_alloc_tmp (rspamd_mempool_t * pool, gsize size); * @param size bytes to allocate * @return pointer to allocated object */ -void * rspamd_mempool_alloc0 (rspamd_mempool_t * pool, gsize size); +void * rspamd_mempool_alloc0 (rspamd_mempool_t * pool, gsize size) + RSPAMD_ATTR_ALLOC_SIZE(2) RSPAMD_ATTR_ALLOC_ALIGN(MEM_ALIGNMENT) RSPAMD_ATTR_RETURNS_NONNUL; /** * Get memory and set it to zero @@ -155,7 +182,7 @@ void * rspamd_mempool_alloc0 (rspamd_mempool_t * pool, gsize size); * @param size bytes to allocate * @return pointer to allocated object */ -void * rspamd_mempool_alloc0_tmp (rspamd_mempool_t * pool, gsize size); +void * rspamd_mempool_alloc0_tmp (rspamd_mempool_t * pool, gsize size) RSPAMD_ATTR_RETURNS_NONNUL; /** * Cleanup temporary data in pool @@ -168,7 +195,8 @@ void rspamd_mempool_cleanup_tmp (rspamd_mempool_t * pool); * @param src source string * @return pointer to newly created string that is copy of src */ -gchar * rspamd_mempool_strdup (rspamd_mempool_t * pool, const gchar *src); +gchar * rspamd_mempool_strdup (rspamd_mempool_t * pool, const gchar *src) + RSPAMD_ATTR_ALLOC_ALIGN(MEM_ALIGNMENT); /** * Make a copy of fixed string in pool as null terminated string @@ -177,7 +205,7 @@ gchar * rspamd_mempool_strdup (rspamd_mempool_t * pool, const gchar *src); * @return pointer to newly created string that is copy of src */ gchar * rspamd_mempool_fstrdup (rspamd_mempool_t * pool, - const struct f_str_s *src); + const struct f_str_s *src) RSPAMD_ATTR_ALLOC_ALIGN(MEM_ALIGNMENT); struct f_str_tok; @@ -188,15 +216,17 @@ struct f_str_tok; * @return pointer to newly created string that is copy of src */ gchar * rspamd_mempool_ftokdup (rspamd_mempool_t *pool, - const struct f_str_tok *src); + const struct f_str_tok *src) RSPAMD_ATTR_ALLOC_ALIGN(MEM_ALIGNMENT); /** * Allocate piece of shared memory * @param pool memory pool object * @param size bytes to allocate */ -void * rspamd_mempool_alloc_shared (rspamd_mempool_t * pool, gsize size); -void * rspamd_mempool_alloc0_shared (rspamd_mempool_t *pool, gsize size); +void * rspamd_mempool_alloc_shared (rspamd_mempool_t * pool, gsize size) + RSPAMD_ATTR_ALLOC_SIZE(2) RSPAMD_ATTR_ALLOC_ALIGN(MEM_ALIGNMENT) RSPAMD_ATTR_RETURNS_NONNUL; +void * rspamd_mempool_alloc0_shared (rspamd_mempool_t *pool, gsize size) + RSPAMD_ATTR_ALLOC_SIZE(2) RSPAMD_ATTR_ALLOC_ALIGN(MEM_ALIGNMENT) RSPAMD_ATTR_RETURNS_NONNUL; /** * Add destructor callback to pool * @param pool memory pool object diff --git a/src/libutil/multipattern.c b/src/libutil/multipattern.c index 94b5398b3..e4a39d5fe 100644 --- a/src/libutil/multipattern.c +++ b/src/libutil/multipattern.c @@ -193,6 +193,12 @@ rspamd_multipattern_pattern_filter (const gchar *pattern, gsize len, gchar *ret = NULL; #ifdef WITH_HYPERSCAN if (rspamd_hs_check ()) { + gint gl_flags = RSPAMD_REGEXP_ESCAPE_ASCII; + + if (flags & RSPAMD_MULTIPATTERN_UTF8) { + gl_flags |= RSPAMD_REGEXP_ESCAPE_UTF; + } + if (flags & RSPAMD_MULTIPATTERN_TLD) { ret = rspamd_multipattern_escape_tld_hyperscan (pattern, len, dst_len); } @@ -201,10 +207,11 @@ rspamd_multipattern_pattern_filter (const gchar *pattern, gsize len, *dst_len = rspamd_strlcpy (ret, pattern, len + 1); } else if (flags & RSPAMD_MULTIPATTERN_GLOB) { - ret = rspamd_str_regexp_escape (pattern, len, dst_len, TRUE); + ret = rspamd_str_regexp_escape (pattern, len, dst_len, + gl_flags | RSPAMD_REGEXP_ESCAPE_GLOB); } else { - ret = rspamd_str_regexp_escape (pattern, len, dst_len, FALSE); + ret = rspamd_str_regexp_escape (pattern, len, dst_len, gl_flags); } return ret; diff --git a/src/libutil/printf.c b/src/libutil/printf.c index 7ee4d35e6..148b49d9e 100644 --- a/src/libutil/printf.c +++ b/src/libutil/printf.c @@ -602,7 +602,6 @@ rspamd_vprintf_common (rspamd_printf_append_func func, rspamd_ftok_t *tok; GString *gs; GError *err; - gboolean bv; while (*fmt) { @@ -969,12 +968,6 @@ rspamd_vprintf_common (rspamd_printf_append_func func, continue; - case 'B': - bv = (gboolean) va_arg (args, double); - RSPAMD_PRINTF_APPEND (bv ? "true" : "false", bv ? 4 : 5); - - continue; - case 'p': ui64 = (uintptr_t) va_arg (args, void *); hex = 2; diff --git a/src/libutil/printf.h b/src/libutil/printf.h index 73787d3a5..86947f67c 100644 --- a/src/libutil/printf.h +++ b/src/libutil/printf.h @@ -32,7 +32,6 @@ * %[0][width][.width]F long double * %[0][width][.width]g double * %[0][width][.width]G long double - * %B boolean (true or false) * %P pid_t * %r rlim_t * %p void * diff --git a/src/libutil/regexp.c b/src/libutil/regexp.c index c8f4faa4d..4eb0361ec 100644 --- a/src/libutil/regexp.c +++ b/src/libutil/regexp.c @@ -297,6 +297,7 @@ rspamd_regexp_new (const gchar *pattern, const gchar *flags, const gchar *start = pattern, *end, *flags_str = NULL; gchar *err_str; rspamd_regexp_t *res; + gboolean explicit_utf = FALSE; PCRE_T *r; gchar sep = 0, *real_pattern; #ifndef WITH_PCRE2 @@ -378,11 +379,13 @@ rspamd_regexp_new (const gchar *pattern, const gchar *flags, break; case 'u': rspamd_flags &= ~RSPAMD_REGEXP_FLAG_RAW; + rspamd_flags |= RSPAMD_REGEXP_FLAG_UTF; #ifndef WITH_PCRE2 regexp_flags |= PCRE_FLAG(UTF8); #else regexp_flags |= PCRE_FLAG(UTF); #endif + explicit_utf = TRUE; break; case 'O': /* We optimize all regexps by default */ @@ -390,6 +393,7 @@ rspamd_regexp_new (const gchar *pattern, const gchar *flags, break; case 'r': rspamd_flags |= RSPAMD_REGEXP_FLAG_RAW; + rspamd_flags &= ~RSPAMD_REGEXP_FLAG_UTF; #ifndef WITH_PCRE2 regexp_flags &= ~PCRE_FLAG(UTF8); #else @@ -453,7 +457,7 @@ fin: if (rspamd_flags & RSPAMD_REGEXP_FLAG_RAW) { res->raw_re = r; } - else { + else if (!explicit_utf) { #ifndef WITH_PCRE2 res->raw_re = pcre_compile (real_pattern, regexp_flags & ~PCRE_FLAG(UTF8), (const char **)&err_str, &err_off, NULL); @@ -568,7 +572,11 @@ rspamd_regexp_search (rspamd_regexp_t *re, const gchar *text, gsize len, #endif } - g_assert (r != NULL); + if (r == NULL) { + /* Invalid regexp type for the specified input */ + return FALSE; + } + ncaptures = (re->ncaptures + 1) * 3; ovec = g_alloca (sizeof (gint) * ncaptures); diff --git a/src/libutil/regexp.h b/src/libutil/regexp.h index 6b1bd50f9..b982c08f6 100644 --- a/src/libutil/regexp.h +++ b/src/libutil/regexp.h @@ -33,6 +33,7 @@ #define RSPAMD_REGEXP_FLAG_FULL_MATCH (1 << 3) #define RSPAMD_REGEXP_FLAG_PCRE_ONLY (1 << 4) #define RSPAMD_REGEXP_FLAG_DISABLE_JIT (1 << 5) +#define RSPAMD_REGEXP_FLAG_UTF (1 << 6) struct rspamd_config; diff --git a/src/libutil/shingles.c b/src/libutil/shingles.c index 240facc4a..70aa5fe78 100644 --- a/src/libutil/shingles.c +++ b/src/libutil/shingles.c @@ -126,7 +126,7 @@ rspamd_shingles_from_text (GArray *input, rspamd_stat_token_t *word; guint64 val; gint i, j, k; - gsize hlen, beg = 0; + gsize hlen, ilen = 0, beg = 0, widx = 0; enum rspamd_cryptobox_fast_hash_type ht; if (pool != NULL) { @@ -138,10 +138,19 @@ rspamd_shingles_from_text (GArray *input, row = rspamd_fstring_sized_new (256); + for (i = 0; i < input->len; i ++) { + word = &g_array_index (input, rspamd_stat_token_t, i); + + if (!((word->flags & RSPAMD_STAT_TOKEN_FLAG_SKIPPED) + || word->stemmed.len == 0)) { + ilen ++; + } + } + /* Init hashes pipes and keys */ hashes = g_malloc (sizeof (*hashes) * RSPAMD_SHINGLE_SIZE); - hlen = input->len > SHINGLES_WINDOW ? - (input->len - SHINGLES_WINDOW + 1) : 1; + hlen = ilen > SHINGLES_WINDOW ? + (ilen - SHINGLES_WINDOW + 1) : 1; keys = rspamd_shingles_get_keys_cached (key); for (i = 0; i < RSPAMD_SHINGLE_SIZE; i ++) { @@ -150,11 +159,36 @@ rspamd_shingles_from_text (GArray *input, /* Now parse input words into a vector of hashes using rolling window */ if (alg == RSPAMD_SHINGLES_OLD) { - for (i = 0; i <= (gint)input->len; i ++) { - if (i - beg >= SHINGLES_WINDOW || i == (gint)input->len) { + for (i = 0; i <= (gint)ilen; i ++) { + if (i - beg >= SHINGLES_WINDOW || i == (gint)ilen) { for (j = beg; j < i; j ++) { - word = &g_array_index (input, rspamd_stat_token_t, j); - row = rspamd_fstring_append (row, word->begin, word->len); + + word = NULL; + while (widx < input->len) { + word = &g_array_index (input, rspamd_stat_token_t, widx); + + if ((word->flags & RSPAMD_STAT_TOKEN_FLAG_SKIPPED) + || word->stemmed.len == 0) { + widx++; + } + else { + break; + } + } + + if (word == NULL) { + /* Nothing but exceptions */ + for (i = 0; i < RSPAMD_SHINGLE_SIZE; i ++) { + g_free (hashes[i]); + } + + g_free (hashes); + + return NULL; + } + + row = rspamd_fstring_append (row, word->stemmed.begin, + word->stemmed.len); } /* Now we need to create a new row here */ @@ -166,13 +200,14 @@ rspamd_shingles_from_text (GArray *input, } beg++; + widx ++; row = rspamd_fstring_assign (row, "", 0); } } } else { - guint64 res[SHINGLES_WINDOW * RSPAMD_SHINGLE_SIZE], seed; + guint64 window[SHINGLES_WINDOW * RSPAMD_SHINGLE_SIZE], seed; switch (alg) { case RSPAMD_SHINGLES_XXHASH: @@ -186,34 +221,60 @@ rspamd_shingles_from_text (GArray *input, break; } - memset (res, 0, sizeof (res)); - for (i = 0; i <= (gint)input->len; i ++) { - if (i - beg >= SHINGLES_WINDOW || i == (gint)input->len) { + memset (window, 0, sizeof (window)); + for (i = 0; i <= ilen; i ++) { + if (i - beg >= SHINGLES_WINDOW || i == ilen) { for (j = 0; j < RSPAMD_SHINGLE_SIZE; j ++) { /* Shift hashes window to right */ for (k = 0; k < SHINGLES_WINDOW - 1; k ++) { - res[j * SHINGLES_WINDOW + k] = - res[j * SHINGLES_WINDOW + k + 1]; + window[j * SHINGLES_WINDOW + k] = + window[j * SHINGLES_WINDOW + k + 1]; + } + + word = NULL; + + while (widx < input->len) { + word = &g_array_index (input, rspamd_stat_token_t, widx); + + if ((word->flags & RSPAMD_STAT_TOKEN_FLAG_SKIPPED) + || word->stemmed.len == 0) { + widx++; + } + else { + break; + } + } + + if (word == NULL) { + /* Nothing but exceptions */ + for (i = 0; i < RSPAMD_SHINGLE_SIZE; i ++) { + g_free (hashes[i]); + } + + g_free (hashes); + + return NULL; } - word = &g_array_index (input, rspamd_stat_token_t, beg); /* Insert the last element to the pipe */ memcpy (&seed, keys[j], sizeof (seed)); - res[j * SHINGLES_WINDOW + SHINGLES_WINDOW - 1] = + window[j * SHINGLES_WINDOW + SHINGLES_WINDOW - 1] = rspamd_cryptobox_fast_hash_specific (ht, - word->begin, word->len, + word->stemmed.begin, word->stemmed.len, seed); val = 0; for (k = 0; k < SHINGLES_WINDOW; k ++) { - val ^= res[j * SHINGLES_WINDOW + k] >> + val ^= window[j * SHINGLES_WINDOW + k] >> (8 * (SHINGLES_WINDOW - k - 1)); } g_assert (hlen > beg); hashes[j][beg] = val; } - beg++; + + beg ++; + widx ++; } } } diff --git a/src/libutil/str_util.c b/src/libutil/str_util.c index 09817a344..0c63edba2 100644 --- a/src/libutil/str_util.c +++ b/src/libutil/str_util.c @@ -297,18 +297,38 @@ rspamd_ftok_icase_hash (gconstpointer key) } gboolean +rspamd_ftok_equal (gconstpointer v, gconstpointer v2) +{ + const rspamd_ftok_t *f1 = v, *f2 = v2; + + if (f1->len == f2->len && + memcmp (f1->begin, f2->begin, f1->len) == 0) { + return TRUE; + } + + return FALSE; +} + +guint +rspamd_ftok_hash (gconstpointer key) +{ + const rspamd_ftok_t *f = key; + + return t1ha (f->begin, f->len, rspamd_hash_seed ()); +} + +gboolean rspamd_gstring_icase_equal (gconstpointer v, gconstpointer v2) { const GString *f1 = v, *f2 = v2; if (f1->len == f2->len && - rspamd_lc_cmp (f1->str, f2->str, f1->len) == 0) { + rspamd_lc_cmp (f1->str, f2->str, f1->len) == 0) { return TRUE; } return FALSE; } - guint rspamd_gstring_icase_hash (gconstpointer key) { @@ -882,6 +902,128 @@ rspamd_encode_base64_fold (const guchar *in, gsize inlen, gint str_len, return rspamd_encode_base64_common (in, inlen, str_len, outlen, TRUE, how); } +gchar * +rspamd_encode_qp_fold (const guchar *in, gsize inlen, gint str_len, + gsize *outlen, enum rspamd_newlines_type how) +{ + gsize olen = 0, span = 0, i = 0; + gchar *out; + gint ch; + const guchar *end = in + inlen, *p = in; + static const gchar hexdigests[16] = "0123456789ABCDEF"; + + while (p < end) { + ch = *p; + + if (ch < 128 && ch != '\r' && ch != '\n') { + olen ++; + span ++; + } + else { + if (str_len > 0 && span + 5 >= str_len) { + if (how == RSPAMD_TASK_NEWLINES_CRLF) { + /* =\r\n */ + olen += 3; + } + else { + olen += 2; + } + span = 0; + } + + olen += 3; + span += 3; + } + + if (str_len > 0 && span + 3 >= str_len) { + if (how == RSPAMD_TASK_NEWLINES_CRLF) { + /* =\r\n */ + olen += 3; + } + else { + olen += 2; + } + span = 0; + } + + p ++; + } + + out = g_malloc (olen + 1); + p = in; + i = 0; + span = 0; + + while (p < end) { + ch = *p; + + if (ch < 128 && ch != '\r' && ch != '\n') { + out[i++] = ch; + span ++; + } + else { + if (str_len > 0 && span + 5 >= str_len) { + /* Add new line and then continue */ + switch (how) { + default: + case RSPAMD_TASK_NEWLINES_CRLF: + out[i++] = '='; + out[i++] = '\r'; + out[i++] = '\n'; + break; + case RSPAMD_TASK_NEWLINES_LF: + out[i++] = '='; + out[i++] = '\n'; + break; + case RSPAMD_TASK_NEWLINES_CR: + out[i++] = '='; + out[i++] = '\r'; + break; + } + + span = 0; + } + + out[i++] = '='; + out[i++] = hexdigests[((ch >> 4) & 0xF)]; + out[i++] = hexdigests[(ch & 0xF)]; + span += 3; + } + + if (str_len > 0 && span + 3 >= str_len) { + /* Add new line and then continue */ + switch (how) { + default: + case RSPAMD_TASK_NEWLINES_CRLF: + out[i++] = '='; + out[i++] = '\r'; + out[i++] = '\n'; + break; + case RSPAMD_TASK_NEWLINES_LF: + out[i++] = '='; + out[i++] = '\n'; + break; + case RSPAMD_TASK_NEWLINES_CR: + out[i++] = '='; + out[i++] = '\r'; + break; + } + + span = 0; + } + + g_assert (i <= olen); + p ++; + } + + out[i] = '\0'; + + if (outlen) { + *outlen = i; + } + + return out; +} #define MIN3(a, b, c) ((a) < (b) ? ((a) < (c) ? (a) : (c)) : ((b) < (c) ? (b) : (c))) @@ -1428,7 +1570,8 @@ rspamd_string_find_eoh (GString *input, goffset *body_start) got_lf, got_linebreak, got_linebreak_cr, - got_linebreak_lf + got_linebreak_lf, + obs_fws } state = skip_char; g_assert (input != NULL); @@ -1478,7 +1621,9 @@ rspamd_string_find_eoh (GString *input, goffset *body_start) } else if (g_ascii_isspace (*p)) { /* We have \r<space>*, allow to stay in this state */ + c = p; p ++; + state = obs_fws; } else { p++; @@ -1498,7 +1643,9 @@ rspamd_string_find_eoh (GString *input, goffset *body_start) } else if (g_ascii_isspace (*p)) { /* We have \n<space>*, allow to stay in this state */ + c = p; p ++; + state = obs_fws; } else { p++; @@ -1518,7 +1665,9 @@ rspamd_string_find_eoh (GString *input, goffset *body_start) } else if (g_ascii_isspace (*p)) { /* We have <linebreak><space>*, allow to stay in this state */ + c = p; p ++; + state = obs_fws; } else { p++; @@ -1537,6 +1686,8 @@ rspamd_string_find_eoh (GString *input, goffset *body_start) } else if (g_ascii_isspace (*p)) { /* We have \r\n<space>*, allow to keep in this state */ + c = p; + state = obs_fws; p ++; } else { @@ -1552,7 +1703,95 @@ rspamd_string_find_eoh (GString *input, goffset *body_start) } return c - input->str; + case obs_fws: + if (*p == ' ' || *p == '\t') { + p ++; + } + else if (*p == '\r') { + /* Perform lookahead due to #2349 */ + if (end - p > 2) { + if (p[1] == '\n' && g_ascii_isspace (p[2])) { + /* Real obs_fws state, switch */ + c = p; + p ++; + state = got_cr; + } + else if (g_ascii_isspace (p[1])) { + p ++; + state = obs_fws; + } + else { + /* + * newline wsp+ \r <nwsp>, hence: + * c -> eoh + * p + 1 -> body start + */ + if (body_start) { + /* \r\n\r\n */ + *body_start = p - input->str + 1; + } + + return c - input->str; + } + } + else { + /* shortage */ + if (body_start) { + *body_start = p - input->str + 1; + } + + return p - input->str; + } + } + else if (*p == '\n') { + /* Perform lookahead due to #2349 */ + if (end - p > 1) { + if (p[1] == ' ' || p[1] == '\t') { + c = p; + p ++; + state = obs_fws; + } + else if (p[1] == '\r') { + c = p; + p ++; + state = got_lf; + } + else if (p[1] == '\n') { + c = p; + p ++; + state = got_lf; + } + else { + /* + * newline wsp+ \n <nwsp>, hence: + * c -> eoh + * p + 1 -> body start + */ + if (body_start) { + /* \r\n\r\n */ + *body_start = p - input->str + 1; + } + + return c - input->str; + } + + } + else { + /* shortage */ + if (body_start) { + *body_start = p - input->str + 1; + } + + return p - input->str; + } + } + else { + p++; + state = skip_char; + } + break; } + } if (state == got_linebreak_lf) { @@ -2120,25 +2359,71 @@ rspamd_memrchr (const void *m, gint c, gsize len) return NULL; } +struct UConverter * +rspamd_get_utf8_converter (void) +{ + static UConverter *utf8_conv = NULL; + UErrorCode uc_err = U_ZERO_ERROR; + + if (utf8_conv == NULL) { + utf8_conv = ucnv_open ("UTF-8", &uc_err); + if (!U_SUCCESS (uc_err)) { + msg_err ("FATAL error: cannot open converter for utf8: %s", + u_errorName (uc_err)); + + g_assert_not_reached (); + } + + ucnv_setFromUCallBack (utf8_conv, + UCNV_FROM_U_CALLBACK_SUBSTITUTE, + NULL, + NULL, + NULL, + &uc_err); + ucnv_setToUCallBack (utf8_conv, + UCNV_TO_U_CALLBACK_SUBSTITUTE, + NULL, + NULL, + NULL, + &uc_err); + } + + return utf8_conv; +} + + +const struct UNormalizer2 * +rspamd_get_unicode_normalizer (void) +{ +#if U_ICU_VERSION_MAJOR_NUM >= 44 + UErrorCode uc_err = U_ZERO_ERROR; + static const UNormalizer2 *norm = NULL; + + if (norm == NULL) { + norm = unorm2_getInstance (NULL, "nfkc", UNORM2_COMPOSE, &uc_err); + g_assert (U_SUCCESS (uc_err)); + } + + return norm; +#else + /* Old libicu */ + return NULL; +#endif +} + + gboolean rspamd_normalise_unicode_inplace (rspamd_mempool_t *pool, gchar *start, guint *len) { #if U_ICU_VERSION_MAJOR_NUM >= 44 UErrorCode uc_err = U_ZERO_ERROR; - static UConverter *utf8_conv = NULL; - static const UNormalizer2 *norm = NULL; + UConverter *utf8_conv = rspamd_get_utf8_converter (); + const UNormalizer2 *norm = rspamd_get_unicode_normalizer (); gint32 nsym, end; UChar *src = NULL, *dest = NULL; gboolean ret = FALSE; - if (utf8_conv == NULL) { - utf8_conv = ucnv_open ("UTF-8", &uc_err); - g_assert (U_SUCCESS (uc_err)); - norm = unorm2_getInstance (NULL, "nfkc", UNORM2_COMPOSE, &uc_err); - g_assert (U_SUCCESS (uc_err)); - } - /* We first need to convert data to UChars :( */ src = g_malloc ((*len + 1) * sizeof (*src)); nsym = ucnv_toUChars (utf8_conv, src, *len + 1, @@ -2210,10 +2495,10 @@ out: gchar * rspamd_str_regexp_escape (const gchar *pattern, gsize slen, - gsize *dst_len, gboolean allow_glob) + gsize *dst_len, enum rspamd_regexp_escape_flags flags) { const gchar *p, *end = pattern + slen; - gchar *res, *d, t; + gchar *res, *d, t, *tmp_utf = NULL; gsize len; static const gchar hexdigests[16] = "0123456789abcdef"; @@ -2248,20 +2533,46 @@ rspamd_str_regexp_escape (const gchar *pattern, gsize slen, if (g_ascii_isspace (t)) { len ++; } - else if (!g_ascii_isprint (t)) { - /* \\xHH -> 4 symbols */ - len += 3; + else { + if (!(flags & RSPAMD_REGEXP_ESCAPE_UTF)) { + if (!g_ascii_isprint (t)) { + /* \\xHH -> 4 symbols */ + len += 3; + } + } } break; } } + if (flags & RSPAMD_REGEXP_ESCAPE_UTF) { + if (!g_utf8_validate (pattern, slen, NULL)) { + tmp_utf = rspamd_str_make_utf_valid (pattern, slen, NULL); + } + } + if (slen == len) { if (dst_len) { + + if (tmp_utf) { + slen = strlen (tmp_utf); + } + *dst_len = slen; } - return g_strdup (pattern); + + + if (tmp_utf) { + return tmp_utf; + } + else { + return g_strdup (pattern); + } + } + + if (tmp_utf) { + pattern = tmp_utf; } res = g_malloc (len + 1); @@ -2291,7 +2602,7 @@ rspamd_str_regexp_escape (const gchar *pattern, gsize slen, case '*': case '?': case '+': - if (allow_glob) { + if (flags & RSPAMD_REGEXP_ESCAPE_GLOB) { /* Treat * as .* and ? as .? */ *d++ = '.'; } @@ -2303,7 +2614,7 @@ rspamd_str_regexp_escape (const gchar *pattern, gsize slen, if (g_ascii_isspace (t)) { *d++ = '\\'; } - else if (!g_ascii_isgraph (t)) { + else if (!(flags & RSPAMD_REGEXP_ESCAPE_UTF) && !g_ascii_isgraph (t)) { *d++ = '\\'; *d++ = 'x'; *d++ = hexdigests[((t >> 4) & 0xF)]; @@ -2322,5 +2633,67 @@ rspamd_str_regexp_escape (const gchar *pattern, gsize slen, *dst_len = d - res; } + if (tmp_utf) { + g_free (tmp_utf); + } + return res; } + + +gchar * +rspamd_str_make_utf_valid (const gchar *src, gsize slen, gsize *dstlen) +{ + GString *dst; + const gchar *last; + gchar *dchar; + gsize i, valid, prev; + UChar32 uc; + + if (src == NULL) { + return NULL; + } + + if (slen == 0) { + slen = strlen (src); + } + + dst = g_string_sized_new (slen); + i = 0; + last = src; + valid = 0; + prev = 0; + + while (i < slen) { + U8_NEXT (src, i, slen, uc); + + if (uc <= 0) { + if (valid > 0) { + g_string_append_len (dst, last, valid); + } + /* 0xFFFD in UTF8 */ + g_string_append_len (dst, "\357\277\275", 3); + valid = 0; + last = &src[i]; + } + else { + valid += i - prev; + } + + prev = i; + } + + if (valid > 0) { + g_string_append_len (dst, last, valid); + } + + dchar = dst->str; + + if (dstlen) { + *dstlen = dst->len; + } + + g_string_free (dst, FALSE); + + return dchar; +}
\ No newline at end of file diff --git a/src/libutil/str_util.h b/src/libutil/str_util.h index 73637a62c..935c5116d 100644 --- a/src/libutil/str_util.h +++ b/src/libutil/str_util.h @@ -61,6 +61,8 @@ gboolean rspamd_str_equal (gconstpointer v, gconstpointer v2); */ guint rspamd_ftok_icase_hash (gconstpointer key); gboolean rspamd_ftok_icase_equal (gconstpointer v, gconstpointer v2); +guint rspamd_ftok_hash (gconstpointer key); +gboolean rspamd_ftok_equal (gconstpointer v, gconstpointer v2); guint rspamd_gstring_icase_hash (gconstpointer key); gboolean rspamd_gstring_icase_equal (gconstpointer v, gconstpointer v2); @@ -202,6 +204,16 @@ gchar * rspamd_encode_base64_fold (const guchar *in, gsize inlen, gint str_len, gsize *outlen, enum rspamd_newlines_type how); /** + * Encode and fold string using quoted printable encoding + * @param in input + * @param inlen input length + * @param str_len maximum string length (if <= 0 then no lines are split) + * @return freshly allocated base64 encoded value or NULL if input is invalid + */ +gchar * rspamd_encode_qp_fold (const guchar *in, gsize inlen, gint str_len, + gsize *outlen, enum rspamd_newlines_type how); + +/** * Decode quoted-printable encoded buffer, input and output must not overlap * @param in input * @param inlen length of input @@ -384,6 +396,12 @@ rspamd_str_has_8bit (const guchar *beg, gsize len) return FALSE; } +struct UConverter; +struct UConverter *rspamd_get_utf8_converter (void); + +struct UNormalizer2; +const struct UNormalizer2 *rspamd_get_unicode_normalizer (void); + /** * Gets a string in UTF8 and normalises it to NFKC_Casefold form * @param pool optional memory pool used for logging purposes @@ -394,6 +412,11 @@ rspamd_str_has_8bit (const guchar *beg, gsize len) gboolean rspamd_normalise_unicode_inplace (rspamd_mempool_t *pool, gchar *start, guint *len); +enum rspamd_regexp_escape_flags { + RSPAMD_REGEXP_ESCAPE_ASCII = 0, + RSPAMD_REGEXP_ESCAPE_UTF = 1u << 0, + RSPAMD_REGEXP_ESCAPE_GLOB = 1u << 1, +}; /** * Escapes special characters when reading plain data to be processed in pcre * @param pattern pattern to process @@ -404,6 +427,16 @@ gboolean rspamd_normalise_unicode_inplace (rspamd_mempool_t *pool, */ gchar * rspamd_str_regexp_escape (const gchar *pattern, gsize slen, - gsize *dst_len, gboolean allow_glob); + gsize *dst_len, enum rspamd_regexp_escape_flags flags); + +/** + * Returns copy of src (zero terminated) where all unicode is made valid or replaced + * to FFFD characters. Caller must free string after usage + * @param src + * @param slen + * @param dstelen + * @return + */ +gchar * rspamd_str_make_utf_valid (const gchar *src, gsize slen, gsize *dstlen); #endif /* SRC_LIBUTIL_STR_UTIL_H_ */ diff --git a/src/libutil/upstream.c b/src/libutil/upstream.c index 3703bdd19..eb88e501a 100644 --- a/src/libutil/upstream.c +++ b/src/libutil/upstream.c @@ -22,6 +22,8 @@ #include "cryptobox.h" #include "utlist.h" +#include <math.h> + struct upstream_inet_addr_entry { rspamd_inet_addr_t *addr; struct upstream_inet_addr_entry *next; @@ -32,6 +34,14 @@ struct upstream_addr_elt { guint errors; }; +struct upstream_list_watcher { + rspamd_upstream_watch_func func; + GFreeFunc dtor; + gpointer ud; + enum rspamd_upstreams_watch_event events_mask; + struct upstream_list_watcher *next, *prev; +}; + struct upstream { guint weight; guint cur_weight; @@ -58,12 +68,23 @@ struct upstream { ref_entry_t ref; }; +struct upstream_limits { + gdouble revive_time; + gdouble revive_jitter; + gdouble error_time; + gdouble dns_timeout; + guint max_errors; + guint dns_retransmits; +}; + struct upstream_list { struct upstream_ctx *ctx; GPtrArray *ups; GPtrArray *alive; + struct upstream_list_watcher *watchers; rspamd_mutex_t *lock; guint64 hash_seed; + struct upstream_limits limits; guint cur_elt; enum rspamd_upstream_flag flags; enum rspamd_upstream_rotation rot_alg; @@ -72,12 +93,7 @@ struct upstream_list { struct upstream_ctx { struct rdns_resolver *res; struct event_base *ev_base; - guint max_errors; - gdouble revive_time; - gdouble revive_jitter; - gdouble error_time; - gdouble dns_timeout; - guint dns_retransmits; + struct upstream_limits limits; GQueue *upstreams; gboolean configured; rspamd_mempool_t *pool; @@ -102,26 +118,27 @@ static guint default_dns_retransmits = 2; void rspamd_upstreams_library_config (struct rspamd_config *cfg, - struct upstream_ctx *ctx, struct event_base *ev_base, - struct rdns_resolver *resolver) + struct upstream_ctx *ctx, + struct event_base *ev_base, + struct rdns_resolver *resolver) { g_assert (ctx != NULL); g_assert (cfg != NULL); if (cfg->upstream_error_time) { - ctx->error_time = cfg->upstream_error_time; + ctx->limits.error_time = cfg->upstream_error_time; } if (cfg->upstream_max_errors) { - ctx->max_errors = cfg->upstream_max_errors; + ctx->limits.max_errors = cfg->upstream_max_errors; } if (cfg->upstream_revive_time) { - ctx->revive_time = cfg->upstream_max_errors; + ctx->limits.revive_time = cfg->upstream_max_errors; } if (cfg->dns_retransmits) { - ctx->dns_retransmits = cfg->dns_retransmits; + ctx->limits.dns_retransmits = cfg->dns_retransmits; } if (cfg->dns_timeout) { - ctx->dns_timeout = cfg->dns_timeout; + ctx->limits.dns_timeout = cfg->dns_timeout; } ctx->ev_base = ev_base; @@ -161,12 +178,12 @@ rspamd_upstreams_library_init (void) struct upstream_ctx *ctx; ctx = g_malloc0 (sizeof (*ctx)); - ctx->error_time = default_error_time; - ctx->max_errors = default_max_errors; - ctx->dns_retransmits = default_dns_retransmits; - ctx->dns_timeout = default_dns_timeout; - ctx->revive_jitter = default_revive_jitter; - ctx->revive_time = default_revive_time; + ctx->limits.error_time = default_error_time; + ctx->limits.max_errors = default_max_errors; + ctx->limits.dns_retransmits = default_dns_retransmits; + ctx->limits.dns_timeout = default_dns_timeout; + ctx->limits.revive_jitter = default_revive_jitter; + ctx->limits.revive_time = default_revive_time; ctx->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "upstreams"); @@ -375,14 +392,14 @@ rspamd_upstream_resolve_addrs (const struct upstream_list *ls, if (up->name[0] != '/') { if (rdns_make_request_full (up->ctx->res, rspamd_upstream_dns_cb, up, - up->ctx->dns_timeout, up->ctx->dns_retransmits, + ls->limits.dns_timeout, ls->limits.dns_retransmits, 1, up->name, RDNS_REQUEST_A) != NULL) { up->dns_requests ++; REF_RETAIN (up); } if (rdns_make_request_full (up->ctx->res, rspamd_upstream_dns_cb, up, - up->ctx->dns_timeout, up->ctx->dns_retransmits, + ls->limits.dns_timeout, ls->limits.dns_retransmits, 1, up->name, RDNS_REQUEST_AAAA) != NULL) { up->dns_requests ++; REF_RETAIN (up); @@ -398,6 +415,7 @@ rspamd_upstream_set_inactive (struct upstream_list *ls, struct upstream *up) guint i; struct upstream *cur; struct timeval tv; + struct upstream_list_watcher *w; RSPAMD_UPSTREAM_LOCK (ls->lock); g_ptr_array_remove_index (ls->alive, up->active_idx); @@ -418,12 +436,18 @@ rspamd_upstream_set_inactive (struct upstream_list *ls, struct upstream *up) event_base_set (up->ctx->ev_base, &up->ev); } - ntim = rspamd_time_jitter (up->ctx->revive_time, - up->ctx->revive_jitter); + ntim = rspamd_time_jitter (ls->limits.revive_time, + ls->limits.revive_jitter); double_to_tv (ntim, &tv); event_add (&up->ev, &tv); } + DL_FOREACH (up->ls->watchers, w) { + if (w->events_mask & RSPAMD_UPSTREAM_WATCH_OFFLINE) { + w->func (up, RSPAMD_UPSTREAM_WATCH_OFFLINE, up->errors, w->ud); + } + } + RSPAMD_UPSTREAM_UNLOCK (ls->lock); } @@ -433,6 +457,7 @@ rspamd_upstream_fail (struct upstream *up, gboolean addr_failure) gdouble error_rate, max_error_rate; gdouble sec_last, sec_cur; struct upstream_addr_elt *addr_elt; + struct upstream_list_watcher *w; if (up->ctx && up->active_idx != -1) { sec_cur = rspamd_get_ticks (FALSE); @@ -442,6 +467,12 @@ rspamd_upstream_fail (struct upstream *up, gboolean addr_failure) /* We have the first error */ up->last_fail = sec_cur; up->errors = 1; + + DL_FOREACH (up->ls->watchers, w) { + if (w->events_mask & RSPAMD_UPSTREAM_WATCH_FAILURE) { + w->func (up, RSPAMD_UPSTREAM_WATCH_FAILURE, 1, w->ud); + } + } } else { sec_last = up->last_fail; @@ -449,10 +480,16 @@ rspamd_upstream_fail (struct upstream *up, gboolean addr_failure) if (sec_cur >= sec_last) { up->errors ++; + DL_FOREACH (up->ls->watchers, w) { + if (w->events_mask & RSPAMD_UPSTREAM_WATCH_FAILURE) { + w->func (up, RSPAMD_UPSTREAM_WATCH_FAILURE, up->errors, w->ud); + } + } + if (sec_cur > sec_last) { error_rate = ((gdouble)up->errors) / (sec_cur - sec_last); - max_error_rate = ((gdouble)up->ctx->max_errors) / - up->ctx->error_time; + max_error_rate = ((gdouble)up->ls->limits.max_errors) / + up->ls->limits.error_time; } else { error_rate = 1; @@ -467,7 +504,7 @@ rspamd_upstream_fail (struct upstream *up, gboolean addr_failure) } else { /* Just re-resolve addresses */ - if (sec_cur - sec_last > up->ctx->revive_time) { + if (sec_cur - sec_last > up->ls->limits.revive_time) { up->errors = 0; rspamd_upstream_resolve_addrs (up->ls, up); } @@ -492,6 +529,7 @@ void rspamd_upstream_ok (struct upstream *up) { struct upstream_addr_elt *addr_elt; + struct upstream_list_watcher *w; RSPAMD_UPSTREAM_LOCK (up->lock); if (up->errors > 0 && up->active_idx != -1) { @@ -502,6 +540,12 @@ rspamd_upstream_ok (struct upstream *up) addr_elt = g_ptr_array_index (up->addrs.addr, up->addrs.cur); addr_elt->errors = 0; } + + DL_FOREACH (up->ls->watchers, w) { + if (w->events_mask & RSPAMD_UPSTREAM_WATCH_SUCCESS) { + w->func (up, RSPAMD_UPSTREAM_WATCH_SUCCESS, 0, w->ud); + } + } } RSPAMD_UPSTREAM_UNLOCK (up->lock); @@ -531,6 +575,18 @@ rspamd_upstreams_create (struct upstream_ctx *ctx) ls->ctx = ctx; ls->rot_alg = RSPAMD_UPSTREAM_UNDEF; + if (ctx) { + ls->limits = ctx->limits; + } + else { + ls->limits.error_time = default_error_time; + ls->limits.max_errors = default_max_errors; + ls->limits.dns_retransmits = default_dns_retransmits; + ls->limits.dns_timeout = default_dns_timeout; + ls->limits.revive_jitter = default_revive_jitter; + ls->limits.revive_time = default_revive_time; + } + return ls; } @@ -812,6 +868,7 @@ rspamd_upstreams_destroy (struct upstream_list *ups) { guint i; struct upstream *up; + struct upstream_list_watcher *w, *tmp; if (ups != NULL) { g_ptr_array_free (ups->alive, TRUE); @@ -822,6 +879,13 @@ rspamd_upstreams_destroy (struct upstream_list *ups) REF_RELEASE (up); } + DL_FOREACH_SAFE (ups->watchers, w, tmp) { + if (w->dtor) { + w->dtor (w->ud); + } + g_free (w); + } + g_ptr_array_free (ups->ups, TRUE); rspamd_mutex_free (ups->lock); g_free (ups); @@ -833,6 +897,7 @@ rspamd_upstream_restore_cb (gpointer elt, gpointer ls) { struct upstream *up = (struct upstream *)elt; struct upstream_list *ups = (struct upstream_list *)ls; + struct upstream_list_watcher *w; /* Here the upstreams list is already locked */ RSPAMD_UPSTREAM_LOCK (up->lock); @@ -843,6 +908,13 @@ rspamd_upstream_restore_cb (gpointer elt, gpointer ls) g_ptr_array_add (ups->alive, up); up->active_idx = ups->alive->len - 1; RSPAMD_UPSTREAM_UNLOCK (up->lock); + + DL_FOREACH (up->ls->watchers, w) { + if (w->events_mask & RSPAMD_UPSTREAM_WATCH_ONLINE) { + w->func (up, RSPAMD_UPSTREAM_WATCH_ONLINE, up->errors, w->ud); + } + } + /* For revive event */ REF_RELEASE (up); } @@ -1070,3 +1142,58 @@ rspamd_upstreams_foreach (struct upstream_list *ups, cb (up, i, ud); } } + +void +rspamd_upstreams_set_limits (struct upstream_list *ups, + gdouble revive_time, + gdouble revive_jitter, + gdouble error_time, + gdouble dns_timeout, + guint max_errors, + guint dns_retransmits) +{ + g_assert (ups != NULL); + + if (!isnan (revive_time)) { + ups->limits.revive_time = revive_time; + } + + if (!isnan (revive_jitter)) { + ups->limits.revive_jitter = revive_jitter; + } + + if (!isnan (error_time)) { + ups->limits.error_time = error_time; + } + + if (!isnan (dns_timeout)) { + ups->limits.dns_timeout = dns_timeout; + } + + if (max_errors > 0) { + ups->limits.max_errors = max_errors; + } + + if (dns_retransmits > 0) { + ups->limits.dns_retransmits = dns_retransmits; + } +} + +void rspamd_upstreams_add_watch_callback (struct upstream_list *ups, + enum rspamd_upstreams_watch_event events, + rspamd_upstream_watch_func func, + GFreeFunc dtor, + gpointer ud) +{ + struct upstream_list_watcher *nw; + + g_assert ((events & RSPAMD_UPSTREAM_WATCH_ALL) != 0); + + nw = g_malloc (sizeof (*nw)); + nw->func = func; + nw->events_mask = events; + nw->ud = ud; + nw->dtor = dtor; + + DL_APPEND (ups->watchers, nw); +} diff --git a/src/libutil/upstream.h b/src/libutil/upstream.h index 3bc2132da..5c0c92afc 100644 --- a/src/libutil/upstream.h +++ b/src/libutil/upstream.h @@ -83,6 +83,24 @@ void rspamd_upstreams_set_flags (struct upstream_list *ups, enum rspamd_upstream_flag flags); /** + * Sets custom limits for upstreams + * @param ups + * @param revive_time + * @param revive_jitter + * @param error_time + * @param dns_timeout + * @param max_errors + * @param dns_retransmits + */ +void rspamd_upstreams_set_limits (struct upstream_list *ups, + gdouble revive_time, + gdouble revive_jitter, + gdouble error_time, + gdouble dns_timeout, + guint max_errors, + guint dns_retransmits); + +/** * Sets rotation policy for upstreams list * @param ups * @param rot @@ -163,6 +181,32 @@ typedef void (*rspamd_upstream_traverse_func) (struct upstream *up, guint idx, void rspamd_upstreams_foreach (struct upstream_list *ups, rspamd_upstream_traverse_func cb, void *ud); +enum rspamd_upstreams_watch_event { + RSPAMD_UPSTREAM_WATCH_SUCCESS = 1u << 0, + RSPAMD_UPSTREAM_WATCH_FAILURE = 1u << 1, + RSPAMD_UPSTREAM_WATCH_OFFLINE = 1u << 2, + RSPAMD_UPSTREAM_WATCH_ONLINE = 1u << 3, + RSPAMD_UPSTREAM_WATCH_ALL = (1u << 0) | (1u << 1) | (1u << 2) | (1u << 3), +}; + +typedef void (*rspamd_upstream_watch_func) (struct upstream *up, + enum rspamd_upstreams_watch_event event, + guint cur_errors, + void *ud); + +/** + * Adds new watcher to the upstreams list + * @param ups + * @param events + * @param func + * @param ud + */ +void rspamd_upstreams_add_watch_callback (struct upstream_list *ups, + enum rspamd_upstreams_watch_event events, + rspamd_upstream_watch_func func, + GFreeFunc free_func, + gpointer ud); + /** * Returns the current IP address of the upstream * @param up diff --git a/src/libutil/util.c b/src/libutil/util.c index b17766144..68fbd5443 100644 --- a/src/libutil/util.c +++ b/src/libutil/util.c @@ -1794,6 +1794,16 @@ restart: #endif } +#ifdef HAVE_CLOCK_GETTIME +# ifdef CLOCK_MONOTONIC_COARSE +# define RSPAMD_FAST_MONOTONIC_CLOCK CLOCK_MONOTONIC_COARSE +# elif defined(CLOCK_MONOTONIC_FAST) +# define RSPAMD_FAST_MONOTONIC_CLOCK CLOCK_MONOTONIC_FAST +# else +# define RSPAMD_FAST_MONOTONIC_CLOCK CLOCK_MONOTONIC +# endif +#endif + gdouble rspamd_get_ticks (gboolean rdtsc_ok) { @@ -1814,7 +1824,7 @@ rspamd_get_ticks (gboolean rdtsc_ok) #endif #ifdef HAVE_CLOCK_GETTIME struct timespec ts; - gint clk_id = CLOCK_MONOTONIC; + gint clk_id = RSPAMD_FAST_MONOTONIC_CLOCK; clock_gettime (clk_id, &ts); @@ -2124,8 +2134,35 @@ rspamd_init_libs (void) rlim.rlim_max = rlim.rlim_cur; setrlimit (RLIMIT_STACK, &rlim); - ctx->libmagic = magic_open (MAGIC_MIME|MAGIC_NO_CHECK_COMPRESS| - MAGIC_NO_CHECK_ELF|MAGIC_NO_CHECK_TAR); + gint magic_flags = 0; + + /* Unless trusty and other crap is supported... */ +#if 0 +#ifdef MAGIC_NO_CHECK_BUILTIN + magic_flags = MAGIC_NO_CHECK_BUILTIN; +#endif +#endif + magic_flags |= MAGIC_MIME|MAGIC_NO_CHECK_COMPRESS| + MAGIC_NO_CHECK_ELF|MAGIC_NO_CHECK_TAR; +#ifdef MAGIC_NO_CHECK_CDF + magic_flags |= MAGIC_NO_CHECK_CDF; +#endif +#ifdef MAGIC_NO_CHECK_ENCODING + magic_flags |= MAGIC_NO_CHECK_ENCODING; +#endif +#ifdef MAGIC_NO_CHECK_TAR + magic_flags |= MAGIC_NO_CHECK_TAR; +#endif +#ifdef MAGIC_NO_CHECK_TEXT + magic_flags |= MAGIC_NO_CHECK_TEXT; +#endif +#ifdef MAGIC_NO_CHECK_TOKENS + magic_flags |= MAGIC_NO_CHECK_TOKENS; +#endif +#ifdef MAGIC_NO_CHECK_JSON + magic_flags |= MAGIC_NO_CHECK_JSON; +#endif + ctx->libmagic = magic_open (magic_flags); ctx->local_addrs = rspamd_inet_library_init (); REF_INIT_RETAIN (ctx, rspamd_deinit_libs); diff --git a/src/lua/lua_common.c b/src/lua/lua_common.c index ab344e484..01d5dc869 100644 --- a/src/lua/lua_common.c +++ b/src/lua/lua_common.c @@ -21,9 +21,12 @@ #include "ottery.h" #include "rspamd_control.h" #include "lua_thread_pool.h" +#include "libstat/stat_api.h" +#include "libserver/rspamd_control.h" + #include <math.h> #include <sys/wait.h> -#include <src/libserver/rspamd_control.h> + /* Lua module init function */ #define MODULE_INIT_FUNC "module_init" @@ -2343,3 +2346,184 @@ rspamd_lua_require_function (lua_State *L, const gchar *modname, return FALSE; } + + +gboolean +rspamd_lua_try_load_redis (lua_State *L, const ucl_object_t *obj, + struct rspamd_config *cfg, gint *ref_id) +{ + gint res_pos, err_idx; + struct rspamd_config **pcfg; + + /* Create results table */ + lua_createtable (L, 0, 0); + res_pos = lua_gettop (L); + lua_pushcfunction (L, &rspamd_lua_traceback); + err_idx = lua_gettop (L); + + /* Obtain function */ + if (!rspamd_lua_require_function (L, "lua_redis", "try_load_redis_servers")) { + msg_err_config ("cannot require lua_redis"); + lua_pop (L, 2); + + return FALSE; + } + + /* Function arguments */ + ucl_object_push_lua (L, obj, false); + pcfg = lua_newuserdata (L, sizeof (*pcfg)); + rspamd_lua_setclass (L, "rspamd{config}", -1); + *pcfg = cfg; + lua_pushvalue (L, res_pos); + + if (lua_pcall (L, 3, 1, err_idx) != 0) { + GString *tb; + + tb = lua_touserdata (L, -1); + msg_err_config ("cannot call lua try_load_redis_servers script: %s", tb->str); + g_string_free (tb, TRUE); + lua_settop (L, 0); + + return FALSE; + } + + if (lua_toboolean (L, -1)) { + if (ref_id) { + /* Ref table */ + lua_pushvalue (L, res_pos); + *ref_id = luaL_ref (L, LUA_REGISTRYINDEX); + lua_settop (L, 0); + } + else { + /* Leave it on the stack */ + lua_settop (L, res_pos); + } + + return TRUE; + } + else { + lua_settop (L, 0); + } + + return FALSE; +} + +void +rspamd_lua_push_full_word (lua_State *L, rspamd_stat_token_t *w) +{ + gint fl_cnt; + + lua_createtable (L, 4, 0); + + if (w->stemmed.len > 0) { + lua_pushlstring (L, w->stemmed.begin, w->stemmed.len); + lua_rawseti (L, -2, 1); + } + else { + lua_pushstring (L, ""); + lua_rawseti (L, -2, 1); + } + + if (w->normalized.len > 0) { + lua_pushlstring (L, w->normalized.begin, w->normalized.len); + lua_rawseti (L, -2, 2); + } + else { + lua_pushstring (L, ""); + lua_rawseti (L, -2, 2); + } + + if (w->original.len > 0) { + lua_pushlstring (L, w->original.begin, w->original.len); + lua_rawseti (L, -2, 3); + } + else { + lua_pushstring (L, ""); + lua_rawseti (L, -2, 3); + } + + /* Flags part */ + fl_cnt = 1; + lua_createtable (L, 4, 0); + + if (w->flags & RSPAMD_STAT_TOKEN_FLAG_NORMALISED) { + lua_pushstring (L, "normalised"); + lua_rawseti (L, -2, fl_cnt ++); + } + if (w->flags & RSPAMD_STAT_TOKEN_FLAG_BROKEN_UNICODE) { + lua_pushstring (L, "broken_unicode"); + lua_rawseti (L, -2, fl_cnt ++); + } + if (w->flags & RSPAMD_STAT_TOKEN_FLAG_UTF) { + lua_pushstring (L, "utf"); + lua_rawseti (L, -2, fl_cnt ++); + } + if (w->flags & RSPAMD_STAT_TOKEN_FLAG_TEXT) { + lua_pushstring (L, "text"); + lua_rawseti (L, -2, fl_cnt ++); + } + if (w->flags & RSPAMD_STAT_TOKEN_FLAG_HEADER) { + lua_pushstring (L, "header"); + lua_rawseti (L, -2, fl_cnt ++); + } + if (w->flags & (RSPAMD_STAT_TOKEN_FLAG_META|RSPAMD_STAT_TOKEN_FLAG_LUA_META)) { + lua_pushstring (L, "meta"); + lua_rawseti (L, -2, fl_cnt ++); + } + if (w->flags & RSPAMD_STAT_TOKEN_FLAG_STOP_WORD) { + lua_pushstring (L, "stop_word"); + lua_rawseti (L, -2, fl_cnt ++); + } + if (w->flags & RSPAMD_STAT_TOKEN_FLAG_INVISIBLE_SPACES) { + lua_pushstring (L, "invisible_spaces"); + lua_rawseti (L, -2, fl_cnt ++); + } + if (w->flags & RSPAMD_STAT_TOKEN_FLAG_STEMMED) { + lua_pushstring (L, "stemmed"); + lua_rawseti (L, -2, fl_cnt ++); + } + + lua_rawseti (L, -2, 4); +} + +gint +rspamd_lua_push_words (lua_State *L, GArray *words, + enum rspamd_lua_words_type how) +{ + rspamd_stat_token_t *w; + guint i, cnt; + + lua_createtable (L, words->len, 0); + + for (i = 0, cnt = 1; i < words->len; i ++) { + w = &g_array_index (words, rspamd_stat_token_t, i); + + switch (how) { + case RSPAMD_LUA_WORDS_STEM: + if (w->stemmed.len > 0) { + lua_pushlstring (L, w->stemmed.begin, w->stemmed.len); + lua_rawseti (L, -2, cnt ++); + } + break; + case RSPAMD_LUA_WORDS_NORM: + if (w->normalized.len > 0) { + lua_pushlstring (L, w->normalized.begin, w->normalized.len); + lua_rawseti (L, -2, cnt ++); + } + break; + case RSPAMD_LUA_WORDS_RAW: + if (w->original.len > 0) { + lua_pushlstring (L, w->original.begin, w->original.len); + lua_rawseti (L, -2, cnt ++); + } + break; + case RSPAMD_LUA_WORDS_FULL: + rspamd_lua_push_full_word (L, w); + /* Push to the resulting vector */ + lua_rawseti (L, -2, cnt ++); + break; + } + } + + return 1; +}
\ No newline at end of file diff --git a/src/lua/lua_common.h b/src/lua/lua_common.h index fccbf5115..4c82be640 100644 --- a/src/lua/lua_common.h +++ b/src/lua/lua_common.h @@ -7,6 +7,7 @@ #include <lua.h> #include <lauxlib.h> #include <lualib.h> +#include <stdbool.h> #include "rspamd.h" #include "ucl.h" @@ -423,6 +424,39 @@ void rspamd_lua_add_ref_dtor (lua_State *L, rspamd_mempool_t *pool, gboolean rspamd_lua_require_function (lua_State *L, const gchar *modname, const gchar *funcname); +/** + * Tries to load redis server definition from ucl object specified + * @param L + * @param obj + * @param cfg + * @return + */ +gboolean rspamd_lua_try_load_redis (lua_State *L, const ucl_object_t *obj, + struct rspamd_config *cfg, gint *ref_id); + +struct rspamd_stat_token_s; +/** + * Pushes a single word into Lua + * @param L + * @param word + */ +void rspamd_lua_push_full_word (lua_State *L, struct rspamd_stat_token_s *word); + +enum rspamd_lua_words_type { + RSPAMD_LUA_WORDS_STEM = 0, + RSPAMD_LUA_WORDS_NORM, + RSPAMD_LUA_WORDS_RAW, + RSPAMD_LUA_WORDS_FULL +}; +/** + * Pushes words (rspamd_stat_token_t) to Lua + * @param L + * @param words + * @param how + */ +gint rspamd_lua_push_words (lua_State *L, GArray *words, + enum rspamd_lua_words_type how); + /* Paths defs */ #define RSPAMD_CONFDIR_INDEX "CONFDIR" #define RSPAMD_LOCAL_CONFDIR_INDEX "LOCAL_CONFDIR" diff --git a/src/lua/lua_config.c b/src/lua/lua_config.c index eee3861bc..1965b4903 100644 --- a/src/lua/lua_config.c +++ b/src/lua/lua_config.c @@ -1055,13 +1055,11 @@ struct lua_callback_data { gint ref; } callback; gboolean cb_is_ref; + + /* Dynamic data */ gint stack_level; gint order; -}; - -struct lua_watcher_data { - struct lua_callback_data *cbd; - gint cb_ref; + struct rspamd_symcache_item *item; }; /* @@ -1093,128 +1091,23 @@ rspamd_compare_order_func (gconstpointer a, gconstpointer b) return cb2->order - cb1->order; } -static void -lua_watcher_callback (gpointer session_data, gpointer ud) -{ - struct rspamd_task *task = session_data, **ptask; - struct lua_watcher_data *wd = ud; - lua_State *L; - gint level, nresults, err_idx, ret; - GString *tb; - struct rspamd_symbol_result *s; - - L = wd->cbd->L; - level = lua_gettop (L); - lua_pushcfunction (L, &rspamd_lua_traceback); - err_idx = lua_gettop (L); +static void lua_metric_symbol_callback_return (struct thread_entry *thread_entry, + int ret); - level ++; - lua_rawgeti (L, LUA_REGISTRYINDEX, wd->cb_ref); - - ptask = lua_newuserdata (L, sizeof (struct rspamd_task *)); - rspamd_lua_setclass (L, "rspamd{task}", -1); - *ptask = task; - - if ((ret = lua_pcall (L, 1, LUA_MULTRET, err_idx)) != 0) { - tb = lua_touserdata (L, -1); - msg_err_task ("call to (%s) failed (%d): %v", - wd->cbd->symbol, ret, tb); - - if (tb) { - g_string_free (tb, TRUE); - } - } - else { - nresults = lua_gettop (L) - level; - - if (nresults >= 1) { - /* Function returned boolean, so maybe we need to insert result? */ - gint res = 0; - gint i; - gdouble flag = 1.0; - gint type; - struct lua_watcher_data *nwd; - - type = lua_type (L, level + 1); - - if (type == LUA_TBOOLEAN) { - res = lua_toboolean (L, level + 1); - } - else if (type == LUA_TFUNCTION) { - /* Function returned a closure that should be watched for */ - nwd = rspamd_mempool_alloc (task->task_pool, sizeof (*nwd)); - lua_pushvalue (L, level + 1); - nwd->cb_ref = luaL_ref (L, LUA_REGISTRYINDEX); - nwd->cbd = wd->cbd; - rspamd_session_watcher_push_callback (task->s, - rspamd_session_get_watcher (task->s), - lua_watcher_callback, nwd); - /* - * We immediately pop watcher since we have not registered - * any async events from here - */ - rspamd_session_watcher_pop (task->s, - rspamd_session_get_watcher (task->s)); - } - else { - res = (gint)lua_tonumber (L, level + 1); - } - - if (res) { - gint first_opt = 2; - - if (lua_type (L, level + 2) == LUA_TNUMBER) { - flag = lua_tonumber (L, level + 2); - /* Shift opt index */ - first_opt = 3; - } - else { - flag = res; - } - - s = rspamd_task_insert_result (task, - wd->cbd->symbol, flag, NULL); - - if (s) { - guint last_pos = lua_gettop (L); - - for (i = level + first_opt; i <= last_pos; i++) { - if (lua_type (L, i) == LUA_TSTRING) { - const char *opt = lua_tostring (L, i); - - rspamd_task_add_result_option (task, s, opt); - } - else if (lua_type (L, i) == LUA_TTABLE) { - lua_pushvalue (L, i); - - for (lua_pushnil (L); lua_next (L, -2); lua_pop (L, 1)) { - const char *opt = lua_tostring (L, -1); - - rspamd_task_add_result_option (task, s, opt); - } - - lua_pop (L, 1); - } - } - } - } - } - } - - lua_settop (L, err_idx - 1); -} - -static void lua_metric_symbol_callback_return (struct thread_entry *thread_entry, int ret); - -static void lua_metric_symbol_callback_error (struct thread_entry *thread_entry, int ret, const char *msg); +static void lua_metric_symbol_callback_error (struct thread_entry *thread_entry, + int ret, + const char *msg); static void -lua_metric_symbol_callback (struct rspamd_task *task, gpointer ud) +lua_metric_symbol_callback (struct rspamd_task *task, + struct rspamd_symcache_item *item, + gpointer ud) { struct lua_callback_data *cd = ud; struct rspamd_task **ptask; struct thread_entry *thread_entry; + rspamd_symcache_item_async_inc (task, item, "lua symbol"); thread_entry = lua_thread_pool_get_for_task (task); g_assert(thread_entry->cd == NULL); @@ -1222,6 +1115,7 @@ lua_metric_symbol_callback (struct rspamd_task *task, gpointer ud) lua_State *thread = thread_entry->lua_state; cd->stack_level = lua_gettop (thread); + cd->item = item; if (cd->cb_is_ref) { lua_rawgeti (thread, LUA_REGISTRYINDEX, cd->callback.ref); @@ -1241,11 +1135,15 @@ lua_metric_symbol_callback (struct rspamd_task *task, gpointer ud) } static void -lua_metric_symbol_callback_error (struct thread_entry *thread_entry, int ret, const char *msg) +lua_metric_symbol_callback_error (struct thread_entry *thread_entry, + int ret, + const char *msg) { struct lua_callback_data *cd = thread_entry->cd; struct rspamd_task *task = thread_entry->task; msg_err_task ("call to (%s) failed (%d): %s", cd->symbol, ret, msg); + + rspamd_symcache_item_async_dec_check (task, cd->item, "lua symbol"); } static void @@ -1268,7 +1166,6 @@ lua_metric_symbol_callback_return (struct thread_entry *thread_entry, int ret) gint i; gdouble flag = 1.0; gint type; - struct lua_watcher_data *wd; type = lua_type (L, cd->stack_level + 1); @@ -1276,20 +1173,7 @@ lua_metric_symbol_callback_return (struct thread_entry *thread_entry, int ret) res = lua_toboolean (L, cd->stack_level + 1); } else if (type == LUA_TFUNCTION) { - /* Function returned a closure that should be watched for */ - wd = rspamd_mempool_alloc (task->task_pool, sizeof (*wd)); - lua_pushvalue (L /*cd->L*/, cd->stack_level + 1); - wd->cb_ref = luaL_ref (L, LUA_REGISTRYINDEX); - wd->cbd = cd; - rspamd_session_watcher_push_callback (task->s, - rspamd_session_get_watcher (task->s), - lua_watcher_callback, wd); - /* - * We immediately pop watcher since we have not registered - * any async events from here - */ - rspamd_session_watcher_pop (task->s, - rspamd_session_get_watcher (task->s)); + g_assert_not_reached (); } else { res = lua_tonumber (L, cd->stack_level + 1); @@ -1340,6 +1224,7 @@ lua_metric_symbol_callback_return (struct thread_entry *thread_entry, int ret) g_assert (lua_gettop (L) == cd->stack_level); /* we properly cleaned up the stack */ cd->stack_level = 0; + rspamd_symcache_item_async_dec_check (task, cd->item, "lua symbol"); } static gint @@ -1361,7 +1246,7 @@ rspamd_register_symbol_fromlua (lua_State *L, priority = 1; } - if ((ret = rspamd_symbols_cache_find_symbol (cfg->cache, name)) != -1) { + if ((ret = rspamd_symcache_find_symbol (cfg->cache, name)) != -1) { if (optional) { msg_debug_config ("duplicate symbol: %s, skip registering", name); @@ -1418,7 +1303,7 @@ rspamd_register_symbol_fromlua (lua_State *L, cd->symbol = rspamd_mempool_strdup (cfg->cfg_pool, name); } - ret = rspamd_symbols_cache_add_symbol (cfg->cache, + ret = rspamd_symcache_add_symbol (cfg->cache, name, priority, lua_metric_symbol_callback, @@ -1442,7 +1327,7 @@ rspamd_register_symbol_fromlua (lua_State *L, cd->symbol = rspamd_mempool_strdup (cfg->cfg_pool, name); } - ret = rspamd_symbols_cache_add_symbol (cfg->cache, + ret = rspamd_symcache_add_symbol (cfg->cache, name, priority, lua_metric_symbol_callback, @@ -1458,7 +1343,7 @@ rspamd_register_symbol_fromlua (lua_State *L, lua_settop (L, err_idx - 1); } else { - ret = rspamd_symbols_cache_add_symbol (cfg->cache, + ret = rspamd_symcache_add_symbol (cfg->cache, name, priority, NULL, @@ -1835,7 +1720,7 @@ lua_config_register_symbols (lua_State *L) while (lua_next (L, -2)) { lua_pushvalue (L, -2); sym = luaL_checkstring (L, -2); - rspamd_symbols_cache_add_symbol (cfg->cache, sym, + rspamd_symcache_add_symbol (cfg->cache, sym, 0, NULL, NULL, SYMBOL_TYPE_VIRTUAL, ret); lua_pop (L, 2); @@ -1844,7 +1729,7 @@ lua_config_register_symbols (lua_State *L) } else if (lua_type (L, i) == LUA_TSTRING) { sym = luaL_checkstring (L, i); - rspamd_symbols_cache_add_symbol (cfg->cache, sym, + rspamd_symcache_add_symbol (cfg->cache, sym, 0, NULL, NULL, SYMBOL_TYPE_VIRTUAL, ret); } @@ -1874,7 +1759,7 @@ lua_config_register_virtual_symbol (lua_State * L) } if (name) { - ret = rspamd_symbols_cache_add_symbol (cfg->cache, name, + ret = rspamd_symcache_add_symbol (cfg->cache, name, weight > 0 ? 0 : -1, NULL, NULL, SYMBOL_TYPE_VIRTUAL, parent); } @@ -2035,9 +1920,9 @@ lua_config_register_dependency (lua_State * L) if (child_id > 0 && parent != NULL) { if (skip_squeeze || !rspamd_lua_squeeze_dependency (L, cfg, - rspamd_symbols_cache_symbol_by_id (cfg->cache, child_id), + rspamd_symcache_symbol_by_id (cfg->cache, child_id), parent)) { - rspamd_symbols_cache_add_dependency (cfg->cache, child_id, parent); + rspamd_symcache_add_dependency (cfg->cache, child_id, parent); } } } @@ -2052,7 +1937,7 @@ lua_config_register_dependency (lua_State * L) if (child != NULL && parent != NULL) { if (skip_squeeze || !rspamd_lua_squeeze_dependency (L, cfg, child, parent)) { - rspamd_symbols_cache_add_delayed_dependency (cfg->cache, child, + rspamd_symcache_add_delayed_dependency (cfg->cache, child, parent); } @@ -2348,7 +2233,7 @@ lua_config_add_composite (lua_State * L) composite); if (new) { - rspamd_symbols_cache_add_symbol (cfg->cache, name, + rspamd_symcache_add_symbol (cfg->cache, name, 0, NULL, NULL, SYMBOL_TYPE_COMPOSITE, -1); } @@ -2494,7 +2379,9 @@ lua_config_newindex (lua_State *L) /* Here we pop function from the stack, so no lua_pop is required */ condref = luaL_ref (L, LUA_REGISTRYINDEX); - rspamd_symbols_cache_add_condition (cfg->cache, id, L, condref); + g_assert (name != NULL); + rspamd_symcache_add_condition_delayed (cfg->cache, + name, L, condref); } else { lua_pop (L, 1); @@ -2609,7 +2496,7 @@ lua_config_add_condition (lua_State *L) lua_pushvalue (L, 3); condref = luaL_ref (L, LUA_REGISTRYINDEX); - ret = rspamd_symbols_cache_add_condition_delayed (cfg->cache, sym, L, + ret = rspamd_symcache_add_condition_delayed (cfg->cache, sym, L, condref); if (!ret) { @@ -2631,7 +2518,7 @@ lua_config_set_peak_cb (lua_State *L) if (cfg && lua_type (L, 2) == LUA_TFUNCTION) { lua_pushvalue (L, 2); condref = luaL_ref (L, LUA_REGISTRYINDEX); - rspamd_symbols_cache_set_peak_callback (cfg->cache, + rspamd_symcache_set_peak_callback (cfg->cache, condref); } @@ -2646,7 +2533,7 @@ lua_config_enable_symbol (lua_State *L) const gchar *sym = luaL_checkstring (L, 2); if (cfg && sym) { - rspamd_symbols_cache_enable_symbol (cfg->cache, sym); + rspamd_symcache_enable_symbol_perm (cfg->cache, sym); } else { return luaL_error (L, "invalid arguments"); @@ -2663,7 +2550,7 @@ lua_config_disable_symbol (lua_State *L) const gchar *sym = luaL_checkstring (L, 2); if (cfg && sym) { - rspamd_symbols_cache_disable_symbol (cfg->cache, sym); + rspamd_symcache_disable_symbol_perm (cfg->cache, sym); } else { return luaL_error (L, "invalid arguments"); @@ -2978,7 +2865,7 @@ lua_config_get_symbols_count (lua_State *L) guint res = 0; if (cfg != NULL) { - res = rspamd_symbols_cache_stats_symbols_count (cfg->cache); + res = rspamd_symcache_stats_symbols_count (cfg->cache); } else { return luaL_error (L, "invalid arguments"); @@ -2997,7 +2884,7 @@ lua_config_get_symbols_cksum (lua_State *L) guint64 res = 0, *pres; if (cfg != NULL) { - res = rspamd_symbols_cache_get_cksum (cfg->cache); + res = rspamd_symcache_get_cksum (cfg->cache); } else { return luaL_error (L, "invalid arguments"); @@ -3018,7 +2905,7 @@ lua_config_get_symbols_counters (lua_State *L) ucl_object_t *counters; if (cfg != NULL) { - counters = rspamd_symbols_cache_counters (cfg->cache); + counters = rspamd_symcache_counters (cfg->cache); ucl_object_push_lua (L, counters, true); ucl_object_unref (counters); } @@ -3078,7 +2965,7 @@ lua_config_get_symbol_callback (lua_State *L) struct lua_callback_data *cbd; if (cfg != NULL && sym != NULL) { - abs_cbdata = rspamd_symbols_cache_get_cbdata (cfg->cache, sym); + abs_cbdata = rspamd_symcache_get_cbdata (cfg->cache, sym); if (abs_cbdata == NULL || abs_cbdata->magic != rspamd_lua_callback_magic) { lua_pushnil (L); @@ -3111,7 +2998,7 @@ lua_config_set_symbol_callback (lua_State *L) struct lua_callback_data *cbd; if (cfg != NULL && sym != NULL && lua_type (L, 3) == LUA_TFUNCTION) { - abs_cbdata = rspamd_symbols_cache_get_cbdata (cfg->cache, sym); + abs_cbdata = rspamd_symcache_get_cbdata (cfg->cache, sym); if (abs_cbdata == NULL || abs_cbdata->magic != rspamd_lua_callback_magic) { lua_pushboolean (L, FALSE); @@ -3148,7 +3035,7 @@ lua_config_get_symbol_stat (lua_State *L) guint hits; if (cfg != NULL && sym != NULL) { - if (!rspamd_symbols_cache_stat_symbol (cfg->cache, sym, &freq, + if (!rspamd_symcache_stat_symbol (cfg->cache, sym, &freq, &stddev, &tm, &hits)) { lua_pushnil (L); } diff --git a/src/lua/lua_cryptobox.c b/src/lua/lua_cryptobox.c index cda4912de..70f29fa91 100644 --- a/src/lua/lua_cryptobox.c +++ b/src/lua/lua_cryptobox.c @@ -26,11 +26,13 @@ * print(h:hex()) */ + #include "lua_common.h" #include "libcryptobox/cryptobox.h" #include "libcryptobox/keypair.h" #include "libcryptobox/keypair_private.h" #include "unix-std.h" +#include "contrib/libottery/ottery.h" struct rspamd_lua_cryptobox_hash { rspamd_cryptobox_hash_state_t *h; @@ -75,6 +77,8 @@ LUA_FUNCTION_DEF (cryptobox, encrypt_memory); LUA_FUNCTION_DEF (cryptobox, encrypt_file); LUA_FUNCTION_DEF (cryptobox, decrypt_memory); LUA_FUNCTION_DEF (cryptobox, decrypt_file); +LUA_FUNCTION_DEF (cryptobox, encrypt_cookie); +LUA_FUNCTION_DEF (cryptobox, decrypt_cookie); static const struct luaL_reg cryptoboxlib_f[] = { LUA_INTERFACE_DEF (cryptobox, verify_memory), @@ -85,6 +89,8 @@ static const struct luaL_reg cryptoboxlib_f[] = { LUA_INTERFACE_DEF (cryptobox, encrypt_file), LUA_INTERFACE_DEF (cryptobox, decrypt_memory), LUA_INTERFACE_DEF (cryptobox, decrypt_file), + LUA_INTERFACE_DEF (cryptobox, encrypt_cookie), + LUA_INTERFACE_DEF (cryptobox, decrypt_cookie), {NULL, NULL} }; @@ -1822,6 +1828,198 @@ lua_cryptobox_decrypt_file (lua_State *L) return 2; } +#define RSPAMD_CRYPTOBOX_AES_BLOCKSIZE 16 +#define RSPAMD_CRYPTOBOX_AES_KEYSIZE 16 + +/*** + * @function rspamd_cryptobox.encrypt_cookie(secret_key, secret_cookie) + * Specialised function that performs AES-CTR encryption of the provided cookie + * ``` + * e := base64(nonce||aesencrypt(nonce, secret_cookie)) + * nonce := uint32_le(unix_timestamp)||random_64bit + * aesencrypt := aes_ctr(nonce, secret_key) ^ pad(secret_cookie) + * pad := secret_cookie || 0^(32-len(secret_cookie)) + * ``` + * @param {string} secret_key secret key as a hex string (must be 16 bytes in raw or 32 in hex) + * @param {string} secret_cookie secret cookie as a string for up to 31 character + * @return {string} e function value for this sk and cookie + */ +static gint +lua_cryptobox_encrypt_cookie (lua_State *L) +{ + guchar aes_block[RSPAMD_CRYPTOBOX_AES_BLOCKSIZE], *blk; + guchar padded_cookie[RSPAMD_CRYPTOBOX_AES_BLOCKSIZE]; + guchar nonce[RSPAMD_CRYPTOBOX_AES_BLOCKSIZE]; + guchar aes_key[RSPAMD_CRYPTOBOX_AES_KEYSIZE]; + guchar result[RSPAMD_CRYPTOBOX_AES_BLOCKSIZE * 2]; + guint32 ts; + + const gchar *sk, *cookie; + gsize sklen, cookie_len; + gint bklen; + + sk = lua_tolstring (L, 1, &sklen); + cookie = lua_tolstring (L, 2, &cookie_len); + + if (sk && cookie) { + if (sklen == 32) { + /* Hex */ + rspamd_decode_hex_buf (sk, sklen, aes_key, sizeof (aes_key)); + } + else if (sklen == RSPAMD_CRYPTOBOX_AES_KEYSIZE) { + /* Raw */ + memcpy (aes_key, sk, sizeof (aes_key)); + } + else { + return luaL_error (L, "invalid keysize %d", (gint)sklen); + } + + if (cookie_len > sizeof (padded_cookie) - 1) { + return luaL_error (L, "cookie is too long %d", (gint)padded_cookie); + } + + /* Fill nonce */ + ottery_rand_bytes (nonce, sizeof (guint64) + sizeof (guint32)); + ts = (guint32)rspamd_get_calendar_ticks (); + ts = GUINT32_TO_LE (ts); + memcpy (nonce + sizeof (guint64) + sizeof (guint32), &ts, sizeof (ts)); + + /* Prepare padded cookie */ + memset (padded_cookie, 0, sizeof (padded_cookie)); + memcpy (padded_cookie, cookie, cookie_len); + + /* Perform AES CTR via AES ECB on nonce */ + EVP_CIPHER_CTX *ctx; + ctx = EVP_CIPHER_CTX_new (); + EVP_EncryptInit_ex (ctx, EVP_aes_128_ecb (), NULL, aes_key, NULL); + EVP_CIPHER_CTX_set_padding (ctx, 0); + + bklen = sizeof (aes_block); + blk = aes_block; + g_assert (EVP_EncryptUpdate (ctx, blk, &bklen, nonce, sizeof (nonce))); + blk += bklen; + g_assert (EVP_EncryptFinal_ex(ctx, blk, &bklen)); + EVP_CIPHER_CTX_free (ctx); + + /* Encode result */ + memcpy (result, nonce, sizeof (nonce)); + for (guint i = 0; i < sizeof (aes_block); i ++) { + result[i + sizeof (nonce)] = padded_cookie[i] ^ aes_block[i]; + } + + gsize rlen; + gchar *res = rspamd_encode_base64 (result, sizeof (result), + 0, &rlen); + + lua_pushlstring (L, res, rlen); + g_free (res); + rspamd_explicit_memzero (aes_key, sizeof (aes_key)); + rspamd_explicit_memzero (aes_block, sizeof (aes_block)); + } + else { + return luaL_error (L, "invalid arguments"); + } + + return 1; +} + +/*** + * @function rspamd_cryptobox.decrypt_cookie(secret_key, encrypted_cookie) + * Specialised function that performs AES-CTR decryption of the provided cookie in form + * ``` + * e := base64(nonce||aesencrypt(nonce, secret_cookie)) + * nonce := int32_le(unix_timestamp)||random_96bit + * aesencrypt := aes_ctr(nonce, secret_key) ^ pad(secret_cookie) + * pad := secret_cookie || 0^(32-len(secret_cookie)) + * ``` + * @param {string} secret_key secret key as a hex string (must be 16 bytes in raw or 32 in hex) + * @param {string} encrypted_cookie encrypted cookie as a base64 encoded string + * @return {string+number} decrypted value of the cookie and the cookie timestamp + */ +static gint +lua_cryptobox_decrypt_cookie (lua_State *L) +{ + guchar *blk; + guchar nonce[RSPAMD_CRYPTOBOX_AES_BLOCKSIZE]; + guchar aes_key[RSPAMD_CRYPTOBOX_AES_KEYSIZE]; + guchar *src; + guint32 ts; + + const gchar *sk, *cookie; + gsize sklen, cookie_len; + gint bklen; + + sk = lua_tolstring (L, 1, &sklen); + cookie = lua_tolstring (L, 2, &cookie_len); + + if (sk && cookie) { + if (sklen == 32) { + /* Hex */ + rspamd_decode_hex_buf (sk, sklen, aes_key, sizeof (aes_key)); + } + else if (sklen == RSPAMD_CRYPTOBOX_AES_KEYSIZE) { + /* Raw */ + memcpy (aes_key, sk, sizeof (aes_key)); + } + else { + return luaL_error (L, "invalid keysize %d", (gint)sklen); + } + + src = g_malloc (cookie_len); + + rspamd_cryptobox_base64_decode (cookie, cookie_len, src, &cookie_len); + + if (cookie_len != RSPAMD_CRYPTOBOX_AES_BLOCKSIZE * 2) { + g_free (src); + lua_pushnil (L); + + return 1; + } + + /* Perform AES CTR via AES ECB on nonce */ + EVP_CIPHER_CTX *ctx; + ctx = EVP_CIPHER_CTX_new (); + /* As per CTR definition, we use encrypt for both encrypt and decrypt */ + EVP_EncryptInit_ex (ctx, EVP_aes_128_ecb (), NULL, aes_key, NULL); + EVP_CIPHER_CTX_set_padding (ctx, 0); + + /* Copy time */ + memcpy (&ts, src + sizeof (guint64) + sizeof (guint32), sizeof (ts)); + ts = GUINT32_FROM_LE (ts); + bklen = sizeof (nonce); + blk = nonce; + g_assert (EVP_EncryptUpdate (ctx, blk, &bklen, src, + RSPAMD_CRYPTOBOX_AES_BLOCKSIZE)); + blk += bklen; + g_assert (EVP_EncryptFinal_ex (ctx, blk, &bklen)); + EVP_CIPHER_CTX_free (ctx); + + /* Decode result */ + for (guint i = 0; i < RSPAMD_CRYPTOBOX_AES_BLOCKSIZE; i ++) { + src[i + sizeof (nonce)] ^= nonce[i]; + } + + if (src[RSPAMD_CRYPTOBOX_AES_BLOCKSIZE * 2 - 1] != '\0') { + /* Bad cookie */ + lua_pushnil (L); + lua_pushnil (L); + } + else { + lua_pushstring (L, src + sizeof (nonce)); + lua_pushnumber (L, ts); + } + + rspamd_explicit_memzero (src, RSPAMD_CRYPTOBOX_AES_BLOCKSIZE * 2); + g_free (src); + rspamd_explicit_memzero (aes_key, sizeof (aes_key)); + } + else { + return luaL_error (L, "invalid arguments"); + } + + return 2; +} + static gint lua_load_pubkey (lua_State * L) { diff --git a/src/lua/lua_dns.c b/src/lua/lua_dns.c index fe4b7ff2e..7b07f4003 100644 --- a/src/lua/lua_dns.c +++ b/src/lua/lua_dns.c @@ -23,14 +23,15 @@ static const struct luaL_reg dns_f[] = { {NULL, NULL} }; -void -lua_dns_callback (struct rdns_reply *reply, void *arg); +static const gchar *M = "rspamd lua dns"; + +void lua_dns_callback (struct rdns_reply *reply, void *arg); struct lua_rspamd_dns_cbdata { struct thread_entry *thread; struct rspamd_task *task; struct rspamd_dns_resolver *resolver; - struct rspamd_async_watcher *w; + struct rspamd_symcache_item *item; struct rspamd_async_session *s; }; @@ -50,8 +51,13 @@ lua_dns_request (lua_State *L) /* Check arguments */ if (!rspamd_lua_parse_table_arguments (L, 1, &err, - "*name=S;task=U{task};*type=S;forced=B;session=U{session};config=U{config}", - &to_resolve, &task, &type_str, &forced, &session, &cfg)) { + "*name=S;task=U{task};*type=S;forced=B;session=U{session};config=U{config}", + &to_resolve, + &task, + &type_str, + &forced, + &session, + &cfg)) { if (err) { ret = luaL_error (L, "invalid arguments: %s", err->message); @@ -104,36 +110,40 @@ lua_dns_request (lua_State *L) if (task == NULL) { - ret = make_dns_request (cfg->dns_resolver, + ret = (make_dns_request (cfg->dns_resolver, session, pool, lua_dns_callback, cbdata, type, - to_resolve); + to_resolve) != NULL); } else { - if (forced) { + if (forced) { ret = make_dns_request_task_forced (task, - lua_dns_callback, - cbdata, - type, - to_resolve); + lua_dns_callback, + cbdata, + type, + to_resolve); } else { ret = make_dns_request_task (task, - lua_dns_callback, - cbdata, - type, - to_resolve); + lua_dns_callback, + cbdata, + type, + to_resolve); } } if (ret) { cbdata->thread = lua_thread_pool_get_running_entry (cfg->lua_thread_pool); cbdata->s = session; - cbdata->w = rspamd_session_get_watcher (session); - rspamd_session_watcher_push (session); + + if (task) { + cbdata->item = rspamd_symcache_get_cur_item (task); + rspamd_symcache_item_async_inc (task, cbdata->item, M); + } + return lua_thread_yield (cbdata->thread, 0); } else { @@ -166,8 +176,8 @@ lua_dns_callback (struct rdns_reply *reply, void *arg) lua_thread_resume (cbdata->thread, 2); - if (cbdata->s) { - rspamd_session_watcher_pop (cbdata->s, cbdata->w); + if (cbdata->item) { + rspamd_symcache_item_async_dec_check (cbdata->task, cbdata->item, M); } } diff --git a/src/lua/lua_dns_resolver.c b/src/lua/lua_dns_resolver.c index e7b6eeaab..ffc4cd738 100644 --- a/src/lua/lua_dns_resolver.c +++ b/src/lua/lua_dns_resolver.c @@ -37,10 +37,12 @@ local function symbol_callback(task) end end - task:get_resolver():resolve_a(task:get_session(), task:get_mempool(), - host, dns_cb) + task:get_resolver():resolve_a({task = task, name = host, callback = dns_cb}) end */ + +static const gchar *M = "rspamd lua dns resolver"; + struct rspamd_dns_resolver * lua_check_dns_resolver (lua_State * L); void luaopen_dns_resolver (lua_State * L); @@ -86,7 +88,7 @@ struct lua_dns_cbdata { gint cbref; gchar *to_resolve; gchar *user_str; - struct rspamd_async_watcher *w; + struct rspamd_symcache_item *item; struct rspamd_async_session *s; }; @@ -184,6 +186,11 @@ lua_dns_resolver_callback (struct rdns_reply *reply, gpointer arg) lua_pushboolean (L, reply->authenticated); + if (cd->item) { + /* We also need to restore the item in case there are some chains */ + rspamd_symcache_set_cur_item (cd->task, cd->item); + } + if (lua_pcall (L, 6, 0, err_idx) != 0) { tb = lua_touserdata (L, -1); @@ -199,8 +206,8 @@ lua_dns_resolver_callback (struct rdns_reply *reply, gpointer arg) luaL_unref (L, LUA_REGISTRYINDEX, cd->cbref); lua_thread_pool_restore_callback (&cbs); - if (cd->s) { - rspamd_session_watcher_pop (cd->s, cd->w); + if (cd->item) { + rspamd_symcache_item_async_dec_check (cd->task, cd->item, M); } if (!cd->pool) { @@ -345,6 +352,7 @@ lua_dns_resolver_resolve_common (lua_State *L, struct rspamd_task *task = NULL; GError *err = NULL; gboolean forced = FALSE; + struct rspamd_symcache_item *item = NULL; /* Check arguments */ if (!rspamd_lua_parse_table_arguments (L, first, &err, @@ -365,6 +373,7 @@ lua_dns_resolver_resolve_common (lua_State *L, if (task) { pool = task->task_pool; session = task->s; + item = rspamd_symcache_get_cur_item (task); } if (to_resolve != NULL) { @@ -432,8 +441,6 @@ lua_dns_resolver_resolve_common (lua_State *L, if (session) { cbdata->s = session; - cbdata->w = rspamd_session_get_watcher (session); - rspamd_session_watcher_push (session); } } else { @@ -441,6 +448,13 @@ lua_dns_resolver_resolve_common (lua_State *L, } } else { + /* Fail-safety as this function can, in theory, call + * lua_dns_resolver_callback without switching to the event loop + */ + if (item) { + rspamd_symcache_item_async_inc (task, item, M); + } + if (forced) { ret = make_dns_request_task_forced (task, lua_dns_resolver_callback, @@ -457,13 +471,22 @@ lua_dns_resolver_resolve_common (lua_State *L, if (ret) { cbdata->s = session; - cbdata->w = rspamd_session_get_watcher (session); - rspamd_session_watcher_push (session); + + + if (item) { + cbdata->item = item; + rspamd_symcache_item_async_inc (task, item, M); + } /* callback was set up */ lua_pushboolean (L, TRUE); - } else { + } + else { lua_pushnil (L); } + + if (item) { + rspamd_symcache_item_async_dec_check (task, item, M); + } } } else { @@ -484,12 +507,15 @@ err: } /*** - * @method resolver:resolve_a(session, pool, host, callback) + * @method resolver:resolve_a(table) * Resolve A record for a specified host. - * @param {async_session} session asynchronous session normally associated with rspamd task (`task:get_session()`) - * @param {mempool} pool memory pool for storing intermediate data - * @param {string} host name to resolve - * @param {function} callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)` + * Table elements: + * * `task` - task element (preferred, required to track dependencies) -or- + * * `session` - asynchronous session normally associated with rspamd task (`task:get_session()`) + * * `mempool` - pool memory pool for storing intermediate data + * * `name` - host name to resolve + * * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)` + * * `forced` - true if needed to override notmal limit for DNS requests * @return {boolean} `true` if DNS request has been scheduled */ static int @@ -511,12 +537,15 @@ lua_dns_resolver_resolve_a (lua_State *L) } /*** - * @method resolver:resolve_ptr(session, pool, ip, callback) + * @method resolver:resolve_ptr(table) * Resolve PTR record for a specified host. - * @param {async_session} session asynchronous session normally associated with rspamd task (`task:get_session()`) - * @param {mempool} pool memory pool for storing intermediate data - * @param {string} ip name to resolve in string form (e.g. '8.8.8.8' or '2001:dead::') - * @param {function} callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)` + * Table elements: + * * `task` - task element (preferred, required to track dependencies) -or- + * * `session` - asynchronous session normally associated with rspamd task (`task:get_session()`) + * * `mempool` - pool memory pool for storing intermediate data + * * `name` - host name to resolve + * * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)` + * * `forced` - true if needed to override notmal limit for DNS requests * @return {boolean} `true` if DNS request has been scheduled */ static int @@ -538,12 +567,15 @@ lua_dns_resolver_resolve_ptr (lua_State *L) } /*** - * @method resolver:resolve_txt(session, pool, host, callback) + * @method resolver:resolve_txt(table) * Resolve TXT record for a specified host. - * @param {async_session} session asynchronous session normally associated with rspamd task (`task:get_session()`) - * @param {mempool} pool memory pool for storing intermediate data - * @param {string} host name to get TXT record for - * @param {function} callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)` + * Table elements: + * * `task` - task element (preferred, required to track dependencies) -or- + * * `session` - asynchronous session normally associated with rspamd task (`task:get_session()`) + * * `mempool` - pool memory pool for storing intermediate data + * * `name` - host name to resolve + * * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)` + * * `forced` - true if needed to override notmal limit for DNS requests * @return {boolean} `true` if DNS request has been scheduled */ static int @@ -565,12 +597,15 @@ lua_dns_resolver_resolve_txt (lua_State *L) } /*** - * @method resolver:resolve_mx(session, pool, host, callback) + * @method resolver:resolve_mx(table) * Resolve MX record for a specified host. - * @param {async_session} session asynchronous session normally associated with rspamd task (`task:get_session()`) - * @param {mempool} pool memory pool for storing intermediate data - * @param {string} host name to get MX record for - * @param {function} callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)` + * Table elements: + * * `task` - task element (preferred, required to track dependencies) -or- + * * `session` - asynchronous session normally associated with rspamd task (`task:get_session()`) + * * `mempool` - pool memory pool for storing intermediate data + * * `name` - host name to resolve + * * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)` + * * `forced` - true if needed to override notmal limit for DNS requests * @return {boolean} `true` if DNS request has been scheduled */ static int @@ -592,12 +627,15 @@ lua_dns_resolver_resolve_mx (lua_State *L) } /*** - * @method resolver:resolve_ns(session, pool, host, callback) + * @method resolver:resolve_ns(table) * Resolve NS records for a specified host. - * @param {async_session} session asynchronous session normally associated with rspamd task (`task:get_session()`) - * @param {mempool} pool memory pool for storing intermediate data - * @param {string} host name to get NS records for - * @param {function} callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)` + * Table elements: + * * `task` - task element (preferred, required to track dependencies) -or- + * * `session` - asynchronous session normally associated with rspamd task (`task:get_session()`) + * * `mempool` - pool memory pool for storing intermediate data + * * `name` - host name to resolve + * * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)` + * * `forced` - true if needed to override notmal limit for DNS requests * @return {boolean} `true` if DNS request has been scheduled */ static int diff --git a/src/lua/lua_http.c b/src/lua/lua_http.c index 3f3ee72e0..2ca01d8a5 100644 --- a/src/lua/lua_http.c +++ b/src/lua/lua_http.c @@ -45,6 +45,8 @@ local function symbol_callback(task) #define MAX_HEADERS_SIZE 8192 +static const gchar *M = "rspamd lua http"; + LUA_FUNCTION_DEF (http, request); static const struct luaL_reg httplib_m[] = { @@ -60,10 +62,11 @@ static const struct luaL_reg httplib_m[] = { struct lua_http_cbdata { struct rspamd_http_connection *conn; struct rspamd_async_session *session; - struct rspamd_async_watcher *w; + struct rspamd_symcache_item *item; struct rspamd_http_message *msg; struct event_base *ev_base; struct rspamd_config *cfg; + struct rspamd_task *task; struct timeval tv; struct rspamd_cryptobox_keypair *local_kp; struct rspamd_cryptobox_pubkey *peer_pk; @@ -146,13 +149,13 @@ static void lua_http_maybe_free (struct lua_http_cbdata *cbd) { if (cbd->session) { - if (cbd->w) { - /* We still need to clear watcher */ - rspamd_session_watcher_pop (cbd->session, cbd->w); - } if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_RESOLVED) { /* Event is added merely for resolved events */ + if (cbd->item) { + rspamd_symcache_item_async_dec_check (cbd->task, cbd->item, M); + } + rspamd_session_remove_event (cbd->session, lua_http_fin, cbd); } } @@ -174,6 +177,11 @@ lua_http_push_error (struct lua_http_cbdata *cbd, const char *err) lua_rawgeti (L, LUA_REGISTRYINDEX, cbd->cbref); lua_pushstring (L, err); + + if (cbd->item) { + rspamd_symcache_set_cur_item (cbd->task, cbd->item); + } + if (lua_pcall (L, 1, 0, 0) != 0) { msg_info ("callback call failed: %s", lua_tostring (L, -1)); lua_pop (L, 1); @@ -204,7 +212,6 @@ lua_http_finish_handler (struct rspamd_http_connection *conn, { struct lua_http_cbdata *cbd = (struct lua_http_cbdata *)conn->ud; struct rspamd_http_header *h, *htmp; - struct rspamd_async_watcher *existing_watcher = NULL; const gchar *body; gsize body_len; @@ -258,9 +265,9 @@ lua_http_finish_handler (struct rspamd_http_connection *conn, lua_settable (L, -3); } - if (cbd->w) { + if (cbd->item) { /* Replace watcher to deal with nested calls */ - existing_watcher = rspamd_session_replace_watcher (cbd->session, cbd->w); + rspamd_symcache_set_cur_item (cbd->task, cbd->item); } if (lua_pcall (L, 4, 0, 0) != 0) { @@ -268,11 +275,6 @@ lua_http_finish_handler (struct rspamd_http_connection *conn, lua_pop (L, 1); } - if (cbd->w) { - /* Restore existing watcher */ - rspamd_session_replace_watcher (cbd->session, existing_watcher); - } - lua_http_maybe_free (cbd); lua_thread_pool_restore_callback (&lcbd); @@ -292,7 +294,6 @@ lua_http_resume_handler (struct rspamd_http_connection *conn, const gchar *body; gsize body_len; struct rspamd_http_header *h, *htmp; - struct rspamd_async_watcher *existing_watcher = NULL; if (err) { lua_pushstring (L, err); @@ -355,17 +356,12 @@ lua_http_resume_handler (struct rspamd_http_connection *conn, lua_settable (L, -3); } - if (cbd->w) { + if (cbd->item) { /* Replace watcher to deal with nested calls */ - existing_watcher = rspamd_session_replace_watcher (cbd->session, cbd->w); + rspamd_symcache_set_cur_item (cbd->task, cbd->item); } lua_thread_resume (cbd->thread, 2); - - if (cbd->w) { - /* Restore existing watcher */ - rspamd_session_replace_watcher (cbd->session, existing_watcher); - } } static gboolean @@ -424,19 +420,26 @@ lua_http_make_connection (struct lua_http_cbdata *cbd) cbd->auth); } - rspamd_http_connection_write_message (cbd->conn, cbd->msg, - cbd->host, cbd->mime_type, cbd, fd, - &cbd->tv, cbd->ev_base); - /* Message is now owned by a connection object */ - cbd->msg = NULL; - if (cbd->session) { - rspamd_session_add_event (cbd->session, cbd->w, + rspamd_session_add_event (cbd->session, (event_finalizer_t) lua_http_fin, cbd, - g_quark_from_static_string ("lua http")); + M); cbd->flags |= RSPAMD_LUA_HTTP_FLAG_RESOLVED; } + if (cbd->item) { + rspamd_symcache_item_async_inc (cbd->task, cbd->item, M); + } + + struct rspamd_http_message *msg = cbd->msg; + + /* Message is now owned by a connection object */ + cbd->msg = NULL; + + rspamd_http_connection_write_message (cbd->conn, msg, + cbd->host, cbd->mime_type, cbd, fd, + &cbd->tv, cbd->ev_base); + return TRUE; } @@ -465,8 +468,14 @@ lua_http_dns_handler (struct rdns_reply *reply, gpointer ud) if (!lua_http_make_connection (cbd)) { lua_http_push_error (cbd, "unable to make connection to the host"); lua_http_maybe_free (cbd); + + return; } } + + if (cbd->item) { + rspamd_symcache_item_async_dec_check (cbd->task, cbd->item, M); + } } static void @@ -863,10 +872,13 @@ lua_http_request (lua_State *L) return 1; } if (task == NULL && cfg == NULL) { - return luaL_error (L, "Bad params to rspamd_http:request(): either task or config should be set"); + return luaL_error (L, + "Bad params to rspamd_http:request(): either task or config should be set"); } + if (ev_base == NULL) { - return luaL_error (L, "Bad params to rspamd_http:request(): ev_base isn't passed"); + return luaL_error (L, + "Bad params to rspamd_http:request(): ev_base isn't passed"); } cbd = g_malloc0 (sizeof (*cbd)); @@ -883,6 +895,11 @@ lua_http_request (lua_State *L) cbd->max_size = max_size; cbd->url = url; cbd->auth = auth; + cbd->task = task; + + if (task) { + cbd->item = rspamd_symcache_get_cur_item (task); + } if (msg->host) { cbd->host = rspamd_fstring_cstr (msg->host); @@ -900,9 +917,6 @@ lua_http_request (lua_State *L) if (session) { cbd->session = session; - - cbd->w = rspamd_session_get_watcher (cbd->session); - rspamd_session_watcher_push_specific (cbd->session, cbd->w); } if (rspamd_parse_inet_address (&cbd->addr, msg->host->str, msg->host->len)) { @@ -911,10 +925,6 @@ lua_http_request (lua_State *L) lua_http_maybe_free (cbd); lua_pushboolean (L, FALSE); - if (cbd->w) { - rspamd_session_watcher_pop (cbd->session, cbd->w); - } - return 1; } } @@ -930,10 +940,6 @@ lua_http_request (lua_State *L) lua_pushboolean (L, FALSE); g_free (to_resolve); - if (cbd->w) { - rspamd_session_watcher_pop (cbd->session, cbd->w); - } - return 1; } @@ -948,12 +954,11 @@ lua_http_request (lua_State *L) lua_http_maybe_free (cbd); lua_pushboolean (L, FALSE); - if (cbd->w) { - rspamd_session_watcher_pop (cbd->session, cbd->w); - } - return 1; } + else if (cbd->item) { + rspamd_symcache_item_async_inc (cbd->task, cbd->item, M); + } } } diff --git a/src/lua/lua_map.c b/src/lua/lua_map.c index 9cf50f1a2..98b025ea2 100644 --- a/src/lua/lua_map.c +++ b/src/lua/lua_map.c @@ -24,6 +24,9 @@ * This module is used to manage rspamd maps and map like objects * * @module rspamd_map + * + * All maps could be obtained by function `rspamd_config:get_maps()` + * Also see [`lua_maps` module description](lua_maps.html). */ /*** diff --git a/src/lua/lua_mimepart.c b/src/lua/lua_mimepart.c index 78c3e05b9..50cdaa7b7 100644 --- a/src/lua/lua_mimepart.c +++ b/src/lua/lua_mimepart.c @@ -132,8 +132,16 @@ LUA_FUNCTION_DEF (textpart, get_stats); LUA_FUNCTION_DEF (textpart, get_words_count); /*** - * @method mime_part:get_words() - * Get words in the part + * @method mime_part:get_words([how]) + * Get words in the part. Optional `how` argument defines type of words returned: + * - `stem`: stemmed words (default) + * - `norm`: normalised words (utf normalised + lowercased) + * - `raw`: raw words in utf (if possible) + * - `full`: list of tables, each table has the following fields: + * - [1] - stemmed word + * - [2] - normalised word + * - [3] - raw word + * - [4] - flags (table of strings) * @return {table/strings} words in the part */ LUA_FUNCTION_DEF (textpart, get_words); @@ -162,6 +170,13 @@ LUA_FUNCTION_DEF (textpart, get_html); * @return {string} short abbreviation (such as `ru`) for the script's language */ LUA_FUNCTION_DEF (textpart, get_language); + +/*** + * @method text_part:get_charset() + * Returns part real charset + * @return {string} charset of the part + */ +LUA_FUNCTION_DEF (textpart, get_charset); /*** * @method text_part:get_languages() * Returns array of tables of all languages detected for a part: @@ -205,6 +220,7 @@ static const struct luaL_reg textpartlib_m[] = { LUA_INTERFACE_DEF (textpart, is_html), LUA_INTERFACE_DEF (textpart, get_html), LUA_INTERFACE_DEF (textpart, get_language), + LUA_INTERFACE_DEF (textpart, get_charset), LUA_INTERFACE_DEF (textpart, get_languages), LUA_INTERFACE_DEF (textpart, get_mimepart), LUA_INTERFACE_DEF (textpart, get_stats), @@ -285,6 +301,15 @@ LUA_FUNCTION_DEF (mimepart, get_header_full); * @return {number} number of header's occurrencies or 0 if not found */ LUA_FUNCTION_DEF (mimepart, get_header_count); + +/*** + * @method mimepart:get_raw_headers() + * Get all undecoded headers of a mime part as a string + * @return {rspamd_text} all raw headers for a message as opaque text + */ +LUA_FUNCTION_DEF (mimepart, get_raw_headers); + + /*** * @method mime_part:get_content() * Get the parsed content of part @@ -318,6 +343,20 @@ LUA_FUNCTION_DEF (mimepart, get_type); LUA_FUNCTION_DEF (mimepart, get_type_full); /*** + * @method mime_part:get_detected_type() + * Extract content-type string of the mime part. Use libmagic detection + * @return {string,string} content type in form 'type','subtype' + */ +LUA_FUNCTION_DEF (mimepart, get_detected_type); + +/*** + * @method mime_part:get_detected_type_full() + * Extract content-type string of the mime part with all attributes. Use libmagic detection + * @return {string,string,table} content type in form 'type','subtype', {attrs} + */ +LUA_FUNCTION_DEF (mimepart, get_detected_type_full); + +/*** * @method mime_part:get_cte() * Extract content-transfer-encoding for a part * @return {string} content transfer encoding (e.g. `base64` or `7bit`) @@ -355,6 +394,12 @@ LUA_FUNCTION_DEF (mimepart, get_image); * @return {bool} true if a part is an archive */ LUA_FUNCTION_DEF (mimepart, is_archive); +/*** + * @method mime_part:is_attachment() + * Returns true if mime part looks like an attachment + * @return {bool} true if a part looks like an attachment + */ +LUA_FUNCTION_DEF (mimepart, is_attachment); /*** * @method mime_part:get_archive() @@ -377,6 +422,20 @@ LUA_FUNCTION_DEF (mimepart, get_archive); */ LUA_FUNCTION_DEF (mimepart, is_multipart); /*** + * @method mime_part:is_message() + * Returns true if mime part is a message part (message/rfc822) + * @return {bool} true if a part is is a message part + */ +LUA_FUNCTION_DEF (mimepart, is_message); +/*** + * @method mime_part:get_boundary() + * Returns boundary for a part (extracted from parent multipart for normal parts and + * from the part itself for multipart) + * @return {string} boundary value or nil + */ +LUA_FUNCTION_DEF (mimepart, get_boundary); + +/*** * @method mime_part:get_children() * Returns rspamd_mimepart table of part's childer. Returns nil if mime part is not multipart * or a message part. @@ -402,6 +461,13 @@ LUA_FUNCTION_DEF (mimepart, get_text); * @return {string} 128 characters hex string with digest of the part */ LUA_FUNCTION_DEF (mimepart, get_digest); + +/*** + * @method mime_part:get_id() + * Returns the order of the part in parts list + * @return {number} index of the part (starting from 1 as it is Lua API) + */ +LUA_FUNCTION_DEF (mimepart, get_id); /*** * @method mime_part:is_broken() * Returns true if mime part has incorrectly specified content type @@ -421,6 +487,13 @@ LUA_FUNCTION_DEF (mimepart, is_broken); * @param {table} params optional parameters */ LUA_FUNCTION_DEF (mimepart, headers_foreach); +/*** + * @method mime_part:get_parent() + * Returns parent part for this part + * @return {rspamd_mimepart} parent part or nil + */ +LUA_FUNCTION_DEF (mimepart, get_parent); + static const struct luaL_reg mimepartlib_m[] = { LUA_INTERFACE_DEF (mimepart, get_content), @@ -428,22 +501,30 @@ static const struct luaL_reg mimepartlib_m[] = { LUA_INTERFACE_DEF (mimepart, get_length), LUA_INTERFACE_DEF (mimepart, get_type), LUA_INTERFACE_DEF (mimepart, get_type_full), + LUA_INTERFACE_DEF (mimepart, get_detected_type), + LUA_INTERFACE_DEF (mimepart, get_detected_type_full), LUA_INTERFACE_DEF (mimepart, get_cte), LUA_INTERFACE_DEF (mimepart, get_filename), + LUA_INTERFACE_DEF (mimepart, get_boundary), LUA_INTERFACE_DEF (mimepart, get_header), LUA_INTERFACE_DEF (mimepart, get_header_raw), LUA_INTERFACE_DEF (mimepart, get_header_full), LUA_INTERFACE_DEF (mimepart, get_header_count), + LUA_INTERFACE_DEF (mimepart, get_raw_headers), LUA_INTERFACE_DEF (mimepart, is_image), LUA_INTERFACE_DEF (mimepart, get_image), LUA_INTERFACE_DEF (mimepart, is_archive), LUA_INTERFACE_DEF (mimepart, get_archive), LUA_INTERFACE_DEF (mimepart, is_multipart), + LUA_INTERFACE_DEF (mimepart, is_message), LUA_INTERFACE_DEF (mimepart, get_children), + LUA_INTERFACE_DEF (mimepart, get_parent), LUA_INTERFACE_DEF (mimepart, is_text), LUA_INTERFACE_DEF (mimepart, is_broken), + LUA_INTERFACE_DEF (mimepart, is_attachment), LUA_INTERFACE_DEF (mimepart, get_text), LUA_INTERFACE_DEF (mimepart, get_digest), + LUA_INTERFACE_DEF (mimepart, get_id), LUA_INTERFACE_DEF (mimepart, headers_foreach), {"__tostring", rspamd_lua_class_tostring}, {NULL, NULL} @@ -725,7 +806,7 @@ lua_textpart_get_words_count (lua_State *L) lua_pushinteger (L, 0); } else { - lua_pushinteger (L, part->utf_words->len); + lua_pushinteger (L, part->nwords); } return 1; @@ -736,8 +817,7 @@ lua_textpart_get_words (lua_State *L) { LUA_TRACE_POINT; struct rspamd_mime_text_part *part = lua_check_textpart (L); - rspamd_stat_token_t *w; - guint i; + enum rspamd_lua_words_type how = RSPAMD_LUA_WORDS_STEM; if (part == NULL) { return luaL_error (L, "invalid arguments"); @@ -747,14 +827,27 @@ lua_textpart_get_words (lua_State *L) lua_createtable (L, 0, 0); } else { - lua_createtable (L, part->utf_words->len, 0); - - for (i = 0; i < part->utf_words->len; i ++) { - w = &g_array_index (part->utf_words, rspamd_stat_token_t, i); + if (lua_type (L, 2) == LUA_TSTRING) { + const gchar *how_str = lua_tostring (L, 2); - lua_pushlstring (L, w->begin, w->len); - lua_rawseti (L, -2, i + 1); + if (strcmp (how_str, "stem") == 0) { + how = RSPAMD_LUA_WORDS_STEM; + } + else if (strcmp (how_str, "norm") == 0) { + how = RSPAMD_LUA_WORDS_NORM; + } + else if (strcmp (how_str, "raw") == 0) { + how = RSPAMD_LUA_WORDS_RAW; + } + else if (strcmp (how_str, "full") == 0) { + how = RSPAMD_LUA_WORDS_FULL; + } + else { + return luaL_error (L, "unknown words type: %s", how_str); + } } + + return rspamd_lua_push_words (L, part->utf_words, how); } return 1; @@ -834,6 +927,28 @@ lua_textpart_get_language (lua_State * L) } static gint +lua_textpart_get_charset (lua_State * L) +{ + LUA_TRACE_POINT; + struct rspamd_mime_text_part *part = lua_check_textpart (L); + + if (part != NULL) { + if (part->real_charset != NULL) { + lua_pushstring (L, part->real_charset); + return 1; + } + else { + lua_pushnil (L); + } + } + else { + return luaL_error (L, "invalid arguments"); + } + + return 1; +} + +static gint lua_textpart_get_languages (lua_State * L) { LUA_TRACE_POINT; @@ -878,8 +993,8 @@ struct lua_shingle_data { #define STORE_TOKEN(i, t) do { \ if ((i) < part->utf_words->len) { \ word = &g_array_index (part->utf_words, rspamd_stat_token_t, (i)); \ - sd->t.begin = word->begin; \ - sd->t.len = word->len; \ + sd->t.begin = word->stemmed.begin; \ + sd->t.len = word->stemmed.len; \ } \ }while (0) @@ -938,7 +1053,8 @@ lua_textpart_get_fuzzy_hashes (lua_State * L) for (i = 0; i < part->utf_words->len; i ++) { word = &g_array_index (part->utf_words, rspamd_stat_token_t, i); - rspamd_cryptobox_hash_update (&st, word->begin, word->len); + rspamd_cryptobox_hash_update (&st, + word->stemmed.begin, word->stemmed.len); } rspamd_cryptobox_hash_final (&st, digest); @@ -1123,48 +1239,49 @@ lua_mimepart_get_length (lua_State * L) } static gint -lua_mimepart_get_type_common (lua_State * L, gboolean full) +lua_mimepart_get_type_common (lua_State * L, struct rspamd_content_type *ct, + gboolean full) { - struct rspamd_mime_part *part = lua_check_mimepart (L); + GHashTableIter it; gpointer k, v; struct rspamd_content_type_param *param; - if (part == NULL) { + if (ct == NULL) { lua_pushnil (L); lua_pushnil (L); return 2; } - lua_pushlstring (L, part->ct->type.begin, part->ct->type.len); - lua_pushlstring (L, part->ct->subtype.begin, part->ct->subtype.len); + lua_pushlstring (L, ct->type.begin, ct->type.len); + lua_pushlstring (L, ct->subtype.begin, ct->subtype.len); if (!full) { return 2; } - lua_createtable (L, 0, 2 + (part->ct->attrs ? - g_hash_table_size (part->ct->attrs) : 0)); + lua_createtable (L, 0, 2 + (ct->attrs ? + g_hash_table_size (ct->attrs) : 0)); - if (part->ct->charset.len > 0) { + if (ct->charset.len > 0) { lua_pushstring (L, "charset"); - lua_pushlstring (L, part->ct->charset.begin, part->ct->charset.len); + lua_pushlstring (L, ct->charset.begin, ct->charset.len); lua_settable (L, -3); } - if (part->ct->boundary.len > 0) { - lua_pushstring (L, "charset"); - lua_pushlstring (L, part->ct->boundary.begin, part->ct->boundary.len); + if (ct->boundary.len > 0) { + lua_pushstring (L, "boundary"); + lua_pushlstring (L, ct->boundary.begin, ct->boundary.len); lua_settable (L, -3); } - if (part->ct->attrs) { - g_hash_table_iter_init (&it, part->ct->attrs); + if (ct->attrs) { + g_hash_table_iter_init (&it, ct->attrs); while (g_hash_table_iter_next (&it, &k, &v)) { param = v; - if (param->name.len > 0 && param->name.len > 0) { + if (param->name.len > 0 && param->value.len > 0) { /* TODO: think about multiple values here */ lua_pushlstring (L, param->name.begin, param->name.len); lua_pushlstring (L, param->value.begin, param->value.len); @@ -1180,14 +1297,52 @@ static gint lua_mimepart_get_type (lua_State * L) { LUA_TRACE_POINT; - return lua_mimepart_get_type_common (L, FALSE); + struct rspamd_mime_part *part = lua_check_mimepart (L); + + if (part == NULL) { + return luaL_error (L, "invalid arguments"); + } + + return lua_mimepart_get_type_common (L, part->ct, FALSE); } static gint lua_mimepart_get_type_full (lua_State * L) { LUA_TRACE_POINT; - return lua_mimepart_get_type_common (L, TRUE); + struct rspamd_mime_part *part = lua_check_mimepart (L); + + if (part == NULL) { + return luaL_error (L, "invalid arguments"); + } + + return lua_mimepart_get_type_common (L, part->ct, TRUE); +} + +static gint +lua_mimepart_get_detected_type (lua_State * L) +{ + LUA_TRACE_POINT; + struct rspamd_mime_part *part = lua_check_mimepart (L); + + if (part == NULL) { + return luaL_error (L, "invalid arguments"); + } + + return lua_mimepart_get_type_common (L, part->detected_ct, FALSE); +} + +static gint +lua_mimepart_get_detected_type_full (lua_State * L) +{ + LUA_TRACE_POINT; + struct rspamd_mime_part *part = lua_check_mimepart (L); + + if (part == NULL) { + return luaL_error (L, "invalid arguments"); + } + + return lua_mimepart_get_type_common (L, part->detected_ct, TRUE); } static gint @@ -1223,6 +1378,35 @@ lua_mimepart_get_filename (lua_State * L) } static gint +lua_mimepart_get_boundary (lua_State * L) +{ + LUA_TRACE_POINT; + struct rspamd_mime_part *part = lua_check_mimepart (L), *parent; + + if (part == NULL) { + return luaL_error (L, "invalid arguments"); + } + + if (IS_CT_MULTIPART (part->ct)) { + lua_pushlstring (L, part->specific.mp->boundary.begin, + part->specific.mp->boundary.len); + } + else { + parent = part->parent_part; + + if (!parent || !IS_CT_MULTIPART (parent->ct)) { + lua_pushnil (L); + } + else { + lua_pushlstring (L, parent->specific.mp->boundary.begin, + parent->specific.mp->boundary.len); + } + } + + return 1; +} + +static gint lua_mimepart_get_header_common (lua_State *L, enum rspamd_lua_task_header_type how) { struct rspamd_mime_part *part = lua_check_mimepart (L); @@ -1273,6 +1457,28 @@ lua_mimepart_get_header_count (lua_State * L) } static gint +lua_mimepart_get_raw_headers (lua_State *L) +{ + LUA_TRACE_POINT; + struct rspamd_mime_part *part = lua_check_mimepart (L); + struct rspamd_lua_text *t; + + if (part) { + t = lua_newuserdata (L, sizeof (*t)); + rspamd_lua_setclass (L, "rspamd{text}", -1); + t->start = part->raw_headers_str; + t->len = part->raw_headers_len; + t->flags = 0; + } + else { + return luaL_error (L, "invalid arguments"); + } + + + return 1; +} + +static gint lua_mimepart_is_image (lua_State * L) { LUA_TRACE_POINT; @@ -1318,6 +1524,52 @@ lua_mimepart_is_multipart (lua_State * L) } static gint +lua_mimepart_is_message (lua_State * L) +{ + LUA_TRACE_POINT; + struct rspamd_mime_part *part = lua_check_mimepart (L); + + if (part == NULL) { + return luaL_error (L, "invalid arguments"); + } + + lua_pushboolean (L, IS_CT_MESSAGE (part->ct) ? true : false); + + return 1; +} + +static gint +lua_mimepart_is_attachment (lua_State * L) +{ + LUA_TRACE_POINT; + struct rspamd_mime_part *part = lua_check_mimepart (L); + + if (part == NULL) { + return luaL_error (L, "invalid arguments"); + } + + if (!(part->flags & (RSPAMD_MIME_PART_IMAGE|RSPAMD_MIME_PART_TEXT))) { + if (part->cd && part->cd->type == RSPAMD_CT_ATTACHMENT) { + lua_pushboolean (L, true); + } + else { + if (part->cd && part->cd->filename.len > 0) { + /* We still have filename and it is not an image */ + lua_pushboolean (L, true); + } + else { + lua_pushboolean (L, false); + } + } + } + else { + lua_pushboolean (L, false); + } + + return 1; +} + +static gint lua_mimepart_is_text (lua_State * L) { LUA_TRACE_POINT; @@ -1411,13 +1663,13 @@ lua_mimepart_get_children (lua_State * L) return luaL_error (L, "invalid arguments"); } - if (!IS_CT_MULTIPART (part->ct) || part->specific.mp.children == NULL) { + if (!IS_CT_MULTIPART (part->ct) || part->specific.mp->children == NULL) { lua_pushnil (L); } else { - lua_createtable (L, part->specific.mp.children->len, 0); + lua_createtable (L, part->specific.mp->children->len, 0); - PTR_ARRAY_FOREACH (part->specific.mp.children, i, cur) { + PTR_ARRAY_FOREACH (part->specific.mp->children, i, cur) { pcur = lua_newuserdata (L, sizeof (*pcur)); *pcur = cur; rspamd_lua_setclass (L, "rspamd{mimepart}", -1); @@ -1428,6 +1680,29 @@ lua_mimepart_get_children (lua_State * L) return 1; } +static gint +lua_mimepart_get_parent (lua_State * L) +{ + LUA_TRACE_POINT; + struct rspamd_mime_part *part = lua_check_mimepart (L); + struct rspamd_mime_part **pparent; + + if (part == NULL) { + return luaL_error (L, "invalid arguments"); + } + + if (part->parent_part) { + pparent = lua_newuserdata (L, sizeof (*pparent)); + *pparent = part->parent_part; + rspamd_lua_setclass (L, "rspamd{mimepart}", -1); + } + else { + lua_pushnil (L); + } + + return 1; +} + static gint lua_mimepart_get_text (lua_State * L) @@ -1472,6 +1747,21 @@ lua_mimepart_get_digest (lua_State * L) } static gint +lua_mimepart_get_id (lua_State * L) +{ + LUA_TRACE_POINT; + struct rspamd_mime_part *part = lua_check_mimepart (L); + + if (part == NULL) { + return luaL_error (L, "invalid arguments"); + } + + lua_pushnumber (L, part->id); + + return 1; +} + +static gint lua_mimepart_headers_foreach (lua_State *L) { LUA_TRACE_POINT; diff --git a/src/lua/lua_redis.c b/src/lua/lua_redis.c index 65c0609f9..4335a3467 100644 --- a/src/lua/lua_redis.c +++ b/src/lua/lua_redis.c @@ -22,6 +22,8 @@ #define REDIS_DEFAULT_TIMEOUT 1.0 +static const gchar *M = "rspamd lua redis"; + /*** * @module rspamd_redis * This module implements redis asynchronous client for rspamd LUA API. @@ -94,12 +96,12 @@ struct lua_redis_request_specific_userdata; struct lua_redis_userdata { redisAsyncContext *ctx; struct rspamd_task *task; + struct rspamd_symcache_item *item; struct rspamd_async_session *s; struct event_base *ev_base; struct rspamd_config *cfg; struct rspamd_redis_pool *pool; gchar *server; - gchar *reqline; struct lua_redis_request_specific_userdata *specific; gdouble timeout; guint16 port; @@ -119,7 +121,6 @@ struct lua_redis_request_specific_userdata { guint nargs; gchar **args; gsize *arglens; - struct rspamd_async_watcher *w; struct lua_redis_userdata *c; struct lua_redis_ctx *ctx; struct lua_redis_request_specific_userdata *next; @@ -140,8 +141,9 @@ struct lua_redis_ctx { struct lua_redis_result { gboolean is_error; gint result_ref; - struct rspamd_async_watcher *w; + struct rspamd_symcache_item *item; struct rspamd_async_session *s; + struct rspamd_task *task; struct lua_redis_request_specific_userdata *sp_ud; }; @@ -280,6 +282,10 @@ lua_redis_push_error (const gchar *err, /* Data is nil */ lua_pushnil (cbs.L); + if (ud->item) { + rspamd_symcache_set_cur_item (ud->task, ud->item); + } + if (lua_pcall (cbs.L, 2, 0, 0) != 0) { msg_info ("call to callback failed: %s", lua_tostring (cbs.L, -1)); lua_pop (cbs.L, 1); @@ -291,7 +297,10 @@ lua_redis_push_error (const gchar *err, sp_ud->flags |= LUA_REDIS_SPECIFIC_REPLIED; if (connected && ud->s) { - rspamd_session_watcher_pop (ud->s, sp_ud->w); + if (ud->item) { + rspamd_symcache_item_async_dec_check (ud->task, ud->item, M); + } + rspamd_session_remove_event (ud->s, lua_redis_fin, sp_ud); } else { @@ -363,6 +372,10 @@ lua_redis_push_data (const redisReply *r, struct lua_redis_ctx *ctx, /* Data */ lua_redis_push_reply (cbs.L, r, ctx->flags & LUA_REDIS_TEXTDATA); + if (ud->item) { + rspamd_symcache_set_cur_item (ud->task, ud->item); + } + if (lua_pcall (cbs.L, 2, 0, 0) != 0) { msg_info ("call to callback failed: %s", lua_tostring (cbs.L, -1)); lua_pop (cbs.L, 1); @@ -374,7 +387,10 @@ lua_redis_push_data (const redisReply *r, struct lua_redis_ctx *ctx, sp_ud->flags |= LUA_REDIS_SPECIFIC_REPLIED; if (ud->s) { - rspamd_session_watcher_pop (ud->s, sp_ud->w); + if (ud->item) { + rspamd_symcache_item_async_dec_check (ud->task, ud->item, M); + } + rspamd_session_remove_event (ud->s, lua_redis_fin, sp_ud); } else { @@ -491,8 +507,16 @@ lua_redis_cleanup_events (struct lua_redis_ctx *ctx) while (!g_queue_is_empty (ctx->events_cleanup)) { struct lua_redis_result *result = g_queue_pop_head (ctx->events_cleanup); - rspamd_session_watcher_pop (result->s, result->w); - rspamd_session_remove_event (result->s, lua_redis_fin, result->sp_ud); + if (result->item) { + rspamd_symcache_item_async_dec_check (result->task, result->item, M); + } + + if (result->s) { + rspamd_session_remove_event (result->s, lua_redis_fin, result->sp_ud); + } + else { + lua_redis_fin (result->sp_ud); + } g_free (result); } @@ -584,7 +608,8 @@ lua_redis_callback_sync (redisAsyncContext *ac, gpointer r, gpointer priv) result->result_ref = luaL_ref (L, LUA_REGISTRYINDEX); result->s = ud->s; - result->w = sp_ud->w; + result->item = ud->item; + result->task = ud->task; result->sp_ud = sp_ud; g_queue_push_tail (ctx->replies, result); @@ -812,13 +837,6 @@ rspamd_lua_redis_prepare_connection (lua_State *L, gint *pcbref, gboolean is_asy } lua_pop (L, 1); - lua_pushstring (L, "ev_base"); - lua_gettable (L, -2); - if (lua_type (L, -1) == LUA_TUSERDATA) { - ev_base = lua_check_ev_base (L, -1); - } - lua_pop (L, 1); - if (cfg && ev_base) { ret = TRUE; } @@ -911,12 +929,17 @@ rspamd_lua_redis_prepare_connection (lua_State *L, gint *pcbref, gboolean is_asy ctx->events_cleanup = g_queue_new (); } + ud->s = session; ud->cfg = cfg; ud->pool = cfg->redis_pool; ud->ev_base = ev_base; ud->task = task; + if (task) { + ud->item = rspamd_symcache_get_cur_item (task); + } + ret = TRUE; } else { @@ -1020,6 +1043,7 @@ lua_redis_make_request (lua_State *L) &sp_ud->nargs); lua_pop (L, 1); LL_PREPEND (ud->specific, sp_ud); + ret = redisAsyncCommandArgv (ud->ctx, lua_redis_callback, sp_ud, @@ -1029,12 +1053,13 @@ lua_redis_make_request (lua_State *L) if (ret == REDIS_OK) { if (ud->s) { - rspamd_session_add_event (ud->s, NULL, lua_redis_fin, sp_ud, g_quark_from_static_string ("lua redis")); - sp_ud->w = rspamd_session_get_watcher (ud->s); - rspamd_session_watcher_push (ud->s); - } - else { - sp_ud->w = NULL; + rspamd_session_add_event (ud->s, + lua_redis_fin, sp_ud, + M); + + if (ud->item) { + rspamd_symcache_item_async_inc (ud->task, ud->item, M); + } } REDIS_RETAIN (ctx); /* Cleared by fin event */ @@ -1224,7 +1249,7 @@ lua_redis_make_request_sync (lua_State *L) * @param {task} task worker task object * @param {ip|string} host server address * @param {number} timeout timeout in seconds for request (1.0 by default) - * @return {redis} new connection object or nil if connection failed + * @return {boolean,redis} new connection object or nil if connection failed */ static int lua_redis_connect (lua_State *L) @@ -1255,11 +1280,12 @@ lua_redis_connect (lua_State *L) return 2; } + lua_pushboolean (L, TRUE); pctx = lua_newuserdata (L, sizeof (ctx)); *pctx = ctx; rspamd_lua_setclass (L, "rspamd{redis}", -1); - return 1; + return 2; } /*** @@ -1273,7 +1299,6 @@ static int lua_redis_connect_sync (lua_State *L) { LUA_TRACE_POINT; - rspamd_inet_addr_t *ip = NULL; gdouble timeout = REDIS_DEFAULT_TIMEOUT; struct lua_redis_ctx *ctx, **pctx; @@ -1295,13 +1320,8 @@ lua_redis_connect_sync (lua_State *L) pctx = lua_newuserdata (L, sizeof (ctx)); *pctx = ctx; rspamd_lua_setclass (L, "rspamd{redis}", -1); - } else { - if (ip) { - rspamd_inet_address_free (ip); - } - lua_pushboolean (L, FALSE); lua_pushstring (L, "bad arguments for redis request"); return 2; @@ -1396,18 +1416,27 @@ lua_redis_add_cmd (lua_State *L) if (ret == REDIS_OK) { if (ud->s) { - rspamd_session_add_event (ud->s, NULL, lua_redis_fin, sp_ud, g_quark_from_static_string ("lua redis")); - sp_ud->w = rspamd_session_get_watcher (ud->s); - rspamd_session_watcher_push (ud->s); + rspamd_session_add_event (ud->s, + lua_redis_fin, + sp_ud, + M); + + if (ud->item) { + rspamd_symcache_item_async_inc (ud->task, ud->item, M); + } } double_to_tv (sp_ud->c->timeout, &tv); + if (IS_ASYNC (ctx)) { - event_set (&sp_ud->timeout, -1, EV_TIMEOUT, lua_redis_timeout, sp_ud); + event_set (&sp_ud->timeout, -1, EV_TIMEOUT, + lua_redis_timeout, sp_ud); } else { - event_set (&sp_ud->timeout, -1, EV_TIMEOUT, lua_redis_timeout_sync, sp_ud); + event_set (&sp_ud->timeout, -1, EV_TIMEOUT, + lua_redis_timeout_sync, sp_ud); } + event_base_set (ud->ev_base, &sp_ud->timeout); event_add (&sp_ud->timeout, &tv); REDIS_RETAIN (ctx); diff --git a/src/lua/lua_regexp.c b/src/lua/lua_regexp.c index e88dc1bcc..4e233448b 100644 --- a/src/lua/lua_regexp.c +++ b/src/lua/lua_regexp.c @@ -29,6 +29,8 @@ */ LUA_FUNCTION_DEF (regexp, create); +LUA_FUNCTION_DEF (regexp, import_glob); +LUA_FUNCTION_DEF (regexp, import_plain); LUA_FUNCTION_DEF (regexp, create_cached); LUA_FUNCTION_DEF (regexp, get_cached); LUA_FUNCTION_DEF (regexp, get_pattern); @@ -58,6 +60,8 @@ static const struct luaL_reg regexplib_m[] = { }; static const struct luaL_reg regexplib_f[] = { LUA_INTERFACE_DEF (regexp, create), + LUA_INTERFACE_DEF (regexp, import_glob), + LUA_INTERFACE_DEF (regexp, import_plain), LUA_INTERFACE_DEF (regexp, get_cached), LUA_INTERFACE_DEF (regexp, create_cached), {NULL, NULL} @@ -133,22 +137,144 @@ lua_regexp_create (lua_State *L) flags_str = luaL_checkstring (L, 2); } - re = rspamd_regexp_new (string, flags_str, &err); - if (re == NULL) { - lua_pushnil (L); - msg_info ("cannot parse regexp: %s, error: %s", - string, - err == NULL ? "undefined" : err->message); - g_error_free (err); + if (string) { + re = rspamd_regexp_new (string, flags_str, &err); + if (re == NULL) { + lua_pushnil (L); + msg_info ("cannot parse regexp: %s, error: %s", + string, + err == NULL ? "undefined" : err->message); + g_error_free (err); + } else { + new = g_malloc0 (sizeof (struct rspamd_lua_regexp)); + new->re = re; + new->re_pattern = g_strdup (string); + new->module = rspamd_lua_get_module_name (L); + pnew = lua_newuserdata (L, sizeof (struct rspamd_lua_regexp *)); + rspamd_lua_setclass (L, "rspamd{regexp}", -1); + *pnew = new; + } + } + else { + return luaL_error (L, "invalid arguments"); + } + + return 1; +} + +/*** + * @function rspamd_regexp.import_glob(glob_pattern[, flags]) + * Creates new rspamd_regexp from glob + * @param {string} pattern pattern to build regexp. + * @param {string} flags optional flags to create regular expression + * @return {regexp} regexp argument that is *not* automatically destroyed + * @example + * local regexp = require "rspamd_regexp" + * + * local re = regexp.import_glob('ab*', 'i') + */ +static int +lua_regexp_import_glob (lua_State *L) +{ + LUA_TRACE_POINT; + rspamd_regexp_t *re; + struct rspamd_lua_regexp *new, **pnew; + const gchar *string, *flags_str = NULL; + gchar *escaped; + gsize pat_len; + GError *err = NULL; + + string = luaL_checklstring (L, 1, &pat_len); + + if (lua_gettop (L) == 2) { + flags_str = luaL_checkstring (L, 2); + } + + if (string) { + escaped = rspamd_str_regexp_escape (string, pat_len, NULL, + RSPAMD_REGEXP_ESCAPE_GLOB|RSPAMD_REGEXP_ESCAPE_UTF); + + re = rspamd_regexp_new (escaped, flags_str, &err); + + if (re == NULL) { + lua_pushnil (L); + msg_info ("cannot parse regexp: %s, error: %s", + string, + err == NULL ? "undefined" : err->message); + g_error_free (err); + g_free (escaped); + } + else { + new = g_malloc0 (sizeof (struct rspamd_lua_regexp)); + new->re = re; + new->re_pattern = escaped; + new->module = rspamd_lua_get_module_name (L); + pnew = lua_newuserdata (L, sizeof (struct rspamd_lua_regexp *)); + rspamd_lua_setclass (L, "rspamd{regexp}", -1); + *pnew = new; + } + } + else { + return luaL_error (L, "invalid arguments"); + } + + return 1; +} + +/*** + * @function rspamd_regexp.import_plain(plain_string[, flags]) + * Creates new rspamd_regexp from plain string (escaping specials) + * @param {string} pattern pattern to build regexp. + * @param {string} flags optional flags to create regular expression + * @return {regexp} regexp argument that is *not* automatically destroyed + * @example + * local regexp = require "rspamd_regexp" + * + * local re = regexp.import_plain('exact_string_with*', 'i') + */ +static int +lua_regexp_import_plain (lua_State *L) +{ + LUA_TRACE_POINT; + rspamd_regexp_t *re; + struct rspamd_lua_regexp *new, **pnew; + const gchar *string, *flags_str = NULL; + gchar *escaped; + gsize pat_len; + GError *err = NULL; + + string = luaL_checklstring (L, 1, &pat_len); + + if (lua_gettop (L) == 2) { + flags_str = luaL_checkstring (L, 2); + } + + if (string) { + escaped = rspamd_str_regexp_escape (string, pat_len, NULL, + RSPAMD_REGEXP_ESCAPE_ASCII); + + re = rspamd_regexp_new (escaped, flags_str, &err); + + if (re == NULL) { + lua_pushnil (L); + msg_info ("cannot parse regexp: %s, error: %s", + string, + err == NULL ? "undefined" : err->message); + g_error_free (err); + g_free (escaped); + } + else { + new = g_malloc0 (sizeof (struct rspamd_lua_regexp)); + new->re = re; + new->re_pattern = escaped; + new->module = rspamd_lua_get_module_name (L); + pnew = lua_newuserdata (L, sizeof (struct rspamd_lua_regexp *)); + rspamd_lua_setclass (L, "rspamd{regexp}", -1); + *pnew = new; + } } else { - new = g_malloc0 (sizeof (struct rspamd_lua_regexp)); - new->re = re; - new->re_pattern = g_strdup (string); - new->module = rspamd_lua_get_module_name (L); - pnew = lua_newuserdata (L, sizeof (struct rspamd_lua_regexp *)); - rspamd_lua_setclass (L, "rspamd{regexp}", -1); - *pnew = new; + return luaL_error (L, "invalid arguments"); } return 1; @@ -175,19 +301,24 @@ lua_regexp_get_cached (lua_State *L) flags_str = luaL_checkstring (L, 2); } - re = rspamd_regexp_cache_query (NULL, string, flags_str); + if (string) { + re = rspamd_regexp_cache_query (NULL, string, flags_str); - if (re) { - new = g_malloc0 (sizeof (struct rspamd_lua_regexp)); - new->re = rspamd_regexp_ref (re); - new->re_pattern = g_strdup (string); - new->module = rspamd_lua_get_module_name (L); - pnew = lua_newuserdata (L, sizeof (struct rspamd_lua_regexp *)); - rspamd_lua_setclass (L, "rspamd{regexp}", -1); - *pnew = new; + if (re) { + new = g_malloc0 (sizeof (struct rspamd_lua_regexp)); + new->re = rspamd_regexp_ref (re); + new->re_pattern = g_strdup (string); + new->module = rspamd_lua_get_module_name (L); + pnew = lua_newuserdata (L, sizeof (struct rspamd_lua_regexp *)); + rspamd_lua_setclass (L, "rspamd{regexp}", -1); + *pnew = new; + } + else { + lua_pushnil (L); + } } else { - lua_pushnil (L); + return luaL_error (L, "invalid arguments"); } return 1; @@ -222,36 +353,40 @@ lua_regexp_create_cached (lua_State *L) flags_str = luaL_checkstring (L, 2); } - re = rspamd_regexp_cache_query (NULL, string, flags_str); + if (string) { + re = rspamd_regexp_cache_query (NULL, string, flags_str); - if (re) { - new = g_malloc0 (sizeof (struct rspamd_lua_regexp)); - new->re = rspamd_regexp_ref (re); - new->re_pattern = g_strdup (string); - new->module = rspamd_lua_get_module_name (L); - pnew = lua_newuserdata (L, sizeof (struct rspamd_lua_regexp *)); - - rspamd_lua_setclass (L, "rspamd{regexp}", -1); - *pnew = new; - } - else { - re = rspamd_regexp_cache_create (NULL, string, flags_str, &err); - if (re == NULL) { - lua_pushnil (L); - msg_info ("cannot parse regexp: %s, error: %s", - string, - err == NULL ? "undefined" : err->message); - g_error_free (err); - } - else { + if (re) { new = g_malloc0 (sizeof (struct rspamd_lua_regexp)); new->re = rspamd_regexp_ref (re); new->re_pattern = g_strdup (string); new->module = rspamd_lua_get_module_name (L); pnew = lua_newuserdata (L, sizeof (struct rspamd_lua_regexp *)); + rspamd_lua_setclass (L, "rspamd{regexp}", -1); *pnew = new; } + else { + re = rspamd_regexp_cache_create (NULL, string, flags_str, &err); + if (re == NULL) { + lua_pushnil (L); + msg_info ("cannot parse regexp: %s, error: %s", + string, + err == NULL ? "undefined" : err->message); + g_error_free (err); + } else { + new = g_malloc0 (sizeof (struct rspamd_lua_regexp)); + new->re = rspamd_regexp_ref (re); + new->re_pattern = g_strdup (string); + new->module = rspamd_lua_get_module_name (L); + pnew = lua_newuserdata (L, sizeof (struct rspamd_lua_regexp *)); + rspamd_lua_setclass (L, "rspamd{regexp}", -1); + *pnew = new; + } + } + } + else { + return luaL_error (L, "invalid arguments"); } return 1; diff --git a/src/lua/lua_rsa.c b/src/lua/lua_rsa.c index 17675e548..58607f92c 100644 --- a/src/lua/lua_rsa.c +++ b/src/lua/lua_rsa.c @@ -557,10 +557,30 @@ static gint lua_rsa_signature_base64 (lua_State *L) { rspamd_fstring_t *sig = lua_check_rsa_sign (L, 1); + guint boundary = 0; gchar *b64; gsize outlen; + enum rspamd_newlines_type how = RSPAMD_TASK_NEWLINES_CRLF; - b64 = rspamd_encode_base64 (sig->str, sig->len, 0, &outlen); + if (lua_isnumber (L, 2)) { + boundary = lua_tonumber (L, 2); + } + + if (lua_isstring (L, 3)) { + const gchar *how_str = lua_tostring (L, 3); + + if (strcmp (how_str, "cr") == 0) { + how = RSPAMD_TASK_NEWLINES_CR; + } + else if (strcmp (how_str, "lf") == 0) { + how = RSPAMD_TASK_NEWLINES_LF; + } + else { + how = RSPAMD_TASK_NEWLINES_CRLF; + } + } + + b64 = rspamd_encode_base64_fold (sig->str, sig->len, boundary, &outlen, how); if (b64) { lua_pushlstring (L, b64, outlen); @@ -589,14 +609,15 @@ lua_rsa_verify_memory (lua_State *L) RSA *rsa; rspamd_fstring_t *signature; const gchar *data; + gsize sz; gint ret; rsa = lua_check_rsa_pubkey (L, 1); signature = lua_check_rsa_sign (L, 2); - data = luaL_checkstring (L, 3); + data = luaL_checklstring (L, 3, &sz); if (rsa != NULL && signature != NULL && data != NULL) { - ret = RSA_verify (NID_sha256, data, strlen (data), + ret = RSA_verify (NID_sha256, data, sz, signature->str, signature->len, rsa); if (ret == 0) { @@ -631,14 +652,15 @@ lua_rsa_sign_memory (lua_State *L) RSA *rsa; rspamd_fstring_t *signature, **psig; const gchar *data; + gsize sz; gint ret; rsa = lua_check_rsa_privkey (L, 1); - data = luaL_checkstring (L, 2); + data = luaL_checklstring (L, 2, &sz); if (rsa != NULL && data != NULL) { signature = rspamd_fstring_sized_new (RSA_size (rsa)); - ret = RSA_sign (NID_sha256, data, strlen (data), + ret = RSA_sign (NID_sha256, data, sz, signature->str, (guint *)&signature->len, rsa); if (ret != 1) { diff --git a/src/lua/lua_task.c b/src/lua/lua_task.c index 5a6be50f0..e9ac358d8 100644 --- a/src/lua/lua_task.c +++ b/src/lua/lua_task.c @@ -21,10 +21,12 @@ #include "unix-std.h" #include "libmime/smtp_parsers.h" #include "libserver/mempool_vars_internal.h" +#include "libserver/dkim.h" #include "libserver/task.h" #include "libstat/stat_api.h" +#include "libutil/map_helpers.h" + #include <math.h> -#include <src/libserver/task.h> /*** * @module rspamd_task @@ -50,13 +52,13 @@ end /*** * @function rspamd_task.load_from_file(filename[, cfg]) * Loads a message from specific file - * @return {boolean},{rspamd_task|error} status + new task or error message + * @return {boolean,rspamd_task|error} status + new task or error message */ LUA_FUNCTION_DEF (task, load_from_file); /*** * @function rspamd_task.load_from_string(message[, cfg]) * Loads a message from specific file - * @return {boolean},{rspamd_task|error} status + new task or error message + * @return {boolean,rspamd_task|error} status + new task or error message */ LUA_FUNCTION_DEF (task, load_from_string); @@ -235,7 +237,19 @@ LUA_FUNCTION_DEF (task, get_text_parts); * @return {table rspamd_mime_part} list of mime parts */ LUA_FUNCTION_DEF (task, get_parts); - +/*** + * @method task:get_meta_words([how='stem']) + * Get meta words from task (subject and displayed names) + * - `stem`: stemmed words (default) + * - `norm`: normalised words (utf normalised + lowercased) + * - `raw`: raw words in utf (if possible) + * - `full`: list of tables, each table has the following fields: + * - [1] - stemmed word + * - [2] - normalised word + * - [3] - raw word + * - [4] - flags (table of strings) + */ +LUA_FUNCTION_DEF (task, get_meta_words); /*** * @method task:get_request_header(name) * Get value of a HTTP request header. @@ -534,6 +548,26 @@ LUA_FUNCTION_DEF (task, get_images); */ LUA_FUNCTION_DEF (task, get_archives); /*** + * @method task:get_dkim_results() + * Returns list of all dkim check results as table of maps. Callee must ensure that + * dkim checks have been completed by adding dependency on `DKIM_TRACE` symbol. + * Fields in map: + * + * * `result` - string result of check: + * - `reject` + * - `allow` + * - `tempfail` + * - `permfail` + * - `not found` + * - `bad record` + * * `domain` - dkim domain + * * `selector` - dkim selector + * * `bhash` - short version of b tag (8 base64 symbols) + * * `fail_reason` - reason of failure (if applicable) + * @return {list of maps} dkim check results + */ +LUA_FUNCTION_DEF (task, get_dkim_results); +/*** * @method task:get_symbol(name) * Searches for a symbol `name` in all metrics results and returns a list of tables * one per metric that describes the symbol inserted. Please note that this function @@ -584,6 +618,20 @@ LUA_FUNCTION_DEF (task, get_symbols_tokens); */ LUA_FUNCTION_DEF (task, has_symbol); /*** + * @method task:has_symbol(name) + * Fast path to check if a specified symbol is in the task's results + * @param {string} name symbol's name + * @return {boolean} `true` if symbol has been found + */ +LUA_FUNCTION_DEF (task, enable_symbol); +/*** + * @method task:has_symbol(name) + * Fast path to check if a specified symbol is in the task's results + * @param {string} name symbol's name + * @return {boolean} `true` if symbol has been found + */ +LUA_FUNCTION_DEF (task, disable_symbol); +/*** * @method task:get_date(type[, gmt]) * Returns timestamp for a connection or for a MIME message. This function can be called with a * single table arguments with the following fields: @@ -620,15 +668,28 @@ LUA_FUNCTION_DEF (task, get_date); LUA_FUNCTION_DEF (task, get_message_id); LUA_FUNCTION_DEF (task, get_timeval); /*** + * @method task:get_metric_result() + * Get full result of a metric as a table: + * - `score`: current score + * - `action`: current action as a string + * - `nnegative`: number of negative rules matched + * - `npositive`: number of positive rules matched + * - `positive_score`: total score for positive rules + * - `negative_score`: total score for negative rules + * - `passthrough`: set to true if message has a passthrough result + * @return {table} metric result + */ +LUA_FUNCTION_DEF (task, get_metric_result); +/*** * @method task:get_metric_score(name) - * Get the current score of metric `name`. Should be used in post-filters only. + * Get the current score of metric `name` (must be nil or 'default') . Should be used in idempotent filters only. * @param {string} name name of a metric - * @return {table} table containing the current score and required score of the metric + * @return {number,number} 2 numbers containing the current score and required score of the metric */ LUA_FUNCTION_DEF (task, get_metric_score); /*** * @method task:get_metric_action(name) - * Get the current action of metric `name`. Should be used in post-filters only. + * Get the current action of metric `name` (must be nil or 'default'). Should be used in idempotent filters only. * @param {string} name name of a metric * @return {string} the current action of the metric as a string */ @@ -912,6 +973,17 @@ LUA_FUNCTION_DEF (task, get_newlines_type); */ LUA_FUNCTION_DEF (task, get_stat_tokens); +/*** + * @method task:lookup_words(map, function({o, n, s, f}) ... end) + * Matches words in a task (including meta words) against some map (set, regexp and so on) + * and call the specified function with a table containing 4 values: + * - [1] - stemmed word + * - [2] - normalised word + * - [3] - raw word + * - [4] - flags (table of strings) + */ +LUA_FUNCTION_DEF (task, lookup_words); + static const struct luaL_reg tasklib_f[] = { LUA_INTERFACE_DEF (task, load_from_file), LUA_INTERFACE_DEF (task, load_from_string), @@ -977,15 +1049,19 @@ static const struct luaL_reg tasklib_m[] = { LUA_INTERFACE_DEF (task, set_hostname), LUA_INTERFACE_DEF (task, get_images), LUA_INTERFACE_DEF (task, get_archives), + LUA_INTERFACE_DEF (task, get_dkim_results), LUA_INTERFACE_DEF (task, get_symbol), LUA_INTERFACE_DEF (task, get_symbols), LUA_INTERFACE_DEF (task, get_symbols_all), LUA_INTERFACE_DEF (task, get_symbols_numeric), LUA_INTERFACE_DEF (task, get_symbols_tokens), LUA_INTERFACE_DEF (task, has_symbol), + LUA_INTERFACE_DEF (task, enable_symbol), + LUA_INTERFACE_DEF (task, disable_symbol), LUA_INTERFACE_DEF (task, get_date), LUA_INTERFACE_DEF (task, get_message_id), LUA_INTERFACE_DEF (task, get_timeval), + LUA_INTERFACE_DEF (task, get_metric_result), LUA_INTERFACE_DEF (task, get_metric_score), LUA_INTERFACE_DEF (task, get_metric_action), LUA_INTERFACE_DEF (task, set_metric_score), @@ -1011,6 +1087,8 @@ static const struct luaL_reg tasklib_m[] = { LUA_INTERFACE_DEF (task, disable_action), LUA_INTERFACE_DEF (task, get_newlines_type), LUA_INTERFACE_DEF (task, get_stat_tokens), + LUA_INTERFACE_DEF (task, get_meta_words), + LUA_INTERFACE_DEF (task, lookup_words), {"__tostring", rspamd_lua_class_tostring}, {NULL, NULL} }; @@ -1052,6 +1130,7 @@ static const struct luaL_reg archivelib_m[] = { }; /* Blob methods */ +LUA_FUNCTION_DEF (text, fromstring); LUA_FUNCTION_DEF (text, len); LUA_FUNCTION_DEF (text, str); LUA_FUNCTION_DEF (text, ptr); @@ -1059,6 +1138,11 @@ LUA_FUNCTION_DEF (text, save_in_file); LUA_FUNCTION_DEF (text, take_ownership); LUA_FUNCTION_DEF (text, gc); +static const struct luaL_reg textlib_f[] = { + LUA_INTERFACE_DEF (text, fromstring), + {NULL, NULL} +}; + static const struct luaL_reg textlib_m[] = { LUA_INTERFACE_DEF (text, len), LUA_INTERFACE_DEF (text, str), @@ -1289,7 +1373,7 @@ lua_task_load_from_file (lua_State * L) err = strerror (errno); } else { - task = rspamd_task_new (NULL, cfg, NULL, NULL); + task = rspamd_task_new (NULL, cfg, NULL, NULL, NULL); task->msg.begin = map; task->msg.len = sz; rspamd_mempool_add_destructor (task->task_pool, @@ -1342,7 +1426,7 @@ lua_task_load_from_string (lua_State * L) } } - task = rspamd_task_new (NULL, cfg, NULL, NULL); + task = rspamd_task_new (NULL, cfg, NULL, NULL, NULL); task->msg.begin = g_strdup (str_message); task->msg.len = message_len; rspamd_mempool_add_destructor (task->task_pool, lua_task_free_dtor, task); @@ -1591,7 +1675,8 @@ lua_task_set_pre_result (lua_State * L) /* Keep compatibility here :( */ ucl_object_replace_key (task->messages, - ucl_object_fromstring (message), "smtp_message", 0, + ucl_object_fromstring_common (message, 0, UCL_STRING_RAW), + "smtp_message", 0, false); } else { @@ -1668,7 +1753,8 @@ lua_task_append_message (lua_State * L) } ucl_object_insert_key (task->messages, - ucl_object_fromstring (message), category, 0, + ucl_object_fromstring_common (message, 0, UCL_STRING_RAW), + category, 0, true); } else { @@ -2022,6 +2108,12 @@ rspamd_lua_push_header (lua_State *L, struct rspamd_mime_header *rh, rspamd_lua_table_set (L, "value", rh->value); } + if (rh->raw_len > 0) { + lua_pushstring (L, "raw"); + lua_pushlstring (L, rh->raw_value, rh->raw_len); + lua_settable (L, -3); + } + if (rh->decoded) { rspamd_lua_table_set (L, "decoded", rh->decoded); } @@ -3361,6 +3453,90 @@ lua_task_get_archives (lua_State *L) return 1; } +static gint +lua_task_get_dkim_results (lua_State *L) +{ + LUA_TRACE_POINT; + struct rspamd_task *task = lua_check_task (L, 1); + guint nelt = 0, i; + struct rspamd_dkim_check_result **pres, **cur; + + if (task) { + if (!lua_task_get_cached (L, task, "dkim_results", 0)) { + pres = rspamd_mempool_get_variable (task->task_pool, + RSPAMD_MEMPOOL_DKIM_CHECK_RESULTS); + + if (pres == NULL) { + lua_newtable (L); + } + else { + for (cur = pres; *cur != NULL; cur ++) { + nelt ++; + } + + lua_createtable (L, nelt, 0); + + for (i = 0; i < nelt; i ++) { + struct rspamd_dkim_check_result *res = pres[i]; + const gchar *result_str = "unknown"; + + lua_createtable (L, 0, 4); + + switch (res->rcode) { + case DKIM_CONTINUE: + result_str = "allow"; + break; + case DKIM_REJECT: + result_str = "reject"; + break; + case DKIM_TRYAGAIN: + result_str = "tempfail"; + break; + case DKIM_NOTFOUND: + result_str = "not found"; + break; + case DKIM_RECORD_ERROR: + result_str = "bad record"; + break; + case DKIM_PERM_ERROR: + result_str = "permanent error"; + break; + default: + break; + } + + rspamd_lua_table_set (L, "result", result_str); + + if (res->domain) { + rspamd_lua_table_set (L, "domain", res->domain); + } + + if (res->selector) { + rspamd_lua_table_set (L, "selector", res->selector); + } + + if (res->short_b) { + rspamd_lua_table_set (L, "bhash", res->short_b); + } + + if (res->fail_reason) { + rspamd_lua_table_set (L, "fail_reason", res->fail_reason); + } + + lua_rawseti (L, -2, i + 1); + } + } + + lua_task_set_cached (L, task, "dkim_results", -1, 0); + } + } + else { + return luaL_error (L, "invalid arguments"); + } + + return 1; +} + static inline gboolean lua_push_symbol_result (lua_State *L, struct rspamd_task *task, @@ -3501,6 +3677,48 @@ lua_task_has_symbol (lua_State *L) } static gint +lua_task_enable_symbol (lua_State *L) +{ + LUA_TRACE_POINT; + struct rspamd_task *task = lua_check_task (L, 1); + const gchar *symbol; + gboolean found = FALSE; + + symbol = luaL_checkstring (L, 2); + + if (task && symbol) { + found = rspamd_symcache_enable_symbol (task, task->cfg->cache, symbol); + lua_pushboolean (L, found); + } + else { + return luaL_error (L, "invalid arguments"); + } + + return 1; +} + +static gint +lua_task_disable_symbol (lua_State *L) +{ + LUA_TRACE_POINT; + struct rspamd_task *task = lua_check_task (L, 1); + const gchar *symbol; + gboolean found = FALSE; + + symbol = luaL_checkstring (L, 2); + + if (task && symbol) { + found = rspamd_symcache_disable_symbol (task, task->cfg->cache, symbol); + lua_pushboolean (L, found); + } + else { + return luaL_error (L, "invalid arguments"); + } + + return 1; +} + +static gint lua_task_get_symbols (lua_State *L) { LUA_TRACE_POINT; @@ -3593,7 +3811,7 @@ lua_task_get_symbols_numeric (lua_State *L) kh_foreach_value_ptr (mres->symbols, s, { if (!(s->flags & RSPAMD_SYMBOL_RESULT_IGNORED)) { - id = rspamd_symbols_cache_find_symbol (task->cfg->cache, + id = rspamd_symcache_find_symbol (task->cfg->cache, s->name); lua_pushinteger (L, id); lua_rawseti (L, -3, i); @@ -3667,8 +3885,8 @@ lua_task_get_symbols_tokens (lua_State *L) } lua_createtable (L, - rspamd_symbols_cache_stats_symbols_count (task->cfg->cache), 0); - rspamd_symbols_cache_foreach (task->cfg->cache, tokens_foreach_cb, &cbd); + rspamd_symcache_stats_symbols_count (task->cfg->cache), 0); + rspamd_symcache_foreach (task->cfg->cache, tokens_foreach_cb, &cbd); return 1; } @@ -3958,6 +4176,8 @@ lua_task_has_flag (lua_State *L) RSPAMD_TASK_FLAG_SKIP_PROCESS); LUA_TASK_GET_FLAG (flag, "milter", RSPAMD_TASK_FLAG_MILTER); + LUA_TASK_GET_FLAG (flag, "bad_unicode", + RSPAMD_TASK_FLAG_BAD_UNICODE); if (!found) { msg_warn_task ("unknown flag requested: %s", flag); @@ -4173,7 +4393,7 @@ lua_task_set_settings (lua_State *L) } } - rspamd_symbols_cache_process_settings (task, task->cfg->cache); + rspamd_symcache_process_settings (task, task->cfg->cache); } else { return luaL_error (L, "invalid arguments"); @@ -4514,6 +4734,63 @@ lua_task_process_regexp (lua_State *L) } static gint +lua_task_get_metric_result (lua_State *L) +{ + LUA_TRACE_POINT; + struct rspamd_task *task = lua_check_task (L, 1); + struct rspamd_metric_result *metric_res; + + if (task) { + metric_res = task->result; + + /* Fields added: + * - `score`: current score + * - `action`: current action as a string + * - `nnegative`: number of negative rules matched + * - `npositive`: number of positive rules matched + * - `positive_score`: total score for positive rules + * - `negative_score`: total score for negative rules + * - `passthrough`: set to true if message has a passthrough result + */ + lua_createtable (L, 0, 7); + + lua_pushstring (L, "score"); + lua_pushnumber (L, metric_res->score); + lua_settable (L, -3); + + lua_pushstring (L, "action"); + lua_pushstring (L, rspamd_action_to_str ( + rspamd_check_action_metric (task, metric_res))); + lua_settable (L, -3); + + lua_pushstring (L, "nnegative"); + lua_pushnumber (L, metric_res->nnegative); + lua_settable (L, -3); + + lua_pushstring (L, "npositive"); + lua_pushnumber (L, metric_res->npositive); + lua_settable (L, -3); + + lua_pushstring (L, "positive_score"); + lua_pushnumber (L, metric_res->positive_score); + lua_settable (L, -3); + + lua_pushstring (L, "negative_score"); + lua_pushnumber (L, metric_res->negative_score); + lua_settable (L, -3); + + lua_pushstring (L, "passthrough"); + lua_pushboolean (L, !!(metric_res->passthrough_result != NULL)); + lua_settable (L, -3); + } + else { + return luaL_error (L, "invalid arguments"); + } + + return 1; +} + +static gint lua_task_get_metric_score (lua_State *L) { LUA_TRACE_POINT; @@ -4678,13 +4955,13 @@ lua_push_stat_token (lua_State *L, rspamd_token_t *tok) if (tok->t1) { lua_pushstring (L, "t1"); - lua_pushlstring (L, tok->t1->begin, tok->t1->len); + lua_pushlstring (L, tok->t1->stemmed.begin, tok->t1->stemmed.len); lua_settable (L, -3); } if (tok->t2) { lua_pushstring (L, "t2"); - lua_pushlstring (L, tok->t2->begin, tok->t2->len); + lua_pushlstring (L, tok->t2->stemmed.begin, tok->t2->stemmed.len); lua_settable (L, -3); } @@ -4717,8 +4994,8 @@ lua_push_stat_token (lua_State *L, rspamd_token_t *tok) lua_pushboolean (L, true); lua_settable (L, -3); } - if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_SUBJECT) { - lua_pushstring (L, "subject"); + if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_HEADER) { + lua_pushstring (L, "header"); lua_pushboolean (L, true); lua_settable (L, -3); } @@ -4930,6 +5207,152 @@ lua_task_headers_foreach (lua_State *L) return 0; } +static gint +lua_task_get_meta_words (lua_State *L) +{ + LUA_TRACE_POINT; + struct rspamd_task *task = lua_check_task (L, 1); + enum rspamd_lua_words_type how = RSPAMD_LUA_WORDS_STEM; + + if (task == NULL) { + return luaL_error (L, "invalid arguments"); + } + + if (task->meta_words == NULL) { + lua_createtable (L, 0, 0); + } + else { + if (lua_type (L, 2) == LUA_TSTRING) { + const gchar *how_str = lua_tostring (L, 2); + + if (strcmp (how_str, "stem") == 0) { + how = RSPAMD_LUA_WORDS_STEM; + } + else if (strcmp (how_str, "norm") == 0) { + how = RSPAMD_LUA_WORDS_NORM; + } + else if (strcmp (how_str, "raw") == 0) { + how = RSPAMD_LUA_WORDS_RAW; + } + else if (strcmp (how_str, "full") == 0) { + how = RSPAMD_LUA_WORDS_FULL; + } + else { + return luaL_error (L, "unknown words type: %s", how_str); + } + } + + return rspamd_lua_push_words (L, task->meta_words, how); + } + + return 1; +} + +static guint +lua_lookup_words_array (lua_State *L, + gint cbpos, + struct rspamd_task *task, + struct rspamd_lua_map *map, + GArray *words) +{ + rspamd_stat_token_t *tok; + guint i, nmatched = 0; + gint err_idx; + gboolean matched; + const gchar *key; + gsize keylen; + + for (i = 0; i < words->len; i ++) { + tok = &g_array_index (words, rspamd_stat_token_t, i); + + matched = FALSE; + + if (tok->normalized.len == 0) { + continue; + } + + key = tok->normalized.begin; + keylen = tok->normalized.len; + + switch (map->type) { + case RSPAMD_LUA_MAP_SET: + case RSPAMD_LUA_MAP_HASH: + /* We know that tok->normalized is zero terminated in fact */ + if (rspamd_match_hash_map (map->data.hash, key)) { + matched = TRUE; + } + break; + case RSPAMD_LUA_MAP_REGEXP: + case RSPAMD_LUA_MAP_REGEXP_MULTIPLE: + if (rspamd_match_regexp_map_single (map->data.re_map, key, + keylen)) { + matched = TRUE; + } + break; + default: + g_assert_not_reached (); + break; + } + + if (matched) { + nmatched ++; + + lua_pushcfunction (L, &rspamd_lua_traceback); + err_idx = lua_gettop (L); + lua_pushvalue (L, cbpos); /* Function */ + rspamd_lua_push_full_word (L, tok); + + if (lua_pcall (L, 1, 0, err_idx) != 0) { + GString *tb = lua_touserdata (L, -1); + msg_err_task ("cannot call callback function for lookup words: %s", + tb->str); + g_string_free (tb, TRUE); + } + + lua_settop (L, err_idx - 1); + } + } + + return nmatched; +} + +static gint +lua_task_lookup_words (lua_State *L) +{ + LUA_TRACE_POINT; + struct rspamd_task *task = lua_check_task (L, 1); + struct rspamd_lua_map *map = lua_check_map (L, 2); + struct rspamd_mime_text_part *tp; + + guint i, matches = 0; + + if (task == NULL || map == NULL || lua_type (L, 3) != LUA_TFUNCTION) { + return luaL_error (L, "invalid arguments"); + } + + if (map->type != RSPAMD_LUA_MAP_SET && + map->type != RSPAMD_LUA_MAP_REGEXP && + map->type != RSPAMD_LUA_MAP_HASH && + map->type != RSPAMD_LUA_MAP_REGEXP_MULTIPLE) { + return luaL_error (L, "invalid map type"); + } + + PTR_ARRAY_FOREACH (task->text_parts, i, tp) { + if (tp->utf_words) { + matches += lua_lookup_words_array (L, 3, task, map, tp->utf_words); + } + } + + if (task->meta_words) { + matches += lua_lookup_words_array (L, 3, task, map, task->meta_words); + } + + lua_pushinteger (L, matches); + + return 1; +} + + /* Image functions */ static gint lua_image_get_width (lua_State *L) @@ -5145,6 +5568,32 @@ lua_archive_get_filename (lua_State *L) /* Text methods */ static gint +lua_text_fromstring (lua_State *L) +{ + LUA_TRACE_POINT; + const gchar *str; + gsize l = 0; + struct rspamd_lua_text *t; + + str = luaL_checklstring (L, 1, &l); + + if (str) { + t = lua_newuserdata (L, sizeof (*t)); + t->start = g_malloc (l + 1); + rspamd_strlcpy ((char *)t->start, str, l + 1); + t->len = l; + t->flags = RSPAMD_TEXT_FLAG_OWN; + rspamd_lua_setclass (L, "rspamd{text}", -1); + } + else { + return luaL_error (L, "invalid arguments"); + } + + + return 1; +} + +static gint lua_text_len (lua_State *L) { LUA_TRACE_POINT; @@ -5229,28 +5678,37 @@ lua_text_save_in_file (lua_State *L) struct rspamd_lua_text *t = lua_check_text (L, 1); const gchar *fname = NULL; guint mode = 00644; - gint fd; + gint fd = -1; + gboolean need_close = FALSE; if (t != NULL) { if (lua_type (L, 2) == LUA_TSTRING) { fname = luaL_checkstring (L, 2); + + if (lua_type (L, 3) == LUA_TNUMBER) { + mode = lua_tonumber (L, 3); + } } - if (lua_type (L, 3) == LUA_TNUMBER) { - mode = lua_tonumber (L, 3); + else if (lua_type (L, 2) == LUA_TNUMBER) { + /* Created fd */ + fd = lua_tonumber (L, 2); } - if (fname) { - fd = rspamd_file_xopen (fname, O_CREAT | O_WRONLY | O_EXCL, mode, 0); + if (fd == -1) { + if (fname) { + fd = rspamd_file_xopen (fname, O_CREAT | O_WRONLY | O_EXCL, mode, 0); - if (fd == -1) { - lua_pushboolean (L, false); - lua_pushstring (L, strerror (errno)); + if (fd == -1) { + lua_pushboolean (L, false); + lua_pushstring (L, strerror (errno)); - return 2; + return 2; + } + need_close = TRUE; + } + else { + fd = STDOUT_FILENO; } - } - else { - fd = STDOUT_FILENO; } if (write (fd, t->start, t->len) == -1) { @@ -5264,7 +5722,7 @@ lua_text_save_in_file (lua_State *L) return 2; } - if (fd != STDOUT_FILENO) { + if (need_close) { close (fd); } @@ -5309,6 +5767,16 @@ lua_load_task (lua_State * L) return 1; } +static gint +lua_load_text (lua_State * L) +{ + lua_newtable (L); + luaL_register (L, NULL, textlib_f); + + return 1; +} + + static void luaopen_archive (lua_State * L) { @@ -5339,6 +5807,8 @@ luaopen_text (lua_State *L) { rspamd_lua_new_class (L, "rspamd{text}", textlib_m); lua_pop (L, 1); + + rspamd_lua_add_preload (L, "rspamd_text", lua_load_text); } void diff --git a/src/lua/lua_tcp.c b/src/lua/lua_tcp.c index f9c1a477d..047bfe444 100644 --- a/src/lua/lua_tcp.c +++ b/src/lua/lua_tcp.c @@ -19,6 +19,8 @@ #include "unix-std.h" #include <math.h> +static const gchar *M = "rspamd lua tcp"; + /*** * @module rspamd_tcp * Rspamd TCP module represents generic TCP asynchronous client available from LUA code. @@ -336,11 +338,11 @@ struct lua_tcp_cbdata { guint port; guint flags; gchar tag[7]; - struct rspamd_async_watcher *w; struct event ev; struct lua_tcp_dtor *dtors; ref_entry_t ref; struct rspamd_task *task; + struct rspamd_symcache_item *item; struct thread_entry *thread; struct rspamd_config *cfg; gboolean eof; @@ -482,10 +484,10 @@ lua_tcp_maybe_free (struct lua_tcp_cbdata *cbd) * Object is owned by lua and will be destroyed on __gc() */ - if (cbd->w) { - rspamd_session_watcher_pop (cbd->session, cbd->w); + if (cbd->item) { + rspamd_symcache_item_async_dec_check (cbd->task, cbd->item, M); + cbd->item = NULL; } - cbd->w = NULL; if (cbd->async_ev) { rspamd_session_remove_event (cbd->session, lua_tcp_void_finalyser, cbd); @@ -494,10 +496,10 @@ lua_tcp_maybe_free (struct lua_tcp_cbdata *cbd) cbd->async_ev = NULL; } else { - if (cbd->w) { - rspamd_session_watcher_pop (cbd->session, cbd->w); + if (cbd->item) { + rspamd_symcache_item_async_dec_check (cbd->task, cbd->item, M); + cbd->item = NULL; } - cbd->w = NULL; if (cbd->async_ev) { rspamd_session_remove_event (cbd->session, lua_tcp_fin, cbd); @@ -525,7 +527,6 @@ lua_tcp_push_error (struct lua_tcp_cbdata *cbd, gboolean is_fatal, struct lua_tcp_handler *hdl; gint cbref, top; struct lua_callback_state cbs; - struct rspamd_async_watcher *existing_watcher = NULL; lua_State *L; if (cbd->thread) { @@ -572,21 +573,14 @@ lua_tcp_push_error (struct lua_tcp_cbdata *cbd, gboolean is_fatal, rspamd_lua_setclass (L, "rspamd{tcp}", -1); TCP_RETAIN (cbd); - if (cbd->w) { - /* Replace watcher to deal with nested calls */ - existing_watcher = rspamd_session_replace_watcher (cbd->session, cbd->w); + if (cbd->item) { + rspamd_symcache_set_cur_item (cbd->task, cbd->item); } if (lua_pcall (L, 3, 0, 0) != 0) { msg_info ("callback call failed: %s", lua_tostring (L, -1)); } - - if (cbd->w) { - /* Restore existing watcher */ - rspamd_session_replace_watcher (cbd->session, existing_watcher); - } - lua_settop (L, top); TCP_RELEASE (cbd); @@ -617,7 +611,6 @@ lua_tcp_push_data (struct lua_tcp_cbdata *cbd, const guint8 *str, gsize len) gint cbref, arg_cnt, top; struct lua_callback_state cbs; lua_State *L; - struct rspamd_async_watcher *existing_watcher = NULL; if (cbd->thread) { lua_tcp_resume_thread (cbd, str, len); @@ -663,20 +656,14 @@ lua_tcp_push_data (struct lua_tcp_cbdata *cbd, const guint8 *str, gsize len) TCP_RETAIN (cbd); - if (cbd->w) { - /* Replace watcher to deal with nested calls */ - existing_watcher = rspamd_session_replace_watcher (cbd->session, cbd->w); + if (cbd->item) { + rspamd_symcache_set_cur_item (cbd->task, cbd->item); } if (lua_pcall (L, arg_cnt, 0, 0) != 0) { msg_info ("callback call failed: %s", lua_tostring (L, -1)); } - if (cbd->w) { - /* Restore existing watcher */ - rspamd_session_replace_watcher (cbd->session, existing_watcher); - } - lua_settop (L, top); TCP_RELEASE (cbd); } @@ -720,7 +707,6 @@ lua_tcp_resume_thread (struct lua_tcp_cbdata *cbd, const guint8 *str, gsize len) lua_State *L = cbd->thread->lua_state; struct lua_tcp_handler *hdl; - struct rspamd_async_watcher *existing_watcher = NULL; hdl = g_queue_peek_head (cbd->handlers); @@ -735,18 +721,12 @@ lua_tcp_resume_thread (struct lua_tcp_cbdata *cbd, const guint8 *str, gsize len) lua_tcp_shift_handler (cbd); lua_thread_pool_set_running_entry (cbd->cfg->lua_thread_pool, cbd->thread); - if (cbd->w) { - /* Replace watcher to deal with nested calls */ - existing_watcher = rspamd_session_replace_watcher (cbd->session, cbd->w); + if (cbd->item) { + rspamd_symcache_set_cur_item (cbd->task, cbd->item); } lua_thread_resume (cbd->thread, 2); - if (cbd->w) { - /* Restore existing watcher */ - rspamd_session_replace_watcher (cbd->session, existing_watcher); - } - TCP_RELEASE (cbd); } @@ -915,7 +895,15 @@ lua_tcp_process_read_handler (struct lua_tcp_cbdata *cbd, else { /* Plan new read */ msg_debug_tcp ("NOT found TCP stop pattern"); - lua_tcp_plan_read (cbd); + + if (!cbd->eof) { + lua_tcp_plan_read (cbd); + } + else { + /* Got session finished but no stop pattern */ + lua_tcp_push_error (cbd, TRUE, + "IO read error: connection terminated"); + } } } } @@ -977,7 +965,7 @@ lua_tcp_process_read (struct lua_tcp_cbdata *cbd, lua_tcp_process_read_handler (cbd, rh, TRUE); } else { - lua_tcp_push_error (cbd, FALSE, "IO read error: connection terminated"); + lua_tcp_push_error (cbd, TRUE, "IO read error: connection terminated"); } lua_tcp_plan_handler_event (cbd, FALSE, TRUE); @@ -1042,7 +1030,6 @@ lua_tcp_handler (int fd, short what, gpointer ud) if (cbd->connect_cb != -1) { struct lua_tcp_cbdata **pcbd; gint top; - struct rspamd_async_watcher *existing_watcher = NULL; lua_thread_pool_prepare_callback (cbd->cfg->lua_thread_pool, &cbs); L = cbs.L; @@ -1054,26 +1041,16 @@ lua_tcp_handler (int fd, short what, gpointer ud) TCP_RETAIN (cbd); rspamd_lua_setclass (L, "rspamd{tcp}", -1); - if (cbd->w) { - /* Replace watcher to deal with nested calls */ - existing_watcher = rspamd_session_replace_watcher ( - cbd->session, cbd->w); + if (cbd->item) { + rspamd_symcache_set_cur_item (cbd->task, cbd->item); } if (lua_pcall (L, 1, 0, 0) != 0) { msg_info ("callback call failed: %s", lua_tostring (L, -1)); } - if (cbd->w) { - /* Restore existing watcher */ - rspamd_session_replace_watcher (cbd->session, - existing_watcher); - } - lua_settop (L, top); - TCP_RELEASE (cbd); - lua_thread_pool_restore_callback (&cbs); } } @@ -1194,8 +1171,7 @@ lua_tcp_register_event (struct lua_tcp_cbdata *cbd) if (cbd->session) { event_finalizer_t fin = IS_SYNC (cbd) ? lua_tcp_void_finalyser : lua_tcp_fin; - cbd->async_ev = rspamd_session_add_event (cbd->session, NULL, fin, cbd, - g_quark_from_static_string ("lua tcp")); + cbd->async_ev = rspamd_session_add_event (cbd->session, fin, cbd, M); if (!cbd->async_ev) { return FALSE; @@ -1208,12 +1184,8 @@ lua_tcp_register_event (struct lua_tcp_cbdata *cbd) static void lua_tcp_register_watcher (struct lua_tcp_cbdata *cbd) { - if (cbd->session) { - cbd->w = rspamd_session_get_watcher (cbd->session); - - if (cbd->w) { - rspamd_session_watcher_push (cbd->session); - } + if (cbd->item) { + rspamd_symcache_item_async_inc (cbd->task, cbd->item, M); } } @@ -1241,6 +1213,7 @@ lua_tcp_make_connection (struct lua_tcp_cbdata *cbd) return FALSE; } +#if 0 if (!(cbd->flags & LUA_TCP_FLAG_RESOLVED)) { /* We come here without resolving, so we need to add a watcher */ lua_tcp_register_watcher (cbd); @@ -1248,6 +1221,7 @@ lua_tcp_make_connection (struct lua_tcp_cbdata *cbd) else { cbd->flags |= LUA_TCP_FLAG_RESOLVED; } +#endif lua_tcp_register_event (cbd); @@ -1590,6 +1564,11 @@ lua_tcp_request (lua_State *L) } cbd->task = task; + + if (task) { + cbd->item = rspamd_symcache_get_cur_item (task); + } + cbd->cfg = cfg; h = rspamd_random_uint64_fast (); rspamd_snprintf (cbd->tag, sizeof (cbd->tag), "%uxL", h); @@ -1668,6 +1647,8 @@ lua_tcp_request (lua_State *L) if (rspamd_parse_inet_address (&cbd->addr, host, 0)) { rspamd_inet_address_set_port (cbd->addr, port); /* Host is numeric IP, no need to resolve */ + lua_tcp_register_watcher (cbd); + if (!lua_tcp_make_connection (cbd)) { lua_pushboolean (L, FALSE); @@ -1860,6 +1841,8 @@ lua_tcp_connect_sync (lua_State *L) } } else { + cbd->item = rspamd_symcache_get_cur_item (task); + if (!make_dns_request_task (task, lua_tcp_dns_handler, cbd, RDNS_REQUEST_A, host)) { lua_pushboolean (L, FALSE); diff --git a/src/lua/lua_upstream.c b/src/lua/lua_upstream.c index 854bfafd9..1a4d6b128 100644 --- a/src/lua/lua_upstream.c +++ b/src/lua/lua_upstream.c @@ -56,6 +56,7 @@ LUA_FUNCTION_DEF (upstream_list, all_upstreams); LUA_FUNCTION_DEF (upstream_list, get_upstream_by_hash); LUA_FUNCTION_DEF (upstream_list, get_upstream_round_robin); LUA_FUNCTION_DEF (upstream_list, get_upstream_master_slave); +LUA_FUNCTION_DEF (upstream_list, add_watcher); static const struct luaL_reg upstream_list_m[] = { @@ -63,6 +64,7 @@ static const struct luaL_reg upstream_list_m[] = { LUA_INTERFACE_DEF (upstream_list, get_upstream_round_robin), LUA_INTERFACE_DEF (upstream_list, get_upstream_master_slave), LUA_INTERFACE_DEF (upstream_list, all_upstreams), + LUA_INTERFACE_DEF (upstream_list, add_watcher), {"__tostring", rspamd_lua_class_tostring}, {"__gc", lua_upstream_list_destroy}, {NULL, NULL} @@ -290,7 +292,7 @@ lua_upstream_list_get_upstream_by_hash (lua_State *L) } } else { - lua_pushnil (L); + return luaL_error (L, "invalid arguments"); } return 1; @@ -322,7 +324,7 @@ lua_upstream_list_get_upstream_round_robin (lua_State *L) } } else { - lua_pushnil (L); + return luaL_error (L, "invalid arguments"); } return 1; @@ -356,7 +358,7 @@ lua_upstream_list_get_upstream_master_slave (lua_State *L) } } else { - lua_pushnil (L); + return luaL_error (L, "invalid arguments"); } return 1; @@ -390,12 +392,173 @@ lua_upstream_list_all_upstreams (lua_State *L) rspamd_upstreams_foreach (upl, lua_upstream_inserter, L); } else { - lua_pushnil (L); + return luaL_error (L, "invalid arguments"); } return 1; } +static inline enum rspamd_upstreams_watch_event +lua_str_to_upstream_flag (const gchar *str) +{ + enum rspamd_upstreams_watch_event fl = 0; + + if (strcmp (str, "success") == 0) { + fl = RSPAMD_UPSTREAM_WATCH_SUCCESS; + } + else if (strcmp (str, "failure") == 0) { + fl = RSPAMD_UPSTREAM_WATCH_FAILURE; + } + else if (strcmp (str, "online") == 0) { + fl = RSPAMD_UPSTREAM_WATCH_ONLINE; + } + else if (strcmp (str, "offline") == 0) { + fl = RSPAMD_UPSTREAM_WATCH_OFFLINE; + } + else { + msg_err ("invalid flag: %s", str); + } + + return fl; +} + +static inline const gchar * +lua_upstream_flag_to_str (enum rspamd_upstreams_watch_event fl) +{ + const gchar *res = "unknown"; + + /* Works with single flags, not combinations */ + if (fl & RSPAMD_UPSTREAM_WATCH_SUCCESS) { + res = "success"; + } + else if (fl & RSPAMD_UPSTREAM_WATCH_FAILURE) { + res = "failure"; + } + else if (fl & RSPAMD_UPSTREAM_WATCH_ONLINE) { + res = "online"; + } + else if (fl & RSPAMD_UPSTREAM_WATCH_OFFLINE) { + res = "offline"; + } + else { + msg_err ("invalid flag: %d", fl); + } + + return res; +} + +struct rspamd_lua_upstream_watcher_cbdata { + lua_State *L; + gint cbref; + struct upstream_list *upl; +}; + +static void +lua_upstream_watch_func (struct upstream *up, + enum rspamd_upstreams_watch_event event, + guint cur_errors, + void *ud) +{ + struct rspamd_lua_upstream_watcher_cbdata *cdata = + (struct rspamd_lua_upstream_watcher_cbdata *)ud; + lua_State *L; + struct upstream **pup; + const gchar *what; + gint err_idx; + + L = cdata->L; + what = lua_upstream_flag_to_str (event); + lua_pushcfunction (L, &rspamd_lua_traceback); + err_idx = lua_gettop (L); + + lua_rawgeti (L, LUA_REGISTRYINDEX, cdata->cbref); + lua_pushstring (L, what); + pup = lua_newuserdata (L, sizeof (*pup)); + *pup = up; + rspamd_lua_setclass (L, "rspamd{upstream}", -1); + lua_pushinteger (L, cur_errors); + + if (lua_pcall (L, 3, 0, err_idx) != 0) { + GString *tb = lua_touserdata (L, -1); + msg_err ("cannot call watch function for upstream: %s", tb->str); + g_string_free (tb, TRUE); + lua_settop (L, 0); + + return; + } + + lua_settop (L, 0); +} + +static void +lua_upstream_watch_dtor (gpointer ud) +{ + struct rspamd_lua_upstream_watcher_cbdata *cdata = + (struct rspamd_lua_upstream_watcher_cbdata *)ud; + + luaL_unref (cdata->L, LUA_REGISTRYINDEX, cdata->cbref); + g_free (cdata); +} + +/*** + * @method upstream_list:add_watcher(what, cb) + * Add new watcher to the upstream lists events (table or a string): + * - `success` - called whenever upstream successfully used + * - `failure` - called on upstream error + * - `online` - called when upstream is being taken online from offline + * - `offline` - called when upstream is being taken offline from online + * Callback is a function: function(what, upstream, cur_errors) ... end + * @example +ups:add_watcher('success', function(what, up, cur_errors) ... end) +ups:add_watcher({'online', 'offline'}, function(what, up, cur_errors) ... end) + * @return nothing + */ +static gint +lua_upstream_list_add_watcher (lua_State *L) +{ + LUA_TRACE_POINT; + struct upstream_list *upl; + + upl = lua_check_upstream_list (L); + if (upl && + (lua_type (L, 2) == LUA_TTABLE || lua_type (L, 2) == LUA_TSTRING) && + lua_type (L, 3) == LUA_TFUNCTION) { + + enum rspamd_upstreams_watch_event flags = 0; + struct rspamd_lua_upstream_watcher_cbdata *cdata; + + if (lua_type (L, 2) == LUA_TSTRING) { + flags = lua_str_to_upstream_flag (lua_tostring (L, 2)); + } + else { + for (lua_pushnil (L); lua_next (L, -2); lua_pop (L, 1)) { + if (lua_isstring (L, -1)) { + flags |= lua_str_to_upstream_flag (lua_tostring (L, -1)); + } + else { + lua_pop (L, 1); + + return luaL_error (L, "invalid arguments"); + } + } + } + + cdata = g_malloc0 (sizeof (*cdata)); + lua_pushvalue (L, 3); /* callback */ + cdata->cbref = luaL_ref (L, LUA_REGISTRYINDEX); + cdata->L = L; + cdata->upl = upl; + + rspamd_upstreams_add_watch_callback (upl, flags, + lua_upstream_watch_func, lua_upstream_watch_dtor, cdata); + } + else { + return luaL_error (L, "invalid arguments"); + } + + return 0; +} + static gint lua_load_upstream_list (lua_State * L) { diff --git a/src/lua/lua_util.c b/src/lua/lua_util.c index 925cb07ca..0fd442709 100644 --- a/src/lua/lua_util.c +++ b/src/lua/lua_util.c @@ -18,7 +18,10 @@ #include "tokenizers/tokenizers.h" #include "unix-std.h" #include "contrib/zstd/zstd.h" +#include "contrib/uthash/utlist.h" #include "libmime/email_addr.h" +#include "libmime/content_type.h" +#include "libmime/mime_headers.h" #include "linenoise.h" #include <math.h> #include <glob.h> @@ -59,6 +62,14 @@ LUA_FUNCTION_DEF (util, config_from_ucl); */ LUA_FUNCTION_DEF (util, encode_base64); /*** + * @function util.encode_qp(input[, str_len, [newlines_type]]) + * Encodes data in quouted printable breaking lines if needed + * @param {text or string} input input data + * @param {number} str_len optional size of lines or 0 if split is not needed + * @return {rspamd_text} encoded data chunk + */ +LUA_FUNCTION_DEF (util, encode_qp); +/*** * @function util.decode_base64(input) * Decodes data from base64 ignoring whitespace characters * @param {text or string} input data to decode; if `rspamd{text}` is used then the string is modified **in-place** @@ -539,6 +550,29 @@ LUA_FUNCTION_DEF (util, caseless_hash_fast); */ LUA_FUNCTION_DEF (util, get_hostname); +/*** + * @function util.parse_content_type(ct_string, mempool) + * Parses content-type string to a table: + * - `type` + * - `subtype` + * - `charset` + * - `boundary` + * - other attributes + * + * @param {string} ct_string content type as string + * @param {rspamd_mempool} mempool needed to store temporary data (e.g. task pool) + * @return table or nil if cannot parse content type + */ +LUA_FUNCTION_DEF (util, parse_content_type); + +/*** + * @function util.mime_header_encode(hdr) + * Encodes header if needed + * @param {string} hdr input header + * @return encoded header + */ +LUA_FUNCTION_DEF (util, mime_header_encode); + static const struct luaL_reg utillib_f[] = { LUA_INTERFACE_DEF (util, create_event_base), @@ -546,6 +580,7 @@ static const struct luaL_reg utillib_f[] = { LUA_INTERFACE_DEF (util, config_from_ucl), LUA_INTERFACE_DEF (util, process_message), LUA_INTERFACE_DEF (util, encode_base64), + LUA_INTERFACE_DEF (util, encode_qp), LUA_INTERFACE_DEF (util, decode_base64), LUA_INTERFACE_DEF (util, encode_base32), LUA_INTERFACE_DEF (util, decode_base32), @@ -591,6 +626,8 @@ static const struct luaL_reg utillib_f[] = { LUA_INTERFACE_DEF (util, umask), LUA_INTERFACE_DEF (util, isatty), LUA_INTERFACE_DEF (util, get_hostname), + LUA_INTERFACE_DEF (util, parse_content_type), + LUA_INTERFACE_DEF (util, mime_header_encode), LUA_INTERFACE_DEF (util, pack), LUA_INTERFACE_DEF (util, unpack), LUA_INTERFACE_DEF (util, packsize), @@ -734,7 +771,7 @@ lua_util_config_from_ucl (lua_State *L) cfg->lua_state = L; cfg->rcl_obj = obj; - cfg->cache = rspamd_symbols_cache_new (cfg); + cfg->cache = rspamd_symcache_new (cfg); top = rspamd_rcl_config_init (cfg, NULL); if (!rspamd_rcl_parse (top, cfg, cfg, cfg->cfg_pool, cfg->rcl_obj, &err)) { @@ -743,6 +780,11 @@ lua_util_config_from_ucl (lua_State *L) lua_pushnil (L); } else { + + if (int_options & RSPAMD_CONFIG_INIT_LIBS) { + cfg->libs_ctx = rspamd_init_libs (); + } + rspamd_config_post_load (cfg, int_options); pcfg = lua_newuserdata (L, sizeof (struct rspamd_config *)); rspamd_lua_setclass (L, "rspamd{config}", -1); @@ -780,8 +822,7 @@ lua_util_process_message (lua_State *L) if (cfg != NULL && message != NULL) { base = event_init (); rspamd_init_filters (cfg, FALSE); - task = rspamd_task_new (NULL, cfg, NULL, NULL); - task->ev_base = base; + task = rspamd_task_new (NULL, cfg, NULL, NULL, base); task->msg.begin = rspamd_mempool_alloc (task->task_pool, mlen); rspamd_strlcpy ((gpointer)task->msg.begin, message, mlen); task->msg.len = mlen; @@ -899,6 +940,70 @@ lua_util_encode_base64 (lua_State *L) } static gint +lua_util_encode_qp (lua_State *L) +{ + LUA_TRACE_POINT; + struct rspamd_lua_text *t; + const gchar *s = NULL; + gchar *out; + gsize inlen, outlen; + guint str_lim = 0; + + if (lua_type (L, 1) == LUA_TSTRING) { + s = luaL_checklstring (L, 1, &inlen); + } + else if (lua_type (L, 1) == LUA_TUSERDATA) { + t = lua_check_text (L, 1); + + if (t != NULL) { + s = t->start; + inlen = t->len; + } + } + + if (lua_gettop (L) > 1) { + str_lim = luaL_checknumber (L, 2); + } + + if (s == NULL) { + lua_pushnil (L); + } + else { + enum rspamd_newlines_type how = RSPAMD_TASK_NEWLINES_CRLF; + + if (lua_type (L, 3) == LUA_TSTRING) { + const gchar *how_str = lua_tostring (L, 3); + + if (g_ascii_strcasecmp (how_str, "cr") == 0) { + how = RSPAMD_TASK_NEWLINES_CR; + } + else if (g_ascii_strcasecmp (how_str, "lf") == 0) { + how = RSPAMD_TASK_NEWLINES_LF; + } + else if (g_ascii_strcasecmp (how_str, "crlf") != 0) { + return luaL_error (L, "invalid newline style: %s", how_str); + } + } + + out = rspamd_encode_qp_fold (s, inlen, str_lim, &outlen, how); + + if (out != NULL) { + t = lua_newuserdata (L, sizeof (*t)); + rspamd_lua_setclass (L, "rspamd{text}", -1); + t->start = out; + t->len = outlen; + /* Need destruction */ + t->flags = RSPAMD_TEXT_FLAG_OWN; + } + else { + lua_pushnil (L); + } + } + + return 1; +} + +static gint lua_util_decode_base64 (lua_State *L) { LUA_TRACE_POINT; @@ -1116,7 +1221,7 @@ lua_util_tokenize_text (lua_State *L) ex = g_malloc0 (sizeof (*ex)); ex->pos = pos; ex->len = ex_len; - ex->type = RSPAMD_EXCEPTION_URL; + ex->type = RSPAMD_EXCEPTION_GENERIC; exceptions = g_list_prepend (exceptions, ex); } } @@ -1140,7 +1245,7 @@ lua_util_tokenize_text (lua_State *L) &utxt, RSPAMD_TOKENIZE_UTF, NULL, exceptions, - NULL); + NULL, NULL); if (res == NULL) { lua_pushnil (L); @@ -1150,7 +1255,7 @@ lua_util_tokenize_text (lua_State *L) for (i = 0; i < res->len; i ++) { w = &g_array_index (res, rspamd_stat_token_t, i); - lua_pushlstring (L, w->begin, w->len); + lua_pushlstring (L, w->original.begin, w->original.len); lua_rawseti (L, -2, i + 1); } } @@ -1873,7 +1978,7 @@ lua_util_create_file (lua_State *L) mode = lua_tointeger (L, 2); } - fd = rspamd_file_xopen (fpath, O_RDWR | O_CREAT | O_EXCL, mode, 0); + fd = rspamd_file_xopen (fpath, O_RDWR | O_CREAT | O_TRUNC, mode, 0); if (fd == -1) { lua_pushnil (L); @@ -2432,6 +2537,98 @@ lua_util_get_hostname (lua_State *L) } static gint +lua_util_parse_content_type (lua_State *L) +{ + LUA_TRACE_POINT; + gsize len; + const gchar *ct_str = luaL_checklstring (L, 1, &len); + rspamd_mempool_t *pool = rspamd_lua_check_mempool (L, 2); + struct rspamd_content_type *ct; + + if (!ct_str || !pool) { + return luaL_error (L, "invalid arguments"); + } + + ct = rspamd_content_type_parse (ct_str, len, pool); + + if (ct == NULL) { + lua_pushnil (L); + } + else { + GHashTableIter it; + gpointer k, v; + + lua_createtable (L, 0, 4 + (ct->attrs ? g_hash_table_size (ct->attrs) : 0)); + + if (ct->type.len > 0) { + lua_pushstring (L, "type"); + lua_pushlstring (L, ct->type.begin, ct->type.len); + lua_settable (L, -3); + } + + if (ct->subtype.len > 0) { + lua_pushstring (L, "subtype"); + lua_pushlstring (L, ct->subtype.begin, ct->subtype.len); + lua_settable (L, -3); + } + + if (ct->charset.len > 0) { + lua_pushstring (L, "charset"); + lua_pushlstring (L, ct->charset.begin, ct->charset.len); + lua_settable (L, -3); + } + + if (ct->orig_boundary.len > 0) { + lua_pushstring (L, "boundary"); + lua_pushlstring (L, ct->orig_boundary.begin, ct->orig_boundary.len); + lua_settable (L, -3); + } + + if (ct->attrs) { + g_hash_table_iter_init (&it, ct->attrs); + + while (g_hash_table_iter_next (&it, &k, &v)) { + struct rspamd_content_type_param *param = + (struct rspamd_content_type_param *)v, *cur; + guint i = 1; + + lua_pushlstring (L, param->name.begin, param->name.len); + lua_createtable (L, 1, 0); + + DL_FOREACH (param, cur) { + lua_pushlstring (L, cur->value.begin, cur->value.len); + lua_rawseti (L, -2, i++); + } + + lua_settable (L, -3); + } + } + } + + return 1; +} + + +static gint +lua_util_mime_header_encode (lua_State *L) +{ + LUA_TRACE_POINT; + gsize len; + const gchar *hdr = luaL_checklstring (L, 1, &len); + gchar *encoded; + + if (!hdr) { + return luaL_error (L, "invalid arguments"); + } + + encoded = rspamd_mime_header_encode (hdr, len); + lua_pushstring (L, encoded); + g_free (encoded); + + return 1; +} + +static gint lua_util_is_valid_utf8 (lua_State *L) { LUA_TRACE_POINT; @@ -2454,7 +2651,7 @@ static gint lua_util_readline (lua_State *L) { LUA_TRACE_POINT; - const gchar *prompt = NULL; + const gchar *prompt = ""; gchar *input; if (lua_type (L, 1) == LUA_TSTRING) { diff --git a/src/plugins/chartable.c b/src/plugins/chartable.c index 414647153..b6e42457a 100644 --- a/src/plugins/chartable.c +++ b/src/plugins/chartable.c @@ -85,8 +85,12 @@ chartable_get_context (struct rspamd_config *cfg) chartable_module.ctx_offset); } -static void chartable_symbol_callback (struct rspamd_task *task, void *unused); -static void chartable_url_symbol_callback (struct rspamd_task *task, void *unused); +static void chartable_symbol_callback (struct rspamd_task *task, + struct rspamd_symcache_item *item, + void *unused); +static void chartable_url_symbol_callback (struct rspamd_task *task, + struct rspamd_symcache_item *item, + void *unused); gint chartable_module_init (struct rspamd_config *cfg, struct module_ctx **ctx) @@ -146,14 +150,14 @@ chartable_module_config (struct rspamd_config *cfg) chartable_module_ctx->threshold = DEFAULT_THRESHOLD; } - rspamd_symbols_cache_add_symbol (cfg->cache, + rspamd_symcache_add_symbol (cfg->cache, chartable_module_ctx->symbol, 0, chartable_symbol_callback, NULL, SYMBOL_TYPE_NORMAL, -1); - rspamd_symbols_cache_add_symbol (cfg->cache, + rspamd_symcache_add_symbol (cfg->cache, chartable_module_ctx->url_symbol, 0, chartable_url_symbol_callback, @@ -354,12 +358,12 @@ rspamd_chartable_process_word_utf (struct rspamd_task *task, guint *ncap, struct chartable_ctx *chartable_module_ctx) { - const gchar *p, *end; + const UChar32 *p, *end; gdouble badness = 0.0; UChar32 uc; UBlockCode sc; gint last_is_latin = -1; - guint same_script_count = 0, nsym = 0, i = 0; + guint same_script_count = 0, nsym = 0; enum { start_process = 0, got_alpha, @@ -367,13 +371,13 @@ rspamd_chartable_process_word_utf (struct rspamd_task *task, got_unknown, } state = start_process, prev_state = start_process; - p = w->begin; - end = p + w->len; + p = w->unicode.begin; + end = p + w->unicode.len; /* We assume that w is normalized */ - while (p + i < end) { - U8_NEXT (p, i, w->len, uc); + while (p < end) { + uc = *p++; if (((gint32)uc) < 0) { break; @@ -460,7 +464,8 @@ rspamd_chartable_process_word_utf (struct rspamd_task *task, } } - msg_debug_chartable ("word %*s, badness: %.2f", (gint)w->len, w->begin, + msg_debug_chartable ("word %*s, badness: %.2f", + (gint)w->normalized.len, w->normalized.begin, badness); return badness; @@ -486,11 +491,11 @@ rspamd_chartable_process_word_ascii (struct rspamd_task *task, got_unknown, } state = start_process; - p = w->begin; - end = p + w->len; + p = w->normalized.begin; + end = p + w->normalized.len; last_sc = 0; - if (w->len > chartable_module_ctx->max_word_len) { + if (w->normalized.len > chartable_module_ctx->max_word_len) { return 0.0; } @@ -545,7 +550,8 @@ rspamd_chartable_process_word_ascii (struct rspamd_task *task, badness = 4.0; } - msg_debug_chartable ("word %*s, badness: %.2f", (gint)w->len, w->begin, + msg_debug_chartable ("word %*s, badness: %.2f", + (gint)w->normalized.len, w->normalized.begin, badness); return badness; @@ -568,9 +574,9 @@ rspamd_chartable_process_part (struct rspamd_task *task, for (i = 0; i < part->utf_words->len; i++) { w = &g_array_index (part->utf_words, rspamd_stat_token_t, i); - if (w->len > 0 && (w->flags & RSPAMD_STAT_TOKEN_FLAG_TEXT)) { + if ((w->flags & RSPAMD_STAT_TOKEN_FLAG_TEXT)) { - if (IS_PART_UTF (part)) { + if (w->flags & RSPAMD_STAT_TOKEN_FLAG_UTF) { cur_score += rspamd_chartable_process_word_utf (task, w, FALSE, &ncap, chartable_module_ctx); } @@ -588,7 +594,7 @@ rspamd_chartable_process_part (struct rspamd_task *task, */ part->capital_letters += ncap; - cur_score /= (gdouble)part->utf_words->len; + cur_score /= (gdouble)part->nwords; if (cur_score > 2.0) { cur_score = 2.0; @@ -602,7 +608,9 @@ rspamd_chartable_process_part (struct rspamd_task *task, } static void -chartable_symbol_callback (struct rspamd_task *task, void *unused) +chartable_symbol_callback (struct rspamd_task *task, + struct rspamd_symcache_item *item, + void *unused) { guint i; struct rspamd_mime_text_part *part; @@ -613,59 +621,40 @@ chartable_symbol_callback (struct rspamd_task *task, void *unused) rspamd_chartable_process_part (task, part, chartable_module_ctx); } - if (task->subject != NULL) { - GArray *words; + if (task->meta_words != NULL) { rspamd_stat_token_t *w; - guint i; - gdouble cur_score = 0.0; - - UText utxt = UTEXT_INITIALIZER; - UErrorCode uc_err = U_ZERO_ERROR; - gsize slen = strlen (task->subject); - - utext_openUTF8 (&utxt, - task->subject, - slen, - &uc_err); - - words = rspamd_tokenize_text (task->subject, slen, - &utxt, - RSPAMD_TOKENIZE_UTF, - NULL, - NULL, - NULL); - - if (words && words->len > 0) { - for (i = 0; i < words->len; i++) { - w = &g_array_index (words, rspamd_stat_token_t, i); - cur_score += rspamd_chartable_process_word_utf (task, w, FALSE, - NULL, chartable_module_ctx); - } + gdouble cur_score = 0; + gsize arlen = task->meta_words->len; - cur_score /= (gdouble)words->len; - - if (cur_score > 2.0) { - cur_score = 2.0; - } + for (i = 0; i < arlen; i++) { + w = &g_array_index (task->meta_words, rspamd_stat_token_t, i); + cur_score += rspamd_chartable_process_word_utf (task, w, FALSE, + NULL, chartable_module_ctx); + } - if (cur_score > chartable_module_ctx->threshold) { - rspamd_task_insert_result (task, chartable_module_ctx->symbol, - cur_score, "subject"); + cur_score /= (gdouble)arlen; - } + if (cur_score > 2.0) { + cur_score = 2.0; } - if (words) { - g_array_free (words, TRUE); - } + if (cur_score > chartable_module_ctx->threshold) { + rspamd_task_insert_result (task, chartable_module_ctx->symbol, + cur_score, "subject"); - utext_close (&utxt); + } } + + rspamd_symcache_finalize_item (task, item); } static void -chartable_url_symbol_callback (struct rspamd_task *task, void *unused) +chartable_url_symbol_callback (struct rspamd_task *task, + struct rspamd_symcache_item *item, + void *unused) { + /* XXX: TODO: unbreak module once URLs unicode project is over */ +#if 0 struct rspamd_url *u; GHashTableIter it; gpointer k, v; @@ -684,10 +673,10 @@ chartable_url_symbol_callback (struct rspamd_task *task, void *unused) } if (u->hostlen > 0) { - w.begin = u->host; - w.len = u->hostlen; + w.stemmed.begin = u->host; + w.stemmed.len = u->hostlen; - if (g_utf8_validate (w.begin, w.len, NULL)) { + if (g_utf8_validate (w.stemmed.begin, w.stemmed.len, NULL)) { cur_score += rspamd_chartable_process_word_utf (task, &w, TRUE, NULL, chartable_module_ctx); } @@ -709,10 +698,10 @@ chartable_url_symbol_callback (struct rspamd_task *task, void *unused) } if (u->hostlen > 0) { - w.begin = u->host; - w.len = u->hostlen; + w.stemmed.begin = u->host; + w.stemmed.len = u->hostlen; - if (g_utf8_validate (w.begin, w.len, NULL)) { + if (g_utf8_validate (w.stemmed.begin, w.stemmed.len, NULL)) { cur_score += rspamd_chartable_process_word_utf (task, &w, TRUE, NULL, chartable_module_ctx); } @@ -728,4 +717,6 @@ chartable_url_symbol_callback (struct rspamd_task *task, void *unused) cur_score, NULL); } +#endif + rspamd_symcache_finalize_item (task, item); } diff --git a/src/plugins/dkim_check.c b/src/plugins/dkim_check.c index 1784612f0..6ea567178 100644 --- a/src/plugins/dkim_check.c +++ b/src/plugins/dkim_check.c @@ -50,6 +50,8 @@ #define DEFAULT_TIME_JITTER 60 #define DEFAULT_MAX_SIGS 5 +static const gchar *M = "rspamd dkim plugin"; + static const gchar default_sign_headers[] = "" "(o)from:(o)sender:(o)reply-to:(o)subject:(o)date:(o)message-id:" "(o)to:(o)cc:(o)mime-version:(o)content-type:(o)content-transfer-encoding:" @@ -83,15 +85,19 @@ struct dkim_check_result { rspamd_dkim_context_t *ctx; rspamd_dkim_key_t *key; struct rspamd_task *task; - gint res; + struct rspamd_dkim_check_result *res; gdouble mult_allow; gdouble mult_deny; - struct rspamd_async_watcher *w; + struct rspamd_symcache_item *item; struct dkim_check_result *next, *prev, *first; }; -static void dkim_symbol_callback (struct rspamd_task *task, void *unused); -static void dkim_sign_callback (struct rspamd_task *task, void *unused); +static void dkim_symbol_callback (struct rspamd_task *task, + struct rspamd_symcache_item *item, + void *unused); +static void dkim_sign_callback (struct rspamd_task *task, + struct rspamd_symcache_item *item, + void *unused); static gint lua_dkim_sign_handler (lua_State *L); static gint lua_dkim_verify_handler (lua_State *L); @@ -498,50 +504,50 @@ dkim_module_config (struct rspamd_config *cfg) return TRUE; } - cb_id = rspamd_symbols_cache_add_symbol (cfg->cache, + cb_id = rspamd_symcache_add_symbol (cfg->cache, "DKIM_CHECK", 0, dkim_symbol_callback, NULL, SYMBOL_TYPE_CALLBACK, -1); - rspamd_symbols_cache_add_symbol (cfg->cache, + rspamd_symcache_add_symbol (cfg->cache, dkim_module_ctx->symbol_reject, 0, NULL, NULL, - SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE, + SYMBOL_TYPE_VIRTUAL | SYMBOL_TYPE_FINE, cb_id); - rspamd_symbols_cache_add_symbol (cfg->cache, + rspamd_symcache_add_symbol (cfg->cache, dkim_module_ctx->symbol_na, 0, NULL, NULL, - SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE, + SYMBOL_TYPE_VIRTUAL | SYMBOL_TYPE_FINE, cb_id); - rspamd_symbols_cache_add_symbol (cfg->cache, + rspamd_symcache_add_symbol (cfg->cache, dkim_module_ctx->symbol_permfail, 0, NULL, NULL, - SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE, + SYMBOL_TYPE_VIRTUAL | SYMBOL_TYPE_FINE, cb_id); - rspamd_symbols_cache_add_symbol (cfg->cache, + rspamd_symcache_add_symbol (cfg->cache, dkim_module_ctx->symbol_tempfail, 0, NULL, NULL, - SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE, + SYMBOL_TYPE_VIRTUAL | SYMBOL_TYPE_FINE, cb_id); - rspamd_symbols_cache_add_symbol (cfg->cache, + rspamd_symcache_add_symbol (cfg->cache, dkim_module_ctx->symbol_allow, 0, NULL, NULL, - SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE, + SYMBOL_TYPE_VIRTUAL | SYMBOL_TYPE_FINE, cb_id); - rspamd_symbols_cache_add_symbol (cfg->cache, + rspamd_symcache_add_symbol (cfg->cache, "DKIM_TRACE", 0, NULL, NULL, - SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_NOSTAT, + SYMBOL_TYPE_VIRTUAL | SYMBOL_TYPE_NOSTAT, cb_id); rspamd_config_add_symbol (cfg, "DKIM_TRACE", @@ -579,12 +585,12 @@ dkim_module_config (struct rspamd_config *cfg) cfg->cfg_pool, dkim_module_ctx->sign_condition_ref); - rspamd_symbols_cache_add_symbol (cfg->cache, + rspamd_symcache_add_symbol (cfg->cache, "DKIM_SIGN", 0, dkim_sign_callback, NULL, - SYMBOL_TYPE_CALLBACK|SYMBOL_TYPE_FINE, + SYMBOL_TYPE_CALLBACK | SYMBOL_TYPE_FINE, -1); msg_info_config ("init condition script for DKIM signing"); @@ -592,7 +598,7 @@ dkim_module_config (struct rspamd_config *cfg) * Allow dkim signing to be executed only after dkim check */ if (cb_id > 0) { - rspamd_symbols_cache_add_delayed_dependency (cfg->cache, + rspamd_symcache_add_delayed_dependency (cfg->cache, "DKIM_SIGN", dkim_module_ctx->symbol_reject); } @@ -816,7 +822,7 @@ lua_dkim_sign_handler (lua_State *L) if (pubkey != NULL) { /* Also check if private and public keys match */ rspamd_dkim_key_t *pk; - gsize keylen = strlen (pubkey); + keylen = strlen (pubkey); pk = rspamd_dkim_parse_key (pubkey, &keylen, NULL); @@ -942,6 +948,7 @@ dkim_module_check (struct dkim_check_result *res) const gchar *strict_value; struct dkim_check_result *first, *cur = NULL; struct dkim_ctx *dkim_module_ctx = dkim_get_context (res->task->cfg); + struct rspamd_task *task = res->task; first = res->first; @@ -950,8 +957,8 @@ dkim_module_check (struct dkim_check_result *res) continue; } - if (cur->key != NULL && cur->res == -1) { - cur->res = rspamd_dkim_check (cur->ctx, cur->key, cur->task); + if (cur->key != NULL && cur->res == NULL) { + cur->res = rspamd_dkim_check (cur->ctx, cur->key, task); if (dkim_module_ctx->dkim_domains != NULL) { /* Perform strict check */ @@ -972,60 +979,85 @@ dkim_module_check (struct dkim_check_result *res) if (cur->ctx == NULL) { continue; } - if (cur->res == -1) { + if (cur->res == NULL) { /* Still need a key */ all_done = FALSE; } } if (all_done) { + /* Create zero terminated array of results */ + struct rspamd_dkim_check_result **pres; + guint nres = 0, i = 0; + + DL_FOREACH (first, cur) { + if (cur->ctx == NULL || cur->res == NULL) { + continue; + } + + nres ++; + } + + pres = rspamd_mempool_alloc (task->task_pool, sizeof (*pres) * (nres + 1)); + pres[nres] = NULL; + DL_FOREACH (first, cur) { const gchar *symbol = NULL, *trace = NULL; gdouble symbol_weight = 1.0; - if (cur->ctx == NULL) { + if (cur->ctx == NULL || cur->res == NULL) { continue; } - if (cur->res == DKIM_REJECT) { + + pres[i++] = cur->res; + + if (cur->res->rcode == DKIM_REJECT) { symbol = dkim_module_ctx->symbol_reject; trace = "-"; symbol_weight = cur->mult_deny * 1.0; } - else if (cur->res == DKIM_CONTINUE) { + else if (cur->res->rcode == DKIM_CONTINUE) { symbol = dkim_module_ctx->symbol_allow; trace = "+"; symbol_weight = cur->mult_allow * 1.0; } - else if (cur->res == DKIM_PERM_ERROR) { + else if (cur->res->rcode == DKIM_PERM_ERROR) { trace = "~"; symbol = dkim_module_ctx->symbol_permfail; } - else if (cur->res == DKIM_TRYAGAIN) { + else if (cur->res->rcode == DKIM_TRYAGAIN) { trace = "?"; symbol = dkim_module_ctx->symbol_tempfail; } if (symbol != NULL) { const gchar *domain = rspamd_dkim_get_domain (cur->ctx); + const gchar *selector = rspamd_dkim_get_selector (cur->ctx); gsize tracelen; gchar *tracebuf; - tracelen = strlen (domain) + 3; /* :<trace>\0 */ - tracebuf = rspamd_mempool_alloc (cur->task->task_pool, + tracelen = strlen (domain) + strlen (selector) + 4; + tracebuf = rspamd_mempool_alloc (task->task_pool, tracelen); rspamd_snprintf (tracebuf, tracelen, "%s:%s", domain, trace); rspamd_task_insert_result (cur->task, - symbol, - symbol_weight, - domain); - rspamd_task_insert_result (cur->task, "DKIM_TRACE", 0.0, tracebuf); + + rspamd_snprintf (tracebuf, tracelen, "%s:s=%s", domain, selector); + rspamd_task_insert_result (task, + symbol, + symbol_weight, + tracebuf); } + } - rspamd_session_watcher_pop (res->task->s, res->w); + + rspamd_mempool_set_variable (task->task_pool, + RSPAMD_MEMPOOL_DKIM_CHECK_RESULTS, + pres, NULL); } } @@ -1064,10 +1096,12 @@ dkim_module_key_handler (rspamd_dkim_key_t *key, if (err != NULL) { if (err->code == DKIM_SIGERROR_NOKEY) { - res->res = DKIM_TRYAGAIN; + res->res = rspamd_dkim_create_result (ctx, DKIM_TRYAGAIN, task); + res->res->fail_reason = "DNS error when getting key"; } else { - res->res = DKIM_PERM_ERROR; + res->res = rspamd_dkim_create_result (ctx, DKIM_PERM_ERROR, task); + res->res->fail_reason = "invalid DKIM record"; } } } @@ -1080,7 +1114,9 @@ dkim_module_key_handler (rspamd_dkim_key_t *key, } static void -dkim_symbol_callback (struct rspamd_task *task, void *unused) +dkim_symbol_callback (struct rspamd_task *task, + struct rspamd_symcache_item *item, + void *unused) { GPtrArray *hlist; rspamd_dkim_context_t *ctx; @@ -1112,15 +1148,21 @@ dkim_symbol_callback (struct rspamd_task *task, void *unused) || (!dkim_module_ctx->check_local && rspamd_inet_address_is_local (task->from_addr, TRUE))) { msg_info_task ("skip DKIM checks for local networks and authorized users"); + rspamd_symcache_finalize_item (task, item); + return; } /* Check whitelist */ if (rspamd_match_radix_map_addr (dkim_module_ctx->whitelist_ip, task->from_addr) != NULL) { msg_info_task ("skip DKIM checks for whitelisted address"); + rspamd_symcache_finalize_item (task, item); + return; } + rspamd_symcache_item_async_inc (task, item, M); + /* Now check if a message has its signature */ hlist = rspamd_message_get_header_array (task, RSPAMD_DKIM_SIGNHEADER, @@ -1137,10 +1179,11 @@ dkim_symbol_callback (struct rspamd_task *task, void *unused) cur = rspamd_mempool_alloc0 (task->task_pool, sizeof (*cur)); cur->first = res; - cur->res = -1; + cur->res = NULL; cur->task = task; cur->mult_allow = 1.0; cur->mult_deny = 1.0; + cur->item = item; ctx = rspamd_create_dkim_context (rh->decoded, task->task_pool, @@ -1148,6 +1191,15 @@ dkim_symbol_callback (struct rspamd_task *task, void *unused) RSPAMD_DKIM_NORMAL, &err); + if (res == NULL) { + res = cur; + res->first = res; + res->prev = res; + } + else { + DL_APPEND (res, cur); + } + if (ctx == NULL) { if (err != NULL) { msg_info_task ("<%s> cannot parse DKIM context: %e", @@ -1197,17 +1249,6 @@ dkim_symbol_callback (struct rspamd_task *task, void *unused) } } - if (res == NULL) { - res = cur; - res->first = res; - res->prev = res; - res->w = rspamd_session_get_watcher (task->s); - } - else { - cur->w = res->w; - DL_APPEND (res, cur); - } - checked ++; if (checked > dkim_module_ctx->max_sigs) { @@ -1226,13 +1267,16 @@ dkim_symbol_callback (struct rspamd_task *task, void *unused) } if (res != NULL) { - rspamd_session_watcher_push (task->s); dkim_module_check (res); } + + rspamd_symcache_item_async_dec_check (task, item, M); } static void -dkim_sign_callback (struct rspamd_task *task, void *unused) +dkim_sign_callback (struct rspamd_task *task, + struct rspamd_symcache_item *item, + void *unused) { lua_State *L; struct rspamd_task **ptask; @@ -1285,6 +1329,7 @@ dkim_sign_callback (struct rspamd_task *task, void *unused) msg_err_task ("invalid return value from sign condition: %e", err); g_error_free (err); + rspamd_symcache_finalize_item (task, item); return; } @@ -1307,6 +1352,7 @@ dkim_sign_callback (struct rspamd_task *task, void *unused) lua_settop (L, 0); luaL_error (L, "unknown key type: %s", key_type); + rspamd_symcache_finalize_item (task, item); return; } @@ -1321,6 +1367,7 @@ dkim_sign_callback (struct rspamd_task *task, void *unused) if (arc_idx == 0) { lua_settop (L, 0); luaL_error (L, "no arc idx specified"); + rspamd_symcache_finalize_item (task, item); return; } @@ -1330,12 +1377,14 @@ dkim_sign_callback (struct rspamd_task *task, void *unused) if (arc_cv == NULL) { lua_settop (L, 0); luaL_error (L, "no arc cv specified"); + rspamd_symcache_finalize_item (task, item); return; } if (arc_idx == 0) { lua_settop (L, 0); luaL_error (L, "no arc idx specified"); + rspamd_symcache_finalize_item (task, item); return; } @@ -1344,6 +1393,7 @@ dkim_sign_callback (struct rspamd_task *task, void *unused) lua_settop (L, 0); luaL_error (L, "unknown sign type: %s", sign_type_str); + rspamd_symcache_finalize_item (task, item); return; } @@ -1376,6 +1426,7 @@ dkim_sign_callback (struct rspamd_task *task, void *unused) msg_err_task ("cannot load dkim key %s: %e", lru_key, err); g_error_free (err); + rspamd_symcache_finalize_item (task, item); return; } @@ -1400,6 +1451,7 @@ dkim_sign_callback (struct rspamd_task *task, void *unused) msg_err_task ("cannot load dkim key %s: %e", lru_key, err); g_error_free (err); + rspamd_symcache_finalize_item (task, item); return; } @@ -1419,6 +1471,7 @@ dkim_sign_callback (struct rspamd_task *task, void *unused) msg_err_task ("cannot create sign context: %e", err); g_error_free (err); + rspamd_symcache_finalize_item (task, item); return; } @@ -1446,9 +1499,13 @@ dkim_sign_callback (struct rspamd_task *task, void *unused) if (!sign) { msg_debug_task ("skip signing as dkim condition callback returned" " false"); + rspamd_symcache_finalize_item (task, item); + return; } } + + rspamd_symcache_finalize_item (task, item); } struct rspamd_dkim_lua_verify_cbdata { @@ -1461,7 +1518,7 @@ struct rspamd_dkim_lua_verify_cbdata { static void dkim_module_lua_push_verify_result (struct rspamd_dkim_lua_verify_cbdata *cbd, - gint code, GError *err) + struct rspamd_dkim_check_result *res, GError *err) { struct rspamd_task **ptask, *task; const gchar *error_str = "unknown error"; @@ -1469,7 +1526,7 @@ dkim_module_lua_push_verify_result (struct rspamd_dkim_lua_verify_cbdata *cbd, task = cbd->task; - switch (code) { + switch (res->rcode) { case DKIM_CONTINUE: error_str = NULL; success = TRUE; @@ -1525,13 +1582,19 @@ dkim_module_lua_push_verify_result (struct rspamd_dkim_lua_verify_cbdata *cbd, lua_pushstring (cbd->L, error_str); if (cbd->ctx) { - lua_pushstring (cbd->L, rspamd_dkim_get_domain (cbd->ctx)); + lua_pushstring (cbd->L, res->domain); + lua_pushstring (cbd->L, res->selector); + lua_pushstring (cbd->L, res->short_b); + lua_pushstring (cbd->L, res->fail_reason); } else { lua_pushnil (cbd->L); + lua_pushnil (cbd->L); + lua_pushnil (cbd->L); + lua_pushnil (cbd->L); } - if (lua_pcall (cbd->L, 4, 0, 0) != 0) { + if (lua_pcall (cbd->L, 7, 0, 0) != 0) { msg_err_task ("call to verify callback failed: %s", lua_tostring (cbd->L, -1)); lua_pop (cbd->L, 1); @@ -1542,14 +1605,14 @@ dkim_module_lua_push_verify_result (struct rspamd_dkim_lua_verify_cbdata *cbd, static void dkim_module_lua_on_key (rspamd_dkim_key_t *key, - gsize keylen, - rspamd_dkim_context_t *ctx, - gpointer ud, - GError *err) + gsize keylen, + rspamd_dkim_context_t *ctx, + gpointer ud, + GError *err) { struct rspamd_dkim_lua_verify_cbdata *cbd = ud; struct rspamd_task *task; - gint ret; + struct rspamd_dkim_check_result *res; struct dkim_ctx *dkim_module_ctx; task = cbd->task; @@ -1576,16 +1639,22 @@ dkim_module_lua_on_key (rspamd_dkim_key_t *key, if (err != NULL) { if (err->code == DKIM_SIGERROR_NOKEY) { - dkim_module_lua_push_verify_result (cbd, DKIM_TRYAGAIN, err); + res = rspamd_dkim_create_result (ctx, DKIM_TRYAGAIN, task); + res->fail_reason = "DNS error when getting key"; + } else { - dkim_module_lua_push_verify_result (cbd, DKIM_PERM_ERROR, err); + res = rspamd_dkim_create_result (ctx, DKIM_PERM_ERROR, task); + res->fail_reason = "invalid DKIM record"; } } else { - dkim_module_lua_push_verify_result (cbd, DKIM_TRYAGAIN, NULL); + res = rspamd_dkim_create_result (ctx, DKIM_TRYAGAIN, task); + res->fail_reason = "DNS error when getting key"; } + dkim_module_lua_push_verify_result (cbd, res, err); + if (err) { g_error_free (err); } @@ -1593,8 +1662,8 @@ dkim_module_lua_on_key (rspamd_dkim_key_t *key, return; } - ret = rspamd_dkim_check (cbd->ctx, cbd->key, cbd->task); - dkim_module_lua_push_verify_result (cbd, ret, NULL); + res = rspamd_dkim_check (cbd->ctx, cbd->key, cbd->task); + dkim_module_lua_push_verify_result (cbd, res, NULL); } static gint @@ -1605,7 +1674,7 @@ lua_dkim_verify_handler (lua_State *L) rspamd_dkim_context_t *ctx; struct rspamd_dkim_lua_verify_cbdata *cbd; rspamd_dkim_key_t *key; - gint ret; + struct rspamd_dkim_check_result *ret; GError *err = NULL; const gchar *type_str = NULL; enum rspamd_dkim_type type = RSPAMD_DKIM_NORMAL; diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c index 23aeacb66..bfb6b4d72 100644 --- a/src/plugins/fuzzy_check.c +++ b/src/plugins/fuzzy_check.c @@ -45,11 +45,9 @@ #include "libutil/http_private.h" #include "libstat/stat_api.h" #include <math.h> +#include <src/libmime/message.h> #define DEFAULT_SYMBOL "R_FUZZY_HASH" -#define DEFAULT_UPSTREAM_ERROR_TIME 10 -#define DEFAULT_UPSTREAM_DEAD_TIME 300 -#define DEFAULT_UPSTREAM_MAXERRORS 10 #define DEFAULT_IO_TIMEOUT 500 #define DEFAULT_RETRANSMITS 3 @@ -58,6 +56,7 @@ #define RSPAMD_FUZZY_PLUGIN_VERSION RSPAMD_FUZZY_VERSION static const gint rspamd_fuzzy_hash_len = 5; +static const gchar *M = "fuzzy check"; struct fuzzy_ctx; struct fuzzy_mapping { @@ -66,33 +65,26 @@ struct fuzzy_mapping { double weight; }; -struct fuzzy_mime_type { - rspamd_regexp_t *type_re; - rspamd_regexp_t *subtype_re; -}; - struct fuzzy_rule { struct upstream_list *servers; const gchar *symbol; const gchar *algorithm_str; const gchar *name; + const ucl_object_t *ucl_obj; enum rspamd_shingle_alg alg; GHashTable *mappings; - GPtrArray *mime_types; GPtrArray *fuzzy_headers; GString *hash_key; GString *shingles_key; struct rspamd_cryptobox_keypair *local_key; struct rspamd_cryptobox_pubkey *peer_key; double max_score; - guint32 min_bytes; gboolean read_only; gboolean skip_unknown; - gboolean fuzzy_images; - gboolean short_text_direct_hash; gint learn_condition_cb; struct rspamd_hash_map_helper *skip_map; struct fuzzy_ctx *ctx; + gint lua_id; }; struct fuzzy_ctx { @@ -101,15 +93,13 @@ struct fuzzy_ctx { GPtrArray *fuzzy_rules; struct rspamd_config *cfg; const gchar *default_symbol; - guint32 min_hash_len; struct rspamd_radix_map_helper *whitelist; struct rspamd_keypair_cache *keypairs_cache; - gdouble text_multiplier; - guint32 min_bytes; - guint32 min_height; - guint32 min_width; guint32 io_timeout; guint32 retransmits; + gint check_mime_part_ref; /* Lua callback */ + gint process_rule_ref; /* Lua callback */ + gint cleanup_rules_ref; gboolean enabled; }; @@ -131,8 +121,8 @@ struct fuzzy_client_session { GPtrArray *commands; GPtrArray *results; struct rspamd_task *task; + struct rspamd_symcache_item *item; struct upstream *server; - rspamd_inet_addr_t *addr; struct fuzzy_rule *rule; struct event ev; struct event timev; @@ -149,7 +139,6 @@ struct fuzzy_learn_session { struct rspamd_http_connection_entry *http_entry; struct rspamd_async_session *session; struct upstream *server; - rspamd_inet_addr_t *addr; struct fuzzy_rule *rule; struct rspamd_task *task; struct event ev; @@ -170,14 +159,17 @@ struct fuzzy_learn_session { struct fuzzy_cmd_io { guint32 tag; guint32 flags; - struct rspamd_fuzzy_cmd cmd; struct iovec io; + struct rspamd_mime_part *part; + struct rspamd_fuzzy_cmd cmd; }; static const char *default_headers = "Subject,Content-Type,Reply-To,X-Mailer"; -static void fuzzy_symbol_callback (struct rspamd_task *task, void *unused); +static void fuzzy_symbol_callback (struct rspamd_task *task, + struct rspamd_symcache_item *item, + void *unused); /* Initialization */ gint fuzzy_check_module_init (struct rspamd_config *cfg, @@ -246,10 +238,10 @@ parse_flags (struct fuzzy_rule *rule, /* Add flag to hash table */ g_hash_table_insert (rule->mappings, GINT_TO_POINTER (map->fuzzy_flag), map); - rspamd_symbols_cache_add_symbol (cfg->cache, + rspamd_symcache_add_symbol (cfg->cache, map->symbol, 0, NULL, NULL, - SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE, + SYMBOL_TYPE_VIRTUAL | SYMBOL_TYPE_FINE, cb_id); } else { @@ -265,53 +257,6 @@ parse_flags (struct fuzzy_rule *rule, } } - -static GPtrArray * -parse_mime_types (struct rspamd_config *cfg, const gchar *str) -{ - gchar **strvec, *p; - gint num, i; - struct fuzzy_mime_type *type; - GPtrArray *res; - - strvec = g_strsplit_set (str, ",", 0); - num = g_strv_length (strvec); - res = g_ptr_array_sized_new (num); - - for (i = 0; i < num; i++) { - g_strstrip (strvec[i]); - - if ((p = strchr (strvec[i], '/')) != NULL) { - type = rspamd_mempool_alloc (cfg->cfg_pool, - sizeof (struct fuzzy_mime_type)); - type->type_re = rspamd_regexp_from_glob (strvec[i], p - strvec[i], - NULL); - type->subtype_re = rspamd_regexp_from_glob (p + 1, 0, NULL); - rspamd_mempool_add_destructor (cfg->cfg_pool, - (rspamd_mempool_destruct_t)rspamd_regexp_unref, - type->type_re); - rspamd_mempool_add_destructor (cfg->cfg_pool, - (rspamd_mempool_destruct_t)rspamd_regexp_unref, - type->subtype_re); - g_ptr_array_add (res, type); - } - else { - type = rspamd_mempool_alloc (cfg->cfg_pool, - sizeof (struct fuzzy_mime_type)); - type->type_re = rspamd_regexp_from_glob (strvec[i], 0, NULL); - rspamd_mempool_add_destructor (cfg->cfg_pool, - (rspamd_mempool_destruct_t)rspamd_regexp_unref, - type->type_re); - type->subtype_re = NULL; - g_ptr_array_add (res, type); - } - } - - g_strfreev (strvec); - - return res; -} - static GPtrArray * parse_fuzzy_headers (struct rspamd_config *cfg, const gchar *str) { @@ -334,39 +279,6 @@ parse_fuzzy_headers (struct rspamd_config *cfg, const gchar *str) return res; } -static gboolean -fuzzy_check_content_type (struct fuzzy_rule *rule, struct rspamd_content_type *ct) -{ - struct fuzzy_mime_type *ft; - guint i; - - PTR_ARRAY_FOREACH (rule->mime_types, i, ft) { - if (ft->type_re) { - - if (ct->type.len > 0 && - rspamd_regexp_match (ft->type_re, - ct->type.begin, - ct->type.len, - TRUE)) { - if (ft->subtype_re) { - if (ct->subtype.len > 0 && - rspamd_regexp_match (ft->subtype_re, - ct->subtype.begin, - ct->subtype.len, - TRUE)) { - return TRUE; - } - } - else { - return TRUE; - } - } - } - } - - return FALSE; -} - static double fuzzy_normalize (gint32 in, double weight) { @@ -431,6 +343,7 @@ fuzzy_parse_rule (struct rspamd_config *cfg, const ucl_object_t *obj, rule = fuzzy_rule_new (fuzzy_module_ctx->default_symbol, cfg->cfg_pool); + rule->ucl_obj = obj; rule->ctx = fuzzy_module_ctx; rule->learn_condition_cb = -1; rule->alg = RSPAMD_SHINGLES_OLD; @@ -445,36 +358,6 @@ fuzzy_parse_rule (struct rspamd_config *cfg, const ucl_object_t *obj, (void **)&rule->skip_map); } - if ((value = ucl_object_lookup (obj, "mime_types")) != NULL) { - it = NULL; - while ((cur = ucl_object_iterate (value, &it, value->type == UCL_ARRAY)) - != NULL) { - GPtrArray *tmp; - guint i; - gpointer ptr; - - tmp = parse_mime_types (cfg, ucl_obj_tostring (cur)); - - if (tmp) { - if (rule->mime_types) { - PTR_ARRAY_FOREACH (tmp, i, ptr) { - g_ptr_array_add (rule->mime_types, ptr); - } - - g_ptr_array_free (tmp, TRUE); - } - else { - rule->mime_types = tmp; - } - } - } - - if (rule->mime_types) { - rspamd_mempool_add_destructor (cfg->cfg_pool, - rspamd_ptr_array_free_hard, rule->mime_types); - } - } - if ((value = ucl_object_lookup (obj, "headers")) != NULL) { it = NULL; while ((cur = ucl_object_iterate (value, &it, value->type == UCL_ARRAY)) @@ -514,10 +397,6 @@ fuzzy_parse_rule (struct rspamd_config *cfg, const ucl_object_t *obj, rule->max_score = ucl_obj_todouble (value); } - if ((value = ucl_object_lookup (obj, "min_bytes")) != NULL) { - rule->min_bytes = ucl_obj_toint (value); - } - if ((value = ucl_object_lookup (obj, "symbol")) != NULL) { rule->symbol = ucl_obj_tostring (value); } @@ -538,14 +417,6 @@ fuzzy_parse_rule (struct rspamd_config *cfg, const ucl_object_t *obj, rule->skip_unknown = ucl_obj_toboolean (value); } - if ((value = ucl_object_lookup (obj, "short_text_direct_hash")) != NULL) { - rule->short_text_direct_hash = ucl_obj_toboolean (value); - } - - if ((value = ucl_object_lookup (obj, "fuzzy_images")) != NULL) { - rule->fuzzy_images = ucl_obj_toboolean (value); - } - if ((value = ucl_object_lookup (obj, "algorithm")) != NULL) { rule->algorithm_str = ucl_object_tostring (value); @@ -684,10 +555,10 @@ fuzzy_parse_rule (struct rspamd_config *cfg, const ucl_object_t *obj, g_ptr_array_add (fuzzy_module_ctx->fuzzy_rules, rule); if (rule->symbol != fuzzy_module_ctx->default_symbol) { - rspamd_symbols_cache_add_symbol (cfg->cache, rule->symbol, + rspamd_symcache_add_symbol (cfg->cache, rule->symbol, 0, NULL, NULL, - SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE, + SYMBOL_TYPE_VIRTUAL | SYMBOL_TYPE_FINE, cb_id); } @@ -699,6 +570,34 @@ fuzzy_parse_rule (struct rspamd_config *cfg, const ucl_object_t *obj, rule->algorithm_str); } + /* + * Process rule in Lua + */ + gint err_idx, ret; + GString *tb; + lua_State *L = (lua_State *)cfg->lua_state; + + lua_pushcfunction (L, &rspamd_lua_traceback); + err_idx = lua_gettop (L); + lua_rawgeti (L, LUA_REGISTRYINDEX, fuzzy_module_ctx->process_rule_ref); + ucl_object_push_lua (L, obj, true); + + if ((ret = lua_pcall (L, 1, 1, err_idx)) != 0) { + tb = lua_touserdata (L, -1); + msg_err_config ("call to process_rule lua " + "script failed (%d): %v", ret, tb); + + if (tb) { + g_string_free (tb, TRUE); + } + rule->lua_id = -1; + } + else { + rule->lua_id = lua_tonumber (L, -1); + } + + lua_settop (L, 0); + rspamd_mempool_add_destructor (cfg->cfg_pool, fuzzy_free_rule, rule); @@ -718,6 +617,9 @@ fuzzy_check_module_init (struct rspamd_config *cfg, struct module_ctx **ctx) fuzzy_module_ctx->keypairs_cache = rspamd_keypair_cache_new (32); fuzzy_module_ctx->fuzzy_rules = g_ptr_array_new (); fuzzy_module_ctx->cfg = cfg; + fuzzy_module_ctx->process_rule_ref = -1; + fuzzy_module_ctx->check_mime_part_ref = -1; + fuzzy_module_ctx->cleanup_rules_ref = -1; rspamd_mempool_add_destructor (cfg->cfg_pool, (rspamd_mempool_destruct_t)rspamd_mempool_delete, @@ -1005,58 +907,78 @@ fuzzy_check_module_config (struct rspamd_config *cfg) } fuzzy_module_ctx->enabled = TRUE; + fuzzy_module_ctx->check_mime_part_ref = -1; + fuzzy_module_ctx->process_rule_ref = -1; + fuzzy_module_ctx->cleanup_rules_ref = -1; - if ((value = - rspamd_config_get_module_opt (cfg, "fuzzy_check", "symbol")) != NULL) { - fuzzy_module_ctx->default_symbol = ucl_obj_tostring (value); + /* Interact with lua_fuzzy */ + if (luaL_dostring (L, "return require \"lua_fuzzy\"") != 0) { + msg_err_config ("cannot require lua_fuzzy: %s", + lua_tostring (L, -1)); + fuzzy_module_ctx->enabled = FALSE; } else { - fuzzy_module_ctx->default_symbol = DEFAULT_SYMBOL; - } + if (lua_type (L, -1) != LUA_TTABLE) { + msg_err_config ("lua fuzzy must return " + "table and not %s", + lua_typename (L, lua_type (L, -1))); + fuzzy_module_ctx->enabled = FALSE; + } else { + lua_pushstring (L, "process_rule"); + lua_gettable (L, -2); + + if (lua_type (L, -1) != LUA_TFUNCTION) { + msg_err_config ("process_rule must return " + "function and not %s", + lua_typename (L, lua_type (L, -1))); + fuzzy_module_ctx->enabled = FALSE; + } + else { + fuzzy_module_ctx->process_rule_ref = luaL_ref (L, LUA_REGISTRYINDEX); + } - if ((value = - rspamd_config_get_module_opt (cfg, "fuzzy_check", - "min_length")) != NULL) { - fuzzy_module_ctx->min_hash_len = ucl_obj_toint (value); - } - else { - fuzzy_module_ctx->min_hash_len = 0; - } + lua_pushstring (L, "check_mime_part"); + lua_gettable (L, -2); - if ((value = - rspamd_config_get_module_opt (cfg, "fuzzy_check", - "min_bytes")) != NULL) { - fuzzy_module_ctx->min_bytes = ucl_obj_toint (value); - } - else { - fuzzy_module_ctx->min_bytes = 0; - } + if (lua_type (L, -1) != LUA_TFUNCTION) { + msg_err_config ("check_mime_part must return " + "function and not %s", + lua_typename (L, lua_type (L, -1))); + fuzzy_module_ctx->enabled = FALSE; + } + else { + fuzzy_module_ctx->check_mime_part_ref = luaL_ref (L, LUA_REGISTRYINDEX); + } - if ((value = - rspamd_config_get_module_opt (cfg, "fuzzy_check", - "text_multiplier")) != NULL) { - fuzzy_module_ctx->text_multiplier = ucl_object_todouble (value); - } - else { - fuzzy_module_ctx->text_multiplier = 2.0; - } + lua_pushstring (L, "cleanup_rules"); + lua_gettable (L, -2); - if ((value = - rspamd_config_get_module_opt (cfg, "fuzzy_check", - "min_height")) != NULL) { - fuzzy_module_ctx->min_height = ucl_obj_toint (value); + if (lua_type (L, -1) != LUA_TFUNCTION) { + msg_err_config ("cleanup_rules must return " + "function and not %s", + lua_typename (L, lua_type (L, -1))); + fuzzy_module_ctx->enabled = FALSE; + } + else { + fuzzy_module_ctx->cleanup_rules_ref = luaL_ref (L, LUA_REGISTRYINDEX); + } + } } - else { - fuzzy_module_ctx->min_height = 0; + + lua_settop (L, 0); + + if (!fuzzy_module_ctx->enabled) { + return TRUE; } + if ((value = - rspamd_config_get_module_opt (cfg, "fuzzy_check", - "min_width")) != NULL) { - fuzzy_module_ctx->min_width = ucl_obj_toint (value); + rspamd_config_get_module_opt (cfg, "fuzzy_check", "symbol")) != NULL) { + fuzzy_module_ctx->default_symbol = ucl_obj_tostring (value); } else { - fuzzy_module_ctx->min_width = 0; + fuzzy_module_ctx->default_symbol = DEFAULT_SYMBOL; } + if ((value = rspamd_config_get_module_opt (cfg, "fuzzy_check", "timeout")) != NULL) { fuzzy_module_ctx->io_timeout = ucl_obj_todouble (value) * 1000; @@ -1088,10 +1010,10 @@ fuzzy_check_module_config (struct rspamd_config *cfg) if ((value = rspamd_config_get_module_opt (cfg, "fuzzy_check", "rule")) != NULL) { - cb_id = rspamd_symbols_cache_add_symbol (cfg->cache, - "FUZZY_CALLBACK", 0, fuzzy_symbol_callback, NULL, - SYMBOL_TYPE_CALLBACK|SYMBOL_TYPE_FINE, - -1); + cb_id = rspamd_symcache_add_symbol (cfg->cache, + "FUZZY_CALLBACK", 0, fuzzy_symbol_callback, NULL, + SYMBOL_TYPE_CALLBACK | SYMBOL_TYPE_FINE, + -1); /* * Here we can have 2 possibilities: @@ -1137,6 +1059,10 @@ fuzzy_check_module_config (struct rspamd_config *cfg) } } } + + /* We want that to check bad mime attachments */ + rspamd_symcache_add_delayed_dependency (cfg->cache, + "FUZZY_CALLBACK", "MIME_TYPES_CALLBACK"); } if (fuzzy_module_ctx->fuzzy_rules == NULL) { @@ -1163,7 +1089,7 @@ fuzzy_check_module_config (struct rspamd_config *cfg) lua_settable (L, -3); } - lua_pop (L, 1); /* Remove global function */ + lua_settop (L, 0); return res; } @@ -1171,6 +1097,43 @@ fuzzy_check_module_config (struct rspamd_config *cfg) gint fuzzy_check_module_reconfig (struct rspamd_config *cfg) { + struct fuzzy_ctx *fuzzy_module_ctx = fuzzy_get_context (cfg); + + if (fuzzy_module_ctx->cleanup_rules_ref != -1) { + /* Sync lua_fuzzy rules */ + gint err_idx, ret; + GString *tb; + lua_State *L = (lua_State *)cfg->lua_state; + + lua_pushcfunction (L, &rspamd_lua_traceback); + err_idx = lua_gettop (L); + lua_rawgeti (L, LUA_REGISTRYINDEX, fuzzy_module_ctx->cleanup_rules_ref); + + if ((ret = lua_pcall (L, 0, 0, err_idx)) != 0) { + tb = lua_touserdata (L, -1); + msg_err_config ("call to cleanup_rules lua " + "script failed (%d): %v", ret, tb); + + if (tb) { + g_string_free (tb, TRUE); + } + } + + luaL_unref (cfg->lua_state, LUA_REGISTRYINDEX, + fuzzy_module_ctx->cleanup_rules_ref); + lua_settop (L, 0); + } + + if (fuzzy_module_ctx->check_mime_part_ref != -1) { + luaL_unref (cfg->lua_state, LUA_REGISTRYINDEX, + fuzzy_module_ctx->check_mime_part_ref); + } + + if (fuzzy_module_ctx->process_rule_ref != -1) { + luaL_unref (cfg->lua_state, LUA_REGISTRYINDEX, + fuzzy_module_ctx->process_rule_ref); + } + return fuzzy_check_module_config (cfg); } @@ -1364,18 +1327,71 @@ fuzzy_cmd_set_cached (struct fuzzy_rule *rule, rspamd_mempool_set_variable (pool, key, data, NULL); } +static gboolean +fuzzy_rule_check_mimepart (struct rspamd_task *task, + struct fuzzy_rule *rule, + struct rspamd_mime_part *part, + gboolean *need_check, + gboolean *fuzzy_check) +{ + if (rule->lua_id != -1 && rule->ctx->check_mime_part_ref != -1) { + gint err_idx, ret; + GString *tb; + lua_State *L = (lua_State *)task->cfg->lua_state; + struct rspamd_task **ptask; + struct rspamd_mime_part **ppart; + + lua_pushcfunction (L, &rspamd_lua_traceback); + err_idx = lua_gettop (L); + lua_rawgeti (L, LUA_REGISTRYINDEX, rule->ctx->check_mime_part_ref); + + ptask = lua_newuserdata (L, sizeof (*ptask)); + *ptask = task; + rspamd_lua_setclass (L, "rspamd{task}", -1); + + ppart = lua_newuserdata (L, sizeof (*ppart)); + *ppart = part; + rspamd_lua_setclass (L, "rspamd{mimepart}", -1); + + lua_pushnumber (L, rule->lua_id); + + if ((ret = lua_pcall (L, 3, 2, err_idx)) != 0) { + tb = lua_touserdata (L, -1); + msg_err_task ("call to check_mime_part lua " + "script failed (%d): %v", ret, tb); + + if (tb) { + g_string_free (tb, TRUE); + } + ret = FALSE; + } + else { + ret = TRUE; + *need_check = lua_toboolean (L, -2); + *fuzzy_check = lua_toboolean (L, -1); + } + + lua_settop (L, 0); + + return ret; + } + + return FALSE; +} + /* * Create fuzzy command from a text part */ static struct fuzzy_cmd_io * fuzzy_cmd_from_text_part (struct rspamd_task *task, - struct fuzzy_rule *rule, - int c, - gint flag, - guint32 weight, - gboolean short_text, - rspamd_mempool_t *pool, - struct rspamd_mime_text_part *part) + struct fuzzy_rule *rule, + int c, + gint flag, + guint32 weight, + gboolean short_text, + rspamd_mempool_t *pool, + struct rspamd_mime_text_part *part, + struct rspamd_mime_part *mp) { struct rspamd_fuzzy_shingle_cmd *shcmd = NULL; struct rspamd_fuzzy_cmd *cmd = NULL; @@ -1389,7 +1405,7 @@ fuzzy_cmd_from_text_part (struct rspamd_task *task, GArray *words; struct fuzzy_cmd_io *io; - cached = fuzzy_cmd_get_cached (rule, pool, part); + cached = fuzzy_cmd_get_cached (rule, pool, mp); if (cached) { /* Copy cached */ @@ -1443,7 +1459,12 @@ fuzzy_cmd_from_text_part (struct rspamd_task *task, for (i = 0; i < words->len; i ++) { word = &g_array_index (words, rspamd_stat_token_t, i); - rspamd_cryptobox_hash_update (&st, word->begin, word->len); + + if (!((word->flags & RSPAMD_STAT_TOKEN_FLAG_SKIPPED) + || word->stemmed.len == 0)) { + rspamd_cryptobox_hash_update (&st, word->stemmed.begin, + word->stemmed.len); + } } rspamd_cryptobox_hash_final (&st, shcmd->basic.digest); @@ -1472,10 +1493,11 @@ fuzzy_cmd_from_text_part (struct rspamd_task *task, * Since it is copied when obtained from the cache, it is safe to use * it this way. */ - fuzzy_cmd_set_cached (rule, pool, part, cached); + fuzzy_cmd_set_cached (rule, pool, mp, cached); } io = rspamd_mempool_alloc (pool, sizeof (*io)); + io->part = mp; if (!short_text) { shcmd->basic.tag = ottery_rand_uint32 (); @@ -1536,11 +1558,12 @@ fuzzy_cmd_from_text_part (struct rspamd_task *task, static struct fuzzy_cmd_io * fuzzy_cmd_from_image_part (struct fuzzy_rule *rule, - int c, - gint flag, - guint32 weight, - rspamd_mempool_t *pool, - struct rspamd_image *img) + int c, + gint flag, + guint32 weight, + rspamd_mempool_t *pool, + struct rspamd_image *img, + struct rspamd_mime_part *mp) { struct rspamd_fuzzy_shingle_cmd *shcmd; struct rspamd_fuzzy_encrypted_shingle_cmd *encshcmd; @@ -1548,7 +1571,7 @@ fuzzy_cmd_from_image_part (struct fuzzy_rule *rule, struct rspamd_shingle *sh; struct rspamd_cached_shingles *cached; - cached = fuzzy_cmd_get_cached (rule, pool, img); + cached = fuzzy_cmd_get_cached (rule, pool, mp); if (cached) { /* Copy cached */ @@ -1598,7 +1621,7 @@ fuzzy_cmd_from_image_part (struct fuzzy_rule *rule, cached = rspamd_mempool_alloc (pool, sizeof (*cached)); cached->sh = sh; memcpy (cached->digest, shcmd->basic.digest, sizeof (cached->digest)); - fuzzy_cmd_set_cached (rule, pool, img, cached); + fuzzy_cmd_set_cached (rule, pool, mp, cached); } shcmd->basic.tag = ottery_rand_uint32 (); @@ -1611,6 +1634,7 @@ fuzzy_cmd_from_image_part (struct fuzzy_rule *rule, } io = rspamd_mempool_alloc (pool, sizeof (*io)); + io->part = mp; io->tag = shcmd->basic.tag; io->flags = FUZZY_CMD_FLAG_IMAGE; memcpy (&io->cmd, &shcmd->basic, sizeof (io->cmd)); @@ -1631,11 +1655,12 @@ fuzzy_cmd_from_image_part (struct fuzzy_rule *rule, static struct fuzzy_cmd_io * fuzzy_cmd_from_data_part (struct fuzzy_rule *rule, - int c, - gint flag, - guint32 weight, - rspamd_mempool_t *pool, - guchar digest[rspamd_cryptobox_HASHBYTES]) + int c, + gint flag, + guint32 weight, + rspamd_mempool_t *pool, + guchar digest[rspamd_cryptobox_HASHBYTES], + struct rspamd_mime_part *mp) { struct rspamd_fuzzy_cmd *cmd; struct rspamd_fuzzy_encrypted_cmd *enccmd = NULL; @@ -1662,6 +1687,7 @@ fuzzy_cmd_from_data_part (struct fuzzy_rule *rule, io = rspamd_mempool_alloc (pool, sizeof (*io)); io->flags = 0; io->tag = cmd->tag; + io->part = mp; memcpy (&io->cmd, cmd, sizeof (io->cmd)); if (rule->peer_key) { @@ -2106,6 +2132,9 @@ fuzzy_check_session_is_completed (struct fuzzy_client_session *session) if (nreplied == session->commands->len) { fuzzy_insert_metric_results (session->task, session->results); + if (session->item) { + rspamd_symcache_item_async_dec_check (session->task, session->item, M); + } rspamd_session_remove_event (session->task->s, fuzzy_io_fin, session); return TRUE; @@ -2174,11 +2203,16 @@ fuzzy_check_io_callback (gint fd, short what, void *arg) /* Error state */ msg_err_task ("got error on IO with server %s(%s), on %s, %d, %s", rspamd_upstream_name (session->server), - rspamd_inet_address_to_string_pretty (session->addr), + rspamd_inet_address_to_string_pretty ( + rspamd_upstream_addr (session->server)), session->state == 1 ? "read" : "write", errno, strerror (errno)); rspamd_upstream_fail (session->server, FALSE); + + if (session->item) { + rspamd_symcache_item_async_dec_check (session->task, session->item, M); + } rspamd_session_remove_event (session->task->s, fuzzy_io_fin, session); } else { @@ -2215,9 +2249,13 @@ fuzzy_check_timer_callback (gint fd, short what, void *arg) if (session->retransmits >= session->rule->ctx->retransmits) { msg_err_task ("got IO timeout with server %s(%s), after %d retransmits", rspamd_upstream_name (session->server), - rspamd_inet_address_to_string_pretty (session->addr), + rspamd_inet_address_to_string_pretty ( + rspamd_upstream_addr (session->server)), session->retransmits); rspamd_upstream_fail (session->server, FALSE); + if (session->item) { + rspamd_symcache_item_async_dec_check (session->task, session->item, M); + } rspamd_session_remove_event (session->task->s, fuzzy_io_fin, session); } else { @@ -2285,7 +2323,7 @@ fuzzy_controller_io_callback (gint fd, short what, void *arg) session->task->message_id, strerror (errno)); if (*(session->err) == NULL) { g_set_error (session->err, - g_quark_from_static_string ("fuzzy check"), + g_quark_from_static_string (M), errno, "read socket error: %s", strerror (errno)); } ret = return_error; @@ -2349,7 +2387,7 @@ fuzzy_controller_io_callback (gint fd, short what, void *arg) if (*(session->err) == NULL) { g_set_error (session->err, - g_quark_from_static_string ("fuzzy check"), + g_quark_from_static_string (M), rep->v1.value, "fuzzy hash is skipped"); } } @@ -2368,7 +2406,7 @@ fuzzy_controller_io_callback (gint fd, short what, void *arg) if (*(session->err) == NULL) { g_set_error (session->err, - g_quark_from_static_string ("fuzzy check"), + g_quark_from_static_string (M), rep->v1.value, "process fuzzy error"); } } @@ -2397,7 +2435,7 @@ fuzzy_controller_io_callback (gint fd, short what, void *arg) if (!fuzzy_cmd_vector_to_wire (fd, session->commands)) { if (*(session->err) == NULL) { g_set_error (session->err, - g_quark_from_static_string ("fuzzy check"), + g_quark_from_static_string (M), errno, "write socket error: %s", strerror (errno)); } ret = return_error; @@ -2420,7 +2458,8 @@ fuzzy_controller_io_callback (gint fd, short what, void *arg) else if (ret == return_error) { msg_err_task ("got error in IO with server %s(%s), %d, %s", rspamd_upstream_name (session->server), - rspamd_inet_address_to_string_pretty (session->addr), + rspamd_inet_address_to_string_pretty ( + rspamd_upstream_addr (session->server)), errno, strerror (errno)); rspamd_upstream_fail (session->server, FALSE); } @@ -2523,7 +2562,8 @@ fuzzy_controller_timer_callback (gint fd, short what, void *arg) msg_err_task_check ("got IO timeout with server %s(%s), " "after %d retransmits", rspamd_upstream_name (session->server), - rspamd_inet_address_to_string_pretty (session->addr), + rspamd_inet_address_to_string_pretty ( + rspamd_upstream_addr (session->server)), session->retransmits); if (session->session) { @@ -2582,18 +2622,12 @@ fuzzy_generate_commands (struct rspamd_task *task, struct fuzzy_rule *rule, struct rspamd_mime_part *mime_part; struct rspamd_image *image; struct fuzzy_cmd_io *io, *cur; - guint i, j, min_bytes = 0; + guint i, j; GPtrArray *res; + gboolean check_part, fuzzy_check; res = g_ptr_array_sized_new (task->parts->len + 1); - if (rule->min_bytes) { - min_bytes = rule->min_bytes; - } - else { - min_bytes = rule->ctx->min_bytes; - } - if (c == FUZZY_STAT) { io = fuzzy_cmd_stat (rule, c, flag, value, task->task_pool); if (io) { @@ -2603,212 +2637,63 @@ fuzzy_generate_commands (struct rspamd_task *task, struct fuzzy_rule *rule, goto end; } - if (G_LIKELY (!(flags & FUZZY_CHECK_FLAG_NOTEXT))) { - for (i = 0; i < task->text_parts->len; i ++) { - gdouble fac; - gboolean short_text = FALSE; + PTR_ARRAY_FOREACH (task->parts, i, mime_part) { + check_part = FALSE; + fuzzy_check = FALSE; - part = g_ptr_array_index (task->text_parts, i); + if (fuzzy_rule_check_mimepart (task, rule, mime_part, &check_part, + &fuzzy_check)) { + io = NULL; - if (IS_PART_EMPTY (part)) { - continue; - } + if (check_part) { + if (mime_part->flags & RSPAMD_MIME_PART_TEXT && + !(flags & FUZZY_CHECK_FLAG_NOTEXT)) { + part = mime_part->specific.txt; - /* Check length of part */ - fac = rule->ctx->text_multiplier * part->utf_content->len; - if ((double)min_bytes > fac) { - if (!rule->short_text_direct_hash) { - msg_info_task ( - "<%s>, part is shorter than %d bytes: %.0f " - "(%d * %.2f bytes), " - "skip fuzzy check", - task->message_id, min_bytes, - fac, - part->utf_content->len, - rule->ctx->text_multiplier); - continue; - } - else { - msg_info_task ( - "<%s>, part is shorter than %d bytes: %.0f " - "(%d * %.2f bytes), " - "use direct hash", - task->message_id, min_bytes, - fac, - part->utf_content->len, - rule->ctx->text_multiplier); - short_text = TRUE; + io = fuzzy_cmd_from_text_part (task, rule, + c, + flag, + value, + !fuzzy_check, + task->task_pool, + part, + mime_part); } - } + else if (mime_part->flags & RSPAMD_MIME_PART_IMAGE && + !(flags & FUZZY_CHECK_FLAG_NOIMAGES)) { + image = mime_part->specific.img; - if (part->utf_words == NULL || - part->utf_words->len == 0) { - msg_info_task ("<%s>, part hash empty, skip fuzzy check", - task->message_id); - continue; - } - - if (rule->ctx->min_hash_len != 0 && - part->utf_words->len < - rule->ctx->min_hash_len) { - if (!rule->short_text_direct_hash) { - msg_info_task ( - "<%s>, part hash is shorter than %d symbols, " - "skip fuzzy check", - task->message_id, - rule->ctx->min_hash_len); - continue; + io = fuzzy_cmd_from_data_part (rule, c, flag, value, + task->task_pool, + image->parent->digest, + mime_part); + io->flags |= FUZZY_CMD_FLAG_IMAGE; } else { - msg_info_task ( - "<%s>, part hash is shorter than %d symbols, " - "use direct hash", - task->message_id, - rule->ctx->min_hash_len); - short_text = TRUE; - } - } - - io = fuzzy_cmd_from_text_part (task, rule, - c, - flag, - value, - short_text, - task->task_pool, - part); - - if (io) { - gboolean skip_existing = FALSE; - - PTR_ARRAY_FOREACH (res, j, cur) { - if (memcmp (cur->cmd.digest, io->cmd.digest, - sizeof (io->cmd.digest)) == 0) { - skip_existing = TRUE; - break; - } - } - - if (!skip_existing) { - g_ptr_array_add (res, io); + io = fuzzy_cmd_from_data_part (rule, c, flag, value, + task->task_pool, + mime_part->digest, mime_part); } - } - } - } - - /* Process other parts and images */ - for (i = 0; i < task->parts->len; i ++) { - mime_part = g_ptr_array_index (task->parts, i); + if (io) { + gboolean skip_existing = FALSE; - if (mime_part->flags & RSPAMD_MIME_PART_IMAGE) { - - if (G_LIKELY (!(flags & FUZZY_CHECK_FLAG_NOIMAGES))) { - image = mime_part->specific.img; - - if (image->data->len > 0) { - /* Check: - * - min height - * - min width - * - min bytes - */ - - if ((rule->ctx->min_height == 0 || - image->height >= rule->ctx->min_height) && - (rule->ctx->min_width == 0 || - image->width >= rule->ctx->min_width) && - (min_bytes == 0 || - mime_part->parsed_data.len >= min_bytes)) { - io = fuzzy_cmd_from_data_part (rule, c, flag, value, - task->task_pool, - image->parent->digest); - if (io) { - gboolean skip_existing = FALSE; - - PTR_ARRAY_FOREACH (res, j, cur) { - if (memcmp (cur->cmd.digest, io->cmd.digest, - sizeof (io->cmd.digest)) == 0) { - skip_existing = TRUE; - break; - } - } - - if (!skip_existing) { - g_ptr_array_add (res, io); - } - } - - if (rule->fuzzy_images) { - /* Try to normalize image */ - if (!image->is_normalized) { - rspamd_image_normalize (task, image); - } - } - - if (image->is_normalized) { - io = fuzzy_cmd_from_image_part (rule, c, flag, - value, - task->task_pool, - image); - if (io) { - gboolean skip_existing = FALSE; - - PTR_ARRAY_FOREACH (res, j, cur) { - if (memcmp (cur->cmd.digest, io->cmd.digest, - sizeof (io->cmd.digest)) == 0) { - skip_existing = TRUE; - break; - } - } - - if (!skip_existing) { - g_ptr_array_add (res, io); - } - } + PTR_ARRAY_FOREACH (res, j, cur) { + if (memcmp (cur->cmd.digest, io->cmd.digest, + sizeof (io->cmd.digest)) == 0) { + skip_existing = TRUE; + break; } } - } - } - - continue; - } - - if (G_LIKELY (!(flags & FUZZY_CHECK_FLAG_NOIMAGES))) { - if (mime_part->ct && - !(mime_part->flags & (RSPAMD_MIME_PART_TEXT|RSPAMD_MIME_PART_IMAGE)) && - mime_part->parsed_data.len > 0 && - fuzzy_check_content_type (rule, mime_part->ct)) { - if (min_bytes == 0 || mime_part->parsed_data.len >= min_bytes) { - io = fuzzy_cmd_from_data_part (rule, c, flag, value, - task->task_pool, - mime_part->digest); - if (io) { - gboolean skip_existing = FALSE; - - PTR_ARRAY_FOREACH (res, j, cur) { - if (memcmp (cur->cmd.digest, io->cmd.digest, - sizeof (io->cmd.digest)) == 0) { - skip_existing = TRUE; - break; - } - } - if (!skip_existing) { - g_ptr_array_add (res, io); - } + if (!skip_existing) { + g_ptr_array_add (res, io); } } } } } - /* Process metadata */ -#if 0 - io = fuzzy_cmd_from_task_meta (rule, c, flag, value, - task->task_pool, task); - if (io) { - g_ptr_array_add (res, io); - } -#endif end: if (res->len == 0) { g_ptr_array_free (res, TRUE); @@ -2856,7 +2741,6 @@ register_fuzzy_client_call (struct rspamd_task *task, session->fd = sock; session->server = selected; session->rule = rule; - session->addr = addr; session->results = g_ptr_array_sized_new (32); event_set (&session->ev, sock, EV_WRITE, fuzzy_check_io_callback, @@ -2869,8 +2753,12 @@ register_fuzzy_client_call (struct rspamd_task *task, event_base_set (session->task->ev_base, &session->timev); event_add (&session->timev, &session->tv); - rspamd_session_add_event (task->s, NULL, fuzzy_io_fin, session, - g_quark_from_static_string ("fuzzy check")); + rspamd_session_add_event (task->s, fuzzy_io_fin, session, M); + session->item = rspamd_symcache_get_cur_item (task); + + if (session->item) { + rspamd_symcache_item_async_inc (task, session->item, M); + } } } } @@ -2878,7 +2766,9 @@ register_fuzzy_client_call (struct rspamd_task *task, /* This callback is called when we check message in fuzzy hashes storage */ static void -fuzzy_symbol_callback (struct rspamd_task *task, void *unused) +fuzzy_symbol_callback (struct rspamd_task *task, + struct rspamd_symcache_item *item, + void *unused) { struct fuzzy_rule *rule; guint i; @@ -2886,6 +2776,8 @@ fuzzy_symbol_callback (struct rspamd_task *task, void *unused) struct fuzzy_ctx *fuzzy_module_ctx = fuzzy_get_context (task->cfg); if (!fuzzy_module_ctx->enabled) { + rspamd_symcache_finalize_item (task, item); + return; } @@ -2896,10 +2788,14 @@ fuzzy_symbol_callback (struct rspamd_task *task, void *unused) msg_info_task ("<%s>, address %s is whitelisted, skip fuzzy check", task->message_id, rspamd_inet_address_to_string (task->from_addr)); + rspamd_symcache_finalize_item (task, item); + return; } } + rspamd_symcache_item_async_inc (task, item, M); + PTR_ARRAY_FOREACH (fuzzy_module_ctx->fuzzy_rules, i, rule) { commands = fuzzy_generate_commands (task, rule, FUZZY_CHECK, 0, 0, 0); @@ -2907,6 +2803,8 @@ fuzzy_symbol_callback (struct rspamd_task *task, void *unused) register_fuzzy_client_call (task, rule, commands); } } + + rspamd_symcache_item_async_dec_check (task, item, M); } void @@ -2963,7 +2861,6 @@ register_fuzzy_controller_call (struct rspamd_http_connection_entry *entry, msec_to_tv (fuzzy_module_ctx->io_timeout, &s->tv); s->task = task; - s->addr = addr; s->commands = commands; s->http_entry = entry; s->server = selected; @@ -3009,9 +2906,9 @@ fuzzy_process_handler (struct rspamd_http_connection_entry *conn_ent, struct fuzzy_ctx *fuzzy_module_ctx; /* Prepare task */ - task = rspamd_task_new (session->wrk, session->cfg, NULL, session->lang_det); + task = rspamd_task_new (session->wrk, session->cfg, NULL, + session->lang_det, conn_ent->rt->ev_base); task->cfg = ctx->cfg; - task->ev_base = conn_ent->rt->ev_base; saved = rspamd_mempool_alloc0 (session->pool, sizeof (gint)); err = rspamd_mempool_alloc0 (session->pool, sizeof (GError *)); fuzzy_module_ctx = fuzzy_get_context (ctx->cfg); @@ -3030,6 +2927,7 @@ fuzzy_process_handler (struct rspamd_http_connection_entry *conn_ent, rspamd_task_free (task); rspamd_controller_send_error (conn_ent, 400, "Message processing error"); + return; } @@ -3325,7 +3223,6 @@ fuzzy_check_send_lua_learn (struct fuzzy_rule *rule, msec_to_tv (rule->ctx->io_timeout, &s->tv); s->task = task; - s->addr = addr; s->commands = commands; s->http_entry = NULL; s->server = selected; @@ -3343,7 +3240,10 @@ fuzzy_check_send_lua_learn (struct fuzzy_rule *rule, event_base_set (s->task->ev_base, &s->timev); event_add (&s->timev, &s->tv); - rspamd_session_add_event (task->s, NULL, fuzzy_lua_fin, s, g_quark_from_static_string ("fuzzy check")); + rspamd_session_add_event (task->s, + fuzzy_lua_fin, + s, + M); (*saved)++; ret = 1; diff --git a/src/plugins/lua/antivirus.lua b/src/plugins/lua/antivirus.lua index 025e36043..ed3d93e79 100644 --- a/src/plugins/lua/antivirus.lua +++ b/src/plugins/lua/antivirus.lua @@ -15,11 +15,10 @@ limitations under the License. ]] -- local rspamd_logger = require "rspamd_logger" -local rspamd_util = require "rspamd_util" local rspamd_regexp = require "rspamd_regexp" -local tcp = require "rspamd_tcp" -local upstream_list = require "rspamd_upstream_list" local lua_util = require "lua_util" +local fun = require "fun" +local lua_antivirus = require "lua_antivirus" local redis_params local N = "antivirus" @@ -69,760 +68,6 @@ antivirus { return end -local default_message = '${SCANNER}: virus found: "${VIRUS}"' - -local function match_patterns(default_sym, found, patterns) - if type(patterns) ~= 'table' then return default_sym end - if not patterns[1] then - for sym, pat in pairs(patterns) do - if pat:match(found) then - return sym - end - end - return default_sym - else - for _, p in ipairs(patterns) do - for sym, pat in pairs(p) do - if pat:match(found) then - return sym - end - end - end - return default_sym - end -end - -local function yield_result(task, rule, vname) - local all_whitelisted = true - if type(vname) == 'string' then - local symname = match_patterns(rule['symbol'], vname, rule['patterns']) - if rule['whitelist'] and rule['whitelist']:get_key(vname) then - rspamd_logger.infox(task, '%s: "%s" is in whitelist', rule['type'], vname) - return - end - task:insert_result(symname, 1.0, vname) - rspamd_logger.infox(task, '%s: virus found: "%s"', rule['type'], vname) - elseif type(vname) == 'table' then - for _, vn in ipairs(vname) do - local symname = match_patterns(rule['symbol'], vn, rule['patterns']) - if rule['whitelist'] and rule['whitelist']:get_key(vn) then - rspamd_logger.infox(task, '%s: "%s" is in whitelist', rule['type'], vn) - else - all_whitelisted = false - task:insert_result(symname, 1.0, vn) - rspamd_logger.infox(task, '%s: virus found: "%s"', rule['type'], vn) - end - end - end - if rule['action'] then - if type(vname) == 'table' then - if all_whitelisted then return end - vname = table.concat(vname, '; ') - end - task:set_pre_result(rule['action'], - lua_util.template(rule.message or 'Rejected', { - SCANNER = rule['type'], - VIRUS = vname, - }), N) - end -end - -local function clamav_config(opts) - local clamav_conf = { - scan_mime_parts = true; - scan_text_mime = false; - scan_image_mime = false; - default_port = 3310, - log_clean = false, - timeout = 15.0, - retransmits = 2, - cache_expire = 3600, -- expire redis in one hour - message = default_message, - } - - for k,v in pairs(opts) do - clamav_conf[k] = v - end - - if not clamav_conf.prefix then - clamav_conf.prefix = 'rs_cl' - end - - if not clamav_conf['servers'] then - rspamd_logger.errx(rspamd_config, 'no servers defined') - - return nil - end - - clamav_conf['upstreams'] = upstream_list.create(rspamd_config, - clamav_conf['servers'], - clamav_conf.default_port) - - if clamav_conf['upstreams'] then - return clamav_conf - end - - rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', - clamav_conf['servers']) - return nil -end - -local function fprot_config(opts) - local fprot_conf = { - scan_mime_parts = true; - scan_text_mime = false; - scan_image_mime = false; - default_port = 10200, - timeout = 15.0, - log_clean = false, - retransmits = 2, - cache_expire = 3600, -- expire redis in one hour - message = default_message, - } - - for k,v in pairs(opts) do - fprot_conf[k] = v - end - - if not fprot_conf.prefix then - fprot_conf.prefix = 'rs_fp' - end - - if not fprot_conf['servers'] then - rspamd_logger.errx(rspamd_config, 'no servers defined') - - return nil - end - - fprot_conf['upstreams'] = upstream_list.create(rspamd_config, - fprot_conf['servers'], - fprot_conf.default_port) - - if fprot_conf['upstreams'] then - return fprot_conf - end - - rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', - fprot_conf['servers']) - return nil -end - -local function sophos_config(opts) - local sophos_conf = { - scan_mime_parts = true; - scan_text_mime = false; - scan_image_mime = false; - default_port = 4010, - timeout = 15.0, - log_clean = false, - retransmits = 2, - cache_expire = 3600, -- expire redis in one hour - message = default_message, - savdi_report_encrypted = false, - savdi_report_oversize = false, - } - - for k,v in pairs(opts) do - sophos_conf[k] = v - end - - if not sophos_conf.prefix then - sophos_conf.prefix = 'rs_sp' - end - - if not sophos_conf['servers'] then - rspamd_logger.errx(rspamd_config, 'no servers defined') - - return nil - end - - sophos_conf['upstreams'] = upstream_list.create(rspamd_config, - sophos_conf['servers'], - sophos_conf.default_port) - - if sophos_conf['upstreams'] then - return sophos_conf - end - - rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', - sophos_conf['servers']) - return nil -end - -local function savapi_config(opts) - local savapi_conf = { - scan_mime_parts = true; - scan_text_mime = false; - scan_image_mime = false; - default_port = 4444, -- note: You must set ListenAddress in savapi.conf - product_id = 0, - log_clean = false, - timeout = 15.0, - retransmits = 2, - cache_expire = 3600, -- expire redis in one hour - message = default_message, - } - - for k,v in pairs(opts) do - savapi_conf[k] = v - end - - if not savapi_conf.prefix then - savapi_conf.prefix = 'rs_ap' - end - - if not savapi_conf['servers'] then - rspamd_logger.errx(rspamd_config, 'no servers defined') - - return nil - end - - savapi_conf['upstreams'] = upstream_list.create(rspamd_config, - savapi_conf['servers'], - savapi_conf.default_port) - - if savapi_conf['upstreams'] then - return savapi_conf - end - - rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', - savapi_conf['servers']) - return nil -end - -local function message_not_too_large(task, content, rule) - local max_size = tonumber(rule['max_size']) - if not max_size then return true end - if #content > max_size then - rspamd_logger.infox("skip %s AV check as it is too large: %s (%s is allowed)", - rule.type, #content, max_size) - return false - end - return true -end - -local function need_av_check(task, content, rule) - return message_not_too_large(task, content, rule) -end - -local function check_av_cache(task, digest, rule, fn) - local key = digest - - local function redis_av_cb(err, data) - if data and type(data) == 'string' then - -- Cached - if data ~= 'OK' then - lua_util.debugm(N, task, 'got cached result for %s: %s', key, data) - data = rspamd_str_split(data, '\x30') - yield_result(task, rule, data) - else - lua_util.debugm(N, task, 'got cached result for %s: %s', key, data) - end - else - if err then - rspamd_logger.errx(task, 'Got error checking cache: %1', err) - end - fn() - end - end - - if redis_params then - - key = rule['prefix'] .. key - - if rspamd_redis_make_request(task, - redis_params, -- connect params - key, -- hash key - false, -- is write - redis_av_cb, --callback - 'GET', -- command - {key} -- arguments) - ) then - return true - end - end - - return false -end - -local function save_av_cache(task, digest, rule, to_save) - local key = digest - - local function redis_set_cb(err) - -- Do nothing - if err then - rspamd_logger.errx(task, 'failed to save virus cache for %s -> "%s": %s', - to_save, key, err) - else - lua_util.debugm(N, task, 'saved cached result for %s: %s', key, to_save) - end - end - - if type(to_save) == 'table' then - to_save = table.concat(to_save, '\x30') - end - - if redis_params then - key = rule['prefix'] .. key - - rspamd_redis_make_request(task, - redis_params, -- connect params - key, -- hash key - true, -- is write - redis_set_cb, --callback - 'SETEX', -- command - { key, rule['cache_expire'], to_save } - ) - end - - return false -end - -local function fprot_check(task, content, digest, rule) - local function fprot_check_uncached () - local upstream = rule.upstreams:get_upstream_round_robin() - local addr = upstream:get_addr() - local retransmits = rule.retransmits - local scan_id = task:get_queue_id() - if not scan_id then scan_id = task:get_uid() end - local header = string.format('SCAN STREAM %s SIZE %d\n', scan_id, - #content) - local footer = '\n' - - local function fprot_callback(err, data) - if err then - -- set current upstream to fail because an error occurred - upstream:fail() - - -- retry with another upstream until retransmits exceeds - if retransmits > 0 then - - retransmits = retransmits - 1 - - -- Select a different upstream! - upstream = rule.upstreams:get_upstream_round_robin() - addr = upstream:get_addr() - - lua_util.debugm(N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr) - - tcp.request({ - task = task, - host = addr:to_string(), - port = addr:get_port(), - timeout = rule['timeout'], - callback = fprot_callback, - data = { header, content, footer }, - stop_pattern = '\n' - }) - else - rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type']) - task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and retransmits exceed') - end - else - upstream:ok() - data = tostring(data) - local cached - local clean = string.match(data, '^0 <clean>') - if clean then - cached = 'OK' - if rule['log_clean'] then - rspamd_logger.infox(task, '%s [%s]: message or mime_part is clean', rule['symbol'], rule['type']) - end - else - -- returncodes: 1: infected, 2: suspicious, 3: both, 4-255: some error occured - -- see http://www.f-prot.com/support/helpfiles/unix/appendix_c.html for more detail - local vname = string.match(data, '^[1-3] <[%w%s]-: (.-)>') - if not vname then - rspamd_logger.errx(task, 'Unhandled response: %s', data) - else - yield_result(task, rule, vname) - cached = vname - end - end - if cached then - save_av_cache(task, digest, rule, cached) - end - end - end - - tcp.request({ - task = task, - host = addr:to_string(), - port = addr:get_port(), - timeout = rule['timeout'], - callback = fprot_callback, - data = { header, content, footer }, - stop_pattern = '\n' - }) - end - - if need_av_check(task, content, rule) then - if check_av_cache(task, digest, rule, fprot_check_uncached) then - return - else - fprot_check_uncached() - end - end -end - -local function clamav_check(task, content, digest, rule) - local function clamav_check_uncached () - local upstream = rule.upstreams:get_upstream_round_robin() - local addr = upstream:get_addr() - local retransmits = rule.retransmits - local header = rspamd_util.pack("c9 c1 >I4", "zINSTREAM", "\0", - #content) - local footer = rspamd_util.pack(">I4", 0) - - local function clamav_callback(err, data) - if err then - - -- set current upstream to fail because an error occurred - upstream:fail() - - -- retry with another upstream until retransmits exceeds - if retransmits > 0 then - - retransmits = retransmits - 1 - - -- Select a different upstream! - upstream = rule.upstreams:get_upstream_round_robin() - addr = upstream:get_addr() - - lua_util.debugm(N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr) - - tcp.request({ - task = task, - host = addr:to_string(), - port = addr:get_port(), - timeout = rule['timeout'], - callback = clamav_callback, - data = { header, content, footer }, - stop_pattern = '\0' - }) - else - rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type']) - task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and retransmits exceed') - end - - else - upstream:ok() - data = tostring(data) - local cached - lua_util.debugm(N, task, '%s [%s]: got reply: %s', rule['symbol'], rule['type'], data) - if data == 'stream: OK' then - cached = 'OK' - if rule['log_clean'] then - rspamd_logger.infox(task, '%s [%s]: message or mime_part is clean', rule['symbol'], rule['type']) - else - lua_util.debugm(N, task, '%s [%s]: message or mime_part is clean', rule['symbol'], rule['type']) - end - else - local vname = string.match(data, 'stream: (.+) FOUND') - if vname then - yield_result(task, rule, vname) - cached = vname - else - rspamd_logger.errx(task, 'unhandled response: %s', data) - task:insert_result(rule['symbol_fail'], 0.0, 'unhandled response') - end - end - if cached then - save_av_cache(task, digest, rule, cached) - end - end - end - - tcp.request({ - task = task, - host = addr:to_string(), - port = addr:get_port(), - timeout = rule['timeout'], - callback = clamav_callback, - data = { header, content, footer }, - stop_pattern = '\0' - }) - end - - if need_av_check(task, content, rule) then - if check_av_cache(task, digest, rule, clamav_check_uncached) then - return - else - clamav_check_uncached() - end - end -end - -local function sophos_check(task, content, digest, rule) - local function sophos_check_uncached () - local upstream = rule.upstreams:get_upstream_round_robin() - local addr = upstream:get_addr() - local retransmits = rule.retransmits - local protocol = 'SSSP/1.0\n' - local streamsize = string.format('SCANDATA %d\n', #content) - local bye = 'BYE\n' - - local function sophos_callback(err, data, conn) - - if err then - - -- set current upstream to fail because an error occurred - upstream:fail() - - -- retry with another upstream until retransmits exceeds - if retransmits > 0 then - - retransmits = retransmits - 1 - - -- Select a different upstream! - upstream = rule.upstreams:get_upstream_round_robin() - addr = upstream:get_addr() - - lua_util.debugm(N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr) - - tcp.request({ - task = task, - host = addr:to_string(), - port = addr:get_port(), - timeout = rule['timeout'], - callback = sophos_callback, - data = { protocol, streamsize, content, bye } - }) - else - rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type']) - task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and retransmits exceed') - end - else - upstream:ok() - data = tostring(data) - lua_util.debugm(N, task, '%s [%s]: got reply: %s', rule['symbol'], rule['type'], data) - local vname = string.match(data, 'VIRUS (%S+) ') - if vname then - yield_result(task, rule, vname) - save_av_cache(task, digest, rule, vname) - else - if string.find(data, 'DONE OK') then - if rule['log_clean'] then - rspamd_logger.infox(task, '%s [%s]: message or mime_part is clean', rule['symbol'], rule['type']) - else - lua_util.debugm(N, task, '%s [%s]: message or mime_part is clean', rule['symbol'], rule['type']) - end - save_av_cache(task, digest, rule, 'OK') - -- not finished - continue - elseif string.find(data, 'ACC') or string.find(data, 'OK SSSP') then - conn:add_read(sophos_callback) - -- set pseudo virus if configured, else do nothing since it's no fatal - elseif string.find(data, 'FAIL 0212') then - rspamd_logger.infox(task, 'Message is ENCRYPTED (0212 SOPHOS_SAVI_ERROR_FILE_ENCRYPTED): %s', data) - if rule['savdi_report_encrypted'] then - yield_result(task, rule, "SAVDI_FILE_ENCRYPTED") - save_av_cache(task, digest, rule, "SAVDI_FILE_ENCRYPTED") - end - -- set pseudo virus if configured, else set fail since part was not scanned - elseif string.find(data, 'REJ 4') then - if rule['savdi_report_oversize'] then - rspamd_logger.infox(task, 'SAVDI: Message is OVERSIZED (SSSP reject code 4): %s', data) - yield_result(task, rule, "SAVDI_FILE_OVERSIZED") - save_av_cache(task, digest, rule, "SAVDI_FILE_OVERSIZED") - else - rspamd_logger.errx(task, 'SAVDI: Message is OVERSIZED (SSSP reject code 4): %s', data) - task:insert_result(rule['symbol_fail'], 0.0, 'Message is OVERSIZED (SSSP reject code 4):' .. data) - end - -- excplicitly set REJ1 message when SAVDIreports a protocol error - elseif string.find(data, 'REJ 1') then - rspamd_logger.errx(task, 'SAVDI (Protocol error (REJ 1)): %s', data) - task:insert_result(rule['symbol_fail'], 0.0, 'SAVDI (Protocol error (REJ 1)):' .. data) - else - rspamd_logger.errx(task, 'unhandled response: %s', data) - task:insert_result(rule['symbol_fail'], 0.0, 'unhandled response') - end - - end - end - end - - tcp.request({ - task = task, - host = addr:to_string(), - port = addr:get_port(), - timeout = rule['timeout'], - callback = sophos_callback, - data = { protocol, streamsize, content, bye } - }) - end - - if need_av_check(task, content, rule) then - if check_av_cache(task, digest, rule, sophos_check_uncached) then - return - else - sophos_check_uncached() - end - end -end - -local function savapi_check(task, content, digest, rule) - local function savapi_check_uncached () - local upstream = rule.upstreams:get_upstream_round_robin() - local addr = upstream:get_addr() - local retransmits = rule.retransmits - local message_file = task:store_in_file(tonumber("0644", 8)) - local vnames = {} - - -- Forward declaration for recursive calls - local savapi_scan1_cb - - local function savapi_fin_cb(err, conn) - local vnames_reordered = {} - -- Swap table - for virus,_ in pairs(vnames) do - table.insert(vnames_reordered, virus) - end - lua_util.debugm(N, task, "%s: number of virus names found %s", rule['type'], #vnames_reordered) - if #vnames_reordered > 0 then - local vname = {} - for _,virus in ipairs(vnames_reordered) do - table.insert(vname, virus) - end - - yield_result(task, rule, vname) - save_av_cache(task, digest, rule, vname) - end - if conn then - conn:close() - end - end - - local function savapi_scan2_cb(err, data, conn) - local result = tostring(data) - lua_util.debugm(N, task, "%s: got reply: %s", rule['type'], result) - - -- Terminal response - clean - if string.find(result, '200') or string.find(result, '210') then - if rule['log_clean'] then - rspamd_logger.infox(task, '%s: message or mime_part is clean', rule['type']) - end - save_av_cache(task, digest, rule, 'OK') - conn:add_write(savapi_fin_cb, 'QUIT\n') - - -- Terminal response - infected - elseif string.find(result, '319') then - conn:add_write(savapi_fin_cb, 'QUIT\n') - - -- Non-terminal response - elseif string.find(result, '310') then - local virus - virus = result:match "310.*<<<%s(.*)%s+;.*;.*" - if not virus then - virus = result:match "310%s(.*)%s+;.*;.*" - if not virus then - rspamd_logger.errx(task, "%s: virus result unparseable: %s", rule['type'], result) - return - end - end - -- Store unique virus names - vnames[virus] = 1 - -- More content is expected - conn:add_write(savapi_scan1_cb, '\n') - end - end - - savapi_scan1_cb = function(err, conn) - conn:add_read(savapi_scan2_cb, '\n') - end - - -- 100 PRODUCT:xyz - local function savapi_greet2_cb(err, data, conn) - local result = tostring(data) - if string.find(result, '100 PRODUCT') then - lua_util.debugm(N, task, "%s: scanning file: %s", rule['type'], message_file) - conn:add_write(savapi_scan1_cb, {string.format('SCAN %s\n', message_file)}) - else - rspamd_logger.errx(task, '%s: invalid product id %s', rule['type'], rule['product_id']) - conn:add_write(savapi_fin_cb, 'QUIT\n') - end - end - - local function savapi_greet1_cb(err, conn) - conn:add_read(savapi_greet2_cb, '\n') - end - - local function savapi_callback_init(err, data, conn) - if err then - - -- set current upstream to fail because an error occurred - upstream:fail() - - -- retry with another upstream until retransmits exceeds - if retransmits > 0 then - - retransmits = retransmits - 1 - - -- Select a different upstream! - upstream = rule.upstreams:get_upstream_round_robin() - addr = upstream:get_addr() - - lua_util.debugm(N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr) - - tcp.request({ - task = task, - host = addr:to_string(), - port = addr:get_port(), - timeout = rule['timeout'], - callback = savapi_callback_init, - stop_pattern = {'\n'}, - }) - else - rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type']) - task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and retransmits exceed') - end - else - upstream:ok() - local result = tostring(data) - - -- 100 SAVAPI:4.0 greeting - if string.find(result, '100') then - conn:add_write(savapi_greet1_cb, {string.format('SET PRODUCT %s\n', rule['product_id'])}) - end - end - end - - tcp.request({ - task = task, - host = addr:to_string(), - port = addr:get_port(), - timeout = rule['timeout'], - callback = savapi_callback_init, - stop_pattern = {'\n'}, - }) - end - - if need_av_check(task, content, rule) then - if check_av_cache(task, digest, rule, savapi_check_uncached) then - return - else - savapi_check_uncached() - end - end -end - -local av_types = { - clamav = { - configure = clamav_config, - check = clamav_check - }, - fprot = { - configure = fprot_config, - check = fprot_check - }, - sophos = { - configure = sophos_config, - check = sophos_check - }, - savapi = { - configure = savapi_config, - check = savapi_check - }, -} local function add_antivirus_rule(sym, opts) if not opts['type'] then @@ -830,8 +75,14 @@ local function add_antivirus_rule(sym, opts) return nil end - if not opts['symbol'] then opts['symbol'] = sym end - local cfg = av_types[opts['type']] + if not opts['symbol'] then opts['symbol'] = sym:upper() end + local cfg = lua_antivirus.av_types[opts['type']] + + if not cfg then + rspamd_logger.errx(rspamd_config, 'unknown antivirus type: %s', + opts['type']) + return nil + end if not opts['symbol_fail'] then opts['symbol_fail'] = string.upper(opts['type']) .. '_FAIL' @@ -845,15 +96,10 @@ local function add_antivirus_rule(sym, opts) end -- WORKAROUND for deprecated attachments_only - if not cfg then - rspamd_logger.errx(rspamd_config, 'unknown antivirus type: %s', - opts['type']) - end - local rule = cfg.configure(opts) rule.type = opts.type rule.symbol_fail = opts.symbol_fail - + rule.redis_params = redis_params if not rule then rspamd_logger.errx(rspamd_config, 'cannot configure %s for %s', @@ -890,21 +136,20 @@ local function add_antivirus_rule(sym, opts) if rule.scan_mime_parts then local parts = task:get_parts() or {} - for _,p in ipairs(parts) do - if ( - (p:is_image() and rule.scan_image_mime) - or (p:is_text() and rule.scan_text_mime) - or (p:is_multipart() and rule.scan_text_mime) - or (not p:is_image() and not p:is_text() and not p:is_multipart()) - ) then + local filter_func = function(p) + return (rule.scan_image_mime and p:is_image()) + or (rule.scan_text_mime and p:is_text()) + or (p:is_attachment()) + end - local content = p:get_content() + fun.each(function(p) + local content = p:get_content() - if content and #content > 0 then - cfg.check(task, content, p:get_digest(), rule) - end + if content and #content > 0 then + cfg.check(task, content, p:get_digest(), rule) end - end + end, fun.filter(filter_func, parts)) + else cfg.check(task, task:get_content(), task:get_digest(), rule) end @@ -917,8 +162,10 @@ if opts and type(opts) == 'table' then redis_params = rspamd_parse_redis_server('antivirus') local has_valid = false for k, m in pairs(opts) do - if type(m) == 'table' and m['type'] and m['servers'] then + if type(m) == 'table' and m.servers then + if not m.type then m.type = k end local cb = add_antivirus_rule(k, m) + if not cb then rspamd_logger.errx(rspamd_config, 'cannot add rule: "' .. k .. '"') else diff --git a/src/plugins/lua/arc.lua b/src/plugins/lua/arc.lua index eeae65289..e940db619 100644 --- a/src/plugins/lua/arc.lua +++ b/src/plugins/lua/arc.lua @@ -483,9 +483,16 @@ local function arc_sign_seal(task, params, header) sha_ctx:update(s) lua_util.debugm(N, task, 'initial update signature with header: %s', s) + local nl_type + if task:has_flag("milter") then + nl_type = "lf" + else + nl_type = task:get_newlines_type() + end + local sig = rspamd_rsa.sign_memory(privkey, sha_ctx:bin()) cur_arc_seal = string.format('%s%s', cur_arc_seal, - sig:base64()) + sig:base64(70, nl_type)) task:set_milter_reply({ add_headers = { @@ -591,7 +598,7 @@ local function arc_signing_cb(task) if err and err == 'No such file or directory' then lua_util.debugm(N, task, 'cannot read key from %s: %s', p.key, err) else - rspamd_logger.warnx(N, task, 'cannot read key from %s: %s', p.key, err) + rspamd_logger.warnx(task, 'cannot read key from %s: %s', p.key, err) end return false end diff --git a/src/plugins/lua/asn.lua b/src/plugins/lua/asn.lua index 86f1c42d2..2e5b8466e 100644 --- a/src/plugins/lua/asn.lua +++ b/src/plugins/lua/asn.lua @@ -74,8 +74,9 @@ local function asn_check(task) asn_set(parts[1], parts[2], parts[3]) end - task:get_resolver():resolve_txt(task:get_session(), task:get_mempool(), - req_name, rspamd_dns_cb) + task:get_resolver():resolve_txt({task = task, + name = req_name, + callback = rspamd_dns_cb}) end local ip = task:get_from_ip() diff --git a/src/plugins/lua/bayes_expiry.lua b/src/plugins/lua/bayes_expiry.lua index 9495cf0cd..d15ed43ac 100644 --- a/src/plugins/lua/bayes_expiry.lua +++ b/src/plugins/lua/bayes_expiry.lua @@ -44,15 +44,19 @@ local function check_redis_classifier(cls, cfg) if cls.new_schema then local symbol_spam, symbol_ham local expiry = (cls.expiry or cls.expire) + if type(expiry) == 'table' then + expiry = expiry[1] + end + if cls.lazy then settings.lazy = cls.lazy end -- Load symbols from statfiles - local statfiles = cls.statfile - for _,stf in ipairs(statfiles) do - local symbol = stf.symbol or 'undefined' + + local function check_statfile_table(tbl, def_sym) + local symbol = tbl.symbol or def_sym local spam - if stf.spam then - spam = stf.spam + if tbl.spam then + spam = tbl.spam else if string.match(symbol:upper(), 'SPAM') then spam = true @@ -68,7 +72,27 @@ local function check_redis_classifier(cls, cfg) end end - if not symbol_spam or not symbol_ham or not expiry then + local statfiles = cls.statfile + if statfiles[1] then + for _,stf in ipairs(statfiles) do + if not stf.symbol then + for k,v in pairs(stf) do + check_statfile_table(v, k) + end + else + check_statfile_table(stf, 'undefined') + end + end + else + for stn,stf in pairs(statfiles) do + check_statfile_table(stf, stn) + end + end + + if not symbol_spam or not symbol_ham or type(expiry) ~= 'number' then + logger.debugm(N, rspamd_config, + 'disable expiry for classifier %s: no expiry %s', + symbol_spam, cls) return end -- Now try to load redis_params if needed @@ -88,6 +112,9 @@ local function check_redis_classifier(cls, cfg) return end + logger.debugm(N, rspamd_config, "enabled expiry for %s/%s -> %s expiry", + symbol_spam, symbol_ham, expiry) + table.insert(settings.classifiers, { symbol_spam = symbol_spam, symbol_ham = symbol_ham, @@ -391,24 +418,17 @@ rspamd_config:add_on_load(function (_, ev_base, worker) local unique_redis_params = {} -- Push redis script to all unique redis servers for _,cls in ipairs(settings.classifiers) do - local seen = false - for _,rp in ipairs(unique_redis_params) do - if lutil.table_cmp(rp, cls.redis_params) then - seen = true - end - end - - if not seen then - table.insert(unique_redis_params, cls.redis_params) + if not unique_redis_params[cls.redis_params.hash] then + unique_redis_params[cls.redis_params.hash] = cls.redis_params end end - for _,rp in ipairs(unique_redis_params) do + for h,rp in pairs(unique_redis_params) do local script_id = lredis.add_redis_script(lutil.template(expiry_script, template), rp) for _,cls in ipairs(settings.classifiers) do - if lutil.table_cmp(rp, cls.redis_params) then + if cls.redis_params.hash == h then cls.script = script_id end end diff --git a/src/plugins/lua/clickhouse.lua b/src/plugins/lua/clickhouse.lua index d95325b24..5fc642760 100644 --- a/src/plugins/lua/clickhouse.lua +++ b/src/plugins/lua/clickhouse.lua @@ -50,6 +50,7 @@ local settings = { full_urls = false, from_tables = nil, enable_symbols = false, + database = 'default', use_https = false, use_gzip = true, allow_local = false, @@ -285,6 +286,7 @@ local function clickhouse_send_data(task) end local function clickhouse_collect(task) + if task:has_flag('skip') then return end if not settings.allow_local and rspamd_lua_utils.is_rspamc_or_controller(task) then return end for _,sym in ipairs(settings.stop_symbols) do @@ -488,27 +490,26 @@ local function clickhouse_collect(task) table.insert(row, {}) end + local flatten_urls = function(f, ...) + return fun.totable(fun.map(function(k,v) return f(k,v) end, ...)) + end + -- Urls step - local urls_tlds = {} local urls_urls = {} if task:has_urls(false) then + for _,u in ipairs(task:get_urls(false)) do - urls_tlds[u:get_tld()] = true if settings['full_urls'] then - urls_urls[u:get_text()] = true + urls_urls[u:get_text()] = u else - urls_urls[u:get_host()] = true + urls_urls[u:get_host()] = u end end - end - - local flatten_urls = function(...) - return fun.totable(fun.map(function(k,_) return k end, ...)) - end - if #urls_tlds > 0 then - table.insert(row, flatten_urls(urls_tlds)) - table.insert(row, flatten_urls(urls_urls)) + -- Get tlds + table.insert(row, flatten_urls(function(_,u) return u:get_tld() end, urls_urls)) + -- Get hosts/full urls + table.insert(row, flatten_urls(function(k, _) return k end, urls_urls)) else table.insert(row, {}) table.insert(row, {}) @@ -516,9 +517,10 @@ local function clickhouse_collect(task) -- Emails step if task:has_urls(true) then - table.insert(row, flatten_urls(fun.map(function(u) - return string.format('%s@%s', u:get_user(), u:get_host()),true - end, task:get_emails()))) + table.insert(row, flatten_urls(function(k, _) return k end, + fun.map(function(u) + return string.format('%s@%s', u:get_user(), u:get_host()),true + end, task:get_emails()))) else table.insert(row, {}) end diff --git a/src/plugins/lua/clustering.lua b/src/plugins/lua/clustering.lua index d6c78ef79..a5d72fc64 100644 --- a/src/plugins/lua/clustering.lua +++ b/src/plugins/lua/clustering.lua @@ -186,18 +186,21 @@ local function clusterting_filter_cb(task, rule) end local function clusterting_idempotent_cb(task, rule) - local action = task:get_action() + if task:has_flag('skip') then return end + if not rule.allow_local and lua_util.is_rspamc_or_controller(task) then return end + + local verdict = lua_util.get_task_verdict(task) local score - if action == 'no action' then + if verdict == 'ham' then score = rule.ham_mult - elseif action == 'reject' then + elseif verdict == 'spam' then score = rule.spam_mult - elseif action == 'add header' or action == 'rewrite subject' then + elseif verdict == 'junk' then score = rule.junk_mult else - rspamd_logger.debugm(N, task, 'skip rule %s, action=%s', - rule.name, action) + rspamd_logger.debugm(N, task, 'skip rule %s, verdict=%s', + rule.name, verdict) return end diff --git a/src/plugins/lua/dcc.lua b/src/plugins/lua/dcc.lua index 311dc608e..8c5dddeeb 100644 --- a/src/plugins/lua/dcc.lua +++ b/src/plugins/lua/dcc.lua @@ -128,10 +128,11 @@ local function check_dcc (task) else rspamd_logger.errx(task, 'failed to scan, maximum retransmits exceed') - upstream:fail() + if upstream then upstream:fail() end end else -- Parse the response + if upstream then upstream:ok() end local _,_,result,disposition,header = tostring(data):find("(.-)\n(.-)\n(.-)\n") lua_util.debugm(N, task, 'DCC result=%1 disposition=%2 header="%3"', result, disposition, header) diff --git a/src/plugins/lua/dkim_signing.lua b/src/plugins/lua/dkim_signing.lua index 8d621bbb2..77acc2f61 100644 --- a/src/plugins/lua/dkim_signing.lua +++ b/src/plugins/lua/dkim_signing.lua @@ -163,7 +163,7 @@ local function dkim_signing_cb(task) if err and err == 'No such file or directory' then lua_util.debugm(N, task, 'cannot read key from "%s": %s', p.key, err) else - rspamd_logger.warnx(N, task, 'cannot read key from "%s": %s', p.key, err) + rspamd_logger.warnx(task, 'cannot read key from "%s": %s', p.key, err) end return false end diff --git a/src/plugins/lua/dmarc.lua b/src/plugins/lua/dmarc.lua index b55c5d41f..1bac6fb29 100644 --- a/src/plugins/lua/dmarc.lua +++ b/src/plugins/lua/dmarc.lua @@ -415,12 +415,23 @@ local function dmarc_validate_policy(task, policy, hdrfromdom, dmarc_esld) policy.domain .. ' : ' .. reason_str, policy.dmarc_policy) disposition = what else - if (math.random(100) > policy.pct) then + local coin = math.random(100) + if (coin > policy.pct) then if (not no_sampling_domains or not no_sampling_domains:get_key(policy.domain)) then - task:insert_result(dmarc_symbols['softfail'], 1.0, + + if what == 'reject' then + disposition = 'quarantine' + else + disposition = 'softfail' + end + + task:insert_result(dmarc_symbols[disposition], 1.0, policy.domain .. ' : ' .. reason_str, policy.dmarc_policy, "sampled_out") sampled_out = true + lua_util.debugm(N, task, + 'changed dmarc policy from %s to %s, sampled out: %s < %s', + what, disposition, coin, policy.pct) else task:insert_result(dmarc_symbols[what], 1.0, policy.domain .. ' : ' .. reason_str, policy.dmarc_policy, "local_policy") diff --git a/src/plugins/lua/elastic.lua b/src/plugins/lua/elastic.lua index 87e01063a..aa3702112 100644 --- a/src/plugins/lua/elastic.lua +++ b/src/plugins/lua/elastic.lua @@ -20,7 +20,6 @@ local rspamd_http = require "rspamd_http" local lua_util = require "lua_util" local util = require "rspamd_util" local ucl = require "ucl" -local hash = require "rspamd_cryptobox_hash" local rspamd_redis = require "lua_redis" local upstream_list = require "rspamd_upstream_list" @@ -30,6 +29,7 @@ end local rows = {} local nrows = 0 +local failed_sends = 0 local elastic_template local redis_params local N = "elastic" @@ -37,7 +37,7 @@ local E = {} local connect_prefix = 'http://' local enabled = true local settings = { - limit = 10, + limit = 500, index_pattern = 'rspamd-%Y.%m.%d', template_file = rspamd_paths['PLUGINSDIR'] .. '/elastic/rspamd_template.json', kibana_file = rspamd_paths['PLUGINSDIR'] ..'/elastic/kibana.json', @@ -52,6 +52,7 @@ local settings = { user = nil, password = nil, no_ssl_verify = false, + max_fail = 3, } local function read_file(path) @@ -78,34 +79,7 @@ local function elastic_send_data(task) local push_url = connect_prefix .. ip_addr .. '/'..es_index..'/_bulk' local bulk_json = table.concat(tbl, "\n") - local function http_index_data_callback(err, code, body, _) - -- todo error handling we may store the rows it into redis and send it again late - lua_util.debugm(N, task, "After create data %1", body) - if code ~= 200 then - rspamd_logger.infox(task, "cannot push data to elastic backend (%s): %s (%s)", - push_url, err, code) - if settings['failover'] then - local h = hash.create() - h:update(bulk_json) - local key = settings['key_prefix'] ..es_index..":".. h:base32():sub(1, 20) - local data = util.zstd_compress(bulk_json) - local function redis_set_cb(rerr) - if rerr ~=nil then - rspamd_logger.errx(task, 'redis_set_cb received error: %1', rerr) - end - end - rspamd_redis.make_request(task, - redis_params, -- connect params - key, -- hash key - true, -- is write - redis_set_cb, --callback - 'SETEX', -- command - {key, tostring(settings['expire']), data} -- arguments - ) - end - end - end - rspamd_http.request({ + local err, response = rspamd_http.request({ url = push_url, headers = { ['Content-Type'] = 'application/x-ndjson', @@ -117,11 +91,27 @@ local function elastic_send_data(task) no_ssl_verify = settings.no_ssl_verify, user = settings.user, password = settings.password, - callback = http_index_data_callback, timeout = settings.timeout, }) + if err then + rspamd_logger.infox(task, "cannot push data to elastic backend (%s): %s; failed attempts: %s/%s", + push_url, err, failed_sends, settings.max_fail) + else + if response.code ~= 200 then + rspamd_logger.infox(task, "cannot push data to elastic backend (%s): wrong http code %s (%s); failed attempts: %s/%s", + push_url, err, response.code, failed_sends, settings.max_fail) + else + lua_util.debugm(N, task, "successfully sent %s (%s bytes) rows to ES", + nrows, #bulk_json) + + return true + end + end + + return false end + local function get_general_metadata(task) local r = {} local ip_addr = task:get_ip() @@ -175,6 +165,7 @@ local function get_general_metadata(task) else r.from = 'unknown' end + local syminf = task:get_symbols_all() r.symbols = syminf r.asn = {} @@ -182,6 +173,7 @@ local function get_general_metadata(task) r.asn.country = pool:get_variable("country") or 'unknown' r.asn.asn = pool:get_variable("asn") or 0 r.asn.ipnet = pool:get_variable("ipnet") or 'unknown' + local function process_header(name) local hdr = task:get_header_full(name) if hdr then @@ -208,15 +200,30 @@ end local function elastic_collect(task) if not enabled then return end + if task:has_flag('skip') then return end if not settings.allow_local and lua_util.is_rspamc_or_controller(task) then return end + local row = {['rspamd_meta'] = get_general_metadata(task), ['@timestamp'] = tostring(util.get_time() * 1000)} table.insert(rows, row) nrows = nrows + 1 if nrows > settings['limit'] then - elastic_send_data(task) - nrows = 0 - rows = {} + lua_util.debugm(N, task, 'send elastic search rows: %s', nrows) + if elastic_send_data(task) then + nrows = 0 + rows = {} + failed_sends = 0; + else + failed_sends = failed_sends + 1 + + if failed_sends > settings.max_fail then + rspamd_logger.errx(task, 'cannot send %s rows to ES %s times, stop trying', + nrows, failed_sends) + nrows = 0 + rows = {} + failed_sends = 0; + end + end end end @@ -244,6 +251,7 @@ local function check_elastic_server(cfg, ev_base, _) for _,plugin in pairs(value['plugins']) do if plugin['name'] == 'ingest-geoip' then plugin_found = true + lua_util.debugm(N, "ingest-geoip plugin has been found") end end if not plugin_found then @@ -274,7 +282,7 @@ end -- import ingest pipeline and kibana dashboard/visualization local function initial_setup(cfg, ev_base, worker) - if not (worker:get_name() == 'controller' and worker:get_index() == 0) then return end + if not worker:is_primary_controller() then return end local upstream = settings.upstream:get_upstream_round_robin() local ip_addr = upstream:get_addr():to_string(true) diff --git a/src/plugins/lua/emails.lua b/src/plugins/lua/emails.lua index 282f53bb4..1727e49e1 100644 --- a/src/plugins/lua/emails.lua +++ b/src/plugins/lua/emails.lua @@ -185,14 +185,15 @@ local function gen_check_emails(rule) end local replyto = get_raw_header('Reply-To') - if not replyto then return false end - local rt = util.parse_mail_address(replyto, task:get_mempool()) - - if rt and rt[1] then - rspamd_lua_utils.remove_email_aliases(rt[1]) - if not checked[rt[1].addr] then - check_email_rule(task, rule, rt[1]) - checked[rt[1].addr] = true + if replyto then + local rt = util.parse_mail_address(replyto, task:get_mempool()) + + if rt and rt[1] then + rspamd_lua_utils.remove_email_aliases(rt[1]) + if not checked[rt[1].addr] then + check_email_rule(task, rule, rt[1]) + checked[rt[1].addr] = true + end end end end diff --git a/src/plugins/lua/forged_recipients.lua b/src/plugins/lua/forged_recipients.lua index 887b1bf82..8abc55a50 100644 --- a/src/plugins/lua/forged_recipients.lua +++ b/src/plugins/lua/forged_recipients.lua @@ -40,36 +40,38 @@ local function check_forged_headers(task) if not mime_rcpt then return elseif #mime_rcpt == 0 then - local sra = smtp_rcpt[1].addr .. (#smtp_rcpt > 1 and ' ...' or '') - task:insert_result(symbol_rcpt, score, '', sra) return end -- Find pair for each smtp recipient in To or Cc headers for _,sr in ipairs(smtp_rcpt) do res = false for _,mr in ipairs(mime_rcpt) do - if mr['addr'] and sr['addr'] and - string.lower(mr['addr']) == string.lower(sr['addr']) then + if mr.addr and mr.addr ~= '' then + if sr['addr'] and + string.lower(mr['addr']) == string.lower(sr['addr']) then + res = true + break + elseif delivered_to and delivered_to == mr['addr'] then + -- allow alias expansion and forwarding (Postfix) + res = true + break + elseif auser and auser == sr['addr'] then + -- allow user to BCC themselves + res = true + break + elseif ((smtp_from or E)[1] or E).addr and + smtp_from[1]['addr'] == sr['addr'] then + -- allow sender to BCC themselves + res = true + break + elseif mr['user'] and sr['user'] and + string.lower(mr['user']) == string.lower(sr['user']) then + -- If we have the same username but for another domain, then + -- lower the overall score + score = score / 2 + end + else res = true - break - elseif delivered_to and delivered_to == mr['addr'] then - -- allow alias expansion and forwarding (Postfix) - res = true - break - elseif auser and auser == sr['addr'] then - -- allow user to BCC themselves - res = true - break - elseif ((smtp_from or E)[1] or E).addr and - smtp_from[1]['addr'] == sr['addr'] then - -- allow sender to BCC themselves - res = true - break - elseif mr['user'] and sr['user'] and - string.lower(mr['user']) == string.lower(sr['user']) then - -- If we have the same username but for another domain, then - -- lower the overall score - score = score / 2 end end if not res then diff --git a/src/plugins/lua/greylist.lua b/src/plugins/lua/greylist.lua index 7427f999e..5a1f6c0f9 100644 --- a/src/plugins/lua/greylist.lua +++ b/src/plugins/lua/greylist.lua @@ -43,7 +43,7 @@ end local redis_params local whitelisted_ip -local whitelist_domains_map = nil +local whitelist_domains_map local toint =math.ifloor or math.floor local settings = { expire = 86400, -- 1 day by default @@ -56,6 +56,8 @@ local settings = { ipv4_mask = 19, -- Mask bits for ipv4 ipv6_mask = 64, -- Mask bits for ipv6 report_time = false, -- Tell when greylisting is epired (appended to `message`) + check_local = false, + check_authed = false, } local rspamd_logger = require "rspamd_logger" @@ -132,7 +134,7 @@ local function envelope_key(task) end -- Returns pair of booleans: found,greylisted -local function check_time(task, tm, type) +local function check_time(task, tm, type, now) local t = tonumber(tm) if not t then @@ -140,7 +142,6 @@ local function check_time(task, tm, type) return false,false end - local now = rspamd_util.get_time() if now - t < settings['timeout'] then return true,true else @@ -154,7 +155,10 @@ end local function greylist_message(task, end_time, why) task:insert_result(settings['symbol'], 0.0, 'greylisted', end_time, why) - if rspamd_lua_utils.is_rspamc_or_controller(task) then return end + if not settings.check_local and rspamd_lua_utils.is_rspamc_or_controller(task) then + return + end + if settings.message_func then task:set_pre_result(settings['action'], settings.message_func(task, end_time), N) @@ -172,7 +176,9 @@ end local function greylist_check(task) local ip = task:get_ip() - if task:get_user() or (ip and ip:is_local()) then + if ((not settings.check_authed and task:get_user()) or + (not settings.check_local and ip and ip:is_local())) then + rspamd_logger.infox(task, "skip greylisting for local networks and/or authorized users"); return end @@ -195,36 +201,54 @@ local function greylist_check(task) local greylisted_meta = false if data then + local end_time_body,end_time_meta + local now = rspamd_util.get_time() + if data[1] and type(data[1]) ~= 'userdata' then - ret_body,greylisted_body = check_time(task, data[1], 'body') + local tm = tonumber(data[1]) or now + ret_body,greylisted_body = check_time(task, data[1], 'body', now) if greylisted_body then - local end_time = rspamd_util.time_to_string(rspamd_util.get_time() - + settings['timeout']) - task:get_mempool():set_variable("grey_greylisted_body", end_time) + end_time_body = tm + settings['timeout'] + task:get_mempool():set_variable("grey_greylisted_body", + rspamd_util.time_to_string(end_time_body)) end end + if data[2] and type(data[2]) ~= 'userdata' then if not ret_body or greylisted_body then - ret_meta,greylisted_meta = check_time(task, data[2], 'meta') + local tm = tonumber(data[2]) or now + ret_meta,greylisted_meta = check_time(task, data[2], 'meta', now) if greylisted_meta then - local end_time = rspamd_util.time_to_string(rspamd_util.get_time() - + settings['timeout']) - task:get_mempool():set_variable("grey_greylisted_meta", end_time) + end_time_meta = tm + settings['timeout'] + task:get_mempool():set_variable("grey_greylisted_meta", + rspamd_util.time_to_string(end_time_meta)) end end end + local how + local end_time_str + if not ret_body and not ret_meta then - local end_time = rspamd_util.time_to_string(rspamd_util.get_time() - + settings['timeout']) - task:get_mempool():set_variable("grey_greylisted", end_time) + -- no record found + task:get_mempool():set_variable("grey_greylisted", 'true') elseif greylisted_body and greylisted_meta then - local end_time = rspamd_util.time_to_string(rspamd_util.get_time() + - settings['timeout']) - rspamd_logger.infox(task, 'greylisted until "%s"', - end_time) - greylist_message(task, end_time, 'too early') + end_time_str = rspamd_util.time_to_string( + math.min(end_time_body, end_time_meta)) + how = 'meta and body' + elseif greylisted_body then + end_time_str = rspamd_util.time_to_string(end_time_body) + how = 'body only' + elseif greylisted_meta then + end_time_str = rspamd_util.time_to_string(end_time_meta) + how = 'meta only' + end + + if how and end_time_str then + rspamd_logger.infox(task, 'greylisted until "%s" (%s)', + end_time_str, how) + greylist_message(task, end_time_str, 'too early') end elseif err then rspamd_logger.errx(task, 'got error while getting greylisting keys: %1', err) @@ -265,7 +289,8 @@ local function greylist_set(task) end end - if task:get_user() or (ip and ip:is_local()) then + if ((not settings.check_authed and task:get_user()) or + (not settings.check_local and ip and ip:is_local())) then if action == 'greylist' then -- We are going to accept message rspamd_logger.infox(task, 'Downgrading metric action from "greylist" to "no action"') @@ -331,7 +356,7 @@ local function greylist_set(task) is_whitelisted, rspamd_util.time_to_string(rspamd_util.get_time() + settings['expire'])) - if is_rspamc then return end + if not settings.check_local and is_rspamc then return end ret,conn,upstream = rspamd_redis_make_request(task, redis_params, -- connect params @@ -350,7 +375,7 @@ local function greylist_set(task) rspamd_logger.errx(task, 'got error while connecting to redis') end elseif do_greylisting or do_greylisting_required then - if is_rspamc then return end + if not settings.check_local and is_rspamc then return end local t = tostring(toint(rspamd_util.get_time())) local end_time = rspamd_util.time_to_string(t + settings['timeout']) rspamd_logger.infox(task, 'greylisted until "%s", new record', end_time) diff --git a/src/plugins/lua/metadata_exporter.lua b/src/plugins/lua/metadata_exporter.lua index 57fe0f105..951142368 100644 --- a/src/plugins/lua/metadata_exporter.lua +++ b/src/plugins/lua/metadata_exporter.lua @@ -688,6 +688,7 @@ end local function gen_exporter(rule) return function (task) + if task:has_flag('skip') then return end local selector = rule.selector or 'default' local selected = selectors[selector](task) if selected then diff --git a/src/plugins/lua/milter_headers.lua b/src/plugins/lua/milter_headers.lua index 209985bbd..7659a3fe6 100644 --- a/src/plugins/lua/milter_headers.lua +++ b/src/plugins/lua/milter_headers.lua @@ -25,6 +25,7 @@ local logger = require "rspamd_logger" local util = require "rspamd_util" local N = 'milter_headers' local lua_util = require "lua_util" +local ts = require("tableshape").types local E = {} local HOSTNAME = util.get_hostname() @@ -518,9 +519,35 @@ local function milter_headers(task) end end -local opts = rspamd_config:get_all_opt(N) or rspamd_config:get_all_opt('rmilter_headers') +local config_schema = ts.shape{ + use = ts.array_of(ts.string) + ts.string / function(s) return {s} end, + remove_upstream_spam_flag = ts.boolean:is_optional(), + extended_spam_headers = ts.boolean:is_optional(), + skip_local = ts.boolean:is_optional(), + skip_authenticated = ts.boolean:is_optional(), + local_headers = ts.array_of(ts.string):is_optional(), + authenticated_headers = ts.array_of(ts.string):is_optional(), + extended_headers_rcpt = + (ts.array_of(ts.string) + ts.string / function(s) return {s} end):is_optional(), + custom = ts.map_of(ts.string, ts.string):is_optional(), +} + +local opts = rspamd_config:get_all_opt(N) or + rspamd_config:get_all_opt('rmilter_headers') + if not opts then return end +-- Process config +do + local res,err = config_schema:transform(opts) + if not res then + logger.errx(rspamd_config, 'invalid config for %s: %s', N, err) + return + else + opts = res + end +end + local have_routine = {} local function activate_routine(s) if settings.routines[s] or custom_routines[s] then @@ -537,33 +564,28 @@ local function activate_routine(s) logger.errx(rspamd_config, 'routine "%s" does not exist', s) end end -if opts['remove_upstream_spam_flag'] then activate_routine('remove-spam-flag') end -if opts['extended_spam_headers'] then + +if opts.remove_upstream_spam_flag ~= nil then + settings.remove_upstream_spam_flag = opts.remove_upstream_spam_flag +end + +if opts.extended_spam_headers then activate_routine('x-spamd-result') activate_routine('x-rspamd-server') activate_routine('x-rspamd-queue-id') end -if type(opts['use']) == 'string' then - opts['use'] = {opts['use']} -elseif (type(opts['use']) == 'table' and not opts['use'][1] and not next(active_routines)) then - logger.debugm(N, rspamd_config, 'no functions are enabled') - return -end -if type(opts['use']) ~= 'table' then - logger.errx(rspamd_config, 'unexpected type for "use" option: %s', type(opts['use'])) - return -end -if type(opts['local_headers']) == 'table' and opts['local_headers'][1] then - for _, h in ipairs(opts['local_headers']) do + +if opts.local_headers then + for _, h in ipairs(opts.local_headers) do settings.local_headers[h] = true end end -if type(opts['authenticated_headers']) == 'table' and opts['authenticated_headers'][1] then - for _, h in ipairs(opts['authenticated_headers']) do +if opts.authenticated_headers then + for _, h in ipairs(opts.authenticated_headers) do settings.authenticated_headers[h] = true end end -if type(opts['custom']) == 'table' then +if opts.custom then for k, v in pairs(opts['custom']) do local f, err = load(v) if not f then @@ -573,27 +595,35 @@ if type(opts['custom']) == 'table' then end end end + if type(opts['skip_local']) == 'boolean' then settings.skip_local = opts['skip_local'] end + if type(opts['skip_authenticated']) == 'boolean' then settings.skip_authenticated = opts['skip_authenticated'] end + for _, s in ipairs(opts['use']) do if not have_routine[s] then activate_routine(s) end end + +if settings.remove_upstream_spam_flag then + activate_routine('remove-spam-flag') +end + if (#active_routines < 1) then logger.errx(rspamd_config, 'no active routines') return end -logger.infox(rspamd_config, 'active routines [%s]', table.concat(active_routines, ',')) -if type(opts['extended_headers_rcpt']) == 'string' then - opts['extended_headers_rcpt'] = {opts['extended_headers_rcpt']} -end -if type(opts['extended_headers_rcpt']) == 'table' and opts['extended_headers_rcpt'][1] then - for _, e in ipairs(opts['extended_headers_rcpt']) do + +logger.infox(rspamd_config, 'active routines [%s]', + table.concat(active_routines, ',')) + +if opts.extended_headers_rcpt then + for _, e in ipairs(opts.extended_headers_rcpt) do if string.find(e, '^[^@]+@[^@]+$') then if not settings.extended_headers_rcpt.addr then settings.extended_headers_rcpt.addr = {} @@ -617,6 +647,7 @@ if type(opts['extended_headers_rcpt']) == 'table' and opts['extended_headers_rcp end end end + rspamd_config:register_symbol({ name = 'MILTER_HEADERS', type = 'idempotent', diff --git a/src/plugins/lua/mime_types.lua b/src/plugins/lua/mime_types.lua index e8ce709da..36407dac0 100644 --- a/src/plugins/lua/mime_types.lua +++ b/src/plugins/lua/mime_types.lua @@ -599,7 +599,7 @@ local full_extensions_map = { {"roff", "application/x-troff"}, {"rpm", "audio/x-pn-realaudio-plugin"}, {"rqy", "text/x-ms-rqy"}, - {"rtf", {"application/rtf","application/msword"}}, + {"rtf", {"application/rtf","application/msword", "text/richtext"}}, {"rtx", "text/richtext"}, {"rvt", "application/octet-stream" }, {"ruleset", "application/xml"}, @@ -813,7 +813,7 @@ local function check_mime_type(task) return ext[1],ext[2],parts end - local function check_filename(fname, ct, is_archive) + local function check_filename(fname, ct, is_archive, part) local ext,ext2,parts = gen_extension(fname) -- ext is the last extension, LOWERCASED -- ext2 is the one before last extension LOWERCASED @@ -834,12 +834,16 @@ local function check_mime_type(task) -- Double extension + bad extension == VERY bad task:insert_result(settings['symbol_double_extension'], badness_mult, string.format(".%s.%s", ext2, ext)) + task:insert_result('MIME_TRACE', 0.0, + string.format("%s:%s", part:get_id(), '-')) return end end if badness_mult then -- Just bad extension task:insert_result(settings['symbol_bad_extension'], badness_mult, ext) + task:insert_result('MIME_TRACE', 0.0, + string.format("%s:%s", part:get_id(), '-')) end end @@ -861,6 +865,8 @@ local function check_mime_type(task) if settings['archive_extensions'][ext] then -- Archive in archive task:insert_result(settings['symbol_archive_in_archive'], 1.0, ext) + task:insert_result('MIME_TRACE', 0.0, + string.format("%s:%s", part:get_id(), '-')) end else if ext2 then @@ -871,7 +877,10 @@ local function check_mime_type(task) -- Exclude multipart archive extensions, e.g. .zip.001 and not string.match(ext, '^%d+$') then - task:insert_result(settings['symbol_archive_in_archive'], 1.0, string.format(".%s.%s", ext2, ext)) + task:insert_result(settings['symbol_archive_in_archive'], + 1.0, string.format(".%s.%s", ext2, ext)) + task:insert_result('MIME_TRACE', 0.0, + string.format("%s:%s", part:get_id(), '-')) end else check_extension(settings['bad_extensions'][ext], nil) @@ -902,17 +911,24 @@ local function check_mime_type(task) if parts then for _,p in ipairs(parts) do local mtype,subtype = p:get_type() + local dtype,dsubtype = p:get_detected_type() if not mtype then task:insert_result(settings['symbol_unknown'], 1.0, 'missing content type') + task:insert_result('MIME_TRACE', 0.0, + string.format("%s:%s", p:get_id(), '~')) else -- Check for attachment local filename = p:get_filename() local ct = string.format('%s/%s', mtype, subtype):lower() + local detected_ct + if dtype and dsubtype then + detected_ct = string.format('%s/%s', dtype, dsubtype) + end if filename then filename = filename:gsub('[^%s%g]', '?') - check_filename(filename, ct, false) + check_filename(filename, ct, false, p) end if p:is_archive() then @@ -932,6 +948,8 @@ local function check_mime_type(task) if arch:is_encrypted() then task:insert_result(settings['symbol_encrypted_archive'], 1.0, filename) + task:insert_result('MIME_TRACE', 0.0, + string.format("%s:%s", p:get_id(), '-')) end if check then @@ -944,28 +962,60 @@ local function check_mime_type(task) end if f['encrypted'] then - task:insert_result(settings['symbol_encrypted_archive'], 1.0, f['name']) + task:insert_result(settings['symbol_encrypted_archive'], + 1.0, f['name']) + task:insert_result('MIME_TRACE', 0.0, + string.format("%s:%s", p:get_id(), '-')) end if f['name'] then - check_filename(f['name'], nil, true) + check_filename(f['name'], nil, true, p) end end end end if map then - local v = map:get_key(ct) + local v + local detected_different = false + if detected_ct and detected_ct ~= ct then + v = map:get_key(detected_ct) + detected_different = true + else + v = map:get_key(ct) + end if v then local n = tonumber(v) - if n > 0 then - task:insert_result(settings['symbol_bad'], n, ct) - elseif n < 0 then - task:insert_result(settings['symbol_good'], -n, ct) + if n then + if n > 0 then + if detected_different then + -- Penalize case + n = n * 1.5 + task:insert_result(settings['symbol_bad'], n, + string.format('%s:%s', ct, detected_ct)) + else + task:insert_result(settings['symbol_bad'], n, ct) + end + task:insert_result('MIME_TRACE', 0.0, + string.format("%s:%s", p:get_id(), '-')) + elseif n < 0 then + task:insert_result(settings['symbol_good'], -n, ct) + task:insert_result('MIME_TRACE', 0.0, + string.format("%s:%s", p:get_id(), '+')) + else + -- Neutral content type + task:insert_result('MIME_TRACE', 0.0, + string.format("%s:%s", p:get_id(), '~')) + end + else + logger.warnx(task, 'unknown value: "%s" for content type %s in the map', + v, ct) end else task:insert_result(settings['symbol_unknown'], 1.0, ct) + task:insert_result('MIME_TRACE', 0.0, + string.format("%s:%s", p:get_id(), '~')) end end end @@ -1071,6 +1121,13 @@ if opts then parent = id, group = 'mime_types', }) + rspamd_config:register_symbol({ + type = 'virtual,nostat', + name = 'MIME_TRACE', + parent = id, + group = 'mime_types', + score = 0, + }) else lua_util.disable_module(N, "config") end diff --git a/src/plugins/lua/neural.lua b/src/plugins/lua/neural.lua index 71ae3afb6..43d7bb127 100644 --- a/src/plugins/lua/neural.lua +++ b/src/plugins/lua/neural.lua @@ -904,6 +904,8 @@ local function check_anns(rule, _, ev_base) end local function ann_push_vector(task) + if task:has_flag('skip') then return end + if not settings.allow_local and lua_util.is_rspamc_or_controller(task) then return end local scores = task:get_metric_score() for _,rule in pairs(settings.rules) do local sid = "0" diff --git a/src/plugins/lua/ratelimit.lua b/src/plugins/lua/ratelimit.lua index 61d19966b..f2358a483 100644 --- a/src/plugins/lua/ratelimit.lua +++ b/src/plugins/lua/ratelimit.lua @@ -28,6 +28,7 @@ local lua_maps = require "lua_maps" local lua_util = require "lua_util" local rspamd_hash = require "rspamd_cryptobox_hash" local lua_selectors = require "lua_selectors" +local ts = require("tableshape").types -- A plugin that implements ratelimits using redis @@ -84,23 +85,26 @@ local bucket_check_script = [[ if last < tonumber(KEYS[2]) then local rate = tonumber(KEYS[3]) dynr = tonumber(redis.call('HGET', KEYS[1], 'dr')) / 10000.0 + if dynr == 0 then dynr = 0.0001 end rate = rate * dynr leaked = ((now - last) * rate) + if leaked > burst then leaked = burst end burst = burst - leaked redis.call('HINCRBYFLOAT', KEYS[1], 'b', -(leaked)) redis.call('HSET', KEYS[1], 'l', KEYS[2]) end + + dynb = tonumber(redis.call('HGET', KEYS[1], 'db')) / 10000.0 + if dynb == 0 then dynb = 0.0001 end + + if burst > 0 and (burst + 1) > tonumber(KEYS[4]) * dynb then + return {1, tostring(burst), tostring(dynr), tostring(dynb), tostring(leaked)} + end else burst = 0 redis.call('HSET', KEYS[1], 'b', '0') end - dynb = tonumber(redis.call('HGET', KEYS[1], 'db')) / 10000.0 - - if (burst + 1) > tonumber(KEYS[4]) * dynb then - return {1, tostring(burst), tostring(dynr), tostring(dynb), tostring(leaked)} - end - return {0, tostring(burst), tostring(dynr), tostring(dynb), tostring(leaked)} ]] local bucket_check_id @@ -132,20 +136,55 @@ local bucket_update_script = [[ return {1, 1, 1} end - local burst = tonumber(redis.call('HGET', KEYS[1], 'b')) - local db = tonumber(redis.call('HGET', KEYS[1], 'db')) / 10000 - local dr = tonumber(redis.call('HGET', KEYS[1], 'dr')) / 10000 + local dr, db = 1.0, 1.0 - if dr < tonumber(KEYS[5]) and dr > 1.0 / tonumber(KEYS[5]) then - dr = dr * tonumber(KEYS[3]) - redis.call('HSET', KEYS[1], 'dr', tostring(math.floor(dr * 10000))) + if tonumber(KEYS[5]) > 1 then + local rate_mult = tonumber(KEYS[3]) + local rate_limit = tonumber(KEYS[5]) + dr = tonumber(redis.call('HGET', KEYS[1], 'dr')) / 10000 + + if rate_mult > 1.0 and dr < rate_limit then + dr = dr * rate_mult + if dr > 0.0001 then + redis.call('HSET', KEYS[1], 'dr', tostring(math.floor(dr * 10000))) + else + redis.call('HSET', KEYS[1], 'dr', '1') + end + elseif rate_mult < 1.0 and dr > (1.0 / rate_limit) then + dr = dr * rate_mult + if dr > 0.0001 then + redis.call('HSET', KEYS[1], 'dr', tostring(math.floor(dr * 10000))) + else + redis.call('HSET', KEYS[1], 'dr', '1') + end + end end - if db < tonumber(KEYS[6]) and db > 1.0 / tonumber(KEYS[6]) then - db = db * tonumber(KEYS[4]) - redis.call('HSET', KEYS[1], 'db', tostring(math.floor(db * 10000))) + if tonumber(KEYS[6]) > 1 then + local rate_mult = tonumber(KEYS[4]) + local rate_limit = tonumber(KEYS[6]) + db = tonumber(redis.call('HGET', KEYS[1], 'db')) / 10000 + + if rate_mult > 1.0 and db < rate_limit then + db = db * rate_mult + if db > 0.0001 then + redis.call('HSET', KEYS[1], 'db', tostring(math.floor(db * 10000))) + else + redis.call('HSET', KEYS[1], 'db', '1') + end + elseif rate_mult < 1.0 and db > (1.0 / rate_limit) then + db = db * rate_mult + if db > 0.0001 then + redis.call('HSET', KEYS[1], 'db', tostring(math.floor(db * 10000))) + else + redis.call('HSET', KEYS[1], 'db', '1') + end + end end + local burst = tonumber(redis.call('HGET', KEYS[1], 'b')) + if burst < 0 then burst = 0 end + redis.call('HINCRBYFLOAT', KEYS[1], 'b', 1) redis.call('HSET', KEYS[1], 'l', KEYS[2]) redis.call('EXPIRE', KEYS[1], KEYS[7]) @@ -154,8 +193,8 @@ local bucket_update_script = [[ ]] local bucket_update_id --- message_func(task, limit_type, prefix, bucket) -local message_func = function(_, limit_type, _, _) +-- message_func(task, limit_type, prefix, bucket, limit_key) +local message_func = function(_, limit_type, _, _, _) return string.format('Ratelimit "%s" exceeded', limit_type) end @@ -229,48 +268,67 @@ local function parse_string_limit(lim, no_error) return nil end +local function str_to_rate(str) + local divider,divisor = parse_string_limit(str, false) + + if not divisor then + rspamd_logger.errx(rspamd_config, 'bad rate string: %s', str) + + return nil + end + + return divisor / divider +end + +local bucket_schema = ts.shape{ + burst = ts.number + ts.string / lua_util.dehumanize_number, + rate = ts.number + ts.string / str_to_rate +} + local function parse_limit(name, data) - local buckets = {} if type(data) == 'table' then - -- 3 cases here: + -- 2 cases here: -- * old limit in format [burst, rate] - -- * vector of strings in Andrew's string format + -- * vector of strings in Andrew's string format (removed from 1.8.2) -- * proper bucket table if #data == 2 and tonumber(data[1]) and tonumber(data[2]) then -- Old style ratelimit rspamd_logger.warnx(rspamd_config, 'old style ratelimit for %s', name) if tonumber(data[1]) > 0 and tonumber(data[2]) > 0 then - table.insert(buckets, { + return { burst = data[1], rate = data[2] - }) + } elseif data[1] ~= 0 then rspamd_logger.warnx(rspamd_config, 'invalid numbers for %s', name) else rspamd_logger.infox(rspamd_config, 'disable limit %s, burst is zero', name) end + + return nil else - -- Recursively map parse_limit and flatten the list - fun.each(function(l) - -- Flatten list - for _,b in ipairs(l) do table.insert(buckets, b) end - end, fun.map(function(d) return parse_limit(d, name) end, data)) + local parsed_bucket,err = bucket_schema:transform(data) + + if not parsed_bucket or err then + rspamd_logger.errx(rspamd_config, 'cannot parse bucket for %s: %s; original value: %s', + name, err, data) + else + return parsed_bucket + end end elseif type(data) == 'string' then local rep_rate, burst = parse_string_limit(data) - + rspamd_logger.warnx(rspamd_config, 'old style rate bucket config detected for %s: %s', + name, data) if rep_rate and burst then - table.insert(buckets, { + return { burst = burst, - rate = 1.0 / rep_rate -- reciprocal - }) + rate = burst / rep_rate -- reciprocal + } end end - -- Filter valid - return fun.totable(fun.filter(function(val) - return type(val.burst) == 'number' and type(val.rate) == 'number' - end, buckets)) + return nil end --- Check whether this addr is bounce @@ -461,8 +519,6 @@ local function limit_to_prefixes(task, k, v, prefixes) end end end - - end return n @@ -515,8 +571,8 @@ local function ratelimit_cb(task) if ret then local bucket = parse_limit(k, bd) - if bucket[1] then - prefixes[redis_key] = make_prefix(redis_key, k, bucket[1]) + if bucket then + prefixes[redis_key] = make_prefix(redis_key, k, bucket) end nprefixes = nprefixes + 1 else @@ -525,7 +581,7 @@ local function ratelimit_cb(task) end end - local function gen_check_cb(prefix, bucket, lim_name) + local function gen_check_cb(prefix, bucket, lim_name, lim_key) return function(err, data) if err then rspamd_logger.errx('cannot check limit %s: %s %s', prefix, err, data) @@ -538,25 +594,26 @@ local function ratelimit_cb(task) if data[1] == 1 then -- set symbol only and do NOT soft reject if settings.symbol then - task:insert_result(settings.symbol, 0.0, lim_name .. "(" .. prefix .. ")") + task:insert_result(settings.symbol, 0.0, + string.format('%s(%s)', lim_name, lim_key)) rspamd_logger.infox(task, - 'set_symbol_only: ratelimit "%s(%s)" exceeded, (%s / %s): %s (%s:%s dyn)', + 'set_symbol_only: ratelimit "%s(%s)" exceeded, (%s / %s): %s (%s:%s dyn); redis key: %s', lim_name, prefix, bucket.burst, bucket.rate, - data[2], data[3], data[4]) + data[2], data[3], data[4], lim_key) return -- set INFO symbol and soft reject elseif settings.info_symbol then task:insert_result(settings.info_symbol, 1.0, - lim_name .. "(" .. prefix .. ")") + string.format('%s(%s)', lim_name, lim_key)) end rspamd_logger.infox(task, - 'ratelimit "%s(%s)" exceeded, (%s / %s): %s (%s:%s dyn)', + 'ratelimit "%s(%s)" exceeded, (%s / %s): %s (%s:%s dyn); redis key: %s', lim_name, prefix, bucket.burst, bucket.rate, - data[2], data[3], data[4]) + data[2], data[3], data[4], lim_key) task:set_pre_result('soft reject', - message_func(task, lim_name, prefix, bucket), N) + message_func(task, lim_name, prefix, bucket, lim_key), N) end end end @@ -579,7 +636,7 @@ local function ratelimit_cb(task) value.name, pr, value.hash, bucket.burst, bucket.rate) lua_redis.exec_redis_script(bucket_check_id, {key = value.hash, task = task, is_write = true}, - gen_check_cb(pr, bucket, value.name), + gen_check_cb(pr, bucket, value.name, value.hash), {value.hash, tostring(now), tostring(rate), tostring(bucket.burst), tostring(settings.expire)}) end @@ -587,6 +644,8 @@ local function ratelimit_cb(task) end local function ratelimit_update_cb(task) + if task:has_flag('skip') then return end + if not settings.allow_local and lua_util.is_rspamc_or_controller(task) then return end local prefixes = task:cache_get('ratelimit_prefixes') if prefixes then @@ -596,7 +655,7 @@ local function ratelimit_update_cb(task) return end - local is_spam = not (task:get_metric_action() == 'no action') + local verdict = lua_util.get_task_verdict(task) -- Update each bucket for k, v in pairs(prefixes) do @@ -615,12 +674,15 @@ local function ratelimit_update_cb(task) end local now = rspamd_util.get_time() now = lua_util.round(now * 1000.0) -- Get milliseconds - local mult_burst = bucket.ham_factor_burst or 1.0 - local mult_rate = bucket.ham_factor_burst or 1.0 + local mult_burst = 1.0 + local mult_rate = 1.0 - if is_spam then + if verdict == 'spam' or verdict == 'junk' then mult_burst = bucket.spam_factor_burst or 1.0 mult_rate = bucket.spam_factor_rate or 1.0 + elseif verdict == 'ham' then + mult_burst = bucket.ham_factor_burst or 1.0 + mult_rate = bucket.ham_factor_rate or 1.0 end lua_redis.exec_redis_script(bucket_update_id, @@ -645,42 +707,72 @@ if opts then if opts['rates'] and type(opts['rates']) == 'table' then -- new way of setting limits fun.each(function(t, lim) - local buckets - if type(lim) == 'table' and lim.selector and lim.bucket then - local selector = lua_selectors.parse_selector(rspamd_config, lim.selector) - if not selector then - rspamd_logger.errx(rspamd_config, 'bad ratelimit selector for %s: "%s"', - t, lim.selector) - return - end - local bucket = parse_limit(t, lim.bucket) + local buckets = {} + + if type(lim) == 'table' and lim.bucket then - if not bucket then - rspamd_logger.errx(rspamd_config, 'bad ratelimit bucket for %s: "%s"', - t, lim.bucket) - return + if lim.bucket[1] then + for _,bucket in ipairs(lim.bucket) do + local b = parse_limit(t, bucket) + + if not b then + rspamd_logger.errx(rspamd_config, 'bad ratelimit bucket for %s: "%s"', + t, b) + return + end + + table.insert(buckets, b) + end + else + local bucket = parse_limit(t, lim.bucket) + + if not bucket then + rspamd_logger.errx(rspamd_config, 'bad ratelimit bucket for %s: "%s"', + t, lim.bucket) + return + end + + buckets = {bucket} end + settings.limits[t] = { - selector = selector, - buckets = bucket + buckets = buckets } + if lim.selector then + local selector = lua_selectors.parse_selector(rspamd_config, lim.selector) + if not selector then + rspamd_logger.errx(rspamd_config, 'bad ratelimit selector for %s: "%s"', + t, lim.selector) + settings.limits[t] = nil + return + end + + settings.limits[t].selector = selector + end else + rspamd_logger.warnx(rspamd_config, 'old syntax for ratelimits: %s', lim) buckets = parse_limit(t, lim) - if buckets and #buckets > 0 then + if buckets then settings.limits[t] = { - buckets = buckets + buckets = {buckets} } end end end, opts['rates']) end - local enabled_limits = fun.totable(fun.map(function(t) - return t + -- Display what's enabled + fun.each(function(s) + rspamd_logger.infox(rspamd_config, 'enabled ratelimit: %s', s) + end, fun.map(function(n,d) + return string.format('%s [%s]', n, + table.concat(fun.totable(fun.map(function(v) + return string.format('%s msgs burst, %s msgs/sec rate', + v.burst, v.rate) + end, d.buckets)), '; ') + ) end, settings.limits)) - rspamd_logger.infox(rspamd_config, - 'enabled rate buckets: [%1]', table.concat(enabled_limits, ',')) -- Ret, ret, ret: stupid legacy stuff: -- If we have a string with commas then load it as as static map diff --git a/src/plugins/lua/rbl.lua b/src/plugins/lua/rbl.lua index 53f537ac0..3bbf46ff3 100644 --- a/src/plugins/lua/rbl.lua +++ b/src/plugins/lua/rbl.lua @@ -24,6 +24,7 @@ local rspamd_logger = require 'rspamd_logger' local rspamd_util = require 'rspamd_util' local fun = require 'fun' local lua_util = require 'lua_util' +local ts = require("tableshape").types -- This plugin implements various types of RBL checks -- Documentation can be found here: @@ -32,49 +33,49 @@ local lua_util = require 'lua_util' local E = {} local N = 'rbl' -local rbls = {} -local local_exclusions = nil +local local_exclusions local default_monitored = '1.0.0.127' -local symbols = { - dkim_allow_symbol = 'R_DKIM_ALLOW', -} - -local dkim_config = rspamd_config:get_all_opt("dkim") -if (dkim_config or E).symbol_allow then - symbols['dkim_allow_symbol'] = dkim_config['symbol_allow'] -end - local function validate_dns(lstr) if lstr:match('%.%.') then + -- two dots in a row return false end for v in lstr:gmatch('[^%.]+') do if not v:match('^[%w-]+$') or v:len() > 63 or v:match('^-') or v:match('-$') then + -- too long label or weird labels return false end end return true end -local hash_alg = { - sha1 = true, - md5 = true, - sha256 = true, - sha384 = true, - sha512 = true, -} +local function maybe_make_hash(data, rule) + if rule.hash then + local h = hash.create_specific(rule.hash, data) + local s + if rule.hash_format then + if rule.hash_format == 'base32' then + s = h:base32() + elseif rule.hash_format == 'base64' then + s = h:base64() + else + s = h:hex() + end + else + s = h:hex() + end + + if rule.hash_len then + s = s:sub(1, rule.hash_len) + end -local function make_hash(data, specific) - local h - if not hash_alg[specific] then - h = hash.create(data) + return s else - h = hash.create_specific(specific, data) + return data end - return h:hex() end local function is_excluded_ip(rip) @@ -84,8 +85,8 @@ local function is_excluded_ip(rip) return false end -local function ip_to_rbl(ip, rbl) - return table.concat(ip:inversed_str_octets(), '.') .. '.' .. rbl +local function ip_to_rbl(ip) + return table.concat(ip:inversed_str_octets(), '.') end local function gen_check_rcvd_conditions(rbl, received_total) @@ -155,328 +156,305 @@ local function gen_check_rcvd_conditions(rbl, received_total) end end -local function rbl_cb (task) - local function gen_rbl_callback(rule) - return function (_, to_resolve, results, err) - if err and (err ~= 'requested record is not found' and err ~= 'no records with this name') then - rspamd_logger.errx(task, 'error looking up %s: %s', to_resolve, err) +local function rbl_dns_process(task, rbl, to_resolve, results, err) + if err and (err ~= 'requested record is not found' and + err ~= 'no records with this name') then + rspamd_logger.errx(task, 'error looking up %s: %s', to_resolve, err) + end + if not results then + lua_util.debugm(N, task, + 'DNS RESPONSE: label=%1 results=%2 error=%3 rbl=%4', + to_resolve, false, err, rbl.symbol) + return + else + lua_util.debugm(N, task, + 'DNS RESPONSE: label=%1 results=%2 error=%3 rbl=%4', + to_resolve, true, err, rbl.symbol) + end + + if rbl.returncodes == nil and rbl.symbol ~= nil then + task:insert_result(rbl.symbol, 1, to_resolve) + return + end + + for _,result in ipairs(results) do + local ipstr = result:to_string() + lua_util.debugm(N, task, '%s DNS result %s', to_resolve, ipstr) + local foundrc = false + -- Check return codes + for s,i in pairs(rbl.returncodes) do + for _,v in ipairs(i) do + if string.find(ipstr, '^' .. v .. '$') then + foundrc = true + task:insert_result(s, 1, to_resolve .. ' : ' .. ipstr) + break + end end - if not results then - lua_util.debugm(N, task, 'DNS RESPONSE: label=%1 results=%2 error=%3 rbl=%4', to_resolve, false, err, rule['rbls'][1]['symbol']) - return + end + if not foundrc then + if rbl.unknown and rbl.symbol then + task:insert_result(rbl.symbol, 1, to_resolve) else - lua_util.debugm(N, task, 'DNS RESPONSE: label=%1 results=%2 error=%3 rbl=%4', to_resolve, true, err, rule['rbls'][1]['symbol']) + rspamd_logger.errx(task, 'RBL %1 returned unknown result: %2', + rbl.rbl, ipstr) end + end + end +end - for _,rbl in ipairs(rule.rbls) do - if rbl['returncodes'] == nil and rbl['symbol'] ~= nil then - task:insert_result(rbl['symbol'], 1, to_resolve) - return - end - for _,result in pairs(results) do - local ipstr = result:to_string() - local foundrc - lua_util.debugm(N, task, '%s DNS result %s', to_resolve, ipstr) - for s,i in pairs(rbl['returncodes']) do - if type(i) == 'string' then - if string.find(ipstr, '^' .. i .. '$') then - foundrc = i - task:insert_result(s, 1, to_resolve .. ' : ' .. ipstr) - break - end - elseif type(i) == 'table' then - for _,v in pairs(i) do - if string.find(ipstr, '^' .. v .. '$') then - foundrc = v - task:insert_result(s, 1, to_resolve .. ' : ' .. ipstr) - break - end - end - end - end - if not foundrc then - if rbl['unknown'] and rbl['symbol'] then - task:insert_result(rbl['symbol'], 1, to_resolve) - else - rspamd_logger.errx(task, 'RBL %1 returned unknown result: %2', - rbl['rbl'], ipstr) - end - end - end +local function gen_rbl_callback(rule) + -- Here, we have functional approach: we form a pipeline of functions + -- f1, f2, ... fn. Each function accepts task and return boolean value + -- that allows to process pipeline further + -- Each function in the pipeline can add something to `dns_req` vector as a side effect + + local function add_dns_request(req, forced, requests_table) + if requests_table[req] then + -- Duplicate request + if forced and not requests_table[req].forced then + requests_table[req].forced = true end + else + local nreq = { + forced = forced, + n = string.format('%s.%s', + maybe_make_hash(req, rule), + rule.rbl) + } + requests_table[req] = nreq end end - local params = {} -- indexed by rbl name + local function is_alive(_, _) + if rule.monitored then + if not rule.monitored:alive() then + return false + end + end - local function gen_rbl_rule(to_resolve, rbl) - lua_util.debugm(N, task, 'DNS REQUEST: label=%1 rbl=%2', to_resolve, rbl['symbol']) - if not params[to_resolve] then - local nrule = { - to_resolve = to_resolve, - rbls = {rbl}, - forced = true, - } - nrule.callback = gen_rbl_callback(nrule) - params[to_resolve] = nrule - else - table.insert(params[to_resolve].rbls, rbl) + return true + end + + local function check_user(task, _) + if task:get_user() then + return false end - return params[to_resolve] + return true end - local havegot = { - emails = {}, - received = {} - } - local notgot = {} + local function check_local(task, _) + local ip = task:get_from_ip() - local alive_rbls = fun.filter(function(_, rbl) - if rbl.monitored then - if not rbl.monitored:alive() then - return false - end + if not ip:is_valid() then + ip = nil + end + + if ip and ip:is_local() or is_excluded_ip(ip) then + return false end return true - end, rbls) - - -- Now exclude rbls, that are disabled by configuration - local enabled_rbls = fun.filter(function(_, rbl) - if rbl['exclude_users'] then - if not havegot['user'] and not notgot['user'] then - havegot['user'] = task:get_user() - if havegot['user'] == nil then - notgot['user'] = true - end - end - if havegot['user'] ~= nil then - return false - end + end + + local function check_helo(task, requests_table) + local helo = task:get_helo() + + if not helo then + return false end - if (rbl['exclude_local'] or rbl['exclude_private_ips']) and not notgot['from'] then - if not havegot['from'] then - havegot['from'] = task:get_from_ip() - if not havegot['from']:is_valid() then - notgot['from'] = true + add_dns_request(helo, true, requests_table) + end + + local function check_dkim(task, requests_table) + local das = task:get_symbol('DKIM_TRACE') + local mime_from_domain + local ret = false + + if das and das[1] and das[1].options then + + if rule.dkim_match_from then + -- We check merely mime from + mime_from_domain = ((task:get_from('mime') or E)[1] or E).domain + if mime_from_domain then + mime_from_domain = rspamd_util.get_tld(mime_from_domain) end end - if havegot['from'] and not notgot['from'] and ((rbl['exclude_local'] and - is_excluded_ip(havegot['from'])) or (rbl['exclude_private_ips'] and - havegot['from']:is_local())) then - return false - end - end - -- Helo checks - if rbl['helo'] then - if notgot['helo'] then - return false - end - if not havegot['helo'] then - if rbl['hash'] then - havegot['helo'] = task:get_helo() - if havegot['helo'] then - havegot['helo'] = make_hash(havegot['helo'], rbl['hash']) + for _, d in ipairs(das[1].options) do + + local domain,result = d:match('^([^%:]*):([%+%-%~])$') + + -- We must ignore bad signatures, omg + if domain and result and result == '+' then + if rule.dkim_match_from then + -- We check merely mime from + local domain_tld = domain + if not rule.dkim_domainonly then + -- Adjust + domain_tld = rspamd_util.get_tld(domain) + end + + if mime_from_domain and mime_from_domain == domain_tld then + add_dns_request(domain_tld, true, requests_table) + ret = true + end else - notgot['helo'] = true - return false - end - else - havegot['helo'] = task:get_helo() - if havegot['helo'] == nil or not validate_dns(havegot['helo']) then - havegot['helo'] = nil - notgot['helo'] = true - return false + if rule.dkim_domainonly then + add_dns_request(rspamd_util.get_tld(domain), false, requests_table) + ret = true + else + add_dns_request(domain, false, requests_table) + ret = true + end end end end - elseif rbl['dkim'] then - -- DKIM checks - if notgot['dkim'] then - return false - end - if not havegot['dkim'] then - local das = task:get_symbol(symbols['dkim_allow_symbol']) - if ((das or E)[1] or E).options then - havegot['dkim'] = das[1]['options'] + end + + return ret + end + + local function check_emails(task, requests_table) + local emails = task:get_emails() + + if not emails then + return false + end + + for _,email in ipairs(emails) do + if rule.emails_domainonly then + add_dns_request(email:get_tld(), false, requests_table) + else + if rule.hash then + -- Leave @ as is + add_dns_request(string.format('%s@%s', + email:get_user(), email:get_domain()), false, requests_table) else - notgot['dkim'] = true - return false - end - end - elseif rbl['emails'] then - -- Emails checks - if notgot['emails'] then - return false - end - if #havegot['emails'] == 0 then - havegot['emails'] = task:get_emails() - if havegot['emails'] == nil then - notgot['emails'] = true - havegot['emails'] = {} - return false - end - end - elseif rbl['from'] then - if notgot['from'] then - return false - end - if not havegot['from'] then - havegot['from'] = task:get_from_ip() - if not havegot['from']:is_valid() then - notgot['from'] = true - return false - end - end - elseif rbl['received'] then - if notgot['received'] then - return false - end - if #havegot['received'] == 0 then - havegot['received'] = task:get_received_headers() - if next(havegot['received']) == nil then - notgot['received'] = true - havegot['received'] = {} - return false - end - end - elseif rbl['rdns'] then - if notgot['rdns'] then - return false - end - if not havegot['rdns'] then - havegot['rdns'] = task:get_hostname() - if havegot['rdns'] == nil or havegot['rdns'] == 'unknown' then - notgot['rdns'] = true - return false + -- Replace @ with . + add_dns_request(string.format('%s.%s', + email:get_user(), email:get_domain()), false, requests_table) end end end return true - end, alive_rbls) - - -- Now we iterate over enabled rbls and fill params - -- Helo RBLs - fun.each(function(_, rbl) - local to_resolve = havegot['helo'] .. '.' .. rbl['rbl'] - gen_rbl_rule(to_resolve, rbl) - end, - fun.filter(function(_, rbl) - if rbl['helo'] then return true end - return false - end, enabled_rbls)) + end - -- DKIM RBLs - fun.each(function(_, rbl) - for _, d in ipairs(havegot['dkim']) do - if rbl['dkim_domainonly'] then - d = rspamd_util.get_tld(d) - end - local to_resolve = d .. '.' .. rbl['rbl'] - gen_rbl_rule(to_resolve, rbl) + local function check_from(task, requests_table) + local ip = task:get_from_ip() + + if not ip or not ip:is_valid() then + return true end - end, - fun.filter(function(_, rbl) - if rbl['dkim'] then return true end - return false - end, enabled_rbls)) - - -- Emails RBLs - fun.each(function(_, rbl) - if rbl['emails'] == 'domain_only' then - local cleanList = {} - for _, email in ipairs(havegot['emails']) do - cleanList[email:get_host()] = true - end - for k in pairs(cleanList) do - local to_resolve - if rbl['hash'] then - to_resolve = make_hash(tostring(k), rbl['hash']) .. '.' .. rbl['rbl'] - else - to_resolve = k .. '.' .. rbl['rbl'] - end - gen_rbl_rule(to_resolve, rbl) - end - else - for _, email in ipairs(havegot['emails']) do - local to_resolve - if rbl['hash'] then - to_resolve = make_hash(email:get_user() .. '@' .. email:get_host(), rbl['hash']) .. '.' .. rbl['rbl'] - else - local upart = email:get_user() - if validate_dns(upart) then - to_resolve = upart .. '.' .. email:get_host() .. '.' .. rbl['rbl'] - end - end - if to_resolve then - gen_rbl_rule(to_resolve, rbl) - end - end + if (ip:get_version() == 6 and rule.ipv6) or + (ip:get_version() == 4 and rule.ipv4) then + add_dns_request(ip_to_rbl(ip), true, requests_table) end - end, - fun.filter(function(_, rbl) - if rbl['emails'] then return true end - return false - end, enabled_rbls)) - - -- RDNS lists - fun.each(function(_, rbl) - local to_resolve = havegot['rdns'] .. '.' .. rbl['rbl'] - gen_rbl_rule(to_resolve, rbl) - end, - fun.filter(function(_, rbl) - if rbl['rdns'] then return true end - return false - end, enabled_rbls)) - - -- From lists - fun.each(function(_, rbl) - if (havegot['from']:get_version() == 6 and rbl['ipv6']) or - (havegot['from']:get_version() == 4 and rbl['ipv4']) then - local to_resolve = ip_to_rbl(havegot['from'], rbl['rbl']) - gen_rbl_rule(to_resolve, rbl) - end - end, - fun.filter(function(_, rbl) - if rbl['from'] then return true end - return false - end, enabled_rbls)) - havegot['received'] = fun.filter(function(h) - return not h['flags']['artificial'] - end, havegot['received']):totable() + return true + end + + local function check_received(task, requests_table) + local received = fun.filter(function(h) + return not h['flags']['artificial'] + end, task:get_received_headers()):totable() - local received_total = #havegot['received'] - -- Received lists - fun.each(function(_, rbl) - local check_conditions = gen_check_rcvd_conditions(rbl, received_total) - for pos,rh in ipairs(havegot['received']) do + local received_total = #received + local check_conditions = gen_check_rcvd_conditions(rule, received_total) + + for pos,rh in ipairs(received) do if check_conditions(rh, pos) then - local to_resolve = ip_to_rbl(rh['real_ip'], rbl['rbl']) - local rule = gen_rbl_rule(to_resolve, rbl) - -- Disable forced for received resolving, as we have no control on - -- those headers count - rule.forced = false + add_dns_request(ip_to_rbl(rh.real_ip), false, requests_table) end end - end, - fun.filter(function(_, rbl) - if rbl['received'] then return true end - return false - end, enabled_rbls)) - local r = task:get_resolver() - for _,p in pairs(params) do - r:resolve_a({ - task = task, - name = p.to_resolve, - callback = p.callback, - forced = p.forced - }) + return true + end + + local function check_rdns(task, requests_table) + local hostname = task:get_hostname() + if hostname == nil or hostname == 'unknown' then + return false + end + + add_dns_request(hostname, true, requests_table) + + return true + end + + -- Create function pipeline depending on rbl settings + local pipeline = { + is_alive, -- generic for all + } + + if rule.exclude_users then + pipeline[#pipeline + 1] = check_user + end + + if rule.exclude_local or rule.exclude_private_ips then + pipeline[#pipeline + 1] = check_local + end + + if rule.helo then + pipeline[#pipeline + 1] = check_helo + end + + if rule.dkim then + pipeline[#pipeline + 1] = check_dkim + end + + if rule.emails then + pipeline[#pipeline + 1] = check_emails + end + + if rule.from then + pipeline[#pipeline + 1] = check_from + end + + if rule.received then + pipeline[#pipeline + 1] = check_received + end + + if rule.rdns then + pipeline[#pipeline + 1] = check_rdns + end + + return function(task) + -- DNS requests to issue (might be hashed afterwards) + local dns_req = {} + + local function rbl_dns_callback(_, to_resolve, results, err) + rbl_dns_process(task, rule, to_resolve, results, err) + end + + -- Execute functions pipeline + for _,f in ipairs(pipeline) do + if not f(task, dns_req) then + lua_util.debugm(N, task, "skip rbl check: %s; pipeline condition returned false", + rule.symbol) + return + end + end + + -- Now check all DNS requests pending and emit them + local r = task:get_resolver() + for name,p in pairs(dns_req) do + if validate_dns(p.n) then + lua_util.debugm(N, task, "rbl %s; resolve %s -> %s", + rule.symbol, name, p.n) + r:resolve_a({ + task = task, + name = p.n, + callback = rbl_dns_callback, + forced = p.forced + }) + else + rspamd_logger.warnx(task, 'cannot send invalid DNS request %s for %s', + p.n, rule.symbol) + end + end end end @@ -491,26 +469,28 @@ end -- Plugin defaults should not be changed - override these in config -- New defaults should not alter behaviour local default_defaults = { - ['default_enabled'] = {[1] = true, [2] = 'enabled'}, - ['default_ipv4'] = {[1] = true, [2] = 'ipv4'}, - ['default_ipv6'] = {[1] = false, [2] = 'ipv6'}, - ['default_received'] = {[1] = true, [2] = 'received'}, - ['default_from'] = {[1] = false, [2] = 'from'}, - ['default_unknown'] = {[1] = false, [2] = 'unknown'}, - ['default_rdns'] = {[1] = false, [2] = 'rdns'}, - ['default_helo'] = {[1] = false, [2] = 'helo'}, - ['default_dkim'] = {[1] = false, [2] = 'dkim'}, - ['default_dkim_domainonly'] = {[1] = true, [2] = 'dkim_domainonly'}, - ['default_emails'] = {[1] = false, [2] = 'emails'}, - ['default_exclude_private_ips'] = {[1] = true, [2] = 'exclude_private_ips'}, - ['default_exclude_users'] = {[1] = false, [2] = 'exclude_users'}, - ['default_exclude_local'] = {[1] = true, [2] = 'exclude_local'}, - ['default_is_whitelist'] = {[1] = false, [2] = 'is_whitelist'}, - ['default_ignore_whitelist'] = {[1] = false, [2] = 'ignore_whitelists'}, + ['default_enabled'] = true, + ['default_ipv4'] = true, + ['default_ipv6'] = true, + ['default_received'] = false, + ['default_from'] = true, + ['default_unknown'] = false, + ['default_rdns'] = false, + ['default_helo'] = false, + ['default_dkim'] = false, + ['default_dkim_domainonly'] = true, + ['default_emails'] = false, + ['default_emails_domainonly'] = false, + ['default_exclude_private_ips'] = true, + ['default_exclude_users'] = false, + ['default_exclude_local'] = true, + ['default_is_whitelist'] = false, + ['default_ignore_whitelist'] = false, } +-- Enrich with defaults for default, default_v in pairs(default_defaults) do if opts[default] == nil then - opts[default] = default_v[1] + opts[default] = default_v end end @@ -521,136 +501,175 @@ end local white_symbols = {} local black_symbols = {} -local need_dkim = false -local id = rspamd_config:register_symbol({ - type = 'callback', - callback = rbl_cb, - name = 'RBL_CALLBACK', - flags = 'empty,nice' +local rule_schema = ts.shape({ + enabled = ts.boolean:is_optional(), + disabled = ts.boolean:is_optional(), + rbl = ts.string, + symbol = ts.string:is_optional(), + returncodes = ts.map_of( + ts.string / string.upper, + ( + ts.array_of(ts.string) + (ts.string / function(s) + return { s } + end) + ) + ):is_optional(), + whitelist_exception = ( + ts.array_of(ts.string) + (ts.string / function(s) return {s} end) + ):is_optional(), + local_exclude_ip_map = ts.string:is_optional(), + hash = ts.one_of{"sha1", "sha256", "sha384", "sha512", "md5", "blake2"}:is_optional(), + hash_format = ts.one_of{"hex", "base32", "base64"}:is_optional(), + hash_len = (ts.integer + ts.string / tonumber):is_optional(), +}, { + extra_fields = ts.map_of(ts.string, ts.boolean) }) -local is_monitored = {} -local rbls_count = 0 -for key,rbl in pairs(opts['rbls']) do - (function() - if type(rbl) ~= 'table' or rbl['disabled'] then - rspamd_logger.infox(rspamd_config, 'disable rbl "%s"', key) - return - end - for default, default_v in pairs(default_defaults) do - if(rbl[default_v[2]] == nil) then - rbl[default_v[2]] = opts[default] - end - end - if not rbl['enabled'] then return end - if type(rbl['returncodes']) == 'table' then - for s,_ in pairs(rbl['returncodes']) do - if type(rspamd_config.get_api_version) ~= 'nil' then - rspamd_config:register_symbol({ - name = s, - parent = id, - type = 'virtual' - }) +local monitored_addresses = {} - if rbl['dkim'] then - need_dkim = true - end - if(rbl['is_whitelist']) then - if type(rbl['whitelist_exception']) == 'string' then - if (rbl['whitelist_exception'] ~= s) then - table.insert(white_symbols, s) - end - elseif type(rbl['whitelist_exception']) == 'table' then - local foundException = false - for _, e in pairs(rbl['whitelist_exception']) do - if e == s then - foundException = true - break - end - end - if not foundException then - table.insert(white_symbols, s) - end - else - table.insert(white_symbols, s) - end - else - if rbl['ignore_whitelists'] == false then - table.insert(black_symbols, s) - end - end - end - end - end - if not rbl['symbol'] and - ((rbl['returncodes'] and rbl['unknown']) or - (not rbl['returncodes'])) then - rbl['symbol'] = key - end - if rbl['symbol'] then +local function add_rbl(key, rbl) + if not rbl.symbol then + rbl.symbol = key:upper() + end + + local flags_tbl = {'no_squeeze'} + if rbl.is_whitelist then + flags_tbl[#flags_tbl + 1] = 'nice' + end + + if not (rbl.dkim or rbl.emails) then + flags_tbl[#flags_tbl + 1] = 'empty' + end + + local id = rspamd_config:register_symbol{ + type = 'callback', + callback = gen_rbl_callback(rbl), + name = rbl.symbol, + flags = table.concat(flags_tbl, ',') + } + + if rbl.dkim then + rspamd_config:register_dependency(rbl.symbol, 'DKIM_CHECK') + end + + if rbl.returncodes then + for s,_ in pairs(rbl['returncodes']) do rspamd_config:register_symbol({ - name = rbl['symbol'], + name = s, parent = id, type = 'virtual' }) - rbls_count = rbls_count + 1 - if rbl['dkim'] then - need_dkim = true - end - if (rbl['is_whitelist']) then - if type(rbl['whitelist_exception']) == 'string' then - if (rbl['whitelist_exception'] ~= rbl['symbol']) then - table.insert(white_symbols, rbl['symbol']) - end - elseif type(rbl['whitelist_exception']) == 'table' then - local foundException = false - for _, e in pairs(rbl['whitelist_exception']) do - if e == rbl['symbol'] then - foundException = true - break - end - end - if not foundException then - table.insert(white_symbols, rbl['symbol']) - end - else - table.insert(white_symbols, rbl['symbol']) + if rbl.is_whitelist then + if rbl.whitelist_exception then + local foundException = false + for _, e in ipairs(rbl.whitelist_exception) do + if e == s then + foundException = true + break end + end + if not foundException then + table.insert(white_symbols, s) + end + else + table.insert(white_symbols, s) + end else - if rbl['ignore_whitelists'] == false then - table.insert(black_symbols, rbl['symbol']) + if rbl.ignore_whitelist == false then + table.insert(black_symbols, s) end end end - if rbl['rbl'] then - if not rbl['disable_monitoring'] and not rbl['is_whitelist'] and not is_monitored[rbl['rbl']] then - is_monitored[rbl['rbl']] = true - rbl.monitored = rspamd_config:register_monitored(rbl['rbl'], 'dns', + end + + if not rbl.is_whitelist and rbl.ignore_whitelist == false then + table.insert(black_symbols, rbl.symbol) + end + -- Process monitored + if not rbl.disable_monitoring and not rbl.is_whitelist then + if not monitored_addresses[rbl.rbl] then + monitored_addresses[rbl.rbl] = true + rbl.monitored = rspamd_config:register_monitored(rbl['rbl'], 'dns', { rcode = 'nxdomain', - prefix = rbl['monitored_address'] or default_monitored + prefix = rbl.monitored_address or default_monitored }) + end + end +end + +for key,rbl in pairs(opts.rbls or opts.rules) do + if type(rbl) ~= 'table' or rbl.disabled == true or rbl.enabled == false then + rspamd_logger.infox(rspamd_config, 'disable rbl "%s"', key) + else + for default,_ in pairs(default_defaults) do + local rbl_opt = default:sub(#('default_') + 1) + if rbl[rbl_opt] == nil then + rbl[rbl_opt] = opts[default] end + end - rbls[key] = rbl + local res,err = rule_schema:transform(rbl) + if not res then + rspamd_logger.errx(rspamd_config, 'invalid config for %s: %s, RBL is DISABLED', + key, err) + else + add_rbl(key, res) end - end)() + end -- rbl.enabled end -if rbls_count == 0 then - lua_util.disable_module(N, "config") -end +-- We now create two symbols: +-- * RBL_CALLBACK_WHITE that depends on all symbols white +-- * RBL_CALLBACK that depends on all symbols black to participate in depends chains + +local function rbl_callback_white(task) + local found_whitelist = false + for _, w in ipairs(white_symbols) do + if task:has_symbol(w) then + lua_util.debugm(N, task,'found whitelist %s', w) + found_whitelist = true + break + end + end -for _, w in pairs(white_symbols) do - for _, b in pairs(black_symbols) do - local csymbol = 'RBL_COMPOSITE_' .. w .. '_' .. b - rspamd_config:set_metric_symbol(csymbol, 0, 'Autogenerated composite') - rspamd_config:add_composite(csymbol, w .. ' & ' .. b) + if found_whitelist then + -- Disable all symbols black + for _, b in ipairs(black_symbols) do + lua_util.debugm(N, task,'disable %s, whitelist found', b) + task:disable_symbol(b) + end end + lua_util.debugm(N, task, "finished rbl whitelists processing") +end + +local function rbl_callback_fin(task) + -- Do nothing + lua_util.debugm(N, task, "finished rbl processing") +end + +rspamd_config:register_symbol{ + type = 'callback', + callback = rbl_callback_white, + name = 'RBL_CALLBACK_WHITE', + flags = 'nice,empty,no_squeeze' +} + +rspamd_config:register_symbol{ + type = 'callback', + callback = rbl_callback_fin, + name = 'RBL_CALLBACK', + flags = 'empty,no_squeeze' +} + +for _, w in ipairs(white_symbols) do + rspamd_config:register_dependency('RBL_CALLBACK_WHITE', w) end -if need_dkim then - rspamd_config:register_dependency('RBL_CALLBACK', symbols['dkim_allow_symbol']) + +for _, b in ipairs(black_symbols) do + rspamd_config:register_dependency(b, 'RBL_CALLBACK_WHITE') + rspamd_config:register_dependency('RBL_CALLBACK', b) end diff --git a/src/plugins/lua/replies.lua b/src/plugins/lua/replies.lua index fe15211ef..234a41ca3 100644 --- a/src/plugins/lua/replies.lua +++ b/src/plugins/lua/replies.lua @@ -37,6 +37,10 @@ local settings = { score = -4, -- Default score use_auth = true, use_local = true, + cookie = nil, + cookie_key = nil, + cookie_is_pattern = false, + cookie_valid_time = '2w', -- 2 weeks by default } local N = "replies" @@ -128,16 +132,123 @@ local function replies_set(task) end end +local function replies_check_cookie(task) + local function cookie_matched(extra, ts) + local dt = task:get_date{format = 'connect', gmt = true} + + if dt < ts then + rspamd_logger.infox(task, 'ignore cookie as its date is in future') + + return + end + + if settings.cookie_valid_time then + if dt - ts > settings.cookie_valid_time then + rspamd_logger.infox(task, + 'ignore cookie as its timestamp is too old: %s (%s current time)', + ts, dt) + + return + end + end + + if extra then + task:insert_result(settings['symbol'], 1.0, + string.format('cookie:%s:%s', extra, ts)) + else + task:insert_result(settings['symbol'], 1.0, + string.format('cookie:%s', ts)) + end + if settings['action'] ~= nil then + local ip_addr = task:get_ip() + if (settings.use_auth and + task:get_user()) or + (settings.use_local and ip_addr and ip_addr:is_local()) then + rspamd_logger.infox(task, "not forcing action for local network or authorized user"); + else + task:set_pre_result(settings['action'], settings['message'], N) + end + end + end + + -- If in-reply-to header not present return + local irt = task:get_header('in-reply-to') + if irt == nil then + return + end + + local cr = require "rspamd_cryptobox" + -- Extract user part if needed + local extracted_cookie = irt:match('^%<?([^@]+)@.*$') + if not extracted_cookie then + -- Assume full message id as a cookie + extracted_cookie = irt + end + + local dec_cookie,ts = cr.decrypt_cookie(settings.cookie_key, extracted_cookie) + + if dec_cookie then + -- We have something that looks like a cookie + if settings.cookie_is_pattern then + local m = dec_cookie:match(settings.cookie) + + if m then + cookie_matched(m, ts) + end + else + -- Direct match + if dec_cookie == settings.cookie then + cookie_matched(nil, ts) + end + end + end +end + local opts = rspamd_config:get_all_opt('replies') if not (opts and type(opts) == 'table') then rspamd_logger.infox(rspamd_config, 'module is unconfigured') return end if opts then + settings = lua_util.override_defaults(settings, opts) redis_params = lua_redis.parse_redis_server('replies') if not redis_params then - rspamd_logger.infox(rspamd_config, 'no servers are specified, disabling module') - lua_util.disable_module(N, "redis") + if not (settings.cookie and settings.cookie_key) then + rspamd_logger.infox(rspamd_config, 'no servers are specified, disabling module') + lua_util.disable_module(N, "redis") + else + -- Cookies mode + -- Check key sanity: + local pattern = {'^'} + for i=1,32 do pattern[i + 1] = '[a-zA-Z0-9]' end + pattern[34] = '$' + if not settings.cookie_key:match(table.concat(pattern, '')) then + rspamd_logger.errx(rspamd_config, + 'invalid cookies key: %s, must be 32 hex digits', settings.cookie_key) + lua_util.disable_module(N, "config") + + return + end + + if settings.cookie_valid_time then + settings.cookie_valid_time = lua_util.parse_time_interval(settings.cookie_valid_time) + end + + local id = rspamd_config:register_symbol({ + name = 'REPLIES_CHECK', + type = 'prefilter,nostat', + callback = replies_check_cookie, + priority = 10, + group = "replies" + }) + rspamd_config:register_symbol({ + name = settings['symbol'], + parent = id, + type = 'virtual', + score = settings.score, + group = "replies", + }) + end else rspamd_config:register_symbol({ name = 'REPLIES_SET', @@ -161,8 +272,4 @@ if opts then group = "replies", }) end - - for k,v in pairs(opts) do - settings[k] = v - end end diff --git a/src/plugins/lua/reputation.lua b/src/plugins/lua/reputation.lua index 7831f2770..374771c9b 100644 --- a/src/plugins/lua/reputation.lua +++ b/src/plugins/lua/reputation.lua @@ -38,10 +38,9 @@ local redis_params = nil local default_expiry = 864000 -- 10 day by default local keymap_schema = ts.shape{ - ['reject'] = ts.string, - ['add header'] = ts.string, - ['rewrite subject'] = ts.string, - ['no action'] = ts.string + ['spam'] = ts.string, + ['junk'] = ts.string, + ['ham'] = ts.string, } -- Get reputation from ham/spam/probable hits @@ -109,7 +108,8 @@ local function gen_dkim_queries(task, rule) local dom,res = lpeg.match(gr, opt) if dom and res then - ret[dom] = res + local tld = rspamd_util.get_tld(dom) + ret[tld] = res end end end @@ -165,14 +165,14 @@ local function dkim_reputation_filter(task, rule) end local function dkim_reputation_idempotent(task, rule) - local action = task:get_metric_action() + local verdict = lua_util.get_task_verdict(task) local token = { } local cfg = rule.selector.config local need_set = false -- TODO: take metric score into consideration - local k = cfg.keys_map[action] + local k = cfg.keys_map[verdict] if k then token[k] = 1.0 @@ -218,10 +218,9 @@ local dkim_selector = { -- s is for spam, -- p is for probable spam keys_map = { - ['reject'] = 's', - ['add header'] = 'p', - ['rewrite subject'] = 'p', - ['no action'] = 'h' + ['spam'] = 's', + ['junk'] = 'p', + ['ham'] = 'h' }, symbol = 'DKIM_SCORE', -- symbol to be inserted lower_bound = 10, -- minimum number of messages to be scored @@ -312,14 +311,14 @@ local function url_reputation_filter(task, rule) end local function url_reputation_idempotent(task, rule) - local action = task:get_metric_action() + local verdict = lua_util.get_task_verdict(task) local token = { } local cfg = rule.selector.config local need_set = false -- TODO: take metric score into consideration - local k = cfg.keys_map[action] + local k = cfg.keys_map[verdict] if k then token[k] = 1.0 @@ -343,10 +342,9 @@ local url_selector = { -- s is for spam, -- p is for probable spam keys_map = { - ['reject'] = 's', - ['add header'] = 'p', - ['rewrite subject'] = 'p', - ['no action'] = 'h' + ['spam'] = 's', + ['junk'] = 'p', + ['ham'] = 'h' }, symbol = 'URL_SCORE', -- symbol to be inserted lower_bound = 10, -- minimum number of messages to be scored @@ -357,7 +355,7 @@ local url_selector = { outbound = true, inbound = true, }, - dependencies = {"SURBL_CALLBACK"}, + dependencies = {"SURBL_REDIRECTOR_CALLBACK"}, filter = url_reputation_filter, -- used to get scores idempotent = url_reputation_idempotent -- used to set scores } @@ -509,13 +507,13 @@ local function ip_reputation_idempotent(task, rule) end end - local action = task:get_metric_action() + local verdict = lua_util.get_task_verdict(task) local token = { } local need_set = false -- TODO: take metric score into consideration - local k = cfg.keys_map[action] + local k = cfg.keys_map[verdict] if k then token[k] = 1.0 @@ -545,10 +543,9 @@ local ip_selector = { -- s is for spam, -- p is for probable spam keys_map = { - ['reject'] = 's', - ['add header'] = 'p', - ['rewrite subject'] = 'p', - ['no action'] = 'h' + ['spam'] = 's', + ['junk'] = 'p', + ['ham'] = 'h' }, scores = { -- how each component is evaluated ['asn'] = 0.4, @@ -603,7 +600,7 @@ local function spf_reputation_filter(task, rule) end local function spf_reputation_idempotent(task, rule) - local action = task:get_metric_action() + local verdict = lua_util.get_task_verdict(task) local spf_record = task:get_mempool():get_variable('spf_record') local spf_allow = task:has_symbol('R_SPF_ALLOW') local token = { @@ -614,7 +611,7 @@ local function spf_reputation_idempotent(task, rule) if not spf_record or not spf_allow then return end -- TODO: take metric score into consideration - local k = cfg.keys_map[action] + local k = cfg.keys_map[verdict] if k then token[k] = 1.0 @@ -639,10 +636,9 @@ local spf_selector = { -- s is for spam, -- p is for probable spam keys_map = { - ['reject'] = 's', - ['add header'] = 'p', - ['rewrite subject'] = 'p', - ['no action'] = 'h' + ['spam'] = 's', + ['junk'] = 'p', + ['ham'] = 'h' }, symbol = 'SPF_SCORE', -- symbol to be inserted lower_bound = 10, -- minimum number of messages to be scored @@ -707,18 +703,20 @@ local function generic_reputation_filter(task, rule) if selector_res then if type(selector_res) == 'table' then fun.each(function(e) - lua_util.debugm(N, task, 'check generic reputation %s', e) + lua_util.debugm(N, task, 'check generic reputation (%s) %s', + rule['symbol'], e) rule.backend.get_token(task, rule, e, tokens_cb) end, selector_res) else - lua_util.debugm(N, task, 'check generic reputation %s', selector_res) + lua_util.debugm(N, task, 'check generic reputation (%s) %s', + rule['symbol'], selector_res) rule.backend.get_token(task, rule, selector_res, tokens_cb) end end end local function generic_reputation_idempotent(task, rule) - local action = task:get_metric_action() + local verdict = lua_util.get_task_verdict(task) local cfg = rule.selector.config local need_set = false local token = {} @@ -726,7 +724,7 @@ local function generic_reputation_idempotent(task, rule) local selector_res = cfg.selector(task) if not selector_res then return end - local k = cfg.keys_map[action] + local k = cfg.keys_map[verdict] if k then token[k] = 1.0 @@ -736,13 +734,13 @@ local function generic_reputation_idempotent(task, rule) if need_set then if type(selector_res) == 'table' then fun.each(function(e) - lua_util.debugm(N, task, 'set generic selector %s = %s', - e, token) + lua_util.debugm(N, task, 'set generic selector (%s) %s = %s', + rule['symbol'], e, token) rule.backend.set_token(task, rule, e, token) end, selector_res) else - lua_util.debugm(N, task, 'set generic selector %s = %s', - selector_res, token) + lua_util.debugm(N, task, 'set generic selector (%s) %s = %s', + rule['symbol'], selector_res, token) rule.backend.set_token(task, rule, selector_res, token) end end @@ -767,10 +765,9 @@ local generic_selector = { -- s is for spam, -- p is for probable spam keys_map = { - ['reject'] = 's', - ['add header'] = 'p', - ['rewrite subject'] = 'p', - ['no action'] = 'h' + ['spam'] = 's', + ['junk'] = 'p', + ['ham'] = 'h' }, lower_bound = 10, -- minimum number of messages to be scored min_score = nil, @@ -990,22 +987,22 @@ local function reputation_redis_get_token(task, rule, token, continuation_cb) values[data[i]] = ndata end end - lua_util.debugm(N, task, 'got values for key %s -> %s', - key, values) + lua_util.debugm(N, task, 'rule %s - got values for key %s -> %s', + rule['symbol'], key, values) continuation_cb(nil, key, values) else - rspamd_logger.errx(task, 'invalid type while getting reputation keys %s: %s', - key, type(data)) + rspamd_logger.errx(task, 'rule %s - invalid type while getting reputation keys %s: %s', + rule['symbol'], key, type(data)) continuation_cb("invalid type", key, nil) end elseif err then - rspamd_logger.errx(task, 'got error while getting reputation keys %s: %s', - key, err) + rspamd_logger.errx(task, 'rule %s - got error while getting reputation keys %s: %s', + rule['symbol'], key, err) continuation_cb(err, key, nil) else - rspamd_logger.errx(task, 'got error while getting reputation keys %s: %s', - key, "unknown error") + rspamd_logger.errx(task, 'rule %s - got error while getting reputation keys %s: %s', + rule['symbol'], key, "unknown error") continuation_cb("unknown error", key, nil) end end @@ -1024,8 +1021,8 @@ local function reputation_redis_set_token(task, rule, token, values, continuatio local function redis_set_cb(err, data) if err then - rspamd_logger.errx(task, 'got error while setting reputation keys %s: %s', - key, err) + rspamd_logger.errx(task, 'rule %s - got error while setting reputation keys %s: %s', + rule['symbol'], key, err) if continuation_cb then continuation_cb(err, key) end @@ -1042,8 +1039,8 @@ local function reputation_redis_set_token(task, rule, token, values, continuatio table.insert(args, k) table.insert(args, v) end - lua_util.debugm(N, task, 'set values for key %s -> %s', - key, values) + lua_util.debugm(N, task, 'rule %s - set values for key %s -> %s', + rule['symbol'], key, values) local ret = lua_redis.exec_redis_script(rule.backend.script_set, {task = task, is_write = true}, redis_set_cb, @@ -1217,7 +1214,7 @@ local function parse_rule(name, tbl) local symbol = name if tbl.symbol then - symbol = name + symbol = tbl.symbol end rule.symbol = symbol diff --git a/src/plugins/lua/settings.lua b/src/plugins/lua/settings.lua index 970062d3b..a6ce955c1 100644 --- a/src/plugins/lua/settings.lua +++ b/src/plugins/lua/settings.lua @@ -193,25 +193,40 @@ local function check_settings(task) end local function check_specific_setting(rule_name, rule, ip, client_ip, from, rcpt, - user, auth_user) + user, auth_user, hostname, matched) local res = false - if rule['authenticated'] then + if rule.authenticated then if auth_user then res = true + matched[#matched + 1] = 'authenticated' end if not res then return nil end end - if rule['ip'] then + if rule['local'] then if not ip or not ip:is_valid() then return nil end - for _, i in ipairs(rule['ip']) do + + if ip:is_local() then + matched[#matched + 1] = 'local' + res = true + else + return nil + end + end + + if rule.ip then + if not ip or not ip:is_valid() then + return nil + end + for _, i in ipairs(rule.ip) do res = check_ip_setting(i, ip) if res then + matched[#matched + 1] = 'ip' break end end @@ -220,13 +235,14 @@ local function check_settings(task) end end - if rule['client_ip'] then + if rule.client_ip then if not client_ip or not client_ip:is_valid() then return nil end - for _, i in ipairs(rule['client_ip']) do + for _, i in ipairs(rule.client_ip) do res = check_ip_setting(i, client_ip) if res then + matched[#matched + 1] = 'client_ip' break end end @@ -235,13 +251,14 @@ local function check_settings(task) end end - if rule['from'] then + if rule.from then if not from then return nil end - for _, i in ipairs(rule['from']) do + for _, i in ipairs(rule.from) do res = check_addr_setting(i, from) if res then + matched[#matched + 1] = 'from' break end end @@ -250,13 +267,15 @@ local function check_settings(task) end end - if rule['rcpt'] then + if rule.rcpt then if not rcpt then return nil end - for _, i in ipairs(rule['rcpt']) do + for _, i in ipairs(rule.rcpt) do res = check_addr_setting(i, rcpt) + if res then + matched[#matched + 1] = 'rcpt' break end end @@ -265,13 +284,14 @@ local function check_settings(task) end end - if rule['user'] then + if rule.user then if not user then return nil end - for _, i in ipairs(rule['user']) do + for _, i in ipairs(rule.user) do res = check_addr_setting(i, user) if res then + matched[#matched + 1] = 'user' break end end @@ -280,11 +300,28 @@ local function check_settings(task) end end - if rule['request_header'] then - for k, v in pairs(rule['request_header']) do + if rule.hostname then + if #hostname == 0 then + return nil + end + for _, i in ipairs(rule.hostname) do + res = check_addr_setting(i, hostname) + if res then + matched[#matched + 1] = 'hostname' + break + end + end + if not res then + return nil + end + end + + if rule.request_header then + for k, v in pairs(rule.request_header) do local h = task:get_request_header(k) res = (h and v:match(h)) if res then + matched[#matched + 1] = 'req_header: ' .. k break end end @@ -293,13 +330,14 @@ local function check_settings(task) end end - if rule['header'] then - for _, e in ipairs(rule['header']) do + if rule.header then + for _, e in ipairs(rule.header) do for k, v in pairs(e) do for _, p in ipairs(v) do local h = task:get_header(k) res = (h and p:match(h)) if res then + matched[#matched + 1] = 'header: ' .. k break end end @@ -316,21 +354,11 @@ local function check_settings(task) end end - if rule['selector'] then - local sel = selectors_cache[rule_name] - if not sel then - sel = lua_selectors.create_selector_closure(rspamd_config, rule.selector, - rule.delimiter or "") + if rule.selector then + res = fun.all(function(s) return s(task) end, rule.selector) - if sel then - selectors_cache[rule_name] = sel - end - end - - if sel then - if sel(task) then - res = true - end + if res then + matched[#matched + 1] = 'selector' end end @@ -361,6 +389,7 @@ local function check_settings(task) local from = task:get_from() local rcpt = task:get_recipients() local uname = task:get_user() + local hostname = task:get_hostname() or '' local user = {} if uname then user[1] = {} @@ -380,19 +409,23 @@ local function check_settings(task) for pri = max_pri,1,-1 do if not applied and settings[pri] then for _,s in ipairs(settings[pri]) do - local rule = check_specific_setting(s.name, s.rule, ip, client_ip, from, rcpt, user, uname) - if rule then - rspamd_logger.infox(task, "<%1> apply settings according to rule %2", - task:get_message_id(), s.name) - if rule['apply'] then - apply_settings(task, rule['apply']) + local matched = {} + local rule = check_specific_setting(s.name, s.rule, + ip, client_ip, from, rcpt, user, uname, hostname, matched) + + -- Can use xor here but more complicated for reading + if (rule and not s.rule.inverse) or (not rule and s.rule.inverse) then + rspamd_logger.infox(task, "<%s> apply settings according to rule %s (%s matched)", + task:get_message_id(), s.name, table.concat(matched, ',')) + if s.rule['apply'] then + apply_settings(task, s.rule['apply']) applied = true end - if rule['symbols'] then + if s.rule['symbols'] then -- Add symbols, specified in the settings fun.each(function(val) task:insert_result(val, 1.0) - end, rule['symbols']) + end, s.rule['symbols']) end end end @@ -556,9 +589,21 @@ local function process_settings_table(tbl) out['user'] = check_table(elt['user'], user) end end + if elt['hostname'] then + local hostname = process_addr(elt['hostname']) + if hostname then + out['hostname'] = check_table(elt['hostname'], hostname) + end + end if elt['authenticated'] then out['authenticated'] = true end + if elt['local'] then + out['local'] = true + end + if elt['inverse'] then + out['inverse'] = true + end if elt['request_header'] then local rho = {} for k, v in pairs(elt['request_header']) do @@ -602,6 +647,26 @@ local function process_settings_table(tbl) end end + if elt['selector'] then + local sel = selectors_cache[name] + if not sel then + sel = lua_selectors.create_selector_closure(rspamd_config, elt.selector, + elt.delimiter or "") + + if sel then + selectors_cache[name] = sel + end + end + + if sel then + if out.selector then + table.insert(out['selector'], sel) + else + out['selector'] = {sel} + end + end + end + -- Now we must process actions if elt['symbols'] then out['symbols'] = elt['symbols'] end if elt['id'] then diff --git a/src/plugins/lua/whitelist.lua b/src/plugins/lua/whitelist.lua index 1c8612386..b9dce612b 100644 --- a/src/plugins/lua/whitelist.lua +++ b/src/plugins/lua/whitelist.lua @@ -162,9 +162,11 @@ local function whitelist_cb(symbol, rule, task) if dkim_opts then fun.each(function(val) if val[2] == '+' then - find_domain(val[1], 'dkim_success') + local tld = rspamd_util.get_tld(val[1]) + find_domain(tld, 'dkim_success') elseif val[2] == '-' then - find_domain(val[1], 'dkim_fail') + local tld = rspamd_util.get_tld(val[1]) + find_domain(tld, 'dkim_fail') end end, fun.map(function(s) diff --git a/src/plugins/regexp.c b/src/plugins/regexp.c index 92cccc338..897bbd277 100644 --- a/src/plugins/regexp.c +++ b/src/plugins/regexp.c @@ -39,7 +39,9 @@ struct regexp_ctx { gsize max_size; }; -static void process_regexp_item (struct rspamd_task *task, void *user_data); +static void process_regexp_item (struct rspamd_task *task, + struct rspamd_symcache_item *item, + void *user_data); /* Initialization */ @@ -170,7 +172,7 @@ regexp_module_config (struct rspamd_config *cfg) res = FALSE; } else { - rspamd_symbols_cache_add_symbol (cfg->cache, + rspamd_symcache_add_symbol (cfg->cache, cur_item->symbol, 0, process_regexp_item, @@ -187,12 +189,12 @@ regexp_module_config (struct rspamd_config *cfg) cur_item->symbol = ucl_object_key (value); cur_item->lua_function = ucl_object_toclosure (value); - rspamd_symbols_cache_add_symbol (cfg->cache, - cur_item->symbol, - 0, - process_regexp_item, - cur_item, - SYMBOL_TYPE_NORMAL, -1); + rspamd_symcache_add_symbol (cfg->cache, + cur_item->symbol, + 0, + process_regexp_item, + cur_item, + SYMBOL_TYPE_NORMAL, -1); nlua ++; } else if (value->type == UCL_OBJECT) { @@ -243,7 +245,7 @@ regexp_module_config (struct rspamd_config *cfg) } if (cur_item && (is_lua || valid_expression)) { - id = rspamd_symbols_cache_add_symbol (cfg->cache, + id = rspamd_symcache_add_symbol (cfg->cache, cur_item->symbol, 0, process_regexp_item, @@ -255,8 +257,10 @@ regexp_module_config (struct rspamd_config *cfg) if (elt != NULL && ucl_object_type (elt) == UCL_USERDATA) { struct ucl_lua_funcdata *conddata; + g_assert (cur_item->symbol != NULL); conddata = ucl_object_toclosure (elt); - rspamd_symbols_cache_add_condition (cfg->cache, id, + rspamd_symcache_add_condition_delayed (cfg->cache, + cur_item->symbol, conddata->L, conddata->idx); } @@ -275,39 +279,96 @@ regexp_module_config (struct rspamd_config *cfg) elt = ucl_object_lookup (value, "score"); if (elt) { - score = ucl_object_todouble (elt); + if (ucl_object_type (elt) != UCL_FLOAT && ucl_object_type (elt) != UCL_INT) { + msg_err_config ( + "score attribute is not numeric for symbol: '%s'", + cur_item->symbol); + + res = FALSE; + } + else { + score = ucl_object_todouble (elt); + } } elt = ucl_object_lookup (value, "one_shot"); if (elt) { - if (ucl_object_toboolean (elt)) { - nshots = 1; + if (ucl_object_type (elt) != UCL_BOOLEAN) { + msg_err_config ( + "one_shot attribute is not numeric for symbol: '%s'", + cur_item->symbol); + + res = FALSE; + } + else { + if (ucl_object_toboolean (elt)) { + nshots = 1; + } } } if ((elt = ucl_object_lookup (value, "any_shot")) != NULL) { - if (ucl_object_toboolean (elt)) { - nshots = -1; + if (ucl_object_type (elt) != UCL_BOOLEAN) { + msg_err_config ( + "any_shot attribute is not numeric for symbol: '%s'", + cur_item->symbol); + + res = FALSE; + } + else { + if (ucl_object_toboolean (elt)) { + nshots = -1; + } } } if ((elt = ucl_object_lookup (value, "nshots")) != NULL) { - nshots = ucl_object_toint (elt); + if (ucl_object_type (elt) != UCL_FLOAT && ucl_object_type (elt) != UCL_INT) { + msg_err_config ( + "nshots attribute is not numeric for symbol: '%s'", + cur_item->symbol); + + res = FALSE; + } + else { + nshots = ucl_object_toint (elt); + } } elt = ucl_object_lookup (value, "one_param"); if (elt) { - if (ucl_object_toboolean (elt)) { - flags |= RSPAMD_SYMBOL_FLAG_ONEPARAM; + if (ucl_object_type (elt) != UCL_BOOLEAN) { + msg_err_config ( + "one_param attribute is not numeric for symbol: '%s'", + cur_item->symbol); + + res = FALSE; + } + else { + if (ucl_object_toboolean (elt)) { + flags |= RSPAMD_SYMBOL_FLAG_ONEPARAM; + } } } elt = ucl_object_lookup (value, "priority"); if (elt) { - priority = ucl_object_toint (elt); + if (ucl_object_type (elt) != UCL_FLOAT && ucl_object_type (elt) != UCL_INT) { + msg_err_config ( + "priority attribute is not numeric for symbol: '%s'", + cur_item->symbol); + + res = FALSE; + } + else { + priority = ucl_object_toint (elt); + } + } + else { + priority = ucl_object_get_priority (value) + 1; } rspamd_config_add_symbol (cfg, cur_item->symbol, @@ -415,7 +476,9 @@ rspamd_lua_call_expression_func (struct ucl_lua_funcdata *lua_data, static void -process_regexp_item (struct rspamd_task *task, void *user_data) +process_regexp_item (struct rspamd_task *task, + struct rspamd_symcache_item *symcache_item, + void *user_data) { struct regexp_module_item *item = user_data; gint res = FALSE; @@ -449,4 +512,6 @@ process_regexp_item (struct rspamd_task *task, void *user_data) if (res) { rspamd_task_insert_result (task, item->symbol, res, NULL); } + + rspamd_symcache_finalize_item (task, symcache_item); } diff --git a/src/plugins/spf.c b/src/plugins/spf.c index 46160878f..eedaf6c2e 100644 --- a/src/plugins/spf.c +++ b/src/plugins/spf.c @@ -45,6 +45,8 @@ #define DEFAULT_SYMBOL_NA "R_SPF_NA" #define DEFAULT_CACHE_SIZE 2048 +static const gchar *M = "rspamd spf plugin"; + struct spf_ctx { struct module_ctx ctx; const gchar *symbol_fail; @@ -62,7 +64,9 @@ struct spf_ctx { gboolean check_authed; }; -static void spf_symbol_callback (struct rspamd_task *task, void *unused); +static void spf_symbol_callback (struct rspamd_task *task, + struct rspamd_symcache_item *item, + void *unused); /* Initialization */ gint spf_module_init (struct rspamd_config *cfg, struct module_ctx **ctx); @@ -295,38 +299,38 @@ spf_module_config (struct rspamd_config *cfg) &spf_module_ctx->whitelist_ip, NULL); } - cb_id = rspamd_symbols_cache_add_symbol (cfg->cache, - spf_module_ctx->symbol_fail, - 0, - spf_symbol_callback, - NULL, - SYMBOL_TYPE_NORMAL|SYMBOL_TYPE_FINE|SYMBOL_TYPE_EMPTY, -1); - rspamd_symbols_cache_add_symbol (cfg->cache, + cb_id = rspamd_symcache_add_symbol (cfg->cache, + spf_module_ctx->symbol_fail, + 0, + spf_symbol_callback, + NULL, + SYMBOL_TYPE_NORMAL | SYMBOL_TYPE_FINE | SYMBOL_TYPE_EMPTY, -1); + rspamd_symcache_add_symbol (cfg->cache, spf_module_ctx->symbol_softfail, 0, NULL, NULL, SYMBOL_TYPE_VIRTUAL, cb_id); - rspamd_symbols_cache_add_symbol (cfg->cache, + rspamd_symcache_add_symbol (cfg->cache, spf_module_ctx->symbol_permfail, 0, NULL, NULL, SYMBOL_TYPE_VIRTUAL, cb_id); - rspamd_symbols_cache_add_symbol (cfg->cache, + rspamd_symcache_add_symbol (cfg->cache, spf_module_ctx->symbol_na, 0, NULL, NULL, SYMBOL_TYPE_VIRTUAL, cb_id); - rspamd_symbols_cache_add_symbol (cfg->cache, + rspamd_symcache_add_symbol (cfg->cache, spf_module_ctx->symbol_neutral, 0, NULL, NULL, SYMBOL_TYPE_VIRTUAL, cb_id); - rspamd_symbols_cache_add_symbol (cfg->cache, + rspamd_symcache_add_symbol (cfg->cache, spf_module_ctx->symbol_allow, 0, NULL, NULL, SYMBOL_TYPE_VIRTUAL, cb_id); - rspamd_symbols_cache_add_symbol (cfg->cache, + rspamd_symcache_add_symbol (cfg->cache, spf_module_ctx->symbol_dnsfail, 0, NULL, NULL, SYMBOL_TYPE_VIRTUAL, @@ -510,7 +514,7 @@ spf_plugin_callback (struct spf_resolved *record, struct rspamd_task *task, gpointer ud) { struct spf_resolved *l; - struct rspamd_async_watcher *w = ud; + struct rspamd_symcache_item *item = (struct rspamd_symcache_item *)ud; struct spf_ctx *spf_module_ctx = spf_get_context (task->cfg); if (record && record->na) { @@ -560,16 +564,17 @@ spf_plugin_callback (struct spf_resolved *record, struct rspamd_task *task, spf_record_unref (l); } - rspamd_session_watcher_pop (task->s, w); + rspamd_symcache_item_async_dec_check (task, item, M); } static void -spf_symbol_callback (struct rspamd_task *task, void *unused) +spf_symbol_callback (struct rspamd_task *task, + struct rspamd_symcache_item *item, + void *unused) { const gchar *domain; struct spf_resolved *l; - struct rspamd_async_watcher *w; gint *dmarc_checks; struct spf_ctx *spf_module_ctx = spf_get_context (task->cfg); @@ -591,6 +596,7 @@ spf_symbol_callback (struct rspamd_task *task, void *unused) if (rspamd_match_radix_map_addr (spf_module_ctx->whitelist_ip, task->from_addr) != NULL) { + rspamd_symcache_finalize_item (task, item); return; } @@ -598,10 +604,13 @@ spf_symbol_callback (struct rspamd_task *task, void *unused) || (!spf_module_ctx->check_local && rspamd_inet_address_is_local (task->from_addr, TRUE))) { msg_info_task ("skip SPF checks for local networks and authorized users"); + rspamd_symcache_finalize_item (task, item); + return; } domain = rspamd_spf_get_domain (task); + rspamd_symcache_item_async_inc (task, item, M); if (domain) { if ((l = @@ -612,9 +621,8 @@ spf_symbol_callback (struct rspamd_task *task, void *unused) spf_record_unref (l); } else { - w = rspamd_session_get_watcher (task->s); - if (!rspamd_spf_resolve (task, spf_plugin_callback, w)) { + if (!rspamd_spf_resolve (task, spf_plugin_callback, item)) { msg_info_task ("cannot make spf request for [%s]", task->message_id); rspamd_task_insert_result (task, @@ -623,8 +631,10 @@ spf_symbol_callback (struct rspamd_task *task, void *unused) "(SPF): spf DNS fail"); } else { - rspamd_session_watcher_push (task->s); + rspamd_symcache_item_async_inc (task, item, M); } } } + + rspamd_symcache_item_async_dec_check (task, item, M); } diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c index c27e5c858..4bc17db20 100644 --- a/src/plugins/surbl.c +++ b/src/plugins/surbl.c @@ -64,6 +64,8 @@ INIT_LOG_MODULE(surbl) +static const gchar *M = "surbl"; + #define DEFAULT_SURBL_WEIGHT 10 #define DEFAULT_REDIRECTOR_READ_TIMEOUT 5.0 #define DEFAULT_SURBL_SYMBOL "SURBL_DNS" @@ -108,7 +110,7 @@ struct dns_param { struct rspamd_task *task; gchar *host_resolve; struct suffix_item *suffix; - struct rspamd_async_watcher *w; + struct rspamd_symcache_item *item; struct surbl_module_ctx *ctx; }; @@ -120,7 +122,7 @@ struct redirector_param { struct rspamd_http_connection *conn; GHashTable *tree; struct suffix_item *suffix; - struct rspamd_async_watcher *w; + struct rspamd_symcache_item *item; gint sock; guint redirector_requests; }; @@ -136,8 +138,12 @@ static const guint64 rspamd_surbl_cb_magic = 0xe09b8536f80de0d1ULL; static const gchar *rspamd_surbl_default_monitored = "facebook.com"; static const guint default_max_redirected_urls = 10; -static void surbl_test_url (struct rspamd_task *task, void *user_data); -static void surbl_test_redirector (struct rspamd_task *task, void *user_data); +static void surbl_test_url (struct rspamd_task *task, + struct rspamd_symcache_item *item, + void *user_data); +static void surbl_test_redirector (struct rspamd_task *task, + struct rspamd_symcache_item *item, + void *user_data); static void surbl_dns_callback (struct rdns_reply *reply, gpointer arg); static void surbl_dns_ip_callback (struct rdns_reply *reply, gpointer arg); static void process_dns_results (struct rspamd_task *task, @@ -605,7 +611,7 @@ register_bit_symbols (struct rspamd_config *cfg, struct suffix_item *suffix, while (g_hash_table_iter_next (&it, &k, &v)) { bit = v; - rspamd_symbols_cache_add_symbol (cfg->cache, bit->symbol, + rspamd_symcache_add_symbol (cfg->cache, bit->symbol, 0, NULL, NULL, SYMBOL_TYPE_VIRTUAL, parent_id); msg_debug_config ("bit: %d", bit->bit); @@ -614,13 +620,13 @@ register_bit_symbols (struct rspamd_config *cfg, struct suffix_item *suffix, else if (suffix->bits != NULL) { for (i = 0; i < suffix->bits->len; i++) { bit = &g_array_index (suffix->bits, struct surbl_bit_item, i); - rspamd_symbols_cache_add_symbol (cfg->cache, bit->symbol, + rspamd_symcache_add_symbol (cfg->cache, bit->symbol, 0, NULL, NULL, SYMBOL_TYPE_VIRTUAL, parent_id); } } else { - rspamd_symbols_cache_add_symbol (cfg->cache, suffix->symbol, + rspamd_symcache_add_symbol (cfg->cache, suffix->symbol, 0, NULL, NULL, SYMBOL_TYPE_VIRTUAL, parent_id); } @@ -756,10 +762,30 @@ surbl_module_parse_rule (const ucl_object_t* value, struct rspamd_config* cfg) continue; } - cb_id = rspamd_symbols_cache_add_symbol (cfg->cache, "SURBL_CALLBACK", + GString *sym = g_string_sized_new (127); + gchar *p; + + rspamd_printf_gstring (sym, "SURBL_%s", + new_suffix->suffix); + + p = sym->str; + + while (*p) { + if (*p == '.') { + *p = '_'; + } + else { + *p = g_ascii_toupper (*p); + } + + p ++; + } + + cb_id = rspamd_symcache_add_symbol (cfg->cache, sym->str, 0, surbl_test_url, new_suffix, SYMBOL_TYPE_CALLBACK, -1); - rspamd_symbols_cache_add_dependency (cfg->cache, cb_id, + rspamd_symcache_add_dependency (cfg->cache, cb_id, SURBL_REDIRECTOR_CALLBACK); + g_string_free (sym, TRUE); nrules++; new_suffix->callback_id = cb_id; cur = ucl_object_lookup (cur_rule, "bits"); @@ -889,7 +915,7 @@ surbl_module_parse_rule (const ucl_object_t* value, struct rspamd_config* cfg) if (new_suffix->symbol) { /* Register just a symbol itself */ - rspamd_symbols_cache_add_symbol (cfg->cache, + rspamd_symcache_add_symbol (cfg->cache, new_suffix->symbol, 0, NULL, NULL, SYMBOL_TYPE_VIRTUAL, cb_id); nrules++; @@ -943,7 +969,7 @@ surbl_module_config (struct rspamd_config *cfg) lua_pop (L, 1); /* Remove global function */ - (void)rspamd_symbols_cache_add_symbol (cfg->cache, SURBL_REDIRECTOR_CALLBACK, + (void) rspamd_symcache_add_symbol (cfg->cache, SURBL_REDIRECTOR_CALLBACK, 0, surbl_test_redirector, NULL, SYMBOL_TYPE_CALLBACK, -1); @@ -967,9 +993,9 @@ surbl_module_config (struct rspamd_config *cfg) rspamd_config_get_module_opt (cfg, "surbl", "redirector_symbol")) != NULL) { surbl_module_ctx->redirector_symbol = ucl_obj_tostring (value); - rspamd_symbols_cache_add_symbol (cfg->cache, - surbl_module_ctx->redirector_symbol, - 0, NULL, NULL, SYMBOL_TYPE_COMPOSITE, -1); + rspamd_symcache_add_symbol (cfg->cache, + surbl_module_ctx->redirector_symbol, + 0, NULL, NULL, SYMBOL_TYPE_COMPOSITE, -1); } else { surbl_module_ctx->redirector_symbol = NULL; @@ -988,7 +1014,7 @@ surbl_module_config (struct rspamd_config *cfg) surbl_module_ctx->use_tags = ucl_obj_toboolean (value); } else { - surbl_module_ctx->use_tags = TRUE; + surbl_module_ctx->use_tags = FALSE; } if ((value = @@ -1084,7 +1110,7 @@ surbl_module_config (struct rspamd_config *cfg) } if (cur_suffix->options & SURBL_OPTION_CHECKDKIM) { - rspamd_symbols_cache_add_dependency (cfg->cache, + rspamd_symcache_add_dependency (cfg->cache, cur_suffix->callback_id, "DKIM_TRACE"); } @@ -1319,6 +1345,7 @@ format_surbl_request (rspamd_mempool_t * pool, static void make_surbl_requests (struct rspamd_url *url, struct rspamd_task *task, + struct rspamd_symcache_item *item, struct suffix_item *suffix, gboolean forced, GHashTable *tree, struct surbl_ctx *surbl_module_ctx) @@ -1375,8 +1402,8 @@ make_surbl_requests (struct rspamd_url *url, struct rspamd_task *task, if (make_dns_request_task (task, surbl_dns_ip_callback, (void *) param, RDNS_REQUEST_A, surbl_req)) { - param->w = rspamd_session_get_watcher (task->s); - rspamd_session_watcher_push (task->s); + param->item = item; + rspamd_symcache_item_async_inc (task, item, M); } } } @@ -1402,8 +1429,8 @@ make_surbl_requests (struct rspamd_url *url, struct rspamd_task *task, if (make_dns_request_task (task, surbl_dns_callback, (void *) param, RDNS_REQUEST_A, surbl_req)) { - param->w = rspamd_session_get_watcher (task->s); - rspamd_session_watcher_push (task->s); + param->item = item; + rspamd_symcache_item_async_inc (task, item, M); } } else if (err != NULL) { @@ -1428,6 +1455,7 @@ process_dns_results (struct rspamd_task *task, gboolean got_result = FALSE; struct surbl_bit_item *bit; struct in_addr ina; + struct surbl_ctx *surbl_module_ctx = surbl_get_context (task->cfg); if (suffix->ips && g_hash_table_size (suffix->ips) > 0) { @@ -1438,7 +1466,10 @@ process_dns_results (struct rspamd_task *task, resolved_name, suffix->suffix, bit->bit); rspamd_task_insert_result (task, bit->symbol, 1, resolved_name); - rspamd_url_add_tag (uri, "surbl", bit->symbol, task->task_pool); + + if (surbl_module_ctx->use_tags) { + rspamd_url_add_tag (uri, "surbl", bit->symbol, task->task_pool); + } got_result = TRUE; } } @@ -1458,7 +1489,10 @@ process_dns_results (struct rspamd_task *task, resolved_name, suffix->suffix, bit->bit); rspamd_task_insert_result (task, bit->symbol, 1, resolved_name); - rspamd_url_add_tag (uri, "surbl", bit->symbol, task->task_pool); + + if (surbl_module_ctx->use_tags) { + rspamd_url_add_tag (uri, "surbl", bit->symbol, task->task_pool); + } } } } @@ -1469,7 +1503,10 @@ process_dns_results (struct rspamd_task *task, task->message_id, resolved_name, suffix->suffix); rspamd_task_insert_result (task, suffix->symbol, 1, resolved_name); - rspamd_url_add_tag (uri, "surbl", suffix->symbol, task->task_pool); + + if (surbl_module_ctx->use_tags) { + rspamd_url_add_tag (uri, "surbl", suffix->symbol, task->task_pool); + } } else { ina.s_addr = addr; @@ -1508,7 +1545,7 @@ surbl_dns_callback (struct rdns_reply *reply, gpointer arg) param->suffix->suffix); } - rspamd_session_watcher_pop (param->task->s, param->w); + rspamd_symcache_item_async_dec_check (param->task, param->item, M); } static void @@ -1547,7 +1584,7 @@ surbl_dns_ip_callback (struct rdns_reply *reply, gpointer arg) if (make_dns_request_task (task, surbl_dns_callback, param, RDNS_REQUEST_A, to_resolve->str)) { - rspamd_session_watcher_push_specific (task->s, param->w); + rspamd_symcache_item_async_inc (param->task, param->item, M); } g_string_free (to_resolve, TRUE); @@ -1561,7 +1598,7 @@ surbl_dns_ip_callback (struct rdns_reply *reply, gpointer arg) } - rspamd_session_watcher_pop (param->task->s, param->w); + rspamd_symcache_item_async_dec_check (param->task, param->item, M); } static void @@ -1569,6 +1606,10 @@ free_redirector_session (void *ud) { struct redirector_param *param = (struct redirector_param *)ud; + if (param->item) { + rspamd_symcache_item_async_dec_check (param->task, param->item, M); + } + rspamd_http_connection_unref (param->conn); close (param->sock); } @@ -1595,12 +1636,14 @@ surbl_redirector_finish (struct rspamd_http_connection *conn, { struct redirector_param *param = (struct redirector_param *)conn->ud; struct rspamd_task *task; + struct surbl_ctx *surbl_module_ctx; gint r, urllen; struct rspamd_url *redirected_url, *existing; const rspamd_ftok_t *hdr; gchar *urlstr; task = param->task; + surbl_module_ctx = surbl_get_context (task->cfg); if (msg->code == 200) { hdr = rspamd_http_message_find_header (msg, "Uri"); @@ -1630,8 +1673,10 @@ surbl_redirector_finish (struct rspamd_http_connection *conn, existing->count ++; } - rspamd_url_add_tag (param->url, "redirector", urlstr, - task->task_pool); + if (surbl_module_ctx->use_tags) { + rspamd_url_add_tag (param->url, "redirector", urlstr, + task->task_pool); + } } else { msg_info_surbl ("cannot parse redirector reply: %s", urlstr); @@ -1677,6 +1722,7 @@ register_redirector_call (struct rspamd_url *url, struct rspamd_task *task, msg_info_surbl ("<%s> cannot create tcp socket failed: %s", task->message_id, strerror (errno)); + return; } @@ -1700,7 +1746,14 @@ register_redirector_call (struct rspamd_url *url, struct rspamd_task *task, timeout = rspamd_mempool_alloc (task->task_pool, sizeof (struct timeval)); double_to_tv (surbl_module_ctx->read_timeout, timeout); - rspamd_session_add_event (task->s, NULL, free_redirector_session, param, g_quark_from_static_string ("surbl")); + rspamd_session_add_event (task->s, + free_redirector_session, param, + M); + param->item = rspamd_symcache_get_cur_item (task); + + if (param->item) { + rspamd_symcache_item_async_inc (param->task, param->item, M); + } rspamd_http_connection_write_message (param->conn, msg, NULL, NULL, param, s, timeout, task->ev_base); @@ -1738,6 +1791,9 @@ surbl_test_tags (struct rspamd_task *task, struct redirector_param *param, /* We know results for this URL */ DL_FOREACH (tag, cur) { + msg_info_surbl ("<%s> domain [%s] is in surbl %s (tags)", + task->message_id, + ftld, cur->data); rspamd_task_insert_result (task, cur->data, 1, ftld); } @@ -1819,6 +1875,7 @@ surbl_tree_redirector_callback (gpointer key, gpointer value, void *data) *purl = url; rspamd_lua_setclass (L, "rspamd{url}", -1); lua_pushlightuserdata (L, nparam); + rspamd_symcache_set_cur_item (task, param->item); if (lua_pcall (L, 3, 0, 0) != 0) { msg_err_task ("cannot call for redirector script: %s", @@ -1826,8 +1883,7 @@ surbl_tree_redirector_callback (gpointer key, gpointer value, void *data) lua_pop (L, 1); } else { - nparam->w = rspamd_session_get_watcher (task->s); - rspamd_session_watcher_push (task->s); + nparam->item = param->item; } } else { @@ -1851,10 +1907,16 @@ surbl_tree_url_callback (gpointer key, gpointer value, void *data) return; } + if (url->flags & RSPAMD_URL_FLAG_HTML_DISPLAYED) { + /* Skip urls that are displayed only */ + return; + } + task = param->task; surbl_module_ctx = param->ctx; - msg_debug_surbl ("check url %*s", url->urllen, url->string); + msg_debug_surbl ("check url %*s in %s", url->urllen, url->string, + param->suffix->suffix); if (surbl_module_ctx->use_tags && surbl_test_tags (param->task, param, url)) { return; @@ -1865,12 +1927,14 @@ surbl_tree_url_callback (gpointer key, gpointer value, void *data) return; } - make_surbl_requests (url, param->task, param->suffix, FALSE, + make_surbl_requests (url, param->task, param->item, param->suffix, FALSE, param->tree, surbl_module_ctx); } static void -surbl_test_url (struct rspamd_task *task, void *user_data) +surbl_test_url (struct rspamd_task *task, + struct rspamd_symcache_item *item, + void *user_data) { struct redirector_param *param; struct suffix_item *suffix = user_data; @@ -1883,6 +1947,8 @@ surbl_test_url (struct rspamd_task *task, void *user_data) if (!rspamd_monitored_alive (suffix->m)) { msg_info_surbl ("disable surbl %s as it is reported to be offline", suffix->suffix); + rspamd_symcache_finalize_item (task, item); + return; } @@ -1891,11 +1957,15 @@ surbl_test_url (struct rspamd_task *task, void *user_data) param->suffix = suffix; param->tree = g_hash_table_new (rspamd_strcase_hash, rspamd_strcase_equal); param->ctx = surbl_module_ctx; + param->item = item; + rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t)g_hash_table_unref, param->tree); g_hash_table_foreach (task->urls, surbl_tree_url_callback, param); + rspamd_symcache_item_async_inc (task, item, M); + /* We also need to check and process img URLs */ if (suffix->options & SURBL_OPTION_CHECKIMAGES) { for (i = 0; i < task->text_parts->len; i ++) { @@ -1940,10 +2010,14 @@ surbl_test_url (struct rspamd_task *task, void *user_data) } } } + + rspamd_symcache_item_async_dec_check (task, item, M); } static void -surbl_test_redirector (struct rspamd_task *task, void *user_data) +surbl_test_redirector (struct rspamd_task *task, + struct rspamd_symcache_item *item, + void *user_data) { struct redirector_param *param; guint i, j; @@ -1953,14 +2027,19 @@ surbl_test_redirector (struct rspamd_task *task, void *user_data) struct surbl_ctx *surbl_module_ctx = surbl_get_context (task->cfg); if (!surbl_module_ctx->use_redirector || !surbl_module_ctx->redirector_tlds) { + rspamd_symcache_finalize_item (task, item); + return; } + rspamd_symcache_item_async_inc (task, item, M); + param = rspamd_mempool_alloc0 (task->task_pool, sizeof (*param)); param->task = task; param->suffix = NULL; param->redirector_requests = 0; param->ctx = surbl_module_ctx; + param->item = item; g_hash_table_foreach (task->urls, surbl_tree_redirector_callback, param); /* We also need to check and process img URLs */ @@ -1984,6 +2063,8 @@ surbl_test_redirector (struct rspamd_task *task, void *user_data) } } } + + rspamd_symcache_item_async_dec_check (task, item, M); } @@ -2097,12 +2178,14 @@ surbl_continue_process_handler (lua_State *L) gsize urllen; struct rspamd_url *redirected_url; gchar *urlstr; + struct surbl_ctx *surbl_module_ctx; nurl = lua_tolstring (L, 1, &urllen); param = (struct redirector_param *)lua_topointer (L, 2); if (param != NULL) { task = param->task; + surbl_module_ctx = surbl_get_context (task->cfg); if (nurl != NULL) { msg_info_surbl ("<%s> got reply from redirector: '%*s' -> '%*s'", @@ -2125,8 +2208,10 @@ surbl_continue_process_handler (lua_State *L) redirected_url->flags |= RSPAMD_URL_FLAG_REDIRECTED; } - rspamd_url_add_tag (param->url, "redirector", urlstr, - task->task_pool); + if (surbl_module_ctx->use_tags) { + rspamd_url_add_tag (param->url, "redirector", urlstr, + task->task_pool); + } } else { msg_info_surbl ("<%s> could not resolve '%*s' on redirector", @@ -2139,9 +2224,6 @@ surbl_continue_process_handler (lua_State *L) param->task->message_id, param->url->urllen, param->url->string); } - - rspamd_session_watcher_pop (task->s, param->w); - param->w = NULL; } else { return luaL_error (L, "invalid arguments"); diff --git a/src/ragel/smtp_received.rl b/src/ragel/smtp_received.rl index b13259fed..f43ad167c 100644 --- a/src/ragel/smtp_received.rl +++ b/src/ragel/smtp_received.rl @@ -24,12 +24,9 @@ ( address_literal >Real_Domain_Start %Real_Domain_End FWS "(" TCP_info ")" ) | address_literal >Real_IP_Start %Real_IP_End; # Not RFC conforming, but many MTA try this - exim_real_ip = "[" (IPv4_addr|IPv6_simple) >IP4_start %IP4_end "]" - >Real_IP_Start %Real_IP_End (":" digit{1,4})?; - exim_content = exim_real_ip; ccontent = ctext | FWS | '(' @{ fcall balanced_ccontent; }; balanced_ccontent := ccontent* ')' @{ fret; }; - comment = "(" (FWS? ccontent|exim_content)* FWS? ")"; + comment = "(" ((FWS? ccontent)* FWS?) >Comment_Start %Comment_End ")"; CFWS = ((FWS? comment)+ FWS?) | FWS; From_domain = "FROM"i FWS Extended_Domain >From_Start %From_End; diff --git a/src/ragel/smtp_received_parser.rl b/src/ragel/smtp_received_parser.rl index 565a20b7f..836a02384 100644 --- a/src/ragel/smtp_received_parser.rl +++ b/src/ragel/smtp_received_parser.rl @@ -100,33 +100,19 @@ } action Real_IP_Start { - if (real_ip_end == NULL) { + if (real_ip_end == NULL && real_ip_start == NULL) { real_ip_start = p; } } action Real_IP_End { - if (ip_start && ip_end && ip_end > ip_start) { - real_ip_start = ip_start; - real_ip_end = ip_end; - } - else { - real_ip_end = p; - } - - ip_start = NULL; - ip_end = NULL; - } - action Reported_IP_Start { - reported_ip_start = p; - } - action Reported_IP_End { - - if (ip_start && ip_end && ip_end > ip_start) { - reported_ip_start = ip_start; - reported_ip_end = ip_end; - } - else { - reported_ip_end = p; + if (real_ip_end == NULL && real_ip_start != NULL) { + if (ip_start && ip_end && ip_end > ip_start) { + real_ip_start = ip_start; + real_ip_end = ip_end; + } + else { + real_ip_end = p; + } } ip_start = NULL; @@ -225,6 +211,21 @@ } } + action Comment_Start { + cstart = p; + } + + action Comment_End { + cend = p; + + if (cend && cstart && cend > cstart) { + rspamd_smtp_maybe_process_smtp_comment (task, cstart, cend - cstart, rh); + } + + cend = NULL; + cstart = NULL; + } + include smtp_whitespace "smtp_whitespace.rl"; include smtp_ip "smtp_ip.rl"; include smtp_date "smtp_date.rl"; @@ -246,9 +247,8 @@ rspamd_smtp_received_parse (struct rspamd_task *task, const char *data, size_t l const char *real_domain_start, *real_domain_end, *real_ip_start, *real_ip_end, *reported_domain_start, *reported_domain_end, - *reported_ip_start, *reported_ip_end, *ip_start, *ip_end, *date_start, - *for_start, *for_end, *tmp; + *for_start, *for_end, *tmp, *cstart, *cend; struct tm tm; const char *p = data, *pe = data + len, *eof; int cs, in_v6 = 0, *stack = NULL; @@ -269,13 +269,13 @@ rspamd_smtp_received_parse (struct rspamd_task *task, const char *data, size_t l real_ip_end = NULL; reported_domain_start = NULL; reported_domain_end = NULL; - reported_ip_start = NULL; - reported_ip_end = NULL; ip_start = NULL; ip_end = NULL; date_start = NULL; for_start = NULL; for_end = NULL; + cstart = NULL; + cend = NULL; rh->type = RSPAMD_RECEIVED_UNKNOWN; memset (&for_addr, 0, sizeof (for_addr)); @@ -290,10 +290,9 @@ rspamd_smtp_received_parse (struct rspamd_task *task, const char *data, size_t l rh->real_ip = rspamd_mempool_alloc (task->task_pool, tmplen + 1); rspamd_strlcpy (rh->real_ip, real_ip_start, tmplen + 1); } - if (reported_ip_end && reported_ip_start && reported_ip_end > reported_ip_start) { - tmplen = reported_ip_end - reported_ip_start; - rh->from_ip = rspamd_mempool_alloc (task->task_pool, tmplen + 1); - rspamd_strlcpy (rh->from_ip, reported_ip_start, tmplen + 1); + + if (!rh->real_ip && rh->comment_ip) { + rh->real_ip = rh->comment_ip; } if (rh->real_ip && !rh->from_ip) { diff --git a/src/rspamadm/commands.c b/src/rspamadm/commands.c index 81aafcdc9..5b0b4bb5a 100644 --- a/src/rspamadm/commands.c +++ b/src/rspamadm/commands.c @@ -91,9 +91,7 @@ rspamadm_fill_internal_commands (GPtrArray *dest) static void lua_thread_str_error_cb (struct thread_entry *thread, int ret, const char *msg) { - const struct rspamadm_command *cmd = thread->cd; - - msg_err ("call to rspamadm lua script %s failed (%d): %s", cmd->name, + msg_err ("call to rspamadm lua script failed (%d): %s", ret, msg); } diff --git a/src/rspamadm/configdump.c b/src/rspamadm/configdump.c index 962089437..a255994a5 100644 --- a/src/rspamadm/configdump.c +++ b/src/rspamadm/configdump.c @@ -143,8 +143,8 @@ rspamadm_add_doc_elt (const ucl_object_t *obj, const ucl_object_t *doc_obj, elt = ucl_object_lookup (doc_obj, "required"); if (elt) { - rspamd_printf_fstring (&comment, " * Required: %B", - ucl_object_toboolean (elt)); + rspamd_printf_fstring (&comment, " * Required: %s", + ucl_object_toboolean (elt) ? "true" : "false"); cur_comment = ucl_object_fromstring_common (comment->str, comment->len, 0); rspamd_fstring_erase (comment, 0, comment->len); DL_APPEND (nobj, cur_comment); @@ -279,7 +279,7 @@ rspamadm_configdump (gint argc, gchar **argv, const struct rspamadm_command *cmd pworker++; } - cfg->cache = rspamd_symbols_cache_new (cfg); + cfg->cache = rspamd_symcache_new (cfg); cfg->compiled_modules = modules; cfg->compiled_workers = workers; cfg->cfg_name = config; diff --git a/src/rspamadm/configtest.c b/src/rspamadm/configtest.c index 72a8f4945..db9a8d604 100644 --- a/src/rspamadm/configtest.c +++ b/src/rspamadm/configtest.c @@ -136,7 +136,7 @@ rspamadm_configtest (gint argc, gchar **argv, const struct rspamadm_command *cmd (void) g_quark_from_static_string ((*pworker)->name); pworker++; } - cfg->cache = rspamd_symbols_cache_new (cfg); + cfg->cache = rspamd_symcache_new (cfg); cfg->compiled_modules = modules; cfg->compiled_workers = workers; cfg->cfg_name = config; @@ -156,7 +156,7 @@ rspamadm_configtest (gint argc, gchar **argv, const struct rspamadm_command *cmd ret = rspamd_config_post_load (cfg, RSPAMD_CONFIG_INIT_SYMCACHE); } - if (!rspamd_symbols_cache_validate (rspamd_main->cfg->cache, + if (ret && !rspamd_symcache_validate (rspamd_main->cfg->cache, rspamd_main->cfg, FALSE)) { ret = FALSE; diff --git a/src/rspamadm/control.c b/src/rspamadm/control.c index 50f515a5d..0e39ed703 100644 --- a/src/rspamadm/control.c +++ b/src/rspamadm/control.c @@ -132,8 +132,8 @@ rspamd_control_finish_handler (struct rspamd_http_connection *conn, } else { if (strcmp (cbdata->path, "/fuzzystat") == 0) { - rspamadm_execute_lua_ucl_subr (cbdata->argc, - cbdata->argv, + rspamadm_execute_lua_ucl_subr (cbdata->argc - 1, + &cbdata->argv[1], obj, "fuzzy_stat", TRUE); diff --git a/src/rspamadm/lua_repl.c b/src/rspamadm/lua_repl.c index a43527823..6d1501100 100644 --- a/src/rspamadm/lua_repl.c +++ b/src/rspamadm/lua_repl.c @@ -431,7 +431,7 @@ rspamadm_lua_message_handler (lua_State *L, gint argc, gchar **argv) rspamd_printf ("cannot open %s: %s\n", argv[i], strerror (errno)); } else { - task = rspamd_task_new (NULL, rspamd_main->cfg, NULL, NULL); + task = rspamd_task_new (NULL, rspamd_main->cfg, NULL, NULL, NULL); if (!rspamd_task_load_message (task, NULL, map, len)) { rspamd_printf ("cannot load %s\n", argv[i]); diff --git a/src/rspamadm/rspamadm.c b/src/rspamadm/rspamadm.c index 301fa168f..ef52af1e3 100644 --- a/src/rspamadm/rspamadm.c +++ b/src/rspamadm/rspamadm.c @@ -272,7 +272,7 @@ rspamadm_execute_lua_ucl_subr (gint argc, gchar **argv, for (i = 1; i < argc; i ++) { lua_pushstring (L, argv[i]); - lua_rawseti (L, -1, i); + lua_rawseti (L, -2, i); } /* Push results */ diff --git a/src/rspamd.c b/src/rspamd.c index fdc9b1b49..88b44d773 100644 --- a/src/rspamd.c +++ b/src/rspamd.c @@ -43,6 +43,9 @@ #ifdef HAVE_SYS_WAIT_H #include <sys/wait.h> #endif +#ifdef HAVE_SYS_RESOURCE_H +#include <sys/resource.h> +#endif #ifdef HAVE_LIBUTIL_H #include <libutil.h> #endif @@ -277,7 +280,7 @@ reread_config (struct rspamd_main *rspamd_main) struct rspamd_config *tmp_cfg, *old_cfg; gchar *cfg_file; - rspamd_symbols_cache_save (rspamd_main->cfg->cache); + rspamd_symcache_save (rspamd_main->cfg->cache); tmp_cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_DEFAULT); tmp_cfg->libs_ctx = rspamd_main->cfg->libs_ctx; REF_RETAIN (tmp_cfg->libs_ctx); @@ -1028,12 +1031,29 @@ rspamd_cld_handler (gint signo, short what, gpointer arg) g_strsignal (WTERMSIG (res))); } else { +#ifdef HAVE_SYS_RESOURCE_H + struct rlimit rlmt; + (void)getrlimit (RLIMIT_CORE, &rlmt); + msg_warn_main ( "%s process %P terminated abnormally by signal: %s" - " but NOT created core file", + " but NOT created core file (throttled=%s); " + "core file limits: %L current, %L max", g_quark_to_string (cur->type), cur->pid, - g_strsignal (WTERMSIG (res))); + g_strsignal (WTERMSIG (res)), + cur->cores_throttled ? "yes" : "no", + (gint64)rlmt.rlim_cur, + (gint64)rlmt.rlim_max); +#else + msg_warn_main ( + "%s process %P terminated abnormally by signal: %s" + " but NOT created core file (throttled=%s); ", + g_quark_to_string (cur->type), + cur->pid, + g_strsignal (WTERMSIG (res)), + cur->cores_throttled ? "yes" : "no"); +#endif } #else msg_warn_main ( @@ -1063,6 +1083,8 @@ rspamd_cld_handler (gint signo, short what, gpointer arg) if (need_refork) { /* Fork another worker in replace of dead one */ rspamd_check_core_limits (rspamd_main); + + rspamd_fork_delayed (cur->cf, cur->index, rspamd_main); } } @@ -1301,7 +1323,7 @@ main (gint argc, gchar **argv, gchar **env) res = TRUE; - if (!rspamd_symbols_cache_validate (rspamd_main->cfg->cache, + if (!rspamd_symcache_validate (rspamd_main->cfg->cache, rspamd_main->cfg, FALSE)) { res = FALSE; diff --git a/src/rspamd.h b/src/rspamd.h index 409c051b3..74f08c2d3 100644 --- a/src/rspamd.h +++ b/src/rspamd.h @@ -72,6 +72,7 @@ struct rspamd_worker { guint index; /**< index number */ guint nconns; /**< current connections count */ gboolean wanna_die; /**< worker is terminating */ + gboolean cores_throttled; /**< set to true if cores throttling took place */ gdouble start_time; /**< start time */ struct rspamd_main *srv; /**< pointer to server structure */ GQuark type; /**< process type */ @@ -281,6 +282,7 @@ struct rspamd_main { enum rspamd_exception_type { RSPAMD_EXCEPTION_NEWLINE = 0, RSPAMD_EXCEPTION_URL, + RSPAMD_EXCEPTION_GENERIC, }; /** * Structure to point exception in text from processing @@ -288,6 +290,7 @@ enum rspamd_exception_type { struct rspamd_process_exception { goffset pos; guint len; + gpointer ptr; enum rspamd_exception_type type; }; diff --git a/src/rspamd_proxy.c b/src/rspamd_proxy.c index 88bc03bf0..aff866bb4 100644 --- a/src/rspamd_proxy.c +++ b/src/rspamd_proxy.c @@ -38,6 +38,8 @@ #include "libmime/lang_detection.h" #include "contrib/zstd/zstd.h" +#include <math.h> + #ifdef HAVE_NETINET_TCP_H #include <netinet/tcp.h> /* for TCP_NODELAY */ #endif @@ -416,6 +418,7 @@ rspamd_proxy_parse_upstream (rspamd_mempool_t *pool, if (elt) { up->u = rspamd_upstreams_create (ctx->cfg->ups_ctx); + if (!rspamd_upstreams_from_ucl (up->u, elt, 11333, NULL)) { g_set_error (err, rspamd_proxy_quark (), 100, "upstream has bad hosts definition"); @@ -1691,7 +1694,8 @@ rspamd_proxy_self_scan (struct rspamd_proxy_session *session) msg = session->client_message; task = rspamd_task_new (session->worker, session->ctx->cfg, - session->pool, session->ctx->lang_det); + session->pool, session->ctx->lang_det, + session->ctx->ev_base); task->flags |= RSPAMD_TASK_FLAG_MIME; task->sock = -1; @@ -1707,7 +1711,6 @@ rspamd_proxy_self_scan (struct rspamd_proxy_session *session) task->resolver = session->ctx->resolver; /* TODO: allow to disable autolearn in protocol */ task->flags |= RSPAMD_TASK_FLAG_LEARN_AUTO; - task->ev_base = session->ctx->ev_base; task->s = rspamd_session_create (task->task_pool, rspamd_proxy_task_fin, NULL, (event_finalizer_t )rspamd_task_free, task); data = rspamd_http_message_get_body (msg, &len); @@ -1745,6 +1748,17 @@ rspamd_proxy_self_scan (struct rspamd_proxy_session *session) double_to_tv (session->ctx->default_upstream->timeout, &task_tv); event_add (&task->timeout_ev, &task_tv); } + else if (session->ctx->has_self_scan) { + if (session->ctx->cfg->task_timeout > 0) { + struct timeval task_tv; + + event_set (&task->timeout_ev, -1, EV_TIMEOUT, rspamd_task_timeout, + task); + event_base_set (session->ctx->ev_base, &task->timeout_ev); + double_to_tv (session->ctx->cfg->task_timeout, &task_tv); + event_add (&task->timeout_ev, &task_tv); + } + } session->master_conn->task = task; rspamd_task_process (task, RSPAMD_TASK_PROCESS_ALL); @@ -2157,6 +2171,34 @@ proxy_rotate_key (gint fd, short what, void *arg) rspamd_keypair_unref (kp); } +static void +adjust_upstreams_limits (struct rspamd_proxy_ctx *ctx) +{ + struct rspamd_http_upstream *backend; + gpointer k, v; + GHashTableIter it; + + /* + * We set error time equal to max_retries * backend_timeout and max_errors + * to max_retries - 1 + * + * So if we failed to scan a message on a backend for some reasons, we + * will try to re-resolve it faster + */ + + g_hash_table_iter_init (&it, ctx->upstreams); + + while (g_hash_table_iter_next (&it, &k, &v)) { + backend = (struct rspamd_http_upstream *)v; + + if (!backend->self_scan && backend->u) { + rspamd_upstreams_set_limits (backend->u, + NAN, NAN, ctx->max_retries * backend->timeout, NAN, + ctx->max_retries - 1, 0); + } + } +} + void start_rspamd_proxy (struct rspamd_worker *worker) { struct rspamd_proxy_ctx *ctx = worker->ctx; @@ -2207,6 +2249,7 @@ start_rspamd_proxy (struct rspamd_worker *worker) { rspamd_lua_run_postloads (ctx->cfg->lua_state, ctx->cfg, ctx->ev_base, worker); + adjust_upstreams_limits (ctx); event_base_loop (ctx->ev_base, 0); rspamd_worker_block_signals (); diff --git a/src/worker.c b/src/worker.c index 2ce029779..5147e0a2d 100644 --- a/src/worker.c +++ b/src/worker.c @@ -35,14 +35,13 @@ #include "utlist.h" #include "libutil/http_private.h" #include "libmime/lang_detection.h" +#include <math.h> #include "unix-std.h" #include "lua/lua_common.h" /* 60 seconds for worker's IO */ #define DEFAULT_WORKER_IO_TIMEOUT 60000 -/* Timeout for task processing */ -#define DEFAULT_TASK_TIMEOUT 8.0 gpointer init_worker (struct rspamd_config *cfg); void start_worker (struct rspamd_worker *worker); @@ -97,9 +96,8 @@ rspamd_worker_call_finish_handlers (struct rspamd_worker *worker) if (cfg->finish_callbacks) { ctx = worker->ctx; /* Create a fake task object for async events */ - task = rspamd_task_new (worker, cfg, NULL, NULL); + task = rspamd_task_new (worker, cfg, NULL, NULL, ctx->ev_base); task->resolver = ctx->resolver; - task->ev_base = ctx->ev_base; task->flags |= RSPAMD_TASK_FLAG_PROCESSING; task->s = rspamd_session_create (task->task_pool, rspamd_worker_finalize, @@ -144,6 +142,26 @@ rspamd_task_timeout (gint fd, short what, gpointer ud) if (!(task->processed_stages & RSPAMD_TASK_STAGE_FILTERS)) { msg_info_task ("processing of task timed out, forced processing"); + + if (task->cfg->soft_reject_on_timeout) { + struct rspamd_metric_result *res = task->result; + + if (rspamd_check_action_metric (task, res) != METRIC_ACTION_REJECT) { + rspamd_add_passthrough_result (task, + METRIC_ACTION_SOFT_REJECT, + 0, + NAN, + "timeout processing message", + "task timeout"); + + ucl_object_replace_key (task->messages, + ucl_object_fromstring_common ("timeout processing message", + 0, UCL_STRING_RAW), + "smtp_message", 0, + false); + } + } + task->processed_stages |= RSPAMD_TASK_STAGE_FILTERS; rspamd_session_cleanup (task->s); rspamd_task_process (task, RSPAMD_TASK_PROCESS_ALL); @@ -343,7 +361,7 @@ accept_socket (gint fd, short what, void *arg) struct rspamd_worker_ctx *ctx; struct rspamd_task *task; rspamd_inet_addr_t *addr; - gint nfd; + gint nfd, http_opts = 0; ctx = worker->ctx; @@ -364,7 +382,7 @@ accept_socket (gint fd, short what, void *arg) return; } - task = rspamd_task_new (worker, ctx->cfg, NULL, ctx->lang_det); + task = rspamd_task_new (worker, ctx->cfg, NULL, ctx->lang_det, ctx->ev_base); msg_info_task ("accepted connection from %s port %d, task ptr: %p", rspamd_inet_address_to_string (addr), @@ -387,15 +405,18 @@ accept_socket (gint fd, short what, void *arg) /* TODO: allow to disable autolearn in protocol */ task->flags |= RSPAMD_TASK_FLAG_LEARN_AUTO; + if (ctx->encrypted_only && !rspamd_inet_address_is_local (addr, FALSE)) { + http_opts = RSPAMD_HTTP_REQUIRE_ENCRYPTION; + } + task->http_conn = rspamd_http_connection_new (rspamd_worker_body_handler, rspamd_worker_error_handler, rspamd_worker_finish_handler, - 0, + http_opts, RSPAMD_HTTP_SERVER, ctx->keys_cache, NULL); rspamd_http_connection_set_max_size (task->http_conn, task->cfg->max_message); - task->ev_base = ctx->ev_base; worker->nconns++; rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t)reduce_tasks_count, worker); @@ -536,7 +557,7 @@ init_worker (struct rspamd_config *cfg) ctx->is_mime = TRUE; ctx->timeout = DEFAULT_WORKER_IO_TIMEOUT; ctx->cfg = cfg; - ctx->task_timeout = DEFAULT_TASK_TIMEOUT; + ctx->task_timeout = NAN; rspamd_rcl_register_worker_option (cfg, type, @@ -549,30 +570,13 @@ init_worker (struct rspamd_config *cfg) rspamd_rcl_register_worker_option (cfg, type, - "http", - rspamd_rcl_parse_struct_boolean, - ctx, - G_STRUCT_OFFSET (struct rspamd_worker_ctx, is_http), - 0, - "Deprecated: always true now"); - - rspamd_rcl_register_worker_option (cfg, - type, - "json", + "encrypted_only", rspamd_rcl_parse_struct_boolean, ctx, - G_STRUCT_OFFSET (struct rspamd_worker_ctx, is_json), + G_STRUCT_OFFSET (struct rspamd_worker_ctx, encrypted_only), 0, "Deprecated: always true now"); - rspamd_rcl_register_worker_option (cfg, - type, - "allow_learn", - rspamd_rcl_parse_struct_boolean, - ctx, - G_STRUCT_OFFSET (struct rspamd_worker_ctx, allow_learn), - 0, - "Deprecated: disabled and forgotten"); rspamd_rcl_register_worker_option (cfg, type, @@ -592,9 +596,7 @@ init_worker (struct rspamd_config *cfg) G_STRUCT_OFFSET (struct rspamd_worker_ctx, task_timeout), RSPAMD_CL_FLAG_TIME_FLOAT, - "Maximum task processing time, default: " - G_STRINGIFY(DEFAULT_TASK_TIMEOUT) - " seconds"); + "Maximum task processing time, default: 8.0 seconds"); rspamd_rcl_register_worker_option (cfg, type, @@ -670,9 +672,18 @@ start_worker (struct rspamd_worker *worker) ctx->cfg = worker->srv->cfg; ctx->ev_base = rspamd_prepare_worker (worker, "normal", accept_socket); msec_to_tv (ctx->timeout, &ctx->io_tv); - rspamd_symbols_cache_start_refresh (worker->srv->cfg->cache, ctx->ev_base, + rspamd_symcache_start_refresh (worker->srv->cfg->cache, ctx->ev_base, worker); + if (isnan (ctx->task_timeout)) { + if (isnan (ctx->cfg->task_timeout)) { + ctx->task_timeout = 0; + } + else { + ctx->task_timeout = ctx->cfg->task_timeout; + } + } + ctx->resolver = dns_resolver_init (worker->srv->logger, ctx->ev_base, worker->srv->cfg); diff --git a/src/worker_private.h b/src/worker_private.h index fe4a6c4b3..f07a95b41 100644 --- a/src/worker_private.h +++ b/src/worker_private.h @@ -40,12 +40,8 @@ struct rspamd_worker_ctx { struct timeval io_tv; /* Detect whether this worker is mime worker */ gboolean is_mime; - /* HTTP worker */ - gboolean is_http; - /* JSON output */ - gboolean is_json; - /* Allow learning through worker */ - gboolean allow_learn; + /* Allow encrypted requests only using network */ + gboolean encrypted_only; /* Limit of tasks */ guint32 max_tasks; /* Maximum time for task processing */ diff --git a/test/functional/cases/103_password.robot b/test/functional/cases/103_password.robot index 2ae6dec8f..c750369bf 100644 --- a/test/functional/cases/103_password.robot +++ b/test/functional/cases/103_password.robot @@ -20,7 +20,7 @@ PASSWORD - PBKDF PASSWORD - PBKDF WRONG [Setup] Password Setup ${PBKDF_PASSWORD} ${result} = Run Rspamc -h ${LOCAL_ADDR}:${PORT_CONTROLLER} -P q1q1 stat - Check Rspamc ${result} Unauthorized + Should Be Equal As Integers ${result.rc} 1 PASSWORD - CATENA [Setup] Password Setup ${CATENA_PASSWORD} @@ -30,7 +30,7 @@ PASSWORD - CATENA PASSWORD - CATENA WRONG [Setup] Password Setup ${CATENA_PASSWORD} ${result} = Run Rspamc -h ${LOCAL_ADDR}:${PORT_CONTROLLER} -P q stat - Check Rspamc ${result} Unauthorized + Should Be Equal As Integers ${result.rc} 1 PASSWORD - ENABLE [Setup] Password Setup ${CATENA_PASSWORD} @@ -40,12 +40,12 @@ PASSWORD - ENABLE PASSWORD - ENABLE WITH NORMAL [Setup] Password Setup ${CATENA_PASSWORD} ${result} = Run Rspamc -h ${LOCAL_ADDR}:${PORT_CONTROLLER} -P nq1 stat_reset - Check Rspamc ${result} Unauthorized + Should Be Equal As Integers ${result.rc} 1 PASSWORD - ENABLE INCORRECT [Setup] Password Setup ${CATENA_PASSWORD} ${result} = Run Rspamc -h ${LOCAL_ADDR}:${PORT_CONTROLLER} -P q2q2 stat_reset - Check Rspamc ${result} Unauthorized + Should Be Equal As Integers ${result.rc} 1 *** Keywords *** Password Setup diff --git a/test/functional/cases/109_composites.robot b/test/functional/cases/109_composites.robot new file mode 100644 index 000000000..d24c505e9 --- /dev/null +++ b/test/functional/cases/109_composites.robot @@ -0,0 +1,65 @@ +*** Settings *** +Suite Setup Generic Setup +Suite Teardown Simple Teardown +Library ${TESTDIR}/lib/rspamd.py +Resource ${TESTDIR}/lib/rspamd.robot +Variables ${TESTDIR}/lib/vars.py + +*** Variables *** +${CONFIG} ${TESTDIR}/configs/composites.conf +${LUA_SCRIPT} ${TESTDIR}/lua/composites.lua +${MESSAGE} ${TESTDIR}/messages/spam_message.eml +${RSPAMD_SCOPE} Suite + +*** Test Cases *** +Composites - Score + ${result} = Scan Message With Rspamc ${MESSAGE} + Check Rspamc ${result} ${SPACE}50.00 / 0.00 + +Composites - Expressions + ${result} = Scan Message With Rspamc ${MESSAGE} + Check Rspamc ${result} EXPRESSIONS (5.00) + Should Contain ${result.stdout} EXPRESSIONS_B (0.00) + +Composites - Policy: remove_weight + ${result} = Scan Message With Rspamc ${MESSAGE} + Check Rspamc ${result} ${SPACE}POLICY_REMOVE_WEIGHT (5.00) + Should Not Contain ${result.stdout} ${SPACE}POLICY_REMOVE_WEIGHT_A (1.00) + Should Contain ${result.stdout} ${SPACE}POLICY_REMOVE_WEIGHT_B (0.00) + +Composites - Policy: force removing + ${result} = Scan Message With Rspamc ${MESSAGE} + Check Rspamc ${result} ${SPACE}POLICY_FORCE_REMOVE (5.00) + Should Contain ${result.stdout} ${SPACE}POLICY_FORCE_REMOVE_A (1.00) + Should Not Contain ${result.stdout} ${SPACE}POLICY_FORCE_REMOVE_B + +Composites - Policy: leave + ${result} = Scan Message With Rspamc ${MESSAGE} + Check Rspamc ${result} ${SPACE}POLICY_LEAVE (5.00) + Should Not Contain ${result.stdout} ${SPACE}POLICY_LEAVE_A + Should Contain ${result.stdout} ${SPACE}POLICY_LEAVE_B (1.00) + +Composites - Default policy: remove_weight + ${result} = Scan Message With Rspamc ${MESSAGE} + Check Rspamc ${result} DEFAULT_POLICY_REMOVE_WEIGHT (5.00) + Should Contain ${result.stdout} DEFAULT_POLICY_REMOVE_WEIGHT_A (0.00) + Should Contain ${result.stdout} DEFAULT_POLICY_REMOVE_WEIGHT_B (0.00) + +Composites - Default policy: remove_symbol + ${result} = Scan Message With Rspamc ${MESSAGE} + Check Rspamc ${result} DEFAULT_POLICY_REMOVE_SYMBOL (5.00) + Should Not Contain ${result.stdout} DEFAULT_POLICY_REMOVE_SYMBOL_ + +Composites - Default policy: leave + ${result} = Scan Message With Rspamc ${MESSAGE} + Check Rspamc ${result} DEFAULT_POLICY_LEAVE (5.00) + Should Contain ${result.stdout} DEFAULT_POLICY_LEAVE_A (1.00) + Should Contain ${result.stdout} DEFAULT_POLICY_LEAVE_B (1.00) + +Composites - Symbol groups + ${result} = Scan Message With Rspamc ${MESSAGE} + Check Rspamc ${result} SYMBOL_GROUPS (5.00) + Should Contain ${result.stdout} POSITIVE_A (-1.00) + Should Contain ${result.stdout} ANY_A (-1.00) + Should Contain ${result.stdout} NEGATIVE_B (1.00) + Should Not Contain ${result.stdout} NEGATIVE_A diff --git a/test/functional/cases/110_statistics/redis-keyed-siphash.robot b/test/functional/cases/110_statistics/redis-keyed-siphash.robot index ec95efcd1..09d369d54 100644 --- a/test/functional/cases/110_statistics/redis-keyed-siphash.robot +++ b/test/functional/cases/110_statistics/redis-keyed-siphash.robot @@ -15,6 +15,3 @@ Learn Relearn Relearn Test - -Empty Part - Empty Part Test diff --git a/test/functional/cases/110_statistics/redis-keyed-xxhash.robot b/test/functional/cases/110_statistics/redis-keyed-xxhash.robot index 5a7b2daf3..e95b4cebc 100644 --- a/test/functional/cases/110_statistics/redis-keyed-xxhash.robot +++ b/test/functional/cases/110_statistics/redis-keyed-xxhash.robot @@ -15,6 +15,3 @@ Learn Relearn Relearn Test - -Empty Part - Empty Part Test diff --git a/test/functional/cases/110_statistics/redis-plain-siphash.robot b/test/functional/cases/110_statistics/redis-plain-siphash.robot index d436b1a68..deb0c9559 100644 --- a/test/functional/cases/110_statistics/redis-plain-siphash.robot +++ b/test/functional/cases/110_statistics/redis-plain-siphash.robot @@ -14,6 +14,3 @@ Learn Relearn Relearn Test - -Empty Part - Empty Part Test diff --git a/test/functional/cases/110_statistics/redis-plain-xxhash.robot b/test/functional/cases/110_statistics/redis-plain-xxhash.robot index 0c45ac16b..379f39767 100644 --- a/test/functional/cases/110_statistics/redis-plain-xxhash.robot +++ b/test/functional/cases/110_statistics/redis-plain-xxhash.robot @@ -14,6 +14,3 @@ Learn Relearn Relearn Test - -Empty Part - Empty Part Test diff --git a/test/functional/cases/115_dmarc.robot b/test/functional/cases/115_dmarc.robot index c68927115..597a6a330 100644 --- a/test/functional/cases/115_dmarc.robot +++ b/test/functional/cases/115_dmarc.robot @@ -77,6 +77,16 @@ DMARC NA NXDOMAIN ... -i 37.48.67.26 --from foo@mom.za.org Check Rspamc ${result} DMARC_NA +DMARC PCT ZERO REJECT + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/pct_none.eml + ... -i 37.48.67.26 --from foo@mom.za.org + Check Rspamc ${result} DMARC_POLICY_QUARANTINE + +DMARC PCT ZERO SP QUARANTINE + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/pct_none1.eml + ... -i 37.48.67.26 --from foo@mom.za.org + Check Rspamc ${result} DMARC_POLICY_SOFTFAIL + DKIM PERMFAIL NXDOMAIN ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim2.eml ... -i 37.48.67.26 diff --git a/test/functional/cases/210_clickhouse/001_migration.robot b/test/functional/cases/210_clickhouse/001_migration.robot index c2db5b3ec..bf6d48084 100644 --- a/test/functional/cases/210_clickhouse/001_migration.robot +++ b/test/functional/cases/210_clickhouse/001_migration.robot @@ -11,6 +11,7 @@ Test Teardown Clickhosue Teardown *** Variables *** ${CONFIG} ${TESTDIR}/configs/clickhouse.conf ${RSPAMD_SCOPE} Suite +${CLICKHOUSE_PORT} ${18123} *** Test Cases *** Initial schema @@ -68,7 +69,7 @@ Clickhouse Setup ${result} = Run Process clickhouse-server --daemon --config-file\=${TMPDIR}/clickhouse-config.xml --pid-file\=${TMPDIR}/clickhouse.pid Run Keyword If ${result.rc} != 0 Log ${result.stderr} Should Be Equal As Integers ${result.rc} 0 - Wait Until Keyword Succeeds 5 sec 1 sec Check Pidfile ${TMPDIR}/clickhouse.pid timeout=5 sec + Wait Until Keyword Succeeds 5 sec 50 ms TCP Connect localhost ${CLICKHOUSE_PORT} Set Suite Variable ${TMPDIR} ${TMPDIR} diff --git a/test/functional/cases/210_clickhouse/clickhouse.py b/test/functional/cases/210_clickhouse/clickhouse.py index 4e95eadf6..f5e4646cd 100644 --- a/test/functional/cases/210_clickhouse/clickhouse.py +++ b/test/functional/cases/210_clickhouse/clickhouse.py @@ -8,7 +8,7 @@ __client = None class Client: def __init__(self): - self.port = 18123 + self.port = BuiltIn().get_variable_value('${CLICKHOUSE_PORT}', default=18123) def get_query_string(self): return "http://localhost:%d/?default_format=JSONEachRow" % (self.port) diff --git a/test/functional/cases/280_rules.robot b/test/functional/cases/280_rules.robot new file mode 100644 index 000000000..bb2012237 --- /dev/null +++ b/test/functional/cases/280_rules.robot @@ -0,0 +1,44 @@ +*** Settings *** +Test Setup Rules Setup +Test Teardown Rules Teardown +Library ${TESTDIR}/lib/rspamd.py +Resource ${TESTDIR}/lib/rspamd.robot +Variables ${TESTDIR}/lib/vars.py + +*** Variables *** +${CONFIG} ${TESTDIR}/configs/plugins.conf +${MESSAGE1} ${TESTDIR}/messages/fws_fn.eml +${MESSAGE2} ${TESTDIR}/messages/fws_fp.eml +${MESSAGE3} ${TESTDIR}/messages/fws_tp.eml +${MESSAGE4} ${TESTDIR}/messages/broken_richtext.eml +${URL_TLD} ${TESTDIR}/../lua/unit/test_tld.dat +${RSPAMD_SCOPE} Test + + +*** Test Cases *** +Broken MIME + ${result} = Scan Message With Rspamc ${MESSAGE3} + Check Rspamc ${result} MISSING_SUBJECT + +Issue 2584 + ${result} = Scan Message With Rspamc ${MESSAGE1} + Check Rspamc ${result} BROKEN_CONTENT_TYPE inverse=1 + Should Not Contain ${result.stdout} MISSING_SUBJECT + Should Not Contain ${result.stdout} R_MISSING_CHARSET + +Issue 2349 + ${result} = Scan Message With Rspamc ${MESSAGE2} + Check Rspamc ${result} MULTIPLE_UNIQUE_HEADERS inverse=1 + +Broken Rich Text + ${result} = Scan Message With Rspamc ${MESSAGE4} + Check Rspamc ${result} BROKEN_CONTENT_TYPE + +*** Keywords *** +Rules Setup + ${PLUGIN_CONFIG} = Get File ${TESTDIR}/configs/regexp.conf + Set Suite Variable ${PLUGIN_CONFIG} + Generic Setup PLUGIN_CONFIG + +Rules Teardown + Normal Teardown diff --git a/test/functional/cases/290_greylist.robot b/test/functional/cases/290_greylist.robot new file mode 100644 index 000000000..64896fe2b --- /dev/null +++ b/test/functional/cases/290_greylist.robot @@ -0,0 +1,39 @@ +*** Settings *** +Suite Setup Greylist Setup +Suite Teardown Greylist Teardown +Library ${TESTDIR}/lib/rspamd.py +Resource ${TESTDIR}/lib/rspamd.robot +Variables ${TESTDIR}/lib/vars.py + +*** Variables *** +${CONFIG} ${TESTDIR}/configs/plugins.conf +${MESSAGE} ${TESTDIR}/messages/spam_message.eml +${REDIS_SCOPE} Suite +${RSPAMD_SCOPE} Suite +${URL_TLD} ${TESTDIR}/../lua/unit/test_tld.dat + +*** Test Cases *** +GREYLIST NEW + ${result} = Scan Message With Rspamc ${MESSAGE} + Check Rspamc ${result} GREYLIST (0.00)[greylisted + +GREYLIST EARLY + ${result} = Scan Message With Rspamc ${MESSAGE} + Check Rspamc ${result} GREYLIST (0.00)[greylisted + +GREYLIST PASS + Sleep 4s Wait greylisting timeout + ${result} = Scan Message With Rspamc ${MESSAGE} + Check Rspamc ${result} GREYLIST (0.00)[pass + +*** Keywords *** +Greylist Setup + ${PLUGIN_CONFIG} = Get File ${TESTDIR}/configs/greylist.conf + Set Suite Variable ${PLUGIN_CONFIG} + Generic Setup PLUGIN_CONFIG + Run Redis + +Greylist Teardown + Normal Teardown + Shutdown Process With Children ${REDIS_PID} + Terminate All Processes kill=True
\ No newline at end of file diff --git a/test/functional/cases/300_rbl.robot b/test/functional/cases/300_rbl.robot new file mode 100644 index 000000000..8ac96915e --- /dev/null +++ b/test/functional/cases/300_rbl.robot @@ -0,0 +1,47 @@ +*** Settings *** +Suite Setup Rbl Setup +Suite Teardown Rbl Teardown +Library ${TESTDIR}/lib/rspamd.py +Resource ${TESTDIR}/lib/rspamd.robot +Variables ${TESTDIR}/lib/vars.py + +*** Variables *** +${CONFIG} ${TESTDIR}/configs/plugins.conf +${MESSAGE} ${TESTDIR}/messages/spam_message.eml +${RSPAMD_SCOPE} Suite +${URL_TLD} ${TESTDIR}/../lua/unit/test_tld.dat + +*** Test Cases *** +RBL FROM MISS + ${result} = Scan Message With Rspamc ${MESSAGE} -i 1.2.3.4 + Check Rspamc ${result} FAKE_RBL_CODE_2 inverse=True + +RBL FROM HIT + ${result} = Scan Message With Rspamc ${MESSAGE} -i 4.3.2.1 + Check Rspamc ${result} FAKE_RBL_CODE_2 + +RBL FROM MULTIPLE HIT + ${result} = Scan Message With Rspamc ${MESSAGE} -i 4.3.2.3 + Check Rspamc ${result} FAKE_RBL_CODE_2 FAKE_RBL_CODE_3 + +RBL FROM UNKNOWN HIT + ${result} = Scan Message With Rspamc ${MESSAGE} -i 4.3.2.2 + Check Rspamc ${result} FAKE_RBL_UNKNOWN + +RBL RECEIVED HIT + ${result} = Scan Message With Rspamc ${MESSAGE} -i 8.8.8.8 + Check Rspamc ${result} FAKE_RECEIVED_RBL_CODE_3 + +RBL FROM HIT WL + ${result} = Scan Message With Rspamc ${MESSAGE} -i 4.3.2.4 + Check Rspamc ${result} FAKE_RBL_CODE_2 inverse=True + +*** Keywords *** +Rbl Setup + ${PLUGIN_CONFIG} = Get File ${TESTDIR}/configs/rbl.conf + Set Suite Variable ${PLUGIN_CONFIG} + Generic Setup PLUGIN_CONFIG + +Rbl Teardown + Normal Teardown + Terminate All Processes kill=True
\ No newline at end of file diff --git a/test/functional/configs/clickhouse-config.xml b/test/functional/configs/clickhouse-config.xml index 10530a580..b210748ba 100644 --- a/test/functional/configs/clickhouse-config.xml +++ b/test/functional/configs/clickhouse-config.xml @@ -10,7 +10,7 @@ <!-- <console>1</console> --> <!-- Default behavior is autodetection (log to console if not daemon mode and is tty) --> </logger> <!--display_name>production</display_name--> <!-- It is the name that will be shown in the client --> - <http_port>18123</http_port> + <http_port>${CLICKHOUSE_PORT}</http_port> <tcp_port>19000</tcp_port> <!-- For HTTPS and SSL over native protocol. --> diff --git a/test/functional/configs/composites.conf b/test/functional/configs/composites.conf new file mode 100644 index 000000000..8b79d5ae7 --- /dev/null +++ b/test/functional/configs/composites.conf @@ -0,0 +1,69 @@ +options = { + pidfile = "${TMPDIR}/rspamd.pid" +} +logging = { + type = "file", + level = "debug" + filename = "${TMPDIR}/rspamd.log" +} + +worker { + type = normal + bind_socket = ${LOCAL_ADDR}:${PORT_NORMAL} + count = 1 + task_timeout = 60s; +} +worker { + type = controller + bind_socket = ${LOCAL_ADDR}:${PORT_CONTROLLER} + count = 1 + secure_ip = ["127.0.0.1", "::1"]; + stats_path = "${TMPDIR}/stats.ucl" +} +lua = "${TESTDIR}/lua/test_coverage.lua"; +lua = ${LUA_SCRIPT}; + +composites { + EXPRESSIONS { + expression = "(EXPRESSIONS_A | ~EXPRESSIONS_B) & !EXPRESSIONS_C"; + score = 5.0; + } + + POLICY_REMOVE_WEIGHT { + expression = "POLICY_REMOVE_WEIGHT_A and ~POLICY_REMOVE_WEIGHT_B"; + score = 5.0; + } + POLICY_FORCE_REMOVE { + expression = "POLICY_FORCE_REMOVE_A & ^POLICY_FORCE_REMOVE_B"; + score = 5.0; + } + POLICY_FORCE_REMOVE_LEAVE { + expression = "-POLICY_FORCE_REMOVE_A and -POLICY_FORCE_REMOVE_B"; + score = 5.0; + } + POLICY_LEAVE { + expression = "POLICY_LEAVE_A & -POLICY_LEAVE_B"; + score = 5.0; + } + + DEFAULT_POLICY_REMOVE_WEIGHT { + expression = "DEFAULT_POLICY_REMOVE_WEIGHT_A and DEFAULT_POLICY_REMOVE_WEIGHT_B"; + score = 5.0; + policy = "remove_weight"; + } + DEFAULT_POLICY_REMOVE_SYMBOL { + expression = "DEFAULT_POLICY_REMOVE_SYMBOL_A & DEFAULT_POLICY_REMOVE_SYMBOL_B"; + score = 5.0; + policy = "remove_symbol"; + } + DEFAULT_POLICY_LEAVE { + expression = "DEFAULT_POLICY_LEAVE_A & DEFAULT_POLICY_LEAVE_B"; + score = 5.0; + policy = "leave"; + } + + SYMBOL_GROUPS { + expression = "!g+:positive & g-:negative & -g:any"; + score = 5.0; + } +} diff --git a/test/functional/configs/fuzzy.conf b/test/functional/configs/fuzzy.conf index 3eb554dc8..21a5dfbb3 100644 --- a/test/functional/configs/fuzzy.conf +++ b/test/functional/configs/fuzzy.conf @@ -63,6 +63,8 @@ fuzzy_check { retransmits = 10; rule { + min_bytes = 0; + min_length = 0; algorithm = "${ALGORITHM}"; servers = "${LOCAL_ADDR}:${PORT_FUZZY}"; symbol = "R_TEST_FUZZY"; diff --git a/test/functional/configs/greylist.conf b/test/functional/configs/greylist.conf new file mode 100644 index 000000000..5e12accf9 --- /dev/null +++ b/test/functional/configs/greylist.conf @@ -0,0 +1,11 @@ +redis { + servers = "${REDIS_ADDR}:${REDIS_PORT}"; +} +greylist { + check_local = true; + timeout = 4; +} + +actions { + greylist = 1; +} diff --git a/test/functional/configs/plugins.conf b/test/functional/configs/plugins.conf index ecff3eabf..d7a5b009c 100644 --- a/test/functional/configs/plugins.conf +++ b/test/functional/configs/plugins.conf @@ -243,6 +243,11 @@ options = { rcode = 'nxdomain'; }, { + name = "_dmarc.zero_pct.com", + type = "txt"; + replies = ["v=DMARC1; p=reject; sp=quarantine; pct=0"]; + }, + { name = "example.com", type = "txt"; replies = ["$Id: example.com 4415 2015-08-24 20:12:23Z davids $", "v=spf1 -all"]; @@ -476,7 +481,48 @@ options = { name = "fail8.org.org.za", type = "txt"; replies = ["v=spf1 ip4:8.8.8.8 a:www.dnssec-failed.org -all"]; - }]; + }, + { + name = "1.2.3.4.fake.rbl"; + type = "a"; + replies = ["127.0.0.2"]; + }, + { + name = "2.2.3.4.fake.rbl"; + type = "a"; + replies = ["127.0.0.10"]; + }, + { + name = "3.2.3.4.fake.rbl"; + type = "a"; + replies = ["127.0.0.2", "127.0.0.3"]; + }, + { + name = "4.2.3.4.fake.rbl"; + type = "a"; + replies = ["127.0.0.2"]; + }, + { + name = "4.2.3.4.fake.wl"; + type = "a"; + replies = ["127.0.0.2"]; + }, + { + name = "4.3.2.1.fake.rbl"; + type = "a"; + rcode = 'nxdomain'; + }, + { + name = "1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.e.f.fake.rbl"; + type = "a"; + replies = ["127.0.0.2"]; + }, + { + name = "131.193.18.151.fake.rbl"; + type = "a"; + replies = ["127.0.0.3"]; + } + ]; } } logging = { diff --git a/test/functional/configs/rbl.conf b/test/functional/configs/rbl.conf new file mode 100644 index 000000000..a66deea6a --- /dev/null +++ b/test/functional/configs/rbl.conf @@ -0,0 +1,43 @@ +rbl { + rbls { + fake { + from = true; + ipv4 = true; + ipv6 = true; + rbl = "fake.rbl"; + symbol = "FAKE_RBL_UNKNOWN"; + unknown = true; + returncodes = { + "FAKE_RBL_CODE_2" = "127.0.0.2"; + "FAKE_RBL_CODE_3" = "127.0.0.3"; + } + } + fake_received { + from = false; + ipv4 = true; + ipv6 = true; + received = true; + rbl = "fake.rbl"; + symbol = "FAKE_RECEIVED_RBL_UNKNOWN"; + unknown = true; + returncodes = { + "FAKE_RECEIVED_RBL_CODE_2" = "127.0.0.2"; + "FAKE_RECEIVED_RBL_CODE_3" = "127.0.0.3"; + } + } + fake_whitelist { + from = true; + ipv4 = true; + ipv6 = true; + received = true; + is_whitelist = true; + rbl = "fake.wl"; + symbol = "FAKE_WL_RBL_UNKNOWN"; + unknown = true; + returncodes = { + "FAKE_WL_RBL_CODE_2" = "127.0.0.2"; + "FAKE_WL_RBL_CODE_3" = "127.0.0.3"; + } + } + } +}
\ No newline at end of file diff --git a/test/functional/lib/rspamd.py b/test/functional/lib/rspamd.py index e0454e347..bd3e0c382 100644 --- a/test/functional/lib/rspamd.py +++ b/test/functional/lib/rspamd.py @@ -5,15 +5,11 @@ import os.path import psutil import glob import pwd -import re import shutil import signal import socket -import errno import sys import tempfile -import time -import subprocess from robot.libraries.BuiltIn import BuiltIn from robot.api import logger @@ -163,41 +159,45 @@ def spamc(addr, port, filename): return r.decode('utf-8') def TCP_Connect(addr, port): + """Attempts to open a TCP connection to specified address:port + + Example: + | Wait Until Keyword Succeeds | 5s | 10ms | TCP Connect | localhost | 8080 | + """ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(5) # seconds s.connect((addr, port)) + s.close() def update_dictionary(a, b): a.update(b) return a -def shutdown_process(process): - i = 0 - while i < 100: - try: - os.kill(process.pid, signal.SIGTERM) - except OSError as e: - assert e.errno == errno.ESRCH - return - if process.status() == psutil.STATUS_ZOMBIE: - return +TERM_TIMEOUT = 10 # wait after sending a SIGTERM signal +KILL_WAIT = 20 # additional wait after sending a SIGKILL signal - i += 1 - time.sleep(0.1) +def shutdown_process(process): + # send SIGTERM + process.terminate() - while i < 200: + try: + process.wait(TERM_TIMEOUT) + return + except psutil.TimeoutExpired: + logger.info( "PID {} is not termianated in {} seconds, sending SIGKILL..." % + (process.pid, TERM_TIMEOUT)) try: - os.kill(process.pid, signal.SIGKILL) - except OSError as e: - assert e.errno == errno.ESRCH - return - - if process.status() == psutil.STATUS_ZOMBIE: + # send SIGKILL + process.kill() + except psutil.NoSuchProcess: + # process exited just befor we tried to kill return - i += 1 - time.sleep(0.1) - assert False, "Failed to shutdown process %d (%s)" % (process.pid, process.name()) + try: + process.wait(KILL_WAIT) + except psutil.TimeoutExpired: + raise RuntimeError("Failed to shutdown process %d (%s)" % (process.pid, process.name())) def shutdown_process_with_children(pid): @@ -295,41 +295,79 @@ def python3_which(cmd, mode=os.F_OK | os.X_OK, path=None): return None +def _merge_luacov_stats(statsfile, coverage): + """ + Reads a coverage stats file written by luacov and merges coverage data to + 'coverage' dict: { src_file: hits_list } + + Format of the file defined in: + https://github.com/keplerproject/luacov/blob/master/src/luacov/stats.lua + """ + with open(statsfile, 'rb') as fh: + while True: + # max_line:filename + line = fh.readline().rstrip() + if not line: + break + + max_line, src_file = line.split(':') + counts = [int(x) for x in fh.readline().split()] + assert len(counts) == int(max_line) + + if src_file in coverage: + # enlarge list if needed: lenght of list in different luacov.stats.out files may differ + old_len = len(coverage[src_file]) + new_len = len(counts) + if new_len > old_len: + coverage[src_file].extend([0] * (new_len - old_len)) + # sum execution counts for each line + for l, exe_cnt in enumerate(counts): + coverage[src_file][l] += exe_cnt + else: + coverage[src_file] = counts + + +def _dump_luacov_stats(statsfile, coverage): + """ + Saves data to the luacov stats file. Existing file is overwritted if exists. + """ + src_files = sorted(coverage) + + with open(statsfile, 'wb') as fh: + for src in src_files: + stats = " ".join(str(n) for n in coverage[src]) + fh.write("%s:%s\n%s\n" % (len(coverage[src]), src, stats)) + + +# File used by luacov to collect coverage stats +LUA_STATSFILE = "luacov.stats.out" + + def collect_lua_coverage(): - if python3_which("luacov-coveralls") is None: - logger.info("luacov-coveralls not found, will not collect Lua coverage") - return + """ + Merges ${TMPDIR}/*.luacov.stats.out into luacov.stats.out + Example: + | Collect Lua Coverage | + """ # decided not to do optional coverage so far #if not 'ENABLE_LUA_COVERAGE' in os.environ['HOME']: # logger.info("ENABLE_LUA_COVERAGE is not present in env, will not collect Lua coverage") # return - current_directory = os.getcwd() - report_file = current_directory + "/lua_coverage_report.json" - old_report = current_directory + "/lua_coverage_report.json.old" - tmp_dir = BuiltIn().get_variable_value("${TMPDIR}") - coverage_files = glob.glob('%s/*.luacov.stats.out' % (tmp_dir)) - for stat_file in coverage_files: - shutil.move(stat_file, "luacov.stats.out") - # logger.console("statfile: " + stat_file) + coverage = {} + input_files = [] - if (os.path.isfile(report_file)): - shutil.move(report_file, old_report) - p = subprocess.Popen(["luacov-coveralls", "-o", report_file, "-j", old_report, "--merge", "--dryrun"], - stdout = subprocess.PIPE, stderr= subprocess.PIPE) - output,error = p.communicate() - - logger.info("luacov-coveralls stdout: " + output) - logger.info("luacov-coveralls stderr: " + error) - os.remove(old_report) - else: - p = subprocess.Popen(["luacov-coveralls", "-o", report_file, "--dryrun"], stdout = subprocess.PIPE, stderr= subprocess.PIPE) - output,error = p.communicate() - - logger.info("luacov-coveralls stdout: " + output) - logger.info("luacov-coveralls stderr: " + error) - os.remove("luacov.stats.out") + for f in glob.iglob("%s/*.luacov.stats.out" % tmp_dir): + _merge_luacov_stats(f, coverage) + input_files.append(f) + if input_files: + if os.path.isfile(LUA_STATSFILE): + _merge_luacov_stats(LUA_STATSFILE, coverage) + _dump_luacov_stats(LUA_STATSFILE, coverage) + logger.info("%s merged into %s" % (", ".join(input_files), LUA_STATSFILE)) + else: + logger.info("no *.luacov.stats.out files found in %s" % tmp_dir) diff --git a/test/functional/lua/composites.lua b/test/functional/lua/composites.lua new file mode 100644 index 000000000..66d595cd8 --- /dev/null +++ b/test/functional/lua/composites.lua @@ -0,0 +1,126 @@ +rspamd_config:register_symbol({ + name = 'EXPRESSIONS_B', + score = 1.0, + callback = function() + return true, 'Fires always' + end +}) + +rspamd_config:register_symbol({ + name = 'POLICY_REMOVE_WEIGHT_A', + score = 1.0, + callback = function() + return true, 'Fires always' + end +}) +rspamd_config:register_symbol({ + name = 'POLICY_REMOVE_WEIGHT_B', + score = 1.0, + callback = function() + return true, 'Fires always' + end +}) +rspamd_config:register_symbol({ + name = 'POLICY_FORCE_REMOVE_A', + score = 1.0, + callback = function() + return true, 'Fires always' + end +}) +rspamd_config:register_symbol({ + name = 'POLICY_FORCE_REMOVE_B', + score = 1.0, + callback = function() + return true, 'Fires always' + end +}) +rspamd_config:register_symbol({ + name = 'POLICY_LEAVE_A', + score = 1.0, + callback = function() + return true, 'Fires always' + end +}) +rspamd_config:register_symbol({ + name = 'POLICY_LEAVE_B', + score = 1.0, + callback = function() + return true, 'Fires always' + end +}) + +rspamd_config:register_symbol({ + name = 'DEFAULT_POLICY_REMOVE_WEIGHT_A', + score = 1.0, + callback = function() + return true, 'Fires always' + end +}) +rspamd_config:register_symbol({ + name = 'DEFAULT_POLICY_REMOVE_WEIGHT_B', + score = 1.0, + callback = function() + return true, 'Fires always' + end +}) +rspamd_config:register_symbol({ + name = 'DEFAULT_POLICY_REMOVE_SYMBOL_A', + score = 1.0, + callback = function() + return true, 'Fires always' + end +}) +rspamd_config:register_symbol({ + name = 'DEFAULT_POLICY_REMOVE_SYMBOL_B', + score = 1.0, + callback = function() + return true, 'Fires always' + end +}) +rspamd_config:register_symbol({ + name = 'DEFAULT_POLICY_LEAVE_A', + score = 1.0, + callback = function() + return true, 'Fires always' + end +}) +rspamd_config:register_symbol({ + name = 'DEFAULT_POLICY_LEAVE_B', + score = 1.0, + callback = function() + return true, 'Fires always' + end +}) + +rspamd_config:register_symbol({ + name = 'POSITIVE_A', + score = -1.0, + group = "positive", + callback = function() + return true, 'Fires always' + end +}) +rspamd_config:register_symbol({ + name = 'NEGATIVE_A', + score = -1.0, + group = "negative", + callback = function() + return true, 'Fires always' + end +}) +rspamd_config:register_symbol({ + name = 'NEGATIVE_B', + score = 1.0, + group = "negative", + callback = function() + return true, 'Fires always' + end +}) +rspamd_config:register_symbol({ + name = 'ANY_A', + score = -1.0, + group = "any", + callback = function() + return true, 'Fires always' + end +}) diff --git a/test/functional/lua/prepostfilters.lua b/test/functional/lua/prepostfilters.lua index 4f14d0222..c87c95849 100644 --- a/test/functional/lua/prepostfilters.lua +++ b/test/functional/lua/prepostfilters.lua @@ -16,7 +16,7 @@ for i = 1,10 do end if task:has_symbol('TEST_PRE') then local r = task:get_resolver() - r:resolve_a(task:get_session(), task:get_mempool(), 'example.com', dns_cb) + r:resolve_a({task = task, name = 'example.com', callback = dns_cb}) end end }) diff --git a/test/functional/messages/broken_richtext.eml b/test/functional/messages/broken_richtext.eml new file mode 100644 index 000000000..e4786c104 --- /dev/null +++ b/test/functional/messages/broken_richtext.eml @@ -0,0 +1,27 @@ +From: user@example.com
+Message-ID: <XXX@yyy>
+MIME-Version: 1.0
+To: user@example.com
+Subject: Hi
+Content-Type: multipart/mixed; boundary=
+ "xxx"
+
+
+--xxx
+Content-Type: text/plain
+Content-Transfer-Encoding: 7bit
+
+Hi
+
+--xxx
+Content-Type: text/richtext
+Content-Description: eicar.zip
+Content-Transfer-Encoding: base64
+Content-Disposition: attachment; filename="eicar.zip"
+
+UEsDBAoAAAAAAOCYuCg8z1FoRAAAAEQAAAAJAAAAZWljYXIuY29tWDVPIVAlQEFQ
+WzRcUFpYNTQoUF4pN0NDKTd9JEVJQ0FSLVNUQU5EQVJELUFOVElWSVJVUy1URVNU
+LUZJTEUhJEgrSCpQSwECFAAKAAAAAADgmLgoPM9RaEQAAABEAAAACQAAAAAAAAAB
+ACAA/4EAAAAAZWljYXIuY29tUEsFBgAAAAABAAEANwAAAGsAAAAAAA==
+
+--xxx--
diff --git a/test/functional/messages/dmarc/pct_none.eml b/test/functional/messages/dmarc/pct_none.eml new file mode 100644 index 000000000..50fe33106 --- /dev/null +++ b/test/functional/messages/dmarc/pct_none.eml @@ -0,0 +1,17 @@ +Date: Tue, 09 Aug 2016 10:01:27 +0200 +Message-ID: <20160809100128@rspamd.tk> +From: Rspamd <foo@zero_pct.com> +To: foo@rspamd.tk +Subject: hello +Content-Type: text/plain; charset=utf-8; format=flowed; DelSp=Yes +MIME-Version: 1.0 +Content-Disposition: inline +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=rspamd.tk; s=testdkim; + t=1470729879; h=from:subject:date:message-id:to:mime-version:content-type; + bh=7HkRgYnNru3SR2EWfgWU8yhM0MOH6ZZrPoEIgNIh8wc=; + b=kTIV4jcgv9sWFh2JFrS/+PcNxiloituqjmHHqeJOTfa+/9C+Er8BjnMysTJyYVq36Gnv0OZDgLr3Yy4YP5Lzbt1M9ZdN5cJqO7yn1N7wyaGfkt++b09rIYBy5Dkk7OWyP3cDThqDzv8C9heSvqBSEsirFsbt3Wx2g/hWiJlnjew= + + +hello + + diff --git a/test/functional/messages/dmarc/pct_none1.eml b/test/functional/messages/dmarc/pct_none1.eml new file mode 100644 index 000000000..a0cd9fd6a --- /dev/null +++ b/test/functional/messages/dmarc/pct_none1.eml @@ -0,0 +1,17 @@ +Date: Tue, 09 Aug 2016 10:01:27 +0200 +Message-ID: <20160809100128@rspamd.tk> +From: Rspamd <foo@subdomain.zero_pct.com> +To: foo@rspamd.tk +Subject: hello +Content-Type: text/plain; charset=utf-8; format=flowed; DelSp=Yes +MIME-Version: 1.0 +Content-Disposition: inline +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=rspamd.tk; s=testdkim; + t=1470729879; h=from:subject:date:message-id:to:mime-version:content-type; + bh=7HkRgYnNru3SR2EWfgWU8yhM0MOH6ZZrPoEIgNIh8wc=; + b=kTIV4jcgv9sWFh2JFrS/+PcNxiloituqjmHHqeJOTfa+/9C+Er8BjnMysTJyYVq36Gnv0OZDgLr3Yy4YP5Lzbt1M9ZdN5cJqO7yn1N7wyaGfkt++b09rIYBy5Dkk7OWyP3cDThqDzv8C9heSvqBSEsirFsbt3Wx2g/hWiJlnjew= + + +hello + + diff --git a/test/functional/messages/fws_fn.eml b/test/functional/messages/fws_fn.eml new file mode 100644 index 000000000..67601bea5 --- /dev/null +++ b/test/functional/messages/fws_fn.eml @@ -0,0 +1,30 @@ +From: <admin@aaa.example.com> +To: <ragamuffin@bbb.example.com> +Subject: Test content type +Date: Tue, 17 Jul 2018 18:06:01 +0200 (CEST) +Message-ID: <20180717160601.B25511E37E7@aaa.example.com> +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="sd8f70as98f70as98f70as98f70as98df70as98df7as90f87as90d8f7" + +--sd8f70as98f70as98f70as98f70as98df70as98df7as90f87as90d8f7 +Content-Transfer-Encoding: quoted-printable +Content-Type: text/html; charset=utf-8 +Content-Disposition: inline + +<html> +<body style=3D"font-family:arial"> +Hello world +</body> +</html> + +--sd8f70as98f70as98f70as98f70as98df70as98df7as90f87as90d8f7 +Content-ID: <qrcode> +Content-Transfer-Encoding: base64 +Content-Type: image/jpeg +Content-Disposition: inline + +alskfhaslkfjhlaskfjhlaskjfhlaskjfhlaksjfhklasjfhklasdjfhlask +lksjdfhalskjfhklasjfhlaskfhlaskfhklasfhklasdjfhlkasdjfhklasd + +--sd8f70as98f70as98f70as98f70as98df70as98df7as90f87as90d8f7-- + diff --git a/test/functional/messages/fws_fp.eml b/test/functional/messages/fws_fp.eml new file mode 100644 index 000000000..2c669c9e4 --- /dev/null +++ b/test/functional/messages/fws_fp.eml @@ -0,0 +1,56 @@ +From: Sender <sender@example.com> +Mime-Version: 1.0 (1.0) +Date: Fri, 5 Oct 2018 19:56:40 -0400 +Message-Id: <E7015E63-E006-4E30-9313-851CB7F2424E> +To: Receiver <receiver@example.com> +X-Spam_report: Spam detection software, running on the system "www.kamailio.org", + has NOT identified this incoming email as spam. The original + message has been attached to this so you can view it or label + similar future email. If you have any questions, see + the administrator of that system for details. + + Content preview: Hereâ��s an old one from 5 years ago. The principles remain + the same. https://www.fredposner.com/1457/kamailio-behind-nat/ -- fred [...] + + + Content analysis details: (-1.9 points, 5.5 required) + + pts rule name description + ---- ---------------------- -------------------------------------------------- + -1.9 BAYES_00 BODY: Bayes spam probability is 0 to 1% + [score: 0.0000] + 0.0 HTML_MESSAGE BODY: HTML included in message + 0.0 MIME_QP_LONG_LINE RAW: Quoted-printable line longer than 76 chars +List-Id: yYY +Subject: XXX + +--===============1364639178== +Content-Type: multipart/alternative; + boundary=Apple-Mail-C9403573-708A-4D69-BF6C-F2730C368461 +Content-Transfer-Encoding: 7bit + + +--Apple-Mail-C9403573-708A-4D69-BF6C-F2730C368461 +Content-Type: text/plain; + charset=utf-8 +Content-Transfer-Encoding: quoted-printable + +xxx + +--Apple-Mail-C9403573-708A-4D69-BF6C-F2730C368461 +Content-Type: text/html; + charset=utf-8 +Content-Transfer-Encoding: quoted-printable + +yyy +--Apple-Mail-C9403573-708A-4D69-BF6C-F2730C368461-- + + +--===============1364639178== +Content-Type: text/plain; charset="utf-8" +MIME-Version: 1.0 +Content-Transfer-Encoding: base64 +Content-Disposition: inline + +zzzz +--===============1364639178==-- diff --git a/test/functional/messages/fws_tp.eml b/test/functional/messages/fws_tp.eml new file mode 100644 index 000000000..fbe2f901c --- /dev/null +++ b/test/functional/messages/fws_tp.eml @@ -0,0 +1,56 @@ +From: Sender <sender@example.com> +Mime-Version: 1.0 (1.0) +Date: Fri, 5 Oct 2018 19:56:40 -0400 +Message-Id: <E7015E63-E006-4E30-9313-851CB7F2424E> +To: Receiver <receiver@example.com> +X-Spam_report: Spam detection software, running on the system "www.kamailio.org", + has NOT identified this incoming email as spam. The original + message has been attached to this so you can view it or label + similar future email. If you have any questions, see + the administrator of that system for details. + + Content preview: Hereâ��s an old one from 5 years ago. The principles remain + the same. https://www.fredposner.com/1457/kamailio-behind-nat/ -- fred [...] + + + Content analysis details: (-1.9 points, 5.5 required) + + pts rule name description + ---- ---------------------- -------------------------------------------------- + -1.9 BAYES_00 BODY: Bayes spam probability is 0 to 1% + [score: 0.0000] + 0.0 HTML_MESSAGE BODY: HTML included in message + 0.0 MIME_QP_LONG_LINE RAW: Quoted-printable line longer than 76 chars +List-Id: yYY +Subject: XXX + +--===============1364639178== +Content-Type: multipart/alternative; + boundary=Apple-Mail-C9403573-708A-4D69-BF6C-F2730C368461 +Content-Transfer-Encoding: 7bit + + +--Apple-Mail-C9403573-708A-4D69-BF6C-F2730C368461 +Content-Type: text/plain; + charset=utf-8 +Content-Transfer-Encoding: quoted-printable + +xxx + +--Apple-Mail-C9403573-708A-4D69-BF6C-F2730C368461 +Content-Type: text/html; + charset=utf-8 +Content-Transfer-Encoding: quoted-printable + +yyy +--Apple-Mail-C9403573-708A-4D69-BF6C-F2730C368461-- + + +--===============1364639178== +Content-Type: text/plain; charset="utf-8" +MIME-Version: 1.0 +Content-Transfer-Encoding: base64 +Content-Disposition: inline + +zzzz +--===============1364639178==-- diff --git a/test/lua/unit/received.lua b/test/lua/unit/received.lua index 6c133279f..ac21c0e83 100644 --- a/test/lua/unit/received.lua +++ b/test/lua/unit/received.lua @@ -5,17 +5,18 @@ context("Received headers parser", function() ffi.cdef[[ struct received_header { - char *from_hostname; - char *from_ip; - char *real_hostname; - char *real_ip; - char *by_hostname; - char *for_mbox; - void *addr; - void *hdr; - long timestamp; - int type; - int flags; + char *from_hostname; + char *from_ip; + char *real_hostname; + char *real_ip; + char *by_hostname; + char *for_mbox; + char *comment_ip; + void *addr; + void *hdr; + long timestamp; + int type; + int flags; }; struct rspamd_task * rspamd_task_new(struct rspamd_worker *worker, struct rspamd_config *cfg); int rspamd_smtp_received_parse (struct rspamd_task *task, diff --git a/test/lua/unit/rsa.lua b/test/lua/unit/rsa.lua index 962077242..900c8a53f 100644 --- a/test/lua/unit/rsa.lua +++ b/test/lua/unit/rsa.lua @@ -12,7 +12,7 @@ context("RSA signature verification test", function() local signature = 'test.sig' local test_dir = string.gsub(debug.getinfo(1).source, "^@(.+/)[^/]+$", "%1") local rsa_key, rsa_sig - + test("RSA sign", function() -- Signing test rsa_key = rsa_privkey.load_file(string.format('%s/%s', test_dir, privkey)) @@ -25,7 +25,7 @@ context("RSA signature verification test", function() assert_not_nil(sig) sig:save(string.format('%s/%s', test_dir, signature), true) end) - + test("RSA verify", function() -- Verifying test local h = hash.create_specific('sha256') diff --git a/test/lua/unit/selectors.lua b/test/lua/unit/selectors.lua index 8053912c0..2c2ebfb5b 100644 --- a/test/lua/unit/selectors.lua +++ b/test/lua/unit/selectors.lua @@ -32,43 +32,43 @@ context("Selectors test", function() local cases = { ["ip"] = { - selector = "ip", + selector = "ip", expect = {"198.172.22.91"}}, ["header Subject"] = { - selector = "header(Subject)", + selector = "header(Subject)", expect = {"Second, lower-cased header subject"}}, ["header Subject lower"] = { - selector = "header(Subject).lower", + selector = "header(Subject).lower", expect = {"second, lower-cased header subject"}}, ["header full Subject lower"] = { - selector = "header(Subject, 'full').lower", + selector = "header(Subject, 'full').lower", expect = {{"second, lower-cased header subject", "test subject"}}}, ["header full strong Subject"] = { - selector = "header(Subject, 'full,strong')", + selector = "header(Subject, 'full,strong')", expect = {{"Test subject"}}}, ["header full strong lower-cased Subject"] = { - selector = "header(subject, 'full,strong')", + selector = "header(subject, 'full,strong')", expect = {{"Second, lower-cased header subject"}}}, ["digest"] = { - selector = "digest", - expect = {"2216397bc061bb6968e1836f3680fed0"}}, + selector = "digest", + expect = {"c459a21bd1f33fb4ba035481f46ef0c7"}}, ["user"] = { - selector = "user", + selector = "user", expect = {"cool user name"}}, ["from"] = { - selector = "from", + selector = "from", expect = {"whoknows@nowhere.com"}}, ["rcpts"] = { - selector = "rcpts", + selector = "rcpts", expect = {{"nobody@example.com", "no-one@example.com"}}}, ["1st rcpts"] = { @@ -80,7 +80,7 @@ context("Selectors test", function() expect = {"nobody@example.com"}}, ["first rcpts"] = { - selector = "rcpts.first", + selector = "rcpts.first", expect = {"nobody@example.com"}}, ["first addr rcpts"] = { @@ -88,7 +88,7 @@ context("Selectors test", function() expect = {"nobody@example.com"}}, ["to"] = { - selector = "to", + selector = "to", expect = {"nobody@example.com"}}, ["attachments"] = { @@ -106,7 +106,7 @@ context("Selectors test", function() expect = {{"f.zip", "f2.zip"}}}, ["helo"] = { - selector = "helo", + selector = "helo", expect = {"hello mail"}}, ["received by hostname"] = { @@ -114,7 +114,7 @@ context("Selectors test", function() expect = {{"server.chat-met-vreemden.nl"}}}, ["urls"] = { - selector = "urls", + selector = "urls", expect = {{"http://example.net"}}}, ["emails"] = { @@ -134,11 +134,11 @@ context("Selectors test", function() expect = {"1"}}, ["time"] = { - selector = "time", + selector = "time", expect = {"1537364211"}}, ["request_header"] = { - selector = "request_header(hdr1)", + selector = "request_header(hdr1)", expect = {"value1"}}, ["get_host"] = { diff --git a/test/lua/unit/test.data b/test/lua/unit/test.data index 05ec159ee..696972ef4 100644 --- a/test/lua/unit/test.data +++ b/test/lua/unit/test.data @@ -1,19 +1,10 @@ -Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas et adipiscing nunc, non auctor justo. Nunc consectetur nunc eget nibh pulvinar, quis consequat tortor mollis. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. In eu lectus in mi sodales fermentum. Pellentesque id sapien in augue sodales porta laoreet vel odio. Nam semper massa turpis, sit amet tempus nulla scelerisque non. Sed porttitor sapien est, vitae imperdiet nulla blandit vel. - -Proin venenatis neque urna, sit amet elementum risus accumsan in. Nam lectus augue, porttitor a accumsan porta, dignissim ut magna. Cras eleifend orci at eros porttitor posuere. Vestibulum semper accumsan purus, vel varius nisl porta sed. Duis id consectetur ligula, id vestibulum erat. Ut ultricies viverra ipsum sagittis semper. Proin cursus ante at justo pharetra placerat. Donec ultrices eros orci, sed congue libero mattis nec. Ut fringilla quis mauris non condimentum. Proin ante risus, imperdiet et elit vitae, aliquet egestas tortor. - -Curabitur tempus, lacus in vehicula pulvinar, odio nisl porta sem, sit amet aliquet metus tellus rhoncus nulla. Morbi condimentum enim vitae lorem imperdiet, ac feugiat nisl suscipit. Proin eleifend laoreet tortor. Sed id risus elementum, sollicitudin nibh et, luctus augue. Cras consequat nec diam in volutpat. Nam tempor nisi nec risus tempus varius quis eget mi. Nunc mauris dolor, imperdiet eget nisi ut, aliquet vulputate massa. Sed consequat diam vitae malesuada vulputate. - -Duis ac mollis ante, et suscipit eros. Aenean vel risus nec nisi lacinia aliquam. Phasellus auctor hendrerit facilisis. Etiam at dolor vestibulum, varius quam tempus, adipiscing dolor. Vestibulum erat dui, semper eget orci eget, iaculis laoreet ipsum. Nullam id quam at odio pharetra vestibulum. Morbi porttitor sagittis nunc sed pellentesque. Quisque semper orci ac metus molestie egestas. - -Donec mattis commodo magna sit amet pellentesque. In et varius urna. Suspendisse eget tellus nulla. Suspendisse potenti. Praesent a ante sit amet tellus pellentesque ornare id sit amet eros. Donec sed dignissim ligula, nec condimentum nulla. Etiam fermentum leo sit amet tincidunt fringilla. In semper lacus sit amet laoreet volutpat. Curabitur ultricies sem in dui eleifend, at aliquam sapien adipiscing. Aliquam erat volutpat. Maecenas felis eros, posuere quis placerat a, imperdiet ac lectus. Phasellus volutpat lorem sed nulla vehicula pulvinar. Pellentesque ipsum ante, posuere id eros non, vehicula commodo augue. Proin vitae tempor lectus, at commodo elit. Donec luctus commodo erat. Curabitur faucibus, arcu tincidunt vehicula rutrum, quam turpis hendrerit sapien, nec vehicula est est ut turpis. - -Suspendisse elementum ligula nibh. Nullam interdum tempor metus sed ultrices. Proin nec venenatis tellus. Donec non tortor at ipsum ultrices sagittis. Integer lectus augue, lacinia quis ante iaculis, iaculis volutpat odio. Fusce porttitor quis nisi vitae condimentum. Nam luctus placerat fringilla. Praesent vulputate aliquet convallis. Nunc vel sagittis sem. Vestibulum vitae ante a ipsum pulvinar faucibus pulvinar ac nulla. Morbi mattis nulla et dui egestas, vitae volutpat nisi pretium. Ut ullamcorper ac orci nec lacinia. Proin ac condimentum diam. Nulla blandit, tellus sit amet volutpat suscipit, purus eros iaculis odio, in euismod nulla nisl at urna. Aliquam erat volutpat. Pellentesque hendrerit dui eget lobortis congue. - -Etiam condimentum adipiscing sollicitudin. Curabitur at rhoncus sem. Donec dignissim, erat sodales malesuada fringilla, metus est vehicula nulla, nec volutpat lacus neque ut libero. Suspendisse potenti. Nam vel facilisis massa. Morbi eget quam urna. Suspendisse quis lacus eget justo placerat pulvinar sit amet a nunc. Cras porta magna nec risus feugiat, pharetra rhoncus mauris convallis. Integer pretium dolor massa, a consectetur velit laoreet sed. Nunc rutrum convallis orci, sit amet congue sapien iaculis at. - -Etiam leo nibh, ultricies eget dictum non, euismod nec quam. Cras luctus mauris odio, in dapibus risus rhoncus et. Integer ullamcorper elit neque. Nam vel nisl sit amet nunc gravida laoreet. Suspendisse sed neque id nulla tempus laoreet. Aliquam interdum lorem quis erat lacinia, in congue velit euismod. Morbi sit amet viverra dolor, lacinia pretium massa. Proin eu rutrum augue, sit amet condimentum enim. Pellentesque sodales, justo quis gravida lacinia, nulla dolor accumsan sapien, non hendrerit urna dui quis nibh. In posuere non elit eu hendrerit. - -Nulla eget justo ultrices tellus tristique cursus. Maecenas in sem et tellus pulvinar rhoncus quis ac urna. Vestibulum imperdiet nisl nibh, at suscipit mi varius eu. Vestibulum adipiscing tellus sed ipsum vehicula, vitae convallis felis ornare. Sed commodo varius adipiscing. Pellentesque eget magna diam. Etiam ac ante ligula. Proin mattis quis odio id placerat. Nam luctus, sapien rhoncus iaculis elementum, diam magna condimentum velit, eu pellentesque orci tortor eget dui. Fusce adipiscing feugiat nunc, ac iaculis lorem ultrices sit amet. Proin vel felis eget nulla consectetur congue vel sed sapien. Morbi sed aliquam neque. Interdum et malesuada fames ac ante ipsum primis in faucibus. In volutpat velit ut est sodales, eget porttitor massa faucibus. - -Nunc vel fermentum augue, vitae convallis metus. Pellentesque accumsan ipsum nec condimentum pharetra. Quisque id nisi egestas, ultrices mi nec, faucibus orci. Aenean aliquam, ipsum lobortis tristique facilisis, neque purus commodo est, semper semper nunc metus quis ligula. Praesent faucibus est vel quam eleifend, a accumsan diam tempus. Integer pellentesque rhoncus vehicula. Donec sit amet mollis sem. Vivamus et ipsum at justo iaculis imperdiet a vel felis. Morbi felis augue, condimentum sit amet ligula et, mollis adipiscing purus. In vulputate nibh nec mattis fermentum. In ornare felis in aliquam cursus. Suspendisse magna mi, vehicula ut egestas eu, egestas non eros. Nunc non sem massa. Pellentesque mattis, lacus sit amet luctus scelerisque, lacus est ultricies erat, ut tincidunt nisl orci id leo. Maecenas ut massa volutpat, aliquet odio placerat, luctus massa. Ut sed odio tincidunt, luctus arcu in, dictum leo. +RLvXs8ZWOYXidwy4RSErSJFAGiRhimvMhHNIOzbxkkDC1IQz03tf9jvglA45PXAb +AyYIMAlMn1DrRCwsGKV/u8EEjkO34ujwirJ6ytbiZkjTnANBhGtZdjMCfsEUIY9a +y35d3CeKZF9KaRdlWRDJdfBbZE9mn4rSUQ1X0+HweUZ3AmMHwWLa9SB+ii7ysEEl ++6QLqHczu7K0Ji3LVKI+NzPJOWmWWCHjJyhs8HsuHpUrJ3iSeLxfW0TD8x6eZ52C +EWC0BbR32vtquw8r5O+yR6hbBUJj8mTqTs3yAaTEs8Q+7y5uFuGsv+0NrmEOASyT +NvGaxODKLO1A/8kXXsko3I3hZOoi+9GG/eAncMRWtdwllE/KqZfp9uzi5aYh1MMb +px4SFqH5FQfvveZwGgEl9+BCkRQIptqv1fMlWouy35n5AeHkfflyyA4wC6iwgJAL +R5R95Y8y2UPWoRkB+HFvoEryCNrkdC1QmW07n5shHO9NzNk34tQIzfjvYwcPi2yy +3e/YNr3jyKOs86jTK6z9M/4htai/OxuF34rGS9pau/NINrDOpCNNy4zDgsQkvm5l +H4CzhH5tNvYaog== diff --git a/test/lua/unit/utf.lua b/test/lua/unit/utf.lua index e22eb2a2f..277d99e41 100644 --- a/test/lua/unit/utf.lua +++ b/test/lua/unit/utf.lua @@ -5,36 +5,68 @@ context("UTF8 check functions", function() ffi.cdef[[ void rspamd_str_lc_utf8 (char *str, unsigned int size); void rspamd_str_lc (char *str, unsigned int size); + char * rspamd_str_make_utf_valid (const char *src, size_t slen, size_t *dstlen); ]] - test("UTF lowercase", function() - local cases = { - {"АбЫрвАлг", "абырвалг"}, - {"АAБBвc", "аaбbвc"} - } - - for _,c in ipairs(cases) do + local cases = { + {"АбЫрвАлг", "абырвалг"}, + {"АAБBвc", "аaбbвc"}, + --{"STRASSE", "straße"}, XXX: NYI + {"KEÇİ", "keçi"}, + } + + for i,c in ipairs(cases) do + test("UTF lowercase " .. tostring(i), function() local buf = ffi.new("char[?]", #c[1] + 1) ffi.copy(buf, c[1]) ffi.C.rspamd_str_lc_utf8(buf, #c[1]) local s = ffi.string(buf) assert_equal(s, c[2]) - end - end) - test("ASCII lowercase", function() - local cases = { - {"AbCdEf", "abcdef"}, - {"A", "a"}, - {"AaAa", "aaaa"}, - {"AaAaAaAa", "aaaaaaaa"} - } - - for _,c in ipairs(cases) do + end) + end + + cases = { + {"AbCdEf", "abcdef"}, + {"A", "a"}, + {"AaAa", "aaaa"}, + {"AaAaAaAa", "aaaaaaaa"} + } + + for i,c in ipairs(cases) do + test("ASCII lowercase " .. tostring(i), function() local buf = ffi.new("char[?]", #c[1] + 1) ffi.copy(buf, c[1]) ffi.C.rspamd_str_lc(buf, #c[1]) local s = ffi.string(buf) assert_equal(s, c[2]) - end - end) + end) + end + + cases = { + {'тест', 'тест'}, + {'\200\213\202', '���'}, + {'тест\200\213\202test', 'тест���test'}, + {'\200\213\202test', '���test'}, + {'\200\213\202test\200\213\202', '���test���'}, + {'тест\200\213\202test\200\213\202', 'тест���test���'}, + {'тест\200\213\202test\200\213\202тест', 'тест���test���тест'}, + } + + local NULL = ffi.new 'void*' + for i,c in ipairs(cases) do + test("Unicode make valid " .. tostring(i), function() + local buf = ffi.new("char[?]", #c[1] + 1) + ffi.copy(buf, c[1]) + + local s = ffi.string(ffi.C.rspamd_str_make_utf_valid(buf, #c[1], NULL)) + local function to_hex(s) + return (s:gsub('.', function (c) + return string.format('%02X', string.byte(c)) + end)) + end + print(to_hex(s)) + print(to_hex(c[2])) + assert_equal(s, c[2]) + end) + end end)
\ No newline at end of file diff --git a/test/rspamd_lua_pcall_vs_resume_test.c b/test/rspamd_lua_pcall_vs_resume_test.c index 95cf77cdf..8252dffb0 100644 --- a/test/rspamd_lua_pcall_vs_resume_test.c +++ b/test/rspamd_lua_pcall_vs_resume_test.c @@ -53,7 +53,11 @@ test_resume(lua_State *L, gint function_call) for (i = 0; i < N; i ++) { lua_rawgeti (L, LUA_REGISTRYINDEX, function_call); +#if LUA_VERSION_NUM < 503 lua_resume (L, 0); +#else + lua_resume (L, NULL, 0); +#endif lua_pop (L, 1); } @@ -75,7 +79,11 @@ test_resume_get_thread(gint function_call) ent = lua_thread_pool_get_for_config (rspamd_main->cfg); lua_rawgeti (ent->lua_state, LUA_REGISTRYINDEX, function_call); +#if LUA_VERSION_NUM < 503 lua_resume (ent->lua_state, 0); +#else + lua_resume (ent->lua_state, NULL, 0); +#endif lua_pop (ent->lua_state, 1); lua_thread_pool_return (rspamd_main->cfg->lua_thread_pool, ent); @@ -99,7 +107,11 @@ test_resume_get_new_thread(gint function_call) ent = lua_thread_pool_get_for_task (rspamd_main->cfg->lua_thread_pool); lua_rawgeti (ent->lua_state, LUA_REGISTRYINDEX, function_call); +#if LUA_VERSION_NUM < 503 lua_resume (ent->lua_state, 0); +#else + lua_resume (ent->lua_state, NULL, 0); +#endif lua_pop (ent->lua_state, 1); /* lua_thread_pool_return (rspamd_main->cfg->lua_thread_pool, ent); */ diff --git a/test/rspamd_test_suite.c b/test/rspamd_test_suite.c index 10a5d941b..6ca8d6465 100644 --- a/test/rspamd_test_suite.c +++ b/test/rspamd_test_suite.c @@ -1,6 +1,7 @@ #include "config.h" #include "rspamd.h" #include "libstat/stat_api.h" +#include "lua/lua_common.h" #include "tests.h" struct rspamd_main *rspamd_main = NULL; @@ -31,6 +32,7 @@ main (int argc, char **argv) memset (rspamd_main, 0, sizeof (struct rspamd_main)); rspamd_main->server_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL); cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_DEFAULT); + cfg->libs_ctx = rspamd_init_libs (); rspamd_main->cfg = cfg; cfg->cfg_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL); cfg->log_type = RSPAMD_LOG_CONSOLE; @@ -51,8 +53,7 @@ main (int argc, char **argv) exit (1); } - cfg->libs_ctx = rspamd_init_libs (); - + rspamd_lua_set_path ((lua_State *)cfg->lua_state, NULL, NULL); base = event_init (); rspamd_stat_init (cfg, base); rspamd_url_init (NULL); diff --git a/test/tools/dump_coveralls.py b/test/tools/dump_coveralls.py new file mode 100755 index 000000000..c453d0511 --- /dev/null +++ b/test/tools/dump_coveralls.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +# Small tool to dump JSON payload for coveralls.io API + +import json +from operator import itemgetter +import os +import sys + + +def warn(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + +def dump_file(json_file): + """Dumps coveralls.io API payload stored in json_file + Returns: 0 if successful, 1 otherwise + """ + try: + with open(json_file, encoding='utf8') as f: + data = json.load(f) + except OSError as err: + warn(err) + return os.EX_DATAERR + except json.decoder.JSONDecodeError: + warn("{}: json parsing error".format(json_file)) + return 1 + + if 'source_files' not in data: + warn("{}: no source_files, not a coveralls.io payload?".format(json_file)) + return 1 + + print("{} ({} soource files)".format(json_file, len(data['source_files']))) + + for src_file in sorted(data['source_files'], key=itemgetter('name')): + covered_lines = not_skipped_lines = 0 + for cnt in src_file['coverage']: + if cnt is None: + continue + not_skipped_lines += 1 + if cnt > 0: + covered_lines += 1 + if not_skipped_lines > 0: + coverage = "{:.0%}".format(covered_lines / not_skipped_lines) + else: + coverage = 'N/A' + + print("\t{:>3} {}".format(coverage, src_file['name'])) + + return 0 + + +def main(): + if (len(sys.argv) < 2): + warn("usage: {} file.json ...".format(sys.argv[0])) + return os.EX_USAGE + + exit_status = 0 + for f in sys.argv[1:]: + exit_status += dump_file(f) + + return exit_status + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/test/tools/gcov_coveralls.py b/test/tools/gcov_coveralls.py new file mode 100755 index 000000000..71aa48b7b --- /dev/null +++ b/test/tools/gcov_coveralls.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +""" +Script to save coverage info for C source files in JSON for coveralls.io + +When C code compiled with --coverage flag, for each object files *.gcno is +generated, it contains information to reconstruct the basic block graphs and +assign source line numbers to blocks + +When binary executed *.gcda file is written on exit, with same base name as +corresponding *.gcno file. It contains some summary information, counters, e.t.c. + +gcov(1) utility can be used to get information from *.gcda file and write text +reports to *.gocov file (one file for each source file from which object was compiled). + +The script finds *.gcno files, uses gcov to generate *.gcov files, parses them +and accomulates statistics for all source files. + +This script was written with quite a few assumptions: + + * Code was build using absolute path to source directory (and absolute path + stored in object file debug sylmbols). + + * Current directory is writable and there is no useful *.gcov files in it + (becase they will be deleted). + + * Object file has same base name as *.gcno file (e. g. foo.c.gcno and foo.c.o). + This is the case for cmake builds, but probably not for other build systems + + * Source file names contain only ASCII characters. +""" + +import argparse +from collections import defaultdict +from glob import glob +import hashlib +import json +import os +from os.path import isabs, join, normpath, relpath +import os.path +import subprocess +import sys + + +def warn(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + +def parse_gcov_file(gcov_file): + """Parses the content of .gcov file written by gcov --intermediate-format + + Returns: + str: Source file name + dict: coverage info { line_number: hits } + """ + count = {} + with open(gcov_file) as fh: + for line in fh: + tag, value = line.split(':') + if tag == 'file': + src_file = value.rstrip() + elif tag == 'lcount': + line_num, exec_count = value.split(',') + count[int(line_num)] = int(exec_count) + + return src_file, count + + +def run_gcov(filename, coverage, args): + """ * run gcov on given file + * parse generated .gcov files and update coverage structure + * store source file md5 (if not yet stored) + * delete .gcov files + """ + if args.verbose: + warn("calling:", 'gcov', '--intermediate-format', filename) + stdout = None + else: + # gcov is noisy and don't have quit flag so redirect stdout to /dev/null + stdout = subprocess.DEVNULL + + subprocess.check_call(['gcov', '--intermediate-format', filename], stdout=stdout) + + for gcov_file in glob('*.gcov'): + if args.verbose: + warn('parsing', gcov_file) + src_file, count = parse_gcov_file(gcov_file) + os.remove(gcov_file) + + if src_file not in coverage: + coverage[src_file] = defaultdict(int, count) + else: + # sum execution counts + for line, exe_cnt in count.items(): + coverage[src_file][line] += exe_cnt + + +def main(): + parser = argparse.ArgumentParser( + description='Save gcov coverage results in JSON file for coveralls.io.') + parser.add_argument( + '-v', + '--verbose', + action="store_true", + help='Display additional informaton and gcov command output.') + parser.add_argument( + '-e', + '--exclude', + action='append', + metavar='DIR', + help= + ("Don't look for .gcno/.gcda files in this directories (repeat option to skip several directories). " + "Path is relative to the dirictory where script was started, e. g. '.git'")) + parser.add_argument( + '-p', + '--prefix', + action='append', + help= + ("Strip this prefix from absolute path to source file. " + "If this option is provided, then only files with given prefixex in absolute path " + "will be added to coverage (option can be repeated).")) + parser.add_argument( + '--out', + type=argparse.FileType('w'), + required=True, + metavar='FILE', + help='Save JSON payload to this file') + args = parser.parse_args() + + # ensure that there is no unrelated .gcov files in current directory + for gcov_file in glob('*.gcov'): + os.remove(gcov_file) + warn("Warning: {} deleted".format(gcov_file)) + + # dict { src_file_name: {line1: exec_count1, line2: exec_count2, ...} } + coverage = {} + + # find . -name '*.gcno' (respecting args.exclude) + for root, dirs, files in os.walk('.'): + for f in files: + # Usually gcov called with a source file as an argument, but this + # name used only to find .gcno and .gcda files. To find source + # file information from debug symbols is used. So we can call gcov + # on .gcno file. + if f.endswith('.gcno'): + run_gcov(join(root, f), coverage, args) + + # don't look into excluded dirs + for subdir in dirs: + # path relative to start dir + path = normpath(join(root, subdir)) + if path in args.exclude: + if args.verbose: + warn('directory "{}" excluded'.format(path)) + dirs.remove(subdir) + + # prepare JSON pyload for coveralls.io API + # https://docs.coveralls.io/api-introduction + coveralls_data = {'source_files': []} + + for src_file in coverage: + # filter by prefix and save path with stripped prefix + src_file_rel = src_file + if args.prefix and isabs(src_file): + for prefix in args.prefix: + if src_file.startswith(prefix): + src_file_rel = relpath(src_file, start=prefix) + break + else: + # skip file outside given prefixes + # it can be e. g. library include file + if args.verbose: + warn('file "{}" is not mathced by prefix, skipping'.format(src_file)) + continue + + try: + with open(src_file, mode='rb') as fh: + line_count = sum(1 for _ in fh) + fh.seek(0) + md5 = hashlib.md5(fh.read()).hexdigest() + except OSError as err: + # skip files for which source file is not available + warn(err, 'not adding to coverage') + continue + + coverage_array = [None] * line_count + + for line_num, exe_cnt in coverage[src_file].items(): + # item at index 0 representing the coverage for line 1 of the source code + assert 1 <= line_num <= line_count + coverage_array[line_num - 1] = exe_cnt + + coveralls_data['source_files'].append({ + 'name': src_file_rel, + 'coverage': coverage_array, + 'source_digest': md5 + }) + + args.out.write(json.dumps(coveralls_data)) + + if args.verbose: + warn('Coverage for {} source files was written'.format( + len(coveralls_data['source_files']))) + + +if __name__ == '__main__': + main() diff --git a/test/tools/http_put.py b/test/tools/http_put.py new file mode 100755 index 000000000..de2dad3d7 --- /dev/null +++ b/test/tools/http_put.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +""" +Small script to upload file using HTTP PUT +""" + +import argparse +import os +import sys + +import requests + + +def main(): + parser = argparse.ArgumentParser( + description='Upload a file usgin HTTP PUT method', + epilog=( + "To use HTTP Auth set HTTP_PUT_AUTH environment variable to user:password\n" + "Example: %(prog)s file1 file2 https://example.com/dir/")) + parser.add_argument( + "file", type=argparse.FileType('rb'), nargs='+', help="File to upload") + parser.add_argument( + "dir_url", help="Remote URL (path to a directory, must include a trailing /)") + args = parser.parse_args() + + if not args.dir_url.endswith('/'): + parser.error("URL must end with /") + + http_auth = os.getenv('HTTP_PUT_AUTH') + if http_auth: + user, password = http_auth.split(':') + auth = (user, password) + else: + auth = None + + exit_code = 0 + + for fh in args.file: + try: + r = requests.put(args.dir_url + fh.name, data=fh, auth=auth, timeout=(45, 90)) + r.raise_for_status() + print("{} uploaded to {}".format(fh.name, r.url)) + except (requests.exceptions.ConnectionError, + requests.exceptions.HTTPError) as err: + print(err, file=sys.stderr) + exit_code = 1 + + return exit_code + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/test/functional/util/merge_coveralls.py b/test/tools/merge_coveralls.py index e1498ac01..8fad0f55b 100755 --- a/test/functional/util/merge_coveralls.py +++ b/test/tools/merge_coveralls.py @@ -37,12 +37,13 @@ path_mapping = [ ] parser = argparse.ArgumentParser(description='') -parser.add_argument('--input', type=str, required=True, nargs='+', help='input files') -parser.add_argument('--output', type=str, required=True, help='output file)') -parser.add_argument('--root', type=str, required=False, default="/rspamd/src/github.com/rspamd/rspamd", help='repository root)') -parser.add_argument('--install-dir', type=str, required=False, default="/rspamd/install", help='install root)') -parser.add_argument('--build-dir', type=str, required=False, default="/rspamd/build", help='build root)') -parser.add_argument('--token', type=str, help='If present, the file will be uploaded to coveralls)') +parser.add_argument('--input', required=True, nargs='+', help='input files') +parser.add_argument('--output', help='output file)') +parser.add_argument('--root', default="/rspamd/src/github.com/rspamd/rspamd", help='repository root)') +parser.add_argument('--install-dir', default="/rspamd/install", help='install root)') +parser.add_argument('--build-dir', default="/rspamd/build", help='build root)') +parser.add_argument('--token', help='If present, the file will be uploaded to coveralls)') + def merge_coverage_vectors(c1, c2): assert(len(c1) == len(c2)) @@ -63,7 +64,6 @@ def merge_coverage_vectors(c1, c2): def normalize_name(name): - orig_name = name name = os.path.normpath(name) if not os.path.isabs(name): name = os.path.abspath(repository_root + "/" + name) @@ -86,11 +86,6 @@ def merge(files, j1): else: sf['name'] = name files[name] = sf - if not ('source' in sf): - path = "%s/%s" % (repository_root, sf['name']) - if os.path.isfile(path): - with open(path) as f: - files[name]['source'] = f.read() return files @@ -128,29 +123,49 @@ if __name__ == '__main__': if 'service_job_id' not in j1 and 'service_job_id' in j2: j1['service_job_id'] = j2['service_job_id'] - if not j1['service_job_id'] and 'CIRCLE_BUILD_NUM' in os.environ: - j1['service_job_id'] = os.environ['CIRCLE_BUILD_NUM'] - elif not j1['service_job_id'] and 'DRONE_PREV_BUILD_NUMBER' in os.environ: - j1['service_job_id'] = os.environ['DRONE_PREV_BUILD_NUMBER'] - if 'CIRCLECI' in os.environ and os.environ['CIRCLECI']: + if os.getenv('CIRCLECI'): j1['service_name'] = 'circleci' - elif 'DRONE' in os.environ and os.environ['DRONE']: + j1['service_job_id'] = os.getenv('CIRCLE_BUILD_NUM') + elif os.getenv('CI') == 'drone': j1['service_name'] = 'drone' + j1['service_branch'] = os.getenv('CI_COMMIT_BRANCH') + j1['service_build_url'] = os.getenv('DRONE_BUILD_LINK') + j1['service_job_id'] = os.getenv('CI_JOB_NUMBER') + j1['service_number'] = os.getenv('CI_BUILD_NUMBER') + j1['commit_sha'] = os.getenv('CI_COMMIT_SHA') + if os.getenv('CI_BUILD_EVENT') == 'pull_request': + j1['service_pull_request'] = os.getenv('CI_PULL_REQUEST') + # git data can be filled by cpp-coveralls, but in our layout it can't find repo + # so we can override git info witout merging + j1['git'] = { + 'head': { + 'id': j1['commit_sha'], + 'author_email': os.getenv('CI_COMMIT_AUTHOR_EMAIL'), + 'message': os.getenv('CI_COMMIT_MESSAGE') + }, + 'branch': j1['service_branch'], + 'remotes': [{ + 'name': 'origin', + 'url': os.getenv('CI_REPO_REMOTE') + }] + } + j1['source_files'] = list(files.values()) - with open(args.output, 'w') as f: - f.write(json.dumps(j1)) + if args.output: + with open(args.output, 'w') as f: + f.write(json.dumps(j1)) if args.token: j1['repo_token'] = args.token print("sending data to coveralls...") r = requests.post('https://coveralls.io/api/v1/jobs', files={"json_file": json.dumps(j1)}) - response = json.loads(r.text) - print("uploaded %s\nmessage:%s" % (response['url'], response['message'])) + response = r.json() + print("[coveralls] %s" % response['message']) + if 'url' in response: + print("[coveralls] Uploaded to %s" % response['url']) # post https://coveralls.io/api/v1/jobs # print args - - diff --git a/utils/asn.pl b/utils/asn.pl index 11bb6746b..b5f2ca41e 100644 --- a/utils/asn.pl +++ b/utils/asn.pl @@ -16,14 +16,14 @@ $LWP::Simple::ua->show_progress(1); $Net::MRT::USE_RFC4760 = -1; my %config = ( - asn_sources => [ - 'ftp://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest', - 'ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest', - 'ftp://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest', - 'ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest', - 'ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest' - ], - bgp_sources => ['http://data.ris.ripe.net/rrc00/latest-bview.gz'] + asn_sources => [ + 'ftp://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest', + 'ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest', + 'ftp://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest', + 'ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest', + 'ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest' + ], + bgp_sources => ['http://data.ris.ripe.net/rrc00/latest-bview.gz'] ); my $download_asn = 0; @@ -38,171 +38,169 @@ my $v4_zone = "asn.rspamd.com"; my $v6_zone = "asn6.rspamd.com"; my $v4_file = "asn.zone"; my $v6_file = "asn6.zone"; -my $ns_servers = ["asn-ns.rspamd.com", "asn-ns2.rspamd.com"]; +my $ns_servers = [ "asn-ns.rspamd.com", "asn-ns2.rspamd.com" ]; GetOptions( - "download-asn" => \$download_asn, - "download-bgp" => \$download_bgp, - "4!" => \$v4, - "6!" => \$v6, - "parse!" => \$parse, - "target=s" => \$download_target, - "zone-v4=s" => \$v4_zone, - "zone-v6=s" => \$v6_zone, - "file-v4=s" => \$v4_file, - "file-v6=s" => \$v6_file, - "ns-server=s@" => \$ns_servers, - "help|?" => \$help, - "man" => \$man + "download-asn" => \$download_asn, + "download-bgp" => \$download_bgp, + "4!" => \$v4, + "6!" => \$v6, + "parse!" => \$parse, + "target=s" => \$download_target, + "zone-v4=s" => \$v4_zone, + "zone-v6=s" => \$v6_zone, + "file-v4=s" => \$v4_file, + "file-v6=s" => \$v6_file, + "ns-server=s@" => \$ns_servers, + "help|?" => \$help, + "man" => \$man ) or pod2usage(2); pod2usage(1) if $help; pod2usage( -exitval => 0, -verbose => 2 ) if $man; sub download_file { - my ($u) = @_; + my ($u) = @_; - print "Fetching $u\n"; - my $ff = File::Fetch->new( uri => $u ); - my $where = $ff->fetch( to => $download_target ) or die $ff->error; + print "Fetching $u\n"; + my $ff = File::Fetch->new( uri => $u ); + my $where = $ff->fetch( to => $download_target ) or die $ff->error; - return $where; + return $where; } if ($download_asn) { - foreach my $u ( @{ $config{'asn_sources'} } ) { - download_file($u); - } + foreach my $u ( @{ $config{'asn_sources'} } ) { + download_file($u); + } } if ($download_bgp) { - foreach my $u ( @{ $config{'bgp_sources'} } ) { - download_file($u); - } + foreach my $u ( @{ $config{'bgp_sources'} } ) { + download_file($u); + } } if ( !$parse ) { - exit 0; + exit 0; } my $v4_fh; my $v6_fh; if ($v4) { - open( $v4_fh, ">", $v4_file ) or die "Cannot open $v4_file for writing: $!"; - print $v4_fh - "\$SOA 43200 $ns_servers->[0] support.rspamd.com 0 600 300 86400 300\n"; - foreach my $ns (@{$ns_servers}) { - print $v4_fh "\$NS 43200 $ns\n"; - } + open( $v4_fh, ">", $v4_file ) or die "Cannot open $v4_file for writing: $!"; + print $v4_fh "\$SOA 43200 $ns_servers->[0] support.rspamd.com 0 600 300 86400 300\n"; + foreach my $ns ( @{$ns_servers} ) { + print $v4_fh "\$NS 43200 $ns\n"; + } } if ($v6) { - open( $v6_fh, ">", $v6_file ) or die "Cannot open $v6_file for writing: $!"; - print $v6_fh - "\$SOA 43200 $ns_servers->[0] support.rspamd.com 0 600 300 86400 300\n"; - foreach my $ns (@{$ns_servers}) { - print $v6_fh "\$NS 43200 $ns\n"; - } + open( $v6_fh, ">", $v6_file ) or die "Cannot open $v6_file for writing: $!"; + print $v6_fh "\$SOA 43200 $ns_servers->[0] support.rspamd.com 0 600 300 86400 300\n"; + foreach my $ns ( @{$ns_servers} ) { + print $v6_fh "\$NS 43200 $ns\n"; + } } # Now load BGP data my $networks = {}; foreach my $u ( @{ $config{'bgp_sources'} } ) { - my $parsed = URI->new($u); - my $fname = $download_target . '/' . basename( $parsed->path ); - open( my $fh, "<:gzip", $fname ) - or die "Cannot open $fname: $!"; - - while ( my $dd = eval { Net::MRT::mrt_read_next($fh) } ) { - if ( $dd->{'prefix'} && $dd->{'bits'} ) { - next if $dd->{'subtype'} == 2 and !$v4; - next if $dd->{'subtype'} == 4 and !$v6; - my $entry = $dd->{'entries'}->[0]; - my $net = $dd->{'prefix'} . '/' . $dd->{'bits'}; - if ( $entry && $entry->{'AS_PATH'} ) { - my $as = $entry->{'AS_PATH'}->[-1]; - if (ref($as) eq "ARRAY") { - $as = @{$as}[0]; + my $parsed = URI->new($u); + my $fname = $download_target . '/' . basename( $parsed->path ); + open( my $fh, "<:gzip", $fname ) + or die "Cannot open $fname: $!"; + + while ( my $dd = eval { Net::MRT::mrt_read_next($fh) } ) { + if ( $dd->{'prefix'} && $dd->{'bits'} ) { + next if $dd->{'subtype'} == 2 and !$v4; + next if $dd->{'subtype'} == 4 and !$v6; + my $entry = $dd->{'entries'}->[0]; + my $net = $dd->{'prefix'} . '/' . $dd->{'bits'}; + if ( $entry && $entry->{'AS_PATH'} ) { + my $as = $entry->{'AS_PATH'}->[-1]; + if ( ref($as) eq "ARRAY" ) { + $as = @{$as}[0]; + } + + if ( !$networks->{$as} ) { + if ( $dd->{'subtype'} == 2 ) { + $networks->{$as} = { nets_v4 => [$net], nets_v6 => [] }; + } + else { + $networks->{$as} = { nets_v6 => [$net], nets_v4 => [] }; + } + } + else { + if ( $dd->{'subtype'} == 2 ) { + push @{ $networks->{$as}->{'nets_v4'} }, $net; + } + else { + push @{ $networks->{$as}->{'nets_v6'} }, $net; + } + } + } } - - if ( !$networks->{$as} ) { - if ( $dd->{'subtype'} == 2 ) { - $networks->{$as} = { nets_v4 => [$net], nets_v6 => [] }; - } - else { - $networks->{$as} = { nets_v6 => [$net], nets_v4 => [] }; - } - } - else { - if ( $dd->{'subtype'} == 2 ) { - push @{ $networks->{$as}->{'nets_v4'} }, $net; - } - else { - push @{ $networks->{$as}->{'nets_v6'} }, $net; - } - } - } } - } } # Now roughly detect countries foreach my $u ( @{ $config{'asn_sources'} } ) { - my $parsed = URI->new($u); - my $fname = $download_target . '/' . basename( $parsed->path ); - open( my $fh, "<", $fname ) or die "Cannot open $fname: $!"; - - while (<$fh>) { - next if /^\#/; - chomp; - my @elts = split /\|/; - - if ( $elts[2] eq 'asn' && $elts[3] ne '*' ) { - my $as_start = int( $elts[3] ); - my $as_end = $as_start + int( $elts[4] ); - - for ( my $as = $as_start ; $as < $as_end ; $as++ ) { - my $real_as = $as; - - if (ref($as) eq "ARRAY") { - $real_as = @{$as}[0]; + my $parsed = URI->new($u); + my $fname = $download_target . '/' . basename( $parsed->path ); + open( my $fh, "<", $fname ) or die "Cannot open $fname: $!"; + + while (<$fh>) { + next if /^\#/; + chomp; + my @elts = split /\|/; + + if ( $elts[2] eq 'asn' && $elts[3] ne '*' ) { + my $as_start = int( $elts[3] ); + my $as_end = $as_start + int( $elts[4] ); + + for ( my $as = $as_start ; $as < $as_end ; $as++ ) { + my $real_as = $as; + + if ( ref($as) eq "ARRAY" ) { + $real_as = @{$as}[0]; + } + + if ( $networks->{"$real_as"} ) { + $networks->{"$real_as"}->{'country'} = $elts[1]; + $networks->{"$real_as"}->{'rir'} = $elts[0]; + } + } } - - if ( $networks->{"$real_as"} ) { - $networks->{"$real_as"}->{'country'} = $elts[1]; - $networks->{"$real_as"}->{'rir'} = $elts[0]; - } - } } - } } while ( my ( $k, $v ) = each( %{$networks} ) ) { - if ($v4) { - foreach my $n ( @{ $v->{'nets_v4'} } ) { - - # "15169 | 8.8.8.0/24 | US | arin |" for 8.8.8.8 - if ( $v->{'country'} ) { - printf $v4_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, $v->{'country'}, $v->{'rir'}; - } - else { - printf $v4_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, 'UN', 'UN'; - } + if ($v4) { + foreach my $n ( @{ $v->{'nets_v4'} } ) { + + # "15169 | 8.8.8.0/24 | US | arin |" for 8.8.8.8 + if ( $v->{'country'} ) { + printf $v4_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, $v->{'country'}, $v->{'rir'}; + } + else { + printf $v4_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, 'UN', 'UN'; + } + } } - } - if ($v6) { - foreach my $n ( @{ $v->{'nets_v6'} } ) { - - # "15169 | 8.8.8.0/24 | US | arin |" for 8.8.8.8 - if ( $v->{'country'} ) { - printf $v6_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, $v->{'country'}, $v->{'rir'}; - } - else { - printf $v6_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, 'UN', 'UN'; - } + if ($v6) { + foreach my $n ( @{ $v->{'nets_v6'} } ) { + + # "15169 | 8.8.8.0/24 | US | arin |" for 8.8.8.8 + if ( $v->{'country'} ) { + printf $v6_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, $v->{'country'}, $v->{'rir'}; + } + else { + printf $v6_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, 'UN', 'UN'; + } + } } - } } __END__ diff --git a/utils/cgp_rspamd.pl b/utils/cgp_rspamd.pl index b1d30b905..0070cf4a5 100644 --- a/utils/cgp_rspamd.pl +++ b/utils/cgp_rspamd.pl @@ -21,278 +21,276 @@ my $request_timeout = 15; # 15 seconds by default my $reject_message = "Spam message rejected"; GetOptions( - "host=s" => \$rspamd_host, - "header=s" => \$header, - "reject-message=s" => \$reject_message, - "max-size=i" => \$max_size, - "timeout=f" => \$request_timeout, - "help|?" => \$help, - "man" => \$man + "host=s" => \$rspamd_host, + "header=s" => \$header, + "reject-message=s" => \$reject_message, + "max-size=i" => \$max_size, + "timeout=f" => \$request_timeout, + "help|?" => \$help, + "man" => \$man ) or pod2usage(2); pod2usage(1) if $help; pod2usage( -exitval => 0, -verbose => 2 ) if $man; my $main_domain = cgp_main_domain(); -my $scanned = 0; +my $scanned = 0; # Turn off bufferization as required by CGP $| = 1; sub cgp_main_domain { - if ( open(my $fh, 'Settings/Main.settings') ) { - while (<$fh>) { - if ( /^\s+DomainName\s+=\s+([^;]+);/ ) { - return $1; - } + if ( open( my $fh, 'Settings/Main.settings' ) ) { + while (<$fh>) { + if (/^\s+DomainName\s+=\s+([^;]+);/) { + return $1; + } + } } - } } sub cgp_string { - my ($in) = @_; + my ($in) = @_; - $in =~ s/\"/\\"/g; - $in =~ s/\n/\\n/gms; - $in =~ s/\r/\\r/mgs; - $in =~ s/\t/ /g; + $in =~ s/\"/\\"/g; + $in =~ s/\n/\\n/gms; + $in =~ s/\r/\\r/mgs; + $in =~ s/\t/ /g; - return "\"$in\""; + return "\"$in\""; } sub rspamd_scan { - my ( $tag, $file ) = @_; - - my $http_callback = sub { - my ( $body, $hdr ) = @_; - - if ( $hdr && $hdr->{Status} =~ /^2/ ) { - my $js = eval('decode_json($body)'); - $scanned++; - - if ( !$js ) { - print "* Rspamd: Bad response for $file: invalid JSON: parse error\n"; - print "$tag FAILURE\n"; - } - else { - my $def = $js; - my $headers = ""; - - if ( !$def ) { - print -"* Rspamd: Bad response for $file: invalid JSON: default is missing\n"; - print "$tag FAILURE\n"; - } - else { - my $action = $def->{'action'}; - my $id = $js->{'message-id'}; - - my $symbols = ""; - while ( my ( $k, $s ) = each( %{$def->{'symbols'}}) ) { - $symbols .= sprintf "%s(%.2f);", $k, $s->{'score'}; - } - - printf -"* Rspamd: Scanned %s; id: <%s>; Score: %.2f / %.2f; Symbols: [%s]\n", - $file, $id, $def->{'score'}, $def->{'required_score'}, $symbols; - - if ( $js->{'dkim-signature'} ) { - $headers .= "DKIM-Signature: " . $js->{'dkim-signature'}; - } - - if ( $js->{'milter'} ) { - my $block = $js->{'milter'}; - - if ( $block->{'add_headers'} ) { - while ( my ( $h, $v ) = each( %{ $block->{'add_headers'} } ) ) { - if (ref($v) eq 'HASH') { - if ($headers eq "") { - $headers .= "$h: $v->{value}"; - } - else { - $headers .= "\\e$h: $v->{value}"; - } + my ( $tag, $file ) = @_; + + my $http_callback = sub { + my ( $body, $hdr ) = @_; + + if ( $hdr && $hdr->{Status} =~ /^2/ ) { + my $js = eval('decode_json($body)'); + $scanned++; + + if ( !$js ) { + print "* Rspamd: Bad response for $file: invalid JSON: parse error\n"; + print "$tag FAILURE\n"; + } + else { + my $def = $js; + my $headers = ""; + + if ( !$def ) { + print "* Rspamd: Bad response for $file: invalid JSON: default is missing\n"; + print "$tag FAILURE\n"; } else { - if ($headers eq "") { - $headers .= "$h: $v"; - } - else { - $headers .= "\\e$h: $v"; - } + my $action = $def->{'action'}; + my $id = $js->{'message-id'}; + + my $symbols = ""; + while ( my ( $k, $s ) = each( %{ $def->{'symbols'} } ) ) { + $symbols .= sprintf "%s(%.2f);", $k, $s->{'score'}; + } + + printf + "* Rspamd: Scanned %s; id: <%s>; Score: %.2f / %.2f; Symbols: [%s]\n", + $file, $id, $def->{'score'}, $def->{'required_score'}, $symbols; + + if ( $js->{'dkim-signature'} ) { + $headers .= "DKIM-Signature: " . $js->{'dkim-signature'}; + } + + if ( $js->{'milter'} ) { + my $block = $js->{'milter'}; + + if ( $block->{'add_headers'} ) { + while ( my ( $h, $v ) = each( %{ $block->{'add_headers'} } ) ) { + if ( ref($v) eq 'HASH' ) { + if ( $headers eq "" ) { + $headers .= "$h: $v->{value}"; + } + else { + $headers .= "\\e$h: $v->{value}"; + } + } + else { + if ( $headers eq "" ) { + $headers .= "$h: $v"; + } + else { + $headers .= "\\e$h: $v"; + } + } + } + } + } + + if ( $action eq 'reject' ) { + print "$tag DISCARD\n"; + return; + } + elsif ( $action eq 'add header' || $action eq 'rewrite subject' ) { + if ( $headers eq "" ) { + $headers .= "$header"; + } + else { + $headers .= "\\e$header"; + } + } + elsif ( $action eq 'soft reject' ) { + print "$tag REJECT Try again later\n"; + return; + } + + if ( $headers eq "" ) { + print "$tag OK\n"; + } + else { + print "$tag ADDHEADER " . cgp_string($headers) . " OK\n"; + } } - } } - } - - if ( $action eq 'reject' ) { - print "$tag DISCARD\n"; - return; - } - elsif ( $action eq 'add header' || $action eq 'rewrite subject' ) { - if ( $headers eq "" ) { - $headers .= "$header"; + } + else { + if ($hdr) { + print "* Rspamd: Bad response for $file: HTTP error: $hdr->{Status} $hdr->{Reason}\n"; } else { - $headers .= "\\e$header"; + print "* Rspamd: Bad response for $file: IO error: $!\n"; } - } - elsif ( $action eq 'soft reject' ) { - print "$tag REJECT Try again later\n"; - return; - } - - if ( $headers eq "" ) { - print "$tag OK\n"; - } - else { - print "$tag ADDHEADER " . cgp_string($headers) . " OK\n"; - } + print "$tag FAILURE\n"; } - } + }; + + if ($local) { + + # Use file scan + # XXX: not implemented now due to CGP queue format + http_get( + "http://$rspamd_host/symbols?file=$file", + timeout => $request_timeout, + $http_callback + ); } else { - if ($hdr) { - print -"* Rspamd: Bad response for $file: HTTP error: $hdr->{Status} $hdr->{Reason}\n"; - } - else { - print "* Rspamd: Bad response for $file: IO error: $!\n"; - } - print "$tag FAILURE\n"; - } - }; - - if ($local) { - - # Use file scan - # XXX: not implemented now due to CGP queue format - http_get( - "http://$rspamd_host/symbols?file=$file", - timeout => $request_timeout, - $http_callback - ); - } - else { - my $sb = stat($file); - - if ( !$sb || $sb->size > $max_size ) { - if ($sb) { - print "* File $file is too large: " . $sb->size . "\n$tag FAILURE\n"; - - } - else { - print "* Cannot stat $file: $!\n$tag FAILURE\n"; - } - return; - } - aio_load( - $file, - sub { - my ($data) = @_; - - if ( !$data ) { - print "* Cannot open $file: $!\n$tag FAILURE\n"; - return; - } + my $sb = stat($file); + + if ( !$sb || $sb->size > $max_size ) { + if ($sb) { + print "* File $file is too large: " . $sb->size . "\n$tag FAILURE\n"; - # Parse CGP format - $data =~ s/^((?:[^\n]*\n)*?)\n(.*)$/$2/ms; - my @envelope = split /\n/, $1; - chomp(@envelope); - my $from; - my @rcpts; - my $ip; - my $user; - - foreach my $elt (@envelope) { - if ( $elt =~ /^P\s[^<]*(<[^>]*>).*$/ ) { - $from = $1; - } - elsif ( $elt =~ /^R\s[^<]*(<[^>]*>).*$/ ) { - push @rcpts, $1; - } - elsif ( $elt =~ /^S (?:<([^>]+)> )?(?:SMTP|HTTPU?|AIRSYNC|XIMSS) \[([0-9a-f.:]+)\]/ ) { - if ($1) { - $user = $1; - } - if ($2) { - $ip = $2; } - } - elsif ( $elt =~ /^S (?:<([^>]+)> )?(?:DSN|GROUP|LIST|PBX|PIPE|RULE) \[0\.0\.0\.0\]/ ) { - if ($1) { - $user = $1; + else { + print "* Cannot stat $file: $!\n$tag FAILURE\n"; } - $ip = '127.2.4.7'; - } + return; } + aio_load( + $file, + sub { + my ($data) = @_; + + if ( !$data ) { + print "* Cannot open $file: $!\n$tag FAILURE\n"; + return; + } - my $headers = {}; - if ( $file =~ /\/([^\/.]+)\.msg$/ ) { - $headers->{'Queue-ID'} = $1; - } - if ($from) { - $headers->{From} = $from; - } - if ( scalar(@rcpts) > 0 ) { + # Parse CGP format + $data =~ s/^((?:[^\n]*\n)*?)\n(.*)$/$2/ms; + my @envelope = split /\n/, $1; + chomp(@envelope); + my $from; + my @rcpts; + my $ip; + my $user; + + foreach my $elt (@envelope) { + if ( $elt =~ /^P\s[^<]*(<[^>]*>).*$/ ) { + $from = $1; + } + elsif ( $elt =~ /^R\s[^<]*(<[^>]*>).*$/ ) { + push @rcpts, $1; + } + elsif ( $elt =~ /^S (?:<([^>]+)> )?(?:SMTP|HTTPU?|AIRSYNC|XIMSS) \[([0-9a-f.:]+)\]/ ) { + if ($1) { + $user = $1; + } + if ($2) { + $ip = $2; + } + } + elsif ( $elt =~ /^S (?:<([^>]+)> )?(?:DSN|GROUP|LIST|PBX|PIPE|RULE) \[0\.0\.0\.0\]/ ) { + if ($1) { + $user = $1; + } + $ip = '127.2.4.7'; + } + } - # XXX: Anyevent cannot parse headers with multiple values - $headers->{Rcpt} = join(',', @rcpts); - } - if ($ip) { - $headers->{IP} = $ip; - } - if ($user) { - $headers->{User} = $user; - } - if ($main_domain) { - $headers->{'MTA-Tag'} = $main_domain; - } + my $headers = {}; + if ( $file =~ /\/([^\/.]+)\.msg$/ ) { + $headers->{'Queue-ID'} = $1; + } + if ($from) { + $headers->{From} = $from; + } + if ( scalar(@rcpts) > 0 ) { + + # XXX: Anyevent cannot parse headers with multiple values + $headers->{Rcpt} = join( ',', @rcpts ); + } + if ($ip) { + $headers->{IP} = $ip; + } + if ($user) { + $headers->{User} = $user; + } + if ($main_domain) { + $headers->{'MTA-Tag'} = $main_domain; + } - http_post( - "http://$rspamd_host/checkv2", $data, - timeout => $request_timeout, - headers => $headers, - $http_callback + http_post( + "http://$rspamd_host/checkv2", $data, + timeout => $request_timeout, + headers => $headers, + $http_callback + ); + } ); - } - ); - } + } } # Show informational message print "* Rspamd CGP filter has been started\n"; my $w = AnyEvent->io( - fh => \*STDIN, - poll => 'r', - cb => sub { - chomp( my $input = <STDIN> ); - - if ( $input =~ /^(\d+)\s+(\S+)(\s+(\S+)\s*)?$/ ) { - my $tag = $1; - my $cmd = $2; - - if ( $cmd eq "INTF" ) { - print "$input\n"; - } - elsif ( $cmd eq "FILE" && $4 ) { - my $file = $4; - print "* Scanning file $file\n"; - rspamd_scan $tag, $file; - } - elsif ( $cmd eq "QUIT" ) { - print "* Terminating after scanning of $scanned files\n"; - print "$tag OK\n"; - exit 0; - } - else { - print "* Unknown command $cmd\n"; - print "$tag FAILURE\n"; - } + fh => \*STDIN, + poll => 'r', + cb => sub { + chomp( my $input = <STDIN> ); + + if ( $input =~ /^(\d+)\s+(\S+)(\s+(\S+)\s*)?$/ ) { + my $tag = $1; + my $cmd = $2; + + if ( $cmd eq "INTF" ) { + print "$input\n"; + } + elsif ( $cmd eq "FILE" && $4 ) { + my $file = $4; + print "* Scanning file $file\n"; + rspamd_scan $tag, $file; + } + elsif ( $cmd eq "QUIT" ) { + print "* Terminating after scanning of $scanned files\n"; + print "$tag OK\n"; + exit 0; + } + else { + print "* Unknown command $cmd\n"; + print "$tag FAILURE\n"; + } + } } - } ); EV::run; @@ -326,8 +324,7 @@ Specifies Rspamd host to use for scanning =item B<--header> -Specifies the header that should be added when Rspamd action is B<add header> -or B<rewrite subject>. +Specifies the header that should be added when Rspamd action is B<add header> or B<rewrite subject>. =item B<--reject-message> @@ -353,12 +350,8 @@ Prints the manual page and exits. =head1 DESCRIPTION -B<cgp_rspamd> is intended to scan messages processed with B<CommunigatePro> MTA -on some Rspamd scanner. It reads standard input and parses CGP helpers -protocol. On scan requests, this filter can query Rspamd to process a message. -B<cgp_rspamd> can tell CGP to add header or reject SPAM messages depending on -Rspamd scan result. - -=back +B<cgp_rspamd> is intended to scan messages processed with B<CommunigatePro> MTA on some Rspamd scanner. It reads +standard input and parses CGP helpers protocol. On scan requests, this filter can query Rspamd to process a message. +B<cgp_rspamd> can tell CGP to add header or reject SPAM messages depending on Rspamd scan result. =cut diff --git a/utils/classifier_test.pl b/utils/classifier_test.pl index 2dbb4e903..994ad8ce8 100644 --- a/utils/classifier_test.pl +++ b/utils/classifier_test.pl @@ -30,438 +30,428 @@ my $man; my $help; GetOptions( - "spam|s=s" => \$spam_dir, - "ham|h=s" => \$ham_dir, - "spam-symbol=s" => \$spam_symbol, - "ham-symbol=s" => \$ham_symbol, - "classifier|c=s" => \$classifier, - "timeout|t=f" => \$timeout, - "parallel|p=i" => \$parallel, - "train-fraction|t=f" => \$train_fraction, - "bogofilter|b" => \$use_bogofilter, - "dspam|d" => \$use_dspam, - "check-only" => \$check_only, - "help|?" => \$help, - "man" => \$man + "spam|s=s" => \$spam_dir, + "ham|h=s" => \$ham_dir, + "spam-symbol=s" => \$spam_symbol, + "ham-symbol=s" => \$ham_symbol, + "classifier|c=s" => \$classifier, + "timeout|t=f" => \$timeout, + "parallel|p=i" => \$parallel, + "train-fraction|t=f" => \$train_fraction, + "bogofilter|b" => \$use_bogofilter, + "dspam|d" => \$use_dspam, + "check-only" => \$check_only, + "help|?" => \$help, + "man" => \$man ) or pod2usage(2); pod2usage(1) if $help; pod2usage( -exitval => 0, -verbose => 2 ) if $man; sub read_dir_files { - my ( $dir, $target ) = @_; - opendir( my $dh, $dir ) or die "cannot open dir $dir: $!"; - while ( my $file = readdir $dh ) { - if ( -f "$dir/$file" ) { - push @{$target}, "$dir/$file"; + my ( $dir, $target ) = @_; + opendir( my $dh, $dir ) or die "cannot open dir $dir: $!"; + while ( my $file = readdir $dh ) { + if ( -f "$dir/$file" ) { + push @{$target}, "$dir/$file"; + } } - } } sub shuffle_array { - my ($ar) = @_; + my ($ar) = @_; - for ( my $i = 0 ; $i < scalar @{$ar} ; $i++ ) { - if ( $i > 1 ) { - my $sel = int( rand( $i - 1 ) ); - ( @{$ar}[$i], @{$ar}[$sel] ) = ( @{$ar}[$sel], @{$ar}[$i] ); + for ( my $i = 0 ; $i < scalar @{$ar} ; $i++ ) { + if ( $i > 1 ) { + my $sel = int( rand( $i - 1 ) ); + ( @{$ar}[$i], @{$ar}[$sel] ) = ( @{$ar}[$sel], @{$ar}[$i] ); + } } - } } sub learn_rspamc { - my ( $files, $spam ) = @_; - my $processed = 0; - - my $cmd = $spam ? "learn_spam" : "learn_ham"; - my $args_quoted = shell_quote @{$files}; - open( - my $p, -"$rspamc -t $timeout -c $classifier --compact -j -n $parallel $cmd $args_quoted |" - ) or die "cannot spawn $rspamc: $!"; - - while (<$p>) { - my $res = eval('decode_json($_)'); - if ( $res && $res->{'success'} ) { - $processed++; + my ( $files, $spam ) = @_; + my $processed = 0; + + my $cmd = $spam ? "learn_spam" : "learn_ham"; + my $args_quoted = shell_quote @{$files}; + open( my $p, "$rspamc -t $timeout -c $classifier --compact -j -n $parallel $cmd $args_quoted |" ) + or die "cannot spawn $rspamc: $!"; + + while (<$p>) { + my $res = eval('decode_json($_)'); + if ( $res && $res->{'success'} ) { + $processed++; + } } - } - return $processed; + return $processed; } sub learn_bogofilter { - my ( $files, $spam ) = @_; - my $processed = 0; - - foreach my $f ( @{$files} ) { - my $args_quoted = shell_quote $f; - my $fl = $spam ? "-s" : "-n"; - `$bogofilter -I $args_quoted $fl`; - if ( $? == 0 ) { - $processed++; + my ( $files, $spam ) = @_; + my $processed = 0; + + foreach my $f ( @{$files} ) { + my $args_quoted = shell_quote $f; + my $fl = $spam ? "-s" : "-n"; + `$bogofilter -I $args_quoted $fl`; + if ( $? == 0 ) { + $processed++; + } } - } - return $processed; + return $processed; } sub learn_dspam { - my ( $files, $spam ) = @_; - my $processed = 0; - - foreach my $f ( @{$files} ) { - my $args_quoted = shell_quote $f; - my $fl = $spam ? "--class=spam" : "--class=innocent"; - open( my $p, - "|$dspam --user nobody --source=corpus --stdout --mode=toe $fl" ) - or die "cannot run $dspam: $!"; - - open( my $inp, "< $f" ); - while (<$inp>) { - print $p $_; + my ( $files, $spam ) = @_; + my $processed = 0; + + foreach my $f ( @{$files} ) { + my $args_quoted = shell_quote $f; + my $fl = $spam ? "--class=spam" : "--class=innocent"; + open( my $p, "|$dspam --user nobody --source=corpus --stdout --mode=toe $fl" ) + or die "cannot run $dspam: $!"; + + open( my $inp, "< $f" ); + while (<$inp>) { + print $p $_; + } } - } - return $processed; + return $processed; } sub learn_samples { - my ( $ar_ham, $ar_spam ) = @_; - my $len; - my $processed = 0; - my $total = 0; - my $learn_func; - - my @files_spam; - my @files_ham; - - if ($use_dspam) { - $learn_func = \&learn_dspam; - } - elsif ($use_bogofilter) { - $learn_func = \&learn_bogofilter; - } - else { - $learn_func = \&learn_rspamc; - } - - $len = int( scalar @{$ar_ham} * $train_fraction ); - my @cur_vec; - - # Shuffle spam and ham samples - for ( my $i = 0 ; $i < $len ; $i++ ) { - if ( $i > 0 && ( $i % $parallel == 0 || $i == $len - 1 ) ) { - push @cur_vec, @{$ar_ham}[$i]; - push @files_ham, [@cur_vec]; - @cur_vec = (); - $total++; - } - else { - push @cur_vec, @{$ar_ham}[$i]; - } - } - - $len = int( scalar @{$ar_spam} * $train_fraction ); - @cur_vec = (); - for ( my $i = 0 ; $i < $len ; $i++ ) { - if ( $i > 0 && ( $i % $parallel == 0 || $i == $len - 1 ) ) { - push @cur_vec, @{$ar_spam}[$i]; - push @files_spam, [@cur_vec]; - @cur_vec = (); - $total++; - } - else { - push @cur_vec, @{$ar_spam}[$i]; + my ( $ar_ham, $ar_spam ) = @_; + my $len; + my $processed = 0; + my $total = 0; + my $learn_func; + + my @files_spam; + my @files_ham; + + if ($use_dspam) { + $learn_func = \&learn_dspam; } - } - - for ( my $i = 0 ; $i < $total ; $i++ ) { - my $args; - my $spam; - - if ( $i % 2 == 0 ) { - $args = pop @files_spam; - - if ( !$args ) { - $args = pop @files_ham; - $spam = 0; - } - else { - $spam = 1; - } + elsif ($use_bogofilter) { + $learn_func = \&learn_bogofilter; } else { - $args = pop @files_ham; - if ( !$args ) { - $args = pop @files_spam; - $spam = 1; - } - else { - $spam = 0; - } + $learn_func = \&learn_rspamc; } - my $r = $learn_func->( $args, $spam ); - if ($r) { - $processed += $r; + $len = int( scalar @{$ar_ham} * $train_fraction ); + my @cur_vec; + + # Shuffle spam and ham samples + for ( my $i = 0 ; $i < $len ; $i++ ) { + if ( $i > 0 && ( $i % $parallel == 0 || $i == $len - 1 ) ) { + push @cur_vec, @{$ar_ham}[$i]; + push @files_ham, [@cur_vec]; + @cur_vec = (); + $total++; + } + else { + push @cur_vec, @{$ar_ham}[$i]; + } } - } - return $processed; -} + $len = int( scalar @{$ar_spam} * $train_fraction ); + @cur_vec = (); + for ( my $i = 0 ; $i < $len ; $i++ ) { + if ( $i > 0 && ( $i % $parallel == 0 || $i == $len - 1 ) ) { + push @cur_vec, @{$ar_spam}[$i]; + push @files_spam, [@cur_vec]; + @cur_vec = (); + $total++; + } + else { + push @cur_vec, @{$ar_spam}[$i]; + } + } -sub check_rspamc { - my ( $files, $spam, $fp_cnt, $fn_cnt, $detected_cnt ) = @_; + for ( my $i = 0 ; $i < $total ; $i++ ) { + my $args; + my $spam; - my $args_quoted = shell_quote @{$files}; - my $processed = 0; + if ( $i % 2 == 0 ) { + $args = pop @files_spam; - open( - my $p, -"$rspamc -t $timeout -n $parallel --header=\"Settings: {symbols_enabled=[BAYES_SPAM]}\" --compact -j $args_quoted |" - ) or die "cannot spawn $rspamc: $!"; - - while (<$p>) { - my $res = eval('decode_json($_)'); - if ( $res && $res->{'default'} ) { - $processed++; - - if ($spam) { - if ( $res->{'default'}->{$ham_symbol} ) { - my $m = $res->{'default'}->{$ham_symbol}->{'options'}->[0]; - if ( $m && $m =~ /^(\d+(?:\.\d+)?)%$/ ) { - my $percentage = int($1); - if ( $percentage >= $rspamc_prob_trigger ) { - $$fp_cnt++; + if ( !$args ) { + $args = pop @files_ham; + $spam = 0; + } + else { + $spam = 1; } - } - else { - $$fp_cnt++; - } - } - elsif ( !$res->{'default'}->{$spam_symbol} ) { - $$fn_cnt++; } else { - $$detected_cnt++; - } - } - else { - if ( $res->{'default'}->{$spam_symbol} ) { - my $m = $res->{'default'}->{$spam_symbol}->{'options'}->[0]; - if ( $m && $m =~ /^(\d+(?:\.\d+)?)%$/ ) { - - my $percentage = int($1); - if ( $percentage >= $rspamc_prob_trigger ) { - $$fp_cnt++; + $args = pop @files_ham; + if ( !$args ) { + $args = pop @files_spam; + $spam = 1; + } + else { + $spam = 0; } - } - else { - $$fp_cnt++; - } - } - elsif ( !$res->{'default'}->{$ham_symbol} ) { - $$fn_cnt++; } - else { - $$detected_cnt++; + + my $r = $learn_func->( $args, $spam ); + if ($r) { + $processed += $r; } - } } - } - return $processed; + return $processed; } -sub check_bogofilter { - my ( $files, $spam, $fp_cnt, $fn_cnt, $detected_cnt ) = @_; - my $processed = 0; +sub check_rspamc { + my ( $files, $spam, $fp_cnt, $fn_cnt, $detected_cnt ) = @_; - foreach my $f ( @{$files} ) { - my $args_quoted = shell_quote $f; + my $args_quoted = shell_quote @{$files}; + my $processed = 0; - open( my $p, "$bogofilter -t -I $args_quoted |" ) - or die "cannot spawn $bogofilter: $!"; + open( + my $p, +"$rspamc -t $timeout -n $parallel --header=\"Settings: {symbols_enabled=[BAYES_SPAM]}\" --compact -j $args_quoted |" + ) or die "cannot spawn $rspamc: $!"; while (<$p>) { - if ( $_ =~ /^([SHU])\s+.*$/ ) { - $processed++; - - if ($spam) { - if ( $1 eq 'H' ) { - $$fp_cnt++; - } - elsif ( $1 eq 'U' ) { - $$fn_cnt++; - } - else { - $$detected_cnt++; - } - } - else { - if ( $1 eq 'S' ) { - $$fp_cnt++; - } - elsif ( $1 eq 'U' ) { - $$fn_cnt++; - } - else { - $$detected_cnt++; - } + my $res = eval('decode_json($_)'); + if ( $res && $res->{'default'} ) { + $processed++; + + if ($spam) { + if ( $res->{'default'}->{$ham_symbol} ) { + my $m = $res->{'default'}->{$ham_symbol}->{'options'}->[0]; + if ( $m && $m =~ /^(\d+(?:\.\d+)?)%$/ ) { + my $percentage = int($1); + if ( $percentage >= $rspamc_prob_trigger ) { + $$fp_cnt++; + } + } + else { + $$fp_cnt++; + } + } + elsif ( !$res->{'default'}->{$spam_symbol} ) { + $$fn_cnt++; + } + else { + $$detected_cnt++; + } + } + else { + if ( $res->{'default'}->{$spam_symbol} ) { + my $m = $res->{'default'}->{$spam_symbol}->{'options'}->[0]; + if ( $m && $m =~ /^(\d+(?:\.\d+)?)%$/ ) { + + my $percentage = int($1); + if ( $percentage >= $rspamc_prob_trigger ) { + $$fp_cnt++; + } + } + else { + $$fp_cnt++; + } + } + elsif ( !$res->{'default'}->{$ham_symbol} ) { + $$fn_cnt++; + } + else { + $$detected_cnt++; + } + } } - } } - } - return $processed; + return $processed; } -sub check_dspam { - my ( $files, $spam, $fp_cnt, $fn_cnt, $detected_cnt ) = @_; - my $processed = 0; +sub check_bogofilter { + my ( $files, $spam, $fp_cnt, $fn_cnt, $detected_cnt ) = @_; + my $processed = 0; + + foreach my $f ( @{$files} ) { + my $args_quoted = shell_quote $f; + + open( my $p, "$bogofilter -t -I $args_quoted |" ) + or die "cannot spawn $bogofilter: $!"; + + while (<$p>) { + if ( $_ =~ /^([SHU])\s+.*$/ ) { + $processed++; + + if ($spam) { + if ( $1 eq 'H' ) { + $$fp_cnt++; + } + elsif ( $1 eq 'U' ) { + $$fn_cnt++; + } + else { + $$detected_cnt++; + } + } + else { + if ( $1 eq 'S' ) { + $$fp_cnt++; + } + elsif ( $1 eq 'U' ) { + $$fn_cnt++; + } + else { + $$detected_cnt++; + } + } + } + } + } - foreach my $f ( @{$files} ) { - my $args_quoted = shell_quote $f; + return $processed; +} - my $pid = open2( *Reader, *Writer, - "$dspam --user nobody --classify --stdout --mode=notrain" ); - open( my $inp, "< $f" ); - while (<$inp>) { - print Writer $_; - } - close Writer; +sub check_dspam { + my ( $files, $spam, $fp_cnt, $fn_cnt, $detected_cnt ) = @_; + my $processed = 0; - while (<Reader>) { - if ( $_ =~ -qr(^X-DSPAM-Result: nobody; result="([^"]+)"; class="[^"]+"; probability=(\d+(?:\.\d+)?).*$) - ) - { - $processed++; - my $percentage = int($2 * 100.0); + foreach my $f ( @{$files} ) { + my $args_quoted = shell_quote $f; - if ($spam) { - if ( $1 eq 'Innocent') { - if ( $percentage <= (100 - $rspamc_prob_trigger) ) { - $$fp_cnt++; - } - } - elsif ( $1 ne 'Spam' ) { - $$fn_cnt++; - } - else { - $$detected_cnt++; - } + my $pid = open2( *Reader, *Writer, "$dspam --user nobody --classify --stdout --mode=notrain" ); + open( my $inp, "< $f" ); + while (<$inp>) { + print Writer $_; } - else { - if ( $1 eq 'Spam' ) { - if ( $percentage >= $rspamc_prob_trigger ) { - $$fp_cnt++; + close Writer; + + while (<Reader>) { + if ( $_ =~ qr(^X-DSPAM-Result: nobody; result="([^"]+)"; class="[^"]+"; probability=(\d+(?:\.\d+)?).*$) ) { + $processed++; + my $percentage = int( $2 * 100.0 ); + + if ($spam) { + if ( $1 eq 'Innocent' ) { + if ( $percentage <= ( 100 - $rspamc_prob_trigger ) ) { + $$fp_cnt++; + } + } + elsif ( $1 ne 'Spam' ) { + $$fn_cnt++; + } + else { + $$detected_cnt++; + } + } + else { + if ( $1 eq 'Spam' ) { + if ( $percentage >= $rspamc_prob_trigger ) { + $$fp_cnt++; + } + } + elsif ( $1 ne 'Innocent' ) { + $$fn_cnt++; + } + else { + $$detected_cnt++; + } + } } - } - elsif ( $1 ne 'Innocent' ) { - $$fn_cnt++; - } - else { - $$detected_cnt++; - } } - } + close Reader; + waitpid( $pid, 0 ); } - close Reader; - waitpid( $pid, 0 ); - } - return $processed; + return $processed; } sub cross_validate { - my ($hr) = @_; - my $args = ""; - my $processed = 0; - my $fp_spam = 0; - my $fn_spam = 0; - my $fp_ham = 0; - my $fn_ham = 0; - my $total_spam = 0; - my $total_ham = 0; - my $detected_spam = 0; - my $detected_ham = 0; - my $i = 0; - my $len = scalar keys %{$hr}; - my @files_spam; - my @files_ham; - my @cur_spam; - my @cur_ham; - my $check_func; - - if ($use_dspam) { - $check_func = \&check_dspam; - } - elsif ($use_bogofilter) { - $check_func = \&check_bogofilter; - } - else { - $check_func = \&check_rspamc; - } - - while ( my ( $fn, $spam ) = each( %{$hr} ) ) { - if ($spam) { - if ( scalar @cur_spam >= $parallel || $i == $len - 1 ) { - push @cur_spam, $fn; - push @files_spam, [@cur_spam]; - @cur_spam = (); - } - else { - push @cur_spam, $fn; - } + my ($hr) = @_; + my $args = ""; + my $processed = 0; + my $fp_spam = 0; + my $fn_spam = 0; + my $fp_ham = 0; + my $fn_ham = 0; + my $total_spam = 0; + my $total_ham = 0; + my $detected_spam = 0; + my $detected_ham = 0; + my $i = 0; + my $len = scalar keys %{$hr}; + my @files_spam; + my @files_ham; + my @cur_spam; + my @cur_ham; + my $check_func; + + if ($use_dspam) { + $check_func = \&check_dspam; + } + elsif ($use_bogofilter) { + $check_func = \&check_bogofilter; } else { - if ( scalar @cur_ham >= $parallel || $i == $len - 1 ) { - push @cur_ham, $fn; - push @files_ham, [@cur_ham]; - @cur_ham = (); - } - else { - push @cur_ham, $fn; - } + $check_func = \&check_rspamc; + } + + while ( my ( $fn, $spam ) = each( %{$hr} ) ) { + if ($spam) { + if ( scalar @cur_spam >= $parallel || $i == $len - 1 ) { + push @cur_spam, $fn; + push @files_spam, [@cur_spam]; + @cur_spam = (); + } + else { + push @cur_spam, $fn; + } + } + else { + if ( scalar @cur_ham >= $parallel || $i == $len - 1 ) { + push @cur_ham, $fn; + push @files_ham, [@cur_ham]; + @cur_ham = (); + } + else { + push @cur_ham, $fn; + } + } } - } - shuffle_array( \@files_spam ); + shuffle_array( \@files_spam ); - foreach my $fn (@files_spam) { - my $r = $check_func->( $fn, 1, \$fp_ham, \$fn_spam, \$detected_spam ); - $total_spam += $r; - $processed += $r; - } + foreach my $fn (@files_spam) { + my $r = $check_func->( $fn, 1, \$fp_ham, \$fn_spam, \$detected_spam ); + $total_spam += $r; + $processed += $r; + } - shuffle_array( \@files_ham ); + shuffle_array( \@files_ham ); - foreach my $fn (@files_ham) { - my $r = $check_func->( $fn, 0, \$fp_spam, \$fn_ham, \$detected_ham ); - $total_ham += $r; - $processed += $r; - } + foreach my $fn (@files_ham) { + my $r = $check_func->( $fn, 0, \$fp_spam, \$fn_ham, \$detected_ham ); + $total_ham += $r; + $processed += $r; + } - printf "Scanned %d messages + printf "Scanned %d messages %d spam messages (%d detected) -%d ham messages (%d detected)\n", - $processed, $total_spam, $detected_spam, $total_ham, $detected_ham; +%d ham messages (%d detected)\n", $processed, $total_spam, $detected_spam, $total_ham, $detected_ham; - printf "\nHam FP rate: %.2f%% (%d messages) -Ham FN rate: %.2f%% (%d messages)\n", - $fp_ham / $total_ham * 100.0, $fp_ham, - $fn_ham / $total_ham * 100.0, $fn_ham; + printf "\nHam FP rate: %.2f%% (%d messages) +Ham FN rate: %.2f%% (%d messages)\n", $fp_ham / $total_ham * 100.0, $fp_ham, $fn_ham / $total_ham * 100.0, $fn_ham; - printf "\nSpam FP rate: %.2f%% (%d messages) + printf "\nSpam FP rate: %.2f%% (%d messages) Spam FN rate: %.2f%% (%d messages)\n", - $fp_spam / $total_spam * 100.0, $fp_spam, - $fn_spam / $total_spam * 100.0, $fn_spam; + $fp_spam / $total_spam * 100.0, $fp_spam, + $fn_spam / $total_spam * 100.0, $fn_spam; } if ( !$spam_dir || !$ham_dir ) { - die "spam or/and ham directories are not specified"; + die "spam or/and ham directories are not specified"; } my @spam_samples; @@ -473,24 +463,23 @@ shuffle_array( \@spam_samples ); shuffle_array( \@ham_samples ); if ( !$check_only ) { - my $learned = 0; - my $t0 = [gettimeofday]; - $learned = learn_samples( \@ham_samples, \@spam_samples ); - my $t1 = [gettimeofday]; + my $learned = 0; + my $t0 = [gettimeofday]; + $learned = learn_samples( \@ham_samples, \@spam_samples ); + my $t1 = [gettimeofday]; - printf "Learned classifier, %d items processed, %.2f seconds elapsed\n", - $learned, tv_interval( $t0, $t1 ); + printf "Learned classifier, %d items processed, %.2f seconds elapsed\n", $learned, tv_interval( $t0, $t1 ); } my %validation_set; my $len = int( scalar @spam_samples * $train_fraction ); for ( my $i = $len ; $i < scalar @spam_samples ; $i++ ) { - $validation_set{ $spam_samples[$i] } = 1; + $validation_set{ $spam_samples[$i] } = 1; } $len = int( scalar @ham_samples * $train_fraction ); for ( my $i = $len ; $i < scalar @spam_samples ; $i++ ) { - $validation_set{ $ham_samples[$i] } = 0; + $validation_set{ $ham_samples[$i] } = 0; } cross_validate( \%validation_set ); @@ -544,8 +533,7 @@ Prints the manual page and exits. =head1 DESCRIPTION -B<classifier_test.pl> is intended to test Rspamd classifier for false positives, -false negatives and other parameters. It uses half of the corpus for training -and half for cross-validation. +B<classifier_test.pl> is intended to test Rspamd classifier for false positives, false negatives and other parameters. +It uses half of the corpus for training and half for cross-validation. =cut diff --git a/utils/fann_train.pl b/utils/fann_train.pl index 46b539489..2ce422eb4 100755 --- a/utils/fann_train.pl +++ b/utils/fann_train.pl @@ -8,28 +8,28 @@ use warnings FATAL => 'all'; use AI::FANN qw(:all); use Getopt::Std; -my %sym_idx; # Symbols by index -my %sym_names; # Symbols by name -my $num = 1; # Number of symbols +my %sym_idx; # Symbols by index +my %sym_names; # Symbols by name +my $num = 1; # Number of symbols my @spam; my @ham; -my $max_samples = -1; -my $split = 1; -my $preprocessed = 0; # output is in format <score>:<0|1>:<SYM1,...SYMN> -my $score_spam = 12; -my $score_ham = -6; +my $max_samples = -1; +my $split = 1; +my $preprocessed = 0; # output is in format <score>:<0|1>:<SYM1,...SYMN> +my $score_spam = 12; +my $score_ham = -6; sub process { - my ($input, $spam, $ham) = @_; + my ( $input, $spam, $ham ) = @_; my $samples = 0; - while(<$input>) { - if (!$preprocessed) { + while (<$input>) { + if ( !$preprocessed ) { if (/^.*rspamd_task_write_log.*: \[(-?\d+\.?\d*)\/(\d+\.?\d*)\]\s*\[(.+)\].*$/) { - if ($1 > $score_spam) { + if ( $1 > $score_spam ) { $_ = "$1:1: $3"; } - elsif ($1 < $score_ham) { + elsif ( $1 < $score_ham ) { $_ = "$1:0: $3\n"; } else { @@ -47,7 +47,7 @@ sub process { my $is_spam = 0; - if ($2 == 1) { + if ( $2 == 1 ) { $is_spam = 1; } @@ -56,13 +56,13 @@ sub process { foreach my $sym (@ar) { chomp $sym; - if (!$sym_idx{$sym}) { - $sym_idx{$sym} = $num; + if ( !$sym_idx{$sym} ) { + $sym_idx{$sym} = $num; $sym_names{$num} = $sym; $num++; } - $sample{$sym_idx{$sym}} = 1; + $sample{ $sym_idx{$sym} } = 1; } if ($is_spam) { @@ -73,32 +73,31 @@ sub process { } $samples++; - if ($max_samples > 0 && $samples > $max_samples) { + if ( $max_samples > 0 && $samples > $max_samples ) { return; } } } # Shuffle array -sub fisher_yates_shuffle -{ +sub fisher_yates_shuffle { my $array = shift; - my $i = @$array; + my $i = @$array; while ( --$i ) { my $j = int rand( $i + 1 ); - @$array[$i, $j] = @$array[$j, $i]; + @$array[ $i, $j ] = @$array[ $j, $i ]; } } # Train network sub train { - my ($ann, $sample, $result) = @_; + my ( $ann, $sample, $result ) = @_; my @row; - for (my $i = 1; $i < $num; $i++) { - if ($sample->{$i}) { + for ( my $i = 1 ; $i < $num ; $i++ ) { + if ( $sample->{$i} ) { push @row, 1; } else { @@ -108,16 +107,16 @@ sub train { #print "@row -> @{$result}\n"; - $ann->train(\@row, \@{$result}); + $ann->train( \@row, \@{$result} ); } sub test { - my ($ann, $sample) = @_; + my ( $ann, $sample ) = @_; my @row; - for (my $i = 1; $i < $num; $i++) { - if ($sample->{$i}) { + for ( my $i = 1 ; $i < $num ; $i++ ) { + if ( $sample->{$i} ) { push @row, 1; } else { @@ -125,117 +124,120 @@ sub test { } } - my $ret = $ann->run(\@row); + my $ret = $ann->run( \@row ); return $ret; } my %opts; -getopts('o:i:s:n:t:hpS:H:', \%opts); +getopts( 'o:i:s:n:t:hpS:H:', \%opts ); -if ($opts{'h'}) { +if ( $opts{'h'} ) { print "$0 [-i input] [-o output] [-s scores] [-n max_samples] [-S spam_score] [-H ham_score] [-ph]\n"; exit; } my $input = *STDIN; -if ($opts{'i'}) { - open($input, '<', $opts{'i'}) or die "cannot open $opts{i}"; +if ( $opts{'i'} ) { + open( $input, '<', $opts{'i'} ) or die "cannot open $opts{i}"; } -if ($opts{'n'}) { +if ( $opts{'n'} ) { $max_samples = $opts{'n'}; } -if ($opts{'t'}) { +if ( $opts{'t'} ) { + # Test split $split = $opts{'t'}; } -if ($opts{'p'}) { +if ( $opts{'p'} ) { $preprocessed = 1; } -if ($opts{'H'}) { +if ( $opts{'H'} ) { $score_ham = $opts{'H'}; } -if ($opts{'S'}) { +if ( $opts{'S'} ) { $score_spam = $opts{'S'}; } # ham_prob, spam_prob my @spam_out = (1); -my @ham_out = (0); +my @ham_out = (0); -process($input, \@spam, \@ham); -fisher_yates_shuffle(\@spam); -fisher_yates_shuffle(\@ham); +process( $input, \@spam, \@ham ); +fisher_yates_shuffle( \@spam ); +fisher_yates_shuffle( \@ham ); -my $nspam = int(scalar(@spam) / $split); -my $nham = int(scalar(@ham) / $split); +my $nspam = int( scalar(@spam) / $split ); +my $nham = int( scalar(@ham) / $split ); -my $ann = AI::FANN->new_standard($num - 1, ($num + 2) / 2, 1); +my $ann = AI::FANN->new_standard( $num - 1, ( $num + 2 ) / 2, 1 ); my @train_data; + # Train ANN -for (my $i = 0; $i < $nham; $i++) { +for ( my $i = 0 ; $i < $nham ; $i++ ) { push @train_data, [ $ham[$i], \@ham_out ]; } -for (my $i = 0; $i < $nspam; $i++) { +for ( my $i = 0 ; $i < $nspam ; $i++ ) { push @train_data, [ $spam[$i], \@spam_out ]; } -fisher_yates_shuffle(\@train_data); +fisher_yates_shuffle( \@train_data ); foreach my $train_row (@train_data) { - train($ann, @{$train_row}[0], @{$train_row}[1]); + train( $ann, @{$train_row}[0], @{$train_row}[1] ); } print "Trained $nspam SPAM and $nham HAM samples\n"; # Now run fann -if ($split > 1) { - my $sample = 0.0; +if ( $split > 1 ) { + my $sample = 0.0; my $correct = 0.0; - for (my $i = $nham; $i < $nham * $split; $i++) { - my $ret = test($ann, $ham[$i]); + for ( my $i = $nham ; $i < $nham * $split ; $i++ ) { + my $ret = test( $ann, $ham[$i] ); + #print "@{$ret}\n"; - if (@{$ret}[0] < 0.5) { + if ( @{$ret}[0] < 0.5 ) { $correct++; } $sample++; } - print "Tested $sample HAM samples, correct matched: $correct, rate: ".($correct / $sample)."\n"; + print "Tested $sample HAM samples, correct matched: $correct, rate: " . ( $correct / $sample ) . "\n"; - $sample = 0.0; + $sample = 0.0; $correct = 0.0; - for (my $i = $nspam; $i < $nspam * $split; $i++) { - my $ret = test($ann, $spam[$i]); + for ( my $i = $nspam ; $i < $nspam * $split ; $i++ ) { + my $ret = test( $ann, $spam[$i] ); + #print "@{$ret}\n"; - if (@{$ret}[0] > 0.5) { + if ( @{$ret}[0] > 0.5 ) { $correct++; } $sample++; } - print "Tested $sample SPAM samples, correct matched: $correct, rate: ".($correct / $sample)."\n"; + print "Tested $sample SPAM samples, correct matched: $correct, rate: " . ( $correct / $sample ) . "\n"; } -if ($opts{'o'}) { - $ann->save($opts{'o'}) or die "cannot save ann into $opts{o}"; +if ( $opts{'o'} ) { + $ann->save( $opts{'o'} ) or die "cannot save ann into $opts{o}"; } -if ($opts{'s'}) { - open(my $scores, '>', - $opts{'s'}) or die "cannot open score file $opts{'s'}"; +if ( $opts{'s'} ) { + open( my $scores, '>', $opts{'s'} ) or die "cannot open score file $opts{'s'}"; print $scores "{"; - for (my $i = 1; $i < $num; $i++) { + for ( my $i = 1 ; $i < $num ; $i++ ) { my $n = $i - 1; - if ($i != $num - 1) { + if ( $i != $num - 1 ) { print $scores "\"$sym_names{$i}\":$n,"; } else { diff --git a/utils/redirector.pl.in b/utils/redirector.pl.in index 3bd6ab112..4fbe3844a 100755 --- a/utils/redirector.pl.in +++ b/utils/redirector.pl.in @@ -34,22 +34,22 @@ eval "require SWF::Element" or $with_swf = 0; # p5-SWF-File my $DEBUG = grep { $_ eq '-debug' } @ARGV; our %cfg = ( - port => 8080, - max_size => 102400, - http_timeout => 5, - max_rec => 5, - pidfile => '/tmp/redirector.pid', - do_log => 0, - debug => 0, - redis_server => 'localhost:6379', - - facility => LOG_LOCAL3, # syslog facility - log_level => LOG_INFO, - digest_bits => 256, - cache_expire => 3600, - user => '@RSPAMD_USER@', - group => '@RSPAMD_GROUP@', - cfg_file => '@CMAKE_INSTALL_PREFIX@/etc/rspamd-redirector.conf', + port => 8080, + max_size => 102400, + http_timeout => 5, + max_rec => 5, + pidfile => '/tmp/redirector.pid', + do_log => 0, + debug => 0, + redis_server => 'localhost:6379', + + facility => LOG_LOCAL3, # syslog facility + log_level => LOG_INFO, + digest_bits => 256, + cache_expire => 3600, + user => '@RSPAMD_USER@', + group => '@RSPAMD_GROUP@', + cfg_file => '@CMAKE_INSTALL_PREFIX@/etc/rspamd-redirector.conf', ); our $do_reopen_log = 0; @@ -59,468 +59,442 @@ our $redis_conn; # Read file into string sub read_file { - my ($file) = @_; + my ($file) = @_; - open( IN, $file ) or _log( LOG_ALERT, "Can't open $file: $!" ); - local $/; - my $content = <IN>; - close IN; + open( IN, $file ) or _log( LOG_ALERT, "Can't open $file: $!" ); + local $/; + my $content = <IN>; + close IN; - return $content; + return $content; } # Write log line: sub _log { - my ( $l, $w, @s ) = @_; - - if ($DEBUG) { - printf STDERR $w . "\n", @s; - } - else { - syslog( $l, $w . "\n", @s ) if ( $l <= $cfg{'log_level'} ); - } - - if ( $l == LOG_ALERT ) { - die $w; - } + my ( $l, $w, @s ) = @_; + + if ($DEBUG) { + printf STDERR $w . "\n", @s; + } + else { + syslog( $l, $w . "\n", @s ) if ( $l <= $cfg{'log_level'} ); + } + + if ( $l == LOG_ALERT ) { + die $w; + } } # Init swf parser sub swf_init_parser { - if ($with_swf) { - $swf_parser = SWF::Parser->new( 'tag-callback' => \&swf_tag_callback ); - } + if ($with_swf) { + $swf_parser = SWF::Parser->new( 'tag-callback' => \&swf_tag_callback ); + } } # Checking for SWF url sub swf_search_get_url { - my $actions = shift; - my $saved_pool_str = ""; - - for my $action (@$actions) { - if ( $action->tag_name eq 'ActionConstantPool' ) { - my $pool = $action->ConstantPool; - for my $string (@$pool) { - if ( $string =~ /^https?:\/\// ) { - $saved_pool_str = $string->value; - } - } - } - elsif ( $action->tag_name eq 'ActionGetURL2' ) { - if ( $saved_pool_str ne "" ) { - $saved_swf_url = $saved_pool_str; - } - } - elsif ( $action->tag_name =~ 'ActionGetURL' ) { - $saved_swf_url = $action->UrlString->value; - } - } + my $actions = shift; + my $saved_pool_str = ""; + + for my $action (@$actions) { + if ( $action->tag_name eq 'ActionConstantPool' ) { + my $pool = $action->ConstantPool; + for my $string (@$pool) { + if ( $string =~ /^https?:\/\// ) { + $saved_pool_str = $string->value; + } + } + } + elsif ( $action->tag_name eq 'ActionGetURL2' ) { + if ( $saved_pool_str ne "" ) { + $saved_swf_url = $saved_pool_str; + } + } + elsif ( $action->tag_name =~ 'ActionGetURL' ) { + $saved_swf_url = $action->UrlString->value; + } + } } # SWF check tag utility sub swf_check_tag { - my ( $t, $stream ) = @_; - my ($tagname) = $t->tag_name; - - for ($tagname) { - ( /^Do(Init)?Action$/ or /^DefineButton$/ ) and do { - swf_search_get_url( $t->Actions ); - last; - }; - /^PlaceObject2$/ and do { - for my $ca ( @{ $t->ClipActions } ) { - swf_search_get_url( $ca->Actions ); - } - last; - }; - /^DefineButton2$/ and do { - for my $ba ( @{ $t->Actions } ) { - swf_search_get_url( $ba->Actions ); - } - last; - }; - /^DefineSprite$/ and do { - for my $tag ( @{ $t->ControlTags } ) { - swf_search_get_url( $tag, $stream ); - } - last; - }; - } + my ( $t, $stream ) = @_; + my ($tagname) = $t->tag_name; + + for ($tagname) { + ( /^Do(Init)?Action$/ or /^DefineButton$/ ) and do { + swf_search_get_url( $t->Actions ); + last; + }; + /^PlaceObject2$/ and do { + for my $ca ( @{ $t->ClipActions } ) { + swf_search_get_url( $ca->Actions ); + } + last; + }; + /^DefineButton2$/ and do { + for my $ba ( @{ $t->Actions } ) { + swf_search_get_url( $ba->Actions ); + } + last; + }; + /^DefineSprite$/ and do { + for my $tag ( @{ $t->ControlTags } ) { + swf_search_get_url( $tag, $stream ); + } + last; + }; + } } # Callback for swf parser sub swf_tag_callback { - my ( $self, $tag, $length, $stream ) = @_; - my $t = SWF::Element::Tag->new( Tag => $tag, Length => $length ); - my ($tagname) = $t->tag_name; + my ( $self, $tag, $length, $stream ) = @_; + my $t = SWF::Element::Tag->new( Tag => $tag, Length => $length ); + my ($tagname) = $t->tag_name; - return - unless $tagname eq 'DoAction' - or $tagname eq 'DoInitAction' - or $tagname eq 'PlaceObject2' - or $tagname eq 'DefineButton' - or $tagname eq 'DefineButton2' - or $tagname eq 'DefineSprite'; + return + unless $tagname eq 'DoAction' + or $tagname eq 'DoInitAction' + or $tagname eq 'PlaceObject2' + or $tagname eq 'DefineButton' + or $tagname eq 'DefineButton2' + or $tagname eq 'DefineSprite'; - if ( $tagname eq 'DefineSprite' ) { + if ( $tagname eq 'DefineSprite' ) { - # Tags in the sprite are not unpacked here. + # Tags in the sprite are not unpacked here. - $t->shallow_unpack($stream); - $t->TagStream->parse( callback => \&swf_tag_callback ); - return; + $t->shallow_unpack($stream); + $t->TagStream->parse( callback => \&swf_tag_callback ); + return; - } - elsif ( $tagname eq 'PlaceObject2' ) { + } + elsif ( $tagname eq 'PlaceObject2' ) { - # Most of PlaceObject2 tags don't have ClipActions. + # Most of PlaceObject2 tags don't have ClipActions. - $t->lookahead_Flags($stream); - return unless $t->PlaceFlagHasClipActions; - } + $t->lookahead_Flags($stream); + return unless $t->PlaceFlagHasClipActions; + } - # unpack the tag and search actions. + # unpack the tag and search actions. - $t->unpack($stream); - swf_check_tag($t); + $t->unpack($stream); + swf_check_tag($t); } # Check url from redis cache first sub redis_check_url { - my ($url) = @_; + my ($url) = @_; - my $context = Digest->new("SHA-512"); - $context->add($url); - return $redis_conn->get( $context->digest() ); + my $context = Digest->new("SHA-512"); + $context->add($url); + return $redis_conn->get( $context->digest() ); } # Write url to redis key sub redis_cache_url { - my ( $url, $url_real ) = @_; - - if ( $url ne $url_real ) { - my $context = Digest->new("SHA-512"); - $context->add($url); - if (!$redis_conn->setex( $context->digest(), $cfg{cache_expire}, $url_real)) { - _log(LOG_INFO, "cannot save redirect from $url to $url_real in redis"); - } - } + my ( $url, $url_real ) = @_; + + if ( $url ne $url_real ) { + my $context = Digest->new("SHA-512"); + $context->add($url); + if ( !$redis_conn->setex( $context->digest(), $cfg{cache_expire}, $url_real ) ) { + _log( LOG_INFO, "cannot save redirect from $url to $url_real in redis" ); + } + } } sub create_response { - my ( $code, $uri ) = @_; + my ( $code, $uri ) = @_; - my $new_response; + my $new_response; - if ($uri) { - $new_response = HTTP::Response->new( $code, 'OK' ); - $new_response->header( "Uri", $uri ); - $new_response->content($uri); - $new_response->content_length( length($uri) ); - } - else { - $new_response = HTTP::Response->new($code); - $new_response->content_length(0); - } + if ($uri) { + $new_response = HTTP::Response->new( $code, 'OK' ); + $new_response->header( "Uri", $uri ); + $new_response->content($uri); + $new_response->content_length( length($uri) ); + } + else { + $new_response = HTTP::Response->new($code); + $new_response->content_length(0); + } - $new_response->header( "Connection", "Close" ); - $new_response->header( "Proxy-Connection", "Close" ); + $new_response->header( "Connection", "Close" ); + $new_response->header( "Proxy-Connection", "Close" ); - return $new_response; + return $new_response; } # POE http client callback sub process_client { - my ( $kernel, $heap ) = @_[ KERNEL, HEAP ]; - - my $http_request = $_[ARG0]->[0]; - my $rec = $_[ARG0]->[1][0]; - my $http_response = $_[ARG1]->[0]; - my $base_url = $_[ARG0]->[1][1]; - $saved_swf_url = ""; - - if ( $rec == 0 ) { - $base_url = $http_request->uri; - } - else { - - # Check cache for each url - my $redirect = redis_check_url( $http_request->uri ); - if ($redirect) { - _log( LOG_INFO, - "Memcached redirect from %s to %s for request from: %s", - $http_response->base, $redirect, $heap->{remote_ip} ); - my $new_response = create_response( 200, $redirect ); - - # Avoid sending the response if the client has gone away. - $heap->{client}->put($new_response) if defined $heap->{client}; - - # Shut down the client's connection when the response is sent. - return; - } - } - - if ($do_reopen_log) { - $do_reopen_log = 0; - reopen_log(); - } - - if ( $rec > $cfg{max_rec} ) { - _log( LOG_INFO, - "Max recursion exceeded: %d from %s to %s for request from: %s", - $rec, $base_url, $http_request->uri, $heap->{remote_ip} ); - - # Write to cache - redis_cache_url( $base_url, $http_request->uri ); - my $new_response = create_response( 200, $http_request->uri ); - - # Avoid sending the response if the client has gone away. - $heap->{client}->put($new_response) if defined $heap->{client}; - - # Shut down the client's connection when the response is sent. - $kernel->yield("shutdown"); - return; - } - - # Detect HTTP redirects - if ( $http_response->is_redirect ) { - my $redirect = $http_response->header('Location'); - if ($redirect) { - if ( $redirect =~ /^https?:\/\// ) { - _log( LOG_INFO, - "HTML redirect from %s to %s for request from: %s", - $http_response->base, $redirect, $heap->{remote_ip} ); - my $request = HTTP::Request->new( 'GET', $redirect ); - $request->header( "Connection", "close" ); - $request->header( "Proxy-Connection", "close" ); - $kernel->post( "cl", "request", "got_response", $request, - [ $rec + 1, $base_url ] ); - return; - } - else { - _log( LOG_INFO, -"ignoring internal redirect from %s to %s for request from: %s", - $http_request->uri, - $redirect, - $heap->{remote_ip} - ); - my $new_response = create_response( 200, $http_request->uri ); - - # Avoid sending the response if the client has gone away. - $heap->{client}->put($new_response) if defined $heap->{client}; - - # Shut down the client's connection when the response is sent. - $kernel->yield("shutdown"); - return; - } - } - } - elsif ( $http_response->code != 200 ) { - _log( LOG_INFO, "HTTP response was %d, for request to %s", - $http_response->code, $http_request->uri ); - my $new_response; - if ($rec == 0) { - $new_response = create_response( $http_response->code ); - } - else { - redis_cache_url( $base_url, $http_request->uri ); - $new_response = create_response( 200, $http_request->uri ); - } - - # Avoid sending the response if the client has gone away. - $heap->{client}->put($new_response) if defined $heap->{client}; - - # Shut down the client's connection when the response is sent. - $kernel->yield("shutdown"); - return; - } - my $response_type = $http_response->content_type(); - if ( $response_type =~ /^text/i ) { - my $content = $http_response->decoded_content(); - my $p = HTML::HeadParser->new($http_response); - $p->parse($content); - my $expire = $http_response->header('Refresh'); - if ( $http_response->is_redirect || $expire ) { - my $redirect; - if ($expire) { - $expire =~ /URL=(\S+)/; - $redirect = $1; - } - else { - $redirect = $http_response->header('Location'); - } - if ($redirect) { - if ( $redirect =~ /^https?:\/\// ) { - _log( LOG_INFO, - "HTML redirect from %s to %s for request from: %s", - $http_response->base, - $redirect, - $heap->{remote_ip} - ); - my $request = HTTP::Request->new( 'GET', $redirect ); - $request->header( "Connection", "close" ); - $request->header( "Proxy-Connection", "close" ); - $kernel->post( "cl", "request", "got_response", $request, - [ $rec + 1, $base_url ] ); - return; - } - else { - _log( LOG_INFO, -"ignoring internal redirect from %s to %s for request from: %s", - $http_response->base, - $redirect, - $heap->{remote_ip} - ); - } - } - } - if ( $content =~ /location\s*=\s*["']*(https?:\/\/[^"'\s]+)["']*/im ) { - my $redir = uri_unescape($1); - _log( LOG_INFO, "js redirect from %s to %s for request from: %s", - $http_response->base, $1, $heap->{remote_ip} ); - my $request = HTTP::Request->new( 'GET', $redir ); - $request->header( "Connection", "close" ); - $request->header( "Proxy-Connection", "close" ); - $kernel->post( "cl", "request", "got_response", $request, - [ $rec + 1, $base_url ] ); - return; - } - } - elsif ( - $with_swf - && ( - $response_type eq 'application/x-shockwave-flash' - || ( $http_request->uri =~ /\.swf(\?.*)?$/i - && $http_response->code == 200 ) - ) - ) - { - my $content = $http_response->decoded_content(); - $swf_parser->parse($content); - - if ( $saved_swf_url ne "" ) { - _log( LOG_INFO, "flash redirect from %s to %s for request from: %s", - $http_response->base, $saved_swf_url, $heap->{remote_ip} ); - my $request = HTTP::Request->new( 'GET', $saved_swf_url ); - - # Reset swf redirect global variable - $saved_swf_url = ""; - $request->header( "Connection", "close" ); - $request->header( "Proxy-Connection", "close" ); - $kernel->post( "cl", "request", "got_response", $request, - [ $rec + 1, $base_url ] ); - return; - } - } - else { - _log( LOG_INFO, - "response wasn't text request from: %s, response is: %s", - $heap->{remote_ip}, $response_type ); - } - - _log( LOG_INFO, "redirect from %s to %s for request from: %s", - $base_url, $http_request->uri, $heap->{remote_ip} ); - - # Write to cache - redis_cache_url( $base_url, $http_request->uri ); - my $new_response = - create_response( $http_response->code, $http_request->uri ); - - # Avoid sending the response if the client has gone away. - $heap->{client}->put($new_response) if defined $heap->{client}; - - # Shut down the client's connection when the response is sent. - $kernel->yield("shutdown"); + my ( $kernel, $heap ) = @_[ KERNEL, HEAP ]; + + my $http_request = $_[ARG0]->[0]; + my $rec = $_[ARG0]->[1][0]; + my $http_response = $_[ARG1]->[0]; + my $base_url = $_[ARG0]->[1][1]; + $saved_swf_url = ""; + + if ( $rec == 0 ) { + $base_url = $http_request->uri; + } + else { + + # Check cache for each url + my $redirect = redis_check_url( $http_request->uri ); + if ($redirect) { + _log( LOG_INFO, "Memcached redirect from %s to %s for request from: %s", + $http_response->base, $redirect, $heap->{remote_ip} ); + my $new_response = create_response( 200, $redirect ); + + # Avoid sending the response if the client has gone away. + $heap->{client}->put($new_response) if defined $heap->{client}; + + # Shut down the client's connection when the response is sent. + return; + } + } + + if ($do_reopen_log) { + $do_reopen_log = 0; + reopen_log(); + } + + if ( $rec > $cfg{max_rec} ) { + _log( LOG_INFO, "Max recursion exceeded: %d from %s to %s for request from: %s", + $rec, $base_url, $http_request->uri, $heap->{remote_ip} ); + + # Write to cache + redis_cache_url( $base_url, $http_request->uri ); + my $new_response = create_response( 200, $http_request->uri ); + + # Avoid sending the response if the client has gone away. + $heap->{client}->put($new_response) if defined $heap->{client}; + + # Shut down the client's connection when the response is sent. + $kernel->yield("shutdown"); + return; + } + + # Detect HTTP redirects + if ( $http_response->is_redirect ) { + my $redirect = $http_response->header('Location'); + if ($redirect) { + if ( $redirect =~ /^https?:\/\// ) { + _log( LOG_INFO, "HTML redirect from %s to %s for request from: %s", + $http_response->base, $redirect, $heap->{remote_ip} ); + my $request = HTTP::Request->new( 'GET', $redirect ); + $request->header( "Connection", "close" ); + $request->header( "Proxy-Connection", "close" ); + $kernel->post( "cl", "request", "got_response", $request, [ $rec + 1, $base_url ] ); + return; + } + else { + _log( LOG_INFO, "ignoring internal redirect from %s to %s for request from: %s", + $http_request->uri, $redirect, $heap->{remote_ip} ); + my $new_response = create_response( 200, $http_request->uri ); + + # Avoid sending the response if the client has gone away. + $heap->{client}->put($new_response) if defined $heap->{client}; + + # Shut down the client's connection when the response is sent. + $kernel->yield("shutdown"); + return; + } + } + } + elsif ( $http_response->code != 200 ) { + _log( LOG_INFO, "HTTP response was %d, for request to %s", $http_response->code, $http_request->uri ); + my $new_response; + if ( $rec == 0 ) { + $new_response = create_response( $http_response->code ); + } + else { + redis_cache_url( $base_url, $http_request->uri ); + $new_response = create_response( 200, $http_request->uri ); + } + + # Avoid sending the response if the client has gone away. + $heap->{client}->put($new_response) if defined $heap->{client}; + + # Shut down the client's connection when the response is sent. + $kernel->yield("shutdown"); + return; + } + my $response_type = $http_response->content_type(); + if ( $response_type =~ /^text/i ) { + my $content = $http_response->decoded_content(); + my $p = HTML::HeadParser->new($http_response); + $p->parse($content); + my $expire = $http_response->header('Refresh'); + if ( $http_response->is_redirect || $expire ) { + my $redirect; + if ($expire) { + $expire =~ /URL=(\S+)/; + $redirect = $1; + } + else { + $redirect = $http_response->header('Location'); + } + if ($redirect) { + if ( $redirect =~ /^https?:\/\// ) { + _log( LOG_INFO, "HTML redirect from %s to %s for request from: %s", + $http_response->base, $redirect, $heap->{remote_ip} ); + my $request = HTTP::Request->new( 'GET', $redirect ); + $request->header( "Connection", "close" ); + $request->header( "Proxy-Connection", "close" ); + $kernel->post( "cl", "request", "got_response", $request, [ $rec + 1, $base_url ] ); + return; + } + else { + _log( LOG_INFO, "ignoring internal redirect from %s to %s for request from: %s", + $http_response->base, $redirect, $heap->{remote_ip} ); + } + } + } + if ( $content =~ /location\s*=\s*["']*(https?:\/\/[^"'\s]+)["']*/im ) { + my $redir = uri_unescape($1); + _log( LOG_INFO, "js redirect from %s to %s for request from: %s", + $http_response->base, $1, $heap->{remote_ip} ); + my $request = HTTP::Request->new( 'GET', $redir ); + $request->header( "Connection", "close" ); + $request->header( "Proxy-Connection", "close" ); + $kernel->post( "cl", "request", "got_response", $request, [ $rec + 1, $base_url ] ); + return; + } + } + elsif ( + $with_swf + && ( + $response_type eq 'application/x-shockwave-flash' + || ( $http_request->uri =~ /\.swf(\?.*)?$/i + && $http_response->code == 200 ) + ) + ) + { + my $content = $http_response->decoded_content(); + $swf_parser->parse($content); + + if ( $saved_swf_url ne "" ) { + _log( LOG_INFO, "flash redirect from %s to %s for request from: %s", + $http_response->base, $saved_swf_url, $heap->{remote_ip} ); + my $request = HTTP::Request->new( 'GET', $saved_swf_url ); + + # Reset swf redirect global variable + $saved_swf_url = ""; + $request->header( "Connection", "close" ); + $request->header( "Proxy-Connection", "close" ); + $kernel->post( "cl", "request", "got_response", $request, [ $rec + 1, $base_url ] ); + return; + } + } + else { + _log( LOG_INFO, "response wasn't text request from: %s, response is: %s", $heap->{remote_ip}, $response_type ); + } + + _log( LOG_INFO, "redirect from %s to %s for request from: %s", $base_url, $http_request->uri, $heap->{remote_ip} ); + + # Write to cache + redis_cache_url( $base_url, $http_request->uri ); + my $new_response = create_response( $http_response->code, $http_request->uri ); + + # Avoid sending the response if the client has gone away. + $heap->{client}->put($new_response) if defined $heap->{client}; + + # Shut down the client's connection when the response is sent. + $kernel->yield("shutdown"); } sub process_input { - my ( $kernel, $heap, $request ) = @_[ KERNEL, HEAP, ARG0 ]; - - if ( $request->isa("HTTP::Response") ) { - $heap->{client}->put($request); - $kernel->yield("shutdown"); - return; - } - - my $domain; - if ( $request->uri =~ /^http:\/\/([^\/]+)\// ) { - my @parts = split( /\./, $1 ); - my $c1 = pop @parts; - my $c2 = pop @parts; - $domain = "$c2.$c1"; - } - - if ( - ( - defined( $cfg{check_regexp} ) && $request->uri !~ $cfg{check_regexp} - ) - || ( defined( $cfg{check_domains} ) - && scalar( grep { $_ eq $domain } @{ $cfg{check_domains} } ) == 0 ) - ) - { - my $new_response = create_response( 200, $request->uri ); - - # Avoid sending the response if the client has gone away. - $heap->{client}->put($new_response) if defined $heap->{client}; - $kernel->yield("shutdown"); - - # Shut down the client's connection when the response is sent. - return; - } - - # Check cache first - my $redirect = redis_check_url( $request->uri ); - if ($redirect) { - _log( LOG_INFO, "Memcached redirect from %s to %s for request from: %s", - $request->uri, $redirect, $heap->{remote_ip} ); - my $new_response = create_response( 200, $redirect ); - - # Avoid sending the response if the client has gone away. - $heap->{client}->put($new_response) if defined $heap->{client}; - $kernel->yield("shutdown"); - - # Shut down the client's connection when the response is sent. - return; - } - - # Start http request - my $new_request = HTTP::Request->new( 'GET', $request->uri ); - $new_request->header( "Connection", "close" ); - $new_request->header( "Proxy-Connection", "close" ); - $kernel->post( "cl", "request", "got_response", $new_request, [ 0, "" ] ); + my ( $kernel, $heap, $request ) = @_[ KERNEL, HEAP, ARG0 ]; + + if ( $request->isa("HTTP::Response") ) { + $heap->{client}->put($request); + $kernel->yield("shutdown"); + return; + } + + my $domain; + if ( $request->uri =~ /^http:\/\/([^\/]+)\// ) { + my @parts = split( /\./, $1 ); + my $c1 = pop @parts; + my $c2 = pop @parts; + $domain = "$c2.$c1"; + } + + if ( + ( defined( $cfg{check_regexp} ) && $request->uri !~ $cfg{check_regexp} ) + || ( defined( $cfg{check_domains} ) + && scalar( grep { $_ eq $domain } @{ $cfg{check_domains} } ) == 0 ) + ) + { + my $new_response = create_response( 200, $request->uri ); + + # Avoid sending the response if the client has gone away. + $heap->{client}->put($new_response) if defined $heap->{client}; + $kernel->yield("shutdown"); + + # Shut down the client's connection when the response is sent. + return; + } + + # Check cache first + my $redirect = redis_check_url( $request->uri ); + if ($redirect) { + _log( LOG_INFO, "Memcached redirect from %s to %s for request from: %s", + $request->uri, $redirect, $heap->{remote_ip} ); + my $new_response = create_response( 200, $redirect ); + + # Avoid sending the response if the client has gone away. + $heap->{client}->put($new_response) if defined $heap->{client}; + $kernel->yield("shutdown"); + + # Shut down the client's connection when the response is sent. + return; + } + + # Start http request + my $new_request = HTTP::Request->new( 'GET', $request->uri ); + $new_request->header( "Connection", "close" ); + $new_request->header( "Proxy-Connection", "close" ); + $kernel->post( "cl", "request", "got_response", $new_request, [ 0, "" ] ); } sub sig_DIE { - my ( $sig, $ex ) = @_[ ARG0, ARG1 ]; - _log( LOG_ERR, "$$: error in $ex->{event}: $ex->{error_str}" ); - $poe_kernel->sig_handled(); - - # Send the signal to session that sent the original event. - if ( $ex->{source_session} ne $_[SESSION] ) { - $poe_kernel->signal( $ex->{source_session}, 'DIE', $sig, $ex ); - } + my ( $sig, $ex ) = @_[ ARG0, ARG1 ]; + _log( LOG_ERR, "$$: error in $ex->{event}: $ex->{error_str}" ); + $poe_kernel->sig_handled(); + + # Send the signal to session that sent the original event. + if ( $ex->{source_session} ne $_[SESSION] ) { + $poe_kernel->signal( $ex->{source_session}, 'DIE', $sig, $ex ); + } } sub sig_CLD { - my ( $heap, $child_pid ) = @_[ HEAP, ARG1 ]; - return 0; + my ( $heap, $child_pid ) = @_[ HEAP, ARG1 ]; + return 0; } ############################### Main code fragment ################################## # Do daemonization if ( !$DEBUG ) { - Proc::Daemon::Init; - POE::Kernel->has_forked; - setlogsock('unix'); - openlog( 'redirector', 'ndelay,pid', $cfg{'facility'} ); + Proc::Daemon::Init; + POE::Kernel->has_forked; + setlogsock('unix'); + openlog( 'redirector', 'ndelay,pid', $cfg{'facility'} ); } # Try to eval config file if ( -f $cfg{cfg_file} ) { - my $config = read_file( $cfg{cfg_file} ); - eval $config; + my $config = read_file( $cfg{cfg_file} ); + eval $config; } _log( LOG_ALERT, "Process is already started, check $cfg{pidfile}" ) @@ -528,18 +502,18 @@ _log( LOG_ALERT, "Process is already started, check $cfg{pidfile}" ) # Drop privilleges if ( $> == 0 ) { - my $uid = getpwnam( $cfg{user} ) - or _log( LOG_ALERT, "user $cfg{user} unknown" ); - my $gid = getgrnam( $cfg{group} ) - or _log( LOG_ALERT, "group $cfg{group} unknown" ); - $< = $> = $uid; - $) = $( = $gid; + my $uid = getpwnam( $cfg{user} ) + or _log( LOG_ALERT, "user $cfg{user} unknown" ); + my $gid = getgrnam( $cfg{group} ) + or _log( LOG_ALERT, "group $cfg{group} unknown" ); + $< = $> = $uid; + $) = $( = $gid; } if ( !$DEBUG ) { - _log( LOG_ALERT, "Cannot write to pidfile $cfg{pidfile}" ) - if !open( PID, "> $cfg{pidfile}" ); - close(PID); + _log( LOG_ALERT, "Cannot write to pidfile $cfg{pidfile}" ) + if !open( PID, "> $cfg{pidfile}" ); + close(PID); } # Reopen log on SIGUSR1 @@ -551,40 +525,40 @@ $SIG{QUIT} = sub { $poe_kernel->stop(); }; $SIG{PIPE} = 'IGNORE'; if ( !$DEBUG ) { - Proc::PidUtil::make_pidfile( $cfg{pidfile}, $$ ) - or _log( LOG_ALERT, "Cannot write pidfile $cfg{pidfile}" ); + Proc::PidUtil::make_pidfile( $cfg{pidfile}, $$ ) + or _log( LOG_ALERT, "Cannot write pidfile $cfg{pidfile}" ); } # Init redis connection _log( LOG_INFO, "Starting redis connection" ); $redis_conn = Redis::Fast->new( - server => $cfg{redis_server}, - reconnect => 60, - every => 500_000, - encoding => undef, + server => $cfg{redis_server}, + reconnect => 60, + every => 500_000, + encoding => undef, ); # POE part POE::Component::Client::HTTP->spawn( - Alias => 'cl', - MaxSize => $cfg{max_size}, # Remove for unlimited page sizes - Timeout => $cfg{http_timeout}, - ConnectionManager => POE::Component::Client::Keepalive->new( - max_per_host => 256, - max_open => 1024, - keep_alive => 1, - timeout => $cfg{http_timeout}, - ), + Alias => 'cl', + MaxSize => $cfg{max_size}, # Remove for unlimited page sizes + Timeout => $cfg{http_timeout}, + ConnectionManager => POE::Component::Client::Keepalive->new( + max_per_host => 256, + max_open => 1024, + keep_alive => 1, + timeout => $cfg{http_timeout}, + ), ); _log( LOG_INFO, "Starting HTTP server" ); POE::Component::Server::TCP->new( - Alias => "", - Port => $cfg{port}, - ClientFilter => 'POE::Filter::HTTPD', + Alias => "", + Port => $cfg{port}, + ClientFilter => 'POE::Filter::HTTPD', - ClientInput => \&process_input, - InlineStates => { got_response => \&process_client, }, + ClientInput => \&process_input, + InlineStates => { got_response => \&process_client, }, ); swf_init_parser(); @@ -597,6 +571,6 @@ exit 0; ############################## Final block #################################### END { - _log( LOG_NOTICE, 'redirector stopped' ); - closelog(); + _log( LOG_NOTICE, 'redirector stopped' ); + closelog(); } diff --git a/utils/rspamd_stats.pl b/utils/rspamd_stats.pl index ac7b1349f..a6d603709 100755 --- a/utils/rspamd_stats.pl +++ b/utils/rspamd_stats.pl @@ -15,20 +15,20 @@ my @symbols_bidirectional; my @symbols_groups; my @symbols_ignored; my %groups; -my $reject_score = 15.0; -my $junk_score = 6.0; -my $diff_alpha = 0.1; -my $correlations = 0; -my $nrelated = 10; -my $log_file = ""; +my $reject_score = 15.0; +my $junk_score = 6.0; +my $diff_alpha = 0.1; +my $correlations = 0; +my $nrelated = 10; +my $log_file = ""; my $search_pattern = ""; -my $startTime=""; +my $startTime = ""; my $endTime; my $num_logs; my $exclude_logs = 0; -my $man = 0; -my $json = 0; -my $help = 0; +my $man = 0; +my $json = 0; +my $help = 0; # Associate file extensions with decompressors my %decompressor = ( @@ -39,44 +39,43 @@ my %decompressor = ( ); GetOptions( - "reject-score|r=f" => \$reject_score, - "junk-score|j=f" => \$junk_score, - "symbol|s=s@" => \@symbols_search, - "symbol-bidir|S=s@" => \@symbols_bidirectional, - "exclude|X=s@" => \@symbols_exclude, - "ignore=s@" => \@symbols_ignored, - "group|g=s@" => \@symbols_groups, - "log|l=s" => \$log_file, - "alpha-score|alpha|a=f" => \$diff_alpha, - "correlations|c" => \$correlations, - "nrelated=i" => \$nrelated, - "search-pattern=s" => \$search_pattern, - "start=s" => \$startTime, - "end=s" => \$endTime, - "num-logs|n=i" => \$num_logs, - "exclude-logs|x=i" => \$exclude_logs, - "json|j" => \$json, - "help|?" => \$help, - "man" => \$man + "reject-score|r=f" => \$reject_score, + "junk-score|j=f" => \$junk_score, + "symbol|s=s@" => \@symbols_search, + "symbol-bidir|S=s@" => \@symbols_bidirectional, + "exclude|X=s@" => \@symbols_exclude, + "ignore=s@" => \@symbols_ignored, + "group|g=s@" => \@symbols_groups, + "log|l=s" => \$log_file, + "alpha-score|alpha|a=f" => \$diff_alpha, + "correlations|c" => \$correlations, + "nrelated=i" => \$nrelated, + "search-pattern=s" => \$search_pattern, + "start=s" => \$startTime, + "end=s" => \$endTime, + "num-logs|n=i" => \$num_logs, + "exclude-logs|x=i" => \$exclude_logs, + "json|j" => \$json, + "help|?" => \$help, + "man" => \$man ) or pod2usage(2); pod2usage(1) if $help; -pod2usage(-exitval => 0, -verbose => 2) if $man; - +pod2usage( -exitval => 0, -verbose => 2 ) if $man; # Global vars -my $total = 0; -my $total_spam = 0; -my $total_junk = 0; -my $junk_symbols = 0; -my $spam_symbols = 0; -my $ham_symbols = 0; +my $total = 0; +my $total_spam = 0; +my $total_junk = 0; +my $junk_symbols = 0; +my $spam_symbols = 0; +my $ham_symbols = 0; my $ham_spam_change = 0; my $ham_junk_change = 0; my %sym_res; my $rspamd_log; -my $enabled = 0; -my $log_file_num = 1; +my $enabled = 0; +my $log_file_num = 1; my $spinner_update_time = 0; my %action; @@ -91,688 +90,690 @@ foreach ( $startTime, $endTime ) { $_ = &normalized_time($_) } # Convert bidirectional symbols foreach my $s (@symbols_bidirectional) { - $bidir_match{$s} = { - spam => "${s}_SPAM", - ham => "${s}_HAM", - }; - push @symbols_search, $s unless grep /^$s$/, @symbols_search; + $bidir_match{$s} = { + spam => "${s}_SPAM", + ham => "${s}_HAM", + }; + push @symbols_search, $s unless grep /^$s$/, @symbols_search; } # Deal with groups my $group_id = 0; foreach my $g (@symbols_groups) { - my @symbols = split /,/,$g; - my $group_name = "group$group_id"; + my @symbols = split /,/, $g; + my $group_name = "group$group_id"; - foreach my $s (@symbols) { - $groups{$s} = $group_name; - push @symbols_search, $s unless grep /^$s$/, @symbols_search; - } + foreach my $s (@symbols) { + $groups{$s} = $group_name; + push @symbols_search, $s unless grep /^$s$/, @symbols_search; + } } @symbols_search = '.*' unless @symbols_search; -if ($log_file eq '-' || $log_file eq '') { - $rspamd_log = \*STDIN; - &ProcessLog(); +if ( $log_file eq '-' || $log_file eq '' ) { + $rspamd_log = \*STDIN; + &ProcessLog(); } elsif ( -d "$log_file" ) { - my $log_dir = "$log_file"; + my $log_dir = "$log_file"; - my @logs = &GetLogfilesList($log_dir); + my @logs = &GetLogfilesList($log_dir); - # Process logs - foreach (@logs) { - my $ext = (/[^.]+\.?([^.]*?)$/)[0]; - my $dc = $decompressor{$ext} || 'cat'; + # Process logs + foreach (@logs) { + my $ext = (/[^.]+\.?([^.]*?)$/)[0]; + my $dc = $decompressor{$ext} || 'cat'; - open( $rspamd_log, "-|", "$dc $log_dir/$_" ) - or die "cannot execute $dc $log_dir/$_ : $!"; + open( $rspamd_log, "-|", "$dc $log_dir/$_" ) + or die "cannot execute $dc $log_dir/$_ : $!"; - printf {interactive(*STDERR)} "\033[J Parsing log files: [%d/%d] %s\033[G", $log_file_num++, scalar @logs, $_; - $spinner_update_time = 0; # Force spinner update - &spinner; + printf { interactive(*STDERR) } "\033[J Parsing log files: [%d/%d] %s\033[G", $log_file_num++, scalar @logs, + $_; + $spinner_update_time = 0; # Force spinner update + &spinner; - &ProcessLog; + &ProcessLog; - close($rspamd_log) - or warn "cannot close $dc $log_dir/$_: $!"; - } - print {interactive(*STDERR)} "\033[J\033[G"; # Progress indicator clean-up + close($rspamd_log) + or warn "cannot close $dc $log_dir/$_: $!"; + } + print { interactive(*STDERR) } "\033[J\033[G"; # Progress indicator clean-up } else { - my $ext = ($log_file =~ /[^.]+\.?([^.]*?)$/)[0]; - my $dc = $decompressor{$ext} || 'cat'; - open( $rspamd_log, "-|", "$dc $log_file" ) - or die "cannot execute $dc $log_file : $!"; - $spinner_update_time = 0; # Force spinner update - &spinner; - &ProcessLog(); + my $ext = ( $log_file =~ /[^.]+\.?([^.]*?)$/ )[0]; + my $dc = $decompressor{$ext} || 'cat'; + open( $rspamd_log, "-|", "$dc $log_file" ) + or die "cannot execute $dc $log_file : $!"; + $spinner_update_time = 0; # Force spinner update + &spinner; + &ProcessLog(); } -my $total_ham = $total - ($total_spam + $total_junk); +my $total_ham = $total - ( $total_spam + $total_junk ); if ($json) { - print "{"; - &Summary(); - print '"symbols":{'; - &SymbolsStat(); - print "}}\n"; + print "{"; + &Summary(); + print '"symbols":{'; + &SymbolsStat(); + print "}}\n"; } else { - &SymbolsStat(); - &Summary(); + &SymbolsStat(); + &Summary(); } exit; sub IsIgnored { - my ($sym) = @_; + my ($sym) = @_; - foreach my $ex (@symbols_ignored) { - if ($sym =~ /^$ex$/) { - return 1; + foreach my $ex (@symbols_ignored) { + if ( $sym =~ /^$ex$/ ) { + return 1; + } } - } - return 0; + return 0; } sub GenRelated { - my ($htb, $target_sym) = @_; - - my @result; - my $i = 0; - foreach my $sym (sort { $htb->{$b} <=> $htb->{$a} } keys %{$htb}) { - if ($sym ne $target_sym) { - my @elt = ($sym, $htb->{$sym}); - push @result, \@elt; - $i ++; - } + my ( $htb, $target_sym ) = @_; + + my @result; + my $i = 0; + foreach my $sym ( sort { $htb->{$b} <=> $htb->{$a} } keys %{$htb} ) { + if ( $sym ne $target_sym ) { + my @elt = ( $sym, $htb->{$sym} ); + push @result, \@elt; + $i++; + } - last if $i > $nrelated; - } + last if $i > $nrelated; + } - return \@result; + return \@result; } sub StringifyRelated { - my ($ar, $total) = @_; - return join("\n", (map { sprintf "\t%s(%s: %.1f%%)", - $_->[0], $_->[1], $_->[1] / ($total * 1.0) * 100.0 } @{$ar})); + my ( $ar, $total ) = @_; + return + join( "\n", ( map { sprintf "\t%s(%s: %.1f%%)", $_->[0], $_->[1], $_->[1] / ( $total * 1.0 ) * 100.0 } @{$ar} ) ); } sub SymbolsStat { - if ($total > 0) { - my $has_comma = 0; - while (my ($s, $r) = each(%sym_res)) { - if ($r->{hits} > 0) { - my $th = $r->{hits}; - my $sh = $r->{spam_hits}; - my $jh = $r->{junk_hits}; - my $hh = $r->{hits} - $sh - $jh; - my $htp = $hh * 100.0 / $total_ham if $total_ham != 0; - my $stp = $sh * 100.0 / $total_spam if $total_spam != 0; - my $jtp = $jh * 100.0 / $total_junk if $total_junk != 0; - - if ($json) { - if ($has_comma) { - print ","; - } - else { - $has_comma = 1; - } - print "\"$s\":{"; - JsonObjectElt("avg_weight", $r->{'weight'},"%.4f"); - print ","; - JsonObjectElt("hits", $th, "%d"); - print ","; - JsonObjectElt("hits_percentage", $th/$total, "%.4f"); - print ","; - JsonObjectElt("spam_hits", $sh, "%d"); - print ","; - JsonObjectElt("spam_to_total", $sh/$th, "%.4f"); - print ","; - JsonObjectElt("spam_percentage", $stp/100.0 || 0, "%.4f"); - print ","; - JsonObjectElt("ham_hits", $hh, "%d"); - print ","; - JsonObjectElt("ham_to_total", $hh/$th, "%.4f"); - print ","; - JsonObjectElt("ham_percentage", $htp/100.0 || 0, "%.4f"); - print ","; - JsonObjectElt("junk_hits", $jh, "%d"); - print ","; - JsonObjectElt("junk_to_total", $jh/$th, "%.4f"); - print ","; - JsonObjectElt("junk_percentage", $jtp/100.0 || 0, "%.4f"); - } - else { - printf "%s avg. weight %.3f, hits %d(%.3f%%): + if ( $total > 0 ) { + my $has_comma = 0; + while ( my ( $s, $r ) = each(%sym_res) ) { + if ( $r->{hits} > 0 ) { + my $th = $r->{hits}; + my $sh = $r->{spam_hits}; + my $jh = $r->{junk_hits}; + my $hh = $r->{hits} - $sh - $jh; + my ( $htp, $stp, $jtp ); + $htp = $hh * 100.0 / $total_ham if $total_ham != 0; + $stp = $sh * 100.0 / $total_spam if $total_spam != 0; + $jtp = $jh * 100.0 / $total_junk if $total_junk != 0; + + if ($json) { + if ($has_comma) { + print ","; + } + else { + $has_comma = 1; + } + print "\"$s\":{"; + JsonObjectElt( "avg_weight", $r->{'weight'}, "%.4f" ); + print ","; + JsonObjectElt( "hits", $th, "%d" ); + print ","; + JsonObjectElt( "hits_percentage", $th / $total, "%.4f" ); + print ","; + JsonObjectElt( "spam_hits", $sh, "%d" ); + print ","; + JsonObjectElt( "spam_to_total", $sh / $th, "%.4f" ); + print ","; + JsonObjectElt( "spam_percentage", $stp / 100.0 || 0, "%.4f" ); + print ","; + JsonObjectElt( "ham_hits", $hh, "%d" ); + print ","; + JsonObjectElt( "ham_to_total", $hh / $th, "%.4f" ); + print ","; + JsonObjectElt( "ham_percentage", $htp / 100.0 || 0, "%.4f" ); + print ","; + JsonObjectElt( "junk_hits", $jh, "%d" ); + print ","; + JsonObjectElt( "junk_to_total", $jh / $th, "%.4f" ); + print ","; + JsonObjectElt( "junk_percentage", $jtp / 100.0 || 0, "%.4f" ); + } + else { + printf "%s avg. weight %.3f, hits %d(%.3f%%): Ham %7.3f%%, %6d/%-6d (%7.3f%%) Spam %7.3f%%, %6d/%-6d (%7.3f%%) Junk %7.3f%%, %6d/%-6d (%7.3f%%) -", - $s, $r->{weight} / $r->{hits}, $th, ($th / $total * 100), - ($hh / $th * 100), $hh, $total_ham, ($htp or 0), - ($sh / $th * 100), $sh, $total_spam, ($stp or 0), - ($jh / $th * 100), $jh, $total_junk, ($jtp or 0); - } - - my $schp = $r->{spam_change} / $total_spam * 100.0 if $total_spam; - my $jchp = $r->{junk_change} / $total_junk * 100.0 if $total_junk; - - if ($r->{weight} != 0) { - if (!$json) { - if ($r->{weight} > 0) { - printf " +", $s, $r->{weight} / $r->{hits}, $th, ( $th / $total * 100 ), + ( $hh / $th * 100 ), $hh, $total_ham, ( $htp or 0 ), + ( $sh / $th * 100 ), $sh, $total_spam, ( $stp or 0 ), + ( $jh / $th * 100 ), $jh, $total_junk, ( $jtp or 0 ); + } + my ( $schp, $jchp ); + $schp = $r->{spam_change} / $total_spam * 100.0 if $total_spam; + $jchp = $r->{junk_change} / $total_junk * 100.0 if $total_junk; + + if ( $r->{weight} != 0 ) { + if ( !$json ) { + if ( $r->{weight} > 0 ) { + printf " Spam changes (ham/junk -> spam): %6d/%-6d (%7.3f%%) Spam changes / total spam hits: %6d/%-6d (%7.3f%%) Junk changes (ham -> junk): %6d/%-6d (%7.3f%%) Junk changes / total junk hits: %6d/%-6d (%7.3f%%) ", - $r->{spam_change}, $th, ($r->{spam_change} / $th * 100), - $r->{spam_change}, $total_spam, ($schp or 0), - $r->{junk_change}, $th, ($r->{junk_change} / $th * 100), - $r->{junk_change}, $total_junk, ($jchp or 0); - } - else { - printf " + $r->{spam_change}, $th, ( $r->{spam_change} / $th * 100 ), + $r->{spam_change}, $total_spam, ( $schp or 0 ), + $r->{junk_change}, $th, ( $r->{junk_change} / $th * 100 ), + $r->{junk_change}, $total_junk, ( $jchp or 0 ); + } + else { + printf " Spam changes (spam -> junk/ham): %6d/%-6d (%7.3f%%) Spam changes / total spam hits : %6d/%-6d (%7.3f%%) Junk changes (junk -> ham) : %6d/%-6d (%7.3f%%) Junk changes / total junk hits : %6d/%-6d (%7.3f%%) ", - $r->{spam_change}, $th, ($r->{spam_change} / $th * 100), - $r->{spam_change}, $total_spam, ($schp or 0), - $r->{junk_change}, $th, ($r->{junk_change} / $th * 100), - $r->{junk_change}, $total_junk, ($jchp or 0); - } - } - else { - print ","; - JsonObjectElt("spam_change", $r->{spam_change}, "%.4f"); - print ","; - JsonObjectElt("junk_change", $r->{junk_change}, "%.4f"); - } - } - - if ($correlations) { - - my $spam_related = GenRelated($r->{symbols_met_spam}, $s); - my $junk_related = GenRelated($r->{symbols_met_junk}, $s); - my $ham_related = GenRelated($r->{symbols_met_ham}, $s); - - if (!$json) { - print "Correlations report:\n"; - - while (my ($cs, $hits) = each %{$r->{corr}}) { - my $corr_prob = $r->{'hits'} / $total; - my $merged_hits = 0; - if($r->{symbols_met_spam}->{$cs}) { - $merged_hits += $r->{symbols_met_spam}->{$cs}; - } - if($r->{symbols_met_junk}->{$cs}) { - $merged_hits += $r->{symbols_met_junk}->{$cs}; - } - if($r->{symbols_met_ham}->{$cs}) { - $merged_hits += $r->{symbols_met_ham}->{$cs}; - } - - if ($merged_hits > 0) { - printf "Probability of %s when %s fires: %.3f\n", $cs, $s, - (($merged_hits / $total) / $corr_prob); - } - } + $r->{spam_change}, $th, ( $r->{spam_change} / $th * 100 ), + $r->{spam_change}, $total_spam, ( $schp or 0 ), + $r->{junk_change}, $th, ( $r->{junk_change} / $th * 100 ), + $r->{junk_change}, $total_junk, ( $jchp or 0 ); + } + } + else { + print ","; + JsonObjectElt( "spam_change", $r->{spam_change}, "%.4f" ); + print ","; + JsonObjectElt( "junk_change", $r->{junk_change}, "%.4f" ); + } + } - print "Related symbols report:\n"; - printf "Top related in spam:\n %s\n", StringifyRelated($spam_related, - $r->{spam_hits}); - printf "Top related in junk:\n %s\n", StringifyRelated($junk_related, - $r->{junk_hits}); - printf "Top related in ham:\n %s\n", StringifyRelated($ham_related, - $r->{hits} - $r->{spam_hits} - $r->{junk_hits}); - } - else { - print ","; - print "\"correllations\":{"; + if ($correlations) { + + my $spam_related = GenRelated( $r->{symbols_met_spam}, $s ); + my $junk_related = GenRelated( $r->{symbols_met_junk}, $s ); + my $ham_related = GenRelated( $r->{symbols_met_ham}, $s ); + + if ( !$json ) { + print "Correlations report:\n"; + + while ( my ( $cs, $hits ) = each %{ $r->{corr} } ) { + my $corr_prob = $r->{'hits'} / $total; + my $merged_hits = 0; + if ( $r->{symbols_met_spam}->{$cs} ) { + $merged_hits += $r->{symbols_met_spam}->{$cs}; + } + if ( $r->{symbols_met_junk}->{$cs} ) { + $merged_hits += $r->{symbols_met_junk}->{$cs}; + } + if ( $r->{symbols_met_ham}->{$cs} ) { + $merged_hits += $r->{symbols_met_ham}->{$cs}; + } + + if ( $merged_hits > 0 ) { + printf "Probability of %s when %s fires: %.3f\n", $cs, $s, + ( ( $merged_hits / $total ) / $corr_prob ); + } + } + + print "Related symbols report:\n"; + printf "Top related in spam:\n %s\n", StringifyRelated( $spam_related, $r->{spam_hits} ); + printf "Top related in junk:\n %s\n", StringifyRelated( $junk_related, $r->{junk_hits} ); + printf "Top related in ham:\n %s\n", + StringifyRelated( $ham_related, $r->{hits} - $r->{spam_hits} - $r->{junk_hits} ); + } + else { + print ","; + print "\"correllations\":{"; + + my $has_comma_ = 0; + while ( my ( $cs, $hits ) = each %{ $r->{corr} } ) { + if ($has_comma_) { + print ","; + } + else { + $has_comma_ = 1; + } + my $corr_prob = $hits / $total; + my $sym_prob = $r->{hits} / $total; + JsonObjectElt( $cs, ( $corr_prob / $sym_prob ), "%.4f" ); + } + + print "}"; + } + } - my $has_comma_ = 0; - while (my ($cs, $hits) = each %{$r->{corr}}) { - if ($has_comma_) { - print ","; - } - else { - $has_comma_ = 1; - } - my $corr_prob = $hits / $total; - my $sym_prob = $r->{hits} / $total; - JsonObjectElt($cs, ($corr_prob / $sym_prob) ,"%.4f"); + print "}" if $json; + } + else { + print "Symbol $s has not been met\n" if !$json; } - print "}"; - } + print '-' x 80 . "\n" if !$json; } - - print "}" if $json; - } - else { - print "Symbol $s has not been met\n" if !$json; - } - - print '-' x 80 . "\n" if !$json; } - } } sub Summary() { - if (!$json) { - print " + if ( !$json ) { + print " === Summary ", '=' x 68, " Messages scanned: $total"; - printf " [ %s / %s ] + printf " [ %s / %s ] ", $timeStamp{'start'}, $timeStamp{'end'} - if defined $timeStamp{'start'}; - say ''; - printf "%11s: %6.2f%%, %d\n", $_, 100 * $action{$_} / $total, $action{$_} - for sort keys %action; - say ''; - printf "scan time min/avg/max = %.2f/%.2f/%.2f s -", $scanTime{'min'} / 1000, - ($total) ? $scanTime{'total'} / $total / 1000 : undef, - $scanTime{'max'} / 1000 - if exists $scanTime{'min'}; - say '=' x 80; - } - else { - JsonObjectElt("total", $total, "%d"); - print ","; - - if (defined $timeStamp{'start'}) { - JsonObjectElt("start", $timeStamp{'start'}); - print ","; + if defined $timeStamp{'start'}; + say ''; + printf "%11s: %6.2f%%, %d\n", $_, 100 * $action{$_} / $total, $action{$_} for sort keys %action; + say ''; + printf "scan time min/avg/max = %.2f/%.2f/%.2f s +", $scanTime{'min'} / 1000, ($total) ? $scanTime{'total'} / $total / 1000 : undef, $scanTime{'max'} / 1000 + if exists $scanTime{'min'}; + say '=' x 80; } + else { + JsonObjectElt( "total", $total, "%d" ); + print ","; - if (defined $timeStamp{'end'}) { - JsonObjectElt("end", $timeStamp{'end'}); - print ","; - } + if ( defined $timeStamp{'start'} ) { + JsonObjectElt( "start", $timeStamp{'start'} ); + print ","; + } - print "\"actions\":{"; + if ( defined $timeStamp{'end'} ) { + JsonObjectElt( "end", $timeStamp{'end'} ); + print ","; + } - my $has_comma = 0; - foreach my $a (sort keys %action) { - if ($has_comma) { - print ","; - } - else { - $has_comma = 1; - } - JsonObjectElt($a, $action{$a}, "%d"); + print "\"actions\":{"; + + my $has_comma = 0; + foreach my $a ( sort keys %action ) { + if ($has_comma) { + print ","; + } + else { + $has_comma = 1; + } + JsonObjectElt( $a, $action{$a}, "%d" ); + } + print "},"; } - print "},"; - } } sub ProcessRelated { - my ($symbols, $target, $source) = @_; + my ( $symbols, $target, $source ) = @_; - foreach my $s (@{$symbols}) { - $s =~ /^([^\(]+)(\(([^\)]+)\))?/; - my $sym_name = $1; - my $sym_score = 0; + foreach my $s ( @{$symbols} ) { + $s =~ /^([^\(]+)(\(([^\)]+)\))?/; + my $sym_name = $1; + my $sym_score = 0; - if ($groups{$sym_name}) { - $sym_name = $groups{$sym_name}; - } + if ( $groups{$sym_name} ) { + $sym_name = $groups{$sym_name}; + } - next if ($source eq $sym_name); + next if ( $source eq $sym_name ); - next if IsIgnored($sym_name); + next if IsIgnored($sym_name); - if ($2) { - $sym_score = $3 * 1.0; + if ($2) { + $sym_score = $3 * 1.0; - if (abs($sym_score) < $diff_alpha) { - next; - } + if ( abs($sym_score) < $diff_alpha ) { + next; + } + + my $bm = $bidir_match{$sym_name}; + if ($bm) { + if ( $sym_score >= 0 ) { + $sym_name = $bm->{'spam'}; + } + else { + $sym_name = $bm->{'ham'}; + } + } + } - my $bm = $bidir_match{$sym_name}; - if ($bm) { - if ($sym_score >= 0) { - $sym_name = $bm->{'spam'}; + if ( exists( $target->{$sym_name} ) ) { + $target->{$sym_name}++; } else { - $sym_name = $bm->{'ham'}; + $target->{$sym_name} = 1; } - } - } - - if (exists($target->{$sym_name})) { - $target->{$sym_name} ++; - } - else { - $target->{$sym_name} = 1; } - } } sub ProcessLog { - my ( $ts_format, @line ) = &log_time_format($rspamd_log); + my ( $ts_format, @line ) = &log_time_format($rspamd_log); - while() { - last if eof $rspamd_log; - $_ = (@line) ? shift @line : <$rspamd_log>; + while () { + last if eof $rspamd_log; + $_ = (@line) ? shift @line : <$rspamd_log>; - if (!$enabled && ($search_pattern eq "" || /$search_pattern/)) { - $enabled = 1; - } - - next if !$enabled; - - if (/^.*rspamd_task_write_log.*$/) { - &spinner; - my $ts; - if ( $ts_format eq 'syslog' ) { - $ts = syslog2iso( join ' ', ( split /\s+/ )[ 0 .. 2 ] ); - } elsif ( $ts_format eq 'syslog5424' ) { - /^([0-9-]+)T([0-9:]+)/; - $ts = "$1 $2"; - } else { - $ts = join ' ', ( split /\s+/ )[ 0 .. 1 ]; - } - - next if ( $ts lt $startTime ); - next if ( defined $endTime && $ts gt $endTime ); - - if ($_ !~ /\(([^()]+)\): \[(NaN|-?\d+(?:\.\d+)?)\/(-?\d+(?:\.\d+)?)\]\s+\[([^\]]+)\].+? time: (\d+\.\d+)ms real/) { - #print "BAD: $_\n"; - next; - } - - my @symbols = split /(?:\{[^}]*\})?(?:$|,)/, $4; - my $scan_time = $5; - my $act = $1; - my $score = $2 * 1.0; - my $skip = 0; - - foreach my $ex (@symbols_exclude) { - my @found = grep {/^$ex/} @symbols; - - if (scalar(@found) > 0) { - $skip = 1; - last; + if ( !$enabled && ( $search_pattern eq "" || /$search_pattern/ ) ) { + $enabled = 1; } - } - - next if ( $skip != 0 ); - - if (defined($timeStamp{'end'})) { - $timeStamp{'end'} = $ts if ( $ts gt $timeStamp{'end'} ); - } - else { - $timeStamp{'end'} = $ts; - } - - if (defined($timeStamp{'start'})) { - $timeStamp{'start'} = $ts if ( $ts lt $timeStamp{'start'} ); - } - else { - $timeStamp{'start'} = $ts; - } - - $scanTime{'min'} = $scan_time if ( !exists $scanTime{'min'} || $scanTime{'min'} > $scan_time ); - $scanTime{'max'} = $scan_time if ( $scanTime{'max'} < $scan_time ); - $scanTime{'total'} += $scan_time; - - $action{$act}++; - $total ++; - if ($score >= $reject_score) { - $total_spam ++; - } - elsif ($score >= $junk_score) { - $total_junk ++; - } + next if !$enabled; - my @sym_names; - - foreach my $s (@symbols_search) { - my @selected = grep /$s/, @symbols; + if (/^.*rspamd_task_write_log.*$/) { + &spinner; + my $ts; + if ( $ts_format eq 'syslog' ) { + $ts = syslog2iso( join ' ', ( split /\s+/ )[ 0 .. 2 ] ); + } + elsif ( $ts_format eq 'syslog5424' ) { + /^([0-9-]+)T([0-9:]+)/; + $ts = "$1 $2"; + } + else { + $ts = join ' ', ( split /\s+/ )[ 0 .. 1 ]; + } - if (scalar(@selected) > 0) { + next if ( $ts lt $startTime ); + next if ( defined $endTime && $ts gt $endTime ); - foreach my $sym (@selected) { - $sym =~ /^([^\(]+)(\(([^\)]+)\))?/; - my $sym_name = $1; - my $sym_score = 0; - my $orig_name = $sym_name; + if ( $_ !~ + /\(([^()]+)\): \[(NaN|-?\d+(?:\.\d+)?)\/(-?\d+(?:\.\d+)?)\]\s+\[([^\]]+)\].+? time: (\d+\.\d+)ms real/ ) + { + #print "BAD: $_\n"; + next; + } - if ($2) { - $sym_score = $3 * 1.0; + my @symbols = split /(?:\{[^}]*\})?(?:$|,)/, $4; + my $scan_time = $5; + my $act = $1; + my $score = $2 * 1.0; + my $skip = 0; - if (abs($sym_score) < $diff_alpha) { - next; - } + foreach my $ex (@symbols_exclude) { + my @found = grep { /^$ex/ } @symbols; - my $bm = $bidir_match{$sym_name}; - if ($bm) { - if ($sym_score >= 0) { - $sym_name = $bm->{'spam'}; + if ( scalar(@found) > 0 ) { + $skip = 1; + last; } - else { - $sym_name = $bm->{'ham'}; - } - } } - next if $orig_name !~ /^$s/; + next if ( $skip != 0 ); - if ($groups{$s}) { - # Replace with group - $sym_name = $groups{$s}; + if ( defined( $timeStamp{'end'} ) ) { + $timeStamp{'end'} = $ts if ( $ts gt $timeStamp{'end'} ); + } + else { + $timeStamp{'end'} = $ts; } - push @sym_names, $sym_name; - - if (!$sym_res{$sym_name}) { - $sym_res{$sym_name} = { - hits => 0, - spam_hits => 0, - junk_hits => 0, - spam_change => 0, - junk_change => 0, - weight => 0, - corr => {}, - symbols_met_spam => {}, - symbols_met_ham => {}, - symbols_met_junk => {}, - }; + if ( defined( $timeStamp{'start'} ) ) { + $timeStamp{'start'} = $ts if ( $ts lt $timeStamp{'start'} ); + } + else { + $timeStamp{'start'} = $ts; } - my $r = $sym_res{$sym_name}; + $scanTime{'min'} = $scan_time if ( !exists $scanTime{'min'} || $scanTime{'min'} > $scan_time ); + $scanTime{'max'} = $scan_time if ( $scanTime{'max'} < $scan_time ); + $scanTime{'total'} += $scan_time; - $r->{hits} ++; - $r->{weight} += $sym_score; - my $is_spam = 0; - my $is_junk = 0; + $action{$act}++; + $total++; - if ($score >= $reject_score) { - $is_spam = 1; - $r->{spam_hits} ++; - if ($correlations) { - ProcessRelated(\@symbols, $r->{symbols_met_spam}, $sym_name); - } - } - elsif ($score >= $junk_score) { - $is_junk = 1; - $r->{junk_hits} ++; - if ($correlations) { - ProcessRelated(\@symbols, $r->{symbols_met_junk}, $sym_name); - } + if ( $score >= $reject_score ) { + $total_spam++; } - else { - if ($correlations) { - ProcessRelated(\@symbols, $r->{symbols_met_ham}, $sym_name); - } + elsif ( $score >= $junk_score ) { + $total_junk++; } - if ($sym_score != 0) { - my $score_without = $score - $sym_score; - - if ($sym_score > 0) { - if ($is_spam && $score_without < $reject_score) { - $r->{spam_change} ++; - } - if ($is_junk && $score_without < $junk_score) { - $r->{junk_change} ++; - } - } - else { - if (!$is_spam && $score_without >= $reject_score) { - $r->{spam_change} ++; + my @sym_names; + + foreach my $s (@symbols_search) { + my @selected = grep /$s/, @symbols; + + if ( scalar(@selected) > 0 ) { + + foreach my $sym (@selected) { + $sym =~ /^([^\(]+)(\(([^\)]+)\))?/; + my $sym_name = $1; + my $sym_score = 0; + my $orig_name = $sym_name; + + if ($2) { + $sym_score = $3 * 1.0; + + if ( abs($sym_score) < $diff_alpha ) { + next; + } + + my $bm = $bidir_match{$sym_name}; + if ($bm) { + if ( $sym_score >= 0 ) { + $sym_name = $bm->{'spam'}; + } + else { + $sym_name = $bm->{'ham'}; + } + } + } + + next if $orig_name !~ /^$s/; + + if ( $groups{$s} ) { + + # Replace with group + $sym_name = $groups{$s}; + } + + push @sym_names, $sym_name; + + if ( !$sym_res{$sym_name} ) { + $sym_res{$sym_name} = { + hits => 0, + spam_hits => 0, + junk_hits => 0, + spam_change => 0, + junk_change => 0, + weight => 0, + corr => {}, + symbols_met_spam => {}, + symbols_met_ham => {}, + symbols_met_junk => {}, + }; + } + + my $r = $sym_res{$sym_name}; + + $r->{hits}++; + $r->{weight} += $sym_score; + my $is_spam = 0; + my $is_junk = 0; + + if ( $score >= $reject_score ) { + $is_spam = 1; + $r->{spam_hits}++; + if ($correlations) { + ProcessRelated( \@symbols, $r->{symbols_met_spam}, $sym_name ); + } + } + elsif ( $score >= $junk_score ) { + $is_junk = 1; + $r->{junk_hits}++; + if ($correlations) { + ProcessRelated( \@symbols, $r->{symbols_met_junk}, $sym_name ); + } + } + else { + if ($correlations) { + ProcessRelated( \@symbols, $r->{symbols_met_ham}, $sym_name ); + } + } + + if ( $sym_score != 0 ) { + my $score_without = $score - $sym_score; + + if ( $sym_score > 0 ) { + if ( $is_spam && $score_without < $reject_score ) { + $r->{spam_change}++; + } + if ( $is_junk && $score_without < $junk_score ) { + $r->{junk_change}++; + } + } + else { + if ( !$is_spam && $score_without >= $reject_score ) { + $r->{spam_change}++; + } + if ( !$is_junk && $score_without >= $junk_score ) { + $r->{junk_change}++; + } + } + } + } # End foreach symbols selected } - if (!$is_junk && $score_without >= $junk_score) { - $r->{junk_change} ++; - } - } } - } # End foreach symbols selected - } - } - - if ($correlations) { - foreach my $sym (@sym_names) { - next if IsIgnored($sym); - my $r = $sym_res{$sym}; - - foreach my $corr_sym (@sym_names) { - if ($corr_sym ne $sym) { - if ($r->{'corr'}->{$corr_sym}) { - $r->{'corr'}->{$corr_sym} ++; - } - else { - $r->{'corr'}->{$corr_sym} = 1; - } + + if ($correlations) { + foreach my $sym (@sym_names) { + next if IsIgnored($sym); + my $r = $sym_res{$sym}; + + foreach my $corr_sym (@sym_names) { + if ( $corr_sym ne $sym ) { + if ( $r->{'corr'}->{$corr_sym} ) { + $r->{'corr'}->{$corr_sym}++; + } + else { + $r->{'corr'}->{$corr_sym} = 1; + } + } + } + } # End of correlations check } - } - } # End of correlations check - } + } } - } } sub JsonObjectElt() { - my ($k, $v) = @_; - my $f = defined $_[2] ? $_[2] : '%s'; + my ( $k, $v ) = @_; + my $f = defined $_[2] ? $_[2] : '%s'; - if ($f eq "%s") { - $f = "\"%s\""; - } + if ( $f eq "%s" ) { + $f = "\"%s\""; + } - printf "\"%s\":$f", $k, $v; + printf "\"%s\":$f", $k, $v; } sub GetLogfilesList { - my ($dir) = @_; - opendir( DIR, $dir ) or die $!; + my ($dir) = @_; + opendir( DIR, $dir ) or die $!; - my $pattern = join( '|', keys %decompressor ); - my $re = qr/\.[0-9]+(?:\.(?:$pattern))?/; + my $pattern = join( '|', keys %decompressor ); + my $re = qr/\.[0-9]+(?:\.(?:$pattern))?/; - # Add unnumbered logs first - my @logs = - grep { -f "$dir/$_" && !/$re/ } readdir(DIR); + # Add unnumbered logs first + my @logs = + grep { -f "$dir/$_" && !/$re/ } readdir(DIR); - # Add numbered logs - rewinddir(DIR); - push( @logs, - ( sort numeric ( grep { -f "$dir/$_" && /$re/ } readdir(DIR) ) ) ); + # Add numbered logs + rewinddir(DIR); + push( @logs, ( sort numeric ( grep { -f "$dir/$_" && /$re/ } readdir(DIR) ) ) ); - closedir(DIR); + closedir(DIR); - # Select required logs and revers their order - @logs = - reverse - splice( @logs, $exclude_logs, $num_logs ||= @logs - $exclude_logs ); + # Select required logs and revers their order + @logs = + reverse splice( @logs, $exclude_logs, $num_logs ||= @logs - $exclude_logs ); - # Loop through array printing out filenames - print {interactive(*STDERR)} "\nLog files to process:\n"; - foreach my $file (@logs) { - print {interactive(*STDERR)} " $file\n"; - } - print {interactive(*STDERR)} "\n"; + # Loop through array printing out filenames + print { interactive(*STDERR) } "\nLog files to process:\n"; + foreach my $file (@logs) { + print { interactive(*STDERR) } " $file\n"; + } + print { interactive(*STDERR) } "\n"; - return @logs; + return @logs; } sub log_time_format { - my $fh = shift; - my ( $format, $line ); - while (<$fh>) { - $line = $_; - - # 2017-08-08 00:00:01 #66984( - # 2017-08-08 00:00:01.001 #66984( - if (/^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d(\.\d{3})? #\d+\(/) { - $format = 'rspamd'; - last; - } + my $fh = shift; + my ( $format, $line ); + while (<$fh>) { + $line = $_; + + # 2017-08-08 00:00:01 #66984( + # 2017-08-08 00:00:01.001 #66984( + if (/^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d(\.\d{3})? #\d+\(/) { + $format = 'rspamd'; + last; + } - # Aug 8 00:02:50 #66986( - elsif (/^\w{3} (?:\s?\d|\d\d) \d\d:\d\d:\d\d #\d+\(/) { - $format = 'syslog'; - last; - } + # Aug 8 00:02:50 #66986( + elsif (/^\w{3} (?:\s?\d|\d\d) \d\d:\d\d:\d\d #\d+\(/) { + $format = 'syslog'; + last; + } - # Aug 8 00:02:50 hostname rspamd[66986] - elsif (/^\w{3} (?:\s?\d|\d\d) \d\d:\d\d:\d\d \S+ rspamd\[\d+\]/) { - $format = 'syslog'; - last; - } + # Aug 8 00:02:50 hostname rspamd[66986] + elsif (/^\w{3} (?:\s?\d|\d\d) \d\d:\d\d:\d\d \S+ rspamd\[\d+\]/) { + $format = 'syslog'; + last; + } - # 2018-04-16T06:25:46.012590+02:00 rspamd rspamd[12968] - elsif(/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,6})?(Z|[-+]\d{2}:\d{2}) \S+ rspamd\[\d+\]/) { - $format = 'syslog5424'; - last; - } + # 2018-04-16T06:25:46.012590+02:00 rspamd rspamd[12968] + elsif (/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,6})?(Z|[-+]\d{2}:\d{2}) \S+ rspamd\[\d+\]/) { + $format = 'syslog5424'; + last; + } - # Skip newsyslog messages - # Aug 8 00:00:00 hostname newsyslog[63284]: logfile turned over - elsif ( /^\w{3} (?:\s?\d|\d\d) \d\d:\d\d:\d\d\ \S+ newsyslog\[\d+\]: logfile turned over$/ ) { - next; - } - # Skip journalctl messages - # -- Logs begin at Mon 2018-01-15 11:16:24 MSK, end at Fri 2018-04-27 09:10:30 MSK. -- - elsif ( /^-- Logs begin at \w{3} \d{4}-\d\d-\d\d \d\d:\d\d:\d\d [A-Z]{3}, end at \w{3} \d{4}-\d\d-\d\d \d\d:\d\d:\d\d [A-Z]{3}\. --$/ ) { - next; - } - else { - print "Unknown log format\n"; - exit 1; + # Skip newsyslog messages + # Aug 8 00:00:00 hostname newsyslog[63284]: logfile turned over + elsif (/^\w{3} (?:\s?\d|\d\d) \d\d:\d\d:\d\d\ \S+ newsyslog\[\d+\]: logfile turned over$/) { + next; + } + + # Skip journalctl messages + # -- Logs begin at Mon 2018-01-15 11:16:24 MSK, end at Fri 2018-04-27 09:10:30 MSK. -- + elsif ( +/^-- Logs begin at \w{3} \d{4}-\d\d-\d\d \d\d:\d\d:\d\d [A-Z]{3}, end at \w{3} \d{4}-\d\d-\d\d \d\d:\d\d:\d\d [A-Z]{3}\. --$/ + ) + { + next; + } + else { + print "Unknown log format\n"; + exit 1; + } } - } - return ( $format, $line ); + return ( $format, $line ); } sub normalized_time { - return undef - if !defined( $_ = shift ); + return + if !defined( $_ = shift ); - /^\d\d(?::\d\d){0,2}$/ - ? sprintf '%04d-%02d-%02d %s', 1900 + (localtime)[5], 1 + (localtime)[4], - (localtime)[3], $_ - : $_; + /^\d\d(?::\d\d){0,2}$/ + ? sprintf '%04d-%02d-%02d %s', 1900 + (localtime)[5], 1 + (localtime)[4], (localtime)[3], $_ + : $_; } sub numeric { - $a =~ /\.(\d+)\./; - my $a_num = $1; - $b =~ /\.(\d+)\./; - my $b_num = $1; + $a =~ /\.(\d+)\./; + my $a_num = $1; + $b =~ /\.(\d+)\./; + my $b_num = $1; - $a_num <=> $b_num; + $a_num <=> $b_num; } sub spinner { @@ -780,7 +781,7 @@ sub spinner { return if ( ( time - $spinner_update_time ) < 1 ); $spinner_update_time = time; - printf {interactive(*STDERR)} "%s\r", $spinner[ $spinner_update_time % @spinner ]; + printf { interactive(*STDERR) } "%s\r", $spinner[ $spinner_update_time % @spinner ]; select()->flush(); } @@ -788,33 +789,32 @@ sub spinner { # using current year as syslog does not record the year (nor the timezone) # or the last year if the guessed time is in the future. sub syslog2iso { - my %month_map; - @month_map{qw(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec)} = 0 .. 11; - - my ( $month, @t ) = - $_[0] =~ m/^(\w{3}) \s\s? (\d\d?) \s (\d\d):(\d\d):(\d\d)/x; - my $epoch = - timelocal( ( reverse @t ), $month_map{$month}, 1900 + (localtime)[5] ); - sprintf '%04d-%02d-%02d %02d:%02d:%02d', - 1900 + (localtime)[5] - ( $epoch > time ), - $month_map{$month} + 1, @t; + my %month_map; + @month_map{qw(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec)} = 0 .. 11; + + my ( $month, @t ) = $_[0] =~ m/^(\w{3}) \s\s? (\d\d?) \s (\d\d):(\d\d):(\d\d)/x; + my $epoch = + timelocal( ( reverse @t ), $month_map{$month}, 1900 + (localtime)[5] ); + sprintf '%04d-%02d-%02d %02d:%02d:%02d', 1900 + (localtime)[5] - ( $epoch > time ), $month_map{$month} + 1, @t; } ### Imported from IO::Interactive 1.022 Perl module sub is_interactive { - my ($out_handle) = (@_, select); # Default to default output handle + ## no critic (ProhibitInteractiveTest) + + my ($out_handle) = ( @_, select ); # Default to default output handle # Not interactive if output is not to terminal... return 0 if not -t $out_handle; # If *ARGV is opened, we're interactive if... - if ( tied(*ARGV) or defined(fileno(ARGV)) ) { # this is what 'Scalar::Util::openhandle *ARGV' boils down to + if ( tied(*ARGV) or defined( fileno(ARGV) ) ) { # this is what 'Scalar::Util::openhandle *ARGV' boils down to # ...it's currently opened to the magic '-' file return -t *STDIN if defined $ARGV && $ARGV eq '-'; # ...it's at end-of-file and the next file is the magic '-' file - return @ARGV>0 && $ARGV[0] eq '-' && -t *STDIN if eof *ARGV; + return @ARGV > 0 && $ARGV[0] eq '-' && -t *STDIN if eof *ARGV; # ...it's directly attached to the terminal return -t *ARGV; @@ -828,17 +828,18 @@ sub is_interactive { } ### Imported from IO::Interactive 1.022 Perl module -local (*DEV_NULL, *DEV_NULL2); +local ( *DEV_NULL, *DEV_NULL2 ); my $dev_null; + BEGIN { pipe *DEV_NULL, *DEV_NULL2 - or die "Internal error: can't create null filehandle"; + or die "Internal error: can't create null filehandle"; $dev_null = \*DEV_NULL; } ### Imported from IO::Interactive 1.022 Perl module sub interactive { - my ($out_handle) = (@_, \*STDOUT); # Default to STDOUT + my ($out_handle) = ( @_, \*STDOUT ); # Default to STDOUT return &is_interactive ? $out_handle : $dev_null; } @@ -875,13 +876,11 @@ rspamd_stats [options] [--symbol=SYM1 [--symbol=SYM2...]] [--log file] =item B<--log> -Specifies log file or directory to read data from. -If a directory is specified B<rspamd_stats> analyses files in the directory -including known compressed file types. Number of log files can be limited using -B<--num-logs> and B<--exclude-logs> options. This assumes that files in the log -directory have B<newsyslog(8)>- or B<logrotate(8)>-like name format with numeric -indexes. Files without indexes (generally it is merely one file) are considered -the most recent and files with lower indexes are considered newer. +Specifies log file or directory to read data from. If a directory is specified B<rspamd_stats> analyses files in the +directory including known compressed file types. Number of log files can be limited using B<--num-logs> and +B<--exclude-logs> options. This assumes that files in the log directory have B<newsyslog(8)>- or B<logrotate(8)>-like +name format with numeric indexes. Files without indexes (generally it is merely one file) are considered the most +recent and files with lower indexes are considered newer. =item B<--reject-score> @@ -909,7 +908,8 @@ Number of latest logs to exclude (0 by default). =item B<--correlations> -Additionally print correlation rate for each symbol displayed. This routine calculates merely paired correlations between symbols. +Additionally print correlation rate for each symbol displayed. This routine calculates merely paired correlations +between symbols. =item B<--search-pattern> @@ -917,20 +917,19 @@ Do not process input unless finding the specified regular expression. Useful to =item B<--exclude> -Exclude log lines if certain symbols are fired (e.g. GTUBE). You may specify this option multiple time to skip multiple symbols. +Exclude log lines if certain symbols are fired (e.g. GTUBE). You may specify this option multiple time to skip multiple +symbols. =item B<--start> -Select log entries after this time. Format: C<YYYY-MM-DD HH:MM:SS> (can be -truncated to any desired accuracy). If used with B<--end> select entries between -B<--start> and B<--end>. The omitted date defaults to the current date if you +Select log entries after this time. Format: C<YYYY-MM-DD HH:MM:SS> (can be truncated to any desired accuracy). If used +with B<--end> select entries between B<--start> and B<--end>. The omitted date defaults to the current date if you supply the time. =item B<--end> -Select log entries before this time. Format: C<YYYY-MM-DD HH:MM:SS> (can be -truncated to any desired accuracy). If used with B<--start> select entries between -B<--start> and B<--end>. The omitted date defaults to the current date if you +Select log entries before this time. Format: C<YYYY-MM-DD HH:MM:SS> (can be truncated to any desired accuracy). If used +with B<--start> select entries between B<--start> and B<--end>. The omitted date defaults to the current date if you supply the time. =item B<--help> @@ -986,7 +985,8 @@ B<total ham hits>: overall number of B<HAM> messages =item 4. -B<ham with symbol percentage>: percentage of number of hits with specified symbol in B<HAM> messages divided by total number of B<HAM> messages. +B<ham with symbol percentage>: percentage of number of hits with specified symbol in B<HAM> messages divided by total +number of B<HAM> messages. =back |