diff options
220 files changed, 12097 insertions, 3844 deletions
diff --git a/.eslintrc.json b/.eslintrc.json index 1dace30fa..2dc3ebc51 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -21,9 +21,9 @@ "id-length": ["error", { "min": 1 }], "indent": ["error", 4, { "SwitchCase": 1 }], // "max-len": ["error", { "code": 120 }], - "key-spacing": ["error", { - "singleLine": { "afterColon": false } - }], + // "key-spacing": ["error", { + // "singleLine": { "afterColon": false } + // }], "line-comment-position": "off", "max-params": ["warn", 6], "max-statements": ["warn", 44], @@ -63,10 +63,12 @@ // Temporarily disabled rules "func-style": "off", "function-paren-newline": "off", + "key-spacing": "off", "max-len": "off", "max-lines": "off", "max-lines-per-function": "off", "no-invalid-this": "off", + "prefer-exponentiation-operator": "off", "sort-keys": "off", "sort-vars": "off" } diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 01948df04..d34d9ebb7 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -19,6 +19,7 @@ Do you want to ask a question? Are you looking for support? Here are the places * Read about bug reporting in general: https://rspamd.com/doc/faq.html#how-to-report-bugs-found-in-rspamd * Enabled relevant debugging logs: https://rspamd.com/doc/faq.html#how-to-debug-some-module-in-rspamd * Checked the FAQs about Core files in case of fatal crash: https://rspamd.com/doc/faq.html#how-to-figure-out-why-rspamd-process-crashed + * Tried ASAN package and obtained the ASAN report (if possible): https://rspamd.com/doc/faq.html#asan-builds * Checked that your issue isn't already filed: https://github.com/issues?utf8=%E2%9C%93&q=is%3Aissue+user%3Arspamd * Checked that there is not already an experimental package or master branch diff --git a/.github/ISSUE_TEMPLATE/support_question.md b/.github/ISSUE_TEMPLATE/support_question.md new file mode 100644 index 000000000..87407834e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/support_question.md @@ -0,0 +1,15 @@ +--- +name: Support Question +about: If you have a question 💬, please check out our Mailing lists or other support channels! + +--- + +--------------^ Click "Preview" for a nicer view! +We primarily use GitHub as an issue tracker; for usage and support questions, please check out these resources below. Thanks! + +--- + +* Mailing list: https://lists.rspamd.com/mailman/listinfo (you can subscribe on that page or browse the archives) +* Telegram channel: http://t.me/rspamd +* Also have a look at the following page for more information on how to get support: + https://rspamd.com/support.html diff --git a/AUTHORS.md b/AUTHORS.md index ac0a68e7d..6fad668df 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -1,6 +1,6 @@ # Project Authors and Contributors -Rspamd was created by [Vsevolod Stakhov](https://github.com/vstakov) while working in [Rambler Mail](https://mail.rambler.ru/) +Rspamd was created by [Vsevolod Stakhov](https://github.com/vstakov). ## Authors @@ -26,5 +26,4 @@ Rspamd was created by [Vsevolod Stakhov](https://github.com/vstakov) while worki ## Special thanks to * [Mimecast](https://mimecast.com) -* [Rambler Mail](https://mail.rambler.ru/) * [Locaweb](https://locaweb.com.br) diff --git a/CMakeLists.txt b/CMakeLists.txt index c6b446cac..155e317ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.9 FATAL_ERROR) SET(RSPAMD_VERSION_MAJOR 2) -SET(RSPAMD_VERSION_MINOR 2) +SET(RSPAMD_VERSION_MINOR 3) # Keep two digits all the time SET(RSPAMD_VERSION_MAJOR_NUM ${RSPAMD_VERSION_MAJOR}0) @@ -21,11 +21,12 @@ ENDIF() SET(RSPAMD_VERSION "${RSPAMD_VERSION_MAJOR}.${RSPAMD_VERSION_MINOR}") -PROJECT(rspamd VERSION "${RSPAMD_VERSION}" LANGUAGES C ASM) +PROJECT(rspamd VERSION "${RSPAMD_VERSION}" LANGUAGES C CXX ASM) # This is supported merely with cmake 3.1 SET(CMAKE_C_STANDARD 11) SET(CMAKE_C_STANDARD_REQUIRED ON) +LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/") SET(RSPAMD_MASTER_SITE_URL "https://rspamd.com") @@ -40,11 +41,6 @@ SET(RSPAMD_WORKER_CONTROLLER "*:11334") SET_PROPERTY(GLOBAL PROPERTY ALLOW_DUPLICATE_CUSTOM_TARGETS 1) ############################# OPTIONS SECTION ############################################# - -OPTION(ENABLE_OPTIMIZATION "Enable extra optimizations [default: OFF]" OFF) -OPTION(SKIP_RELINK_RPATH "Skip relinking and full RPATH for the install tree" OFF) -OPTION(ENABLE_GPERF_TOOLS "Enable google perftools [default: OFF]" OFF) -OPTION(ENABLE_STATIC "Enable static compiling [default: OFF]" OFF) OPTION(ENABLE_LUAJIT "Link with libluajit [default: ON]" ON) OPTION(ENABLE_URL_INCLUDE "Enable urls in ucl includes (requires libcurl or libfetch) [default: OFF]" OFF) OPTION(NO_SHARED "Build internal libs static [default: ON]" ON) @@ -52,152 +48,12 @@ OPTION(INSTALL_WEBUI "Install web interface [default: ON]" OPTION(WANT_SYSTEMD_UNITS "Install systemd unit files on Linux [default: OFF]" OFF) OPTION(ENABLE_SNOWBALL "Enable snowball stemmer [default: ON]" ON) OPTION(ENABLE_CLANG_PLUGIN "Enable clang static analysing plugin [default: OFF]" OFF) -OPTION(ENABLE_HYPERSCAN "Enable hyperscan for fast regexp processing [default: OFF]" OFF) OPTION(ENABLE_PCRE2 "Enable pcre2 instead of pcre [default: OFF]" OFF) OPTION(ENABLE_JEMALLOC "Build rspamd with jemalloc allocator [default: OFF]" OFF) -OPTION(ENABLE_COVERAGE "Build rspamd with code coverage options [default: OFF]" OFF) -OPTION(ENABLE_FULL_DEBUG "Build rspamd with all possible debug [default: OFF]" OFF) OPTION(ENABLE_UTILS "Build rspamd internal utils [default: OFF]" OFF) OPTION(ENABLE_LIBUNWIND "Use libunwind to print crash traces [default: OFF]" OFF) OPTION(ENABLE_LUA_TRACE "Trace all Lua C API invocations [default: OFF]" OFF) OPTION(ENABLE_LUA_REPL "Enables Lua repl (requires C++11 compiler) [default: ON]" ON) -OPTION(ENABLE_BLAS "Enables libopenblas support [default: OFF]" OFF) - - -INCLUDE(FindArch.cmake) -TARGET_ARCHITECTURE(ARCH) - -INCLUDE(FindRagel.cmake) -IF(NOT RAGEL_FOUND) - MESSAGE(FATAL_ERROR "Ragel is required to build rspamd") -ENDIF() - -IF (NOT "${ARCH}" STREQUAL "x86_64") - MESSAGE(STATUS "Hyperscan support is possible only for x86_64 architecture") - SET(ENABLE_HYPERSCAN "OFF") -ENDIF() - -IF ("${ARCH}" STREQUAL "x86_64") - MESSAGE(STATUS "Enable sse2 on x86_64 architecture") - IF((CMAKE_C_COMPILER_ID MATCHES "GNU") OR (CMAKE_C_COMPILER_ID MATCHES "Clang")) - ADD_COMPILE_OPTIONS(-msse2) - ELSEIF(CMAKE_C_COMPILER_ID MATCHES "Intel") - ADD_COMPILE_OPTIONS(/QxSSE2) - ELSEIF((CMAKE_C_COMPILER_ID MATCHES "MSVC")) - ADD_COMPILE_OPTIONS(/arch:SSE2) - ENDIF() -ENDIF() - -IF(ENABLE_PCRE2 MATCHES "ON") - SET(WITH_PCRE2 1) - # For utf8 API - LIST(APPEND CMAKE_REQUIRED_DEFINITIONS "-DPCRE2_CODE_UNIT_WIDTH=8") -ENDIF() -# Build optimized code for following CPU (default i386) -#SET(CPU_TUNE "i686") - -# Now CMAKE_INSTALL_PREFIX is a base prefix for everything -# CONFDIR - for configuration -# LOCAL_CONFDIR - for local configuration -# MANDIR - for manual pages -# RUNDIR - for runtime files -# DBDIR - for static files -# LOGDIR - for log files -# EXAMPLESDIR - for examples files - -IF(NOT CONFDIR) - SET(CONFDIR "${CMAKE_INSTALL_PREFIX}/etc/rspamd") -ENDIF(NOT CONFDIR) - -IF(NOT LOCAL_CONFDIR) - SET(LOCAL_CONFDIR "${CONFDIR}") -ENDIF(NOT LOCAL_CONFDIR) - -IF(NOT MANDIR) - SET(MANDIR "${CMAKE_INSTALL_PREFIX}/share/man") -ENDIF(NOT MANDIR) - -IF(NOT RUNDIR) - SET(RUNDIR "/var/run/rspamd") -ENDIF(NOT RUNDIR) - -IF(NOT DBDIR) - SET(DBDIR "/var/lib/rspamd") -ENDIF(NOT DBDIR) - -IF(NOT LOGDIR) - SET(LOGDIR "/var/log/rspamd") -ENDIF(NOT LOGDIR) - -IF(NOT SHAREDIR) - SET(SHAREDIR "${CMAKE_INSTALL_PREFIX}/share/rspamd") -ENDIF(NOT SHAREDIR) - -IF(NOT EXAMPLESDIR) - SET(EXAMPLESDIR "${CMAKE_INSTALL_PREFIX}/share/examples/rspamd") -ENDIF(NOT EXAMPLESDIR) - -IF(NOT LUALIBDIR) - SET(LUALIBDIR "${SHAREDIR}/lualib") -ENDIF(NOT LUALIBDIR) - -IF(NOT PLUGINSDIR) - SET(PLUGINSDIR "${SHAREDIR}/plugins") -ENDIF(NOT PLUGINSDIR) - -IF(NOT RULESDIR) - SET(RULESDIR "${SHAREDIR}/rules") -ENDIF(NOT RULESDIR) - -IF(NOT WWWDIR) - SET(WWWDIR "${SHAREDIR}/www") -ENDIF(NOT WWWDIR) - -# Set libdir -IF(NOT LIBDIR) - SET(RSPAMD_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib/rspamd") -ELSE(NOT LIBDIR) - SET(RSPAMD_LIBDIR "${LIBDIR}") -ENDIF(NOT LIBDIR) -SET(CMAKE_MACOSX_RPATH ON) -SET(CMAKE_INSTALL_RPATH "${RSPAMD_LIBDIR}") - -# Set includedir -IF(NOT INCLUDEDIR) - SET(INCLUDEDIR include/rspamd) -ENDIF(NOT INCLUDEDIR) - -IF(NOT SYSTEMDDIR) - SET(SYSTEMDDIR ${CMAKE_INSTALL_PREFIX}/lib/systemd/system) -ENDIF(NOT SYSTEMDDIR) - -SET(RSPAMD_DEFAULT_INCLUDE_PATHS "/opt;/usr;/usr/local;/opt/local;/usr/pkg;/opt/csw;/sw") -SET(RSPAMD_DEFAULT_LIBRARY_PATHS "/usr/local;/usr/pkg;/usr;/Library/Frameworks;/sw;/opt/local;/opt/csw;/opt") - -IF(ENABLE_STATIC MATCHES "ON") - MESSAGE(STATUS "Static build of rspamd implies that the target binary will be *GPL* licensed") - SET(GPL_RSPAMD_BINARY 1) - SET(CMAKE_SKIP_INSTALL_RPATH ON) - SET(BUILD_STATIC 1) - SET(CMAKE_FIND_LIBRARY_SUFFIXES ".a") - SET(BUILD_SHARED_LIBRARIES OFF) - SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static") - SET(LINK_TYPE "STATIC") - SET(NO_SHARED "ON") - # Dirty hack for cmake - SET(CMAKE_EXE_LINK_DYNAMIC_C_FLAGS) # remove -Wl,-Bdynamic - SET(CMAKE_EXE_LINK_DYNAMIC_CXX_FLAGS) - SET(CMAKE_SHARED_LIBRARY_C_FLAGS) # remove -fPIC - SET(CMAKE_SHARED_LIBRARY_CXX_FLAGS) - SET(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS) # remove -rdynamic - SET(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS) -ELSE(ENABLE_STATIC MATCHES "ON") - IF (NO_SHARED MATCHES "OFF") - SET(LINK_TYPE "SHARED") - ELSE(NO_SHARED MATCHES "OFF") - SET(LINK_TYPE "STATIC") - ENDIF (NO_SHARED MATCHES "OFF") -ENDIF (ENABLE_STATIC MATCHES "ON") ############################# INCLUDE SECTION ############################################# @@ -209,275 +65,30 @@ INCLUDE(CheckCSourceRuns) INCLUDE(CheckLibraryExists) INCLUDE(CheckCCompilerFlag) INCLUDE(CMakeParseArguments) +INCLUDE(FindArch) +INCLUDE(AsmOp) +INCLUDE(FindRagel) +INCLUDE(FindLua) +INCLUDE(ProcessPackage) + +IF(NOT RAGEL_FOUND) + MESSAGE(FATAL_ERROR "Ragel is required to build rspamd") +ENDIF() FIND_PACKAGE(PkgConfig REQUIRED) FIND_PACKAGE(Perl REQUIRED) -############################# MACRO SECTION ############################################# - -# Find lua installation -MACRO(FindLua) - # Find lua libraries - UNSET(LUA_INCLUDE_DIR CACHE) - UNSET(LUA_LIBRARY CACHE) - CMAKE_PARSE_ARGUMENTS(LUA "" "VERSION_MAJOR;VERSION_MINOR;ROOT" "" ${ARGN}) +INCLUDE(Toolset) +INCLUDE(Sanitizer) - IF(NOT LUA_VERSION_MAJOR OR NOT LUA_VERSION_MINOR) - MESSAGE(FATAL_ERROR "Invalid FindLua invocation: ${ARGN}") - ENDIF() - - IF(ENABLE_LUAJIT MATCHES "ON") - MESSAGE(STATUS "Check for luajit ${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}") - FIND_PATH(LUA_INCLUDE_DIR luajit.h - HINTS - "${RSPAMD_SEARCH_PATH}" "${LUA_ROOT}" - $ENV{LUA_DIR} - PATH_SUFFIXES "include/luajit-2.0" - "include/luajit-2.1" - "include/luajit${LUA_VERSION_MAJOR}${LUA_VERSION_MINOR}" - "include/luajit${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" - "include/luajit-${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" - "include/luajit-${LUA_VERSION_MAJOR}_${LUA_VERSION_MINOR}-2.0" - "include/luajit-${LUA_VERSION_MAJOR}_${LUA_VERSION_MINOR}-2.1" - "include/luajit" - "include/lua${LUA_VERSION_MAJOR}${LUA_VERSION_MINOR}" - "include/lua${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" - "include/lua-${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" - include/lua include - PATHS ${RSPAMD_DEFAULT_INCLUDE_PATHS} - ) - FIND_LIBRARY(LUA_LIBRARY - NAMES luajit - "luajit-2.0" - "luajit2.0" - "luajit${LUA_VERSION_MAJOR}${LUA_VERSION_MINOR}" - "luajit${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" - "luajit-${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" - HINTS - "${RSPAMD_SEARCH_PATH}" "${LUA_ROOT}" - $ENV{LUA_DIR} - PATH_SUFFIXES lib64 lib - PATHS ${RSPAMD_DEFAULT_LIBRARY_PATHS} - DOC "Lua library" - ) - - IF(NOT LUA_LIBRARY OR NOT LUA_INCLUDE_DIR) - MESSAGE(STATUS "Fallback from luajit to plain lua") - SET(ENABLE_LUAJIT "OFF") - MESSAGE(STATUS "Check for lua ${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}") - FIND_PATH(LUA_INCLUDE_DIR lua.h - HINTS - "${RSPAMD_SEARCH_PATH}" "${LUA_ROOT}" - $ENV{LUA_DIR} - PATH_SUFFIXES "include/lua${LUA_VERSION_MAJOR}${LUA_VERSION_MINOR}" - "include/lua${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" - "include/lua-${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" - include/lua include - PATHS ${RSPAMD_DEFAULT_INCLUDE_PATHS} - ) - FIND_LIBRARY(LUA_LIBRARY - NAMES lua - "lua${LUA_VERSION_MAJOR}${LUA_VERSION_MINOR}" - "lua${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" - "lua-${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" - HINTS - "${RSPAMD_SEARCH_PATH}" "${LUA_ROOT}" - $ENV{LUA_DIR} - PATH_SUFFIXES lib64 lib - PATHS ${RSPAMD_DEFAULT_LIBRARY_PATHS} - DOC "Lua library" - ) - ELSE() - SET(WITH_LUAJIT 1) - ENDIF() - ELSE(ENABLE_LUAJIT MATCHES "ON") - MESSAGE(STATUS "Check for lua ${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}") - FIND_PATH(LUA_INCLUDE_DIR lua.h - HINTS - "${RSPAMD_SEARCH_PATH}" "${LUA_ROOT}" - $ENV{LUA_DIR} - PATH_SUFFIXES "include/lua${LUA_VERSION_MAJOR}${LUA_VERSION_MINOR}" - "include/lua${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" - "include/lua-${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" - include/lua include - PATHS ${RSPAMD_DEFAULT_INCLUDE_PATHS} - ) - FIND_LIBRARY(LUA_LIBRARY - NAMES lua - "lua${LUA_VERSION_MAJOR}${LUA_VERSION_MINOR}" - "lua${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" - "lua-${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" - HINTS - "${RSPAMD_SEARCH_PATH}" "${LUA_ROOT}" - $ENV{LUA_DIR} - PATH_SUFFIXES lib64 lib - PATHS ${RSPAMD_DEFAULT_LIBRARY_PATHS} - DOC "Lua library" - ) - ENDIF(ENABLE_LUAJIT MATCHES "ON") - - IF(LUA_LIBRARY AND LUA_INCLUDE_DIR) - SET(LUA_FOUND 1) - IF(NOT LUA_VERSION_MAJOR OR NOT LUA_VERSION_MINOR) - SET(LUA_VERSION_MAJOR ${LUA_VERSION_MAJOR}) - SET(LUA_VERSION_MINOR ${LUA_VERSION_MINOR}) - ENDIF(NOT LUA_VERSION_MAJOR OR NOT LUA_VERSION_MINOR) - IF(ENABLE_LUAJIT MATCHES "ON") - MESSAGE(STATUS "Found luajit ${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR} in lib:${LUA_LIBRARY}, headers:${LUA_INCLUDE_DIR}") - ELSE(ENABLE_LUAJIT MATCHES "ON") - MESSAGE(STATUS "Found lua ${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR} in lib:${LUA_LIBRARY}, headers:${LUA_INCLUDE_DIR}") - ENDIF(ENABLE_LUAJIT MATCHES "ON") - ENDIF(LUA_LIBRARY AND LUA_INCLUDE_DIR) -ENDMACRO() - -FUNCTION(INSTALL_IF_NOT_EXISTS src dest destname suffix) - IF(NOT IS_ABSOLUTE "${src}") - SET(src "${CMAKE_CURRENT_SOURCE_DIR}/${src}") - ENDIF() - GET_FILENAME_COMPONENT(src_name "${src}" NAME) - GET_FILENAME_COMPONENT(dest_name "${destname}" NAME) - IF(NOT IS_ABSOLUTE "${dest}") - SET(dest "${CMAKE_INSTALL_PREFIX}/${dest}") - ENDIF() - INSTALL(CODE " - IF(NOT EXISTS \"\$ENV{DESTDIR}${dest}/${dest_name}${suffix}\") - #FILE(INSTALL \"${src}\" DESTINATION \"${dest}\") - MESSAGE(STATUS \"Installing: \$ENV{DESTDIR}${dest}/${dest_name}${suffix}\") - EXECUTE_PROCESS(COMMAND \${CMAKE_COMMAND} -E copy \"${src}\" - \"\$ENV{DESTDIR}${dest}/${dest_name}${suffix}\" - RESULT_VARIABLE copy_result - ERROR_VARIABLE error_output) - IF(copy_result) - MESSAGE(FATAL_ERROR \${error_output}) - ENDIF() - ELSE() - MESSAGE(STATUS \"Skipping : \$ENV{DESTDIR}${dest}/${dest_name}${suffix}\") - ENDIF() - ") -ENDFUNCTION(INSTALL_IF_NOT_EXISTS) - -# Process required package by using FindPackage and calling for INCLUDE_DIRECTORIES and -# setting list of required libraries -# Usage: -# ProcessPackage(VAR [OPTIONAL] [ROOT path] [INCLUDE path] -# [LIBRARY path] [INCLUDE_SUFFIXES path1 path2 ...] [LIB_SUFFIXES path1 path2 ...] -# [MODULES module1 module2 ...]) -# params: -# OPTIONAL - do not fail if a package has not been found -# ROOT - defines root directory for a package -# INCLUDE - name of the include file to check -# LIBRARY - name of the library to check -# INCLUDE_SUFFIXES - list of include suffixes (relative to ROOT) -# LIB_SUFFIXES - list of library suffixes -# MODULES - modules to search using pkg_config -MACRO(ProcessPackage PKG_NAME) - - CMAKE_PARSE_ARGUMENTS(PKG "OPTIONAL;OPTIONAL_INCLUDE" "ROOT;INCLUDE" - "LIBRARY;INCLUDE_SUFFIXES;LIB_SUFFIXES;MODULES;LIB_OUTPUT" ${ARGN}) - - IF(NOT PKG_LIBRARY) - SET(PKG_LIBRARY "${PKG_NAME}") - ENDIF() - IF(NOT PKG_INCLUDE) - SET(PKG_INCLUDE "${PKG_NAME}.h") - ENDIF() - IF(NOT PKG_LIB_OUTPUT) - SET(PKG_LIB_OUTPUT RSPAMD_REQUIRED_LIBRARIES) - ENDIF() +INCLUDE(ArchDep) +INCLUDE(Paths) - IF(NOT PKG_ROOT AND PKG_MODULES) - PKG_SEARCH_MODULE(${PKG_NAME} ${PKG_MODULES}) - ENDIF() - - IF(${PKG_NAME}_FOUND) - MESSAGE(STATUS "Found package ${PKG_NAME} in pkg-config modules ${PKG_MODULES}") - SET(WITH_${PKG_NAME} 1 CACHE INTERNAL "") - IF(ENABLE_STATIC MATCHES "ON") - SET(_XPREFIX "${PKG_NAME}_STATIC") - ELSE(ENABLE_STATIC MATCHES "ON") - SET(_XPREFIX "${PKG_NAME}") - ENDIF(ENABLE_STATIC MATCHES "ON") - FOREACH(_arg ${${_XPREFIX}_INCLUDE_DIRS}) - INCLUDE_DIRECTORIES("${_arg}") - SET(${PKG_NAME}_INCLUDE "${_arg}" CACHE INTERNAL "") - ENDFOREACH(_arg ${${_XPREFIX}_INCLUDE_DIRS}) - FOREACH(_arg ${${_XPREFIX}_LIBRARY_DIRS}) - LINK_DIRECTORIES("${_arg}") - SET(${PKG_NAME}_LIBRARY_PATH "${_arg}" CACHE INTERNAL "") - ENDFOREACH(_arg ${${_XPREFIX}_LIBRARY_DIRS}) - # Handle other CFLAGS and LDFLAGS - FOREACH(_arg ${${_XPREFIX}_CFLAGS_OTHER}) - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_arg}") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_arg}") - ENDFOREACH(_arg ${${_XPREFIX}_CFLAGS_OTHER}) - FOREACH(_arg ${${_XPREFIX}_LDFLAGS_OTHER}) - SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${_arg}") - ENDFOREACH(_arg ${${_XPREFIX}_LDFLAGS_OTHER}) - LIST(APPEND ${PKG_LIB_OUTPUT} "${${_XPREFIX}_LIBRARIES}") - INCLUDE_DIRECTORIES(${${_XPREFIX}_INCLUDEDIR}) - ELSE() - IF(NOT ${PKG_NAME}_GUESSED) - # Try some more heuristic - FIND_LIBRARY(_lib NAMES ${PKG_LIBRARY} - HINTS ${PKG_ROOT} ${RSPAMD_SEARCH_PATH} - PATH_SUFFIXES ${PKG_LIB_SUFFIXES} lib64 lib - PATHS ${RSPAMD_DEFAULT_LIBRARY_PATHS}) - IF(NOT _lib) - IF(PKG_OPTIONAL) - MESSAGE(STATUS "Cannot find library ${PKG_LIBRARY} for package ${PKG_NAME}, ignoring") - ELSE() - MESSAGE(FATAL_ERROR "Cannot find library ${PKG_LIBRARY} for package ${PKG_NAME}") - ENDIF() - ENDIF(NOT _lib) - - FIND_PATH(_incl ${PKG_INCLUDE} - HINTS ${PKG_ROOT} ${RSPAMD_SEARCH_PATH} - PATH_SUFFIXES ${PKG_INCLUDE_SUFFIXES} include - PATHS {RSPAMD_DEFAULT_INCLUDE_PATHS}) - IF(NOT _incl) - IF(PKG_OPTIONAL OR PKG_OPTIONAL_INCLUDE) - MESSAGE(STATUS "Cannot find header ${PKG_INCLUDE} for package ${PKG_NAME}") - ELSE() - MESSAGE(FATAL_ERROR "Cannot find header ${PKG_INCLUDE} for package ${PKG_NAME}") - ENDIF() - ELSE() - STRING(REGEX REPLACE "/[^/]+$" "" _incl_path "${PKG_INCLUDE}") - STRING(REGEX REPLACE "${_incl_path}/$" "" _stripped_incl "${_incl}") - INCLUDE_DIRECTORIES("${_stripped_incl}") - SET(${PKG_NAME}_INCLUDE "${_stripped_incl}" CACHE INTERNAL "") - ENDIF(NOT _incl) - - IF(_lib) - # We need to apply heuristic to find the real dir name - GET_FILENAME_COMPONENT(_lib_path "${_lib}" PATH) - LINK_DIRECTORIES("${_lib_path}") - LIST(APPEND ${PKG_LIB_OUTPUT} ${_lib}) - SET(${PKG_NAME}_LIBRARY_PATH "${_lib_path}" CACHE INTERNAL "") - SET(${PKG_NAME}_LIBRARY "${_lib}" CACHE INTERNAL "") - ENDIF() - - IF(_incl AND _lib) - MESSAGE(STATUS "Found package ${PKG_NAME} in '${_lib_path}' (${_lib}) and '${_stripped_incl}' (${PKG_INCLUDE}).") - SET(${PKG_NAME}_GUESSED 1 CACHE INTERNAL "") - SET(WITH_${PKG_NAME} 1 CACHE INTERNAL "") - ELSEIF(_lib) - IF(PKG_OPTIONAL_INCLUDE) - SET(${PKG_NAME}_GUESSED 1 INTERNAL "") - SET(WITH_${PKG_NAME} 1 INTERNAL "") - ENDIF() - MESSAGE(STATUS "Found incomplete package ${PKG_NAME} in '${_lib_path}' (${_lib}); no includes.") - ENDIF() - ELSE() - MESSAGE(STATUS "Found package ${PKG_NAME} (cached)") - INCLUDE_DIRECTORIES("${${PKG_NAME}_INCLUDE}") - LINK_DIRECTORIES("${${PKG_NAME}_LIBRARY_PATH}") - LIST(APPEND ${PKG_LIB_OUTPUT} "${${PKG_NAME}_LIBRARY}") - ENDIF() - ENDIF(${PKG_NAME}_FOUND) - - UNSET(_lib CACHE) - UNSET(_incl CACHE) -ENDMACRO(ProcessPackage name) +IF(ENABLE_PCRE2 MATCHES "ON") + SET(WITH_PCRE2 1) + # For utf8 API + LIST(APPEND CMAKE_REQUIRED_DEFINITIONS "-DPCRE2_CODE_UNIT_WIDTH=8") +ENDIF() ############################# CONFIG SECTION ############################################# # Initial set @@ -505,84 +116,8 @@ INCLUDE_DIRECTORIES("${CMAKE_SOURCE_DIR}/" "${CMAKE_BINARY_DIR}/src" #Stored in the binary dir "${CMAKE_BINARY_DIR}/src/libcryptobox") -SET(POE_LOOP "Loop::IO_Poll") SET(TAR "tar") - -# Platform specific configuration -IF(CMAKE_SYSTEM_NAME MATCHES "^.*BSD$|DragonFly") - ADD_DEFINITIONS(-DFREEBSD -D_BSD_SOURCE) - CONFIGURE_FILE(freebsd/rspamd.sh.in freebsd/rspamd @ONLY) - MESSAGE(STATUS "Configuring for BSD system") - # Find util library - ProcessPackage(LIBUTIL LIBRARY util INCLUDE libutil.h - ROOT ${LIBUTIL_ROOT_DIR} OPTIONAL) - IF(WITH_LIBUTIL) - SET(HAVE_LIBUTIL_H 1) - LIST(APPEND CMAKE_REQUIRED_LIBRARIES util) - CHECK_FUNCTION_EXISTS(pidfile_open HAVE_PIDFILE) - CHECK_FUNCTION_EXISTS(pidfile_fileno HAVE_PIDFILE_FILENO) - ENDIF() - IF(CMAKE_SYSTEM_NAME MATCHES "^NetBSD$") - LIST(APPEND CMAKE_REQUIRED_LIBRARIES rt) - ENDIF() - SET(POE_LOOP "Loop::Kqueue") - SET(TAR "gtar") -ENDIF() - -IF(CMAKE_SYSTEM_NAME STREQUAL "Darwin") - ADD_DEFINITIONS(-D_BSD_SOURCE -DDARWIN) - SET(CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS} -undefined dynamic_lookup") - IF(ENABLE_LUAJIT MATCHES "ON") - SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -pagezero_size 10000 -image_base 100000000") - ENDIF(ENABLE_LUAJIT MATCHES "ON") - MESSAGE(STATUS "Configuring for Darwin") - SET(TAR "gnutar") - SET(CMAKE_FIND_FRAMEWORK "NEVER") -ENDIF(CMAKE_SYSTEM_NAME STREQUAL "Darwin") - -IF(CMAKE_SYSTEM_NAME STREQUAL "Linux") - ADD_DEFINITIONS(-D_GNU_SOURCE -DLINUX) - # Workaround with architecture specific includes - #IF(IS_DIRECTORY "/usr/include/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/") - # INCLUDE_DIRECTORIES("/usr/include/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/") - # LIST(APPEND CMAKE_REQUIRED_INCLUDES "/usr/include/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/") - #ENDIF(IS_DIRECTORY "/usr/include/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/") - - LIST(APPEND CMAKE_REQUIRED_LIBRARIES rt) - LIST(APPEND CMAKE_REQUIRED_LIBRARIES dl) - LIST(APPEND CMAKE_REQUIRED_LIBRARIES resolv) - MESSAGE(STATUS "Configuring for Linux") - IF(EXISTS "/etc/debian_version") - SET(LINUX_START_SCRIPT "rspamd_debian.in") - ELSE(EXISTS "/etc/debian_version") - SET(LINUX_START_SCRIPT "rspamd_rh.in") - ENDIF(EXISTS "/etc/debian_version") - SET(POE_LOOP "XS::Loop::EPoll") -ENDIF(CMAKE_SYSTEM_NAME STREQUAL "Linux") - -IF(CMAKE_SYSTEM_NAME STREQUAL "SunOS") - - IF("${CMAKE_C_COMPILER_ID}" MATCHES SunPro) - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xa -xregs=no%frameptr -xstrconst -xc99") - IF(ENABLE_OPTIMIZATION MATCHES "ON") - SET(CMAKE_C_OPT_FLAGS "-fast -xdepend") - ELSE(ENABLE_OPTIMIZATION MATCHES "ON") - SET(CMAKE_C_OPT_FLAGS "-xO0") - ENDIF(ENABLE_OPTIMIZATION MATCHES "ON") - ENDIF("${CMAKE_C_COMPILER_ID}" MATCHES SunPro) - - ADD_DEFINITIONS(-D__EXTENSIONS__ -DSOLARIS -D_POSIX_SOURCE -D_POSIX_C_SOURCE=200112) - LIST(APPEND CMAKE_REQUIRED_LIBRARIES rt) - LIST(APPEND CMAKE_REQUIRED_LIBRARIES dl) - LIST(APPEND CMAKE_REQUIRED_LIBRARIES resolv) - LIST(APPEND CMAKE_REQUIRED_LIBRARIES nsl) - LIST(APPEND CMAKE_REQUIRED_LIBRARIES socket) - LIST(APPEND CMAKE_REQUIRED_LIBRARIES umem) - # Ugly hack, but FindOpenSSL on Solaris does not link with libcrypto - SET(CMAKE_VERBOSE_MAKEFILE ON) - SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH FALSE) - SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib:${RSPAMD_LIBDIR}") -ENDIF(CMAKE_SYSTEM_NAME STREQUAL "SunOS") +INCLUDE(OSDep) # Now find libraries and headers LIST(APPEND RSPAMD_REQUIRED_LIBRARIES "m") @@ -612,7 +147,7 @@ ELSE(ENABLE_LUAJIT MATCHES "ON") ENDIF(NOT LUA_FOUND) ENDIF(ENABLE_LUAJIT MATCHES "ON") -IF(ENABLE_JEMALLOC MATCHES "ON") +IF(ENABLE_JEMALLOC MATCHES "ON" AND NOT SANITIZE) ProcessPackage(JEMALLOC LIBRARY jemalloc_pic jemalloc INCLUDE jemalloc.h INCLUDE_SUFFIXES include/jemalloc ROOT ${JEMALLOC_ROOT_DIR}) SET(WITH_JEMALLOC "1") @@ -655,162 +190,9 @@ ProcessPackage(SODIUM LIBRARY sodium INCLUDE sodium.h INCLUDE_SUFFIXES include/libsodium include/sodium ROOT ${LIBSODIUM_ROOT_DIR} MODULES libsodium>=1.0.0) -IF(ENABLE_BLAS MATCHES "ON") -ProcessPackage(BLAS OPTIONAL_INCLUDE LIBRARY openblas blas - INCLUDE cblas.h INCLUDE_SUFFIXES include/openblas - include/blas - ROOT ${BLAS_ROOT_DIR} - LIB_OUTPUT BLAS_REQUIRED_LIBRARIES) -ENDIF() - -IF(WITH_BLAS) - MESSAGE(STATUS "Use openblas to accelerate kann") - IF(NOT BLAS_INCLUDE) - FIND_FILE(HAVE_CBLAS_H HINTS "${RSPAMD_SEARCH_PATH}" - NAMES cblas.h - DOC "Path to cblas.h header") - IF(NOT HAVE_CBLAS_H) - MESSAGE(STATUS "Blas header cblas.h has not been found, use internal workaround") - ELSE() - ADD_DEFINITIONS(-DHAVE_CBLAS_H) - ENDIF() - ELSE() - ADD_DEFINITIONS(-DHAVE_CBLAS_H) - ENDIF() - ADD_DEFINITIONS(-DHAVE_CBLAS) -ENDIF(WITH_BLAS) - -IF(ENABLE_HYPERSCAN MATCHES "ON") - ProcessPackage(HYPERSCAN LIBRARY hs INCLUDE hs.h INCLUDE_SUFFIXES - hs include/hs - ROOT ${HYPERSCAN_ROOT_DIR} MODULES libhs) - SET(WITH_HYPERSCAN 1) - - # For static linking with Hyperscan we need to link using CXX - IF (ENABLE_HYPERSCAN MATCHES "ON") - IF(${HYPERSCAN_LIBRARY} MATCHES ".*[.]a$" OR STATIC_HYPERSCAN) - ENABLE_LANGUAGE(CXX) - SET(USE_CXX_LINKER 1) - ENDIF() - ENDIF() -ENDIF() - -#Check for openssl (required for dkim) -SET(HAVE_OPENSSL 1) - -# Google performance tools - -IF(ENABLE_GPERF_TOOLS MATCHES "ON") - ProcessPackage(GPERF LIBRARY profiler INCLUDE profiler.h INCLUDE_SUFFIXES include/google - ROOT ${GPERF_ROOT_DIR}) - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-omit-frame-pointer") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer") - SET(WITH_GPERF_TOOLS 1) -ENDIF(ENABLE_GPERF_TOOLS MATCHES "ON") - -CHECK_C_COMPILER_FLAG(-Wall SUPPORT_WALL) -CHECK_C_COMPILER_FLAG(-W SUPPORT_W) -CHECK_C_COMPILER_FLAG(-Wpointer-arith SUPPORT_WPOINTER) -CHECK_C_COMPILER_FLAG(-Wno-unused-parameter SUPPORT_WPARAM) -CHECK_C_COMPILER_FLAG(-Wno-unused-function SUPPORT_WFUNCTION) -CHECK_C_COMPILER_FLAG(-Wno-strict-aliasing SUPPORT_WSTRICT_ALIASING) -CHECK_C_COMPILER_FLAG(-Wunused-variable SUPPORT_WUNUSED_VAR) -CHECK_C_COMPILER_FLAG(-Wno-pointer-sign SUPPORT_WPOINTER_SIGN) -CHECK_C_COMPILER_FLAG(-Wno-sign-compare SUPPORT_WSIGN_COMPARE) -CHECK_C_COMPILER_FLAG(-Wstrict-prototypes SUPPORT_WSTRICT_PROTOTYPES) -CHECK_C_COMPILER_FLAG(-pedantic SUPPORT_PEDANTIC_FLAG) -CHECK_C_COMPILER_FLAG(-Wno-unused-const-variable SUPPORT_WNO_UNUSED_CONST) -# GCC 6 specific -CHECK_C_COMPILER_FLAG(-Wnull-dereference SUPPORT_WNULL_DEREFERENCE) -CHECK_C_COMPILER_FLAG(-Wduplicated-cond SUPPORT_WDUPLICATED_COND) -# GCC 7 specific -CHECK_C_COMPILER_FLAG(-Wimplicit-fallthrough SUPPORT_WIMPLICIT_FALLTHROUGH) - -IF(SUPPORT_W) - ADD_COMPILE_OPTIONS("-W") -ENDIF(SUPPORT_W) -IF(SUPPORT_WALL) - ADD_COMPILE_OPTIONS("-Wall") -ENDIF(SUPPORT_WALL) -IF(SUPPORT_WPOINTER) - ADD_COMPILE_OPTIONS("-Wpointer-arith") -ENDIF(SUPPORT_WPOINTER) -IF(SUPPORT_WPARAM) - ADD_COMPILE_OPTIONS("-Wno-unused-parameter") -ENDIF(SUPPORT_WPARAM) -IF(SUPPORT_WFUNCTION) - ADD_COMPILE_OPTIONS("-Wno-unused-function") -ENDIF(SUPPORT_WFUNCTION) -IF(SUPPORT_WUNUSED_VAR) - ADD_COMPILE_OPTIONS("-Wunused-variable") -ENDIF(SUPPORT_WUNUSED_VAR) -IF(SUPPORT_WPOINTER_SIGN) - ADD_COMPILE_OPTIONS("-Wno-pointer-sign") -ENDIF(SUPPORT_WPOINTER_SIGN) -IF(SUPPORT_WSTRICT_PROTOTYPES) - ADD_COMPILE_OPTIONS("-Wstrict-prototypes") -ENDIF(SUPPORT_WSTRICT_PROTOTYPES) -IF(SUPPORT_WSTRICT_ALIASING) - ADD_COMPILE_OPTIONS("-Wno-strict-aliasing") - ADD_COMPILE_OPTIONS("-fno-strict-aliasing") -ENDIF(SUPPORT_WSTRICT_ALIASING) -#IF(SUPPORT_PEDANTIC_FLAG) -# SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -pedantic") -#ENDIF(SUPPORT_PEDANTIC_FLAG) -IF(SUPPORT_WNULL_DEREFERENCE) - ADD_COMPILE_OPTIONS("-Wnull-dereference") -ENDIF() -IF(SUPPORT_WDUPLICATED_COND) - ADD_COMPILE_OPTIONS("-Wduplicated-cond") -ENDIF() -IF(SUPPORT_WLOGICAL_OP) - ADD_COMPILE_OPTIONS("-Wlogical-op") -ENDIF() -IF(SUPPORT_WNO_UNUSED_CONST) - ADD_COMPILE_OPTIONS("-Wno-unused-const-variable") -ENDIF() -IF(SUPPORT_WSIGN_COMPARE) - ADD_COMPILE_OPTIONS("-Wno-sign-compare") -ENDIF() -IF(SUPPORT_WIMPLICIT_FALLTHROUGH) - ADD_COMPILE_OPTIONS("-Wno-implicit-fallthrough") -ENDIF(SUPPORT_WIMPLICIT_FALLTHROUGH) - -CHECK_C_COMPILER_FLAG(-fPIC SUPPORT_FPIC) -IF(SUPPORT_FPIC) - ADD_COMPILE_OPTIONS("-fPIC") -ENDIF(SUPPORT_FPIC) - - # Optimization flags -IF(NOT CMAKE_C_OPT_FLAGS) - IF(ENABLE_OPTIMIZATION MATCHES "ON") - SET(CMAKE_C_OPT_FLAGS "-g -O2") - IF(${CMAKE_VERSION} VERSION_GREATER "3.9.0") - CMAKE_POLICY(SET CMP0069 NEW) - INCLUDE(CheckIPOSupported) - check_ipo_supported(RESULT SUPPORT_LTO OUTPUT LTO_DIAG ) - if(SUPPORT_LTO) - SET(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE) - else() - message(WARNING "IPO is not supported: ${LTO_DIAG}") - endif() - ENDIF() - ELSE(ENABLE_OPTIMIZATION MATCHES "ON") - IF(ENABLE_FULL_DEBUG MATCHES "ON") - ADD_DEFINITIONS(-DFULL_DEBUG) - SET(CMAKE_C_OPT_FLAGS "-g -Og") - ELSE(ENABLE_FULL_DEBUG MATCHES "ON") - SET(CMAKE_C_OPT_FLAGS "-g -O2") - ENDIF(ENABLE_FULL_DEBUG MATCHES "ON") - ENDIF(ENABLE_OPTIMIZATION MATCHES "ON") -ENDIF(NOT CMAKE_C_OPT_FLAGS) - -IF(ENABLE_COVERAGE) - SET(CMAKE_C_OPT_FLAGS "-g -Og") - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --coverage") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --coverage") - SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --coverage") -ENDIF(ENABLE_COVERAGE) +include (CompilerWarnings) +include (Hyperscan) +include (Openblas) IF(ENABLE_LUA_TRACE) SET(WITH_LUA_TRACE 1) @@ -1204,6 +586,7 @@ ADD_SUBDIRECTORY(contrib/lua-lpeg) ADD_SUBDIRECTORY(contrib/t1ha) ADD_SUBDIRECTORY(contrib/libev) ADD_SUBDIRECTORY(contrib/kann) +ADD_SUBDIRECTORY(contrib/fastutf8) IF (NOT WITH_LUAJIT) ADD_SUBDIRECTORY(contrib/lua-bit) @@ -1235,7 +618,6 @@ ADD_SUBDIRECTORY(utils) ############################ TARGETS SECTION ############################### - CONFIGURE_FILE(config.h.in src/config.h) ##################### INSTALLATION ########################################## @@ -1254,8 +636,8 @@ INSTALL(CODE "FILE(MAKE_DIRECTORY \$ENV{DESTDIR}${RULESDIR})") LIST(LENGTH CONFFILES CONFLIST_COUNT) MATH(EXPR CONFLIST_MAX ${CONFLIST_COUNT}-1) -FILE(GLOB_RECURSE CONF_FILES RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/conf" - "${CMAKE_CURRENT_SOURCE_DIR}/conf/*" ) +FILE(GLOB_RECURSE CONF_FILES RELATIVE "${CMAKE_SOURCE_DIR}/conf" + "${CMAKE_SOURCE_DIR}/conf/*" ) FOREACH(CONF_FILE ${CONF_FILES}) GET_FILENAME_COMPONENT(_rp ${CONF_FILE} PATH) INSTALL(CODE "FILE(MAKE_DIRECTORY \$ENV{DESTDIR}${CONFDIR}/${_rp})") @@ -1263,16 +645,6 @@ FOREACH(CONF_FILE ${CONF_FILES}) DESTINATION ${CONFDIR}/${_rp}) ENDFOREACH(CONF_FILE) -SET(MAIN_CONF "conf/rspamd.conf") -IF(BUILD_PORT) - INSTALL_IF_NOT_EXISTS(${MAIN_CONF} ${CONFDIR} "rspamd.conf" ".sample") -ELSE(BUILD_PORT) - INSTALL_IF_NOT_EXISTS(${MAIN_CONF} ${CONFDIR} "rspamd.conf" "") -ENDIF(BUILD_PORT) -IF(INSTALL_EXAMPLES MATCHES "ON") - INSTALL(FILES ${MAIN_CONF} DESTINATION ${EXAMPLESDIR}) -ENDIF(INSTALL_EXAMPLES MATCHES "ON") - # Lua plugins FILE(GLOB LUA_PLUGINS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/src/plugins/lua" @@ -1357,7 +729,7 @@ ENDIF(NOT DEBIAN_BUILD) find_program(_PVS_STUDIO "pvs-studio-analyzer") if(_PVS_STUDIO) - include(${CMAKE_SOURCE_DIR}/PVS-Studio.cmake) + include(PVS-Studio) pvs_studio_add_target(TARGET ${PROJECT_NAME}.analyze ANALYZE ${PROJECT_NAME} rspamd-server rspamadm rspamc OUTPUT FORMAT errorfile @@ -1,3 +1,41 @@ +2.2: 19 Nov 2019 + * [Conf] Antivirus: Fix the default config + * [Feature] Add verdict library in lua + * [Feature] Allow exception when choosing upstream + * [Feature] Allow to disable symbols from the metric config + * [Feature] Allow to limit maps per specific worker + * [Feature] Always validate Rspamd protocol output + * [Feature] Antivirus: Add preliminary virustotal support + * [Feature] Clickhouse: Rework Clickhouse collection logic + * [Feature] Improve base64 usage + * [Feature] Shutdown timeout is now associated with task timeout + * [Fix] #3129 Multiple classifiers on redis working incorrectly + * [Fix] Allow real upstreams configuration + * [Fix] Another try to fix slow callbacks and timers + * [Fix] Check results of write message as SSL can bork them + * [Fix] Clickhouse: Avoid potential races in collection + * [Fix] Clickhouse: Fix periodic script + * [Fix] Fail DNS upstream on each retransmit attempt + * [Fix] Fix consistent hashing when upstreams are marked inactive + * [Fix] Fix issues found + * [Fix] Fix off-by-one in retries for the proxy + * [Fix] Fix termination + * [Fix] Fix upstreams exclusion logic + * [Fix] Fix utf8 validation for symbols options and empty strings + * [Fix] Oops, fix maps reload + * [Fix] Rbl: Allow utf8 lookups for IDN domains + * [Fix] Sigh, another try to fix brain-damaged openssl + * [Project] Add fast utf8 validation library + * [Project] Use own utf8 validation instead of glib + * [Rework] Another phase of finish actions rework + * [Rework] Further cmake system rework + * [Rework] Further isolation of the controller's functions + * [Rework] Make cmake structure more modular + * [Rework] Move cmake modules to a dedicated path + * [Rework] Replace controller functions by any scanner worker if needed + * [Rework] Rework final scripts logic + * [Rework] Rewrite rspamd_str_make_utf_valid function + 2.1: 28 Oct 2019 * [Conf] Update neural.conf * [CritFix] Fix dkim verification for multiple headers listed diff --git a/cmake/ArchDep.cmake b/cmake/ArchDep.cmake new file mode 100644 index 000000000..7b9a84491 --- /dev/null +++ b/cmake/ArchDep.cmake @@ -0,0 +1,101 @@ +TARGET_ARCHITECTURE(ARCH) + +SET(ASM_CODE " + .macro TEST1 op + \\op %eax, %eax + .endm + TEST1 xorl + ") +ASM_OP(HAVE_SLASHMACRO "slash macro convention") + +SET(ASM_CODE " + .macro TEST1 op + $0 %eax, %eax + .endm + TEST1 xorl + ") +ASM_OP(HAVE_DOLLARMACRO "dollar macro convention") + +# For now we support only x86_64 architecture with optimizations +IF("${ARCH}" STREQUAL "x86_64") + IF(NOT HAVE_SLASHMACRO AND NOT HAVE_DOLLARMACRO) + MESSAGE(FATAL_ERROR "Your assembler cannot compile macros, please check your CMakeFiles/CMakeError.log") + ENDIF() + + SET(ASM_CODE "vpaddq %ymm0, %ymm0, %ymm0") + ASM_OP(HAVE_AVX2 "avx2") + # Handle broken compilers, sigh... + IF(HAVE_AVX2) + CHECK_C_SOURCE_COMPILES( + " +#include <stddef.h> +#pragma GCC push_options +#pragma GCC target(\"avx2\") +#ifndef __SSE2__ +#define __SSE2__ +#endif +#ifndef __SSE__ +#define __SSE__ +#endif +#ifndef __SSE4_2__ +#define __SSE4_2__ +#endif +#ifndef __SSE4_1__ +#define __SSE4_1__ +#endif +#ifndef __SSEE3__ +#define __SSEE3__ +#endif +#ifndef __AVX__ +#define __AVX__ +#endif +#ifndef __AVX2__ +#define __AVX2__ +#endif + +#ifndef __clang__ +#if __GNUC__ < 6 +#error Broken due to compiler bug +#endif +#endif + +#include <immintrin.h> +static void foo(const char* a) __attribute__((__target__(\"avx2\"))); +static void foo(const char* a) +{ + __m256i str = _mm256_loadu_si256((__m256i *)a); + __m256i t = _mm256_loadu_si256((__m256i *)a + 1); + _mm256_add_epi8(str, t); +} +int main(int argc, char** argv) { + foo(argv[0]); +}" HAVE_AVX2_C_COMPILER) + IF(NOT HAVE_AVX2_C_COMPILER) + MESSAGE(STATUS "Your compiler has broken AVX2 support") + UNSET(HAVE_AVX2 CACHE) + ENDIF() + ENDIF() + SET(ASM_CODE "vpaddq %xmm0, %xmm0, %xmm0") + ASM_OP(HAVE_AVX "avx") + SET(ASM_CODE "pmuludq %xmm0, %xmm0") + ASM_OP(HAVE_SSE2 "sse2") + SET(ASM_CODE "lddqu 0(%esi), %xmm0") + ASM_OP(HAVE_SSE3 "sse3") + SET(ASM_CODE "pshufb %xmm0, %xmm0") + ASM_OP(HAVE_SSSE3 "ssse3") + SET(ASM_CODE "pblendw \$0, %xmm0, %xmm0") + ASM_OP(HAVE_SSE41 "sse41") + SET(ASM_CODE "crc32 %eax, %eax") + ASM_OP(HAVE_SSE42 "sse42") +ENDIF() + +IF ("${ARCH}" STREQUAL "x86_64") + MESSAGE(STATUS "Enable sse2 on x86_64 architecture") + IF((CMAKE_C_COMPILER_ID MATCHES "GNU") OR (CMAKE_C_COMPILER_ID MATCHES "Clang")) + ADD_COMPILE_OPTIONS(-msse2) + ELSEIF(CMAKE_C_COMPILER_ID MATCHES "Intel") + ADD_COMPILE_OPTIONS(/QxSSE2) + ELSEIF((CMAKE_C_COMPILER_ID MATCHES "MSVC")) + ADD_COMPILE_OPTIONS(/arch:SSE2) + ENDIF() +ENDIF()
\ No newline at end of file diff --git a/src/libcryptobox/AsmOp.cmake b/cmake/AsmOp.cmake index bcf9d996a..bcf9d996a 100644 --- a/src/libcryptobox/AsmOp.cmake +++ b/cmake/AsmOp.cmake diff --git a/cmake/CompilerWarnings.cmake b/cmake/CompilerWarnings.cmake new file mode 100644 index 000000000..a9797f539 --- /dev/null +++ b/cmake/CompilerWarnings.cmake @@ -0,0 +1,72 @@ +CHECK_C_COMPILER_FLAG(-Wall SUPPORT_WALL) +CHECK_C_COMPILER_FLAG(-W SUPPORT_W) +CHECK_C_COMPILER_FLAG(-Wpointer-arith SUPPORT_WPOINTER) +CHECK_C_COMPILER_FLAG(-Wno-unused-parameter SUPPORT_WPARAM) +CHECK_C_COMPILER_FLAG(-Wno-unused-function SUPPORT_WFUNCTION) +CHECK_C_COMPILER_FLAG(-Wno-strict-aliasing SUPPORT_WSTRICT_ALIASING) +CHECK_C_COMPILER_FLAG(-Wunused-variable SUPPORT_WUNUSED_VAR) +CHECK_C_COMPILER_FLAG(-Wno-pointer-sign SUPPORT_WPOINTER_SIGN) +CHECK_C_COMPILER_FLAG(-Wno-sign-compare SUPPORT_WSIGN_COMPARE) +CHECK_C_COMPILER_FLAG(-Wstrict-prototypes SUPPORT_WSTRICT_PROTOTYPES) +CHECK_C_COMPILER_FLAG(-pedantic SUPPORT_PEDANTIC_FLAG) +CHECK_C_COMPILER_FLAG(-Wno-unused-const-variable SUPPORT_WNO_UNUSED_CONST) +# GCC 6 specific +CHECK_C_COMPILER_FLAG(-Wnull-dereference SUPPORT_WNULL_DEREFERENCE) +CHECK_C_COMPILER_FLAG(-Wduplicated-cond SUPPORT_WDUPLICATED_COND) +# GCC 7 specific +CHECK_C_COMPILER_FLAG(-Wimplicit-fallthrough SUPPORT_WIMPLICIT_FALLTHROUGH) + +IF(SUPPORT_W) + ADD_COMPILE_OPTIONS("-W") +ENDIF(SUPPORT_W) +IF(SUPPORT_WALL) + ADD_COMPILE_OPTIONS("-Wall") +ENDIF(SUPPORT_WALL) +IF(SUPPORT_WPOINTER) + ADD_COMPILE_OPTIONS("-Wpointer-arith") +ENDIF(SUPPORT_WPOINTER) +IF(SUPPORT_WPARAM) + ADD_COMPILE_OPTIONS("-Wno-unused-parameter") +ENDIF(SUPPORT_WPARAM) +IF(SUPPORT_WFUNCTION) + ADD_COMPILE_OPTIONS("-Wno-unused-function") +ENDIF(SUPPORT_WFUNCTION) +IF(SUPPORT_WUNUSED_VAR) + ADD_COMPILE_OPTIONS("-Wunused-variable") +ENDIF(SUPPORT_WUNUSED_VAR) +IF(SUPPORT_WPOINTER_SIGN) + ADD_COMPILE_OPTIONS("-Wno-pointer-sign") +ENDIF(SUPPORT_WPOINTER_SIGN) +IF(SUPPORT_WSTRICT_PROTOTYPES) + ADD_COMPILE_OPTIONS("-Wstrict-prototypes") +ENDIF(SUPPORT_WSTRICT_PROTOTYPES) +IF(SUPPORT_WSTRICT_ALIASING) + ADD_COMPILE_OPTIONS("-Wno-strict-aliasing") + ADD_COMPILE_OPTIONS("-fno-strict-aliasing") +ENDIF(SUPPORT_WSTRICT_ALIASING) +#IF(SUPPORT_PEDANTIC_FLAG) +# SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -pedantic") +#ENDIF(SUPPORT_PEDANTIC_FLAG) +IF(SUPPORT_WNULL_DEREFERENCE) + ADD_COMPILE_OPTIONS("-Wnull-dereference") +ENDIF() +IF(SUPPORT_WDUPLICATED_COND) + ADD_COMPILE_OPTIONS("-Wduplicated-cond") +ENDIF() +IF(SUPPORT_WLOGICAL_OP) + ADD_COMPILE_OPTIONS("-Wlogical-op") +ENDIF() +IF(SUPPORT_WNO_UNUSED_CONST) + ADD_COMPILE_OPTIONS("-Wno-unused-const-variable") +ENDIF() +IF(SUPPORT_WSIGN_COMPARE) + ADD_COMPILE_OPTIONS("-Wno-sign-compare") +ENDIF() +IF(SUPPORT_WIMPLICIT_FALLTHROUGH) + ADD_COMPILE_OPTIONS("-Wno-implicit-fallthrough") +ENDIF(SUPPORT_WIMPLICIT_FALLTHROUGH) + +CHECK_C_COMPILER_FLAG(-fPIC SUPPORT_FPIC) +IF(SUPPORT_FPIC) + ADD_COMPILE_OPTIONS("-fPIC") +ENDIF(SUPPORT_FPIC)
\ No newline at end of file diff --git a/FindArch.cmake b/cmake/FindArch.cmake index 277289248..277289248 100644 --- a/FindArch.cmake +++ b/cmake/FindArch.cmake diff --git a/cmake/FindLua.cmake b/cmake/FindLua.cmake new file mode 100644 index 000000000..7aa3e8b82 --- /dev/null +++ b/cmake/FindLua.cmake @@ -0,0 +1,114 @@ +# Find lua installation +MACRO(FindLua) + # Find lua libraries + UNSET(LUA_INCLUDE_DIR CACHE) + UNSET(LUA_LIBRARY CACHE) + CMAKE_PARSE_ARGUMENTS(LUA "" "VERSION_MAJOR;VERSION_MINOR;ROOT" "" ${ARGN}) + + IF(NOT LUA_VERSION_MAJOR OR NOT LUA_VERSION_MINOR) + MESSAGE(FATAL_ERROR "Invalid FindLua invocation: ${ARGN}") + ENDIF() + + IF(ENABLE_LUAJIT MATCHES "ON") + MESSAGE(STATUS "Check for luajit ${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}") + FIND_PATH(LUA_INCLUDE_DIR luajit.h + HINTS + "${RSPAMD_SEARCH_PATH}" "${LUA_ROOT}" + $ENV{LUA_DIR} + PATH_SUFFIXES "include/luajit-2.0" + "include/luajit-2.1" + "include/luajit${LUA_VERSION_MAJOR}${LUA_VERSION_MINOR}" + "include/luajit${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" + "include/luajit-${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" + "include/luajit-${LUA_VERSION_MAJOR}_${LUA_VERSION_MINOR}-2.0" + "include/luajit-${LUA_VERSION_MAJOR}_${LUA_VERSION_MINOR}-2.1" + "include/luajit" + "include/lua${LUA_VERSION_MAJOR}${LUA_VERSION_MINOR}" + "include/lua${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" + "include/lua-${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" + include/lua include + PATHS ${RSPAMD_DEFAULT_INCLUDE_PATHS} + ) + FIND_LIBRARY(LUA_LIBRARY + NAMES luajit + "luajit-2.0" + "luajit2.0" + "luajit${LUA_VERSION_MAJOR}${LUA_VERSION_MINOR}" + "luajit${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" + "luajit-${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" + HINTS + "${RSPAMD_SEARCH_PATH}" "${LUA_ROOT}" + $ENV{LUA_DIR} + PATH_SUFFIXES lib64 lib + PATHS ${RSPAMD_DEFAULT_LIBRARY_PATHS} + DOC "Lua library" + ) + + IF(NOT LUA_LIBRARY OR NOT LUA_INCLUDE_DIR) + MESSAGE(STATUS "Fallback from luajit to plain lua") + SET(ENABLE_LUAJIT "OFF") + MESSAGE(STATUS "Check for lua ${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}") + FIND_PATH(LUA_INCLUDE_DIR lua.h + HINTS + "${RSPAMD_SEARCH_PATH}" "${LUA_ROOT}" + $ENV{LUA_DIR} + PATH_SUFFIXES "include/lua${LUA_VERSION_MAJOR}${LUA_VERSION_MINOR}" + "include/lua${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" + "include/lua-${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" + include/lua include + PATHS ${RSPAMD_DEFAULT_INCLUDE_PATHS} + ) + FIND_LIBRARY(LUA_LIBRARY + NAMES lua + "lua${LUA_VERSION_MAJOR}${LUA_VERSION_MINOR}" + "lua${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" + "lua-${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" + HINTS + "${RSPAMD_SEARCH_PATH}" "${LUA_ROOT}" + $ENV{LUA_DIR} + PATH_SUFFIXES lib64 lib + PATHS ${RSPAMD_DEFAULT_LIBRARY_PATHS} + DOC "Lua library" + ) + ELSE() + SET(WITH_LUAJIT 1) + ENDIF() + ELSE(ENABLE_LUAJIT MATCHES "ON") + MESSAGE(STATUS "Check for lua ${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}") + FIND_PATH(LUA_INCLUDE_DIR lua.h + HINTS + "${RSPAMD_SEARCH_PATH}" "${LUA_ROOT}" + $ENV{LUA_DIR} + PATH_SUFFIXES "include/lua${LUA_VERSION_MAJOR}${LUA_VERSION_MINOR}" + "include/lua${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" + "include/lua-${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" + include/lua include + PATHS ${RSPAMD_DEFAULT_INCLUDE_PATHS} + ) + FIND_LIBRARY(LUA_LIBRARY + NAMES lua + "lua${LUA_VERSION_MAJOR}${LUA_VERSION_MINOR}" + "lua${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" + "lua-${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" + HINTS + "${RSPAMD_SEARCH_PATH}" "${LUA_ROOT}" + $ENV{LUA_DIR} + PATH_SUFFIXES lib64 lib + PATHS ${RSPAMD_DEFAULT_LIBRARY_PATHS} + DOC "Lua library" + ) + ENDIF(ENABLE_LUAJIT MATCHES "ON") + + IF(LUA_LIBRARY AND LUA_INCLUDE_DIR) + SET(LUA_FOUND 1) + IF(NOT LUA_VERSION_MAJOR OR NOT LUA_VERSION_MINOR) + SET(LUA_VERSION_MAJOR ${LUA_VERSION_MAJOR}) + SET(LUA_VERSION_MINOR ${LUA_VERSION_MINOR}) + ENDIF(NOT LUA_VERSION_MAJOR OR NOT LUA_VERSION_MINOR) + IF(ENABLE_LUAJIT MATCHES "ON") + MESSAGE(STATUS "Found luajit ${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR} in lib:${LUA_LIBRARY}, headers:${LUA_INCLUDE_DIR}") + ELSE(ENABLE_LUAJIT MATCHES "ON") + MESSAGE(STATUS "Found lua ${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR} in lib:${LUA_LIBRARY}, headers:${LUA_INCLUDE_DIR}") + ENDIF(ENABLE_LUAJIT MATCHES "ON") + ENDIF(LUA_LIBRARY AND LUA_INCLUDE_DIR) +ENDMACRO()
\ No newline at end of file diff --git a/FindRagel.cmake b/cmake/FindRagel.cmake index a058b7fb1..a058b7fb1 100644 --- a/FindRagel.cmake +++ b/cmake/FindRagel.cmake diff --git a/cmake/Hyperscan.cmake b/cmake/Hyperscan.cmake new file mode 100644 index 000000000..b8f83a3bb --- /dev/null +++ b/cmake/Hyperscan.cmake @@ -0,0 +1,19 @@ +option (ENABLE_HYPERSCAN "Enable hyperscan for fast regexp processing [default: OFF]" OFF) + +if (ENABLE_HYPERSCAN MATCHES "ON") + if (NOT ("${ARCH}" STREQUAL "x86_64" OR "${ARCH}" STREQUAL "i386")) + MESSAGE(FATAL_ERROR "Hyperscan is supported only on x86_64/i386 architectures") + endif () + ProcessPackage (HYPERSCAN LIBRARY hs INCLUDE hs.h INCLUDE_SUFFIXES + hs include/hs + ROOT ${HYPERSCAN_ROOT_DIR} MODULES libhs) + set (WITH_HYPERSCAN 1) + + # For static linking with Hyperscan we need to link using CXX + if (ENABLE_HYPERSCAN MATCHES "ON") + if (${HYPERSCAN_LIBRARY} MATCHES ".*[.]a$" OR STATIC_HYPERSCAN) + enable_language (CXX) + set (USE_CXX_LINKER 1) + endif () + endif () +endif ()
\ No newline at end of file diff --git a/cmake/OSDep.cmake b/cmake/OSDep.cmake new file mode 100644 index 000000000..93f814675 --- /dev/null +++ b/cmake/OSDep.cmake @@ -0,0 +1,75 @@ +# Platform specific configuration +IF(CMAKE_SYSTEM_NAME MATCHES "^.*BSD$|DragonFly") + ADD_DEFINITIONS(-DFREEBSD -D_BSD_SOURCE) + CONFIGURE_FILE(freebsd/rspamd.sh.in freebsd/rspamd @ONLY) + MESSAGE(STATUS "Configuring for BSD system") + # Find util library + ProcessPackage(LIBUTIL LIBRARY util INCLUDE libutil.h + ROOT ${LIBUTIL_ROOT_DIR} OPTIONAL) + IF(WITH_LIBUTIL) + SET(HAVE_LIBUTIL_H 1) + LIST(APPEND CMAKE_REQUIRED_LIBRARIES util) + CHECK_FUNCTION_EXISTS(pidfile_open HAVE_PIDFILE) + CHECK_FUNCTION_EXISTS(pidfile_fileno HAVE_PIDFILE_FILENO) + ENDIF() + IF(CMAKE_SYSTEM_NAME MATCHES "^NetBSD$") + LIST(APPEND CMAKE_REQUIRED_LIBRARIES rt) + ENDIF() + SET(POE_LOOP "Loop::Kqueue") + SET(TAR "gtar") +ENDIF() + +IF(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + ADD_DEFINITIONS(-D_BSD_SOURCE -DDARWIN) + SET(CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS} -undefined dynamic_lookup") + IF(ENABLE_LUAJIT MATCHES "ON") + SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -pagezero_size 10000 -image_base 100000000") + ENDIF(ENABLE_LUAJIT MATCHES "ON") + MESSAGE(STATUS "Configuring for Darwin") + SET(TAR "gnutar") + SET(CMAKE_FIND_FRAMEWORK "NEVER") +ENDIF(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + +IF(CMAKE_SYSTEM_NAME STREQUAL "Linux") + ADD_DEFINITIONS(-D_GNU_SOURCE -DLINUX) + # Workaround with architecture specific includes + #IF(IS_DIRECTORY "/usr/include/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/") + # INCLUDE_DIRECTORIES("/usr/include/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/") + # LIST(APPEND CMAKE_REQUIRED_INCLUDES "/usr/include/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/") + #ENDIF(IS_DIRECTORY "/usr/include/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/") + + LIST(APPEND CMAKE_REQUIRED_LIBRARIES rt) + LIST(APPEND CMAKE_REQUIRED_LIBRARIES dl) + LIST(APPEND CMAKE_REQUIRED_LIBRARIES resolv) + MESSAGE(STATUS "Configuring for Linux") + IF(EXISTS "/etc/debian_version") + SET(LINUX_START_SCRIPT "rspamd_debian.in") + ELSE(EXISTS "/etc/debian_version") + SET(LINUX_START_SCRIPT "rspamd_rh.in") + ENDIF(EXISTS "/etc/debian_version") + SET(POE_LOOP "XS::Loop::EPoll") +ENDIF(CMAKE_SYSTEM_NAME STREQUAL "Linux") + +IF(CMAKE_SYSTEM_NAME STREQUAL "SunOS") + + IF("${CMAKE_C_COMPILER_ID}" MATCHES SunPro) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xa -xregs=no%frameptr -xstrconst -xc99") + IF(ENABLE_OPTIMIZATION MATCHES "ON") + SET(CMAKE_C_OPT_FLAGS "-fast -xdepend") + ELSE(ENABLE_OPTIMIZATION MATCHES "ON") + SET(CMAKE_C_OPT_FLAGS "-xO0") + ENDIF(ENABLE_OPTIMIZATION MATCHES "ON") + ENDIF("${CMAKE_C_COMPILER_ID}" MATCHES SunPro) + + ADD_DEFINITIONS(-D__EXTENSIONS__ -DSOLARIS -D_POSIX_SOURCE -D_POSIX_C_SOURCE=200112) + LIST(APPEND CMAKE_REQUIRED_LIBRARIES rt) + LIST(APPEND CMAKE_REQUIRED_LIBRARIES dl) + LIST(APPEND CMAKE_REQUIRED_LIBRARIES resolv) + LIST(APPEND CMAKE_REQUIRED_LIBRARIES nsl) + LIST(APPEND CMAKE_REQUIRED_LIBRARIES socket) + LIST(APPEND CMAKE_REQUIRED_LIBRARIES umem) + # Ugly hack, but FindOpenSSL on Solaris does not link with libcrypto + SET(CMAKE_VERBOSE_MAKEFILE ON) + SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH FALSE) + SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib:${RSPAMD_LIBDIR}") +ENDIF(CMAKE_SYSTEM_NAME STREQUAL "SunOS")
\ No newline at end of file diff --git a/cmake/Openblas.cmake b/cmake/Openblas.cmake new file mode 100644 index 000000000..e2afa92c5 --- /dev/null +++ b/cmake/Openblas.cmake @@ -0,0 +1,26 @@ +option (ENABLE_BLAS "Enable openblas for fast neural network processing [default: OFF]" OFF) + +IF(ENABLE_BLAS MATCHES "ON") + ProcessPackage(BLAS OPTIONAL_INCLUDE LIBRARY openblas blas + INCLUDE cblas.h INCLUDE_SUFFIXES include/openblas + include/blas + ROOT ${BLAS_ROOT_DIR} + LIB_OUTPUT BLAS_REQUIRED_LIBRARIES) +ENDIF() + +IF(WITH_BLAS) + MESSAGE(STATUS "Use openblas to accelerate kann") + IF(NOT BLAS_INCLUDE) + FIND_FILE(HAVE_CBLAS_H HINTS "${RSPAMD_SEARCH_PATH}" + NAMES cblas.h + DOC "Path to cblas.h header") + IF(NOT HAVE_CBLAS_H) + MESSAGE(STATUS "Blas header cblas.h has not been found, use internal workaround") + ELSE() + ADD_DEFINITIONS(-DHAVE_CBLAS_H) + ENDIF() + ELSE() + ADD_DEFINITIONS(-DHAVE_CBLAS_H) + ENDIF() + ADD_DEFINITIONS(-DHAVE_CBLAS) +ENDIF(WITH_BLAS)
\ No newline at end of file diff --git a/PVS-Studio.cmake b/cmake/PVS-Studio.cmake index 6001f3337..6001f3337 100644 --- a/PVS-Studio.cmake +++ b/cmake/PVS-Studio.cmake diff --git a/cmake/Paths.cmake b/cmake/Paths.cmake new file mode 100644 index 000000000..475c6f0fe --- /dev/null +++ b/cmake/Paths.cmake @@ -0,0 +1,77 @@ +# Now CMAKE_INSTALL_PREFIX is a base prefix for everything +# CONFDIR - for configuration +# LOCAL_CONFDIR - for local configuration +# MANDIR - for manual pages +# RUNDIR - for runtime files +# DBDIR - for static files +# LOGDIR - for log files +# EXAMPLESDIR - for examples files + +IF(NOT CONFDIR) + SET(CONFDIR "${CMAKE_INSTALL_PREFIX}/etc/rspamd") +ENDIF(NOT CONFDIR) + +IF(NOT LOCAL_CONFDIR) + SET(LOCAL_CONFDIR "${CONFDIR}") +ENDIF(NOT LOCAL_CONFDIR) + +IF(NOT MANDIR) + SET(MANDIR "${CMAKE_INSTALL_PREFIX}/share/man") +ENDIF(NOT MANDIR) + +IF(NOT RUNDIR) + SET(RUNDIR "/var/run/rspamd") +ENDIF(NOT RUNDIR) + +IF(NOT DBDIR) + SET(DBDIR "/var/lib/rspamd") +ENDIF(NOT DBDIR) + +IF(NOT LOGDIR) + SET(LOGDIR "/var/log/rspamd") +ENDIF(NOT LOGDIR) + +IF(NOT SHAREDIR) + SET(SHAREDIR "${CMAKE_INSTALL_PREFIX}/share/rspamd") +ENDIF(NOT SHAREDIR) + +IF(NOT EXAMPLESDIR) + SET(EXAMPLESDIR "${CMAKE_INSTALL_PREFIX}/share/examples/rspamd") +ENDIF(NOT EXAMPLESDIR) + +IF(NOT LUALIBDIR) + SET(LUALIBDIR "${SHAREDIR}/lualib") +ENDIF(NOT LUALIBDIR) + +IF(NOT PLUGINSDIR) + SET(PLUGINSDIR "${SHAREDIR}/plugins") +ENDIF(NOT PLUGINSDIR) + +IF(NOT RULESDIR) + SET(RULESDIR "${SHAREDIR}/rules") +ENDIF(NOT RULESDIR) + +IF(NOT WWWDIR) + SET(WWWDIR "${SHAREDIR}/www") +ENDIF(NOT WWWDIR) + +# Set libdir +IF(NOT LIBDIR) + SET(RSPAMD_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib/rspamd") +ELSE(NOT LIBDIR) + SET(RSPAMD_LIBDIR "${LIBDIR}") +ENDIF(NOT LIBDIR) +SET(CMAKE_MACOSX_RPATH ON) +SET(CMAKE_INSTALL_RPATH "${RSPAMD_LIBDIR}") + +# Set includedir +IF(NOT INCLUDEDIR) + SET(INCLUDEDIR include/rspamd) +ENDIF(NOT INCLUDEDIR) + +IF(NOT SYSTEMDDIR) + SET(SYSTEMDDIR ${CMAKE_INSTALL_PREFIX}/lib/systemd/system) +ENDIF(NOT SYSTEMDDIR) + +SET(RSPAMD_DEFAULT_INCLUDE_PATHS "/opt;/usr;/usr/local;/opt/local;/usr/pkg;/opt/csw;/sw") +SET(RSPAMD_DEFAULT_LIBRARY_PATHS "/usr/local;/usr/pkg;/usr;/Library/Frameworks;/sw;/opt/local;/opt/csw;/opt") diff --git a/cmake/ProcessPackage.cmake b/cmake/ProcessPackage.cmake new file mode 100644 index 000000000..a46fd85b4 --- /dev/null +++ b/cmake/ProcessPackage.cmake @@ -0,0 +1,122 @@ +# Process required package by using FindPackage and calling for INCLUDE_DIRECTORIES and +# setting list of required libraries +# Usage: +# ProcessPackage(VAR [OPTIONAL] [ROOT path] [INCLUDE path] +# [LIBRARY path] [INCLUDE_SUFFIXES path1 path2 ...] [LIB_SUFFIXES path1 path2 ...] +# [MODULES module1 module2 ...]) +# params: +# OPTIONAL - do not fail if a package has not been found +# ROOT - defines root directory for a package +# INCLUDE - name of the include file to check +# LIBRARY - name of the library to check +# INCLUDE_SUFFIXES - list of include suffixes (relative to ROOT) +# LIB_SUFFIXES - list of library suffixes +# MODULES - modules to search using pkg_config +MACRO(ProcessPackage PKG_NAME) + + CMAKE_PARSE_ARGUMENTS(PKG "OPTIONAL;OPTIONAL_INCLUDE" "ROOT;INCLUDE" + "LIBRARY;INCLUDE_SUFFIXES;LIB_SUFFIXES;MODULES;LIB_OUTPUT" ${ARGN}) + + IF(NOT PKG_LIBRARY) + SET(PKG_LIBRARY "${PKG_NAME}") + ENDIF() + IF(NOT PKG_INCLUDE) + SET(PKG_INCLUDE "${PKG_NAME}.h") + ENDIF() + IF(NOT PKG_LIB_OUTPUT) + SET(PKG_LIB_OUTPUT RSPAMD_REQUIRED_LIBRARIES) + ENDIF() + + IF(NOT PKG_ROOT AND PKG_MODULES) + PKG_SEARCH_MODULE(${PKG_NAME} ${PKG_MODULES}) + ENDIF() + + IF(${PKG_NAME}_FOUND) + MESSAGE(STATUS "Found package ${PKG_NAME} in pkg-config modules ${PKG_MODULES}") + SET(WITH_${PKG_NAME} 1 CACHE INTERNAL "") + IF(ENABLE_STATIC MATCHES "ON") + SET(_XPREFIX "${PKG_NAME}_STATIC") + ELSE(ENABLE_STATIC MATCHES "ON") + SET(_XPREFIX "${PKG_NAME}") + ENDIF(ENABLE_STATIC MATCHES "ON") + FOREACH(_arg ${${_XPREFIX}_INCLUDE_DIRS}) + INCLUDE_DIRECTORIES("${_arg}") + SET(${PKG_NAME}_INCLUDE "${_arg}" CACHE INTERNAL "") + ENDFOREACH(_arg ${${_XPREFIX}_INCLUDE_DIRS}) + FOREACH(_arg ${${_XPREFIX}_LIBRARY_DIRS}) + LINK_DIRECTORIES("${_arg}") + SET(${PKG_NAME}_LIBRARY_PATH "${_arg}" CACHE INTERNAL "") + ENDFOREACH(_arg ${${_XPREFIX}_LIBRARY_DIRS}) + # Handle other CFLAGS and LDFLAGS + FOREACH(_arg ${${_XPREFIX}_CFLAGS_OTHER}) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_arg}") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_arg}") + ENDFOREACH(_arg ${${_XPREFIX}_CFLAGS_OTHER}) + FOREACH(_arg ${${_XPREFIX}_LDFLAGS_OTHER}) + SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${_arg}") + ENDFOREACH(_arg ${${_XPREFIX}_LDFLAGS_OTHER}) + LIST(APPEND ${PKG_LIB_OUTPUT} "${${_XPREFIX}_LIBRARIES}") + INCLUDE_DIRECTORIES(${${_XPREFIX}_INCLUDEDIR}) + ELSE() + IF(NOT ${PKG_NAME}_GUESSED) + # Try some more heuristic + FIND_LIBRARY(_lib NAMES ${PKG_LIBRARY} + HINTS ${PKG_ROOT} ${RSPAMD_SEARCH_PATH} + PATH_SUFFIXES ${PKG_LIB_SUFFIXES} lib64 lib + PATHS ${RSPAMD_DEFAULT_LIBRARY_PATHS}) + IF(NOT _lib) + IF(PKG_OPTIONAL) + MESSAGE(STATUS "Cannot find library ${PKG_LIBRARY} for package ${PKG_NAME}, ignoring") + ELSE() + MESSAGE(FATAL_ERROR "Cannot find library ${PKG_LIBRARY} for package ${PKG_NAME}") + ENDIF() + ENDIF(NOT _lib) + + FIND_PATH(_incl ${PKG_INCLUDE} + HINTS ${PKG_ROOT} ${RSPAMD_SEARCH_PATH} + PATH_SUFFIXES ${PKG_INCLUDE_SUFFIXES} include + PATHS {RSPAMD_DEFAULT_INCLUDE_PATHS}) + IF(NOT _incl) + IF(PKG_OPTIONAL OR PKG_OPTIONAL_INCLUDE) + MESSAGE(STATUS "Cannot find header ${PKG_INCLUDE} for package ${PKG_NAME}") + ELSE() + MESSAGE(FATAL_ERROR "Cannot find header ${PKG_INCLUDE} for package ${PKG_NAME}") + ENDIF() + ELSE() + STRING(REGEX REPLACE "/[^/]+$" "" _incl_path "${PKG_INCLUDE}") + STRING(REGEX REPLACE "${_incl_path}/$" "" _stripped_incl "${_incl}") + INCLUDE_DIRECTORIES("${_stripped_incl}") + SET(${PKG_NAME}_INCLUDE "${_stripped_incl}" CACHE INTERNAL "") + ENDIF(NOT _incl) + + IF(_lib) + # We need to apply heuristic to find the real dir name + GET_FILENAME_COMPONENT(_lib_path "${_lib}" PATH) + LINK_DIRECTORIES("${_lib_path}") + LIST(APPEND ${PKG_LIB_OUTPUT} ${_lib}) + SET(${PKG_NAME}_LIBRARY_PATH "${_lib_path}" CACHE INTERNAL "") + SET(${PKG_NAME}_LIBRARY "${_lib}" CACHE INTERNAL "") + ENDIF() + + IF(_incl AND _lib) + MESSAGE(STATUS "Found package ${PKG_NAME} in '${_lib_path}' (${_lib}) and '${_stripped_incl}' (${PKG_INCLUDE}).") + SET(${PKG_NAME}_GUESSED 1 CACHE INTERNAL "") + SET(WITH_${PKG_NAME} 1 CACHE INTERNAL "") + ELSEIF(_lib) + IF(PKG_OPTIONAL_INCLUDE) + SET(${PKG_NAME}_GUESSED 1 INTERNAL "") + SET(WITH_${PKG_NAME} 1 INTERNAL "") + ENDIF() + MESSAGE(STATUS "Found incomplete package ${PKG_NAME} in '${_lib_path}' (${_lib}); no includes.") + ENDIF() + ELSE() + MESSAGE(STATUS "Found package ${PKG_NAME} (cached)") + INCLUDE_DIRECTORIES("${${PKG_NAME}_INCLUDE}") + LINK_DIRECTORIES("${${PKG_NAME}_LIBRARY_PATH}") + LIST(APPEND ${PKG_LIB_OUTPUT} "${${PKG_NAME}_LIBRARY}") + ENDIF() + ENDIF(${PKG_NAME}_FOUND) + + UNSET(_lib CACHE) + UNSET(_incl CACHE) +ENDMACRO(ProcessPackage name)
\ No newline at end of file diff --git a/cmake/Sanitizer.cmake b/cmake/Sanitizer.cmake new file mode 100644 index 000000000..fc96fec8e --- /dev/null +++ b/cmake/Sanitizer.cmake @@ -0,0 +1,52 @@ +# Ported from Clickhouse: https://github.com/ClickHouse/ClickHouse/blob/master/cmake/sanitize.cmake + +option (SANITIZE "Enable sanitizer: address, memory, undefined" "") +set (SAN_FLAGS "${SAN_FLAGS} -g -fno-omit-frame-pointer -DSANITIZER") +# O1 is normally set by clang, and -Og by gcc +if (COMPILER_GCC) + set (SAN_FLAGS "${SAN_FLAGS} -Og") +else () + set (SAN_FLAGS "${SAN_FLAGS} -O1") +endif () +if (SANITIZE) + if (ENABLE_JEMALLOC MATCHES "ON") + message (STATUS "Jemalloc support is useless in case of build with sanitizers") + set (ENABLE_JEMALLOC "OFF") + endif () + if (SANITIZE STREQUAL "address") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=address -fsanitize-address-use-after-scope") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=address -fsanitize-address-use-after-scope") + if (COMPILER_GCC) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -static-libasan") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libasan") + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address -fsanitize-address-use-after-scope") + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libasan") + endif () + + elseif (SANITIZE STREQUAL "memory") + set (MSAN_FLAGS "-fsanitize=memory -fsanitize-memory-track-origins -fno-optimize-sibling-calls") + + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${MSAN_FLAGS}") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${MSAN_FLAGS}") + + if (COMPILER_GCC) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -static-libmsan") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libmsan") + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=memory") + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libmsan") + endif () + + elseif (SANITIZE STREQUAL "undefined") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all") + if (COMPILER_GCC) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -static-libubsan") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libubsan") + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined") + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libubsan") + endif () + + message (FATAL_ERROR "Unknown sanitizer type: ${SANITIZE}") + endif () + message (STATUS "Add sanitizer: ${SANITIZE}") +endif()
\ No newline at end of file diff --git a/cmake/Toolset.cmake b/cmake/Toolset.cmake new file mode 100644 index 000000000..37019bc47 --- /dev/null +++ b/cmake/Toolset.cmake @@ -0,0 +1,186 @@ +option (ENABLE_FAST_MATH "Build rspamd with fast math compiler flag [default: ON]" ON) + +if(CMAKE_C_COMPILER_ID STREQUAL "GNU") + SET (COMPILER_GCC 1) +elseif(CMAKE_C_COMPILER_ID MATCHES "Clang|AppleClang") + SET (COMPILER_CLANG 1) +endif() + +SET (COMPILER_FAST_MATH "") +if (ENABLE_FAST_MATH MATCHES "ON") + # We need to keep nans and infinities, so cannot keep all fast math there + IF (COMPILER_CLANG) + SET (COMPILER_FAST_MATH "-fassociative-math -freciprocal-math -fno-signed-zeros -ffp-contract=fast") + ELSE() + SET (COMPILER_FAST_MATH "-funsafe-math-optimizations -fno-math-errno") + ENDIF () +endif () + +if (CMAKE_GENERATOR STREQUAL "Ninja") + # Turn on colored output. https://github.com/ninja-build/ninja/wiki/FAQ + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-color=always") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-color=always") +endif () + +if (COMPILER_GCC) + # Require minimum version of gcc + set (GCC_MINIMUM_VERSION 4) + if (CMAKE_C_COMPILER_VERSION VERSION_LESS ${GCC_MINIMUM_VERSION} AND NOT CMAKE_VERSION VERSION_LESS 2.8.9) + message (FATAL_ERROR "GCC version must be at least ${GCC_MINIMUM_VERSION}.") + endif () +elseif (COMPILER_CLANG) + # Require minimum version of clang + set (CLANG_MINIMUM_VERSION 4) + if (CMAKE_C_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION}) + message (FATAL_ERROR "Clang version must be at least ${CLANG_MINIMUM_VERSION}.") + endif () + ADD_COMPILE_OPTIONS(-Wno-unused-command-line-argument) +else () + message (WARNING "You are using an unsupported compiler ${CMAKE_C_COMPILER_ID}. Compilation has only been tested with Clang 4+ and GCC 4+.") +endif () + +option(LINKER_NAME "Linker name or full path") + +find_program(LLD_PATH NAMES "ld.lld" "lld") +find_program(GOLD_PATH NAMES "ld.gold" "gold") + +if(NOT LINKER_NAME) + if(LLD_PATH) + set(LINKER_NAME "lld") + elseif(GOLD_PATH) + set(LINKER_NAME "gold") + else() + message(STATUS "Use generic 'ld' as a linker") + endif() +endif() + +if(LINKER_NAME) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}") + set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}") + + message(STATUS "Using custom linker by name: ${LINKER_NAME}") +endif () + +option (ENABLE_STATIC "Enable static compiling [default: OFF]" OFF) + +if (ENABLE_STATIC MATCHES "ON") + MESSAGE(STATUS "Static build of rspamd implies that the target binary will be *GPL* licensed") + SET(GPL_RSPAMD_BINARY 1) + SET(CMAKE_SKIP_INSTALL_RPATH ON) + SET(BUILD_STATIC 1) + SET(CMAKE_FIND_LIBRARY_SUFFIXES ".a") + SET(BUILD_SHARED_LIBRARIES OFF) + SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static") + SET(LINK_TYPE "STATIC") + SET(NO_SHARED "ON") + # Dirty hack for cmake + SET(CMAKE_EXE_LINK_DYNAMIC_C_FLAGS) # remove -Wl,-Bdynamic + SET(CMAKE_EXE_LINK_DYNAMIC_CXX_FLAGS) + SET(CMAKE_SHARED_LIBRARY_C_FLAGS) # remove -fPIC + SET(CMAKE_SHARED_LIBRARY_CXX_FLAGS) + SET(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS) # remove -rdynamic + SET(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS) +else () + if (NO_SHARED MATCHES "OFF") + SET(LINK_TYPE "SHARED") + else () + SET(LINK_TYPE "STATIC") + endif () +endif () + +# Google performance tools +option (ENABLE_GPERF_TOOLS "Enable google perftools [default: OFF]" OFF) +if (ENABLE_GPERF_TOOLS MATCHES "ON") + ProcessPackage(GPERF LIBRARY profiler INCLUDE profiler.h INCLUDE_SUFFIXES include/google + ROOT ${GPERF_ROOT_DIR}) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-omit-frame-pointer") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer") + set (WITH_GPERF_TOOLS 1) +endif (ENABLE_GPERF_TOOLS MATCHES "ON") + +# Legacy options support +option (ENABLE_COVERAGE "Build rspamd with code coverage options [default: OFF]" OFF) +option (ENABLE_OPTIMIZATION "Enable extra optimizations [default: OFF]" OFF) +option (SKIP_RELINK_RPATH "Skip relinking and full RPATH for the install tree" OFF) +option (ENABLE_FULL_DEBUG "Build rspamd with all possible debug [default: OFF]" OFF) + +if(NOT CMAKE_BUILD_TYPE) + if (ENABLE_FULL_DEBUG MATCHES "ON") + set(CMAKE_BUILD_TYPE Debug CACHE STRING "" FORCE) + endif() + if (ENABLE_COVERAGE MATCHES "ON") + set(CMAKE_BUILD_TYPE Coverage CACHE STRING "" FORCE) + endif() + if (ENABLE_OPTIMIZATION MATCHES "ON") + set(CMAKE_BUILD_TYPE Release CACHE STRING "" FORCE) + endif() +endif() + +if (CMAKE_CONFIGURATION_TYPES) # multiconfig generator? + set (CMAKE_CONFIGURATION_TYPES "Debug;RelWithDebInfo;Release;Coverage" CACHE STRING "" FORCE) +else() + if (NOT CMAKE_BUILD_TYPE) + if (NOT SANITIZE) + message(STATUS "Defaulting to release build.") + set(CMAKE_BUILD_TYPE Release CACHE STRING "" FORCE) + else () + message(STATUS "Defaulting to debug build due to sanitizers being enabled.") + set(CMAKE_BUILD_TYPE Debug CACHE STRING "" FORCE) + endif () + endif() + set_property(CACHE CMAKE_BUILD_TYPE PROPERTY HELPSTRING "Choose the type of build") + set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug;Release;Coverage") +endif() + +string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC) +message (STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE_UC}") +set(CMAKE_C_FLAGS_COVERAGE "${CMAKE_C_FLAGS} -O1 --coverage -fno-inline -fno-default-inline -fno-inline-small-functions ${COMPILER_FAST_MATH}") +set(CMAKE_CXX_FLAGS_COVERAGE "${CMAKE_CXX_FLAGS} -O1 --coverage -fno-inline -fno-default-inline -fno-inline-small-functions ${COMPILER_FAST_MATH}") + +if (COMPILER_GCC) + # GCC flags + set (COMPILER_DEBUG_FLAGS "-g -ggdb -g3 -ggdb3") + set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELEASE} -O2 ${COMPILER_FAST_MATH} ${COMPILER_DEBUG_FLAGS}") + set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELEASE} -O2 ${COMPILER_FAST_MATH} ${COMPILER_DEBUG_FLAGS}") + + set (CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O3 ${COMPILER_FAST_MATH} -fomit-frame-pointer") + set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 ${COMPILER_FAST_MATH} -fomit-frame-pointer") + + set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -Og ${COMPILER_FAST_MATH} ${COMPILER_DEBUG_FLAGS}") + set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Og ${COMPILER_FAST_MATH} ${COMPILER_DEBUG_FLAGS}") +else () + # Clang flags + if (LINKER_NAME MATCHES "lld") + set (COMPILER_DEBUG_FLAGS "-g -glldb -gdwarf-aranges -gdwarf-5") + else () + set (COMPILER_DEBUG_FLAGS "-g -glldb -gdwarf-aranges -gdwarf-4") + endif () + set (CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O2 -fomit-frame-pointer ${COMPILER_FAST_MATH}") + set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O2 -fomit-frame-pointer ${COMPILER_FAST_MATH}") + + set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELEASE} -O2 ${COMPILER_FAST_MATH} ${COMPILER_DEBUG_FLAGS}") + set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELEASE} -O2 ${COMPILER_FAST_MATH} ${COMPILER_DEBUG_FLAGS}") + + set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 ${COMPILER_FAST_MATH} ${COMPILER_DEBUG_FLAGS}") + set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 ${COMPILER_FAST_MATH} ${COMPILER_DEBUG_FLAGS}") +endif() + + +if (CMAKE_BUILD_TYPE_UC MATCHES "COVERAGE") + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --coverage") +elseif (CMAKE_BUILD_TYPE_UC MATCHES "RELEASE") + if (${CMAKE_VERSION} VERSION_GREATER "3.9.0") + cmake_policy (SET CMP0069 NEW) + include (CheckIPOSupported) + check_ipo_supported (RESULT SUPPORT_LTO OUTPUT LTO_DIAG ) + if (SUPPORT_LTO) + set (CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE) + message (STATUS "Enable IPO for the ${CMAKE_BUILD_TYPE} build") + else () + message(WARNING "IPO is not supported: ${LTO_DIAG}") + endif () + endif () +endif () + +message (STATUS "Final CFLAGS: ${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE_UC}}") +message (STATUS "Final CXXFLAGS: ${CMAKE_CXX_FLAGS} ${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE_UC}}")
\ No newline at end of file diff --git a/conf/groups.conf b/conf/groups.conf index bf783cc2f..dcea1bcd0 100644 --- a/conf/groups.conf +++ b/conf/groups.conf @@ -116,5 +116,11 @@ group "external_services" { .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/external_services_group.conf" } +group "content" { + .include "$CONFDIR/scores.d/content_group.conf" + .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/content_group.conf" + .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/content_group.conf" +} + .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/groups.conf" .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/groups.conf" diff --git a/conf/modules.d/spamtrap.conf b/conf/modules.d/spamtrap.conf index e5665e15f..d0e70f6b7 100644 --- a/conf/modules.d/spamtrap.conf +++ b/conf/modules.d/spamtrap.conf @@ -24,7 +24,7 @@ spamtrap { # Optionally set an action #action = "no action"; # A map file containing regexp entries for spamtrap emails and domains - #map = file://$LOCAL_CONFDIR/maps.d/spamtrap.map + #map = file://$LOCAL_CONFDIR/local.d/maps.d/spamtrap.map # Name of the symbol #symbol = "SPAMTRAP"; # A score for this module diff --git a/conf/options.inc b/conf/options.inc index f600fdb9c..78fd3bee6 100644 --- a/conf/options.inc +++ b/conf/options.inc @@ -13,7 +13,7 @@ # # Relevant documentation: https://rspamd.com/doc/configuration/options.html -filters = "chartable,dkim,spf,regexp,fuzzy_check"; +filters = "chartable,dkim,regexp,fuzzy_check"; raw_mode = false; one_shot = false; cache_file = "$DBDIR/symbols.cache"; @@ -57,6 +57,9 @@ words_decay = 600; # Write statistics about rspamd usage to the round-robin database rrd = "${DBDIR}/rspamd.rrd"; +# Write statistics for `rspamc` here +stats_file = "${DBDIR}/stats.ucl"; + # Local networks local_addrs = [192.168.0.0/16, 10.0.0.0/8, 172.16.0.0/12, fd00::/8, 169.254.0.0/16, fe80::/10]; hs_cache_dir = "${DBDIR}/"; diff --git a/conf/scores.d/content_group.conf b/conf/scores.d/content_group.conf new file mode 100644 index 000000000..b53ec31d0 --- /dev/null +++ b/conf/scores.d/content_group.conf @@ -0,0 +1,37 @@ +# Content matching rules +# +# Please don't modify this file as your changes might be overwritten with +# the next update. +# +# You can modify '$LOCAL_CONFDIR/rspamd.conf.local.override' to redefine +# parameters defined on the top level +# +# You can modify '$LOCAL_CONFDIR/rspamd.conf.local' to add +# parameters defined on the top level +# +# For specific modules or configuration you can also modify +# '$LOCAL_CONFDIR/local.d/file.conf' - to add your options or rewrite defaults +# '$LOCAL_CONFDIR/override.d/file.conf' - to override the defaults +# +# See https://rspamd.com/doc/tutorials/writing_rules.html for details + +description = "Content rules"; + +symbols = { + "PDF_ENCRYPTED" { + weight = 0.3; + description = "There is an encrypted PDF in the message"; + one_shot = true; + } + "PDF_JAVASCRIPT" { + weight = 0.1; + description = "There is an PDF with JavaScript in the message"; + one_shot = true; + } + "PDF_SUSPICIOUS" { + weight = 4.5; + description = "There is an PDF with suspicious properties in the message"; + one_shot = true; + } +} + diff --git a/config.h.in b/config.h.in index 1ab588506..04a5a8010 100644 --- a/config.h.in +++ b/config.h.in @@ -66,7 +66,6 @@ #cmakedefine HAVE_OCLOEXEC 1 #cmakedefine HAVE_ONOFOLLOW 1 #cmakedefine HAVE_OPENMEMSTREAM 1 -#cmakedefine HAVE_OPENSSL 1 #cmakedefine HAVE_O_DIRECT 1 #cmakedefine HAVE_PATH_MAX 1 @@ -432,4 +431,6 @@ extern uint64_t ottery_rand_uint64(void); #error incompatible compiler found, need gcc > 2.7 or clang #endif +#define HAVE_OPENSSL 1 + #endif diff --git a/contrib/aho-corasick/CMakeLists.txt b/contrib/aho-corasick/CMakeLists.txt index 93c51a146..2c431b5b8 100644 --- a/contrib/aho-corasick/CMakeLists.txt +++ b/contrib/aho-corasick/CMakeLists.txt @@ -1,12 +1,6 @@ SET(AHOCORASICSRC acism_create.c acism.c) -IF(ENABLE_FULL_DEBUG MATCHES "OFF") -if ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3") -endif () -ENDIF() - IF(NOT GPL_RSPAMD_BINARY) ADD_LIBRARY(rspamd-actrie SHARED ${AHOCORASICSRC}) target_link_libraries(rspamd-actrie glib-2.0) diff --git a/contrib/fastutf8/CMakeLists.txt b/contrib/fastutf8/CMakeLists.txt new file mode 100644 index 000000000..f2570bc50 --- /dev/null +++ b/contrib/fastutf8/CMakeLists.txt @@ -0,0 +1,13 @@ +SET(UTFSRC ${CMAKE_CURRENT_SOURCE_DIR}/fastutf8.c) +IF(HAVE_AVX2) + SET(UTFSRC ${UTFSRC} ${CMAKE_CURRENT_SOURCE_DIR}/avx2.c) + MESSAGE(STATUS "UTF8: AVX2 support is added") +ENDIF() +IF(HAVE_SSE41) + SET(UTFSRC ${UTFSRC} ${CMAKE_CURRENT_SOURCE_DIR}/sse41.c) + MESSAGE(STATUS "UTF8: SSE41 support is added") +ENDIF() + +CONFIGURE_FILE(platform_config.h.in platform_config.h) + +ADD_LIBRARY(rspamd-fastutf8 STATIC ${UTFSRC})
\ No newline at end of file diff --git a/contrib/fastutf8/LICENSE b/contrib/fastutf8/LICENSE new file mode 100644 index 000000000..9b5471be2 --- /dev/null +++ b/contrib/fastutf8/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2019 Yibo Cai +Copyright (c) 2019 Vsevolod Stakhov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE.
\ No newline at end of file diff --git a/contrib/fastutf8/avx2.c b/contrib/fastutf8/avx2.c new file mode 100644 index 000000000..765c62fdb --- /dev/null +++ b/contrib/fastutf8/avx2.c @@ -0,0 +1,314 @@ +/* + * MIT License + * + * Copyright (c) 2019 Yibo Cai + * Copyright (c) 2019 Vsevolod Stakhov + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "config.h" +#include "fastutf8.h" +#include "platform_config.h" + + +#ifndef __clang__ +#pragma GCC push_options +#pragma GCC target("avx2") +#endif + +#ifndef __SSE2__ +#define __SSE2__ +#endif +#ifndef __SSE__ +#define __SSE__ +#endif +#ifndef __SSE4_2__ +#define __SSE4_2__ +#endif +#ifndef __SSE4_1__ +#define __SSE4_1__ +#endif +#ifndef __SSEE3__ +#define __SSEE3__ +#endif +#ifndef __AVX__ +#define __AVX__ +#endif +#ifndef __AVX2__ +#define __AVX2__ +#endif + +#include <immintrin.h> + +/* + * Map high nibble of "First Byte" to legal character length minus 1 + * 0x00 ~ 0xBF --> 0 + * 0xC0 ~ 0xDF --> 1 + * 0xE0 ~ 0xEF --> 2 + * 0xF0 ~ 0xFF --> 3 + */ +static const int8_t _first_len_tbl[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 3, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 3, +}; + +/* Map "First Byte" to 8-th item of range table (0xC2 ~ 0xF4) */ +static const int8_t _first_range_tbl[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, +}; + +/* + * Range table, map range index to min and max values + * Index 0 : 00 ~ 7F (First Byte, ascii) + * Index 1,2,3: 80 ~ BF (Second, Third, Fourth Byte) + * Index 4 : A0 ~ BF (Second Byte after E0) + * Index 5 : 80 ~ 9F (Second Byte after ED) + * Index 6 : 90 ~ BF (Second Byte after F0) + * Index 7 : 80 ~ 8F (Second Byte after F4) + * Index 8 : C2 ~ F4 (First Byte, non ascii) + * Index 9~15 : illegal: i >= 127 && i <= -128 + */ +static const int8_t _range_min_tbl[] = { + 0x00, 0x80, 0x80, 0x80, 0xA0, 0x80, 0x90, 0x80, + 0xC2, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, + 0x00, 0x80, 0x80, 0x80, 0xA0, 0x80, 0x90, 0x80, + 0xC2, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, +}; +static const int8_t _range_max_tbl[] = { + 0x7F, 0xBF, 0xBF, 0xBF, 0xBF, 0x9F, 0xBF, 0x8F, + 0xF4, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x7F, 0xBF, 0xBF, 0xBF, 0xBF, 0x9F, 0xBF, 0x8F, + 0xF4, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, +}; + +/* + * Tables for fast handling of four special First Bytes(E0,ED,F0,F4), after + * which the Second Byte are not 80~BF. It contains "range index adjustment". + * +------------+---------------+------------------+----------------+ + * | First Byte | original range| range adjustment | adjusted range | + * +------------+---------------+------------------+----------------+ + * | E0 | 2 | 2 | 4 | + * +------------+---------------+------------------+----------------+ + * | ED | 2 | 3 | 5 | + * +------------+---------------+------------------+----------------+ + * | F0 | 3 | 3 | 6 | + * +------------+---------------+------------------+----------------+ + * | F4 | 4 | 4 | 8 | + * +------------+---------------+------------------+----------------+ + */ +/* index1 -> E0, index14 -> ED */ +static const int8_t _df_ee_tbl[] = { + 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, + 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, +}; +/* index1 -> F0, index5 -> F4 */ +static const int8_t _ef_fe_tbl[] = { + 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; + +static inline __m256i push_last_byte_of_a_to_b(__m256i a, __m256i b) + __attribute__((__target__("avx2"))); +static inline __m256i push_last_byte_of_a_to_b(__m256i a, __m256i b) +{ + return _mm256_alignr_epi8(b, _mm256_permute2x128_si256(a, b, 0x21), 15); +} + +static inline __m256i push_last_2bytes_of_a_to_b(__m256i a, __m256i b) + __attribute__((__target__("avx2"))); +static inline __m256i push_last_2bytes_of_a_to_b(__m256i a, __m256i b) +{ + return _mm256_alignr_epi8(b, _mm256_permute2x128_si256(a, b, 0x21), 14); +} + +static inline __m256i push_last_3bytes_of_a_to_b(__m256i a, __m256i b) + __attribute__((__target__("avx2"))); +static inline __m256i push_last_3bytes_of_a_to_b(__m256i a, __m256i b) +{ + return _mm256_alignr_epi8(b, _mm256_permute2x128_si256(a, b, 0x21), 13); +} + +off_t rspamd_fast_utf8_validate_avx2 (const unsigned char *data, size_t len) + __attribute__((__target__("avx2"))); + +/* 5x faster than naive method */ +/* Return 0 - success, -1 - error, >0 - first error char(if RET_ERR_IDX = 1) */ +off_t rspamd_fast_utf8_validate_avx2 (const unsigned char *data, size_t len) +{ + off_t err_pos = 1; + + if (len >= 32) { + __m256i prev_input = _mm256_set1_epi8 (0); + __m256i prev_first_len = _mm256_set1_epi8 (0); + + /* Cached tables */ + const __m256i first_len_tbl = + _mm256_lddqu_si256 ((const __m256i *) _first_len_tbl); + const __m256i first_range_tbl = + _mm256_lddqu_si256 ((const __m256i *) _first_range_tbl); + const __m256i range_min_tbl = + _mm256_lddqu_si256 ((const __m256i *) _range_min_tbl); + const __m256i range_max_tbl = + _mm256_lddqu_si256 ((const __m256i *) _range_max_tbl); + const __m256i df_ee_tbl = + _mm256_lddqu_si256 ((const __m256i *) _df_ee_tbl); + const __m256i ef_fe_tbl = + _mm256_lddqu_si256 ((const __m256i *) _ef_fe_tbl); + + __m256i error = _mm256_set1_epi8 (0); + + while (len >= 32) { + const __m256i input = _mm256_lddqu_si256 ((const __m256i *) data); + + /* high_nibbles = input >> 4 */ + const __m256i high_nibbles = + _mm256_and_si256 (_mm256_srli_epi16 (input, 4), _mm256_set1_epi8 (0x0F)); + + /* first_len = legal character length minus 1 */ + /* 0 for 00~7F, 1 for C0~DF, 2 for E0~EF, 3 for F0~FF */ + /* first_len = first_len_tbl[high_nibbles] */ + __m256i first_len = _mm256_shuffle_epi8 (first_len_tbl, high_nibbles); + + /* First Byte: set range index to 8 for bytes within 0xC0 ~ 0xFF */ + /* range = first_range_tbl[high_nibbles] */ + __m256i range = _mm256_shuffle_epi8 (first_range_tbl, high_nibbles); + + /* Second Byte: set range index to first_len */ + /* 0 for 00~7F, 1 for C0~DF, 2 for E0~EF, 3 for F0~FF */ + /* range |= (first_len, prev_first_len) << 1 byte */ + range = _mm256_or_si256 ( + range, push_last_byte_of_a_to_b (prev_first_len, first_len)); + + /* Third Byte: set range index to saturate_sub(first_len, 1) */ + /* 0 for 00~7F, 0 for C0~DF, 1 for E0~EF, 2 for F0~FF */ + __m256i tmp1, tmp2; + + /* tmp1 = saturate_sub(first_len, 1) */ + tmp1 = _mm256_subs_epu8 (first_len, _mm256_set1_epi8 (1)); + /* tmp2 = saturate_sub(prev_first_len, 1) */ + tmp2 = _mm256_subs_epu8 (prev_first_len, _mm256_set1_epi8 (1)); + + /* range |= (tmp1, tmp2) << 2 bytes */ + range = _mm256_or_si256 (range, push_last_2bytes_of_a_to_b (tmp2, tmp1)); + + /* Fourth Byte: set range index to saturate_sub(first_len, 2) */ + /* 0 for 00~7F, 0 for C0~DF, 0 for E0~EF, 1 for F0~FF */ + /* tmp1 = saturate_sub(first_len, 2) */ + tmp1 = _mm256_subs_epu8 (first_len, _mm256_set1_epi8 (2)); + /* tmp2 = saturate_sub(prev_first_len, 2) */ + tmp2 = _mm256_subs_epu8 (prev_first_len, _mm256_set1_epi8 (2)); + /* range |= (tmp1, tmp2) << 3 bytes */ + range = _mm256_or_si256 (range, push_last_3bytes_of_a_to_b (tmp2, tmp1)); + + /* + * Now we have below range indices caluclated + * Correct cases: + * - 8 for C0~FF + * - 3 for 1st byte after F0~FF + * - 2 for 1st byte after E0~EF or 2nd byte after F0~FF + * - 1 for 1st byte after C0~DF or 2nd byte after E0~EF or + * 3rd byte after F0~FF + * - 0 for others + * Error cases: + * 9,10,11 if non ascii First Byte overlaps + * E.g., F1 80 C2 90 --> 8 3 10 2, where 10 indicates error + */ + + /* Adjust Second Byte range for special First Bytes(E0,ED,F0,F4) */ + /* Overlaps lead to index 9~15, which are illegal in range table */ + __m256i shift1, pos, range2; + /* shift1 = (input, prev_input) << 1 byte */ + shift1 = push_last_byte_of_a_to_b (prev_input, input); + pos = _mm256_sub_epi8 (shift1, _mm256_set1_epi8 (0xEF)); + /* + * shift1: | EF F0 ... FE | FF 00 ... ... DE | DF E0 ... EE | + * pos: | 0 1 15 | 16 17 239| 240 241 255| + * pos-240: | 0 0 0 | 0 0 0 | 0 1 15 | + * pos+112: | 112 113 127| >= 128 | >= 128 | + */ + tmp1 = _mm256_subs_epu8 (pos, _mm256_set1_epi8 ((char)240)); + range2 = _mm256_shuffle_epi8 (df_ee_tbl, tmp1); + tmp2 = _mm256_adds_epu8 (pos, _mm256_set1_epi8 (112)); + range2 = _mm256_add_epi8 (range2, _mm256_shuffle_epi8 (ef_fe_tbl, tmp2)); + + range = _mm256_add_epi8 (range, range2); + + /* Load min and max values per calculated range index */ + __m256i minv = _mm256_shuffle_epi8 (range_min_tbl, range); + __m256i maxv = _mm256_shuffle_epi8 (range_max_tbl, range); + + /* Check value range */ + error = _mm256_cmpgt_epi8(minv, input); + error = _mm256_or_si256(error, _mm256_cmpgt_epi8(input, maxv)); + /* 5% performance drop from this conditional branch */ + if (!_mm256_testz_si256(error, error)) { + break; + } + + prev_input = input; + prev_first_len = first_len; + + data += 32; + len -= 32; + err_pos += 32; + } + + /* Error in first 16 bytes */ + if (err_pos == 1) { + goto do_naive; + } + + /* Find previous token (not 80~BF) */ + int32_t token4 = _mm256_extract_epi32 (prev_input, 7); + const int8_t *token = (const int8_t *) &token4; + int lookahead = 0; + + if (token[3] > (int8_t) 0xBF) { + lookahead = 1; + } + else if (token[2] > (int8_t) 0xBF) { + lookahead = 2; + } + else if (token[1] > (int8_t) 0xBF) { + lookahead = 3; + } + + data -= lookahead; + len += lookahead; + err_pos -= lookahead; + } + + /* Check remaining bytes with naive method */ +do_naive: + if (len > 0) { + off_t err_pos2 = rspamd_fast_utf8_validate_ref (data, len); + + if (err_pos2) { + return err_pos + err_pos2 - 1; + } + } + + return 0; +} + +#ifndef __clang__ +#pragma GCC pop_options +#endif + diff --git a/contrib/fastutf8/fastutf8.c b/contrib/fastutf8/fastutf8.c new file mode 100644 index 000000000..2a5a9983c --- /dev/null +++ b/contrib/fastutf8/fastutf8.c @@ -0,0 +1,160 @@ +/* + * MIT License + * + * Copyright (c) 2019 Yibo Cai + * Copyright (c) 2019 Vsevolod Stakhov + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "fastutf8.h" +#include "platform_config.h" + + +/* + * http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94 + * + * Table 3-7. Well-Formed UTF-8 Byte Sequences + * + * +--------------------+------------+-------------+------------+-------------+ + * | Code Points | First Byte | Second Byte | Third Byte | Fourth Byte | + * +--------------------+------------+-------------+------------+-------------+ + * | U+0000..U+007F | 00..7F | | | | + * +--------------------+------------+-------------+------------+-------------+ + * | U+0080..U+07FF | C2..DF | 80..BF | | | + * +--------------------+------------+-------------+------------+-------------+ + * | U+0800..U+0FFF | E0 | A0..BF | 80..BF | | + * +--------------------+------------+-------------+------------+-------------+ + * | U+1000..U+CFFF | E1..EC | 80..BF | 80..BF | | + * +--------------------+------------+-------------+------------+-------------+ + * | U+D000..U+D7FF | ED | 80..9F | 80..BF | | + * +--------------------+------------+-------------+------------+-------------+ + * | U+E000..U+FFFF | EE..EF | 80..BF | 80..BF | | + * +--------------------+------------+-------------+------------+-------------+ + * | U+10000..U+3FFFF | F0 | 90..BF | 80..BF | 80..BF | + * +--------------------+------------+-------------+------------+-------------+ + * | U+40000..U+FFFFF | F1..F3 | 80..BF | 80..BF | 80..BF | + * +--------------------+------------+-------------+------------+-------------+ + * | U+100000..U+10FFFF | F4 | 80..8F | 80..BF | 80..BF | + * +--------------------+------------+-------------+------------+-------------+ + */ + +/* Return 0 - success, >0 - index (1 based) of first error char */ +off_t +rspamd_fast_utf8_validate_ref (const unsigned char *data, size_t len) +{ + off_t err_pos = 1; + + while (len) { + int bytes; + const unsigned char byte1 = data[0]; + + /* 00..7F */ + if (byte1 <= 0x7F) { + bytes = 1; + /* C2..DF, 80..BF */ + } + else if (len >= 2 && byte1 >= 0xC2 && byte1 <= 0xDF && + (signed char) data[1] <= (signed char) 0xBF) { + bytes = 2; + } + else if (len >= 3) { + const unsigned char byte2 = data[1]; + + /* Is byte2, byte3 between 0x80 ~ 0xBF */ + const int byte2_ok = (signed char) byte2 <= (signed char) 0xBF; + const int byte3_ok = (signed char) data[2] <= (signed char) 0xBF; + + if (byte2_ok && byte3_ok && + /* E0, A0..BF, 80..BF */ + ((byte1 == 0xE0 && byte2 >= 0xA0) || + /* E1..EC, 80..BF, 80..BF */ + (byte1 >= 0xE1 && byte1 <= 0xEC) || + /* ED, 80..9F, 80..BF */ + (byte1 == 0xED && byte2 <= 0x9F) || + /* EE..EF, 80..BF, 80..BF */ + (byte1 >= 0xEE && byte1 <= 0xEF))) { + bytes = 3; + } + else if (len >= 4) { + /* Is byte4 between 0x80 ~ 0xBF */ + const int byte4_ok = (signed char) data[3] <= (signed char) 0xBF; + + if (byte2_ok && byte3_ok && byte4_ok && + /* F0, 90..BF, 80..BF, 80..BF */ + ((byte1 == 0xF0 && byte2 >= 0x90) || + /* F1..F3, 80..BF, 80..BF, 80..BF */ + (byte1 >= 0xF1 && byte1 <= 0xF3) || + /* F4, 80..8F, 80..BF, 80..BF */ + (byte1 == 0xF4 && byte2 <= 0x8F))) { + bytes = 4; + } + else { + return err_pos; + } + } + else { + return err_pos; + } + } + else { + return err_pos; + } + + len -= bytes; + err_pos += bytes; + data += bytes; + } + + return 0; +} + +/* Prototypes */ +#ifdef HAVE_SSSE3 +extern off_t rspamd_fast_utf8_validate_sse41 (const unsigned char *data, size_t len); +#endif +#ifdef HAVE_AVX2 +extern off_t rspamd_fast_utf8_validate_avx2 (const unsigned char *data, size_t len); +#endif + +static off_t (*validate_func) (const unsigned char *data, size_t len) = + rspamd_fast_utf8_validate_ref; + + +void +rspamd_fast_utf8_library_init (unsigned flags) +{ +#ifdef HAVE_SSSE3 + if (flags & RSPAMD_FAST_UTF8_FLAG_SSE41) { + validate_func = rspamd_fast_utf8_validate_sse41; + } +#endif +#ifdef HAVE_AVX2 + if (flags & RSPAMD_FAST_UTF8_FLAG_AVX2) { + validate_func = rspamd_fast_utf8_validate_avx2; + } +#endif +} + +off_t +rspamd_fast_utf8_validate (const unsigned char *data, size_t len) +{ + return len >= 64 ? + validate_func (data, len) : + rspamd_fast_utf8_validate_ref (data, len); +}
\ No newline at end of file diff --git a/contrib/fastutf8/fastutf8.h b/contrib/fastutf8/fastutf8.h new file mode 100644 index 000000000..001499ab3 --- /dev/null +++ b/contrib/fastutf8/fastutf8.h @@ -0,0 +1,59 @@ +/* + * MIT License + * + * Copyright (c) 2019 Yibo Cai + * Copyright (c) 2019 Vsevolod Stakhov + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef RSPAMD_FASTUTF8_H +#define RSPAMD_FASTUTF8_H + +#include <sys/types.h> +#include <stdbool.h> +#include <stdint.h> + +enum rspamd_fast_utf8_cpu_flags { + RSPAMD_FAST_UTF8_FLAG_SSE41 = 1u << 0u, + RSPAMD_FAST_UTF8_FLAG_AVX2 = 1u << 1u, +}; + +/** + * Called to init codecs + * @param flags + */ +void rspamd_fast_utf8_library_init (unsigned flags); + +/** + * Called to validate input using fast codec + * @param data + * @param len + * @return + */ +off_t rspamd_fast_utf8_validate (const unsigned char *data, size_t len); + +/** + * Use plain C implementation + * @param data + * @param len + * @return + */ +off_t rspamd_fast_utf8_validate_ref (const unsigned char *data, size_t len); + +#endif diff --git a/contrib/fastutf8/platform_config.h.in b/contrib/fastutf8/platform_config.h.in new file mode 100644 index 000000000..301234e1e --- /dev/null +++ b/contrib/fastutf8/platform_config.h.in @@ -0,0 +1,12 @@ +#ifndef PLATFORM_H_CONFIG +#define PLATFORM_H_CONFIG + +#define ARCH "${ARCH}" +#define CMAKE_ARCH_${ARCH} 1 + +#ifdef __x86_64__ +#cmakedefine HAVE_AVX2 1 +#cmakedefine HAVE_SSE41 1 +#endif + +#endif
\ No newline at end of file diff --git a/contrib/fastutf8/sse41.c b/contrib/fastutf8/sse41.c new file mode 100644 index 000000000..df338cf27 --- /dev/null +++ b/contrib/fastutf8/sse41.c @@ -0,0 +1,272 @@ +/* + * MIT License + * + * Copyright (c) 2019 Yibo Cai + * Copyright (c) 2019 Vsevolod Stakhov + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "config.h" +#include "fastutf8.h" +#include "platform_config.h" + +#ifndef __clang__ +#pragma GCC push_options +#pragma GCC target("sse4.1") +#endif + +#ifndef __SSE2__ +#define __SSE2__ +#endif +#ifndef __SSE__ +#define __SSE__ +#endif +#ifndef __SSEE3__ +#define __SSEE3__ +#endif +#ifndef __SSE4_1__ +#define __SSE4_1__ +#endif + +#include <smmintrin.h> + +/* + * Map high nibble of "First Byte" to legal character length minus 1 + * 0x00 ~ 0xBF --> 0 + * 0xC0 ~ 0xDF --> 1 + * 0xE0 ~ 0xEF --> 2 + * 0xF0 ~ 0xFF --> 3 + */ +static const int8_t _first_len_tbl[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 3, +}; + +/* Map "First Byte" to 8-th item of range table (0xC2 ~ 0xF4) */ +static const int8_t _first_range_tbl[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, +}; + +/* + * Range table, map range index to min and max values + * Index 0 : 00 ~ 7F (First Byte, ascii) + * Index 1,2,3: 80 ~ BF (Second, Third, Fourth Byte) + * Index 4 : A0 ~ BF (Second Byte after E0) + * Index 5 : 80 ~ 9F (Second Byte after ED) + * Index 6 : 90 ~ BF (Second Byte after F0) + * Index 7 : 80 ~ 8F (Second Byte after F4) + * Index 8 : C2 ~ F4 (First Byte, non ascii) + * Index 9~15 : illegal: i >= 127 && i <= -128 + */ +static const int8_t _range_min_tbl[] = { + 0x00, 0x80, 0x80, 0x80, 0xA0, 0x80, 0x90, 0x80, + 0xC2, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, +}; +static const int8_t _range_max_tbl[] = { + 0x7F, 0xBF, 0xBF, 0xBF, 0xBF, 0x9F, 0xBF, 0x8F, + 0xF4, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, +}; + +/* + * Tables for fast handling of four special First Bytes(E0,ED,F0,F4), after + * which the Second Byte are not 80~BF. It contains "range index adjustment". + * +------------+---------------+------------------+----------------+ + * | First Byte | original range| range adjustment | adjusted range | + * +------------+---------------+------------------+----------------+ + * | E0 | 2 | 2 | 4 | + * +------------+---------------+------------------+----------------+ + * | ED | 2 | 3 | 5 | + * +------------+---------------+------------------+----------------+ + * | F0 | 3 | 3 | 6 | + * +------------+---------------+------------------+----------------+ + * | F4 | 4 | 4 | 8 | + * +------------+---------------+------------------+----------------+ + */ +/* index1 -> E0, index14 -> ED */ +static const int8_t _df_ee_tbl[] = { + 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, +}; +/* index1 -> F0, index5 -> F4 */ +static const int8_t _ef_fe_tbl[] = { + 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; + +off_t +rspamd_fast_utf8_validate_sse41 (const unsigned char *data, size_t len) + __attribute__((__target__("sse4.1"))); + +/* Return 0 - success, >0 - first error char(if RET_ERR_IDX = 1) */ +off_t +rspamd_fast_utf8_validate_sse41 (const unsigned char *data, size_t len) +{ + off_t err_pos = 1; + + if (len >= 16) { + __m128i prev_input = _mm_set1_epi8 (0); + __m128i prev_first_len = _mm_set1_epi8 (0); + + /* Cached tables */ + const __m128i first_len_tbl = + _mm_lddqu_si128 ((const __m128i *) _first_len_tbl); + const __m128i first_range_tbl = + _mm_lddqu_si128 ((const __m128i *) _first_range_tbl); + const __m128i range_min_tbl = + _mm_lddqu_si128 ((const __m128i *) _range_min_tbl); + const __m128i range_max_tbl = + _mm_lddqu_si128 ((const __m128i *) _range_max_tbl); + const __m128i df_ee_tbl = + _mm_lddqu_si128 ((const __m128i *) _df_ee_tbl); + const __m128i ef_fe_tbl = + _mm_lddqu_si128 ((const __m128i *) _ef_fe_tbl); + + __m128i error = _mm_set1_epi8 (0); + + while (len >= 16) { + const __m128i input = _mm_lddqu_si128 ((const __m128i *) data); + + /* high_nibbles = input >> 4 */ + const __m128i high_nibbles = + _mm_and_si128 (_mm_srli_epi16 (input, 4), _mm_set1_epi8 (0x0F)); + + /* first_len = legal character length minus 1 */ + /* 0 for 00~7F, 1 for C0~DF, 2 for E0~EF, 3 for F0~FF */ + /* first_len = first_len_tbl[high_nibbles] */ + __m128i first_len = _mm_shuffle_epi8 (first_len_tbl, high_nibbles); + + /* First Byte: set range index to 8 for bytes within 0xC0 ~ 0xFF */ + /* range = first_range_tbl[high_nibbles] */ + __m128i range = _mm_shuffle_epi8 (first_range_tbl, high_nibbles); + + /* Second Byte: set range index to first_len */ + /* 0 for 00~7F, 1 for C0~DF, 2 for E0~EF, 3 for F0~FF */ + /* range |= (first_len, prev_first_len) << 1 byte */ + range = _mm_or_si128 ( + range, _mm_alignr_epi8(first_len, prev_first_len, 15)); + + /* Third Byte: set range index to saturate_sub(first_len, 1) */ + /* 0 for 00~7F, 0 for C0~DF, 1 for E0~EF, 2 for F0~FF */ + __m128i tmp1, tmp2; + /* tmp1 = saturate_sub(first_len, 1) */ + tmp1 = _mm_subs_epu8 (first_len, _mm_set1_epi8 (1)); + /* tmp2 = saturate_sub(prev_first_len, 1) */ + tmp2 = _mm_subs_epu8 (prev_first_len, _mm_set1_epi8 (1)); + /* range |= (tmp1, tmp2) << 2 bytes */ + range = _mm_or_si128 (range, _mm_alignr_epi8(tmp1, tmp2, 14)); + + /* Fourth Byte: set range index to saturate_sub(first_len, 2) */ + /* 0 for 00~7F, 0 for C0~DF, 0 for E0~EF, 1 for F0~FF */ + /* tmp1 = saturate_sub(first_len, 2) */ + tmp1 = _mm_subs_epu8 (first_len, _mm_set1_epi8 (2)); + /* tmp2 = saturate_sub(prev_first_len, 2) */ + tmp2 = _mm_subs_epu8 (prev_first_len, _mm_set1_epi8 (2)); + /* range |= (tmp1, tmp2) << 3 bytes */ + range = _mm_or_si128 (range, _mm_alignr_epi8(tmp1, tmp2, 13)); + + /* + * Now we have below range indices caluclated + * Correct cases: + * - 8 for C0~FF + * - 3 for 1st byte after F0~FF + * - 2 for 1st byte after E0~EF or 2nd byte after F0~FF + * - 1 for 1st byte after C0~DF or 2nd byte after E0~EF or + * 3rd byte after F0~FF + * - 0 for others + * Error cases: + * 9,10,11 if non ascii First Byte overlaps + * E.g., F1 80 C2 90 --> 8 3 10 2, where 10 indicates error + */ + + /* Adjust Second Byte range for special First Bytes(E0,ED,F0,F4) */ + /* Overlaps lead to index 9~15, which are illegal in range table */ + __m128i shift1, pos, range2; + /* shift1 = (input, prev_input) << 1 byte */ + shift1 = _mm_alignr_epi8(input, prev_input, 15); + pos = _mm_sub_epi8 (shift1, _mm_set1_epi8 (0xEF)); + /* + * shift1: | EF F0 ... FE | FF 00 ... ... DE | DF E0 ... EE | + * pos: | 0 1 15 | 16 17 239| 240 241 255| + * pos-240: | 0 0 0 | 0 0 0 | 0 1 15 | + * pos+112: | 112 113 127| >= 128 | >= 128 | + */ + tmp1 = _mm_subs_epu8 (pos, _mm_set1_epi8 ((char)240)); + range2 = _mm_shuffle_epi8 (df_ee_tbl, tmp1); + tmp2 = _mm_adds_epu8 (pos, _mm_set1_epi8 (112)); + range2 = _mm_add_epi8 (range2, _mm_shuffle_epi8 (ef_fe_tbl, tmp2)); + + range = _mm_add_epi8 (range, range2); + + /* Load min and max values per calculated range index */ + __m128i minv = _mm_shuffle_epi8 (range_min_tbl, range); + __m128i maxv = _mm_shuffle_epi8 (range_max_tbl, range); + + /* Check value range */ + error = _mm_cmplt_epi8(input, minv); + error = _mm_or_si128(error, _mm_cmpgt_epi8(input, maxv)); + /* 5% performance drop from this conditional branch */ + if (!_mm_testz_si128(error, error)) { + break; + } + + prev_input = input; + prev_first_len = first_len; + + data += 16; + len -= 16; + err_pos += 16; + } + + /* Error in first 16 bytes */ + if (err_pos == 1) { + goto do_naive; + } + + /* Find previous token (not 80~BF) */ + int32_t token4 = _mm_extract_epi32 (prev_input, 3); + const int8_t *token = (const int8_t *) &token4; + int lookahead = 0; + + if (token[3] > (int8_t) 0xBF) { + lookahead = 1; + } + else if (token[2] > (int8_t) 0xBF) { + lookahead = 2; + } + else if (token[1] > (int8_t) 0xBF) { + lookahead = 3; + } + + data -= lookahead; + len += lookahead; + err_pos -= lookahead; + } + + do_naive: + if (len > 0) { + off_t err_pos2 = rspamd_fast_utf8_validate_ref (data, len); + + if (err_pos2) { + return err_pos + err_pos2 - 1; + } + } + + return 0; +} + +#ifndef __clang__ +#pragma GCC pop_options +#endif
\ No newline at end of file diff --git a/contrib/fpconv/CMakeLists.txt b/contrib/fpconv/CMakeLists.txt index 0d27dbb7f..b3305250b 100644 --- a/contrib/fpconv/CMakeLists.txt +++ b/contrib/fpconv/CMakeLists.txt @@ -1,11 +1,6 @@ SET(FPCONVSRC fpconv.c) SET(FTPCONV_COMPILE_FLAGS "-DRSPAMD_LIB") -IF(ENABLE_FULL_DEBUG MATCHES "OFF") - if ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") - set(FTPCONV_COMPILE_FLAGS "${FTPCONV_COMPILE_FLAGS} -O3") - endif () -ENDIF() ADD_LIBRARY(rspamd-fpconv STATIC ${FPCONVSRC}) SET_TARGET_PROPERTIES(rspamd-fpconv PROPERTIES VERSION ${RSPAMD_VERSION}) diff --git a/contrib/hiredis/CMakeLists.txt b/contrib/hiredis/CMakeLists.txt index f8b233996..1e056319f 100644 --- a/contrib/hiredis/CMakeLists.txt +++ b/contrib/hiredis/CMakeLists.txt @@ -6,13 +6,6 @@ SET(HIREDISSRC async.c sds.c) SET(HIREDIS_CFLAGS "") - -IF(ENABLE_FULL_DEBUG MATCHES "OFF") -IF("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") - SET(HIREDIS_CFLAGS "${HIREDIS_CFLAGS} -O3") -ENDIF() -ENDIF() - ADD_LIBRARY(rspamd-hiredis STATIC ${HIREDISSRC}) SET_TARGET_PROPERTIES(rspamd-hiredis PROPERTIES COMPILE_FLAGS "${HIREDIS_CFLAGS}")
\ No newline at end of file diff --git a/contrib/http-parser/CMakeLists.txt b/contrib/http-parser/CMakeLists.txt index 499c85e93..a5da7010c 100644 --- a/contrib/http-parser/CMakeLists.txt +++ b/contrib/http-parser/CMakeLists.txt @@ -2,11 +2,6 @@ SET(HTTPSRC http_parser.c) SET(HTTP_COMPILE_FLAGS "-DRSPAMD_LIB") -IF(ENABLE_FULL_DEBUG MATCHES "OFF") -if ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") - set(HTTP_COMPILE_FLAGS "${HTTP_COMPILE_FLAGS} -O3") -endif () -ENDIF() ADD_LIBRARY(rspamd-http-parser STATIC ${HTTPSRC}) SET_TARGET_PROPERTIES(rspamd-http-parser PROPERTIES VERSION ${RSPAMD_VERSION}) diff --git a/contrib/kann/CMakeLists.txt b/contrib/kann/CMakeLists.txt index 2bf32c92e..b3a1d547c 100644 --- a/contrib/kann/CMakeLists.txt +++ b/contrib/kann/CMakeLists.txt @@ -1,11 +1,5 @@ SET(LIBKANNSRC kautodiff.c kann.c) -IF(ENABLE_FULL_DEBUG MATCHES "OFF") - if ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3") - endif () -ENDIF() - ADD_LIBRARY(rspamd-kann SHARED ${LIBKANNSRC}) IF(WITH_BLAS) diff --git a/contrib/lc-btrie/CMakeLists.txt b/contrib/lc-btrie/CMakeLists.txt index b520bda86..a5fe9e53d 100644 --- a/contrib/lc-btrie/CMakeLists.txt +++ b/contrib/lc-btrie/CMakeLists.txt @@ -2,10 +2,5 @@ SET(LCTRIESRC btrie.c) ADD_LIBRARY(lcbtrie STATIC ${LCTRIESRC}) SET(LCTRIE_CFLAGS "-DBUILD_RSPAMD") -IF(ENABLE_FULL_DEBUG MATCHES "OFF") -if ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") - SET(LCTRIE_CFLAGS "${LCTRIE_CFLAGS} -O3") -endif () -ENDIF() set_target_properties(lcbtrie PROPERTIES COMPILE_FLAGS "${LCTRIE_CFLAGS}")
\ No newline at end of file diff --git a/contrib/libev/CMakeLists.txt b/contrib/libev/CMakeLists.txt index e98ff126e..db380db95 100644 --- a/contrib/libev/CMakeLists.txt +++ b/contrib/libev/CMakeLists.txt @@ -47,12 +47,6 @@ CHECK_LIBRARY_EXISTS(rt clock_gettime "" HAVE_LIBRT) CHECK_LIBRARY_EXISTS(rt clock_gettime "" HAVE_CLOCK_GETTIME) CHECK_LIBRARY_EXISTS(m ceil "" HAVE_LIBM) -IF(ENABLE_FULL_DEBUG MATCHES "OFF") -if ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3") -endif () -ENDIF() - CONFIGURE_FILE(config.h.in libev-config.h) ADD_LIBRARY(rspamd-ev SHARED ${LIBEVSRC}) diff --git a/contrib/libottery/CMakeLists.txt b/contrib/libottery/CMakeLists.txt index 4a6ee31db..b8536f2f1 100644 --- a/contrib/libottery/CMakeLists.txt +++ b/contrib/libottery/CMakeLists.txt @@ -7,10 +7,5 @@ SET(OTTERYSRC chacha_merged.c aes_cryptobox.c) ADD_LIBRARY(ottery STATIC ${OTTERYSRC}) -SET(OTTERY_CFLAGS "-DBUILD_RSPAMD -DOTTERY_NO_PID_CHECK -DOTTERY_NO_INIT_CHECK") -IF(ENABLE_FULL_DEBUG MATCHES "OFF") -if ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") - SET(OTTERY_CFLAGS "${OTTERY_CFLAGS} -O3") -endif () -ENDIF() +SET(OTTERY_CFLAGS "-DBUILD_RSPAMD -DOTTERY_NO_PID_CHECK -DOTTERY_NO_INIT_CHECK -DOTTERY_NO_WIPE_STACK") set_target_properties(ottery PROPERTIES COMPILE_FLAGS "${OTTERY_CFLAGS}")
\ No newline at end of file diff --git a/contrib/librdns/rdns.h b/contrib/librdns/rdns.h index b563c7ea3..5ea8f5952 100644 --- a/contrib/librdns/rdns.h +++ b/contrib/librdns/rdns.h @@ -164,11 +164,12 @@ struct rdns_upstream_context { void *data; struct rdns_upstream_elt* (*select)(const char *name, size_t len, void *ups_data); - struct rdns_upstream_elt* (*select_retransmit)(const char *name, - size_t len, void *ups_data); + struct rdns_upstream_elt* (*select_retransmit)(const char *name, size_t len, + struct rdns_upstream_elt* prev_elt, + void *ups_data); unsigned int (*count)(void *ups_data); void (*ok)(struct rdns_upstream_elt *elt, void *ups_data); - void (*fail)(struct rdns_upstream_elt *elt, void *ups_data); + void (*fail)(struct rdns_upstream_elt *elt, void *ups_data, const char *reason); }; /** diff --git a/contrib/librdns/resolver.c b/contrib/librdns/resolver.c index 2cc3695a7..aec0e9381 100644 --- a/contrib/librdns/resolver.c +++ b/contrib/librdns/resolver.c @@ -332,14 +332,15 @@ rdns_process_timer (void *arg) req->retransmits --; resolver = req->resolver; + if (req->resolver->ups && req->io->srv->ups_elt) { + req->resolver->ups->fail (req->io->srv->ups_elt, + req->resolver->ups->data, "timeout waiting reply"); + } + else { + UPSTREAM_FAIL (req->io->srv, time (NULL)); + } + if (req->retransmits == 0) { - if (req->resolver->ups && req->io->srv->ups_elt) { - req->resolver->ups->fail (req->io->srv->ups_elt, - req->resolver->ups->data); - } - else { - UPSTREAM_FAIL (req->io->srv, time (NULL)); - } rep = rdns_make_reply (req, RDNS_RC_TIMEOUT); rdns_request_unschedule (req); @@ -371,8 +372,11 @@ rdns_process_timer (void *arg) if (resolver->ups) { struct rdns_upstream_elt *elt; - elt = resolver->ups->select_retransmit (req->requested_names[0].name, - req->requested_names[0].len, resolver->ups->data); + elt = resolver->ups->select_retransmit ( + req->requested_names[0].name, + req->requested_names[0].len, + req->io->srv->ups_elt, + resolver->ups->data); if (elt) { serv = elt->server; @@ -423,7 +427,7 @@ rdns_process_timer (void *arg) else if (r == -1) { if (req->resolver->ups && req->io->srv->ups_elt) { req->resolver->ups->fail (req->io->srv->ups_elt, - req->resolver->ups->data); + req->resolver->ups->data, "cannot send retransmit after timeout"); } else { UPSTREAM_FAIL (req->io->srv, time (NULL)); @@ -533,7 +537,7 @@ rdns_process_retransmit (int fd, void *arg) else if (r == -1) { if (req->resolver->ups && req->io->srv->ups_elt) { req->resolver->ups->fail (req->io->srv->ups_elt, - req->resolver->ups->data); + req->resolver->ups->data, "retransmit send failed"); } else { UPSTREAM_FAIL (req->io->srv, time (NULL)); @@ -551,6 +555,43 @@ rdns_process_retransmit (int fd, void *arg) } } +struct rdns_server * +rdns_select_request_upstream (struct rdns_resolver *resolver, + struct rdns_request *req, + bool is_retransmit, + struct rdns_server *prev_serv) +{ + struct rdns_server *serv = NULL; + + if (resolver->ups) { + struct rdns_upstream_elt *elt; + + if (is_retransmit && prev_serv) { + elt = resolver->ups->select_retransmit (req->requested_names[0].name, + req->requested_names[0].len, + prev_serv->ups_elt, + resolver->ups->data); + } + else { + elt = resolver->ups->select (req->requested_names[0].name, + req->requested_names[0].len, resolver->ups->data); + } + + if (elt) { + serv = elt->server; + serv->ups_elt = elt; + } + else { + UPSTREAM_SELECT_ROUND_ROBIN (resolver->servers, serv); + } + } + else { + UPSTREAM_SELECT_ROUND_ROBIN (resolver->servers, serv); + } + + return serv; +} + #define align_ptr(p, a) \ (guint8 *) (((uintptr_t) (p) + ((uintptr_t) a - 1)) & ~((uintptr_t) a - 1)) @@ -737,30 +778,14 @@ rdns_make_request_full ( /* Add EDNS RR */ rdns_add_edns0 (req); - req->retransmits = repeats; + req->retransmits = repeats ? repeats : 1; req->timeout = timeout; req->state = RDNS_REQUEST_NEW; } req->async = resolver->async; - if (resolver->ups) { - struct rdns_upstream_elt *elt; - - elt = resolver->ups->select (req->requested_names[0].name, - req->requested_names[0].len, resolver->ups->data); - - if (elt) { - serv = elt->server; - serv->ups_elt = elt; - } - else { - UPSTREAM_SELECT_ROUND_ROBIN (resolver->servers, serv); - } - } - else { - UPSTREAM_SELECT_ROUND_ROBIN (resolver->servers, serv); - } + serv = rdns_select_request_upstream (resolver, req, false, NULL); if (serv == NULL) { rdns_warn ("cannot find suitable server for request"); @@ -776,24 +801,54 @@ rdns_make_request_full ( req->io->sock, req); } else { - req->io->uses++; - /* Now send request to server */ - r = rdns_send_request (req, req->io->sock, true); + do { + r = rdns_send_request (req, req->io->sock, true); - if (r == -1) { - rdns_info ("cannot send DNS request: %s", strerror (errno)); - REF_RELEASE (req); + if (r == -1) { + req->retransmits --; /* It must be > 0 */ + + if (req->retransmits > 0) { + if (resolver->ups && serv->ups_elt) { + resolver->ups->fail (serv->ups_elt, resolver->ups->data, + "send IO error"); + } + else { + UPSTREAM_FAIL (serv, time (NULL)); + } + + serv = rdns_select_request_upstream (resolver, req, + true, serv); + + if (serv == NULL) { + rdns_warn ("cannot find suitable server for request"); + REF_RELEASE (req); + return NULL; + } + + req->io = serv->io_channels[ottery_rand_uint32 () % serv->io_cnt]; + } + else { + rdns_info ("cannot send DNS request: %s", strerror (errno)); + REF_RELEASE (req); - if (resolver->ups && serv->ups_elt) { - resolver->ups->fail (serv->ups_elt, resolver->ups->data); + if (resolver->ups && serv->ups_elt) { + resolver->ups->fail (serv->ups_elt, resolver->ups->data, + "send IO error"); + } + else { + UPSTREAM_FAIL (serv, time (NULL)); + } + + return NULL; + } } else { - UPSTREAM_FAIL (serv, time (NULL)); + /* All good */ + req->io->uses++; + break; } - - return NULL; - } + } while (req->retransmits > 0); } REF_RETAIN (req->io); diff --git a/contrib/libucl/ucl_chartable.h b/contrib/libucl/ucl_chartable.h index db9f02900..043b62689 100644 --- a/contrib/libucl/ucl_chartable.h +++ b/contrib/libucl/ucl_chartable.h @@ -27,7 +27,7 @@ #include "ucl_internal.h" static const unsigned int ucl_chartable[256] = { -UCL_CHARACTER_VALUE_END, UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED, +UCL_CHARACTER_JSON_UNSAFE|UCL_CHARACTER_VALUE_END|UCL_CHARACTER_UCL_UNSAFE, UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED, UCL_CHARACTER_JSON_UNSAFE|UCL_CHARACTER_UCL_UNSAFE, diff --git a/contrib/libucl/ucl_emitter.c b/contrib/libucl/ucl_emitter.c index d37cdda40..687e6cdae 100644 --- a/contrib/libucl/ucl_emitter.c +++ b/contrib/libucl/ucl_emitter.c @@ -756,6 +756,9 @@ ucl_elt_string_write_json (const char *str, size_t size, func->ucl_emitter_append_len (c, len, func->ud); } switch (*p) { + case '\0': + func->ucl_emitter_append_len ("\\u0000", 6, func->ud); + break; case '\n': func->ucl_emitter_append_len ("\\n", 2, func->ud); break; diff --git a/contrib/libucl/ucl_util.c b/contrib/libucl/ucl_util.c index 5ef83e31b..830aaa14c 100644 --- a/contrib/libucl/ucl_util.c +++ b/contrib/libucl/ucl_util.c @@ -2244,6 +2244,7 @@ ucl_object_fromstring_common (const char *str, size_t len, enum ucl_string_flags if (ucl_test_character (*p, UCL_CHARACTER_JSON_UNSAFE | UCL_CHARACTER_WHITESPACE_UNSAFE)) { switch (*p) { case '\v': + case '\0': escaped_len += 5; break; case ' ': @@ -2279,6 +2280,14 @@ ucl_object_fromstring_common (const char *str, size_t len, enum ucl_string_flags *d++ = '\\'; *d = 'f'; break; + case '\0': + *d++ = '\\'; + *d++ = 'u'; + *d++ = '0'; + *d++ = '0'; + *d++ = '0'; + *d = '0'; + break; case '\v': *d++ = '\\'; *d++ = 'u'; diff --git a/contrib/lua-lpeg/CMakeLists.txt b/contrib/lua-lpeg/CMakeLists.txt index 2362aac9c..92dd0182d 100644 --- a/contrib/lua-lpeg/CMakeLists.txt +++ b/contrib/lua-lpeg/CMakeLists.txt @@ -4,12 +4,6 @@ SET(LPEGSRC lpcap.c lptree.c lpvm.c) -IF(ENABLE_FULL_DEBUG MATCHES "OFF") -if ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") - SET(LPEG_CFLAGS "${LPEG_CFLAGS} -O3") -endif () -ENDIF() - SET(LIB_TYPE STATIC) ADD_LIBRARY(rspamd-lpeg ${LIB_TYPE} ${LPEGSRC}) set_target_properties(rspamd-lpeg PROPERTIES COMPILE_FLAGS "${LPEG_CFLAGS}") diff --git a/contrib/lua-lpeg/lptree.c b/contrib/lua-lpeg/lptree.c index cb8342459..ffc03f770 100644 --- a/contrib/lua-lpeg/lptree.c +++ b/contrib/lua-lpeg/lptree.c @@ -6,6 +6,7 @@ #include <ctype.h> #include <limits.h> #include <string.h> +#include <src/lua/lua_common.h> #include "lua.h" @@ -1155,9 +1156,25 @@ static int lp_match (lua_State *L) { #endif const char *r; size_t l; + const char *s; + Pattern *p = (getpatt(L, 1, NULL), getpattern(L, 1)); Instruction *code = (p->code != NULL) ? p->code : prepcompile(L, p, 1); - const char *s = luaL_checklstring(L, SUBJIDX, &l); + + if (lua_type (L, SUBJIDX) == LUA_TSTRING) { + s = luaL_checklstring (L, SUBJIDX, &l); + } + else if (lua_type (L, SUBJIDX) == LUA_TUSERDATA) { + struct rspamd_lua_text *t = lua_check_text (L, SUBJIDX); + if (!t) { + return luaL_error (L, "invalid argument (not a text)"); + } + s = t->start; + l = t->len; + } + else { + return luaL_error (L, "invalid argument"); + } size_t i = initposition(L, l); int ptop = lua_gettop(L), rs; lua_pushnil(L); /* initialize subscache */ diff --git a/contrib/lua-lpeg/lpvm.c b/contrib/lua-lpeg/lpvm.c index 4ef424579..6058bf5b1 100644 --- a/contrib/lua-lpeg/lpvm.c +++ b/contrib/lua-lpeg/lpvm.c @@ -306,27 +306,39 @@ const char *match (lua_State *L, const char *o, const char *s, const char *e, continue; } case IChar: { - if ((byte)*s == p->i.aux && s < e) { p++; s++; } + if (s < e && (byte)*s == p->i.aux) { p++; s++; } else goto fail; continue; } case ITestChar: { - if ((byte)*s == p->i.aux && s < e) p += 2; + if (s < e && (byte)*s == p->i.aux) p += 2; else p += getoffset(p); continue; } case ISet: { - int c = (byte)*s; - if (testchar((p+1)->buff, c) && s < e) - { p += CHARSETINSTSIZE; s++; } - else goto fail; + if (s < e) { + int c = (byte) *s; + if (testchar((p + 1)->buff, c)) { + p += CHARSETINSTSIZE; + s++; + } + else goto fail; + } + else { + goto fail; + } continue; } case ITestSet: { - int c = (byte)*s; - if (testchar((p + 2)->buff, c) && s < e) - p += 1 + CHARSETINSTSIZE; - else p += getoffset(p); + if (s < e) { + int c = (byte) *s; + if (testchar((p + 2)->buff, c)) + p += 1 + CHARSETINSTSIZE; + else p += getoffset(p); + } + else { + p += getoffset(p); + } continue; } case IBehind: { diff --git a/contrib/replxx/CMakeLists.txt b/contrib/replxx/CMakeLists.txt index a1b40dfdb..b391639f5 100644 --- a/contrib/replxx/CMakeLists.txt +++ b/contrib/replxx/CMakeLists.txt @@ -1,6 +1,5 @@ # -*- mode: CMAKE; -*- project( replxx VERSION 0.0.2 LANGUAGES CXX C ) -message(STATUS "Build mode: ${CMAKE_BUILD_TYPE}") # INFO set(REPLXX_DISPLAY_NAME "replxx") diff --git a/contrib/t1ha/CMakeLists.txt b/contrib/t1ha/CMakeLists.txt index 491010ff9..c8de483f4 100644 --- a/contrib/t1ha/CMakeLists.txt +++ b/contrib/t1ha/CMakeLists.txt @@ -4,9 +4,3 @@ SET(T1HASRC t1ha1.c ADD_LIBRARY(rspamd-t1ha STATIC ${T1HASRC}) SET_TARGET_PROPERTIES(rspamd-t1ha PROPERTIES VERSION ${RSPAMD_VERSION}) ADD_DEFINITIONS("-DT1HA_USE_FAST_ONESHOT_READ=0") - -IF(ENABLE_FULL_DEBUG MATCHES "OFF") - if ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") - SET_TARGET_PROPERTIES(rspamd-t1ha PROPERTIES COMPILE_FLAGS "-O3") - endif () -ENDIF() diff --git a/contrib/zstd/CMakeLists.txt b/contrib/zstd/CMakeLists.txt index 2cccee0c1..120e179ba 100644 --- a/contrib/zstd/CMakeLists.txt +++ b/contrib/zstd/CMakeLists.txt @@ -22,9 +22,3 @@ SET(ZSTDSRC ADD_LIBRARY(rspamd-zstd STATIC ${ZSTDSRC}) ADD_DEFINITIONS(-DZSTD_STATIC_LINKING_ONLY) - -IF(ENABLE_FULL_DEBUG MATCHES "OFF") -if ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") - SET_TARGET_PROPERTIES(rspamd-zstd PROPERTIES COMPILE_FLAGS "-O3") -endif () -ENDIF() diff --git a/debian/rules b/debian/rules index 325cf5e3c..372aaffcd 100755 --- a/debian/rules +++ b/debian/rules @@ -5,6 +5,7 @@ .PHONY: override_dh_strip export DEB_LDFLAGS_MAINT_APPEND = -Wl,--as-needed +export ASAN_OPTIONS=detect_leaks=0 override_dh_auto_configure: dh_auto_configure -- -DCONFDIR=/etc/rspamd \ @@ -21,8 +22,6 @@ override_dh_auto_configure: -DDEBIAN_BUILD=1 \ -DINSTALL_EXAMPLES=ON \ -DENABLE_JEMALLOC=ON \ - -DENABLE_OPTIMIZATION=OFF \ - -DENABLE_FULL_DEBUG=OFF \ -DENABLE_GD=OFF \ -DENABLE_PCRE2=OFF \ -DENABLE_LUAJIT=ON \ diff --git a/doc/Makefile b/doc/Makefile index 888dc6b4b..6c99a9288 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -38,7 +38,7 @@ lua_clickhouse: lua_selectors: $(LLUADOC) < ../lualib/lua_selectors/init.lua > markdown/lua/lua_selectors.md -lua_clickhouse: +lua_mime: $(LLUADOC) < ../lualib/lua_mime.lua > markdown/lua/lua_mime.md rspamd_regexp: ../src/lua/lua_regexp.c diff --git a/lualib/lua_content/ical.lua b/lualib/lua_content/ical.lua new file mode 100644 index 000000000..bb2f52771 --- /dev/null +++ b/lualib/lua_content/ical.lua @@ -0,0 +1,84 @@ +--[[ +Copyright (c) 2019, Vsevolod Stakhov <vsevolod@highsecure.ru> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +local l = require 'lpeg' +local lua_util = require "lua_util" +local N = "lua_content" + +local ical_grammar + +local function gen_grammar() + if not ical_grammar then + local wsp = l.P" " + local crlf = l.P"\r"^-1 * l.P"\n" + local eol = (crlf * #crlf) + (crlf - (crlf^-1 * wsp)) + local name = l.C((l.P(1) - (l.P":"))^1) / function(v) return (v:gsub("[\n\r]+%s","")) end + local value = l.C((l.P(1) - eol)^0) / function(v) return (v:gsub("[\n\r]+%s","")) end + ical_grammar = name * ":" * wsp^0 * value * eol + end + + return ical_grammar +end + +local exports = {} + +local function extract_text_data(specific) + local fun = require "fun" + + local tbl = fun.totable(fun.map(function(e) return e[2]:lower() end, specific.elts)) + return table.concat(tbl, '\n') +end + +local function process_ical(input, _, task) + local control={n='\n', r=''} + local rspamd_url = require "rspamd_url" + local escaper = l.Ct((gen_grammar() / function(key, value) + value = value:gsub("\\(.)", control) + key = key:lower() + local local_urls = rspamd_url.all(task:get_mempool(), value) + + if local_urls and #local_urls > 0 then + for _,u in ipairs(local_urls) do + lua_util.debugm(N, task, 'ical: found URL in ical %s', + tostring(u)) + task:inject_url(u) + end + end + lua_util.debugm(N, task, 'ical: ical key %s = "%s"', + key, value) + return {key, value} + end)^1) + + local elts = escaper:match(input) + + if not elts then + return nil + end + + return { + tag = 'ical', + extract_text = extract_text_data, + elts = elts + } +end + +--[[[ +-- @function lua_ical.process(input) +-- Returns all values from ical as a plain text. Names are completely ignored. +--]] +exports.process = process_ical + +return exports
\ No newline at end of file diff --git a/lualib/lua_content/init.lua b/lualib/lua_content/init.lua new file mode 100644 index 000000000..c23ca9d09 --- /dev/null +++ b/lualib/lua_content/init.lua @@ -0,0 +1,97 @@ +--[[ +Copyright (c) 2019, Vsevolod Stakhov <vsevolod@highsecure.ru> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +--[[[ +-- @module lua_content +-- This module contains content processing logic +--]] + + +local exports = {} +local N = "lua_content" +local lua_util = require "lua_util" + +local content_modules = { + ical = { + mime_type = "text/calendar", + module = require "lua_content/ical", + extensions = {'ical'}, + output = "text" + }, + pdf = { + mime_type = "application/pdf", + module = require "lua_content/pdf", + extensions = {'pdf'}, + output = "table" + }, +} + +local modules_by_mime_type +local modules_by_extension + +local function init() + modules_by_mime_type = {} + modules_by_extension = {} + for k,v in pairs(content_modules) do + if v.mime_type then + modules_by_mime_type[v.mime_type] = {k, v} + end + if v.extensions then + for _,ext in ipairs(v.extensions) do + modules_by_extension[ext] = {k, v} + end + end + end +end + +exports.maybe_process_mime_part = function(part, task) + if not modules_by_mime_type then + init() + end + + local ctype, csubtype = part:get_type() + local mt = string.format("%s/%s", ctype or 'application', + csubtype or 'octet-stream') + local pair = modules_by_mime_type[mt] + + if not pair then + local ext = part:get_detected_ext() + + if ext then + pair = modules_by_extension[ext] + end + end + + if pair then + lua_util.debugm(N, task, "found known content of type %s: %s", + mt, pair[1]) + + local data = pair[2].module.process(part:get_content(), part, task) + + if data then + lua_util.debugm(N, task, "extracted content from %s: %s type", + pair[1], type(data)) + part:set_specific(data) + else + lua_util.debugm(N, task, "failed to extract anything from %s", + pair[1]) + end + end + +end + + +return exports
\ No newline at end of file diff --git a/lualib/lua_content/pdf.lua b/lualib/lua_content/pdf.lua new file mode 100644 index 000000000..460938f8a --- /dev/null +++ b/lualib/lua_content/pdf.lua @@ -0,0 +1,1040 @@ +--[[ +Copyright (c) 2019, Vsevolod Stakhov <vsevolod@highsecure.ru> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +--[[[ +-- @module lua_content/pdf +-- This module contains some heuristics for PDF files +--]] + +local rspamd_trie = require "rspamd_trie" +local rspamd_util = require "rspamd_util" +local rspamd_text = require "rspamd_text" +local rspamd_url = require "rspamd_url" +local bit = require "bit" +local N = "lua_content" +local lua_util = require "lua_util" +local rspamd_regexp = require "rspamd_regexp" +local lpeg = require "lpeg" +local pdf_patterns = { + trailer = { + patterns = { + [[\ntrailer\r?\n]] + } + }, + javascript = { + patterns = { + [[\/JS(?:[\s/><])]], + [[\/JavaScript(?:[\s/><])]], + } + }, + openaction = { + patterns = { + [[\/OpenAction(?:[\s/><])]], + [[\/AA(?:[\s/><])]], + } + }, + suspicious = { + patterns = { + [[netsh\s]], + [[echo\s]], + [[\/[A-Za-z]*#\d\d(?:[#A-Za-z<>/\s])]], -- Hex encode obfuscation + } + }, + start_object = { + patterns = { + [=[[\r\n\0]\s*\d+\s+\d+\s+obj[\r\n]]=] + } + }, + end_object = { + patterns = { + [=[endobj[\r\n]]=] + } + }, + start_stream = { + patterns = { + [=[>\s*stream[\r\n]]=], + } + }, + end_stream = { + patterns = { + [=[endstream[\r\n]]=] + } + } +} + +local pdf_text_patterns = { + start = { + patterns = { + [[\sBT\s]] + } + }, + stop = { + patterns = { + [[\sET\b]] + } + } +} + +-- index[n] -> +-- t[1] - pattern, +-- t[2] - key in patterns table, +-- t[3] - value in patterns table +-- t[4] - local pattern index +local pdf_indexes = {} +local pdf_text_indexes = {} + +local pdf_trie +local pdf_text_trie + +local exports = {} + +local config = { + max_extraction_size = 512 * 1024, + max_processing_size = 32 * 1024, + enabled = true, +} + +-- Used to process patterns found in PDF +-- positions for functional processors should be a iter/table from trie matcher in form +---- [{n1, pat_idx1}, ... {nn, pat_idxn}] where +---- pat_idxn is pattern index and n1 ... nn are match positions +local processors = {} +-- PDF objects outer grammar in LPEG style (performing table captures) +local pdf_outer_grammar +local pdf_text_grammar + +-- Used to match objects +local object_re = rspamd_regexp.create_cached([=[/(\d+)\s+(\d+)\s+obj\s*/]=]) + +local function config_module() + local opts = rspamd_config:get_all_opt('lua_content') + + if opts and opts.pdf then + config = lua_util.override_defaults(config, opts.pdf) + end +end + +local function compile_tries() + local default_compile_flags = bit.bor(rspamd_trie.flags.re, + rspamd_trie.flags.dot_all, + rspamd_trie.flags.no_start) + local function compile_pats(patterns, indexes, compile_flags) + local strs = {} + for what,data in pairs(patterns) do + for i,pat in ipairs(data.patterns) do + strs[#strs + 1] = pat + indexes[#indexes + 1] = {what, data, pat, i} + end + end + + return rspamd_trie.create(strs, compile_flags or default_compile_flags) + end + + if not pdf_trie then + pdf_trie = compile_pats(pdf_patterns, pdf_indexes) + end + if not pdf_text_trie then + pdf_text_trie = compile_pats(pdf_text_patterns, pdf_text_indexes) + end +end + +-- Returns a table with generic grammar elements for PDF +local function generic_grammar_elts() + local P = lpeg.P + local R = lpeg.R + local S = lpeg.S + local V = lpeg.V + local C = lpeg.C + local D = R'09' -- Digits + + local grammar_elts = {} + + -- Helper functions + local function pdf_hexstring_unescape(s) + local function ue(cc) + return string.char(tonumber(cc, 16)) + end + if #s % 2 == 0 then + -- Sane hex string + return s:gsub('..', ue) + end + + -- WTF hex string + -- Append '0' to it and unescape... + return s:sub(1, #s - 1):gsub('..' , ue) .. (s:sub(#s) .. '0'):gsub('..' , ue) + end + + local function pdf_string_unescape(s) + -- TODO: add unescaping logic + return s + end + + local function pdf_id_unescape(s) + return (s:gsub('#%d%d', function (cc) + return string.char(tonumber(cc:sub(2), 16)) + end)) + end + + local delim = S'()<>[]{}/%' + grammar_elts.ws = S'\0 \r\n\t\f' + local hex = R'af' + R'AF' + D + -- Comments. + local eol = P'\r\n' + '\n' + local line = (1 - S'\r\n\f')^0 * eol^-1 + grammar_elts.comment = P'%' * line + + -- Numbers. + local sign = S'+-'^-1 + local decimal = D^1 + local float = D^1 * P'.' * D^0 + P'.' * D^1 + grammar_elts.number = C(sign * (float + decimal)) / tonumber + + -- String + grammar_elts.str = P{ "(" * C(((1 - S"()") + V(1))^0) / pdf_string_unescape * ")" } + grammar_elts.hexstr = P{"<" * C(hex^0) / pdf_hexstring_unescape * ">"} + + -- Identifier + grammar_elts.id = P{'/' * C((1-(delim + grammar_elts.ws))^1) / pdf_id_unescape} + + -- Booleans (who care about them?) + grammar_elts.boolean = C(P("true") + P("false")) + + -- Stupid references + grammar_elts.ref = lpeg.Ct{lpeg.Cc("%REF%") * C(D^1) * " " * C(D^1) * " " * "R"} + + return grammar_elts +end + + +-- Generates a grammar to parse outer elements (external objects in PDF notation) +local function gen_outer_grammar() + local V = lpeg.V + local gen = generic_grammar_elts() + + return lpeg.P{ + "EXPR"; + EXPR = gen.ws^0 * V("ELT")^0 * gen.ws^0, + ELT = V("ARRAY") + V("DICT") + V("ATOM"), + ATOM = gen.ws^0 * (gen.comment + gen.boolean + gen.ref + + gen.number + V("STRING") + gen.id) * gen.ws^0, + DICT = "<<" * gen.ws^0 * lpeg.Cf(lpeg.Ct("") * V("KV_PAIR")^0, rawset) * gen.ws^0 * ">>", + KV_PAIR = lpeg.Cg(gen.id * gen.ws^0 * V("ELT") * gen.ws^0), + ARRAY = "[" * gen.ws^0 * lpeg.Ct(V("ELT")^0) * gen.ws^0 * "]", + STRING = lpeg.P{gen.str + gen.hexstr}, + } +end + +-- Graphic state in PDF +local function gen_graphics_unary() + local P = lpeg.P + local S = lpeg.S + + return P("q") + P("Q") + P("h") + + S("WSsFfBb") * P("*")^0 + P("n") + +end +local function gen_graphics_binary() + local P = lpeg.P + local S = lpeg.S + + return S("gGwJjMi") + + P("M") + P("ri") + P("gs") + + P("CS") + P("cs") + P("sh") +end +local function gen_graphics_ternary() + local P = lpeg.P + local S = lpeg.S + + return P("d") + P("m") + S("lm") +end +local function gen_graphics_nary() + local P = lpeg.P + local S = lpeg.S + + return P("SC") + P("sc") + P("SCN") + P("scn") + P("k") + P("K") + P("re") + S("cvy") + + P("RG") + P("rg") +end + +-- Generates a grammar to parse text blocks (between BT and ET) +local function gen_text_grammar() + local V = lpeg.V + local P = lpeg.P + local C = lpeg.C + local gen = generic_grammar_elts() + + local empty = "" + local unary_ops = C("T*") / "\n" + + C(gen_graphics_unary()) / empty + local binary_ops = P("Tc") + P("Tw") + P("Tz") + P("TL") + P("Tr") + P("Ts") + + gen_graphics_binary() + local ternary_ops = P("TD") + P("Td") + gen_graphics_ternary() + local nary_op = P("Tm") + gen_graphics_nary() + local text_binary_op = P("Tj") + P("TJ") + P("'") + local text_quote_op = P('"') + local font_op = P("Tf") + + return lpeg.P{ + "EXPR"; + EXPR = gen.ws^0 * lpeg.Ct(V("COMMAND")^0), + COMMAND = (V("UNARY") + V("BINARY") + V("TERNARY") + V("NARY") + V("TEXT") + + V("FONT") + gen.comment) * gen.ws^0, + UNARY = unary_ops, + BINARY = V("ARG") / empty * gen.ws^1 * binary_ops, + TERNARY = V("ARG") / empty * gen.ws^1 * V("ARG") / empty * gen.ws^1 * ternary_ops, + NARY = (gen.number / 0 * gen.ws^1)^1 * (gen.id / empty * gen.ws^0)^-1 * nary_op, + ARG = V("ARRAY") + V("DICT") + V("ATOM"), + ATOM = (gen.comment + gen.boolean + gen.ref + + gen.number + V("STRING") + gen.id), + DICT = "<<" * gen.ws^0 * lpeg.Cf(lpeg.Ct("") * V("KV_PAIR")^0, rawset) * gen.ws^0 * ">>", + KV_PAIR = lpeg.Cg(gen.id * gen.ws^0 * V("ARG") * gen.ws^0), + ARRAY = "[" * gen.ws^0 * lpeg.Ct(V("ARG")^0) * gen.ws^0 * "]", + STRING = lpeg.P{gen.str + gen.hexstr}, + TEXT = (V("TEXT_ARG") * gen.ws^1 * text_binary_op) + + (V("ARG") / 0 * gen.ws^1 * V("ARG") / 0 * gen.ws^1 * V("TEXT_ARG") * gen.ws^1 * text_quote_op), + FONT = (V("FONT_ARG") * gen.ws^1 * (gen.number / 0) * gen.ws^1 * font_op), + FONT_ARG = lpeg.Ct(lpeg.Cc("%font%") * gen.id), + TEXT_ARG = lpeg.Ct(V("STRING")) + V("TEXT_ARRAY"), + TEXT_ARRAY = "[" * + lpeg.Ct(((gen.ws^0 * (gen.ws^0 * (gen.number / 0)^0 * gen.ws^0 * (gen.str + gen.hexstr)))^1)) * gen.ws^0 * "]", + } +end + +-- Call immediately on require +compile_tries() +config_module() +pdf_outer_grammar = gen_outer_grammar() +pdf_text_grammar = gen_text_grammar() + +local function extract_text_data(specific) + return nil -- NYI +end + +-- Generates index for major/minor pair +local function obj_ref(major, minor) + return major * 10.0 + 1.0 / (minor + 1.0) +end + +-- Return indirect object reference (if needed) +local function maybe_dereference_object(elt, pdf, task) + if type(elt) == 'table' and elt[1] == '%REF%' then + local ref = obj_ref(elt[2], elt[3]) + + if pdf.ref[ref] then + -- No recursion! + return pdf.ref[ref] + else + lua_util.debugm(N, task, 'cannot dereference %s:%s -> %s', + elt[2], elt[3], obj_ref(elt[2], elt[3])) + return nil + end + end + + return elt +end + +-- Enforced dereference +local function dereference_object(elt, pdf) + if type(elt) == 'table' and elt[1] == '%REF%' then + local ref = obj_ref(elt[2], elt[3]) + + if pdf.ref[ref] then + -- Not a dict but the object! + return pdf.ref[ref] + end + end + + return nil +end + +-- Apply PDF stream filter +local function apply_pdf_filter(input, filt) + if filt == 'FlateDecode' then + return rspamd_util.inflate(input, config.max_extraction_size) + end + + return nil +end + +-- Conditionally apply a pipeline of stream filters and return uncompressed data +local function maybe_apply_filter(dict, data) + local uncompressed = data + + if dict.Filter then + local filt = dict.Filter + if type(filt) == 'string' then + filt = {filt} + end + + for _,f in ipairs(filt) do + uncompressed = apply_pdf_filter(uncompressed, f) + + if not uncompressed then break end + end + end + + return uncompressed +end + +-- Conditionally extract stream data from object and attach it as obj.uncompressed +local function maybe_extract_object_stream(obj, pdf, task) + local dict = obj.dict + if dict.Filter and dict.Length then + local len = math.min(obj.stream.len, + tonumber(maybe_dereference_object(dict.Length, pdf, task)) or 0) + local real_stream = obj.stream.data:span(1, len) + + local uncompressed = maybe_apply_filter(dict, real_stream) + + if uncompressed then + obj.uncompressed = uncompressed + lua_util.debugm(N, task, 'extracted object %s:%s: (%s -> %s)', + obj.major, obj.minor, len, uncompressed:len()) + return obj.uncompressed + else + lua_util.debugm(N, task, 'cannot extract object %s:%s; len = %s; filter = %s', + obj.major, obj.minor, len, dict.Filter) + end + end +end + + +local function parse_object_grammar(obj, task, pdf) + -- Parse grammar + local obj_dict_span + if obj.stream then + obj_dict_span = obj.data:span(1, obj.stream.start - obj.start) + else + obj_dict_span = obj.data + end + + if obj_dict_span:len() < config.max_processing_size then + local ret,obj_or_err = pcall(pdf_outer_grammar.match, pdf_outer_grammar, obj_dict_span) + + if ret then + if obj.stream then + obj.dict = obj_or_err + lua_util.debugm(N, task, 'stream object %s:%s is parsed to: %s', + obj.major, obj.minor, obj_or_err) + else + -- Direct object + pdf.ref[obj_ref(obj.major, obj.minor)] = obj_or_err + if type(obj_or_err) == 'table' then + obj.dict = obj_or_err + obj.uncompressed = obj_or_err + lua_util.debugm(N, task, 'direct object %s:%s is parsed to: %s', + obj.major, obj.minor, obj_or_err) + else + obj.dict = {} + obj.uncompressed = obj_or_err + end + end + else + lua_util.debugm(N, task, 'object %s:%s cannot be parsed: %s', + obj.major, obj.minor, obj_or_err) + end + else + lua_util.debugm(N, task, 'object %s:%s cannot be parsed: too large %s', + obj.major, obj.minor, obj_dict_span:len()) + end +end + +-- Extracts font data and process /ToUnicode mappings +local function process_font(task, pdf, font, fname) + local dict = font + if font.dict then + dict = font.dict + end + + if type(dict) == 'table' and dict.ToUnicode then + local cmap = maybe_dereference_object(dict.ToUnicode, pdf, task) + + if cmap and cmap.dict then + maybe_extract_object_stream(cmap, pdf, task) + lua_util.debugm(N, task, 'found cmap for font %s: %s', + fname, cmap.uncompressed) + end + end +end + +local function process_dict(task, pdf, obj, dict) + if not obj.type and type(dict) == 'table' then + if dict.Type and type(dict.Type) == 'string' then + -- Common stuff + obj.type = dict.Type + else + -- Fucking pdf, we need to guess a type (or ignore that crap)... + lua_util.debugm(N, task, 'no explicit type for %s:%s', + obj.major, obj.minor) + if dict.Parent then + -- Guess by parent + local parent = dereference_object(dict.Parent, pdf) + + if parent and parent.type then + if parent.type == 'Catalog' then + obj.type = 'Pages' + elseif parent.type == 'Pages' then + obj.type = 'Page' + end + + if obj.type then + lua_util.debugm(N, task, 'guessed type for %s:%s (%s) from parent %s:%s (%s)', + obj.major, obj.minor, obj.type, parent.major, parent.minor, parent.type) + + end + end + end + end + + lua_util.debugm(N, task, 'process stream dictionary for object %s:%s -> %s', + obj.major, obj.minor, obj.type) + local contents = dict.Contents + if contents and type(contents) == 'table' then + if contents[1] == '%REF%' then + -- Single reference + contents = {contents} + end + obj.contents = {} + + for _,c in ipairs(contents) do + local cobj = maybe_dereference_object(c, pdf, task) + if cobj then + obj.contents[#obj.contents + 1] = cobj + cobj.parent = obj + cobj.type = 'content' + end + end + + lua_util.debugm(N, task, 'found content objects for %s:%s -> %s', + obj.major, obj.minor, #obj.contents) + end + + local resources = dict.Resources + if resources and type(resources) == 'table' then + obj.resources = maybe_dereference_object(resources, pdf, task) + else + -- Fucking pdf: we need to inherit from parent + resources = {} + if dict.Parent then + local parent = maybe_dereference_object(dict.Parent, pdf, task) + + if parent and type(parent) == 'table' and parent.dict then + if parent.resources then + lua_util.debugm(N, task, 'propagated resources from %s:%s to %s:%s', + parent.major, parent.minor, obj.major, obj.minor) + resources = parent.resources + end + end + end + + obj.resources = resources + end + + local fonts = obj.resources.Font + + if fonts and type(fonts) == 'table' then + obj.fonts = {} + for k,v in pairs(fonts) do + obj.fonts[k] = maybe_dereference_object(v, pdf, task) + + if obj.fonts[k] then + local font = obj.fonts[k] + process_font(task, pdf, font, k) + lua_util.debugm(N, task, 'found font "%s" for object %s:%s -> %s', + k, obj.major, obj.minor, font) + end + end + end + + lua_util.debugm(N, task, 'found resources for object %s:%s: %s', + obj.major, obj.minor, obj.resources) + + if dict.Type == 'FontDescriptor' then + + lua_util.debugm(N, task, "obj %s:%s is a font descriptor", + obj.major, obj.minor) + + local stream_ref + if dict.FontFile then + stream_ref = dereference_object(dict.FontFile, pdf) + end + if dict.FontFile2 then + stream_ref = dereference_object(dict.FontFile2, pdf) + end + if dict.FontFile3 then + stream_ref = dereference_object(dict.FontFile3, pdf) + end + + if stream_ref then + if not stream_ref.dict then + stream_ref.dict = {} + end + stream_ref.dict.type = 'font_data' + stream_ref.dict.ignore = true + + lua_util.debugm(N, task, "obj %s:%s is a font data stream", + stream_ref.major, stream_ref.minor) + end + end + end +end + +-- This function is intended to unpack objects from ObjStm crappy structure +local compound_obj_grammar +local function compound_obj_grammar_gen() + if not compound_obj_grammar then + local gen = generic_grammar_elts() + compound_obj_grammar = gen.ws^0 * (gen.comment * gen.ws^1)^0 * + lpeg.Ct(lpeg.Ct(gen.number * gen.ws^1 * gen.number * gen.ws^0)^1) + end + + return compound_obj_grammar +end +local function pdf_compound_object_unpack(_, uncompressed, pdf, task, first) + -- First, we need to parse data line by line likely to find a line + -- that consists of pairs of numbers + compound_obj_grammar_gen() + local elts = compound_obj_grammar:match(uncompressed) + if elts and #elts > 0 then + lua_util.debugm(N, task, 'compound elts (chunk length %s): %s', + #uncompressed, elts) + + for i,pair in ipairs(elts) do + local obj_number,offset = pair[1], pair[2] + + offset = offset + first + if offset < #uncompressed then + local span_len + if i == #elts then + span_len = #uncompressed - offset + else + span_len = (elts[i + 1][2] + first) - offset + end + + if span_len > 0 then + local obj = { + major = obj_number, + minor = 0, -- Implicit + data = uncompressed:span(offset + 1, span_len), + ref = obj_ref(obj_number, 0) + } + parse_object_grammar(obj, task, pdf) + + if obj.dict then + pdf.objects[#pdf.objects + 1] = obj + end + end + end + end + end +end + +-- PDF 1.5 ObjStmt +local function extract_pdf_compound_objects(task, pdf) + for _,obj in ipairs(pdf.objects or {}) do + if obj.stream and obj.dict and type(obj.dict) == 'table' then + local t = obj.dict.Type + if t and t == 'ObjStm' then + -- We are in troubles sir... + local nobjs = tonumber(maybe_dereference_object(obj.dict.N, pdf, task)) + local first = tonumber(maybe_dereference_object(obj.dict.First, pdf, task)) + + if nobjs and first then + --local extend = maybe_dereference_object(obj.dict.Extends, pdf, task) + lua_util.debugm(N, task, 'extract ObjStm with %s objects (%s first) %s extend', + nobjs, first, obj.dict.Extends) + + local uncompressed = maybe_extract_object_stream(obj, pdf, task) + + if uncompressed then + pdf_compound_object_unpack(obj, uncompressed, pdf, task, first) + end + else + lua_util.debugm(N, task, 'ObjStm object %s:%s has bad dict: %s', + obj.major, obj.minor, obj.dict) + end + end + end + end +end + +-- This function arranges starts and ends of all objects and process them into initial +-- set of objects +local function extract_outer_objects(task, input, pdf) + local start_pos, end_pos = 1, 1 + local obj_count = 0 + while start_pos <= #pdf.start_objects and end_pos <= #pdf.end_objects do + local first = pdf.start_objects[start_pos] + local last = pdf.end_objects[end_pos] + + -- 7 is length of `endobj\n` + if first + 7 < last then + local len = last - first - 7 + + -- Also get the starting span and try to match it versus obj re to get numbers + local obj_line_potential = first - 32 + if obj_line_potential < 1 then obj_line_potential = 1 end + + if end_pos > 1 and pdf.end_objects[end_pos - 1] >= obj_line_potential then + obj_line_potential = pdf.end_objects[end_pos - 1] + 1 + end + + local obj_line_span = input:span(obj_line_potential, first - obj_line_potential + 1) + local matches = object_re:search(obj_line_span, true, true) + + if matches and matches[1] then + local nobj = { + start = first, + len = len, + data = input:span(first, len), + major = tonumber(matches[1][2]), + minor = tonumber(matches[1][3]), + } + pdf.objects[obj_count + 1] = nobj + if nobj.major and nobj.minor then + -- Add reference + local ref = obj_ref(nobj.major, nobj.minor) + nobj.ref = ref -- Our internal reference + pdf.ref[ref] = nobj + end + end + + obj_count = obj_count + 1 + start_pos = start_pos + 1 + end_pos = end_pos + 1 + elseif first > last then + end_pos = end_pos + 1 + else + start_pos = start_pos + 1 + end_pos = end_pos + 1 + end + end +end + +-- This function attaches streams to objects and processes outer pdf grammar +local function attach_pdf_streams(task, input, pdf) + if pdf.start_streams and pdf.end_streams then + local start_pos, end_pos = 1, 1 + + for _,obj in ipairs(pdf.objects) do + while start_pos <= #pdf.start_streams and end_pos <= #pdf.end_streams do + local first = pdf.start_streams[start_pos] + local last = pdf.end_streams[end_pos] + last = last - 10 -- Exclude endstream\n pattern + lua_util.debugm(N, task, "start: %s, end: %s; obj: %s-%s", + first, last, obj.start, obj.start + obj.len) + if first > obj.start and last < obj.start + obj.len and last > first then + -- In case if we have fake endstream :( + while pdf.end_streams[end_pos + 1] and pdf.end_streams[end_pos + 1] < obj.start + obj.len do + end_pos = end_pos + 1 + last = pdf.end_streams[end_pos] + end + -- Strip the first \n + while first < last do + local chr = input:at(first) + if chr ~= 13 and chr ~= 10 then break end + first = first + 1 + end + local len = last - first + obj.stream = { + start = first, + len = len, + data = input:span(first, len) + } + start_pos = start_pos + 1 + end_pos = end_pos + 1 + break + elseif first < obj.start then + start_pos = start_pos + 1 + elseif last > obj.start + obj.len then + -- Not this object + break + else + start_pos = start_pos + 1 + end_pos = end_pos + 1 + end + end + if obj.stream then + lua_util.debugm(N, task, 'found object %s:%s %s start %s len, %s stream start, %s stream length', + obj.major, obj.minor, obj.start, obj.len, obj.stream.start, obj.stream.len) + else + lua_util.debugm(N, task, 'found object %s:%s %s start %s len, no stream', + obj.major, obj.minor, obj.start, obj.len) + end + end + end +end + +-- Processes PDF objects: extracts streams, object numbers, process outer grammar, +-- augment object types +local function postprocess_pdf_objects(task, input, pdf) + pdf.objects = {} -- objects table + pdf.ref = {} -- references table + extract_outer_objects(task, input, pdf) + + -- Now we have objects and we need to attach streams that are in bounds + attach_pdf_streams(task, input, pdf) + -- Parse grammar for outer objects + for _,obj in ipairs(pdf.objects) do + if obj.ref then + parse_object_grammar(obj, task, pdf) + end + end + extract_pdf_compound_objects(task, pdf) + + -- Now we might probably have all objects being processed + for _,obj in ipairs(pdf.objects) do + if obj.dict then + -- Types processing + process_dict(task, pdf, obj, obj.dict) + end + end +end + +local function offsets_to_blocks(starts, ends, out) + local start_pos, end_pos = 1, 1 + + while start_pos <= #starts and end_pos <= #ends do + local first = starts[start_pos] + local last = ends[end_pos] + + if first < last then + local len = last - first + out[#out + 1] = { + start = first, + len = len, + } + start_pos = start_pos + 1 + end_pos = end_pos + 1 + elseif first > last then + end_pos = end_pos + 1 + else + -- Not ordered properly! + break + end + end +end + +local function search_text(task, pdf) + for _,obj in ipairs(pdf.objects) do + if obj.type == 'Page' and obj.contents then + local text = {} + for _,tobj in ipairs(obj.contents) do + maybe_extract_object_stream(tobj, pdf, task) + local matches = pdf_text_trie:match(tobj.uncompressed or '') + if matches then + local text_blocks = {} + local starts = {} + local ends = {} + + for npat,matched_positions in pairs(matches) do + if npat == 1 then + for _,pos in ipairs(matched_positions) do + starts[#starts + 1] = pos + end + else + for _,pos in ipairs(matched_positions) do + ends[#ends + 1] = pos + end + end + end + + offsets_to_blocks(starts, ends, text_blocks) + for _,bl in ipairs(text_blocks) do + if bl.len > 2 then + -- To remove \s+ET\b pattern (it can leave trailing space or not but it doesn't matter) + bl.len = bl.len - 2 + end + + bl.data = tobj.uncompressed:span(bl.start, bl.len) + --lua_util.debugm(N, task, 'extracted text from object %s:%s: %s', + -- tobj.major, tobj.minor, bl.data) + + if bl.len < config.max_processing_size then + local ret,obj_or_err = pcall(pdf_text_grammar.match, pdf_text_grammar, + bl.data) + + if ret then + text[#text + 1] = obj_or_err + lua_util.debugm(N, task, 'attached %s from content object %s:%s to %s:%s', + obj_or_err, tobj.major, tobj.minor, obj.major, obj.minor) + else + lua_util.debugm(N, task, 'object %s:%s cannot be parsed: %s', + obj.major, obj.minor, obj_or_err) + end + + end + end + end + end + + -- Join all text data together + if #text > 0 then + obj.text = rspamd_text.fromtable(text) + lua_util.debugm(N, task, 'object %s:%s is parsed to: %s', + obj.major, obj.minor, obj.text) + end + end + end +end + +-- This function searches objects for `/URI` key and parses it's content +local function search_urls(task, pdf) + local function recursive_object_traverse(obj, dict, rec) + if rec > 10 then + lua_util.debugm(N, task, 'object %s:%s recurses too much', + obj.major, obj.minor) + return + end + + for k,v in pairs(dict) do + if type(v) == 'table' then + recursive_object_traverse(obj, v, rec + 1) + elseif k == 'URI' then + v = maybe_dereference_object(v, pdf, task) + if type(v) == 'string' then + local url = rspamd_url.create(task:get_mempool(), v) + + if url then + lua_util.debugm(N, task, 'found url %s in object %s:%s', + v, obj.major, obj.minor) + task:inject_url(url) + end + end + end + end + end + + for _,obj in ipairs(pdf.objects) do + if obj.dict then + recursive_object_traverse(obj, obj.dict, 0) + end + end +end + +local function process_pdf(input, _, task) + + if not config.enabled then + -- Skip processing + return {} + end + + local matches = pdf_trie:match(input) + + if matches then + local pdf_output = { + tag = 'pdf', + extract_text = extract_text_data, + } + local grouped_processors = {} + for npat,matched_positions in pairs(matches) do + local index = pdf_indexes[npat] + + local proc_key,loc_npat = index[1], index[4] + + if not grouped_processors[proc_key] then + grouped_processors[proc_key] = { + processor_func = processors[proc_key], + offsets = {}, + } + end + local proc = grouped_processors[proc_key] + -- Fill offsets + for _,pos in ipairs(matched_positions) do + proc.offsets[#proc.offsets + 1] = {pos, loc_npat} + end + end + + for name,processor in pairs(grouped_processors) do + -- Sort by offset + lua_util.debugm(N, task, "pdf: process group %s with %s matches", + name, #processor.offsets) + table.sort(processor.offsets, function(e1, e2) return e1[1] < e2[1] end) + processor.processor_func(input, task, processor.offsets, pdf_output) + end + + pdf_output.flags = {} + + if pdf_output.start_objects and pdf_output.end_objects then + -- Postprocess objects + postprocess_pdf_objects(task, input, pdf_output) + search_text(task, pdf_output) + search_urls(task, pdf_output) + else + pdf_output.flags.no_objects = true + end + + return pdf_output + end +end + +-- Processes the PDF trailer +processors.trailer = function(input, task, positions, output) + local last_pos = positions[#positions] + + local last_span = input:span(last_pos[1]) + for line in last_span:lines(true) do + if line:find('/Encrypt ') then + lua_util.debugm(N, task, "pdf: found encrypted line in trailer: %s", + line) + output.encrypted = true + end + end +end + +processors.javascript = function(_, task, _, output) + lua_util.debugm(N, task, "pdf: found javascript tag") + output.javascript = true +end + +processors.openaction = function(_, task, _, output) + lua_util.debugm(N, task, "pdf: found openaction tag") + output.openaction = true +end + +processors.suspicious = function(_, task, _, output) + lua_util.debugm(N, task, "pdf: found a suspicious pattern") + output.suspicious = true +end + +local function generic_table_inserter(positions, output, output_key) + if not output[output_key] then + output[output_key] = {} + end + local shift = #output[output_key] + for i,pos in ipairs(positions) do + output[output_key][i + shift] = pos[1] + end +end + +processors.start_object = function(_, task, positions, output) + generic_table_inserter(positions, output, 'start_objects') +end + +processors.end_object = function(_, task, positions, output) + generic_table_inserter(positions, output, 'end_objects') +end + +processors.start_stream = function(_, task, positions, output) + generic_table_inserter(positions, output, 'start_streams') +end + +processors.end_stream = function(_, task, positions, output) + generic_table_inserter(positions, output, 'end_streams') +end + +exports.process = process_pdf + +return exports
\ No newline at end of file diff --git a/lualib/lua_dkim_tools.lua b/lualib/lua_dkim_tools.lua index 90bff13d5..42b595670 100644 --- a/lualib/lua_dkim_tools.lua +++ b/lualib/lua_dkim_tools.lua @@ -211,9 +211,9 @@ local function prepare_dkim_signing(N, task, settings) end local function is_skip_sign() - return (settings.sign_networks and not is_sign_networks) and - (settings.auth_only and not is_authed) and - (settings.sign_local and not is_local) + return not (settings.sign_networks and is_sign_networks) and + not (settings.auth_only and is_authed) and + not (settings.sign_local and is_local) end if hdom then diff --git a/lualib/lua_ical.lua b/lualib/lua_ical.lua deleted file mode 100644 index 4f6b61919..000000000 --- a/lualib/lua_ical.lua +++ /dev/null @@ -1,47 +0,0 @@ ---[[ -Copyright (c) 2019, Vsevolod Stakhov <vsevolod@highsecure.ru> - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -]]-- - -local l = require 'lpeg' - -local wsp = l.P" " -local crlf = l.P"\r"^-1 * l.P"\n" -local eol = (crlf * #crlf) + (crlf - (crlf^-1 * wsp)) -local name = l.C((l.P(1) - (l.P":"))^1) / function(v) return (v:gsub("[\n\r]+%s","")) end -local value = l.C((l.P(1) - eol)^0) / function(v) return (v:gsub("[\n\r]+%s","")) end -local elt = name * ":" * wsp^0 * value * eol - -local exports = {} - -local function ical_txt_values(input) - local control={n='\n', r='\r'} - local escaper = l.Ct((elt / function(_,b) return (b:gsub("\\(.)", control)) end)^1) - - local values = escaper:match(input) - - if not values then - return nil - end - - return table.concat(values, "\n") -end - ---[[[ --- @function lua_ical.ical_txt_values(input) --- Returns all values from ical as a plain text. Names are completely ignored. ---]] -exports.ical_txt_values = ical_txt_values - -return exports
\ No newline at end of file diff --git a/lualib/lua_magic/heuristics.lua b/lualib/lua_magic/heuristics.lua index 3da6a84ab..d9d408170 100644 --- a/lualib/lua_magic/heuristics.lua +++ b/lualib/lua_magic/heuristics.lua @@ -61,18 +61,24 @@ local zip_patterns = { local txt_trie local txt_patterns = { html = { - [[(?i)\s*<html]], - [[(?i)\s*<\!DOCTYPE HTML]], - [[(?i)\s*<xml]], - [[(?i)\s*<body]], - [[(?i)\s*<table]], - [[(?i)\s*<a]], - [[(?i)\s*<p]], - [[(?i)\s*<div]], - [[(?i)\s*<span]], + {[[(?i)\s*<html]], 30}, + {[[(?i)\s*<\!DOCTYPE HTML]], 30}, + {[[(?i)\s*<xml]], 20}, + {[[(?i)\s*<body]], 20}, + {[[(?i)\s*<table]], 20}, + {[[(?i)\s*<a]], 10}, + {[[(?i)\s*<p]], 10}, + {[[(?i)\s*<div]], 10}, + {[[(?i)\s*<span]], 10}, }, csv = { - [[(?:[-a-zA-Z0-9_]+\s*,){2,}(?:[-a-zA-Z0-9_]+,?[ ]*[\r\n])]] + {[[(?:[-a-zA-Z0-9_]+\s*,){2,}(?:[-a-zA-Z0-9_]+,?[ ]*[\r\n])]], 20} + }, + ics = { + {[[^BEGIN:VCALENDAR\r?\n]], 40}, + }, + vcf = { + {[[^BEGIN:VCARD\r?\n]], 40}, }, } @@ -95,7 +101,7 @@ local function compile_tries() for _,pat in ipairs(pats) do -- These are utf16 strings in fact... strs[#strs + 1] = transform_func(pat) - indexes[#indexes + 1] = ext + indexes[#indexes + 1] = {ext, pat} end end @@ -131,7 +137,7 @@ local function compile_tries() function(pat) return pat end) -- Text patterns at the initial fragment txt_trie = compile_pats(txt_patterns, txt_patterns_indexes, - function(pat) return pat end, + function(pat_tbl) return pat_tbl[1] end, bit.bor(rspamd_trie.flags.re, rspamd_trie.flags.dot_all, rspamd_trie.flags.no_start)) @@ -184,8 +190,8 @@ local function detect_ole_format(input, log_obj) for n,_ in pairs(matches) do if msoffice_clsid_indexes[n] then lua_util.debugm(N, log_obj, "found valid clsid for %s", - msoffice_clsid_indexes[n]) - return true,msoffice_clsid_indexes[n] + msoffice_clsid_indexes[n][1]) + return true,msoffice_clsid_indexes[n][1] end end end @@ -195,7 +201,7 @@ local function detect_ole_format(input, log_obj) if matches then for n,_ in pairs(matches) do if msoffice_patterns_indexes[n] then - return true,msoffice_patterns_indexes[n] + return true,msoffice_patterns_indexes[n][1] end end end @@ -295,8 +301,8 @@ local function detect_archive_flaw(part, arch, log_obj) for n,_ in pairs(matches) do if zip_patterns_indexes[n] then lua_util.debugm(N, log_obj, "found zip pattern for %s", - zip_patterns_indexes[n]) - return zip_patterns_indexes[n],40 + zip_patterns_indexes[n][1]) + return zip_patterns_indexes[n][1],40 end end end @@ -392,11 +398,11 @@ exports.text_part_heuristic = function(part, log_obj) if matches then -- Require at least 2 occurrences of those patterns for n,positions in pairs(matches) do - local ext = txt_patterns_indexes[n] + local ext,weight = txt_patterns_indexes[n][1], txt_patterns_indexes[n][2][2] if ext then - res[ext] = (res[ext] or 0) + 20 * #positions + res[ext] = (res[ext] or 0) + weight * #positions lua_util.debugm(N, log_obj, "found txt pattern for %s: %s, total: %s", - ext, #positions, res[ext]) + ext, weight * #positions, res[ext]) end end diff --git a/lualib/lua_magic/types.lua b/lualib/lua_magic/types.lua index 2ee3e62d7..d15eec6e1 100644 --- a/lualib/lua_magic/types.lua +++ b/lualib/lua_magic/types.lua @@ -282,6 +282,16 @@ local types = { ct = 'text/csv', av_check = false, }, + ics = { + type = 'text', + ct = 'text/calendar', + av_check = false, + }, + vcf = { + type = 'text', + ct = 'text/vcard', + av_check = false, + }, eml = { type = 'message', ct = 'message/rfc822', diff --git a/lualib/lua_scanners/common.lua b/lualib/lua_scanners/common.lua index 117945dd4..f286d963a 100644 --- a/lualib/lua_scanners/common.lua +++ b/lualib/lua_scanners/common.lua @@ -104,9 +104,9 @@ local function yield_result(task, rule, vname, dyn_weight, is_fail) rspamd_logger.infox(task, '%s: "%s" is in whitelist', rule.log_prefix, tm) else all_whitelisted = false - task:insert_result(symname, symscore, tm) rspamd_logger.infox(task, '%s: result - %s: "%s - score: %s"', rule.log_prefix, threat_info, tm, symscore) + task:insert_result(symname, symscore, tm) end end @@ -116,7 +116,7 @@ local function yield_result(task, rule, vname, dyn_weight, is_fail) lua_util.template(rule.message or 'Rejected', { SCANNER = rule.name, VIRUS = threat_table, - }), rule.name) + }), rule.name, nil, nil, 'least') end end diff --git a/lualib/lua_scanners/p0f.lua b/lualib/lua_scanners/p0f.lua index 056c0ad8b..06953660d 100644 --- a/lualib/lua_scanners/p0f.lua +++ b/lualib/lua_scanners/p0f.lua @@ -99,8 +99,19 @@ local function p0f_check(task, ip, rule) task:get_mempool():set_variable('os_fingerprint', os_string, link_type, uptime_min, distance) - common.yield_result(task, rule, { - os_string, link_type, 'distance:' .. distance }, 0.0) + if link_type and #link_type > 0 then + common.yield_result(task, rule, { + os_string, + 'link=' .. link_type, + 'distance=' .. distance}, + 0.0) + else + common.yield_result(task, rule, { + os_string, + 'link=unknown', + 'distance=' .. distance}, + 0.0) + end return data end @@ -115,17 +126,24 @@ local function p0f_check(task, ip, rule) end end + if err then + rspamd_logger.errx(task, 'p0f received an error: %s', err) + common.yield_result(task, rule, 'Error getting result: ' .. err, + 0.0, 'fail') + return + end + data = parse_p0f_response(data) if rule.redis_params and data then local key = rule.prefix .. ip:to_string() local ret = lua_redis.redis_make_request(task, - rule.redis_params, - key, - true, - redis_set_cb, - 'SETEX', - { key, tostring(rule.expire), data } + rule.redis_params, + key, + true, + redis_set_cb, + 'SETEX', + { key, tostring(rule.expire), data } ) if not ret then diff --git a/lualib/lua_scanners/sophos.lua b/lualib/lua_scanners/sophos.lua index 9da931c4e..200466b93 100644 --- a/lualib/lua_scanners/sophos.lua +++ b/lualib/lua_scanners/sophos.lua @@ -141,8 +141,8 @@ local function sophos_check(task, content, digest, rule) conn:add_read(sophos_callback) elseif string.find(data, 'FAIL 0212') then rspamd_logger.warnx(task, 'Message is encrypted (FAIL 0212): %s', data) - common.yield_result(task, rule, 'SAVDI: Message is encrypted (FAIL 0212)', 0.0, 'fail') - cached = 'ENCRYPTED' + common.yield_result(task, rule, 'SAVDI: Message is encrypted (FAIL 0212)', 0.0, 'encrypted') + cached = 'encrypted' elseif string.find(data, 'REJ 4') then rspamd_logger.warnx(task, 'Message is oversized (REJ 4): %s', data) common.yield_result(task, rule, 'SAVDI: Message oversized (REJ 4)', 0.0, 'fail') diff --git a/lualib/lua_scanners/virustotal.lua b/lualib/lua_scanners/virustotal.lua index 8775100ba..6bbdf94fd 100644 --- a/lualib/lua_scanners/virustotal.lua +++ b/lualib/lua_scanners/virustotal.lua @@ -130,7 +130,7 @@ local function virustotal_check(task, content, digest, rule) if res then local obj = parser:get_object() - if not obj.positives then + if not obj.positives or type(obj.positives) ~= 'number' then if obj.response_code then if obj.response_code == 0 then cached = 'OK' @@ -153,35 +153,36 @@ local function virustotal_check(task, content, digest, rule) task:insert_result(rule.symbol_fail, 1.0, 'Bad JSON reply: no `positives` element') return end - end - if obj.positives < rule.minimum_engines then - lua_util.debugm(rule.name, task, '%s: hash %s has not enough hits: %s where %s is min', - rule.log_prefix, obj.positives, rule.minimum_engines) - -- TODO: add proper hashing! - cached = 'OK' else - if obj.positives > rule.full_score_engines then - dyn_score = 1.0 + if obj.positives < rule.minimum_engines then + lua_util.debugm(rule.name, task, '%s: hash %s has not enough hits: %s where %s is min', + rule.log_prefix, obj.positives, rule.minimum_engines) + -- TODO: add proper hashing! + cached = 'OK' else - local norm_pos = obj.positives - rule.minimum_engines - dyn_score = norm_pos / (rule.full_score_engines - rule.minimum_engines) - end + if obj.positives > rule.full_score_engines then + dyn_score = 1.0 + else + local norm_pos = obj.positives - rule.minimum_engines + dyn_score = norm_pos / (rule.full_score_engines - rule.minimum_engines) + end - if dyn_score < 0 or dyn_score > 1 then - dyn_score = 1.0 + if dyn_score < 0 or dyn_score > 1 then + dyn_score = 1.0 + end + local sopt = string.format("%s:%s/%s", + hash, obj.positives, obj.total) + common.yield_result(task, rule, sopt, dyn_score) + cached = sopt end - local sopt = string.format("%s:%s/%s", - hash, obj.positives, obj.total) - common.yield_result(task, rule, sopt, dyn_score) - cached = sopt end else + -- not res rspamd_logger.errx(task, 'invalid JSON reply: %s, body: %s, headers: %s', json_err, body, headers) task:insert_result(rule.symbol_fail, 1.0, 'Bad JSON reply: ' .. json_err) return end - end if cached then diff --git a/lualib/lua_selectors/transforms.lua b/lualib/lua_selectors/transforms.lua index ffc9acd00..b0c912deb 100644 --- a/lualib/lua_selectors/transforms.lua +++ b/lualib/lua_selectors/transforms.lua @@ -54,7 +54,7 @@ local transform_function = { ['list'] = true, }, ['process'] = function(inp, t) - return fun.nth(#inp, inp),pure_type(t) + return fun.nth(fun.length(inp), inp),pure_type(t) end, ['description'] = 'Returns the last element', }, diff --git a/lualib/lua_util.lua b/lualib/lua_util.lua index bda8b0c02..89a4016b2 100644 --- a/lualib/lua_util.lua +++ b/lualib/lua_util.lua @@ -423,6 +423,30 @@ end exports.list_to_hash = list_to_hash --[[[ +-- @function lua_util.nkeys(table|gen, param, state) +-- Returns number of keys in a table (i.e. from both the array and hash parts combined) +-- @param {table} list numerically-indexed table or string, which is treated as a one-element list +-- @return {number} number of keys +-- @example +-- print(lua_util.nkeys({})) -- 0 +-- print(lua_util.nkeys({ "a", nil, "b" })) -- 2 +-- print(lua_util.nkeys({ dog = 3, cat = 4, bird = nil })) -- 2 +-- print(lua_util.nkeys({ "a", dog = 3, cat = 4 })) -- 3 +-- +--]] +local function nkeys(gen, param, state) + local n = 0 + if not param then + for _,_ in pairs(gen) do n = n + 1 end + else + for _,_ in fun.iter(gen, param, state) do n = n + 1 end + end + return n +end + +exports.nkeys = nkeys + +--[[[ -- @function lua_util.parse_time_interval(str) -- Parses human readable time interval -- Accepts 's' for seconds, 'm' for minutes, 'h' for hours, 'd' for days, @@ -1269,4 +1293,62 @@ exports.toboolean = function(v) end end +---[[[ +-- @function lua_util.config_check_local_or_authed(config, modname) +-- Reads check_local and check_authed from the config as this is used in many modules +-- @param {rspamd_config} config `rspamd_config` global +-- @param {name} module name +-- @return {boolean} v converted to boolean +--]]] +exports.config_check_local_or_authed = function(rspamd_config, modname, def_local, def_authed) + local check_local = def_local or false + local check_authed = def_authed or false + + local function try_section(where) + local ret = false + local opts = rspamd_config:get_all_opt(where) + if type(opts) == 'table' then + if type(opts['check_local']) == 'boolean' then + check_local = opts['check_local'] + ret = true + end + if type(opts['check_authed']) == 'boolean' then + check_authed = opts['check_authed'] + ret = true + end + end + + return ret + end + + if not try_section(modname) then + try_section('options') + end + + return {check_local, check_authed} +end + +---[[[ +-- @function lua_util.is_skip_local_or_authed(task, conf[, ip]) +-- Returns `true` if local or authenticated task should be skipped for this module +-- @param {rspamd_task} task +-- @param {table} conf table returned from `config_check_local_or_authed` +-- @param {rspamd_ip} ip optional ip address (can be obtained from a task) +-- @return {boolean} true if check should be skipped +--]]] +exports.is_skip_local_or_authed = function(task, conf, ip) + if not ip then + ip = task:get_from_ip() + end + if not conf then + conf = {false, false} + end + if ((not conf[2] and task:get_user()) or + (not conf[1] and type(ip) == 'userdata' and ip:is_local())) then + return true + end + + return false +end + return exports diff --git a/lualib/rspamadm/dns_tool.lua b/lualib/rspamadm/dns_tool.lua index cd512ab9a..f45f4a4a3 100644 --- a/lualib/rspamadm/dns_tool.lua +++ b/lualib/rspamadm/dns_tool.lua @@ -17,8 +17,8 @@ limitations under the License. local argparse = require "argparse" local rspamd_logger = require "rspamd_logger" -local lua_util = require "lua_util" local ansicolors = require "ansicolors" +local bit = require "bit" local parser = argparse() :name "rspamadm dns_tool" @@ -84,14 +84,16 @@ local function load_config(opts) end local function spf_handler(opts) - local lua_spf = require("lua_ffi").spf + local rspamd_spf = require "rspamd_spf" local rspamd_task = require "rspamd_task" + local rspamd_ip = require "rspamd_ip" local task = rspamd_task:create(rspamd_config, rspamadm_ev_base) task:set_session(rspamadm_session) task:set_resolver(rspamadm_dns_resolver) if opts.ip then + opts.ip = rspamd_ip.fromstring(opts.ip) task:set_from_ip(opts.ip) end @@ -104,56 +106,90 @@ local function spf_handler(opts) os.exit(1) end + local function flag_to_str(fl) + if bit.band(fl, rspamd_spf.flags.temp_fail) ~= 0 then + return "temporary failure" + elseif bit.band(fl, rspamd_spf.flags.perm_fail) ~= 0 then + return "permanent failure" + elseif bit.band(fl, rspamd_spf.flags.na) ~= 0 then + return "no spf record" + end + + return "unknown flag: " .. tostring(fl) + end + local function display_spf_results(elt, colored) local dec = function(e) return e end + local policy_decode = function(e) + if e == rspamd_spf.policy.fail then + return 'reject' + elseif e == rspamd_spf.policy.pass then + return 'pass' + elseif e == rspamd_spf.policy.soft_fail then + return 'soft fail' + elseif e == rspamd_spf.policy.neutral then + return 'neutral' + end + + return 'unknown' + end if colored then dec = function(e) return highlight(e) end - if elt.res == 'pass' then + if elt.result == rspamd_spf.policy.pass then dec = function(e) return green(e) end - elseif elt.res == 'fail' then + elseif elt.result == rspamd_spf.policy.fail then dec = function(e) return red(e) end end end - printf('%s: %s', highlight('Result'), dec(elt.res)) - printf('%s: %s', highlight('Network'), dec(elt.ipnet)) + printf('%s: %s', highlight('Policy'), dec(policy_decode(elt.result))) + printf('%s: %s', highlight('Network'), dec(elt.addr)) - if elt.spf_str then - printf('%s: %s', highlight('Original'), elt.spf_str) + if elt.str then + printf('%s: %s', highlight('Original'), elt.str) end end - local function cb(success, res, matched) - if success then + local function cb(record, flags, err) + if record then + local result, flag_or_policy, error_or_addr + if opts.ip then + result, flag_or_policy, error_or_addr = record:check_ip(opts.ip) + end if opts.ip and not opts.all then - if matched then - display_spf_results(matched, true) + if result then + display_spf_results(error_or_addr, true) else - printf('Not matched') + printf('Not matched: %s', error_or_addr) end os.exit(0) end - printf('SPF record for %s; digest: %s', - highlight(opts.domain or opts.from), highlight(res.digest)) - for _,elt in ipairs(res.addrs) do - if lua_util.table_cmp(elt, matched) then - printf("%s", highlight('*** Matched ***')) - display_spf_results(elt, true) - printf('------') - else - display_spf_results(elt, false) - printf('------') + if result then + printf('SPF record for %s; digest: %s', + highlight(opts.domain or opts.from), highlight(record:get_digest())) + for _,elt in ipairs(record:get_elts()) do + if result and error_or_addr and elt.str and elt.str == error_or_addr.str then + printf("%s", highlight('*** Matched ***')) + display_spf_results(elt, true) + printf('------') + else + display_spf_results(elt, false) + printf('------') + end end + else + printf('Error getting SPF record: %s (%s flag)', err, + flag_to_str(flag_or_policy or flags)) end else - printf('Cannot get SPF record: %s', res) + printf('Cannot get SPF record: %s', err) end end - lua_spf.spf_resolve(task, cb) + rspamd_spf.resolve(task, cb) end local function handler(args) diff --git a/lualib/rspamadm/vault.lua b/lualib/rspamadm/vault.lua index 0dadaaeb4..d0b448a8d 100644 --- a/lualib/rspamadm/vault.lua +++ b/lualib/rspamadm/vault.lua @@ -290,6 +290,7 @@ local function create_and_push_key(opts, domain, existing) url = uri, method = 'put', headers = { + ['Content-Type'] = 'application/json', ['X-Vault-Token'] = opts.token }, body = { @@ -493,6 +494,7 @@ local function roll_handler(opts, domain) url = uri, method = 'put', headers = { + ['Content-Type'] = 'application/json', ['X-Vault-Token'] = opts.token }, body = { @@ -564,4 +566,4 @@ return { handler = handler, description = parser._description, name = 'vault' -}
\ No newline at end of file +} diff --git a/rules/content.lua b/rules/content.lua new file mode 100644 index 000000000..718fd22c1 --- /dev/null +++ b/rules/content.lua @@ -0,0 +1,88 @@ +--[[ +Copyright (c) 2019, Vsevolod Stakhov <vsevolod@highsecure.ru> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +local function process_pdf_specific(task, part, specific) + local suspicious_factor = 0 + if specific.encrypted then + task:insert_result('PDF_ENCRYPTED', 1.0, part:get_filename()) + suspicious_factor = suspicious_factor + 0.1 + if specific.openaction then + suspicious_factor = suspicious_factor + 0.5 + end + end + + if specific.javascript then + task:insert_result('PDF_JAVASCRIPT', 1.0, part:get_filename()) + suspicious_factor = suspicious_factor + 0.1 + if specific.openaction then + suspicious_factor = suspicious_factor + 0.5 + end + end + + if specific.suspicious then + suspicious_factor = suspicious_factor + 0.7 + end + + if suspicious_factor > 0.5 then + if suspicious_factor > 1.0 then suspicious_factor = 1.0 end + task:insert_result('PDF_SUSPICIOUS', suspicious_factor, part:get_filename()) + end +end + +local tags_processors = { + pdf = process_pdf_specific +} + +local function process_specific_cb(task) + local parts = task:get_parts() or {} + + for _,p in ipairs(parts) do + if p:is_specific() then + local data = p:get_specific() + + if data and type(data) == 'table' and data.tag then + if tags_processors[data.tag] then + tags_processors[data.tag](task, p, data) + end + end + end + end +end + +local id = rspamd_config:register_symbol{ + type = 'callback', + name = 'SPECIFIC_CONTENT_CHECK', + callback = process_specific_cb +} + +rspamd_config:register_symbol{ + type = 'virtual', + name = 'PDF_ENCRYPTED', + parent = id, + groups = {"content", "pdf"}, +} +rspamd_config:register_symbol{ + type = 'virtual', + name = 'PDF_JAVASCRIPT', + parent = id, + groups = {"content", "pdf"}, +} +rspamd_config:register_symbol{ + type = 'virtual', + name = 'PDF_SUSPICIOUS', + parent = id, + groups = {"content", "pdf"}, +} diff --git a/rules/misc.lua b/rules/misc.lua index 7d3682271..5dcf6ea05 100644 --- a/rules/misc.lua +++ b/rules/misc.lua @@ -685,12 +685,14 @@ local check_encrypted_name = rspamd_config:register_symbol{ local function check_part(part) if part:is_multipart() then local children = part:get_children() or {} + local text_kids = {} for _,cld in ipairs(children) do if cld:is_multipart() then check_part(cld) elseif cld:is_text() then seen_text = true + text_kids[#text_kids + 1] = cld else local type,subtype,_ = cld:get_type_full() @@ -712,6 +714,17 @@ local check_encrypted_name = rspamd_config:register_symbol{ end end end + if seen_text and seen_encrypted then + -- Ensure that our seen text is not really part of pgp #3205 + for _,tp in ipairs(text_kids) do + local t,_ = tp:get_type() + seen_text = false -- reset temporary + if t and t == 'text' then + seen_text = true + break + end + end + end end end end diff --git a/rules/regexp/headers.lua b/rules/regexp/headers.lua index 25f4b725a..b70284a2a 100644 --- a/rules/regexp/headers.lua +++ b/rules/regexp/headers.lua @@ -315,7 +315,8 @@ local hotmail_baydav_msgid = 'Message-Id=/^<?BAY\\d+-DAV\\d+[A-Z0-9]{25}\\@phx\\ -- Sympatico message id local sympatico_msgid = 'Message-Id=/^<?BAYC\\d+-PASMTP\\d+[A-Z0-9]{25}\\@CEZ\\.ICE>?$/H' -- Mailman message id -local mailman_msgid = 'Message-ID=/^<mailman\\.\\d+\\.\\d+\\.\\d+\\..+\\@\\S+>$/H' +-- https://bazaar.launchpad.net/~mailman-coders/mailman/2.1/view/head:/Mailman/Utils.py#L811 +local mailman_msgid = [[Message-ID=/^<mailman\.\d+\.\d+\.\d+\.[-+.:=\w]+@[-a-zA-Z\d.]+>$/H]] -- Message id seems to be forged local unusable_msgid = string.format('(%s | %s | %s | %s | %s | %s)', lyris_ezml_remailer, wacky_sendmail_version, iplanet_messaging_server, hotmail_baydav_msgid, sympatico_msgid, mailman_msgid) diff --git a/rules/rspamd.lua b/rules/rspamd.lua index e82eee4fa..8ce90b0d0 100644 --- a/rules/rspamd.lua +++ b/rules/rspamd.lua @@ -37,6 +37,7 @@ dofile(local_rules .. '/http_headers.lua') dofile(local_rules .. '/forwarding.lua') dofile(local_rules .. '/mid.lua') dofile(local_rules .. '/bitcoin.lua') +dofile(local_rules .. '/content.lua') if rspamd_util.file_exists(local_conf .. '/rspamd.local.lua') then dofile(local_conf .. '/rspamd.local.lua') diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1c03b1239..9a34d2ac4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -96,11 +96,10 @@ SET(RSPAMDSRC controller.c SET(PLUGINSSRC plugins/regexp.c plugins/chartable.c plugins/fuzzy_check.c - plugins/spf.c plugins/dkim_check.c libserver/rspamd_control.c) -SET(MODULES_LIST regexp chartable fuzzy_check spf dkim) +SET(MODULES_LIST regexp chartable fuzzy_check dkim) SET(WORKERS_LIST normal controller fuzzy rspamd_proxy) IF (ENABLE_HYPERSCAN MATCHES "ON") LIST(APPEND WORKERS_LIST "hs_helper") @@ -184,6 +183,7 @@ TARGET_LINK_LIBRARIES(rspamd-server rspamd-cdb) TARGET_LINK_LIBRARIES(rspamd-server rspamd-lpeg) TARGET_LINK_LIBRARIES(rspamd-server lcbtrie) TARGET_LINK_LIBRARIES(rspamd-server rspamd-zstd) +TARGET_LINK_LIBRARIES(rspamd-server rspamd-fastutf8) IF (ENABLE_CLANG_PLUGIN MATCHES "ON") ADD_DEPENDENCIES(rspamd-server rspamd-clang) @@ -206,7 +206,7 @@ IF (ENABLE_HYPERSCAN MATCHES "ON") TARGET_LINK_LIBRARIES(rspamd-server hs) ENDIF() -IF (WITH_BLAS) +IF(WITH_BLAS) TARGET_LINK_LIBRARIES(rspamd-server ${BLAS_REQUIRED_LIBRARIES}) ENDIF() diff --git a/src/controller.c b/src/controller.c index 7b6ecff4a..adbc2b848 100644 --- a/src/controller.c +++ b/src/controller.c @@ -38,8 +38,6 @@ /* 60 seconds for worker's IO */ #define DEFAULT_WORKER_IO_TIMEOUT 60000 -#define DEFAULT_STATS_PATH RSPAMD_DBDIR "/stats.ucl" - /* HTTP paths */ #define PATH_AUTH "/auth" #define PATH_SYMBOLS "/symbols" @@ -105,7 +103,6 @@ INIT_LOG_MODULE(controller) #define COLOR_REJECT "#CB4B4B" #define COLOR_TOTAL "#9440ED" -static const ev_tstamp rrd_update_time = 1.0; static const guint64 rspamd_controller_ctx_magic = 0xf72697805e6941faULL; extern void fuzzy_stat_command (struct rspamd_task *task); @@ -162,9 +159,6 @@ struct rspamd_controller_worker_ctx { /* Static files dir */ gchar *static_files_dir; - /* Saved statistics path */ - gchar *saved_stats_path; - /* Custom commands registered by plugins */ GHashTable *custom_commands; @@ -177,9 +171,7 @@ struct rspamd_controller_worker_ctx { /* Local keypair */ gpointer key; - ev_timer rrd_event; struct rspamd_rrd_file *rrd; - ev_timer save_stats_event; struct rspamd_lang_detector *lang_det; gdouble task_timeout; }; @@ -1522,7 +1514,7 @@ rspamd_controller_handle_lua_history (lua_State *L, if (lua_isfunction (L, -1)) { task = rspamd_task_new (session->ctx->worker, session->cfg, - session->pool, ctx->lang_det, ctx->event_loop); + session->pool, ctx->lang_det, ctx->event_loop, FALSE); task->resolver = ctx->resolver; task->s = rspamd_session_create (session->pool, @@ -1819,7 +1811,7 @@ rspamd_controller_handle_lua (struct rspamd_http_connection_entry *conn_ent, } task = rspamd_task_new (session->ctx->worker, session->cfg, session->pool, - ctx->lang_det, ctx->event_loop); + ctx->lang_det, ctx->event_loop, FALSE); task->resolver = ctx->resolver; task->s = rspamd_session_create (session->pool, @@ -2004,7 +1996,7 @@ rspamd_controller_handle_learn_common ( } task = rspamd_task_new (session->ctx->worker, session->cfg, session->pool, - session->ctx->lang_det, ctx->event_loop); + session->ctx->lang_det, ctx->event_loop, FALSE); task->resolver = ctx->resolver; task->s = rspamd_session_create (session->pool, @@ -2103,7 +2095,7 @@ rspamd_controller_handle_scan (struct rspamd_http_connection_entry *conn_ent, } task = rspamd_task_new (session->ctx->worker, session->cfg, session->pool, - ctx->lang_det, ctx->event_loop); + ctx->lang_det, ctx->event_loop, FALSE); task->resolver = ctx->resolver; task->s = rspamd_session_create (session->pool, @@ -2134,6 +2126,7 @@ rspamd_controller_handle_scan (struct rspamd_http_connection_entry *conn_ent, ev_timer_init (&task->timeout_ev, rspamd_task_timeout, ctx->task_timeout, ctx->task_timeout); ev_timer_start (task->event_loop, &task->timeout_ev); + ev_set_priority (&task->timeout_ev, EV_MAXPRI); } end: @@ -2592,7 +2585,7 @@ rspamd_controller_handle_stat_common ( ctx = session->ctx; task = rspamd_task_new (session->ctx->worker, session->cfg, session->pool, - ctx->lang_det, ctx->event_loop); + ctx->lang_det, ctx->event_loop, FALSE); task->resolver = ctx->resolver; cbdata = rspamd_mempool_alloc0 (session->pool, sizeof (*cbdata)); cbdata->conn_ent = conn_ent; @@ -2994,7 +2987,7 @@ rspamd_controller_handle_lua_plugin (struct rspamd_http_connection_entry *conn_e } task = rspamd_task_new (session->ctx->worker, session->cfg, session->pool, - ctx->lang_det, ctx->event_loop); + ctx->lang_det, ctx->event_loop, FALSE); task->resolver = ctx->resolver; task->s = rspamd_session_create (session->pool, @@ -3096,7 +3089,7 @@ rspamd_controller_accept_socket (EV_P_ ev_io *w, int revents) session = g_malloc0 (sizeof (struct rspamd_controller_session)); session->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), - "csession"); + "csession", 0); session->ctx = ctx; session->cfg = ctx->cfg; session->lang_det = ctx->lang_det; @@ -3110,181 +3103,6 @@ rspamd_controller_accept_socket (EV_P_ ev_io *w, int revents) } static void -rspamd_controller_rrd_update (EV_P_ ev_timer *w, int revents) -{ - struct rspamd_controller_worker_ctx *ctx = - (struct rspamd_controller_worker_ctx *)w->data; - struct rspamd_stat *stat; - GArray ar; - gdouble points[METRIC_ACTION_MAX]; - GError *err = NULL; - guint i; - - g_assert (ctx->rrd != NULL); - stat = ctx->srv->stat; - - for (i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i ++) { - points[i] = stat->actions_stat[i]; - } - - ar.data = (gchar *)points; - ar.len = sizeof (points); - - if (!rspamd_rrd_add_record (ctx->rrd, &ar, rspamd_get_calendar_ticks (), - &err)) { - msg_err_ctx ("cannot update rrd file: %e", err); - g_error_free (err); - } - - /* Plan new event */ - ev_timer_again (ctx->event_loop, &ctx->rrd_event); -} - -static void -rspamd_controller_load_saved_stats (struct rspamd_controller_worker_ctx *ctx) -{ - struct ucl_parser *parser; - ucl_object_t *obj; - const ucl_object_t *elt, *subelt; - struct rspamd_stat *stat, stat_copy; - gint i; - - g_assert (ctx->saved_stats_path != NULL); - - if (access (ctx->saved_stats_path, R_OK) == -1) { - msg_err_ctx ("cannot load controller stats from %s: %s", - ctx->saved_stats_path, strerror (errno)); - return; - } - - parser = ucl_parser_new (0); - - if (!ucl_parser_add_file (parser, ctx->saved_stats_path)) { - msg_err_ctx ("cannot parse controller stats from %s: %s", - ctx->saved_stats_path, ucl_parser_get_error (parser)); - ucl_parser_free (parser); - - return; - } - - obj = ucl_parser_get_object (parser); - ucl_parser_free (parser); - - stat = ctx->srv->stat; - memcpy (&stat_copy, stat, sizeof (stat_copy)); - - elt = ucl_object_lookup (obj, "scanned"); - - if (elt != NULL && ucl_object_type (elt) == UCL_INT) { - stat_copy.messages_scanned = ucl_object_toint (elt); - } - - elt = ucl_object_lookup (obj, "learned"); - - if (elt != NULL && ucl_object_type (elt) == UCL_INT) { - stat_copy.messages_learned = ucl_object_toint (elt); - } - - elt = ucl_object_lookup (obj, "actions"); - - if (elt != NULL) { - for (i = METRIC_ACTION_REJECT; i <= METRIC_ACTION_NOACTION; i++) { - subelt = ucl_object_lookup (elt, rspamd_action_to_str (i)); - - if (subelt && ucl_object_type (subelt) == UCL_INT) { - stat_copy.actions_stat[i] = ucl_object_toint (subelt); - } - } - } - - elt = ucl_object_lookup (obj, "connections_count"); - - if (elt != NULL && ucl_object_type (elt) == UCL_INT) { - stat_copy.connections_count = ucl_object_toint (elt); - } - - elt = ucl_object_lookup (obj, "control_connections_count"); - - if (elt != NULL && ucl_object_type (elt) == UCL_INT) { - stat_copy.control_connections_count = ucl_object_toint (elt); - } - - ucl_object_unref (obj); - memcpy (stat, &stat_copy, sizeof (stat_copy)); -} - -static void -rspamd_controller_store_saved_stats (struct rspamd_controller_worker_ctx *ctx) -{ - struct rspamd_stat *stat; - ucl_object_t *top, *sub; - struct ucl_emitter_functions *efuncs; - gint i, fd; - - g_assert (ctx->saved_stats_path != NULL); - - fd = open (ctx->saved_stats_path, O_WRONLY|O_CREAT|O_TRUNC, 00644); - - if (fd == -1) { - msg_err_ctx ("cannot open for writing controller stats from %s: %s", - ctx->saved_stats_path, strerror (errno)); - return; - } - - if (rspamd_file_lock (fd, FALSE) == -1) { - msg_err_ctx ("cannot lock controller stats in %s: %s", - ctx->saved_stats_path, strerror (errno)); - close (fd); - - return; - } - - stat = ctx->srv->stat; - - top = ucl_object_typed_new (UCL_OBJECT); - ucl_object_insert_key (top, ucl_object_fromint ( - stat->messages_scanned), "scanned", 0, false); - ucl_object_insert_key (top, ucl_object_fromint ( - stat->messages_learned), "learned", 0, false); - - if (stat->messages_scanned > 0) { - sub = ucl_object_typed_new (UCL_OBJECT); - for (i = METRIC_ACTION_REJECT; i <= METRIC_ACTION_NOACTION; i++) { - ucl_object_insert_key (sub, - ucl_object_fromint (stat->actions_stat[i]), - rspamd_action_to_str (i), 0, false); - } - ucl_object_insert_key (top, sub, "actions", 0, false); - } - - ucl_object_insert_key (top, - ucl_object_fromint (stat->connections_count), "connections", 0, false); - ucl_object_insert_key (top, - ucl_object_fromint (stat->control_connections_count), - "control_connections", 0, false); - - - efuncs = ucl_object_emit_fd_funcs (fd); - ucl_object_emit_full (top, UCL_EMIT_JSON_COMPACT, - efuncs, NULL); - - ucl_object_unref (top); - rspamd_file_unlock (fd, FALSE); - close (fd); - ucl_object_emit_funcs_free (efuncs); -} - -static void -rspamd_controller_stats_save_periodic (EV_P_ ev_timer *w, int revents) -{ - struct rspamd_controller_worker_ctx *ctx = - (struct rspamd_controller_worker_ctx *)w->data; - - rspamd_controller_store_saved_stats (ctx); - ev_timer_again (EV_A_ w); -} - -static void rspamd_controller_password_sane (struct rspamd_controller_worker_ctx *ctx, const gchar *password, const gchar *type) { @@ -3417,16 +3235,6 @@ init_controller_worker (struct rspamd_config *cfg) rspamd_rcl_register_worker_option (cfg, type, - "stats_path", - rspamd_rcl_parse_struct_string, - ctx, - G_STRUCT_OFFSET (struct rspamd_controller_worker_ctx, - saved_stats_path), - 0, - "Directory where controller saves server's statistics between restarts"); - - rspamd_rcl_register_worker_option (cfg, - type, "task_timeout", rspamd_rcl_parse_struct_time, ctx, @@ -3564,20 +3372,6 @@ lua_csession_send_string (lua_State *L) } static void -rspamd_controller_on_terminate (struct rspamd_worker *worker) -{ - struct rspamd_controller_worker_ctx *ctx = worker->ctx; - - rspamd_controller_store_saved_stats (ctx); - - if (ctx->rrd) { - msg_info ("closing rrd file: %s", ctx->rrd->filename); - ev_timer_stop (ctx->event_loop, &ctx->rrd_event); - rspamd_rrd_close (ctx->rrd); - } -} - -static void rspamd_plugin_cbdata_dtor (gpointer p) { struct rspamd_controller_plugin_cbdata *cbd = p; @@ -3691,7 +3485,6 @@ start_controller_worker (struct rspamd_worker *worker) GHashTableIter iter; gpointer key, value; guint i; - const ev_tstamp save_stats_interval = 60; /* 1 minute */ gpointer m; g_assert (rspamd_worker_check_context (worker->ctx, rspamd_controller_ctx_magic)); @@ -3721,43 +3514,13 @@ start_controller_worker (struct rspamd_worker *worker) if (ctx->secure_ip != NULL) { rspamd_config_radix_from_ucl (ctx->cfg, ctx->secure_ip, "Allow unauthenticated requests from these addresses", - &ctx->secure_map, NULL); - } - - if (ctx->saved_stats_path == NULL) { - /* Assume default path */ - ctx->saved_stats_path = rspamd_mempool_strdup (worker->srv->cfg->cfg_pool, - DEFAULT_STATS_PATH); + &ctx->secure_map, + NULL, + worker); } - rspamd_controller_load_saved_stats (ctx); ctx->lang_det = ctx->cfg->lang_det; - /* RRD collector */ - if (ctx->cfg->rrd_file && worker->index == 0) { - GError *rrd_err = NULL; - - ctx->rrd = rspamd_rrd_file_default (ctx->cfg->rrd_file, &rrd_err); - - if (ctx->rrd) { - ctx->rrd_event.data = ctx; - ev_timer_init (&ctx->rrd_event, rspamd_controller_rrd_update, - rrd_update_time, rrd_update_time); - ev_timer_start (ctx->event_loop, &ctx->rrd_event); - } - else if (rrd_err) { - msg_err ("cannot load rrd from %s: %e", ctx->cfg->rrd_file, - rrd_err); - g_error_free (rrd_err); - } - else { - msg_err ("cannot load rrd from %s: unknown error", ctx->cfg->rrd_file); - } - } - else { - ctx->rrd = NULL; - } - rspamd_controller_password_sane (ctx, ctx->password, "normal password"); rspamd_controller_password_sane (ctx, ctx->enable_password, "enable " "password"); @@ -3890,27 +3653,7 @@ start_controller_worker (struct rspamd_worker *worker) rspamd_symcache_start_refresh (worker->srv->cfg->cache, ctx->event_loop, worker); rspamd_stat_init (worker->srv->cfg, ctx->event_loop); - - if (worker->index == 0) { - if (!ctx->cfg->disable_monitored) { - rspamd_worker_init_monitored (worker, ctx->event_loop, ctx->resolver); - } - - rspamd_map_watch (worker->srv->cfg, ctx->event_loop, - ctx->resolver, worker, TRUE); - - /* Schedule periodic stats saving, see #1823 */ - ctx->save_stats_event.data = ctx; - ev_timer_init (&ctx->save_stats_event, - rspamd_controller_stats_save_periodic, - save_stats_interval, save_stats_interval); - ev_timer_start (ctx->event_loop, &ctx->save_stats_event); - } - else { - rspamd_map_watch (worker->srv->cfg, ctx->event_loop, - ctx->resolver, worker, FALSE); - } - + rspamd_worker_init_controller (worker, &ctx->rrd); rspamd_lua_run_postloads (ctx->cfg->lua_state, ctx->cfg, ctx->event_loop, worker); #ifdef WITH_HYPERSCAN @@ -3923,7 +3666,7 @@ start_controller_worker (struct rspamd_worker *worker) /* Start event loop */ ev_loop (ctx->event_loop, 0); rspamd_worker_block_signals (); - rspamd_controller_on_terminate (worker); + rspamd_controller_on_terminate (worker, ctx->rrd); rspamd_stat_close (); rspamd_http_router_free (ctx->http); diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c index f7aec3e27..bc13b7049 100644 --- a/src/fuzzy_storage.c +++ b/src/fuzzy_storage.c @@ -112,6 +112,7 @@ struct fuzzy_key_stat { guint64 deleted; guint64 errors; rspamd_lru_hash_t *last_ips; + ref_entry_t ref; }; struct rspamd_fuzzy_mirror { @@ -230,10 +231,14 @@ struct rspamd_updates_cbdata { GArray *updates_pending; struct rspamd_fuzzy_storage_ctx *ctx; gchar *source; + gboolean final; }; static void rspamd_fuzzy_write_reply (struct fuzzy_session *session); +static gboolean rspamd_fuzzy_process_updates_queue ( + struct rspamd_fuzzy_storage_ctx *ctx, + const gchar *source, gboolean final); static gboolean rspamd_fuzzy_check_ratelimit (struct fuzzy_session *session) @@ -376,6 +381,16 @@ fuzzy_key_stat_dtor (gpointer p) if (st->last_ips) { rspamd_lru_hash_destroy (st->last_ips); } + + g_free (st); +} + +static void +fuzzy_key_stat_unref (gpointer p) +{ + struct fuzzy_key_stat *st = p; + + REF_RELEASE (st); } static void @@ -383,8 +398,12 @@ fuzzy_key_dtor (gpointer p) { struct fuzzy_key *key = p; - if (key->stat) { - fuzzy_key_stat_dtor (key->stat); + if (key) { + if (key->stat) { + REF_RELEASE (key->stat); + } + + g_free (key); } } @@ -458,6 +477,11 @@ rspamd_fuzzy_updates_cb (gboolean success, rspamd_fuzzy_backend_version (ctx->backend, source, fuzzy_update_version_callback, g_strdup (source)); ctx->updates_failed = 0; + + if (cbdata->final || ctx->worker->state != rspamd_worker_state_running) { + /* Plan exit */ + ev_break (ctx->event_loop, EVBREAK_ALL); + } } else { if (++ctx->updates_failed > ctx->updates_maxfail) { @@ -466,6 +490,11 @@ rspamd_fuzzy_updates_cb (gboolean success, cbdata->updates_pending->len, ctx->updates_maxfail); ctx->updates_failed = 0; + + if (cbdata->final || ctx->worker->state != rspamd_worker_state_running) { + /* Plan exit */ + ev_break (ctx->event_loop, EVBREAK_ALL); + } } else { msg_err ("cannot commit update transaction to fuzzy backend, " @@ -478,12 +507,14 @@ rspamd_fuzzy_updates_cb (gboolean success, g_array_append_vals (ctx->updates_pending, cbdata->updates_pending->data, cbdata->updates_pending->len); - } - } - if (ctx->worker->wanna_die) { - /* Plan exit */ - ev_break (ctx->event_loop, EVBREAK_ALL); + if (cbdata->final) { + /* Try one more time */ + rspamd_fuzzy_process_updates_queue (cbdata->ctx, cbdata->source, + cbdata->final); + + } + } } g_array_free (cbdata->updates_pending, TRUE); @@ -491,16 +522,17 @@ rspamd_fuzzy_updates_cb (gboolean success, g_free (cbdata); } -static void +static gboolean rspamd_fuzzy_process_updates_queue (struct rspamd_fuzzy_storage_ctx *ctx, - const gchar *source, gboolean forced) + const gchar *source, gboolean final) { struct rspamd_updates_cbdata *cbdata; - if ((forced ||ctx->updates_pending->len > 0)) { + if (ctx->updates_pending->len > 0) { cbdata = g_malloc (sizeof (*cbdata)); cbdata->ctx = ctx; + cbdata->final = final; cbdata->updates_pending = ctx->updates_pending; ctx->updates_pending = g_array_sized_new (FALSE, FALSE, sizeof (struct fuzzy_peer_cmd), @@ -509,7 +541,14 @@ rspamd_fuzzy_process_updates_queue (struct rspamd_fuzzy_storage_ctx *ctx, rspamd_fuzzy_backend_process_updates (ctx->backend, cbdata->updates_pending, source, rspamd_fuzzy_updates_cb, cbdata); + return TRUE; } + else if (final) { + /* No need to sync */ + ev_break (ctx->event_loop, EVBREAK_ALL); + } + + return FALSE; } static void @@ -862,10 +901,12 @@ rspamd_fuzzy_process_command (struct fuzzy_session *session) if (ip_stat == NULL) { naddr = rspamd_inet_address_copy (session->addr); ip_stat = g_malloc0 (sizeof (*ip_stat)); + REF_INIT_RETAIN (ip_stat, fuzzy_key_stat_dtor); rspamd_lru_hash_insert (session->key_stat->last_ips, naddr, ip_stat, -1, 0); } + REF_RETAIN (ip_stat); session->ip_stat = ip_stat; } @@ -1145,6 +1186,11 @@ fuzzy_session_destroy (gpointer d) rspamd_inet_address_free (session->addr); rspamd_explicit_memzero (session->nm, sizeof (session->nm)); session->worker->nconns--; + + if (session->ip_stat) { + REF_RELEASE (session->ip_stat); + } + g_free (session); } @@ -1165,8 +1211,6 @@ accept_fuzzy_socket (EV_P_ ev_io *w, int revents) if (revents == EV_READ) { for (;;) { - worker->nconns++; - r = rspamd_inet_address_recvfrom (w->fd, buf, sizeof (buf), @@ -1195,6 +1239,7 @@ accept_fuzzy_socket (EV_P_ ev_io *w, int revents) session->ctx = worker->ctx; session->time = (guint64) time (NULL); session->addr = addr; + worker->nconns++; if (rspamd_fuzzy_cmd_from_wire (buf, r, session)) { /* Check shingles count sanity */ @@ -1571,12 +1616,14 @@ fuzzy_parse_keypair (rspamd_mempool_t *pool, return FALSE; } - key = rspamd_mempool_alloc0 (pool, sizeof (*key)); + key = g_malloc0 (sizeof (*key)); key->key = kp; - keystat = rspamd_mempool_alloc0 (pool, sizeof (*keystat)); + keystat = g_malloc0 (sizeof (*keystat)); + REF_INIT_RETAIN (keystat, fuzzy_key_stat_dtor); /* Hash of ip -> fuzzy_key_stat */ keystat->last_ips = rspamd_lru_hash_new_full (1024, - (GDestroyNotify) rspamd_inet_address_free, fuzzy_key_stat_dtor, + (GDestroyNotify) rspamd_inet_address_free, + fuzzy_key_stat_unref, rspamd_inet_address_hash, rspamd_inet_address_equal); key->stat = keystat; pk = rspamd_keypair_component (kp, RSPAMD_KEYPAIR_COMPONENT_PK, @@ -1952,7 +1999,7 @@ start_fuzzy (struct rspamd_worker *worker) if (ctx->update_map != NULL) { rspamd_config_radix_from_ucl (worker->srv->cfg, ctx->update_map, "Allow fuzzy updates from specified addresses", - &ctx->update_ips, NULL); + &ctx->update_ips, NULL, worker); } if (ctx->skip_map != NULL) { @@ -1963,7 +2010,8 @@ start_fuzzy (struct rspamd_worker *worker) rspamd_kv_list_read, rspamd_kv_list_fin, rspamd_kv_list_dtor, - (void **)&ctx->skip_hashes)) == NULL) { + (void **)&ctx->skip_hashes, + worker)) == NULL) { msg_warn_config ("cannot load hashes list from %s", ucl_object_tostring (ctx->skip_map)); } @@ -1975,14 +2023,18 @@ start_fuzzy (struct rspamd_worker *worker) if (ctx->blocked_map != NULL) { rspamd_config_radix_from_ucl (worker->srv->cfg, ctx->blocked_map, "Block fuzzy requests from the specific IPs", - &ctx->blocked_ips, NULL); + &ctx->blocked_ips, + NULL, + worker); } /* Create radix trees */ if (ctx->ratelimit_whitelist_map != NULL) { rspamd_config_radix_from_ucl (worker->srv->cfg, ctx->ratelimit_whitelist_map, "Skip ratelimits from specific ip addresses/networks", - &ctx->ratelimit_whitelist, NULL); + &ctx->ratelimit_whitelist, + NULL, + worker); } /* Ratelimits */ @@ -1996,7 +2048,8 @@ start_fuzzy (struct rspamd_worker *worker) ctx->resolver = rspamd_dns_resolver_init (worker->srv->logger, ctx->event_loop, worker->srv->cfg); - rspamd_map_watch (worker->srv->cfg, ctx->event_loop, ctx->resolver, worker, 0); + rspamd_map_watch (worker->srv->cfg, ctx->event_loop, + ctx->resolver, worker, RSPAMD_MAP_WATCH_WORKER); /* Get peer pipe */ memset (&srv_cmd, 0, sizeof (srv_cmd)); @@ -2020,8 +2073,16 @@ start_fuzzy (struct rspamd_worker *worker) } if (worker->index == 0 && ctx->updates_pending->len > 0) { - rspamd_fuzzy_process_updates_queue (ctx, local_db_name, FALSE); - ev_loop (ctx->event_loop, 0); + + msg_info_config ("start another event loop to sync fuzzy storage"); + + if (rspamd_fuzzy_process_updates_queue (ctx, local_db_name, TRUE)) { + ev_loop (ctx->event_loop, 0); + msg_info_config ("sync cycle is done"); + } + else { + msg_info_config ("no need to sync"); + } } rspamd_fuzzy_backend_close (ctx->backend); @@ -2034,6 +2095,10 @@ start_fuzzy (struct rspamd_worker *worker) rspamd_keypair_cache_destroy (ctx->keypair_cache); } + if (ctx->ratelimit_buckets) { + rspamd_lru_hash_destroy (ctx->ratelimit_buckets); + } + REF_RELEASE (ctx->cfg); rspamd_log_close (worker->srv->logger, TRUE); diff --git a/src/libcryptobox/CMakeLists.txt b/src/libcryptobox/CMakeLists.txt index 8ab390fa1..272701b53 100644 --- a/src/libcryptobox/CMakeLists.txt +++ b/src/libcryptobox/CMakeLists.txt @@ -1,118 +1,25 @@ -INCLUDE(AsmOp.cmake) - -TARGET_ARCHITECTURE(ARCH) - SET(CHACHASRC ${CMAKE_CURRENT_SOURCE_DIR}/chacha20/chacha.c ${CMAKE_CURRENT_SOURCE_DIR}/chacha20/ref.c) SET(BASE64SRC ${CMAKE_CURRENT_SOURCE_DIR}/base64/ref.c ${CMAKE_CURRENT_SOURCE_DIR}/base64/base64.c) -SET(ASM_CODE " - .macro TEST1 op - \\op %eax, %eax - .endm - TEST1 xorl - ") -ASM_OP(HAVE_SLASHMACRO "slash macro convention") - -SET(ASM_CODE " - .macro TEST1 op - $0 %eax, %eax - .endm - TEST1 xorl - ") -ASM_OP(HAVE_DOLLARMACRO "dollar macro convention") - -# For now we support only x86_64 architecture with optimizations -IF("${ARCH}" STREQUAL "x86_64") - IF(NOT HAVE_SLASHMACRO AND NOT HAVE_DOLLARMACRO) - MESSAGE(FATAL_ERROR "Your assembler cannot compile macros, please check your CMakeFiles/CMakeError.log") - ENDIF() - - SET(ASM_CODE "vpaddq %ymm0, %ymm0, %ymm0") - ASM_OP(HAVE_AVX2 "avx2") - # Handle broken compilers, sigh... - IF(HAVE_AVX2) - CHECK_C_SOURCE_COMPILES( -" -#include <stddef.h> -#pragma GCC push_options -#pragma GCC target(\"avx2\") -#ifndef __SSE2__ -#define __SSE2__ -#endif -#ifndef __SSE__ -#define __SSE__ -#endif -#ifndef __SSE4_2__ -#define __SSE4_2__ -#endif -#ifndef __SSE4_1__ -#define __SSE4_1__ -#endif -#ifndef __SSEE3__ -#define __SSEE3__ -#endif -#ifndef __AVX__ -#define __AVX__ -#endif -#ifndef __AVX2__ -#define __AVX2__ -#endif - -#ifndef __clang__ -#if __GNUC__ < 6 -#error Broken due to compiler bug -#endif -#endif - -#include <immintrin.h> -static void foo(const char* a) __attribute__((__target__(\"avx2\"))); -static void foo(const char* a) -{ - __m256i str = _mm256_loadu_si256((__m256i *)a); - __m256i t = _mm256_loadu_si256((__m256i *)a + 1); - _mm256_add_epi8(str, t); -} -int main(int argc, char** argv) { - foo(argv[0]); -}" HAVE_AVX2_C_COMPILER) - IF(NOT HAVE_AVX2_C_COMPILER) - MESSAGE(STATUS "Your compiler has broken AVX2 support") - UNSET(HAVE_AVX2 CACHE) - ENDIF() - ENDIF() - SET(ASM_CODE "vpaddq %xmm0, %xmm0, %xmm0") - ASM_OP(HAVE_AVX "avx") - SET(ASM_CODE "pmuludq %xmm0, %xmm0") - ASM_OP(HAVE_SSE2 "sse2") - SET(ASM_CODE "lddqu 0(%esi), %xmm0") - ASM_OP(HAVE_SSE3 "sse3") - SET(ASM_CODE "pshufb %xmm0, %xmm0") - ASM_OP(HAVE_SSSE3 "ssse3") - SET(ASM_CODE "pblendw \$0, %xmm0, %xmm0") - ASM_OP(HAVE_SSE41 "sse41") - SET(ASM_CODE "crc32 %eax, %eax") - ASM_OP(HAVE_SSE42 "sse42") -ENDIF() - IF(HAVE_AVX2) SET(CHACHASRC ${CHACHASRC} ${CMAKE_CURRENT_SOURCE_DIR}/chacha20/avx2.S) SET(BASE64SRC ${BASE64SRC} ${CMAKE_CURRENT_SOURCE_DIR}/base64/avx2.c) - MESSAGE(STATUS "AVX2 support is added") + MESSAGE(STATUS "Cryptobox: AVX2 support is added (chacha20, avx2)") ENDIF(HAVE_AVX2) IF(HAVE_AVX) SET(CHACHASRC ${CHACHASRC} ${CMAKE_CURRENT_SOURCE_DIR}/chacha20/avx.S) - MESSAGE(STATUS "AVX support is added") + MESSAGE(STATUS "Cryptobox: AVX support is added (chacha20)") ENDIF(HAVE_AVX) IF(HAVE_SSE2) SET(CHACHASRC ${CHACHASRC} ${CMAKE_CURRENT_SOURCE_DIR}/chacha20/sse2.S) - MESSAGE(STATUS "SSE2 support is added") + MESSAGE(STATUS "Cryptobox: SSE2 support is added (chacha20)") ENDIF(HAVE_SSE2) IF(HAVE_SSE42) SET(BASE64SRC ${BASE64SRC} ${CMAKE_CURRENT_SOURCE_DIR}/base64/sse42.c) - MESSAGE(STATUS "SSE42 support is added") + MESSAGE(STATUS "Cryptobox: SSE42 support is added (base64)") ENDIF(HAVE_SSE42) CONFIGURE_FILE(platform_config.h.in platform_config.h) diff --git a/src/libcryptobox/base64/avx2.c b/src/libcryptobox/base64/avx2.c index 80f3b9972..432149a29 100644 --- a/src/libcryptobox/base64/avx2.c +++ b/src/libcryptobox/base64/avx2.c @@ -144,6 +144,7 @@ dec_reshuffle (__m256i in) const __m256i eq_2F = _mm256_cmpeq_epi8(str, mask_2F); \ const __m256i roll = _mm256_shuffle_epi8(lut_roll, _mm256_add_epi8(eq_2F, hi_nibbles)); \ if (!_mm256_testz_si256(lo, hi)) { \ + seen_error = true; \ break; \ } \ str = _mm256_add_epi8(str, roll); \ @@ -168,12 +169,15 @@ base64_decode_avx2 (const char *in, size_t inlen, uint8_t q, carry; size_t outl = 0; size_t leftover = 0; + bool seen_error = false; repeat: switch (leftover) { for (;;) { case 0: - INNER_LOOP_AVX2 + if (G_LIKELY (!seen_error)) { + INNER_LOOP_AVX2 + } if (inlen-- == 0) { ret = 1; @@ -267,6 +271,7 @@ repeat: } if (inlen > 0) { + seen_error = false; goto repeat; } } diff --git a/src/libcryptobox/base64/base64.c b/src/libcryptobox/base64/base64.c index 03ca99786..efa356252 100644 --- a/src/libcryptobox/base64/base64.c +++ b/src/libcryptobox/base64/base64.c @@ -19,9 +19,10 @@ #include "base64.h" #include "platform_config.h" #include "str_util.h" +#include "util.h" #include "contrib/libottery/ottery.h" -extern unsigned long cpu_config; +extern unsigned cpu_config; const uint8_t base64_table_dec[256] = { @@ -46,20 +47,21 @@ base64_table_dec[256] = static const char base64_alphabet[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz+/"; typedef struct base64_impl { - unsigned long cpu_flags; + unsigned short enabled; + unsigned short min_len; + unsigned int cpu_flags; const char *desc; - int (*decode) (const char *in, size_t inlen, unsigned char *out, size_t *outlen); } base64_impl_t; #define BASE64_DECLARE(ext) \ int base64_decode_##ext(const char *in, size_t inlen, unsigned char *out, size_t *outlen); -#define BASE64_IMPL(cpuflags, desc, ext) \ - {(cpuflags), desc, base64_decode_##ext} +#define BASE64_IMPL(cpuflags, min_len, desc, ext) \ + {0, (min_len), (cpuflags), desc, base64_decode_##ext} BASE64_DECLARE(ref); -#define BASE64_REF BASE64_IMPL(0, "ref", ref) +#define BASE64_REF BASE64_IMPL(0, 0, "ref", ref) #ifdef RSPAMD_HAS_TARGET_ATTR # if defined(HAVE_SSE42) @@ -67,7 +69,7 @@ int base64_decode_sse42 (const char *in, size_t inlen, unsigned char *out, size_t *outlen) __attribute__((__target__("sse4.2"))); BASE64_DECLARE(sse42); -# define BASE64_SSE42 BASE64_IMPL(CPUID_SSE42, "sse42", sse42) +# define BASE64_SSE42 BASE64_IMPL(CPUID_SSE42, 24, "sse42", sse42) # endif #endif @@ -77,74 +79,66 @@ int base64_decode_avx2 (const char *in, size_t inlen, unsigned char *out, size_t *outlen) __attribute__((__target__("avx2"))); BASE64_DECLARE(avx2); -# define BASE64_AVX2 BASE64_IMPL(CPUID_AVX2, "avx2", avx2) +# define BASE64_AVX2 BASE64_IMPL(CPUID_AVX2, 128, "avx2", avx2) # endif #endif -static const base64_impl_t base64_list[] = { +static base64_impl_t base64_list[] = { BASE64_REF, -#ifdef BASE64_AVX2 - BASE64_AVX2, -#endif #ifdef BASE64_SSE42 BASE64_SSE42, #endif +#ifdef BASE64_AVX2 + BASE64_AVX2, +#endif }; -static const base64_impl_t *base64_opt = &base64_list[0]; static const base64_impl_t *base64_ref = &base64_list[0]; const char * base64_load (void) { guint i; + const base64_impl_t *opt_impl = base64_ref; + + /* Enable reference */ + base64_list[0].enabled = true; if (cpu_config != 0) { - for (i = 0; i < G_N_ELEMENTS (base64_list); i++) { + for (i = 1; i < G_N_ELEMENTS (base64_list); i++) { if (base64_list[i].cpu_flags & cpu_config) { - base64_opt = &base64_list[i]; - break; + base64_list[i].enabled = true; + opt_impl = &base64_list[i]; } } } - return base64_opt->desc; + return opt_impl->desc; } gboolean rspamd_cryptobox_base64_decode (const gchar *in, gsize inlen, guchar *out, gsize *outlen) { - if (inlen > 256) { - /* - * For SIMD base64 decoding we need really large inputs with no - * garbadge such as newlines - * Otherwise, naive version is MUCH faster - */ - - if (rspamd_memcspn (in, base64_alphabet, 256) == 256) { - return base64_opt->decode (in, inlen, out, outlen); - } - else { - /* Garbage found */ - return base64_ref->decode (in, inlen, out, outlen); + const base64_impl_t *opt_impl = base64_ref; + + for (gint i = G_N_ELEMENTS (base64_list) - 1; i > 0; i --) { + if (base64_list[i].enabled && base64_list[i].min_len <= inlen) { + opt_impl = &base64_list[i]; + break; } } - else { - /* Small input, use reference version */ - return base64_ref->decode (in, inlen, out, outlen); - } - g_assert_not_reached (); + return opt_impl->decode (in, inlen, out, outlen); } -size_t -base64_test (bool generic, size_t niters, size_t len) +double +base64_test (bool generic, size_t niters, size_t len, size_t str_len) { size_t cycles; guchar *in, *out, *tmp; - const base64_impl_t *impl; + gdouble t1, t2, total = 0; gsize outlen; g_assert (len > 0); @@ -152,22 +146,35 @@ base64_test (bool generic, size_t niters, size_t len) tmp = g_malloc (len); ottery_rand_bytes (in, len); - impl = generic ? &base64_list[0] : base64_opt; + out = rspamd_encode_base64_fold (in, len, str_len, &outlen, + RSPAMD_TASK_NEWLINES_CRLF); - out = rspamd_encode_base64 (in, len, 0, &outlen); - impl->decode (out, outlen, tmp, &len); + if (generic) { + base64_list[0].decode (out, outlen, tmp, &len); + } + else { + rspamd_cryptobox_base64_decode (out, outlen, tmp, &len); + } g_assert (memcmp (in, tmp, len) == 0); for (cycles = 0; cycles < niters; cycles ++) { - impl->decode (out, outlen, in, &len); + t1 = rspamd_get_ticks (TRUE); + if (generic) { + base64_list[0].decode (out, outlen, tmp, &len); + } + else { + rspamd_cryptobox_base64_decode (out, outlen, tmp, &len); + } + t2 = rspamd_get_ticks (TRUE); + total += t2 - t1; } g_free (in); g_free (tmp); g_free (out); - return cycles; + return total; } diff --git a/src/libcryptobox/base64/sse42.c b/src/libcryptobox/base64/sse42.c index 1d1287ad2..806dd5298 100644 --- a/src/libcryptobox/base64/sse42.c +++ b/src/libcryptobox/base64/sse42.c @@ -118,6 +118,7 @@ static inline __m128i dec_reshuffle (__m128i in) 'A','Z', \ 'a','z'); \ if (_mm_cmpistrc(range, str, _SIDD_UBYTE_OPS | _SIDD_CMP_RANGES | _SIDD_NEGATIVE_POLARITY)) { \ + seen_error = true; \ break; \ } \ __m128i indices = _mm_subs_epu8(str, _mm_set1_epi8(46)); \ @@ -150,12 +151,15 @@ base64_decode_sse42 (const char *in, size_t inlen, uint8_t q, carry; size_t outl = 0; size_t leftover = 0; + bool seen_error = false; repeat: switch (leftover) { for (;;) { case 0: - INNER_LOOP_SSE42 + if (G_LIKELY (!seen_error)) { + INNER_LOOP_SSE42 + } if (inlen-- == 0) { ret = 1; @@ -249,6 +253,7 @@ repeat: } if (inlen > 0) { + seen_error = false; goto repeat; } } diff --git a/src/libcryptobox/chacha20/chacha.c b/src/libcryptobox/chacha20/chacha.c index fad83e823..e4543d3b8 100644 --- a/src/libcryptobox/chacha20/chacha.c +++ b/src/libcryptobox/chacha20/chacha.c @@ -27,7 +27,7 @@ #include "chacha.h" #include "platform_config.h" -extern unsigned long cpu_config; +extern unsigned cpu_config; typedef struct chacha_impl_t { unsigned long cpu_flags; diff --git a/src/libcryptobox/cryptobox.c b/src/libcryptobox/cryptobox.c index e4549096f..414dbdfa1 100644 --- a/src/libcryptobox/cryptobox.c +++ b/src/libcryptobox/cryptobox.c @@ -57,7 +57,7 @@ #include <sodium.h> -unsigned long cpu_config = 0; +unsigned cpu_config = 0; static gboolean cryptobox_loaded = FALSE; diff --git a/src/libmime/archives.c b/src/libmime/archives.c index 8c7e4ea90..32a251c94 100644 --- a/src/libmime/archives.c +++ b/src/libmime/archives.c @@ -67,7 +67,8 @@ rspamd_archive_file_try_utf (struct rspamd_task *task, struct rspamd_charset_converter *conv; UConverter *utf8_converter; - conv = rspamd_mime_get_converter_cached (charset, &uc_err); + conv = rspamd_mime_get_converter_cached (charset, task->task_pool, + FALSE, &uc_err); utf8_converter = rspamd_get_utf8_converter (); if (conv == NULL) { @@ -277,7 +278,7 @@ rspamd_archive_process_zip (struct rspamd_task *task, cd += fname_len + comment_len + extra_len + cd_basic_len; } - part->flags |= RSPAMD_MIME_PART_ARCHIVE; + part->part_type = RSPAMD_MIME_PART_ARCHIVE; part->specific.arch = arch; if (part->cd) { @@ -509,7 +510,7 @@ rspamd_archive_process_rar_v4 (struct rspamd_task *task, const guchar *start, } end: - part->flags |= RSPAMD_MIME_PART_ARCHIVE; + part->part_type = RSPAMD_MIME_PART_ARCHIVE; part->specific.arch = arch; arch->archive_name = &part->cd->filename; arch->size = part->parsed_data.len; @@ -733,7 +734,7 @@ rspamd_archive_process_rar (struct rspamd_task *task, } end: -part->flags |= RSPAMD_MIME_PART_ARCHIVE; + part->part_type = RSPAMD_MIME_PART_ARCHIVE; part->specific.arch = arch; if (part->cd != NULL) { arch->archive_name = &part->cd->filename; @@ -1673,7 +1674,7 @@ rspamd_archive_process_7zip (struct rspamd_task *task, while ((p = rspamd_7zip_read_next_section (task, p, end, arch)) != NULL); - part->flags |= RSPAMD_MIME_PART_ARCHIVE; + part->part_type = RSPAMD_MIME_PART_ARCHIVE; part->specific.arch = arch; if (part->cd != NULL) { arch->archive_name = &part->cd->filename; @@ -1823,7 +1824,7 @@ rspamd_archive_process_gzip (struct rspamd_task *task, set: /* Set archive data */ - part->flags |= RSPAMD_MIME_PART_ARCHIVE; + part->part_type = RSPAMD_MIME_PART_ARCHIVE; part->specific.arch = arch; if (part->cd) { @@ -1917,7 +1918,7 @@ rspamd_archives_process (struct rspamd_task *task) const guchar gz_magic[] = {0x1F, 0x8B}; PTR_ARRAY_FOREACH (MESSAGE_FIELD (task, parts), i, part) { - if (!(part->flags & (RSPAMD_MIME_PART_TEXT|RSPAMD_MIME_PART_IMAGE))) { + if (part->part_type == RSPAMD_MIME_PART_UNDEFINED) { if (part->parsed_data.len > 0) { if (rspamd_archive_cheat_detect (part, "zip", zip_magic, sizeof (zip_magic))) { @@ -1936,8 +1937,8 @@ rspamd_archives_process (struct rspamd_task *task) rspamd_archive_process_gzip (task, part); } - if (IS_CT_TEXT (part->ct) && - (part->flags & RSPAMD_MIME_PART_ARCHIVE) && + if (part->ct && (part->ct->flags & RSPAMD_CONTENT_TYPE_TEXT) && + part->part_type == RSPAMD_MIME_PART_ARCHIVE && part->specific.arch) { struct rspamd_archive *arch = part->specific.arch; diff --git a/src/libmime/content_type.h b/src/libmime/content_type.h index 2e3bf5e40..49bba4269 100644 --- a/src/libmime/content_type.h +++ b/src/libmime/content_type.h @@ -34,10 +34,6 @@ enum rspamd_content_type_flags { RSPAMD_CONTENT_TYPE_MISSING = 1 << 5, }; -#define IS_CT_MULTIPART(ct) ((ct) && ((ct)->flags & RSPAMD_CONTENT_TYPE_MULTIPART)) -#define IS_CT_TEXT(ct) ((ct) && ((ct)->flags & RSPAMD_CONTENT_TYPE_TEXT)) -#define IS_CT_MESSAGE(ct) ((ct) &&((ct)->flags & RSPAMD_CONTENT_TYPE_MESSAGE)) - enum rspamd_content_param_flags { RSPAMD_CONTENT_PARAM_NORMAL = 0, RSPAMD_CONTENT_PARAM_RFC2231 = (1 << 0), diff --git a/src/libmime/email_addr.c b/src/libmime/email_addr.c index 24b1d0111..4e09de6fb 100644 --- a/src/libmime/email_addr.c +++ b/src/libmime/email_addr.c @@ -110,6 +110,7 @@ rspamd_email_address_add (rspamd_mempool_t *pool, guint nlen; elt = g_malloc0 (sizeof (*elt)); + rspamd_mempool_notify_alloc (pool, sizeof (*elt)); if (addr != NULL) { memcpy (elt, addr, sizeof (*addr)); @@ -132,6 +133,7 @@ rspamd_email_address_add (rspamd_mempool_t *pool, /* We need to unquote addr */ nlen = elt->domain_len + elt->user_len + 2; elt->addr = g_malloc (nlen + 1); + rspamd_mempool_notify_alloc (pool, nlen + 1); elt->addr_len = rspamd_snprintf ((char *)elt->addr, nlen, "%*s@%*s", (gint)elt->user_len, elt->user, (gint)elt->domain_len, elt->domain); @@ -143,6 +145,7 @@ rspamd_email_address_add (rspamd_mempool_t *pool, elt->name = rspamd_mime_header_decode (pool, name->str, name->len, NULL); } + rspamd_mempool_notify_alloc (pool, name->len); g_ptr_array_add (ar, elt); } @@ -481,6 +484,7 @@ rspamd_email_address_from_mime (rspamd_mempool_t *pool, break; } + rspamd_mempool_notify_alloc (pool, cpy->len); g_string_free (ns, TRUE); return res; diff --git a/src/libmime/images.c b/src/libmime/images.c index faa7a6b2e..218e947fc 100644 --- a/src/libmime/images.c +++ b/src/libmime/images.c @@ -52,7 +52,7 @@ rspamd_images_process (struct rspamd_task *task) struct rspamd_mime_part *part; PTR_ARRAY_FOREACH (MESSAGE_FIELD (task, parts), i, part) { - if (!(part->flags & (RSPAMD_MIME_PART_TEXT|RSPAMD_MIME_PART_ARCHIVE))) { + if (part->part_type == RSPAMD_MIME_PART_UNDEFINED) { if (part->detected_type && strcmp (part->detected_type, "image") == 0 && part->parsed_data.len > 0) { @@ -610,7 +610,7 @@ process_image (struct rspamd_task *task, struct rspamd_mime_part *part) img->parent = part; - part->flags |= RSPAMD_MIME_PART_IMAGE; + part->part_type = RSPAMD_MIME_PART_IMAGE; part->specific.img = img; } } @@ -715,7 +715,7 @@ rspamd_images_link (struct rspamd_task *task) guint i; PTR_ARRAY_FOREACH (MESSAGE_FIELD (task, parts), i, part) { - if (part->flags & RSPAMD_MIME_PART_IMAGE) { + if (part->part_type == RSPAMD_MIME_PART_IMAGE) { rspamd_image_process_part (task, part); } } diff --git a/src/libmime/message.c b/src/libmime/message.c index f3aba6001..95a1ab708 100644 --- a/src/libmime/message.c +++ b/src/libmime/message.c @@ -489,6 +489,12 @@ rspamd_strip_newlines_parse (struct rspamd_task *task, } static void +rspamd_u_text_dtor (void *p) +{ + utext_close ((UText *)p); +} + +static void rspamd_normalize_text_part (struct rspamd_task *task, struct rspamd_mime_text_part *part) { @@ -535,7 +541,7 @@ rspamd_normalize_text_part (struct rspamd_task *task, } else { rspamd_mempool_add_destructor (task->task_pool, - (rspamd_mempool_destruct_t)utext_close, + rspamd_u_text_dtor, &part->utf_stripped_text); } } @@ -543,6 +549,8 @@ rspamd_normalize_text_part (struct rspamd_task *task, rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t) free_byte_array_callback, part->utf_stripped_content); + rspamd_mempool_notify_alloc (task->task_pool, + part->utf_stripped_content->len); rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t) rspamd_ptr_array_free_hard, part->newlines); @@ -688,71 +696,8 @@ rspamd_message_process_plain_text_part (struct rspamd_task *task, rspamd_mime_text_part_maybe_convert (task, text_part); if (text_part->utf_raw_content != NULL) { - /* Check for ical */ - rspamd_ftok_t cal_ct; - - /* - * TODO: If we want to process more than that, we need - * to create some generic framework that accepts a part - * and returns a processed data - */ - RSPAMD_FTOK_ASSIGN (&cal_ct, "calendar"); - - if (rspamd_ftok_casecmp (&cal_ct, &text_part->mime_part->ct->subtype) == 0) { - lua_State *L = task->cfg->lua_state; - gint err_idx; - - lua_pushcfunction (L, &rspamd_lua_traceback); - err_idx = lua_gettop (L); - - /* Obtain function */ - if (!rspamd_lua_require_function (L, "lua_ical", "ical_txt_values")) { - msg_err_task ("cannot require lua_ical.ical_txt_values"); - lua_settop (L, err_idx - 1); - - return FALSE; - } - - lua_pushlstring (L, text_part->utf_raw_content->data, - text_part->utf_raw_content->len); - - if (lua_pcall (L, 1, 1, err_idx) != 0) { - msg_err_task ("cannot call lua lua_ical.ical_txt_values: %s", - lua_tostring (L, -1)); - lua_settop (L, err_idx - 1); - - return FALSE; - } - - if (lua_type (L, -1) == LUA_TSTRING) { - const char *ndata; - gsize nsize; - - ndata = lua_tolstring (L, -1, &nsize); - text_part->utf_content = g_byte_array_sized_new (nsize); - g_byte_array_append (text_part->utf_content, ndata, nsize); - rspamd_mempool_add_destructor (task->task_pool, - (rspamd_mempool_destruct_t) free_byte_array_callback, - text_part->utf_content); - } - else if (lua_type (L, -1) == LUA_TNIL) { - msg_info_task ("cannot convert text/calendar to plain text"); - text_part->utf_content = text_part->utf_raw_content; - } - else { - msg_err_task ("invalid return type when calling lua_ical.ical_txt_values: %s", - lua_typename (L, lua_type (L, -1))); - lua_settop (L, err_idx - 1); - - return FALSE; - } - - lua_settop (L, err_idx - 1); - } - else { - /* Just have the same content */ - text_part->utf_content = text_part->utf_raw_content; - } + /* Just have the same content */ + text_part->utf_content = text_part->utf_raw_content; } else { /* @@ -815,9 +760,10 @@ rspamd_message_process_text_part_maybe (struct rspamd_task *task, gboolean found_html = FALSE, found_txt = FALSE, straight_ct = FALSE; enum rspamd_action_type act; - if ((IS_CT_TEXT (mime_part->ct) && (straight_ct = TRUE)) || - (mime_part->detected_type && - strcmp (mime_part->detected_type, "text") == 0)) { + if (((mime_part->ct && (mime_part->ct->flags & RSPAMD_CONTENT_TYPE_TEXT)) && + (straight_ct = TRUE)) || + (mime_part->detected_type && + strcmp (mime_part->detected_type, "text") == 0)) { found_txt = TRUE; html_tok.begin = "html"; @@ -866,7 +812,7 @@ rspamd_message_process_text_part_maybe (struct rspamd_task *task, } g_ptr_array_add (MESSAGE_FIELD (task, text_parts), text_part); - mime_part->flags |= RSPAMD_MIME_PART_TEXT; + mime_part->part_type = RSPAMD_MIME_PART_TEXT; mime_part->specific.txt = text_part; act = rspamd_check_gtube (task, text_part); @@ -1001,7 +947,7 @@ rspamd_message_from_data (struct rspamd_task *task, const guchar *start, } else { /* Check sanity */ - if (IS_CT_TEXT (part->ct)) { + if (part->ct && (part->ct->flags & RSPAMD_CONTENT_TYPE_TEXT)) { RSPAMD_FTOK_FROM_STR (&srch, "application"); if (rspamd_ftok_cmp (&ct->type, &srch) == 0) { @@ -1056,11 +1002,18 @@ rspamd_message_dtor (struct rspamd_message *msg) rspamd_message_headers_unref (p->raw_headers); } - if (IS_CT_MULTIPART (p->ct)) { + if (IS_PART_MULTIPART (p)) { if (p->specific.mp->children) { g_ptr_array_free (p->specific.mp->children, TRUE); } } + + if (p->part_type == RSPAMD_MIME_PART_CUSTOM_LUA && + p->specific.lua_specific.cbref != -1) { + luaL_unref (msg->task->cfg->lua_state, + LUA_REGISTRYINDEX, + p->specific.lua_specific.cbref); + } } PTR_ARRAY_FOREACH (msg->text_parts, i, tp) { @@ -1098,6 +1051,7 @@ rspamd_message_new (struct rspamd_task *task) msg->parts = g_ptr_array_sized_new (4); msg->text_parts = g_ptr_array_sized_new (2); + msg->task = task; REF_INIT_RETAIN (msg, rspamd_message_dtor); @@ -1363,7 +1317,7 @@ rspamd_message_process (struct rspamd_task *task) guint tw, *ptw, dw; struct rspamd_mime_part *part; lua_State *L = NULL; - gint func_pos = -1; + gint magic_func_pos = -1, content_func_pos = -1, old_top = -1, funcs_top = -1; if (task->cfg) { L = task->cfg->lua_state; @@ -1371,20 +1325,38 @@ rspamd_message_process (struct rspamd_task *task) rspamd_archives_process (task); + if (L) { + old_top = lua_gettop (L); + } + if (L && rspamd_lua_require_function (L, "lua_magic", "detect_mime_part")) { - func_pos = lua_gettop (L); + magic_func_pos = lua_gettop (L); } else { msg_err_task ("cannot require lua_magic.detect_mime_part"); } + if (L && rspamd_lua_require_function (L, + "lua_content", "maybe_process_mime_part")) { + content_func_pos = lua_gettop (L); + } + else { + msg_err_task ("cannot require lua_content.maybe_process_mime_part"); + } + + if (L) { + funcs_top = lua_gettop (L); + } + PTR_ARRAY_FOREACH (MESSAGE_FIELD (task, parts), i, part) { - if (func_pos != -1 && part->parsed_data.len > 0) { + if (magic_func_pos != -1 && part->parsed_data.len > 0) { struct rspamd_mime_part **pmime; struct rspamd_task **ptask; - lua_pushvalue (L, func_pos); + lua_pushcfunction (L, &rspamd_lua_traceback); + gint err_idx = lua_gettop (L); + lua_pushvalue (L, magic_func_pos); pmime = lua_newuserdata (L, sizeof (struct rspamd_mime_part *)); rspamd_lua_setclass (L, "rspamd{mimepart}", -1); *pmime = part; @@ -1392,7 +1364,7 @@ rspamd_message_process (struct rspamd_task *task) rspamd_lua_setclass (L, "rspamd{task}", -1); *ptask = task; - if (lua_pcall (L, 2, 2, 0) != 0) { + if (lua_pcall (L, 2, 2, err_idx) != 0) { msg_err_task ("cannot detect type: %s", lua_tostring (L, -1)); } else { @@ -1432,17 +1404,39 @@ rspamd_message_process (struct rspamd_task *task) } } - lua_settop (L, func_pos); + lua_settop (L, funcs_top); + } + + /* Now detect content */ + if (content_func_pos != -1 && part->parsed_data.len > 0 && + part->part_type == RSPAMD_MIME_PART_UNDEFINED) { + struct rspamd_mime_part **pmime; + struct rspamd_task **ptask; + + lua_pushcfunction (L, &rspamd_lua_traceback); + gint err_idx = lua_gettop (L); + lua_pushvalue (L, content_func_pos); + pmime = lua_newuserdata (L, sizeof (struct rspamd_mime_part *)); + rspamd_lua_setclass (L, "rspamd{mimepart}", -1); + *pmime = part; + ptask = lua_newuserdata (L, sizeof (struct rspamd_task *)); + rspamd_lua_setclass (L, "rspamd{task}", -1); + *ptask = task; + + if (lua_pcall (L, 2, 0, err_idx) != 0) { + msg_err_task ("cannot detect content: %s", lua_tostring (L, -1)); + } + + lua_settop (L, funcs_top); } - if (!(part->flags & (RSPAMD_MIME_PART_IMAGE|RSPAMD_MIME_PART_ARCHIVE)) && - (!part->ct || !(part->ct->flags & (RSPAMD_CONTENT_TYPE_MULTIPART|RSPAMD_CONTENT_TYPE_MESSAGE)))) { + if (part->part_type == RSPAMD_MIME_PART_UNDEFINED) { rspamd_message_process_text_part_maybe (task, part); } } - if (func_pos != -1) { - lua_settop (L, func_pos - 1); + if (old_top != -1) { + lua_settop (L, old_top); } /* Calculate average words length and number of short words */ diff --git a/src/libmime/message.h b/src/libmime/message.h index f3a8315fc..c11b273eb 100644 --- a/src/libmime/message.h +++ b/src/libmime/message.h @@ -30,14 +30,25 @@ struct rspamd_image; struct rspamd_archive; enum rspamd_mime_part_flags { - RSPAMD_MIME_PART_TEXT = (1 << 0), RSPAMD_MIME_PART_ATTACHEMENT = (1 << 1), - RSPAMD_MIME_PART_IMAGE = (1 << 2), - RSPAMD_MIME_PART_ARCHIVE = (1 << 3), RSPAMD_MIME_PART_BAD_CTE = (1 << 4), - RSPAMD_MIME_PART_MISSING_CTE = (1 << 5) + RSPAMD_MIME_PART_MISSING_CTE = (1 << 5), }; +enum rspamd_mime_part_type { + RSPAMD_MIME_PART_UNDEFINED = 0, + RSPAMD_MIME_PART_MULTIPART, + RSPAMD_MIME_PART_MESSAGE, + RSPAMD_MIME_PART_TEXT, + RSPAMD_MIME_PART_ARCHIVE, + RSPAMD_MIME_PART_IMAGE, + RSPAMD_MIME_PART_CUSTOM_LUA +}; + +#define IS_PART_MULTIPART(part) ((part) && ((part)->part_type == RSPAMD_MIME_PART_MULTIPART)) +#define IS_PART_TEXT(part) ((part) && ((part)->part_type == RSPAMD_MIME_PART_TEXT)) +#define IS_PART_MESSAGE(part) ((part) &&((part)->part_type == RSPAMD_MIME_PART_MESSAGE)) + enum rspamd_cte { RSPAMD_CTE_UNKNOWN = 0, RSPAMD_CTE_7BIT = 1, @@ -54,6 +65,19 @@ struct rspamd_mime_multipart { rspamd_ftok_t boundary; }; +enum rspamd_lua_specific_type { + RSPAMD_LUA_PART_TEXT, + RSPAMD_LUA_PART_STRING, + RSPAMD_LUA_PART_TABLE, + RSPAMD_LUA_PART_FUNCTION, + RSPAMD_LUA_PART_UNKNOWN, +}; + +struct rspamd_lua_specific_part { + gint cbref; + enum rspamd_lua_specific_type type; +}; + struct rspamd_mime_part { struct rspamd_content_type *ct; struct rspamd_content_type *detected_ct; @@ -72,6 +96,7 @@ struct rspamd_mime_part { enum rspamd_cte cte; guint flags; + enum rspamd_mime_part_type part_type; guint id; union { @@ -79,6 +104,7 @@ struct rspamd_mime_part { struct rspamd_mime_text_part *txt; struct rspamd_image *img; struct rspamd_archive *arch; + struct rspamd_lua_specific_part lua_specific; } specific; guchar digest[rspamd_cryptobox_HASHBYTES]; @@ -153,6 +179,7 @@ struct rspamd_message { GHashTable *emails; /**< list of parsed emails */ struct rspamd_mime_headers_table *raw_headers; /**< list of raw headers */ struct rspamd_mime_header *headers_order; /**< order of raw headers */ + struct rspamd_task *task; GPtrArray *rcpt_mime; GPtrArray *from_mime; guchar digest[16]; diff --git a/src/libmime/mime_encoding.c b/src/libmime/mime_encoding.c index 0fbba54b2..1f130325e 100644 --- a/src/libmime/mime_encoding.c +++ b/src/libmime/mime_encoding.c @@ -22,6 +22,7 @@ #include "libserver/task.h" #include "mime_encoding.h" #include "message.h" +#include "contrib/fastutf8/fastutf8.h" #include <unicode/ucnv.h> #include <unicode/ucsdet.h> #if U_ICU_VERSION_MAJOR_NUM >= 44 @@ -35,7 +36,7 @@ #define RSPAMD_CHARSET_FLAG_ASCII (1 << 1) #define RSPAMD_CHARSET_CACHE_SIZE 32 -#define RSPAMD_CHARSET_MAX_CONTENT 128 +#define RSPAMD_CHARSET_MAX_CONTENT 512 #define SET_PART_RAW(part) ((part)->flags &= ~RSPAMD_MIME_TEXT_PART_FLAG_UTF) #define SET_PART_UTF(part) ((part)->flags |= RSPAMD_MIME_TEXT_PART_FLAG_UTF) @@ -134,7 +135,10 @@ rspamd_converter_to_uchars (struct rspamd_charset_converter *cnv, struct rspamd_charset_converter * -rspamd_mime_get_converter_cached (const gchar *enc, UErrorCode *err) +rspamd_mime_get_converter_cached (const gchar *enc, + rspamd_mempool_t *pool, + gboolean is_canon, + UErrorCode *err) { const gchar *canon_name; static rspamd_lru_hash_t *cache; @@ -146,7 +150,19 @@ rspamd_mime_get_converter_cached (const gchar *enc, UErrorCode *err) rspamd_str_equal); } - canon_name = ucnv_getStandardName (enc, "IANA", err); + if (enc == NULL) { + return NULL; + } + + if (!is_canon) { + rspamd_ftok_t cset_tok; + + RSPAMD_FTOK_FROM_STR (&cset_tok, enc); + canon_name = rspamd_mime_detect_charset (&cset_tok, pool); + } + else { + canon_name = enc; + } if (canon_name == NULL) { return NULL; @@ -238,6 +254,7 @@ rspamd_mime_detect_charset (const rspamd_ftok_t *in, rspamd_mempool_t *pool) { gchar *ret = NULL, *h, *t; struct rspamd_charset_substitution *s; + const gchar *cset; UErrorCode uc_err = U_ZERO_ERROR; if (sub_hash == NULL) { @@ -267,10 +284,28 @@ rspamd_mime_detect_charset (const rspamd_ftok_t *in, rspamd_mempool_t *pool) s = g_hash_table_lookup (sub_hash, ret); if (s) { - return ucnv_getStandardName (s->canon, "IANA", &uc_err); + ret = (char *)s->canon; } - return ucnv_getStandardName (ret, "IANA", &uc_err); + /* Just fucking stupid */ + cset = ucnv_getCanonicalName (ret, "MIME", &uc_err); + + if (cset == NULL) { + uc_err = U_ZERO_ERROR; + cset = ucnv_getCanonicalName (ret, "IANA", &uc_err); + } + + if (cset == NULL) { + uc_err = U_ZERO_ERROR; + cset = ucnv_getCanonicalName (ret, "WINDOWS", &uc_err); + } + + if (cset == NULL) { + uc_err = U_ZERO_ERROR; + cset = ucnv_getCanonicalName (ret, "JAVA", &uc_err); + } + + return cset; } gchar * @@ -286,7 +321,7 @@ rspamd_mime_text_to_utf8 (rspamd_mempool_t *pool, UConverter *utf8_converter; struct rspamd_charset_converter *conv; - conv = rspamd_mime_get_converter_cached (in_enc, &uc_err); + conv = rspamd_mime_get_converter_cached (in_enc, pool, TRUE, &uc_err); utf8_converter = rspamd_get_utf8_converter (); if (conv == NULL) { @@ -350,7 +385,8 @@ rspamd_mime_text_part_utf8_convert (struct rspamd_task *task, UConverter *utf8_converter; struct rspamd_charset_converter *conv; - conv = rspamd_mime_get_converter_cached (charset, &uc_err); + conv = rspamd_mime_get_converter_cached (charset, task->task_pool, + TRUE, &uc_err); utf8_converter = rspamd_get_utf8_converter (); if (conv == NULL) { @@ -409,6 +445,7 @@ rspamd_mime_text_part_utf8_convert (struct rspamd_task *task, gboolean rspamd_mime_to_utf8_byte_array (GByteArray *in, GByteArray *out, + rspamd_mempool_t *pool, const gchar *enc) { gint32 r, clen, dlen; @@ -418,6 +455,24 @@ rspamd_mime_to_utf8_byte_array (GByteArray *in, struct rspamd_charset_converter *conv; rspamd_ftok_t charset_tok; + if (in == NULL || in->len == 0) { + return FALSE; + } + + if (enc == NULL) { + /* Assume utf ? */ + if (rspamd_fast_utf8_validate (in->data, in->len) == 0) { + g_byte_array_set_size (out, in->len); + memcpy (out->data, in->data, out->len); + + return TRUE; + } + else { + /* Bad stuff, keep out */ + return FALSE; + } + } + RSPAMD_FTOK_FROM_STR (&charset_tok, enc); if (rspamd_mime_charset_utf_check (&charset_tok, (gchar *)in->data, in->len, @@ -429,7 +484,7 @@ rspamd_mime_to_utf8_byte_array (GByteArray *in, } utf8_converter = rspamd_get_utf8_converter (); - conv = rspamd_mime_get_converter_cached (enc, &uc_err); + conv = rspamd_mime_get_converter_cached (enc, pool, TRUE, &uc_err); if (conv == NULL) { return FALSE; @@ -468,36 +523,38 @@ rspamd_mime_to_utf8_byte_array (GByteArray *in, void rspamd_mime_charset_utf_enforce (gchar *in, gsize len) { - const gchar *end, *p; - gsize remain = len; + gchar *p, *end; + goffset err_offset; + UChar32 uc = 0; /* Now we validate input and replace bad characters with '?' symbol */ p = in; + end = in + len; - while (remain > 0 && !g_utf8_validate (p, remain, &end)) { - gchar *valid; + while (p < end && len > 0 && (err_offset = rspamd_fast_utf8_validate (p, len)) > 0) { + err_offset --; /* As it returns it 1 indexed */ + gint32 cur_offset = err_offset; - if (end >= in + len) { - if (p < in + len) { - memset ((gchar *)p, '?', (in + len) - p); - } - break; - } + while (cur_offset < len) { + gint32 tmp = cur_offset; - valid = g_utf8_find_next_char (end, in + len); + U8_NEXT (p, cur_offset, len, uc); - if (!valid) { - valid = in + len; + if (uc > 0) { + /* Fill string between err_offset and tmp with `?` character */ + memset (p + err_offset, '?', tmp - err_offset); + break; + } } - if (valid > end) { - memset ((gchar *)end, '?', valid - end); - p = valid; - remain = (in + len) - p; - } - else { + if (uc < 0) { + /* Fill till the end */ + memset (p + err_offset, '?', len - err_offset); break; } + + p += cur_offset; + len = end - p; } } @@ -568,28 +625,30 @@ rspamd_mime_charset_utf_check (rspamd_ftok_t *charset, * corner cases */ if (content_check) { - real_charset = rspamd_mime_charset_find_by_content (in, - MIN (RSPAMD_CHARSET_MAX_CONTENT, len)); + if (rspamd_fast_utf8_validate (in, len) != 0) { + real_charset = rspamd_mime_charset_find_by_content (in, + MIN (RSPAMD_CHARSET_MAX_CONTENT, len)); - if (real_charset) { + if (real_charset) { - if (rspamd_regexp_match (utf_compatible_re, - real_charset, strlen (real_charset), TRUE)) { - RSPAMD_FTOK_ASSIGN (charset, UTF8_CHARSET); + if (rspamd_regexp_match (utf_compatible_re, + real_charset, strlen (real_charset), TRUE)) { + RSPAMD_FTOK_ASSIGN (charset, UTF8_CHARSET); - return TRUE; - } - else { - charset->begin = real_charset; - charset->len = strlen (real_charset); + return TRUE; + } + else { + charset->begin = real_charset; + charset->len = strlen (real_charset); - return FALSE; + return FALSE; + } } + + rspamd_mime_charset_utf_enforce (in, len); } } - rspamd_mime_charset_utf_enforce (in, len); - return TRUE; } @@ -615,6 +674,8 @@ rspamd_mime_text_part_maybe_convert (struct rspamd_task *task, part_content = g_byte_array_sized_new (text_part->parsed.len); memcpy (part_content->data, text_part->parsed.begin, text_part->parsed.len); part_content->len = text_part->parsed.len; + rspamd_mempool_notify_alloc (task->task_pool, + part_content->len); rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t)g_byte_array_unref, part_content); diff --git a/src/libmime/mime_encoding.h b/src/libmime/mime_encoding.h index 5224d33fb..22f0ee818 100644 --- a/src/libmime/mime_encoding.h +++ b/src/libmime/mime_encoding.h @@ -47,7 +47,7 @@ const gchar *rspamd_mime_detect_charset (const rspamd_ftok_t *in, * @param pool * @param input * @param len - * @param in_enc + * @param in_enc canon charset * @param olen * @param err * @return @@ -57,14 +57,20 @@ gchar *rspamd_mime_text_to_utf8 (rspamd_mempool_t *pool, gsize *olen, GError **err); /** - * Converts data from `in` to `out`, returns `FALSE` if `enc` is not a valid iconv charset + * Converts data from `in` to `out`, + * returns `FALSE` if `enc` is not a valid iconv charset + * + * This function, in fact, copies `in` from `out` replacing out content in + * total. * @param in * @param out - * @param enc + * @param enc validated canonical charset name. If NULL, then utf8 check is done only * @return */ gboolean rspamd_mime_to_utf8_byte_array (GByteArray *in, - GByteArray *out, const gchar *enc); + GByteArray *out, + rspamd_mempool_t *pool, + const gchar *enc); /** * Maybe convert part to utf-8 @@ -83,7 +89,8 @@ void rspamd_mime_text_part_maybe_convert (struct rspamd_task *task, * @return */ gboolean rspamd_mime_charset_utf_check (rspamd_ftok_t *charset, - gchar *in, gsize len, gboolean content_check); + gchar *in, gsize len, + gboolean content_check); /** * Ensure that all characters in string are valid utf8 chars or replace them @@ -93,14 +100,18 @@ gboolean rspamd_mime_charset_utf_check (rspamd_ftok_t *charset, */ void rspamd_mime_charset_utf_enforce (gchar *in, gsize len); -/** - * Gets cached converter - * @param enc - * @param err - * @return - */ + /** + * Gets cached converter + * @param enc input encoding + * @param pool pool to use for temporary normalisation + * @param is_canon TRUE if normalisation is needed + * @param err output error + * @return converter + */ struct rspamd_charset_converter *rspamd_mime_get_converter_cached ( const gchar *enc, + rspamd_mempool_t *pool, + gboolean is_canon, UErrorCode *err); /** diff --git a/src/libmime/mime_expressions.c b/src/libmime/mime_expressions.c index 7354b3aeb..ea46233ec 100644 --- a/src/libmime/mime_expressions.c +++ b/src/libmime/mime_expressions.c @@ -1309,6 +1309,19 @@ struct addr_list { guint addrlen; }; +static gint +addr_list_cmp_func (const void *a, const void *b) +{ + const struct addr_list *addra = (struct addr_list *)a, + *addrb = (struct addr_list *)b; + + if (addra->addrlen != addrb->addrlen) { + return addra->addrlen - addrb->addrlen; + } + + return memcmp (addra->addr, addrb->addr, addra->addrlen); +} + #define COMPARE_RCPT_LEN 3 #define MIN_RCPT_TO_COMPARE 7 @@ -1320,7 +1333,7 @@ rspamd_recipients_distance (struct rspamd_task *task, GArray * args, struct rspamd_email_address *cur; double threshold; struct addr_list *ar; - gint num, i, j, hits = 0, total = 0; + gint num, i, hits = 0; if (args == NULL) { msg_warn_task ("no parameters to function"); @@ -1356,27 +1369,31 @@ rspamd_recipients_distance (struct rspamd_task *task, GArray * args, ar = rspamd_mempool_alloc0 (task->task_pool, num * sizeof (struct addr_list)); /* Fill array */ + num = 0; PTR_ARRAY_FOREACH (MESSAGE_FIELD (task, rcpt_mime), i, cur) { - ar[i].name = cur->addr; - ar[i].namelen = cur->addr_len; - ar[i].addr = cur->domain; - ar[i].addrlen = cur->domain_len; + if (cur->addr_len > COMPARE_RCPT_LEN) { + ar[num].name = cur->addr; + ar[num].namelen = cur->addr_len; + ar[num].addr = cur->domain; + ar[num].addrlen = cur->domain_len; + num ++; + } } + qsort (ar, num, sizeof (*ar), addr_list_cmp_func); + /* Cycle all elements in array */ for (i = 0; i < num; i++) { - for (j = i + 1; j < num; j++) { - if (ar[i].namelen >= COMPARE_RCPT_LEN && ar[j].namelen >= COMPARE_RCPT_LEN && - rspamd_lc_cmp (ar[i].name, ar[j].name, COMPARE_RCPT_LEN) == 0) { - /* Common name part */ - hits++; + if (i < num - 1) { + if (ar[i].namelen == ar[i + 1].namelen) { + if (rspamd_lc_cmp (ar[i].name, ar[i + 1].name, COMPARE_RCPT_LEN) == 0) { + hits++; + } } - - total++; } } - if ((hits * num / 2.) / (double)total >= threshold) { + if ((hits * num / 2.) / (double)num >= threshold) { return TRUE; } @@ -1480,7 +1497,7 @@ rspamd_compare_transfer_encoding (struct rspamd_task * task, } PTR_ARRAY_FOREACH (MESSAGE_FIELD (task, parts), i, part) { - if (IS_CT_TEXT (part->ct)) { + if (IS_PART_TEXT (part)) { if (part->cte == cte) { return TRUE; } @@ -1800,7 +1817,7 @@ rspamd_content_type_compare_param (struct rspamd_task * task, * If user did not specify argument, let's assume that he wants * recursive search if mime part is multipart/mixed */ - if (IS_CT_MULTIPART (cur_part->ct)) { + if (IS_PART_MULTIPART (cur_part)) { recursive = TRUE; } } @@ -1880,7 +1897,7 @@ rspamd_content_type_has_param (struct rspamd_task * task, * If user did not specify argument, let's assume that he wants * recursive search if mime part is multipart/mixed */ - if (IS_CT_MULTIPART (cur_part->ct)) { + if (IS_PART_MULTIPART (cur_part)) { recursive = TRUE; } } @@ -1955,7 +1972,7 @@ rspamd_content_type_check (struct rspamd_task *task, * If user did not specify argument, let's assume that he wants * recursive search if mime part is multipart/mixed */ - if (IS_CT_MULTIPART (ct)) { + if (IS_PART_MULTIPART (cur_part)) { recursive = TRUE; } } diff --git a/src/libmime/mime_headers.c b/src/libmime/mime_headers.c index b024bd7b1..aed7575eb 100644 --- a/src/libmime/mime_headers.c +++ b/src/libmime/mime_headers.c @@ -512,9 +512,12 @@ rspamd_mime_headers_process (struct rspamd_task *task, } static void -rspamd_mime_header_maybe_save_token (rspamd_mempool_t *pool, GString *out, - GByteArray *token, GByteArray *decoded_token, - rspamd_ftok_t *old_charset, rspamd_ftok_t *new_charset) +rspamd_mime_header_maybe_save_token (rspamd_mempool_t *pool, + GString *out, + GByteArray *token, + GByteArray *decoded_token, + rspamd_ftok_t *old_charset, + rspamd_ftok_t *new_charset) { if (new_charset->len == 0) { g_assert_not_reached (); @@ -538,14 +541,22 @@ rspamd_mime_header_maybe_save_token (rspamd_mempool_t *pool, GString *out, } /* We need to flush and decode old token to out string */ - if (rspamd_mime_to_utf8_byte_array (token, decoded_token, + if (rspamd_mime_to_utf8_byte_array (token, decoded_token, pool, rspamd_mime_detect_charset (new_charset, pool))) { g_string_append_len (out, decoded_token->data, decoded_token->len); } /* We also reset buffer */ g_byte_array_set_size (token, 0); - /* Propagate charset */ + /* + * Propagate charset + * + * Here are dragons: we save the original charset to allow buffers concat + * in the condition at the beginning of the function. + * However, it will likely cause unnecessary calls for + * `rspamd_mime_detect_charset` which could be relatively expensive. + * But we ignore that for now... + */ memcpy (old_charset, new_charset, sizeof (*old_charset)); } @@ -776,6 +787,7 @@ rspamd_mime_header_decode (rspamd_mempool_t *pool, const gchar *in, g_byte_array_free (token, TRUE); g_byte_array_free (decoded, TRUE); rspamd_mime_header_sanity_check (out); + rspamd_mempool_notify_alloc (pool, out->len); ret = g_string_free (out, FALSE); rspamd_mempool_add_destructor (pool, g_free, ret); diff --git a/src/libmime/mime_parser.c b/src/libmime/mime_parser.c index 0b682879d..d989a8e2e 100644 --- a/src/libmime/mime_parser.c +++ b/src/libmime/mime_parser.c @@ -448,6 +448,30 @@ rspamd_mime_part_get_cd (struct rspamd_task *task, struct rspamd_mime_part *part cd->lc_data, &cd->filename); break; } + else if (part->ct) { + /* + * Even in case of malformed Content-Disposition, we can still + * fall back to Content-Type + */ + cd = rspamd_mempool_alloc0 (task->task_pool, sizeof (*cd)); + cd->type = RSPAMD_CT_INLINE; + + /* We can also have content dispositon definitions in Content-Type */ + if (part->ct->attrs) { + RSPAMD_FTOK_ASSIGN (&srch, "name"); + found = g_hash_table_lookup (part->ct->attrs, &srch); + + if (!found) { + RSPAMD_FTOK_ASSIGN (&srch, "filename"); + found = g_hash_table_lookup (part->ct->attrs, &srch); + } + + if (found) { + cd->type = RSPAMD_CT_ATTACHMENT; + memcpy (&cd->filename, &found->value, sizeof (cd->filename)); + } + } + } } } @@ -508,13 +532,14 @@ rspamd_mime_parse_normal_part (struct rspamd_task *task, } } - if (IS_CT_TEXT (part->ct)) { + if (part->ct && (part->ct->flags & RSPAMD_CONTENT_TYPE_TEXT)) { /* Need to copy text as we have couple of in-place change functions */ parsed = rspamd_fstring_sized_new (part->raw_data.len); parsed->len = part->raw_data.len; memcpy (parsed->str, part->raw_data.begin, parsed->len); part->parsed_data.begin = parsed->str; part->parsed_data.len = parsed->len; + rspamd_mempool_notify_alloc (task->task_pool, parsed->len); rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t)rspamd_fstring_free, parsed); } @@ -531,6 +556,7 @@ rspamd_mime_parse_normal_part (struct rspamd_task *task, parsed->len = r; part->parsed_data.begin = parsed->str; part->parsed_data.len = parsed->len; + rspamd_mempool_notify_alloc (task->task_pool, parsed->len); rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t)rspamd_fstring_free, parsed); } @@ -542,6 +568,7 @@ rspamd_mime_parse_normal_part (struct rspamd_task *task, parsed->len = part->raw_data.len; part->parsed_data.begin = parsed->str; part->parsed_data.len = parsed->len; + rspamd_mempool_notify_alloc (task->task_pool, parsed->len); rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t)rspamd_fstring_free, parsed); } @@ -553,6 +580,7 @@ rspamd_mime_parse_normal_part (struct rspamd_task *task, parsed->str, &parsed->len); part->parsed_data.begin = parsed->str; part->parsed_data.len = parsed->len; + rspamd_mempool_notify_alloc (task->task_pool, parsed->len); rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t)rspamd_fstring_free, parsed); break; @@ -560,6 +588,7 @@ rspamd_mime_parse_normal_part (struct rspamd_task *task, parsed = rspamd_fstring_sized_new (part->raw_data.len / 4 * 3 + 12); r = rspamd_decode_uue_buf (part->raw_data.begin, part->raw_data.len, parsed->str, parsed->allocated); + rspamd_mempool_notify_alloc (task->task_pool, parsed->len); rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t)rspamd_fstring_free, parsed); if (r != -1) { @@ -573,6 +602,7 @@ rspamd_mime_parse_normal_part (struct rspamd_task *task, part->cte = RSPAMD_CTE_8BIT; parsed->len = MIN (part->raw_data.len, parsed->allocated); memcpy (parsed->str, part->raw_data.begin, parsed->len); + rspamd_mempool_notify_alloc (task->task_pool, parsed->len); part->parsed_data.begin = parsed->str; part->parsed_data.len = parsed->len; } @@ -720,6 +750,7 @@ rspamd_mime_process_multipart_node (struct rspamd_task *task, if (sel->flags & RSPAMD_CONTENT_TYPE_MULTIPART) { st->nesting ++; g_ptr_array_add (st->stack, npart); + npart->part_type = RSPAMD_MIME_PART_MULTIPART; npart->specific.mp = rspamd_mempool_alloc0 (task->task_pool, sizeof (struct rspamd_mime_multipart)); memcpy (&npart->specific.mp->boundary, &sel->orig_boundary, @@ -729,6 +760,7 @@ rspamd_mime_process_multipart_node (struct rspamd_task *task, else if (sel->flags & RSPAMD_CONTENT_TYPE_MESSAGE) { st->nesting ++; g_ptr_array_add (st->stack, npart); + npart->part_type = RSPAMD_MIME_PART_MESSAGE; if ((ret = rspamd_mime_parse_normal_part (task, npart, st, err)) == RSPAMD_MIME_PARSE_OK) { @@ -1005,7 +1037,7 @@ rspamd_mime_preprocess_cb (struct rspamd_multipattern *mp, bend++; /* \r\n */ - if (*bend == '\n') { + if (bend < end && *bend == '\n') { bend++; } } @@ -1366,6 +1398,7 @@ rspamd_mime_parse_message (struct rspamd_task *task, if (sel->flags & RSPAMD_CONTENT_TYPE_MULTIPART) { g_ptr_array_add (nst->stack, npart); nst->nesting ++; + npart->part_type = RSPAMD_MIME_PART_MULTIPART; npart->specific.mp = rspamd_mempool_alloc0 (task->task_pool, sizeof (struct rspamd_mime_multipart)); memcpy (&npart->specific.mp->boundary, &sel->orig_boundary, @@ -1375,6 +1408,7 @@ rspamd_mime_parse_message (struct rspamd_task *task, else if (sel->flags & RSPAMD_CONTENT_TYPE_MESSAGE) { if ((ret = rspamd_mime_parse_normal_part (task, npart, nst, err)) == RSPAMD_MIME_PARSE_OK) { + npart->part_type = RSPAMD_MIME_PART_MESSAGE; ret = rspamd_mime_parse_message (task, npart, nst, err); } } diff --git a/src/libmime/scan_result.c b/src/libmime/scan_result.c index 2ffe6e7ff..7a8451f9c 100644 --- a/src/libmime/scan_result.c +++ b/src/libmime/scan_result.c @@ -21,6 +21,7 @@ #include "lua/lua_common.h" #include "libserver/cfg_file_private.h" #include "libmime/scan_result_private.h" +#include "contrib/fastutf8/fastutf8.h" #include <math.h> #include "contrib/uthash/utlist.h" @@ -289,19 +290,9 @@ insert_metric_result (struct rspamd_task *task, single = TRUE; } - /* Now check for the duplicate options */ - if (opt && s->options) { - k = kh_get (rspamd_options_hash, s->options, opt); + s->nshots ++; - if (k == kh_end (s->options)) { - rspamd_task_add_result_option (task, s, opt); - } - else { - s->nshots ++; - } - } - else { - s->nshots ++; + if (opt) { rspamd_task_add_result_option (task, s, opt); } @@ -516,15 +507,40 @@ rspamd_task_add_result_option (struct rspamd_task *task, { struct rspamd_symbol_option *opt; gboolean ret = FALSE; - gchar *opt_cpy; + gchar *opt_cpy = NULL; + gsize vlen; khiter_t k; gint r; if (s && val) { + if (s->opts_len < 0) { + /* Cannot add more options, give up */ + msg_debug_task ("cannot add more options to symbol %s when adding option %s", + s->name, val); + return FALSE; + } + if (!s->options) { s->options = kh_init (rspamd_options_hash); } + vlen = strlen (val); + + if (vlen + s->opts_len > task->cfg->max_opts_len) { + /* Add truncated option */ + msg_info_task ("cannot add more options to symbol %s when adding option %s", + s->name, val); + val = "..."; + vlen = 3; + s->opts_len = -1; + } + + if (rspamd_fast_utf8_validate (val, vlen) != 0) { + opt_cpy = rspamd_str_make_utf_valid (val, vlen, &vlen, + task->task_pool); + val = opt_cpy; + } + if (!(s->sym && (s->sym->flags & RSPAMD_SYMBOL_FLAG_ONEPARAM)) && kh_size (s->options) < task->cfg->default_max_shots) { /* Append new options */ @@ -532,7 +548,11 @@ rspamd_task_add_result_option (struct rspamd_task *task, if (k == kh_end (s->options)) { opt = rspamd_mempool_alloc0 (task->task_pool, sizeof (*opt)); - opt_cpy = rspamd_mempool_strdup (task->task_pool, val); + + if (opt_cpy == NULL) { + opt_cpy = rspamd_mempool_strdup (task->task_pool, val); + } + k = kh_put (rspamd_options_hash, s->options, opt_cpy, &r); kh_value (s->options, k) = opt; @@ -543,15 +563,12 @@ rspamd_task_add_result_option (struct rspamd_task *task, } } else { - opt = rspamd_mempool_alloc0 (task->task_pool, sizeof (*opt)); - opt_cpy = rspamd_mempool_strdup (task->task_pool, val); - k = kh_put (rspamd_options_hash, s->options, opt_cpy, &r); - - kh_value (s->options, k) = opt; - opt->option = opt_cpy; - DL_APPEND (s->opts_head, opt); + /* Skip addition */ + ret = FALSE; + } - ret = TRUE; + if (ret && s->opts_len >= 0) { + s->opts_len += vlen; } } else if (!val) { diff --git a/src/libmime/scan_result.h b/src/libmime/scan_result.h index b5f76baf7..3b222fffb 100644 --- a/src/libmime/scan_result.h +++ b/src/libmime/scan_result.h @@ -39,6 +39,7 @@ struct rspamd_symbol_result { struct rspamd_symbol_option *opts_head; /**< head of linked list of options */ const gchar *name; struct rspamd_symbol *sym; /**< symbol configuration */ + gssize opts_len; /**< total size of all options (negative if truncated option is added) */ guint nshots; enum rspamd_symbol_result_flags flags; }; diff --git a/src/libserver/async_session.c b/src/libserver/async_session.c index cec2963aa..1bd4c77e9 100644 --- a/src/libserver/async_session.c +++ b/src/libserver/async_session.c @@ -240,7 +240,9 @@ rspamd_session_remove_event_full (struct rspamd_async_session *session, kh_del (rspamd_events_hash, session->events, k); /* Remove event */ - fin (ud); + if (fin) { + fin (ud); + } rspamd_session_pending (session); } diff --git a/src/libserver/cfg_file.h b/src/libserver/cfg_file.h index 1cf4a1faf..de6f37766 100644 --- a/src/libserver/cfg_file.h +++ b/src/libserver/cfg_file.h @@ -127,9 +127,10 @@ struct rspamd_symbols_group { enum rspamd_symbol_flags { RSPAMD_SYMBOL_FLAG_NORMAL = 0, - RSPAMD_SYMBOL_FLAG_IGNORE = (1 << 1), + RSPAMD_SYMBOL_FLAG_IGNORE_METRIC = (1 << 1), RSPAMD_SYMBOL_FLAG_ONEPARAM = (1 << 2), RSPAMD_SYMBOL_FLAG_UNGROUPPED = (1 << 3), + RSPAMD_SYMBOL_FLAG_DISABLED = (1 << 4), }; /** @@ -261,6 +262,8 @@ enum rspamd_log_format_type { RSPAMD_LOG_SETTINGS_ID, RSPAMD_LOG_GROUPS, RSPAMD_LOG_PUBLIC_GROUPS, + RSPAMD_LOG_MEMPOOL_SIZE, + RSPAMD_LOG_MEMPOOL_WASTE, }; enum rspamd_log_format_flags { @@ -438,6 +441,7 @@ struct rspamd_config { gchar *rrd_file; /**< rrd file to store statistics */ gchar *history_file; /**< file to save rolling history */ + gchar *stats_file; /**< file to save stats */ gchar *tld_file; /**< file to load effective tld list from */ gchar *hs_cache_dir; /**< directory to save hyperscan databases */ gchar *events_backend; /**< string representation of the events backend used */ @@ -466,6 +470,7 @@ struct rspamd_config { guint full_gc_iters; /**< iterations between full gc cycle */ guint max_lua_urls; /**< maximum number of urls to be passed to Lua */ guint max_blas_threads; /**< maximum threads for openblas when learning ANN */ + guint max_opts_len; /**< maximum length for all options for a symbol */ GList *classify_headers; /**< list of headers using for statistics */ struct module_s **compiled_modules; /**< list of compiled C modules */ @@ -493,6 +498,7 @@ struct rspamd_config { struct rspamd_config_settings_elt *setting_ids; /**< preprocessed settings ids */ struct rspamd_lang_detector *lang_det; /**< language detector */ + struct rspamd_worker *cur_worker; /**< set dynamically by each worker */ ref_entry_t ref; /**< reference counter */ }; @@ -755,7 +761,8 @@ gboolean rspamd_config_radix_from_ucl (struct rspamd_config *cfg, const ucl_object_t *obj, const gchar *description, struct rspamd_radix_map_helper **target, - GError **err); + GError **err, + struct rspamd_worker *worker); /** * Adds new settings id to be preprocessed diff --git a/src/libserver/cfg_rcl.c b/src/libserver/cfg_rcl.c index e55076894..817f7efc5 100644 --- a/src/libserver/cfg_rcl.c +++ b/src/libserver/cfg_rcl.c @@ -456,11 +456,10 @@ rspamd_rcl_symbol_handler (rspamd_mempool_t *pool, const ucl_object_t *obj, const gchar *description = NULL; gdouble score = NAN; guint priority = 1, flags = 0; - gint nshots; + gint nshots = 0; g_assert (key != NULL); cfg = sd->cfg; - nshots = cfg->default_max_shots; if ((elt = ucl_object_lookup (obj, "one_shot")) != NULL) { if (ucl_object_type (elt) != UCL_BOOLEAN) { @@ -520,7 +519,23 @@ rspamd_rcl_symbol_handler (rspamd_mempool_t *pool, const ucl_object_t *obj, } if (ucl_object_toboolean (elt)) { - flags |= RSPAMD_SYMBOL_FLAG_IGNORE; + flags |= RSPAMD_SYMBOL_FLAG_IGNORE_METRIC; + } + } + + if ((elt = ucl_object_lookup (obj, "enabled")) != NULL) { + if (ucl_object_type (elt) != UCL_BOOLEAN) { + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "enabled attribute is not boolean for symbol: '%s'", + key); + + return FALSE; + } + + if (ucl_object_toboolean (elt)) { + flags |= RSPAMD_SYMBOL_FLAG_DISABLED; } } @@ -1949,6 +1964,12 @@ rspamd_rcl_config_init (struct rspamd_config *cfg, GHashTable *skip_sections) RSPAMD_CL_FLAG_STRING_PATH, "Path to RRD file"); rspamd_rcl_add_default_handler (sub, + "stats_file", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_config, stats_file), + RSPAMD_CL_FLAG_STRING_PATH, + "Path to stats file"); + rspamd_rcl_add_default_handler (sub, "history_file", rspamd_rcl_parse_struct_string, G_STRUCT_OFFSET (struct rspamd_config, history_file), @@ -2208,6 +2229,12 @@ rspamd_rcl_config_init (struct rspamd_config *cfg, GHashTable *skip_sections) RSPAMD_CL_FLAG_INT_32, "Maximum number of Blas threads for learning neural networks (default: 1)"); rspamd_rcl_add_default_handler (sub, + "max_opts_len", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, max_opts_len), + RSPAMD_CL_FLAG_INT_32, + "Maximum size of all options for a single symbol (default: 4096)"); + rspamd_rcl_add_default_handler (sub, "events_backend", rspamd_rcl_parse_struct_string, G_STRUCT_OFFSET (struct rspamd_config, events_backend), diff --git a/src/libserver/cfg_utils.c b/src/libserver/cfg_utils.c index 59840847b..37a3d4578 100644 --- a/src/libserver/cfg_utils.c +++ b/src/libserver/cfg_utils.c @@ -133,7 +133,7 @@ rspamd_config_new (enum rspamd_config_init_flags flags) struct rspamd_config *cfg; rspamd_mempool_t *pool; - pool = rspamd_mempool_new (8 * 1024 * 1024, "cfg"); + pool = rspamd_mempool_new (8 * 1024 * 1024, "cfg", 0); cfg = rspamd_mempool_alloc0 (pool, sizeof (*cfg)); /* Allocate larger pool for cfg */ cfg->cfg_pool = pool; @@ -197,6 +197,7 @@ rspamd_config_new (enum rspamd_config_init_flags flags) cfg->cache_reload_time = 30.0; cfg->max_lua_urls = 1024; cfg->max_blas_threads = 1; + cfg->max_opts_len = 4096; /* Default log line */ cfg->log_format_str = "id: <$mid>,$if_qid{ qid: <$>,}$if_ip{ ip: $,}" @@ -518,6 +519,12 @@ rspamd_config_process_var (struct rspamd_config *cfg, const rspamd_ftok_t *var, else if (rspamd_ftok_cstr_equal (&tok, "settings_id", TRUE)) { type = RSPAMD_LOG_SETTINGS_ID; } + else if (rspamd_ftok_cstr_equal (&tok, "mempool_size", TRUE)) { + type = RSPAMD_LOG_MEMPOOL_SIZE; + } + else if (rspamd_ftok_cstr_equal (&tok, "mempool_waste", TRUE)) { + type = RSPAMD_LOG_MEMPOOL_WASTE; + } else { msg_err_config ("unknown log variable: %T", &tok); return FALSE; @@ -1149,7 +1156,8 @@ rspamd_include_map_handler (const guchar *data, gsize len, rspamd_ucl_read_cb, rspamd_ucl_fin_cb, rspamd_ucl_dtor_cb, - (void **)pcbdata) != NULL; + (void **)pcbdata, + NULL) != NULL; } /* @@ -1598,7 +1606,7 @@ rspamd_config_new_symbol (struct rspamd_config *cfg, const gchar *symbol, sym_def->weight_ptr = score_ptr; sym_def->name = rspamd_mempool_strdup (cfg->cfg_pool, symbol); sym_def->flags = flags; - sym_def->nshots = nshots; + sym_def->nshots = nshots != 0 ? nshots : cfg->default_max_shots; sym_def->groups = g_ptr_array_sized_new (1); rspamd_mempool_add_destructor (cfg->cfg_pool, rspamd_ptr_array_free_hard, sym_def->groups); @@ -1700,6 +1708,11 @@ rspamd_config_add_symbol (struct rspamd_config *cfg, description); } + /* Or nshots in case of non-default setting */ + if (nshots != 0 && sym_def->nshots == cfg->default_max_shots) { + sym_def->nshots = nshots; + } + return FALSE; } else { @@ -1720,7 +1733,16 @@ rspamd_config_add_symbol (struct rspamd_config *cfg, } sym_def->flags = flags; - sym_def->nshots = nshots; + + if (nshots != 0) { + sym_def->nshots = nshots; + } + else { + /* Do not reset unless we have exactly lower priority */ + if (sym_def->priority < priority) { + sym_def->nshots = cfg->default_max_shots; + } + } if (description) { sym_def->description = rspamd_mempool_strdup (cfg->cfg_pool, @@ -2166,10 +2188,11 @@ rspamd_config_get_action_by_type (struct rspamd_config *cfg, gboolean rspamd_config_radix_from_ucl (struct rspamd_config *cfg, - const ucl_object_t *obj, - const gchar *description, - struct rspamd_radix_map_helper **target, - GError **err) + const ucl_object_t *obj, + const gchar *description, + struct rspamd_radix_map_helper **target, + GError **err, + struct rspamd_worker *worker) { ucl_type_t type; ucl_object_iter_t it = NULL; @@ -2193,8 +2216,10 @@ rspamd_config_radix_from_ucl (struct rspamd_config *cfg, rspamd_radix_read, rspamd_radix_fin, rspamd_radix_dtor, - (void **)target) == NULL) { - g_set_error (err, g_quark_from_static_string ("rspamd-config"), + (void **)target, + worker) == NULL) { + g_set_error (err, + g_quark_from_static_string ("rspamd-config"), EINVAL, "bad map definition %s for %s", str, ucl_object_key (obj)); return FALSE; @@ -2218,8 +2243,10 @@ rspamd_config_radix_from_ucl (struct rspamd_config *cfg, rspamd_radix_read, rspamd_radix_fin, rspamd_radix_dtor, - (void **)target) == NULL) { - g_set_error (err, g_quark_from_static_string ("rspamd-config"), + (void **)target, + worker) == NULL) { + g_set_error (err, + g_quark_from_static_string ("rspamd-config"), EINVAL, "bad map object for %s", ucl_object_key (obj)); return FALSE; } diff --git a/src/libserver/dkim.c b/src/libserver/dkim.c index 4b66ebbd5..b3efd2149 100644 --- a/src/libserver/dkim.c +++ b/src/libserver/dkim.c @@ -1032,6 +1032,17 @@ rspamd_create_dkim_context (const gchar *sig, if (!parser_funcs[param](ctx, c, tlen, err)) { state = DKIM_STATE_ERROR; } + if (state == DKIM_STATE_ERROR) { + /* + * We need to return from here as state machine won't + * do any more steps after p == end + */ + if (err) { + msg_info_dkim ("dkim parse failed: %e", *err); + } + + return NULL; + } /* Finish processing */ p++; } diff --git a/src/libserver/dns.c b/src/libserver/dns.c index 710f96b3a..505e44a7a 100644 --- a/src/libserver/dns.c +++ b/src/libserver/dns.c @@ -30,11 +30,13 @@ static struct rdns_upstream_elt* rspamd_dns_select_upstream (const char *name, size_t len, void *ups_data); static struct rdns_upstream_elt* rspamd_dns_select_upstream_retransmit ( const char *name, - size_t len, void *ups_data); + size_t len, + struct rdns_upstream_elt *prev_elt, + void *ups_data); static void rspamd_dns_upstream_ok (struct rdns_upstream_elt *elt, void *ups_data); static void rspamd_dns_upstream_fail (struct rdns_upstream_elt *elt, - void *ups_data); + void *ups_data, const gchar *reason); static unsigned int rspamd_dns_upstream_count (void *ups_data); static struct rdns_upstream_context rspamd_ups_ctx = { @@ -815,12 +817,20 @@ rspamd_dns_select_upstream (const char *name, static struct rdns_upstream_elt* rspamd_dns_select_upstream_retransmit ( const char *name, - size_t len, void *ups_data) + size_t len, + struct rdns_upstream_elt *prev_elt, + void *ups_data) { struct upstream_list *ups = ups_data; struct upstream *up; - up = rspamd_upstream_get_forced (ups, RSPAMD_UPSTREAM_RANDOM, name, len); + if (prev_elt) { + up = rspamd_upstream_get_except (ups, (struct upstream *)prev_elt->lib_data, + RSPAMD_UPSTREAM_MASTER_SLAVE, name, len); + } + else { + up = rspamd_upstream_get_forced (ups, RSPAMD_UPSTREAM_RANDOM, name, len); + } if (up) { msg_debug ("select forced %s", rspamd_upstream_name (up)); @@ -842,11 +852,11 @@ rspamd_dns_upstream_ok (struct rdns_upstream_elt *elt, static void rspamd_dns_upstream_fail (struct rdns_upstream_elt *elt, - void *ups_data) + void *ups_data, const gchar *reason) { struct upstream *up = elt->lib_data; - rspamd_upstream_fail (up, FALSE); + rspamd_upstream_fail (up, FALSE, reason); } static unsigned int diff --git a/src/libserver/dynamic_cfg.c b/src/libserver/dynamic_cfg.c index 45c6838ec..a39778ec2 100644 --- a/src/libserver/dynamic_cfg.c +++ b/src/libserver/dynamic_cfg.c @@ -283,7 +283,7 @@ init_dynamic_config (struct rspamd_config *cfg) json_config_read_cb, json_config_fin_cb, json_config_dtor_cb, - (void **)pjb)) { + (void **)pjb, NULL)) { msg_err ("cannot add map for configuration %s", cfg->dynamic_conf); } } diff --git a/src/libserver/fuzzy_backend_redis.c b/src/libserver/fuzzy_backend_redis.c index ed0813edf..7070773b3 100644 --- a/src/libserver/fuzzy_backend_redis.c +++ b/src/libserver/fuzzy_backend_redis.c @@ -444,7 +444,7 @@ rspamd_fuzzy_redis_shingles_callback (redisAsyncContext *c, gpointer r, msg_err_redis_session ("error getting shingles: %s", c->errstr); } - rspamd_upstream_fail (session->up, FALSE); + rspamd_upstream_fail (session->up, FALSE, strerror (errno)); } rspamd_fuzzy_redis_session_dtor (session, FALSE); @@ -582,7 +582,7 @@ rspamd_fuzzy_redis_check_callback (redisAsyncContext *c, gpointer r, msg_err_redis_session ("error getting hashes: %s", c->errstr); } - rspamd_upstream_fail (session->up, FALSE); + rspamd_upstream_fail (session->up, FALSE, strerror (errno)); } rspamd_fuzzy_redis_session_dtor (session, FALSE); @@ -651,7 +651,7 @@ rspamd_fuzzy_backend_check_redis (struct rspamd_fuzzy_backend *bk, rspamd_inet_address_get_port (addr)); if (session->ctx == NULL) { - rspamd_upstream_fail (up, TRUE); + rspamd_upstream_fail (up, TRUE, strerror (errno)); rspamd_fuzzy_redis_session_dtor (session, TRUE); if (cb) { @@ -721,7 +721,7 @@ rspamd_fuzzy_redis_count_callback (redisAsyncContext *c, gpointer r, msg_err_redis_session ("error getting count: %s", c->errstr); } - rspamd_upstream_fail (session->up, FALSE); + rspamd_upstream_fail (session->up, FALSE, strerror (errno)); } rspamd_fuzzy_redis_session_dtor (session, FALSE); @@ -776,7 +776,7 @@ rspamd_fuzzy_backend_count_redis (struct rspamd_fuzzy_backend *bk, rspamd_inet_address_get_port (addr)); if (session->ctx == NULL) { - rspamd_upstream_fail (up, TRUE); + rspamd_upstream_fail (up, TRUE, strerror (errno)); rspamd_fuzzy_redis_session_dtor (session, TRUE); if (cb) { @@ -844,7 +844,7 @@ rspamd_fuzzy_redis_version_callback (redisAsyncContext *c, gpointer r, msg_err_redis_session ("error getting version: %s", c->errstr); } - rspamd_upstream_fail (session->up, FALSE); + rspamd_upstream_fail (session->up, FALSE, strerror (errno)); } rspamd_fuzzy_redis_session_dtor (session, FALSE); @@ -900,7 +900,7 @@ rspamd_fuzzy_backend_version_redis (struct rspamd_fuzzy_backend *bk, rspamd_inet_address_get_port (addr)); if (session->ctx == NULL) { - rspamd_upstream_fail (up, FALSE); + rspamd_upstream_fail (up, FALSE, strerror (errno)); rspamd_fuzzy_redis_session_dtor (session, TRUE); if (cb) { @@ -1334,7 +1334,7 @@ rspamd_fuzzy_redis_update_callback (redisAsyncContext *c, gpointer r, msg_err_redis_session ("error sending update to redis: %s", c->errstr); } - rspamd_upstream_fail (session->up, FALSE); + rspamd_upstream_fail (session->up, FALSE, strerror (errno)); } rspamd_fuzzy_redis_session_dtor (session, FALSE); @@ -1460,7 +1460,7 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk, rspamd_inet_address_get_port (addr)); if (session->ctx == NULL) { - rspamd_upstream_fail (up, TRUE); + rspamd_upstream_fail (up, TRUE, strerror (errno)); rspamd_fuzzy_redis_session_dtor (session, TRUE); if (cb) { diff --git a/src/libserver/fuzzy_backend_sqlite.c b/src/libserver/fuzzy_backend_sqlite.c index 90c0db70d..0f9b3c1ee 100644 --- a/src/libserver/fuzzy_backend_sqlite.c +++ b/src/libserver/fuzzy_backend_sqlite.c @@ -431,7 +431,8 @@ rspamd_fuzzy_backend_sqlite_open_db (const gchar *path, GError **err) bk = g_malloc0 (sizeof (*bk)); bk->path = g_strdup (path); bk->expired = 0; - bk->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "fuzzy_backend"); + bk->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), + "fuzzy_backend", 0); bk->db = rspamd_sqlite3_open_or_create (bk->pool, bk->path, create_tables_sql, 1, err); diff --git a/src/libserver/html.c b/src/libserver/html.c index de6e104b4..a6b037861 100644 --- a/src/libserver/html.c +++ b/src/libserver/html.c @@ -60,7 +60,7 @@ static struct html_tag_def tag_defs[] = { TAG_DEF(Tag_APPLET, "applet", (CM_OBJECT | CM_IMG | CM_INLINE | CM_PARAM)), TAG_DEF(Tag_AREA, "area", (CM_BLOCK | CM_EMPTY | FL_HREF)), TAG_DEF(Tag_B, "b", (CM_INLINE|FL_BLOCK)), - TAG_DEF(Tag_BASE, "base", (CM_HEAD | CM_EMPTY | FL_HREF)), + TAG_DEF(Tag_BASE, "base", (CM_HEAD | CM_EMPTY)), TAG_DEF(Tag_BASEFONT, "basefont", (CM_INLINE | CM_EMPTY)), TAG_DEF(Tag_BDO, "bdo", (CM_INLINE)), TAG_DEF(Tag_BIG, "big", (CM_INLINE)), @@ -815,8 +815,6 @@ rspamd_html_process_tag (rspamd_mempool_t *pool, struct html_content *hc, return TRUE; } } - - parent->content_length += tag->content_length; } if (hc->total_tags < max_tags) { @@ -1801,6 +1799,7 @@ rspamd_html_process_img_tag (rspamd_mempool_t *pool, struct html_tag *tag, if (hc->images == NULL) { hc->images = g_ptr_array_sized_new (4); + rspamd_mempool_notify_alloc (pool, 4 * sizeof (gpointer) + sizeof (GPtrArray)); rspamd_mempool_add_destructor (pool, rspamd_ptr_array_free_hard, hc->images); } @@ -2370,6 +2369,7 @@ rspamd_html_process_block_tag (rspamd_mempool_t *pool, struct html_tag *tag, if (hc->blocks == NULL) { hc->blocks = g_ptr_array_sized_new (64); + rspamd_mempool_notify_alloc (pool, 64 * sizeof (gpointer) + sizeof (GPtrArray)); rspamd_mempool_add_destructor (pool, rspamd_ptr_array_free_hard, hc->blocks); } @@ -2772,13 +2772,6 @@ rspamd_html_process_part_full (rspamd_mempool_t *pool, struct html_content *hc, p ++; } else { - if (content_tag) { - if (content_tag->content == NULL) { - content_tag->content = c; - } - - content_tag->content_length += p - c; - } state = tag_begin; } break; @@ -2796,24 +2789,35 @@ rspamd_html_process_part_full (rspamd_mempool_t *pool, struct html_content *hc, if (need_decode) { goffset old_offset = dest->len; + if (content_tag) { + if (content_tag->content_length == 0) { + content_tag->content_offset = old_offset; + } + } + g_byte_array_append (dest, c, (p - c)); len = rspamd_html_decode_entitles_inplace ( dest->data + old_offset, p - c); dest->len = dest->len + len - (p - c); + + if (content_tag) { + content_tag->content_length += len; + } } else { len = p - c; - g_byte_array_append (dest, c, len); - } - if (content_tag) { - if (content_tag->content == NULL) { - content_tag->content = c; + if (content_tag) { + if (content_tag->content_length == 0) { + content_tag->content_offset = dest->len; + } + + content_tag->content_length += len; } - content_tag->content_length += p - c + 1; + g_byte_array_append (dest, c, len); } } @@ -2826,6 +2830,20 @@ rspamd_html_process_part_full (rspamd_mempool_t *pool, struct html_content *hc, if (dest->len > 0 && !g_ascii_isspace (dest->data[dest->len - 1])) { g_byte_array_append (dest, " ", 1); + if (content_tag) { + if (content_tag->content_length == 0) { + /* + * Special case + * we have a space at the beginning but + * we have no set content_offset + * so we need to do it here + */ + content_tag->content_offset = dest->len; + } + else { + content_tag->content_length++; + } + } } save_space = FALSE; } @@ -2837,24 +2855,34 @@ rspamd_html_process_part_full (rspamd_mempool_t *pool, struct html_content *hc, if (need_decode) { goffset old_offset = dest->len; + if (content_tag) { + if (content_tag->content_length == 0) { + content_tag->content_offset = dest->len; + } + } + g_byte_array_append (dest, c, (p - c)); len = rspamd_html_decode_entitles_inplace ( dest->data + old_offset, p - c); dest->len = dest->len + len - (p - c); + + if (content_tag) { + content_tag->content_length += len; + } } else { len = p - c; - g_byte_array_append (dest, c, len); - } + if (content_tag) { + if (content_tag->content_length == 0) { + content_tag->content_offset = dest->len; + } - if (content_tag) { - if (content_tag->content == NULL) { - content_tag->content = c; + content_tag->content_length += len; } - content_tag->content_length += p - c; + g_byte_array_append (dest, c, len); } } @@ -2874,10 +2902,6 @@ rspamd_html_process_part_full (rspamd_mempool_t *pool, struct html_content *hc, continue; } - if (content_tag) { - content_tag->content_length ++; - } - p ++; break; @@ -2947,6 +2971,21 @@ rspamd_html_process_part_full (rspamd_mempool_t *pool, struct html_content *hc, if (cur_tag->id == Tag_BR || cur_tag->id == Tag_HR) { if (dest->len > 0 && dest->data[dest->len - 1] != '\n') { g_byte_array_append (dest, "\r\n", 2); + + if (content_tag) { + if (content_tag->content_length == 0) { + /* + * Special case + * we have a \r\n at the beginning but + * we have no set content_offset + * so we need to do it here + */ + content_tag->content_offset = dest->len; + } + else { + content_tag->content_length += 2; + } + } } save_space = FALSE; } @@ -2956,6 +2995,21 @@ rspamd_html_process_part_full (rspamd_mempool_t *pool, struct html_content *hc, cur_tag->id == Tag_DIV)) { if (dest->len > 0 && dest->data[dest->len - 1] != '\n') { g_byte_array_append (dest, "\r\n", 2); + + if (content_tag) { + if (content_tag->content_length == 0) { + /* + * Special case + * we have a \r\n at the beginning but + * we have no set content_offset + * so we need to get it here + */ + content_tag->content_offset = dest->len; + } + else { + content_tag->content_length += 2; + } + } } save_space = FALSE; } @@ -3031,28 +3085,20 @@ rspamd_html_process_part_full (rspamd_mempool_t *pool, struct html_content *hc, } } else if (cur_tag->id == Tag_BASE && !(cur_tag->flags & (FL_CLOSING))) { - struct html_tag *prev_tag = NULL; - - if (cur_level && cur_level->parent) { - prev_tag = cur_level->parent->data; - } - /* - * Base is allowed only within head tag but we slightly - * relax that + * Base is allowed only within head tag but HTML is retarded */ - if (!prev_tag || prev_tag->id == Tag_HEAD || - prev_tag->id == Tag_HTML) { + if (hc->base_url == NULL) { url = rspamd_html_process_url_tag (pool, cur_tag, hc); if (url != NULL) { - if (hc->base_url == NULL) { - /* We have a base tag available */ - hc->base_url = url; - } - + msg_debug_html ("got valid base tag"); + hc->base_url = url; cur_tag->extra = url; } + else { + msg_debug_html ("got invalid base tag!"); + } } } @@ -3112,6 +3158,7 @@ rspamd_html_process_part_full (rspamd_mempool_t *pool, struct html_content *hc, } g_queue_free (styles_blocks); + hc->parsed = dest; return dest; } diff --git a/src/libserver/html.h b/src/libserver/html.h index 86a266a62..b369bd890 100644 --- a/src/libserver/html.h +++ b/src/libserver/html.h @@ -107,9 +107,9 @@ struct html_block { struct html_tag { gint id; gint flags; - guint content_length; struct html_tag_component name; - const gchar *content; + guint content_length; + goffset content_offset; GQueue *params; gpointer extra; /** Additional data associated with tag (e.g. image) */ GNode *parent; @@ -127,6 +127,7 @@ struct html_content { guchar *tags_seen; GPtrArray *images; GPtrArray *blocks; + GByteArray *parsed; }; /* diff --git a/src/libserver/milter.c b/src/libserver/milter.c index 5cef1dfa2..bc33708ff 100644 --- a/src/libserver/milter.c +++ b/src/libserver/milter.c @@ -955,6 +955,8 @@ rspamd_milter_handle_session (struct rspamd_milter_session *session, if (r == -1) { if (errno == EAGAIN || errno == EINTR) { rspamd_milter_plan_io (session, priv, EV_READ); + + return TRUE; } else { /* Fatal IO error */ @@ -964,6 +966,10 @@ rspamd_milter_handle_session (struct rspamd_milter_session *session, priv->err_cb (priv->fd, session, priv->ud, err); REF_RELEASE (session); g_error_free (err); + + REF_RELEASE (session); + + return FALSE; } } else if (r == 0) { @@ -973,6 +979,10 @@ rspamd_milter_handle_session (struct rspamd_milter_session *session, priv->err_cb (priv->fd, session, priv->ud, err); REF_RELEASE (session); g_error_free (err); + + REF_RELEASE (session); + + return FALSE; } else { priv->parser.buf->len += r; @@ -1023,6 +1033,8 @@ rspamd_milter_handle_session (struct rspamd_milter_session *session, REF_RELEASE (session); g_error_free (err); + REF_RELEASE (session); + return FALSE; } } @@ -1034,6 +1046,8 @@ rspamd_milter_handle_session (struct rspamd_milter_session *session, REF_RELEASE (session); g_error_free (err); + REF_RELEASE (session); + return FALSE; } else { @@ -1060,6 +1074,7 @@ rspamd_milter_handle_session (struct rspamd_milter_session *session, case RSPAMD_MILTER_WANNA_DIE: /* We are here after processing everything, so release session */ REF_RELEASE (session); + return FALSE; break; } @@ -1090,7 +1105,7 @@ rspamd_milter_handle_socket (gint fd, ev_tstamp timeout, priv->parser.buf = rspamd_fstring_sized_new (RSPAMD_MILTER_MESSAGE_CHUNK + 5); priv->event_loop = ev_base; priv->state = RSPAMD_MILTER_READ_MORE; - priv->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "milter"); + priv->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "milter", 0); priv->discard_on_reject = milter_ctx->discard_on_reject; priv->quarantine_on_reject = milter_ctx->quarantine_on_reject; priv->ev.timeout = timeout; diff --git a/src/libserver/protocol.c b/src/libserver/protocol.c index 0786f4860..9ffdcd5c7 100644 --- a/src/libserver/protocol.c +++ b/src/libserver/protocol.c @@ -26,6 +26,7 @@ #include "unix-std.h" #include "protocol_internal.h" #include "libserver/mempool_vars_internal.h" +#include "contrib/fastutf8/fastutf8.h" #include "task.h" #include <math.h> @@ -873,15 +874,15 @@ rspamd_protocol_extended_url (struct rspamd_task *task, obj = ucl_object_typed_new (UCL_OBJECT); - elt = ucl_object_fromlstring (encoded, enclen); + elt = ucl_object_fromstring_common (encoded, enclen, 0); ucl_object_insert_key (obj, elt, "url", 0, false); if (url->tldlen > 0) { - elt = ucl_object_fromlstring (url->tld, url->tldlen); + elt = ucl_object_fromstring_common (url->tld, url->tldlen, 0); ucl_object_insert_key (obj, elt, "tld", 0, false); } if (url->hostlen > 0) { - elt = ucl_object_fromlstring (url->host, url->hostlen); + elt = ucl_object_fromstring_common (url->host, url->hostlen, 0); ucl_object_insert_key (obj, elt, "host", 0, false); } @@ -922,16 +923,13 @@ urls_protocol_cb (gpointer key, gpointer value, gpointer ud) return; } - const gchar *end = NULL; + goffset err_offset; - if (g_utf8_validate (url->host, url->hostlen, &end)) { - obj = ucl_object_fromlstring (url->host, url->hostlen); - } - else if (end - url->host > 0) { - obj = ucl_object_fromlstring (url->host, end - url->host); + if ((err_offset = rspamd_fast_utf8_validate (url->host, url->hostlen)) == 0) { + obj = ucl_object_fromstring_common (url->host, url->hostlen, 0); } else { - return; + obj = ucl_object_fromstring_common (url->host, err_offset - 1, 0); } } else { @@ -2029,7 +2027,21 @@ rspamd_protocol_write_reply (struct rspamd_task *task, ev_tstamp timeout) reply = rspamd_fstring_sized_new (256); rspamd_ucl_emit_fstring (top, UCL_EMIT_JSON_COMPACT, &reply); ucl_object_unref (top); - rspamd_http_message_set_body_from_fstring_steal (msg, reply); + + /* We also need to validate utf8 */ + if (rspamd_fast_utf8_validate (reply->str, reply->len) != 0) { + gsize valid_len; + gchar *validated; + + /* We copy reply several times here but it should be a rare case */ + validated = rspamd_str_make_utf_valid (reply->str, reply->len, + &valid_len, task->task_pool); + rspamd_http_message_set_body (msg, validated, valid_len); + rspamd_fstring_free (reply); + } + else { + rspamd_http_message_set_body_from_fstring_steal (msg, reply); + } } else { msg->status = rspamd_fstring_new_init ("OK", 2); diff --git a/src/libserver/re_cache.c b/src/libserver/re_cache.c index a9fc2270b..6889861cc 100644 --- a/src/libserver/re_cache.c +++ b/src/libserver/re_cache.c @@ -30,9 +30,12 @@ #ifdef WITH_HYPERSCAN #include "hs.h" +#endif + #include "unix-std.h" #include <signal.h> #include <stdalign.h> +#include <math.h> #include "contrib/libev/ev.h" #ifndef WITH_PCRE2 @@ -41,12 +44,12 @@ #include <pcre2.h> #endif +#include "contrib/fastutf8/fastutf8.h" + #ifdef HAVE_SYS_WAIT_H #include <sys/wait.h> #endif -#endif - #define msg_err_re_cache(...) rspamd_default_log_function (G_LOG_LEVEL_CRITICAL, \ "re_cache", cache->hash, \ G_STRFUNC, \ @@ -536,7 +539,7 @@ rspamd_re_cache_process_pcre (struct rspamd_re_runtime *rt, const gchar *start = NULL, *end = NULL; guint max_hits = rspamd_regexp_get_maxhits (re); guint64 id = rspamd_regexp_get_cache_id (re); - gdouble t1, t2, pr; + gdouble t1 = NAN, t2, pr; const gdouble slow_time = 1e8; if (in == NULL) { @@ -585,10 +588,11 @@ rspamd_re_cache_process_pcre (struct rspamd_re_runtime *rt, rt->stat.regexp_matched += r; } - if (pr > 0.9) { + if (!isnan (t1)) { t2 = rspamd_get_ticks (TRUE); if (t2 - t1 > slow_time) { + rspamd_symcache_enable_profile (task); msg_info_task ("regexp '%16s' took %.0f ticks to execute", rspamd_regexp_get_pattern (re), t2 - t1); } @@ -988,7 +992,7 @@ rspamd_re_cache_process_headers_list (struct rspamd_task *task, in = (const guchar *)cur->value; lenvec[i] = strlen (cur->value); - if (!g_utf8_validate (in, lenvec[i], NULL)) { + if (rspamd_fast_utf8_validate (in, lenvec[i]) != 0) { raw = TRUE; } } @@ -1627,12 +1631,12 @@ rspamd_re_cache_is_finite (struct rspamd_re_cache *cache, gdouble wait_time; const gint max_tries = 10; gint tries = 0, rc; + void (*old_hdl)(int); wait_time = max_time / max_tries; /* We need to restore SIGCHLD processing */ - signal (SIGCHLD, SIG_DFL); + old_hdl = signal (SIGCHLD, SIG_DFL); cld = fork (); - g_assert (cld != -1); if (cld == 0) { /* Try to compile pattern */ @@ -1651,7 +1655,7 @@ rspamd_re_cache_is_finite (struct rspamd_re_cache *cache, exit (EXIT_SUCCESS); } - else { + else if (cld > 0) { double_to_ts (wait_time, &ts); while ((rc = waitpid (cld, &status, WNOHANG)) == 0 && tries ++ < max_tries) { @@ -1661,7 +1665,7 @@ rspamd_re_cache_is_finite (struct rspamd_re_cache *cache, /* Child has been terminated */ if (rc > 0) { /* Forget about SIGCHLD after this point */ - signal (SIGCHLD, SIG_IGN); + signal (SIGCHLD, old_hdl); if (WIFEXITED (status) && WEXITSTATUS (status) == EXIT_SUCCESS) { return TRUE; @@ -1681,9 +1685,15 @@ rspamd_re_cache_is_finite (struct rspamd_re_cache *cache, msg_err_re_cache ( "cannot approximate %s to hyperscan: timeout waiting", rspamd_regexp_get_pattern (re)); - signal (SIGCHLD, SIG_IGN); + signal (SIGCHLD, old_hdl); } } + else { + msg_err_re_cache ( + "cannot approximate %s to hyperscan: fork failed: %s", + rspamd_regexp_get_pattern (re), strerror (errno)); + signal (SIGCHLD, old_hdl); + } return FALSE; } @@ -2018,8 +2028,6 @@ rspamd_re_cache_compile_timer_cb (EV_P_ ev_timer *w, int revents ) g_free (hs_flags); } - fsync (fd); - /* Now rename temporary file to the new .hs file */ rspamd_snprintf (npath, sizeof (path), "%s%c%s.hs", cbdata->cache_dir, G_DIR_SEPARATOR, re_class->hash); diff --git a/src/libserver/redis_pool.c b/src/libserver/redis_pool.c index 0d310d968..57fc734f9 100644 --- a/src/libserver/redis_pool.c +++ b/src/libserver/redis_pool.c @@ -26,12 +26,18 @@ struct rspamd_redis_pool_elt; +enum rspamd_redis_pool_connection_state { + RSPAMD_REDIS_POOL_CONN_INACTIVE = 0, + RSPAMD_REDIS_POOL_CONN_ACTIVE, + RSPAMD_REDIS_POOL_CONN_FINALISING +}; + struct rspamd_redis_pool_connection { struct redisAsyncContext *ctx; struct rspamd_redis_pool_elt *elt; GList *entry; ev_timer timeout; - gboolean active; + enum rspamd_redis_pool_connection_state state; gchar tag[MEMPOOL_UID_LEN]; ref_entry_t ref; }; @@ -99,7 +105,7 @@ rspamd_redis_pool_get_key (const gchar *db, const gchar *password, static void rspamd_redis_pool_conn_dtor (struct rspamd_redis_pool_connection *conn) { - if (conn->active) { + if (conn->state == RSPAMD_REDIS_POOL_CONN_ACTIVE) { msg_debug_rpool ("active connection removed"); if (conn->ctx) { @@ -126,7 +132,7 @@ rspamd_redis_pool_conn_dtor (struct rspamd_redis_pool_connection *conn) redisAsyncContext *ac = conn->ctx; /* To prevent on_disconnect here */ - conn->active = TRUE; + conn->state = RSPAMD_REDIS_POOL_CONN_FINALISING; g_hash_table_remove (conn->elt->pool->elts_by_ctx, ac); conn->ctx = NULL; ac->onDisconnect = NULL; @@ -171,15 +177,57 @@ rspamd_redis_pool_elt_dtor (gpointer p) } static void +rspamd_redis_on_quit (redisAsyncContext *c, gpointer r, gpointer priv) +{ + struct rspamd_redis_pool_connection *conn = + (struct rspamd_redis_pool_connection *)priv; + + msg_debug_rpool ("quit command reply for the connection %p, refcount: %d", + conn->ctx, conn->ref.refcount); + /* + * The connection will be freed by hiredis itself as we are here merely after + * quit command has succeeded and we have timer being set already. + * The problem is that when this callback is called, our connection is likely + * dead, so probably even on_disconnect callback has been already called... + * + * Hence, the connection might already be freed, so even (conn) pointer may be + * inaccessible. + * + * TODO: Use refcounts to prevent this stuff to happen, the problem is how + * to handle Redis timeout on `quit` command in fact... The good thing is that + * it will not likely happen. + */ +} + +static void rspamd_redis_conn_timeout (EV_P_ ev_timer *w, int revents) { struct rspamd_redis_pool_connection *conn = (struct rspamd_redis_pool_connection *)w->data; - g_assert (!conn->active); - msg_debug_rpool ("scheduled removal of connection %p, refcount: %d", - conn->ctx, conn->ref.refcount); - REF_RELEASE (conn); + g_assert (conn->state != RSPAMD_REDIS_POOL_CONN_ACTIVE); + + if (conn->state == RSPAMD_REDIS_POOL_CONN_INACTIVE) { + msg_debug_rpool ("scheduled soft removal of connection %p, refcount: %d", + conn->ctx, conn->ref.refcount); + /* Prevent reusing */ + if (conn->entry) { + g_queue_unlink (conn->elt->inactive, conn->entry); + conn->entry = NULL; + } + + conn->state = RSPAMD_REDIS_POOL_CONN_FINALISING; + ev_timer_again (EV_A_ w); + redisAsyncCommand (conn->ctx, rspamd_redis_on_quit, conn, "QUIT"); + } + else { + /* Finalising by timeout */ + ev_timer_stop (EV_A_ w); + msg_debug_rpool ("final removal of connection %p, refcount: %d", + conn->ctx, conn->ref.refcount); + REF_RELEASE (conn); + } + } static void @@ -205,7 +253,7 @@ rspamd_redis_pool_schedule_timeout (struct rspamd_redis_pool_connection *conn) conn->timeout.data = conn; ev_timer_init (&conn->timeout, rspamd_redis_conn_timeout, - real_timeout, 0.0); + real_timeout, real_timeout / 2.0); ev_timer_start (conn->elt->pool->event_loop, &conn->timeout); } @@ -219,8 +267,7 @@ rspamd_redis_pool_on_disconnect (const struct redisAsyncContext *ac, int status, * Here, we know that redis itself will free this connection * so, we need to do something very clever about it */ - - if (!conn->active) { + if (conn->state != RSPAMD_REDIS_POOL_CONN_ACTIVE) { /* Do nothing for active connections as it is already handled somewhere */ if (conn->ctx) { msg_debug_rpool ("inactive connection terminated: %s, refs: %d", @@ -261,7 +308,7 @@ rspamd_redis_pool_new_connection (struct rspamd_redis_pool *pool, conn = g_malloc0 (sizeof (*conn)); conn->entry = g_list_prepend (NULL, conn); conn->elt = elt; - conn->active = TRUE; + conn->state = RSPAMD_REDIS_POOL_CONN_ACTIVE; g_hash_table_insert (elt->pool->elts_by_ctx, ctx, conn); g_queue_push_head_link (elt->active, conn->entry); @@ -275,10 +322,12 @@ rspamd_redis_pool_new_connection (struct rspamd_redis_pool *pool, conn); if (password) { - redisAsyncCommand (ctx, NULL, NULL, "AUTH %s", password); + redisAsyncCommand (ctx, NULL, NULL, + "AUTH %s", password); } if (db) { - redisAsyncCommand (ctx, NULL, NULL, "SELECT %s", db); + redisAsyncCommand (ctx, NULL, NULL, + "SELECT %s", db); } } @@ -307,8 +356,8 @@ rspamd_redis_pool_init (void) struct rspamd_redis_pool *pool; pool = g_malloc0 (sizeof (*pool)); - pool->elts_by_key = g_hash_table_new_full (g_int64_hash, g_int64_equal, NULL, - rspamd_redis_pool_elt_dtor); + pool->elts_by_key = g_hash_table_new_full (g_int64_hash, g_int64_equal, + NULL, rspamd_redis_pool_elt_dtor); pool->elts_by_ctx = g_hash_table_new (g_direct_hash, g_direct_equal); return pool; @@ -349,11 +398,11 @@ rspamd_redis_pool_connect (struct rspamd_redis_pool *pool, if (g_queue_get_length (elt->inactive) > 0) { conn_entry = g_queue_pop_head_link (elt->inactive); conn = conn_entry->data; - g_assert (!conn->active); + g_assert (conn->state != RSPAMD_REDIS_POOL_CONN_ACTIVE); if (conn->ctx->err == REDIS_OK) { ev_timer_stop (elt->pool->event_loop, &conn->timeout); - conn->active = TRUE; + conn->state = RSPAMD_REDIS_POOL_CONN_ACTIVE; g_queue_push_tail_link (elt->active, conn_entry); msg_debug_rpool ("reused existing connection to %s:%d: %p", ip, port, conn->ctx); @@ -404,7 +453,7 @@ rspamd_redis_pool_release_connection (struct rspamd_redis_pool *pool, conn = g_hash_table_lookup (pool->elts_by_ctx, ctx); if (conn != NULL) { - g_assert (conn->active); + g_assert (conn->state == RSPAMD_REDIS_POOL_CONN_ACTIVE); if (ctx->err != REDIS_OK) { /* We need to terminate connection forcefully */ @@ -418,7 +467,7 @@ rspamd_redis_pool_release_connection (struct rspamd_redis_pool *pool, /* Just move it to the inactive queue */ g_queue_unlink (conn->elt->active, conn->entry); g_queue_push_head_link (conn->elt->inactive, conn->entry); - conn->active = FALSE; + conn->state = RSPAMD_REDIS_POOL_CONN_INACTIVE; rspamd_redis_pool_schedule_timeout (conn); msg_debug_rpool ("mark connection %p inactive", conn->ctx); } diff --git a/src/libserver/rspamd_symcache.c b/src/libserver/rspamd_symcache.c index 60f405e12..212ae76e7 100644 --- a/src/libserver/rspamd_symcache.c +++ b/src/libserver/rspamd_symcache.c @@ -99,12 +99,12 @@ struct rspamd_symcache_id_list { struct rspamd_symcache_item { /* This block is likely shared */ - struct item_stat *st; + struct rspamd_symcache_item_stat *st; guint64 last_count; struct rspamd_counter_data *cd; gchar *symbol; - enum rspamd_symbol_type type; + gint type; /* Callback data */ union { @@ -115,6 +115,7 @@ struct rspamd_symcache_item { } normal; struct { gint parent; + struct rspamd_symcache_item *parent_item; } virtual; } specific; @@ -144,17 +145,6 @@ struct rspamd_symcache_item { GPtrArray *container; }; -struct item_stat { - struct rspamd_counter_data time_counter; - gdouble avg_time; - gdouble weight; - guint hits; - guint64 total_hits; - struct rspamd_counter_data frequency_counter; - gdouble avg_frequency; - gdouble stddev_frequency; -}; - struct rspamd_symcache { /* Hash table for fast access */ GHashTable *items_by_symbol; @@ -214,6 +204,7 @@ struct cache_savepoint { guint version; guint items_inflight; gboolean profile; + gboolean has_slow; gdouble profile_start; struct rspamd_scan_result *rs; @@ -324,8 +315,7 @@ rspamd_symcache_find_filter (struct rspamd_symcache *cache, if (item != NULL) { if (resolve_parent && item->is_virtual && !(item->type & SYMBOL_TYPE_GHOST)) { - item = g_ptr_array_index (cache->items_by_id, - item->specific.virtual.parent); + item =item->specific.virtual.parent_item; } return item; @@ -338,7 +328,7 @@ const gchar * rspamd_symcache_get_parent (struct rspamd_symcache *cache, const gchar *symbol) { - struct rspamd_symcache_item *item; + struct rspamd_symcache_item *item, *parent; g_assert (cache != NULL); @@ -351,8 +341,15 @@ rspamd_symcache_get_parent (struct rspamd_symcache *cache, if (item != NULL) { if (item->is_virtual && !(item->type & SYMBOL_TYPE_GHOST)) { - item = g_ptr_array_index (cache->items_by_id, - item->specific.virtual.parent); + parent = item->specific.virtual.parent_item; + + if (!parent) { + item->specific.virtual.parent_item = g_ptr_array_index (cache->items_by_id, + item->specific.virtual.parent); + parent = item->specific.virtual.parent_item; + } + + item = parent; } return item->symbol; @@ -893,6 +890,7 @@ rspamd_symcache_load_items (struct rspamd_symcache *cache, const gchar *name) g_assert (item->specific.virtual.parent < (gint)cache->items_by_id->len); parent = g_ptr_array_index (cache->items_by_id, item->specific.virtual.parent); + item->specific.virtual.parent_item = parent; if (parent->st->weight < item->st->weight) { parent->st->weight = item->st->weight; @@ -1160,6 +1158,8 @@ rspamd_symcache_add_symbol (struct rspamd_symcache *cache, else { item->is_virtual = TRUE; item->specific.virtual.parent = parent; + item->specific.virtual.parent_item = + g_ptr_array_index (cache->items_by_id, parent); item->id = cache->virtual->len; g_ptr_array_add (cache->virtual, item); item->container = cache->virtual; @@ -1320,7 +1320,7 @@ rspamd_symcache_new (struct rspamd_config *cfg) cache = g_malloc0 (sizeof (struct rspamd_symcache)); cache->static_pool = - rspamd_mempool_new (rspamd_mempool_suggest_size (), "symcache"); + rspamd_mempool_new (rspamd_mempool_suggest_size (), "symcache", 0); cache->items_by_symbol = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); cache->items_by_id = g_ptr_array_new (); @@ -1428,6 +1428,7 @@ rspamd_symcache_validate_cb (gpointer k, gpointer v, gpointer ud) g_assert (item->specific.virtual.parent < (gint) cache->items_by_id->len); parent = g_ptr_array_index (cache->items_by_id, item->specific.virtual.parent); + item->specific.virtual.parent_item = parent; if (fabs (parent->st->weight) < fabs (item->st->weight)) { parent->st->weight = item->st->weight; @@ -1495,7 +1496,8 @@ rspamd_symcache_validate (struct rspamd_symcache *cache, ignore_symbol = FALSE; sym_def = v; - if (sym_def && (sym_def->flags & RSPAMD_SYMBOL_FLAG_IGNORE)) { + if (sym_def && (sym_def->flags & + (RSPAMD_SYMBOL_FLAG_IGNORE_METRIC|RSPAMD_SYMBOL_FLAG_DISABLED))) { ignore_symbol = TRUE; } @@ -1512,6 +1514,13 @@ rspamd_symcache_validate (struct rspamd_symcache *cache, } } } + else if (sym_def->flags & RSPAMD_SYMBOL_FLAG_DISABLED) { + item = g_hash_table_lookup (cache->items_by_symbol, k); + + if (item) { + item->enabled = FALSE; + } + } } return ret; @@ -1590,14 +1599,18 @@ rspamd_symcache_is_item_allowed (struct rspamd_task *task, { const gchar *what = "execution"; + if (!exec_only) { + what = "symbol insertion"; + } + /* Static checks */ if (!item->enabled || (RSPAMD_TASK_IS_EMPTY (task) && !(item->type & SYMBOL_TYPE_EMPTY)) || (item->type & SYMBOL_TYPE_MIME_ONLY && !RSPAMD_TASK_IS_MIME(task))) { if (!item->enabled) { - msg_debug_cache_task ("skipping check of %s as it is permanently disabled", - item->symbol); + msg_debug_cache_task ("skipping %s of %s as it is permanently disabled", + what, item->symbol); return FALSE; } @@ -1615,10 +1628,6 @@ rspamd_symcache_is_item_allowed (struct rspamd_task *task, } } - if (!exec_only) { - what = "symbol insertion"; - } - /* Settings checks */ if (task->settings_elt != 0) { guint32 id = task->settings_elt->id; @@ -1753,7 +1762,8 @@ rspamd_symcache_check_symbol (struct rspamd_task *task, msg_debug_cache_task ("execute %s, %d", item->symbol, item->id); if (checkpoint->profile) { - dyn_item->start_msec = (rspamd_get_virtual_ticks () - + ev_now_update_if_cheap (task->event_loop); + dyn_item->start_msec = (ev_now (task->event_loop) - checkpoint->profile_start) * 1e3; } @@ -1906,14 +1916,15 @@ rspamd_symcache_make_checkpoint (struct rspamd_task *task, rspamd_symcache_order_unref, checkpoint->order); /* Calculate profile probability */ + ev_now_update_if_cheap (task->event_loop); ev_tstamp now = ev_now (task->event_loop); + checkpoint->profile_start = now; if ((cache->last_profile == 0.0 || now > cache->last_profile + PROFILE_MAX_TIME) || (task->msg.len >= PROFILE_MESSAGE_SIZE_THRESHOLD) || (rspamd_random_double_fast () >= (1 - PROFILE_PROBABILITY))) { msg_debug_cache_task ("enable profiling of symbols for task"); checkpoint->profile = TRUE; - checkpoint->profile_start = rspamd_get_virtual_ticks (); cache->last_profile = now; } @@ -2021,7 +2032,8 @@ rspamd_symcache_process_settings (struct rspamd_task *task, gboolean rspamd_symcache_process_symbols (struct rspamd_task *task, - struct rspamd_symcache *cache, gint stage) + struct rspamd_symcache *cache, + gint stage) { struct rspamd_symcache_item *item = NULL; struct rspamd_symcache_dynamic_item *dyn_item; @@ -2060,6 +2072,11 @@ rspamd_symcache_process_symbols (struct rspamd_task *task, if (!CHECK_START_BIT (checkpoint, dyn_item) && !CHECK_FINISH_BIT (checkpoint, dyn_item)) { + + if (checkpoint->has_slow) { + /* Delay */ + return FALSE; + } /* Check priorities */ if (saved_priority == G_MININT) { saved_priority = item->priority; @@ -2099,6 +2116,11 @@ rspamd_symcache_process_symbols (struct rspamd_task *task, if (!CHECK_START_BIT (checkpoint, dyn_item) && !CHECK_FINISH_BIT (checkpoint, dyn_item)) { /* Check priorities */ + if (checkpoint->has_slow) { + /* Delay */ + return FALSE; + } + if (saved_priority == G_MININT) { saved_priority = item->priority; } @@ -2151,6 +2173,11 @@ rspamd_symcache_process_symbols (struct rspamd_task *task, rspamd_symcache_check_symbol (task, cache, item, checkpoint); + + if (checkpoint->has_slow) { + /* Delay */ + return FALSE; + } } if (!(item->type & SYMBOL_TYPE_FINE)) { @@ -2185,6 +2212,11 @@ rspamd_symcache_process_symbols (struct rspamd_task *task, /* Check priorities */ all_done = FALSE; + if (checkpoint->has_slow) { + /* Delay */ + return FALSE; + } + if (saved_priority == G_MININT) { saved_priority = item->priority; } @@ -2218,6 +2250,11 @@ rspamd_symcache_process_symbols (struct rspamd_task *task, if (!CHECK_START_BIT (checkpoint, dyn_item) && !CHECK_FINISH_BIT (checkpoint, dyn_item)) { /* Check priorities */ + if (checkpoint->has_slow) { + /* Delay */ + return FALSE; + } + if (saved_priority == G_MININT) { saved_priority = item->priority; } @@ -2752,14 +2789,15 @@ rspamd_symcache_is_checked (struct rspamd_task *task, void rspamd_symcache_disable_symbol_perm (struct rspamd_symcache *cache, - const gchar *symbol) + const gchar *symbol, + gboolean resolve_parent) { struct rspamd_symcache_item *item; g_assert (cache != NULL); g_assert (symbol != NULL); - item = rspamd_symcache_find_filter (cache, symbol, true); + item = rspamd_symcache_find_filter (cache, symbol, resolve_parent); if (item) { item->enabled = FALSE; @@ -2930,7 +2968,7 @@ rspamd_symcache_disable_symbol (struct rspamd_task *task, void rspamd_symcache_foreach (struct rspamd_symcache *cache, - void (*func) (gint, const gchar *, gint, gpointer), + void (*func) (struct rspamd_symcache_item *, gpointer), gpointer ud) { struct rspamd_symcache_item *item; @@ -2941,7 +2979,7 @@ rspamd_symcache_foreach (struct rspamd_symcache *cache, while (g_hash_table_iter_next (&it, &k, &v)) { item = (struct rspamd_symcache_item *)v; - func (item->id, item->symbol, item->type, ud); + func (item, ud); } } @@ -2977,6 +3015,72 @@ rspamd_symcache_set_cur_item (struct rspamd_task *task, return ex; } +struct rspamd_symcache_delayed_cbdata { + struct rspamd_symcache_item *item; + struct rspamd_task *task; + struct rspamd_async_event *event; + struct ev_timer tm; +}; + +static void +rspamd_symcache_delayed_item_cb (EV_P_ ev_timer *w, int what) +{ + struct rspamd_symcache_delayed_cbdata *cbd = + (struct rspamd_symcache_delayed_cbdata *)w->data; + struct rspamd_symcache_item *item; + struct rspamd_task *task; + struct cache_dependency *rdep; + struct cache_savepoint *checkpoint; + struct rspamd_symcache_dynamic_item *dyn_item; + guint i; + + item = cbd->item; + task = cbd->task; + checkpoint = task->checkpoint; + checkpoint->has_slow = FALSE; + cbd->event = NULL; + ev_timer_stop (EV_A_ w); + + /* Process all reverse dependencies */ + PTR_ARRAY_FOREACH (item->rdeps, i, rdep) { + if (rdep->item) { + dyn_item = rspamd_symcache_get_dynamic (checkpoint, rdep->item); + if (!CHECK_START_BIT (checkpoint, dyn_item)) { + msg_debug_cache_task ("check item %d(%s) rdep of %s ", + rdep->item->id, rdep->item->symbol, item->symbol); + + if (!rspamd_symcache_check_deps (task, task->cfg->cache, + rdep->item, + checkpoint, 0, FALSE)) { + msg_debug_cache_task ("blocked execution of %d(%s) rdep of %s " + "unless deps are resolved", + rdep->item->id, rdep->item->symbol, item->symbol); + } + else { + rspamd_symcache_check_symbol (task, task->cfg->cache, + rdep->item, + checkpoint); + } + } + } + } + + rspamd_session_remove_event (task->s, NULL, cbd); +} + +static void +rspamd_delayed_timer_dtor (gpointer d) +{ + struct rspamd_symcache_delayed_cbdata *cbd = + (struct rspamd_symcache_delayed_cbdata *)d; + + if (cbd->event) { + ev_timer_stop (cbd->task->event_loop, &cbd->tm); + /* Event has not been executed */ + rspamd_session_remove_event (cbd->task->s, NULL, cbd); + cbd->event = NULL; + } +} /** * Finalize the current async element potentially calling its deps @@ -2990,6 +3094,7 @@ rspamd_symcache_finalize_item (struct rspamd_task *task, struct rspamd_symcache_dynamic_item *dyn_item; gdouble diff; guint i; + gboolean enable_slow_timer = FALSE; const gdouble slow_diff_limit = 300; /* Sanity checks */ @@ -3018,11 +3123,24 @@ rspamd_symcache_finalize_item (struct rspamd_task *task, checkpoint->cur_item = NULL; if (checkpoint->profile) { - diff = ((rspamd_get_virtual_ticks () - checkpoint->profile_start) * 1e3 - + ev_now_update_if_cheap (task->event_loop); + diff = ((ev_now (task->event_loop) - checkpoint->profile_start) * 1e3 - dyn_item->start_msec); + if (diff > slow_diff_limit) { - msg_info_task ("slow rule: %s(%d): %.2f ms", item->symbol, item->id, - diff); + + if (!checkpoint->has_slow) { + checkpoint->has_slow = TRUE; + enable_slow_timer = TRUE; + msg_info_task ("slow rule: %s(%d): %.2f ms; enable slow timer delay", + item->symbol, item->id, + diff); + } + else { + msg_info_task ("slow rule: %s(%d): %.2f ms", + item->symbol, item->id, + diff); + } } if (G_UNLIKELY (RSPAMD_TASK_IS_PROFILING (task))) { @@ -3034,6 +3152,38 @@ rspamd_symcache_finalize_item (struct rspamd_task *task, } } + if (enable_slow_timer) { + struct rspamd_symcache_delayed_cbdata *cbd = rspamd_mempool_alloc (task->task_pool, + sizeof (*cbd)); + /* Add timer to allow something else to be executed */ + ev_timer *tm = &cbd->tm; + + cbd->event = rspamd_session_add_event (task->s, NULL, cbd, + "symcache"); + + /* + * If no event could be added, then we are already in the destruction + * phase. So the main issue is to deal with has slow here + */ + if (cbd->event) { + ev_timer_init (tm, rspamd_symcache_delayed_item_cb, 0.1, 0.0); + ev_set_priority (tm, EV_MINPRI); + rspamd_mempool_add_destructor (task->task_pool, + rspamd_delayed_timer_dtor, cbd); + + cbd->task = task; + cbd->item = item; + tm->data = cbd; + ev_timer_start (task->event_loop, tm); + } + else { + /* Just reset as no timer is added */ + checkpoint->has_slow = FALSE; + } + + return; + } + /* Process all reverse dependencies */ PTR_ARRAY_FOREACH (item->rdeps, i, rdep) { if (rdep->item) { @@ -3505,7 +3655,7 @@ rspamd_symcache_process_settings_elt (struct rspamd_symcache *cache, } } -enum rspamd_symbol_type +gint rspamd_symcache_item_flags (struct rspamd_symcache_item *item) { if (item) { @@ -3513,4 +3663,95 @@ rspamd_symcache_item_flags (struct rspamd_symcache_item *item) } return 0; +} + +const gchar* +rspamd_symcache_item_name (struct rspamd_symcache_item *item) +{ + return item ? item->symbol : NULL; +} + +const struct rspamd_symcache_item_stat * +rspamd_symcache_item_stat (struct rspamd_symcache_item *item) +{ + return item ? item->st : NULL; +} + +gboolean +rspamd_symcache_item_is_enabled (struct rspamd_symcache_item *item) +{ + if (item) { + if (!item->enabled) { + return FALSE; + } + + if (item->is_virtual && item->specific.virtual.parent_item != NULL) { + return rspamd_symcache_item_is_enabled (item->specific.virtual.parent_item); + } + + return TRUE; + } + + return FALSE; +} + +struct rspamd_symcache_item * rspamd_symcache_item_get_parent ( + struct rspamd_symcache_item *item) +{ + if (item && item->is_virtual && item->specific.virtual.parent_item != NULL) { + return item->specific.virtual.parent_item; + } + + return NULL; +} + +const GPtrArray* +rspamd_symcache_item_get_deps (struct rspamd_symcache_item *item) +{ + struct rspamd_symcache_item *parent; + + if (item) { + parent = rspamd_symcache_item_get_parent (item); + + if (parent) { + item = parent; + } + + return item->deps; + } + + return NULL; +} + +const GPtrArray* +rspamd_symcache_item_get_rdeps (struct rspamd_symcache_item *item) +{ + struct rspamd_symcache_item *parent; + + if (item) { + parent = rspamd_symcache_item_get_parent (item); + + if (parent) { + item = parent; + } + + return item->rdeps; + } + + return NULL; +} + +void +rspamd_symcache_enable_profile (struct rspamd_task *task) +{ + struct cache_savepoint *checkpoint = task->checkpoint; + + if (checkpoint && !checkpoint->profile) { + ev_now_update_if_cheap (task->event_loop); + ev_tstamp now = ev_now (task->event_loop); + checkpoint->profile_start = now; + + msg_debug_cache_task ("enable profiling of symbols for task"); + checkpoint->profile = TRUE; + } }
\ No newline at end of file diff --git a/src/libserver/rspamd_symcache.h b/src/libserver/rspamd_symcache.h index 6542e76ce..c220b1ccc 100644 --- a/src/libserver/rspamd_symcache.h +++ b/src/libserver/rspamd_symcache.h @@ -68,6 +68,17 @@ struct rspamd_abstract_callback_data { char data[]; }; +struct rspamd_symcache_item_stat { + struct rspamd_counter_data time_counter; + gdouble avg_time; + gdouble weight; + guint hits; + guint64 total_hits; + struct rspamd_counter_data frequency_counter; + gdouble avg_frequency; + gdouble stddev_frequency; +}; + /** * Creates new cache structure * @return @@ -242,7 +253,8 @@ void rspamd_symcache_add_delayed_dependency (struct rspamd_symcache *cache, * @param symbol */ void rspamd_symcache_disable_symbol_perm (struct rspamd_symcache *cache, - const gchar *symbol); + const gchar *symbol, + gboolean resolve_parent); /** * Enable specific symbol in the cache @@ -356,8 +368,7 @@ gboolean rspamd_symcache_disable_symbol (struct rspamd_task *task, * @param ud */ void rspamd_symcache_foreach (struct rspamd_symcache *cache, - void (*func) (gint /* id */, const gchar * /* name */, - gint /* flags */, gpointer /* userdata */), + void (*func) (struct rspamd_symcache_item *item, gpointer /* userdata */), gpointer ud); /** @@ -515,8 +526,54 @@ gboolean rspamd_symcache_is_item_allowed (struct rspamd_task *task, * @param item * @return */ -enum rspamd_symbol_type rspamd_symcache_item_flags (struct rspamd_symcache_item *item); +gint rspamd_symcache_item_flags (struct rspamd_symcache_item *item); +/** + * Returns cache item name + * @param item + * @return + */ +const gchar* rspamd_symcache_item_name (struct rspamd_symcache_item *item); +/** + * Returns the current item stat + * @param item + * @return + */ +const struct rspamd_symcache_item_stat * + rspamd_symcache_item_stat (struct rspamd_symcache_item *item); +/** + * Returns if an item is enabled (for virutal it also means that parent should be enabled) + * @param item + * @return + */ +gboolean rspamd_symcache_item_is_enabled (struct rspamd_symcache_item *item); +/** + * Returns parent for virtual symbols (or NULL) + * @param item + * @return + */ +struct rspamd_symcache_item * rspamd_symcache_item_get_parent ( + struct rspamd_symcache_item *item); +/** + * Returns direct deps for an element + * @param item + * @return array of struct rspamd_symcache_item * + */ +const GPtrArray* rspamd_symcache_item_get_deps ( + struct rspamd_symcache_item *item); +/** + * Returns direct reverse deps for an element + * @param item + * @return array of struct rspamd_symcache_item * + */ +const GPtrArray* rspamd_symcache_item_get_rdeps ( + struct rspamd_symcache_item *item); + +/** + * Enable profiling for task (e.g. when a slow rule has been found) + * @param task + */ +void rspamd_symcache_enable_profile (struct rspamd_task *task); #ifdef __cplusplus } #endif diff --git a/src/libserver/spf.c b/src/libserver/spf.c index 46a6dd0a3..053269007 100644 --- a/src/libserver/spf.c +++ b/src/libserver/spf.c @@ -67,6 +67,7 @@ struct rspamd_spf_library_ctx { guint max_dns_requests; guint min_cache_ttl; gboolean disable_ipv6; + rspamd_lru_hash_t *spf_hash; }; struct rspamd_spf_library_ctx *spf_lib_ctx = NULL; @@ -110,6 +111,10 @@ struct rspamd_spf_library_ctx *spf_lib_ctx = NULL; rspamd_spf_log_id, "spf", rec->task->task_pool->tag.uid, \ G_STRFUNC, \ __VA_ARGS__) +#define msg_debug_spf_flatten(...) rspamd_conditional_debug_fast_num_id (NULL, NULL, \ + rspamd_spf_log_id, "spf", (flat)->digest, \ + G_STRFUNC, \ + __VA_ARGS__) INIT_LOG_MODULE(spf) @@ -149,15 +154,26 @@ RSPAMD_CONSTRUCTOR(rspamd_spf_lib_ctx_ctor) { } RSPAMD_DESTRUCTOR(rspamd_spf_lib_ctx_dtor) { + if (spf_lib_ctx->spf_hash) { + rspamd_lru_hash_destroy (spf_lib_ctx->spf_hash); + } g_free (spf_lib_ctx); spf_lib_ctx = NULL; } +static void +spf_record_cached_unref_dtor (gpointer p) +{ + struct spf_resolved *flat = (struct spf_resolved *)p; + + _spf_record_unref (flat, "LRU cache"); +} + void spf_library_config (const ucl_object_t *obj) { const ucl_object_t *value; - guint64 ival; + gint64 ival; bool bval; if (obj == NULL) { @@ -188,6 +204,32 @@ spf_library_config (const ucl_object_t *obj) } } + if ((value = ucl_object_find_key (obj, "disable_ipv6")) != NULL) { + if (ucl_object_toboolean_safe (value, &bval)) { + spf_lib_ctx->disable_ipv6 = bval; + } + } + + if (spf_lib_ctx->spf_hash) { + rspamd_lru_hash_destroy (spf_lib_ctx->spf_hash); + spf_lib_ctx->spf_hash = NULL; + } + + if ((value = ucl_object_find_key (obj, "spf_cache_size")) != NULL) { + if (ucl_object_toint_safe (value, &ival) && ival > 0) { + spf_lib_ctx->spf_hash = rspamd_lru_hash_new ( + ival, + g_free, + spf_record_cached_unref_dtor); + } + } + else { + /* Preserve compatibility */ + spf_lib_ctx->spf_hash = rspamd_lru_hash_new ( + 2048, + g_free, + spf_record_cached_unref_dtor); + } } static gboolean start_spf_parse (struct spf_record *rec, @@ -396,7 +438,9 @@ rspamd_spf_process_reference (struct spf_resolved *target, continue; } if (cur->flags & RSPAMD_SPF_FLAG_PERMFAIL) { - target->flags |= RSPAMD_SPF_RESOLVED_PERM_FAILED; + if (cur->flags & RSPAMD_SPF_FLAG_REDIRECT) { + target->flags |= RSPAMD_SPF_RESOLVED_PERM_FAILED; + } continue; } if (cur->flags & RSPAMD_SPF_FLAG_NA) { @@ -554,12 +598,32 @@ static void rspamd_spf_maybe_return (struct spf_record *rec) { struct spf_resolved *flat; + struct rspamd_task *task = rec->task; if (rec->requests_inflight == 0 && !rec->done) { flat = rspamd_spf_record_flatten (rec); rspamd_spf_record_postprocess (flat, rec->task); + + if (flat->ttl > 0 && flat->flags == 0) { + + if (spf_lib_ctx->spf_hash) { + rspamd_lru_hash_insert (spf_lib_ctx->spf_hash, + g_strdup (flat->domain), + spf_record_ref (flat), + flat->timestamp, flat->ttl); + + msg_info_task ("stored record for %s (0x%xuL) in LRU cache for %d seconds, " + "%d/%d elements in the cache", + flat->domain, + flat->digest, + flat->ttl, + rspamd_lru_hash_size (spf_lib_ctx->spf_hash), + rspamd_lru_hash_capacity (spf_lib_ctx->spf_hash)); + } + } + rec->callback (flat, rec->task, rec->cbdata); - REF_RELEASE (flat); + spf_record_unref (flat); rec->done = TRUE; } } @@ -1750,16 +1814,36 @@ expand_spf_macro (struct spf_record *rec, struct spf_resolved_element *resolved, len += INET6_ADDRSTRLEN - 1; break; case 's': - len += strlen (rec->sender); + if (rec->sender) { + len += strlen (rec->sender); + } + else { + len += sizeof ("unknown") - 1; + } break; case 'l': - len += strlen (rec->local_part); + if (rec->local_part) { + len += strlen (rec->local_part); + } + else { + len += sizeof ("unknown") - 1; + } break; case 'o': - len += strlen (rec->sender_domain); + if (rec->sender_domain) { + len += strlen (rec->sender_domain); + } + else { + len += sizeof ("unknown") - 1; + } break; case 'd': - len += strlen (resolved->cur_domain); + if (resolved->cur_domain) { + len += strlen (resolved->cur_domain); + } + else { + len += sizeof ("unknown") - 1; + } break; case 'v': len += sizeof ("in-addr") - 1; @@ -1768,6 +1852,9 @@ expand_spf_macro (struct spf_record *rec, struct spf_resolved_element *resolved, if (task->helo) { len += strlen (task->helo); } + else { + len += sizeof ("unknown") - 1; + } break; default: msg_info_spf ( @@ -2013,8 +2100,10 @@ expand_spf_macro (struct spf_record *rec, struct spf_resolved_element *resolved, /* Read current element and try to parse record */ static gboolean -parse_spf_record (struct spf_record *rec, struct spf_resolved_element *resolved, - const gchar *elt) +spf_process_element (struct spf_record *rec, + struct spf_resolved_element *resolved, + const gchar *elt, + const gchar **elts) { struct spf_addr *addr = NULL; gboolean res = FALSE; @@ -2110,7 +2199,33 @@ parse_spf_record (struct spf_record *rec, struct spf_resolved_element *resolved, /* redirect */ if (g_ascii_strncasecmp (begin, SPF_REDIRECT, sizeof (SPF_REDIRECT) - 1) == 0) { - res = parse_spf_redirect (rec, resolved, addr); + /* + * According to https://tools.ietf.org/html/rfc7208#section-6.1 + * There must be no ALL element anywhere in the record, + * redirect must be ignored + */ + gboolean ignore_redirect = FALSE; + + for (const gchar **tmp = elts; *tmp != NULL; tmp ++) { + if (g_ascii_strcasecmp ((*tmp) + 1, "all") == 0) { + ignore_redirect = TRUE; + break; + } + } + + if (!ignore_redirect) { + res = parse_spf_redirect (rec, resolved, addr); + } + else { + msg_info_spf ("ignore SPF redirect (%s) for domain %s as there is also all element", + begin, rec->sender_domain); + + /* Pop the current addr as it is ignored */ + g_ptr_array_remove_index_fast (resolved->elts, + resolved->elts->len - 1); + + return TRUE; + } } else { msg_info_spf ("spf error for domain %s: bad spf command %s", @@ -2219,7 +2334,7 @@ start_spf_parse (struct spf_record *rec, struct spf_resolved_element *resolved, cur_elt = elts; while (*cur_elt) { - parse_spf_record (rec, resolved, *cur_elt); + spf_process_element (rec, resolved, *cur_elt, (const gchar **)elts); cur_elt++; } @@ -2293,13 +2408,7 @@ spf_dns_callback (struct rdns_reply *reply, gpointer arg) rspamd_spf_maybe_return (rec); } -struct rspamd_spf_cred { - gchar *local_part; - gchar *domain; - gchar *sender; -}; - -struct rspamd_spf_cred * +static struct rspamd_spf_cred * rspamd_spf_cache_domain (struct rspamd_task *task) { struct rspamd_email_address *addr; @@ -2344,10 +2453,9 @@ rspamd_spf_cache_domain (struct rspamd_task *task) return cred; } -const gchar * -rspamd_spf_get_domain (struct rspamd_task *task) +struct rspamd_spf_cred * +rspamd_spf_get_cred (struct rspamd_task *task) { - gchar *domain = NULL; struct rspamd_spf_cred *cred; cred = rspamd_mempool_get_variable (task->task_pool, @@ -2357,6 +2465,17 @@ rspamd_spf_get_domain (struct rspamd_task *task) cred = rspamd_spf_cache_domain (task); } + return cred; +} + +const gchar * +rspamd_spf_get_domain (struct rspamd_task *task) +{ + gchar *domain = NULL; + struct rspamd_spf_cred *cred; + + cred = rspamd_spf_get_cred (task); + if (cred) { domain = cred->domain; } @@ -2366,22 +2485,30 @@ rspamd_spf_get_domain (struct rspamd_task *task) gboolean rspamd_spf_resolve (struct rspamd_task *task, spf_cb_t callback, - gpointer cbdata) + gpointer cbdata, struct rspamd_spf_cred *cred) { struct spf_record *rec; - struct rspamd_spf_cred *cred; - - cred = rspamd_mempool_get_variable (task->task_pool, - RSPAMD_MEMPOOL_SPF_DOMAIN); - - if (!cred) { - cred = rspamd_spf_cache_domain (task); - } if (!cred || !cred->domain) { return FALSE; } + /* First lookup in the hash */ + if (spf_lib_ctx->spf_hash) { + struct spf_resolved *cached; + + cached = rspamd_lru_hash_lookup (spf_lib_ctx->spf_hash, cred->domain, + task->task_timestamp); + + if (cached) { + cached->flags |= RSPAMD_SPF_FLAG_CACHED; + callback (cached, task, cbdata); + + return TRUE; + } + } + + rec = rspamd_mempool_alloc0 (task->task_pool, sizeof (struct spf_record)); rec->task = task; rec->callback = callback; @@ -2410,16 +2537,16 @@ rspamd_spf_resolve (struct rspamd_task *task, spf_cb_t callback, } struct spf_resolved * -spf_record_ref (struct spf_resolved *rec) +_spf_record_ref (struct spf_resolved *flat, const gchar *loc) { - REF_RETAIN (rec); - return rec; + REF_RETAIN (flat); + return flat; } void -spf_record_unref (struct spf_resolved *rec) +_spf_record_unref (struct spf_resolved *flat, const gchar *loc) { - REF_RELEASE (rec); + REF_RELEASE (flat); } gchar * diff --git a/src/libserver/spf.h b/src/libserver/spf.h index 9fcb01d1e..451cb5003 100644 --- a/src/libserver/spf.h +++ b/src/libserver/spf.h @@ -22,6 +22,21 @@ typedef enum spf_mech_e { SPF_NEUTRAL } spf_mech_t; +static inline gchar spf_mech_char (spf_mech_t mech) +{ + switch (mech) { + case SPF_FAIL: + return '-'; + case SPF_SOFT_FAIL: + return '~'; + case SPF_PASS: + return '+'; + case SPF_NEUTRAL: + default: + return '?'; + } +} + typedef enum spf_action_e { SPF_RESOLVE_MX, SPF_RESOLVE_A, @@ -45,6 +60,7 @@ typedef enum spf_action_e { #define RSPAMD_SPF_FLAG_NA (1u << 9u) #define RSPAMD_SPF_FLAG_PERMFAIL (1u << 10u) #define RSPAMD_SPF_FLAG_RESOLVED (1u << 11u) +#define RSPAMD_SPF_FLAG_CACHED (1u << 12u) /** Default SPF limits for avoiding abuse **/ #define SPF_MAX_NESTING 10 @@ -84,28 +100,38 @@ struct spf_resolved { ref_entry_t ref; /* Refcounting */ }; +struct rspamd_spf_cred { + gchar *local_part; + gchar *domain; + gchar *sender; +}; /* * Resolve spf record for specified task and call a callback after resolution fails/succeed */ -gboolean rspamd_spf_resolve (struct rspamd_task *task, spf_cb_t callback, - gpointer cbdata); +gboolean rspamd_spf_resolve (struct rspamd_task *task, + spf_cb_t callback, + gpointer cbdata, + struct rspamd_spf_cred *cred); /* * Get a domain for spf for specified task */ const gchar *rspamd_spf_get_domain (struct rspamd_task *task); - +struct rspamd_spf_cred *rspamd_spf_get_cred (struct rspamd_task *task); /* * Increase refcount */ -struct spf_resolved *spf_record_ref (struct spf_resolved *rec); - +struct spf_resolved *_spf_record_ref (struct spf_resolved *rec, const gchar *loc); +#define spf_record_ref(rec) \ + _spf_record_ref ((rec), G_STRLOC) /* * Decrease refcount */ -void spf_record_unref (struct spf_resolved *rec); +void _spf_record_unref (struct spf_resolved *rec, const gchar *loc); +#define spf_record_unref(rec) \ + _spf_record_unref((rec), G_STRLOC) /** * Prints address + mask in a freshly allocated string (must be freed) diff --git a/src/libserver/task.c b/src/libserver/task.c index 2fb9bf1d9..c1fcc752f 100644 --- a/src/libserver/task.c +++ b/src/libserver/task.c @@ -60,17 +60,20 @@ rspamd_task_quark (void) * Create new task */ struct rspamd_task * -rspamd_task_new (struct rspamd_worker *worker, struct rspamd_config *cfg, +rspamd_task_new (struct rspamd_worker *worker, + struct rspamd_config *cfg, rspamd_mempool_t *pool, struct rspamd_lang_detector *lang_det, - struct ev_loop *event_loop) + struct ev_loop *event_loop, + gboolean debug_mem) { struct rspamd_task *new_task; rspamd_mempool_t *task_pool; guint flags = 0; if (pool == NULL) { - task_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "task"); + task_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), + "task", debug_mem ? RSPAMD_MEMPOOL_DEBUG : 0); flags |= RSPAMD_TASK_FLAG_OWN_POOL; } else { @@ -724,14 +727,6 @@ rspamd_task_process (struct rspamd_task *task, guint stages) case RSPAMD_TASK_STAGE_FILTERS: all_done = rspamd_symcache_process_symbols (task, task->cfg->cache, st); break; - case RSPAMD_TASK_STAGE_IDEMPOTENT: - /* Stop task timeout */ - if (ev_can_stop (&task->timeout_ev)) { - ev_timer_stop (task->event_loop, &task->timeout_ev); - } - - all_done = rspamd_symcache_process_symbols (task, task->cfg->cache, st); - break; case RSPAMD_TASK_STAGE_PROCESS_MESSAGE: if (!(task->flags & RSPAMD_TASK_FLAG_SKIP_PROCESS)) { @@ -814,6 +809,15 @@ rspamd_task_process (struct rspamd_task *task, guint stages) rspamd_make_composites (task); break; + case RSPAMD_TASK_STAGE_IDEMPOTENT: + /* Stop task timeout */ + if (ev_can_stop (&task->timeout_ev)) { + ev_timer_stop (task->event_loop, &task->timeout_ev); + } + + all_done = rspamd_symcache_process_symbols (task, task->cfg->cache, st); + break; + case RSPAMD_TASK_STAGE_DONE: task->processed_stages |= RSPAMD_TASK_STAGE_DONE; break; @@ -1155,6 +1159,7 @@ rspamd_task_log_metric_res (struct rspamd_task *task, rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t)rspamd_fstring_free, symbuf); + rspamd_mempool_notify_alloc (task->task_pool, symbuf->len); res.begin = symbuf->str; res.len = symbuf->len; break; @@ -1200,6 +1205,7 @@ rspamd_task_log_metric_res (struct rspamd_task *task, rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t) rspamd_fstring_free, symbuf); + rspamd_mempool_notify_alloc (task->task_pool, symbuf->len); res.begin = symbuf->str; res.len = symbuf->len; break; @@ -1544,6 +1550,18 @@ rspamd_task_log_variable (struct rspamd_task *task, var.len = sizeof (undef) - 1; } break; + case RSPAMD_LOG_MEMPOOL_SIZE: + var.len = rspamd_snprintf (numbuf, sizeof (numbuf), + "%Hz", + rspamd_mempool_get_used_size (task->task_pool)); + var.begin = numbuf; + break; + case RSPAMD_LOG_MEMPOOL_WASTE: + var.len = rspamd_snprintf (numbuf, sizeof (numbuf), + "%Hz", + rspamd_mempool_get_wasted_size (task->task_pool)); + var.begin = numbuf; + break; default: var = rspamd_task_log_metric_res (task, lf); break; @@ -1839,4 +1857,125 @@ rspamd_task_stage_name (enum rspamd_task_stage stg) } return ret; +} + +void +rspamd_task_timeout (EV_P_ ev_timer *w, int revents) +{ + struct rspamd_task *task = (struct rspamd_task *)w->data; + + if (!(task->processed_stages & RSPAMD_TASK_STAGE_FILTERS)) { + ev_now_update_if_cheap (task->event_loop); + msg_info_task ("processing of task time out: %.1fs spent; %.1fs limit; " + "forced processing", + ev_now (task->event_loop) - task->task_timestamp, + w->repeat); + + if (task->cfg->soft_reject_on_timeout) { + struct rspamd_action *action, *soft_reject; + + action = rspamd_check_action_metric (task); + + if (action->action_type != METRIC_ACTION_REJECT) { + soft_reject = rspamd_config_get_action_by_type (task->cfg, + METRIC_ACTION_SOFT_REJECT); + rspamd_add_passthrough_result (task, + soft_reject, + 0, + NAN, + "timeout processing message", + "task timeout", + 0); + + ucl_object_replace_key (task->messages, + ucl_object_fromstring_common ("timeout processing message", + 0, UCL_STRING_RAW), + "smtp_message", 0, + false); + } + } + + ev_timer_again (EV_A_ w); + task->processed_stages |= RSPAMD_TASK_STAGE_FILTERS; + rspamd_session_cleanup (task->s); + rspamd_task_process (task, RSPAMD_TASK_PROCESS_ALL); + rspamd_session_pending (task->s); + } + else { + /* Postprocessing timeout */ + msg_info_task ("post-processing of task time out: %.1f second spent; forced processing", + ev_now (task->event_loop) - task->task_timestamp); + + if (task->cfg->soft_reject_on_timeout) { + struct rspamd_action *action, *soft_reject; + + action = rspamd_check_action_metric (task); + + if (action->action_type != METRIC_ACTION_REJECT) { + soft_reject = rspamd_config_get_action_by_type (task->cfg, + METRIC_ACTION_SOFT_REJECT); + rspamd_add_passthrough_result (task, + soft_reject, + 0, + NAN, + "timeout post-processing message", + "task timeout", + 0); + + ucl_object_replace_key (task->messages, + ucl_object_fromstring_common ("timeout post-processing message", + 0, UCL_STRING_RAW), + "smtp_message", 0, + false); + } + } + + ev_timer_stop (EV_A_ w); + task->processed_stages |= RSPAMD_TASK_STAGE_DONE; + rspamd_session_cleanup (task->s); + rspamd_task_process (task, RSPAMD_TASK_PROCESS_ALL); + rspamd_session_pending (task->s); + } +} + +void +rspamd_worker_guard_handler (EV_P_ ev_io *w, int revents) +{ + struct rspamd_task *task = (struct rspamd_task *)w->data; + gchar fake_buf[1024]; + gssize r; + + r = read (w->fd, fake_buf, sizeof (fake_buf)); + + if (r > 0) { + msg_warn_task ("received extra data after task is loaded, ignoring"); + } + else { + if (r == 0) { + /* + * Poor man approach, that might break things in case of + * shutdown (SHUT_WR) but sockets are so bad that there's no + * reliable way to distinguish between shutdown(SHUT_WR) and + * close. + */ + if (task->cmd != CMD_CHECK_V2 && task->cfg->enable_shutdown_workaround) { + msg_info_task ("workaround for shutdown enabled, please update " + "your client, this support might be removed in future"); + shutdown (w->fd, SHUT_RD); + ev_io_stop (task->event_loop, &task->guard_ev); + } + else { + msg_err_task ("the peer has closed connection unexpectedly"); + rspamd_session_destroy (task->s); + } + } + else if (errno != EAGAIN) { + msg_err_task ("the peer has closed connection unexpectedly: %s", + strerror (errno)); + rspamd_session_destroy (task->s); + } + else { + return; + } + } }
\ No newline at end of file diff --git a/src/libserver/task.h b/src/libserver/task.h index a96e2ac05..50e07b23f 100644 --- a/src/libserver/task.h +++ b/src/libserver/task.h @@ -225,7 +225,8 @@ struct rspamd_task *rspamd_task_new (struct rspamd_worker *worker, struct rspamd_config *cfg, rspamd_mempool_t *pool, struct rspamd_lang_detector *lang_det, - struct ev_loop *event_loop); + struct ev_loop *event_loop, + gboolean debug_mem); /** * Destroy task object and remove its IO dispatcher if it exists @@ -374,6 +375,16 @@ gboolean rspamd_task_set_finish_time (struct rspamd_task *task); */ const gchar *rspamd_task_stage_name (enum rspamd_task_stage stg); +/* + * Called on forced timeout + */ +void rspamd_task_timeout (EV_P_ ev_timer *w, int revents); + +/* + * Called on unexpected IO error (e.g. ECONNRESET) + */ +void rspamd_worker_guard_handler (EV_P_ ev_io *w, int revents); + #ifdef __cplusplus } #endif diff --git a/src/libserver/url.c b/src/libserver/url.c index 985df24ac..866cb4c22 100644 --- a/src/libserver/url.c +++ b/src/libserver/url.c @@ -1021,6 +1021,14 @@ rspamd_web_parse (struct http_parser_url *u, const gchar *str, gsize len, st = parse_path; c = p + 1; } + else if (*p == '?') { + st = parse_query; + c = p + 1; + } + else if (*p == '#') { + st = parse_part; + c = p + 1; + } else if (p != last) { goto out; } @@ -1359,6 +1367,14 @@ rspamd_web_parse (struct http_parser_url *u, const gchar *str, gsize len, c = p + 1; st = parse_query; } + else if (t == '#') { + /* No query, just fragment */ + if (p - c != 0) { + SET_U (u, UF_PATH); + } + c = p + 1; + st = parse_part; + } else if (is_url_end (t)) { goto set; } @@ -1550,6 +1566,66 @@ rspamd_tld_trie_callback (struct rspamd_multipattern *mp, return 0; } +static void +rspamd_url_regen_from_inet_addr (struct rspamd_url *uri, const void *addr, int af, + rspamd_mempool_t *pool) +{ + gchar *strbuf, *p; + gsize slen = uri->urllen - uri->hostlen; + goffset r = 0; + + if (af == AF_INET) { + slen += INET_ADDRSTRLEN; + } + else { + slen += INET6_ADDRSTRLEN; + } + + /* Allocate new string to build it from IP */ + strbuf = rspamd_mempool_alloc (pool, slen + 1); + r += rspamd_snprintf (strbuf + r, slen - r, "%*s", + (gint)(uri->host - uri->string), + uri->string); + uri->host = strbuf + r; + inet_ntop (af, addr, strbuf + r, slen - r + 1); + uri->hostlen = strlen (uri->host); + r += uri->hostlen; + uri->tld = uri->host; + uri->tldlen = uri->hostlen; + uri->flags |= RSPAMD_URL_FLAG_NUMERIC; + + /* Reconstruct URL */ + if (uri->datalen > 0) { + p = strbuf + r + 1; + r += rspamd_snprintf (strbuf + r, slen - r, "/%*s", + (gint)uri->datalen, + uri->data); + uri->data = p; + } + else { + /* Add trailing slash if needed */ + r += rspamd_snprintf (strbuf + r, slen - r, "/"); + } + + if (uri->querylen > 0) { + p = strbuf + r + 1; + r += rspamd_snprintf (strbuf + r, slen - r, "?%*s", + (gint)uri->querylen, + uri->query); + uri->query = p; + } + if (uri->fragmentlen > 0) { + p = strbuf + r + 1; + r += rspamd_snprintf (strbuf + r, slen - r, "#%*s", + (gint)uri->fragmentlen, + uri->fragment); + uri->fragment = p; + } + + uri->string = strbuf; + uri->urllen = r; +} + static gboolean rspamd_url_is_ip (struct rspamd_url *uri, rspamd_mempool_t *pool) { @@ -1577,23 +1653,11 @@ rspamd_url_is_ip (struct rspamd_url *uri, rspamd_mempool_t *pool) } if (rspamd_parse_inet_address_ip4 (p, end - p, &in4)) { - uri->host = rspamd_mempool_alloc (pool, INET_ADDRSTRLEN + 1); - memset (uri->host, 0, INET_ADDRSTRLEN + 1); - inet_ntop (AF_INET, &in4, uri->host, INET_ADDRSTRLEN); - uri->hostlen = strlen (uri->host); - uri->tld = uri->host; - uri->tldlen = uri->hostlen; - uri->flags |= RSPAMD_URL_FLAG_NUMERIC; + rspamd_url_regen_from_inet_addr (uri, &in4, AF_INET, pool); ret = TRUE; } else if (rspamd_parse_inet_address_ip6 (p, end - p, &in6)) { - uri->host = rspamd_mempool_alloc (pool, INET6_ADDRSTRLEN + 1); - memset (uri->host, 0, INET6_ADDRSTRLEN + 1); - inet_ntop (AF_INET6, &in6, uri->host, INET6_ADDRSTRLEN); - uri->hostlen = strlen (uri->host); - uri->tld = uri->host; - uri->tldlen = uri->hostlen; - uri->flags |= RSPAMD_URL_FLAG_NUMERIC; + rspamd_url_regen_from_inet_addr (uri, &in6, AF_INET6, pool); ret = TRUE; } else { @@ -1693,26 +1757,16 @@ rspamd_url_is_ip (struct rspamd_url *uri, rspamd_mempool_t *pool) if (check_num) { if (dots <= 4) { memcpy (&in4, &n, sizeof (in4)); - uri->host = rspamd_mempool_alloc (pool, INET_ADDRSTRLEN + 1); - memset (uri->host, 0, INET_ADDRSTRLEN + 1); - inet_ntop (AF_INET, &in4, uri->host, INET_ADDRSTRLEN); - uri->hostlen = strlen (uri->host); - uri->tld = uri->host; - uri->tldlen = uri->hostlen; - uri->flags |= RSPAMD_URL_FLAG_NUMERIC | RSPAMD_URL_FLAG_OBSCURED; + rspamd_url_regen_from_inet_addr (uri, &in4, AF_INET, pool); + uri->flags |= RSPAMD_URL_FLAG_OBSCURED; ret = TRUE; } else if (end - c > (gint) sizeof (buf) - 1) { rspamd_strlcpy (buf, c, end - c + 1); if (inet_pton (AF_INET6, buf, &in6) == 1) { - uri->host = rspamd_mempool_alloc (pool, INET6_ADDRSTRLEN + 1); - memset (uri->host, 0, INET6_ADDRSTRLEN + 1); - inet_ntop (AF_INET6, &in6, uri->host, INET6_ADDRSTRLEN); - uri->hostlen = strlen (uri->host); - uri->tld = uri->host; - uri->tldlen = uri->hostlen; - uri->flags |= RSPAMD_URL_FLAG_NUMERIC | RSPAMD_URL_FLAG_OBSCURED; + rspamd_url_regen_from_inet_addr (uri, &in6, AF_INET6, pool); + uri->flags |= RSPAMD_URL_FLAG_OBSCURED; ret = TRUE; } } @@ -1756,7 +1810,7 @@ rspamd_url_shift (struct rspamd_url *uri, gsize nlen, old_shift = uri->hostlen; uri->hostlen -= shift; memmove (uri->host + uri->hostlen, uri->host + old_shift, - uri->datalen + uri->querylen + uri->fragmentlen); + uri->datalen + uri->querylen + uri->fragmentlen + 1); uri->urllen -= shift; uri->flags |= RSPAMD_URL_FLAG_HOSTENCODED; break; @@ -1771,7 +1825,7 @@ rspamd_url_shift (struct rspamd_url *uri, gsize nlen, old_shift = uri->datalen; uri->datalen -= shift; memmove (uri->data + uri->datalen, uri->data + old_shift, - uri->querylen + uri->fragmentlen); + uri->querylen + uri->fragmentlen + 1); uri->urllen -= shift; uri->flags |= RSPAMD_URL_FLAG_PATHENCODED; break; @@ -1786,7 +1840,7 @@ rspamd_url_shift (struct rspamd_url *uri, gsize nlen, old_shift = uri->querylen; uri->querylen -= shift; memmove (uri->query + uri->querylen, uri->query + old_shift, - uri->fragmentlen); + uri->fragmentlen + 1); uri->urllen -= shift; uri->flags |= RSPAMD_URL_FLAG_QUERYENCODED; break; @@ -2124,6 +2178,21 @@ rspamd_url_parse (struct rspamd_url *uri, } } } + + /* Replace stupid '\' with '/' after schema */ + if (uri->protocol & (PROTOCOL_HTTP|PROTOCOL_HTTPS|PROTOCOL_FTP) && + uri->protocollen > 0 && uri->urllen > uri->protocollen + 2) { + + gchar *pos = &uri->string[uri->protocollen], *host_start = uri->host; + + while (pos < host_start) { + if (*pos == '\\') { + *pos = '/'; + uri->flags |= RSPAMD_URL_FLAG_OBSCURED; + } + pos ++; + } + } } else if (uri->protocol & PROTOCOL_TELEPHONE) { /* We need to normalise phone number: remove all spaces and braces */ @@ -2862,7 +2931,7 @@ rspamd_url_trie_generic_callback_common (struct rspamd_multipattern *mp, pool = cb->pool; if ((matcher->flags & URL_FLAG_NOHTML) && cb->how == RSPAMD_URL_FIND_STRICT) { - /* Do not try to match non-html like urls in html texts */ + /* Do not try to match non-html like urls in html texts, continue matching */ return 0; } @@ -2888,6 +2957,7 @@ rspamd_url_trie_generic_callback_common (struct rspamd_multipattern *mp, } if (!rspamd_url_trie_is_match (matcher, pos, text + len, newline_pos)) { + /* Mismatch, continue */ return 0; } @@ -2934,7 +3004,8 @@ rspamd_url_trie_generic_callback_common (struct rspamd_multipattern *mp, if (cb->func) { if (!cb->func (url, cb->start - text, (m.m_begin + m.m_len) - text, cb->funcd)) { - return FALSE; + /* We need to stop here in any case! */ + return -1; } } } @@ -3009,9 +3080,10 @@ rspamd_url_text_part_callback (struct rspamd_url *url, gsize start_offset, if (cbd->part->utf_stripped_content && cbd->url_len > cbd->part->utf_stripped_content->len * 10) { - /* Absurdic case, stop here now */ - msg_err_task ("part has too many URLs, we cannot process more: %z", - cbd->url_len); + /* Absurd case, stop here now */ + msg_err_task ("part has too many URLs, we cannot process more: %z url len; " + "%d stripped content length", + cbd->url_len, cbd->part->utf_stripped_content->len); return FALSE; } @@ -3026,6 +3098,17 @@ rspamd_url_text_part_callback (struct rspamd_url *url, gsize start_offset, } if (target_tbl) { + /* Also check max urls */ + if (cbd->task->cfg && cbd->task->cfg->max_lua_urls > 0) { + if (g_hash_table_size (target_tbl) > cbd->task->cfg->max_lua_urls) { + msg_err_task ("part has too many URLs, we cannot process more: " + "%d urls extracted ", + (guint)g_hash_table_size (target_tbl)); + + return FALSE; + } + } + if ((existing = g_hash_table_lookup (target_tbl, url)) == NULL) { url->flags |= RSPAMD_URL_FLAG_FROM_TEXT; g_hash_table_insert (target_tbl, url, url); @@ -3594,13 +3677,13 @@ rspamd_url_encode (struct rspamd_url *url, gsize *pdlen, } if (url->querylen > 0) { - *d++ = '/'; + *d++ = '?'; ENCODE_URL_COMPONENT ((guchar *)url->query, url->querylen, RSPAMD_URL_FLAGS_QUERYSAFE); } if (url->fragmentlen > 0) { - *d++ = '/'; + *d++ = '#'; ENCODE_URL_COMPONENT ((guchar *)url->fragment, url->fragmentlen, RSPAMD_URL_FLAGS_FRAGMENTSAFE); } diff --git a/src/libserver/worker_util.c b/src/libserver/worker_util.c index ddf74136d..e519cb985 100644 --- a/src/libserver/worker_util.c +++ b/src/libserver/worker_util.c @@ -25,6 +25,7 @@ #include "libutil/map_private.h" #include "libutil/http_private.h" #include "libutil/http_router.h" +#include "libutil/rrd.h" #ifdef WITH_GPERF_TOOLS #include <gperftools/profiler.h> @@ -62,6 +63,7 @@ #endif #include "contrib/libev/ev.h" +#include "libstat/stat_api.h" /* Forward declaration */ static void rspamd_worker_heartbeat_start (struct rspamd_worker *, @@ -113,58 +115,138 @@ rspamd_worker_check_finished (EV_P_ ev_timer *w, int revents) } } +static gboolean +rspamd_worker_finalize (gpointer user_data) +{ + struct rspamd_task *task = user_data; + + if (!(task->flags & RSPAMD_TASK_FLAG_PROCESSING)) { + msg_info_task ("finishing actions has been processed, terminating"); + /* ev_break (task->event_loop, EVBREAK_ALL); */ + task->worker->state = rspamd_worker_wanna_die; + rspamd_session_destroy (task->s); + + return TRUE; + } + + return FALSE; +} + +gboolean +rspamd_worker_call_finish_handlers (struct rspamd_worker *worker) +{ + struct rspamd_task *task; + struct rspamd_config *cfg = worker->srv->cfg; + struct rspamd_abstract_worker_ctx *ctx; + struct rspamd_config_cfg_lua_script *sc; + + if (cfg->on_term_scripts) { + ctx = (struct rspamd_abstract_worker_ctx *)worker->ctx; + /* Create a fake task object for async events */ + task = rspamd_task_new (worker, cfg, NULL, NULL, ctx->event_loop, FALSE); + task->resolver = ctx->resolver; + task->flags |= RSPAMD_TASK_FLAG_PROCESSING; + task->s = rspamd_session_create (task->task_pool, + rspamd_worker_finalize, + NULL, + (event_finalizer_t) rspamd_task_free, + task); + + DL_FOREACH (cfg->on_term_scripts, sc) { + lua_call_finish_script (sc, task); + } + + task->flags &= ~RSPAMD_TASK_FLAG_PROCESSING; + + if (rspamd_session_pending (task->s)) { + return TRUE; + } + } + + return FALSE; +} + static void rspamd_worker_terminate_handlers (struct rspamd_worker *w) { - guint i; - gboolean (*cb)(struct rspamd_worker *); - struct rspamd_abstract_worker_ctx *actx; - struct ev_loop *final_gift, *orig_loop; - static ev_timer margin_call; - static int nchecks = 0; - - if (w->finish_actions->len == 0) { - /* Nothing to do */ - return; + if (w->nconns == 0 && + (!(w->flags & RSPAMD_WORKER_SCANNER) || w->srv->cfg->on_term_scripts == NULL)) { + /* + * We are here either: + * - No active connections are represented + * - No term scripts are registered + * - Worker is not a scanner, so it can die safely + */ + w->state = rspamd_worker_wanna_die; } + else { + if (w->nconns > 0) { + /* + * Wait until all connections are terminated + */ + w->state = rspamd_worker_wait_connections; + } + else { + /* + * Start finish scripts + */ + if (w->state != rspamd_worker_wait_final_scripts) { + w->state = rspamd_worker_wait_final_scripts; - actx = (struct rspamd_abstract_worker_ctx *)w->ctx; - - /* - * Here are dragons: - * - we create a new loop - * - we set a new ev_loop for worker via injection over rspamd_abstract_worker_ctx - * - then we run finish actions - * - then we create a special timer to kill worker if it fails to finish - */ - final_gift = ev_loop_new (EVBACKEND_ALL); - orig_loop = actx->event_loop; - actx->event_loop = final_gift; - margin_call.data = &nchecks; - ev_timer_init (&margin_call, rspamd_worker_check_finished, 0.1, - 0.1); - ev_timer_start (final_gift, &margin_call); - - for (i = 0; i < w->finish_actions->len; i ++) { - cb = g_ptr_array_index (w->finish_actions, i); - cb (w); - } - - ev_run (final_gift, 0); - ev_loop_destroy (final_gift); - /* Restore original loop */ - actx->event_loop = orig_loop; + if ((w->flags & RSPAMD_WORKER_SCANNER) && + rspamd_worker_call_finish_handlers (w)) { + msg_info ("performing async finishing actions"); + w->state = rspamd_worker_wait_final_scripts; + } + else { + /* + * We are done now + */ + msg_info ("no async finishing actions, terminating"); + w->state = rspamd_worker_wanna_die; + } + } + } + } } static void rspamd_worker_on_delayed_shutdown (EV_P_ ev_timer *w, int revents) { + struct rspamd_worker *worker = (struct rspamd_worker *)w->data; + + worker->state = rspamd_worker_wanna_die; + ev_timer_stop (EV_A_ w); ev_break (loop, EVBREAK_ALL); #ifdef WITH_GPERF_TOOLS ProfilerStop (); #endif } +static void +rspamd_worker_shutdown_check (EV_P_ ev_timer *w, int revents) +{ + struct rspamd_worker *worker = (struct rspamd_worker *)w->data; + + if (worker->state != rspamd_worker_wanna_die) { + rspamd_worker_terminate_handlers (worker); + + if (worker->state == rspamd_worker_wanna_die) { + /* We are done, kill event loop */ + ev_timer_stop (EV_A_ w); + ev_break (EV_A_ EVBREAK_ALL); + } + else { + /* Try again later */ + ev_timer_again (EV_A_ w); + } + } + else { + ev_timer_stop (EV_A_ w); + ev_break (EV_A_ EVBREAK_ALL); + } +} + /* * Config reload is designed by sending sigusr2 to active workers and pending shutdown of them */ @@ -172,25 +254,34 @@ static gboolean rspamd_worker_usr2_handler (struct rspamd_worker_signal_handler *sigh, void *arg) { /* Do not accept new connections, preparing to end worker's process */ - if (!sigh->worker->wanna_die) { - static ev_timer shutdown_ev; + if (sigh->worker->state == rspamd_worker_state_running) { + static ev_timer shutdown_ev, shutdown_check_ev; ev_tstamp shutdown_ts; shutdown_ts = MAX (SOFT_SHUTDOWN_TIME, sigh->worker->srv->cfg->task_timeout * 2.0); rspamd_worker_ignore_signal (sigh); - sigh->worker->wanna_die = TRUE; - rspamd_worker_terminate_handlers (sigh->worker); + sigh->worker->state = rspamd_worker_state_terminating; + rspamd_default_log_function (G_LOG_LEVEL_INFO, sigh->worker->srv->server_pool->tag.tagname, sigh->worker->srv->server_pool->tag.uid, G_STRFUNC, "worker's shutdown is pending in %.2f sec", shutdown_ts); + + /* Soft shutdown timer */ + shutdown_ev.data = sigh->worker; ev_timer_init (&shutdown_ev, rspamd_worker_on_delayed_shutdown, shutdown_ts, 0.0); ev_timer_start (sigh->event_loop, &shutdown_ev); + + /* This timer checks if we are ready to die and is called frequently */ + shutdown_check_ev.data = sigh->worker; + ev_timer_init (&shutdown_check_ev, rspamd_worker_shutdown_check, + 0.5, 0.5); + ev_timer_start (sigh->event_loop, &shutdown_check_ev); rspamd_worker_stop_accept (sigh->worker); } @@ -216,10 +307,15 @@ rspamd_worker_usr1_handler (struct rspamd_worker_signal_handler *sigh, void *arg static gboolean rspamd_worker_term_handler (struct rspamd_worker_signal_handler *sigh, void *arg) { - if (!sigh->worker->wanna_die) { - static ev_timer shutdown_ev; + if (sigh->worker->state == rspamd_worker_state_running) { + static ev_timer shutdown_ev, shutdown_check_ev; + ev_tstamp shutdown_ts; + + shutdown_ts = MAX (SOFT_SHUTDOWN_TIME, + sigh->worker->srv->cfg->task_timeout * 2.0); rspamd_worker_ignore_signal (sigh); + sigh->worker->state = rspamd_worker_state_terminating; rspamd_default_log_function (G_LOG_LEVEL_INFO, sigh->worker->srv->server_pool->tag.tagname, sigh->worker->srv->server_pool->tag.uid, @@ -227,12 +323,26 @@ rspamd_worker_term_handler (struct rspamd_worker_signal_handler *sigh, void *arg "terminating after receiving signal %s", g_strsignal (sigh->signo)); - rspamd_worker_terminate_handlers (sigh->worker); - sigh->worker->wanna_die = 1; - ev_timer_init (&shutdown_ev, rspamd_worker_on_delayed_shutdown, - 0.0, 0.0); - ev_timer_start (sigh->event_loop, &shutdown_ev); rspamd_worker_stop_accept (sigh->worker); + rspamd_worker_terminate_handlers (sigh->worker); + + /* Check if we are ready to die */ + if (sigh->worker->state != rspamd_worker_wanna_die) { + /* This timer is called when we have no choices but to die */ + shutdown_ev.data = sigh->worker; + ev_timer_init (&shutdown_ev, rspamd_worker_on_delayed_shutdown, + shutdown_ts, 0.0); + ev_timer_start (sigh->event_loop, &shutdown_ev); + + /* This timer checks if we are ready to die and is called frequently */ + shutdown_check_ev.data = sigh->worker; + ev_timer_init (&shutdown_check_ev, rspamd_worker_shutdown_check, + 0.5, 0.5); + ev_timer_start (sigh->event_loop, &shutdown_check_ev); + } + else { + ev_break (sigh->event_loop, EVBREAK_ALL); + } } /* Stop reacting on signals */ @@ -862,7 +972,6 @@ rspamd_fork_worker (struct rspamd_main *rspamd_main, REF_RETAIN (cf); wrk->index = index; wrk->ctx = cf->ctx; - wrk->finish_actions = g_ptr_array_new (); wrk->ppid = getpid (); wrk->pid = fork (); wrk->cores_throttled = rspamd_main->cores_throttling; @@ -949,6 +1058,7 @@ rspamd_fork_worker (struct rspamd_main *rspamd_main, close (wrk->srv_pipe[0]); rspamd_socket_nonblocking (wrk->control_pipe[1]); rspamd_socket_nonblocking (wrk->srv_pipe[1]); + rspamd_main->cfg->cur_worker = wrk; /* Execute worker */ cf->worker->worker_start_func (wrk); exit (EXIT_FAILURE); @@ -1377,7 +1487,7 @@ rspamd_check_termination_clause (struct rspamd_main *rspamd_main, { gboolean need_refork = TRUE; - if (wrk->wanna_die || rspamd_main->wanna_die) { + if (wrk->state != rspamd_worker_state_running || rspamd_main->wanna_die) { /* Do not refork workers that are intended to be terminated */ need_refork = FALSE; } @@ -1508,4 +1618,381 @@ rspamd_worker_check_context (gpointer ctx, guint64 magic) struct rspamd_abstract_worker_ctx *actx = (struct rspamd_abstract_worker_ctx*)ctx; return actx->magic == magic; +} + +static gboolean +rspamd_worker_log_pipe_handler (struct rspamd_main *rspamd_main, + struct rspamd_worker *worker, gint fd, + gint attached_fd, + struct rspamd_control_command *cmd, + gpointer ud) +{ + struct rspamd_config *cfg = ud; + struct rspamd_worker_log_pipe *lp; + struct rspamd_control_reply rep; + + memset (&rep, 0, sizeof (rep)); + rep.type = RSPAMD_CONTROL_LOG_PIPE; + + if (attached_fd != -1) { + lp = g_malloc0 (sizeof (*lp)); + lp->fd = attached_fd; + lp->type = cmd->cmd.log_pipe.type; + + DL_APPEND (cfg->log_pipes, lp); + msg_info ("added new log pipe"); + } + else { + rep.reply.log_pipe.status = ENOENT; + msg_err ("cannot attach log pipe: invalid fd"); + } + + if (write (fd, &rep, sizeof (rep)) != sizeof (rep)) { + msg_err ("cannot write reply to the control socket: %s", + strerror (errno)); + } + + return TRUE; +} + +static gboolean +rspamd_worker_monitored_handler (struct rspamd_main *rspamd_main, + struct rspamd_worker *worker, gint fd, + gint attached_fd, + struct rspamd_control_command *cmd, + gpointer ud) +{ + struct rspamd_control_reply rep; + struct rspamd_monitored *m; + struct rspamd_monitored_ctx *mctx = worker->srv->cfg->monitored_ctx; + struct rspamd_config *cfg = ud; + + memset (&rep, 0, sizeof (rep)); + rep.type = RSPAMD_CONTROL_MONITORED_CHANGE; + + if (cmd->cmd.monitored_change.sender != getpid ()) { + m = rspamd_monitored_by_tag (mctx, cmd->cmd.monitored_change.tag); + + if (m != NULL) { + rspamd_monitored_set_alive (m, cmd->cmd.monitored_change.alive); + rep.reply.monitored_change.status = 1; + msg_info_config ("updated monitored status for %s: %s", + cmd->cmd.monitored_change.tag, + cmd->cmd.monitored_change.alive ? "alive" : "dead"); + } else { + msg_err ("cannot find monitored by tag: %*s", 32, + cmd->cmd.monitored_change.tag); + rep.reply.monitored_change.status = 0; + } + } + + if (write (fd, &rep, sizeof (rep)) != sizeof (rep)) { + msg_err ("cannot write reply to the control socket: %s", + strerror (errno)); + } + + return TRUE; +} + +void +rspamd_worker_init_scanner (struct rspamd_worker *worker, + struct ev_loop *ev_base, + struct rspamd_dns_resolver *resolver, + struct rspamd_lang_detector **plang_det) +{ + rspamd_stat_init (worker->srv->cfg, ev_base); +#ifdef WITH_HYPERSCAN + rspamd_control_worker_add_cmd_handler (worker, + RSPAMD_CONTROL_HYPERSCAN_LOADED, + rspamd_worker_hyperscan_ready, + NULL); +#endif + rspamd_control_worker_add_cmd_handler (worker, + RSPAMD_CONTROL_LOG_PIPE, + rspamd_worker_log_pipe_handler, + worker->srv->cfg); + rspamd_control_worker_add_cmd_handler (worker, + RSPAMD_CONTROL_MONITORED_CHANGE, + rspamd_worker_monitored_handler, + worker->srv->cfg); + + *plang_det = worker->srv->cfg->lang_det; +} + +void +rspamd_controller_store_saved_stats (struct rspamd_main *rspamd_main, + struct rspamd_config *cfg) +{ + struct rspamd_stat *stat; + ucl_object_t *top, *sub; + struct ucl_emitter_functions *efuncs; + gint i, fd; + gchar fpath[PATH_MAX]; + + if (cfg->stats_file == NULL) { + return; + } + + rspamd_snprintf (fpath, sizeof (fpath), "%s.XXXXXXXX", cfg->stats_file); + fd = g_mkstemp_full (fpath, O_WRONLY|O_TRUNC, 00644); + + if (fd == -1) { + msg_err_config ("cannot open for writing controller stats from %s: %s", + fpath, strerror (errno)); + return; + } + + stat = rspamd_main->stat; + + top = ucl_object_typed_new (UCL_OBJECT); + ucl_object_insert_key (top, ucl_object_fromint ( + stat->messages_scanned), "scanned", 0, false); + ucl_object_insert_key (top, ucl_object_fromint ( + stat->messages_learned), "learned", 0, false); + + if (stat->messages_scanned > 0) { + sub = ucl_object_typed_new (UCL_OBJECT); + for (i = METRIC_ACTION_REJECT; i <= METRIC_ACTION_NOACTION; i++) { + ucl_object_insert_key (sub, + ucl_object_fromint (stat->actions_stat[i]), + rspamd_action_to_str (i), 0, false); + } + ucl_object_insert_key (top, sub, "actions", 0, false); + } + + ucl_object_insert_key (top, + ucl_object_fromint (stat->connections_count), + "connections", 0, false); + ucl_object_insert_key (top, + ucl_object_fromint (stat->control_connections_count), + "control_connections", 0, false); + + efuncs = ucl_object_emit_fd_funcs (fd); + if (!ucl_object_emit_full (top, UCL_EMIT_JSON_COMPACT, + efuncs, NULL)) { + msg_err_config ("cannot write stats to %s: %s", + fpath, strerror (errno)); + + unlink (fpath); + } + else { + if (rename (fpath, cfg->stats_file) == -1) { + msg_err_config ("cannot rename stats from %s to %s: %s", + fpath, cfg->stats_file, strerror (errno)); + } + } + + ucl_object_unref (top); + close (fd); + ucl_object_emit_funcs_free (efuncs); +} + +static ev_timer rrd_timer; + +void +rspamd_controller_on_terminate (struct rspamd_worker *worker, + struct rspamd_rrd_file *rrd) +{ + struct rspamd_abstract_worker_ctx *ctx; + + ctx = (struct rspamd_abstract_worker_ctx *)worker->ctx; + rspamd_controller_store_saved_stats (worker->srv, worker->srv->cfg); + + if (rrd) { + ev_timer_stop (ctx->event_loop, &rrd_timer); + msg_info ("closing rrd file: %s", rrd->filename); + rspamd_rrd_close (rrd); + } +} + +static void +rspamd_controller_load_saved_stats (struct rspamd_main *rspamd_main, + struct rspamd_config *cfg) +{ + struct ucl_parser *parser; + ucl_object_t *obj; + const ucl_object_t *elt, *subelt; + struct rspamd_stat *stat, stat_copy; + gint i; + + if (cfg->stats_file == NULL) { + return; + } + + if (access (cfg->stats_file, R_OK) == -1) { + msg_err_config ("cannot load controller stats from %s: %s", + cfg->stats_file, strerror (errno)); + return; + } + + parser = ucl_parser_new (0); + + if (!ucl_parser_add_file (parser, cfg->stats_file)) { + msg_err_config ("cannot parse controller stats from %s: %s", + cfg->stats_file, ucl_parser_get_error (parser)); + ucl_parser_free (parser); + + return; + } + + obj = ucl_parser_get_object (parser); + ucl_parser_free (parser); + + stat = rspamd_main->stat; + memcpy (&stat_copy, stat, sizeof (stat_copy)); + + elt = ucl_object_lookup (obj, "scanned"); + + if (elt != NULL && ucl_object_type (elt) == UCL_INT) { + stat_copy.messages_scanned = ucl_object_toint (elt); + } + + elt = ucl_object_lookup (obj, "learned"); + + if (elt != NULL && ucl_object_type (elt) == UCL_INT) { + stat_copy.messages_learned = ucl_object_toint (elt); + } + + elt = ucl_object_lookup (obj, "actions"); + + if (elt != NULL) { + for (i = METRIC_ACTION_REJECT; i <= METRIC_ACTION_NOACTION; i++) { + subelt = ucl_object_lookup (elt, rspamd_action_to_str (i)); + + if (subelt && ucl_object_type (subelt) == UCL_INT) { + stat_copy.actions_stat[i] = ucl_object_toint (subelt); + } + } + } + + elt = ucl_object_lookup (obj, "connections_count"); + + if (elt != NULL && ucl_object_type (elt) == UCL_INT) { + stat_copy.connections_count = ucl_object_toint (elt); + } + + elt = ucl_object_lookup (obj, "control_connections_count"); + + if (elt != NULL && ucl_object_type (elt) == UCL_INT) { + stat_copy.control_connections_count = ucl_object_toint (elt); + } + + ucl_object_unref (obj); + memcpy (stat, &stat_copy, sizeof (stat_copy)); +} + +struct rspamd_controller_periodics_cbdata { + struct rspamd_worker *worker; + struct rspamd_rrd_file *rrd; + struct rspamd_stat *stat; + ev_timer save_stats_event; +}; + +static void +rspamd_controller_rrd_update (EV_P_ ev_timer *w, int revents) +{ + struct rspamd_controller_periodics_cbdata *cbd = + (struct rspamd_controller_periodics_cbdata *)w->data; + struct rspamd_stat *stat; + GArray ar; + gdouble points[METRIC_ACTION_MAX]; + GError *err = NULL; + guint i; + + g_assert (cbd->rrd != NULL); + stat = cbd->stat; + + for (i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i ++) { + points[i] = stat->actions_stat[i]; + } + + ar.data = (gchar *)points; + ar.len = sizeof (points); + + if (!rspamd_rrd_add_record (cbd->rrd, &ar, rspamd_get_calendar_ticks (), + &err)) { + msg_err ("cannot update rrd file: %e", err); + g_error_free (err); + } + + /* Plan new event */ + ev_timer_again (EV_A_ w); +} + +static void +rspamd_controller_stats_save_periodic (EV_P_ ev_timer *w, int revents) +{ + struct rspamd_controller_periodics_cbdata *cbd = + (struct rspamd_controller_periodics_cbdata *)w->data; + + rspamd_controller_store_saved_stats (cbd->worker->srv, cbd->worker->srv->cfg); + ev_timer_again (EV_A_ w); +} + +void +rspamd_worker_init_controller (struct rspamd_worker *worker, + struct rspamd_rrd_file **prrd) +{ + struct rspamd_abstract_worker_ctx *ctx; + static const ev_tstamp rrd_update_time = 1.0; + + ctx = (struct rspamd_abstract_worker_ctx *)worker->ctx; + rspamd_controller_load_saved_stats (worker->srv, worker->srv->cfg); + + if (worker->index == 0) { + /* Enable periodics and other stuff */ + static struct rspamd_controller_periodics_cbdata cbd; + const ev_tstamp save_stats_interval = 60; /* 1 minute */ + + memset (&cbd, 0, sizeof (cbd)); + cbd.save_stats_event.data = &cbd; + cbd.worker = worker; + cbd.stat = worker->srv->stat; + + ev_timer_init (&cbd.save_stats_event, + rspamd_controller_stats_save_periodic, + save_stats_interval, save_stats_interval); + ev_timer_start (ctx->event_loop, &cbd.save_stats_event); + + rspamd_map_watch (worker->srv->cfg, ctx->event_loop, + ctx->resolver, worker, + RSPAMD_MAP_WATCH_PRIMARY_CONTROLLER); + + if (prrd != NULL) { + if (ctx->cfg->rrd_file && worker->index == 0) { + GError *rrd_err = NULL; + + *prrd = rspamd_rrd_file_default (ctx->cfg->rrd_file, &rrd_err); + + if (*prrd) { + cbd.rrd = *prrd; + rrd_timer.data = &cbd; + ev_timer_init (&rrd_timer, rspamd_controller_rrd_update, + rrd_update_time, rrd_update_time); + ev_timer_start (ctx->event_loop, &rrd_timer); + } + else if (rrd_err) { + msg_err ("cannot load rrd from %s: %e", ctx->cfg->rrd_file, + rrd_err); + g_error_free (rrd_err); + } + else { + msg_err ("cannot load rrd from %s: unknown error", + ctx->cfg->rrd_file); + } + } + else { + *prrd = NULL; + } + } + + if (!ctx->cfg->disable_monitored) { + rspamd_worker_init_monitored (worker, + ctx->event_loop, ctx->resolver); + } + } + else { + rspamd_map_watch (worker->srv->cfg, ctx->event_loop, + ctx->resolver, worker, RSPAMD_MAP_WATCH_SCANNER); + } }
\ No newline at end of file diff --git a/src/libserver/worker_util.h b/src/libserver/worker_util.h index 6fbda0b4a..298243961 100644 --- a/src/libserver/worker_util.h +++ b/src/libserver/worker_util.h @@ -237,6 +237,38 @@ void rspamd_worker_throttle_accept_events (gint sock, void *data); gboolean rspamd_check_termination_clause (struct rspamd_main *rspamd_main, struct rspamd_worker *wrk, int status); +/** + * Call for final scripts for a worker + * @param worker + * @return + */ +gboolean rspamd_worker_call_finish_handlers (struct rspamd_worker *worker); + +struct rspamd_rrd_file; +/** + * Terminate controller worker + * @param worker + */ +void rspamd_controller_on_terminate (struct rspamd_worker *worker, + struct rspamd_rrd_file *rrd); + +/** + * Inits controller worker + * @param worker + * @param ev_base + * @param prrd + */ +void rspamd_worker_init_controller (struct rspamd_worker *worker, + struct rspamd_rrd_file **prrd); + +/** + * Saves stats + * @param rspamd_main + * @param cfg + */ +void rspamd_controller_store_saved_stats (struct rspamd_main *rspamd_main, + struct rspamd_config *cfg); + #ifdef WITH_HYPERSCAN struct rspamd_control_command; diff --git a/src/libstat/backends/redis_backend.c b/src/libstat/backends/redis_backend.c index ec65a133f..bc245bf0b 100644 --- a/src/libstat/backends/redis_backend.c +++ b/src/libstat/backends/redis_backend.c @@ -964,7 +964,7 @@ rspamd_redis_stat_keys (redisAsyncContext *c, gpointer r, gpointer priv) msg_err ("cannot get keys to gather stat: unknown error"); } - rspamd_upstream_fail (cbdata->selected, FALSE); + rspamd_upstream_fail (cbdata->selected, FALSE, c->errstr); rspamd_redis_async_cbdata_cleanup (cbdata); redis_elt->cbdata = NULL; } @@ -1124,7 +1124,7 @@ rspamd_redis_timeout (EV_P_ ev_timer *w, int revents) msg_err_task_check ("connection to redis server %s timed out", rspamd_upstream_name (rt->selected)); - rspamd_upstream_fail (rt->selected, FALSE); + rspamd_upstream_fail (rt->selected, FALSE, "timeout"); if (rt->redis) { redis = rt->redis; @@ -1225,7 +1225,7 @@ rspamd_redis_processed (redisAsyncContext *c, gpointer r, gpointer priv) rspamd_upstream_name (rt->selected), c->errstr); if (rt->redis) { - rspamd_upstream_fail (rt->selected, FALSE); + rspamd_upstream_fail (rt->selected, FALSE, c->errstr); } if (!rt->err) { @@ -1353,7 +1353,7 @@ rspamd_redis_connected (redisAsyncContext *c, gpointer r, gpointer priv) else if (rt->has_event) { msg_err_task ("error getting reply from redis server %s: %s", rspamd_upstream_name (rt->selected), c->errstr); - rspamd_upstream_fail (rt->selected, FALSE); + rspamd_upstream_fail (rt->selected, FALSE, c->errstr); if (!rt->err) { g_set_error (&rt->err, rspamd_redis_stat_quark (), c->err, @@ -1384,7 +1384,7 @@ rspamd_redis_learned (redisAsyncContext *c, gpointer r, gpointer priv) rspamd_upstream_name (rt->selected), c->errstr); if (rt->redis) { - rspamd_upstream_fail (rt->selected, FALSE); + rspamd_upstream_fail (rt->selected, FALSE, c->errstr); } if (!rt->err) { diff --git a/src/libstat/learn_cache/redis_cache.c b/src/libstat/learn_cache/redis_cache.c index 0df3783ab..301232d28 100644 --- a/src/libstat/learn_cache/redis_cache.c +++ b/src/libstat/learn_cache/redis_cache.c @@ -114,7 +114,7 @@ rspamd_redis_cache_timeout (EV_P_ ev_timer *w, int revents) msg_err_task ("connection to redis server %s timed out", rspamd_upstream_name (rt->selected)); - rspamd_upstream_fail (rt->selected, FALSE); + rspamd_upstream_fail (rt->selected, FALSE, "timeout"); if (rt->has_event) { rspamd_session_remove_event (task->s, rspamd_redis_cache_fin, rt); @@ -166,7 +166,7 @@ rspamd_stat_cache_redis_get (redisAsyncContext *c, gpointer r, gpointer priv) rspamd_upstream_ok (rt->selected); } else { - rspamd_upstream_fail (rt->selected, FALSE); + rspamd_upstream_fail (rt->selected, FALSE, c->errstr); } if (rt->has_event) { @@ -188,7 +188,7 @@ rspamd_stat_cache_redis_set (redisAsyncContext *c, gpointer r, gpointer priv) rspamd_upstream_ok (rt->selected); } else { - rspamd_upstream_fail (rt->selected, FALSE); + rspamd_upstream_fail (rt->selected, FALSE, c->errstr); } if (rt->has_event) { diff --git a/src/libstat/stat_process.c b/src/libstat/stat_process.c index 603e0ac96..6bd17f38f 100644 --- a/src/libstat/stat_process.c +++ b/src/libstat/stat_process.c @@ -142,6 +142,7 @@ rspamd_stat_process_tokenize (struct rspamd_stat_ctx *st_ctx, task->tokens = g_ptr_array_sized_new (reserved_len); rspamd_mempool_add_destructor (task->task_pool, rspamd_ptr_array_free_hard, task->tokens); + rspamd_mempool_notify_alloc (task->task_pool, reserved_len * sizeof (gpointer)); pdiff = rspamd_mempool_get_variable (task->task_pool, "parts_distance"); PTR_ARRAY_FOREACH (MESSAGE_FIELD (task, text_parts), i, part) { diff --git a/src/libutil/addr.c b/src/libutil/addr.c index 15480a9ad..03e7e9918 100644 --- a/src/libutil/addr.c +++ b/src/libutil/addr.c @@ -1440,8 +1440,18 @@ rspamd_parse_host_port_priority (const gchar *str, portbuf, 0, pool); } else { + const gchar *second_semicolon = strchr (p + 1, ':'); + name = str; - namelen = p - str; + + if (second_semicolon) { + /* name + port part excluding priority */ + namelen = second_semicolon - str; + } + else { + /* Full ip/name + port */ + namelen = strlen (str); + } if (!rspamd_check_port_priority (p, default_port, priority, portbuf, sizeof (portbuf), pool)) { diff --git a/src/libutil/fstring.h b/src/libutil/fstring.h index 6102215d4..730a59c1c 100644 --- a/src/libutil/fstring.h +++ b/src/libutil/fstring.h @@ -37,6 +37,7 @@ typedef struct f_str_s { #define RSPAMD_FSTRING_DATA(s) ((s)->str) #define RSPAMD_FSTRING_LEN(s) ((s)->len) +#define RSPAMD_FSTRING_LIT(lit) rspamd_fstring_new_init((lit), sizeof(lit) - 1) typedef struct f_str_tok { gsize len; diff --git a/src/libutil/http_connection.c b/src/libutil/http_connection.c index ca87c205a..a71c6fe75 100644 --- a/src/libutil/http_connection.c +++ b/src/libutil/http_connection.c @@ -1197,7 +1197,7 @@ rspamd_http_connection_new_client (struct rspamd_http_context *ctx, msg_info ("cannot connect to http proxy %s: %s", rspamd_inet_address_to_string_pretty (proxy_addr), strerror (errno)); - rspamd_upstream_fail (up, TRUE); + rspamd_upstream_fail (up, TRUE, strerror (errno)); return NULL; } diff --git a/src/libutil/logger.c b/src/libutil/logger.c index 31d018533..4fc14c550 100644 --- a/src/libutil/logger.c +++ b/src/libutil/logger.c @@ -579,7 +579,9 @@ rspamd_set_logger (struct rspamd_config *cfg, rspamd_config_radix_from_ucl (cfg, cfg->debug_ip_map, "IP addresses for which debug logs are enabled", - &logger->debug_ip, NULL); + &logger->debug_ip, + NULL, + NULL); } else if (logger->debug_ip) { rspamd_map_helper_destroy_radix (logger->debug_ip); @@ -1366,6 +1368,42 @@ rspamd_conditional_debug_fast (rspamd_logger_t *rspamd_log, } } +void +rspamd_conditional_debug_fast_num_id (rspamd_logger_t *rspamd_log, + rspamd_inet_addr_t *addr, + guint mod_id, const gchar *module, guint64 id, + const gchar *function, const gchar *fmt, ...) +{ + static gchar logbuf[LOGBUF_LEN], idbuf[64]; + va_list vp; + gchar *end; + + if (rspamd_log == NULL) { + rspamd_log = default_logger; + } + + if (rspamd_logger_need_log (rspamd_log, G_LOG_LEVEL_DEBUG, mod_id) || + rspamd_log->is_debug) { + if (rspamd_log->debug_ip && addr != NULL) { + if (rspamd_match_radix_map_addr (rspamd_log->debug_ip, addr) + == NULL) { + return; + } + } + + rspamd_snprintf (idbuf, sizeof (idbuf), "%XuL", id); + va_start (vp, fmt); + end = rspamd_vsnprintf (logbuf, sizeof (logbuf), fmt, vp); + *end = '\0'; + va_end (vp); + rspamd_log->log_func (module, idbuf, + function, + G_LOG_LEVEL_DEBUG | RSPAMD_LOG_FORCED, + logbuf, + rspamd_log); + } +} + /** * Wrapper for glib logger */ diff --git a/src/libutil/logger.h b/src/libutil/logger.h index 1dff75211..dd980445e 100644 --- a/src/libutil/logger.h +++ b/src/libutil/logger.h @@ -133,7 +133,13 @@ void rspamd_conditional_debug (rspamd_logger_t *logger, void rspamd_conditional_debug_fast (rspamd_logger_t *logger, rspamd_inet_addr_t *addr, - guint mod_id, const gchar *module, const gchar *id, + guint mod_id, + const gchar *module, const gchar *id, + const gchar *function, const gchar *fmt, ...); +void rspamd_conditional_debug_fast_num_id (rspamd_logger_t *logger, + rspamd_inet_addr_t *addr, + guint mod_id, + const gchar *module, guint64 id, const gchar *function, const gchar *fmt, ...); /** diff --git a/src/libutil/map.c b/src/libutil/map.c index 441b408ba..e5aae11ea 100644 --- a/src/libutil/map.c +++ b/src/libutil/map.c @@ -970,10 +970,14 @@ rspamd_map_periodic_dtor (struct map_periodic_cbdata *periodic) g_atomic_int_set (periodic->map->locked, 0); msg_debug_map ("unlocked map %s", periodic->map->name); - if (!periodic->map->wrk->wanna_die) { + if (periodic->map->wrk->state == rspamd_worker_state_running) { rspamd_map_schedule_periodic (periodic->map, RSPAMD_SYMBOL_RESULT_NORMAL); } + else { + msg_debug_map ("stop scheduling periodics for %s; terminating state", + periodic->map->name); + } } g_free (periodic); @@ -1001,8 +1005,12 @@ rspamd_map_schedule_periodic (struct rspamd_map *map, int how) gdouble timeout; struct map_periodic_cbdata *cbd; - if (map->scheduled_check || (map->wrk && map->wrk->wanna_die)) { - /* Do not schedule check if some check is already scheduled */ + if (map->scheduled_check || (map->wrk && + map->wrk->state != rspamd_worker_state_running)) { + /* + * Do not schedule check if some check is already scheduled or + * if worker is going to die + */ return; } @@ -1573,8 +1581,9 @@ rspamd_map_read_http_cached_file (struct rspamd_map *map, msg_info_map ("read cached data for %s from %s, %uz bytes; next check at: %s;" " last modified on: %s; etag: %V", - bk->uri, path, - st.st_size - header.data_off, + bk->uri, + path, + (size_t)(st.st_size - header.data_off), ncheck_buf, lm_buf, htdata->etag); @@ -1856,7 +1865,8 @@ rspamd_map_process_periodic (struct map_periodic_cbdata *cbd) map->scheduled_check = NULL; if (!map->file_only && !cbd->locked) { - if (!g_atomic_int_compare_and_exchange (cbd->map->locked, 0, 1)) { + if (!g_atomic_int_compare_and_exchange (cbd->map->locked, + 0, 1)) { msg_debug_map ( "don't try to reread map %s as it is locked by other process, " "will reread it later", cbd->map->name); @@ -1897,7 +1907,7 @@ rspamd_map_process_periodic (struct map_periodic_cbdata *cbd) return; } - if (!(cbd->map->wrk && cbd->map->wrk->wanna_die)) { + if (cbd->map->wrk && cbd->map->wrk->state == rspamd_worker_state_running) { bk = g_ptr_array_index (cbd->map->backends, cbd->cur_backend); g_assert (bk != NULL); @@ -1976,22 +1986,40 @@ rspamd_map_watch (struct rspamd_config *cfg, struct ev_loop *event_loop, struct rspamd_dns_resolver *resolver, struct rspamd_worker *worker, - gboolean active_http) + enum rspamd_map_watch_type how) { GList *cur = cfg->maps; struct rspamd_map *map; struct rspamd_map_backend *bk; guint i; + g_assert (how > RSPAMD_MAP_WATCH_MIN && how < RSPAMD_MAP_WATCH_MAX); + /* First of all do synced read of data */ while (cur) { map = cur->data; map->event_loop = event_loop; map->r = resolver; - map->wrk = worker; - if (active_http) { - map->active_http = active_http; + if (map->wrk == NULL && how != RSPAMD_MAP_WATCH_WORKER) { + /* Generic scanner map */ + map->wrk = worker; + + if (how == RSPAMD_MAP_WATCH_PRIMARY_CONTROLLER) { + map->active_http = TRUE; + } + else { + map->active_http = FALSE; + } + } + else if (map->wrk != NULL && map->wrk == worker) { + /* Map is bound to a specific worker */ + map->active_http = TRUE; + } + else { + /* Skip map for this worker as irrelevant */ + cur = g_list_next (cur); + continue; } if (!map->active_http) { @@ -2581,7 +2609,8 @@ rspamd_map_add (struct rspamd_config *cfg, map_cb_t read_callback, map_fin_cb_t fin_callback, map_dtor_t dtor, - void **user_data) + void **user_data, + struct rspamd_worker *worker) { struct rspamd_map *map; struct rspamd_map_backend *bk; @@ -2608,6 +2637,7 @@ rspamd_map_add (struct rspamd_config *cfg, map->locked = rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint)); map->backends = g_ptr_array_sized_new (1); + map->wrk = worker; rspamd_mempool_add_destructor (cfg->cfg_pool, rspamd_ptr_array_free_hard, map->backends); g_ptr_array_add (map->backends, bk); @@ -2654,7 +2684,8 @@ rspamd_map_add_from_ucl (struct rspamd_config *cfg, map_cb_t read_callback, map_fin_cb_t fin_callback, map_dtor_t dtor, - void **user_data) + void **user_data, + struct rspamd_worker *worker) { ucl_object_iter_t it = NULL; const ucl_object_t *cur, *elt; @@ -2667,7 +2698,7 @@ rspamd_map_add_from_ucl (struct rspamd_config *cfg, if (ucl_object_type (obj) == UCL_STRING) { /* Just a plain string */ return rspamd_map_add (cfg, ucl_object_tostring (obj), description, - read_callback, fin_callback, dtor, user_data); + read_callback, fin_callback, dtor, user_data, worker); } map = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_map)); @@ -2680,6 +2711,7 @@ rspamd_map_add_from_ucl (struct rspamd_config *cfg, map->locked = rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint)); map->backends = g_ptr_array_new (); + map->wrk = worker; rspamd_mempool_add_destructor (cfg->cfg_pool, rspamd_ptr_array_free_hard, map->backends); map->poll_timeout = cfg->map_timeout; diff --git a/src/libutil/map.h b/src/libutil/map.h index 9e09ab8fe..ce49bacbb 100644 --- a/src/libutil/map.h +++ b/src/libutil/map.h @@ -70,7 +70,8 @@ struct rspamd_map *rspamd_map_add (struct rspamd_config *cfg, map_cb_t read_callback, map_fin_cb_t fin_callback, map_dtor_t dtor, - void **user_data); + void **user_data, + struct rspamd_worker *worker); /** * Add map from ucl @@ -81,7 +82,16 @@ struct rspamd_map *rspamd_map_add_from_ucl (struct rspamd_config *cfg, map_cb_t read_callback, map_fin_cb_t fin_callback, map_dtor_t dtor, - void **user_data); + void **user_data, + struct rspamd_worker *worker); + +enum rspamd_map_watch_type { + RSPAMD_MAP_WATCH_MIN = 9, + RSPAMD_MAP_WATCH_PRIMARY_CONTROLLER, + RSPAMD_MAP_WATCH_SCANNER, + RSPAMD_MAP_WATCH_WORKER, + RSPAMD_MAP_WATCH_MAX +}; /** * Start watching of maps by adding events to libevent event loop @@ -90,7 +100,7 @@ void rspamd_map_watch (struct rspamd_config *cfg, struct ev_loop *event_loop, struct rspamd_dns_resolver *resolver, struct rspamd_worker *worker, - gboolean active_http); + enum rspamd_map_watch_type how); /** * Preloads maps where all backends are file diff --git a/src/libutil/map_helpers.c b/src/libutil/map_helpers.c index a9bd8d70e..d179d44f5 100644 --- a/src/libutil/map_helpers.c +++ b/src/libutil/map_helpers.c @@ -20,6 +20,7 @@ #include "radix.h" #include "rspamd.h" #include "cryptobox.h" +#include "contrib/fastutf8/fastutf8.h" #ifdef WITH_HYPERSCAN #include "hs.h" @@ -630,11 +631,11 @@ rspamd_map_helper_new_hash (struct rspamd_map *map) if (map) { pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), - map->tag); + map->tag, 0); } else { pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), - NULL); + NULL, 0); } htb = rspamd_mempool_alloc0 (pool, sizeof (*htb)); @@ -687,11 +688,11 @@ rspamd_map_helper_new_radix (struct rspamd_map *map) if (map) { pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), - map->tag); + map->tag, 0); } else { pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), - NULL); + NULL, 0); } r = rspamd_mempool_alloc0 (pool, sizeof (*r)); @@ -745,7 +746,7 @@ rspamd_map_helper_new_regexp (struct rspamd_map *map, rspamd_mempool_t *pool; pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), - map->tag); + map->tag, 0); re_map = rspamd_mempool_alloc0 (pool, sizeof (*re_map)); re_map->pool = pool; @@ -1189,7 +1190,7 @@ rspamd_match_regexp_map_single (struct rspamd_regexp_map_helper *map, } if (map->map_flags & RSPAMD_REGEXP_MAP_FLAG_UTF) { - if (g_utf8_validate (in, len, NULL)) { + if (rspamd_fast_utf8_validate (in, len) == 0) { validated = TRUE; } } @@ -1280,7 +1281,7 @@ rspamd_match_regexp_map_all (struct rspamd_regexp_map_helper *map, g_assert (in != NULL); if (map->map_flags & RSPAMD_REGEXP_MAP_FLAG_UTF) { - if (g_utf8_validate (in, len, NULL)) { + if (rspamd_fast_utf8_validate (in, len) == 0) { validated = TRUE; } } diff --git a/src/libutil/mem_pool.c b/src/libutil/mem_pool.c index c01ce0c2c..d17c72bb8 100644 --- a/src/libutil/mem_pool.c +++ b/src/libutil/mem_pool.c @@ -22,6 +22,7 @@ #include "khash.h" #include "cryptobox.h" #include "contrib/uthash/utlist.h" +#include "mem_pool_internal.h" #ifdef WITH_JEMALLOC #include <jemalloc/jemalloc.h> @@ -57,22 +58,6 @@ #undef MEMORY_GREEDY -#define ENTRY_LEN 128 -#define ENTRY_NELTS 64 - -struct entry_elt { - guint32 fragmentation; - guint32 leftover; -}; - -struct rspamd_mempool_entry_point { - gchar src[ENTRY_LEN]; - guint32 cur_suggestion; - guint32 cur_elts; - struct entry_elt elts[ENTRY_NELTS]; -}; - - static inline uint32_t rspamd_entry_hash (const char *str) { @@ -233,11 +218,11 @@ rspamd_mempool_chain_new (gsize size, enum rspamd_mempool_chain_type pool_type) optimal_size = sys_alloc_size (total_size); #endif total_size = MAX (total_size, optimal_size); - map = malloc (total_size); + gint ret = posix_memalign (&map, MIN_MEM_ALIGNMENT, total_size); - if (map == NULL) { - g_error ("%s: failed to allocate %"G_GSIZE_FORMAT" bytes", - G_STRLOC, total_size); + if (ret != 0 || map == NULL) { + g_error ("%s: failed to allocate %"G_GSIZE_FORMAT" bytes: %d - %s", + G_STRLOC, total_size, ret, strerror (errno)); abort (); } @@ -249,7 +234,6 @@ rspamd_mempool_chain_new (gsize size, enum rspamd_mempool_chain_type pool_type) chain->pos = align_ptr (chain->begin, MIN_MEM_ALIGNMENT); chain->slice_size = total_size - sizeof (struct _pool_chain); - chain->lock = NULL; return chain; } @@ -268,7 +252,7 @@ rspamd_mempool_get_chain (rspamd_mempool_t * pool, { g_assert (pool_type >= 0 && pool_type < RSPAMD_MEMPOOL_MAX); - return pool->pools[pool_type]; + return pool->priv->pools[pool_type]; } static void @@ -279,7 +263,7 @@ rspamd_mempool_append_chain (rspamd_mempool_t * pool, g_assert (pool_type >= 0 && pool_type < RSPAMD_MEMPOOL_MAX); g_assert (chain != NULL); - LL_PREPEND (pool->pools[pool_type], chain); + LL_PREPEND (pool->priv->pools[pool_type], chain); } /** @@ -288,7 +272,7 @@ rspamd_mempool_append_chain (rspamd_mempool_t * pool, * @return new memory pool object */ rspamd_mempool_t * -rspamd_mempool_new_ (gsize size, const gchar *tag, const gchar *loc) +rspamd_mempool_new_ (gsize size, const gchar *tag, gint flags, const gchar *loc) { rspamd_mempool_t *new_pool; gpointer map; @@ -345,19 +329,67 @@ rspamd_mempool_new_ (gsize size, const gchar *tag, const gchar *loc) env_checked = TRUE; } - new_pool = g_malloc0 (sizeof (rspamd_mempool_t)); - new_pool->entry = rspamd_mempool_get_entry (loc); - new_pool->destructors = g_array_sized_new (FALSE, FALSE, - sizeof (struct _pool_destructors), 32); - /* Set it upon first call of set variable */ + struct rspamd_mempool_entry_point *entry = rspamd_mempool_get_entry (loc); + gsize total_size; + + if (size == 0 && entry) { + size = entry->cur_suggestion; + } + + total_size = sizeof (rspamd_mempool_t) + + sizeof (struct rspamd_mempool_specific) + + MIN_MEM_ALIGNMENT + + sizeof (struct _pool_chain) + + size; - if (size == 0) { - new_pool->elt_len = new_pool->entry->cur_suggestion; + if (G_UNLIKELY (flags & RSPAMD_MEMPOOL_DEBUG)) { + total_size += sizeof (GHashTable *); + } + /* + * Memory layout: + * struct rspamd_mempool_t + * <optional debug hash table> + * struct rspamd_mempool_specific + * struct _pool_chain + * alignment (if needed) + * memory chunk + */ + guchar *mem_chunk; + gint ret = posix_memalign ((void **)&mem_chunk, MIN_MEM_ALIGNMENT, + total_size); + gsize priv_offset; + + if (ret != 0 || mem_chunk == NULL) { + g_error ("%s: failed to allocate %"G_GSIZE_FORMAT" bytes: %d - %s", + G_STRLOC, total_size, ret, strerror (errno)); + abort (); + } + + /* Set memory layout */ + new_pool = (rspamd_mempool_t *)mem_chunk; + if (G_UNLIKELY (flags & RSPAMD_MEMPOOL_DEBUG)) { + /* Allocate debug table */ + GHashTable *debug_tbl; + + debug_tbl = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); + memcpy (mem_chunk + sizeof (rspamd_mempool_t), &debug_tbl, + sizeof (GHashTable *)); + priv_offset = sizeof (rspamd_mempool_t) + sizeof (GHashTable *); } else { - new_pool->elt_len = size; + priv_offset = sizeof (rspamd_mempool_t); } + new_pool->priv = (struct rspamd_mempool_specific *)(mem_chunk + + priv_offset); + /* Zero memory for specific and for the first chain */ + memset (new_pool->priv, 0, sizeof (struct rspamd_mempool_specific) + + sizeof (struct _pool_chain)); + + new_pool->priv->entry = entry; + new_pool->priv->elt_len = size; + new_pool->priv->flags = flags; + if (tag) { rspamd_strlcpy (new_pool->tag.tagname, tag, sizeof (new_pool->tag.tagname)); } @@ -375,17 +407,64 @@ rspamd_mempool_new_ (gsize size, const gchar *tag, const gchar *loc) mem_pool_stat->pools_allocated++; + /* Now we can attach one chunk to speed up simple allocations */ + struct _pool_chain *nchain; + + nchain = (struct _pool_chain *) + (mem_chunk + + priv_offset + + sizeof (struct rspamd_mempool_specific)); + + guchar *unaligned = mem_chunk + + priv_offset + + sizeof (struct rspamd_mempool_specific) + + sizeof (struct _pool_chain); + + nchain->slice_size = size; + nchain->begin = unaligned; + nchain->slice_size = size; + nchain->pos = align_ptr (unaligned, MIN_MEM_ALIGNMENT); + new_pool->priv->pools[RSPAMD_MEMPOOL_NORMAL] = nchain; + new_pool->priv->used_memory = size; + + /* Adjust stats */ + g_atomic_int_add (&mem_pool_stat->bytes_allocated, + (gint)size); + g_atomic_int_add (&mem_pool_stat->chunks_allocated, 1); + return new_pool; } static void * memory_pool_alloc_common (rspamd_mempool_t * pool, gsize size, - enum rspamd_mempool_chain_type pool_type) + enum rspamd_mempool_chain_type pool_type, + const gchar *loc) RSPAMD_ATTR_ALLOC_SIZE(2) RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT) RSPAMD_ATTR_RETURNS_NONNUL; + +void +rspamd_mempool_notify_alloc_ (rspamd_mempool_t *pool, gsize size, const gchar *loc) +{ + if (pool && G_UNLIKELY (pool->priv->flags & RSPAMD_MEMPOOL_DEBUG)) { + GHashTable *debug_tbl = *(GHashTable **)(((guchar *)pool + sizeof (*pool))); + gpointer ptr; + + ptr = g_hash_table_lookup (debug_tbl, loc); + + if (ptr) { + ptr = GSIZE_TO_POINTER (GPOINTER_TO_SIZE (ptr) + size); + } + else { + ptr = GSIZE_TO_POINTER (size); + } + + g_hash_table_insert (debug_tbl, (gpointer) loc, ptr); + } +} + static void * memory_pool_alloc_common (rspamd_mempool_t * pool, gsize size, - enum rspamd_mempool_chain_type pool_type) + enum rspamd_mempool_chain_type pool_type, const gchar *loc) { guint8 *tmp; struct _pool_chain *new, *cur; @@ -393,17 +472,23 @@ memory_pool_alloc_common (rspamd_mempool_t * pool, gsize size, if (pool) { POOL_MTX_LOCK (); + pool->priv->used_memory += size; + + if (G_UNLIKELY (pool->priv->flags & RSPAMD_MEMPOOL_DEBUG)) { + rspamd_mempool_notify_alloc_ (pool, size, loc); + } + if (always_malloc && pool_type != RSPAMD_MEMPOOL_SHARED) { void *ptr; ptr = g_malloc (size); POOL_MTX_UNLOCK (); - if (pool->trash_stack == NULL) { - pool->trash_stack = g_ptr_array_sized_new (128); + if (pool->priv->trash_stack == NULL) { + pool->priv->trash_stack = g_ptr_array_sized_new (128); } - g_ptr_array_add (pool->trash_stack, ptr); + g_ptr_array_add (pool->priv->trash_stack, ptr); return ptr; } @@ -416,18 +501,22 @@ memory_pool_alloc_common (rspamd_mempool_t * pool, gsize size, } if (cur == NULL || free < size) { + if (free < size) { + pool->priv->wasted_memory += free; + } + /* Allocate new chain element */ - if (pool->elt_len >= size + MIN_MEM_ALIGNMENT) { - pool->entry->elts[pool->entry->cur_elts].fragmentation += size; - new = rspamd_mempool_chain_new (pool->elt_len, + if (pool->priv->elt_len >= size + MIN_MEM_ALIGNMENT) { + pool->priv->entry->elts[pool->priv->entry->cur_elts].fragmentation += size; + new = rspamd_mempool_chain_new (pool->priv->elt_len, pool_type); } else { mem_pool_stat->oversized_chunks++; g_atomic_int_add (&mem_pool_stat->fragmented_size, free); - pool->entry->elts[pool->entry->cur_elts].fragmentation += free; - new = rspamd_mempool_chain_new (size + pool->elt_len, pool_type); + pool->priv->entry->elts[pool->priv->entry->cur_elts].fragmentation += free; + new = rspamd_mempool_chain_new (size + pool->priv->elt_len, pool_type); } /* Connect to pool subsystem */ @@ -453,39 +542,21 @@ memory_pool_alloc_common (rspamd_mempool_t * pool, gsize size, void * -rspamd_mempool_alloc (rspamd_mempool_t * pool, gsize size) -{ - return memory_pool_alloc_common (pool, size, RSPAMD_MEMPOOL_NORMAL); -} - -void * -rspamd_mempool_alloc_tmp (rspamd_mempool_t * pool, gsize size) -{ - return memory_pool_alloc_common (pool, size, RSPAMD_MEMPOOL_TMP); -} - -void * -rspamd_mempool_alloc0 (rspamd_mempool_t * pool, gsize size) +rspamd_mempool_alloc_ (rspamd_mempool_t * pool, gsize size, const gchar *loc) { - void *pointer = rspamd_mempool_alloc (pool, size); - - memset (pointer, 0, size); - - return pointer; + return memory_pool_alloc_common (pool, size, RSPAMD_MEMPOOL_NORMAL, loc); } void * -rspamd_mempool_alloc0_tmp (rspamd_mempool_t * pool, gsize size) +rspamd_mempool_alloc0_ (rspamd_mempool_t * pool, gsize size, const gchar *loc) { - void *pointer = rspamd_mempool_alloc_tmp (pool, size); - + void *pointer = rspamd_mempool_alloc_ (pool, size, loc); memset (pointer, 0, size); return pointer; } - void * -rspamd_mempool_alloc0_shared (rspamd_mempool_t * pool, gsize size) +rspamd_mempool_alloc0_shared_ (rspamd_mempool_t * pool, gsize size, const gchar *loc) { void *pointer = rspamd_mempool_alloc_shared (pool, size); @@ -494,14 +565,14 @@ rspamd_mempool_alloc0_shared (rspamd_mempool_t * pool, gsize size) } void * -rspamd_mempool_alloc_shared (rspamd_mempool_t * pool, gsize size) +rspamd_mempool_alloc_shared_ (rspamd_mempool_t * pool, gsize size, const gchar *loc) { - return memory_pool_alloc_common (pool, size, RSPAMD_MEMPOOL_SHARED); + return memory_pool_alloc_common (pool, size, RSPAMD_MEMPOOL_SHARED, loc); } gchar * -rspamd_mempool_strdup (rspamd_mempool_t * pool, const gchar *src) +rspamd_mempool_strdup_ (rspamd_mempool_t * pool, const gchar *src, const gchar *loc) { gsize len; gchar *newstr; @@ -511,7 +582,7 @@ rspamd_mempool_strdup (rspamd_mempool_t * pool, const gchar *src) } len = strlen (src); - newstr = rspamd_mempool_alloc (pool, len + 1); + newstr = rspamd_mempool_alloc_ (pool, len + 1, loc); memcpy (newstr, src, len); newstr[len] = '\0'; @@ -519,7 +590,8 @@ rspamd_mempool_strdup (rspamd_mempool_t * pool, const gchar *src) } gchar * -rspamd_mempool_fstrdup (rspamd_mempool_t * pool, const struct f_str_s *src) +rspamd_mempool_fstrdup_ (rspamd_mempool_t * pool, const struct f_str_s *src, + const gchar *loc) { gchar *newstr; @@ -527,7 +599,7 @@ rspamd_mempool_fstrdup (rspamd_mempool_t * pool, const struct f_str_s *src) return NULL; } - newstr = rspamd_mempool_alloc (pool, src->len + 1); + newstr = rspamd_mempool_alloc_ (pool, src->len + 1, loc); memcpy (newstr, src->str, src->len); newstr[src->len] = '\0'; @@ -535,7 +607,8 @@ rspamd_mempool_fstrdup (rspamd_mempool_t * pool, const struct f_str_s *src) } gchar * -rspamd_mempool_ftokdup (rspamd_mempool_t *pool, const rspamd_ftok_t *src) +rspamd_mempool_ftokdup_ (rspamd_mempool_t *pool, const rspamd_ftok_t *src, + const gchar *loc) { gchar *newstr; @@ -543,7 +616,7 @@ rspamd_mempool_ftokdup (rspamd_mempool_t *pool, const rspamd_ftok_t *src) return NULL; } - newstr = rspamd_mempool_alloc (pool, src->len + 1); + newstr = rspamd_mempool_alloc_ (pool, src->len + 1, loc); memcpy (newstr, src->begin, src->len); newstr[src->len] = '\0'; @@ -557,15 +630,25 @@ rspamd_mempool_add_destructor_full (rspamd_mempool_t * pool, const gchar *function, const gchar *line) { - struct _pool_destructors cur; + struct _pool_destructors *cur; POOL_MTX_LOCK (); - cur.func = func; - cur.data = data; - cur.function = function; - cur.loc = line; + cur = rspamd_mempool_alloc_ (pool, sizeof (*cur), line); + cur->func = func; + cur->data = data; + cur->function = function; + cur->loc = line; + cur->next = NULL; + + if (pool->priv->dtors_tail) { + pool->priv->dtors_tail->next = cur; + pool->priv->dtors_tail = cur; + } + else { + pool->priv->dtors_head = cur; + pool->priv->dtors_tail = cur; + } - g_array_append_val (pool->destructors, cur); POOL_MTX_UNLOCK (); } @@ -576,11 +659,8 @@ rspamd_mempool_replace_destructor (rspamd_mempool_t * pool, void *new_data) { struct _pool_destructors *tmp; - guint i; - - for (i = 0; i < pool->destructors->len; i ++) { - tmp = &g_array_index (pool->destructors, struct _pool_destructors, i); + LL_FOREACH (pool->priv->dtors_head, tmp) { if (tmp->func == func && tmp->data == old_data) { tmp->func = func; tmp->data = new_data; @@ -615,11 +695,7 @@ rspamd_mempool_adjust_entry (struct rspamd_mempool_entry_point *e) sel_pos = sz[50 + jitter]; sel_neg = sz[4 + jitter]; - if (sel_neg > 0) { - /* We need to increase our suggestion */ - e->cur_suggestion *= (1 + (((double)sel_pos) / e->cur_suggestion)) * 1.5; - } - else if (-sel_neg > sel_pos) { + if (-sel_neg > sel_pos) { /* We need to reduce current suggestion */ e->cur_suggestion /= (1 + (((double)-sel_neg) / e->cur_suggestion)) * 1.5; } @@ -643,23 +719,35 @@ void rspamd_mempool_destructors_enforce (rspamd_mempool_t *pool) { struct _pool_destructors *destructor; - guint i; POOL_MTX_LOCK (); - for (i = 0; i < pool->destructors->len; i ++) { - destructor = &g_array_index (pool->destructors, struct _pool_destructors, i); + LL_FOREACH (pool->priv->dtors_head, destructor) { /* Avoid calling destructors for NULL pointers */ if (destructor->data != NULL) { destructor->func (destructor->data); } } - pool->destructors->len = 0; + pool->priv->dtors_head = pool->priv->dtors_tail = NULL; POOL_MTX_UNLOCK (); } +struct mempool_debug_elt { + gsize sz; + const gchar *loc; +}; + +static gint +rspamd_mempool_debug_elt_cmp (const void *a, const void *b) +{ + const struct mempool_debug_elt *e1 = a, *e2 = b; + + /* Inverse order */ + return (gint)((gssize)e2->sz) - ((gssize)e1->sz); +} + void rspamd_mempool_delete (rspamd_mempool_t * pool) { @@ -671,38 +759,74 @@ rspamd_mempool_delete (rspamd_mempool_t * pool) POOL_MTX_LOCK (); - cur = NULL; + cur = pool->priv->pools[RSPAMD_MEMPOOL_NORMAL]; + + if (G_UNLIKELY (pool->priv->flags & RSPAMD_MEMPOOL_DEBUG)) { + GHashTable *debug_tbl = *(GHashTable **)(((guchar *)pool) + sizeof (*pool)); + /* Show debug info */ + gsize ndtor = 0; + LL_COUNT (pool->priv->dtors_head, destructor, ndtor); + msg_info_pool ("destructing of the memory pool %p; elt size = %z; " + "used memory = %Hz; wasted memory = %Hd; " + "vars = %z; destructors = %z", + pool, + pool->priv->elt_len, + pool->priv->used_memory, + pool->priv->wasted_memory, + pool->priv->variables ? g_hash_table_size (pool->priv->variables) : (gsize)0, + ndtor); + + GHashTableIter it; + gpointer k, v; + GArray *sorted_debug_size = g_array_sized_new (FALSE, FALSE, + sizeof (struct mempool_debug_elt), + g_hash_table_size (debug_tbl)); + + g_hash_table_iter_init (&it, debug_tbl); + + while (g_hash_table_iter_next (&it, &k, &v)) { + struct mempool_debug_elt e; + e.loc = (const gchar *)k; + e.sz = GPOINTER_TO_SIZE (v); + g_array_append_val (sorted_debug_size, e); + } + + g_array_sort (sorted_debug_size, rspamd_mempool_debug_elt_cmp); + + for (guint _i = 0; _i < sorted_debug_size->len; _i ++) { + struct mempool_debug_elt *e; - if (pool->pools[RSPAMD_MEMPOOL_NORMAL] != NULL) { - cur = pool->pools[RSPAMD_MEMPOOL_NORMAL]; + e = &g_array_index (sorted_debug_size, struct mempool_debug_elt, _i); + msg_info_pool ("allocated %Hz from %s", e->sz, e->loc); + } + + g_array_free (sorted_debug_size, TRUE); + g_hash_table_unref (debug_tbl); } if (cur && mempool_entries) { - pool->entry->elts[pool->entry->cur_elts].leftover = + pool->priv->entry->elts[pool->priv->entry->cur_elts].leftover = pool_chain_free (cur); - pool->entry->cur_elts = (pool->entry->cur_elts + 1) % - G_N_ELEMENTS (pool->entry->elts); + pool->priv->entry->cur_elts = (pool->priv->entry->cur_elts + 1) % + G_N_ELEMENTS (pool->priv->entry->elts); - if (pool->entry->cur_elts == 0) { - rspamd_mempool_adjust_entry (pool->entry); + if (pool->priv->entry->cur_elts == 0) { + rspamd_mempool_adjust_entry (pool->priv->entry); } } /* Call all pool destructors */ - for (i = 0; i < pool->destructors->len; i ++) { - destructor = &g_array_index (pool->destructors, struct _pool_destructors, i); + LL_FOREACH (pool->priv->dtors_head, destructor) { /* Avoid calling destructors for NULL pointers */ if (destructor->data != NULL) { destructor->func (destructor->data); } } - g_array_free (pool->destructors, TRUE); - - for (i = 0; i < G_N_ELEMENTS (pool->pools); i ++) { - if (pool->pools[i]) { - LL_FOREACH_SAFE (pool->pools[i], cur, tmp) { + for (i = 0; i < G_N_ELEMENTS (pool->priv->pools); i ++) { + if (pool->priv->pools[i]) { + LL_FOREACH_SAFE (pool->priv->pools[i], cur, tmp) { g_atomic_int_add (&mem_pool_stat->bytes_allocated, -((gint)cur->slice_size)); g_atomic_int_add (&mem_pool_stat->chunks_allocated, -1); @@ -713,51 +837,31 @@ rspamd_mempool_delete (rspamd_mempool_t * pool) munmap ((void *)cur, len); } else { - free (cur); /* Not g_free as we use system allocator */ + /* The last pool is special, it is a part of the initial chunk */ + if (cur->next != NULL) { + free (cur); /* Not g_free as we use system allocator */ + } } } } } - if (pool->variables) { - g_hash_table_destroy (pool->variables); + if (pool->priv->variables) { + g_hash_table_destroy (pool->priv->variables); } - if (pool->trash_stack) { - for (i = 0; i < pool->trash_stack->len; i++) { - ptr = g_ptr_array_index (pool->trash_stack, i); + if (pool->priv->trash_stack) { + for (i = 0; i < pool->priv->trash_stack->len; i++) { + ptr = g_ptr_array_index (pool->priv->trash_stack, i); g_free (ptr); } - g_ptr_array_free (pool->trash_stack, TRUE); - } - - g_atomic_int_inc (&mem_pool_stat->pools_freed); - POOL_MTX_UNLOCK (); - g_free (pool); -} - -void -rspamd_mempool_cleanup_tmp (rspamd_mempool_t * pool) -{ - struct _pool_chain *cur, *tmp; - - POOL_MTX_LOCK (); - - if (pool->pools[RSPAMD_MEMPOOL_TMP]) { - LL_FOREACH_SAFE (pool->pools[RSPAMD_MEMPOOL_TMP], cur, tmp) { - g_atomic_int_add (&mem_pool_stat->bytes_allocated, - -((gint)cur->slice_size)); - g_atomic_int_add (&mem_pool_stat->chunks_allocated, -1); - - free (cur); - } - - pool->pools[RSPAMD_MEMPOOL_TMP] = NULL; + g_ptr_array_free (pool->priv->trash_stack, TRUE); } g_atomic_int_inc (&mem_pool_stat->pools_freed); POOL_MTX_UNLOCK (); + free (pool); /* allocated by posix_memalign */ } void @@ -1009,11 +1113,11 @@ rspamd_mempool_set_variable (rspamd_mempool_t *pool, gpointer value, rspamd_mempool_destruct_t destructor) { - if (pool->variables == NULL) { - pool->variables = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); + if (pool->priv->variables == NULL) { + pool->priv->variables = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); } - g_hash_table_insert (pool->variables, rspamd_mempool_strdup (pool, + g_hash_table_insert (pool->priv->variables, rspamd_mempool_strdup (pool, name), value); if (destructor != NULL) { rspamd_mempool_add_destructor (pool, destructor, value); @@ -1023,18 +1127,18 @@ rspamd_mempool_set_variable (rspamd_mempool_t *pool, gpointer rspamd_mempool_get_variable (rspamd_mempool_t *pool, const gchar *name) { - if (pool->variables == NULL) { + if (pool->priv->variables == NULL) { return NULL; } - return g_hash_table_lookup (pool->variables, name); + return g_hash_table_lookup (pool->priv->variables, name); } void rspamd_mempool_remove_variable (rspamd_mempool_t *pool, const gchar *name) { - if (pool->variables != NULL) { - g_hash_table_remove (pool->variables, name); + if (pool->priv->variables != NULL) { + g_hash_table_remove (pool->priv->variables, name); } } @@ -1079,3 +1183,15 @@ rspamd_mempool_glist_append (rspamd_mempool_t *pool, GList *l, gpointer p) return l; } + +gsize +rspamd_mempool_get_used_size (rspamd_mempool_t *pool) +{ + return pool->priv->used_memory; +} + +gsize +rspamd_mempool_get_wasted_size (rspamd_mempool_t *pool) +{ + return pool->priv->wasted_memory; +}
\ No newline at end of file diff --git a/src/libutil/mem_pool.h b/src/libutil/mem_pool.h index 5a663e35f..1554edbd2 100644 --- a/src/libutil/mem_pool.h +++ b/src/libutil/mem_pool.h @@ -1,3 +1,19 @@ +/*- + * Copyright 2019 Vsevolod Stakhov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /** * @file mem_pool.h * \brief Memory pools library. @@ -50,17 +66,8 @@ struct f_str_s; #define MEMPOOL_TAG_LEN 20 #define MEMPOOL_UID_LEN 20 +/* All pointers are aligned as this variable */ #define MIN_MEM_ALIGNMENT sizeof (guint64) -#define align_ptr(p, a) \ - (guint8 *) (((uintptr_t) (p) + ((uintptr_t) a - 1)) & ~((uintptr_t) a - 1)) - -enum rspamd_mempool_chain_type { - RSPAMD_MEMPOOL_NORMAL = 0, - RSPAMD_MEMPOOL_TMP, - RSPAMD_MEMPOOL_SHARED, - RSPAMD_MEMPOOL_MAX -}; - /** * Destructor type definition */ @@ -88,27 +95,6 @@ typedef pthread_rwlock_t rspamd_mempool_rwlock_t; #endif /** - * Pool page structure - */ -struct _pool_chain { - guint8 *begin; /**< begin of pool chain block */ - guint8 *pos; /**< current start of free space in block */ - gsize slice_size; /**< length of block */ - rspamd_mempool_mutex_t *lock; - struct _pool_chain *next; -}; - -/** - * Destructors list item structure - */ -struct _pool_destructors { - rspamd_mempool_destruct_t func; /**< pointer to destructor */ - void *data; /**< data to free */ - const gchar *function; /**< function from which this destructor was added */ - const gchar *loc; /**< line number */ -}; - -/** * Tag to use for logging purposes */ struct rspamd_mempool_tag { @@ -116,18 +102,18 @@ struct rspamd_mempool_tag { gchar uid[MEMPOOL_UID_LEN]; /**< unique id */ }; +enum rspamd_mempool_flags { + RSPAMD_MEMPOOL_DEBUG = (1u << 0u), +}; + /** * Memory pool type */ struct rspamd_mempool_entry_point; struct rspamd_mutex_s; +struct rspamd_mempool_specific; typedef struct memory_pool_s { - struct _pool_chain *pools[RSPAMD_MEMPOOL_MAX]; - GArray *destructors; - GPtrArray *trash_stack; - GHashTable *variables; /**< private memory pool variables */ - gsize elt_len; /**< size of an element */ - struct rspamd_mempool_entry_point *entry; + struct rspamd_mempool_specific *priv; struct rspamd_mempool_tag tag; /**< memory pool tag */ } rspamd_mempool_t; @@ -151,9 +137,11 @@ typedef struct memory_pool_stat_s { * @param size size of pool's page * @return new memory pool object */ -rspamd_mempool_t *rspamd_mempool_new_ (gsize size, const gchar *tag, const gchar *loc); +rspamd_mempool_t *rspamd_mempool_new_ (gsize size, const gchar *tag, gint flags, + const gchar *loc); -#define rspamd_mempool_new(size, tag) rspamd_mempool_new_((size), (tag), G_STRLOC) +#define rspamd_mempool_new(size, tag, flags) \ + rspamd_mempool_new_((size), (tag), (flags), G_STRLOC) /** * Get memory from pool @@ -161,16 +149,20 @@ rspamd_mempool_t *rspamd_mempool_new_ (gsize size, const gchar *tag, const gchar * @param size bytes to allocate * @return pointer to allocated object */ -void *rspamd_mempool_alloc (rspamd_mempool_t *pool, gsize size) +void *rspamd_mempool_alloc_ (rspamd_mempool_t *pool, gsize size, const gchar *loc) RSPAMD_ATTR_ALLOC_SIZE(2) RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT) RSPAMD_ATTR_RETURNS_NONNUL; +#define rspamd_mempool_alloc(pool, size) \ + rspamd_mempool_alloc_((pool), (size), G_STRLOC) /** - * Get memory from temporary pool - * @param pool memory pool object - * @param size bytes to allocate - * @return pointer to allocated object + * Notify external memory usage for memory pool + * @param pool + * @param size + * @param loc */ -void *rspamd_mempool_alloc_tmp (rspamd_mempool_t *pool, gsize size) RSPAMD_ATTR_RETURNS_NONNUL; +void rspamd_mempool_notify_alloc_ (rspamd_mempool_t *pool, gsize size, const gchar *loc); +#define rspamd_mempool_notify_alloc(pool, size) \ + rspamd_mempool_notify_alloc_((pool), (size), G_STRLOC) /** * Get memory and set it to zero @@ -178,21 +170,10 @@ void *rspamd_mempool_alloc_tmp (rspamd_mempool_t *pool, gsize size) RSPAMD_ATTR_ * @param size bytes to allocate * @return pointer to allocated object */ -void *rspamd_mempool_alloc0 (rspamd_mempool_t *pool, gsize size) +void *rspamd_mempool_alloc0_ (rspamd_mempool_t *pool, gsize size, const gchar *loc) RSPAMD_ATTR_ALLOC_SIZE(2) RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT) RSPAMD_ATTR_RETURNS_NONNUL; - -/** - * Get memory and set it to zero - * @param pool memory pool object - * @param size bytes to allocate - * @return pointer to allocated object - */ -void *rspamd_mempool_alloc0_tmp (rspamd_mempool_t *pool, gsize size) RSPAMD_ATTR_RETURNS_NONNUL; - -/** - * Cleanup temporary data in pool - */ -void rspamd_mempool_cleanup_tmp (rspamd_mempool_t *pool); +#define rspamd_mempool_alloc0(pool, size) \ + rspamd_mempool_alloc0_((pool), (size), G_STRLOC) /** * Make a copy of string in pool @@ -200,8 +181,10 @@ void rspamd_mempool_cleanup_tmp (rspamd_mempool_t *pool); * @param src source string * @return pointer to newly created string that is copy of src */ -gchar *rspamd_mempool_strdup (rspamd_mempool_t *pool, const gchar *src) +gchar *rspamd_mempool_strdup_ (rspamd_mempool_t *pool, const gchar *src, const gchar *loc) RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT); +#define rspamd_mempool_strdup(pool, src) \ + rspamd_mempool_strdup_ ((pool), (src), G_STRLOC) /** * Make a copy of fixed string in pool as null terminated string @@ -209,8 +192,12 @@ RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT); * @param src source string * @return pointer to newly created string that is copy of src */ -gchar *rspamd_mempool_fstrdup (rspamd_mempool_t *pool, - const struct f_str_s *src) RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT); +gchar *rspamd_mempool_fstrdup_ (rspamd_mempool_t *pool, + const struct f_str_s *src, + const gchar *loc) +RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT); +#define rspamd_mempool_fstrdup(pool, src) \ + rspamd_mempool_fstrdup_ ((pool), (src), G_STRLOC) struct f_str_tok; @@ -220,19 +207,27 @@ struct f_str_tok; * @param src source string * @return pointer to newly created string that is copy of src */ -gchar *rspamd_mempool_ftokdup (rspamd_mempool_t *pool, - const struct f_str_tok *src) RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT); +gchar *rspamd_mempool_ftokdup_ (rspamd_mempool_t *pool, + const struct f_str_tok *src, + const gchar *loc) +RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT); +#define rspamd_mempool_ftokdup(pool, src) \ + rspamd_mempool_ftokdup_ ((pool), (src), G_STRLOC) /** * Allocate piece of shared memory * @param pool memory pool object * @param size bytes to allocate */ -void *rspamd_mempool_alloc_shared (rspamd_mempool_t *pool, gsize size) +void *rspamd_mempool_alloc_shared_ (rspamd_mempool_t *pool, gsize size, const gchar *loc) RSPAMD_ATTR_ALLOC_SIZE(2) RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT) RSPAMD_ATTR_RETURNS_NONNUL; +#define rspamd_mempool_alloc_shared(pool, size) \ + rspamd_mempool_alloc_shared_((pool), (size), G_STRLOC) -void *rspamd_mempool_alloc0_shared (rspamd_mempool_t *pool, gsize size) +void *rspamd_mempool_alloc0_shared_ (rspamd_mempool_t *pool, gsize size, const gchar *loc) RSPAMD_ATTR_ALLOC_SIZE(2) RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT) RSPAMD_ATTR_RETURNS_NONNUL; +#define rspamd_mempool_alloc0_shared(pool, size) \ + rspamd_mempool_alloc0_shared_((pool), (size), G_STRLOC) /** * Add destructor callback to pool @@ -342,6 +337,9 @@ void rspamd_mempool_stat_reset (void); gsize rspamd_mempool_suggest_size_ (const char *loc); +gsize rspamd_mempool_get_used_size (rspamd_mempool_t *pool); +gsize rspamd_mempool_get_wasted_size (rspamd_mempool_t *pool); + /** * Set memory pool variable * @param pool memory pool object @@ -349,8 +347,10 @@ gsize rspamd_mempool_suggest_size_ (const char *loc); * @param gpointer value value of variable * @param destructor pointer to function-destructor */ -void rspamd_mempool_set_variable (rspamd_mempool_t *pool, const gchar *name, - gpointer value, rspamd_mempool_destruct_t destructor); +void rspamd_mempool_set_variable (rspamd_mempool_t *pool, + const gchar *name, + gpointer value, + rspamd_mempool_destruct_t destructor); /** * Get memory pool variable diff --git a/src/libutil/mem_pool_internal.h b/src/libutil/mem_pool_internal.h new file mode 100644 index 000000000..1f253e790 --- /dev/null +++ b/src/libutil/mem_pool_internal.h @@ -0,0 +1,81 @@ +/*- + * Copyright 2019 Vsevolod Stakhov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RSPAMD_MEM_POOL_INTERNAL_H +#define RSPAMD_MEM_POOL_INTERNAL_H + +/* + * Internal memory pool stuff + */ + +#define align_ptr(p, a) \ + ((guint8 *) ((uintptr_t) (p) + ((-(intptr_t)(p)) & ((a) - 1)))) + +enum rspamd_mempool_chain_type { + RSPAMD_MEMPOOL_NORMAL = 0, + RSPAMD_MEMPOOL_SHARED, + RSPAMD_MEMPOOL_MAX +}; +#define ENTRY_LEN 128 +#define ENTRY_NELTS 64 + +struct entry_elt { + guint32 fragmentation; + guint32 leftover; +}; + +struct rspamd_mempool_entry_point { + gchar src[ENTRY_LEN]; + guint32 cur_suggestion; + guint32 cur_elts; + struct entry_elt elts[ENTRY_NELTS]; +}; + +/** + * Destructors list item structure + */ +struct _pool_destructors { + rspamd_mempool_destruct_t func; /**< pointer to destructor */ + void *data; /**< data to free */ + const gchar *function; /**< function from which this destructor was added */ + const gchar *loc; /**< line number */ + struct _pool_destructors *next; +}; + +struct rspamd_mempool_specific { + struct _pool_chain *pools[RSPAMD_MEMPOOL_MAX]; + struct _pool_destructors *dtors_head, *dtors_tail; + GPtrArray *trash_stack; + GHashTable *variables; /**< private memory pool variables */ + struct rspamd_mempool_entry_point *entry; + gsize elt_len; /**< size of an element */ + gsize used_memory; + guint wasted_memory; + gint flags; +}; + +/** + * Pool page structure + */ +struct _pool_chain { + guint8 *begin; /**< begin of pool chain block */ + guint8 *pos; /**< current start of free space in block */ + gsize slice_size; /**< length of block */ + struct _pool_chain *next; +}; + + +#endif diff --git a/src/libutil/radix.c b/src/libutil/radix.c index 3ea6c6b8c..8eb05cd1f 100644 --- a/src/libutil/radix.c +++ b/src/libutil/radix.c @@ -130,7 +130,7 @@ radix_create_compressed (void) return NULL; } - tree->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL); + tree->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL, 0); tree->size = 0; tree->duplicates = 0; tree->tree = btrie_init (tree->pool); diff --git a/src/libutil/regexp.c b/src/libutil/regexp.c index 9bef43501..f36bd04f9 100644 --- a/src/libutil/regexp.c +++ b/src/libutil/regexp.c @@ -199,7 +199,7 @@ rspamd_regexp_post_process (rspamd_regexp_t *r) pcre2_jit_stack_assign (r->mcontext, NULL, global_re_cache->jstack); } - if (r->re != r->raw_re && !(r->flags & RSPAMD_REGEXP_FLAG_DISABLE_JIT)) { + if (r->raw_re && r->re != r->raw_re && !(r->flags & RSPAMD_REGEXP_FLAG_DISABLE_JIT)) { if (pcre2_jit_compile (r->raw_re, jit_flags) < 0) { msg_debug ("jit compilation of %s is not supported", r->pattern); r->flags |= RSPAMD_REGEXP_FLAG_DISABLE_JIT; @@ -209,6 +209,7 @@ rspamd_regexp_post_process (rspamd_regexp_t *r) msg_debug ("jit compilation of raw %s is not supported", r->pattern); } else if (!(r->flags & RSPAMD_REGEXP_FLAG_DISABLE_JIT)) { + g_assert (r->raw_mcontext != NULL); pcre2_jit_stack_assign (r->raw_mcontext, NULL, global_re_cache->jstack); } } diff --git a/src/libutil/str_util.c b/src/libutil/str_util.c index 866ef52d8..8fcaca484 100644 --- a/src/libutil/str_util.c +++ b/src/libutil/str_util.c @@ -27,6 +27,8 @@ #endif #include <math.h> +#include "contrib/fastutf8/fastutf8.h" + const guchar lc_map[256] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, @@ -2932,8 +2934,8 @@ rspamd_str_regexp_escape (const gchar *pattern, gsize slen, } if (flags & RSPAMD_REGEXP_ESCAPE_UTF) { - if (!g_utf8_validate (pattern, slen, NULL)) { - tmp_utf = rspamd_str_make_utf_valid (pattern, slen, NULL); + if (rspamd_fast_utf8_validate (pattern, slen) != 0) { + tmp_utf = rspamd_str_make_utf_valid (pattern, slen, NULL, NULL); } } @@ -3050,61 +3052,117 @@ rspamd_str_regexp_escape (const gchar *pattern, gsize slen, gchar * -rspamd_str_make_utf_valid (const guchar *src, gsize slen, gsize *dstlen) +rspamd_str_make_utf_valid (const guchar *src, gsize slen, + gsize *dstlen, + rspamd_mempool_t *pool) { - GString *dst; - const gchar *last; - gchar *dchar; - gsize valid, prev; UChar32 uc; - gint32 i; + goffset err_offset; + const guchar *p; + gchar *dst, *d; + gsize remain = slen, dlen = 0; if (src == NULL) { return NULL; } if (slen == 0) { - slen = strlen (src); + if (dstlen) { + *dstlen = 0; + } + + return pool ? rspamd_mempool_strdup (pool, "") : g_strdup (""); } - dst = g_string_sized_new (slen); - i = 0; - last = src; - valid = 0; - prev = 0; + p = src; + dlen = slen + 1; /* As we add '\0' */ + + /* Check space required */ + while (remain > 0 && (err_offset = rspamd_fast_utf8_validate (p, remain)) > 0) { + gint i = 0; + + err_offset --; /* As it returns it 1 indexed */ + p += err_offset; + remain -= err_offset; + dlen += err_offset; - while (i < slen) { - U8_NEXT (src, i, slen, uc); + /* Each invalid character of input requires 3 bytes of output (+2 bytes) */ + while (i < remain) { + U8_NEXT (p, i, remain, uc); - if (uc <= 0) { - if (valid > 0) { - g_string_append_len (dst, last, valid); + if (uc < 0) { + dlen += 2; + } + else { + break; } - /* 0xFFFD in UTF8 */ - g_string_append_len (dst, "\357\277\275", 3); - valid = 0; - last = &src[i]; - } - else { - valid += i - prev; } - prev = i; + p += i; + remain -= i; + } + + if (pool) { + dst = rspamd_mempool_alloc (pool, dlen + 1); + } + else { + dst = g_malloc (dlen + 1); + } + + p = src; + d = dst; + remain = slen; + + while (remain > 0 && (err_offset = rspamd_fast_utf8_validate (p, remain)) > 0) { + /* Copy valid */ + err_offset --; /* As it returns it 1 indexed */ + memcpy (d, p, err_offset); + d += err_offset; + + /* Append 0xFFFD for each bad character */ + gint i = 0; + + p += err_offset; + remain -= err_offset; + + while (i < remain) { + gint old_i = i; + U8_NEXT (p, i, remain, uc); + + if (uc < 0) { + *d++ = '\357'; + *d++ = '\277'; + *d++ = '\275'; + } + else { + /* Adjust p and remaining stuff and go to the outer cycle */ + i = old_i; + break; + } + } + /* + * Now p is the first valid utf8 character and remain is the rest of the string + * so we can continue our loop + */ + p += i; + remain -= i; } - if (valid > 0) { - g_string_append_len (dst, last, valid); + if (err_offset == 0 && remain > 0) { + /* Last piece */ + memcpy (d, p, remain); + d += remain; } - dchar = dst->str; + /* Last '\0' */ + g_assert (dlen > d - dst); + *d = '\0'; if (dstlen) { - *dstlen = dst->len; + *dstlen = d - dst; } - g_string_free (dst, FALSE); - - return dchar; + return dst; } gsize diff --git a/src/libutil/str_util.h b/src/libutil/str_util.h index 7891a8e54..c08dd55bb 100644 --- a/src/libutil/str_util.h +++ b/src/libutil/str_util.h @@ -527,7 +527,7 @@ rspamd_str_regexp_escape (const gchar *pattern, gsize slen, * @param dstelen * @return */ -gchar *rspamd_str_make_utf_valid (const guchar *src, gsize slen, gsize *dstlen); +gchar *rspamd_str_make_utf_valid (const guchar *src, gsize slen, gsize *dstlen, rspamd_mempool_t *pool); /** * Strips characters in `strip_chars` from start and end of the GString @@ -561,7 +561,8 @@ gchar ** rspamd_string_len_split (const gchar *in, gsize len, #define IS_ZERO_WIDTH_SPACE(uc) ((uc) == 0x200B || \ (uc) == 0x200C || \ (uc) == 0x200D || \ - (uc) == 0xFEFF) + (uc) == 0xFEFF || \ + (uc) == 0x00AD) #define IS_OBSCURED_CHAR(uc) (((uc) >= 0x200B && (uc) <= 0x200F) || \ ((uc) >= 0x2028 && (uc) <= 0x202F) || \ ((uc) >= 0x205F && (uc) <= 0x206F) || \ diff --git a/src/libutil/upstream.c b/src/libutil/upstream.c index 227ea4442..196d9cde8 100644 --- a/src/libutil/upstream.c +++ b/src/libutil/upstream.c @@ -258,7 +258,7 @@ rspamd_upstreams_library_init (void) ctx = g_malloc0 (sizeof (*ctx)); memcpy (&ctx->limits, &default_limits, sizeof (ctx->limits)); ctx->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), - "upstreams"); + "upstreams", 0); ctx->upstreams = g_queue_new (); REF_INIT_RETAIN (ctx, rspamd_upstream_ctx_dtor); @@ -735,13 +735,19 @@ rspamd_upstream_set_inactive (struct upstream_list *ls, struct upstream *upstrea } void -rspamd_upstream_fail (struct upstream *upstream, gboolean addr_failure) +rspamd_upstream_fail (struct upstream *upstream, + gboolean addr_failure, + const gchar *reason) { gdouble error_rate = 0, max_error_rate = 0; gdouble sec_last, sec_cur; struct upstream_addr_elt *addr_elt; struct upstream_list_watcher *w; + msg_debug_upstream ("upstream %s failed; reason: %s", + upstream->name, + reason); + if (upstream->ctx && upstream->active_idx != -1) { sec_cur = rspamd_get_ticks (FALSE); @@ -780,27 +786,38 @@ rspamd_upstream_fail (struct upstream *upstream, gboolean addr_failure) if (error_rate > max_error_rate) { /* Remove upstream from the active list */ if (upstream->ls->ups->len > 1) { - msg_debug_upstream ("mark upstream %s inactive: %.2f " + msg_debug_upstream ("mark upstream %s inactive; " + "reason: %s; %.2f " "error rate (%d errors), " "%.2f max error rate, " "%.1f first error time, " "%.1f current ts, " "%d upstreams left", - upstream->name, error_rate, upstream->errors, - max_error_rate, sec_last, sec_cur, + upstream->name, + reason, + error_rate, + upstream->errors, + max_error_rate, + sec_last, + sec_cur, upstream->ls->alive->len - 1); rspamd_upstream_set_inactive (upstream->ls, upstream); upstream->errors = 0; } else { msg_debug_upstream ("cannot mark last alive upstream %s " - "inactive: %.2f " + "inactive; reason: %s; %.2f " "error rate (%d errors), " "%.2f max error rate, " "%.1f first error time, " "%.1f current ts", - upstream->name, error_rate, upstream->errors, - max_error_rate, sec_last, sec_cur); + upstream->name, + reason, + error_rate, + upstream->errors, + max_error_rate, + sec_last, + sec_cur); /* Just re-resolve addresses */ if (sec_cur - sec_last > upstream->ls->limits->revive_time) { upstream->errors = 0; @@ -819,7 +836,8 @@ rspamd_upstream_fail (struct upstream *upstream, gboolean addr_failure) if (addr_failure) { /* Also increase count of errors for this specific address */ if (upstream->addrs.addr) { - addr_elt = g_ptr_array_index (upstream->addrs.addr, upstream->addrs.cur); + addr_elt = g_ptr_array_index (upstream->addrs.addr, + upstream->addrs.cur); addr_elt->errors++; } } @@ -829,29 +847,30 @@ rspamd_upstream_fail (struct upstream *upstream, gboolean addr_failure) } void -rspamd_upstream_ok (struct upstream *up) +rspamd_upstream_ok (struct upstream *upstream) { struct upstream_addr_elt *addr_elt; struct upstream_list_watcher *w; - RSPAMD_UPSTREAM_LOCK (up); - if (up->errors > 0 && up->active_idx != -1) { + RSPAMD_UPSTREAM_LOCK (upstream); + if (upstream->errors > 0 && upstream->active_idx != -1) { /* We touch upstream if and only if it is active */ - up->errors = 0; + msg_debug_upstream ("reset errors on upstream %s (was %ud)", upstream->name, upstream->errors); + upstream->errors = 0; - if (up->addrs.addr) { - addr_elt = g_ptr_array_index (up->addrs.addr, up->addrs.cur); + if (upstream->addrs.addr) { + addr_elt = g_ptr_array_index (upstream->addrs.addr, upstream->addrs.cur); addr_elt->errors = 0; } - DL_FOREACH (up->ls->watchers, w) { + DL_FOREACH (upstream->ls->watchers, w) { if (w->events_mask & RSPAMD_UPSTREAM_WATCH_SUCCESS) { - w->func (up, RSPAMD_UPSTREAM_WATCH_SUCCESS, 0, w->ud); + w->func (upstream, RSPAMD_UPSTREAM_WATCH_SUCCESS, 0, w->ud); } } } - RSPAMD_UPSTREAM_UNLOCK (up); + RSPAMD_UPSTREAM_UNLOCK (upstream); } void @@ -1309,18 +1328,30 @@ rspamd_upstream_restore_cb (gpointer elt, gpointer ls) } static struct upstream* -rspamd_upstream_get_random (struct upstream_list *ups) +rspamd_upstream_get_random (struct upstream_list *ups, + struct upstream *except) { - guint idx = ottery_rand_range (ups->alive->len - 1); + for (;;) { + guint idx = ottery_rand_range (ups->alive->len - 1); + struct upstream *up; + + up = g_ptr_array_index (ups->alive, idx); - return g_ptr_array_index (ups->alive, idx); + if (except && up == except) { + continue; + } + + return up; + } } static struct upstream* -rspamd_upstream_get_round_robin (struct upstream_list *ups, gboolean use_cur) +rspamd_upstream_get_round_robin (struct upstream_list *ups, + struct upstream *except, + gboolean use_cur) { guint max_weight = 0, min_checked = G_MAXUINT; - struct upstream *up, *selected = NULL, *min_checked_sel = NULL; + struct upstream *up = NULL, *selected = NULL, *min_checked_sel = NULL; guint i; /* Select upstream with the maximum cur_weight */ @@ -1328,6 +1359,11 @@ rspamd_upstream_get_round_robin (struct upstream_list *ups, gboolean use_cur) for (i = 0; i < ups->alive->len; i ++) { up = g_ptr_array_index (ups->alive, i); + + if (except != NULL && up == except) { + continue; + } + if (use_cur) { if (up->cur_weight > max_weight) { selected = up; @@ -1395,18 +1431,15 @@ rspamd_consistent_hash (guint64 key, guint32 nbuckets) } static struct upstream* -rspamd_upstream_get_hashed (struct upstream_list *ups, const guint8 *key, guint keylen) +rspamd_upstream_get_hashed (struct upstream_list *ups, + struct upstream *except, + const guint8 *key, guint keylen) { guint64 k; guint32 idx; static const guint max_tries = 20; struct upstream *up = NULL; - if (ups->alive->len == 1) { - /* Fast path */ - return g_ptr_array_index (ups->alive, 0); - } - /* Generate 64 bits input key */ k = rspamd_cryptobox_fast_hash_specific (RSPAMD_CRYPTOBOX_XXHASH64, key, keylen, ups->hash_seed); @@ -1419,8 +1452,8 @@ rspamd_upstream_get_hashed (struct upstream_list *ups, const guint8 *key, guint idx = rspamd_consistent_hash (k, ups->ups->len); up = g_ptr_array_index (ups->ups, idx); - if (up->active_idx < 0) { - /* Found inactive upstream */ + if (up->active_idx < 0 || (except != NULL && up == except)) { + /* Found inactive or excluded upstream */ k = mum_hash_step (k, ups->hash_seed); } else { @@ -1434,7 +1467,7 @@ rspamd_upstream_get_hashed (struct upstream_list *ups, const guint8 *key, guint } /* We failed to find any active upstream */ - up = rspamd_upstream_get_random (ups); + up = rspamd_upstream_get_random (ups, except); msg_info ("failed to find hashed upstream for %s, fallback to random: %s", ups->ups_line, up->name); @@ -1443,8 +1476,10 @@ rspamd_upstream_get_hashed (struct upstream_list *ups, const guint8 *key, guint static struct upstream* rspamd_upstream_get_common (struct upstream_list *ups, - enum rspamd_upstream_rotation default_type, - const guchar *key, gsize keylen, gboolean forced) + struct upstream* except, + enum rspamd_upstream_rotation default_type, + const guchar *key, gsize keylen, + gboolean forced) { enum rspamd_upstream_rotation type; struct upstream *up = NULL; @@ -1458,6 +1493,12 @@ rspamd_upstream_get_common (struct upstream_list *ups, } RSPAMD_UPSTREAM_UNLOCK (ups); + if (ups->alive->len == 1 && default_type != RSPAMD_UPSTREAM_SEQUENTIAL) { + /* Fast path */ + up = g_ptr_array_index (ups->alive, 0); + goto end; + } + if (!forced) { type = ups->rot_alg != RSPAMD_UPSTREAM_UNDEF ? ups->rot_alg : default_type; } @@ -1473,16 +1514,16 @@ rspamd_upstream_get_common (struct upstream_list *ups, switch (type) { default: case RSPAMD_UPSTREAM_RANDOM: - up = rspamd_upstream_get_random (ups); + up = rspamd_upstream_get_random (ups, except); break; case RSPAMD_UPSTREAM_HASHED: - up = rspamd_upstream_get_hashed (ups, key, keylen); + up = rspamd_upstream_get_hashed (ups, except, key, keylen); break; case RSPAMD_UPSTREAM_ROUND_ROBIN: - up = rspamd_upstream_get_round_robin (ups, TRUE); + up = rspamd_upstream_get_round_robin (ups, except, TRUE); break; case RSPAMD_UPSTREAM_MASTER_SLAVE: - up = rspamd_upstream_get_round_robin (ups, FALSE); + up = rspamd_upstream_get_round_robin (ups, except, FALSE); break; case RSPAMD_UPSTREAM_SEQUENTIAL: if (ups->cur_elt >= ups->alive->len) { @@ -1494,6 +1535,7 @@ rspamd_upstream_get_common (struct upstream_list *ups, break; } +end: if (up) { up->checked ++; } @@ -1506,7 +1548,7 @@ rspamd_upstream_get (struct upstream_list *ups, enum rspamd_upstream_rotation default_type, const guchar *key, gsize keylen) { - return rspamd_upstream_get_common (ups, default_type, key, keylen, FALSE); + return rspamd_upstream_get_common (ups, NULL, default_type, key, keylen, FALSE); } struct upstream* @@ -1514,7 +1556,15 @@ rspamd_upstream_get_forced (struct upstream_list *ups, enum rspamd_upstream_rotation forced_type, const guchar *key, gsize keylen) { - return rspamd_upstream_get_common (ups, forced_type, key, keylen, TRUE); + return rspamd_upstream_get_common (ups, NULL, forced_type, key, keylen, TRUE); +} + +struct upstream *rspamd_upstream_get_except (struct upstream_list *ups, + struct upstream *except, + enum rspamd_upstream_rotation default_type, + const guchar *key, gsize keylen) +{ + return rspamd_upstream_get_common (ups, except, default_type, key, keylen, FALSE); } void diff --git a/src/libutil/upstream.h b/src/libutil/upstream.h index 4dd69e0dd..89bcd9b52 100644 --- a/src/libutil/upstream.h +++ b/src/libutil/upstream.h @@ -60,7 +60,7 @@ void rspamd_upstreams_library_config (struct rspamd_config *cfg, /** * Add an error to an upstream */ -void rspamd_upstream_fail (struct upstream *upstream, gboolean addr_failure); +void rspamd_upstream_fail (struct upstream *upstream, gboolean addr_failure, const gchar *reason); /** * Increase upstream successes count @@ -284,6 +284,17 @@ struct upstream *rspamd_upstream_get_forced (struct upstream_list *ups, const guchar *key, gsize keylen); /** + * Get new upstream from the list excepting the upstream specified + * @param ups upstream list + * @param type type of rotation algorithm, for `RSPAMD_UPSTREAM_HASHED` it is required to specify `key` and `keylen` as arguments + * @return + */ +struct upstream *rspamd_upstream_get_except (struct upstream_list *ups, + struct upstream *except, + enum rspamd_upstream_rotation default_type, + const guchar *key, gsize keylen); + +/** * Re-resolve addresses for all upstreams registered */ void rspamd_upstream_reresolve (struct upstream_ctx *ctx); diff --git a/src/libutil/util.c b/src/libutil/util.c index 5ada3a27e..9c788587a 100644 --- a/src/libutil/util.c +++ b/src/libutil/util.c @@ -86,6 +86,7 @@ #include "cryptobox.h" #include "zlib.h" #include "contrib/uthash/utlist.h" +#include "contrib/fastutf8/fastutf8.h" /* Check log messages intensity once per minute */ #define CHECK_TIME 60 @@ -2334,6 +2335,18 @@ rspamd_init_libs (void) #endif } + /* Configure utf8 library */ + guint utf8_flags = 0; + + if ((ctx->crypto_ctx->cpu_config & CPUID_SSE41)) { + utf8_flags |= RSPAMD_FAST_UTF8_FLAG_SSE41; + } + if ((ctx->crypto_ctx->cpu_config & CPUID_AVX2)) { + utf8_flags |= RSPAMD_FAST_UTF8_FLAG_AVX2; + } + + rspamd_fast_utf8_library_init (utf8_flags); + g_assert (ottery_init (ottery_cfg) == 0); #ifdef HAVE_LOCALE_H @@ -2432,7 +2445,9 @@ rspamd_config_libs (struct rspamd_external_libs_ctx *ctx, if (cfg->local_addrs) { rspamd_config_radix_from_ucl (cfg, cfg->local_addrs, "Local addresses", - ctx->local_addrs, NULL); + ctx->local_addrs, + NULL, + NULL); } if (cfg->ssl_ca_path) { diff --git a/src/lua/CMakeLists.txt b/src/lua/CMakeLists.txt index 2730702b2..30f5008fa 100644 --- a/src/lua/CMakeLists.txt +++ b/src/lua/CMakeLists.txt @@ -30,6 +30,7 @@ SET(LUASRC ${CMAKE_CURRENT_SOURCE_DIR}/lua_common.c ${CMAKE_CURRENT_SOURCE_DIR}/lua_udp.c ${CMAKE_CURRENT_SOURCE_DIR}/lua_text.c ${CMAKE_CURRENT_SOURCE_DIR}/lua_worker.c - ${CMAKE_CURRENT_SOURCE_DIR}/lua_kann.c) + ${CMAKE_CURRENT_SOURCE_DIR}/lua_kann.c + ${CMAKE_CURRENT_SOURCE_DIR}/lua_spf.c) SET(RSPAMD_LUA ${LUASRC} PARENT_SCOPE)
\ No newline at end of file diff --git a/src/lua/lua_common.c b/src/lua/lua_common.c index 2e34c42dd..248f1dbc3 100644 --- a/src/lua/lua_common.c +++ b/src/lua/lua_common.c @@ -941,6 +941,7 @@ rspamd_lua_init (bool wipe_mem) luaopen_udp (L); luaopen_worker (L); luaopen_kann (L); + luaopen_spf (L); #ifndef WITH_LUAJIT rspamd_lua_add_preload (L, "bit", luaopen_bit); lua_settop (L, 0); @@ -1829,8 +1830,14 @@ rspamd_lua_get_traceback_string (lua_State *L, luaL_Buffer *buf) { const gchar *msg = lua_tostring (L, -1); - luaL_addstring (buf, msg); - lua_pop (L, 1); /* Error string */ + if (msg) { + luaL_addstring (buf, msg); + lua_pop (L, 1); /* Error string */ + } + else { + luaL_addstring (buf, "unknown error"); + } + luaL_addstring (buf, "; trace:"); rspamd_lua_traceback_string (L, buf); } @@ -2106,11 +2113,14 @@ gboolean rspamd_lua_require_function (lua_State *L, const gchar *modname, const gchar *funcname) { - gint table_pos; + gint table_pos, err_pos; + lua_pushcfunction (L, &rspamd_lua_traceback); + err_pos = lua_gettop (L); lua_getglobal (L, "require"); if (lua_isnil (L, -1)) { + lua_remove (L, err_pos); lua_pop (L, 1); return FALSE; @@ -2120,13 +2130,21 @@ rspamd_lua_require_function (lua_State *L, const gchar *modname, /* Now try to call */ if (lua_pcall (L, 1, 1, 0) != 0) { + lua_remove (L, err_pos); + msg_warn ("require of %s.%s failed: %s", modname, + funcname, lua_tostring (L, -1)); lua_pop (L, 1); return FALSE; } + lua_remove (L, err_pos); + /* Now we should have a table with results */ if (!lua_istable (L, -1)) { + msg_warn ("require of %s.%s failed: not a table but %s", modname, + funcname, lua_typename (L, lua_type (L, -1))); + lua_pop (L, 1); return FALSE; @@ -2142,6 +2160,10 @@ rspamd_lua_require_function (lua_State *L, const gchar *modname, return TRUE; } + else { + msg_warn ("require of %s.%s failed: not a function but %s", modname, + funcname, lua_typename (L, lua_type (L, -1))); + } lua_pop (L, 2); @@ -2324,4 +2346,35 @@ rspamd_lua_push_words (lua_State *L, GArray *words, } return 1; +} + +gchar * +rspamd_lua_get_module_name (lua_State *L) +{ + lua_Debug d; + gchar *p; + gchar func_buf[128]; + + if (lua_getstack (L, 1, &d) == 1) { + (void) lua_getinfo (L, "Sl", &d); + if ((p = strrchr (d.short_src, '/')) == NULL) { + p = d.short_src; + } + else { + p++; + } + + if (strlen (p) > 20) { + rspamd_snprintf (func_buf, sizeof (func_buf), "%10s...]:%d", p, + d.currentline); + } + else { + rspamd_snprintf (func_buf, sizeof (func_buf), "%s:%d", p, + d.currentline); + } + + return g_strdup (func_buf); + } + + return NULL; }
\ No newline at end of file diff --git a/src/lua/lua_common.h b/src/lua/lua_common.h index 9878cc521..935a7c7d7 100644 --- a/src/lua/lua_common.h +++ b/src/lua/lua_common.h @@ -231,7 +231,7 @@ struct rspamd_lua_ip *lua_check_ip (lua_State *L, gint pos); struct rspamd_lua_text *lua_check_text (lua_State *L, gint pos); /* Creates and *pushes* new rspamd text, data is copied if RSPAMD_TEXT_FLAG_OWN is in flags*/ struct rspamd_lua_text *lua_new_text (lua_State *L, const gchar *start, - gsize len, guint flags); + gsize len, gboolean own); struct rspamd_lua_regexp *lua_check_regexp (lua_State *L, gint pos); @@ -346,6 +346,8 @@ void luaopen_worker (lua_State *L); void luaopen_kann (lua_State *L); +void luaopen_spf (lua_State *L); + void rspamd_lua_dostring (const gchar *line); double rspamd_lua_normalize (struct rspamd_config *cfg, @@ -557,6 +559,13 @@ enum rspamd_lua_words_type { gint rspamd_lua_push_words (lua_State *L, GArray *words, enum rspamd_lua_words_type how); +/** + * Returns newly allocated name for caller module name + * @param L + * @return + */ +gchar *rspamd_lua_get_module_name (lua_State *L); + /* Paths defs */ #define RSPAMD_CONFDIR_INDEX "CONFDIR" #define RSPAMD_LOCAL_CONFDIR_INDEX "LOCAL_CONFDIR" diff --git a/src/lua/lua_config.c b/src/lua/lua_config.c index 33873d8ab..266dbd111 100644 --- a/src/lua/lua_config.c +++ b/src/lua/lua_config.c @@ -431,9 +431,10 @@ LUA_FUNCTION_DEF (config, add_condition); LUA_FUNCTION_DEF (config, enable_symbol); /*** - * @method rspamd_config:disable_symbol(symbol) + * @method rspamd_config:disable_symbol(symbol, [disable_parent=true]) * Disables execution for the specified symbol * @param {string} symbol symbol's name + * @param {boolean} disable_parent if true then disable parent execution in case of a virtual symbol */ LUA_FUNCTION_DEF (config, disable_symbol); @@ -454,6 +455,15 @@ LUA_FUNCTION_DEF (config, get_symbol_parent); LUA_FUNCTION_DEF (config, get_group_symbols); /*** + * @method rspamd_config:get_groups([need_private]) + * Returns list of all groups defined + * @param {boolean} need_private optional flag to include private groups + * @available 2.3+ + * @return {list|table} list of all groups + */ +LUA_FUNCTION_DEF (config, get_groups); + +/*** * @method rspamd_config:register_settings_id(name, symbols_enabled, symbols_disabled) * Register new static settings id in config * @param {string} name id name (not numeric!) @@ -578,7 +588,7 @@ rspamd_config:add_on_load(function(cfg, ev_base) rspamd_config:add_periodic(ev_base, 1.0, function(cfg, ev_base) local logger = require "rspamd_logger" logger.infox(cfg, "periodic function") - return true -- if return false, then the periodic event is removed + return true -- if return numeric, a new interval is set. if return false, then the periodic event is removed end) end) */ @@ -871,6 +881,7 @@ static const struct luaL_reg configlib_m[] = { LUA_INTERFACE_DEF (config, get_symbols_counters), {"get_symbols_scores", lua_config_get_symbols}, LUA_INTERFACE_DEF (config, get_symbols), + LUA_INTERFACE_DEF (config, get_groups), LUA_INTERFACE_DEF (config, get_symbol_callback), LUA_INTERFACE_DEF (config, set_symbol_callback), LUA_INTERFACE_DEF (config, get_symbol_stat), @@ -2318,7 +2329,7 @@ lua_config_set_metric_symbol (lua_State * L) nshots = 1; } if (strstr (flags_str, "ignore") != NULL) { - flags |= RSPAMD_SYMBOL_FLAG_IGNORE; + flags |= RSPAMD_SYMBOL_FLAG_IGNORE_METRIC; } if (strstr (flags_str, "one_param") != NULL) { flags |= RSPAMD_SYMBOL_FLAG_ONEPARAM; @@ -2888,9 +2899,14 @@ lua_config_disable_symbol (lua_State *L) LUA_TRACE_POINT; struct rspamd_config *cfg = lua_check_config (L, 1); const gchar *sym = luaL_checkstring (L, 2); + gboolean disable_parent = TRUE; if (cfg && sym) { - rspamd_symcache_disable_symbol_perm (cfg->cache, sym); + if (lua_isboolean (L, 3)) { + disable_parent = lua_toboolean (L, 3); + } + + rspamd_symcache_disable_symbol_perm (cfg->cache, sym, disable_parent); } else { return luaL_error (L, "invalid arguments"); @@ -3186,6 +3202,14 @@ lua_periodic_callback_finish (struct thread_entry *thread, int ret) lua_pop (L, 1); /* Return value */ } + + if (periodic->cfg->cur_worker) { + if (periodic->cfg->cur_worker->state != rspamd_worker_state_running) { + /* We are terminating, no more periodics */ + plan_more = FALSE; + } + } + if (plan_more) { if (periodic->need_jitter) { timeout = rspamd_time_jitter (timeout, 0.0); @@ -3368,7 +3392,7 @@ lua_metric_symbol_inserter (gpointer k, gpointer v, gpointer ud) lua_pushstring (L, "flags"); lua_createtable (L, 0, 3); - if (s->flags & RSPAMD_SYMBOL_FLAG_IGNORE) { + if (s->flags & RSPAMD_SYMBOL_FLAG_IGNORE_METRIC) { lua_pushstring (L, "ignore"); lua_pushboolean (L, true); lua_settable (L, -3); @@ -3383,6 +3407,11 @@ lua_metric_symbol_inserter (gpointer k, gpointer v, gpointer ud) lua_pushboolean (L, true); lua_settable (L, -3); } + if (s->flags & RSPAMD_SYMBOL_FLAG_DISABLED) { + lua_pushstring (L, "disabled"); + lua_pushboolean (L, true); + lua_settable (L, -3); + } if (s->cache_item) { guint sflags = rspamd_symcache_get_symbol_flags (cbd->cfg->cache, sym); @@ -3645,6 +3674,53 @@ lua_config_get_group_symbols (lua_State *L) } static gint +lua_config_get_groups (lua_State *L) +{ + LUA_TRACE_POINT; + struct rspamd_config *cfg = lua_check_config (L, 1); + gboolean need_private; + struct rspamd_symbols_group *gr; + GHashTableIter it; + gpointer k, v; + + if (cfg) { + if (lua_isboolean (L, 2)) { + need_private = lua_toboolean (L, 2); + } + else { + need_private = !(cfg->public_groups_only); + } + + lua_createtable (L, 0, g_hash_table_size (cfg->groups)); + g_hash_table_iter_init (&it, cfg->groups); + + while (g_hash_table_iter_next (&it, &k, &v)) { + gr = (struct rspamd_symbols_group *)v; + + if (need_private || (gr->flags & RSPAMD_SYMBOL_GROUP_PUBLIC)) { + lua_createtable (L, 0, 4); + + lua_pushstring (L, gr->description); + lua_setfield (L, -2, "description"); + lua_pushnumber (L, gr->max_score); + lua_setfield (L, -2, "max_score"); + lua_pushboolean (L, (gr->flags & RSPAMD_SYMBOL_GROUP_PUBLIC) != 0); + lua_setfield (L, -2, "is_public"); + /* TODO: maybe push symbols as well */ + + /* Parent table indexed by group name */ + lua_setfield (L, -2, gr->name); + } + } + } + else { + return luaL_error (L, "invalid arguments"); + } + + return 1; +} + +static gint lua_config_register_finish_script (lua_State *L) { LUA_TRACE_POINT; diff --git a/src/lua/lua_expression.c b/src/lua/lua_expression.c index 60ee8fdf7..b2addd30c 100644 --- a/src/lua/lua_expression.c +++ b/src/lua/lua_expression.c @@ -36,12 +36,12 @@ local rspamd_mempool = require "rspamd_mempool" local function parse_func(str) -- extract token till the first space character - local token = table.join('', take_while(function(s) return s ~= ' ' end, str)) + local token = table.concat(totable(take_while(function(s) return s ~= ' ' end, iter(str)))) -- Return token name return token end -local function process_func(token, task) +local function process_func(token) -- Do something using token and task end diff --git a/src/lua/lua_html.c b/src/lua/lua_html.c index 39a4a77a0..c0e07de36 100644 --- a/src/lua/lua_html.c +++ b/src/lua/lua_html.c @@ -186,12 +186,17 @@ lua_check_html (lua_State * L, gint pos) return ud ? *((struct html_content **)ud) : NULL; } -static struct html_tag * +struct lua_html_tag { + struct html_content *html; + struct html_tag *tag; +}; + +static struct lua_html_tag * lua_check_html_tag (lua_State * L, gint pos) { void *ud = rspamd_lua_check_udata (L, pos, "rspamd{html_tag}"); luaL_argcheck (L, ud != NULL, pos, "'html_tag' expected"); - return ud ? *((struct html_tag **)ud) : NULL; + return ud ? ((struct lua_html_tag *)ud) : NULL; } static gint @@ -263,7 +268,7 @@ static void lua_html_push_image (lua_State *L, struct html_image *img) { LUA_TRACE_POINT; - struct html_tag **ptag; + struct lua_html_tag *ltag; struct rspamd_url **purl; lua_newtable (L); @@ -298,8 +303,9 @@ lua_html_push_image (lua_State *L, struct html_image *img) if (img->tag) { lua_pushstring (L, "tag"); - ptag = lua_newuserdata (L, sizeof (gpointer)); - *ptag = img->tag; + ltag = lua_newuserdata (L, sizeof (struct lua_html_tag)); + ltag->tag = img->tag; + ltag->html = NULL; rspamd_lua_setclass (L, "rspamd{html_tag}", -1); lua_settable (L, -3); } @@ -440,6 +446,7 @@ lua_html_get_blocks (lua_State *L) struct lua_html_traverse_ud { lua_State *L; + struct html_content *html; gint cbref; GHashTable *tags; gboolean any; @@ -449,15 +456,17 @@ static gboolean lua_html_node_foreach_cb (GNode *n, gpointer d) { struct lua_html_traverse_ud *ud = d; - struct html_tag *tag = n->data, **ptag; + struct html_tag *tag = n->data; + struct lua_html_tag *ltag; if (tag && (ud->any || g_hash_table_lookup (ud->tags, GSIZE_TO_POINTER (mum_hash64 (tag->id, 0))))) { lua_rawgeti (ud->L, LUA_REGISTRYINDEX, ud->cbref); - ptag = lua_newuserdata (ud->L, sizeof (*ptag)); - *ptag = tag; + ltag = lua_newuserdata (ud->L, sizeof (*ltag)); + ltag->tag = tag; + ltag->html = ud->html; rspamd_lua_setclass (ud->L, "rspamd{html_tag}", -1); lua_pushinteger (ud->L, tag->content_length); @@ -489,6 +498,7 @@ lua_html_foreach_tag (lua_State *L) ud.tags = g_hash_table_new (g_direct_hash, g_direct_equal); ud.any = FALSE; + ud.html = hc; if (lua_type (L, 2) == LUA_TSTRING) { tagname = luaL_checkstring (L, 2); @@ -556,11 +566,11 @@ static gint lua_html_tag_get_type (lua_State *L) { LUA_TRACE_POINT; - struct html_tag *tag = lua_check_html_tag (L, 1); + struct lua_html_tag *ltag = lua_check_html_tag (L, 1); const gchar *tagname; - if (tag != NULL) { - tagname = rspamd_html_tag_by_id (tag->id); + if (ltag != NULL) { + tagname = rspamd_html_tag_by_id (ltag->tag->id); if (tagname) { lua_pushstring (L, tagname); @@ -580,15 +590,16 @@ static gint lua_html_tag_get_parent (lua_State *L) { LUA_TRACE_POINT; - struct html_tag *tag = lua_check_html_tag (L, 1), **ptag; + struct lua_html_tag *ltag = lua_check_html_tag (L, 1), *ptag; GNode *node; - if (tag != NULL) { - node = tag->parent; + if (ltag != NULL) { + node = ltag->tag->parent; if (node && node->data) { - ptag = lua_newuserdata (L, sizeof (gpointer)); - *ptag = node->data; + ptag = lua_newuserdata (L, sizeof (*ptag)); + ptag->tag = node->data; + ptag->html = ltag->html; rspamd_lua_setclass (L, "rspamd{html_tag}", -1); } else { @@ -606,33 +617,33 @@ static gint lua_html_tag_get_flags (lua_State *L) { LUA_TRACE_POINT; - struct html_tag *tag = lua_check_html_tag (L, 1); + struct lua_html_tag *ltag = lua_check_html_tag (L, 1); gint i = 1; - if (tag) { + if (ltag->tag) { /* Push flags */ lua_createtable (L, 4, 0); - if (tag->flags & FL_CLOSING) { + if (ltag->tag->flags & FL_CLOSING) { lua_pushstring (L, "closing"); lua_rawseti (L, -2, i++); } - if (tag->flags & FL_HREF) { + if (ltag->tag->flags & FL_HREF) { lua_pushstring (L, "href"); lua_rawseti (L, -2, i++); } - if (tag->flags & FL_CLOSED) { + if (ltag->tag->flags & FL_CLOSED) { lua_pushstring (L, "closed"); lua_rawseti (L, -2, i++); } - if (tag->flags & FL_BROKEN) { + if (ltag->tag->flags & FL_BROKEN) { lua_pushstring (L, "broken"); lua_rawseti (L, -2, i++); } - if (tag->flags & FL_XML) { + if (ltag->tag->flags & FL_XML) { lua_pushstring (L, "xml"); lua_rawseti (L, -2, i++); } - if (tag->flags & RSPAMD_HTML_FLAG_UNBALANCED) { + if (ltag->tag->flags & RSPAMD_HTML_FLAG_UNBALANCED) { lua_pushstring (L, "unbalanced"); lua_rawseti (L, -2, i++); } @@ -648,15 +659,16 @@ static gint lua_html_tag_get_content (lua_State *L) { LUA_TRACE_POINT; - struct html_tag *tag = lua_check_html_tag (L, 1); + struct lua_html_tag *ltag = lua_check_html_tag (L, 1); struct rspamd_lua_text *t; - if (tag) { - if (tag->content && tag->content_length) { + if (ltag) { + if (ltag->html && ltag->tag->content_length && + ltag->html->parsed->len >= ltag->tag->content_offset + ltag->tag->content_length) { t = lua_newuserdata (L, sizeof (*t)); rspamd_lua_setclass (L, "rspamd{text}", -1); - t->start = tag->content; - t->len = tag->content_length; + t->start = ltag->html->parsed->data + ltag->tag->content_offset; + t->len = ltag->tag->content_length; t->flags = 0; } else { @@ -674,10 +686,10 @@ static gint lua_html_tag_get_content_length (lua_State *L) { LUA_TRACE_POINT; - struct html_tag *tag = lua_check_html_tag (L, 1); + struct lua_html_tag *ltag = lua_check_html_tag (L, 1); - if (tag) { - lua_pushinteger (L, tag->content_length); + if (ltag) { + lua_pushinteger (L, ltag->tag->content_length); } else { return luaL_error (L, "invalid arguments"); @@ -690,24 +702,24 @@ static gint lua_html_tag_get_extra (lua_State *L) { LUA_TRACE_POINT; - struct html_tag *tag = lua_check_html_tag (L, 1); + struct lua_html_tag *ltag = lua_check_html_tag (L, 1); struct html_image *img; struct rspamd_url **purl; - if (tag) { - if (tag->extra) { - if (tag->flags & FL_HREF) { + if (ltag) { + if (ltag->tag->extra) { + if ((ltag->tag->flags & FL_HREF) || ltag->tag->id == Tag_BASE) { /* For A that's URL */ purl = lua_newuserdata (L, sizeof (gpointer)); - *purl = tag->extra; + *purl = ltag->tag->extra; rspamd_lua_setclass (L, "rspamd{url}", -1); } - else if (tag->id == Tag_IMG) { - img = tag->extra; + else if (ltag->tag->id == Tag_IMG) { + img = ltag->tag->extra; lua_html_push_image (L, img); } - else if (tag->flags & FL_BLOCK) { - lua_html_push_block (L, tag->extra); + else if (ltag->tag->flags & FL_BLOCK) { + lua_html_push_block (L, ltag->tag->extra); } else { /* Unknown extra ? */ diff --git a/src/lua/lua_http.c b/src/lua/lua_http.c index f7dd01e87..6ad5e6d21 100644 --- a/src/lua/lua_http.c +++ b/src/lua/lua_http.c @@ -467,6 +467,11 @@ static void lua_http_dns_handler (struct rdns_reply *reply, gpointer ud) { struct lua_http_cbdata *cbd = (struct lua_http_cbdata *)ud; + struct rspamd_symcache_item *item; + struct rspamd_task *task; + + task = cbd->task; + item = cbd->item; if (reply->code != RDNS_RC_NOERROR) { lua_http_push_error (cbd, "unable to resolve host"); @@ -497,8 +502,8 @@ lua_http_dns_handler (struct rdns_reply *reply, gpointer ud) REF_RELEASE (cbd); } - if (cbd->item) { - rspamd_symcache_item_async_dec_check (cbd->task, cbd->item, M); + if (item) { + rspamd_symcache_item_async_dec_check (task, item, M); } } diff --git a/src/lua/lua_ip.c b/src/lua/lua_ip.c index f873c4515..fb6845519 100644 --- a/src/lua/lua_ip.c +++ b/src/lua/lua_ip.c @@ -187,6 +187,8 @@ static const struct luaL_reg iplib_m[] = { static const struct luaL_reg iplib_f[] = { LUA_INTERFACE_DEF (ip, from_string), + {"fromstring", lua_ip_from_string}, + {"fromip", lua_ip_copy}, {"from_ip", lua_ip_copy}, {NULL, NULL} }; diff --git a/src/lua/lua_map.c b/src/lua/lua_map.c index bead7ae4a..13674e6b1 100644 --- a/src/lua/lua_map.c +++ b/src/lua/lua_map.c @@ -161,7 +161,8 @@ lua_config_add_radix_map (lua_State *L) rspamd_radix_read, rspamd_radix_fin, rspamd_radix_dtor, - (void **)&map->data.radix)) == NULL) { + (void **)&map->data.radix, + NULL)) == NULL) { msg_warn_config ("invalid radix map %s", map_line); lua_pushnil (L); @@ -218,7 +219,8 @@ lua_config_radix_from_config (lua_State *L) rspamd_radix_read, rspamd_radix_fin, rspamd_radix_dtor, - (void **)&map->data.radix)) == NULL) { + (void **)&map->data.radix, + NULL)) == NULL) { msg_err_config ("invalid radix map static"); lua_pushnil (L); ucl_object_unref (fake_obj); @@ -279,7 +281,8 @@ lua_config_radix_from_ucl (lua_State *L) rspamd_radix_read, rspamd_radix_fin, rspamd_radix_dtor, - (void **)&map->data.radix)) == NULL) { + (void **)&map->data.radix, + NULL)) == NULL) { msg_err_config ("invalid radix map static"); lua_pushnil (L); ucl_object_unref (fake_obj); @@ -324,7 +327,8 @@ lua_config_add_hash_map (lua_State *L) rspamd_kv_list_read, rspamd_kv_list_fin, rspamd_kv_list_dtor, - (void **)&map->data.hash)) == NULL) { + (void **)&map->data.hash, + NULL)) == NULL) { msg_warn_config ("invalid set map %s", map_line); lua_pushnil (L); return 1; @@ -364,7 +368,8 @@ lua_config_add_kv_map (lua_State *L) rspamd_kv_list_read, rspamd_kv_list_fin, rspamd_kv_list_dtor, - (void **)&map->data.hash)) == NULL) { + (void **)&map->data.hash, + NULL)) == NULL) { msg_warn_config ("invalid hash map %s", map_line); lua_pushnil (L); @@ -529,7 +534,8 @@ lua_config_add_map (lua_State *L) lua_map_read, lua_map_fin, lua_map_dtor, - (void **)&map->data.cbdata)) == NULL) { + (void **)&map->data.cbdata, + NULL)) == NULL) { if (cbidx != -1) { luaL_unref (L, LUA_REGISTRYINDEX, cbidx); @@ -554,7 +560,8 @@ lua_config_add_map (lua_State *L) rspamd_kv_list_read, rspamd_kv_list_fin, rspamd_kv_list_dtor, - (void **)&map->data.hash)) == NULL) { + (void **)&map->data.hash, + NULL)) == NULL) { lua_pushnil (L); ucl_object_unref (map_obj); @@ -571,7 +578,8 @@ lua_config_add_map (lua_State *L) rspamd_kv_list_read, rspamd_kv_list_fin, rspamd_kv_list_dtor, - (void **)&map->data.hash)) == NULL) { + (void **)&map->data.hash, + NULL)) == NULL) { lua_pushnil (L); ucl_object_unref (map_obj); @@ -588,7 +596,8 @@ lua_config_add_map (lua_State *L) rspamd_radix_read, rspamd_radix_fin, rspamd_radix_dtor, - (void **)&map->data.radix)) == NULL) { + (void **)&map->data.radix, + NULL)) == NULL) { lua_pushnil (L); ucl_object_unref (map_obj); @@ -605,7 +614,8 @@ lua_config_add_map (lua_State *L) rspamd_regexp_list_read_single, rspamd_regexp_list_fin, rspamd_regexp_list_dtor, - (void **) &map->data.re_map)) == NULL) { + (void **) &map->data.re_map, + NULL)) == NULL) { lua_pushnil (L); ucl_object_unref (map_obj); @@ -622,7 +632,8 @@ lua_config_add_map (lua_State *L) rspamd_regexp_list_read_multiple, rspamd_regexp_list_fin, rspamd_regexp_list_dtor, - (void **) &map->data.re_map)) == NULL) { + (void **) &map->data.re_map, + NULL)) == NULL) { lua_pushnil (L); ucl_object_unref (map_obj); @@ -639,7 +650,8 @@ lua_config_add_map (lua_State *L) rspamd_glob_list_read_single, rspamd_regexp_list_fin, rspamd_regexp_list_dtor, - (void **) &map->data.re_map)) == NULL) { + (void **) &map->data.re_map, + NULL)) == NULL) { lua_pushnil (L); ucl_object_unref (map_obj); @@ -656,7 +668,8 @@ lua_config_add_map (lua_State *L) rspamd_glob_list_read_multiple, rspamd_regexp_list_fin, rspamd_regexp_list_dtor, - (void **) &map->data.re_map)) == NULL) { + (void **) &map->data.re_map, + NULL)) == NULL) { lua_pushnil (L); ucl_object_unref (map_obj); diff --git a/src/lua/lua_mempool.c b/src/lua/lua_mempool.c index 06dcd2d5c..c376ee701 100644 --- a/src/lua/lua_mempool.c +++ b/src/lua/lua_mempool.c @@ -154,7 +154,7 @@ lua_mempool_create (lua_State *L) { LUA_TRACE_POINT; struct memory_pool_s *mempool = rspamd_mempool_new ( - rspamd_mempool_suggest_size (), "lua"), **pmempool; + rspamd_mempool_suggest_size (), "lua", 0), **pmempool; if (mempool) { pmempool = lua_newuserdata (L, sizeof (struct memory_pool_s *)); diff --git a/src/lua/lua_mimepart.c b/src/lua/lua_mimepart.c index 01c64ae64..c221cbfa3 100644 --- a/src/lua/lua_mimepart.c +++ b/src/lua/lua_mimepart.c @@ -14,6 +14,7 @@ * limitations under the License. */ +#include <src/libmime/message.h> #include "lua_common.h" #include "libmime/message.h" #include "libmime/lang_detection.h" @@ -522,6 +523,26 @@ LUA_FUNCTION_DEF (mimepart, headers_foreach); */ LUA_FUNCTION_DEF (mimepart, get_parent); +/*** + * @method mime_part:get_specific() + * Returns specific lua content for this part + * @return {any} specific lua content + */ +LUA_FUNCTION_DEF (mimepart, get_specific); + +/*** + * @method mime_part:set_specific(<any>) + * Sets a specific content for this part + * @return {any} previous specific lua content (or nil) + */ +LUA_FUNCTION_DEF (mimepart, set_specific); + +/*** + * @method mime_part:is_specific(<any>) + * Returns true if part has specific lua content + * @return {boolean} flag + */ +LUA_FUNCTION_DEF (mimepart, is_specific); static const struct luaL_reg mimepartlib_m[] = { LUA_INTERFACE_DEF (mimepart, get_content), @@ -555,6 +576,9 @@ static const struct luaL_reg mimepartlib_m[] = { LUA_INTERFACE_DEF (mimepart, get_digest), LUA_INTERFACE_DEF (mimepart, get_id), LUA_INTERFACE_DEF (mimepart, headers_foreach), + LUA_INTERFACE_DEF (mimepart, get_specific), + LUA_INTERFACE_DEF (mimepart, set_specific), + LUA_INTERFACE_DEF (mimepart, is_specific), {"__tostring", rspamd_lua_class_tostring}, {NULL, NULL} }; @@ -1564,14 +1588,14 @@ lua_mimepart_get_boundary (lua_State * L) return luaL_error (L, "invalid arguments"); } - if (IS_CT_MULTIPART (part->ct)) { + if (IS_PART_MULTIPART (part)) { lua_pushlstring (L, part->specific.mp->boundary.begin, part->specific.mp->boundary.len); } else { parent = part->parent_part; - if (!parent || !IS_CT_MULTIPART (parent->ct)) { + if (!parent || !IS_PART_MULTIPART (parent)) { lua_pushnil (L); } else { @@ -1670,7 +1694,7 @@ lua_mimepart_is_image (lua_State * L) return luaL_error (L, "invalid arguments"); } - lua_pushboolean (L, (part->flags & RSPAMD_MIME_PART_IMAGE) ? true : false); + lua_pushboolean (L, part->part_type == RSPAMD_MIME_PART_IMAGE); return 1; } @@ -1685,7 +1709,7 @@ lua_mimepart_is_archive (lua_State * L) return luaL_error (L, "invalid arguments"); } - lua_pushboolean (L, (part->flags & RSPAMD_MIME_PART_ARCHIVE) ? true : false); + lua_pushboolean (L, part->part_type == RSPAMD_MIME_PART_ARCHIVE); return 1; } @@ -1700,7 +1724,7 @@ lua_mimepart_is_multipart (lua_State * L) return luaL_error (L, "invalid arguments"); } - lua_pushboolean (L, IS_CT_MULTIPART (part->ct) ? true : false); + lua_pushboolean (L, IS_PART_MULTIPART (part) ? true : false); return 1; } @@ -1715,7 +1739,7 @@ lua_mimepart_is_message (lua_State * L) return luaL_error (L, "invalid arguments"); } - lua_pushboolean (L, IS_CT_MESSAGE (part->ct) ? true : false); + lua_pushboolean (L, IS_PART_MESSAGE (part) ? true : false); return 1; } @@ -1730,7 +1754,7 @@ lua_mimepart_is_attachment (lua_State * L) return luaL_error (L, "invalid arguments"); } - if (!(part->flags & (RSPAMD_MIME_PART_IMAGE))) { + if (part->part_type != RSPAMD_MIME_PART_IMAGE) { if (part->cd && part->cd->type == RSPAMD_CT_ATTACHMENT) { lua_pushboolean (L, true); } @@ -1761,7 +1785,7 @@ lua_mimepart_is_text (lua_State * L) return luaL_error (L, "invalid arguments"); } - lua_pushboolean (L, (part->flags & RSPAMD_MIME_PART_TEXT) ? true : false); + lua_pushboolean (L, part->part_type == RSPAMD_MIME_PART_TEXT); return 1; } @@ -1798,7 +1822,7 @@ lua_mimepart_get_image (lua_State * L) return luaL_error (L, "invalid arguments"); } - if (!(part->flags & RSPAMD_MIME_PART_IMAGE) || part->specific.img == NULL) { + if (part->part_type != RSPAMD_MIME_PART_IMAGE || part->specific.img == NULL) { lua_pushnil (L); } else { @@ -1821,7 +1845,7 @@ lua_mimepart_get_archive (lua_State * L) return luaL_error (L, "invalid arguments"); } - if (!(part->flags & RSPAMD_MIME_PART_ARCHIVE) || part->specific.arch == NULL) { + if (part->part_type != RSPAMD_MIME_PART_ARCHIVE || part->specific.arch == NULL) { lua_pushnil (L); } else { @@ -1845,7 +1869,7 @@ lua_mimepart_get_children (lua_State * L) return luaL_error (L, "invalid arguments"); } - if (!IS_CT_MULTIPART (part->ct) || part->specific.mp->children == NULL) { + if (!IS_PART_MULTIPART (part) || part->specific.mp->children == NULL) { lua_pushnil (L); } else { @@ -1897,7 +1921,7 @@ lua_mimepart_get_text (lua_State * L) return luaL_error (L, "invalid arguments"); } - if (!(part->flags & RSPAMD_MIME_PART_TEXT) || part->specific.txt == NULL) { + if (part->part_type != RSPAMD_MIME_PART_TEXT || part->specific.txt == NULL) { lua_pushnil (L); } else { @@ -2025,6 +2049,101 @@ lua_mimepart_headers_foreach (lua_State *L) return 0; } +static gint +lua_mimepart_get_specific (lua_State * L) +{ + LUA_TRACE_POINT; + struct rspamd_mime_part *part = lua_check_mimepart (L); + + if (part == NULL) { + return luaL_error (L, "invalid arguments"); + } + + if (part->part_type != RSPAMD_MIME_PART_CUSTOM_LUA) { + lua_pushnil (L); + } + else { + lua_rawgeti (L, LUA_REGISTRYINDEX, part->specific.lua_specific.cbref); + } + + return 1; +} + +static gint +lua_mimepart_is_specific (lua_State * L) +{ + LUA_TRACE_POINT; + struct rspamd_mime_part *part = lua_check_mimepart (L); + + if (part == NULL) { + return luaL_error (L, "invalid arguments"); + } + + lua_pushboolean (L, part->part_type == RSPAMD_MIME_PART_CUSTOM_LUA); + + return 1; +} + +static gint +lua_mimepart_set_specific (lua_State * L) +{ + LUA_TRACE_POINT; + struct rspamd_mime_part *part = lua_check_mimepart (L); + + if (part == NULL || lua_isnil (L, 2)) { + return luaL_error (L, "invalid arguments"); + } + + if (part->part_type != RSPAMD_MIME_PART_UNDEFINED && + part->part_type != RSPAMD_MIME_PART_CUSTOM_LUA) { + return luaL_error (L, + "internal error: trying to set specific lua content on part of type %d", + part->part_type); + } + + if (part->part_type == RSPAMD_MIME_PART_CUSTOM_LUA) { + /* Push old specific data */ + lua_rawgeti (L, LUA_REGISTRYINDEX, part->specific.lua_specific.cbref); + luaL_unref (L, LUA_REGISTRYINDEX, part->specific.lua_specific.cbref); + } + else { + part->part_type = RSPAMD_MIME_PART_CUSTOM_LUA; + lua_pushnil (L); + } + + /* Now, we push argument on the position 2 and save its reference */ + lua_pushvalue (L, 2); + part->specific.lua_specific.cbref = luaL_ref (L, LUA_REGISTRYINDEX); + /* Now stack has just a return value as luaL_ref removes value from stack */ + + gint ltype = lua_type (L, 2); + + switch (ltype) { + case LUA_TTABLE: + part->specific.lua_specific.type = RSPAMD_LUA_PART_TABLE; + break; + case LUA_TSTRING: + part->specific.lua_specific.type = RSPAMD_LUA_PART_STRING; + break; + case LUA_TUSERDATA: + if (rspamd_lua_check_udata_maybe (L, 2, "rspamd{text}")) { + part->specific.lua_specific.type = RSPAMD_LUA_PART_TEXT; + } + else { + part->specific.lua_specific.type = RSPAMD_LUA_PART_UNKNOWN; + } + break; + case LUA_TFUNCTION: + part->specific.lua_specific.type = RSPAMD_LUA_PART_FUNCTION; + break; + default: + part->specific.lua_specific.type = RSPAMD_LUA_PART_UNKNOWN; + break; + } + + return 1; +} + void luaopen_textpart (lua_State * L) { diff --git a/src/lua/lua_regexp.c b/src/lua/lua_regexp.c index c0458a876..b782ef7f1 100644 --- a/src/lua/lua_regexp.c +++ b/src/lua/lua_regexp.c @@ -81,37 +81,6 @@ lua_check_regexp (lua_State * L, gint pos) return ud ? *((struct rspamd_lua_regexp **)ud) : NULL; } -static gchar * -rspamd_lua_get_module_name (lua_State *L) -{ - lua_Debug d; - gchar *p; - gchar func_buf[128]; - - if (lua_getstack (L, 1, &d) == 1) { - (void) lua_getinfo (L, "Sl", &d); - if ((p = strrchr (d.short_src, '/')) == NULL) { - p = d.short_src; - } - else { - p++; - } - - if (strlen (p) > 20) { - rspamd_snprintf (func_buf, sizeof (func_buf), "%10s...]:%d", p, - d.currentline); - } - else { - rspamd_snprintf (func_buf, sizeof (func_buf), "%s:%d", p, - d.currentline); - } - - return g_strdup (func_buf); - } - - return NULL; -} - /*** * @function rspamd_regexp.create(pattern[, flags]) * Creates new rspamd_regexp @@ -891,7 +860,7 @@ luaopen_regexp (lua_State * L) { if (!regexp_static_pool) { regexp_static_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), - "regexp_lua_pool"); + "regexp_lua_pool", 0); } rspamd_lua_new_class (L, "rspamd{regexp}", regexplib_m); diff --git a/src/lua/lua_spf.c b/src/lua/lua_spf.c new file mode 100644 index 000000000..22f1ad2b2 --- /dev/null +++ b/src/lua/lua_spf.c @@ -0,0 +1,614 @@ +/*- + * Copyright 2019 Vsevolod Stakhov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * @file lua_spf.c + * This module exports spf functions to Lua + */ + +#include "lua_common.h" +#include "libserver/spf.h" +#include "libutil/ref.h" + +#define SPF_RECORD_CLASS "rspamd{spf_record}" + +LUA_FUNCTION_DEF (spf, resolve); +LUA_FUNCTION_DEF (spf, config); + +LUA_FUNCTION_DEF (spf_record, check_ip); +LUA_FUNCTION_DEF (spf_record, dtor); +LUA_FUNCTION_DEF (spf_record, get_domain); +LUA_FUNCTION_DEF (spf_record, get_elts); +LUA_FUNCTION_DEF (spf_record, get_ttl); +LUA_FUNCTION_DEF (spf_record, get_timestamp); +LUA_FUNCTION_DEF (spf_record, get_digest); + +static luaL_reg rspamd_spf_f[] = { + LUA_INTERFACE_DEF (spf, resolve), + LUA_INTERFACE_DEF (spf, config), + {NULL, NULL}, +}; + +static luaL_reg rspamd_spf_record_m[] = { + LUA_INTERFACE_DEF (spf_record, check_ip), + LUA_INTERFACE_DEF (spf_record, get_domain), + LUA_INTERFACE_DEF (spf_record, get_ttl), + LUA_INTERFACE_DEF (spf_record, get_digest), + LUA_INTERFACE_DEF (spf_record, get_elts), + LUA_INTERFACE_DEF (spf_record, get_timestamp), + {"__gc", lua_spf_record_dtor}, + {NULL, NULL}, +}; + +struct rspamd_lua_spf_cbdata { + struct rspamd_task *task; + lua_State *L; + struct rspamd_symcache_item *item; + gint cbref; + ref_entry_t ref; +}; + +static gint +lua_load_spf (lua_State * L) +{ + lua_newtable (L); + + /* Create integer arguments to check SPF results */ + lua_newtable (L); + lua_pushinteger (L, SPF_FAIL); + lua_setfield (L, -2, "fail"); + lua_pushinteger (L, SPF_PASS); + lua_setfield (L, -2, "pass"); + lua_pushinteger (L, SPF_NEUTRAL); + lua_setfield (L, -2, "neutral"); + lua_pushinteger (L, SPF_SOFT_FAIL); + lua_setfield (L, -2, "soft_fail"); + + lua_setfield (L, -2, "policy"); + + /* Flags stuff */ + lua_newtable (L); + + lua_pushinteger (L, RSPAMD_SPF_RESOLVED_TEMP_FAILED); + lua_setfield (L, -2, "temp_fail"); + lua_pushinteger (L, RSPAMD_SPF_RESOLVED_NA); + lua_setfield (L, -2, "na"); + lua_pushinteger (L, RSPAMD_SPF_RESOLVED_PERM_FAILED); + lua_setfield (L, -2, "perm_fail"); + lua_pushinteger (L, RSPAMD_SPF_FLAG_CACHED); + lua_setfield (L, -2, "cached"); + + lua_setfield (L, -2, "flags"); + + luaL_register (L, NULL, rspamd_spf_f); + + return 1; +} + +void luaopen_spf (lua_State *L) +{ + rspamd_lua_new_class (L, SPF_RECORD_CLASS, rspamd_spf_record_m); + lua_pop (L, 1); /* No need in metatable... */ + + rspamd_lua_add_preload (L, "rspamd_spf", lua_load_spf); + lua_settop (L, 0); +} + +static void +lua_spf_push_result (struct rspamd_lua_spf_cbdata *cbd, gint code_flags, + struct spf_resolved *resolved, const gchar *err) +{ + g_assert (cbd != NULL); + REF_RETAIN (cbd); + + lua_pushcfunction (cbd->L, &rspamd_lua_traceback); + gint err_idx = lua_gettop (cbd->L); + + lua_rawgeti (cbd->L, LUA_REGISTRYINDEX, cbd->cbref); + + if (resolved) { + struct spf_resolved **presolved; + + presolved = lua_newuserdata (cbd->L, sizeof (*presolved)); + rspamd_lua_setclass (cbd->L, SPF_RECORD_CLASS, -1); + *presolved = spf_record_ref (resolved); + } + else { + lua_pushnil (cbd->L); + } + + lua_pushinteger (cbd->L, code_flags); + + if (err) { + lua_pushstring (cbd->L, err); + } + else { + lua_pushnil (cbd->L); + } + + if (lua_pcall (cbd->L, 3, 0, err_idx) != 0) { + struct rspamd_task *task = cbd->task; + + msg_err_task ("cannot call callback function for spf: %s", + lua_tostring (cbd->L, -1)); + } + + lua_settop (cbd->L, err_idx - 1); + + REF_RELEASE (cbd); +} + +static void +lua_spf_dtor (struct rspamd_lua_spf_cbdata *cbd) +{ + if (cbd) { + luaL_unref (cbd->L, LUA_REGISTRYINDEX, cbd->cbref); + if (cbd->item) { + rspamd_symcache_item_async_dec_check (cbd->task, cbd->item, + "lua_spf"); + } + } +} + +static void +spf_lua_lib_callback (struct spf_resolved *record, struct rspamd_task *task, + gpointer ud) +{ + struct rspamd_lua_spf_cbdata *cbd = (struct rspamd_lua_spf_cbdata *)ud; + + if (record) { + if ((record->flags & RSPAMD_SPF_RESOLVED_NA)) { + lua_spf_push_result (cbd, RSPAMD_SPF_RESOLVED_NA, NULL, + "no SPF record"); + } + else if (record->elts->len == 0) { + if (record->flags & RSPAMD_SPF_RESOLVED_PERM_FAILED) { + lua_spf_push_result (cbd, RSPAMD_SPF_RESOLVED_PERM_FAILED, NULL, + "bad SPF record"); + } + else if ((record->flags & RSPAMD_SPF_RESOLVED_TEMP_FAILED)) { + lua_spf_push_result (cbd, RSPAMD_SPF_RESOLVED_TEMP_FAILED, NULL, + "temporary DNS error"); + } + else { + lua_spf_push_result (cbd, RSPAMD_SPF_RESOLVED_PERM_FAILED, NULL, + "empty SPF record"); + } + } + else if (record->domain) { + spf_record_ref (record); + lua_spf_push_result (cbd, record->flags, record, NULL); + spf_record_unref (record); + } + else { + lua_spf_push_result (cbd, RSPAMD_SPF_RESOLVED_PERM_FAILED, NULL, + "internal error: non empty record for no domain"); + } + } + else { + lua_spf_push_result (cbd, RSPAMD_SPF_RESOLVED_PERM_FAILED, NULL, + "internal error: no record"); + } + + REF_RELEASE (cbd); +} + +/*** + * @function rspamd_spf.resolve(task, callback) + * Resolves SPF credentials for a task + * @param {rspamd_task} task task + * @param {function} callback callback that is called on spf resolution +*/ +gint +lua_spf_resolve (lua_State * L) +{ + struct rspamd_task *task = lua_check_task (L, 1); + + if (task && lua_isfunction (L, 2)) { + struct rspamd_lua_spf_cbdata *cbd = rspamd_mempool_alloc0 (task->task_pool, + sizeof (*cbd)); + struct rspamd_spf_cred *spf_cred; + + cbd->task = task; + cbd->L = L; + lua_pushvalue (L, 2); + cbd->cbref = luaL_ref (L, LUA_REGISTRYINDEX); + /* TODO: make it as an optional parameter */ + spf_cred = rspamd_spf_get_cred (task); + cbd->item = rspamd_symcache_get_cur_item (task); + + if (cbd->item) { + rspamd_symcache_item_async_inc (task, cbd->item, "lua_spf"); + } + REF_INIT_RETAIN (cbd, lua_spf_dtor); + + if (!rspamd_spf_resolve (task, spf_lua_lib_callback, cbd, spf_cred)) { + msg_info_task ("cannot make spf request for %s", + spf_cred ? spf_cred->domain : "empty domain"); + if (spf_cred) { + lua_spf_push_result (cbd, RSPAMD_SPF_RESOLVED_TEMP_FAILED, + NULL, "DNS failed"); + } + else { + lua_spf_push_result (cbd, RSPAMD_SPF_RESOLVED_NA, + NULL, "No domain"); + } + REF_RELEASE (cbd); + } + } + else { + return luaL_error (L, "invalid arguments"); + } + + return 0; +} + +static gint +lua_spf_record_dtor (lua_State *L) +{ + struct spf_resolved *record = + * (struct spf_resolved **)rspamd_lua_check_udata (L, 1, + SPF_RECORD_CLASS); + + if (record) { + spf_record_unref (record); + } + + return 0; +} + +static void +lua_spf_push_spf_addr (lua_State *L, struct spf_addr *addr) +{ + gchar *addr_mask; + + lua_createtable (L, 0, 4); + + lua_pushinteger (L, addr->mech); + lua_setfield (L, -2, "result"); + lua_pushinteger (L, addr->flags); + lua_setfield (L, -2, "flags"); + + if (addr->spf_string) { + lua_pushstring (L, addr->spf_string); + lua_setfield (L, -2, "str"); + } + + addr_mask = spf_addr_mask_to_string (addr); + + if (addr_mask) { + lua_pushstring (L, addr_mask); + lua_setfield (L, -2, "addr"); + g_free (addr_mask); + } +} + +static gint +spf_check_element (lua_State *L, struct spf_resolved *rec, struct spf_addr *addr, + struct rspamd_lua_ip *ip) +{ + gboolean res = FALSE; + const guint8 *s, *d; + guint af, mask, bmask, addrlen; + + + if (addr->flags & RSPAMD_SPF_FLAG_TEMPFAIL) { + /* Ignore failed addresses */ + + return -1; + } + + af = rspamd_inet_address_get_af (ip->addr); + /* Basic comparing algorithm */ + if (((addr->flags & RSPAMD_SPF_FLAG_IPV6) && af == AF_INET6) || + ((addr->flags & RSPAMD_SPF_FLAG_IPV4) && af == AF_INET)) { + d = rspamd_inet_address_get_hash_key (ip->addr, &addrlen); + + if (af == AF_INET6) { + s = (const guint8 *)addr->addr6; + mask = addr->m.dual.mask_v6; + } + else { + s = (const guint8 *)addr->addr4; + mask = addr->m.dual.mask_v4; + } + + /* Compare the first bytes */ + bmask = mask / CHAR_BIT; + if (mask > addrlen * CHAR_BIT) { + /* XXX: add logging */ + } + else if (memcmp (s, d, bmask) == 0) { + if (bmask * CHAR_BIT < mask) { + /* Compare the remaining bits */ + s += bmask; + d += bmask; + mask = (0xff << (CHAR_BIT - (mask - bmask * 8))) & 0xff; + + if ((*s & mask) == (*d & mask)) { + res = TRUE; + } + } + else { + res = TRUE; + } + } + } + else { + if (addr->flags & RSPAMD_SPF_FLAG_ANY) { + res = TRUE; + } + else { + res = FALSE; + } + } + + if (res) { + if (addr->flags & RSPAMD_SPF_FLAG_ANY) { + if (rec->flags & RSPAMD_SPF_RESOLVED_PERM_FAILED) { + lua_pushboolean (L, false); + lua_pushinteger (L, RSPAMD_SPF_RESOLVED_PERM_FAILED); + lua_pushfstring (L, "%cany", spf_mech_char (addr->mech)); + } + else if (rec->flags & RSPAMD_SPF_RESOLVED_TEMP_FAILED) { + lua_pushboolean (L, false); + lua_pushinteger (L, RSPAMD_SPF_RESOLVED_TEMP_FAILED); + lua_pushfstring (L, "%cany", spf_mech_char (addr->mech)); + } + else { + lua_pushboolean (L, true); + lua_pushinteger (L, addr->mech); + lua_spf_push_spf_addr (L, addr); + } + } + else { + lua_pushboolean (L, true); + lua_pushinteger (L, addr->mech); + lua_spf_push_spf_addr (L, addr); + } + + return 3; + } + + return -1; +} + +/*** + * @method rspamd_spf_record:check_ip(ip) + * Checks the processed record versus a specific IP address. This function + * returns 3 values normally: + * 1. Boolean check result + * 2. If result is `false` then the second value is the error flag (e.g. rspamd_spf.flags.temp_fail), otherwise it will be an SPF method + * 3. If result is `false` then this will be an error string, otherwise - an SPF string (e.g. `mx` or `ip4:x.y.z.1`) + * @param {rspamd_ip|string} ip address + * @return {result,flag_or_policy,error_or_addr} - triplet +*/ +static gint +lua_spf_record_check_ip (lua_State *L) +{ + struct spf_resolved *record = + * (struct spf_resolved **)rspamd_lua_check_udata (L, 1, + SPF_RECORD_CLASS); + struct rspamd_lua_ip *ip = NULL; + gint nres = 0; + gboolean need_free_ip = FALSE; + + if (lua_type (L, 2) == LUA_TUSERDATA) { + ip = lua_check_ip (L, 2); + } + else if (lua_type (L, 2) == LUA_TSTRING) { + const gchar *ip_str; + gsize iplen; + + ip = g_malloc0 (sizeof (struct rspamd_lua_ip)); + ip_str = lua_tolstring (L, 2, &iplen); + + if (!rspamd_parse_inet_address (&ip->addr, + ip_str, iplen, RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) { + g_free (ip); + ip = NULL; + } + else { + need_free_ip = TRUE; + } + } + + if (record && ip && ip->addr) { + for (guint i = 0; i < record->elts->len; i ++) { + struct spf_addr *addr = &g_array_index (record->elts, struct spf_addr, i); + if ((nres = spf_check_element (L, record, addr, ip)) > 0) { + if (need_free_ip) { + g_free (ip); + } + + return nres; + } + } + } + else { + if (need_free_ip) { + g_free (ip); + } + + return luaL_error (L, "invalid arguments"); + } + + if (need_free_ip) { + g_free (ip); + } + + /* If we are here it means that there is no ALL record */ + /* + * According to https://tools.ietf.org/html/rfc7208#section-4.7 it means + * SPF neutral + */ + struct spf_addr fake_all; + + fake_all.mech = SPF_NEUTRAL; + fake_all.flags = RSPAMD_SPF_FLAG_ANY; + fake_all.spf_string = "all"; + + lua_pushboolean (L, true); + lua_pushinteger (L, SPF_NEUTRAL); + lua_spf_push_spf_addr (L, &fake_all); + + return 3; +} + +/*** + * @method rspamd_spf_record:get_domain() + * Returns domain for the specific spf record +*/ +static gint +lua_spf_record_get_domain (lua_State *L) +{ + struct spf_resolved *record = + *(struct spf_resolved **) rspamd_lua_check_udata (L, 1, + SPF_RECORD_CLASS); + + if (record) { + lua_pushstring (L, record->domain); + } + else { + return luaL_error (L, "invalid arguments"); + } + + return 1; +} + +/*** + * @method rspamd_spf_record:get_ttl() + * Returns ttl for the specific spf record +*/ +static gint +lua_spf_record_get_ttl (lua_State *L) +{ + struct spf_resolved *record = + *(struct spf_resolved **) rspamd_lua_check_udata (L, 1, + SPF_RECORD_CLASS); + + if (record) { + lua_pushinteger (L, record->ttl); + } + else { + return luaL_error (L, "invalid arguments"); + } + + return 1; +} + +/*** + * @method rspamd_spf_record:get_timestamp() + * Returns ttl for the specific spf record +*/ +static gint +lua_spf_record_get_timestamp (lua_State *L) +{ + struct spf_resolved *record = + *(struct spf_resolved **) rspamd_lua_check_udata (L, 1, + SPF_RECORD_CLASS); + + if (record) { + lua_pushnumber (L, record->timestamp); + } + else { + return luaL_error (L, "invalid arguments"); + } + + return 1; +} + +/*** + * @method rspamd_spf_record:get_digest() + * Returns string hex representation of the record digest (fast hash function) +*/ +static gint +lua_spf_record_get_digest (lua_State *L) +{ + struct spf_resolved *record = + *(struct spf_resolved **) rspamd_lua_check_udata (L, 1, + SPF_RECORD_CLASS); + + if (record) { + gchar hexbuf[64]; + + rspamd_snprintf (hexbuf, sizeof (hexbuf), "%xuL", record->digest); + lua_pushstring (L, hexbuf); + } + else { + return luaL_error (L, "invalid arguments"); + } + + return 1; +} + +/*** + * @method rspamd_spf_record:get_elts() + * Returns a list of all elements in an SPF record. Each element is a table with the + * following fields: + * + * - result - mech flag from rspamd_spf.results + * - flags - all flags + * - addr - address and mask as a string + * - str - string representation (if available) +*/ +static gint +lua_spf_record_get_elts (lua_State *L) +{ + struct spf_resolved *record = + *(struct spf_resolved **) rspamd_lua_check_udata (L, 1, + SPF_RECORD_CLASS); + + if (record) { + guint i; + struct spf_addr *addr; + + lua_createtable (L, record->elts->len, 0); + + for (i = 0; i < record->elts->len; i ++) { + addr = (struct spf_addr *)&g_array_index (record->elts, + struct spf_addr, i); + lua_spf_push_spf_addr (L, addr); + + lua_rawseti (L, -2, i + 1); + } + } + else { + return luaL_error (L, "invalid arguments"); + } + + return 1; +} + +/*** + * @function rspamd_spf.config(object) + * Configures SPF library according to the UCL config + * @param {table} object configuration object +*/ +gint +lua_spf_config (lua_State * L) +{ + ucl_object_t *config_obj = ucl_object_lua_import (L, 1); + + if (config_obj) { + spf_library_config (config_obj); + ucl_object_unref (config_obj); /* As we copy data all the time */ + } + else { + return luaL_error (L, "invalid arguments"); + } + + return 0; +}
\ No newline at end of file diff --git a/src/lua/lua_task.c b/src/lua/lua_task.c index e59bd3685..774bb0120 100644 --- a/src/lua/lua_task.c +++ b/src/lua/lua_task.c @@ -1615,7 +1615,7 @@ lua_task_load_from_file (lua_State * L) } } - task = rspamd_task_new (NULL, cfg, NULL, NULL, NULL); + task = rspamd_task_new (NULL, cfg, NULL, NULL, NULL, FALSE); task->msg.begin = data->str; task->msg.len = data->len; rspamd_mempool_add_destructor (task->task_pool, @@ -1629,7 +1629,7 @@ lua_task_load_from_file (lua_State * L) if (!map) { err = strerror (errno); } else { - task = rspamd_task_new (NULL, cfg, NULL, NULL, NULL); + task = rspamd_task_new (NULL, cfg, NULL, NULL, NULL, FALSE); task->msg.begin = map; task->msg.len = sz; rspamd_mempool_add_destructor (task->task_pool, @@ -1683,7 +1683,7 @@ lua_task_load_from_string (lua_State * L) } } - task = rspamd_task_new (NULL, cfg, NULL, NULL, NULL); + task = rspamd_task_new (NULL, cfg, NULL, NULL, NULL, FALSE); task->msg.begin = g_malloc (message_len); memcpy ((gchar *)task->msg.begin, str_message, message_len); task->msg.len = message_len; @@ -1729,7 +1729,7 @@ lua_task_create (lua_State * L) } } - task = rspamd_task_new (NULL, cfg, NULL, NULL, ev_base); + task = rspamd_task_new (NULL, cfg, NULL, NULL, ev_base, FALSE); task->flags |= RSPAMD_TASK_FLAG_EMPTY; ptask = lua_newuserdata (L, sizeof (*ptask)); @@ -1861,27 +1861,54 @@ lua_task_insert_result (lua_State * L) luaL_checkstring (L, args_start)); weight = luaL_checknumber (L, args_start + 1); top = lua_gettop (L); - s = rspamd_task_insert_result_full (task, symbol_name, weight, NULL, flags); + s = rspamd_task_insert_result_full (task, symbol_name, weight, + NULL, flags); /* Get additional options */ if (s) { for (i = args_start + 2; i <= top; i++) { - if (lua_type (L, i) == LUA_TSTRING) { + gint ltype = lua_type (L, i); + + if (ltype == LUA_TSTRING) { param = luaL_checkstring (L, i); rspamd_task_add_result_option (task, s, param); } - else if (lua_type (L, i) == LUA_TTABLE) { + else if (ltype == LUA_TTABLE) { lua_pushvalue (L, i); lua_pushnil (L); while (lua_next (L, -2)) { - param = lua_tostring (L, -1); - rspamd_task_add_result_option (task, s, param); + if (lua_isstring (L, -1)) { + param = lua_tostring (L, -1); + rspamd_task_add_result_option (task, s, param); + } + else { + const gchar *tname = lua_typename (L, lua_type (L, -1)); + lua_pop (L, 2); + + return luaL_error (L, "not a string option in a table " + "when adding symbol %s: %s type", + s->name, tname); + } + lua_pop (L, 1); } lua_pop (L, 1); } + else if (ltype == LUA_TNIL) { + /* We have received a NULL option, it is not good but not a fatal error */ + msg_info_task ("nil option when adding symbol %s at pos %d", + s->name, i); + continue; + } + else { + const gchar *tname = lua_typename (L, ltype); + + return luaL_error (L, "not a string/table option " + "when adding symbol %s: %s type", + s->name, tname); + } } } @@ -2090,7 +2117,7 @@ lua_task_append_message (lua_State * L) { LUA_TRACE_POINT; struct rspamd_task *task = lua_check_task (L, 1); - const gchar *message = luaL_checkstring (L, 2), *category; + const gchar *category; if (task != NULL) { if (lua_type (L, 3) == LUA_TSTRING) { @@ -2101,7 +2128,7 @@ lua_task_append_message (lua_State * L) } ucl_object_insert_key (task->messages, - ucl_object_fromstring_common (message, 0, UCL_STRING_RAW), + ucl_object_lua_import (L, 2), category, 0, true); } @@ -2839,7 +2866,13 @@ lua_task_get_received_headers (lua_State * L) const gchar *proto; guint k = 1; - if (task && task->message) { + if (task) { + if (!task->message) { + /* No message - no received */ + lua_newtable (L); + return 1; + } + if (!lua_task_get_cached (L, task, "received")) { lua_createtable (L, 0, 0); @@ -3917,28 +3950,47 @@ lua_task_set_from_ip (lua_State *L) { LUA_TRACE_POINT; struct rspamd_task *task = lua_check_task (L, 1); - gsize len; - const gchar *ip_str = luaL_checklstring (L, 2, &len); rspamd_inet_addr_t *addr = NULL; - if (!task || !ip_str) { - lua_pushstring (L, "invalid parameters"); - return lua_error (L); + if (!task) { + return luaL_error (L, "no task"); } else { - if (!rspamd_parse_inet_address (&addr, - ip_str, - len, - RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) { - msg_warn_task ("cannot get IP from received header: '%s'", - ip_str); - } - else { - if (task->from_addr) { - rspamd_inet_address_free (task->from_addr); + if (lua_type (L, 2) == LUA_TSTRING) { + gsize len; + const gchar *ip_str = lua_tolstring (L, 2, &len); + + if (!rspamd_parse_inet_address (&addr, + ip_str, + len, + RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) { + return luaL_error (L, "invalid IP string: %s", ip_str); } + else { + if (task->from_addr) { + rspamd_inet_address_free (task->from_addr); + } - task->from_addr = addr; + task->from_addr = addr; + } + } + else if (lua_type (L, 2) == LUA_TUSERDATA) { + struct rspamd_lua_ip *ip = lua_check_ip (L, 2); + + if (ip && ip->addr) { + if (task->from_addr) { + rspamd_inet_address_free (task->from_addr); + } + + task->from_addr = rspamd_inet_address_copy (ip->addr); + } + else { + return luaL_error (L, "invalid IP object"); + } + } + else { + return luaL_error (L, "invalid IP argument type: %s", lua_typename (L, + lua_type (L, 2))); } } @@ -4110,7 +4162,7 @@ lua_task_get_images (lua_State *L) lua_createtable (L, MESSAGE_FIELD (task, parts)->len, 0); PTR_ARRAY_FOREACH (MESSAGE_FIELD (task, parts), i, part) { - if (part->flags & RSPAMD_MIME_PART_IMAGE) { + if (part->part_type == RSPAMD_MIME_PART_IMAGE) { pimg = lua_newuserdata (L, sizeof (struct rspamd_image *)); rspamd_lua_setclass (L, "rspamd{image}", -1); *pimg = part->specific.img; @@ -4147,7 +4199,7 @@ lua_task_get_archives (lua_State *L) lua_createtable (L, MESSAGE_FIELD (task, parts)->len, 0); PTR_ARRAY_FOREACH (MESSAGE_FIELD (task, parts), i, part) { - if (part->flags & RSPAMD_MIME_PART_ARCHIVE) { + if (part->part_type == RSPAMD_MIME_PART_ARCHIVE) { parch = lua_newuserdata (L, sizeof (struct rspamd_archive *)); rspamd_lua_setclass (L, "rspamd{archive}", -1); *parch = part->specific.arch; @@ -4597,10 +4649,15 @@ struct tokens_foreach_cbdata { }; static void -tokens_foreach_cb (gint id, const gchar *sym, gint flags, gpointer ud) +tokens_foreach_cb (struct rspamd_symcache_item *item, gpointer ud) { struct tokens_foreach_cbdata *cbd = ud; struct rspamd_symbol_result *s; + gint flags; + const gchar *sym; + + sym = rspamd_symcache_item_name (item); + flags = rspamd_symcache_item_flags (item); if (flags & SYMBOL_TYPE_NOSTAT) { return; @@ -4849,14 +4906,19 @@ lua_task_get_timeval (lua_State *L) struct timeval tv; if (task != NULL) { - double_to_tv (task->task_timestamp, &tv); - lua_createtable (L, 0, 2); - lua_pushstring (L, "tv_sec"); - lua_pushinteger (L, (lua_Integer)tv.tv_sec); - lua_settable (L, -3); - lua_pushstring (L, "tv_usec"); - lua_pushinteger (L, (lua_Integer)tv.tv_usec); - lua_settable (L, -3); + if (lua_isboolean (L, 2) && !!lua_toboolean (L, 2)) { + lua_pushnumber (L, task->task_timestamp); + } + else { + double_to_tv (task->task_timestamp, &tv); + lua_createtable (L, 0, 2); + lua_pushstring (L, "tv_sec"); + lua_pushinteger (L, (lua_Integer) tv.tv_sec); + lua_settable (L, -3); + lua_pushstring (L, "tv_usec"); + lua_pushinteger (L, (lua_Integer) tv.tv_usec); + lua_settable (L, -3); + } } else { return luaL_error (L, "invalid arguments"); diff --git a/src/lua/lua_tcp.c b/src/lua/lua_tcp.c index 09d572199..1edd7127e 100644 --- a/src/lua/lua_tcp.c +++ b/src/lua/lua_tcp.c @@ -1768,6 +1768,7 @@ lua_tcp_request (lua_State *L) if (rspamd_session_blocked (session)) { lua_tcp_push_error (cbd, TRUE, "async session is the blocked state"); TCP_RELEASE (cbd); + cbd->item = NULL; /* To avoid decrease with no watcher */ lua_pushboolean (L, FALSE); return 1; @@ -1784,6 +1785,7 @@ lua_tcp_request (lua_State *L) lua_tcp_push_error (cbd, TRUE, "cannot connect to the host: %s", host); lua_pushboolean (L, FALSE); + /* No reset of the item as watcher has been registered */ TCP_RELEASE (cbd); return 1; @@ -1795,7 +1797,7 @@ lua_tcp_request (lua_State *L) RDNS_REQUEST_A, host)) { lua_tcp_push_error (cbd, TRUE, "cannot resolve host: %s", host); lua_pushboolean (L, FALSE); - + cbd->item = NULL; /* To avoid decrease with no watcher */ TCP_RELEASE (cbd); return 1; @@ -1809,6 +1811,7 @@ lua_tcp_request (lua_State *L) RDNS_REQUEST_A, host)) { lua_tcp_push_error (cbd, TRUE, "cannot resolve host: %s", host); lua_pushboolean (L, FALSE); + cbd->item = NULL; /* To avoid decrease with no watcher */ TCP_RELEASE (cbd); diff --git a/src/lua/lua_text.c b/src/lua/lua_text.c index 328d6e8d5..5c01a239b 100644 --- a/src/lua/lua_text.c +++ b/src/lua/lua_text.c @@ -75,6 +75,21 @@ LUA_FUNCTION_DEF (text, save_in_file); */ LUA_FUNCTION_DEF (text, span); /*** + * @method rspamd_text:lines([stringify]) + * Returns an iter over all lines as rspamd_text objects or as strings if `stringify` is true + * @param {boolean} stringify stringify lines + * @return {iterator} iterator triplet + */ +LUA_FUNCTION_DEF (text, lines); +/*** + * @method rspamd_text:split(regexp, [stringify]) + * Returns an iter over all encounters of the specific regexp as rspamd_text objects or as strings if `stringify` is true + * @param {rspamd_regexp} regexp regexp (pcre syntax) used for splitting + * @param {boolean} stringify stringify lines + * @return {iterator} iterator triplet + */ +LUA_FUNCTION_DEF (text, split); +/*** * @method rspamd_text:at(pos) * Returns a byte at the position `pos` * @param {integer} pos index @@ -104,6 +119,8 @@ static const struct luaL_reg textlib_m[] = { LUA_INTERFACE_DEF (text, take_ownership), LUA_INTERFACE_DEF (text, save_in_file), LUA_INTERFACE_DEF (text, span), + LUA_INTERFACE_DEF (text, lines), + LUA_INTERFACE_DEF (text, split), LUA_INTERFACE_DEF (text, at), LUA_INTERFACE_DEF (text, bytes), {"write", lua_text_save_in_file}, @@ -123,25 +140,31 @@ lua_check_text (lua_State * L, gint pos) } struct rspamd_lua_text * -lua_new_text (lua_State *L, const gchar *start, gsize len, guint flags) +lua_new_text (lua_State *L, const gchar *start, gsize len, gboolean own) { struct rspamd_lua_text *t; t = lua_newuserdata (L, sizeof (*t)); + t->flags = 0; - if (len > 0 && (flags & RSPAMD_TEXT_FLAG_OWN)) { + if (own) { gchar *storage; - storage = g_malloc (len); - memcpy (storage, start, len); - t->start = storage; + if (len > 0) { + storage = g_malloc (len); + memcpy (storage, start, len); + t->start = storage; + t->flags = RSPAMD_TEXT_FLAG_OWN; + } + else { + t->start = ""; + } } else { t->start = start; } t->len = len; - t->flags = flags; rspamd_lua_setclass (L, "rspamd{text}", -1); return t; @@ -163,7 +186,7 @@ lua_text_fromstring (lua_State *L) transparent = lua_toboolean (L, 2); } - lua_new_text (L, str, l, transparent ? 0 : RSPAMD_TEXT_FLAG_OWN); + lua_new_text (L, str, l, !transparent); } else { return luaL_error (L, "invalid arguments"); @@ -173,31 +196,24 @@ lua_text_fromstring (lua_State *L) return 1; } -static gint -lua_text_fromtable (lua_State *L) +#define MAX_REC 10 + +static void +lua_text_tbl_length (lua_State *L, gsize dlen, gsize *dest, guint rec) { - LUA_TRACE_POINT; - const gchar *delim = "", *st; - struct rspamd_lua_text *t, *elt; - gsize textlen = 0, dlen, stlen, tblen; - gchar *dest; + gsize tblen, stlen; + struct rspamd_lua_text *elt; - if (!lua_istable (L, 1)) { - return luaL_error (L, "invalid arguments"); - } + if (rec > MAX_REC) { + luaL_error (L, "lua_text_tbl_length: recursion limit exceeded"); - if (lua_type (L, 2) == LUA_TSTRING) { - delim = lua_tolstring (L, 2, &dlen); - } - else { - dlen = 0; + return; } - /* Calculate length needed */ - tblen = rspamd_lua_table_size (L, 1); + tblen = rspamd_lua_table_size (L, -1); - for (guint i = 0; i < tblen; i ++) { - lua_rawgeti (L, 1, i + 1); + for (gsize i = 0; i < tblen; i ++) { + lua_rawgeti (L, -1, i + 1); if (lua_type (L, -1) == LUA_TSTRING) { #if LUA_VERSION_NUM >= 502 @@ -205,55 +221,114 @@ lua_text_fromtable (lua_State *L) #else stlen = lua_objlen (L, -1); #endif - textlen += stlen; + (*dest) += stlen; } - else { - elt = lua_check_text (L, -1); + else if (lua_type (L, -1) == LUA_TUSERDATA){ + elt = (struct rspamd_lua_text *)lua_touserdata (L, -1); if (elt) { - textlen += elt->len; + (*dest) += elt->len; } } + else if (lua_type (L, -1) == LUA_TTABLE) { + lua_text_tbl_length (L, dlen, dest, rec + 1); + } if (i != tblen - 1) { - textlen += dlen; + (*dest) += dlen; } lua_pop (L, 1); } +} - /* Allocate new text */ - t = lua_newuserdata (L, sizeof (*t)); - dest = g_malloc (textlen); - t->start = dest; - t->len = textlen; - t->flags = RSPAMD_TEXT_FLAG_OWN; - rspamd_lua_setclass (L, "rspamd{text}", -1); +static void +lua_text_tbl_append (lua_State *L, + const gchar *delim, + gsize dlen, + gchar **dest, + guint rec) +{ + const gchar *st; + gsize tblen, stlen; + struct rspamd_lua_text *elt; + + if (rec > MAX_REC) { + luaL_error (L, "lua_text_tbl_length: recursion limit exceeded"); + + return; + } + + tblen = rspamd_lua_table_size (L, -1); for (guint i = 0; i < tblen; i ++) { - lua_rawgeti (L, 1, i + 1); + lua_rawgeti (L, -1, i + 1); if (lua_type (L, -1) == LUA_TSTRING) { st = lua_tolstring (L, -1, &stlen); - memcpy (dest, st, stlen); - dest += stlen; + memcpy ((*dest), st, stlen); + (*dest) += stlen; } - else { - elt = lua_check_text (L, -1); + else if (lua_type (L, -1) == LUA_TUSERDATA){ + elt = (struct rspamd_lua_text *)lua_touserdata (L, -1); if (elt) { - memcpy (dest, elt->start, elt->len); - dest += elt->len; + memcpy ((*dest), elt->start, elt->len); + (*dest) += elt->len; } } + else if (lua_type (L, -1) == LUA_TTABLE) { + lua_text_tbl_append (L, delim, dlen, dest, rec + 1); + } if (dlen && i != tblen - 1) { - memcpy (dest, delim, dlen); - dest += dlen; + memcpy ((*dest), delim, dlen); + (*dest) += dlen; } lua_pop (L, 1); } +} + +static gint +lua_text_fromtable (lua_State *L) +{ + LUA_TRACE_POINT; + const gchar *delim = ""; + struct rspamd_lua_text *t; + gsize textlen = 0, dlen, oldtop = lua_gettop (L); + gchar *dest; + + if (!lua_istable (L, 1)) { + return luaL_error (L, "invalid arguments"); + } + + if (lua_type (L, 2) == LUA_TSTRING) { + delim = lua_tolstring (L, 2, &dlen); + } + else { + dlen = 0; + } + + /* Calculate length needed */ + lua_pushvalue (L, 1); + lua_text_tbl_length (L, dlen, &textlen, 0); + lua_pop (L, 1); + + /* Allocate new text */ + t = lua_newuserdata (L, sizeof (*t)); + dest = g_malloc (textlen); + t->start = dest; + t->len = textlen; + t->flags = RSPAMD_TEXT_FLAG_OWN; + rspamd_lua_setclass (L, "rspamd{text}", -1); + + lua_pushvalue (L, 1); + lua_text_tbl_append (L, delim, dlen, &dest, 0); + lua_pop (L, 1); /* Table arg */ + + gint newtop = lua_gettop (L); + g_assert ( newtop== oldtop + 1); return 1; } @@ -341,7 +416,7 @@ lua_text_span (lua_State *L) { LUA_TRACE_POINT; struct rspamd_lua_text *t = lua_check_text (L, 1); - gint start = lua_tointeger (L, 2), len = -1; + gint64 start = lua_tointeger (L, 2), len = -1; if (t && start >= 1 && start <= t->len) { if (lua_isnumber (L, 3)) { @@ -355,7 +430,123 @@ lua_text_span (lua_State *L) return luaL_error (L, "invalid length"); } - lua_new_text (L, t->start + (start - 1), len, 0); + lua_new_text (L, t->start + (start - 1), len, FALSE); + } + else { + if (!t) { + return luaL_error (L, "invalid arguments, text required"); + } + else { + return luaL_error (L, "invalid arguments: start offset %d " + "is larger than text len %d", (int)start, (int)t->len); + } + } + + return 1; +} + +static gint64 +rspamd_lua_text_push_line (lua_State *L, + struct rspamd_lua_text *t, + gint64 start_offset, + const gchar *sep_pos, + gboolean stringify) +{ + const gchar *start; + gsize len; + gint64 ret; + + start = t->start + start_offset; + len = sep_pos ? (sep_pos - start) : (t->len - start_offset); + ret = start_offset + len; + + /* Trim line */ + while (len > 0) { + if (start[len - 1] == '\r' || start[len - 1] == '\n') { + len --; + } + else { + break; + } + } + + if (stringify) { + lua_pushlstring (L, start, len); + } + else { + struct rspamd_lua_text *ntext; + + ntext = lua_newuserdata (L, sizeof (*ntext)); + rspamd_lua_setclass (L, "rspamd{text}", -1); + ntext->start = start; + ntext->len = len; + ntext->flags = 0; /* Not own as it must be owned by a top object */ + } + + return ret; +} + +static gint +rspamd_lua_text_readline (lua_State *L) +{ + struct rspamd_lua_text *t = lua_touserdata (L, lua_upvalueindex (1)); + gboolean stringify = lua_toboolean (L, lua_upvalueindex (2)); + gint64 pos = lua_tointeger (L, lua_upvalueindex (3)); + + if (pos < 0) { + return luaL_error (L, "invalid pos: %d", (gint)pos); + } + + if (pos >= t->len) { + /* We are done */ + return 0; + } + + const gchar *sep_pos; + + /* We look just for `\n` ignoring `\r` as it is very rare nowadays */ + sep_pos = memchr (t->start + pos, '\n', t->len - pos); + + if (sep_pos == NULL) { + /* Either last `\n` or `\r` separated text */ + sep_pos = memchr (t->start + pos, '\r', t->len - pos); + } + + pos = rspamd_lua_text_push_line (L, t, pos, sep_pos, stringify); + + /* Skip separators */ + while (pos < t->len) { + if (t->start[pos] == '\n' || t->start[pos] == '\r') { + pos ++; + } + else { + break; + } + } + + /* Update pos */ + lua_pushinteger (L, pos); + lua_replace (L, lua_upvalueindex (3)); + + return 1; +} + +static gint +lua_text_lines (lua_State *L) +{ + LUA_TRACE_POINT; + struct rspamd_lua_text *t = lua_check_text (L, 1); + gboolean stringify = FALSE; + + if (t) { + if (lua_isboolean (L, 2)) { + stringify = lua_toboolean (L, 2); + } + + lua_pushvalue (L, 1); + lua_pushboolean (L, stringify); + lua_pushinteger (L, 0); /* Current pos */ + lua_pushcclosure (L, rspamd_lua_text_readline, 3); } else { return luaL_error (L, "invalid arguments"); @@ -365,6 +556,163 @@ lua_text_span (lua_State *L) } static gint +rspamd_lua_text_regexp_split (lua_State *L) { + struct rspamd_lua_text *t = lua_touserdata (L, lua_upvalueindex (1)), + *new_t; + struct rspamd_lua_regexp *re = *(struct rspamd_lua_regexp **) + lua_touserdata (L, lua_upvalueindex (2)); + gboolean stringify = lua_toboolean (L, lua_upvalueindex (3)); + gint64 pos = lua_tointeger (L, lua_upvalueindex (4)); + gboolean matched; + + if (pos < 0) { + return luaL_error (L, "invalid pos: %d", (gint) pos); + } + + if (pos >= t->len) { + /* We are done */ + return 0; + } + + const gchar *start, *end, *old_start; + + end = t->start + pos; + + for (;;) { + old_start = end; + + matched = rspamd_regexp_search (re->re, t->start, t->len, &start, &end, FALSE, + NULL); + + if (matched) { + if (start - old_start > 0) { + if (stringify) { + lua_pushlstring (L, old_start, start - old_start); + } + else { + new_t = lua_newuserdata (L, sizeof (*t)); + rspamd_lua_setclass (L, "rspamd{text}", -1); + new_t->start = old_start; + new_t->len = start - old_start; + new_t->flags = 0; + } + + break; + } + else { + if (start == end) { + matched = FALSE; + break; + } + /* + * All match separators (e.g. starting separator, + * we need to skip it). Continue iterations. + */ + } + } + else { + /* No match, stop */ + break; + } + } + + if (!matched && (t->len > 0 && (end == NULL || end < t->start + t->len))) { + /* No more matches, but we might need to push the last element */ + if (end == NULL) { + end = t->start; + } + /* No separators, need to push the whole remaining part */ + if (stringify) { + lua_pushlstring (L, end, (t->start + t->len) - end); + } + else { + new_t = lua_newuserdata (L, sizeof (*t)); + rspamd_lua_setclass (L, "rspamd{text}", -1); + new_t->start = end; + new_t->len = (t->start + t->len) - end; + new_t->flags = 0; + } + + pos = t->len; + } + else { + + pos = end - t->start; + } + + /* Update pos */ + lua_pushinteger (L, pos); + lua_replace (L, lua_upvalueindex (4)); + + return 1; +} + +static gint +lua_text_split (lua_State *L) +{ + LUA_TRACE_POINT; + struct rspamd_lua_text *t = lua_check_text (L, 1); + struct rspamd_lua_regexp *re; + gboolean stringify = FALSE, own_re = FALSE; + + if (lua_type (L, 2) == LUA_TUSERDATA) { + re = lua_check_regexp (L, 2); + } + else { + rspamd_regexp_t *c_re; + GError *err = NULL; + + c_re = rspamd_regexp_new (lua_tostring (L, 2), NULL, &err); + if (c_re == NULL) { + + gint ret = luaL_error (L, "cannot parse regexp: %s, error: %s", + lua_tostring (L, 2), + err == NULL ? "undefined" : err->message); + if (err) { + g_error_free (err); + } + + return ret; + } + + re = g_malloc0 (sizeof (struct rspamd_lua_regexp)); + re->re = c_re; + re->re_pattern = g_strdup (lua_tostring (L, 2)); + re->module = rspamd_lua_get_module_name (L); + own_re = TRUE; + } + + if (t && re) { + if (lua_isboolean (L, 3)) { + stringify = lua_toboolean (L, 3); + } + + /* Upvalues */ + lua_pushvalue (L, 1); /* text */ + + if (own_re) { + struct rspamd_lua_regexp **pre; + pre = lua_newuserdata (L, sizeof (struct rspamd_lua_regexp *)); + rspamd_lua_setclass (L, "rspamd{regexp}", -1); + *pre = re; + } + else { + lua_pushvalue (L, 2); /* regexp */ + } + + lua_pushboolean (L, stringify); + lua_pushinteger (L, 0); /* Current pos */ + lua_pushcclosure (L, rspamd_lua_text_regexp_split, 4); + } + else { + return luaL_error (L, "invalid arguments"); + } + + return 1; +} + + +static gint lua_text_at (lua_State *L) { LUA_TRACE_POINT; diff --git a/src/lua/lua_upstream.c b/src/lua/lua_upstream.c index 7ba77839f..2f04d7059 100644 --- a/src/lua/lua_upstream.c +++ b/src/lua/lua_upstream.c @@ -129,14 +129,22 @@ lua_upstream_fail (lua_State *L) LUA_TRACE_POINT; struct upstream *up = lua_check_upstream (L); gboolean fail_addr = FALSE; + const gchar *reason = "unknown"; if (up) { if (lua_isboolean (L, 2)) { fail_addr = lua_toboolean (L, 2); + + if (lua_isstring (L, 3)) { + reason = lua_tostring (L, 3); + } + } + else if (lua_isstring (L, 2)) { + reason = lua_tostring (L, 2); } - rspamd_upstream_fail (up, fail_addr); + rspamd_upstream_fail (up, fail_addr, reason); } return 0; diff --git a/src/lua/lua_url.c b/src/lua/lua_url.c index cc76f06e8..0bd4f1c7e 100644 --- a/src/lua/lua_url.c +++ b/src/lua/lua_url.c @@ -731,7 +731,7 @@ lua_url_create (lua_State *L) } else { own_pool = TRUE; - pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "url"); + pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "url", 0); text = luaL_checklstring (L, 1, &length); } @@ -789,9 +789,7 @@ lua_url_table_inserter (struct rspamd_url *url, gsize start_offset, lua_url = lua_newuserdata (L, sizeof (struct rspamd_lua_url)); rspamd_lua_setclass (L, "rspamd{url}", -1); lua_url->url = url; - lua_pushinteger (L, n + 1); - lua_pushlstring (L, url->string, url->urllen); - lua_settable (L, -3); + lua_rawseti (L, -2, n + 1); return TRUE; } diff --git a/src/lua/lua_util.c b/src/lua/lua_util.c index 1ea8d380c..0b52cfbdc 100644 --- a/src/lua/lua_util.c +++ b/src/lua/lua_util.c @@ -34,6 +34,7 @@ #include "unicode/uspoof.h" #include "unicode/uscript.h" +#include "contrib/fastutf8/fastutf8.h" /*** * @module rspamd_util @@ -382,15 +383,26 @@ LUA_FUNCTION_DEF (util, zstd_compress); LUA_FUNCTION_DEF (util, zstd_decompress); /*** - * @function util.gzip_decompress(data) + * @function util.gzip_decompress(data, [size_limit]) * Decompresses input using gzip algorithm * * @param {string/rspamd_text} data compressed data - * @return {error,rspamd_text} pair of error + decompressed text + * @param {integer} size_limit optional size limit + * @return {rspamd_text} decompressed text */ LUA_FUNCTION_DEF (util, gzip_decompress); /*** + * @function util.inflate(data, [size_limit]) + * Decompresses input using inflate algorithm + * + * @param {string/rspamd_text} data compressed data + * @param {integer} size_limit optional size limit + * @return {rspamd_text} decompressed text + */ +LUA_FUNCTION_DEF (util, inflate); + +/*** * @function util.gzip_compress(data) * Compresses input using gzip compression * @@ -661,6 +673,7 @@ static const struct luaL_reg utillib_f[] = { LUA_INTERFACE_DEF (util, zstd_decompress), LUA_INTERFACE_DEF (util, gzip_compress), LUA_INTERFACE_DEF (util, gzip_decompress), + LUA_INTERFACE_DEF (util, inflate), LUA_INTERFACE_DEF (util, normalize_prob), LUA_INTERFACE_DEF (util, caseless_hash), LUA_INTERFACE_DEF (util, caseless_hash_fast), @@ -872,7 +885,7 @@ lua_util_process_message (lua_State *L) if (cfg != NULL && message != NULL) { base = ev_loop_new (EVFLAG_SIGNALFD|EVBACKEND_ALL); rspamd_init_filters (cfg, FALSE); - task = rspamd_task_new (NULL, cfg, NULL, NULL, base); + task = rspamd_task_new (NULL, cfg, NULL, NULL, base, FALSE); task->msg.begin = rspamd_mempool_alloc (task->task_pool, mlen); rspamd_strlcpy ((gpointer)task->msg.begin, message, mlen); task->msg.len = mlen; @@ -1385,7 +1398,7 @@ lua_util_parse_html (lua_State *L) } if (start != NULL) { - pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL); + pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL, 0); hc = rspamd_mempool_alloc0 (pool, sizeof (*hc)); in = g_byte_array_sized_new (len); g_byte_array_append (in, start, len); @@ -1455,7 +1468,8 @@ lua_util_parse_addr (lua_State *L) } } else { - pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "lua util"); + pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), + "lua util", 0); own_pool = TRUE; } @@ -1659,7 +1673,8 @@ lua_util_parse_mail_address (lua_State *L) } } else { - pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "lua util"); + pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), + "lua util", 0); own_pool = TRUE; } @@ -2315,7 +2330,7 @@ lua_util_gzip_compress (lua_State *L) static gint -lua_util_gzip_decompress (lua_State *L) +lua_util_zlib_inflate (lua_State *L, int windowBits) { LUA_TRACE_POINT; struct rspamd_lua_text *t = NULL, *res, tmp; @@ -2324,6 +2339,7 @@ lua_util_gzip_decompress (lua_State *L) gint rc; guchar *p; gsize remain; + gssize size_limit = -1; if (lua_type (L, 1) == LUA_TSTRING) { t = &tmp; @@ -2338,11 +2354,21 @@ lua_util_gzip_decompress (lua_State *L) return luaL_error (L, "invalid arguments"); } - sz = t->len; + if (lua_type (L, 2) == LUA_TNUMBER) { + size_limit = lua_tointeger (L, 2); + if (size_limit <= 0) { + return luaL_error (L, "invalid arguments (size_limit)"); + } + + sz = MIN (t->len * 2, size_limit); + } + else { + sz = t->len * 2; + } memset (&strm, 0, sizeof (strm)); /* windowBits +16 to decode gzip, zlib 1.2.0.4+ */ - rc = inflateInit2 (&strm, MAX_WBITS + 16); + rc = inflateInit2 (&strm, windowBits); if (rc != Z_OK) { return luaL_error (L, "cannot init zlib"); @@ -2363,7 +2389,7 @@ lua_util_gzip_decompress (lua_State *L) strm.avail_out = remain; strm.next_out = p; - rc = inflate (&strm, Z_FINISH); + rc = inflate (&strm, Z_NO_FLUSH); if (rc != Z_OK && rc != Z_BUF_ERROR) { if (rc == Z_STREAM_END) { @@ -2382,10 +2408,21 @@ lua_util_gzip_decompress (lua_State *L) res->len = strm.total_out; if (strm.avail_out == 0 && strm.avail_in != 0) { + + if (size_limit > 0 || res->len >= G_MAXUINT32 / 2) { + if (res->len > size_limit || res->len >= G_MAXUINT32 / 2) { + lua_pop (L, 1); /* Text will be freed here */ + lua_pushnil (L); + inflateEnd (&strm); + + return 1; + } + } + /* Need to allocate more */ remain = res->len; - res->start = g_realloc ((gpointer)res->start, strm.avail_in + sz); - sz = strm.avail_in + sz; + res->start = g_realloc ((gpointer)res->start, res->len * 2); + sz = res->len * 2; p = (guchar *)res->start + remain; remain = sz - remain; } @@ -2394,9 +2431,19 @@ lua_util_gzip_decompress (lua_State *L) inflateEnd (&strm); res->len = strm.total_out; - return 2; + return 1; +} +static gint +lua_util_gzip_decompress (lua_State *L) +{ + return lua_util_zlib_inflate (L, MAX_WBITS + 16); } +static gint +lua_util_inflate (lua_State *L) +{ + return lua_util_zlib_inflate (L, MAX_WBITS); +} static gint lua_util_normalize_prob (lua_State *L) @@ -2716,10 +2763,13 @@ lua_util_is_utf_outside_range(lua_State *L) return 1; } - rspamd_lru_hash_insert(validators, creation_hash_key, validator, 0, 0); + rspamd_lru_hash_insert(validators, creation_hash_key, validator, + 0, 0); } - ret = uspoof_checkUTF8 (validator, string_to_check, len_of_string, NULL, &uc_err); + gint32 pos = 0; + ret = uspoof_checkUTF8 (validator, string_to_check, len_of_string, &pos, + &uc_err); } else { return luaL_error (L, "invalid arguments"); @@ -2855,10 +2905,33 @@ lua_util_is_valid_utf8 (lua_State *L) const gchar *str; gsize len; - str = lua_tolstring (L, 1, &len); + if (lua_isstring (L, 1)) { + str = lua_tolstring (L, 1, &len); + } + else { + struct rspamd_lua_text *t = lua_check_text (L, 1); + + if (t) { + str = t->start; + len = t->len; + } + else { + return luaL_error (L, "invalid arguments (text expected)"); + } + } if (str) { - lua_pushboolean (L, g_utf8_validate (str, len, NULL)); + goffset error_offset = rspamd_fast_utf8_validate (str, len); + + if (error_offset == 0) { + lua_pushboolean (L, true); + } + else { + lua_pushboolean (L, false); + lua_pushnumber (L, error_offset); + + return 2; + } } else { return luaL_error (L, "invalid arguments"); diff --git a/src/lua/lua_worker.c b/src/lua/lua_worker.c index 4a3e4e908..17266ae96 100644 --- a/src/lua/lua_worker.c +++ b/src/lua/lua_worker.c @@ -423,7 +423,7 @@ lua_worker_add_control_handler (lua_State *L) } rspamd_mempool_t *pool = rspamd_mempool_new ( - rspamd_mempool_suggest_size (), "lua_control"); + rspamd_mempool_suggest_size (), "lua_control", 0); cbd = rspamd_mempool_alloc0 (pool, sizeof (*cbd)); cbd->pool = pool; cbd->event_loop = event_loop; diff --git a/src/plugins/dkim_check.c b/src/plugins/dkim_check.c index 410a38309..133feef2f 100644 --- a/src/plugins/dkim_check.c +++ b/src/plugins/dkim_check.c @@ -435,7 +435,7 @@ dkim_module_config (struct rspamd_config *cfg) rspamd_config_get_module_opt (cfg, "dkim", "whitelist")) != NULL) { rspamd_config_radix_from_ucl (cfg, value, "DKIM whitelist", - &dkim_module_ctx->whitelist_ip, NULL); + &dkim_module_ctx->whitelist_ip, NULL, NULL); } if ((value = @@ -445,7 +445,8 @@ dkim_module_config (struct rspamd_config *cfg) rspamd_kv_list_read, rspamd_kv_list_fin, rspamd_kv_list_dtor, - (void **)&dkim_module_ctx->dkim_domains)) { + (void **)&dkim_module_ctx->dkim_domains, + NULL)) { msg_warn_config ("cannot load dkim domains list from %s", ucl_object_tostring (value)); } @@ -461,7 +462,8 @@ dkim_module_config (struct rspamd_config *cfg) rspamd_kv_list_read, rspamd_kv_list_fin, rspamd_kv_list_dtor, - (void **)&dkim_module_ctx->dkim_domains)) { + (void **)&dkim_module_ctx->dkim_domains, + NULL)) { msg_warn_config ("cannot load dkim domains list from %s", ucl_object_tostring (value)); } @@ -538,7 +540,7 @@ dkim_module_config (struct rspamd_config *cfg) 0.0, "DKIM check callback", "policies", - RSPAMD_SYMBOL_FLAG_IGNORE, + RSPAMD_SYMBOL_FLAG_IGNORE_METRIC, 1, 1); rspamd_config_add_symbol_group (cfg, "DKIM_CHECK", "dkim"); @@ -585,7 +587,7 @@ dkim_module_config (struct rspamd_config *cfg) 0.0, "DKIM trace symbol", "policies", - RSPAMD_SYMBOL_FLAG_IGNORE, + RSPAMD_SYMBOL_FLAG_IGNORE_METRIC, 1, 1); rspamd_config_add_symbol_group (cfg, "DKIM_TRACE", "dkim"); @@ -1119,7 +1121,8 @@ dkim_symbol_callback (struct rspamd_task *task, GError *err = NULL; struct rspamd_mime_header *rh, *rh_cur; struct dkim_check_result *res = NULL, *cur; - guint checked = 0, *dmarc_checks; + guint checked = 0; + gdouble *dmarc_checks; struct dkim_ctx *dkim_module_ctx = dkim_get_context (task->cfg); /* Allow dmarc */ diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c index 5f706e237..e8f02652d 100644 --- a/src/plugins/fuzzy_check.c +++ b/src/plugins/fuzzy_check.c @@ -359,7 +359,8 @@ fuzzy_parse_rule (struct rspamd_config *cfg, const ucl_object_t *obj, rspamd_kv_list_read, rspamd_kv_list_fin, rspamd_kv_list_dtor, - (void **)&rule->skip_map); + (void **)&rule->skip_map, + NULL); } if ((value = ucl_object_lookup (obj, "headers")) != NULL) { @@ -617,7 +618,8 @@ fuzzy_check_module_init (struct rspamd_config *cfg, struct module_ctx **ctx) fuzzy_module_ctx = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct fuzzy_ctx)); - fuzzy_module_ctx->fuzzy_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL); + fuzzy_module_ctx->fuzzy_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), + NULL, 0); /* TODO: this should match rules count actually */ fuzzy_module_ctx->keypairs_cache = rspamd_keypair_cache_new (32); fuzzy_module_ctx->fuzzy_rules = g_ptr_array_new (); @@ -1042,7 +1044,9 @@ fuzzy_check_module_config (struct rspamd_config *cfg) rspamd_config_get_module_opt (cfg, "fuzzy_check", "whitelist")) != NULL) { rspamd_config_radix_from_ucl (cfg, value, "Fuzzy whitelist", - &fuzzy_module_ctx->whitelist, NULL); + &fuzzy_module_ctx->whitelist, + NULL, + NULL); } else { fuzzy_module_ctx->whitelist = NULL; @@ -1060,7 +1064,7 @@ fuzzy_check_module_config (struct rspamd_config *cfg) 0.0, "Fuzzy check callback", "fuzzy", - RSPAMD_SYMBOL_FLAG_IGNORE, + RSPAMD_SYMBOL_FLAG_IGNORE_METRIC, 1, 1); @@ -2145,7 +2149,7 @@ fuzzy_insert_metric_results (struct rspamd_task *task, GPtrArray *results) if (task->message) { PTR_ARRAY_FOREACH (MESSAGE_FIELD (task, text_parts), i, tp) { - if (!IS_PART_EMPTY (tp) && tp->utf_words->len > 0) { + if (!IS_PART_EMPTY (tp) && tp->utf_words != NULL && tp->utf_words->len > 0) { seen_text_part = TRUE; if (tp->utf_stripped_text.magic == UTEXT_MAGIC) { @@ -2252,7 +2256,7 @@ fuzzy_check_timer_callback (gint fd, short what, void *arg) rspamd_inet_address_to_string_pretty ( rspamd_upstream_addr_cur (session->server)), session->retransmits); - rspamd_upstream_fail (session->server, TRUE); + rspamd_upstream_fail (session->server, TRUE, "timeout"); if (session->item) { rspamd_symcache_item_async_dec_check (session->task, session->item, M); @@ -2334,7 +2338,7 @@ fuzzy_check_io_callback (gint fd, short what, void *arg) session->state == 1 ? "read" : "write", errno, strerror (errno)); - rspamd_upstream_fail (session->server, TRUE); + rspamd_upstream_fail (session->server, TRUE, strerror (errno)); if (session->item) { rspamd_symcache_item_async_dec_check (session->task, session->item, M); @@ -2375,7 +2379,7 @@ fuzzy_controller_timer_callback (gint fd, short what, void *arg) task = session->task; if (session->retransmits >= session->rule->ctx->retransmits) { - rspamd_upstream_fail (session->server, TRUE); + rspamd_upstream_fail (session->server, TRUE, "timeout"); msg_err_task_check ("got IO timeout with server %s(%s), " "after %d retransmits", rspamd_upstream_name (session->server), @@ -2591,7 +2595,7 @@ fuzzy_controller_io_callback (gint fd, short what, void *arg) rspamd_inet_address_to_string_pretty ( rspamd_upstream_addr_cur (session->server)), errno, strerror (errno)); - rspamd_upstream_fail (session->server, FALSE); + rspamd_upstream_fail (session->server, FALSE, strerror (errno)); } /* @@ -2714,7 +2718,7 @@ fuzzy_generate_commands (struct rspamd_task *task, struct fuzzy_rule *rule, io = NULL; if (check_part) { - if (mime_part->flags & RSPAMD_MIME_PART_TEXT && + if (mime_part->part_type == RSPAMD_MIME_PART_TEXT && !(flags & FUZZY_CHECK_FLAG_NOTEXT)) { part = mime_part->specific.txt; @@ -2727,7 +2731,7 @@ fuzzy_generate_commands (struct rspamd_task *task, struct fuzzy_rule *rule, part, mime_part); } - else if (mime_part->flags & RSPAMD_MIME_PART_IMAGE && + else if (mime_part->part_type == RSPAMD_MIME_PART_IMAGE && !(flags & FUZZY_CHECK_FLAG_NOIMAGES)) { image = mime_part->specific.img; @@ -2795,7 +2799,7 @@ register_fuzzy_client_call (struct rspamd_task *task, rspamd_inet_address_to_string_pretty (addr), errno, strerror (errno)); - rspamd_upstream_fail (selected, TRUE); + rspamd_upstream_fail (selected, TRUE, strerror (errno)); g_ptr_array_free (commands, TRUE); } else { /* Create session for a socket */ @@ -2921,7 +2925,7 @@ register_fuzzy_controller_call (struct rspamd_http_connection_entry *entry, rspamd_inet_address_to_string_pretty (addr), rule->name, strerror (errno)); - rspamd_upstream_fail (selected, TRUE); + rspamd_upstream_fail (selected, TRUE, strerror (errno)); } else { s = @@ -2975,7 +2979,7 @@ fuzzy_process_handler (struct rspamd_http_connection_entry *conn_ent, /* Prepare task */ task = rspamd_task_new (session->wrk, session->cfg, NULL, - session->lang_det, conn_ent->rt->event_loop); + session->lang_det, conn_ent->rt->event_loop, FALSE); task->cfg = ctx->cfg; saved = rspamd_mempool_alloc0 (session->pool, sizeof (gint)); err = rspamd_mempool_alloc0 (session->pool, sizeof (GError *)); @@ -3284,7 +3288,7 @@ fuzzy_check_send_lua_learn (struct fuzzy_rule *rule, if ((sock = rspamd_inet_address_connect (addr, SOCK_DGRAM, TRUE)) == -1) { - rspamd_upstream_fail (selected, TRUE); + rspamd_upstream_fail (selected, TRUE, strerror (errno)); } else { s = rspamd_mempool_alloc0 (task->task_pool, diff --git a/src/plugins/lua/antivirus.lua b/src/plugins/lua/antivirus.lua index 34b0c6947..5d7268b06 100644 --- a/src/plugins/lua/antivirus.lua +++ b/src/plugins/lua/antivirus.lua @@ -16,6 +16,7 @@ limitations under the License. local rspamd_logger = require "rspamd_logger" local lua_util = require "lua_util" +local lua_redis = require "lua_redis" local fun = require "fun" local lua_antivirus = require("lua_scanners").filter('antivirus') local common = require "lua_scanners/common" @@ -119,6 +120,12 @@ local function add_antivirus_rule(sym, opts) rule.patterns = common.create_regex_table(opts.patterns or {}) rule.patterns_fail = common.create_regex_table(opts.patterns_fail or {}) + lua_redis.register_prefix(rule.prefix .. '_*', N, + string.format('Antivirus cache for rule "%s"', + rule.type), { + type = 'string', + }) + if opts.whitelist then rule.whitelist = rspamd_config:add_hash_map(opts.whitelist) end @@ -142,7 +149,7 @@ end -- Registration local opts = rspamd_config:get_all_opt(N) if opts and type(opts) == 'table' then - redis_params = rspamd_parse_redis_server(N) + redis_params = lua_redis.parse_redis_server(N) local has_valid = false for k, m in pairs(opts) do if type(m) == 'table' then diff --git a/src/plugins/lua/bayes_expiry.lua b/src/plugins/lua/bayes_expiry.lua index e5eb471d4..84c11644d 100644 --- a/src/plugins/lua/bayes_expiry.lua +++ b/src/plugins/lua/bayes_expiry.lua @@ -259,23 +259,26 @@ local expiry_script = [[ 0,0,0,0,0,0,0,0,0,0,0 for _,key in ipairs(keys) do - local values = redis.call('HMGET', key, 'H', 'S') - local ham = tonumber(values[1]) or 0 - local spam = tonumber(values[2]) or 0 - local ttl = redis.call('TTL', key) - tokens[key] = { - ham, - spam, - ttl - } - local total = spam + ham - sum = sum + total - sum_squares = sum_squares + total * total - nelts = nelts + 1 - - for k,v in pairs({['ham']=ham, ['spam']=spam, ['total']=total}) do - if tonumber(v) > 19 then v = 20 end - occurr[k][v] = occurr[k][v] and occurr[k][v] + 1 or 1 + local t = redis.call('TYPE', key)["ok"] + if t == 'hash' then + local values = redis.call('HMGET', key, 'H', 'S') + local ham = tonumber(values[1]) or 0 + local spam = tonumber(values[2]) or 0 + local ttl = redis.call('TTL', key) + tokens[key] = { + ham, + spam, + ttl + } + local total = spam + ham + sum = sum + total + sum_squares = sum_squares + total * total + nelts = nelts + 1 + + for k,v in pairs({['ham']=ham, ['spam']=spam, ['total']=total}) do + if tonumber(v) > 19 then v = 20 end + occurr[k][v] = occurr[k][v] and occurr[k][v] + 1 or 1 + end end end diff --git a/src/plugins/lua/dmarc.lua b/src/plugins/lua/dmarc.lua index c2fc4d9bb..390abd41c 100644 --- a/src/plugins/lua/dmarc.lua +++ b/src/plugins/lua/dmarc.lua @@ -23,8 +23,7 @@ local rspamd_url = require "rspamd_url" local rspamd_util = require "rspamd_util" local rspamd_redis = require "lua_redis" local lua_util = require "lua_util" -local check_local = false -local check_authed = false +local auth_and_local_conf if confighelp then return @@ -45,6 +44,7 @@ local report_settings = { smtp_port = 25, retries = 2, from_name = 'Rspamd', + msgid_from = 'rspamd', } local report_template = [[From: "{= from_name =}" <{= from_addr =}> To: {= rcpt =} @@ -559,7 +559,7 @@ local function dmarc_callback(task) local hfromdom = ((from or E)[1] or E).domain local dmarc_domain local ip_addr = task:get_ip() - local dmarc_checks = task:get_mempool():get_variable('dmarc_checks', 'int') or 0 + local dmarc_checks = task:get_mempool():get_variable('dmarc_checks', 'double') or 0 local seen_invalid = false if dmarc_checks ~= 2 then @@ -567,8 +567,7 @@ local function dmarc_callback(task) return end - if ((not check_authed and task:get_user()) or - (not check_local and ip_addr and ip_addr:is_local())) then + if lua_util.is_skip_local_or_authed(task, auth_and_local_conf, ip_addr) then rspamd_logger.infox(task, "skip DMARC checks for local networks and authorized users") return end @@ -709,29 +708,14 @@ local function dmarc_callback(task) end -local function try_opts(where) - local ret = false - local opts = rspamd_config:get_all_opt(where) - if type(opts) == 'table' then - if type(opts['check_local']) == 'boolean' then - check_local = opts['check_local'] - ret = true - end - if type(opts['check_authed']) == 'boolean' then - check_authed = opts['check_authed'] - ret = true - end - end - - return ret -end - -if not try_opts(N) then try_opts('options') end - local opts = rspamd_config:get_all_opt('dmarc') if not opts or type(opts) ~= 'table' then return end + +auth_and_local_conf = lua_util.config_check_local_or_authed(rspamd_config, N, + false, false) + no_sampling_domains = rspamd_map_add(N, 'no_sampling_domains', 'map', 'Domains not to apply DMARC sampling to') no_reporting_domains = rspamd_map_add(N, 'no_reporting_domains', 'map', 'Domains not to apply DMARC reporting to') @@ -943,7 +927,7 @@ if opts['reporting'] == true then submitter = report_settings.domain, report_id = report_id, report_date = rspamd_util.time_to_string(rspamd_util.get_time()), - message_id = rspamd_util.random_hex(12) .. '@rspamd', + message_id = rspamd_util.random_hex(16) .. '@' .. report_settings.msgid_from, report_start = report_start, report_end = report_end }, true) diff --git a/src/plugins/lua/external_services.lua b/src/plugins/lua/external_services.lua index 0a9f39ad9..8e29accbe 100644 --- a/src/plugins/lua/external_services.lua +++ b/src/plugins/lua/external_services.lua @@ -17,6 +17,7 @@ limitations under the License. local rspamd_logger = require "rspamd_logger" local lua_util = require "lua_util" +local lua_redis = require "lua_redis" local fun = require "fun" local lua_scanners = require("lua_scanners").filter('scanner') local common = require "lua_scanners/common" @@ -142,6 +143,12 @@ local function add_scanner_rule(sym, opts) rule.redis_params = redis_params + lua_redis.register_prefix(rule.prefix .. '_*', N, + string.format('External services cache for rule "%s"', + rule.type), { + type = 'string', + }) + -- if any mime_part filter defined, do not scan all attachments if opts.mime_parts_filter_regex ~= nil or opts.mime_parts_filter_ext ~= nil then @@ -185,7 +192,7 @@ end -- Registration local opts = rspamd_config:get_all_opt(N) if opts and type(opts) == 'table' then - redis_params = rspamd_parse_redis_server(N) + redis_params = lua_redis.parse_redis_server(N) local has_valid = false for k, m in pairs(opts) do if type(m) == 'table' and m.servers then diff --git a/src/plugins/lua/force_actions.lua b/src/plugins/lua/force_actions.lua index 4fa4fd7fb..fb863a23b 100644 --- a/src/plugins/lua/force_actions.lua +++ b/src/plugins/lua/force_actions.lua @@ -30,7 +30,7 @@ local rspamd_cryptobox_hash = require "rspamd_cryptobox_hash" local rspamd_expression = require "rspamd_expression" local rspamd_logger = require "rspamd_logger" -local function gen_cb(expr, act, pool, message, subject, raction, honor, limit) +local function gen_cb(expr, act, pool, message, subject, raction, honor, limit, least) local function parse_atom(str) local atom = table.concat(fun.totable(fun.take_while(function(c) @@ -78,10 +78,14 @@ local function gen_cb(expr, act, pool, message, subject, raction, honor, limit) if subject then task:set_metric_subject(subject) end + + local flags = "" + if least then flags = "least" end + if type(message) == 'string' then - task:set_pre_result(act, message, N) + task:set_pre_result(act, message, N, nil, nil, flags) else - task:set_pre_result(act, nil, N) + task:set_pre_result(act, nil, N, nil, nil, flags) end return true, act end @@ -138,10 +142,11 @@ local function configure_module() local subject = sett.subject local message = sett.message local lim = sett.limit or 0 + local least = sett.least or false local raction = lua_util.list_to_hash(sett.require_action) local honor = lua_util.list_to_hash(sett.honor_action) local cb, atoms = gen_cb(expr, action, rspamd_config:get_mempool(), - message, subject, raction, honor, lim) + message, subject, raction, honor, lim, least) if cb and atoms then local t = {} if (raction or honor) then diff --git a/src/plugins/lua/greylist.lua b/src/plugins/lua/greylist.lua index 3c6335ea1..671b2be69 100644 --- a/src/plugins/lua/greylist.lua +++ b/src/plugins/lua/greylist.lua @@ -464,6 +464,14 @@ if opts then rspamd_logger.infox(rspamd_config, 'no servers are specified, disabling module') rspamd_lua_utils.disable_module(N, "redis") else + lua_redis.register_prefix(settings.key_prefix .. 'b[a-z0-9]{20}', N, + 'Greylisting elements (body hashes)"', { + type = 'string', + }) + lua_redis.register_prefix(settings.key_prefix .. 'm[a-z0-9]{20}', N, + 'Greylisting elements (meta hashes)"', { + type = 'string', + }) rspamd_config:register_symbol({ name = 'GREYLIST_SAVE', type = 'postfilter', diff --git a/src/plugins/lua/history_redis.lua b/src/plugins/lua/history_redis.lua index 8a23a75f8..cc26a281e 100644 --- a/src/plugins/lua/history_redis.lua +++ b/src/plugins/lua/history_redis.lua @@ -251,7 +251,9 @@ if opts then priority = 150 }) lua_redis.register_prefix(settings.key_prefix .. hostname, N, - "Redis history") + "Redis history", { + type = 'list', + }) rspamd_plugins['history'] = { handler = handle_history_request } diff --git a/src/plugins/lua/maps_stats.lua b/src/plugins/lua/maps_stats.lua index 725629a77..46a769f34 100644 --- a/src/plugins/lua/maps_stats.lua +++ b/src/plugins/lua/maps_stats.lua @@ -105,7 +105,17 @@ if opts then end redis_params = lua_redis.parse_redis_server(N, opts) - +-- XXX, this is a poor approach as not all maps are defined here... +local tmaps = rspamd_config:get_maps() +for _,m in ipairs(tmaps) do + if m:get_uri() ~= 'static' then + lua_redis.register_prefix(settings.prefix .. m:get_uri(), N, + 'Maps stats data', { + type = 'zlist', + persistent = true, + }) + end +end if redis_params then rspamd_config:add_on_load(function (_, ev_base, worker) diff --git a/src/plugins/lua/multimap.lua b/src/plugins/lua/multimap.lua index 13316583c..3a53d24cb 100644 --- a/src/plugins/lua/multimap.lua +++ b/src/plugins/lua/multimap.lua @@ -590,7 +590,12 @@ local function multimap_callback(task, rule) if opt then - task:insert_result(forced, symbol, score, opt) + if type(opt) == 'table' then + task:insert_result(forced, symbol, score, fun.totable(fun.map(tostring, opt))) + else + task:insert_result(forced, symbol, score, tostring(opt)) + end + else task:insert_result(forced, symbol, score) end @@ -861,6 +866,12 @@ local function multimap_callback(task, rule) end end end, + user = function() + local user = task:get_user() + if user then + match_rule(rule, user) + end + end, filename = function() local parts = task:get_parts() @@ -1032,6 +1043,7 @@ local function add_multimap_rule(key, newrule) symbol_options = true, filename = true, url = true, + user = true, content = true, hostname = true, asn = true, diff --git a/src/plugins/lua/mx_check.lua b/src/plugins/lua/mx_check.lua index 61d1cd1b8..d67524c21 100644 --- a/src/plugins/lua/mx_check.lua +++ b/src/plugins/lua/mx_check.lua @@ -23,6 +23,7 @@ local rspamd_logger = require "rspamd_logger" local rspamd_tcp = require "rspamd_tcp" local rspamd_util = require "rspamd_util" local lua_util = require "lua_util" +local lua_redis = require "lua_redis" local N = "mx_check" local fun = require "fun" @@ -267,15 +268,18 @@ if not (opts and type(opts) == 'table') then return end if opts then - redis_params = rspamd_parse_redis_server('mx_check') + redis_params = lua_redis.parse_redis_server('mx_check') if not redis_params then rspamd_logger.errx(rspamd_config, 'no redis servers are specified, disabling module') lua_util.disable_module(N, "redis") return end - for k,v in pairs(opts) do - settings[k] = v - end + + settings = lua_util.override_defaults(settings, opts) + lua_redis.register_prefix(settings.key_prefix .. '*', N, + 'MX check cache', { + type = 'string', + }) local id = rspamd_config:register_symbol({ name = settings.symbol_bad_mx, diff --git a/src/plugins/lua/neural.lua b/src/plugins/lua/neural.lua index e64751ef2..2d1d33ec5 100644 --- a/src/plugins/lua/neural.lua +++ b/src/plugins/lua/neural.lua @@ -1236,7 +1236,29 @@ local function process_rules_settings() lua_redis.register_prefix(selt.prefix, N, string.format('NN prefix for rule "%s"; settings id "%s"', - rule.prefix, selt.name), {persistent = true}) + rule.prefix, selt.name), { + persistent = true, + type = 'zlist', + }) + -- Versions + lua_redis.register_prefix(selt.prefix .. '_\\d+', N, + string.format('NN storage for rule "%s"; settings id "%s"', + rule.prefix, selt.name), { + persistent = true, + type = 'hash', + }) + lua_redis.register_prefix(selt.prefix .. '_\\d+_spam', N, + string.format('NN learning set (spam) for rule "%s"; settings id "%s"', + rule.prefix, selt.name), { + persistent = true, + type = 'list', + }) + lua_redis.register_prefix(selt.prefix .. '_\\d+_ham', N, + string.format('NN learning set (spam) for rule "%s"; settings id "%s"', + rule.prefix, selt.name), { + persistent = true, + type = 'list', + }) end for _,rule in pairs(settings.rules) do diff --git a/src/plugins/lua/once_received.lua b/src/plugins/lua/once_received.lua index c8c47ddb1..4f1e67c72 100644 --- a/src/plugins/lua/once_received.lua +++ b/src/plugins/lua/once_received.lua @@ -65,8 +65,11 @@ local function check_quantity_received (task) end task:insert_result(symbol_rdns, 1) else - rspamd_logger.infox(task, 'SMTP resolver failed to resolve: %1 is %2', + rspamd_logger.infox(task, 'source hostname has not been passed to Rspamd from MTA, ' .. + 'but we could resolve source IP address PTR %s as "%s"', to_resolve, results[1]) + task:set_hostname(results[1]) + if good_hosts then for _,gh in ipairs(good_hosts) do if string.find(results[1], gh) then diff --git a/src/plugins/lua/p0f.lua b/src/plugins/lua/p0f.lua index f7fed7886..3242e73b0 100644 --- a/src/plugins/lua/p0f.lua +++ b/src/plugins/lua/p0f.lua @@ -78,6 +78,11 @@ rule = p0f.configure(opts) if rule then rule.redis_params = lua_redis.parse_redis_server(N) + lua_redis.register_prefix(rule.prefix .. '*', N, + 'P0f check cache', { + type = 'string', + }) + local id = rspamd_config:register_symbol({ name = 'P0F_CHECK', type = 'prefilter,nostat', diff --git a/src/plugins/lua/rbl.lua b/src/plugins/lua/rbl.lua index 521e63708..f1bd88047 100644 --- a/src/plugins/lua/rbl.lua +++ b/src/plugins/lua/rbl.lua @@ -324,16 +324,21 @@ local function gen_rbl_callback(rule) if requests_table[req] then -- Duplicate request - if forced and not requests_table[req].forced then - requests_table[req].forced = true + local nreq = requests_table[req] + if forced and not nreq.forced then + nreq.forced = true end + + return true,nreq -- Duplicate else + local nreq + local resolve_ip = rule.resolve_ip and not is_ip if rule.process_script then local processed = rule.process_script(req, rule.rbl, task, resolve_ip) if processed then - local nreq = { + nreq = { forced = forced, n = processed, orig = req_str, @@ -356,7 +361,7 @@ local function gen_rbl_callback(rule) to_resolve = orign end - local nreq = { + nreq = { forced = forced, n = to_resolve, orig = req_str, @@ -365,7 +370,7 @@ local function gen_rbl_callback(rule) } requests_table[req] = nreq end - + return false, nreq end end @@ -712,15 +717,19 @@ local function gen_rbl_callback(rule) local nresolved = 0 -- This is called when doing resolve_ip phase... - local function gen_rbl_ip_dns_callback(_) + local function gen_rbl_ip_dns_callback(orig_resolve_table_elt) return function(_, _, results, err) if not err then for _,dns_res in ipairs(results) do -- Check if we have rspamd{ip} userdata if type(dns_res) == 'userdata' then -- Add result as an actual RBL request - add_dns_request(task, dns_res, false, true, - resolved_req) + local dup,nreq = add_dns_request(task, dns_res, false, true, + resolved_req, orig_resolve_table_elt.what) + -- Add original name + if not dup then + nreq.orig = nreq.orig .. ':' .. orig_resolve_table_elt.n + end end end end @@ -759,7 +768,7 @@ local function gen_rbl_callback(rule) if r:resolve_a({ task = task, name = req.n, - callback = gen_rbl_ip_dns_callback(req.orig), + callback = gen_rbl_ip_dns_callback(req), forced = req.forced }) then nresolved = nresolved + 1 diff --git a/src/plugins/lua/replies.lua b/src/plugins/lua/replies.lua index dcba6cc81..6c3459b4f 100644 --- a/src/plugins/lua/replies.lua +++ b/src/plugins/lua/replies.lua @@ -159,7 +159,7 @@ local function replies_set(task) true, -- is write redis_set_cb, --callback 'SETEX', -- command - {key, tostring(settings['expire']), value:lower()} -- arguments + {key, tostring(math.floor(settings['expire'])), value:lower()} -- arguments ) if not ret then rspamd_logger.errx(task, "redis request wasn't scheduled") diff --git a/src/plugins/lua/spf.lua b/src/plugins/lua/spf.lua new file mode 100644 index 000000000..10daa0d2b --- /dev/null +++ b/src/plugins/lua/spf.lua @@ -0,0 +1,243 @@ +--[[ +Copyright (c) 2019, Vsevolod Stakhov <vsevolod@highsecure.ru> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +local N = "spf" +local lua_util = require "lua_util" +local rspamd_spf = require "rspamd_spf" +local bit = require "bit" +local rspamd_logger = require "rspamd_logger" + +if confighelp then + rspamd_config:add_example(nil, N, + 'Performs SPF checks', + [[ +spf { + # Enable module + enabled = true + # Number of elements in the cache of parsed SPF records + spf_cache_size = 2048; + # Default max expire for an element in this cache + spf_cache_expire = 1d; + # Whitelist IPs from checks + whitelist = "/path/to/some/file"; + # Maximum number of recursive DNS subrequests (e.g. includes chanin length) + max_dns_nesting = 10; + # Maximum count of DNS requests per record + max_dns_requests = 30; + # Minimum TTL enforced for all elements in SPF records + min_cache_ttl = 5m; + # Disable all IPv6 lookups + disable_ipv6 = false; + # Use IP address from a received header produced by this relay (using by attribute) + external_relay = "192.168.1.1"; +} + ]]) + return +end + +local symbols = { + fail = "R_SPF_FAIL", + softfail = "R_SPF_SOFTFAIL", + neutral = "R_SPF_NEUTRAL", + allow = "R_SPF_ALLOW", + dnsfail = "R_SPF_DNSFAIL", + permfail = "R_SPF_PERMFAIL", + na = "R_SPF_NA", +} + +local default_config = { + spf_cache_size = 2048, + max_dns_nesting = 10, + max_dns_requests = 30, + whitelist = nil, + min_cache_ttl = 60 * 5, + disable_ipv6 = false, + symbols = symbols, + external_relay = nil, +} + +local local_config = rspamd_config:get_all_opt('spf') +local auth_and_local_conf = lua_util.config_check_local_or_authed(rspamd_config, N, + false, false) + +if local_config then + local_config = lua_util.override_defaults(default_config, local_config) +else + local_config = default_config +end + +local function spf_check_callback(task) + + local ip + + if local_config.external_relay then + -- Search received headers to get header produced by an external relay + local rh = task:get_received_headers() or {} + local found = false + + for i,hdr in ipairs(rh) do + if hdr.real_ip and hdr.real_ip == local_config.external_relay then + -- We can use the next header as a source of IP address + if rh[i + 1] then + local nhdr = rh[i + 1] + lua_util.debugm(N, task, 'found external relay %s at received header number %s -> %s', + local_config.external_relay, i, nhdr.real_ip) + + if nhdr.real_ip then + ip = nhdr.real_ip + found = true + end + end + + break + end + end + if not found then + rspamd_logger.warnx(task, "cannot find external relay with IP %s", + local_config.external_relay) + ip = task:get_from_ip() + end + else + ip = task:get_from_ip() + end + + local function flag_to_symbol(fl) + if bit.band(fl, rspamd_spf.flags.temp_fail) ~= 0 then + return local_config.symbols.dnsfail + elseif bit.band(fl, rspamd_spf.flags.perm_fail) ~= 0 then + return local_config.symbols.permfail + elseif bit.band(fl, rspamd_spf.flags.na) ~= 0 then + return local_config.symbols.na + end + + return 'SPF_UNKNOWN' + end + + local function policy_decode(res) + if res == rspamd_spf.policy.fail then + return local_config.symbols.fail,'-' + elseif res == rspamd_spf.policy.pass then + return local_config.symbols.allow,'+' + elseif res == rspamd_spf.policy.soft_fail then + return local_config.symbols.softfail,'~' + elseif res == rspamd_spf.policy.neutral then + return local_config.symbols.neutral,'?' + end + + return 'SPF_UNKNOWN','?' + end + + local function spf_resolved_cb(record, flags, err) + lua_util.debugm(N, task, 'got spf results: %s flags, %s err', + flags, err) + + if record then + local result, flag_or_policy, error_or_addr = record:check_ip(ip) + + lua_util.debugm(N, task, + 'checked ip %s: result=%s, flag_or_policy=%s, error_or_addr=%s', + ip, flags, err, error_or_addr) + + if result then + local sym,code = policy_decode(flag_or_policy) + local opt = string.format('%s%s', code, error_or_addr.str or '???') + if bit.band(flags, rspamd_spf.flags.cached) ~= 0 then + opt = opt .. ':c' + rspamd_logger.infox(task, + "use cached record for %s (0x%s) in LRU cache for %s seconds", + record:get_domain(), + record:get_digest(), + record:get_ttl() - math.floor(task:get_timeval(true) - + record:get_timestamp())); + end + task:insert_result(sym, 1.0, opt) + else + local sym = flag_to_symbol(flag_or_policy) + task:insert_result(sym, 1.0, error_or_addr) + end + else + local sym = flag_to_symbol(flags) + task:insert_result(sym, 1.0, err) + end + end + + if ip then + if local_config.whitelist and ip and local_config.whitelist:get_key(ip) then + rspamd_logger.infox(task, 'whitelisted SPF checks from %s', + tostring(ip)) + return + end + + if lua_util.is_skip_local_or_authed(task, auth_and_local_conf, ip) then + rspamd_logger.infox(task, 'skip SPF checks for local networks and authorized users') + return + end + + rspamd_spf.resolve(task, spf_resolved_cb) + else + lua_util.debugm(N, task, "spf checks are not possible as no source IP address is defined") + end + + -- FIXME: we actually need to set this variable when we really checked SPF + -- However, the old C module has set it all the times + -- Hence, we follow the same rule for now. It should be better designed at some day + local mpool = task:get_mempool() + local dmarc_checks = mpool:get_variable('dmarc_checks', 'double') or 0 + dmarc_checks = dmarc_checks + 1 + mpool:set_variable('dmarc_checks', dmarc_checks) +end + +-- Register all symbols and init rspamd_spf library +rspamd_spf.config(local_config) +local sym_id = rspamd_config:register_symbol{ + name = 'SPF_CHECK', + type = 'callback', + flags = 'fine,empty', + groups = {'policies','spf'}, + score = 0.0, + callback = spf_check_callback +} + +if local_config.whitelist then + local lua_maps = require "lua_maps" + + local_config.whitelist = lua_maps.map_add_from_ucl(local_config.whitelist, + "radix", "SPF whitelist map") +end + +if local_config.external_relay then + local rspamd_ip = require "rspamd_ip" + local ip = rspamd_ip.from_string(local_config.external_relay) + + if not ip or not ip:is_valid() then + rspamd_logger.errx(rspamd_config, "invalid external relay IP: %s", + local_config.external_relay) + local_config.external_relay = nil + else + local_config.external_relay = ip + end +end + +for _,sym in pairs(local_config.symbols) do + rspamd_config:register_symbol{ + name = sym, + type = 'virtual', + parent = sym_id, + groups = {'policies', 'spf'}, + } +end + + diff --git a/src/plugins/lua/url_redirector.lua b/src/plugins/lua/url_redirector.lua index 4952dc25f..ba7d77649 100644 --- a/src/plugins/lua/url_redirector.lua +++ b/src/plugins/lua/url_redirector.lua @@ -343,6 +343,16 @@ if opts then settings.redirector_hosts_map = lua_maps.map_add_from_ucl(settings.redirector_hosts_map, 'set', 'Redirectors definitions') + lua_redis.register_prefix(settings.key_prefix .. '[a-z0-9]{32}', N, + 'URL redirector hashes', { + type = 'string', + }) + if settings.top_urls_key then + lua_redis.register_prefix(settings.top_urls_key, N, + 'URL redirector top urls', { + type = 'zlist', + }) + end local id = rspamd_config:register_symbol{ name = 'URL_REDIRECTOR_CHECK', type = 'callback,prefilter', diff --git a/src/plugins/lua/whitelist.lua b/src/plugins/lua/whitelist.lua index 11c01134b..6bd819e88 100644 --- a/src/plugins/lua/whitelist.lua +++ b/src/plugins/lua/whitelist.lua @@ -152,7 +152,7 @@ local function whitelist_cb(symbol, rule, task) local tld = rspamd_util.get_tld(helo) if tld then - find_domain(tld) + find_domain(tld, 'spf') end end end diff --git a/src/plugins/spf.c b/src/plugins/spf.c deleted file mode 100644 index 3c02eafbe..000000000 --- a/src/plugins/spf.c +++ /dev/null @@ -1,722 +0,0 @@ -/*- - * Copyright 2016 Vsevolod Stakhov - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/***MODULE:spf - * rspamd module that checks spf records of incoming email - * - * Allowed options: - * - symbol_allow (string): symbol to insert (default: 'R_SPF_ALLOW') - * - symbol_fail (string): symbol to insert (default: 'R_SPF_FAIL') - * - symbol_softfail (string): symbol to insert (default: 'R_SPF_SOFTFAIL') - * - symbol_na (string): symbol to insert (default: 'R_SPF_NA') - * - symbol_dnsfail (string): symbol to insert (default: 'R_SPF_DNSFAIL') - * - symbol_permfail (string): symbol to insert (default: 'R_SPF_PERMFAIL') - * - whitelist (map): map of whitelisted networks - */ - - -#include "config.h" -#include "libmime/message.h" -#include "libserver/spf.h" -#include "libutil/hash.h" -#include "libutil/map.h" -#include "libutil/map_helpers.h" -#include "rspamd.h" -#include "libserver/mempool_vars_internal.h" - -#define DEFAULT_SYMBOL_FAIL "R_SPF_FAIL" -#define DEFAULT_SYMBOL_SOFTFAIL "R_SPF_SOFTFAIL" -#define DEFAULT_SYMBOL_NEUTRAL "R_SPF_NEUTRAL" -#define DEFAULT_SYMBOL_ALLOW "R_SPF_ALLOW" -#define DEFAULT_SYMBOL_DNSFAIL "R_SPF_DNSFAIL" -#define DEFAULT_SYMBOL_PERMFAIL "R_SPF_PERMFAIL" -#define DEFAULT_SYMBOL_NA "R_SPF_NA" -#define DEFAULT_CACHE_SIZE 2048 - -static const gchar *M = "rspamd spf plugin"; - -struct spf_ctx { - struct module_ctx ctx; - const gchar *symbol_fail; - const gchar *symbol_softfail; - const gchar *symbol_neutral; - const gchar *symbol_allow; - const gchar *symbol_dnsfail; - const gchar *symbol_na; - const gchar *symbol_permfail; - - struct rspamd_radix_map_helper *whitelist_ip; - rspamd_lru_hash_t *spf_hash; - - gboolean check_local; - gboolean check_authed; -}; - -static void spf_symbol_callback (struct rspamd_task *task, - struct rspamd_symcache_item *item, - void *unused); - -/* Initialization */ -gint spf_module_init (struct rspamd_config *cfg, struct module_ctx **ctx); -gint spf_module_config (struct rspamd_config *cfg); -gint spf_module_reconfig (struct rspamd_config *cfg); - -module_t spf_module = { - "spf", - spf_module_init, - spf_module_config, - spf_module_reconfig, - NULL, - RSPAMD_MODULE_VER, - (guint)-1, -}; - -static inline struct spf_ctx * -spf_get_context (struct rspamd_config *cfg) -{ - return (struct spf_ctx *)g_ptr_array_index (cfg->c_modules, - spf_module.ctx_offset); -} - - -gint -spf_module_init (struct rspamd_config *cfg, struct module_ctx **ctx) -{ - struct spf_ctx *spf_module_ctx; - - spf_module_ctx = rspamd_mempool_alloc0 (cfg->cfg_pool, - sizeof (*spf_module_ctx)); - *ctx = (struct module_ctx *)spf_module_ctx; - - rspamd_rcl_add_doc_by_path (cfg, - NULL, - "SPF check plugin", - "spf", - UCL_OBJECT, - NULL, - 0, - NULL, - 0); - - rspamd_rcl_add_doc_by_path (cfg, - "spf", - "Map of IP addresses that should be excluded from SPF checks (in addition to `local_networks`)", - "whitelist", - UCL_STRING, - NULL, - 0, - NULL, - 0); - rspamd_rcl_add_doc_by_path (cfg, - "spf", - "Symbol that is added if SPF check is successful", - "symbol_allow", - UCL_STRING, - NULL, - 0, - NULL, - 0); - rspamd_rcl_add_doc_by_path (cfg, - "spf", - "Symbol that is added if SPF policy is set to 'deny'", - "symbol_fail", - UCL_STRING, - NULL, - 0, - NULL, - 0); - rspamd_rcl_add_doc_by_path (cfg, - "spf", - "Symbol that is added if SPF policy is set to 'undefined'", - "symbol_softfail", - UCL_STRING, - NULL, - 0, - NULL, - 0); - rspamd_rcl_add_doc_by_path (cfg, - "spf", - "Symbol that is added if SPF policy is set to 'neutral'", - "symbol_neutral", - UCL_STRING, - NULL, - 0, - NULL, - 0); - rspamd_rcl_add_doc_by_path (cfg, - "spf", - "Symbol that is added if SPF policy is failed due to DNS failure", - "symbol_dnsfail", - UCL_STRING, - NULL, - 0, - NULL, - 0); - rspamd_rcl_add_doc_by_path (cfg, - "spf", - "Symbol that is added if no SPF policy is found", - "symbol_na", - UCL_STRING, - NULL, - 0, - NULL, - 0); - rspamd_rcl_add_doc_by_path (cfg, - "spf", - "Symbol that is added if SPF policy is invalid", - "symbol_permfail", - UCL_STRING, - NULL, - 0, - NULL, - 0); - rspamd_rcl_add_doc_by_path (cfg, - "spf", - "Size of SPF parsed records cache", - "spf_cache_size", - UCL_INT, - NULL, - 0, - NULL, - 0); - - rspamd_rcl_add_doc_by_path (cfg, - "spf", - "Minimum cached records TTL, 0 to disable (default: 5min)", - "min_cache_ttl", - UCL_INT, - NULL, - RSPAMD_CL_FLAG_UINT, - NULL, - 0); - rspamd_rcl_add_doc_by_path (cfg, - "spf", - "Maximum number of nested requests (default: " G_STRINGIFY(SPF_MAX_NESTING) ")", - "max_dns_nesting", - UCL_INT, - NULL, - RSPAMD_CL_FLAG_UINT, - NULL, - 0); - rspamd_rcl_add_doc_by_path (cfg, - "spf", - "Maximum number of dns requests to resolve SPF (default: " G_STRINGIFY(SPF_MAX_DNS_REQUESTS) ")", - "max_dns_requests", - UCL_INT, - NULL, - RSPAMD_CL_FLAG_UINT, - NULL, - 0); - rspamd_rcl_add_doc_by_path (cfg, - "spf", - "Disable ipv6 resolving when doing SPF resolution", - "disable_ipv6", - UCL_BOOLEAN, - NULL, - 0, - NULL, - 0); - - return 0; -} - - -gint -spf_module_config (struct rspamd_config *cfg) -{ - const ucl_object_t *value; - gint res = TRUE, cb_id; - guint cache_size; - struct spf_ctx *spf_module_ctx = spf_get_context (cfg); - - if (!rspamd_config_is_module_enabled (cfg, "spf")) { - return TRUE; - } - - spf_module_ctx->whitelist_ip = NULL; - - value = rspamd_config_get_module_opt (cfg, "spf", "check_local"); - - if (value == NULL) { - rspamd_config_get_module_opt (cfg, "options", "check_local"); - } - - if (value != NULL) { - spf_module_ctx->check_local = ucl_obj_toboolean (value); - } - else { - spf_module_ctx->check_local = FALSE; - } - - value = rspamd_config_get_module_opt (cfg, "spf", "check_authed"); - - if (value == NULL) { - rspamd_config_get_module_opt (cfg, "options", "check_authed"); - } - - if (value != NULL) { - spf_module_ctx->check_authed = ucl_obj_toboolean (value); - } - else { - spf_module_ctx->check_authed = FALSE; - } - if ((value = - rspamd_config_get_module_opt (cfg, "spf", "symbol_fail")) != NULL) { - spf_module_ctx->symbol_fail = ucl_obj_tostring (value); - } - else { - spf_module_ctx->symbol_fail = DEFAULT_SYMBOL_FAIL; - } - if ((value = - rspamd_config_get_module_opt (cfg, "spf", "symbol_softfail")) != NULL) { - spf_module_ctx->symbol_softfail = ucl_obj_tostring (value); - } - else { - spf_module_ctx->symbol_softfail = DEFAULT_SYMBOL_SOFTFAIL; - } - if ((value = - rspamd_config_get_module_opt (cfg, "spf", "symbol_neutral")) != NULL) { - spf_module_ctx->symbol_neutral = ucl_obj_tostring (value); - } - else { - spf_module_ctx->symbol_neutral = DEFAULT_SYMBOL_NEUTRAL; - } - if ((value = - rspamd_config_get_module_opt (cfg, "spf", "symbol_allow")) != NULL) { - spf_module_ctx->symbol_allow = ucl_obj_tostring (value); - } - else { - spf_module_ctx->symbol_allow = DEFAULT_SYMBOL_ALLOW; - } - if ((value = - rspamd_config_get_module_opt (cfg, "spf", "symbol_dnsfail")) != NULL) { - spf_module_ctx->symbol_dnsfail = ucl_obj_tostring (value); - } - else { - spf_module_ctx->symbol_dnsfail = DEFAULT_SYMBOL_DNSFAIL; - } - if ((value = - rspamd_config_get_module_opt (cfg, "spf", "symbol_na")) != NULL) { - spf_module_ctx->symbol_na = ucl_obj_tostring (value); - } - else { - spf_module_ctx->symbol_na = DEFAULT_SYMBOL_NA; - } - if ((value = - rspamd_config_get_module_opt (cfg, "spf", "symbol_permfail")) != NULL) { - spf_module_ctx->symbol_permfail = ucl_obj_tostring (value); - } - else { - spf_module_ctx->symbol_permfail = DEFAULT_SYMBOL_PERMFAIL; - } - if ((value = - rspamd_config_get_module_opt (cfg, "spf", "spf_cache_size")) != NULL) { - cache_size = ucl_obj_toint (value); - } - else { - cache_size = DEFAULT_CACHE_SIZE; - } - - spf_library_config (ucl_obj_get_key (cfg->rcl_obj, "spf")); - - if ((value = - rspamd_config_get_module_opt (cfg, "spf", "whitelist")) != NULL) { - - rspamd_config_radix_from_ucl (cfg, value, "SPF whitelist", - &spf_module_ctx->whitelist_ip, NULL); - } - - cb_id = rspamd_symcache_add_symbol (cfg->cache, - "SPF_CHECK", - 0, - spf_symbol_callback, - NULL, - SYMBOL_TYPE_CALLBACK | SYMBOL_TYPE_FINE | SYMBOL_TYPE_EMPTY, -1); - rspamd_config_add_symbol (cfg, - "SPF_CHECK", - 0.0, - "SPF check callback", - "policies", - RSPAMD_SYMBOL_FLAG_IGNORE, - 1, - 1); - rspamd_config_add_symbol_group (cfg, "SPF_CHECK", "spf"); - - rspamd_symcache_add_symbol (cfg->cache, - spf_module_ctx->symbol_fail, 0, - NULL, NULL, - SYMBOL_TYPE_VIRTUAL, - cb_id); - rspamd_symcache_add_symbol (cfg->cache, - spf_module_ctx->symbol_softfail, 0, - NULL, NULL, - SYMBOL_TYPE_VIRTUAL, - cb_id); - rspamd_symcache_add_symbol (cfg->cache, - spf_module_ctx->symbol_permfail, 0, - NULL, NULL, - SYMBOL_TYPE_VIRTUAL, - cb_id); - rspamd_symcache_add_symbol (cfg->cache, - spf_module_ctx->symbol_na, 0, - NULL, NULL, - SYMBOL_TYPE_VIRTUAL, - cb_id); - rspamd_symcache_add_symbol (cfg->cache, - spf_module_ctx->symbol_neutral, 0, - NULL, NULL, - SYMBOL_TYPE_VIRTUAL, - cb_id); - rspamd_symcache_add_symbol (cfg->cache, - spf_module_ctx->symbol_allow, 0, - NULL, NULL, - SYMBOL_TYPE_VIRTUAL, - cb_id); - rspamd_symcache_add_symbol (cfg->cache, - spf_module_ctx->symbol_dnsfail, 0, - NULL, NULL, - SYMBOL_TYPE_VIRTUAL, - cb_id); - - if (cache_size > 0) { - spf_module_ctx->spf_hash = rspamd_lru_hash_new ( - cache_size, - NULL, - (GDestroyNotify) spf_record_unref); - rspamd_mempool_add_destructor (cfg->cfg_pool, - (rspamd_mempool_destruct_t)rspamd_lru_hash_destroy, - spf_module_ctx->spf_hash); - } - - rspamd_mempool_add_destructor (cfg->cfg_pool, - (rspamd_mempool_destruct_t)rspamd_map_helper_destroy_radix, - spf_module_ctx->whitelist_ip); - - msg_info_config ("init internal spf module"); - - return res; -} - -gint -spf_module_reconfig (struct rspamd_config *cfg) -{ - return spf_module_config (cfg); -} - -static gboolean -spf_check_element (struct spf_resolved *rec, struct spf_addr *addr, - struct rspamd_task *task, gboolean cached) -{ - gboolean res = FALSE; - const guint8 *s, *d; - gchar *spf_result; - guint af, mask, bmask, addrlen; - const gchar *spf_message, *spf_symbol; - struct spf_ctx *spf_module_ctx = spf_get_context (task->cfg); - - if (task->from_addr == NULL) { - return FALSE; - } - - if (addr->flags & RSPAMD_SPF_FLAG_TEMPFAIL) { - /* Ignore failed addresses */ - return FALSE; - } - - af = rspamd_inet_address_get_af (task->from_addr); - /* Basic comparing algorithm */ - if (((addr->flags & RSPAMD_SPF_FLAG_IPV6) && af == AF_INET6) || - ((addr->flags & RSPAMD_SPF_FLAG_IPV4) && af == AF_INET)) { - d = rspamd_inet_address_get_hash_key (task->from_addr, &addrlen); - - if (af == AF_INET6) { - s = (const guint8 *)addr->addr6; - mask = addr->m.dual.mask_v6; - } - else { - s = (const guint8 *)addr->addr4; - mask = addr->m.dual.mask_v4; - } - - /* Compare the first bytes */ - bmask = mask / CHAR_BIT; - if (mask > addrlen * CHAR_BIT) { - msg_info_task ("bad mask length: %d", mask); - } - else if (memcmp (s, d, bmask) == 0) { - if (bmask * CHAR_BIT < mask) { - /* Compare the remaining bits */ - s += bmask; - d += bmask; - mask = (0xff << (CHAR_BIT - (mask - bmask * 8))) & 0xff; - - if ((*s & mask) == (*d & mask)) { - res = TRUE; - } - } - else { - res = TRUE; - } - } - } - else { - if (addr->flags & RSPAMD_SPF_FLAG_ANY) { - res = TRUE; - } - else { - res = FALSE; - } - } - - if (res) { - spf_result = rspamd_mempool_alloc (task->task_pool, - strlen (addr->spf_string) + 5); - - switch (addr->mech) { - case SPF_FAIL: - spf_symbol = spf_module_ctx->symbol_fail; - spf_result[0] = '-'; - spf_message = "(SPF): spf fail"; - if (addr->flags & RSPAMD_SPF_FLAG_ANY) { - if (rec->flags & RSPAMD_SPF_RESOLVED_PERM_FAILED) { - msg_info_task ("do not apply SPF failed policy, as we have " - "some addresses unresolved"); - spf_symbol = spf_module_ctx->symbol_permfail; - } - else if (rec->flags & RSPAMD_SPF_RESOLVED_TEMP_FAILED) { - msg_info_task ("do not apply SPF failed policy, as we have " - "some addresses unresolved"); - spf_symbol = spf_module_ctx->symbol_dnsfail; - spf_message = "(SPF): spf DNS fail"; - } - } - break; - case SPF_SOFT_FAIL: - spf_symbol = spf_module_ctx->symbol_softfail; - spf_message = "(SPF): spf softfail"; - spf_result[0] = '~'; - - if (addr->flags & RSPAMD_SPF_FLAG_ANY) { - if (rec->flags & RSPAMD_SPF_RESOLVED_PERM_FAILED) { - msg_info_task ("do not apply SPF failed policy, as we have " - "some addresses unresolved"); - spf_symbol = spf_module_ctx->symbol_permfail; - } - else if (rec->flags & RSPAMD_SPF_RESOLVED_TEMP_FAILED) { - msg_info_task ("do not apply SPF failed policy, as we have " - "some addresses unresolved"); - spf_symbol = spf_module_ctx->symbol_dnsfail; - spf_message = "(SPF): spf DNS fail"; - } - } - break; - case SPF_NEUTRAL: - spf_symbol = spf_module_ctx->symbol_neutral; - spf_message = "(SPF): spf neutral"; - spf_result[0] = '?'; - break; - default: - spf_symbol = spf_module_ctx->symbol_allow; - spf_message = "(SPF): spf allow"; - spf_result[0] = '+'; - break; - } - - gint r = rspamd_strlcpy (spf_result + 1, addr->spf_string, - strlen (addr->spf_string) + 1); - - if (cached) { - rspamd_strlcpy (spf_result + r + 1, ":c", 3); - } - - rspamd_task_insert_result (task, - spf_symbol, - 1, - spf_result); - ucl_object_insert_key (task->messages, - ucl_object_fromstring (spf_message), "spf", 0, - false); - - return TRUE; - } - - return FALSE; -} - -static void -spf_check_list (struct spf_resolved *rec, struct rspamd_task *task, gboolean cached) -{ - guint i; - struct spf_addr *addr; - struct spf_ctx *spf_module_ctx = spf_get_context (task->cfg); - - if (cached) { - msg_info_task ("use cached record for %s (0x%xuL) in LRU cache for %d seconds, " - "%d/%d elements in the cache", - rec->domain, - rec->digest, - rec->ttl - (guint)(task->task_timestamp - rec->timestamp), - rspamd_lru_hash_size (spf_module_ctx->spf_hash), - rspamd_lru_hash_capacity (spf_module_ctx->spf_hash)); - } - - for (i = 0; i < rec->elts->len; i ++) { - addr = &g_array_index (rec->elts, struct spf_addr, i); - if (spf_check_element (rec, addr, task, cached)) { - break; - } - } -} - -static void -spf_plugin_callback (struct spf_resolved *record, struct rspamd_task *task, - gpointer ud) -{ - struct spf_resolved *l = NULL; - struct rspamd_symcache_item *item = (struct rspamd_symcache_item *)ud; - struct spf_ctx *spf_module_ctx = spf_get_context (task->cfg); - - if (record && (record->flags & RSPAMD_SPF_RESOLVED_NA)) { - rspamd_task_insert_result (task, - spf_module_ctx->symbol_na, - 1, - NULL); - } - else if (record && record->elts->len == 0 && (record->flags & RSPAMD_SPF_RESOLVED_TEMP_FAILED)) { - rspamd_task_insert_result (task, - spf_module_ctx->symbol_dnsfail, - 1, - NULL); - } - else if (record && record->elts->len == 0 && (record->flags & RSPAMD_SPF_RESOLVED_PERM_FAILED)) { - rspamd_task_insert_result (task, - spf_module_ctx->symbol_permfail, - 1, - NULL); - } - else if (record && record->elts->len == 0) { - rspamd_task_insert_result (task, - spf_module_ctx->symbol_permfail, - 1, - NULL); - } - else if (record && record->domain) { - - spf_record_ref (record); - - if (!spf_module_ctx->spf_hash || - (l = rspamd_lru_hash_lookup (spf_module_ctx->spf_hash, - record->domain, task->task_timestamp)) == NULL) { - l = record; - - if (record->ttl > 0 && record->flags == 0) { - - if (spf_module_ctx->spf_hash) { - rspamd_lru_hash_insert (spf_module_ctx->spf_hash, - record->domain, spf_record_ref (l), - record->timestamp, record->ttl); - - msg_info_task ("stored record for %s (0x%xuL) in LRU cache for %d seconds, " - "%d/%d elements in the cache", - record->domain, - record->digest, - record->ttl, - rspamd_lru_hash_size (spf_module_ctx->spf_hash), - rspamd_lru_hash_capacity (spf_module_ctx->spf_hash)); - } - } - - } - - spf_record_ref (l); - spf_check_list (l, task, FALSE); - spf_record_unref (l); - - spf_record_unref (record); - } - - rspamd_symcache_item_async_dec_check (task, item, M); -} - - -static void -spf_symbol_callback (struct rspamd_task *task, - struct rspamd_symcache_item *item, - void *unused) -{ - const gchar *domain; - struct spf_resolved *l; - gint *dmarc_checks; - struct spf_ctx *spf_module_ctx = spf_get_context (task->cfg); - - /* Allow dmarc */ - dmarc_checks = rspamd_mempool_get_variable (task->task_pool, - RSPAMD_MEMPOOL_DMARC_CHECKS); - - if (dmarc_checks) { - (*dmarc_checks) ++; - } - else { - dmarc_checks = rspamd_mempool_alloc (task->task_pool, - sizeof (*dmarc_checks)); - *dmarc_checks = 1; - rspamd_mempool_set_variable (task->task_pool, - RSPAMD_MEMPOOL_DMARC_CHECKS, - dmarc_checks, NULL); - } - - if (rspamd_match_radix_map_addr (spf_module_ctx->whitelist_ip, - task->from_addr) != NULL) { - rspamd_symcache_finalize_item (task, item); - return; - } - - if ((!spf_module_ctx->check_authed && task->user != NULL) - || (!spf_module_ctx->check_local && - rspamd_inet_address_is_local (task->from_addr, TRUE))) { - msg_info_task ("skip SPF checks for local networks and authorized users"); - rspamd_symcache_finalize_item (task, item); - - return; - } - - domain = rspamd_spf_get_domain (task); - rspamd_symcache_item_async_inc (task, item, M); - - if (domain) { - if (spf_module_ctx->spf_hash && - (l = rspamd_lru_hash_lookup (spf_module_ctx->spf_hash, domain, - task->task_timestamp)) != NULL) { - spf_record_ref (l); - spf_check_list (l, task, TRUE); - spf_record_unref (l); - } - else { - - if (!rspamd_spf_resolve (task, spf_plugin_callback, item)) { - msg_info_task ("cannot make spf request for %s", domain); - rspamd_task_insert_result (task, - spf_module_ctx->symbol_dnsfail, - 1, - "(SPF): spf DNS fail"); - } - else { - rspamd_symcache_item_async_inc (task, item, M); - } - } - } - - rspamd_symcache_item_async_dec_check (task, item, M); -} diff --git a/src/rspamadm/CMakeLists.txt b/src/rspamadm/CMakeLists.txt index 925471619..51ce1bc1b 100644 --- a/src/rspamadm/CMakeLists.txt +++ b/src/rspamadm/CMakeLists.txt @@ -11,7 +11,7 @@ SET(RSPAMADMSRC rspamadm.c lua_repl.c dkim_keygen.c ${CMAKE_BINARY_DIR}/src/workers.c - ${CMAKE_BINARY_DIR}/src/modules.c + #${CMAKE_BINARY_DIR}/src/modules.c - defined in rspamdserver ${CMAKE_SOURCE_DIR}/src/controller.c ${CMAKE_SOURCE_DIR}/src/fuzzy_storage.c ${CMAKE_SOURCE_DIR}/src/worker.c diff --git a/src/rspamadm/lua_repl.c b/src/rspamadm/lua_repl.c index ee22c4868..bceed5855 100644 --- a/src/rspamadm/lua_repl.c +++ b/src/rspamadm/lua_repl.c @@ -431,7 +431,7 @@ rspamadm_lua_message_handler (lua_State *L, gint argc, gchar **argv) rspamd_printf ("cannot open %s: %s\n", argv[i], strerror (errno)); } else { - task = rspamd_task_new (NULL, rspamd_main->cfg, NULL, NULL, NULL); + task = rspamd_task_new (NULL, rspamd_main->cfg, NULL, NULL, NULL, FALSE); if (!rspamd_task_load_message (task, NULL, map, len)) { rspamd_printf ("cannot load %s\n", argv[i]); diff --git a/src/rspamadm/rspamadm.c b/src/rspamadm/rspamadm.c index 661b468af..775b505ee 100644 --- a/src/rspamadm/rspamadm.c +++ b/src/rspamadm/rspamadm.c @@ -396,7 +396,7 @@ main (gint argc, gchar **argv, gchar **env) rspamd_main->pid = getpid (); rspamd_main->type = process_quark; rspamd_main->server_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), - "rspamadm"); + "rspamadm", 0); rspamadm_fill_internal_commands (all_commands); help_command.command_data = all_commands; diff --git a/src/rspamd.c b/src/rspamd.c index 495da45d9..f25884f6f 100644 --- a/src/rspamd.c +++ b/src/rspamd.c @@ -691,8 +691,8 @@ kill_old_workers (gpointer key, gpointer value, gpointer unused) rspamd_main = w->srv; - if (!w->wanna_die) { - w->wanna_die = TRUE; + if (w->state == rspamd_worker_state_running) { + w->state = rspamd_worker_state_terminating; kill (w->pid, SIGUSR2); ev_io_stop (rspamd_main->event_loop, &w->srv_ev); msg_info_main ("send signal to worker %P", w->pid); @@ -950,7 +950,7 @@ rspamd_term_handler (struct ev_loop *loop, ev_signal *w, int revents) shutdown_ts = MAX (SOFT_SHUTDOWN_TIME, rspamd_main->cfg->task_timeout * 2.0); msg_info_main ("catch termination signal, waiting for children for %.2f seconds", - shutdown_ts); + valgrind_mode ? shutdown_ts * 10 : shutdown_ts); /* Stop srv events to avoid false notifications */ g_hash_table_foreach (rspamd_main->workers, stop_srv_ev, rspamd_main); rspamd_pass_signal (rspamd_main->workers, SIGTERM); @@ -1095,10 +1095,6 @@ rspamd_cld_handler (EV_P_ ev_child *w, struct rspamd_main *rspamd_main, rspamd_control_broadcast_srv_cmd (rspamd_main, &cmd, wrk->pid); } - if (wrk->finish_actions) { - g_ptr_array_free (wrk->finish_actions, TRUE); - } - need_refork = rspamd_check_termination_clause (wrk->srv, wrk, w->rstatus); if (need_refork) { @@ -1190,7 +1186,7 @@ main (gint argc, gchar **argv, gchar **env) rspamd_main = (struct rspamd_main *) g_malloc0 (sizeof (struct rspamd_main)); rspamd_main->server_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), - "main"); + "main", 0); rspamd_main->stat = rspamd_mempool_alloc0_shared (rspamd_main->server_pool, sizeof (struct rspamd_stat)); rspamd_main->cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_DEFAULT); diff --git a/src/rspamd.h b/src/rspamd.h index d32681359..773be7c56 100644 --- a/src/rspamd.h +++ b/src/rspamd.h @@ -82,6 +82,14 @@ struct rspamd_worker_heartbeat { gint64 nbeats; /**< positive for beats received, negative for beats missed */ }; +enum rspamd_worker_state { + rspamd_worker_state_running = 0, + rspamd_worker_state_terminating, + rspamd_worker_wait_connections, + rspamd_worker_wait_final_scripts, + rspamd_worker_wanna_die +}; + /** * Worker process structure */ @@ -90,7 +98,7 @@ struct rspamd_worker { pid_t ppid; /**< pid of parent */ guint index; /**< index number */ guint nconns; /**< current connections count */ - gboolean wanna_die; /**< worker is terminating */ + enum rspamd_worker_state state; /**< current worker state */ gboolean cores_throttled; /**< set to true if cores throttling took place */ gdouble start_time; /**< start time */ struct rspamd_main *srv; /**< pointer to server structure */ @@ -108,7 +116,6 @@ struct rspamd_worker { struct rspamd_worker_heartbeat hb; /**< heartbeat data */ gpointer control_data; /**< used by control protocol to handle commands */ gpointer tmp_data; /**< used to avoid race condition to deal with control messages */ - GPtrArray *finish_actions; /**< called when worker is terminated */ ev_child cld_ev; /**< to allow reaping */ rspamd_worker_term_cb term_handler; /**< custom term handler */ }; diff --git a/src/rspamd_proxy.c b/src/rspamd_proxy.c index 227e20c99..7e460c040 100644 --- a/src/rspamd_proxy.c +++ b/src/rspamd_proxy.c @@ -1172,7 +1172,7 @@ proxy_session_refresh (struct rspamd_proxy_session *session) session->client_addr = NULL; nsession->ctx = session->ctx; nsession->worker = session->worker; - nsession->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "proxy"); + nsession->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "proxy", 0); nsession->client_sock = session->client_sock; session->client_sock = -1; nsession->mirror_conns = g_ptr_array_sized_new (nsession->ctx->mirrors->len); @@ -1293,7 +1293,7 @@ proxy_backend_mirror_error_handler (struct rspamd_http_connection *conn, GError msg_info_session ("abnormally closing connection from backend: %s:%s, " "error: %e", bk_conn->name, - rspamd_inet_address_to_string ( + rspamd_inet_address_to_string_pretty ( rspamd_upstream_addr_cur (bk_conn->up)), err); @@ -1301,7 +1301,7 @@ proxy_backend_mirror_error_handler (struct rspamd_http_connection *conn, GError bk_conn->err = rspamd_mempool_strdup (session->pool, err->message); } - rspamd_upstream_fail (bk_conn->up, FALSE); + rspamd_upstream_fail (bk_conn->up, FALSE, err ? err->message : "unknown"); proxy_backend_close_connection (bk_conn); REF_RELEASE (bk_conn->s); @@ -1378,7 +1378,7 @@ proxy_open_mirror_connections (struct rspamd_proxy_session *session) if (bk_conn->backend_sock == -1) { msg_err_session ("cannot connect upstream for %s", m->name); - rspamd_upstream_fail (bk_conn->up, TRUE); + rspamd_upstream_fail (bk_conn->up, TRUE, strerror (errno)); continue; } @@ -1474,8 +1474,36 @@ proxy_client_write_error (struct rspamd_proxy_session *session, gint code, } else { reply = rspamd_http_new_message (HTTP_RESPONSE); - reply->code = code; - reply->status = rspamd_fstring_new_init (status, strlen (status)); + + switch (code) { + case ETIMEDOUT: + reply->code = 504; + reply->status = RSPAMD_FSTRING_LIT ("Gateway timeout"); + break; + case ECONNRESET: + case ECONNABORTED: + reply->code = 502; + reply->status = RSPAMD_FSTRING_LIT ("Gateway connection reset"); + break; + case ECONNREFUSED: + reply->code = 502; + reply->status = RSPAMD_FSTRING_LIT ("Gateway connection refused"); + break; + default: + if (code >= 300) { + /* Likely HTTP error */ + reply->code = code; + reply->status = rspamd_fstring_new_init (status, strlen (status)); + } + else { + reply->code = 502; + reply->status = RSPAMD_FSTRING_LIT ("Unknown gateway error: "); + reply->status = rspamd_fstring_append (reply->status, + status, strlen (status)); + } + break; + } + rspamd_http_connection_write_message (session->client_conn, reply, NULL, NULL, session, session->ctx->timeout); @@ -1489,18 +1517,18 @@ proxy_backend_master_error_handler (struct rspamd_http_connection *conn, GError struct rspamd_proxy_session *session; session = bk_conn->s; - msg_info_session ("abnormally closing connection from backend: %s, error: %e," - " retries left: %d", - rspamd_inet_address_to_string ( - rspamd_upstream_addr_cur (session->master_conn->up)), - err, - session->ctx->max_retries - session->retries); session->retries ++; - rspamd_upstream_fail (bk_conn->up, FALSE); + msg_info_session ("abnormally closing connection from backend: %s, error: %e," + " retries left: %d", + rspamd_inet_address_to_string_pretty ( + rspamd_upstream_addr_cur (session->master_conn->up)), + err, + session->ctx->max_retries - session->retries); + rspamd_upstream_fail (bk_conn->up, FALSE, err ? err->message : "unknown"); proxy_backend_close_connection (session->master_conn); - if (session->ctx->max_retries && - session->retries > session->ctx->max_retries) { + if (session->ctx->max_retries > 0 && + session->retries >= session->ctx->max_retries) { msg_err_session ("cannot connect to upstream, maximum retries " "has been reached: %d", session->retries); /* Terminate session immediately */ @@ -1726,7 +1754,7 @@ rspamd_proxy_self_scan (struct rspamd_proxy_session *session) msg = session->client_message; task = rspamd_task_new (session->worker, session->ctx->cfg, session->pool, session->ctx->lang_det, - session->ctx->event_loop); + session->ctx->event_loop, FALSE); task->flags |= RSPAMD_TASK_FLAG_MIME; if (session->ctx->milter) { @@ -1844,8 +1872,25 @@ retry: goto err; } - session->master_conn->up = rspamd_upstream_get (backend->u, - RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0); + /* Provide hash key if hashing based on source address is desired */ + guint hash_len; + gpointer hash_key = rspamd_inet_address_get_hash_key (session->client_addr, + &hash_len); + + if (session->ctx->max_retries > 1 && + session->retries == session->ctx->max_retries) { + + session->master_conn->up = rspamd_upstream_get_except (backend->u, + session->master_conn->up, + RSPAMD_UPSTREAM_ROUND_ROBIN, + hash_key, hash_len); + } + else { + session->master_conn->up = rspamd_upstream_get (backend->u, + RSPAMD_UPSTREAM_ROUND_ROBIN, + hash_key, hash_len); + } + session->master_conn->timeout = backend->timeout; if (session->master_conn->up == NULL) { @@ -1861,10 +1906,11 @@ retry: if (session->master_conn->backend_sock == -1) { msg_err_session ("cannot connect upstream: %s(%s)", host ? hostbuf : "default", - rspamd_inet_address_to_string ( + rspamd_inet_address_to_string_pretty ( rspamd_upstream_addr_cur ( session->master_conn->up))); - rspamd_upstream_fail (session->master_conn->up, TRUE); + rspamd_upstream_fail (session->master_conn->up, TRUE, + strerror (errno)); session->retries ++; goto retry; } @@ -2123,7 +2169,7 @@ proxy_accept_socket (EV_P_ ev_io *w, int revents) session->mirror_conns = g_ptr_array_sized_new (ctx->mirrors->len); session->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), - "proxy"); + "proxy", 0); session->ctx = ctx; session->worker = worker; @@ -2216,6 +2262,7 @@ void start_rspamd_proxy (struct rspamd_worker *worker) { struct rspamd_proxy_ctx *ctx = worker->ctx; + gboolean is_controller = FALSE; g_assert (rspamd_worker_check_context (worker->ctx, rspamd_rspamd_proxy_magic)); ctx->cfg = worker->srv->cfg; @@ -2225,7 +2272,6 @@ start_rspamd_proxy (struct rspamd_worker *worker) ctx->resolver = rspamd_dns_resolver_init (worker->srv->logger, ctx->event_loop, worker->srv->cfg); - rspamd_map_watch (worker->srv->cfg, ctx->event_loop, ctx->resolver, worker, 0); rspamd_upstreams_library_config (worker->srv->cfg, ctx->cfg->ups_ctx, ctx->event_loop, ctx->resolver->r); @@ -2241,6 +2287,42 @@ start_rspamd_proxy (struct rspamd_worker *worker) rspamd_worker_init_scanner (worker, ctx->event_loop, ctx->resolver, &ctx->lang_det); + if (worker->index == 0) { + /* + * If there are no controllers and no normal workers, + * then pretend that we are a controller + */ + gboolean controller_seen = FALSE; + GList *cur; + + cur = worker->srv->cfg->workers; + + while (cur) { + struct rspamd_worker_conf *cf; + + cf = (struct rspamd_worker_conf *)cur->data; + if ((cf->type == g_quark_from_static_string ("controller")) || + (cf->type == g_quark_from_static_string ("normal"))) { + + if (cf->enabled && cf->count >= 0) { + controller_seen = TRUE; + break; + } + } + + cur = g_list_next (cur); + } + + if (!controller_seen) { + msg_info ("no controller or normal workers defined, execute " + "controller periodics in this worker"); + worker->flags |= RSPAMD_WORKER_CONTROLLER; + is_controller = TRUE; + } + } + } + else { + worker->flags &= ~RSPAMD_WORKER_SCANNER; } if (worker->srv->cfg->enable_sessions_cache) { @@ -2257,6 +2339,20 @@ start_rspamd_proxy (struct rspamd_worker *worker) ctx->milter_ctx.cfg = ctx->cfg; rspamd_milter_init_library (&ctx->milter_ctx); + if (is_controller) { + rspamd_worker_init_controller (worker, NULL); + } + else { + if (ctx->has_self_scan) { + rspamd_map_watch (worker->srv->cfg, ctx->event_loop, ctx->resolver, + worker, RSPAMD_MAP_WATCH_SCANNER); + } + else { + rspamd_map_watch (worker->srv->cfg, ctx->event_loop, ctx->resolver, + worker, RSPAMD_MAP_WATCH_WORKER); + } + } + rspamd_lua_run_postloads (ctx->cfg->lua_state, ctx->cfg, ctx->event_loop, worker); adjust_upstreams_limits (ctx); @@ -2268,6 +2364,10 @@ start_rspamd_proxy (struct rspamd_worker *worker) rspamd_stat_close (); } + if (is_controller) { + rspamd_controller_on_terminate (worker, NULL); + } + REF_RELEASE (ctx->cfg); rspamd_log_close (worker->srv->logger, TRUE); diff --git a/src/worker.c b/src/worker.c index 04447feea..7d9550249 100644 --- a/src/worker.c +++ b/src/worker.c @@ -27,16 +27,13 @@ #include "libserver/dns.h" #include "libmime/message.h" #include "rspamd.h" -#include "keypairs_cache.h" #include "libstat/stat_api.h" #include "libserver/worker_util.h" #include "libserver/rspamd_control.h" #include "worker_private.h" -#include "utlist.h" #include "libutil/http_private.h" -#include "libmime/lang_detection.h" +#include "libserver/cfg_file_private.h" #include <math.h> -#include <src/libserver/cfg_file_private.h> #include "unix-std.h" #include "lua/lua_common.h" @@ -57,68 +54,27 @@ worker_t normal_worker = { }; #define msg_err_ctx(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \ - "controller", ctx->cfg->cfg_pool->tag.uid, \ + "worker", ctx->cfg->cfg_pool->tag.uid, \ G_STRFUNC, \ __VA_ARGS__) #define msg_warn_ctx(...) rspamd_default_log_function (G_LOG_LEVEL_WARNING, \ - "controller", ctx->cfg->cfg_pool->tag.uid, \ + "worker", ctx->cfg->cfg_pool->tag.uid, \ G_STRFUNC, \ __VA_ARGS__) #define msg_info_ctx(...) rspamd_default_log_function (G_LOG_LEVEL_INFO, \ - "controller", ctx->cfg->cfg_pool->tag.uid, \ + "worker", ctx->cfg->cfg_pool->tag.uid, \ G_STRFUNC, \ __VA_ARGS__) -static gboolean -rspamd_worker_finalize (gpointer user_data) -{ - struct rspamd_task *task = user_data; - - if (!(task->flags & RSPAMD_TASK_FLAG_PROCESSING)) { - msg_info_task ("finishing actions has been processed, terminating"); - /* ev_break (task->event_loop, EVBREAK_ALL); */ - rspamd_session_destroy (task->s); - - return TRUE; - } - - return FALSE; -} - -static gboolean -rspamd_worker_call_finish_handlers (struct rspamd_worker *worker) -{ +struct rspamd_worker_session { + gint64 magic; struct rspamd_task *task; - struct rspamd_config *cfg = worker->srv->cfg; - struct rspamd_abstract_worker_ctx *ctx; - struct rspamd_config_cfg_lua_script *sc; - - if (cfg->on_term_scripts) { - ctx = worker->ctx; - /* Create a fake task object for async events */ - task = rspamd_task_new (worker, cfg, NULL, NULL, ctx->event_loop); - task->resolver = ctx->resolver; - task->flags |= RSPAMD_TASK_FLAG_PROCESSING; - task->s = rspamd_session_create (task->task_pool, - rspamd_worker_finalize, - NULL, - (event_finalizer_t) rspamd_task_free, - task); - - DL_FOREACH (cfg->on_term_scripts, sc) { - lua_call_finish_script (sc, task); - } - - task->flags &= ~RSPAMD_TASK_FLAG_PROCESSING; - - if (rspamd_session_pending (task->s)) { - return TRUE; - } - } - - return FALSE; -} - + gint fd; + rspamd_inet_addr_t *addr; + struct rspamd_worker_ctx *ctx; + struct rspamd_http_connection *http_conn; + struct rspamd_worker *worker; +}; /* * Reduce number of tasks proceeded */ @@ -129,139 +85,89 @@ reduce_tasks_count (gpointer arg) worker->nconns --; - if (worker->wanna_die && worker->nconns == 0) { + if (worker->state == rspamd_worker_wait_connections && worker->nconns == 0) { + + worker->state = rspamd_worker_wait_final_scripts; msg_info ("performing finishing actions"); - rspamd_worker_call_finish_handlers (worker); + + if (rspamd_worker_call_finish_handlers (worker)) { + worker->state = rspamd_worker_wait_final_scripts; + } + else { + worker->state = rspamd_worker_wanna_die; + } + } + else if (worker->state != rspamd_worker_state_running) { + worker->state = rspamd_worker_wait_connections; } } -void -rspamd_task_timeout (EV_P_ ev_timer *w, int revents) +static gint +rspamd_worker_body_handler (struct rspamd_http_connection *conn, + struct rspamd_http_message *msg, + const gchar *chunk, gsize len) { - struct rspamd_task *task = (struct rspamd_task *)w->data; - - if (!(task->processed_stages & RSPAMD_TASK_STAGE_FILTERS)) { - msg_info_task ("processing of task time out: %.1f second spent; forced processing", - ev_now (task->event_loop) - task->task_timestamp); - - if (task->cfg->soft_reject_on_timeout) { - struct rspamd_action *action, *soft_reject; - - action = rspamd_check_action_metric (task); - - if (action->action_type != METRIC_ACTION_REJECT) { - soft_reject = rspamd_config_get_action_by_type (task->cfg, - METRIC_ACTION_SOFT_REJECT); - rspamd_add_passthrough_result (task, - soft_reject, - 0, - NAN, - "timeout processing message", - "task timeout", - 0); - - ucl_object_replace_key (task->messages, - ucl_object_fromstring_common ("timeout processing message", - 0, UCL_STRING_RAW), - "smtp_message", 0, - false); - } - } + struct rspamd_worker_session *session = (struct rspamd_worker_session *)conn->ud; + struct rspamd_task *task; + struct rspamd_worker_ctx *ctx; + const rspamd_ftok_t *hv_tok; + gboolean debug_mempool = FALSE; - ev_timer_again (EV_A_ w); - task->processed_stages |= RSPAMD_TASK_STAGE_FILTERS; - rspamd_session_cleanup (task->s); - rspamd_task_process (task, RSPAMD_TASK_PROCESS_ALL); - rspamd_session_pending (task->s); - } - else { - /* Postprocessing timeout */ - msg_info_task ("post-processing of task time out: %.1f second spent; forced processing", - ev_now (task->event_loop) - task->task_timestamp); - - if (task->cfg->soft_reject_on_timeout) { - struct rspamd_action *action, *soft_reject; - - action = rspamd_check_action_metric (task); - - if (action->action_type != METRIC_ACTION_REJECT) { - soft_reject = rspamd_config_get_action_by_type (task->cfg, - METRIC_ACTION_SOFT_REJECT); - rspamd_add_passthrough_result (task, - soft_reject, - 0, - NAN, - "timeout post-processing message", - "task timeout", - 0); - - ucl_object_replace_key (task->messages, - ucl_object_fromstring_common ("timeout post-processing message", - 0, UCL_STRING_RAW), - "smtp_message", 0, - false); - } - } + ctx = session->ctx; - ev_timer_stop (EV_A_ w); - task->processed_stages |= RSPAMD_TASK_STAGE_DONE; - rspamd_session_cleanup (task->s); - rspamd_task_process (task, RSPAMD_TASK_PROCESS_ALL); - rspamd_session_pending (task->s); + /* Check debug */ + if ((hv_tok = rspamd_http_message_find_header (msg, "Memory")) != NULL) { + rspamd_ftok_t cmp; + + RSPAMD_FTOK_ASSIGN (&cmp, "debug"); + + if (rspamd_ftok_cmp (hv_tok, &cmp) == 0) { + debug_mempool = TRUE; + } } -} -void -rspamd_worker_guard_handler (EV_P_ ev_io *w, int revents) -{ - struct rspamd_task *task = (struct rspamd_task *)w->data; - gchar fake_buf[1024]; - gssize r; + task = rspamd_task_new (session->worker, + session->ctx->cfg, NULL, session->ctx->lang_det, + session->ctx->event_loop, + debug_mempool); + session->task = task; - r = read (w->fd, fake_buf, sizeof (fake_buf)); + msg_info_task ("accepted connection from %s port %d, task ptr: %p", + rspamd_inet_address_to_string (session->addr), + rspamd_inet_address_get_port (session->addr), + task); - if (r > 0) { - msg_warn_task ("received extra data after task is loaded, ignoring"); + /* Copy some variables */ + if (ctx->is_mime) { + task->flags |= RSPAMD_TASK_FLAG_MIME; } else { - if (r == 0) { - /* - * Poor man approach, that might break things in case of - * shutdown (SHUT_WR) but sockets are so bad that there's no - * reliable way to distinguish between shutdown(SHUT_WR) and - * close. - */ - if (task->cmd != CMD_CHECK_V2 && task->cfg->enable_shutdown_workaround) { - msg_info_task ("workaround for shutdown enabled, please update " - "your client, this support might be removed in future"); - shutdown (w->fd, SHUT_RD); - ev_io_stop (task->event_loop, &task->guard_ev); - } - else { - msg_err_task ("the peer has closed connection unexpectedly"); - rspamd_session_destroy (task->s); - } - } - else if (errno != EAGAIN) { - msg_err_task ("the peer has closed connection unexpectedly: %s", - strerror (errno)); - rspamd_session_destroy (task->s); - } - else { - return; - } + task->flags &= ~RSPAMD_TASK_FLAG_MIME; } -} -static gint -rspamd_worker_body_handler (struct rspamd_http_connection *conn, - struct rspamd_http_message *msg, - const gchar *chunk, gsize len) -{ - struct rspamd_task *task = (struct rspamd_task *) conn->ud; - struct rspamd_worker_ctx *ctx; + /* We actually transfer ownership from session to task here */ + task->sock = session->fd; + task->client_addr = session->addr; + task->worker = session->worker; + task->http_conn = session->http_conn; - ctx = task->worker->ctx; + task->resolver = ctx->resolver; + /* TODO: allow to disable autolearn in protocol */ + task->flags |= RSPAMD_TASK_FLAG_LEARN_AUTO; + + session->worker->nconns++; + rspamd_mempool_add_destructor (task->task_pool, + (rspamd_mempool_destruct_t)reduce_tasks_count, + session->worker); + + /* Session memory is also now handled by task */ + rspamd_mempool_add_destructor (task->task_pool, + (rspamd_mempool_destruct_t)g_free, + session); + + /* Set up async session */ + task->s = rspamd_session_create (task->task_pool, rspamd_task_fin, + rspamd_task_restore, (event_finalizer_t )rspamd_task_free, task); if (!rspamd_protocol_handle_request (task, msg)) { msg_err_task ("cannot handle request: %e", task->err); @@ -285,12 +191,15 @@ rspamd_worker_body_handler (struct rspamd_http_connection *conn, ev_timer_init (&task->timeout_ev, rspamd_task_timeout, ctx->task_timeout, ctx->task_timeout); + ev_set_priority (&task->timeout_ev, EV_MAXPRI); ev_timer_start (task->event_loop, &task->timeout_ev); } /* Set socket guard */ task->guard_ev.data = task; - ev_io_init (&task->guard_ev, rspamd_worker_guard_handler, task->sock, EV_READ); + ev_io_init (&task->guard_ev, + rspamd_worker_guard_handler, + task->sock, EV_READ); ev_io_start (task->event_loop, &task->guard_ev); rspamd_task_process (task, RSPAMD_TASK_PROCESS_ALL); @@ -301,43 +210,84 @@ rspamd_worker_body_handler (struct rspamd_http_connection *conn, static void rspamd_worker_error_handler (struct rspamd_http_connection *conn, GError *err) { - struct rspamd_task *task = (struct rspamd_task *) conn->ud; + struct rspamd_worker_session *session = (struct rspamd_worker_session *)conn->ud; + struct rspamd_task *task; struct rspamd_http_message *msg; rspamd_fstring_t *reply; - msg_info_task ("abnormally closing connection from: %s, error: %e", - rspamd_inet_address_to_string (task->client_addr), err); - if (task->processed_stages & RSPAMD_TASK_STAGE_REPLIED) { - /* Terminate session immediately */ - rspamd_session_destroy (task->s); + /* + * This function can be called with both struct rspamd_worker_session * + * and struct rspamd_task * + * + * The first case is when we read message and it is controlled by this code; + * the second case is when a reply is written and we do not control it normally, + * as it is managed by `rspamd_protocol_reply` in protocol.c + * + * Hence, we need to distinguish our arguments... + * + * The approach here is simple: + * - struct rspamd_worker_session starts with gint64 `magic` and we set it to + * MAX_INT64 + * - struct rspamd_task starts with a pointer (or pointer + command on 32 bit system) + * + * The idea is simple: no sane pointer would reach MAX_INT64, so if this field + * is MAX_INT64 then it is our session, and if it is not then it is a task. + */ + + if (session->magic == G_MAXINT64) { + task = session->task; } else { - task->processed_stages |= RSPAMD_TASK_STAGE_REPLIED; - msg = rspamd_http_new_message (HTTP_RESPONSE); + task = (struct rspamd_task *)conn->ud; + } + + + if (task) { + msg_info_task ("abnormally closing connection from: %s, error: %e", + rspamd_inet_address_to_string_pretty (task->client_addr), err); - if (err) { - msg->status = rspamd_fstring_new_init (err->message, - strlen (err->message)); - msg->code = err->code; + if (task->processed_stages & RSPAMD_TASK_STAGE_REPLIED) { + /* Terminate session immediately */ + rspamd_session_destroy (task->s); } else { - msg->status = rspamd_fstring_new_init ("Internal error", - strlen ("Internal error")); - msg->code = 500; - } + task->processed_stages |= RSPAMD_TASK_STAGE_REPLIED; + msg = rspamd_http_new_message (HTTP_RESPONSE); - msg->date = time (NULL); - - reply = rspamd_fstring_sized_new (msg->status->len + 16); - rspamd_printf_fstring (&reply, "{\"error\":\"%V\"}", msg->status); - rspamd_http_message_set_body_from_fstring_steal (msg, reply); - rspamd_http_connection_reset (task->http_conn); - rspamd_http_connection_write_message (task->http_conn, - msg, - NULL, - "application/json", - task, - 1.0); + if (err) { + msg->status = rspamd_fstring_new_init (err->message, + strlen (err->message)); + msg->code = err->code; + } + else { + msg->status = rspamd_fstring_new_init ("Internal error", + strlen ("Internal error")); + msg->code = 500; + } + + msg->date = time (NULL); + + reply = rspamd_fstring_sized_new (msg->status->len + 16); + rspamd_printf_fstring (&reply, "{\"error\":\"%V\"}", msg->status); + rspamd_http_message_set_body_from_fstring_steal (msg, reply); + rspamd_http_connection_reset (task->http_conn); + rspamd_http_connection_write_message (task->http_conn, + msg, + NULL, + "application/json", + task, + 1.0); + } + } + else { + /* If there was no task, then session is unmanaged */ + msg_info ("no data received from: %s, error: %e", + rspamd_inet_address_to_string_pretty (session->addr), err); + rspamd_http_connection_reset (session->http_conn); + rspamd_http_connection_unref (session->http_conn); + rspamd_inet_address_free (session->addr); + close (session->fd); + g_free (session); } } @@ -345,16 +295,38 @@ static gint rspamd_worker_finish_handler (struct rspamd_http_connection *conn, struct rspamd_http_message *msg) { - struct rspamd_task *task = (struct rspamd_task *) conn->ud; + struct rspamd_worker_session *session = (struct rspamd_worker_session *)conn->ud; + struct rspamd_task *task; - if (task->processed_stages & RSPAMD_TASK_STAGE_REPLIED) { - /* We are done here */ - msg_debug_task ("normally closing connection from: %s", - rspamd_inet_address_to_string (task->client_addr)); - rspamd_session_destroy (task->s); + /* Read the comment to rspamd_worker_error_handler */ + + if (session->magic == G_MAXINT64) { + task = session->task; + } + else { + task = (struct rspamd_task *)conn->ud; + } + + if (task) { + if (task->processed_stages & RSPAMD_TASK_STAGE_REPLIED) { + /* We are done here */ + msg_debug_task ("normally closing connection from: %s", + rspamd_inet_address_to_string (task->client_addr)); + rspamd_session_destroy (task->s); + } + else if (task->processed_stages & RSPAMD_TASK_STAGE_DONE) { + rspamd_session_pending (task->s); + } } - else if (task->processed_stages & RSPAMD_TASK_STAGE_DONE) { - rspamd_session_pending (task->s); + else { + /* If there was no task, then session is unmanaged */ + msg_info ("no data received from: %s, closing connection", + rspamd_inet_address_to_string_pretty (session->addr)); + rspamd_inet_address_free (session->addr); + rspamd_http_connection_reset (session->http_conn); + rspamd_http_connection_unref (session->http_conn); + close (session->fd); + g_free (session); } return 0; @@ -368,7 +340,7 @@ accept_socket (EV_P_ ev_io *w, int revents) { struct rspamd_worker *worker = (struct rspamd_worker *) w->data; struct rspamd_worker_ctx *ctx; - struct rspamd_task *task; + struct rspamd_worker_session *session; rspamd_inet_addr_t *addr; gint nfd, http_opts = 0; @@ -392,132 +364,38 @@ accept_socket (EV_P_ ev_io *w, int revents) return; } - task = rspamd_task_new (worker, ctx->cfg, NULL, ctx->lang_det, ctx->event_loop); - - msg_info_task ("accepted connection from %s port %d, task ptr: %p", - rspamd_inet_address_to_string (addr), - rspamd_inet_address_get_port (addr), - task); - - /* Copy some variables */ - if (ctx->is_mime) { - task->flags |= RSPAMD_TASK_FLAG_MIME; - } - else { - task->flags &= ~RSPAMD_TASK_FLAG_MIME; - } - - task->sock = nfd; - task->client_addr = addr; - - worker->srv->stat->connections_count++; - task->resolver = ctx->resolver; - /* TODO: allow to disable autolearn in protocol */ - task->flags |= RSPAMD_TASK_FLAG_LEARN_AUTO; + session = g_malloc0 (sizeof (*session)); + session->magic = G_MAXINT64; + session->addr = addr; + session->fd = nfd; + session->ctx = ctx; + session->worker = worker; if (ctx->encrypted_only && !rspamd_inet_address_is_local (addr, FALSE)) { http_opts = RSPAMD_HTTP_REQUIRE_ENCRYPTION; } - task->http_conn = rspamd_http_connection_new_server ( + session->http_conn = rspamd_http_connection_new_server ( ctx->http_ctx, nfd, rspamd_worker_body_handler, rspamd_worker_error_handler, rspamd_worker_finish_handler, http_opts); - rspamd_http_connection_set_max_size (task->http_conn, task->cfg->max_message); - worker->nconns++; - rspamd_mempool_add_destructor (task->task_pool, - (rspamd_mempool_destruct_t)reduce_tasks_count, worker); - /* Set up async session */ - task->s = rspamd_session_create (task->task_pool, rspamd_task_fin, - rspamd_task_restore, (event_finalizer_t )rspamd_task_free, task); + worker->srv->stat->connections_count++; + rspamd_http_connection_set_max_size (session->http_conn, + ctx->cfg->max_message); if (ctx->key) { - rspamd_http_connection_set_key (task->http_conn, ctx->key); + rspamd_http_connection_set_key (session->http_conn, ctx->key); } - rspamd_http_connection_read_message (task->http_conn, - task, + rspamd_http_connection_read_message (session->http_conn, + session, ctx->timeout); } -static gboolean -rspamd_worker_log_pipe_handler (struct rspamd_main *rspamd_main, - struct rspamd_worker *worker, gint fd, - gint attached_fd, - struct rspamd_control_command *cmd, - gpointer ud) -{ - struct rspamd_config *cfg = ud; - struct rspamd_worker_log_pipe *lp; - struct rspamd_control_reply rep; - - memset (&rep, 0, sizeof (rep)); - rep.type = RSPAMD_CONTROL_LOG_PIPE; - - if (attached_fd != -1) { - lp = g_malloc0 (sizeof (*lp)); - lp->fd = attached_fd; - lp->type = cmd->cmd.log_pipe.type; - - DL_APPEND (cfg->log_pipes, lp); - msg_info ("added new log pipe"); - } - else { - rep.reply.log_pipe.status = ENOENT; - msg_err ("cannot attach log pipe: invalid fd"); - } - - if (write (fd, &rep, sizeof (rep)) != sizeof (rep)) { - msg_err ("cannot write reply to the control socket: %s", - strerror (errno)); - } - - return TRUE; -} - -static gboolean -rspamd_worker_monitored_handler (struct rspamd_main *rspamd_main, - struct rspamd_worker *worker, gint fd, - gint attached_fd, - struct rspamd_control_command *cmd, - gpointer ud) -{ - struct rspamd_control_reply rep; - struct rspamd_monitored *m; - struct rspamd_monitored_ctx *mctx = worker->srv->cfg->monitored_ctx; - struct rspamd_config *cfg = ud; - - memset (&rep, 0, sizeof (rep)); - rep.type = RSPAMD_CONTROL_MONITORED_CHANGE; - - if (cmd->cmd.monitored_change.sender != getpid ()) { - m = rspamd_monitored_by_tag (mctx, cmd->cmd.monitored_change.tag); - - if (m != NULL) { - rspamd_monitored_set_alive (m, cmd->cmd.monitored_change.alive); - rep.reply.monitored_change.status = 1; - msg_info_config ("updated monitored status for %s: %s", - cmd->cmd.monitored_change.tag, - cmd->cmd.monitored_change.alive ? "alive" : "dead"); - } else { - msg_err ("cannot find monitored by tag: %*s", 32, - cmd->cmd.monitored_change.tag); - rep.reply.monitored_change.status = 0; - } - } - - if (write (fd, &rep, sizeof (rep)) != sizeof (rep)) { - msg_err ("cannot write reply to the control socket: %s", - strerror (errno)); - } - - return TRUE; -} - gpointer init_worker (struct rspamd_config *cfg) { @@ -596,46 +474,6 @@ init_worker (struct rspamd_config *cfg) return ctx; } -static gboolean -rspamd_worker_on_terminate (struct rspamd_worker *worker) -{ - if (worker->nconns == 0) { - msg_info ("performing finishing actions"); - if (rspamd_worker_call_finish_handlers (worker)) { - return TRUE; - } - } - - return FALSE; -} - -void -rspamd_worker_init_scanner (struct rspamd_worker *worker, - struct ev_loop *ev_base, - struct rspamd_dns_resolver *resolver, - struct rspamd_lang_detector **plang_det) -{ - rspamd_stat_init (worker->srv->cfg, ev_base); - g_ptr_array_add (worker->finish_actions, - (gpointer) rspamd_worker_on_terminate); -#ifdef WITH_HYPERSCAN - rspamd_control_worker_add_cmd_handler (worker, - RSPAMD_CONTROL_HYPERSCAN_LOADED, - rspamd_worker_hyperscan_ready, - NULL); -#endif - rspamd_control_worker_add_cmd_handler (worker, - RSPAMD_CONTROL_LOG_PIPE, - rspamd_worker_log_pipe_handler, - worker->srv->cfg); - rspamd_control_worker_add_cmd_handler (worker, - RSPAMD_CONTROL_MONITORED_CHANGE, - rspamd_worker_monitored_handler, - worker->srv->cfg); - - *plang_det = worker->srv->cfg->lang_det; -} - /* * Start worker process */ @@ -643,6 +481,7 @@ void start_worker (struct rspamd_worker *worker) { struct rspamd_worker_ctx *ctx = worker->ctx; + gboolean is_controller = FALSE; g_assert (rspamd_worker_check_context (worker->ctx, rspamd_worker_magic)); ctx->cfg = worker->srv->cfg; @@ -662,7 +501,6 @@ start_worker (struct rspamd_worker *worker) ctx->resolver = rspamd_dns_resolver_init (worker->srv->logger, ctx->event_loop, worker->srv->cfg); - rspamd_map_watch (worker->srv->cfg, ctx->event_loop, ctx->resolver, worker, 0); rspamd_upstreams_library_config (worker->srv->cfg, ctx->cfg->ups_ctx, ctx->event_loop, ctx->resolver->r); @@ -673,12 +511,54 @@ start_worker (struct rspamd_worker *worker) ctx->http_ctx); rspamd_worker_init_scanner (worker, ctx->event_loop, ctx->resolver, &ctx->lang_det); + + if (worker->index == 0) { + /* If there are no controllers, then pretend that we are a controller */ + gboolean controller_seen = FALSE; + GList *cur; + + cur = worker->srv->cfg->workers; + + while (cur) { + struct rspamd_worker_conf *cf; + + cf = (struct rspamd_worker_conf *)cur->data; + if (cf->type == g_quark_from_static_string ("controller")) { + if (cf->enabled && cf->count >= 0) { + controller_seen = TRUE; + break; + } + } + + cur = g_list_next (cur); + } + + if (!controller_seen) { + msg_info_ctx ("no controller workers defined, execute " + "controller periodics in this worker"); + worker->flags |= RSPAMD_WORKER_CONTROLLER; + is_controller = TRUE; + } + } + + if (is_controller) { + rspamd_worker_init_controller (worker, NULL); + } + else { + rspamd_map_watch (worker->srv->cfg, ctx->event_loop, ctx->resolver, + worker, RSPAMD_MAP_WATCH_SCANNER); + } + rspamd_lua_run_postloads (ctx->cfg->lua_state, ctx->cfg, ctx->event_loop, worker); ev_loop (ctx->event_loop, 0); rspamd_worker_block_signals (); + if (is_controller) { + rspamd_controller_on_terminate (worker, NULL); + } + rspamd_stat_close (); REF_RELEASE (ctx->cfg); rspamd_log_close (worker->srv->logger, TRUE); diff --git a/src/worker_private.h b/src/worker_private.h index cef2c9a19..62fec96f1 100644 --- a/src/worker_private.h +++ b/src/worker_private.h @@ -65,16 +65,6 @@ void rspamd_worker_init_scanner (struct rspamd_worker *worker, struct rspamd_dns_resolver *resolver, struct rspamd_lang_detector **plang_det); -/* - * Called on forced timeout - */ -void rspamd_task_timeout (EV_P_ ev_timer *w, int revents); - -/* - * Called on unexpected IO error (e.g. ECONNRESET) - */ -void rspamd_worker_guard_handler (EV_P_ ev_io *w, int revents); - #ifdef __cplusplus } #endif diff --git a/test/functional/cases/102_multimap.robot b/test/functional/cases/102_multimap.robot index b495b5394..c953970dc 100644 --- a/test/functional/cases/102_multimap.robot +++ b/test/functional/cases/102_multimap.robot @@ -22,8 +22,13 @@ ${URL4} ${TESTDIR}/messages/url4.eml ${URL5} ${TESTDIR}/messages/url5.eml ${URL_TLD} ${TESTDIR}/../lua/unit/test_tld.dat ${FREEMAIL_CC} ${TESTDIR}/messages/freemailcc.eml +${URL_ICS} ${TESTDIR}/messages/ics.eml *** Test Cases *** +URL_ICS + ${result} = Scan Message With Rspamc ${URL_ICS} + Check Rspamc ${result} Urls: ["test.com"] + MAP - DNSBL HIT ${result} = Scan Message With Rspamc ${MESSAGE} -i 127.0.0.2 Check Rspamc ${result} DNSBL_MAP @@ -326,6 +331,8 @@ FREEMAIL_CC ${result} = Scan Message With Rspamc ${FREEMAIL_CC} Check Rspamc ${result} FREEMAIL_CC (19.00)[test.com, test1.com, test2.com, test3.com, test4.com, test5.com, test6.com, test7.com, test8.com, test9.com, test10.com, test11.com, test12.com, test13.com, test14.com] + + *** Keywords *** Multimap Setup ${PLUGIN_CONFIG} = Get File ${TESTDIR}/configs/multimap.conf diff --git a/test/functional/cases/115_dmarc.robot b/test/functional/cases/115_dmarc.robot index 597a6a330..a3c60f83f 100644 --- a/test/functional/cases/115_dmarc.robot +++ b/test/functional/cases/115_dmarc.robot @@ -87,138 +87,6 @@ DMARC PCT ZERO SP QUARANTINE ... -i 37.48.67.26 --from foo@mom.za.org Check Rspamc ${result} DMARC_POLICY_SOFTFAIL -DKIM PERMFAIL NXDOMAIN - ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim2.eml - ... -i 37.48.67.26 - Check Rspamc ${result} R_DKIM_PERMFAIL - -DKIM PERMFAIL BAD RECORD - ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml - ... -i 37.48.67.26 - Check Rspamc ${result} R_DKIM_PERMFAIL - -DKIM TEMPFAIL SERVFAIL UNALIGNED - ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim3.eml - ... -i 37.48.67.26 - Check Rspamc ${result} R_DKIM_TEMPFAIL - Should Contain ${result.stdout} DMARC_POLICY_SOFTFAIL - -DKIM NA NOSIG - ${result} = Scan Message With Rspamc ${TESTDIR}/messages/utf.eml - ... -i 37.48.67.26 - Check Rspamc ${result} R_DKIM_NA - -SPF PERMFAIL UNRESOLVEABLE INCLUDE - ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml - ... -i 37.48.67.26 -F x@fail3.org.org.za - Check Rspamc ${result} R_SPF_PERMFAIL - -SPF DNSFAIL FAILED INCLUDE UNALIGNED - ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml - ... -i 8.8.8.8 -F x@fail2.org.org.za - Check Rspamc ${result} R_SPF_DNSFAIL - Should Contain ${result.stdout} DMARC_POLICY_SOFTFAIL - -SPF ALLOW UNRESOLVEABLE INCLUDE - ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml - ... -i 8.8.8.8 -F x@fail3.org.org.za - Check Rspamc ${result} R_SPF_ALLOW - -SPF ALLOW FAILED INCLUDE - ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml - ... -i 8.8.4.4 -F x@fail2.org.org.za - Check Rspamc ${result} R_SPF_ALLOW - -SPF NA NA - ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml - ... -i 8.8.8.8 -F x@za - Check Rspamc ${result} R_SPF_NA - -SPF NA NOREC - ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml - ... -i 8.8.8.8 -F x@co.za - Check Rspamc ${result} R_SPF_NA - -SPF NA NXDOMAIN - ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml - ... -i 8.8.8.8 -F x@zzzzaaaa - Check Rspamc ${result} R_SPF_NA - -SPF PERMFAIL UNRESOLVEABLE REDIRECT - ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml - ... -i 8.8.8.8 -F x@fail4.org.org.za - Check Rspamc ${result} R_SPF_PERMFAIL - -SPF REDIRECT NO USEABLE ELEMENTS - ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml - ... -i 8.8.8.8 -F x@fail10.org.org.za - Check Rspamc ${result} R_SPF_PERMFAIL - -SPF DNSFAIL FAILED REDIRECT - ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml - ... -i 8.8.8.8 -F x@fail1.org.org.za - Check Rspamc ${result} R_SPF_DNSFAIL - -SPF PERMFAIL NO USEABLE ELEMENTS - ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml - ... -i 8.8.8.8 -F x@fail5.org.org.za - Check Rspamc ${result} R_SPF_PERMFAIL - -SPF FAIL - ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml - ... -i 8.8.8.8 -F x@example.net - Check Rspamc ${result} R_SPF_FAIL - -SPF PERMFAIL UNRESOLVEABLE MX - ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml - ... -i 1.2.3.4 -F x@fail6.org.org.za - Check Rspamc ${result} R_SPF_PERMFAIL - -SPF PERMFAIL UNRESOLVEABLE A - ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml - ... -i 1.2.3.4 -F x@fail7.org.org.za - Check Rspamc ${result} R_SPF_PERMFAIL - -SPF DNSFAIL FAILED A - ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml - ... -i 1.2.3.4 -F x@fail8.org.org.za - Check Rspamc ${result} R_SPF_DNSFAIL - -SPF DNSFAIL FAILED MX - ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml - ... -i 1.2.3.4 -F x@fail9.org.org.za - Check Rspamc ${result} R_SPF_DNSFAIL - -SPF DNSFAIL FAILED RECORD - ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml - ... -i 1.2.3.4 -F x@www.dnssec-failed.org - Check Rspamc ${result} R_SPF_DNSFAIL - -SPF PASS INCLUDE - ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml - ... -i 8.8.8.8 -F x@pass1.org.org.za - Check Rspamc ${result} R_SPF_ALLOW - -SPF PTRS - ${result} = Scan Message With Rspamc /dev/null - ... -i 88.99.142.95 -F foo@crazyspf.cacophony.za.org - Check Rspamc ${result} R_SPF_ALLOW - ${result} = Scan Message With Rspamc /dev/null - ... -i 128.66.0.1 -F foo@crazyspf.cacophony.za.org - Check Rspamc ${result} R_SPF_PERMFAIL - ${result} = Scan Message With Rspamc /dev/null - ... -i 209.85.216.182 -F foo@crazyspf.cacophony.za.org - Check Rspamc ${result} R_SPF_FAIL - #${result} = Scan Message With Rspamc /dev/null - #... -i 98.138.91.166 -F foo@crazyspf.cacophony.za.org - #Check Rspamc ${result} R_SPF_ALLOW - #${result} = Scan Message With Rspamc /dev/null - #... -i 98.138.91.167 -F foo@crazyspf.cacophony.za.org - #Check Rspamc ${result} R_SPF_ALLOW - #${result} = Scan Message With Rspamc /dev/null - #... -i 98.138.91.168 -F foo@crazyspf.cacophony.za.org - #Check Rspamc ${result} R_SPF_ALLOW - *** Keywords *** DMARC Setup ${PLUGIN_CONFIG} = Get File ${TESTDIR}/configs/dmarc.conf diff --git a/test/functional/cases/130_dkim.robot b/test/functional/cases/116_dkim.robot index ad0b27ac4..79a40b1e6 100644 --- a/test/functional/cases/130_dkim.robot +++ b/test/functional/cases/116_dkim.robot @@ -1,15 +1,36 @@ *** Settings *** -Suite Setup Generic Setup +Suite Setup DKIM Setup Suite Teardown Simple Teardown Library ${TESTDIR}/lib/rspamd.py Resource ${TESTDIR}/lib/rspamd.robot Variables ${TESTDIR}/lib/vars.py *** Variables *** -${CONFIG} ${TESTDIR}/configs/dkim.conf +${CONFIG} ${TESTDIR}/configs/plugins.conf ${RSPAMD_SCOPE} Suite +${URL_TLD} ${TESTDIR}/../../contrib/publicsuffix/effective_tld_names.dat *** Test Cases *** +DKIM PERMFAIL NXDOMAIN + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim2.eml + ... -i 37.48.67.26 + Check Rspamc ${result} R_DKIM_PERMFAIL + +DKIM PERMFAIL BAD RECORD + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml + ... -i 37.48.67.26 + Check Rspamc ${result} R_DKIM_PERMFAIL + +DKIM TEMPFAIL SERVFAIL UNALIGNED + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim3.eml + ... -i 37.48.67.26 + Check Rspamc ${result} R_DKIM_TEMPFAIL + +DKIM NA NOSIG + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/utf.eml + ... -i 37.48.67.26 + Check Rspamc ${result} R_DKIM_NA + DKIM Sign Set Suite Variable ${RAN_SIGNTEST} 0 ${result} = Scan Message With Rspamc ${TESTDIR}/messages/spam_message.eml --mime --header=dodkim=1 @@ -29,4 +50,10 @@ DKIM Verify ED25519 PASS DKIM Verify ED25519 REJECT ${result} = Scan Message With Rspamc ${TESTDIR}/messages/ed25519-broken.eml - Check Rspamc ${result} R_DKIM_REJECT
\ No newline at end of file + Check Rspamc ${result} R_DKIM_REJECT + +*** Keywords *** +DKIM Setup + ${PLUGIN_CONFIG} = Get File ${TESTDIR}/configs/dkim.conf + Set Suite Variable ${PLUGIN_CONFIG} + Generic Setup PLUGIN_CONFIG diff --git a/test/functional/cases/117_spf.robot b/test/functional/cases/117_spf.robot new file mode 100644 index 000000000..08733471b --- /dev/null +++ b/test/functional/cases/117_spf.robot @@ -0,0 +1,143 @@ +*** Settings *** +Suite Setup SPF Setup +Suite Teardown Simple Teardown +Library ${TESTDIR}/lib/rspamd.py +Resource ${TESTDIR}/lib/rspamd.robot +Variables ${TESTDIR}/lib/vars.py + +*** Variables *** +${CONFIG} ${TESTDIR}/configs/plugins.conf +${RSPAMD_SCOPE} Suite +${URL_TLD} ${TESTDIR}/../../contrib/publicsuffix/effective_tld_names.dat + +*** Test Cases *** +SPF FAIL UNRESOLVEABLE INCLUDE + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml + ... -i 37.48.67.26 -F x@fail3.org.org.za + Check Rspamc ${result} R_SPF_FAIL + +SPF DNSFAIL FAILED INCLUDE UNALIGNED + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml + ... -i 8.8.8.8 -F x@fail2.org.org.za + Check Rspamc ${result} R_SPF_DNSFAIL + Should Contain ${result.stdout} DMARC_POLICY_SOFTFAIL + +SPF ALLOW UNRESOLVEABLE INCLUDE + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml + ... -i 8.8.8.8 -F x@fail3.org.org.za + Check Rspamc ${result} R_SPF_ALLOW + +SPF ALLOW FAILED INCLUDE + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml + ... -i 8.8.4.4 -F x@fail2.org.org.za + Check Rspamc ${result} R_SPF_ALLOW + +SPF NA NA + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml + ... -i 8.8.8.8 -F x@za + Check Rspamc ${result} R_SPF_NA + +SPF NA NOREC + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml + ... -i 8.8.8.8 -F x@co.za + Check Rspamc ${result} R_SPF_NA + +SPF NA NXDOMAIN + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml + ... -i 8.8.8.8 -F x@zzzzaaaa + Check Rspamc ${result} R_SPF_NA + +SPF PERMFAIL UNRESOLVEABLE REDIRECT + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml + ... -i 8.8.8.8 -F x@fail4.org.org.za + Check Rspamc ${result} R_SPF_PERMFAIL + +SPF REDIRECT NO USEABLE ELEMENTS + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml + ... -i 8.8.8.8 -F x@fail10.org.org.za + Check Rspamc ${result} R_SPF_PERMFAIL + +SPF DNSFAIL FAILED REDIRECT + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml + ... -i 8.8.8.8 -F x@fail1.org.org.za + Check Rspamc ${result} R_SPF_DNSFAIL + +SPF PERMFAIL NO USEABLE ELEMENTS + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml + ... -i 8.8.8.8 -F x@fail5.org.org.za + Check Rspamc ${result} R_SPF_PERMFAIL + +SPF FAIL + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml + ... -i 8.8.8.8 -F x@example.net + Check Rspamc ${result} R_SPF_FAIL + +SPF FAIL UNRESOLVEABLE MX + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml + ... -i 1.2.3.4 -F x@fail6.org.org.za + Check Rspamc ${result} R_SPF_FAIL + +SPF FAIL UNRESOLVEABLE A + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml + ... -i 1.2.3.4 -F x@fail7.org.org.za + Check Rspamc ${result} R_SPF_FAIL + +SPF DNSFAIL FAILED A + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml + ... -i 1.2.3.4 -F x@fail8.org.org.za + Check Rspamc ${result} R_SPF_DNSFAIL + +SPF DNSFAIL FAILED MX + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml + ... -i 1.2.3.4 -F x@fail9.org.org.za + Check Rspamc ${result} R_SPF_DNSFAIL + +SPF DNSFAIL FAILED RECORD + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml + ... -i 1.2.3.4 -F x@www.dnssec-failed.org + Check Rspamc ${result} R_SPF_DNSFAIL + +SPF PASS INCLUDE + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml + ... -i 8.8.8.8 -F x@pass1.org.org.za + Check Rspamc ${result} R_SPF_ALLOW + +SPF PTRS + ${result} = Scan Message With Rspamc /dev/null + ... -i 88.99.142.95 -F foo@crazyspf.cacophony.za.org + Check Rspamc ${result} R_SPF_ALLOW + ${result} = Scan Message With Rspamc /dev/null + ... -i 128.66.0.1 -F foo@crazyspf.cacophony.za.org + Check Rspamc ${result} R_SPF_FAIL + ${result} = Scan Message With Rspamc /dev/null + ... -i 209.85.216.182 -F foo@crazyspf.cacophony.za.org + Check Rspamc ${result} R_SPF_FAIL + #${result} = Scan Message With Rspamc /dev/null + #... -i 98.138.91.166 -F foo@crazyspf.cacophony.za.org + #Check Rspamc ${result} R_SPF_ALLOW + #${result} = Scan Message With Rspamc /dev/null + #... -i 98.138.91.167 -F foo@crazyspf.cacophony.za.org + #Check Rspamc ${result} R_SPF_ALLOW + #${result} = Scan Message With Rspamc /dev/null + #... -i 98.138.91.168 -F foo@crazyspf.cacophony.za.org + #Check Rspamc ${result} R_SPF_ALLOW + +SPF PERMFAIL REDIRECT WITHOUT SPF + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim4.eml + ... -i 192.0.2.1 -F a@fail1.org.org.za + Check Rspamc ${result} R_SPF_DNSFAIL + +SPF EXTERNAL RELAY + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/external_relay.eml + Should contain ${result.stdout} R_SPF_ALLOW (1.00)[+ip4:37.48.67.26] + +SPF UPPERCASE + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/dmarc/bad_dkim1.eml + ... -i 8.8.8.8 -F x@fail11.org.org.za + Check Rspamc ${result} R_SPF_ALLOW + +*** Keywords *** +SPF Setup + ${PLUGIN_CONFIG} = Get File ${TESTDIR}/configs/dmarc.conf + Set Suite Variable ${PLUGIN_CONFIG} + Generic Setup PLUGIN_CONFIG diff --git a/test/functional/cases/125_map_reload.robot b/test/functional/cases/125_map_reload.robot index 2afaf2ab2..4973b090d 100644 --- a/test/functional/cases/125_map_reload.robot +++ b/test/functional/cases/125_map_reload.robot @@ -18,7 +18,6 @@ CHECK HIT AND MISS Check Rspamc ${result} MAP_SET_HIT_AND_MISS (1.00)[example.com] WRITE NEW MAP - Sleep 1s Wait for new time ${TMP_FILE} = Make Temporary File Copy File ${TESTDIR}/configs/maps/domains.list.2 ${TMP_FILE} Move File ${TMP_FILE} ${MAP_FILE} diff --git a/test/functional/cases/161_p0f.robot b/test/functional/cases/161_p0f.robot index 9023b639d..edf47da4c 100644 --- a/test/functional/cases/161_p0f.robot +++ b/test/functional/cases/161_p0f.robot @@ -19,6 +19,8 @@ p0f MISS Run Dummy p0f ${result} = Scan Message With Rspamc ${MESSAGE} --ip 1.1.1.1 Check Rspamc ${result} P0F + Check Rspamc ${result} Linux 3.11 and newer + Check Rspamc ${result} Ethernet or modem Check Rspamc ${result} WINDOWS inverse=1 Check Rspamc ${result} P0F_FAIL inverse=1 Shutdown p0f @@ -26,11 +28,11 @@ p0f MISS p0f HIT Run Dummy p0f ${P0F_SOCKET} windows ${result} = Scan Message With Rspamc ${MESSAGE} --ip 1.1.1.2 - Check Rspamc ${result} P0F inverse=1 + Check Rspamc ${result} P0F Check Rspamc ${result} P0F_FAIL inverse=1 - Check Rspamc ${result} ETHER - Check Rspamc ${result} DISTGE10 + Check Rspamc ${result} Ethernet or modem Check Rspamc ${result} WINDOWS + Check Rspamc ${result} Linux 3.11 and newer inverse=1 Shutdown p0f p0f MISS CACHE @@ -56,9 +58,10 @@ p0f NO REDIS Run Dummy p0f ${result} = Scan Message With Rspamc ${MESSAGE} --ip 1.1.1.5 Check Rspamc ${result} P0F - Check Rspamc ${result} ETHER - Check Rspamc ${result} DISTGE10 + Check Rspamc ${result} Linux 3.11 and newer + Check Rspamc ${result} Ethernet or modem Check Rspamc ${result} P0F_FAIL inverse=1 + Should Contain ${result.stdout} distance=10 Shutdown p0f p0f NO MATCH @@ -80,7 +83,7 @@ p0f BAD RESPONSE Run Dummy p0f ${P0F_SOCKET} windows bad_response ${result} = Scan Message With Rspamc ${MESSAGE} --ip 1.1.1.8 Check Rspamc ${result} P0F_FAIL - Check Rspamc ${result} Malformed Response + Check Rspamc ${result} Error getting result: IO read error: connection terminated Check Rspamc ${result} WINDOWS inverse=1 Shutdown p0f diff --git a/test/functional/cases/210_clickhouse/001_migration.robot b/test/functional/cases/210_clickhouse/001_migration.robot index 816162409..7d8d0bb82 100644 --- a/test/functional/cases/210_clickhouse/001_migration.robot +++ b/test/functional/cases/210_clickhouse/001_migration.robot @@ -40,13 +40,14 @@ Migration Column should exist rspamd Groups.Scores Schema version should be 8 -Retention - Upload new schema ${TESTDIR}/data/schema_2/schema.sql - Insert data rspamd ${TESTDIR}/data/schema_2/data.rspamd.sql - Assert rows count rspamd 56 - Prepare rspamd - Sleep 2 #TODO: replace this check with waiting until migration finishes - Assert rows count rspamd 30 +# Eventually broken +#Retention +# Upload new schema ${TESTDIR}/data/schema_2/schema.sql +# Insert data rspamd ${TESTDIR}/data/schema_2/data.rspamd.sql +# Assert rows count rspamd 56 +# Prepare rspamd +# Sleep 2 #TODO: replace this check with waiting until migration finishes +# Assert rows count rspamd 30 *** Keywords *** Clickhouse Setup diff --git a/test/functional/cases/280_rules.robot b/test/functional/cases/280_rules.robot index 882fc7275..b6b347313 100644 --- a/test/functional/cases/280_rules.robot +++ b/test/functional/cases/280_rules.robot @@ -13,6 +13,8 @@ ${MESSAGE2} ${TESTDIR}/messages/fws_fp.eml ${MESSAGE3} ${TESTDIR}/messages/fws_tp.eml ${MESSAGE4} ${TESTDIR}/messages/broken_richtext.eml ${MESSAGE5} ${TESTDIR}/messages/badboundary.eml +${MESSAGE6} ${TESTDIR}/messages/pdf_encrypted.eml +${MESSAGE7} ${TESTDIR}/messages/pdf_js.eml ${URL_TLD} ${TESTDIR}/../lua/unit/test_tld.dat ${RSPAMD_SCOPE} Test @@ -45,6 +47,15 @@ Broken boundary ${result} = Scan Message With Rspamc ${MESSAGE4} Check Rspamc ${result} BROKEN_CONTENT_TYPE +PDF encrypted + ${result} = Scan Message With Rspamc ${MESSAGE6} + Check Rspamc ${result} PDF_ENCRYPTED + +PDF javascript + ${result} = Scan Message With Rspamc ${MESSAGE7} + Check Rspamc ${result} PDF_JAVASCRIPT + + *** Keywords *** Rules Setup ${PLUGIN_CONFIG} = Get File ${TESTDIR}/configs/regexp.conf diff --git a/test/functional/cases/340_surbl.robot b/test/functional/cases/340_surbl.robot index 5137223bf..8c7fcc3c1 100644 --- a/test/functional/cases/340_surbl.robot +++ b/test/functional/cases/340_surbl.robot @@ -11,6 +11,12 @@ ${RSPAMD_SCOPE} Suite ${URL_TLD} ${TESTDIR}/../lua/unit/test_tld.dat *** Test Cases *** +SURBL resolve ip + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/url7.eml + Should Contain ${result.stdout} URIBL_SBL_CSS (1.00)[8.8.8.9:example.ru + Should Contain ${result.stdout} URIBL_XBL (1.00)[8.8.8.8:example.ru + Should Contain ${result.stdout} URIBL_PBL (1.00)[8.8.8.8:example.ru + SURBL Example.com domain ${result} = Scan Message With Rspamc ${TESTDIR}/messages/url4.eml Should Contain ${result.stdout} RSPAMD_URIBL @@ -141,6 +147,9 @@ SURBL IDN Punycode domain Should Not Contain ${result.stdout} DBL_PHISH Should Not Contain ${result.stdout} URIBL_BLACK +SURBL html entity ­ + ${result} = Scan Message With Rspamc ${TESTDIR}/messages/url10.eml + Should Contain ${result.stdout} RSPAMD_URIBL *** Keywords *** Surbl Setup diff --git a/test/functional/cases/350_magic.robot b/test/functional/cases/350_magic.robot index ecb08390c..b3a760086 100644 --- a/test/functional/cases/350_magic.robot +++ b/test/functional/cases/350_magic.robot @@ -68,4 +68,6 @@ Magic detections bundle 1 Should Contain ${result.stdout} MAGIC_SYM_JAR_52 Should Contain ${result.stdout} MAGIC_SYM_APK_53 Should Contain ${result.stdout} MAGIC_SYM_BAT_54 + Should Contain ${result.stdout} MAGIC_SYM_ICS_55 + Should Contain ${result.stdout} MAGIC_SYM_VCF_56 diff --git a/test/functional/configs/dmarc.conf b/test/functional/configs/dmarc.conf index ddfc1ac61..08a542c70 100644 --- a/test/functional/configs/dmarc.conf +++ b/test/functional/configs/dmarc.conf @@ -1 +1,4 @@ dmarc { } +spf { + external_relay = 192.168.1.1; +}
\ No newline at end of file diff --git a/test/functional/configs/multimap.conf b/test/functional/configs/multimap.conf index 804284b11..b12f796e8 100644 --- a/test/functional/configs/multimap.conf +++ b/test/functional/configs/multimap.conf @@ -1,5 +1,6 @@ asn { } +spf {} redis { servers = "${REDIS_ADDR}:${REDIS_PORT}"; expand_keys = true; diff --git a/test/functional/configs/plugins.conf b/test/functional/configs/plugins.conf index 1587ce584..025f6c55f 100644 --- a/test/functional/configs/plugins.conf +++ b/test/functional/configs/plugins.conf @@ -1,5 +1,5 @@ options = { - filters = ["spf", "dkim", "regexp"] + filters = [ "dkim", "regexp"] url_tld = "${URL_TLD}" pidfile = "${TMPDIR}/rspamd.pid" lua_path = "${INSTALLROOT}/share/rspamd/lib/?.lua" @@ -14,6 +14,16 @@ options = { replies = ["k=ed25519; p=yi50DjK5O9pqbFpNHklsv9lqaS0ArSYu02qp1S0DW1Y="]; }, { + name = "brisbane._domainkey.football.example.com"; + type = txt; + replies = ["v=DKIM1; k=ed25519; p=11qYAYKxCrfVS/7TyWQHOg7hcvPapiMlrwIaaPcHURo="]; + }, + { + name = "test._domainkey.football.example.com"; + type = txt; + replies = ["v=DKIM1; k=rsa; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDkHlOQoBTzWRiGs5V6NpP3idY6Wk08a5qhdR6wy5bdOKb2jLQiY/J16JYi0Qvx/byYzCNb3W91y3FutACDfzwQ/BC/e/8uBsCR+yz1Lxj+PL6lHvqMKrM3rG4hstT5QjvHO9PzoxZyVYLzBfO2EeC3Ip3G+2kryOTIKT+l/K4w3QIDAQAB"], + }, + { name = "dkim._domainkey.cacophony.za.org", type = "txt"; replies = ["v=DKIM1; k=rsa; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDXtxBE5IiNRMcq2/lc2zErfdCvDFyQNBnMjbOjBQrPST2k4fdGbtpe5Iu5uS01Met+dAEf94XL8I0hwmYw+n70PP834zfJGi2egwGqrakpaWsCDPvIJZLkxJCJKQRA/zrQ622uEXdvYixVbsEGVw7U4wAGSmT5rU2eU1y63AlOlQIDAQAB"]; @@ -199,6 +209,11 @@ options = { replies = ["v=spf1 redirect=fail5.org.org.za"]; }, { + name = "fail11.org.org.za", + type = "txt"; + replies = ["v=sPF1 ip4:8.8.8.8 -all"]; + }, + { name = "fail5.org.org.za", type = "txt"; replies = ["v=spf1 OMGBARF"]; @@ -209,6 +224,16 @@ options = { replies = ["v=spf1 ip4:8.8.8.8 a -all"]; }, { + name = "trusted.com", + type = "txt"; + replies = ["v=spf1 ip4:192.168.1.1"]; + }, + { + name = "external.com", + type = "txt"; + replies = ["v=spf1 ip4:37.48.67.26"]; + }, + { name = "co.za", type = "txt"; rcode = 'norec'; @@ -540,6 +565,12 @@ options = { replies = ["127.0.0.2"]; }, { + # testtest.com + name = "rcf1ecxtxrrpfncqzsdaiezjkf7f1rzz.test.uribl"; + type = a; + replies = ["127.0.0.2"]; + }, + { name = "jhcszdsmo3wuj5mp8t38kdisdmr3ib3q.test.uribl"; type = a; replies = ["127.0.0.2"]; @@ -667,6 +698,7 @@ worker { modules { path = "${TESTDIR}/../../src/plugins/lua/" } +spf {} lua = "${TESTDIR}/lua/test_coverage.lua"; lua = "${INSTALLROOT}/share/rspamd/rules/rspamd.lua" ${PLUGIN_CONFIG} diff --git a/test/functional/configs/whitelist.conf b/test/functional/configs/whitelist.conf index 29df415ab..d7592fc35 100644 --- a/test/functional/configs/whitelist.conf +++ b/test/functional/configs/whitelist.conf @@ -1,5 +1,5 @@ dmarc {} - +spf {} whitelist { rules { diff --git a/test/functional/messages/dmarc/bad_dkim4.eml b/test/functional/messages/dmarc/bad_dkim4.eml new file mode 100644 index 000000000..6d57aaf5b --- /dev/null +++ b/test/functional/messages/dmarc/bad_dkim4.eml @@ -0,0 +1,17 @@ +Date: Tue, 09 Aug 2016 10:01:27 +0200 +Message-ID: <20160809100127@rspamd.tk> +From: Rspamd <a@fail1.org.org.za> +To: foo@rspamd.tk +Subject: hello +Content-Type: text/plain; charset=utf-8; format=flowed; DelSp=Yes +MIME-Version: 1.0 +Content-Disposition: inline +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=yoni.za.org; s=testdkim1; + t=1470729879; h=from:subject:date:message-id:to:mime-version:content-type; + bh=7HkRgYnNru3SR2EWfgWU8yhM0MOH6ZZrPoEIgNIh8wc=; + b=kTIV4jcgv9sWFh2JFrS/+PcNxiloituqjmHHqeJOTfa+/9C+Er8BjnMysTJyYVq36Gnv0OZDgLr3Yy4YP5Lzbt1M9ZdN5cJqO7yn1N7wyaGfkt++b09rIYBy5Dkk7OWyP3cDThqDzv8C9heSvqBSEsirFsbt3Wx2g/hWiJlnjew= + + +hello + + diff --git a/test/functional/messages/external_relay.eml b/test/functional/messages/external_relay.eml new file mode 100644 index 000000000..14ac8ae2c --- /dev/null +++ b/test/functional/messages/external_relay.eml @@ -0,0 +1,16 @@ +Return-path: root@external.com +Received: from trusted.com (trusted.com [192.168.1.1]) by + example.com with LMTP id MJX+NoRd5F2caAAAzslS3g for <test@example.com>; + Thu, 5 Dec 2019 18:22:18 +0300 +Received: from external.com (external.com [37.48.67.26]) by + trusted.com (Postfix) with ESMTP id C018DA00021; + Thu, 5 Dec 2019 18:22:18 +0300 +To: test@example.com +From: root@external.com +Subject: test Sat, 26 Jan 2019 12:04:58 +0100 +Message-Id: <20190126120458.015328@srv.example.com> +Date: Sat, 26 Jan 2019 12:04:58 +0100 +MIME-Version: 1.0 +Content-Type: multipart/mixed + +dsadas diff --git a/test/functional/messages/gargantua.eml b/test/functional/messages/gargantua.eml index 11cc95d23..d86f99ad5 100644 --- a/test/functional/messages/gargantua.eml +++ b/test/functional/messages/gargantua.eml @@ -23466,4 +23466,34 @@ KioqKioqKioqKioqKioqKioqKioNCmVjaG8gICogICAgICAgIEdvb2RieWUhICAg ICAgICAqDQplY2hvICAqKioqKioqKioqKioqKioqKioqKioqKioqKg0KZWNoby4N CnBhdXNlDQpzZXQgcGF0aD0lb2xkcGF0aCUNCmNvbG9y +--XXX +Content-Type: text/calendar +Content-Transfer-Encoding: base64 +X-Real-Type: ics + +QkVHSU46VkNBTEVOREFSDQpQUk9ESUQ6LS8vVGVzdCBJbmMvL1Rlc3QgQ2FsZW5k +YXIgNzAuOTA1NC8vRU4NClZFUlNJT046Mi4wDQpDQUxTQ0FMRTpHUkVHT1JJQU4N +Ck1FVEhPRDpQVUJMSVNIDQpYLVdSLUNBTE5BTUU6dGVzdEB0ZXN0LmNvbQ0KWC1X +Ui1USU1FWk9ORTpFdXJvcGUvTW9zY293DQpCRUdJTjpWRVZFTlQNCkRUU1RBUlQ6 +MjAxNjA1MTZUMDUwMDAwWg0KRFRFTkQ6MjAxNjA1MTZUMDUxNTAwWg0KRFRTVEFN +UDoyMDE5MTEyMVQxNDAxMTFaDQpPUkdBTklaRVI7Q049dGVzdGVyQHRlc3QuY29t +Om1haWx0bzp0ZXN0ZXJAdGVzdC5jb20NClVJRDo4M2YybThjZGw1YXNjZmtmbHBs +dDhxdWp2NEB0ZXN0LmNvbQ0KQVRURU5ERUU7Q1VUWVBFPUlORElWSURVQUw7Uk9M +RT1SRVEtUEFSVElDSVBBTlQ7UEFSVFNUQVQ9QUNDRVBURUQ7Q049dGVzdA0KIEBn +bWFpbC5jb207WC1OVU0tR1VFU1RTPTA6bWFpbHRvOnRlc3RAdGVzdC5jb20NCkNS +RUFURUQ6MjAxNjA0MjBUMjAyMzM2Wg0KREVTQ1JJUFRJT046DQpMQVNULU1PRElG +SUVEOjIwMTYwNDIwVDIwMjMzNloNCkxPQ0FUSU9OOg0KU0VRVUVOQ0U6MA0KU1RB +VFVTOkNPTkZJUk1FRA0KU1VNTUFSWTpaYXJ5YWRrYQ0KVFJBTlNQOk9QQVFVRQ0K +RU5EOlZFVkVOVA0K + +--XXX +Content-Type: text/vcard +Content-Transfer-Encoding: base64 +X-Real-Type: vcf + +QkVHSU46VkNBUkQNClZFUlNJT046My4wDQpGTjrQkNCy0YLQvtC60L7RgNC10LXR +hg0KTjo70JDQstGC0L7QutC+0YDQtdC10YY7OzsNClRFTDtUWVBFPUNFTEw6MjYy +LTc3Mg0KVEVMO1RZUEU9SE9NRTorNzkwMDUyNjI3NzINCkVORDpWQ0FSRA0K + + --XXX--
\ No newline at end of file diff --git a/test/functional/messages/ics.eml b/test/functional/messages/ics.eml new file mode 100644 index 000000000..54563d041 --- /dev/null +++ b/test/functional/messages/ics.eml @@ -0,0 +1,53 @@ +Return-Path: <root@srv.example.com> +To: test@example.com +From: root@srv.example.com +Subject: test Sat, 26 Jan 2019 12:04:58 +0100 +Message-Id: <20190126120458.015328@srv.example.com> +Date: Sat, 26 Jan 2019 12:04:58 +0100 +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="_----------=_1574345186400022" + +--_----------=_1574345186400022 +Content-Type: multipart/alternative; boundary="_----------=_1574345186400023" + +This is a multi-part message in MIME format. + +--_----------=_1574345186400023 +Content-Disposition: inline +Content-Transfer-Encoding: quoted-printable +Content-Type: text/plain; charset="utf-8"; format="flowed" + + + +--_----------=_1574345186400023 +Content-Disposition: inline +Content-Transfer-Encoding: quoted-printable +Content-Type: text/html; charset="utf-8" + +<div><br data-mce-bogus=3D"1"></div>= + +--_----------=_1574345186400023-- + +--_----------=_1574345186400022 +Content-Disposition: attachment; filename="test2.ics" +Content-Transfer-Encoding: base64 +Content-Type: text/calendar; name="test.ics" + +QkVHSU46VkNBTEVOREFSDQpNRVRIT0Q6UFVCTElTSA0KVkVSU0lPTjoyLjANClgt +V1ItQ0FMTkFNRTp0ZXN0DQpQUk9ESUQ6LS8vQXBwbGUgSW5jLi8vTWFjIE9TIFgg +MTAuMTQuMy8vRU4NClgtQVBQTEUtQ0FMRU5EQVItQ09MT1I6IzFEOUJGNg0KWC1X +Ui1USU1FWk9ORTpFdXJvcGUvTW9zY293DQpDQUxTQ0FMRTpHUkVHT1JJQU4NCkJF +R0lOOlZUSU1FWk9ORQ0KVFpJRDpFdXJvcGUvTW9zY293DQpCRUdJTjpTVEFOREFS +RA0KVFpPRkZTRVRGUk9NOiswMjMwMTcNCkRUU1RBUlQ6MjAwMTAxMDFUMDAwMDAw +DQpUWk5BTUU6R01UKzMNClRaT0ZGU0VUVE86KzAyMzAxNw0KRU5EOlNUQU5EQVJE +DQpFTkQ6VlRJTUVaT05FDQpCRUdJTjpWRVZFTlQNCkNSRUFURUQ6MjAxOTExMjRU +MTA0ODE3Wg0KVUlEOkU3QUIxQkY2LTkyOTctNDJFNS05Njc4LTA1Nzk3QzM0OEVF +NA0KRFRFTkQ7VFpJRD1FdXJvcGUvTW9zY293OjIwMTkxMTIyVDEwMDAwMA0KVFJB +TlNQOk9QQVFVRQ0KWC1BUFBMRS1UUkFWRUwtQURWSVNPUlktQkVIQVZJT1I6QVVU +T01BVElDDQpTVU1NQVJZOmh0dHA6Ly90ZXN0LmNvbQ0KTEFTVC1NT0RJRklFRDoy +MDE5MTEyNFQxMDQ4MzFaDQpEVFNUQU1QOjIwMTkxMTI0VDEwNDgzMVoNCkRUU1RB +UlQ7VFpJRD1FdXJvcGUvTW9zY293OjIwMTkxMTIyVDA5MDAwMA0KU0VRVUVOQ0U6 +MA0KRU5EOlZFVkVOVA0KRU5EOlZDQUxFTkRBUg0K + + +--_----------=_1574345186400022--
\ No newline at end of file diff --git a/test/functional/messages/pdf_encrypted.eml b/test/functional/messages/pdf_encrypted.eml new file mode 100644 index 000000000..190903f3e --- /dev/null +++ b/test/functional/messages/pdf_encrypted.eml @@ -0,0 +1,692 @@ +Return-Path: <root@srv.example.com> +To: test@example.com +From: root@srv.example.com +Subject: test Sat, 26 Jan 2019 12:04:58 +0100 +Message-Id: <20190126120458.015328@srv.example.com> +Date: Sat, 26 Jan 2019 12:04:58 +0100 +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="_----------=_1574345186400022" + +--_----------=_1574345186400022 +Content-Type: multipart/alternative; boundary="_----------=_1574345186400023" + +This is a multi-part message in MIME format. + +--_----------=_1574345186400023 +Content-Disposition: inline +Content-Transfer-Encoding: quoted-printable +Content-Type: text/plain; charset="utf-8"; format="flowed" + + +--_----------=_1574345186400023 +Content-Disposition: inline +Content-Transfer-Encoding: quoted-printable +Content-Type: text/html; charset="utf-8" + +<div><br data-mce-bogus=3D"1"></div>= + +--_----------=_1574345186400023-- + +--_----------=_1574345186400022 +Content-Disposition: attachment; filename="990777.pdf" +Content-Transfer-Encoding: base64 +Content-Type: application/pdf; name="990777.pdf" + +JVBERi0xLjUKJeLjz9MKMSAwIG9iaiAKPDwKL0xhbmcgKMdWjlC/KQovUGFnZXMgMiAwIFIKL1R5 +cGUgL0NhdGFsb2cKPj4KZW5kb2JqIAoyIDAgb2JqIAo8PAovS2lkcyBbMyAwIFJdCi9Db3VudCAx +Ci9UeXBlIC9QYWdlcwo+PgplbmRvYmogCjMgMCBvYmogCjw8Ci9Hcm91cCAKPDwKL0NTIC9EZXZp +Y2VSR0IKL1R5cGUgL0dyb3VwCi9TIC9UcmFuc3BhcmVuY3kKPj4KL1BhcmVudCAyIDAgUgovUmVz +b3VyY2VzIAo8PAovWE9iamVjdCAKPDwKL1gwIDQgMCBSCj4+Ci9Gb250IAo8PAovRjIgNSAwIFIK +L0YxIDYgMCBSCj4+Cj4+Ci9NZWRpYUJveCBbMCAwIDYxMiA3OTJdCi9Db250ZW50cyA3IDAgUgov +VHlwZSAvUGFnZQo+PgplbmRvYmogCjcgMCBvYmogCjw8Ci9GaWx0ZXIgL0ZsYXRlRGVjb2RlCi9M +ZW5ndGggMjk2NAo+PgpzdHJlYW0KSv0zY+7m39IMcD92XE9hHmKDGw5NINuRdGhau3oGWLi54qga +NuDlMWNdgnTziYVZIRKcIh8AhJV+ySjqg5SSeAFmg3POIzBWnjMBd10mpLmvtmmsnjaJTCfzv1MP +UmUs45NDAtnAKSeH47ARczIiXyEdo/tlL3QQicDb7sQcvvr6DedaZZ3+Vu1H8B4/HloiJjFT7sxa +okkAt60kPCtNsQoQUlpVt99KXhnIBM8u1E0voCK2xLbUKGTU5zMprCTeXhfclPj4yFi7X9m4IJRW +rXoFTR/SYNyA+t0LNN9Vx4a1oMatRenyZ5NvGQHktNiYrKGxK7SKcKtGqwwl/39CNbHsWGNqcKd3 +bpC8P8ZiN6MgTI8BUFJdv19x3xF57zmt0W91GbKRBR67xaGTrr7GI18eoca/6FPbQcPM+H1JiQ5O +ZLk35I6XIlD3prYHgXC9dij2YcCob8l6NkPIeVwd1LV0RlV+EnaDyjinPfpjr4YH/pPAuOAh47br +tSl1xdH6gWlV++jJCgTmHhsPwPEuOEteYl1Zr3vaVVal0LAqJYul8j8OxB4dKDcEsepAtFnAvi5t +1Kf/FxBqBSqIodzxquUqYRvEPqGdpt11O1BrAwzmA8nLSzo/pkPljRSEmImdhO7842PpJ4yK6gQp +Shh2sx358J5q0YQTZAXL9L5qZiH9fM+jy26KiZ5sNHpuqAxqm6KWxO+OfW5eJFL4EJq5WZ4irkMv +Cc58SO9fY8tT1HMrMVpMmaOMb3JfzBfaOOrSaBS0vBTFDPbJh+PTu0TfoBk5Bxqe1ydTEFrdtNJk +pohHw/0UgcNaOXeQM2e8Qe7WegWVy/4d7JNVr8KqHxM03TMbEBBom0NpDonW1eRAbq4b8DQPZUOW +1U9/BJPuW3u1C+fDO57qTH0ynANuc+kygxoZN8DBJv1h9wGBaVrYt5qOLKQdXZBXCzTgXGqNoGEH +UHuxxBD90YVYg2PfCFd8RRGeyiMOlljEVe2WUmHctvT1RzG29VRqlFpjQBKDuCP1sIOOmawj/NBH +Oe8ewZ8aXDRa35L7mb+H4nVBtRenV4UnEXT5HPEAz2YA9GxakL4fwTVkcEsO/q49LXrrIX7Sl5Ja +yBqtGgQAlljureXq3LorO9HgCE9rtX4OdD7/Qul+LxfIQCAYQVqVQtGwO9JbBp/LA/wgE3qKLBAD +v40uNvR7j9LRfRkQGK2q3yKO5VoK0DR5YiHMHvylpvt2+nvGZ3+iScjJXivAmNPHye4+K2rzuECf +O7fLRed0E7gkHXaaGAiTaAsS9EAxFjmHc88yKWt8PmuXfdG5b6JqvizDgGX0bTHbLdpeMTZG1cGH +9tB4K4jPIyktyQp/1GU61mstggxPCt33v9/KmHuwk27ON0FsrXE7FbAkCi8JoNtBJ8+pQdJ6Wlu7 +xhvoSEshHqsRsudANCR8YHCfI1dsZxd+sqgb5EA/9dfKuTELp6KFWaVmMR8idgaMbTpglNe/+bPi +vkH+Yb+g1ZRkJjLcuUiMaM+SdCX6VDgJg5sRWihfXEqZQ+/Ld/NishGgY6nTW9Y49eXWsYWh2pgL +/t6G4s/EpYuLYZy2hQkZfRNU0TFMH8oYoXSS7cnZCDI9MLmTZzowUS7YqvOn/xBbXma5Dhq6hQtF +s1X2CF+3vTTAQYjVMyf+eWIm52LHlzmNg3Sj4LmGMy2IwMODZ7HDzBLBSNMqSk2SIeJagh3xTOvR +8uZgK7GPXX/PJjTOgR3IkYG8SPuIMMV2HTXMYsWH1L5sSRnhwkW+7cxqzy2eV/fGbnY34GEuHN/H +bs+ZX+TO+WsYIbhalN+0ZriqoAn1U3//ScOP4Kr1sw3bXcvRDXN2m3cAuAEkpoaGKULybLpfm1Av +C8pLekhtS+sRX6+yHI23Asmh2s5INJLglOZigeQEBMxjAq2osjr8RUuwdHc4G9Az++ibTUr4uqJ0 +JP84KWoWAkrbtqhS7laN435YwWwavOECA4IZ74fx+ofm3mP1pGvMiO7k0eLkGskJnzXMQf8VfHSo +kt4aS/62n4wuppjBgPaZv1vGYhK3Wl2QJp3x8u+aRDNZ6IWqxD5L4TAT1ovuyAjoQBG1lHhZJ78s +IFCkb5jIPeoaookRKmPgVLbT7h062HJdCHJfeBZ1MfTY16wOFEURl/KapT/Ydadqk433sJCtgbWy +H3A3DfrW+VtkkZNjZlggHOo4fTq59q3pkSlfYl0s8S3hVVOZwOZew8GAxuTeJavggrsJMfPMLy2+ +Ps4q1fcJZFV+4hjiKt1SN5X6zKwXWKoxINjOzZBRjTn1+xrhq5fFwJ1dPatlPNdtWz0VGnVKWe6T +WZJxytspffXANXIxbtaaUeOwaVuLIiT5ypZ3maspUufsMH6jooSdvrt3CweWqDLzVeQbKrFNBECx +Au4IXV8V752NeUzpSE4nLqKgwnBPp7UCNXYIzu7NYbwT7PP1cAbqTL5vSVOjBpeeMgw6tVddNt3h +XMiAQ03ndKFnUuBqhdt6B6rYJdl9SzZ31x0sA7cdEKRzdod2VbUCiQmjC5FQ6PWYY5pu3mhgNaaS +rUQCUOlt+UisQIDoN+Z1ee2oWFkpkcnhRD7QlaueAaB9nNFh7AFseftBifYglpxOART6/jYcqrfa +CCv9PPgK0w0T/DXxENejkSW6mP6LyfdbJ8+P/NezhTAcIXhzKMSkgetsTb6NJCwpoCqCqqyDbsyV +Qyoc1/fjHM/UEQTA2XEWby6S9IOnJe8JeL713t4NaYpD442lQwZXXWH1B/UgE021dAAxx5LF0puv ++DD+mztSQw4Y0rMPaDtUfTX/NRXAbxF7QpLA0dB0pTtGbSItqgwDFEKL8NU4pzw/QuwzLEC8MQY4 +odqHVWR/d8GUBkJEPNV+6K6DCkCw5nbaNnDw+JFy4LlUSi+Re5d0YYAc+2UWUmyOXkk2LrhGtDtk +kLmWY+nXWuQXi14Y70H3hjfbfvMJVVi+RKv/CV888BmKk8QIkkELIvh6m+YBWSaaBtTH/o6xnW2Z +Ur0Hxp4dBKbgH+nw2atxcHJ4X1BshfTYRjAKHV6cS1cbpwPHxW+8cgfLgdvACI/SK8nQ1zEB5FqG +sPIAbQspTnRkZogO1TdioGDOrWrnsPqk96DVlPcFL3HcgRsT20V4xOogFtqsFpxR5gDFNCYx3qwR +550iIRZ6J3Yalh1Uy0bTbR3oo1Fny/xGqEhHRNgwzm/TvVJHW9l/Y6fjlZTmFhOqvGqzx/OBvFtp +W0Trt0/dUe35j9weoLHuLBhBCbVlqgUTIZx9C0EOSWJ30Qg4eqhS7HLPAJh3qWTGkqeIh4xS1wd3 +G7hUfPd24EWF7kjcYQ5eBeeBAkPFqgNpt/WO4wMNiSbmw5oXN/ayJZnIW2+zDfzfF8+PWpoFZlRz +qShf79fOW8asFml8Yqs02N6i5E4qLri5vhL6PLk/BqGEnFPBuF6cpkcHDEOvsYYLTcPmeMNjdacf +Xb8XcH/qxOKN6d0G9ODvYzMoqyg6LneIPAeBi2u1uSf1J69HnwfK4FdawJVaG87icGO7QrnNJxY0 +JV+HCF3geF19qXedKSI97EDUgc8KHxQnrZpc/3KA+smM5SjRks6C/rglrpaNxSzYjAXLc5rIaCCG +/ds1ZVGjRVLvGmP6Cm7xIPzB5VF8gymtYmmnHPN44GxWQyOnyvukDJyt4sYB0kHfedTkKeGYrELA +qEk+KlhH0vqDRUzsebIzQEfYpimaSV8iz45djF/RD4xRd34VDh5SSaN6t60+YH03r65cdXVV042w +GrhtJmeM3a83i9C94cPxldhQ8OtZmnoP+LfFLbyEZEHq1AHRQ+S49Co9NJhUzlZ+3sMqKKwuP7Hj +Y2bjOmLq4UVXzLTn+fVDlNh6k2JTVDK87NzERpZyVzFnQMv4KRml6ObfiMHMWKyzWcLYrb5MYrtD +MFgwpuqglZ8Fw+FD9wZ/+BzLv/zZCmVuZHN0cmVhbSAKZW5kb2JqIAo2IDAgb2JqIAo8PAovTGFz +dENoYXIgMTIyCi9CYXNlRm9udCAvQkFQUlBUK0NhbGlicmkKL1N1YnR5cGUgL1RydWVUeXBlCi9X +aWR0aHMgOCAwIFIKL0ZvbnREZXNjcmlwdG9yIDkgMCBSCi9FbmNvZGluZyAvV2luQW5zaUVuY29k +aW5nCi9UeXBlIC9Gb250Ci9GaXJzdENoYXIgMzIKPj4KZW5kb2JqIAo5IDAgb2JqIAo8PAovRm9u +dE5hbWUgL0JBUFJQVCtDYWxpYnJpCi9TdGVtViA1MgovRm9udEZpbGUyIDEwIDAgUgovQXNjZW50 +IDc1MAovRmxhZ3MgMzIKL0ZvbnRXZWlnaHQgNDAwCi9YSGVpZ2h0IDI1MAovRGVzY2VudCAtMjUw +Ci9BdmdXaWR0aCA1MjEKL0l0YWxpY0FuZ2xlIDAKL01heFdpZHRoIDE3NDMKL0ZvbnRCQm94IFst +NTAzIC0yNTAgMTI0MCA3NTBdCi9UeXBlIC9Gb250RGVzY3JpcHRvcgovQ2FwSGVpZ2h0IDc1MAo+ +PgplbmRvYmogCjEwIDAgb2JqIAo8PAovRmlsdGVyIC9GbGF0ZURlY29kZQovTGVuZ3RoIDE5MDcz +Ci9MZW5ndGgxIDM4OTM2Cj4+CnN0cmVhbQrYusqSy3UuFRDwkgedfQ4hAJh8HzhH5nOlUX6RoQw0 +Ucw8rnKp03J52nsi+HEmsYIYLDTyUesFVgXSUcwuB7vc45ylYtKTJVedyIj1e98w58x6whw4lHVC +z+ff6wbuQVCBq236rlL/+1GFPkyx33wMgbVcn0PRMmCe3PJvPrxu/8neNhh3o27WVhJPCJIUbjP1 +DEWrE4L3z/Qpixl4ipe8VZOmh3Ph/9TJ5s9JYmcd3V/NA79ZFmmgofkN3Ai1razQYXoNXFScQZbM +Vcwi44/qttcTkr/+hIWHtiBc1UXI0RyH3MWE6/nL1DYrLF4jPcnrLJMRSicbEAejIIhFxcbywDtF +NYIaSdDdJ3NZXRS3xdo84iNhJWTtyxSLLDB4XZbVrWij8iV7Xk+mGoRYF+3kCguRnGVDu/3ea+oc +Z64z6KZpI5WsSSiKYCQ2qMRWJSSNPtNa4jxaFwREig0hHlf6oZ5j7eYrMXvvbykws1AfNAWOi9Sn +nAf9CoFjI/btOyakHUVOZdeRkgA23yPAhq1G9lSVMIlJgb7hKJNmVoK7BoG2Dbp3esjoANahOzZw +xsvqjhlQGLthE69FFfD54oOtKZ1kd3FAHi33568mnHB1CBDM2mZe3xqVeCse2uRxrO1CeStW/bqo +bk6JFI60oyISb+J5yS+Agk3qmp4/jw2o9Pt9O7b7kLIwgv/x8SpKIG4OW97h5g8enG69L8dPCF1D +hyf/BpiNUZNl8tTlNDOQ63NOTc1vvuFTrtbifIletzbrPxM137DOnGNLHjA+oRaW7DLnhPh3KdBv +JBITIFFsRY85KBeN1GF/FD6b4blMJXyfAhRHixju0d4PcppPmDwSnMTXCrXJmJJ0h2lziRYuVDur +o//PKrekvbD6qmXjvFq1xsBJ+3hiNyxzRy0gfy4dGyRuvf+6bDWsvlvPkqMX8y/yYlfg6FVn3vR/ +Ou9jT5omWLVbNLbdwLeh+4p3M04FBsYSy3aL8FKkB06Lsa04AcJPCIfxWnZvDpsyF2HkHzI3mxCW +cbKRUvBSub/6ib8diw/oB5SKEclS+DMgfsmThg13Co4n7w3pbueZCHoi7aNZipIQs8gR6WQKCdXV +ftgkFBdy+YFl8f6Ht9dcccHHL52EYATg9wq9CyW8GKNMMY4a/4MF5mjUjibVEqcHQgY/A4qAVLn7 +ZqGSVxfAmycq5VD1cM/7Xqg0kwbMzw9W9yypzatGNGXCqnKzOX4qUxWtruQPnKnAm8BfuJ0Jxisx +y7t/Fjp3TyP+U8qIzdi+jsW1YhHrDJSfXgr7hAfhdynoxgt7yX+zG/X+H0VdvIHuscSuPclWCUg9 +TJy/KnB9BzYNjYUfrBdZYrNzkW2XFt7JocfEJv+Mf/0x+Qr6NFOrwmWih26tiUiHCC07+7NXyIp3 +QUR4vYnBvwvHUsuJOXEGp8pPBLyNZp+1cMdoP2CyarLplSkPLHVFY77nhhFdjvE27oH6s56NPxs/ +JUsM1T85LbDITk1EuGG4GHqyur6DJQ1GaU2cAhGpH+zqUAFJdCAmWaEH6MVKQJKx57E2oQ9lsR9j +zCjBStkSyKVfDHteDM6lmCJfUKf/wcIE1kwn2BscALMvhs//r+mX5HtZRh7HU+hO2PiKhyqilWGK +5TXzws6f42s9tRjRt8DbyTDmRO4vNeiJwZp3MQCopEX4AqalTPSUAWInT53YfkRpXad2HmsRl/V3 +CkQRMR2wTBv4z/AZCEHte0fhww7m1B738wShKECh7SE67yJ/iMnYOvJ2BgB8KFf7LJM7p0V8NnAu +OTHyhaRavbuZto1YfKei8J21+dOASUsqoWEEbg0+i78qcvLUeKaHKC7/9kkQlqklV9g5/930xGLI +9Tz1XvK/iIe8etddEJXxzYVq9cQ9EGSp29mKmWj9PnGoeMiORt7vfMeKEgDNwUfH+xcX0wgCiJfL +4r1RcP5CNw2s3sgwj53sRO0n0aeosJCYlchoIa5Min3WxVw7JAmSrgcCy7ghKH7lXBKXbOQ3oICT +w1poPrWOkkl4p1C5JMYAe2SKgIfU4zChWRfWpF2d6r1DSCiv/Tj6FbxItmht+Vgwsts5+hq6U91+ +4vYb9Wo3pJe9N9PfhyP1KMNmBHgF3/tVhaVIf1yLhmkHFabqD0uUOIYcMNnPNIQl0WKk/T2uoBn2 +2nTznWdZWSFzG74huW6X7aDhQyCWSiPbWh/qrY6YcPaHeaIh4VoD2PR+H+UlOZT8h7ct+LKcLXwm +C5BPm1cSeqcMaDouCNhNx4+PcQmbK4+w/rlI3SiObeBxpbaVdP/zz+W7JxIpzg57/xFkgUVJXp77 +IEChIu5Y9z1w/Q8SOaeMhH6jbRprK/VNo0WGHXq/AjqLE0x9//MgzgzOw15h/EsyxhEvAPCP6EVE +JPdoikRhEXT+dfeUl29OT+rusVtrP4LPdq3CXbJ8rKUQt/R8mHE8rHn/4HysfmUAx+T50kJgp6D2 +kGaNRDngKm4hr2nF2ZFzW5Jkfk9MgpmfUJ/mgvvsUJ8EVHofV0NvdocsncrvkN4qlUACG7GknkHa +5ZueSsPQ/WPCwbd6SAxyj9DPHUa4YYplgGvJGGrqPf68tr/NJ+elrBkxObDRn5qWocJVeV4rDyV0 +k9IZA4rjO2114cTufbxQkmgNt+cdPW1qK04NFZsddo/5nN9hCdMJteJTM9w3JyI+Nxs8FtFRzi7m +Zv2yxgpZmRRqOdColg0jqHfo+qbrFQB7kJdMe0S5BY0Xdbew4vd2N2lucY0yKwjDyGRIr9Rk6q+P +yU/t3QWjJ9H64StzyK87WsAkxV9z0IJhFkeH/a2PoKpxKCy9WknrSe5bKR/7cWr0iE5B4gvpj0zp +Vikkdo3TR0najttgnaZTcpk+ryWBRjtrVL6glgkkyVkTDS2bU+7wyhR0zFo1Jx9UtwY+Fu4M4S8b +7xM3s5C25C4PwmmE3tzz72YTPj5WT5XNblBjvjd470ZxTpMqiOWGKGXvP4NaZ8/OZBzI3P2cCOwc +Fqi+UAEF8YHg+ZpxnqWFsKPJ0tQgIIWK/ySiJTd3eM80MiGNMZbQ0ob91MlrOjrRWTm+HUhWP87s +8mFy0a7SdIau1Mm9iQ+l3bfaD92n+KXAnzF9leNAOIAaqJBcODqCH/G3fZyNTDVY80tsHY7UfOuO +XKQk+LXybYKNj+xVOb/H+NV6tLZaB/4yJQv56xvrkGEQTe+AXgp+Nhb/YjFKkofltmllI5YT48S/ +aHJbRv+b84NRkLIueTk5Pz8n4dsc0J0sguZ8ZZpltuopyXcCiKOXWJ9KBwAPN4FhAaVBmLO/EivQ ++ZS1GcqGxXHARcKMa0+enJI2Vz59IKC5ySAWENro/3UBYEFc7+a7xPZV4i8PD/qgQ1vRE3hclXQc +vwUQTAsa8FnrgGKTJ4x6km6RhsqrtB+7XxgJabEA3UuM0wJHP8XdErDo+xcH8DkeA+rtNQyUKdQW +CBBr7ZMVNncDsmMSRX4Jw+d9ojBvUZo6uuKnZfJzQrHpAze2Zyyti3IzlhCJen8fU4vJc3XkY8sY +YF0Da3FIhBAVWF93NOyLdq3Stkqt828nDG/xsozIEIi43++DinqhjyzL4qMjIfJ1d1JFaMKFD0a9 +04+NUr6vTWVuPJbjAN2Yc1U0jQkozgyl4B3z5cn6/UTPcMMOTafGhuT7HeqEeUkqM8GQZtGWk7mI +QRShxEv/E7ooulLkb1uRMu3vr/AK0yk/dzIPyryy7jF3AaVXNWR3AszjJUi3DLT8ptCBI6+dKDLK +qrh8HZqSy+JQjBRQJcJth1EHBF8mF/oS3pzexDli/h4Ihd9xulTn0gz/aP1MDQzHNP+xOuvur+9B +65m61N+ScoqWyqnPRUfNoSrvRxqH6dwSi1GIaSVhKVOYQUZRBfSuEDNnFc7rZnKDT1FNVATTD+rw +W57SuGXD834D4wABjVWNBnLtsGrN1NjMVQ4mMPTgdONVsLx/i1CYebQ07ZWhW4GhUqNtPdl4KZzp +/JvGrUjGVOOus4kORRUbqqyge/kE5ybKxL6LLay0iKVNRMgxiOlhSdnELMJjva9CUYq1rx4F8z0u +vhrKyqRr00FuBvUO40xBTScq0ycukLI78w4TNCpT4+FmgyUfRTsikxmxU/5gHRegEz8gz/P5DYCV +ItZwBP18YgaBReuL5IyFgx6IUF6fTeZ4QLwFIn/5LlxNPc4i1p9cZUD8nOtEmwzs2/Ih+yKzIjgz +2xcLyx4XqyUDl2CAbUnpXwLbRmM3xeAio7j1lep4K0+V0+MBC0S0g97KZlBYRSutW85C2EhcZEGi +nQmsORhJH/Lz52zyIp+GWKkyAeBzWYxfm+fxzATeF0rpoorePIrPWVcvtAWxD3le60LKNn/H8+BB +6BTBUFFq4+yK1EFLZ5voH8hqFT+VHXzIhxGXWmJHU7YPrMz8ZXiN7UNYGv1OrLDOJX0bEnVuo8aV +O/bjGJrSw1CfXy/NvHeijsiuNAWewyEQSMoaBG52gucbmWi8H+RbT+Th77cm6NlGR68xkaDP88Eg +UIMnB1UiPH95uLFDtXFBAz8l/yvgQ/aDjEQ+98lMITZHWFtk4T2ACQOjLY+o19CzOSW0OyHEb1W3 +IEfxSwRtQgyoUPt7fLnnqJF8o72kqQGk9xKSIP3qW+4DSZVnwhvi2RRz2hMpVlwSlTf0ExypiE9V +Z073LY5U+O3zy/d5al03X9PV6U3FfElkmOwSzZkAaQ2cN0Ds0V53AetW6RpVKCPRxuIzStV0Qexv +oFonPAkEMtUqJMaqz5YzNyYPAIPAsrACTrksqGM3Qly2wwW/K+0DDy437yzP1jKOyTT7B68K5RMK +0jrb8V+vwhfTCPbhqSMcCBuA9ir/DHIVpvY+mryLHJ1ZEA8YeF5fEb6NATms38ZM6cJ4u1nWu/M5 +Y9Vak03L2MgbNvIAvLljLlCoT4UJ9jLDyG78c1BVXJqHdwXd3xOj4Rddlm08YjMGaZrk7wQ/XUJM +ihAT33QDojB0SpWbXvzWkCLn4zBMMOGV779ZrGU656a/2ElnuqWwTHIYOuESJfMfwmEjhFbJJ4EP +AS0RQUVh/fdbGuNY4RgSNCRXLDbOK18AlW0X7SpOPK2B1ajgvVMPHzx2deOZCMg2vi0fIKv1DcDi +VoBtu28Stk5J0iABGcpVhdhvpVkRNkzOrrH1ZVhG4WqmxI6JLqf55lcVl9kIwcTcSKN12iWSBvRQ +UNMihlm/WJUsi7Bo6YGYsWacrHG8PgArAXPJ60ELq7NQaoIYZ5c4SAYC8io9IwcTz04rwEtbKNT/ +8XAxVMN5LZ9xCQxv7etHKcfuCh+w8zswG8YbMSl2a4TqiwFB5hXfbNhQWULRVO7javp7RAyX2I0h +yc/xABGyT3XoDe8hIqKyWihbi1ywb5Fuh2hGZxpAwvk2U7QLb4q6lqUBWHsCyQUbEYZqVErCvIy3 +CkJjCGM/BJjtCxf475bAZOeZAL4bSNAlqLpkPAD/WvAq3V2YUTHaBcSl6F8FC2GgZSyRSptNLnKc +4Ce0q7+qdYCacma/MLPWyTgIx6IbXdKta0cu67jUDProzXGskrSpi2BrMik4WI91rG76rthRiZ4v +7H7PWVYzQuGZ4szGINxx2i6VwSR/MW44m4VK+sLtx68vUd98DZ3LJyqMvAxAE8/WXPAzJB7dQJQh +PX6+h0L1LPXosEO72j+seE0quMydbgDyzEGuP6YTCSAwd1Zoqi8AUAph+aeFxPAhCszeZHzRbQYA +vkLR2jopZoT6FmXtXhz97ldTdlad9vYvtSYHKCS6N5/GDzYxkR/chqJMuT/n4dfL029FcvRxZ17I +6XqKDEUZ0AVihDj30FeK0GT3r9VAgHR4CIEf+X1hbGFtyYoMIVZGq0iinL4AM8UFK9Hje+xPSuPb +ugMuX+hc5Fvow2NO4qbAXTguFQFcVqJdIcKXPYJR8oSUv6I+g75ywmLZnlx8BbE2fGRx3Crqqo0a +QuDzIyiVo/X7y8hkxWREv8NnxMqi2+GTwgL/zBYitqBsegURtdSG75e79nc15L6GGSG1vhOr5haf +ZFp35Jq1AeYn0PNRsNsOZQzGNS4jOmVvBE8DgKPCIngcus7UAyYURYzOwprOkTvTj3nT+GFv0EvG +UpKMt0JfsCJNpd/k74kHB8pwuRLSp7K6dHsVVBpWsFuGayL1pN09XJFvV8NobbrqLuHcAqvot6Jw +lDrCMlvaXc9pzYKrjejCho9ITdKeQ8yuAS3kdpGJw+VjyQ8o6wAeaN0XPfYkGXYbHNSruFxkvtJY +npqqYZhS9mFUlAl/ebp+P1H9rO3d0ya0abZ+7mTCEgXcgljzpPnZqqrn+qGeWjNd67d46BeQu2f8 +YIwsjEpYQBqMtBSacYd/e2W8J6VHJEGoq+Ai3fRIb05TqjdxP7ToQNdMIvWZyBiAfz9SMwtquNLf +usqwUqUM+Ntj7KRjtfGoH9IXdaZDdcH8apw95OT4ukgWkTYBvsQldU6n9GbhoaYip3feFe+JoLiJ +mbM/X06PPpW/DU5lKm8jAC50iMvUhQxrTsON85yVFP4iQhgGFHBafw/phPhcE4Tn4lTiTTo55H31 +zqsj3zkzQeLOD1Py9zQJ9FNfbRqHFjJPSnpgIac2hIjLaps0sLx+h+MFCwtoSldv7+ea5nu1m2gd +4DcSGLzULLkjDbK/0nffowT4tQcb4p0I6EfrGep1SbeYuyckdCl5jVc4DGoeW6gz5Kt3CHY7fdvy +/tPX44NMbbKuuFBr83ody8Bu9AlaZZ6Z1Ejc00URF7nxlESa7yEvTvL36LUXwrMVipFbjedkKBb5 +gU1DDV616Ab0mt3obHCQ275uWIeXv0ZDbfjsTsuVFJCBdO7AMeTIvCeWxsaD4P7zjPm/UoZijI8L +QlTSt8/oZm+JYSmza+co+n1FyrkkB6jRqAFLgj5ifUusPmSLZnNJTXKy+C5ON+8phL0ySUh8x5xt +YFtXoGmvv5mzxKaib880f6GFNArKsVlIhzJBtWBK0tgqGTSCDB03YZ68bngSNxxhQhB+filXmW2i +OBXkxfLtUEJJxZaFrYggWUWBStrajHOr4aLo6/1/CwtpmAkr8f4jNbmLSngNiD3B7vrgGSktPuea +24FSwdPVvKZp5uyVxsxBSwMzr9+hEIPgWGmXZfQ18DksydUDdvoqIZ353foKGJQsWXrjIxNGUIi6 ++j9iBK70+FI9kFITG/4CvoX9eTUekyUM9sZAdtvf7Ty/pvNMIoNzLQC7NxkGwPTC6pQkCfBLFiTS +GtxkZ05JDnaXEtqTO3VGxWjj97EAW/PMc/jot1fwg37MSZJK57JHTHCie2dW0p9l0IEBmzIzqb7T +4WWplG1bwG9FBDLT/FCwt1aV4h1jTgSDTFlWCm4TcItzEJ6cC3r05MSMEm8uMRo4cikLIRjt7gbh +x+KFh1k/gePLF2uXzwdin7UXGZz4n+v6ltbN+xelHEY7tWCULTojHetXFPqGzLC9nbPoTiSIjMiP +VVMmityBRj6K6GbX3ZWorGXtTI474S+jO/RDmRPecx0GGfMMmi+VUNsgDTLMs1UYeTsKHJGcZl0X +IwnbUgCJYD8wYAWSQyoaeywzHjfBwy4J2eMso/wqL0RfrH3kymoLUeoRi/O7ufN6rAA/DX3iaT7Z +0ixFrrVnmS1seg3AZkGyA9vC31qQlEaK0P1URJnadoGi7RIHZdEL9heHc/aoWXjj7WZTsWz5x7tJ +565k1XTzjU5Zr703Lxg8/JPIsxVYeqDkQEfbwEU+UAOrEGIGH1EjAYNTjk0zc7VzFXP1rNV0MpGp +eegKIDKPrAKNGAVtC3Y13pNZvDPv05gpuIabcv2XNezCkC9111yCoxxNUFVfLsIzPJ/EPkkeGRU1 +W97fulLcHPBDlT2ol2HRCJ8PGsKxoMeuRZXQ1zSdS6SdvsOZYF8DRxsbYZK1ug4KUMTq5lOi/EOf +u1q8YRsJxLap/Ah2PUs27OhB3UBR93RfAb9GuNPoCzFXR74NXpgU8QeG1/BlvSanNc60qSPQurog +whRv7TaC03Vqh0VXbn1zpte7jAIYx8K5wEloVOm4veRUGayouz6FHjA4CBfTaoRkZ1+BFWC8bPeb +OE7P2SXcgBdoEnhhyP6tB15fxw1atZ5YLmj52VHCbM0fGLzPoZ/yCqXwRM5cgjr55XjK3Na5nWFg +R4j2B0m0CqSjYDVSEyQYulfFWodDHxFjeQGRl3EbhttkLTJYAnKWVXBfnqGBI8G7h8Fmz0c342i7 +uKJJOVRLqJCYxpDKjsmEo70buc1gKXQTHhK2N7ms2fGmPp0pxXR3xj2+vyfaHBvm3wqD+/K8FzX9 +gBCgNqxzqKNJhWKLzVCyMSkp+AUFIu9qm0vbPYx8LVeEmhMg7redbxV36shEbSQNtStmym9pM+XE +nIWdqo+ALNicz/KCkWsWm0Xp8V+AVlQ95Pto874T1d1FqB69WYMkdwwyB+0VmaOoqDz8YS8k0ckh +3vY0vD54iIEN2jfTlGmiLOUrpPtagofdGtDx6VKzAKF0iGywhGJSgsaqwilmAVKEQ5oHFI8v68bp +dhCcMQeRs4J+iXsQlXNcqy+2wOu7ywqEUXtCeDOyg4fMp9rTPQ/n3vgQKlJQTrANUylYJiWOFfYT +AuYJsLuegleR2qi7OjUq/n08n5pKOSXI8iJU0n9277HVyAgEumIxFb8h76wWrDMxjy+xuzAZxZU8 +Jf8DlyoxK2EfuLdj46F0y7G6Vqy6XVfcjo6q+JYhcHsdv51DtsNccYtQ0GHlnk4ppz44yhHtAfcT +dEs/HCvRVrOUknr7khIF4Mu9HJfJrav+KhFf2mfq3dHOSCq5QVLNQEfDTKadh/1aaIeFFpFEjrp5 +bJ9tZYp8XT6PNwsJsp83IURJBrboPOfBMPU/m84InciG/3DDM7UERoSt/krkqDQLBDa7IFALeKU4 +2jZhX3utdAp7AQZTDDnTVve/KJ0QHBEwXDvhy/Zt+jXbJjotaN53aOGSB9C/RYJUJK/L6AnrGMIy +pby/oFTEins3DNxyObUrywgcOQi0J6qutpHozdL3k9/YHSAqve0IoHBE5kKMFt4oByG6ha1B9gym +8XsSdfrZRjbzXipLH4VZp5fq2SOG+PDHQYpwUHsFsel/IMqQAGDuLcl/2HfVn2UyhTJh+cRXY6J8 +R2x32I0i9WvKRNE+XOMKrpOh1K/zyanKEYgQ3eFsDm+VGykJK3jgaEqFhy8v/jgcwlntEQtefZsP +Pur/4io1iL36UTsTqQAWPdemjpHo7GSRZtsfqIKSy3DXJS6pvrnMmyBtZg4tJDPDRetNxpwUqYDx +7ls0eUhnR6K17htxbRme9cdn/AS8nbzcrtyqlOyIQBsxculdFFXFBWNcmuTMhryV5Kgpo19SzlWS +CcrFYLG3bZqSdmJIDqsO643oJyDI6szuSS2HLBEktZ0Pq7KC6mrno3qz/sSP0glMSVBWHVf0gG1q +f+OENXURBJ7EhEuP7AdqaFop/ajTjyAfoK3+Bei7/7UPoAM/jAEHzQx0uVrxVnZO+53GGHGKzJ09 +uoFu3BML6RnoFjqlN/4cdhEcw1tmjzMLrb+WaW0JKHurRU5+3g5OaqTAHiSA974xaO1FUpEAhW/l +m3RJqkvoHijtyF+5F5nkxW5BTVsjoCcTww4JmGZatQWukHqLEvYlTlnr8JiYMaFke5X2nAjWfvAu +MR/ftIoM61oZ+PGOgoJ5QhpTVwLe0aygpGghslUMAnO/qiAk4RIXT1+GW3N1PtJ8oSgIbdLdWfGy +Ri9JjPoGFyMjEZxcXHGUp470+0AxxFIjW8k1Ttx0/WEHwaUFzYzSgbO50K4lJ6xGKcdI/CvmBrdQ +m0+HSvQREfrszOpZvZB0qLGvf802NlmIBMrcYa0s2yuBTsy+Oq+S0UMk4EKxeVoF0n7QxAfT2Lxk +SjXKQ3CVDeDAIpALNH3sA0TlgNa2GVeFe+IfH4haBxdeqZn/eTOVmyVFKcgt9oQKiy/YAO17Pzvz +wAskbuCNzGVC+QTGVxmkJLnooSp/WNEoLfs7QFVYdCifAvvfJ36cnQIKgHYlG37+n32zP/pvInlR +K653hClUeZaIk7eyEj1BWVp/4kmLbiWJCjGpxv/YpLUnCbX06MRS3yvQZ9dUhAt+HsLwZjRM7JRa +Y9OPjSHPBgxXPPFx00G0Kd+fAUN1LegrFXlXOxJiBn21dNETPdRejuXzKqDpif6PBVcNwDZPpzWc +OBdpHdsg0DgHF3ele948GrEDQZSNMjRL31KFZdZx+jrJFU0kM1eDnoPC6EU95u9Ed7Sr8C6dHgUh ++pyzR36iCEqIYKJQdHSpKzVq/HPPvthTgcZEXTE7x8YZRVSHxhlHecoNRbs6V7TlAR9FkIf3vB9P +c/owQSV3DXfJA6gQbe/+eo1iIIPi0j/hdb0bR+Ix47R7p0KYPPG8n1BkLL1+eFvO18M22jzLTifP +yZ+pQ4T1UmOR+4qW2+0QhHdvXOR2LHEQ7IuvrAlP/kcPDyNLP/IQ7Obawi/4YUi7Ki/22ZXr4+kA +NwmxzgSzLgbbilBdaR7J3I3E1bs2ZspTFr60ahXVwo2lQtxHHZ3avkm8UBfX2IrkeGthNt7LkeDV +MYo1mlxt2oXHChhXSvDxMziuFMUgNf2hEo1HjRm8oOlggkV2Oytjdh9B4Iyy4dJd9CrD5RvVa0/E +POA/90DDfs/cJzgWygsbhDumkdrHOM08VQRUP5rVPVBoFLtzDy1oM9JF38/D8HXbDwhfBeHbMa2u +6UQXaJXB5YrPajJaKPqfMuL6F3biT803ow5lvRfoEarlPfy1VE88lBTiErD6E+jfNKTqo2cEDqF1 +X8OcZvpnUsa+e3k8b/+JYt8ffLU9H9tos4IaS/f95rvhrpEiyo7fuf3kOVB9GHVeoUWce+RUe1ET +ESJhDjk5INRXNUBSLFJk4P44v1WjpCq9QFhX8rBsf03UA8r/L3oOz/n0BhFLPhHs2jDdbIoVnp5A +z5cb6z6R68aqprPTWF4lNt7SGkT8ZoGQTJl+7DHq+CG/CVmdH8rO77Zp5RxVpz4jkKPSB6M9oXse +nMWFlZDfeAmuojxFlG0+F8IcmF2xWwpshQ5TY9y1/tSVHmuBnEVH/yiRgMlP4XS+x4mnrJIiSu3g +emiJ3DXHudMi/Nuht0niIMyypCY93EWvkItDxymuzlNNrN9kftY9+S9TEVsSGw4AMFx9TAAL6kYh +qQT38SZCjtGWl0qepDXV0fAtQ451i3Tvb4t1+K6Vv5hCeTsxjNXbiSnXiVVwriPW8q9xe5HXbomM +5j0cJiCzQOrVPbpJi4hFs/9CpBtNbkoaqCyCh/RUH37SanxICqBhGyj07I9srhiI/f061Ke9EIwv +/W2POtNnY3gpjug9YLy7Vwml7MZnsdtKC58P+WW6dDgpeLznN4bxZBGLPSj10w0HCAco8zDfZdLs +wepxBN8EWD2YO8R9Xzd83NshecFGKe4POOymV29sJ9ZEFb1DKo1FeXxjYCq39vcBU66m+A1s44uJ +9PNUSpCx01SVhYJfHOSfis9EZdtP0oXWX/+JKUNuGdvIgboGFLjR/liekwSmJODq+hDkYVcWyJvF +GnNDbtBEowEOPuhw7pNtJx7eCa2xVhXpqknWgnvhlCiNvq9nfKqx2J7pd+sLwNe6l+dfdyNN56Od +vP2/jDNULjcpPM2oYnPab0VTXB+pHZ4JatNLrpPOiTmC0DSO3XsncppF7doqg+jIU9PdXAMzmTsr +NzC4KOITQhvX2rYQF4oarQPzT6luK+mb46zADKvWfNLEyltPOypFZuaKyP8RJcrWdZg2UwJvX517 +2Qwb9TOu2cpaMcu11LWwPJ8B57IDeh/+ZFq8eBmKy4vVPfzHg0ODb7d+Mq+RNAvu4s9iTGTRP9mw +c0jXBmVaIawQCsWmB5H9D7Jdp6a5YftjqAc7ORLhkXaNHJg8ApYRxbST03IsmyoY5Mslln7PXxaE +VjhLw7cAkaSCVK3dvCeUzEaehBssXZMYu4F88yxad0jZhtGPPj8uIgI0zjHtIEJ1HVwuIVX5wbCF +fOgoV6/xsKZjOD1Rkg/sRmkF92AsBZch9hTw+1zO7mxqeb5OoGy04ktRNyuKRY8c7bR+wyosR4NT +Xod9/7Hdly29rQGJ5Ski7MLzD9LJPRtuOntmyhJu2nWr0luwnIePxWcyQyiuojiuXxhjkydWMqUl +rJmu5AqnWoFDq052OHmi8yAi5Zdk06E0+DYkM+LyzFCx0XC/uFr3o3La62FmSwnMuYBHmCDEQc99 +dyfCx1/t2LKnBdT0b966q9W14oQrzMxzGjGKFXFdgVTAz68YUou4Cq+YZPb5GU5GgMOg3OfIV0Vh +rISlI+NBc8/uoniPmhh83TW3h189u5hGVNv+kJwnXzKRrnq6lgW0myzlPmXBPFAHNAnMv5Ubr8S/ +EiIi1QcuU721xGbhrzNJy8n0oNOMDOqN+mlewZT7bQO2Ix1JXlPc0iFxfJmLNBGOC36BYsoDyfK/ +71IhxxajUVpH2Ew8PUrtbm6y6SAP/IKCftrDKQPloOMPmmSwEM5Cx1NQ0g3W4KIhapL8bLxngCsg +CfYosIVEy8W8O00kw68az5t3e2jsKQbgzCyvwRDsawYuXHzY87P8CBhgCaelRlmLP5345r8fW6Io +cITGS/KiYo4U+vK+iXj6zCGDkWJbcvSYyBitJEe12I8C+tIJbCgdaPV+1HjiUs02UlumY75ZFLXU +ovdS5pi3ZWxmc0ZaVbdG3/l8Mq+1N2+uHqdlf0rI5pJkFXKhyBJPGSw7u6yFEH9E94oUiA+HRKxw +NzaIvBp5dao9NJA2FMJa1x293pPBGNn9IQAm23Xc0BIZMB86x2fsO9ZaXf4IQkp2LJyzPYt4lvC+ +6nUwx+Ek1uS23ucVB0LM81LCs3qCsxXdsYhQR2i13ifUOXWhUpWkhAT5iUFR83f28pn+dRN0JSZu +O6P0NtPkbSJi1MVyv36qVRg/yU1p/7gITBsu6D0g1oYwr5U3GHylemPWMMH5rFQfJSwOSOZfyY5a +GJ9PBZ9GPgUrfLFsytJgN4iqoGM0XsHBRGydzfDZyjKXlDKwcnDvKPdIM+iGclOadYSMg172UORs +RRVKi3XM30V/2w0b9cmyjUX1+FgbFAi42OlmD1C0rBZ/zPur66lXKq67aKbGjPsDmdAoGtfYy3yx +N7ep5M2jJ5NNjJjeNf9gJ6ov4YekATgceIs1Ku6ueftgXZ3wr3feJKKCfgO28EINEXUb5F6SzoqF +MtNYZpM3QbFDm4VdJKvryKUS5cD3AlRMRmvPBQNstw6JTt3WuiazrFdLtFtpXgtXbB0yz7nWCwhH +jHAE6XFfUu5m8szs095vws37PwcVlJugp7NpTFpVMkvb7G4i6KgJVtmeTAd87+wJjkt+if3hwfxg +m9fGf1DovySpeG34EOxTDiY6RZ/FLpJpvg8IsZqxvwKoNCe7kBRRqIPTJvbItT7gBJAhmvBSmBSL +2HTXA2Z4M4pv+hWFFocg9fCs/zzo5F4HnKG0+RG3dUkz4gs2ui/pkOREVY5ion4YjnMspWxW1yh2 +sgn3iJa99x8swKnzDlmX6fnz7B5A4GZdUVOE9SI8c5F5lzsLgMIZvG4PiNRazNtm5X1G3kLypE7p +UJhdhPjb/iQPBHxoDWR0sf5CtyzDna5SJZGbjI5FLISO+Q7RW/awlff8auI+jhEHLiy0PrFs2l4y +jKUrWQmLiQ7/F4DIjuXm0Tw/0I/UB9m3JqIs11qJbjYiJK5oP9bhfmBdHQFEy1VtGaUZMZ6SqXAC +NbbNogllHDIlMFSIJVc14NR1axz89+UW6oGno2eaABwSAFTWtvB1I0WXqqtX1UMJ50tlYTqk/Nrj +YoAOn9Eh1oTrx0JG738p7CLGTZXZuI0JehdtLxLwYDG9zRgOWYy2EVSAhA5bAKoQEwcREjuBrSTa +Rb4AUMixnRRIpXcoD5ok46Zn2Uth+LrYKVe65DH1J6e6GeqQ7594F8euKJRBirQA8STpJ7MzO2bC +4qKmitkiv2qnUodzCJOJxIlQFj7xBfI0OoXi/DzaaPLfWxlNIrgqghmDmMYvt4inCqmH8zZintd/ +7n546Gk3V4+5q0ErqfPVsXPeK3v4v58YpGXOEAoLLnVrtg535if7o5mhtQ/bnmyJJc9NS+4Vuy1A +qZ+6dn+eowCxusTb76nOQ7SVNLcvUxxFiQStL0IFe+uF+QONyTUJs83E6tQqQ4i9LlNdtG08zeag +KVM2i39IlgN7FxZJel5rcxiWztazHwY+lp7EhFX881x4wGsZ2UZeDSRw06Vpp6xq+Wk+TMEIZUNF +k51SVw1H1rPcHUepjDcjPwyLP/eQFscQW9w2vvv9CoamL/cFwD9rirT2gVuc77rvSK4h3sudhLNY +fy9MhbgtN4BwNwBuY4ktawW2qH6h3Q5RtCgxD5w2/SbIrd3+5CV9nu82hkzgwYsGIq3kJQVLE4je +4JNw7YQ3zp543UCjT/djuwBXbssdSCCC5qXEqP5kSRfzwBA0faxE4qCO38pahe8xQ8hgoHL3OijV +yD4SQG+C1SwSeccOwXlCrRIoa3DWVYi2sjnlvT2AnBbACwwNa3lJd+h2+xDhJS+gJvJ3/cwd5WGQ +14Wk//zV+yBMwBrlCyGWBHjmaXmFJw2Np0g39YiKM1W3fblN3gAe78BIR5MOQiOvDmu5ucqsvK2u +ma+8jpo+I+UN+9fg6b4bxqQc3S+0bjB45O9ZGga58W95PJ0GkSHyPFMhPtnG0/sgudMHeYnIW7bu +27pG1HQi0HLPEaCkIZMz1Aj+p2rAXMPfUkb+cqiYejq2jTViaOiiYPAwD2AMjmdTb5L6u3/nnqLF +BgE1zw3HmdGSpTIVZo/k36oAlExs7UvEpGyN5BLtcCXkYdGMjf+PATFMt62C1Tt0/6DHy35rpU8X +dJszaWuTetTWmOxKE8X+5s/y2tE9okGVhe0uzYHHWiylyTsytuIOs3X4Qp1GO+J0ZY/loI71pccu +pJPWvYrGUQ1TGb6JCWsjpgHOT3xVg592f2hfJnpz89un/9XUuUEYEDOYq75pveVI5GUE02eHfsht +0nx/V2hFe1JI9I404E8y5fsxP5pmXcUSpb4EvadrmvWOOOKfBBgkme9W49Np3fZckBPmeNO7iYq/ +5bPKh8U2qEc4PTlwGI1ajuHW4205Ef9jgAhfJ0SBf+/0i3nJqdUFfSKh+oVk+Z1UHZiExrQZRJWm +a8Mg8SPImXOeii4FjqLQrG7c+Zqefx553yJSBE9Z3RuESrkAysUeeoG48WiFCKYql7IO+lu2Hw7z +9KX+r27pcV4nwtUTCVDzzcuBMlQo0b1Axp88Nz4swOOi5Whu561YynUrjo/ZXbfAPd9dS50m/mD0 +5vnj3ZX11oD/GYEPyiAVYd7TMZNgg1Fv8mDjoNAnpRQBVQrYTktVpjHStD+yBK6IJDTo4pFF4Hcq +22KTuqd1TjS7N2VPfeHxgIxPGSKz8iz6IyW8xJvRVzXc/xsY8i3ieucfrD6VYw3cCKN0jeMWgJyG +flpkObA1I7lgiOPplKPV+hx+KYFATAWxIBE1+Yer24koXNBaedIrgiI7X27lYqd/w4SmNbAzzHnh +1Q3unC3zrDwlZQHMTKDB4anZUyvbcl6/33tFDqPksKT2VYGhAAdZKliB+oGYVDluOrpfxektUX/o +ezKDIkl+Nyi9sxq9MtjRkQ2qMXYKcc7J1cc6Q9QbVoyCyyqB8gL1TbkXRo8aHBtw3ynRRqPAmNNv +ktj+iSPBTlZyDTw9rHbLKJ3GHPPuzIN6mzJhn2bhMg1cg3JCBxUUl6oAOrTcDaY1R0qdJ84hwdxp +OgI4FW50qD9MVmtIvd1ERowuRHVRsszDvfHW8UtRBfg13FgboTJkNntv1ovmBAFr2O6bt0h7n7Z5 +3Bbe9nh3f1Tp89AfSlNFjvCp+RPTH1snIex/PCx/m783+TA+vexv5rVdnsMVD1DlTxIoTx+n7QND +aoVdpwYYDVg+tZIICPmRAK0aLoZYHio+cwksiuAtZiIMGis428z6ATvKKy7B4mAkvH5XgVJRjfPV +TXQZn4npFnJ0L9sP28o4ICNwmNK7qdSXZfdrhafElH6dkJTlcUXS/NaA1fZ7kEEkBXgXxZTXEY33 +xneWrgD0e1q+CwKECh26vgpIpX/OIyIR1H2O0om22X4BNbiVpziWuiZ7/ZINS9b+0RPk9iQ1UfIP +yZYxtF4ZLd1E0dO/gokBF3S9CSlNl+RsxTt9Wy5Nnw7L57BdankXHW57VMaW7SWezpJqIby4hwKe +l2REwuBGZxqs+XbuvxMr/fHKUvTGjtSwl/H2NIQnfYlaOe3Ap/hIqK9uXWcuZrdC+NPj0QSjsjwJ +VuQxnm399CQgXaw+3/0kHSY4dLYdsdtFHMT8F9D83OA8sJ7LP8c6yRN5UIjeJ+MQGwumDkncTGad +fpaWLGy+rdWvwqb+9mDVUzy7ZU6eMU9tcSSv4p3/vz6T2N5JpUFp+La8RwcETEtUfiVhe63gIuH2 +vBSCciCxLv8VVACo44ObstvrKVv58Rkgn1PU3b83hI4mmzsHpDYMMSW3A3wQ/yvsHSh8X6dXwZ0S +TVpvyvYs90+Uze/WsFtHXrb6bBMNSz7+TJvxuOY63o0gU6adjVyp1SX4/L60bMl2VkWL0mW5faii +nfaDGrbc3NLwnWSWdXAzWguEzO+KlnK4JiiyHy3vzxyrvJRgAY3pa9LPigweSP9opW8zQGK9CoZs +3cAuQxCcbdcie8mIJYkUVLcm4cOa7R/crdskb13iT6KyD6b8hO7Ev1ooR+r2YIDhyMzGH6yKoedK +6K9kqUimHgU+mrrA8vK5AwqE4SBfhePKd33VGDSAzxyqS/rfUqkSWfAjUi2nQHfeMPaoe9du1JtZ +CaXKEo4t8KanV1oPH3zNgkqV6iuO29CQEhC4iizkSCRFyY8lUdOglfAKlLGfs2Cp5OwMMr62UhPc +GtbEsftzX4KsAfdW57RrnAtTMAfkXostsjOYbaPjnnTlEhQKIXrJF4DV15rf99p7a+APNufKSRSK +Px3b+nwqnGMAD9n4yFCocGbxjRI+HoyxE5wNFHVnEI88/gqlNRNpRBZNXZLKARgJsNGmeEX3QT1Q +2nPVVd0SwWB4az7/2cEbSAHkyipdaa0nPV3kWR1U8BJOqkoWaMqHJ8WvFRfHImYAfq097eNANQCB +iu5/XYX92S42pyRKfSfdt0uWPInupfc4FbOXts5JlPPWJJJy10ZIWmqFP4W3MerPe/xwiBnf1xUT +BmZvJVobNAw0a0TeU5ZtbIjbFLQULCfcKu+nubjrYQFQqkfGIeDaTHOj/VEzuVRYUmCqw1fWkm/T +30JQz+Wnjf/Qkyn/pzEkS3ApHE6JTgUr7aKX+51VpArN8lTxqqvwaXvt8HEOUZ3RGXEGfJcNIkEw +M8cWGbNfLF90178nB8JNoZdaIT+5y3Lg/Wh4rcfuwaHF7D/7ZCMRHHcR9KTO29Ba6vUCDR4OO2X+ +0ttspKorfpO19yHawntLKa8oCnsaUPx1o05c5A9KSMpv6cAqxjuzjdLkrj97EtBnikIIW4CzR6P9 +Vw6OMumdy6d2VsAu93kDB4lZI+aeoLYOn9fNtSJdleN5KaLL2vgdsI2L2YPHlnFUU8ag+9zbWiZA +DLOmO99Ac/WDzZn0eH8tePAjz6Snr6h04n7bT4LclbfpQFJMz/7L/xbGSaEGON3VSzgm+kSRTtX2 +CTlXT0zjrfc3d0I3PDDqCJm4IfkDqsdIsYIwl2Y8sp12kBJp1aPO/oTUQ9XLQDULYf1dmWjbrGIP +gg4VZrB6cPho7pUV2Xf0VlqisX6lVquplEFluIRKlhhtFcxgjRwB78YtgsdTawE9x5mzprDwpGEV +TiBp+MczAkJbX2FSizBH01RYDbYHr45mub8b59XVS9xOePXw/2rIfUI0/9pgDXyv+YOBQ1sS08Al +q8hCsfMwiFwqA3qc5tM0qP3iWxm8meyDWzu9mFEplglR9v8BAKxnxV7tpJeFcME+41ZFp+jyaI4M +vdqVQnw5XtTulQfKDRMEgfgvXFPzn5avkwioWB1spDRd5Oq2Am9jZidFZRw6fuQqfkOlfiGWP7QT +fcyRXQx1x0JtdMp1rfC+K8jCWIU4eG+mnVZecUSSfsY2Au8vuIyG1YVy3bVFgqKOoTWttbxV8obY +P2fB/YyQ2F7Rj/fHKdoARgud5vzc137vFOv3V694Eh0EdTKPe/vliFt0kT3WMsl+qmkeNAVTuxa9 +Rd1V/luMP66VWo7DT9LQ/8v4rCECh5ASNKa6MYjTeiuchmqsbFOEMDXouJlMV6JZtmj898O7D10O +Ds9BcvNCdWERfReQ+6ZBgZiLK+RMhxsbKsRmbvchKYqN9FDd5dhN6L2iwlIBoPR5qY8cGk5C+Ah4 +fh6iTIxqE3eILzR4dn+icD2EBlgIMUs/20IIdsXPlw7rAkHPps+yY9x8xynlVaAhNHS5uqxh90Zp +nPtfE5MtERYvTaZYTRUmViw7DXvbAFZeD1nyEdFYybZ7R5REJL2lrvwfDb4CUiD5kPF6U2Swdy/p +9VkhQ7BdGcHB487nBdcsXdDPbupUgBXoKXYof+CJ5xrcFJ/rpo6ZDyht7SNruRzbRzG2VEQh7oDq +W032XiNmVgkTJBsDfQykPUpn1+65HZDzuA8cbEr6XfWa1bu2sWa0nYcbDr/EZ8zi7z3MIBPxLzTZ +/J7SxKs4uPi3eip7JLWI6svfFWEpQ+ydtZxQjboxn2llIT99sBLSP5Ri8l84iLrA5Faf+S9LRjie +eAeFz1YQy6XKDkw0/Job8CUZtceUtAa+daK2c92o/hqdxWEb3bppkS/smy+O03l5cqoiXkLxY4DL +C/L+lcEB1bUxnwCtVMRV1GZ2DwnhmUtso4d3tcT1xPNaI/nJAqivaXdjf+tjhvATCuGQwbGbuTGT +TGYiC3B9MMBDIVq0d16gtF3HYGA1KJyEgvFOKPgGUHrcSQK6hZ5f9vKImFFTaibqVsFtMhftWq/l +/6IBVXTcc7HYzm+Cy/KaRAuhbE92SxvvIm8ywyLInMtTQBgo3SCt2M7qAMcA4UPCNh7nd/y2BOQT +nUXuPWOjqpW97DRCJZtvkuLxEeeQxtgn5VWwu/2U+m8kP9d45Dxn+T26roqHFdK4YOP5iTRZUrRw +3uCuJEnoWzVhmdgUbQ4xLutF+Rd4WrQUbAAZTJBLygGZ93D23SlLuoqzcZdp/Lw7+YUzAr4FvMLC +fu6XKaJv3ShdZ6Rj5fzEpxmzKlJE6wk7a9Loz2qf8vFGcQvFCP4iVd4zRXV9XSIrNxuJYsriNR3i +CEpUIxCJt7wt1tioqf1grtf3cldor3LttNOR7l//gCqE7e9OfBxUGa6omM+5ZNR/XVh1jZM3vdZc +4cUTyCnAKcU2Og5Gwe0qSQCm3Qlq4rip9HzypbpKBJ3XjS5BEL4T8ztn724hUYGHAq+LHvK6cD6I +Gf8WllcuiqbnuotRxXlHmGPpFwk7gvqDAPiBg0NmRFYqMBdV3hgheghaLQ9VIm7l9Za18Fd4yCi7 +5fKbskS4D2HNuHpxUAOMijuheDG+y6kJsWFLzu5YAIY4VzxIkKRUPKU0234urC2RUWUZxl0zNgeH +F/zy48HtUXoMEoaQRcnUWmklJl1U+EtZ2x/veVZzRGG+xt8K13sqFGq6HkWoZAVn7fwQg170Ngel +TRYHMlpDjAGGVEbduDGibQSePsPherf1wOyF3Tu0Q1Beh8ypDhNZRQSG7XnfKhzPNiZXrPDYc1zm +MHq7uQ4iv7GdkOJCk7RcEB27eW0Iuu6lXmqUPLFfBYITjiJiT9Dlrt999XFEorSOYIVG4MdFlZI3 +frGQGyl4wiQVFRiOVhWcrHMpzNzKiQuqtLsDpJ4VpNAmbxJSRsFfuWR9g8Wc7mTXWH/u1/BWpdyq +m1oP7IWH3M0gNXdrXSrFW7BGaA4XBUBMBWCY5VI4FGcXYDWuGU8qoTRZEUIkvF4pfc8lQlXweu5B +mBeSg4yyyqgPJDblQ1kbWdw1Az4OHr9N+mSJHwk2ZFkUg+SKPt1/WWMW2kstwQ8rJyzS3rojRQCF +PeKNNw1A5B6XwJ/5EMWkQEvW3177FlvCC+FzJeNOYhdp6kRIMFQ45qj1ryAIbqNBtQjn6hVAhvzB +wZwZDcKlwGWy9/YScbdxtsPWpf3+PozHmgdnmUctpOuaIoe8hNEd/M97PAfuBbxhzP6VtbOy0Rhj +8LqeCm1cPavxxb+hppKXcOQ51yx08kqJoxQU37KtHD87P7uwk6HZVvdwypvd8goNT0QyNECNhJ91 +fbP4MWX4089KohzrWMwxCVTJqWH+VbVfQ0G08iVjKfT7sNS9ZD2AnWuXDTRU7bNxLznMMwmsemcR +/yHrkSqkFkZIV947Cmp5+/gkefwS5359y9GsV4M7JPcnZ3sQggD8n2UIeJ/60sJ+NVpaXp4PmTOg +AJOkmW7NSbzDnax7P4hWf0FQgfF8yLT/nmChQh71va6VspBXgdRoF7uHl7xMyloEffF8i/yPYEme +2Oeo98j9yuDBgLfyZ39kV+bfUZJUo77bVBoHCgKMzzDzoOYQhh7fCflN/vgSTGsjj0VQruqFa9GQ +bHOhcdccNjYmFd4EPRVVRkn5PL67gvoO1+Z6ZGfS3ZA7M4XEv/Y8BZTUbsvCOmY0Ac/oeky2BH4a +N9Ses0U8zZg60gK999anwCtX7DROW3euQA1Sx8Ur4Avso64kaWziyiQNMakovdr5mZ5+Ys8E5NPN +ECtaYYC4eXqQD8kIeCttwrVImVoJQ8BueHJDlrIPXYA8uIHedl/ZWp037TO8xTelPQCLrBEOAp7e +gmNx8+G7BhpWWAhWW+NoI6XGMabL0NCWVJbK52GQCWkx0Ky5EYIbie1e+OzyfZ2jdZOWWqP6ENIG +m7rzTf5yaVR/s8PFHD5HNi0PeUX1h+MbA4I22RG1ErCLo06wxzN9j8FC6H3nLPqEx2JxPiKcX6qZ +p7WLG+UwtrREz5KxRsDfKUtP2ugbuhyNufd+dOdokjxr8DVufv/0AaiQHTDGHVU8YPaDSLyPQFMB +Rs+JZUzwQSaP84dILAx7nDWD6Wx8Cjj+I56f4mmiCm6yMEbY1CcNCAuJxaXuoNdelmbvhQATdi56 +yDczmOLrRXx45xUZU/FO6FMc2ffg/jxuBeP325XgchjTlSHEKOP+8eeexZTpCN5xUy+q9inX3CA6 +EQme3wV1NS4k79Kbc3I/98L+H/XJ8Ry4FWYlxTFTbBoYyzsRL2Kqg3M7htaqUPv4RCvO8bVpK8Fg +Fo7z8DGs8YXuEOVNi3YqcGYeA9uYGgpEAyJMETKxthHKjYeeP/1mxhQSSGDZf+6vjtdtarvudLVp +SSuVOOty/fxdlHkDjTHuxc7PDvWcKpezPYM19ijPCT0ZeUEU1cNDlWZMmwoJWNucbUDMHn4GdpuJ +Yr3I1SjeO2I7gv8PsriE2NvjPvasy+bbz5JzjxPIsB4HnFN8M4SPqwAcYBqHYMJLIQ8tgE6BbtH9 +MSev4vQeq/sEG3DrNqwuZWJhrkoPxy7hTSThixtsnRzb8rQ0P76BL+5Wf2WOHwc1DOdUPIfujtUH +YpsCruS3fcfdCnerGJ2SuElNGM3Z2IH3wtQhtT1J9OlLOv5MIYyIFuEZtOW3OeCP3NnLLyFy4Sjj +FgacdjJW/LL+mBjawkpxl2R0d+EviMZ0nPr61zeMaLf6j5HxrozkKZRMYkr8TxYU9YzxGwAHnQFq +ZfGXSfmvhx+7yF5yNmN3bjc0rHiFp/ZbeZ01WuOmJMn47B4sr4gBuARzNe22aUzfkaWgTqG4JxI5 +KE2Mv9v11HllgRhJLd3QqXst08l36vm0WlexZ/W+yhMUIn1TK4OC0CCsmdPgfpgYEUf4560yTtTL +P9gWsjfHnc+HBkz3Vgm+XFjP3y6hjFGGlnEuTcu6U+I4Xz3VJdZvZl2ciraCMTwOzfkKdSGREz/e +9YvIaqRAPt49MFlVXn4GTaH+MGAP176lwYoBfdVHvV5HQIe6TMtkGWIDTZ1Cen3UVZy9wk4PeSop +6ARrakuM7Eg80JeLGpXNjDDSKTUv9eTLrdVFpQSJxDHTaQGpD0xBOlwgBKV2O0jMUHbTp3O+qkf4 +rRpdb9HU5QNSuwojwmqt4PeHt67Bxz6hnvP5lmJbDKxsjTiYsscJbAmrcTZxApW8KDLb0F3zkFms +ZJSDi9tARaZtcksRtM74qBTHJXCisOB21ZJiNtuLsA/kD8XNjZv+GqBHkWI8pBPE+EQnsjRpa6yi +nT/rPSkvRENl6EYNAo0Ew1fhUBnVpgj8IJZ9IfhKIAiCtLpz+uSnSGB2zXi78TsKwnaGWg/FwnXy +ZzeAPNGEBSYsZ6fxvJJVz5ysB4ga03kk3SRwEgjKBU9WjvN+/vN7dsFgMPf6QaBNF1/D0sohcbDI +dFGVHTfjqXxdhafdDICsvEkty3WIif255qKQsBRxFT/J5x1AQOMXAzZ+ZOU2G1/kkhwmXDUdDMIf +SFb1Wd4zu30DpOlzUlZVmDgArfYybGAG6y6P61abPIauLwDVA8QQqD3zS7z43u27gcpWMhD1gUO4 +y4ZrBbVzqZehCdSs7pZ5qpT/tt4Zt2Q0kQ6IRdQ/T95e7j3a9Ud60Iqgpvo4LGf3tjFrG/eAawg/ +tKS82GBUuoapYSdP9C5byea0FeyudPqxFqqDCPVovxC5sPhlVpFl5gkzMGQrx6NFBbTmQ0LkIvT8 +JoLh5pcm0vefDCNl5P3zYTcjGhgj6xb8Ghnqi7XKbvERJrRI0JtJR+hDa2iz3Fn76U9JG+rkvTeO +RPFZ/XlMy1eRssPwUw2z9Lfy0dCDu7STNFT16qv4v782/1XA0hVWjyOhGgAvjeLvX1uN2edi/o38 +rWC0yOpBIQaHyovaPf/jm9IsinvvfG/BPPw7okPu4ohn+ucBmX0yjH+JeuoKY/6lFH9UvFuIKktF +OM/Gh1y5ycySO3eZuuFVrnLReMi9AirsYfdya3dBkwz7ADi//V8h0Xi9TsrVRhvM9BEGuT0YXyQ9 +32/N5ZWozKPr4bEuX1CnZFvHezv70I+zGX9R0EoapV6TxjuaudZdqHJwJ3bQlvO6IB1feOdnrJhS +uwaM4U8WKMg5MtdmrpzIMkhCObSp6qYrbSIHVWV0KgQ8kWVgSwhxKwax1D2KJg1V4DuCXlI+bYP6 +P64lfGOEgEQ0kjYNzLia2NUxv1gWSt4TAL74veiGLJZ/EOqwHl/ZWFsb6V+y6tAknyb+EoMME+lU +fTIZfaix08yNXcUTscMLQzIxfEYFfm7yGLgjs4xiFqtYSh7fJfHgVj51ziK39m3VMcc3O88LBM0M +ACE3/1CFDxHHqC91syLC7SAl2jHCMRFCKEW9HSuMM6f1gYOUgTkr+yz+Tdw1q57GA52WjTlcMvzN +NbhJ+k13WrfaMWN2/+U5PwzxI0Ezirwca8V9bKczXdwHe9ct4BlBJitxfAIdQ0ik1cMp3YeWbHob +DCst0Hm/4G1Xw18jfkmjRb6JgEC4fHlpLcULXGNaSdSiD+AZvzCVGadOwHr+/d6KNaTK3l6TPHXi +/CqTZm/VsFt727rjiTtvkDczCvdpS3dOr/E8fWWURThZ4wBGXnxZ8Z5AhRMMxSOkMgYFb4QtrbM2 +V2vPyf1WqzcpQpJXm5pBj/YS7vuY7cM2NPrV+NcQlFrAJro9qKfBZXRNU0ESESgCiUZcR463CDmj +v1RuMszf5P4Bos0mevBzFA1RreBFZf8gy/x6EA1X1JwEzsyRkqA86g3wbyyrQvIKKg7RONd+oh6v +17z+6Ob+vllcXQOYnYPmkktJGgNMFYTmc0bFFZ/OvWI5e/OPy9PmPyQBYGDnywawJdhEXk2rCB4B +3UMrAk+uyvzZCI7EDDicqW9Ao4UdrriF1wGdW5sUk4pUDpJ3g/n2XdKV34G4/Cs4frJbPg2ieLGQ +VFGGLEhzzZjnOt7eoLrUGKLkpGRI05Pl0qUAog7+Hlh1pSm1HDEXJXStYVUsQtPIsxWOn1Qgcsa8 +znQ2Sc/4ZgGrvX29AjUTd5EIGT9nndmRC2jqG3Q2Zvtx04cRsYMQuHwujhbF0/jamR3YCOJNQsF5 +ue3LT6fjzq2em4HyO0L71YZ4SgYHeD1+eSmSSFI1MBolriT5nQ/PUHz9DmHMPOv4p4CljTPZBqMZ +amF0FEbbqTW95tG+PvV2/RKkYEbH6T4Mhd3sorR1iFvO8DJFDa8twdSg7v16f6TMstXu5CwfGljo +//it9eJA5DnXhDnUiQGhd+WUnmKnA03zwKWt1p7WNwnvrktVnA0BKb4LOdaQixxx8dZ4LO1FOVJB +RTkGbm2WCfXQB54kX1Uj4pOEAcA0+HIbspGq7MrXWS4QUxpxQgX7bSzbvSNKTNxyBYgZwLjoPQ50 +nVucd4ydce/SBRm4bRQ6KAbiH+vPqPoD7t7cseyb6pwObz4uh713c9pnli9T7AXr1uTlmy247Arc +j0gOwGcvvGVMp49URjRp2jMSSBgqqZ71rkp0LfYh6k3MMQzQllnm/Nn8I5gkKiH4JnAtUKL1WN3y +xtrH23LQd2UuwVx1gzFfYrAiphMHh4rI0+cGwAGgV9MQBKIrShryWeBOL43SANrydwOh4fXR6q95 +SxChd0KK1zpOFy9nJyZfU+Lzec2+xtaCU7xxV6K4HJSd1VfFNI+l4BNDeAJjI2PcXslanMKfJzJQ +n6okQFmhDEoLv9ak5w5SXhwvfYGpBUHmJ5AqFOhv9NEdv1N8gk716gU6Z8f/1PnH+n0sTicqWUeb +fpDTrWpyVkm7/7uiJ/6uck4GAAIKmGdLQLlIMBFj/6TvJsVCrnOff+F04PzIUrwmYaCpQmJS4UkR +436FVxtWJO82cSaBlOr5LdRYy4Q6OcnPDhx9r8FMwc0W+bjbQ8m77cuWfh7HuTGcmrxTYBrrkAcO +4drCCrzxLccZ/cs/+0iW6zXnaA3wrm+A4qJJnfliLalX/9cweNfLMSQ9T2zaLHotE72dflv71bDP +ZGw9eLRfGHRja1rbjDZrsDbC1Ii5wS45G5V93fcjpO1bucaCKNi3OoMeOSslpWr1bTtI4Jt2W1Gr ++Bcb/siZ/038hOojoNyB+ssfO1MzLuetoA5dWUWceKJH6qi5XwVvmUlHNaYhSezByMS1CwEnQGu0 +ExMrvZIvcn8QZe6fh0qfvL00XNqKZWra0zs1BbZiJZnisygy5f/JbnrXF423kWpjWFJdad7bXq5m +PkVwHonXUbOb0reaBNThwyErvcEMCWxmtFDFL9Gko24UfJwYjnT2bhZw/B0kwmn68zsH0Qhp76Hd +y0I1PJkHVOb4xNo7IHKHKyWeH9dQi+6Itur9CWUstzyH23ma7TrHAXOsr7aigGBv9ac9sy7VfYYo +j1lbTQplbmRzdHJlYW0gCmVuZG9iaiAKOCAwIG9iaiBbMjI2IDMyNiA0MDEgMCA1MDcgMCAwIDIy +MSAzMDMgMzAzIDAgNDk4IDI1MCAzMDYgMjUyIDAgNTA3IDUwNyA1MDcgNTA3IDUwNyA1MDcgMCAw +IDUwNyAwIDI2OCAwIDAgMCAwIDQ2MyAwIDAgNTQ0IDUzMyA2MTUgNDg4IDQ1OSA2MzEgNjIzIDI1 +MiAwIDUyMCA0MjAgODU1IDY0NiAwIDUxNyA2NzMgNTQzIDQ1OSA0ODcgMCA1NjcgODkwIDAgNDg3 +IDQ2OCAwIDAgMCAwIDAgMCA0NzkgNTI1IDQyMyA1MjUgNDk4IDMwNSA0NzEgNTI1IDIzMCAyMzkg +NDU1IDIzMCA3OTkgNTI1IDUyNyA1MjUgNTI1IDM0OSAzOTEgMzM1IDUyNSA0NTIgNzE1IDQzMyA0 +NTMgMzk1XQplbmRvYmogCjUgMCBvYmogCjw8Ci9MYXN0Q2hhciAxMjEKL0Jhc2VGb250IC9BSkNL +UEorQ2FsaWJyaSxCb2xkCi9TdWJ0eXBlIC9UcnVlVHlwZQovV2lkdGhzIDExIDAgUgovRm9udERl +c2NyaXB0b3IgMTIgMCBSCi9FbmNvZGluZyAvV2luQW5zaUVuY29kaW5nCi9UeXBlIC9Gb250Ci9G +aXJzdENoYXIgMzIKPj4KZW5kb2JqIAoxMiAwIG9iaiAKPDwKL0ZvbnROYW1lIC9BSkNLUEorQ2Fs +aWJyaSxCb2xkCi9TdGVtViA1MwovRm9udEZpbGUyIDEzIDAgUgovQXNjZW50IDc1MAovRmxhZ3Mg +MzIKL0ZvbnRXZWlnaHQgNzAwCi9YSGVpZ2h0IDI1MAovRGVzY2VudCAtMjUwCi9BdmdXaWR0aCA1 +MzYKL0l0YWxpY0FuZ2xlIDAKL01heFdpZHRoIDE3NTkKL0ZvbnRCQm94IFstNTE5IC0yNTAgMTI0 +MCA3NTBdCi9UeXBlIC9Gb250RGVzY3JpcHRvcgovQ2FwSGVpZ2h0IDc1MAo+PgplbmRvYmogCjEz +IDAgb2JqIAo8PAovRmlsdGVyIC9GbGF0ZURlY29kZQovTGVuZ3RoIDExMDUzCi9MZW5ndGgxIDI1 +MzA0Cj4+CnN0cmVhbQrC9s36b9MLlCRTrcwvwoHtDSMwrggVXnGkk2ubKPINmAD+iv2g56GePOvD +ZD22cbU9A9RDS6CpYyAXceIzuQBQvmSzzD37tr1Lg8mdljB7EQBZg7JhAo+a8On5j8K4SWns2BFy +iKKzDJYTM0MCDgwte2QQu81IozNXxb0Qb5tpWjH97ZK+8kFbarUhJu3P4gErg9K2UrZZzGODk2J4 +pwDHFS/1lSlxohae/WBfCDFFDEp0MXZZUxm2PvqbKWtkA3vCaM8/kA48gF8m6iDbEX27cfYCA7kb +vtTuE2Zb63HwfXfFEJA8sKPGxlFkQadfzwKtp1Bt+WIdmM3BR7VmUZkACkkMs6A1S6sxPjOKIjQO +DvREElgA6JnEw6mGseKeRgRmiYGHaqK73ykfhKYyhlEinhhzbIT7gs2/ou5fwsS3E18BxyI+jUQl +d7AVJl99rTsZSfRjJyG3oCanuCOgy7QQLF6DUsAzKLlyYe+m14Kp4UuOpmAwPFyVrTAzBreYjyMQ +1mkZEfiu1/9V73PrV2G7KbPvs7zuLe091OTZzzWtTJ5dSSCuqafPHd2XHMhcahDc1uP6X8IraURm +6kVSXyt64H/8TlGUDP3aU/1LoKRonc4rXM/uE0JT+rGqlXdXr9Gs/f4U6dxAMcRdns65QUyI6FW6 +BgG4JyyuA1MvEpTJ4zT/Uji9u6zRTMcHpNBa4lXfdhRQTwB7Lg9XYWwdkexNSfoIb+APYUXbs+4H +iXaAYbV2J8203IwCxT6j3wnzDJJOpKT01rsENQA52UHWM7cEAgoQm13PmdEQHDTQh61psgi5oK/q +UrTRjosR0TCt8xFXz0zAO5ePMX/ZRpgn1FgF9EAnSmzBm7Q4/MOf30YdKDzM+q2q9k9ZKkEHZHEY +HtNsWXraxHEOaxskDm1PpssdHVbq+bKIDlFHZMAdJOAd8A6tNEuaBeHmz3vYa5Gi5PwQMpelsQ6g +F1BwVYb+0awinWRxxpHQdaSWmM+nt9Gd2rbItw08ERTlYo0MJe9TBvhB4jOCMsVT2UebmIBy6744 +bVt2y/FQPcXPt3dSxQ4T/m74aNAvsDuc/wmN9Zp5Hly7cjlzf3JrAv4eJD3vE6fRt5TtEZyFDNkj +h2UP9vW+ZEoRdw/5S3o85wgXFdvTmAUKQPG2eDuV8/T+Y5bwucTDQH2+PCpnjw7cpz5Mz2L1WSID ++4A9yeuhDdrPykVx6czuBDXbmpozg37sZ4o4rYKE4Te4/1A9lvQ/3JBJ9EMZo1d1f2VhisZYf9XE +f/rnLAFnquREyoTwmSRQd/G128PyqTpNVU9ruIlI2783iXYLx2tzl1x5PS0F1WhM2vVlCqfzSzHk +koFW4P+ciUv1rqXF2Hoa8WrxMgcqMOgyvKF7PJwNr7AL6cnG69f9PEcxwNpobnrg+p8W8GdMRGRR +pFZYC2kdzz3tyjhUDFQOQFnYEZe/f1TVrmJcOBZBMXibNYPkDmGBpvSJNcbiQkIA7zbcgw9gOHVN +Y03qBymmCAbQWSYIvQ5pp8FZ2Tk98wKsMTqKjc1ji+ZUTfeGF+cRPMiweQev3iNabuEbP2LMUPvb +eAC2RITndRkObSHYX2Y5G8jixBgZDQsmW1HiO3/nhAtNIZkEELYihhHPsPIVupi6Pp4R75ShuuY2 +v7PpNDeZL2mDHCDBwGrVII2J//05PCTe3bEGWncQ66eXCzR8h5DRXciVm0Roit7L3YaVagjF1OJI +Jf5oZ/0ZgD9vPbmoVaDt16/GTlFmUSjT+DWTYVLhGoVLr1KCSuzhE68Lo+jPXmTGcFGrefBX4PuK +r0WIKwYTK0mfDxvfLDQhMGDb2IzJRbXaBsGIq1Eu1yDF6xLT1x3K+TRSQyXLml/XVhHZiJqPtuBE +lHH6jFJSZYZLPi0RxaWA43FwZ9UZkX1EKQRIJAhQYtvNjSQQXMMiad/WtUNElfvD7AMv7FQ+7Pvc +QC+Sj3IdXAD6N9mHgjd3MLm45rr/Rxsrx2vl4KT6xK/4/ZT8+KQRNjhiKh/dXdGFtYZPRrRHJoOQ +yozAj4T1ao8LiNWsSpBRhbyQvs32qcgjwXDcBR0QHQXWSSL2OWQ9OJ5tCxYDpkURaX6/aZorIJCS +/+A32f1fCzTunWrs7GU7wAtKZDi/fEVVSljR2758fKnRrJ0G+DKVPODSrtZdVF6Ee3vDrJVnzeXa +1g0RvTD3nAOTQ3VuZ5sPo+J0H+vn2UTNlNkKjR0XsZk3r3w/KLVjZMMW90n8TVePhL/lpFHydcKZ +lrs3liXShXqls5stwuIj2m1Prv3TPI8BBjNP7hDOFYkbaxGwG7b/NleKx+qLEYgMKQP1iX58EfPw +ri2By1tz0+0cD2jYikTjUmzuzhNeNwkvi68KUiLc+euS8P9WE08qQFE4MNb/GFqjmdd0bWoG5th0 +6vgbs6K5KCd8YYA6PCGVtyLFdhSEHAZCzlK3QrE4lU2HYwjCim2kCnyBUclr658h+m3ajorJB6oA +Gd+Vn2t11FF6i8dl21+Qi4WgrQ3ZM9pLqL+vA2sboTSsUhOyHV5QfiWc63E7CZf6D23oLQdPgnw6 +z6EebtAAFOrxfe8q8+TptXJ/wbBFc1uuVtWcSdmJ/vJfIFddL/8C7kgKXhprFQzVpM/a0p0x0Eg4 +eH/NvHKAE9atK0sOFk0vhSeFRKQhePlM5LKEPgsJGqUsMLbRk3uryiItIjFo8I4gRZTtH//DHH2m +s9RvpAbRGt0m89yoSS/Dbpdf7iVpB8Qckf4EvEf6D4MXPN2c1mg25aPVD/YkioIh9jwJ0ujMY22D +yRzFf7GQ89OoFJRd2x8D/VZrWeIiEKnGysxE6/lvw2mAsioIUc8WtiFH+k1QG3D6ycdVZ7e0rbXi +7tyNPtzJTFmVdaC1uQULV63GcdaOLqpEkgDKLAnH8gjIDsVwJrhTw/MK4uHOcN0+ax9OlEi/xO9U +xaXTkOgAO1lQv10eigP3MCdT/qEEJ6Ylpg5wZ8XHQ9d6K69eDDCQldDvqCQwxFRFAhP95CtQm6pT +zgyJ+D/A5pv9iVnBjNMOZUFdzZTfJhTTHbzhbrD52wyGn+HmP6U9qcUvZUD8B99sWKwNgeyGKtqD +R9wyGhG8NH2g7mjDCM7DZ/hnh0Hj5+9gy0rGH5fuyBnY6NLLmUTLV59YU6jYXVHek7pXFQ8hCykw +SaD74Lq9RoxoJRuEZyBS16QrW0u2wGq3WAIJVbPMdiYKuujHYcFj+wN9UkzzwHTTP1q0fKxeqz47 +wVhypcqUT87mSaGAGvnz7n36hZnhSSWMnMu7nSXTuVsoz6CLQuffwvWOJo5WdP5h0ypvF3uPsj2g +K+2+pK0tTCMwBuba5JXHXUqNUe5USY2mFHkH8s6FLbJ4OAs9QSysfdp4vJIrEwd4/zKx2eT+2+Nj +Ioq+DM1faMjWESTnJW4BPztEzTr4q4ikx1qt8HOSXoDPWbwSkhb/KzEGwgI9hfxqPYRk3Wf6oh44 +vDSoRvxiy9nww2CqzJBEuUYajgyXY03DK/TwVvJoA0ayBCP1f11qupFOKIJjbHQbzG60+8wgh9At +Nzv3fFeZCn2Tgwd0oKuyNyAgk3BwKQC7fDYk6Wj3RTR2DsBN5OLFKslu+KLWGKBwz9FAZAqUAuy4 +R9ez+BaURu6Ct94wXslFeQoj4SKZ9auvbkE4pTi0uYm/dhDi70rmEUYieJ1IBQwnq8FbsqpSL0GB +UhGakE9yGvtPMBhvaO1P8WucB52Os/yXqfrqML3C3tW6JKuAh7jvXkNr7ZPpeU7ERlQFIG53dl1U +EHIumywsJmFeWQqEAAX9rU7mpxm4ptbOIhhIOF1M4i6kGww2nJGo01yT097WC+WAjsOqi7PMK5rl +fnM5qV1KbQbqKVyDyQhwBBrZ2xIwzk2z3Cutj+Cedk05nOsOljp4y6VuudHSP7WBSYYTqB0tgSnX +pO0csf3u7M8qmumslyqr8f/rLEWrRZJ/8PCnNr+gBsnGImwhldF9+wTviXQla1kv9imBHNP1E0/S +OR6yCnB/TNlrwIDAoAaRk6XEmI4MA9wN/x7C0NYz+88jgvbh3Gqrn1hMucThcEcDBywalx16TUBc +XYd0R3EJyqKtkuUhU6dfSVn0BNPJvLUbuBQsmX0/tntio3MKHo3eFZgDoH8LI0brnYEIwtaZBK1B +sBoSQ/CgRYHbRTFTlX4o/GpurRQAL8FWdMSaR0dDjWE62lbxQTX+teUm6VUrZn6FoTyWsrbDmH99 +jqkn+SWzDP3T1Xi6hofIwBI01WZ+DccPgS9tiaAYkrgtxte7faDOePoGbdUN2QPe0VaAKP772rrv +OUFK0TTgxQlSQ6aZQFJeGEORx1u9p4h3Qnxn3ZXIEF7T/2lTEiCAQaDo0Ni8Q+LAEwUcy0ZyBaSw +kvpecsOeAeP0vXbSJij4rn9idU2tM+xeIPzFETL/gz7NfSp36XMY2SEoSOL/xUGin9T6QriCDdJr +dx/MUwxmlj+2WLwEAK05q/gupDaSoUi40S5tRC0v6GM5A1h9+LBiQi7v1ZH/0PRhdkYQCTEU15lA +aUnQkNcOCRw3lP4nmXwTNI3el4t4+YziObeovzAQV2x+ebEZqzlMXTLLyvo5oNkPosopIcpyfHrR +MZC8xWKWI9goJDI2FN9XpqZ4ZNYjPmljLxYPS+d/yxyWNuEs7TLNreqHxS6PyY2ESKxQgbiKhSTZ +b38KVHsqOfusshdv15XlAEl2YdeQQMcCgkzpXRJ0KdXzJjFyV+1w5J/e+jLlejel+b26uO396LoQ +NOSd+28Q1/9ryJCn25/f9b65onCFXQdon8aSkX/UWjbX5+ciHbaokm0prFuaNIVQF8n6tNtN9+Sq +a4r7Yhe89t9f+Cxu8O5OlvzZ+FVRFEoccoVADj0K6kfWGSF9LE5TPa0jXIQgaq+Vu/x3h2BwOYZk +pRQr9hOSaI6FECc1qNXywtu2MewL2kQwEUO9aWKiII+4ybPC+1XabGMHcuReRDDb9mXa5m81sDod +lsgXWLdUfi1o1aS+gGrneH9T6dISRm9vdlpv3NJcXTGc+vY8Rfc5CIT3jtXJHL3doOEw665mUt2y +qpZBrWxbnuFRg2NUqhUjx8Frv8yGf/8yBI9gebF1mdcERIs1S/sR3PZcFbx1fblyM650IJUIIUvm +PAMVgX7eYRPzyX46+cUExzqFOLAEYM+6xe9Rk5HX3ldpau1Dz92bPCkH8E4CyiaP7PWFEC/BZZu6 +npU0tjFrU664+6uG6qgjC61mDz0iT9SmjvXrqbfL1AFYolDI2YmwjKtl2CO73FjgpFO0OO9MN/qv +SOdfEo5/io/v9+dnlpbC7K2T64gsDidV+VvYVKLOdZs1oc68u4QJcM/blKLjIJ0f1FoM38TZp8gC +mQcoTBg6SNh4zriRTw766W+inNagUgOrW0xYyihPPdGRzb/JGmljqEFOaY5/P4QPaQixLSQOg30T +hFOq+81oejBgovW4CuzKoBpp1OigC5CyPGTjPdvpM4HpmuqRoxntu9b1ipQfMQWd4UH24po+fYP7 +5R2HpNG/tdG+Ubx1pJ8o2g4SCyd3b3U9vsuGhcj93D041vrMFIP19f1a1N8QIIQ3tpKn38EDsB2s +fwTp+mHy42nVqdNuOuxn5SCTnr12gMO/F2FlUC/F3ZFFMtkUZm+vcR7Camtp2Pr1/pv9mrFxd7I8 +d607AuxaGpVhp6kCPy++JqfjjVShtkHCAusuRdc6pCZw222CVNprgOuSGRMY47W/le2h/Hvm8vF9 +gjdoeC24c4EwB/MRYWa6LJCdpLxAvv3IH8W1X3HKBz8AFLFCat6+gHMpjXt1M7Ec++lvnf31eXU7 +4q1WHGfj/HVt6J4fzr7ns4L4i7ksqZMRENMD4E1jDroqhRXQmcsdSGMxiwdBZ45YWtUd6nNd3xwg +9JPIxSSYHsBx03IVtwNoxqWeR18t9RgnEmb9q83nxarDp0JENJEMKNXPmgu97//zvyAOrPvzrm9i +FIl5+vT67oyHTQVzmxu5Iz7/Q4WGfzeHm8zOkQ7froz/fBGPEV38WkEwNzJV7ErRjE7A4T6ZfpM2 +i6qWxVPOYhsoFjshglDgeJm0sXYliZJqa2QTBj5FlUGoiN51RdYKpUtNTme0+Wx8qAOER07GrmOs +gNiypmJrlrwW0nnRIJJxiTolElys1t/37o+zM2RpoyqYWenZkehC+goI+0xUsPKzS518d5t1Gmsf +UZwjMPED0EotzMkmWGTsU5HRGdyHOj/cf0DysBGullp/iOipozxGwjOXfiZB7lItlT5IZsk/CaPc +C3aRxaOuLngfne0PGLDpbSNUvboWkj1qGgW9BF15nEnUFUD38nkkiK1GbsWgeGdefDEGOTdYPMzK +47GFgFL67VBgxNICckSZNJmLP+7VoCN+QKZjhmmGbF11lk+WTSR8VhyVkAghRqTLQp6HOavOYbay +jP8Bam13rzQWSv6pSVmaZhmHMNs63jKaMg6loTMtBqh+THaS0f1Ebvr4wxTqiNQmqCHpeM02n7UO +EZOx5Pi42zu4u8VJ/h3Wl68mmhdRfoIfo6JRbTYIACXQORkFJl068/pZJfV4k4mystiKv4MyyNBC +U2AqCLGqmQL2BlzxU/YJflfEZfp1lNUL9HbVbF3HnK/0K7DRor7WwTEpbRvkyjW6MoT2XIqcjDXn +uS/rnmWdnL20LBZc9G0wq2eZ4NyBgjseuV0rSaBIUDWJMWm4nZIcGjDmG3OvYKqEG6uoI1HrM3oc +Xno1jKWX6IT4XvIV2M8sgkt+1UWmSuU5VNKFDBCgq8jxuPhr84KtEI1Bfi/NYNkULS2Z06J9DUhl +jm+EYPxT12dbszkf0yYjOMOK6uEf/jgG+oIy/MmD4IXkOwHAMYVdmaao9tuEF1w6TNpNronZd92T +DgKdh07p1z4fC2evDOEMrl1dR5q0HXOGZVrSh3ibwpNVyYOyNuWM0sSHPx/c+t7KHmofvtlCmwv+ +8GkKePTg6PNJ7EpjqXc0BnMsl9J3P+bOGgZr6+k3r5HT21qSopwYn9EfH0YIQ8NpWTGXcLqWcpcM +bQKvNgN1TJy6EQBKWjR+j8T6oDb7y/rDhbs/8arGMPFBqjJZRGIhW2lCjnjmdf+KSkWLARFA1si5 +FefuEbLl+E4rlLeJ07fNcQqML5+3HkLjOy6n7hKbfIaAyhKqm04vPlT0QKrP0qLWO9yVXoH5nBB0 +A078/83+9WrvaKBJstduDel+jmaVkwwGMRBCybJEZxHNcr9MLQ9bEAfJHjucQW2+QaJHuq95JQ0P +gCoQkR1H8tsWK4FdtnHBUV/vSBJeUvxXTFqoW1TgkY6Yzdblau5chUN3Jqr1cMDAw4cEispBUIwV +wT5/RYMZhyT4zKXAbWlchcVzBZCjbd26+2ZcVuboi2LQh95+pwPV0m5V+oY+J63VmHCXtgxV8mZ1 +MCNbLVQU7ZHB133whOIZY5sid3n742NIlwUf+MA1FRru/03vMTc/ZDs2o/uRbuFIqOZ8NIy9lTiO +IdfK/+sj6WgNGANRnR8YMRn6ytzMMYfJ2AwEtkSrv2zqzeo2gXfQrQ3Ct/4cacaNjqm48XnmTFMS +0pfypZG4E/tJOgPw7gZdG9v2p1/5NyQwskBTrZGyzztQ36RsZCLDkqKu7NWLc0+B+lUsy2v4R9uU +L8PyNJwOb7rqjsHiOtxUho+kZU3EukNlJx6krUMM3Uw5GZbQgjov1FFlHJAOYFzW1ZRvVXKVgbBW +qY78H5wqQHmWnSO73TYwjDR7XBbgBNR/rnfK3grucrHkzBCBoF0M+JB829AqugTbltE4x/Hc2Z+a +ODEA35AZzwgZwgJkWy9A84Fb7b80EIc7jPigKZR8WeSlmpYD0nbvdA50fKCLOopWUadJLupJPq+z +CbEnmHjL/VmLgPUPBTHqrvNAtEbpziW+g9RRqBc9jq4Hw8kQfJhqcwxdUX52OgukfDc8AFRstALG +JlBBl2p8yOPMTTAWpRlhKfXkEYZXO8JbukOx+IbYNZOUWfGXkAt2NAYPpQKgqsP+A5yGhjkJWDno +6+7gpgZjJCC0xhKtWnyS8BJ7MckgA4sNsFbQ0atis6nbuTE7Y0BZiCZHJnpyxlqhz7OYdFDupaKN +P6tynwyQQMSyweG6nG/DBsXn8qI1azSLCAuqzfyVDMEeJWbaEg4etJj1cH584YUi8gmi/gILEzon +QsD/hAhl63SF3lF0kJ18veMa1ipg50DIhlNQfitg5BJzjJpEV78RGSVFUScfso9sgmSR5KYuO7PE +qYjutjb+53jTP5THvRkpZJftG7kFh1/p0zi/nd0m6KGitky3Sv+BGvV9ct5fUGgGnSsUHPntX3EC +youA3zQJA+zJH+3TDrRWQlkDUcoxXCBo0614jnRJJDD7BYIsmOLHSkZFnUStfOMoHG0IH9xA+64B +u6MZsswNL/2RKg60+Xbyd9WXAtofOJuzlA4aS+pm3jZmodbfIo/Y5q1tUhEU92ISKCrLX4m/hERb +dFTXHO1mRTPuPpvyp6nNOwsl2Q+LAmyO2t1vDH5ynyCLkL3SADk1oSmDj+ssLXFCrVtl08AAaWJp +MauRq8Eyi/eFofTRKHCPDswPFCyjvpZb3hfdlrlWR3Iy92oJfosuxOZTTkwQgu6qfrzhcZsPN4YB +KC3evYvUe8ERAEbAUxJkaw+CYIZQUiUDAYJiRonOHjWn1fmPfEPIBAR4hRC/HZIrFHjC8m8O+fXu +Kt4V3qCtESup/iyzv2IzcGyxBpBtaOK+DVc40kMeLqZ6PeWAwiGZgykABkxWRUCb75zFXfIip0Vl +G56puawuIQ8yfDWvz7v62mJxa6MpKG5nD+YRJE1C7MNDB6XYdK+AaYkRbpD8yC3E1Ar9eRr20/mL +YKwXiXRLQIwcW4AX/fYa3ti9LgMm+C8r5yhSDWKMXnhhLCwtXcknTcDPWVcOOresXyILTmzHVn08 +meRtBOSD0kU2/2CxKMYp6Hzjo/gtoQb9ii9wmcSFUEJPIzkF3tIGlfD1Sf46U3I323udRMizRq2f +0iqkPTmJ/NQJUJpdWcVWdGPO8tYgYTX2aoMreaZSRVPtZyPR68OdfhHOpqxV4J6PLXa+PKXV+T8B +t7GhazY2h5gQhZvbCs1unDHfW2NmlE//33+QvAqom8V4eCQwUKScDji7V45G4z/U3p+qsB1W+9q4 +2nk9CBzLDBwazl6izdh1QrpmFK+A9pNVujwf64yzNeAGc01xu8PQnKVyPuuwVf7SepphyARY2hLl +80rXkA+hW5vjIEPGHv5E0IGlzZIId25WC3bBytuLaILjkQ053juMxvyOD3J8XCyzs4d/5QaKRNxH +6d01ewYY79KLJyhqhoqXQ0zDqdJkyr9Tsl2NZmiP6MNG1nBoUYHMhIcbTBFYWXQcXBWs/MRHlrh5 +d9hmmgXke6VHS5XaFJBn73oAew6nuoAQqrCVK1gwYhHfHTeH9iV1srF0am8qnzv9Ct+tafgocjOz +6VXiL/wtPnc/RPnDeiunRaImUOhSCHnszNnU4EjEoB8wf5zwRgiXWMoHbc6+rQmV7rrrXJ70F5bz +z6scYwwolaGpR1YfYNyd56aLVuuffyTaWrSJxgXRuhrxLXh2rSdvXz+tuGd9Nt+ns3z6RLtBUNmu +Dy2320kbcIA97cuywM5E59QRcS+mlLk5b5Rms6NuapMCo5/08ruZAjH63u0XuARIqb/ZgKW9+IEb +PU9psUF+e5d9uxSPB40A80vNgOj01SSKR8iSXGvHhl2cpPxQ3YYMVxsKe2iv6TdLZlFQLon0aW8a +HmsND0Zs00ugnQXyC/DnYjWtkJ0cF0G7IZgmxLcR3ScagarXUC7asowALOSztznAs62iJ/aeAvfL +W18SgmkxHApjCBPSUWLufSb4VF6/Vz+NKx1BXYWvZWxEx7dswrc5C59tllKijnsSdrg2tG0Pc8wI +j6QqDVuSfTPgyLop+pmWrOFu7LwkI1Gw0dwWLCitkfIO+2ZYwJRBmeJb7zSm7CE/mKdHH4lMJOO+ +zjZ5VorMbD5zGxTjUnRFvQv+yMrY3YZ6MxzqLSuqTxowJR6XrdBM36ykzNoXffW9F9aBMJTqSlP1 +8bBVhSKKbPFobdvyh98KlEnUljBkTwg3AMZlPQQzlVJGaIRLHF/HgQ7LcF3v3OxVY58WToaCsb9W +5tahHL1Sah8lGElIilfsRnRdko7coEu2gfRa3KgaahfUwfvu2M0XZIAv9XUtAKengbLZeJVB/yeu +Y+bzabQuQDYArNUq5KNCt0/e+vteyydM5vLvATJaSIrx2GfK437ZFaBQmB+R+G0Aa9ofSUjkoLmS +R0zo8R6VPHJpZpqodQgcHRA6RH2W9slDVbbdwuxFowscvZkyg0s8XevJm95iv/q56pJcGv62lrDl +lCsiec3/9iuFz7GxwOmrty27Y0qyBS2LS9M/+IeoG38vIZNmSko7gXXrwIoWMN2hApN9a99bEPzK +Ph/Yxbtl2fg3cf4jjcpB/MWn/HFrC2RrYttdGVaeHGppTNijHQQoSHtd3Tto0o2o84a6tif1874v +QPyKQiYA1EDhGbJXcw+8IFq8U1npt0rH+pTD9S3w+3Y6vgmQg+1sLBl9IRJ1CI7dAZtSOocbQRjU +BEkBAgDSAFPXQpJL9wN4CJu7YRx3dzbmYEVmxiXBkJtvs/WsdgG1fqUV+hN0XCzCgf6wXQ/i72zM +hwTcH9y6WkG5SjzFZYTYeZpnXD+H+kIZX7X/8VX23d37MoJtrrFVBrVRVKbE34S/08Mu2P8dWZoy +cgWG+LJMVTIloYDwtvak1MOLsCdjBht4556ZB3LPxnZNnUiA0FHWhtr4Wjr8ipgrWOSOXOjYymmE +1Co11V8LAz/znaGGha+GmY0gDeP5cOugmzyQrj9kR8D7osdTIpYFvNJPK9zIFcgP1hPoNfhVCRJA +H/PERcgkXtTRID94sLUhh98wxK9GdcgArwUTnradBN+1kLq8kJue5L+1Zry82C417FGOcMHc/Tqv +YxdO0Fmw0umiFGIXPhR0q2ArBOigPtQt+aZooydGH7wQ1r1MqQoq7h9lAYAP5IUM1T92PPx6Hzj+ +jAhDgCKu9kNhHWdrRztzLX10B4aB62mPu/cHSvdW6ge1kfWO34HKk+VnGWsS354ujNNt1iJ/WgjM +M5Mg5NNUCd66R/u3hXoprhQvErO7ZwCtIDZIbu8csQ6Hjp+1VrraFV/n+1iZBHS3UAmeTSTAJrms +h8zKaK/c7y57cWDRKIj9mbso2pCRYw0DadkXgJRl6hzu7bbEar73ykHclBPBfz8RbtapNmtyhlNk +pocTelxpTWltOD5BpD9o0W9gB2Ibp8OZv85h3lYUudWhMr38onwwuMdmIdzUyjO13PlSC7v91kcQ +NIa534DJ6uS0uCuhDKXouQrMdH+mrK9U8G09HZFjdldFG79sVOp6NX93Sc1zPl0k7wab3pm+PD6p +j5aZNjAlPDJUgAaGFFyvChhNlOs+mWZ0e9kW+iw1+qj4Ud7PhU5qK0s+SPn3kAbeOHxT6TQwixy8 +iMbluTZA4JdPRtRpdasV3AmIAw9n87/XRCZUY6XU62uFo22ZrhhA5pPdlt8Fjl+jlhzdC/+MMXB3 +7K7G5Cbpnaw6GlNrQgRwFvNgD+x5Mkh1FCyh7NDuLEzFJE4bayD9s2yvzCuYQ4Tdmnns+hncSLBh +Le8Abx/L8Qdi2R/hU/kK92kSja7mC4qfYQJ1Uf+yP14U49O/1ilAP/F7wePceJBEdl4ColCx5ujY +fzy0ZWMFPv7yv/zIMEfFnu8Fetkqe/D9BBbwmVbu2YxxSk8xLqaHYSNDAxL4mNxc8tnW9+fNG0EV +SF0pkKAqwsKwQG8fdAnlzDaqznmkmRhSVGlYNhFeIcbQJ6O/TXKGY9EESwzSXoSGXaFqY5VTT4fU +dMapXDjlMwtyaeXnn4BCv/3UUhs5XjJz5SWJ+WyRUy6IrVszdiw7BnaAEXy4VSRyBGpo4DLluoGL +7npEpB5D6aiOOyeX8Y7gq+27hIrhJWq4oht6W/jGoAHaSaviZWshW590DH+1ZJ6pVIf1v6U1e8mX +y9dx2BrUrY9FZXMLPPwJJefXVjseDhoQGyZ6h4FznG2VEBxaHwJ29x+Bv6NCmY+ou5wCN9ZYYKM0 +56vfwBywuDy1YjT1Pgi2MZnPv9wxoUTqQOtiYClMq2zw7V6gOLaQrPKEeiXowJYaWv8Jr8epS26k +W/d3uFNHNTEI4ml/5XjBRRjc+HnkThYQ3O4vKasClQHCtSf3nksGHRus9CJLKDsib73TRpZe+ERL +wiFsHfdQPBEUmJVzxy9UEcLFtykhG0wpyYbRbvHKZEnNV7yzuf9VDjVnzgBCNRkEkjA5Er7yYuJ6 +nsz/xwP9QGvsmZ0+0VvF3uPvohLvHcg6hGlZyMgdSmWIapqrjOGKY6RN8p7cLTiCKb+zYKbGg3j0 +bw3bQU8UDdnoqIWZZ3/7ahKdsYI1X9qori+pJqwjULNYIX3epl5N79MJHNV3j0Ci315TBulj17oh +juNJhQ/L1OA8PW0VRJtkVd/kCMzSaCkBoxhJEt24R3hHcQun7FkwF+osb2XLAO1TZUCU+XqwtjtE +mD8jguVPtESvWgZ2mYJo0RByBCC4ndwuD4vr00P462JaxoYN4IEEjnOVkQn6kespR1yAMXWwS+t7 +cAqhb8ajdnHUiySiCCZTqHWXELzGdd5m3jvkXW9eAjKY0ewV5G19kMpdrU7BvXRXY62LGiLQdN1L +FmZ0uiST3Q3oWFF3FYKqwBaXXrO5um5iQFxwvJQH5F7Tqri+OeUb5oBThtHnq51axS2kbBzVIxKe +fGBlGXEYywcioVg3OeTbGqPMTwDc5P8URpNZI0J0h+Yc+6dzcf5DtPci6sFz3BF0mD0uY3pPE1wi +dY2yQw0kNGshto/z1LnpdFbxImrxNE01gmv9kjNLjL/a4krVVigChGtYlJEPQhlz6P6Qgc+K5GOm +zjLNZtyAS7Pif907JOpZpMU7Ye98pxBlHQwwrEKvF9InzfwSN6v17ZXgaA5SMTjsTgYml/tHDle2 +chIFEIk1ZBr+gDMN5ZjbnhhVhdPa1YwLjiIlcURkblcnSwFw3eC3myuw9Y2WUg2cW0nDLUnGVtaM +zFD2OpwhToblsBbuiAsGUNUOEZQL2YpENso1t5pLj4z6y6I53vmtPJM/SRWT/t31AxiL7Dice5B9 +NXLLjH3eYw58AcnpoLUtNJ/dXznO5k7qTJn8EMU6hlvIwrDaO8DlNEhvdFsSWHBYn2El/zip9LY/ +nKfrgO80IQbfjWIhGoy+ctZnjQ90B7Unf1wcCn/nSCFl+xDtS9s3Ks9lszWLzit5ADsGe9Bll6cX +QEBd0C7iAj5D8SG9x9izhMudyj4Qd5pd/uwu4s4HPiWI8qwAK5aT7zZNahlLIQSMH6iyBRSEBskq +IxkKUiPuecS1jEumfeL5F/CwmYf2lgN7TYbih1vQUoIwqpOT0H/sxFGDTXLroTRGHR2VVHIpJvgn +l++lgbcJkW5AddVtu5fqw9PufdZdAEovQeEUhiECnu6Qxf95tJnJtkzFEh86HXSWepsfozIUz+Ut +sHZVxQr01mpEGMsa+zJHcvN/S6BD8sFZ6ZKgL+Glgm8AlZqb+yohkGEiPhmKFKGHs65+bY9ciRhN +dmLOCRZdgZi8EYIjw25X6aLfAB1cPwh64ciQ1sGqAt/yd2EemNV7cq9+JFhmJnb8CLp68NPxTWx1 +28mupoBAeyn7b5zyyLra0FcwGausEfvwBIdibqcmZsRJerj7BhZi4thM6/DPRquemfMJEvnFLy13 +3rKblgGr5d07sbwtgwCQ1OfjbEmLc8MlAJQ+iT1Vh0wBa4yISbXXLUKrG//9TBRtMk2cKzKCz0jV +ec8+VH7NOv5cZIGKYZG0jAYUnt16Je/zdoemWix3dHq1ZU7h/ttTek9XygKnh5V6Y+COycNEJewG +wz2lXJvD8NnsgwtNg0N9gmNeR4XzR9MdoxSXulskNrgIiV/YMemtpoFpQHnmNQ098PBbNwXnfWCi +OHxH8xHtcTuLcamIBYqaPqUgsw0TB3lu22YWDJmDtzA29fRQo/xeGztSPeARjNhB+9mzv5Dg6xIT +4gGWIoBa+ReS5dCCTNalEVEdafw/3BxF3ATXfF4dbABKetlRNjHOCu7DK3oL+DBTirTVTXW3wy97 +NzwaNA09TEL8ezyfbnU9lDbtTJA38rjH4ikhHlWjxwWx33oQkDKNr4OWgy2V28Wpxa6u8wka4nL7 +OAxdzt851TYDXtCO8E+yA4me14THakjeskiCsf3i7YvxirI8JLATVNHlc8LV1FRwAr6V2E1VXP18 +v/BlbeRK/o60CjvqDhhx7jb29oFxsJilIXuiMRebTlGYNAQcGGssr4KbwCkm9zNLLnF8x6gUgu5h +ZOX49xjM0V9UVuw5dg7ZEY1mTJNzbUud0kSZu3W7ZHVimk7y7e1Cby6W/6eMSPFHQjY6a1GwhHFU +wMSWxpDzcltBEutaWgfS5o9GFa2ihRAEv3RJhJ6tmmUelT0g8rYZdTRhSbwRSLSVn3aIeAbFMb9r +khbgNwVx9l076i5OC9ilLA26FzcG+NyvMUpcHnkdpIGpvOdNGh7DsnkwaLGfxv5GW4cayKF5VgiV +wirDzCcy7njqhGddt/JgOSxQgQ/KSNc4T/P7vLgQMG6bk9p9TqqEHI/mUlih8+ROYjoV/hyb3lJ7 +tuvoP97m693UCmVuZHN0cmVhbSAKZW5kb2JqIAoxMSAwIG9iaiBbMjI2IDMyNiAwIDAgMCAwIDAg +MCAzMTIgMzEyIDAgMCAyNTggMCAyNjcgMCAwIDAgMCAwIDUwNyAwIDAgMCA1MDcgMCAwIDAgMCAw +IDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAyNjcgMCAwIDAgMCAwIDAgMCAwIDAgMCA0OTUgMCAwIDAg +MCA1MjAgMCAwIDAgMCAwIDAgMCA0OTQgNTM3IDQxOCA1MzcgNTAzIDMxNiA0NzQgNTM3IDI0NiAw +IDAgMjQ2IDgxMyA1MzcgNTM4IDUzNyAwIDM1NSAzOTkgMzQ3IDUzNyA0NzMgNzQ1IDAgNDc0XQpl +bmRvYmogCjQgMCBvYmogCjw8Ci9XaWR0aCAxMTQKL0JpdHNQZXJDb21wb25lbnQgOAovSW50ZXJw +b2xhdGUgZmFsc2UKL0hlaWdodCAxMTUKL1N1YnR5cGUgL0ltYWdlCi9GaWx0ZXIgL0ZsYXRlRGVj +b2RlCi9MZW5ndGggMTE3NwovVHlwZSAvWE9iamVjdAovQ29sb3JTcGFjZSAvRGV2aWNlUkdCCj4+ +CnN0cmVhbQp1br5FLjda1eD0zjtyAvE+/lTlB0Dp6bsMLbxjDhVJe5KGBEh2bGZM+87+10OBJx7v +20LR6d6ro3zRvq5rM1pmCLOQmGGS5rcffyhdOnzsbeYQ9e1Lp+wonfzbbxGEm6K7V+6AClW9aGey +ixjx6jkwBWZkg6IuzRpFyXgNj6iE22WuH2YuS1jPnRTdbYPy+8YCEJwt4CpBYS4uJlYqnY79bfCa +BU96Hs1emDXJjFp/3IHK61M1fNOO7eeb5Gze8+kUqE8MQioCRm+Gy7veXfjf22cuh8LH9HP0Qbeq +YTPQX/7yEo7qdylXQNQMq2Cvy8Ms+UyzNs3Rbq5Zc3EgjZKSM/2urW2M7PWLvrdvuVd8LzH1+yaB +tsDbQKfLkj+TL+bxNkWFy/R+8wKJTaSd7vUVbjF2x7dGzhqTyrq3Yc3lTy//+UKgFwzjdPKPSU11 +4LybpTwR6PeHg9pWlEhra+m0/ovLPV/QAyPhF/0b3c7MT5n1CC78iQwvgm9QZH8+7duTnGW7rw9u +cdWJGeS4S6n6x/kldfl+8GvsqzsHizts/C6DYnkwH2JYSXnIMcne4Q3C7z5gJxDLnO/l/rfrzkIO +wzJn172IiNpcT/astSp1G35a7W21Mw/dWMQYy5L/GMhDdoPxnLRAhhAm6ajFThd92qj25EafimYw +owUth7IFZv/3akjnQ5A8RMQDOavphwtIlCXGo6Rjtz/omr67M4AML0XmOQDFRf6iikfVWkrsSzw3 +BEB6a2FgOmYPWFE/kFqmOlBsiyP26PMyir7/tBxtKjgzwkrzMfUitUTlT0PD9DEwcBud0mTwtr7k +gzTrFXRFrB60z6GNFZIOf2dCBpa/VXPvcyfK6jm1MM04UWlPGHpoRYCmmUr56foDzVOmXNMj0OJF +85MZn/1Vfm3e28yC10NzaUiZO8imcfC6x89oiahlVXmOK2INa9XtMMoKLE/XApcOf1Q+VoEzSJEm +xYEx1/bNyKHKSiYKlr/LFS7X5N09B11lztkX7nVdknVLCj82r9eNcrZ8zo5lB/UEzg/C6wmgxFgJ +pHGQhY17ZL0d0wrHfl82+kkoW5m4eWrneNIz+MZkIyKoTt1XLUuhnAoXe1GPIhQqEjhJ4fRL71Il +RSC2+SN+ZuGuN+j5+esV+iNJiXsw8D5HwpKlybC5R6bQG9m2CFFGiZDUck4lucnLtgfBXmIIimzx +I7j65osM9FH4FB5qk8K9yqhldW90jtQgsWw9k0Y14ykBALpjxf0pi/jkks33zTrHU9BzDWyhice+ +eXpPHkdswQ9SuJOSXUr7oWqcY+6BnsW4KuzkTZeHgsofPnpyQ+ebRDqbpgvkdu3s1kwCfb0MNTT5 +4EuZBeHkL3jhcgrKwHPQiFLftxCaXMG9XPX8RwbjSZGnzHj+lfW3XHDE4V/zAA3sxCmku67IjEYI +adyy6R3s52pXEmmbqHcXZlH46jfdvQG3GRJsR50oHRCXyKM+XrRIRLfYXhMgiDjvsyY7UJ1oQOEy +b3lsxK0Sm7NwfjtWsIFWLUlQiu3LXTKrc60OIy0VkUQkOIn8a2pEOb5hFhIVCmVuZHN0cmVhbSAK +ZW5kb2JqIAoxNCAwIG9iaiAKPDwKL1IgMwovUCAtMzkwNAovTyAoEl+d7xqkNcRcYiGTMAQ9qsP5 +2Q5mlp8QaZAWL5ISodlEKQovRmlsdGVyIC9TdGFuZGFyZAovTGVuZ3RoIDEyOAovViAyCi9VICi0 +adZcdFxy+EWzWcIvpVL+80kAAAAAAAAAAAAAAAAAAAAAKQo+PgplbmRvYmogCjE1IDAgb2JqIAo8 +PAovQ3JlYXRvciA8NmI0ZTRkOGQxNjRlZTZmN2QxMmQ5OTlhMzM4YjBhNjdmY2JhMjk4N2ZiNDhh +ZjMwMWZmNjM0MmM1OGU3NWQ3MmY2OTQwMWFhNTg5MGFmZWYwODU4PgovUHJvZHVjZXIgKFt+fb0m +ftbH4R2pqgO7OlcpCi9Nb2REYXRlICgvdH+9J3bXx+AVqK0BuT5epikKL0NyZWF0aW9uRGF0ZSAo +L3R/vSd218fgFamjArk7UtGKEaDLeIgpCj4+CmVuZG9iaiB4cmVmCjAgMTYKMDAwMDAwMDAwMCA2 +NTUzNSBmIAowMDAwMDAwMDE1IDAwMDAwIG4gCjAwMDAwMDAwODAgMDAwMDAgbiAKMDAwMDAwMDEz +OSAwMDAwMCBuIAowMDAwMDM1MTc2IDAwMDAwIG4gCjAwMDAwMjMzMjkgMDAwMDAgbiAKMDAwMDAw +MzQwNCAwMDAwMCBuIAowMDAwMDAwMzY0IDAwMDAwIG4gCjAwMDAwMjI5OTUgMDAwMDAgbiAKMDAw +MDAwMzU3NSAwMDAwMCBuIAowMDAwMDAzODI5IDAwMDAwIG4gCjAwMDAwMzQ5MTMgMDAwMDAgbiAK +MDAwMDAyMzUwNyAwMDAwMCBuIAowMDAwMDIzNzY3IDAwMDAwIG4gCjAwMDAwMzY1NDUgMDAwMDAg +biAKMDAwMDAzNjY5NyAwMDAwMCBuIAp0cmFpbGVyCgo8PAovRW5jcnlwdCAxNCAwIFIKL0luZm8g +MTUgMCBSCi9Sb290IDEgMCBSCi9TaXplIDE2Ci9JRCBbPDg5MGFhMzdhMmRjMGE2YzljNzhmODY0 +NzhlNDNkMzg5PjwyNWU3ZTAwZTc5ZWFiZjM2Y2ZjZTdjNTM2YjgwNGJmOT5dCj4+CnN0YXJ0eHJl +ZgozNjkxNAolJUVPRgo= + diff --git a/test/functional/messages/pdf_js.eml b/test/functional/messages/pdf_js.eml new file mode 100644 index 000000000..0eb7ba116 --- /dev/null +++ b/test/functional/messages/pdf_js.eml @@ -0,0 +1,1800 @@ +Return-Path: <root@srv.example.com> +To: test@example.com +From: root@srv.example.com +Subject: test Sat, 26 Jan 2019 12:04:58 +0100 +Message-Id: <20190126120458.015328@srv.example.com> +Date: Sat, 26 Jan 2019 12:04:58 +0100 +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="_----------=_1574345186400022" + +--_----------=_1574345186400022 +Content-Type: multipart/alternative; boundary="_----------=_1574345186400023" + +This is a multi-part message in MIME format. + +--_----------=_1574345186400023 +Content-Disposition: inline +Content-Transfer-Encoding: quoted-printable +Content-Type: text/plain; charset="utf-8"; format="flowed" + + +--_----------=_1574345186400023 +Content-Disposition: inline +Content-Transfer-Encoding: quoted-printable +Content-Type: text/html; charset="utf-8" + +<div><br data-mce-bogus=3D"1"></div>= + +--_----------=_1574345186400023-- + +--_----------=_1574345186400022 +Content-Disposition: attachment; filename="DynamicEmail_AcroForm.pdf" +Content-Transfer-Encoding: base64 +Content-Type: application/pdf; name="DynamicEmail_AcroForm.pdf" + +JVBERi0xLjUNJeLjz9MNCjEgMCBvYmo8PC9OdW1zWzAgMiAwIFJdPj4NZW5kb2Jq +DTIgMCBvYmo8PC9TL0Q+Pg1lbmRvYmoNMyAwIG9iajw8L0NvdW50IDEvS2lkc1s5 +IDAgUl0vVHlwZS9QYWdlcz4+DWVuZG9iag01IDAgb2JqPDwvTW9kRGF0ZShEOjIw +MDQxMDI4MTgxNjQ1KzEwJzAwJykvQ3JlYXRpb25EYXRlKEQ6MjAwNDEwMjgxODEz +MzgrMTAnMDAnKS9UaXRsZShQbGFuZXQgUERGIEphdmFTY3JpcHQgTGVhcm5pbmcg +Q2VudGVyIEV4YW1wbGUgIzIpL0NyZWF0b3IoUFNjcmlwdDUuZGxsIFZlcnNpb24g +NS4yLjIpL0F1dGhvcihDaHJpcyBEYWhsLCBBUlRTIFBERiBHbG9iYWwgU2Vydmlj +ZXMpL1Byb2R1Y2VyKEFjcm9iYXQgRGlzdGlsbGVyIDYuMC4xIFwoV2luZG93c1wp +KT4+DWVuZG9iag03IDAgb2JqPDwvUGFnZXMgMyAwIFIvVHlwZS9DYXRhbG9nL05h +bWVzIDM4IDAgUi9QYWdlTGFiZWxzIDEgMCBSL0Fjcm9Gb3JtIDMyIDAgUi9NZXRh +ZGF0YSA0MyAwIFIvRklDTDpFbmZvY3VzIDI5IDAgUj4+DWVuZG9iag05IDAgb2Jq +PDwvQW5ub3RzIDM3IDAgUi9Db250ZW50c1sxMiAwIFIgMTMgMCBSIDE0IDAgUiAx +NSAwIFIgMTYgMCBSIDE3IDAgUiAyMSAwIFIgMjIgMCBSXS9UeXBlL1BhZ2UvUGFy +ZW50IDMgMCBSL1JvdGF0ZSAwL01lZGlhQm94WzAgMCA2MTIgNzkyXS9Dcm9wQm94 +WzAgMCA2MTIgNzkyXS9UcmltQm94WzAgMCAzODYuMzI3MDI2IDc4Ljg3NjAwN10v +UmVzb3VyY2VzIDEwIDAgUj4+DWVuZG9iag0xMCAwIG9iajw8L0NvbG9yU3BhY2U8 +PC9DczggMjAgMCBSL0NzNiAxMSAwIFI+Pi9Gb250PDwvRjEgMTggMCBSL0YyIDE5 +IDAgUj4+L1Byb2NTZXRbL1BERi9UZXh0XS9FeHRHU3RhdGU8PC9HUzEgMjcgMCBS +L0dTMiAyOCAwIFI+Pj4+DWVuZG9iag0xMSAwIG9ialsvSUNDQmFzZWQgMjMgMCBS +XQ1lbmRvYmoNMTIgMCBvYmo8PC9MZW5ndGggNzc2OS9GaWx0ZXIvRmxhdGVEZWNv +ZGU+PnN0cmVhbQ0KSIlsV0uOJLsN3Pcp6gQ5+pGS1m/hvX0CozD2DNDtB888+Pwm +GaHMVHVjMGiQRaYofoKh/759+/v393/+9fN/3//48/3PXz8/vv/16+fz8evn4+3b +H7/18fz9SP7vaI/fz/+8ffvbP/Lj37/f8iEPMymlHXn2R7e/pffHh2nq0eowTT50 +lMe7acoh/VWTSzZNParM0NhvWjebdIxUXjRJzqOgmClvmnyUOU1jRyoVfZbNxH5K +40Uze707WViqm8l+zfe3f5luHr217erjmGMPedhF92t1u6huV+9HSX2z0aM22TRy +yKxbRHq0Ji+a0bZr9KPW/abDMlheNJLH3cnusKd0v2ZcvcpRxlb12g4p2yVqtYu+ +avai13J0yZtJPsZ81eT9EqaJEl+KctRxv4MpxtguYUfLVj8POGm5O7WjSdtM9lvi +5rqV2HrU+mI+vBH7aI9f392oWcyzhdG0j39AUyc0uTxcThCHTohNIdf6eIZDSmic +PtUs7IRGWWbIA8m1/+GQzHClu4dBXwURyIoT1AKBgyAEtS+42JZ/RJiO2uspwyG3 +dlnUabeT60STR8Lv3ofmYRoVJGtOWEjXyHmem9hKo8PSqHVlfCBD7hZMyBNFG3Md +0XVGU6WcEURgTT2ygYUHnUqD3Nu6hsKiNGS2RmZNnvCQGCxr01JXattk4yJVc42w +RxXl5u/Wh8+3u6YZQL1vGolTTW49ZG3CDsBwdJ45K37v1pURA4LuA9WRfoqwb102 +g6q4VVf4W+9CLqv+OSEi9Q7zcg58QQWJVCTOJ4epnv3UvN80NiCWKmhGJyo1pv/E +bISVB3Csmh5hVMWs1rrST1kK+7ieMpOr82aRrXOvwXVZJxAjl1UO7IlqV/QCFg6X +J7dscmvwKOf4EYduM/1ZMwNCftx0Bknn4FMxLCi4JQFajSGcbKysEd2UmHiT9ey/ +gDPTsP9wQZMrarumZKTlIdFdPjeTw4256Q1JBdbZnM3l0bJAQ4s6IUvvm+zJZuGo +qSxtK4iq7EeWthxEOqcfFirYV7nr435Nl+ExZDOYpfADOHEyAluQCzWVMQhwNVdk +pqY1qWd5noHVcigny3L7AUXOS/EeCqFB8VNNrJgTqTnk1pUywjZNpkeDRw0YrIEs +LpeRTxkeeepmkVPfzkiZw5jQoO2Y+GQbfrF2jELAEcidcNLypAM0V//aNNSvNecK +PL0+a8zW0J6ajrMdsu6xYNpM5nwmLjcLnjOfBq6bSubEUu7wyHImqAIFUHpLYSon +8iClhJ1OhyqLNsChVTrwSEmXDA8pc7eQvstj/V7ooYSmMnCGnuxqPq7WQvKer80W +hKKNo2YSiooG7JG3pXG5kzuob1GThbt/kYVul4NmBp0w4qdrUDaxzUkHaAwAM5q8 +kxl+1lylX16fNd6f0Bjh7flLzeVlFFP6l5rr9OX1WWMYNj9pHpBwSy0UyYYF+DOC +AkQaRoMFaaPTD5dLPvEMDqXiE9XZiMs8oXQJuRbiT1selfDjZMTEpkS0CQchPLkM +B4DNZaELwBJi6rVvEDcOgmRpuObgNikKcShD7LSfRN3XZBt3TJWaoeuBhkM7Vx1T +qZ3cketxkKd5iyL3bVHBiovfmzqAVmxNJ/rYbx+hKVlOjcupcuna3yK+CfmVHo0u +ifzPQjc8dJnHBskxkQQVA7pkf/GJX9Tt03jVcFM6fwoZy7h5ev3AShnA65oxr7Vn +cmcBakUMo7ECU+kx2VhFec20ajpDzimfNX9GYvDatK4YYbAetdleVJHJ1TTa6SCl +bRZaSLS91QX5wpF0WGtS4uf1cvTq3UXw2ZumNXx/8DxnA/H9wWEDZ3ZNu8YvIlLy +CUYkZD2a6dBG5/ji0i3RoNWQ6yIoSofCtGpHTJmjohNpTTyxl9UOnI0+UOrOOviT +ISqbsGRTWg5C3ph9uqKfsNmicJfYZj4bjpw60WGx8szuAs8cua8TBjST7UaEjdeM +h6igz+k8YSiw00lwNFcwB8NS1i63ed85nriQa0HeW8cJlZVUrae8aq03i0Lu4V+Y +IWflieCO4m3DM0XDwml0yDGl3gOFMS8PWYwoBSi5z+ivGlKi6IdCgmoyD6my3ifr +o4XvE2WghU8oFQZOnuLvVXjkvJYezkjrkdSQzClcQrraevDp1wf7HtKEvZKV9bm6 +WkgOl0XrXHQdcp24Ra9rMmtGR+gEWpROuc5Nlr48Tk1WwgVfMwqPWilXWWd0nOrP +04iK71mih5CN1rpmDdNbo9KYbsoJwzmYuDKWxyzUxEXLyWBtO0DmGT7FLN+ARVZ4 +tMJxDKAFtvi4+nMLHpr5cqoosCoRYOgm881y01R+c8H76lwZJE5g5nEqkM5bA73N +R1/AksM+yJmvUN6jYJl2RoGb5njIRmYa92CfCyvn2p7Eyty2vXjfnNimamSb2THb +j5vGIg78U9JwXzsS8nolpozGUfsuzunSwyLzbrHFRYJgXlvcNU4Z7hYySVX93WJy +m/xCb/Soc5GNEhZlZaPNkEHCLH95nQGNYV8NeiKGPDG4nzUngzy9vtIk7nqLJRE7 +A50tWjL+PBDLwqkilbFoh0fNuN8g0gEbxaj6jp7KLl4WakPJSAPGrArsclBxLxM3 +hOTHva4tVs5e5+fbj3BZhMGx/+OmWdOltvv4/GOdJtcMX5yuSXIDR+EGdyhEbjT3 +U4aHtLZZYJOYnFh7guuqYy2DJyBXRRY811vlHchkqzz4NSpWar7Np8TL6mS/fgYg +odQVZOUAg1sJxxX8OS7BISCf9ovn3WKQs5cxkDry58ZEKB9GJZJ5L0ev8/FanhhX +NTKyCPII8qvJrnBp3jfNlKXJpLu+PFzkPA5bRy4nJd/NmLZpRxJ7ghCbXCmXHvLg +tHVSHNOUi2O73Pk40OikaWtoYcI6QwkBGQYid7Catkv6Daxck9frIiwGWzrHznO5 +CyB0EhAHX7heE1g0EmCJIbnkKcsDGlsewe1GsIZYUANfkNVseMKYRrn08JQYkaLg +ZjJCXsSMj4nBjeZcDLlNjeCBe2dCReEJ00AjEzzg0OQCi0hkuqACHr3OzWIsj4oz +Jh1SNK93QOkn6kWHFKJgRcegpSzInukBjU/SiCZbrflZs1rzx83P9p0l+t7ANs+Y +VteQQKs/Zj0aWZhS0K/keL1c8RNDYnTsgoteNfZvOUVmqMhmIIvTdUWOKctyqINI +mODgqQykDERfsrXSqsFSdFaNrwrQ0EmG73CwTpBCThFkysoqcsOoee3htgZirgcf +LWavF8p5VtYTclTmKWc+MpnJPABa1bfoS3UAOc24DtpPNbaEaWSOU/MODbm81zBk +bqtgUCY3Qcf1hP6x3hvouNgCLhd0WLAbkwu/2PESVN+25Wbhm49xBaqZXPE7WYZp +RkfbO8UNucGi6SaWwNdNg+Xvgc+vFN6KORR+SPlScyGyLd2FdS+ay6twZj9rrsOX +11can/T/s10tSZbjIPAqdYIOSwghnadjImZRvZn7LwbIxLZe9RIMFn+SkxPJCbrx +DNtRuFM4TdTlOr0eeX144zKwGbVAOT0zcc1908zE0kOib3a+10XmMrGZ5uqDBr/j +sy5+nqBRYLHYB8XfJfedZTjz6KsY+vnaIrl+tl7mJfzfPyFkV14BtSL+JOd6Yd4+ +vYsoIJbk5A6JKynejumKoWVZJfsXV6Uk0bCTcAfMmN3owJh07o7XIHq0bdCr7oSJ +EvP1sImuBRJYWpFsO+iNO+DmcNLkHzD9RofGMs5Pym8etGYjbY5E5waKRRs+bHTa +JXRCOiqmxfCMCNQGaqBnwVmaFKOIcNbDnUFuz7qxgBvP+vmdWRA2VkmMpVxI+MGU +9m5F55QNAUCc3G3QJicbcSlMDvn2INWRWyRUiBNfHA79EUPe6UUMrdcGLTX0y27b ++KsumGFCLBqxDLtXoV8qzAKrUTBOK83K1ZWOc3UBBgWncRIoQiNCjYkneL0agh/B +FnkEIhmV/42CNT6pymzNwUBcENBdq6u/6KfkHo5RYxZSpYYZyl5W1dDatXoUFaGD +jQAnVIpkQZzfx6D6hTBp52brVQ+6CeXw/zmJzw1Rs3ux3fXDTTbwR+J5MegHOsP6 +pfzaZAwI0J8qlULvCsh0Fxw43y9OwOhenIbpYRtG1M064fXmoJGyQchpeNSMgHcr +neSwsgrjFCIsFo9Orh5hXDeHZq/qGZ0afEPqDyzQ9wRNBGC+OnhPbbfnz8FRjTIf +3LSejvxr7GBYvics9b15MRSZQJ8+Rsuu+IMQRoMODXksSYnuGXr9oLvL43gicrle +RvRf8jKxqHAB0sKesK9HVzD1778DZEB+cRnL+Hqs8wmWkS/7QcPjy14CFYD4gbxC +JBlKKCCIYUP7eoLMvv1Iw++A085rOsnrTA04khA+6QFPEhQGTU9fhm7GAgKX1onT +QTeicZa+mzzJyTqOzVIQCLFZWnCbCgTTeU0G2emovWMHGgpqp4ReHJsBakyInm9A +b3GgMYG5IZwW2mgL+Vo1ucuLXtNe8c9+MQyGimwGq4STN6AU/uEgCRKcxE3ai0Yy +qEHOit1tBcY6cHG+QTSwS6HGECLbZXBMddCUFy0vpIp+wyYxDkaFxhDeAL1sUl4N +bSMbyn+2BS9mzerVKn1NDwnjMK0/rLE+3thtHlZsLgxYGTXFBTJmFeE9kMfXU5QR +iPEqW4/UqrJFYUcsv96dsHKZn52BQba5jYEh/7w4IZX4+eZ4JnLsOE0Ykdsyaazb +vpGz4EAjTpqkBYUzdIFme2gTaozF0puQGOzZ2UCy2mO6Q0FKQHdK9FXnKozqNwX5 +VjiF3y+zQ//uaYFGoEa9uzzo28IJslxYlF/HjeMMpPQn54ltcSK2Lf96XYS2mbOg +Oas2kmw+f9crtk4vecXWsFGf0DqDniO0Tl/tCa35aGpHaJ3DNxEac2Q2X6GzgHbv +2JpXoBwC2io3MHFw+s0+qCGFInOPOy00MSlcuk9si/OEcv4af2Xch+Ct85MTK4wM +IGjE3s0u1L/g9xzcT1qhtGxJ32ANZq/1wPrMlsyvN8xfPhDmISElwfzitsIOhIYa +OE0hMSe25sU/WEeFLB4GXnhrUoKlLZ1/QL824ylRzSZXGcFmG28jvT3XOtxweFur +mxKr8Vrp+MNuda2Ewoojk9eKV1fS/GUL3Od0U8a6GTUC2OIIk5ToXJE74IXT0tCw +q9Ubwsq1aNCg52t9vWipJ8B4iuYcf98xEtNEu+vGR2K0FrEb9WLm2iendY48zeKK +b1MOGUezV//gXPouUWdsHo7FaTlVc8xPMmz3QyQA9/rgbKJPKsXKPprhw024vhM8 +vF33IXhONee0frrlfcK2Kte90TL7j8z0wtGDo976clg089A4OWscbsScOD31ltP+ +wdHai5OMfYb0dDNdF/X9cWRdho+DwwkRtuKbcyZd/D7UY+aL3z77k9NOJ5yz98Ho +xG7lgzPWOpwQ4fn3cLxzeAFSyQG2jkPk9BKezyPFV7SXN14Uoq3x9d8/ITRaApu8 +Lfznf8DhaRZQN+gL5AoEHCQH9MJgC07dOz64+vAXBukYdE4vBDfgaij4cbcr3JYC +VglR0LUTZFNB2d4BeEecepRPC73968DKQy44AEyUkM2VzxedxiZsWYeu4ZypCNbe +kFADpvKh9CYHNt2LM70q8wfEgRaYOuiNpK1dTxjHVkCDNMIWIYKmG1fHem44JsKN +SRQ+EFkZQtAADR3Y2IINEKHktRAjNHNRLRxWZbr5HafYizN8QH0fHM1XnR5EVENZ +ARzPfHMLLxavyrSB58lCdggqg4T8MD0EZPICmtDvhCXWK/+N4GlGhUU6F5GKIpA8 +0hRYKkK97eZ8vzjeIB4qcBb3cR8M/z2zYVZbwmWqNEMmelWkwk9aO+tYbprBnfsl +0RLiV5sGPTe3Z690FIjbcYCMzuaK4PaDHgMa/W4/zqFXT//k7Bwh/754PpLuxidj +uVFQq0tlLWVnY2WtrKaLgXd63vW3AIQX62/yyFuC3FaXrKs0NKsr+mazudE3NhBU +zDrvs10agzfapIRs0Gp20BFsJo4cYWpHr0vyeLKPUsAl1LMV0uw6FG1+vd0MGhpL +D4Hd627Ei5sW+IKsqTlpg2KuNl6vclWn3unJm26ow3AC/pYZcwaPDEXr+u6lQI9X +nRT0iUpLehjxt8Bs5zRqDGgIEb1mU8ReaTcNjVZ3CCUaL7h6o+6twC6hMX5t/HKs +cGz4OcWBo6CN42QA/92cp34DK/6dc6/AW+snx2V92pNzY087bEG3Oc3+vLjc3Hj2 +/LXg7tUbO5a08dbQO0BCBLwR0n71e/IgpBw7RgXRgg1QGIT5nU/q9dDQ0L5PCbWT +XvW9U2NyNPWFN+aNrvbXU1oI3u/PYktAMfzGaQQUggK0jFtxgjZihxlbdMQBaQdY +8MtMwNkJJxz4zWqUgxx7UwEcH4ANRW5Ehj85T+pL6ycn6hOcxfPpJ+fRcoip9lfO +83pp/eT4DNs/OF+g4OXsJImGFfNnJQTIMKwBCcLGgB9B93bPMyh0wS8k0EjQfKGb +Ji2d82eUhnD8BBhxckxOtA0F5XgKGgoYNo/ErAF2wSYTO0ac36/tfjNJbpM+Qa5J +E43ym1P3M9iOHS8hZ8060PCocdUxlNOIHbkeF3FalChiPwoKChx/F3UOWvU1fVHH +v/1JTm96c4K+hEs3jlqNTci/WBa6XsR/brrPw6D5bIIcJwlQ0aBFx8Wn4WjIX+uT +w00Z+ClpLOMR4Y0HhTQGb3DWftae08YEiMCGNZiBPamxWVh90s2rcrqTble7c/47 +A4Nr06tipUAdtc0vqoxkFc00Kmgfh8TsBNpR6op44Ukq1JrU/FyXY2TvTQLPvjhj +4P+L7wUayP8vNhswc3DG035p0SSeoEVK1DMbFcYyti+cHhcFhiQtBVAmFTrDOg02 +NbbK3AjrxRetVzmwN2wh1cY8xMmQmb2wZK+rFJS4sUV3ZT1hs2XiHnLsdhccMfVF +hULljdUFnLma1QsLnM1y44TNayZMnIDP1/3CmpidAYKzuBI5+Cxl7trY750TgUta +OuI+DC8IMzmn3HTler4kOrFH/GEn3SZfBHbUKBu+qTMlAkYnnV0aNdBpc2loIaLr +f7arJTl0XQXO7yqyglOWEEJaz6lXdQfJ5O5/8IBubDnJEIwsxKdpEpTizLLvGlKi +rIdOguoyLxGt/aR+2rmfTDrauUJNpePkKbGv4kRrNfRwx1VL0kAwt3IIzSrrxdXP +Fuse0ob9JCuzXVWtJIdlMYyDziDLxitMqjOloSLmBlp0oyz7JavViVvTJuGC28zE +CRHKonWH4dZYT9Mr7rNEDyUbFaleQ/dKZhrdTflCcy4Grq86sTs1+dB+M1ifDpB5 +R3Qx07dg0SZOjM52TKAFtkS7xrqFE7NxcxIkeE4iwJovmTvLoRH+s+C9KlcXiROY +ed4KpIvSQG1z6UtYCtgHOYsRynd0DFOjF3hpy0U2IzM4B20XVu6ansTKNl5z8Zyc +mKbTyTaj47Zfh8Y9TvybpOExdjTl2hKvhsKZ/l/cY2pp0fi2nOKqSTCfKR6aoAyn +hW5S1dhbXB6bf7DBE7KLbPS06BWNsVMGCfP4tboDGsc+SXqijjzZuD81N4O8T/2m +uTjr3ZeL2Jno7N6S8bcFXwqnugp9mYYT0vC+RaQDNqpT9Td6TlZxWUxvSnqaMOZZ +YJWDikeaOCG0fZx5HTly3nn++8+/eaQIQ2D/16Gp7po++7j+MU+bY4YbZ2guPcBR +OcEDChGb2eyWcULHeFlgkrh8MfcE18qj9MUbEKuuBc9yZD6ATF+ZB79Gxrq0oz81 +N6ub/cYdgIQu5aSwgcGtlO0K/pyPYBOQT8fD29tikbP3tRA68ufBQEwuRj2DeabD +ZH98T0+263QyUgR5Jfmdlz/h0Xy+NFtL00h3Y3iEyH5cPo5Cvib5bkO3bb+S2JOE +2GWh3C3lxW4zUhzX9Idjh2xcDmZW0vYxVJhQd0xCQIOB6glW22eJHWAVmlbbRVos +lnTLmReyKSB0ExAXN9zICSwGCbBmkzzy1joBjQ+P5HYrWUMOqIU/aBUbVhjXTA49 +rBIrQ5TcTFfKRcy4TCxOtOBiiO01CB54dyNUdN6wHTQawQMHhj5gkYG8HqjACZP9 +slh1QnDH5oErizcqoNuNelkhnSgoqBiUlDtpjSegiU5aWWRVmj81VZr/Hud83nmg +zwL2fka3hoYEesYyG95oYUpHvZLjWX/8J4Zk6/gDi14N1m+/RUao68tAi9PZRIwp +ax2QRSS8cCBCmUiZiF6yl1LloBTGrHGrAA3dZPgBB3WDdnKKJFOeVtUDo/Yzh0c1 +xK6Fjxbb5EG5iEqtkEsYp9a4ZDKSbQG0JKbot+wAcoZzHZTfnDklXKN73ZpPaMjl +I4cpc1olg3J5KCrOLtSP195CxeUUCLmjwpLduNz5R8MmOGPa9sMiJh/9SlRzWfCd +LMM1y1D2QXFTHrAY8yX2xNeXBsM/HN+/KaIUWyrikv6r5kFkH7qFdd80z6nOnv2p +eS6vU79potPfmkhOyI1r2I7CnUI0UbfrfPXI7cMbl4HNqAXL6ZmJa+5bZiaWviz6 +Zud7XWQuk5tpjj6c4Hd81sXPEzIKLAb7oPlZcp9ZhjOXvoqhr68tkutr62Vewv/9 +L4zsyi2gRsRXaq6D8/bpXUQDsRQnZ0hsSXF3oCtAy7JK9h+OSkmhYSZhD5iB3ejA +QDp/jtcgerRtyKv2hIkS8/Gwya4FFhhakWx7yRt7wK0h0uQfgH6j48Qy4iftNxda +s5E+R6JzAsWgjTdsdNolfIR0VEwL8IwI1ARqkGfRWboUUEQ66+HOILdn3FjQjWf8 +/M0sCBurLMZSDiT8YEo7W9E15UMQEBd3G/TJxUZeCpfDvj1MdeQUiSPkiYeGoD8C +5F1e5NB6bchSoF9+28ZfdcENE3LRiGX4vYr98sAsshoF47LSrRxd+XCOLtCg0DQi +gSI0IjwxcQW3V0PwI9gij0Eko/K/UbDGK1WZrTkYiAsGumt09UN+Su7RGE/MYqo8 +YYayl1U1tHaNHkVF6GAj4BEqJbIg3t/H4PELYdLOydarHnSTyuH/c5KfG6Jm92C7 +64eTbOCP5PNiOB/sDOOX9mtTMWDA91SpFHtXUKa74KD5PDRBo3tpGtDDNpyonXXi +1ZtAI+WDUNNwqRkJ71Y+kmBlFcYpZFgsHp0cPcK4boJmr+oZnSd4h9QfWKAngiYD +MB8d3Ke2+/P10qhGmQ9OWk9H/jVmMDzfE5763LwYikygo4/Rsyv+IKTRkOOEPJ6k +RfcMHT/o/uTxuiJyuQ4n+h85XCwpngBrYU/Yx3NWgPr330EyYL84jGV8PN45gmXk +y3/IePFlh0EFIH4gR4gkQ4kDCGL40D6eILNvv6Xhb9Bp1zWd1HWmBhpJCp/ywEuS +FIbMlx6ObsYCBpfWitMhN7Jxlr67PKnJOo7JUhQIsVladJsHSKZzmwyx86F2xg4y +Dqi9LfQibAapMSF7vgm9xYLGBOaEcFnooy3kaxVy1yt6ob3in/1iGAwV2QxeCZE3 +qBT+4SQJFkTiJu2QkQyeoGbF7LYiYx28OO8gG9h1oGAIke0yCFMdMu1F6xVSRb/h +kxiBUXFiCHeAXj4pt4a2kQ3lP9vCK2Zh9WqVvqYvCyOY1h/WWN/u2G2+vNgcGPAy +aooDZMwqwhuQx8dTlBGIcZStR2pV2aKwI5YfZyesHObvzgCQbU5jcMivQxNWyZ9v +jWciYcdl0oiclilj3PaNnIUGJ2KlSVlQOEMXZLaHNuGJsVh6ExaDPTsbRFZ7oDsO +SBnoTou+al2FU/2WYN+Kp/D7ZfY6f/e04ESwRr27POTbwwmxnrBov147jiuQ0p+a +J7alidi2/Ot1kdpmzkImVm0k2Rx/1xFbl5ccsTVM1Ce0ruDLEVqXr/aE1hya2iu0 +ruGdCI05M5tH6Cyo3Rlb8wqUl4G2yg1cHES/2QdPSLHInOMuC11MCZvuE9vSPKGc +f8avinsRvM/81MQIowIMGrF3t4v1L7x7Ds4nrVBatqRPsAa313pofWZL5sdJ85cD +wnxZSFkwv9itMANxQg2aprCYE1Pz4h+so0IWFwMvvDVpwdKWzj+gX5txlahmk6uc +YLON00lvz7Vez3B6W6ObFqtxW+n4w261rcSBFUsmtxWvrpT5yxa8z+WmjHUznghi +iyVM0qJzRO6gFy5LQ8OuVncIK9eiQUOex/g6ZKkroHiK5g1/nwGJw5McU0+mc88Z +kKjeWrEVUfOZmlQQWdULKwIecfNdoHmQpMRYBYbP7P1893zF2yV2CWyT3dlN/r4V +NdyxkIbGR3c4qu66PTdGFeROtEFSl+vHI8djjcObFpuLmgh22lu2HCd///n4vwAD +ABb+jmYNCmVuZHN0cmVhbQ1lbmRvYmoNMTMgMCBvYmo8PC9MZW5ndGggODc0My9G +aWx0ZXIvRmxhdGVEZWNvZGU+PnN0cmVhbQ0KSIlsV7uOXbsN7ecrzg9kX4mUSKl2 +ESBAAgQuUgSpnBcuxhdIXOT3s8jF/RjbGAxmFjcpiW9Sdjtc18tHP9aQlyw/pCnw +PNQ2sIDeL/zlTXwdq9nN4fOYAeXQMRO2tV849BhDQ8D20b0I8/UOwjpcJEWa9ZfY +PFYXnqgO3HGUXhhHTD90jptjjqP1nifIxqVTj7EG7wAfJIYfHnoU5T0pfW8+tOPd +Q4/l+9ZDN/AHTRXv0vngkGMuv1UF7t2eqorxjltXmYePfesqkna+NOnrUP+gaw/d +Hrr2Bt3ooH4h9WPguGAvgh69BXdJy8Zb7+MJ44U4BnzndzxvLTwHqniDilAA/gI2 +3EYj4OHhrKBID5VA0a0ps0IGZnLLEw1evOxYOHyhx/Z5c8ATKrx06X7dvsIVapD4 +51t/xc+3L7+9hQqjhYb9EFjuK3WwcVFExmEWHND9A4aREB2hdVFGO3Zy1Al4GX5f +1x2F6YU9+4MDvh52n3Di+46LUq84Tzhf+Z0eoWV74ecYpedGXIZ8RFsPPUHpkReg +iPOVO3wbWHbizKPCX94udz84et4JvCN6/NiyE4+e8QBKuAYEZEkgs57QbT0w4rpp +8Z8UVUkO7/TdxOsDL0aLTSkBXmmHb5qh+aKzC4vzgF0RDUNtT9O2MpyLEm+mzJ4z +cRc+CoZalOjIS8CRJQhwjcTLKCBS/CRATc00BcXyTtjBecTsZSjCp2++vP07RcYe +mXnLtOLS5SK83wQoAGs/CDh9eobhGBlT3SP5GpJRE8+dpgBFJTg6tVLJQiBt0nD9 +guXK3h8MULezcnQLO8ID4ombUwLZrpUWES1RDPie3TWxGfEqZxYFCvSRKobM1A8U +xMqaSZGtee0U4tFH4qwPYYRe7xYhwbKEOQQKzwqxOvCMKYT59AcHDLyIp1uaqkVJ +iivFyuNqfKa40L8tOR7/L9x/xtNJmZbNAzcsYxObwlSULDGIVassoIAuL6VOQlup +tC22o+40iiyG5PZVpl0VtL5WGn9WTE+hc9SJpZ3O6PmmMOhMjlapOYY/8IQ7rSRO +ipBBqm66Elsm70C0awVI28aQ3tQiW7ke5hVhMRwUptozmuqDYzFHbLMeNeEBPrVc +iXt5QzurZN3oI7E1nrDtrHHZwaLOemc5MOLuzJO2WIelHgWnN1K0klvz2QN1wl93 +JqNL9FPimdvvHzsSevmc7PJQ7msSNKtoEt6TIDUEpRbA3YoB9gzcajrZPZOg5q+k +hG+AzQrrSDwqAANTQnQ8OJC1a9UdO7ErT1iatgZl1DPdKIGGz8APOyA59uoMcva0 +i3KVtAjPOX9GifudlJL6kRJ/NYcIfGtVa5o83hK1MKpdvLXxVFnn6zl94YsI9SPD +VKP6TRKblH2asZpG9U17kX+VOau0MazC4FUKcybpVy0OHdJjfmv5JX3cx/zAIeKF +e+LsGYHNS2K04jByjNJoT0ZFxlXOGKfEI9IiDMXiIQzN6IqIPusVXKSIwSJMmKg8 +FoZelQ6p6UTVdKa5xeyMWK9mKDESXjh6vpQAKTstD+/NyMqfUm6fn1I/oShm1JTC +0y1GyZ9QLinT7D4/ody3n1I/oWCsQO5/pBjilZh6jvB4Yg4mwh0ClHMuzUoTWDkk +bNSkwJytfHrxryqvhjsDszJhiIjpy859AbjxBsSMUyKGKZMaXGaWrsC+5MIU2K0/ +OGC4SQlNJLOOG9lVQIlFJClRFIDn4oA4RsJz3htLSoC2JeVpbQSUWFGs+VUDAuus +KtE7H8WVxN3q1atVhI7S8xHD38d0jMaxeNqwq5N8TUrPObB6y4LDCPGbsFvV7JGB +vpD5rVfvCA5MU511vsXWCNw6E0EuCVIQ+6xToPTdv6NI9kRExeSpo9PE4e3AczAu +lkudak4nuZNjGeMg6PnyptfwGRIdY0ONnysZosZdXgT2XX7OWgTTtLpTorIAq1Ag +94AwpqwLU2I/FoWlOVVccbUibojV0oeLVudE7Mnh2+5gf+DVTomTsktAOa7ETJEH +Mrn66sU/Y9iNBDS+QRsTMtp14NZqvpHSITpbEkprq1V1xiAZVjB6ajKjI4Q6j4hR +Lwy5lldNoKHPGlGug2sGj1ybznbhatlP5w9ObTEw0Nk9q6vn4zKkqk2Lrw94aT+D +Ts+hmhxc2lDMYriMgJssts5pNgJ5s3G0tpMjBpdslhRYm81tcLgMNZzdzTb1ntWr +MoOB97ndcvNI23GZ6NE9T1sHRl1I58wb031rPjliiqwTlNj5uSkFZurLlhvuQ/hX +C1mxfgD7Yjs0PyX8HKss9zZQrPfvKMyVGKR4ysig2Pm/1N4QmyRPzDyD6WGxeONW +mp5pgDcT6qDlB0ajUY1mUcnNwWZkUGMIrT1jsoXDLFtJiUE206DVdlOZN2qlskpN +hV3swQHT1/A0nWFttdz4mQdjkEGNAlpXSobgjSszT0KNioFrO/HYjuLAzdHR3EvC +amCdyyocaOWxmavkH+3UQRfHVTXmLsf4lUNRBsuYlRW9LNvqThFaUoUndKOl56Sn +6wq4btYKJZOOVqamLWIa2q8ShkI0ydEy0aISVTWIGfSBo0KFxEnBWakGcHaQaJ8j +71jVfvu4QrRxt1iDQT3StKMMB86cOmIX2aU4Fz+EzqTiU6trLWVIsf3GVVVX267G +t2j751b2XeeMbvofrDQtWgkm9dyj+m5Ztn+XG8p+/fcfb395/fb2y6dv9vr0+QXe +veCv7/9+/vSnt/b6w6u9fn1F6Xj97zVef3z99W/t9fe3X37/WV7/+vYmG0EStaPu ++npTsA7i5e9vn9/+jHNanJsr1SmKtfOYneaZyOuvSenK9U5hHu2hHZu4oFoEbsr5 +ts0odig6CIQyIM5QaL60BgNwBpayT2BINBhhrZujoXa3ugMZqA1DZHWyGA9CIrpW +tXR80RaVia1mo/XceFXluCmo4q9LPpYTBOt1Q2whrSd/vSGWhB4SK+f5wEtmapEj +QmHq3bJEnxw4S3riGFDTUs4Tc0MKwsydS3LWC7wjNmOTCaVg+uG8MuvGTYiO5fBj +UKRWvZvSna0lGkfgVq0milFcspTjvGbfT4+q1SaXDJFbWfbL4bUYRgqSvxVDrGih +p58rw04401Awh58BIVKdyTxNtwtLBUjkwolDIh47bo62USgnT4g3As90X0+1w107 +99R8hCRDo1li4A3nRWFNFZtUyM1BitC72W4RHdtn4lwmowf0XQJtcwrAvJwcrerj +Gv4Bq49TIilRAJVnrip33fioitmudipRU2GsSIH3qoWlM+aW6YVpqD36gyMMcmJP +3GtyjS5czqgKWkGqpwB9OdoqeGVzq4lrRIDhUTFH/IChjVlFIGvIjxRWlXdURO3R +PlioG7Ikqgg8DrdFC0U5BMv/+a5yJEt2HOb/U9QFZkK7xPOUM0b1/d0hllxeV8Q3 +wZSUEhcQbNPib2eMNsjotqREzeLsbXDWoExO8gFuS2yvGDRw8BK55/t7a4yWRqV8 +b8vc3f3G2FFudueKGu7+PqEiRq8/1Hzlc4GKDjyfC95YT9B6WZBx7evaHy4Fn54/ +bSp5/x5aMiXUfT1m9n4eYIwH4InttQIu8AGXi/wHLocLLbmyZB4X5/3m+Po7CN// +/I9e1dS7OSn8+bAs7up37sfe9Hssq/A+fM2wfumKi3WkVruy5vTaueWWAk5rzdNR +cCgEbmEc145qSdZm0E0YDonBD3fchbEjactK8lox7CikFHDzHVrZ3gFhzb/OytCE +GaKglhKfbW18xpVb05XRtWItrZgoxsQzVEtNLQKWJu9HbK4YZIT085ofGJHQjmFW +wZCrE8Q607eaHrkaRRZvMVVURUfuLgo4s+kZHkV3uX4RU5wxp1xTzrYqs6uuqtzd +rhpVxNVZt9UPzYFgKTw72o21I+Z4rcDkGK8TMuTn8x+ZFPO8bpFJ4zPn0AnrItdz +V4pvdabODBdvQedj0i67ymxSWJ7L8bqyPn3NDPisCyi/3tHtFSH0hT8vy0rqzWzu +jf2BF01UqYEoYKk3YeEgBzWULugd8TJDoeG2i6GEcc9sj8yzZ0Uzy0WfxFQq4F7n +8lEPh2Wjwo6nuGQ9VsPhi6gCpafSUnfc/J0s39JNrIdfltGpDX/+uc/5bQEhpRdk +mWs8CqZhthSD9abbV6ug4brKUexoxWLoQ9MNKBCcnR5rs91YPi21vlYgIaVyUPWI +SSlixSO+Ssto5gf8447iYWf9O87kyfzvpFZOW/rhz4cl4OfEY/iUrtiqdDIelZFJ +S/HcFaG3bfM12Ae497ixvFHnfq3InVtMVlk6iPU2T03HEoqE/zjy+BBZR3kHX4Ll ++xW0TNkWDtooqraxlxPoEhJBvPalO3zEqd6gOw4t71GVoWfc2Dm958eKS/mQhNJP +9biYOXXRt5CZKN+vx/OpvvLkv2OjWk3N35TBuypem6upXCDJE8+prB+oi8TDZaHS +w3FanzMFv9e4alfrUXFSE1y/OCywaNDKcL2h4g4QfOKxJVhCQhS0UTw+7c4VZbnY +sSFpep4LYn028NOe7/k2z1bsUripLzDIo/iT33QgUzHmTju+9A98whtkWKTizh+p +n+ylPxR7umiCgDe6NCVVE3WhamfGID5b6mGv40e0IR2Fy3WSuFQKxj1I6FCR1H55 +aYUKqZVDxzcm2FE7R6i7ZZgGvrelU7a+LAvNF9cpas8xhIdLr4iIEG5meZIESSFd +cImGECYRPbID0av9tSLD1z1mjEM8qnC181I4h8oZ4yQDelytEJQIpB562ra3e9m3 +5eexgFHjsmhoy5pB60MEumTlbIPXaPq8SvctdpMORSAYAc9f2y9fPT4oN70347Xi +qKA5duKX8KL+USV+MHzxnfsEv59pz7QXTOKZw8tlufvPq55/W1DhP2TpywbXu+iP +RReaobYN5SsymclkVV/q4GMjxC1xrvTbRzUQS0Uq9kpcVdZ96/sZVyLUkCpgY1hs +kKKrzQD467RzsvyWTqCuB65mu10/cFnj2mG5s5dXhEl8ttcv79kPZ9kz7Lz413pk +ml5Zv16yDe+b+7UiC6J7PirCvWlHPxdhzjLdOLjg6hPzuE6f4JClR+Xk9G+T46yW +Ip2iImM6M8v2begj2OlIq+j3AwrCuaOZY6AXiidBn8CzhhUQYTmCs1dvkCVDdNgT +09IstR+Lo04lNzCvSMRG1y12c4Q6a3cW8166A210YoLQCiQi3unPgzGe8E28v8/u +5WiSibfDMY7Xh70NlpyNEkEpUYlbU0qswnClpW9bMqGBZ9lOqk28qgJ81H/SsqtS +t0B+AC9lNtrLC3bV8csyty6xjjIZyoe/dGaf069LUQugHkPP6OoftW4+O4oKGppV +715FK9rUihbVaoBujqkCHw7D0v6xFIXu3dDZwKUKL2dPUCfDcqBWxuE0xvYjKDrI +zlKOs6cd9QEIBODaJfMmhMSFwY6S7C9LVK0YKMYsD7oNyUXZnAWiCWmAIDstc+va +uwQxNMidHcZybIP4vFcgNIN4o9XPnmQjvEQAadnH/1iIdrZ6LYAWA1wYJhM3Z+xl +yUxnK0qDtfuNq3sPB1j8oUuqb76ia7iBH9aVQIfNrKYQ7rx15xhS1VPznWf0Gzsd +MG2+VmC0IO6qo/sEeT8t1d6u2gAuImyHrj1zEUO9K4W2nokWwYwqwie0YRb3z728 +YRSNQctJ1qfePWt84C6R9LJUcClO6MqpyH/zH6ERxk0HlxoaBXYVP2jYCMld8AfR +kAhHhvgEjkkTuSU905dKVYUINXPxhfggLV38cMLjynQGFSmm4oRIin7JFWbQlL6Z +GF4TT49MmpAmtEqo1opyLI4nMRTbjRcioA0yoIU6J0VhldXd6S5QO+y6Ugt1h9qc +UOdpL3+3G3aphfyVkE8eRAfCC29hnro3C7nrTCjamdwyNBlszXFzUfio++Id6L7i +zRkPhLBdXm+/dK0GIUs48njxdkY65OeU0uGJN3ifM1xWTB7MVw/+5otijGfFyqIJ +VeqAZFnJuqFfhER4WnZReowU2cAxJaDhthsmbUqq3pZFaYLqT1aztHoss8qCbCbe +biDMsEPFRD/ZkRh41SWDxZmt3bPpxoyVT61VCbCGeE9qXPPt4IrhEwYa9USj9rgq +5Q93LQsbuB/hr9fUtIhH1BvLnfcZXvF5QlKJo69GmoZRL31m/7dw8xY+/j6je0cc +XWItR0jdnY00YbN8O33Y/8MCJJoCNu2pwASUeLVjUdm9g1xqy8/LMijxZFl7ydIU +d3gVuDkP+lHqN/umeVrL4pi6R9EBBfMhXyrxVfrlmxmqtmAuJmHvKoyGDmd26z8R +E9zN+kPNbrp/UaaoPD/L9xtzwpX/IJ708x9ZuvKZCjrxMYYy4ZlDI0yt/umYKkrm +auK6zafUW6Huaay86ud8rCin+gRl//GJJa5sX55azxyqj9p0SRf6IK3oGd+/HvZD ++sJujZErXf2HlsUMluVHlupyJiGc/1rTgEYTdQ9oSzItLa34zND6YsbaUIYrCXyd +G2MHBNXnimEHL+iPxBLQuAlfvzBwLYfkC3DrZWMvQon4vJs62m25R8Q1nlHg08JJ +VS/3rt8WzKyZqrJcnQ9+u68CAt6Eyr8kDpNfqlcKtlST0KSJWxWuaK6Js/NIGZlf +0x0hadSbdviAUbr9aYJe2xEo87p3ZQSaxdZkJaaf/Mu5rphJCT0r5rxW6IRlFQq7 +duzLN1M7dnTrt/N1ZxaxN7xTjem3lzqHHZzptwclxissG2zWPgKVwua6vsKykwFX ++7BcvTxnsPTmD0/mLPBa8/l33SgogaHoRibpf/Ln0P4Vo93B2IkVFMe+D+68MbS+ +ryzD60/e89tyb8p/n5Q87ZYOefDpVJEXWSUcIe4KqNHExfRZJsN4koJFpwffk16r +fNmRiyf59FjxkBHSMKc2BKKeuBf1w84fJJ26OW11zAMqdNdFZSY+Q24G9z9YA5d2 +XBbeCSQkukgFyT8MsdaR/MAdVD+Nb64cnVheQ1j773rCK0GuqJ/QqyXys1rCXrJ+ +gXe/6VZITEnuzRURVjwt+T3jMPYNvxmY0+trQUrIEF7n/2xXSZJkOxHc1ynqAoDm +4QSsMRYcoA2DRfU3A+5vhg+hl8rq3qVHSk+hUAzuWbj3ePDtIzBJc6QWZnFdSMPp +I0kpiSMffIAhg6KmAmwadxnKnsGOl/bn7f43ir+fYsM0XaYHXrCyXVwxspQKoApu +kHv2WN5bCNM+tKDOmGllxJUNV4kNKRob2xRjZPYFzEHLZB7jwY5iauNaUXFpd84U +6V2773jijqezoZIK42W36gOpNPMbPsl2LIgnmSg/oDE6/pwNaygCUm6fMJzgbQw5 +nfZ8kTXgcZM5JkK//0cQVg0y14TLjP9Nv2Fp0+SrdafOCKrVSZKAZ+wYcQZTx/Rt +LD918hFTFYerFG9Y4yRPKd6wuxbU5BJNXl+DN+c71Qi/XrBJU9pwWJuvOIKZNHE1 +nVZPkznnp2grOXddaZegccOpZ4kCnPdJthE0bngH+bVwBKXkIG2muAx0CxbX9C4r +51djfGuc1GX/+SgZ4xC2zjrYn3knBfhPrAAwPnTyf3z+8QF6gB5w67dNofTSbwuE +bF36bVGu+nKh32DJ49JvC9nZIueUtQc/Cu6xWMLxA+0l4QBnf5Nw8KHOl4SDh6Vc +Eg5XqKXfEm6DaZdLwm3W0SXhgPv0iSHhdos3DAkHXKv7MgfDAx8J91gewQYv16zf +LfuScMC7XxIOFxETekm4TaJ8STjgmi8Jx8fZbxKOzxr1JQkHvOYl4RCcnN4kHMPV +LgG2qSnaq6aB16vEHc2crqJndNu1HzIn/rWAg2HuaCo1oh8uSMABc1BdAg6WOi4B +x/cpn4+AAxxuQ6HfYFjp0m/bhODRb3zN9KbfYCmlfF767bE8+o27uhmx9Bu/mi79 +BjyHUz/0Gx0rl34DbsFpqN900Tf9htDseek3hqp/PvKNocxv8m1Tu5bPR77hcfZ+ +ybf36pV8O7n/km+05Eu+8ZvZKyTfgFMx5bB8YwbUS74xQzzZpM2YUuNNvSHp5rjU +G9N0fD7ijQVbjUO8MfOPCxRvqpRLvBGPN/H27Vriixs0P9pm9C6U5xG7jA5kU0Q3 +8d6guyWodXIRk1W06NxYgPSJ965FcIzIkGm3j4Udfylnglbchl1i+pD4bE4hl8eu +xnm4+lZUMJxYrqdJ5gHc+ohi6MIjJmQ3bYVlDVva3Lqnp3y3IAI+XSPqZQbP5Xge +WrCy6600OoVA7vrgHwptTOmzYFSfUDiTN9/NB0xHEk97EwngPNcz1m88I7Nflt22 +V5zAqdrwxcC55fDJcxUBy3Yi+m1hdwVsMXeooLyhLJNzfC6uKdhacRS6P9DWiDiN +7DOlG4FbUIm+vKOET6PFS8zsEya59h7KeM6JVJ1yaTbhGq+Nd8+2jN6dZMs8NXGy +X7jPfJIuLGtowexRrdUb1opi3TmcSsW8cjQ7VbKrc83pBJtWpmnuc42YsCc/UmjX +Vn3vVvejbR2pVcz4x3CClFAAUgh6m3prCL7vvjQE8MihIaYzZB+Zol7WUlIO0gmU +JeFRqxw0xHuYBpS+YoMsHNp4ti9ZVmrfLHKMFmQJ8SjGKHThNkMolMeNYraywU+I +8/Q36TBx6IAUacfmleOFlm86rB1K8s1bEJxi2QpLOQ0zRe77k7VEteUHOrF7bm8L +ynAOlGmcokPneGHmun1IyoGJLLK0oJy88TwPfAyjuqes0CI9O0V2CoUW3AWHJvfs +2oadKl7h8YmdGvLrEGcliVdk5xDZmEphO245NqS+Ik7McVnydGR1JJwZjuwyS1ua +M3ybouLAbVDyem2pvEmCofRQg5hylRvy04V25WNnsX51AHSpG6cxYoctrEuvyHNE +JzROQZey2I5ONSs4+TSW5cHALFEGhgBp4r/KuGZi4u6La+agHc4WVedLX6gb15h8 +fgrHKebi29ykXPjbB5IYcgpkhNncORUThwnC+L8ff3z85a9/z5//+t8HKBJrsOPR +UfU/H4wWlcmhDsbVcIXNG3QiCNNPUez82eGT4iBWt4hH6p8kiVv/ooeLYe4AWJnl +fvzHeNZnG8v3+ibeG1l6TkyPJ+n4yDXH/y9+uYCw/ILNQ29c0BTEnIe+OQl6FZgj +nJxFuHfdIEFHADXs4H1GOQhri5jI69+NeJ+dII1L/zGdFKaKPOKhcIkMEy8rhy4w +RAwOfm7z9lqkTBoWCh4L5OeDoS6xEDswKvr0pXAYkh7XBhpoomQdAuJ84hxa2VCJ +ZBzVccfQIt1AFwmktRwW598l+sWdaCBAueo/ykys5SDZDob+rUWBQ3t7gZWnlwbe +Vf+SKfMq6EdE8Bkop+61ma/TRTrYvJqjiMBsj2IgipAfDMKGalTCVF67aSMpF9kb +n2qoS/z4CLYHXDGbOZr972RMAoGEFi8N2MGouBHfA1roBaQN6IB4qNybPeBw4sMV +upezXpHsnc6yaowUW5ZY/IlbVyVAzwoe1QS/0rZjsJK+WjEU2COWEEcGO2vXZ7Mz +NDCdp5YVG5P3BYXGxg0zUEMHI/EaAjueJW/92acyY2PIdTMrTpXaD5L3ba/rX/JW +7kQu8N4RsNwj41gI00UznUUkVutCJYISMGcvHXIhZS3l+MYj7RIuFLxdH9I8dIi5 +MKjLGOvF32PEylmMfaR/FyX82kKmLsxMKAbgpnyfWYCJzSGHNsQMEjffpnHA7Bck +Alupt6oQS4SJ6GttvStwY2JuVydSeuwHNSWU1xo3LSUBBkhDb526muY6tTHxA7g3 +51Cpp9lO8aanEfP51/OffrBlDd+wijUrfTnB2KKz/s1NmyYL860DcfaI+Gl1g48/ +P3IquFF7ta2c8OE9bwOkUD07iJsU7WWooju8fbeB3fkd13J94t2Jr49/yw3SY7YF +BP2n3WjtGL60YjXla8HbfX3fwoaLykU7H9ftjoHPRkaac3J6/sYQ18WW7oCwyG1Y +LIVfDOgxQxjHlf47fM54NnwzPCF8d1wRebmOr/oqJaqu+5isEnsde29wODhZdMxA +4vAbrBn1QV+keYR9x8/TZ2Qkx+7L0KkS8EFW3Nf3E3woaiGtcxXKiImWx4GKFf/9 +J5eUgv+UDxPM++fL8GQMDB7D7ItEyyOOQycXsBfPzSIyTEOpoj4c5oQtacRnRIqw +j3qgl3OmPf83xxGfK4Z5iFpUNSIaNC8wJMGLBJd8mynfcEtzPoauxiyoHMX81reL +krqXGq7s5qbTp1wdHqtUiIS1zHhRr64xALfCz/X1MehiS+2tjeKLDTVbMgq7VpqW +zxauVTVSsu8HTk3tuIkNFdWq3d0jwldJS3/O1M9VSLOQTE2uMDzdwkVPUtaB5yrt +/r9kb95+wfj4Ei1jBuzRnKnTCaGjh0H36Borx2IZlrqOs2kOUzMtJ0kl6CuO8vTf +40kmlzumuxx9zXwlkzmsGYGuvdfrf6asu2c1HEUw7xIxXZ7Bu/Oi3fyaddZvWJrz +4zGk7GTbS9NtpSq46PuIAU/DnJp3z+nVw3D6yZrfO/ccy8fwOGz++qpKxrW8mzqP +uTnncWapr7Vp37OZW6ntBZu4lpeHoSen1/ZYzNWHzaQZsOKqpqica93JmcqRKAp7 +K89o1KvMcv3LmUBUnXtkpYRPQiwLF2gtZYCqjKKmfn7rRj/UoCDh3qbkMTyNsEKQ +lP4bAzK0ujbJJ/JvDEONxVtYp78aUO/lbcsvhmdwvHuqBlyLtdIkq/p548XAEjJ0 +ZOVNcDTdLIn40TCzbkbJQbj+z3aVJIkRg7Cv5AnesOE9ueb/5yAJd/dUzVEYY7ND +07ScyQQYLS4Eexq7f85zoWrzvT01/73S01WaX/X61B5x//bA83AL71XHumz2EZ5T +pF92Pn44sj1fy/p1zvt1QWlK1d5zKn5vl2Fe6Y/h9PpjVv3ta/W/aOCg+PkE0SVk ++XXFDPRZvxAe9/6UIfdqCfyILcIrJfP02K8E9E0RuHlm5KU9AddgZFo2I0C7oziL +VhJOmgQCaagsEBq4Hek2UT7mhWDPArP793zcouWE7M4Y+nex22CbQkEA3KauNQRr +zbN+2ZeqVHBZAmFOPoc2zucGayCKFT+jGpi5UvebdoC+pWzUeUg1/aVHab7VMet0 +hXi7EU5jp4+9i3to7/Uus/LplNF0m17IW2sVOwmvt3+4Vt7GuKfiNDjCXcKiy/6R +gI0T4wseQSfhMNOFnIN4lx+dew82E/SqhDABYD+Es8eFYl9+vue7zhkl2LQ0ZT3S +ncL7MR67aarKWPjAefrlFoFTFYTrNtY//mXw1Ncu9jE44nmIvS/T1EXUagYr4Yeh +jxlsOM1gh3AcWWVtVtU1drFPdTIkOE0cqrr2RVHZUISMMwxpuOzquZgm8Zbbpyfz +L13JI7M1U3KYFJkK75p0kmBq4ff8KBt8yi5RuRKKoVwvQ8mPv+ZGu5mKaHCAS/Ni +yS6cqihzQvGaGUEfpayl8F6yqqtq9PD62mn82jC51KTJKMVWGxeWy7Z9z7vVbSuf +6bRdD6s7dpOP4lCT9kVlUILMMgxokBvMQqwqDCSTc2+c7cOcnBU4OwS9zNuFxjWn +Rpomg8zGuEGBo/kU8dFOcdsggWUxkbYUC/9CvFTcIkj0Yqb7Fm9l+Q67/5hbE4sy +9Szti6XjjH6hlOy+3vO0kGsFK1t60/hDU//5L8AApYeXyQ0KZW5kc3RyZWFtDWVu +ZG9iag0xNCAwIG9iajw8L0xlbmd0aCA4Mjc5L0ZpbHRlci9GbGF0ZURlY29kZT4+ +c3RyZWFtDQpIiWxXTdIlqwqc31WcFVSoCOp6OuLFHXTvf/ogE7VO329WiZbyl4BV +xmPVPiqPqnyqwy4S0FoHbAo4Wv38+ud//9TeHukVO4p+/kDQZOGEPj4BayMsHbDU +FrCP4SfUXp+5FIK+PgGHTUAxQOP9Abm9mbzXSwHU0K+XZypO59nlGQW6zTGwas1t +609d6w3N9nYI7GlVP78hKONHPBawK+N+0OGmti1Y9oPAnjE7BO6u9hPeV5wf/hL0 +Z3hcvgTd7wgYDuruFvpD1oCJc7u3W4egDKzrClSMaIy+YQbjCgLysDIrQ2fKoyZ2 +e2h5WhWGtmkFHPY5ieGauYW//s6U30gefRrDN90df66gP7PRfH16gw4K+/QxHipj +As6utL7jDntKp0rIJoNbwoIlgD0Ncsjtw534Wl9CC9XR8GiP7drY7dlfJy6HPxxO +3KUy3nBI39spmMu4HqrrU3J7DdUVPuT2AtxmgyqztYBdJzU3/GyTvtdnRWq7vdrg +CO0zoAhhj8PMVWu5vRnzTirW68Lfy+aFoeHengJrC+sSQRjP6hmEj07fIrl5DglB +75U+dy46VFNAU92QPp/d7rqbLdy+vdhw1mzpFVjigkYn6sJd0hmCMbG9Nc3tEPhf +SR//IfI//mv4QaJ8DBQwBMGp7HCWmsqtqEYuoNdnhRtWo2WWbmh7N5S767Dcf07L +ywQcqjtmHZdb75/jN1etyBtKW2+/uoMyI2pY4iEEvSJ1UYz61mUshFx2/ggQ907F +r9J3JosweeAEY2KOudKnM6DaDgFY4UmAw6aX3IU81hf0tG/cfgTSBcfNQYq1lQFE +fR8nwBJO8wqOIuIBGKzv9Kh2S0Qjo/Te1SLsFT2LQctesBPTCtZ1rSQFm0MlfNUe +9DL19iH91ctcMKSdXqbRIOT0Moda9NXLXCBznl7mEKTLXuWwRvrcXrY8M8btZcur +k91ettyBenvZcr7p7WUO67q97MDdy1Jw+0jwTX4WZDdzdaq+m9dCRvxXcLpXOKzK +j4J9y/nlL8HtZ0fAfrYYxd3QHK53P1vP0nn6md8HjmXDgsfnq6FFRGy817MboqNF +APdhM7ebttPRIvxsL+hoOz1OR/vOF3Q09YFlosl1j8ufK7gu8jYo80dB8WygQFvd +dA1oEyNBlPiAU7Bq5KcL1hgsDGGjuPIoBBM2yCPFNoztrlr7WtdOWCagGbVTy+2D +IS/cvdBGMA6GHxU3az2bC0rJYvsG93B6qTy98/8ylbpxtbaZ/7eF26txe+UwFIR1 +VJhKdW27az2rDRoHiuYdTos0ck3YvF3QUbVmpU9xc1QI/owQeInrPXdLlpYFcnzH +laEeD9g6hYH2QjY5G38C5BQdrT/6bM7AjWnpDlysY9HdNbt7REYAG//uLGShZm7H +eswIqBs6BVFYLIqroM9EFRicvNrAulZwwKJgbxgtvuX2FFgeZ8aQRSOJ0zk5CCeP +uF0RBlsdylhnfqFERiz7hjRV13ivr8EMWhV+an3trvULXlRmZxlctyCsgpcvGJdw +ewqKZ9BvalfbXwKprDdUNyakqDa2ACN1w1Xry9VO+TqwvlgB0L0iMto3pHWI3F23 +keWGcJW2p1GqG5mOyEyYI8ZOSQSgdVu2s+v3P/++LeuLzck4TujJ0Ugz7BhCV884 +PsjIJLOZnpUdGekszVwXFoKGyASR5ob0zLR5191RxodK6+lIjhQrd4uCb0v1c5JG +QdK/bEHvNX+ojPbikwuaZv8LUKQeQlnBwHYJ5YLspMEnR5KvxuCLRZMYbz6tvRt0 +8lap89JpPlPfbJpof4dNkz1+synhZdMRWJ6mYpdNkx3jsskvl0smV4xOJ1kWp9FD +Jov+ddfDDV0OmdxJtb24ZPkYTC45VM5x4NKFyaUr2NRZrrR8CyrzmVxa+7FGLvnA +MsebS+HmernkrZx1CFyxGIfsxaUdtbuufR0uOcy3anLJBYtMBZciXUhskGmn06bT +d3qRUMe8JFS4s+qLUHHnkkOogGUcQoWC7DhJqIgOX10glEOU8U2YCOabTz739Peq +cgggncKX/UWnxSq26cSs2Wz6MoRsiiC8BlmLAWqeQdZhYcvAIGue7Nl/OMi6YPZ+ +BlmLkfkOqkam3EHWBaDzXvfGrO0Msg6TrDzbO7PWM8g61KFnkL0wB9ktOAOkC8rX +ZPsScJB1deqor7k19Ov6g+DMYOEvWz8K9i3nl78EZ5C9AgyyAcc6g6zDmObuJGvx +iJEzyYZTZZ1JNVzOMTgn2QjJFSBkHPgxyUZAa39NssavPcmGOaWeSTbT4wyyX+mC +4cYimeoO6R8IJItohMlRZQFWAcyXjmUt0Z0R5ikZMNNvjPmGc9TcnoK1Go9rrBSL +p2OEDeX36TLh6ejG0HVy3snLMHMScvsQe69P5Xbh6YueoyqGaheqzPBzsJt1A63k +QM7eFwuioqxJnoB95k0eM+eDbcVHWOKz6qAX8eJzRsR7Fma3gKr1eFGxPpm34YmB +/41+koH/p/L2zv+XbcPRLnxwHuknxpRUO1C2cq+QMwsW73PrnRZ/rqC7O8iCqFtw +piLL134xCQLtNa9z3qEDhhd9PRNRQLlxdmRM0sIGMqK/2V2PIYCzmQLVuekVm73b +NTaAYETAxQYQtfvCQWuvYMZEEbCz+eX2xueCV9fcXhWDZos+7pqthtkKA4dDNbZW +tnUXlML6jV67+K40DJ9wWkFx0eSfu5XD1JCK9bbw97J5YczKe3sKDJ1yecRQ71bP +GMTd08MsuR0PxOmFqdLpkeETOgcceiB1D/rsZTdcuLz92HB455DhAtjigkY/2sJd +0hmFOXFYa5rbIRhe/JBA8UPkqNfmoL7DTjga/2/RagZaSMYBkF5fFWetRrtGuqHV +NASq3XXY7T+n3ag5fpNuu4s3x/Bq75/jNVesyBtKjitHUDIjGkezERU2UlfxOO1b +l8mXrez8yTmQm5diUIsWQSu7ZPrACaMj2cZc9OnC8KwsQqORF/G0idOalzKObfqC +nvltb0+BdLJqjTOH4bKCijZOgDsLYrwWQMLBPkCfmiaglZKTARdr51jQsxg0Plvb +zsvBJqFrJSs48FfCV+3BVDPiFcG+sfBGcEFZ/AWm+FtVsjovwDGySdGtPlUUlOc+ +Fetd9OjnEJwnjO1CR+91PzZPa0A9tdWRuytrfxT7gGVxGg5SXmil5/bC4TwnjhEn +yQ8CWvElUKSFq4dqKYXaLc4HNePmARU+XZXGNtqOFhu2d9swfVPlvT72w7YDTr56 +ymQ91DNwRAt2WFnJm8nnBKqD5b9u5K5xSu7+RyB7+L7B/o9gQhAJMYIgoDDSYcMJ +i4OZxrLYAauiMpXUyetpQ+WK12FAqViv0ekdxuswIbcry+Zet5ju799R+r9OH1+X +j/WlWsJxNhNbz2X8q/o6eiBruRtXDwyQRzHXf4yrOCG3w7C7DrPv33DL+/R02779 +/2xXS5JluQrbSq6gwsZf1lPRET3Imrz9Dx6S8Dm+WT3KFPb1wRiEKB+eldfvf7m7 +Stz3LMk0vA+7WCF/G96H/TiDD7uDy/u6BpgwwIEzwAQcKl0OMAF73dcAE4aWR0Jn +BIR0PANKQBLmM8Cs6M/zMQD2sp8BJmAm3srNNYmJ8YzpS4MnVdULc4A5hmdwWNEq +1/xvgwYYuKMXzXlluUj5L8MTVATM5n8azleen/wwPAPMa+AAAygC4QATcJd2DTAw +rGd+WRhv5zOfIOL7Gl8C5zR0lls2HYg1vN+6ppedeiqnFzz+WM/0cpLjGV8+s0UJ +FJeYpLPdmJe7azrDIwIUy86DA8PoObDK2/hPXE95F5Cip7GyAMvaB2K7Pdu5blS3 +OG4DRr/uXPXCCgvDaIwkphzAptBM9OsDg1HFzK9h5nEs9yndgdNNSkLSE183FuD0 +TmdSa24WjnGuSair9nGtI2Bdv64MVK31yJbfDCO+g68vrQ/pYOjbC1bNga8BQ9U3 +vcMs8mlwdbkqd6vGsT6dMGNj/oZa1VZ1na0xl+IlvIc5oW7Hl3vXh9AUWpll/ex2 +fcw2L2OWOkmQGRU3qnm1K8O+QYbv5bqLtWKxDBlY2fjk5o7VFO0JVRi9z3a6txTc +dtz3LGKtW+G68XHwlgcpNqvdq8P4sGY9I8mGWzw3c9iMT6MjnawJx5iEHzeh+NrR +Ica4mD4MqygAmFICZvBYMjEASB/MJef2r9l4VaXtjrSlZOUzBpr8elvKG4eb8AZK +J5CorBQjanlPDSxh6Js5OxjF0JFdHWjaDTFMaTsNGKcq1/vkMIU64Om1arZquT0b +5Nq6mA+NIHJ8ahITJwYeGklytTVNL7py7Ro4qp4gIjbVWxmEWFgajcC3Accm59vs +uZ2GTZb81gNIcxg9xwNUjl4rP7c1mPWevjVtH3VyfSQc2r78IF3bWn2XI2ibcqez +TIP7ByFkX2z34HubGq6+gGwySD007gVLP7tpCJaJG/PHXamKjroxKJA0tqT1dmXa +oF7nExXm1sY8BM/FpgprnDbrOIbv1/AIjs9UZr/wSmZoqSQTdibWN/FKLohvAbpo +x5AT/nCYUiawra1+ZVzuyVmYUgJiwEyo7S5dr/WocpGQQNu8TOMUBINa4YCuDDRF +SpBUDwS71tydhmZcHoNlNkaerTivunI3hVgEXrdam7/1qVsgrxLK7Vbmvc6EAaFM +QmZ7wOeWZWp7RjBPq0TgEH5Yt4xliXXUJh5AwfnbAEXNaojzh1TzZRBtDO9pmG3d +hrjvqsfA8Dgpsmd4nItVDOZ4EqUlw9M16w70gwsVScpjiIdCr+GPJZ4qj96aW/ey +86oSoGPKk5oSuSphhidSJLv3a5XyHtrVFfeWYnqeUPauuVWBX53iBXT8kfekeY8i +6+ui+TDM9BU0H9A1dpLnfSoJHp4PQ2YgOC/Qri/PA4528bwvCsBD9AFt2MP0Advs +F9OHIZUymT7gkLQm078wmf4Ykunx6zEfpsfpEtXJ9GEgDSXTh7OeGSbXp8YiMX3g +Uea1Sp0hpg9U+0X0rpo4RB9wzfUQfcCx+kX0x/AQPX4gcibR8wnsIXp8bdeL6OHM +y/Nw9KV5x4BQL55HyM3e9QjZWg/PB4QsvHg+/qv74Xl41F6ef+Dh+TQcnkcHrA/P +x+Fe98XzYWCqJc/DNY0U5Hm4rho6RB/Nt6yb6NPwEP1nMovoo7mkVo9C/UPD1nSH +STEQ43OqBTMQJ4MpQRsGqHAYfHB95mi49g13VsNjcDetu/Sp5+maguo6p++h1jU6 +16k1oH7Hl5UYW6VXjaIMBlPxnfVm2h6qA7BXKeXc3F2uRMYCji71Zu2GJPcLt6nd +XfwEUteXYjUaF1MWBoxmYUCiKsiEk4WHa3P7GE9Y0NVifXOeZSQWfz8Vp935+822 +FhDzmVEv6ffeeBeo1OtN498bthPV68mRBVbiBdpLcgef7mIFJbhvA3TzPAbAwQJs +m2BJeBVyQ2h0sh+KJD4IiEEPVRJ5fMHCzvgYokqi+z6/jioZ4z0dnKjT+W0IdX2b +fgVJN/iVfgv+5sUwaaTh+zWckvgMBcaYZwtodzI6cWppx6Dw7P7seJyY1BOANqr6 +aDrcC/vqWI3Lfac+9xciHPVsl6FF63h+vTjHPoev54b58UUJ9LhGJn48J7rCQfz9 +86pKjeCssk9AQjqOaCYdmVe//vcPdtTIQ9EMJr8/NKg44vvVyOogkPAK0Jqg8fXC +UIq0J96vRjue+hrKMOAwOxDb67Od61GWztcaW9A2T/eiUEeqL1GiIRjO6kZ3HvOF +I+rEcnsaQGr6tdovgofTJ2U55gY4E1+ffEgMPYAudkfE4Wpv9UD57uVaT1WMX1fC +MYZOt4zMVvsuS+uuxB12oUoefnExtEi6Vv2HoTESq8pXjp+dPAaYgTF/4ywqrroL +hUjUhrmeZdYDtZ3P9q77pqQvPD1uivG1U0/I2966nmVzfU761rqgq4+RHX/kF0vy +vV73kSln6pSqYmBj4SOfAWuXvDZlmSeHrHPdLWFVtTwlu4wvEw9p/UDtrmu86/Hu +S3RgCu2uTJPimSUzhT746yQNCGt+/bjJb9bTOBo0lOMfYshD9KKOX4zwckkbN0IF +Z1SlTdRmlsyEc53HJNkDZsF06nIYzP1ez9MKkAqJX1YKh1NNpQ3qDjily53llzDn +vgtj3uOPu7QxeigON8mZfjyvhZEpfAUMl4SV20FNDKOpqyBMkJWI1uCbw8AiwNAg +yMiKG+IvM2AWOz83Su1p9euG3T/gHp/b99b2vpgTxXR4W+RNiQu8kXpWqDYu18yg +zpt4OUj3Xlav1dG0WkxRIKp23qt29pbS9QKuEdabHgxtEMm153mwQZLppsLIqEfz +yajW2qVI82vGVF7Pm4wksZrZdFNc3GTvm+LGr17f5XCjMipr6EG8+Pk0dsetShXl +4XBFi3T7osHqyd0Si0Hl3zeurCsETFpydUETtJE/N2lfdARCqb+h3c3WgdoNoXet +dzVBnwmHKe/PdpQvvsa6wRWZ2iV/PSfprrbcTYxi6nmV0aeGUNPx8mYMucqZEkW5 +TyTU5ro+plUVMDLhru9I2GyYubyqtB9LEum+xR79pPkeIhdtH91FAFkVDzP9JhFn +EeIRPMkqq7Jmwg2SbOxwTzpiK9gkr+CCwsWVCdQ564AjtNzUdhDdC7ZTOIlD2GhZ +j6BM7Ts71imcMfPF88ubN/Mq3tt6AcCk0WnvOm6h9Qxj3U8yZcE77yXuGsoXJGPG +7Y2SSH6R1hDLqP0/ryH8DsM3DU1Zw5a6OIHhCKGlMCXpY3RUtlMdQHo3ebMITe0W +ENtDthe710eus//G2Ghq13n6Dif1tTW4vvSixg7zwLbq2S4DFSpO94v2w5luov2Z +2xWb7dot3UFFtaX2AuXRIXPdlV2bYcAIgZ6wFJW22CJ6vnlMEpN6bpDdluZDtN4b ++TxBpCGYgZohThNNdzbq9UtSL0O4NLpOTrp8gMnf7qFbNFHdHn4i7u1en0tE2RST +Ld5crsz2eBkTc34B1VSK1KEe+c/FPDsxBAYTx3+J0ks+jw+1pq6QLldj2unYFONj +jKCfuoctpUITgwPmY+17uagLcjTLlXJi78aI1aG32WoV5Ua5kyCYqjUeWsUgk6W2 +xaXsLPJgOG/bMl9mZQNs+/9sl0mWJCkMRK+SJ6gHiPE89XpXuen7L1pmJoaozlWE +Ae4OQsNXyNIlt++uSYooaCbdNoUPo9ek5eTpS72kD1RhMWvE0qWBmeYr8alYrQG9 +2ZQ5Z9firHTTV9sbaVmdj0K0v1w2d47fedtN+3LZEDOgTYr4z3NzmmzIOm3MEvRd +5fVOpvhINsw/pQToBWSWfFHPv+dyRGFAInbZH8Z0WZsdxnRZVj17dUnaOWcpTsT3 +qK56yocxXTLjH8b0ARU/IqarnWlBcVtuxLya9YzPjpNr8O72Iia+bRcxXa50ERPn +6ONBTBiptgcxOXARk/IiJqRgLBjzGQBjUpbDmI8UY94BMiZkaocxeUPzMqbrme0w +Jq5I/SNuHDdkW2lxiZZGs0mNKxkT96NsFJAJM40LmTDjLAcyYeT5QuYeOJC57R6Q +idfVdiATn2v1gUxupxzIxF7VjZEiXbb8QiYM8VCmm6nUeSjTZa0PZLrudiHT5Shz +Q+ZWBzL3wIbMqwmZkHYhE7LNBzIxMC5kupyirRVbm+1Ap5bPNd/5pe5qhXctxU1Q +ZjHCzKZMyDUOZbrMgT8Wq6kPZWK9NkfKxOvbpUzsxcZDmThLPpTpqt10BCP3/IZ4 ++WXW3vlSL2a6ZAE7mIkBvZuYCU+fhzI/UhMpc8fhocwbmEGZ8P5UN2W6ssOYSEf2 +MiaiXnTelVGmWImMeaUY82oyJp7VFchRWaUPY2JASZuMCUe+CIlNhW/kvbqK1fd8 +17lXZNFI8YGYiHc7hIkgsEuYnxZShsd7lYaXjBYDHihTDuGFbfykzeNaeqj5bBXH +BaJxvjMW2i/R0ojDe9osATGL07ZERFYpWwR1GLbJ2fd8p638YfomUom+HIHVcVnY +mmaxCjEMetrSg1iQsgd08j8cSKP9MAAuz58DY+iVsDIy3jBuHoDQ1K9q80M4hyYT +smeepayso0/bMkyT2zufulgJxbwAdosSuSLXDa3lpFyXfWXVga99SY0p+/e5tHu0 +j2v/Q09YBEfEl/vwNwfaVAlmBnWOGVGRK6VFdQ/Pc2gyZQLG5/qV8w1155rR3lQw +FYp33moV91TKLKcfsRgkBmqi3YHRSrf0oSN7JPMY6LTnHw7UnH8eGLLHJMI0ByQ3 +sgY6Et3/Bm4cuL16/nFgf+U88tdAZc78GKi18NgEfvRMMspQ9Oa57TDV9+Q0ZFRT +ZukjLN62jBu5A7yxSESscX6f+2Uzlkdh0PW2oNbRv45zgKhKLH68hQ5kyd9PJM9e +ab/fge7xCVkV3Am9qsupnsI6fdQ8BavAd/8yZBvtNEPmuavXp1nyAfLknW/KWxN5 +0lCSlcbKXr6Mn+uAevO6pzzU8qIMOLFGY5htOIHtIZWY+JeXhOBusXQ08R3i1uVM +bCfRZj4SVtFyDng8A/H5tJqJpH11OXNV+4AvC1YLujzuK+sSn32jHORYznOhLsoM +a3OzhVVUvnsYpZUdw7LoPDG5LX5idt/Ino/7MpL6uU9shbl733d48/aGSvP95S0s +4D60FrNW9iN+3wGQedUH2WdNuYfJfU3ejt3JvXPj9BAEIbAsmFlSB7ea3/m29GiW +lfRZ6/vKkrihTV0p4o/bGldq23EHGkD/c552h8kl3q6qUea+hKXMXtuXtlaV9+1u +XVLnFHVrOk69Hw6r3JfTZvvTx8Da2Ie9iQMG/jk18PsOnCxn/nCyHwcCCKyR1zYQ +uBx2eMDVVJkKILBOMN1AYCiWt+C7rPnwgZYPdSGaH+qdgghcFsFJEIEPVOXkmG7K +IUSCLQ8S7IFTJv1rK/UfBg4S3AEigSEb5YME2K3NBwl8oJd+kADHi8tl6ugvAYRx +6rjz/toDBLDrsAcIDB3jOECAa1B+BRDsWzpEsAfu0T6unhm9OpGkrJCt8IY9gELM +h6pnMWUbZHhXPfOdbnMoEwLIcSs6EnUHiLLqDwUhSNXcQmGxx5DdWW87Ai0GVU9F +sKA3I2T43bY0H4A90L5VR2LmvNnoE65VFLnW6TsL7EHuLlNSNp7ah2c1BW9G/+Ny +JFJk6+3Kzl5Ly2PAuG+XnVfW0au5XIn+t8aKrSTgrbstSh42g0PDDbVxtKzDd1Bi +NW/IaQjhi2Pi1JNdFI0wCqT1FlbhXuYv8JgbFBTgqrnH0d5XyuDofM482pVKKVVg +0cmiqLtsg9O2ND87P0U/dTfJnTvLamH3wGBzI7+ZOluT3+CjrjqCqKrWuRwq9b43 +Jiq3RNXeUdQhUQn2USR18DnHnfc2NPFbMAfNVigtXC2zo8SthNGVvMo4yuNu7iuQ +rihIfHKon2x6c15qGev2yyVqGna8GJ4X2y62ZdxA8Bnn6y46ueZ7BZWUHVew6Mhm +cQWLb2ct/wxd5vY6GAFEA2M0u72X2MCrmqtmCiJ0vC5tcjXqy2+uLqYegtHtmXdc +WKjIg0diuft8bu+8KYRbklTTC/tpdYq9AWsq8ifnF28wJM4e7hS69sKXpZ53A8B3 +q7ZPFfuKPhGXhm6NGyOtZS6OAuLmEmPDSsyTrZLZqSPi65ck8wEphJIdYU/lPN7F +vDnmWWI7g/DK2c5yNWpTy3F+5my9vauBzDlWI8Ex5WvvAQraWCkhtDRF71F55ikX +L0kGQzFmpdiXFWUycZu+rnNXK+4uDdpkzL08qfTUojIQJveoiRuoKuJ9xOdUBce5 +kKlKNXO4kvo18hQ8T0Ax674SlJcz7/lPJD9apSwi/SYQ9IFgBqNN5yYkFoNQfgO9 +x+rAxqLMNC9GNq0vCsqhj6EfhBRgYEAQvIb21hbdfMXeei1banm0TXt+BDP3kH3I +7ffyKUpl0Ex6CWQ8vNQdWqwVvA+rcZDZWUE788Ok9RGATRvt6hOauj/sXKW96lP2 +oD5sXNsb3Z7a83rnYQ8+XHiDqyuV5O2NS5lFiWaKsk5M3LTERiBiEFewIlEpJnO4 +G3LTUt6V+0wmwsnE5Zkoq4Sf3DLUMsU0sgRyB3P2kZGyjy5J04OPZv1XW5BO1KDA +8DIVrSnxICvrmFn2h4wEOsY7b2InBZnHtngMjqTlXf2S0taQq8ARdybZJlJ6X9uX +wG/JI9xjyi3U/ez//oMVzcFHddOv9vvVuABK3IvfFYqby45M5leb6eo+AJ7CgKdK +ysX5jFzlcsF/JLHcq2Eadx5Q1u7Thfn3ebs3nzbv112Oevd25DirpXuNaQuceF7u ++XHu5fy4D+TYSxobR87WJeOkONqdHwIIPR2GuW8/htPXj1m1t9fqdG+MjPU0WXvg +9FQ8T/9h4ND65ztI6yyP+X0tAi6/D8WA2gwNZCs/DuwPn0f+GkDQfWpD6EMOE8Iu +SZFbXwwCRD1Jb6IuQUkukxyFPpTKitUzEzMzqi2CQwBsKGPw/szlptrNJE6nqkPz +VRiKNFTIB2tLLUfX+MzPxLe3tCRnFz73WL7ksrj3Ap7Q62qXzEu0rRS5B+DBspQP +UCJ+8bQmkYXw6karrdTiW3PSYVfW/Cx0uQWqwMbH2FLL25jvfFU4zK6nrQVP92On +cj207rYDzegj85h7uX34AKjSfhzYfnQe+Wvg9demXbpZvBX4fgd6kUV7moIhWWwJ +Gqzr0P7yySABJBX+Mohmw3Lf/hxbYjnCd77zASUT2Zi9hRglXA9dFp/vFW+PCuiU +kRdlzTyftRnLGx/PzG5ebJFguv4uEiJ+tHIGYaJXcrkKWctGfyWMouUccA8p5UtP +syKNZPF2yjr3ttn0qYXitoa63flsGzUnx/Io7KPr66lFS2AyShMk9DBhr6zc7DFg +UVE8IWBb/EDCvpE9H/flL0/53qcx9hUwvG/fOvfyn6CpAZS0DBXQUgukHAW1X8Ft +d0OgK3MRAqCWvwnEQiNwDQpuq4Oaq+Aq0xjSTgQ5z9gMUoeCZS0sINUtqI8JTv1m +MC40+ZiZIsubG0IrYEMwF9LRMzYDOU4BIMAAFyx3Dw0KZW5kc3RyZWFtDWVuZG9i +ag0xNSAwIG9iajw8L0xlbmd0aCA4MjA3L0ZpbHRlci9GbGF0ZURlY29kZT4+c3Ry +ZWFtDQpIiWxXTZIuIQrcv1P0CSpEwZ/zvIiJWXTffzuQiZbfm15VJKCl/CRYbT5S +55fpY7N/VYe1ScCp44X2iMnX3z+XQK1Bj9X2DKkOx7NqDbhcGubDFf3L+iNq0Gsf +AStWj0dG3zDM+zOnvvr+9Hat7o+2e3M3q/r+3M3qdbQDefT/+IL1zL6+rD261tfP +K+jP8N9+/6m9PFb7r4JiIwWjYElT95fDOXBi0+lQniLQ9l7jjC6QbthwDOhbbQFn +U0BdtiHNu3zoRyMs3H0aN7Oe5mtMCEqY10cKYK+EVWFuss3nXPSg4TK+/8T+Rbh/ +nwI4DdCoFTo8jjsZj07zxmj5lQPVMRLRuKx1tOVZSlsxuk0QnNV5tPL0gaNNoVvx +a3t64WpEwdzpmuYQMJLf/4b2O6Lt119yR3sL3uC2x3jk/xM0vycFO7ryFWgVwG5x +CX1KxrbyEuoJyXCsBb2WK5hRYusOtrpP7NWbr+Z2rQJqQzDMpWHuzmjQp3o0/Ku0 +8UK3GTOtITgeckH5BZsnh3xgJqqjOIwHaDScfXmNRfiK5Nnx81oHtJ1FUpcANl40 +4PaL3PoyssoXvaqZOGABF0yaR94F7Av6Ur9OiDxRhqQ1BO/FPuLOVOigs2AjQyYc +PBHIoJ0WMHI04Ioym14V9PzwXw8IZlx3IInBfQ0wOI+I1q1+qDVK+l2s85/Nzcb7 +c4ervUc7cBxr4q6p5uL4nM35oTl+zs85mn/GOCcnojUu9qpx7b043fJuftzGnx+n +8mi3z//++S/t17wLMgVv/cVW9ovgDe7HHgjuKE8zxHv0Ftu6IDLJBT0o2mFp4Arr +cY1FEnQopOjlNS7Ysgv0Nitg64AqB9JcZL76ebYrhCN6UvybPp1+A55uLuh14EJr +thd6gbS0Tqy9Qm0dsPfBzdXYnGuaL3ZA8rcfbQEKzBed57wycZbwVAOrmoL/IQCv +WtcvQnJCHA5wkYTrXq9g7e71eUNdH5D58QrmpHldbDaVuwvLWiStg1tAAzz8Ulij +lXmM6kYMQjDyq21GLSjC3WDsPMnL7iYSdVHGYAw2n4ygDThhzG1uHEu0MunS707F +GYbFrtxH/k7BjUN3QjXD4aLH4awDeoxUkX9lbMiL17qO3qFx6BnGmAz2lPg3zVdl +iwqnxsDB+WS+yEPAtroFnl+YXl4swS4BK2AQMqASsuWEoKN0VhDCyJbu0Ggu2jek +ucx+66M9BOwJhzD1t3kTxe+idAJO/j1X64RWWloDeyE1zbu0jlLq0RBje54GbBRn +qca6nPsuhVRCUphjnSoOL+u6qtwFnXW89da4tgJhjA0K0bRW7m2d1q1PkkAWxktQ +oMJdiYpM/blLUzLnPE0izt54V5ISpvSZHKYVypEss7yXwlmploEhPqbLC7ZdPIlr +oboKWIXJ2oowBTbdtckgGrc2jhFLyI6dMViyzQfZc+vnoj7dWMrY+ZRFL7gXCAyl +MJiP6bjXSxjhRxRl45iicJwXfeekGLQxgk4wHvQWO3iNz3pTmLErbcYz8goZMWDM +mwlpXl9teP8QXEC04jPDuqDlGJTmwko1nCxhNNWa5ilQJA3CE32/CRCGvRgi0ngF +SXh/QKH7JOw1Y9FcKq/R24Z/6ZX2qp2pCjZri3BGMvkIMmgd77DKVh3/HqzLgesc +GFNtT/MUSIyesZo0b+gKYYd0WrryLBq15fcbnSFRZMjsvEkd7K9sgeFl1nlk0Pef +c9dwF0Pa2QR15XZM2FksfycGDhRRHId1pY3pcaUPSzGGe/CxuQUzyoxDv/v0mwK+ +qhbvZ2xj0S4uOLXn/3NCk57mwijHwyeOE87wIamMfdxgvcE3Q1xnMA0WEzIqlYj5 +mHNravHkcsgkkIY/9T53znAajPcfIE/SZNywvOac8efWT15b9u7oUppkH2NFTvk8 +i/KxWGUffGyYhce3ZU0/lJYNnF6KJyU203RL41OrKb2cjyGtDOMVMxDDdGpZ50X4 +8wo4DH0jbeMt85tAOzqKL9FxyoaCMeYvgqg3YO9BzX7D+Y93wT+CM9F+Hvw7UvI9 +uu/Kq4DUfFw3lkTMffcx7gUYimfdZcUXz8aKJvoNgVpnVTqvO+zoH9VjHmhyLmBN +zub8jCILngiIOdOhNsCo7IQ0D4a89Kvh3zUGv6m8TbwSEWsXtAKat3gBBVzYbkQj +3zAepLLNKYi2P+PIZHDj5sXoGRl5lkkCwMUaagGcnRcpumFelBMw9RWjS0D3Fdwy +OTjmRatXLc3pw5G7CT1acvrVNO5k8Mw2p6xhvwji6TchcMeW9SmopCJbui36+hSs +0rYAvuL0qOmdzlE1BgD6ctFdE+4J0uR7QG5UOBRtgQfKf8rFbM8xisbmHPcm2wri +KpyKFM7Ol1JUfJx8SSJ6vtZ+aXNWiy4efu8cxtiBwpWDA5jR8cphrjdm75v7JAcn +ucmONVlRXkr5ukCcBw/jbCML0Oi1ZqDGOTBQhyBa+QyyhlOFaBXbMKy9z4q8+ulO +J7cNwiWgiLXQH+eC/5Cy8fPF7mqYIQ+MdmppngKLx06s7iw+3N2foxyyJyeBmY+Z +HE0dnmaLe+jde8MtfJ1S3x++bKcCdOHsVhiE6KzkjDqhFw6s3fSFnjJ8Db6Cqdy7 +dWVmcvVgqpe6XR7PgGheK0MSs5fPQ4VOrByH7FyzC4qpkBHdENuNzhBNbqbwKbsv +WrTlz/jkwnTtCC254yH2T/JgZgjHcBIMP/1QoJNBq3QcQxgPm0ANsNpIVyR918K7 +r5pNo37AeqzBpnhOxnGKsQjzsD15/NyF3C2TV48eGSP1ovncgG4rbb1ad2sjz3cm +UxJ7O5kdM150kUwmYx/ovd1w8cm1BcbXZkChk4wxbD3JZh+msjI66GViUo66UAYt +Bq6EWZT009brwGQ/Kt3UlJM/p/PI7YL1XeiIeGPE32Pe/owoSGP5g3FenOG4cgIC +Zyzhwyg5w6FVvTjDBb3XwxkOB5MKpOBwXZSxqqfKeLXRjfVQhsMlF2Msb1t5Vfw6 +20QyxoaHMV4BGCNW93oYYwW5josxXGCkTzCGQ8n5JDgh7tHmxRnhFjIO9cVn035Y +w2G4+2WNVR4+6UAajoTvWpDGhoc0XgFIIxb3wxmORrGLMyJiHK7BGREST7TNGX6R +Wu3ijPCyyMsZcVPOyyANXz65G0gj4lnaRRqRAPayhkOZ9bDGR/aANPy4iw+TJI0Q +qB7SCNj6YQ3AizQCVz2kASiHNC5Yj7Uc0ojT8JVk+6xMgSSNuEuthzTCc4wSaCFi +zoksacM9VUj91Ff27qSNSOYhF21ERnHi08woW4c1DkrSSJycgaV6OCO27vXijPi1 +yOGMOBgpBpyAopOLMyKIVW698gUAzggvtHFxRuR2qYczwovDDmd8BpScYXT7eYUs +T64l15C3FnliC3xC4iC+5/9WPOdVrheCLwF1uPuMAmc+HZ+CaveKj2PgTREHIf01 +d9APdtUhW/ANi8Hxs3qq/I/tKkmy7MZh+zpFncAhUtR0HocjepG18f0XTRDU8NO5 +yQxQ+k8SJ4Bf33+CscJHsr9qfHdFCgPnXYfXlKt476LRS7pXJOCYKdMg8mDohW/1 +IRewLipCT3xA6XVDbIe8rnfdc6FE62k2A0Zb80NCb8IgNKzF7SWzcD4I3YubS2g0 +iNWvuFoZ+oPBQr9/GLr31bhtUuEimhxq2sqnag+aVuN6ZQwxOQG2tlE6Zo13edac +v2a6lTysI7aj8OL3Jnip72Nxh18YI3+pWm6uHy9tLM//GG4G7ih/x4g686CzX/i1 +Y748Br+RSvwGfbDxzvCW67ucAYmWcHJYjIb38D1R4v2Q5/RWxG5wyiHk9hQ1ex2t +JBhtBRzG7VNz+9xKlV+flDUYdh9Yh+3tNFgE2qGl4M6vT3bSvXtwKOrC5dbJaIs/ +rhRYY+Xuyt1jcbeOxTbN3dlIZ1aAe5ENTAr9pmyOFkl14By5OzCUtsRyHRz+Ii86 +JbL/i+kVhtliCJqVfpCQIKu2fEfdMN857F1fg+uCyneS7UyqzhC5uovDpucMUBCK +Z0HrAZmB7LMbtxC6X4GN2qoYf62T79LFsyQF+Q7BolrKd4yROq0zOZj/mof5y0p9 +163w13EWAtbow51rytlPGte1Rp8omTwblu2mMHhMMv5G1h5dmEtkq7IzMxqmM4rF +wwpHJJv5zklo26fGuQHSFjDGGFcEUtOpkQ0lM9NnQdbciuRySK016ouQ1bmbBskI +pSqMVILwj9Y7yv62Zmv3pGEEjkiki+2KxPDKtHe95JwZ5Y5WRE05W2ZmHxw9tce6 +rdjemPVP7/k72tEi3/VoAH/CYHYMX6+h+rPcICUigcFwJl7jv/g0SVEoqx9wHnD2 +f2Kwh3zg7nevIqRWYyELxaKFuPH3O0ayoUP5EwGV5ILUArQpG3J7G+1d7wmNv2bD +0rF3T1KRLp6NsQKHeyjyWQ5WFEtgMjqfIdQ84OCR3ybkucIk6CtPQgcN/p58xrhJ +gkeSrjNJ3EV9yl0vnD5rtA5AoRgYbW8vEnmwIqHXVrw1pMiBczK/P3IkWMwft5Ze +tXYMN8pOtM1+MHirGpIGZWfzB/0GNA5yDRXtsFFVDBa8UIDggxF41Gb0qhVP9j4v +fUNsZ20/6zXXC6Fx5sQEwu2N3Wjm9t54OpqPwzFJAGPs7YVqmG/xzy82q8Wf6wr9 +N+MtmDwaO5vmz0XCsCo/XzS2o/HhZXUm2g8fz2qjdJ6dXqs8qc/tJuUshD4CKMph +qP8+MfATKCu34QiWz8Ay1t6wWpg8xf4ELkynPvFWb2BscA2t1+Fo7DhUXG5ok1IU +pOGwrnmUqVAzX+UqnRJir7vLK5ulBYL2jwamuVv5tbiKI4nyLf0D1lH3bhrA44CV +Wmh2HhUX9WDUta9C0giWwM3JOKAQ94Ks4IxO2Qo35cQz5DdhZWAsYSQI1Ore3phB +e3uEDRX5oKZnNzNE+DVhepZKWIy0Lpo+X5XMu2MSlaWYLRAQaxtyd239XU9BIlPC +Lasf2qZbRo+XxbAAt/V1ExLSiX4hI8GwohDBSF9haKtTwzCmk91lZH4UigYIz7yd +CCtXePvZj5hyONl3Umy5L+q4Yszh0OwadHOwrZ+drXD5Y8N1zM7ljZae7O3CG4c0 +cHz4eg1IpYCT9BMJuXzQIp+c30PUh54WrnfyUeHpdhB3G4edvdw4iSmEIyDFunJc +gWEl2XF7z4+3GXDk19e+y8gJNF/Cuu3Q6rgZGaUZLxq9GmzFnxaSl7Vxvbpr1vWQ +2FvTLrdsvOtNSG6dvzahZKk7gmrULFkZfbHdMKC3G/3963+/TiV6CNxLf349pQnR ++xUpFEMh86VxXIJkA7TBkW/pqQXOINmdZny6TXkhtAd3pyHK0H+rDH7LfjFzmNq7 +M5yieRWOamVxe++6Ibczs/byZOaI8CqLrUxs95NSOAdOdhC5mfDhodB9WjyXxu7z +LqL8aGeF7rTw7z+xQV1vVcg6T6k/F/un0CQAUeYITg04IIHnngTdMFesG9qfVu9U +8etGFIIkEDZXd7o9qxDx8dMWMLIan265e6AX4ugR6+BoXAxcumHcO3cTz1zFb93X +TjLn25jYeu6OswcTe18Mg0699ybk9njWXs9H49fzOmVEg6RTwmkj/j0unqjnD49H +ZsPS5VFZ23BEFd6j9QfDIfTPbwShKzh1XkJ3bGMdQlcIMjuE7lBMH0JXb/wsyGiZ +mtooi9thm/Mpfm2RIs96kXEYXb0Q20PojjvHnbiKxbi3Cf3CJPRrCEJ3ONQOoTtc +4+Fz3ISqK/gcF29187l28srh8zDMw+eA69B5IHnoPAx66Dxg3XT+INLANQSdA1La +Bp0jHlUeOkdEGGXZEWmXz5Uq8vI5Hmb6rjebh88dVlsPn7tBbB0+h9daO3wOn/b+ +8DkM86Fz7J/10DmcTBk/Mjus1YfOFS6bh85x+VEPXfvTlF9POld8x+76oHBOOndY +OUIknbuhNTl0ruiph80T3SgQHy4/OKgciNQQVA649KFyNwwOfEHlgDYOVwOuA7l9 +mr7rMwfTqP0UbIfLYZh2OrhC4Y3D5coufLl8G5LL43PjkDku0+chczxFD5crKFAP +l2+X7nL9rs/dgCQ964zj5nKHs/TL5QifHCpH40mNMRns24fY8LIID5XfqkwqR4LE +PMPUmXKoXDFszYfKUQaU98HlSLzRD5dfmFx+DZVpPMo6XA442sPlOI3hDC6PGjzk +jVsWSrPkcpSs9nddRQ6Zo74p7JLMYWCySXZko1s0ff54Kei8igvJ9jDFNswYmNxx +Vf8qo/5gwKE1DO4K+uub4XDJ5ynBJRXDINs1z/W8HsZEBOB26H2gDx5xDGdG4nn8 +qmcShZ15gAFHChQNF7qBKpHLCAhpZBIaOWmVyGo3SIpfHbFeCnOh9Qt9E3PlGnp+ +TjgRlan8OptYZcvE6dGFumvJuBmnrWkjYBfbkA8tc9x19wMHqo6icDjq4sd5F3ep +8HAQT8Wsx8mx6QuFneIaineAL16Og+hjUIpcsI3DqZFccBmuWwqTa72ORm7yNcbO +VJS7h7UNuTvCttcxPE4yNmEMfxYUQ190YSMDNyFdmrDo5u+dTV6isj1xkusLDeK+ +zVbLfLNqNDBH3SDJvHR21WggK8+Tkb6tOzYawVJhEk6GUiM2PRpoQr62tvWuZ56g +bdOVkSdlZZrMYFIQ6k4a/GUKPi+JIrbiXztK6k+8vzHPxmTNYp7Ux+A/qRL+7YUG +8Ya13h3iBNs+f4J6y1O+vh8bVW0t1NktazfoGlnWDqJzZF2bN2CrT2G7oYuewjYL +5b4r16HUp67N2eAaAPWUtUFQ61PWbmhJiyhrh1ZPVSc6RX1wz281SouoaXyafsma +9oOFSiSK2mGjI6No/daljqeoDUkq7/qc7RS1of3oU9RusJR4g+ttjFPUF2ZRX0PW +MG6n7ZvB7BQ13Ma6jKI2eHM9Rb29nFXtcLZ1qhYxyaplVe+Y3fWQu1nVeKz0p6qR +EXaKGslCZomi3sm0q/ozuaKs7+uyrHFCG09Zw1DbKWuHIzWrzrzffMraQOvjlLVR +N+2yjWDqU9Z+/ph615F+p6rDl/1WNVw96inrnTdZ159PYV07X5P58JiCZPWEdNnc +3SH//oMtrcR0gem04v2J/RUtnu94UC5WlJTDNehw5HITUgD0QWSbG0LEGSW/w8Yx +6f9sV0uSbSkI3Mpbgl/E9bzoWdWk9z9okgT1VtfgRtxEjyK/BFQrQPa5QNy96SAu +N/bY8CZhE1/t4pFvgsHGd6K5BFTfvtDQX7jHzO2sgtUMAtg5lXV7kJ/efJCaLAcm +KCPobrmua+3jd4Oj+epK1QfLN/qlLxesPlPgT1OOA40vE6Z4GXFZH/45SM1V5cdo +xg8EZ2g+hYKOEoGvQV42c/AlFaXJ6Lzkw4sF30Tjz4eY9w3tonQJgoowX9Le9Y6G +xD7e9GBVv1k5nZqgiG9XFEILCN2+fQkhypBB8db6CNQHSQaUFwBrB/300nzVCzYu +Q6U0uOWEkzeT06LKVUVVNjhpYxmaMB6+5103qyE67bhOuJrDKNMm2Nvftiee2p0f +THUMfAe1wfBIXCr3Fnevlu4nb1Z4tBg8WZdXgXszyXvRYZPerixMWGeeDwbyZsVR +zzG7anogylqpiZLAF1VpjNuG+SEhsjy3h2Bi/sNxm0nYeZlyjNKRaTALe+c5IydJ +2owG6TtQxI7sZ7Vtok7nYnYCPLGwOaKhtKOyrM1Rsf/5LERevTBfLXm6EmQ66X/g +e4xXbMQxKAEOTp5kshml2QSe/gb58Vr6Ql01t1OwPX9H0pbPTX4ZC9nK09dimz+H +r6uwu4zLtibCbvBrvcvwFXd7WFtJapzxauwebEy2V6qZfZO2/kJpuZu4C3d3dZKr +4EC/yuxqMSUjthckZHPru+LqUNyF6OcW4JxpFilcV9Y5WKL790I7LfF19Vpkx/F7 +45j4Xqe/paywE31a935hT6s+LveO0NJV93wZKgSXojZflBS1jZsuQ21a5zLUthBe +l6G2KSyXgrYrfilKihH5PusGRzsMZWiyewmGMsFif+IMBUjNnKEuDIZKQTAUYOmH +oXA6J4hgKBN09h7OUAaLjstQm0XmMNTOohMMhf39CPxlKoei8DLRh6JMIGMfioJy +4zBUoENQBzs/4dO5kp9w8n7oCc8g4Sj31vLQj/mjv+yEV8i7LCzDZKcdRyc52Zgi +csnJQqHUS05quS0vOYXgktMmT5OcNie4JCe7S8dLTttz5pCTaYp6lORjkTT7Q04w +w1mFvaYeapLq7e6lJhP0Q0wGgv2dmS4MaroC5yZ8q+OQk8Gm/SEnqR76SU64u/dD +TqaZMkqCnLAul51w3DjkhLukP+QEXVQOORmcDFknp4SHnK7AyQlfl3rICZoWecgJ +ytXDTZ6M47CPGXwkisgJtuAqaOZwkzmz95ebNpvNJCerKTovOX0UIWcnsRIWZdjs +ZHVJ8CgGI9OtsfL9xOZyuwgCs2ibr8COJ5v21ULw3OHlUIT12GYA5bUhEH8t4Bre ++8HcFhGDKgh9aQnalQHffH31JyjVvFTfkFVLPr3rmxNGfm02rPs9fftIGHdbInMQ +pV4BqTc3h6Dq/nO+tUCQfs8eXhy43e/u5KnUjM5NxdP1+a5YjUfnp2GUe3SYLK8+ +FqZiHwb/i6ERB7YQNffBEfQlfgMbMN42p9PN1J66xZAzuXeP02bkQwjpART/Zz1U +5cdWv8p8D99szuNus9l41DoQaodNh4fOjO3R2ZfnbHua7gwGRtbCZJuamff10ZyQ +79yj3XV+dr6GWfZ7+mLmx+Vh0NTsw+CefstSg20qRtvvKzjZZAKt/ReBOX54tV9W +45b8IuChXz9v8QRcYEB/x/RrEyoqEpBworBS1Je9pjsllOrUZ4JaPKEKGB6Q/FPR +2RnspSXk9sHtuT64Pb+e8uN0787v7bLm1e3AdXYTy4hlfox4PYfblZrbB8kTvelR +zVRe66pOGC/l9lyv3J5fu2Hu6cdwvD2tStUek3sCQqCv74ktj5WeXuzx/ie4jn1P +cL+q9SxNWPHds4r2i4xgcwXgqJyLrD8CbJw+Bh9sgpiTMJwYKrNnI9uXcVOsdu8d +TDCHvutNlewzHFZWLNrSUlFJg25Lg5UkiR76QpGzvXryNFP4ywV9yu+CRXOYOoUN +22gp0PqL4JgU9irzV0Hecj75IRjeJH8Iht0CKFEahUbBzOGd+gqjrRGksHxdl28v +QgexkS9yHDL3u4yaCYi+32Bnn88+E+4dFHS6d3aWBpTODI4zRXxGCwOoW0R8BBCG +nCeAOtv1DCCMSPoGEPqDG0A2X7UbQAanvgFkAp9Lcr1aq3gDyKCPYRFABr0fjwAy +GGb3ALowAigFx3GK9lR+FzCATJ263gCCfrP+IrjxYvZa81dB3nI++SE4AXQFHkCA +egPIYJ9vAJngdBWLRh1yI8RM3t8AMoe0/S7v9gSQubO9AWTerk8A2WPKugEUwXED +6CNaGEDjZR8foYyO0VXaE//9x7dYH8ot6NG/XSAzKgWAzwEWBFscNpLXbHyQEcNi +W4SBRtEl8T6wi6LAt4TYLme7rxsZi6+j2QBsrHCbY50aO9OaDbOXwbXdQIJWJeHM +CecKxI+b3gdZS4yxAoezgNsrQpfZfV0wYEK15s2lemph7twJ+VJdz7pZYgi/rg69 +suN06mJmZQ9dFtd396FF0PZeWO0rbg9BsR70y7UbFmEfgslihKYWUBgMQ7bD5XHV +9oehzf+Vr2mNTSBGEYOzJ+LbFktTLu9Y97OtV1jKnmaHsqN0ukV9Xfh1H4TKFojD +3meAfYFs7+PGnhFzDWQ2k0xx5/Ydq9PatbJbawyzzfFx9fRN2cxqLnNMaO4aeHIk +5G40Z2fdTMWWtTUadqkzf9lhSek+nW60PBk1ppjH4MdLvI3cVrfGaQy+r+B0jdsy +9IXsxEMAOJbDbgYEXKOyYfMyvS0k2PfsgnWLITbFKLAPrCw5KRDv3/Ex67CYeXH4 +FjbFcfiw8HYB2BzQq3zYBqrRzrXX2L6YFAVJgdOnJ6BaHgJivppJAobHoLHM/oCT +sK6LoMCO3SEY1rgA2o8RzZtIhCqp+NpeO6TRap3MCCfA4io74d+fTvKg3M2bWjy9 +0W/dhryagvDb7ikAZBQivHBjZTmZTGmoUDyQZHK9c7vOfqH5qaUpQ9AjCno77Oen +q7fSMntsLyTD5duRXRyNmqPZA/y9L3P89fOlH49H6ZwRtC44g4ydufaHYLCjpID6 +e00oVH+yr+onsFbZpFVlVKt/jAnhgTs3q57IsE3ba6ONhDyak9vYx9DI53lcX6Tn +U9xpWhP+/flUp8mNcPekrqbc9yuQhjMkq7vXR4MqjAOh76y8lsasgikWm064Gtst +dUtPiO2W2r2/63OSWao4XDx9tdy+mWTihrZO3uNo1u2os23q02nNBChFyFFMdAYF +TZ/438UiuvaMrYuGnV4brPkdnr19yQthFG53wXCi5Neen6tQrRWEpDu2S3UBWjDq +xVbnUdtSKCoJX2VBJrRB9Al8sroYfYKERebc2ejSnnfySXufxjb9kevhLfRJ9XoT +qjCewtvR8mUsWBSu+udHrPjsuDF6iossJb+vALVs+IWRKd7mGOycC3rURYyobJ6n +L/ve4nmvVudqwogdre+6NCahk4VxbY+j02dldF5Nn9YssutCKh5eiAo96dMySHa1 +xemaLExlpHtk1jHpByWxzn51J+Q79a7Go/PbMMo9O4yWdx8bj/j6MTl4dxRrpqv7 +UZ13IVju9WUWAIpxRyytAEdzApgeUxA0IWNYWQAsjDDz8ig2C5WRELttnhn6rrcI +UMtkwAJatavn8u3FUt2/33YH4OIUgR7ygVJGbF+sCJxj/hPIQGjFiyYAqn8NUQVM +DUzBLjAxgyQEA4j7TKFNOHCVABIwg+RGU4hqCzNwUFlaGoH9bmhgCOMmcykABBgA +uPSI8A0KZW5kc3RyZWFtDWVuZG9iag0xNiAwIG9iajw8L0xlbmd0aCA4MjM0L0Zp +bHRlci9GbGF0ZURlY29kZT4+c3RyZWFtDQpIiWxXOZJjOw70+xS6wChILFzO0/En +xqh2/v2NARIg+FRdlpQgHhesCWn0prFfyu+96SUGpY+XyruRAKow4Nqv379cMFxd +3l1jfW6s02DA3QGla6pDMN5zyevLBPzW2X8Q8Fv2hiA/+UGwIPivieQ9qXT+XMHd +Vt+8+o+CpjMFOvAJy3g5nLQcqizANcXhGISX2Id9Y8M5Xw77hPpiAWQpGOoy5nNd +V6y3BThbbKYj1RcTBC3U98BlB7ww3wDaj/J0p+h7k+Ippsc4rPXYXVkBlwIK4+xu +D4zvmXF6H6FO8bW9Dw+Lm/RwuJ2iUqtmpR17dQUcQ3GTMVJb/WuzXg+j4mhTa/E1 +fGA/IqkOwfX1h2PD1+stZqQU/ae9h061zexI7a9///nl5pl732j43y/pprYrHjxA +9psJR9NiP8k09kfcmaCZZx93sYNdYJ+Ya7+wxwonZkx9noLbdgv9hdsuRmR2T43l +AosaA03UgW4zTzetHpCm28MEuiAQXVhniSuZURz2Rge6umWnHVzrZDGJJ+ly2N9r +YHU3JKMJVJHbRBPr0rrDoeNCNUdTqqdg5HaaQUWxuSBoeK28S5dImC24izKC1tX8 +6i0C3mG8VBY/15dfzb/usFNnid3jLl5RcPiMZZWNmyk9YSdJ7RQ0gufsNm3PT0GP +Mjd73Jb6RN2y4uYwLUP7GlpR9frE+uIeRXKHX+SgeBvcdpdRaAyOgNOjyPaSnbdt +UTSJFl5DgrO8olRA2ZP6scQNsC8Ee71OtiLmbJFWCCK2TdCgMTmsPSnqR0SZRvGY +fFwTlY56LEdlJXjGHSkHhmmmPtbNUoryQRSG5agXbac2eZR4UdFXBY2iDn57yG/k +03xrQxCvSOQjEHsiip8JULjNQMP3mO8t8A1Pv/2yEzQaVMTGMrvuY36H/pk7ZzMg +6ltAV9+WsXzXtxkjAkGBeM7YfKS2dnzuZQ1w4SyF2QvOaBJXsFAODAoSrqW6WGfy +sNad6hxhTUjBbfEON0axsKuPTNeVL6WWHY9gmCVwK3PA2ZBxXrB/h2FHtCDuWJeN +r7c3iQOn+e+op2AgyP13OdySXnip2T6taP8suEwg0sPo5jx12yugR0PCeGj3l591 +ezfH+jEjYa+sDdtegmWhsOLaWGc/ixoKlkEibH4EExzlC98vb2L2/h1mHV42PazC +h4J3rdbzNDJnuWCFDzo22xQP22kF6vlwWPmuK2HvlQ/nFUepHpcJBEPkVWbzkOcn +5Cw0JYiabBYYGhQlDNGiRMu5edt6iiquHvQndHt0eSdDoRzkqIUDtiDSpoVrWBBx +qMGNTECMSGXfzCBHGOtF3tiPcmC2vHLUvRadWrlB8rxblnOdAnlPcx7l1p/R8cKc +S0aisIfO/VhlRucVyUqQO2X3NKe0Hc14Z0Kgu84e8FF4UIuIoyuxsRnUWRM0xifq +TYyMqFJAJ3EGV4vbTfQ8E3hq4kIb6zqCgcBqhPhNGOpOVWu9W2kacX8BHEElYu+O +munXdXrqsEeV87ddOEapnxqPJDA/gCz8JJjoIX6dHmlisZACD4O/BMWP3GAf7P8K +zin1yTeBN+bxKRA7xWEnPiUXMJ65jolJUZF7C4/wytY70+LrwPTImM/1OaKTe/l2 +B2p06rZSfSu28ziA+6O3dM+3Ex52taion/HylSG014OqluCayHwk40dBDi7krW/X +4GJwBwnF4ELWLoLc5+BC3i6kBheDIlyDicHR5TG4kHMEeq6vaO8YXAzuqY/BhUa0 +7BxcyPv+qMGFvLroY3Rx9bYfo4tv2KhGFz+uBhe/SuPH4OKXbVqDi0HN9Q3E+4wx ++e4I8VxtUdAwuLjRokbl4GICNP4cXBw2rsHl+KAGlyOoYeHTseFrq5ptPUYBEzi7 +jVHAAAbEHAVoxxBWo4AJWps1CpB3911Un7KR1ChggqOOdW9mUqOAQQrSmKOA+WHP +VaOAwUVSo8CBNQpcAUYBfE01C/juYz9mAT99zJoFDG69XN+uKnc0iLvvcGusb5Sc +MwsYVJ2PWcAEKyIUw4CbefYaBi7MYeAKkvv77fr+JuBVw4DBwbuGAYMrK87+MHUO +AwYpigI6mHtmVEMLdXjuru/g85gG/LH9MQx4WGQRcW7uIRMFCsPACakzDHyGGIaB ++7ocBvwE7Y9hwAW0axggJ1tc04Dfb9NjHCBvslTjgMGRBB/eCdZ15wFaqIa1br6f +UvMAjHnHATf1mDUOnLjJceDzKWjBbLkTHT5zijnMjZwykB0fOWWwh3Uyp9gK+lyV +UwZH5ZAjjg6aKcW9tLFuLXOMSimDPbpzphS34NeZUgbH1kqpAyulrgApha+lUsp3 +j4E3U8pP51Up5XejmzJ+98aPlDLBDC/EOvu8cjKK/UXrkVFu1RjqkFEGV9A9ZNSF +mVFXkAlkt+H9iWMEQz4Z0t4rnwz6E24+HTtnPjEFT8t8MWgHPvIpvXaXl85KJ7sa +yH3lk4dEG5VPHi6R28inE04nnz7DC/lUb8t0clPm1BXp5IJKJj+/ayWT3y5pdCST +++UQoFhHGc9kMUjRwjKZTJAUPtbNUqKVTLBkv9nkhuZV2XRCJrPp4yGRTNaW93rw +WRNIUkxEjQfIKD5rkMZ68FkTdO3FZw020uKrbK10P+isYaXnKukuNmuwBxePna09 +R3KBzRpcQePAZi9MNnsExSJNwDR/FgSbtQNlrgd59esp/yAoIubmmuNHwTmlPvkm +KDZ7BWCzDtcsNsuecfKgsyaY2R6czrpJZRRddYPHKJV01h3Cz2WvRIfNujfPXiu1 +pY9is/6abGvOZk9wFJv9jBYwHJ5BmsKnfyAYWUPdT4Z61F9lQA0KM7KOZH1nfOQw +R5g51xOu2VM9Bd6g8DWjwLbNeVjYbp7dx0YpIBWsz5gjKA8D7wzo6tbVhe+6tTHN +5zMgSfb81GaOu6yJ5eg7i/iBgoAX5JGqMRV2VCA7x5ANFkPONfwZhELk11wNcPhI +izc7Uj0mUS9GPlhG0LoVFJ+PsJF3F40iCRt45/fTNL9Hm7CqNdNG4c/Is4J8LPpw +NyJArCIFi8155giMFgwkiVj99iJyBWbSiSThiSQRC2Ov6Fdj4Dl/C4pti8Lufwum +bcrPU4yxj5330MFPjRTcPT7eguepdeX+pBsmGO1QeAM8L90w2OVJ4cXYWb8U3uC4 +dEKS6BXdkFXaWDce2bTohsG+n3RDLDLWpRsGJ18Gf2DRjSsA3cDXXHTDd18PtmFY +orKBbRhcQ4pN2NVZnmzDBKtfNuKGWJfAGwRxKrphgilcdMONvC7duDDpxhUkvfDb +8TcczAt8wy2nRTcMpWGSbhw7J92wu/cHn3C3zCfdSK/d5RWsDnTDX/pkGx4RctmG +QQ1mA7Zxoumwjc/oAtuopyXb8APGk7wjyIpuGGp6ubsHXHvSDRNM7UU3DGrQEdAJ +9yM/ubsJ2rrc3d2+LneHJR90w+08L3lHzFzy/vES0A3teGfmlyWSNU/7zmtRf/37 +D1T4JPowg/25gioFbkvf5C+BBVvjFEg8Eh4yY2+E5vRLeozEk+dG6JpgJVFmd5EF +FUek6wakNg909f+zXSVZkus47Cp1gnwaSek8f1t1/20TAGUrsmNnUIMpjiDy51pH +6WLiePwcsOeoqt2hg+hbce32rUnV+g3rmLmdgpH1JE63rY7WdHnJfthSl71YiybX +wa/6Uz/i2O7tqi98+sf67EneKiEJUict0vbadHuF4VB4VNyYsx+O+g+hexwx2Dj/ +yREzeb1Mj6OkI40w80pdEuqpnoFrwi37TTu4pcwrK1HX571sSqzajHDxR73qKZY5 +R39tIlC4B4YHuqcHUjCWTF6Ksg0Ugvcmvx4nGhQtFQQHSiUNTYgikVBP3JibzvpQ ++hsZFg1QmF5ZsqHz0uVaHpbJdwV9KSeML+Ord4ErqI5mLj0Ca/CHn//JhuA/DF1T +cQqe4lnEO5d99zfw909b7U6MKC213etZK1Y1wq1x1uU/KxodkCBxe8Cu6RaEEHCm +MnPldnPeDmIMuKi6PpeGHPG4EGynCydqmNWfwjd3kNoH1aF7hcMvwaJ0lFHFbObV +aiTrKO0q/i22UakM4XUpHYnSa27vIn5u0rTOTJmeJuH2aWlAlPScrmRPf8alY+9n +mjr+OOvprU56cpwJVRSo6eucM04kRAoymT8iRckcfFd9Bm769woiZsfQ/9REyU4C +svYMctxUTwPb5Or2TO2mt/hBGTrzY3nlHIKUDbMV/bjbiYXWmn4ts/Y2pZi/UIqn +FyQYU07l6YiX2vJ2Ow1UyqzCsEdRpm5W35SV6m9G58vOsl59zqZR3rvTaPnv18Yj +D18mZ8c0zDb0+hK7tvBoo5udaaPQ4qSxCU1un4opQ3FXkKzJ9ZYMFHQkYMmYymyI +qaHNdz2Y+xADRQEK2NS+fXpuL2oomF2QS1vz07ALQSNlmsoHCu9fHp5rfRFE8kVM +fgjAgwFtKA6KlHPV9SpSGYKV2Tj11qKnc1jE05UfW8XV9MhrfSavA9WBYVs9qa3t +K+e/qfXtSie4Id0UhaKqBqXgfVy8aqwvgmf4eF39f4J1phGbmlmfYesI3mtNqfRF +UMJpErgqcaebLG5gW5l0ucc+XmjqyiGoW4qjKgXsM9nSIJzKh6U6EwIb8153Mf9V +dPtKn2d8YthVW0DTD1hZ9a0JNc+eYHn5VuruxsER92nsKFXXe82ZZhJa5c+r0g/a +asqppu0j17delnm/z8N7e1ZNTcVEZQyDjjLbpFq8oJLKsd0FNFUVKzpNJ0QTGiO3 +U/A6+8OzcnaUlvnh7BS8vt28/pugR2WUwNU2JzhewC3ya0gRj9qXvm18RQhacuPN +5dFfXwe01i5fO7rletdr1EDGgoPNBEwWPhtzIgQmqpzLLmKMMSZRmEvcM/FjnsC1 +9i+CaPK1fgoYpwER73BQFGSqqrCspabuq4upNdfbRLU49+DlrV5zEQwzx71elTQN +XAdm9QwddoUQrNlP5NELCsv25/FRRIrX3EzBGwsfjmcseDzL/AyK/yiYQ7X4D0Bv +7AIcGwKSw3bmJrSP1mgq6/FIQEumjzkgIEnrMyd4f7ZzvbM94bolWNUUdpFfo+72 +HHmhTJTppcIPenbgPKT2FRivw2mRUBBu3N4UJGulMqNNsdDB9ZUsE80+dG/7IL3U +FXRaBq9fOlwJh8pFV8sIgalMF9d6aoZ296DaRm4WLm0x4nowLv8lUFPwKlWHJpiB +OdRzYILGr5nVwKqeUmt7JpqA40F6mSVpyuWlZlh4eU4accfYqWxLpooIRLjodB+C +DCfORP/9Dq+/4H/v60bco4irituTgvHPJN5dti6lHbJDBcXvi/fjGmZM1WoOC8qn +UHONO9/iXf3Kt/5mqyxrKjxlpyWnGN5mWcuQmewyv15CWrWCqQ9lmJHcehCzOBoM +LNouH7fjte0SxBEMujkpQAA2dB8JwUZ9u49wBNJf/v7+LRN7hep4aHQuWvnFC08B +dCBMCkCG1o2mydQLARteCDAVBeRUFF0UibuQon4gtodVa3vXI1reswG83XeDx433 +33DRq9dB/uwlRO+/Ts553ezkoNpdCauUoE5OJz06C+qJfNO7zhef02mR9+5jMP36 +saYUu43NKQeSNa8eewRPS+Vj6hfBU64/75BXw2hL01ak/L9XEOoqEpK4gFn/AbIp +2l6ccOwcBFkDQ9AzbrS7rncwDJgkPAfDZc9xroP5rac5LMweKt9Vl0fzcNWlweWm +OdGikDzwDIaXwLq2d/GLAn5xLrczqvHnWwVYuoyleo2+CdXVRgH10GrzXu/JODDs +wC6u1Zq7VevZ8QP5qofM0OLqK6Zh5QhA2Vr6wLOKTN1u6jwYzfCvZIr9uEDXtWLp +A9WjpXeIN7V2nm3rWo9ni9dWdMhlx4ZbzCjMthdLqzMvo9Go06CQXPD1wVKHXW1y +fbP0T2t5ObtKT5uauAwGF/3cy3ymJKiuKQpQD23V7/Xu6jMuI8221LMeHwgPGWJ1 +xVJa6UoDVt4dNdEYylZY8EKgAXaiYQeaSguMAxsEVHGvYhCCXnM+3Vyvaz+RHrAo +jU4mBLNa482ETUJDXjQIm3dRKG2ujSm6GE47QlrdGVz4hWZnOwVoaJxFVtT9Xb8L +XIUi1BksdegCEnj1L4Kn2MBc4xs+/3gO/BKAg9inYIzGd/iUS0wmWVX+W+dZazE8 +apE/VA2KCVWRimLHH22Pe71vbQ+3033nrpXbp37W5V1zBabLe4yM0Ky33H2FCuvq +jpniLUf/boEhFQJOF+kASQooB3RjsO5o2SPpYARzQAy5p9rsoJ7zKkaBxR3PqioV +xrsNFqpSo61oYipzKFQb8e8aXDdhF//vc+V28gCj1QFBvoxfPkWpSFUgWINUZoIH +AapQdLcbZmU4ApEUnWah8NLzdnoLRVjbzShoPrk++5L7LrU766a281mRMKa/FyWM +Hg2bqIlZms+KneSVNeebqzL2k6rHF2c5PYU76uNIaEIqfvycUXyiYDCgfkUJ+3zc +V7KGThK+IxCNBqyL6c0ZJ2DfjMWegd7CAeLXk8sYL2Ao5BTeknYbLZ8+2se6ufJg +VhlKv84mCUOqgC+ZtSqnOMIcKM21OwWYynSYj/Xa8vJ5WHi6IWdTkAeoJiI9+6u5 +oN7Jl531fPY5nGZ5L0+znZ8/Vh55+jK6yv9kz3r51hG8FQ4UoX0VlOhfEjBqI50G +7BU5qRlwogUF3CIJiCfo6D+aNMgSAjWZY7EGOShVIm2e9rFsyVfKIsy0zxAHpRAD +YbsK7ixCYk2QbR5V4GznzKa+/5cCl/lL1fXWWWTKmoSzy/hp7VBWVLSatvdc33qY +dKk7N6OBn9V4QTqSAWxMYXIhqRY6d7pK9c306yirRafpgig7Y+R2Ch4O/OlX1er1 +g0A8g80DOT8ACXK4CLinX8PH3qzgZzgJWOt6BpeArfVrsAlBb36vj3jGdXqs9Xk7 +OM/5eaB9qfZAfzYLc4Z4znLAOFc/88f5dc4nR7GcXo7iz3RzHvau89nndJrlvf0x +m/5+bCrNLoOr6MXxO9UI37yKh8z1RfA69ToPl84S+W+q/z1uBCYfQAcIecDSNWkg +DUr9ETmbFYkI7EODSrRjwPmSM8BxdwQIko9pvcRtVdRQEDSfv3ZuD25hXHf8NSCG +J7QMlKADQS5yd+KBrsPD67A7Xq76vkbL7bsxX5CoVG0TVm6vdEfAtqRL2KlzfQ4k +ugTMp4nKBcg6MKkbkDhMaef46CI19c8Nx/6ADIpLsJa2tz3FgXR7PZmfu2vJOiHd +typgG0TeDpIPZrYLrXaVlFaarGAqR82OlUhkou3JBe7U639sV0eSHbsR3P9TzAXE +aHjgGFrpAAzthoqQ7r9QmqrufuTfzJssmC6TZXAigGP5kbNz+3Ap7vXIS+F2FJ+I +wnE5nCs+Z01XTz614fGtBJ/iddht2L5OQhte/YLzevWwMz0TAXpEGzN3Hz89m5za +csC79oMQgmlTQsBBSpbcuNBhhH41XZzeCbthHXl8mqxrab1cHneGtxfPQoTeHs+e +XPer5sxAMbXt3N2KX3hKHM5h/ngc7h58Sovdwkij1sOUZt0mn2K83sqI8lSlGmrE +lymeEaMiiJOZw3DyqO8cx9TnLM71Ee89ZWXFb1SQHtu7R8oxvb0tl4DIi7s4sf7d +acgonKhXkZclGMcSpVmLKUIGKWg7ylevWltBoIKhyE9AL5flYZrZ9sCWmRO4Xl6u +DoKZ2q7m+GepaxlQXz2K419cGVWVDKOQunLm+j5eDyde102myHjNsq5dSIN6zMXw +2uOjn6ryTEhFga+GX48Apg67DSnpVyMfNIT7UrmsHAoAjynlh9y4MIH5mVGUS5xb +mj85Bbtn7TJmbI/qm+vHsJIqF7q436AtMhVjlh85g/kO2KzbYg49kGXO2yUYbl08 +Xf2i2su3D9WsEY0CU8FSbZ/iO3pt0SS0mqEoAZiqC0/Nyd8SiKoW6LaiMmOCw7Sj +SXZePZQra7rGhXIuqI5sQLhz7TTdgsZreHpruhhhStHiukaa4jGn2xBtPSpoy5E0 +TEM+1vm0W27BQJdnnC3OiABHuzeHHcDlKWZNw+GhZ+4S24fHt4af4BMv3j+62TS0 +2seOjwmdeZNJo94giag5sxRw2KN9l4Ters6d6/DYFuzNcAzBcmp4dHteO8PsOQwU +nTXekIzz9hBcJbYXtxjmNy8TmiWZui61nPvjinVdDldzrP1Y0vJp7qa+W95kl/fh +7WffjCzH6Fj8LFzcDrjEzMpRI2FXh47tFnic2nrSqQ34Y9OFeN+GNtfaoYYW/kGh +Nh/6qoHs83a9V4unvuboHhf8mwsrxjpSRpVFcMqDH6VI1alcP2zmwQW/bjw9awBx +BuPELDC2orOnPgaBkpnEg7wUt8rgBqAXgzqlOrtyGb1qtucwBgSnTF7eMmX4aRjo +t5/VCmitvTkE6uR5tntyzrsRjStV0bfZ6srXrZkdHHqn99OqWAyT82S45Lk5HJZf +Tu9arbev1VJ5nd+NuOXXCzeSqXBic7G1XtcxxXZLzYZ7IokPGDP4RZ6HGUa2+Zr1 +Wa7O3DyMUSCq/s7tHJ/z20D70SsRtY69ysORey8n8evipZYcNNiqdot9I9VC3Pej +tZFt7Pu9WsfrLD2y33fTYef+tJ2Zar1dTer/969zfU080jD+9laUqf9oSBzk+//+ +/de/vv6DC5nA450eIUgiskffJIXz1geFMfowkZPiKB70UyYAu9x8JwhK13zlD3KX +81Oenp5mnttZNk9+mz1rPnoFfBLkFpiJcTZpGnc/NI5vJ81Ds8iBUPzOkLArVsPo +PBpOea4Ol+Wnbw9bsQ+HO0dw4fhIkltgNoKird9MxZvS408SGdqNh+es5/3JgjDl +yRK0h7GedRjOnpOnUT/7+25ya92f5svqpdcNM01CkISMw0FWX/1QOb6cVA+9MhFC +7ydPaNd8r9dRn9N0ysft4bP4djg0FftwuLsEOnqr7zwIQfINc4Cp3wXG+mAqmut8 +MfmAL8+MAbhmfeVBRQ0dT54ATjbJOA14xuvyippgz+EPgAbDUCvhnQaPQGzLs8HF +uPqman45qJx6Bc9D7TsLwqpYDZMzC8IlTxaEwzILbv86Cz7c7SzAhaO+s+AWmG2Y +ONrNREwjfpQmUaFcf2XB+dFfJA9L7iSAoZfLgdZpt3XVYcDexutyCMa5WwWdth69 +HhhZkIIgWx42E/Pum6j57SByapY0D82fNDgenZ712p/Tcst5pwGdVu40CI9mGnx4 +nGnwz79KR/fCpIafgtb7CwI2rE0BRyrCjsGrN32YsOJqwLU5J5Z+6baO5xbGaMKN +dgO4934gk7nFdgk4tB6tn0ME6utufgrz4PDd+DbqcPcLhXCztfFRw7N4mVJzWqXd +0BgmGH8L8xnZt34I+SjhZcd3XUK1xqcGUoWa+Etdal1j3gg2tJ1qHZl4vHfAVDrA +aCFQ8M++Rmw+dGdTqsndiKe8azjxwfD+z9/D8Y0YFRwdl+xYsyhC4G0pKfh+BEOv +iw8BXwiCGN4B29yGIFnn6D/1UQhQuCEoeGYWmdJp0dlHkBSWvTW2D4d8Fa8vuIlS +o4PoEYUDcOuQuxA7ora13DAWEU4MUwG5Gz5f41n3fM/DohPCs4e/XGL76XL+UYzB +e9lZ63ohsp+bjR+3gQr77wQ4sT4xmUM4tiwZ09eXIzfV2kKZdRTq0JX9jpmiSIel +hvYLZ/R7vesdSriG4KzafmI7E4wfu+RG/E5/u1EVfHLIL/3Y6bimys1TaYaq3RSi +XYVY6GRWbB6C5eyvm27jR5wMtHfNzQ8bTVD8b0eUbYKGADfCBjqSb0+FaU+GCVXM +xnB8IAyCjWmFDtQ1ZfrS+jalICzjUiswwmZgpfm9Ouq8jwLOaTbOFrv3dfzprfW9 +nMuYHm4Iz7SS2y0oTCTCpsxpsX3RrXCVCxY/180f69KLfLcQHCk+dPjsGWb2o1rY +5OqDHuliWAlRyjHgAu4qQ1XYGkvWVbfWF/mFCqaSdcMoJDc+qsu4rYjrtTsC5aiE +9JOqTNXO1bY9XoZKKQoFNWezDWg7O3j3Wp/V682G7qhPK0MkUyCo9CKeJlsUmvwY +YFsq68O1OAVscpX04QHSm03Qfj0u9CSVoiBd+HTy18bW10qEuF/uGrG7hiPKDtND +Oa0/ppeg6lyC16ixfSD29GsvX7fjoNobzDrfbkV/Xo7wsh3XMpsaHMCycSV92nYT +Sv5Ut6SwA6MdYU8qn2r2LHlROYX6CePtQxF3jxNOVWKgDDXHQKCNF7hKbp0uHVrq +09lVV4RO+XI9oaW7UICQMfI9DY6iBljaehc9ePdV846oTtiH3Vd0eKery9HyPkHh +4xJYDF9Vh9MMtWMb4H04gELEge+jwkMwr/UWwPl75hELrlHeAmR3cbecZmPLbpmC +aOLPkQ89vjHt8tBYKg3nsmosjiUFVq3RVyAPPPz9+xFV2slEqO4KnJkpuOx9UKNM +BMP+Q2En2rW6o6gkQLC8e/WpdZYpQjJiBr8Mvf1atwA9HInhyaYWwTndoI7Ji8Qo +Ck9ZXesqEWxJ8w3P7LFdgqlTNBfVY5e/Fyx5iAo11YXdTgiKh7g/BNdWeOmw9ie+ +v5EH/hAwGdpbgJHDHi492wlRvbank/Rw9fTSL+/ul4bhNg2Hp5c2c/vc/b2uks3Z +Zzt+edmI7We6LcINiv30bMRqktyAbu4Xn2RJ/pBcbFhIuF9vwam+cg9VJA3zE1Po +WVmBeGVF8ZO7kJdEZ+S4jmqp+fiZ3iGYc7/X+SQgLFS3q16rm6lWQtAufft0Lffo +fkx2wDmLNRmxeyuQ9DzRsd7MzDIRva7DlydWCEqLKWZrvSxl8JTnbkivxHYL5JSh +tysrlC725LtT6RNOX1V67OoI7/Vo7fHb22WVpzebrAwMk8ulyrPD2yx8/R4J6c/V +nnwNfz/56nDkcoSKd5wnlBwYS7BDoU4qBxGYn/vrN6L8ZAXDDerKFB1w5+v/AgwA +63GGYQ0KZW5kc3RyZWFtDWVuZG9iag0xNyAwIG9iajw8L0xlbmd0aCA4MTg1L0Zp +bHRlci9GbGF0ZURlY29kZT4+c3RyZWFtDQpIiXxXS5IkuQrc9ynyBGkSQh/OUzZj +b9G1mfsvHrijkHK6bFZhDgRC/FWHvNuUl/Z31fqqDnXqS/W9FtBYAcbQ19evwGvN +IKhOZ7d3bT1gawaoMjcM8faeddx8G8q/Q7m+66zU3iHucnPx7Aa+rrRMb6hqKZ6E +2V/82V463qVYKm8BRUbaYmMGQVVeNG0F7P1YDkRhXCy5eev9K51yNKfL9smPQ2e/ +Ea3+2+VdsPu93QIbr/IefYpDN8+p//wFkfleVf5TZPqpYlvk2wn1PcfGvx/sV3BN +xE3sB8J8l1WvP/7Ebc4PPMYLIKLnlm5oPaBZC684YYWD17u66YAjkDQiI7PRh36b +UlsQIjMcVQXqZoDNPetwrJ7S6s51whzkj1EDhtMChkkJKW7tsJvHibAMwqXQlYZ7 +7AeVL4pL5E2cRdgWbJltiyeheq78BkEMP9RJ/ZJ+MP5foVyK5O9lwRrZxgn5lY6w ++UBeZc5x80cD9JQF7B3KSm+PoxAkU3pGGfFIzYPCmV+/LsJOAPdkJO8fhJND+csf +hJ2Gf+OKs887U5Nw1HolV/2BMN6taxK6IZuHxkXVo1EDLu2AVlCg1tEaZocHQqFX +ZkCpcENF9nUPmWxI8V4++EMISwecSmXp1e4JhlwvheJmaBaGTB+uBT5adYtPVqII +feT6YW2rVN/ZTxryrTNk3myk5++NnUxRaH6X5HtPCohsJfyCp2yNw/dm2iiO/PJm +a2xeo6V4HxCvlY7F6R0OfeLgBmtNcRBOl/kILuNt777Q12tjvO3dBn4yC6WGUeNw +WRy5vBcACWbMXGivobeDG003oABVm4lCdm7ZYE5WasPomkwGt0IKb+q502FWFwVf +DcPKejvQvb8sxZNg1DYUbo4hA+WKGO+O5GcjX80qLBmCTonJFVYX3ZBX7DHoDt86 +tBksN2+S0D1pifuz8OxBNkxxw9RuqFJTPAlN2JAmWs8HAeXgRVJpbYveGzU1ANUw +tLOWZu4AEfhAJhzpMmG65AYgM6+GeB3+6Pi7QffCaMcGMNLYWhaD0nGZhgVh6Hhd +ibTK3J44mfX71//uy03/fMMAEyWBGeqEAn+WRmevmtXI/BojA7tDUzn5K9nhtIAI +jXdWsw3pnKXr8N1Xyj1CarqSbSyLc+J+fnbPhOXegAz8uAk2heWzhd7LSnJClg4q +afl4qPOppBWLybxKyQnax1NLDjGKWC7LZ3D0lV1LjrcwuUV1F9MqPu7GVUxOUFlP +MTkUbo0opg2fYjoEozZlhqOYQvldS3E0I4RiCrtKe4rF4Vr1Kia/VQyJh+8TkhMJ +xbRi2vermpwgqz3VtBrqZlfTgVlNh5DF49aaLx2fBD9pV1NYv+ypJocy26mm7eUs +J4eztKdclkDJKacdssNXFgjKKQI+rmpasaj2p5oiW/gzyulkU5bTZ3qhnM7tspzC +nVcxxYm5ZTc6u1t7iinMY1tsj/VWn2JyWIs8xRKRnP0qJieMq9jCVUwTFBM8edWS +41rnU0wna0qm4HURVpNikVW3wvOF1WQRCxJ+UyJGhN85huFSVnNsgB1wKQpmFcbS +Q85g9PCFT+JG96I2E3omtZXSIISBA/zCNWHrFqwkwmL0o7mxoEK87gxrYGsNgVwT +K2sWjMIzgLzFxMIpI83qWFYGas/z1jiauMw4YTS4cRSahXnuVuoFYynZ4iDEG43q +MGwUi3xAVIv3kAxS3wNbYrGHCw0OLnTwFREGKdozJKLRfR+CG+zxjOuZ66g/EHzh +spoEW1zYIoxWuJBNZF9AWZMeQBidoNzvxPMp4KjgS+yWDldZG4Z45OE6/IoajNVV +gKyjp0++JE1wc6xz7gCHjXtsgS0Jw70zxRuHb6h5/vaCjsV0a482PNOWMdikeTgs +YweH4Xb6e9xL9OJqpIWjiLPDVvJNulK6Kv5edEIp0GxhR3gcVhV2YEKG7Pe/Y8i+ +QhGvVY/I96+jYY6+Q4asGpbHjdwDMoADO8Uo2zjuID2v0rjpdfgsbrY2pPSsH3wT +tNzI+3BamcKuKDu+gh5sg3xhiKrpDXU94iTMkXzBFLcYydCOfIgtncZgf/I21haM +mewCKmm71Q0p3nKbT74sqNPCqNWV/XbHOF5QWzsKAV6PIH7EAMVmDYl0XmabsPC8 +8riYp9pYPxA8wXUTlDuNxUCzCBLfdkRzcdGtmFAWXiUB3nSvTjQnaQLIyg4U0v2d +lQbuYFb4v0T04zJqdp+wcY3ZwO6dfo65vqGHhYN1E56HTJxt+gMhbJCb4GakrTzA +dQG2yVdc5mhnRcXDa/GmDJVOXrw8iH5Z+aqb6bedFnacGmG1FFfGWZT8xqBUdLCM +Uew0NcUl30l5t4/Ax6tNimdiee4fAI9WD5dPxH/+gojvQhXDoEYz986wXNAXAL94 +Snha5JD2dP8GoSsfr56CAZVOifdkwMbVPN6oXxAX7nrFiyhg5WoXj4wL6rAUT0LM +Q/zNMSmFsMcV/EWCt24QwuRIZonTJ3crzM1Ao69nqEpZ7AvJdSUVqqOVBByx5C3k +HKVtQDqyQYrxneCGDbuhzgHxhyAtxdU4sKjdsAYtaal8dXpNBtiz6s6VgL3Z9jnF +Y/WM5JipjUtZhRP9rMmFN33uTzBDtkToAoq7Cw1u3ZBd4yFEZVNdPuAUfy8MEKyU +lsZovkl6Bb9wH5hl0ekyNgzxmMpy+AN95/k7alxu7X4pbef06Lo07aCOdh7CGytC +FHsYy7sxWWqszuGWmaob98k4C0dzcMtiZl6J/uUjDiSFH/2kzP3J4e01hF+W5Mzj +1YwjcFTZV6d4DOaArfNlMCc9VeqGFEfH2Py1nx3amRTS+xmZiAOn2hzCpOHuah7M +B0aUJcWToJ18FbTPeElA+8hnQU3xWjOpaPsyvhJK2s4Vs5Rte+NU3PxYxfA3fWvW +d+PNukUKjsYkSD9rBuZyOyOx0Daj3Lw6v2/CWLx864PVSSjc8Mx2zpYsZlQnqxht +JRuFrA3zNqY3Hz3s/F36vLX79Yqd0+Nyk7a1G+rQ5/JzD0H8XbHMlsYCKoMddLU0 +RiqaxxRjFjESo7PlDXIDZhZ1OXy/uU7+zThX/r2yb4XneHry068i9fUvv39hIPgG +wg4fvej7ECa6kFRfWHrl62QCNkWbrR1p64SYFE4Y0Rmrr1Nt7KbsqLfzDvLUdW59 +uIKm8Pzqbw6RW3Xjg3Ef3XxF02NYQhpOcRA6+tnzt8/tcZQrvE1TcHgszOv1WJZt +bRv+tL19r83PW++/0ytHe3qNh2+XpmWfLkdB1Fz9nyhsQseyFHAWYQ0xCotXe4yr +uVA3GC+adz1XAaLwkv5w/d79/CncOh7F/tihS3ms+0wvoxKeCDyEeHo+f0efnEd5 +1NHY2YBUKpW5Ars8/OvOnLqvKKoXt8j5s6BQL8UFr9fn4Me5NOvD1ygCz5VVPtyf +hJNrdheBb2s670x1/91FoJhRO82jD/arCNhF/893lSRp0uqwq/QJOhKMGc7ztlX3 +3z5LMglfdcW/ypAZ0niU92o0KBUrHQWFsPtqlJN5fo1y4kexhMcFKdhJkKczCfLy +E6b58x3GqdkO8lT8JEG+a6/nq/fptMq5Pa2WP0+TnvS8TK4kiPuq315IwY439Eo7 +0dhEPd5gDe32FGnUPpkH4zzf8maBs4Ce1dRUR4NSPOu6GZNMPz/uyvatVsLjg1eg +eMvTOxrz8hOt+fMdzalZRvobPJkHeNXJknhyvY7CInZfDYtdGfgaWIp92FuJEN3m ++c8Boj6R4/VMpBB08ZoCEhVwjS5Thtdr+WsiMVQ/YJdHDcw3YP6uB18mbIKDza9G +hRjJS6K1BqxNFG8tQnM2P+81t6s3opAAedWg6+uG8IZ2p8ARvjjMxBrUO361RAr2 +1YwmkDinppnhJWgWnzU30jP9KfeyFfITTH+AxbLwSZNnl5dVZEIG9qA3Ab2zAngv +ud31uxb184uCMVgcHW6mC9RXu4xchiajVlI7k2AW47qPXZ+AxuinWsEQ5a1lsJlm +rhFsEnA2wmbkQjV6cMkeDk1MJSFestoNre3tVR5bo3KdhulUCLeTSWmGlC59aSDI +aCAqU4E21O49jWqbj0PwdQSK3K+fofzF6A52bjwElvV9C8YsPNRE0OKDB4ZcA41T +iQj0yV6LyIISoR1LYsNm3wNDa0SuipS28LBle1cj5FfVTUR18J4+5cIYdjqXZytc +d2daYWK6oKlqHEFH2cDpRjZBn3Z5OP5YtiZrye5DanfNAmCUgWrfKF+4xrVaqqgN +CmnAx5ihVTeHfVybq6w3xY1ZV2FMzXP5SLyOaYN0gO3Bqf1XQXFlQpTLPn4I0ACj +JqySAq8fgq5ZQgLaZopgyzYTMQNzy5KDOdw0jALWwaY2+YIXmllupwAP7X90uioe +5IchHlf6et3KEJ1dujxtvv0Wqnu5+jECavq9XtFZApp88dQM7239KQ5L1SOYbKl7 +C16Rz2ZQV5jCdn6AU7EyRC0PLdQNLMri6v/VL+ztyplRR5AZZXFYky4yKlCd882o +gG3YlVEh6D53SpkzATJpLOxV7aRUYLN6rfpYO6UCjd6ulArBautNKYvQUo4wpQ7M +lDoCphRO284o3P30K6MMQVXejApYMu71xOHlZBReOP1a7aW8GWVv+CqjYJ8kH1X2 +ME2LzCgYs40rpbCu8SwzKJ7h4pv/CDKDDPTEfwj0uswgmL38ELi/GQXTqH4go+L+ +uq6ECtwffxPK1PR2Qh2YCbUFmVA4/aaTvcmW6USXzjedoNY1w0Ftm1c6IZiGnXUn +5d7pBMuXK5tgytp2NiGI9zgJ+BH1zCabjLTDbrcgqXzAWQ/RtxUXtmsQCAErWQ4K +AWdpuy+3GEBLPV078HzWuxrDkyYtHg3Y1rquDsFY/v464Gr1VWzDl99uQTL5fVo0 +f1/+TgH75zklbM3SyFvx44R813ZCvjpPb6u8t2+r6efbpKnZp8k5Y+C+p95eSEGS +ecD5Mv1A3eYZBALnUzknhPJP3TPEfsmeMAJzWnlXZx33yaXKP7cHSC/zv7BZPUpt +eHsgBSTy+3TS/H35Owbsn+eYsDXTDHFCR4WNj9pYL57vURikXTenufLHx7hU69PW +TIJWY7ipVzc4guwGIVjPy6/C78+hV4Hq8KsZhKD13QsC9EOgkJca8NQLUBVeugVk +s+xegE+/6RXU8UOv+D306sDsBUfAXoDT9tIrXP7MqxmEoE57mwFCu7/lPhAC5G0G +eOEc12orh14FNDu9oL1smr0gYPHDrmBKv1oBlu1mVy0L2i+CrPxISp8/BM9NpmD1 +D7oFQ/thVzDNeNkVDCqimc0AifscdhXQu7/N4MBsBluQzYCnD7vC7eNmV/Tq2w2g +2Sn/1LxddQjRNPxeZlnLZgDLF7u6AaxZX26FIK6HW31GvRIhepUsZ/GCbwqeyi7Z +eANyiKbpjEAwRa5Ok/e6SMJgshIaU76a37Dl01/BHE3XacirD2Dc4qwPrcuwkdAZ +G7Vwva/TTeYOnOw1GDvbtTqXGAZ7zfqr6bG5MjZqQlPtWZ3Lzvox+7pQGz03C1fL +vaZ+mTc3U2RVy90mQa/aTmoaBuWTV1iqbINr+86WkdfJ/oUWjJ85c3ilwaPysTEM +5laUxdbUaOYNa9m6DNWiMfO2yTrW8nQVEy3ZK2bUgKXoKbQhmeqOvTDWcxMV+MNO +j4SBmp3ToKLtvh0W8fN3WCRH2guG7nWHVgoaXZSsKV5uCp3euwwzcvvQAFjY6MD2 +ZNZZ//yIc7ZgiJAcTl7wfQScLHjEVPRH1+sQxCjO0aby9dreXNunnmssp9Gppm0o +25rbve56fWMtjzg1lf60bbhCbXv0yvVs1KvbgXB03dslaJ7rKGDRCVrX7Y1PHavs +XFkZVkoWE0F5HulWVD0B9dQpJq31wWrA07ItEgW3z20ZK1zvprRNQzd65sPuckWo +y4xCdn5fuE+9fcI2yE5B+GH8XWsHrWcqm95qrpKSRaJslC/ZFUdOY/naZ2PosHnf +HS+RJvr1YKvlKbth6+19eFaK3L5IUx5TBLlmpDl3eesqDaPKMFNe6Jl9j1YBM4LU +Z/d6jmCj6t1JMmdWLZjNVJq1rpfUWv58WpydwMO5mFpqNNUGJ7hjpomWEWEIgFbP +1j4JHzUW5G78zKP3PYxPw88C9tXEK+eBEQHDcjsF8c+IS54Wn0BHxOVO3l9UrPFz +aTvgNCjW5WJUaweBaacPBPYxE38RF6fLYT2cnvQJqi2uMhklf9QU+F3/MWn1eD8Q +YTxzNwWh/NL2VpjRlko+asSKPPz34eBWfVLrBxS+qnZ6ZwVP6//vpzu+6KHBso64 +EGk9ggL6CNj48Ipe7QhG9kdTM/AMZiR747p2N511JUKrubdrb66Ose6TqPAfNy9x +9f3ntflZvaFoxysYnN/e07j1un3scrL/Hh9d3rM4gLttxQW1W6PAXs4EHpdJ7qur +du9f06BHscvgrFQQ9SRcfftAIfRocOBr/sU68PXzBjl2st5cl6bgXBJd3/svgs5K +JIF42kSpCbQEFzzQnxjMkqXRYSGwhyUVJQawTWUcCnbAPjbC7hLn/CyXOJy3LUIv +F/8JPIormQeXZ2EjNuTFhtE75K8teA0UgtLLLwIk0LwFShHCp4lBSfslNm9r5mMn +4ifqQm1cH5pAHLUJb+9zw22bcq/n+OIuUz2Vz2nsELA0UXWaPccKw9OPk4rmii04 +wfDheQZDx8RJUrViiPimwAcjaGJkDNi64FMIrYmDDRaOEGDsggCtP2BR2Rmofh1T +Xt8Q22PQW36vw02cWgth0Tiy1sjtT5I6mj4GDBcHYhy9cKkhbkEYL3jKFwWcjX4T +DHnbRONBp1YKyLN+EWQywGRefxGcv+SRfwTRaordguBbje/unZOSd1llsNC32dIK +5JoheLR7VVG73tLkY8N0STbHXDfNh4b5EHBf5rm9Lc2PYBHw/2waJ+3PGx8hrSu3 +XwGjGIpCvFzUmwUlBM8Y24+1q97By8sIexWsUjiJSzwT1LYPddCA0KeTOhBgL/LM +3sXI68lV5k40QRmuPoo2pwmofON6VctcYK0bYppZuT0FS7eZgsnQu3H5LoGeu5cy +da1CPY1xVJg2UZ9EzQD1xiQVWh+aJ+IwNUcLECMZUmWCI3B+wDJIJsewBf5yYNPs +cgRWSUKg29N/CFBTEYBSFiyNdL8TVtrF123jCIGit8iIhjkK/kAPE9TL6K69Hs78 +P9vVjh1bDgLzWYVX8I6EPqD1vDNZdzL7DwaqkK7admSDaF3EpyiU5YarfWBmufWZ +5ias9cCwKJWB27E6PKVk3AU+a+sVQ/F5nC5CViyGSgWb2hUV0Q30gA8NECsy0sNM +7E4Nme2oPCa8E48jkXLjtf835T5vk2DPuEohIpOoRqAHh8XIeh3J8ViB10vAi+dC +8T7DeSsetPHRO9YvCv8ouJuoD40OF6cXVYjplPlUDrFl3TLdrugLtYr55uIcZx6H +aGLXeHbF6vM592md4zvYsovV6N1AD7oC25krCs17Ry6WUAR7nmDkNBdWfvx5QVGZ +y1Z5f6mdERZ4tzjuYyLSO6sckVF+Lmqee0Lwtlq3mG/nkrPPhRO1V94e3RrOzJbm +hb+vNfK3+PVIo36dRAzsq3+fzDzc4iO7AE8Nym1Xwrfi5NcVVscvisPGXLEyikHH +1Mdof+iYi630i465IoKz6ZiL4e9JaPue8M6pvM879s/Nx1ycc12EzBWm/RAyF5fO +Q8i2eAjZVpwYuQKs+YfiELKtSELmIihuErJwlwwxCZkrSrNDyOJ982nweL3chAzR +uY9b9v9grKrNh4+5XMbDyCIRbRxG9iQqGdlWPG/7yD4LQgGFjniF9RAbBmbcisnh +YsvpOSFVzb2PDjEtmKXhvqdFD91ySS8y5mIvH6eVU9sKxVJlfzfMY5UFlBd82lGL +c0HslgbHwqPQsXBuxPngari8YbutbaUvEl8b2eaTK8JAreEZHeaDlCiCJDgP/SsV +GHA25esWraaIRrW1zu85Uavl+eKAnR8iF7etiAjRfBhLjN/qhix3zqxIkbEoFjMm +HD+j8ceJISHy6ZZssTFlkwjZ12IkeiLczpoQvxt8nbsCBUmMAMCZYm1njQoVllyG +fqINcB0Bd838HKfNyctsi/Ce9TQPi1EiyMNyIjBV7/OgeBgGkQWf9YRyI1N1hRQj +7TGcN47epvqInoVE31QETVY85Sg6gM7FSqai/FxrSVzO71mDAtwwTOkQ07tO8xBp +3rXf56OiL30FpMjlQ7JG7M9Mno32sU1GW/5aSYu6WJqnorSaj5kDs2EBV/3+xj4f +dLbXxuYcO3ZJhAeDVefVyx7q9tHrzjDZzTxX8tOW3amg3nF7rbvgF9e+SfPcJE9/ +HIj6G5xs92NkYqNWNmjPsvOaQMOJbVzCGKisYQXvLllG3godt62glFGTDHXQpEuc +JHCPYhSej0zE4q8Dya8yiHIbzCu/TYIqlagHhkYxAfU6jlfwOMMo61QUrXvuXpWB +Ck6Jity4cqIEvmeVOwfXqTcUtQDALVrInOAsikGfXDRjGvlBV+jkRInMuDibnTS7 +OJJvsAxcUVlVLAMnKMo0g40tBwnZe1s8Zzk9g8MVHbMck/C6DuJwxJX9nIqz93oZ +rv6rrCgKd6dEmXk1OshQUUlNfiiS/ER81k/5fGL/4IciVp92K7ycGGBuI5Pxkslt +xXZ4E0N6oTHrPtZKBLv3LWYyHgWS1Suzb0zlvgst7K5jVrsi5ki8hGtY1/Z1KiMm +6krzq1ReWT0r0u3d7jPifSvq4JUrIMyxokf1iNvNEBuL2xU18DqQsONcCs67QAJz +h0TjXu/TUfr908jpx9Wz2/3paf1x7IjheJqTZK68Hb9Wot++3WHKti+DnFZ5OTxT +NPZxnCKt8a7nvHJK6RWU5+4M2v52hnR79hFygF+o7DDW96N4ilQImD8Uhwh+3sHk +tj8DqNh9bXtf8pLww3lrbrESje9YzD6YA/PFYvBgtgaLcEmVHD+mhTkMzy2FsX+E +Du9jLRQD9CxYEp8jE+bOg8hYgt8Y2S52uAmxW6MnI63nyKWL5krHJeDZgg3g14EE +NF+jk7oZztfC4JjoiiNGVNKcCkQl4XvkxYBqte21ciIHpsGtbGbTx22Hcg7o/awY +cnSb8xdPjiRwGme0PWJJ7FFhA9TvwDDD/aBwZmMfZ6oa2OlJZQdt/PvPSfUGKdZB +oK59fVYJy9HRejWqFgrnKCpa0S9eGEqIb9/TciaIuXMF8zJ4Z4hrMExt8SlcAhoJ +oitG0/tcjSDXjYFaeftOV12LH2c6gy3BtX6LnfvVo1CmFEy1lJVXY5hIVmV8GhXf +0ajhGCfDePyGxFfiWXmab94/zZg8V2fI9qdPQHV8fYs3p7pjx9Pe70fxAIBtAPum +8Kq0lorJxrBYC1xkVy3gpzGu0ZIsqeWty/tiNLvYhB3cKsTNy7OkcvDn+SqA5c7l +McQ2+Gne7ooovXAu9loXJ3eK5tvuET0hxNutOOjmilLsF4U3uf+5FB5hTOXl7wRm +iNJ/Yy7bsvQ/1plIn9B8DCRsoKYX6FeKGZ1u93kpXMHGZGi1c+OCtVPzvXINHE/h +WhKPfxJVp6U5FA+Wf2QfWL4aQkXG/IZclWji7rtUkty7O0u4sQXlgz+uUFqrw0SI +Yzwcz8VmdqGLK8oaz3n16yavo3g+HMbVu4SO4dsVqxS2mlsaommdCqwN8ePk6QGR +cbUAiyqxwRUrxFYH/IxHeJfA1h/RYDvItCJCkK2j/CGjq2zKF8WcCjVFlFxA5v45 +WN6qluf4GhaWS+SM3wpPotJcZg7TLyaH9VA0rYuyIBZ9Xyy2xnToFhj9Ue0+bRyV +PbA9gtAGS6vtEHWUYuuMv7JQJXM3Bvqu2DZPhXKtQszhjGUKFm5bM79WQV1K25XU +hDhQspLIsoC+8RJysNp3TsSp6jlv4NRx3WAOlI1gc5sb6f/MoC7Gqak+oudgtjTP +2TAc5163IorMFyvv0kb6XClyeco9LhSsvli+IO61juaV5iHSvE65z6VwdZspcq5J +VkhAPsk72iY4PcX8NblDJ1k4cmlA8zDv3KQAqMEquGcNulpLZUuOHQojIRgMlY2r +g/2vjLvDnRDcp6NQRE82H0PEklp3qfPuoEMQeycCZGccYAJ9yEbkjvq+GrNnxQU8 +oa3FsoYmGrMSuxrZWMkScuqNHC5RFiSjrEtucTZN61TE/IUIrtAWfyyqdwH4x8jw +hPjSjfmvhMXJ5Uvqtg74vM5NeZ4oupL+6S7msrjLVcZJdjFuQNlBwshfAz2VoO8D +0BcWz7gXu9fLf//CZJJzCEb1GwqALxUvWrSQFS4q+EcPcBoQe5LOnAu+z1RjTXac +KwmrIVwpeuISmVLhYBJTD78W8nReXrAgiKQxKb9GTU0OSF/uECtHrwAhH5usXj/2 +yUn5hd+2AVlm+tUAK3MZxKAJ2DWYGLfjtJ4l/Iqv0M1+if4M3eaNhBP7QfxamCeB +OVJgul+sDGC3DKAhvIXPuNIRSWwl4LVC5bXzfhTurNf8yxWxibZfFB6DVVOBVUAx +00PEGhKQKy42kvt4fSQxFJXhEE9OiK3jXJzPh0hWJ2D3rcRu8JxGZxCQTSBOQyB1 +bOtFVuvPbsUrVBhYeJJiBFZhnYqO/Wv/uGNOn7sDNTXvxre9GvPbRDSV9HpLfCP4 +5DmtRskTjIi0pE8Ga/lDPmMD4VOiFbwQ1mwgf5pCZr5e3xP4cjw7NhN9//7nukTn +2BnLFWjlJxeeOnsmkOvSLNu/tc4eCf85KAbC5q97RD6+j/t4jrEhL8QYAVHMZYcK +C2wAhzKsAjytq99iN9n1QIVO3o5fD3jI2yep6sjb5zASd8N5J63vwl+LHDFrs677 +vLBRe5kIzKrkxZlmjxQjzdujERj4zOSVB7ab+9kOZr4fhRPwyXz+L3MERwEgwADp +LnxbDQplbmRzdHJlYW0NZW5kb2JqDTE4IDAgb2JqPDwvVHlwZS9Gb250L0VuY29k +aW5nL1dpbkFuc2lFbmNvZGluZy9CYXNlRm9udC9IZWx2ZXRpY2EtQm9sZC9GaXJz +dENoYXIgMzIvTGFzdENoYXIgMTIwL1N1YnR5cGUvVHlwZTEvRm9udERlc2NyaXB0 +b3IgMjUgMCBSL1dpZHRoc1syNzggMzMzIDQ3NCA1NTYgNTU2IDg4OSA3MjIgMjM4 +IDMzMyAzMzMgMzg5IDU4NCAyNzggMzMzIDI3OCAyNzggNTU2IDU1NiA1NTYgNTU2 +IDU1NiA1NTYgNTU2IDU1NiA1NTYgNTU2IDMzMyAzMzMgNTg0IDU4NCA1ODQgNjEx +IDk3NSA3MjIgNzIyIDcyMiA3MjIgNjY3IDYxMSA3NzggNzIyIDI3OCA1NTYgNzIy +IDYxMSA4MzMgNzIyIDc3OCA2NjcgNzc4IDcyMiA2NjcgNjExIDcyMiA2NjcgOTQ0 +IDY2NyA2NjcgNjExIDMzMyAyNzggMzMzIDU4NCA1NTYgMzMzIDU1NiA2MTEgNTU2 +IDYxMSA1NTYgMzMzIDYxMSA2MTEgMjc4IDI3OCA1NTYgMjc4IDg4OSA2MTEgNjEx +IDYxMSA2MTEgMzg5IDU1NiAzMzMgNjExIDU1NiA3NzggNTU2XT4+DWVuZG9iag0x +OSAwIG9iajw8L1R5cGUvRm9udC9FbmNvZGluZy9XaW5BbnNpRW5jb2RpbmcvQmFz +ZUZvbnQvSGVsdmV0aWNhL0ZpcnN0Q2hhciAzMi9MYXN0Q2hhciAxMTgvU3VidHlw +ZS9UeXBlMS9Gb250RGVzY3JpcHRvciAyNiAwIFIvV2lkdGhzWzI3OCAyNzggMzU1 +IDU1NiA1NTYgODg5IDY2NyAxOTEgMzMzIDMzMyAzODkgNTg0IDI3OCAzMzMgMjc4 +IDI3OCA1NTYgNTU2IDU1NiA1NTYgNTU2IDU1NiA1NTYgNTU2IDU1NiA1NTYgMjc4 +IDI3OCA1ODQgNTg0IDU4NCA1NTYgMTAxNSA2NjcgNjY3IDcyMiA3MjIgNjY3IDYx +MSA3NzggNzIyIDI3OCA1MDAgNjY3IDU1NiA4MzMgNzIyIDc3OCA2NjcgNzc4IDcy +MiA2NjcgNjExIDcyMiA2NjcgOTQ0IDY2NyA2NjcgNjExIDI3OCAyNzggMjc4IDQ2 +OSA1NTYgMzMzIDU1NiA1NTYgNTAwIDU1NiA1NTYgMjc4IDU1NiA1NTYgMjIyIDIy +MiA1MDAgMjIyIDgzMyA1NTYgNTU2IDU1NiA1NTYgMzMzIDUwMCAyNzggNTU2IDUw +MF0+Pg1lbmRvYmoNMjAgMCBvYmpbL1NlcGFyYXRpb24vQWNyb0Zha2VHcmF5IDEx +IDAgUiAyNCAwIFJdDWVuZG9iag0yMSAwIG9iajw8L0xlbmd0aCA4NDEwL0ZpbHRl +ci9GbGF0ZURlY29kZT4+c3RyZWFtDQpIiWxXS7IkKQ7c9yneCdIAid952qZtFq82 +c//FyN0FEa+qV5kuFIA+SHIr88vHZ3r9+v7LyvzUMSDY1b8AzfaXz08R6g2g1vH1 +N5XH5mrdWt7YbH6atYDrU90OhPr69Nre63MPfi20t+HgtbX5/tS1IRjTvgDbWoDm ++4H9s9tMdQqMm37ztBZLfwr6Z672FoxPyevqgBr6hJXHlZWXL8b7eV2yNRwEOGX6 +7HZguiZ2fa376IRtP24d/JF6q/y+udZ5dq0TgGZGfKj4tijE9rL5n7+s9k8pcVDc +Pg769Qhi57LxUQ0Py6+/CcIvldueT/4Q3IOrf/6Ej3r7jDr/RfAcyQ9+g7ry9+82 +yKxIhsHg+DKatT/WuEfv4bCAPuk/HAQ4FL3RO9wWgun8fuzO9TUZvYksD7gVesBQ +b+Vju9/1gNO4XZ+dcG+Zo1Rt9dMGTy9hMKBt2j9Gf8PmLdUp8M/iKkIewG1o7+6A +1XdeZYSF4ewVRwC2yF74vsmO5olkpvV1V+NJSXetSshDWzHqrsjkAUFvnastjAu4 +e31gD4e3VKcgktK57K660fXx3HzZ1kZepBpLAV4z44FSEF8V+dMQnvVZ/di4F7D1 +iQyAlZP6ZWR8jJViDkVvLJYdH8dsV6WpOqwNJviKB/tb6vz913+pX9yZoC2TqURF +QsAymUozwqINENaAvXseV3D9EBRdpxSWIj9fJ6zzqEtQ44cQyRbRz8sq/OOG0Ivy +Ycn0sDGQaesVNgjJb7XVs0qvctWHshT3wc43qWFPCGZm0piM/x4PiiQtNyYSeNR8 +QjnIc2ugVVO1bdXiMblaCx9X8XxrCgdgvsWy3ut96O3IO95ce1lqn0JfM5h0/Wrp +yxvIv1EomoX72qlN8W4Lij1sjKz+33+ogncLnyL0F0Qis1Y15Inchsxu43ipz024 +ej4ahSA6o/HZ+MRrjjpe+Hy9G6ENO1DqQ4/urK86tZ3gHlxF4YA6mhC/XwVBQoui ++t77gU9huQLfTeuMGRIJeyup9Z4DT1dn6Vr2xRJXTLA2RaXPVE8Bnt43BW7rCGjY +MhUEwaV+t0vNqxUfGiTyapajgr2hz3EskWAg7PgaVRG7Tu0OO0NarjGNDcO03AvL +C3KZMWnrwGOLv9elXVdGWHvXrquEScPVjZzrHTNPlqaAtll69lypTsHCu1U+9cG7 +jT2pP3X2zPRCbQbcdvOJl11WFYfds1IynfqrbIbVKptc1I2xVVNyeCNEOsuluAGs +9sX1ZSzZ29obqsY9GPMPIN8R5hLupFzZ9SjjdSD45+h3u4wTlSnuK9Vd7dJMew9j +L6lTX0898T3vVZbmiyH17SrHefGEbRx1CvBmlFp0Ury4pq97yf6a2q2wYy6Wucgs +VFRjiaLD/cJMnfljvRRBRmvIaQHnzmQYQx24jS/VFqor0bIMsXBZNOasg2pKISga +GzaGa4uP6ISNFA+01JPW5kkhGKySK84D6l3L0RcB2R8Fpc1h9qzHfLZVdFcjdM3F +TTU4BLUzvt21XlQLFm+WMHxVd6qngI0GX6uZY5zj7oNZi0Fa6tBD4uI5xt2q7ZPV +sERZq5yHV/QauYqpibosQgFn41uzoZvcSbIg5wNWTIFRVVBuH1j0+I4gwoQiGNvt +mX1uEY62VG5H3qWpHZOhGMLOVlBH2pFDcztmViYiJv1vBqGqVaABI0aTfQeVgJtZ +19DQMmRbU7rH7IGz62AEZ171lT0cckJkeqXLPBPKFsO62OngPFEYDL+EjSW3LHvD +mjlzBd6lbiroE/QN91mqg8VPoBiKktatoqqa1nX3A2WdFX+vVw2IZEDhuo1uMvkj +X67KCXBVZeDUXcZPaFc9BTXX+cjRnJSQeza1rpqXqY31qW89Jqv0ba/nbdUDpT7l +x7Oera+b/FREBPuu6ZiajdPlZxQu1k5l9CtqqgxZVS73CcHel/rYvPvRV9Ht5nhR +nxC4qCWpT8Bul+sADlmT1MfQal/r6zAh1vKALGiX+oRgt3Wpj8VApuJN6vPAnFCO +gNQH3+bACuqDvet+UR+cjTgk9YmrbkwsIjcwa+3X85oa2nMV3p2X+gS04S/yE4Ky +6iU/hrK6L/k58JKfIxD5iY+r2CV7eEDv80V+EDB0hSQ/ph56yA/MmONV0VCt6ov8 +QMHmnTAQIe1G8oPdNAol+UH8yzrkB7lhfsnPz9xRXRgYih/yA1ftcckPoGZIkh/C ++iI/EKgokvwQ2iU/Lyjy8whIfnB4t0N+cNlqL/KDIE6/5Ae5vPrhN8hcXw/7CTzX +eK1iAjvsx5be8GU/IWhjXfYDr/shPwku97mY1Idf9kt+sHP1h/3gYFVish/c6qE7 +em72Yj+I37T3emt+6Q8ckGxnnETdvR36g3C2cenPj1CyVniNfjAO/fn1CMKWSKZv +vsBR7V8FGLtC4E0GMkGFve4/sbiBBOyTfwjuGeeTPwS65/fvF/9Goh6Rtk1bOjO9 +d3ZQLzFbr/fBPz75pkeMQ+nLI9FI57xtGjVBFA0TE2BfS4nB6QT/5B7DdBq/RXMY +Ww3yWqkyJrMBeirnqPrGKbWrXurr4WolerzwuiarhTrEpOLjLn2/YRsj1VPA3sLu +T4dM3j0OW1zdZ3PO4VFIljZvYgQViZomJpSlc+33+ijj9mAO0GrodaR60+G7yo/F +9x36OFCzZfchx6QAVVBphEmOG/Y+Mw7MujG2jsuh0GseN6S+qnF9JZRpz1Qoy3vx +1+oSMcLkhCLOjZ0NFliDB28xFC8wK39D86Oc8cTIwG9zUkB3RJjZJFQqSWGGZrTM +hcnXWpeiVYoSLx0KLriP4PsRPI/kRy4zvXs5+YZZ5xcFVSMsTA9UzrAB2/dnZ3OQ +U3cQAPaW0hqX5yZs1t/QNU4/gjVduxlj0NDY4qymuuGqXSFwkdcZ4xbg6qfxWa8c +IW7IAs96F4NB9KaQbUJLLtuZqiHoW6mK+h2Qzz7uNfYb+hypnoJmUh+LsOTuPefi +ZqnuS05rUjeNSA6rO+jMPB5P9cIkaPNsxywwWQKmSN5WU3upIM+YpgFz0FgPAEGQ +qghBumSK1np+5yayoW6Ia7UuIlvpXvKHJKrwNtLkEtkIBkN51iON9n6+jrTZr80R ++NXu4QFHm7raC6K+nLRKAakN4G6XqyDpWjtUVuq7m1y2eXhVy2xgoz8znCMMRKuf +hv3rETgJAhNvdPV6GYdAchZox3ipe5f6prU2p1xlFyoze/mxPmW8YxwIuLYGjfRs +Y/Hm4U3p4k1XswciyO3ETQLvWi+aJtAosPsuyhaxB5zumU96Rd2m2k3effqBUi8a +VbVeWDn4tVw7TIFZxzE+NPyZ0iD97BmYl9sVCVxXT009+hEMBi/uv00vU3C5iNze +eb+RD9lkblfPwQnK2nlgmqOOpvWi8nW+DnO2v3eP+cDnc3rhC+bd7A0xlKT1ErT0 +jjoYKihdV0TM1v/Zro4jy24keKcVY8EEtLCH1+Ze1v+ITQH1uBN9+J0FPAAls2pX +t1E9Nqb5y09bc5fzLzmmW92OK/NdbmNNfdZ72I1jVSzYbUwX5vbrsWpSJf1YXY1f +LeoRHh4onnA2EeRV+cUDQDLcIQII+kMEgM1uExFcuIjgCkQEPC7nSwRlN5CbCNii +90sESI3aT60HH87+EAFbq/iszlouE6AopP4yAaj38gDUy/3ywIGbB47APNDc620e +aJ4oLw+03x76TANte9Q0gJTq46UB/BfKpQGeFi8P4JQwXx6A63O/PLBmxs0EB24u +WILNBmu+3WyAu3N72aApcE49Z0K3W+2ret+HDVC9XjaAv+63iJwZ37OzcvvcnX/X +NA4ZbHjI4ApEBoQzHzJg4KXykEFdfeImA4Sx51mTwSfMXYIg8sC6yWAJNhlAmdYv +GaC9cvneZFDcDWwyQGNpdV3tYarykgEn23e1pXm5oKptfLiAc9ylAs58+VLBgpcK +jsBUwI/LpQJ0sV8q4MK8VMAqPW6px8N7e6mAg0y663T5uFQAs3iQ2VRAs41LBcfK +ZbnlMbr9wOf2lwqOwFSA97s6uxjjJtfmXaxx0y3luCeMW+irZpKHCKDMrjh2Wn+J +AMrM9p4NB5Vx76b76iWCAzcRHEFatik1Xiage9PLBKxn+TIB3zZuqa+aBx4mwIg4 +47teR7tUwMSYLxVUB+SmgmNVU8HH6qYCzFxDMRqKHXEEHT8/EvTmhoiRADMVzx2c +S+pQxSrOEz5g/M5O967V6qmlFCHRmJC3zlnO6vydnCw1CZXVhQ3bYbrQF3ERYQ+K +wFDLC3Mca/sStDi0Xt1Uqh5OJzmCO5a1Oxb3lN3PHsllxCqWmBdaGpb2rKbuVaUh +7GFaSFvHGHRVSj45rLPCMqar/lLSMUj+arJ943xW/yiIqD8/uqDF/hXUpoBiIKwd +/SOYqiFLIMu6F+m2zXSjUposiRuTjcWLCUtSWzeowYU557VdAvgJQeOvVzzYD3NR +z+oN6FaxR/NTIsvMZhao/fIONHunFPihenNekdiTx92yjTkWq8nUfWhmcf37hL0y +oaHnMlOtTLiClQkQtJsJQB4LlAgtuaM7iQABc25lQuPYd2K9JWfxzgTgMetZzXsI +YCYAZTPmygQIaswnEwBbqCcTLlyZcAXKBMBiRpU3M9wznkyAIDyZgIexzqxYB8qx +3kygimU8q9HjhDKBBnFwp61jCONkAqw311lhGTO+qcD1N+7ZoNY/Clbc06ZxfgXl +kwjc0ecrgKGdjUoEGtZ9dbdpxjcRHLQnEVD2U7uJcOBOhCXYicCv80kEnp7Hkwjy +6s0E+mHOE+t4egvxyQUq9/Ro9EQ7qUBT0xMnFWjd0XcqMIyfVPjGvVNheLLjeVmp +0MxSXc/90Q6G20ewhsUMG1KAklvbs6OjdIXyFYwxn096VJv17Egeo64gwrtzP8xn +cEa5Alw7og5lV+WHzVVT1o6Pcj/Ut8PJYdob6go7nSyTIC3A7tYeBgObEbIToDWT +fAdBSX4VaJUwVl+A/O0MgLEQN7MZGmd1DUP44RwAWF0mOOVwd3ZSoRYgWwjd1896 +EVqCuDd76Ay5//Knnumqb1LiI7rdt/Dq6JEVvYPeVfJpYwB7HxtayRxum9M5imh7 +aDQRR4jhvsRvaUqEoqlIBh4qQpOl4sKc69q9BGna0ZwK/4hH/eAQu9+uTONACZCH +Mo0Nry1em2vvzHaI62eRSeGtg5aerr57WWFc3GZT7eb2J6/d0zzSghWLLuQaQG9c +jW2UJ85+2AhLEfXGI4p1GExWbSRVKT7IjlNDSWhLq3B2DgSecU4wxlCv2zk/KApq +XG4ddUMbp4X6rpf6+Trnz+mwdHxux+tL9dscQ48yKiUd8VJF2pwL/7mCU8n7EFX9 +vwA1B9RhQXGgZroAbYBHgygH4Ts2pF1N/9/aPVegygcT8e0WuFRBzYSG3I5Osaaz +PlClsrYH0vpgOyPzeaIDVl/ZHA6AY+mfHggXjrVbuHhsAmxNZYVdMWF26DFm/JTp +jobVmpAjxipU1GuVim6GkSHGu56rSyEbAcDgmrQqCcYZFxolJPqfmm5J+3hJE9r2 +AseVKcfBC2G15La7NNPAt33CUaqut4ljmHr2EfkHqHi3hhRDKx5Lu+voz4tbnmSX +lemXzLy2N29naMuD6u3jGA8qnogO5hBzPkXazn20Ob3tl8iITUnmhykUmZPn4YZW +s8/0rrdZXfTSsopOx5i3PRZcFKPNonLg/umGfRxt7X48QKIKv/D3u/z679//+Sup +b/+FvqJA7X/+SiT5X6A/BuYPIFq7XF4BDAQTrA+8A4784AIKx75SLIh4eH8F4fd8 +T/g8geWMn3Sohn4hh65XZTLpwjwiqdKDhONcdzwfUEk+tLf6arYEV5WuovUIOIXN +LSDsyFdAVq+EXzZT+ImBRY8CZjW2kZoJKzqgRu/UF3JW83YJMEIgdwThbcC9HT0F +YY5rNzvc5vqky4MsMKtfBinQEN1QwEawqcAQTRQzoDbXt+jJAOtcR7PcNrJl0TK5 +mLZDzOohqE2NHMTdbLhe9zI4y1cQMJs1kvKy6pQnYuFTUC9KsWOy4Hp4Hn4LJyit +l160LlBjE2A+LejNHGne9Wi9ZCIXW8G8toe6BNHr1rOGaeiQIVzb4yeyPxHkwOzq +LfnMOBVUXfPTEsgYa73ZEREVnUq2tPyUEEs8pno9Ta0PhNQDp6rtEeCY4YBMWTAn +R4H0R/6ndbqiL9nsWaFZm3XVsw0fXS34uQLr8fNvTZVRaJtZ9W9GbQFOyeRcxcqs +fxCcFGPnwPD4P8H9BKle8x8Exy/fd/hpKC4WDaTYP1dQRUU/ErBfo09pebZYSesV +BibMQ0VL6QTYijaTqwjHFGQdAMtF9JtG3IyU7fNZZZtINLtg01pR307MlOVFxcvD +3zbk5gNH2GcvwaTHxb5K7ACPE3ZFcNCApsuGIpxNDCFpCTCxnJGd48rzvLSszvTG +6GPz7YQYlYrQUrv+/G2LDgUnmkMts1wwoUO7ECHpbLqCmrPWK9ow1inMA/IGpgpV +ov2WFFVryWVyQBMP1WJFUyobWtMa2rvuul0D9UYS9qyz8tDuYQZiiZ1ab1mQGUM4 +o2I05rK2S8D0KoodKDGkSxpJHyRDR8fYuVTGdlpfqrcVEUIt2ClpHujdoZa7DvN3 +p1C3UxYKZUdm8mW5alnRBgVzf2FaXjgCdvP6OiphSSY6PMuHM/a1fRQl/Uh+eTBF +ibH49NxdvLaiHKQoyFYtVxe36OijXtC+7+icTDnsmvZCMuGlYi+UFW8pLy+0qeq/ +do9azHiGszl01UIl9efmEcXydHojE+q4sCr6vH0Jyj6uKv1DdQA1ki1L046I1MSC +rElycHcBcjlY7cs2YS17DeZn4AEYpabPalopwqkPeKVb0Hdy+6eecQRC5MFp/am+ +W3BK6cjm2ivATfF0c3AV3Fndz0RBd3bBYQvcmkiCMUA4nLGFWbIh2b2u7UvAtk5f +ux6wG9DhwcUkeDva7CqPVLQmhMWVjC0Un97dJhD+LV1SHVvwcwWHBb72EDvTAH5D +ht/+0aVh9C0gTM2cVqJg7orX6sTnoxSgLQ0t16IAZU910TSVb8FQM6Vvx3CNMGSt +YWKOfXaqLhK+OozNcX734dFH+8OBW2BFfv6tqjhwwLTjpectuPGA22v8CuYoN0Cm +SIswGioDGS9WGZPlUAAlNhkzaNxj4WQSAdagyjazXA7BcLVpYHhUPWSXII0PlJJo +PjglIRAFw9z/Y7tasiRZddhWagn8bGA9fd6sanL3P3iWZCLI6h5lChxg/JWHpEfy +YZj/wKgGs6d4LkBzfCxU4bqAVW13+kpVtkYQ1NhA7JOmqhiQsRrJ5iOly+7KxU07 +LGWxd5nFNiPRpqVZOjPXKgIFteQJ1Mfmgn9+u0lxi3pLd49B9nIWjA1cjpo5Q+VC +GF3EAAvUqZF1OEhIwAnPRYGp50HKpgrGv8HKRKz7vuGxbS5MNbjz9eQE+Ry+SKf1 +/uztHeY8quk1j+qCf36/lqG7W/j+rm1n4YnUHS8r+x8LT4rsHmlr/1hAH5r5STKY +Xwthh9buT/5aeKrOp6ZQPuaaKKt0VnEo30uLIYcBgdEAsCuaPDoUYBGdNUZbL/WI +o18CdvRgNJBeIm67n+YCaMPevZqqwC07otQVlWz9gcG8cO1e3NZDFvpLIHZ240gL +6UUqGVW7Q1hzAcKpCoJJIZz6TOms7g39MmBTM77+bz+i6hKTWR8nbfq++0g11HKy +ZCCUxNTifDyxlYcGwjSsIIcG0lb+se/TxAONcE8FUZd4E3GL0+MYQJ3WhhPton64 +e0pzAW6mrIhY5DgPchHC7brINZetVMs9W2d6dGnEKZKuGagk+XilqxTaeXRT63D5 +dQ+Kl+1pIwwLJGmd+6g7bGr+wtcBzwIrBb/uqpRTpzc6flnqUrfmoqkYy2lmdKle +mh2ol9ax7v2mKjW23q0qNKaltFXWXTQKmqlzrEIZYh4Y70J9zMQQ52scQXn9rP9Y +CApmgwuZi38tKDmVr4PsN1M4TsCcO0Br6td//6NE9A0NVugiP/eCxyeAe0zNRoFC +nUV/HKOEmGj1dOwHJRLN2G0QemsHQhwxVd/9xaR7vl6KLZwuceSGLi8IxYjuuaWa +3xCqS3ycCdO+nq8tqLzn6aw9u7ZUxpuY7ppf0k3kttmruyDE821nP1+Or/21zFAb +euyGy/vXY1VjRP6y+p9ojDhguYI1SOfPuyAiD7gVQW5p6KZ4W6ldY/ypmIgr4fZW +37cISnpZe/fR3+z9Oi3xHh75qRLKu5n7j15CUjtlRWZNJ/cqj9Wvx/2YG44wGiuY +dNvUc4mV1up3+ABK7b7t3m+131+X2e/T02Tn9sfCUu3D4Jg7ekVFoIs9euXPu3Ba +cVhB5PSvBQyCngtdKTKj2ACyNcXLiqCrnhktG3gN01gaOtVomCpw6D6ArfiBEG/x +eX/3I+PnFG3fhJ3Sg7Qd2AcbEmg74Fzs+LN+wDqPOBfCy0hOfK30gQkAwYQ5ph1d +1p6nCwMODXVMj3jYnPVKHxpi3vtNzHDILKW/xBF2n/PMoEBj82rb9vXLS0ye4wWj +a3/kBaUfKDVgdvUeAfU4JcI/31I5B2EBoQHlEyJ54aR+kF6OOfLZRqvV4VVOG12d +QX0mFmy9uQ8fqgutByAG13GwK0S1i9LEAD4nbwX4UaSJvJam4FlqCOXVWkhvnMq1 +s+2dwVHWSJPw4zI8xcFZsVDWlwyqocRkwZxZFynYLw+w+VSPTxgCc7O1VHDyJnaI +MzyC3RRAuCEsMp7ZAkca5Uj6nPtjXQFkJ4AyvtjW3u24tXLf+Nr4VfS61xTv4pCr +Lu73Qh8XPu+BPVPpWfCu48ahEoOQmjoLUSpTaZ7G241PgFvL5sNbunXI6863OHwv +oJqZVmq7n6Z3hEkV5tcN+IwXijW+C5uKBBwZMIJVxammcNEUUY9DVNoa49pOwLTH +P2OMe79l8RYqPEojLyyyTIPXSovZVCjLnC6DzJHSYzH4LKvq4PNRZk32V9zPjIxq +dlhiaka4q/S+OEnAvS5KEo8ebu+2MyDwMe6Zaoxx7dDJ8VjLucq43z1J2QVf8+dC +6BPW+L4Xepv6QG280ddv236+H5UlubE94FpVZN3GAUNQ0jb6vW9kQ63oW1dZQs+U +tGcVa5KeumlIsSx5K2VXzlf5Cu9LBXlJEd9vuZ6sslc1R0jYW+zTqCdTFZtXJof5 +Pvc9G4spPazosnqSZ1QNmejyqDo2VWT09VWE1Cwy/VBALOtS5mOtnIAQI+L+e21F +0GpiPAptkw/nqifcunjnUmVC0+FcUW84NDK9C73p9FGPz4m0WeeRPk6t483CSK8l +8Wn1QImvtu/9XfW1Cq6KWO2nlGAA5GUqNVWNrKXhLzuJIC310Wue6J/zRIseVado +QoNxz0IostEYsDA1Ks3wU289+hEvtVAFsGsKYywETOY6QJ8Crqn0ARphk6zSFB5y +/bNrg5/2PQnd1aBEdxpCQiPZ0P6y7NL7hqucw3MBoUGoUbOAUeAwteayR4q7SkF1 +nQ4ujYaA+gXNVeO7Onk8jAUPsY7jWlhrqDw6IfkIeK/M0sjxHOXDuI9QQj0s/sJg +22WneC6A9gBa3BKQDD4gWfcOmniUQQEByUW7gg/ApTbtCd0b8lFQTx1r3PvTJY6h +raERVB6m8I+F3smpu2/ugzkHbL4IF7wSymgGPAuL8f/NBTM+piHCGzgboSIkohG6 +B1zHbzPf7tpH6Ab0Ir9EHU4k6YL0OtudVAsfI0ECnrPKOOFZKD66cZ8hF0NPnzds +6YdnoZYU39SFFAunl61ZYKb4alV9TcG/5eSuAG1VrcnOQ5uzNXnXy7qmgQhDPlud +3uaJz104OvQtL6Duo5cPeWGozVV1VHjJGd8pvTqjdVXBPURRKgtiizQwTVqMZleK +xw9Y7YEaICWeC+MclyXOFEAm3jf3iYg2nm4D/5pKkvJK1f64Z/V+9sLa2VaE8FKc +0jJFtrGtKIFc/W4qGa+SxjrYYwQZGobC+KhyW3V2ZAbu6D5VDXIQjuq6TLdFeLv0 +HHhxZOISma2N0Es9EOILBP/Zngxl8uZK2Hqqt9OcW4NbYwyDjYi6jgv6E5TPAmJb +X79DI05vGipNQQmSwVo0ma8RrRe7Ck2t1otexUtLX+/+FhNO5hZwij/B839o2OKu +qSSeGrCJRzvo7QtLmynOhbe5RK7osX8vRFv9WJgl9dcoODDltsUSi6Yx0vJtqx2j +teN5GisrTRt+835gvnb6ux9FbbEZy5YoiWr1TxjMKr4HHoUgapkH9esJMtCNlq+9 +ou4bXIavYV6CFCkQ62DnsIjHby2Ayxz3hoacRBZvXJG5W/NAzfcuuQu9gJG4RfHL +8e48UOJd4me/rnl/Xfzj9DC2vZdPFgBqlvB9ykmz7fPQjZ93IUSd7u2NI9ffCxHF +nripqIZCAXqOFUMwQ9k39YsF+mPSCIBrK7aRd71Hc28HQjz65Nr3ftP+2oI6yo6w +qeCuFHa1dDTwQJkH7p7SY9JPvkm5Y6GTbsyVV2l64jugB9mECpQ01bCVehdW643G +Ea/adRx4Xt3vfcvibjKZRlsXbYJBKxfGlg3LFrEZ+riIfqpWHywPfv926Te9HAH+ +8P8far/VyI5PoxPsT7z6UBmQcZy94JLAwPaxEC1mjnPJ969LpcbirBrdeJuCLRc2 +xwLA2slMSOECttFFY9giYwFkDkQnbgIk0YicR94F9OEHSnyVee9v5Pb5eh/W1J2M +r2fHwO3GfXZ/JPeLpLqkcwFTz/Nx6OD2Hh7ZLOpwLl8a0I5q8SDGT6ouKPHkUGe/ +iyQNNJhjGJw+U5yGWyTgl51XaV+/zM4BDR+oXZ60z4XXnfGgOf+x8MbZxxl08IjA +1+QYA8YPsaZSA0MP9H+qq+BIYhiE/a+Kq+AGgo1JPdt/D2tJOLf7FDgMAVsg6Efq +yiAcrTq1pGzD1Td5yG+Rj0gd/lefGnZjaKjHbXvGLmnUizBuHge/4Pgee5oDzvb0 +VEnomAOTIk3NbcOyYDALka5QVM/r6NBLe+DahMLETAMaW+KGq8aBSvzO+vdfm8oF +8QYGhISeppgGZdJSWphwG+aif91EsAJ59mkaQFTsEs7nFFu7eqDkzBUtJgnV5mmB +VhMruU2pOuvgusSC+pWML//o4KgyChEizXlaZs24LOO9mNhgYg/0rK4qDZN6VN+6 +dqwOvSQ1q0Njc8NW4bosmYRXdOJVB75+ft8CDAALM4tvDQplbmRzdHJlYW0NZW5k +b2JqDTIyIDAgb2JqPDwvTGVuZ3RoIDg1NDcvRmlsdGVyL0ZsYXRlRGVjb2RlPj5z +dHJlYW0NCkiJbFdNsiapCp3XKu4KbogiynoqouMN6u5/2nAOZvp1vVkeIJV/cGj/ +lr6/bH73Mb5GwL4G4OyA2hVw+9fvX0mYqydh9AX+GhNwK6A3T6gmJQ6Cfpt+/fn1 +T1Dse8fJQdnTvn5AsA7CavaVcGxJOF0BpRtgCOWJ83sviKv6V0JwVTrAkINSNs4c +L1fDNB61JOD4dl1QpEwb3yYLtrYFvk54wvSCFtwjXoTV/Yt/h6fWt+jk6emYFUKr +dOnxFYRlG8rYtoTeFZr3LgfSzKX75YcbDMetSScpzlKhLqHEwmltd7A9YxqelX3D +1leJO2PuLhGW1GbS1r8Ja30SViv1FVDdAHdnGhztW4MrZU9aF2onhGdnRGAcWMbu +efPdkUV0ZbhckHK9UsCepHSaZwPWqMjXk2BxiR3xK+P+/PofrbF0yIxY/jCppoIg +A+YGIcN3ghsKRs4E3LgxcmI63N2lDJgMiOgAXxtyobcT3H1guUf3yw/N/Po7snZ9 +nB7Otuv2UH5v6lbwseU3SmxRm/xNYV4R0kOwLpNQEC7J+yPJJhN/pLH7W+hPiQzN ++3fIMR5oETuST95o7ajUeUdzx+n95u/N46DujpRC8GTROv9uE6d3KOPxG6O99g2H +jRIvgg6KN2PHcYq3BmUyW6mMK5S1RtuW43ZDvwrdDeLpcYpPsNemrrMhUXejtE6c +lcn+G37cDb5vTj/qRhbp2Dfc00schIjzcvAtLQnqWIBb0QJs9NKlMQ92px+Hz7cj +bOh4dYzNqjn88JOT3zrgYL/Z1Y88ihDHbSF/TYgv1JxH+4K4zVniIETnEEUOxQ8C +7dvg/7PanzivG05Tn7CxgfV+4sQKmZM5Mu3AyqEqmeJbx3Edl2XY4CrZJ+XUOytI +KlAoqLb2C0N3P64qwt51Omt9ClVj6U8/CSp5eBB6mSbwhFaCalsn6BQ3zoUxeNwe +wrkx6FjF6Y0JOlt8oTd68gOKsM+mH1+Y6U3xIohSGXf2ti0VRUDz4/bsaDnzMFwd +VfdMxI32dg3MjS5y8Tk+q+xReAGXVjp71lqcLYb8TZUxXpndVxNCX9pZxF/Rr8LQ +6Eo7RmR4LgRHODJSanPmX4Qo47g62qVEZichvWy3RIQwxulFCC0iz7PlNgMhbfw4 +NAIm//nFI/DUi9giAV4c8yWyJadOW6WWxVR8BS67sON4ix3FL0vjyugY75XOKF4E +iSyQy9Ig+LzV9o4Cvgjje0QJvZZ6bBjabwlFzX78ouav4oElL3lw7IKxRryWhhbi +lys+DKOpkea+XlPzTv3QO5wV+X8Roo5SidfUhYy6JJhYFyHaQx+3qY4B9khoa1HR ++vmLR3t8NY8aj7XxxRHktOw1Nco/Q/II3IalqbjD4C2NTvlzEzLjE84GZ1nUIuDG +AcuypyXBFvJsx/BPuBYu3Px5+0EpHAkh+nAjDmLvr4Lefh0tsZ/392qJ7Xy8ij0w +Fad4EbJJX3/38XG6zHN63d7Sha9q4ZlL8TbKyjKruGUzf318co5+fMarH49SsQ+P +/0YQoml0/DCiJn9AWBMp2TVNjzW8Dd6wAXVCXJrhwqwahjW6bkLR/Wgbr5W1L1t6 ++HNeXJ3GswSw1729hKU025Ru2hmDdcONcf0QYhWc5MsA7MbDB9pBrgaUVgfBhIcv +NDlUQRrRwdy7l5GeqY1JDRSNO3E6ClAB1y6Imhdf52cSuljx0f26jg+4/BFnd1wU +z21H+fRAdGJvy/dJmyVu2bCD4FQ9B06u6NlyA+Y7syDFhcqQHwES59902uoo4nVC +MHGZZQtKHw5oNumy3K7xvDzhUrTg1ummdDl6iliHPFpQN+NFjTNjWP3tgtmlrdJo +gK8Z00yyNQ6kGcvt5StGaR6Hco4c4twTo3gk/IZdDp/mwk+71rxht5IGjtQyWvIQ +jrijBC3bbkATtopzV+6byFOqZoYsnqVadcWEFM8N4+Y7W+Ak3GOdxkPxXEzyNnBr +0JqmJpn60LIz1geP7mWGD4j3zr83FZc8JBWlWVLFmpeysBVs1bd0FTvbW9jx1/aL +29zenpBUtpC2T4qzaVU9cDvoSLKPfvQ7HpdVfeknrQaFabf6qGrUCcLczPhshVnp +cGDkVTbI2ChUKtE2a7t35vQa4DfMhYIKnShehOXFRz/DrMuzbDH053CfFd0FXdpi +7IW12SszRE7v1HXz1RCBOaqU1ziJVKW+kRvsWVkD4NuqRnK8xM4eHukcFU7HFeEd +8tWy/iLkgtaLMNAtMidbLuyoYUHRGUdRPinY6KKpSK1HAr4viGv29Za/rQNTPHvB +B39swizlgLOW2Sklnk+WvM4pjjEVUAg3G4Ic6ZnjK1QWJv/C+xZ9dPC2gR6ezTNV +GdX3zu/oP8Zu2/JRBj8s5FhY1veBtHzNmz25CpUb1HDTtJIdaDYq5NbNvdHFCEA8 +XoaXtNQEdfaij6hij5JYfgdinwvgDwhtcOHACHeqm79swP1sEHmF43GTBBRgvHBs +PaUcm/boV6HHY87k4cYeiWswmvDqyzp2OiXfkFBL4YfNnV2ZtQdyPpc4Z09ox4Bt +Dra/8GLhhyoZ+nhGDTuEzNO/CE9+p6fup8pDeC6pP/6LZ24PH7hlVwgo9Ow2OlrY +99aW8lY3Dg7kdLx0uUObEVaBmz2R4E59+IuvC46piNt7GMWdHXqlEhl1ZQtYX1dS +bE61zyw5iTM7t8Oo25+XEOaJlX/yDZqnZOcJ6I1XAMbiSiSIeOBe+quBrbwRAzLg +mZvSS3zNffM5QizXiEDOVXFx0ZTY75i3MC+XQyqWI+aFzgX+ECamGU6btQ0eVVBm +yqmQqjqnhlNcBfvgzOIMOIjMSrpxxqze4ZWt4GdmwmdsLQxTYGOOSa4+6WP2g6F6 +wzlPnIqwcwrk3+hrOcrzpsktSuTRpN691Ht07EF4vAU00wMpvtVffri0syWnwukz +Z8vdu5w4jOWSo1863RG3VwjMwZWtJQ5CmBLif0BQXxwnDNIQWLYyLfI6ZVP1Udr5 +4rJa6bN7PwUPY9YDKT7ZlA+f6ygsTE9MMtvxVN8YCGtSGuuGcaS+cPrxVBEUtRWH +G1TrCEpo2nFYayfzXbFGeOWPG+E+uYydI9fPcizKDEts+nUhnWyUXzfgLNU7d8Jo +QsYwwMRIbe8vjNR/wlAEcwZ5KepGs3vkZaz5fBiULo2vMOHtTZ5NDhl0tf/0w7SL +2/lUbOXTtsZ5zzGVd71OlS1pNV6dN3x2H6wsEitPNtKOxvHzEt6GXT//H4L4LMJQ +1EBHZHP81NPAAa3xf6eO0U9qLcmxHHD7VSHRCWtHqQrKlrBvvnflUpKhifxgyvrC +SiTGaZ0V4OBbPf32gyIL+Kh5cEca2Bn0u19HK9YiauJ0Hzbvo1hFpvR+4pZm6cNN +mznwj0sIKyMyVRg25H44tAbHpnhngvSlJQ4Cg/Yv21WOZUmOw/w+RZ6gnjZqOU+9 +GavSmfsbQwBkKH51OpkflEKiuIJ//vbiH1D43OMv9JXvf16n7DrSbUEPkNTQoQe5 +lQ42RH3DVv7LJGBhRFeQdfYML+6EMlZSb66byr1/HbabitN90stbNKzKx4vlve/2 +hit6Rwim6FV+6/WdVTLOdm/u3D6CHW5FUFO32G/NBSM+18c63z1FhtIuaBZplyF6 +qdISVs2S++EGpZxzjrHlTnEAZyVN3mcpc84Swxor33lSXrECBqQK4Iub7TejbOco +GzG4fz1MctTlDLffoHJyL5jRDX7eb/SjyfSbGw/M1AlBZlZ8nWkXp9+0jNszbaVZ +5nTofXMez7rL/uLab8GAQfq7oMBgEtiXrKkCsQVp7cVO+/ua33mlMXla8XJsPwie +CeB67F8CuFD59ojA/r6vACUmP4oBaUmv0ava5ZTTa3TiHmq2IgE6tMOyFaUMeRlI +SAabbbyXY3hiSDo9F4+3k747i7tJjh1G/tTyggY9tTswrZwfe6ZinMrDvRkGa9Td +YJFanqqZhiIfegvplUX8XMvxaPDTdY2CDEyj0GhotEoNU6NlQ/l0AXOtNY1Y8px7 +pTlLWePlyhTk/OHOP+MH/LS//OAHQTj6CjgsAKKQwFCdEDpvTwQ8CRBR7bMYKCvg +4kRlINMNjIsQVtT2iRbhGbN0Ngm9T4Wwa6vqUJvs57fefxjaaC+OyCM9nyB1OECS +BLUbPPK1TjccMXrAo8NOHr70fVnavlERcJmjTqYG1LU7BP5Qnwv+8PNT9fKl41FR +AbdOW5PbLe4i03VYpYqMhJkBap9E2ttPey83o8WHEON1o1iEhTbvgeEyXrb/Wxcu +doTf/7wE6XzYcP4geOInvvgbZwD+V5bprDfDffn9FhgN63At9c5KWNUQliYvF7TB +Or8R/g67btj6eMxE2mya8mI1+3B8qovuydv6++bgNKlXQOgd2yUgfblft/4+vFoe +HncXtNurGIPtUVut7D4qVuPF+WlY5B4dFsubw56p18veqhMxkLkd/Lvvt6AVJbX3 +kynSjhRxWEXhaoEG3uiiQ2hUapqxyPiM65w0Qn2fcfb7cfZ8jVWMIXoNipzDqout +KgaHpjT/h9LQUB056jRMYxdGo04B2Oz50tdq1LBBnu76z9TFalWj/pJmTyOG3rZe +fRrPqvO9vsT8Bx4LozStFm1fv4qG21VpwiYSgL+O+ta9VmNz1/hofmd4oIlKMTeX +yo7fWXX36dFdd3pgi8aF/fm7hf3FHwG1t/Zz151QTubsavLANJ5k9YSNhrZ31EeH +wX0KanHC64FHgJjFx4MOqpjreDgj44RNwWbp3sVSNxnU7IFEoGtC4QB131jdYvQU +wiRrqxVbmLQqqBdDBRRJgfT1VwZkUpz2jEvfVzBj/PAqeQjZ4h0OsY4TWYgmFUwM +NxwxzwwW79ZrvoPpkELkei/OcwUrUe+qs2qELmCvWuKnDlfnXaSKDjlUQhf6oFfv +SlszgL/SYRWBMsS3wyYy20/s7hoBKkpSR8s2GALPvSj8ewVWtd41VMbuRvJU9ojN +MYVpscyory+tPdWbxSP5Ki9cU0awEWVthU2i7KVNqii8yqIbtNanrIS9n6IT7ojV +8NWQXdKX0GSHL4uGEjS3Gxlt2tdfgfIbdNj3TxQxiEiHr4Cpg/swxngWkMc4Y6li +kbOmduKNayhY+pKhWFn8LeoOgBE79ghgiFaCGBrhKHF6D0OxbvntW7FDVUZEmhAU +Dx8wFMgI8kuPFMQqTq4qUq3FbjSMHP+olyrN2FdvQenNd9312l5fh1We09NqcXva +NHT7tDlTuDvbqS/6G/hhKx0sa/0gmGSQEmi2qmgXvfMkZN04hM0iK+k2F5DGLs0K +Di36R9P2pRo/RH67K9baXffAbszpzrTzsN9LrWjGdhusIA3u7Egb3l6mvl5HI9La +sf0cKlMwcXV4W8cxTb0qT7WfIt09f7tsgVMdLikj3Tz5W3/r7lVwnbvuTpoyLaqW +wzU1drbYHSPXQbfqSwQFbam9YRN9SMEzpvj3o/wkwNA2PwUcPhx2Y2QZmi20FQ2d +YUu0vPG0PDwvIjMf31vCMM58Lw/FIQmewwaHobju2F1N1RbudUjCBbYgvx7l0FAX +S8F9nAeCeuBfgvvaR0AuAVh5g01FBkcWvNYiLs+pem1jIO76ek4Xob2v7c/7Yr2F +cUxRXft6P9ezQlVAz8UgtO9zn5TK54bgvu6dpRwJ3P8bjNb1aEpcj6v5CPDNYUfB +cDZWCOz0t+DjDB47vHotcQ+vL98UYBGFHKo6bJ11saMUOqxlZ2P7zTvIKJyzYMAC +rCQ2sG2iyQErdktgfG5xfiQfkLJ492pL2ckUgXKFGdbYp7w31palCdCsJ8T2puc9 +pSsFqqUSiBG+sAoEyoGjIaeSTjo0FW4TB4BgyG04BVAE4hNVsqVH4KYoWifDd0OR +uzcxRU+bVWN7HSzV+dbTTJywE86h5ijijMdv+q0crY9DN5UwDeddh7PF9tJ5eimy +9J6ML+y6aKkSXMFcOrxWDZ8zdGlcjeh1AZuqCziCVTcvU3U0bafPB1ibHtpEKzt3 +uV1WjIREORWp7MGFGKc8W6tctGFFbxufcETEPILaZWVq5lJkMx1MuEqezgbgrmoy +OrtREyNx2OyBsgtUzWWQraGPRwQzWrRnwohgX6RGbkxjMkx0dFet1jcErdL2EEzk +Br5GMnhdQt10uLcMU3boUobomcmjTcy+TbnUNutnr6n6ED/qDK+S7EmkzmF4ODY3 +HuoOH1EDxKTmC8EDsbnHyIDf08RlLSxELc7oqUVRmFZVE1MSlFDqVYtIGnEgaQOT +5/st6K3GO+jveDQpiVs4bYTe5oISmh857ByTe0mSCCOn7GN96Sq2KsTxnDrdMng6 +q2k9KhdNDuvlA46R1SkErGn4miyBXBKHs+2TsEROdbmfXd/ze3SlgWCxljAytu27 +7g8fU1/r4VZMp6cf+uxKI1kxjDqjOryMLj/AVpbc4FsFZbSHLOB+0ZU1Bdc4qqRZ +UVgzwFuVOiY20NcHrE8BCkEpCsBZWb22jTi9vgh51iuvrV3Vr7AHGUpBqi4o0w7t +znUyrvvxsqDM6Ym9TZfL9EdPK2W+Yar+CPQ0fN3uwys7zbWLXy4+v6K0Xiv/ZXSy +d4bTQwzcLGwxHpgeDf/7D3d47iXlb/RUCJzuccz03WI2k1XLG7sUOKaoNhzKngvG +gncuEXSafsoxgtguyvOsLzWY9YtagliL+zftBjUW4WaVXxqdnBKzQyyVX+jSYvsJ +Eokc2WpuUxOEwzq5GsV5e9Fiu65wF+BhbWFyPkjz6EvACsSPGQQtttep/hkxs8lJ +WNio6FHDm/ult5fLZqH3FHvtU2bgQDCeVzf1z2pplaLqqzum6s5gFKfFiWRwOiRW +w1uD4fd4E5ooeMPbrvf8esWGqvRHqCjPnY+cJ7W+3wL1PJPxjSwHkLedWVM3jSRr +KFiayUx8t79kzITavvu860v8dQSbwJCnw3vGziGf4gvdxIUeGhFpAbMkhMA1Gjp7 +iOh1Zu1ShUNWZqSVTfq/+ExotpMoPpoLSnO+7K4XDVorjLTfZ4fN8u6waGr2YXEl +uRP5Qru1SOEQHDKMPxRMxOe/BIu9SIK1TWmMo44ag2ciShmoivLy0HEuqEeTa+9c +76oBTGqHLF5P0lvVe3O98hwadROKCJ9tsftosAXjMEw2tEZDOU1oNB12PwJW/Mpa +gJhA78XRjRE35oyzO/PA83VR060QKwgSA4/pCfXOoojkultFDSX4QTCwSz6PR5c4 +FvLEYYu7tPvlJWVPeMG1tXRcVAamrF/HyuWWkLL/Z7taduQ6bui+v6KBLOwsZNeD +9eDWlr0wDNhI5gcGg0kgo3usSHKSzzd5DqvunUkgjBpk1a1i8XF4mGi7l3BEIQfR +YpQKDgNQtxiOKHI32fleV7JuxxDzSybJ1lyWo0pdA4SLwo6KUeoQR13boehkZvga +MazCqGRhN5flV20sAKHfR9r1AtM1n8qnBU081rNU9rnwS6mE/RFh0EQFKsadCrED +LsLpJqYc209RwJTbPGGAx2blHXJhJAqIhIUUvZ9cuhmLUkadcalroEleFSbyYrKM +ah1koTU3s89y1bkksd6HQBNnOeWMKzoNm9zdAKv8OcQZIQ+FtaWoHcySDudxuHCk +JL6aKYkzZM+0tCjEge2VraOBoDzRSZUNVBxCICOoGYTIxRhwJ0Vu1rG+Zk4Ub30t +hkETPbQncejeDoUMbq/ESNAhxIe0KLXYXhJOH8qApWBGqJRKt1Lky3tkVKwLqcFA +zzdHEOvLclPvJGVCNw4yNoakk3TUuQI2AJ2p0Evh9RGMwacFARZ25grePzAI8TJp +AGoOQpVFPOgNz7ygT7JiUk/L0SntNKGXJnHZr/bdlhhk2QqnWrHnQN52FkuP3ZCd +G/ApW7G2E2t7V4oS+LO+7pGrBcuokAbDGicr2fhgijnO65VHg9q4SCRrkRyWwiRE +fIVwlO1COxo6mkWvxeZQ1KLxDKFlpfD7SmNyy7StkDFFwTbUOapb4FSdu3zNN+1U +27z0WOw1H7Bg5neCSJqxu7FBrILoJIEi19eIhOaxys+9LwFSUFi9GhzeLsQl4eDC +rEZgLKEIWo0AXyVHptWGci/oD5aJIyitHqK9dpdMKIZyvZLCdo0KSpnRX6fLVEaU +JdUTxUxsGcQ1FwNCZz2vq3K9Bv6muXIpyp0gSuSSlS+kEK88BcrUrFHNN3NRfz0Y +tfENObu/675l84fJN8idIZmO7iYCO+wSiDYARG1k+muSMgm/MrHyxT0XiC1vkds5 +K+z1GRkOeJzo0356tBzdTNq9q4u2TGTaFu15azdDnRNPm4BoK/SwhW1AIjOmzSPk +9MrtFehiwWgQy0RfaMGxbNqZZKjOR0wcA+JEMMxtjUyhxO5GaAZLci9P4qfIWWxt +HR6KiUywryeYp1eOXzUBliXSyEwh+gIw3FQ58VB7qOYzT51maj+vayOaOjcyr+VC +bJ4z3IipwOG1YV3wcYkYdO7OHAmXouO9NyiEih5RKlX3UOAxGiQxWsM4XSSJCQTu +1MOt/hZdYgSt1/N6JeOaQEUPGilWWjEuMZI0bs8jb850iD6ZRDZTIV5ofrrCtoKo +TBK1DsbL7XMAiTQyaOLXEiWyOYGIeBsKx4Ke29vpmQY2xG7mjgQoQMvtRKraGYYx +mNxaDtFhesUhFB3gqCRFQnd5UEi4pi5TKkm8l7ObGgNDOJ1tKi2f10CV8GF0g8bE +10UTV3JOMkE0Si8MSECyV+gDyOqGsIINPnPdocjsTKBp3ahjAiZkkGcnHmSSRGBT +gHCFOSaWJHu06T446Wn0MYUEiY31VGg96I0x9oLDEF8ThU+Z4BM+R6HVgGVtKfj5 +Uji541mSgnZ0Hp2E82AJS1JiVU9a4vW3OI6Js/cTBbKHkVBxuRgAk185TvTg132l +Wq9MNbscE0Ylz3Ia2g/RTM8a20PRi88Kbq0CK86Kwb6awm/M/IRqzPb0yabxxs0j +8zV9zj2keVT0PMQdUeO6xZQjX6fUOX4OqbF7kkNMR2JPGGYIJr+VUBzxnt5m2M1J +xnqeVbIi5+wK+NFkcAxX4G7jncy6RvrbSgn7wrtz2T8IYplJmHjcxI/HMi+R3kGp +H+uSCHKRk6W1BVLcjgc6xOp1J47Z1piVx1tYUt7+52IB90MxMObeoBh+xf8oHLRL +KJRjDuaRRgsM9eFiJ7Y4rzSmvqVFtAzAp2FYsHsvrk7qFiK3Y5I81rXHMOCnO4mh +cY0lbmmb2ILQCzqmCLRDijJQCnntztAb9QAR9vPobbAov43edprqpiBDUfi0zamN +Kzp38yur8saXpbbEeHmZ5/XMqSn8wJJnk3efEhEE9si6Grm/QmDmVY3tg/zGw3h7 +G9cbQj3oOkDzHW+tpAorsOaWpGfFBHKij9A9yrnv2KGLbhyflL74/u3NpTBjJOLn +BGW4nxXiKWticbiyxuJJ5aIn/gRbfsL22rC9e6MxUQTrI3N780ynyO3gFsf68K5y +fD29qZ1Pd1g63a5FzraF6LZzeyhAXI6vMTgdp6dlS1yeEg+Hac7QymE6RW4XebUe +D19fh2P24eG3dXd4dVl28jqmJ/+A/SxKfyl2MPGc/n8UO81en8H4CnY74hoJvJ8V +nrs3KAqpe3ZvDPkmZj7Aho09yqZMFmcKsEFTYLcNbZwKnGCYJEvwvYb2uqdNk4RF +ATpgYua1jQPGyFa37A1ol9moHShX8TMPccjaDkUPL8XXhg0+A6zTDQT7skXaIBJh +OVwJXu12p3ri3f6swLVYZ7ra8wadMgL1uN1eWPoG0RGcxMQCqRbeTFBcCh8hRkQg +swd1DAN+GpEqS9wWRHiuEBQ2HQYg+leJCJDCu8jNgOS1bg/PCzNdCo7eyCZMUdn+ +qleKiUTI5Jke0hGBJaMh+JeIT0YRWRdOSAUNj/rFypE3w87o/J1ZJK2GRKN7DJdc +HSSO3rNdRJH7qNmW95nRA5li+VyY8dc36Y8eO6yA2zjRVlMM2azVpBh8mabB0zZr +HWPNrmCtI1p/sFIT26gn1jrcef1Yt6bEEmGGWA+qbbNWE1MZm7UOiVEJrPWQgrUu +RbBWfFw2a/Wj2UyDtfrVuW3W6oaRGIDumSibw9LuNWpiebBZhjdM7GymwVpNMWfZ +rHUYDJZNWkPanHXLwVDNstbnGwWTCZTVLZ9zU1YTUX6bsh4+BmU1sUZueZ6MTsq2 +KesRMq5bQKtszuoB59wRnNUUMmVzVs+WPDZnXdm0Oevr9AJnXe9bnNWdeWKsfuPQ +zVhNxJwYjNXNk3JirG6+bsJq0qCrCWhOqdqJsA7Hx3ZejySZkY4zUJOEdXiQ+ias +yJm+Ceurh7CYrBVzHvCx8H4oLFamuEHRlZOYe9FEJIq5F6JTGBZzZgkoc8aigdTR +Ncd1hFbNAFkit49azuuTMyuqzETlcDMSvDGTZQqvcyKQFghNcJItKmFoKTy2cVoB +zSs9bGE9+hRLWwQvF+XuihmzoR+ZVDF1td5jcwo0RZQn4bGxpE0crK9UomIacyJ3 +FhROtbCInMVGunooJjLKQlBRy4m1OSs5eN6WRPOi3UVPrVHtWz23Ro1OwnX3aMps +jYU+RAu0bjDDh1WoANcyVpWIIxGBzu2Z+bcUnn/N08cUwnktQlTYn0BC/DZi3NAa +xgG17OGRPUPph3gakooit7cY32JdyPonkt9iFn02LU/FeDdaeApTH4rnJDZdngqF +ADrtdCFcIyhKcuQwvDJfkWwa6aMCsq9zZTLyJwDX/ZqhGErHoGCtXmv4tSG7Gk03 +xSDE1s4wAMwaoWaJlvk7DKHoyiAP9mPx6vKYsOjL1NjOjpvBGsxWXUMGE6ifSKFy +dNyrpdajm1ousvpzuNyStWeS08FMHrzZ4/4afRyQvnu4fPtjvubrwz8uBgjJ/tlP +Se4XM70n/8L8+3C/fPv9Z8vQz7bj89PLJV0fnvy//1y+/vX2+PL85frr+x+vPz3+ ++/HvT58+fPxy/fn58dPLh5d/Xr9/fvny/On614ffLu+MP1kBvsvmxuvD+8vXP/z3 +8f7x9nz9S7m+u371tz9ezif8/nJ9//vTH3f7/vrLx+eXr/yIHx4u/7rk64frpfSK +aNmfFaReHXosyT89X/4ku+p5EISB6M6vuA1HaK1BBhODLiZiDCQOxgFpCX4EEgv6 +932lRCAOTdrm+vJ67+7aO1Fl2UYJ2CZRDKI7zO645wd2tKfzxSM5gRDMvDV8ISY7 +T0c7R+sl9uclxhHwCCrho1KY3zS8NNtkzZSnz2zKC49bnpx3a8RdzxXwI9S5byKa +TVFV2IN2ZILOOCD0IUhsU+AFvuseCoGxjQ/pNqS0VGNn5rVU1Golqahf1JQ3TU2m +H5RnFV1Vpw6aQc+Kg9Jh1CnqtpKEAfnctXxnVY7zqzHsapDIQKB7WyJJRxiDrXZ/ +V6CvAAMAvjaLtA0KZW5kc3RyZWFtDWVuZG9iag0yMyAwIG9iajw8L0xlbmd0aCAy +NTc1L0ZpbHRlci9GbGF0ZURlY29kZS9OIDMvQWx0ZXJuYXRlL0RldmljZVJHQj4+ +c3RyZWFtDQpIiZyWeVRTdxbHf2/JnpCVsMNjDVuAsAaQNWxhkR0EUQhJCAESQkjY +BUFEBRRFRISqlTLWbXRGT0WdLq5jrQ7WferSA/Uw6ug4tBbXjp0XOEedTmem0+8f +7/c593fv793fvfed8wCgJ6WqtdUwCwCN1qDPSozFFhUUYqQJAAMKIAIRADJ5rS4t +OyEH4JLGS7Ba3An8i55eB5BpvSJMysAw8P+JLdfpDQBAGTgHKJS1cpw7ca6qN+hM +9hmceaWVJoZRE+vxBHG2NLFqnr3nfOY52sQKjVaBsylnnUKjMPFpnFfXGZU4I6k4 +d9WplfU4X8XZpcqoUeP83BSrUcpqAUDpJrtBKS/H2Q9nuj4nS4LzAgDIdNU7XPoO +G5QNBtOlJNW6Rr1aVW7A3OUemCg0VIwlKeurlAaDMEMmr5TpFZikWqOTaRsBmL/z +nDim2mJ4kYNFocHBQn8f0TuF+q+bv1Cm3s7Tk8y5nkH8C29tP+dXPQqAeBavzfq3 +ttItAIyvBMDy5luby/sAMPG+Hb74zn34pnkpNxh0Yb6+9fX1Pmql3MdU0Df6nw6/ +QO+8z8d03JvyYHHKMpmxyoCZ6iavrqo26rFanUyuxIQ/HeJfHfjzeXhnKcuUeqUW +j8jDp0ytVeHt1irUBnW1FlNr/1MTf2XYTzQ/17i4Y68Br9gHsC7yAPK3CwDl0gBS +tA3fgd70LZWSBzLwNd/h3vzczwn691PhPtOjVq2ai5Nk5WByo75ufs/0WQICoAIm +4AErYA+cgTsQAn8QAsJBNIgHySAd5IACsBTIQTnQAD2oBy2gHXSBHrAebALDYDsY +A7vBfnAQjIOPwQnwR3AefAmugVtgEkyDh2AGPAWvIAgiQQyIC1lBDpAr5AX5Q2Io +EoqHUqEsqAAqgVSQFjJCLdAKqAfqh4ahHdBu6PfQUegEdA66BH0FTUEPoO+glzAC +02EebAe7wb6wGI6BU+AceAmsgmvgJrgTXgcPwaPwPvgwfAI+D1+DJ+GH8CwCEBrC +RxwRISJGJEg6UoiUIXqkFelGBpFRZD9yDDmLXEEmkUfIC5SIclEMFaLhaBKai8rR +GrQV7UWH0V3oYfQ0egWdQmfQ1wQGwZbgRQgjSAmLCCpCPaGLMEjYSfiIcIZwjTBN +eEokEvlEATGEmEQsIFYQm4m9xK3EA8TjxEvEu8RZEolkRfIiRZDSSTKSgdRF2kLa +R/qMdJk0TXpOppEdyP7kBHIhWUvuIA+S95A/JV8m3yO/orAorpQwSjpFQWmk9FHG +KMcoFynTlFdUNlVAjaDmUCuo7dQh6n7qGept6hMajeZEC6Vl0tS05bQh2u9on9Om +aC/oHLonXUIvohvp6+gf0o/Tv6I/YTAYboxoRiHDwFjH2M04xfia8dyMa+ZjJjVT +mLWZjZgdNrts9phJYboyY5hLmU3MQeYh5kXmIxaF5caSsGSsVtYI6yjrBmuWzWWL +2OlsDbuXvYd9jn2fQ+K4ceI5Ck4n5wPOKc5dLsJ15kq4cu4K7hj3DHeaR+QJeFJe +Ba+H91veBG/GnGMeaJ5n3mA+Yv6J+SQf4bvxpfwqfh//IP86/6WFnUWMhdJijcV+ +i8sWzyxtLKMtlZbdlgcsr1m+tMKs4q0qrTZYjVvdsUatPa0zreutt1mfsX5kw7MJ +t5HbdNsctLlpC9t62mbZNtt+YHvBdtbO3i7RTme3xe6U3SN7vn20fYX9gP2n9g8c +uA6RDmqHAYfPHP6KmWMxWBU2hJ3GZhxtHZMcjY47HCccXzkJnHKdOpwOON1xpjqL +ncucB5xPOs+4OLikubS47HW56UpxFbuWu252Pev6zE3glu+2ym3c7b7AUiAVNAn2 +Cm67M9yj3GvcR92vehA9xB6VHls9vvSEPYM8yz1HPC96wV7BXmqvrV6XvAneod5a +71HvG0K6MEZYJ9wrnPLh+6T6dPiM+zz2dfEt9N3ge9b3tV+QX5XfmN8tEUeULOoQ +HRN95+/pL/cf8b8awAhICGgLOBLwbaBXoDJwW+Cfg7hBaUGrgk4G/SM4JFgfvD/4 +QYhLSEnIeyE3xDxxhrhX/HkoITQ2tC3049AXYcFhhrCDYX8PF4ZXhu8Jv79AsEC5 +YGzB3QinCFnEjojJSCyyJPL9yMkoxyhZ1GjUN9HO0YrondH3YjxiKmL2xTyO9YvV +x34U+0wSJlkmOR6HxCXGdcdNxHPic+OH479OcEpQJexNmEkMSmxOPJ5ESEpJ2pB0 +Q2onlUt3S2eSQ5KXJZ9OoadkpwynfJPqmapPPZYGpyWnbUy7vdB1oXbheDpIl6Zv +TL+TIcioyfhDJjEzI3Mk8y9ZoqyWrLPZ3Ozi7D3ZT3Nic/pybuW65xpzT+Yx84ry +duc9y4/L78+fXOS7aNmi8wXWBeqCI4WkwrzCnYWzi+MXb1o8XRRU1FV0fYlgScOS +c0utl1Yt/aSYWSwrPlRCKMkv2VPygyxdNiqbLZWWvlc6I5fIN8sfKqIVA4oHyghl +v/JeWURZf9l9VYRqo+pBeVT5YPkjtUQ9rP62Iqlie8WzyvTKDyt/rMqvOqAha0o0 +R7UcbaX2dLV9dUP1JZ2Xrks3WRNWs6lmRp+i31kL1S6pPWLg4T9TF4zuxpXGqbrI +upG65/V59Yca2A3ahguNno1rGu81JTT9phltljefbHFsaW+ZWhazbEcr1FraerLN +ua2zbXp54vJd7dT2yvY/dfh19Hd8vyJ/xbFOu87lnXdXJq7c22XWpe+6sSp81fbV +6Gr16ok1AWu2rHndrej+osevZ7Dnh1557xdrRWuH1v64rmzdRF9w37b1xPXa9dc3 +RG3Y1c/ub+q/uzFt4+EBbKB74PtNxZvODQYObt9M3WzcPDmU+k8ApAFb/pi4mSSZ +kJn8mmia1ZtCm6+cHJyJnPedZJ3SnkCerp8dn4uf+qBpoNihR6G2oiailqMGo3aj +5qRWpMelOKWpphqmi6b9p26n4KhSqMSpN6mpqhyqj6sCq3Wr6axcrNCtRK24ri2u +oa8Wr4uwALB1sOqxYLHWskuywrM4s660JbSctRO1irYBtnm28Ldot+C4WbjRuUq5 +wro7urW7LrunvCG8m70VvY++Cr6Evv+/er/1wHDA7MFnwePCX8Lbw1jD1MRRxM7F +S8XIxkbGw8dBx7/IPci8yTrJuco4yrfLNsu2zDXMtc01zbXONs62zzfPuNA50LrR +PNG+0j/SwdNE08bUSdTL1U7V0dZV1tjXXNfg2GTY6Nls2fHadtr724DcBdyK3RDd +lt4c3qLfKd+v4DbgveFE4cziU+Lb42Pj6+Rz5PzlhOYN5pbnH+ep6DLovOlG6dDq +W+rl63Dr++yG7RHtnO4o7rTvQO/M8Fjw5fFy8f/yjPMZ86f0NPTC9VD13vZt9vv3 +ivgZ+Kj5OPnH+lf65/t3/Af8mP0p/br+S/7c/23//wIMAPeE8/sKDQplbmRzdHJl +YW0NZW5kb2JqDTI0IDAgb2JqPDwvU2l6ZVsyNTVdL0xlbmd0aCA3ODAvRmlsdGVy +L0ZsYXRlRGVjb2RlL0RlY29kZVswIDEgMCAxIDAgMV0vUmFuZ2VbMCAxIDAgMSAw +IDFdL0JpdHNQZXJTYW1wbGUgOC9Eb21haW5bMCAxXS9FbmNvZGVbMCAyNTRdL0Z1 +bmN0aW9uVHlwZSAwPj5zdHJlYW0NCkiJAP0CAv0jHyAkICElIiImIyQnJCUoJSYq +JygrKCksKSotKiwuLC0wLS4xLzAyMDEzMTM0MzQ2NDU3NTc4Nzg5ODo6OTs8Ozw9 +PD4+PT8/PkBAQEFBQUJCQkNDQ0REREVFRUZGRkdHRkhIR0lISEpJSUtKSkxLS01M +TE5NTU9OTlBPT1FQUFJRUVNSUlRTU1VTVFZUVVdVVlhWV1lXWFpYWVtZWlxaW11b +XF5bXF9cXWBdXmBeX2FfYGJgYWNgYWRhYmViY2ZjZGZkZWdkZmhlZ2lmZ2pnaGto +aWtpamxpa21qa25rbG9sbXBtbnBtb3FucHJvcHNwcXRwcnRxc3VydHZzdHd0dXh0 +dnh1d3l2d3p3eHt3eXt4enx5en16e356fH97fX98fYB8foF9f4J+gIJ/gIN/gYSA +goWBg4WCg4aChIeDhYiEhoiFhomFh4qGiIuHiYuIioyIio2Ji46KjI6LjY+LjZCM +jpGNj5GOkJKOkJOPkZSQkpWRk5WRk5aSlJeTlZiUlpmVl5qWmJuXmZuXmZyYmp2Z +m56anJ6anZ+bnaCcnqGdn6GeoKKeoKOfoaSgoqWho6WhpKaipKejpaikpqmlp6ml +qKqmqKunqayoqq2pq62prK6qrK+rrbCsrrCtr7GtsLKusLOvsbSwsrSxs7WytLay +tLeztbi0tri1t7m2uLq2uLu3uby4ury5u726vL67vb+7vcC8vsC9v8G+wMK/wcPA +wcTAwsTBw8XCxMbDxcfExsjFxsjFx8nGyMrHycvIyszJy8zKy83LzM7Lzc/MztDN +z9DOz9HP0NLP0dPQ0tPR09TS09XT1NbU1dfU1tfV19jW19nX2NrY2dvZ2tvZ29za +293b3N7c3d7d3t/e3+De3+Hf4OHg4eLh4uPi4+Tj5OXj5OXk5ebl5ufm5+jn6Ojo +6Onp6erp6uvq6+zr7Ozs7e3t7e7t7u/u7/Dv8PDw8fHx8fLy8vPz8/Pz9PT09fX1 +9vb29vf39/j4+Pn5+fr6+/v8/Pz9/f3///8CDAADYLy/Cg0KZW5kc3RyZWFtDWVu +ZG9iag0yNSAwIG9iajw8L1R5cGUvRm9udERlc2NyaXB0b3IvRm9udEJCb3hbLTE3 +MCAtMjI4IDEwMDMgOTYyXS9Gb250TmFtZS9IZWx2ZXRpY2EtQm9sZC9GbGFncyAy +NjIxNzYvU3RlbVYgMTQwL1N0ZW1IIDE0MC9DYXBIZWlnaHQgNzE4L1hIZWlnaHQg +NTMyL0FzY2VudCA3MTgvRGVzY2VudCAtMjA3L0l0YWxpY0FuZ2xlIDA+Pg1lbmRv +YmoNMjYgMCBvYmo8PC9UeXBlL0ZvbnREZXNjcmlwdG9yL0ZvbnRCQm94Wy0xNjYg +LTIyNSAxMDAwIDkzMV0vRm9udE5hbWUvSGVsdmV0aWNhL0ZsYWdzIDMyL1N0ZW1W +IDg4L1N0ZW1IIDg4L0NhcEhlaWdodCA3MTgvWEhlaWdodCA1MjMvQXNjZW50IDcx +OC9EZXNjZW50IC0yMDcvSXRhbGljQW5nbGUgMD4+DWVuZG9iag0yNyAwIG9iajw8 +L1R5cGUvRXh0R1N0YXRlL1NBIGZhbHNlL09QIGZhbHNlL1NNIDAuMDIvb3AgZmFs +c2UvT1BNIDE+Pg1lbmRvYmoNMjggMCBvYmo8PC9UeXBlL0V4dEdTdGF0ZS9TQSB0 +cnVlL09QIGZhbHNlL1NNIDAuMDIvb3AgZmFsc2UvT1BNIDE+Pg1lbmRvYmoNMjkg +MCBvYmo8PC9QaXRTdG9wIDMwIDAgUj4+DWVuZG9iag0zMCAwIG9iajw8L0NDIDMx +IDAgUj4+DWVuZG9iag0zMSAwIG9iajw8Pj4NZW5kb2JqDTMyIDAgb2JqPDwvRmll +bGRzWzM2IDAgUl0vRFI8PC9FbmNvZGluZzw8L1BERkRvY0VuY29kaW5nIDM1IDAg +Uj4+L0ZvbnQ8PC9IZWx2IDM0IDAgUi9aYURiIDMzIDAgUj4+Pj4vREEoL0hlbHYg +MCBUZiAwIGcgKT4+DWVuZG9iag0zMyAwIG9iajw8L1R5cGUvRm9udC9OYW1lL1ph +RGIvQmFzZUZvbnQvWmFwZkRpbmdiYXRzL1N1YnR5cGUvVHlwZTE+Pg1lbmRvYmoN +MzQgMCBvYmo8PC9UeXBlL0ZvbnQvTmFtZS9IZWx2L0VuY29kaW5nIDM1IDAgUi9C +YXNlRm9udC9IZWx2ZXRpY2EvU3VidHlwZS9UeXBlMT4+DWVuZG9iag0zNSAwIG9i +ajw8L1R5cGUvRW5jb2RpbmcvRGlmZmVyZW5jZXNbMjQvYnJldmUvY2Fyb24vY2ly +Y3VtZmxleC9kb3RhY2NlbnQvaHVuZ2FydW1sYXV0L29nb25lay9yaW5nL3RpbGRl +IDM5L3F1b3Rlc2luZ2xlIDk2L2dyYXZlIDEyOC9idWxsZXQvZGFnZ2VyL2RhZ2dl +cmRibC9lbGxpcHNpcy9lbWRhc2gvZW5kYXNoL2Zsb3Jpbi9mcmFjdGlvbi9ndWls +c2luZ2xsZWZ0L2d1aWxzaW5nbHJpZ2h0L21pbnVzL3BlcnRob3VzYW5kL3F1b3Rl +ZGJsYmFzZS9xdW90ZWRibGxlZnQvcXVvdGVkYmxyaWdodC9xdW90ZWxlZnQvcXVv +dGVyaWdodC9xdW90ZXNpbmdsYmFzZS90cmFkZW1hcmsvZmkvZmwvTHNsYXNoL09F +L1NjYXJvbi9ZZGllcmVzaXMvWmNhcm9uL2RvdGxlc3NpL2xzbGFzaC9vZS9zY2Fy +b24vemNhcm9uIDE2MC9FdXJvIDE2NC9jdXJyZW5jeSAxNjYvYnJva2VuYmFyIDE2 +OC9kaWVyZXNpcy9jb3B5cmlnaHQvb3JkZmVtaW5pbmUgMTcyL2xvZ2ljYWxub3Qv +Lm5vdGRlZi9yZWdpc3RlcmVkL21hY3Jvbi9kZWdyZWUvcGx1c21pbnVzL3R3b3N1 +cGVyaW9yL3RocmVlc3VwZXJpb3IvYWN1dGUvbXUgMTgzL3BlcmlvZGNlbnRlcmVk +L2NlZGlsbGEvb25lc3VwZXJpb3Ivb3JkbWFzY3VsaW5lIDE4OC9vbmVxdWFydGVy +L29uZWhhbGYvdGhyZWVxdWFydGVycyAxOTIvQWdyYXZlL0FhY3V0ZS9BY2lyY3Vt +ZmxleC9BdGlsZGUvQWRpZXJlc2lzL0FyaW5nL0FFL0NjZWRpbGxhL0VncmF2ZS9F +YWN1dGUvRWNpcmN1bWZsZXgvRWRpZXJlc2lzL0lncmF2ZS9JYWN1dGUvSWNpcmN1 +bWZsZXgvSWRpZXJlc2lzL0V0aC9OdGlsZGUvT2dyYXZlL09hY3V0ZS9PY2lyY3Vt +ZmxleC9PdGlsZGUvT2RpZXJlc2lzL211bHRpcGx5L09zbGFzaC9VZ3JhdmUvVWFj +dXRlL1VjaXJjdW1mbGV4L1VkaWVyZXNpcy9ZYWN1dGUvVGhvcm4vZ2VybWFuZGJs +cy9hZ3JhdmUvYWFjdXRlL2FjaXJjdW1mbGV4L2F0aWxkZS9hZGllcmVzaXMvYXJp +bmcvYWUvY2NlZGlsbGEvZWdyYXZlL2VhY3V0ZS9lY2lyY3VtZmxleC9lZGllcmVz +aXMvaWdyYXZlL2lhY3V0ZS9pY2lyY3VtZmxleC9pZGllcmVzaXMvZXRoL250aWxk +ZS9vZ3JhdmUvb2FjdXRlL29jaXJjdW1mbGV4L290aWxkZS9vZGllcmVzaXMvZGl2 +aWRlL29zbGFzaC91Z3JhdmUvdWFjdXRlL3VjaXJjdW1mbGV4L3VkaWVyZXNpcy95 +YWN1dGUvdGhvcm4veWRpZXJlc2lzXT4+DWVuZG9iag0zNiAwIG9iajw8L0YgNC9U +eXBlL0Fubm90L1JlY3RbMjYwLjI1MTAwNyA1MDIuNDk4OTAxIDM2OS43NTE0MDQg +NTQwLjc0OTAyM10vRlQvVHgvU3VidHlwZS9XaWRnZXQvUCA5IDAgUi9UKHRvZGF5 +c0RhdGUpL1EgMS9EQSgvSGVsdiAxMiBUZiAwIGcpL01LPDw+Pj4+DWVuZG9iag0z +NyAwIG9ialszNiAwIFJdDWVuZG9iag0zOCAwIG9iajw8L0phdmFTY3JpcHQgMzkg +MCBSPj4NZW5kb2JqDTM5IDAgb2JqPDwvTmFtZXNbKGRvY09wZW5lZCk0MCAwIFJd +Pj4NZW5kb2JqDTQwIDAgb2JqPDwvUy9KYXZhU2NyaXB0L0pTIDQyIDAgUj4+DWVu +ZG9iag00MSAwIG9iaiAzMTkNZW5kb2JqDTQyIDAgb2JqPDwvTGVuZ3RoIDQxIDAg +Ui9GaWx0ZXJbL0ZsYXRlRGVjb2RlXT4+c3RyZWFtDQpIiXRSQWrDMBA82+A/bHWy +odgPCDkESq99wzraxCqKZKRVQij5Q5/clVxDklKBDxrN7IxGHgY4YzA4WgL2ENkH +gstEPFEA7ffpRI5hwggjkQM/kyMNaAOhvoIP4Dw3tYyAcbeAH4WyaeqmPiS3Z+Nd +HrTAbdfUX/moMof2UQEvW1AcEinhVEKqqmF4jPCZIt/nyJxsrWELji7whkxtt1nh +mPdylNjYfg7GsW7V6TRoPVxlqVfQXckJT0t8IzHoLHf+8peA89yjpcCt2o0+ca7O +uCjAIhIrDwdDVme9WiLxZGJ/JH7PeKvYa7yWiKrrz2hTjlr2JVO5vYhLkqemZHgu +CqK4iqOn6L75VxKSAzyicU39pFrrzWFu8pGN9F/T6wM/li2qW84m7D1aC/KT3D8t +rO/d1Heo2P0IMACaArkEDQplbmRzdHJlYW0NZW5kb2JqDTQzIDAgb2JqPDwvTGVu +Z3RoIDM0MjcvVHlwZS9NZXRhZGF0YS9TdWJ0eXBlL1hNTD4+c3RyZWFtDQo8P3hw +YWNrZXQgYmVnaW49J++7vycgaWQ9J1c1TTBNcENlaGlIenJlU3pOVGN6a2M5ZCc/ +Pgo8P2Fkb2JlLXhhcC1maWx0ZXJzIGVzYz0iQ1JMRiI/Pg0KPHg6eG1wbWV0YSB4 +bWxuczp4PSdhZG9iZTpuczptZXRhLycgeDp4bXB0az0nWE1QIHRvb2xraXQgMi45 +LjEtMTQsIGZyYW1ld29yayAxLjYnPg0KPHJkZjpSREYgeG1sbnM6cmRmPSdodHRw +Oi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjJyB4bWxuczpp +WD0naHR0cDovL25zLmFkb2JlLmNvbS9pWC8xLjAvJz4NCjxyZGY6RGVzY3JpcHRp +b24gcmRmOmFib3V0PSd1dWlkOjI4NWVhMTI5LWZlYmUtNDgyYS1iYzFhLTVkYTZj +ZWIwNTg2NicgeG1sbnM6cGRmPSdodHRwOi8vbnMuYWRvYmUuY29tL3BkZi8xLjMv +JyBwZGY6UHJvZHVjZXI9J0Fjcm9iYXQgRGlzdGlsbGVyIDYuMC4xIChXaW5kb3dz +KSc+PC9yZGY6RGVzY3JpcHRpb24+DQo8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91 +dD0ndXVpZDoyODVlYTEyOS1mZWJlLTQ4MmEtYmMxYS01ZGE2Y2ViMDU4NjYnIHht +bG5zOnhhcD0naHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wLycgeGFwOk1vZGlm +eURhdGU9JzIwMDQtMTAtMjhUMTg6MTY6NDUrMTA6MDAnIHhhcDpDcmVhdGVEYXRl +PScyMDA0LTEwLTI4VDE4OjEzOjM4KzEwOjAwJyB4YXA6TWV0YWRhdGFEYXRlPScy +MDA0LTEwLTI4VDE4OjE2OjQ1KzEwOjAwJyB4YXA6Q3JlYXRvclRvb2w9J1BTY3Jp +cHQ1LmRsbCBWZXJzaW9uIDUuMi4yJz48L3JkZjpEZXNjcmlwdGlvbj4NCjxyZGY6 +RGVzY3JpcHRpb24gcmRmOmFib3V0PSd1dWlkOjI4NWVhMTI5LWZlYmUtNDgyYS1i +YzFhLTVkYTZjZWIwNTg2NicgeG1sbnM6eGFwTU09J2h0dHA6Ly9ucy5hZG9iZS5j +b20veGFwLzEuMC9tbS8nIHhhcE1NOkRvY3VtZW50SUQ9J3V1aWQ6MmNhMjBjZmEt +M2FlZi00MTU1LThkOWMtYTM4ZWJjNDM3MjVmJy8+DQo8cmRmOkRlc2NyaXB0aW9u +IHJkZjphYm91dD0ndXVpZDoyODVlYTEyOS1mZWJlLTQ4MmEtYmMxYS01ZGE2Y2Vi +MDU4NjYnIHhtbG5zOmRjPSdodHRwOi8vcHVybC5vcmcvZGMvZWxlbWVudHMvMS4x +LycgZGM6Zm9ybWF0PSdhcHBsaWNhdGlvbi9wZGYnPjxkYzp0aXRsZT48cmRmOkFs +dD48cmRmOmxpIHhtbDpsYW5nPSd4LWRlZmF1bHQnPlBsYW5ldCBQREYgSmF2YVNj +cmlwdCBMZWFybmluZyBDZW50ZXIgRXhhbXBsZSAjMjwvcmRmOmxpPjwvcmRmOkFs +dD48L2RjOnRpdGxlPjxkYzpjcmVhdG9yPjxyZGY6U2VxPjxyZGY6bGk+Q2hyaXMg +RGFobCwgQVJUUyBQREYgR2xvYmFsIFNlcnZpY2VzPC9yZGY6bGk+PC9yZGY6U2Vx +PjwvZGM6Y3JlYXRvcj48L3JkZjpEZXNjcmlwdGlvbj4NCjwvcmRmOlJERj4NCjwv +eDp4bXBtZXRhPg0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAog +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgIAo8P3hw +YWNrZXQgZW5kPSd3Jz8+DQplbmRzdHJlYW0NZW5kb2JqDXhyZWYNCjAgNDQNCjAw +MDAwMDAwMDQgNjU1MzUgZg0KMDAwMDAwMDAxNiAwMDAwMCBuDQowMDAwMDAwMDQ5 +IDAwMDAwIG4NCjAwMDAwMDAwNzIgMDAwMDAgbg0KMDAwMDAwMDAwNiAwMDAwMSBm +DQowMDAwMDAwMTIyIDAwMDAwIG4NCjAwMDAwMDAwMDggMDAwMDEgZg0KMDAwMDAw +MDM5NSAwMDAwMCBuDQowMDAwMDAwMDAwIDAwMDAxIGYNCjAwMDAwMDA1MjEgMDAw +MDAgbg0KMDAwMDAwMDc0NiAwMDAwMCBuDQowMDAwMDAwODg3IDAwMDAwIG4NCjAw +MDAwMDA5MjEgMDAwMDAgbg0KMDAwMDAwODc2MCAwMDAwMCBuDQowMDAwMDE3NTcz +IDAwMDAwIG4NCjAwMDAwMjU5MjIgMDAwMDAgbg0KMDAwMDAzNDE5OSAwMDAwMCBu +DQowMDAwMDQyNTAzIDAwMDAwIG4NCjAwMDAwNTA3NTggMDAwMDAgbg0KMDAwMDA1 +MTI2MyAwMDAwMCBuDQowMDAwMDUxNzU2IDAwMDAwIG4NCjAwMDAwNTE4MTIgMDAw +MDAgbg0KMDAwMDA2MDI5MiAwMDAwMCBuDQowMDAwMDY4OTA5IDAwMDAwIG4NCjAw +MDAwNzE1NzggMDAwMDAgbg0KMDAwMDA3MjUzMyAwMDAwMCBuDQowMDAwMDcyNzIz +IDAwMDAwIG4NCjAwMDAwNzI5MDIgMDAwMDAgbg0KMDAwMDA3Mjk3OCAwMDAwMCBu +DQowMDAwMDczMDUzIDAwMDAwIG4NCjAwMDAwNzMwODggMDAwMDAgbg0KMDAwMDA3 +MzExOCAwMDAwMCBuDQowMDAwMDczMTM4IDAwMDAwIG4NCjAwMDAwNzMyNjggMDAw +MDAgbg0KMDAwMDA3MzM0NCAwMDAwMCBuDQowMDAwMDczNDMzIDAwMDAwIG4NCjAw +MDAwNzQ2MjQgMDAwMDAgbg0KMDAwMDA3NDc4MyAwMDAwMCBuDQowMDAwMDc0ODA3 +IDAwMDAwIG4NCjAwMDAwNzQ4NDUgMDAwMDAgbg0KMDAwMDA3NDg5MCAwMDAwMCBu +DQowMDAwMDc0OTMzIDAwMDAwIG4NCjAwMDAwNzQ5NTMgMDAwMDAgbg0KMDAwMDA3 +NTM0NiAwMDAwMCBuDQp0cmFpbGVyDQo8PC9TaXplIDQ0L1Jvb3QgNyAwIFIvSW5m +byA1IDAgUi9JRFs8NTQ4YTEwMTkxOWUxNmI1ZmRhOTMyZTcyZmFjNDYyYjc+PDBm +MzAxYmVmZTJkMzA2NGU5OWZkODRjZWUxNjAyOGM4Pl0+Pg0Kc3RhcnR4cmVmDQo3 +ODg1MA0KJSVFT0YNCjUgMCBvYmo8PC9Nb2REYXRlKEQ6MjAwNDEwMjgxODE3NDYr +MTAnMDAnKS9DcmVhdGlvbkRhdGUoRDoyMDA0MTAyODE4MTMzOCsxMCcwMCcpL1Rp +dGxlKFBsYW5ldCBQREYgSmF2YVNjcmlwdCBMZWFybmluZyBDZW50ZXIgRXhhbXBs +ZSAjMikvQ3JlYXRvcihQU2NyaXB0NS5kbGwgVmVyc2lvbiA1LjIuMikvQXV0aG9y +KENocmlzIERhaGwsIEFSVFMgUERGIEdsb2JhbCBTZXJ2aWNlcykvUHJvZHVjZXIo +QWNyb2JhdCBEaXN0aWxsZXIgNi4wLjEgXChXaW5kb3dzXCkpPj4NZW5kb2JqDTcg +MCBvYmo8PC9QYWdlcyAzIDAgUi9UeXBlL0NhdGFsb2cvT3BlbkFjdGlvbiA0NiAw +IFIvTmFtZXMgMzggMCBSL1BhZ2VMYWJlbHMgMSAwIFIvQWNyb0Zvcm0gMzIgMCBS +L01ldGFkYXRhIDQ3IDAgUi9GSUNMOkVuZm9jdXMgMjkgMCBSPj4NZW5kb2JqDTM2 +IDAgb2JqPDwvRiA0L1R5cGUvQW5ub3QvUmVjdFsyNjAuMjUxMDA3IDUwMi40OTg5 +MDEgMzY5Ljc1MTQwNCA1NDAuNzQ5MDIzXS9GVC9UeC9TdWJ0eXBlL1dpZGdldC9Q +IDkgMCBSL1QodG9kYXlzRGF0ZSkvVigxMC8yOC8yMDA0KS9BUDw8L04gNDUgMCBS +Pj4vUSAxL0RBKC9IZWx2IDEyIFRmIDAgZykvTUs8PD4+Pj4NZW5kb2JqDTQ0IDAg +b2JqIG51bGwNZW5kb2JqDTQ1IDAgb2JqPDwvTGVuZ3RoIDExNS9UeXBlL1hPYmpl +Y3QvQkJveFswLjAgMC4wIDEwOS41MDAzOTcgMzguMjUwMTIyXS9SZXNvdXJjZXM8 +PC9Gb250PDwvSGVsdiAzNCAwIFI+Pi9Qcm9jU2V0Wy9QREYvVGV4dF0+Pi9TdWJ0 +eXBlL0Zvcm0vRm9ybVR5cGUgMS9NYXRyaXhbMS4wIDAuMCAwLjAgMS4wIDAuMCAw +LjBdPj5zdHJlYW0NCi9UeCBCTUMgCnEKMSAxIDEwNy41MDA0IDM2LjI1MDEgcmUK +VwpuCkJUCi9IZWx2IDEyIFRmCjIgMTQuNjczIFRkCjEzLjg3MjEgVEwKMjIuNzI2 +MiAwIFRkCigxMC8yOC8yMDA0KSBUagpFVApRCkVNQwoNCmVuZHN0cmVhbQ1lbmRv +YmoNNDYgMCBvYmo8PC9EWzkgMCBSL1hZWiAtMzI3NjggLTMyNzY4IDEuMF0vUy9H +b1RvPj4NZW5kb2JqDTQ3IDAgb2JqPDwvTGVuZ3RoIDM0MjcvVHlwZS9NZXRhZGF0 +YS9TdWJ0eXBlL1hNTD4+c3RyZWFtDQo8P3hwYWNrZXQgYmVnaW49J++7vycgaWQ9 +J1c1TTBNcENlaGlIenJlU3pOVGN6a2M5ZCc/Pgo8P2Fkb2JlLXhhcC1maWx0ZXJz +IGVzYz0iQ1JMRiI/Pg0KPHg6eG1wbWV0YSB4bWxuczp4PSdhZG9iZTpuczptZXRh +LycgeDp4bXB0az0nWE1QIHRvb2xraXQgMi45LjEtMTQsIGZyYW1ld29yayAxLjYn +Pg0KPHJkZjpSREYgeG1sbnM6cmRmPSdodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAy +LzIyLXJkZi1zeW50YXgtbnMjJyB4bWxuczppWD0naHR0cDovL25zLmFkb2JlLmNv +bS9pWC8xLjAvJz4NCjxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0PSd1dWlkOjY5 +YmJhMDBhLTdjYzUtNGEwMS04MjNmLTUxMzM0Y2EzZmEwNicgeG1sbnM6cGRmPSdo +dHRwOi8vbnMuYWRvYmUuY29tL3BkZi8xLjMvJyBwZGY6UHJvZHVjZXI9J0Fjcm9i +YXQgRGlzdGlsbGVyIDYuMC4xIChXaW5kb3dzKSc+PC9yZGY6RGVzY3JpcHRpb24+ +DQo8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0ndXVpZDo2OWJiYTAwYS03Y2M1 +LTRhMDEtODIzZi01MTMzNGNhM2ZhMDYnIHhtbG5zOnhhcD0naHR0cDovL25zLmFk +b2JlLmNvbS94YXAvMS4wLycgeGFwOk1vZGlmeURhdGU9JzIwMDQtMTAtMjhUMTg6 +MTc6NDYrMTA6MDAnIHhhcDpDcmVhdGVEYXRlPScyMDA0LTEwLTI4VDE4OjEzOjM4 +KzEwOjAwJyB4YXA6TWV0YWRhdGFEYXRlPScyMDA0LTEwLTI4VDE4OjE3OjQ2KzEw +OjAwJyB4YXA6Q3JlYXRvclRvb2w9J1BTY3JpcHQ1LmRsbCBWZXJzaW9uIDUuMi4y +Jz48L3JkZjpEZXNjcmlwdGlvbj4NCjxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0 +PSd1dWlkOjY5YmJhMDBhLTdjYzUtNGEwMS04MjNmLTUxMzM0Y2EzZmEwNicgeG1s +bnM6eGFwTU09J2h0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8nIHhhcE1N +OkRvY3VtZW50SUQ9J3V1aWQ6MmNhMjBjZmEtM2FlZi00MTU1LThkOWMtYTM4ZWJj +NDM3MjVmJy8+DQo8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0ndXVpZDo2OWJi +YTAwYS03Y2M1LTRhMDEtODIzZi01MTMzNGNhM2ZhMDYnIHhtbG5zOmRjPSdodHRw +Oi8vcHVybC5vcmcvZGMvZWxlbWVudHMvMS4xLycgZGM6Zm9ybWF0PSdhcHBsaWNh +dGlvbi9wZGYnPjxkYzp0aXRsZT48cmRmOkFsdD48cmRmOmxpIHhtbDpsYW5nPSd4 +LWRlZmF1bHQnPlBsYW5ldCBQREYgSmF2YVNjcmlwdCBMZWFybmluZyBDZW50ZXIg +RXhhbXBsZSAjMjwvcmRmOmxpPjwvcmRmOkFsdD48L2RjOnRpdGxlPjxkYzpjcmVh +dG9yPjxyZGY6U2VxPjxyZGY6bGk+Q2hyaXMgRGFobCwgQVJUUyBQREYgR2xvYmFs +IFNlcnZpY2VzPC9yZGY6bGk+PC9yZGY6U2VxPjwvZGM6Y3JlYXRvcj48L3JkZjpE +ZXNjcmlwdGlvbj4NCjwvcmRmOlJERj4NCjwveDp4bXBtZXRhPg0KICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAg +ICAgICAgICAgICAgICAgICAgICAgIAo8P3hwYWNrZXQgZW5kPSd3Jz8+DQplbmRz +dHJlYW0NZW5kb2JqDXhyZWYNCjUgMQ0KMDAwMDA3OTg4NSAwMDAwMCBuDQo3IDEN +CjAwMDAwODAxNTggMDAwMDAgbg0KMzYgMQ0KMDAwMDA4MDMwMiAwMDAwMCBuDQo0 +NCA0DQowMDAwMDgwNDkxIDAwMDAwIG4NCjAwMDAwODA1MTIgMDAwMDAgbg0KMDAw +MDA4MDgzNSAwMDAwMCBuDQowMDAwMDgwODkzIDAwMDAwIG4NCnRyYWlsZXINCjw8 +L1NpemUgNDgvUm9vdCA3IDAgUi9JbmZvIDUgMCBSL0lEWzw1NDhhMTAxOTE5ZTE2 +YjVmZGE5MzJlNzJmYWM0NjJiNz48ZTVlN2QzZmFiM2MyYzA0MDhmOTZmZDU1ZmE2 +NzgzZWI+XS9QcmV2IDc4ODUwID4+DQpzdGFydHhyZWYNCjg0Mzk3DQolJUVPRg0K + diff --git a/test/functional/messages/url10.eml b/test/functional/messages/url10.eml new file mode 100644 index 000000000..3371def85 --- /dev/null +++ b/test/functional/messages/url10.eml @@ -0,0 +1,6 @@ +Content-Type: text/html +Content-Transfer-Encoding: quoted-printable +From: user@example.com +To: undisclosed-recipients;; + +<a href="https:/\test­test.­com/redirect?url=https%3A%2F%2Fexample%2Ecom&urlhash=rH0t#100xp@example.com">click here!</a>
\ No newline at end of file diff --git a/test/lua/unit/base64.lua b/test/lua/unit/base64.lua index 5e22e188c..43606e91e 100644 --- a/test/lua/unit/base64.lua +++ b/test/lua/unit/base64.lua @@ -1,6 +1,7 @@ context("Base64 encoding", function() local ffi = require("ffi") local util = require("rspamd_util") + local logger = require "rspamd_logger" ffi.cdef[[ void rspamd_cryptobox_init (void); void ottery_rand_bytes(void *buf, size_t n); @@ -10,8 +11,8 @@ context("Base64 encoding", function() size_t str_len, size_t *outlen); void g_free(void *ptr); int memcmp(const void *a1, const void *a2, size_t len); - size_t base64_test (bool generic, size_t niters, size_t len); - double rspamd_get_ticks (void); + double base64_test (bool generic, size_t niters, size_t len, size_t str_len); + double rspamd_get_ticks (int); ]] ffi.C.rspamd_cryptobox_init() @@ -98,7 +99,7 @@ vehemence of any carnal pleasure.]] assert_equal(orig, tostring(dec), "fuzz test failed for length: " .. #orig) end end) - test("Base64 fuzz test (ffi)", function() + test("Base64 fuzz test (ffi)", function() for i = 1,1000 do local b, l = random_buf(4096) local nl = ffi.new("size_t [1]") @@ -117,52 +118,75 @@ vehemence of any carnal pleasure.]] local speed_iters = 10000 - test("Base64 test reference vectors 1K", function() - local t1 = ffi.C.rspamd_get_ticks() - local res = ffi.C.base64_test(true, speed_iters, 1024) - local t2 = ffi.C.rspamd_get_ticks() + local function perform_base64_speed_test(chunk, is_reference, line_len) + local ticks = ffi.C.base64_test(is_reference, speed_iters, chunk, line_len) + local what = 'Optimized' + if is_reference then + what = 'Reference' + end + logger.messagex("%s base64 %s chunk (%s line len): %s ticks per iter, %s ticks per byte", + what, chunk, line_len, + ticks / speed_iters, ticks / speed_iters / chunk) - print("Reference base64 (1K): " .. tostring(t2 - t1) .. " sec") + return 1 + end + test("Base64 test reference vectors 78", function() + local res = perform_base64_speed_test(78, true, 0) assert_not_equal(res, 0) end) - test("Base64 test optimized vectors 1K", function() - local t1 = ffi.C.rspamd_get_ticks() - local res = ffi.C.base64_test(false, speed_iters, 1024) - local t2 = ffi.C.rspamd_get_ticks() - - print("Optimized base64 (1K): " .. tostring(t2 - t1) .. " sec") + test("Base64 test optimized vectors 78", function() + local res = perform_base64_speed_test(78, false, 0) assert_not_equal(res, 0) end) - test("Base64 test reference vectors 512", function() - local t1 = ffi.C.rspamd_get_ticks() - local res = ffi.C.base64_test(true, speed_iters, 512) - local t2 = ffi.C.rspamd_get_ticks() - print("Reference base64 (512): " .. tostring(t2 - t1) .. " sec") + test("Base64 test reference vectors 512", function() + local res = perform_base64_speed_test(512, true, 0) assert_not_equal(res, 0) end) test("Base64 test optimized vectors 512", function() - local t1 = ffi.C.rspamd_get_ticks() - local res = ffi.C.base64_test(false, speed_iters, 512) - local t2 = ffi.C.rspamd_get_ticks() + local res = perform_base64_speed_test(512, false, 0) + assert_not_equal(res, 0) + end) + test("Base64 test reference vectors 512 (78 line len)", function() + local res = perform_base64_speed_test(512, true, 78) + assert_not_equal(res, 0) + end) + test("Base64 test optimized vectors 512 (78 line len)", function() + local res = perform_base64_speed_test(512, false, 78) + assert_not_equal(res, 0) + end) - print("Optimized base64 (512): " .. tostring(t2 - t1) .. " sec") + test("Base64 test reference vectors 1K", function() + local res = perform_base64_speed_test(1024, true, 0) + assert_not_equal(res, 0) + end) + test("Base64 test optimized vectors 1K", function() + local res = perform_base64_speed_test(1024, false, 0) + assert_not_equal(res, 0) + end) + test("Base64 test reference vectors 1K (78 line len)", function() + local res = perform_base64_speed_test(1024, true, 78) + assert_not_equal(res, 0) + end) + test("Base64 test optimized vectors 1K (78 line len)", function() + local res = perform_base64_speed_test(1024, false, 78) assert_not_equal(res, 0) end) - test("Base64 test reference vectors 10K", function() - local t1 = ffi.C.rspamd_get_ticks() - local res = ffi.C.base64_test(true, speed_iters / 100, 10240) - local t2 = ffi.C.rspamd_get_ticks() - print("Reference base64 (10K): " .. tostring(t2 - t1) .. " sec") + test("Base64 test reference vectors 10K", function() + local res = perform_base64_speed_test(10 * 1024, true, 0) assert_not_equal(res, 0) end) test("Base64 test optimized vectors 10K", function() - local t1 = ffi.C.rspamd_get_ticks() - local res = ffi.C.base64_test(false, speed_iters / 100, 10240) - local t2 = ffi.C.rspamd_get_ticks() - - print("Optimized base64 (10K): " .. tostring(t2 - t1) .. " sec") + local res = perform_base64_speed_test(10 * 1024, false, 0) + assert_not_equal(res, 0) + end) + test("Base64 test reference vectors 10K (78 line len)", function() + local res = perform_base64_speed_test(10 * 1024, true, 78) + assert_not_equal(res, 0) + end) + test("Base64 test optimized vectors 10K (78 line len)", function() + local res = perform_base64_speed_test(10 * 1024, false, 78) assert_not_equal(res, 0) end) end) diff --git a/test/lua/unit/rfc2047.lua b/test/lua/unit/rfc2047.lua index 0b8baf624..6054c5ca2 100644 --- a/test/lua/unit/rfc2047.lua +++ b/test/lua/unit/rfc2047.lua @@ -29,7 +29,7 @@ context("RFC2047 decoding", function() ffi.cdef[[ const char * rspamd_mime_header_decode (void *pool, const char *in, size_t inlen); - void * rspamd_mempool_new_ (size_t sz, const char *name, const char *strloc); + void * rspamd_mempool_new_ (size_t sz, const char *name, int flags, const char *strloc); void rspamd_mempool_delete (void *pool); ]] @@ -46,7 +46,7 @@ context("RFC2047 decoding", function() {"=?windows-1251?B?xO7q8+zl7fIuc2NyLnV1ZQ==?=", "Документ.scr.uue"}, } - local pool = ffi.C.rspamd_mempool_new_(4096, "lua", "rfc2047.lua:49") + local pool = ffi.C.rspamd_mempool_new_(4096, "lua", 0, "rfc2047.lua:49") for _,c in ipairs(cases) do local res = ffi.C.rspamd_mime_header_decode(pool, c[1], #c[1]) @@ -60,7 +60,7 @@ context("RFC2047 decoding", function() end) test("Fuzz test for rfc2047 tokens", function() local util = require("rspamd_util") - local pool = ffi.C.rspamd_mempool_new_(4096, "lua", "rfc2047.lua:63") + local pool = ffi.C.rspamd_mempool_new_(4096, "lua", 0, "rfc2047.lua:63") local str = "Тест Тест Тест Тест Тест" for i = 0,1000 do diff --git a/test/lua/unit/selectors.lua b/test/lua/unit/selectors.lua index e8e8c0b47..0aa0bab47 100644 --- a/test/lua/unit/selectors.lua +++ b/test/lua/unit/selectors.lua @@ -141,7 +141,17 @@ context("Selectors test", function() ["received by hostname"] = { selector = "received:by_hostname", - expect = {{"server.chat-met-vreemden.nl"}}}, + expect = {{"server1.chat-met-vreemden.nl", "server2.chat-met-vreemden.nl"}}}, + + ["received by hostname last"] = { + selector = "received:by_hostname.last", + expect = {"server2.chat-met-vreemden.nl"} + }, + + ["received by hostname first"] = { + selector = "received:by_hostname.first", + expect = {"server1.chat-met-vreemden.nl"} + }, ["urls"] = { selector = "urls", @@ -247,6 +257,10 @@ context("Selectors test", function() selector = "rcpts.nth(2).lower", expect = {'no-one@example.com'}}, + ["transformation last"] = { + selector = "rcpts.last.lower", + expect = {'no-one@example.com'}}, + ["transformation substring"] = { selector = "header(Subject, strong).substring(6)", expect = {'subject'}}, @@ -305,8 +319,12 @@ end) --[=========[ ******************* message ******************* ]=========] msg = [[ -Received: from ca-18-193-131.service.infuturo.it ([151.18.193.131] helo=User) - by server.chat-met-vreemden.nl with esmtpa (Exim 4.76) +Received: from ca-18-193-131.service1.infuturo.it ([151.18.193.131] helo=User) + by server1.chat-met-vreemden.nl with esmtpa (Exim 4.76) + (envelope-from <upwest201diana@outlook.com>) + id 1ZC1sl-0006b4-TU; Mon, 06 Jul 2015 10:36:08 +0200 +Received: from ca-18-193-131.service2.infuturo.it ([151.18.193.132] helo=User) + by server2.chat-met-vreemden.nl with esmtpa (Exim 4.76) (envelope-from <upwest201diana@outlook.com>) id 1ZC1sl-0006b4-TU; Mon, 06 Jul 2015 10:36:08 +0200 From: <whoknows@nowhere.com> diff --git a/test/lua/unit/url.lua b/test/lua/unit/url.lua index a748c4de8..7f337c8b2 100644 --- a/test/lua/unit/url.lua +++ b/test/lua/unit/url.lua @@ -25,6 +25,8 @@ context("URL check functions", function() {"http://user:password@тест2.РФ:18 text", {"тест2.рф", "user"}}, {"somebody@example.com", {"example.com", "somebody"}}, {"https://127.0.0.1/abc text", {"127.0.0.1", nil}}, + {"https:\\\\127.0.0.1/abc text", {"127.0.0.1", nil}}, + {"https:\\\\127.0.0.1", {"127.0.0.1", nil}}, {"https://127.0.0.1 text", {"127.0.0.1", nil}}, {"https://[::1]:1", {"::1", nil}}, {"https://user:password@[::1]:1", {"::1", nil}}, @@ -54,10 +56,22 @@ context("URL check functions", function() end cases = { - {"http://%30%78%63%30%2e%30%32%35%30.01", true, { --0xc0.0250.01 + {'http://example.net/?arg=%23#fragment', true, { + host = 'example.net', fragment = 'fragment', query = 'arg=#', + }}, + {"http:/\\[::eeee:192.168.0.1]/#test", true, { + host = '::eeee:c0a8:1', fragment = 'test' + }}, + {"http:/\\[::eeee:192.168.0.1]#test", true, { + host = '::eeee:c0a8:1', fragment = 'test' + }}, + {"http:/\\[::eeee:192.168.0.1]?test", true, { + host = '::eeee:c0a8:1', query = 'test' + }}, + {"http:\\\\%30%78%63%30%2e%30%32%35%30.01", true, { --0xc0.0250.01 host = '192.168.0.1', }}, - {"http://www.google.com/foo?bar=baz#", true, { + {"http:/\\www.google.com/foo?bar=baz#", true, { host = 'www.google.com', path = 'foo', query = 'bar=baz', tld = 'google.com' }}, {"http://[www.google.com]/", false}, @@ -78,17 +92,14 @@ context("URL check functions", function() {"http://0.0xFFFFFF", true, { host = '0.255.255.255' }}, - {"http://030052000001", true, { - host = '192.168.0.1' - }}, - {"http://0xc0.052000001", true, { + {"http:/\\030052000001", true, { host = '192.168.0.1' }}, - {"http://192.168.0.1.", true, { + {"http:\\/0xc0.052000001", true, { host = '192.168.0.1' }}, - {"http://[::eeee:192.168.0.1]", true, { - host = '::eeee:c0a8:1' + {"http://192.168.0.1.?foo", true, { + host = '192.168.0.1', query = 'foo', }}, {"http://twitter.com#test", true, { host = 'twitter.com', fragment = 'test' @@ -102,9 +113,9 @@ context("URL check functions", function() for i,c in ipairs(cases) do local res = url.create(pool, c[1]) - test("Parse urls " .. i, function() + test("Parse url: " .. c[1], function() if c[2] then - assert_not_nil(res, "cannot parse " .. c[1]) + assert_not_nil(res, "we are able to parse url: " .. c[1]) local uf = res:to_table() diff --git a/test/lua/unit/utf.lua b/test/lua/unit/utf.lua index 75dd33977..34217afa4 100644 --- a/test/lua/unit/utf.lua +++ b/test/lua/unit/utf.lua @@ -5,7 +5,14 @@ context("UTF8 check functions", function() ffi.cdef[[ unsigned int rspamd_str_lc_utf8 (char *str, unsigned int size); unsigned int rspamd_str_lc (char *str, unsigned int size); - char * rspamd_str_make_utf_valid (const char *src, size_t slen, size_t *dstlen); + void rspamd_fast_utf8_library_init (unsigned flags); + void ottery_rand_bytes(void *buf, size_t n); + double rspamd_get_ticks(int allow); + size_t rspamd_fast_utf8_validate (const unsigned char *data, size_t len); + size_t rspamd_fast_utf8_validate_ref (const unsigned char *data, size_t len); + size_t rspamd_fast_utf8_validate_sse41 (const unsigned char *data, size_t len); + size_t rspamd_fast_utf8_validate_avx2 (const unsigned char *data, size_t len); + char * rspamd_str_make_utf_valid (const char *src, size_t slen, size_t *dstlen, void *); ]] local cases = { @@ -58,7 +65,7 @@ context("UTF8 check functions", function() local buf = ffi.new("char[?]", #c[1] + 1) ffi.copy(buf, c[1]) - local s = ffi.string(ffi.C.rspamd_str_make_utf_valid(buf, #c[1], NULL)) + local s = ffi.string(ffi.C.rspamd_str_make_utf_valid(buf, #c[1], NULL, NULL)) local function to_hex(s) return (s:gsub('.', function (c) return string.format('%02X', string.byte(c)) @@ -69,4 +76,127 @@ context("UTF8 check functions", function() assert_equal(s, c[2]) end) end + + -- Enable sse and avx2 + ffi.C.rspamd_fast_utf8_library_init(3) + local valid_cases = { + "a", + "\xc3\xb1", + "\xe2\x82\xa1", + "\xf0\x90\x8c\xbc", + "안녕하세요, 세상" + } + for i,c in ipairs(valid_cases) do + test("Unicode validate success: " .. tostring(i), function() + local buf = ffi.new("char[?]", #c + 1) + ffi.copy(buf, c) + + local ret = ffi.C.rspamd_fast_utf8_validate(buf, #c) + assert_equal(ret, 0) + end) + end + local invalid_cases = { + "\xc3\x28", + "\xa0\xa1", + "\xe2\x28\xa1", + "\xe2\x82\x28", + "\xf0\x28\x8c\xbc", + "\xf0\x90\x28\xbc", + "\xf0\x28\x8c\x28", + "\xc0\x9f", + "\xf5\xff\xff\xff", + "\xed\xa0\x81", + "\xf8\x90\x80\x80\x80", + "123456789012345\xed", + "123456789012345\xf1", + "123456789012345\xc2", + "\xC2\x7F" + } + for i,c in ipairs(invalid_cases) do + test("Unicode validate fail: " .. tostring(i), function() + local buf = ffi.new("char[?]", #c + 1) + ffi.copy(buf, c) + + local ret = ffi.C.rspamd_fast_utf8_validate(buf, #c) + assert_not_equal(ret, 0) + end) + end + + local speed_iters = 10000 + local function test_size(buflen, is_valid, impl) + local logger = require "rspamd_logger" + local test_str + if is_valid then + test_str = table.concat(valid_cases) + else + test_str = table.concat(valid_cases) .. table.concat(invalid_cases) + end + + local buf = ffi.new("char[?]", buflen) + if #test_str < buflen then + local t = {} + local len = #test_str + while len < buflen do + t[#t + 1] = test_str + len = len + #test_str + end + test_str = table.concat(t) + end + ffi.copy(buf, test_str:sub(1, buflen)) + + local tm = 0 + + for _=1,speed_iters do + if impl == 'ref' then + local t1 = ffi.C.rspamd_get_ticks(1) + ffi.C.rspamd_fast_utf8_validate_ref(buf, buflen) + local t2 = ffi.C.rspamd_get_ticks(1) + tm = tm + (t2 - t1) + elseif impl == 'sse' then + local t1 = ffi.C.rspamd_get_ticks(1) + ffi.C.rspamd_fast_utf8_validate_sse41(buf, buflen) + local t2 = ffi.C.rspamd_get_ticks(1) + tm = tm + (t2 - t1) + else + local t1 = ffi.C.rspamd_get_ticks(1) + ffi.C.rspamd_fast_utf8_validate_avx2(buf, buflen) + local t2 = ffi.C.rspamd_get_ticks(1) + tm = tm + (t2 - t1) + end + end + + logger.messagex("%s utf8 %s check (valid = %s): %s ticks per iter, %s ticks per byte", + impl, buflen, is_valid, + tm / speed_iters, tm / speed_iters / buflen) + + return 0 + end + + for _,sz in ipairs({78, 512, 65535}) do + test(string.format("Utf8 test %s %d buffer, %s", 'ref', sz, 'valid'), function() + local res = test_size(sz, true, 'ref') + assert_equal(res, 0) + end) + test(string.format("Utf8 test %s %d buffer, %s", 'ref', sz, 'invalid'), function() + local res = test_size(sz, false, 'ref') + assert_equal(res, 0) + end) + test(string.format("Utf8 test %s %d buffer, %s", 'sse', sz, 'valid'), function() + local res = test_size(sz, true, 'sse') + assert_equal(res, 0) + end) + test(string.format("Utf8 test %s %d buffer, %s", 'sse', sz, 'invalid'), function() + local res = test_size(sz, false, 'sse') + assert_equal(res, 0) + end) + test(string.format("Utf8 test %s %d buffer, %s", 'avx2', sz, 'valid'), function() + local res = test_size(sz, true, 'avx2') + assert_equal(res, 0) + end) + test(string.format("Utf8 test %s %d buffer, %s", 'avx2', sz, 'invalid'), function() + local res = test_size(sz, false, 'avx2') + assert_equal(res, 0) + end) + end + end)
\ No newline at end of file diff --git a/test/rspamd_dns_test.c b/test/rspamd_dns_test.c index 6b12746ae..25c28528e 100644 --- a/test/rspamd_dns_test.c +++ b/test/rspamd_dns_test.c @@ -70,11 +70,11 @@ rspamd_dns_test_func () cfg = (struct rspamd_config *)g_malloc (sizeof (struct rspamd_config)); bzero (cfg, sizeof (struct rspamd_config)); - cfg->cfg_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL); + cfg->cfg_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL, 0); cfg->dns_retransmits = 2; cfg->dns_timeout = 0.5; - pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL); + pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL, 0); s = rspamd_session_create (pool, session_fin, NULL, NULL, NULL); diff --git a/test/rspamd_mem_pool_test.c b/test/rspamd_mem_pool_test.c index 61b9a7a94..61b4efe3f 100644 --- a/test/rspamd_mem_pool_test.c +++ b/test/rspamd_mem_pool_test.c @@ -20,7 +20,7 @@ rspamd_mem_pool_test_func () pid_t pid; int ret; - pool = rspamd_mempool_new (sizeof (TEST_BUF), NULL); + pool = rspamd_mempool_new (sizeof (TEST_BUF), NULL, 0); tmp = rspamd_mempool_alloc (pool, sizeof (TEST_BUF)); tmp2 = rspamd_mempool_alloc (pool, sizeof (TEST_BUF) * 2); tmp3 = rspamd_mempool_alloc_shared (pool, sizeof (TEST_BUF)); @@ -32,8 +32,8 @@ rspamd_mem_pool_test_func () g_assert (strncmp (tmp, TEST_BUF, sizeof (TEST_BUF)) == 0); g_assert (strncmp (tmp2, TEST2_BUF, sizeof (TEST2_BUF)) == 0); g_assert (strncmp (tmp3, TEST_BUF, sizeof (TEST_BUF)) == 0); - + rspamd_mempool_delete (pool); rspamd_mempool_stat (&st); - + } diff --git a/test/rspamd_radix_test.c b/test/rspamd_radix_test.c index 5a42b36e5..dda0bff36 100644 --- a/test/rspamd_radix_test.c +++ b/test/rspamd_radix_test.c @@ -148,7 +148,7 @@ rspamd_btrie_test_vec (void) gsize i; gpointer val; - pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "btrie"); + pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "btrie", 0); tree = btrie_init (pool); while (t->ip != NULL) { @@ -238,7 +238,7 @@ rspamd_radix_test_func (void) addrs[i].mask6 = ottery_rand_range(128); } - pool = rspamd_mempool_new (65536, "btrie"); + pool = rspamd_mempool_new (65536, "btrie", 0); btrie = btrie_init (pool); msg_notice ("btrie performance (%z elts)", nelts); diff --git a/test/rspamd_test_suite.c b/test/rspamd_test_suite.c index db12a1a0c..d7b660642 100644 --- a/test/rspamd_test_suite.c +++ b/test/rspamd_test_suite.c @@ -31,11 +31,11 @@ main (int argc, char **argv) rspamd_main = (struct rspamd_main *)g_malloc (sizeof (struct rspamd_main)); memset (rspamd_main, 0, sizeof (struct rspamd_main)); - rspamd_main->server_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL); + rspamd_main->server_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL, 0); cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_DEFAULT); cfg->libs_ctx = rspamd_init_libs (); rspamd_main->cfg = cfg; - cfg->cfg_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL); + cfg->cfg_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL, 0); cfg->log_type = RSPAMD_LOG_CONSOLE; cfg->log_level = G_LOG_LEVEL_MESSAGE; diff --git a/test/rspamd_upstream_test.c b/test/rspamd_upstream_test.c index bbe47fc04..263a28e75 100644 --- a/test/rspamd_upstream_test.c +++ b/test/rspamd_upstream_test.c @@ -106,7 +106,7 @@ rspamd_upstream_test_func (void) next_addr = rspamd_upstream_addr_next (up); g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET6); /* Test errors with IPv6 */ - rspamd_upstream_fail (up, TRUE); + rspamd_upstream_fail (up, TRUE, NULL); /* Now we should have merely IPv4 addresses in rotation */ addr = rspamd_upstream_addr_next (up); for (i = 0; i < 256; i++) { @@ -169,7 +169,7 @@ rspamd_upstream_test_func (void) up = rspamd_upstream_get (ls, RSPAMD_UPSTREAM_MASTER_SLAVE, NULL, 0); for (i = 0; i < 100; i ++) { - rspamd_upstream_fail (up, TRUE); + rspamd_upstream_fail (up, TRUE, NULL); } g_assert (rspamd_upstreams_alive (ls) == 2); diff --git a/utils/cgp_rspamd.pl b/utils/cgp_rspamd.pl index 0070cf4a5..6898d268f 100644 --- a/utils/cgp_rspamd.pl +++ b/utils/cgp_rspamd.pl @@ -137,7 +137,7 @@ sub rspamd_scan { } } elsif ( $action eq 'soft reject' ) { - print "$tag REJECT Try again later\n"; + print "$tag REJECTED Try again later\n"; return; } |