diff options
| author | Andrew Lewis <nerf@judo.za.org> | 2014-01-01 18:00:55 +0200 |
|---|---|---|
| committer | Andrew Lewis <nerf@judo.za.org> | 2014-01-01 18:00:55 +0200 |
| commit | b9a65473e352d23485377ca9f895e2716cd624fe (patch) | |
| tree | 35dc4e2973153eb00b5979098de17631b7d6495b | |
| parent | 932f68cf1607335a4cd820b781466b8545b1a084 (diff) | |
| parent | 3d97b8ca01fea56ba9371cb3992a9cb2c791030e (diff) | |
| download | rspamd-b9a65473e352d23485377ca9f895e2716cd624fe.tar.gz rspamd-b9a65473e352d23485377ca9f895e2716cd624fe.zip | |
Merge remote-tracking branch 'upstream/master'
99 files changed, 10099 insertions, 4498 deletions
@@ -45,3 +45,12 @@ bf9307c5d78bd9b06e9fcae008db959aad1e62d2 0.5.4 34ec644af33ca2040b6e24f5ceadd311b502618e 0.5.6 e11d0fa6d58038a59ed5ed5cba04ec2b1089d61b 0.6.0 fd359d2ef59b1c0e35ca6c1db7f0b4cb5cf7e9fd 0.6.1 +e34a90b92b248cca1b8cb03052084e5e441ec21d 0.6.2 +28685d33512977c42fdfb3000ee05a28f6097794 0.6.3 +7efe36ce9272d6c6c521041ec8c587ee79f7ba7e 0.6.4 +7efe36ce9272d6c6c521041ec8c587ee79f7ba7e 0.6.4 +0000000000000000000000000000000000000000 0.6.4 +0000000000000000000000000000000000000000 0.6.4 +93e5d880b3898e8531f5968c295fca4bec845d68 0.6.4 +d7ffd19265d09c7b5b13409a730a3a987ea36f26 0.6.5 +3d18c18889ffd933cbfde99ede231f8869c3b8d8 0.6.6 diff --git a/CMakeLists.txt b/CMakeLists.txt index 8937fe937..ffbc24974 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,10 +10,10 @@ PROJECT(rspamd C) SET(RSPAMD_VERSION_MAJOR 0) SET(RSPAMD_VERSION_MINOR 6) -SET(RSPAMD_VERSION_PATCH 1) +SET(RSPAMD_VERSION_PATCH 6) SET(RSPAMD_VERSION "${RSPAMD_VERSION_MAJOR}.${RSPAMD_VERSION_MINOR}.${RSPAMD_VERSION_PATCH}") -SET(RSPAMD_MASTER_SITE_URL "http://bitbucket.org/vstakhov/rspamd") +SET(RSPAMD_MASTER_SITE_URL "https://rspamd.com") IF(NOT RSPAMD_USER) SET(RSPAMD_USER "nobody") @@ -28,7 +28,6 @@ OPTION(DEBUG_MODE "Enable debug output [default: ON]" OPTION(ENABLE_OPTIMIZATION "Enable optimization [default: OFF]" OFF) OPTION(SKIP_RELINK_RPATH "Skip relinking and full RPATH for the install tree" OFF) OPTION(ENABLE_REDIRECTOR "Enable redirector install [default: OFF]" OFF) -OPTION(ENABLE_PROFILING "Enable profiling [default: OFF]" OFF) OPTION(ENABLE_GPERF_TOOLS "Enable google perftools [default: OFF]" OFF) OPTION(ENABLE_STATIC "Enable static compiling [default: OFF]" OFF) OPTION(ENABLE_LUAJIT "Link with libluajit [default: OFF]" OFF) @@ -36,8 +35,10 @@ OPTION(ENABLE_JUDY "Find and link with Judy library [default: ON]" OPTION(ENABLE_DB "Find and link with DB library [default: OFF]" OFF) OPTION(ENABLE_SQLITE "Find and link with sqlite3 library [default: OFF]" OFF) OPTION(ENABLE_HIREDIS "Find and link with external redis library [default: OFF]" OFF) +OPTION(ENABLE_URL_INCLUDE "Enable urls in ucl includes (requires libcurl or libfetch) [default: OFF]" OFF) OPTION(NO_SHARED "Build internal libs static [default: OFF]" OFF) OPTION(FORCE_GMIME24 "Link with gmime2.4 [default: OFF]" OFF) +OPTION(INSTALL_EXAMPLES "Install examples [default: OFF]" OFF) # Build optimized code for following CPU (default i386) #SET(CPU_TUNE "i686") @@ -159,6 +160,7 @@ MACRO(AddModules MLIST WLIST) #ENDIF(NOT EXISTS "src/modules.c") ENDMACRO(AddModules MLIST WLIST) +# Find lua installation MACRO(FindLua _major _minor) # Find lua libraries MESSAGE(STATUS "Check for lua ${_major}.${_minor}") @@ -260,6 +262,33 @@ FUNCTION(INSTALL_IF_NOT_EXISTS src dest suffix) ") ENDFUNCTION(INSTALL_IF_NOT_EXISTS) +# Process required package by using FindPackage and calling for INCLUDE_DIRECTORIES and +# setting list of required libraries +MACRO(ProcessPackage var _name0) + PKG_SEARCH_MODULE(${var} REQUIRED "${_name0}" ${ARGN}) + IF(${var}_FOUND) + SET(WITH_${var} 1) + IF(ENABLE_STATIC MATCHES "ON") + SET(_XPREFIX "${var}_STATIC") + ELSE(ENABLE_STATIC MATCHES "ON") + SET(_XPREFIX "${var}") + ENDIF(ENABLE_STATIC MATCHES "ON") + FOREACH(_arg ${${_XPREFIX}_INCLUDE_DIRS}) + INCLUDE_DIRECTORIES("${_arg}") + ENDFOREACH(_arg ${${_XPREFIX}_INCLUDE_DIRS}) + FOREACH(_arg ${${_XPREFIX}_LIBRARY_DIRS}) + LINK_DIRECTORIES("${_arg}") + ENDFOREACH(_arg ${${_XPREFIX}_LIBRARY_DIRS}) + # Handle other CFLAGS and LDFLAGS + FOREACH(_arg ${${_XPREFIX}_CFLAGS_OTHER}) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_arg}") + ENDFOREACH(_arg ${${_XPREFIX}_CFLAGS_OTHER}) + FOREACH(_arg ${${_XPREFIX}_LDFLAGS_OTHER}) + SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${_arg}") + ENDFOREACH(_arg ${${_XPREFIX}_LDFLAGS_OTHER}) + LIST(APPEND RSPAMD_REQUIRED_LIBRARIES "${${_XPREFIX}_LIBRARIES}") + ENDIF(${var}_FOUND) +ENDMACRO(ProcessPackage name) ############################# CONFIG SECTION ############################################# # Initial set @@ -296,20 +325,21 @@ IF(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") /usr/lib /usr/local/lib DOC "Path where the libintl library can be found") - LIST(APPEND CMAKE_REQUIRED_LIBRARIES ${LIBINTL_LIBRARY}) + LIST(APPEND RSPAMD_REQUIRED_LIBRARIES ${LIBINTL_LIBRARY}) MESSAGE(STATUS "Configuring for FreeBSD") ENDIF(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") IF(CMAKE_SYSTEM_NAME STREQUAL "Darwin") SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_BSD_SOURCE -DDARWIN") + SET(CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS} -undefined dynamic_lookup") FIND_LIBRARY(LIBINTL_LIBRARY NAMES intl PATHS /lib /opt/lib /usr/lib /usr/local/lib DOC "Path where the libintl library can be found") - LIST(APPEND CMAKE_REQUIRED_LIBRARIES ${LIBINTL_LIBRARY}) + LIST(APPEND RSPAMD_REQUIRED_LIBRARIES ${LIBINTL_LIBRARY}) MESSAGE(STATUS "Configuring for Darwin") ENDIF(CMAKE_SYSTEM_NAME STREQUAL "Darwin") @@ -327,6 +357,10 @@ IF(CMAKE_SYSTEM_NAME STREQUAL "Linux") LIST(APPEND CMAKE_REQUIRED_LIBRARIES rt) LIST(APPEND CMAKE_REQUIRED_LIBRARIES dl) + #XXX: gio bug workaround + IF(ENABLE_STATIC MATCHES "ON") + LIST(APPEND CMAKE_REQUIRED_LIBRARIES selinux) + ENDIF(ENABLE_STATIC MATCHES "ON") MESSAGE(STATUS "Configuring for Linux") IF(EXISTS "/etc/debian_version") SET(LINUX_START_SCRIPT "rspamd_debian.in") @@ -396,27 +430,9 @@ ELSE(NOT LUA_FOUND) INCLUDE_DIRECTORIES("${LUA_INCLUDE_DIR}") ENDIF(NOT LUA_FOUND) -# Check and link to pcre -pkg_check_modules(PCRE REQUIRED libpcre) -IF(PCRE_INCLUDE_DIRS) - INCLUDE_DIRECTORIES("${PCRE_INCLUDE_DIRS}") -ENDIF(PCRE_INCLUDE_DIRS) -IF(PCRE_LIBRARY_DIRS) - LINK_DIRECTORIES("${PCRE_LIBRARY_DIRS}") -ENDIF(PCRE_LIBRARY_DIRS) - IF(ENABLE_SQLITE MATCHES "ON") # Find optional sqlite3 support - pkg_check_modules(SQLITE sqlite3>=3.6.0) - IF(SQLITE_FOUND) - SET(WITH_SQLITE 1) - ENDIF(SQLITE_FOUND) - IF(SQLITE_INCLUDE_DIRS) - INCLUDE_DIRECTORIES("${SQLITE_INCLUDE_DIRS}") - ENDIF(SQLITE_INCLUDE_DIRS) - IF(SQLITE_LIBRARY_DIRS) - LINK_DIRECTORIES("${SQLITE_LIBRARY_DIRS}") - ENDIF(SQLITE_LIBRARY_DIRS) + ProcessPackage(SQLITE sqlite3>=3.6.0) ENDIF(ENABLE_SQLITE MATCHES "ON") #Check for openssl (required for dkim) @@ -425,129 +441,19 @@ IF(OPENSSL_FOUND) INCLUDE_DIRECTORIES("${OPENSSL_INCLUDE_DIR}") ENDIF(OPENSSL_FOUND) -IF(ENABLE_STATIC MATCHES "ON") - pkg_check_modules(GLIB2 REQUIRED glib-2.0>=2.12 gthread-2.0 gmodule-2.0) - SET(LINK_TYPE "STATIC") -ELSE(ENABLE_STATIC MATCHES "ON") - pkg_check_modules(GLIB2 REQUIRED glib-2.0>=2.12 gthread-2.0 gmodule-2.0) - IF(NO_SHARED MATCHES "OFF") - SET(LINK_TYPE "SHARED") - ELSE(NO_SHARED MATCHES "OFF") - SET(LINK_TYPE "STATIC") - ENDIF(NO_SHARED MATCHES "OFF") -ENDIF(ENABLE_STATIC MATCHES "ON") -SET(GLIB2_VERSION "${GLIB2_glib-2.0_VERSION}") - -IF(GLIB2_VERSION VERSION_GREATER "2.30.0") - pkg_check_modules(LIBFFI libffi) - IF(LIBFFI_FOUND) - SET(GLIB2_LDFLAGS "${GLIB2_LDFLAGS};${LIBFFI_LDFLAGS}") - SET(GLIB2_LIBRARIES "${GLIB2_LIBRARIES};${LIBFFI_LIBRARIES}") - SET(GLIB2_STATIC_LDFLAGS "${GLIB2_STATIC_LDFLAGS};${LIBFFI_STATIC_LDFLAGS}") - SET(GLIB2_CFLAGS "${GLIB2_CFLAGS};${LIBFFI_CFLAGS}") - ENDIF(LIBFFI_FOUND) -ENDIF(GLIB2_VERSION VERSION_GREATER "2.30.0") -pkg_check_modules(GMIME2 gmime-2.0) +ProcessPackage(GLIB2 glib-2.0>=2.12) +ProcessPackage(GTHREAD gthread-2.0) IF(ENABLE_HIREDIS MATCHES "ON") # Try to find hiredis library - pkg_check_modules(HIREDIS REQUIRED libhiredis) - IF(HIREDIS_INCLUDE_DIRS) - INCLUDE_DIRECTORIES("${HIREDIS_INCLUDE_DIRS}") - ENDIF(HIREDIS_INCLUDE_DIRS) - IF(HIREDIS_LIBRARY_DIRS) - LINK_DIRECTORIES("${HIREDIS_LIBRARY_DIRS}") - ENDIF(HIREDIS_LIBRARY_DIRS) - IF(HIREDIS_FOUND) - SET(WITH_SYSTEM_HIREDIS 1) - ENDIF(HIREDIS_FOUND) + ProcessPackage(HIREDIS libhiredis) ENDIF(ENABLE_HIREDIS MATCHES "ON") -# Try to link with gmime24 -IF(NOT GMIME2_FOUND OR FORCE_GMIME24) - pkg_check_modules(GMIME24 gmime-2.4) - IF(NOT GMIME24_FOUND) - pkg_check_modules(GMIME24 REQUIRED gmime-2.6) - ENDIF(NOT GMIME24_FOUND) - SET(GMIME24 "yes") - # Gmime2 - FOREACH(arg ${GMIME24_CFLAGS}) - SET(GMIME_CFLAGS "${GMIME_CFLAGS} ${arg}") - ENDFOREACH(arg ${GMIME24_CFLAGS}) - - IF(ENABLE_STATIC MATCHES "ON") - FOREACH(arg ${GMIME24_STATIC_LDFLAGS}) - SET(GMIME_LDFLAGS "${GMIME_LDFLAGS} ${arg}") - ENDFOREACH(arg ${GMIME24_LDFLAGS}) - ELSE(ENABLE_STATIC MATCHES "ON") - FOREACH(arg ${GMIME24_LDFLAGS}) - SET(GMIME_LDFLAGS "${GMIME_LDFLAGS} ${arg}") - ENDFOREACH(arg ${GMIME24_LDFLAGS}) - ENDIF(ENABLE_STATIC MATCHES "ON") - IF(GMIME24_INCLUDE_DIRS) - INCLUDE_DIRECTORIES(${GMIME24_INCLUDE_DIRS}) - ENDIF(GMIME24_INCLUDE_DIRS) - IF(GMIME24_LIBRARY_DIRS) - LINK_DIRECTORIES(${GMIME24_LIBRARY_DIRS}) - ENDIF(GMIME24_LIBRARY_DIRS) -ELSE(NOT GMIME2_FOUND OR FORCE_GMIME24) - # Gmime2 - FOREACH(arg ${GMIME2_CFLAGS}) - SET(GMIME_CFLAGS "${GMIME_CFLAGS} ${arg}") - ENDFOREACH(arg ${GMIME2_CFLAGS}) - - IF(ENABLE_STATIC MATCHES "ON") - FOREACH(arg ${GMIME2_STATIC_LDFLAGS}) - SET(GMIME_LDFLAGS "${GMIME_LDFLAGS} ${arg}") - ENDFOREACH(arg ${GMIME2_LDFLAGS}) - ELSE(ENABLE_STATIC MATCHES "ON") - FOREACH(arg ${GMIME2_LDFLAGS}) - SET(GMIME_LDFLAGS "${GMIME_LDFLAGS} ${arg}") - ENDFOREACH(arg ${GMIME2_LDFLAGS}) - ENDIF(ENABLE_STATIC MATCHES "ON") - IF(GMIME2_INCLUDE_DIRS) - INCLUDE_DIRECTORIES(${GMIME2_INCLUDE_DIRS}) - ENDIF(GMIME2_INCLUDE_DIRS) - IF(GMIME2_LIBRARY_DIRS) - LINK_DIRECTORIES(${GMIME2_LIBRARY_DIRS}) - ENDIF(GMIME2_LIBRARY_DIRS) -ENDIF(NOT GMIME2_FOUND OR FORCE_GMIME24) - -# Make from ; separated list normal space separated list -# Glib2 -FOREACH(arg ${GLIB2_CFLAGS}) - SET(GLIB_CFLAGS "${GLIB_CFLAGS} ${arg}") -ENDFOREACH(arg ${GLIB2_CFLAGS}) - -IF(ENABLE_STATIC MATCHES "ON") - FOREACH(arg ${GLIB2_STATIC_LDFLAGS}) - SET(GLIB_LDFLAGS "${GLIB_LDFLAGS} ${arg}") - ENDFOREACH(arg ${GLIB2_LDFLAGS}) -ELSE(ENABLE_STATIC MATCHES "ON") - FOREACH(arg ${GLIB2_LDFLAGS}) - SET(GLIB_LDFLAGS "${GLIB_LDFLAGS} ${arg}") - ENDFOREACH(arg ${GLIB2_LDFLAGS}) -ENDIF(ENABLE_STATIC MATCHES "ON") +ProcessPackage(GMIME2 gmime-2.6 gmime-2.4 gmime-2.0) +IF(GMIME2_VERSION VERSION_GREATER "2.4.0") + SET(GMIME24 1) +ENDIF(GMIME2_VERSION VERSION_GREATER "2.4.0") - -INCLUDE_DIRECTORIES(${GLIB2_INCLUDE_DIRS}) -LINK_DIRECTORIES(${GLIB2_LIBRARY_DIRS}) - - -FIND_LIBRARY(LIBZ_LIBRARY NAMES z PATH_SUFFIXES lib64 lib - PATHS - ~/Library/Frameworks - /Library/Frameworks - /usr/local - /usr - /sw - /opt/local - /opt/csw - /opt - DOC "Path where the libz library can be found") -IF(NOT LIBZ_LIBRARY) - MESSAGE(FATAL_ERROR "libz is required for libgmime") -ENDIF(NOT LIBZ_LIBRARY) # Check for libevent FIND_LIBRARY(LIBEVENT_LIBRARY NAMES event PATH_SUFFIXES lib64 lib @@ -631,20 +537,6 @@ IF(ENABLE_DB MATCHES "ON") ENDIF(LIBDB_LIBRARY) ENDIF(ENABLE_DB MATCHES "ON") -IF(ENABLE_PROFILING MATCHES "ON") - SET(WITH_PROFILER 1) - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pg") - SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -pg") -ENDIF(ENABLE_PROFILING MATCHES "ON") - -# Static build - -IF(ENABLE_STATIC MATCHES "ON") - SET(BUILD_STATIC 1) - SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static -pthread") - MESSAGE(STATUS "Static build of rspamd, no custom plugins support") -ENDIF(ENABLE_STATIC MATCHES "ON") - # Google performance tools IF(ENABLE_GPERF_TOOLS MATCHES "ON") @@ -692,7 +584,7 @@ IF(LIBUTIL_LIBRARY) /usr/local/include DOC "Path to libutil header") IF(HAVE_LIBUTIL_H) - LIST(APPEND CMAKE_REQUIRED_LIBRARIES util) + LIST(APPEND RSPAMD_REQUIRED_LIBRARIES util) CHECK_FUNCTION_EXISTS(pidfile_open HAVE_PIDFILE) CHECK_FUNCTION_EXISTS(pidfile_fileno HAVE_PIDFILE_FILENO) ENDIF(HAVE_LIBUTIL_H) @@ -700,31 +592,49 @@ ENDIF(LIBUTIL_LIBRARY) # Find libfetch (for FreeBSD) -FIND_LIBRARY(LIBFETCH_LIBRARY NAMES fetch PATHS PATH_SUFFIXES lib64 lib - PATHS - ~/Library/Frameworks - /Library/Frameworks - /usr/local - /usr - /sw - /opt/local - /opt/csw - /opt - DOC "Path where the libfetch library can be found") -IF(LIBFETCH_LIBRARY) - FIND_FILE(HAVE_FETCH_H NAMES fetch.h PATHS /usr/include - /opt/include - /usr/local/include - DOC "Path to libfetch header") -ELSE(LIBFETCH_LIBRARY) - # Try to find libcurl - INCLUDE(FindCURL) - IF(NOT CURL_FOUND) - MESSAGE(WARNING "Neither libcurl nor libfetch were found, no support of URL includes in configuration") - ELSE(NOT CURL_FOUND) - INCLUDE_DIRECTORIES("${CURL_INCLUDE_DIRS}") - ENDIF(NOT CURL_FOUND) -ENDIF(LIBFETCH_LIBRARY) +IF(ENABLE_URL_INCLUDE MATCHES "ON") + FIND_LIBRARY(LIBFETCH_LIBRARY NAMES fetch PATHS PATH_SUFFIXES lib64 lib + PATHS + ~/Library/Frameworks + /Library/Frameworks + /usr/local + /usr + /sw + /opt/local + /opt/csw + /opt + DOC "Path where the libfetch library can be found") + IF(LIBFETCH_LIBRARY) + FIND_FILE(HAVE_FETCH_H NAMES fetch.h PATHS /usr/include + /opt/include + /usr/local/include + DOC "Path to libfetch header") + ELSE(LIBFETCH_LIBRARY) + # Try to find libcurl + ProcessPackage(CURL libcurl) + IF(NOT CURL_FOUND) + MESSAGE(WARNING "Neither libcurl nor libfetch were found, no support of URL includes in configuration") + ENDIF(NOT CURL_FOUND) + ENDIF(LIBFETCH_LIBRARY) +ENDIF(ENABLE_URL_INCLUDE MATCHES "ON") + +# Static build + +IF(ENABLE_STATIC MATCHES "ON") + SET(BUILD_STATIC 1) + SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static -pthread") + MESSAGE(STATUS "Static build of rspamd, no custom plugins support") + SET(LINK_TYPE "STATIC") + SET(NO_SHARED "ON") +ELSE(ENABLE_STATIC MATCHES "ON") + IF(NO_SHARED MATCHES "OFF") + SET(LINK_TYPE "SHARED") + ELSE(NO_SHARED MATCHES "OFF") + SET(LINK_TYPE "STATIC") + ENDIF(NO_SHARED MATCHES "OFF") +ENDIF(ENABLE_STATIC MATCHES "ON") + +LIST(APPEND RSPAMD_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}") # Process with warn flags SET(CMAKE_C_WARN_FLAGS "") @@ -862,6 +772,7 @@ CHECK_FUNCTION_EXISTS(wait4 HAVE_WAIT4) CHECK_FUNCTION_EXISTS(waitpid HAVE_WAITPID) CHECK_FUNCTION_EXISTS(flock HAVE_FLOCK) CHECK_FUNCTION_EXISTS(tanhl HAVE_TANHL) +CHECK_FUNCTION_EXISTS(tanh HAVE_TANH) CHECK_FUNCTION_EXISTS(expl HAVE_EXPL) CHECK_FUNCTION_EXISTS(exp2l HAVE_EXP2L) CHECK_FUNCTION_EXISTS(sendfile HAVE_SENDFILE) @@ -924,9 +835,6 @@ IF(NOT HAVE_COMPATIBLE_QUEUE_H) INCLUDE_DIRECTORIES(compat) ENDIF(NOT HAVE_COMPATIBLE_QUEUE_H) - -SET(CONTRIBSRC "") - IF(NOT DESTDIR) SET(DESTDIR $ENV{DESTDIR}) ENDIF(NOT DESTDIR) @@ -1038,7 +946,7 @@ SET(CONFFILES ######################### LINK SECTION ############################### -ADD_EXECUTABLE(rspamd ${RSPAMDSRC} ${CONTRIBSRC} ${PLUGINSSRC}) +ADD_EXECUTABLE(rspamd ${RSPAMDSRC} ${PLUGINSSRC}) SET_TARGET_PROPERTIES(rspamd PROPERTIES LINKER_LANGUAGE C) SET_TARGET_PROPERTIES(rspamd PROPERTIES COMPILE_FLAGS "-DRSPAMD_MAIN") IF(NOT DEBIAN_BUILD) @@ -1061,33 +969,14 @@ ENDIF(HAVE_LIBEVENT2) IF(WITH_DB) TARGET_LINK_LIBRARIES(rspamd db) ENDIF(WITH_DB) -IF(SQLITE_LIBRARIES) - TARGET_LINK_LIBRARIES(rspamd ${SQLITE_LIBRARIES}) -ENDIF(SQLITE_LIBRARIES) IF(OPENSSL_FOUND) TARGET_LINK_LIBRARIES(rspamd ${OPENSSL_LIBRARIES}) ENDIF(OPENSSL_FOUND) -TARGET_LINK_LIBRARIES(rspamd ${PCRE_LIBRARIES}) -IF(GMIME24) - TARGET_LINK_LIBRARIES(rspamd ${GMIME24_LIBRARIES}) -ELSE(GMIME24) - TARGET_LINK_LIBRARIES(rspamd ${GMIME2_LIBRARIES}) -ENDIF(GMIME24) -TARGET_LINK_LIBRARIES(rspamd ${GLIB2_LIBRARIES}) IF(HAVE_FETCH_H) TARGET_LINK_LIBRARIES(rspamd fetch) -ELSE(HAVE_FETCH_H) - IF(CURL_FOUND) - TARGET_LINK_LIBRARIES(rspamd ${CURL_LIBRARIES}) - ENDIF(CURL_FOUND) ENDIF(HAVE_FETCH_H) -TARGET_LINK_LIBRARIES(rspamd ${CMAKE_REQUIRED_LIBRARIES}) - -IF(ENABLE_STATIC MATCHES "ON") - TARGET_LINK_LIBRARIES(rspamd ${PCRE_LIBRARIES}) - TARGET_LINK_LIBRARIES(rspamd "z") -ENDIF(ENABLE_STATIC MATCHES "ON") +TARGET_LINK_LIBRARIES(rspamd ${RSPAMD_REQUIRED_LIBRARIES}) IF(ENABLE_LUAJIT MATCHES "ON") TARGET_LINK_LIBRARIES(rspamd "${LUAJIT_LIBRARY}") @@ -1124,6 +1013,9 @@ FOREACH(CONF_IDX RANGE ${CONFLIST_MAX}) ELSE(BUILD_PORT) INSTALL_IF_NOT_EXISTS(${CONF_FILE} ${CONFDIR} "") ENDIF(BUILD_PORT) + IF(INSTALL_EXAMPLES MATCHES "ON") + INSTALL(FILES ${CONF_FILE} DESTINATION ${EXAMPLESDIR}) + ENDIF(INSTALL_EXAMPLES MATCHES "ON") ENDFOREACH(CONF_IDX RANGE ${CONFLIST_MAX}) # Lua plugins @@ -1148,4 +1040,4 @@ ENDFOREACH(LUA_CONF) # Manual pages INSTALL(FILES "doc/rspamd.8" DESTINATION ${MANDIR}/man8) -INSTALL(FILES "doc/rspamc.1" DESTINATION ${MANDIR}/man1)
\ No newline at end of file +INSTALL(FILES "doc/rspamc.1" DESTINATION ${MANDIR}/man1) @@ -1,3 +1,62 @@ +0.6.6: + * Removed issue with BUFSIZ limitation in the controller output + * Simplify logging symbols escaping + * Adjusted weights for several rules + * Improve spamhaus rbl support + * Removed PBL for received headers checks + * Added hfilter module that performs various HELO and IP checks. + * Rspamd can now be reloaded by HUP signal + * Fuzzy storage should expire hashes properly + * Build system has been reworked for better supportof pkg-config + * Various minor bugfixes + +0.6.5: + * Fixed critical bug in DNS resolver, introduced in 0.6.4 + * Improved multimap and rbl plugins to skip + * Add dns_sockets option for tuning sockets per server in DNS resolver + * Improved packages for rspamd + +0.6.4: + * Added io channels for DNS request to balance load and reduce id + collisions chance + * Fixed a bug in SPF filter that may cause core dump in specific + circumstances + * FIxed default config for rbl module + * It is possible to get a list of rspamc commands with their descriptions + * Added SORBS bl to the default config + * 2tld file for surbl module has been significantly extended + * Perl modules has been removed from the code. + * Fixed an issue in libucl when parsing macros + +0.6.3: + * Fixed issues with DNS: + - labels decompression algorithm was fixed; + - added resolve_mx to lua API; + - fixed modules that use DNS. + * Lua modules once_received and emails reworked for new resolver API and UCL. + * Debian package was polished. + * Fixed a bug in fuzzy_check module that prevents correct processing messages + without valid parts. + +0.6.2: + * Fuzzy check module has been reworked: + - now fuzzy_check operates with a group of rules, that define which + servers sre to be checked; + - it is possible to specify read_only groups to avoid learning errors; + - turn fuzzy_check to one_shot mode permanently; + - fuzzy_check module configuration is now incompatible with the previous + versions. + * Imported bugfixes from libucl. + * Fixed whitelist plugin. + * Fixed statfiles resizing. + * Improved logging initialization order. + * Fixed race condition in the controller worker. + +0.6.1: + * Critical bugfixes: + - fixed build system; + - fixed in_class setting in bayes learning; + 0.6.0: * Use UCL instead xml for configuration (https://github.com/vstakhov/libucl) * Fix statistics module normalization diff --git a/centos/rspamd.spec b/centos/rspamd.spec index b7e6259b2..57ab3b071 100644 --- a/centos/rspamd.spec +++ b/centos/rspamd.spec @@ -7,32 +7,43 @@ %define USE_JUDY 0 +%if 0%{?suse_version} +%define __cmake cmake +%define __install install +%define __make make +%define __chown chown +%endif + Name: rspamd -Version: 0.6.1 +Version: 0.6.6 Release: 1 Summary: Rapid spam filtering system Group: System Environment/Daemons # BSD License (two clause) # http://www.freebsd.org/copyright/freebsd-license.html -License: BSD -URL: https://bitbucket.org/vstakhov/rspamd/ +License: BSD2c +URL: https://rspamd.com BuildRoot: %{_tmppath}/%{name}-%{version}-%{release} -%if USE_JUDY -BuildRequires: cmake,glib2-devel,gmime-devel,libevent-devel,openssl-devel,lua-devel,Judy-devel -Requires: glib2,gmime,lua,Judy,libevent +%if "%{USE_JUDY}" == "1" +BuildRequires: cmake,glib2-devel,gmime-devel,libevent-devel,openssl-devel,lua-devel,Judy-devel,pcre-devel +Requires: lua, logrotate %else -BuildRequires: cmake,glib2-devel,gmime-devel,libevent-devel,openssl-devel,lua-devel -Requires: glib2,gmime,lua,libevent +BuildRequires: cmake,glib2-devel,gmime-devel,libevent-devel,openssl-devel,lua-devel,pcre-devel +Requires: lua, logrotate %endif # for /user/sbin/useradd +%if 0%{?suse_version} +Requires(pre): shadow, %insserv_prereq, %fillup_prereq +%else Requires(pre): shadow-utils Requires(post): chkconfig # for /sbin/service Requires(preun): chkconfig, initscripts Requires(postun): initscripts +%endif -Source0: http://cdn.bitbucket.org/vstakhov/rspamd/downloads/%{name}-%{version}.tar.gz +Source0: https://rspamd.com/downloads/%{name}-%{version}.tar.gz Source1: %{name}.init Source2: %{name}.logrotate @@ -45,7 +56,6 @@ lua. %setup -q %build -rm -rf %{buildroot} %{__cmake} \ -DCMAKE_INSTALL_PREFIX=%{_prefix} \ -DCONFDIR=%{_sysconfdir}/rspamd \ @@ -61,7 +71,7 @@ rm -rf %{buildroot} -DDEBIAN_BUILD=1 \ -DRSPAMD_GROUP=%{rspamd_group} \ -DRSPAMD_USER=%{rspamd_user} \ -%if USE_JUDY +%if "%{USE_JUDY}" == "1" -DENABLE_JUDY=ON %else -DENABLE_JUDY=OFF @@ -73,15 +83,36 @@ rm -rf %{buildroot} %{__make} install DESTDIR=%{buildroot} INSTALLDIRS=vendor %{__install} -p -D -m 0755 %{SOURCE1} %{buildroot}%{_initrddir}/%{name} + +%if 0%{?suse_version} +mkdir -p %{buildroot}%{_sbindir} +ln -sf %{_initrddir}/rspamd %{buildroot}%{_sbindir}/rcrspamd +%endif + %{__install} -p -D -m 0644 %{SOURCE2} %{buildroot}%{_sysconfdir}/logrotate.d/%{name} %{__install} -d -p -m 0755 %{buildroot}%{rspamd_logdir} -%{__install} -o %{rspamd_user} -g %{rspamd_group} -d -p -m 0755 %{buildroot}%{rspamd_home} +%{__install} -d -p -m 0755 %{buildroot}%{rspamd_home} %clean rm -rf %{buildroot} %pre -%{_sbindir}/useradd -c "Rspamd user" -s /bin/false -r -d %{rspamd_home} %{rspamd_user} 2>/dev/null || : +%{_sbindir}/groupadd -r %{rspamd_group} 2>/dev/null || : +%{_sbindir}/useradd -g %{rspamd_group} -c "Rspamd user" -s /bin/false -r -d %{rspamd_home} %{rspamd_user} 2>/dev/null || : + +%if 0%{?suse_version} + +%post +%fillup_and_insserv rspamd + +%preun +%stop_on_removal rspamd + +%postun +%restart_on_update rspamd +%insserv_cleanup + +%else %post /sbin/chkconfig --add %{name} @@ -97,6 +128,8 @@ if [ $1 -ge 1 ]; then /sbin/service %{name} condrestart > /dev/null 2>&1 || : fi +%endif + %files %defattr(-,root,root,-) %{_initrddir}/%{name} @@ -114,29 +147,52 @@ fi %config(noreplace) %{rspamd_confdir}/workers.conf %config(noreplace) %{_sysconfdir}/logrotate.d/%{name} %dir %{rspamd_logdir} +%dir %{rspamd_home} +%dir %{rspamd_confdir}/lua/regexp +%dir %{rspamd_confdir}/lua %dir %{rspamd_confdir} -%attr(755, %{rspamd_user}, %{rspamd_group}) %dir %{rspamd_home} +%dir %{rspamd_pluginsdir}/lua +%dir %{rspamd_pluginsdir} %config(noreplace) %{rspamd_confdir}/2tld.inc %config(noreplace) %{rspamd_confdir}/surbl-whitelist.inc -%config(noreplace) %{rspamd_pluginsdir}/lua/forged_recipients.lua -%config(noreplace) %{rspamd_pluginsdir}/lua/maillist.lua -%config(noreplace) %{rspamd_pluginsdir}/lua/multimap.lua -%config(noreplace) %{rspamd_pluginsdir}/lua/once_received.lua -%config(noreplace) %{rspamd_pluginsdir}/lua/rbl.lua -%config(noreplace) %{rspamd_pluginsdir}/lua/ratelimit.lua -%config(noreplace) %{rspamd_pluginsdir}/lua/whitelist.lua -%config(noreplace) %{rspamd_pluginsdir}/lua/phishing.lua -%config(noreplace) %{rspamd_pluginsdir}/lua/trie.lua -%config(noreplace) %{rspamd_pluginsdir}/lua/emails.lua -%config(noreplace) %{rspamd_pluginsdir}/lua/ip_score.lua -%config(noreplace) %{rspamd_confdir}/lua/regexp/drugs.lua -%config(noreplace) %{rspamd_confdir}/lua/regexp/fraud.lua -%config(noreplace) %{rspamd_confdir}/lua/regexp/headers.lua -%config(noreplace) %{rspamd_confdir}/lua/regexp/lotto.lua -%config(noreplace) %{rspamd_confdir}/lua/rspamd.lua -%config(noreplace) %{rspamd_confdir}/lua/rspamd.classifiers.lua +%{rspamd_pluginsdir}/lua/forged_recipients.lua +%{rspamd_pluginsdir}/lua/maillist.lua +%{rspamd_pluginsdir}/lua/multimap.lua +%{rspamd_pluginsdir}/lua/once_received.lua +%{rspamd_pluginsdir}/lua/rbl.lua +%{rspamd_pluginsdir}/lua/ratelimit.lua +%{rspamd_pluginsdir}/lua/whitelist.lua +%{rspamd_pluginsdir}/lua/phishing.lua +%{rspamd_pluginsdir}/lua/trie.lua +%{rspamd_pluginsdir}/lua/emails.lua +%{rspamd_pluginsdir}/lua/ip_score.lua +%{rspamd_confdir}/lua/regexp/drugs.lua +%{rspamd_confdir}/lua/regexp/fraud.lua +%{rspamd_confdir}/lua/regexp/headers.lua +%{rspamd_confdir}/lua/regexp/lotto.lua +%{rspamd_confdir}/lua/rspamd.lua +%{rspamd_confdir}/lua/hfilter.lua +%{rspamd_confdir}/lua/rspamd.classifiers.lua +%if 0%{?suse_version} +%{_sbindir}/rcrspamd +%endif %changelog +* Fri Dec 27 2013 Vsevolod Stakhov <vsevolod-at-highsecure.ru> 0.6.6-1 +- Update to 0.6.6. + +* Fri Dec 20 2013 Vsevolod Stakhov <vsevolod-at-highsecure.ru> 0.6.5-1 +- Update to 0.6.5. + +* Wed Dec 18 2013 Vsevolod Stakhov <vsevolod-at-highsecure.ru> 0.6.4-1 +- Update to 0.6.4. + +* Tue Dec 10 2013 Vsevolod Stakhov <vsevolod-at-highsecure.ru> 0.6.3-1 +- Update to 0.6.3. + +* Fri Dec 06 2013 Vsevolod Stakhov <vsevolod-at-highsecure.ru> 0.6.2-1 +- Update to 0.6.2. + * Tue Nov 19 2013 Vsevolod Stakhov <vsevolod-at-highsecure.ru> 0.6.0-1 - Update to 0.6.0. diff --git a/centos/sources/rspamd.init b/centos/sources/rspamd.init index 161bbc396..78714a935 100644 --- a/centos/sources/rspamd.init +++ b/centos/sources/rspamd.init @@ -1,11 +1,20 @@ #!/bin/sh # # rspamd - this script starts and stops the rspamd daemon +### BEGIN INIT INFO +# Provides: rspamd +# Required-Start: $remote_fs $network +# Required-Stop: $network $remote_fs +# Default-Start: 5 +# Default-Stop: +# Short-Description: Rspamd daemon +# Description: Rspamd spam filtering daemon process +### END INIT INFO # # chkconfig: - 85 15 # description: rspamd is a spam filtering system # processname: rspamd -# config: /etc/rspamd/rspamd.xml +# config: /etc/rspamd/rspamd.conf # config: /etc/sysconfig/rspamd # pidfile: /var/run/rspamd/rspamd.pid @@ -21,7 +30,7 @@ rspamd="/usr/bin/rspamd" prog=$(basename $rspamd) -RSPAMD_CONF_FILE="/etc/rspamd.xml" +RSPAMD_CONF_FILE="/etc/rspamd/rspamd.conf" RSPAMD_USER="rspamd" RSPAMD_GROUP="rspamd" diff --git a/conf/2tld.inc b/conf/2tld.inc index 299f96229..f2b36f2fc 100644 --- a/conf/2tld.inc +++ b/conf/2tld.inc @@ -952,3 +952,6263 @@ blogindo.net o-f.com topfreewebhosting.com freeadsensehost.com + +#ac +com.ac +edu.ac +gov.ac +net.ac +mil.ac +org.ac +#ad +nom.ad +#ae +co.ae +net.ae +org.ae +sch.ae +ac.ae +gov.ae +mil.ae +#aero +accident-investigation.aero +accident-prevention.aero +aerobatic.aero +aeroclub.aero +aerodrome.aero +agents.aero +aircraft.aero +airline.aero +airport.aero +air-surveillance.aero +airtraffic.aero +air-traffic-control.aero +ambulance.aero +amusement.aero +association.aero +author.aero +ballooning.aero +broker.aero +caa.aero +cargo.aero +catering.aero +certification.aero +championship.aero +charter.aero +civilaviation.aero +club.aero +conference.aero +consultant.aero +consulting.aero +control.aero +council.aero +crew.aero +design.aero +dgca.aero +educator.aero +emergency.aero +engine.aero +engineer.aero +entertainment.aero +equipment.aero +exchange.aero +express.aero +federation.aero +flight.aero +freight.aero +fuel.aero +gliding.aero +government.aero +groundhandling.aero +group.aero +hanggliding.aero +homebuilt.aero +insurance.aero +journal.aero +journalist.aero +leasing.aero +logistics.aero +magazine.aero +maintenance.aero +marketplace.aero +media.aero +microlight.aero +modelling.aero +navigation.aero +parachuting.aero +paragliding.aero +passenger-association.aero +pilot.aero +press.aero +production.aero +recreation.aero +repbody.aero +res.aero +research.aero +rotorcraft.aero +safety.aero +scientist.aero +services.aero +show.aero +skydiving.aero +software.aero +student.aero +taxi.aero +trader.aero +trading.aero +trainer.aero +union.aero +workinggroup.aero +works.aero +#af +gov.af +com.af +org.af +net.af +edu.af +#ag +com.ag +org.ag +net.ag +co.ag +nom.ag +#ai +off.ai +com.ai +net.ai +org.ai +#al +com.al +edu.al +gov.al +mil.al +net.al +org.al +#am +#an +com.an +net.an +org.an +edu.an +#ao +ed.ao +gv.ao +og.ao +co.ao +pb.ao +it.ao +#aq +#ar +com.ar +gov.ar +int.ar +mil.ar +net.ar +org.ar +#arpa +e164.arpa +in-addr.arpa +ip6.arpa +iris.arpa +uri.arpa +urn.arpa +#as +gov.as +#asia +#at +ac.at +co.at +gv.at +or.at +#au +com.au +net.au +org.au +edu.au +gov.au +asn.au +id.au +csiro.au +info.au +conf.au +oz.au +act.au +nsw.au +nt.au +qld.au +sa.au +tas.au +vic.au +wa.au +act.edu.au +nsw.edu.au +nt.edu.au +qld.edu.au +sa.edu.au +tas.edu.au +vic.edu.au +wa.edu.au +act.gov.au +nt.gov.au +qld.gov.au +sa.gov.au +tas.gov.au +vic.gov.au +wa.gov.au +#aw +com.aw +#ax +#az +com.az +net.az +int.az +gov.az +org.az +edu.az +info.az +pp.az +mil.az +name.az +pro.az +biz.az +#ba +org.ba +net.ba +edu.ba +gov.ba +mil.ba +unsa.ba +unbi.ba +co.ba +com.ba +rs.ba +#bb +biz.bb +com.bb +edu.bb +gov.bb +info.bb +net.bb +org.bb +store.bb +#bd +com.bd +edu.bd +net.bd +gov.bd +org.bd +mil.bd +#be +ac.be +#bf +gov.bf +#bg +a.bg +b.bg +c.bg +d.bg +e.bg +f.bg +g.bg +h.bg +i.bg +j.bg +k.bg +l.bg +m.bg +n.bg +o.bg +p.bg +q.bg +r.bg +s.bg +t.bg +u.bg +v.bg +w.bg +x.bg +y.bg +z.bg +0.bg +1.bg +2.bg +3.bg +4.bg +5.bg +6.bg +7.bg +8.bg +9.bg +#bh +com.bh +edu.bh +net.bh +org.bh +gov.bh +#bi +co.bi +com.bi +edu.bi +or.bi +org.bi +#biz +#bj +asso.bj +barreau.bj +gouv.bj +#bm +com.bm +edu.bm +gov.bm +net.bm +org.bm +#bn +com.bn +edu.bn +org.bn +net.bn +#bo +com.bo +edu.bo +gov.bo +gob.bo +int.bo +org.bo +net.bo +mil.bo +tv.bo +#br +adm.br +adv.br +agr.br +am.br +arq.br +art.br +ato.br +b.br +bio.br +blog.br +bmd.br +cim.br +cng.br +cnt.br +com.br +coop.br +ecn.br +eco.br +edu.br +emp.br +eng.br +esp.br +etc.br +eti.br +far.br +flog.br +fm.br +fnd.br +fot.br +fst.br +g12.br +ggf.br +gov.br +imb.br +ind.br +inf.br +jor.br +jus.br +leg.br +lel.br +mat.br +med.br +mil.br +mus.br +net.br +nom.br +not.br +ntr.br +odo.br +org.br +ppg.br +pro.br +psc.br +psi.br +qsl.br +radio.br +rec.br +slg.br +srv.br +taxi.br +teo.br +tmp.br +trd.br +tur.br +tv.br +vet.br +vlog.br +wiki.br +zlg.br +#bs +com.bs +net.bs +org.bs +edu.bs +gov.bs +#bt +com.bt +edu.bt +gov.bt +net.bt +org.bt +#bv +#bw +co.bw +org.bw +#by +gov.by +mil.by +com.by +of.by +#bz +com.bz +net.bz +org.bz +edu.bz +gov.bz +#ca +ab.ca +bc.ca +mb.ca +nb.ca +nf.ca +nl.ca +ns.ca +nt.ca +nu.ca +on.ca +pe.ca +qc.ca +sk.ca +yk.ca +gc.ca +#cat +#cc +#cd +gov.cd +#cf +#cg +#ch +#ci +org.ci +or.ci +com.ci +co.ci +edu.ci +ed.ci +ac.ci +net.ci +go.ci +asso.ci +aeroport.ci +int.ci +presse.ci +md.ci +gouv.ci +#ck +.co.ck +.org.ck +.edu.ck +.gov.ck +.net.ck +.gen.ck +.biz.ck +.info.ck +#cl +gov.cl +gob.cl +co.cl +mil.cl +#cm +gov.cm +#cn +ac.cn +com.cn +edu.cn +gov.cn +net.cn +org.cn +mil.cn +ah.cn +bj.cn +cq.cn +fj.cn +gd.cn +gs.cn +gz.cn +gx.cn +ha.cn +hb.cn +he.cn +hi.cn +hl.cn +hn.cn +jl.cn +js.cn +jx.cn +ln.cn +nm.cn +nx.cn +qh.cn +sc.cn +sd.cn +sh.cn +sn.cn +sx.cn +tj.cn +xj.cn +xz.cn +yn.cn +zj.cn +hk.cn +mo.cn +tw.cn +#co +arts.co +com.co +edu.co +firm.co +gov.co +info.co +int.co +mil.co +net.co +nom.co +org.co +rec.co +web.co +#com +#coop +#cr +ac.cr +co.cr +ed.cr +fi.cr +go.cr +or.cr +sa.cr +#cu +com.cu +edu.cu +org.cu +net.cu +gov.cu +inf.cu +#cv +#cw +com.cw +edu.cw +net.cw +org.cw +#cx +gov.cx +#cy +com.cy +biz.cy +info.cy +ltd.cy +pro.cy +net.cy +org.cy +name.cy +tm.cy +ac.cy +ekloges.cy +press.cy +parliament.cy +#cz +#de +#dj +#dk +#dm +com.dm +net.dm +org.dm +edu.dm +gov.dm +#do +art.do +com.do +edu.do +gob.do +gov.do +mil.do +net.do +org.do +sld.do +web.do +#dz +com.dz +org.dz +net.dz +gov.dz +edu.dz +asso.dz +pol.dz +art.dz +#ec +com.ec +info.ec +net.ec +fin.ec +k12.ec +med.ec +pro.ec +org.ec +edu.ec +gov.ec +gob.ec +mil.ec +#edu +#ee +edu.ee +gov.ee +riik.ee +lib.ee +med.ee +com.ee +pri.ee +aip.ee +org.ee +fie.ee +eg +com.eg +edu.eg +eun.eg +gov.eg +mil.eg +name.eg +net.eg +org.eg +sci.eg +#er +com.er +edu.er +gov.er +mil.er +net.er +org.er +ind.er +#es +com.es +nom.es +org.es +gob.es +edu.es +#et +com.et +gov.et +org.et +edu.et +net.et +biz.et +name.et +info.et +#eu +#fi +aland.fi +#fj +ac.fj +biz.fj +com.fj +info.fj +mil.fj +name.fj +net.fj +org.fj +pro.fj +#fk +co.fk +org.fk +gov.fk +ac.fk +nom.fk +net.fk +#fm +#fo +#fr +com.fr +asso.fr +nom.fr +prd.fr +presse.fr +tm.fr +aeroport.fr +assedic.fr +avocat.fr +avoues.fr +cci.fr +chambagri.fr +chirurgiens-dentistes.fr +experts-comptables.fr +geometre-expert.fr +gouv.fr +greta.fr +huissier-justice.fr +medecin.fr +notaires.fr +pharmacien.fr +port.fr +veterinaire.fr +#ga +#gd +#ge +com.ge +edu.ge +gov.ge +org.ge +mil.ge +net.ge +pvt.ge +#gf +#gg +co.gg +org.gg +net.gg +sch.gg +gov.gg +#gh +com.gh +edu.gh +gov.gh +org.gh +mil.gh +#gi +com.gi +ltd.gi +gov.gi +mod.gi +edu.gi +org.gi +#gl +#gm +ac.gn +com.gn +edu.gn +gov.gn +org.gn +net.gn +#gov +#gp +com.gp +net.gp +mobi.gp +edu.gp +org.gp +asso.gp +#gq +#gr +com.gr +edu.gr +net.gr +org.gr +gov.gr +#gs +#gt +com.gt +edu.gt +gob.gt +ind.gt +mil.gt +net.gt +org.gt +#gu +com.gu +net.gu +gov.gu +org.gu +edu.gu +#gw +#gy +co.gy +com.gy +net.gy +#hk +com.hk +edu.hk +gov.hk +idv.hk +net.hk +org.hk +#hm +#hn +com.hn +edu.hn +org.hn +net.hn +mil.hn +gob.hn +#hr +iz.hr +from.hr +name.hr +com.hr +#ht +com.ht +shop.ht +firm.ht +info.ht +adult.ht +net.ht +pro.ht +org.ht +med.ht +art.ht +coop.ht +pol.ht +asso.ht +edu.ht +rel.ht +gouv.ht +perso.ht +#hu +co.hu +info.hu +org.hu +priv.hu +sport.hu +tm.hu +2000.hu +agrar.hu +bolt.hu +casino.hu +city.hu +erotica.hu +erotika.hu +film.hu +forum.hu +games.hu +hotel.hu +ingatlan.hu +jogasz.hu +konyvelo.hu +lakas.hu +media.hu +news.hu +reklam.hu +sex.hu +shop.hu +suli.hu +szex.hu +tozsde.hu +utazas.hu +video.hu +#id +ac.id +biz.id +co.id +go.id +mil.id +my.id +net.id +or.id +sch.id +web.id +#ie +gov.ie +#il +ac.il +co.il +org.il +net.il +k12.il +gov.il +muni.il +idf.il +#im +co.im +ltd.co.im +plc.co.im +net.im +gov.im +org.im +nic.im +ac.im +#in +co.in +firm.in +net.in +org.in +gen.in +ind.in +nic.in +ac.in +edu.in +res.in +gov.in +mil.in +#info +#int +eu.int +#io +com.io +#iq +gov.iq +edu.iq +mil.iq +com.iq +org.iq +net.iq +#ir +ac.ir +co.ir +gov.ir +id.ir +net.ir +org.ir +sch.ir +#is +net.is +com.is +edu.is +gov.is +org.is +int.is +#it +gov.it +edu.it +agrigento.it +ag.it +alessandria.it +al.it +ancona.it +an.it +aosta.it +aoste.it +ao.it +arezzo.it +ar.it +ascoli-piceno.it +ascolipiceno.it +ap.it +asti.it +at.it +avellino.it +av.it +bari.it +ba.it +andria-barletta-trani.it +andriabarlettatrani.it +trani-barletta-andria.it +tranibarlettaandria.it +barletta-trani-andria.it +barlettatraniandria.it +andria-trani-barletta.it +andriatranibarletta.it +trani-andria-barletta.it +traniandriabarletta.it +bt.it +belluno.it +bl.it +benevento.it +bn.it +bergamo.it +bg.it +biella.it +bi.it +bologna.it +bo.it +bolzano.it +bozen.it +balsan.it +alto-adige.it +altoadige.it +suedtirol.it +bz.it +brescia.it +bs.it +brindisi.it +br.it +cagliari.it +ca.it +caltanissetta.it +cl.it +campobasso.it +cb.it +carboniaiglesias.it +carbonia-iglesias.it +iglesias-carbonia.it +iglesiascarbonia.it +ci.it +caserta.it +ce.it +catania.it +ct.it +catanzaro.it +cz.it +chieti.it +ch.it +como.it +co.it +cosenza.it +cs.it +cremona.it +cr.it +crotone.it +kr.it +cuneo.it +cn.it +dell-ogliastra.it +dellogliastra.it +ogliastra.it +og.it +enna.it +en.it +ferrara.it +fe.it +fermo.it +fm.it +firenze.it +florence.it +fi.it +foggia.it +fg.it +forli-cesena.it +forlicesena.it +cesena-forli.it +cesenaforli.it +fc.it +frosinone.it +fr.it +genova.it +genoa.it +ge.it +gorizia.it +go.it +grosseto.it +gr.it +imperia.it +im.it +isernia.it +is.it +laquila.it +aquila.it +aq.it +la-spezia.it +laspezia.it +sp.it +latina.it +lt.it +lecce.it +le.it +lecco.it +lc.it +livorno.it +li.it +lodi.it +lo.it +lucca.it +lu.it +macerata.it +mc.it +mantova.it +mn.it +massa-carrara.it +massacarrara.it +carrara-massa.it +carraramassa.it +ms.it +matera.it +mt.it +medio-campidano.it +mediocampidano.it +campidano-medio.it +campidanomedio.it +vs.it +messina.it +me.it +milano.it +milan.it +mi.it +modena.it +mo.it +monza.it +monza-brianza.it +monzabrianza.it +monzaebrianza.it +monzaedellabrianza.it +monza-e-della-brianza.it +mb.it +napoli.it +naples.it +na.it +novara.it +no.it +nuoro.it +nu.it +oristano.it +or.it +padova.it +padua.it +pd.it +palermo.it +pa.it +parma.it +pr.it +pavia.it +pv.it +perugia.it +pg.it +pescara.it +pe.it +pesaro-urbino.it +pesarourbino.it +urbino-pesaro.it +urbinopesaro.it +pu.it +piacenza.it +pc.it +pisa.it +pi.it +pistoia.it +pt.it +pordenone.it +pn.it +potenza.it +pz.it +prato.it +po.it +ragusa.it +rg.it +ravenna.it +ra.it +reggio-calabria.it +reggiocalabria.it +rc.it +reggio-emilia.it +reggioemilia.it +re.it +rieti.it +ri.it +rimini.it +rn.it +roma.it +rome.it +rm.it +rovigo.it +ro.it +salerno.it +sa.it +sassari.it +ss.it +savona.it +sv.it +siena.it +si.it +siracusa.it +sr.it +sondrio.it +so.it +taranto.it +ta.it +tempio-olbia.it +tempioolbia.it +olbia-tempio.it +olbiatempio.it +ot.it +teramo.it +te.it +terni.it +tr.it +torino.it +turin.it +to.it +trapani.it +tp.it +trento.it +trentino.it +tn.it +treviso.it +tv.it +trieste.it +ts.it +udine.it +ud.it +varese.it +va.it +venezia.it +venice.it +ve.it +verbania.it +vb.it +vercelli.it +vc.it +verona.it +vr.it +vibo-valentia.it +vibovalentia.it +vv.it +vicenza.it +vi.it +viterbo.it +vt.it +#je +co.je +org.je +net.je +sch.je +gov.je +#jm +edu.jm +gov.jm +com.jm +net.jm +org.jm +#jo +com.jo +org.jo +net.jo +edu.jo +sch.jo +gov.jo +mil.jo +name.jo +#jobs +#jp +ac.jp +ad.jp +co.jp +ed.jp +go.jp +gr.jp +lg.jp +ne.jp +or.jp +aichi.jp +akita.jp +aomori.jp +chiba.jp +ehime.jp +fukui.jp +fukuoka.jp +fukushima.jp +gifu.jp +gunma.jp +hiroshima.jp +hokkaido.jp +hyogo.jp +ibaraki.jp +ishikawa.jp +iwate.jp +kagawa.jp +kagoshima.jp +kanagawa.jp +kochi.jp +kumamoto.jp +kyoto.jp +mie.jp +miyagi.jp +miyazaki.jp +nagano.jp +nagasaki.jp +nara.jp +niigata.jp +oita.jp +okayama.jp +okinawa.jp +osaka.jp +saga.jp +saitama.jp +shiga.jp +shimane.jp +shizuoka.jp +tochigi.jp +tokushima.jp +tokyo.jp +tottori.jp +toyama.jp +wakayama.jp +yamagata.jp +yamaguchi.jp +yamanashi.jp +kawasaki.jp +kitakyushu.jp +kobe.jp +nagoya.jp +sapporo.jp +sendai.jp +yokohama.jp +aisai.aichi.jp +ama.aichi.jp +anjo.aichi.jp +asuke.aichi.jp +chiryu.aichi.jp +chita.aichi.jp +fuso.aichi.jp +gamagori.aichi.jp +handa.aichi.jp +hazu.aichi.jp +hekinan.aichi.jp +higashiura.aichi.jp +ichinomiya.aichi.jp +inazawa.aichi.jp +inuyama.aichi.jp +isshiki.aichi.jp +iwakura.aichi.jp +kanie.aichi.jp +kariya.aichi.jp +kasugai.aichi.jp +kira.aichi.jp +kiyosu.aichi.jp +komaki.aichi.jp +konan.aichi.jp +kota.aichi.jp +mihama.aichi.jp +miyoshi.aichi.jp +nagakute.aichi.jp +nishio.aichi.jp +nisshin.aichi.jp +obu.aichi.jp +oguchi.aichi.jp +oharu.aichi.jp +okazaki.aichi.jp +owariasahi.aichi.jp +seto.aichi.jp +shikatsu.aichi.jp +shinshiro.aichi.jp +shitara.aichi.jp +tahara.aichi.jp +takahama.aichi.jp +tobishima.aichi.jp +toei.aichi.jp +togo.aichi.jp +tokai.aichi.jp +tokoname.aichi.jp +toyoake.aichi.jp +toyohashi.aichi.jp +toyokawa.aichi.jp +toyone.aichi.jp +toyota.aichi.jp +tsushima.aichi.jp +yatomi.aichi.jp +akita.akita.jp +daisen.akita.jp +fujisato.akita.jp +gojome.akita.jp +hachirogata.akita.jp +happou.akita.jp +higashinaruse.akita.jp +honjo.akita.jp +honjyo.akita.jp +ikawa.akita.jp +kamikoani.akita.jp +kamioka.akita.jp +katagami.akita.jp +kazuno.akita.jp +kitaakita.akita.jp +kosaka.akita.jp +kyowa.akita.jp +misato.akita.jp +mitane.akita.jp +moriyoshi.akita.jp +nikaho.akita.jp +noshiro.akita.jp +odate.akita.jp +oga.akita.jp +ogata.akita.jp +semboku.akita.jp +yokote.akita.jp +yurihonjo.akita.jp +aomori.aomori.jp +gonohe.aomori.jp +hachinohe.aomori.jp +hashikami.aomori.jp +hiranai.aomori.jp +hirosaki.aomori.jp +itayanagi.aomori.jp +kuroishi.aomori.jp +misawa.aomori.jp +mutsu.aomori.jp +nakadomari.aomori.jp +noheji.aomori.jp +oirase.aomori.jp +owani.aomori.jp +rokunohe.aomori.jp +sannohe.aomori.jp +shichinohe.aomori.jp +shingo.aomori.jp +takko.aomori.jp +towada.aomori.jp +tsugaru.aomori.jp +tsuruta.aomori.jp +abiko.chiba.jp +asahi.chiba.jp +chonan.chiba.jp +chosei.chiba.jp +choshi.chiba.jp +chuo.chiba.jp +funabashi.chiba.jp +futtsu.chiba.jp +hanamigawa.chiba.jp +ichihara.chiba.jp +ichikawa.chiba.jp +ichinomiya.chiba.jp +inzai.chiba.jp +isumi.chiba.jp +kamagaya.chiba.jp +kamogawa.chiba.jp +kashiwa.chiba.jp +katori.chiba.jp +katsuura.chiba.jp +kimitsu.chiba.jp +kisarazu.chiba.jp +kozaki.chiba.jp +kujukuri.chiba.jp +kyonan.chiba.jp +matsudo.chiba.jp +midori.chiba.jp +mihama.chiba.jp +minamiboso.chiba.jp +mobara.chiba.jp +mutsuzawa.chiba.jp +nagara.chiba.jp +nagareyama.chiba.jp +narashino.chiba.jp +narita.chiba.jp +noda.chiba.jp +oamishirasato.chiba.jp +omigawa.chiba.jp +onjuku.chiba.jp +otaki.chiba.jp +sakae.chiba.jp +sakura.chiba.jp +shimofusa.chiba.jp +shirako.chiba.jp +shiroi.chiba.jp +shisui.chiba.jp +sodegaura.chiba.jp +sosa.chiba.jp +tako.chiba.jp +tateyama.chiba.jp +togane.chiba.jp +tohnosho.chiba.jp +tomisato.chiba.jp +urayasu.chiba.jp +yachimata.chiba.jp +yachiyo.chiba.jp +yokaichiba.chiba.jp +yokoshibahikari.chiba.jp +yotsukaido.chiba.jp +ainan.ehime.jp +honai.ehime.jp +ikata.ehime.jp +imabari.ehime.jp +iyo.ehime.jp +kamijima.ehime.jp +kihoku.ehime.jp +kumakogen.ehime.jp +masaki.ehime.jp +matsuno.ehime.jp +matsuyama.ehime.jp +namikata.ehime.jp +niihama.ehime.jp +ozu.ehime.jp +saijo.ehime.jp +seiyo.ehime.jp +shikokuchuo.ehime.jp +tobe.ehime.jp +toon.ehime.jp +uchiko.ehime.jp +uwajima.ehime.jp +yawatahama.ehime.jp +echizen.fukui.jp +eiheiji.fukui.jp +fukui.fukui.jp +ikeda.fukui.jp +katsuyama.fukui.jp +mihama.fukui.jp +minamiechizen.fukui.jp +obama.fukui.jp +ohi.fukui.jp +ono.fukui.jp +sabae.fukui.jp +sakai.fukui.jp +takahama.fukui.jp +tsuruga.fukui.jp +wakasa.fukui.jp +ashiya.fukuoka.jp +buzen.fukuoka.jp +chikugo.fukuoka.jp +chikuho.fukuoka.jp +chikujo.fukuoka.jp +chikushino.fukuoka.jp +chikuzen.fukuoka.jp +chuo.fukuoka.jp +dazaifu.fukuoka.jp +fukuchi.fukuoka.jp +hakata.fukuoka.jp +higashi.fukuoka.jp +hirokawa.fukuoka.jp +hisayama.fukuoka.jp +iizuka.fukuoka.jp +inatsuki.fukuoka.jp +kaho.fukuoka.jp +kasuga.fukuoka.jp +kasuya.fukuoka.jp +kawara.fukuoka.jp +keisen.fukuoka.jp +koga.fukuoka.jp +kurate.fukuoka.jp +kurogi.fukuoka.jp +kurume.fukuoka.jp +minami.fukuoka.jp +miyako.fukuoka.jp +miyama.fukuoka.jp +miyawaka.fukuoka.jp +mizumaki.fukuoka.jp +munakata.fukuoka.jp +nakagawa.fukuoka.jp +nakama.fukuoka.jp +nishi.fukuoka.jp +nogata.fukuoka.jp +ogori.fukuoka.jp +okagaki.fukuoka.jp +okawa.fukuoka.jp +oki.fukuoka.jp +omuta.fukuoka.jp +onga.fukuoka.jp +onojo.fukuoka.jp +oto.fukuoka.jp +saigawa.fukuoka.jp +sasaguri.fukuoka.jp +shingu.fukuoka.jp +shinyoshitomi.fukuoka.jp +shonai.fukuoka.jp +soeda.fukuoka.jp +sue.fukuoka.jp +tachiarai.fukuoka.jp +tagawa.fukuoka.jp +takata.fukuoka.jp +toho.fukuoka.jp +toyotsu.fukuoka.jp +tsuiki.fukuoka.jp +ukiha.fukuoka.jp +umi.fukuoka.jp +usui.fukuoka.jp +yamada.fukuoka.jp +yame.fukuoka.jp +yanagawa.fukuoka.jp +yukuhashi.fukuoka.jp +aizubange.fukushima.jp +aizumisato.fukushima.jp +aizuwakamatsu.fukushima.jp +asakawa.fukushima.jp +bandai.fukushima.jp +date.fukushima.jp +fukushima.fukushima.jp +furudono.fukushima.jp +futaba.fukushima.jp +hanawa.fukushima.jp +higashi.fukushima.jp +hirata.fukushima.jp +hirono.fukushima.jp +iitate.fukushima.jp +inawashiro.fukushima.jp +ishikawa.fukushima.jp +iwaki.fukushima.jp +izumizaki.fukushima.jp +kagamiishi.fukushima.jp +kaneyama.fukushima.jp +kawamata.fukushima.jp +kitakata.fukushima.jp +kitashiobara.fukushima.jp +koori.fukushima.jp +koriyama.fukushima.jp +kunimi.fukushima.jp +miharu.fukushima.jp +mishima.fukushima.jp +namie.fukushima.jp +nango.fukushima.jp +nishiaizu.fukushima.jp +nishigo.fukushima.jp +okuma.fukushima.jp +omotego.fukushima.jp +ono.fukushima.jp +otama.fukushima.jp +samegawa.fukushima.jp +shimogo.fukushima.jp +shirakawa.fukushima.jp +showa.fukushima.jp +soma.fukushima.jp +sukagawa.fukushima.jp +taishin.fukushima.jp +tamakawa.fukushima.jp +tanagura.fukushima.jp +tenei.fukushima.jp +yabuki.fukushima.jp +yamato.fukushima.jp +yamatsuri.fukushima.jp +yanaizu.fukushima.jp +yugawa.fukushima.jp +anpachi.gifu.jp +ena.gifu.jp +gifu.gifu.jp +ginan.gifu.jp +godo.gifu.jp +gujo.gifu.jp +hashima.gifu.jp +hichiso.gifu.jp +hida.gifu.jp +higashishirakawa.gifu.jp +ibigawa.gifu.jp +ikeda.gifu.jp +kakamigahara.gifu.jp +kani.gifu.jp +kasahara.gifu.jp +kasamatsu.gifu.jp +kawaue.gifu.jp +kitagata.gifu.jp +mino.gifu.jp +minokamo.gifu.jp +mitake.gifu.jp +mizunami.gifu.jp +motosu.gifu.jp +nakatsugawa.gifu.jp +ogaki.gifu.jp +sakahogi.gifu.jp +seki.gifu.jp +sekigahara.gifu.jp +shirakawa.gifu.jp +tajimi.gifu.jp +takayama.gifu.jp +tarui.gifu.jp +toki.gifu.jp +tomika.gifu.jp +wanouchi.gifu.jp +yamagata.gifu.jp +yaotsu.gifu.jp +yoro.gifu.jp +annaka.gunma.jp +chiyoda.gunma.jp +fujioka.gunma.jp +higashiagatsuma.gunma.jp +isesaki.gunma.jp +itakura.gunma.jp +kanna.gunma.jp +kanra.gunma.jp +katashina.gunma.jp +kawaba.gunma.jp +kiryu.gunma.jp +kusatsu.gunma.jp +maebashi.gunma.jp +meiwa.gunma.jp +midori.gunma.jp +minakami.gunma.jp +naganohara.gunma.jp +nakanojo.gunma.jp +nanmoku.gunma.jp +numata.gunma.jp +oizumi.gunma.jp +ora.gunma.jp +ota.gunma.jp +shibukawa.gunma.jp +shimonita.gunma.jp +shinto.gunma.jp +showa.gunma.jp +takasaki.gunma.jp +takayama.gunma.jp +tamamura.gunma.jp +tatebayashi.gunma.jp +tomioka.gunma.jp +tsukiyono.gunma.jp +tsumagoi.gunma.jp +ueno.gunma.jp +yoshioka.gunma.jp +asaminami.hiroshima.jp +daiwa.hiroshima.jp +etajima.hiroshima.jp +fuchu.hiroshima.jp +fukuyama.hiroshima.jp +hatsukaichi.hiroshima.jp +higashihiroshima.hiroshima.jp +hongo.hiroshima.jp +jinsekikogen.hiroshima.jp +kaita.hiroshima.jp +kui.hiroshima.jp +kumano.hiroshima.jp +kure.hiroshima.jp +mihara.hiroshima.jp +miyoshi.hiroshima.jp +naka.hiroshima.jp +onomichi.hiroshima.jp +osakikamijima.hiroshima.jp +otake.hiroshima.jp +saka.hiroshima.jp +sera.hiroshima.jp +seranishi.hiroshima.jp +shinichi.hiroshima.jp +shobara.hiroshima.jp +takehara.hiroshima.jp +abashiri.hokkaido.jp +abira.hokkaido.jp +aibetsu.hokkaido.jp +akabira.hokkaido.jp +akkeshi.hokkaido.jp +asahikawa.hokkaido.jp +ashibetsu.hokkaido.jp +ashoro.hokkaido.jp +assabu.hokkaido.jp +atsuma.hokkaido.jp +bibai.hokkaido.jp +biei.hokkaido.jp +bifuka.hokkaido.jp +bihoro.hokkaido.jp +biratori.hokkaido.jp +chippubetsu.hokkaido.jp +chitose.hokkaido.jp +date.hokkaido.jp +ebetsu.hokkaido.jp +embetsu.hokkaido.jp +eniwa.hokkaido.jp +erimo.hokkaido.jp +esan.hokkaido.jp +esashi.hokkaido.jp +fukagawa.hokkaido.jp +fukushima.hokkaido.jp +furano.hokkaido.jp +furubira.hokkaido.jp +haboro.hokkaido.jp +hakodate.hokkaido.jp +hamatonbetsu.hokkaido.jp +hidaka.hokkaido.jp +higashikagura.hokkaido.jp +higashikawa.hokkaido.jp +hiroo.hokkaido.jp +hokuryu.hokkaido.jp +hokuto.hokkaido.jp +honbetsu.hokkaido.jp +horokanai.hokkaido.jp +horonobe.hokkaido.jp +ikeda.hokkaido.jp +imakane.hokkaido.jp +ishikari.hokkaido.jp +iwamizawa.hokkaido.jp +iwanai.hokkaido.jp +kamifurano.hokkaido.jp +kamikawa.hokkaido.jp +kamishihoro.hokkaido.jp +kamisunagawa.hokkaido.jp +kamoenai.hokkaido.jp +kayabe.hokkaido.jp +kembuchi.hokkaido.jp +kikonai.hokkaido.jp +kimobetsu.hokkaido.jp +kitahiroshima.hokkaido.jp +kitami.hokkaido.jp +kiyosato.hokkaido.jp +koshimizu.hokkaido.jp +kunneppu.hokkaido.jp +kuriyama.hokkaido.jp +kuromatsunai.hokkaido.jp +kushiro.hokkaido.jp +kutchan.hokkaido.jp +kyowa.hokkaido.jp +mashike.hokkaido.jp +matsumae.hokkaido.jp +mikasa.hokkaido.jp +minamifurano.hokkaido.jp +mombetsu.hokkaido.jp +moseushi.hokkaido.jp +mukawa.hokkaido.jp +muroran.hokkaido.jp +naie.hokkaido.jp +nakagawa.hokkaido.jp +nakasatsunai.hokkaido.jp +nakatombetsu.hokkaido.jp +nanae.hokkaido.jp +nanporo.hokkaido.jp +nayoro.hokkaido.jp +nemuro.hokkaido.jp +niikappu.hokkaido.jp +niki.hokkaido.jp +nishiokoppe.hokkaido.jp +noboribetsu.hokkaido.jp +numata.hokkaido.jp +obihiro.hokkaido.jp +obira.hokkaido.jp +oketo.hokkaido.jp +okoppe.hokkaido.jp +otaru.hokkaido.jp +otobe.hokkaido.jp +otofuke.hokkaido.jp +otoineppu.hokkaido.jp +oumu.hokkaido.jp +ozora.hokkaido.jp +pippu.hokkaido.jp +rankoshi.hokkaido.jp +rebun.hokkaido.jp +rikubetsu.hokkaido.jp +rishiri.hokkaido.jp +rishirifuji.hokkaido.jp +saroma.hokkaido.jp +sarufutsu.hokkaido.jp +shakotan.hokkaido.jp +shari.hokkaido.jp +shibecha.hokkaido.jp +shibetsu.hokkaido.jp +shikabe.hokkaido.jp +shikaoi.hokkaido.jp +shimamaki.hokkaido.jp +shimizu.hokkaido.jp +shimokawa.hokkaido.jp +shinshinotsu.hokkaido.jp +shintoku.hokkaido.jp +shiranuka.hokkaido.jp +shiraoi.hokkaido.jp +shiriuchi.hokkaido.jp +sobetsu.hokkaido.jp +sunagawa.hokkaido.jp +taiki.hokkaido.jp +takasu.hokkaido.jp +takikawa.hokkaido.jp +takinoue.hokkaido.jp +teshikaga.hokkaido.jp +tobetsu.hokkaido.jp +tohma.hokkaido.jp +tomakomai.hokkaido.jp +tomari.hokkaido.jp +toya.hokkaido.jp +toyako.hokkaido.jp +toyotomi.hokkaido.jp +toyoura.hokkaido.jp +tsubetsu.hokkaido.jp +tsukigata.hokkaido.jp +urakawa.hokkaido.jp +urausu.hokkaido.jp +uryu.hokkaido.jp +utashinai.hokkaido.jp +wakkanai.hokkaido.jp +wassamu.hokkaido.jp +yakumo.hokkaido.jp +yoichi.hokkaido.jp +aioi.hyogo.jp +akashi.hyogo.jp +ako.hyogo.jp +amagasaki.hyogo.jp +aogaki.hyogo.jp +asago.hyogo.jp +ashiya.hyogo.jp +awaji.hyogo.jp +fukusaki.hyogo.jp +goshiki.hyogo.jp +harima.hyogo.jp +himeji.hyogo.jp +ichikawa.hyogo.jp +inagawa.hyogo.jp +itami.hyogo.jp +kakogawa.hyogo.jp +kamigori.hyogo.jp +kamikawa.hyogo.jp +kasai.hyogo.jp +kasuga.hyogo.jp +kawanishi.hyogo.jp +miki.hyogo.jp +minamiawaji.hyogo.jp +nishinomiya.hyogo.jp +nishiwaki.hyogo.jp +ono.hyogo.jp +sanda.hyogo.jp +sannan.hyogo.jp +sasayama.hyogo.jp +sayo.hyogo.jp +shingu.hyogo.jp +shinonsen.hyogo.jp +shiso.hyogo.jp +sumoto.hyogo.jp +taishi.hyogo.jp +taka.hyogo.jp +takarazuka.hyogo.jp +takasago.hyogo.jp +takino.hyogo.jp +tamba.hyogo.jp +tatsuno.hyogo.jp +toyooka.hyogo.jp +yabu.hyogo.jp +yashiro.hyogo.jp +yoka.hyogo.jp +yokawa.hyogo.jp +ami.ibaraki.jp +asahi.ibaraki.jp +bando.ibaraki.jp +chikusei.ibaraki.jp +daigo.ibaraki.jp +fujishiro.ibaraki.jp +hitachi.ibaraki.jp +hitachinaka.ibaraki.jp +hitachiomiya.ibaraki.jp +hitachiota.ibaraki.jp +ibaraki.ibaraki.jp +ina.ibaraki.jp +inashiki.ibaraki.jp +itako.ibaraki.jp +iwama.ibaraki.jp +joso.ibaraki.jp +kamisu.ibaraki.jp +kasama.ibaraki.jp +kashima.ibaraki.jp +kasumigaura.ibaraki.jp +koga.ibaraki.jp +miho.ibaraki.jp +mito.ibaraki.jp +moriya.ibaraki.jp +naka.ibaraki.jp +namegata.ibaraki.jp +oarai.ibaraki.jp +ogawa.ibaraki.jp +omitama.ibaraki.jp +ryugasaki.ibaraki.jp +sakai.ibaraki.jp +sakuragawa.ibaraki.jp +shimodate.ibaraki.jp +shimotsuma.ibaraki.jp +shirosato.ibaraki.jp +sowa.ibaraki.jp +suifu.ibaraki.jp +takahagi.ibaraki.jp +tamatsukuri.ibaraki.jp +tokai.ibaraki.jp +tomobe.ibaraki.jp +tone.ibaraki.jp +toride.ibaraki.jp +tsuchiura.ibaraki.jp +tsukuba.ibaraki.jp +uchihara.ibaraki.jp +ushiku.ibaraki.jp +yachiyo.ibaraki.jp +yamagata.ibaraki.jp +yawara.ibaraki.jp +yuki.ibaraki.jp +anamizu.ishikawa.jp +hakui.ishikawa.jp +hakusan.ishikawa.jp +kaga.ishikawa.jp +kahoku.ishikawa.jp +kanazawa.ishikawa.jp +kawakita.ishikawa.jp +komatsu.ishikawa.jp +nakanoto.ishikawa.jp +nanao.ishikawa.jp +nomi.ishikawa.jp +nonoichi.ishikawa.jp +noto.ishikawa.jp +shika.ishikawa.jp +suzu.ishikawa.jp +tsubata.ishikawa.jp +tsurugi.ishikawa.jp +uchinada.ishikawa.jp +wajima.ishikawa.jp +fudai.iwate.jp +fujisawa.iwate.jp +hanamaki.iwate.jp +hiraizumi.iwate.jp +hirono.iwate.jp +ichinohe.iwate.jp +ichinoseki.iwate.jp +iwaizumi.iwate.jp +iwate.iwate.jp +joboji.iwate.jp +kamaishi.iwate.jp +kanegasaki.iwate.jp +karumai.iwate.jp +kawai.iwate.jp +kitakami.iwate.jp +kuji.iwate.jp +kunohe.iwate.jp +kuzumaki.iwate.jp +miyako.iwate.jp +mizusawa.iwate.jp +morioka.iwate.jp +ninohe.iwate.jp +noda.iwate.jp +ofunato.iwate.jp +oshu.iwate.jp +otsuchi.iwate.jp +rikuzentakata.iwate.jp +shiwa.iwate.jp +shizukuishi.iwate.jp +sumita.iwate.jp +takizawa.iwate.jp +tanohata.iwate.jp +tono.iwate.jp +yahaba.iwate.jp +yamada.iwate.jp +ayagawa.kagawa.jp +higashikagawa.kagawa.jp +kanonji.kagawa.jp +kotohira.kagawa.jp +manno.kagawa.jp +marugame.kagawa.jp +mitoyo.kagawa.jp +naoshima.kagawa.jp +sanuki.kagawa.jp +tadotsu.kagawa.jp +takamatsu.kagawa.jp +tonosho.kagawa.jp +uchinomi.kagawa.jp +utazu.kagawa.jp +zentsuji.kagawa.jp +akune.kagoshima.jp +amami.kagoshima.jp +hioki.kagoshima.jp +isa.kagoshima.jp +isen.kagoshima.jp +izumi.kagoshima.jp +kagoshima.kagoshima.jp +kanoya.kagoshima.jp +kawanabe.kagoshima.jp +kinko.kagoshima.jp +kouyama.kagoshima.jp +makurazaki.kagoshima.jp +matsumoto.kagoshima.jp +minamitane.kagoshima.jp +nakatane.kagoshima.jp +nishinoomote.kagoshima.jp +satsumasendai.kagoshima.jp +soo.kagoshima.jp +tarumizu.kagoshima.jp +yusui.kagoshima.jp +aikawa.kanagawa.jp +atsugi.kanagawa.jp +ayase.kanagawa.jp +chigasaki.kanagawa.jp +ebina.kanagawa.jp +fujisawa.kanagawa.jp +hadano.kanagawa.jp +hakone.kanagawa.jp +hiratsuka.kanagawa.jp +isehara.kanagawa.jp +kaisei.kanagawa.jp +kamakura.kanagawa.jp +kiyokawa.kanagawa.jp +matsuda.kanagawa.jp +minamiashigara.kanagawa.jp +miura.kanagawa.jp +nakai.kanagawa.jp +ninomiya.kanagawa.jp +odawara.kanagawa.jp +oi.kanagawa.jp +oiso.kanagawa.jp +sagamihara.kanagawa.jp +samukawa.kanagawa.jp +tsukui.kanagawa.jp +yamakita.kanagawa.jp +yamato.kanagawa.jp +yokosuka.kanagawa.jp +yugawara.kanagawa.jp +zama.kanagawa.jp +zushi.kanagawa.jp +aki.kochi.jp +geisei.kochi.jp +hidaka.kochi.jp +higashitsuno.kochi.jp +ino.kochi.jp +kagami.kochi.jp +kami.kochi.jp +kitagawa.kochi.jp +kochi.kochi.jp +mihara.kochi.jp +motoyama.kochi.jp +muroto.kochi.jp +nahari.kochi.jp +nakamura.kochi.jp +nankoku.kochi.jp +nishitosa.kochi.jp +niyodogawa.kochi.jp +ochi.kochi.jp +okawa.kochi.jp +otoyo.kochi.jp +otsuki.kochi.jp +sakawa.kochi.jp +sukumo.kochi.jp +susaki.kochi.jp +tosa.kochi.jp +tosashimizu.kochi.jp +toyo.kochi.jp +tsuno.kochi.jp +umaji.kochi.jp +yasuda.kochi.jp +yusuhara.kochi.jp +amakusa.kumamoto.jp +arao.kumamoto.jp +aso.kumamoto.jp +choyo.kumamoto.jp +gyokuto.kumamoto.jp +hitoyoshi.kumamoto.jp +kamiamakusa.kumamoto.jp +kashima.kumamoto.jp +kikuchi.kumamoto.jp +kosa.kumamoto.jp +kumamoto.kumamoto.jp +mashiki.kumamoto.jp +mifune.kumamoto.jp +minamata.kumamoto.jp +minamioguni.kumamoto.jp +nagasu.kumamoto.jp +nishihara.kumamoto.jp +oguni.kumamoto.jp +ozu.kumamoto.jp +sumoto.kumamoto.jp +takamori.kumamoto.jp +uki.kumamoto.jp +uto.kumamoto.jp +yamaga.kumamoto.jp +yamato.kumamoto.jp +yatsushiro.kumamoto.jp +ayabe.kyoto.jp +fukuchiyama.kyoto.jp +higashiyama.kyoto.jp +ide.kyoto.jp +ine.kyoto.jp +joyo.kyoto.jp +kameoka.kyoto.jp +kamo.kyoto.jp +kita.kyoto.jp +kizu.kyoto.jp +kumiyama.kyoto.jp +kyotamba.kyoto.jp +kyotanabe.kyoto.jp +kyotango.kyoto.jp +maizuru.kyoto.jp +minami.kyoto.jp +minamiyamashiro.kyoto.jp +miyazu.kyoto.jp +muko.kyoto.jp +nagaokakyo.kyoto.jp +nakagyo.kyoto.jp +nantan.kyoto.jp +oyamazaki.kyoto.jp +sakyo.kyoto.jp +seika.kyoto.jp +tanabe.kyoto.jp +uji.kyoto.jp +ujitawara.kyoto.jp +wazuka.kyoto.jp +yamashina.kyoto.jp +yawata.kyoto.jp +asahi.mie.jp +inabe.mie.jp +ise.mie.jp +kameyama.mie.jp +kawagoe.mie.jp +kiho.mie.jp +kisosaki.mie.jp +kiwa.mie.jp +komono.mie.jp +kumano.mie.jp +kuwana.mie.jp +matsusaka.mie.jp +meiwa.mie.jp +mihama.mie.jp +minamiise.mie.jp +misugi.mie.jp +miyama.mie.jp +nabari.mie.jp +shima.mie.jp +suzuka.mie.jp +tado.mie.jp +taiki.mie.jp +taki.mie.jp +tamaki.mie.jp +toba.mie.jp +tsu.mie.jp +udono.mie.jp +ureshino.mie.jp +watarai.mie.jp +yokkaichi.mie.jp +furukawa.miyagi.jp +higashimatsushima.miyagi.jp +ishinomaki.miyagi.jp +iwanuma.miyagi.jp +kakuda.miyagi.jp +kami.miyagi.jp +kawasaki.miyagi.jp +kesennuma.miyagi.jp +marumori.miyagi.jp +matsushima.miyagi.jp +minamisanriku.miyagi.jp +misato.miyagi.jp +murata.miyagi.jp +natori.miyagi.jp +ogawara.miyagi.jp +ohira.miyagi.jp +onagawa.miyagi.jp +osaki.miyagi.jp +rifu.miyagi.jp +semine.miyagi.jp +shibata.miyagi.jp +shichikashuku.miyagi.jp +shikama.miyagi.jp +shiogama.miyagi.jp +shiroishi.miyagi.jp +tagajo.miyagi.jp +taiwa.miyagi.jp +tome.miyagi.jp +tomiya.miyagi.jp +wakuya.miyagi.jp +watari.miyagi.jp +yamamoto.miyagi.jp +zao.miyagi.jp +aya.miyazaki.jp +ebino.miyazaki.jp +gokase.miyazaki.jp +hyuga.miyazaki.jp +kadogawa.miyazaki.jp +kawaminami.miyazaki.jp +kijo.miyazaki.jp +kitagawa.miyazaki.jp +kitakata.miyazaki.jp +kitaura.miyazaki.jp +kobayashi.miyazaki.jp +kunitomi.miyazaki.jp +kushima.miyazaki.jp +mimata.miyazaki.jp +miyakonojo.miyazaki.jp +miyazaki.miyazaki.jp +morotsuka.miyazaki.jp +nichinan.miyazaki.jp +nishimera.miyazaki.jp +nobeoka.miyazaki.jp +saito.miyazaki.jp +shiiba.miyazaki.jp +shintomi.miyazaki.jp +takaharu.miyazaki.jp +takanabe.miyazaki.jp +takazaki.miyazaki.jp +tsuno.miyazaki.jp +achi.nagano.jp +agematsu.nagano.jp +anan.nagano.jp +aoki.nagano.jp +asahi.nagano.jp +azumino.nagano.jp +chikuhoku.nagano.jp +chikuma.nagano.jp +chino.nagano.jp +fujimi.nagano.jp +hakuba.nagano.jp +hara.nagano.jp +hiraya.nagano.jp +iida.nagano.jp +iijima.nagano.jp +iiyama.nagano.jp +iizuna.nagano.jp +ikeda.nagano.jp +ikusaka.nagano.jp +ina.nagano.jp +karuizawa.nagano.jp +kawakami.nagano.jp +kiso.nagano.jp +kisofukushima.nagano.jp +kitaaiki.nagano.jp +komagane.nagano.jp +komoro.nagano.jp +matsukawa.nagano.jp +matsumoto.nagano.jp +miasa.nagano.jp +minamiaiki.nagano.jp +minamimaki.nagano.jp +minamiminowa.nagano.jp +minowa.nagano.jp +miyada.nagano.jp +miyota.nagano.jp +mochizuki.nagano.jp +nagano.nagano.jp +nagawa.nagano.jp +nagiso.nagano.jp +nakagawa.nagano.jp +nakano.nagano.jp +nozawaonsen.nagano.jp +obuse.nagano.jp +ogawa.nagano.jp +okaya.nagano.jp +omachi.nagano.jp +omi.nagano.jp +ookuwa.nagano.jp +ooshika.nagano.jp +otaki.nagano.jp +otari.nagano.jp +sakae.nagano.jp +sakaki.nagano.jp +saku.nagano.jp +sakuho.nagano.jp +shimosuwa.nagano.jp +shinanomachi.nagano.jp +shiojiri.nagano.jp +suwa.nagano.jp +suzaka.nagano.jp +takagi.nagano.jp +takamori.nagano.jp +takayama.nagano.jp +tateshina.nagano.jp +tatsuno.nagano.jp +togakushi.nagano.jp +togura.nagano.jp +tomi.nagano.jp +ueda.nagano.jp +wada.nagano.jp +yamagata.nagano.jp +yamanouchi.nagano.jp +yasaka.nagano.jp +yasuoka.nagano.jp +chijiwa.nagasaki.jp +futsu.nagasaki.jp +goto.nagasaki.jp +hasami.nagasaki.jp +hirado.nagasaki.jp +iki.nagasaki.jp +isahaya.nagasaki.jp +kawatana.nagasaki.jp +kuchinotsu.nagasaki.jp +matsuura.nagasaki.jp +nagasaki.nagasaki.jp +obama.nagasaki.jp +omura.nagasaki.jp +oseto.nagasaki.jp +saikai.nagasaki.jp +sasebo.nagasaki.jp +seihi.nagasaki.jp +shimabara.nagasaki.jp +shinkamigoto.nagasaki.jp +togitsu.nagasaki.jp +tsushima.nagasaki.jp +unzen.nagasaki.jp +ando.nara.jp +gose.nara.jp +heguri.nara.jp +higashiyoshino.nara.jp +ikaruga.nara.jp +ikoma.nara.jp +kamikitayama.nara.jp +kanmaki.nara.jp +kashiba.nara.jp +kashihara.nara.jp +katsuragi.nara.jp +kawai.nara.jp +kawakami.nara.jp +kawanishi.nara.jp +koryo.nara.jp +kurotaki.nara.jp +mitsue.nara.jp +miyake.nara.jp +nara.nara.jp +nosegawa.nara.jp +oji.nara.jp +ouda.nara.jp +oyodo.nara.jp +sakurai.nara.jp +sango.nara.jp +shimoichi.nara.jp +shimokitayama.nara.jp +shinjo.nara.jp +soni.nara.jp +takatori.nara.jp +tawaramoto.nara.jp +tenkawa.nara.jp +tenri.nara.jp +uda.nara.jp +yamatokoriyama.nara.jp +yamatotakada.nara.jp +yamazoe.nara.jp +yoshino.nara.jp +aga.niigata.jp +agano.niigata.jp +gosen.niigata.jp +itoigawa.niigata.jp +izumozaki.niigata.jp +joetsu.niigata.jp +kamo.niigata.jp +kariwa.niigata.jp +kashiwazaki.niigata.jp +minamiuonuma.niigata.jp +mitsuke.niigata.jp +muika.niigata.jp +murakami.niigata.jp +myoko.niigata.jp +nagaoka.niigata.jp +niigata.niigata.jp +ojiya.niigata.jp +omi.niigata.jp +sado.niigata.jp +sanjo.niigata.jp +seiro.niigata.jp +seirou.niigata.jp +sekikawa.niigata.jp +shibata.niigata.jp +tagami.niigata.jp +tainai.niigata.jp +tochio.niigata.jp +tokamachi.niigata.jp +tsubame.niigata.jp +tsunan.niigata.jp +uonuma.niigata.jp +yahiko.niigata.jp +yoita.niigata.jp +yuzawa.niigata.jp +beppu.oita.jp +bungoono.oita.jp +bungotakada.oita.jp +hasama.oita.jp +hiji.oita.jp +himeshima.oita.jp +hita.oita.jp +kamitsue.oita.jp +kokonoe.oita.jp +kuju.oita.jp +kunisaki.oita.jp +kusu.oita.jp +oita.oita.jp +saiki.oita.jp +taketa.oita.jp +tsukumi.oita.jp +usa.oita.jp +usuki.oita.jp +yufu.oita.jp +akaiwa.okayama.jp +asakuchi.okayama.jp +bizen.okayama.jp +hayashima.okayama.jp +ibara.okayama.jp +kagamino.okayama.jp +kasaoka.okayama.jp +kibichuo.okayama.jp +kumenan.okayama.jp +kurashiki.okayama.jp +maniwa.okayama.jp +misaki.okayama.jp +nagi.okayama.jp +niimi.okayama.jp +nishiawakura.okayama.jp +okayama.okayama.jp +satosho.okayama.jp +setouchi.okayama.jp +shinjo.okayama.jp +shoo.okayama.jp +soja.okayama.jp +takahashi.okayama.jp +tamano.okayama.jp +tsuyama.okayama.jp +wake.okayama.jp +yakage.okayama.jp +aguni.okinawa.jp +ginowan.okinawa.jp +ginoza.okinawa.jp +gushikami.okinawa.jp +haebaru.okinawa.jp +higashi.okinawa.jp +hirara.okinawa.jp +iheya.okinawa.jp +ishigaki.okinawa.jp +ishikawa.okinawa.jp +itoman.okinawa.jp +izena.okinawa.jp +kadena.okinawa.jp +kin.okinawa.jp +kitadaito.okinawa.jp +kitanakagusuku.okinawa.jp +kumejima.okinawa.jp +kunigami.okinawa.jp +minamidaito.okinawa.jp +motobu.okinawa.jp +nago.okinawa.jp +naha.okinawa.jp +nakagusuku.okinawa.jp +nakijin.okinawa.jp +nanjo.okinawa.jp +nishihara.okinawa.jp +ogimi.okinawa.jp +okinawa.okinawa.jp +onna.okinawa.jp +shimoji.okinawa.jp +taketomi.okinawa.jp +tarama.okinawa.jp +tokashiki.okinawa.jp +tomigusuku.okinawa.jp +tonaki.okinawa.jp +urasoe.okinawa.jp +uruma.okinawa.jp +yaese.okinawa.jp +yomitan.okinawa.jp +yonabaru.okinawa.jp +yonaguni.okinawa.jp +zamami.okinawa.jp +abeno.osaka.jp +chihayaakasaka.osaka.jp +chuo.osaka.jp +daito.osaka.jp +fujiidera.osaka.jp +habikino.osaka.jp +hannan.osaka.jp +higashiosaka.osaka.jp +higashisumiyoshi.osaka.jp +higashiyodogawa.osaka.jp +hirakata.osaka.jp +ibaraki.osaka.jp +ikeda.osaka.jp +izumi.osaka.jp +izumiotsu.osaka.jp +izumisano.osaka.jp +kadoma.osaka.jp +kaizuka.osaka.jp +kanan.osaka.jp +kashiwara.osaka.jp +katano.osaka.jp +kawachinagano.osaka.jp +kishiwada.osaka.jp +kita.osaka.jp +kumatori.osaka.jp +matsubara.osaka.jp +minato.osaka.jp +minoh.osaka.jp +misaki.osaka.jp +moriguchi.osaka.jp +neyagawa.osaka.jp +nishi.osaka.jp +nose.osaka.jp +osakasayama.osaka.jp +sakai.osaka.jp +sayama.osaka.jp +sennan.osaka.jp +settsu.osaka.jp +shijonawate.osaka.jp +shimamoto.osaka.jp +suita.osaka.jp +tadaoka.osaka.jp +taishi.osaka.jp +tajiri.osaka.jp +takaishi.osaka.jp +takatsuki.osaka.jp +tondabayashi.osaka.jp +toyonaka.osaka.jp +toyono.osaka.jp +yao.osaka.jp +ariake.saga.jp +arita.saga.jp +fukudomi.saga.jp +genkai.saga.jp +hamatama.saga.jp +hizen.saga.jp +imari.saga.jp +kamimine.saga.jp +kanzaki.saga.jp +karatsu.saga.jp +kashima.saga.jp +kitagata.saga.jp +kitahata.saga.jp +kiyama.saga.jp +kouhoku.saga.jp +kyuragi.saga.jp +nishiarita.saga.jp +ogi.saga.jp +omachi.saga.jp +ouchi.saga.jp +saga.saga.jp +shiroishi.saga.jp +taku.saga.jp +tara.saga.jp +tosu.saga.jp +yoshinogari.saga.jp +arakawa.saitama.jp +asaka.saitama.jp +chichibu.saitama.jp +fujimi.saitama.jp +fujimino.saitama.jp +fukaya.saitama.jp +hanno.saitama.jp +hanyu.saitama.jp +hasuda.saitama.jp +hatogaya.saitama.jp +hatoyama.saitama.jp +hidaka.saitama.jp +higashichichibu.saitama.jp +higashimatsuyama.saitama.jp +honjo.saitama.jp +ina.saitama.jp +iruma.saitama.jp +iwatsuki.saitama.jp +kamiizumi.saitama.jp +kamikawa.saitama.jp +kamisato.saitama.jp +kasukabe.saitama.jp +kawagoe.saitama.jp +kawaguchi.saitama.jp +kawajima.saitama.jp +kazo.saitama.jp +kitamoto.saitama.jp +koshigaya.saitama.jp +kounosu.saitama.jp +kuki.saitama.jp +kumagaya.saitama.jp +matsubushi.saitama.jp +minano.saitama.jp +misato.saitama.jp +miyashiro.saitama.jp +miyoshi.saitama.jp +moroyama.saitama.jp +nagatoro.saitama.jp +namegawa.saitama.jp +niiza.saitama.jp +ogano.saitama.jp +ogawa.saitama.jp +ogose.saitama.jp +okegawa.saitama.jp +omiya.saitama.jp +otaki.saitama.jp +ranzan.saitama.jp +ryokami.saitama.jp +saitama.saitama.jp +sakado.saitama.jp +satte.saitama.jp +sayama.saitama.jp +shiki.saitama.jp +shiraoka.saitama.jp +soka.saitama.jp +sugito.saitama.jp +toda.saitama.jp +tokigawa.saitama.jp +tokorozawa.saitama.jp +tsurugashima.saitama.jp +urawa.saitama.jp +warabi.saitama.jp +yashio.saitama.jp +yokoze.saitama.jp +yono.saitama.jp +yorii.saitama.jp +yoshida.saitama.jp +yoshikawa.saitama.jp +yoshimi.saitama.jp +aisho.shiga.jp +gamo.shiga.jp +higashiomi.shiga.jp +hikone.shiga.jp +koka.shiga.jp +konan.shiga.jp +kosei.shiga.jp +koto.shiga.jp +kusatsu.shiga.jp +maibara.shiga.jp +moriyama.shiga.jp +nagahama.shiga.jp +nishiazai.shiga.jp +notogawa.shiga.jp +omihachiman.shiga.jp +otsu.shiga.jp +ritto.shiga.jp +ryuoh.shiga.jp +takashima.shiga.jp +takatsuki.shiga.jp +torahime.shiga.jp +toyosato.shiga.jp +yasu.shiga.jp +akagi.shimane.jp +ama.shimane.jp +gotsu.shimane.jp +hamada.shimane.jp +higashiizumo.shimane.jp +hikawa.shimane.jp +hikimi.shimane.jp +izumo.shimane.jp +kakinoki.shimane.jp +masuda.shimane.jp +matsue.shimane.jp +misato.shimane.jp +nishinoshima.shimane.jp +ohda.shimane.jp +okinoshima.shimane.jp +okuizumo.shimane.jp +shimane.shimane.jp +tamayu.shimane.jp +tsuwano.shimane.jp +unnan.shimane.jp +yakumo.shimane.jp +yasugi.shimane.jp +yatsuka.shimane.jp +arai.shizuoka.jp +atami.shizuoka.jp +fuji.shizuoka.jp +fujieda.shizuoka.jp +fujikawa.shizuoka.jp +fujinomiya.shizuoka.jp +fukuroi.shizuoka.jp +gotemba.shizuoka.jp +haibara.shizuoka.jp +hamamatsu.shizuoka.jp +higashiizu.shizuoka.jp +ito.shizuoka.jp +iwata.shizuoka.jp +izu.shizuoka.jp +izunokuni.shizuoka.jp +kakegawa.shizuoka.jp +kannami.shizuoka.jp +kawanehon.shizuoka.jp +kawazu.shizuoka.jp +kikugawa.shizuoka.jp +kosai.shizuoka.jp +makinohara.shizuoka.jp +matsuzaki.shizuoka.jp +minamiizu.shizuoka.jp +mishima.shizuoka.jp +morimachi.shizuoka.jp +nishiizu.shizuoka.jp +numazu.shizuoka.jp +omaezaki.shizuoka.jp +shimada.shizuoka.jp +shimizu.shizuoka.jp +shimoda.shizuoka.jp +shizuoka.shizuoka.jp +susono.shizuoka.jp +yaizu.shizuoka.jp +yoshida.shizuoka.jp +ashikaga.tochigi.jp +bato.tochigi.jp +haga.tochigi.jp +ichikai.tochigi.jp +iwafune.tochigi.jp +kaminokawa.tochigi.jp +kanuma.tochigi.jp +karasuyama.tochigi.jp +kuroiso.tochigi.jp +mashiko.tochigi.jp +mibu.tochigi.jp +moka.tochigi.jp +motegi.tochigi.jp +nasu.tochigi.jp +nasushiobara.tochigi.jp +nikko.tochigi.jp +nishikata.tochigi.jp +nogi.tochigi.jp +ohira.tochigi.jp +ohtawara.tochigi.jp +oyama.tochigi.jp +sakura.tochigi.jp +sano.tochigi.jp +shimotsuke.tochigi.jp +shioya.tochigi.jp +takanezawa.tochigi.jp +tochigi.tochigi.jp +tsuga.tochigi.jp +ujiie.tochigi.jp +utsunomiya.tochigi.jp +yaita.tochigi.jp +aizumi.tokushima.jp +anan.tokushima.jp +ichiba.tokushima.jp +itano.tokushima.jp +kainan.tokushima.jp +komatsushima.tokushima.jp +matsushige.tokushima.jp +mima.tokushima.jp +minami.tokushima.jp +miyoshi.tokushima.jp +mugi.tokushima.jp +nakagawa.tokushima.jp +naruto.tokushima.jp +sanagochi.tokushima.jp +shishikui.tokushima.jp +tokushima.tokushima.jp +wajiki.tokushima.jp +adachi.tokyo.jp +akiruno.tokyo.jp +akishima.tokyo.jp +aogashima.tokyo.jp +arakawa.tokyo.jp +bunkyo.tokyo.jp +chiyoda.tokyo.jp +chofu.tokyo.jp +chuo.tokyo.jp +edogawa.tokyo.jp +fuchu.tokyo.jp +fussa.tokyo.jp +hachijo.tokyo.jp +hachioji.tokyo.jp +hamura.tokyo.jp +higashikurume.tokyo.jp +higashimurayama.tokyo.jp +higashiyamato.tokyo.jp +hino.tokyo.jp +hinode.tokyo.jp +hinohara.tokyo.jp +inagi.tokyo.jp +itabashi.tokyo.jp +katsushika.tokyo.jp +kita.tokyo.jp +kiyose.tokyo.jp +kodaira.tokyo.jp +koganei.tokyo.jp +kokubunji.tokyo.jp +komae.tokyo.jp +koto.tokyo.jp +kouzushima.tokyo.jp +kunitachi.tokyo.jp +machida.tokyo.jp +meguro.tokyo.jp +minato.tokyo.jp +mitaka.tokyo.jp +mizuho.tokyo.jp +musashimurayama.tokyo.jp +musashino.tokyo.jp +nakano.tokyo.jp +nerima.tokyo.jp +ogasawara.tokyo.jp +okutama.tokyo.jp +ome.tokyo.jp +oshima.tokyo.jp +ota.tokyo.jp +setagaya.tokyo.jp +shibuya.tokyo.jp +shinagawa.tokyo.jp +shinjuku.tokyo.jp +suginami.tokyo.jp +sumida.tokyo.jp +tachikawa.tokyo.jp +taito.tokyo.jp +tama.tokyo.jp +toshima.tokyo.jp +chizu.tottori.jp +hino.tottori.jp +kawahara.tottori.jp +koge.tottori.jp +kotoura.tottori.jp +misasa.tottori.jp +nanbu.tottori.jp +nichinan.tottori.jp +sakaiminato.tottori.jp +tottori.tottori.jp +wakasa.tottori.jp +yazu.tottori.jp +yonago.tottori.jp +asahi.toyama.jp +fuchu.toyama.jp +fukumitsu.toyama.jp +funahashi.toyama.jp +himi.toyama.jp +imizu.toyama.jp +inami.toyama.jp +johana.toyama.jp +kamiichi.toyama.jp +kurobe.toyama.jp +nakaniikawa.toyama.jp +namerikawa.toyama.jp +nanto.toyama.jp +nyuzen.toyama.jp +oyabe.toyama.jp +taira.toyama.jp +takaoka.toyama.jp +tateyama.toyama.jp +toga.toyama.jp +tonami.toyama.jp +toyama.toyama.jp +unazuki.toyama.jp +uozu.toyama.jp +yamada.toyama.jp +arida.wakayama.jp +aridagawa.wakayama.jp +gobo.wakayama.jp +hashimoto.wakayama.jp +hidaka.wakayama.jp +hirogawa.wakayama.jp +inami.wakayama.jp +iwade.wakayama.jp +kainan.wakayama.jp +kamitonda.wakayama.jp +katsuragi.wakayama.jp +kimino.wakayama.jp +kinokawa.wakayama.jp +kitayama.wakayama.jp +koya.wakayama.jp +koza.wakayama.jp +kozagawa.wakayama.jp +kudoyama.wakayama.jp +kushimoto.wakayama.jp +mihama.wakayama.jp +misato.wakayama.jp +nachikatsuura.wakayama.jp +shingu.wakayama.jp +shirahama.wakayama.jp +taiji.wakayama.jp +tanabe.wakayama.jp +wakayama.wakayama.jp +yuasa.wakayama.jp +yura.wakayama.jp +asahi.yamagata.jp +funagata.yamagata.jp +higashine.yamagata.jp +iide.yamagata.jp +kahoku.yamagata.jp +kaminoyama.yamagata.jp +kaneyama.yamagata.jp +kawanishi.yamagata.jp +mamurogawa.yamagata.jp +mikawa.yamagata.jp +murayama.yamagata.jp +nagai.yamagata.jp +nakayama.yamagata.jp +nanyo.yamagata.jp +nishikawa.yamagata.jp +obanazawa.yamagata.jp +oe.yamagata.jp +oguni.yamagata.jp +ohkura.yamagata.jp +oishida.yamagata.jp +sagae.yamagata.jp +sakata.yamagata.jp +sakegawa.yamagata.jp +shinjo.yamagata.jp +shirataka.yamagata.jp +shonai.yamagata.jp +takahata.yamagata.jp +tendo.yamagata.jp +tozawa.yamagata.jp +tsuruoka.yamagata.jp +yamagata.yamagata.jp +yamanobe.yamagata.jp +yonezawa.yamagata.jp +yuza.yamagata.jp +abu.yamaguchi.jp +hagi.yamaguchi.jp +hikari.yamaguchi.jp +hofu.yamaguchi.jp +iwakuni.yamaguchi.jp +kudamatsu.yamaguchi.jp +mitou.yamaguchi.jp +nagato.yamaguchi.jp +oshima.yamaguchi.jp +shimonoseki.yamaguchi.jp +shunan.yamaguchi.jp +tabuse.yamaguchi.jp +tokuyama.yamaguchi.jp +toyota.yamaguchi.jp +ube.yamaguchi.jp +yuu.yamaguchi.jp +chuo.yamanashi.jp +doshi.yamanashi.jp +fuefuki.yamanashi.jp +fujikawa.yamanashi.jp +fujikawaguchiko.yamanashi.jp +fujiyoshida.yamanashi.jp +hayakawa.yamanashi.jp +hokuto.yamanashi.jp +ichikawamisato.yamanashi.jp +kai.yamanashi.jp +kofu.yamanashi.jp +koshu.yamanashi.jp +kosuge.yamanashi.jp +minami-alps.yamanashi.jp +minobu.yamanashi.jp +nakamichi.yamanashi.jp +nanbu.yamanashi.jp +narusawa.yamanashi.jp +nirasaki.yamanashi.jp +nishikatsura.yamanashi.jp +oshino.yamanashi.jp +otsuki.yamanashi.jp +showa.yamanashi.jp +tabayama.yamanashi.jp +tsuru.yamanashi.jp +uenohara.yamanashi.jp +yamanakako.yamanashi.jp +yamanashi.yamanashi.jp +#ke +co.ke +or.ke +ne.ke +go.ke +ac.ke +sc.ke +me.ke +mobi.ke +info.ke +#kg +org.kg +net.kg +com.kg +edu.kg +gov.kg +mil.kg +#kh +per.kh +com.kh +edu.kh +gov.kh +mil.kh +net.kh +org.kh +#ki +edu.ki +biz.ki +net.ki +org.ki +gov.ki +info.ki +com.ki +#km +org.km +nom.km +gov.km +prd.km +tm.km +edu.km +mil.km +ass.km +com.km +coop.km +asso.km +presse.km +medecin.km +notaires.km +pharmaciens.km +veterinaire.km +gouv.km +#kn +net.kn +org.kn +edu.kn +gov.kn +com.kp +edu.kp +gov.kp +org.kp +rep.kp +tra.kp +#kr +ac.kr +co.kr +es.kr +go.kr +hs.kr +kg.kr +mil.kr +ms.kr +ne.kr +or.kr +pe.kr +re.kr +sc.kr +busan.kr +chungbuk.kr +chungnam.kr +daegu.kr +daejeon.kr +gangwon.kr +gwangju.kr +gyeongbuk.kr +gyeonggi.kr +gyeongnam.kr +incheon.kr +jeju.kr +jeonbuk.kr +jeonnam.kr +seoul.kr +ulsan.kr +#kw +.com.kw +.edu.kw +.gov.kw +.net.kw +.org.kw +.mil.kw +#ky +edu.ky +gov.ky +com.ky +org.ky +net.ky +#kz +org.kz +edu.kz +net.kz +gov.kz +mil.kz +com.kz +#la +int.la +net.la +info.la +edu.la +gov.la +per.la +com.la +org.la +com.lb +edu.lb +gov.lb +net.lb +org.lb +#lc +com.lc +net.lc +co.lc +org.lc +edu.lc +gov.lc +#li +#lk +gov.lk +sch.lk +net.lk +int.lk +com.lk +org.lk +edu.lk +ngo.lk +soc.lk +web.lk +ltd.lk +assn.lk +grp.lk +hotel.lk +com.lr +edu.lr +gov.lr +org.lr +net.lr +#ls +co.ls +org.ls +#lt +gov.lt +#lu +#lv +com.lv +edu.lv +gov.lv +org.lv +mil.lv +id.lv +net.lv +asn.lv +conf.lv +#ly +com.ly +net.ly +gov.ly +plc.ly +edu.ly +sch.ly +med.ly +org.ly +id.ly +#ma +co.ma +net.ma +gov.ma +org.ma +ac.ma +press.ma +#mc +tm.mc +asso.mc +#md +#me +co.me +net.me +org.me +edu.me +ac.me +gov.me +its.me +priv.me +#mg +org.mg +nom.mg +gov.mg +prd.mg +tm.mg +edu.mg +mil.mg +com.mg +#mh +#mil +#mk +com.mk +org.mk +net.mk +edu.mk +gov.mk +inf.mk +name.mk +#ml +com.ml +edu.ml +gouv.ml +gov.ml +net.ml +org.ml +presse.ml +#mm +net.mm +com.mm +edu.mm +org.mm +gov.mm +#mn +gov.mn +edu.mn +org.mn +#mo +com.mo +net.mo +org.mo +edu.mo +gov.mo +#mobi +#mp +#mq +#mr +gov.mr +#ms +#mt +mt +org.mt +com.mt +gov.mt +edu.mt +net.mt +#mu +com.mu +net.mu +org.mu +gov.mu +ac.mu +co.mu +or.mu +#museum +academy.museum +agriculture.museum +air.museum +airguard.museum +alabama.museum +alaska.museum +amber.museum +ambulance.museum +american.museum +americana.museum +americanantiques.museum +americanart.museum +amsterdam.museum +and.museum +annefrank.museum +anthro.museum +anthropology.museum +antiques.museum +aquarium.museum +arboretum.museum +archaeological.museum +archaeology.museum +architecture.museum +art.museum +artanddesign.museum +artcenter.museum +artdeco.museum +arteducation.museum +artgallery.museum +arts.museum +artsandcrafts.museum +asmatart.museum +assassination.museum +assisi.museum +association.museum +astronomy.museum +atlanta.museum +austin.museum +australia.museum +automotive.museum +aviation.museum +axis.museum +badajoz.museum +baghdad.museum +bahn.museum +bale.museum +baltimore.museum +barcelona.museum +baseball.museum +basel.museum +baths.museum +bauern.museum +beauxarts.museum +beeldengeluid.museum +bellevue.museum +bergbau.museum +berkeley.museum +berlin.museum +bern.museum +bible.museum +bilbao.museum +bill.museum +birdart.museum +birthplace.museum +bonn.museum +boston.museum +botanical.museum +botanicalgarden.museum +botanicgarden.museum +botany.museum +brandywinevalley.museum +brasil.museum +bristol.museum +british.museum +britishcolumbia.museum +broadcast.museum +brunel.museum +brussel.museum +brussels.museum +bruxelles.museum +building.museum +burghof.museum +bus.museum +bushey.museum +cadaques.museum +california.museum +cambridge.museum +can.museum +canada.museum +capebreton.museum +carrier.museum +cartoonart.museum +casadelamoneda.museum +castle.museum +castres.museum +celtic.museum +center.museum +chattanooga.museum +cheltenham.museum +chesapeakebay.museum +chicago.museum +children.museum +childrens.museum +childrensgarden.museum +chiropractic.museum +chocolate.museum +christiansburg.museum +cincinnati.museum +cinema.museum +circus.museum +civilisation.museum +civilization.museum +civilwar.museum +clinton.museum +clock.museum +coal.museum +coastaldefence.museum +cody.museum +coldwar.museum +collection.museum +colonialwilliamsburg.museum +coloradoplateau.museum +columbia.museum +columbus.museum +communication.museum +communications.museum +community.museum +computer.museum +computerhistory.museum +comunicacoes.museum +contemporary.museum +contemporaryart.museum +convent.museum +copenhagen.museum +corporation.museum +correios-e-telecomunicacoes.museum +corvette.museum +costume.museum +countryestate.museum +county.museum +crafts.museum +cranbrook.museum +creation.museum +cultural.museum +culturalcenter.museum +culture.museum +cyber.museum +cymru.museum +dali.museum +dallas.museum +database.museum +ddr.museum +decorativearts.museum +delaware.museum +delmenhorst.museum +denmark.museum +depot.museum +design.museum +detroit.museum +dinosaur.museum +discovery.museum +dolls.museum +donostia.museum +durham.museum +eastafrica.museum +eastcoast.museum +education.museum +educational.museum +egyptian.museum +eisenbahn.museum +elburg.museum +elvendrell.museum +embroidery.museum +encyclopedic.museum +england.museum +entomology.museum +environment.museum +environmentalconservation.museum +epilepsy.museum +essex.museum +estate.museum +ethnology.museum +exeter.museum +exhibition.museum +family.museum +farm.museum +farmequipment.museum +farmers.museum +farmstead.museum +field.museum +figueres.museum +filatelia.museum +film.museum +fineart.museum +finearts.museum +finland.museum +flanders.museum +florida.museum +force.museum +fortmissoula.museum +fortworth.museum +foundation.museum +francaise.museum +frankfurt.museum +franziskaner.museum +freemasonry.museum +freiburg.museum +fribourg.museum +frog.museum +fundacio.museum +furniture.museum +gallery.museum +garden.museum +gateway.museum +geelvinck.museum +gemological.museum +geology.museum +georgia.museum +giessen.museum +glas.museum +glass.museum +gorge.museum +grandrapids.museum +graz.museum +guernsey.museum +halloffame.museum +hamburg.museum +handson.museum +harvestcelebration.museum +hawaii.museum +health.museum +heimatunduhren.museum +hellas.museum +helsinki.museum +hembygdsforbund.museum +heritage.museum +histoire.museum +historical.museum +historicalsociety.museum +historichouses.museum +historisch.museum +historisches.museum +history.museum +historyofscience.museum +horology.museum +house.museum +humanities.museum +illustration.museum +imageandsound.museum +indian.museum +indiana.museum +indianapolis.museum +indianmarket.museum +intelligence.museum +interactive.museum +iraq.museum +iron.museum +isleofman.museum +jamison.museum +jefferson.museum +jerusalem.museum +jewelry.museum +jewish.museum +jewishart.museum +jfk.museum +journalism.museum +judaica.museum +judygarland.museum +juedisches.museum +juif.museum +karate.museum +karikatur.museum +kids.museum +koebenhavn.museum +koeln.museum +kunst.museum +kunstsammlung.museum +kunstunddesign.museum +labor.museum +labour.museum +lajolla.museum +lancashire.museum +landes.museum +lans.museum +lans.museum +larsson.museum +lewismiller.museum +lincoln.museum +linz.museum +living.museum +livinghistory.museum +localhistory.museum +london.museum +losangeles.museum +louvre.museum +loyalist.museum +lucerne.museum +luxembourg.museum +luzern.museum +mad.museum +madrid.museum +mallorca.museum +manchester.museum +mansion.museum +mansions.museum +manx.museum +marburg.museum +maritime.museum +maritimo.museum +maryland.museum +marylhurst.museum +media.museum +medical.museum +medizinhistorisches.museum +meeres.museum +memorial.museum +mesaverde.museum +michigan.museum +midatlantic.museum +military.museum +mill.museum +miners.museum +mining.museum +minnesota.museum +missile.museum +missoula.museum +modern.museum +moma.museum +money.museum +monmouth.museum +monticello.museum +montreal.museum +moscow.museum +motorcycle.museum +muenchen.museum +muenster.museum +mulhouse.museum +muncie.museum +museet.museum +museumcenter.museum +museumvereniging.museum +music.museum +national.museum +nationalfirearms.museum +nationalheritage.museum +nativeamerican.museum +naturalhistory.museum +naturalhistorymuseum.museum +naturalsciences.museum +nature.museum +naturhistorisches.museum +natuurwetenschappen.museum +naumburg.museum +naval.museum +nebraska.museum +neues.museum +newhampshire.museum +newjersey.museum +newmexico.museum +newport.museum +newspaper.museum +newyork.museum +niepce.museum +norfolk.museum +north.museum +nrw.museum +nuernberg.museum +nuremberg.museum +nyc.museum +nyny.museum +oceanographic.museum +oceanographique.museum +omaha.museum +online.museum +ontario.museum +openair.museum +oregon.museum +oregontrail.museum +otago.museum +oxford.museum +pacific.museum +paderborn.museum +palace.museum +paleo.museum +palmsprings.museum +panama.museum +paris.museum +pasadena.museum +pharmacy.museum +philadelphia.museum +philadelphiaarea.museum +philately.museum +phoenix.museum +photography.museum +pilots.museum +pittsburgh.museum +planetarium.museum +plantation.museum +plants.museum +plaza.museum +portal.museum +portland.museum +portlligat.museum +posts-and-telecommunications.museum +preservation.museum +presidio.museum +press.museum +project.museum +public.museum +pubol.museum +quebec.museum +railroad.museum +railway.museum +research.museum +resistance.museum +riodejaneiro.museum +rochester.museum +rockart.museum +roma.museum +russia.museum +saintlouis.museum +salem.museum +salvadordali.museum +salzburg.museum +sandiego.museum +sanfrancisco.museum +santabarbara.museum +santacruz.museum +santafe.museum +saskatchewan.museum +satx.museum +savannahga.museum +schlesisches.museum +schoenbrunn.museum +schokoladen.museum +school.museum +schweiz.museum +science.museum +scienceandhistory.museum +scienceandindustry.museum +sciencecenter.museum +sciencecenters.museum +science-fiction.museum +sciencehistory.museum +sciences.museum +sciencesnaturelles.museum +scotland.museum +seaport.museum +settlement.museum +settlers.museum +shell.museum +sherbrooke.museum +sibenik.museum +silk.museum +ski.museum +skole.museum +society.museum +sologne.museum +soundandvision.museum +southcarolina.museum +southwest.museum +space.museum +spy.museum +square.museum +stadt.museum +stalbans.museum +starnberg.museum +state.museum +stateofdelaware.museum +station.museum +steam.museum +steiermark.museum +stjohn.museum +stockholm.museum +stpetersburg.museum +stuttgart.museum +suisse.museum +surgeonshall.museum +surrey.museum +svizzera.museum +sweden.museum +sydney.museum +tank.museum +tcm.museum +technology.museum +telekommunikation.museum +television.museum +texas.museum +textile.museum +theater.museum +time.museum +timekeeping.museum +topology.museum +torino.museum +touch.museum +town.museum +transport.museum +tree.museum +trolley.museum +trust.museum +trustee.museum +uhren.museum +ulm.museum +undersea.museum +university.museum +usa.museum +usantiques.museum +usarts.museum +uscountryestate.museum +usculture.museum +usdecorativearts.museum +usgarden.museum +ushistory.museum +ushuaia.museum +uslivinghistory.museum +utah.museum +uvic.museum +valley.museum +vantaa.museum +versailles.museum +viking.museum +village.museum +virginia.museum +virtual.museum +virtuel.museum +vlaanderen.museum +volkenkunde.museum +wales.museum +wallonie.museum +war.museum +washingtondc.museum +watchandclock.museum +watch-and-clock.museum +western.museum +westfalen.museum +whaling.museum +wildlife.museum +williamsburg.museum +windmill.museum +workshop.museum +york.museum +yorkshire.museum +yosemite.museum +youth.museum +zoological.museum +zoology.museum +#mv +aero.mv +biz.mv +com.mv +coop.mv +edu.mv +gov.mv +info.mv +int.mv +mil.mv +museum.mv +name.mv +net.mv +org.mv +pro.mv +#mw +ac.mw +biz.mw +co.mw +com.mw +coop.mw +edu.mw +gov.mw +int.mw +museum.mw +net.mw +org.mw +#mx +com.mx +org.mx +gob.mx +edu.mx +net.mx +#my +com.my +net.my +org.my +gov.my +edu.my +mil.my +name.my +#mz +adv.mz +ac.mz +co.mz +org.mz +gov.mz +edu.mz +#na +info.na +pro.na +name.na +school.na +or.na +dr.na +us.na +mx.na +ca.na +in.na +cc.na +tv.na +ws.na +mobi.na +co.na +com.na +org.na +#name +#nc +asso.nc +#ne +#net +#nf +com.nf +net.nf +per.nf +rec.nf +web.nf +arts.nf +firm.nf +info.nf +other.nf +store.nf +ac.ng +com.ng +edu.ng +gov.ng +net.ng +org.ng +#ni +gob.ni +co.ni +ac.ni +org.ni +nom.ni +net.ni +mil.ni +#nl +bv.nl +#no +fhs.no +vgs.no +fylkesbibl.no +folkebibl.no +museum.no +idrett.no +priv.no +mil.no +stat.no +dep.no +kommune.no +herad.no +aa.no +ah.no +bu.no +fm.no +hl.no +hm.no +jan-mayen.no +mr.no +nl.no +nt.no +of.no +ol.no +oslo.no +rl.no +sf.no +st.no +svalbard.no +tm.no +tr.no +va.no +vf.no +gs.aa.no +gs.ah.no +gs.bu.no +gs.fm.no +gs.hl.no +gs.hm.no +gs.jan-mayen.no +gs.mr.no +gs.nl.no +gs.nt.no +gs.of.no +gs.ol.no +gs.oslo.no +gs.rl.no +gs.sf.no +gs.st.no +gs.svalbard.no +gs.tm.no +gs.tr.no +gs.va.no +gs.vf.no +akrehamn.no +akrehamn.no +algard.no +algard.no +arna.no +brumunddal.no +bryne.no +bronnoysund.no +bronnoysund.no +drobak.no +drobak.no +egersund.no +fetsund.no +floro.no +floro.no +fredrikstad.no +hokksund.no +honefoss.no +honefoss.no +jessheim.no +jorpeland.no +jorpeland.no +kirkenes.no +kopervik.no +krokstadelva.no +langevag.no +langevag.no +leirvik.no +mjondalen.no +mjondalen.no +mo-i-rana.no +mosjoen.no +mosjoen.no +nesoddtangen.no +orkanger.no +osoyro.no +osoyro.no +raholt.no +raholt.no +sandnessjoen.no +sandnessjoen.no +skedsmokorset.no +slattum.no +spjelkavik.no +stathelle.no +stavern.no +stjordalshalsen.no +stjordalshalsen.no +tananger.no +tranby.no +vossevangen.no +afjord.no +afjord.no +agdenes.no +al.no +al.no +alesund.no +alesund.no +alstahaug.no +alta.no +alta.no +alaheadju.no +alaheadju.no +alvdal.no +amli.no +amli.no +amot.no +amot.no +andebu.no +andoy.no +andoy.no +andasuolo.no +ardal.no +ardal.no +aremark.no +arendal.no +as.no +aseral.no +aseral.no +asker.no +askim.no +askvoll.no +askoy.no +askoy.no +asnes.no +asnes.no +audnedaln.no +aukra.no +aure.no +aurland.no +aurskog-holand.no +aurskog-holand.no +austevoll.no +austrheim.no +averoy.no +averoy.no +balestrand.no +ballangen.no +balat.no +balat.no +balsfjord.no +bahccavuotna.no +bahccavuotna.no +bamble.no +bardu.no +beardu.no +beiarn.no +bajddar.no +bajddar.no +baidar.no +baidar.no +berg.no +bergen.no +berlevag.no +berlevag.no +bearalvahki.no +bearalvahki.no +bindal.no +birkenes.no +bjarkoy.no +bjarkoy.no +bjerkreim.no +bjugn.no +bodo.no +bodo.no +badaddja.no +badaddja.no +budejju.no +bokn.no +bremanger.no +bronnoy.no +bronnoy.no +bygland.no +bykle.no +barum.no +bo.telemark.no +bo.telemark.no +bo.nordland.no +bo.nordland.no +bievat.no +bievat.no +bomlo.no +bomlo.no +batsfjord.no +batsfjord.no +bahcavuotna.no +bahcavuotna.no +dovre.no +drammen.no +drangedal.no +dyroy.no +dyroy.no +donna.no +donna.no +eid.no +eidfjord.no +eidsberg.no +eidskog.no +eidsvoll.no +eigersund.no +elverum.no +enebakk.no +engerdal.no +etne.no +etnedal.no +evenes.no +evenassi.no +evenassi.no +evje-og-hornnes.no +farsund.no +fauske.no +fuossko.no +fuoisku.no +fedje.no +fet.no +finnoy.no +finnoy.no +fitjar.no +fjaler.no +fjell.no +flakstad.no +flatanger.no +flekkefjord.no +flesberg.no +flora.no +fla.no +fla.no +folldal.no +forsand.no +fosnes.no +frei.no +frogn.no +froland.no +frosta.no +frana.no +froya.no +froya.no +fusa.no +fyresdal.no +forde.no +forde.no +gamvik.no +gangaviika.no +gaular.no +gausdal.no +gildeskal.no +gildeskal.no +giske.no +gjemnes.no +gjerdrum.no +gjerstad.no +gjesdal.no +gjovik.no +gjovik.no +gloppen.no +gol.no +gran.no +grane.no +granvin.no +gratangen.no +grimstad.no +grong.no +kraanghke.no +kraanghke.no +grue.no +gulen.no +hadsel.no +halden.no +halsa.no +hamar.no +hamaroy.no +habmer.no +habmer.no +hapmir.no +hapmir.no +hammerfest.no +hammarfeasta.no +hammarfeasta.no +haram.no +hareid.no +harstad.no +hasvik.no +aknoluokta.no +hattfjelldal.no +aarborte.no +haugesund.no +hemne.no +hemnes.no +hemsedal.no +heroy.more-og-romsdal.no +heroy.more-og-romsdal.no +heroy.nordland.no +heroy.nordland.no +hitra.no +hjartdal.no +hjelmeland.no +hobol.no +hobol.no +hof.no +hol.no +hole.no +holmestrand.no +holtalen.no +holtalen.no +hornindal.no +horten.no +hurdal.no +hurum.no +hvaler.no +hyllestad.no +hagebostad.no +hoyanger.no +hoyanger.no +hoylandet.no +hoylandet.no +ha.no +ha.no +ibestad.no +inderoy.no +inderoy.no +iveland.no +jevnaker.no +jondal.no +jolster.no +jolster.no +karasjok.no +karasjohka.no +karasjohka.no +karlsoy.no +galsa.no +galsa.no +karmoy.no +karmoy.no +kautokeino.no +guovdageaidnu.no +klepp.no +klabu.no +kongsberg.no +kongsvinger.no +kragero.no +kragero.no +kristiansand.no +kristiansund.no +krodsherad.no +krodsherad.no +kvalsund.no +rahkkeravju.no +rahkkeravju.no +kvam.no +kvinesdal.no +kvinnherad.no +kviteseid.no +kvitsoy.no +kvitsoy.no +kvafjord.no +giehtavuoatna.no +kvanangen.no +navuotna.no +navuotna.no +kafjord.no +kafjord.no +gaivuotna.no +gaivuotna.no +larvik.no +lavangen.no +lavagis.no +loabat.no +loabat.no +lebesby.no +davvesiida.no +leikanger.no +leirfjord.no +leka.no +leksvik.no +lenvik.no +leangaviika.no +lesja.no +levanger.no +lier.no +lierne.no +lillehammer.no +lillesand.no +lindesnes.no +lindas.no +lindas.no +lom.no +loppa.no +lahppi.no +lahppi.no +lund.no +lunner.no +luroy.no +luroy.no +luster.no +lyngdal.no +lyngen.no +ivgu.no +lardal.no +lerdal.no +lodingen.no +lodingen.no +lorenskog.no +lorenskog.no +loten.no +loten.no +malvik.no +masoy.no +masoy.no +muosat.no +muosat.no +mandal.no +marker.no +marnardal.no +masfjorden.no +meland.no +meldal.no +melhus.no +meloy.no +meloy.no +meraker.no +meraker.no +moareke.no +moareke.no +midsund.no +midtre-gauldal.no +modalen.no +modum.no +molde.no +moskenes.no +moss.no +mosvik.no +malselv.no +malselv.no +malatvuopmi.no +malatvuopmi.no +namdalseid.no +aejrie.no +namsos.no +namsskogan.no +naamesjevuemie.no +naamesjevuemie.no +laakesvuemie.no +nannestad.no +narvik.no +narviika.no +naustdal.no +nedre-eiker.no +nes.akershus.no +nes.buskerud.no +nesna.no +nesodden.no +nesseby.no +unjarga.no +unjarga.no +nesset.no +nissedal.no +nittedal.no +nord-aurdal.no +nord-fron.no +nord-odal.no +norddal.no +nordkapp.no +davvenjarga.no +davvenjarga.no +nordre-land.no +nordreisa.no +raisa.no +raisa.no +nore-og-uvdal.no +notodden.no +naroy.no +notteroy.no +notteroy.no +odda.no +oksnes.no +oksnes.no +oppdal.no +oppegard.no +oppegard.no +orkdal.no +orland.no +orland.no +orskog.no +orskog.no +orsta.no +orsta.no +os.hedmark.no +os.hordaland.no +osen.no +osteroy.no +osteroy.no +ostre-toten.no +ostre-toten.no +overhalla.no +ovre-eiker.no +ovre-eiker.no +oyer.no +oyer.no +oygarden.no +oygarden.no +oystre-slidre.no +oystre-slidre.no +porsanger.no +porsangu.no +porsgrunn.no +radoy.no +radoy.no +rakkestad.no +rana.no +ruovat.no +randaberg.no +rauma.no +rendalen.no +rennebu.no +rennesoy.no +rennesoy.no +rindal.no +ringebu.no +ringerike.no +ringsaker.no +rissa.no +risor.no +risor.no +roan.no +rollag.no +rygge.no +ralingen.no +rodoy.no +rodoy.no +romskog.no +romskog.no +roros.no +roros.no +rost.no +rost.no +royken.no +royken.no +royrvik.no +royrvik.no +rade.no +rade.no +salangen.no +siellak.no +saltdal.no +salat.no +salat.no +salat.no +samnanger.no +sande.more-og-romsdal.no +sande.more-og-romsdal.no +sande.vestfold.no +sandefjord.no +sandnes.no +sandoy.no +sandoy.no +sarpsborg.no +sauda.no +sauherad.no +sel.no +selbu.no +selje.no +seljord.no +sigdal.no +siljan.no +sirdal.no +skaun.no +skedsmo.no +ski.no +skien.no +skiptvet.no +skjervoy.no +skjervoy.no +skierva.no +skierva.no +skjak.no +skjak.no +skodje.no +skanland.no +skanland.no +skanit.no +skanit.no +smola.no +smola.no +snillfjord.no +snasa.no +snasa.no +snoasa.no +snaase.no +snaase.no +sogndal.no +sokndal.no +sola.no +solund.no +songdalen.no +sortland.no +spydeberg.no +stange.no +stavanger.no +steigen.no +steinkjer.no +stjordal.no +stjordal.no +stokke.no +stor-elvdal.no +stord.no +stordal.no +storfjord.no +omasvuotna.no +strand.no +stranda.no +stryn.no +sula.no +suldal.no +sund.no +sunndal.no +surnadal.no +sveio.no +svelvik.no +sykkylven.no +sogne.no +sogne.no +somna.no +somna.no +sondre-land.no +sondre-land.no +sor-aurdal.no +sor-aurdal.no +sor-fron.no +sor-fron.no +sor-odal.no +sor-odal.no +sor-varanger.no +sor-varanger.no +matta-varjjat.no +matta-varjjat.no +sorfold.no +sorfold.no +sorreisa.no +sorreisa.no +sorum.no +sorum.no +tana.no +deatnu.no +time.no +tingvoll.no +tinn.no +tjeldsund.no +dielddanuorri.no +tjome.no +tjome.no +tokke.no +tolga.no +torsken.no +tranoy.no +tranoy.no +tromso.no +tromso.no +tromsa.no +romsa.no +trondheim.no +troandin.no +trysil.no +trana.no +trogstad.no +trogstad.no +tvedestrand.no +tydal.no +tynset.no +tysfjord.no +divtasvuodna.no +divttasvuotna.no +tysnes.no +tysvar.no +tonsberg.no +tonsberg.no +ullensaker.no +ullensvang.no +ulvik.no +utsira.no +vadso.no +vadso.no +cahcesuolo.no +cahcesuolo.no +vaksdal.no +valle.no +vang.no +vanylven.no +vardo.no +vardo.no +varggat.no +varggat.no +vefsn.no +vaapste.no +vega.no +vegarshei.no +vegarshei.no +vennesla.no +verdal.no +verran.no +vestby.no +vestnes.no +vestre-slidre.no +vestre-toten.no +vestvagoy.no +vestvagoy.no +vevelstad.no +vik.no +vikna.no +vindafjord.no +volda.no +voss.no +varoy.no +vagan.no +vagan.no +voagat.no +vagsoy.no +vagsoy.no +vaga.no +vaga.no +valer.ostfold.no +valer.ostfold.no +valer.hedmark.no +valer.hedmark.no +#np +com.np +edu.np +gov.np +mil.np +net.np +org.np +name.np +pro.np +info.np +#nr +biz.nr +info.nr +gov.nr +edu.nr +org.nr +net.nr +com.nr +#nu +#nz +ac.nz +co.nz +geek.nz +gen.nz +kiwi.nz +maori.nz +net.nz +org.nz +school.nz +cri.nz +govt.nz +iwi.nz +parliament.nz +mil.nz +health.nz +#om +om +co.om +com.om +org.om +net.om +edu.om +gov.om +museum.om +pro.om +med.om +#org +#pa +ac.pa +gob.pa +com.pa +org.pa +sld.pa +edu.pa +net.pa +ing.pa +abo.pa +med.pa +nom.pa +#pe +edu.pe +gob.pe +nom.pe +mil.pe +org.pe +com.pe +net.pe +#pf +com.pf +org.pf +edu.pf +#pg +com.pg +net.pg +ac.pg +gov.pg +mil.pg +org.pg. +#ph +com.ph +net.ph +org.ph +gov.ph +edu.ph +ngo.ph +mil.ph +i.ph +#pk +com.pk +net.pk +edu.pk +org.pk +fam.pk +biz.pk +web.pk +gov.pk +gob.pk +gok.pk +gon.pk +gop.pk +gos.pk +info.pk +#pl +aid.pl +agro.pl +atm.pl +auto.pl +biz.pl +com.pl +edu.pl +gmina.pl +gsm.pl +info.pl +mail.pl +miasta.pl +media.pl +mil.pl +net.pl +nieruchomosci.pl +nom.pl +org.pl +pc.pl +powiat.pl +priv.pl +realestate.pl +rel.pl +sex.pl +shop.pl +sklep.pl +sos.pl +szkola.pl +targi.pl +tm.pl +tourism.pl +travel.pl +turystyka.pl +6bone.pl +art.pl +mbone.pl +gov.pl +uw.gov.pl +um.gov.pl +ug.gov.pl +upow.gov.pl +starostwo.gov.pl +so.gov.pl +sr.gov.pl +po.gov.pl +pa.gov.pl +ngo.pl +irc.pl +usenet.pl +augustow.pl +babia-gora.pl +bedzin.pl +beskidy.pl +bialowieza.pl +bialystok.pl +bielawa.pl +bieszczady.pl +boleslawiec.pl +bydgoszcz.pl +bytom.pl +cieszyn.pl +czeladz.pl +czest.pl +dlugoleka.pl +elblag.pl +elk.pl +glogow.pl +gniezno.pl +gorlice.pl +grajewo.pl +ilawa.pl +jaworzno.pl +jelenia-gora.pl +jgora.pl +kalisz.pl +kazimierz-dolny.pl +karpacz.pl +kartuzy.pl +kaszuby.pl +katowice.pl +kepno.pl +ketrzyn.pl +klodzko.pl +kobierzyce.pl +kolobrzeg.pl +konin.pl +konskowola.pl +kutno.pl +lapy.pl +lebork.pl +legnica.pl +lezajsk.pl +limanowa.pl +lomza.pl +lowicz.pl +lubin.pl +lukow.pl +malbork.pl +malopolska.pl +mazowsze.pl +mazury.pl +mielec.pl +mielno.pl +mragowo.pl +naklo.pl +nowaruda.pl +nysa.pl +olawa.pl +olecko.pl +olkusz.pl +olsztyn.pl +opoczno.pl +opole.pl +ostroda.pl +ostroleka.pl +ostrowiec.pl +ostrowwlkp.pl +pila.pl +pisz.pl +podhale.pl +podlasie.pl +polkowice.pl +pomorze.pl +pomorskie.pl +prochowice.pl +pruszkow.pl +przeworsk.pl +pulawy.pl +radom.pl +rawa-maz.pl +rybnik.pl +rzeszow.pl +sanok.pl +sejny.pl +siedlce.pl +slask.pl +slupsk.pl +sosnowiec.pl +stalowa-wola.pl +skoczow.pl +starachowice.pl +stargard.pl +suwalki.pl +swidnica.pl +swiebodzin.pl +swinoujscie.pl +szczecin.pl +szczytno.pl +tarnobrzeg.pl +tgory.pl +turek.pl +tychy.pl +ustka.pl +walbrzych.pl +warmia.pl +warszawa.pl +waw.pl +wegrow.pl +wielun.pl +wlocl.pl +wloclawek.pl +wodzislaw.pl +wolomin.pl +wroclaw.pl +zachpomor.pl +zagan.pl +zarow.pl +zgora.pl +zgorzelec.pl +gda.pl +gdansk.pl +gdynia.pl +med.pl +sopot.pl +gliwice.pl +krakow.pl +poznan.pl +wroc.pl +zakopane.pl +#pm +#pn +gov.pn +co.pn +org.pn +edu.pn +net.pn +#post +#pr +com.pr +net.pr +org.pr +gov.pr +edu.pr +isla.pr +pro.pr +biz.pr +info.pr +name.pr +est.pr +prof.pr +ac.pr +#pro +aca.pro +bar.pro +cpa.pro +jur.pro +law.pro +med.pro +eng.pro +#ps +edu.ps +gov.ps +sec.ps +plo.ps +com.ps +org.ps +net.ps +#pt +net.pt +gov.pt +org.pt +edu.pt +int.pt +publ.pt +com.pt +nome.pt +#pw +co.pw +ne.pw +or.pw +ed.pw +go.pw +belau.pw +#py +com.py +coop.py +edu.py +gov.py +mil.py +net.py +org.py +#qa +com.qa +edu.qa +gov.qa +mil.qa +name.qa +net.qa +org.qa +sch.qa +#re +com.re +asso.re +nom.re +#ro +com.ro +org.ro +tm.ro +nt.ro +nom.ro +info.ro +rec.ro +arts.ro +firm.ro +store.ro +www.ro +#rs +co.rs +org.rs +edu.rs +ac.rs +gov.rs +in.rs +#ru +ac.ru +com.ru +edu.ru +int.ru +net.ru +org.ru +pp.ru +adygeya.ru +altai.ru +amur.ru +arkhangelsk.ru +astrakhan.ru +bashkiria.ru +belgorod.ru +bir.ru +bryansk.ru +buryatia.ru +cbg.ru +chel.ru +chelyabinsk.ru +chita.ru +chukotka.ru +chuvashia.ru +dagestan.ru +dudinka.ru +e-burg.ru +grozny.ru +irkutsk.ru +ivanovo.ru +izhevsk.ru +jar.ru +joshkar-ola.ru +kalmykia.ru +kaluga.ru +kamchatka.ru +karelia.ru +kazan.ru +kchr.ru +kemerovo.ru +khabarovsk.ru +khakassia.ru +khv.ru +kirov.ru +koenig.ru +komi.ru +kostroma.ru +krasnoyarsk.ru +kuban.ru +kurgan.ru +kursk.ru +lipetsk.ru +magadan.ru +mari.ru +mari-el.ru +marine.ru +mordovia.ru +mosreg.ru +msk.ru +murmansk.ru +nalchik.ru +nnov.ru +nov.ru +novosibirsk.ru +nsk.ru +omsk.ru +orenburg.ru +oryol.ru +palana.ru +penza.ru +perm.ru +pskov.ru +ptz.ru +rnd.ru +ryazan.ru +sakhalin.ru +samara.ru +saratov.ru +simbirsk.ru +smolensk.ru +spb.ru +stavropol.ru +stv.ru +surgut.ru +tambov.ru +tatarstan.ru +tom.ru +tomsk.ru +tsaritsyn.ru +tsk.ru +tula.ru +tuva.ru +tver.ru +tyumen.ru +udm.ru +udmurtia.ru +ulan-ude.ru +vladikavkaz.ru +vladimir.ru +vladivostok.ru +volgograd.ru +vologda.ru +voronezh.ru +vrn.ru +vyatka.ru +yakutia.ru +yamal.ru +yaroslavl.ru +yekaterinburg.ru +yuzhno-sakhalinsk.ru +amursk.ru +baikal.ru +cmw.ru +fareast.ru +jamal.ru +kms.ru +k-uralsk.ru +kustanai.ru +kuzbass.ru +magnitka.ru +mytis.ru +nakhodka.ru +nkz.ru +norilsk.ru +oskol.ru +pyatigorsk.ru +rubtsovsk.ru +snz.ru +syzran.ru +vdonsk.ru +zgrad.ru +gov.ru +mil.ru +test.ru +#rw +gov.rw +net.rw +edu.rw +ac.rw +com.rw +co.rw +int.rw +mil.rw +gouv.rw +#sa +com.sa +net.sa +org.sa +gov.sa +med.sa +pub.sa +edu.sa +sch.sa +#sb +com.sb +edu.sb +gov.sb +net.sb +org.sb +#sc +com.sc +gov.sc +net.sc +org.sc +edu.sc +#sd +com.sd +net.sd +org.sd +edu.sd +med.sd +tv.sd +gov.sd +info.sd +#se +a.se +ac.se +b.se +bd.se +brand.se +c.se +d.se +e.se +f.se +fh.se +fhsk.se +fhv.se +g.se +h.se +i.se +k.se +komforb.se +kommunalforbund.se +komvux.se +l.se +lanbib.se +m.se +n.se +naturbruksgymn.se +o.se +org.se +p.se +parti.se +pp.se +press.se +r.se +s.se +sshn.se +t.se +tm.se +u.se +w.se +x.se +y.se +z.se +#sg +com.sg +net.sg +org.sg +gov.sg +edu.sg +per.sg +#sh +com.sh +net.sh +gov.sh +org.sh +mil.sh +#si +#sk +#sl +com.sl +net.sl +edu.sl +gov.sl +org.sl +#sm +#sn +art.sn +com.sn +edu.sn +gouv.sn +org.sn +perso.sn +univ.sn +#so +com.so +net.so +org.so +#sr +#st +co.st +com.st +consulado.st +edu.st +embaixada.st +gov.st +mil.st +net.st +org.st +principe.st +saotome.st +store.st +#su +#sv +edu.sv +gob.sv +com.sv +org.sv +red.sv +#sx +gov.sx +#sy +edu.sy +gov.sy +net.sy +mil.sy +com.sy +org.sy +#sz +co.sz +ac.sz +org.sz +#tc +#td +#tel +#tf +#tg +#th +ac.th +co.th +go.th +in.th +mi.th +net.th +or.th +#tj +ac.tj +biz.tj +co.tj +com.tj +edu.tj +go.tj +gov.tj +int.tj +mil.tj +name.tj +net.tj +nic.tj +org.tj +test.tj +web.tj +#tk +#tl +gov.tl +#tm +com.tm +co.tm +org.tm +net.tm +nom.tm +gov.tm +mil.tm +edu.tm +#tn +com.tn +ens.tn +fin.tn +gov.tn +ind.tn +intl.tn +nat.tn +net.tn +org.tn +info.tn +perso.tn +tourism.tn +edunet.tn +rnrt.tn +rns.tn +rnu.tn +mincom.tn +agrinet.tn +defense.tn +turen.tn +#to +com.to +gov.to +net.to +org.to +edu.to +mil.to +#tr +com.tr +info.tr +biz.tr +net.tr +org.tr +web.tr +gen.tr +av.tr +dr.tr +bbs.tr +name.tr +tel.tr +gov.tr +bel.tr +pol.tr +mil.tr +k12.tr +edu.tr +#travel +#tt +co.tt +com.tt +org.tt +net.tt +biz.tt +info.tt +pro.tt +int.tt +coop.tt +jobs.tt +mobi.tt +travel.tt +museum.tt +aero.tt +name.tt +gov.tt +edu.tt +#tv +#tw +edu.tw +gov.tw +mil.tw +com.tw +net.tw +org.tw +idv.tw +game.tw +ebiz.tw +club.tw +ac.tz +co.tz +go.tz +hotel.tz +info.tz +me.tz +mil.tz +mobi.tz +ne.tz +or.tz +sc.tz +tv.tz +#ua +com.ua +edu.ua +gov.ua +in.ua +net.ua +org.ua +cherkassy.ua +cherkasy.ua +chernigov.ua +chernihiv.ua +chernivtsi.ua +chernovtsy.ua +ck.ua +cn.ua +cr.ua +crimea.ua +cv.ua +dn.ua +dnepropetrovsk.ua +dnipropetrovsk.ua +dominic.ua +donetsk.ua +dp.ua +if.ua +ivano-frankivsk.ua +kh.ua +kharkiv.ua +kharkov.ua +kherson.ua +khmelnitskiy.ua +khmelnytskyi.ua +kiev.ua +kirovograd.ua +km.ua +kr.ua +krym.ua +ks.ua +kv.ua +kyiv.ua +lg.ua +lt.ua +lugansk.ua +lutsk.ua +lv.ua +lviv.ua +mk.ua +mykolaiv.ua +nikolaev.ua +od.ua +odesa.ua +odessa.ua +pl.ua +poltava.ua +rivne.ua +rovno.ua +rv.ua +sb.ua +sebastopol.ua +sevastopol.ua +sm.ua +sumy.ua +te.ua +ternopil.ua +uz.ua +uzhgorod.ua +vinnica.ua +vinnytsia.ua +vn.ua +volyn.ua +yalta.ua +zaporizhzhe.ua +zaporizhzhia.ua +zhitomir.ua +zhytomyr.ua +zp.ua +zt.ua +co.ua +pp.ua +#ug +co.ug +or.ug +ac.ug +sc.ug +go.ug +ne.ug +com.ug +org.ug +#uk +ac.uk +co.uk +gov.uk +judiciary.uk +ltd.uk +me.uk +mod.uk +net.uk +nhs.uk +nic.uk +org.uk +parliament.uk +plc.uk +police.uk +sch.uk +bl.uk +jet.uk +nls.uk +#us +dni.us +fed.us +isa.us +kids.us +nsn.us +ak.us +al.us +ar.us +as.us +az.us +ca.us +co.us +ct.us +dc.us +de.us +fl.us +ga.us +gu.us +hi.us +ia.us +id.us +il.us +in.us +ks.us +ky.us +la.us +ma.us +md.us +me.us +mi.us +mn.us +mo.us +ms.us +mt.us +nc.us +nd.us +ne.us +nh.us +nj.us +nm.us +nv.us +ny.us +oh.us +ok.us +or.us +pa.us +pr.us +ri.us +sc.us +sd.us +tn.us +tx.us +ut.us +vi.us +vt.us +va.us +wa.us +wi.us +wv.us +wy.us +k12.ak.us +k12.al.us +k12.ar.us +k12.as.us +k12.az.us +k12.ca.us +k12.co.us +k12.ct.us +k12.dc.us +k12.de.us +k12.fl.us +k12.ga.us +k12.gu.us +k12.ia.us +k12.id.us +k12.il.us +k12.in.us +k12.ks.us +k12.ky.us +k12.la.us +k12.ma.us +k12.md.us +k12.me.us +k12.mi.us +k12.mn.us +k12.mo.us +k12.ms.us +k12.mt.us +k12.nc.us +k12.nd.us +k12.ne.us +k12.nh.us +k12.nj.us +k12.nm.us +k12.nv.us +k12.ny.us +k12.oh.us +k12.ok.us +k12.or.us +k12.pa.us +k12.pr.us +k12.ri.us +k12.sc.us +k12.sd.us +k12.tn.us +k12.tx.us +k12.ut.us +k12.vi.us +k12.vt.us +k12.va.us +k12.wa.us +k12.wi.us +k12.wv.us +k12.wy.us +cc.ak.us +cc.al.us +cc.ar.us +cc.as.us +cc.az.us +cc.ca.us +cc.co.us +cc.ct.us +cc.dc.us +cc.de.us +cc.fl.us +cc.ga.us +cc.gu.us +cc.hi.us +cc.ia.us +cc.id.us +cc.il.us +cc.in.us +cc.ks.us +cc.ky.us +cc.la.us +cc.ma.us +cc.md.us +cc.me.us +cc.mi.us +cc.mn.us +cc.mo.us +cc.ms.us +cc.mt.us +cc.nc.us +cc.nd.us +cc.ne.us +cc.nh.us +cc.nj.us +cc.nm.us +cc.nv.us +cc.ny.us +cc.oh.us +cc.ok.us +cc.or.us +cc.pa.us +cc.pr.us +cc.ri.us +cc.sc.us +cc.sd.us +cc.tn.us +cc.tx.us +cc.ut.us +cc.vi.us +cc.vt.us +cc.va.us +cc.wa.us +cc.wi.us +cc.wv.us +cc.wy.us +lib.ak.us +lib.al.us +lib.ar.us +lib.as.us +lib.az.us +lib.ca.us +lib.co.us +lib.ct.us +lib.dc.us +lib.de.us +lib.fl.us +lib.ga.us +lib.gu.us +lib.hi.us +lib.ia.us +lib.id.us +lib.il.us +lib.in.us +lib.ks.us +lib.ky.us +lib.la.us +lib.ma.us +lib.md.us +lib.me.us +lib.mi.us +lib.mn.us +lib.mo.us +lib.ms.us +lib.mt.us +lib.nc.us +lib.nd.us +lib.ne.us +lib.nh.us +lib.nj.us +lib.nm.us +lib.nv.us +lib.ny.us +lib.oh.us +lib.ok.us +lib.or.us +lib.pa.us +lib.pr.us +lib.ri.us +lib.sc.us +lib.sd.us +lib.tn.us +lib.tx.us +lib.ut.us +lib.vi.us +lib.vt.us +lib.va.us +lib.wa.us +lib.wi.us +lib.wv.us +lib.wy.us +pvt.k12.ma.us +chtr.k12.ma.us +paroch.k12.ma.us +#uy +com.uy +edu.uy +gub.uy +mil.uy +net.uy +org.uy +#uz +co.uz +com.uz +net.uz +org.uz +#va +#vc +com.vc +net.vc +org.vc +gov.vc +mil.vc +edu.vc +#ve +co.ve +com.ve +e12.ve +edu.ve +gov.ve +info.ve +mil.ve +net.ve +org.ve +web.ve +#vg +#vi +co.vi +com.vi +k12.vi +net.vi +org.vi +#vn +com.vn +net.vn +org.vn +edu.vn +gov.vn +int.vn +ac.vn +biz.vn +info.vn +name.vn +pro.vn +health.vn +#vu +#wf +#ws +com.ws +net.ws +org.ws +gov.ws +edu.ws +#yt +#xxx +#ye +com.ye +co.ye +ltd.ye +me.ye +net.ye +org.ye +plc.ye +gov.ye +#za +ac.za +co.za +edu.za +gov.za +law.za +mil.za +nom.za +org.za +school.za +ecape.school.za +fs.school.za +gp.school.za +kzn.school.za +mpm.za +ncape.school.za +lp.school.za +nw.school.za +wcape.school.za +alt.za +net.work.za +ngo.za +tm.za +web.za +agric.za +grondar.za +inca.za +nis.za +#zm +ac.zm +co.zm +com.zm +edu.zm +gov.zm +net.zm +org.zm +sch.zm +#zw +co.zw +org.zw +gov.zw +ac.zw +cloudfront.net +compute.amazonaws.com +us-east-1.amazonaws.com +compute-1.amazonaws.com +z-1.compute-1.amazonaws.com +z-2.compute-1.amazonaws.com +ap-northeast-1.compute.amazonaws.com +ap-southeast-1.compute.amazonaws.com +ap-southeast-2.compute.amazonaws.com +eu-west-1.compute.amazonaws.com +sa-east-1.compute.amazonaws.com +us-gov-west-1.compute.amazonaws.com +us-west-1.compute.amazonaws.com +us-west-2.compute.amazonaws.com +elasticbeanstalk.com +elb.amazonaws.com +s3.amazonaws.com +s3-us-west-2.amazonaws.com +s3-us-west-1.amazonaws.com +s3-eu-west-1.amazonaws.com +s3-ap-southeast-1.amazonaws.com +s3-ap-southeast-2.amazonaws.com +s3-ap-northeast-1.amazonaws.com +s3-sa-east-1.amazonaws.com +s3-us-gov-west-1.amazonaws.com +s3-fips-us-gov-west-1.amazonaws.com +s3-website-us-east-1.amazonaws.com +s3-website-us-west-2.amazonaws.com +s3-website-us-west-1.amazonaws.com +s3-website-eu-west-1.amazonaws.com +s3-website-ap-southeast-1.amazonaws.com +s3-website-ap-southeast-2.amazonaws.com +s3-website-ap-northeast-1.amazonaws.com +s3-website-sa-east-1.amazonaws.com +s3-website-us-gov-west-1.amazonaws.com +betainabox.com +ae.org +ar.com +br.com +cn.com +com.de +de.com +eu.com +gb.com +gb.net +gr.com +hu.com +hu.net +jp.net +jpn.com +kr.com +no.com +qc.com +ru.com +sa.com +se.com +se.net +uk.com +uk.net +us.com +us.org +uy.com +za.com +c.la +cloudcontrolled.com +cloudcontrolapp.com +co.ca +co.nl +co.no +dreamhosters.com +dyndns-at-home.com +dyndns-at-work.com +dyndns-blog.com +dyndns-free.com +dyndns-home.com +dyndns-ip.com +dyndns-mail.com +dyndns-office.com +dyndns-pics.com +dyndns-remote.com +dyndns-server.com +dyndns-web.com +dyndns-wiki.com +dyndns-work.com +dyndns.biz +dyndns.info +dyndns.org +dyndns.tv +at-band-camp.net +ath.cx +barrel-of-knowledge.info +barrell-of-knowledge.info +better-than.tv +blogdns.com +blogdns.net +blogdns.org +blogsite.org +boldlygoingnowhere.org +broke-it.net +buyshouses.net +cechire.com +dnsalias.com +dnsalias.net +dnsalias.org +dnsdojo.com +dnsdojo.net +dnsdojo.org +does-it.net +doesntexist.com +doesntexist.org +dontexist.com +dontexist.net +dontexist.org +doomdns.com +doomdns.org +dvrdns.org +dyn-o-saur.com +dynalias.com +dynalias.net +dynalias.org +dynathome.net +dyndns.ws +endofinternet.net +endofinternet.org +endoftheinternet.org +est-a-la-maison.com +est-a-la-masion.com +est-le-patron.com +est-mon-blogueur.com +for-better.biz +for-more.biz +for-our.info +for-some.biz +for-the.biz +forgot.her.name +forgot.his.name +from-ak.com +from-al.com +from-ar.com +from-az.net +from-ca.com +from-co.net +from-ct.com +from-dc.com +from-de.com +from-fl.com +from-ga.com +from-hi.com +from-ia.com +from-id.com +from-il.com +from-in.com +from-ks.com +from-ky.com +from-la.net +from-ma.com +from-md.com +from-me.org +from-mi.com +from-mn.com +from-mo.com +from-ms.com +from-mt.com +from-nc.com +from-nd.com +from-ne.com +from-nh.com +from-nj.com +from-nm.com +from-nv.com +from-ny.net +from-oh.com +from-ok.com +from-or.com +from-pa.com +from-pr.com +from-ri.com +from-sc.com +from-sd.com +from-tn.com +from-tx.com +from-ut.com +from-va.com +from-vt.com +from-wa.com +from-wi.com +from-wv.com +from-wy.com +ftpaccess.cc +fuettertdasnetz.de +game-host.org +game-server.cc +getmyip.com +gets-it.net +go.dyndns.org +gotdns.com +gotdns.org +groks-the.info +groks-this.info +ham-radio-op.net +here-for-more.info +hobby-site.com +hobby-site.org +home.dyndns.org +homedns.org +homeftp.net +homeftp.org +homeip.net +homelinux.com +homelinux.net +homelinux.org +homeunix.com +homeunix.net +homeunix.org +iamallama.com +in-the-band.net +is-a-anarchist.com +is-a-blogger.com +is-a-bookkeeper.com +is-a-bruinsfan.org +is-a-bulls-fan.com +is-a-candidate.org +is-a-caterer.com +is-a-celticsfan.org +is-a-chef.com +is-a-chef.net +is-a-chef.org +is-a-conservative.com +is-a-cpa.com +is-a-cubicle-slave.com +is-a-democrat.com +is-a-designer.com +is-a-doctor.com +is-a-financialadvisor.com +is-a-geek.com +is-a-geek.net +is-a-geek.org +is-a-green.com +is-a-guru.com +is-a-hard-worker.com +is-a-hunter.com +is-a-knight.org +is-a-landscaper.com +is-a-lawyer.com +is-a-liberal.com +is-a-libertarian.com +is-a-linux-user.org +is-a-llama.com +is-a-musician.com +is-a-nascarfan.com +is-a-nurse.com +is-a-painter.com +is-a-patsfan.org +is-a-personaltrainer.com +is-a-photographer.com +is-a-player.com +is-a-republican.com +is-a-rockstar.com +is-a-socialist.com +is-a-soxfan.org +is-a-student.com +is-a-teacher.com +is-a-techie.com +is-a-therapist.com +is-an-accountant.com +is-an-actor.com +is-an-actress.com +is-an-anarchist.com +is-an-artist.com +is-an-engineer.com +is-an-entertainer.com +is-by.us +is-certified.com +is-found.org +is-gone.com +is-into-anime.com +is-into-cars.com +is-into-cartoons.com +is-into-games.com +is-leet.com +is-lost.org +is-not-certified.com +is-saved.org +is-slick.com +is-uberleet.com +is-very-bad.org +is-very-evil.org +is-very-good.org +is-very-nice.org +is-very-sweet.org +is-with-theband.com +isa-geek.com +isa-geek.net +isa-geek.org +isa-hockeynut.com +issmarterthanyou.com +isteingeek.de +istmein.de +kicks-ass.net +kicks-ass.org +knowsitall.info +land-4-sale.us +lebtimnetz.de +leitungsen.de +likes-pie.com +likescandy.com +merseine.nu +mine.nu +misconfused.org +mypets.ws +myphotos.cc +neat-url.com +office-on-the.net +on-the-web.tv +podzone.net +podzone.org +readmyblog.org +saves-the-whales.com +scrapper-site.net +scrapping.cc +selfip.biz +selfip.com +selfip.info +selfip.net +selfip.org +sells-for-less.com +sells-for-u.com +sells-it.net +sellsyourhome.org +servebbs.com +servebbs.net +servebbs.org +serveftp.net +serveftp.org +servegame.org +shacknet.nu +simple-url.com +space-to-rent.com +stuff-4-sale.org +stuff-4-sale.us +teaches-yoga.com +thruhere.net +traeumtgerade.de +webhop.biz +webhop.info +webhop.net +webhop.org +worse-than.tv +writesthisblog.com +a.ssl.fastly.net +b.ssl.fastly.net +global.ssl.fastly.net +a.prod.fastly.net +global.prod.fastly.net +github.io +ro.com +appspot.com +blogspot.be +blogspot.bj +blogspot.ca +blogspot.cf +blogspot.ch +blogspot.co.at +blogspot.co.il +blogspot.co.nz +blogspot.co.uk +blogspot.com +blogspot.com.ar +blogspot.com.au +blogspot.com.br +blogspot.com.es +blogspot.cv +blogspot.cz +blogspot.de +blogspot.dk +blogspot.fi +blogspot.fr +blogspot.gr +blogspot.hk +blogspot.hu +blogspot.ie +blogspot.in +blogspot.it +blogspot.jp +blogspot.kr +blogspot.mr +blogspot.mx +blogspot.nl +blogspot.no +blogspot.pt +blogspot.re +blogspot.ro +blogspot.se +blogspot.sg +blogspot.sk +blogspot.td +blogspot.tw +codespot.com +googleapis.com +googlecode.com +herokuapp.com +herokussl.com +iki.fi +biz.at +info.at +co.pl +nyc.mn +operaunite.com +rhcloud.com +priv.at +za.net +za.org + diff --git a/conf/lua/hfilter.lua b/conf/lua/hfilter.lua new file mode 100644 index 000000000..b68061143 --- /dev/null +++ b/conf/lua/hfilter.lua @@ -0,0 +1,310 @@ +-- +-- Copyright (c) 2013, Alexey Savelyev +-- E-mail: info@homeweb.ru +-- WWW: http://homeweb.ru +-- + +--Rating for checks_hellohost and checks_hello: 5 - very hard, 4 - hard, 3 - meduim, 2 - low, 1 - very low +local checks_hellohost = { +['[.-]dynamic[.-]'] = 5, ['dynamic[.-][0-9]'] = 5, ['[0-9][.-]?dynamic'] = 5, +['[.-]dyn[.-]'] = 5, ['dyn[.-][0-9]'] = 5, ['[0-9][.-]?dyn'] = 5, +['[.-]clients?[.-]'] = 5, ['clients?[.-][0-9]'] = 5, ['[0-9][.-]?clients?'] = 5, +['[.-]dynip[.-]'] = 5, ['dynip[.-][0-9]'] = 5, ['[0-9][.-]?dynip'] = 5, +['[.-]broadband[.-]'] = 5, ['broadband[.-][0-9]'] = 5, ['[0-9][.-]?broadband'] = 5, +['[.-]broad[.-]'] = 5, ['broad[.-][0-9]'] = 5, ['[0-9][.-]?broad'] = 5, +['[.-]bredband[.-]'] = 5, ['bredband[.-][0-9]'] = 5, ['[0-9][.-]?bredband'] = 5, +['[.-]nat[.-]'] = 5, ['nat[.-][0-9]'] = 5, ['[0-9][.-]?nat'] = 5, +['[.-]pptp[.-]'] = 5, ['pptp[.-][0-9]'] = 5, ['[0-9][.-]?pptp'] = 5, +['[.-]pppoe[.-]'] = 5, ['pppoe[.-][0-9]'] = 5, ['[0-9][.-]?pppoe'] = 5, +['[.-]ppp[.-]'] = 5, ['ppp[.-][0-9]'] = 5, ['[0-9][.-]?ppp'] = 5, +['[.-][a|x]?dsl[.-]'] = 4, ['[a|x]?dsl[.-]?[0-9]'] = 4, ['[0-9][.-]?[a|x]?dsl'] = 4, +['[.-][a|x]?dsl-dynamic[.-]'] = 5, ['[a|x]?dsl-dynamic[.-]?[0-9]'] = 5, ['[0-9][.-]?[a|x]?dsl-dynamic'] = 5, +['[.-][a|x]?dsl-line[.-]'] = 4, ['[a|x]?dsl-line[.-]?[0-9]'] = 4, ['[0-9][.-]?[a|x]?dsl-line'] = 4, +['[.-]dhcp[.-]'] = 5, ['dhcp[.-][0-9]'] = 5, ['[0-9][.-]?dhcp'] = 5, +['[.-]catv[.-]'] = 5, ['catv[.-][0-9]'] = 5, ['[0-9][.-]?catv'] = 5, +['[.-]wifi[.-]'] = 5, ['wifi[.-][0-9]'] = 5, ['[0-9][.-]?wifi'] = 5, +['[.-]unused-addr[.-]'] = 3, ['unused-addr[.-][0-9]'] = 3, ['[0-9][.-]?unused-addr'] = 3, +['[.-]dial-?up[.-]'] = 5, ['dial-?up[.-][0-9]'] = 5, ['[0-9][.-]?dial-?up'] = 5, +['[.-]gprs[.-]'] = 5, ['gprs[.-][0-9]'] = 5, ['[0-9][.-]?gprs'] = 5, +['[.-]cdma[.-]'] = 5, ['cdma[.-][0-9]'] = 5, ['[0-9][.-]?cdma'] = 5, +['[.-]homeuser[.-]'] = 5, ['homeuser[.-][0-9]'] = 5, ['[0-9][.-]?homeuser'] = 5, +['[.-]in-?addr[.-]'] = 4, ['in-?addr[.-][0-9]'] = 4, ['[0-9][.-]?in-?addr'] = 4, +['[.-]pool[.-]'] = 4, ['pool[.-][0-9]'] = 4, ['[0-9][.-]?pool'] = 4, +['[.-]cable[.-]'] = 3, ['cable[.-][0-9]'] = 3, ['[0-9][.-]?cable'] = 3, +['[.-]host[.-]'] = 2, ['host[.-][0-9]'] = 2, ['[0-9][.-]?host'] = 2, +['[.-]customers[.-]'] = 1, ['customers[.-][0-9]'] = 1, ['[0-9][.-]?customers'] = 1 +} + +local checks_hello = { +['localhost$'] = 5, ['\\.hfilter\\.ru'] = 5, ['^\\[*84\\.47\\.176\\.(70|71)'] = 5, ['^\\[*81\\.26\\.148\\.(66|67|68|69|70|71|72|73|74|75|76|77|79)'] = 5, +['^(dsl)?(device|speedtouch)\\.lan$'] = 5, +['\\.(lan|local|home|localdomain|intra|in-addr.arpa|priv|online|user|veloxzon)$'] = 5, +['^\\[*127\\.'] = 5, ['^\\[*10\\.'] = 5, ['^\\[*172\\.16\\.'] = 5, ['^\\[*192\\.168\\.'] = 5, +--bareip +['^\\[*\\d+[x.-]\\d+[x.-]\\d+[x.-]\\d+\\]*$'] = 4 +} + +local function trim1(s) + return (s:gsub("^%s*(.-)%s*$", "%1")) +end + +local function check_regexp(str, regexp_text) + local re = regexp.get_cached(regexp_text) + if not re then re = regexp.create(regexp_text, 'i') end + if re:match(str) then return true end +return false +end + +local function split(str, delim, maxNb) + -- Eliminate bad cases... + if string.find(str, delim) == nil then + return { str } + end + if maxNb == nil or maxNb < 1 then + maxNb = 0 -- No limit + end + local result = {} + local pat = "(.-)" .. delim .. "()" + local nb = 0 + local lastPos + for part, pos in string.gmatch(str, pat) do + nb = nb + 1 + result[nb] = part + lastPos = pos + if nb == maxNb then break end + end + -- Handle the last field + if nb ~= maxNb then + result[nb + 1] = string.sub(str, lastPos) + end + return result +end + +local function check_fqdn(domain) + if check_regexp(domain, '(?=^.{4,255}$)(^((?!-)[a-zA-Z0-9-]{1,63}(?<!-)\\.)+[a-zA-Z]{2,63}$)') then + return true + end +return false +end + +-- host: host for check +-- symbol_suffix: suffix for symbol +-- eq_ip: ip for comparing or empty string +-- eq_host: host for comparing or empty string +local function check_host(task, host, symbol_suffix, eq_ip, eq_host) + + local function check_host_cb_mx_a(resolver, to_resolve, results, err) + task:inc_dns_req() + if not results then + task:insert_result('HFILTER_' .. symbol_suffix .. '_NORESOLVE_MX', 1.0) + end + end + local function check_host_cb_mx(resolver, to_resolve, results, err) + task:inc_dns_req() + if not results then + task:insert_result('HFILTER_' .. symbol_suffix .. '_NORES_A_OR_MX', 1.0) + else + for _,mx in pairs(results) do + if mx['name'] then + task:get_resolver():resolve_a(task:get_session(), task:get_mempool(), mx['name'], check_host_cb_mx_a) + end + end + end + end + local function check_host_cb_a(resolver, to_resolve, results, err) + task:inc_dns_req() + + if not results then + task:get_resolver():resolve_mx(task:get_session(), task:get_mempool(), host, check_host_cb_mx) + elseif eq_ip ~= '' then + for _,result in pairs(results) do + if result:to_string() == eq_ip then + return true + end + end + task:insert_result('HFILTER_' .. symbol_suffix .. '_IP_A', 1.0) + end + end + + if host then + host = string.lower(host) + else + return false + end + if eq_host then + eq_host = string.lower(eq_host) + else + eq_host = '' + end + + if check_fqdn(host) then + if eq_host == '' or eq_host ~= host then + task:get_resolver():resolve_a(task:get_session(), task:get_mempool(), host, check_host_cb_a) + end + else + task:insert_result('HFILTER_' .. symbol_suffix .. '_NOT_FQDN', 1.0) + end + +return true +end + +-- +local function hfilter(task) + local recvh = task:get_received_headers() + + if table.maxn(recvh) == 0 then + return false + end + + --IP-- + local ip = false + local rip = task:get_from_ip() + if rip then + ip = rip:to_string() + if ip and ip == '0.0.0.0' then + ip = false + end + end + + --HOSTNAME-- + local r = recvh[1] + local hostname = false + local hostname_lower = false + if r['real_hostname'] and ( r['real_hostname'] ~= 'unknown' or not check_regexp(r['real_hostname'], '^\\d+\\.\\d+\\.\\d+\\.\\d+$') ) then + hostname = r['real_hostname'] + hostname_lower = string.lower(hostname) + end + + --HELO-- + local helo = task:get_helo() + local helo_lower = false + if helo then + helo_lower = string.lower(helo) + else + helo = false + helo_lower = false + end + + --MESSAGE-- + local message = task:get_message() + + --RULES--RULES--RULES-- + + -- Check's HELO + local checks_hello_found = false + if helo then + -- Regexp check HELO + for regexp,weight in pairs(checks_hello) do + if check_regexp(helo_lower, regexp) then + task:insert_result('HFILTER_HELO_' .. weight, 1.0) + checks_hello_found = true + break + end + end + if not checks_hello_found then + for regexp,weight in pairs(checks_hellohost) do + if check_regexp(helo_lower, regexp) then + task:insert_result('HFILTER_HELO_' .. weight, 1.0) + checks_hello_found = true + break + end + end + end + + --FQDN check HELO + if ip then + check_host(task, helo, 'HELO', ip, hostname) + end + end + + -- + local function check_hostname(hostname_res) + -- Check regexp HOSTNAME + for regexp,weight in pairs(checks_hellohost) do + if check_regexp(hostname_res, regexp) then + task:insert_result('HFILTER_HOSTNAME_' .. weight, 1.0) + break + end + end + end + local function hfilter_hostname_ptr(resolver, to_resolve, results, err) + task:inc_dns_req() + if results then + check_hostname(results[1]) + end + end + + -- Check's HOSTNAME + if hostname then + if not checks_hello_found then + check_hostname(hostname) + end + else + task:insert_result('HFILTER_HOSTNAME_NOPTR', 1.00) + if ip and not checks_hello_found then + task:get_resolver():resolve_ptr(task:get_session(), task:get_mempool(), ip, hfilter_hostname_ptr) + end + end + + -- MAILFROM checks -- + local from = task:get_from() + if from then + --FROM host check + for _,fr in ipairs(from) do + local fr_split = split(fr['addr'], '@', 0) + if table.maxn(fr_split) == 2 then + check_host(task, fr_split[2], 'FROMHOST', '', '') + end + end + end + + --Message ID host check + local message_id = task:get_message_id() + if message_id then + local mid_split = split(message_id, '@', 0) + if table.maxn(mid_split) == 2 and not string.find(mid_split[2], "local") then + if not check_fqdn(mid_split[2]) then + task:insert_result('HFILTER_MID_NOT_FQDN', 1.00) + end + end + end + + -- Links checks + local parts = task:get_text_parts() + if parts then + --One text part-- + if table.maxn(parts) > 0 and parts[1]:get_content() then + local part_text = trim1(parts[1]:get_content()) + local total_part_len = string.len(part_text) + if total_part_len > 0 then + local urls = task:get_urls() + if urls then + local total_url_len = 0 + for _,url in ipairs(urls) do + total_url_len = total_url_len + string.len(url:get_text()) + end + if total_url_len > 0 then + if total_url_len + 7 > total_part_len then + task:insert_result('HFILTER_URL_ONLY', 1.00) + else + if not string.find(part_text, "\n") then + task:insert_result('HFILTER_URL_ONELINE', 1.00) + end + end + end + end + end + end + end + + return false +end + +rspamd_config:register_symbols(hfilter, 1.0, +"HFILTER_HELO_1", "HFILTER_HELO_2", "HFILTER_HELO_3", "HFILTER_HELO_4", "HFILTER_HELO_5", +"HFILTER_HOSTNAME_1", "HFILTER_HOSTNAME_2", "HFILTER_HOSTNAME_3", "HFILTER_HOSTNAME_4", "HFILTER_HOSTNAME_5", +"HFILTER_HELO_NORESOLVE_MX", "HFILTER_HELO_NORES_A_OR_MX", "HFILTER_HELO_IP_A", "HFILTER_HELO_NOT_FQDN", +"HFILTER_FROMHOST_NORESOLVE_MX", "HFILTER_FROMHOST_NORES_A_OR_MX", "HFILTER_FROMHOST_NOT_FQDN", +"HFILTER_MID_NOT_FQDN", +"HFILTER_HOSTNAME_NOPTR", +"HFILTER_URL_ONLY", "HFILTER_URL_ONELINE"); diff --git a/conf/lua/rspamd.lua b/conf/lua/rspamd.lua index 822f6447f..e8b585837 100644 --- a/conf/lua/rspamd.lua +++ b/conf/lua/rspamd.lua @@ -68,7 +68,6 @@ reconf['DATE_IN_PAST'] = function(task) return false end - local function file_exists(filename) local file = io.open(filename) if file then @@ -79,6 +78,10 @@ local function file_exists(filename) end end +if file_exists('hfilter.lua') then + dofile('hfilter.lua') +end + if file_exists('rspamd.local.lua') then dofile('rspamd.local.lua') end diff --git a/conf/metrics.conf b/conf/metrics.conf index 727bea6da..13b9796af 100644 --- a/conf/metrics.conf +++ b/conf/metrics.conf @@ -18,7 +18,7 @@ metric { name = "FORGED_OUTLOOK_TAGS"; } symbol { - weight = 5.0; + weight = 0.30; description = "Sender is forged (different From: header and smtp MAIL FROM: addresses)"; name = "FORGED_SENDER"; } @@ -288,7 +288,7 @@ metric { name = "MISSING_MID"; } symbol { - weight = 3.0; + weight = 1.0; description = "Recipients are not the same as RCPT TO: mail command"; name = "FORGED_RECIPIENTS"; } @@ -322,20 +322,71 @@ metric { description = "One received header with 'bad' patterns inside"; name = "ONCE_RECEIVED_STRICT"; } + symbol { name = "RBL_SPAMHAUS"; weight = 0.0; description = "From address is listed in zen"; } + symbol { name = "RBL_SPAMHAUS_SBL"; weight = 2.0; description = "From address is listed in zen sbl"; } + symbol { name = "RBL_SPAMHAUS_CSS"; weight = 2.0; description = "From address is listed in zen css"; } + symbol { name = "RBL_SPAMHAUS_XBL"; weight = 4.0; description = "From address is listed in zen xbl"; } + symbol { name = "RBL_SPAMHAUS_PBL"; weight = 2.0; description = "From address is listed in zen pbl"; } + symbol { name = "RECEIVED_SPAMHAUS_XBL"; weight = 3.0; description = "Received address is listed in zen pbl"; } + symbol { + weight = 2.0; + description = "From address is listed in senderscore.com BL"; + name = "RBL_SENDERSCORE"; + } + symbol { + weight = 2.0; + description = "From address is listed in mailspike.com BL"; + name = "RBL_MAILSPIKE"; + } symbol { weight = 1.0; - description = "Received headers contains addresses from zen spamhaus RBL"; - name = "RBL_ZEN"; + name = "RBL_SORBS"; + description = "From address is listed in SORBS RBL"; + } + symbol { + weight = 2.5; + name = "RBL_SORBS_HTTP"; + description = "List of Open HTTP Proxy Servers."; + } + symbol { + weight = 2.5; + name = "RBL_SORBS_SOCKS"; + description = "List of Open SOCKS Proxy Servers."; } symbol { weight = 1.0; - description = "Received headers contains addresses from senderscore.com RBL"; - name = "RBL_SENDERSCORE"; + name = "RBL_SORBS_MISC"; + description = "List of open Proxy Servers not listed in the SOCKS or HTTP lists."; + } + symbol { + weight = 3.0; + name = "RBL_SORBS_SMTP"; + description = "List of Open SMTP relay servers."; + } + symbol { + weight = 1.5; + name = "RBL_SORBS_RECENT"; + description = "List of hosts that have been noted as sending spam/UCE/UBE to the admins of SORBS within the last 28 days (includes new.spam.dnsbl.sorbs.net)."; + } + symbol { + weight = 0.4; + name = "RBL_SORBS_WEB"; + description = "List of web (WWW) servers which have spammer abusable vulnerabilities (e.g. FormMail scripts)"; + } + symbol { + weight = 2.0; + name = "RBL_SORBS_DUL"; + description = "Dynamic IP Address ranges (NOT a Dial Up list!)"; } symbol { weight = 1.0; - description = "Received headers contains addresses from mailspike.com RBL"; - name = "RBL_MAILSPIKE"; + name = "RBL_SORBS_BLOCK"; + description = "List of hosts demanding that they never be tested by SORBS."; + } + symbol { + weight = 1.0; + name = "RBL_SORBS_ZOMBIE"; + description = "List of networks hijacked from their original owners, some of which have already used for spamming."; } symbol { weight = 3.0; @@ -408,22 +459,22 @@ metric { name = "BAYES_HAM"; } symbol { - weight = 1.0; + weight = 5.0; description = "Generic fuzzy hash match"; - name = "R_FUZZY"; + name = "FUZZY_UNKNOWN"; } symbol { - weight = 1.0; + weight = 10.0; description = "Denied fuzzy hash"; name = "FUZZY_DENIED"; } symbol { - weight = 1.0; + weight = 5.0; description = "Probable fuzzy hash"; name = "FUZZY_PROB"; } symbol { - weight = 1.0; + weight = -2.1; description = "Whitelisted fuzzy hash"; name = "FUZZY_WHITE"; } @@ -493,11 +544,16 @@ metric { name = "WS_SURBL_MULTI"; } symbol { - weight = 9.500000; + weight = 4.500000; description = "rambler.ru uribl"; name = "RAMBLER_URIBL"; } symbol { + weight = 5.500000; + description = "DBL uribl"; + name = "DBL"; + } + symbol { weight = 7.5; description = "uribl.com black url"; name = "URIBL_BLACK"; @@ -637,4 +693,26 @@ metric { description = "Message date is in past"; name = "DATE_IN_PAST"; } +# hfilter symbols + symbol { weight = 1.00; name = "HFILTER_HELO_1"; description = "Helo host checks (very low)"; } + symbol { weight = 2.00; name = "HFILTER_HELO_2"; description = "Helo host checks (low)"; } + symbol { weight = 3.00; name = "HFILTER_HELO_3"; description = "Helo host checks (medium)"; } + symbol { weight = 3.50; name = "HFILTER_HELO_4"; description = "Helo host checks (hard)"; } + symbol { weight = 4.00; name = "HFILTER_HELO_5"; description = "Helo host checks (very hard)"; } + symbol { weight = 1.00; name = "HFILTER_HOSTNAME_1"; description = "Hostname checks (very low)"; } + symbol { weight = 2.00; name = "HFILTER_HOSTNAME_2"; description = "Hostname checks (low)"; } + symbol { weight = 3.00; name = "HFILTER_HOSTNAME_3"; description = "Hostname checks (medium)"; } + symbol { weight = 3.50; name = "HFILTER_HOSTNAME_4"; description = "Hostname checks (hard)"; } + symbol { weight = 4.00; name = "HFILTER_HOSTNAME_5"; description = "Hostname checks (very hard)"; } + symbol { weight = 1.50; name = "HFILTER_HELO_NORESOLVE_MX"; description = "MX found in Helo and no resolve"; } + symbol { weight = 2.00; name = "HFILTER_HELO_NORES_A_OR_MX"; description = "Helo no resolve to A or MX"; } + symbol { weight = 2.50; name = "HFILTER_HELO_IP_A"; description = "Helo A IP != hostname IP"; } + symbol { weight = 3.00; name = "HFILTER_HELO_NOT_FQDN"; description = "Helo not FQDN"; } + symbol { weight = 1.50; name = "HFILTER_FROMHOST_NORESOLVE_MX"; description = "MX found in FROM host and no resolve"; } + symbol { weight = 3.00; name = "HFILTER_FROMHOST_NORES_A_OR_MX"; description = "FROM host no resolve to A or MX"; } + symbol { weight = 4.00; name = "HFILTER_FROMHOST_NOT_FQDN"; description = "FROM host not FQDN"; } + symbol { weight = 0.50; name = "HFILTER_MID_NOT_FQDN"; description = "Message-id host not FQDN"; } + symbol { weight = 4.00; name = "HFILTER_HOSTNAME_NOPTR"; description = "No PTR for IP"; } + symbol { weight = 3.50; name = "HFILTER_URL_ONLY"; description = "URL only in body"; } + symbol { weight = 2.00; name = "HFILTER_URL_ONELINE"; description = "One line URL and text in body"; } } diff --git a/conf/modules.conf b/conf/modules.conf index 2b7d5b8b8..41296b7bf 100644 --- a/conf/modules.conf +++ b/conf/modules.conf @@ -1,24 +1,28 @@ # Rspamd modules configuration fuzzy_check { - servers = "highsecure.ru:11335"; - symbol = "R_FUZZY"; min_bytes = 300; - max_score = 10; - mime_types = "application/pdf"; - fuzzy_map = { - FUZZY_DENIED { - weight = 10.0; - flag = 1 - } - FUZZY_PROB { - weight = 5.0; - flag = 2 - } - FUZZY_WHITE { - weight = -2.1; - flag = 3 - } - } + rule { + servers = "highsecure.ru:11335"; + symbol = "FUZZY_UNKNOWN"; + mime_types = "application/pdf"; + max_score = 20.0; + read_only = yes; + skip_unknown = yes; + fuzzy_map = { + FUZZY_DENIED { + max_score = 20.0; + flag = 1 + } + FUZZY_PROB { + max_score = 10.0; + flag = 2 + } + FUZZY_WHITE { + max_score = 2.0; + flag = 3 + } + } + } } forged_recipients { symbol_sender = "FORGED_SENDER"; @@ -67,28 +71,34 @@ rbl { default_from = true; rbls { - spamhaus_zen { - symbol = "RBL_ZEN"; - rbl = "zen.spamhaus.org"; - ipv4 = true; - ipv6 = true; - } - spamhaus_pbl { - symbol = "RECEIVED_PBL"; - rbl = "pbl.spamhaus.org"; - ipv4 = true; - ipv6 = true; - received = true; - from = false; - } - spamhaus_pbl { - symbol = "RECEIVED_XBL"; + + spamhaus { + symbol = "RBL_SPAMHAUS"; + rbl = "zen.spamhaus.org"; + ipv4 = true; + ipv6 = true; + unknown = false; + returncodes { + RBL_SPAMHAUS_SBL = "127.0.0.2"; + RBL_SPAMHAUS_CSS = "127.0.0.3"; + RBL_SPAMHAUS_XBL = "127.0.0.4"; + RBL_SPAMHAUS_XBL = "127.0.0.5"; + RBL_SPAMHAUS_XBL = "127.0.0.6"; + RBL_SPAMHAUS_XBL = "127.0.0.7"; + RBL_SPAMHAUS_PBL = "127.0.0.10"; + RBL_SPAMHAUS_PBL = "127.0.0.11"; + } + } + + spamhaus_xbl { + symbol = "RECEIVED_SPAMHAUS_XBL"; rbl = "xbl.spamhaus.org"; ipv4 = true; ipv6 = true; received = true; from = false; } + mailspike { symbol = "RBL_MAILSPIKE"; rbl = "bl.mailspike.net"; @@ -97,6 +107,22 @@ rbl { symbol = "RBL_SENDERSCORE"; rbl = "bl.score.senderscore.com"; } + sorbs { + symbol = "RBL_SORBS"; + rbl = "dnsbl.sorbs.net"; + returncodes { + #http://www.sorbs.net/general/using.shtml + RBL_SORBS_HTTP = "127.0.0.2" + RBL_SORBS_SOCKS = "127.0.0.3" + RBL_SORBS_MISC = "127.0.0.4" + RBL_SORBS_SMTP = "127.0.0.5" + RBL_SORBS_RECENT = "127.0.0.6" + RBL_SORBS_WEB = "127.0.0.7" + RBL_SORBS_DUL = "127.0.0.10" + RBL_SORBS_BLOCK = "127.0.0.8" + RBL_SORBS_ZOMBIE = "127.0.0.9" + } + } } } diff --git a/conf/options.conf b/conf/options.conf index 841372ec2..1dd1ebfb8 100644 --- a/conf/options.conf +++ b/conf/options.conf @@ -6,6 +6,7 @@ options { raw_mode = false; one_shot = false; dns_timeout = 1s; + dns_sockets = 16; dns_retransmits = 5; cache_file = "$DBDIR/symbols.cache"; map_watch_interval = 1min; diff --git a/config.h.in b/config.h.in index c7e44d846..e387d5d1c 100644 --- a/config.h.in +++ b/config.h.in @@ -145,6 +145,7 @@ #cmakedefine HAVE_FLOCK 1 #cmakedefine HAVE_TANHL 1 +#cmakedefine HAVE_TANH 1 #cmakedefine HAVE_EXPL 1 #cmakedefine HAVE_EXP2L 1 diff --git a/contrib/lgpl/CMakeLists.txt b/contrib/lgpl/CMakeLists.txt index 56cedff89..39191620a 100644 --- a/contrib/lgpl/CMakeLists.txt +++ b/contrib/lgpl/CMakeLists.txt @@ -12,9 +12,7 @@ IF(GLIB2_VERSION VERSION_LESS "2.16") ADD_LIBRARY(glibadditions SHARED ${LIBGLIBSRC}) SET_TARGET_PROPERTIES(glibadditions PROPERTIES COMPILE_FLAGS "-I${CMAKE_SOURCE_DIR}/contrib/lgpl") - TARGET_LINK_LIBRARIES(glibadditions ${CMAKE_REQUIRED_LIBRARIES}) - TARGET_LINK_LIBRARIES(glibadditions ${GLIB2_LIBRARIES}) - TARGET_LINK_LIBRARIES(glibadditions pcre) + TARGET_LINK_LIBRARIES(glibadditions ${RSPAMD_REQUIRED_LIBRARIES}) SET_TARGET_PROPERTIES(glibadditions PROPERTIES VERSION ${RSPAMD_VERSION}) diff --git a/debian/changelog b/debian/changelog index 60f8eddfc..3d34bc28c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,33 @@ +rspamd (0.6.6-1) unstable; urgency=low + + * Upgrade to 0.6.6 + + -- Vsevolod Stakhov <vsevolod@highsecure.ru> Fri, 27 Dec 2013 14:55:00 +0000 + +rspamd (0.6.5-1) unstable; urgency=low + + * Upgrade to 0.6.5 + + -- Vsevolod Stakhov <vsevolod@highsecure.ru> Wed, 20 Dec 2013 15:50:00 +0000 + +rspamd (0.6.4-1) unstable; urgency=low + + * Upgrade to 0.6.4 + + -- Vsevolod Stakhov <vsevolod@highsecure.ru> Wed, 18 Dec 2013 15:24:00 +0000 + +rspamd (0.6.3-1) unstable; urgency=low + + * Upgrade to 0.6.3 + + -- Vsevolod Stakhov <vsevolod@highsecure.ru> Tue, 10 Dec 2013 19:17:00 +0000 + +rspamd (0.6.2-1) unstable; urgency=low + + * Upgrade to 0.6.2 + + -- Vsevolod Stakhov <vsevolod@highsecure.ru> Wed, 4 Dec 2013 16:43:00 +0000 + rspamd (0.6.1-1) unstable; urgency=low * Upgrade to 0.6.1 diff --git a/debian/control b/debian/control index 9b67feece..0611624c4 100644 --- a/debian/control +++ b/debian/control @@ -1,9 +1,9 @@ Source: rspamd Section: mail -Priority: extra +Priority: optional Maintainer: Vsevolod Stakhov <vsevolod@highsecure.ru> -Build-Depends: debhelper (>= 7.0.50~), dpkg-dev (>= 1.16.1~), cmake, libevent-dev (>= 1.3), libglib2.0-dev (>= 2.16.0), libgmime-2.6-dev, liblua5.2-dev, libpcre3-dev, cdbs, libssl-dev (>= 1.0), libjudy-dev, libcurl4-openssl-dev -Standards-Version: 3.9.3 +Build-Depends: debhelper (>= 7.0.50~), dpkg-dev (>= 1.16.1~), cmake, libevent-dev (>= 1.3), libglib2.0-dev (>= 2.16.0), libgmime-2.6-dev, liblua5.2-dev | liblua5.1-dev | liblua5.1-0-dev, libpcre3-dev, cdbs, libssl-dev (>= 1.0), libcurl4-openssl-dev +Standards-Version: 3.9.5 Homepage: https://bitbucket.org/vstakhov/rspamd/ Vcs-Hg: https://bitbucket.org/vstakhov/rspamd/ Vcs-Browser: https://bitbucket.org/vstakhov/rspamd/src diff --git a/debian/postrm b/debian/postrm index 4af96e131..14735e0a2 100644 --- a/debian/postrm +++ b/debian/postrm @@ -1,24 +1,9 @@ #!/bin/sh # postrm script for rspamd -# -# see: dh_installdeb(1) +#DEBHELPER# set -e -# summary of how this script can be called: -# * <postrm> `remove' -# * <postrm> `purge' -# * <old-postrm> `upgrade' <new-version> -# * <new-postrm> `failed-upgrade' <old-version> -# * <new-postrm> `abort-install' -# * <new-postrm> `abort-install' <old-version> -# * <new-postrm> `abort-upgrade' <old-version> -# * <disappearer's-postrm> `disappear' <overwriter> -# <overwriter-version> -# for details, see http://www.debian.org/doc/debian-policy/ or -# the debian-policy package - - case "$1" in purge|remove|abort-install|disappear) # find first and last SYSTEM_UID numbers @@ -81,11 +66,6 @@ case "$1" in ;; esac -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. - -#DEBHELPER# - exit 0 diff --git a/debian/preinst b/debian/preinst index 9127115e1..3588f2f5b 100644 --- a/debian/preinst +++ b/debian/preinst @@ -1,19 +1,9 @@ #!/bin/sh -# preinst script for rmilter -# -# see: dh_installdeb(1) +# preinst script for rspamd +#DEBHELPER# set -e -# summary of how this script can be called: -# * <new-preinst> `install' -# * <new-preinst> `install' <old-version> -# * <new-preinst> `upgrade' <old-version> -# * <old-preinst> `abort-upgrade' <new-version> -# for details, see http://www.debian.org/doc/debian-policy/ or -# the debian-policy package - - case "$1" in install) SERVER_HOME=/var/lib/rspamd @@ -77,9 +67,4 @@ case "$1" in ;; esac -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. - -#DEBHELPER# - exit 0 diff --git a/debian/rules b/debian/rules index 2a50cc64b..ce09d5128 100755 --- a/debian/rules +++ b/debian/rules @@ -2,20 +2,25 @@ #export DEB_BUILD_MAINT_OPTIONS=hardening=+all +export DEB_LDFLAGS_MAINT_APPEND=-Wl,--as-needed + include /usr/share/dpkg/buildflags.mk include /usr/share/cdbs/1/rules/debhelper.mk include /usr/share/cdbs/1/class/cmake.mk -DEB_CMAKE_NORMAL_ARGS+= -DCONFDIR=/etc/rspamd \ +DEB_CMAKE_NORMAL_ARGS+= -DCONFDIR=/etc/rspamd \ -DMANDIR=/usr/share/man \ -DRUNDIR=/var/lib/rspamd \ -DDBDIR=/var/lib/rspamd \ -DLOGDIR=/var/log/rspamd \ -DPLUGINSDIR=/usr/share/rspamd \ + -DEXAMPLESDIR=/usr/share/examples/rspamd \ -DLIBDIR=/usr/lib \ -DINCLUDEDIR=/usr/include \ -DNO_SHARED=ON \ -DDEBIAN_BUILD=1 \ + -DINSTALL_EXAMPLES=ON \ + -DFORCE_GMIME24=ON \ -DRSPAMD_GROUP=rspamd \ -DRSPAMD_USER=rspamd clean:: diff --git a/debian/source.lintian-overrides b/debian/source.lintian-overrides deleted file mode 100644 index 56d0deeaa..000000000 --- a/debian/source.lintian-overrides +++ /dev/null @@ -1,3 +0,0 @@ -# Actually rspamd does not contain any gpl code except of these package, -# so ignore unsafe link with openssl -rspamd: possible-gpl-code-linked-with-openssl diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 000000000..bcc5b3a9f --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,12 @@ +# A simple makefile to generate documentation from .md using pandoc + +PANDOC ?= pandoc + +all: man + +man: rspamd.8 rspamc.1 + +rspamd.8: rspamd.8.md + $(PANDOC) -s -f markdown -t man -o rspamd.8 rspamd.8.md +rspamc.1: rspamc.1.md + $(PANDOC) -s -f markdown -t man -o rspamc.1 rspamc.1.md diff --git a/doc/makeman.sh b/doc/makeman.sh deleted file mode 100755 index 176f37b25..000000000 --- a/doc/makeman.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh -# Makes manual pages from pods - -POD2MAN="pod2man" -VERSION="unknown" -CMAKEFILE="../CMakeLists.txt" - -# Extract release version -if [ -f ${CMAKEFILE} ] ; then - _mjver=`fgrep 'SET(RSPAMD_VERSION_MAJOR' ${CMAKEFILE} | sed -e 's/^.*RSPAMD_VERSION_MAJOR \([0-9]\).*/\1/'` - _miver=`fgrep 'SET(RSPAMD_VERSION_MINOR' ${CMAKEFILE} | sed -e 's/^.*RSPAMD_VERSION_MINOR \([0-9]\).*/\1/'` - _pver=`fgrep 'SET(RSPAMD_VERSION_PATCH' ${CMAKEFILE} | sed -e 's/^.*RSPAMD_VERSION_PATCH \([0-9]\).*/\1/'` - VERSION="${_mjver}.${_miver}.${_pver}" -fi - -pod2man -c "Rspamd documentation" -n rspamd -s 8 -r "rspamd-${VERSION}" < rspamd.pod > rspamd.8 -pod2man -c "Rspamd documentation" -n rspamc -s 1 -r "rspamd-${VERSION}" < rspamc.pod > rspamc.1 diff --git a/doc/markdown/architecture/index.md b/doc/markdown/architecture/index.md new file mode 100644 index 000000000..33669f2d5 --- /dev/null +++ b/doc/markdown/architecture/index.md @@ -0,0 +1,117 @@ +# Rspamd architecture + +## Introduction + +Rspamd is a universal spam filtering system based on event-driven processing +model. It means that rspamd is intended not to block anywhere in the code. To +process messages rspamd uses a set of so called `rules`. Each `rule` is a symbolic +name associated with some message property. For example, we can define the following +rules: + +- SPF_ALLOW - means that a message is validated by SPF; +- BAYES_SPAM - means that a message is statistically considered as spam; +- FORGED_OUTLOOK_MID - message ID seems to be forged for Outlook MUA. + +Rules are defined by [modules](../modules/). So far, if there is a module that +performs SPF checks it may define several rules according to SPF policy: + +- SPF_ALLOW - a sender is allowed to send messages for this domain; +- SPF_DENY - a sender is denied by SPF policy; +- SPF_SOFTFAIL - there is no affinity defined by SPF policy. + +Rspamd supports two main types of modules: internal written in C and external +written in Lua. There is no real difference between these two types with the exception +that C modules are embeded all the time and can be enabled in `filters` attribute +in the `options` section of the config: + +~~~nginx +options { + filters = "regexp,surbl,spf,dkim,fuzzy_check,chartable,email"; + ... +} +~~~ + +## Metrics + +Rules in rspamd, defines merely a logic of checks, however it is required to +set up weights for each rule. Weight means `significance` in terms of rspamd. So +far, rules with greater absolute value of weight are considered as more important +than the recent rules. The weight of rules is defined in `metrics`. Each metric +is a set of grouped rules with specific weights. For example, we may define the +following weights for our SPF rules: + +- SPF_ALLOW: -1 +- SPF_DENY: 2 +- SPF_SOFTFAIL: 0.5 + +Positive weights means that this rule turns message to more spammy, while negative +means the opposite. + +### Rules scheduler + +To avoid unnecessary checks rspamd uses scheduler of rules for each message. So far, +if a message is considered as `definite spam` then further checks are not performed. +This scheduler is rather naive and it performs the following logic: + +- select negative rules *before* positive ones to prevent false positives; +- prefer rules with the following characteristics: + - frequent rules; + - rules with more weight; + - faster rules + +These optimizations can filter definite spam more quickly than a generic queue. + +## Actions + +Another important property of metrics is their actions set. This set defines recommended +actions for a message if it reach a certain score defined by all rules triggered. +Rspamd defines the following actions: + +- **No action**: a message is likely ham; +- **Greylist**: greylist message is it is not certainly ham; +- **Add header**: a message is likely spam, so add a specific header; +- **Rewrite subject**: a message is likely spam, so rewrite its subject; +- **Reject**: a message is very likely spam, so reject it completely + +These actions are just recommendations for MTA and are not to be strictly followed. +For all actions that are greater or equal than *greylist* it is recommended to +perform explicit greylisting. *Add header* and *rewrite subject* actions are very +close in semantics and are both considered as `probable spam`. `Reject` is a +strong rule that usually means that a message should be really rejected by MTA. +The triggering score for these actions should be specified according to their logic +priorities. If two actions have the same weight, the result is unspecified. + +## Rules weight + +The weights of rules is not necessarily constant. For example, for statistics rules +we have no certain confidence if a message is spam or not. We have some probability +instead. To allow fuzzy rules weight, rspamd supports `dynamic weights`. Generally, +it means that a rule may add a dynamic range from 0 to a defined weight in the metric. +So far if we define symbol `BAYES_SPAM` with weight 5.0, then this rule can add +a resulting symbol with weight from 0 to 5.0. To distribute values in the proper +way, rspamd usually uses some sort of Sigma function to provide fair distribution curve. +Nevertheless, the most of rspamd rules uses static weights with the exception of +fuzzy rules. + +## Statistic + +Rspamd uses statistic algorithms to precise the final score of a message. Currently, +the only algorithm defined is OSB-Bayes. You may find the concrete details of this +algorithm in the following [paper](http://osbf-lua.luaforge.net/papers/osbf-eddc.pdf). +Rspamd uses window size of 5 words in its classification. During classification procedure, +rspamd split a message to a set of tokens. + +Tokens are separated by punctiation or space characters. Short tokens (less than 3 symbols) are ignored. For each token rspamd +calculates two non-cryptographic hashes used subsequently as indices. All these tokens +are stored in memory-mapped files called `statistic files` (or `statfiles`). Each statfile +is a set of token chains, indexed by the first hash. A new token may be inserted to some +chain, and if this chain is full then rspamd tries to expire less significant tokens to +insert a new one. It is possible to obtain the current state of tokens by running + + rspamc stat` + +command that asks controller for free and used tokens in each statfile. +Please note that if a statfile is close to be completely filled then during subsequent +learning you will loose existing data. Therefore, it is recommended to increase size for +such statfiles. + diff --git a/doc/markdown/configuration/index.md b/doc/markdown/configuration/index.md new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/doc/markdown/configuration/index.md diff --git a/doc/markdown/index.md b/doc/markdown/index.md new file mode 100644 index 000000000..181dc956e --- /dev/null +++ b/doc/markdown/index.md @@ -0,0 +1,22 @@ +# Rspamd documentation project + +## Introduction +Rspamd is a fast and advanced spam filtering system. It is based on event-driven processing model +allowing to work with multiple messages simultaneously without blocking anywhere during messages +processing. Rspamd contains various modules shipped in the default distribution and permits to be +extended with the own custom modules and rules written in [Lua](http://lua.org) programming language. +Rspamd uses complex estimation system based on a set of rules, each of those rules has its own score and +the final score of a message is defined by a sum of rules' scores that were true for that message. This approach +is similar to other complex spam filtering systems, such as [SpamAssassin](http://spamassassin.apache.org). +At the same time, rspamd uses fuzzy logic to process unknown messages. That includes fuzzy hashes and +statistics module. + +## Table of Contents +This document contains the basic documentation for rspamd spam filtering system. It is divided into the following +parts: + +- [Architecture](architecture/) presents the architecture of rspamd and how spam filtering is performed +- [Rspamd configuration](configuration/) describes principles of rspamd configuration +- [Modules](modules/) chapter lists rspamd modules and defines their configuration attributes +- [Workers](workers/) section describes workers that are implemented in the rspamd +- [Lua API](lua/) explains how to extend rspamd with own lua modules
\ No newline at end of file diff --git a/doc/markdown/lua/index.md b/doc/markdown/lua/index.md new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/doc/markdown/lua/index.md diff --git a/doc/markdown/modules/chartable.md b/doc/markdown/modules/chartable.md new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/doc/markdown/modules/chartable.md diff --git a/doc/markdown/modules/dkim.md b/doc/markdown/modules/dkim.md new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/doc/markdown/modules/dkim.md diff --git a/doc/markdown/modules/emails.md b/doc/markdown/modules/emails.md new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/doc/markdown/modules/emails.md diff --git a/doc/markdown/modules/forged_recipients.md b/doc/markdown/modules/forged_recipients.md new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/doc/markdown/modules/forged_recipients.md diff --git a/doc/markdown/modules/fuzzy_check.md b/doc/markdown/modules/fuzzy_check.md new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/doc/markdown/modules/fuzzy_check.md diff --git a/doc/markdown/modules/index.md b/doc/markdown/modules/index.md new file mode 100644 index 000000000..63c55ed2b --- /dev/null +++ b/doc/markdown/modules/index.md @@ -0,0 +1,64 @@ +# Rspamd modules + +Rspamd ships with a set of modules. Some modules are written in C to speedup +complex procedures while others are written in lua to reduce code size. +Actually, new modules are encouraged to be written in lua and add the essential +support to the Lua API itself. Truly speaking, lua modules are very close to +C modules in terms of performance. However, lua modules can be written and loaded +dynamically. + +## C Modules + +C modules provides core functionality of rspamd and are actually statically linked +to the main rspamd code. C modules are defined in the `options` section of rspamd +configuration. If no `filters` attribute is defined then all modules are disabled. +The default configuration enables all modules explicitly: + +~~~nginx +filters = "chartable,dkim,spf,surbl,regexp,fuzzy_check"; +~~~ + +Here is the list of C modules available: + +- [regexp](regexp.md): the core module that allow to define regexp rules, +rspamd internal functions and lua rules. +- [surbl](surbl.md): this module extracts URLs from messages and check them against +public DNS black lists to filter messages with malicious URLs. +- [spf](spf.md): checks SPF records for messages processed. +- [dkim](dkim.md): performs DKIM signatures checks. +- [fuzzy_check](fuzzy_check.md): checks messages fuzzy hashes against public blacklists. +- [chartable](chartable.md): checks character sets of text parts in messages. + +## Lua modules + +Lua modules are dynamically loaded on rspamd startup and are reloaded on rspamd +reconfiguration. Should you want to write a lua module consult with the +[Lua API documentation](../lua/). To define path to lua modules there is a special section +named `modules` in rspamd: + +~~~nginx +modules { + path = "/path/to/dir/"; + path = "/path/to/module.lua"; + path = "$PLUGINSDIR/lua"; +} +~~~ + +If a path is a directory then rspamd scans it for `*.lua" pattern and load all +files matched. + +Here is the list of Lua modules shipped with rspamd: + +- [multimap](multimap.md) - a complex module that operates with different types +of maps. +- [rbl](rbl.md) - a plugin that checks messages against DNS blacklist based on +either SMTP FROM addresses or on information from `Received` headers. +- [emails](emails.md) - extract emails from a message and checks it against DNS +blacklists. +- [maillist](maillist.md) - determines the common mailing list signatures in a message. +- [once_received](once_received.md) - detects messages with a single `Received` headers +and performs some additional checks for such messages. +- [phishing](phishing.md) - detects messages with phished URLs. +- [ratelimit](ratelimit.md) - implements leaked bucket algorithm for ratelimiting and +uses `redis` to store data. +- [trie](trie.md) - uses suffix trie for extra-fast patterns lookup in messages.
\ No newline at end of file diff --git a/doc/markdown/modules/maillist.md b/doc/markdown/modules/maillist.md new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/doc/markdown/modules/maillist.md diff --git a/doc/markdown/modules/multimap.md b/doc/markdown/modules/multimap.md new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/doc/markdown/modules/multimap.md diff --git a/doc/markdown/modules/once_received.md b/doc/markdown/modules/once_received.md new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/doc/markdown/modules/once_received.md diff --git a/doc/markdown/modules/phishing.md b/doc/markdown/modules/phishing.md new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/doc/markdown/modules/phishing.md diff --git a/doc/markdown/modules/ratelimit.md b/doc/markdown/modules/ratelimit.md new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/doc/markdown/modules/ratelimit.md diff --git a/doc/markdown/modules/rbl.md b/doc/markdown/modules/rbl.md new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/doc/markdown/modules/rbl.md diff --git a/doc/markdown/modules/regexp.md b/doc/markdown/modules/regexp.md new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/doc/markdown/modules/regexp.md diff --git a/doc/markdown/modules/spf.md b/doc/markdown/modules/spf.md new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/doc/markdown/modules/spf.md diff --git a/doc/markdown/modules/surbl.md b/doc/markdown/modules/surbl.md new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/doc/markdown/modules/surbl.md diff --git a/doc/markdown/modules/trie.md b/doc/markdown/modules/trie.md new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/doc/markdown/modules/trie.md diff --git a/doc/markdown/workers/index.md b/doc/markdown/workers/index.md new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/doc/markdown/workers/index.md diff --git a/doc/rspamc.1 b/doc/rspamc.1 index 3ca41ac98..73be9afb4 100644 --- a/doc/rspamc.1 +++ b/doc/rspamc.1 @@ -1,260 +1,212 @@ -.\" Automatically generated by Pod::Man 2.25 (Pod::Simple 3.16) -.\" -.\" Standard preamble: -.\" ======================================================================== -.de Sp \" Vertical space (when we can't use .PP) -.if t .sp .5v -.if n .sp -.. -.de Vb \" Begin verbatim text -.ft CW -.nf -.ne \\$1 -.. -.de Ve \" End verbatim text -.ft R -.fi -.. -.\" Set up some character translations and predefined strings. \*(-- will -.\" give an unbreakable dash, \*(PI will give pi, \*(L" will give a left -.\" double quote, and \*(R" will give a right double quote. \*(C+ will -.\" give a nicer C++. Capital omega is used to do unbreakable dashes and -.\" therefore won't be available. \*(C` and \*(C' expand to `' in nroff, -.\" nothing in troff, for use with C<>. -.tr \(*W- -.ds C+ C\v'-.1v'\h'-1p'\s-2+\h'-1p'+\s0\v'.1v'\h'-1p' -.ie n \{\ -. ds -- \(*W- -. ds PI pi -. if (\n(.H=4u)&(1m=24u) .ds -- \(*W\h'-12u'\(*W\h'-12u'-\" diablo 10 pitch -. if (\n(.H=4u)&(1m=20u) .ds -- \(*W\h'-12u'\(*W\h'-8u'-\" diablo 12 pitch -. ds L" "" -. ds R" "" -. ds C` "" -. ds C' "" -'br\} -.el\{\ -. ds -- \|\(em\| -. ds PI \(*p -. ds L" `` -. ds R" '' -'br\} -.\" -.\" Escape single quotes in literal strings from groff's Unicode transform. -.ie \n(.g .ds Aq \(aq -.el .ds Aq ' -.\" -.\" If the F register is turned on, we'll generate index entries on stderr for -.\" titles (.TH), headers (.SH), subsections (.SS), items (.Ip), and index -.\" entries marked with X<> in POD. Of course, you'll have to process the -.\" output yourself in some meaningful fashion. -.ie \nF \{\ -. de IX -. tm Index:\\$1\t\\n%\t"\\$2" -.. -. nr % 0 -. rr F -.\} -.el \{\ -. de IX -.. -.\} -.\" -.\" Accent mark definitions (@(#)ms.acc 1.5 88/02/08 SMI; from UCB 4.2). -.\" Fear. Run. Save yourself. No user-serviceable parts. -. \" fudge factors for nroff and troff -.if n \{\ -. ds #H 0 -. ds #V .8m -. ds #F .3m -. ds #[ \f1 -. ds #] \fP -.\} -.if t \{\ -. ds #H ((1u-(\\\\n(.fu%2u))*.13m) -. ds #V .6m -. ds #F 0 -. ds #[ \& -. ds #] \& -.\} -. \" simple accents for nroff and troff -.if n \{\ -. ds ' \& -. ds ` \& -. ds ^ \& -. ds , \& -. ds ~ ~ -. ds / -.\} -.if t \{\ -. ds ' \\k:\h'-(\\n(.wu*8/10-\*(#H)'\'\h"|\\n:u" -. ds ` \\k:\h'-(\\n(.wu*8/10-\*(#H)'\`\h'|\\n:u' -. ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'^\h'|\\n:u' -. ds , \\k:\h'-(\\n(.wu*8/10)',\h'|\\n:u' -. ds ~ \\k:\h'-(\\n(.wu-\*(#H-.1m)'~\h'|\\n:u' -. ds / \\k:\h'-(\\n(.wu*8/10-\*(#H)'\z\(sl\h'|\\n:u' -.\} -. \" troff and (daisy-wheel) nroff accents -.ds : \\k:\h'-(\\n(.wu*8/10-\*(#H+.1m+\*(#F)'\v'-\*(#V'\z.\h'.2m+\*(#F'.\h'|\\n:u'\v'\*(#V' -.ds 8 \h'\*(#H'\(*b\h'-\*(#H' -.ds o \\k:\h'-(\\n(.wu+\w'\(de'u-\*(#H)/2u'\v'-.3n'\*(#[\z\(de\v'.3n'\h'|\\n:u'\*(#] -.ds d- \h'\*(#H'\(pd\h'-\w'~'u'\v'-.25m'\f2\(hy\fP\v'.25m'\h'-\*(#H' -.ds D- D\\k:\h'-\w'D'u'\v'-.11m'\z\(hy\v'.11m'\h'|\\n:u' -.ds th \*(#[\v'.3m'\s+1I\s-1\v'-.3m'\h'-(\w'I'u*2/3)'\s-1o\s+1\*(#] -.ds Th \*(#[\s+2I\s-2\h'-\w'I'u*3/5'\v'-.3m'o\v'.3m'\*(#] -.ds ae a\h'-(\w'a'u*4/10)'e -.ds Ae A\h'-(\w'A'u*4/10)'E -. \" corrections for vroff -.if v .ds ~ \\k:\h'-(\\n(.wu*9/10-\*(#H)'\s-2\u~\d\s+2\h'|\\n:u' -.if v .ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'\v'-.4m'^\v'.4m'\h'|\\n:u' -. \" for low resolution devices (crt and lpr) -.if \n(.H>23 .if \n(.V>19 \ -\{\ -. ds : e -. ds 8 ss -. ds o a -. ds d- d\h'-1'\(ga -. ds D- D\h'-1'\(hy -. ds th \o'bp' -. ds Th \o'LP' -. ds ae ae -. ds Ae AE -.\} -.rm #[ #] #H #V #F C -.\" ======================================================================== -.\" -.IX Title "rspamc 1" -.TH rspamc 1 "2013-02-02" "rspamd-0.5.4" "Rspamd documentation" -.\" For nroff, turn off justification. Always turn off hyphenation; it makes -.\" way too many mistakes in technical documents. -.if n .ad l -.nh -.SH "NAME" -rspamc \- a simple client for rspamd spam filtering system -.SH "SYNOPSIS" -.IX Header "SYNOPSIS" -rspamc [\fB\-h\fR \fIhost[:port]\fR] [\fB\-p\fR] [\fB\-v\fR] [\fB\-b\fR \fIbind_address\fR] [\fB\-u\fR \fIuser\fR] -[\fB\-F\fR \fIfrom\fR] [\fB\-r\fR \fIrcpt\fR] [\fB\-d\fR \fIdeliver-to\fR] -[\fB\-i\fR \fIip\fR] [\fB\-c\fR \fIclassifier\fR] [\fB\-w\fR \fIweight\fR] -[\fB\-P\fR \fIpassword\fR] [\fB\-f\fR \fIflag\fR] [\fB\-t\fR \fItimeout\fR] [command] [file [file ...]] +.TH RSPAMC 1 "" "Rspamd User Manual" +.SH NAME +.PP +rspamc - rspamd command line client +.SH SYNOPSIS +.PP +rspamc [\f[I]options\f[]] [\f[I]command\f[]] [\f[I]input-file\f[]]... +.PP +rspamc --help +.SH DESCRIPTION +.PP +Rspamc is a simple client for checking messages using rspamd or for +learning rspamd by messages. +Rspamc supports the following commands: +.IP \[bu] 2 +Scan commands: +.RS 2 +.IP \[bu] 2 +\f[I]symbols\f[]: scan message and show symbols (default command) +.RE +.IP \[bu] 2 +Control commands +.RS 2 +.IP \[bu] 2 +\f[I]learn_spam\f[]: learn message as spam +.IP \[bu] 2 +\f[I]learn_ham\f[]: learn message as ham +.IP \[bu] 2 +\f[I]fuzzy_add\f[]: add message to fuzzy storage (check \f[C]-f\f[] and +\f[C]-w\f[] options for this command) +.IP \[bu] 2 +\f[I]fuzzy_del\f[]: delete message from fuzzy storage (check \f[C]-f\f[] +option for this command) +.IP \[bu] 2 +\f[I]stat\f[]: show rspamd statistics +.IP \[bu] 2 +\f[I]stat_reset\f[]: show and reset rspamd statistics (useful for +graphs) +.IP \[bu] 2 +\f[I]counters\f[]: display rspamd symbols statistics +.IP \[bu] 2 +\f[I]uptime\f[]: show rspamd uptime +.IP \[bu] 2 +\f[I]add_symbol\f[]: add or modify symbol settings in rspamd +.IP \[bu] 2 +\f[I]add_action\f[]: add or modify action settings +.RE +.PP +Control commands that modifies rspamd state are considered as privileged +and basically requires a password to be specified with \f[C]-P\f[] +option (see \f[B]OPTIONS\f[], below, for details). +This depends on a controller\[aq]s settings and is discussed in +\f[C]rspamd-workers\f[] page. +.PP +\f[C]Input\ files\f[] may be either regular file(s) or a directory to +scan. +If no files are specified rspamc reads from the standard input. +Controller commands usually does not accept any input, however learn* +and fuzzy* commands requires input. +.SH OPTIONS +.PP +-h \f[I]host[:port]\f[], --connect=\f[I]host[:port]\f[] Specify host and +port +.PP +-P \f[I]password\f[], --password=\f[I]password\f[] Specify control +password +.TP +.B -c \f[I]name\f[], --classifier=\f[I]name\f[] +Classifier to learn spam or ham (bayes is used by default) +.RS +.RE +.TP +.B -w \f[I]weight\f[], --weight=\f[I]weight\f[] +Weight for fuzzy operations +.RS +.RE +.TP +.B -f \f[I]number\f[], --flag=\f[I]number\f[] +Flag for fuzzy operations +.RS +.RE +.TP +.B -p, --pass +Pass all filters +.RS +.RE +.TP +.B -v, --verbose +More verbose output +.RS +.RE +.TP +.B -i \f[I]ip address\f[], --ip=\f[I]ip address\f[] +Emulate that message was received from specified ip address +.RS +.RE +.TP +.B -u \f[I]username\f[], --user=\f[I]username\f[] +Emulate that message was from specified user +.RS +.RE +.TP +.B -d \f[I]user\@domain\f[], --deliver=\f[I]user\@domain\f[] +Emulate that message is delivered to specified user +.RS +.RE +.TP +.B -F \f[I]user\@domain\f[], --from=\f[I]user\@domain\f[] +Emulate that message is from specified user +.RS +.RE +.TP +.B -r \f[I]user\@domain\f[], --rcpt=\f[I]user\@domain\f[] +Emulate that message is for specified user +.RS +.RE +.TP +.B -t \f[I]seconds\f[], --timeout=\f[I]seconds\f[] +Timeout for waiting for a reply +.RS +.RE +.TP +.B -b \f[I]host:port\f[], --bind=\f[I]host:port\f[] +Bind to specified ip address +.RS +.RE +.TP +.B --commands +List available commands +.RS +.RE +.SH RETURN VALUE +.PP +On exit rspamc returns \f[C]0\f[] if operation was successfull and an +error code otherwise. +.SH EXAMPLES .PP -rspamc [\fB\-\-help\fR] -.SH "DESCRIPTION" -.IX Header "DESCRIPTION" -\&\fBRspamc\fR is a simple client for checking messages using rspamd or for learning rspamd by messages. -\&\fBRspamc\fR has several mandatory options for learning: \fIpassword\fR and \fIstatfile\fR. -.SH "OPTIONS" -.IX Header "OPTIONS" -.IP "\fB\-h\fR \fIhost[:port]\fR, \fB\-\-connect\fR \fIhost[:port]\fR" 4 -.IX Item "-h host[:port], --connect host[:port]" -Specify host and port for connecting to rspamd server. Default host is \fIlocalhost\fR and -default port is \fI11333\fR for checking messages and \fI11334\fR for learning and statistic. -Also it is possible to specify a unix socket for all operations (for example: -\&\fBrspamc\fR \fB\-h\fR /path/to/soket) -.IP "\fB\-b\fR \fIlocal_ip\fR, \fB\-\-bind\fR \fIlocal_ip\fR" 4 -.IX Item "-b local_ip, --bind local_ip" -Specify explicit \s-1IP\s0 address to bind a client for operations. -.IP "\fB\-u\fR \fIuser\fR, \fB\-\-user\fR \fIuser\fR" 4 -.IX Item "-u user, --user user" -Specify username for connection with rspamd server. -.IP "\fB\-F\fR \fIfrom_addr\fR, \fB\-\-from\fR \fIfrom_addr\fR" 4 -.IX Item "-F from_addr, --from from_addr" -Specify \s-1SMTP\s0 \s-1FROM\s0 address for connection with rspamd server. -.IP "\fB\-r\fR \fIrcpt_addr\fR, \fB\-\-rcpt\fR \fIrcpt_addr\fR" 4 -.IX Item "-r rcpt_addr, --rcpt rcpt_addr" -Specify \s-1SMTP\s0 \s-1RCPT\s0 \s-1TO\s0 address for connection with rspamd server. -.IP "\fB\-d\fR \fIdeliver_addr\fR, \fB\-\-deliver\fR \fIdeliver_addr\fR" 4 -.IX Item "-d deliver_addr, --deliver deliver_addr" -Specify real delivery address for connection with rspamd server. -.IP "\fB\-p\fR, \fB\-\-pass\-all\fR" 4 -.IX Item "-p, --pass-all" -Pass all filters when checking messages. Ignored in case of learning. -.IP "\fB\-v\fR, \fB\-\-verbose\fR" 4 -.IX Item "-v, --verbose" -Be more verbose while displaying results. For example show descriptions of symbols. -.IP "\fB\-P\fR \fIpassword\fR, \fB\-\-password\fR \fIpassword\fR" 4 -.IX Item "-P password, --password password" -Specify controller's password. Mandatory option for learning. -.IP "\fB\-c\fR \fIclassifier\fR, \fB\-\-classifier\fR \fIclassifier\fR" 4 -.IX Item "-c classifier, --classifier classifier" -Specify classifier to learn message. Mandatory option for learning. Bayes classifier is used by default if this option is omitted. -.IP "\fB\-i\fR \fIip\fR, \fB\-\-ip\fR \fIip\fR" 4 -.IX Item "-i ip, --ip ip" -Add \s-1IP\s0 header when scanning message. Useful for checking messages and emulating that client comes from -specific \s-1IP\s0 address. -.IP "\fB\-w\fR \fIweight\fR, \fB\-\-weight\fR \fIweight\fR" 4 -.IX Item "-w weight, --weight weight" -Weight of message for fuzzy operations. -.IP "\fB\-f\fR \fIflag\fR, \fB\-\-flag\fR \fIflag\fR" 4 -.IX Item "-f flag, --flag flag" -Flag of list for fuzzy operations. -.IP "\fB\-t\fR \fItimeout\fR, \fB\-\-timeout\fR \fItimeout\fR" 4 -.IX Item "-t timeout, --timeout timeout" -Timeout in seconds for all operations. Default value is 5 seconds. -.SH "RETURN VALUE" -.IX Header "RETURN VALUE" -On exit \fBrspamc\fR returns 0 if operation was successfull and error code otherwise. -.SH "EXAMPLES" -.IX Header "EXAMPLES" Check stdin: -.PP -.Vb 1 -\& rspamc < some_file -.Ve +.IP +.nf +\f[C] +rspamc\ <\ some_file +\f[] +.fi .PP Check files: -.PP -.Vb 1 -\& rspamc symbols file1 file2 file3 -.Ve +.IP +.nf +\f[C] +rspamc\ symbols\ file1\ file2\ file3 +\f[] +.fi .PP Learn files: -.PP -.Vb 1 -\& rspamc \-P pass learn_spam file1 file2 file3 -.Ve +.IP +.nf +\f[C] +rspamc\ -P\ pass\ learn_spam\ file1\ file2\ file3 +\f[] +.fi .PP Add fuzzy hash to set 2: -.PP -.Vb 1 -\& rspamc \-P pass \-f 2 \-w 10 fuzzy_add file1 file2 -.Ve +.IP +.nf +\f[C] +rspamc\ -P\ pass\ -f\ 2\ -w\ 10\ fuzzy_add\ file1\ file2 +\f[] +.fi .PP Delete fuzzy hash from other server: -.PP -.Vb 1 -\& rspamc \-P pass \-h hostname:11334 \-f 2 fuzzy_del file1 file2 -.Ve +.IP +.nf +\f[C] +rspamc\ -P\ pass\ -h\ hostname:11334\ -f\ 2\ fuzzy_del\ file1\ file2 +\f[] +.fi .PP Get statistics: -.PP -.Vb 1 -\& rspamc stat -.Ve +.IP +.nf +\f[C] +rspamc\ stat +\f[] +.fi .PP Get uptime: +.IP +.nf +\f[C] +rspamc\ uptime +\f[] +.fi .PP -.Vb 1 -\& rspamc uptime -.Ve -.PP -Add custom rule's weight: -.PP -.Vb 1 -\& rspamc add_symbol test 1.5 -.Ve -.PP -Add custom action's weight: +Add custom rule\[aq]s weight: +.IP +.nf +\f[C] +rspamc\ add_symbol\ test\ 1.5 +\f[] +.fi .PP -.Vb 1 -\& rspamc add_action reject 7.1 -.Ve -.SH "AUTHOR" -.IX Header "AUTHOR" -Vsevolod Stakhov <vsevolod@highsecure.ru> -.SH "COPYRIGHT AND LICENSE" -.IX Header "COPYRIGHT AND LICENSE" -Copyright 2011\-2012 by Vsevolod Stakhov <vsevolod@highsecure.ru>. +Add custom action\[aq]s weight: +.IP +.nf +\f[C] +rspamc\ add_action\ reject\ 7.1 +\f[] +.fi +.SH SEE ALSO .PP -This program is free software; you may redistribute it and/or modify it -under the terms of \s-1BSD\s0 license. +Rspamd documentation and source codes may be downloaded from +<https://rspamd.com/>. diff --git a/doc/rspamc.1.md b/doc/rspamc.1.md new file mode 100644 index 000000000..531e0802d --- /dev/null +++ b/doc/rspamc.1.md @@ -0,0 +1,134 @@ +% RSPAMC(1) Rspamd User Manual + +# NAME + +rspamc - rspamd command line client + +# SYNOPSIS + +rspamc [*options*] [*command*] [*input-file*]... + +rspamc --help + +# DESCRIPTION + +Rspamc is a simple client for checking messages using rspamd or for learning rspamd by messages. +Rspamc supports the following commands: + +* Scan commands: + * *symbols*: scan message and show symbols (default command) +* Control commands + * *learn_spam*: learn message as spam + * *learn_ham*: learn message as ham + * *fuzzy_add*: add message to fuzzy storage (check `-f` and `-w` options for this command) + * *fuzzy_del*: delete message from fuzzy storage (check `-f` option for this command) + * *stat*: show rspamd statistics + * *stat_reset*: show and reset rspamd statistics (useful for graphs) + * *counters*: display rspamd symbols statistics + * *uptime*: show rspamd uptime + * *add_symbol*: add or modify symbol settings in rspamd + * *add_action*: add or modify action settings + +Control commands that modifies rspamd state are considered as privileged and basically requires a password +to be specified with `-P` option (see **OPTIONS**, below, for details). +This depends on a controller's settings and is discussed in `rspamd-workers` page. + +`Input files` may be either regular file(s) or a directory to scan. If no files are specified rspamc reads +from the standard input. Controller commands usually does not accept any input, however learn* and fuzzy* commands +requires input. + +# OPTIONS + +-h *host[:port]*, \--connect=*host[:port]* + Specify host and port + +-P *password*, \--password=*password* + Specify control password + +-c *name*, \--classifier=*name* +: Classifier to learn spam or ham (bayes is used by default) + +-w *weight*, \--weight=*weight* +: Weight for fuzzy operations + +-f *number*, \--flag=*number* +: Flag for fuzzy operations + +-p, \--pass +: Pass all filters + +-v, \--verbose +: More verbose output + +-i *ip address*, \--ip=*ip address* +: Emulate that message was received from specified ip address + +-u *username*, \--user=*username* +: Emulate that message was from specified user + +-d *user@domain*, \--deliver=*user@domain* +: Emulate that message is delivered to specified user + +-F *user@domain*, \--from=*user@domain* +: Emulate that message is from specified user + +-r *user@domain*, \--rcpt=*user@domain* +: Emulate that message is for specified user + +-t *seconds*, \--timeout=*seconds* +: Timeout for waiting for a reply + +-b *host:port*, \--bind=*host:port* +: Bind to specified ip address + +\--commands +: List available commands + +# RETURN VALUE + +On exit rspamc returns `0` if operation was successfull and an error code otherwise. + +# EXAMPLES + +Check stdin: + + rspamc < some_file + +Check files: + + rspamc symbols file1 file2 file3 + +Learn files: + + rspamc -P pass learn_spam file1 file2 file3 + +Add fuzzy hash to set 2: + + rspamc -P pass -f 2 -w 10 fuzzy_add file1 file2 + +Delete fuzzy hash from other server: + + rspamc -P pass -h hostname:11334 -f 2 fuzzy_del file1 file2 + +Get statistics: + + rspamc stat + +Get uptime: + + rspamc uptime + +Add custom rule's weight: + + rspamc add_symbol test 1.5 + +Add custom action's weight: + + rspamc add_action reject 7.1 + +# SEE ALSO + +Rspamd documentation and source codes may be downloaded from +<https://rspamd.com/>. + +[rspamd-workers]: https://rspamd.com/doc/rspamd-workers.html
\ No newline at end of file diff --git a/doc/rspamc.pod b/doc/rspamc.pod deleted file mode 100644 index 73b43b649..000000000 --- a/doc/rspamc.pod +++ /dev/null @@ -1,138 +0,0 @@ -=head1 NAME - -rspamc - a simple client for rspamd spam filtering system - -=head1 SYNOPSIS - -rspamc [B<-h> I<host[:port]>] [B<-p>] [B<-v>] [B<-b> I<bind_address>] [B<-u> I<user>] -[B<-F> I<from>] [B<-r> I<rcpt>] [B<-d> I<deliver-to>] -[B<-i> I<ip>] [B<-c> I<classifier>] [B<-w> I<weight>] -[B<-P> I<password>] [B<-f> I<flag>] [B<-t> I<timeout>] [command] [file [file ...]] - -rspamc [B<--help>] - -=head1 DESCRIPTION - -B<Rspamc> is a simple client for checking messages using rspamd or for learning rspamd by messages. -B<Rspamc> has several mandatory options for learning: I<password> and I<statfile>. - -=head1 OPTIONS - -=over 4 - -=item B<-h> I<host[:port]>, B<--connect> I<host[:port]> - -Specify host and port for connecting to rspamd server. Default host is I<localhost> and -default port is I<11333> for checking messages and I<11334> for learning and statistic. -Also it is possible to specify a unix socket for all operations (for example: -B<rspamc> B<-h> /path/to/soket) - -=item B<-b> I<local_ip>, B<--bind> I<local_ip> - -Specify explicit IP address to bind a client for operations. - -=item B<-u> I<user>, B<--user> I<user> - -Specify username for connection with rspamd server. - -=item B<-F> I<from_addr>, B<--from> I<from_addr> - -Specify SMTP FROM address for connection with rspamd server. - -=item B<-r> I<rcpt_addr>, B<--rcpt> I<rcpt_addr> - -Specify SMTP RCPT TO address for connection with rspamd server. - -=item B<-d> I<deliver_addr>, B<--deliver> I<deliver_addr> - -Specify real delivery address for connection with rspamd server. - -=item B<-p>, B<--pass-all> - -Pass all filters when checking messages. Ignored in case of learning. - -=item B<-v>, B<--verbose> - -Be more verbose while displaying results. For example show descriptions of symbols. - -=item B<-P> I<password>, B<--password> I<password> - -Specify controller's password. Mandatory option for learning. - -=item B<-c> I<classifier>, B<--classifier> I<classifier> - -Specify classifier to learn message. Mandatory option for learning. Bayes classifier is used by default if this option is omitted. - -=item B<-i> I<ip>, B<--ip> I<ip> - -Add IP header when scanning message. Useful for checking messages and emulating that client comes from -specific IP address. - -=item B<-w> I<weight>, B<--weight> I<weight> - -Weight of message for fuzzy operations. - -=item B<-f> I<flag>, B<--flag> I<flag> - -Flag of list for fuzzy operations. - -=item B<-t> I<timeout>, B<--timeout> I<timeout> - -Timeout in seconds for all operations. Default value is 5 seconds. - -=back - -=head1 RETURN VALUE - -On exit B<rspamc> returns 0 if operation was successfull and error code otherwise. - -=head1 EXAMPLES - -Check stdin: - - rspamc < some_file - -Check files: - - rspamc symbols file1 file2 file3 - -Learn files: - - rspamc -P pass learn_spam file1 file2 file3 - -Add fuzzy hash to set 2: - - rspamc -P pass -f 2 -w 10 fuzzy_add file1 file2 - -Delete fuzzy hash from other server: - - rspamc -P pass -h hostname:11334 -f 2 fuzzy_del file1 file2 - -Get statistics: - - rspamc stat - -Get uptime: - - rspamc uptime - -Add custom rule's weight: - - rspamc add_symbol test 1.5 - -Add custom action's weight: - - rspamc add_action reject 7.1 - -=head1 AUTHOR - -Vsevolod Stakhov <vsevolod@highsecure.ru> - -=head1 COPYRIGHT AND LICENSE - -Copyright 2011-2012 by Vsevolod Stakhov <vsevolod@highsecure.ru>. - -This program is free software; you may redistribute it and/or modify it -under the terms of BSD license. - -=cut diff --git a/doc/rspamd.8 b/doc/rspamd.8 index c34e553fd..37650ebaa 100644 --- a/doc/rspamd.8 +++ b/doc/rspamd.8 @@ -1,190 +1,138 @@ -.\" Automatically generated by Pod::Man 2.25 (Pod::Simple 3.16) -.\" -.\" Standard preamble: -.\" ======================================================================== -.de Sp \" Vertical space (when we can't use .PP) -.if t .sp .5v -.if n .sp -.. -.de Vb \" Begin verbatim text -.ft CW +.TH RSPAMD 8 "" "Rspamd User Manual" +.SH NAME +.PP +rspamd - main daemon for rapid spam filtering system +.SH SYNOPSIS +.PP +rspamd [\f[I]options\f[]]... +.PP +rspamd --help +.SH DESCRIPTION +.PP +Rspamd filtering system is designed to be fast, modular and easily +scalable system. +Rspamd core is written in \f[C]C\f[] language using event driven +processing model. +Plugins for rspamd can be written in \f[C]Lua\f[] programming language. +Rspamd is designed to process connections completely asynchronous and do +not block anywhere in code. +.SH OPTIONS +.TP +.B -t, --config-test +Perform config test and exit +.RS +.RE +.TP +.B -f, --no-fork +Do not daemonize main process +.RS +.RE +.TP +.B -c \f[I]path\f[], --config=\f[I]path\f[] +Specify config file(s) +.RS +.RE +.TP +.B -u \f[I]username\f[], --user=\f[I]username\f[] +User to run rspamd as +.RS +.RE +.TP +.B -g \f[I]groupname\f[], --group=\f[I]groupname\f[] +Group to run rspamd as +.RS +.RE +.TP +.B -p \f[I]path\f[], --pid=\f[I]path\f[] +Path to pidfile +.RS +.RE +.TP +.B -C, --dump-cache +Dump symbols cache stats and exit +.RS +.RE +.TP +.B -d, --debug +Force debug output +.RS +.RE +.TP +.B -i, --insecure +Ignore running workers as privileged users (insecure) +.RS +.RE +.TP +.B --test-lua=\f[I]path\f[] +Specify lua file(s) to test +.RS +.RE +.TP +.B --sign-config=\f[I]path\f[] +Specify config file(s) to sign +.RS +.RE +.TP +.B --private-key=\f[I]path\f[] +Specify private key to sign +.RS +.RE +.TP +.B --convert-config=\f[I]path\f[] +Convert configuration to UCL +.RS +.RE +.SH EXAMPLES +.PP +Run rspamd daemon with default configuration: +.IP .nf -.ne \\$1 -.. -.de Ve \" End verbatim text -.ft R +\f[C] +rspamd +\f[] .fi -.. -.\" Set up some character translations and predefined strings. \*(-- will -.\" give an unbreakable dash, \*(PI will give pi, \*(L" will give a left -.\" double quote, and \*(R" will give a right double quote. \*(C+ will -.\" give a nicer C++. Capital omega is used to do unbreakable dashes and -.\" therefore won't be available. \*(C` and \*(C' expand to `' in nroff, -.\" nothing in troff, for use with C<>. -.tr \(*W- -.ds C+ C\v'-.1v'\h'-1p'\s-2+\h'-1p'+\s0\v'.1v'\h'-1p' -.ie n \{\ -. ds -- \(*W- -. ds PI pi -. if (\n(.H=4u)&(1m=24u) .ds -- \(*W\h'-12u'\(*W\h'-12u'-\" diablo 10 pitch -. if (\n(.H=4u)&(1m=20u) .ds -- \(*W\h'-12u'\(*W\h'-8u'-\" diablo 12 pitch -. ds L" "" -. ds R" "" -. ds C` "" -. ds C' "" -'br\} -.el\{\ -. ds -- \|\(em\| -. ds PI \(*p -. ds L" `` -. ds R" '' -'br\} -.\" -.\" Escape single quotes in literal strings from groff's Unicode transform. -.ie \n(.g .ds Aq \(aq -.el .ds Aq ' -.\" -.\" If the F register is turned on, we'll generate index entries on stderr for -.\" titles (.TH), headers (.SH), subsections (.SS), items (.Ip), and index -.\" entries marked with X<> in POD. Of course, you'll have to process the -.\" output yourself in some meaningful fashion. -.ie \nF \{\ -. de IX -. tm Index:\\$1\t\\n%\t"\\$2" -.. -. nr % 0 -. rr F -.\} -.el \{\ -. de IX -.. -.\} -.\" -.\" Accent mark definitions (@(#)ms.acc 1.5 88/02/08 SMI; from UCB 4.2). -.\" Fear. Run. Save yourself. No user-serviceable parts. -. \" fudge factors for nroff and troff -.if n \{\ -. ds #H 0 -. ds #V .8m -. ds #F .3m -. ds #[ \f1 -. ds #] \fP -.\} -.if t \{\ -. ds #H ((1u-(\\\\n(.fu%2u))*.13m) -. ds #V .6m -. ds #F 0 -. ds #[ \& -. ds #] \& -.\} -. \" simple accents for nroff and troff -.if n \{\ -. ds ' \& -. ds ` \& -. ds ^ \& -. ds , \& -. ds ~ ~ -. ds / -.\} -.if t \{\ -. ds ' \\k:\h'-(\\n(.wu*8/10-\*(#H)'\'\h"|\\n:u" -. ds ` \\k:\h'-(\\n(.wu*8/10-\*(#H)'\`\h'|\\n:u' -. ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'^\h'|\\n:u' -. ds , \\k:\h'-(\\n(.wu*8/10)',\h'|\\n:u' -. ds ~ \\k:\h'-(\\n(.wu-\*(#H-.1m)'~\h'|\\n:u' -. ds / \\k:\h'-(\\n(.wu*8/10-\*(#H)'\z\(sl\h'|\\n:u' -.\} -. \" troff and (daisy-wheel) nroff accents -.ds : \\k:\h'-(\\n(.wu*8/10-\*(#H+.1m+\*(#F)'\v'-\*(#V'\z.\h'.2m+\*(#F'.\h'|\\n:u'\v'\*(#V' -.ds 8 \h'\*(#H'\(*b\h'-\*(#H' -.ds o \\k:\h'-(\\n(.wu+\w'\(de'u-\*(#H)/2u'\v'-.3n'\*(#[\z\(de\v'.3n'\h'|\\n:u'\*(#] -.ds d- \h'\*(#H'\(pd\h'-\w'~'u'\v'-.25m'\f2\(hy\fP\v'.25m'\h'-\*(#H' -.ds D- D\\k:\h'-\w'D'u'\v'-.11m'\z\(hy\v'.11m'\h'|\\n:u' -.ds th \*(#[\v'.3m'\s+1I\s-1\v'-.3m'\h'-(\w'I'u*2/3)'\s-1o\s+1\*(#] -.ds Th \*(#[\s+2I\s-2\h'-\w'I'u*3/5'\v'-.3m'o\v'.3m'\*(#] -.ds ae a\h'-(\w'a'u*4/10)'e -.ds Ae A\h'-(\w'A'u*4/10)'E -. \" corrections for vroff -.if v .ds ~ \\k:\h'-(\\n(.wu*9/10-\*(#H)'\s-2\u~\d\s+2\h'|\\n:u' -.if v .ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'\v'-.4m'^\v'.4m'\h'|\\n:u' -. \" for low resolution devices (crt and lpr) -.if \n(.H>23 .if \n(.V>19 \ -\{\ -. ds : e -. ds 8 ss -. ds o a -. ds d- d\h'-1'\(ga -. ds D- D\h'-1'\(hy -. ds th \o'bp' -. ds Th \o'LP' -. ds ae ae -. ds Ae AE -.\} -.rm #[ #] #H #V #F C -.\" ======================================================================== -.\" -.IX Title "rspamd 8" -.TH rspamd 8 "2013-02-02" "rspamd-0.5.4" "Rspamd documentation" -.\" For nroff, turn off justification. Always turn off hyphenation; it makes -.\" way too many mistakes in technical documents. -.if n .ad l -.nh -.SH "NAME" -rspamd \- main daemon for rspamd spam filtering system -.SH "SYNOPSIS" -.IX Header "SYNOPSIS" -rspamd [\fB\-c\fR \fIconfig_file\fR] [\fB\-f\fR] -[\fB\-u\fR \fIuser\fR] [\fB\-g\fR \fIgroup\fR] [\fB\-p\fR \fIpidfile\fR] -[\fB\-t\fR] [\fB\-d\fR] .PP -rspamd [\fB\-\-help\fR] +Run rspamd in foreground with custom configuration: +.IP +.nf +\f[C] +rspamd\ -f\ -c\ ~/rspamd.conf +\f[] +.fi .PP -rspamd [\fB\-t\fR] +Run rspamd specifying user and group: +.IP +.nf +\f[C] +rspamd\ -u\ rspamd\ -g\ rspamd\ -c\ /etc/rspamd/rspamd.conf +\f[] +.fi .PP -rspamd [\fB\-C\fR] -.SH "DESCRIPTION" -.IX Header "DESCRIPTION" -\&\fBRspamd\fR filtering system is designed to be fast, modular and easily extendable system. -\&\fBRspamd\fR core is written in C language using event driven paradigma. -Plugins for \fBrspamd\fR can be written in lua. -\&\fBRspamd\fR is designed to process connections completely asynchronous and do not block anywhere in code. -.SH "OPTIONS" -.IX Header "OPTIONS" -.IP "\fB\-c\fR \fIconfig_file\fR, \fB\-\-config\fR \fIconfig_file\fR" 4 -.IX Item "-c config_file, --config config_file" -Specify the path where rspamd config is placed. Default is rspamd.xml. -.IP "\fB\-u\fR \fIuser\fR, \fB\-\-user\fR \fIuser\fR" 4 -.IX Item "-u user, --user user" -Specify user rspamd run as. It is possible only when rspamd is launched by super-user as it -calls \fIsetuid\fR\|(2) after spawning workers. -.IP "\fB\-g\fR \fIgroup\fR, \fB\-\-group\fR \fIgroup\fR" 4 -.IX Item "-g group, --group group" -Specify group rspamd run as. -.IP "\fB\-p\fR \fIpidfile\fR, \fB\-\-pidfile\fR \fIpidfile\fR" 4 -.IX Item "-p pidfile, --pidfile pidfile" -Path to pid file where rspamd pid would be stored. Directory containing pidfile must be -writeable by \fBrspamd\fR. -.IP "\fB\-f\fR, \fB\-\-no\-fork\fR" 4 -.IX Item "-f, --no-fork" -Do not daemonize after launch. Usable for debugging purposes. -.IP "\fB\-t\fR, \fB\-\-config\-test\fR" 4 -.IX Item "-t, --config-test" -Just perform test of configuration. Return zero exit code when configuration is \s-1OK\s0. -.IP "\fB\-C\fR, \fB\-\-counters\fR" 4 -.IX Item "-C, --counters" -Show counters for all symbols. Usable when symbols cache is saved. -.IP "\fB\-d\fR, \fB\-\-debug\fR" 4 -.IX Item "-d, --debug" -Turn on debugging mode in logging. -.SH "RETURN VALUE" -.IX Header "RETURN VALUE" -On exit \fBrspamd\fR returns 0 if operation was successfull and error code otherwise. -.SH "AUTHOR" -.IX Header "AUTHOR" -Vsevolod Stakhov <vsevolod@highsecure.ru> -.SH "COPYRIGHT AND LICENSE" -.IX Header "COPYRIGHT AND LICENSE" -Copyright 2011 by Vsevolod Stakhov <vsevolod@highsecure.ru>. +Test lua scripts using rspamd API: +.IP +.nf +\f[C] +rspamd\ --test-lua=~/test1.lua\ --test-lua=~/test2.lua +\f[] +.fi +.PP +Sign config files for \f[C].includes\f[] macro: +.IP +.nf +\f[C] +rspamd\ --private-key=sign.key\ --sign-config=rspamd.conf +\f[] +.fi +.PP +Convert old \f[C]XML\f[] config to the \f[C]UCL\f[] format (since +0.6.0): +.IP +.nf +\f[C] +rspamd\ -c\ /etc/rspamd.xml\ --convert-config=/etc/rspamd/rspamd.conf +\f[] +.fi +.SH SEE ALSO .PP -This program is free software; you may redistribute it and/or modify it -under the terms of \s-1BSD\s0 license. +Rspamd documentation and source codes may be downloaded from +<https://rspamd.com/>. diff --git a/doc/rspamd.8.md b/doc/rspamd.8.md new file mode 100644 index 000000000..2e6d60b48 --- /dev/null +++ b/doc/rspamd.8.md @@ -0,0 +1,90 @@ +% RSPAMD(8) Rspamd User Manual + +# NAME + +rspamd - main daemon for rapid spam filtering system + +# SYNOPSIS + +rspamd [*options*]... + +rspamd --help + +# DESCRIPTION + +Rspamd filtering system is designed to be fast, modular and easily scalable system. +Rspamd core is written in `C` language using event driven processing model. +Plugins for rspamd can be written in `Lua` programming language. +Rspamd is designed to process connections completely asynchronous and do not block anywhere in code. + +# OPTIONS + +-t, \--config-test +: Perform config test and exit + +-f, \--no-fork +: Do not daemonize main process + +-c *path*, \--config=*path* +: Specify config file(s) + +-u *username*, \--user=*username* +: User to run rspamd as + +-g *groupname*, \--group=*groupname* +: Group to run rspamd as + +-p *path*, \--pid=*path* +: Path to pidfile + +-C, \--dump-cache +: Dump symbols cache stats and exit + +-d, \--debug +: Force debug output + +-i, \--insecure +: Ignore running workers as privileged users (insecure) + +\--test-lua=*path* +: Specify lua file(s) to test + +\--sign-config=*path* +: Specify config file(s) to sign + +\--private-key=*path* +: Specify private key to sign + +\--convert-config=*path* +: Convert configuration to UCL + +# EXAMPLES + +Run rspamd daemon with default configuration: + + rspamd + +Run rspamd in foreground with custom configuration: + + rspamd -f -c ~/rspamd.conf + +Run rspamd specifying user and group: + + rspamd -u rspamd -g rspamd -c /etc/rspamd/rspamd.conf + +Test lua scripts using rspamd API: + + rspamd --test-lua=~/test1.lua --test-lua=~/test2.lua + +Sign config files for `.includes` macro: + + rspamd --private-key=sign.key --sign-config=rspamd.conf + +Convert old `XML` config to the `UCL` format (since 0.6.0): + + rspamd -c /etc/rspamd.xml --convert-config=/etc/rspamd/rspamd.conf + +# SEE ALSO + +Rspamd documentation and source codes may be downloaded from +<https://rspamd.com/>.
\ No newline at end of file diff --git a/doc/rspamd.pod b/doc/rspamd.pod deleted file mode 100644 index 0e03d5c82..000000000 --- a/doc/rspamd.pod +++ /dev/null @@ -1,79 +0,0 @@ -=head1 NAME - -rspamd - main daemon for rspamd spam filtering system - -=head1 SYNOPSIS - -rspamd [B<-c> I<config_file>] [B<-f>] -[B<-u> I<user>] [B<-g> I<group>] [B<-p> I<pidfile>] -[B<-t>] [B<-d>] - -rspamd [B<--help>] - -rspamd [B<-t>] - -rspamd [B<-C>] - -=head1 DESCRIPTION - -B<Rspamd> filtering system is designed to be fast, modular and easily extendable system. -B<Rspamd> core is written in C language using event driven paradigma. -Plugins for B<rspamd> can be written in lua. -B<Rspamd> is designed to process connections completely asynchronous and do not block anywhere in code. - -=head1 OPTIONS - -=over 4 - -=item B<-c> I<config_file>, B<--config> I<config_file> - -Specify the path where rspamd config is placed. Default is rspamd.xml. - -=item B<-u> I<user>, B<--user> I<user> - -Specify user rspamd run as. It is possible only when rspamd is launched by super-user as it -calls setuid(2) after spawning workers. - -=item B<-g> I<group>, B<--group> I<group> - -Specify group rspamd run as. - -=item B<-p> I<pidfile>, B<--pidfile> I<pidfile> - -Path to pid file where rspamd pid would be stored. Directory containing pidfile must be -writeable by B<rspamd>. - -=item B<-f>, B<--no-fork> - -Do not daemonize after launch. Usable for debugging purposes. - -=item B<-t>, B<--config-test> - -Just perform test of configuration. Return zero exit code when configuration is OK. - -=item B<-C>, B<--counters> - -Show counters for all symbols. Usable when symbols cache is saved. - -=item B<-d>, B<--debug> - -Turn on debugging mode in logging. - -=back - -=head1 RETURN VALUE - -On exit B<rspamd> returns 0 if operation was successfull and error code otherwise. - -=head1 AUTHOR - -Vsevolod Stakhov <vsevolod@highsecure.ru> - -=head1 COPYRIGHT AND LICENSE - -Copyright 2011 by Vsevolod Stakhov <vsevolod@highsecure.ru>. - -This program is free software; you may redistribute it and/or modify it -under the terms of BSD license. - -=cut
\ No newline at end of file diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 51e8b218a..b236be14a 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -70,10 +70,9 @@ IF(CMAKE_COMPILER_IS_GNUCC) SET_TARGET_PROPERTIES(rspamd-util PROPERTIES COMPILE_FLAGS "-fno-strict-aliasing") ENDIF(CMAKE_COMPILER_IS_GNUCC) -TARGET_LINK_LIBRARIES(rspamd-util ${CMAKE_REQUIRED_LIBRARIES}) +TARGET_LINK_LIBRARIES(rspamd-util ${RSPAMD_REQUIRED_LIBRARIES}) TARGET_LINK_LIBRARIES(rspamd-util pcre) TARGET_LINK_LIBRARIES(rspamd-util rspamd-ucl) -TARGET_LINK_LIBRARIES(rspamd-util ${GLIB2_LIBRARIES}) TARGET_LINK_LIBRARIES(rspamd-util event) IF(NOT DEBIAN_BUILD) @@ -101,16 +100,16 @@ IF(NOT DEBIAN_BUILD) SET_TARGET_PROPERTIES(rspamdclient_static PROPERTIES COMPILE_FLAGS "-fno-strict-aliasing") ENDIF(CMAKE_COMPILER_IS_GNUCC) TARGET_LINK_LIBRARIES(rspamdclient rspamd-util) - TARGET_LINK_LIBRARIES(rspamdclient_static ${CMAKE_REQUIRED_LIBRARIES}) - TARGET_LINK_LIBRARIES(rspamdclient_static ${GLIB2_LIBRARIES}) + TARGET_LINK_LIBRARIES(rspamdclient ${RSPAMD_REQUIRED_LIBRARIES}) + TARGET_LINK_LIBRARIES(rspamdclient_static rspamd-util) + TARGET_LINK_LIBRARIES(rspamdclient_static ${RSPAMD_REQUIRED_LIBRARIES}) ELSE(NOT DEBIAN_BUILD) ADD_LIBRARY(rspamdclient STATIC ${LIBRSPAMDCLIENTSRC}) IF(CMAKE_COMPILER_IS_GNUCC) SET_TARGET_PROPERTIES(rspamdclient PROPERTIES COMPILE_FLAGS "-fno-strict-aliasing") ENDIF(CMAKE_COMPILER_IS_GNUCC) TARGET_LINK_LIBRARIES(rspamdclient rspamd-util) - TARGET_LINK_LIBRARIES(rspamdclient ${CMAKE_REQUIRED_LIBRARIES}) - TARGET_LINK_LIBRARIES(rspamdclient ${GLIB2_LIBRARIES}) + TARGET_LINK_LIBRARIES(rspamdclient ${RSPAMD_REQUIRED_LIBRARIES}) ENDIF(NOT DEBIAN_BUILD) IF(NOT DEBIAN_BUILD) @@ -159,9 +158,6 @@ ENDIF(CMAKE_COMPILER_IS_GNUCC) IF(WITH_DB) TARGET_LINK_LIBRARIES(rspamd-server db) ENDIF(WITH_DB) -IF(SQLITE_LIBRARIES) - TARGET_LINK_LIBRARIES(rspamd-server ${SQLITE_LIBRARIES}) -ENDIF(SQLITE_LIBRARIES) IF(OPENSSL_FOUND) TARGET_LINK_LIBRARIES(rspamd-server ${OPENSSL_LIBRARIES}) @@ -186,12 +182,6 @@ IF(CMAKE_COMPILER_IS_GNUCC) SET_TARGET_PROPERTIES(rspamd-mime PROPERTIES COMPILE_FLAGS "-DRSPAMD_LIB -fno-strict-aliasing") ENDIF(CMAKE_COMPILER_IS_GNUCC) -IF(GMIME24) - TARGET_LINK_LIBRARIES(rspamd-mime ${GMIME24_LIBRARIES}) -ELSE(GMIME24) - TARGET_LINK_LIBRARIES(rspamd-mime ${GMIME2_LIBRARIES}) -ENDIF(GMIME24) - IF(NO_SHARED MATCHES "OFF") INSTALL(TARGETS rspamd-mime LIBRARY DESTINATION ${LIBDIR} diff --git a/perl/MANIFEST b/perl/MANIFEST deleted file mode 100644 index 49ff80978..000000000 --- a/perl/MANIFEST +++ /dev/null @@ -1,3 +0,0 @@ -Makefile.PL -MANIFEST -lib/Mail/Rspamd/Client.pm diff --git a/perl/Makefile.PL.in b/perl/Makefile.PL.in deleted file mode 100644 index 625e69310..000000000 --- a/perl/Makefile.PL.in +++ /dev/null @@ -1,11 +0,0 @@ -use ExtUtils::MakeMaker; -WriteMakefile( - AUTHOR => 'Vsevolod Stakhov <vsevolod@highsecure.ru>', - VERSION_FROM => 'lib/Mail/Rspamd/Client.pm', # finds $VERSION - PREREQ_PM => { - "IO::String" => 0, - "Term::ReadKey" => 0, - "XML::Parser" => 0, - "IO::Socket" => 0, - }, - ); diff --git a/perl/lib/Mail/Rspamd/Client.pm b/perl/lib/Mail/Rspamd/Client.pm deleted file mode 100644 index 9353c2468..000000000 --- a/perl/lib/Mail/Rspamd/Client.pm +++ /dev/null @@ -1,1390 +0,0 @@ - -=head1 NAME - -Mail::Rspamd::Client - Client for rspamd Protocol - - -=head1 SYNOPSIS - - my $client = new Mail::Rspamd::Client($config); - - if ($client->ping()) { - $self->{error} = "Ping is ok\n"; - } - - my $result = $client->check($testmsg); - - if ($result->{'default'}->{isspam} eq 'True') { - do something with spam message here - } - -=head1 DESCRIPTION - -Mail::Rspamd::Client is a module that provides a perl implementation for -the spamd protocol. - -=cut - -package Mail::Rspamd::Client; - -use IO::Socket; -use Carp; - -use vars qw($VERSION); -$VERSION = "1.02"; - -my $EOL = "\015\012"; -my $BLANK = $EOL x 2; -my $PROTOVERSION = 'RSPAMC/1.2'; - -=head1 PUBLIC METHODS - -=head2 new - -public class (Mail::Rspamd::Client) new (\% $args) - -Description: -This method creates a new Mail::Rspamd::Client object. - -=cut - -sub new { - my ($class, $args) = @_; - - $class = ref($class) || $class; - - my $self = {}; - - # with a sockets_path set then it makes no sense to set host and port - if ($args->{hosts}) { - $self->{hosts} = $args->{hosts}; - $self->{alive_hosts} = $self->{hosts}; - } - - if ($args->{username}) { - $self->{username} = $args->{username}; - } - if ($args->{ip}) { - $self->{ip} = $args->{ip}; - } - if ($args->{from}) { - $self->{from} = $args->{from}; - } - if ($args->{subject}) { - $self->{subject} = $args->{subject}; - } - if ($args->{rcpt}) { - $self->{rcpt} = $args->{rcpt}; - } - if ($args->{deliver_to}) { - $self->{deliver_to} = $args->{deliver_to}; - } - if ($args->{timeout}) { - $self->{timeout} = $args->{timeout}; - } - else { - $self->{timeout} = 5; - } - if ($args->{password}) { - $self->{password} = $args->{password}; - } - if ($args->{statfile}) { - $self->{statfile} = $args->{statfile}; - } - if ($args->{weight}) { - $self->{weight} = $args->{weight}; - } - else { - $self->{weight} = 1; - } - if ($args->{pass_all}) { - $self->{pass_all} = 1; - } - if ($args->{imap_search}) { - $self->{imap_search} = $args->{imap_search}; - } - else { - $self->{imap_search} = 'ALL'; - } - - if ($args->{command}) { - if ($args->{command} =~ /(SYMBOLS|PROCESS|CHECK|URLS|EMAILS)/i) { - $self->{'command'} = $1; - $self->{'control'} = 0; - } - elsif ($args->{command} =~ /(STAT|LEARN|SHUTDOWN|RELOAD|UPTIME|COUNTERS|FUZZY_ADD|FUZZY_DEL|WEIGHTS)/i) { - $self->{'command'} = $1; - $self->{'control'} = 1; - } - } - - $self->{error} = ""; - - bless($self, $class); - - $self; -} - - -sub make_ssl_socket { - my ($host, $port) = @_; - - eval { - require IO::Socket::SSL; - IO::Socket::SSL->import(LIST); - } or croak "IO::Socket::SSL required for imaps"; - - return IO::Socket::SSL->new("$host:$port"); -} - - - -=head2 process_item - -public instance (\%) process_item (String $item) - -Description: -Do specified command for a single file, path or IMAP folder - -The return value is a hash reference containing results of each command for each server from cluster - -=cut - -sub process_item { - my $self = shift; - my $item = shift; - my $cb = shift; - - if (defined ($item)) { - if ($item =~ qr|^imap(s?):user:([^:]+):password:([^:]*):host:([^:]+):mbox:(.+)$|) { - return $self->_process_imap ($1, $2, $3, $4, $5, $cb); - } - elsif (-f $item) { - return $self->_process_file ($item, $cb); - } - elsif (-d $item) { - return $self->_process_directory ($item, $cb); - } - else { - warn "urecognized argument: $item"; - } - } - undef; -} - -=head2 process_path - -public instance (\%) process_path () - -Description: -Do specified command for each file in path or message in IMAP folder - -The return value is a hash reference containing results of each command for each server from cluster - -=cut -sub process_path { - my $self = shift; - my $cb = shift; - my %res; - - foreach (@_) { - $res{$_} = $self->process_item($_, $cb); - } - - return \%res; -} - -=head2 do_all_cmd - -public instance (\%) do_all_cmd (String $msg) - -Description: -This method makes a call to the the whole rspamd cluster and call specified command -(in $self->{command}). - -The return value is a hash reference containing results of each command for each server from cluster - -=cut - -sub do_all_cmd { - my ($self, $input) = @_; - - my %res; - - if (!$self->{'hosts'} || scalar (@{ $self->{'hosts'} }) == 0) { - $res{'error'} = 'Hosts list is empty'; - $res{'error_code'} = 404; - } - else { - foreach my $hostdef (@{ $self->{'hosts'} }) { - $self->_clear_errors(); - - my $remote = $self->_create_connection($hostdef); - - if (! $remote) { - $res{$hostdef}->{error_code} = 404; - $res{$hostdef}->{error} = "Cannot connect to $hostdef"; - } - else { - if ($self->{'control'}) { - $res{$hostdef} = $self->_do_control_command ($remote, $input); - } - else { - $res{$hostdef} = $self->_do_rspamc_command ($remote, $input); - } - } - } - } - - return \%res; -} - -=head2 do_cmd - -public instance (\%) do_cmd (String $msg) - -Description: -This method makes a call to a single rspamd server from a cluster -(in $self->{command}). - -The return value is a hash reference containing results of each command for each server from cluster - -=cut - -sub do_cmd { - my ($self, $input) = @_; - - my $res; - - if (!$self->{'hosts'} || scalar (@{ $self->{'hosts'} }) == 0) { - $res->{'error'} = 'Hosts list is empty'; - $res->{'error_code'} = 404; - } - else { - $self->_clear_errors(); - - my $remote = $self->_create_connection(); - - if (! $remote) { - $res->{error_code} = 404; - $res->{error} = "Cannot connect to " . $remote; - } - else { - if ($self->{'control'}) { - $res = $self->_do_control_command ($remote, $input); - } - else { - $res = $self->_do_rspamc_command ($remote, $input); - } - } - } - - return $res; -} - - -=head2 check - -public instance (\%) check (String $msg) - -Description: -This method makes a call to the spamd server and depending on the value of -C<$is_check_p> either calls PROCESS or CHECK. - -The return value is a hash reference containing metrics indexed by name. Each metric -is hash that contains data: - -=over -=item * -isspam - -=item * -score - -=item * -threshold - -=item * -symbols - array of symbols - -=back - -=cut - -sub check { - my ($self, $msg) = @_; - - $self->{command} = 'CHECK'; - $self->{control} = 0; - - return $self->do_cmd ($msg); -} - -=head2 symbols - -public instance (\%) symbols (String $msg) - -Description: -This method makes a call to the spamd server - -The return value is a hash reference containing metrics indexed by name. Each metric -is hash that contains data: - -=over -=item * -isspam - -=item * -score - -=item * -threshold - -=item * -symbols - array of symbols - -=back - -=cut - -sub symbols { - my ($self, $msg) = @_; - - $self->{command} = 'SYMBOLS'; - $self->{control} = 0; - - return $self->do_cmd ($msg); -} - -=head2 process - -public instance (\%) process (String $msg) - -Description: -This method makes a call to the spamd server - -The return value is a hash reference containing metrics indexed by name. Each metric -is hash that contains data: - -=over -=item * -isspam - -=item * -score - -=item * -threshold - -=item * -symbols - array of symbols - -=back - -=cut -sub process { - my ($self, $msg) = @_; - - $self->{command} = 'PROCESS'; - $self->{control} = 0; - - return $self->do_cmd ($msg); -} - -=head2 urls - -public instance (\%) urls (String $msg) - -Description: -This method makes a call to the spamd server - -The return value is a hash reference containing metrics indexed by name. Each metric -is hash that contains data: - -urls - list of all urls in message -=cut -sub urls { - my ($self, $msg) = @_; - - $self->{command} = 'URLS'; - $self->{control} = 0; - - return $self->do_cmd ($msg); -} - - -=head2 learn - -public instance (\%) learn (String $msg) - -Description: -This method makes a call to the spamd learning a statfile with message. - -=cut - -sub learn { - my ($self, $msg) = @_; - - $self->{command} = 'learn'; - $self->{control} = 1; - - return $self->do_cmd ($msg); -} - -=head2 weights - -public instance (\%) weights (String $msg) - -Description: -This method makes a call to the spamd showing weights of message by each statfile. - -=cut -sub weights { - my ($self, $msg) = @_; - - $self->{command} = 'weights'; - $self->{control} = 1; - - return $self->do_cmd ($msg); -} - -=head2 fuzzy_add - -public instance (\%) fuzzy_add (String $msg) - -Description: -This method makes a call to the spamd adding specified message to fuzzy storage. - -=cut -sub fuzzy_add { - my ($self, $msg) = @_; - - $self->{command} = 'fuzzy_add'; - $self->{control} = 1; - - return $self->do_cmd ($msg); -} -=head2 fuzzy_del - -public instance (\%) fuzzy_add (String $msg) - -Description: -This method makes a call to the spamd removing specified message from fuzzy storage. - -=cut -sub fuzzy_del { - my ($self, $msg) = @_; - - $self->{command} = 'fuzzy_del'; - $self->{control} = 1; - - return $self->do_cmd ($msg); -} - -=head2 stat - -public instance (\%) stat () - -Description: -This method makes a call to the spamd and get statistics. - -=cut -sub stat { - my ($self) = @_; - - $self->{command} = 'stat'; - $self->{control} = 1; - - return $self->do_cmd (undef); -} -=head2 uptime - -public instance (\%) uptime () - -Description: -This method makes a call to the spamd and get uptime. - -=cut -sub uptime { - my ($self) = @_; - - $self->{command} = 'uptime'; - $self->{control} = 1; - - return $self->do_cmd (undef); -} -=head2 counters - -public instance (\%) counters () - -Description: -This method makes a call to the spamd and get counters. - -=cut -sub counters { - my ($self) = @_; - - $self->{command} = 'counters'; - $self->{control} = 1; - - return $self->do_cmd (undef); -} - -=head2 ping - -public instance (Boolean) ping () - -Description: -This method performs a server ping and returns 0 or 1 depending on -if the server responded correctly. - -=cut - -sub ping { - my $self = shift; - my $host = shift; - - my $remote; - $self->{control} = 0; - if (defined($host)) { - $remote = $self->_create_connection($host); - } - else { - # Create connection to random host from cluster - $remote = $self->_create_connection(); - } - - return undef unless $remote; - local $SIG{PIPE} = 'IGNORE'; - - if (!(syswrite($remote, "PING $PROTOVERSION$EOL"))) { - $self->_mark_dead($remote); - close($remote); - return 0; - } - syswrite($remote, $EOL); - - return undef unless $self->_get_io_readiness($remote, 0); - my $line; - sysread ($remote, $line, 255); - close $remote; - return undef unless $line; - - my ($version, $resp_code, $resp_msg) = $self->_parse_response_line($line); - return 0 unless (defined($resp_msg) && $resp_msg eq 'PONG'); - - return 1; -} - -=head1 PRIVATE METHODS - -=head2 _connect_host -private instance (IO::Socket) _create_host ($def) - -Description: -This method sets up a proper IO::Socket connection based on the arguments -used when greating the client object. - -On failure, it sets an internal error code and returns undef. -=cut - -sub _connect_host { - my ($self, $hostdef) = @_; - - my $remote; - - if ($hostdef =~ /^\//) { - if (! socket ($remote, PF_UNIX, SOCK_STREAM, 0)) { - carp "Cannot create unix socket\n"; - return undef; - } - my $sun = sockaddr_un($hostdef); - if (!connect ($remote, $sun)) { - carp "Cannot connect to socket $hostdef\n"; - close $remote; - return undef; - } - } - elsif ($hostdef =~ /^\s*(([^:]+):(\d+))\s*$/) { - my $peer_addr = $2; - if ($2 eq '*') { - $peer_addr = '127.0.0.1'; - } - $remote = IO::Socket::INET->new( Proto => "tcp", - PeerAddr => $peer_addr, - PeerPort => $3, - Blocking => 0, - ); - # Get write readiness - if (defined ($remote)) { - if ($self->_get_io_readiness($remote, 1) != 0) { - return $remote; - } - else { - close ($remote); - return undef; - } - } - } - elsif ($hostdef =~ /^\s*([^:]+)\s*$/) { - my $peer_addr = $1; - if ($1 eq '*') { - $peer_addr = '127.0.0.1'; - } - $remote = IO::Socket::INET->new( Proto => "tcp", - PeerAddr => $peer_addr, - PeerPort => $self->{control} ? 11334 : 11333, - Blocking => 0, - ); - # Get write readiness - if (defined ($remote)) { - if ($self->_get_io_readiness($remote, 1) != 0) { - return $remote; - } - else { - close ($remote); - return undef; - } - } - } - - unless ($remote) { - $self->{error} = "Failed to create connection to spamd daemon: $!\n"; - return undef; - } - $remote; - -} - -=head2 _create_connection -private instance (IO::Socket) _create_connection () - -Description: -This method sets up a proper IO::Socket connection based on the arguments -used when greating the client object. - -On failure, it sets an internal error code and returns undef. - -=cut - -sub _create_connection { - my ($self, $hostdef) = @_; - - my $tries = 0; - - if (!defined ($hostdef)) { - my $server; - - do { - $server = $self->_select_server(); - $tries ++; - - my $remote = $self->_connect_host ($server); - - return $remote if $remote; - } while ($tries < 5); - - return undef; - } - - return $self->_connect_host ($hostdef); -} - -=head2 _auth - -private instance (IO::Socket) _auth (Socket sock) - -Description: -This method do control auth. - -On failure this method returns 0 - -=cut -sub _auth { - my ($self, $sock) = @_; - - local $SIG{PIPE} = 'IGNORE'; - - if (!(syswrite($sock, "password $self->{password}$EOL"))) { - $self->_mark_dead($remote); - return 0; - } - - return 0 unless $self->_get_io_readiness($sock, 0); - - if (sysread($sock, $reply, 255)) { - if ($reply =~ /^password accepted/) { - return 0 unless $self->_get_io_readiness($sock, 0); - # read "END" - sysread($sock, $reply, 255); - return 1; - } - } - - return 0; - -} - -=head2 _revive_dead - -private instance (IO::Socket) _revive_dead () - -Description: -This method marks dead upstreams as alive - -=cut -sub _revive_dead { - my ($self) = @_; - - my $now = time(); - foreach my $s ($self->{dead_hosts}) { - # revive after minute of downtime - if (defined($s->{dead}) && $s->{dead} == 1 && $now - $s->{t} > 60) { - $s->{dead} = 0; - push(@{$self->{alive_hosts}}, $s->{host}); - } - } - - 1; -} - -=head2 _select_server - -private instance (IO::Socket) _select_server () - -Description: -This method returns one server from rspamd cluster or undef if there are no suitable ones - -=cut -sub _select_server { - my($self) = @_; - - return undef unless $self->{alive_hosts}; - - $self->_revive_dead(); - my $alive_num = scalar(@{$self->{alive_hosts}}); - if (!$alive_num) { - $self->{alive_hosts} = $self->{hosts}; - $self->{dead_hosts} = (); - $alive_num = scalar($self->{alive_hosts}); - } - - my $selected = $self->{alive_hosts}[int(rand($alive_num))]; - - $selected; -} - - -=head2 _select_server - -private instance (IO::Socket) _mark_dead (String server) - -Description: -This method marks upstream as dead for some time. It can be revived by _revive_dead method - -=cut -sub _mark_dead { - my ($self, $server) = @_; - - return undef unless $self->{hosts}; - my $now = time(); - $self->{dead_hosts}->{$server} = { - host => $server, - dead => 1, - t => $now, - }; - for (my $i = 0; $i < scalar (@{$self->{alive_hosts}}); $i ++) { - if ($self->{alive_hosts} == $server) { - splice(@{$self->{alive_hosts}}, $i, 1); - last; - } - } -} - -=head2 _get_io_readiness - -private instance (IO::Socket) _mark_dead (String server) - -Description: -This method marks upstream as dead for some time. It can be revived by _revive_dead method - -=cut -sub _get_io_readiness { - my ($self, $sock, $is_write) = @_; - my $w = ''; - vec($w, fileno($sock), 1) = 1; - - if ($is_write) { - return select(undef, $w, undef, $self->{timeout}); - } - else { - return select($w, undef,undef, $self->{timeout}); - } - - undef; -} - -=head2 _parse_response_line - -private instance (@) _parse_response_line (String $line) - -Description: -This method parses the initial response line/header from the server -and returns its parts. - -We have this as a seperate method in case we ever decide to get fancy -with the response line. - -=cut - -sub _parse_response_line { - my ($self, $line) = @_; - - $line =~ s/\r?\n$//; - return split(/\s+/, $line, 3); -} - -sub _write_message { - my $self = shift; - my $remote = shift; - my $message = shift; - my $len = shift; - - my $written = 0; - - while ($written < $len) { - last unless ($self->_get_io_readiness($remote, 1)); - my $cur = syswrite $remote, $message, $len, $written; - - last if ($cur <= 0); - $written += $cur; - } - - return $written == $len; -} - -=head2 _clear_errors - -private instance () _clear_errors () - -Description: -This method clears out any current errors. - -=cut - -sub _clear_errors { - my ($self) = @_; - - $self->{resp_code} = undef; - $self->{resp_msg} = undef; - $self->{error} = undef; -} - -# Currently just read stdin for user's message and pass it to rspamd -sub _do_rspamc_command { - my ($self, $remote, $msg) = @_; - - my %metrics; - my ($in, $res); - - my $msgsize = length($msg); - - local $SIG{PIPE} = 'IGNORE'; - - if (!(syswrite($remote, "$self->{command} $PROTOVERSION$EOL"))) { - $self->_mark_dead($remote); - my %r = ( - error => 'cannot connect to rspamd', - error_code => 502, - ); - close($remote); - return \%r; - } - syswrite $remote, "Content-length: $msgsize$EOL"; - syswrite $remote, "User: $self->{username}$EOL" if (exists($self->{username})); - syswrite $remote, "From: $self->{from}$EOL" if (exists($self->{from})); - syswrite $remote, "IP: $self->{ip}$EOL" if (exists($self->{ip})); - syswrite $remote, "Deliver-To: $self->{deliver_to}$EOL" if (exists($self->{deliver_to})); - syswrite $remote, "Subject: $self->{subject}$EOL" if (exists($self->{subject})); - syswrite $remote, "Pass: all$EOL" if (exists($self->{pass_all}) && $self->{pass_all}); - if (ref $self->{rcpt} eq "ARRAY") { - foreach ($self->{rcpt}) { - syswrite $remote, "Rcpt: $_ $EOL"; - } - } - syswrite $remote, $EOL; - - if (! $self->_write_message($remote, $msg, $msgsize)) { - my %r = ( - error => 'error writing message to rspamd', - error_code => 502, - ); - close $remote; - return \%r; - } - - #syswrite $remote, $EOL; - - unless ($self->_get_io_readiness($remote, 0)) { - close $remote; - my %r = ( - error => 'timed out while waiting for reply', - error_code => 502, - ); - return \%r; - } - - my $offset = 0; - do { - $res = sysread($remote, $in, 512, $offset); - if (!defined ($res)) { - close $remote; - my %r = ( - error => 'IO error while reading data from socket: ' . $!, - error_code => 503, - ); - return \%r; - } - if ($res > 0 && $res < 512) { - $self->_get_io_readiness($remote, 0); - } - $offset += $res; - } while ($res > 0); - - my ($version, $resp_code, $resp_msg) = $self->_parse_response_line($in); - - $self->{resp_code} = $resp_code; - $self->{resp_msg} = $resp_msg; - - unless (defined($resp_code) && $resp_code == 0) { - close $remote; - my %r = ( - error => 'invalid reply', - error_code => 500, - ); - return \%r - } - - my $cur_metric; - my @lines = split (/^/, $in); - if (lc $self->{'command'} eq 'urls') { - $metrics{'default'} = { - isspam => 'false', - score => 0, - threshold => 0, - symbols => [], - urls => [], - messages => [], - action => 'reject', - }; - foreach my $line (@lines) { - if ($line =~ /^Urls: (.+)$/) { - @{ $metrics{'default'}->{'urls'} } = split /,\s+/, $1; - } - } - } - else { - foreach my $line (@lines) { - if ($line =~ m!Metric: (\S+); (\S+); (\S+) / (\S+) (/ (\S+))?!) { - $metrics{$1} = { - isspam => $2, - score => $3 + 0, - threshold => $4 + 0, - reject_score => $6, - symbols => [], - urls => [], - messages => [], - action => 'no action', - }; - $cur_metric = $1; - } - elsif ($line =~ /^Symbol: (\S+);\s*(.+?)\s*$/ && $cur_metric) { - # Line with parameters - my $symref = $metrics{$cur_metric}->{'symbols'}; - push(@$symref, "$1($2)"); - } - elsif ($line =~ /^Symbol: (\S+?)\s*$/ && $cur_metric) { - my $symref = $metrics{$cur_metric}->{'symbols'}; - push(@$symref, $1); - } - elsif ($line =~ /^Urls: (.+?)\s*$/ && $cur_metric) { - @{ $metrics{$cur_metric}->{'urls'} } = split /,\s+/, $1; - } - elsif ($line =~ /^Message: (.+?)\s*$/ && $cur_metric) { - my $symref = $metrics{$cur_metric}->{'messages'}; - push(@$symref, $1); - } - elsif ($line =~ /^Action: (.+?)\s*$/ && $cur_metric) { - $metrics{$cur_metric}->{'action'} = $1; - } - elsif ($line =~ /^${EOL}$/) { - last; - } - } - } - - close $remote; - - return \%metrics; - -} - - -sub _do_control_command { - my ($self, $remote, $msg) = @_; - - local $SIG{PIPE} = 'IGNORE'; - my %res = ( - error_code => 0, - error => '', - ); - - unless ($self->_get_io_readiness($remote, 0)) { - $res{error} = "Timeout while reading data from socket"; - $res{error_code} = 501; - close($remote); - return \%res; - } - - # Read greeting first - if (defined (my $greeting = <$remote>)) { - if ($greeting !~ /^Rspamd version/) { - $res{error} = "Not rspamd greeting line $greeting"; - $res{error_code} = 500; - close($remote); - return \%res; - } - } - - if ($self->{'command'} =~ /^learn$/i) { - if (!$self->{'statfile'}) { - $res{error} = "Statfile is not specified to learn command"; - $res{error_code} = 500; - close($remote); - return \%res; - } - - if ($self->_auth ($remote)) { - my $len = length ($msg); - syswrite $remote, "learn $self->{statfile} $len -m $self->{weight}" . $EOL; - if (! $self->_write_message($remote, $msg, length($msg))) { - $res{error} = 'error writing message to rspamd'; - $res{error_code} = 502; - close $remote; - return \%res; - } - unless ($self->_get_io_readiness($remote, 0)) { - $res{error} = "Timeout while reading data from socket"; - $res{error_code} = 501; - close($remote); - return \%res; - } - if (defined (my $reply = <$remote>)) { - if ($reply =~ /^learn ok, sum weight: ([0-9.]+)/) { - $res{error} = "Learn succeed. Sum weight: $1\n"; - close($remote); - return \%res; - } - else { - $res{error_code} = 500; - $res{error} = "Learn failed: $reply\n"; - close($remote); - return \%res; - } - } - } - else { - $res{error_code} = 403; - $res{error} = "Authentication failed\n"; - close($remote); - return \%res; - } - } - elsif ($self->{'command'} =~ /^weights$/i) { - if (!$self->{'statfile'}) { - $res{error_code} = 500; - $res{error} = "Statfile is not specified to weights command"; - close($remote); - return \%res; - } - - my $len = length ($msg); - $res{error} = "Sending $len bytes...\n"; - syswrite $remote, "weights $self->{'statfile'} $len" . $EOL; - if (! $self->_write_message($remote, $msg, length($msg))) { - $res{error} = 'error writing message to rspamd'; - $res{error_code} = 502; - close $remote; - return \%res; - } - unless ($self->_get_io_readiness($remote, 0)) { - $res{error} = "Timeout while reading data from socket"; - $res{error_code} = 501; - close($remote); - return \%res; - } - while (defined (my $reply = <$remote>)) { - last if $reply =~ /^END/; - $res{error} .= $reply; - } - } - elsif ($self->{'command'} =~ /(reload|shutdown)/i) { - if ($self->_auth ($remote)) { - syswrite $remote, $self->{'command'} . $EOL; - unless ($self->_get_io_readiness($remote, 0)) { - $res{error} = "Timeout while reading data from socket"; - $res{error_code} = 501; - close($remote); - return \%res; - } - while (defined (my $line = <$remote>)) { - last if $line =~ /^END/; - $res{error} .= $line; - } - } - else { - $res{error_code} = 403; - $res{error} = "Authentication failed\n"; - close($remote); - return \%res; - } - } - elsif ($self->{'command'} =~ /(fuzzy_add|fuzzy_del)/i) { - if ($self->_auth ($remote)) { - my $len = length ($msg); - syswrite $remote, $self->{'command'} . " $len $self->{'weight'}" . $EOL; - if (! $self->_write_message($remote, $msg, length($msg))) { - $res{error} = 'error writing message to rspamd'; - $res{error_code} = 502; - close $remote; - return \%res; - } - unless ($self->_get_io_readiness($remote, 0)) { - $res{error} = "Timeout while reading data from socket"; - $res{error_code} = 501; - close($remote); - return \%res; - } - if (defined (my $reply = <$remote>)) { - if ($reply =~ /^OK/) { - $res{error} = $self->{'command'} . " succeed\n"; - close($remote); - return \%res; - } - else { - $res{error_code} = 500; - $res{error} = $self->{'command'} . " failed\n"; - close($remote); - return \%res; - } - } - } - else { - $res{error_code} = 403; - $res{error} = "Authentication failed\n"; - close($remote); - return \%res; - } - - } - else { - syswrite $remote, $self->{'command'} . $EOL; - unless ($self->_get_io_readiness($remote, 0)) { - $res{error} = "Timeout while reading data from socket"; - $res{error_code} = 501; - close($remote); - return \%res; - } - while (defined (my $line = <$remote>)) { - last if $line =~ /^END/; - $res{error} .= $line; - } - } - - close($remote); - return \%res; -} - -sub _process_file { - my $self = shift; - my $file = shift; - my $cb = shift; - my $res; - - open(FILE, "< $file") or return; - - my $input; - while (defined (my $line = <FILE>)) { - $input .= $line; - } - - close FILE; - $res = $self->do_all_cmd ($input); - if (defined ($cb) && $res) { - $cb->($file, $res); - } -} - -sub _process_directory { - my $self = shift; - my $dir = shift; - my $cb = shift; - - opendir (DIR, $dir) or return; - - while (defined (my $file = readdir (DIR))) { - $file = "$dir/$file"; - if (-f $file) { - $self->_process_file ($file, $cb); - } - } - closedir (DIR); -} - -sub _check_imap_reply { - my $self = shift; - my $sock = shift; - my $seq = shift; - - my $input; - - while (defined ($input = <$sock>)) { - chomp $input; - if ($input =~ /BAD|NO (.+)$/) { - $_[0] = $1; - return 0; - } - next if ($input =~ /^\*/); - if ($input =~ /^$seq OK/) { - return 1; - } - - $_[0] = $input; - return 0; - } - - $_[0] = "timeout"; - - return 0; -} - -sub _parse_imap_body { - my $self = shift; - my $sock = shift; - my $seq = shift; - my $input; - my $got_body = 0; - - while (defined (my $line = <$sock>)) { - if (!$got_body && $line =~ /^\*/) { - $got_body = 1; - next; - } - if ($line =~ /^$seq OK/) { - return $input; - } - elsif ($got_body) { - $input .= $line; - next; - } - - return undef; - } - - return undef; - -} - -sub _parse_imap_sequences { - my $self = shift; - my $sock = shift; - my $seq = shift; - my $input; - - while (defined ($input = <$sock>)) { - chomp $input; - if ($input =~ /^\* SEARCH (.+)$/) { - @res = split (/\s/, $1); - next; - } - elsif ($input =~ /^$seq OK/) { - return \@res; - } - return undef; - } - -} - -sub _process_imap { - my ($self, $ssl, $user, $password, $host, $mbox, $cb) = @_; - my $seq = 1; - my $sock; - my $res; - - if (!$password) { - eval { - require Term::ReadKey; - Term::ReadKey->import( qw(ReadMode ReadLine) ); - print "Enter IMAP password: "; - ReadMode(2); - $password = ReadLine(0); - chomp $password; - ReadMode(0); - print "\n"; - } or croak "cannot get password. Check that Term::ReadKey is installed"; - } - - # Stupid code that does not take care of timeouts etc, just trying to extract messages - if ($ssl) { - $sock = $self->_make_ssl_socket ($host, 'imaps'); - } - else { - $sock = IO::Socket::INET->new( Proto => "tcp", - PeerAddr => $host, - PeerPort => 'imap', - Blocking => 1, - ); - } - unless ($sock) { - $self->{error} = "Cannot connect to imap server: $!"; - return; - } - my $reply = <$sock>; - if (!defined ($reply) || $reply !~ /^\* OK/) { - $self->{error} = "Imap server is not ready"; - return; - } - syswrite $sock, "$seq LOGIN $user $password$EOL"; - if (!$self->_check_imap_reply ($sock, $seq, $reply)) { - $self->{error} = "Cannot login to imap server: $reply"; - return; - } - $seq ++; - syswrite $sock, "$seq SELECT $mbox$EOL"; - if (!$self->_check_imap_reply ($sock, $seq, $reply)) { - $self->{error} = "Cannot select mbox $mbox: $reply"; - return; - } - $seq ++; - syswrite $sock, "$seq SEARCH $self->{imap_search}$EOL"; - my $messages; - if (!defined ($messages = $self->_parse_imap_sequences ($sock, $seq))) { - $self->{error} = "Cannot make search"; - return; - } - $seq ++; - foreach my $message (@{ $messages }){ - syswrite $sock, "$seq FETCH $message body[]$EOL"; - if (defined (my $input = $self->_parse_imap_body ($sock, $seq))) { - $self->do_all_cmd ($input); - if (defined ($cb) && $res) { - $cb->($seq, $res); - } - } - $seq ++; - } - syswrite $sock, "$seq LOGOUT$EOL"; - close $sock; -} - -1; diff --git a/perl/lib/Mail/Rspamd/Config.pm b/perl/lib/Mail/Rspamd/Config.pm deleted file mode 100644 index 7f3d8fefd..000000000 --- a/perl/lib/Mail/Rspamd/Config.pm +++ /dev/null @@ -1,593 +0,0 @@ -=head1 NAME - -Mail::Rspamd::Config - Utilities for rspamd configuration - -=head1 SYNOPSIS - -=head1 DESCRIPTION - -Mail::Rspamd::Config is a module that provides a perl implementation for -configuring rspamd. - -=cut - -package Mail::Rspamd::Config; - -use Carp; -use XML::Parser; - -use vars qw($VERSION); -$VERSION = "1.02"; - -use constant PARSER_STATE_START => 0; -use constant PARSER_STATE_MAIN => 1; -use constant PARSER_STATE_WORKER => 2; -use constant PARSER_STATE_MODULE => 3; -use constant PARSER_STATE_CLASSIFIER => 4; -use constant PARSER_STATE_STATFILE => 5; -use constant PARSER_STATE_LOGGING => 6; -use constant PARSER_STATE_METRIC => 8; -use constant PARSER_STATE_VIEW => 9; -use constant PARSER_STATE_MODULES => 10; -use constant PARSER_STATE_END => -1; - - -=head1 PUBLIC METHODS - -=head2 new - -public class (Mail::Rspamd::Config) new (\% $args) - -Description: -This method creates a new Mail::Rspamd::Config object. - -=cut - -sub new { - my ($class, $args) = @_; - - $class = ref($class) || $class; - - my $self = { - workers => [], - modules => {}, - classifiers => {}, - metrics => {}, - options => {}, - variables => {}, - logging => {}, - lua => [], - composites => {}, - paths => [], - views => [], - parser_state => { - state => PARSER_STATE_START, - valid => 1, - }, - }; - - if (defined ($args->{'file'})) { - $self->{'file'} = $args->{'file'} - } - - - bless($self, $class); - - $self; -} - -=head2 load - -public load (String $file) - -Description: -Loads rspamd config file and parses it. - -=cut - -sub load { - my ($self, $file) = @_; - - if (defined ($file)) { - $self->{'file'} = $file; - } - - if (!defined ($self->{'file'}) || ! -f $self->{'file'}) { - carp 'cannot open file specified'; - return undef; - } - - my $parser = new XML::Parser(Handlers => {Start => sub { $self->_handle_start_element(@_) }, - End => sub { $self->_handle_end_element(@_) }, - Char => sub { $self->_handle_text(@_) } }); - - $parser->parsefile($self->{file}); -} - -=head2 save - -public save (String $file) - -Description: -Dumps rspamd config to xml file. - -=cut - -sub save { - my ($self, $file) = @_; - - if (defined ($file)) { - $self->{'file'} = $file; - } - - if (!defined ($self->{'file'})) { - carp 'cannot open file specified'; - return undef; - } - - $self->_dump(); -} - -=head2 _handle_start_element - -private _handle_start_element($parser, $element, [attr, value...]) - -Description: -Handle start xml tag of rspamd - -=cut -sub _handle_start_element { - my ($self, $parser, $element, @attrs) = @_; - - - if ($self->{parser_state}->{valid}) { - # Start element - $self->{parser_state}->{element} = lc $element; - - if ($self->{parser_state}->{state} == PARSER_STATE_START) { - if (lc $element eq 'rspamd') { - $self->{parser_state}->{state} = PARSER_STATE_MAIN; - } - else { - $self->{parser_state}->{valid} = 0; - $self->{error} = 'Start element missing, it must be <rspamd>, but is <' . $element . '>'; - } - } - # Main section - elsif ($self->{parser_state}->{state} == PARSER_STATE_MAIN) { - my $lce = lc $element; - if ($lce eq 'logging') { - $self->{parser_state}->{state} = PARSER_STATE_LOGGING; - } - elsif ($lce eq 'worker') { - $self->{parser_state}->{state} = PARSER_STATE_WORKER; - $self->{parser_state}->{worker} = { options => {} }; - } - elsif ($lce eq 'view') { - $self->{parser_state}->{state} = PARSER_STATE_VIEW; - $self->{parser_state}->{view} = {}; - } - elsif ($lce eq 'metric') { - $self->{parser_state}->{state} = PARSER_STATE_METRIC; - $self->{parser_state}->{metric} = { symbols => {} }; - } - elsif ($lce eq 'module') { - $self->{parser_state}->{state} = PARSER_STATE_MODULE; - $self->_get_attr('name', 'name', 1, @attrs); - $self->{parser_state}->{module} = {}; - } - elsif ($lce eq 'classifier') { - $self->{parser_state}->{state} = PARSER_STATE_CLASSIFIER; - $self->_get_attr('type', 'type', 1, @attrs); - $self->{parser_state}->{classifier} = { statfiles => []}; - } - elsif ($lce eq 'variable') { - $self->_get_attr('name', 'name', 1, @attrs); - } - elsif ($lce eq 'lua') { - $self->_get_attr('src', 'src', 1, @attrs); - } - elsif ($lce eq 'composite') { - $self->_get_attr('name', 'name', 1, @attrs); - } - elsif ($lce eq 'modules') { - $self->{parser_state}->{state} = PARSER_STATE_MODULES; - } - else { - # Other element - $self->{parser_state}->{element} = $lce; - } - } - elsif ($self->{parser_state}->{state} == PARSER_STATE_MODULE) { - my $lce = lc $element; - if ($lce eq 'option') { - $self->_get_attr('name', 'option', 1, @attrs); - } - else { - $self->{parser_state}->{valid} = 0; - $self->{error} = 'Invalid tag <' . $lce . '> in module section'; - } - } - elsif ($self->{parser_state}->{state} == PARSER_STATE_METRIC) { - my $lce = lc $element; - if ($lce eq 'symbol') { - $self->_get_attr('weight', 'weight', 1, @attrs); - } - } - elsif ($self->{parser_state}->{state} == PARSER_STATE_CLASSIFIER) { - my $lce = lc $element; - if ($lce eq 'statfile') { - $self->{parser_state}->{state} = PARSER_STATE_STATFILE; - } - } - elsif ($self->{parser_state}->{state} == PARSER_STATE_WORKER) { - my $lce = lc $element; - if ($lce eq 'param') { - $self->_get_attr('name', 'name', 1, @attrs); - } - } - elsif ($self->{parser_state}->{state} == PARSER_STATE_END) { - # Tags after end element - $self->{parser_state}->{valid} = 0; - $self->{error} = 'Invalid tag <' . $element . '> after end tag'; - } - else { - # On other states just set element - } - } -} - - -=head2 _handle_end_element - -private _handle_end_element($parser, $element) - -Description: -Handle end xml tag of rspamd - -=cut -sub _handle_end_element { - my ($self, $parser, $element) = @_; - - if ($self->{parser_state}->{valid}) { - my $lce = lc $element; - if ($self->{parser_state}->{state} == PARSER_STATE_MAIN) { - if ($lce eq 'rspamd') { - $self->{parser_state}->{state} = PARSER_STATE_END; - } - } - elsif ($self->{parser_state}->{state} == PARSER_STATE_WORKER) { - if ($lce eq 'worker') { - push(@{$self->{workers}}, $self->{parser_state}->{worker}); - $self->{parser_state}->{state} = PARSER_STATE_MAIN; - $self->{parser_state}->{worker} = undef; - } - } - elsif ($self->{parser_state}->{state} == PARSER_STATE_CLASSIFIER) { - if ($lce eq 'classifier') { - $self->{classifiers}->{ $self->{parser_state}->{type} } = $self->{parser_state}->{classifier}; - $self->{parser_state}->{state} = PARSER_STATE_MAIN; - $self->{parser_state}->{classifier} = undef; - } - } - elsif ($self->{parser_state}->{state} == PARSER_STATE_METRIC) { - if ($lce eq 'metric') { - if (exists ($self->{parser_state}->{metric}->{name})) { - $self->{metrics}->{ $self->{parser_state}->{metric}->{name} } = $self->{parser_state}->{metric}; - $self->{parser_state}->{state} = PARSER_STATE_MAIN; - $self->{parser_state}->{metric} = undef; - } - else { - $self->{parser_state}->{valid} = 0; - $self->{error} = 'Metric must have <name> tag'; - } - } - } - elsif ($self->{parser_state}->{state} == PARSER_STATE_STATFILE) { - if ($lce eq 'statfile') { - push(@{$self->{parser_state}->{classifier}->{statfiles}}, $self->{parser_state}->{statfile}); - $self->{parser_state}->{state} = PARSER_STATE_CLASSIFIER; - $self->{parser_state}->{statfile} = undef; - } - } - elsif ($self->{parser_state}->{state} == PARSER_STATE_MODULE) { - if ($lce eq 'module') { - $self->{modules}->{ $self->{parser_state}->{name} } = $self->{parser_state}->{module}; - $self->{parser_state}->{state} = PARSER_STATE_MAIN; - $self->{parser_state}->{module} = undef; - } - } - elsif ($self->{parser_state}->{state} == PARSER_STATE_LOGGING) { - if ($lce eq 'logging') { - $self->{parser_state}->{state} = PARSER_STATE_MAIN; - } - } - elsif ($self->{parser_state}->{state} == PARSER_STATE_VIEW) { - if ($lce eq 'view') { - push(@{$self->{views}}, $self->{parser_state}->{view}); - $self->{parser_state}->{state} = PARSER_STATE_MAIN; - $self->{parser_state}->{view} = undef; - } - } - elsif ($self->{parser_state}->{state} == PARSER_STATE_MODULES) { - if ($lce eq 'modules') { - $self->{parser_state}->{state} = PARSER_STATE_MAIN; - } - } - } -} - -=head2 _handle_text - -private _handle_text($parser, $string) - -Description: -Handle data of xml tag - -=cut -sub _handle_text { - my ($self, $parser, $string) = @_; - - my $data; - - if (defined ($string) && $string =~ /^\s*(\S*(?:\s+\S+)*)\s*$/) { - $data = $1; - } - else { - return undef; - } - if (!$data) { - return undef; - } - - if ($self->{parser_state}->{valid}) { - if ($self->{parser_state}->{state} == PARSER_STATE_MAIN) { - if ($self->{parser_state}->{element} eq 'variable') { - $self->{variables}->{ $self->{parser_state}->{name} } = $data; - } - elsif ($self->{parser_state}->{element} eq 'composite') { - $self->{composites}->{ $self->{parser_state}->{name} } = $data; - } - elsif ($self->{parser_state}->{element} eq 'lua') { - push(@{$self->{lua}}, $self->{parser_state}->{src}); - } - else { - $self->{options}->{ $self->{parser_state}->{element} } = $data; - } - } - elsif ($self->{parser_state}->{state} == PARSER_STATE_LOGGING) { - $self->{logging}->{ $self->{parser_state}->{element} } = $data; - } - elsif ($self->{parser_state}->{state} == PARSER_STATE_WORKER) { - if ($self->{parser_state}->{element} eq 'param' || $self->{parser_state}->{element} eq 'option') { - $self->{parser_state}->{worker}->{options}->{$self->{parser_state}->{name}} = $data; - } - else { - $self->{parser_state}->{worker}->{ $self->{parser_state}->{element} } = $data; - } - } - elsif ($self->{parser_state}->{state} == PARSER_STATE_CLASSIFIER) { - $self->{parser_state}->{classifier}->{ $self->{parser_state}->{element} } = $data; - } - elsif ($self->{parser_state}->{state} == PARSER_STATE_STATFILE) { - $self->{parser_state}->{statfile}->{ $self->{parser_state}->{element} } = $data; - } - elsif ($self->{parser_state}->{state} == PARSER_STATE_MODULE) { - $self->{parser_state}->{module}->{ $self->{parser_state}->{option} } = $data; - } - elsif ($self->{parser_state}->{state} == PARSER_STATE_VIEW) { - $self->{parser_state}->{view}->{ $self->{parser_state}->{option} } = $data; - } - elsif ($self->{parser_state}->{state} == PARSER_STATE_MODULES) { - push(@{$self->{paths}}, $data); - } - elsif ($self->{parser_state}->{state} == PARSER_STATE_METRIC) { - if ($self->{parser_state}->{element} eq 'symbol') { - $self->{parser_state}->{metric}->{symbols}->{ $data } = $self->{parser_state}->{weight}; - } - else { - $self->{parser_state}->{metric}->{ $self->{parser_state}->{element} } = $data; - } - } - } -} - -=head2 _get_attr - -private _get_attr($name, $hash_name, $required, @attrs) - -Description: -Extract specified attr and put it to parser_state - -=cut -sub _get_attr { - my ($self, $name, $hash_name, $required, @attrs) = @_; - my $found = 0; - my $param = 1; - - foreach (@attrs) { - if ($found) { - $self->{parser_state}->{$hash_name} = $_; - last; - } - else { - if ($param) { - if (lc $_ eq $name) { - $found = 1; - } - $param = 0; - } - else { - $param = 1; - } - } - } - - if (!$found && $required) { - $self->{error} = "Attribute '$name' is required for tag '$self->{parser_state}->{element}'"; - $self->{parser_state}->{'valid'} = 0; - } -} - -=head2 _dump - -private _dump() - -Description: -Dumps rspamd config to xml file - -=cut -sub _dump { - my ($self) = @_; - - open(XML, "> $self->{file}") or carp "cannot open file '$self->file'"; - - print XML "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<rspamd>\n"; - - print XML "<!-- Main section -->\n"; - while(my ($k, $v) = each (%{$self->{options}})) { - my $ek = $self->_xml_escape($k); - print XML "<$ek>" . $self->_xml_escape($v) . "</$ek>\n"; - } - foreach my $lua(@{$self->{lua}}) { - print XML "<lua src=\"". $self->_xml_escape($lua) ."\">lua</lua>\n"; - } - print XML "<!-- End of main section -->\n\n"; - - print XML "<!-- Variables section -->\n"; - while(my ($k, $v) = each (%{$self->{variables}})) { - my $ek = $self->_xml_escape($k); - print XML "<variable name=\"$ek\">" . $self->_xml_escape($v) . "</variable>\n"; - } - print XML "<!-- End of variables section -->\n\n"; - - print XML "<!-- Composites section -->\n"; - while(my ($k, $v) = each (%{$self->{composites}})) { - my $ek = $self->_xml_escape($k); - print XML "<composite name=\"$ek\">" . $self->_xml_escape($v) . "</composite>\n"; - } - print XML "<!-- End of composites section -->\n\n"; - - print XML "<!-- Workers section -->\n"; - foreach my $worker (@{$self->{workers}}) { - print XML "<worker>\n"; - while (my ($k, $v) = each (%{$worker})) { - my $ek = $self->_xml_escape($k); - if ($k eq 'options') { - while (my ($kk, $vv) = each (%{$v})) { - print XML " <param name=\"". $self->_xml_escape($kk) ."\">" . $self->_xml_escape($vv) . "</param>\n"; - } - } - else { - print XML " <$ek>" . $self->_xml_escape($v) . "</$ek>\n"; - } - } - print XML "</worker>\n"; - } - print XML "<!-- End of workers section -->\n\n"; - - print XML "<!-- Metrics section -->\n"; - while (my ($k, $v) = each (%{$self->{metrics}})) { - print XML "<metric name=\"". $self->_xml_escape($k) ."\">\n"; - while (my ($kk, $vv) = each (%{ $v })) { - my $ek = $self->_xml_escape($kk); - if ($ek eq 'symbols') { - while (my ($sym, $weight) = each (%{ $vv })) { - print XML " <symbol weight=\"". $self->_xml_escape($weight) ."\">" . $self->_xml_escape($sym) . "</symbol>\n"; - } - } - else { - print XML " <$ek>" . $self->_xml_escape($vv) . "</$ek>\n"; - } - } - print XML "</metric>\n"; - } - print XML "<!-- End of metrics section -->\n\n"; - - print XML "<!-- Logging section -->\n<logging>\n"; - while (my ($k, $v) = each (%{$self->{logging}})) { - my $ek = $self->_xml_escape($k); - print XML " <$ek>" . $self->_xml_escape($v) . "</$ek>\n"; - } - print XML "</logging>\n<!-- End of logging section -->\n\n"; - - print XML "<!-- Classifiers section -->\n"; - while (my ($type, $classifier) = each(%{$self->{classifiers}})) { - print XML "<classifier type=\"". $self->_xml_escape($type) ."\">\n"; - while (my ($k, $v) = each (%{$classifier})) { - my $ek = $self->_xml_escape($k); - if ($k eq 'statfiles') { - foreach my $statfile (@{$v}) { - print XML " <statfile>\n"; - while (my ($kk, $vv) = each (%{$statfile})) { - my $ekk = $self->_xml_escape($kk); - print XML " <$ekk>" . $self->_xml_escape($vv) . "</$ekk>\n"; - } - print XML " </statfile>\n"; - } - } - else { - print XML " <$ek>" . $self->_xml_escape($v) . "</$ek>\n"; - } - } - print XML "</classifier>\n"; - } - print XML "<!-- End of classifiers section -->\n\n"; - - print XML "<!-- Modules section -->\n"; - while (my ($name, $module) = each(%{$self->{modules}})) { - print XML "<module name=\"". $self->_xml_escape($name) ."\">\n"; - while (my ($k, $v) = each (%{$module})) { - my $ek = $self->_xml_escape($k); - print XML " <option name=\"$ek\">" . $self->_xml_escape($v) . "</option>\n"; - } - print XML "</module>\n"; - } - print XML "<!-- End of modules section -->\n\n"; - - print XML "<!-- Paths section -->\n<modules>\n"; - foreach my $module(@{$self->{paths}}) { - print XML " <module>" . $self->_xml_escape($module) . "</module>\n"; - } - print XML "</modules>\n<!-- End of paths section -->\n\n"; - - print XML "</rspamd>\n"; -} - -=head2 _xml_escape - -private _xml_escape() - -Description: -Escapes characters in xml string - -=cut -sub _xml_escape { - my $data = $_[1]; - if ($data =~ /[\&\<\>\"]/) { - $data =~ s/\&/\&\;/g; - $data =~ s/\</\<\;/g; - $data =~ s/\>/\>\;/g; - $data =~ s/\"/\"\;/g; - } - return $data; -} - -=head2 _xml_unescape - -private _xml_unescape() - -Description: -Unescapes characters in xml string - -=cut -sub _xml_unescape { - my $data = $_[1]; - if ($data =~ /\&|\<|\>|\"/) { - $data =~ s/\&/\&/g; - $data =~ s/\<\;/\</g; - $data =~ s/\>\;/\>/g; - $data =~ s/\"\;/\"/g; - } - return $data; -} diff --git a/src/buffer.c b/src/buffer.c index 59dd55d3e..73bb6e654 100644 --- a/src/buffer.c +++ b/src/buffer.c @@ -690,9 +690,9 @@ rspamd_dispatcher_write (rspamd_io_dispatcher_t * d, } gboolean rspamd_dispatcher_write_string (rspamd_io_dispatcher_t *d, - GString *str, - gboolean delayed, - gboolean free_on_write) + GString *str, + gboolean delayed, + gboolean free_on_write) { struct rspamd_out_buffer_s *newbuf; diff --git a/src/cfg_file.h b/src/cfg_file.h index d53829675..41c37bd34 100644 --- a/src/cfg_file.h +++ b/src/cfg_file.h @@ -36,9 +36,6 @@ #define DEFAULT_SCORE 10.0 #define DEFAULT_REJECT_SCORE 999.0 -#define yyerror parse_err -#define yywarn parse_warn - struct expression; struct tokenizer; struct classifier; @@ -379,6 +376,7 @@ struct config_file { guint32 dns_retransmits; /**< maximum retransmits count */ guint32 dns_throttling_errors; /**< maximum errors for starting resolver throttling */ guint32 dns_throttling_time; /**< time in seconds for DNS throttling */ + guint32 dns_io_per_server; /**< number of sockets per DNS server */ GList *nameservers; /**< list of nameservers or NULL to parse resolv.conf */ }; @@ -415,7 +413,7 @@ gboolean parse_host_priority (memory_pool_t *pool, const gchar *str, gchar **add * @param type type of credits * @return 1 if line was successfully parsed and 0 in case of error */ -gboolean parse_bind_line (struct config_file *cfg, struct worker_conf *cf, gchar *str); +gboolean parse_bind_line (struct config_file *cfg, struct worker_conf *cf, const gchar *str); /** * Init default values @@ -502,7 +500,9 @@ gboolean parse_normalizer (struct config_file *cfg, struct statfile *st, const g /* * Read XML configuration file */ -gboolean read_rspamd_config (struct config_file *cfg, const gchar *filename, const gchar *convert_to); +gboolean read_rspamd_config (struct config_file *cfg, + const gchar *filename, const gchar *convert_to, + rspamd_rcl_section_fin_t logger_fin, gpointer logger_ud); /* * Register symbols of classifiers inside metrics diff --git a/src/cfg_rcl.c b/src/cfg_rcl.c index b27dedbdd..e8cb66800 100644 --- a/src/cfg_rcl.c +++ b/src/cfg_rcl.c @@ -437,7 +437,6 @@ rspamd_rcl_worker_handler (struct config_file *cfg, ucl_object_t *obj, const gchar *worker_type, *worker_bind; GQuark qtype; struct worker_conf *wrk; - struct rspamd_worker_bind_conf *bcf; struct rspamd_worker_cfg_parser *wparser; struct rspamd_worker_param_parser *whandler; @@ -475,13 +474,10 @@ rspamd_rcl_worker_handler (struct config_file *cfg, ucl_object_t *obj, if (!ucl_object_tostring_safe (cur, &worker_bind)) { continue; } - bcf = memory_pool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_worker_bind_conf)); - if (!parse_host_port_priority (cfg->cfg_pool, worker_bind, &bcf->bind_host, - &bcf->bind_port, NULL)) { + if (!parse_bind_line (cfg, wrk, worker_bind)) { g_set_error (err, CFG_RCL_ERROR, EINVAL, "cannot parse bind line: %s", worker_bind); return FALSE; } - LL_PREPEND (wrk->bind_conf, bcf); } } @@ -1037,6 +1033,8 @@ rspamd_rcl_config_init (void) G_STRUCT_OFFSET (struct config_file, dns_timeout), RSPAMD_CL_FLAG_TIME_INTEGER); rspamd_rcl_add_default_handler (sub, "dns_retransmits", rspamd_rcl_parse_struct_integer, G_STRUCT_OFFSET (struct config_file, dns_retransmits), RSPAMD_CL_FLAG_INT_32); + rspamd_rcl_add_default_handler (sub, "dns_sockets", rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct config_file, dns_io_per_server), RSPAMD_CL_FLAG_INT_32); rspamd_rcl_add_default_handler (sub, "raw_mode", rspamd_rcl_parse_struct_boolean, G_STRUCT_OFFSET (struct config_file, raw_mode), 0); rspamd_rcl_add_default_handler (sub, "one_shot", rspamd_rcl_parse_struct_boolean, @@ -1198,6 +1196,9 @@ rspamd_read_rcl_config (struct rspamd_rcl_section *top, } } } + if (cur->fin) { + cur->fin (cfg, cur->fin_ud); + } } cfg->rcl_obj = obj; diff --git a/src/cfg_rcl.h b/src/cfg_rcl.h index 272272ab4..39ce2fc43 100644 --- a/src/cfg_rcl.h +++ b/src/cfg_rcl.h @@ -66,6 +66,13 @@ struct rspamd_rcl_struct_parser { typedef gboolean (*rspamd_rcl_handler_t) (struct config_file *cfg, ucl_object_t *obj, gpointer ud, struct rspamd_rcl_section *section, GError **err); +/** + * A handler type that is called at the end of section parsing + * @param cfg configuration + * @param ud user data + */ +typedef void (*rspamd_rcl_section_fin_t)(struct config_file *cfg, gpointer ud); + struct rspamd_rcl_default_handler_data { struct rspamd_rcl_struct_parser pd; const gchar *key; @@ -82,6 +89,8 @@ struct rspamd_rcl_section { UT_hash_handle hh; /** hash handle */ struct rspamd_rcl_section *subsections; /**< hash table of subsections */ struct rspamd_rcl_default_handler_data *default_parser; /**< generic parsing fields */ + rspamd_rcl_section_fin_t fin; /** called at the end of section parsing */ + gpointer fin_ud; }; /** diff --git a/src/cfg_utils.c b/src/cfg_utils.c index 765ff7f34..8e57ea1c9 100644 --- a/src/cfg_utils.c +++ b/src/cfg_utils.c @@ -164,7 +164,7 @@ parse_host_priority (memory_pool_t *pool, const gchar *str, gchar **addr, guint } gboolean -parse_bind_line (struct config_file *cfg, struct worker_conf *cf, gchar *str) +parse_bind_line (struct config_file *cfg, struct worker_conf *cf, const gchar *str) { struct rspamd_worker_bind_conf *cnf; @@ -175,34 +175,6 @@ parse_bind_line (struct config_file *cfg, struct worker_conf *cf, gchar *str) cnf->bind_port = DEFAULT_BIND_PORT; if (str[0] == '/' || str[0] == '.') { -#ifdef HAVE_DIRNAME - /* Try to check path of bind credit */ - struct stat st; - gint fd; - gchar *copy = memory_pool_strdup (cfg->cfg_pool, str); - if (stat (copy, &st) == -1) { - if (errno == ENOENT) { - if ((fd = open (str, O_RDWR | O_TRUNC | O_CREAT, S_IWUSR | S_IRUSR)) == -1) { - msg_err ("cannot open path %s for making socket, %s", str, strerror (errno)); - return FALSE; - } - else { - close (fd); - unlink (str); - } - } - else { - msg_err ("cannot stat path %s for making socket, %s", str, strerror (errno)); - return 0; - } - } - else { - if (unlink (str) == -1) { - msg_err ("cannot remove path %s for making socket, %s", str, strerror (errno)); - return 0; - } - } -#endif cnf->bind_host = memory_pool_strdup (cfg->cfg_pool, str); cnf->is_unix = TRUE; LL_PREPEND (cf->bind_conf, cnf); @@ -233,6 +205,8 @@ init_defaults (struct config_file *cfg) /* After 20 errors do throttling for 10 seconds */ cfg->dns_throttling_errors = 20; cfg->dns_throttling_time = 10000; + /* 16 sockets per DNS server */ + cfg->dns_io_per_server = 16; cfg->statfile_sync_interval = 60000; cfg->statfile_sync_timeout = 20000; @@ -635,12 +609,14 @@ internal_normalizer_func (struct config_file *cfg, long double score, void *data } #ifdef HAVE_TANHL return max * tanhl (score / max); -#else +#elif defined(HAVE_TANHL) /* * As some implementations of libm does not support tanhl, try to use * tanh */ return max * tanh ((double) (score / max)); +#else + return score < max ? score / max : max; #endif } @@ -776,7 +752,9 @@ rspamd_ucl_add_conf_variables (struct ucl_parser *parser) } gboolean -read_rspamd_config (struct config_file *cfg, const gchar *filename, const gchar *convert_to) +read_rspamd_config (struct config_file *cfg, const gchar *filename, + const gchar *convert_to, rspamd_rcl_section_fin_t logger_fin, + gpointer logger_ud) { struct stat st; gint fd; @@ -784,7 +762,7 @@ read_rspamd_config (struct config_file *cfg, const gchar *filename, const gchar const gchar *ext; GMarkupParseContext *ctx; GError *err = NULL; - struct rspamd_rcl_section *top; + struct rspamd_rcl_section *top, *logger; gboolean res, is_xml = FALSE; struct rspamd_xml_userdata ud; struct ucl_parser *parser; @@ -866,6 +844,12 @@ read_rspamd_config (struct config_file *cfg, const gchar *filename, const gchar top = rspamd_rcl_config_init (); err = NULL; + HASH_FIND_STR(top, "logging", logger); + if (logger != NULL) { + logger->fin = logger_fin; + logger->fin_ud = logger_ud; + } + if (!rspamd_read_rcl_config (top, cfg, cfg->rcl_obj, &err)) { msg_err ("rcl parse error: %s", err->message); return FALSE; diff --git a/src/chacha_private.h b/src/chacha_private.h new file mode 100644 index 000000000..5034f4955 --- /dev/null +++ b/src/chacha_private.h @@ -0,0 +1,228 @@ +/* +chacha-merged.c version 20080118 +D. J. Bernstein +Public domain. +*/ + +#ifndef CHACHA_PRIVATE_H_ +#define CHACHA_PRIVATE_H_ + +#include <stddef.h> + +typedef unsigned char u8; +typedef unsigned int u32; + +typedef struct +{ + u32 input[16]; /* could be compressed */ +} chacha_ctx; + +#define U8C(v) (v##U) +#define U32C(v) (v##U) + +#define U8V(v) ((u8)(v) & U8C(0xFF)) +#define U32V(v) ((u32)(v) & U32C(0xFFFFFFFF)) + +#define ROTL32(v, n) \ + (U32V((v) << (n)) | ((v) >> (32 - (n)))) + +#define U8TO32_LITTLE(p) \ + (((u32)((p)[0]) ) | \ + ((u32)((p)[1]) << 8) | \ + ((u32)((p)[2]) << 16) | \ + ((u32)((p)[3]) << 24)) + +#define U32TO8_LITTLE(p, v) \ + do { \ + (p)[0] = U8V((v) ); \ + (p)[1] = U8V((v) >> 8); \ + (p)[2] = U8V((v) >> 16); \ + (p)[3] = U8V((v) >> 24); \ + } while (0) + +#define ROTATE(v,c) (ROTL32(v,c)) +#define XOR(v,w) ((v) ^ (w)) +#define PLUS(v,w) (U32V((v) + (w))) +#define PLUSONE(v) (PLUS((v),1)) + +#define QUARTERROUND(a,b,c,d) \ + a = PLUS(a,b); d = ROTATE(XOR(d,a),16); \ + c = PLUS(c,d); b = ROTATE(XOR(b,c),12); \ + a = PLUS(a,b); d = ROTATE(XOR(d,a), 8); \ + c = PLUS(c,d); b = ROTATE(XOR(b,c), 7); + +static const char sigma[16] = "expand 32-byte k"; +static const char tau[16] = "expand 16-byte k"; + +static void chacha_keysetup(chacha_ctx *x, const u8 *k, u32 kbits, u32 ivbits) +{ + const char *constants; + + x->input[4] = U8TO32_LITTLE(k + 0); + x->input[5] = U8TO32_LITTLE(k + 4); + x->input[6] = U8TO32_LITTLE(k + 8); + x->input[7] = U8TO32_LITTLE(k + 12); + if (kbits == 256) { /* recommended */ + k += 16; + constants = sigma; + } + else { /* kbits == 128 */ + constants = tau; + } + x->input[8] = U8TO32_LITTLE(k + 0); + x->input[9] = U8TO32_LITTLE(k + 4); + x->input[10] = U8TO32_LITTLE(k + 8); + x->input[11] = U8TO32_LITTLE(k + 12); + x->input[0] = U8TO32_LITTLE(constants + 0); + x->input[1] = U8TO32_LITTLE(constants + 4); + x->input[2] = U8TO32_LITTLE(constants + 8); + x->input[3] = U8TO32_LITTLE(constants + 12); +} + +static void chacha_ivsetup(chacha_ctx *x, const u8 *iv) +{ + x->input[12] = 0; + x->input[13] = 0; + x->input[14] = U8TO32_LITTLE(iv + 0); + x->input[15] = U8TO32_LITTLE(iv + 4); +} + +static void chacha_encrypt_bytes(chacha_ctx *x, const u8 *m, u8 *c, u32 bytes) +{ + u32 x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15; + u32 j0, j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15; + u8 *ctarget = NULL; + u8 tmp[64]; + unsigned int i; + + if (!bytes) + return; + + j0 = x->input[0]; + j1 = x->input[1]; + j2 = x->input[2]; + j3 = x->input[3]; + j4 = x->input[4]; + j5 = x->input[5]; + j6 = x->input[6]; + j7 = x->input[7]; + j8 = x->input[8]; + j9 = x->input[9]; + j10 = x->input[10]; + j11 = x->input[11]; + j12 = x->input[12]; + j13 = x->input[13]; + j14 = x->input[14]; + j15 = x->input[15]; + + for (;;) { + if (bytes < 64) { + for (i = 0; i < bytes; ++i) + tmp[i] = m[i]; + m = tmp; + ctarget = c; + c = tmp; + } + x0 = j0; + x1 = j1; + x2 = j2; + x3 = j3; + x4 = j4; + x5 = j5; + x6 = j6; + x7 = j7; + x8 = j8; + x9 = j9; + x10 = j10; + x11 = j11; + x12 = j12; + x13 = j13; + x14 = j14; + x15 = j15; + for (i = 20; i > 0; i -= 2) { + QUARTERROUND( x0, x4, x8, x12) + QUARTERROUND( x1, x5, x9, x13) + QUARTERROUND( x2, x6, x10, x14) + QUARTERROUND( x3, x7, x11, x15) + QUARTERROUND( x0, x5, x10, x15) + QUARTERROUND( x1, x6, x11, x12) + QUARTERROUND( x2, x7, x8, x13) + QUARTERROUND( x3, x4, x9, x14) + } + x0 = PLUS(x0,j0); + x1 = PLUS(x1,j1); + x2 = PLUS(x2,j2); + x3 = PLUS(x3,j3); + x4 = PLUS(x4,j4); + x5 = PLUS(x5,j5); + x6 = PLUS(x6,j6); + x7 = PLUS(x7,j7); + x8 = PLUS(x8,j8); + x9 = PLUS(x9,j9); + x10 = PLUS(x10,j10); + x11 = PLUS(x11,j11); + x12 = PLUS(x12,j12); + x13 = PLUS(x13,j13); + x14 = PLUS(x14,j14); + x15 = PLUS(x15,j15); + +#ifndef KEYSTREAM_ONLY + x0 = XOR(x0,U8TO32_LITTLE(m + 0)); + x1 = XOR(x1,U8TO32_LITTLE(m + 4)); + x2 = XOR(x2,U8TO32_LITTLE(m + 8)); + x3 = XOR(x3,U8TO32_LITTLE(m + 12)); + x4 = XOR(x4,U8TO32_LITTLE(m + 16)); + x5 = XOR(x5,U8TO32_LITTLE(m + 20)); + x6 = XOR(x6,U8TO32_LITTLE(m + 24)); + x7 = XOR(x7,U8TO32_LITTLE(m + 28)); + x8 = XOR(x8,U8TO32_LITTLE(m + 32)); + x9 = XOR(x9,U8TO32_LITTLE(m + 36)); + x10 = XOR(x10,U8TO32_LITTLE(m + 40)); + x11 = XOR(x11,U8TO32_LITTLE(m + 44)); + x12 = XOR(x12,U8TO32_LITTLE(m + 48)); + x13 = XOR(x13,U8TO32_LITTLE(m + 52)); + x14 = XOR(x14,U8TO32_LITTLE(m + 56)); + x15 = XOR(x15,U8TO32_LITTLE(m + 60)); +#endif + + j12 = PLUSONE(j12); + if (!j12) { + j13 = PLUSONE(j13); + /* stopping at 2^70 bytes per nonce is user's responsibility */ + } + + U32TO8_LITTLE(c + 0, x0); + U32TO8_LITTLE(c + 4, x1); + U32TO8_LITTLE(c + 8, x2); + U32TO8_LITTLE(c + 12, x3); + U32TO8_LITTLE(c + 16, x4); + U32TO8_LITTLE(c + 20, x5); + U32TO8_LITTLE(c + 24, x6); + U32TO8_LITTLE(c + 28, x7); + U32TO8_LITTLE(c + 32, x8); + U32TO8_LITTLE(c + 36, x9); + U32TO8_LITTLE(c + 40, x10); + U32TO8_LITTLE(c + 44, x11); + U32TO8_LITTLE(c + 48, x12); + U32TO8_LITTLE(c + 52, x13); + U32TO8_LITTLE(c + 56, x14); + U32TO8_LITTLE(c + 60, x15); + + if (bytes <= 64) { + if (bytes < 64) { + for (i = 0; i < bytes; ++i) + ctarget[i] = c[i]; + } + x->input[12] = j12; + x->input[13] = j13; + return; + } + bytes -= 64; + c += 64; +#ifndef KEYSTREAM_ONLY + m += 64; +#endif + } +} + +#endif /* CHACHA_PRIVATE_H_ */ diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index b8430db7f..1d48a202f 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -4,17 +4,18 @@ SET(RSPAMCSRC rspamc.c) ADD_EXECUTABLE(rspamc ${RSPAMCSRC}) SET_TARGET_PROPERTIES(rspamc PROPERTIES COMPILE_FLAGS "-I${CMAKE_SOURCE_DIR}/lib") TARGET_LINK_LIBRARIES(rspamc rspamd-util) -TARGET_LINK_LIBRARIES(rspamc rspamdclient) -TARGET_LINK_LIBRARIES(rspamc pcre) +IF(ENABLE_STATIC MATCHES "ON") + TARGET_LINK_LIBRARIES(rspamc rspamdclient_static) +ELSE(ENABLE_STATIC MATCHES "ON") + TARGET_LINK_LIBRARIES(rspamc rspamdclient) +ENDIF(ENABLE_STATIC MATCHES "ON") IF(GLIB_COMPAT) TARGET_LINK_LIBRARIES(rspamc glibadditions) ENDIF(GLIB_COMPAT) IF(OPENSSL_FOUND) TARGET_LINK_LIBRARIES(rspamc ${OPENSSL_LIBRARIES}) ENDIF(OPENSSL_FOUND) -TARGET_LINK_LIBRARIES(rspamc ${GLIB2_LIBRARIES}) -TARGET_LINK_LIBRARIES(rspamc ${CMAKE_REQUIRED_LIBRARIES}) -TARGET_LINK_LIBRARIES(rspamc m) +TARGET_LINK_LIBRARIES(rspamc ${RSPAMD_REQUIRED_LIBRARIES}) IF(NOT DEBIAN_BUILD) SET_TARGET_PROPERTIES(rspamc PROPERTIES VERSION ${RSPAMD_VERSION}) ENDIF(NOT DEBIAN_BUILD) diff --git a/src/client/rspamc.c b/src/client/rspamc.c index 9279682e4..3ed9f1dd4 100644 --- a/src/client/rspamc.c +++ b/src/client/rspamc.c @@ -46,6 +46,7 @@ static gint timeout = 5; static gboolean pass_all; static gboolean tty = FALSE; static gboolean verbose = FALSE; +static gboolean print_commands = FALSE; static struct rspamd_client *client = NULL; static GOptionEntry entries[] = @@ -64,6 +65,7 @@ static GOptionEntry entries[] = { "rcpt", 'r', 0, G_OPTION_ARG_STRING, &rcpt, "Emulate that message is for specified user", NULL }, { "timeout", 't', 0, G_OPTION_ARG_INT, &timeout, "Timeout for waiting for a reply", NULL }, { "bind", 'b', 0, G_OPTION_ARG_STRING, &local_addr, "Bind to specified ip address", NULL }, + { "commands", 0, 0, G_OPTION_ARG_NONE, &print_commands, "List available commands", NULL }, { NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL } }; @@ -82,6 +84,92 @@ enum rspamc_command { RSPAMC_COMMAND_ADD_ACTION }; +struct { + enum rspamc_command cmd; + const char *name; + const char *description; + gboolean is_controller; + gboolean is_privileged; +} rspamc_command_help[] = { + { + .cmd = RSPAMC_COMMAND_SYMBOLS, + .name = "symbols", + .description = "scan message and show symbols (default command)", + .is_controller = FALSE, + .is_privileged = FALSE + }, + { + .cmd = RSPAMC_COMMAND_LEARN_SPAM, + .name = "learn_spam", + .description = "learn message as spam", + .is_controller = TRUE, + .is_privileged = TRUE + }, + { + .cmd = RSPAMC_COMMAND_LEARN_HAM, + .name = "learn_ham", + .description = "learn message as ham", + .is_controller = TRUE, + .is_privileged = TRUE + }, + { + .cmd = RSPAMC_COMMAND_FUZZY_ADD, + .name = "fuzzy_add", + .description = "add message to fuzzy storage (check -f and -w options for this command)", + .is_controller = TRUE, + .is_privileged = TRUE + }, + { + .cmd = RSPAMC_COMMAND_FUZZY_DEL, + .name = "fuzzy_del", + .description = "delete message from fuzzy storage (check -f option for this command)", + .is_controller = TRUE, + .is_privileged = TRUE + }, + { + .cmd = RSPAMC_COMMAND_STAT, + .name = "stat", + .description = "show rspamd statistics", + .is_controller = TRUE, + .is_privileged = FALSE + }, + { + .cmd = RSPAMC_COMMAND_STAT_RESET, + .name = "stat_reset", + .description = "show and reset rspamd statistics (useful for graphs)", + .is_controller = TRUE, + .is_privileged = TRUE + }, + { + .cmd = RSPAMC_COMMAND_COUNTERS, + .name = "counters", + .description = "display rspamd symbols statistics", + .is_controller = TRUE, + .is_privileged = FALSE + }, + { + .cmd = RSPAMC_COMMAND_UPTIME, + .name = "uptime", + .description = "show rspamd uptime", + .is_controller = TRUE, + .is_privileged = FALSE + }, + { + .cmd = RSPAMC_COMMAND_ADD_SYMBOL, + .name = "add_symbol", + .description = "add or modify symbol settings in rspamd", + .is_controller = TRUE, + .is_privileged = TRUE + }, + { + .cmd = RSPAMC_COMMAND_ADD_ACTION, + .name = "add_action", + .description = "add or modify action settings", + .is_controller = TRUE, + .is_privileged = TRUE + } +}; + /* * Parse command line */ @@ -151,6 +239,30 @@ check_rspamc_command (const gchar *cmd) return RSPAMC_COMMAND_UNKNOWN; } +static void +print_commands_list (void) +{ + guint i; + + PRINT_FUNC ("Rspamc commands summary:\n"); + for (i = 0; i < G_N_ELEMENTS (rspamc_command_help); i ++) { + if (tty) { + PRINT_FUNC (" \033[1m%10s\033[0m (%7s%1s)\t%s\n", rspamc_command_help[i].name, + rspamc_command_help[i].is_controller ? "control" : "normal", + rspamc_command_help[i].is_privileged ? "*" : "", + rspamc_command_help[i].description); + } + else { + PRINT_FUNC (" %10s (%7s%1s)\t%s\n", rspamc_command_help[i].name, + rspamc_command_help[i].is_controller ? "control" : "normal", + rspamc_command_help[i].is_privileged ? "*" : "", + rspamc_command_help[i].description); + } + } + PRINT_FUNC ("\n* is for privileged commands that may need password (see -P option)\n"); + PRINT_FUNC ("control commands use port 11334 while normal use 11333 by default (see -h option)\n"); +} + /* * Parse connect_str and add server to librspamdclient */ @@ -798,6 +910,13 @@ main (gint argc, gchar **argv, gchar **env) read_cmd_line (&argc, &argv); + tty = isatty (STDOUT_FILENO); + + if (print_commands) { + print_commands_list (); + exit (EXIT_SUCCESS); + } + if (local_addr) { if (inet_aton (local_addr, &ina) != 0) { client = rspamd_client_init_binded (&ina); @@ -812,7 +931,6 @@ main (gint argc, gchar **argv, gchar **env) } rspamd_set_timeout (client, 1000, timeout * 1000); - tty = isatty (STDOUT_FILENO); /* Now read other args from argc and argv */ if (argc == 1) { /* No args, just read stdin */ diff --git a/src/controller.c b/src/controller.c index d59cc2972..d3f0f2855 100644 --- a/src/controller.c +++ b/src/controller.c @@ -221,7 +221,8 @@ free_session (void *ud) } static gboolean -restful_write_reply (gint error_code, const gchar *err_message, const gchar *buf, gsize buflen, rspamd_io_dispatcher_t *d) +restful_write_reply (gint error_code, const gchar *err_message, + const gchar *buf, gsize buflen, rspamd_io_dispatcher_t *d) { static gchar hbuf[256]; gint r; @@ -249,6 +250,36 @@ restful_write_reply (gint error_code, const gchar *err_message, const gchar *buf return TRUE; } +static gboolean +restful_write_reply_string (gint error_code, const gchar *err_message, + GString *buf, rspamd_io_dispatcher_t *d) +{ + static gchar hbuf[256]; + gint r; + + r = rspamd_snprintf (hbuf, sizeof (hbuf), + "HTTP/1.0 %d %s" CRLF "Version: " RVERSION CRLF, + error_code, err_message ? err_message : "OK"); + if (buf->len > 0) { + r += rspamd_snprintf (hbuf + r, sizeof (hbuf) - r, "Content-Length: %z" CRLF, buf->len); + } + r += rspamd_snprintf (hbuf + r, sizeof (hbuf) - r, CRLF); + + if (buf != NULL) { + if (!rspamd_dispatcher_write (d, hbuf, r, TRUE, TRUE)) { + return FALSE; + } + return rspamd_dispatcher_write_string (d, buf, FALSE, TRUE); + } + else { + if (!rspamd_dispatcher_write (d, hbuf, r, FALSE, TRUE)) { + return FALSE; + } + } + + return TRUE; +} + static gint check_auth (struct controller_command *cmd, struct controller_session *session) { @@ -428,19 +459,16 @@ process_sync_command (struct controller_session *session, gchar **args) static gboolean process_counters_command (struct controller_session *session) { - gchar out_buf[BUFSIZ]; GList *cur; struct cache_item *item; struct symbols_cache *cache; - gint r; + GString *out; cache = session->cfg->cache; + out = g_string_sized_new (BUFSIZ); if (!session->restful) { - r = rspamd_snprintf (out_buf, sizeof (out_buf), "Rspamd counters." CRLF); - } - else { - r = 0; + rspamd_printf_gstring (out, "Rspamd counters:" CRLF); } if (cache != NULL) { @@ -448,7 +476,7 @@ process_counters_command (struct controller_session *session) while (cur) { item = cur->data; if (!item->is_callback) { - r += rspamd_snprintf (out_buf + r, sizeof (out_buf) - r, "%s %.2f %d %.3f" CRLF, + rspamd_printf_gstring (out, "%s %.2f %d %.3f" CRLF, item->s->symbol, item->s->weight, item->s->frequency, item->s->avg_time); } @@ -458,7 +486,7 @@ process_counters_command (struct controller_session *session) while (cur) { item = cur->data; if (!item->is_callback) { - r += rspamd_snprintf (out_buf + r, sizeof (out_buf) - r, "%s %.2f %d %.3f" CRLF, + rspamd_printf_gstring (out, "%s %.2f %d %.3f" CRLF, item->s->symbol, item->s->weight, item->s->frequency, item->s->avg_time); } @@ -467,18 +495,18 @@ process_counters_command (struct controller_session *session) } if (!session->restful) { - return rspamd_dispatcher_write (session->dispatcher, out_buf, r, FALSE, FALSE); + return rspamd_dispatcher_write_string (session->dispatcher, out, FALSE, TRUE); } else { - return restful_write_reply (200, NULL, out_buf, r, session->dispatcher); + return restful_write_reply_string (200, NULL, out, session->dispatcher); } } static gboolean process_stat_command (struct controller_session *session, gboolean do_reset) { - gchar out_buf[BUFSIZ]; - gint r, i; + GString *out; + gint i; guint64 used, total, rev, ham = 0, spam = 0; time_t ti; memory_pool_stat_t mem_st; @@ -486,14 +514,16 @@ process_stat_command (struct controller_session *session, gboolean do_reset) stat_file_t *statfile; struct statfile *st; GList *cur_cl, *cur_st; - struct rspamd_stat *stat; + struct rspamd_stat *stat, stat_copy; memory_pool_stat (&mem_st); - stat = session->worker->srv->stat; - r = rspamd_snprintf (out_buf, sizeof (out_buf), "Messages scanned: %ud" CRLF, stat->messages_scanned); - if (session->worker->srv->stat->messages_scanned > 0) { + memcpy (&stat_copy, session->worker->srv->stat, sizeof (stat_copy)); + stat = &stat_copy; + out = g_string_sized_new (BUFSIZ); + rspamd_printf_gstring (out, "Messages scanned: %ud" CRLF, stat->messages_scanned); + if (stat->messages_scanned > 0) { for (i = METRIC_ACTION_REJECT; i <= METRIC_ACTION_NOACTION; i ++) { - r += rspamd_snprintf (out_buf + r, sizeof (out_buf) - r, "Messages with action %s: %ud, %.2f%%" CRLF, + rspamd_printf_gstring (out, "Messages with action %s: %ud, %.2f%%" CRLF, str_action_metric (i), stat->actions_stat[i], (double)stat->actions_stat[i] / (double)stat->messages_scanned * 100.); if (i < METRIC_ACTION_GREYLIST) { @@ -503,26 +533,26 @@ process_stat_command (struct controller_session *session, gboolean do_reset) ham += stat->actions_stat[i]; } if (do_reset) { - stat->actions_stat[i] = 0; + session->worker->srv->stat->actions_stat[i] = 0; } } - r += rspamd_snprintf (out_buf + r, sizeof (out_buf) - r, "Messages treated as spam: %ud, %.2f%%" CRLF, spam, + rspamd_printf_gstring (out, "Messages treated as spam: %ud, %.2f%%" CRLF, spam, (double)spam / (double)stat->messages_scanned * 100.); - r += rspamd_snprintf (out_buf + r, sizeof (out_buf) - r, "Messages treated as ham: %ud, %.2f%%" CRLF, ham, + rspamd_printf_gstring (out, "Messages treated as ham: %ud, %.2f%%" CRLF, ham, (double)ham / (double)stat->messages_scanned * 100.); } - r += rspamd_snprintf (out_buf + r, sizeof (out_buf) - r, "Messages learned: %ud" CRLF, stat->messages_learned); - r += rspamd_snprintf (out_buf + r, sizeof (out_buf) - r, "Connections count: %ud" CRLF, stat->connections_count); - r += rspamd_snprintf (out_buf + r, sizeof (out_buf) - r, "Control connections count: %ud" CRLF, stat->control_connections_count); - r += rspamd_snprintf (out_buf + r, sizeof (out_buf) - r, "Pools allocated: %z" CRLF, mem_st.pools_allocated); - r += rspamd_snprintf (out_buf + r, sizeof (out_buf) - r, "Pools freed: %z" CRLF, mem_st.pools_freed); - r += rspamd_snprintf (out_buf + r, sizeof (out_buf) - r, "Bytes allocated: %z" CRLF, mem_st.bytes_allocated); - r += rspamd_snprintf (out_buf + r, sizeof (out_buf) - r, "Memory chunks allocated: %z" CRLF, mem_st.chunks_allocated); - r += rspamd_snprintf (out_buf + r, sizeof (out_buf) - r, "Shared chunks allocated: %z" CRLF, mem_st.shared_chunks_allocated); - r += rspamd_snprintf (out_buf + r, sizeof (out_buf) - r, "Chunks freed: %z" CRLF, mem_st.chunks_freed); - r += rspamd_snprintf (out_buf + r, sizeof (out_buf) - r, "Oversized chunks: %z" CRLF, mem_st.oversized_chunks); - r += rspamd_snprintf (out_buf + r, sizeof (out_buf) - r, "Fuzzy hashes stored: %ud" CRLF, stat->fuzzy_hashes); - r += rspamd_snprintf (out_buf + r, sizeof (out_buf) - r, "Fuzzy hashes expired: %ud" CRLF, stat->fuzzy_hashes_expired); + rspamd_printf_gstring (out, "Messages learned: %ud" CRLF, stat->messages_learned); + rspamd_printf_gstring (out, "Connections count: %ud" CRLF, stat->connections_count); + rspamd_printf_gstring (out, "Control connections count: %ud" CRLF, stat->control_connections_count); + rspamd_printf_gstring (out, "Pools allocated: %z" CRLF, mem_st.pools_allocated); + rspamd_printf_gstring (out, "Pools freed: %z" CRLF, mem_st.pools_freed); + rspamd_printf_gstring (out, "Bytes allocated: %z" CRLF, mem_st.bytes_allocated); + rspamd_printf_gstring (out, "Memory chunks allocated: %z" CRLF, mem_st.chunks_allocated); + rspamd_printf_gstring (out, "Shared chunks allocated: %z" CRLF, mem_st.shared_chunks_allocated); + rspamd_printf_gstring (out, "Chunks freed: %z" CRLF, mem_st.chunks_freed); + rspamd_printf_gstring (out, "Oversized chunks: %z" CRLF, mem_st.oversized_chunks); + rspamd_printf_gstring (out, "Fuzzy hashes stored: %ud" CRLF, stat->fuzzy_hashes); + rspamd_printf_gstring (out, "Fuzzy hashes expired: %ud" CRLF, stat->fuzzy_hashes_expired); /* Now write statistics for each statfile */ cur_cl = g_list_first (session->cfg->classifiers); while (cur_cl) { @@ -539,14 +569,14 @@ process_stat_command (struct controller_session *session, gboolean do_reset) statfile_get_revision (statfile, &rev, &ti); if (total != (guint64)-1 && used != (guint64)-1) { if (st->label) { - r += rspamd_snprintf (out_buf + r, sizeof (out_buf) - r, + rspamd_printf_gstring (out, "Statfile: %s <%s> (version %uL); length: %Hz; free blocks: %uL; total blocks: %uL; free: %.2f%%" CRLF, st->symbol, st->label, rev, st->size, (total - used), total, (double)((double)(total - used) / (double)total) * 100.); } else { - r += rspamd_snprintf (out_buf + r, sizeof (out_buf) - r, + rspamd_printf_gstring (out, "Statfile: %s (version %uL); length: %Hz; free blocks: %uL; total blocks: %uL; free: %.2f%%" CRLF, st->symbol, rev, st->size, (total - used), total, @@ -560,17 +590,17 @@ process_stat_command (struct controller_session *session, gboolean do_reset) } if (do_reset) { - stat->messages_scanned = 0; - stat->messages_learned = 0; - stat->connections_count = 0; - stat->control_connections_count = 0; + session->worker->srv->stat->messages_scanned = 0; + session->worker->srv->stat->messages_learned = 0; + session->worker->srv->stat->connections_count = 0; + session->worker->srv->stat->control_connections_count = 0; } if (!session->restful) { - return rspamd_dispatcher_write (session->dispatcher, out_buf, r, FALSE, FALSE); + return rspamd_dispatcher_write_string (session->dispatcher, out, FALSE, TRUE); } else { - return restful_write_reply (200, NULL, out_buf, r, session->dispatcher); + return restful_write_reply_string (200, NULL, out, session->dispatcher); } } @@ -1604,10 +1634,12 @@ controller_read_socket (f_str_t * in, void *arg) session->state = STATE_REPLY; break; case STATE_OTHER: + rspamd_dispatcher_pause (session->dispatcher); if (session->other_handler) { - session->other_handler (session, in); + if (!session->other_handler (session, in)) { + return FALSE; + } } - rspamd_dispatcher_pause (session->dispatcher); break; case STATE_WAIT: rspamd_dispatcher_pause (session->dispatcher); @@ -1618,9 +1650,14 @@ controller_read_socket (f_str_t * in, void *arg) } if (session->state == STATE_REPLY || session->state == STATE_QUIT) { - rspamd_dispatcher_restore (session->dispatcher); + /* In case of normal session we restore read state, for restful session we need to terminate immediately */ + if (!session->restful) { + rspamd_dispatcher_restore (session->dispatcher); + } + else { + return FALSE; + } } - return TRUE; } diff --git a/src/dkim.c b/src/dkim.c index c65fb59fd..d864165ac 100644 --- a/src/dkim.c +++ b/src/dkim.c @@ -950,37 +950,51 @@ rspamd_dkim_simple_body_step (GChecksum *ck, const gchar **start, guint remain) static gboolean rspamd_dkim_canonize_body (rspamd_dkim_context_t *ctx, const gchar *start, const gchar *end) { + const gchar *p; + if (start == NULL) { /* Empty body */ - g_checksum_update (ctx->body_hash, CRLF, sizeof (CRLF) - 1); + if (ctx->body_canon_type == DKIM_CANON_SIMPLE) { + g_checksum_update (ctx->body_hash, CRLF, sizeof (CRLF) - 1); + } + else { + g_checksum_update (ctx->body_hash, "", 0); + } } else { - end --; - while (end > start + 2) { - if (*end == '\n' && *(end - 1) == '\r' && *(end - 2) == '\n') { - end -= 2; + /* Strip extra ending CRLF */ + p = end - 1; + while (p >= start + 2) { + if (*p == '\n' && *(p - 1) == '\r' && *(p - 2) == '\n') { + p -= 2; } - else if (*end == '\n' && *(end - 1) == '\n') { - end --; + else if (*p == '\n' && *(p - 1) == '\n') { + p --; } - else if (*end == '\r' && *(end - 1) == '\r') { - end --; + else if (*p == '\r' && *(p - 1) == '\r') { + p --; } else { break; } } + end = p + 1; if (end == start || end == start + 2) { /* Empty body */ - g_checksum_update (ctx->body_hash, CRLF, sizeof (CRLF) - 1); + if (ctx->body_canon_type == DKIM_CANON_SIMPLE) { + g_checksum_update (ctx->body_hash, CRLF, sizeof (CRLF) - 1); + } + else { + g_checksum_update (ctx->body_hash, "", 0); + } } else { if (ctx->body_canon_type == DKIM_CANON_SIMPLE) { /* Simple canonization */ - while (rspamd_dkim_simple_body_step (ctx->body_hash, &start, end - start + 1)); + while (rspamd_dkim_simple_body_step (ctx->body_hash, &start, end - start)); } else { - while (rspamd_dkim_relaxed_body_step (ctx->body_hash, &start, end - start + 1)); + while (rspamd_dkim_relaxed_body_step (ctx->body_hash, &start, end - start)); } } return TRUE; @@ -1,6 +1,5 @@ /* - * Copyright (c) 2009-2012, Vsevolod Stakhov - * Copyright (c) 2008, 2009, 2010 William Ahern + * Copyright (c) 2009-2013, Vsevolod Stakhov * * All rights reserved. * @@ -24,15 +23,11 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -/* - * Rspamd resolver library is based on code written by William Ahern. - * - * The original library can be found at: http://25thandclement.com/~william/projects/dns.c.html - */ - #include "config.h" #include "dns.h" #include "main.h" +#include "utlist.h" +#include "chacha_private.h" #ifdef HAVE_OPENSSL #include <openssl/rand.h> #endif @@ -52,19 +47,70 @@ static const unsigned initial_bias = 72; static const gint dns_port = 53; - -#ifdef HAVE_ARC4RANDOM -#define DNS_RANDOM arc4random -#elif defined HAVE_RANDOM -#define DNS_RANDOM random -#else -#define DNS_RANDOM rand -#endif - #define UDP_PACKET_SIZE 4096 #define DNS_COMPRESSION_BITS 0xC0 +static void dns_retransmit_handler (gint fd, short what, void *arg); + +/* + * DNS permutor utilities + */ + +#define PERMUTOR_BUF_SIZE 32768 +#define PERMUTOR_KSIZE 32 +#define PERMUTOR_IVSIZE 8 + +struct dns_permutor { + chacha_ctx ctx; + guchar perm_buf[PERMUTOR_BUF_SIZE]; + guint pos; +}; + +/** + * Init chacha20 context + * @param p + */ +static void +dns_permutor_init (struct dns_permutor *p) +{ + /* Init random key and IV */ + rspamd_random_bytes (p->perm_buf, sizeof (p->perm_buf)); + + /* Setup ctx */ + chacha_keysetup (&p->ctx, p->perm_buf, PERMUTOR_KSIZE * 8, 0); + chacha_ivsetup (&p->ctx, p->perm_buf + PERMUTOR_KSIZE); + + chacha_encrypt_bytes (&p->ctx, p->perm_buf, p->perm_buf, sizeof (p->perm_buf)); + + p->pos = 0; +} + +static struct dns_permutor * +dns_permutor_new (memory_pool_t *pool) +{ + struct dns_permutor *new; + + new = memory_pool_alloc0 (pool, sizeof (struct dns_permutor)); + dns_permutor_init (new); + + return new; +} + +static guint16 +dns_permutor_generate_id (struct dns_permutor *p) +{ + guint16 id; + if (p->pos + sizeof (guint16) >= sizeof (p->perm_buf)) { + dns_permutor_init (p); + } + + memcpy (&id, &p->perm_buf[p->pos], sizeof (guint16)); + p->pos += sizeof (guint16); + + return id; +} + /* Punycode utility */ static guint digit(unsigned n) { @@ -200,240 +246,6 @@ punycode_label_toascii(const gunichar *in, gsize in_len, gchar *out, return TRUE; } - -/* - * P E R M U T A T I O N G E N E R A T O R - */ - -#define DNS_K_TEA_BLOCK_SIZE 8 -#define DNS_K_TEA_CYCLES 32 -#define DNS_K_TEA_MAGIC 0x9E3779B9U - -static void dns_retransmit_handler (gint fd, short what, void *arg); - - -static void -dns_k_tea_init(struct dns_k_tea *tea, guint32 key[], guint cycles) -{ - memcpy(tea->key, key, sizeof tea->key); - - tea->cycles = (cycles)? cycles : DNS_K_TEA_CYCLES; -} /* dns_k_tea_init() */ - - -static void -dns_k_tea_encrypt (struct dns_k_tea *tea, guint32 v[], guint32 *w) -{ - guint32 y, z, sum, n; - - y = v[0]; - z = v[1]; - sum = 0; - - for (n = 0; n < tea->cycles; n++) { - sum += DNS_K_TEA_MAGIC; - y += ((z << 4) + tea->key[0]) ^ (z + sum) ^ ((z >> 5) + tea->key[1]); - z += ((y << 4) + tea->key[2]) ^ (y + sum) ^ ((y >> 5) + tea->key[3]); - } - - w[0] = y; - w[1] = z; - -} /* dns_k_tea_encrypt() */ - - -/* - * Permutation generator, based on a Luby-Rackoff Feistel construction. - * - * Specifically, this is a generic balanced Feistel block cipher using TEA - * (another block cipher) as the pseudo-random function, F. At best it's as - * strong as F (TEA), notwithstanding the seeding. F could be AES, SHA-1, or - * perhaps Bernstein's Salsa20 core; I am naively trying to keep things - * simple. - * - * The generator can create a permutation of any set of numbers, as long as - * the size of the set is an even power of 2. This limitation arises either - * out of an inherent property of balanced Feistel constructions, or by my - * own ignorance. I'll tackle an unbalanced construction after I wrap my - * head around Schneier and Kelsey's paper. - * - * CAVEAT EMPTOR. IANAC. - */ -#define DNS_K_PERMUTOR_ROUNDS 8 - - - -static inline guint -dns_k_permutor_powof (guint n) -{ - guint m, i = 0; - - for (m = 1; m < n; m <<= 1, i++); - - return i; -} /* dns_k_permutor_powof() */ - -static void -dns_k_permutor_init (struct dns_k_permutor *p, guint low, guint high) -{ - guint32 key[DNS_K_TEA_KEY_SIZE / sizeof (guint32)]; - guint width, i; - - p->stepi = 0; - - p->length = (high - low) + 1; - p->limit = high; - - width = dns_k_permutor_powof (p->length); - width += width % 2; - - p->shift = width / 2; - p->mask = (1U << p->shift) - 1; - p->rounds = DNS_K_PERMUTOR_ROUNDS; - -#ifndef HAVE_OPENSSL - for (i = 0; i < G_N_ELEMENTS (key); i++) { - key[i] = DNS_RANDOM (); - } -#else - if (RAND_bytes ((unsigned char *)key, sizeof (key)) != 1) { - for (i = 0; i < G_N_ELEMENTS (key); i++) { - key[i] = DNS_RANDOM (); - } - } -#endif - dns_k_tea_init (&p->tea, key, 0); - -} /* dns_k_permutor_init() */ - - -static guint -dns_k_permutor_F (struct dns_k_permutor *p, guint k, guint x) -{ - guint32 in[DNS_K_TEA_BLOCK_SIZE / sizeof (guint32)], out[DNS_K_TEA_BLOCK_SIZE / sizeof (guint32)]; - - memset(in, '\0', sizeof in); - - in[0] = k; - in[1] = x; - - dns_k_tea_encrypt (&p->tea, in, out); - - return p->mask & out[0]; -} /* dns_k_permutor_F() */ - - -static guint -dns_k_permutor_E (struct dns_k_permutor *p, guint n) -{ - guint l[2], r[2]; - guint i; - - i = 0; - l[i] = p->mask & (n >> p->shift); - r[i] = p->mask & (n >> 0); - - do { - l[(i + 1) % 2] = r[i % 2]; - r[(i + 1) % 2] = l[i % 2] ^ dns_k_permutor_F(p, i, r[i % 2]); - - i++; - } while (i < p->rounds - 1); - - return ((l[i % 2] & p->mask) << p->shift) | ((r[i % 2] & p->mask) << 0); -} /* dns_k_permutor_E() */ - - -static guint -dns_k_permutor_D (struct dns_k_permutor *p, guint n) -{ - guint l[2], r[2]; - guint i; - - i = p->rounds - 1; - l[i % 2] = p->mask & (n >> p->shift); - r[i % 2] = p->mask & (n >> 0); - - do { - i--; - - r[i % 2] = l[(i + 1) % 2]; - l[i % 2] = r[(i + 1) % 2] ^ dns_k_permutor_F(p, i, l[(i + 1) % 2]); - } while (i > 0); - - return ((l[i % 2] & p->mask) << p->shift) | ((r[i % 2] & p->mask) << 0); -} /* dns_k_permutor_D() */ - - -static guint -dns_k_permutor_step(struct dns_k_permutor *p) -{ - guint n; - - do { - n = dns_k_permutor_E(p, p->stepi++); - } while (n >= p->length); - - return n + (p->limit + 1 - p->length); -} /* dns_k_permutor_step() */ - - -/* - * Simple permutation box. Useful for shuffling rrsets from an iterator. - * Uses AES s-box to provide good diffusion. - */ -static guint16 -dns_k_shuffle16 (guint16 n, guint s) -{ - static const guint8 sbox[256] = - { 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, - 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, - 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, - 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, - 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, - 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, - 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, - 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, - 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, - 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, - 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, - 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, - 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, - 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, - 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, - 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, - 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, - 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, - 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, - 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, - 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, - 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, - 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, - 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, - 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, - 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, - 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, - 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, - 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, - 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, - 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, - 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16 }; - guchar a, b; - guint i; - - a = 0xff & (n >> 0); - b = 0xff & (n >> 8); - - for (i = 0; i < 4; i++) { - a ^= 0xff & s; - a = sbox[a] ^ b; - b = sbox[b] ^ a; - s >>= 8; - } - - return ((0xff00 & (a << 8)) | (0x00ff & (b << 0))); -} /* dns_k_shuffle16() */ - struct dns_request_key { guint16 id; guint16 port; @@ -496,7 +308,7 @@ make_dns_header (struct rspamd_dns_request *req) /* Set DNS header values */ header = (struct dns_header *)req->packet; memset (header, 0 , sizeof (struct dns_header)); - header->qid = dns_k_permutor_step (req->resolver->permutor); + header->qid = dns_permutor_generate_id (req->resolver->permutor); header->rd = 1; header->qdcount = htons (1); req->pos += sizeof (struct dns_header); @@ -767,6 +579,7 @@ static gint send_dns_request (struct rspamd_dns_request *req) { gint r; + struct rspamd_dns_server *serv = req->io->srv; r = send (req->sock, req->packet, req->pos, 0); if (r == -1) { @@ -774,20 +587,23 @@ send_dns_request (struct rspamd_dns_request *req) event_set (&req->io_event, req->sock, EV_WRITE, dns_retransmit_handler, req); event_base_set (req->resolver->ev_base, &req->io_event); event_add (&req->io_event, &req->tv); - register_async_event (req->session, (event_finalizer_t)event_del, &req->io_event, g_quark_from_static_string ("dns resolver")); + register_async_event (req->session, (event_finalizer_t)event_del, &req->io_event, + g_quark_from_static_string ("dns resolver")); return 0; } else { - msg_err ("send failed: %s for server %s", strerror (errno), req->server->name); - upstream_fail (&req->server->up, req->time); + msg_err ("send failed: %s for server %s", strerror (errno), serv->name); + upstream_fail (&serv->up, req->time); return -1; } } else if (r < req->pos) { + msg_err ("incomplete send over UDP socket, seems to be internal bug"); event_set (&req->io_event, req->sock, EV_WRITE, dns_retransmit_handler, req); event_base_set (req->resolver->ev_base, &req->io_event); event_add (&req->io_event, &req->tv); - register_async_event (req->session, (event_finalizer_t)event_del, &req->io_event, g_quark_from_static_string ("dns resolver")); + register_async_event (req->session, (event_finalizer_t)event_del, &req->io_event, + g_quark_from_static_string ("dns resolver")); return 0; } @@ -800,22 +616,24 @@ dns_fin_cb (gpointer arg) struct rspamd_dns_request *req = arg; event_del (&req->timer_event); - g_hash_table_remove (req->resolver->requests, &req->id); + g_hash_table_remove (req->io->requests, &req->id); } static guint8 * decompress_label (guint8 *begin, guint16 *len, guint16 max) { - guint16 offset = DNS_COMPRESSION_BITS; - offset = (*len) ^ (offset << 8); + guint16 offset = (*len); if (offset > max) { + msg_info ("invalid DNS compression pointer: %d max is %d", (gint)offset, (gint)max); return NULL; } *len = *(begin + offset); return begin + offset; } +#define UNCOMPRESS_DNS_OFFSET(p) (((*(p)) ^ DNS_COMPRESSION_BITS) << 8) + *((p) + 1) + static guint8 * dns_request_reply_cmp (struct rspamd_dns_request *req, guint8 *in, gint len) { @@ -844,10 +662,9 @@ dns_request_reply_cmp (struct rspamd_dns_request *req, guint8 *in, gint len) } /* This may be compressed, so we need to decompress it */ if (len1 & DNS_COMPRESSION_BITS) { - len1 = ((*p) << 8) + *(p + 1); + len1 = UNCOMPRESS_DNS_OFFSET(p); l1 = decompress_label (in, &len1, len); if (l1 == NULL) { - msg_info ("invalid DNS pointer"); return NULL; } decompressed ++; @@ -859,7 +676,7 @@ dns_request_reply_cmp (struct rspamd_dns_request *req, guint8 *in, gint len) p += len1; } if (len2 & DNS_COMPRESSION_BITS) { - len2 = ((*p) << 8) + *(p + 1); + len2 = UNCOMPRESS_DNS_OFFSET(p); l2 = decompress_label (req->packet, &len2, len); if (l2 == NULL) { msg_info ("invalid DNS pointer"); @@ -899,14 +716,15 @@ dns_request_reply_cmp (struct rspamd_dns_request *req, guint8 *in, gint len) #define MAX_RECURSION_LEVEL 10 static gboolean -dns_parse_labels (guint8 *in, gchar **target, guint8 **pos, struct rspamd_dns_reply *rep, gint *remain, gboolean make_name) +dns_parse_labels (guint8 *in, gchar **target, guint8 **pos, struct rspamd_dns_reply *rep, + gint *remain, gboolean make_name) { guint16 namelen = 0; - guint8 *p = *pos, *begin = *pos, *l, *t, *end = *pos + *remain; + guint8 *p = *pos, *begin = *pos, *l, *t, *end = *pos + *remain, *new_pos = *pos; guint16 llen; - gint offset = -1; - gint length = *remain; + gint length = *remain, new_remain = *remain; gint ptrs = 0, labels = 0; + gboolean got_compression = FALSE; /* First go through labels and calculate name length */ while (p - begin < length) { @@ -916,34 +734,52 @@ dns_parse_labels (guint8 *in, gchar **target, guint8 **pos, struct rspamd_dns_re } llen = *p; if (llen == 0) { + if (!got_compression) { + /* In case of compression we have already decremented the processing position */ + new_remain -= sizeof (guint8); + new_pos += sizeof (guint8); + } break; } - else if (llen & DNS_COMPRESSION_BITS) { - ptrs ++; - llen = ((*p) << 8) + *(p + 1); - l = decompress_label (in, &llen, end - in); - if (l == NULL) { - msg_info ("invalid DNS pointer"); - return FALSE; - } - if (offset < 0) { - /* Set offset strictly */ - offset = p - begin + 2; + else if ((llen & DNS_COMPRESSION_BITS)) { + if (end - p > 1) { + ptrs ++; + llen = UNCOMPRESS_DNS_OFFSET(p); + l = decompress_label (in, &llen, end - in); + if (l == NULL) { + msg_info ("invalid DNS pointer"); + return FALSE; + } + if (!got_compression) { + /* Our label processing is finished actually */ + new_remain -= sizeof (guint16); + new_pos += sizeof (guint16); + got_compression = TRUE; + } + if (l < in || l > begin + length) { + msg_warn ("invalid pointer in DNS packet"); + return FALSE; + } + begin = l; + length = end - begin; + p = l + *l + 1; + namelen += *l; + labels ++; } - if (l < in || l > begin + length) { - msg_warn ("invalid pointer in DNS packet"); + else { + msg_warn ("DNS packet has incomplete compressed label, input length: %d bytes, remain: %d", + *remain, new_remain); return FALSE; } - begin = l; - length = end - begin; - p = l + *l + 1; - namelen += *l; - labels ++; } else { - namelen += *p; - p += *p + 1; + namelen += llen; + p += llen + 1; labels ++; + if (!got_compression) { + new_remain -= llen + 1; + new_pos += llen + 1; + } } } @@ -962,7 +798,7 @@ dns_parse_labels (guint8 *in, gchar **target, guint8 **pos, struct rspamd_dns_re break; } else if (llen & DNS_COMPRESSION_BITS) { - llen = ((*p) << 8) + *(p + 1); + llen = UNCOMPRESS_DNS_OFFSET(p); l = decompress_label (in, &llen, end - in); begin = l; length = end - begin; @@ -980,11 +816,8 @@ dns_parse_labels (guint8 *in, gchar **target, guint8 **pos, struct rspamd_dns_re } *(t - 1) = '\0'; end: - if (offset < 0) { - offset = p - begin + 1; - } - *remain -= offset; - *pos += offset; + *remain = new_remain; + *pos = new_pos; return TRUE; } @@ -1004,8 +837,8 @@ dns_parse_rr (guint8 *in, union rspamd_reply_element *elt, guint8 **pos, struct msg_info ("bad RR name"); return -1; } - if ((gint)(p - *pos) >= (gint)(*remain - sizeof (guint16) * 5) || *remain <= 0) { - msg_info ("stripped dns reply"); + if (*remain < (gint)sizeof (guint16) * 6) { + msg_info ("stripped dns reply: %d bytes remain", *remain); return -1; } GET16 (type); @@ -1023,6 +856,7 @@ dns_parse_rr (guint8 *in, union rspamd_reply_element *elt, guint8 **pos, struct if (!(datalen & 0x3) && datalen <= *remain) { memcpy (&elt->a.addr[0], p, sizeof (struct in_addr)); p += datalen; + *remain -= datalen; parsed = TRUE; } else { @@ -1035,11 +869,13 @@ dns_parse_rr (guint8 *in, union rspamd_reply_element *elt, guint8 **pos, struct case DNS_T_AAAA: if (rep->request->type != DNS_REQUEST_AAA) { p += datalen; + *remain -= datalen; } else { if (datalen == sizeof (struct in6_addr) && datalen <= *remain) { memcpy (&elt->aaa.addr, p, sizeof (struct in6_addr)); p += datalen; + *remain -= datalen; parsed = TRUE; } else { @@ -1052,6 +888,7 @@ dns_parse_rr (guint8 *in, union rspamd_reply_element *elt, guint8 **pos, struct case DNS_T_PTR: if (rep->request->type != DNS_REQUEST_PTR) { p += datalen; + *remain -= datalen; } else { if (! dns_parse_labels (in, &elt->ptr.name, &p, rep, remain, TRUE)) { @@ -1064,10 +901,10 @@ dns_parse_rr (guint8 *in, union rspamd_reply_element *elt, guint8 **pos, struct case DNS_T_MX: if (rep->request->type != DNS_REQUEST_MX) { p += datalen; + *remain -= datalen; } else { GET16 (elt->mx.priority); - datalen -= sizeof (guint16); if (! dns_parse_labels (in, &elt->mx.name, &p, rep, remain, TRUE)) { msg_info ("invalid labels in MX record"); return -1; @@ -1078,6 +915,7 @@ dns_parse_rr (guint8 *in, union rspamd_reply_element *elt, guint8 **pos, struct case DNS_T_TXT: if (rep->request->type != DNS_REQUEST_TXT) { p += datalen; + *remain -= datalen; } else { elt->txt.data = memory_pool_alloc (rep->request->pool, datalen + 1); @@ -1091,6 +929,7 @@ dns_parse_rr (guint8 *in, union rspamd_reply_element *elt, guint8 **pos, struct memcpy (elt->txt.data + copied, p + 1, txtlen); copied += txtlen; p += txtlen + 1; + *remain -= txtlen + 1; } else { break; @@ -1103,6 +942,7 @@ dns_parse_rr (guint8 *in, union rspamd_reply_element *elt, guint8 **pos, struct case DNS_T_SPF: if (rep->request->type != DNS_REQUEST_SPF) { p += datalen; + *remain -= datalen; } else { copied = 0; @@ -1113,6 +953,7 @@ dns_parse_rr (guint8 *in, union rspamd_reply_element *elt, guint8 **pos, struct memcpy (elt->txt.data + copied, p + 1, txtlen); copied += txtlen; p += txtlen + 1; + *remain -= txtlen + 1; } else { break; @@ -1125,6 +966,7 @@ dns_parse_rr (guint8 *in, union rspamd_reply_element *elt, guint8 **pos, struct case DNS_T_SRV: if (rep->request->type != DNS_REQUEST_SRV) { p += datalen; + *remain -= datalen; } else { if (p - *pos > (gint)(*remain - sizeof (guint16) * 3)) { @@ -1144,13 +986,14 @@ dns_parse_rr (guint8 *in, union rspamd_reply_element *elt, guint8 **pos, struct case DNS_T_CNAME: /* Skip cname records */ p += datalen; + *remain -= datalen; break; default: msg_debug ("unexpected RR type: %d", type); p += datalen; + *remain -= datalen; break; } - *remain -= datalen; *pos = p; if (parsed) { @@ -1160,12 +1003,13 @@ dns_parse_rr (guint8 *in, union rspamd_reply_element *elt, guint8 **pos, struct } static gboolean -dns_parse_reply (guint8 *in, gint r, struct rspamd_dns_resolver *resolver, +dns_parse_reply (gint sock, guint8 *in, gint r, struct rspamd_dns_resolver *resolver, struct rspamd_dns_request **req_out, struct rspamd_dns_reply **_rep) { struct dns_header *header = (struct dns_header *)in; struct rspamd_dns_request *req; struct rspamd_dns_reply *rep; + struct rspamd_dns_io_channel *ioc; union rspamd_reply_element *elt; guint8 *pos; guint16 id; @@ -1177,10 +1021,17 @@ dns_parse_reply (guint8 *in, gint r, struct rspamd_dns_resolver *resolver, return FALSE; } + /* Find io channel */ + if ((ioc = g_hash_table_lookup (resolver->io_channels, GINT_TO_POINTER (sock))) == NULL) { + msg_err ("io channel is not found for this resolver: %d", sock); + return FALSE; + } + /* Now try to find corresponding request */ id = header->qid; - if ((req = g_hash_table_lookup (resolver->requests, &id)) == NULL) { + if ((req = g_hash_table_lookup (ioc->requests, &id)) == NULL) { /* No such requests found */ + msg_debug ("DNS request with id %d has not been found for IO channel", (gint)id); return FALSE; } *req_out = req; @@ -1188,7 +1039,9 @@ dns_parse_reply (guint8 *in, gint r, struct rspamd_dns_resolver *resolver, * Now we have request and query data is now at the end of header, so compare * request QR section and reply QR section */ - if ((pos = dns_request_reply_cmp (req, in + sizeof (struct dns_header), r - sizeof (struct dns_header))) == NULL) { + if ((pos = dns_request_reply_cmp (req, in + sizeof (struct dns_header), + r - sizeof (struct dns_header))) == NULL) { + msg_debug ("DNS request with id %d is for different query, ignoring", (gint)id); return FALSE; } /* @@ -1261,12 +1114,12 @@ dns_read_cb (gint fd, short what, void *arg) /* First read packet from socket */ r = read (fd, in, sizeof (in)); if (r > (gint)(sizeof (struct dns_header) + sizeof (struct dns_query))) { - if (dns_parse_reply (in, r, resolver, &req, &rep)) { + if (dns_parse_reply (fd, in, r, resolver, &req, &rep)) { /* Decrease errors count */ if (rep->request->resolver->errors > 0) { rep->request->resolver->errors --; } - upstream_ok (&rep->request->server->up, rep->request->time); + upstream_ok (&rep->request->io->srv->up, rep->request->time); rep->request->func (rep, rep->request->arg); remove_normal_event (req->session, dns_fin_cb, req); } @@ -1278,73 +1131,44 @@ dns_timer_cb (gint fd, short what, void *arg) { struct rspamd_dns_request *req = arg; struct rspamd_dns_reply *rep; + struct rspamd_dns_server *serv; gint r; /* Retransmit dns request */ req->retransmits ++; + serv = req->io->srv; if (req->retransmits >= req->resolver->max_retransmits) { - msg_err ("maximum number of retransmits expired for resolving %s of type %s", req->requested_name, dns_strtype (req->type)); - rep = memory_pool_alloc0 (req->pool, sizeof (struct rspamd_dns_reply)); - rep->request = req; - rep->code = DNS_RC_SERVFAIL; - upstream_fail (&rep->request->server->up, rep->request->time); + msg_err ("maximum number of retransmits expired for resolving %s of type %s", + req->requested_name, dns_strtype (req->type)); dns_check_throttling (req->resolver); req->resolver->errors ++; - - req->func (rep, req->arg); - remove_normal_event (req->session, dns_fin_cb, req); - - return; - } - /* Select other server */ - if (req->resolver->is_master_slave) { - req->server = (struct rspamd_dns_server *)get_upstream_master_slave (req->resolver->servers, - req->resolver->servers_num, sizeof (struct rspamd_dns_server), - req->time, DEFAULT_UPSTREAM_ERROR_TIME, DEFAULT_UPSTREAM_DEAD_TIME, DEFAULT_UPSTREAM_MAXERRORS); - } - else { - req->server = (struct rspamd_dns_server *)get_upstream_round_robin (req->resolver->servers, - req->resolver->servers_num, sizeof (struct rspamd_dns_server), - req->time, DEFAULT_UPSTREAM_ERROR_TIME, DEFAULT_UPSTREAM_DEAD_TIME, DEFAULT_UPSTREAM_MAXERRORS); - } - if (req->server == NULL) { - rep = memory_pool_alloc0 (req->pool, sizeof (struct rspamd_dns_reply)); - rep->request = req; - rep->code = DNS_RC_SERVFAIL; - - req->func (rep, req->arg); - remove_normal_event (req->session, dns_fin_cb, req); - return; + goto err; } - - if (req->server->sock == -1) { - req->server->sock = make_universal_socket (req->server->name, dns_port, SOCK_DGRAM, TRUE, FALSE, FALSE); - } - req->sock = req->server->sock; if (req->sock == -1) { - rep = memory_pool_alloc0 (req->pool, sizeof (struct rspamd_dns_reply)); - rep->request = req; - rep->code = DNS_RC_SERVFAIL; - upstream_fail (&rep->request->server->up, rep->request->time); - - req->func (rep, req->arg); - remove_normal_event (req->session, dns_fin_cb, req); - - return; + goto err; } /* Add other retransmit event */ r = send_dns_request (req); if (r == -1) { - rep = memory_pool_alloc0 (req->pool, sizeof (struct rspamd_dns_reply)); - rep->request = req; - rep->code = DNS_RC_SERVFAIL; - upstream_fail (&rep->request->server->up, rep->request->time); - req->func (rep, req->arg); - remove_normal_event (req->session, dns_fin_cb, req); - return; + goto err; } + + msg_debug ("retransmit DNS request with ID %d", (int)req->id); evtimer_add (&req->timer_event, &req->tv); + + return; +err: + msg_debug ("error on retransmitting DNS request with ID %d", (int)req->id); + rep = memory_pool_alloc0 (req->pool, sizeof (struct rspamd_dns_reply)); + rep->request = req; + rep->code = DNS_RC_SERVFAIL; + if (serv) { + upstream_fail (&serv->up, rep->request->time); + } + req->func (rep, req->arg); + remove_normal_event (req->session, dns_fin_cb, req); + return; } static void @@ -1352,9 +1176,11 @@ dns_retransmit_handler (gint fd, short what, void *arg) { struct rspamd_dns_request *req = arg; struct rspamd_dns_reply *rep; + struct rspamd_dns_server *serv; gint r; remove_normal_event (req->session, (event_finalizer_t)event_del, &req->io_event); + serv = req->io->srv; if (what == EV_WRITE) { /* Retransmit dns request */ @@ -1365,7 +1191,7 @@ dns_retransmit_handler (gint fd, short what, void *arg) rep = memory_pool_alloc0 (req->pool, sizeof (struct rspamd_dns_reply)); rep->request = req; rep->code = DNS_RC_SERVFAIL; - upstream_fail (&rep->request->server->up, rep->request->time); + upstream_fail (&serv->up, rep->request->time); req->resolver->errors ++; dns_check_throttling (req->resolver); @@ -1378,7 +1204,7 @@ dns_retransmit_handler (gint fd, short what, void *arg) rep = memory_pool_alloc0 (req->pool, sizeof (struct rspamd_dns_reply)); rep->request = req; rep->code = DNS_RC_SERVFAIL; - upstream_fail (&rep->request->server->up, rep->request->time); + upstream_fail (&serv->up, rep->request->time); req->func (rep, req->arg); } @@ -1390,8 +1216,9 @@ dns_retransmit_handler (gint fd, short what, void *arg) evtimer_add (&req->timer_event, &req->tv); /* Add request to hash table */ - g_hash_table_insert (req->resolver->requests, &req->id, req); - register_async_event (req->session, (event_finalizer_t)dns_fin_cb, req, g_quark_from_static_string ("dns resolver")); + g_hash_table_insert (req->io->requests, &req->id, req); + register_async_event (req->session, (event_finalizer_t)dns_fin_cb, + req, g_quark_from_static_string ("dns resolver")); } } } @@ -1403,11 +1230,17 @@ make_dns_request (struct rspamd_dns_resolver *resolver, { va_list args; struct rspamd_dns_request *req; + struct rspamd_dns_server *serv; struct in_addr *addr; const gchar *name, *service, *proto; gint r; + const gint max_id_cycles = 32; struct dns_header *header; + /* If no DNS servers defined silently return FALSE */ + if (resolver->servers_num == 0) { + return FALSE; + } /* Check throttling */ if (resolver->throttling) { return FALSE; @@ -1464,24 +1297,29 @@ make_dns_request (struct rspamd_dns_resolver *resolver, req->retransmits = 0; req->time = time (NULL); if (resolver->is_master_slave) { - req->server = (struct rspamd_dns_server *)get_upstream_master_slave (resolver->servers, + serv = (struct rspamd_dns_server *)get_upstream_master_slave (resolver->servers, resolver->servers_num, sizeof (struct rspamd_dns_server), req->time, DEFAULT_UPSTREAM_ERROR_TIME, DEFAULT_UPSTREAM_DEAD_TIME, DEFAULT_UPSTREAM_MAXERRORS); } else { - req->server = (struct rspamd_dns_server *)get_upstream_round_robin (resolver->servers, + serv = (struct rspamd_dns_server *)get_upstream_round_robin (resolver->servers, resolver->servers_num, sizeof (struct rspamd_dns_server), req->time, DEFAULT_UPSTREAM_ERROR_TIME, DEFAULT_UPSTREAM_DEAD_TIME, DEFAULT_UPSTREAM_MAXERRORS); } - if (req->server == NULL) { + if (serv == NULL) { msg_err ("cannot find suitable server for request"); return FALSE; } - if (req->server->sock == -1) { - req->server->sock = make_universal_socket (req->server->name, dns_port, SOCK_DGRAM, TRUE, FALSE, FALSE); + /* Now select IO channel */ + + req->io = serv->cur_io_channel; + if (req->io == NULL) { + msg_err ("cannot find suitable io channel for the server %s", serv->name); + return FALSE; } - req->sock = req->server->sock; + serv->cur_io_channel = serv->cur_io_channel->next; + req->sock = req->io->sock; if (req->sock == -1) { return FALSE; @@ -1496,18 +1334,23 @@ make_dns_request (struct rspamd_dns_resolver *resolver, r = send_dns_request (req); if (r == 1) { - /* Add timer event */ - evtimer_add (&req->timer_event, &req->tv); - /* Add request to hash table */ - while (g_hash_table_lookup (resolver->requests, &req->id)) { + r = 0; + while (g_hash_table_lookup (req->io->requests, &req->id)) { /* Check for unique id */ header = (struct dns_header *)req->packet; - header->qid = dns_k_permutor_step (resolver->permutor); + header->qid = dns_permutor_generate_id (resolver->permutor); req->id = header->qid; + if (++r > max_id_cycles) { + msg_err ("cannot generate new id for server %s", serv->name); + return FALSE; + } } - g_hash_table_insert (resolver->requests, &req->id, req); - register_async_event (session, (event_finalizer_t)dns_fin_cb, req, g_quark_from_static_string ("dns resolver")); + /* Add timer event */ + evtimer_add (&req->timer_event, &req->tv); + g_hash_table_insert (req->io->requests, &req->id, req); + register_async_event (session, (event_finalizer_t)dns_fin_cb, req, + g_quark_from_static_string ("dns resolver")); } else if (r == -1) { return FALSE; @@ -1584,14 +1427,14 @@ dns_resolver_init (struct event_base *ev_base, struct config_file *cfg) GList *cur; struct rspamd_dns_resolver *new; gchar *begin, *p, *err, addr_holder[16]; - gint priority, i; + gint priority, i, j; struct rspamd_dns_server *serv; + struct rspamd_dns_io_channel *ioc; new = memory_pool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_dns_resolver)); new->ev_base = ev_base; - new->requests = g_hash_table_new (dns_id_hash, dns_id_equal); - new->permutor = memory_pool_alloc (cfg->cfg_pool, sizeof (struct dns_k_permutor)); - dns_k_permutor_init (new->permutor, 0, G_MAXUINT16); + new->permutor = dns_permutor_new (cfg->cfg_pool); + new->io_channels = g_hash_table_new (g_direct_hash, g_direct_equal); new->static_pool = cfg->cfg_pool; new->request_timeout = cfg->dns_timeout; new->max_retransmits = cfg->dns_retransmits; @@ -1602,7 +1445,7 @@ dns_resolver_init (struct event_base *ev_base, struct config_file *cfg) /* Parse resolv.conf */ if (! parse_resolv_conf (new) || new->servers_num == 0) { msg_err ("cannot parse resolv.conf and no nameservers defined, so no ways to resolve addresses"); - return NULL; + return new; } } else { @@ -1657,22 +1500,33 @@ dns_resolver_init (struct event_base *ev_base, struct config_file *cfg) msg_err ("no valid nameservers defined, try to parse resolv.conf"); if (! parse_resolv_conf (new) || new->servers_num == 0) { msg_err ("cannot parse resolv.conf and no nameservers defined, so no ways to resolve addresses"); - return NULL; + return new; } } } - /* Now init all servers */ + /* Now init io channels to all servers */ for (i = 0; i < new->servers_num; i ++) { serv = &new->servers[i]; - serv->sock = make_universal_socket (serv->name, dns_port, SOCK_DGRAM, TRUE, FALSE, FALSE); - if (serv->sock == -1) { - msg_warn ("cannot create socket to server %s", serv->name); - } - else { - event_set (&serv->ev, serv->sock, EV_READ | EV_PERSIST, dns_read_cb, new); - event_base_set (new->ev_base, &serv->ev); - event_add (&serv->ev, NULL); + for (j = 0; j < (gint)cfg->dns_io_per_server; j ++) { + ioc = memory_pool_alloc (new->static_pool, sizeof (struct rspamd_dns_io_channel)); + ioc->sock = make_universal_socket (serv->name, dns_port, SOCK_DGRAM, TRUE, FALSE, FALSE); + if (ioc->sock == -1) { + msg_warn ("cannot create socket to server %s", serv->name); + } + else { + ioc->requests = g_hash_table_new (dns_id_hash, dns_id_equal); + memory_pool_add_destructor (new->static_pool, (pool_destruct_func)g_hash_table_unref, + ioc->requests); + ioc->srv = serv; + ioc->resolver = new; + event_set (&ioc->ev, ioc->sock, EV_READ | EV_PERSIST, dns_read_cb, new); + event_base_set (new->ev_base, &ioc->ev); + event_add (&ioc->ev, NULL); + CDL_PREPEND (serv->io_channels, ioc); + serv->cur_io_channel = ioc; + g_hash_table_insert (new->io_channels, GINT_TO_POINTER (ioc->sock), ioc); + } } } @@ -1,3 +1,28 @@ +/* + * Copyright (c) 2013, Vsevolod Stakhov + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + #ifndef RSPAMD_DNS_H #define RSPAMD_DNS_H @@ -17,38 +42,39 @@ struct rspamd_dns_reply; struct config_file; typedef void (*dns_callback_type) (struct rspamd_dns_reply *reply, gpointer arg); + /** - * Implements DNS server + * Represents DNS server */ struct rspamd_dns_server { struct upstream up; /**< upstream structure */ gchar *name; /**< name of DNS server */ + struct rspamd_dns_io_channel *io_channels; + struct rspamd_dns_io_channel *cur_io_channel; +}; + +/** + * IO channel for a specific DNS server + */ +struct rspamd_dns_io_channel { + struct rspamd_dns_server *srv; + struct rspamd_dns_resolver *resolver; gint sock; /**< persistent socket */ struct event ev; + GHashTable *requests; /**< requests in flight */ + struct rspamd_dns_io_channel *prev, *next; }; -#define DNS_K_TEA_KEY_SIZE 16 - -struct dns_k_tea { - guint32 key[DNS_K_TEA_KEY_SIZE / sizeof (guint32)]; - guint cycles; -}; /* struct dns_k_tea */ - -struct dns_k_permutor { - guint stepi, length, limit; - guint shift, mask, rounds; - - struct dns_k_tea tea; -}; +struct dns_permutor; struct rspamd_dns_resolver { struct rspamd_dns_server servers[MAX_SERVERS]; gint servers_num; /**< number of DNS servers registered */ - GHashTable *requests; /**< requests in flight */ - struct dns_k_permutor *permutor; /**< permutor for randomizing request id */ + struct dns_permutor *permutor; /**< permutor for randomizing request id */ guint request_timeout; guint max_retransmits; guint max_errors; + GHashTable *io_channels; /**< hash of io chains indexed by socket */ memory_pool_t *static_pool; /**< permament pool (cfg_pool) */ gboolean throttling; /**< dns servers are busy */ gboolean is_master_slave; /**< if this is true, then select upstreams as master/slave */ @@ -74,7 +100,7 @@ enum rspamd_request_type { struct rspamd_dns_request { memory_pool_t *pool; /**< pool associated with request */ struct rspamd_dns_resolver *resolver; - struct rspamd_dns_server *server; + struct rspamd_dns_io_channel *io; dns_callback_type func; gpointer arg; struct event timer_event; @@ -234,12 +260,12 @@ struct dns_query { /* Rspamd DNS API */ -/* +/** * Init DNS resolver, params are obtained from a config file or system file /etc/resolv.conf */ struct rspamd_dns_resolver *dns_resolver_init (struct event_base *ev_base, struct config_file *cfg); -/* +/** * Make a DNS request * @param resolver resolver object * @param session async session to register event @@ -254,12 +280,12 @@ gboolean make_dns_request (struct rspamd_dns_resolver *resolver, struct rspamd_async_session *session, memory_pool_t *pool, dns_callback_type cb, gpointer ud, enum rspamd_request_type type, ...); -/* +/** * Get textual presentation of DNS error code */ const gchar *dns_strerror (enum dns_rcode rcode); -/* +/** * Get textual presentation of DNS request type */ const gchar *dns_strtype (enum rspamd_request_type type); diff --git a/src/filter.c b/src/filter.c index c62a69084..b1448f173 100644 --- a/src/filter.c +++ b/src/filter.c @@ -89,7 +89,7 @@ insert_metric_result (struct worker_task *task, struct metric *metric, const gch weight = g_hash_table_lookup (metric->symbols, symbol); if (weight == NULL) { - w = 1.0 * flag; + w = 0.0; } else { w = (*weight) * flag; @@ -116,12 +116,16 @@ insert_metric_result (struct worker_task *task, struct metric *metric, const gch w *= metric_res->grow_factor; metric_res->grow_factor *= metric->grow_factor; } - else if (w > 0) { - metric_res->grow_factor = metric->grow_factor; - } s->score += w; metric_res->score += w; } + else { + if (fabs (s->score) < fabs (w)) { + /* Replace less weight with a bigger one */ + metric_res->score = metric_res->score - s->score + w; + s->score = w; + } + } } else { s = memory_pool_alloc (task->task_pool, sizeof (struct symbol)); diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c index 1f47b0dde..9bd4d49d3 100644 --- a/src/fuzzy_storage.c +++ b/src/fuzzy_storage.c @@ -63,6 +63,8 @@ /* Current version of fuzzy hash file format */ #define CURRENT_FUZZY_VERSION 1 +#define INVALID_NODE_TIME (guint64)-1 + /* Init functions */ gpointer init_fuzzy (struct config_file *cfg); void start_fuzzy (struct rspamd_worker *worker); @@ -96,7 +98,7 @@ static sig_atomic_t wanna_die = 0; struct rspamd_fuzzy_storage_ctx { gboolean use_judy; char *hashfile; - guint32 expire; + gdouble expire; guint32 frequent_score; guint32 max_mods; radix_tree_t *update_ips; @@ -121,6 +123,7 @@ struct fuzzy_session { gint fd; u_char *pos; socklen_t salen; + guint64 time; union { struct sockaddr ss; struct sockaddr_storage sa; @@ -173,23 +176,15 @@ expire_nodes (gpointer *to_expire, gint expired_num, for (i = 0; i < expired_num; i ++) { #ifdef WITH_JUDY - PPvoid_t pvalue; - gpointer data; - if (ctx->use_judy) { node = (struct rspamd_fuzzy_node *)to_expire[i]; + if (node->time != INVALID_NODE_TIME) { + server_stat->fuzzy_hashes_expired ++; + } + server_stat->fuzzy_hashes --; JudySLDel (&jtree, node->h.hash_pipe, PJE0); bloom_del (bf, node->h.hash_pipe); g_slice_free1 (sizeof (struct rspamd_fuzzy_node), node); - - pvalue = JudySLGet (jtree, node->h.hash_pipe, PJE0); - if (pvalue) { - data = *pvalue; - JudySLDel (&jtree, node->h.hash_pipe, PJE0); - g_slice_free1 (sizeof (struct rspamd_fuzzy_node), data); - server_stat->fuzzy_hashes_expired ++; - server_stat->fuzzy_hashes --; - } } else { #endif @@ -198,7 +193,9 @@ expire_nodes (gpointer *to_expire, gint expired_num, head = hashes[node->h.block_size % BUCKETS]; g_queue_delete_link (head, cur); bloom_del (bf, node->h.hash_pipe); - server_stat->fuzzy_hashes_expired++; + if (node->time != INVALID_NODE_TIME) { + server_stat->fuzzy_hashes_expired ++; + } server_stat->fuzzy_hashes--; g_slice_free1 (sizeof(struct rspamd_fuzzy_node), node); #ifdef WITH_JUDY @@ -277,7 +274,7 @@ sync_cache (gpointer ud) pvalue = JudySLFirst (jtree, indexbuf, PJE0); while (pvalue) { node = *((struct rspamd_fuzzy_node **)pvalue); - if (now - node->time > expire) { + if (node->time == INVALID_NODE_TIME || now - node->time > expire) { if (nodes_expired == NULL) { nodes_expired = g_malloc (max_expired * sizeof (gpointer)); } @@ -558,7 +555,8 @@ read_hashes_file (struct rspamd_worker *wrk) } static inline struct rspamd_fuzzy_node * -check_hash_node (GQueue *hash, fuzzy_hash_t *s, gint update_value, struct rspamd_fuzzy_storage_ctx *ctx) +check_hash_node (GQueue *hash, fuzzy_hash_t *s, gint update_value, + guint64 time, struct rspamd_fuzzy_storage_ctx *ctx) { GList *cur; struct rspamd_fuzzy_node *h; @@ -571,8 +569,17 @@ check_hash_node (GQueue *hash, fuzzy_hash_t *s, gint update_value, struct rspamd pvalue = JudySLGet (jtree, s->hash_pipe, PJE0); if (pvalue != NULL) { h = *((struct rspamd_fuzzy_node **)pvalue); - /* Also check block size */ - if (h->h.block_size== s->block_size) { + + if (h->time == INVALID_NODE_TIME) { + /* Node is expired */ + return NULL; + } + else if (update_value == 0 && time - h->time > ctx->expire) { + h->time = INVALID_NODE_TIME; + server_stat->fuzzy_hashes_expired ++; + return NULL; + } + else if (h->h.block_size== s->block_size) { msg_info ("fuzzy hash was found in judy tree"); if (update_value) { h->value += update_value; @@ -588,10 +595,18 @@ check_hash_node (GQueue *hash, fuzzy_hash_t *s, gint update_value, struct rspamd h = cur->data; if ((prob = fuzzy_compare_hashes (&h->h, s)) > LEV_LIMIT) { msg_info ("fuzzy hash was found, probability %d%%", prob); - if (update_value) { + if (h->time == INVALID_NODE_TIME) { + return NULL; + } + else if (update_value) { msg_info ("new hash weight: %d", h->value); h->value += update_value; } + else if (time - h->time > ctx->expire) { + h->time = INVALID_NODE_TIME; + server_stat->fuzzy_hashes_expired ++; + return NULL; + } return h; } cur = g_list_next (cur); @@ -602,9 +617,17 @@ check_hash_node (GQueue *hash, fuzzy_hash_t *s, gint update_value, struct rspamd h = cur->data; if ((prob = fuzzy_compare_hashes (&h->h, s)) > LEV_LIMIT) { msg_info ("fuzzy hash was found, probability %d%%", prob); - if (update_value) { - h->value += update_value; + if (h->time == INVALID_NODE_TIME) { + return NULL; + } + else if (update_value) { msg_info ("new hash weight: %d", h->value); + h->value += update_value; + } + else if (time - h->time > ctx->expire) { + h->time = INVALID_NODE_TIME; + server_stat->fuzzy_hashes_expired ++; + return NULL; } if (h->value > (gint)ctx->frequent_score) { g_queue_unlink (hash, cur); @@ -623,7 +646,7 @@ check_hash_node (GQueue *hash, fuzzy_hash_t *s, gint update_value, struct rspamd } static gint -process_check_command (struct fuzzy_cmd *cmd, gint *flag, struct rspamd_fuzzy_storage_ctx *ctx) +process_check_command (struct fuzzy_cmd *cmd, gint *flag, guint64 time, struct rspamd_fuzzy_storage_ctx *ctx) { fuzzy_hash_t s; struct rspamd_fuzzy_node *h; @@ -637,7 +660,7 @@ process_check_command (struct fuzzy_cmd *cmd, gint *flag, struct rspamd_fuzzy_st s.block_size = cmd->blocksize; rspamd_rwlock_reader_lock (ctx->tree_lock); - h = check_hash_node (hashes[cmd->blocksize % BUCKETS], &s, 0, ctx); + h = check_hash_node (hashes[cmd->blocksize % BUCKETS], &s, 0, time, ctx); rspamd_rwlock_reader_unlock (ctx->tree_lock); if (h == NULL) { @@ -650,24 +673,28 @@ process_check_command (struct fuzzy_cmd *cmd, gint *flag, struct rspamd_fuzzy_st } static gboolean -update_hash (struct fuzzy_cmd *cmd, struct rspamd_fuzzy_storage_ctx *ctx) +update_hash (struct fuzzy_cmd *cmd, guint64 time, struct rspamd_fuzzy_storage_ctx *ctx) { fuzzy_hash_t s; - gboolean r; + struct rspamd_fuzzy_node *n; memcpy (s.hash_pipe, cmd->hash, sizeof (s.hash_pipe)); s.block_size = cmd->blocksize; mods ++; rspamd_rwlock_writer_lock (ctx->tree_lock); - r = check_hash_node (hashes[cmd->blocksize % BUCKETS], &s, cmd->value, ctx) != NULL; + n = check_hash_node (hashes[cmd->blocksize % BUCKETS], &s, cmd->value, time, ctx); rspamd_rwlock_writer_unlock (ctx->tree_lock); - return r; + if (n != NULL) { + n->time = time; + return TRUE; + } + return FALSE; } static gboolean -process_write_command (struct fuzzy_cmd *cmd, struct rspamd_fuzzy_storage_ctx *ctx) +process_write_command (struct fuzzy_cmd *cmd, guint64 time, struct rspamd_fuzzy_storage_ctx *ctx) { struct rspamd_fuzzy_node *h; #ifdef WITH_JUDY @@ -675,7 +702,7 @@ process_write_command (struct fuzzy_cmd *cmd, struct rspamd_fuzzy_storage_ctx *c #endif if (bloom_check (bf, cmd->hash)) { - if (update_hash (cmd, ctx)) { + if (update_hash (cmd, time, ctx)) { return TRUE; } } @@ -684,7 +711,7 @@ process_write_command (struct fuzzy_cmd *cmd, struct rspamd_fuzzy_storage_ctx *c h = g_slice_alloc (sizeof (struct rspamd_fuzzy_node)); memcpy (&h->h.hash_pipe, &cmd->hash, sizeof (cmd->hash)); h->h.block_size = cmd->blocksize; - h->time = (guint64) time (NULL); + h->time = time; h->value = cmd->value; h->flag = cmd->flag; #ifdef WITH_JUDY @@ -764,7 +791,7 @@ delete_hash (GQueue *hash, fuzzy_hash_t *s, struct rspamd_fuzzy_storage_ctx *ctx } static gboolean -process_delete_command (struct fuzzy_cmd *cmd, struct rspamd_fuzzy_storage_ctx *ctx) +process_delete_command (struct fuzzy_cmd *cmd, guint64 time, struct rspamd_fuzzy_storage_ctx *ctx) { fuzzy_hash_t s; gboolean res = FALSE; @@ -817,7 +844,7 @@ check_fuzzy_client (struct fuzzy_session *session) #define CMD_PROCESS(x) \ do { \ -if (process_##x##_command (&session->cmd, session->worker->ctx)) { \ +if (process_##x##_command (&session->cmd, session->time, session->worker->ctx)) { \ if (sendto (session->fd, "OK" CRLF, sizeof ("OK" CRLF) - 1, 0, &session->client_addr.ss, session->salen) == -1) { \ msg_err ("error while writing reply: %s", strerror (errno)); \ } \ @@ -837,7 +864,7 @@ process_fuzzy_command (struct fuzzy_session *session) switch (session->cmd.cmd) { case FUZZY_CHECK: - r = process_check_command (&session->cmd, &flag, session->worker->ctx); + r = process_check_command (&session->cmd, &flag, session->time, session->worker->ctx); if (r != 0) { r = rspamd_snprintf (buf, sizeof (buf), "OK %d %d" CRLF, r, flag); if (sendto (session->fd, buf, r, 0, @@ -910,6 +937,7 @@ accept_fuzzy_socket (gint fd, short what, void *arg) session.pos = (u_char *) & session.cmd; session.salen = sizeof (session.client_addr); session.ctx = worker->ctx; + session.time = (guint64)time (NULL); /* Got some data */ if (what == EV_READ) { @@ -1022,7 +1050,7 @@ init_fuzzy (struct config_file *cfg) rspamd_rcl_register_worker_option (cfg, type, "expire", rspamd_rcl_parse_struct_time, ctx, - G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, expire), RSPAMD_CL_FLAG_TIME_UINT_32); + G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, expire), RSPAMD_CL_FLAG_TIME_FLOAT); rspamd_rcl_register_worker_option (cfg, type, "use_judy", rspamd_rcl_parse_struct_boolean, ctx, diff --git a/src/logger.c b/src/logger.c index 9dd999c61..c0f2e4888 100644 --- a/src/logger.c +++ b/src/logger.c @@ -144,6 +144,22 @@ direct_write_log_line (rspamd_logger_t *rspamd_log, void *data, gint count, gboo } } +static void +rspamd_escape_log_string (gchar *str) +{ + guchar *p = (guchar *)str; + + while (*p) { + if ((*p & 0x80) || !g_ascii_isprint (*p)) { + *p = '?'; + } + else if (*p == '\n' || *p == '\r') { + *p = ' '; + } + p ++; + } +} + /* Logging utility functions */ gint open_log_priv (rspamd_logger_t *rspamd_log, uid_t uid, gid_t gid) @@ -259,7 +275,7 @@ reopen_log (rspamd_logger_t *logger) * Setup logger */ void -rspamd_set_logger (enum rspamd_log_type type, GQuark ptype, struct rspamd_main *rspamd) +rspamd_set_logger (struct config_file *cfg, GQuark ptype, struct rspamd_main *rspamd) { gchar **strvec, *p, *err; gint num, i, k; @@ -271,7 +287,7 @@ rspamd_set_logger (enum rspamd_log_type type, GQuark ptype, struct rspamd_main * memset (rspamd->logger, 0, sizeof (rspamd_logger_t)); } - rspamd->logger->type = type; + rspamd->logger->type = cfg->log_type; rspamd->logger->pid = getpid (); rspamd->logger->process_type = ptype; @@ -282,7 +298,7 @@ rspamd_set_logger (enum rspamd_log_type type, GQuark ptype, struct rspamd_main * g_mutex_init (rspamd->logger->mtx); #endif - switch (type) { + switch (cfg->log_type) { case RSPAMD_LOG_CONSOLE: rspamd->logger->log_func = file_log_function; rspamd->logger->fd = STDERR_FILENO; @@ -295,7 +311,7 @@ rspamd_set_logger (enum rspamd_log_type type, GQuark ptype, struct rspamd_main * break; } - rspamd->logger->cfg = rspamd->cfg; + rspamd->logger->cfg = cfg; /* Set up buffer */ if (rspamd->cfg->log_buffered) { if (rspamd->cfg->log_buf_size != 0) { @@ -382,7 +398,7 @@ void rspamd_common_log_function (rspamd_logger_t *rspamd_log, GLogLevelFlags log_level, const gchar *function, const gchar *fmt, ...) { - static gchar logbuf[BUFSIZ], escaped_logbuf[BUFSIZ]; + static gchar logbuf[BUFSIZ]; va_list vp; u_char *end; @@ -391,9 +407,9 @@ rspamd_common_log_function (rspamd_logger_t *rspamd_log, GLogLevelFlags log_leve va_start (vp, fmt); end = rspamd_vsnprintf (logbuf, sizeof (logbuf), fmt, vp); *end = '\0'; - (void)rspamd_escape_string (escaped_logbuf, logbuf, sizeof (escaped_logbuf)); + rspamd_escape_log_string (logbuf); va_end (vp); - rspamd_log->log_func (NULL, function, log_level, escaped_logbuf, FALSE, rspamd_log); + rspamd_log->log_func (NULL, function, log_level, logbuf, FALSE, rspamd_log); g_mutex_unlock (rspamd_log->mtx); } } @@ -402,7 +418,7 @@ void rspamd_default_log_function (GLogLevelFlags log_level, const gchar *function, const gchar *fmt, ...) { - static gchar logbuf[BUFSIZ], escaped_logbuf[BUFSIZ]; + static gchar logbuf[BUFSIZ]; va_list vp; u_char *end; @@ -412,9 +428,9 @@ rspamd_default_log_function (GLogLevelFlags log_level, va_start (vp, fmt); end = rspamd_vsnprintf (logbuf, sizeof (logbuf), fmt, vp); *end = '\0'; - (void)rspamd_escape_string (escaped_logbuf, logbuf, sizeof (escaped_logbuf)); + rspamd_escape_log_string (logbuf); va_end (vp); - fprintf (stderr, "%s\n", escaped_logbuf); + fprintf (stderr, "%s\n", logbuf); } } else if (log_level <= default_logger->cfg->log_level) { @@ -422,9 +438,9 @@ rspamd_default_log_function (GLogLevelFlags log_level, va_start (vp, fmt); end = rspamd_vsnprintf (logbuf, sizeof (logbuf), fmt, vp); *end = '\0'; - (void)rspamd_escape_string (escaped_logbuf, logbuf, sizeof (escaped_logbuf)); + rspamd_escape_log_string (logbuf); va_end (vp); - default_logger->log_func (NULL, function, log_level, escaped_logbuf, FALSE, default_logger); + default_logger->log_func (NULL, function, log_level, logbuf, FALSE, default_logger); g_mutex_unlock (default_logger->mtx); } } @@ -687,7 +703,7 @@ file_log_function (const gchar * log_domain, const gchar *function, GLogLevelFla void rspamd_conditional_debug (rspamd_logger_t *rspamd_log, guint32 addr, const gchar *function, const gchar *fmt, ...) { - static gchar logbuf[BUFSIZ], escaped_logbuf[BUFSIZ]; + static gchar logbuf[BUFSIZ]; va_list vp; u_char *end; @@ -698,9 +714,9 @@ rspamd_conditional_debug (rspamd_logger_t *rspamd_log, guint32 addr, const gchar va_start (vp, fmt); end = rspamd_vsnprintf (logbuf, sizeof (logbuf), fmt, vp); *end = '\0'; - (void)rspamd_escape_string (escaped_logbuf, logbuf, sizeof (escaped_logbuf)); + rspamd_escape_log_string (logbuf); va_end (vp); - rspamd_log->log_func (NULL, function, G_LOG_LEVEL_DEBUG, escaped_logbuf, TRUE, rspamd_log); + rspamd_log->log_func (NULL, function, G_LOG_LEVEL_DEBUG, logbuf, TRUE, rspamd_log); g_mutex_unlock (rspamd_log->mtx); } } @@ -710,13 +726,11 @@ rspamd_conditional_debug (rspamd_logger_t *rspamd_log, guint32 addr, const gchar void rspamd_glib_log_function (const gchar *log_domain, GLogLevelFlags log_level, const gchar *message, gpointer arg) { - gchar escaped_logbuf[BUFSIZ]; rspamd_logger_t *rspamd_log = arg; if (rspamd_log->enabled) { g_mutex_lock (rspamd_log->mtx); - (void)rspamd_escape_string (escaped_logbuf, message, sizeof (escaped_logbuf)); - rspamd_log->log_func (log_domain, NULL, log_level, escaped_logbuf, FALSE, rspamd_log); + rspamd_log->log_func (log_domain, NULL, log_level, message, FALSE, rspamd_log); g_mutex_unlock (rspamd_log->mtx); } } diff --git a/src/logger.h b/src/logger.h index 676b30131..50b9fe41b 100644 --- a/src/logger.h +++ b/src/logger.h @@ -15,7 +15,7 @@ typedef struct rspamd_logger_s rspamd_logger_t; /** * Init logger */ -void rspamd_set_logger (enum rspamd_log_type type, GQuark ptype, struct rspamd_main *main); +void rspamd_set_logger (struct config_file *cfg, GQuark ptype, struct rspamd_main *main); /** * Open log file or initialize other structures */ diff --git a/src/lua/lua_config.c b/src/lua/lua_config.c index 48686ff03..7987fa00e 100644 --- a/src/lua/lua_config.c +++ b/src/lua/lua_config.c @@ -42,6 +42,7 @@ LUA_FUNCTION_DEF (config, add_hash_map); LUA_FUNCTION_DEF (config, add_kv_map); LUA_FUNCTION_DEF (config, get_classifier); LUA_FUNCTION_DEF (config, register_symbol); +LUA_FUNCTION_DEF (config, register_symbols); LUA_FUNCTION_DEF (config, register_virtual_symbol); LUA_FUNCTION_DEF (config, register_callback_symbol); LUA_FUNCTION_DEF (config, register_callback_symbol_priority); @@ -60,6 +61,7 @@ static const struct luaL_reg configlib_m[] = { LUA_INTERFACE_DEF (config, add_kv_map), LUA_INTERFACE_DEF (config, get_classifier), LUA_INTERFACE_DEF (config, register_symbol), + LUA_INTERFACE_DEF (config, register_symbols), LUA_INTERFACE_DEF (config, register_virtual_symbol), LUA_INTERFACE_DEF (config, register_callback_symbol), LUA_INTERFACE_DEF (config, register_callback_symbol_priority), @@ -596,7 +598,52 @@ lua_config_register_symbol (lua_State * L) } memory_pool_add_destructor (cfg->cfg_pool, (pool_destruct_func)lua_destroy_cfg_symbol, cd); } - return 1; + return 0; +} + +static gint +lua_config_register_symbols (lua_State *L) +{ + struct config_file *cfg = lua_check_config (L); + struct lua_callback_data *cd; + gint i, top; + gchar *sym; + gdouble weight = 1.0; + + if (lua_gettop (L) < 3) { + msg_err ("not enough arguments to register a function"); + return 0; + } + if (cfg) { + cd = memory_pool_alloc (cfg->cfg_pool, sizeof (struct lua_callback_data)); + if (lua_type (L, 2) == LUA_TSTRING) { + cd->callback.name = memory_pool_strdup (cfg->cfg_pool, luaL_checkstring (L, 2)); + cd->cb_is_ref = FALSE; + } + else { + lua_pushvalue (L, 2); + /* Get a reference */ + cd->callback.ref = luaL_ref (L, LUA_REGISTRYINDEX); + cd->cb_is_ref = TRUE; + } + if (lua_type (L, 3) == LUA_TNUMBER) { + weight = lua_tonumber (L, 3); + top = 4; + } + else { + top = 3; + } + sym = memory_pool_strdup (cfg->cfg_pool, luaL_checkstring (L, top)); + cd->symbol = sym; + cd->L = L; + register_symbol (&cfg->cache, sym, weight, lua_metric_symbol_callback, cd); + for (i = top; i < lua_gettop (L); i ++) { + sym = memory_pool_strdup (cfg->cfg_pool, luaL_checkstring (L, i + 1)); + register_virtual_symbol (&cfg->cache, sym, weight); + } + } + + return 0; } static gint @@ -613,7 +660,7 @@ lua_config_register_virtual_symbol (lua_State * L) register_virtual_symbol (&cfg->cache, name, weight); } } - return 1; + return 0; } static gint @@ -645,7 +692,7 @@ lua_config_register_callback_symbol (lua_State * L) } memory_pool_add_destructor (cfg->cfg_pool, (pool_destruct_func)lua_destroy_cfg_symbol, cd); } - return 1; + return 0; } static gint @@ -681,7 +728,7 @@ lua_config_register_callback_symbol_priority (lua_State * L) memory_pool_add_destructor (cfg->cfg_pool, (pool_destruct_func)lua_destroy_cfg_symbol, cd); } - return 1; + return 0; } diff --git a/src/lua/lua_dns.c b/src/lua/lua_dns.c index c56ff4548..540fc7713 100644 --- a/src/lua/lua_dns.c +++ b/src/lua/lua_dns.c @@ -33,6 +33,7 @@ LUA_FUNCTION_DEF (dns_resolver, init); LUA_FUNCTION_DEF (dns_resolver, resolve_a); LUA_FUNCTION_DEF (dns_resolver, resolve_ptr); LUA_FUNCTION_DEF (dns_resolver, resolve_txt); +LUA_FUNCTION_DEF (dns_resolver, resolve_mx); static const struct luaL_reg dns_resolverlib_f[] = { LUA_INTERFACE_DEF (dns_resolver, init), @@ -43,6 +44,7 @@ static const struct luaL_reg dns_resolverlib_m[] = { LUA_INTERFACE_DEF (dns_resolver, resolve_a), LUA_INTERFACE_DEF (dns_resolver, resolve_ptr), LUA_INTERFACE_DEF (dns_resolver, resolve_txt), + LUA_INTERFACE_DEF (dns_resolver, resolve_mx), {"__tostring", lua_class_tostring}, {NULL, NULL} }; @@ -128,6 +130,24 @@ lua_dns_callback (struct rspamd_dns_reply *reply, gpointer arg) lua_pushnil (cd->L); } + else if (reply->type == DNS_REQUEST_MX) { + lua_newtable (cd->L); + cur = reply->elements; + while (cur) { + elt = cur->data; + /* mx['name'], mx['priority'] */ + lua_newtable (cd->L); + lua_set_table_index (cd->L, "name", elt->mx.name); + lua_pushstring (cd->L, "priority"); + lua_pushnumber (cd->L, elt->mx.priority); + lua_settable (cd->L, -3); + + lua_rawseti (cd->L, -2, ++i); + cur = g_list_next (cur); + } + lua_pushnil (cd->L); + + } else { lua_pushnil (cd->L); lua_pushstring (cd->L, "Unknown reply type"); @@ -288,6 +308,21 @@ lua_dns_resolver_resolve_txt (lua_State *L) return 1; } +static int +lua_dns_resolver_resolve_mx (lua_State *L) +{ + struct rspamd_dns_resolver *dns_resolver = lua_check_dns_resolver (L); + + if (dns_resolver) { + return lua_dns_resolver_resolve_common (L, dns_resolver, DNS_REQUEST_MX); + } + else { + lua_pushnil (L); + } + + return 1; +} + gint luaopen_dns_resolver (lua_State * L) { diff --git a/src/main.c b/src/main.c index a4240aa9c..a9bf5c7e3 100644 --- a/src/main.c +++ b/src/main.c @@ -299,17 +299,14 @@ drop_priv (struct rspamd_main *rspamd) } static void -config_logger (struct rspamd_main *rspamd, GQuark type, gboolean is_fatal) +config_logger (struct config_file *cfg, gpointer ud) { - rspamd_set_logger (rspamd->cfg->log_type, type, rspamd); - if (open_log_priv (rspamd->logger, rspamd->workers_uid, rspamd->workers_gid) == -1) { - if (is_fatal) { - fprintf (stderr, "Fatal error, cannot open logfile, exiting\n"); - exit (EXIT_FAILURE); - } - else { - msg_err ("cannot log to file, logfile unaccessable"); - } + struct rspamd_main *rm = ud; + + rspamd_set_logger (cfg, g_quark_try_string ("main"), rm); + if (open_log_priv (rm->logger, rm->workers_uid, rm->workers_gid) == -1) { + fprintf (stderr, "Fatal error, cannot open logfile, exiting\n"); + exit (EXIT_FAILURE); } } @@ -366,7 +363,6 @@ reread_config (struct rspamd_main *rspamd) gchar *cfg_file; GList *l; struct filter *filt; - GQuark type; tmp_cfg = (struct config_file *)g_malloc (sizeof (struct config_file)); if (tmp_cfg) { @@ -380,6 +376,7 @@ reread_config (struct rspamd_main *rspamd) memory_pool_add_destructor (tmp_cfg->cfg_pool, (pool_destruct_func)lua_close, tmp_cfg->lua_state); if (! load_rspamd_config (tmp_cfg, FALSE)) { + rspamd_set_logger (rspamd_main->cfg, g_quark_try_string ("main"), rspamd_main); msg_err ("cannot parse new config file, revert to old one"); free_config (tmp_cfg); } @@ -393,12 +390,11 @@ reread_config (struct rspamd_main *rspamd) if (is_debug) { rspamd->cfg->log_level = G_LOG_LEVEL_DEBUG; } - type = g_quark_try_string ("main"); - config_logger (rspamd, type, FALSE); /* Pre-init of cache */ rspamd->cfg->cache = g_new0 (struct symbols_cache, 1); rspamd->cfg->cache->static_pool = memory_pool_new (memory_pool_get_size ()); rspamd->cfg->cache->cfg = rspamd->cfg; + rspamd_main->cfg->cache->items_by_symbol = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); /* Perform modules configuring */ l = g_list_first (rspamd->cfg->filters); @@ -723,7 +719,8 @@ load_rspamd_config (struct config_file *cfg, gboolean init_modules) struct filter *filt; struct module_ctx *cur_module = NULL; - if (! read_rspamd_config (cfg, cfg->cfg_name, convert_config)) { + if (! read_rspamd_config (cfg, cfg->cfg_name, convert_config, + config_logger, rspamd_main)) { return FALSE; } @@ -996,10 +993,6 @@ main (gint argc, gchar **argv, gchar **env) GList *l; worker_t **pworker; GQuark type; -#ifdef HAVE_OPENSSL - gchar rand_bytes[sizeof (guint32)]; - guint32 rand_seed; -#endif #ifdef HAVE_SA_SIGINFO signals_info = g_queue_new (); @@ -1038,7 +1031,7 @@ main (gint argc, gchar **argv, gchar **env) rspamd_main->cfg->log_level = G_LOG_LEVEL_DEBUG; } else { - rspamd_main->cfg->log_level = G_LOG_LEVEL_INFO; + rspamd_main->cfg->log_level = G_LOG_LEVEL_WARNING; } type = g_quark_from_static_string ("main"); @@ -1054,23 +1047,16 @@ main (gint argc, gchar **argv, gchar **env) #ifdef HAVE_OPENSSL ERR_load_crypto_strings (); - /* Init random generator */ - if (RAND_bytes (rand_bytes, sizeof (rand_bytes)) != 1) { - msg_err ("cannot seed random generator using openssl: %s, using time", ERR_error_string (ERR_get_error (), NULL)); - g_random_set_seed (time (NULL)); - } - else { - memcpy (&rand_seed, rand_bytes, sizeof (guint32)); - g_random_set_seed (rand_seed); - } - OpenSSL_add_all_algorithms (); OpenSSL_add_all_digests (); OpenSSL_add_all_ciphers (); #endif + rspamd_prng_seed (); + /* First set logger to console logger */ - rspamd_set_logger (RSPAMD_LOG_CONSOLE, type, rspamd_main); + rspamd_main->cfg->log_type = RSPAMD_LOG_CONSOLE; + rspamd_set_logger (rspamd_main->cfg, type, rspamd_main); (void)open_log (rspamd_main->logger); g_log_set_default_handler (rspamd_glib_log_function, rspamd_main->logger); @@ -1159,8 +1145,6 @@ main (gint argc, gchar **argv, gchar **env) rlim.rlim_cur = 100 * 1024 * 1024; setrlimit (RLIMIT_STACK, &rlim); - config_logger (rspamd_main, type, TRUE); - /* Create rolling history */ rspamd_main->history = rspamd_roll_history_new (rspamd_main->server_pool); diff --git a/src/main.h b/src/main.h index 29c8a9f0c..e3a13ee65 100644 --- a/src/main.h +++ b/src/main.h @@ -122,20 +122,11 @@ struct process_exception { }; /** - * Union that would be used for storing sockaddrs - */ -union sa_union { - struct sockaddr_storage ss; - struct sockaddr_in s4; - struct sockaddr_in6 s6; -}; - -/** * Control session object */ struct controller_command; struct controller_session; -typedef void (*controller_func_t)(gchar **args, struct controller_session *session); +typedef gboolean (*controller_func_t)(gchar **args, struct controller_session *session); struct controller_session { struct rspamd_worker *worker; /**< pointer to worker structure (controller in fact) */ @@ -168,7 +159,7 @@ struct controller_session { f_str_t *learn_buf; /**< learn input */ GList *parts; /**< extracted mime parts */ gint in_class; /**< positive or negative learn */ - void (*other_handler)(struct controller_session *session, + gboolean (*other_handler)(struct controller_session *session, f_str_t *in); /**< other command handler to execute at the end of processing */ void *other_data; /**< and its data */ controller_func_t custom_handler; /**< custom command handler */ diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c index b79ecc452..ba07acd2f 100644 --- a/src/plugins/fuzzy_check.c +++ b/src/plugins/fuzzy_check.c @@ -60,72 +60,81 @@ #define DEFAULT_PORT 11335 struct storage_server { - struct upstream up; - gchar *name; - gchar *addr; - guint16 port; + struct upstream up; + gchar *name; + gchar *addr; + guint16 port; }; struct fuzzy_mapping { - guint64 fuzzy_flag; - const gchar *symbol; + guint64 fuzzy_flag; + const gchar *symbol; double weight; }; struct fuzzy_mime_type { - gchar *type; - gchar *subtype; + gchar *type; + gchar *subtype; +}; + +struct fuzzy_rule { + struct storage_server *servers; + gint servers_num; + const gchar *symbol; + GHashTable *mappings; + GList *mime_types; + double max_score; + gboolean read_only; + gboolean skip_unknown; }; struct fuzzy_ctx { - gint (*filter) (struct worker_task * task); - const gchar *symbol; - struct storage_server *servers; - gint servers_num; - memory_pool_t *fuzzy_pool; - double max_score; - guint32 min_hash_len; - radix_tree_t *whitelist; - GHashTable *mappings; - GList *mime_types; - guint32 min_bytes; - guint32 min_height; - guint32 min_width; - guint32 io_timeout; + gint (*filter) (struct worker_task * task); + memory_pool_t *fuzzy_pool; + GList *fuzzy_rules; + const gchar *default_symbol; + guint32 min_hash_len; + radix_tree_t *whitelist; + guint32 min_bytes; + guint32 min_height; + guint32 min_width; + guint32 io_timeout; }; struct fuzzy_client_session { - gint state; - fuzzy_hash_t *h; - struct event ev; - struct timeval tv; - struct worker_task *task; - struct storage_server *server; - gint fd; + gint state; + fuzzy_hash_t *h; + struct event ev; + struct timeval tv; + struct worker_task *task; + struct storage_server *server; + struct fuzzy_rule *rule; + gint fd; }; struct fuzzy_learn_session { - struct event ev; - fuzzy_hash_t *h; - gint cmd; - gint value; - gint flag; - gint *saved; - GError **err; - struct timeval tv; - struct controller_session *session; - struct storage_server *server; - struct worker_task *task; - gint fd; + struct event ev; + fuzzy_hash_t *h; + gint cmd; + gint value; + gint flag; + gint *saved; + GError **err; + struct timeval tv; + struct controller_session *session; + struct storage_server *server; + struct fuzzy_rule *rule; + struct worker_task *task; + gint fd; }; -static struct fuzzy_ctx *fuzzy_module_ctx = NULL; -static const gchar hex_digits[] = "0123456789abcdef"; +static struct fuzzy_ctx *fuzzy_module_ctx = NULL; +static const gchar hex_digits[] = "0123456789abcdef"; -static gint fuzzy_mime_filter (struct worker_task *task); -static void fuzzy_symbol_callback (struct worker_task *task, void *unused); -static void fuzzy_add_handler (gchar **args, struct controller_session *session); -static void fuzzy_delete_handler (gchar **args, struct controller_session *session); +static void fuzzy_symbol_callback (struct worker_task *task, void *unused); +static gboolean fuzzy_add_handler (gchar **args, struct controller_session *session); +static gboolean fuzzy_delete_handler (gchar **args, + struct controller_session *session); /* Initialization */ gint fuzzy_check_module_init (struct config_file *cfg, struct module_ctx **ctx); @@ -139,82 +148,36 @@ module_t fuzzy_check_module = { fuzzy_check_module_reconfig }; -/* Flags string is in format <numeric_flag>:<SYMBOL>:weight[, <numeric_flag>:<SYMBOL>:weight...] */ static void -parse_flags_string_old (struct config_file *cfg, const gchar *str) -{ - gchar **strvec, *item, *err_str, **map_str; - gint num, i, t; - struct fuzzy_mapping *map; - - strvec = g_strsplit_set (str, ", ;", 0); - num = g_strv_length (strvec); - - for (i = 0; i < num; i ++) { - item = strvec[i]; - map_str = g_strsplit_set (item, ":", 3); - t = g_strv_length (map_str); - if (t != 3 && t != 2) { - msg_err ("invalid fuzzy mapping: %s", item); - } - else { - map = memory_pool_alloc (fuzzy_module_ctx->fuzzy_pool, sizeof (struct fuzzy_mapping)); - map->symbol = memory_pool_strdup (fuzzy_module_ctx->fuzzy_pool, map_str[1]); - - errno = 0; - map->fuzzy_flag = strtol (map_str[0], &err_str, 10); - if (errno != 0 || (err_str && *err_str != '\0')) { - msg_info ("cannot parse flag %s: %s", map_str[0], strerror (errno)); - continue; - } - else if (t == 2) { - /* Weight is skipped in definition */ - map->weight = fuzzy_module_ctx->max_score; - } - else { - map->weight = strtol (map_str[2], &err_str, 10); - - } - /* Add flag to hash table */ - g_hash_table_insert (fuzzy_module_ctx->mappings, GINT_TO_POINTER(map->fuzzy_flag), map); - register_virtual_symbol (&cfg->cache, map->symbol, map->weight); - } - g_strfreev (map_str); - } - - g_strfreev (strvec); -} - -static void -parse_flags_string (struct config_file *cfg, ucl_object_t *val) +parse_flags (struct fuzzy_rule *rule, struct config_file *cfg, ucl_object_t *val) { ucl_object_t *elt; struct fuzzy_mapping *map; const gchar *sym = NULL; if (val->type == UCL_STRING) { - parse_flags_string_old (cfg, ucl_obj_tostring (val)); + msg_err ("string mappings are deprecated and no longer supported, use new style configuration"); } else if (val->type == UCL_OBJECT) { - elt = ucl_obj_get_key (val, "symbol"); + elt = ucl_object_find_key (val, "symbol"); if (elt == NULL || !ucl_object_tostring_safe (elt, &sym)) { sym = ucl_object_key (val); } if (sym != NULL) { map = memory_pool_alloc (fuzzy_module_ctx->fuzzy_pool, sizeof (struct fuzzy_mapping)); map->symbol = sym; - elt = ucl_obj_get_key (val, "flag"); + elt = ucl_object_find_key (val, "flag"); if (elt != NULL && ucl_obj_toint_safe (elt, &map->fuzzy_flag)) { - elt = ucl_obj_get_key (val, "weight"); + elt = ucl_object_find_key (val, "max_score"); if (elt != NULL) { map->weight = ucl_obj_todouble (elt); } else { - map->weight = fuzzy_module_ctx->max_score; + map->weight = rule->max_score; } /* Add flag to hash table */ - g_hash_table_insert (fuzzy_module_ctx->mappings, GINT_TO_POINTER (map->fuzzy_flag), map); - register_virtual_symbol (&cfg->cache, map->symbol, map->weight); + g_hash_table_insert (rule->mappings, GINT_TO_POINTER (map->fuzzy_flag), map); + register_virtual_symbol (&cfg->cache, map->symbol, 1.0); } else { msg_err ("fuzzy_map parameter has no flag definition"); @@ -261,12 +224,12 @@ parse_mime_types (const gchar *str) } static gboolean -fuzzy_check_content_type (GMimeContentType *type) +fuzzy_check_content_type (struct fuzzy_rule *rule, GMimeContentType *type) { struct fuzzy_mime_type *ft; GList *cur; - cur = fuzzy_module_ctx->mime_types; + cur = rule->mime_types; while (cur) { ft = cur->data; if (g_mime_content_type_is_type (type, ft->type, ft->subtype)) { @@ -279,7 +242,7 @@ fuzzy_check_content_type (GMimeContentType *type) } static void -parse_servers_string (const gchar *str) +parse_servers_string (struct fuzzy_rule *rule, const gchar *str) { gchar **strvec; gint i, num; @@ -288,18 +251,18 @@ parse_servers_string (const gchar *str) strvec = g_strsplit_set (str, ",", 0); num = g_strv_length (strvec); - fuzzy_module_ctx->servers = memory_pool_alloc0 (fuzzy_module_ctx->fuzzy_pool, sizeof (struct storage_server) * num); + rule->servers = memory_pool_alloc0 (fuzzy_module_ctx->fuzzy_pool, sizeof (struct storage_server) * num); for (i = 0; i < num; i++) { g_strstrip (strvec[i]); - cur = &fuzzy_module_ctx->servers[fuzzy_module_ctx->servers_num]; + cur = &rule->servers[rule->servers_num]; if (parse_host_port (fuzzy_module_ctx->fuzzy_pool, strvec[i], &cur->addr, &cur->port)) { if (cur->port == 0) { cur->port = DEFAULT_PORT; } cur->name = memory_pool_strdup (fuzzy_module_ctx->fuzzy_pool, strvec[i]); - fuzzy_module_ctx->servers_num++; + rule->servers_num++; } } @@ -310,32 +273,25 @@ parse_servers_string (const gchar *str) static double fuzzy_normalize (gint32 in, double weight) { - double ms = weight, ams = fabs (ms), ain = fabs (in); - - if (ams > 0.001) { - if (ain < ams / 2.) { - return in; - } - else if (ain < ams * 2.) { - ain = ain / 3. + ams / 3.; - return in > 0 ? ain : -(ain); - } - else { - return in > 0 ? ms : -(ms); - } + if (weight == 0) { + return 0; } - - return (double)in; +#ifdef HAVE_TANH + return tanh (G_E * (double)in / weight); +#else + return (in < weight ? in / weight : weight); +#endif } static const gchar * fuzzy_to_string (fuzzy_hash_t *h) { static gchar strbuf [FUZZY_HASHLEN * 2 + 1]; + const int max_print = 5; gint i; guint8 byte; - for (i = 0; i < FUZZY_HASHLEN; i ++) { + for (i = 0; i < max_print; i ++) { byte = h->hash_pipe[i]; if (byte == '\0') { break; @@ -343,22 +299,102 @@ fuzzy_to_string (fuzzy_hash_t *h) strbuf[i * 2] = hex_digits[byte >> 4]; strbuf[i * 2 + 1] = hex_digits[byte & 0xf]; } - - strbuf[i * 2] = '\0'; + if (i == max_print) { + memcpy (&strbuf[i * 2], "...", 4); + } + else { + strbuf[i * 2] = '\0'; + } return strbuf; } +static struct fuzzy_rule * +fuzzy_rule_new (const char *default_symbol, memory_pool_t *pool) +{ + struct fuzzy_rule *rule; + + rule = memory_pool_alloc0 (pool, sizeof (struct fuzzy_rule)); + + rule->mappings = g_hash_table_new (g_direct_hash, g_direct_equal); + rule->symbol = default_symbol; + memory_pool_add_destructor (pool, (pool_destruct_func)g_hash_table_unref, rule->mappings); + rule->read_only = FALSE; + + return rule; +} + +static gint +fuzzy_parse_rule (struct config_file *cfg, ucl_object_t *obj) +{ + ucl_object_t *value, *cur; + struct fuzzy_rule *rule; + ucl_object_iter_t it = NULL; + + if (obj->type != UCL_OBJECT) { + msg_err ("invalid rule definition"); + return -1; + } + + rule = fuzzy_rule_new (fuzzy_module_ctx->default_symbol, fuzzy_module_ctx->fuzzy_pool); + + if ((value = ucl_object_find_key (obj, "mime_types")) != NULL) { + if (value->type == UCL_ARRAY) { + value = value->value.av; + } + LL_FOREACH (value, cur) { + rule->mime_types = g_list_concat (rule->mime_types, + parse_mime_types (ucl_obj_tostring (cur))); + } + } + + if ((value = ucl_object_find_key (obj, "max_score")) != NULL) { + rule->max_score = ucl_obj_todouble (value); + } + if ((value = ucl_object_find_key (obj, "symbol")) != NULL) { + rule->symbol = ucl_obj_tostring (value); + } + if ((value = ucl_object_find_key (obj, "read_only")) != NULL) { + rule->read_only = ucl_obj_toboolean (value); + } + if ((value = ucl_object_find_key (obj, "skip_unknown")) != NULL) { + rule->skip_unknown = ucl_obj_toboolean (value); + } + + if ((value = ucl_object_find_key (obj, "servers")) != NULL) { + if (value->type == UCL_ARRAY) { + value = value->value.av; + } + LL_FOREACH (value, cur) { + parse_servers_string (rule, ucl_obj_tostring (cur)); + } + } + if ((value = ucl_object_find_key (obj, "fuzzy_map")) != NULL) { + while ((cur = ucl_iterate_object (value, &it, true)) != NULL) { + parse_flags (rule, cfg, cur); + } + } + + if (rule->servers_num == 0) { + msg_err ("no servers defined for fuzzy rule with symbol: %s", rule->symbol); + return -1; + } + else { + fuzzy_module_ctx->fuzzy_rules = g_list_prepend (fuzzy_module_ctx->fuzzy_rules, rule); + if (rule->symbol != fuzzy_module_ctx->default_symbol) { + register_virtual_symbol (&cfg->cache, rule->symbol, 1.0); + } + } + + return 0; +} + gint fuzzy_check_module_init (struct config_file *cfg, struct module_ctx **ctx) { fuzzy_module_ctx = g_malloc0 (sizeof (struct fuzzy_ctx)); - fuzzy_module_ctx->filter = fuzzy_mime_filter; fuzzy_module_ctx->fuzzy_pool = memory_pool_new (memory_pool_get_size ()); - fuzzy_module_ctx->servers = NULL; - fuzzy_module_ctx->servers_num = 0; - fuzzy_module_ctx->mappings = g_hash_table_new (g_direct_hash, g_direct_equal); *ctx = (struct module_ctx *)fuzzy_module_ctx; @@ -369,20 +405,13 @@ gint fuzzy_check_module_config (struct config_file *cfg) { ucl_object_t *value, *cur; - ucl_object_iter_t it = NULL; - gint res = TRUE; + gint res = TRUE; if ((value = get_module_opt (cfg, "fuzzy_check", "symbol")) != NULL) { - fuzzy_module_ctx->symbol = ucl_obj_tostring (value); - } - else { - fuzzy_module_ctx->symbol = DEFAULT_SYMBOL; - } - if ((value = get_module_opt (cfg, "fuzzy_check", "max_score")) != NULL) { - fuzzy_module_ctx->max_score = ucl_obj_todouble (value); + fuzzy_module_ctx->default_symbol = ucl_obj_tostring (value); } else { - fuzzy_module_ctx->max_score = 0.; + fuzzy_module_ctx->default_symbol = DEFAULT_SYMBOL; } if ((value = get_module_opt (cfg, "fuzzy_check", "min_length")) != NULL) { @@ -415,11 +444,6 @@ fuzzy_check_module_config (struct config_file *cfg) else { fuzzy_module_ctx->io_timeout = DEFAULT_IO_TIMEOUT; } - if ((value = get_module_opt (cfg, "fuzzy_check", "mime_types")) != NULL) { - LL_FOREACH (value, cur) { - fuzzy_module_ctx->mime_types = parse_mime_types (ucl_obj_tostring (cur)); - } - } if ((value = get_module_opt (cfg, "fuzzy_check", "whitelist")) != NULL) { fuzzy_module_ctx->whitelist = radix_tree_create (); @@ -433,21 +457,24 @@ fuzzy_check_module_config (struct config_file *cfg) fuzzy_module_ctx->whitelist = NULL; } - if ((value = get_module_opt (cfg, "fuzzy_check", "servers")) != NULL) { + if ((value = get_module_opt (cfg, "fuzzy_check", "rule")) != NULL) { LL_FOREACH (value, cur) { - parse_servers_string (ucl_obj_tostring (cur)); - } - } - if ((value = get_module_opt (cfg, "fuzzy_check", "fuzzy_map")) != NULL) { - while ((cur = ucl_iterate_object (value, &it, true)) != NULL) { - parse_flags_string (cfg, cur); + if (fuzzy_parse_rule (cfg, cur) == -1) { + return -1; + } } } - register_symbol (&cfg->cache, fuzzy_module_ctx->symbol, fuzzy_module_ctx->max_score, fuzzy_symbol_callback, NULL); + if (fuzzy_module_ctx->fuzzy_rules != NULL) { + register_callback_symbol (&cfg->cache, fuzzy_module_ctx->default_symbol, + 1.0, fuzzy_symbol_callback, NULL); - register_custom_controller_command ("fuzzy_add", fuzzy_add_handler, TRUE, TRUE); - register_custom_controller_command ("fuzzy_del", fuzzy_delete_handler, TRUE, TRUE); + register_custom_controller_command ("fuzzy_add", fuzzy_add_handler, TRUE, TRUE); + register_custom_controller_command ("fuzzy_del", fuzzy_delete_handler, TRUE, TRUE); + } + else { + msg_warn ("fuzzy module is enabled but no rules are defined"); + } return res; } @@ -456,11 +483,9 @@ gint fuzzy_check_module_reconfig (struct config_file *cfg) { memory_pool_delete (fuzzy_module_ctx->fuzzy_pool); - fuzzy_module_ctx->servers = NULL; - fuzzy_module_ctx->servers_num = 0; + fuzzy_module_ctx->fuzzy_pool = memory_pool_new (memory_pool_get_size ()); - g_hash_table_remove_all (fuzzy_module_ctx->mappings); return fuzzy_check_module_config (cfg); } @@ -518,21 +543,24 @@ fuzzy_io_callback (gint fd, short what, void *arg) } *err_str = '\0'; /* Get mapping by flag */ - if ((map = g_hash_table_lookup (fuzzy_module_ctx->mappings, GINT_TO_POINTER (flag))) == NULL) { + if ((map = g_hash_table_lookup (session->rule->mappings, GINT_TO_POINTER (flag))) == NULL) { /* Default symbol and default weight */ - symbol = fuzzy_module_ctx->symbol; - nval = fuzzy_normalize (value, fuzzy_module_ctx->max_score); + symbol = session->rule->symbol; + nval = fuzzy_normalize (value, session->rule->max_score); } else { /* Get symbol and weight from map */ symbol = map->symbol; nval = fuzzy_normalize (value, map->weight); } - msg_info ("<%s>, found fuzzy hash '%s' with weight: %.2f, in list: %d", - session->task->message_id, fuzzy_to_string (session->h), flag, nval); - rspamd_snprintf (buf, sizeof (buf), "%d: %d / %.2f", flag, value, nval); - insert_result (session->task, symbol, nval, g_list_prepend (NULL, + msg_info ("<%s>, found fuzzy hash '%s' with weight: %.2f, in list: %s:%d%s", + session->task->message_id, fuzzy_to_string (session->h), nval, symbol, + flag, map == NULL ? "(unknown)" : ""); + if (map != NULL || !session->rule->skip_unknown) { + rspamd_snprintf (buf, sizeof (buf), "%d: %d / %.2f", flag, value, nval); + insert_result_single (session->task, symbol, nval, g_list_prepend (NULL, memory_pool_strdup (session->task->task_pool, buf))); + } } goto ok; } @@ -544,7 +572,7 @@ fuzzy_io_callback (gint fd, short what, void *arg) return; err: - msg_err ("got error on IO with server %s:%d, %d, %s", session->server->name, session->server->port, errno, strerror (errno)); + msg_err ("got error on IO with server %s, %d, %s", session->server->name, errno, strerror (errno)); ok: remove_normal_event (session->task->s, fuzzy_io_fin, session); } @@ -564,8 +592,10 @@ fuzzy_learn_callback (gint fd, short what, void *arg) struct fuzzy_learn_session *session = arg; struct fuzzy_cmd cmd; gchar buf[512]; + const gchar *cmd_name; gint r; + cmd_name = (session->cmd == FUZZY_WRITE ? "add" : "delete"); if (what == EV_WRITE) { /* Send command to storage */ cmd.blocksize = session->h->block_size; @@ -575,7 +605,9 @@ fuzzy_learn_callback (gint fd, short what, void *arg) cmd.flag = session->flag; if (write (fd, &cmd, sizeof (struct fuzzy_cmd)) == -1) { if (*(session->err) == NULL) { - g_set_error (session->err, g_quark_from_static_string ("fuzzy check"), 404, "write socket error: %s", strerror (errno)); + g_set_error (session->err, + g_quark_from_static_string ("fuzzy check"), + errno, "write socket error: %s", strerror (errno)); } goto err; } @@ -587,34 +619,46 @@ fuzzy_learn_callback (gint fd, short what, void *arg) } else if (what == EV_READ) { if (read (fd, buf, sizeof (buf)) == -1) { - msg_info ("cannot add fuzzy hash for message <%s>", session->task->message_id); + msg_info ("cannot %s fuzzy hash for message <%s>, list %s:%d", cmd_name, + session->task->message_id, session->rule->symbol, session->flag); if (*(session->err) == NULL) { - g_set_error (session->err, g_quark_from_static_string ("fuzzy check"), 404, "read socket error: %s", strerror (errno)); + g_set_error (session->err, + g_quark_from_static_string ("fuzzy check"), + errno, "read socket error: %s", strerror (errno)); } goto err; } else if (buf[0] == 'O' && buf[1] == 'K') { - msg_info ("added fuzzy hash '%s' to list: %d for message <%s>", - fuzzy_to_string (session->h), session->flag, session->task->message_id); + msg_info ("%s fuzzy hash '%s', list: %s:%d for message <%s>", cmd_name, + fuzzy_to_string (session->h), session->rule->symbol, + session->flag, session->task->message_id); goto ok; } else { - msg_info ("cannot add fuzzy hash for message <%s>", session->task->message_id); + msg_info ("cannot %s fuzzy hash '%s' for message <%s>, list %s:%d", cmd_name, + fuzzy_to_string (session->h), session->task->message_id, + session->rule->symbol, session->flag); if (*(session->err) == NULL) { - g_set_error (session->err, g_quark_from_static_string ("fuzzy check"), 500, "add fuzzy error"); + g_set_error (session->err, + g_quark_from_static_string ("fuzzy check"), EINVAL, "%s fuzzy error", cmd_name); } goto ok; } } else { errno = ETIMEDOUT; + if (*(session->err) == NULL) { + g_set_error (session->err, + g_quark_from_static_string ("fuzzy check"), EINVAL, "%s fuzzy, IO timeout", cmd_name); + } goto err; } return; err: - msg_err ("got error in IO with server %s:%d, %d, %s", session->server->name, session->server->port, errno, strerror (errno)); + msg_err ("got error in IO with server %s, %d, %s", + session->server->name, errno, strerror (errno)); ok: if (--(*(session->saved)) == 0) { session->session->state = STATE_REPLY; @@ -648,7 +692,7 @@ ok: } static inline void -register_fuzzy_call (struct worker_task *task, fuzzy_hash_t *h) +register_fuzzy_call (struct worker_task *task, struct fuzzy_rule *rule, fuzzy_hash_t *h) { struct fuzzy_client_session *session; struct storage_server *selected; @@ -656,11 +700,12 @@ register_fuzzy_call (struct worker_task *task, fuzzy_hash_t *h) /* Get upstream */ #ifdef HAVE_CLOCK_GETTIME - selected = (struct storage_server *)get_upstream_by_hash (fuzzy_module_ctx->servers, fuzzy_module_ctx->servers_num, + selected = (struct storage_server *)get_upstream_by_hash (rule->servers, rule->servers_num, sizeof (struct storage_server), task->ts.tv_sec, - DEFAULT_UPSTREAM_ERROR_TIME, DEFAULT_UPSTREAM_DEAD_TIME, DEFAULT_UPSTREAM_MAXERRORS, h->hash_pipe, sizeof (h->hash_pipe)); + DEFAULT_UPSTREAM_ERROR_TIME, DEFAULT_UPSTREAM_DEAD_TIME, DEFAULT_UPSTREAM_MAXERRORS, + h->hash_pipe, sizeof (h->hash_pipe)); #else - selected = (struct storage_server *)get_upstream_by_hash (fuzzy_module_ctx->servers, fuzzy_module_ctx->servers_num, + selected = (struct storage_server *)get_upstream_by_hash (rule->servers, rule->servers_num, sizeof (struct storage_server), task->tv.tv_sec, DEFAULT_UPSTREAM_ERROR_TIME, DEFAULT_UPSTREAM_DEAD_TIME, DEFAULT_UPSTREAM_MAXERRORS, h->hash_pipe, sizeof (h->hash_pipe)); #endif @@ -678,15 +723,15 @@ register_fuzzy_call (struct worker_task *task, fuzzy_hash_t *h) session->task = task; session->fd = sock; session->server = selected; + session->rule = rule; event_add (&session->ev, &session->tv); register_async_event (task->s, fuzzy_io_fin, session, g_quark_from_static_string ("fuzzy check")); } } } -/* This callback is called when we check message via fuzzy hashes storage */ static void -fuzzy_symbol_callback (struct worker_task *task, void *unused) +fuzzy_check_rule (struct worker_task *task, struct fuzzy_rule *rule) { struct mime_text_part *part; struct mime_part *mime_part; @@ -696,25 +741,6 @@ fuzzy_symbol_callback (struct worker_task *task, void *unused) GList *cur; fuzzy_hash_t *fake_fuzzy; - - /* Check whitelist */ -#ifdef HAVE_INET_PTON - if (fuzzy_module_ctx->whitelist && !task->from_addr.ipv6 && task->from_addr.d.in4.s_addr != INADDR_NONE) { - if (radix32tree_find (fuzzy_module_ctx->whitelist, ntohl ((guint32) task->from_addr.d.in4.s_addr)) != RADIX_NO_VALUE) { - msg_info ("<%s>, address %s is whitelisted, skip fuzzy check", - task->message_id, inet_ntoa (task->from_addr.d.in4)); - return; - } - } -#else - if (fuzzy_module_ctx->whitelist && task->from_addr.s_addr != 0) { - if (radix32tree_find (fuzzy_module_ctx->whitelist, ntohl ((guint32) task->from_addr.s_addr)) != RADIX_NO_VALUE) { - msg_info ("<%s>, address %s is whitelisted, skip fuzzy check", - task->message_id, inet_ntoa (task->from_addr)); - return; - } - } -#endif cur = task->text_parts; while (cur) { @@ -747,8 +773,8 @@ fuzzy_symbol_callback (struct worker_task *task, void *unused) continue; } - register_fuzzy_call (task, part->fuzzy); - register_fuzzy_call (task, part->double_fuzzy); + register_fuzzy_call (task, rule, part->fuzzy); + register_fuzzy_call (task, rule, part->double_fuzzy); cur = g_list_next (cur); } @@ -763,7 +789,7 @@ fuzzy_symbol_callback (struct worker_task *task, void *unused) /* Construct fake fuzzy hash */ fake_fuzzy = memory_pool_alloc0 (task->task_pool, sizeof (fuzzy_hash_t)); rspamd_strlcpy (fake_fuzzy->hash_pipe, checksum, sizeof (fake_fuzzy->hash_pipe)); - register_fuzzy_call (task, fake_fuzzy); + register_fuzzy_call (task, rule, fake_fuzzy); g_free (checksum); } } @@ -774,13 +800,14 @@ fuzzy_symbol_callback (struct worker_task *task, void *unused) cur = task->parts; while (cur) { mime_part = cur->data; - if (mime_part->content->len > 0 && fuzzy_check_content_type (mime_part->type)) { + if (mime_part->content->len > 0 && fuzzy_check_content_type (rule, mime_part->type)) { if (fuzzy_module_ctx->min_bytes <= 0 || mime_part->content->len >= fuzzy_module_ctx->min_bytes) { - checksum = g_compute_checksum_for_data (G_CHECKSUM_MD5, mime_part->content->data, mime_part->content->len); + checksum = g_compute_checksum_for_data (G_CHECKSUM_MD5, + mime_part->content->data, mime_part->content->len); /* Construct fake fuzzy hash */ fake_fuzzy = memory_pool_alloc0 (task->task_pool, sizeof (fuzzy_hash_t)); rspamd_strlcpy (fake_fuzzy->hash_pipe, checksum, sizeof (fake_fuzzy->hash_pipe)); - register_fuzzy_call (task, fake_fuzzy); + register_fuzzy_call (task, rule, fake_fuzzy); g_free (checksum); } } @@ -788,41 +815,64 @@ fuzzy_symbol_callback (struct worker_task *task, void *unused) } } +/* This callback is called when we check message via fuzzy hashes storage */ +static void +fuzzy_symbol_callback (struct worker_task *task, void *unused) +{ + struct fuzzy_rule *rule; + GList *cur; + + /* Check whitelist */ +#ifdef HAVE_INET_PTON + if (fuzzy_module_ctx->whitelist && !task->from_addr.ipv6 && task->from_addr.d.in4.s_addr != INADDR_NONE) { + if (radix32tree_find (fuzzy_module_ctx->whitelist, ntohl ((guint32) task->from_addr.d.in4.s_addr)) != RADIX_NO_VALUE) { + msg_info ("<%s>, address %s is whitelisted, skip fuzzy check", + task->message_id, inet_ntoa (task->from_addr.d.in4)); + return; + } + } +#else + if (fuzzy_module_ctx->whitelist && task->from_addr.s_addr != 0) { + if (radix32tree_find (fuzzy_module_ctx->whitelist, ntohl ((guint32) task->from_addr.s_addr)) != RADIX_NO_VALUE) { + msg_info ("<%s>, address %s is whitelisted, skip fuzzy check", + task->message_id, inet_ntoa (task->from_addr)); + return; + } + } +#endif + + cur = fuzzy_module_ctx->fuzzy_rules; + while (cur) { + rule = cur->data; + fuzzy_check_rule (task, rule); + cur = g_list_next (cur); + } +} + static inline gboolean -register_fuzzy_controller_call (struct controller_session *session, struct worker_task *task, fuzzy_hash_t *h, - gint cmd, gint value, gint flag, gint *saved, GError **err) +register_fuzzy_controller_call (struct controller_session *session, + struct fuzzy_rule *rule, struct worker_task *task, fuzzy_hash_t *h, + gint cmd, gint value, gint flag, gint *saved, GError **err) { struct fuzzy_learn_session *s; struct storage_server *selected; - gint sock, r; - gchar out_buf[BUFSIZ]; + gint sock; /* Get upstream */ #ifdef HAVE_CLOCK_GETTIME - selected = (struct storage_server *)get_upstream_by_hash (fuzzy_module_ctx->servers, fuzzy_module_ctx->servers_num, + selected = (struct storage_server *)get_upstream_by_hash (rule->servers, rule->servers_num, sizeof (struct storage_server), task->ts.tv_sec, - DEFAULT_UPSTREAM_ERROR_TIME, DEFAULT_UPSTREAM_DEAD_TIME, DEFAULT_UPSTREAM_MAXERRORS, h->hash_pipe, sizeof (h->hash_pipe)); + DEFAULT_UPSTREAM_ERROR_TIME, DEFAULT_UPSTREAM_DEAD_TIME, DEFAULT_UPSTREAM_MAXERRORS, + h->hash_pipe, sizeof (h->hash_pipe)); #else - selected = (struct storage_server *)get_upstream_by_hash (fuzzy_module_ctx->servers, fuzzy_module_ctx->servers_num, + selected = (struct storage_server *)get_upstream_by_hash (rule->servers, rule->servers_num, sizeof (struct storage_server), task->tv.tv_sec, - DEFAULT_UPSTREAM_ERROR_TIME, DEFAULT_UPSTREAM_DEAD_TIME, DEFAULT_UPSTREAM_MAXERRORS, h->hash_pipe, sizeof (h->hash_pipe)); + DEFAULT_UPSTREAM_ERROR_TIME, DEFAULT_UPSTREAM_DEAD_TIME, DEFAULT_UPSTREAM_MAXERRORS, + h->hash_pipe, sizeof (h->hash_pipe)); #endif if (selected) { /* Create UDP socket */ if ((sock = make_universal_socket (selected->addr, selected->port, SOCK_DGRAM, TRUE, FALSE, FALSE)) == -1) { - msg_warn ("cannot connect to %s, %d, %s", selected->name, errno, strerror (errno)); - session->state = STATE_REPLY; - if (session->restful) { - r = rspamd_snprintf (out_buf, sizeof (out_buf), "HTTP/1.0 404 No hashes have been written" CRLF CRLF); - } - else { - r = rspamd_snprintf (out_buf, sizeof (out_buf), "no hashes have been written" CRLF "END" CRLF); - } - if (! rspamd_dispatcher_write (session->dispatcher, out_buf, r, FALSE, FALSE)) { - return FALSE; - } - free_task (task, FALSE); - rspamd_dispatcher_restore (session->dispatcher); return FALSE; } else { @@ -841,27 +891,120 @@ register_fuzzy_controller_call (struct controller_session *session, struct worke s->saved = saved; s->fd = sock; s->err = err; + s->rule = rule; event_add (&s->ev, &s->tv); (*saved)++; register_async_event (session->s, fuzzy_learn_fin, s, g_quark_from_static_string ("fuzzy check")); return TRUE; } } + return FALSE; } -static void -fuzzy_process_handler (struct controller_session *session, f_str_t * in) +static gboolean +fuzzy_process_rule (struct controller_session *session, struct fuzzy_rule *rule, + struct worker_task *task, GError **err, gint cmd, gint flag, gint value, gint *saved) { - struct worker_task *task; struct mime_text_part *part; struct mime_part *mime_part; struct rspamd_image *image; - GList *cur; - GError **err; - gint r, cmd = 0, value = 0, flag = 0, *saved, *sargs; - gchar out_buf[BUFSIZ], *checksum; + GList *cur; + gchar *checksum; fuzzy_hash_t fake_fuzzy; + gboolean processed = FALSE; + + /* Plan new event for writing */ + cur = task->text_parts; + + while (cur) { + part = cur->data; + if (part->is_empty || part->fuzzy == NULL || part->fuzzy->hash_pipe[0] == '\0' || + (fuzzy_module_ctx->min_bytes > 0 && part->content->len < fuzzy_module_ctx->min_bytes)) { + /* Skip empty parts */ + cur = g_list_next (cur); + continue; + } + if (! register_fuzzy_controller_call (session, rule, task, + part->fuzzy, cmd, value, flag, saved, err)) { + return FALSE; + } + if (! register_fuzzy_controller_call (session, rule, task, + part->double_fuzzy, cmd, value, flag, saved, err)) { + /* Cannot write hash */ + return FALSE; + } + processed = TRUE; + cur = g_list_next (cur); + } + + /* Process images */ + cur = task->images; + while (cur) { + image = cur->data; + if (image->data->len > 0) { + if (fuzzy_module_ctx->min_height <= 0 || image->height >= fuzzy_module_ctx->min_height) { + if (fuzzy_module_ctx->min_width <= 0 || image->width >= fuzzy_module_ctx->min_width) { + checksum = g_compute_checksum_for_data (G_CHECKSUM_MD5, image->data->data, image->data->len); + /* Construct fake fuzzy hash */ + fake_fuzzy.block_size = 0; + memset (fake_fuzzy.hash_pipe, 0, sizeof (fake_fuzzy.hash_pipe)); + rspamd_strlcpy (fake_fuzzy.hash_pipe, checksum, sizeof (fake_fuzzy.hash_pipe)); + if (! register_fuzzy_controller_call (session, rule, task, + &fake_fuzzy, cmd, value, flag, saved, err)) { + g_free (checksum); + return FALSE; + } + + msg_info ("save hash of image: [%s] to list: %d", checksum, flag); + g_free (checksum); + processed = TRUE; + } + } + } + cur = g_list_next (cur); + } + /* Process other parts */ + cur = task->parts; + while (cur) { + mime_part = cur->data; + if (mime_part->content->len > 0 && fuzzy_check_content_type (rule, mime_part->type)) { + if (fuzzy_module_ctx->min_bytes <= 0 || mime_part->content->len >= fuzzy_module_ctx->min_bytes) { + checksum = g_compute_checksum_for_data (G_CHECKSUM_MD5, + mime_part->content->data, mime_part->content->len); + /* Construct fake fuzzy hash */ + fake_fuzzy.block_size = 0; + memset (fake_fuzzy.hash_pipe, 0, sizeof (fake_fuzzy.hash_pipe)); + rspamd_strlcpy (fake_fuzzy.hash_pipe, checksum, sizeof (fake_fuzzy.hash_pipe)); + if (! register_fuzzy_controller_call (session, rule, task, + &fake_fuzzy, cmd, value, flag, saved, err)) { + return FALSE; + } + msg_info ("save hash of part of type: %s/%s: [%s] to list %d", + mime_part->type->type, mime_part->type->subtype, + checksum, flag); + g_free (checksum); + processed = TRUE; + } + } + cur = g_list_next (cur); + } + + memory_pool_add_destructor (session->session_pool, (pool_destruct_func)free_task_soft, task); + + return processed; +} + +static gboolean +fuzzy_process_handler (struct controller_session *session, f_str_t * in) +{ + struct fuzzy_rule *rule; + gboolean processed = FALSE, res = TRUE; + GList *cur; + struct worker_task *task; + GError **err; + gint r, cmd = 0, value = 0, flag = 0, *saved, *sargs; + gchar out_buf[BUFSIZ]; /* Extract arguments */ if (session->other_data) { @@ -870,17 +1013,17 @@ fuzzy_process_handler (struct controller_session *session, f_str_t * in) value = sargs[1]; flag = sargs[2]; } - + /* Prepare task */ task = construct_task (session->worker); session->other_data = task; session->state = STATE_WAIT; - + /* Allocate message from string */ task->msg = memory_pool_alloc (task->task_pool, sizeof (f_str_t)); task->msg->begin = in->begin; task->msg->len = in->len; - + saved = memory_pool_alloc0 (session->session_pool, sizeof (gint)); err = memory_pool_alloc0 (session->session_pool, sizeof (GError *)); @@ -899,132 +1042,33 @@ fuzzy_process_handler (struct controller_session *session, f_str_t * in) msg_warn ("write error"); } rspamd_dispatcher_restore (session->dispatcher); - return; + return FALSE; } - else { - /* Plan new event for writing */ - cur = task->text_parts; + cur = fuzzy_module_ctx->fuzzy_rules; + while (cur && res) { + rule = cur->data; - while (cur) { - part = cur->data; - if (part->is_empty || part->fuzzy == NULL || part->fuzzy->hash_pipe[0] == '\0' || - (fuzzy_module_ctx->min_bytes > 0 && part->content->len < fuzzy_module_ctx->min_bytes)) { - /* Skip empty parts */ - cur = g_list_next (cur); - continue; - } - if (! register_fuzzy_controller_call (session, task, part->fuzzy, cmd, value, flag, saved, err)) { - /* Cannot write hash */ - session->state = STATE_REPLY; - if (session->restful) { - r = rspamd_snprintf (out_buf, sizeof (out_buf), "HTTP/1.0 500 Cannot write fuzzy hash" CRLF CRLF); - } - else { - r = rspamd_snprintf (out_buf, sizeof (out_buf), "cannot write fuzzy hash" CRLF "END" CRLF); - } - if (! rspamd_dispatcher_write (session->dispatcher, out_buf, r, FALSE, FALSE)) { - return; - } - rspamd_dispatcher_restore (session->dispatcher); - free_task (task, FALSE); - return; - } - if (! register_fuzzy_controller_call (session, task, part->double_fuzzy, cmd, value, flag, saved, err)) { - /* Cannot write hash */ - session->state = STATE_REPLY; - if (session->restful) { - r = rspamd_snprintf (out_buf, sizeof (out_buf), "HTTP/1.0 500 Cannot write fuzzy hash" CRLF CRLF); - } - else { - r = rspamd_snprintf (out_buf, sizeof (out_buf), "cannot write fuzzy hash" CRLF "END" CRLF); - } - if (! rspamd_dispatcher_write (session->dispatcher, out_buf, r, FALSE, FALSE)) { - return; - } - free_task (task, FALSE); - rspamd_dispatcher_restore (session->dispatcher); - return; - } + if (rule->read_only) { cur = g_list_next (cur); + continue; } - /* Process images */ - cur = task->images; - while (cur) { - image = cur->data; - if (image->data->len > 0) { - if (fuzzy_module_ctx->min_height <= 0 || image->height >= fuzzy_module_ctx->min_height) { - if (fuzzy_module_ctx->min_width <= 0 || image->width >= fuzzy_module_ctx->min_width) { - checksum = g_compute_checksum_for_data (G_CHECKSUM_MD5, image->data->data, image->data->len); - /* Construct fake fuzzy hash */ - fake_fuzzy.block_size = 0; - bzero (fake_fuzzy.hash_pipe, sizeof (fake_fuzzy.hash_pipe)); - rspamd_strlcpy (fake_fuzzy.hash_pipe, checksum, sizeof (fake_fuzzy.hash_pipe)); - if (! register_fuzzy_controller_call (session, task, &fake_fuzzy, cmd, value, flag, saved, err)) { - /* Cannot write hash */ - session->state = STATE_REPLY; - if (session->restful) { - r = rspamd_snprintf (out_buf, sizeof (out_buf), "HTTP/1.0 500 Cannot write fuzzy hash" CRLF CRLF); - } - else { - r = rspamd_snprintf (out_buf, sizeof (out_buf), "cannot write fuzzy hash" CRLF "END" CRLF); - } - g_free (checksum); - free_task (task, FALSE); - if (! rspamd_dispatcher_write (session->dispatcher, out_buf, r, FALSE, FALSE)) { - return; - } - rspamd_dispatcher_restore (session->dispatcher); - return; - } - - msg_info ("save hash of image: [%s] to list: %d", checksum, flag); - g_free (checksum); - } - } - } + + /* Check for flag */ + if (g_hash_table_lookup (rule->mappings, GINT_TO_POINTER (flag)) == NULL) { cur = g_list_next (cur); + continue; } - /* Process other parts */ - cur = task->parts; - while (cur) { - mime_part = cur->data; - if (mime_part->content->len > 0 && fuzzy_check_content_type (mime_part->type)) { - if (fuzzy_module_ctx->min_bytes <= 0 || mime_part->content->len >= fuzzy_module_ctx->min_bytes) { - checksum = g_compute_checksum_for_data (G_CHECKSUM_MD5, mime_part->content->data, mime_part->content->len); - /* Construct fake fuzzy hash */ - fake_fuzzy.block_size = 0; - bzero (fake_fuzzy.hash_pipe, sizeof (fake_fuzzy.hash_pipe)); - rspamd_strlcpy (fake_fuzzy.hash_pipe, checksum, sizeof (fake_fuzzy.hash_pipe)); - if (! register_fuzzy_controller_call (session, task, &fake_fuzzy, cmd, value, flag, saved, err)) { - /* Cannot write hash */ - session->state = STATE_REPLY; - if (session->restful) { - r = rspamd_snprintf (out_buf, sizeof (out_buf), "HTTP/1.0 500 Cannot write fuzzy hash" CRLF CRLF); - } - else { - r = rspamd_snprintf (out_buf, sizeof (out_buf), "cannot write fuzzy hash" CRLF "END" CRLF); - } - g_free (checksum); - free_task (task, FALSE); - if (! rspamd_dispatcher_write (session->dispatcher, out_buf, r, FALSE, FALSE)) { - return; - } - rspamd_dispatcher_restore (session->dispatcher); - return; - } - msg_info ("save hash of part of type: %s/%s: [%s] to list %d", - mime_part->type->type, mime_part->type->subtype, - checksum, flag); - g_free (checksum); - } - } - cur = g_list_next (cur); + + res = fuzzy_process_rule (session, rule, task, err, cmd, flag, value, saved); + + if (res) { + processed = TRUE; } - } - memory_pool_add_destructor (session->session_pool, (pool_destruct_func)free_task_soft, task); + cur = g_list_next (cur); + } - if (*saved == 0) { + if (!res) { session->state = STATE_REPLY; if (session->restful) { r = rspamd_snprintf (out_buf, sizeof (out_buf), "HTTP/1.0 404 No hashes have been written" CRLF CRLF); @@ -1033,13 +1077,28 @@ fuzzy_process_handler (struct controller_session *session, f_str_t * in) r = rspamd_snprintf (out_buf, sizeof (out_buf), "no hashes have been written" CRLF "END" CRLF); } if (! rspamd_dispatcher_write (session->dispatcher, out_buf, r, FALSE, FALSE)) { - return; + return FALSE; } - rspamd_dispatcher_restore (session->dispatcher); + return FALSE; + } + else if (!processed) { + session->state = STATE_REPLY; + if (session->restful) { + r = rspamd_snprintf (out_buf, sizeof (out_buf), "HTTP/1.0 404 No fuzzy rules matched" CRLF CRLF); + } + else { + r = rspamd_snprintf (out_buf, sizeof (out_buf), "no fuzzy rules matched" CRLF "END" CRLF); + } + if (! rspamd_dispatcher_write (session->dispatcher, out_buf, r, FALSE, FALSE)) { + return FALSE; + } + return FALSE; } + + return TRUE; } -static void +static gboolean fuzzy_controller_handler (gchar **args, struct controller_session *session, gint cmd) { gchar *arg, out_buf[BUFSIZ], *err_str; @@ -1053,22 +1112,22 @@ fuzzy_controller_handler (gchar **args, struct controller_session *session, gint msg_info ("empty content length"); r = rspamd_snprintf (out_buf, sizeof (out_buf), "HTTP/1.0 500 Fuzzy command requires Content-Length" CRLF CRLF); if (! rspamd_dispatcher_write (session->dispatcher, out_buf, r, FALSE, FALSE)) { - return; + return FALSE; } session->state = STATE_REPLY; rspamd_dispatcher_restore (session->dispatcher); - return; + return FALSE; } errno = 0; size = strtoul (arg, &err_str, 10); if (errno != 0 || (err_str && *err_str != '\0')) { r = rspamd_snprintf (out_buf, sizeof (out_buf), "HTTP/1.0 500 Learn size is invalid" CRLF CRLF); if (! rspamd_dispatcher_write (session->dispatcher, out_buf, r, FALSE, FALSE)) { - return; + return FALSE; } session->state = STATE_REPLY; rspamd_dispatcher_restore (session->dispatcher); - return; + return FALSE; } arg = g_hash_table_lookup (session->kwargs, "value"); if (arg) { @@ -1096,20 +1155,20 @@ fuzzy_controller_handler (gchar **args, struct controller_session *session, gint msg_info ("empty content length"); r = rspamd_snprintf (out_buf, sizeof (out_buf), "fuzzy command requires length as argument" CRLF "END" CRLF); if (! rspamd_dispatcher_write (session->dispatcher, out_buf, r, FALSE, FALSE)) { - return; + return FALSE; } session->state = STATE_REPLY; - return; + return FALSE; } errno = 0; size = strtoul (arg, &err_str, 10); if (errno != 0 || (err_str && *err_str != '\0')) { r = rspamd_snprintf (out_buf, sizeof (out_buf), "learn size is invalid" CRLF); if (! rspamd_dispatcher_write (session->dispatcher, out_buf, r, FALSE, FALSE)) { - return; + return FALSE; } session->state = STATE_REPLY; - return; + return FALSE; } /* Process value */ arg = args[1]; @@ -1142,23 +1201,18 @@ fuzzy_controller_handler (gchar **args, struct controller_session *session, gint sargs[1] = value; sargs[2] = flag; session->other_data = sargs; + + return TRUE; } -static void +static gboolean fuzzy_add_handler (gchar **args, struct controller_session *session) { - fuzzy_controller_handler (args, session, FUZZY_WRITE); + return fuzzy_controller_handler (args, session, FUZZY_WRITE); } -static void +static gboolean fuzzy_delete_handler (gchar **args, struct controller_session *session) { - fuzzy_controller_handler (args, session, FUZZY_DEL); -} - -static gint -fuzzy_mime_filter (struct worker_task *task) -{ - /* XXX: remove this */ - return 0; + return fuzzy_controller_handler (args, session, FUZZY_DEL); } diff --git a/src/plugins/lua/emails.lua b/src/plugins/lua/emails.lua index 8b4ba5970..162809541 100644 --- a/src/plugins/lua/emails.lua +++ b/src/plugins/lua/emails.lua @@ -5,40 +5,16 @@ -- symbol = sym2, dnsbl = bl.somehost.com, domain_only = no local rules = {} -function split(str, delim, maxNb) - -- Eliminate bad cases... - if string.find(str, delim) == nil then - return { str } - end - if maxNb == nil or maxNb < 1 then - maxNb = 0 -- No limit - end - local result = {} - local pat = "(.-)" .. delim .. "()" - local nb = 0 - local lastPos - for part, pos in string.gmatch(str, pat) do - nb = nb + 1 - result[nb] = part - lastPos = pos - if nb == maxNb then break end - end - -- Handle the last field - if nb ~= maxNb then - result[nb + 1] = string.sub(str, lastPos) - end - return result -end - -function emails_dns_cb(task, to_resolve, results, err, symbol) - if results then - rspamd_logger.info(string.format('<%s> email: [%s] resolved for symbol: %s', task:get_message():get_message_id(), to_resolve, symbol)) - task:insert_result(symbol, 1) - end -end - -- Check rule for a single email -function check_email_rule(task, rule, addr) +local function check_email_rule(task, rule, addr) + local function emails_dns_cb(resolver, to_resolve, results, err) + task:inc_dns_req() + if results then + rspamd_logger.info(string.format('<%s> email: [%s] resolved for symbol: %s', + task:get_message():get_message_id(), to_resolve, rule['symbol'])) + task:insert_result(rule['symbol'], 1) + end + end if rule['dnsbl'] then local to_resolve = '' if rule['domain_only'] then @@ -46,26 +22,29 @@ function check_email_rule(task, rule, addr) else to_resolve = string.format('%s.%s.%s', addr:get_user(), addr:get_host(), rule['dnsbl']) end - task:resolve_dns_a(to_resolve, 'emails_dns_cb', rule['symbol']) + task:get_resolver():resolve_a(task:get_session(), task:get_mempool(), + to_resolve, emails_dns_cb) elseif rule['map'] then if rule['domain_only'] then local key = addr:get_host() if rule['map']:get_key(key) then task:insert_result(rule['symbol'], 1) - rspamd_logger.info(string.format('<%s> email: \'%s\' is found in list: %s', task:get_message():get_message_id(), key, rule['symbol'])) + rspamd_logger.info(string.format('<%s> email: \'%s\' is found in list: %s', + task:get_message():get_message_id(), key, rule['symbol'])) end else local key = string.format('%s@%s', addr:get_user(), addr:get_host()) if rule['map']:get_key(key) then task:insert_result(rule['symbol'], 1) - rspamd_logger.info(string.format('<%s> email: \'%s\' is found in list: %s', task:get_message():get_message_id(), key, rule['symbol'])) + rspamd_logger.info(string.format('<%s> email: \'%s\' is found in list: %s', + task:get_message():get_message_id(), key, rule['symbol'])) end end end end -- Check email -function check_emails(task) +local function check_emails(task) local emails = task:get_emails() local checked = {} if emails then @@ -81,40 +60,6 @@ function check_emails(task) end end --- Add rule to ruleset -local function add_emails_rule(key, obj) - local newrule = { - name = nil, - dnsbl = nil, - map = nil, - domain_only = false, - symbol = key - } - for name,value in pairs(obj) do - if name == 'dnsbl' then - newrule['dnsbl'] = value - newrule['name'] = value - elseif name == 'map' then - newrule['name'] = value - newrule['map'] = rspamd_config:add_hash_map (newrule['name']) - elseif name == 'symbol' then - newrule['symbol'] = value - elseif name == 'domain_only' then - newrule['domain_only'] = value - else - rspamd_logger.err('invalid rule option: '.. name) - return nil - end - - end - if not newrule['symbol'] or (not newrule['map'] and not newrule['dnsbl']) then - rspamd_logger.err('incomplete rule') - return nil - end - table.insert(rules, newrule) - return newrule -end - -- Registration if type(rspamd_config.get_api_version) ~= 'nil' then @@ -125,19 +70,23 @@ if type(rspamd_config.get_api_version) ~= 'nil' then end end -local opts = rspamd_config:get_all_opt('emails') -if opts then - for k,m in pairs(opts) do - if type(m) ~= 'table' then - rspamd_logger.err('parameter ' .. k .. ' is invalid, must be an object') - else - local rule = add_emails_rule(k, m) - if not rule then - rspamd_logger.err('cannot add rule: "'..k..'"') +local opts = rspamd_config:get_all_opt('emails', 'rule') +if opts and type(opts) == 'table' then + for k,v in pairs(opts) do + if k == 'rule' and type(v) == 'table' then + local rule = v + if not rule['symbol'] then + rule['symbol'] = k + end + if rule['map'] then + rule['name'] = rule['map'] + rule['map'] = rspamd_config:add_hash_map (rule['name']) + end + if not rule['symbol'] or (not rule['map'] and not rule['dnsbl']) then + rspamd_logger.err('incomplete rule') else - if type(rspamd_config.get_api_version) ~= 'nil' then - rspamd_config:register_virtual_symbol(rule['symbol'], 1.0) - end + table.insert(rules, rule) + rspamd_config:register_virtual_symbol(rule['symbol'], 1.0) end end end @@ -146,8 +95,8 @@ end if table.maxn(rules) > 0 then -- add fake symbol to check all maps inside a single callback if type(rspamd_config.get_api_version) ~= 'nil' then - rspamd_config:register_callback_symbol('EMAILS', 1.0, 'check_emails') + rspamd_config:register_callback_symbol('EMAILS', 1.0, check_emails) else - rspamd_config:register_symbol('EMAILS', 1.0, 'check_emails') + rspamd_config:register_symbol('EMAILS', 1.0, check_emails) end end diff --git a/src/plugins/lua/forged_recipients.lua b/src/plugins/lua/forged_recipients.lua index a46776f91..9ad47a259 100644 --- a/src/plugins/lua/forged_recipients.lua +++ b/src/plugins/lua/forged_recipients.lua @@ -37,7 +37,7 @@ function check_forged_headers(task) end -- Check sender local smtp_from = task:get_from() - if smtp_form then + if smtp_from then local mime_from = task:get_from_headers() if not mime_from or not (string.lower(mime_from[1]['addr']) == string.lower(smtp_from[1]['addr'])) then task:insert_result(symbol_sender, 1) diff --git a/src/plugins/lua/multimap.lua b/src/plugins/lua/multimap.lua index 229d594cf..1711c212e 100644 --- a/src/plugins/lua/multimap.lua +++ b/src/plugins/lua/multimap.lua @@ -66,7 +66,7 @@ local function check_multimap(task) end elseif rule['type'] == 'dnsbl' then local ip = task:get_from_ip() - if ip and ip ~= "0.0.0.0" then + if ip and ip:to_string() ~= "0.0.0.0" then if ip:get_version() == 6 and rule['ipv6'] then task:get_resolver():resolve_a(task:get_session(), task:get_mempool(), ip_to_rbl(ip, rule['map']), multimap_rbl_cb, rule['map']) diff --git a/src/plugins/lua/once_received.lua b/src/plugins/lua/once_received.lua index 403646489..e2d4496ac 100644 --- a/src/plugins/lua/once_received.lua +++ b/src/plugins/lua/once_received.lua @@ -6,33 +6,34 @@ local symbol_strict = nil local bad_hosts = {} local good_hosts = {} -function recv_dns_cb(task, to_resolve, results, err) - if not results then - task:insert_result(symbol_strict, 1) - else - rspamd_logger.info(string.format('SMTP resolver failed to resolve: %s is %s', to_resolve, results[1])) - local i = true - for _,h in ipairs(bad_hosts) do - if string.find(results[1], h) then - -- Check for good hostname - if good_hosts then - for _,gh in ipairs(good_hosts) do - if string.find(results[1], gh) then - i = false - break +local function check_quantity_received (task) + local function recv_dns_cb(resolver, to_resolve, results, err) + task:inc_dns_req() + if not results then + task:insert_result(symbol_strict, 1) + else + rspamd_logger.info(string.format('SMTP resolver failed to resolve: %s is %s', to_resolve, results[1])) + local i = true + for _,h in ipairs(bad_hosts) do + if string.find(results[1], h) then + -- Check for good hostname + if good_hosts then + for _,gh in ipairs(good_hosts) do + if string.find(results[1], gh) then + i = false + break + end end end - end - if i then - task:insert_result(symbol_strict, 1, h) - return + if i then + task:insert_result(symbol_strict, 1, h) + return + end end end end end -end -function check_quantity_received (task) local recvh = task:get_received_headers() if table.maxn(recvh) <= 1 then task:insert_result(symbol, 1) @@ -43,10 +44,13 @@ function check_quantity_received (task) return end -- Unresolved host - if not r['real_hostname'] or string.lower(r['real_hostname']) == 'unknown' or string.match(r['real_hostname'], '^%d+%.%d+%.%d+%.%d+$') then + if not r['real_hostname'] or string.lower(r['real_hostname']) == 'unknown' or + string.match(r['real_hostname'], '^%d+%.%d+%.%d+%.%d+$') then + if r['real_ip'] then -- Try to resolve it again - task:resolve_dns_ptr(r['real_ip'], 'recv_dns_cb') + task:get_resolver():resolve_ptr(task:get_session(), task:get_mempool(), + tostring(r['real_ip']), recv_dns_cb) else task:insert_result(symbol_strict, 1) end @@ -91,7 +95,7 @@ end local opts = rspamd_config:get_all_opt('once_received') if opts then if opts['symbol'] then - symbol = opts['symbol'] + local symbol = opts['symbol'] for n,v in pairs(opts) do if n == 'symbol_strict' then @@ -115,6 +119,6 @@ if opts then end -- Register symbol's callback - rspamd_config:register_symbol(symbol, 1.0, 'check_quantity_received') + rspamd_config:register_symbol(symbol, 1.0, check_quantity_received) end end diff --git a/src/plugins/lua/rbl.lua b/src/plugins/lua/rbl.lua index cb827e854..e7e720908 100644 --- a/src/plugins/lua/rbl.lua +++ b/src/plugins/lua/rbl.lua @@ -78,7 +78,7 @@ local function rbl_cb (task) end local rip = task:get_from_ip() - if(rip:to_string() ~= "0.0.0.0") then + if rip and (rip:to_string() ~= '0.0.0.0') then for k,rbl in pairs(rbls) do if (rip:get_version() == 6 and rbl['ipv6'] and rbl['from']) or (rip:get_version() == 4 and rbl['ipv4'] and rbl['from']) then @@ -89,7 +89,7 @@ local function rbl_cb (task) end local recvh = task:get_received_headers() for _,rh in ipairs(recvh) do - if rh['real_ip'] then + if rh['real_ip'] and rh['real_ip']:to_string() ~= '0.0.0.0' then for k,rbl in pairs(rbls) do if (rh['real_ip']:get_version() == 6 and rbl['ipv6'] and rbl['received']) or (rh['real_ip']:get_version() == 4 and rbl['ipv4'] and rbl['received']) then diff --git a/src/plugins/lua/whitelist.lua b/src/plugins/lua/whitelist.lua index 0f4c41f85..7a196ec89 100644 --- a/src/plugins/lua/whitelist.lua +++ b/src/plugins/lua/whitelist.lua @@ -22,10 +22,14 @@ function check_whitelist (task) -- check client's from domain local from = task:get_from() if from then - local _,_,domain = string.find(from, '@(.+)>?$') - local key = h:get_key(domain) - if key then - task:insert_result(symbol_from, 1) + local from_addr = from[1]['addr'] + + if from_addr then + local _,_,domain = string.find(from_addr, '@(.+)>?$') + local key = h:get_key(domain) + if key then + task:insert_result(symbol_from, 1) + end end end end diff --git a/src/printf.c b/src/printf.c index b03e1475f..d72ec95c8 100644 --- a/src/printf.c +++ b/src/printf.c @@ -169,128 +169,136 @@ rspamd_sprintf_num (gchar *buf, gchar *last, guint64 ui64, gchar zero, return ((gchar *)memcpy (buf, p, len)) + len; } -gint -rspamd_fprintf (FILE *f, const gchar *fmt, ...) +struct rspamd_printf_char_buf { + char *begin; + char *pos; + glong remain; +}; + +static glong +rspamd_printf_append_char (const gchar *buf, glong buflen, gpointer ud) { - va_list args; - gchar buf[BUFSIZ]; - gint r; + struct rspamd_printf_char_buf *dst = (struct rspamd_printf_char_buf *)ud; + glong wr; - va_start (args, fmt); - rspamd_vsnprintf (buf, sizeof (buf), fmt, args); - va_end (args); + if (dst->remain <= 0) { + return dst->remain; + } - r = fprintf (f, "%s", buf); + wr = MIN (dst->remain, buflen); + memcpy (dst->pos, buf, wr); + dst->remain -= wr; + dst->pos += wr; - return r; + return wr; } -gint -rspamd_log_fprintf (FILE *f, const gchar *fmt, ...) +static glong +rspamd_printf_append_file (const gchar *buf, glong buflen, gpointer ud) +{ + FILE *dst = (FILE *)ud; + + return fwrite (buf, 1, buflen, dst); +} + +static glong +rspamd_printf_append_gstring (const gchar *buf, glong buflen, gpointer ud) +{ + GString *dst = (GString *)ud; + + g_string_append_len (dst, buf, buflen); + + return buflen; +} + +glong +rspamd_fprintf (FILE *f, const gchar *fmt, ...) { va_list args; - gchar buf[BUFSIZ]; - gint r; + glong r; va_start (args, fmt); - rspamd_vsnprintf (buf, sizeof (buf), fmt, args); + r = rspamd_vprintf_common (rspamd_printf_append_file, f, fmt, args); va_end (args); - r = fprintf (f, "%s\n", buf); - fflush (f); - - return r; + return r; } -gint -rspamd_sprintf (gchar *buf, const gchar *fmt, ...) +glong +rspamd_log_fprintf (FILE *f, const gchar *fmt, ...) { - gchar *p; va_list args; + glong r; va_start (args, fmt); - p = rspamd_vsnprintf (buf, /* STUB */ 65536, fmt, args); + r = rspamd_vprintf_common (rspamd_printf_append_file, f, fmt, args); va_end (args); - return p - buf; + fflush (f); + + return r; } -gint +glong rspamd_snprintf (gchar *buf, glong max, const gchar *fmt, ...) { - gchar *p; - va_list args; + gchar *r; + va_list args; va_start (args, fmt); - p = rspamd_vsnprintf (buf, max - 1, fmt, args); + r = rspamd_vsnprintf (buf, max, fmt, args); va_end (args); - *p = '\0'; - return p - buf; + return (r - buf); } gchar * -rspamd_escape_string (gchar *dst, const gchar *src, glong len) +rspamd_vsnprintf (gchar *buf, glong max, const gchar *fmt, va_list args) { - gchar *buf = dst, *last = dst + len; - guint8 c; - const gchar *p = src; - gunichar uc; + struct rspamd_printf_char_buf dst; - if (len <= 0) { - return dst; - } + dst.begin = buf; + dst.pos = dst.begin; + dst.remain = max - 1; + (void)rspamd_vprintf_common (rspamd_printf_append_char, &dst, fmt, args); + *dst.pos = '\0'; - while (*p && buf < last) { - /* Detect utf8 */ - uc = g_utf8_get_char_validated (p, last - buf); - if (uc > 0) { - c = g_unichar_to_utf8 (uc, buf); - buf += c; - p += c; - } - else { - c = *p ++; - if (G_UNLIKELY ((c & 0x80))) { - c &= 0x7F; - if (last - buf >= 3) { - *buf++ = 'M'; - *buf++ = '-'; - } - } - if (G_UNLIKELY ( g_ascii_iscntrl (c))) { - if (c == '\n') { - *buf++ = ' '; - } - else if (c == '\t') { - *buf++ = '\t'; - } - else { - *buf++ = '^'; - if (buf != last) { - *buf++ = c ^ 0100; - } - } - } - else { - *buf++ = c; - } - } - } + return dst.pos; +} - *buf = '\0'; +glong +rspamd_printf_gstring (GString *s, const gchar *fmt, ...) +{ + va_list args; + glong r; - return buf; + va_start (args, fmt); + r = rspamd_vprintf_common (rspamd_printf_append_gstring, s, fmt, args); + va_end (args); + + return r; } -gchar * -rspamd_vsnprintf (gchar *buf, glong max, const gchar *fmt, va_list args) +#define RSPAMD_PRINTF_APPEND(buf, len) \ + do { \ + wr = func ((buf), (len), apd); \ + if (wr <= 0) { \ + goto oob; \ + } \ + written += wr; \ + fmt ++; \ + buf_start = fmt; \ + } while(0) + +glong +rspamd_vprintf_common (rspamd_printf_append_func func, gpointer apd, const gchar *fmt, va_list args) { - gchar *p, zero, *last; + gchar zero, numbuf[G_ASCII_DTOSTR_BUF_SIZE], *p, *last, c; + const gchar *buf_start = fmt; gint d; long double f, scale; - size_t len, slen; + glong written = 0, wr, slen; gint64 i64; guint64 ui64; guint width, sign, hex, humanize, bytes, frac_width, i; @@ -298,13 +306,7 @@ rspamd_vsnprintf (gchar *buf, glong max, const gchar *fmt, va_list args) GString *gs; gboolean bv; - if (max <= 0) { - return buf; - } - - last = buf + max; - - while (*fmt && buf < last) { + while (*fmt) { /* * "buf < last" means that we could copy at least one character: @@ -313,6 +315,15 @@ rspamd_vsnprintf (gchar *buf, glong max, const gchar *fmt, va_list args) if (*fmt == '%') { + /* Append what we have in buf */ + if (fmt > buf_start) { + wr = func (buf_start, fmt - buf_start, apd); + if (wr <= 0) { + goto oob; + } + written += wr; + } + i64 = 0; ui64 = 0; @@ -323,7 +334,7 @@ rspamd_vsnprintf (gchar *buf, glong max, const gchar *fmt, va_list args) bytes = 0; humanize = 0; frac_width = 0; - slen = (size_t) -1; + slen = -1; while (*fmt >= '0' && *fmt <= '9') { width = width * 10 + *fmt++ - '0'; @@ -376,10 +387,10 @@ rspamd_vsnprintf (gchar *buf, glong max, const gchar *fmt, va_list args) case '*': d = (gint)va_arg (args, gint); if (G_UNLIKELY (d < 0)) { - msg_err ("crititcal error: size is less than 0"); - g_assert (0); + msg_err ("critical error: size is less than 0"); + return 0; } - slen = (size_t)d; + slen = (glong)d; fmt++; continue; @@ -395,61 +406,28 @@ rspamd_vsnprintf (gchar *buf, glong max, const gchar *fmt, va_list args) case 'V': v = va_arg (args, f_str_t *); - - len = v->len; - len = (buf + len < last) ? len : (size_t) (last - buf); - - buf = ((gchar *)memcpy (buf, v->begin, len)) + len; - fmt++; + RSPAMD_PRINTF_APPEND (v->begin, v->len); continue; case 'v': gs = va_arg (args, GString *); - len = gs->len; - len = (buf + len < last) ? len : (size_t) (last - buf); - - buf = ((gchar *)memcpy (buf, gs->str, len)) + len; - fmt++; - break; - - case 's': - p = va_arg(args, gchar *); - if (p == NULL) { - p = "(NULL)"; - } - - if (slen == (size_t) -1) { - while (*p && buf < last) { - *buf++ = *p++; - } - - } else { - len = (buf + slen < last) ? slen : (size_t) (last - buf); - - buf = ((gchar *)memcpy (buf, p, len)) + len; - } - - fmt++; + RSPAMD_PRINTF_APPEND (gs->str, gs->len); continue; - case 'S': - p = va_arg(args, gchar *); + case 's': + p = va_arg (args, gchar *); if (p == NULL) { p = "(NULL)"; } - if (slen == (size_t) -1) { - buf = rspamd_escape_string (buf, p, last - buf); - - } else { - len = (buf + slen < last) ? slen : (size_t) (last - buf); - - buf = rspamd_escape_string (buf, p, len); + if (slen == -1) { + /* NULL terminated string */ + slen = strlen (p); } - fmt++; + RSPAMD_PRINTF_APPEND (p, slen); continue; @@ -510,57 +488,28 @@ rspamd_vsnprintf (gchar *buf, glong max, const gchar *fmt, va_list args) case 'f': - f = (double) va_arg (args, double); - if (f < 0) { - *buf++ = '-'; - f = -f; + case 'F': + if (*fmt == 'f') { + f = (long double) va_arg (args, double); } - - ui64 = (gint64) f; - - buf = rspamd_sprintf_num (buf, last, ui64, zero, 0, width); - - if (frac_width) { - - if (buf < last) { - *buf++ = '.'; - } - - scale = 1.0; - - for (i = 0; i < frac_width; i++) { - scale *= 10.0; - } - - /* - * (gint64) cast is required for msvc6: - * it can not convert guint64 to double - */ - ui64 = (guint64) ((f - (gint64) ui64) * scale); - - buf = rspamd_sprintf_num (buf, last, ui64, '0', 0, frac_width); + else { + f = (long double) va_arg (args, long double); } - - fmt++; - - continue; - - case 'F': - f = (long double) va_arg (args, long double); - + p = numbuf; + last = p + sizeof (numbuf); if (f < 0) { - *buf++ = '-'; + *p++ = '-'; f = -f; } ui64 = (gint64) f; - buf = rspamd_sprintf_num (buf, last, ui64, zero, 0, width); + p = rspamd_sprintf_num (p, last, ui64, zero, 0, width); if (frac_width) { - if (buf < last) { - *buf++ = '.'; + if (p < last) { + *p++ = '.'; } scale = 1.0; @@ -575,50 +524,32 @@ rspamd_vsnprintf (gchar *buf, glong max, const gchar *fmt, va_list args) */ ui64 = (guint64) ((f - (gint64) ui64) * scale); - buf = rspamd_sprintf_num (buf, last, ui64, '0', 0, frac_width); + p = rspamd_sprintf_num (p, last, ui64, '0', 0, frac_width); } - fmt++; + slen = p - numbuf; + RSPAMD_PRINTF_APPEND (numbuf, slen); continue; case 'g': - f = (long double) va_arg (args, double); - - if (f < 0) { - *buf++ = '-'; - f = -f; - } - g_ascii_formatd (buf, last - buf, "%g", (double)f); - buf += strlen (buf); - fmt++; - - continue; - - case 'b': - bv = (gboolean) va_arg (args, double); - if (bv) { - len = MIN (last - buf, 4); - memcpy (buf, "true", len); + case 'G': + if (*fmt == 'g') { + f = (long double) va_arg (args, double); } else { - len = MIN (last - buf, 5); - memcpy (buf, "false", len); + f = (long double) va_arg (args, long double); } - fmt++; - continue; + g_ascii_formatd (numbuf, sizeof (numbuf), "%g", (double)f); + slen = strlen (numbuf); + RSPAMD_PRINTF_APPEND (numbuf, slen); - case 'G': - f = (long double) va_arg (args, long double); + continue; - if (f < 0) { - *buf++ = '-'; - f = -f; - } - g_ascii_formatd (buf, last - buf, "%g", (double)f); - buf += strlen (buf); - fmt++; + case 'b': + bv = (gboolean) va_arg (args, double); + RSPAMD_PRINTF_APPEND (bv ? "true" : "false", bv ? 4 : 5); continue; @@ -631,39 +562,43 @@ rspamd_vsnprintf (gchar *buf, glong max, const gchar *fmt, va_list args) break; case 'c': - d = va_arg (args, gint); - *buf++ = (gchar) (d & 0xff); - fmt++; + c = va_arg (args, gint); + c &= 0xff; + RSPAMD_PRINTF_APPEND (&c, 1); continue; case 'Z': - *buf++ = '\0'; - fmt++; + c = '\0'; + RSPAMD_PRINTF_APPEND (&c, 1); continue; case 'N': - *buf++ = LF; - fmt++; + c = LF; + RSPAMD_PRINTF_APPEND (&c, 1); continue; case '%': - *buf++ = '%'; - fmt++; + c = '%'; + RSPAMD_PRINTF_APPEND (&c, 1); continue; default: - *buf++ = *fmt++; + c = *fmt; + RSPAMD_PRINTF_APPEND (&c, 1); continue; } + /* Print number */ + p = numbuf; + last = p + sizeof (numbuf); if (sign) { if (i64 < 0) { - *buf++ = '-'; + *p++ = '-'; ui64 = (guint64) -i64; } else { @@ -672,19 +607,29 @@ rspamd_vsnprintf (gchar *buf, glong max, const gchar *fmt, va_list args) } if (!humanize) { - buf = rspamd_sprintf_num (buf, last, ui64, zero, hex, width); + p = rspamd_sprintf_num (p, last, ui64, zero, hex, width); } else { - buf = rspamd_humanize_number (buf, last, ui64, bytes); + p = rspamd_humanize_number (p, last, ui64, bytes); } + slen = p - numbuf; + RSPAMD_PRINTF_APPEND (numbuf, slen); + } else { fmt++; + } + } - } else { - *buf++ = *fmt++; + /* Finish buffer */ + if (fmt > buf_start) { + wr = func (buf_start, fmt - buf_start, apd); + if (wr <= 0) { + goto oob; } + written += wr; } - return buf; +oob: + return written; } diff --git a/src/printf.h b/src/printf.h index 9d3921c3a..a4e03791d 100644 --- a/src/printf.h +++ b/src/printf.h @@ -47,7 +47,6 @@ * %V f_str_t * * %v GString * * %s null-terminated string - * %S ascii null-terminated string * %*s length and string * %Z '\0' * %N '\n' @@ -55,20 +54,22 @@ * %% % * */ -gint rspamd_sprintf (gchar *buf, const gchar *fmt, ...); -gint rspamd_fprintf (FILE *f, const gchar *fmt, ...); -gint rspamd_log_fprintf (FILE *f, const gchar *fmt, ...); -gint rspamd_snprintf (gchar *buf, glong max, const gchar *fmt, ...); -gchar *rspamd_vsnprintf (gchar *buf, glong max, const gchar *fmt, va_list args); -/* - * Escape rspamd string to write it to log file or other 7 bit prefferable places - * - * @param dst destination string - * @param src source string - * @param len length of destination buffer - * @return pointer to end of buffer +/** + * Callback used for common printf operations + * @param buf buffer to append + * @param buflen lenght of the buffer + * @param ud opaque pointer + * @return number of characters written */ -gchar * rspamd_escape_string (gchar *dst, const gchar *src, glong len); +typedef glong (*rspamd_printf_append_func)(const gchar *buf, glong buflen, gpointer ud); + +glong rspamd_fprintf (FILE *f, const gchar *fmt, ...); +glong rspamd_log_fprintf (FILE *f, const gchar *fmt, ...); +glong rspamd_snprintf (gchar *buf, glong max, const gchar *fmt, ...); +gchar *rspamd_vsnprintf (gchar *buf, glong max, const gchar *fmt, va_list args); +glong rspamd_printf_gstring (GString *s, const gchar *fmt, ...); + +glong rspamd_vprintf_common (rspamd_printf_append_func func, gpointer apd, const gchar *fmt, va_list args); #endif /* PRINTF_H_ */ @@ -181,7 +181,7 @@ spf_addr_find (GList *addrs, gpointer to_find) addr = cur->data; if (addr->is_list) { if ((res = spf_addr_find (addr->data.list, to_find)) != NULL) { - return res; + return cur; } } else { diff --git a/src/statfile.c b/src/statfile.c index 9930e0fe0..9df60fd56 100644 --- a/src/statfile.c +++ b/src/statfile.c @@ -235,7 +235,7 @@ statfile_pool_reindex (statfile_pool_t * pool, gchar *filename, size_t old_size, } pos = map + (sizeof (struct stat_file) - sizeof (struct stat_file_block)); - while (pos - map < (gint)old_size) { + while (old_size - (pos - map) >= sizeof (struct stat_file_block)) { block = (struct stat_file_block *)pos; if (block->hash1 != 0 && block->value != 0) { statfile_pool_set_block_common (pool, new, block->hash1, block->hash2, 0, block->value, FALSE); @@ -307,11 +307,15 @@ statfile_pool_open (statfile_pool_t * pool, gchar *filename, size_t size, gboole } memory_pool_lock_mutex (pool->lock); - if (!forced && abs (st.st_size - size) > (gint)sizeof (struct stat_file)) { + if (!forced && labs (size - st.st_size) > (long)sizeof (struct stat_file) * 2 + && size > sizeof (struct stat_file)) { memory_pool_unlock_mutex (pool->lock); - msg_warn ("need to reindex statfile old size: %Hz, new size: %Hz", st.st_size, size); + msg_warn ("need to reindex statfile old size: %Hz, new size: %Hz", (size_t)st.st_size, size); return statfile_pool_reindex (pool, filename, st.st_size, size); } + else if (size < sizeof (struct stat_file)) { + msg_err ("requested to shrink statfile to %Hz but it is too small", size); + } new_file = &pool->files[pool->opened++]; bzero (new_file, sizeof (stat_file_t)); diff --git a/src/symbols_cache.c b/src/symbols_cache.c index c60554f8f..d6c17390d 100644 --- a/src/symbols_cache.c +++ b/src/symbols_cache.c @@ -30,7 +30,7 @@ #include "view.h" #include "cfg_file.h" -#define WEIGHT_MULT 2.0 +#define WEIGHT_MULT 4.0 #define FREQUENCY_MULT 10.0 #define TIME_MULT -1.0 @@ -58,6 +58,7 @@ cache_logic_cmp (const void *p1, const void *p2) { const struct cache_item *i1 = p1, *i2 = p2; double w1, w2; + double weight1, weight2; double f1 = 0, f2 = 0; if (i1->priority == 0 && i2->priority == 0) { @@ -65,8 +66,10 @@ cache_logic_cmp (const void *p1, const void *p2) f1 = ((double)i1->s->frequency * nsymbols) / (double)total_frequency; f2 = ((double)i2->s->frequency * nsymbols) / (double)total_frequency; } - w1 = abs (i1->s->weight) * WEIGHT_MULT + f1 * FREQUENCY_MULT + i1->s->avg_time * TIME_MULT; - w2 = abs (i2->s->weight) * WEIGHT_MULT + f2 * FREQUENCY_MULT + i2->s->avg_time * TIME_MULT; + weight1 = i1->metric_weight == 0 ? i1->s->weight : i1->metric_weight; + weight2 = i2->metric_weight == 0 ? i2->s->weight : i2->metric_weight; + w1 = abs (weight1) * WEIGHT_MULT + f1 * FREQUENCY_MULT + i1->s->avg_time * TIME_MULT; + w2 = abs (weight2) * WEIGHT_MULT + f2 * FREQUENCY_MULT + i2->s->avg_time * TIME_MULT; } else { /* Strict sorting */ @@ -704,6 +707,34 @@ check_debug_symbol (struct config_file *cfg, const gchar *symbol) return FALSE; } +static void +rspamd_symbols_cache_metric_cb (gpointer k, gpointer v, gpointer ud) +{ + struct symbols_cache *cache = (struct symbols_cache *)ud; + GList *cur; + const gchar *sym = k; + gdouble weight = *(gdouble *)v; + struct cache_item *item; + + cur = cache->negative_items; + while (cur) { + item = cur->data; + if (strcmp (item->s->symbol, sym) == 0) { + item->metric_weight = weight; + return; + } + cur = g_list_next (cur); + } + cur = cache->static_items; + while (cur) { + item = cur->data; + if (strcmp (item->s->symbol, sym) == 0) { + item->metric_weight = weight; + return; + } + cur = g_list_next (cur); + } +} gboolean validate_cache (struct symbols_cache *cache, struct config_file *cfg, gboolean strict) @@ -787,6 +818,14 @@ validate_cache (struct symbols_cache *cache, struct config_file *cfg, gboolean s g_list_free (metric_symbols); #endif /* GLIB_COMPAT */ + /* Now adjust symbol weights according to default metric */ + if (cfg->default_metric != NULL) { + g_hash_table_foreach (cfg->default_metric->symbols, rspamd_symbols_cache_metric_cb, cache); + /* Resort caches */ + cache->negative_items = g_list_sort (cache->negative_items, cache_logic_cmp); + cache->static_items = g_list_sort (cache->static_items, cache_logic_cmp); + } + return TRUE; } diff --git a/src/symbols_cache.h b/src/symbols_cache.h index 15883f361..658852a32 100644 --- a/src/symbols_cache.h +++ b/src/symbols_cache.h @@ -43,6 +43,7 @@ struct cache_item { /* Priority */ gint priority; + gdouble metric_weight; }; diff --git a/src/ucl/include/ucl.h b/src/ucl/include/ucl.h index f7390c9a7..632c6e170 100644 --- a/src/ucl/include/ucl.h +++ b/src/ucl/include/ucl.h @@ -104,7 +104,8 @@ enum ucl_type { UCL_STRING, //!< UCL_STRING UCL_BOOLEAN, //!< UCL_BOOLEAN UCL_TIME, //!< UCL_TIME - UCL_USERDATA //!< UCL_USERDATA + UCL_USERDATA, //!< UCL_USERDATA + UCL_NULL //!< UCL_NULL }; /** @@ -201,6 +202,26 @@ ucl_object_new (void) if (new != NULL) { memset (new, 0, sizeof (ucl_object_t)); new->ref = 1; + new->type = UCL_NULL; + } + return new; +} + +/** + * Create new object with type specified + * @param type type of a new object + * @return new object + */ +static inline ucl_object_t* ucl_object_typed_new (unsigned int type) UCL_WARN_UNUSED_RESULT; +static inline ucl_object_t * +ucl_object_typed_new (unsigned int type) +{ + ucl_object_t *new; + new = malloc (sizeof (ucl_object_t)); + if (new != NULL) { + memset (new, 0, sizeof (ucl_object_t)); + new->ref = 1; + new->type = (type <= UCL_NULL ? type : UCL_NULL); } return new; } @@ -754,6 +775,16 @@ unsigned char *ucl_object_emit (ucl_object_t *obj, enum ucl_emitter emit_type); */ bool ucl_pubkey_add (struct ucl_parser *parser, const unsigned char *key, size_t len); +/** + * Set FILENAME and CURDIR variables in parser + * @param parser parser object + * @param filename filename to set or NULL to set FILENAME to "undef" and CURDIR to getcwd() + * @param need_expand perform realpath() if this variable is true and filename is not NULL + * @return true if variables has been set + */ +bool ucl_parser_set_filevars (struct ucl_parser *parser, const char *filename, + bool need_expand); + typedef void* ucl_object_iter_t; /** diff --git a/src/ucl/src/ucl_emitter.c b/src/ucl/src/ucl_emitter.c index c7d14dc8a..d0e62436d 100644 --- a/src/ucl/src/ucl_emitter.c +++ b/src/ucl/src/ucl_emitter.c @@ -38,6 +38,7 @@ static void ucl_elt_write_rcl (ucl_object_t *obj, UT_string *buf, unsigned int t bool start_tabs, bool is_top, bool expand_array); static void ucl_elt_write_yaml (ucl_object_t *obj, UT_string *buf, unsigned int tabs, bool start_tabs, bool compact, bool expand_array); +static void ucl_elt_array_write_yaml (ucl_object_t *obj, UT_string *buf, unsigned int tabs, bool start_tabs, bool is_top); /** * Add tabulation to the output buffer @@ -256,6 +257,12 @@ ucl_elt_write_json (ucl_object_t *obj, UT_string *buf, unsigned int tabs, bool s } ucl_elt_string_write_json (obj->value.sv, obj->len, buf); break; + case UCL_NULL: + if (start_tabs) { + ucl_add_tabs (buf, tabs, compact); + } + utstring_printf (buf, "null"); + break; case UCL_OBJECT: ucl_elt_obj_write_json (obj, buf, tabs, start_tabs, compact); break; @@ -440,6 +447,12 @@ ucl_elt_write_rcl (ucl_object_t *obj, UT_string *buf, unsigned int tabs, } ucl_elt_string_write_json (obj->value.sv, obj->len, buf); break; + case UCL_NULL: + if (start_tabs) { + ucl_add_tabs (buf, tabs, false); + } + utstring_printf (buf, "null"); + break; case UCL_OBJECT: ucl_elt_obj_write_rcl (obj, buf, tabs, start_tabs, is_top); break; @@ -471,6 +484,19 @@ ucl_object_emit_rcl (ucl_object_t *obj) } +static void +ucl_obj_write_yaml (ucl_object_t *obj, UT_string *buf, unsigned int tabs, bool start_tabs) +{ + bool is_array = (obj->next != NULL); + + if (is_array) { + ucl_elt_array_write_yaml (obj, buf, tabs, start_tabs, false); + } + else { + ucl_elt_write_yaml(obj, buf, tabs, start_tabs, false, true); + } +} + /** * Write a single object to the buffer * @param obj object to write @@ -486,25 +512,20 @@ ucl_elt_obj_write_yaml (ucl_object_t *obj, UT_string *buf, unsigned int tabs, bo ucl_add_tabs (buf, tabs, is_top); } if (!is_top) { - utstring_append_len (buf, ": {\n", 4); + utstring_append_len (buf, "{\n", 2); } while ((cur = ucl_hash_iterate (obj->value.ov, &it))) { ucl_add_tabs (buf, tabs + 1, is_top); - if (cur->flags & UCL_OBJECT_NEED_KEY_ESCAPE) { + if (cur->keylen > 0) { ucl_elt_string_write_json (cur->key, cur->keylen, buf); } else { - utstring_append_len (buf, cur->key, cur->keylen); - } - if (cur->type != UCL_OBJECT && cur->type != UCL_ARRAY) { - utstring_append_len (buf, " : ", 3); - } - else { - utstring_append_c (buf, ' '); + utstring_append_len (buf, "null", 4); } - ucl_elt_write_yaml (cur, buf, is_top ? tabs : tabs + 1, false, false, true); - if (cur->type != UCL_OBJECT && cur->type != UCL_ARRAY) { + utstring_append_len(buf, ": ", 2); + ucl_obj_write_yaml (cur, buf, is_top ? tabs : tabs + 1, false); + if (ucl_hash_iter_has_next(it)) { if (!is_top) { utstring_append_len (buf, ",\n", 2); } @@ -586,6 +607,12 @@ ucl_elt_write_yaml (ucl_object_t *obj, UT_string *buf, unsigned int tabs, } ucl_elt_string_write_json (obj->value.sv, obj->len, buf); break; + case UCL_NULL: + if (start_tabs) { + ucl_add_tabs (buf, tabs, false); + } + utstring_printf (buf, "null"); + break; case UCL_OBJECT: ucl_elt_obj_write_yaml (obj, buf, tabs, start_tabs, is_top); break; diff --git a/src/ucl/src/ucl_hash.c b/src/ucl/src/ucl_hash.c index d644da4f4..a3711deb8 100644 --- a/src/ucl/src/ucl_hash.c +++ b/src/ucl/src/ucl_hash.c @@ -96,6 +96,9 @@ ucl_hash_search (ucl_hash_t* hashlin, const char *key, unsigned keylen) { ucl_hash_node_t *found; + if (hashlin == NULL) { + return NULL; + } HASH_FIND (hh, hashlin->buckets, key, keylen, found); if (found) { diff --git a/src/ucl/src/ucl_internal.h b/src/ucl/src/ucl_internal.h index 21b1aa53e..78b52edbd 100644 --- a/src/ucl/src/ucl_internal.h +++ b/src/ucl/src/ucl_internal.h @@ -96,6 +96,7 @@ struct ucl_macro { struct ucl_stack { ucl_object_t *obj; struct ucl_stack *next; + int level; }; struct ucl_chunk { @@ -209,33 +210,33 @@ ucl_maybe_parse_boolean (ucl_object_t *obj, const unsigned char *start, size_t l bool ret = false, val = false; if (len == 5) { - if (tolower (p[0]) == 'f' && strncasecmp (p, "false", 5) == 0) { + if ((p[0] == 'f' || p[0] == 'F') && strncasecmp (p, "false", 5) == 0) { ret = true; val = false; } } else if (len == 4) { - if (tolower (p[0]) == 't' && strncasecmp (p, "true", 4) == 0) { + if ((p[0] == 't' || p[0] == 'T') && strncasecmp (p, "true", 4) == 0) { ret = true; val = true; } } else if (len == 3) { - if (tolower (p[0]) == 'y' && strncasecmp (p, "yes", 3) == 0) { + if ((p[0] == 'y' || p[0] == 'Y') && strncasecmp (p, "yes", 3) == 0) { ret = true; val = true; } - if (tolower (p[0]) == 'o' && strncasecmp (p, "off", 3) == 0) { + else if ((p[0] == 'o' || p[0] == 'O') && strncasecmp (p, "off", 3) == 0) { ret = true; val = false; } } else if (len == 2) { - if (tolower (p[0]) == 'n' && strncasecmp (p, "no", 2) == 0) { + if ((p[0] == 'n' || p[0] == 'N') && strncasecmp (p, "no", 2) == 0) { ret = true; val = false; } - else if (tolower (p[0]) == 'o' && strncasecmp (p, "on", 2) == 0) { + else if ((p[0] == 'o' || p[0] == 'O') && strncasecmp (p, "on", 2) == 0) { ret = true; val = true; } diff --git a/src/ucl/src/ucl_parser.c b/src/ucl/src/ucl_parser.c index 2e4816874..0441a121c 100644 --- a/src/ucl/src/ucl_parser.c +++ b/src/ucl/src/ucl_parser.c @@ -86,10 +86,21 @@ ucl_chunk_restore_state (struct ucl_chunk *chunk, struct ucl_parser_saved_state static inline void ucl_set_err (struct ucl_chunk *chunk, int code, const char *str, UT_string **err) { - ucl_create_err (err, "error on line %d at column %d: '%s', character: '%c'", - chunk->line, chunk->column, str, *chunk->pos); + if (isgraph (*chunk->pos)) { + ucl_create_err (err, "error on line %d at column %d: '%s', character: '%c'", + chunk->line, chunk->column, str, *chunk->pos); + } + else { + ucl_create_err (err, "error on line %d at column %d: '%s', character: '0x%02x'", + chunk->line, chunk->column, str, (int)*chunk->pos); + } } +/** + * Skip all comments from the current pos resolving nested and multiline comments + * @param parser + * @return + */ static bool ucl_skip_comments (struct ucl_parser *parser) { @@ -233,6 +244,16 @@ ucl_lex_is_comment (const unsigned char c1, const unsigned char c2) return false; } +/** + * Check variable found + * @param parser + * @param ptr + * @param remain + * @param out_len + * @param strict + * @param found + * @return + */ static inline const char * ucl_check_variable_safe (struct ucl_parser *parser, const char *ptr, size_t remain, size_t *out_len, bool strict, bool *found) @@ -263,6 +284,15 @@ ucl_check_variable_safe (struct ucl_parser *parser, const char *ptr, size_t rema return ptr; } +/** + * Check for a variable in a given string + * @param parser + * @param ptr + * @param remain + * @param out_len + * @param vars_found + * @return + */ static const char * ucl_check_variable (struct ucl_parser *parser, const char *ptr, size_t remain, size_t *out_len, bool *vars_found) { @@ -309,6 +339,14 @@ ucl_check_variable (struct ucl_parser *parser, const char *ptr, size_t remain, s return ret; } +/** + * Expand a single variable + * @param parser + * @param ptr + * @param remain + * @param dest + * @return + */ static const char * ucl_expand_single_variable (struct ucl_parser *parser, const char *ptr, size_t remain, unsigned char **dest) @@ -353,6 +391,14 @@ ucl_expand_single_variable (struct ucl_parser *parser, const char *ptr, return ret; } +/** + * Expand variables in string + * @param parser + * @param dst + * @param src + * @param in_len + * @return + */ static ssize_t ucl_expand_variable (struct ucl_parser *parser, unsigned char **dst, const char *src, size_t in_len) @@ -400,6 +446,18 @@ ucl_expand_variable (struct ucl_parser *parser, unsigned char **dst, return out_len; } +/** + * Store or copy pointer to the trash stack + * @param parser parser object + * @param src src string + * @param dst destination buffer (trash stack pointer) + * @param dst_const const destination pointer (e.g. value of object) + * @param in_len input length + * @param need_unescape need to unescape source (and copy it) + * @param need_lowercase need to lowercase value (and copy) + * @param need_expand need to expand variables (and copy as well) + * @return output length (excluding \0 symbol) + */ static inline ssize_t ucl_copy_or_store_ptr (struct ucl_parser *parser, const unsigned char *src, unsigned char **dst, @@ -448,6 +506,47 @@ ucl_copy_or_store_ptr (struct ucl_parser *parser, return ret; } +/** + * Create and append an object at the specified level + * @param parser + * @param is_array + * @param level + * @return + */ +static inline ucl_object_t * +ucl_add_parser_stack (ucl_object_t *obj, struct ucl_parser *parser, bool is_array, int level) +{ + struct ucl_stack *st; + + if (!is_array) { + if (obj == NULL) { + obj = ucl_object_typed_new (UCL_OBJECT); + } + else { + obj->type = UCL_OBJECT; + } + obj->value.ov = ucl_hash_create (); + parser->state = UCL_STATE_KEY; + } + else { + if (obj == NULL) { + obj = ucl_object_typed_new (UCL_ARRAY); + } + else { + obj->type = UCL_ARRAY; + } + parser->state = UCL_STATE_VALUE; + } + + st = UCL_ALLOC (sizeof (struct ucl_stack)); + st->obj = obj; + st->level = level; + LL_PREPEND (parser->stack, st); + parser->cur_obj = obj; + + return obj; +} + int ucl_maybe_parse_number (ucl_object_t *obj, const char *start, const char *end, const char **pos, bool allow_double, bool number_bytes) @@ -781,12 +880,13 @@ ucl_lex_json_string (struct ucl_parser *parser, * @return true if a key has been parsed */ static bool -ucl_parse_key (struct ucl_parser *parser, struct ucl_chunk *chunk) +ucl_parse_key (struct ucl_parser *parser, struct ucl_chunk *chunk, bool *next_key, bool *end_of_object) { - const unsigned char *p, *c = NULL, *end; + const unsigned char *p, *c = NULL, *end, *t; const char *key; bool got_quote = false, got_eq = false, got_semicolon = false, - need_unescape = false, ucl_escape = false, var_expand = false; + need_unescape = false, ucl_escape = false, var_expand = false, + got_content = false, got_sep = false; ucl_object_t *nobj, *tobj; ucl_hash_t *container; ssize_t keylen; @@ -805,23 +905,39 @@ ucl_parse_key (struct ucl_parser *parser, struct ucl_chunk *chunk) * A key must start with alpha, number, '/' or '_' and end with space character */ if (c == NULL) { - if (ucl_lex_is_comment (p[0], p[1])) { + if (chunk->remain >= 2 && ucl_lex_is_comment (p[0], p[1])) { if (!ucl_skip_comments (parser)) { return false; } p = chunk->pos; } + else if (ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) { + ucl_chunk_skipc (chunk, p); + } else if (ucl_test_character (*p, UCL_CHARACTER_KEY_START)) { /* The first symbol */ c = p; ucl_chunk_skipc (chunk, p); + got_content = true; } else if (*p == '"') { /* JSON style key */ c = p + 1; got_quote = true; + got_content = true; ucl_chunk_skipc (chunk, p); } + else if (*p == '}') { + /* We have actually end of an object */ + *end_of_object = true; + return true; + } + else if (*p == '.') { + ucl_chunk_skipc (chunk, p); + parser->prev_state = parser->state; + parser->state = UCL_STATE_MACRO_NAME; + return true; + } else { /* Invalid identifier */ ucl_set_err (chunk, UCL_ESYNTAX, "key must begin with a letter", &parser->err); @@ -832,6 +948,7 @@ ucl_parse_key (struct ucl_parser *parser, struct ucl_chunk *chunk) /* Parse the body of a key */ if (!got_quote) { if (ucl_test_character (*p, UCL_CHARACTER_KEY)) { + got_content = true; ucl_chunk_skipc (chunk, p); } else if (ucl_test_character (*p, UCL_CHARACTER_KEY_SEP)) { @@ -856,11 +973,14 @@ ucl_parse_key (struct ucl_parser *parser, struct ucl_chunk *chunk) } } - if (p >= chunk->end) { + if (p >= chunk->end && got_content) { ucl_set_err (chunk, UCL_ESYNTAX, "unfinished key", &parser->err); return false; } - + else if (!got_content) { + return true; + } + *end_of_object = false; /* We are now at the end of the key, need to parse the rest */ while (p < chunk->end) { if (ucl_test_character (*p, UCL_CHARACTER_WHITESPACE)) { @@ -886,7 +1006,7 @@ ucl_parse_key (struct ucl_parser *parser, struct ucl_chunk *chunk) return false; } } - else if (ucl_lex_is_comment (p[0], p[1])) { + else if (chunk->remain >= 2 && ucl_lex_is_comment (p[0], p[1])) { /* Check for comment */ if (!ucl_skip_comments (parser)) { return false; @@ -899,11 +1019,41 @@ ucl_parse_key (struct ucl_parser *parser, struct ucl_chunk *chunk) } } - if (p >= chunk->end) { + if (p >= chunk->end && got_content) { ucl_set_err (chunk, UCL_ESYNTAX, "unfinished key", &parser->err); return false; } + got_sep = got_semicolon || got_eq; + + if (!got_sep) { + /* + * Maybe we have more keys nested, so search for termination character. + * Possible choices: + * 1) key1 key2 ... keyN [:=] value <- we treat that as error + * 2) key1 ... keyN {} or [] <- we treat that as nested objects + * 3) key1 value[;,\n] <- we treat that as linear object + */ + t = p; + *next_key = false; + while (ucl_test_character (*t, UCL_CHARACTER_WHITESPACE)) { + t ++; + } + /* Check first non-space character after a key */ + if (*t != '{' && *t != '[') { + while (t < chunk->end) { + if (*t == ',' || *t == ';' || *t == '\n' || *t == '\r') { + break; + } + else if (*t == '{' || *t == '[') { + *next_key = true; + break; + } + t ++; + } + } + } + /* Create a new object */ nobj = ucl_object_new (); keylen = ucl_copy_or_store_ptr (parser, c, &nobj->trash_stack[UCL_TRASH_KEY], @@ -989,7 +1139,7 @@ ucl_parse_string_value (struct ucl_parser *parser, *var_expand = true; } - if (ucl_lex_is_atom_end (*p) || ucl_lex_is_comment (p[0], p[1])) { + if (ucl_lex_is_atom_end (*p) || (chunk->remain >= 2 && ucl_lex_is_comment (p[0], p[1]))) { break; } ucl_chunk_skipc (chunk, p); @@ -1064,7 +1214,6 @@ static bool ucl_parse_value (struct ucl_parser *parser, struct ucl_chunk *chunk) { const unsigned char *p, *c; - struct ucl_stack *st; ucl_object_t *obj = NULL, *t; unsigned int stripped_spaces; int str_len; @@ -1107,27 +1256,14 @@ ucl_parse_value (struct ucl_parser *parser, struct ucl_chunk *chunk) break; case '{': /* We have a new object */ - obj->type = UCL_OBJECT; - obj->value.ov = ucl_hash_create (); - parser->state = UCL_STATE_KEY; - st = UCL_ALLOC (sizeof (struct ucl_stack)); - st->obj = obj; - LL_PREPEND (parser->stack, st); - parser->cur_obj = obj; + obj = ucl_add_parser_stack (obj, parser, false, parser->stack->level); ucl_chunk_skipc (chunk, p); return true; break; case '[': /* We have a new array */ - obj = parser->cur_obj; - obj->type = UCL_ARRAY; - - parser->state = UCL_STATE_VALUE; - st = UCL_ALLOC (sizeof (struct ucl_stack)); - st->obj = obj; - LL_PREPEND (parser->stack, st); - parser->cur_obj = obj; + obj = ucl_add_parser_stack (obj, parser, true, parser->stack->level); ucl_chunk_skipc (chunk, p); return true; @@ -1168,7 +1304,7 @@ ucl_parse_value (struct ucl_parser *parser, struct ucl_chunk *chunk) default: /* Skip any spaces and comments */ if (ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE) || - ucl_lex_is_comment (p[0], p[1])) { + (chunk->remain >= 2 && ucl_lex_is_comment (p[0], p[1]))) { while (p < chunk->end && ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) { ucl_chunk_skipc (chunk, p); } @@ -1206,8 +1342,11 @@ ucl_parse_value (struct ucl_parser *parser, struct ucl_chunk *chunk) ucl_set_err (chunk, 0, "string value must not be empty", &parser->err); return false; } - - if (!ucl_maybe_parse_boolean (obj, c, str_len)) { + else if (str_len == 4 && memcmp (c, "null", 4) == 0) { + obj->len = 0; + obj->type = UCL_NULL; + } + else if (!ucl_maybe_parse_boolean (obj, c, str_len)) { obj->type = UCL_STRING; if ((str_len = ucl_copy_or_store_ptr (parser, c, &obj->trash_stack[UCL_TRASH_VALUE], &obj->value.sv, str_len, false, false, var_expand)) == -1) { @@ -1238,6 +1377,7 @@ ucl_parse_after_value (struct ucl_parser *parser, struct ucl_chunk *chunk) const unsigned char *p; bool got_sep = false; struct ucl_stack *st; + int last_level; p = chunk->pos; @@ -1246,7 +1386,7 @@ ucl_parse_after_value (struct ucl_parser *parser, struct ucl_chunk *chunk) /* Skip whitespaces */ ucl_chunk_skipc (chunk, p); } - else if (ucl_lex_is_comment (p[0], p[1])) { + else if (chunk->remain >= 2 && ucl_lex_is_comment (p[0], p[1])) { /* Skip comment */ if (!ucl_skip_comments (parser)) { return false; @@ -1258,16 +1398,24 @@ ucl_parse_after_value (struct ucl_parser *parser, struct ucl_chunk *chunk) else if (ucl_test_character (*p, UCL_CHARACTER_VALUE_END)) { if (*p == '}' || *p == ']') { if (parser->stack == NULL) { - ucl_set_err (chunk, UCL_ESYNTAX, "unexpected } detected", &parser->err); + ucl_set_err (chunk, UCL_ESYNTAX, "end of array or object detected without corresponding start", &parser->err); return false; } if ((*p == '}' && parser->stack->obj->type == UCL_OBJECT) || (*p == ']' && parser->stack->obj->type == UCL_ARRAY)) { - /* Pop object from a stack */ + /* Pop all nested objects from a stack */ st = parser->stack; + last_level = st->level; parser->stack = st->next; UCL_FREE (sizeof (struct ucl_stack), st); + + while (parser->stack != NULL && last_level > 0 && parser->stack->level == last_level) { + st = parser->stack; + parser->stack = st->next; + last_level = st->level; + UCL_FREE (sizeof (struct ucl_stack), st); + } } else { ucl_set_err (chunk, UCL_ESYNTAX, "unexpected terminating symbol detected", &parser->err); @@ -1391,11 +1539,23 @@ ucl_state_machine (struct ucl_parser *parser) { ucl_object_t *obj; struct ucl_chunk *chunk = parser->chunks; - struct ucl_stack *st; const unsigned char *p, *c = NULL, *macro_start = NULL; unsigned char *macro_escaped; size_t macro_len = 0; struct ucl_macro *macro = NULL; + bool next_key = false, end_of_object = false; + + if (parser->top_obj == NULL) { + if (*chunk->pos == '[') { + obj = ucl_add_parser_stack (NULL, parser, true, 0); + } + else { + obj = ucl_add_parser_stack (NULL, parser, false, 0); + } + parser->top_obj = obj; + parser->cur_obj = obj; + parser->state = UCL_STATE_INIT; + } p = chunk->pos; while (chunk->pos < chunk->end) { @@ -1406,6 +1566,7 @@ ucl_state_machine (struct ucl_parser *parser) * if we got [ or { correspondingly or can just treat new data as * a key of newly created object */ + obj = parser->cur_obj; if (!ucl_skip_comments (parser)) { parser->prev_state = parser->state; parser->state = UCL_STATE_ERROR; @@ -1413,25 +1574,16 @@ ucl_state_machine (struct ucl_parser *parser) } else { p = chunk->pos; - obj = ucl_object_new (); if (*p == '[') { parser->state = UCL_STATE_VALUE; - obj->type = UCL_ARRAY; ucl_chunk_skipc (chunk, p); } else { parser->state = UCL_STATE_KEY; - obj->type = UCL_OBJECT; - obj->value.ov = ucl_hash_create (); if (*p == '{') { ucl_chunk_skipc (chunk, p); } - }; - parser->cur_obj = obj; - parser->top_obj = obj; - st = UCL_ALLOC (sizeof (struct ucl_stack)); - st->obj = obj; - LL_PREPEND (parser->stack, st); + } } break; case UCL_STATE_KEY: @@ -1444,13 +1596,24 @@ ucl_state_machine (struct ucl_parser *parser) parser->state = UCL_STATE_AFTER_VALUE; continue; } - if (!ucl_parse_key (parser, chunk)) { + if (!ucl_parse_key (parser, chunk, &next_key, &end_of_object)) { parser->prev_state = parser->state; parser->state = UCL_STATE_ERROR; return false; } - if (parser->state != UCL_STATE_MACRO_NAME) { - parser->state = UCL_STATE_VALUE; + if (end_of_object) { + p = chunk->pos; + parser->state = UCL_STATE_AFTER_VALUE; + continue; + } + else if (parser->state != UCL_STATE_MACRO_NAME) { + if (next_key && parser->stack->obj->type == UCL_OBJECT) { + /* Parse more keys and nest objects accordingly */ + obj = ucl_add_parser_stack (parser->cur_obj, parser, false, parser->stack->level + 1); + } + else { + parser->state = UCL_STATE_VALUE; + } } else { c = chunk->pos; @@ -1504,7 +1667,7 @@ ucl_state_machine (struct ucl_parser *parser) /* Now we need to skip all spaces */ while (p < chunk->end) { if (!ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) { - if (ucl_lex_is_comment (p[0], p[1])) { + if (chunk->remain >= 2 && ucl_lex_is_comment (p[0], p[1])) { /* Skip comment */ if (!ucl_skip_comments (parser)) { return false; @@ -1526,8 +1689,8 @@ ucl_state_machine (struct ucl_parser *parser) return false; } macro_len = ucl_expand_variable (parser, ¯o_escaped, macro_start, macro_len); - parser->state = UCL_STATE_AFTER_VALUE; - if (macro_escaped == macro_start) { + parser->state = parser->prev_state; + if (macro_escaped == NULL) { if (!macro->handler (macro_start, macro_len, macro->ud)) { return false; } @@ -1565,6 +1728,9 @@ ucl_parser_new (int flags) new->flags = flags; + /* Initial assumption about filevars */ + ucl_parser_set_filevars (new, NULL, false); + return new; } @@ -1587,16 +1753,51 @@ void ucl_parser_register_variable (struct ucl_parser *parser, const char *var, const char *value) { - struct ucl_variable *new; + struct ucl_variable *new = NULL, *cur; - new = UCL_ALLOC (sizeof (struct ucl_variable)); - memset (new, 0, sizeof (struct ucl_variable)); - new->var = strdup (var); - new->var_len = strlen (var); - new->value = strdup (value); - new->value_len = strlen (value); + if (var == NULL) { + return; + } - LL_PREPEND (parser->variables, new); + /* Find whether a variable already exists */ + LL_FOREACH (parser->variables, cur) { + if (strcmp (cur->var, var) == 0) { + new = cur; + break; + } + } + + if (value == NULL) { + + if (new != NULL) { + /* Remove variable */ + LL_DELETE (parser->variables, new); + free (new->var); + free (new->value); + UCL_FREE (sizeof (struct ucl_variable), new); + } + else { + /* Do nothing */ + return; + } + } + else { + if (new == NULL) { + new = UCL_ALLOC (sizeof (struct ucl_variable)); + memset (new, 0, sizeof (struct ucl_variable)); + new->var = strdup (var); + new->var_len = strlen (var); + new->value = strdup (value); + new->value_len = strlen (value); + + LL_PREPEND (parser->variables, new); + } + else { + free (new->value); + new->value = strdup (value); + new->value_len = strlen (value); + } + } } bool diff --git a/src/ucl/src/ucl_util.c b/src/ucl/src/ucl_util.c index 0df611961..470d29163 100644 --- a/src/ucl/src/ucl_util.c +++ b/src/ucl/src/ucl_util.c @@ -25,6 +25,8 @@ #include "ucl_internal.h" #include "ucl_chartable.h" +#include <libgen.h> /* For dirname */ + #ifdef HAVE_OPENSSL #include <openssl/err.h> #include <openssl/sha.h> @@ -224,7 +226,7 @@ ucl_copy_value_trash (ucl_object_t *obj) ucl_object_t* ucl_parser_get_object (struct ucl_parser *parser) { - if (parser->state != UCL_STATE_INIT && parser->state != UCL_STATE_ERROR) { + if (parser->state != UCL_STATE_ERROR) { return ucl_object_ref (parser->top_obj); } @@ -288,7 +290,7 @@ ucl_pubkey_add (struct ucl_parser *parser, const unsigned char *key, size_t len) return false; #else # if (OPENSSL_VERSION_NUMBER < 0x10000000L) - ucl_create_err (err, "cannot check signatures, openssl version is unsupported"); + ucl_create_err (&parser->err, "cannot check signatures, openssl version is unsupported"); return EXIT_FAILURE; # else struct ucl_pubkey *nkey; @@ -439,24 +441,31 @@ ucl_fetch_file (const unsigned char *filename, unsigned char **buf, size_t *bufl int fd; struct stat st; - if (stat (filename, &st) == -1) { + if (stat (filename, &st) == -1 || !S_ISREG (st.st_mode)) { ucl_create_err (err, "cannot stat file %s: %s", filename, strerror (errno)); return false; } - if ((fd = open (filename, O_RDONLY)) == -1) { - ucl_create_err (err, "cannot open file %s: %s", - filename, strerror (errno)); - return false; + if (st.st_size == 0) { + /* Do not map empty files */ + *buf = ""; + *buflen = 0; } - if ((*buf = mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) { + else { + if ((fd = open (filename, O_RDONLY)) == -1) { + ucl_create_err (err, "cannot open file %s: %s", + filename, strerror (errno)); + return false; + } + if ((*buf = mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) { + close (fd); + ucl_create_err (err, "cannot mmap file %s: %s", + filename, strerror (errno)); + return false; + } + *buflen = st.st_size; close (fd); - ucl_create_err (err, "cannot mmap file %s: %s", - filename, strerror (errno)); - return false; } - *buflen = st.st_size; - close (fd); return true; } @@ -528,6 +537,7 @@ ucl_include_url (const unsigned char *data, size_t len, size_t buflen = 0; struct ucl_chunk *chunk; char urlbuf[PATH_MAX]; + int prev_state; snprintf (urlbuf, sizeof (urlbuf), "%.*s", (int)len, data); @@ -548,13 +558,20 @@ ucl_include_url (const unsigned char *data, size_t len, ucl_create_err (&parser->err, "cannot verify url %s: %s", urlbuf, ERR_error_string (ERR_get_error (), NULL)); - munmap (sigbuf, siglen); + if (siglen > 0) { + munmap (sigbuf, siglen); + } return false; } - munmap (sigbuf, siglen); + if (siglen > 0) { + munmap (sigbuf, siglen); + } #endif } + prev_state = parser->state; + parser->state = UCL_STATE_INIT; + res = ucl_parser_add_chunk (parser, buf, buflen); if (res == true) { /* Remove chunk from the stack */ @@ -564,6 +581,8 @@ ucl_include_url (const unsigned char *data, size_t len, UCL_FREE (sizeof (struct ucl_chunk), chunk); } } + + parser->state = prev_state; free (buf); return res; @@ -586,6 +605,7 @@ ucl_include_file (const unsigned char *data, size_t len, unsigned char *buf = NULL; size_t buflen; char filebuf[PATH_MAX], realbuf[PATH_MAX]; + int prev_state; snprintf (filebuf, sizeof (filebuf), "%.*s", (int)len, data); if (realpath (filebuf, realbuf) == NULL) { @@ -612,13 +632,22 @@ ucl_include_file (const unsigned char *data, size_t len, ucl_create_err (&parser->err, "cannot verify file %s: %s", filebuf, ERR_error_string (ERR_get_error (), NULL)); - munmap (sigbuf, siglen); + if (siglen > 0) { + munmap (sigbuf, siglen); + } return false; } - munmap (sigbuf, siglen); + if (siglen > 0) { + munmap (sigbuf, siglen); + } #endif } + ucl_parser_set_filevars (parser, realbuf, false); + + prev_state = parser->state; + parser->state = UCL_STATE_INIT; + res = ucl_parser_add_chunk (parser, buf, buflen); if (res == true) { /* Remove chunk from the stack */ @@ -628,7 +657,12 @@ ucl_include_file (const unsigned char *data, size_t len, UCL_FREE (sizeof (struct ucl_chunk), chunk); } } - munmap (buf, buflen); + + parser->state = prev_state; + + if (buflen > 0) { + munmap (buf, buflen); + } return res; } @@ -676,19 +710,60 @@ ucl_includes_handler (const unsigned char *data, size_t len, void* ud) } bool +ucl_parser_set_filevars (struct ucl_parser *parser, const char *filename, bool need_expand) +{ + char realbuf[PATH_MAX], *curdir; + + if (filename != NULL) { + if (need_expand) { + if (realpath (filename, realbuf) == NULL) { + return false; + } + } + else { + ucl_strlcpy (realbuf, filename, sizeof (realbuf)); + } + + /* Define variables */ + ucl_parser_register_variable (parser, "FILENAME", realbuf); + curdir = dirname (realbuf); + ucl_parser_register_variable (parser, "CURDIR", curdir); + } + else { + /* Set everything from the current dir */ + curdir = getcwd (realbuf, sizeof (realbuf)); + ucl_parser_register_variable (parser, "FILENAME", "undef"); + ucl_parser_register_variable (parser, "CURDIR", curdir); + } + + return true; +} + +bool ucl_parser_add_file (struct ucl_parser *parser, const char *filename) { unsigned char *buf; size_t len; bool ret; + char realbuf[PATH_MAX]; + + if (realpath (filename, realbuf) == NULL) { + ucl_create_err (&parser->err, "cannot open file %s: %s", + filename, + strerror (errno)); + return false; + } - if (!ucl_fetch_file (filename, &buf, &len, &parser->err)) { + if (!ucl_fetch_file (realbuf, &buf, &len, &parser->err)) { return false; } + ucl_parser_set_filevars (parser, realbuf, false); ret = ucl_parser_add_chunk (parser, buf, len); - munmap (buf, len); + if (len > 0) { + munmap (buf, len); + } return ret; } @@ -883,6 +958,17 @@ ucl_object_insert_key_common (ucl_object_t *top, ucl_object_t *elt, top->type = UCL_OBJECT; } + if (top->type != UCL_OBJECT) { + /* It is possible to convert NULL type to an object */ + if (top->type == UCL_NULL) { + top->type = UCL_OBJECT; + } + else { + /* Refuse converting of other object types */ + return top; + } + } + if (top->value.ov == NULL) { top->value.ov = ucl_hash_create (); } diff --git a/src/ucl/src/xxhash.c b/src/ucl/src/xxhash.c index bb4c639aa..5869503be 100644 --- a/src/ucl/src/xxhash.c +++ b/src/ucl/src/xxhash.c @@ -273,7 +273,7 @@ U32 XXH32(const void* input, int len, U32 seed) XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
# if !defined(XXH_USE_UNALIGNED_ACCESS)
- if ((((size_t)input) & 3)) // Input is aligned, let's leverage the speed advantage
+ if (!(((size_t)input) & 3)) // Input is aligned, let's leverage the speed advantage
{
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
diff --git a/src/ucl/tests/rcl_test.json.xz b/src/ucl/tests/rcl_test.json.xz Binary files differdeleted file mode 100644 index 98c3559ad..000000000 --- a/src/ucl/tests/rcl_test.json.xz +++ /dev/null @@ -371,7 +371,7 @@ struct url_matcher matchers[] = { { ".zm", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, { ".zw", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, /* Likely emails */ - { "@", "mailto://",url_email_start, url_email_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH } + { "@", "mailto://",url_email_start, url_email_end, URL_FLAG_NOHTML } }; struct url_match_scanner { @@ -1456,7 +1456,7 @@ url_email_start (const gchar *begin, const gchar *end, const gchar *pos, url_mat else { p = pos + strlen (match->pattern); if (is_domain (*p)) { - match->m_begin = p; + match->m_begin = pos; return TRUE; } } @@ -1467,15 +1467,24 @@ static gboolean url_email_end (const gchar *begin, const gchar *end, const gchar *pos, url_match_t *match) { const gchar *p; + gboolean got_at = FALSE; p = pos + strlen (match->pattern); + if (*pos == '@') { + got_at = TRUE; + } - while (p < end && (is_domain (*p) || *p == '_' || (*p == '.' && p + 1 < end && is_domain (*(p + 1))))) { + while (p < end && (is_domain (*p) || *p == '_' + || (*p == '@' && !got_at) || + (*p == '.' && p + 1 < end && is_domain (*(p + 1))))) { + if (*p == '@') { + got_at = TRUE; + } p ++; } match->m_len = p - match->m_begin; match->add_prefix = TRUE; - return TRUE; + return got_at; } void diff --git a/src/util.c b/src/util.c index bc2206579..776e807d1 100644 --- a/src/util.c +++ b/src/util.c @@ -31,6 +31,11 @@ #include "filter.h" #include "message.h" +#ifdef HAVE_OPENSSL +#include <openssl/rand.h> +#include <openssl/err.h> +#endif + /* Check log messages intensity once per minute */ #define CHECK_TIME 60 /* More than 2 log messages per second */ @@ -215,7 +220,8 @@ accept_from_socket (gint listen_sock, struct sockaddr *addr, socklen_t * len) gint make_unix_socket (const gchar *path, struct sockaddr_un *addr, gint type, gboolean is_server, gboolean async) { - gint fd, s_error, r, optlen, serrno, on = 1; + gint fd = -1, s_error, r, optlen, serrno, on = 1; + struct stat st; if (path == NULL) return -1; @@ -227,10 +233,25 @@ make_unix_socket (const gchar *path, struct sockaddr_un *addr, gint type, gboole addr->sun_len = SUN_LEN (addr); #endif + if (is_server) { + /* Unlink socket if it exists already */ + if (lstat (addr->sun_path, &st) != -1) { + if (S_ISSOCK (st.st_mode)) { + if (unlink (addr->sun_path) == -1) { + msg_warn ("unlink %s failed: %d, '%s'", addr->sun_path, errno, strerror (errno)); + goto out; + } + } + else { + msg_warn ("%s is not a socket", addr->sun_path); + goto out; + } + } + } fd = socket (PF_LOCAL, type, 0); if (fd == -1) { - msg_warn ("socket failed: %d, '%s'", errno, strerror (errno)); + msg_warn ("socket failed %s: %d, '%s'", addr->sun_path, errno, strerror (errno)); return -1; } @@ -240,7 +261,7 @@ make_unix_socket (const gchar *path, struct sockaddr_un *addr, gint type, gboole /* Set close on exec */ if (fcntl (fd, F_SETFD, FD_CLOEXEC) == -1) { - msg_warn ("fcntl failed: %d, '%s'", errno, strerror (errno)); + msg_warn ("fcntl failed %s: %d, '%s'", addr->sun_path, errno, strerror (errno)); goto out; } if (is_server) { @@ -253,14 +274,14 @@ make_unix_socket (const gchar *path, struct sockaddr_un *addr, gint type, gboole if (r == -1) { if (errno != EINPROGRESS) { - msg_warn ("bind/connect failed: %d, '%s'", errno, strerror (errno)); + msg_warn ("bind/connect failed %s: %d, '%s'", addr->sun_path, errno, strerror (errno)); goto out; } if (!async) { /* Try to poll */ if (poll_sync_socket (fd, CONNECT_TIMEOUT * 1000, POLLOUT) <= 0) { errno = ETIMEDOUT; - msg_warn ("bind/connect failed: timeout"); + msg_warn ("bind/connect failed %s: timeout", addr->sun_path); goto out; } else { @@ -286,7 +307,9 @@ make_unix_socket (const gchar *path, struct sockaddr_un *addr, gint type, gboole out: serrno = errno; - close (fd); + if (fd != -1) { + close (fd); + } errno = serrno; return (-1); } @@ -310,18 +333,11 @@ make_universal_socket (const gchar *credits, guint16 port, gchar portbuf[8]; if (*credits == '/') { - r = stat (credits, &st); if (is_server) { - if (r == -1) { - return make_unix_socket (credits, &un, type, is_server, async); - } - else { - /* Unix socket exists, it must be unlinked first */ - errno = EEXIST; - return -1; - } + return make_unix_socket (credits, &un, type, is_server, async); } else { + r = stat (credits, &st); if (r == -1) { /* Unix socket doesn't exists it must be created first */ errno = ENOENT; @@ -394,18 +410,11 @@ make_universal_sockets_list (const gchar *credits, guint16 port, cur = strv; while (*cur != NULL) { if (*credits == '/') { - r = stat (credits, &st); if (is_server) { - if (r == -1) { - fd = make_unix_socket (credits, &un, type, is_server, async); - } - else { - /* Unix socket exists, it must be unlinked first */ - errno = EEXIST; - goto err; - } + fd = make_unix_socket (credits, &un, type, is_server, async); } else { + r = stat (credits, &st); if (r == -1) { /* Unix socket doesn't exists it must be created first */ errno = ENOENT; @@ -2386,6 +2395,51 @@ restart: return p - buf; #endif } + +void +rspamd_random_bytes (gchar *buf, gsize buflen) +{ + gint fd; + gsize i; +#ifdef HAVE_OPENSSL + + /* Init random generator */ + if (RAND_bytes (buf, buflen) != 1) { + msg_err ("cannot seed random generator using openssl: %s, using time", + ERR_error_string (ERR_get_error (), NULL)); + goto fallback; + } +#else + goto fallback; +#endif + return; + +fallback: + /* Try to use /dev/random if no openssl is found */ + fd = open ("/dev/random", O_RDONLY); + if (fd != -1) { + if (read (fd, buf, buflen) == (gssize)buflen) { + close (fd); + return; + } + close (fd); + } + /* No /dev/random */ + g_random_set_seed (time (NULL)); + for (i = 0; i < buflen; i ++) { + buf[i] = g_random_int () & 0xff; + } +} + +void +rspamd_prng_seed (void) +{ + guint32 rand_seed = 0; + + rspamd_random_bytes ((gchar *)&rand_seed, sizeof (rand_seed)); + g_random_set_seed (rand_seed); +} + /* * vi:ts=4 */ diff --git a/src/util.h b/src/util.h index f2b7132af..c36587cab 100644 --- a/src/util.h +++ b/src/util.h @@ -14,6 +14,17 @@ struct workq; struct statfile; struct classifier_config; +/** + * Union that is used for storing sockaddrs + */ +union sa_union { + struct sockaddr_storage ss; + struct sockaddr sa; + struct sockaddr_in s4; + struct sockaddr_in6 s6; + struct sockaddr_un su; +}; + /* * Create socket and bind or connect it to specified address and port */ @@ -440,16 +451,15 @@ time_t parse_http_date (const gchar *header, gsize len); gint rspamd_read_passphrase (gchar *buf, gint size, gint rwflag, gpointer key); /** - * Expand path that may contain configuration variables: - * $CONFDIR - configuration directory - * $RUNDIR - local states directory - * $DBDIR - databases dir - * $LOGDIR - logs dir - * $PLUGINSDIR - plugins dir - * $PREFIX - installation prefix - * $VERSION - rspamd version - * @param pool to use - * @param path path to expand + * Seed glib prng using openssl if possible + */ +void rspamd_prng_seed (void); + +/** + * Generate random bytes using the most suitable generator + * @param buf + * @param buflen */ +void rspamd_random_bytes (gchar *buf, gsize buflen); #endif diff --git a/src/worker.c b/src/worker.c index bb43afba8..95355bdd3 100644 --- a/src/worker.c +++ b/src/worker.c @@ -502,8 +502,9 @@ accept_socket (gint fd, short what, void *arg) struct rspamd_worker_ctx *ctx; union sa_union su; struct worker_task *new_task; + char ip_str[INET6_ADDRSTRLEN + 1]; - socklen_t addrlen = sizeof (su.ss); + socklen_t addrlen = sizeof (su); gint nfd; ctx = worker->ctx; @@ -514,7 +515,7 @@ accept_socket (gint fd, short what, void *arg) } if ((nfd = - accept_from_socket (fd, (struct sockaddr *) &su.ss, &addrlen)) == -1) { + accept_from_socket (fd, &su.sa, &addrlen)) == -1) { msg_warn ("accept failed: %s", strerror (errno)); return; } @@ -525,16 +526,21 @@ accept_socket (gint fd, short what, void *arg) new_task = construct_task (worker); - if (su.ss.ss_family == AF_UNIX) { + if (su.sa.sa_family == AF_UNIX) { msg_info ("accepted connection from unix socket"); new_task->client_addr.s_addr = INADDR_NONE; } - else if (su.ss.ss_family == AF_INET) { + else if (su.sa.sa_family == AF_INET) { msg_info ("accepted connection from %s port %d", inet_ntoa (su.s4.sin_addr), ntohs (su.s4.sin_port)); memcpy (&new_task->client_addr, &su.s4.sin_addr, sizeof (struct in_addr)); } + else if (su.sa.sa_family == AF_INET6) { + msg_info ("accepted connection from %s port %d", + inet_ntop (su.sa.sa_family, &su.s6.sin6_addr, ip_str, sizeof (ip_str)), + ntohs (su.s6.sin6_port)); + } /* Copy some variables */ new_task->sock = nfd; diff --git a/src/worker_util.c b/src/worker_util.c index e8d8f7423..599d098c9 100644 --- a/src/worker_util.c +++ b/src/worker_util.c @@ -152,7 +152,6 @@ free_task (struct worker_task *task, gboolean is_soft) if (task->received) { g_list_free (task->received); } - memory_pool_delete (task->task_pool); if (task->dispatcher) { if (is_soft) { /* Plan dispatcher shutdown */ @@ -165,6 +164,7 @@ free_task (struct worker_task *task, gboolean is_soft) if (task->sock != -1) { close (task->sock); } + memory_pool_delete (task->task_pool); g_slice_free1 (sizeof (struct worker_task), task); } } |