From 8f5a7be536a5334e14a62115edd66e9c60e30b63 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Tue, 12 Mar 2019 11:29:08 +0000
Subject: [Minor] Neural: Add safe guard for local learning spawning

---
 src/plugins/lua/neural.lua | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'src')

diff --git a/src/plugins/lua/neural.lua b/src/plugins/lua/neural.lua
index 43d7bb127..4e268962c 100644
--- a/src/plugins/lua/neural.lua
+++ b/src/plugins/lua/neural.lua
@@ -680,6 +680,8 @@ local function train_ann(rule, _, ev_base, elt, worker)
             return st
           end
 
+          rule.learning_spawned = true
+
           worker:spawn_process{
             func = train_torch,
             on_complete = ann_trained_torch,
-- 
cgit v1.2.3


From ae698ea973b832bffa5aa3c98fddcf29031073f2 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Tue, 12 Mar 2019 11:36:01 +0000
Subject: [Minor] Fuzzy_check: Do not penalise score for large images

---
 src/plugins/fuzzy_check.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c
index 672a6dc63..a0e4df012 100644
--- a/src/plugins/fuzzy_check.c
+++ b/src/plugins/fuzzy_check.c
@@ -1875,6 +1875,8 @@ fuzzy_insert_result (struct fuzzy_client_session *session,
 	struct fuzzy_client_result *res;
 	gboolean is_fuzzy = FALSE;
 	gchar hexbuf[rspamd_cryptobox_HASHBYTES * 2 + 1];
+	/* Discriminate scores for small images */
+	static const guint short_image_limit = 32 * 1024;
 
 	/* Get mapping by flag */
 	if ((map =
@@ -1902,7 +1904,10 @@ fuzzy_insert_result (struct fuzzy_client_session *session,
 	nval = fuzzy_normalize (rep->v1.value, weight);
 
 	if (io && (io->flags & FUZZY_CMD_FLAG_IMAGE)) {
-		nval *= rspamd_normalize_probability (rep->v1.prob, 0.5);
+		if (!io->part || io->part->parsed_data.len <= short_image_limit) {
+			nval *= rspamd_normalize_probability (rep->v1.prob, 0.5);
+		}
+
 		type = "img";
 		res->type = FUZZY_RESULT_IMG;
 	}
-- 
cgit v1.2.3


From a3f9ececff684f274a8bf0215416ca7db5d10ea1 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Tue, 12 Mar 2019 14:57:03 +0000
Subject: [Minor] Document keepalive attribute

---
 src/lua/lua_http.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'src')

diff --git a/src/lua/lua_http.c b/src/lua/lua_http.c
index 4378aea89..72eec70ad 100644
--- a/src/lua/lua_http.c
+++ b/src/lua/lua_http.c
@@ -536,6 +536,7 @@ lua_http_push_headers (lua_State *L, struct rspamd_http_message *msg)
  * @param {resolver} resolver to perform DNS-requests. Usually got from either `task` or `config`
  * @param {boolean} gzip if true, body of the requests will be compressed
  * @param {boolean} no_ssl_verify disable SSL peer checks
+ * @param {boolean} keepalive enable keep-alive pool
  * @param {string} user for HTTP authentication
  * @param {string} password for HTTP authentication, only if "user" present
  * @return {boolean} `true`, in **async** mode, if a request has been successfully scheduled. If this value is `false` then some error occurred, the callback thus will not be called.
-- 
cgit v1.2.3


From c4567b40b87002369ab8b1c429dabfe5abf361ef Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Tue, 12 Mar 2019 15:08:27 +0000
Subject: [Minor] Fix docs

---
 src/lua/lua_udp.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/lua/lua_udp.c b/src/lua/lua_udp.c
index aac22695f..8a862f16a 100644
--- a/src/lua/lua_udp.c
+++ b/src/lua/lua_udp.c
@@ -35,7 +35,7 @@ rspamd_config.SYM = function(task)
   udp.sento{
     host = addr, -- must be ip address object (e.g. received by upstream module)
     port = 500,
-    data = {'str1, 'str2'}, -- can be table, string or rspamd_text
+    data = {'str1', 'str2'}, -- can be table, string or rspamd_text
     timeout = 0.5, -- default = 1s
     task = task, -- if has task
     session = session, -- optional
-- 
cgit v1.2.3


From 05cbd5a1b9dfa972460e2724faea63594add750f Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Wed, 13 Mar 2019 09:07:37 +0000
Subject: [Fix] SURBL: Fix regression in surbl module

---
 src/plugins/surbl.c | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

(limited to 'src')

diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c
index e5080b864..316ca7f62 100644
--- a/src/plugins/surbl.c
+++ b/src/plugins/surbl.c
@@ -1362,10 +1362,8 @@ format_surbl_request (rspamd_mempool_t * pool,
 		}
 	}
 
-	if (url->surbl == NULL) {
-		url->surbl = result;
-		url->surbllen = r;
-	}
+	url->surbl = result;
+	url->surbllen = r;
 
 	if (!forced &&
 			rspamd_match_hash_map (surbl_module_ctx->whitelist, result) != NULL) {
-- 
cgit v1.2.3


From ab0f47f4efc84898a3fb18b7990ae719b0c7884e Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Wed, 13 Mar 2019 14:58:59 +0000
Subject: [Minor] Attach PVS analyze target

---
 CMakeLists.txt     | 8 ++++++++
 src/CMakeLists.txt | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)

(limited to 'src')

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c01bba2d5..32a796276 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1411,3 +1411,11 @@ IF(NOT DEBIAN_BUILD)
 	ADD_CUSTOM_TARGET(check DEPENDS rspamd-test)
 	ADD_CUSTOM_TARGET(run-test DEPENDS check COMMAND test/rspamd-test)
 ENDIF(NOT DEBIAN_BUILD)
+
+
+# PVS Studio
+include(${CMAKE_SOURCE_DIR}/PVS-Studio.cmake)
+pvs_studio_add_target(TARGET ${PROJECT_NAME}.analyze ALL
+		ANALYZE ${PROJECT_NAME} rspamd-server rspamadm rspamc
+		OUTPUT FORMAT errorfile
+		LOG target_${PROJECT_NAME}.err)
\ No newline at end of file
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 256feb522..22d28e770 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -222,4 +222,4 @@ ENDIF(NOT DEBIAN_BUILD)
 TARGET_LINK_LIBRARIES(rspamd rspamd-server)
 
 INSTALL(TARGETS rspamd RUNTIME DESTINATION bin)
-INSTALL(TARGETS rspamd-server LIBRARY DESTINATION ${RSPAMD_LIBDIR})
+INSTALL(TARGETS rspamd-server LIBRARY DESTINATION ${RSPAMD_LIBDIR})
\ No newline at end of file
-- 
cgit v1.2.3


From f7aee39a9dc81ae437e1dcc651a7335a7ae071d3 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Wed, 13 Mar 2019 15:01:18 +0000
Subject: [Fix] Fix various issues found by PVS Studio

https://www.viva64.com/
---
 PVS-Studio.cmake                | 547 ++++++++++++++++++++++++++++++++++++++++
 src/client/rspamc.c             |  14 +-
 src/fuzzy_storage.c             |   2 +
 src/libmime/archives.c          |   5 +-
 src/libserver/dkim.c            |   2 +
 src/libserver/milter.c          |   6 +-
 src/libserver/rspamd_symcache.c |   1 -
 src/libserver/task.c            |   2 +-
 src/libutil/http_connection.c   |  19 +-
 src/libutil/logger.c            |   2 +-
 src/libutil/map.c               |   2 +-
 src/libutil/mem_pool.c          |  18 +-
 src/libutil/multipattern.c      |   1 -
 src/libutil/rrd.c               |  13 +-
 src/libutil/shingles.c          |   4 +
 src/lua/lua_common.c            |   4 +-
 src/lua/lua_config.c            |   6 +
 src/lua/lua_html.c              |   3 -
 src/lua/lua_http.c              |   4 +
 src/lua/lua_thread_pool.c       |   2 +-
 src/lua/lua_util.c              |   4 +-
 src/plugins/surbl.c             |   3 -
 22 files changed, 607 insertions(+), 57 deletions(-)
 create mode 100644 PVS-Studio.cmake

(limited to 'src')

diff --git a/PVS-Studio.cmake b/PVS-Studio.cmake
new file mode 100644
index 000000000..6001f3337
--- /dev/null
+++ b/PVS-Studio.cmake
@@ -0,0 +1,547 @@
+# 2006-2008 (c) Viva64.com Team
+# 2008-2018 (c) OOO "Program Verification Systems"
+#
+# Version 12
+# Apache 2.0 license
+
+cmake_minimum_required(VERSION 2.8.12)
+cmake_policy(SET CMP0054 NEW)
+
+if (PVS_STUDIO_AS_SCRIPT)
+    # This code runs at build time.
+    # It executes pvs-studio-analyzer and propagates its return value.
+
+    set(in_cl_params FALSE)
+    set(additional_args)
+
+    foreach (arg ${PVS_STUDIO_COMMAND})
+        if (NOT in_cl_params)
+            if ("${arg}" STREQUAL "--cl-params")
+                set(in_cl_params TRUE)
+            endif ()
+        else ()
+            # A workaround for macOS frameworks (e.g. QtWidgets.framework)
+            # You can test this workaround on this project: https://github.com/easyaspi314/MidiEditor/tree/gba
+            if (APPLE AND "${arg}" MATCHES "^-I(.*)\\.framework$")
+                STRING(REGEX REPLACE "^-I(.*)\\.framework$" "\\1.framework" framework "${arg}")
+                if (IS_ABSOLUTE "${framework}")
+                    get_filename_component(framework "${framework}" DIRECTORY)
+                    list(APPEND additional_args "-iframework")
+                    list(APPEND additional_args "${framework}")
+                endif ()
+            endif ()
+        endif ()
+    endforeach ()
+
+    execute_process(COMMAND ${PVS_STUDIO_COMMAND} ${additional_args}
+            ERROR_VARIABLE error
+            RESULT_VARIABLE result)
+
+    set(stderr_type "")
+
+    if (result)
+        set(stderr_type FATAL_ERROR)
+    endif ()
+
+    if (result OR error)
+        message(${stderr_type} "${error}")
+    endif ()
+
+    return()
+endif ()
+
+if(__PVS_STUDIO_INCLUDED)
+    return()
+endif()
+set(__PVS_STUDIO_INCLUDED TRUE)
+
+set(PVS_STUDIO_SCRIPT "${CMAKE_CURRENT_LIST_FILE}")
+
+function (pvs_studio_log TEXT)
+    if (PVS_STUDIO_DEBUG)
+        message("PVS-Studio: ${TEXT}")
+    endif ()
+endfunction ()
+
+function (pvs_studio_relative_path VAR ROOT FILEPATH)
+    set("${VAR}" "${FILEPATH}" PARENT_SCOPE)
+    if ("${FILEPATH}" MATCHES "^/.*$" OR "${FILEPATH}" MATCHES "^.:/.*$")
+        file(RELATIVE_PATH RPATH "${ROOT}" "${FILEPATH}")
+        if (NOT "${RPATH}" MATCHES "^\\.\\..*$")
+            set("${VAR}" "${RPATH}" PARENT_SCOPE)
+        endif ()
+    endif ()
+endfunction ()
+
+function (pvs_studio_join_path VAR DIR1 DIR2)
+    if ("${DIR2}" MATCHES "^(/|~|.:/).*$" OR "${DIR1}" STREQUAL "")
+        set("${VAR}" "${DIR2}" PARENT_SCOPE)
+    else ()
+        set("${VAR}" "${DIR1}/${DIR2}" PARENT_SCOPE)
+    endif ()
+endfunction ()
+
+macro (pvs_studio_append_flags_from_property CXX C DIR PREFIX)
+    if (NOT "${PROPERTY}" STREQUAL "NOTFOUND" AND NOT "${PROPERTY}" STREQUAL "PROPERTY-NOTFOUND")
+        foreach (PROP ${PROPERTY})
+            pvs_studio_join_path(PROP "${DIR}" "${PROP}")
+
+            if (APPLE AND "${PREFIX}" STREQUAL "-I" AND IS_ABSOLUTE "${PROP}" AND "${PROP}" MATCHES "\\.framework$")
+                get_filename_component(FRAMEWORK "${PROP}" DIRECTORY)
+                list(APPEND "${CXX}" "-iframework")
+                list(APPEND "${CXX}" "${FRAMEWORK}")
+                list(APPEND "${C}" "-iframework")
+                list(APPEND "${C}" "${FRAMEWORK}")
+                pvs_studio_log("framework: ${FRAMEWORK}")
+            elseif (NOT "${PROP}" STREQUAL "")
+                list(APPEND "${CXX}" "${PREFIX}${PROP}")
+                list(APPEND "${C}" "${PREFIX}${PROP}")
+            endif()
+        endforeach ()
+    endif ()
+endmacro ()
+
+macro (pvs_studio_append_standard_flag FLAGS STANDARD)
+    if ("${STANDARD}" MATCHES "^(99|11|14|17)$")
+        if ("${PVS_STUDIO_PREPROCESSOR}" MATCHES "gcc|clang")
+            list(APPEND "${FLAGS}" "-std=c++${STANDARD}")
+        endif ()
+    endif ()
+endmacro ()
+
+function (pvs_studio_set_directory_flags DIRECTORY CXX C)
+    set(CXX_FLAGS "${${CXX}}")
+    set(C_FLAGS "${${C}}")
+
+    get_directory_property(PROPERTY DIRECTORY "${DIRECTORY}" INCLUDE_DIRECTORIES)
+    pvs_studio_append_flags_from_property(CXX_FLAGS C_FLAGS "${DIRECTORY}" "-I")
+
+    get_directory_property(PROPERTY DIRECTORY "${DIRECTORY}" COMPILE_DEFINITIONS)
+    pvs_studio_append_flags_from_property(CXX_FLAGS C_FLAGS "" "-D")
+
+    set("${CXX}" "${CXX_FLAGS}" PARENT_SCOPE)
+    set("${C}" "${C_FLAGS}" PARENT_SCOPE)
+endfunction ()
+
+function (pvs_studio_set_target_flags TARGET CXX C)
+    set(CXX_FLAGS "${${CXX}}")
+    set(C_FLAGS "${${C}}")
+
+    set(prop_incdirs "$<TARGET_PROPERTY:${TARGET},INCLUDE_DIRECTORIES>")
+    list(APPEND CXX_FLAGS "$<$<BOOL:${prop_incdirs}>:-I$<JOIN:${prop_incdirs},$<SEMICOLON>-I>>")
+    list(APPEND C_FLAGS "$<$<BOOL:${prop_incdirs}>:-I$<JOIN:${prop_incdirs},$<SEMICOLON>-I>>")
+
+    set(prop_compdefs "$<TARGET_PROPERTY:${TARGET},COMPILE_DEFINITIONS>")
+    list(APPEND CXX_FLAGS "$<$<BOOL:${prop_compdefs}>:-D$<JOIN:${prop_compdefs},$<SEMICOLON>-D>>")
+    list(APPEND C_FLAGS "$<$<BOOL:${prop_compdefs}>:-D$<JOIN:${prop_compdefs},$<SEMICOLON>-D>>")
+
+    set("${CXX}" "${CXX_FLAGS}" PARENT_SCOPE)
+    set("${C}" "${C_FLAGS}" PARENT_SCOPE)
+endfunction ()
+
+function (pvs_studio_set_source_file_flags SOURCE)
+    set(LANGUAGE "")
+
+    string(TOLOWER "${SOURCE}" SOURCE_LOWER)
+    if ("${LANGUAGE}" STREQUAL "" AND "${SOURCE_LOWER}" MATCHES "^.*\\.(c|cpp|cc|cx|cxx|cp|c\\+\\+)$")
+        if ("${SOURCE}" MATCHES "^.*\\.c$")
+            set(LANGUAGE C)
+        else ()
+            set(LANGUAGE CXX)
+        endif ()
+    endif ()
+
+    if ("${LANGUAGE}" STREQUAL "C")
+        set(CL_PARAMS ${PVS_STUDIO_C_FLAGS} ${PVS_STUDIO_TARGET_C_FLAGS} -DPVS_STUDIO)
+    elseif ("${LANGUAGE}" STREQUAL "CXX")
+        set(CL_PARAMS ${PVS_STUDIO_CXX_FLAGS} ${PVS_STUDIO_TARGET_CXX_FLAGS} -DPVS_STUDIO)
+    endif ()
+
+    set(PVS_STUDIO_LANGUAGE "${LANGUAGE}" PARENT_SCOPE)
+    set(PVS_STUDIO_CL_PARAMS "${CL_PARAMS}" PARENT_SCOPE)
+endfunction ()
+
+function (pvs_studio_analyze_file SOURCE SOURCE_DIR BINARY_DIR)
+    set(PLOGS ${PVS_STUDIO_PLOGS})
+    pvs_studio_set_source_file_flags("${SOURCE}")
+
+    get_filename_component(SOURCE "${SOURCE}" REALPATH)
+
+    get_source_file_property(PROPERTY "${SOURCE}" HEADER_FILE_ONLY)
+    if (PROPERTY)
+        return()
+    endif ()
+
+    pvs_studio_relative_path(SOURCE_RELATIVE "${SOURCE_DIR}" "${SOURCE}")
+    pvs_studio_join_path(SOURCE "${SOURCE_DIR}" "${SOURCE}")
+
+    set(LOG "${BINARY_DIR}/PVS-Studio/${SOURCE_RELATIVE}.plog")
+    get_filename_component(LOG "${LOG}" REALPATH)
+    get_filename_component(PARENT_DIR "${LOG}" DIRECTORY)
+
+    if (EXISTS "${SOURCE}" AND NOT TARGET "${LOG}" AND NOT "${PVS_STUDIO_LANGUAGE}" STREQUAL "")
+        # A workaround to support implicit dependencies for ninja generators.
+        set(depPvsArg)
+        set(depCommandArg)
+        if (CMAKE_VERSION VERSION_GREATER 3.6 AND "${CMAKE_GENERATOR}" STREQUAL "Ninja")
+            pvs_studio_relative_path(relLog "${CMAKE_BINARY_DIR}" "${LOG}")
+            set(depPvsArg --dep-file "${LOG}.d" --dep-file-target "${relLog}")
+            set(depCommandArg DEPFILE "${LOG}.d")
+        endif ()
+
+        # https://public.kitware.com/Bug/print_bug_page.php?bug_id=14353
+        # https://public.kitware.com/Bug/file/5436/expand_command.cmake
+        #
+        # It is a workaround to expand generator expressions.
+        set(cmdline "${PVS_STUDIO_BIN}" analyze
+                --output-file "${LOG}"
+                --source-file "${SOURCE}"
+                ${depPvsArg}
+                ${PVS_STUDIO_ARGS}
+                --cl-params "${PVS_STUDIO_CL_PARAMS}" "${SOURCE}")
+
+        string(REPLACE ";" "$<SEMICOLON>" cmdline "${cmdline}")
+        set(pvscmd "${CMAKE_COMMAND}"
+                -D PVS_STUDIO_AS_SCRIPT=TRUE
+                -D "PVS_STUDIO_COMMAND=${cmdline}"
+                -P "${PVS_STUDIO_SCRIPT}"
+                )
+
+        add_custom_command(OUTPUT "${LOG}"
+                COMMAND "${CMAKE_COMMAND}" -E make_directory "${PARENT_DIR}"
+                COMMAND "${CMAKE_COMMAND}" -E remove_directory "${LOG}"
+                COMMAND ${pvscmd}
+                WORKING_DIRECTORY "${BINARY_DIR}"
+                DEPENDS "${SOURCE}" "${PVS_STUDIO_CONFIG}"
+                IMPLICIT_DEPENDS "${PVS_STUDIO_LANGUAGE}" "${SOURCE}"
+                ${depCommandArg}
+                VERBATIM
+                COMMENT "Analyzing ${PVS_STUDIO_LANGUAGE} file ${SOURCE_RELATIVE}")
+        list(APPEND PLOGS "${LOG}")
+    endif ()
+    set(PVS_STUDIO_PLOGS "${PLOGS}" PARENT_SCOPE)
+endfunction ()
+
+function (pvs_studio_analyze_target TARGET DIR)
+    set(PVS_STUDIO_PLOGS "${PVS_STUDIO_PLOGS}")
+    set(PVS_STUDIO_TARGET_CXX_FLAGS "")
+    set(PVS_STUDIO_TARGET_C_FLAGS "")
+
+    get_target_property(PROPERTY "${TARGET}" SOURCES)
+    pvs_studio_relative_path(BINARY_DIR "${CMAKE_SOURCE_DIR}" "${DIR}")
+    if ("${BINARY_DIR}" MATCHES "^/.*$")
+        pvs_studio_join_path(BINARY_DIR "${CMAKE_BINARY_DIR}" "PVS-Studio/__${BINARY_DIR}")
+    else ()
+        pvs_studio_join_path(BINARY_DIR "${CMAKE_BINARY_DIR}" "${BINARY_DIR}")
+    endif ()
+
+    file(MAKE_DIRECTORY "${BINARY_DIR}")
+
+    pvs_studio_set_directory_flags("${DIR}" PVS_STUDIO_TARGET_CXX_FLAGS PVS_STUDIO_TARGET_C_FLAGS)
+    pvs_studio_set_target_flags("${TARGET}" PVS_STUDIO_TARGET_CXX_FLAGS PVS_STUDIO_TARGET_C_FLAGS)
+
+    if (NOT "${PROPERTY}" STREQUAL "NOTFOUND" AND NOT "${PROPERTY}" STREQUAL "PROPERTY-NOTFOUND")
+        foreach (SOURCE ${PROPERTY})
+            pvs_studio_join_path(SOURCE "${DIR}" "${SOURCE}")
+            pvs_studio_analyze_file("${SOURCE}" "${DIR}" "${BINARY_DIR}")
+        endforeach ()
+    endif ()
+
+    set(PVS_STUDIO_PLOGS "${PVS_STUDIO_PLOGS}" PARENT_SCOPE)
+endfunction ()
+
+set(PVS_STUDIO_RECURSIVE_TARGETS)
+set(PVS_STUDIO_RECURSIVE_TARGETS_NEW)
+
+macro(pvs_studio_get_recursive_targets TARGET)
+    get_target_property(libs "${TARGET}" LINK_LIBRARIES)
+    foreach (lib IN LISTS libs)
+        list(FIND PVS_STUDIO_RECURSIVE_TARGETS "${lib}" index)
+        if (TARGET "${lib}" AND "${index}" STREQUAL -1)
+            get_target_property(target_type "${lib}" TYPE)
+            if (NOT "${target_type}" STREQUAL "INTERFACE_LIBRARY")
+                list(APPEND PVS_STUDIO_RECURSIVE_TARGETS "${lib}")
+                list(APPEND PVS_STUDIO_RECURSIVE_TARGETS_NEW "${lib}")
+                pvs_studio_get_recursive_targets("${lib}")
+            endif ()
+        endif ()
+    endforeach ()
+endmacro()
+
+option(PVS_STUDIO_DISABLE OFF "Disable PVS-Studio targets")
+option(PVS_STUDIO_DEBUG OFF "Add debug info")
+
+# pvs_studio_add_target
+# Target options:
+# ALL                           add PVS-Studio target to default build (default: off)
+# TARGET target                 name of analysis target (default: pvs)
+# ANALYZE targets...            targets to analyze
+# RECURSIVE                     analyze target's dependencies (requires CMake 3.5+)
+# COMPILE_COMMANDS              use compile_commands.json instead of targets (specified by the 'ANALYZE' option) to determine files for analysis
+#                               (set CMAKE_EXPORT_COMPILE_COMMANDS, available only for Makefile and Ninja generators)
+#
+# Output options:
+# OUTPUT                        prints report to stdout
+# LOG path                      path to report (default: ${CMAKE_CURRENT_BINARY_DIR}/PVS-Studio.log)
+# FORMAT format                 format of report
+# MODE mode                     analyzers/levels filter (default: GA:1,2)
+# HIDE_HELP                     do not print help message
+#
+# Analyzer options:
+# PLATFORM name                 linux32/linux64 (default: linux64)
+# PREPROCESSOR name             preprocessor type: gcc/clang (default: auto detected)
+# LICENSE path                  path to PVS-Studio.lic (default: ~/.config/PVS-Studio/PVS-Studio.lic)
+# CONFIG path                   path to PVS-Studio.cfg
+# CFG_TEXT text                 embedded PVS-Studio.cfg
+# KEEP_COMBINED_PLOG            do not delete combined plog file *.pvs.raw for further processing with plog-converter
+#
+# Misc options:
+# DEPENDS targets..             additional target dependencies
+# SOURCES path...               list of source files to analyze
+# BIN path                      path to pvs-studio-analyzer (Unix) or CompilerCommandsAnalyzer.exe (Windows)
+# CONVERTER path                path to plog-converter (Unix) or HtmlGenerator.exe (Windows)
+# C_FLAGS flags...              additional C_FLAGS
+# CXX_FLAGS flags...            additional CXX_FLAGS
+# ARGS args...                  additional pvs-studio-analyzer/CompilerCommandsAnalyzer.exe flags
+function (pvs_studio_add_target)
+    macro (default VAR VALUE)
+        if ("${${VAR}}" STREQUAL "")
+            set("${VAR}" "${VALUE}")
+        endif ()
+    endmacro ()
+
+    set(PVS_STUDIO_SUPPORTED_PREPROCESSORS "gcc|clang|visualcpp")
+    if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
+        set(DEFAULT_PREPROCESSOR "clang")
+    elseif (MSVC)
+        set(DEFAULT_PREPROCESSOR "visualcpp")
+    else ()
+        set(DEFAULT_PREPROCESSOR "gcc")
+    endif ()
+
+    set(OPTIONAL OUTPUT ALL RECURSIVE HIDE_HELP KEEP_COMBINED_PLOG COMPILE_COMMANDS)
+    set(SINGLE LICENSE CONFIG TARGET LOG FORMAT BIN CONVERTER PLATFORM PREPROCESSOR CFG_TEXT)
+    set(MULTI SOURCES C_FLAGS CXX_FLAGS ARGS DEPENDS ANALYZE MODE)
+    cmake_parse_arguments(PVS_STUDIO "${OPTIONAL}" "${SINGLE}" "${MULTI}" ${ARGN})
+
+    if ("${PVS_STUDIO_CONFIG}" STREQUAL "" OR NOT "${PVS_STUDIO_CFG_TEXT}" STREQUAL "")
+        set(PVS_STUDIO_EMPTY_CONFIG ON)
+    else ()
+        set(PVS_STUDIO_EMPTY_CONFIG OFF)
+    endif ()
+
+    default(PVS_STUDIO_CFG_TEXT "analysis-mode=31")
+    default(PVS_STUDIO_CONFIG "${CMAKE_BINARY_DIR}/PVS-Studio.cfg")
+    default(PVS_STUDIO_C_FLAGS "")
+    default(PVS_STUDIO_CXX_FLAGS "")
+    default(PVS_STUDIO_TARGET "pvs")
+    default(PVS_STUDIO_LOG "PVS-Studio.log")
+
+    set(PATHS)
+    if (WIN32)
+        set(ROOT "PROGRAMFILES(X86)")
+        set(ROOT "$ENV{${ROOT}}/PVS-Studio")
+        string(REPLACE \\ / ROOT "${ROOT}")
+
+        if (EXISTS "${ROOT}")
+            set(PATHS "${ROOT}")
+        endif ()
+
+        default(PVS_STUDIO_BIN "CompilerCommandsAnalyzer.exe")
+        default(PVS_STUDIO_CONVERTER "HtmlGenerator.exe")
+    else ()
+        default(PVS_STUDIO_BIN "pvs-studio-analyzer")
+        default(PVS_STUDIO_CONVERTER "plog-converter")
+    endif ()
+
+    find_program(PVS_STUDIO_BIN_PATH "${PVS_STUDIO_BIN}" ${PATHS})
+    set(PVS_STUDIO_BIN "${PVS_STUDIO_BIN_PATH}")
+
+    if (NOT EXISTS "${PVS_STUDIO_BIN}")
+        message(FATAL_ERROR "pvs-studio-analyzer is not found")
+    endif ()
+
+    find_program(PVS_STUDIO_CONVERTER_PATH "${PVS_STUDIO_CONVERTER}" ${PATHS})
+    set(PVS_STUDIO_CONVERTER "${PVS_STUDIO_CONVERTER_PATH}")
+
+    if (NOT EXISTS "${PVS_STUDIO_CONVERTER}")
+        message(FATAL_ERROR "plog-converter is not found")
+    endif ()
+
+    default(PVS_STUDIO_MODE "GA:1,2")
+    default(PVS_STUDIO_PREPROCESSOR "${DEFAULT_PREPROCESSOR}")
+    if (WIN32)
+        default(PVS_STUDIO_PLATFORM "x64")
+    else ()
+        default(PVS_STUDIO_PLATFORM "linux64")
+    endif ()
+
+    string(REPLACE ";" "+" PVS_STUDIO_MODE "${PVS_STUDIO_MODE}")
+
+    if (PVS_STUDIO_EMPTY_CONFIG)
+        set(PVS_STUDIO_CONFIG_COMMAND "${CMAKE_COMMAND}" -E echo "${PVS_STUDIO_CFG_TEXT}" > "${PVS_STUDIO_CONFIG}")
+    else ()
+        set(PVS_STUDIO_CONFIG_COMMAND "${CMAKE_COMMAND}" -E touch "${PVS_STUDIO_CONFIG}")
+    endif ()
+
+    add_custom_command(OUTPUT "${PVS_STUDIO_CONFIG}"
+            COMMAND ${PVS_STUDIO_CONFIG_COMMAND}
+            WORKING_DIRECTORY "${BINARY_DIR}"
+            COMMENT "Generating PVS-Studio.cfg")
+
+    if (NOT "${PVS_STUDIO_PREPROCESSOR}" MATCHES "^${PVS_STUDIO_SUPPORTED_PREPROCESSORS}$")
+        message(FATAL_ERROR "Preprocessor ${PVS_STUDIO_PREPROCESSOR} isn't supported. Available options: ${PVS_STUDIO_SUPPORTED_PREPROCESSORS}.")
+    endif ()
+
+    pvs_studio_append_standard_flag(PVS_STUDIO_CXX_FLAGS "${CMAKE_CXX_STANDARD}")
+    pvs_studio_set_directory_flags("${CMAKE_CURRENT_SOURCE_DIR}" PVS_STUDIO_CXX_FLAGS PVS_STUDIO_C_FLAGS)
+
+    if (NOT "${PVS_STUDIO_LICENSE}" STREQUAL "")
+        pvs_studio_join_path(PVS_STUDIO_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}" "${PVS_STUDIO_LICENSE}")
+        list(APPEND PVS_STUDIO_ARGS --lic-file "${PVS_STUDIO_LICENSE}")
+    endif ()
+
+    list(APPEND PVS_STUDIO_ARGS --cfg "${PVS_STUDIO_CONFIG}"
+            --platform "${PVS_STUDIO_PLATFORM}"
+            --preprocessor "${PVS_STUDIO_PREPROCESSOR}")
+
+    if (NOT "${CMAKE_CXX_COMPILER}" STREQUAL "")
+        list(APPEND PVS_STUDIO_ARGS --cxx "${CMAKE_CXX_COMPILER}")
+    endif ()
+
+    if (NOT "${CMAKE_C_COMPILER}" STREQUAL "")
+        list(APPEND PVS_STUDIO_ARGS --cc "${CMAKE_C_COMPILER}")
+    endif ()
+
+    set(PVS_STUDIO_PLOGS "")
+
+    set(PVS_STUDIO_RECURSIVE_TARGETS_NEW)
+    if (${PVS_STUDIO_RECURSIVE})
+        foreach (TARGET IN LISTS PVS_STUDIO_ANALYZE)
+            list(APPEND PVS_STUDIO_RECURSIVE_TARGETS_NEW "${TARGET}")
+            pvs_studio_get_recursive_targets("${TARGET}")
+        endforeach ()
+    endif ()
+
+    set(inc_path)
+
+    foreach (TARGET ${PVS_STUDIO_ANALYZE})
+        set(DIR "${CMAKE_CURRENT_SOURCE_DIR}")
+        string(FIND "${TARGET}" ":" DELIM)
+        if ("${DELIM}" GREATER "-1")
+            math(EXPR DELIMI "${DELIM}+1")
+            string(SUBSTRING "${TARGET}" "${DELIMI}" "-1" DIR)
+            string(SUBSTRING "${TARGET}" "0" "${DELIM}" TARGET)
+            pvs_studio_join_path(DIR "${CMAKE_CURRENT_SOURCE_DIR}" "${DIR}")
+        else ()
+            get_target_property(TARGET_SOURCE_DIR "${TARGET}" SOURCE_DIR)
+            if (EXISTS "${TARGET_SOURCE_DIR}")
+                set(DIR "${TARGET_SOURCE_DIR}")
+            endif ()
+        endif ()
+        pvs_studio_analyze_target("${TARGET}" "${DIR}")
+        list(APPEND PVS_STUDIO_DEPENDS "${TARGET}")
+
+        if ("${inc_path}" STREQUAL "")
+            set(inc_path "$<TARGET_PROPERTY:${TARGET},INCLUDE_DIRECTORIES>")
+        else ()
+            set(inc_path "${inc_path}$<SEMICOLON>$<TARGET_PROPERTY:${TARGET},INCLUDE_DIRECTORIES>")
+        endif ()
+    endforeach ()
+
+    foreach (TARGET ${PVS_STUDIO_RECURSIVE_TARGETS_NEW})
+        set(DIR "${CMAKE_CURRENT_SOURCE_DIR}")
+        get_target_property(TARGET_SOURCE_DIR "${TARGET}" SOURCE_DIR)
+        if (EXISTS "${TARGET_SOURCE_DIR}")
+            set(DIR "${TARGET_SOURCE_DIR}")
+        endif ()
+        pvs_studio_analyze_target("${TARGET}" "${DIR}")
+        list(APPEND PVS_STUDIO_DEPENDS "${TARGET}")
+    endforeach ()
+
+    set(PVS_STUDIO_TARGET_CXX_FLAGS "")
+    set(PVS_STUDIO_TARGET_C_FLAGS "")
+    foreach (SOURCE ${PVS_STUDIO_SOURCES})
+        pvs_studio_analyze_file("${SOURCE}" "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}")
+    endforeach ()
+
+    if (PVS_STUDIO_COMPILE_COMMANDS)
+        set(COMPILE_COMMANDS_LOG "${PVS_STUDIO_LOG}.pvs.analyzer.raw")
+        if (NOT CMAKE_EXPORT_COMPILE_COMMANDS)
+            message(FATAL_ERROR "You should set CMAKE_EXPORT_COMPILE_COMMANDS to TRUE")
+        endif ()
+        add_custom_command(
+                OUTPUT "${COMPILE_COMMANDS_LOG}"
+                COMMAND "${PVS_STUDIO_BIN}" analyze -i
+                --output-file "${COMPILE_COMMANDS_LOG}.always"
+                ${PVS_STUDIO_ARGS}
+                COMMENT "Analyzing with PVS-Studio"
+                WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
+                DEPENDS "${PVS_STUDIO_CONFIG}"
+        )
+        list(APPEND PVS_STUDIO_PLOGS_LOGS "${COMPILE_COMMANDS_LOG}.always")
+        list(APPEND PVS_STUDIO_PLOGS_DEPENDENCIES "${COMPILE_COMMANDS_LOG}")
+    endif ()
+
+    pvs_studio_relative_path(LOG_RELATIVE "${CMAKE_BINARY_DIR}" "${PVS_STUDIO_LOG}")
+    if (PVS_STUDIO_PLOGS OR PVS_STUDIO_COMPILE_COMMANDS)
+        if (WIN32)
+            string(REPLACE / \\ PVS_STUDIO_PLOGS "${PVS_STUDIO_PLOGS}")
+        endif ()
+        if (WIN32)
+            set(COMMANDS COMMAND type ${PVS_STUDIO_PLOGS} ${PVS_STUDIO_PLOGS_LOGS} > "${PVS_STUDIO_LOG}" 2>nul)
+        else ()
+            set(COMMANDS COMMAND cat ${PVS_STUDIO_PLOGS} ${PVS_STUDIO_PLOGS_LOGS} > "${PVS_STUDIO_LOG}")
+        endif ()
+        set(COMMENT "Generating ${LOG_RELATIVE}")
+        if (NOT "${PVS_STUDIO_FORMAT}" STREQUAL "" OR PVS_STUDIO_OUTPUT)
+            if ("${PVS_STUDIO_FORMAT}" STREQUAL "")
+                set(PVS_STUDIO_FORMAT "errorfile")
+            endif ()
+            list(APPEND COMMANDS
+                    COMMAND "${CMAKE_COMMAND}" -E remove -f "${PVS_STUDIO_LOG}.pvs.raw"
+                    COMMAND "${CMAKE_COMMAND}" -E rename "${PVS_STUDIO_LOG}" "${PVS_STUDIO_LOG}.pvs.raw"
+                    COMMAND "${PVS_STUDIO_CONVERTER}" -t "${PVS_STUDIO_FORMAT}" "${PVS_STUDIO_LOG}.pvs.raw" -o "${PVS_STUDIO_LOG}" -a "${PVS_STUDIO_MODE}"
+                    )
+            if(NOT PVS_STUDIO_KEEP_COMBINED_PLOG)
+                list(APPEND COMMANDS COMMAND "${CMAKE_COMMAND}" -E remove -f "${PVS_STUDIO_LOG}.pvs.raw")
+            endif()
+        endif ()
+    else ()
+        set(COMMANDS COMMAND "${CMAKE_COMMAND}" -E touch "${PVS_STUDIO_LOG}")
+        set(COMMENT "Generating ${LOG_RELATIVE}: no sources found")
+    endif ()
+
+    if (WIN32)
+        string(REPLACE / \\ PVS_STUDIO_LOG "${PVS_STUDIO_LOG}")
+    endif ()
+
+    add_custom_command(OUTPUT "${PVS_STUDIO_LOG}"
+            ${COMMANDS}
+            COMMENT "${COMMENT}"
+            DEPENDS ${PVS_STUDIO_PLOGS} ${PVS_STUDIO_PLOGS_DEPENDENCIES}
+            WORKING_DIRECTORY "${CMAKE_BINARY_DIR}")
+
+    if (PVS_STUDIO_ALL)
+        set(ALL "ALL")
+    else ()
+        set(ALL "")
+    endif ()
+
+    if (PVS_STUDIO_OUTPUT)
+        if (PVS_STUDIO_HIDE_HELP AND NOT WIN32)
+            set(COMMANDS COMMAND grep -v " error: Help:" ${PVS_STUDIO_LOG} 1>&2 || exit 0)
+        elseif (WIN32)
+            set(COMMANDS COMMAND type "${PVS_STUDIO_LOG}" 1>&2)
+        else ()
+            set(COMMANDS COMMAND cat "${PVS_STUDIO_LOG}" 1>&2)
+        endif()
+    else ()
+        set(COMMANDS "")
+    endif ()
+
+    add_custom_target("${PVS_STUDIO_TARGET}" ${ALL} ${COMMANDS} WORKING_DIRECTORY "${CMAKE_BINARY_DIR}" DEPENDS ${PVS_STUDIO_DEPENDS} "${PVS_STUDIO_LOG}")
+
+    # A workaround to add implicit dependencies of source files from include directories
+    set_target_properties("${PVS_STUDIO_TARGET}" PROPERTIES INCLUDE_DIRECTORIES "${inc_path}")
+endfunction ()
\ No newline at end of file
diff --git a/src/client/rspamc.c b/src/client/rspamc.c
index 0568692b4..927097e18 100644
--- a/src/client/rspamc.c
+++ b/src/client/rspamc.c
@@ -680,15 +680,15 @@ add_options (GQueue *opts)
 
 		if (kv == NULL || kv[1] == NULL) {
 			ADD_CLIENT_HEADER (opts, *hdr, "");
-
-			if (kv) {
-				g_strfreev (kv);
-			}
 		}
 		else {
 			ADD_CLIENT_HEADER (opts, kv[0], kv[1]);
 		}
 
+		if (kv) {
+			g_strfreev (kv);
+		}
+
 		hdr ++;
 	}
 }
@@ -1006,9 +1006,9 @@ rspamc_counters_sort (const ucl_object_t **o1, const ucl_object_t **o2)
 					order2 = ucl_object_toint (elt2);
 				}
 			}
-
-			g_strfreev (args);
 		}
+
+		g_strfreev (args);
 	}
 
 	return (inverse ? (order2 - order1) : (order1 - order2));
@@ -2008,7 +2008,7 @@ main (gint argc, gchar **argv, gchar **env)
 	g_queue_free_full (kwattrs, g_free);
 
 	/* Wait for children processes */
-	cur = g_list_first (children);
+	cur = children ? g_list_first (children) : NULL;
 	ret = 0;
 
 	while (cur) {
diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c
index 7fdce82ae..2054633ad 100644
--- a/src/fuzzy_storage.c
+++ b/src/fuzzy_storage.c
@@ -599,6 +599,7 @@ rspamd_fuzzy_send_update_mirror (struct rspamd_fuzzy_storage_ctx *ctx,
 	conn->mirror = m;
 
 	if (conn->up == NULL) {
+		g_free (conn);
 		msg_err ("cannot select upstream for %s", m->name);
 		return;
 	}
@@ -608,6 +609,7 @@ rspamd_fuzzy_send_update_mirror (struct rspamd_fuzzy_storage_ctx *ctx,
 			SOCK_STREAM, TRUE);
 
 	if (conn->sock == -1) {
+		g_free (conn);
 		msg_err ("cannot connect upstream for %s", m->name);
 		rspamd_upstream_fail (conn->up, TRUE);
 		return;
diff --git a/src/libmime/archives.c b/src/libmime/archives.c
index f546570a9..b84670761 100644
--- a/src/libmime/archives.c
+++ b/src/libmime/archives.c
@@ -1146,12 +1146,14 @@ rspamd_7zip_read_coders_info (struct rspamd_task *task,
 					return NULL;
 				}
 
-				folder_nstreams = g_alloca (sizeof (int) * num_folders);
+				folder_nstreams = g_malloc (sizeof (int) * num_folders);
 
 				for (i = 0; i < num_folders && p != NULL && p < end; i++) {
 					p = rspamd_7zip_read_folder (task, p, end, arch,
 							&folder_nstreams[i], &num_digests);
 				}
+
+				g_free (folder_nstreams);
 			}
 			break;
 		case kCodersUnPackSize:
@@ -1499,6 +1501,7 @@ rspamd_7zip_read_files_info (struct rspamd_task *task,
 					if (fend == NULL || fend - p == 0) {
 						/* Crap instead of fname */
 						msg_debug_archive ("bad 7zip name; %s", G_STRLOC);
+						goto end;
 					}
 
 					res = rspamd_7zip_ucs2_to_utf8 (task, p, fend);
diff --git a/src/libserver/dkim.c b/src/libserver/dkim.c
index 9692d1b93..8100f8ee3 100644
--- a/src/libserver/dkim.c
+++ b/src/libserver/dkim.c
@@ -2664,6 +2664,7 @@ rspamd_dkim_sign_key_load (const gchar *key, gsize len,
 		if (stat (key, &st) != 0) {
 			g_set_error (err, dkim_error_quark (), DKIM_SIGERROR_KEYFAIL,
 					"cannot stat key file: '%s' %s", key, strerror (errno));
+			g_free (nkey);
 
 			return NULL;
 		}
@@ -2674,6 +2675,7 @@ rspamd_dkim_sign_key_load (const gchar *key, gsize len,
 		if (map == NULL) {
 			g_set_error (err, dkim_error_quark (), DKIM_SIGERROR_KEYFAIL,
 					"cannot map key file: '%s' %s", key, strerror (errno));
+			g_free (nkey);
 
 			return NULL;
 		}
diff --git a/src/libserver/milter.c b/src/libserver/milter.c
index 236a4bf75..fcdbcae6c 100644
--- a/src/libserver/milter.c
+++ b/src/libserver/milter.c
@@ -1301,7 +1301,7 @@ rspamd_milter_del_header (struct rspamd_milter_session *session,
 	value.len = 0;
 
 	return rspamd_milter_send_action (session, RSPAMD_MILTER_CHGHEADER,
-			idx, name, value);
+			idx, name, &value);
 }
 
 void
@@ -1935,7 +1935,7 @@ rspamd_milter_send_task_results (struct rspamd_milter_session *session,
 				}
 			}
 
-			rspamd_milter_set_reply (session, rcode, xcode, reply);
+			rspamd_milter_set_reply (session, xcode, rcode, reply);
 		}
 		break;
 	case METRIC_ACTION_SOFT_REJECT:
@@ -1949,7 +1949,7 @@ rspamd_milter_send_task_results (struct rspamd_milter_session *session,
 					sizeof (RSPAMD_MILTER_TEMPFAIL_MESSAGE) - 1);
 		}
 
-		rspamd_milter_set_reply (session, rcode, xcode, reply);
+		rspamd_milter_set_reply (session, xcode, rcode, reply);
 		break;
 
 	case METRIC_ACTION_REWRITE_SUBJECT:
diff --git a/src/libserver/rspamd_symcache.c b/src/libserver/rspamd_symcache.c
index 3cfb15408..64e451f7a 100644
--- a/src/libserver/rspamd_symcache.c
+++ b/src/libserver/rspamd_symcache.c
@@ -1580,7 +1580,6 @@ rspamd_symcache_make_checkpoint (struct rspamd_task *task,
 	checkpoint->pass = RSPAMD_CACHE_PASS_INIT;
 	task->checkpoint = checkpoint;
 
-	task->result = task->result;
 
 	return checkpoint;
 }
diff --git a/src/libserver/task.c b/src/libserver/task.c
index 30bccb81b..c4e69c974 100644
--- a/src/libserver/task.c
+++ b/src/libserver/task.c
@@ -1231,7 +1231,7 @@ rspamd_task_write_ialist (struct rspamd_task *task,
 	struct rspamd_email_address *addr;
 	gint i, nchars = 0, cur_chars;
 
-	if (lim <= 0) {
+	if (addrs && lim <= 0) {
 		lim = addrs->len;
 	}
 
diff --git a/src/libutil/http_connection.c b/src/libutil/http_connection.c
index dc3bfa9c0..42b934d2a 100644
--- a/src/libutil/http_connection.c
+++ b/src/libutil/http_connection.c
@@ -722,10 +722,15 @@ rspamd_http_simple_client_helper (struct rspamd_http_connection *conn)
 	priv = conn->priv;
 	ssl = priv->ssl;
 	priv->ssl = NULL;
-	request_method = priv->msg->method;
-	/* Preserve host for keepalive */
-	prev_host = priv->msg->host;
-	priv->msg->host = NULL;
+
+	/* Preserve data */
+	if (priv->msg) {
+		request_method = priv->msg->method;
+		/* Preserve host for keepalive */
+		prev_host = priv->msg->host;
+		priv->msg->host = NULL;
+	}
+
 	rspamd_http_connection_reset (conn);
 	priv->ssl = ssl;
 
@@ -1988,12 +1993,6 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn
 							""
 							"\r\n", ENCRYPTED_VERSION, bodylen);
 				}
-
-				preludelen = rspamd_snprintf (repbuf, sizeof (repbuf), "%s\r\n"
-						"Content-Length: %z\r\n"
-						"Content-Type: %s\r\n"
-						"\r\n", ENCRYPTED_VERSION, bodylen,
-						mime_type);
 			}
 			else {
 				preludelen = rspamd_snprintf (repbuf, sizeof (repbuf),
diff --git a/src/libutil/logger.c b/src/libutil/logger.c
index 45e99f8ae..7eb00f3fa 100644
--- a/src/libutil/logger.c
+++ b/src/libutil/logger.c
@@ -925,7 +925,7 @@ file_log_function (const gchar *module, const gchar *id,
 	gchar *m;
 	gdouble now;
 
-	struct iovec iov[5];
+	struct iovec iov[6];
 	gulong r = 0, mr = 0;
 	guint64 cksum;
 	size_t mlen, mremain;
diff --git a/src/libutil/map.c b/src/libutil/map.c
index d4687e433..0f7ff4f48 100644
--- a/src/libutil/map.c
+++ b/src/libutil/map.c
@@ -2685,7 +2685,7 @@ rspamd_map_add_from_ucl (struct rspamd_config *cfg,
 			}
 		}
 
-		if (map->backends->len == 0) {
+		if (!map->backends || map->backends->len == 0) {
 			msg_err_config ("map has no urls to be loaded: no valid backends");
 			goto err;
 		}
diff --git a/src/libutil/mem_pool.c b/src/libutil/mem_pool.c
index 2f39a116d..a519013d1 100644
--- a/src/libutil/mem_pool.c
+++ b/src/libutil/mem_pool.c
@@ -490,9 +490,9 @@ void *
 rspamd_mempool_alloc0 (rspamd_mempool_t * pool, gsize size)
 {
 	void *pointer = rspamd_mempool_alloc (pool, size);
-	if (pointer) {
-		memset (pointer, 0, size);
-	}
+
+	memset (pointer, 0, size);
+
 	return pointer;
 }
 
@@ -500,9 +500,9 @@ void *
 rspamd_mempool_alloc0_tmp (rspamd_mempool_t * pool, gsize size)
 {
 	void *pointer = rspamd_mempool_alloc_tmp (pool, size);
-	if (pointer) {
-		memset (pointer, 0, size);
-	}
+
+	memset (pointer, 0, size);
+
 	return pointer;
 }
 
@@ -510,9 +510,8 @@ void *
 rspamd_mempool_alloc0_shared (rspamd_mempool_t * pool, gsize size)
 {
 	void *pointer = rspamd_mempool_alloc_shared (pool, size);
-	if (pointer) {
-		memset (pointer, 0, size);
-	}
+
+	memset (pointer, 0, size);
 	return pointer;
 }
 
@@ -800,7 +799,6 @@ rspamd_mempool_stat (rspamd_mempool_stat_t * st)
 		st->shared_chunks_allocated = mem_pool_stat->shared_chunks_allocated;
 		st->bytes_allocated = mem_pool_stat->bytes_allocated;
 		st->chunks_allocated = mem_pool_stat->chunks_allocated;
-		st->shared_chunks_allocated = mem_pool_stat->shared_chunks_allocated;
 		st->chunks_freed = mem_pool_stat->chunks_freed;
 		st->oversized_chunks = mem_pool_stat->oversized_chunks;
 	}
diff --git a/src/libutil/multipattern.c b/src/libutil/multipattern.c
index 268170512..5f5724a0c 100644
--- a/src/libutil/multipattern.c
+++ b/src/libutil/multipattern.c
@@ -209,7 +209,6 @@ rspamd_multipattern_pattern_filter (const gchar *pattern, gsize len,
 			g_free (tmp);
 		}
 		else if (flags & RSPAMD_MULTIPATTERN_RE) {
-			ret = malloc (len + 1);
 			ret = rspamd_str_regexp_escape (pattern, len, dst_len, gl_flags |
 					RSPAMD_REGEXP_ESCAPE_RE);
 		}
diff --git a/src/libutil/rrd.c b/src/libutil/rrd.c
index 95d78c7f5..d17e823e6 100644
--- a/src/libutil/rrd.c
+++ b/src/libutil/rrd.c
@@ -228,12 +228,6 @@ rspamd_rrd_check_file (const gchar *filename, gboolean need_data, GError **err)
 		close (fd);
 		return FALSE;
 	}
-	if (memcmp (head.version, RRD_VERSION, sizeof (head.version)) != 0) {
-		g_set_error (err,
-				rrd_error_quark (), EINVAL, "rrd head error: invalid version");
-		close (fd);
-		return FALSE;
-	}
 	if (head.float_cookie != RRD_FLOAT_COOKIE) {
 		g_set_error (err,
 				rrd_error_quark (), EINVAL, "rrd head error: another architecture "
@@ -409,16 +403,12 @@ rspamd_rrd_open_common (const gchar *filename, gboolean completed, GError **err)
 
 	file = g_malloc0 (sizeof (struct rspamd_rrd_file));
 
-	if (file == NULL) {
-		g_set_error (err, rrd_error_quark (), ENOMEM, "not enough memory");
-		return NULL;
-	}
-
 	/* Open file */
 	fd = rspamd_rrd_open_exclusive (filename);
 	if (fd == -1) {
 		g_set_error (err,
 			rrd_error_quark (), errno, "rrd open error: %s", strerror (errno));
+		g_free (file);
 		return FALSE;
 	}
 
@@ -426,6 +416,7 @@ rspamd_rrd_open_common (const gchar *filename, gboolean completed, GError **err)
 		g_set_error (err,
 			rrd_error_quark (), errno, "rrd stat error: %s", strerror (errno));
 		rspamd_file_unlock (fd, FALSE);
+		g_free (file);
 		close (fd);
 		return FALSE;
 	}
diff --git a/src/libutil/shingles.c b/src/libutil/shingles.c
index 70aa5fe78..4affb16c3 100644
--- a/src/libutil/shingles.c
+++ b/src/libutil/shingles.c
@@ -184,6 +184,10 @@ rspamd_shingles_from_text (GArray *input,
 
 						g_free (hashes);
 
+						if (pool != NULL) {
+							g_free (res);
+						}
+
 						return NULL;
 					}
 
diff --git a/src/lua/lua_common.c b/src/lua/lua_common.c
index 737455a3c..cff684aad 100644
--- a/src/lua/lua_common.c
+++ b/src/lua/lua_common.c
@@ -1940,7 +1940,6 @@ rspamd_lua_execute_lua_subprocess (lua_State *L,
 		msg_err ("call to subprocess failed: %v", tb);
 		/* Indicate error */
 		wlen = (1ULL << 63) + tb->len;
-		g_string_free (tb, TRUE);
 
 		r = write (cbdata->sp[1], &wlen, sizeof (wlen));
 		if (r == -1) {
@@ -1951,6 +1950,7 @@ rspamd_lua_execute_lua_subprocess (lua_State *L,
 		if (r == -1) {
 			msg_err ("write failed: %s", strerror (errno));
 		}
+		g_string_free (tb, TRUE);
 
 		lua_pop (L, 1);
 	}
@@ -2205,9 +2205,9 @@ lua_worker_spawn_process (lua_State *L)
 
 	if (rspamd_socketpair (cbdata->sp, TRUE) == -1) {
 		msg_err ("cannot spawn socketpair: %s", strerror (errno));
-		g_free (cbdata);
 		luaL_unref (L, LUA_REGISTRYINDEX, cbdata->func_cbref);
 		luaL_unref (L, LUA_REGISTRYINDEX, cbdata->cb_cbref);
+		g_free (cbdata);
 
 		return 0;
 	}
diff --git a/src/lua/lua_config.c b/src/lua/lua_config.c
index e846fd4d7..91c648e6f 100644
--- a/src/lua/lua_config.c
+++ b/src/lua/lua_config.c
@@ -3900,13 +3900,19 @@ lua_config_init_subsystem (lua_State *L)
 							cfg);
 				}
 				else {
+					g_strfreev (parts);
+
 					return luaL_error (L, "no event base specified");
 				}
 			}
 			else {
+				g_strfreev (parts);
+
 				return luaL_error (L, "invalid param: %s", parts[i]);
 			}
 		}
+
+		g_strfreev (parts);
 	}
 	else {
 		return luaL_error (L, "invalid arguments");
diff --git a/src/lua/lua_html.c b/src/lua/lua_html.c
index 63839c286..1b5828564 100644
--- a/src/lua/lua_html.c
+++ b/src/lua/lua_html.c
@@ -249,9 +249,6 @@ lua_html_has_property (lua_State *L)
 		else if (strcmp (propname, "unbalanced") == 0) {
 			ret = hc->flags & RSPAMD_HTML_FLAG_UNBALANCED;
 		}
-		else if (strcmp (propname, "unbalanced") == 0) {
-			ret = hc->flags & RSPAMD_HTML_FLAG_UNBALANCED;
-		}
 		else if (strcmp (propname, "data_urls") == 0) {
 			ret = hc->flags & RSPAMD_HTML_FLAG_HAS_DATA_URLS;
 		}
diff --git a/src/lua/lua_http.c b/src/lua/lua_http.c
index 72eec70ad..e518e6dab 100644
--- a/src/lua/lua_http.c
+++ b/src/lua/lua_http.c
@@ -752,6 +752,10 @@ lua_http_request (lua_State *L)
 							rspamd_fstring_free (body);
 						}
 
+						if (mime_type) {
+							g_free (mime_type);
+						}
+
 						return luaL_error (L, "invalid body argument: %s",
 								lua_typename (L, lua_type (L, -1)));
 					}
diff --git a/src/lua/lua_thread_pool.c b/src/lua/lua_thread_pool.c
index ec9bfbe6f..d10f64579 100644
--- a/src/lua/lua_thread_pool.c
+++ b/src/lua/lua_thread_pool.c
@@ -305,7 +305,7 @@ lua_resume_thread_internal_full (struct thread_entry *thread_entry,
 		}
 		else {
 			tb = rspamd_lua_get_traceback_string (thread_entry->lua_state);
-			if (thread_entry->error_callback) {
+			if (tb && thread_entry->error_callback) {
 				thread_entry->error_callback (thread_entry, ret, tb->str);
 			}
 			else if (thread_entry->task) {
diff --git a/src/lua/lua_util.c b/src/lua/lua_util.c
index d78e230e7..dad8c336b 100644
--- a/src/lua/lua_util.c
+++ b/src/lua/lua_util.c
@@ -1177,7 +1177,7 @@ lua_util_tokenize_text (lua_State *L)
 {
 	LUA_TRACE_POINT;
 	const gchar *in = NULL;
-	gsize len, pos, ex_len, i;
+	gsize len = 0, pos, ex_len, i;
 	GList *exceptions = NULL, *cur;
 	struct rspamd_lua_text *t;
 	struct rspamd_process_exception *ex;
@@ -2243,6 +2243,8 @@ lua_util_gzip_decompress (lua_State *L)
 		return luaL_error (L, "invalid arguments");
 	}
 
+	sz = t->len;
+
 	memset (&strm, 0, sizeof (strm));
 	/* windowBits +16 to decode gzip, zlib 1.2.0.4+ */
 	rc = inflateInit2 (&strm, MAX_WBITS + 16);
diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c
index 316ca7f62..d2632cb81 100644
--- a/src/plugins/surbl.c
+++ b/src/plugins/surbl.c
@@ -1864,9 +1864,6 @@ surbl_test_tags (struct rspamd_task *task, struct redirector_param *param,
 		tld.len = url->tldlen;
 
 		ftld = rspamd_mempool_ftokdup (task->task_pool, &tld);
-	}
-
-	if (tag) {
 		/* We know results for this URL */
 
 		DL_FOREACH (tag, cur) {
-- 
cgit v1.2.3


From dd1b49fbdbf49b16fbb16df44f307e62c122ba1b Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Thu, 14 Mar 2019 12:46:57 +0000
Subject: [Minor] Fix rspamc headers parsing

---
 src/client/rspamc.c       | 19 ++++++++++++++++---
 src/client/rspamdclient.h |  4 ++--
 2 files changed, 18 insertions(+), 5 deletions(-)

(limited to 'src')

diff --git a/src/client/rspamc.c b/src/client/rspamc.c
index 927097e18..08a267f1b 100644
--- a/src/client/rspamc.c
+++ b/src/client/rspamc.c
@@ -74,8 +74,8 @@ static gint retcode = EXIT_SUCCESS;
 #define ADD_CLIENT_HEADER(o, n, v) do { \
     struct rspamd_http_client_header *nh; \
     nh = g_malloc (sizeof (*nh)); \
-    nh->name = (n); \
-    nh->value = (v); \
+    nh->name = g_strdup (n); \
+    nh->value = g_strdup (v); \
     g_queue_push_tail ((o), nh); \
 } while (0)
 
@@ -645,6 +645,7 @@ add_options (GQueue *opts)
 		numbuf = g_string_sized_new (8);
 		rspamd_printf_gstring (numbuf, "%d", weight);
 		ADD_CLIENT_HEADER (opts, "Weight", numbuf->str);
+		g_string_free (numbuf, TRUE);
 	}
 
 	if (fuzzy_symbol != NULL) {
@@ -655,6 +656,7 @@ add_options (GQueue *opts)
 		numbuf = g_string_sized_new (8);
 		rspamd_printf_gstring (numbuf, "%d", flag);
 		ADD_CLIENT_HEADER (opts, "Flag", numbuf->str);
+		g_string_free (numbuf, TRUE);
 	}
 
 	if (extended_urls) {
@@ -1841,6 +1843,17 @@ rspamc_process_dir (struct event_base *ev_base, struct rspamc_command *cmd,
 	event_base_loop (ev_base, 0);
 }
 
+
+static void
+rspamc_kwattr_free (gpointer p)
+{
+	struct rspamd_http_client_header *h = (struct rspamd_http_client_header *)p;
+
+	g_free (h->value);
+	g_free (h->name);
+	g_free (h);
+}
+
 gint
 main (gint argc, gchar **argv, gchar **env)
 {
@@ -2005,7 +2018,7 @@ main (gint argc, gchar **argv, gchar **env)
 
 	event_base_loop (ev_base, 0);
 
-	g_queue_free_full (kwattrs, g_free);
+	g_queue_free_full (kwattrs, rspamc_kwattr_free);
 
 	/* Wait for children processes */
 	cur = children ? g_list_first (children) : NULL;
diff --git a/src/client/rspamdclient.h b/src/client/rspamdclient.h
index c2a3c1886..8a5b3de35 100644
--- a/src/client/rspamdclient.h
+++ b/src/client/rspamdclient.h
@@ -24,8 +24,8 @@ struct rspamd_client_connection;
 struct rspamd_http_message;
 
 struct rspamd_http_client_header {
-	const gchar *name;
-	const gchar *value;
+	gchar *name;
+	gchar *value;
 };
 
 /**
-- 
cgit v1.2.3


From 818e3f818179bd737087cba6abd0484b17f68932 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Thu, 14 Mar 2019 13:50:33 +0000
Subject: [Minor] Fix set_reply signature

---
 src/libserver/milter.c | 6 +++---
 src/libserver/milter.h | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

(limited to 'src')

diff --git a/src/libserver/milter.c b/src/libserver/milter.c
index fcdbcae6c..372359258 100644
--- a/src/libserver/milter.c
+++ b/src/libserver/milter.c
@@ -1138,8 +1138,8 @@ rspamd_milter_handle_socket (gint fd, const struct timeval *tv,
 
 gboolean
 rspamd_milter_set_reply (struct rspamd_milter_session *session,
-		rspamd_fstring_t *xcode,
 		rspamd_fstring_t *rcode,
+		rspamd_fstring_t *xcode,
 		rspamd_fstring_t *reply)
 {
 	GString *buf;
@@ -1935,7 +1935,7 @@ rspamd_milter_send_task_results (struct rspamd_milter_session *session,
 				}
 			}
 
-			rspamd_milter_set_reply (session, xcode, rcode, reply);
+			rspamd_milter_set_reply (session, rcode, xcode, reply);
 		}
 		break;
 	case METRIC_ACTION_SOFT_REJECT:
@@ -1949,7 +1949,7 @@ rspamd_milter_send_task_results (struct rspamd_milter_session *session,
 					sizeof (RSPAMD_MILTER_TEMPFAIL_MESSAGE) - 1);
 		}
 
-		rspamd_milter_set_reply (session, xcode, rcode, reply);
+		rspamd_milter_set_reply (session, rcode, xcode, reply);
 		break;
 
 	case METRIC_ACTION_REWRITE_SUBJECT:
diff --git a/src/libserver/milter.h b/src/libserver/milter.h
index 7f375b018..10bf34d52 100644
--- a/src/libserver/milter.h
+++ b/src/libserver/milter.h
@@ -98,14 +98,14 @@ void * rspamd_milter_update_userdata (struct rspamd_milter_session *session,
 /**
  * Sets SMTP reply string
  * @param session
- * @param xcode
  * @param rcode
+ * @param xcode
  * @param reply
  * @return
  */
 gboolean rspamd_milter_set_reply (struct rspamd_milter_session *session,
-		rspamd_fstring_t *xcode,
 		rspamd_fstring_t *rcode,
+		rspamd_fstring_t *xcode,
 		rspamd_fstring_t *reply);
 
 /**
-- 
cgit v1.2.3


From dbfdd2bf45134a467fed1e625c7bcbe93f199e24 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Thu, 14 Mar 2019 13:59:46 +0000
Subject: [Minor] Sigh, another r<->x

---
 src/libserver/milter.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/libserver/milter.c b/src/libserver/milter.c
index 372359258..188ff42d9 100644
--- a/src/libserver/milter.c
+++ b/src/libserver/milter.c
@@ -1146,7 +1146,7 @@ rspamd_milter_set_reply (struct rspamd_milter_session *session,
 	gboolean ret;
 
 	buf = g_string_sized_new (xcode->len + rcode->len + reply->len + 2);
-	rspamd_printf_gstring (buf, "%V %V %V", xcode, rcode, reply);
+	rspamd_printf_gstring (buf, "%V %V %V", rcode, xcode, reply);
 	ret = rspamd_milter_send_action (session, RSPAMD_MILTER_REPLYCODE,
 		buf);
 	g_string_free (buf, TRUE);
-- 
cgit v1.2.3


From 3a1974cf17f235b7bbb98de714fe0210ddb2c596 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Mon, 18 Mar 2019 12:00:00 +0000
Subject: [Rework] Another iteration of HTTP interfaces rework

---
 src/client/rspamdclient.c      |   5 +-
 src/fuzzy_storage.c            |  10 +--
 src/libserver/rspamd_control.c |   5 +-
 src/libutil/http_connection.c  | 119 ++++++++++++++++++++---------
 src/libutil/http_connection.h  |  63 ++++++++++++++--
 src/libutil/http_router.c      |   5 +-
 src/libutil/map.c              | 168 +++++++++++++++++++----------------------
 src/libutil/map_private.h      |   1 -
 src/lua/lua_http.c             |  16 +---
 src/plugins/surbl.c            |  21 ++----
 src/rspamadm/control.c         |  14 +---
 src/rspamd_proxy.c             |  15 ++--
 src/worker.c                   |   5 +-
 13 files changed, 247 insertions(+), 200 deletions(-)

(limited to 'src')

diff --git a/src/client/rspamdclient.c b/src/client/rspamdclient.c
index a2ff85458..5f831ee64 100644
--- a/src/client/rspamdclient.c
+++ b/src/client/rspamdclient.c
@@ -255,13 +255,12 @@ rspamd_client_init (struct rspamd_http_context *http_ctx,
 	conn->ev_base = ev_base;
 	conn->fd = fd;
 	conn->req_sent = FALSE;
-	conn->http_conn = rspamd_http_connection_new (http_ctx,
-			fd,
+	conn->http_conn = rspamd_http_connection_new_client_socket (http_ctx,
 			rspamd_client_body_handler,
 			rspamd_client_error_handler,
 			rspamd_client_finish_handler,
 			0,
-			RSPAMD_HTTP_CLIENT);
+			fd);
 
 	conn->server_name = g_string_new (name);
 	if (port != 0) {
diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c
index 2054633ad..96fb09c2b 100644
--- a/src/fuzzy_storage.c
+++ b/src/fuzzy_storage.c
@@ -618,14 +618,13 @@ rspamd_fuzzy_send_update_mirror (struct rspamd_fuzzy_storage_ctx *ctx,
 	msg = rspamd_http_new_message (HTTP_REQUEST);
 	rspamd_printf_fstring (&msg->url, "/update_v1/%s", m->name);
 
-	conn->http_conn = rspamd_http_connection_new (
+	conn->http_conn = rspamd_http_connection_new_client_socket (
 			ctx->http_ctx,
-			conn->sock,
 			NULL,
 			fuzzy_mirror_error_handler,
 			fuzzy_mirror_finish_handler,
 			RSPAMD_HTTP_CLIENT_SIMPLE,
-			RSPAMD_HTTP_CLIENT);
+			conn->sock);
 
 	rspamd_http_connection_set_key (conn->http_conn,
 			ctx->sync_keypair);
@@ -1992,14 +1991,13 @@ accept_fuzzy_mirror_socket (gint fd, short what, void *arg)
 	session->name = rspamd_inet_address_to_string (addr);
 	rspamd_random_hex (session->uid, sizeof (session->uid) - 1);
 	session->uid[sizeof (session->uid) - 1] = '\0';
-	http_conn = rspamd_http_connection_new (
+	http_conn = rspamd_http_connection_new_server (
 			ctx->http_ctx,
 			nfd,
 			NULL,
 			rspamd_fuzzy_mirror_error_handler,
 			rspamd_fuzzy_mirror_finish_handler,
-			0,
-			RSPAMD_HTTP_SERVER);
+			0);
 
 	rspamd_http_connection_set_key (http_conn, ctx->sync_keypair);
 	session->ctx = ctx;
diff --git a/src/libserver/rspamd_control.c b/src/libserver/rspamd_control.c
index 149ad4245..12d37cdbc 100644
--- a/src/libserver/rspamd_control.c
+++ b/src/libserver/rspamd_control.c
@@ -518,13 +518,12 @@ rspamd_control_process_client_socket (struct rspamd_main *rspamd_main,
 	session = g_malloc0 (sizeof (*session));
 
 	session->fd = fd;
-	session->conn = rspamd_http_connection_new (rspamd_main->http_ctx,
+	session->conn = rspamd_http_connection_new_server (rspamd_main->http_ctx,
 			fd,
 			NULL,
 			rspamd_control_error_handler,
 			rspamd_control_finish_handler,
-			0,
-			RSPAMD_HTTP_SERVER);
+			0);
 	session->rspamd_main = rspamd_main;
 	session->addr = addr;
 	rspamd_http_connection_read_message (session->conn, session,
diff --git a/src/libutil/http_connection.c b/src/libutil/http_connection.c
index 42b934d2a..417784789 100644
--- a/src/libutil/http_connection.c
+++ b/src/libutil/http_connection.c
@@ -1077,14 +1077,12 @@ rspamd_http_parser_reset (struct rspamd_http_connection *conn)
 }
 
 struct rspamd_http_connection *
-rspamd_http_connection_new (
-		struct rspamd_http_context *ctx,
-		gint fd,
-		rspamd_http_body_handler_t body_handler,
-		rspamd_http_error_handler_t error_handler,
-		rspamd_http_finish_handler_t finish_handler,
-		unsigned opts,
-		enum rspamd_http_connection_type type)
+rspamd_http_connection_new_server (struct rspamd_http_context *ctx,
+								   gint fd,
+								   rspamd_http_body_handler_t body_handler,
+								   rspamd_http_error_handler_t error_handler,
+								   rspamd_http_finish_handler_t finish_handler,
+								   unsigned opts)
 {
 	struct rspamd_http_connection *conn;
 	struct rspamd_http_connection_private *priv;
@@ -1095,7 +1093,7 @@ rspamd_http_connection_new (
 
 	conn = g_malloc0 (sizeof (struct rspamd_http_connection));
 	conn->opts = opts;
-	conn->type = type;
+	conn->type = RSPAMD_HTTP_SERVER;
 	conn->body_handler = body_handler;
 	conn->error_handler = error_handler;
 	conn->finish_handler = finish_handler;
@@ -1112,14 +1110,49 @@ rspamd_http_connection_new (
 	conn->priv = priv;
 	priv->ctx = ctx;
 
-	if (conn->type == RSPAMD_HTTP_CLIENT) {
-		priv->cache = ctx->client_kp_cache;
-		if (ctx->client_kp) {
-			priv->local_key = rspamd_keypair_ref (ctx->client_kp);
-		}
+	priv->cache = ctx->server_kp_cache;
+
+	rspamd_http_parser_reset (conn);
+	priv->parser.data = conn;
+
+	return conn;
+}
+
+struct rspamd_http_connection *
+rspamd_http_connection_new_client_socket (struct rspamd_http_context *ctx,
+								   rspamd_http_body_handler_t body_handler,
+								   rspamd_http_error_handler_t error_handler,
+								   rspamd_http_finish_handler_t finish_handler,
+								   unsigned opts,
+								   gint fd)
+{
+	struct rspamd_http_connection *conn;
+	struct rspamd_http_connection_private *priv;
+
+	g_assert (error_handler != NULL && finish_handler == NULL);
+
+	conn = g_malloc0 (sizeof (struct rspamd_http_connection));
+	conn->opts = opts;
+	conn->type = RSPAMD_HTTP_CLIENT;
+	conn->body_handler = body_handler;
+	conn->error_handler = error_handler;
+	conn->finish_handler = finish_handler;
+	conn->fd = fd;
+	conn->ref = 1;
+	conn->finished = FALSE;
+
+	/* Init priv */
+	if (ctx == NULL) {
+		ctx = rspamd_http_context_default ();
 	}
-	else {
-		priv->cache = ctx->server_kp_cache;
+
+	priv = g_malloc0 (sizeof (struct rspamd_http_connection_private));
+	conn->priv = priv;
+	priv->ctx = ctx;
+
+	priv->cache = ctx->client_kp_cache;
+	if (ctx->client_kp) {
+		priv->local_key = rspamd_keypair_ref (ctx->client_kp);
 	}
 
 	rspamd_http_parser_reset (conn);
@@ -1128,16 +1161,42 @@ rspamd_http_connection_new (
 	return conn;
 }
 
+struct rspamd_http_connection *
+rspamd_http_connection_new_client (struct rspamd_http_context *ctx,
+								   rspamd_http_body_handler_t body_handler,
+								   rspamd_http_error_handler_t error_handler,
+								   rspamd_http_finish_handler_t finish_handler,
+								   unsigned opts,
+								   rspamd_inet_addr_t *addr)
+{
+	gint fd;
+
+	if (error_handler == NULL || finish_handler == NULL) {
+		return NULL;
+	}
+
+	fd = rspamd_inet_address_connect (addr, SOCK_STREAM, TRUE);
+
+	if (fd == -1) {
+		msg_info ("cannot connect to %s: %s", rspamd_inet_address_to_string (addr),
+				strerror (errno));
+		return NULL;
+	}
+
+	return rspamd_http_connection_new_client_socket (ctx,
+			body_handler, error_handler, finish_handler,
+			opts | RSPAMD_HTTP_OWN_SOCKET, fd);
+}
+
 struct rspamd_http_connection *
 rspamd_http_connection_new_keepalive (struct rspamd_http_context *ctx,
-		rspamd_http_body_handler_t body_handler,
-		rspamd_http_error_handler_t error_handler,
-		rspamd_http_finish_handler_t finish_handler,
-		rspamd_inet_addr_t *addr,
-		const gchar *host)
+									  rspamd_http_body_handler_t body_handler,
+									  rspamd_http_error_handler_t error_handler,
+									  rspamd_http_finish_handler_t finish_handler,
+									  rspamd_inet_addr_t *addr,
+									  const gchar *host)
 {
 	struct rspamd_http_connection *conn;
-	gint fd;
 
 	if (error_handler == NULL || finish_handler == NULL) {
 		return NULL;
@@ -1153,18 +1212,10 @@ rspamd_http_connection_new_keepalive (struct rspamd_http_context *ctx,
 		return conn;
 	}
 
-	fd = rspamd_inet_address_connect (addr, SOCK_STREAM, TRUE);
-
-	if (fd == -1) {
-		msg_info ("cannot connect to %s: %s", rspamd_inet_address_to_string (addr),
-				host);
-		return NULL;
-	}
-
-	conn = rspamd_http_connection_new (ctx, fd, body_handler, error_handler,
-			finish_handler,
+	conn = rspamd_http_connection_new_client (ctx,
+			body_handler, error_handler, finish_handler,
 			RSPAMD_HTTP_CLIENT_SIMPLE|RSPAMD_HTTP_CLIENT_KEEP_ALIVE,
-			RSPAMD_HTTP_CLIENT);
+			addr);
 
 	if (conn) {
 		rspamd_http_context_prepare_keepalive (ctx, conn, addr, host);
@@ -1387,7 +1438,7 @@ rspamd_http_connection_free (struct rspamd_http_connection *conn)
 		g_free (priv);
 	}
 
-	if (conn->opts & RSPAMD_HTTP_CLIENT_KEEP_ALIVE) {
+	if (conn->opts & RSPAMD_HTTP_OWN_SOCKET) {
 		/* Fd is owned by a connection */
 		close (conn->fd);
 	}
diff --git a/src/libutil/http_connection.h b/src/libutil/http_connection.h
index a327eec0d..da354ac41 100644
--- a/src/libutil/http_connection.h
+++ b/src/libutil/http_connection.h
@@ -85,6 +85,7 @@ enum rspamd_http_options {
 	RSPAMD_HTTP_CLIENT_SHARED = 1u << 3, /**< Store reply in shared memory */
 	RSPAMD_HTTP_REQUIRE_ENCRYPTION = 1u << 4,
 	RSPAMD_HTTP_CLIENT_KEEP_ALIVE = 1u << 5,
+	RSPAMD_HTTP_OWN_SOCKET = 1u << 6,
 };
 
 typedef int (*rspamd_http_body_handler_t) (struct rspamd_http_connection *conn,
@@ -118,20 +119,33 @@ struct rspamd_http_connection {
 };
 
 /**
- * Create new http connection
- * @param handler_t handler_t for body
- * @param opts options
- * @return new connection structure
+ * Creates a new HTTP server connection from an opened FD returned by accept function
+ * @param ctx
+ * @param fd
+ * @param body_handler
+ * @param error_handler
+ * @param finish_handler
+ * @param opts
+ * @return
  */
-struct rspamd_http_connection *rspamd_http_connection_new (
+struct rspamd_http_connection *rspamd_http_connection_new_server (
 		struct rspamd_http_context *ctx,
 		gint fd,
 		rspamd_http_body_handler_t body_handler,
 		rspamd_http_error_handler_t error_handler,
 		rspamd_http_finish_handler_t finish_handler,
-		unsigned opts,
-		enum rspamd_http_connection_type type);
+		unsigned opts);
 
+/**
+ * Creates or reuses a new keepalive client connection identified by hostname and inet_addr
+ * @param ctx
+ * @param body_handler
+ * @param error_handler
+ * @param finish_handler
+ * @param addr
+ * @param host
+ * @return
+ */
 struct rspamd_http_connection *rspamd_http_connection_new_keepalive (
 		struct rspamd_http_context *ctx,
 		rspamd_http_body_handler_t body_handler,
@@ -140,6 +154,41 @@ struct rspamd_http_connection *rspamd_http_connection_new_keepalive (
 		rspamd_inet_addr_t *addr,
 		const gchar *host);
 
+/**
+ * Creates an ordinary connection using the address specified (if proxy is not set)
+ * @param ctx
+ * @param body_handler
+ * @param error_handler
+ * @param finish_handler
+ * @param opts
+ * @param addr
+ * @return
+ */
+struct rspamd_http_connection *rspamd_http_connection_new_client (
+		struct rspamd_http_context *ctx,
+		rspamd_http_body_handler_t body_handler,
+		rspamd_http_error_handler_t error_handler,
+		rspamd_http_finish_handler_t finish_handler,
+		unsigned opts,
+		rspamd_inet_addr_t *addr);
+
+/**
+ * Creates an ordinary client connection using ready file descriptor (ignores proxy)
+ * @param ctx
+ * @param body_handler
+ * @param error_handler
+ * @param finish_handler
+ * @param opts
+ * @param addr
+ * @return
+ */
+struct rspamd_http_connection *rspamd_http_connection_new_client_socket (
+		struct rspamd_http_context *ctx,
+		rspamd_http_body_handler_t body_handler,
+		rspamd_http_error_handler_t error_handler,
+		rspamd_http_finish_handler_t finish_handler,
+		unsigned opts,
+		gint fd);
 
 /**
  * Set key pointed by an opaque pointer
diff --git a/src/libutil/http_router.c b/src/libutil/http_router.c
index 570d3d5c6..ec0eeb7b4 100644
--- a/src/libutil/http_router.c
+++ b/src/libutil/http_router.c
@@ -506,13 +506,12 @@ rspamd_http_router_handle_socket (struct rspamd_http_connection_router *router,
 	conn->ud = ud;
 	conn->is_reply = FALSE;
 
-	conn->conn = rspamd_http_connection_new (router->ctx,
+	conn->conn = rspamd_http_connection_new_server (router->ctx,
 			fd,
 			NULL,
 			rspamd_http_router_error_handler,
 			rspamd_http_router_finish_handler,
-			0,
-			RSPAMD_HTTP_SERVER);
+			0);
 
 	if (router->key) {
 		rspamd_http_connection_set_key (conn->conn, router->key);
diff --git a/src/libutil/map.c b/src/libutil/map.c
index 0f7ff4f48..fc414ab00 100644
--- a/src/libutil/map.c
+++ b/src/libutil/map.c
@@ -75,76 +75,62 @@ write_http_request (struct http_callback_data *cbd)
 	struct rspamd_map *map;
 
 	map = cbd->map;
+	msg = rspamd_http_new_message (HTTP_REQUEST);
 
-	if (cbd->fd != -1) {
-		close (cbd->fd);
+	if (cbd->bk->protocol == MAP_PROTO_HTTPS) {
+		msg->flags |= RSPAMD_HTTP_FLAG_SSL;
 	}
 
-	cbd->fd = rspamd_inet_address_connect (cbd->addr, SOCK_STREAM, TRUE);
-
-	if (cbd->fd != -1) {
-		msg = rspamd_http_new_message (HTTP_REQUEST);
-
-		if (cbd->bk->protocol == MAP_PROTO_HTTPS) {
-			msg->flags |= RSPAMD_HTTP_FLAG_SSL;
-		}
+	if (cbd->check) {
+		msg->method = HTTP_HEAD;
+	}
 
-		if (cbd->check) {
-			msg->method = HTTP_HEAD;
-		}
+	if (cbd->stage == map_load_file) {
+		msg->url = rspamd_fstring_append (msg->url,
+				cbd->data->path, strlen (cbd->data->path));
 
-		if (cbd->stage == map_load_file) {
-			msg->url = rspamd_fstring_append (msg->url,
-					cbd->data->path, strlen (cbd->data->path));
-
-			if (cbd->check && cbd->stage == map_load_file) {
-				if (cbd->data->last_modified != 0) {
-					rspamd_http_date_format (datebuf, sizeof (datebuf),
-							cbd->data->last_modified);
-					rspamd_http_message_add_header (msg, "If-Modified-Since",
-							datebuf);
-				}
-				if (cbd->data->etag) {
-					rspamd_http_message_add_header_len (msg, "If-None-Match",
-							cbd->data->etag->str, cbd->data->etag->len);
-				}
+		if (cbd->check && cbd->stage == map_load_file) {
+			if (cbd->data->last_modified != 0) {
+				rspamd_http_date_format (datebuf, sizeof (datebuf),
+						cbd->data->last_modified);
+				rspamd_http_message_add_header (msg, "If-Modified-Since",
+						datebuf);
+			}
+			if (cbd->data->etag) {
+				rspamd_http_message_add_header_len (msg, "If-None-Match",
+						cbd->data->etag->str, cbd->data->etag->len);
 			}
 		}
-		else if (cbd->stage == map_load_pubkey) {
-			msg->url = rspamd_fstring_append (msg->url,
-					cbd->data->path, strlen (cbd->data->path));
-			msg->url = rspamd_fstring_append (msg->url, ".pub", 4);
-		}
-		else if (cbd->stage == map_load_signature) {
-			msg->url = rspamd_fstring_append (msg->url,
-					cbd->data->path, strlen (cbd->data->path));
-			msg->url = rspamd_fstring_append (msg->url, ".sig", 4);
-		}
-		else {
-			g_assert_not_reached ();
-		}
-
-		msg->url = rspamd_fstring_append (msg->url, cbd->data->rest,
-					strlen (cbd->data->rest));
-
-		if (cbd->data->userinfo) {
-			rspamd_http_message_add_header (msg, "Authorization",
-					cbd->data->userinfo);
-		}
-
-		MAP_RETAIN (cbd, "http_callback_data");
-		rspamd_http_connection_write_message (cbd->conn,
-				msg,
-				cbd->data->host,
-				NULL,
-				cbd,
-				&cbd->tv);
+	}
+	else if (cbd->stage == map_load_pubkey) {
+		msg->url = rspamd_fstring_append (msg->url,
+				cbd->data->path, strlen (cbd->data->path));
+		msg->url = rspamd_fstring_append (msg->url, ".pub", 4);
+	}
+	else if (cbd->stage == map_load_signature) {
+		msg->url = rspamd_fstring_append (msg->url,
+				cbd->data->path, strlen (cbd->data->path));
+		msg->url = rspamd_fstring_append (msg->url, ".sig", 4);
 	}
 	else {
-		msg_err_map ("cannot connect to %s: %s", cbd->data->host,
-				strerror (errno));
-		cbd->periodic->errored = TRUE;
+		g_assert_not_reached ();
 	}
+
+	msg->url = rspamd_fstring_append (msg->url, cbd->data->rest,
+			strlen (cbd->data->rest));
+
+	if (cbd->data->userinfo) {
+		rspamd_http_message_add_header (msg, "Authorization",
+				cbd->data->userinfo);
+	}
+
+	MAP_RETAIN (cbd, "http_callback_data");
+	rspamd_http_connection_write_message (cbd->conn,
+			msg,
+			cbd->data->host,
+			NULL,
+			cbd,
+			&cbd->tv);
 }
 
 static gboolean
@@ -282,10 +268,6 @@ free_http_cbdata_common (struct http_callback_data *cbd, gboolean plan_new)
 		cbd->conn = NULL;
 	}
 
-	if (cbd->fd != -1) {
-		close (cbd->fd);
-	}
-
 	if (cbd->addr) {
 		rspamd_inet_address_free (cbd->addr);
 	}
@@ -517,7 +499,13 @@ http_map_finish (struct rspamd_http_connection *conn,
 					}
 				}
 
-				rspamd_http_connection_reset (cbd->conn);
+				rspamd_http_connection_unref (cbd->conn);
+				cbd->conn = rspamd_http_connection_new_client (NULL,
+						NULL,
+						http_map_error,
+						http_map_finish,
+						RSPAMD_HTTP_CLIENT_SIMPLE|RSPAMD_HTTP_CLIENT_SHARED,
+						cbd->addr);
 				write_http_request (cbd);
 				MAP_RELEASE (cbd, "http_callback_data");
 
@@ -563,7 +551,13 @@ http_map_finish (struct rspamd_http_connection *conn,
 			}
 
 			cbd->stage = map_load_signature;
-			rspamd_http_connection_reset (cbd->conn);
+			rspamd_http_connection_unref (cbd->conn);
+			cbd->conn = rspamd_http_connection_new_client (NULL,
+					NULL,
+					http_map_error,
+					http_map_finish,
+					RSPAMD_HTTP_CLIENT_SIMPLE|RSPAMD_HTTP_CLIENT_SHARED,
+					cbd->addr);
 			write_http_request (cbd);
 			MAP_RELEASE (cbd, "http_callback_data");
 
@@ -1264,20 +1258,15 @@ rspamd_map_dns_callback (struct rdns_reply *reply, void *arg)
 
 			if (cbd->addr != NULL) {
 				rspamd_inet_address_set_port (cbd->addr, cbd->data->port);
-				/* Try to open a socket */
-				cbd->fd = rspamd_inet_address_connect (cbd->addr, SOCK_STREAM,
-						TRUE);
-
-				if (cbd->fd != -1) {
+				cbd->conn = rspamd_http_connection_new_client (NULL,
+						NULL,
+						http_map_error,
+						http_map_finish,
+						flags,
+						cbd->addr);
+
+				if (cbd->conn != NULL) {
 					cbd->stage = map_load_file;
-					cbd->conn = rspamd_http_connection_new (NULL,
-							cbd->fd,
-							NULL,
-							http_map_error,
-							http_map_finish,
-							flags,
-							RSPAMD_HTTP_CLIENT);
-
 					write_http_request (cbd);
 				}
 				else {
@@ -1623,7 +1612,6 @@ check:
 	cbd->ev_base = map->ev_base;
 	cbd->map = map;
 	cbd->data = data;
-	cbd->fd = -1;
 	cbd->check = check;
 	cbd->periodic = periodic;
 	MAP_RETAIN (periodic, "periodic");
@@ -1638,20 +1626,16 @@ check:
 	/* Send both A and AAAA requests */
 	if (rspamd_parse_inet_address (&cbd->addr, data->host, strlen (data->host))) {
 		rspamd_inet_address_set_port (cbd->addr, cbd->data->port);
-		cbd->fd = rspamd_inet_address_connect (cbd->addr, SOCK_STREAM,
-				TRUE);
+		cbd->conn = rspamd_http_connection_new_client (
+				NULL,
+				NULL,
+				http_map_error,
+				http_map_finish,
+				flags,
+				cbd->addr);
 
-		if (cbd->fd != -1) {
+		if (cbd->conn != NULL) {
 			cbd->stage = map_load_file;
-			cbd->conn = rspamd_http_connection_new (
-					NULL,
-					cbd->fd,
-					NULL,
-					http_map_error,
-					http_map_finish,
-					flags,
-					RSPAMD_HTTP_CLIENT);
-
 			write_http_request (cbd);
 			MAP_RELEASE (cbd, "http_callback_data");
 		}
diff --git a/src/libutil/map_private.h b/src/libutil/map_private.h
index 68415d0e0..b32f0e390 100644
--- a/src/libutil/map_private.h
+++ b/src/libutil/map_private.h
@@ -200,7 +200,6 @@ struct http_callback_data {
 	gsize pubkey_len;
 
 	enum rspamd_map_http_stage stage;
-	gint fd;
 	struct timeval tv;
 
 	ref_entry_t ref;
diff --git a/src/lua/lua_http.c b/src/lua/lua_http.c
index e518e6dab..19cf8f2c9 100644
--- a/src/lua/lua_http.c
+++ b/src/lua/lua_http.c
@@ -368,8 +368,6 @@ lua_http_resume_handler (struct rspamd_http_connection *conn,
 static gboolean
 lua_http_make_connection (struct lua_http_cbdata *cbd)
 {
-	int fd;
-
 	rspamd_inet_address_set_port (cbd->addr, cbd->msg->port);
 
 	if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_KEEP_ALIVE) {
@@ -384,22 +382,14 @@ lua_http_make_connection (struct lua_http_cbdata *cbd)
 				cbd->host);
 	}
 	else {
-		fd = rspamd_inet_address_connect (cbd->addr, SOCK_STREAM, TRUE);
-
-		if (fd == -1) {
-			msg_info ("cannot connect to %V", cbd->msg->host);
-			return FALSE;
-		}
-
-		cbd->fd = fd;
-		cbd->conn = rspamd_http_connection_new (
+		cbd->fd = -1;
+		cbd->conn = rspamd_http_connection_new_client (
 				NULL, /* Default context */
-				fd,
 				NULL,
 				lua_http_error_handler,
 				lua_http_finish_handler,
 				RSPAMD_HTTP_CLIENT_SIMPLE,
-				RSPAMD_HTTP_CLIENT);
+				cbd->addr);
 	}
 
 	if (cbd->conn) {
diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c
index d2632cb81..63a7dd544 100644
--- a/src/plugins/surbl.c
+++ b/src/plugins/surbl.c
@@ -125,7 +125,6 @@ struct redirector_param {
 	GHashTable *tree;
 	struct suffix_item *suffix;
 	struct rspamd_symcache_item *item;
-	gint sock;
 	guint redirector_requests;
 };
 
@@ -1689,7 +1688,6 @@ free_redirector_session (void *ud)
 	}
 
 	rspamd_http_connection_unref (param->conn);
-	close (param->sock);
 }
 
 static void
@@ -1780,7 +1778,6 @@ static void
 register_redirector_call (struct rspamd_url *url, struct rspamd_task *task,
 	const gchar *rule)
 {
-	gint s = -1;
 	struct redirector_param *param;
 	struct timeval *timeout;
 	struct upstream *selected;
@@ -1793,11 +1790,15 @@ register_redirector_call (struct rspamd_url *url, struct rspamd_task *task,
 				RSPAMD_UPSTREAM_ROUND_ROBIN, url->host, url->hostlen);
 
 		if (selected) {
-			s = rspamd_inet_address_connect (rspamd_upstream_addr_next (selected),
-					SOCK_STREAM, TRUE);
+			param->conn = rspamd_http_connection_new_client (NULL,
+					NULL,
+					surbl_redirector_error,
+					surbl_redirector_finish,
+					RSPAMD_HTTP_CLIENT_SIMPLE,
+					rspamd_upstream_addr_next (selected));
 		}
 
-		if (s == -1) {
+		if (param->conn == NULL) {
 			msg_info_surbl ("<%s> cannot create tcp socket failed: %s",
 					task->message_id,
 					strerror (errno));
@@ -1810,17 +1811,9 @@ register_redirector_call (struct rspamd_url *url, struct rspamd_task *task,
 						sizeof (struct redirector_param));
 		param->url = url;
 		param->task = task;
-		param->conn = rspamd_http_connection_new (NULL,
-				s,
-				NULL,
-				surbl_redirector_error,
-				surbl_redirector_finish,
-				RSPAMD_HTTP_CLIENT_SIMPLE,
-				RSPAMD_HTTP_CLIENT);
 		param->ctx = surbl_module_ctx;
 		msg = rspamd_http_new_message (HTTP_REQUEST);
 		msg->url = rspamd_fstring_assign (msg->url, url->string, url->urllen);
-		param->sock = s;
 		param->redirector = selected;
 		timeout = rspamd_mempool_alloc (task->task_pool, sizeof (struct timeval));
 		double_to_tv (surbl_module_ctx->read_timeout, timeout);
diff --git a/src/rspamadm/control.c b/src/rspamadm/control.c
index 0a242e943..1311622c3 100644
--- a/src/rspamadm/control.c
+++ b/src/rspamadm/control.c
@@ -175,7 +175,6 @@ rspamadm_control (gint argc, gchar **argv, const struct rspamadm_command *_cmd)
 	rspamd_inet_addr_t *addr;
 	struct timeval tv;
 	static struct rspamadm_control_cbdata cbdata;
-	gint sock;
 
 	context = g_option_context_new (
 			"control - manage rspamd main control interface");
@@ -230,22 +229,14 @@ rspamadm_control (gint argc, gchar **argv, const struct rspamadm_command *_cmd)
 		exit (1);
 	}
 
-	sock = rspamd_inet_address_connect (addr, SOCK_STREAM, TRUE);
 
-	if (sock == -1) {
-		rspamd_fprintf (stderr, "cannot connect to: %s\n", control_path);
-		rspamd_inet_address_free (addr);
-		exit (1);
-	}
-
-	conn = rspamd_http_connection_new (
+	conn = rspamd_http_connection_new_client (
 			rspamd_main->http_ctx, /* Default context */
-			sock,
 			NULL,
 			rspamd_control_error_handler,
 			rspamd_control_finish_handler,
 			RSPAMD_HTTP_CLIENT_SIMPLE,
-			RSPAMD_HTTP_CLIENT);
+			addr);
 	msg = rspamd_http_new_message (HTTP_REQUEST);
 	msg->url = rspamd_fstring_new_init (path, strlen (path));
 	double_to_tv (timeout, &tv);
@@ -261,5 +252,4 @@ rspamadm_control (gint argc, gchar **argv, const struct rspamadm_command *_cmd)
 
 	rspamd_http_connection_unref (conn);
 	rspamd_inet_address_free (addr);
-	close (sock);
 }
diff --git a/src/rspamd_proxy.c b/src/rspamd_proxy.c
index b6ede29b9..e83426673 100644
--- a/src/rspamd_proxy.c
+++ b/src/rspamd_proxy.c
@@ -1404,14 +1404,13 @@ proxy_open_mirror_connections (struct rspamd_proxy_session *session)
 			rspamd_http_message_add_header (msg, "Settings-ID", m->settings_id);
 		}
 
-		bk_conn->backend_conn = rspamd_http_connection_new (
+		bk_conn->backend_conn = rspamd_http_connection_new_client_socket (
 				session->ctx->http_ctx,
-				bk_conn->backend_sock,
 				NULL,
 				proxy_backend_mirror_error_handler,
 				proxy_backend_mirror_finish_handler,
 				RSPAMD_HTTP_CLIENT_SIMPLE,
-				RSPAMD_HTTP_CLIENT);
+				bk_conn->backend_sock);
 
 		if (m->key) {
 			msg->peer_key = rspamd_pubkey_ref (m->key);
@@ -1831,14 +1830,13 @@ retry:
 			goto err; /* No fallback here */
 		}
 
-		session->master_conn->backend_conn = rspamd_http_connection_new (
+		session->master_conn->backend_conn = rspamd_http_connection_new_client_socket (
 				session->ctx->http_ctx,
-				session->master_conn->backend_sock,
 				NULL,
 				proxy_backend_master_error_handler,
 				proxy_backend_master_finish_handler,
 				RSPAMD_HTTP_CLIENT_SIMPLE,
-				RSPAMD_HTTP_CLIENT);
+				session->master_conn->backend_sock);
 		session->master_conn->flags &= ~RSPAMD_BACKEND_CLOSED;
 		session->master_conn->parser_from_ref = backend->parser_from_ref;
 		session->master_conn->parser_to_ref = backend->parser_to_ref;
@@ -2082,14 +2080,13 @@ proxy_accept_socket (gint fd, short what, void *arg)
 	}
 
 	if (!ctx->milter) {
-		session->client_conn = rspamd_http_connection_new (
+		session->client_conn = rspamd_http_connection_new_server (
 				ctx->http_ctx,
 				nfd,
 				NULL,
 				proxy_client_error_handler,
 				proxy_client_finish_handler,
-				0,
-				RSPAMD_HTTP_SERVER);
+				0);
 
 		if (ctx->key) {
 			rspamd_http_connection_set_key (session->client_conn, ctx->key);
diff --git a/src/worker.c b/src/worker.c
index d81be54a1..77614ec13 100644
--- a/src/worker.c
+++ b/src/worker.c
@@ -412,14 +412,13 @@ accept_socket (gint fd, short what, void *arg)
 		http_opts = RSPAMD_HTTP_REQUIRE_ENCRYPTION;
 	}
 
-	task->http_conn = rspamd_http_connection_new (
+	task->http_conn = rspamd_http_connection_new_server (
 			ctx->http_ctx,
 			nfd,
 			rspamd_worker_body_handler,
 			rspamd_worker_error_handler,
 			rspamd_worker_finish_handler,
-			http_opts,
-			RSPAMD_HTTP_SERVER);
+			http_opts);
 	rspamd_http_connection_set_max_size (task->http_conn, task->cfg->max_message);
 	worker->nconns++;
 	rspamd_mempool_add_destructor (task->task_pool,
-- 
cgit v1.2.3


From 34cc551018df1c174685493a64b496d594cc8eb6 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Mon, 18 Mar 2019 14:07:32 +0000
Subject: [Project] Preliminary support of HTTP proxies

Issue: #572
---
 src/client/rspamc.c           |  2 +-
 src/controller.c              |  3 +-
 src/fuzzy_storage.c           |  2 +-
 src/libutil/http_connection.c |  2 +
 src/libutil/http_context.c    | 94 ++++++++++++++++++++++++++++++++++++++++---
 src/libutil/http_context.h    |  8 +++-
 src/libutil/http_private.h    |  4 ++
 src/libutil/upstream.c        | 52 +++++++++++++-----------
 src/libutil/upstream.h        |  4 ++
 src/plugins/surbl.c           |  2 +-
 src/rspamadm/rspamadm.c       |  3 +-
 src/rspamd.c                  |  2 +-
 src/rspamd_proxy.c            |  3 +-
 src/worker.c                  |  3 +-
 14 files changed, 146 insertions(+), 38 deletions(-)

(limited to 'src')

diff --git a/src/client/rspamc.c b/src/client/rspamc.c
index 08a267f1b..2f572c449 100644
--- a/src/client/rspamc.c
+++ b/src/client/rspamc.c
@@ -1911,7 +1911,7 @@ main (gint argc, gchar **argv, gchar **env)
 	http_config.kp_cache_size_server = 0;
 	http_config.user_agent = user_agent;
 	http_ctx = rspamd_http_context_create_config (&http_config,
-			ev_base);
+			ev_base, NULL);
 
 	/* Ignore sigpipe */
 	sigemptyset (&sigpipe_act.sa_mask);
diff --git a/src/controller.c b/src/controller.c
index ac1acae81..7b48462ee 100644
--- a/src/controller.c
+++ b/src/controller.c
@@ -3776,7 +3776,8 @@ start_controller_worker (struct rspamd_worker *worker)
 			"password");
 
 	/* Accept event */
-	ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base);
+	ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base,
+			ctx->cfg->ups_ctx);
 	ctx->http = rspamd_http_router_new (rspamd_controller_error_handler,
 			rspamd_controller_finish_handler, &ctx->io_tv,
 			ctx->static_files_dir, ctx->http_ctx);
diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c
index 96fb09c2b..e82e9062a 100644
--- a/src/fuzzy_storage.c
+++ b/src/fuzzy_storage.c
@@ -2999,7 +2999,7 @@ start_fuzzy (struct rspamd_worker *worker)
 		ctx->keypair_cache = rspamd_keypair_cache_new (ctx->keypair_cache_size);
 	}
 
-	ctx->http_ctx = rspamd_http_context_create (cfg, ctx->ev_base);
+	ctx->http_ctx = rspamd_http_context_create (cfg, ctx->ev_base, ctx->cfg->ups_ctx);
 
 	if (!ctx->collection_mode) {
 		/*
diff --git a/src/libutil/http_connection.c b/src/libutil/http_connection.c
index 417784789..d782c2d13 100644
--- a/src/libutil/http_connection.c
+++ b/src/libutil/http_connection.c
@@ -49,6 +49,8 @@ enum rspamd_http_priv_flags {
 	RSPAMD_HTTP_CONN_FLAG_RESETED = 1 << 2,
 	RSPAMD_HTTP_CONN_FLAG_TOO_LARGE = 1 << 3,
 	RSPAMD_HTTP_CONN_FLAG_ENCRYPTION_NEEDED = 1 << 4,
+	RSPAMD_HTTP_CONN_FLAG_PROXY = 1 << 5,
+	RSPAMD_HTTP_CONN_FLAG_PROXY_REQUEST = 1 << 6,
 };
 
 #define IS_CONN_ENCRYPTED(c) ((c)->flags & RSPAMD_HTTP_CONN_FLAG_ENCRYPTED)
diff --git a/src/libutil/http_context.c b/src/libutil/http_context.c
index 9182285a3..18c89c6bc 100644
--- a/src/libutil/http_context.c
+++ b/src/libutil/http_context.c
@@ -14,12 +14,14 @@
  * limitations under the License.
  */
 
+#include <contrib/http-parser/http_parser.h>
 #include "http_context.h"
 #include "http_private.h"
 #include "keypair.h"
 #include "keypairs_cache.h"
 #include "cfg_file.h"
 #include "contrib/libottery/ottery.h"
+#include "contrib/http-parser/http_parser.h"
 #include "rspamd.h"
 
 INIT_LOG_MODULE(http_context)
@@ -85,7 +87,8 @@ rspamd_http_context_client_rotate_ev (gint fd, short what, void *arg)
 
 static struct rspamd_http_context*
 rspamd_http_context_new_default (struct rspamd_config *cfg,
-								 struct event_base *ev_base)
+								 struct event_base *ev_base,
+								 struct upstream_ctx *ups_ctx)
 {
 	struct rspamd_http_context *ctx;
 
@@ -100,6 +103,7 @@ rspamd_http_context_new_default (struct rspamd_config *cfg,
 	ctx->config.client_key_rotate_time = default_rotate_time;
 	ctx->config.user_agent = default_user_agent;
 	ctx->config.keepalive_interval = default_keepalive_interval;
+	ctx->ups_ctx = ups_ctx;
 
 	if (cfg) {
 		ctx->ssl_ctx = cfg->libs_ctx->ssl_ctx;
@@ -117,9 +121,63 @@ rspamd_http_context_new_default (struct rspamd_config *cfg,
 	return ctx;
 }
 
+static void
+rspamd_http_context_parse_proxy (struct rspamd_http_context *ctx,
+								 const gchar *name,
+								 struct upstream_list **pls)
+{
+	struct http_parser_url u;
+	struct upstream_list *uls;
+
+	if (!ctx->ups_ctx) {
+		msg_err ("cannot parse http_proxy %s - upstreams context is udefined", name);
+		return;
+	}
+
+	memset (&u, 0, sizeof (u));
+
+	if (http_parser_parse_url (name, strlen (name), 1, &u) == 0) {
+		if (!(u.field_set & (1u << UF_HOST)) || u.port == 0) {
+			msg_err ("cannot parse http(s) proxy %s - invalid host or port", name);
+
+			return;
+		}
+
+		uls = rspamd_upstreams_create (ctx->ups_ctx);
+
+		if (!rspamd_upstreams_parse_line_len (uls,
+				name + u.field_data[UF_HOST].off,
+				u.field_data[UF_HOST].len, u.port, NULL)) {
+			msg_err ("cannot parse http(s) proxy %s - invalid data", name);
+
+			rspamd_upstreams_destroy (uls);
+		}
+		else {
+			*pls = uls;
+			msg_info ("set http(s) proxy to %s", name);
+		}
+	}
+	else {
+		uls = rspamd_upstreams_create (ctx->ups_ctx);
+
+		if (!rspamd_upstreams_parse_line (uls,
+				name, 3128, NULL)) {
+			msg_err ("cannot parse http(s) proxy %s - invalid data", name);
+
+			rspamd_upstreams_destroy (uls);
+		}
+		else {
+			*pls = uls;
+			msg_info ("set http(s) proxy to %s", name);
+		}
+	}
+}
+
 static void
 rspamd_http_context_init (struct rspamd_http_context *ctx)
 {
+
+
 	if (ctx->config.kp_cache_size_client > 0) {
 		ctx->client_kp_cache = rspamd_keypair_cache_new (ctx->config.kp_cache_size_client);
 	}
@@ -140,17 +198,28 @@ rspamd_http_context_init (struct rspamd_http_context *ctx)
 		event_add (&ctx->client_rotate_ev, &tv);
 	}
 
+	if (ctx->config.http_proxy) {
+		rspamd_http_context_parse_proxy (ctx, ctx->config.http_proxy,
+				&ctx->http_proxies);
+	}
+
+	if (ctx->config.https_proxy) {
+		rspamd_http_context_parse_proxy (ctx, ctx->config.https_proxy,
+				&ctx->https_proxies);
+	}
+
 	default_ctx = ctx;
 }
 
 struct rspamd_http_context*
 rspamd_http_context_create (struct rspamd_config *cfg,
-							struct event_base *ev_base)
+							struct event_base *ev_base,
+							struct upstream_ctx *ups_ctx)
 {
 	struct rspamd_http_context *ctx;
 	const ucl_object_t *http_obj;
 
-	ctx = rspamd_http_context_new_default (cfg, ev_base);
+	ctx = rspamd_http_context_new_default (cfg, ev_base, ups_ctx);
 	http_obj = ucl_object_lookup (cfg->rcl_obj, "http");
 
 	if (http_obj) {
@@ -194,6 +263,20 @@ rspamd_http_context_create (struct rspamd_config *cfg,
 			if (keepalive_interval) {
 				ctx->config.keepalive_interval = ucl_object_todouble (keepalive_interval);
 			}
+
+			const ucl_object_t *http_proxy;
+			http_proxy = ucl_object_lookup (client_obj, "http_proxy");
+
+			if (http_proxy) {
+				ctx->config.http_proxy = ucl_object_tostring (http_proxy);
+			}
+
+			const ucl_object_t *https_proxy;
+			https_proxy = ucl_object_lookup (client_obj, "https_proxy");
+
+			if (https_proxy) {
+				ctx->config.https_proxy = ucl_object_tostring (https_proxy);
+			}
 		}
 
 		server_obj = ucl_object_lookup (http_obj, "server");
@@ -262,11 +345,12 @@ rspamd_http_context_free (struct rspamd_http_context *ctx)
 
 struct rspamd_http_context*
 rspamd_http_context_create_config (struct rspamd_http_context_cfg *cfg,
-		struct event_base *ev_base)
+								   struct event_base *ev_base,
+								   struct upstream_ctx *ups_ctx)
 {
 	struct rspamd_http_context *ctx;
 
-	ctx = rspamd_http_context_new_default (NULL, ev_base);
+	ctx = rspamd_http_context_new_default (NULL, ev_base, ups_ctx);
 	memcpy (&ctx->config, cfg, sizeof (*cfg));
 	rspamd_http_context_init (ctx);
 
diff --git a/src/libutil/http_context.h b/src/libutil/http_context.h
index 74e5c69a6..6abd66651 100644
--- a/src/libutil/http_context.h
+++ b/src/libutil/http_context.h
@@ -26,6 +26,7 @@
 struct rspamd_http_context;
 struct rspamd_config;
 struct rspamd_http_message;
+struct upstream_ctx;
 
 struct rspamd_http_context_cfg {
 	guint kp_cache_size_client;
@@ -34,6 +35,8 @@ struct rspamd_http_context_cfg {
 	gdouble keepalive_interval;
 	gdouble client_key_rotate_time;
 	const gchar *user_agent;
+	const gchar *http_proxy;
+	const gchar *https_proxy;
 };
 
 /**
@@ -43,11 +46,12 @@ struct rspamd_http_context_cfg {
  * @return new context used for both client and server HTTP connections
  */
 struct rspamd_http_context* rspamd_http_context_create (struct rspamd_config *cfg,
-		struct event_base *ev_base);
+		struct event_base *ev_base, struct upstream_ctx *ctx);
 
 struct rspamd_http_context* rspamd_http_context_create_config (
 		struct rspamd_http_context_cfg *cfg,
-		struct event_base *ev_base);
+		struct event_base *ev_base,
+		struct upstream_ctx *ctx);
 /**
  * Destroys context
  * @param ctx
diff --git a/src/libutil/http_private.h b/src/libutil/http_private.h
index dd3d0c6a9..dd4ca3435 100644
--- a/src/libutil/http_private.h
+++ b/src/libutil/http_private.h
@@ -22,6 +22,7 @@
 #include "keypair.h"
 #include "keypairs_cache.h"
 #include "ref.h"
+#include "upstream.h"
 #include "khash.h"
 #define HASH_CASELESS
 #include "uthash_strcase.h"
@@ -95,6 +96,9 @@ struct rspamd_http_context {
 	struct rspamd_keypair_cache *client_kp_cache;
 	struct rspamd_cryptobox_keypair *client_kp;
 	struct rspamd_keypair_cache *server_kp_cache;
+	struct upstream_ctx *ups_ctx;
+	struct upstream_list *http_proxies;
+	struct upstream_list *https_proxies;
 	gpointer ssl_ctx;
 	gpointer ssl_ctx_noverify;
 	struct event_base *ev_base;
diff --git a/src/libutil/upstream.c b/src/libutil/upstream.c
index 64d5291fa..3a2b803b4 100644
--- a/src/libutil/upstream.c
+++ b/src/libutil/upstream.c
@@ -788,51 +788,45 @@ rspamd_upstream_add_addr (struct upstream *up, rspamd_inet_addr_t *addr)
 	return TRUE;
 }
 
+#define LEN_CHECK_STARTS_WITH(s, len, lit) \
+	((len) >= sizeof(lit) - 1 && g_ascii_strncasecmp ((s), (lit), sizeof(lit) - 1) == 0)
 gboolean
-rspamd_upstreams_parse_line (struct upstream_list *ups,
-		const gchar *str, guint16 def_port, void *data)
+rspamd_upstreams_parse_line_len (struct upstream_list *ups,
+		const gchar *str, gsize len, guint16 def_port, void *data)
 {
-	const gchar *end = str + strlen (str), *p = str;
+	const gchar *end = str + len, *p = str;
 	const gchar *separators = ";, \n\r\t";
 	gchar *tmp;
-	guint len;
+	guint span_len;
 	gboolean ret = FALSE;
 
-	if (g_ascii_strncasecmp (p, "random:", sizeof ("random:") - 1) == 0) {
+	if (LEN_CHECK_STARTS_WITH(p, len, "random:")) {
 		ups->rot_alg = RSPAMD_UPSTREAM_RANDOM;
 		p += sizeof ("random:") - 1;
 	}
-	else if (g_ascii_strncasecmp (p,
-			"master-slave:",
-			sizeof ("master-slave:") - 1) == 0) {
+	else if (LEN_CHECK_STARTS_WITH(p, len, "master-slave:")) {
 		ups->rot_alg = RSPAMD_UPSTREAM_MASTER_SLAVE;
 		p += sizeof ("master-slave:") - 1;
 	}
-	else if (g_ascii_strncasecmp (p,
-			"round-robin:",
-			sizeof ("round-robin:") - 1) == 0) {
+	else if (LEN_CHECK_STARTS_WITH(p, len, "round-robin:")) {
 		ups->rot_alg = RSPAMD_UPSTREAM_ROUND_ROBIN;
 		p += sizeof ("round-robin:") - 1;
 	}
-	else if (g_ascii_strncasecmp (p,
-			"hash:",
-			sizeof ("hash:") - 1) == 0) {
+	else if (LEN_CHECK_STARTS_WITH(p, len, "hash:")) {
 		ups->rot_alg = RSPAMD_UPSTREAM_HASHED;
 		p += sizeof ("hash:") - 1;
 	}
-	else if (g_ascii_strncasecmp (p,
-			"sequential:",
-			sizeof ("sequential:") - 1) == 0) {
+	else if (LEN_CHECK_STARTS_WITH(p, len, "sequential:")) {
 		ups->rot_alg = RSPAMD_UPSTREAM_SEQUENTIAL;
 		p += sizeof ("sequential:") - 1;
 	}
 
 	while (p < end) {
-		len = strcspn (p, separators);
+		span_len = rspamd_memcspn (p, separators, end - p);
 
-		if (len > 0) {
-			tmp = g_malloc (len + 1);
-			rspamd_strlcpy (tmp, p, len + 1);
+		if (span_len > 0) {
+			tmp = g_malloc (span_len + 1);
+			rspamd_strlcpy (tmp, p, span_len + 1);
 
 			if (rspamd_upstreams_add_upstream (ups, tmp, def_port,
 					RSPAMD_UPSTREAM_PARSE_DEFAULT,
@@ -843,14 +837,26 @@ rspamd_upstreams_parse_line (struct upstream_list *ups,
 			g_free (tmp);
 		}
 
-		p += len;
+		p += span_len;
 		/* Skip separators */
-		p += strspn (p, separators);
+		if (p < end) {
+			p += rspamd_memspn (p, separators, end - p);
+		}
 	}
 
 	return ret;
 }
 
+#undef LEN_CHECK_STARTS_WITH
+
+gboolean
+rspamd_upstreams_parse_line (struct upstream_list *ups,
+							 const gchar *str, guint16 def_port, void *data)
+{
+	return rspamd_upstreams_parse_line_len (ups, str, strlen (str),
+			def_port, data);
+}
+
 gboolean
 rspamd_upstreams_from_ucl (struct upstream_list *ups,
 		const ucl_object_t *in, guint16 def_port, void *data)
diff --git a/src/libutil/upstream.h b/src/libutil/upstream.h
index 4db962765..75d840ce2 100644
--- a/src/libutil/upstream.h
+++ b/src/libutil/upstream.h
@@ -157,6 +157,10 @@ gboolean rspamd_upstreams_parse_line (struct upstream_list *ups,
 		const gchar *str, guint16 def_port, void *data);
 
 
+gboolean rspamd_upstreams_parse_line_len (struct upstream_list *ups,
+										  const gchar *str, gsize len,
+										  guint16 def_port,
+										  void *data);
 /**
  * Parse upstreams list from the UCL object
  * @param ups
diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c
index 63a7dd544..3f1990b7b 100644
--- a/src/plugins/surbl.c
+++ b/src/plugins/surbl.c
@@ -2187,9 +2187,9 @@ surbl_is_redirector_handler (lua_State *L)
 
 	task = lua_check_task (L, 1);
 	url = luaL_checklstring (L, 2, &len);
-	surbl_module_ctx = surbl_get_context (task->cfg);
 
 	if (task && url) {
+		surbl_module_ctx = surbl_get_context (task->cfg);
 		url_cpy = rspamd_mempool_alloc (task->task_pool, len);
 		memcpy (url_cpy, url, len);
 
diff --git a/src/rspamadm/rspamadm.c b/src/rspamadm/rspamadm.c
index c0bb4bc72..c49853ef7 100644
--- a/src/rspamadm/rspamadm.c
+++ b/src/rspamadm/rspamadm.c
@@ -436,7 +436,8 @@ main (gint argc, gchar **argv, gchar **env)
 	(void) dns_resolver_init (rspamd_main->logger,
 			rspamd_main->ev_base,
 			cfg);
-	rspamd_main->http_ctx = rspamd_http_context_create (cfg, rspamd_main->ev_base);
+	rspamd_main->http_ctx = rspamd_http_context_create (cfg, rspamd_main->ev_base,
+			NULL);
 
 	g_log_set_default_handler (rspamd_glib_log_function, rspamd_main->logger);
 	g_set_printerr_handler (rspamd_glib_printerr_function);
diff --git a/src/rspamd.c b/src/rspamd.c
index 27ba5e032..142915df9 100644
--- a/src/rspamd.c
+++ b/src/rspamd.c
@@ -1490,7 +1490,7 @@ main (gint argc, gchar **argv, gchar **env)
 	rspamd_mempool_unlock_mutex (rspamd_main->start_mtx);
 
 	rspamd_main->http_ctx = rspamd_http_context_create (rspamd_main->cfg,
-			ev_base);
+			ev_base, rspamd_main->cfg->ups_ctx);
 
 	if (control_fd != -1) {
 		msg_info_main ("listening for control commands on %s",
diff --git a/src/rspamd_proxy.c b/src/rspamd_proxy.c
index e83426673..446552b81 100644
--- a/src/rspamd_proxy.c
+++ b/src/rspamd_proxy.c
@@ -2177,7 +2177,8 @@ start_rspamd_proxy (struct rspamd_worker *worker)
 	rspamd_upstreams_library_config (worker->srv->cfg, ctx->cfg->ups_ctx,
 			ctx->ev_base, ctx->resolver->r);
 
-	ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base);
+	ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base,
+			ctx->cfg->ups_ctx);
 
 	if (ctx->has_self_scan) {
 		/* Additional initialisation needed */
diff --git a/src/worker.c b/src/worker.c
index 77614ec13..40e3d07f9 100644
--- a/src/worker.c
+++ b/src/worker.c
@@ -692,7 +692,8 @@ start_worker (struct rspamd_worker *worker)
 	rspamd_upstreams_library_config (worker->srv->cfg, ctx->cfg->ups_ctx,
 			ctx->ev_base, ctx->resolver->r);
 
-	ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base);
+	ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base,
+			ctx->cfg->ups_ctx);
 	rspamd_worker_init_scanner (worker, ctx->ev_base, ctx->resolver,
 			&ctx->lang_det);
 	rspamd_lua_run_postloads (ctx->cfg->lua_state, ctx->cfg, ctx->ev_base,
-- 
cgit v1.2.3


From f0dc58b678766902780598c9cdffed8188e070f5 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Mon, 18 Mar 2019 15:32:54 +0000
Subject: [Project] More rework for HTTP proxy support

---
 src/libutil/http_connection.c | 157 +++++++++++++++++++++++++++---------------
 src/libutil/http_connection.h |   1 -
 src/plugins/surbl.c           |   6 +-
 3 files changed, 104 insertions(+), 60 deletions(-)

(limited to 'src')

diff --git a/src/libutil/http_connection.c b/src/libutil/http_connection.c
index d782c2d13..e75e26a45 100644
--- a/src/libutil/http_connection.c
+++ b/src/libutil/http_connection.c
@@ -51,6 +51,7 @@ enum rspamd_http_priv_flags {
 	RSPAMD_HTTP_CONN_FLAG_ENCRYPTION_NEEDED = 1 << 4,
 	RSPAMD_HTTP_CONN_FLAG_PROXY = 1 << 5,
 	RSPAMD_HTTP_CONN_FLAG_PROXY_REQUEST = 1 << 6,
+	RSPAMD_HTTP_CONN_OWN_SOCKET = 1 << 7,
 };
 
 #define IS_CONN_ENCRYPTED(c) ((c)->flags & RSPAMD_HTTP_CONN_FLAG_ENCRYPTED)
@@ -1078,24 +1079,25 @@ rspamd_http_parser_reset (struct rspamd_http_connection *conn)
 	priv->parser_cb.on_message_complete = rspamd_http_on_message_complete;
 }
 
-struct rspamd_http_connection *
-rspamd_http_connection_new_server (struct rspamd_http_context *ctx,
+static struct rspamd_http_connection *
+rspamd_http_connection_new_common (struct rspamd_http_context *ctx,
 								   gint fd,
 								   rspamd_http_body_handler_t body_handler,
 								   rspamd_http_error_handler_t error_handler,
 								   rspamd_http_finish_handler_t finish_handler,
-								   unsigned opts)
+								   unsigned opts,
+								   enum rspamd_http_connection_type type,
+								   enum rspamd_http_priv_flags priv_flags,
+								   struct upstream *proxy_upstream)
 {
 	struct rspamd_http_connection *conn;
 	struct rspamd_http_connection_private *priv;
 
-	if (error_handler == NULL || finish_handler == NULL) {
-		return NULL;
-	}
+	g_assert (error_handler != NULL && finish_handler == NULL);
 
 	conn = g_malloc0 (sizeof (struct rspamd_http_connection));
 	conn->opts = opts;
-	conn->type = RSPAMD_HTTP_SERVER;
+	conn->type = type;
 	conn->body_handler = body_handler;
 	conn->error_handler = error_handler;
 	conn->finish_handler = finish_handler;
@@ -1111,8 +1113,17 @@ rspamd_http_connection_new_server (struct rspamd_http_context *ctx,
 	priv = g_malloc0 (sizeof (struct rspamd_http_connection_private));
 	conn->priv = priv;
 	priv->ctx = ctx;
+	priv->flags = priv_flags;
 
-	priv->cache = ctx->server_kp_cache;
+	if (type == RSPAMD_HTTP_SERVER) {
+		priv->cache = ctx->server_kp_cache;
+	}
+	else {
+		priv->cache = ctx->client_kp_cache;
+		if (ctx->client_kp) {
+			priv->local_key = rspamd_keypair_ref (ctx->client_kp);
+		}
+	}
 
 	rspamd_http_parser_reset (conn);
 	priv->parser.data = conn;
@@ -1120,6 +1131,18 @@ rspamd_http_connection_new_server (struct rspamd_http_context *ctx,
 	return conn;
 }
 
+struct rspamd_http_connection *
+rspamd_http_connection_new_server (struct rspamd_http_context *ctx,
+								   gint fd,
+								   rspamd_http_body_handler_t body_handler,
+								   rspamd_http_error_handler_t error_handler,
+								   rspamd_http_finish_handler_t finish_handler,
+								   unsigned opts)
+{
+	return rspamd_http_connection_new_common (ctx, fd, body_handler,
+			error_handler, finish_handler, opts, RSPAMD_HTTP_SERVER, 0, NULL);
+}
+
 struct rspamd_http_connection *
 rspamd_http_connection_new_client_socket (struct rspamd_http_context *ctx,
 								   rspamd_http_body_handler_t body_handler,
@@ -1128,39 +1151,8 @@ rspamd_http_connection_new_client_socket (struct rspamd_http_context *ctx,
 								   unsigned opts,
 								   gint fd)
 {
-	struct rspamd_http_connection *conn;
-	struct rspamd_http_connection_private *priv;
-
-	g_assert (error_handler != NULL && finish_handler == NULL);
-
-	conn = g_malloc0 (sizeof (struct rspamd_http_connection));
-	conn->opts = opts;
-	conn->type = RSPAMD_HTTP_CLIENT;
-	conn->body_handler = body_handler;
-	conn->error_handler = error_handler;
-	conn->finish_handler = finish_handler;
-	conn->fd = fd;
-	conn->ref = 1;
-	conn->finished = FALSE;
-
-	/* Init priv */
-	if (ctx == NULL) {
-		ctx = rspamd_http_context_default ();
-	}
-
-	priv = g_malloc0 (sizeof (struct rspamd_http_connection_private));
-	conn->priv = priv;
-	priv->ctx = ctx;
-
-	priv->cache = ctx->client_kp_cache;
-	if (ctx->client_kp) {
-		priv->local_key = rspamd_keypair_ref (ctx->client_kp);
-	}
-
-	rspamd_http_parser_reset (conn);
-	priv->parser.data = conn;
-
-	return conn;
+	return rspamd_http_connection_new_common (ctx, fd, body_handler,
+			error_handler, finish_handler, opts, RSPAMD_HTTP_CLIENT, 0, NULL);
 }
 
 struct rspamd_http_connection *
@@ -1173,21 +1165,77 @@ rspamd_http_connection_new_client (struct rspamd_http_context *ctx,
 {
 	gint fd;
 
-	if (error_handler == NULL || finish_handler == NULL) {
-		return NULL;
+	if (opts & RSPAMD_HTTP_CLIENT_ENCRYPTED) {
+		if (ctx->https_proxies) {
+			struct upstream *up = rspamd_upstream_get (ctx->https_proxies,
+					RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0);
+
+			if (up) {
+				rspamd_inet_addr_t *proxy_addr = rspamd_upstream_addr_next (up);
+
+				fd = rspamd_inet_address_connect (proxy_addr, SOCK_STREAM, TRUE);
+
+				if (fd == -1) {
+					msg_info ("cannot connect to https proxy %s: %s",
+							rspamd_inet_address_to_string (proxy_addr),
+							strerror (errno));
+					rspamd_upstream_fail (up, TRUE);
+
+					return NULL;
+				}
+
+				return rspamd_http_connection_new_common (ctx, fd, body_handler,
+						error_handler, finish_handler, opts,
+						RSPAMD_HTTP_CLIENT,
+						RSPAMD_HTTP_CONN_OWN_SOCKET|RSPAMD_HTTP_CONN_FLAG_PROXY,
+						up);
+			}
+		}
+	}
+	else {
+		if (ctx->http_proxies) {
+			struct upstream *up = rspamd_upstream_get (ctx->http_proxies,
+					RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0);
+
+			if (up) {
+				rspamd_inet_addr_t *proxy_addr = rspamd_upstream_addr_next (up);
+
+				fd = rspamd_inet_address_connect (proxy_addr, SOCK_STREAM, TRUE);
+
+				if (fd == -1) {
+					msg_info ("cannot connect to http proxy %s: %s",
+							rspamd_inet_address_to_string (proxy_addr),
+							strerror (errno));
+					rspamd_upstream_fail (up, TRUE);
+
+					return NULL;
+				}
+
+				return rspamd_http_connection_new_common (ctx, fd, body_handler,
+						error_handler, finish_handler, opts,
+						RSPAMD_HTTP_CLIENT,
+						RSPAMD_HTTP_CONN_OWN_SOCKET|RSPAMD_HTTP_CONN_FLAG_PROXY,
+						up);
+			}
+		}
 	}
 
+	/* Unproxied version */
 	fd = rspamd_inet_address_connect (addr, SOCK_STREAM, TRUE);
 
 	if (fd == -1) {
-		msg_info ("cannot connect to %s: %s", rspamd_inet_address_to_string (addr),
+		msg_info ("cannot connect to proxy %s: %s",
+				rspamd_inet_address_to_string (addr),
 				strerror (errno));
+
 		return NULL;
 	}
 
-	return rspamd_http_connection_new_client_socket (ctx,
-			body_handler, error_handler, finish_handler,
-			opts | RSPAMD_HTTP_OWN_SOCKET, fd);
+	return rspamd_http_connection_new_common (ctx, fd, body_handler,
+			error_handler, finish_handler, opts,
+			RSPAMD_HTTP_CLIENT,
+			RSPAMD_HTTP_CONN_OWN_SOCKET,
+			NULL);
 }
 
 struct rspamd_http_connection *
@@ -1200,10 +1248,6 @@ rspamd_http_connection_new_keepalive (struct rspamd_http_context *ctx,
 {
 	struct rspamd_http_connection *conn;
 
-	if (error_handler == NULL || finish_handler == NULL) {
-		return NULL;
-	}
-
 	if (ctx == NULL) {
 		ctx = rspamd_http_context_default ();
 	}
@@ -1437,12 +1481,12 @@ rspamd_http_connection_free (struct rspamd_http_connection *conn)
 			rspamd_pubkey_unref (priv->peer_key);
 		}
 
-		g_free (priv);
-	}
+		if (priv->flags & RSPAMD_HTTP_CONN_OWN_SOCKET) {
+			/* Fd is owned by a connection */
+			close (conn->fd);
+		}
 
-	if (conn->opts & RSPAMD_HTTP_OWN_SOCKET) {
-		/* Fd is owned by a connection */
-		close (conn->fd);
+		g_free (priv);
 	}
 
 	g_free (conn);
@@ -1810,6 +1854,7 @@ rspamd_http_message_write_header (const gchar* mime_type, gboolean encrypted,
 			}
 		}
 		else {
+			/* Normal HTTP/1.1 with Host */
 			if (encrypted) {
 				if (host != NULL) {
 					rspamd_printf_fstring (buf,
diff --git a/src/libutil/http_connection.h b/src/libutil/http_connection.h
index da354ac41..6240772da 100644
--- a/src/libutil/http_connection.h
+++ b/src/libutil/http_connection.h
@@ -85,7 +85,6 @@ enum rspamd_http_options {
 	RSPAMD_HTTP_CLIENT_SHARED = 1u << 3, /**< Store reply in shared memory */
 	RSPAMD_HTTP_REQUIRE_ENCRYPTION = 1u << 4,
 	RSPAMD_HTTP_CLIENT_KEEP_ALIVE = 1u << 5,
-	RSPAMD_HTTP_OWN_SOCKET = 1u << 6,
 };
 
 typedef int (*rspamd_http_body_handler_t) (struct rspamd_http_connection *conn,
diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c
index 3f1990b7b..11d3e35d9 100644
--- a/src/plugins/surbl.c
+++ b/src/plugins/surbl.c
@@ -1788,6 +1788,8 @@ register_redirector_call (struct rspamd_url *url, struct rspamd_task *task,
 
 		selected = rspamd_upstream_get (surbl_module_ctx->redirectors,
 				RSPAMD_UPSTREAM_ROUND_ROBIN, url->host, url->hostlen);
+		param = rspamd_mempool_alloc0 (task->task_pool,
+						sizeof (struct redirector_param));
 
 		if (selected) {
 			param->conn = rspamd_http_connection_new_client (NULL,
@@ -1806,9 +1808,7 @@ register_redirector_call (struct rspamd_url *url, struct rspamd_task *task,
 			return;
 		}
 
-		param =
-				rspamd_mempool_alloc (task->task_pool,
-						sizeof (struct redirector_param));
+
 		param->url = url;
 		param->task = task;
 		param->ctx = surbl_module_ctx;
-- 
cgit v1.2.3


From 02de13af4e812e81c882fa33683429c65a475099 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Mon, 18 Mar 2019 15:56:45 +0000
Subject: [Minor] Simplify host element handling in HTTP message

---
 src/libutil/http_connection.c | 87 ++++++++++++++++---------------------------
 src/libutil/http_message.c    |  4 +-
 src/libutil/http_private.h    |  2 +-
 src/lua/lua_http.c            |  6 +--
 4 files changed, 36 insertions(+), 63 deletions(-)

(limited to 'src')

diff --git a/src/libutil/http_connection.c b/src/libutil/http_connection.c
index e75e26a45..78ec2582f 100644
--- a/src/libutil/http_connection.c
+++ b/src/libutil/http_connection.c
@@ -720,7 +720,7 @@ rspamd_http_simple_client_helper (struct rspamd_http_connection *conn)
 	struct rspamd_http_connection_private *priv;
 	gpointer ssl;
 	gint request_method;
-	rspamd_fstring_t *prev_host;
+	GString *prev_host = NULL;
 
 	priv = conn->priv;
 	ssl = priv->ssl;
@@ -754,7 +754,7 @@ rspamd_http_simple_client_helper (struct rspamd_http_connection *conn)
 	}
 	else {
 		if (prev_host) {
-			rspamd_fstring_free (prev_host);
+			g_string_free (prev_host, TRUE);
 		}
 	}
 }
@@ -1426,8 +1426,7 @@ rspamd_http_connection_copy_msg (struct rspamd_http_message *msg, GError **err)
 	}
 
 	if (msg->host) {
-		new_msg->host = rspamd_fstring_new_init (msg->host->str,
-				msg->host->len);
+		new_msg->host = g_string_new_len (msg->host->str, msg->host->len);
 	}
 
 	new_msg->method = msg->method;
@@ -1812,12 +1811,15 @@ rspamd_http_message_write_header (const gchar* mime_type, gboolean encrypted,
 		}
 	}
 	else {
+
+		/* Client request */
 		if (conn->opts & RSPAMD_HTTP_CLIENT_KEEP_ALIVE) {
 			conn_type = "keep-alive";
 		}
 
 		/* Format request */
-		enclen += msg->url->len + strlen (http_method_str (msg->method)) + 1;
+		enclen += RSPAMD_FSTRING_LEN (msg->url) +
+				strlen (http_method_str (msg->method)) + 1;
 
 		if (host == NULL && msg->host == NULL) {
 			/* Fallback to HTTP/1.0 */
@@ -1855,59 +1857,34 @@ rspamd_http_message_write_header (const gchar* mime_type, gboolean encrypted,
 		}
 		else {
 			/* Normal HTTP/1.1 with Host */
+			if (host == NULL) {
+				host = msg->host->str;
+			}
+
 			if (encrypted) {
-				if (host != NULL) {
-					rspamd_printf_fstring (buf,
-							"%s %s HTTP/1.1\r\n"
-							"Connection: %s\r\n"
-							"Host: %s\r\n"
-							"Content-Length: %z\r\n"
-							"Content-Type: application/octet-stream\r\n",
-							"POST",
-							"/post",
-							conn_type,
-							host,
-							enclen);
-				}
-				else {
-					rspamd_printf_fstring (buf,
-							"%s %s HTTP/1.1\r\n"
-							"Connection: %s\r\n"
-							"Host: %V\r\n"
-							"Content-Length: %z\r\n"
-							"Content-Type: application/octet-stream\r\n",
-							"POST",
-							"/post",
-							conn_type,
-							msg->host,
-							enclen);
-				}
+				rspamd_printf_fstring (buf,
+						"%s %s HTTP/1.1\r\n"
+						"Connection: %s\r\n"
+						"Host: %s\r\n"
+						"Content-Length: %z\r\n"
+						"Content-Type: application/octet-stream\r\n",
+						"POST",
+						"/post",
+						conn_type,
+						host,
+						enclen);
 			}
 			else {
-				if (host != NULL) {
-					rspamd_printf_fstring (buf,
-							"%s %V HTTP/1.1\r\n"
-							"Connection: %s\r\n"
-							"Host: %s\r\n"
-							"Content-Length: %z\r\n",
-							http_method_str (msg->method),
-							msg->url,
-							conn_type,
-							host,
-							bodylen);
-				}
-				else {
-					rspamd_printf_fstring (buf,
-							"%s %V HTTP/1.1\r\n"
-							"Connection: %s\r\n"
-							"Host: %V\r\n"
-							"Content-Length: %z\r\n",
-							http_method_str (msg->method),
-							msg->url,
-							conn_type,
-							msg->host,
-							bodylen);
-				}
+				rspamd_printf_fstring (buf,
+						"%s %V HTTP/1.1\r\n"
+						"Connection: %s\r\n"
+						"Host: %s\r\n"
+						"Content-Length: %z\r\n",
+						http_method_str (msg->method),
+						msg->url,
+						conn_type,
+						host,
+						bodylen);
 
 				if (bodylen > 0) {
 					if (mime_type != NULL) {
diff --git a/src/libutil/http_message.c b/src/libutil/http_message.c
index 0720dc416..13241034c 100644
--- a/src/libutil/http_message.c
+++ b/src/libutil/http_message.c
@@ -104,7 +104,7 @@ rspamd_http_message_from_url (const gchar *url)
 		}
 	}
 
-	msg->host = rspamd_fstring_new_init (host, pu.field_data[UF_HOST].len);
+	msg->host = g_string_new_len (host, pu.field_data[UF_HOST].len);
 	msg->url = rspamd_fstring_append (msg->url, path, pathlen);
 
 	REF_INIT_RETAIN (msg, rspamd_http_message_free);
@@ -489,7 +489,7 @@ rspamd_http_message_free (struct rspamd_http_message *msg)
 		rspamd_fstring_free (msg->status);
 	}
 	if (msg->host != NULL) {
-		rspamd_fstring_free (msg->host);
+		g_string_free (msg->host, TRUE);
 	}
 	if (msg->peer_key != NULL) {
 		rspamd_pubkey_unref (msg->peer_key);
diff --git a/src/libutil/http_private.h b/src/libutil/http_private.h
index dd4ca3435..fbc4c17e3 100644
--- a/src/libutil/http_private.h
+++ b/src/libutil/http_private.h
@@ -43,7 +43,7 @@ struct rspamd_http_header {
  */
 struct rspamd_http_message {
 	rspamd_fstring_t *url;
-	rspamd_fstring_t *host;
+	GString *host;
 	rspamd_fstring_t *status;
 	struct rspamd_http_header *headers;
 
diff --git a/src/lua/lua_http.c b/src/lua/lua_http.c
index 19cf8f2c9..a7b8c0a89 100644
--- a/src/lua/lua_http.c
+++ b/src/lua/lua_http.c
@@ -127,10 +127,6 @@ lua_http_fin (gpointer arg)
 		g_free (cbd->mime_type);
 	}
 
-	if (cbd->host) {
-		g_free (cbd->host);
-	}
-
 	if (cbd->auth) {
 		g_free (cbd->auth);
 	}
@@ -935,7 +931,7 @@ lua_http_request (lua_State *L)
 	}
 
 	if (msg->host) {
-		cbd->host = rspamd_fstring_cstr (msg->host);
+		cbd->host = msg->host->str;
 	}
 
 	if (body) {
-- 
cgit v1.2.3


From 6e95f83a94eac93a4cb5596430e2c03a24536a50 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Mon, 18 Mar 2019 16:00:36 +0000
Subject: [Project] Add support of HTTP proxy in requests

Issue: #572
---
 src/libutil/http_connection.c | 37 +++++++++++++++++++++++++++----------
 1 file changed, 27 insertions(+), 10 deletions(-)

(limited to 'src')

diff --git a/src/libutil/http_connection.c b/src/libutil/http_connection.c
index 78ec2582f..ea664f20d 100644
--- a/src/libutil/http_connection.c
+++ b/src/libutil/http_connection.c
@@ -1862,6 +1862,7 @@ rspamd_http_message_write_header (const gchar* mime_type, gboolean encrypted,
 			}
 
 			if (encrypted) {
+				/* TODO: Add proxy support to HTTPCrypt */
 				rspamd_printf_fstring (buf,
 						"%s %s HTTP/1.1\r\n"
 						"Connection: %s\r\n"
@@ -1875,16 +1876,32 @@ rspamd_http_message_write_header (const gchar* mime_type, gboolean encrypted,
 						enclen);
 			}
 			else {
-				rspamd_printf_fstring (buf,
-						"%s %V HTTP/1.1\r\n"
-						"Connection: %s\r\n"
-						"Host: %s\r\n"
-						"Content-Length: %z\r\n",
-						http_method_str (msg->method),
-						msg->url,
-						conn_type,
-						host,
-						bodylen);
+				if (conn->priv->flags & RSPAMD_HTTP_CONN_FLAG_PROXY) {
+					rspamd_printf_fstring (buf,
+							"%s http://%s:%d/%V HTTP/1.1\r\n"
+							"Connection: %s\r\n"
+							"Host: %s\r\n"
+							"Content-Length: %z\r\n",
+							http_method_str (msg->method),
+							host,
+							msg->port,
+							msg->url,
+							conn_type,
+							host,
+							bodylen);
+				}
+				else {
+					rspamd_printf_fstring (buf,
+							"%s %V HTTP/1.1\r\n"
+							"Connection: %s\r\n"
+							"Host: %s\r\n"
+							"Content-Length: %z\r\n",
+							http_method_str (msg->method),
+							msg->url,
+							conn_type,
+							host,
+							bodylen);
+				}
 
 				if (bodylen > 0) {
 					if (mime_type != NULL) {
-- 
cgit v1.2.3


From 6292e6da71860030d5336c0f751c335bfa5c14a7 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Mon, 18 Mar 2019 16:17:16 +0000
Subject: [Project] Drop HTTPS proxies as they are too intrusive to implement

Issue: #572
---
 src/libutil/http_connection.c | 71 +++++++++++++++----------------------------
 src/libutil/http_context.c    | 16 +++-------
 src/libutil/http_context.h    |  1 -
 src/libutil/http_private.h    |  1 -
 4 files changed, 28 insertions(+), 61 deletions(-)

(limited to 'src')

diff --git a/src/libutil/http_connection.c b/src/libutil/http_connection.c
index ea664f20d..72cc1d239 100644
--- a/src/libutil/http_connection.c
+++ b/src/libutil/http_connection.c
@@ -1165,58 +1165,29 @@ rspamd_http_connection_new_client (struct rspamd_http_context *ctx,
 {
 	gint fd;
 
-	if (opts & RSPAMD_HTTP_CLIENT_ENCRYPTED) {
-		if (ctx->https_proxies) {
-			struct upstream *up = rspamd_upstream_get (ctx->https_proxies,
-					RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0);
+	if (ctx->http_proxies) {
+		struct upstream *up = rspamd_upstream_get (ctx->http_proxies,
+				RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0);
 
-			if (up) {
-				rspamd_inet_addr_t *proxy_addr = rspamd_upstream_addr_next (up);
+		if (up) {
+			rspamd_inet_addr_t *proxy_addr = rspamd_upstream_addr_next (up);
 
-				fd = rspamd_inet_address_connect (proxy_addr, SOCK_STREAM, TRUE);
+			fd = rspamd_inet_address_connect (proxy_addr, SOCK_STREAM, TRUE);
 
-				if (fd == -1) {
-					msg_info ("cannot connect to https proxy %s: %s",
-							rspamd_inet_address_to_string (proxy_addr),
-							strerror (errno));
-					rspamd_upstream_fail (up, TRUE);
-
-					return NULL;
-				}
+			if (fd == -1) {
+				msg_info ("cannot connect to http proxy %s: %s",
+						rspamd_inet_address_to_string (proxy_addr),
+						strerror (errno));
+				rspamd_upstream_fail (up, TRUE);
 
-				return rspamd_http_connection_new_common (ctx, fd, body_handler,
-						error_handler, finish_handler, opts,
-						RSPAMD_HTTP_CLIENT,
-						RSPAMD_HTTP_CONN_OWN_SOCKET|RSPAMD_HTTP_CONN_FLAG_PROXY,
-						up);
+				return NULL;
 			}
-		}
-	}
-	else {
-		if (ctx->http_proxies) {
-			struct upstream *up = rspamd_upstream_get (ctx->http_proxies,
-					RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0);
-
-			if (up) {
-				rspamd_inet_addr_t *proxy_addr = rspamd_upstream_addr_next (up);
 
-				fd = rspamd_inet_address_connect (proxy_addr, SOCK_STREAM, TRUE);
-
-				if (fd == -1) {
-					msg_info ("cannot connect to http proxy %s: %s",
-							rspamd_inet_address_to_string (proxy_addr),
-							strerror (errno));
-					rspamd_upstream_fail (up, TRUE);
-
-					return NULL;
-				}
-
-				return rspamd_http_connection_new_common (ctx, fd, body_handler,
-						error_handler, finish_handler, opts,
-						RSPAMD_HTTP_CLIENT,
-						RSPAMD_HTTP_CONN_OWN_SOCKET|RSPAMD_HTTP_CONN_FLAG_PROXY,
-						up);
-			}
+			return rspamd_http_connection_new_common (ctx, fd, body_handler,
+					error_handler, finish_handler, opts,
+					RSPAMD_HTTP_CLIENT,
+					RSPAMD_HTTP_CONN_OWN_SOCKET|RSPAMD_HTTP_CONN_FLAG_PROXY,
+					up);
 		}
 	}
 
@@ -1878,11 +1849,12 @@ rspamd_http_message_write_header (const gchar* mime_type, gboolean encrypted,
 			else {
 				if (conn->priv->flags & RSPAMD_HTTP_CONN_FLAG_PROXY) {
 					rspamd_printf_fstring (buf,
-							"%s http://%s:%d/%V HTTP/1.1\r\n"
+							"%s %s://%s:%d/%V HTTP/1.1\r\n"
 							"Connection: %s\r\n"
 							"Host: %s\r\n"
 							"Content-Length: %z\r\n",
 							http_method_str (msg->method),
+							(msg->flags & RSPAMD_HTTP_FLAG_SSL) ? "https" : "http",
 							host,
 							msg->port,
 							msg->url,
@@ -2243,6 +2215,11 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn
 
 	priv->flags &= ~RSPAMD_HTTP_CONN_FLAG_RESETED;
 
+	if (priv->flags & RSPAMD_HTTP_CONN_FLAG_PROXY) {
+		/* We need to disable SSL flag! */
+		msg->flags &=~ RSPAMD_HTTP_FLAG_SSL;
+	}
+
 	if (rspamd_event_pending (&priv->ev, EV_TIMEOUT|EV_WRITE|EV_READ)) {
 		event_del (&priv->ev);
 	}
diff --git a/src/libutil/http_context.c b/src/libutil/http_context.c
index 18c89c6bc..b3a308891 100644
--- a/src/libutil/http_context.c
+++ b/src/libutil/http_context.c
@@ -203,11 +203,6 @@ rspamd_http_context_init (struct rspamd_http_context *ctx)
 				&ctx->http_proxies);
 	}
 
-	if (ctx->config.https_proxy) {
-		rspamd_http_context_parse_proxy (ctx, ctx->config.https_proxy,
-				&ctx->https_proxies);
-	}
-
 	default_ctx = ctx;
 }
 
@@ -270,13 +265,6 @@ rspamd_http_context_create (struct rspamd_config *cfg,
 			if (http_proxy) {
 				ctx->config.http_proxy = ucl_object_tostring (http_proxy);
 			}
-
-			const ucl_object_t *https_proxy;
-			https_proxy = ucl_object_lookup (client_obj, "https_proxy");
-
-			if (https_proxy) {
-				ctx->config.https_proxy = ucl_object_tostring (https_proxy);
-			}
 		}
 
 		server_obj = ucl_object_lookup (http_obj, "server");
@@ -340,6 +328,10 @@ rspamd_http_context_free (struct rspamd_http_context *ctx)
 
 	kh_destroy (rspamd_keep_alive_hash, ctx->keep_alive_hash);
 
+	if (ctx->http_proxies) {
+		rspamd_upstreams_destroy (ctx->http_proxies);
+	}
+
 	g_free (ctx);
 }
 
diff --git a/src/libutil/http_context.h b/src/libutil/http_context.h
index 6abd66651..4cf07fb48 100644
--- a/src/libutil/http_context.h
+++ b/src/libutil/http_context.h
@@ -36,7 +36,6 @@ struct rspamd_http_context_cfg {
 	gdouble client_key_rotate_time;
 	const gchar *user_agent;
 	const gchar *http_proxy;
-	const gchar *https_proxy;
 };
 
 /**
diff --git a/src/libutil/http_private.h b/src/libutil/http_private.h
index fbc4c17e3..368715891 100644
--- a/src/libutil/http_private.h
+++ b/src/libutil/http_private.h
@@ -98,7 +98,6 @@ struct rspamd_http_context {
 	struct rspamd_keypair_cache *server_kp_cache;
 	struct upstream_ctx *ups_ctx;
 	struct upstream_list *http_proxies;
-	struct upstream_list *https_proxies;
 	gpointer ssl_ctx;
 	gpointer ssl_ctx_noverify;
 	struct event_base *ev_base;
-- 
cgit v1.2.3


From a8eb1a9f1fd59741f4b4a8c1c79155161f574857 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Mon, 18 Mar 2019 17:23:22 +0000
Subject: [Minor] Add task:get_reply_sender method

---
 src/lua/lua_task.c | 49 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)

(limited to 'src')

diff --git a/src/lua/lua_task.c b/src/lua/lua_task.c
index 8e1a851b4..c762eb173 100644
--- a/src/lua/lua_task.c
+++ b/src/lua/lua_task.c
@@ -424,6 +424,16 @@ LUA_FUNCTION_DEF (task, get_recipients);
  * @return {string} principal recipient
  */
 LUA_FUNCTION_DEF (task, get_principal_recipient);
+/***
+ * @method task:get_reply_sender()
+ * Returns a single string with address that should be used to reply on a message
+ *
+ * - reply-to header
+ * - from header
+ * - smtp from as a last resort
+ * @return {address} email address
+ */
+LUA_FUNCTION_DEF (task, get_sender);
 
 /***
  * @method task:set_recipients([type], {rcpt1, rcpt2...})
@@ -1039,6 +1049,7 @@ static const struct luaL_reg tasklib_m[] = {
 	LUA_INTERFACE_DEF (task, get_recipients),
 	LUA_INTERFACE_DEF (task, set_recipients),
 	LUA_INTERFACE_DEF (task, get_principal_recipient),
+	LUA_INTERFACE_DEF (task, get_reply_sender),
 	LUA_INTERFACE_DEF (task, has_from),
 	LUA_INTERFACE_DEF (task, get_from),
 	LUA_INTERFACE_DEF (task, set_from),
@@ -3274,6 +3285,44 @@ lua_task_get_principal_recipient (lua_State *L)
 	return 1;
 }
 
+static gint
+lua_task_get_reply_sender (lua_State *L)
+{
+	LUA_TRACE_POINT;
+	struct rspamd_task *task = lua_check_task (L, 1);
+	struct rspamd_mime_header *rh;
+
+	if (task) {
+		GPtrArray *ar;
+
+		ar = rspamd_message_get_header_array (task, "Reply-To", false);
+
+		if (ar && ar->len == 1) {
+			rh = (struct rspamd_mime_header *)g_ptr_array_index (ar, 0);
+			lua_pushstring (L, rh->decoded);
+		}
+		else if (task->from_mime && task->from_mime->len == 1) {
+			struct rspamd_email_address *addr;
+
+			addr = (struct rspamd_email_address *)g_ptr_array_index (task->from_mime, 0);
+
+			lua_pushlstring (L, addr->addr, addr->addr_len);
+		}
+		else if (task->from_envelope) {
+			lua_pushlstring (L, task->from_envelope->addr,
+					task->from_envelope->addr_len);
+		}
+		else {
+			lua_pushnil (L);
+		}
+	}
+	else {
+		return luaL_error (L, "invalid arguments");
+	}
+
+	return 1;
+}
+
 static gint
 lua_task_get_user (lua_State *L)
 {
-- 
cgit v1.2.3


From 78a9afb714a54e3c40910973bea1ade927db8cc6 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Mon, 18 Mar 2019 17:23:38 +0000
Subject: [Feature] Replies: Filter replies sender to limit whitelisting to
 direct messages

---
 src/plugins/lua/replies.lua | 69 ++++++++++++++++++++++++++++++++++-----------
 1 file changed, 53 insertions(+), 16 deletions(-)

(limited to 'src')

diff --git a/src/plugins/lua/replies.lua b/src/plugins/lua/replies.lua
index 234a41ca3..48c760eed 100644
--- a/src/plugins/lua/replies.lua
+++ b/src/plugins/lua/replies.lua
@@ -32,6 +32,7 @@ local settings = {
   action = nil,
   expire = 86400, -- 1 day by default
   key_prefix = 'rr',
+  key_size = 20,
   message = 'Message is reply to one we originated',
   symbol = 'REPLY',
   score = -4, -- Default score
@@ -45,21 +46,49 @@ local settings = {
 
 local N = "replies"
 
-local function make_key(goop)
+local function make_key(goop, sz, prefix)
   local h = hash.create()
   h:update(goop)
-  local key = h:base32():sub(1, 20)
-  key = settings['key_prefix'] .. key
+  local key
+  if sz then
+    key = h:base32():sub(1, sz)
+  else
+    key = h:base32()
+  end
+
+  if prefix then
+    key = prefix .. key
+  end
+
   return key
 end
 
 local function replies_check(task)
+  local function check_recipient(stored_rcpt)
+    local real_rcpt = task:get_principal_recipient()
+
+    if real_rcpt then
+      local real_rcpt_h = make_key(real_rcpt:lower(), 8)
+      if real_rcpt_h == stored_rcpt then
+        return true
+      end
+
+      rspamd_logger.infox(task, 'ignoring reply as recipient %s is not matching hash %s',
+          real_rcpt, stored_rcpt)
+    else
+      rspamd_logger.infox(task, 'ignoring reply as recipient cannot be detected for hash %s',
+          stored_rcpt)
+    end
+
+    return false
+  end
+
   local function redis_get_cb(err, data)
     if err ~= nil then
       rspamd_logger.errx(task, 'redis_get_cb received error: %1', err)
       return
     end
-    if data == '1' then
+    if data and check_recipient(data) then
       -- Hash was found
       task:insert_result(settings['symbol'], 1.0)
       if settings['action'] ~= nil then
@@ -80,7 +109,7 @@ local function replies_check(task)
     return
   end
   -- Create hash of in-reply-to and query redis
-  local key = make_key(irt)
+  local key = make_key(irt, settings.key_size, settings.key_prefix)
 
   local ret = lua_redis.redis_make_request(task,
     redis_params, -- connect params
@@ -117,18 +146,26 @@ local function replies_set(task)
     return
   end
   -- Create hash of message-id and store to redis
-  local key = make_key(msg_id)
+  local key = make_key(msg_id, settings.key_size, settings.key_prefix)
   lua_util.debugm(N, task, 'storing message-id for replies check')
-  local ret = lua_redis.redis_make_request(task,
-    redis_params, -- connect params
-    key, -- hash key
-    true, -- is write
-    redis_set_cb, --callback
-    'SETEX', -- command
-    {key, tostring(settings['expire']), "1"} -- arguments
-  )
-  if not ret then
-    rspamd_logger.errx(task, "redis request wasn't scheduled")
+
+  local value = task:get_reply_sender()
+
+  if value then
+    value = make_key(value:lower(), 8)
+    local ret = lua_redis.redis_make_request(task,
+        redis_params, -- connect params
+        key, -- hash key
+        true, -- is write
+        redis_set_cb, --callback
+        'SETEX', -- command
+        {key, tostring(settings['expire']), value:lower()} -- arguments
+    )
+    if not ret then
+      rspamd_logger.errx(task, "redis request wasn't scheduled")
+    end
+  else
+    rspamd_logger.infox(task, "cannot find reply sender address")
   end
 end
 
-- 
cgit v1.2.3


From 066b39426d0e90d9add72ef366ad1a0ae9ed2554 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Mon, 18 Mar 2019 17:29:26 +0000
Subject: [Minor] Oops, fix build

---
 src/lua/lua_task.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/lua/lua_task.c b/src/lua/lua_task.c
index c762eb173..b5ed7dd19 100644
--- a/src/lua/lua_task.c
+++ b/src/lua/lua_task.c
@@ -433,7 +433,7 @@ LUA_FUNCTION_DEF (task, get_principal_recipient);
  * - smtp from as a last resort
  * @return {address} email address
  */
-LUA_FUNCTION_DEF (task, get_sender);
+LUA_FUNCTION_DEF (task, get_reply_sender);
 
 /***
  * @method task:set_recipients([type], {rcpt1, rcpt2...})
-- 
cgit v1.2.3


From e2ff03403bc58f032a8081a81e22c78fd7e1f250 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Mon, 18 Mar 2019 17:49:38 +0000
Subject: [Minor] Another 'oops'

---
 src/libutil/http_connection.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/libutil/http_connection.c b/src/libutil/http_connection.c
index 72cc1d239..a0623dca9 100644
--- a/src/libutil/http_connection.c
+++ b/src/libutil/http_connection.c
@@ -1093,7 +1093,7 @@ rspamd_http_connection_new_common (struct rspamd_http_context *ctx,
 	struct rspamd_http_connection *conn;
 	struct rspamd_http_connection_private *priv;
 
-	g_assert (error_handler != NULL && finish_handler == NULL);
+	g_assert (error_handler != NULL && finish_handler != NULL);
 
 	conn = g_malloc0 (sizeof (struct rspamd_http_connection));
 	conn->opts = opts;
-- 
cgit v1.2.3


From 53cc596ad5031354e4ff95657dc18e4afd9af39f Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Mon, 18 Mar 2019 17:52:48 +0000
Subject: [Minor] Sigh, another oops

---
 src/libutil/http_connection.c | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

(limited to 'src')

diff --git a/src/libutil/http_connection.c b/src/libutil/http_connection.c
index a0623dca9..b82f3c98f 100644
--- a/src/libutil/http_connection.c
+++ b/src/libutil/http_connection.c
@@ -1095,6 +1095,10 @@ rspamd_http_connection_new_common (struct rspamd_http_context *ctx,
 
 	g_assert (error_handler != NULL && finish_handler != NULL);
 
+	if (ctx == NULL) {
+		ctx = rspamd_http_context_default ();
+	}
+
 	conn = g_malloc0 (sizeof (struct rspamd_http_connection));
 	conn->opts = opts;
 	conn->type = type;
@@ -1106,10 +1110,6 @@ rspamd_http_connection_new_common (struct rspamd_http_context *ctx,
 	conn->finished = FALSE;
 
 	/* Init priv */
-	if (ctx == NULL) {
-		ctx = rspamd_http_context_default ();
-	}
-
 	priv = g_malloc0 (sizeof (struct rspamd_http_connection_private));
 	conn->priv = priv;
 	priv->ctx = ctx;
@@ -1165,6 +1165,10 @@ rspamd_http_connection_new_client (struct rspamd_http_context *ctx,
 {
 	gint fd;
 
+	if (ctx == NULL) {
+		ctx = rspamd_http_context_default ();
+	}
+
 	if (ctx->http_proxies) {
 		struct upstream *up = rspamd_upstream_get (ctx->http_proxies,
 				RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0);
-- 
cgit v1.2.3


From 723e9e963380dfa12d5b9e0f7b999535e4917e48 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Mon, 18 Mar 2019 17:53:16 +0000
Subject: [Feature] Greylist: Allow to disable greylisting depending on symbols

---
 src/plugins/lua/greylist.lua | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

(limited to 'src')

diff --git a/src/plugins/lua/greylist.lua b/src/plugins/lua/greylist.lua
index 5a1f6c0f9..1ba80dd80 100644
--- a/src/plugins/lua/greylist.lua
+++ b/src/plugins/lua/greylist.lua
@@ -44,7 +44,7 @@ end
 local redis_params
 local whitelisted_ip
 local whitelist_domains_map
-local toint =math.ifloor or math.floor
+local toint = math.ifloor or math.floor
 local settings = {
   expire = 86400, -- 1 day by default
   timeout = 300, -- 5 minutes by default
@@ -53,6 +53,7 @@ local settings = {
   message = 'Try again later', -- default greylisted message
   symbol = 'GREYLIST',
   action = 'soft reject', -- default greylisted action
+  whitelist_symbols = {}, -- whitelist when specific symbols have been found
   ipv4_mask = 19, -- Mask bits for ipv4
   ipv6_mask = 64, -- Mask bits for ipv6
   report_time = false, -- Tell when greylisting is epired (appended to `message`)
@@ -65,6 +66,7 @@ local rspamd_util = require "rspamd_util"
 local fun = require "fun"
 local hash = require "rspamd_cryptobox_hash"
 local rspamd_lua_utils = require "lua_util"
+local lua_map = require "lua_map"
 local N = "greylist"
 
 local function data_key(task)
@@ -276,6 +278,19 @@ local function greylist_set(task)
   -- Don't do anything if pre-result has been already set
   if task:has_pre_result() then return end
 
+  -- Check whitelist_symbols
+  for _,sym in ipairs(settings.whitelist_symbols) do
+    if task:has_symbol(sym) then
+      rspamd_logger.infox(task, 'skip greylisting as we have found symbol %s', sym)
+      if action == 'greylist' then
+        -- We are going to accept message
+        rspamd_logger.infox(task, 'downgrading metric action from "greylist" to "no action"')
+        task:disable_action('greylist')
+      end
+      return
+    end
+  end
+
   if settings.greylist_min_score then
     local score = task:get_metric_score('default')[1]
     if score < settings.greylist_min_score then
@@ -438,12 +453,12 @@ if opts then
     end
   end
 
-  whitelisted_ip = rspamd_map_add('greylist', 'whitelisted_ip', 'radix',
+  whitelisted_ip = lua_map.rspamd_map_add(N, 'whitelisted_ip', 'radix',
     'Greylist whitelist ip map')
-  whitelist_domains_map = rspamd_map_add('greylist', 'whitelist_domains_url',
+  whitelist_domains_map = lua_map.rspamd_map_add(N, 'whitelist_domains_url',
     'map', 'Greylist whitelist domains map')
 
-  redis_params = rspamd_parse_redis_server('greylist')
+  redis_params = rspamd_parse_redis_server(N)
   if not redis_params then
     rspamd_logger.infox(rspamd_config, 'no servers are specified, disabling module')
     rspamd_lua_utils.disable_module(N, "redis")
-- 
cgit v1.2.3