[Project] Preliminary support of HTTP proxies

Issue: #572

14 files changed, 146 insertions, 38 deletions

diff --git a/src/client/rspamc.c b/src/client/rspamc.c
index 08a267f1b..2f572c449 100644
--- a/src/client/rspamc.c
+++ b/src/client/rspamc.c
@@ -1911,7 +1911,7 @@ main (gint argc, gchar **argv, gchar **env)
 	http_config.kp_cache_size_server = 0;
 	http_config.user_agent = user_agent;
 	http_ctx = rspamd_http_context_create_config (&http_config,
-			ev_base);
+			ev_base, NULL);
 
 	/* Ignore sigpipe */
 	sigemptyset (&sigpipe_act.sa_mask);
diff --git a/src/controller.c b/src/controller.c
index ac1acae81..7b48462ee 100644
--- a/src/controller.c
+++ b/src/controller.c
@@ -3776,7 +3776,8 @@ start_controller_worker (struct rspamd_worker *worker)
 			"password");
 
 	/* Accept event */
-	ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base);
+	ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base,
+			ctx->cfg->ups_ctx);
 	ctx->http = rspamd_http_router_new (rspamd_controller_error_handler,
 			rspamd_controller_finish_handler, &ctx->io_tv,
 			ctx->static_files_dir, ctx->http_ctx);
diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c
index 96fb09c2b..e82e9062a 100644
--- a/src/fuzzy_storage.c
+++ b/src/fuzzy_storage.c
@@ -2999,7 +2999,7 @@ start_fuzzy (struct rspamd_worker *worker)
 		ctx->keypair_cache = rspamd_keypair_cache_new (ctx->keypair_cache_size);
 	}
 
-	ctx->http_ctx = rspamd_http_context_create (cfg, ctx->ev_base);
+	ctx->http_ctx = rspamd_http_context_create (cfg, ctx->ev_base, ctx->cfg->ups_ctx);
 
 	if (!ctx->collection_mode) {
 		/*
diff --git a/src/libutil/http_connection.c b/src/libutil/http_connection.c
index 417784789..d782c2d13 100644
--- a/src/libutil/http_connection.c
+++ b/src/libutil/http_connection.c
@@ -49,6 +49,8 @@ enum rspamd_http_priv_flags {
 	RSPAMD_HTTP_CONN_FLAG_RESETED = 1 << 2,
 	RSPAMD_HTTP_CONN_FLAG_TOO_LARGE = 1 << 3,
 	RSPAMD_HTTP_CONN_FLAG_ENCRYPTION_NEEDED = 1 << 4,
+	RSPAMD_HTTP_CONN_FLAG_PROXY = 1 << 5,
+	RSPAMD_HTTP_CONN_FLAG_PROXY_REQUEST = 1 << 6,
 };
 
 #define IS_CONN_ENCRYPTED(c) ((c)->flags & RSPAMD_HTTP_CONN_FLAG_ENCRYPTED)
diff --git a/src/libutil/http_context.c b/src/libutil/http_context.c
index 9182285a3..18c89c6bc 100644
--- a/src/libutil/http_context.c
+++ b/src/libutil/http_context.c
@@ -14,12 +14,14 @@
  * limitations under the License.
  */
 
+#include <contrib/http-parser/http_parser.h>
 #include "http_context.h"
 #include "http_private.h"
 #include "keypair.h"
 #include "keypairs_cache.h"
 #include "cfg_file.h"
 #include "contrib/libottery/ottery.h"
+#include "contrib/http-parser/http_parser.h"
 #include "rspamd.h"
 
 INIT_LOG_MODULE(http_context)
@@ -85,7 +87,8 @@ rspamd_http_context_client_rotate_ev (gint fd, short what, void *arg)
 
 static struct rspamd_http_context*
 rspamd_http_context_new_default (struct rspamd_config *cfg,
-								 struct event_base *ev_base)
+								 struct event_base *ev_base,
+								 struct upstream_ctx *ups_ctx)
 {
 	struct rspamd_http_context *ctx;
 
@@ -100,6 +103,7 @@ rspamd_http_context_new_default (struct rspamd_config *cfg,
 	ctx->config.client_key_rotate_time = default_rotate_time;
 	ctx->config.user_agent = default_user_agent;
 	ctx->config.keepalive_interval = default_keepalive_interval;
+	ctx->ups_ctx = ups_ctx;
 
 	if (cfg) {
 		ctx->ssl_ctx = cfg->libs_ctx->ssl_ctx;
@@ -118,8 +122,62 @@ rspamd_http_context_new_default (struct rspamd_config *cfg,
 }
 
 static void
+rspamd_http_context_parse_proxy (struct rspamd_http_context *ctx,
+								 const gchar *name,
+								 struct upstream_list **pls)
+{
+	struct http_parser_url u;
+	struct upstream_list *uls;
+
+	if (!ctx->ups_ctx) {
+		msg_err ("cannot parse http_proxy %s - upstreams context is udefined", name);
+		return;
+	}
+
+	memset (&u, 0, sizeof (u));
+
+	if (http_parser_parse_url (name, strlen (name), 1, &u) == 0) {
+		if (!(u.field_set & (1u << UF_HOST)) || u.port == 0) {
+			msg_err ("cannot parse http(s) proxy %s - invalid host or port", name);
+
+			return;
+		}
+
+		uls = rspamd_upstreams_create (ctx->ups_ctx);
+
+		if (!rspamd_upstreams_parse_line_len (uls,
+				name + u.field_data[UF_HOST].off,
+				u.field_data[UF_HOST].len, u.port, NULL)) {
+			msg_err ("cannot parse http(s) proxy %s - invalid data", name);
+
+			rspamd_upstreams_destroy (uls);
+		}
+		else {
+			*pls = uls;
+			msg_info ("set http(s) proxy to %s", name);
+		}
+	}
+	else {
+		uls = rspamd_upstreams_create (ctx->ups_ctx);
+
+		if (!rspamd_upstreams_parse_line (uls,
+				name, 3128, NULL)) {
+			msg_err ("cannot parse http(s) proxy %s - invalid data", name);
+
+			rspamd_upstreams_destroy (uls);
+		}
+		else {
+			*pls = uls;
+			msg_info ("set http(s) proxy to %s", name);
+		}
+	}
+}
+
+static void
 rspamd_http_context_init (struct rspamd_http_context *ctx)
 {
+
+
 	if (ctx->config.kp_cache_size_client > 0) {
 		ctx->client_kp_cache = rspamd_keypair_cache_new (ctx->config.kp_cache_size_client);
 	}
@@ -140,17 +198,28 @@ rspamd_http_context_init (struct rspamd_http_context *ctx)
 		event_add (&ctx->client_rotate_ev, &tv);
 	}
 
+	if (ctx->config.http_proxy) {
+		rspamd_http_context_parse_proxy (ctx, ctx->config.http_proxy,
+				&ctx->http_proxies);
+	}
+
+	if (ctx->config.https_proxy) {
+		rspamd_http_context_parse_proxy (ctx, ctx->config.https_proxy,
+				&ctx->https_proxies);
+	}
+
 	default_ctx = ctx;
 }
 
 struct rspamd_http_context*
 rspamd_http_context_create (struct rspamd_config *cfg,
-							struct event_base *ev_base)
+							struct event_base *ev_base,
+							struct upstream_ctx *ups_ctx)
 {
 	struct rspamd_http_context *ctx;
 	const ucl_object_t *http_obj;
 
-	ctx = rspamd_http_context_new_default (cfg, ev_base);
+	ctx = rspamd_http_context_new_default (cfg, ev_base, ups_ctx);
 	http_obj = ucl_object_lookup (cfg->rcl_obj, "http");
 
 	if (http_obj) {
@@ -194,6 +263,20 @@ rspamd_http_context_create (struct rspamd_config *cfg,
 			if (keepalive_interval) {
 				ctx->config.keepalive_interval = ucl_object_todouble (keepalive_interval);
 			}
+
+			const ucl_object_t *http_proxy;
+			http_proxy = ucl_object_lookup (client_obj, "http_proxy");
+
+			if (http_proxy) {
+				ctx->config.http_proxy = ucl_object_tostring (http_proxy);
+			}
+
+			const ucl_object_t *https_proxy;
+			https_proxy = ucl_object_lookup (client_obj, "https_proxy");
+
+			if (https_proxy) {
+				ctx->config.https_proxy = ucl_object_tostring (https_proxy);
+			}
 		}
 
 		server_obj = ucl_object_lookup (http_obj, "server");
@@ -262,11 +345,12 @@ rspamd_http_context_free (struct rspamd_http_context *ctx)
 
 struct rspamd_http_context*
 rspamd_http_context_create_config (struct rspamd_http_context_cfg *cfg,
-		struct event_base *ev_base)
+								   struct event_base *ev_base,
+								   struct upstream_ctx *ups_ctx)
 {
 	struct rspamd_http_context *ctx;
 
-	ctx = rspamd_http_context_new_default (NULL, ev_base);
+	ctx = rspamd_http_context_new_default (NULL, ev_base, ups_ctx);
 	memcpy (&ctx->config, cfg, sizeof (*cfg));
 	rspamd_http_context_init (ctx);
 
diff --git a/src/libutil/http_context.h b/src/libutil/http_context.h
index 74e5c69a6..6abd66651 100644
--- a/src/libutil/http_context.h
+++ b/src/libutil/http_context.h
@@ -26,6 +26,7 @@
 struct rspamd_http_context;
 struct rspamd_config;
 struct rspamd_http_message;
+struct upstream_ctx;
 
 struct rspamd_http_context_cfg {
 	guint kp_cache_size_client;
@@ -34,6 +35,8 @@ struct rspamd_http_context_cfg {
 	gdouble keepalive_interval;
 	gdouble client_key_rotate_time;
 	const gchar *user_agent;
+	const gchar *http_proxy;
+	const gchar *https_proxy;
 };
 
 /**
@@ -43,11 +46,12 @@ struct rspamd_http_context_cfg {
  * @return new context used for both client and server HTTP connections
  */
 struct rspamd_http_context* rspamd_http_context_create (struct rspamd_config *cfg,
-		struct event_base *ev_base);
+		struct event_base *ev_base, struct upstream_ctx *ctx);
 
 struct rspamd_http_context* rspamd_http_context_create_config (
 		struct rspamd_http_context_cfg *cfg,
-		struct event_base *ev_base);
+		struct event_base *ev_base,
+		struct upstream_ctx *ctx);
 /**
  * Destroys context
  * @param ctx
diff --git a/src/libutil/http_private.h b/src/libutil/http_private.h
index dd3d0c6a9..dd4ca3435 100644
--- a/src/libutil/http_private.h
+++ b/src/libutil/http_private.h
@@ -22,6 +22,7 @@
 #include "keypair.h"
 #include "keypairs_cache.h"
 #include "ref.h"
+#include "upstream.h"
 #include "khash.h"
 #define HASH_CASELESS
 #include "uthash_strcase.h"
@@ -95,6 +96,9 @@ struct rspamd_http_context {
 	struct rspamd_keypair_cache *client_kp_cache;
 	struct rspamd_cryptobox_keypair *client_kp;
 	struct rspamd_keypair_cache *server_kp_cache;
+	struct upstream_ctx *ups_ctx;
+	struct upstream_list *http_proxies;
+	struct upstream_list *https_proxies;
 	gpointer ssl_ctx;
 	gpointer ssl_ctx_noverify;
 	struct event_base *ev_base;
diff --git a/src/libutil/upstream.c b/src/libutil/upstream.c
index 64d5291fa..3a2b803b4 100644
--- a/src/libutil/upstream.c
+++ b/src/libutil/upstream.c
@@ -788,51 +788,45 @@ rspamd_upstream_add_addr (struct upstream *up, rspamd_inet_addr_t *addr)
 	return TRUE;
 }
 
+#define LEN_CHECK_STARTS_WITH(s, len, lit) \
+	((len) >= sizeof(lit) - 1 && g_ascii_strncasecmp ((s), (lit), sizeof(lit) - 1) == 0)
 gboolean
-rspamd_upstreams_parse_line (struct upstream_list *ups,
-		const gchar *str, guint16 def_port, void *data)
+rspamd_upstreams_parse_line_len (struct upstream_list *ups,
+		const gchar *str, gsize len, guint16 def_port, void *data)
 {
-	const gchar *end = str + strlen (str), *p = str;
+	const gchar *end = str + len, *p = str;
 	const gchar *separators = ";, \n\r\t";
 	gchar *tmp;
-	guint len;
+	guint span_len;
 	gboolean ret = FALSE;
 
-	if (g_ascii_strncasecmp (p, "random:", sizeof ("random:") - 1) == 0) {
+	if (LEN_CHECK_STARTS_WITH(p, len, "random:")) {
 		ups->rot_alg = RSPAMD_UPSTREAM_RANDOM;
 		p += sizeof ("random:") - 1;
 	}
-	else if (g_ascii_strncasecmp (p,
-			"master-slave:",
-			sizeof ("master-slave:") - 1) == 0) {
+	else if (LEN_CHECK_STARTS_WITH(p, len, "master-slave:")) {
 		ups->rot_alg = RSPAMD_UPSTREAM_MASTER_SLAVE;
 		p += sizeof ("master-slave:") - 1;
 	}
-	else if (g_ascii_strncasecmp (p,
-			"round-robin:",
-			sizeof ("round-robin:") - 1) == 0) {
+	else if (LEN_CHECK_STARTS_WITH(p, len, "round-robin:")) {
 		ups->rot_alg = RSPAMD_UPSTREAM_ROUND_ROBIN;
 		p += sizeof ("round-robin:") - 1;
 	}
-	else if (g_ascii_strncasecmp (p,
-			"hash:",
-			sizeof ("hash:") - 1) == 0) {
+	else if (LEN_CHECK_STARTS_WITH(p, len, "hash:")) {
 		ups->rot_alg = RSPAMD_UPSTREAM_HASHED;
 		p += sizeof ("hash:") - 1;
 	}
-	else if (g_ascii_strncasecmp (p,
-			"sequential:",
-			sizeof ("sequential:") - 1) == 0) {
+	else if (LEN_CHECK_STARTS_WITH(p, len, "sequential:")) {
 		ups->rot_alg = RSPAMD_UPSTREAM_SEQUENTIAL;
 		p += sizeof ("sequential:") - 1;
 	}
 
 	while (p < end) {
-		len = strcspn (p, separators);
+		span_len = rspamd_memcspn (p, separators, end - p);
 
-		if (len > 0) {
-			tmp = g_malloc (len + 1);
-			rspamd_strlcpy (tmp, p, len + 1);
+		if (span_len > 0) {
+			tmp = g_malloc (span_len + 1);
+			rspamd_strlcpy (tmp, p, span_len + 1);
 
 			if (rspamd_upstreams_add_upstream (ups, tmp, def_port,
 					RSPAMD_UPSTREAM_PARSE_DEFAULT,
@@ -843,14 +837,26 @@ rspamd_upstreams_parse_line (struct upstream_list *ups,
 			g_free (tmp);
 		}
 
-		p += len;
+		p += span_len;
 		/* Skip separators */
-		p += strspn (p, separators);
+		if (p < end) {
+			p += rspamd_memspn (p, separators, end - p);
+		}
 	}
 
 	return ret;
 }
 
+#undef LEN_CHECK_STARTS_WITH
+
+gboolean
+rspamd_upstreams_parse_line (struct upstream_list *ups,
+							 const gchar *str, guint16 def_port, void *data)
+{
+	return rspamd_upstreams_parse_line_len (ups, str, strlen (str),
+			def_port, data);
+}
+
 gboolean
 rspamd_upstreams_from_ucl (struct upstream_list *ups,
 		const ucl_object_t *in, guint16 def_port, void *data)
diff --git a/src/libutil/upstream.h b/src/libutil/upstream.h
index 4db962765..75d840ce2 100644
--- a/src/libutil/upstream.h
+++ b/src/libutil/upstream.h
@@ -157,6 +157,10 @@ gboolean rspamd_upstreams_parse_line (struct upstream_list *ups,
 		const gchar *str, guint16 def_port, void *data);
 
 
+gboolean rspamd_upstreams_parse_line_len (struct upstream_list *ups,
+										  const gchar *str, gsize len,
+										  guint16 def_port,
+										  void *data);
 /**
  * Parse upstreams list from the UCL object
  * @param ups
diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c
index 63a7dd544..3f1990b7b 100644
--- a/src/plugins/surbl.c
+++ b/src/plugins/surbl.c
@@ -2187,9 +2187,9 @@ surbl_is_redirector_handler (lua_State *L)
 
 	task = lua_check_task (L, 1);
 	url = luaL_checklstring (L, 2, &len);
-	surbl_module_ctx = surbl_get_context (task->cfg);
 
 	if (task && url) {
+		surbl_module_ctx = surbl_get_context (task->cfg);
 		url_cpy = rspamd_mempool_alloc (task->task_pool, len);
 		memcpy (url_cpy, url, len);
 
diff --git a/src/rspamadm/rspamadm.c b/src/rspamadm/rspamadm.c
index c0bb4bc72..c49853ef7 100644
--- a/src/rspamadm/rspamadm.c
+++ b/src/rspamadm/rspamadm.c
@@ -436,7 +436,8 @@ main (gint argc, gchar **argv, gchar **env)
 	(void) dns_resolver_init (rspamd_main->logger,
 			rspamd_main->ev_base,
 			cfg);
-	rspamd_main->http_ctx = rspamd_http_context_create (cfg, rspamd_main->ev_base);
+	rspamd_main->http_ctx = rspamd_http_context_create (cfg, rspamd_main->ev_base,
+			NULL);
 
 	g_log_set_default_handler (rspamd_glib_log_function, rspamd_main->logger);
 	g_set_printerr_handler (rspamd_glib_printerr_function);
diff --git a/src/rspamd.c b/src/rspamd.c
index 27ba5e032..142915df9 100644
--- a/src/rspamd.c
+++ b/src/rspamd.c
@@ -1490,7 +1490,7 @@ main (gint argc, gchar **argv, gchar **env)
 	rspamd_mempool_unlock_mutex (rspamd_main->start_mtx);
 
 	rspamd_main->http_ctx = rspamd_http_context_create (rspamd_main->cfg,
-			ev_base);
+			ev_base, rspamd_main->cfg->ups_ctx);
 
 	if (control_fd != -1) {
 		msg_info_main ("listening for control commands on %s",
diff --git a/src/rspamd_proxy.c b/src/rspamd_proxy.c
index e83426673..446552b81 100644
--- a/src/rspamd_proxy.c
+++ b/src/rspamd_proxy.c
@@ -2177,7 +2177,8 @@ start_rspamd_proxy (struct rspamd_worker *worker)
 	rspamd_upstreams_library_config (worker->srv->cfg, ctx->cfg->ups_ctx,
 			ctx->ev_base, ctx->resolver->r);
 
-	ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base);
+	ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base,
+			ctx->cfg->ups_ctx);
 
 	if (ctx->has_self_scan) {
 		/* Additional initialisation needed */
diff --git a/src/worker.c b/src/worker.c
index 77614ec13..40e3d07f9 100644
--- a/src/worker.c
+++ b/src/worker.c
@@ -692,7 +692,8 @@ start_worker (struct rspamd_worker *worker)
 	rspamd_upstreams_library_config (worker->srv->cfg, ctx->cfg->ups_ctx,
 			ctx->ev_base, ctx->resolver->r);
 
-	ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base);
+	ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base,
+			ctx->cfg->ups_ctx);
 	rspamd_worker_init_scanner (worker, ctx->ev_base, ctx->resolver,
 			&ctx->lang_det);
 	rspamd_lua_run_postloads (ctx->cfg->lua_state, ctx->cfg, ctx->ev_base,