[Project] Preliminary support of HTTP proxies

Issue: #572

diff --git a/src/client/rspamc.c b/src/client/rspamc.c
index 08a267f1b841c2724a69cf291a747ff69e7282b8..2f572c449f99dcf8e4def3d85ead8cf1ddcd8190 100644 (file)
--- a/src/client/rspamc.c
+++ b/src/client/rspamc.c
@@ -1911,7 +1911,7 @@ main (gint argc, gchar **argv, gchar **env)
        http_config.kp_cache_size_server = 0;
        http_config.user_agent = user_agent;
        http_ctx = rspamd_http_context_create_config (&http_config,
-                       ev_base);
+                       ev_base, NULL);
 
        /* Ignore sigpipe */
        sigemptyset (&sigpipe_act.sa_mask);

diff --git a/src/controller.c b/src/controller.c
index ac1acae81cf2b2e44fbbd38de020366aada0cca4..7b48462ee9b00d5da08cc87b060de0875f4156ce 100644 (file)
--- a/src/controller.c
+++ b/src/controller.c
@@ -3776,7 +3776,8 @@ start_controller_worker (struct rspamd_worker *worker)
                        "password");
 
        /* Accept event */
-       ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base);
+       ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base,
+                       ctx->cfg->ups_ctx);
        ctx->http = rspamd_http_router_new (rspamd_controller_error_handler,
                        rspamd_controller_finish_handler, &ctx->io_tv,
                        ctx->static_files_dir, ctx->http_ctx);

diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c
index 96fb09c2b430078dfe1cbc706bbcd277d4b2d79e..e82e9062ac06c6ce7200429938f7ce62f06dbc97 100644 (file)
--- a/src/fuzzy_storage.c
+++ b/src/fuzzy_storage.c
@@ -2999,7 +2999,7 @@ start_fuzzy (struct rspamd_worker *worker)
                ctx->keypair_cache = rspamd_keypair_cache_new (ctx->keypair_cache_size);
        }
 
-       ctx->http_ctx = rspamd_http_context_create (cfg, ctx->ev_base);
+       ctx->http_ctx = rspamd_http_context_create (cfg, ctx->ev_base, ctx->cfg->ups_ctx);
 
        if (!ctx->collection_mode) {
                /*

diff --git a/src/libutil/http_connection.c b/src/libutil/http_connection.c
index 417784789cfc66a0473ddc91ffe7f3f0185da821..d782c2d13ac2ad1474ffea9e372edcb4a29eae97 100644 (file)
--- a/src/libutil/http_connection.c
+++ b/src/libutil/http_connection.c
@@ -49,6 +49,8 @@ enum rspamd_http_priv_flags {
        RSPAMD_HTTP_CONN_FLAG_RESETED = 1 << 2,
        RSPAMD_HTTP_CONN_FLAG_TOO_LARGE = 1 << 3,
        RSPAMD_HTTP_CONN_FLAG_ENCRYPTION_NEEDED = 1 << 4,
+       RSPAMD_HTTP_CONN_FLAG_PROXY = 1 << 5,
+       RSPAMD_HTTP_CONN_FLAG_PROXY_REQUEST = 1 << 6,
 };
 
 #define IS_CONN_ENCRYPTED(c) ((c)->flags & RSPAMD_HTTP_CONN_FLAG_ENCRYPTED)

diff --git a/src/libutil/http_context.c b/src/libutil/http_context.c
index 9182285a31f9d971daaae7a19aca9598b31dc7ef..18c89c6bcee9dfc8fe56bfb294db2aa8edb1ea39 100644 (file)
--- a/src/libutil/http_context.c
+++ b/src/libutil/http_context.c
@@ -14,12 +14,14 @@
  * limitations under the License.
  */
 
+#include <contrib/http-parser/http_parser.h>
 #include "http_context.h"
 #include "http_private.h"
 #include "keypair.h"
 #include "keypairs_cache.h"
 #include "cfg_file.h"
 #include "contrib/libottery/ottery.h"
+#include "contrib/http-parser/http_parser.h"
 #include "rspamd.h"
 
 INIT_LOG_MODULE(http_context)
@@ -85,7 +87,8 @@ rspamd_http_context_client_rotate_ev (gint fd, short what, void *arg)
 
 static struct rspamd_http_context*
 rspamd_http_context_new_default (struct rspamd_config *cfg,
-                                                                struct event_base *ev_base)
+                                                                struct event_base *ev_base,
+                                                                struct upstream_ctx *ups_ctx)
 {
        struct rspamd_http_context *ctx;
 
@@ -100,6 +103,7 @@ rspamd_http_context_new_default (struct rspamd_config *cfg,
        ctx->config.client_key_rotate_time = default_rotate_time;
        ctx->config.user_agent = default_user_agent;
        ctx->config.keepalive_interval = default_keepalive_interval;
+       ctx->ups_ctx = ups_ctx;
 
        if (cfg) {
                ctx->ssl_ctx = cfg->libs_ctx->ssl_ctx;
@@ -117,9 +121,63 @@ rspamd_http_context_new_default (struct rspamd_config *cfg,
        return ctx;
 }
 
+static void
+rspamd_http_context_parse_proxy (struct rspamd_http_context *ctx,
+                                                                const gchar *name,
+                                                                struct upstream_list **pls)
+{
+       struct http_parser_url u;
+       struct upstream_list *uls;
+
+       if (!ctx->ups_ctx) {
+               msg_err ("cannot parse http_proxy %s - upstreams context is udefined", name);
+               return;
+       }
+
+       memset (&u, 0, sizeof (u));
+
+       if (http_parser_parse_url (name, strlen (name), 1, &u) == 0) {
+               if (!(u.field_set & (1u << UF_HOST)) || u.port == 0) {
+                       msg_err ("cannot parse http(s) proxy %s - invalid host or port", name);
+
+                       return;
+               }
+
+               uls = rspamd_upstreams_create (ctx->ups_ctx);
+
+               if (!rspamd_upstreams_parse_line_len (uls,
+                               name + u.field_data[UF_HOST].off,
+                               u.field_data[UF_HOST].len, u.port, NULL)) {
+                       msg_err ("cannot parse http(s) proxy %s - invalid data", name);
+
+                       rspamd_upstreams_destroy (uls);
+               }
+               else {
+                       *pls = uls;
+                       msg_info ("set http(s) proxy to %s", name);
+               }
+       }
+       else {
+               uls = rspamd_upstreams_create (ctx->ups_ctx);
+
+               if (!rspamd_upstreams_parse_line (uls,
+                               name, 3128, NULL)) {
+                       msg_err ("cannot parse http(s) proxy %s - invalid data", name);
+
+                       rspamd_upstreams_destroy (uls);
+               }
+               else {
+                       *pls = uls;
+                       msg_info ("set http(s) proxy to %s", name);
+               }
+       }
+}
+
 static void
 rspamd_http_context_init (struct rspamd_http_context *ctx)
 {
+
+
        if (ctx->config.kp_cache_size_client > 0) {
                ctx->client_kp_cache = rspamd_keypair_cache_new (ctx->config.kp_cache_size_client);
        }
@@ -140,17 +198,28 @@ rspamd_http_context_init (struct rspamd_http_context *ctx)
                event_add (&ctx->client_rotate_ev, &tv);
        }
 
+       if (ctx->config.http_proxy) {
+               rspamd_http_context_parse_proxy (ctx, ctx->config.http_proxy,
+                               &ctx->http_proxies);
+       }
+
+       if (ctx->config.https_proxy) {
+               rspamd_http_context_parse_proxy (ctx, ctx->config.https_proxy,
+                               &ctx->https_proxies);
+       }
+
        default_ctx = ctx;
 }
 
 struct rspamd_http_context*
 rspamd_http_context_create (struct rspamd_config *cfg,
-                                                       struct event_base *ev_base)
+                                                       struct event_base *ev_base,
+                                                       struct upstream_ctx *ups_ctx)
 {
        struct rspamd_http_context *ctx;
        const ucl_object_t *http_obj;
 
-       ctx = rspamd_http_context_new_default (cfg, ev_base);
+       ctx = rspamd_http_context_new_default (cfg, ev_base, ups_ctx);
        http_obj = ucl_object_lookup (cfg->rcl_obj, "http");
 
        if (http_obj) {
@@ -194,6 +263,20 @@ rspamd_http_context_create (struct rspamd_config *cfg,
                        if (keepalive_interval) {
                                ctx->config.keepalive_interval = ucl_object_todouble (keepalive_interval);
                        }
+
+                       const ucl_object_t *http_proxy;
+                       http_proxy = ucl_object_lookup (client_obj, "http_proxy");
+
+                       if (http_proxy) {
+                               ctx->config.http_proxy = ucl_object_tostring (http_proxy);
+                       }
+
+                       const ucl_object_t *https_proxy;
+                       https_proxy = ucl_object_lookup (client_obj, "https_proxy");
+
+                       if (https_proxy) {
+                               ctx->config.https_proxy = ucl_object_tostring (https_proxy);
+                       }
                }
 
                server_obj = ucl_object_lookup (http_obj, "server");
@@ -262,11 +345,12 @@ rspamd_http_context_free (struct rspamd_http_context *ctx)
 
 struct rspamd_http_context*
 rspamd_http_context_create_config (struct rspamd_http_context_cfg *cfg,
-               struct event_base *ev_base)
+                                                                  struct event_base *ev_base,
+                                                                  struct upstream_ctx *ups_ctx)
 {
        struct rspamd_http_context *ctx;
 
-       ctx = rspamd_http_context_new_default (NULL, ev_base);
+       ctx = rspamd_http_context_new_default (NULL, ev_base, ups_ctx);
        memcpy (&ctx->config, cfg, sizeof (*cfg));
        rspamd_http_context_init (ctx);
 

diff --git a/src/libutil/http_context.h b/src/libutil/http_context.h
index 74e5c69a643eb08701378155c3a3ca69f0cd65a2..6abd666516ff735047e5cc711b42c05ebacacbde 100644 (file)
--- a/src/libutil/http_context.h
+++ b/src/libutil/http_context.h
@@ -26,6 +26,7 @@
 struct rspamd_http_context;
 struct rspamd_config;
 struct rspamd_http_message;
+struct upstream_ctx;
 
 struct rspamd_http_context_cfg {
        guint kp_cache_size_client;
@@ -34,6 +35,8 @@ struct rspamd_http_context_cfg {
        gdouble keepalive_interval;
        gdouble client_key_rotate_time;
        const gchar *user_agent;
+       const gchar *http_proxy;
+       const gchar *https_proxy;
 };
 
 /**
@@ -43,11 +46,12 @@ struct rspamd_http_context_cfg {
  * @return new context used for both client and server HTTP connections
  */
 struct rspamd_http_context* rspamd_http_context_create (struct rspamd_config *cfg,
-               struct event_base *ev_base);
+               struct event_base *ev_base, struct upstream_ctx *ctx);
 
 struct rspamd_http_context* rspamd_http_context_create_config (
                struct rspamd_http_context_cfg *cfg,
-               struct event_base *ev_base);
+               struct event_base *ev_base,
+               struct upstream_ctx *ctx);
 /**
  * Destroys context
  * @param ctx

diff --git a/src/libutil/http_private.h b/src/libutil/http_private.h
index dd3d0c6a930bb21e6a254f4c430a063918356766..dd4ca3435b2882397213bde33f3b4d8bca9bdf57 100644 (file)
--- a/src/libutil/http_private.h
+++ b/src/libutil/http_private.h
@@ -22,6 +22,7 @@
 #include "keypair.h"
 #include "keypairs_cache.h"
 #include "ref.h"
+#include "upstream.h"
 #include "khash.h"
 #define HASH_CASELESS
 #include "uthash_strcase.h"
@@ -95,6 +96,9 @@ struct rspamd_http_context {
        struct rspamd_keypair_cache *client_kp_cache;
        struct rspamd_cryptobox_keypair *client_kp;
        struct rspamd_keypair_cache *server_kp_cache;
+       struct upstream_ctx *ups_ctx;
+       struct upstream_list *http_proxies;
+       struct upstream_list *https_proxies;
        gpointer ssl_ctx;
        gpointer ssl_ctx_noverify;
        struct event_base *ev_base;

diff --git a/src/libutil/upstream.c b/src/libutil/upstream.c
index 64d5291fa85ec1a08ca77462168b31675305a07e..3a2b803b4fd212361cf11ffc29a60313ea4498a7 100644 (file)
--- a/src/libutil/upstream.c
+++ b/src/libutil/upstream.c
@@ -788,51 +788,45 @@ rspamd_upstream_add_addr (struct upstream *up, rspamd_inet_addr_t *addr)
        return TRUE;
 }
 
+#define LEN_CHECK_STARTS_WITH(s, len, lit) \
+       ((len) >= sizeof(lit) - 1 && g_ascii_strncasecmp ((s), (lit), sizeof(lit) - 1) == 0)
 gboolean
-rspamd_upstreams_parse_line (struct upstream_list *ups,
-               const gchar *str, guint16 def_port, void *data)
+rspamd_upstreams_parse_line_len (struct upstream_list *ups,
+               const gchar *str, gsize len, guint16 def_port, void *data)
 {
-       const gchar *end = str + strlen (str), *p = str;
+       const gchar *end = str + len, *p = str;
        const gchar *separators = ";, \n\r\t";
        gchar *tmp;
-       guint len;
+       guint span_len;
        gboolean ret = FALSE;
 
-       if (g_ascii_strncasecmp (p, "random:", sizeof ("random:") - 1) == 0) {
+       if (LEN_CHECK_STARTS_WITH(p, len, "random:")) {
                ups->rot_alg = RSPAMD_UPSTREAM_RANDOM;
                p += sizeof ("random:") - 1;
        }
-       else if (g_ascii_strncasecmp (p,
-                       "master-slave:",
-                       sizeof ("master-slave:") - 1) == 0) {
+       else if (LEN_CHECK_STARTS_WITH(p, len, "master-slave:")) {
                ups->rot_alg = RSPAMD_UPSTREAM_MASTER_SLAVE;
                p += sizeof ("master-slave:") - 1;
        }
-       else if (g_ascii_strncasecmp (p,
-                       "round-robin:",
-                       sizeof ("round-robin:") - 1) == 0) {
+       else if (LEN_CHECK_STARTS_WITH(p, len, "round-robin:")) {
                ups->rot_alg = RSPAMD_UPSTREAM_ROUND_ROBIN;
                p += sizeof ("round-robin:") - 1;
        }
-       else if (g_ascii_strncasecmp (p,
-                       "hash:",
-                       sizeof ("hash:") - 1) == 0) {
+       else if (LEN_CHECK_STARTS_WITH(p, len, "hash:")) {
                ups->rot_alg = RSPAMD_UPSTREAM_HASHED;
                p += sizeof ("hash:") - 1;
        }
-       else if (g_ascii_strncasecmp (p,
-                       "sequential:",
-                       sizeof ("sequential:") - 1) == 0) {
+       else if (LEN_CHECK_STARTS_WITH(p, len, "sequential:")) {
                ups->rot_alg = RSPAMD_UPSTREAM_SEQUENTIAL;
                p += sizeof ("sequential:") - 1;
        }
 
        while (p < end) {
-               len = strcspn (p, separators);
+               span_len = rspamd_memcspn (p, separators, end - p);
 
-               if (len > 0) {
-                       tmp = g_malloc (len + 1);
-                       rspamd_strlcpy (tmp, p, len + 1);
+               if (span_len > 0) {
+                       tmp = g_malloc (span_len + 1);
+                       rspamd_strlcpy (tmp, p, span_len + 1);
 
                        if (rspamd_upstreams_add_upstream (ups, tmp, def_port,
                                        RSPAMD_UPSTREAM_PARSE_DEFAULT,
@@ -843,14 +837,26 @@ rspamd_upstreams_parse_line (struct upstream_list *ups,
                        g_free (tmp);
                }
 
-               p += len;
+               p += span_len;
                /* Skip separators */
-               p += strspn (p, separators);
+               if (p < end) {
+                       p += rspamd_memspn (p, separators, end - p);
+               }
        }
 
        return ret;
 }
 
+#undef LEN_CHECK_STARTS_WITH
+
+gboolean
+rspamd_upstreams_parse_line (struct upstream_list *ups,
+                                                        const gchar *str, guint16 def_port, void *data)
+{
+       return rspamd_upstreams_parse_line_len (ups, str, strlen (str),
+                       def_port, data);
+}
+
 gboolean
 rspamd_upstreams_from_ucl (struct upstream_list *ups,
                const ucl_object_t *in, guint16 def_port, void *data)

diff --git a/src/libutil/upstream.h b/src/libutil/upstream.h
index 4db9627656d72c5257be1ded77bb895d1c01bd51..75d840ce23cd63712ca0f8a57275087863847788 100644 (file)
--- a/src/libutil/upstream.h
+++ b/src/libutil/upstream.h
@@ -157,6 +157,10 @@ gboolean rspamd_upstreams_parse_line (struct upstream_list *ups,
                const gchar *str, guint16 def_port, void *data);
 
 
+gboolean rspamd_upstreams_parse_line_len (struct upstream_list *ups,
+                                                                                 const gchar *str, gsize len,
+                                                                                 guint16 def_port,
+                                                                                 void *data);
 /**
  * Parse upstreams list from the UCL object
  * @param ups

diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c
index 63a7dd544fb7aa4e0e3b72016cc87b611a6e1c03..3f1990b7b6e40c55d0e1ab8baa6fd6e74d2385f9 100644 (file)
--- a/src/plugins/surbl.c
+++ b/src/plugins/surbl.c
@@ -2187,9 +2187,9 @@ surbl_is_redirector_handler (lua_State *L)
 
        task = lua_check_task (L, 1);
        url = luaL_checklstring (L, 2, &len);
-       surbl_module_ctx = surbl_get_context (task->cfg);
 
        if (task && url) {
+               surbl_module_ctx = surbl_get_context (task->cfg);
                url_cpy = rspamd_mempool_alloc (task->task_pool, len);
                memcpy (url_cpy, url, len);
 

diff --git a/src/rspamadm/rspamadm.c b/src/rspamadm/rspamadm.c
index c0bb4bc7279b0a5dda7421840c20c2c2d892e948..c49853ef7e5ae64e40c7a3c0d636a638b97940de 100644 (file)
--- a/src/rspamadm/rspamadm.c
+++ b/src/rspamadm/rspamadm.c
@@ -436,7 +436,8 @@ main (gint argc, gchar **argv, gchar **env)
        (void) dns_resolver_init (rspamd_main->logger,
                        rspamd_main->ev_base,
                        cfg);
-       rspamd_main->http_ctx = rspamd_http_context_create (cfg, rspamd_main->ev_base);
+       rspamd_main->http_ctx = rspamd_http_context_create (cfg, rspamd_main->ev_base,
+                       NULL);
 
        g_log_set_default_handler (rspamd_glib_log_function, rspamd_main->logger);
        g_set_printerr_handler (rspamd_glib_printerr_function);

diff --git a/src/rspamd.c b/src/rspamd.c
index 27ba5e032aa5db089490a48819e395fb6240f0ee..142915df90d7ab78cb39b74f6363e638d337a6e7 100644 (file)
--- a/src/rspamd.c
+++ b/src/rspamd.c
@@ -1490,7 +1490,7 @@ main (gint argc, gchar **argv, gchar **env)
        rspamd_mempool_unlock_mutex (rspamd_main->start_mtx);
 
        rspamd_main->http_ctx = rspamd_http_context_create (rspamd_main->cfg,
-                       ev_base);
+                       ev_base, rspamd_main->cfg->ups_ctx);
 
        if (control_fd != -1) {
                msg_info_main ("listening for control commands on %s",

diff --git a/src/rspamd_proxy.c b/src/rspamd_proxy.c
index e834266731fbe6a6d365af7f093bbea8cc0711d5..446552b811ba8f2fca20080f2c45f4c176473ec6 100644 (file)
--- a/src/rspamd_proxy.c
+++ b/src/rspamd_proxy.c
@@ -2177,7 +2177,8 @@ start_rspamd_proxy (struct rspamd_worker *worker)
        rspamd_upstreams_library_config (worker->srv->cfg, ctx->cfg->ups_ctx,
                        ctx->ev_base, ctx->resolver->r);
 
-       ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base);
+       ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base,
+                       ctx->cfg->ups_ctx);
 
        if (ctx->has_self_scan) {
                /* Additional initialisation needed */

diff --git a/src/worker.c b/src/worker.c
index 77614ec1390d3dbef19d343ba2dcd6b477447549..40e3d07f9a362827f22f558b62cdd312d2737f2e 100644 (file)
--- a/src/worker.c
+++ b/src/worker.c
@@ -692,7 +692,8 @@ start_worker (struct rspamd_worker *worker)
        rspamd_upstreams_library_config (worker->srv->cfg, ctx->cfg->ups_ctx,
                        ctx->ev_base, ctx->resolver->r);
 
-       ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base);
+       ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base,
+                       ctx->cfg->ups_ctx);
        rspamd_worker_init_scanner (worker, ctx->ev_base, ctx->resolver,
                        &ctx->lang_det);
        rspamd_lua_run_postloads (ctx->cfg->lua_state, ctx->cfg, ctx->ev_base,