From 3f00d108df7c99edd5cb45e34d232a80e159caa3 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Tue, 19 Feb 2019 17:57:13 +0000 Subject: [PATCH] [Project] Adopt plugins, workers, rspamadm and rspamc for http_context --- src/client/rspamc.c | 12 +++++- src/client/rspamdclient.c | 13 +++---- src/client/rspamdclient.h | 12 +++--- src/controller.c | 9 +++-- src/fuzzy_storage.c | 27 +++++++------- src/plugins/surbl.c | 5 +-- src/rspamadm/control.c | 8 ++-- src/rspamadm/lua_repl.c | 5 ++- src/rspamadm/rspamadm.c | 1 + src/rspamd.h | 1 + src/rspamd_proxy.c | 78 ++++++++------------------------------- src/worker.c | 13 +++---- src/worker_private.h | 2 +- 13 files changed, 77 insertions(+), 109 deletions(-) diff --git a/src/client/rspamc.c b/src/client/rspamc.c index 588496d46..d9af90b7f 100644 --- a/src/client/rspamc.c +++ b/src/client/rspamc.c @@ -67,6 +67,7 @@ static gchar *key = NULL; static gchar *user_agent = "rspamc"; static GList *children; static GPatternSpec **exclude_compiled = NULL; +static struct rspamd_http_context *http_ctx; static gint retcode = EXIT_SUCCESS; @@ -1668,7 +1669,7 @@ rspamc_process_input (struct event_base *ev_base, struct rspamc_command *cmd, } - conn = rspamd_client_init (ev_base, hostbuf, port, timeout, key); + conn = rspamd_client_init (http_ctx, ev_base, hostbuf, port, timeout, key); if (conn != NULL) { cbdata = g_malloc0 (sizeof (struct rspamc_callback_data)); @@ -1894,6 +1895,15 @@ main (gint argc, gchar **argv, gchar **env) rspamd_init_libs (); ev_base = event_base_new (); + struct rspamd_http_context_cfg http_config; + + memset (&http_config, 0, sizeof (http_config)); + http_config.kp_cache_size_client = 32; + http_config.kp_cache_size_server = 0; + http_config.user_agent = user_agent; + http_ctx = rspamd_http_context_create_config (&http_config, + ev_base); + /* Ignore sigpipe */ sigemptyset (&sigpipe_act.sa_mask); sigaddset (&sigpipe_act.sa_mask, SIGPIPE); diff --git a/src/client/rspamdclient.c b/src/client/rspamdclient.c index b1b550024..54cd490e6 100644 --- a/src/client/rspamdclient.c +++ b/src/client/rspamdclient.c @@ -241,8 +241,9 @@ rspamd_client_finish_handler (struct rspamd_http_connection *conn, } struct rspamd_client_connection * -rspamd_client_init (struct event_base *ev_base, const gchar *name, - guint16 port, gdouble timeout, const gchar *key) +rspamd_client_init (struct rspamd_http_context *http_ctx, + struct event_base *ev_base, const gchar *name, + guint16 port, gdouble timeout, const gchar *key) { struct rspamd_client_connection *conn; gint fd; @@ -256,14 +257,12 @@ rspamd_client_init (struct event_base *ev_base, const gchar *name, conn->ev_base = ev_base; conn->fd = fd; conn->req_sent = FALSE; - conn->keys_cache = rspamd_keypair_cache_new (32); - conn->http_conn = rspamd_http_connection_new (rspamd_client_body_handler, + conn->http_conn = rspamd_http_connection_new (http_ctx, + rspamd_client_body_handler, rspamd_client_error_handler, rspamd_client_finish_handler, 0, - RSPAMD_HTTP_CLIENT, - conn->keys_cache, - NULL); + RSPAMD_HTTP_CLIENT); conn->server_name = g_string_new (name); if (port != 0) { diff --git a/src/client/rspamdclient.h b/src/client/rspamdclient.h index 129ee39a1..c2a3c1886 100644 --- a/src/client/rspamdclient.h +++ b/src/client/rspamdclient.h @@ -47,6 +47,7 @@ typedef void (*rspamd_client_callback) ( gdouble send_time, GError *err); +struct rspamd_http_context; /** * Start rspamd worker or controller command * @param ev_base event base @@ -56,11 +57,12 @@ typedef void (*rspamd_client_callback) ( * @return */ struct rspamd_client_connection * rspamd_client_init ( - struct event_base *ev_base, - const gchar *name, - guint16 port, - gdouble timeout, - const gchar *key); + struct rspamd_http_context *http_ctx, + struct event_base *ev_base, + const gchar *name, + guint16 port, + gdouble timeout, + const gchar *key); /** * diff --git a/src/controller.c b/src/controller.c index b19ac3db7..a6214655a 100644 --- a/src/controller.c +++ b/src/controller.c @@ -150,6 +150,7 @@ struct rspamd_controller_worker_ctx { rspamd_ftok_t cached_password; rspamd_ftok_t cached_enable_password; /* HTTP server */ + struct rspamd_http_context *http_ctx; struct rspamd_http_connection_router *http; /* Server's start time */ time_t start_time; @@ -3707,7 +3708,6 @@ start_controller_worker (struct rspamd_worker *worker) GHashTableIter iter; gpointer key, value; guint i; - struct rspamd_keypair_cache *cache; struct timeval stv; const guint save_stats_interval = 60 * 1000; /* 1 minute */ gpointer m; @@ -3783,10 +3783,10 @@ start_controller_worker (struct rspamd_worker *worker) "password"); /* Accept event */ - cache = rspamd_keypair_cache_new (256); + ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base); ctx->http = rspamd_http_router_new (rspamd_controller_error_handler, - rspamd_controller_finish_handler, &ctx->io_tv, ctx->ev_base, - ctx->static_files_dir, cache); + rspamd_controller_finish_handler, &ctx->io_tv, + ctx->static_files_dir, ctx->http_ctx); /* Add callbacks for different methods */ rspamd_http_router_add_path (ctx->http, @@ -3949,6 +3949,7 @@ start_controller_worker (struct rspamd_worker *worker) g_hash_table_unref (ctx->plugins); g_hash_table_unref (ctx->custom_commands); + rspamd_http_context_free (ctx->http_ctx); REF_RELEASE (ctx->cfg); rspamd_log_close (worker->srv->logger, TRUE); diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c index d0a4f3602..a02db290b 100644 --- a/src/fuzzy_storage.c +++ b/src/fuzzy_storage.c @@ -173,6 +173,7 @@ struct rspamd_fuzzy_storage_ctx { struct rspamd_cryptobox_keypair *collection_keypair; struct rspamd_cryptobox_pubkey *collection_sign_key; gchar *collection_id_file; + struct rspamd_http_context *http_ctx; struct rspamd_keypair_cache *keypair_cache; rspamd_lru_hash_t *errors_ips; rspamd_lru_hash_t *ratelimit_buckets; @@ -616,13 +617,13 @@ rspamd_fuzzy_send_update_mirror (struct rspamd_fuzzy_storage_ctx *ctx, msg = rspamd_http_new_message (HTTP_REQUEST); rspamd_printf_fstring (&msg->url, "/update_v1/%s", m->name); - conn->http_conn = rspamd_http_connection_new (NULL, + conn->http_conn = rspamd_http_connection_new ( + ctx->http_ctx, + NULL, fuzzy_mirror_error_handler, fuzzy_mirror_finish_handler, RSPAMD_HTTP_CLIENT_SIMPLE, - RSPAMD_HTTP_CLIENT, - ctx->keypair_cache, - NULL); + RSPAMD_HTTP_CLIENT); rspamd_http_connection_set_key (conn->http_conn, ctx->sync_keypair); @@ -1994,13 +1995,13 @@ accept_fuzzy_mirror_socket (gint fd, short what, void *arg) session->name = rspamd_inet_address_to_string (addr); rspamd_random_hex (session->uid, sizeof (session->uid) - 1); session->uid[sizeof (session->uid) - 1] = '\0'; - http_conn = rspamd_http_connection_new (NULL, + http_conn = rspamd_http_connection_new ( + ctx->http_ctx, + NULL, rspamd_fuzzy_mirror_error_handler, rspamd_fuzzy_mirror_finish_handler, 0, - RSPAMD_HTTP_SERVER, - ctx->keypair_cache, - NULL); + RSPAMD_HTTP_SERVER); rspamd_http_connection_set_key (http_conn, ctx->sync_keypair); session->ctx = ctx; @@ -3004,6 +3005,8 @@ start_fuzzy (struct rspamd_worker *worker) ctx->keypair_cache = rspamd_keypair_cache_new (ctx->keypair_cache_size); } + ctx->http_ctx = rspamd_http_context_create (cfg, ctx->ev_base); + if (!ctx->collection_mode) { /* * Open DB and perform VACUUM @@ -3058,8 +3061,8 @@ start_fuzzy (struct rspamd_worker *worker) rspamd_fuzzy_collection_error_handler, rspamd_fuzzy_collection_finish_handler, &ctx->stat_tv, - ctx->ev_base, - NULL, ctx->keypair_cache); + NULL, + ctx->http_ctx); if (ctx->collection_keypair) { rspamd_http_router_set_key (ctx->collection_rt, @@ -3202,8 +3205,6 @@ start_fuzzy (struct rspamd_worker *worker) else if (worker->index == 0) { gint fd; - /* Steal keypairs cache... */ - ctx->collection_rt->cache = NULL; rspamd_http_router_free (ctx->collection_rt); /* Try to save collection id */ @@ -3240,8 +3241,8 @@ start_fuzzy (struct rspamd_worker *worker) rspamd_keypair_cache_destroy (ctx->keypair_cache); } + rspamd_http_context_free (ctx->http_ctx); REF_RELEASE (ctx->cfg); - rspamd_log_close (worker->srv->logger, TRUE); exit (EXIT_SUCCESS); diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c index 5949f5bb6..22088c2e5 100644 --- a/src/plugins/surbl.c +++ b/src/plugins/surbl.c @@ -1734,12 +1734,11 @@ register_redirector_call (struct rspamd_url *url, struct rspamd_task *task, param->url = url; param->task = task; param->conn = rspamd_http_connection_new (NULL, + NULL, surbl_redirector_error, surbl_redirector_finish, RSPAMD_HTTP_CLIENT_SIMPLE, - RSPAMD_HTTP_CLIENT, - NULL, - NULL); + RSPAMD_HTTP_CLIENT); param->ctx = surbl_module_ctx; msg = rspamd_http_new_message (HTTP_REQUEST); msg->url = rspamd_fstring_assign (msg->url, url->string, url->urllen); diff --git a/src/rspamadm/control.c b/src/rspamadm/control.c index 554392e63..c030907f6 100644 --- a/src/rspamadm/control.c +++ b/src/rspamadm/control.c @@ -234,13 +234,13 @@ rspamadm_control (gint argc, gchar **argv, const struct rspamadm_command *_cmd) exit (1); } - conn = rspamd_http_connection_new (NULL, + conn = rspamd_http_connection_new ( + NULL, /* Default context */ + NULL, rspamd_control_error_handler, rspamd_control_finish_handler, RSPAMD_HTTP_CLIENT_SIMPLE, - RSPAMD_HTTP_CLIENT, - NULL, - NULL); + RSPAMD_HTTP_CLIENT); msg = rspamd_http_new_message (HTTP_REQUEST); msg->url = rspamd_fstring_new_init (path, strlen (path)); double_to_tv (timeout, &tv); diff --git a/src/rspamadm/lua_repl.c b/src/rspamadm/lua_repl.c index 6248f2aa2..e91bd34fa 100644 --- a/src/rspamadm/lua_repl.c +++ b/src/rspamadm/lua_repl.c @@ -808,8 +808,9 @@ rspamadm_lua (gint argc, gchar **argv, const struct rspamadm_command *cmd) ctx = g_malloc0 (sizeof (*ctx)); http = rspamd_http_router_new (rspamadm_lua_error_handler, rspamadm_lua_finish_handler, - NULL, ev_base, - NULL, NULL); + NULL, + NULL, + rspamd_main->http_ctx); ctx->L = L; ctx->rt = http; rspamd_http_router_add_path (http, diff --git a/src/rspamadm/rspamadm.c b/src/rspamadm/rspamadm.c index ef52af1e3..c0bb4bc72 100644 --- a/src/rspamadm/rspamadm.c +++ b/src/rspamadm/rspamadm.c @@ -436,6 +436,7 @@ main (gint argc, gchar **argv, gchar **env) (void) dns_resolver_init (rspamd_main->logger, rspamd_main->ev_base, cfg); + rspamd_main->http_ctx = rspamd_http_context_create (cfg, rspamd_main->ev_base); g_log_set_default_handler (rspamd_glib_log_function, rspamd_main->logger); g_set_printerr_handler (rspamd_glib_printerr_function); diff --git a/src/rspamd.h b/src/rspamd.h index c11090a6e..10d3be9fb 100644 --- a/src/rspamd.h +++ b/src/rspamd.h @@ -277,6 +277,7 @@ struct rspamd_main { gboolean cores_throttling; /**< turn off cores when limits are exceeded */ struct roll_history *history; /**< rolling history */ struct event_base *ev_base; + struct rspamd_http_context *http_ctx; }; enum rspamd_exception_type { diff --git a/src/rspamd_proxy.c b/src/rspamd_proxy.c index 31aeceb12..4ecbcf54b 100644 --- a/src/rspamd_proxy.c +++ b/src/rspamd_proxy.c @@ -123,18 +123,14 @@ struct rspamd_proxy_ctx { struct timeval io_tv; /* Encryption key for clients */ struct rspamd_cryptobox_keypair *key; - /* Keys cache */ - struct rspamd_keypair_cache *keys_cache; + /* HTTP context */ + struct rspamd_http_context *http_ctx; /* Upstreams to use */ GHashTable *upstreams; /* Mirrors to send traffic to */ GPtrArray *mirrors; /* Default upstream */ struct rspamd_http_upstream *default_upstream; - /* Local rotating keypair for upstreams */ - struct rspamd_cryptobox_keypair *local_key; - struct event rotate_ev; - gdouble rotate_tm; lua_State *lua_state; /* Array of callback functions called on end of scan to compare results */ GArray *cmp_refs; @@ -746,7 +742,6 @@ init_rspamd_proxy (struct rspamd_config *cfg) ctx->mirrors = g_ptr_array_new (); rspamd_mempool_add_destructor (cfg->cfg_pool, (rspamd_mempool_destruct_t)rspamd_ptr_array_free_hard, ctx->mirrors); - ctx->rotate_tm = DEFAULT_ROTATION_TIME; ctx->cfg = cfg; ctx->lua_state = cfg->lua_state; ctx->cmp_refs = g_array_new (FALSE, FALSE, sizeof (gint)); @@ -763,15 +758,6 @@ init_rspamd_proxy (struct rspamd_config *cfg) G_STRUCT_OFFSET (struct rspamd_proxy_ctx, timeout), RSPAMD_CL_FLAG_TIME_FLOAT, "IO timeout"); - rspamd_rcl_register_worker_option (cfg, - type, - "rotate", - rspamd_rcl_parse_struct_time, - ctx, - G_STRUCT_OFFSET (struct rspamd_proxy_ctx, rotate_tm), - RSPAMD_CL_FLAG_TIME_FLOAT, - "Rotation keys time, default: " - G_STRINGIFY (DEFAULT_ROTATION_TIME) " seconds"); rspamd_rcl_register_worker_option (cfg, type, "keypair", @@ -1418,17 +1404,15 @@ proxy_open_mirror_connections (struct rspamd_proxy_session *session) rspamd_http_message_add_header (msg, "Settings-ID", m->settings_id); } - bk_conn->backend_conn = rspamd_http_connection_new (NULL, + bk_conn->backend_conn = rspamd_http_connection_new ( + session->ctx->http_ctx, + NULL, proxy_backend_mirror_error_handler, proxy_backend_mirror_finish_handler, RSPAMD_HTTP_CLIENT_SIMPLE, - RSPAMD_HTTP_CLIENT, - session->ctx->keys_cache, - NULL); + RSPAMD_HTTP_CLIENT); if (m->key) { - rspamd_http_connection_set_key (bk_conn->backend_conn, - session->ctx->local_key); msg->peer_key = rspamd_pubkey_ref (m->key); } @@ -1851,21 +1835,18 @@ retry: } session->master_conn->backend_conn = rspamd_http_connection_new ( + session->ctx->http_ctx, NULL, proxy_backend_master_error_handler, proxy_backend_master_finish_handler, RSPAMD_HTTP_CLIENT_SIMPLE, - RSPAMD_HTTP_CLIENT, - session->ctx->keys_cache, - NULL); + RSPAMD_HTTP_CLIENT); session->master_conn->flags &= ~RSPAMD_BACKEND_CLOSED; session->master_conn->parser_from_ref = backend->parser_from_ref; session->master_conn->parser_to_ref = backend->parser_to_ref; if (backend->key) { msg->peer_key = rspamd_pubkey_ref (backend->key); - rspamd_http_connection_set_key (session->master_conn->backend_conn, - session->ctx->local_key); } if (backend->settings_id != NULL) { @@ -2105,13 +2086,13 @@ proxy_accept_socket (gint fd, short what, void *arg) } if (!ctx->milter) { - session->client_conn = rspamd_http_connection_new (NULL, + session->client_conn = rspamd_http_connection_new ( + ctx->http_ctx, + NULL, proxy_client_error_handler, proxy_client_finish_handler, 0, - RSPAMD_HTTP_SERVER, - ctx->keys_cache, - NULL); + RSPAMD_HTTP_SERVER); if (ctx->key) { rspamd_http_connection_set_key (session->client_conn, ctx->key); @@ -2158,24 +2139,6 @@ proxy_accept_socket (gint fd, short what, void *arg) } } -static void -proxy_rotate_key (gint fd, short what, void *arg) -{ - struct timeval rot_tv; - struct rspamd_proxy_ctx *ctx = arg; - gpointer kp; - - double_to_tv (ctx->rotate_tm, &rot_tv); - rot_tv.tv_sec += ottery_rand_range (rot_tv.tv_sec); - event_del (&ctx->rotate_ev); - event_add (&ctx->rotate_ev, &rot_tv); - - kp = ctx->local_key; - ctx->local_key = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX, - RSPAMD_CRYPTOBOX_MODE_25519); - rspamd_keypair_unref (kp); -} - static void adjust_upstreams_limits (struct rspamd_proxy_ctx *ctx) { @@ -2205,9 +2168,9 @@ adjust_upstreams_limits (struct rspamd_proxy_ctx *ctx) } void -start_rspamd_proxy (struct rspamd_worker *worker) { +start_rspamd_proxy (struct rspamd_worker *worker) +{ struct rspamd_proxy_ctx *ctx = worker->ctx; - struct timeval rot_tv; ctx->cfg = worker->srv->cfg; ctx->ev_base = rspamd_prepare_worker (worker, "rspamd_proxy", @@ -2222,16 +2185,7 @@ start_rspamd_proxy (struct rspamd_worker *worker) { rspamd_upstreams_library_config (worker->srv->cfg, ctx->cfg->ups_ctx, ctx->ev_base, ctx->resolver->r); - /* XXX: stupid default */ - ctx->keys_cache = rspamd_keypair_cache_new (256); - ctx->local_key = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX, - RSPAMD_CRYPTOBOX_MODE_25519); - - double_to_tv (ctx->rotate_tm, &rot_tv); - rot_tv.tv_sec += ottery_rand_range (rot_tv.tv_sec); - event_set (&ctx->rotate_ev, -1, EV_TIMEOUT, proxy_rotate_key, ctx); - event_base_set (ctx->ev_base, &ctx->rotate_ev); - event_add (&ctx->rotate_ev, &rot_tv); + ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base); if (ctx->has_self_scan) { /* Additional initialisation needed */ @@ -2264,7 +2218,7 @@ start_rspamd_proxy (struct rspamd_worker *worker) { rspamd_stat_close (); } - rspamd_keypair_cache_destroy (ctx->keys_cache); + rspamd_http_context_free (ctx->http_ctx); REF_RELEASE (ctx->cfg); rspamd_log_close (worker->srv->logger, TRUE); diff --git a/src/worker.c b/src/worker.c index 2f7176a8d..8fa02b51a 100644 --- a/src/worker.c +++ b/src/worker.c @@ -414,13 +414,13 @@ accept_socket (gint fd, short what, void *arg) http_opts = RSPAMD_HTTP_REQUIRE_ENCRYPTION; } - task->http_conn = rspamd_http_connection_new (rspamd_worker_body_handler, + task->http_conn = rspamd_http_connection_new ( + ctx->http_ctx, + rspamd_worker_body_handler, rspamd_worker_error_handler, rspamd_worker_finish_handler, http_opts, - RSPAMD_HTTP_SERVER, - ctx->keys_cache, - NULL); + RSPAMD_HTTP_SERVER); rspamd_http_connection_set_max_size (task->http_conn, task->cfg->max_message); worker->nconns++; rspamd_mempool_add_destructor (task->task_pool, @@ -696,8 +696,7 @@ start_worker (struct rspamd_worker *worker) rspamd_upstreams_library_config (worker->srv->cfg, ctx->cfg->ups_ctx, ctx->ev_base, ctx->resolver->r); - /* XXX: stupid default */ - ctx->keys_cache = rspamd_keypair_cache_new (256); + ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base); rspamd_worker_init_scanner (worker, ctx->ev_base, ctx->resolver, &ctx->lang_det); rspamd_lua_run_postloads (ctx->cfg->lua_state, ctx->cfg, ctx->ev_base, @@ -707,7 +706,7 @@ start_worker (struct rspamd_worker *worker) rspamd_worker_block_signals (); rspamd_stat_close (); - rspamd_keypair_cache_destroy (ctx->keys_cache); + rspamd_http_context_free (ctx->http_ctx); REF_RELEASE (ctx->cfg); rspamd_log_close (worker->srv->logger, TRUE); diff --git a/src/worker_private.h b/src/worker_private.h index f07a95b41..398c5d23d 100644 --- a/src/worker_private.h +++ b/src/worker_private.h @@ -49,7 +49,7 @@ struct rspamd_worker_ctx { /* Encryption key */ struct rspamd_cryptobox_keypair *key; /* Keys cache */ - struct rspamd_keypair_cache *keys_cache; + struct rspamd_http_context *http_ctx; /* Language detector */ struct rspamd_lang_detector *lang_det; }; -- 2.39.5