diff options
author | Vsevolod Stakhov <vsevolod@highsecure.ru> | 2019-06-22 13:13:41 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-06-22 13:13:41 +0100 |
commit | c20b2f1b973a01590855af6dd384a320ad8773e5 (patch) | |
tree | ea219d9456516dea7c3be1683d58b1e058a9efe4 /src | |
parent | 57c21062f261eb595f8e64cd32d7df9604b7e754 (diff) | |
parent | 28e34a75931d363e7e85619368fa5c43f606e7d9 (diff) | |
download | rspamd-c20b2f1b973a01590855af6dd384a320ad8773e5.tar.gz rspamd-c20b2f1b973a01590855af6dd384a320ad8773e5.zip |
Merge pull request #2931 from rspamd/libev-migration
[Project] Migrate to libev
Diffstat (limited to 'src')
101 files changed, 2185 insertions, 5793 deletions
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3e6abd4e8..a23c4e505 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -89,11 +89,9 @@ ADD_SUBDIRECTORY(rspamadm) SET(RSPAMDSRC controller.c fuzzy_storage.c - lua_worker.c rspamd.c worker.c - rspamd_proxy.c - log_helper.c) + rspamd_proxy.c) SET(PLUGINSSRC plugins/surbl.c plugins/regexp.c @@ -101,11 +99,10 @@ SET(PLUGINSSRC plugins/surbl.c plugins/fuzzy_check.c plugins/spf.c plugins/dkim_check.c - libserver/rspamd_control.c - lua/lua_fann.c) + libserver/rspamd_control.c) SET(MODULES_LIST surbl regexp chartable fuzzy_check spf dkim) -SET(WORKERS_LIST normal controller fuzzy lua rspamd_proxy log_helper) +SET(WORKERS_LIST normal controller fuzzy rspamd_proxy) IF (ENABLE_HYPERSCAN MATCHES "ON") LIST(APPEND WORKERS_LIST "hs_helper") LIST(APPEND RSPAMDSRC "hs_helper.c") diff --git a/src/client/rspamc.c b/src/client/rspamc.c index 2f572c449..d08a7f620 100644 --- a/src/client/rspamc.c +++ b/src/client/rspamc.c @@ -1609,7 +1609,7 @@ rspamc_client_cb (struct rspamd_client_connection *conn, } static void -rspamc_process_input (struct event_base *ev_base, struct rspamc_command *cmd, +rspamc_process_input (struct ev_loop *ev_base, struct rspamc_command *cmd, FILE *in, const gchar *name, GQueue *attrs) { struct rspamd_client_connection *conn; @@ -1736,7 +1736,7 @@ rspamd_dirent_size (DIR * dirp) } static void -rspamc_process_dir (struct event_base *ev_base, struct rspamc_command *cmd, +rspamc_process_dir (struct ev_loop *ev_base, struct rspamc_command *cmd, const gchar *name, GQueue *attrs) { DIR *d; @@ -1829,7 +1829,7 @@ rspamc_process_dir (struct event_base *ev_base, struct rspamc_command *cmd, if (cur_req >= max_requests) { cur_req = 0; /* Wait for completion */ - event_base_loop (ev_base, 0); + ev_loop (ev_base, 0); } } } @@ -1840,7 +1840,7 @@ rspamc_process_dir (struct event_base *ev_base, struct rspamc_command *cmd, } closedir (d); - event_base_loop (ev_base, 0); + ev_loop (ev_base, 0); } @@ -1863,7 +1863,7 @@ main (gint argc, gchar **argv, gchar **env) GPid cld; struct rspamc_command *cmd; FILE *in = NULL; - struct event_base *ev_base; + struct ev_loop *event_loop; struct stat st; struct sigaction sigpipe_act; gchar **exclude_pattern; @@ -1884,6 +1884,7 @@ main (gint argc, gchar **argv, gchar **env) npatterns = 0; while (exclude_pattern && *exclude_pattern) { + exclude_pattern ++; npatterns ++; } @@ -1902,7 +1903,7 @@ main (gint argc, gchar **argv, gchar **env) } rspamd_init_libs (); - ev_base = event_base_new (); + event_loop = ev_loop_new (EVFLAG_SIGNALFD|EVBACKEND_ALL); struct rspamd_http_context_cfg http_config; @@ -1911,7 +1912,7 @@ main (gint argc, gchar **argv, gchar **env) http_config.kp_cache_size_server = 0; http_config.user_agent = user_agent; http_ctx = rspamd_http_context_create_config (&http_config, - ev_base, NULL); + event_loop, NULL); /* Ignore sigpipe */ sigemptyset (&sigpipe_act.sa_mask); @@ -1972,10 +1973,10 @@ main (gint argc, gchar **argv, gchar **env) if (start_argc == argc) { /* Do command without input or with stdin */ if (empty_input) { - rspamc_process_input (ev_base, cmd, NULL, "empty", kwattrs); + rspamc_process_input (event_loop, cmd, NULL, "empty", kwattrs); } else { - rspamc_process_input (ev_base, cmd, in, "stdin", kwattrs); + rspamc_process_input (event_loop, cmd, in, "stdin", kwattrs); } } else { @@ -1990,7 +1991,7 @@ main (gint argc, gchar **argv, gchar **env) } if (S_ISDIR (st.st_mode)) { /* Directories are processed with a separate limit */ - rspamc_process_dir (ev_base, cmd, argv[i], kwattrs); + rspamc_process_dir (event_loop, cmd, argv[i], kwattrs); cur_req = 0; } else { @@ -1999,24 +2000,24 @@ main (gint argc, gchar **argv, gchar **env) fprintf (stderr, "cannot open file %s\n", argv[i]); exit (EXIT_FAILURE); } - rspamc_process_input (ev_base, cmd, in, argv[i], kwattrs); + rspamc_process_input (event_loop, cmd, in, argv[i], kwattrs); cur_req++; fclose (in); } if (cur_req >= max_requests) { cur_req = 0; /* Wait for completion */ - event_base_loop (ev_base, 0); + ev_loop (event_loop, 0); } } } if (cmd->cmd == RSPAMC_COMMAND_FUZZY_DELHASH) { - rspamc_process_input (ev_base, cmd, NULL, "hashes", kwattrs); + rspamc_process_input (event_loop, cmd, NULL, "hashes", kwattrs); } } - event_base_loop (ev_base, 0); + ev_loop (event_loop, 0); g_queue_free_full (kwattrs, rspamc_kwattr_free); diff --git a/src/client/rspamdclient.c b/src/client/rspamdclient.c index 5f831ee64..c27ff9f0b 100644 --- a/src/client/rspamdclient.c +++ b/src/client/rspamdclient.c @@ -37,8 +37,8 @@ struct rspamd_client_connection { GString *server_name; struct rspamd_cryptobox_pubkey *key; struct rspamd_cryptobox_keypair *keypair; - struct event_base *ev_base; - struct timeval timeout; + struct ev_loop *event_loop; + ev_tstamp timeout; struct rspamd_http_connection *http_conn; gboolean req_sent; gdouble start_time; @@ -118,7 +118,7 @@ rspamd_client_finish_handler (struct rspamd_http_connection *conn, rspamd_http_connection_reset (c->http_conn); rspamd_http_connection_read_message (c->http_conn, c->req, - &c->timeout); + c->timeout); return 0; } else { @@ -240,7 +240,7 @@ rspamd_client_finish_handler (struct rspamd_http_connection *conn, struct rspamd_client_connection * rspamd_client_init (struct rspamd_http_context *http_ctx, - struct event_base *ev_base, const gchar *name, + struct ev_loop *ev_base, const gchar *name, guint16 port, gdouble timeout, const gchar *key) { struct rspamd_client_connection *conn; @@ -252,7 +252,7 @@ rspamd_client_init (struct rspamd_http_context *http_ctx, } conn = g_malloc0 (sizeof (struct rspamd_client_connection)); - conn->ev_base = ev_base; + conn->event_loop = ev_base; conn->fd = fd; conn->req_sent = FALSE; conn->http_conn = rspamd_http_connection_new_client_socket (http_ctx, @@ -267,7 +267,7 @@ rspamd_client_init (struct rspamd_http_context *http_ctx, rspamd_printf_gstring (conn->server_name, ":%d", (int)port); } - double_to_tv (timeout, &conn->timeout); + conn->timeout = timeout; if (key) { conn->key = rspamd_pubkey_from_base32 (key, 0, RSPAMD_KEYPAIR_KEX, @@ -442,11 +442,11 @@ rspamd_client_command (struct rspamd_client_connection *conn, if (compressed) { rspamd_http_connection_write_message (conn->http_conn, req->msg, NULL, "application/x-compressed", req, - &conn->timeout); + conn->timeout); } else { rspamd_http_connection_write_message (conn->http_conn, req->msg, NULL, - "text/plain", req, &conn->timeout); + "text/plain", req, conn->timeout); } return TRUE; diff --git a/src/client/rspamdclient.h b/src/client/rspamdclient.h index 8a5b3de35..9fced29af 100644 --- a/src/client/rspamdclient.h +++ b/src/client/rspamdclient.h @@ -18,7 +18,7 @@ #include "config.h" #include "ucl.h" -#include <event.h> +#include "contrib/libev/ev.h" struct rspamd_client_connection; struct rspamd_http_message; @@ -58,7 +58,7 @@ struct rspamd_http_context; */ struct rspamd_client_connection * rspamd_client_init ( struct rspamd_http_context *http_ctx, - struct event_base *ev_base, + struct ev_loop *ev_base, const gchar *name, guint16 port, gdouble timeout, diff --git a/src/controller.c b/src/controller.c index 8dcbdb33f..46c02d47c 100644 --- a/src/controller.c +++ b/src/controller.c @@ -105,12 +105,8 @@ INIT_LOG_MODULE(controller) #define COLOR_REJECT "#CB4B4B" #define COLOR_TOTAL "#9440ED" -const struct timeval rrd_update_time = { - .tv_sec = 1, - .tv_usec = 0 -}; - -const guint64 rspamd_controller_ctx_magic = 0xf72697805e6941faULL; +const static ev_tstamp rrd_update_time = 1.0; +const static guint64 rspamd_controller_ctx_magic = 0xf72697805e6941faULL; extern void fuzzy_stat_command (struct rspamd_task *task); @@ -132,14 +128,13 @@ worker_t controller_worker = { struct rspamd_controller_worker_ctx { guint64 magic; /* Events base */ - struct event_base *ev_base; + struct ev_loop *event_loop; /* DNS resolver */ struct rspamd_dns_resolver *resolver; /* Config */ struct rspamd_config *cfg; /* END OF COMMON PART */ - guint32 timeout; - struct timeval io_tv; + ev_tstamp timeout; /* Whether we use ssl for this server */ gboolean use_ssl; /* Webui password */ @@ -153,7 +148,7 @@ struct rspamd_controller_worker_ctx { struct rspamd_http_context *http_ctx; struct rspamd_http_connection_router *http; /* Server's start time */ - time_t start_time; + ev_tstamp start_time; /* Main server */ struct rspamd_main *srv; /* SSL cert */ @@ -182,9 +177,9 @@ struct rspamd_controller_worker_ctx { /* Local keypair */ gpointer key; - struct event *rrd_event; + ev_timer rrd_event; struct rspamd_rrd_file *rrd; - struct event save_stats_event; + ev_timer save_stats_event; struct rspamd_lang_detector *lang_det; gdouble task_timeout; }; @@ -732,7 +727,7 @@ rspamd_controller_handle_auth (struct rspamd_http_connection_entry *conn_ent, data[4] = st->actions_stat[METRIC_ACTION_SOFT_REJECT]; /* Get uptime */ - uptime = time (NULL) - session->ctx->start_time; + uptime = ev_time () - session->ctx->start_time; ucl_object_insert_key (obj, ucl_object_fromstring ( RVERSION), "version", 0, false); @@ -1000,7 +995,7 @@ rspamd_controller_handle_get_map (struct rspamd_http_connection_entry *conn_ent, struct rspamd_controller_session *session = conn_ent->ud; GList *cur; struct rspamd_map *map; - struct rspamd_map_backend *bk; + struct rspamd_map_backend *bk = NULL; const rspamd_ftok_t *idstr; struct stat st; gint fd; @@ -1041,7 +1036,7 @@ rspamd_controller_handle_get_map (struct rspamd_http_connection_entry *conn_ent, cur = g_list_next (cur); } - if (!found) { + if (!found || bk == NULL) { msg_info_session ("map not found"); rspamd_controller_send_error (conn_ent, 404, "Map not found"); return 0; @@ -1079,7 +1074,7 @@ rspamd_controller_handle_get_map (struct rspamd_http_connection_entry *conn_ent, rspamd_http_router_insert_headers (conn_ent->rt, reply); rspamd_http_connection_write_message (conn_ent->conn, reply, NULL, "text/plain", conn_ent, - conn_ent->rt->ptv); + conn_ent->rt->timeout); conn_ent->is_reply = TRUE; return 0; @@ -1389,13 +1384,13 @@ rspamd_controller_handle_legacy_history ( row = &copied_rows[row_num]; /* Get only completed rows */ if (row->completed) { - rspamd_localtime (row->tv.tv_sec, &tm); + rspamd_localtime (row->timestamp, &tm); strftime (timebuf, sizeof (timebuf) - 1, "%Y-%m-%d %H:%M:%S", &tm); obj = ucl_object_typed_new (UCL_OBJECT); ucl_object_insert_key (obj, ucl_object_fromstring ( timebuf), "time", 0, false); ucl_object_insert_key (obj, ucl_object_fromint ( - row->tv.tv_sec), "unix_time", 0, false); + row->timestamp), "unix_time", 0, false); ucl_object_insert_key (obj, ucl_object_fromstring ( row->message_id), "id", 0, false); ucl_object_insert_key (obj, ucl_object_fromstring (row->from_addr), @@ -1525,7 +1520,7 @@ rspamd_controller_handle_lua_history (lua_State *L, if (lua_isfunction (L, -1)) { task = rspamd_task_new (session->ctx->worker, session->cfg, - session->pool, ctx->lang_det, ctx->ev_base); + session->pool, ctx->lang_det, ctx->event_loop); task->resolver = ctx->resolver; task->s = rspamd_session_create (session->pool, @@ -1822,7 +1817,7 @@ rspamd_controller_handle_lua (struct rspamd_http_connection_entry *conn_ent, } task = rspamd_task_new (session->ctx->worker, session->cfg, session->pool, - ctx->lang_det, ctx->ev_base); + ctx->lang_det, ctx->event_loop); task->resolver = ctx->resolver; task->s = rspamd_session_create (session->pool, @@ -1939,7 +1934,7 @@ rspamd_controller_scan_reply (struct rspamd_task *task) rspamd_http_connection_reset (conn_ent->conn); rspamd_http_router_insert_headers (conn_ent->rt, msg); rspamd_http_connection_write_message (conn_ent->conn, msg, NULL, - "application/json", conn_ent, conn_ent->rt->ptv); + "application/json", conn_ent, conn_ent->rt->timeout); conn_ent->is_reply = TRUE; } @@ -2004,7 +1999,7 @@ rspamd_controller_handle_learn_common ( } task = rspamd_task_new (session->ctx->worker, session->cfg, session->pool, - session->ctx->lang_det, ctx->ev_base); + session->ctx->lang_det, ctx->event_loop); task->resolver = ctx->resolver; task->s = rspamd_session_create (session->pool, @@ -2102,7 +2097,7 @@ rspamd_controller_handle_scan (struct rspamd_http_connection_entry *conn_ent, } task = rspamd_task_new (session->ctx->worker, session->cfg, session->pool, - ctx->lang_det, ctx->ev_base); + ctx->lang_det, ctx->event_loop); task->resolver = ctx->resolver; task->s = rspamd_session_create (session->pool, @@ -2129,13 +2124,10 @@ rspamd_controller_handle_scan (struct rspamd_http_connection_entry *conn_ent, } if (ctx->task_timeout > 0.0) { - struct timeval task_tv; - - event_set (&task->timeout_ev, -1, EV_TIMEOUT, rspamd_task_timeout, - task); - event_base_set (ctx->ev_base, &task->timeout_ev); - double_to_tv (ctx->task_timeout, &task_tv); - event_add (&task->timeout_ev, &task_tv); + task->timeout_ev.data = task; + ev_timer_init (&task->timeout_ev, rspamd_task_timeout, + ctx->task_timeout, 0.0); + ev_timer_start (task->event_loop, &task->timeout_ev); } end: @@ -2214,6 +2206,7 @@ rspamd_controller_handle_saveactions ( switch (i) { case 0: + default: act = METRIC_ACTION_REJECT; break; case 1: @@ -2408,7 +2401,7 @@ rspamd_controller_handle_savemap (struct rspamd_http_connection_entry *conn_ent, { struct rspamd_controller_session *session = conn_ent->ud; GList *cur; - struct rspamd_map *map; + struct rspamd_map *map = NULL; struct rspamd_map_backend *bk; struct rspamd_controller_worker_ctx *ctx; const rspamd_ftok_t *idstr; @@ -2600,7 +2593,7 @@ rspamd_controller_handle_stat_common ( ctx = session->ctx; task = rspamd_task_new (session->ctx->worker, session->cfg, session->pool, - ctx->lang_det, ctx->ev_base); + ctx->lang_det, ctx->event_loop); task->resolver = ctx->resolver; cbdata = rspamd_mempool_alloc0 (session->pool, sizeof (*cbdata)); cbdata->conn_ent = conn_ent; @@ -2907,7 +2900,7 @@ rspamd_controller_handle_ping (struct rspamd_http_connection_entry *conn_ent, NULL, "text/plain", conn_ent, - conn_ent->rt->ptv); + conn_ent->rt->timeout); conn_ent->is_reply = TRUE; return 0; @@ -2941,7 +2934,7 @@ rspamd_controller_handle_unknown (struct rspamd_http_connection_entry *conn_ent, NULL, "text/plain", conn_ent, - conn_ent->rt->ptv); + conn_ent->rt->timeout); conn_ent->is_reply = TRUE; } else { @@ -2957,7 +2950,7 @@ rspamd_controller_handle_unknown (struct rspamd_http_connection_entry *conn_ent, NULL, "text/plain", conn_ent, - conn_ent->rt->ptv); + conn_ent->rt->timeout); conn_ent->is_reply = TRUE; } @@ -3002,7 +2995,7 @@ rspamd_controller_handle_lua_plugin (struct rspamd_http_connection_entry *conn_e } task = rspamd_task_new (session->ctx->worker, session->cfg, session->pool, - ctx->lang_det, ctx->ev_base); + ctx->lang_det, ctx->event_loop); task->resolver = ctx->resolver; task->s = rspamd_session_create (session->pool, @@ -3081,9 +3074,9 @@ rspamd_controller_finish_handler (struct rspamd_http_connection_entry *conn_ent) } static void -rspamd_controller_accept_socket (gint fd, short what, void *arg) +rspamd_controller_accept_socket (EV_P_ ev_io *w, int revents) { - struct rspamd_worker *worker = (struct rspamd_worker *) arg; + struct rspamd_worker *worker = (struct rspamd_worker *)w->data; struct rspamd_controller_worker_ctx *ctx; struct rspamd_controller_session *session; rspamd_inet_addr_t *addr; @@ -3092,7 +3085,8 @@ rspamd_controller_accept_socket (gint fd, short what, void *arg) ctx = worker->ctx; if ((nfd = - rspamd_accept_from_socket (fd, &addr, worker->accept_events)) == -1) { + rspamd_accept_from_socket (w->fd, &addr, + rspamd_worker_throttle_accept_events, worker->accept_events)) == -1) { msg_warn_ctx ("accept failed: %s", strerror (errno)); return; } @@ -3117,9 +3111,10 @@ rspamd_controller_accept_socket (gint fd, short what, void *arg) } static void -rspamd_controller_rrd_update (gint fd, short what, void *arg) +rspamd_controller_rrd_update (EV_P_ ev_timer *w, int revents) { - struct rspamd_controller_worker_ctx *ctx = arg; + struct rspamd_controller_worker_ctx *ctx = + (struct rspamd_controller_worker_ctx *)w->data; struct rspamd_stat *stat; GArray ar; gdouble points[METRIC_ACTION_MAX]; @@ -3143,8 +3138,7 @@ rspamd_controller_rrd_update (gint fd, short what, void *arg) } /* Plan new event */ - event_del (ctx->rrd_event); - evtimer_add (ctx->rrd_event, &rrd_update_time); + ev_timer_again (ctx->event_loop, &ctx->rrd_event); } static void @@ -3282,11 +3276,13 @@ rspamd_controller_store_saved_stats (struct rspamd_controller_worker_ctx *ctx) } static void -rspamd_controller_stats_save_periodic (int fd, short what, gpointer ud) +rspamd_controller_stats_save_periodic (EV_P_ ev_timer *w, int revents) { - struct rspamd_controller_worker_ctx *ctx = ud; + struct rspamd_controller_worker_ctx *ctx = + (struct rspamd_controller_worker_ctx *)w->data; rspamd_controller_store_saved_stats (ctx); + ev_timer_again (EV_A_ w); } static void @@ -3379,7 +3375,7 @@ init_controller_worker (struct rspamd_config *cfg) ctx, G_STRUCT_OFFSET (struct rspamd_controller_worker_ctx, timeout), - RSPAMD_CL_FLAG_TIME_INTEGER, + RSPAMD_CL_FLAG_TIME_FLOAT, "Protocol timeout"); rspamd_rcl_register_worker_option (cfg, @@ -3480,14 +3476,14 @@ static int lua_csession_get_ev_base (lua_State *L) { struct rspamd_http_connection_entry *c = lua_check_controller_entry (L, 1); - struct event_base **pbase; + struct ev_loop **pbase; struct rspamd_controller_session *s; if (c) { s = c->ud; - pbase = lua_newuserdata (L, sizeof (struct event_base *)); + pbase = lua_newuserdata (L, sizeof (struct ev_loop *)); rspamd_lua_setclass (L, "rspamd{ev_base}", -1); - *pbase = s->ctx->ev_base; + *pbase = s->ctx->event_loop; } else { return luaL_error (L, "invalid arguments"); @@ -3568,7 +3564,7 @@ lua_csession_send_string (lua_State *L) return 0; } -static gboolean +static void rspamd_controller_on_terminate (struct rspamd_worker *worker) { struct rspamd_controller_worker_ctx *ctx = worker->ctx; @@ -3577,11 +3573,9 @@ rspamd_controller_on_terminate (struct rspamd_worker *worker) if (ctx->rrd) { msg_info ("closing rrd file: %s", ctx->rrd->filename); - event_del (ctx->rrd_event); + ev_timer_stop (ctx->event_loop, &ctx->rrd_event); rspamd_rrd_close (ctx->rrd); } - - return FALSE; } static void @@ -3698,16 +3692,14 @@ start_controller_worker (struct rspamd_worker *worker) GHashTableIter iter; gpointer key, value; guint i; - struct timeval stv; - const guint save_stats_interval = 60 * 1000; /* 1 minute */ + const ev_tstamp save_stats_interval = 60; /* 1 minute */ gpointer m; - ctx->ev_base = rspamd_prepare_worker (worker, + ctx->event_loop = rspamd_prepare_worker (worker, "controller", rspamd_controller_accept_socket); - msec_to_tv (ctx->timeout, &ctx->io_tv); - ctx->start_time = time (NULL); + ctx->start_time = ev_time (); ctx->worker = worker; ctx->cfg = worker->srv->cfg; ctx->srv = worker->srv; @@ -3738,8 +3730,6 @@ start_controller_worker (struct rspamd_worker *worker) DEFAULT_STATS_PATH); } - g_ptr_array_add (worker->finish_actions, - (gpointer)rspamd_controller_on_terminate); rspamd_controller_load_saved_stats (ctx); ctx->lang_det = ctx->cfg->lang_det; @@ -3750,10 +3740,10 @@ start_controller_worker (struct rspamd_worker *worker) ctx->rrd = rspamd_rrd_file_default (ctx->cfg->rrd_file, &rrd_err); if (ctx->rrd) { - ctx->rrd_event = g_malloc0 (sizeof (*ctx->rrd_event)); - evtimer_set (ctx->rrd_event, rspamd_controller_rrd_update, ctx); - event_base_set (ctx->ev_base, ctx->rrd_event); - event_add (ctx->rrd_event, &rrd_update_time); + ctx->rrd_event.data = ctx; + ev_timer_init (&ctx->rrd_event, rspamd_controller_rrd_update, + rrd_update_time, rrd_update_time); + ev_timer_start (ctx->event_loop, &ctx->rrd_event); } else if (rrd_err) { msg_err ("cannot load rrd from %s: %e", ctx->cfg->rrd_file, @@ -3773,10 +3763,10 @@ start_controller_worker (struct rspamd_worker *worker) "password"); /* Accept event */ - ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base, + ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->event_loop, ctx->cfg->ups_ctx); ctx->http = rspamd_http_router_new (rspamd_controller_error_handler, - rspamd_controller_finish_handler, &ctx->io_tv, + rspamd_controller_finish_handler, ctx->timeout, ctx->static_files_dir, ctx->http_ctx); /* Add callbacks for different methods */ @@ -3889,41 +3879,48 @@ start_controller_worker (struct rspamd_worker *worker) rspamd_controller_handle_unknown); ctx->resolver = rspamd_dns_resolver_init (worker->srv->logger, - ctx->ev_base, + ctx->event_loop, worker->srv->cfg); rspamd_upstreams_library_config (worker->srv->cfg, worker->srv->cfg->ups_ctx, - ctx->ev_base, ctx->resolver->r); - rspamd_symcache_start_refresh (worker->srv->cfg->cache, ctx->ev_base, + ctx->event_loop, ctx->resolver->r); + rspamd_symcache_start_refresh (worker->srv->cfg->cache, ctx->event_loop, worker); - rspamd_stat_init (worker->srv->cfg, ctx->ev_base); + rspamd_stat_init (worker->srv->cfg, ctx->event_loop); if (worker->index == 0) { if (!ctx->cfg->disable_monitored) { - rspamd_worker_init_monitored (worker, ctx->ev_base, ctx->resolver); + rspamd_worker_init_monitored (worker, ctx->event_loop, ctx->resolver); } - rspamd_map_watch (worker->srv->cfg, ctx->ev_base, + rspamd_map_watch (worker->srv->cfg, ctx->event_loop, ctx->resolver, worker, TRUE); /* Schedule periodic stats saving, see #1823 */ - event_set (&ctx->save_stats_event, -1, EV_PERSIST, + ctx->save_stats_event.data = ctx; + ev_timer_init (&ctx->save_stats_event, rspamd_controller_stats_save_periodic, - ctx); - event_base_set (ctx->ev_base, &ctx->save_stats_event); - msec_to_tv (save_stats_interval, &stv); - evtimer_add (&ctx->save_stats_event, &stv); + save_stats_interval, save_stats_interval); + ev_timer_start (ctx->event_loop, &ctx->save_stats_event); } else { - rspamd_map_watch (worker->srv->cfg, ctx->ev_base, + rspamd_map_watch (worker->srv->cfg, ctx->event_loop, ctx->resolver, worker, FALSE); } - rspamd_lua_run_postloads (ctx->cfg->lua_state, ctx->cfg, ctx->ev_base, worker); + rspamd_lua_run_postloads (ctx->cfg->lua_state, ctx->cfg, ctx->event_loop, worker); + +#ifdef WITH_HYPERSCAN + rspamd_control_worker_add_cmd_handler (worker, + RSPAMD_CONTROL_HYPERSCAN_LOADED, + rspamd_worker_hyperscan_ready, + NULL); +#endif /* Start event loop */ - event_base_loop (ctx->ev_base, 0); + ev_loop (ctx->event_loop, 0); rspamd_worker_block_signals (); + rspamd_controller_on_terminate (worker); rspamd_stat_close (); rspamd_http_router_free (ctx->http); diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c index 43c975459..7913c56b9 100644 --- a/src/fuzzy_storage.c +++ b/src/fuzzy_storage.c @@ -36,8 +36,7 @@ #include "libserver/rspamd_control.h" #include "libutil/hash.h" #include "libutil/map_private.h" -#include "libutil/http_private.h" -#include "libutil/http_router.h" +#include "contrib/uthash/utlist.h" #include "unix-std.h" #include <math.h> @@ -88,7 +87,7 @@ worker_t fuzzy_worker = { init_fuzzy, /* Init function */ start_fuzzy, /* Start function */ RSPAMD_WORKER_HAS_SOCKET, - RSPAMD_WORKER_SOCKET_UDP|RSPAMD_WORKER_SOCKET_TCP, /* Both socket */ + RSPAMD_WORKER_SOCKET_UDP, /* UDP socket */ RSPAMD_WORKER_VER /* Version info */ }; @@ -132,7 +131,7 @@ static const guint64 rspamd_fuzzy_storage_magic = 0x291a3253eb1b3ea5ULL; struct rspamd_fuzzy_storage_ctx { guint64 magic; /* Events base */ - struct event_base *ev_base; + struct ev_loop *event_loop; /* DNS resolver */ struct rspamd_dns_resolver *resolver; /* Config */ @@ -146,34 +145,21 @@ struct rspamd_fuzzy_storage_ctx { struct rspamd_radix_map_helper *blocked_ips; struct rspamd_radix_map_helper *ratelimit_whitelist; - struct rspamd_cryptobox_keypair *sync_keypair; - struct rspamd_cryptobox_pubkey *master_key; - struct timeval master_io_tv; - gdouble master_timeout; - GPtrArray *mirrors; const ucl_object_t *update_map; - const ucl_object_t *masters_map; const ucl_object_t *blocked_map; const ucl_object_t *ratelimit_whitelist_map; - GHashTable *master_flags; guint keypair_cache_size; - gint peer_fd; - struct event peer_ev; - struct event stat_ev; - struct timeval stat_tv; + ev_timer stat_ev; + ev_io peer_ev; + ev_tstamp stat_timeout; /* Local keypair */ struct rspamd_cryptobox_keypair *default_keypair; /* Bad clash, need for parse keypair */ struct fuzzy_key *default_key; GHashTable *keys; gboolean encrypted_only; - gboolean collection_mode; gboolean read_only; - struct rspamd_cryptobox_keypair *collection_keypair; - struct rspamd_cryptobox_pubkey *collection_sign_key; - gchar *collection_id_file; - struct rspamd_http_context *http_ctx; struct rspamd_keypair_cache *keypair_cache; rspamd_lru_hash_t *errors_ips; rspamd_lru_hash_t *ratelimit_buckets; @@ -181,7 +167,8 @@ struct rspamd_fuzzy_storage_ctx { GArray *updates_pending; guint updates_failed; guint updates_maxfail; - guint32 collection_id; + /* Used to send data between workers */ + gint peer_fd; /* Ratelimits */ guint leaky_bucket_ttl; @@ -192,7 +179,6 @@ struct rspamd_fuzzy_storage_ctx { gdouble leaky_bucket_rate; struct rspamd_worker *worker; - struct rspamd_http_connection_router *collection_rt; const ucl_object_t *skip_map; struct rspamd_hash_map_helper *skip_hashes; guchar cookie[COOKIE_SIZE]; @@ -224,14 +210,14 @@ struct fuzzy_session { enum fuzzy_cmd_type cmd_type; gint fd; guint64 time; - struct event io; + struct ev_io io; ref_entry_t ref; struct fuzzy_key_stat *key_stat; guchar nm[rspamd_cryptobox_MAX_NMBYTES]; }; struct fuzzy_peer_request { - struct event io_ev; + ev_io io_ev; struct fuzzy_peer_cmd cmd; }; @@ -241,19 +227,13 @@ struct fuzzy_key { struct fuzzy_key_stat *stat; }; -struct fuzzy_master_update_session { - const gchar *name; - gchar uid[16]; - struct rspamd_http_connection *conn; - struct rspamd_http_message *msg; +struct rspamd_updates_cbdata { + GArray *updates_pending; struct rspamd_fuzzy_storage_ctx *ctx; - const gchar *src; - gchar *psrc; - rspamd_inet_addr_t *addr; - gboolean replied; - gint sock; + gchar *source; }; + static void rspamd_fuzzy_write_reply (struct fuzzy_session *session); static gboolean @@ -261,8 +241,7 @@ rspamd_fuzzy_check_ratelimit (struct fuzzy_session *session) { rspamd_inet_addr_t *masked; struct rspamd_leaky_bucket_elt *elt; - struct timeval tv; - gdouble now; + ev_tstamp now; if (session->ctx->ratelimit_whitelist != NULL) { if (rspamd_match_radix_map_addr (session->ctx->ratelimit_whitelist, @@ -289,15 +268,9 @@ rspamd_fuzzy_check_ratelimit (struct fuzzy_session *session) MIN (MAX (session->ctx->leaky_bucket_mask * 4, 64), 128)); } -#ifdef HAVE_EVENT_NO_CACHE_TIME_FUNC - event_base_gettimeofday_cached (session->ctx->ev_base, &tv); -#else - gettimeofday (&tv, NULL); -#endif - - now = tv_to_double (&tv); + now = ev_now (session->ctx->event_loop); elt = rspamd_lru_hash_lookup (session->ctx->ratelimit_buckets, masked, - tv.tv_sec); + now); if (elt) { gboolean ratelimited = FALSE; @@ -348,7 +321,7 @@ rspamd_fuzzy_check_ratelimit (struct fuzzy_session *session) rspamd_lru_hash_insert (session->ctx->ratelimit_buckets, masked, elt, - tv.tv_sec, + now, session->ctx->leaky_bucket_ttl); } @@ -424,15 +397,6 @@ fuzzy_count_callback (guint64 count, void *ud) ctx->stat.fuzzy_hashes = count; } -struct fuzzy_slave_connection { - struct rspamd_cryptobox_keypair *local_key; - struct rspamd_cryptobox_pubkey *remote_key; - struct upstream *up; - struct rspamd_http_connection *http_conn; - struct rspamd_fuzzy_mirror *mirror; - gint sock; -}; - static void fuzzy_rl_bucket_free (gpointer p) { @@ -443,228 +407,32 @@ fuzzy_rl_bucket_free (gpointer p) } static void -fuzzy_mirror_close_connection (struct fuzzy_slave_connection *conn) -{ - if (conn) { - if (conn->http_conn) { - rspamd_http_connection_reset (conn->http_conn); - rspamd_http_connection_unref (conn->http_conn); - } - - close (conn->sock); - - g_free (conn); - } -} - -struct rspamd_fuzzy_updates_cbdata { - struct rspamd_fuzzy_storage_ctx *ctx; - struct rspamd_http_message *msg; - struct fuzzy_slave_connection *conn; - struct rspamd_fuzzy_mirror *m; - GArray *updates_pending; -}; - -static void -fuzzy_mirror_updates_version_cb (guint64 rev64, void *ud) -{ - struct rspamd_fuzzy_updates_cbdata *cbdata = ud; - struct fuzzy_peer_cmd *io_cmd; - guint32 rev32 = rev64, len; - const gchar *p; - rspamd_fstring_t *reply; - struct fuzzy_slave_connection *conn; - struct rspamd_fuzzy_storage_ctx *ctx; - struct rspamd_http_message *msg; - struct rspamd_fuzzy_mirror *m; - struct timeval tv; - guint i; - - conn = cbdata->conn; - ctx = cbdata->ctx; - msg = cbdata->msg; - m = cbdata->m; - - rev32 = GUINT32_TO_LE (rev32); - len = sizeof (guint32) * 2; /* revision + last chunk */ - - for (i = 0; i < cbdata->updates_pending->len; i ++) { - io_cmd = &g_array_index (cbdata->updates_pending, - struct fuzzy_peer_cmd, i); - - if (io_cmd->is_shingle) { - len += sizeof (guint32) + sizeof (guint32) + - sizeof (struct rspamd_fuzzy_shingle_cmd); - } - else { - len += sizeof (guint32) + sizeof (guint32) + - sizeof (struct rspamd_fuzzy_cmd); - } - } - - reply = rspamd_fstring_sized_new (len); - reply = rspamd_fstring_append (reply, (const char *)&rev32, - sizeof (rev32)); - - for (i = 0; i < cbdata->updates_pending->len; i ++) { - io_cmd = &g_array_index (cbdata->updates_pending, struct fuzzy_peer_cmd, i); - - if (io_cmd->is_shingle) { - len = sizeof (guint32) + - sizeof (struct rspamd_fuzzy_shingle_cmd); - } - else { - len = sizeof (guint32) + - sizeof (struct rspamd_fuzzy_cmd); - } - - p = (const char *)io_cmd; - len = GUINT32_TO_LE (len); - reply = rspamd_fstring_append (reply, (const char *)&len, sizeof (len)); - reply = rspamd_fstring_append (reply, p, len); - } - - /* Last chunk */ - len = 0; - reply = rspamd_fstring_append (reply, (const char *)&len, sizeof (len)); - rspamd_http_message_set_body_from_fstring_steal (msg, reply); - double_to_tv (ctx->sync_timeout, &tv); - rspamd_http_connection_write_message (conn->http_conn, - msg, NULL, NULL, conn, - &tv); - msg_info ("send update request to %s", m->name); - - g_array_free (cbdata->updates_pending, TRUE); - g_free (cbdata); -} - -static void -fuzzy_mirror_updates_to_http (struct rspamd_fuzzy_mirror *m, - struct fuzzy_slave_connection *conn, - struct rspamd_fuzzy_storage_ctx *ctx, - struct rspamd_http_message *msg, - GArray *updates) -{ - - struct rspamd_fuzzy_updates_cbdata *cbdata; - - cbdata = g_malloc (sizeof (*cbdata)); - cbdata->ctx = ctx; - cbdata->msg = msg; - cbdata->conn = conn; - cbdata->m = m; - /* Copy queue */ - cbdata->updates_pending = g_array_sized_new (FALSE, FALSE, - sizeof (struct fuzzy_peer_cmd), updates->len); - g_array_append_vals (cbdata->updates_pending, updates->data, updates->len); - rspamd_fuzzy_backend_version (ctx->backend, local_db_name, - fuzzy_mirror_updates_version_cb, cbdata); -} - -static void -fuzzy_mirror_error_handler (struct rspamd_http_connection *conn, GError *err) -{ - struct fuzzy_slave_connection *bk_conn = conn->ud; - msg_info ("abnormally closing connection from backend: %s:%s, " - "error: %e", - bk_conn->mirror->name, - rspamd_inet_address_to_string (rspamd_upstream_addr_cur (bk_conn->up)), - err); - - fuzzy_mirror_close_connection (bk_conn); -} - -static gint -fuzzy_mirror_finish_handler (struct rspamd_http_connection *conn, - struct rspamd_http_message *msg) +fuzzy_stat_count_callback (guint64 count, void *ud) { - struct fuzzy_slave_connection *bk_conn = conn->ud; - - msg_info ("finished mirror connection to %s", bk_conn->mirror->name); - fuzzy_mirror_close_connection (bk_conn); + struct rspamd_fuzzy_storage_ctx *ctx = ud; - return 0; + ev_timer_again (ctx->event_loop, &ctx->stat_ev); + ctx->stat.fuzzy_hashes = count; } static void -rspamd_fuzzy_send_update_mirror (struct rspamd_fuzzy_storage_ctx *ctx, - struct rspamd_fuzzy_mirror *m, GArray *updates) +rspamd_fuzzy_stat_callback (EV_P_ ev_timer *w, int revents) { - struct fuzzy_slave_connection *conn; - struct rspamd_http_message *msg; - - conn = g_malloc0 (sizeof (*conn)); - conn->up = rspamd_upstream_get (m->u, - RSPAMD_UPSTREAM_MASTER_SLAVE, NULL, 0); - conn->mirror = m; - - if (conn->up == NULL) { - g_free (conn); - msg_err ("cannot select upstream for %s", m->name); - return; - } - - conn->sock = rspamd_inet_address_connect ( - rspamd_upstream_addr_next (conn->up), - SOCK_STREAM, TRUE); - - if (conn->sock == -1) { - g_free (conn); - msg_err ("cannot connect upstream for %s", m->name); - rspamd_upstream_fail (conn->up, TRUE); - return; - } - - msg = rspamd_http_new_message (HTTP_REQUEST); - rspamd_printf_fstring (&msg->url, "/update_v1/%s", m->name); - - conn->http_conn = rspamd_http_connection_new_client_socket ( - ctx->http_ctx, - NULL, - fuzzy_mirror_error_handler, - fuzzy_mirror_finish_handler, - RSPAMD_HTTP_CLIENT_SIMPLE, - conn->sock); - - rspamd_http_connection_set_key (conn->http_conn, - ctx->sync_keypair); - msg->peer_key = rspamd_pubkey_ref (m->key); - fuzzy_mirror_updates_to_http (m, conn, ctx, msg, updates); + struct rspamd_fuzzy_storage_ctx *ctx = + (struct rspamd_fuzzy_storage_ctx *)w->data; + rspamd_fuzzy_backend_count (ctx->backend, fuzzy_stat_count_callback, ctx); } -struct rspamd_updates_cbdata { - GArray *updates_pending; - struct rspamd_fuzzy_storage_ctx *ctx; - gchar *source; -}; static void fuzzy_update_version_callback (guint64 ver, void *ud) { msg_info ("updated fuzzy storage from %s: version: %d", - (const char *)ud, (gint)ver); + (const char *)ud, (gint)ver); g_free (ud); } static void -fuzzy_stat_count_callback (guint64 count, void *ud) -{ - struct rspamd_fuzzy_storage_ctx *ctx = ud; - - event_add (&ctx->stat_ev, &ctx->stat_tv); - ctx->stat.fuzzy_hashes = count; -} - -static void -rspamd_fuzzy_stat_callback (gint fd, gshort what, gpointer ud) -{ - struct rspamd_fuzzy_storage_ctx *ctx = ud; - - event_del (&ctx->stat_ev); - rspamd_fuzzy_backend_count (ctx->backend, fuzzy_stat_count_callback, ctx); -} - -static void rspamd_fuzzy_updates_cb (gboolean success, guint nadded, guint ndeleted, @@ -673,8 +441,6 @@ rspamd_fuzzy_updates_cb (gboolean success, void *ud) { struct rspamd_updates_cbdata *cbdata = ud; - struct rspamd_fuzzy_mirror *m; - guint i; struct rspamd_fuzzy_storage_ctx *ctx; const gchar *source; @@ -684,15 +450,6 @@ rspamd_fuzzy_updates_cb (gboolean success, if (success) { rspamd_fuzzy_backend_count (ctx->backend, fuzzy_count_callback, ctx); - if (ctx->updates_pending->len > 0) { - for (i = 0; i < ctx->mirrors->len; i ++) { - m = g_ptr_array_index (ctx->mirrors, i); - - rspamd_fuzzy_send_update_mirror (ctx, m, - cbdata->updates_pending); - } - } - msg_info ("successfully updated fuzzy storage: %d updates in queue; " "%d pending currently; " "%d added, %d deleted, %d extended, %d duplicates", @@ -727,12 +484,7 @@ rspamd_fuzzy_updates_cb (gboolean success, if (ctx->worker->wanna_die) { /* Plan exit */ - struct timeval tv; - - tv.tv_sec = 0; - tv.tv_usec = 0; - - event_base_loopexit (ctx->ev_base, &tv); + ev_break (ctx->event_loop, EVBREAK_ALL); } g_array_free (cbdata->updates_pending, TRUE); @@ -762,9 +514,9 @@ rspamd_fuzzy_process_updates_queue (struct rspamd_fuzzy_storage_ctx *ctx, } static void -rspamd_fuzzy_reply_io (gint fd, gshort what, gpointer d) +rspamd_fuzzy_reply_io (EV_P_ ev_io *w, int revents) { - struct fuzzy_session *session = d; + struct fuzzy_session *session = (struct fuzzy_session *)w->data; rspamd_fuzzy_write_reply (session); REF_RELEASE (session); @@ -807,10 +559,9 @@ rspamd_fuzzy_write_reply (struct fuzzy_session *session) if (errno == EINTR || errno == EWOULDBLOCK || errno == EAGAIN) { /* Grab reference to avoid early destruction */ REF_RETAIN (session); - event_set (&session->io, session->fd, EV_WRITE, - rspamd_fuzzy_reply_io, session); - event_base_set (session->ctx->ev_base, &session->io); - event_add (&session->io, NULL); + session->io.data = session; + ev_io_init (&session->io, rspamd_fuzzy_reply_io, session->fd, EV_WRITE); + ev_io_start (session->ctx->event_loop, &session->io); } else { msg_err ("error while writing reply: %s", strerror (errno)); @@ -819,22 +570,6 @@ rspamd_fuzzy_write_reply (struct fuzzy_session *session) } static void -fuzzy_peer_send_io (gint fd, gshort what, gpointer d) -{ - struct fuzzy_peer_request *up_req = d; - gssize r; - - r = write (fd, &up_req->cmd, sizeof (up_req->cmd)); - - if (r != sizeof (up_req->cmd)) { - msg_err ("cannot send update request to the peer: %s", strerror (errno)); - } - - event_del (&up_req->io_ev); - g_free (up_req); -} - -static void rspamd_fuzzy_update_stats (struct rspamd_fuzzy_storage_ctx *ctx, enum rspamd_fuzzy_epoch epoch, gboolean matched, @@ -946,6 +681,22 @@ rspamd_fuzzy_make_reply (struct rspamd_fuzzy_cmd *cmd, } static void +fuzzy_peer_send_io (EV_P_ ev_io *w, int revents) +{ + struct fuzzy_peer_request *up_req = (struct fuzzy_peer_request *)w->data; + gssize r; + + r = write (w->fd, &up_req->cmd, sizeof (up_req->cmd)); + + if (r != sizeof (up_req->cmd)) { + msg_err ("cannot send update request to the peer: %s", strerror (errno)); + } + + ev_io_stop (EV_A_ w); + g_free (up_req); +} + +static void rspamd_fuzzy_check_callback (struct rspamd_fuzzy_reply *result, void *ud) { struct fuzzy_session *session = ud; @@ -984,7 +735,7 @@ rspamd_fuzzy_check_callback (struct rspamd_fuzzy_reply *result, void *ud) struct fuzzy_peer_cmd up_cmd; struct fuzzy_peer_request *up_req; - if (session->worker->index == 0 || session->ctx->peer_fd == -1) { + if (session->worker->index == 0) { /* Just add to the queue */ memset (&up_cmd, 0, sizeof (up_cmd)); up_cmd.is_shingle = is_shingle; @@ -1017,10 +768,10 @@ rspamd_fuzzy_check_callback (struct rspamd_fuzzy_reply *result, void *ud) sizeof (up_req->cmd.cmd.shingle.sgl)); } - event_set (&up_req->io_ev, session->ctx->peer_fd, EV_WRITE, - fuzzy_peer_send_io, up_req); - event_base_set (session->ctx->ev_base, &up_req->io_ev); - event_add (&up_req->io_ev, NULL); + up_req->io_ev.data = up_req; + ev_io_init (&up_req->io_ev, fuzzy_peer_send_io, + session->ctx->peer_fd, EV_WRITE); + ev_io_start (session->ctx->event_loop, &up_req->io_ev); } } @@ -1103,17 +854,9 @@ rspamd_fuzzy_process_command (struct fuzzy_session *session) if (cmd->cmd == FUZZY_CHECK) { if (rspamd_fuzzy_check_client (session, FALSE)) { - if (G_UNLIKELY (session->ctx->collection_mode)) { - result.v1.prob = 0; - result.v1.value = 500; - result.v1.flag = 0; - rspamd_fuzzy_make_reply (cmd, &result, session, encrypted, - is_shingle); - } else { - REF_RETAIN (session); - rspamd_fuzzy_backend_check (session->ctx->backend, cmd, - rspamd_fuzzy_check_callback, session); - } + REF_RETAIN (session); + rspamd_fuzzy_backend_check (session->ctx->backend, cmd, + rspamd_fuzzy_check_callback, session); } else { result.v1.value = 403; @@ -1123,18 +866,10 @@ rspamd_fuzzy_process_command (struct fuzzy_session *session) } } else if (cmd->cmd == FUZZY_STAT) { - if (G_UNLIKELY (session->ctx->collection_mode)) { - result.v1.prob = 0; - result.v1.value = 500; - result.v1.flag = 0; - rspamd_fuzzy_make_reply (cmd, &result, session, encrypted, is_shingle); - } - else { - result.v1.prob = 1.0; - result.v1.value = 0; - result.v1.flag = session->ctx->stat.fuzzy_hashes; - rspamd_fuzzy_make_reply (cmd, &result, session, encrypted, is_shingle); - } + result.v1.prob = 1.0; + result.v1.value = 0; + result.v1.flag = session->ctx->stat.fuzzy_hashes; + rspamd_fuzzy_make_reply (cmd, &result, session, encrypted, is_shingle); } else { if (rspamd_fuzzy_check_client (session, TRUE)) { @@ -1169,10 +904,10 @@ rspamd_fuzzy_process_command (struct fuzzy_session *session) (gpointer)&up_req->cmd.cmd.shingle : (gpointer)&up_req->cmd.cmd.normal; memcpy (ptr, cmd, up_len); - event_set (&up_req->io_ev, session->ctx->peer_fd, EV_WRITE, - fuzzy_peer_send_io, up_req); - event_base_set (session->ctx->ev_base, &up_req->io_ev); - event_add (&up_req->io_ev, NULL); + up_req->io_ev.data = up_req; + ev_io_init (&up_req->io_ev, fuzzy_peer_send_io, + session->ctx->peer_fd, EV_WRITE); + ev_io_start (session->ctx->event_loop, &up_req->io_ev); } result.v1.value = 0; @@ -1378,147 +1113,6 @@ rspamd_fuzzy_cmd_from_wire (guchar *buf, guint buflen, struct fuzzy_session *s) return TRUE; } -static void -rspamd_fuzzy_mirror_process_update (struct fuzzy_master_update_session *session, - struct rspamd_http_message *msg, guint our_rev) -{ - const guchar *p; - gsize remain; - gint32 revision; - guint32 len = 0, cnt = 0; - struct fuzzy_peer_cmd cmd; - enum { - read_len = 0, - read_data, - finish_processing - } state = read_len; - - gpointer flag_ptr; - - /* - * Message format: - * <uint32_le> - revision - * <uint32_le> - size of the next element - * <data> - command data - * ... - * <0> - end of data - * ... - ignored - */ - p = rspamd_http_message_get_body (msg, &remain); - - if (p && remain >= sizeof (gint32) * 2) { - memcpy (&revision, p, sizeof (gint32)); - revision = GINT32_TO_LE (revision); - - if (revision <= our_rev) { - msg_err_fuzzy_update ("remote revision: %d is older than ours: %d, " - "refusing update", - revision, our_rev); - - return; - } - else if (revision - our_rev > 1) { - msg_warn_fuzzy_update ("remote revision: %d is newer more than one revision " - "than ours: %d, cold sync is recommended", - revision, our_rev); - } - - remain -= sizeof (gint32); - p += sizeof (gint32); - } - else { - msg_err_fuzzy_update ("short update message, not processing"); - goto err; - } - - while (remain > 0) { - switch (state) { - case read_len: - if (remain < sizeof (guint32)) { - msg_err_fuzzy_update ("short update message while reading " - "length, not processing"); - goto err; - } - - memcpy (&len, p, sizeof (guint32)); - len = GUINT32_TO_LE (len); - remain -= sizeof (guint32); - p += sizeof (guint32); - - if (len == 0) { - remain = 0; - state = finish_processing; - } - else { - state = read_data; - } - break; - case read_data: - if (remain < len) { - msg_err_fuzzy_update ("short update message while reading data, " - "not processing" - " (%zd is available, %d is required)", remain, len); - return; - } - - if (len < sizeof (struct rspamd_fuzzy_cmd) + sizeof (guint32) || - len > sizeof (cmd)) { - /* Bad size command */ - msg_err_fuzzy_update ("incorrect element size: %d, at least " - "%d expected", len, - (gint)(sizeof (struct rspamd_fuzzy_cmd) + sizeof (guint32))); - goto err; - } - - memcpy (&cmd, p, len); - if (cmd.is_shingle && len != sizeof (cmd)) { - /* Short command */ - msg_err_fuzzy_update ("incorrect element size: %d, at least " - "%d expected", len, - (gint)(sizeof (cmd))); - goto err; - } - - if (cmd.is_shingle) { - if ((flag_ptr = g_hash_table_lookup (session->ctx->master_flags, - GUINT_TO_POINTER (cmd.cmd.shingle.basic.flag))) != NULL) { - cmd.cmd.shingle.basic.flag = GPOINTER_TO_UINT (flag_ptr); - } - } - else { - if ((flag_ptr = g_hash_table_lookup (session->ctx->master_flags, - GUINT_TO_POINTER (cmd.cmd.normal.flag))) != NULL) { - cmd.cmd.normal.flag = GPOINTER_TO_UINT (flag_ptr); - } - } - - g_array_append_val (session->ctx->updates_pending, cmd); - - p += len; - remain -= len; - len = 0; - state = read_len; - cnt ++; - break; - case finish_processing: - /* Do nothing */ - remain = 0; - break; - } - } - - - rspamd_fuzzy_process_updates_queue (session->ctx, session->src, TRUE); - msg_info_fuzzy_update ("processed updates from the master %s, " - "%ud operations processed," - " revision: %d (local revision: %d)", - rspamd_inet_address_to_string (session->addr), - cnt, revision, our_rev); - -err: - return; -} - static void fuzzy_session_destroy (gpointer d) @@ -1531,492 +1125,13 @@ fuzzy_session_destroy (gpointer d) g_free (session); } -static void -rspamd_fuzzy_mirror_session_destroy (struct fuzzy_master_update_session *session) -{ - if (session) { - rspamd_http_connection_reset (session->conn); - rspamd_http_connection_unref (session->conn); - rspamd_inet_address_free (session->addr); - close (session->sock); - - if (session->psrc) { - g_free (session->psrc); - } - g_free (session); - } -} - -static void -rspamd_fuzzy_mirror_error_handler (struct rspamd_http_connection *conn, GError *err) -{ - struct fuzzy_master_update_session *session = conn->ud; - - msg_err_fuzzy_update ("abnormally closing connection from: %s, error: %e", - rspamd_inet_address_to_string (session->addr), err); - /* Terminate session immediately */ - rspamd_fuzzy_mirror_session_destroy (session); -} - -static void -rspamd_fuzzy_mirror_send_reply (struct fuzzy_master_update_session *session, - guint code, const gchar *str) -{ - struct rspamd_http_message *msg; - - msg = rspamd_http_new_message (HTTP_RESPONSE); - msg->url = rspamd_fstring_new_init (str, strlen (str)); - msg->code = code; - session->replied = TRUE; - - rspamd_http_connection_reset (session->conn); - rspamd_http_connection_write_message (session->conn, msg, NULL, "text/plain", - session, &session->ctx->master_io_tv); -} - -static void -rspamd_fuzzy_update_version_callback (guint64 version, void *ud) -{ - struct fuzzy_master_update_session *session = ud; - - rspamd_fuzzy_mirror_process_update (session, session->msg, version); - rspamd_fuzzy_mirror_send_reply (session, 200, "OK"); -} - -static gint -rspamd_fuzzy_mirror_finish_handler (struct rspamd_http_connection *conn, - struct rspamd_http_message *msg) -{ - struct fuzzy_master_update_session *session = conn->ud; - const struct rspamd_cryptobox_pubkey *rk; - const gchar *err_str = NULL; - gchar *psrc; - const gchar *src = NULL; - gsize remain; - - if (session->replied) { - rspamd_fuzzy_mirror_session_destroy (session); - - return 0; - } - - /* Check key */ - if (!rspamd_http_connection_is_encrypted (conn)) { - msg_err_fuzzy_update ("refuse unencrypted update from: %s", - rspamd_inet_address_to_string (session->addr)); - err_str = "Unencrypted update is not allowed"; - goto end; - } - else { - - if (session->ctx->master_key) { - rk = rspamd_http_connection_get_peer_key (conn); - g_assert (rk != NULL); - - if (!rspamd_pubkey_equal (rk, session->ctx->master_key)) { - msg_err_fuzzy_update ("refuse unknown pubkey update from: %s", - rspamd_inet_address_to_string (session->addr)); - err_str = "Unknown pubkey"; - goto end; - } - } - else { - msg_warn_fuzzy_update ("no trusted key specified, accept any update from %s", - rspamd_inet_address_to_string (session->addr)); - } - if (!rspamd_http_message_get_body (msg, NULL) || !msg->url - || msg->url->len == 0) { - msg_err_fuzzy_update ("empty update message, not processing"); - err_str = "Empty update"; - - goto end; - } - - /* Detect source from url: /update_v1/<source>, so we look for the last '/' */ - remain = msg->url->len; - psrc = rspamd_fstringdup (msg->url); - src = psrc; - - while (remain--) { - if (src[remain] == '/') { - src = &src[remain + 1]; - break; - } - } - - session->src = src; - session->psrc = psrc; - session->msg = msg; - rspamd_fuzzy_backend_version (session->ctx->backend, src, - rspamd_fuzzy_update_version_callback, session); - - return 0; - } - -end: - rspamd_fuzzy_mirror_send_reply (session, 403, err_str); - - return 0; -} - -struct rspamd_fuzzy_collection_session { - struct rspamd_fuzzy_storage_ctx *ctx; - struct rspamd_worker *worker; - rspamd_inet_addr_t *from_addr; - guchar uid[16]; -}; - -static void -rspamd_fuzzy_collection_error_handler (struct rspamd_http_connection_entry *conn_ent, - GError *err) -{ - struct rspamd_fuzzy_collection_session *session = conn_ent->ud; - - msg_err_fuzzy_collection ("http error occurred: %s", err->message); -} - -static void -rspamd_fuzzy_collection_finish_handler (struct rspamd_http_connection_entry *conn_ent) -{ - struct rspamd_fuzzy_collection_session *session = conn_ent->ud; - - - rspamd_inet_address_free (session->from_addr); - g_free (session); -} - -void -rspamd_fuzzy_collection_send_error (struct rspamd_http_connection_entry *entry, - gint code, const gchar *error_msg, ...) -{ - struct rspamd_http_message *msg; - va_list args; - rspamd_fstring_t *reply; - - msg = rspamd_http_new_message (HTTP_RESPONSE); - - va_start (args, error_msg); - msg->status = rspamd_fstring_new (); - rspamd_vprintf_fstring (&msg->status, error_msg, args); - va_end (args); - - msg->date = time (NULL); - msg->code = code; - reply = rspamd_fstring_sized_new (msg->status->len + 16); - rspamd_printf_fstring (&reply, "%V", msg->status); - rspamd_http_message_set_body_from_fstring_steal (msg, reply); - rspamd_http_connection_reset (entry->conn); - rspamd_http_router_insert_headers (entry->rt, msg); - rspamd_http_connection_write_message (entry->conn, - msg, - NULL, - "text/plain", - entry, - entry->rt->ptv); - entry->is_reply = TRUE; -} - -/* - * Note: this function steals fstring - */ -void -rspamd_fuzzy_collection_send_fstring (struct rspamd_http_connection_entry *entry, - rspamd_fstring_t *fstr) -{ - struct rspamd_http_message *msg; - - msg = rspamd_http_new_message (HTTP_RESPONSE); - msg->status = rspamd_fstring_new_init ("OK", 2); - msg->date = time (NULL); - msg->code = 200; - rspamd_http_message_set_body_from_fstring_steal (msg, fstr); - rspamd_http_connection_reset (entry->conn); - rspamd_http_router_insert_headers (entry->rt, msg); - rspamd_http_connection_write_message (entry->conn, - msg, - NULL, - "application/octet-stream", - entry, - entry->rt->ptv); - entry->is_reply = TRUE; -} - -static int -rspamd_fuzzy_collection_cookie (struct rspamd_http_connection_entry *conn_ent, - struct rspamd_http_message *msg) -{ - struct rspamd_fuzzy_collection_session *session = conn_ent->ud; - rspamd_fstring_t *cookie; - - cookie = rspamd_fstring_new_init (session->ctx->cookie, - sizeof (session->ctx->cookie)); - rspamd_fuzzy_collection_send_fstring (conn_ent, cookie); - - return 0; -} - -static int -rspamd_fuzzy_collection_data (struct rspamd_http_connection_entry *conn_ent, - struct rspamd_http_message *msg) -{ - struct rspamd_fuzzy_collection_session *session = conn_ent->ud; - const rspamd_ftok_t *sign_header; - struct rspamd_fuzzy_storage_ctx *ctx; - guint i; - struct fuzzy_peer_cmd *io_cmd; - rspamd_fstring_t *reply; - GError *err = NULL; - guchar *decoded_signature; - gsize dec_len; - guint32 cmdlen, nupdates = 0; - - sign_header = rspamd_http_message_find_header (msg, "Signature"); - - if (sign_header == NULL) { - rspamd_fuzzy_collection_send_error (conn_ent, 403, "Missing signature"); - - return 0; - } - - ctx = session->ctx; - - if (ctx->collection_sign_key == NULL) { - rspamd_fuzzy_collection_send_error (conn_ent, 500, "Misconfigured signature key"); - - return 0; - } - - decoded_signature = g_malloc (sign_header->len * 2 + 1); - dec_len = rspamd_decode_hex_buf (sign_header->begin, sign_header->len, - decoded_signature, sign_header->len * 2 + 1); - - if (dec_len == -1 || !rspamd_keypair_verify (ctx->collection_sign_key, - ctx->cookie, sizeof (ctx->cookie), - decoded_signature, dec_len, &err)) { - if (err) { - rspamd_fuzzy_collection_send_error (conn_ent, 403, "Signature verification error: %e", - err); - g_error_free (err); - } - else { - rspamd_fuzzy_collection_send_error (conn_ent, 403, "Signature verification error"); - } - - g_free (decoded_signature); - - return 0; - } - - g_free (decoded_signature); - - /* Generate new cookie */ - ottery_rand_bytes (ctx->cookie, sizeof (ctx->cookie)); - - /* Send&Clear updates */ - reply = rspamd_fstring_sized_new (8192); - /* - * Message format: - * <uint32_le> - revision - * <uint32_le> - size of the next element - * <data> - command data - * ... - * <0> - end of data - * ... - ignored - */ - reply = rspamd_fstring_append (reply, (const gchar *)&ctx->collection_id, - sizeof (ctx->collection_id)); - - for (i = 0; i < ctx->updates_pending->len; i ++) { - io_cmd = &g_array_index (ctx->updates_pending, struct fuzzy_peer_cmd, i); - - if (io_cmd->is_shingle) { - cmdlen = sizeof (io_cmd->cmd.shingle) + sizeof (guint32); - - } - else { - cmdlen = sizeof (io_cmd->cmd.normal) + sizeof (guint32); - } - - cmdlen = GUINT32_TO_LE (cmdlen); - reply = rspamd_fstring_append (reply, (const gchar *)&cmdlen, - sizeof (cmdlen)); - reply = rspamd_fstring_append (reply, (const gchar *)io_cmd, - cmdlen); - nupdates ++; - } - - msg_info_fuzzy_collection ("collection %d done, send %d updates", - ctx->collection_id, nupdates); - /* Last command */ - cmdlen = 0; - reply = rspamd_fstring_append (reply, (const gchar *)&cmdlen, - sizeof (cmdlen)); - - ctx->updates_pending->len = 0; - /* Clear failed attempts counter */ - ctx->updates_failed = 0; - ctx->collection_id ++; - rspamd_fuzzy_collection_send_fstring (conn_ent, reply); - - return 0; -} - - -static void -accept_fuzzy_collection_socket (gint fd, short what, void *arg) -{ - struct rspamd_worker *worker = (struct rspamd_worker *)arg; - rspamd_inet_addr_t *addr; - gint nfd; - struct rspamd_fuzzy_storage_ctx *ctx; - struct rspamd_fuzzy_collection_session *session; - - if ((nfd = - rspamd_accept_from_socket (fd, &addr, worker->accept_events)) == -1) { - msg_warn ("accept failed: %s", strerror (errno)); - return; - } - /* Check for EAGAIN */ - if (nfd == 0) { - return; - } - - ctx = worker->ctx; - - if (!ctx->collection_keypair) { - msg_err ("deny request from %s, as no local keypair is specified", - rspamd_inet_address_to_string (addr)); - rspamd_inet_address_free (addr); - close (nfd); - - return; - } - - session = g_malloc0 (sizeof (*session)); - session->ctx = ctx; - session->worker = worker; - rspamd_random_hex (session->uid, sizeof (session->uid) - 1); - session->uid[sizeof (session->uid) - 1] = '\0'; - session->from_addr = addr; - rspamd_http_router_handle_socket (ctx->collection_rt, nfd, session); - msg_info_fuzzy_collection ("accepted connection from %s port %d, session ptr: %p", - rspamd_inet_address_to_string (addr), - rspamd_inet_address_get_port (addr), - session); -} - -static void -rspamd_fuzzy_collection_periodic (gint fd, gshort what, gpointer ud) -{ - struct rspamd_fuzzy_storage_ctx *ctx = ud; - - if (++ctx->updates_failed > ctx->updates_maxfail) { - msg_err ("cannot store more data in workqueue, discard " - "%ud updates after %d missed collection points", - ctx->updates_pending->len, - ctx->updates_maxfail); - ctx->updates_failed = 0; - ctx->updates_pending->len = 0; - /* Regenerate cookie */ - ottery_rand_bytes (ctx->cookie, sizeof (ctx->cookie)); - } - else { - msg_err ("fuzzy data has not been collected in time, " - "%ud updates are still pending, %d updates left", - ctx->updates_pending->len, - ctx->updates_maxfail - ctx->updates_failed); - } - - if (ctx->worker->wanna_die) { - /* Plan exit */ - struct timeval tv; - - tv.tv_sec = 0; - tv.tv_usec = 0; - - event_base_loopexit (ctx->ev_base, &tv); - } -} - - -static void -accept_fuzzy_mirror_socket (gint fd, short what, void *arg) -{ - struct rspamd_worker *worker = (struct rspamd_worker *)arg; - rspamd_inet_addr_t *addr; - gint nfd; - struct rspamd_http_connection *http_conn; - struct rspamd_fuzzy_storage_ctx *ctx; - struct fuzzy_master_update_session *session; - - if ((nfd = - rspamd_accept_from_socket (fd, &addr, worker->accept_events)) == -1) { - msg_warn ("accept failed: %s", strerror (errno)); - return; - } - /* Check for EAGAIN */ - if (nfd == 0) { - return; - } - - ctx = worker->ctx; - - if (!ctx->master_ips) { - msg_err ("deny update request from %s as no masters defined", - rspamd_inet_address_to_string (addr)); - rspamd_inet_address_free (addr); - close (nfd); - - return; - } - else if (rspamd_match_radix_map_addr (ctx->master_ips, addr) == NULL) { - msg_err ("deny update request from %s", - rspamd_inet_address_to_string (addr)); - rspamd_inet_address_free (addr); - close (nfd); - - return; - } - - if (!ctx->sync_keypair) { - msg_err ("deny update request from %s, as no local keypair is specified", - rspamd_inet_address_to_string (addr)); - rspamd_inet_address_free (addr); - close (nfd); - - return; - } - - session = g_malloc0 (sizeof (*session)); - session->name = rspamd_inet_address_to_string (addr); - rspamd_random_hex (session->uid, sizeof (session->uid) - 1); - session->uid[sizeof (session->uid) - 1] = '\0'; - http_conn = rspamd_http_connection_new_server ( - ctx->http_ctx, - nfd, - NULL, - rspamd_fuzzy_mirror_error_handler, - rspamd_fuzzy_mirror_finish_handler, - 0); - - rspamd_http_connection_set_key (http_conn, ctx->sync_keypair); - session->ctx = ctx; - session->conn = http_conn; - session->addr = addr; - session->sock = nfd; - - rspamd_http_connection_read_message (http_conn, - session, - &ctx->master_io_tv); -} - /* * Accept new connection and construct task */ static void -accept_fuzzy_socket (gint fd, short what, void *arg) +accept_fuzzy_socket (EV_P_ ev_io *w, int revents) { - struct rspamd_worker *worker = (struct rspamd_worker *)arg; + struct rspamd_worker *worker = (struct rspamd_worker *)w->data; struct fuzzy_session *session; rspamd_inet_addr_t *addr; gssize r; @@ -2024,12 +1139,12 @@ accept_fuzzy_socket (gint fd, short what, void *arg) guint64 *nerrors; /* Got some data */ - if (what == EV_READ) { + if (revents == EV_READ) { for (;;) { worker->nconns++; - r = rspamd_inet_address_recvfrom (fd, + r = rspamd_inet_address_recvfrom (w->fd, buf, sizeof (buf), 0, @@ -2053,7 +1168,7 @@ accept_fuzzy_socket (gint fd, short what, void *arg) session = g_malloc0 (sizeof (*session)); REF_INIT_RETAIN (session, fuzzy_session_destroy); session->worker = worker; - session->fd = fd; + session->fd = w->fd; session->ctx = worker->ctx; session->time = (guint64) time (NULL); session->addr = addr; @@ -2148,7 +1263,7 @@ rspamd_fuzzy_storage_reload (struct rspamd_main *rspamd_main, memset (&rep, 0, sizeof (rep)); rep.type = RSPAMD_CONTROL_RELOAD; - if ((ctx->backend = rspamd_fuzzy_backend_create (ctx->ev_base, + if ((ctx->backend = rspamd_fuzzy_backend_create (ctx->event_loop, worker->cf->options, rspamd_main->cfg, &err)) == NULL) { msg_err ("cannot open backend after reload: %e", err); @@ -2391,120 +1506,6 @@ rspamd_fuzzy_storage_stat (struct rspamd_main *rspamd_main, } static gboolean -fuzzy_storage_parse_mirror (rspamd_mempool_t *pool, - const ucl_object_t *obj, - gpointer ud, - struct rspamd_rcl_section *section, - GError **err) -{ - const ucl_object_t *elt; - struct rspamd_fuzzy_mirror *up = NULL; - struct rspamd_rcl_struct_parser *pd = ud; - struct rspamd_fuzzy_storage_ctx *ctx; - - ctx = pd->user_struct; - - if (ucl_object_type (obj) != UCL_OBJECT) { - g_set_error (err, g_quark_try_string ("fuzzy"), 100, - "mirror/slave option must be an object"); - - return FALSE; - } - - elt = ucl_object_lookup (obj, "name"); - if (elt == NULL) { - g_set_error (err, g_quark_try_string ("fuzzy"), 100, - "mirror option must have some name definition"); - - return FALSE; - } - - up = g_malloc0 (sizeof (*up)); - up->name = g_strdup (ucl_object_tostring (elt)); - - elt = ucl_object_lookup (obj, "key"); - if (elt != NULL) { - up->key = rspamd_pubkey_from_base32 (ucl_object_tostring (elt), 0, - RSPAMD_KEYPAIR_KEX, RSPAMD_CRYPTOBOX_MODE_25519); - } - - if (up->key == NULL) { - g_set_error (err, g_quark_try_string ("fuzzy"), 100, - "cannot read mirror key"); - - goto err; - } - - elt = ucl_object_lookup (obj, "hosts"); - - if (elt == NULL) { - g_set_error (err, g_quark_try_string ("fuzzy"), 100, - "mirror option must have some hosts definition"); - - goto err; - } - - up->u = rspamd_upstreams_create (ctx->cfg->ups_ctx); - if (!rspamd_upstreams_from_ucl (up->u, elt, 11335, NULL)) { - g_set_error (err, g_quark_try_string ("fuzzy"), 100, - "mirror has bad hosts definition"); - - goto err; - } - - g_ptr_array_add (ctx->mirrors, up); - - return TRUE; - -err: - g_free (up->name); - rspamd_upstreams_destroy (up->u); - - if (up->key) { - rspamd_pubkey_unref (up->key); - } - - g_free (up); - - return FALSE; -} - -static gboolean -fuzzy_storage_parse_master_flags (rspamd_mempool_t *pool, - const ucl_object_t *obj, - gpointer ud, - struct rspamd_rcl_section *section, - GError **err) -{ - const ucl_object_t *cur; - struct rspamd_rcl_struct_parser *pd = ud; - struct rspamd_fuzzy_storage_ctx *ctx; - ucl_object_iter_t it = NULL; - gulong remote_flag; - gint64 local_flag; - - ctx = pd->user_struct; - - if (ucl_object_type (obj) != UCL_OBJECT) { - g_set_error (err, g_quark_try_string ("fuzzy"), 100, - "master_flags option must be an object"); - - return FALSE; - } - - while ((cur = ucl_iterate_object (obj, &it, true)) != NULL) { - if (rspamd_strtoul (cur->key, cur->keylen, &remote_flag) && - ucl_object_toint_safe (cur, (int64_t *)&local_flag)) { - g_hash_table_insert (ctx->master_flags, GUINT_TO_POINTER (remote_flag), - GUINT_TO_POINTER (local_flag)); - } - } - - return TRUE; -} - - -static gboolean fuzzy_parse_keypair (rspamd_mempool_t *pool, const ucl_object_t *obj, gpointer ud, @@ -2599,26 +1600,18 @@ init_fuzzy (struct rspamd_config *cfg) ctx->magic = rspamd_fuzzy_storage_magic; ctx->sync_timeout = DEFAULT_SYNC_TIMEOUT; - ctx->master_timeout = DEFAULT_MASTER_TIMEOUT; ctx->keypair_cache_size = DEFAULT_KEYPAIR_CACHE_SIZE; ctx->keys = g_hash_table_new_full (fuzzy_kp_hash, fuzzy_kp_equal, NULL, fuzzy_key_dtor); rspamd_mempool_add_destructor (cfg->cfg_pool, (rspamd_mempool_destruct_t)g_hash_table_unref, ctx->keys); - ctx->master_flags = g_hash_table_new (g_direct_hash, g_direct_equal); - rspamd_mempool_add_destructor (cfg->cfg_pool, - (rspamd_mempool_destruct_t)g_hash_table_unref, ctx->master_flags); ctx->errors_ips = rspamd_lru_hash_new_full (1024, (GDestroyNotify) rspamd_inet_address_free, g_free, rspamd_inet_address_hash, rspamd_inet_address_equal); rspamd_mempool_add_destructor (cfg->cfg_pool, (rspamd_mempool_destruct_t)rspamd_lru_hash_destroy, ctx->errors_ips); ctx->cfg = cfg; - ctx->mirrors = g_ptr_array_new (); - rspamd_mempool_add_destructor (cfg->cfg_pool, - (rspamd_mempool_destruct_t)rspamd_ptr_array_free_hard, ctx->mirrors); ctx->updates_maxfail = DEFAULT_UPDATES_MAXFAIL; - ctx->collection_id_file = RSPAMD_DBDIR "/fuzzy_collection.id"; ctx->leaky_bucket_mask = DEFAULT_BUCKET_MASK; ctx->leaky_bucket_ttl = DEFAULT_BUCKET_TTL; ctx->max_buckets = DEFAULT_MAX_BUCKETS; @@ -2694,32 +1687,6 @@ init_fuzzy (struct rspamd_config *cfg) 0, "Work in read only mode"); - rspamd_rcl_register_worker_option (cfg, - type, - "master_timeout", - rspamd_rcl_parse_struct_time, - ctx, - G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, master_timeout), - RSPAMD_CL_FLAG_TIME_FLOAT, - "Master protocol IO timeout"); - - rspamd_rcl_register_worker_option (cfg, - type, - "sync_keypair", - rspamd_rcl_parse_struct_keypair, - ctx, - G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, sync_keypair), - 0, - "Encryption key for master/slave updates"); - - rspamd_rcl_register_worker_option (cfg, - type, - "masters", - rspamd_rcl_parse_struct_ucl, - ctx, - G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, masters_map), - 0, - "Allow master/slave updates from the following IP addresses"); rspamd_rcl_register_worker_option (cfg, type, @@ -2730,43 +1697,9 @@ init_fuzzy (struct rspamd_config *cfg) 0, "Block requests from specific networks"); - rspamd_rcl_register_worker_option (cfg, - type, - "master_key", - rspamd_rcl_parse_struct_pubkey, - ctx, - G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, master_key), - 0, - "Allow master/slave updates merely using the specified key"); rspamd_rcl_register_worker_option (cfg, type, - "mirror", - fuzzy_storage_parse_mirror, - ctx, - 0, - RSPAMD_CL_FLAG_MULTIPLE, - "List of slave hosts"); - - rspamd_rcl_register_worker_option (cfg, - type, - "slave", - fuzzy_storage_parse_mirror, - ctx, - 0, - RSPAMD_CL_FLAG_MULTIPLE, - "List of slave hosts"); - - rspamd_rcl_register_worker_option (cfg, - type, - "master_flags", - fuzzy_storage_parse_master_flags, - ctx, - 0, - 0, - "Map of flags in form master_flags = { master_flag = local_flag; ... }; "); - rspamd_rcl_register_worker_option (cfg, - type, "updates_maxfail", rspamd_rcl_parse_struct_integer, ctx, @@ -2775,38 +1708,6 @@ init_fuzzy (struct rspamd_config *cfg) "Maximum number of updates to be failed before discarding"); rspamd_rcl_register_worker_option (cfg, type, - "collection_only", - rspamd_rcl_parse_struct_boolean, - ctx, - G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, collection_mode), - 0, - "Start fuzzy in collection only mode"); - rspamd_rcl_register_worker_option (cfg, - type, - "collection_signkey", - rspamd_rcl_parse_struct_pubkey, - ctx, - G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, collection_sign_key), - RSPAMD_CL_FLAG_SIGNKEY, - "Accept only signed requests with the specified key"); - rspamd_rcl_register_worker_option (cfg, - type, - "collection_keypair", - rspamd_rcl_parse_struct_keypair, - ctx, - G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, collection_keypair), - 0, - "Use the specified keypair to encrypt collection protocol"); - rspamd_rcl_register_worker_option (cfg, - type, - "collection_id_file", - rspamd_rcl_parse_struct_string, - ctx, - G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, collection_id_file), - RSPAMD_CL_FLAG_STRING_PATH, - "Store collection epoch in the desired file"); - rspamd_rcl_register_worker_option (cfg, - type, "skip_hashes", rspamd_rcl_parse_struct_ucl, ctx, @@ -2877,17 +1778,18 @@ init_fuzzy (struct rspamd_config *cfg) } static void -rspamd_fuzzy_peer_io (gint fd, gshort what, gpointer d) +rspamd_fuzzy_peer_io (EV_P_ ev_io *w, int revents) { struct fuzzy_peer_cmd cmd; - struct rspamd_fuzzy_storage_ctx *ctx = d; + struct rspamd_fuzzy_storage_ctx *ctx = + (struct rspamd_fuzzy_storage_ctx *)w->data; gssize r; - r = read (fd, &cmd, sizeof (cmd)); + r = read (w->fd, &cmd, sizeof (cmd)); if (r != sizeof (cmd)) { if (errno == EINTR) { - rspamd_fuzzy_peer_io (fd, what, d); + rspamd_fuzzy_peer_io (EV_A_ w, revents); return; } if (errno != EAGAIN) { @@ -2907,7 +1809,7 @@ fuzzy_peer_rep (struct rspamd_worker *worker, struct rspamd_fuzzy_storage_ctx *ctx = ud; GList *cur; struct rspamd_worker_listen_socket *ls; - struct event *accept_events; + struct rspamd_worker_accept_event *ac_ev; ctx->peer_fd = rep_fd; @@ -2931,30 +1833,17 @@ fuzzy_peer_rep (struct rspamd_worker *worker, rspamd_inet_address_to_string_pretty (ls->addr)); if (ls->type == RSPAMD_WORKER_SOCKET_UDP) { - accept_events = g_malloc0 (sizeof (struct event) * 2); - event_set (&accept_events[0], ls->fd, EV_READ | EV_PERSIST, - accept_fuzzy_socket, worker); - event_base_set (ctx->ev_base, &accept_events[0]); - event_add (&accept_events[0], NULL); - worker->accept_events = g_list_prepend (worker->accept_events, - accept_events); + ac_ev = g_malloc0 (sizeof (*ac_ev)); + ac_ev->accept_ev.data = worker; + ac_ev->event_loop = ctx->event_loop; + ev_io_init (&ac_ev->accept_ev, accept_fuzzy_socket, ls->fd, + EV_READ); + ev_io_start (ctx->event_loop, &ac_ev->accept_ev); + DL_APPEND (worker->accept_events, ac_ev); } - else if (worker->index == 0) { + else { /* We allow TCP listeners only for a update worker */ - accept_events = g_malloc0 (sizeof (struct event) * 2); - - if (ctx->collection_mode) { - event_set (&accept_events[0], ls->fd, EV_READ | EV_PERSIST, - accept_fuzzy_collection_socket, worker); - } - else { - event_set (&accept_events[0], ls->fd, EV_READ | EV_PERSIST, - accept_fuzzy_mirror_socket, worker); - } - event_base_set (ctx->ev_base, &accept_events[0]); - event_add (&accept_events[0], NULL); - worker->accept_events = g_list_prepend (worker->accept_events, - accept_events); + g_assert_not_reached (); } } @@ -2963,10 +1852,9 @@ fuzzy_peer_rep (struct rspamd_worker *worker, if (worker->index == 0 && ctx->peer_fd != -1) { /* Listen for peer requests */ - event_set (&ctx->peer_ev, ctx->peer_fd, EV_READ | EV_PERSIST, - rspamd_fuzzy_peer_io, ctx); - event_base_set (ctx->ev_base, &ctx->peer_ev); - event_add (&ctx->peer_ev, NULL); + ctx->peer_ev.data = ctx; + ev_io_init (&ctx->peer_ev, rspamd_fuzzy_peer_io, ctx->peer_fd, EV_READ); + ev_io_start (ctx->event_loop, &ctx->peer_ev); } } @@ -2981,140 +1869,53 @@ start_fuzzy (struct rspamd_worker *worker) struct rspamd_srv_command srv_cmd; struct rspamd_config *cfg = worker->srv->cfg; - ctx->ev_base = rspamd_prepare_worker (worker, + ctx->event_loop = rspamd_prepare_worker (worker, "fuzzy", NULL); ctx->peer_fd = -1; ctx->worker = worker; ctx->cfg = worker->srv->cfg; - double_to_tv (ctx->master_timeout, &ctx->master_io_tv); - ctx->resolver = rspamd_dns_resolver_init (worker->srv->logger, - ctx->ev_base, + ctx->event_loop, worker->srv->cfg); rspamd_upstreams_library_config (worker->srv->cfg, ctx->cfg->ups_ctx, - ctx->ev_base, ctx->resolver->r); + ctx->event_loop, ctx->resolver->r); if (ctx->keypair_cache_size > 0) { /* Create keypairs cache */ ctx->keypair_cache = rspamd_keypair_cache_new (ctx->keypair_cache_size); } - ctx->http_ctx = rspamd_http_context_create (cfg, ctx->ev_base, ctx->cfg->ups_ctx); - - if (!ctx->collection_mode) { - /* - * Open DB and perform VACUUM - */ - if ((ctx->backend = rspamd_fuzzy_backend_create (ctx->ev_base, - worker->cf->options, cfg, &err)) == NULL) { - msg_err ("cannot open backend: %e", err); - if (err) { - g_error_free (err); - } - exit (EXIT_SUCCESS); - } - - rspamd_fuzzy_backend_count (ctx->backend, fuzzy_count_callback, ctx); - - if (worker->index == 0) { - ctx->updates_pending = g_array_sized_new (FALSE, FALSE, - sizeof (struct fuzzy_peer_cmd), 1024); - rspamd_fuzzy_backend_start_update (ctx->backend, ctx->sync_timeout, - rspamd_fuzzy_storage_periodic_callback, ctx); + if ((ctx->backend = rspamd_fuzzy_backend_create (ctx->event_loop, + worker->cf->options, cfg, &err)) == NULL) { + msg_err ("cannot open backend: %e", err); + if (err) { + g_error_free (err); } - - double_to_tv (ctx->sync_timeout, &ctx->stat_tv); - event_set (&ctx->stat_ev, -1, EV_TIMEOUT, rspamd_fuzzy_stat_callback, ctx); - event_base_set (ctx->ev_base, &ctx->stat_ev); - event_add (&ctx->stat_ev, &ctx->stat_tv); - - /* Register custom reload and stat commands for the control socket */ - rspamd_control_worker_add_cmd_handler (worker, RSPAMD_CONTROL_RELOAD, - rspamd_fuzzy_storage_reload, ctx); - rspamd_control_worker_add_cmd_handler (worker, RSPAMD_CONTROL_FUZZY_STAT, - rspamd_fuzzy_storage_stat, ctx); - rspamd_control_worker_add_cmd_handler (worker, RSPAMD_CONTROL_FUZZY_SYNC, - rspamd_fuzzy_storage_sync, ctx); + exit (EXIT_SUCCESS); } - else { - /* - * In collection mode we do a different thing: - * we collect fuzzy hashes in the updates queue and ignore all read commands - */ - if (worker->index == 0) { - ctx->updates_pending = g_array_sized_new (FALSE, FALSE, - sizeof (struct fuzzy_peer_cmd), 1024); - double_to_tv (ctx->sync_timeout, &ctx->stat_tv); - event_set (&ctx->stat_ev, -1, EV_TIMEOUT|EV_PERSIST, - rspamd_fuzzy_collection_periodic, ctx); - event_base_set (ctx->ev_base, &ctx->stat_ev); - event_add (&ctx->stat_ev, &ctx->stat_tv); - - ctx->collection_rt = rspamd_http_router_new ( - rspamd_fuzzy_collection_error_handler, - rspamd_fuzzy_collection_finish_handler, - &ctx->stat_tv, - NULL, - ctx->http_ctx); - - if (ctx->collection_keypair) { - rspamd_http_router_set_key (ctx->collection_rt, - ctx->collection_keypair); - } - - /* Try to load collection id */ - if (ctx->collection_id_file) { - gint fd; - fd = rspamd_file_xopen (ctx->collection_id_file, O_RDONLY, 0, - FALSE); + rspamd_fuzzy_backend_count (ctx->backend, fuzzy_count_callback, ctx); - if (fd == -1) { - if (errno != ENOENT) { - msg_err ("cannot open collection id from %s: %s", - ctx->collection_id_file, strerror (errno)); - } - - ctx->collection_id = 0; - } - else { - if (read (fd, &ctx->collection_id, - sizeof (ctx->collection_id)) == -1) { - msg_err ("cannot read collection id from %s: %s", - ctx->collection_id_file, strerror (errno)); - ctx->collection_id = 0; - } - - close (fd); - } - } - /* Generate new cookie */ - ottery_rand_bytes (ctx->cookie, sizeof (ctx->cookie)); - /* Register paths */ - rspamd_http_router_add_path (ctx->collection_rt, - "/cookie", - rspamd_fuzzy_collection_cookie); - rspamd_http_router_add_path (ctx->collection_rt, - "/data", - rspamd_fuzzy_collection_data); - } + if (worker->index == 0) { + ctx->updates_pending = g_array_sized_new (FALSE, FALSE, + sizeof (struct fuzzy_peer_cmd), 1024); + rspamd_fuzzy_backend_start_update (ctx->backend, ctx->sync_timeout, + rspamd_fuzzy_storage_periodic_callback, ctx); } - if (ctx->mirrors && ctx->mirrors->len != 0) { - if (ctx->sync_keypair == NULL) { - GString *pk_str = NULL; - - ctx->sync_keypair = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX, - RSPAMD_CRYPTOBOX_MODE_25519); - pk_str = rspamd_keypair_print (ctx->sync_keypair, - RSPAMD_KEYPAIR_COMPONENT_PK|RSPAMD_KEYPAIR_BASE32); - msg_warn_config ("generating new temporary keypair for communicating" - " with slave hosts, pk is %s", pk_str->str); - g_string_free (pk_str, TRUE); - } - } + ctx->stat_ev.data = ctx; + ev_timer_init (&ctx->stat_ev, rspamd_fuzzy_stat_callback, ctx->sync_timeout, + ctx->sync_timeout); + ev_timer_start (ctx->event_loop, &ctx->stat_ev); + /* Register custom reload and stat commands for the control socket */ + rspamd_control_worker_add_cmd_handler (worker, RSPAMD_CONTROL_RELOAD, + rspamd_fuzzy_storage_reload, ctx); + rspamd_control_worker_add_cmd_handler (worker, RSPAMD_CONTROL_FUZZY_STAT, + rspamd_fuzzy_storage_stat, ctx); + rspamd_control_worker_add_cmd_handler (worker, RSPAMD_CONTROL_FUZZY_SYNC, + rspamd_fuzzy_storage_sync, ctx); /* Create radix trees */ if (ctx->update_map != NULL) { @@ -3123,12 +1924,6 @@ start_fuzzy (struct rspamd_worker *worker) &ctx->update_ips, NULL); } - if (ctx->masters_map != NULL) { - rspamd_config_radix_from_ucl (worker->srv->cfg, ctx->masters_map, - "Allow fuzzy master/slave updates from specified addresses", - &ctx->master_ips, NULL); - } - if (ctx->skip_map != NULL) { struct rspamd_map *m; @@ -3168,9 +1963,9 @@ start_fuzzy (struct rspamd_worker *worker) /* Maps events */ ctx->resolver = rspamd_dns_resolver_init (worker->srv->logger, - ctx->ev_base, + ctx->event_loop, worker->srv->cfg); - rspamd_map_watch (worker->srv->cfg, ctx->ev_base, ctx->resolver, worker, 0); + rspamd_map_watch (worker->srv->cfg, ctx->event_loop, ctx->resolver, worker, 0); /* Get peer pipe */ memset (&srv_cmd, 0, sizeof (srv_cmd)); @@ -3180,64 +1975,35 @@ start_fuzzy (struct rspamd_worker *worker) memset (srv_cmd.cmd.spair.pair_id, 0, sizeof (srv_cmd.cmd.spair.pair_id)); memcpy (srv_cmd.cmd.spair.pair_id, "fuzzy", sizeof ("fuzzy")); - rspamd_srv_send_command (worker, ctx->ev_base, &srv_cmd, -1, + rspamd_srv_send_command (worker, ctx->event_loop, &srv_cmd, -1, fuzzy_peer_rep, ctx); - event_base_loop (ctx->ev_base, 0); + ev_loop (ctx->event_loop, 0); rspamd_worker_block_signals (); - if (worker->index == 0 && ctx->updates_pending->len > 0) { - if (!ctx->collection_mode) { - rspamd_fuzzy_process_updates_queue (ctx, local_db_name, FALSE); - event_base_loop (ctx->ev_base, 0); + if (ctx->peer_fd != -1) { + if (worker->index == 0) { + ev_io_stop (ctx->event_loop, &ctx->peer_ev); } + close (ctx->peer_fd); } - if (!ctx->collection_mode) { - rspamd_fuzzy_backend_close (ctx->backend); + if (worker->index == 0 && ctx->updates_pending->len > 0) { + rspamd_fuzzy_process_updates_queue (ctx, local_db_name, FALSE); + ev_loop (ctx->event_loop, 0); } - else if (worker->index == 0) { - gint fd; - - rspamd_http_router_free (ctx->collection_rt); - - /* Try to save collection id */ - fd = rspamd_file_xopen (ctx->collection_id_file, - O_WRONLY | O_CREAT | O_TRUNC, 00644, 0); - - if (fd == -1) { - msg_err ("cannot open collection id to store in %s: %s", - ctx->collection_id_file, strerror (errno)); - } - else { - if (write (fd, &ctx->collection_id, - sizeof (ctx->collection_id)) == -1) { - msg_err ("cannot store collection id in %s: %s", - ctx->collection_id_file, strerror (errno)); - } - close (fd); - } - } + rspamd_fuzzy_backend_close (ctx->backend); if (worker->index == 0) { g_array_free (ctx->updates_pending, TRUE); } - if (ctx->peer_fd != -1) { - if (worker->index == 0) { - event_del (&ctx->peer_ev); - } - close (ctx->peer_fd); - } - if (ctx->keypair_cache) { rspamd_keypair_cache_destroy (ctx->keypair_cache); } - struct rspamd_http_context *http_ctx = ctx->http_ctx; REF_RELEASE (ctx->cfg); - rspamd_http_context_free (http_ctx); rspamd_log_close (worker->srv->logger, TRUE); exit (EXIT_SUCCESS); diff --git a/src/hs_helper.c b/src/hs_helper.c index 94a46af8c..f83a9d429 100644 --- a/src/hs_helper.c +++ b/src/hs_helper.c @@ -47,7 +47,7 @@ static const guint64 rspamd_hs_helper_magic = 0x22d310157a2288a0ULL; struct hs_helper_ctx { guint64 magic; /* Events base */ - struct event_base *ev_base; + struct ev_loop *event_loop; /* DNS resolver */ struct rspamd_dns_resolver *resolver; /* Config */ @@ -57,7 +57,7 @@ struct hs_helper_ctx { gboolean loaded; gdouble max_time; gdouble recompile_time; - struct event recompile_timer; + ev_timer recompile_timer; }; static gpointer @@ -216,7 +216,7 @@ rspamd_rs_compile (struct hs_helper_ctx *ctx, struct rspamd_worker *worker, * XXX: now we just sleep for 5 seconds to ensure that */ if (!ctx->loaded) { - sleep (5); + ev_sleep (5.0); ctx->loaded = TRUE; } @@ -226,7 +226,7 @@ rspamd_rs_compile (struct hs_helper_ctx *ctx, struct rspamd_worker *worker, sizeof (srv_cmd.cmd.hs_loaded.cache_dir)); srv_cmd.cmd.hs_loaded.forced = forced; - rspamd_srv_send_command (worker, ctx->ev_base, &srv_cmd, -1, NULL, NULL); + rspamd_srv_send_command (worker, ctx->event_loop, &srv_cmd, -1, NULL, NULL); return TRUE; } @@ -258,26 +258,23 @@ rspamd_hs_helper_reload (struct rspamd_main *rspamd_main, } static void -rspamd_hs_helper_timer (gint fd, short what, gpointer ud) +rspamd_hs_helper_timer (EV_P_ ev_timer *w, int revents) { - struct rspamd_worker *worker = ud; + struct rspamd_worker *worker = (struct rspamd_worker *)w->data; struct hs_helper_ctx *ctx; - struct timeval tv; double tim; ctx = worker->ctx; tim = rspamd_time_jitter (ctx->recompile_time, 0); - double_to_tv (tim, &tv); - event_del (&ctx->recompile_timer); + w->repeat = tim; rspamd_rs_compile (ctx, worker, FALSE); - event_add (&ctx->recompile_timer, &tv); + ev_timer_again (EV_A_ w); } static void start_hs_helper (struct rspamd_worker *worker) { struct hs_helper_ctx *ctx = worker->ctx; - struct timeval tv; double tim; ctx->cfg = worker->srv->cfg; @@ -289,7 +286,7 @@ start_hs_helper (struct rspamd_worker *worker) ctx->hs_dir = RSPAMD_DBDIR "/"; } - ctx->ev_base = rspamd_prepare_worker (worker, + ctx->event_loop = rspamd_prepare_worker (worker, "hs_helper", NULL); @@ -301,13 +298,12 @@ start_hs_helper (struct rspamd_worker *worker) rspamd_control_worker_add_cmd_handler (worker, RSPAMD_CONTROL_RECOMPILE, rspamd_hs_helper_reload, ctx); - event_set (&ctx->recompile_timer, -1, EV_TIMEOUT, rspamd_hs_helper_timer, - worker); - event_base_set (ctx->ev_base, &ctx->recompile_timer); + ctx->recompile_timer.data = worker; tim = rspamd_time_jitter (ctx->recompile_time, 0); - double_to_tv (tim, &tv); - event_add (&ctx->recompile_timer, &tv); - event_base_loop (ctx->ev_base, 0); + ev_timer_init (&ctx->recompile_timer, rspamd_hs_helper_timer, tim, 0.0); + ev_timer_start (ctx->event_loop, &ctx->recompile_timer); + + ev_loop (ctx->event_loop, 0); rspamd_worker_block_signals (); rspamd_log_close (worker->srv->logger, TRUE); diff --git a/src/libcryptobox/curve25519/base_constants.h b/src/libcryptobox/curve25519/base_constants.h index 0eaef129b..48adfcf03 100644 --- a/src/libcryptobox/curve25519/base_constants.h +++ b/src/libcryptobox/curve25519/base_constants.h @@ -1,4 +1,4 @@ -static const ge_precomp base[32][8] = { +static const ge_precomp event_loop[32][8] = { { { { 25967493,-14356035,29566456,3660896,-12694345,4014787,27544626,-11754271,-6079156,2047605 }, diff --git a/src/libcryptobox/curve25519/ref.c b/src/libcryptobox/curve25519/ref.c index 3ccc7ada1..eb89b2cdc 100644 --- a/src/libcryptobox/curve25519/ref.c +++ b/src/libcryptobox/curve25519/ref.c @@ -1601,14 +1601,14 @@ static void ge_select(ge_precomp *t, int pos, signed char b) unsigned char babs = b - (((-bnegative) & b) * ((signed char) 1 << 1)); ge_precomp_0 (t); - cmov (t, &base[pos][0], equal (babs, 1)); - cmov (t, &base[pos][1], equal (babs, 2)); - cmov (t, &base[pos][2], equal (babs, 3)); - cmov (t, &base[pos][3], equal (babs, 4)); - cmov (t, &base[pos][4], equal (babs, 5)); - cmov (t, &base[pos][5], equal (babs, 6)); - cmov (t, &base[pos][6], equal (babs, 7)); - cmov (t, &base[pos][7], equal (babs, 8)); + cmov (t, &event_loop[pos][0], equal (babs, 1)); + cmov (t, &event_loop[pos][1], equal (babs, 2)); + cmov (t, &event_loop[pos][2], equal (babs, 3)); + cmov (t, &event_loop[pos][3], equal (babs, 4)); + cmov (t, &event_loop[pos][4], equal (babs, 5)); + cmov (t, &event_loop[pos][5], equal (babs, 6)); + cmov (t, &event_loop[pos][6], equal (babs, 7)); + cmov (t, &event_loop[pos][7], equal (babs, 8)); fe_copy (minust.yplusx, t->yminusx); fe_copy (minust.yminusx, t->yplusx); fe_neg (minust.xy2d, t->xy2d); diff --git a/src/libcryptobox/keypairs_cache.c b/src/libcryptobox/keypairs_cache.c index 5e3a13e18..bcba5e247 100644 --- a/src/libcryptobox/keypairs_cache.c +++ b/src/libcryptobox/keypairs_cache.c @@ -14,9 +14,9 @@ * limitations under the License. */ #include "config.h" -#include "rspamd.h" #include "keypairs_cache.h" #include "keypair_private.h" +#include "libutil/util.h" #include "hash.h" struct rspamd_keypair_elt { diff --git a/src/libserver/CMakeLists.txt b/src/libserver/CMakeLists.txt index ccedbcdb3..f0dcae867 100644 --- a/src/libserver/CMakeLists.txt +++ b/src/libserver/CMakeLists.txt @@ -6,7 +6,7 @@ SET(LIBRSPAMDSERVERSRC ${CMAKE_CURRENT_SOURCE_DIR}/dkim.c ${CMAKE_CURRENT_SOURCE_DIR}/dns.c ${CMAKE_CURRENT_SOURCE_DIR}/dynamic_cfg.c - ${CMAKE_CURRENT_SOURCE_DIR}/events.c + ${CMAKE_CURRENT_SOURCE_DIR}/async_session.c ${CMAKE_CURRENT_SOURCE_DIR}/fuzzy_backend.c ${CMAKE_CURRENT_SOURCE_DIR}/fuzzy_backend_sqlite.c ${CMAKE_CURRENT_SOURCE_DIR}/html.c diff --git a/src/libserver/events.c b/src/libserver/async_session.c index 3f6d47112..cec2963aa 100644 --- a/src/libserver/events.c +++ b/src/libserver/async_session.c @@ -17,7 +17,7 @@ #include "rspamd.h" #include "contrib/uthash/utlist.h" #include "contrib/libucl/khash.h" -#include "events.h" +#include "async_session.h" #include "cryptobox.h" #define RSPAMD_SESSION_FLAG_DESTROYING (1 << 1) diff --git a/src/libserver/events.h b/src/libserver/async_session.h index 7e0de8749..92454158a 100644 --- a/src/libserver/events.h +++ b/src/libserver/async_session.h @@ -13,8 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef RSPAMD_EVENTS_H -#define RSPAMD_EVENTS_H +#ifndef RSPAMD_ASYNC_SESSION_H +#define RSPAMD_ASYNC_SESSION_H #include "config.h" #include "mem_pool.h" @@ -107,4 +107,4 @@ guint rspamd_session_events_pending (struct rspamd_async_session *session); */ gboolean rspamd_session_blocked (struct rspamd_async_session *s); -#endif /* RSPAMD_EVENTS_H */ +#endif /*RSPAMD_ASYNC_SESSION_H*/ diff --git a/src/libserver/dkim.h b/src/libserver/dkim.h index fe2b4cc4e..57f761895 100644 --- a/src/libserver/dkim.h +++ b/src/libserver/dkim.h @@ -17,7 +17,7 @@ #define DKIM_H_ #include "config.h" -#include "event.h" +#include "contrib/libev/ev.h" #include "dns.h" #include "ref.h" diff --git a/src/libserver/dns.c b/src/libserver/dns.c index 54b97d06a..5277e2f6c 100644 --- a/src/libserver/dns.c +++ b/src/libserver/dns.c @@ -14,13 +14,14 @@ * limitations under the License. */ -#include <contrib/librdns/rdns.h> -#include <contrib/librdns/dns_private.h> + #include "config.h" #include "dns.h" #include "rspamd.h" #include "utlist.h" -#include "rdns_event.h" +#include "contrib/librdns/rdns.h" +#include "contrib/librdns/dns_private.h" +#include "contrib/librdns/rdns_ev.h" #include "unix-std.h" static const gchar *M = "rspamd dns"; @@ -532,13 +533,13 @@ rspamd_dns_resolver_config_ucl (struct rspamd_config *cfg, struct rspamd_dns_resolver * rspamd_dns_resolver_init (rspamd_logger_t *logger, - struct event_base *ev_base, + struct ev_loop *ev_base, struct rspamd_config *cfg) { struct rspamd_dns_resolver *dns_resolver; dns_resolver = g_malloc0 (sizeof (struct rspamd_dns_resolver)); - dns_resolver->ev_base = ev_base; + dns_resolver->event_loop = ev_base; if (cfg != NULL) { dns_resolver->request_timeout = cfg->dns_timeout; dns_resolver->max_retransmits = cfg->dns_retransmits; @@ -549,7 +550,7 @@ rspamd_dns_resolver_init (rspamd_logger_t *logger, } dns_resolver->r = rdns_resolver_new (); - rdns_bind_libevent (dns_resolver->r, dns_resolver->ev_base); + rdns_bind_libev (dns_resolver->r, dns_resolver->event_loop); if (cfg != NULL) { rdns_resolver_set_log_level (dns_resolver->r, cfg->log_level); diff --git a/src/libserver/dns.h b/src/libserver/dns.h index e1def703d..c744ac42e 100644 --- a/src/libserver/dns.h +++ b/src/libserver/dns.h @@ -19,7 +19,7 @@ #include "config.h" #include "mem_pool.h" -#include "events.h" +#include "async_session.h" #include "logger.h" #include "rdns.h" #include "upstream.h" @@ -28,7 +28,7 @@ struct rspamd_config; struct rspamd_dns_resolver { struct rdns_resolver *r; - struct event_base *ev_base; + struct ev_loop *event_loop; struct upstream_list *ups; struct rspamd_config *cfg; gdouble request_timeout; @@ -41,7 +41,7 @@ struct rspamd_dns_resolver { * Init DNS resolver, params are obtained from a config file or system file /etc/resolv.conf */ struct rspamd_dns_resolver * rspamd_dns_resolver_init (rspamd_logger_t *logger, - struct event_base *ev_base, struct rspamd_config *cfg); + struct ev_loop *ev_base, struct rspamd_config *cfg); struct rspamd_dns_request_ud; /** diff --git a/src/libserver/fuzzy_backend.c b/src/libserver/fuzzy_backend.c index 6de977ff6..f6dec1d6e 100644 --- a/src/libserver/fuzzy_backend.c +++ b/src/libserver/fuzzy_backend.c @@ -105,12 +105,12 @@ struct rspamd_fuzzy_backend { enum rspamd_fuzzy_backend_type type; gdouble expire; gdouble sync; - struct event_base *ev_base; + struct ev_loop *event_loop; rspamd_fuzzy_periodic_cb periodic_cb; void *periodic_ud; const struct rspamd_fuzzy_backend_subr *subr; void *subr_ud; - struct event periodic_event; + ev_timer periodic_event; }; static GQuark @@ -271,7 +271,7 @@ rspamd_fuzzy_backend_close_sqlite (struct rspamd_fuzzy_backend *bk, struct rspamd_fuzzy_backend * -rspamd_fuzzy_backend_create (struct event_base *ev_base, +rspamd_fuzzy_backend_create (struct ev_loop *ev_base, const ucl_object_t *config, struct rspamd_config *cfg, GError **err) @@ -307,7 +307,7 @@ rspamd_fuzzy_backend_create (struct event_base *ev_base, } bk = g_malloc0 (sizeof (*bk)); - bk->ev_base = ev_base; + bk->event_loop = ev_base; bk->expire = expire; bk->type = type; bk->subr = &fuzzy_subrs[type]; @@ -499,17 +499,15 @@ rspamd_fuzzy_backend_periodic_sync (struct rspamd_fuzzy_backend *bk) } static void -rspamd_fuzzy_backend_periodic_cb (gint fd, short what, void *ud) +rspamd_fuzzy_backend_periodic_cb (EV_P_ ev_timer *w, int revents) { - struct rspamd_fuzzy_backend *bk = ud; + struct rspamd_fuzzy_backend *bk = (struct rspamd_fuzzy_backend *)w->data; gdouble jittered; - struct timeval tv; jittered = rspamd_time_jitter (bk->sync, bk->sync / 2.0); - double_to_tv (jittered, &tv); - event_del (&bk->periodic_event); + w->repeat = jittered; rspamd_fuzzy_backend_periodic_sync (bk); - event_add (&bk->periodic_event, &tv); + ev_timer_again (EV_A_ w); } void @@ -519,13 +517,12 @@ rspamd_fuzzy_backend_start_update (struct rspamd_fuzzy_backend *bk, void *ud) { gdouble jittered; - struct timeval tv; g_assert (bk != NULL); if (bk->subr->periodic) { if (bk->sync > 0.0) { - event_del (&bk->periodic_event); + ev_timer_stop (bk->event_loop, &bk->periodic_event); } if (cb) { @@ -536,11 +533,11 @@ rspamd_fuzzy_backend_start_update (struct rspamd_fuzzy_backend *bk, rspamd_fuzzy_backend_periodic_sync (bk); bk->sync = timeout; jittered = rspamd_time_jitter (timeout, timeout / 2.0); - double_to_tv (jittered, &tv); - event_set (&bk->periodic_event, -1, EV_TIMEOUT, - rspamd_fuzzy_backend_periodic_cb, bk); - event_base_set (bk->ev_base, &bk->periodic_event); - event_add (&bk->periodic_event, &tv); + + bk->periodic_event.data = bk; + ev_timer_init (&bk->periodic_event, rspamd_fuzzy_backend_periodic_cb, + jittered, 0.0); + ev_timer_start (bk->event_loop, &bk->periodic_event); } } @@ -551,7 +548,7 @@ rspamd_fuzzy_backend_close (struct rspamd_fuzzy_backend *bk) if (bk->sync > 0.0) { rspamd_fuzzy_backend_periodic_sync (bk); - event_del (&bk->periodic_event); + ev_timer_stop (bk->event_loop, &bk->periodic_event); } bk->subr->close (bk, bk->subr_ud); @@ -559,10 +556,10 @@ rspamd_fuzzy_backend_close (struct rspamd_fuzzy_backend *bk) g_free (bk); } -struct event_base* +struct ev_loop* rspamd_fuzzy_backend_event_base (struct rspamd_fuzzy_backend *backend) { - return backend->ev_base; + return backend->event_loop; } gdouble diff --git a/src/libserver/fuzzy_backend.h b/src/libserver/fuzzy_backend.h index f26f3a582..1519761e0 100644 --- a/src/libserver/fuzzy_backend.h +++ b/src/libserver/fuzzy_backend.h @@ -17,7 +17,7 @@ #define SRC_LIBSERVER_FUZZY_BACKEND_H_ #include "config.h" -#include <event.h> +#include "contrib/libev/ev.h" #include "fuzzy_wire.h" struct rspamd_fuzzy_backend; @@ -44,7 +44,7 @@ typedef gboolean (*rspamd_fuzzy_periodic_cb) (void *ud); * @param err * @return */ -struct rspamd_fuzzy_backend * rspamd_fuzzy_backend_create (struct event_base *ev_base, +struct rspamd_fuzzy_backend * rspamd_fuzzy_backend_create (struct ev_loop *ev_base, const ucl_object_t *config, struct rspamd_config *cfg, GError **err); @@ -106,7 +106,7 @@ void rspamd_fuzzy_backend_start_update (struct rspamd_fuzzy_backend *backend, rspamd_fuzzy_periodic_cb cb, void *ud); -struct event_base* rspamd_fuzzy_backend_event_base (struct rspamd_fuzzy_backend *backend); +struct ev_loop* rspamd_fuzzy_backend_event_base (struct rspamd_fuzzy_backend *backend); gdouble rspamd_fuzzy_backend_get_expire (struct rspamd_fuzzy_backend *backend); /** diff --git a/src/libserver/fuzzy_backend_redis.c b/src/libserver/fuzzy_backend_redis.c index 956979d42..79c712386 100644 --- a/src/libserver/fuzzy_backend_redis.c +++ b/src/libserver/fuzzy_backend_redis.c @@ -71,9 +71,9 @@ enum rspamd_fuzzy_redis_command { struct rspamd_fuzzy_redis_session { struct rspamd_fuzzy_backend_redis *backend; redisAsyncContext *ctx; - struct event timeout; + ev_timer timeout; const struct rspamd_fuzzy_cmd *cmd; - struct event_base *ev_base; + struct ev_loop *event_loop; float prob; gboolean shingles_checked; @@ -143,10 +143,7 @@ rspamd_fuzzy_redis_session_dtor (struct rspamd_fuzzy_redis_session *session, ac, is_fatal); } - if (rspamd_event_pending (&session->timeout, EV_TIMEOUT)) { - event_del (&session->timeout); - } - + ev_timer_stop (session->event_loop, &session->timeout); rspamd_fuzzy_redis_session_free_args (session); REF_RELEASE (session->backend); @@ -276,9 +273,10 @@ rspamd_fuzzy_backend_init_redis (struct rspamd_fuzzy_backend *bk, } static void -rspamd_fuzzy_redis_timeout (gint fd, short what, gpointer priv) +rspamd_fuzzy_redis_timeout (EV_P_ ev_timer *w, int revents) { - struct rspamd_fuzzy_redis_session *session = priv; + struct rspamd_fuzzy_redis_session *session = + (struct rspamd_fuzzy_redis_session *)w->data; redisAsyncContext *ac; static char errstr[128]; @@ -320,12 +318,11 @@ rspamd_fuzzy_redis_shingles_callback (redisAsyncContext *c, gpointer r, struct rspamd_fuzzy_redis_session *session = priv; redisReply *reply = r, *cur; struct rspamd_fuzzy_reply rep; - struct timeval tv; GString *key; struct _rspamd_fuzzy_shingles_helper *shingles, *prev = NULL, *sel = NULL; guint i, found = 0, max_found = 0, cur_found = 0; - event_del (&session->timeout); + ev_timer_stop (session->event_loop, &session->timeout); memset (&rep, 0, sizeof (rep)); if (c->err == 0) { @@ -421,12 +418,11 @@ rspamd_fuzzy_redis_shingles_callback (redisAsyncContext *c, gpointer r, } else { /* Add timeout */ - event_set (&session->timeout, -1, EV_TIMEOUT, + session->timeout.data = session; + ev_timer_init (&session->timeout, rspamd_fuzzy_redis_timeout, - session); - event_base_set (session->ev_base, &session->timeout); - double_to_tv (session->backend->timeout, &tv); - event_add (&session->timeout, &tv); + session->backend->timeout, 0.0); + ev_timer_start (session->event_loop, &session->timeout); } return; @@ -456,7 +452,6 @@ rspamd_fuzzy_redis_shingles_callback (redisAsyncContext *c, gpointer r, static void rspamd_fuzzy_backend_check_shingles (struct rspamd_fuzzy_redis_session *session) { - struct timeval tv; struct rspamd_fuzzy_reply rep; const struct rspamd_fuzzy_shingle_cmd *shcmd; GString *key; @@ -501,11 +496,11 @@ rspamd_fuzzy_backend_check_shingles (struct rspamd_fuzzy_redis_session *session) } else { /* Add timeout */ - event_set (&session->timeout, -1, EV_TIMEOUT, rspamd_fuzzy_redis_timeout, - session); - event_base_set (session->ev_base, &session->timeout); - double_to_tv (session->backend->timeout, &tv); - event_add (&session->timeout, &tv); + session->timeout.data = session; + ev_timer_init (&session->timeout, + rspamd_fuzzy_redis_timeout, + session->backend->timeout, 0.0); + ev_timer_start (session->event_loop, &session->timeout); } } @@ -519,7 +514,7 @@ rspamd_fuzzy_redis_check_callback (redisAsyncContext *c, gpointer r, gulong value; guint found_elts = 0; - event_del (&session->timeout); + ev_timer_stop (session->event_loop, &session->timeout); memset (&rep, 0, sizeof (rep)); if (c->err == 0) { @@ -602,7 +597,6 @@ rspamd_fuzzy_backend_check_redis (struct rspamd_fuzzy_backend *bk, struct rspamd_fuzzy_redis_session *session; struct upstream *up; struct upstream_list *ups; - struct timeval tv; rspamd_inet_addr_t *addr; struct rspamd_fuzzy_reply rep; GString *key; @@ -620,7 +614,7 @@ rspamd_fuzzy_backend_check_redis (struct rspamd_fuzzy_backend *bk, session->prob = 1.0; memcpy (rep.digest, session->cmd->digest, sizeof (rep.digest)); memcpy (session->found_digest, session->cmd->digest, sizeof (rep.digest)); - session->ev_base = rspamd_fuzzy_backend_event_base (bk); + session->event_loop = rspamd_fuzzy_backend_event_base (bk); /* First of all check digest */ session->nargs = 5; @@ -677,11 +671,11 @@ rspamd_fuzzy_backend_check_redis (struct rspamd_fuzzy_backend *bk, } else { /* Add timeout */ - event_set (&session->timeout, -1, EV_TIMEOUT, rspamd_fuzzy_redis_timeout, - session); - event_base_set (session->ev_base, &session->timeout); - double_to_tv (backend->timeout, &tv); - event_add (&session->timeout, &tv); + session->timeout.data = session; + ev_timer_init (&session->timeout, + rspamd_fuzzy_redis_timeout, + session->backend->timeout, 0.0); + ev_timer_start (session->event_loop, &session->timeout); } } } @@ -694,7 +688,7 @@ rspamd_fuzzy_redis_count_callback (redisAsyncContext *c, gpointer r, redisReply *reply = r; gulong nelts; - event_del (&session->timeout); + ev_timer_stop (session->event_loop, &session->timeout); if (c->err == 0) { rspamd_upstream_ok (session->up); @@ -741,7 +735,6 @@ rspamd_fuzzy_backend_count_redis (struct rspamd_fuzzy_backend *bk, struct rspamd_fuzzy_redis_session *session; struct upstream *up; struct upstream_list *ups; - struct timeval tv; rspamd_inet_addr_t *addr; GString *key; @@ -754,7 +747,7 @@ rspamd_fuzzy_backend_count_redis (struct rspamd_fuzzy_backend *bk, session->callback.cb_count = cb; session->cbdata = ud; session->command = RSPAMD_FUZZY_REDIS_COMMAND_COUNT; - session->ev_base = rspamd_fuzzy_backend_event_base (bk); + session->event_loop = rspamd_fuzzy_backend_event_base (bk); session->nargs = 2; session->argv = g_malloc (sizeof (gchar *) * 2); @@ -801,11 +794,11 @@ rspamd_fuzzy_backend_count_redis (struct rspamd_fuzzy_backend *bk, } else { /* Add timeout */ - event_set (&session->timeout, -1, EV_TIMEOUT, rspamd_fuzzy_redis_timeout, - session); - event_base_set (session->ev_base, &session->timeout); - double_to_tv (backend->timeout, &tv); - event_add (&session->timeout, &tv); + session->timeout.data = session; + ev_timer_init (&session->timeout, + rspamd_fuzzy_redis_timeout, + session->backend->timeout, 0.0); + ev_timer_start (session->event_loop, &session->timeout); } } } @@ -818,7 +811,7 @@ rspamd_fuzzy_redis_version_callback (redisAsyncContext *c, gpointer r, redisReply *reply = r; gulong nelts; - event_del (&session->timeout); + ev_timer_stop (session->event_loop, &session->timeout); if (c->err == 0) { rspamd_upstream_ok (session->up); @@ -866,7 +859,6 @@ rspamd_fuzzy_backend_version_redis (struct rspamd_fuzzy_backend *bk, struct rspamd_fuzzy_redis_session *session; struct upstream *up; struct upstream_list *ups; - struct timeval tv; rspamd_inet_addr_t *addr; GString *key; @@ -879,7 +871,7 @@ rspamd_fuzzy_backend_version_redis (struct rspamd_fuzzy_backend *bk, session->callback.cb_version = cb; session->cbdata = ud; session->command = RSPAMD_FUZZY_REDIS_COMMAND_VERSION; - session->ev_base = rspamd_fuzzy_backend_event_base (bk); + session->event_loop = rspamd_fuzzy_backend_event_base (bk); session->nargs = 2; session->argv = g_malloc (sizeof (gchar *) * 2); @@ -926,11 +918,11 @@ rspamd_fuzzy_backend_version_redis (struct rspamd_fuzzy_backend *bk, } else { /* Add timeout */ - event_set (&session->timeout, -1, EV_TIMEOUT, rspamd_fuzzy_redis_timeout, - session); - event_base_set (session->ev_base, &session->timeout); - double_to_tv (backend->timeout, &tv); - event_add (&session->timeout, &tv); + session->timeout.data = session; + ev_timer_init (&session->timeout, + rspamd_fuzzy_redis_timeout, + session->backend->timeout, 0.0); + ev_timer_start (session->event_loop, &session->timeout); } } } @@ -1309,7 +1301,8 @@ rspamd_fuzzy_redis_update_callback (redisAsyncContext *c, gpointer r, { struct rspamd_fuzzy_redis_session *session = priv; redisReply *reply = r; - event_del (&session->timeout); + + ev_timer_stop (session->event_loop, &session->timeout); if (c->err == 0) { rspamd_upstream_ok (session->up); @@ -1356,12 +1349,11 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk, struct rspamd_fuzzy_redis_session *session; struct upstream *up; struct upstream_list *ups; - struct timeval tv; rspamd_inet_addr_t *addr; guint i; GString *key; struct fuzzy_peer_cmd *io_cmd; - struct rspamd_fuzzy_cmd *cmd; + struct rspamd_fuzzy_cmd *cmd = NULL; guint nargs, ncommands, cur_shift; g_assert (backend != NULL); @@ -1445,7 +1437,7 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk, session->command = RSPAMD_FUZZY_REDIS_COMMAND_UPDATES; session->cmd = cmd; session->prob = 1.0; - session->ev_base = rspamd_fuzzy_backend_event_base (bk); + session->event_loop = rspamd_fuzzy_backend_event_base (bk); /* First of all check digest */ session->nargs = nargs; @@ -1550,11 +1542,11 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk, } else { /* Add timeout */ - event_set (&session->timeout, -1, EV_TIMEOUT, rspamd_fuzzy_redis_timeout, - session); - event_base_set (session->ev_base, &session->timeout); - double_to_tv (backend->timeout, &tv); - event_add (&session->timeout, &tv); + session->timeout.data = session; + ev_timer_init (&session->timeout, + rspamd_fuzzy_redis_timeout, + session->backend->timeout, 0.0); + ev_timer_start (session->event_loop, &session->timeout); } } } diff --git a/src/libserver/milter.c b/src/libserver/milter.c index 188ff42d9..c06ad7a99 100644 --- a/src/libserver/milter.c +++ b/src/libserver/milter.c @@ -186,10 +186,7 @@ rspamd_milter_session_dtor (struct rspamd_milter_session *session) priv = session->priv; msg_debug_milter ("destroying milter session"); - if (rspamd_event_pending (&priv->ev, EV_TIMEOUT|EV_WRITE|EV_READ)) { - event_del (&priv->ev); - } - + rspamd_ev_watcher_stop (priv->event_loop, &priv->ev); rspamd_milter_session_reset (session, RSPAMD_MILTER_RESET_ALL); if (priv->parser.buf) { @@ -267,14 +264,7 @@ static inline void rspamd_milter_plan_io (struct rspamd_milter_session *session, struct rspamd_milter_private *priv, gshort what) { - if (rspamd_event_pending (&priv->ev, EV_TIMEOUT|EV_WRITE|EV_READ)) { - event_del (&priv->ev); - } - - event_set (&priv->ev, priv->fd, what, rspamd_milter_io_handler, - session); - event_base_set (priv->ev_base, &priv->ev); - event_add (&priv->ev, priv->ptv); + rspamd_ev_watcher_reschedule (priv->event_loop, &priv->ev, what); } @@ -1083,9 +1073,9 @@ rspamd_milter_handle_session (struct rspamd_milter_session *session, gboolean -rspamd_milter_handle_socket (gint fd, const struct timeval *tv, +rspamd_milter_handle_socket (gint fd, ev_tstamp timeout, rspamd_mempool_t *pool, - struct event_base *ev_base, rspamd_milter_finish finish_cb, + struct ev_loop *ev_base, rspamd_milter_finish finish_cb, rspamd_milter_error error_cb, void *ud) { struct rspamd_milter_session *session; @@ -1103,11 +1093,15 @@ rspamd_milter_handle_socket (gint fd, const struct timeval *tv, priv->err_cb = error_cb; priv->parser.state = st_len_1; priv->parser.buf = rspamd_fstring_sized_new (RSPAMD_MILTER_MESSAGE_CHUNK + 5); - priv->ev_base = ev_base; + priv->event_loop = ev_base; priv->state = RSPAMD_MILTER_READ_MORE; priv->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "milter"); priv->discard_on_reject = milter_ctx->discard_on_reject; priv->quarantine_on_reject = milter_ctx->quarantine_on_reject; + priv->ev.timeout = timeout; + + rspamd_ev_watcher_init (&priv->ev, fd, EV_READ|EV_WRITE, + rspamd_milter_io_handler, session); if (pool) { /* Copy tag */ @@ -1117,14 +1111,6 @@ rspamd_milter_handle_socket (gint fd, const struct timeval *tv, priv->headers = kh_init (milter_headers_hash_t); kh_resize (milter_headers_hash_t, priv->headers, 32); - if (tv) { - memcpy (&priv->tv, tv, sizeof (*tv)); - priv->ptv = &priv->tv; - } - else { - priv->ptv = NULL; - } - session->priv = priv; REF_INIT_RETAIN (session, rspamd_milter_session_dtor); diff --git a/src/libserver/milter.h b/src/libserver/milter.h index 10bf34d52..7906aeadf 100644 --- a/src/libserver/milter.h +++ b/src/libserver/milter.h @@ -20,6 +20,7 @@ #include "fstring.h" #include "addr.h" #include "contrib/libucl/ucl.h" +#include "contrib/libev/ev.h" #include "ref.h" enum rspamd_milter_reply { @@ -41,7 +42,7 @@ enum rspamd_milter_reply { }; struct rspamd_email_address; -struct event_base; +struct ev_loop; struct rspamd_http_message; struct rspamd_config; @@ -81,9 +82,9 @@ typedef void (*rspamd_milter_error) (gint fd, * @param ud * @return */ -gboolean rspamd_milter_handle_socket (gint fd, const struct timeval *tv, +gboolean rspamd_milter_handle_socket (gint fd, ev_tstamp timeout, rspamd_mempool_t *pool, - struct event_base *ev_base, rspamd_milter_finish finish_cb, + struct ev_loop *ev_base, rspamd_milter_finish finish_cb, rspamd_milter_error error_cb, void *ud); /** diff --git a/src/libserver/milter_internal.h b/src/libserver/milter_internal.h index 1e4b7b187..41862a169 100644 --- a/src/libserver/milter_internal.h +++ b/src/libserver/milter_internal.h @@ -19,9 +19,10 @@ #include "config.h" #include "libutil/mem_pool.h" -#include <event.h> +#include "contrib/libev/ev.h" #include "khash.h" #include "libutil/str_util.h" +#include "libutil/libev_helper.h" enum rspamd_milter_state { st_len_1 = 0, @@ -59,11 +60,9 @@ KHASH_INIT (milter_headers_hash_t, char *, GArray *, true, struct rspamd_milter_private { struct rspamd_milter_parser parser; - struct event ev; - struct timeval tv; + struct rspamd_io_ev ev; struct rspamd_milter_outbuf *out_chain; - struct timeval *ptv; - struct event_base *ev_base; + struct ev_loop *event_loop; rspamd_mempool_t *pool; khash_t(milter_headers_hash_t) *headers; gint cur_hdr; diff --git a/src/libserver/monitored.c b/src/libserver/monitored.c index ddef9ffe3..d64ec92fe 100644 --- a/src/libserver/monitored.c +++ b/src/libserver/monitored.c @@ -39,7 +39,7 @@ struct rspamd_monitored_methods { struct rspamd_monitored_ctx { struct rspamd_config *cfg; struct rdns_resolver *resolver; - struct event_base *ev_base; + struct ev_loop *event_loop; GPtrArray *elts; GHashTable *helts; mon_change_cb change_cb; @@ -63,7 +63,7 @@ struct rspamd_monitored { enum rspamd_monitored_flags flags; struct rspamd_monitored_ctx *ctx; struct rspamd_monitored_methods proc; - struct event periodic; + ev_timer periodic; gchar tag[RSPAMD_MONITORED_TAG_LEN]; }; @@ -169,9 +169,9 @@ rspamd_monitored_propagate_success (struct rspamd_monitored *m, gdouble lat) } static void -rspamd_monitored_periodic (gint fd, short what, gpointer ud) +rspamd_monitored_periodic (EV_P_ ev_timer *w, int revents) { - struct rspamd_monitored *m = ud; + struct rspamd_monitored *m = (struct rspamd_monitored *)w->data; struct timeval tv; gdouble jittered; gboolean ret = FALSE; @@ -185,7 +185,8 @@ rspamd_monitored_periodic (gint fd, short what, gpointer ud) } if (ret) { - event_add (&m->periodic, &tv); + m->periodic.repeat = jittered; + ev_timer_again (EV_A_ &m->periodic); } } @@ -427,7 +428,7 @@ rspamd_monitored_ctx_init (void) void rspamd_monitored_ctx_config (struct rspamd_monitored_ctx *ctx, struct rspamd_config *cfg, - struct event_base *ev_base, + struct ev_loop *ev_base, struct rdns_resolver *resolver, mon_change_cb change_cb, gpointer ud) @@ -436,7 +437,7 @@ rspamd_monitored_ctx_config (struct rspamd_monitored_ctx *ctx, guint i; g_assert (ctx != NULL); - ctx->ev_base = ev_base; + ctx->event_loop = ev_base; ctx->resolver = resolver; ctx->cfg = cfg; ctx->initialized = TRUE; @@ -457,10 +458,10 @@ rspamd_monitored_ctx_config (struct rspamd_monitored_ctx *ctx, } -struct event_base * +struct ev_loop * rspamd_monitored_ctx_get_ev_base (struct rspamd_monitored_ctx *ctx) { - return ctx->ev_base; + return ctx->event_loop; } @@ -527,7 +528,7 @@ rspamd_monitored_create_ (struct rspamd_monitored_ctx *ctx, g_ptr_array_add (ctx->elts, m); - if (ctx->ev_base) { + if (ctx->event_loop) { rspamd_monitored_start (m); } @@ -592,30 +593,26 @@ rspamd_monitored_stop (struct rspamd_monitored *m) { g_assert (m != NULL); - if (rspamd_event_pending (&m->periodic, EV_TIMEOUT)) { - event_del (&m->periodic); - } + ev_timer_stop (m->ctx->event_loop, &m->periodic); } void rspamd_monitored_start (struct rspamd_monitored *m) { - struct timeval tv; gdouble jittered; g_assert (m != NULL); msg_debug_mon ("started monitored object %s", m->url); jittered = rspamd_time_jitter (m->ctx->monitoring_interval * m->monitoring_mult, 0.0); - double_to_tv (jittered, &tv); - if (rspamd_event_pending (&m->periodic, EV_TIMEOUT)) { - event_del (&m->periodic); + if (ev_is_active (&m->periodic)) { + ev_timer_stop (m->ctx->event_loop, &m->periodic); } - event_set (&m->periodic, -1, EV_TIMEOUT, rspamd_monitored_periodic, m); - event_base_set (m->ctx->ev_base, &m->periodic); - event_add (&m->periodic, &tv); + m->periodic.data = m; + ev_timer_init (&m->periodic, rspamd_monitored_periodic, jittered, 0.0); + ev_timer_start (m->ctx->event_loop, &m->periodic); } void diff --git a/src/libserver/monitored.h b/src/libserver/monitored.h index 4db41f9c2..0189e0e6c 100644 --- a/src/libserver/monitored.h +++ b/src/libserver/monitored.h @@ -52,12 +52,12 @@ typedef void (*mon_change_cb) (struct rspamd_monitored_ctx *ctx, */ void rspamd_monitored_ctx_config (struct rspamd_monitored_ctx *ctx, struct rspamd_config *cfg, - struct event_base *ev_base, + struct ev_loop *ev_base, struct rdns_resolver *resolver, mon_change_cb change_cb, gpointer ud); -struct event_base *rspamd_monitored_ctx_get_ev_base (struct rspamd_monitored_ctx *ctx); +struct ev_loop *rspamd_monitored_ctx_get_ev_base (struct rspamd_monitored_ctx *ctx); /** * Create monitored object diff --git a/src/libserver/protocol.c b/src/libserver/protocol.c index 7df5b27c5..4f854328f 100644 --- a/src/libserver/protocol.c +++ b/src/libserver/protocol.c @@ -1301,7 +1301,7 @@ rspamd_protocol_write_ucl (struct rspamd_task *task, ucl_object_insert_key (top, ucl_object_fromstring (task->message_id), "message-id", 0, false); ucl_object_insert_key (top, - ucl_object_fromdouble (task->time_real_finish - task->time_real), + ucl_object_fromdouble (task->time_real_finish - task->task_timestamp), "time_real", 0, false); ucl_object_insert_key (top, ucl_object_fromdouble (task->time_virtual_finish - task->time_virtual), @@ -1766,7 +1766,7 @@ rspamd_protocol_write_log_pipe (struct rspamd_task *task) } void -rspamd_protocol_write_reply (struct rspamd_task *task) +rspamd_protocol_write_reply (struct rspamd_task *task, ev_tstamp timeout) { struct rspamd_http_message *msg; const gchar *ctype = "application/json"; @@ -1786,7 +1786,8 @@ rspamd_protocol_write_reply (struct rspamd_task *task) msg->flags |= RSPAMD_HTTP_FLAG_SPAMC; } - msg->date = time (NULL); + ev_now_update (task->event_loop); + msg->date = ev_time (); msg_debug_protocol ("writing reply to client"); if (task->err != NULL) { @@ -1832,7 +1833,7 @@ rspamd_protocol_write_reply (struct rspamd_task *task) rspamd_http_connection_reset (task->http_conn); rspamd_http_connection_write_message (task->http_conn, msg, NULL, - ctype, task, &task->tv); + ctype, task, timeout); task->processed_stages |= RSPAMD_TASK_STAGE_REPLIED; } diff --git a/src/libserver/protocol.h b/src/libserver/protocol.h index 08372d765..2059110fb 100644 --- a/src/libserver/protocol.h +++ b/src/libserver/protocol.h @@ -103,7 +103,7 @@ ucl_object_t * rspamd_protocol_write_ucl (struct rspamd_task *task, * @param task task object * @return 0 if we wrote reply and -1 if there was some error */ -void rspamd_protocol_write_reply (struct rspamd_task *task); +void rspamd_protocol_write_reply (struct rspamd_task *task, ev_tstamp timeout); /** * Convert rspamd output to legacy protocol reply diff --git a/src/libserver/redis_pool.c b/src/libserver/redis_pool.c index 528a990a1..6c74ee6f5 100644 --- a/src/libserver/redis_pool.c +++ b/src/libserver/redis_pool.c @@ -15,12 +15,12 @@ */ #include "config.h" -#include <event.h> +#include "contrib/libev/ev.h" #include "redis_pool.h" #include "cfg_file.h" #include "contrib/hiredis/hiredis.h" #include "contrib/hiredis/async.h" -#include "contrib/hiredis/adapters/libevent.h" +#include "contrib/hiredis/adapters/libev.h" #include "cryptobox.h" #include "logger.h" @@ -30,7 +30,7 @@ struct rspamd_redis_pool_connection { struct redisAsyncContext *ctx; struct rspamd_redis_pool_elt *elt; GList *entry; - struct event timeout; + ev_timer timeout; gboolean active; gchar tag[MEMPOOL_UID_LEN]; ref_entry_t ref; @@ -44,7 +44,7 @@ struct rspamd_redis_pool_elt { }; struct rspamd_redis_pool { - struct event_base *ev_base; + struct ev_loop *event_loop; struct rspamd_config *cfg; GHashTable *elts_by_key; GHashTable *elts_by_ctx; @@ -120,9 +120,7 @@ rspamd_redis_pool_conn_dtor (struct rspamd_redis_pool_connection *conn) else { msg_debug_rpool ("inactive connection removed"); - if (rspamd_event_pending (&conn->timeout, EV_TIMEOUT)) { - event_del (&conn->timeout); - } + ev_timer_stop (conn->elt->pool->event_loop, &conn->timeout); if (conn->ctx && !(conn->ctx->c.flags & REDIS_FREEING)) { redisAsyncContext *ac = conn->ctx; @@ -173,9 +171,10 @@ rspamd_redis_pool_elt_dtor (gpointer p) } static void -rspamd_redis_conn_timeout (gint fd, short what, gpointer p) +rspamd_redis_conn_timeout (EV_P_ ev_timer *w, int revents) { - struct rspamd_redis_pool_connection *conn = p; + struct rspamd_redis_pool_connection *conn = + (struct rspamd_redis_pool_connection *)w->data; g_assert (!conn->active); msg_debug_rpool ("scheduled removal of connection %p, refcount: %d", @@ -186,7 +185,6 @@ rspamd_redis_conn_timeout (gint fd, short what, gpointer p) static void rspamd_redis_pool_schedule_timeout (struct rspamd_redis_pool_connection *conn) { - struct timeval tv; gdouble real_timeout; guint active_elts; @@ -203,10 +201,12 @@ rspamd_redis_pool_schedule_timeout (struct rspamd_redis_pool_connection *conn) msg_debug_rpool ("scheduled connection %p cleanup in %.1f seconds", conn->ctx, real_timeout); - double_to_tv (real_timeout, &tv); - event_set (&conn->timeout, -1, EV_TIMEOUT, rspamd_redis_conn_timeout, conn); - event_base_set (conn->elt->pool->ev_base, &conn->timeout); - event_add (&conn->timeout, &tv); + + conn->timeout.data = conn; + ev_timer_init (&conn->timeout, + rspamd_redis_conn_timeout, + real_timeout, 0.0); + ev_timer_start (conn->elt->pool->event_loop, &conn->timeout); } static void @@ -270,7 +270,7 @@ rspamd_redis_pool_new_connection (struct rspamd_redis_pool *pool, REF_INIT_RETAIN (conn, rspamd_redis_pool_conn_dtor); msg_debug_rpool ("created new connection to %s:%d: %p", ip, port, ctx); - redisLibeventAttach (ctx, pool->ev_base); + redisLibevAttach (pool->event_loop, ctx); redisAsyncSetDisconnectCallback (ctx, rspamd_redis_pool_on_disconnect, conn); @@ -317,11 +317,11 @@ rspamd_redis_pool_init (void) void rspamd_redis_pool_config (struct rspamd_redis_pool *pool, struct rspamd_config *cfg, - struct event_base *ev_base) + struct ev_loop *ev_base) { g_assert (pool != NULL); - pool->ev_base = ev_base; + pool->event_loop = ev_base; pool->cfg = cfg; pool->timeout = default_timeout; pool->max_conns = default_max_conns; @@ -339,7 +339,7 @@ rspamd_redis_pool_connect (struct rspamd_redis_pool *pool, struct rspamd_redis_pool_connection *conn; g_assert (pool != NULL); - g_assert (pool->ev_base != NULL); + g_assert (pool->event_loop != NULL); g_assert (ip != NULL); key = rspamd_redis_pool_get_key (db, password, ip, port); @@ -352,7 +352,7 @@ rspamd_redis_pool_connect (struct rspamd_redis_pool *pool, g_assert (!conn->active); if (conn->ctx->err == REDIS_OK) { - event_del (&conn->timeout); + ev_timer_stop (elt->pool->event_loop, &conn->timeout); conn->active = TRUE; g_queue_push_tail_link (elt->active, conn_entry); msg_debug_rpool ("reused existing connection to %s:%d: %p", diff --git a/src/libserver/redis_pool.h b/src/libserver/redis_pool.h index a43b6d7e2..a881a0a38 100644 --- a/src/libserver/redis_pool.h +++ b/src/libserver/redis_pool.h @@ -21,7 +21,7 @@ struct rspamd_redis_pool; struct rspamd_config; struct redisAsyncContext; -struct event_base; +struct ev_loop; /** * Creates new redis pool @@ -36,7 +36,7 @@ struct rspamd_redis_pool *rspamd_redis_pool_init (void); */ void rspamd_redis_pool_config (struct rspamd_redis_pool *pool, struct rspamd_config *cfg, - struct event_base *ev_base); + struct ev_loop *ev_base); /** diff --git a/src/libserver/roll_history.c b/src/libserver/roll_history.c index c9367409d..c70246383 100644 --- a/src/libserver/roll_history.c +++ b/src/libserver/roll_history.c @@ -136,7 +136,7 @@ rspamd_roll_history_update (struct roll_history *history, rspamd_strlcpy (row->from_addr, "unknown", sizeof (row->from_addr)); } - memcpy (&row->tv, &task->tv, sizeof (row->tv)); + row->timestamp = task->task_timestamp; /* Strings */ rspamd_strlcpy (row->message_id, task->message_id, @@ -173,7 +173,7 @@ rspamd_roll_history_update (struct roll_history *history, } } - row->scan_time = task->time_real_finish - task->time_real; + row->scan_time = task->time_real_finish - task->task_timestamp; row->len = task->msg.len; g_atomic_int_set (&row->completed, TRUE); } @@ -282,7 +282,7 @@ rspamd_roll_history_load (struct roll_history *history, const gchar *filename) elt = ucl_object_lookup (cur, "time"); if (elt && ucl_object_type (elt) == UCL_FLOAT) { - double_to_tv (ucl_object_todouble (elt), &row->tv); + row->timestamp = ucl_object_todouble (elt); } elt = ucl_object_lookup (cur, "id"); @@ -391,8 +391,8 @@ rspamd_roll_history_save (struct roll_history *history, const gchar *filename) elt = ucl_object_typed_new (UCL_OBJECT); - ucl_object_insert_key (elt, ucl_object_fromdouble ( - tv_to_double (&row->tv)), "time", 0, false); + ucl_object_insert_key (elt, ucl_object_fromdouble (row->timestamp), + "time", 0, false); ucl_object_insert_key (elt, ucl_object_fromstring (row->message_id), "id", 0, false); ucl_object_insert_key (elt, ucl_object_fromstring (row->symbols), diff --git a/src/libserver/roll_history.h b/src/libserver/roll_history.h index d8a77bfd7..d0f140098 100644 --- a/src/libserver/roll_history.h +++ b/src/libserver/roll_history.h @@ -33,7 +33,7 @@ struct rspamd_task; struct rspamd_config; struct roll_history_row { - struct timeval tv; + ev_tstamp timestamp; gchar message_id[HISTORY_MAX_ID]; gchar symbols[HISTORY_MAX_SYMBOLS]; gchar user[HISTORY_MAX_USER]; diff --git a/src/libserver/rspamd_control.c b/src/libserver/rspamd_control.c index 12d37cdbc..59d1e4234 100644 --- a/src/libserver/rspamd_control.c +++ b/src/libserver/rspamd_control.c @@ -19,6 +19,7 @@ #include "worker_util.h" #include "libutil/http_connection.h" #include "libutil/http_private.h" +#include "libutil/libev_helper.h" #include "unix-std.h" #include "utlist.h" @@ -26,20 +27,14 @@ #include <sys/resource.h> #endif -static struct timeval io_timeout = { - .tv_sec = 30, - .tv_usec = 0 -}; -static struct timeval worker_io_timeout = { - .tv_sec = 0, - .tv_usec = 500000 -}; +static ev_tstamp io_timeout = 30.0; +static ev_tstamp worker_io_timeout = 0.5; struct rspamd_control_session; struct rspamd_control_reply_elt { struct rspamd_control_reply reply; - struct event io_ev; + struct rspamd_io_ev ev; struct rspamd_worker *wrk; gpointer ud; gint attached_fd; @@ -48,6 +43,7 @@ struct rspamd_control_reply_elt { struct rspamd_control_session { gint fd; + struct ev_loop *event_loop; struct rspamd_main *rspamd_main; struct rspamd_http_connection *conn; struct rspamd_control_command cmd; @@ -131,7 +127,7 @@ rspamd_control_send_error (struct rspamd_control_session *session, NULL, "application/json", session, - &io_timeout); + io_timeout); } static void @@ -154,7 +150,7 @@ rspamd_control_send_ucl (struct rspamd_control_session *session, NULL, "application/json", session, - &io_timeout); + io_timeout); } static void @@ -168,7 +164,8 @@ rspamd_control_connection_close (struct rspamd_control_session *session) rspamd_inet_address_to_string (session->addr)); DL_FOREACH_SAFE (session->replies, elt, telt) { - event_del (&elt->io_ev); + rspamd_ev_watcher_stop (session->event_loop, + &elt->ev); g_free (elt); } @@ -358,7 +355,8 @@ rspamd_control_wrk_io (gint fd, short what, gpointer ud) } session->replies_remain --; - event_del (&elt->io_ev); + rspamd_ev_watcher_stop (session->event_loop, + &elt->ev); if (session->replies_remain == 0) { rspamd_control_write_reply (session); @@ -434,12 +432,12 @@ rspamd_control_broadcast_cmd (struct rspamd_main *rspamd_main, rep_elt = g_malloc0 (sizeof (*rep_elt)); rep_elt->wrk = wrk; rep_elt->ud = ud; - event_set (&rep_elt->io_ev, wrk->control_pipe[0], - EV_READ | EV_PERSIST, handler, + rspamd_ev_watcher_init (&rep_elt->ev, + wrk->control_pipe[0], + EV_READ, handler, rep_elt); - event_base_set (rspamd_main->ev_base, - &rep_elt->io_ev); - event_add (&rep_elt->io_ev, &worker_io_timeout); + rspamd_ev_watcher_start (rspamd_main->event_loop, + &rep_elt->ev, worker_io_timeout); DL_APPEND (res, rep_elt); } @@ -526,14 +524,15 @@ rspamd_control_process_client_socket (struct rspamd_main *rspamd_main, 0); session->rspamd_main = rspamd_main; session->addr = addr; + session->event_loop = rspamd_main->event_loop; rspamd_http_connection_read_message (session->conn, session, - &io_timeout); + io_timeout); } struct rspamd_worker_control_data { - struct event io_ev; + ev_io io_ev; struct rspamd_worker *worker; - struct event_base *ev_base; + struct ev_loop *ev_base; struct { rspamd_worker_control_handler handler; gpointer ud; @@ -613,9 +612,10 @@ rspamd_control_default_cmd_handler (gint fd, } static void -rspamd_control_default_worker_handler (gint fd, short what, gpointer ud) +rspamd_control_default_worker_handler (EV_P_ ev_io *w, int revents) { - struct rspamd_worker_control_data *cd = ud; + struct rspamd_worker_control_data *cd = + (struct rspamd_worker_control_data *)w->data; static struct rspamd_control_command cmd; static struct msghdr msg; static struct iovec iov; @@ -631,15 +631,15 @@ rspamd_control_default_worker_handler (gint fd, short what, gpointer ud) msg.msg_iov = &iov; msg.msg_iovlen = 1; - r = recvmsg (fd, &msg, 0); + r = recvmsg (w->fd, &msg, 0); if (r == -1) { msg_err ("cannot read request from the control socket: %s", strerror (errno)); if (errno != EAGAIN && errno != EINTR) { - event_del (&cd->io_ev); - close (fd); + ev_io_stop (cd->ev_base, &cd->io_ev); + close (w->fd); } } else if (r < (gint)sizeof (cmd)) { @@ -647,8 +647,8 @@ rspamd_control_default_worker_handler (gint fd, short what, gpointer ud) (gint)sizeof (cmd)); if (r == 0) { - event_del (&cd->io_ev); - close (fd); + ev_io_stop (cd->ev_base, &cd->io_ev); + close (w->fd); } } else if ((gint)cmd.type >= 0 && cmd.type < RSPAMD_CONTROL_MAX) { @@ -660,13 +660,13 @@ rspamd_control_default_worker_handler (gint fd, short what, gpointer ud) if (cd->handlers[cmd.type].handler) { cd->handlers[cmd.type].handler (cd->worker->srv, cd->worker, - fd, + w->fd, rfd, &cmd, cd->handlers[cmd.type].ud); } else { - rspamd_control_default_cmd_handler (fd, rfd, cd, &cmd); + rspamd_control_default_cmd_handler (w->fd, rfd, cd, &cmd); } } else { @@ -676,7 +676,7 @@ rspamd_control_default_worker_handler (gint fd, short what, gpointer ud) void rspamd_control_worker_add_default_handler (struct rspamd_worker *worker, - struct event_base *ev_base) + struct ev_loop *ev_base) { struct rspamd_worker_control_data *cd; @@ -684,10 +684,10 @@ rspamd_control_worker_add_default_handler (struct rspamd_worker *worker, cd->worker = worker; cd->ev_base = ev_base; - event_set (&cd->io_ev, worker->control_pipe[1], EV_READ | EV_PERSIST, - rspamd_control_default_worker_handler, cd); - event_base_set (ev_base, &cd->io_ev); - event_add (&cd->io_ev, NULL); + cd->io_ev.data = cd; + ev_io_init (&cd->io_ev, rspamd_control_default_worker_handler, + worker->control_pipe[1], EV_READ); + ev_io_start (ev_base, &cd->io_ev); worker->control_data = cd; } @@ -720,26 +720,28 @@ struct rspamd_srv_reply_data { }; static void -rspamd_control_hs_io_handler (gint fd, short what, gpointer ud) +rspamd_control_hs_io_handler (int fd, short what, void *ud) { - struct rspamd_control_reply_elt *elt = ud; + struct rspamd_control_reply_elt *elt = + (struct rspamd_control_reply_elt *)ud; struct rspamd_control_reply rep; /* At this point we just ignore replies from the workers */ (void)read (fd, &rep, sizeof (rep)); - event_del (&elt->io_ev); + rspamd_ev_watcher_stop (elt->wrk->srv->event_loop, &elt->ev); g_free (elt); } static void -rspamd_control_log_pipe_io_handler (gint fd, short what, gpointer ud) +rspamd_control_log_pipe_io_handler (int fd, short what, void *ud) { - struct rspamd_control_reply_elt *elt = ud; + struct rspamd_control_reply_elt *elt = + (struct rspamd_control_reply_elt *)ud; struct rspamd_control_reply rep; /* At this point we just ignore replies from the workers */ (void) read (fd, &rep, sizeof (rep)); - event_del (&elt->io_ev); + rspamd_ev_watcher_stop (elt->wrk->srv->event_loop, &elt->ev); g_free (elt); } @@ -793,8 +795,9 @@ rspamd_control_handle_on_fork (struct rspamd_srv_command *cmd, } } + static void -rspamd_srv_handler (gint fd, short what, gpointer ud) +rspamd_srv_handler (EV_P_ ev_io *w, int revents) { struct rspamd_worker *worker; static struct rspamd_srv_command cmd; @@ -809,8 +812,8 @@ rspamd_srv_handler (gint fd, short what, gpointer ud) struct rspamd_control_command wcmd; gssize r; - if (what == EV_READ) { - worker = ud; + if (revents == EV_READ) { + worker = (struct rspamd_worker *)w->data; srv = worker->srv; iov.iov_base = &cmd; iov.iov_len = sizeof (cmd); @@ -820,7 +823,7 @@ rspamd_srv_handler (gint fd, short what, gpointer ud) msg.msg_iov = &iov; msg.msg_iovlen = 1; - r = recvmsg (fd, &msg, 0); + r = recvmsg (w->fd, &msg, 0); if (r == -1) { msg_err ("cannot read from worker's srv pipe: %s", @@ -831,7 +834,7 @@ rspamd_srv_handler (gint fd, short what, gpointer ud) * Usually this means that a worker is dead, so do not try to read * anything */ - event_del (&worker->srv_ev); + ev_io_stop (EV_A_ w); } else if (r != sizeof (cmd)) { msg_err ("cannot read from worker's srv pipe incomplete command: %d", @@ -919,17 +922,14 @@ rspamd_srv_handler (gint fd, short what, gpointer ud) } /* Now plan write event and send data back */ - event_del (&worker->srv_ev); - event_set (&worker->srv_ev, - worker->srv_pipe[0], - EV_WRITE, - rspamd_srv_handler, - rdata); - event_add (&worker->srv_ev, NULL); + w->data = rdata; + ev_io_stop (EV_A_ w); + ev_io_set (w, worker->srv_pipe[0], EV_WRITE); + ev_io_start (EV_A_ w); } } - else if (what == EV_WRITE) { - rdata = ud; + else if (revents == EV_WRITE) { + rdata = (struct rspamd_srv_reply_data *)w->data; worker = rdata->worker; worker->tmp_data = NULL; /* Avoid race */ srv = rdata->srv; @@ -953,7 +953,7 @@ rspamd_srv_handler (gint fd, short what, gpointer ud) msg.msg_iov = &iov; msg.msg_iovlen = 1; - r = sendmsg (fd, &msg, 0); + r = sendmsg (w->fd, &msg, 0); if (r == -1) { msg_err ("cannot write to worker's srv pipe: %s", @@ -961,28 +961,24 @@ rspamd_srv_handler (gint fd, short what, gpointer ud) } g_free (rdata); - event_del (&worker->srv_ev); - event_set (&worker->srv_ev, - worker->srv_pipe[0], - EV_READ | EV_PERSIST, - rspamd_srv_handler, - worker); - event_add (&worker->srv_ev, NULL); + w->data = worker; + ev_io_stop (EV_A_ w); + ev_io_set (w, worker->srv_pipe[0], EV_READ); + ev_io_start (EV_A_ w); } } void rspamd_srv_start_watching (struct rspamd_main *srv, struct rspamd_worker *worker, - struct event_base *ev_base) + struct ev_loop *ev_base) { g_assert (worker != NULL); worker->tmp_data = NULL; - event_set (&worker->srv_ev, worker->srv_pipe[0], EV_READ | EV_PERSIST, - rspamd_srv_handler, worker); - event_base_set (ev_base, &worker->srv_ev); - event_add (&worker->srv_ev, NULL); + worker->srv_ev.data = worker; + ev_io_init (&worker->srv_ev, rspamd_srv_handler, worker->srv_pipe[0], EV_READ); + ev_io_start (ev_base, &worker->srv_ev); } struct rspamd_srv_request_data { @@ -991,14 +987,14 @@ struct rspamd_srv_request_data { gint attached_fd; struct rspamd_srv_reply rep; rspamd_srv_reply_handler handler; - struct event io_ev; + ev_io io_ev; gpointer ud; }; static void -rspamd_srv_request_handler (gint fd, short what, gpointer ud) +rspamd_srv_request_handler (EV_P_ ev_io *w, int revents) { - struct rspamd_srv_request_data *rd = ud; + struct rspamd_srv_request_data *rd = (struct rspamd_srv_request_data *)w->data; struct msghdr msg; struct iovec iov; guchar fdspace[CMSG_SPACE(sizeof (int))]; @@ -1006,7 +1002,7 @@ rspamd_srv_request_handler (gint fd, short what, gpointer ud) gssize r; gint rfd = -1; - if (what == EV_WRITE) { + if (revents == EV_WRITE) { /* Send request to server */ memset (&msg, 0, sizeof (msg)); @@ -1027,17 +1023,16 @@ rspamd_srv_request_handler (gint fd, short what, gpointer ud) msg.msg_iov = &iov; msg.msg_iovlen = 1; - r = sendmsg (fd, &msg, 0); + r = sendmsg (w->fd, &msg, 0); if (r == -1) { msg_err ("cannot write to server pipe: %s", strerror (errno)); goto cleanup; } - event_del (&rd->io_ev); - event_set (&rd->io_ev, rd->worker->srv_pipe[1], EV_READ, - rspamd_srv_request_handler, rd); - event_add (&rd->io_ev, NULL); + ev_io_stop (EV_A_ w); + ev_io_set (w, rd->worker->srv_pipe[1], EV_READ); + ev_io_start (EV_A_ w); } else { iov.iov_base = &rd->rep; @@ -1048,7 +1043,7 @@ rspamd_srv_request_handler (gint fd, short what, gpointer ud) msg.msg_iov = &iov; msg.msg_iovlen = 1; - r = recvmsg (fd, &msg, 0); + r = recvmsg (w->fd, &msg, 0); if (r == -1) { msg_err ("cannot read from server pipe: %s", strerror (errno)); @@ -1075,13 +1070,14 @@ cleanup: if (rd->handler) { rd->handler (rd->worker, &rd->rep, rfd, rd->ud); } - event_del (&rd->io_ev); + + ev_io_stop (EV_A_ w); g_free (rd); } void rspamd_srv_send_command (struct rspamd_worker *worker, - struct event_base *ev_base, + struct ev_loop *ev_base, struct rspamd_srv_command *cmd, gint attached_fd, rspamd_srv_reply_handler handler, @@ -1102,8 +1098,8 @@ rspamd_srv_send_command (struct rspamd_worker *worker, rd->rep.type = cmd->type; rd->attached_fd = attached_fd; - event_set (&rd->io_ev, worker->srv_pipe[1], EV_WRITE, - rspamd_srv_request_handler, rd); - event_base_set (ev_base, &rd->io_ev); - event_add (&rd->io_ev, NULL); + rd->io_ev.data = rd; + ev_io_init (&rd->io_ev, rspamd_srv_request_handler, + rd->worker->srv_pipe[1], EV_WRITE); + ev_io_start (ev_base, &rd->io_ev); } diff --git a/src/libserver/rspamd_control.h b/src/libserver/rspamd_control.h index ec3962c64..1928ab00f 100644 --- a/src/libserver/rspamd_control.h +++ b/src/libserver/rspamd_control.h @@ -18,7 +18,7 @@ #include "config.h" #include "mem_pool.h" -#include <event.h> +#include "contrib/libev/ev.h" struct rspamd_main; struct rspamd_worker; @@ -199,7 +199,7 @@ void rspamd_control_process_client_socket (struct rspamd_main *rspamd_main, * Register default handlers for a worker */ void rspamd_control_worker_add_default_handler (struct rspamd_worker *worker, - struct event_base *ev_base); + struct ev_loop *ev_base); /** * Register custom handler for a specific control command for this worker @@ -214,7 +214,7 @@ void rspamd_control_worker_add_cmd_handler (struct rspamd_worker *worker, */ void rspamd_srv_start_watching (struct rspamd_main *srv, struct rspamd_worker *worker, - struct event_base *ev_base); + struct ev_loop *ev_base); /** @@ -222,7 +222,7 @@ void rspamd_srv_start_watching (struct rspamd_main *srv, * end */ void rspamd_srv_send_command (struct rspamd_worker *worker, - struct event_base *ev_base, + struct ev_loop *ev_base, struct rspamd_srv_command *cmd, gint attached_fd, rspamd_srv_reply_handler handler, diff --git a/src/libserver/rspamd_symcache.c b/src/libserver/rspamd_symcache.c index 5a4a002c3..aa83024b2 100644 --- a/src/libserver/rspamd_symcache.c +++ b/src/libserver/rspamd_symcache.c @@ -229,10 +229,10 @@ struct cache_savepoint { struct rspamd_cache_refresh_cbdata { gdouble last_resort; - struct event resort_ev; + ev_timer resort_ev; struct rspamd_symcache *cache; struct rspamd_worker *w; - struct event_base *ev_base; + struct ev_loop *event_loop; }; /* weight, frequency, time */ @@ -1577,16 +1577,8 @@ rspamd_symcache_check_symbol (struct rspamd_task *task, if (check) { msg_debug_cache_task ("execute %s, %d", item->symbol, item->id); -#ifdef HAVE_EVENT_NO_CACHE_TIME_FUNC - struct timeval tv; - - event_base_update_cache_time (task->ev_base); - event_base_gettimeofday_cached (task->ev_base, &tv); - t1 = tv_to_double (&tv); -#else - t1 = rspamd_get_ticks (FALSE); -#endif - dyn_item->start_msec = (t1 - task->time_real) * 1e3; + t1 = ev_now (task->event_loop); + dyn_item->start_msec = (t1 - task->time_virtual) * 1e3; dyn_item->async_events = 0; checkpoint->cur_item = item; checkpoint->items_inflight ++; @@ -2173,14 +2165,14 @@ rspamd_symcache_counters (struct rspamd_symcache *cache) } static void -rspamd_symcache_call_peak_cb (struct event_base *ev_base, +rspamd_symcache_call_peak_cb (struct ev_loop *ev_base, struct rspamd_symcache *cache, struct rspamd_symcache_item *item, gdouble cur_value, gdouble cur_err) { lua_State *L = cache->cfg->lua_state; - struct event_base **pbase; + struct ev_loop **pbase; lua_rawgeti (L, LUA_REGISTRYINDEX, cache->peak_cb); pbase = lua_newuserdata (L, sizeof (*pbase)); @@ -2200,11 +2192,11 @@ rspamd_symcache_call_peak_cb (struct event_base *ev_base, } static void -rspamd_symcache_resort_cb (gint fd, short what, gpointer ud) +rspamd_symcache_resort_cb (EV_P_ ev_timer *w, int revents) { - struct timeval tv; gdouble tm; - struct rspamd_cache_refresh_cbdata *cbdata = ud; + struct rspamd_cache_refresh_cbdata *cbdata = + (struct rspamd_cache_refresh_cbdata *)w->data; struct rspamd_symcache *cache; struct rspamd_symcache_item *item; guint i; @@ -2217,10 +2209,8 @@ rspamd_symcache_resort_cb (gint fd, short what, gpointer ud) cur_ticks = rspamd_get_ticks (FALSE); msg_debug_cache ("resort symbols cache, next reload in %.2f seconds", tm); g_assert (cache != NULL); - evtimer_set (&cbdata->resort_ev, rspamd_symcache_resort_cb, cbdata); - event_base_set (cbdata->ev_base, &cbdata->resort_ev); - double_to_tv (tm, &tv); - event_add (&cbdata->resort_ev, &tv); + cbdata->resort_ev.repeat = tm; + ev_timer_again (EV_A_ w); if (rspamd_worker_is_primary_controller (cbdata->w)) { /* Gather stats from shared execution times */ @@ -2263,7 +2253,7 @@ rspamd_symcache_resort_cb (gint fd, short what, gpointer ud) item->frequency_peaks); if (cache->peak_cb != -1) { - rspamd_symcache_call_peak_cb (cbdata->ev_base, + rspamd_symcache_call_peak_cb (cbdata->event_loop, cache, item, cur_value, cur_err); } @@ -2283,36 +2273,41 @@ rspamd_symcache_resort_cb (gint fd, short what, gpointer ud) } } - cbdata->last_resort = cur_ticks; /* We don't do actual sorting due to topological guarantees */ } } +static void +rspamd_symcache_refresh_dtor (void *d) +{ + struct rspamd_cache_refresh_cbdata *cbdata = + (struct rspamd_cache_refresh_cbdata *)d; + + ev_timer_stop (cbdata->event_loop, &cbdata->resort_ev); +} + void rspamd_symcache_start_refresh (struct rspamd_symcache *cache, - struct event_base *ev_base, struct rspamd_worker *w) + struct ev_loop *ev_base, struct rspamd_worker *w) { - struct timeval tv; gdouble tm; struct rspamd_cache_refresh_cbdata *cbdata; cbdata = rspamd_mempool_alloc0 (cache->static_pool, sizeof (*cbdata)); cbdata->last_resort = rspamd_get_ticks (TRUE); - cbdata->ev_base = ev_base; + cbdata->event_loop = ev_base; cbdata->w = w; cbdata->cache = cache; tm = rspamd_time_jitter (cache->reload_time, 0); msg_debug_cache ("next reload in %.2f seconds", tm); g_assert (cache != NULL); - evtimer_set (&cbdata->resort_ev, rspamd_symcache_resort_cb, - cbdata); - event_base_set (ev_base, &cbdata->resort_ev); - double_to_tv (tm, &tv); - event_add (&cbdata->resort_ev, &tv); + cbdata->resort_ev.data = cbdata; + ev_timer_init (&cbdata->resort_ev, rspamd_symcache_resort_cb, + tm, tm); + ev_timer_start (cbdata->event_loop, &cbdata->resort_ev); rspamd_mempool_add_destructor (cache->static_pool, - (rspamd_mempool_destruct_t) event_del, - &cbdata->resort_ev); + rspamd_symcache_refresh_dtor, cbdata); } void @@ -2838,16 +2833,8 @@ rspamd_symcache_finalize_item (struct rspamd_task *task, checkpoint->items_inflight --; checkpoint->cur_item = NULL; -#ifdef HAVE_EVENT_NO_CACHE_TIME_FUNC - struct timeval tv; - event_base_update_cache_time (task->ev_base); - event_base_gettimeofday_cached (task->ev_base, &tv); - t2 = tv_to_double (&tv); -#else - t2 = rspamd_get_ticks (FALSE); -#endif - - diff = ((t2 - task->time_real) * 1e3 - dyn_item->start_msec); + t2 = ev_now (task->event_loop); + diff = ((t2 - task->time_virtual) * 1e3 - dyn_item->start_msec); if (G_UNLIKELY (RSPAMD_TASK_IS_PROFILING (task))) { rspamd_task_profile_set (task, item->symbol, diff); diff --git a/src/libserver/rspamd_symcache.h b/src/libserver/rspamd_symcache.h index ef022b730..7bfb1d093 100644 --- a/src/libserver/rspamd_symcache.h +++ b/src/libserver/rspamd_symcache.h @@ -20,7 +20,7 @@ #include "ucl.h" #include "cfg_file.h" #include <lua.h> -#include <event.h> +#include "contrib/libev/ev.h" struct rspamd_task; struct rspamd_config; @@ -198,7 +198,7 @@ ucl_object_t *rspamd_symcache_counters (struct rspamd_symcache *cache); * @param ev_base */ void rspamd_symcache_start_refresh (struct rspamd_symcache *cache, - struct event_base *ev_base, + struct ev_loop *ev_base, struct rspamd_worker *w); /** diff --git a/src/libserver/task.c b/src/libserver/task.c index 16b33294e..04be61744 100644 --- a/src/libserver/task.c +++ b/src/libserver/task.c @@ -75,7 +75,7 @@ struct rspamd_task * rspamd_task_new (struct rspamd_worker *worker, struct rspamd_config *cfg, rspamd_mempool_t *pool, struct rspamd_lang_detector *lang_det, - struct event_base *ev_base) + struct ev_loop *event_loop) { struct rspamd_task *new_task; @@ -101,24 +101,16 @@ rspamd_task_new (struct rspamd_worker *worker, struct rspamd_config *cfg, } } - new_task->ev_base = ev_base; - -#ifdef HAVE_EVENT_NO_CACHE_TIME_FUNC - if (ev_base) { - event_base_update_cache_time (ev_base); - event_base_gettimeofday_cached (ev_base, &new_task->tv); - new_task->time_real = tv_to_double (&new_task->tv); + new_task->event_loop = event_loop; + if (event_loop) { + new_task->task_timestamp = ev_time (); + new_task->time_virtual = ev_now (event_loop); } else { - gettimeofday (&new_task->tv, NULL); - new_task->time_real = tv_to_double (&new_task->tv); + new_task->task_timestamp = ev_time (); + new_task->time_virtual = rspamd_get_virtual_ticks (); } -#else - gettimeofday (&new_task->tv, NULL); - new_task->time_real = tv_to_double (&new_task->tv); -#endif - new_task->time_virtual = rspamd_get_virtual_ticks (); new_task->time_real_finish = NAN; new_task->time_virtual_finish = NAN; @@ -185,11 +177,13 @@ rspamd_task_new (struct rspamd_worker *worker, struct rspamd_config *cfg, static void rspamd_task_reply (struct rspamd_task *task) { + const ev_tstamp write_timeout = 2.0; + if (task->fin_callback) { task->fin_callback (task, task->fin_arg); } else { - rspamd_protocol_write_reply (task); + rspamd_protocol_write_reply (task, write_timeout); } } @@ -329,13 +323,8 @@ rspamd_task_free (struct rspamd_task *task) g_error_free (task->err); } - if (rspamd_event_pending (&task->timeout_ev, EV_TIMEOUT)) { - event_del (&task->timeout_ev); - } - - if (task->guard_ev) { - event_del (task->guard_ev); - } + ev_timer_stop (task->event_loop, &task->timeout_ev); + ev_io_stop (task->event_loop, &task->guard_ev); if (task->sock != -1) { close (task->sock); @@ -1450,7 +1439,7 @@ rspamd_task_log_variable (struct rspamd_task *task, var.begin = numbuf; break; case RSPAMD_LOG_TIME_REAL: - var.begin = rspamd_log_check_time (task->time_real, + var.begin = rspamd_log_check_time (task->task_timestamp, task->time_real_finish, task->cfg->clock_res); var.len = strlen (var.begin); @@ -1748,25 +1737,9 @@ rspamd_task_profile_get (struct rspamd_task *task, const gchar *key) gboolean rspamd_task_set_finish_time (struct rspamd_task *task) { - struct timeval tv; - if (isnan (task->time_real_finish)) { - -#ifdef HAVE_EVENT_NO_CACHE_TIME_FUNC - if (task->ev_base) { - event_base_update_cache_time (task->ev_base); - event_base_gettimeofday_cached (task->ev_base, &tv); - task->time_real_finish = tv_to_double (&tv); - } - else { - gettimeofday (&tv, NULL); - task->time_real_finish = tv_to_double (&tv); - } -#else - gettimeofday (&tv, NULL); - task->time_real_finish = tv_to_double (&tv); -#endif - task->time_virtual_finish = rspamd_get_virtual_ticks (); + task->time_real_finish = ev_time (); + task->time_virtual_finish = ev_now (task->event_loop); return TRUE; } diff --git a/src/libserver/task.h b/src/libserver/task.h index 263f06719..d581378b7 100644 --- a/src/libserver/task.h +++ b/src/libserver/task.h @@ -18,7 +18,7 @@ #include "config.h" #include "http_connection.h" -#include "events.h" +#include "async_session.h" #include "util.h" #include "mem_pool.h" #include "dns.h" @@ -189,19 +189,18 @@ struct rspamd_task { struct rspamd_config *cfg; /**< pointer to config object */ GError *err; rspamd_mempool_t *task_pool; /**< memory pool for task */ - double time_real; double time_virtual; double time_real_finish; double time_virtual_finish; - struct timeval tv; + ev_tstamp task_timestamp; gboolean (*fin_callback)(struct rspamd_task *task, void *arg); /**< callback for filters finalizing */ void *fin_arg; /**< argument for fin callback */ struct rspamd_dns_resolver *resolver; /**< DNS resolver */ - struct event_base *ev_base; /**< Event base */ - struct event timeout_ev; /**< Global task timeout */ - struct event *guard_ev; /**< Event for input sanity guard */ + struct ev_loop *event_loop; /**< Event base */ + struct ev_timer timeout_ev; /**< Global task timeout */ + struct ev_io guard_ev; /**< Event for input sanity guard */ gpointer checkpoint; /**< Opaque checkpoint data */ ucl_object_t *settings; /**< Settings applied to task */ @@ -220,7 +219,7 @@ struct rspamd_task *rspamd_task_new (struct rspamd_worker *worker, struct rspamd_config *cfg, rspamd_mempool_t *pool, struct rspamd_lang_detector *lang_det, - struct event_base *ev_base); + struct ev_loop *event_loop); /** * Destroy task object and remove its IO dispatcher if it exists */ diff --git a/src/libserver/worker_util.c b/src/libserver/worker_util.c index a3601621f..5c5d41b90 100644 --- a/src/libserver/worker_util.c +++ b/src/libserver/worker_util.c @@ -57,7 +57,9 @@ #include <sys/ucontext.h> #endif -static void rspamd_worker_ignore_signal (int signo); +#include "contrib/libev/ev.h" + +static void rspamd_worker_ignore_signal (struct rspamd_worker_signal_handler *); /** * Return worker's control structure by its type * @param type @@ -82,22 +84,76 @@ rspamd_get_worker_by_type (struct rspamd_config *cfg, GQuark type) return NULL; } -static gboolean +static void +rspamd_worker_check_finished (EV_P_ ev_timer *w, int revents) +{ + int *pnchecks = (int *)w->data; + + if (*pnchecks > SOFT_SHUTDOWN_TIME * 10) { + msg_warn ("terminating worker before finishing of terminate handlers"); + ev_break (EV_A_ EVBREAK_ONE); + } + else { + int refcount = ev_active_cnt (EV_A); + + if (refcount == 1) { + ev_break (EV_A_ EVBREAK_ONE); + } + } +} + +static void rspamd_worker_terminate_handlers (struct rspamd_worker *w) { guint i; gboolean (*cb)(struct rspamd_worker *); - gboolean ret = FALSE; + struct rspamd_abstract_worker_ctx *actx; + struct ev_loop *final_gift, *orig_loop; + static ev_timer margin_call; + static int nchecks = 0; + + if (w->finish_actions->len == 0) { + /* Nothing to do */ + return; + } + + actx = (struct rspamd_abstract_worker_ctx *)w->ctx; + + /* + * Here are dragons: + * - we create a new loop + * - we set a new ev_loop for worker via injection over rspamd_abstract_worker_ctx + * - then we run finish actions + * - then we create a special timer to kill worker if it fails to finish + */ + final_gift = ev_loop_new (EVBACKEND_ALL); + orig_loop = actx->event_loop; + actx->event_loop = final_gift; + margin_call.data = &nchecks; + ev_timer_init (&margin_call, rspamd_worker_check_finished, 0.1, + 0.1); + ev_timer_start (final_gift, &margin_call); for (i = 0; i < w->finish_actions->len; i ++) { cb = g_ptr_array_index (w->finish_actions, i); - if (cb (w)) { - ret = TRUE; - } + cb (w); } - return ret; + ev_run (final_gift, 0); + ev_loop_destroy (final_gift); + /* Restore original loop */ + actx->event_loop = orig_loop; +} + +static void +rspamd_worker_on_delayed_shutdown (EV_P_ ev_timer *w, int revents) +{ + ev_break (loop, EVBREAK_ALL); +#ifdef WITH_GPERF_TOOLS + ProfilerStop (); +#endif } + /* * Config reload is designed by sending sigusr2 to active workers and pending shutdown of them */ @@ -108,7 +164,10 @@ rspamd_worker_usr2_handler (struct rspamd_worker_signal_handler *sigh, void *arg struct timeval tv; if (!sigh->worker->wanna_die) { - rspamd_worker_ignore_signal (SIGUSR2); + static ev_timer shutdown_ev; + + rspamd_worker_ignore_signal (sigh); + tv.tv_sec = SOFT_SHUTDOWN_TIME; tv.tv_usec = 0; sigh->worker->wanna_die = TRUE; @@ -119,7 +178,9 @@ rspamd_worker_usr2_handler (struct rspamd_worker_signal_handler *sigh, void *arg G_STRFUNC, "worker's shutdown is pending in %d sec", SOFT_SHUTDOWN_TIME); - event_base_loopexit (sigh->base, &tv); + ev_timer_init (&shutdown_ev, rspamd_worker_on_delayed_shutdown, + SOFT_SHUTDOWN_TIME, 0.0); + ev_timer_start (sigh->event_loop, &shutdown_ev); rspamd_worker_stop_accept (sigh->worker); } @@ -142,9 +203,10 @@ rspamd_worker_usr1_handler (struct rspamd_worker_signal_handler *sigh, void *arg static gboolean rspamd_worker_term_handler (struct rspamd_worker_signal_handler *sigh, void *arg) { - struct timeval tv; - if (!sigh->worker->wanna_die) { + static ev_timer shutdown_ev; + + rspamd_worker_ignore_signal (sigh); rspamd_default_log_function (G_LOG_LEVEL_INFO, sigh->worker->srv->server_pool->tag.tagname, sigh->worker->srv->server_pool->tag.uid, @@ -152,19 +214,11 @@ rspamd_worker_term_handler (struct rspamd_worker_signal_handler *sigh, void *arg "terminating after receiving signal %s", g_strsignal (sigh->signo)); - tv.tv_usec = 0; - if (rspamd_worker_terminate_handlers (sigh->worker)) { - tv.tv_sec = SOFT_SHUTDOWN_TIME; - } - else { - tv.tv_sec = 0; - } - + rspamd_worker_terminate_handlers (sigh->worker); sigh->worker->wanna_die = 1; - event_base_loopexit (sigh->base, &tv); -#ifdef WITH_GPERF_TOOLS - ProfilerStop (); -#endif + ev_timer_init (&shutdown_ev, rspamd_worker_on_delayed_shutdown, + 0.0, 0.0); + ev_timer_start (sigh->event_loop, &shutdown_ev); rspamd_worker_stop_accept (sigh->worker); } @@ -173,10 +227,10 @@ rspamd_worker_term_handler (struct rspamd_worker_signal_handler *sigh, void *arg } static void -rspamd_worker_signal_handle (int fd, short what, void *arg) +rspamd_worker_signal_handle (EV_P_ ev_signal *w, int revents) { struct rspamd_worker_signal_handler *sigh = - (struct rspamd_worker_signal_handler *) arg; + (struct rspamd_worker_signal_handler *)w->data; struct rspamd_worker_signal_cb *cb, *cbtmp; /* Call all signal handlers registered */ @@ -188,15 +242,14 @@ rspamd_worker_signal_handle (int fd, short what, void *arg) } static void -rspamd_worker_ignore_signal (int signo) +rspamd_worker_ignore_signal (struct rspamd_worker_signal_handler *sigh) { - struct sigaction sig; + sigset_t set; - sigemptyset (&sig.sa_mask); - sigaddset (&sig.sa_mask, signo); - sig.sa_handler = SIG_IGN; - sig.sa_flags = 0; - sigaction (signo, &sig, NULL); + ev_signal_stop (sigh->event_loop, &sigh->ev_sig); + sigemptyset (&set); + sigaddset (&set, sigh->signo); + sigprocmask (SIG_BLOCK, &set, NULL); } static void @@ -222,14 +275,14 @@ rspamd_sigh_free (void *p) g_free (cb); } - event_del (&sigh->ev); + ev_signal_stop (sigh->event_loop, &sigh->ev_sig); rspamd_worker_default_signal (sigh->signo); g_free (sigh); } void rspamd_worker_set_signal_handler (int signo, struct rspamd_worker *worker, - struct event_base *base, + struct ev_loop *event_loop, rspamd_worker_signal_handler handler, void *handler_data) { @@ -242,12 +295,12 @@ rspamd_worker_set_signal_handler (int signo, struct rspamd_worker *worker, sigh = g_malloc0 (sizeof (*sigh)); sigh->signo = signo; sigh->worker = worker; - sigh->base = base; + sigh->event_loop = event_loop; sigh->enabled = TRUE; - signal_set (&sigh->ev, signo, rspamd_worker_signal_handle, sigh); - event_base_set (base, &sigh->ev); - signal_add (&sigh->ev, NULL); + sigh->ev_sig.data = sigh; + ev_signal_init (&sigh->ev_sig, rspamd_worker_signal_handle, signo); + ev_signal_start (event_loop, &sigh->ev_sig); g_hash_table_insert (worker->signal_events, GINT_TO_POINTER (signo), @@ -261,50 +314,32 @@ rspamd_worker_set_signal_handler (int signo, struct rspamd_worker *worker, } void -rspamd_worker_init_signals (struct rspamd_worker *worker, struct event_base *base) +rspamd_worker_init_signals (struct rspamd_worker *worker, + struct ev_loop *event_loop) { - struct sigaction signals; - /* We ignore these signals in the worker */ - rspamd_worker_ignore_signal (SIGPIPE); - rspamd_worker_ignore_signal (SIGALRM); - rspamd_worker_ignore_signal (SIGCHLD); - /* A set of terminating signals */ - rspamd_worker_set_signal_handler (SIGTERM, worker, base, + rspamd_worker_set_signal_handler (SIGTERM, worker, event_loop, rspamd_worker_term_handler, NULL); - rspamd_worker_set_signal_handler (SIGINT, worker, base, + rspamd_worker_set_signal_handler (SIGINT, worker, event_loop, rspamd_worker_term_handler, NULL); - rspamd_worker_set_signal_handler (SIGHUP, worker, base, + rspamd_worker_set_signal_handler (SIGHUP, worker, event_loop, rspamd_worker_term_handler, NULL); /* Special purpose signals */ - rspamd_worker_set_signal_handler (SIGUSR1, worker, base, + rspamd_worker_set_signal_handler (SIGUSR1, worker, event_loop, rspamd_worker_usr1_handler, NULL); - rspamd_worker_set_signal_handler (SIGUSR2, worker, base, + rspamd_worker_set_signal_handler (SIGUSR2, worker, event_loop, rspamd_worker_usr2_handler, NULL); - - /* Unblock all signals processed */ - sigemptyset (&signals.sa_mask); - sigaddset (&signals.sa_mask, SIGTERM); - sigaddset (&signals.sa_mask, SIGINT); - sigaddset (&signals.sa_mask, SIGHUP); - sigaddset (&signals.sa_mask, SIGCHLD); - sigaddset (&signals.sa_mask, SIGUSR1); - sigaddset (&signals.sa_mask, SIGUSR2); - sigaddset (&signals.sa_mask, SIGALRM); - sigaddset (&signals.sa_mask, SIGPIPE); - - sigprocmask (SIG_UNBLOCK, &signals.sa_mask, NULL); } -struct event_base * +struct ev_loop * rspamd_prepare_worker (struct rspamd_worker *worker, const char *name, - void (*accept_handler)(int, short, void *)) + rspamd_accept_handler hdl) { - struct event_base *ev_base; - struct event *accept_events; + struct ev_loop *event_loop; GList *cur; struct rspamd_worker_listen_socket *ls; + struct rspamd_worker_accept_event *accept_ev; #ifdef WITH_PROFILER extern void _start (void), etext (void); @@ -316,65 +351,61 @@ rspamd_prepare_worker (struct rspamd_worker *worker, const char *name, worker->signal_events = g_hash_table_new_full (g_direct_hash, g_direct_equal, NULL, rspamd_sigh_free); - ev_base = event_init (); + event_loop = ev_loop_new (EVFLAG_SIGNALFD); - rspamd_worker_init_signals (worker, ev_base); - rspamd_control_worker_add_default_handler (worker, ev_base); + worker->srv->event_loop = event_loop; + + rspamd_worker_init_signals (worker, event_loop); + rspamd_control_worker_add_default_handler (worker, event_loop); #ifdef WITH_HIREDIS rspamd_redis_pool_config (worker->srv->cfg->redis_pool, - worker->srv->cfg, ev_base); + worker->srv->cfg, event_loop); #endif /* Accept all sockets */ - if (accept_handler) { + if (hdl) { cur = worker->cf->listen_socks; while (cur) { ls = cur->data; if (ls->fd != -1) { - accept_events = g_malloc0 (sizeof (struct event) * 2); - event_set (&accept_events[0], ls->fd, EV_READ | EV_PERSIST, - accept_handler, worker); - event_base_set (ev_base, &accept_events[0]); - event_add (&accept_events[0], NULL); - worker->accept_events = g_list_prepend (worker->accept_events, - accept_events); + accept_ev = g_malloc0 (sizeof (*accept_ev)); + accept_ev->event_loop = event_loop; + accept_ev->accept_ev.data = worker; + ev_io_init (&accept_ev->accept_ev, hdl, ls->fd, EV_READ); + ev_io_start (event_loop, &accept_ev->accept_ev); + + DL_APPEND (worker->accept_events, accept_ev); } cur = g_list_next (cur); } } - return ev_base; + return event_loop; } void rspamd_worker_stop_accept (struct rspamd_worker *worker) { - GList *cur; - struct event *events; + struct rspamd_worker_accept_event *cur, *tmp; /* Remove all events */ - cur = worker->accept_events; - while (cur) { - events = cur->data; + DL_FOREACH_SAFE (worker->accept_events, cur, tmp) { - if (rspamd_event_pending (&events[0], EV_TIMEOUT|EV_READ|EV_WRITE)) { - event_del (&events[0]); + if (ev_is_active (&cur->accept_ev) || ev_is_pending (&cur->accept_ev)) { + ev_io_stop (cur->event_loop, &cur->accept_ev); } - if (rspamd_event_pending (&events[1], EV_TIMEOUT|EV_READ|EV_WRITE)) { - event_del (&events[1]); + + if (ev_is_active (&cur->throttling_ev) || ev_is_pending (&cur->throttling_ev)) { + ev_timer_stop (cur->event_loop, &cur->throttling_ev); } - cur = g_list_next (cur); - g_free (events); + g_free (cur); } - if (worker->accept_events != NULL) { - g_list_free (worker->accept_events); - } /* XXX: we need to do it much later */ #if 0 g_hash_table_iter_init (&it, worker->signal_events); @@ -435,7 +466,7 @@ rspamd_controller_send_error (struct rspamd_http_connection_entry *entry, NULL, "application/json", entry, - entry->rt->ptv); + entry->rt->timeout); entry->is_reply = TRUE; } @@ -467,7 +498,7 @@ rspamd_controller_send_string (struct rspamd_http_connection_entry *entry, NULL, "application/json", entry, - entry->rt->ptv); + entry->rt->timeout); entry->is_reply = TRUE; } @@ -493,7 +524,7 @@ rspamd_controller_send_ucl (struct rspamd_http_connection_entry *entry, NULL, "application/json", entry, - entry->rt->ptv); + entry->rt->timeout); entry->is_reply = TRUE; } @@ -629,11 +660,30 @@ rspamd_worker_set_limits (struct rspamd_main *rspamd_main, } } +static void +rspamd_worker_on_term (EV_P_ ev_child *w, int revents) +{ + struct rspamd_worker *wrk = (struct rspamd_worker *)w->data; + + if (wrk->ppid == getpid ()) { + if (wrk->term_handler) { + wrk->term_handler (EV_A_ w, wrk->srv, wrk); + } + else { + rspamd_check_termination_clause (wrk->srv, wrk, w->rstatus); + } + } + else { + /* Ignore SIGCHLD for not our children... */ + } +} + struct rspamd_worker * rspamd_fork_worker (struct rspamd_main *rspamd_main, - struct rspamd_worker_conf *cf, - guint index, - struct event_base *ev_base) + struct rspamd_worker_conf *cf, + guint index, + struct ev_loop *ev_base, + rspamd_worker_term_cb term_handler) { struct rspamd_worker *wrk; gint rc; @@ -663,6 +713,7 @@ rspamd_fork_worker (struct rspamd_main *rspamd_main, wrk->ppid = getpid (); wrk->pid = fork (); wrk->cores_throttled = rspamd_main->cores_throttling; + wrk->term_handler = term_handler; switch (wrk->pid) { case 0: @@ -682,10 +733,18 @@ rspamd_fork_worker (struct rspamd_main *rspamd_main, evutil_secure_rng_init (); #endif + /* + * Libev stores all signals in a global table, so + * previous handlers must be explicitly detached and forgotten + * before starting a new loop + */ + ev_signal_stop (rspamd_main->event_loop, &rspamd_main->int_ev); + ev_signal_stop (rspamd_main->event_loop, &rspamd_main->term_ev); + ev_signal_stop (rspamd_main->event_loop, &rspamd_main->hup_ev); + ev_signal_stop (rspamd_main->event_loop, &rspamd_main->usr1_ev); /* Remove the inherited event base */ - event_reinit (rspamd_main->ev_base); - event_base_free (rspamd_main->ev_base); - + ev_loop_destroy (rspamd_main->event_loop); + rspamd_main->event_loop = NULL; /* Drop privileges */ rspamd_worker_drop_priv (rspamd_main); /* Set limits */ @@ -721,16 +780,6 @@ rspamd_fork_worker (struct rspamd_main *rspamd_main, rspamd_log_open (rspamd_main->logger); wrk->start_time = rspamd_get_calendar_ticks (); -#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION <= 30)) - # if (GLIB_MINOR_VERSION > 20) - /* Ugly hack for old glib */ - if (!g_thread_get_initialized ()) { - g_thread_init (NULL); - } -# else - g_thread_init (NULL); -# endif -#endif if (cf->bind_conf) { msg_info_main ("starting %s process %P (%d); listen on: %s", cf->worker->name, @@ -765,6 +814,9 @@ rspamd_fork_worker (struct rspamd_main *rspamd_main, rspamd_socket_nonblocking (wrk->control_pipe[0]); rspamd_socket_nonblocking (wrk->srv_pipe[0]); rspamd_srv_start_watching (rspamd_main, wrk, ev_base); + wrk->cld_ev.data = wrk; + ev_child_init (&wrk->cld_ev, rspamd_worker_on_term, wrk->pid, 0); + ev_child_start (rspamd_main->event_loop, &wrk->cld_ev); /* Insert worker into worker's table, pid is index */ g_hash_table_insert (rspamd_main->workers, GSIZE_TO_POINTER ( wrk->pid), wrk); @@ -867,11 +919,10 @@ struct rspamd_worker_session_elt { }; struct rspamd_worker_session_cache { - struct event_base *ev_base; + struct ev_loop *ev_base; GHashTable *cache; struct rspamd_config *cfg; - struct timeval tv; - struct event periodic; + struct ev_timer periodic; }; static gint @@ -885,9 +936,10 @@ rspamd_session_cache_sort_cmp (gconstpointer pa, gconstpointer pb) } static void -rspamd_sessions_cache_periodic (gint fd, short what, gpointer p) +rspamd_sessions_cache_periodic (EV_P_ ev_timer *w, int revents) { - struct rspamd_worker_session_cache *c = p; + struct rspamd_worker_session_cache *c = + (struct rspamd_worker_session_cache *)w->data; GHashTableIter it; gchar timebuf[32]; gpointer k, v; @@ -919,11 +971,13 @@ rspamd_sessions_cache_periodic (gint fd, short what, gpointer p) timebuf); } } + + ev_timer_again (EV_A_ w); } void * rspamd_worker_session_cache_new (struct rspamd_worker *w, - struct event_base *ev_base) + struct ev_loop *ev_base) { struct rspamd_worker_session_cache *c; static const gdouble periodic_interval = 60.0; @@ -933,11 +987,10 @@ rspamd_worker_session_cache_new (struct rspamd_worker *w, c->cache = g_hash_table_new_full (g_direct_hash, g_direct_equal, NULL, g_free); c->cfg = w->srv->cfg; - double_to_tv (periodic_interval, &c->tv); - event_set (&c->periodic, -1, EV_TIMEOUT|EV_PERSIST, - rspamd_sessions_cache_periodic, c); - event_base_set (ev_base, &c->periodic); - event_add (&c->periodic, &c->tv); + c->periodic.data = c; + ev_timer_init (&c->periodic, rspamd_sessions_cache_periodic, periodic_interval, + periodic_interval); + ev_timer_start (ev_base, &c->periodic); return c; } @@ -975,7 +1028,7 @@ rspamd_worker_monitored_on_change (struct rspamd_monitored_ctx *ctx, { struct rspamd_worker *worker = ud; struct rspamd_config *cfg = worker->srv->cfg; - struct event_base *ev_base; + struct ev_loop *ev_base; guchar tag[RSPAMD_MONITORED_TAG_LEN]; static struct rspamd_srv_command srv_cmd; @@ -995,7 +1048,7 @@ rspamd_worker_monitored_on_change (struct rspamd_monitored_ctx *ctx, void rspamd_worker_init_monitored (struct rspamd_worker *worker, - struct event_base *ev_base, + struct ev_loop *ev_base, struct rspamd_dns_resolver *resolver) { rspamd_monitored_ctx_config (worker->srv->cfg->monitored_ctx, @@ -1129,4 +1182,148 @@ rspamd_set_crash_handler (struct rspamd_main *rspamd_main) sigaction (SIGFPE, &sa, NULL); sigaction (SIGSYS, &sa, NULL); #endif -}
\ No newline at end of file +} + +static void +rspamd_enable_accept_event (EV_P_ ev_timer *w, int revents) +{ + struct rspamd_worker_accept_event *ac_ev = + (struct rspamd_worker_accept_event *)w->data; + + ev_timer_stop (EV_A_ w); + ev_io_start (EV_A_ &ac_ev->accept_ev); +} + +void +rspamd_worker_throttle_accept_events (gint sock, void *data) +{ + struct rspamd_worker_accept_event *head, *cur; + const gdouble throttling = 0.5; + + head = (struct rspamd_worker_accept_event *)data; + + DL_FOREACH (head, cur) { + + ev_io_stop (cur->event_loop, &cur->accept_ev); + cur->throttling_ev.data = cur; + ev_timer_init (&cur->throttling_ev, rspamd_enable_accept_event, + throttling, 0.0); + ev_timer_start (cur->event_loop, &cur->throttling_ev); + } +} + +gboolean +rspamd_check_termination_clause (struct rspamd_main *rspamd_main, + struct rspamd_worker *wrk, + int res) +{ + gboolean need_refork = TRUE; + + if (wrk->wanna_die || rspamd_main->wanna_die) { + /* Do not refork workers that are intended to be terminated */ + need_refork = FALSE; + } + + if (WIFEXITED (res) && WEXITSTATUS (res) == 0) { + /* Normal worker termination, do not fork one more */ + msg_info_main ("%s process %P terminated normally", + g_quark_to_string (wrk->type), + wrk->pid); + need_refork = FALSE; + } + else { + if (WIFSIGNALED (res)) { +#ifdef WCOREDUMP + if (WCOREDUMP (res)) { + msg_warn_main ( + "%s process %P terminated abnormally by signal: %s" + " and created core file", + g_quark_to_string (wrk->type), + wrk->pid, + g_strsignal (WTERMSIG (res))); + } + else { +#ifdef HAVE_SYS_RESOURCE_H + struct rlimit rlmt; + (void) getrlimit (RLIMIT_CORE, &rlmt); + + msg_warn_main ( + "%s process %P terminated abnormally by signal: %s" + " but NOT created core file (throttled=%s); " + "core file limits: %L current, %L max", + g_quark_to_string (wrk->type), + wrk->pid, + g_strsignal (WTERMSIG (res)), + wrk->cores_throttled ? "yes" : "no", + (gint64) rlmt.rlim_cur, + (gint64) rlmt.rlim_max); +#else + msg_warn_main ( + "%s process %P terminated abnormally by signal: %s" + " but NOT created core file (throttled=%s); ", + g_quark_to_string (wrk->type), + wrk->pid, + g_strsignal (WTERMSIG (res)), + wrk->cores_throttled ? "yes" : "no"); +#endif + } +#else + msg_warn_main ( + "%s process %P terminated abnormally by signal: %s", + g_quark_to_string (wrk->type), + wrk->pid, + g_strsignal (WTERMSIG (res))); +#endif + if (WTERMSIG (res) == SIGUSR2) { + /* + * It is actually race condition when not started process + * has been requested to be reloaded. + * + * We shouldn't refork on this + */ + need_refork = FALSE; + } + } + else { + msg_warn_main ("%s process %P terminated abnormally " + "with exit code %d", + g_quark_to_string (wrk->type), + wrk->pid, + WEXITSTATUS (res)); + } + } + + return need_refork; +} + +#ifdef WITH_HYPERSCAN +gboolean +rspamd_worker_hyperscan_ready (struct rspamd_main *rspamd_main, + struct rspamd_worker *worker, gint fd, + gint attached_fd, + struct rspamd_control_command *cmd, + gpointer ud) +{ + struct rspamd_control_reply rep; + struct rspamd_re_cache *cache = worker->srv->cfg->re_cache; + + memset (&rep, 0, sizeof (rep)); + rep.type = RSPAMD_CONTROL_HYPERSCAN_LOADED; + + if (!rspamd_re_cache_is_hs_loaded (cache) || cmd->cmd.hs_loaded.forced) { + msg_info ("loading hyperscan expressions after receiving compilation " + "notice: %s", + (!rspamd_re_cache_is_hs_loaded (cache)) ? + "new db" : "forced update"); + rep.reply.hs_loaded.status = rspamd_re_cache_load_hyperscan ( + worker->srv->cfg->re_cache, cmd->cmd.hs_loaded.cache_dir); + } + + if (write (fd, &rep, sizeof (rep)) != sizeof (rep)) { + msg_err ("cannot write reply to the control socket: %s", + strerror (errno)); + } + + return TRUE; +} +#endif
\ No newline at end of file diff --git a/src/libserver/worker_util.h b/src/libserver/worker_util.h index 3186025b3..49bbda62b 100644 --- a/src/libserver/worker_util.h +++ b/src/libserver/worker_util.h @@ -33,9 +33,12 @@ struct rspamd_worker_signal_handler; /** * Init basic signals for a worker * @param worker - * @param base + * @param event_loop */ -void rspamd_worker_init_signals (struct rspamd_worker *worker, struct event_base *base); +void rspamd_worker_init_signals (struct rspamd_worker *worker, struct ev_loop *event_loop); + +typedef void (*rspamd_accept_handler)(struct ev_loop *loop, ev_io *w, int revents); + /** * Prepare worker's startup * @param worker worker structure @@ -44,16 +47,16 @@ void rspamd_worker_init_signals (struct rspamd_worker *worker, struct event_base * @param accept_handler handler of accept event for listen sockets * @return event base suitable for a worker */ -struct event_base * +struct ev_loop * rspamd_prepare_worker (struct rspamd_worker *worker, const char *name, - void (*accept_handler)(int, short, void *)); + rspamd_accept_handler hdl); /** * Set special signal handler for a worker */ void rspamd_worker_set_signal_handler (int signo, struct rspamd_worker *worker, - struct event_base *base, + struct ev_loop *event_loop, rspamd_worker_signal_handler handler, void *handler_data); @@ -124,8 +127,6 @@ void rspamd_controller_send_ucl (struct rspamd_http_connection_entry *entry, */ worker_t * rspamd_get_worker_by_type (struct rspamd_config *cfg, GQuark type); -void rspamd_worker_stop_accept (struct rspamd_worker *worker); - /** * Block signals before terminations */ @@ -162,7 +163,7 @@ gboolean rspamd_worker_is_primary_controller (struct rspamd_worker *w); * @return */ void * rspamd_worker_session_cache_new (struct rspamd_worker *w, - struct event_base *ev_base); + struct ev_loop *ev_base); /** * Adds a new session identified by pointer @@ -185,7 +186,9 @@ void rspamd_worker_session_cache_remove (void *cache, void *ptr); * Fork new worker with the specified configuration */ struct rspamd_worker *rspamd_fork_worker (struct rspamd_main *, - struct rspamd_worker_conf *, guint idx, struct event_base *ev_base); + struct rspamd_worker_conf *, guint idx, + struct ev_loop *ev_base, + rspamd_worker_term_cb term_handler); /** * Sets crash signals handlers if compiled with libunwind @@ -199,9 +202,36 @@ void rspamd_set_crash_handler (struct rspamd_main *); * @param resolver */ void rspamd_worker_init_monitored (struct rspamd_worker *worker, - struct event_base *ev_base, + struct ev_loop *ev_base, struct rspamd_dns_resolver *resolver); +/** + * Performs throttling for accept events + * @param sock + * @param data struct rspamd_worker_accept_event * list + */ +void rspamd_worker_throttle_accept_events (gint sock, void *data); + +/** + * Checks (and logs) the worker's termination status. Returns TRUE if a worker + * should be restarted. + * @param rspamd_main + * @param wrk + * @param status waitpid res + * @return TRUE if refork is desired + */ +gboolean rspamd_check_termination_clause (struct rspamd_main *rspamd_main, + struct rspamd_worker *wrk, int status); + +#ifdef WITH_HYPERSCAN +struct rspamd_control_command; +gboolean rspamd_worker_hyperscan_ready (struct rspamd_main *rspamd_main, + struct rspamd_worker *worker, gint fd, + gint attached_fd, + struct rspamd_control_command *cmd, + gpointer ud); +#endif + #define msg_err_main(...) rspamd_default_log_function (G_LOG_LEVEL_CRITICAL, \ rspamd_main->server_pool->tag.tagname, rspamd_main->server_pool->tag.uid, \ G_STRFUNC, \ diff --git a/src/libstat/backends/redis_backend.c b/src/libstat/backends/redis_backend.c index 5976968a7..e430e491e 100644 --- a/src/libstat/backends/redis_backend.c +++ b/src/libstat/backends/redis_backend.c @@ -22,7 +22,7 @@ #ifdef WITH_HIREDIS #include "hiredis.h" -#include "adapters/libevent.h" +#include "adapters/libev.h" #include "ref.h" #define msg_debug_stat_redis(...) rspamd_conditional_debug_fast (NULL, NULL, \ @@ -70,7 +70,7 @@ struct redis_stat_runtime { struct redis_stat_ctx *ctx; struct rspamd_task *task; struct upstream *selected; - struct event timeout_event; + ev_timer timeout_event; GArray *results; struct rspamd_statfile_config *stcf; gchar *redis_object_expanded; @@ -87,7 +87,7 @@ struct rspamd_redis_stat_cbdata; struct rspamd_redis_stat_elt { struct redis_stat_ctx *ctx; struct rspamd_stat_async_elt *async; - struct event_base *ev_base; + struct ev_loop *event_loop; ucl_object_t *stat; struct rspamd_redis_stat_cbdata *cbdata; }; @@ -986,7 +986,7 @@ rspamd_redis_async_stat_cb (struct rspamd_stat_async_elt *elt, gpointer d) g_assert (cbdata->redis != NULL); - redisLibeventAttach (cbdata->redis, redis_elt->ev_base); + redisLibevAttach (redis_elt->event_loop, cbdata->redis); cbdata->inflight = 1; cbdata->cur = ucl_object_typed_new (UCL_OBJECT); @@ -1019,8 +1019,8 @@ rspamd_redis_fin (gpointer data) rt->has_event = FALSE; /* Stop timeout */ - if (rspamd_event_pending (&rt->timeout_event, EV_TIMEOUT)) { - event_del (&rt->timeout_event); + if (ev_is_active (&rt->timeout_event)) { + ev_timer_stop (rt->task->event_loop, &rt->timeout_event); } if (rt->redis) { @@ -1039,9 +1039,7 @@ rspamd_redis_fin_learn (gpointer data) rt->has_event = FALSE; /* Stop timeout */ - if (rspamd_event_pending (&rt->timeout_event, EV_TIMEOUT)) { - event_del (&rt->timeout_event); - } + ev_timer_stop (rt->task->event_loop, &rt->timeout_event); if (rt->redis) { redis = rt->redis; @@ -1052,9 +1050,9 @@ rspamd_redis_fin_learn (gpointer data) } static void -rspamd_redis_timeout (gint fd, short what, gpointer d) +rspamd_redis_timeout (EV_P_ ev_timer *w, int revents) { - struct redis_stat_runtime *rt = REDIS_RUNTIME (d); + struct redis_stat_runtime *rt = REDIS_RUNTIME (w->data); struct rspamd_task *task; redisAsyncContext *redis; @@ -1444,7 +1442,7 @@ rspamd_redis_init (struct rspamd_stat_ctx *ctx, backend->stcf = stf; st_elt = g_malloc0 (sizeof (*st_elt)); - st_elt->ev_base = ctx->ev_base; + st_elt->event_loop = ctx->event_loop; st_elt->ctx = backend; backend->stat_elt = rspamd_stat_ctx_register_async ( rspamd_redis_async_stat_cb, @@ -1536,7 +1534,7 @@ rspamd_redis_runtime (struct rspamd_task *task, return NULL; } - redisLibeventAttach (rt->redis, task->ev_base); + redisLibevAttach (task->event_loop, rt->redis); rspamd_redis_maybe_auth (ctx, rt->redis); return rt; @@ -1562,7 +1560,6 @@ rspamd_redis_process_tokens (struct rspamd_task *task, { struct redis_stat_runtime *rt = REDIS_RUNTIME (p); rspamd_fstring_t *query; - struct timeval tv; gint ret; const gchar *learned_key = "learns"; @@ -1591,13 +1588,16 @@ rspamd_redis_process_tokens (struct rspamd_task *task, rspamd_session_add_event (task->s, rspamd_redis_fin, rt, M); rt->has_event = TRUE; - if (rspamd_event_pending (&rt->timeout_event, EV_TIMEOUT)) { - event_del (&rt->timeout_event); + + if (ev_is_active (&rt->timeout_event)) { + ev_timer_again (task->event_loop, &rt->timeout_event); + } + else { + rt->timeout_event.data = rt; + ev_timer_init (&rt->timeout_event, rspamd_redis_timeout, + rt->ctx->timeout, 0.); + ev_timer_start (task->event_loop, &rt->timeout_event); } - event_set (&rt->timeout_event, -1, EV_TIMEOUT, rspamd_redis_timeout, rt); - event_base_set (task->ev_base, &rt->timeout_event); - double_to_tv (rt->ctx->timeout, &tv); - event_add (&rt->timeout_event, &tv); query = rspamd_redis_tokens_to_query (task, rt, tokens, rt->ctx->new_schema ? "HGET" : "HMGET", @@ -1628,8 +1628,8 @@ rspamd_redis_finalize_process (struct rspamd_task *task, gpointer runtime, struct redis_stat_runtime *rt = REDIS_RUNTIME (runtime); redisAsyncContext *redis; - if (rspamd_event_pending (&rt->timeout_event, EV_TIMEOUT)) { - event_del (&rt->timeout_event); + if (ev_is_active (&rt->timeout_event)) { + ev_timer_stop (task->event_loop, &rt->timeout_event); } if (rt->redis) { @@ -1653,7 +1653,6 @@ rspamd_redis_learn_tokens (struct rspamd_task *task, GPtrArray *tokens, struct upstream *up; struct upstream_list *ups; rspamd_inet_addr_t *addr; - struct timeval tv; rspamd_fstring_t *query; const gchar *redis_cmd; rspamd_token_t *tok; @@ -1704,7 +1703,7 @@ rspamd_redis_learn_tokens (struct rspamd_task *task, GPtrArray *tokens, g_assert (rt->redis != NULL); - redisLibeventAttach (rt->redis, task->ev_base); + redisLibevAttach (task->event_loop, rt->redis); rspamd_redis_maybe_auth (rt->ctx, rt->redis); /* @@ -1802,13 +1801,15 @@ rspamd_redis_learn_tokens (struct rspamd_task *task, GPtrArray *tokens, rt->has_event = TRUE; /* Set timeout */ - if (rspamd_event_pending (&rt->timeout_event, EV_TIMEOUT)) { - event_del (&rt->timeout_event); + if (ev_is_active (&rt->timeout_event)) { + ev_timer_again (task->event_loop, &rt->timeout_event); + } + else { + rt->timeout_event.data = rt; + ev_timer_init (&rt->timeout_event, rspamd_redis_timeout, + rt->ctx->timeout, 0.); + ev_timer_start (task->event_loop, &rt->timeout_event); } - event_set (&rt->timeout_event, -1, EV_TIMEOUT, rspamd_redis_timeout, rt); - event_base_set (task->ev_base, &rt->timeout_event); - double_to_tv (rt->ctx->timeout, &tv); - event_add (&rt->timeout_event, &tv); return TRUE; } @@ -1827,8 +1828,8 @@ rspamd_redis_finalize_learn (struct rspamd_task *task, gpointer runtime, struct redis_stat_runtime *rt = REDIS_RUNTIME (runtime); redisAsyncContext *redis; - if (rspamd_event_pending (&rt->timeout_event, EV_TIMEOUT)) { - event_del (&rt->timeout_event); + if (ev_is_active (&rt->timeout_event)) { + ev_timer_stop (task->event_loop, &rt->timeout_event); } if (rt->redis) { diff --git a/src/libstat/classifiers/bayes.c b/src/libstat/classifiers/bayes.c index 2b0cf21e8..eca94156c 100644 --- a/src/libstat/classifiers/bayes.c +++ b/src/libstat/classifiers/bayes.c @@ -256,7 +256,7 @@ bayes_classify_token (struct rspamd_classifier *ctx, gboolean bayes_init (struct rspamd_config *cfg, - struct event_base *ev_base, + struct ev_loop *ev_base, struct rspamd_classifier *cl) { cl->cfg->flags |= RSPAMD_FLAG_CLASSIFIER_INTEGER; diff --git a/src/libstat/classifiers/classifiers.h b/src/libstat/classifiers/classifiers.h index fd6daf433..738a5e8c9 100644 --- a/src/libstat/classifiers/classifiers.h +++ b/src/libstat/classifiers/classifiers.h @@ -3,7 +3,7 @@ #include "config.h" #include "mem_pool.h" -#include <event.h> +#include "contrib/libev/ev.h" #define RSPAMD_DEFAULT_CLASSIFIER "bayes" /* Consider this value as 0 */ @@ -19,7 +19,7 @@ struct token_node_s; struct rspamd_stat_classifier { char *name; gboolean (*init_func)(struct rspamd_config *cfg, - struct event_base *ev_base, + struct ev_loop *ev_base, struct rspamd_classifier *cl); gboolean (*classify_func)(struct rspamd_classifier * ctx, GPtrArray *tokens, @@ -35,7 +35,7 @@ struct rspamd_stat_classifier { /* Bayes algorithm */ gboolean bayes_init (struct rspamd_config *cfg, - struct event_base *ev_base, + struct ev_loop *ev_base, struct rspamd_classifier *); gboolean bayes_classify (struct rspamd_classifier *ctx, GPtrArray *tokens, @@ -50,7 +50,7 @@ void bayes_fin (struct rspamd_classifier *); /* Generic lua classifier */ gboolean lua_classifier_init (struct rspamd_config *cfg, - struct event_base *ev_base, + struct ev_loop *ev_base, struct rspamd_classifier *); gboolean lua_classifier_classify (struct rspamd_classifier *ctx, GPtrArray *tokens, diff --git a/src/libstat/classifiers/lua_classifier.c b/src/libstat/classifiers/lua_classifier.c index 83ce7b0e1..21ecba7a1 100644 --- a/src/libstat/classifiers/lua_classifier.c +++ b/src/libstat/classifiers/lua_classifier.c @@ -48,7 +48,7 @@ INIT_LOG_MODULE(luacl) gboolean lua_classifier_init (struct rspamd_config *cfg, - struct event_base *ev_base, + struct ev_loop *ev_base, struct rspamd_classifier *cl) { struct rspamd_lua_classifier_ctx *ctx; diff --git a/src/libstat/learn_cache/redis_cache.c b/src/libstat/learn_cache/redis_cache.c index aea024e06..2313db0b2 100644 --- a/src/libstat/learn_cache/redis_cache.c +++ b/src/libstat/learn_cache/redis_cache.c @@ -21,7 +21,7 @@ #include "cryptobox.h" #include "ucl.h" #include "hiredis.h" -#include "adapters/libevent.h" +#include "adapters/libev.h" #include "lua/lua_common.h" #define REDIS_DEFAULT_TIMEOUT 0.5 @@ -45,7 +45,7 @@ struct rspamd_redis_cache_runtime { struct rspamd_redis_cache_ctx *ctx; struct rspamd_task *task; struct upstream *selected; - struct event timeout_event; + ev_timer timer_ev; redisAsyncContext *redis; gboolean has_event; }; @@ -92,9 +92,7 @@ rspamd_redis_cache_fin (gpointer data) redisAsyncContext *redis; rt->has_event = FALSE; - if (rspamd_event_pending (&rt->timeout_event, EV_TIMEOUT)) { - event_del (&rt->timeout_event); - } + ev_timer_stop (rt->task->event_loop, &rt->timer_ev); if (rt->redis) { redis = rt->redis; @@ -105,9 +103,10 @@ rspamd_redis_cache_fin (gpointer data) } static void -rspamd_redis_cache_timeout (gint fd, short what, gpointer d) +rspamd_redis_cache_timeout (EV_P_ ev_timer *w, int revents) { - struct rspamd_redis_cache_runtime *rt = d; + struct rspamd_redis_cache_runtime *rt = + (struct rspamd_redis_cache_runtime *)w->data; struct rspamd_task *task; task = rt->task; @@ -117,7 +116,7 @@ rspamd_redis_cache_timeout (gint fd, short what, gpointer d) rspamd_upstream_fail (rt->selected, FALSE); if (rt->has_event) { - rspamd_session_remove_event (task->s, rspamd_redis_cache_fin, d); + rspamd_session_remove_event (task->s, rspamd_redis_cache_fin, rt); } } @@ -398,11 +397,12 @@ rspamd_stat_cache_redis_runtime (struct rspamd_task *task, g_assert (rt->redis != NULL); - redisLibeventAttach (rt->redis, task->ev_base); + redisLibevAttach (task->event_loop, rt->redis); /* Now check stats */ - event_set (&rt->timeout_event, -1, EV_TIMEOUT, rspamd_redis_cache_timeout, rt); - event_base_set (task->ev_base, &rt->timeout_event); + rt->timer_ev.data = rt; + ev_timer_init (&rt->timer_ev, rspamd_redis_cache_timeout, + rt->ctx->timeout, 0.0); rspamd_redis_cache_maybe_auth (ctx, rt->redis); if (!learn) { @@ -418,7 +418,6 @@ rspamd_stat_cache_redis_check (struct rspamd_task *task, gpointer runtime) { struct rspamd_redis_cache_runtime *rt = runtime; - struct timeval tv; gchar *h; if (rspamd_session_blocked (task->s)) { @@ -431,8 +430,6 @@ rspamd_stat_cache_redis_check (struct rspamd_task *task, return RSPAMD_LEARN_INGORE; } - double_to_tv (rt->ctx->timeout, &tv); - if (redisAsyncCommand (rt->redis, rspamd_stat_cache_redis_get, rt, "HGET %s %s", rt->ctx->redis_object, h) == REDIS_OK) { @@ -440,7 +437,7 @@ rspamd_stat_cache_redis_check (struct rspamd_task *task, rspamd_redis_cache_fin, rt, M); - event_add (&rt->timeout_event, &tv); + ev_timer_start (rt->task->event_loop, &rt->timer_ev); rt->has_event = TRUE; } @@ -454,7 +451,6 @@ rspamd_stat_cache_redis_learn (struct rspamd_task *task, gpointer runtime) { struct rspamd_redis_cache_runtime *rt = runtime; - struct timeval tv; gchar *h; gint flag; @@ -465,7 +461,6 @@ rspamd_stat_cache_redis_learn (struct rspamd_task *task, h = rspamd_mempool_get_variable (task->task_pool, "words_hash"); g_assert (h != NULL); - double_to_tv (rt->ctx->timeout, &tv); flag = (task->flags & RSPAMD_TASK_FLAG_LEARN_SPAM) ? 1 : -1; if (redisAsyncCommand (rt->redis, rspamd_stat_cache_redis_set, rt, @@ -473,7 +468,7 @@ rspamd_stat_cache_redis_learn (struct rspamd_task *task, rt->ctx->redis_object, h, flag) == REDIS_OK) { rspamd_session_add_event (task->s, rspamd_redis_cache_fin, rt, M); - event_add (&rt->timeout_event, &tv); + ev_timer_start (rt->task->event_loop, &rt->timer_ev); rt->has_event = TRUE; } diff --git a/src/libstat/stat_api.h b/src/libstat/stat_api.h index f9d1aab5a..40a6bc716 100644 --- a/src/libstat/stat_api.h +++ b/src/libstat/stat_api.h @@ -19,7 +19,7 @@ #include "config.h" #include "task.h" #include <lua.h> -#include <event.h> +#include "contrib/libev/ev.h" /** * @file stat_api.h @@ -76,7 +76,7 @@ typedef enum rspamd_stat_result_e { * Initialise statistics modules * @param cfg */ -void rspamd_stat_init (struct rspamd_config *cfg, struct event_base *ev_base); +void rspamd_stat_init (struct rspamd_config *cfg, struct ev_loop *ev_base); /** * Finalize statistics diff --git a/src/libstat/stat_config.c b/src/libstat/stat_config.c index 272a64ddf..bc4c28b5d 100644 --- a/src/libstat/stat_config.c +++ b/src/libstat/stat_config.c @@ -95,7 +95,7 @@ static struct rspamd_stat_cache stat_caches[] = { }; void -rspamd_stat_init (struct rspamd_config *cfg, struct event_base *ev_base) +rspamd_stat_init (struct rspamd_config *cfg, struct ev_loop *ev_base) { GList *cur, *curst; struct rspamd_classifier_config *clf; @@ -163,7 +163,7 @@ rspamd_stat_init (struct rspamd_config *cfg, struct event_base *ev_base) stat_ctx->statfiles = g_ptr_array_new (); stat_ctx->classifiers = g_ptr_array_new (); stat_ctx->async_elts = g_queue_new (); - stat_ctx->ev_base = ev_base; + stat_ctx->event_loop = ev_base; stat_ctx->lua_stat_tokens_ref = -1; /* Interact with lua_stat */ @@ -510,25 +510,24 @@ rspamd_async_elt_dtor (struct rspamd_stat_async_elt *elt) elt->cleanup (elt, elt->ud); } - event_del (&elt->timer_ev); + ev_timer_stop (elt->event_loop, &elt->timer_ev); g_free (elt); } static void -rspamd_async_elt_on_timer (gint fd, short what, gpointer d) +rspamd_async_elt_on_timer (EV_P_ ev_timer *w, int revents) { - struct rspamd_stat_async_elt *elt = d; + struct rspamd_stat_async_elt *elt = (struct rspamd_stat_async_elt *)w->data; gdouble jittered_time; - event_del (&elt->timer_ev); if (elt->enabled) { elt->handler (elt, elt->ud); } jittered_time = rspamd_time_jitter (elt->timeout, 0); - double_to_tv (jittered_time, &elt->tv); - event_add (&elt->timer_ev, &elt->tv); + elt->timer_ev.repeat = jittered_time; + ev_timer_again (EV_A_ w); } struct rspamd_stat_async_elt* @@ -548,21 +547,20 @@ rspamd_stat_ctx_register_async (rspamd_stat_async_handler handler, elt->cleanup = cleanup; elt->ud = d; elt->timeout = timeout; + elt->event_loop = st_ctx->event_loop; REF_INIT_RETAIN (elt, rspamd_async_elt_dtor); /* Enabled by default */ - if (st_ctx->ev_base) { + if (st_ctx->event_loop) { elt->enabled = TRUE; - event_set (&elt->timer_ev, -1, EV_TIMEOUT, rspamd_async_elt_on_timer, elt); - event_base_set (st_ctx->ev_base, &elt->timer_ev); /* * First we set timeval to zero as we want cb to be executed as * fast as possible */ - elt->tv.tv_sec = 0; - elt->tv.tv_usec = 0; - event_add (&elt->timer_ev, &elt->tv); + elt->timer_ev.data = elt; + ev_timer_init (&elt->timer_ev, rspamd_async_elt_on_timer, 0.0, 0.0); + ev_timer_start (st_ctx->event_loop, &elt->timer_ev); } else { elt->enabled = FALSE; diff --git a/src/libstat/stat_internal.h b/src/libstat/stat_internal.h index a547ca93a..50dbae9c1 100644 --- a/src/libstat/stat_internal.h +++ b/src/libstat/stat_internal.h @@ -62,8 +62,8 @@ typedef void (*rspamd_stat_async_cleanup)(struct rspamd_stat_async_elt *elt, struct rspamd_stat_async_elt { rspamd_stat_async_handler handler; rspamd_stat_async_cleanup cleanup; - struct event timer_ev; - struct timeval tv; + struct ev_loop *event_loop; + ev_timer timer_ev; gdouble timeout; gboolean enabled; gpointer ud; @@ -93,7 +93,7 @@ struct rspamd_stat_ctx { struct rspamd_stat_tokenizer *tokenizer; gpointer tkcf; - struct event_base *ev_base; + struct ev_loop *event_loop; }; typedef enum rspamd_learn_cache_result { diff --git a/src/libutil/CMakeLists.txt b/src/libutil/CMakeLists.txt index f86d650f0..5a94a732c 100644 --- a/src/libutil/CMakeLists.txt +++ b/src/libutil/CMakeLists.txt @@ -1,7 +1,7 @@ # Librspamd-util SET(LIBRSPAMDUTILSRC ${CMAKE_CURRENT_SOURCE_DIR}/addr.c - ${CMAKE_CURRENT_SOURCE_DIR}/aio_event.c + ${CMAKE_CURRENT_SOURCE_DIR}/libev_helper.c ${CMAKE_CURRENT_SOURCE_DIR}/bloom.c ${CMAKE_CURRENT_SOURCE_DIR}/expression.c ${CMAKE_CURRENT_SOURCE_DIR}/fstring.c diff --git a/src/libutil/addr.c b/src/libutil/addr.c index 30a9ce66a..112c5d2cd 100644 --- a/src/libutil/addr.c +++ b/src/libutil/addr.c @@ -203,41 +203,10 @@ rspamd_ip_is_valid (const rspamd_inet_addr_t *addr) return ret; } -static void -rspamd_enable_accept_event (gint fd, short what, gpointer d) -{ - struct event *events = d; - - event_del (&events[1]); - event_add (&events[0], NULL); -} - -static void -rspamd_disable_accept_events (gint sock, GList *accept_events) -{ - GList *cur; - struct event *events; - const gdouble throttling = 0.5; - struct timeval tv; - struct event_base *ev_base; - - double_to_tv (throttling, &tv); - - for (cur = accept_events; cur != NULL; cur = g_list_next (cur)) { - events = cur->data; - - ev_base = event_get_base (&events[0]); - event_del (&events[0]); - event_set (&events[1], sock, EV_TIMEOUT, rspamd_enable_accept_event, - events); - event_base_set (ev_base, &events[1]); - event_add (&events[1], &tv); - } -} - gint rspamd_accept_from_socket (gint sock, rspamd_inet_addr_t **target, - GList *accept_events) + rspamd_accept_throttling_handler hdl, + void *hdl_data) { gint nfd, serrno; union sa_union su; @@ -254,7 +223,9 @@ rspamd_accept_from_socket (gint sock, rspamd_inet_addr_t **target, } else if (errno == EMFILE || errno == ENFILE) { /* Temporary disable accept event */ - rspamd_disable_accept_events (sock, accept_events); + if (hdl) { + hdl (sock, hdl_data); + } return 0; } diff --git a/src/libutil/addr.h b/src/libutil/addr.h index bfe586ad1..7efa5e318 100644 --- a/src/libutil/addr.h +++ b/src/libutil/addr.h @@ -221,15 +221,17 @@ int rspamd_inet_address_listen (const rspamd_inet_addr_t *addr, gint type, */ gboolean rspamd_ip_is_valid (const rspamd_inet_addr_t *addr); +typedef void (*rspamd_accept_throttling_handler)(gint, void *); /** * Accept from listening socket filling addr structure * @param sock listening socket - * @param addr allocated inet addr structure - * @param accept_events events for accepting new sockets + * @param target allocated inet addr structure * @return */ -gint rspamd_accept_from_socket (gint sock, rspamd_inet_addr_t **addr, - GList *accept_events); +gint rspamd_accept_from_socket (gint sock, + rspamd_inet_addr_t **target, + rspamd_accept_throttling_handler hdl, + void *hdl_data); /** * Parse host[:port[:priority]] line diff --git a/src/libutil/aio_event.c b/src/libutil/aio_event.c deleted file mode 100644 index 584feb501..000000000 --- a/src/libutil/aio_event.c +++ /dev/null @@ -1,508 +0,0 @@ -/*- - * Copyright 2016 Vsevolod Stakhov - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "config.h" -#include <event.h> -#include "aio_event.h" -#include "rspamd.h" -#include "unix-std.h" - -#ifdef HAVE_SYS_EVENTFD_H -#include <sys/eventfd.h> -#endif - -#ifdef HAVE_AIO_H -#include <aio.h> -#endif - -/* Linux syscall numbers */ -#if defined(__i386__) -# define SYS_io_setup 245 -# define SYS_io_destroy 246 -# define SYS_io_getevents 247 -# define SYS_io_submit 248 -# define SYS_io_cancel 249 -#elif defined(__x86_64__) -# define SYS_io_setup 206 -# define SYS_io_destroy 207 -# define SYS_io_getevents 208 -# define SYS_io_submit 209 -# define SYS_io_cancel 210 -#else -# warning \ - "aio is not supported on this platform, please contact author for details" -# define SYS_io_setup 0 -# define SYS_io_destroy 0 -# define SYS_io_getevents 0 -# define SYS_io_submit 0 -# define SYS_io_cancel 0 -#endif - -#define SYS_eventfd 323 -#define MAX_AIO_EV 64 - -struct io_cbdata { - gint fd; - rspamd_aio_cb cb; - guint64 len; - gpointer buf; - gpointer io_buf; - gpointer ud; -}; - -#ifdef LINUX - -/* Linux specific mappings and utilities to avoid using of libaio */ - -typedef unsigned long aio_context_t; - -typedef enum io_iocb_cmd { - IO_CMD_PREAD = 0, - IO_CMD_PWRITE = 1, - - IO_CMD_FSYNC = 2, - IO_CMD_FDSYNC = 3, - - IO_CMD_POLL = 5, - IO_CMD_NOOP = 6, -} io_iocb_cmd_t; - -#if defined(__LITTLE_ENDIAN) -#define PADDED(x,y) x, y -#elif defined(__BIG_ENDIAN) -#define PADDED(x,y) y, x -#else -#error edit for your odd byteorder. -#endif - -/* - * we always use a 64bit off_t when communicating - * with userland. its up to libraries to do the - * proper padding and aio_error abstraction - */ - -struct iocb { - /* these are internal to the kernel/libc. */ - guint64 aio_data; /* data to be returned in event's data */ - guint32 PADDED (aio_key, aio_reserved1); - /* the kernel sets aio_key to the req # */ - - /* common fields */ - guint16 aio_lio_opcode; /* see IOCB_CMD_ above */ - gint16 aio_reqprio; - guint32 aio_fildes; - - guint64 aio_buf; - guint64 aio_nbytes; - gint64 aio_offset; - - /* extra parameters */ - guint64 aio_reserved2; /* TODO: use this for a (struct sigevent *) */ - - /* flags for the "struct iocb" */ - guint32 aio_flags; - - /* - * if the IOCB_FLAG_RESFD flag of "aio_flags" is set, this is an - * eventfd to signal AIO readiness to - */ - guint32 aio_resfd; -}; - -struct io_event { - guint64 data; /* the data field from the iocb */ - guint64 obj; /* what iocb this event came from */ - gint64 res; /* result code for this event */ - gint64 res2; /* secondary result */ -}; - -/* Linux specific io calls */ -static int -io_setup (guint nr_reqs, aio_context_t *ctx) -{ - return syscall (SYS_io_setup, nr_reqs, ctx); -} - -static int -io_destroy (aio_context_t ctx) -{ - return syscall (SYS_io_destroy, ctx); -} - -static int -io_getevents (aio_context_t ctx, - long min_nr, - long nr, - struct io_event *events, - struct timespec *tmo) -{ - return syscall (SYS_io_getevents, ctx, min_nr, nr, events, tmo); -} - -static int -io_submit (aio_context_t ctx, long n, struct iocb **paiocb) -{ - return syscall (SYS_io_submit, ctx, n, paiocb); -} - -static int -io_cancel (aio_context_t ctx, struct iocb *iocb, struct io_event *result) -{ - return syscall (SYS_io_cancel, ctx, iocb, result); -} - -# ifndef HAVE_SYS_EVENTFD_H -static int -eventfd (guint initval, guint flags) -{ - return syscall (SYS_eventfd, initval); -} -# endif - -#endif - -/** - * AIO context - */ -struct aio_context { - struct event_base *base; - gboolean has_aio; /**< Whether we have aio support on a system */ -#ifdef LINUX - /* Eventfd variant */ - gint event_fd; - struct event eventfd_ev; - aio_context_t io_ctx; -#elif defined(HAVE_AIO_H) - /* POSIX aio */ - struct event rtsigs[128]; -#endif -}; - -#ifdef LINUX -/* Eventfd read callback */ -static void -rspamd_eventfdcb (gint fd, gshort what, gpointer ud) -{ - struct aio_context *ctx = ud; - guint64 ready; - gint done, i; - struct io_event event[32]; - struct timespec ts; - struct io_cbdata *ev_data; - - /* Eventfd returns number of events ready got from kernel */ - if (read (fd, &ready, 8) != 8) { - if (errno == EAGAIN) { - return; - } - msg_err ("eventfd read returned error: %s", strerror (errno)); - } - - ts.tv_sec = 0; - ts.tv_nsec = 0; - - while (ready) { - /* Get events ready */ - done = io_getevents (ctx->io_ctx, 1, 32, event, &ts); - - if (done > 0) { - ready -= done; - - for (i = 0; i < done; i++) { - ev_data = (struct io_cbdata *) (uintptr_t) event[i].data; - /* Call this callback */ - ev_data->cb (ev_data->fd, - event[i].res, - ev_data->len, - ev_data->buf, - ev_data->ud); - if (ev_data->io_buf) { - free (ev_data->io_buf); - } - g_free (ev_data); - } - } - else if (done == 0) { - /* No more events are ready */ - return; - } - else { - msg_err ("io_getevents failed: %s", strerror (errno)); - return; - } - } -} - -#endif - -/** - * Initialize aio with specified event base - */ -struct aio_context * -rspamd_aio_init (struct event_base *base) -{ - struct aio_context *new; - - /* First of all we need to detect which type of aio we can try to use */ - new = g_malloc0 (sizeof (struct aio_context)); - new->base = base; - -#ifdef LINUX - /* On linux we are trying to use io (3) and eventfd for notifying */ - new->event_fd = eventfd (0, 0); - if (new->event_fd == -1) { - msg_err ("eventfd failed: %s", strerror (errno)); - } - else { - /* Set this socket non-blocking */ - if (rspamd_socket_nonblocking (new->event_fd) == -1) { - msg_err ("non blocking for eventfd failed: %s", strerror (errno)); - close (new->event_fd); - } - else { - event_set (&new->eventfd_ev, - new->event_fd, - EV_READ | EV_PERSIST, - rspamd_eventfdcb, - new); - event_base_set (new->base, &new->eventfd_ev); - event_add (&new->eventfd_ev, NULL); - if (io_setup (MAX_AIO_EV, &new->io_ctx) == -1) { - msg_err ("io_setup failed: %s", strerror (errno)); - close (new->event_fd); - } - else { - new->has_aio = TRUE; - } - } - } -#elif defined(HAVE_AIO_H) - /* TODO: implement this */ -#endif - - return new; -} - -/** - * Open file for aio - */ -gint -rspamd_aio_open (struct aio_context *ctx, const gchar *path, int flags) -{ - gint fd = -1; - /* Fallback */ - if (!ctx->has_aio) { - return open (path, flags); - } -#ifdef LINUX - - fd = open (path, flags | O_DIRECT); - - return fd; -#elif defined(HAVE_AIO_H) - fd = open (path, flags); -#endif - - return fd; -} - -/** - * Asynchronous read of file - */ -gint -rspamd_aio_read (gint fd, - gpointer buf, - guint64 len, - guint64 offset, - struct aio_context *ctx, - rspamd_aio_cb cb, - gpointer ud) -{ - gint r = -1; - - if (ctx->has_aio) { -#ifdef LINUX - struct iocb *iocb[1]; - struct io_cbdata *cbdata; - - cbdata = g_malloc0 (sizeof (struct io_cbdata)); - cbdata->cb = cb; - cbdata->buf = buf; - cbdata->len = len; - cbdata->ud = ud; - cbdata->fd = fd; - cbdata->io_buf = NULL; - - iocb[0] = alloca (sizeof (struct iocb)); - memset (iocb[0], 0, sizeof (struct iocb)); - iocb[0]->aio_fildes = fd; - iocb[0]->aio_lio_opcode = IO_CMD_PREAD; - iocb[0]->aio_reqprio = 0; - iocb[0]->aio_buf = (guint64)((uintptr_t)buf); - iocb[0]->aio_nbytes = len; - iocb[0]->aio_offset = offset; - iocb[0]->aio_flags |= (1 << 0) /* IOCB_FLAG_RESFD */; - iocb[0]->aio_resfd = ctx->event_fd; - iocb[0]->aio_data = (guint64)((uintptr_t)cbdata); - - /* Iocb is copied to kernel internally, so it is safe to put it on stack */ - if (io_submit (ctx->io_ctx, 1, iocb) == 1) { - return len; - } - else { - if (errno == EAGAIN || errno == ENOSYS) { - /* Fall back to sync read */ - goto blocking; - } - return -1; - } - -#elif defined(HAVE_AIO_H) -#endif - } - else { - /* Blocking variant */ - goto blocking; -blocking: -#ifdef _LARGEFILE64_SOURCE - r = lseek64 (fd, offset, SEEK_SET); -#else - r = lseek (fd, offset, SEEK_SET); -#endif - if (r > 0) { - r = read (fd, buf, len); - if (r >= 0) { - cb (fd, 0, r, buf, ud); - } - else { - cb (fd, r, -1, buf, ud); - } - } - } - - return r; -} - -/** - * Asynchronous write of file - */ -gint -rspamd_aio_write (gint fd, - gpointer buf, - guint64 len, - guint64 offset, - struct aio_context *ctx, - rspamd_aio_cb cb, - gpointer ud) -{ - gint r = -1; - - if (ctx->has_aio) { -#ifdef LINUX - struct iocb *iocb[1]; - struct io_cbdata *cbdata; - - cbdata = g_malloc0 (sizeof (struct io_cbdata)); - cbdata->cb = cb; - cbdata->buf = buf; - cbdata->len = len; - cbdata->ud = ud; - cbdata->fd = fd; - /* We need to align pointer on boundary of 512 bytes here */ - if (posix_memalign (&cbdata->io_buf, 512, len) != 0) { - return -1; - } - memcpy (cbdata->io_buf, buf, len); - - iocb[0] = alloca (sizeof (struct iocb)); - memset (iocb[0], 0, sizeof (struct iocb)); - iocb[0]->aio_fildes = fd; - iocb[0]->aio_lio_opcode = IO_CMD_PWRITE; - iocb[0]->aio_reqprio = 0; - iocb[0]->aio_buf = (guint64)((uintptr_t)cbdata->io_buf); - iocb[0]->aio_nbytes = len; - iocb[0]->aio_offset = offset; - iocb[0]->aio_flags |= (1 << 0) /* IOCB_FLAG_RESFD */; - iocb[0]->aio_resfd = ctx->event_fd; - iocb[0]->aio_data = (guint64)((uintptr_t)cbdata); - - /* Iocb is copied to kernel internally, so it is safe to put it on stack */ - if (io_submit (ctx->io_ctx, 1, iocb) == 1) { - return len; - } - else { - if (errno == EAGAIN || errno == ENOSYS) { - /* Fall back to sync read */ - goto blocking; - } - return -1; - } - -#elif defined(HAVE_AIO_H) -#endif - } - else { - /* Blocking variant */ - goto blocking; -blocking: -#ifdef _LARGEFILE64_SOURCE - r = lseek64 (fd, offset, SEEK_SET); -#else - r = lseek (fd, offset, SEEK_SET); -#endif - if (r > 0) { - r = write (fd, buf, len); - if (r >= 0) { - cb (fd, 0, r, buf, ud); - } - else { - cb (fd, r, -1, buf, ud); - } - } - } - - return r; -} - -/** - * Close of aio operations - */ -gint -rspamd_aio_close (gint fd, struct aio_context *ctx) -{ - gint r = -1; - - if (ctx->has_aio) { -#ifdef LINUX - struct iocb iocb; - struct io_event ev; - - memset (&iocb, 0, sizeof (struct iocb)); - iocb.aio_fildes = fd; - iocb.aio_lio_opcode = IO_CMD_NOOP; - - /* Iocb is copied to kernel internally, so it is safe to put it on stack */ - r = io_cancel (ctx->io_ctx, &iocb, &ev); - close (fd); - return r; - -#elif defined(HAVE_AIO_H) -#endif - } - - r = close (fd); - - return r; -} diff --git a/src/libutil/aio_event.h b/src/libutil/aio_event.h deleted file mode 100644 index cccbed4e2..000000000 --- a/src/libutil/aio_event.h +++ /dev/null @@ -1,59 +0,0 @@ -/*- - * Copyright 2016 Vsevolod Stakhov - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef AIO_EVENT_H_ -#define AIO_EVENT_H_ - -#include "config.h" - -/** - * AIO context - */ -struct aio_context; - -/** - * Callback for notifying - */ -typedef void (*rspamd_aio_cb) (gint fd, gint res, guint64 len, gpointer data, - gpointer ud); - -/** - * Initialize aio with specified event base - */ -struct aio_context * rspamd_aio_init (struct event_base *base); - -/** - * Open file for aio - */ -gint rspamd_aio_open (struct aio_context *ctx, const gchar *path, int flags); - -/** - * Asynchronous read of file - */ -gint rspamd_aio_read (gint fd, gpointer buf, guint64 len, guint64 offset, - struct aio_context *ctx, rspamd_aio_cb cb, gpointer ud); - -/** - * Asynchronous write of file - */ -gint rspamd_aio_write (gint fd, gpointer buf, guint64 len, guint64 offset, - struct aio_context *ctx, rspamd_aio_cb cb, gpointer ud); - -/** - * Close of aio operations - */ -gint rspamd_aio_close (gint fd, struct aio_context *ctx); - -#endif /* AIO_EVENT_H_ */ diff --git a/src/libutil/http_connection.c b/src/libutil/http_connection.c index aa964f417..9bf7456e7 100644 --- a/src/libutil/http_connection.c +++ b/src/libutil/http_connection.c @@ -25,6 +25,7 @@ #include "ottery.h" #include "keypair_private.h" #include "cryptobox.h" +#include "libutil/libev_helper.h" #include "libutil/ssl_util.h" #include "libserver/url.h" @@ -67,9 +68,8 @@ struct rspamd_http_connection_private { struct rspamd_http_header *header; struct http_parser parser; struct http_parser_settings parser_cb; - struct event ev; - struct timeval tv; - struct timeval *ptv; + struct rspamd_io_ev ev; + ev_tstamp timeout; struct rspamd_http_message *msg; struct iovec *out; guint outlen; @@ -348,9 +348,7 @@ rspamd_http_on_headers_complete (http_parser * parser) if (msg->method == HTTP_HEAD) { /* We don't care about the rest */ - if (rspamd_event_pending (&priv->ev, EV_READ)) { - event_del (&priv->ev); - } + rspamd_ev_watcher_stop (priv->ctx->event_loop, &priv->ev); msg->code = parser->status_code; rspamd_http_connection_ref (conn); @@ -358,7 +356,7 @@ rspamd_http_on_headers_complete (http_parser * parser) if (conn->opts & RSPAMD_HTTP_CLIENT_KEEP_ALIVE) { rspamd_http_context_push_keepalive (conn->priv->ctx, conn, - msg, conn->priv->ctx->ev_base); + msg, conn->priv->ctx->event_loop); rspamd_http_connection_reset (conn); } else { @@ -532,17 +530,14 @@ rspamd_http_on_headers_complete_decrypted (http_parser *parser) if (msg->method == HTTP_HEAD) { /* We don't care about the rest */ - if (rspamd_event_pending (&priv->ev, EV_READ)) { - event_del (&priv->ev); - } - + rspamd_ev_watcher_stop (priv->ctx->event_loop, &priv->ev); msg->code = parser->status_code; rspamd_http_connection_ref (conn); ret = conn->finish_handler (conn, msg); if (conn->opts & RSPAMD_HTTP_CLIENT_KEEP_ALIVE) { rspamd_http_context_push_keepalive (conn->priv->ctx, conn, - msg, conn->priv->ctx->ev_base); + msg, conn->priv->ctx->event_loop); rspamd_http_connection_reset (conn); } else { @@ -692,16 +687,13 @@ rspamd_http_on_message_complete (http_parser * parser) } if (ret == 0) { - if (rspamd_event_pending (&priv->ev, EV_READ)) { - event_del (&priv->ev); - } - + rspamd_ev_watcher_stop (priv->ctx->event_loop, &priv->ev); rspamd_http_connection_ref (conn); ret = conn->finish_handler (conn, priv->msg); if (conn->opts & RSPAMD_HTTP_CLIENT_KEEP_ALIVE) { rspamd_http_context_push_keepalive (conn->priv->ctx, conn, - priv->msg, conn->priv->ctx->ev_base); + priv->msg, conn->priv->ctx->event_loop); rspamd_http_connection_reset (conn); } else { @@ -741,11 +733,11 @@ rspamd_http_simple_client_helper (struct rspamd_http_connection *conn) if (conn->opts & RSPAMD_HTTP_CLIENT_SHARED) { rspamd_http_connection_read_message_shared (conn, conn->ud, - conn->priv->ptv); + conn->priv->timeout); } else { rspamd_http_connection_read_message (conn, conn->ud, - conn->priv->ptv); + conn->priv->timeout); } if (priv->msg) { @@ -835,7 +827,6 @@ rspamd_http_write_helper (struct rspamd_http_connection *conn) else { /* Want to write more */ priv->flags &= ~RSPAMD_HTTP_CONN_FLAG_RESETED; - event_add (&priv->ev, priv->ptv); } return; @@ -1269,10 +1260,7 @@ rspamd_http_connection_reset (struct rspamd_http_connection *conn) if (!(priv->flags & RSPAMD_HTTP_CONN_FLAG_RESETED)) { - if (rspamd_event_pending (&priv->ev, EV_READ|EV_WRITE|EV_TIMEOUT)) { - event_del (&priv->ev); - } - + rspamd_ev_watcher_stop (priv->ctx->event_loop, &priv->ev); rspamd_http_parser_reset (conn); } @@ -1468,7 +1456,7 @@ rspamd_http_connection_free (struct rspamd_http_connection *conn) static void rspamd_http_connection_read_message_common (struct rspamd_http_connection *conn, - gpointer ud, struct timeval *timeout, + gpointer ud, ev_tstamp timeout, gint flags) { struct rspamd_http_connection_private *priv = conn->priv; @@ -1490,42 +1478,30 @@ rspamd_http_connection_read_message_common (struct rspamd_http_connection *conn, priv->flags |= RSPAMD_HTTP_CONN_FLAG_ENCRYPTED; } - if (timeout == NULL) { - priv->ptv = NULL; - } - else { - memmove (&priv->tv, timeout, sizeof (struct timeval)); - priv->ptv = &priv->tv; - } - + priv->timeout = timeout; priv->header = NULL; priv->buf = g_malloc0 (sizeof (*priv->buf)); REF_INIT_RETAIN (priv->buf, rspamd_http_privbuf_dtor); priv->buf->data = rspamd_fstring_sized_new (8192); priv->flags |= RSPAMD_HTTP_CONN_FLAG_NEW_HEADER; - event_set (&priv->ev, - conn->fd, - EV_READ | EV_PERSIST, - rspamd_http_event_handler, - conn); - - event_base_set (priv->ctx->ev_base, &priv->ev); + rspamd_ev_watcher_init (&priv->ev, conn->fd, EV_READ, + rspamd_http_event_handler, conn); + rspamd_ev_watcher_start (priv->ctx->event_loop, &priv->ev, priv->timeout); priv->flags &= ~RSPAMD_HTTP_CONN_FLAG_RESETED; - event_add (&priv->ev, priv->ptv); } void rspamd_http_connection_read_message (struct rspamd_http_connection *conn, - gpointer ud, struct timeval *timeout) + gpointer ud, ev_tstamp timeout) { rspamd_http_connection_read_message_common (conn, ud, timeout, 0); } void rspamd_http_connection_read_message_shared (struct rspamd_http_connection *conn, - gpointer ud, struct timeval *timeout) + gpointer ud, ev_tstamp timeout) { rspamd_http_connection_read_message_common (conn, ud, timeout, RSPAMD_HTTP_FLAG_SHMEM); @@ -1912,7 +1888,7 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn const gchar *host, const gchar *mime_type, gpointer ud, - struct timeval *timeout, + ev_tstamp timeout, gboolean allow_shared) { struct rspamd_http_connection_private *priv = conn->priv; @@ -1930,14 +1906,7 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn conn->ud = ud; priv->msg = msg; - - if (timeout == NULL) { - priv->ptv = NULL; - } - else if (timeout != &priv->tv) { - memcpy (&priv->tv, timeout, sizeof (struct timeval)); - priv->ptv = &priv->tv; - } + priv->timeout = timeout; priv->header = NULL; priv->buf = g_malloc0 (sizeof (*priv->buf)); @@ -2224,16 +2193,12 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn msg->flags &=~ RSPAMD_HTTP_FLAG_SSL; } - if (rspamd_event_pending (&priv->ev, EV_TIMEOUT|EV_WRITE|EV_READ)) { - event_del (&priv->ev); - } + rspamd_ev_watcher_stop (priv->ctx->event_loop, &priv->ev); if (msg->flags & RSPAMD_HTTP_FLAG_SSL) { gpointer ssl_ctx = (msg->flags & RSPAMD_HTTP_FLAG_SSL_NOVERIFY) ? priv->ctx->ssl_ctx_noverify : priv->ctx->ssl_ctx; - event_base_set (priv->ctx->ev_base, &priv->ev); - if (!ssl_ctx) { err = g_error_new (HTTP_ERROR, errno, "ssl message requested " "with no ssl ctx"); @@ -2249,12 +2214,12 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn rspamd_ssl_connection_free (priv->ssl); } - priv->ssl = rspamd_ssl_connection_new (ssl_ctx, priv->ctx->ev_base, + priv->ssl = rspamd_ssl_connection_new (ssl_ctx, priv->ctx->event_loop, !(msg->flags & RSPAMD_HTTP_FLAG_SSL_NOVERIFY)); g_assert (priv->ssl != NULL); if (!rspamd_ssl_connect_fd (priv->ssl, conn->fd, host, &priv->ev, - priv->ptv, rspamd_http_event_handler, + priv->timeout, rspamd_http_event_handler, rspamd_http_ssl_err_handler, conn)) { err = g_error_new (HTTP_ERROR, errno, @@ -2270,10 +2235,9 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn } } else { - event_set (&priv->ev, conn->fd, EV_WRITE, rspamd_http_event_handler, conn); - event_base_set (priv->ctx->ev_base, &priv->ev); - - event_add (&priv->ev, priv->ptv); + rspamd_ev_watcher_init (&priv->ev, conn->fd, EV_WRITE, + rspamd_http_event_handler, conn); + rspamd_ev_watcher_start (priv->ctx->event_loop, &priv->ev, priv->timeout); } } @@ -2283,7 +2247,7 @@ rspamd_http_connection_write_message (struct rspamd_http_connection *conn, const gchar *host, const gchar *mime_type, gpointer ud, - struct timeval *timeout) + ev_tstamp timeout) { rspamd_http_connection_write_message_common (conn, msg, host, mime_type, ud, timeout, FALSE); @@ -2295,7 +2259,7 @@ rspamd_http_connection_write_message_shared (struct rspamd_http_connection *conn const gchar *host, const gchar *mime_type, gpointer ud, - struct timeval *timeout) + ev_tstamp timeout) { rspamd_http_connection_write_message_common (conn, msg, host, mime_type, ud, timeout, TRUE); diff --git a/src/libutil/http_connection.h b/src/libutil/http_connection.h index 6240772da..fc1303446 100644 --- a/src/libutil/http_connection.h +++ b/src/libutil/http_connection.h @@ -31,7 +31,7 @@ #include "http_util.h" #include "addr.h" -#include <event.h> +#include "contrib/libev/ev.h" enum rspamd_http_connection_type { RSPAMD_HTTP_SERVER, @@ -221,12 +221,12 @@ gboolean rspamd_http_connection_is_encrypted (struct rspamd_http_connection *con void rspamd_http_connection_read_message ( struct rspamd_http_connection *conn, gpointer ud, - struct timeval *timeout); + ev_tstamp timeout); void rspamd_http_connection_read_message_shared ( struct rspamd_http_connection *conn, gpointer ud, - struct timeval *timeout); + ev_tstamp timeout); /** * Send reply using initialised connection @@ -241,7 +241,7 @@ void rspamd_http_connection_write_message ( const gchar *host, const gchar *mime_type, gpointer ud, - struct timeval *timeout); + ev_tstamp timeout); void rspamd_http_connection_write_message_shared ( struct rspamd_http_connection *conn, @@ -249,7 +249,7 @@ void rspamd_http_connection_write_message_shared ( const gchar *host, const gchar *mime_type, gpointer ud, - struct timeval *timeout); + ev_tstamp timeout); /** * Free connection structure diff --git a/src/libutil/http_context.c b/src/libutil/http_context.c index 95500aaad..95ab7021c 100644 --- a/src/libutil/http_context.c +++ b/src/libutil/http_context.c @@ -23,6 +23,7 @@ #include "contrib/libottery/ottery.h" #include "contrib/http-parser/http_parser.h" #include "rspamd.h" +#include "libev_helper.h" INIT_LOG_MODULE(http_context) @@ -38,7 +39,7 @@ struct rspamd_http_keepalive_cbdata { struct rspamd_http_context *ctx; GQueue *queue; GList *link; - struct event ev; + struct rspamd_io_ev ev; }; static void @@ -64,20 +65,16 @@ rspamd_http_keepalive_queue_cleanup (GQueue *conns) } static void -rspamd_http_context_client_rotate_ev (gint fd, short what, void *arg) +rspamd_http_context_client_rotate_ev (struct ev_loop *loop, ev_timer *w, int revents) { - struct timeval rot_tv; - struct rspamd_http_context *ctx = arg; + struct rspamd_http_context *ctx = (struct rspamd_http_context *)w->data; gpointer kp; - double_to_tv (ctx->config.client_key_rotate_time, &rot_tv); - rot_tv.tv_sec += ottery_rand_range (rot_tv.tv_sec); + w->repeat = rspamd_time_jitter (ctx->config.client_key_rotate_time, 0); + msg_debug_http_context ("rotate local keypair, next rotate in %.0f seconds", + w->repeat); - msg_debug_http_context ("rotate local keypair, next rotate in %d seconds", - (int)rot_tv.tv_sec); - - event_del (&ctx->client_rotate_ev); - event_add (&ctx->client_rotate_ev, &rot_tv); + ev_timer_again (loop, w); kp = ctx->client_kp; ctx->client_kp = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX, @@ -87,7 +84,7 @@ rspamd_http_context_client_rotate_ev (gint fd, short what, void *arg) static struct rspamd_http_context* rspamd_http_context_new_default (struct rspamd_config *cfg, - struct event_base *ev_base, + struct ev_loop *ev_base, struct upstream_ctx *ups_ctx) { struct rspamd_http_context *ctx; @@ -114,7 +111,7 @@ rspamd_http_context_new_default (struct rspamd_config *cfg, ctx->ssl_ctx_noverify = rspamd_init_ssl_ctx_noverify (); } - ctx->ev_base = ev_base; + ctx->event_loop = ev_base; ctx->keep_alive_hash = kh_init (rspamd_keep_alive_hash); @@ -186,16 +183,14 @@ rspamd_http_context_init (struct rspamd_http_context *ctx) ctx->server_kp_cache = rspamd_keypair_cache_new (ctx->config.kp_cache_size_server); } - if (ctx->config.client_key_rotate_time > 0 && ctx->ev_base) { - struct timeval tv; + if (ctx->config.client_key_rotate_time > 0 && ctx->event_loop) { double jittered = rspamd_time_jitter (ctx->config.client_key_rotate_time, 0); - double_to_tv (jittered, &tv); - event_set (&ctx->client_rotate_ev, -1, EV_TIMEOUT, - rspamd_http_context_client_rotate_ev, ctx); - event_base_set (ctx->ev_base, &ctx->client_rotate_ev); - event_add (&ctx->client_rotate_ev, &tv); + ev_timer_init (&ctx->client_rotate_ev, + rspamd_http_context_client_rotate_ev, jittered, 0); + ev_timer_start (ctx->event_loop, &ctx->client_rotate_ev); + ctx->client_rotate_ev.data = ctx; } if (ctx->config.http_proxy) { @@ -208,7 +203,7 @@ rspamd_http_context_init (struct rspamd_http_context *ctx) struct rspamd_http_context* rspamd_http_context_create (struct rspamd_config *cfg, - struct event_base *ev_base, + struct ev_loop *ev_base, struct upstream_ctx *ups_ctx) { struct rspamd_http_context *ctx; @@ -337,7 +332,7 @@ rspamd_http_context_free (struct rspamd_http_context *ctx) struct rspamd_http_context* rspamd_http_context_create_config (struct rspamd_http_context_cfg *cfg, - struct event_base *ev_base, + struct ev_loop *ev_base, struct upstream_ctx *ups_ctx) { struct rspamd_http_context *ctx; @@ -412,7 +407,7 @@ rspamd_http_context_check_keepalive (struct rspamd_http_context *ctx, struct rspamd_http_connection *conn; cbd = g_queue_pop_head (conns); - event_del (&cbd->ev); + rspamd_ev_watcher_stop (ctx->event_loop, &cbd->ev); conn = cbd->conn; g_free (cbd); @@ -491,6 +486,7 @@ rspamd_http_keepalive_handler (gint fd, short what, gpointer ud) cbdata->conn->keepalive_hash_key->host, cbdata->queue->length); rspamd_http_connection_unref (cbdata->conn); + rspamd_ev_watcher_stop (cbdata->ctx->event_loop, &cbdata->ev); g_free (cbdata); } @@ -498,10 +494,9 @@ void rspamd_http_context_push_keepalive (struct rspamd_http_context *ctx, struct rspamd_http_connection *conn, struct rspamd_http_message *msg, - struct event_base *ev_base) + struct ev_loop *event_loop) { struct rspamd_http_keepalive_cbdata *cbdata; - struct timeval tv; gdouble timeout = ctx->config.keepalive_interval; g_assert (conn->keepalive_hash_key != NULL); @@ -571,17 +566,14 @@ rspamd_http_context_push_keepalive (struct rspamd_http_context *ctx, cbdata->ctx = ctx; conn->finished = FALSE; - event_set (&cbdata->ev, conn->fd, EV_READ|EV_TIMEOUT, + rspamd_ev_watcher_init (&cbdata->ev, conn->fd, EV_READ, rspamd_http_keepalive_handler, cbdata); + rspamd_ev_watcher_start (event_loop, &cbdata->ev, timeout); msg_debug_http_context ("push keepalive element %s (%s), %d connections queued, %.1f timeout", rspamd_inet_address_to_string_pretty (cbdata->conn->keepalive_hash_key->addr), cbdata->conn->keepalive_hash_key->host, cbdata->queue->length, timeout); - - double_to_tv (timeout, &tv); - event_base_set (ev_base, &cbdata->ev); - event_add (&cbdata->ev, &tv); }
\ No newline at end of file diff --git a/src/libutil/http_context.h b/src/libutil/http_context.h index 4cf07fb48..c610ffbbd 100644 --- a/src/libutil/http_context.h +++ b/src/libutil/http_context.h @@ -21,7 +21,7 @@ #include "ucl.h" #include "addr.h" -#include <event.h> +#include "contrib/libev/ev.h" struct rspamd_http_context; struct rspamd_config; @@ -45,11 +45,11 @@ struct rspamd_http_context_cfg { * @return new context used for both client and server HTTP connections */ struct rspamd_http_context* rspamd_http_context_create (struct rspamd_config *cfg, - struct event_base *ev_base, struct upstream_ctx *ctx); + struct ev_loop *ev_base, struct upstream_ctx *ctx); struct rspamd_http_context* rspamd_http_context_create_config ( struct rspamd_http_context_cfg *cfg, - struct event_base *ev_base, + struct ev_loop *ev_base, struct upstream_ctx *ctx); /** * Destroys context @@ -93,6 +93,6 @@ void rspamd_http_context_prepare_keepalive (struct rspamd_http_context *ctx, void rspamd_http_context_push_keepalive (struct rspamd_http_context *ctx, struct rspamd_http_connection *conn, struct rspamd_http_message *msg, - struct event_base *ev_base); + struct ev_loop *ev_base); #endif diff --git a/src/libutil/http_private.h b/src/libutil/http_private.h index 368715891..f5a7dd9cc 100644 --- a/src/libutil/http_private.h +++ b/src/libutil/http_private.h @@ -100,8 +100,8 @@ struct rspamd_http_context { struct upstream_list *http_proxies; gpointer ssl_ctx; gpointer ssl_ctx_noverify; - struct event_base *ev_base; - struct event client_rotate_ev; + struct ev_loop *event_loop; + ev_timer client_rotate_ev; khash_t (rspamd_keep_alive_hash) *keep_alive_hash; }; diff --git a/src/libutil/http_router.c b/src/libutil/http_router.c index ec0eeb7b4..8d5913f0d 100644 --- a/src/libutil/http_router.c +++ b/src/libutil/http_router.c @@ -92,7 +92,7 @@ rspamd_http_router_error_handler (struct rspamd_http_connection *conn, NULL, "text/plain", entry, - entry->rt->ptv); + entry->rt->timeout); entry->is_reply = TRUE; } } @@ -210,7 +210,7 @@ rspamd_http_router_try_file (struct rspamd_http_connection_entry *entry, msg_debug ("requested file %s", realbuf); rspamd_http_connection_write_message (entry->conn, reply_msg, NULL, rspamd_http_router_detect_ct (realbuf), entry, - entry->rt->ptv); + entry->rt->timeout); return TRUE; } @@ -235,7 +235,7 @@ rspamd_http_router_send_error (GError *err, NULL, "text/plain", entry, - entry->rt->ptv); + entry->rt->timeout); } @@ -369,33 +369,25 @@ rspamd_http_router_finish_handler (struct rspamd_http_connection *conn, struct rspamd_http_connection_router * rspamd_http_router_new (rspamd_http_router_error_handler_t eh, rspamd_http_router_finish_handler_t fh, - struct timeval *timeout, + ev_tstamp timeout, const char *default_fs_path, struct rspamd_http_context *ctx) { - struct rspamd_http_connection_router * new; + struct rspamd_http_connection_router *nrouter; struct stat st; - new = g_malloc0 (sizeof (struct rspamd_http_connection_router)); - new->paths = g_hash_table_new_full (rspamd_ftok_icase_hash, + nrouter = g_malloc0 (sizeof (struct rspamd_http_connection_router)); + nrouter->paths = g_hash_table_new_full (rspamd_ftok_icase_hash, rspamd_ftok_icase_equal, rspamd_fstring_mapped_ftok_free, NULL); - new->regexps = g_ptr_array_new (); - new->conns = NULL; - new->error_handler = eh; - new->finish_handler = fh; - new->response_headers = g_hash_table_new_full (rspamd_strcase_hash, + nrouter->regexps = g_ptr_array_new (); + nrouter->conns = NULL; + nrouter->error_handler = eh; + nrouter->finish_handler = fh; + nrouter->response_headers = g_hash_table_new_full (rspamd_strcase_hash, rspamd_strcase_equal, g_free, g_free); - new->ev_base = ctx->ev_base; - - if (timeout) { - new->tv = *timeout; - new->ptv = &new->tv; - } - else { - new->ptv = NULL; - } - - new->default_fs_path = NULL; + nrouter->event_loop = ctx->event_loop; + nrouter->timeout = timeout; + nrouter->default_fs_path = NULL; if (default_fs_path != NULL) { if (stat (default_fs_path, &st) == -1) { @@ -406,14 +398,14 @@ rspamd_http_router_new (rspamd_http_router_error_handler_t eh, msg_err ("path %s is not a directory", default_fs_path); } else { - new->default_fs_path = realpath (default_fs_path, NULL); + nrouter->default_fs_path = realpath (default_fs_path, NULL); } } } - new->ctx = ctx; + nrouter->ctx = ctx; - return new; + return nrouter; } void @@ -517,7 +509,7 @@ rspamd_http_router_handle_socket (struct rspamd_http_connection_router *router, rspamd_http_connection_set_key (conn->conn, router->key); } - rspamd_http_connection_read_message (conn->conn, conn, router->ptv); + rspamd_http_connection_read_message (conn->conn, conn, router->timeout); DL_PREPEND (router->conns, conn); } diff --git a/src/libutil/http_router.h b/src/libutil/http_router.h index 8e8056240..b946067b7 100644 --- a/src/libutil/http_router.h +++ b/src/libutil/http_router.h @@ -44,9 +44,8 @@ struct rspamd_http_connection_router { GHashTable *paths; GHashTable *response_headers; GPtrArray *regexps; - struct timeval tv; - struct timeval *ptv; - struct event_base *ev_base; + ev_tstamp timeout; + struct ev_loop *event_loop; struct rspamd_http_context *ctx; gchar *default_fs_path; rspamd_http_router_handler_t unknown_method_handler; @@ -66,7 +65,7 @@ struct rspamd_http_connection_router { struct rspamd_http_connection_router * rspamd_http_router_new ( rspamd_http_router_error_handler_t eh, rspamd_http_router_finish_handler_t fh, - struct timeval *timeout, + ev_tstamp timeout, const char *default_fs_path, struct rspamd_http_context *ctx); diff --git a/src/libutil/libev_helper.c b/src/libutil/libev_helper.c new file mode 100644 index 000000000..81a23dea6 --- /dev/null +++ b/src/libutil/libev_helper.c @@ -0,0 +1,119 @@ +/*- + * Copyright 2019 Vsevolod Stakhov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "libev_helper.h" + +static void +rspamd_ev_watcher_io_cb (EV_P_ struct ev_io *w, int revents) +{ + struct rspamd_io_ev *ev = (struct rspamd_io_ev *)w->data; + + ev->last_activity = ev_now (EV_A); + ev->cb (ev->io.fd, revents, ev->ud); +} + +static void +rspamd_ev_watcher_timer_cb (EV_P_ struct ev_timer *w, int revents) +{ + struct rspamd_io_ev *ev = (struct rspamd_io_ev *)w->data; + + ev_tstamp after = ev->last_activity - ev_now (EV_A) + ev->timeout; + + if (after < 0.) { + /* Real timeout */ + ev->cb (ev->io.fd, EV_TIMER, ev->ud); + } + else { + /* Start another cycle as there was some activity */ + w->repeat = after; + ev_timer_again (EV_A_ w); + } +} + + +void +rspamd_ev_watcher_init (struct rspamd_io_ev *ev, + int fd, + short what, + rspamd_ev_cb cb, + void *ud) +{ + ev_io_init (&ev->io, rspamd_ev_watcher_io_cb, fd, what); + ev->io.data = ev; + ev_init (&ev->tm, rspamd_ev_watcher_timer_cb); + ev->tm.data = ev; + ev->ud = ud; + ev->cb = cb; +} + +void +rspamd_ev_watcher_start (struct ev_loop *loop, + struct rspamd_io_ev *ev, + ev_tstamp timeout) +{ + g_assert (ev->cb != NULL); + + ev->last_activity = ev_now (EV_A); + ev_io_start (EV_A_ &ev->io); + + if (timeout > 0) { + ev->timeout = timeout; + ev_timer_set (&ev->tm, timeout, 0.0); + ev_timer_start (EV_A_ &ev->tm); + } +} + +void +rspamd_ev_watcher_stop (struct ev_loop *loop, + struct rspamd_io_ev *ev) +{ + if (ev_is_pending (&ev->io) || ev_is_active (&ev->io)) { + ev_io_stop (EV_A_ &ev->io); + } + + if (ev->timeout > 0) { + ev_timer_stop (EV_A_ &ev->tm); + } +} + +void +rspamd_ev_watcher_reschedule (struct ev_loop *loop, + struct rspamd_io_ev *ev, + short what) +{ + g_assert (ev->cb != NULL); + + if (ev_is_pending (&ev->io) || ev_is_active (&ev->io)) { + ev_io_stop (EV_A_ &ev->io); + ev_io_set (&ev->io, ev->io.fd, what); + ev_io_start (EV_A_ &ev->io); + } + else { + ev->io.data = ev; + ev_io_init (&ev->io, rspamd_ev_watcher_io_cb, ev->io.fd, what); + ev_io_start (EV_A_ &ev->io); + } + + if (ev->timeout > 0) { + if (!(ev_is_active (&ev->tm) || ev_is_pending (&ev->tm))) { + ev->tm.data = ev; + ev_timer_init (&ev->tm, rspamd_ev_watcher_timer_cb, ev->timeout, 0.0); + ev_timer_start (EV_A_ &ev->tm); + } + } + + ev->last_activity = ev_now (EV_A); +}
\ No newline at end of file diff --git a/src/libutil/libev_helper.h b/src/libutil/libev_helper.h new file mode 100644 index 000000000..cf52db557 --- /dev/null +++ b/src/libutil/libev_helper.h @@ -0,0 +1,78 @@ +/*- + * Copyright 2019 Vsevolod Stakhov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RSPAMD_LIBEV_HELPER_H +#define RSPAMD_LIBEV_HELPER_H + +#include "config.h" +#include "contrib/libev/ev.h" + +/* + * This module is a little helper to simplify libevent->libev transition + * It allows to create timed IO watchers utilising both + */ + +typedef void (*rspamd_ev_cb)(int fd, short what, void *ud); + +struct rspamd_io_ev { + ev_io io; + ev_timer tm; + rspamd_ev_cb cb; + void *ud; + ev_tstamp last_activity; + ev_tstamp timeout; +}; + +/** + * Initialize watcher similar to event_init + * @param ev + * @param fd + * @param what + * @param cb + * @param ud + */ +void rspamd_ev_watcher_init (struct rspamd_io_ev *ev, + int fd, short what, rspamd_ev_cb cb, void *ud); + +/** + * Start watcher with the specific timeout + * @param loop + * @param ev + * @param timeout + */ +void rspamd_ev_watcher_start (struct ev_loop *loop, + struct rspamd_io_ev *ev, + ev_tstamp timeout); + +/** + * Stops watcher and clean it up + * @param loop + * @param ev + */ +void rspamd_ev_watcher_stop (struct ev_loop *loop, + struct rspamd_io_ev *ev); + +/** + * Convenience function to reschedule watcher with different events + * @param loop + * @param ev + * @param what + */ +void rspamd_ev_watcher_reschedule (struct ev_loop *loop, + struct rspamd_io_ev *ev, + short what); + +#endif diff --git a/src/libutil/map.c b/src/libutil/map.c index fc414ab00..9f43fa253 100644 --- a/src/libutil/map.c +++ b/src/libutil/map.c @@ -23,6 +23,7 @@ #include "http_private.h" #include "rspamd.h" #include "contrib/zstd/zstd.h" +#include "contrib/libev/ev.h" #undef MAP_DEBUG_REFS #ifdef MAP_DEBUG_REFS @@ -44,7 +45,7 @@ static void free_http_cbdata_common (struct http_callback_data *cbd, gboolean plan_new); static void free_http_cbdata_dtor (gpointer p); static void free_http_cbdata (struct http_callback_data *cbd); -static void rspamd_map_periodic_callback (gint fd, short what, void *ud); +static void rspamd_map_process_periodic (struct map_periodic_cbdata *cbd); static void rspamd_map_schedule_periodic (struct rspamd_map *map, gboolean locked, gboolean initial, gboolean errored); static gboolean read_map_file_chunks (struct rspamd_map *map, @@ -130,7 +131,7 @@ write_http_request (struct http_callback_data *cbd) cbd->data->host, NULL, cbd, - &cbd->tv); + cbd->timeout); } static gboolean @@ -274,7 +275,12 @@ free_http_cbdata_common (struct http_callback_data *cbd, gboolean plan_new) MAP_RELEASE (cbd->bk, "rspamd_map_backend"); - MAP_RELEASE (periodic, "periodic"); + + if (periodic) { + /* Detached in case of HTTP error */ + MAP_RELEASE (periodic, "periodic"); + } + g_free (cbd); } @@ -325,17 +331,21 @@ http_map_error (struct rspamd_http_connection *conn, cbd->bk->uri, cbd->addr ? rspamd_inet_address_to_string_pretty (cbd->addr) : "", err); - rspamd_map_periodic_callback (-1, EV_TIMEOUT, cbd->periodic); + MAP_RETAIN (cbd->periodic, "periodic"); + rspamd_map_process_periodic (cbd->periodic); + MAP_RELEASE (cbd->periodic, "periodic"); + /* Detach periodic as rspamd_map_process_periodic will destroy it */ + cbd->periodic = NULL; MAP_RELEASE (cbd, "http_callback_data"); } static void -rspamd_map_cache_cb (gint fd, short what, gpointer ud) +rspamd_map_cache_cb (struct ev_loop *loop, ev_timer *w, int revents) { - struct rspamd_http_map_cached_cbdata *cache_cbd = ud; + struct rspamd_http_map_cached_cbdata *cache_cbd = (struct rspamd_http_map_cached_cbdata *) + w->data; struct rspamd_map *map; struct http_map_data *data; - struct timeval tv; map = cache_cbd->map; data = cache_cbd->data; @@ -349,7 +359,7 @@ rspamd_map_cache_cb (gint fd, short what, gpointer ud) msg_info_map ("cached data is now expired (gen mismatch %L != %L) for %s", cache_cbd->gen, cache_cbd->data->gen, map->name); MAP_RELEASE (cache_cbd->shm, "rspamd_http_map_cached_cbdata"); - event_del (&cache_cbd->timeout); + ev_timer_stop (loop, &cache_cbd->timeout); g_free (cache_cbd); } else if (cache_cbd->data->last_checked >= cache_cbd->last_checked) { @@ -357,17 +367,25 @@ rspamd_map_cache_cb (gint fd, short what, gpointer ud) * We checked map but we have not found anything more recent, * reschedule cache check */ + if (cache_cbd->map->poll_timeout > + ev_now (loop) - cache_cbd->data->last_checked) { + w->repeat = cache_cbd->map->poll_timeout - + (ev_now (loop) - cache_cbd->data->last_checked); + } + else { + w->repeat = cache_cbd->map->poll_timeout; + } + cache_cbd->last_checked = cache_cbd->data->last_checked; msg_debug_map ("cached data is up to date for %s", map->name); - double_to_tv (map->poll_timeout * 2, &tv); - event_add (&cache_cbd->timeout, &tv); + ev_timer_again (loop, &cache_cbd->timeout); } else { data->cur_cache_cbd = NULL; g_atomic_int_set (&data->cache->available, 0); MAP_RELEASE (cache_cbd->shm, "rspamd_http_map_cached_cbdata"); msg_info_map ("cached data is now expired for %s", map->name); - event_del (&cache_cbd->timeout); + ev_timer_stop (loop, &cache_cbd->timeout); g_free (cache_cbd); } } @@ -436,7 +454,6 @@ http_map_finish (struct rspamd_http_connection *conn, struct rspamd_map_backend *bk; struct http_map_data *data; struct rspamd_http_map_cached_cbdata *cache_cbd; - struct timeval tv; const rspamd_ftok_t *expires_hdr, *etag_hdr; char next_check_date[128]; guchar *aux_data, *in = NULL; @@ -456,7 +473,7 @@ http_map_finish (struct rspamd_http_connection *conn, g_atomic_int_set (&data->cache->available, 0); data->cur_cache_cbd = NULL; - rspamd_map_periodic_callback (-1, EV_TIMEOUT, cbd->periodic); + rspamd_map_process_periodic (cbd->periodic); MAP_RELEASE (cbd, "http_callback_data"); return 0; @@ -622,6 +639,8 @@ read_data: } /* Check for expires */ + double cached_timeout = map->poll_timeout * 2; + expires_hdr = rspamd_http_message_find_header (msg, "Expires"); if (expires_hdr) { @@ -635,19 +654,12 @@ read_data: hdate = MIN (map->next_check, hdate); } - double cached_timeout = map->next_check - msg->date + - map->poll_timeout * 2; + cached_timeout = map->next_check - msg->date + + map->poll_timeout * 2; map->next_check = hdate; - double_to_tv (cached_timeout, &tv); - } - else { - double_to_tv (map->poll_timeout * 2, &tv); } } - else { - double_to_tv (map->poll_timeout * 2, &tv); - } /* Check for etag */ etag_hdr = rspamd_http_message_find_header (msg, "ETag"); @@ -682,16 +694,17 @@ read_data: data->cache->last_modified = cbd->data->last_modified; cache_cbd = g_malloc0 (sizeof (*cache_cbd)); cache_cbd->shm = cbd->shmem_data; + cache_cbd->event_loop = cbd->event_loop; cache_cbd->map = map; cache_cbd->data = cbd->data; cache_cbd->last_checked = cbd->data->last_checked; cache_cbd->gen = cbd->data->gen; MAP_RETAIN (cache_cbd->shm, "shmem_data"); - event_set (&cache_cbd->timeout, -1, EV_TIMEOUT, rspamd_map_cache_cb, - cache_cbd); - event_base_set (cbd->ev_base, &cache_cbd->timeout); - event_add (&cache_cbd->timeout, &tv); + ev_timer_init (&cache_cbd->timeout, rspamd_map_cache_cb, cached_timeout, + 0.0); + ev_timer_start (cbd->event_loop, &cache_cbd->timeout); + cache_cbd->timeout.data = cache_cbd; data->cur_cache_cbd = cache_cbd; if (map->next_check) { @@ -700,7 +713,7 @@ read_data: } else { rspamd_http_date_format (next_check_date, sizeof (next_check_date), - time (NULL) + map->poll_timeout); + ev_now (cbd->event_loop) + map->poll_timeout); } @@ -773,7 +786,7 @@ read_data: cbd->periodic->cur_backend ++; munmap (in, dlen); - rspamd_map_periodic_callback (-1, EV_TIMEOUT, cbd->periodic); + rspamd_map_process_periodic (cbd->periodic); } else if (msg->code == 304 && (cbd->check && cbd->stage == map_load_file)) { cbd->data->last_checked = msg->date; @@ -819,13 +832,13 @@ read_data: } else { rspamd_http_date_format (next_check_date, sizeof (next_check_date), - time (NULL) + map->poll_timeout); + ev_now (cbd->event_loop) + map->poll_timeout); } msg_info_map ("data is not modified for server %s, next check at %s", cbd->data->host, next_check_date); cbd->periodic->cur_backend ++; - rspamd_map_periodic_callback (-1, EV_TIMEOUT, cbd->periodic); + rspamd_map_process_periodic (cbd->periodic); } else { msg_info_map ("cannot load map %s from %s: HTTP error %d", @@ -838,7 +851,7 @@ read_data: err: cbd->periodic->errored = 1; - rspamd_map_periodic_callback (-1, EV_TIMEOUT, cbd->periodic); + rspamd_map_process_periodic (cbd->periodic); MAP_RELEASE (cbd, "http_callback_data"); return 0; @@ -951,6 +964,7 @@ read_map_file (struct rspamd_map *map, struct file_map_data *data, } } + ev_stat_stat (map->event_loop, &data->st_ev); len = st.st_size; if (bk->is_signed) { @@ -1045,9 +1059,6 @@ read_map_file (struct rspamd_map *map, struct file_map_data *data, map->read_callback (NULL, 0, &periodic->cbdata, TRUE); } - /* Also update at the read time */ - memcpy (&data->st, &st, sizeof (struct stat)); - return TRUE; } @@ -1143,7 +1154,6 @@ rspamd_map_periodic_dtor (struct map_periodic_cbdata *periodic) map = periodic->map; msg_debug_map ("periodic dtor %p", periodic); - event_del (&periodic->ev); if (periodic->need_modify) { /* We are done */ @@ -1162,6 +1172,16 @@ rspamd_map_periodic_dtor (struct map_periodic_cbdata *periodic) g_free (periodic); } +/* Called on timer execution */ +static void +rspamd_map_periodic_callback (struct ev_loop *loop, ev_timer *w, int revents) +{ + struct map_periodic_cbdata *cbd = (struct map_periodic_cbdata *)w->data; + + ev_timer_stop (loop, w); + rspamd_map_process_periodic (cbd); +} + static void rspamd_map_schedule_periodic (struct rspamd_map *map, gboolean locked, gboolean initial, gboolean errored) @@ -1221,17 +1241,15 @@ rspamd_map_schedule_periodic (struct rspamd_map *map, cbd->cbdata.cur_data = NULL; cbd->cbdata.map = map; cbd->map = map; - map->scheduled_check = TRUE; + map->scheduled_check = cbd; REF_INIT_RETAIN (cbd, rspamd_map_periodic_dtor); - evtimer_set (&cbd->ev, rspamd_map_periodic_callback, cbd); - event_base_set (map->ev_base, &cbd->ev); - + cbd->ev.data = cbd; + ev_timer_init (&cbd->ev, rspamd_map_periodic_callback, jittered_sec, 0.0); + ev_timer_start (map->event_loop, &cbd->ev); msg_debug_map ("schedule new periodic event %p in %.2f seconds", cbd, jittered_sec); - double_to_tv (jittered_sec, &map->tv); - evtimer_add (&cbd->ev, &map->tv); } static void @@ -1286,7 +1304,7 @@ rspamd_map_dns_callback (struct rdns_reply *reply, void *arg) msg_err_map ("cannot resolve %s: %s", cbd->data->host, rdns_strerror (reply->code)); cbd->periodic->errored = 1; - rspamd_map_periodic_callback (-1, EV_TIMEOUT, cbd->periodic); + rspamd_map_process_periodic (cbd->periodic); } } @@ -1567,7 +1585,7 @@ rspamd_map_common_http_callback (struct rspamd_map *map, periodic->need_modify = TRUE; /* Reset the whole chain */ periodic->cur_backend = 0; - rspamd_map_periodic_callback (-1, EV_TIMEOUT, periodic); + rspamd_map_process_periodic (periodic); } else { if (map->active_http) { @@ -1577,7 +1595,7 @@ rspamd_map_common_http_callback (struct rspamd_map *map, else { /* Switch to the next backend */ periodic->cur_backend++; - rspamd_map_periodic_callback (-1, EV_TIMEOUT, periodic); + rspamd_map_process_periodic (periodic); } } @@ -1592,7 +1610,7 @@ rspamd_map_common_http_callback (struct rspamd_map *map, /* Switch to the next backend */ periodic->cur_backend++; data->last_modified = data->cache->last_modified; - rspamd_map_periodic_callback (-1, EV_TIMEOUT, periodic); + rspamd_map_process_periodic (periodic); return; } @@ -1601,7 +1619,7 @@ rspamd_map_common_http_callback (struct rspamd_map *map, else if (!map->active_http) { /* Switch to the next backend */ periodic->cur_backend ++; - rspamd_map_periodic_callback (-1, EV_TIMEOUT, periodic); + rspamd_map_process_periodic (periodic); return; } @@ -1609,7 +1627,7 @@ rspamd_map_common_http_callback (struct rspamd_map *map, check: cbd = g_malloc0 (sizeof (struct http_callback_data)); - cbd->ev_base = map->ev_base; + cbd->event_loop = map->event_loop; cbd->map = map; cbd->data = data; cbd->check = check; @@ -1618,7 +1636,6 @@ check: cbd->bk = bk; MAP_RETAIN (bk, "rspamd_map_backend"); cbd->stage = map_resolve_host2; - double_to_tv (map->cfg->map_timeout, &cbd->tv); REF_INIT_RETAIN (cbd, free_http_cbdata); msg_debug_map ("%s map data from %s", check ? "checking" : "reading", @@ -1673,9 +1690,8 @@ check: } static void -rspamd_map_http_check_callback (gint fd, short what, void *ud) +rspamd_map_http_check_callback (struct map_periodic_cbdata *cbd) { - struct map_periodic_cbdata *cbd = ud; struct rspamd_map *map; struct rspamd_map_backend *bk; @@ -1686,9 +1702,8 @@ rspamd_map_http_check_callback (gint fd, short what, void *ud) } static void -rspamd_map_http_read_callback (gint fd, short what, void *ud) +rspamd_map_http_read_callback (struct map_periodic_cbdata *cbd) { - struct map_periodic_cbdata *cbd = ud; struct rspamd_map *map; struct rspamd_map_backend *bk; @@ -1698,43 +1713,36 @@ rspamd_map_http_read_callback (gint fd, short what, void *ud) } static void -rspamd_map_file_check_callback (gint fd, short what, void *ud) +rspamd_map_file_check_callback (struct map_periodic_cbdata *periodic) { struct rspamd_map *map; - struct map_periodic_cbdata *periodic = ud; struct file_map_data *data; struct rspamd_map_backend *bk; - struct stat st; map = periodic->map; - bk = g_ptr_array_index (map->backends, periodic->cur_backend); data = bk->data.fd; - if (stat (data->filename, &st) != -1 && - (st.st_mtime > data->st.st_mtime || data->st.st_mtime == -1)) { - /* File was modified since last check */ - msg_info_map ("old mtime is %t, new mtime is %t for map file %s", - data->st.st_mtime, st.st_mtime, data->filename); - memcpy (&data->st, &st, sizeof (struct stat)); + if (data->need_modify) { periodic->need_modify = TRUE; periodic->cur_backend = 0; + data->need_modify = FALSE; - rspamd_map_periodic_callback (-1, EV_TIMEOUT, periodic); + rspamd_map_process_periodic (periodic); return; } - /* Switch to the next backend */ + map = periodic->map; + /* Switch to the next backend as the rest is handled by ev_stat */ periodic->cur_backend ++; - rspamd_map_periodic_callback (-1, EV_TIMEOUT, periodic); + rspamd_map_process_periodic (periodic); } static void -rspamd_map_static_check_callback (gint fd, short what, void *ud) +rspamd_map_static_check_callback (struct map_periodic_cbdata *periodic) { struct rspamd_map *map; - struct map_periodic_cbdata *periodic = ud; struct static_map_data *data; struct rspamd_map_backend *bk; @@ -1746,21 +1754,20 @@ rspamd_map_static_check_callback (gint fd, short what, void *ud) periodic->need_modify = TRUE; periodic->cur_backend = 0; - rspamd_map_periodic_callback (-1, EV_TIMEOUT, periodic); + rspamd_map_process_periodic (periodic); return; } /* Switch to the next backend */ periodic->cur_backend ++; - rspamd_map_periodic_callback (-1, EV_TIMEOUT, periodic); + rspamd_map_process_periodic (periodic); } static void -rspamd_map_file_read_callback (gint fd, short what, void *ud) +rspamd_map_file_read_callback (struct map_periodic_cbdata *periodic) { struct rspamd_map *map; - struct map_periodic_cbdata *periodic = ud; struct file_map_data *data; struct rspamd_map_backend *bk; @@ -1777,14 +1784,13 @@ rspamd_map_file_read_callback (gint fd, short what, void *ud) /* Switch to the next backend */ periodic->cur_backend ++; - rspamd_map_periodic_callback (-1, EV_TIMEOUT, periodic); + rspamd_map_process_periodic (periodic); } static void -rspamd_map_static_read_callback (gint fd, short what, void *ud) +rspamd_map_static_read_callback (struct map_periodic_cbdata *periodic) { struct rspamd_map *map; - struct map_periodic_cbdata *periodic = ud; struct static_map_data *data; struct rspamd_map_backend *bk; @@ -1801,18 +1807,17 @@ rspamd_map_static_read_callback (gint fd, short what, void *ud) /* Switch to the next backend */ periodic->cur_backend ++; - rspamd_map_periodic_callback (-1, EV_TIMEOUT, periodic); + rspamd_map_process_periodic (periodic); } static void -rspamd_map_periodic_callback (gint fd, short what, void *ud) +rspamd_map_process_periodic (struct map_periodic_cbdata *cbd) { struct rspamd_map_backend *bk; - struct map_periodic_cbdata *cbd = ud; struct rspamd_map *map; map = cbd->map; - map->scheduled_check = FALSE; + map->scheduled_check = NULL; if (!cbd->locked) { if (!g_atomic_int_compare_and_exchange (cbd->map->locked, 0, 1)) { @@ -1863,13 +1868,13 @@ rspamd_map_periodic_callback (gint fd, short what, void *ud) switch (bk->protocol) { case MAP_PROTO_HTTP: case MAP_PROTO_HTTPS: - rspamd_map_http_read_callback (fd, what, cbd); + rspamd_map_http_read_callback (cbd); break; case MAP_PROTO_FILE: - rspamd_map_file_read_callback (fd, what, cbd); + rspamd_map_file_read_callback (cbd); break; case MAP_PROTO_STATIC: - rspamd_map_static_read_callback (fd, what, cbd); + rspamd_map_static_read_callback (cbd); break; } } else { @@ -1877,34 +1882,70 @@ rspamd_map_periodic_callback (gint fd, short what, void *ud) switch (bk->protocol) { case MAP_PROTO_HTTP: case MAP_PROTO_HTTPS: - rspamd_map_http_check_callback (fd, what, cbd); + rspamd_map_http_check_callback (cbd); break; case MAP_PROTO_FILE: - rspamd_map_file_check_callback (fd, what, cbd); + rspamd_map_file_check_callback (cbd); break; case MAP_PROTO_STATIC: - rspamd_map_static_check_callback (fd, what, cbd); + rspamd_map_static_check_callback (cbd); break; } } } } +static void +rspamd_map_on_stat (struct ev_loop *loop, ev_stat *w, int revents) +{ + struct rspamd_map *map = (struct rspamd_map *)w->data; + + if (w->attr.st_nlink > 0) { + + if (w->attr.st_mtime > w->prev.st_mtime) { + msg_info_map ("old mtime is %t, new mtime is %t for map file %s", + w->prev.st_mtime, w->attr.st_mtime, w->path); + + /* Fire need modify flag */ + struct rspamd_map_backend *bk; + guint i; + + PTR_ARRAY_FOREACH (map->backends, i, bk) { + if (bk->protocol == MAP_PROTO_FILE) { + bk->data.fd->need_modify = TRUE; + } + } + + map->next_check = 0; + + if (map->scheduled_check) { + ev_timer_stop (map->event_loop, &map->scheduled_check->ev); + MAP_RELEASE (map->scheduled_check, "rspamd_map_on_stat"); + map->scheduled_check = NULL; + } + + rspamd_map_schedule_periodic (map, FALSE, TRUE, FALSE); + } + } +} + /* Start watching event for all maps */ void rspamd_map_watch (struct rspamd_config *cfg, - struct event_base *ev_base, + struct ev_loop *event_loop, struct rspamd_dns_resolver *resolver, struct rspamd_worker *worker, gboolean active_http) { GList *cur = cfg->maps; struct rspamd_map *map; + struct rspamd_map_backend *bk; + guint i; /* First of all do synced read of data */ while (cur) { map = cur->data; - map->ev_base = ev_base; + map->event_loop = event_loop; map->r = resolver; map->wrk = worker; @@ -1922,6 +1963,21 @@ rspamd_map_watch (struct rspamd_config *cfg, } } + PTR_ARRAY_FOREACH (map->backends, i, bk) { + bk->event_loop = event_loop; + + if (bk->protocol == MAP_PROTO_FILE) { + struct file_map_data *data; + + data = bk->data.fd; + + ev_stat_init (&data->st_ev, rspamd_map_on_stat, + data->filename, map->poll_timeout * cfg->map_file_watch_multiplier); + data->st_ev.data = map; + ev_stat_start (event_loop, &data->st_ev); + } + } + rspamd_map_schedule_periodic (map, FALSE, TRUE, FALSE); cur = g_list_next (cur); @@ -2215,6 +2271,7 @@ rspamd_map_backend_dtor (struct rspamd_map_backend *bk) switch (bk->protocol) { case MAP_PROTO_FILE: if (bk->data.fd) { + ev_stat_stop (bk->event_loop, &bk->data.fd->st_ev); g_free (bk->data.fd->filename); g_free (bk->data.fd); } @@ -2249,7 +2306,8 @@ rspamd_map_backend_dtor (struct rspamd_map_backend *bk) if (data->cur_cache_cbd) { MAP_RELEASE (data->cur_cache_cbd->shm, "rspamd_http_map_cached_cbdata"); - event_del (&data->cur_cache_cbd->timeout); + ev_timer_stop (data->cur_cache_cbd->event_loop, + &data->cur_cache_cbd->timeout); g_free (data->cur_cache_cbd); data->cur_cache_cbd = NULL; } @@ -2308,7 +2366,6 @@ rspamd_map_parse_backend (struct rspamd_config *cfg, const gchar *map_line) /* Now check for each proto separately */ if (bk->protocol == MAP_PROTO_FILE) { fdata = g_malloc0 (sizeof (struct file_map_data)); - fdata->st.st_mtime = -1; if (access (bk->uri, R_OK) == -1) { if (errno != ENOENT) { diff --git a/src/libutil/map.h b/src/libutil/map.h index acf6eea4e..9f04d4c6c 100644 --- a/src/libutil/map.h +++ b/src/libutil/map.h @@ -2,7 +2,7 @@ #define RSPAMD_MAP_H #include "config.h" -#include <event.h> +#include "contrib/libev/ev.h" #include "ucl.h" #include "mem_pool.h" @@ -79,7 +79,7 @@ struct rspamd_map* rspamd_map_add_from_ucl (struct rspamd_config *cfg, * Start watching of maps by adding events to libevent event loop */ void rspamd_map_watch (struct rspamd_config *cfg, - struct event_base *ev_base, + struct ev_loop *event_loop, struct rspamd_dns_resolver *resolver, struct rspamd_worker *worker, gboolean active_http); diff --git a/src/libutil/map_private.h b/src/libutil/map_private.h index b32f0e390..e08c2dce3 100644 --- a/src/libutil/map_private.h +++ b/src/libutil/map_private.h @@ -54,14 +54,16 @@ enum fetch_proto { */ struct file_map_data { gchar *filename; - struct stat st; + gboolean need_modify; + ev_stat st_ev; }; struct http_map_data; struct rspamd_http_map_cached_cbdata { - struct event timeout; + ev_timer timeout; + struct ev_loop *event_loop; struct rspamd_storage_shmem *shm; struct rspamd_map *map; struct http_map_data *data; @@ -114,6 +116,7 @@ struct rspamd_map_backend { gboolean is_signed; gboolean is_compressed; gboolean is_fallback; + struct ev_loop *event_loop; guint32 id; struct rspamd_cryptobox_pubkey *trusted_pubkey; union rspamd_map_backend_data data; @@ -121,6 +124,8 @@ struct rspamd_map_backend { ref_entry_t ref; }; +struct map_periodic_cbdata; + struct rspamd_map { struct rspamd_dns_resolver *r; struct rspamd_config *cfg; @@ -130,12 +135,12 @@ struct rspamd_map { map_fin_cb_t fin_callback; map_dtor_t dtor; void **user_data; - struct event_base *ev_base; + struct ev_loop *event_loop; struct rspamd_worker *wrk; gchar *description; gchar *name; guint32 id; - gboolean scheduled_check; + struct map_periodic_cbdata *scheduled_check; rspamd_map_tmp_dtor tmp_dtor; gpointer tmp_dtor_data; rspamd_map_traverse_function traverse_function; @@ -143,7 +148,7 @@ struct rspamd_map { gsize nelts; guint64 digest; /* Should we check HTTP or just load cached data */ - struct timeval tv; + ev_tstamp timeout; gdouble poll_timeout; time_t next_check; gboolean active_http; @@ -164,7 +169,7 @@ enum rspamd_map_http_stage { struct map_periodic_cbdata { struct rspamd_map *map; struct map_cb_data cbdata; - struct event ev; + ev_timer ev; gboolean need_modify; gboolean errored; gboolean locked; @@ -183,7 +188,7 @@ struct rspamd_http_file_data { }; struct http_callback_data { - struct event_base *ev_base; + struct ev_loop *event_loop; struct rspamd_http_connection *conn; rspamd_inet_addr_t *addr; struct rspamd_map *map; @@ -191,16 +196,15 @@ struct http_callback_data { struct http_map_data *data; struct map_periodic_cbdata *periodic; struct rspamd_cryptobox_pubkey *pk; - gboolean check; struct rspamd_storage_shmem *shmem_data; struct rspamd_storage_shmem *shmem_sig; struct rspamd_storage_shmem *shmem_pubkey; gsize data_len; gsize sig_len; gsize pubkey_len; - + gboolean check; enum rspamd_map_http_stage stage; - struct timeval tv; + ev_tstamp timeout; ref_entry_t ref; }; diff --git a/src/libutil/ssl_util.c b/src/libutil/ssl_util.c index 95245aa4c..7d4612b3d 100644 --- a/src/libutil/ssl_util.c +++ b/src/libutil/ssl_util.c @@ -45,9 +45,8 @@ struct rspamd_ssl_connection { gboolean verify_peer; SSL *ssl; gchar *hostname; - struct event *ev; - struct event_base *ev_base; - struct timeval *tv; + struct rspamd_io_ev *ev; + struct ev_loop *event_loop; rspamd_ssl_handler_t handler; rspamd_ssl_error_handler_t err_handler; gpointer handler_data; @@ -407,7 +406,7 @@ rspamd_ssl_event_handler (gint fd, short what, gpointer ud) gint ret; GError *err = NULL; - if (what == EV_TIMEOUT) { + if (what == EV_TIMER) { c->shut = ssl_shut_unclean; } @@ -417,7 +416,7 @@ rspamd_ssl_event_handler (gint fd, short what, gpointer ud) ret = SSL_connect (c->ssl); if (ret == 1) { - event_del (c->ev); + rspamd_ev_watcher_stop (c->event_loop, c->ev); /* Verify certificate */ if ((!c->verify_peer) || rspamd_ssl_peer_verify (c)) { c->state = ssl_conn_connected; @@ -437,40 +436,30 @@ rspamd_ssl_event_handler (gint fd, short what, gpointer ud) what = EV_WRITE; } else { + rspamd_ev_watcher_stop (c->event_loop, c->ev); rspamd_tls_set_error (ret, "connect", &err); c->err_handler (c->handler_data, err); g_error_free (err); return; } - event_del (c->ev); - event_set (c->ev, fd, what, rspamd_ssl_event_handler, c); - event_base_set (c->ev_base, c->ev); - event_add (c->ev, c->tv); + rspamd_ev_watcher_reschedule (c->event_loop, c->ev, what); + } break; case ssl_next_read: - event_del (c->ev); - /* Restore handler */ - event_set (c->ev, c->fd, EV_READ|EV_PERSIST, - c->handler, c->handler_data); - event_base_set (c->ev_base, c->ev); - event_add (c->ev, c->tv); + rspamd_ev_watcher_reschedule (c->event_loop, c->ev, EV_READ); c->state = ssl_conn_connected; c->handler (fd, EV_READ, c->handler_data); break; case ssl_next_write: case ssl_conn_connected: - event_del (c->ev); - /* Restore handler */ - event_set (c->ev, c->fd, EV_WRITE, - c->handler, c->handler_data); - event_base_set (c->ev_base, c->ev); - event_add (c->ev, c->tv); + rspamd_ev_watcher_reschedule (c->event_loop, c->ev, what); c->state = ssl_conn_connected; - c->handler (fd, EV_WRITE, c->handler_data); + c->handler (fd, what, c->handler_data); break; default: + rspamd_ev_watcher_stop (c->event_loop, c->ev); g_set_error (&err, rspamd_ssl_quark (), EINVAL, "ssl bad state error: %d", c->state); c->err_handler (c->handler_data, err); @@ -480,7 +469,7 @@ rspamd_ssl_event_handler (gint fd, short what, gpointer ud) } struct rspamd_ssl_connection * -rspamd_ssl_connection_new (gpointer ssl_ctx, struct event_base *ev_base, +rspamd_ssl_connection_new (gpointer ssl_ctx, struct ev_loop *ev_base, gboolean verify_peer) { struct rspamd_ssl_connection *c; @@ -488,7 +477,7 @@ rspamd_ssl_connection_new (gpointer ssl_ctx, struct event_base *ev_base, g_assert (ssl_ctx != NULL); c = g_malloc0 (sizeof (*c)); c->ssl = SSL_new (ssl_ctx); - c->ev_base = ev_base; + c->event_loop = ev_base; c->verify_peer = verify_peer; return c; @@ -497,7 +486,7 @@ rspamd_ssl_connection_new (gpointer ssl_ctx, struct event_base *ev_base, gboolean rspamd_ssl_connect_fd (struct rspamd_ssl_connection *conn, gint fd, - const gchar *hostname, struct event *ev, struct timeval *tv, + const gchar *hostname, struct rspamd_io_ev *ev, ev_tstamp timeout, rspamd_ssl_handler_t handler, rspamd_ssl_error_handler_t err_handler, gpointer handler_data) { @@ -534,17 +523,9 @@ rspamd_ssl_connect_fd (struct rspamd_ssl_connection *conn, gint fd, if (ret == 1) { conn->state = ssl_conn_connected; - if (rspamd_event_pending (ev, EV_TIMEOUT|EV_WRITE|EV_READ)) { - event_del (ev); - } - - event_set (ev, fd, EV_WRITE, rspamd_ssl_event_handler, conn); - - if (conn->ev_base) { - event_base_set (conn->ev_base, ev); - } - - event_add (ev, tv); + rspamd_ev_watcher_stop (conn->event_loop, ev); + rspamd_ev_watcher_init (ev, fd, EV_WRITE, rspamd_ssl_event_handler, conn); + rspamd_ev_watcher_start (conn->event_loop, ev, timeout); } else { ret = SSL_get_error (conn->ssl, ret); @@ -561,13 +542,10 @@ rspamd_ssl_connect_fd (struct rspamd_ssl_connection *conn, gint fd, return FALSE; } - if (rspamd_event_pending (ev, EV_TIMEOUT|EV_WRITE|EV_READ)) { - event_del (ev); - } - - event_set (ev, fd, what, rspamd_ssl_event_handler, conn); - event_base_set (conn->ev_base, ev); - event_add (ev, tv); + rspamd_ev_watcher_stop (conn->event_loop, ev); + rspamd_ev_watcher_init (ev, fd, EV_WRITE|EV_READ, + rspamd_ssl_event_handler, conn); + rspamd_ev_watcher_start (conn->event_loop, ev, timeout); } return TRUE; @@ -638,13 +616,8 @@ rspamd_ssl_read (struct rspamd_ssl_connection *conn, gpointer buf, return -1; } - event_del (conn->ev); - event_set (conn->ev, conn->fd, what, rspamd_ssl_event_handler, conn); - event_base_set (conn->ev_base, conn->ev); - event_add (conn->ev, conn->tv); - + rspamd_ev_watcher_reschedule (conn->event_loop, conn->ev, what); errno = EAGAIN; - } return -1; @@ -713,11 +686,7 @@ rspamd_ssl_write (struct rspamd_ssl_connection *conn, gconstpointer buf, return -1; } - event_del (conn->ev); - event_set (conn->ev, conn->fd, what, rspamd_ssl_event_handler, conn); - event_base_set (conn->ev_base, conn->ev); - event_add (conn->ev, conn->tv); - + rspamd_ev_watcher_reschedule (conn->event_loop, conn->ev, what); errno = EAGAIN; } diff --git a/src/libutil/ssl_util.h b/src/libutil/ssl_util.h index 73a940e00..f7f1652de 100644 --- a/src/libutil/ssl_util.h +++ b/src/libutil/ssl_util.h @@ -18,6 +18,7 @@ #include "config.h" #include "libutil/addr.h" +#include "libutil/libev_helper.h" struct rspamd_ssl_connection; @@ -30,7 +31,7 @@ typedef void (*rspamd_ssl_error_handler_t)(gpointer d, GError *err); * @return opaque connection data */ struct rspamd_ssl_connection * rspamd_ssl_connection_new (gpointer ssl_ctx, - struct event_base *ev_base, gboolean verify_peer); + struct ev_loop *ev_base, gboolean verify_peer); /** * Connects SSL session using the specified (connected) FD @@ -44,7 +45,7 @@ struct rspamd_ssl_connection * rspamd_ssl_connection_new (gpointer ssl_ctx, * @return TRUE if a session has been connected */ gboolean rspamd_ssl_connect_fd (struct rspamd_ssl_connection *conn, gint fd, - const gchar *hostname, struct event *ev, struct timeval *tv, + const gchar *hostname, struct rspamd_io_ev *ev, ev_tstamp timeout, rspamd_ssl_handler_t handler, rspamd_ssl_error_handler_t err_handler, gpointer handler_data); diff --git a/src/libutil/str_util.h b/src/libutil/str_util.h index 8e8898a32..6fbb11ccf 100644 --- a/src/libutil/str_util.h +++ b/src/libutil/str_util.h @@ -83,10 +83,18 @@ gsize rspamd_strlcpy_safe (gchar *dst, const gchar *src, gsize siz); # if __has_feature(address_sanitizer) # define rspamd_strlcpy rspamd_strlcpy_safe # else -# define rspamd_strlcpy rspamd_strlcpy_fast +# ifdef __SANITIZE_ADDRESS__ +# define rspamd_strlcpy rspamd_strlcpy_safe +# else +# define rspamd_strlcpy rspamd_strlcpy_fast +# endif # endif #else -# define rspamd_strlcpy rspamd_strlcpy_fast +# ifdef __SANITIZE_ADDRESS__ +# define rspamd_strlcpy rspamd_strlcpy_safe +# else +# define rspamd_strlcpy rspamd_strlcpy_fast +# endif #endif /** diff --git a/src/libutil/upstream.c b/src/libutil/upstream.c index 3e04e68e9..c445751b4 100644 --- a/src/libutil/upstream.c +++ b/src/libutil/upstream.c @@ -50,7 +50,7 @@ struct upstream { guint dns_requests; gint active_idx; gchar *name; - struct event ev; + ev_timer ev; gdouble last_fail; gpointer ud; struct upstream_list *ls; @@ -92,7 +92,7 @@ struct upstream_list { struct upstream_ctx { struct rdns_resolver *res; - struct event_base *ev_base; + struct ev_loop *event_loop; struct upstream_limits limits; GQueue *upstreams; gboolean configured; @@ -119,7 +119,7 @@ static guint default_dns_retransmits = 2; void rspamd_upstreams_library_config (struct rspamd_config *cfg, struct upstream_ctx *ctx, - struct event_base *ev_base, + struct ev_loop *event_loop, struct rdns_resolver *resolver) { g_assert (ctx != NULL); @@ -141,7 +141,7 @@ rspamd_upstreams_library_config (struct rspamd_config *cfg, ctx->limits.dns_timeout = cfg->dns_timeout; } - ctx->ev_base = ev_base; + ctx->event_loop = event_loop; ctx->res = resolver; ctx->configured = TRUE; } @@ -366,12 +366,12 @@ rspamd_upstream_dns_cb (struct rdns_reply *reply, void *arg) } static void -rspamd_upstream_revive_cb (int fd, short what, void *arg) +rspamd_upstream_revive_cb (struct ev_loop *loop, ev_timer *w, int revents) { - struct upstream *up = (struct upstream *)arg; + struct upstream *up = (struct upstream *)w->data; RSPAMD_UPSTREAM_LOCK (up->lock); - event_del (&up->ev); + ev_timer_stop (loop, w); if (up->ls) { rspamd_upstream_set_active (up->ls, up); } @@ -414,7 +414,6 @@ rspamd_upstream_set_inactive (struct upstream_list *ls, struct upstream *up) gdouble ntim; guint i; struct upstream *cur; - struct timeval tv; struct upstream_list_watcher *w; RSPAMD_UPSTREAM_LOCK (ls->lock); @@ -431,15 +430,14 @@ rspamd_upstream_set_inactive (struct upstream_list *ls, struct upstream *up) rspamd_upstream_resolve_addrs (ls, up); REF_RETAIN (up); - evtimer_set (&up->ev, rspamd_upstream_revive_cb, up); - if (up->ctx->ev_base != NULL && up->ctx->configured) { - event_base_set (up->ctx->ev_base, &up->ev); - } - ntim = rspamd_time_jitter (ls->limits.revive_time, ls->limits.revive_jitter); - double_to_tv (ntim, &tv); - event_add (&up->ev, &tv); + ev_timer_init (&up->ev, rspamd_upstream_revive_cb, ntim, 0); + up->ev.data = up; + + if (up->ctx->event_loop != NULL && up->ctx->configured) { + ev_timer_start (up->ctx->event_loop, &up->ev); + } } DL_FOREACH (up->ls->watchers, w) { @@ -915,9 +913,7 @@ rspamd_upstream_restore_cb (gpointer elt, gpointer ls) /* Here the upstreams list is already locked */ RSPAMD_UPSTREAM_LOCK (up->lock); - if (rspamd_event_pending (&up->ev, EV_TIMEOUT)) { - event_del (&up->ev); - } + ev_timer_stop (up->ctx->event_loop, &up->ev); g_ptr_array_add (ups->alive, up); up->active_idx = ups->alive->len - 1; RSPAMD_UPSTREAM_UNLOCK (up->lock); diff --git a/src/libutil/upstream.h b/src/libutil/upstream.h index 75d840ce2..89ac0ee9e 100644 --- a/src/libutil/upstream.h +++ b/src/libutil/upstream.h @@ -41,7 +41,7 @@ void rspamd_upstreams_library_unref (struct upstream_ctx *ctx); * @param cfg */ void rspamd_upstreams_library_config (struct rspamd_config *cfg, - struct upstream_ctx *ctx, struct event_base *ev_base, + struct upstream_ctx *ctx, struct ev_loop *event_loop, struct rdns_resolver *resolver); /** diff --git a/src/libutil/util.c b/src/libutil/util.c index df10bf912..e7a5c2601 100644 --- a/src/libutil/util.c +++ b/src/libutil/util.c @@ -1612,42 +1612,6 @@ rspamd_thread_func (gpointer ud) return ud; } -/** - * Create new named thread - * @param name name pattern - * @param func function to start - * @param data data to pass to function - * @param err error pointer - * @return new thread object that can be joined - */ -GThread * -rspamd_create_thread (const gchar *name, - GThreadFunc func, - gpointer data, - GError **err) -{ - GThread *new; - struct rspamd_thread_data *td; - static gint32 id; - guint r; - - r = strlen (name); - td = g_malloc (sizeof (struct rspamd_thread_data)); - td->id = ++id; - td->name = g_malloc (r + sizeof ("4294967296")); - td->func = func; - td->data = data; - - rspamd_snprintf (td->name, r + sizeof ("4294967296"), "%s-%d", name, id); -#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION > 32)) - new = g_thread_try_new (td->name, rspamd_thread_func, td, err); -#else - new = g_thread_create (rspamd_thread_func, td, TRUE, err); -#endif - - return new; -} - struct hash_copy_callback_data { gpointer (*key_copy_func)(gconstpointer data, gpointer ud); gpointer (*value_copy_func)(gconstpointer data, gpointer ud); @@ -2570,24 +2534,6 @@ rspamd_constant_memcmp (const guchar *a, const guchar *b, gsize len) return (((gint32)(guint16)((guint32)r + 0x8000) - 0x8000) == 0); } -#if !defined(LIBEVENT_VERSION_NUMBER) || LIBEVENT_VERSION_NUMBER < 0x02000000UL -struct event_base * -event_get_base (struct event *ev) -{ - return ev->ev_base; -} -#endif - -int -rspamd_event_pending (struct event *ev, short what) -{ - if (ev->ev_base == NULL) { - return 0; - } - - return event_pending (ev, what, NULL); -} - int rspamd_file_xopen (const char *fname, int oflags, guint mode, gboolean allow_symlink) diff --git a/src/libutil/util.h b/src/libutil/util.h index 9d12285d4..21e4b320e 100644 --- a/src/libutil/util.h +++ b/src/libutil/util.h @@ -12,7 +12,7 @@ #include <netdb.h> #endif -#include <event.h> +#include "contrib/libev/ev.h" #include <time.h> struct rspamd_config; @@ -263,19 +263,6 @@ void rspamd_mutex_unlock (rspamd_mutex_t *mtx); void rspamd_mutex_free (rspamd_mutex_t *mtx); /** - * Create new named thread - * @param name name pattern - * @param func function to start - * @param data data to pass to function - * @param err error pointer - * @return new thread object that can be joined - */ -GThread * rspamd_create_thread (const gchar *name, - GThreadFunc func, - gpointer data, - GError **err); - -/** * Deep copy of one hash table to another * @param src source hash * @param dst destination hash @@ -426,19 +413,6 @@ void rspamd_random_seed_fast (void); */ gboolean rspamd_constant_memcmp (const guchar *a, const guchar *b, gsize len); -/* Special case for ancient libevent */ -#if !defined(LIBEVENT_VERSION_NUMBER) || LIBEVENT_VERSION_NUMBER < 0x02000000UL -struct event_base * event_get_base (struct event *ev); -#endif -/* CentOS libevent */ -#ifndef evsignal_set -#define evsignal_set(ev, x, cb, arg) \ - event_set((ev), (x), EV_SIGNAL|EV_PERSIST, (cb), (arg)) -#endif - -/* Avoid stupidity in libevent > 1.4 */ -int rspamd_event_pending (struct event *ev, short what); - /** * Open file without following symlinks or special stuff * @param fname filename diff --git a/src/log_helper.c b/src/log_helper.c deleted file mode 100644 index 14a85c5b1..000000000 --- a/src/log_helper.c +++ /dev/null @@ -1,234 +0,0 @@ -/*- - * Copyright 2016 Vsevolod Stakhov - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "config.h" - -#include "libutil/util.h" -#include "libserver/cfg_file.h" -#include "libserver/cfg_rcl.h" -#include "libserver/worker_util.h" -#include "libserver/rspamd_control.h" -#include "libutil/addr.h" -#include "lua/lua_common.h" -#include "unix-std.h" -#include "utlist.h" - -#ifdef HAVE_GLOB_H -#include <glob.h> -#endif - -static gpointer init_log_helper (struct rspamd_config *cfg); -static void start_log_helper (struct rspamd_worker *worker); - -worker_t log_helper_worker = { - "log_helper", /* Name */ - init_log_helper, /* Init function */ - start_log_helper, /* Start function */ - RSPAMD_WORKER_UNIQUE | RSPAMD_WORKER_KILLABLE, - RSPAMD_WORKER_SOCKET_NONE, /* No socket */ - RSPAMD_WORKER_VER /* Version info */ -}; - -static const guint64 rspamd_log_helper_magic = 0x1090bb46aaa74c9aULL; - -/* - * Worker's context - */ -struct log_helper_ctx { - guint64 magic; - /* Events base */ - struct event_base *ev_base; - /* DNS resolver */ - struct rspamd_dns_resolver *resolver; - /* Config */ - struct rspamd_config *cfg; - /* END OF COMMON PART */ - struct event log_ev; - struct rspamd_worker_lua_script *scripts; - lua_State *L; - gint pair[2]; -}; - -static gpointer -init_log_helper (struct rspamd_config *cfg) -{ - struct log_helper_ctx *ctx; - GQuark type; - - type = g_quark_try_string ("log_helper"); - (void)type; - ctx = rspamd_mempool_alloc (cfg->cfg_pool, sizeof (*ctx)); - - ctx->magic = rspamd_log_helper_magic; - ctx->cfg = cfg; - - return ctx; -} - -static void -rspamd_log_helper_read (gint fd, short what, gpointer ud) -{ - struct log_helper_ctx *ctx = ud; - guchar buf[8192]; - gssize r; - guint32 n, i, nextra; - struct rspamd_protocol_log_message_sum *sm; - struct rspamd_worker_lua_script *sc; - struct rspamd_config **pcfg; - struct event_base **pevbase; - - r = read (fd, buf, sizeof (buf)); - - if (r >= (gssize)sizeof (struct rspamd_protocol_log_message_sum)) { - memcpy (&n, buf, sizeof (n)); - memcpy (&nextra, buf + sizeof (n), sizeof (nextra)); - - if (n + nextra != - (r - sizeof (*sm)) / sizeof (struct rspamd_protocol_log_symbol_result)) { - msg_warn ("cannot read data from log pipe: bad length: %d elements " - "announced but %d available", n + nextra, - (gint)((r - sizeof (*sm)) / - sizeof (struct rspamd_protocol_log_symbol_result))); - } - else { - sm = g_malloc (r); - memcpy (sm, buf, r); - - DL_FOREACH (ctx->scripts, sc) { - lua_rawgeti (ctx->L, LUA_REGISTRYINDEX, sc->cbref); - lua_pushnumber (ctx->L, sm->score); - lua_pushnumber (ctx->L, sm->required_score); - - lua_createtable (ctx->L, n, 0); - for (i = 0; i < n; i ++) { - lua_createtable (ctx->L, 2, 0); - lua_pushinteger (ctx->L, sm->results[i].id); - lua_rawseti (ctx->L, -2, 1); - lua_pushnumber (ctx->L, sm->results[i].score); - lua_rawseti (ctx->L, -2, 2); - - lua_rawseti (ctx->L, -2, (i + 1)); - } - - pcfg = lua_newuserdata (ctx->L, sizeof (*pcfg)); - *pcfg = ctx->cfg; - rspamd_lua_setclass (ctx->L, "rspamd{config}", -1); - lua_pushinteger (ctx->L, sm->settings_id); - - lua_createtable (ctx->L, nextra, 0); - for (i = 0; i < nextra; i ++) { - lua_createtable (ctx->L, 2, 0); - lua_pushinteger (ctx->L, sm->results[i + n].id); - lua_rawseti (ctx->L, -2, 1); - lua_pushnumber (ctx->L, sm->results[i + n].score); - lua_rawseti (ctx->L, -2, 2); - - lua_rawseti (ctx->L, -2, (i + 1)); - } - - pevbase = lua_newuserdata (ctx->L, sizeof (*pevbase)); - *pevbase = ctx->ev_base; - rspamd_lua_setclass (ctx->L, "rspamd{ev_base}", -1); - - if (lua_pcall (ctx->L, 7, 0, 0) != 0) { - msg_err ("error executing log handler code: %s", - lua_tostring (ctx->L, -1)); - lua_pop (ctx->L, 1); - } - } - - g_free (sm); - } - } - else if (r == -1) { - if (errno != EAGAIN && errno != EINTR) { - msg_warn ("cannot read data from log pipe: %s", strerror (errno)); - event_del (&ctx->log_ev); - } - } - else if (r == 0) { - msg_warn ("cannot read data from log pipe: EOF"); - event_del (&ctx->log_ev); - } -} - -static void -rspamd_log_helper_reply_handler (struct rspamd_worker *worker, - struct rspamd_srv_reply *rep, gint rep_fd, - gpointer ud) -{ - struct log_helper_ctx *ctx = ud; - - close (ctx->pair[1]); - msg_info ("start waiting for log events"); - event_set (&ctx->log_ev, ctx->pair[0], EV_READ | EV_PERSIST, - rspamd_log_helper_read, ctx); - event_base_set (ctx->ev_base, &ctx->log_ev); - event_add (&ctx->log_ev, NULL); -} - -static void -start_log_helper (struct rspamd_worker *worker) -{ - struct log_helper_ctx *ctx = worker->ctx; - gssize r = -1; - gint nscripts = 0; - struct rspamd_worker_lua_script *tmp; - static struct rspamd_srv_command srv_cmd; - - ctx->ev_base = rspamd_prepare_worker (worker, - "log_helper", - NULL); - ctx->cfg = worker->srv->cfg; - ctx->scripts = worker->cf->scripts; - ctx->L = ctx->cfg->lua_state; - ctx->resolver = rspamd_dns_resolver_init (worker->srv->logger, - ctx->ev_base, - worker->srv->cfg); - rspamd_upstreams_library_config (worker->srv->cfg, ctx->cfg->ups_ctx, - ctx->ev_base, ctx->resolver->r); - - DL_COUNT (worker->cf->scripts, tmp, nscripts); - msg_info ("started log_helper worker with %d scripts", nscripts); - - r = rspamd_socketpair (ctx->pair, FALSE); - - if (r == -1) { - msg_err ("cannot create socketpair: %s, exiting now", strerror (errno)); - /* Prevent new processes spawning */ - exit (EXIT_SUCCESS); - } - - memset (&srv_cmd, 0, sizeof (srv_cmd)); - srv_cmd.type = RSPAMD_SRV_LOG_PIPE; - srv_cmd.cmd.log_pipe.type = RSPAMD_LOG_PIPE_SYMBOLS; - - - /* Wait for startup being completed */ - rspamd_mempool_lock_mutex (worker->srv->start_mtx); - rspamd_srv_send_command (worker, ctx->ev_base, &srv_cmd, ctx->pair[1], - rspamd_log_helper_reply_handler, ctx); - rspamd_mempool_unlock_mutex (worker->srv->start_mtx); - rspamd_lua_run_postloads (ctx->cfg->lua_state, ctx->cfg, ctx->ev_base, - worker); - event_base_loop (ctx->ev_base, 0); - close (ctx->pair[0]); - rspamd_worker_block_signals (); - - REF_RELEASE (ctx->cfg); - rspamd_log_close (worker->srv->logger, TRUE); - - exit (EXIT_SUCCESS); -} diff --git a/src/lua/CMakeLists.txt b/src/lua/CMakeLists.txt index c3f1a84c1..4a2003605 100644 --- a/src/lua/CMakeLists.txt +++ b/src/lua/CMakeLists.txt @@ -22,7 +22,6 @@ SET(LUASRC ${CMAKE_CURRENT_SOURCE_DIR}/lua_common.c ${CMAKE_CURRENT_SOURCE_DIR}/lua_util.c ${CMAKE_CURRENT_SOURCE_DIR}/lua_tcp.c ${CMAKE_CURRENT_SOURCE_DIR}/lua_html.c - ${CMAKE_CURRENT_SOURCE_DIR}/lua_fann.c ${CMAKE_CURRENT_SOURCE_DIR}/lua_sqlite3.c ${CMAKE_CURRENT_SOURCE_DIR}/lua_cryptobox.c ${CMAKE_CURRENT_SOURCE_DIR}/lua_map.c diff --git a/src/lua/lua_cdb.c b/src/lua/lua_cdb.c index a7292da03..5d4c499a7 100644 --- a/src/lua/lua_cdb.c +++ b/src/lua/lua_cdb.c @@ -50,6 +50,7 @@ lua_cdb_create (lua_State *L) struct cdb *cdb, **pcdb; const gchar *filename; gint fd; + struct ev_loop *ev_base = lua_check_ev_base (L, 2); filename = luaL_checkstring (L, 1); /* If file begins with cdb://, just skip it */ @@ -64,13 +65,12 @@ lua_cdb_create (lua_State *L) else { cdb = g_malloc (sizeof (struct cdb)); cdb->filename = g_strdup (filename); - cdb->check_timer_ev = NULL; - cdb->check_timer_tv = NULL; if (cdb_init (cdb, fd) == -1) { msg_warn ("cannot open cdb: %s, %s", filename, strerror (errno)); lua_pushnil (L); } else { + cdb_add_timer (cdb, ev_base, CDB_REFRESH_TIME); pcdb = lua_newuserdata (L, sizeof (struct cdb *)); rspamd_lua_setclass (L, "rspamd{cdb}", -1); *pcdb = cdb; @@ -106,13 +106,6 @@ lua_cdb_lookup (lua_State *L) lua_error (L); return 1; } - /* - * XXX: this code is placed here because event_loop is called inside workers, so start - * monitoring on first check, not on creation - */ - if (cdb->check_timer_ev == NULL) { - cdb_add_timer (cdb, CDB_REFRESH_TIME); - } what = luaL_checkstring (L, 2); if (cdb_find (cdb, what, strlen (what)) > 0) { diff --git a/src/lua/lua_common.c b/src/lua/lua_common.c index e268d6564..689dcd1c4 100644 --- a/src/lua/lua_common.c +++ b/src/lua/lua_common.c @@ -910,7 +910,6 @@ rspamd_lua_init (bool wipe_mem) luaopen_util (L); luaopen_tcp (L); luaopen_html (L); - luaopen_fann (L); luaopen_sqlite3 (L); luaopen_cryptobox (L); luaopen_dns (L); @@ -1828,23 +1827,23 @@ lua_check_session (lua_State * L, gint pos) return ud ? *((struct rspamd_async_session **)ud) : NULL; } -struct event_base* +struct ev_loop* lua_check_ev_base (lua_State * L, gint pos) { void *ud = rspamd_lua_check_udata (L, pos, "rspamd{ev_base}"); luaL_argcheck (L, ud != NULL, pos, "'event_base' expected"); - return ud ? *((struct event_base **)ud) : NULL; + return ud ? *((struct ev_loop **)ud) : NULL; } static void rspamd_lua_run_postloads_error (struct thread_entry *thread, int ret, const char *msg); void rspamd_lua_run_postloads (lua_State *L, struct rspamd_config *cfg, - struct event_base *ev_base, struct rspamd_worker *w) + struct ev_loop *ev_base, struct rspamd_worker *w) { struct rspamd_config_cfg_lua_script *sc; struct rspamd_config **pcfg; - struct event_base **pev_base; + struct ev_loop **pev_base; struct rspamd_worker **pw; /* Execute post load scripts */ diff --git a/src/lua/lua_common.h b/src/lua/lua_common.h index ea07d7717..8919a46fd 100644 --- a/src/lua/lua_common.h +++ b/src/lua/lua_common.h @@ -281,7 +281,6 @@ void luaopen_text (lua_State *L); void luaopen_util (lua_State * L); void luaopen_tcp (lua_State * L); void luaopen_html (lua_State * L); -void luaopen_fann (lua_State *L); void luaopen_sqlite3 (lua_State *L); void luaopen_cryptobox (lua_State *L); void luaopen_dns (lua_State *L); @@ -317,7 +316,7 @@ void rspamd_lua_set_globals (struct rspamd_config *cfg, lua_State *L); struct memory_pool_s * rspamd_lua_check_mempool (lua_State * L, gint pos); struct rspamd_config * lua_check_config (lua_State * L, gint pos); struct rspamd_async_session* lua_check_session (lua_State * L, gint pos); -struct event_base* lua_check_ev_base (lua_State * L, gint pos); +struct ev_loop* lua_check_ev_base (lua_State * L, gint pos); struct rspamd_dns_resolver * lua_check_dns_resolver (lua_State * L, gint pos); /** @@ -420,7 +419,7 @@ void lua_call_finish_script (struct rspamd_config_cfg_lua_script *sc, * @param ev_base */ void rspamd_lua_run_postloads (lua_State *L, struct rspamd_config *cfg, - struct event_base *ev_base, struct rspamd_worker *w); + struct ev_loop *ev_base, struct rspamd_worker *w); void rspamd_lua_run_config_post_init (lua_State *L, struct rspamd_config *cfg); void rspamd_lua_run_config_unload (lua_State *L, struct rspamd_config *cfg); diff --git a/src/lua/lua_config.c b/src/lua/lua_config.c index 27a2e32a8..9f7952cc3 100644 --- a/src/lua/lua_config.c +++ b/src/lua/lua_config.c @@ -3050,21 +3050,21 @@ static void lua_periodic_callback_finish (struct thread_entry *thread, int ret); static void lua_periodic_callback_error (struct thread_entry *thread, int ret, const char *msg); struct rspamd_lua_periodic { - struct event_base *ev_base; + struct ev_loop *event_loop; struct rspamd_config *cfg; lua_State *L; gdouble timeout; - struct event ev; + ev_timer ev; gint cbref; gboolean need_jitter; }; static void -lua_periodic_callback (gint unused_fd, short what, gpointer ud) +lua_periodic_callback (struct ev_loop *loop, ev_timer *w, int revents) { - struct rspamd_lua_periodic *periodic = ud; + struct rspamd_lua_periodic *periodic = (struct rspamd_lua_periodic *)w->data; struct rspamd_config **pcfg, *cfg; - struct event_base **pev_base; + struct ev_loop **pev_base; struct thread_entry *thread; lua_State *L; @@ -3082,9 +3082,8 @@ lua_periodic_callback (gint unused_fd, short what, gpointer ud) *pcfg = cfg; pev_base = lua_newuserdata (L, sizeof (*pev_base)); rspamd_lua_setclass (L, "rspamd{ev_base}", -1); - *pev_base = periodic->ev_base; + *pev_base = periodic->event_loop; - event_del (&periodic->ev); lua_thread_call (thread, 2); } @@ -3094,11 +3093,11 @@ lua_periodic_callback_finish (struct thread_entry *thread, int ret) lua_State *L; struct rspamd_lua_periodic *periodic = thread->cd; gboolean plan_more = FALSE; - struct timeval tv; gdouble timeout = 0.0; L = thread->lua_state; + ev_now_update (periodic->event_loop); #ifdef HAVE_EVENT_NO_CACHE_TIME_FUNC event_base_update_cache_time (periodic->ev_base); #endif @@ -3120,11 +3119,12 @@ lua_periodic_callback_finish (struct thread_entry *thread, int ret) timeout = rspamd_time_jitter (timeout, 0.0); } - double_to_tv (timeout, &tv); - event_add (&periodic->ev, &tv); + periodic->ev.repeat = timeout; + ev_timer_again (periodic->event_loop, &periodic->ev); } else { luaL_unref (L, LUA_REGISTRYINDEX, periodic->cbref); + ev_timer_stop (periodic->event_loop, &periodic->ev); g_free (periodic); } } @@ -3138,7 +3138,7 @@ lua_periodic_callback_error (struct thread_entry *thread, int ret, const char *m msg_err_config ("call to finishing script failed: %s", msg); - lua_periodic_callback_finish(thread, ret); + lua_periodic_callback_finish (thread, ret); } @@ -3147,9 +3147,8 @@ lua_config_add_periodic (lua_State *L) { LUA_TRACE_POINT; struct rspamd_config *cfg = lua_check_config (L, 1); - struct event_base *ev_base = lua_check_ev_base (L, 2); + struct ev_loop *ev_base = lua_check_ev_base (L, 2); gdouble timeout = lua_tonumber (L, 3); - struct timeval tv; struct rspamd_lua_periodic *periodic; gboolean need_jitter = FALSE; @@ -3165,19 +3164,18 @@ lua_config_add_periodic (lua_State *L) periodic->timeout = timeout; periodic->L = L; periodic->cfg = cfg; - periodic->ev_base = ev_base; + periodic->event_loop = ev_base; periodic->need_jitter = need_jitter; lua_pushvalue (L, 4); periodic->cbref = luaL_ref (L, LUA_REGISTRYINDEX); - event_set (&periodic->ev, -1, EV_TIMEOUT, lua_periodic_callback, periodic); - event_base_set (ev_base, &periodic->ev); if (need_jitter) { timeout = rspamd_time_jitter (timeout, 0.0); } - double_to_tv (timeout, &tv); - event_add (&periodic->ev, &tv); + ev_timer_init (&periodic->ev, lua_periodic_callback, timeout, 0.0); + periodic->ev.data = periodic; + ev_timer_start (ev_base, &periodic->ev); return 0; } @@ -3961,7 +3959,7 @@ lua_config_init_subsystem (lua_State *L) rspamd_stat_init (cfg, NULL); } else if (strcmp (parts[i], "dns") == 0) { - struct event_base *ev_base = lua_check_ev_base (L, 3); + struct ev_loop *ev_base = lua_check_ev_base (L, 3); if (ev_base) { cfg->dns_resolver = rspamd_dns_resolver_init (rspamd_logger_get_singleton (), diff --git a/src/lua/lua_dns_resolver.c b/src/lua/lua_dns_resolver.c index 56b6989c0..382e9e985 100644 --- a/src/lua/lua_dns_resolver.c +++ b/src/lua/lua_dns_resolver.c @@ -313,7 +313,7 @@ lua_dns_resolver_init (lua_State *L) { struct rspamd_dns_resolver *resolver, **presolver; struct rspamd_config *cfg, **pcfg; - struct event_base *base, **pbase; + struct ev_loop *base, **pbase; /* Check args */ pbase = rspamd_lua_check_udata (L, 1, "rspamd{ev_base}"); diff --git a/src/lua/lua_fann.c b/src/lua/lua_fann.c deleted file mode 100644 index 40353ea40..000000000 --- a/src/lua/lua_fann.c +++ /dev/null @@ -1,1032 +0,0 @@ -/*- - * Copyright 2016 Vsevolod Stakhov - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "lua_common.h" - -#ifdef WITH_FANN -#include <fann.h> -#endif - -#include "unix-std.h" - -/*** - * @module rspamd_fann - * This module enables [fann](http://libfann.github.io) interaction in rspamd - * Please note, that this module works merely if you have `ENABLE_FANN=ON` option - * definition when building rspamd - */ - -/* - * Fann functions - */ -LUA_FUNCTION_DEF (fann, is_enabled); -LUA_FUNCTION_DEF (fann, create); -LUA_FUNCTION_DEF (fann, create_full); -LUA_FUNCTION_DEF (fann, load_file); -LUA_FUNCTION_DEF (fann, load_data); - -/* - * Fann methods - */ -LUA_FUNCTION_DEF (fann, train); -LUA_FUNCTION_DEF (fann, train_threaded); -LUA_FUNCTION_DEF (fann, test); -LUA_FUNCTION_DEF (fann, save); -LUA_FUNCTION_DEF (fann, data); -LUA_FUNCTION_DEF (fann, get_inputs); -LUA_FUNCTION_DEF (fann, get_outputs); -LUA_FUNCTION_DEF (fann, get_layers); -LUA_FUNCTION_DEF (fann, get_mse); -LUA_FUNCTION_DEF (fann, dtor); - -static const struct luaL_reg fannlib_f[] = { - LUA_INTERFACE_DEF (fann, is_enabled), - LUA_INTERFACE_DEF (fann, create), - LUA_INTERFACE_DEF (fann, create_full), - LUA_INTERFACE_DEF (fann, load_file), - {"load", lua_fann_load_file}, - LUA_INTERFACE_DEF (fann, load_data), - {NULL, NULL} -}; - -static const struct luaL_reg fannlib_m[] = { - LUA_INTERFACE_DEF (fann, train), - LUA_INTERFACE_DEF (fann, train_threaded), - LUA_INTERFACE_DEF (fann, test), - LUA_INTERFACE_DEF (fann, save), - LUA_INTERFACE_DEF (fann, data), - LUA_INTERFACE_DEF (fann, get_inputs), - LUA_INTERFACE_DEF (fann, get_outputs), - LUA_INTERFACE_DEF (fann, get_layers), - LUA_INTERFACE_DEF (fann, get_mse), - {"__gc", lua_fann_dtor}, - {"__tostring", rspamd_lua_class_tostring}, - {NULL, NULL} -}; - -#ifdef WITH_FANN -struct fann * -rspamd_lua_check_fann (lua_State *L, gint pos) -{ - void *ud = rspamd_lua_check_udata (L, pos, "rspamd{fann}"); - luaL_argcheck (L, ud != NULL, pos, "'fann' expected"); - return ud ? *((struct fann **) ud) : NULL; -} -#endif - -/*** - * @function rspamd_fann.is_enabled() - * Checks if fann is enabled for this rspamd build - * @return {boolean} true if fann is enabled - */ -static gint -lua_fann_is_enabled (lua_State *L) -{ -#ifdef WITH_FANN - lua_pushboolean (L, true); -#else - lua_pushboolean (L, false); -#endif - return 1; -} - -/*** - * @function rspamd_fann.create(nlayers, [layer1, ... layern]) - * Creates new neural network with `nlayers` that contains `layer1`...`layern` - * neurons in each layer - * @param {number} nlayers number of layers - * @param {number} layerI number of neurons in each layer - * @return {fann} fann object - */ -static gint -lua_fann_create (lua_State *L) -{ -#ifndef WITH_FANN - return 0; -#else - struct fann *f, **pfann; - guint nlayers, *layers, i; - - nlayers = luaL_checknumber (L, 1); - - if (nlayers > 0) { - layers = g_malloc (nlayers * sizeof (layers[0])); - - if (lua_type (L, 2) == LUA_TNUMBER) { - for (i = 0; i < nlayers; i ++) { - layers[i] = luaL_checknumber (L, i + 2); - } - } - else if (lua_type (L, 2) == LUA_TTABLE) { - for (i = 0; i < nlayers; i ++) { - lua_rawgeti (L, 2, i + 1); - layers[i] = luaL_checknumber (L, -1); - lua_pop (L, 1); - } - } - - f = fann_create_standard_array (nlayers, layers); - fann_set_activation_function_hidden (f, FANN_SIGMOID_SYMMETRIC); - fann_set_activation_function_output (f, FANN_SIGMOID_SYMMETRIC); - fann_set_training_algorithm (f, FANN_TRAIN_INCREMENTAL); - fann_randomize_weights (f, 0, 1); - - if (f != NULL) { - pfann = lua_newuserdata (L, sizeof (gpointer)); - *pfann = f; - rspamd_lua_setclass (L, "rspamd{fann}", -1); - } - else { - lua_pushnil (L); - } - - g_free (layers); - } - else { - lua_pushnil (L); - } - - return 1; -#endif -} - -#ifdef WITH_FANN -static enum fann_activationfunc_enum -string_to_activation_func (const gchar *str) -{ - if (str == NULL) { - return FANN_SIGMOID_SYMMETRIC; - } - if (strcmp (str, "sigmoid") == 0) { - return FANN_SIGMOID; - } - else if (strcmp (str, "elliot") == 0) { - return FANN_ELLIOT; - } - else if (strcmp (str, "elliot_symmetric") == 0) { - return FANN_ELLIOT_SYMMETRIC; - } - else if (strcmp (str, "linear") == 0) { - return FANN_LINEAR; - } - - return FANN_SIGMOID_SYMMETRIC; -} - -static enum fann_train_enum -string_to_learn_alg (const gchar *str) -{ - if (str == NULL) { - return FANN_TRAIN_INCREMENTAL; - } - if (strcmp (str, "rprop") == 0) { - return FANN_TRAIN_RPROP; - } - else if (strcmp (str, "qprop") == 0) { - return FANN_TRAIN_QUICKPROP; - } - else if (strcmp (str, "batch") == 0) { - return FANN_TRAIN_BATCH; - } - - return FANN_TRAIN_INCREMENTAL; -} -/* - * This is needed since libfann provides no versioning macros... - */ -static struct fann_train_data * -rspamd_fann_create_train (guint num_data, guint num_input, guint num_output) -{ - struct fann_train_data *t; - fann_type *inp, *outp; - guint i; - - g_assert (num_data > 0 && num_input > 0 && num_output > 0); - - t = calloc (1, sizeof (*t)); - g_assert (t != NULL); - - t->num_data = num_data; - t->num_input = num_input; - t->num_output = num_output; - - t->input = calloc (num_data, sizeof (fann_type *)); - g_assert (t->input != NULL); - - t->output = calloc (num_data, sizeof (fann_type *)); - g_assert (t->output != NULL); - - inp = calloc (num_data * num_input, sizeof (fann_type)); - g_assert (inp != NULL); - - outp = calloc (num_data * num_output, sizeof (fann_type)); - g_assert (outp != NULL); - - for (i = 0; i < num_data; i ++) { - t->input[i] = inp; - inp += num_input; - t->output[i] = outp; - outp += num_output; - } - - return t; -} - - -#endif - -/*** - * @function rspamd_fann.create_full(params) - * Creates new neural network with parameters: - * - `layers` {table/numbers}: table of layers in form: {N1, N2, N3 ... Nn} where N is number of neurons in a layer - * - `activation_hidden` {string}: activation function type for hidden layers (`tanh` by default) - * - `activation_output` {string}: activation function type for output layer (`tanh` by default) - * - `sparsed` {float}: create sparsed ANN, where number is a coefficient for sparsing - * - `learn` {string}: learning algorithm (quickprop, rprop or incremental) - * - `randomize` {boolean}: randomize weights (true by default) - * @return {fann} fann object - */ -static gint -lua_fann_create_full (lua_State *L) -{ -#ifndef WITH_FANN - return 0; -#else - struct fann *f, **pfann; - guint nlayers, *layers, i; - const gchar *activation_hidden = NULL, *activation_output, *learn_alg = NULL; - gdouble sparsed = 0.0; - gboolean randomize_ann = TRUE; - GError *err = NULL; - - if (lua_type (L, 1) == LUA_TTABLE) { - lua_pushstring (L, "layers"); - lua_gettable (L, 1); - - if (lua_type (L, -1) != LUA_TTABLE) { - return luaL_error (L, "bad layers attribute"); - } - - nlayers = rspamd_lua_table_size (L, -1); - if (nlayers < 2) { - return luaL_error (L, "bad layers attribute"); - } - - layers = g_new0 (guint, nlayers); - - for (i = 0; i < nlayers; i ++) { - lua_rawgeti (L, -1, i + 1); - layers[i] = luaL_checknumber (L, -1); - lua_pop (L, 1); - } - - lua_pop (L, 1); /* Table */ - - if (!rspamd_lua_parse_table_arguments (L, 1, &err, - "sparsed=N;randomize=B;learn=S;activation_hidden=S;activation_output=S", - &sparsed, &randomize_ann, &learn_alg, &activation_hidden, &activation_output)) { - g_free (layers); - - if (err) { - gint r; - - r = luaL_error (L, "invalid arguments: %s", err->message); - g_error_free (err); - return r; - } - else { - return luaL_error (L, "invalid arguments"); - } - } - - if (sparsed != 0.0) { - f = fann_create_standard_array (nlayers, layers); - } - else { - f = fann_create_sparse_array (sparsed, nlayers, layers); - } - - if (f != NULL) { - pfann = lua_newuserdata (L, sizeof (gpointer)); - *pfann = f; - rspamd_lua_setclass (L, "rspamd{fann}", -1); - } - else { - g_free (layers); - return luaL_error (L, "cannot create fann"); - } - - fann_set_activation_function_hidden (f, - string_to_activation_func (activation_hidden)); - fann_set_activation_function_output (f, - string_to_activation_func (activation_output)); - fann_set_training_algorithm (f, string_to_learn_alg (learn_alg)); - - if (randomize_ann) { - fann_randomize_weights (f, 0, 1); - } - - g_free (layers); - } - else { - return luaL_error (L, "bad arguments"); - } - - return 1; -#endif -} - -/*** - * @function rspamd_fann.load(file) - * Loads neural network from the file - * @param {string} file filename where fann is stored - * @return {fann} fann object - */ -static gint -lua_fann_load_file (lua_State *L) -{ -#ifndef WITH_FANN - return 0; -#else - struct fann *f, **pfann; - const gchar *fname; - - fname = luaL_checkstring (L, 1); - - if (fname != NULL) { - f = fann_create_from_file (fname); - - if (f != NULL) { - pfann = lua_newuserdata (L, sizeof (gpointer)); - *pfann = f; - rspamd_lua_setclass (L, "rspamd{fann}", -1); - } - else { - lua_pushnil (L); - } - } - else { - lua_pushnil (L); - } - - return 1; -#endif -} - -/*** - * @function rspamd_fann.load_data(data) - * Loads neural network from the data - * @param {string} file filename where fann is stored - * @return {fann} fann object - */ -static gint -lua_fann_load_data (lua_State *L) -{ -#ifndef WITH_FANN - return 0; -#else - struct fann *f, **pfann; - gint fd; - struct rspamd_lua_text *t; - gchar fpath[PATH_MAX]; - - if (lua_type (L, 1) == LUA_TUSERDATA) { - t = lua_check_text (L, 1); - - if (!t) { - return luaL_error (L, "text required"); - } - } - else { - t = g_alloca (sizeof (*t)); - t->start = lua_tolstring (L, 1, (gsize *)&t->len); - t->flags = 0; - } - - /* We need to save data to file because of libfann stupidity */ - rspamd_strlcpy (fpath, "/tmp/rspamd-fannXXXXXXXXXX", sizeof (fpath)); - fd = mkstemp (fpath); - - if (fd == -1) { - msg_warn ("cannot create tempfile: %s", strerror (errno)); - lua_pushnil (L); - } - else { - if (write (fd, t->start, t->len) == -1) { - msg_warn ("cannot write tempfile: %s", strerror (errno)); - lua_pushnil (L); - unlink (fpath); - close (fd); - - return 1; - } - - f = fann_create_from_file (fpath); - unlink (fpath); - close (fd); - - if (f != NULL) { - pfann = lua_newuserdata (L, sizeof (gpointer)); - *pfann = f; - rspamd_lua_setclass (L, "rspamd{fann}", -1); - } - else { - lua_pushnil (L); - } - } - - return 1; -#endif -} - -/*** - * @function rspamd_fann:data() - * Returns serialized neural network - * @return {rspamd_text} fann data - */ -static gint -lua_fann_data (lua_State *L) -{ -#ifndef WITH_FANN - return 0; -#else - struct fann *f = rspamd_lua_check_fann (L, 1); - gint fd; - struct rspamd_lua_text *res; - gchar fpath[PATH_MAX]; - gpointer map; - gsize sz; - - if (f == NULL) { - return luaL_error (L, "invalid arguments"); - } - - /* We need to save data to file because of libfann stupidity */ - rspamd_strlcpy (fpath, "/tmp/rspamd-fannXXXXXXXXXX", sizeof (fpath)); - fd = mkstemp (fpath); - - if (fd == -1) { - msg_warn ("cannot create tempfile: %s", strerror (errno)); - lua_pushnil (L); - } - else { - if (fann_save (f, fpath) == -1) { - msg_warn ("cannot write tempfile: %s", strerror (errno)); - lua_pushnil (L); - unlink (fpath); - close (fd); - - return 1; - } - - - (void)lseek (fd, 0, SEEK_SET); - map = rspamd_file_xmap (fpath, PROT_READ, &sz, TRUE); - unlink (fpath); - close (fd); - - if (map != NULL) { - res = lua_newuserdata (L, sizeof (*res)); - res->len = sz; - res->start = map; - res->flags = RSPAMD_TEXT_FLAG_OWN|RSPAMD_TEXT_FLAG_MMAPED; - rspamd_lua_setclass (L, "rspamd{text}", -1); - } - else { - lua_pushnil (L); - } - - } - - return 1; -#endif -} - - -/** - * @method rspamd_fann:train(inputs, outputs) - * Trains neural network with samples. Inputs and outputs should be tables of - * equal size, each row in table should be N inputs and M outputs, e.g. - * {0, 1, 1} -> {0} - * @param {table} inputs input samples - * @param {table} outputs output samples - * @return {number} number of samples learned - */ -static gint -lua_fann_train (lua_State *L) -{ -#ifndef WITH_FANN - return 0; -#else - struct fann *f = rspamd_lua_check_fann (L, 1); - guint ninputs, noutputs, j; - fann_type *cur_input, *cur_output; - gboolean ret = FALSE; - - if (f != NULL) { - /* First check sanity, call for table.getn for that */ - ninputs = rspamd_lua_table_size (L, 2); - noutputs = rspamd_lua_table_size (L, 3); - - if (ninputs != fann_get_num_input (f) || - noutputs != fann_get_num_output (f)) { - msg_err ("bad number of inputs(%d, expected %d) and " - "output(%d, expected %d) args for train", - ninputs, fann_get_num_input (f), - noutputs, fann_get_num_output (f)); - } - else { - cur_input = g_malloc (ninputs * sizeof (fann_type)); - - for (j = 0; j < ninputs; j ++) { - lua_rawgeti (L, 2, j + 1); - cur_input[j] = lua_tonumber (L, -1); - lua_pop (L, 1); - } - - cur_output = g_malloc (noutputs * sizeof (fann_type)); - - for (j = 0; j < noutputs; j++) { - lua_rawgeti (L, 3, j + 1); - cur_output[j] = lua_tonumber (L, -1); - lua_pop (L, 1); - } - - fann_train (f, cur_input, cur_output); - g_free (cur_input); - g_free (cur_output); - - ret = TRUE; - } - } - - lua_pushboolean (L, ret); - - return 1; -#endif -} - -#ifdef WITH_FANN -struct lua_fann_train_cbdata { - lua_State *L; - gint pair[2]; - struct fann_train_data *train; - struct fann *f; - gint cbref; - gdouble desired_mse; - guint max_epochs; - GThread *t; - struct event io; -}; - -struct lua_fann_train_reply { - gint errcode; - float mse; - gchar errmsg[128]; -}; - -static void -lua_fann_push_train_result (struct lua_fann_train_cbdata *cbdata, - gint errcode, float mse, const gchar *errmsg) -{ - lua_rawgeti (cbdata->L, LUA_REGISTRYINDEX, cbdata->cbref); - lua_pushnumber (cbdata->L, errcode); - lua_pushstring (cbdata->L, errmsg); - lua_pushnumber (cbdata->L, mse); - - if (lua_pcall (cbdata->L, 3, 0, 0) != 0) { - msg_err ("call to train callback failed: %s", lua_tostring (cbdata->L, -1)); - lua_pop (cbdata->L, 1); - } -} - -static void -lua_fann_thread_notify (gint fd, short what, gpointer ud) -{ - struct lua_fann_train_cbdata *cbdata = ud; - struct lua_fann_train_reply rep; - - if (read (cbdata->pair[0], &rep, sizeof (rep)) == -1) { - if (errno == EAGAIN || errno == EINTR) { - event_add (&cbdata->io, NULL); - return; - } - - lua_fann_push_train_result (cbdata, errno, 0.0, strerror (errno)); - } - else { - lua_fann_push_train_result (cbdata, rep.errcode, rep.mse, rep.errmsg); - } - - g_assert (write (cbdata->pair[0], "", 1) == 1); - g_thread_join (cbdata->t); - close (cbdata->pair[0]); - close (cbdata->pair[1]); - - fann_destroy_train (cbdata->train); - luaL_unref (cbdata->L, LUA_REGISTRYINDEX, cbdata->cbref); - g_free (cbdata); -} - -static void * -lua_fann_train_thread (void *ud) -{ - struct lua_fann_train_cbdata *cbdata = ud; - struct lua_fann_train_reply rep; - gchar repbuf[1]; - - msg_info ("start learning ANN, %d epochs are possible", - cbdata->max_epochs); - rspamd_socket_blocking (cbdata->pair[1]); - fann_train_on_data (cbdata->f, cbdata->train, cbdata->max_epochs, 0, - cbdata->desired_mse); - rep.errcode = 0; - rspamd_strlcpy (rep.errmsg, "OK", sizeof (rep.errmsg)); - rep.mse = fann_get_MSE (cbdata->f); - - if (write (cbdata->pair[1], &rep, sizeof (rep)) == -1) { - msg_err ("cannot write to socketpair: %s", strerror (errno)); - - return NULL; - } - - if (read (cbdata->pair[1], repbuf, sizeof (repbuf)) == -1) { - msg_err ("cannot read from socketpair: %s", strerror (errno)); - - return NULL; - } - - return NULL; -} -#endif -/** - * @method rspamd_fann:train_threaded(inputs, outputs, callback, event_base, {params}) - * Trains neural network with batch of samples. Inputs and outputs should be tables of - * equal size, each row in table should be N inputs and M outputs, e.g. - * {{0, 1, 1}, ...} -> {{0}, {1} ...} - * @param {table} inputs input samples - * @param {table} outputs output samples - * @param {callback} function that is called when train is completed - */ -static gint -lua_fann_train_threaded (lua_State *L) -{ -#ifndef WITH_FANN - return 0; -#else - struct fann *f = rspamd_lua_check_fann (L, 1); - guint ninputs, noutputs, ndata, i, j; - struct lua_fann_train_cbdata *cbdata; - struct event_base *ev_base = lua_check_ev_base (L, 5); - GError *err = NULL; - const guint max_epochs_default = 1000; - const gdouble desired_mse_default = 0.0001; - - if (f != NULL && lua_type (L, 2) == LUA_TTABLE && - lua_type (L, 3) == LUA_TTABLE && lua_type (L, 4) == LUA_TFUNCTION && - ev_base != NULL) { - /* First check sanity, call for table.getn for that */ - ndata = rspamd_lua_table_size (L, 2); - ninputs = fann_get_num_input (f); - noutputs = fann_get_num_output (f); - cbdata = g_malloc0 (sizeof (*cbdata)); - cbdata->L = L; - cbdata->f = f; - cbdata->train = rspamd_fann_create_train (ndata, ninputs, noutputs); - lua_pushvalue (L, 4); - cbdata->cbref = luaL_ref (L, LUA_REGISTRYINDEX); - - if (rspamd_socketpair (cbdata->pair, 0) == -1) { - msg_err ("cannot open socketpair: %s", strerror (errno)); - cbdata->pair[0] = -1; - cbdata->pair[1] = -1; - goto err; - } - - for (i = 0; i < ndata; i ++) { - lua_rawgeti (L, 2, i + 1); - - if (rspamd_lua_table_size (L, -1) != ninputs) { - msg_err ("invalid number of inputs: %d, %d expected", - rspamd_lua_table_size (L, -1), ninputs); - goto err; - } - - for (j = 0; j < ninputs; j ++) { - lua_rawgeti (L, -1, j + 1); - cbdata->train->input[i][j] = lua_tonumber (L, -1); - lua_pop (L, 1); - } - - lua_pop (L, 1); - lua_rawgeti (L, 3, i + 1); - - if (rspamd_lua_table_size (L, -1) != noutputs) { - msg_err ("invalid number of outputs: %d, %d expected", - rspamd_lua_table_size (L, -1), noutputs); - goto err; - } - - for (j = 0; j < noutputs; j++) { - lua_rawgeti (L, -1, j + 1); - cbdata->train->output[i][j] = lua_tonumber (L, -1); - lua_pop (L, 1); - } - } - - cbdata->max_epochs = max_epochs_default; - cbdata->desired_mse = desired_mse_default; - - if (lua_type (L, 5) == LUA_TTABLE) { - rspamd_lua_parse_table_arguments (L, 5, NULL, - "max_epochs=I;desired_mse=N", - &cbdata->max_epochs, &cbdata->desired_mse); - } - - /* Now we can call training in a separate thread */ - rspamd_socket_nonblocking (cbdata->pair[0]); - event_set (&cbdata->io, cbdata->pair[0], EV_READ, lua_fann_thread_notify, - cbdata); - event_base_set (ev_base, &cbdata->io); - /* TODO: add timeout */ - event_add (&cbdata->io, NULL); - cbdata->t = rspamd_create_thread ("fann train", lua_fann_train_thread, - cbdata, &err); - - if (cbdata->t == NULL) { - msg_err ("cannot create training thread: %e", err); - - if (err) { - g_error_free (err); - } - - goto err; - } - } - else { - return luaL_error (L, "invalid arguments"); - } - - return 0; - -err: - if (cbdata->pair[0] != -1) { - close (cbdata->pair[0]); - } - if (cbdata->pair[1] != -1) { - close (cbdata->pair[1]); - } - - fann_destroy_train (cbdata->train); - luaL_unref (L, LUA_REGISTRYINDEX, cbdata->cbref); - g_free (cbdata); - return luaL_error (L, "invalid arguments"); -#endif -} - -/** - * @method rspamd_fann:test(inputs) - * Tests neural network with samples. Inputs is a single sample of input data. - * The function returns table of results, e.g.: - * {0, 1, 1} -> {0} - * @param {table} inputs input sample - * @return {table/number} outputs values - */ -static gint -lua_fann_test (lua_State *L) -{ -#ifndef WITH_FANN - return 0; -#else - struct fann *f = rspamd_lua_check_fann (L, 1); - guint ninputs, noutputs, i, tbl_idx = 2; - fann_type *cur_input, *cur_output; - - if (f != NULL) { - /* First check sanity, call for table.getn for that */ - if (lua_isnumber (L, 2)) { - ninputs = lua_tonumber (L, 2); - tbl_idx = 3; - } - else { - ninputs = rspamd_lua_table_size (L, 2); - - if (ninputs == 0) { - msg_err ("empty inputs number"); - lua_pushnil (L); - - return 1; - } - } - - cur_input = g_malloc0 (ninputs * sizeof (fann_type)); - - for (i = 0; i < ninputs; i++) { - lua_rawgeti (L, tbl_idx, i + 1); - cur_input[i] = lua_tonumber (L, -1); - lua_pop (L, 1); - } - - cur_output = fann_run (f, cur_input); - noutputs = fann_get_num_output (f); - lua_createtable (L, noutputs, 0); - - for (i = 0; i < noutputs; i ++) { - lua_pushnumber (L, cur_output[i]); - lua_rawseti (L, -2, i + 1); - } - - g_free (cur_input); - } - else { - lua_pushnil (L); - } - - return 1; -#endif -} - -/*** - * @method rspamd_fann:get_inputs() - * Returns number of inputs for neural network - * @return {number} number of inputs - */ -static gint -lua_fann_get_inputs (lua_State *L) -{ -#ifndef WITH_FANN - return 0; -#else - struct fann *f = rspamd_lua_check_fann (L, 1); - - if (f != NULL) { - lua_pushnumber (L, fann_get_num_input (f)); - } - else { - lua_pushnil (L); - } - - return 1; -#endif -} - -/*** - * @method rspamd_fann:get_outputs() - * Returns number of outputs for neural network - * @return {number} number of outputs - */ -static gint -lua_fann_get_outputs (lua_State *L) -{ -#ifndef WITH_FANN - return 0; -#else - struct fann *f = rspamd_lua_check_fann (L, 1); - - if (f != NULL) { - lua_pushnumber (L, fann_get_num_output (f)); - } - else { - lua_pushnil (L); - } - - return 1; -#endif -} - -/*** - * @method rspamd_fann:get_mse() - * Returns mean square error for ANN - * @return {number} MSE value - */ -static gint -lua_fann_get_mse (lua_State *L) -{ -#ifndef WITH_FANN - return 0; -#else - struct fann *f = rspamd_lua_check_fann (L, 1); - - if (f != NULL) { - lua_pushnumber (L, fann_get_MSE (f)); - } - else { - lua_pushnil (L); - } - - return 1; -#endif -} - -/*** - * @method rspamd_fann:get_layers() - * Returns array of neurons count for each layer - * @return {table/number} table with number ofr neurons in each layer - */ -static gint -lua_fann_get_layers (lua_State *L) -{ -#ifndef WITH_FANN - return 0; -#else - struct fann *f = rspamd_lua_check_fann (L, 1); - guint nlayers, i, *layers; - - if (f != NULL) { - nlayers = fann_get_num_layers (f); - layers = g_new (guint, nlayers); - fann_get_layer_array (f, layers); - lua_createtable (L, nlayers, 0); - - for (i = 0; i < nlayers; i ++) { - lua_pushnumber (L, layers[i]); - lua_rawseti (L, -2, i + 1); - } - - g_free (layers); - } - else { - lua_pushnil (L); - } - - return 1; -#endif -} - -/*** - * @method rspamd_fann:save(fname) - * Save fann to file named 'fname' - * @param {string} fname filename to save fann into - * @return {boolean} true if ann has been saved - */ -static gint -lua_fann_save (lua_State *L) -{ -#ifndef WITH_FANN - return 0; -#else - struct fann *f = rspamd_lua_check_fann (L, 1); - const gchar *fname = luaL_checkstring (L, 2); - - if (f != NULL && fname != NULL) { - if (fann_save (f, fname) == 0) { - lua_pushboolean (L, true); - } - else { - msg_err ("cannot save ANN to %s: %s", fname, strerror (errno)); - lua_pushboolean (L, false); - } - } - else { - lua_pushnil (L); - } - - return 1; -#endif -} - -static gint -lua_fann_dtor (lua_State *L) -{ -#ifndef WITH_FANN - return 0; -#else - struct fann *f = rspamd_lua_check_fann (L, 1); - - if (f) { - fann_destroy (f); - } - - return 0; -#endif -} - -static gint -lua_load_fann (lua_State * L) -{ - lua_newtable (L); - luaL_register (L, NULL, fannlib_f); - - return 1; -} - -void -luaopen_fann (lua_State * L) -{ - rspamd_lua_new_class (L, "rspamd{fann}", fannlib_m); - lua_pop (L, 1); - - rspamd_lua_add_preload (L, "rspamd_fann", lua_load_fann); -} diff --git a/src/lua/lua_http.c b/src/lua/lua_http.c index 46d5ed412..ec42ab39e 100644 --- a/src/lua/lua_http.c +++ b/src/lua/lua_http.c @@ -67,10 +67,10 @@ struct lua_http_cbdata { struct rspamd_async_session *session; struct rspamd_symcache_item *item; struct rspamd_http_message *msg; - struct event_base *ev_base; + struct ev_loop *event_loop; struct rspamd_config *cfg; struct rspamd_task *task; - struct timeval tv; + ev_tstamp timeout; struct rspamd_cryptobox_keypair *local_kp; struct rspamd_cryptobox_pubkey *peer_pk; rspamd_inet_addr_t *addr; @@ -86,10 +86,10 @@ struct lua_http_cbdata { ref_entry_t ref; }; -static const int default_http_timeout = 5000; +static const gdouble default_http_timeout = 5.0; static struct rspamd_dns_resolver * -lua_http_global_resolver (struct event_base *ev_base) +lua_http_global_resolver (struct ev_loop *ev_base) { static struct rspamd_dns_resolver *global_resolver; @@ -451,7 +451,7 @@ lua_http_make_connection (struct lua_http_cbdata *cbd) rspamd_http_connection_write_message (cbd->conn, msg, cbd->host, cbd->mime_type, cbd, - &cbd->tv); + cbd->timeout); return TRUE; } @@ -562,7 +562,7 @@ static gint lua_http_request (lua_State *L) { LUA_TRACE_POINT; - struct event_base *ev_base; + struct ev_loop *ev_base; struct rspamd_http_message *msg; struct lua_http_cbdata *cbd; struct rspamd_dns_resolver *resolver; @@ -597,7 +597,7 @@ lua_http_request (lua_State *L) cbref = luaL_ref (L, LUA_REGISTRYINDEX); if (lua_gettop (L) >= 3 && rspamd_lua_check_udata_maybe (L, 3, "rspamd{ev_base}")) { - ev_base = *(struct event_base **)lua_touserdata (L, 3); + ev_base = *(struct ev_loop **)lua_touserdata (L, 3); } else { ev_base = NULL; @@ -643,7 +643,7 @@ lua_http_request (lua_State *L) if (lua_type (L, -1) == LUA_TUSERDATA) { task = lua_check_task (L, -1); - ev_base = task->ev_base; + ev_base = task->event_loop; resolver = task->resolver; session = task->s; cfg = task->cfg; @@ -654,7 +654,7 @@ lua_http_request (lua_State *L) lua_pushstring (L, "ev_base"); lua_gettable (L, 1); if (rspamd_lua_check_udata_maybe (L, -1, "rspamd{ev_base}")) { - ev_base = *(struct event_base **)lua_touserdata (L, -1); + ev_base = *(struct ev_loop **)lua_touserdata (L, -1); } else { ev_base = NULL; @@ -717,7 +717,7 @@ lua_http_request (lua_State *L) lua_pushstring (L, "timeout"); lua_gettable (L, 1); if (lua_type (L, -1) == LUA_TNUMBER) { - timeout = lua_tonumber (L, -1) * 1000.; + timeout = lua_tonumber (L, -1); } lua_pop (L, 1); @@ -860,7 +860,7 @@ lua_http_request (lua_State *L) lua_gettable (L, 1); if (lua_type (L, -1) == LUA_TNUMBER) { - max_size = lua_tonumber (L, -1); + max_size = lua_tointeger (L, -1); } lua_pop (L, 1); @@ -943,9 +943,9 @@ lua_http_request (lua_State *L) cbd = g_malloc0 (sizeof (*cbd)); cbd->cbref = cbref; cbd->msg = msg; - cbd->ev_base = ev_base; + cbd->event_loop = ev_base; cbd->mime_type = mime_type; - msec_to_tv (timeout, &cbd->tv); + cbd->timeout = timeout; cbd->fd = -1; cbd->cfg = cfg; cbd->peer_pk = peer_key; diff --git a/src/lua/lua_redis.c b/src/lua/lua_redis.c index 4335a3467..f39168a27 100644 --- a/src/lua/lua_redis.c +++ b/src/lua/lua_redis.c @@ -98,7 +98,7 @@ struct lua_redis_userdata { struct rspamd_task *task; struct rspamd_symcache_item *item; struct rspamd_async_session *s; - struct event_base *ev_base; + struct ev_loop *event_loop; struct rspamd_config *cfg; struct rspamd_redis_pool *pool; gchar *server; @@ -124,7 +124,7 @@ struct lua_redis_request_specific_userdata { struct lua_redis_userdata *c; struct lua_redis_ctx *ctx; struct lua_redis_request_specific_userdata *next; - struct event timeout; + ev_timer timeout_ev; guint flags; }; @@ -184,9 +184,7 @@ lua_redis_dtor (struct lua_redis_ctx *ctx) if (ud->ctx) { LL_FOREACH_SAFE (ud->specific, cur, tmp) { - if (rspamd_event_pending (&cur->timeout, EV_TIMEOUT)) { - event_del (&cur->timeout); - } + ev_timer_stop (ud->event_loop, &cur->timeout_ev); if (!(cur->flags & LUA_REDIS_SPECIFIC_REPLIED)) { is_successful = FALSE; @@ -245,9 +243,7 @@ lua_redis_fin (void *arg) ctx = sp_ud->ctx; - if (rspamd_event_pending (&sp_ud->timeout, EV_TIMEOUT)) { - event_del (&sp_ud->timeout); - } + ev_timer_stop (sp_ud->ctx->async.event_loop, &sp_ud->timeout_ev); msg_debug ("finished redis query %p from session %p", sp_ud, ctx); sp_ud->flags |= LUA_REDIS_SPECIFIC_FINISHED; @@ -556,10 +552,7 @@ lua_redis_callback_sync (redisAsyncContext *ac, gpointer r, gpointer priv) return; } - if (rspamd_event_pending (&sp_ud->timeout, EV_TIMEOUT)) { - event_del (&sp_ud->timeout); - } - + ev_timer_stop (ud->event_loop, &sp_ud->timeout_ev); msg_debug ("got reply from redis: %p for query %p", ac, sp_ud); struct lua_redis_result *result = g_malloc0 (sizeof *result); @@ -630,9 +623,10 @@ lua_redis_callback_sync (redisAsyncContext *ac, gpointer r, gpointer priv) } static void -lua_redis_timeout_sync (int fd, short what, gpointer priv) +lua_redis_timeout_sync (EV_P_ ev_timer *w, int revents) { - struct lua_redis_request_specific_userdata *sp_ud = priv; + struct lua_redis_request_specific_userdata *sp_ud = + (struct lua_redis_request_specific_userdata *)w->data; struct lua_redis_ctx *ctx = sp_ud->ctx; redisAsyncContext *ac; @@ -657,9 +651,10 @@ lua_redis_timeout_sync (int fd, short what, gpointer priv) } static void -lua_redis_timeout (int fd, short what, gpointer u) +lua_redis_timeout (EV_P_ ev_timer *w, int revents) { - struct lua_redis_request_specific_userdata *sp_ud = u; + struct lua_redis_request_specific_userdata *sp_ud = + (struct lua_redis_request_specific_userdata *)w->data; struct lua_redis_ctx *ctx; redisAsyncContext *ac; @@ -790,9 +785,9 @@ lua_redis_parse_args (lua_State *L, gint idx, const gchar *cmd, static struct lua_redis_ctx * rspamd_lua_redis_prepare_connection (lua_State *L, gint *pcbref, gboolean is_async) { - struct lua_redis_ctx *ctx; + struct lua_redis_ctx *ctx = NULL; rspamd_inet_addr_t *ip = NULL; - struct lua_redis_userdata *ud; + struct lua_redis_userdata *ud = NULL; struct rspamd_lua_ip *addr = NULL; struct rspamd_task *task = NULL; const gchar *host; @@ -800,7 +795,7 @@ rspamd_lua_redis_prepare_connection (lua_State *L, gint *pcbref, gboolean is_asy gint cbref = -1; struct rspamd_config *cfg = NULL; struct rspamd_async_session *session = NULL; - struct event_base *ev_base = NULL; + struct ev_loop *ev_base = NULL; gboolean ret = FALSE; guint flags = 0; @@ -850,7 +845,7 @@ rspamd_lua_redis_prepare_connection (lua_State *L, gint *pcbref, gboolean is_asy else { cfg = task->cfg; session = task->s; - ev_base = task->ev_base; + ev_base = task->event_loop; ret = TRUE; } @@ -933,7 +928,7 @@ rspamd_lua_redis_prepare_connection (lua_State *L, gint *pcbref, gboolean is_asy ud->s = session; ud->cfg = cfg; ud->pool = cfg->redis_pool; - ud->ev_base = ev_base; + ud->event_loop = ev_base; ud->task = task; if (task) { @@ -1009,7 +1004,6 @@ lua_redis_make_request (lua_State *L) struct lua_redis_userdata *ud; struct lua_redis_ctx *ctx, **pctx; const gchar *cmd = NULL; - struct timeval tv; gdouble timeout = REDIS_DEFAULT_TIMEOUT; gint cbref = -1; gboolean ret = FALSE; @@ -1064,10 +1058,9 @@ lua_redis_make_request (lua_State *L) REDIS_RETAIN (ctx); /* Cleared by fin event */ ctx->cmds_pending ++; - double_to_tv (timeout, &tv); - event_set (&sp_ud->timeout, -1, EV_TIMEOUT, lua_redis_timeout, sp_ud); - event_base_set (ud->ev_base, &sp_ud->timeout); - event_add (&sp_ud->timeout, &tv); + sp_ud->timeout_ev.data = sp_ud; + ev_timer_init (&sp_ud->timeout_ev, lua_redis_timeout, timeout, 0.0); + ev_timer_start (ud->event_loop, &sp_ud->timeout_ev); ret = TRUE; } else { @@ -1347,7 +1340,6 @@ lua_redis_add_cmd (lua_State *L) const gchar *cmd = NULL; gint args_pos = 2; gint cbref = -1, ret; - struct timeval tv; if (ctx) { if (ctx->flags & LUA_REDIS_TERMINATED) { @@ -1426,19 +1418,18 @@ lua_redis_add_cmd (lua_State *L) } } - double_to_tv (sp_ud->c->timeout, &tv); + sp_ud->timeout_ev.data = sp_ud; if (IS_ASYNC (ctx)) { - event_set (&sp_ud->timeout, -1, EV_TIMEOUT, - lua_redis_timeout, sp_ud); + ev_timer_init (&sp_ud->timeout_ev, lua_redis_timeout, + sp_ud->c->timeout, 0.0); } else { - event_set (&sp_ud->timeout, -1, EV_TIMEOUT, - lua_redis_timeout_sync, sp_ud); + ev_timer_init (&sp_ud->timeout_ev, lua_redis_timeout_sync, + sp_ud->c->timeout, 0.0); } - event_base_set (ud->ev_base, &sp_ud->timeout); - event_add (&sp_ud->timeout, &tv); + ev_timer_start (ud->event_loop, &sp_ud->timeout_ev); REDIS_RETAIN (ctx); ctx->cmds_pending ++; } diff --git a/src/lua/lua_task.c b/src/lua/lua_task.c index d1abe6a2b..0ffe4b8c5 100644 --- a/src/lua/lua_task.c +++ b/src/lua/lua_task.c @@ -1523,7 +1523,7 @@ lua_task_create (lua_State * L) LUA_TRACE_POINT; struct rspamd_task *task = NULL, **ptask; struct rspamd_config *cfg = NULL; - struct event_base *ev_base = NULL; + struct ev_loop *ev_base = NULL; if (lua_type (L, 1) == LUA_TUSERDATA) { gpointer p; @@ -1539,7 +1539,7 @@ lua_task_create (lua_State * L) p = rspamd_lua_check_udata_maybe (L, 2, "rspamd{ev_base}"); if (p) { - ev_base = *(struct event_base **)p; + ev_base = *(struct ev_loop **)p; } } @@ -1610,13 +1610,13 @@ static int lua_task_get_ev_base (lua_State * L) { LUA_TRACE_POINT; - struct event_base **pbase; + struct ev_loop **pbase; struct rspamd_task *task = lua_check_task (L, 1); if (task != NULL) { - pbase = lua_newuserdata (L, sizeof (struct event_base *)); + pbase = lua_newuserdata (L, sizeof (struct ev_loop *)); rspamd_lua_setclass (L, "rspamd{ev_base}", -1); - *pbase = task->ev_base; + *pbase = task->event_loop; } else { return luaL_error (L, "invalid arguments"); @@ -4309,7 +4309,7 @@ lua_task_get_date (lua_State *L) } /* Get GMT date and store it to time_t */ if (type == DATE_CONNECT || type == DATE_CONNECT_STRING) { - tim = (tv_to_msec (&task->tv)) / 1000.; + tim = task->task_timestamp; if (!gmt) { struct tm t; @@ -4399,14 +4399,16 @@ lua_task_get_timeval (lua_State *L) { LUA_TRACE_POINT; struct rspamd_task *task = lua_check_task (L, 1); + struct timeval tv; if (task != NULL) { + double_to_tv (task->task_timestamp, &tv); lua_createtable (L, 0, 2); lua_pushstring (L, "tv_sec"); - lua_pushinteger (L, (lua_Integer)task->tv.tv_sec); + lua_pushinteger (L, (lua_Integer)tv.tv_sec); lua_settable (L, -3); lua_pushstring (L, "tv_usec"); - lua_pushinteger (L, (lua_Integer)task->tv.tv_usec); + lua_pushinteger (L, (lua_Integer)tv.tv_usec); lua_settable (L, -3); } else { @@ -4429,7 +4431,7 @@ lua_task_get_scan_time (lua_State *L) } rspamd_task_set_finish_time (task); - lua_pushnumber (L, task->time_real_finish - task->time_real); + lua_pushnumber (L, task->time_real_finish - task->task_timestamp); lua_pushnumber (L, task->time_virtual_finish - task->time_virtual); if (!set) { diff --git a/src/lua/lua_tcp.c b/src/lua/lua_tcp.c index d0def9ac2..e2f55e78b 100644 --- a/src/lua/lua_tcp.c +++ b/src/lua/lua_tcp.c @@ -64,7 +64,7 @@ rspamd_config:register_symbol({ end local function from_done_cb(err, data, conn) logger.errx(task, 'FROM: got reply: %s, error: %s', data, err) - conn:add_write(rcpt_cb, 'RCPT TO: <hui@yandex.ru>\r\n') + conn:add_write(rcpt_cb, 'RCPT TO: <test@yandex.ru>\r\n') end local function from_cb(err, conn) logger.errx(task, 'written from, error: %s', err) @@ -159,13 +159,6 @@ LUA_FUNCTION_DEF (tcp, connect_sync); * Closes TCP connection */ LUA_FUNCTION_DEF (tcp, close); -/*** - * @method tcp:set_timeout(seconds) - * - * Sets new timeout for a TCP connection in **seconds** - * @param {number} seconds floating point value that specifies new timeout - */ -LUA_FUNCTION_DEF (tcp, set_timeout); /*** * @method tcp:add_read(callback, [pattern]) @@ -210,7 +203,6 @@ static const struct luaL_reg tcp_libf[] = { static const struct luaL_reg tcp_libm[] = { LUA_INTERFACE_DEF (tcp, close), - LUA_INTERFACE_DEF (tcp, set_timeout), LUA_INTERFACE_DEF (tcp, add_read), LUA_INTERFACE_DEF (tcp, add_write), LUA_INTERFACE_DEF (tcp, shift_callback), @@ -227,13 +219,6 @@ static const struct luaL_reg tcp_libm[] = { LUA_FUNCTION_DEF (tcp_sync, close); /*** - * @method set_timeout(seconds) - * - * Sets timeout for IO operations - */ -LUA_FUNCTION_DEF (tcp_sync, set_timeout); - -/*** * @method read_once() * * Performs one read operation. If syscall returned with EAGAIN/EINT, @@ -270,7 +255,6 @@ static void lua_tcp_sync_session_dtor (gpointer ud); static const struct luaL_reg tcp_sync_libm[] = { LUA_INTERFACE_DEF (tcp_sync, close), - LUA_INTERFACE_DEF (tcp_sync, set_timeout), LUA_INTERFACE_DEF (tcp_sync, read_once), LUA_INTERFACE_DEF (tcp_sync, write), LUA_INTERFACE_DEF (tcp_sync, eof), @@ -297,7 +281,7 @@ struct lua_tcp_write_handler { enum lua_tcp_handler_type { LUA_WANT_WRITE = 0, LUA_WANT_READ, - LUA_WANT_CONNECT // used only with sync connections + LUA_WANT_CONNECT }; struct lua_tcp_handler { @@ -342,8 +326,7 @@ struct lua_tcp_dtor { struct lua_tcp_cbdata { struct rspamd_async_session *session; struct rspamd_async_event *async_ev; - struct event_base *ev_base; - struct timeval tv; + struct ev_loop *event_loop; rspamd_inet_addr_t *addr; GByteArray *in; GQueue *handlers; @@ -352,7 +335,7 @@ struct lua_tcp_cbdata { guint port; guint flags; gchar tag[7]; - struct event ev; + struct rspamd_io_ev ev; struct lua_tcp_dtor *dtors; ref_entry_t ref; struct rspamd_task *task; @@ -381,10 +364,10 @@ static void lua_tcp_unregister_event (struct lua_tcp_cbdata *cbd); static void lua_tcp_void_finalyser (gpointer arg) {} -static const int default_tcp_timeout = 5000; +static const gdouble default_tcp_timeout = 5.0; static struct rspamd_dns_resolver * -lua_tcp_global_resolver (struct event_base *ev_base, +lua_tcp_global_resolver (struct ev_loop *ev_base, struct rspamd_config *cfg) { static struct rspamd_dns_resolver *global_resolver; @@ -467,7 +450,7 @@ lua_tcp_fin (gpointer arg) } if (cbd->fd != -1) { - event_del (&cbd->ev); + rspamd_ev_watcher_stop (cbd->event_loop, &cbd->ev); close (cbd->fd); cbd->fd = -1; } @@ -755,15 +738,7 @@ lua_tcp_resume_thread (struct lua_tcp_cbdata *cbd, const guint8 *str, gsize len) static void lua_tcp_plan_read (struct lua_tcp_cbdata *cbd) { - event_del (&cbd->ev); -#ifdef EV_CLOSED - event_set (&cbd->ev, cbd->fd, EV_READ|EV_CLOSED, - lua_tcp_handler, cbd); -#else - event_set (&cbd->ev, cbd->fd, EV_READ, lua_tcp_handler, cbd); -#endif - event_base_set (cbd->ev_base, &cbd->ev); - event_add (&cbd->ev, &cbd->tv); + rspamd_ev_watcher_reschedule (cbd->event_loop, &cbd->ev, EV_READ); } static void @@ -867,7 +842,6 @@ lua_tcp_write_helper (struct lua_tcp_cbdata *cbd) } else { /* Want to write more */ - event_add (&cbd->ev, &cbd->tv); } return; @@ -1033,6 +1007,8 @@ lua_tcp_handler (int fd, short what, gpointer ud) struct lua_tcp_handler *rh = g_queue_peek_head (cbd->handlers); event_type = rh->type; + rspamd_ev_watcher_stop (cbd->event_loop, &cbd->ev); + if (what == EV_READ) { if (cbd->ssl_conn) { r = rspamd_ssl_read (cbd->ssl_conn, inbuf, sizeof (inbuf)); @@ -1100,12 +1076,6 @@ lua_tcp_handler (int fd, short what, gpointer ud) g_assert_not_reached (); } } -#ifdef EV_CLOSED - else if (what == EV_CLOSED) { - lua_tcp_push_error (cbd, TRUE, "Remote peer has closed the connection"); - TCP_RELEASE (cbd); - } -#endif else { lua_tcp_push_error (cbd, TRUE, "IO timeout"); TCP_RELEASE (cbd); @@ -1146,17 +1116,17 @@ lua_tcp_plan_handler_event (struct lua_tcp_cbdata *cbd, gboolean can_read, } } else { - msg_debug_tcp ("plan new read"); if (can_read) { /* We need to plan a new event */ - event_set (&cbd->ev, cbd->fd, EV_READ, lua_tcp_handler, cbd); - event_base_set (cbd->ev_base, &cbd->ev); - event_add (&cbd->ev, &cbd->tv); + msg_debug_tcp ("plan new read"); + rspamd_ev_watcher_reschedule (cbd->event_loop, &cbd->ev, + EV_READ); } else { /* Cannot read more */ + msg_debug_tcp ("cannot read more"); lua_tcp_push_error (cbd, FALSE, "EOF, cannot read more data"); - if (!IS_SYNC(cbd)) { + if (!IS_SYNC (cbd)) { lua_tcp_shift_handler (cbd); lua_tcp_plan_handler_event (cbd, can_read, can_write); } @@ -1172,9 +1142,8 @@ lua_tcp_plan_handler_event (struct lua_tcp_cbdata *cbd, gboolean can_read, if (hdl->h.w.pos < hdl->h.w.total_bytes) { msg_debug_tcp ("plan new write"); if (can_write) { - event_set (&cbd->ev, cbd->fd, EV_WRITE, lua_tcp_handler, cbd); - event_base_set (cbd->ev_base, &cbd->ev); - event_add (&cbd->ev, &cbd->tv); + rspamd_ev_watcher_reschedule (cbd->event_loop, &cbd->ev, + EV_WRITE); } else { /* Cannot write more */ @@ -1192,9 +1161,8 @@ lua_tcp_plan_handler_event (struct lua_tcp_cbdata *cbd, gboolean can_read, } else { /* LUA_WANT_CONNECT */ msg_debug_tcp ("plan new connect"); - event_set (&cbd->ev, cbd->fd, EV_WRITE, lua_tcp_handler, cbd); - event_base_set (cbd->ev_base, &cbd->ev); - event_add (&cbd->ev, &cbd->tv); + rspamd_ev_watcher_reschedule (cbd->event_loop, &cbd->ev, + EV_WRITE); } } } @@ -1289,12 +1257,11 @@ lua_tcp_make_connection (struct lua_tcp_cbdata *cbd) verify_peer = TRUE; } - event_base_set (cbd->ev_base, &cbd->ev); cbd->ssl_conn = - rspamd_ssl_connection_new (ssl_ctx, cbd->ev_base, verify_peer); + rspamd_ssl_connection_new (ssl_ctx, cbd->event_loop, verify_peer); if (!rspamd_ssl_connect_fd (cbd->ssl_conn, fd, cbd->hostname, &cbd->ev, - &cbd->tv, lua_tcp_handler, lua_tcp_ssl_on_error, cbd)) { + cbd->ev.timeout, lua_tcp_handler, lua_tcp_ssl_on_error, cbd)) { lua_tcp_push_error (cbd, TRUE, "ssl connection failed: %s", strerror (errno)); @@ -1305,6 +1272,8 @@ lua_tcp_make_connection (struct lua_tcp_cbdata *cbd) } } else { + rspamd_ev_watcher_init (&cbd->ev, cbd->fd, EV_WRITE, + lua_tcp_handler, cbd); lua_tcp_register_event (cbd); lua_tcp_plan_handler_event (cbd, TRUE, TRUE); } @@ -1431,7 +1400,7 @@ lua_tcp_request (lua_State *L) guint port; gint cbref, tp, conn_cbref = -1; gsize plen = 0; - struct event_base *ev_base; + struct ev_loop *event_loop = NULL; struct lua_tcp_cbdata *cbd; struct rspamd_dns_resolver *resolver = NULL; struct rspamd_async_session *session = NULL; @@ -1453,7 +1422,7 @@ lua_tcp_request (lua_State *L) lua_pushstring (L, "port"); lua_gettable (L, -2); if (lua_type (L, -1) == LUA_TNUMBER) { - port = luaL_checknumber (L, -1); + port = lua_tointeger (L, -1); } else { /* We assume that it is a unix socket */ @@ -1478,7 +1447,7 @@ lua_tcp_request (lua_State *L) lua_gettable (L, -2); if (lua_type (L, -1) == LUA_TUSERDATA) { task = lua_check_task (L, -1); - ev_base = task->ev_base; + event_loop = task->event_loop; resolver = task->resolver; session = task->s; cfg = task->cfg; @@ -1489,10 +1458,10 @@ lua_tcp_request (lua_State *L) lua_pushstring (L, "ev_base"); lua_gettable (L, -2); if (rspamd_lua_check_udata_maybe (L, -1, "rspamd{ev_base}")) { - ev_base = *(struct event_base **)lua_touserdata (L, -1); + event_loop = *(struct ev_loop **)lua_touserdata (L, -1); } else { - ev_base = NULL; + return luaL_error (L, "event loop is required"); } lua_pop (L, 1); @@ -1522,7 +1491,7 @@ lua_tcp_request (lua_State *L) resolver = *(struct rspamd_dns_resolver **)lua_touserdata (L, -1); } else { - resolver = lua_tcp_global_resolver (ev_base, cfg); + resolver = lua_tcp_global_resolver (event_loop, cfg); } lua_pop (L, 1); } @@ -1530,7 +1499,7 @@ lua_tcp_request (lua_State *L) lua_pushstring (L, "timeout"); lua_gettable (L, -2); if (lua_type (L, -1) == LUA_TNUMBER) { - timeout = lua_tonumber (L, -1) * 1000.; + timeout = lua_tonumber (L, -1); } lua_pop (L, 1); @@ -1691,10 +1660,10 @@ lua_tcp_request (lua_State *L) g_queue_push_tail (cbd->handlers, wh); } - cbd->ev_base = ev_base; - msec_to_tv (timeout, &cbd->tv); + cbd->event_loop = event_loop; cbd->fd = -1; cbd->port = port; + cbd->ev.timeout = timeout; if (ssl) { cbd->flags |= LUA_TCP_FLAG_SSL; @@ -1825,7 +1794,7 @@ lua_tcp_connect_sync (lua_State *L) struct rspamd_async_session *session = NULL; struct rspamd_dns_resolver *resolver = NULL; struct rspamd_config *cfg = NULL; - struct event_base *ev_base = NULL; + struct ev_loop *ev_base = NULL; int arguments_validated = rspamd_lua_parse_table_arguments (L, 1, &err, "task=U{task};session=U{session};resolver=U{resolver};ev_base=U{ev_base};" @@ -1859,7 +1828,7 @@ lua_tcp_connect_sync (lua_State *L) if (task) { cfg = task->cfg; - ev_base = task->ev_base; + ev_base = task->event_loop; session = task->s; } if (resolver == NULL) { @@ -1881,9 +1850,8 @@ lua_tcp_connect_sync (lua_State *L) rspamd_snprintf (cbd->tag, sizeof (cbd->tag), "%uxL", h); cbd->handlers = g_queue_new (); - cbd->ev_base = ev_base; + cbd->event_loop = ev_base; cbd->flags |= LUA_TCP_FLAG_SYNC; - double_to_tv (timeout, &cbd->tv); cbd->fd = -1; cbd->port = (guint16)port; @@ -1980,25 +1948,6 @@ lua_tcp_close (lua_State *L) } static gint -lua_tcp_set_timeout (lua_State *L) -{ - LUA_TRACE_POINT; - struct lua_tcp_cbdata *cbd = lua_check_tcp (L, 1); - gdouble seconds = lua_tonumber (L, 2); - - if (cbd == NULL) { - return luaL_error (L, "invalid arguments"); - } - if (!lua_isnumber (L, 2)) { - return luaL_error (L, "invalid arguments: 'seconds' is expected to be number"); - } - - double_to_tv (seconds, &cbd->tv); - - return 0; -} - -static gint lua_tcp_add_read (lua_State *L) { LUA_TRACE_POINT; @@ -2159,7 +2108,7 @@ lua_tcp_sync_close (lua_State *L) cbd->flags |= LUA_TCP_FLAG_FINISHED; if (cbd->fd != -1) { - event_del (&cbd->ev); + rspamd_ev_watcher_stop (cbd->event_loop, &cbd->ev); close (cbd->fd); cbd->fd = -1; } @@ -2175,7 +2124,7 @@ lua_tcp_sync_session_dtor (gpointer ud) if (cbd->fd != -1) { msg_debug ("closing sync TCP connection"); - event_del (&cbd->ev); + rspamd_ev_watcher_stop (cbd->event_loop, &cbd->ev); close (cbd->fd); cbd->fd = -1; } @@ -2188,25 +2137,6 @@ lua_tcp_sync_session_dtor (gpointer ud) } static int -lua_tcp_sync_set_timeout (lua_State *L) -{ - LUA_TRACE_POINT; - struct lua_tcp_cbdata *cbd = lua_check_sync_tcp (L, 1); - gdouble seconds = lua_tonumber (L, 2); - - if (cbd == NULL) { - return luaL_error (L, "invalid arguments: self is not rspamd{tcp_sync}"); - } - if (lua_type (L, 2) != LUA_TNUMBER) { - return luaL_error (L, "invalid arguments: second parameter is expected to be number"); - } - - double_to_tv (seconds, &cbd->tv); - - return 0; -} - -static int lua_tcp_sync_read_once (lua_State *L) { LUA_TRACE_POINT; @@ -2363,12 +2293,11 @@ lua_tcp_starttls (lua_State * L) verify_peer = TRUE; } - event_base_set (cbd->ev_base, &cbd->ev); cbd->ssl_conn = - rspamd_ssl_connection_new (ssl_ctx, cbd->ev_base, verify_peer); + rspamd_ssl_connection_new (ssl_ctx, cbd->event_loop, verify_peer); if (!rspamd_ssl_connect_fd (cbd->ssl_conn, cbd->fd, cbd->hostname, &cbd->ev, - &cbd->tv, lua_tcp_handler, lua_tcp_ssl_on_error, cbd)) { + cbd->ev.timeout, lua_tcp_handler, lua_tcp_ssl_on_error, cbd)) { lua_tcp_push_error (cbd, TRUE, "ssl connection failed: %s", strerror (errno)); } diff --git a/src/lua/lua_udp.c b/src/lua/lua_udp.c index 8a862f16a..94d27bf63 100644 --- a/src/lua/lua_udp.c +++ b/src/lua/lua_udp.c @@ -18,6 +18,7 @@ #include "utlist.h" #include "unix-std.h" #include <math.h> +#include <src/libutil/libev_helper.h> static const gchar *M = "rspamd lua udp"; @@ -59,9 +60,8 @@ static const struct luaL_reg udp_libf[] = { }; struct lua_udp_cbdata { - struct event io; - struct timeval tv; - struct event_base *ev_base; + struct ev_loop *event_loop; + struct rspamd_io_ev ev; struct rspamd_async_event *async_ev; struct rspamd_task *task; rspamd_mempool_t *pool; @@ -115,10 +115,7 @@ lua_udp_cbd_fin (gpointer p) struct lua_udp_cbdata *cbd = (struct lua_udp_cbdata *)p; if (cbd->sock != -1) { - if (cbd->io.ev_base != NULL) { - event_del (&cbd->io); - } - + rspamd_ev_watcher_stop (cbd->event_loop, &cbd->ev); close (cbd->sock); } @@ -264,16 +261,12 @@ lua_udp_io_handler (gint fd, short what, gpointer p) L = cbd->L; - event_del (&cbd->io); - if (what == EV_TIMEOUT) { if (cbd->sent && cbd->retransmits > 0) { r = lua_try_send_request (cbd); if (r == RSPAMD_SENT_OK) { - event_set (&cbd->io, cbd->sock, EV_READ, lua_udp_io_handler, cbd); - event_base_set (cbd->ev_base, &cbd->io); - event_add (&cbd->io, &cbd->tv); + rspamd_ev_watcher_reschedule (cbd->event_loop, &cbd->ev, EV_READ); lua_udp_maybe_register_event (cbd); cbd->retransmits --; } @@ -282,9 +275,7 @@ lua_udp_io_handler (gint fd, short what, gpointer p) } else { cbd->retransmits --; - event_set (&cbd->io, cbd->sock, EV_WRITE, lua_udp_io_handler, cbd); - event_base_set (cbd->ev_base, &cbd->io); - event_add (&cbd->io, &cbd->tv); + rspamd_ev_watcher_reschedule (cbd->event_loop, &cbd->ev, EV_WRITE); } } else { @@ -301,9 +292,7 @@ lua_udp_io_handler (gint fd, short what, gpointer p) if (r == RSPAMD_SENT_OK) { if (cbd->cbref != -1) { - event_set (&cbd->io, cbd->sock, EV_READ, lua_udp_io_handler, cbd); - event_base_set (cbd->ev_base, &cbd->io); - event_add (&cbd->io, &cbd->tv); + rspamd_ev_watcher_reschedule (cbd->event_loop, &cbd->ev, EV_READ); cbd->sent = TRUE; } else { @@ -315,9 +304,7 @@ lua_udp_io_handler (gint fd, short what, gpointer p) } else { cbd->retransmits --; - event_set (&cbd->io, cbd->sock, EV_WRITE, lua_udp_io_handler, cbd); - event_base_set (cbd->ev_base, &cbd->io); - event_add (&cbd->io, &cbd->tv); + rspamd_ev_watcher_reschedule (cbd->event_loop, &cbd->ev, EV_WRITE); } } else if (what == EV_READ) { @@ -358,7 +345,7 @@ lua_udp_sendto (lua_State *L) { LUA_TRACE_POINT; const gchar *host; guint port; - struct event_base *ev_base = NULL; + struct ev_loop *ev_base = NULL; struct lua_udp_cbdata *cbd; struct rspamd_async_session *session = NULL; struct rspamd_task *task = NULL; @@ -371,7 +358,7 @@ lua_udp_sendto (lua_State *L) { lua_gettable (L, -2); if (lua_type (L, -1) == LUA_TNUMBER) { - port = luaL_checknumber (L, -1); + port = lua_tointeger (L, -1); } else { /* We assume that it is a unix socket */ @@ -423,7 +410,7 @@ lua_udp_sendto (lua_State *L) { lua_gettable (L, -2); if (lua_type (L, -1) == LUA_TUSERDATA) { task = lua_check_task (L, -1); - ev_base = task->ev_base; + ev_base = task->event_loop; session = task->s; pool = task->task_pool; } @@ -433,7 +420,7 @@ lua_udp_sendto (lua_State *L) { lua_pushstring (L, "ev_base"); lua_gettable (L, -2); if (rspamd_lua_check_udata_maybe (L, -1, "rspamd{ev_base}")) { - ev_base = *(struct event_base **) lua_touserdata (L, -1); + ev_base = *(struct ev_loop **) lua_touserdata (L, -1); } else { ev_base = NULL; } @@ -472,22 +459,15 @@ lua_udp_sendto (lua_State *L) { } - - if (!ev_base || !pool) { - rspamd_inet_address_free (addr); - - return luaL_error (L, "invalid arguments"); - } - cbd = rspamd_mempool_alloc0 (pool, sizeof (*cbd)); - cbd->ev_base = ev_base; + cbd->event_loop = ev_base; cbd->pool = pool; cbd->s = session; cbd->addr = addr; cbd->sock = rspamd_socket_create (rspamd_inet_address_get_af (addr), SOCK_DGRAM, 0, TRUE); cbd->cbref = -1; - double_to_tv (timeout, &cbd->tv); + cbd->ev.timeout = timeout; if (cbd->sock == -1) { rspamd_inet_address_free (addr); @@ -555,9 +535,9 @@ lua_udp_sendto (lua_State *L) { return 2; } - event_set (&cbd->io, cbd->sock, EV_READ, lua_udp_io_handler, cbd); - event_base_set (cbd->ev_base, &cbd->io); - event_add (&cbd->io, &cbd->tv); + rspamd_ev_watcher_init (&cbd->ev, cbd->sock, EV_READ, + lua_udp_io_handler, cbd); + rspamd_ev_watcher_start (cbd->event_loop, &cbd->ev, timeout); cbd->sent = TRUE; } @@ -571,9 +551,9 @@ lua_udp_sendto (lua_State *L) { return 2; } else { - event_set (&cbd->io, cbd->sock, EV_WRITE, lua_udp_io_handler, cbd); - event_base_set (cbd->ev_base, &cbd->io); - event_add (&cbd->io, &cbd->tv); + rspamd_ev_watcher_init (&cbd->ev, cbd->sock, EV_WRITE, + lua_udp_io_handler, cbd); + rspamd_ev_watcher_start (cbd->event_loop, &cbd->ev, timeout); if (!lua_udp_maybe_register_event (cbd)) { lua_pushboolean (L, false); diff --git a/src/lua/lua_util.c b/src/lua/lua_util.c index e6241b48d..4eaf7b672 100644 --- a/src/lua/lua_util.c +++ b/src/lua/lua_util.c @@ -695,11 +695,11 @@ static gint lua_util_create_event_base (lua_State *L) { LUA_TRACE_POINT; - struct event_base **pev_base; + struct ev_loop **pev_base; - pev_base = lua_newuserdata (L, sizeof (struct event_base *)); + pev_base = lua_newuserdata (L, sizeof (struct ev_loop *)); rspamd_lua_setclass (L, "rspamd{ev_base}", -1); - *pev_base = event_init (); + *pev_base = ev_loop_new (EVFLAG_SIGNALFD|EVBACKEND_ALL); return 1; } @@ -842,13 +842,13 @@ lua_util_process_message (lua_State *L) const gchar *message; gsize mlen; struct rspamd_task *task; - struct event_base *base; + struct ev_loop *base; ucl_object_t *res = NULL; message = luaL_checklstring (L, 2, &mlen); if (cfg != NULL && message != NULL) { - base = event_init (); + base = ev_loop_new (EVFLAG_SIGNALFD|EVBACKEND_ALL); rspamd_init_filters (cfg, FALSE); task = rspamd_task_new (NULL, cfg, NULL, NULL, base); task->msg.begin = rspamd_mempool_alloc (task->task_pool, mlen); @@ -865,7 +865,7 @@ lua_util_process_message (lua_State *L) } else { if (rspamd_task_process (task, RSPAMD_TASK_PROCESS_ALL)) { - event_base_loop (base, 0); + ev_loop (base, 0); if (res != NULL) { ucl_object_push_lua (L, res, true); @@ -885,7 +885,7 @@ lua_util_process_message (lua_State *L) } } - event_base_free (base); + ev_loop_destroy (base); } else { lua_pushnil (L); @@ -3795,14 +3795,14 @@ static int lua_ev_base_loop (lua_State *L) { int flags = 0; - struct event_base *ev_base; + struct ev_loop *ev_base; ev_base = lua_check_ev_base (L, 1); if (lua_isnumber (L, 2)) { - flags = lua_tonumber (L, 2); + flags = lua_tointeger (L, 2); } - int ret = event_base_loop (ev_base, flags); + int ret = ev_run (ev_base, flags); lua_pushinteger (L, ret); return 1; diff --git a/src/lua/lua_worker.c b/src/lua/lua_worker.c index 332d7e663..73f8baea1 100644 --- a/src/lua/lua_worker.c +++ b/src/lua/lua_worker.c @@ -282,8 +282,8 @@ struct rspamd_lua_process_cbdata { GString *out_buf; goffset out_pos; struct rspamd_worker *wrk; - struct event_base *ev_base; - struct event ev; + struct ev_loop *event_loop; + ev_io ev; }; static void @@ -393,9 +393,9 @@ rspamd_lua_cld_handler (struct rspamd_worker_signal_handler *sigh, void *ud) if (!cbdata->replied) { /* We still need to call on_complete callback */ + ev_io_stop (cbdata->event_loop, &cbdata->ev); rspamd_lua_call_on_complete (cbdata->L, cbdata, "Worker has died without reply", NULL, 0); - event_del (&cbdata->ev); } /* Free structures */ @@ -414,7 +414,7 @@ rspamd_lua_cld_handler (struct rspamd_worker_signal_handler *sigh, void *ud) srv_cmd.cmd.on_fork.state = child_dead; srv_cmd.cmd.on_fork.cpid = cbdata->cpid; srv_cmd.cmd.on_fork.ppid = getpid (); - rspamd_srv_send_command (cbdata->wrk, cbdata->ev_base, &srv_cmd, -1, + rspamd_srv_send_command (cbdata->wrk, cbdata->event_loop, &srv_cmd, -1, NULL, NULL); g_free (cbdata); @@ -423,9 +423,10 @@ rspamd_lua_cld_handler (struct rspamd_worker_signal_handler *sigh, void *ud) } static void -rspamd_lua_subprocess_io (gint fd, short what, gpointer ud) +rspamd_lua_subprocess_io (EV_P_ ev_io *w, int revents) { - struct rspamd_lua_process_cbdata *cbdata = ud; + struct rspamd_lua_process_cbdata *cbdata = + (struct rspamd_lua_process_cbdata *)w->data; gssize r; if (cbdata->sz == (guint64)-1) { @@ -436,9 +437,9 @@ rspamd_lua_subprocess_io (gint fd, short what, gpointer ud) sizeof (guint64) - cbdata->io_buf->len); if (r == 0) { + ev_io_stop (cbdata->event_loop, &cbdata->ev); rspamd_lua_call_on_complete (cbdata->L, cbdata, "Unexpected EOF", NULL, 0); - event_del (&cbdata->ev); cbdata->replied = TRUE; kill (cbdata->cpid, SIGTERM); @@ -449,9 +450,9 @@ rspamd_lua_subprocess_io (gint fd, short what, gpointer ud) return; } else { + ev_io_stop (cbdata->event_loop, &cbdata->ev); rspamd_lua_call_on_complete (cbdata->L, cbdata, strerror (errno), NULL, 0); - event_del (&cbdata->ev); cbdata->replied = TRUE; kill (cbdata->cpid, SIGTERM); @@ -481,9 +482,9 @@ rspamd_lua_subprocess_io (gint fd, short what, gpointer ud) cbdata->sz - cbdata->io_buf->len); if (r == 0) { + ev_io_stop (cbdata->event_loop, &cbdata->ev); rspamd_lua_call_on_complete (cbdata->L, cbdata, "Unexpected EOF", NULL, 0); - event_del (&cbdata->ev); cbdata->replied = TRUE; kill (cbdata->cpid, SIGTERM); @@ -494,9 +495,9 @@ rspamd_lua_subprocess_io (gint fd, short what, gpointer ud) return; } else { + ev_io_stop (cbdata->event_loop, &cbdata->ev); rspamd_lua_call_on_complete (cbdata->L, cbdata, strerror (errno), NULL, 0); - event_del (&cbdata->ev); cbdata->replied = TRUE; kill (cbdata->cpid, SIGTERM); @@ -509,6 +510,7 @@ rspamd_lua_subprocess_io (gint fd, short what, gpointer ud) if (cbdata->io_buf->len == cbdata->sz) { gchar rep[4]; + ev_io_stop (cbdata->event_loop, &cbdata->ev); /* Finished reading data */ if (cbdata->is_error) { cbdata->io_buf->str[cbdata->io_buf->len] = '\0'; @@ -520,7 +522,6 @@ rspamd_lua_subprocess_io (gint fd, short what, gpointer ud) NULL, cbdata->io_buf->str, cbdata->io_buf->len); } - event_del (&cbdata->ev); cbdata->replied = TRUE; /* Write reply to the child */ @@ -577,7 +578,7 @@ lua_worker_spawn_process (lua_State *L) actx = w->ctx; cbdata->wrk = w; cbdata->L = L; - cbdata->ev_base = actx->ev_base; + cbdata->event_loop = actx->event_loop; cbdata->sz = (guint64)-1; pid = fork (); @@ -612,7 +613,8 @@ lua_worker_spawn_process (lua_State *L) close (cbdata->sp[0]); /* Here we assume that we can block on writing results */ rspamd_socket_blocking (cbdata->sp[1]); - event_reinit (cbdata->ev_base); + ev_loop_destroy (cbdata->event_loop); + cbdata->event_loop = ev_loop_new (EVFLAG_SIGNALFD); g_hash_table_remove_all (w->signal_events); rspamd_worker_unblock_signals (); rspamd_lua_execute_lua_subprocess (L, cbdata); @@ -639,21 +641,19 @@ lua_worker_spawn_process (lua_State *L) srv_cmd.cmd.on_fork.state = child_create; srv_cmd.cmd.on_fork.cpid = pid; srv_cmd.cmd.on_fork.ppid = getpid (); - rspamd_srv_send_command (w, cbdata->ev_base, &srv_cmd, -1, NULL, NULL); + rspamd_srv_send_command (w, cbdata->event_loop, &srv_cmd, -1, NULL, NULL); close (cbdata->sp[1]); rspamd_socket_nonblocking (cbdata->sp[0]); /* Parent */ - rspamd_worker_set_signal_handler (SIGCHLD, w, cbdata->ev_base, + rspamd_worker_set_signal_handler (SIGCHLD, w, cbdata->event_loop, rspamd_lua_cld_handler, cbdata); /* Add result pipe waiting */ - event_set (&cbdata->ev, cbdata->sp[0], EV_READ | EV_PERSIST, - rspamd_lua_subprocess_io, cbdata); - event_base_set (cbdata->ev_base, &cbdata->ev); - /* TODO: maybe add timeout? */ - event_add (&cbdata->ev, NULL); + ev_io_init (&cbdata->ev, rspamd_lua_subprocess_io, cbdata->sp[0], EV_READ); + cbdata->ev.data = cbdata; + ev_io_start (cbdata->event_loop, &cbdata->ev); return 0; } diff --git a/src/lua_worker.c b/src/lua_worker.c deleted file mode 100644 index df76945ea..000000000 --- a/src/lua_worker.c +++ /dev/null @@ -1,420 +0,0 @@ -/*- - * Copyright 2016 Vsevolod Stakhov - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "config.h" -#include "util.h" -#include "rspamd.h" -#include "libserver/worker_util.h" -#include "protocol.h" -#include "upstream.h" -#include "cfg_file.h" -#include "url.h" -#include "message.h" -#include "map.h" -#include "dns.h" -#include "unix-std.h" - -#include "lua/lua_common.h" - -#ifdef WITH_GPERF_TOOLS -# include <glib/gprintf.h> -#endif - -/* 60 seconds for worker's IO */ -#define DEFAULT_WORKER_IO_TIMEOUT 60000 - -gpointer init_lua_worker (struct rspamd_config *cfg); -void start_lua_worker (struct rspamd_worker *worker); - -worker_t lua_worker = { - "lua", /* Name */ - init_lua_worker, /* Init function */ - start_lua_worker, /* Start function */ - RSPAMD_WORKER_HAS_SOCKET | RSPAMD_WORKER_KILLABLE, - RSPAMD_WORKER_SOCKET_TCP, /* TCP socket */ - RSPAMD_WORKER_VER /* Version info */ -}; - -static const guint64 rspamd_lua_ctx_magic = 0x8055e2652aacf96eULL; -/* - * Worker's context - */ -struct rspamd_lua_worker_ctx { - guint64 magic; - /* Events base */ - struct event_base *ev_base; - /* DNS resolver */ - struct rspamd_dns_resolver *resolver; - /* Config */ - struct rspamd_config *cfg; - /* END OF COMMON PART */ - /* Other params */ - GHashTable *params; - /* Lua script to load */ - gchar *file; - /* Lua state */ - lua_State *L; - /* Callback for accept */ - gint cbref_accept; - /* Callback for finishing */ - gint cbref_fin; - /* The rest options */ - ucl_object_t *opts; -}; - -/* Lua bindings */ -LUA_FUNCTION_DEF (worker, get_ev_base); -LUA_FUNCTION_DEF (worker, register_accept_callback); -LUA_FUNCTION_DEF (worker, register_exit_callback); -LUA_FUNCTION_DEF (worker, get_option); -LUA_FUNCTION_DEF (worker, get_resolver); -LUA_FUNCTION_DEF (worker, get_cfg); - -static const struct luaL_reg lua_workerlib_m[] = { - LUA_INTERFACE_DEF (worker, get_ev_base), - LUA_INTERFACE_DEF (worker, register_accept_callback), - LUA_INTERFACE_DEF (worker, register_exit_callback), - LUA_INTERFACE_DEF (worker, get_option), - LUA_INTERFACE_DEF (worker, get_resolver), - LUA_INTERFACE_DEF (worker, get_cfg), - {"__tostring", rspamd_lua_class_tostring}, - {NULL, NULL} -}; - -/* Basic functions of LUA API for worker object */ -static gint -luaopen_lua_worker (lua_State * L) -{ - rspamd_lua_new_class (L, "rspamd{lua_worker}", lua_workerlib_m); - luaL_register (L, "rspamd_lua_worker", null_reg); - - lua_pop (L, 1); /* remove metatable from stack */ - - return 1; -} - -struct rspamd_lua_worker_ctx * -lua_check_lua_worker (lua_State * L) -{ - void *ud = luaL_checkudata (L, 1, "rspamd{lua_worker}"); - luaL_argcheck (L, ud != NULL, 1, "'lua_worker' expected"); - return ud ? *((struct rspamd_lua_worker_ctx **)ud) : NULL; -} - -static int -lua_worker_get_ev_base (lua_State *L) -{ - struct rspamd_lua_worker_ctx *ctx = lua_check_lua_worker (L); - struct event_base **pbase; - - if (ctx) { - pbase = lua_newuserdata (L, sizeof (struct event_base *)); - rspamd_lua_setclass (L, "rspamd{ev_base}", -1); - *pbase = ctx->ev_base; - } - else { - lua_pushnil (L); - } - - return 1; -} - -static int -lua_worker_register_accept_callback (lua_State *L) -{ - struct rspamd_lua_worker_ctx *ctx = lua_check_lua_worker (L); - - if (ctx) { - if (!lua_isfunction (L, 2)) { - msg_err ("invalid callback passed"); - lua_pushnil (L); - } - else { - lua_pushvalue (L, 2); - ctx->cbref_accept = luaL_ref (L, LUA_REGISTRYINDEX); - return 0; - } - } - else { - lua_pushnil (L); - } - - return 1; -} - -static int -lua_worker_register_exit_callback (lua_State *L) -{ - struct rspamd_lua_worker_ctx *ctx = lua_check_lua_worker (L); - - if (ctx) { - if (!lua_isfunction (L, 2)) { - msg_err ("invalid callback passed"); - lua_pushnil (L); - } - else { - lua_pushvalue (L, 2); - ctx->cbref_fin = luaL_ref (L, LUA_REGISTRYINDEX); - return 0; - } - } - else { - lua_pushnil (L); - } - - return 1; -} - -/* XXX: This functions should be rewritten completely */ -static int -lua_worker_get_option (lua_State *L) -{ - struct rspamd_lua_worker_ctx *ctx = lua_check_lua_worker (L); - const ucl_object_t *val; - const gchar *name; - - if (ctx) { - name = luaL_checkstring (L, 2); - if (name == NULL) { - msg_err ("no name specified"); - lua_pushnil (L); - } - else { - val = ucl_object_lookup (ctx->opts, name); - if (val == NULL) { - lua_pushnil (L); - } - else { - ucl_object_push_lua (L, val, TRUE); - } - } - } - else { - lua_pushnil (L); - } - - return 1; -} - -static int -lua_worker_get_resolver (lua_State *L) -{ - struct rspamd_lua_worker_ctx *ctx = lua_check_lua_worker (L); - struct rspamd_dns_resolver **presolver; - - if (ctx) { - presolver = lua_newuserdata (L, sizeof (gpointer)); - rspamd_lua_setclass (L, "rspamd{resolver}", -1); - *presolver = ctx->resolver; - } - else { - lua_pushnil (L); - } - - return 1; -} - -static int -lua_worker_get_cfg (lua_State *L) -{ - struct rspamd_lua_worker_ctx *ctx = lua_check_lua_worker (L); - struct rspamd_config **pcfg; - - if (ctx) { - pcfg = lua_newuserdata (L, sizeof (gpointer)); - rspamd_lua_setclass (L, "rspamd{config}", -1); - *pcfg = ctx->cfg; - } - else { - lua_pushnil (L); - } - - return 1; -} - -/* End of lua API */ - -/* - * Accept new connection and construct task - */ -static void -lua_accept_socket (gint fd, short what, void *arg) -{ - struct rspamd_worker *worker = (struct rspamd_worker *) arg; - struct rspamd_lua_worker_ctx *ctx, **pctx; - gint nfd; - lua_State *L; - rspamd_inet_addr_t *addr; - - ctx = worker->ctx; - L = ctx->L; - - if ((nfd = - rspamd_accept_from_socket (fd, &addr, worker->accept_events)) == -1) { - msg_warn ("accept failed: %s", strerror (errno)); - return; - } - /* Check for EAGAIN */ - if (nfd == 0) { - return; - } - - msg_info ("accepted connection from %s port %d", - rspamd_inet_address_to_string (addr), - rspamd_inet_address_get_port (addr)); - - /* Call finalizer function */ - lua_rawgeti (L, LUA_REGISTRYINDEX, ctx->cbref_accept); - pctx = lua_newuserdata (L, sizeof (gpointer)); - rspamd_lua_setclass (L, "rspamd{lua_worker}", -1); - *pctx = ctx; - lua_pushinteger (L, nfd); - rspamd_lua_ip_push (L, addr); - lua_pushinteger (L, 0); - - - if (lua_pcall (L, 4, 0, 0) != 0) { - msg_info ("call to worker accept failed: %s", lua_tostring (L, -1)); - lua_pop (L, 1); - } - - rspamd_inet_address_free (addr); - close (nfd); -} - -static gboolean -rspamd_lua_worker_parser (ucl_object_t *obj, gpointer ud) -{ - struct rspamd_lua_worker_ctx *ctx = ud; - - ctx->opts = obj; - - return TRUE; -} - -gpointer -init_lua_worker (struct rspamd_config *cfg) -{ - struct rspamd_lua_worker_ctx *ctx; - GQuark type; - - type = g_quark_try_string ("lua"); - - ctx = rspamd_mempool_alloc (cfg->cfg_pool, - sizeof (struct rspamd_lua_worker_ctx)); - ctx->magic = rspamd_lua_ctx_magic; - ctx->params = g_hash_table_new_full (rspamd_str_hash, - rspamd_str_equal, - g_free, - (GDestroyNotify)g_list_free); - - - rspamd_rcl_register_worker_option (cfg, - type, - "file", - rspamd_rcl_parse_struct_string, - ctx, - G_STRUCT_OFFSET (struct rspamd_lua_worker_ctx, file), - 0, - "Run the following lua script when accepting a connection"); - - rspamd_rcl_register_worker_parser (cfg, type, rspamd_lua_worker_parser, - ctx); - - return ctx; -} - -/* - * Start worker process - */ -void -start_lua_worker (struct rspamd_worker *worker) -{ - struct rspamd_lua_worker_ctx *ctx = worker->ctx, **pctx; - lua_State *L; - -#ifdef WITH_PROFILER - extern void _start (void), etext (void); - monstartup ((u_long) & _start, (u_long) & etext); -#endif - - ctx->ev_base = rspamd_prepare_worker (worker, - "lua_worker", - lua_accept_socket); - - L = worker->srv->cfg->lua_state; - ctx->L = L; - ctx->cfg = worker->srv->cfg; - - ctx->resolver = rspamd_dns_resolver_init (worker->srv->logger, - ctx->ev_base, - worker->srv->cfg); - - /* Open worker's lib */ - luaopen_lua_worker (L); - - if (ctx->file == NULL) { - msg_err ("No lua script defined, so no reason to exist"); - exit (EXIT_SUCCESS); - } - if (access (ctx->file, R_OK) == -1) { - msg_err ("Error reading lua script %s: %s", ctx->file, - strerror (errno)); - exit (EXIT_SUCCESS); - } - - pctx = lua_newuserdata (L, sizeof (gpointer)); - rspamd_lua_setclass (L, "rspamd{lua_worker}", -1); - lua_setglobal (L, "rspamd_lua_worker"); - *pctx = ctx; - - if (luaL_dofile (L, ctx->file) != 0) { - msg_err ("Error executing lua script %s: %s", ctx->file, - lua_tostring (L, -1)); - exit (EXIT_SUCCESS); - } - - if (ctx->cbref_accept == 0) { - msg_err ("No accept function defined, so no reason to exist"); - exit (EXIT_SUCCESS); - } - - rspamd_lua_run_postloads (ctx->cfg->lua_state, ctx->cfg, ctx->ev_base, - worker); - event_base_loop (ctx->ev_base, 0); - rspamd_worker_block_signals (); - - luaL_unref (L, LUA_REGISTRYINDEX, ctx->cbref_accept); - if (ctx->cbref_fin != 0) { - /* Call finalizer function */ - lua_rawgeti (L, LUA_REGISTRYINDEX, ctx->cbref_fin); - pctx = lua_newuserdata (L, sizeof (gpointer)); - rspamd_lua_setclass (L, "rspamd{lua_worker}", -1); - *pctx = ctx; - if (lua_pcall (L, 1, 0, 0) != 0) { - msg_info ("call to worker finalizer failed: %s", lua_tostring (L, - -1)); - lua_pop (L, 1); - } - /* Free resources */ - luaL_unref (L, LUA_REGISTRYINDEX, ctx->cbref_fin); - } - - REF_RELEASE (ctx->cfg); - rspamd_log_close (worker->srv->logger, TRUE); - - exit (EXIT_SUCCESS); -} - diff --git a/src/plugins/dkim_check.c b/src/plugins/dkim_check.c index e1cdc5e98..233ecf1d1 100644 --- a/src/plugins/dkim_check.c +++ b/src/plugins/dkim_check.c @@ -1056,8 +1056,8 @@ dkim_module_key_handler (rspamd_dkim_key_t *key, * lru hash owns this object now */ rspamd_lru_hash_insert (dkim_module_ctx->dkim_hash, - g_strdup (rspamd_dkim_get_dns_key (ctx)), - key, res->task->tv.tv_sec, rspamd_dkim_key_get_ttl (key)); + g_strdup (rspamd_dkim_get_dns_key (ctx)), + key, res->task->task_timestamp, rspamd_dkim_key_get_ttl (key)); /* Release key when task is processed */ rspamd_mempool_add_destructor (res->task->task_pool, dkim_module_key_dtor, res->key); @@ -1210,7 +1210,7 @@ dkim_symbol_callback (struct rspamd_task *task, key = rspamd_lru_hash_lookup (dkim_module_ctx->dkim_hash, rspamd_dkim_get_dns_key (ctx), - task->tv.tv_sec); + task->task_timestamp); if (key != NULL) { cur->key = rspamd_dkim_key_ref (key); @@ -1400,7 +1400,7 @@ dkim_module_lua_on_key (rspamd_dkim_key_t *key, */ rspamd_lru_hash_insert (dkim_module_ctx->dkim_hash, g_strdup (rspamd_dkim_get_dns_key (ctx)), - key, cbd->task->tv.tv_sec, rspamd_dkim_key_get_ttl (key)); + key, cbd->task->task_timestamp, rspamd_dkim_key_get_ttl (key)); /* Release key when task is processed */ rspamd_mempool_add_destructor (cbd->task->task_pool, dkim_module_key_dtor, cbd->key); @@ -1507,7 +1507,7 @@ lua_dkim_verify_handler (lua_State *L) key = rspamd_lru_hash_lookup (dkim_module_ctx->dkim_hash, rspamd_dkim_get_dns_key (ctx), - task->tv.tv_sec); + task->task_timestamp); if (key != NULL) { cbd->key = rspamd_dkim_key_ref (key); diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c index 639b0edf8..75df2a645 100644 --- a/src/plugins/fuzzy_check.c +++ b/src/plugins/fuzzy_check.c @@ -47,6 +47,7 @@ #include "libstat/stat_api.h" #include <math.h> #include <src/libmime/message.h> +#include "libutil/libev_helper.h" #define DEFAULT_SYMBOL "R_FUZZY_HASH" @@ -129,9 +130,8 @@ struct fuzzy_client_session { struct rspamd_symcache_item *item; struct upstream *server; struct fuzzy_rule *rule; - struct event ev; - struct event timev; - struct timeval tv; + struct ev_loop *event_loop; + struct rspamd_io_ev ev; gint state; gint fd; guint retransmits; @@ -146,9 +146,8 @@ struct fuzzy_learn_session { struct upstream *server; struct fuzzy_rule *rule; struct rspamd_task *task; - struct event ev; - struct event timev; - struct timeval tv; + struct ev_loop *event_loop; + struct rspamd_io_ev ev; gint fd; guint retransmits; }; @@ -1185,8 +1184,7 @@ fuzzy_io_fin (void *ud) g_ptr_array_free (session->results, TRUE); } - event_del (&session->ev); - event_del (&session->timev); + rspamd_ev_watcher_stop (session->event_loop, &session->ev); close (session->fd); } @@ -2181,13 +2179,49 @@ fuzzy_check_session_is_completed (struct fuzzy_client_session *session) return FALSE; } +/* Fuzzy check timeout callback */ +static void +fuzzy_check_timer_callback (gint fd, short what, void *arg) +{ + struct fuzzy_client_session *session = arg; + struct rspamd_task *task; + + task = session->task; + + /* We might be here because of other checks being slow */ + if (fuzzy_check_try_read (session) > 0) { + if (fuzzy_check_session_is_completed (session)) { + return; + } + } + + if (session->retransmits >= session->rule->ctx->retransmits) { + msg_err_task ("got IO timeout with server %s(%s), after %d retransmits", + rspamd_upstream_name (session->server), + rspamd_inet_address_to_string_pretty ( + rspamd_upstream_addr_cur (session->server)), + session->retransmits); + rspamd_upstream_fail (session->server, TRUE); + + if (session->item) { + rspamd_symcache_item_async_dec_check (session->task, session->item, M); + } + rspamd_session_remove_event (session->task->s, fuzzy_io_fin, session); + } + else { + /* Plan write event */ + rspamd_ev_watcher_reschedule (session->event_loop, + &session->ev, EV_READ|EV_WRITE); + session->retransmits ++; + } +} + /* Fuzzy check callback */ static void fuzzy_check_io_callback (gint fd, short what, void *arg) { struct fuzzy_client_session *session = arg; struct rspamd_task *task; - struct event_base *ev_base; gint r; enum { @@ -2224,18 +2258,14 @@ fuzzy_check_io_callback (gint fd, short what, void *arg) } } else { - /* Should not happen */ - g_assert (0); + fuzzy_check_timer_callback (fd, what, arg); + return; } if (ret == return_want_more) { /* Processed write, switch to reading */ - ev_base = event_get_base (&session->ev); - event_del (&session->ev); - event_set (&session->ev, fd, EV_READ, - fuzzy_check_io_callback, session); - event_base_set (ev_base, &session->ev); - event_add (&session->ev, NULL); + rspamd_ev_watcher_reschedule (session->event_loop, + &session->ev, EV_READ); } else if (ret == return_error) { /* Error state */ @@ -2258,78 +2288,82 @@ fuzzy_check_io_callback (gint fd, short what, void *arg) /* Read something from network */ if (!fuzzy_check_session_is_completed (session)) { /* Need to read more */ - ev_base = event_get_base (&session->ev); - event_del (&session->ev); - event_set (&session->ev, session->fd, EV_READ, - fuzzy_check_io_callback, session); - event_base_set (ev_base, &session->ev); - event_add (&session->ev, NULL); + rspamd_ev_watcher_reschedule (session->event_loop, + &session->ev, EV_READ); } } } -/* Fuzzy check timeout callback */ + static void -fuzzy_check_timer_callback (gint fd, short what, void *arg) +fuzzy_lua_fin (void *ud) { - struct fuzzy_client_session *session = arg; + struct fuzzy_learn_session *session = ud; + + (*session->saved)--; + + rspamd_ev_watcher_stop (session->event_loop, &session->ev); + close (session->fd); +} + +/* Controller IO */ + +static void +fuzzy_controller_timer_callback (gint fd, short what, void *arg) +{ + struct fuzzy_learn_session *session = arg; struct rspamd_task *task; - struct event_base *ev_base; task = session->task; - /* We might be here because of other checks being slow */ - if (fuzzy_check_try_read (session) > 0) { - if (fuzzy_check_session_is_completed (session)) { - return; - } - } - if (session->retransmits >= session->rule->ctx->retransmits) { - msg_err_task ("got IO timeout with server %s(%s), after %d retransmits", + rspamd_upstream_fail (session->server, TRUE); + msg_err_task_check ("got IO timeout with server %s(%s), " + "after %d retransmits", rspamd_upstream_name (session->server), rspamd_inet_address_to_string_pretty ( rspamd_upstream_addr_cur (session->server)), session->retransmits); - rspamd_upstream_fail (session->server, TRUE); - if (session->item) { - rspamd_symcache_item_async_dec_check (session->task, session->item, M); + if (session->session) { + rspamd_session_remove_event (session->session, fuzzy_lua_fin, + session); + } + else { + if (session->http_entry) { + rspamd_controller_send_error (session->http_entry, + 500, "IO timeout with fuzzy storage"); + } + + if (*session->saved > 0 ) { + (*session->saved)--; + if (*session->saved == 0) { + if (session->http_entry) { + rspamd_task_free (session->task); + } + + session->task = NULL; + } + } + + if (session->http_entry) { + rspamd_http_connection_unref (session->http_entry->conn); + } + + rspamd_ev_watcher_stop (session->event_loop, + &session->ev); + close (session->fd); } - rspamd_session_remove_event (session->task->s, fuzzy_io_fin, session); } else { /* Plan write event */ - ev_base = event_get_base (&session->ev); - event_del (&session->ev); - event_set (&session->ev, fd, EV_WRITE|EV_READ, - fuzzy_check_io_callback, session); - event_base_set (ev_base, &session->ev); - event_add (&session->ev, NULL); - - /* Plan new retransmit timer */ - ev_base = event_get_base (&session->timev); - event_del (&session->timev); - event_base_set (ev_base, &session->timev); - event_add (&session->timev, &session->tv); + rspamd_ev_watcher_reschedule (session->event_loop, + &session->ev, EV_READ|EV_WRITE); session->retransmits ++; } } static void -fuzzy_lua_fin (void *ud) -{ - struct fuzzy_learn_session *session = ud; - - (*session->saved)--; - - event_del (&session->ev); - event_del (&session->timev); - close (session->fd); -} - -/* Controller IO */ -static void fuzzy_controller_io_callback (gint fd, short what, void *arg) { struct fuzzy_learn_session *session = arg; @@ -2340,7 +2374,6 @@ fuzzy_controller_io_callback (gint fd, short what, void *arg) struct fuzzy_cmd_io *io; struct rspamd_fuzzy_cmd *cmd = NULL; const gchar *symbol, *ftype; - struct event_base *ev_base; gint r; enum { return_error = 0, @@ -2355,7 +2388,8 @@ fuzzy_controller_io_callback (gint fd, short what, void *arg) if (what & EV_READ) { if ((r = read (fd, buf, sizeof (buf) - 1)) == -1) { if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) { - event_add (&session->ev, NULL); + rspamd_ev_watcher_reschedule (session->event_loop, + &session->ev, EV_READ); return; } @@ -2482,16 +2516,14 @@ fuzzy_controller_io_callback (gint fd, short what, void *arg) } } else { - g_assert (0); + fuzzy_controller_timer_callback (fd, what, arg); + + return; } if (ret == return_want_more) { - ev_base = event_get_base (&session->ev); - event_del (&session->ev); - event_set (&session->ev, fd, EV_READ, - fuzzy_controller_io_callback, session); - event_base_set (ev_base, &session->ev); - event_add (&session->ev, NULL); + rspamd_ev_watcher_reschedule (session->event_loop, + &session->ev, EV_READ); return; } @@ -2518,8 +2550,7 @@ fuzzy_controller_io_callback (gint fd, short what, void *arg) rspamd_http_connection_unref (session->http_entry->conn); } - event_del (&session->ev); - event_del (&session->timev); + rspamd_ev_watcher_stop (session->event_loop, &session->ev); close (session->fd); if (*session->saved == 0) { @@ -2555,7 +2586,6 @@ cleanup: if (session->http_entry) { ucl_object_t *reply, *hashes; - guint i; gchar hexbuf[rspamd_cryptobox_HASHBYTES * 2 + 1]; reply = ucl_object_typed_new (UCL_OBJECT); @@ -2588,72 +2618,6 @@ cleanup: } -static void -fuzzy_controller_timer_callback (gint fd, short what, void *arg) -{ - struct fuzzy_learn_session *session = arg; - struct rspamd_task *task; - struct event_base *ev_base; - - task = session->task; - - if (session->retransmits >= session->rule->ctx->retransmits) { - rspamd_upstream_fail (session->server, TRUE); - msg_err_task_check ("got IO timeout with server %s(%s), " - "after %d retransmits", - rspamd_upstream_name (session->server), - rspamd_inet_address_to_string_pretty ( - rspamd_upstream_addr_cur (session->server)), - session->retransmits); - - if (session->session) { - rspamd_session_remove_event (session->session, fuzzy_lua_fin, - session); - } - else { - if (session->http_entry) { - rspamd_controller_send_error (session->http_entry, - 500, "IO timeout with fuzzy storage"); - } - - if (*session->saved > 0 ) { - (*session->saved)--; - if (*session->saved == 0) { - if (session->http_entry) { - rspamd_task_free (session->task); - } - - session->task = NULL; - } - } - - if (session->http_entry) { - rspamd_http_connection_unref (session->http_entry->conn); - } - - event_del (&session->ev); - event_del (&session->timev); - close (session->fd); - } - } - else { - /* Plan write event */ - ev_base = event_get_base (&session->ev); - event_del (&session->ev); - event_set (&session->ev, fd, EV_WRITE|EV_READ, - fuzzy_controller_io_callback, session); - event_base_set (ev_base, &session->ev); - event_add (&session->ev, NULL); - - /* Plan new retransmit timer */ - ev_base = event_get_base (&session->timev); - event_del (&session->timev); - event_base_set (ev_base, &session->timev); - event_add (&session->timev, &session->tv); - session->retransmits ++; - } -} - static GPtrArray * fuzzy_generate_commands (struct rspamd_task *task, struct fuzzy_rule *rule, gint c, gint flag, guint32 value, guint flags) @@ -2774,7 +2738,6 @@ register_fuzzy_client_call (struct rspamd_task *task, session = rspamd_mempool_alloc0 (task->task_pool, sizeof (struct fuzzy_client_session)); - msec_to_tv (rule->ctx->io_timeout, &session->tv); session->state = 0; session->commands = commands; session->task = task; @@ -2782,16 +2745,15 @@ register_fuzzy_client_call (struct rspamd_task *task, session->server = selected; session->rule = rule; session->results = g_ptr_array_sized_new (32); + session->event_loop = task->event_loop; - event_set (&session->ev, sock, EV_WRITE, fuzzy_check_io_callback, - session); - event_base_set (session->task->ev_base, &session->ev); - event_add (&session->ev, NULL); - - evtimer_set (&session->timev, fuzzy_check_timer_callback, + rspamd_ev_watcher_init (&session->ev, + sock, + EV_WRITE, + fuzzy_check_io_callback, session); - event_base_set (session->task->ev_base, &session->timev); - event_add (&session->timev, &session->tv); + rspamd_ev_watcher_start (session->event_loop, &session->ev, + ((double)rule->ctx->io_timeout) / 1000.0); rspamd_session_add_event (task->s, fuzzy_io_fin, session, M); session->item = rspamd_symcache_get_cur_item (task); @@ -2881,7 +2843,6 @@ register_fuzzy_controller_call (struct rspamd_http_connection_entry *entry, struct rspamd_controller_session *session = entry->ud; gint sock; gint ret = -1; - struct fuzzy_ctx *fuzzy_module_ctx = fuzzy_get_context (task->cfg); /* Get upstream */ @@ -2899,7 +2860,6 @@ register_fuzzy_controller_call (struct rspamd_http_connection_entry *entry, rspamd_mempool_alloc0 (session->pool, sizeof (struct fuzzy_learn_session)); - msec_to_tv (fuzzy_module_ctx->io_timeout, &s->tv); s->task = task; s->commands = commands; s->http_entry = entry; @@ -2908,17 +2868,17 @@ register_fuzzy_controller_call (struct rspamd_http_connection_entry *entry, s->fd = sock; s->err = err; s->rule = rule; + s->event_loop = task->event_loop; /* We ref connection to avoid freeing before we process fuzzy rule */ rspamd_http_connection_ref (entry->conn); - event_set (&s->ev, sock, EV_WRITE, fuzzy_controller_io_callback, s); - event_base_set (entry->rt->ev_base, &s->ev); - event_add (&s->ev, NULL); - - evtimer_set (&s->timev, fuzzy_controller_timer_callback, + rspamd_ev_watcher_init (&s->ev, + sock, + EV_WRITE, + fuzzy_controller_io_callback, s); - event_base_set (s->task->ev_base, &s->timev); - event_add (&s->timev, &s->tv); + rspamd_ev_watcher_start (s->event_loop, &s->ev, + ((double)rule->ctx->io_timeout) / 1000.0); (*saved)++; ret = 1; @@ -2946,7 +2906,7 @@ fuzzy_process_handler (struct rspamd_http_connection_entry *conn_ent, /* Prepare task */ task = rspamd_task_new (session->wrk, session->cfg, NULL, - session->lang_det, conn_ent->rt->ev_base); + session->lang_det, conn_ent->rt->event_loop); task->cfg = ctx->cfg; saved = rspamd_mempool_alloc0 (session->pool, sizeof (gint)); err = rspamd_mempool_alloc0 (session->pool, sizeof (GError *)); @@ -3258,8 +3218,6 @@ fuzzy_check_send_lua_learn (struct fuzzy_rule *rule, s = rspamd_mempool_alloc0 (task->task_pool, sizeof (struct fuzzy_learn_session)); - - msec_to_tv (rule->ctx->io_timeout, &s->tv); s->task = task; s->commands = commands; s->http_entry = NULL; @@ -3269,14 +3227,15 @@ fuzzy_check_send_lua_learn (struct fuzzy_rule *rule, s->err = err; s->rule = rule; s->session = task->s; + s->event_loop = task->event_loop; - event_set (&s->ev, sock, EV_WRITE, fuzzy_controller_io_callback, s); - event_base_set (task->ev_base, &s->ev); - event_add (&s->ev, NULL); - - evtimer_set (&s->timev, fuzzy_controller_timer_callback, s); - event_base_set (s->task->ev_base, &s->timev); - event_add (&s->timev, &s->tv); + rspamd_ev_watcher_init (&s->ev, + sock, + EV_WRITE, + fuzzy_controller_io_callback, + s); + rspamd_ev_watcher_start (s->event_loop, &s->ev, + ((double)rule->ctx->io_timeout) / 1000.0); rspamd_session_add_event (task->s, fuzzy_lua_fin, @@ -3367,7 +3326,7 @@ static gint fuzzy_lua_learn_handler (lua_State *L) { struct rspamd_task *task = lua_check_task (L, 1); - guint flag = 0, weight = 1.0, send_flags = 0; + guint flag = 0, weight = 1, send_flags = 0; const gchar *symbol; struct fuzzy_ctx *fuzzy_module_ctx = fuzzy_get_context (task->cfg); diff --git a/src/plugins/lua/multimap.lua b/src/plugins/lua/multimap.lua index 68f254c1c..5db8d4680 100644 --- a/src/plugins/lua/multimap.lua +++ b/src/plugins/lua/multimap.lua @@ -22,7 +22,6 @@ end local rules = {} local rspamd_logger = require "rspamd_logger" -local cdb = require "rspamd_cdb" local util = require "rspamd_util" local regexp = require "rspamd_regexp" local rspamd_expression = require "rspamd_expression" @@ -400,6 +399,18 @@ local function multimap_callback(task, rule) local ret = false if r['cdb'] then + if type(r.cdb) == 'string' then + local rspamd_cdb = require "rspamd_cdb" + local cdb = rspamd_cdb.create(r.cdb, task:get_ev_base()) + + if not cdb then + rspamd_logger.infox(task, 'cannot open cdb file %s', r.cdb) + + return false + else + r.cdb = cdb + end + end local srch = value if type(value) == 'userdata' then if value.class == 'rspamd{ip}' then @@ -997,13 +1008,8 @@ local function add_multimap_rule(key, newrule) end -- Check cdb flag if type(newrule['map']) == 'string' and string.find(newrule['map'], '^cdb://.*$') then - newrule['cdb'] = cdb.create(newrule['map']) - if newrule['cdb'] then - ret = true - else - rspamd_logger.warnx(rspamd_config, 'Cannot add rule: map doesn\'t exists: %1', - newrule['map']) - end + newrule['cdb'] = newrule['map'] + ret = true elseif type(newrule['map']) == 'string' and string.find(newrule['map'], '^redis://.*$') then if not redis_params then rspamd_logger.infox(rspamd_config, 'no redis servers are specified, ' .. diff --git a/src/plugins/lua/neural.lua b/src/plugins/lua/neural.lua index 30c4fee0f..dd1b94b3a 100644 --- a/src/plugins/lua/neural.lua +++ b/src/plugins/lua/neural.lua @@ -20,7 +20,6 @@ if confighelp then end local rspamd_logger = require "rspamd_logger" -local rspamd_fann = require "rspamd_fann" local rspamd_util = require "rspamd_util" local lua_redis = require "lua_redis" local lua_util = require "lua_util" @@ -320,14 +319,7 @@ local function create_ann(n, nlayers) return ann else - local layers = {} - local div = 1.0 - for _ = 1, nlayers - 1 do - table.insert(layers, math.floor(n / div)) - div = div * 2 - end - table.insert(layers, 1) - return rspamd_fann.create(nlayers, layers) + assert(false) end end @@ -378,7 +370,7 @@ local function load_or_invalidate_ann(rule, data, id, ev_base) if use_torch then ann = torch.MemoryFile(torch.CharStorage():string(tostring(ann_data))):readObject() else - ann = rspamd_fann.load_data(ann_data) + assert(false) end end @@ -940,7 +932,7 @@ if not (opts and type(opts) == 'table') or not redis_params then return end -if not rspamd_fann.is_enabled() and not use_torch then +if not use_torch then rspamd_logger.errx(rspamd_config, 'neural networks support is not compiled in rspamd, this ' .. 'module is eventually disabled') lua_util.disable_module(N, "fail") diff --git a/src/plugins/spf.c b/src/plugins/spf.c index 6a4004e57..f10807f47 100644 --- a/src/plugins/spf.c +++ b/src/plugins/spf.c @@ -561,7 +561,7 @@ spf_plugin_callback (struct spf_resolved *record, struct rspamd_task *task, spf_record_ref (record); if ((l = rspamd_lru_hash_lookup (spf_module_ctx->spf_hash, - record->domain, task->tv.tv_sec)) == NULL) { + record->domain, task->task_timestamp)) == NULL) { l = record; if (record->ttl > 0 && @@ -571,7 +571,7 @@ spf_plugin_callback (struct spf_resolved *record, struct rspamd_task *task, rspamd_lru_hash_insert (spf_module_ctx->spf_hash, record->domain, spf_record_ref (l), - task->tv.tv_sec, record->ttl); + task->task_timestamp, record->ttl); msg_info_task ("stored record for %s (0x%xuL) in LRU cache for %d seconds, " "%d/%d elements in the cache", @@ -642,7 +642,7 @@ spf_symbol_callback (struct rspamd_task *task, if (domain) { if ((l = rspamd_lru_hash_lookup (spf_module_ctx->spf_hash, domain, - task->tv.tv_sec)) != NULL) { + task->task_timestamp)) != NULL) { spf_record_ref (l); spf_check_list (l, task, TRUE); spf_record_unref (l); diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c index 338bdaa24..3c467615c 100644 --- a/src/plugins/surbl.c +++ b/src/plugins/surbl.c @@ -1815,7 +1815,6 @@ register_redirector_call (struct rspamd_url *url, struct rspamd_task *task, const gchar *rule) { struct redirector_param *param; - struct timeval *timeout; struct upstream *selected; struct rspamd_http_message *msg; struct surbl_ctx *surbl_module_ctx = surbl_get_context (task->cfg); @@ -1851,8 +1850,6 @@ register_redirector_call (struct rspamd_url *url, struct rspamd_task *task, msg = rspamd_http_new_message (HTTP_REQUEST); msg->url = rspamd_fstring_assign (msg->url, url->string, url->urllen); param->redirector = selected; - timeout = rspamd_mempool_alloc (task->task_pool, sizeof (struct timeval)); - double_to_tv (surbl_module_ctx->read_timeout, timeout); rspamd_session_add_event (task->s, free_redirector_session, param, @@ -1864,7 +1861,7 @@ register_redirector_call (struct rspamd_url *url, struct rspamd_task *task, } rspamd_http_connection_write_message (param->conn, msg, NULL, - NULL, param, timeout); + NULL, param, surbl_module_ctx->read_timeout); msg_info_surbl ( "<%s> registered redirector call for %*s to %s, according to rule: %s", diff --git a/src/rspamadm/CMakeLists.txt b/src/rspamadm/CMakeLists.txt index 5be38aa28..3d4f2f490 100644 --- a/src/rspamadm/CMakeLists.txt +++ b/src/rspamadm/CMakeLists.txt @@ -15,10 +15,8 @@ SET(RSPAMADMSRC rspamadm.c ${CMAKE_BINARY_DIR}/src/modules.c ${CMAKE_SOURCE_DIR}/src/controller.c ${CMAKE_SOURCE_DIR}/src/fuzzy_storage.c - ${CMAKE_SOURCE_DIR}/src/lua_worker.c ${CMAKE_SOURCE_DIR}/src/worker.c - ${CMAKE_SOURCE_DIR}/src/rspamd_proxy.c - ${CMAKE_SOURCE_DIR}/src/log_helper.c) + ${CMAKE_SOURCE_DIR}/src/rspamd_proxy.c) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) IF (ENABLE_HYPERSCAN MATCHES "ON") LIST(APPEND RSPAMADMSRC "${CMAKE_SOURCE_DIR}/src/hs_helper.c") diff --git a/src/rspamadm/control.c b/src/rspamadm/control.c index 1311622c3..2cc783c66 100644 --- a/src/rspamadm/control.c +++ b/src/rspamadm/control.c @@ -21,7 +21,7 @@ #include "libutil/http_private.h" #include "addr.h" #include "unix-std.h" -#include <event.h> +#include "contrib/libev/ev.h" #include "libutil/util.h" #include "lua/lua_common.h" @@ -98,7 +98,7 @@ static void rspamd_control_error_handler (struct rspamd_http_connection *conn, GError *err) { rspamd_fprintf (stderr, "Cannot make HTTP request: %e\n", err); - rspamd_http_connection_unref (conn); + ev_break (rspamd_main->event_loop, EVBREAK_ALL); } static gint @@ -111,7 +111,6 @@ rspamd_control_finish_handler (struct rspamd_http_connection *conn, const gchar *body; gsize body_len; struct rspamadm_control_cbdata *cbdata = conn->ud; - struct timeval exit_tv; body = rspamd_http_message_get_body (msg, &body_len); parser = ucl_parser_new (0); @@ -157,9 +156,7 @@ rspamd_control_finish_handler (struct rspamd_http_connection *conn, } end: - exit_tv.tv_sec = 0; - exit_tv.tv_usec = 0; - event_base_loopexit (rspamd_main->ev_base, &exit_tv); + ev_break (rspamd_main->event_loop, EVBREAK_ALL); return 0; } @@ -173,7 +170,6 @@ rspamadm_control (gint argc, gchar **argv, const struct rspamadm_command *_cmd) struct rspamd_http_connection *conn; struct rspamd_http_message *msg; rspamd_inet_addr_t *addr; - struct timeval tv; static struct rspamadm_control_cbdata cbdata; context = g_option_context_new ( @@ -239,16 +235,15 @@ rspamadm_control (gint argc, gchar **argv, const struct rspamadm_command *_cmd) addr); msg = rspamd_http_new_message (HTTP_REQUEST); msg->url = rspamd_fstring_new_init (path, strlen (path)); - double_to_tv (timeout, &tv); cbdata.argc = argc; cbdata.argv = argv; cbdata.path = path; rspamd_http_connection_write_message (conn, msg, NULL, NULL, &cbdata, - &tv); + timeout); - event_base_loop (rspamd_main->ev_base, 0); + ev_loop (rspamd_main->event_loop, 0); rspamd_http_connection_unref (conn); rspamd_inet_address_free (addr); diff --git a/src/rspamadm/lua_repl.c b/src/rspamadm/lua_repl.c index 43c97d01f..59e3db02c 100644 --- a/src/rspamadm/lua_repl.c +++ b/src/rspamadm/lua_repl.c @@ -296,7 +296,7 @@ wait_session_events (void) { /* XXX: it's probably worth to add timeout here - not to wait forever */ while (rspamd_session_events_pending (rspamadm_session) > 0) { - event_base_loop (rspamd_main->ev_base, EVLOOP_ONCE); + ev_loop (rspamd_main->event_loop, EVRUN_ONCE); } } @@ -515,7 +515,7 @@ rspamadm_lua_run_repl (lua_State *L) { gchar *input; gboolean is_multiline = FALSE; - GString *tb; + GString *tb = NULL; guint i; for (;;) { @@ -591,15 +591,16 @@ struct rspamadm_lua_repl_session { }; static void -rspamadm_lua_accept_cb (gint fd, short what, void *arg) +rspamadm_lua_accept_cb (EV_P_ ev_io *w, int revents) { - struct rspamadm_lua_repl_context *ctx = arg; + struct rspamadm_lua_repl_context *ctx = + (struct rspamadm_lua_repl_context *)w->data; rspamd_inet_addr_t *addr; struct rspamadm_lua_repl_session *session; gint nfd; if ((nfd = - rspamd_accept_from_socket (fd, &addr, NULL)) == -1) { + rspamd_accept_from_socket (w->fd, &addr, NULL, NULL)) == -1) { rspamd_fprintf (stderr, "accept failed: %s", strerror (errno)); return; } @@ -793,7 +794,7 @@ rspamadm_lua (gint argc, gchar **argv, const struct rspamadm_command *cmd) /* HTTP Server mode */ GPtrArray *addrs = NULL; gchar *name = NULL; - struct event_base *ev_base; + struct ev_loop *ev_base; struct rspamd_http_connection_router *http; gint fd; struct rspamadm_lua_repl_context *ctx; @@ -804,11 +805,11 @@ rspamadm_lua (gint argc, gchar **argv, const struct rspamadm_command *cmd) exit (EXIT_FAILURE); } - ev_base = rspamd_main->ev_base; + ev_base = rspamd_main->event_loop; ctx = g_malloc0 (sizeof (*ctx)); http = rspamd_http_router_new (rspamadm_lua_error_handler, rspamadm_lua_finish_handler, - NULL, + 0.0, NULL, rspamd_main->http_ctx); ctx->L = L; @@ -822,19 +823,17 @@ rspamadm_lua (gint argc, gchar **argv, const struct rspamadm_command *cmd) fd = rspamd_inet_address_listen (addr, SOCK_STREAM, TRUE); if (fd != -1) { - struct event *ev; + static ev_io ev; - ev = g_malloc0 (sizeof (*ev)); - event_set (ev, fd, EV_READ|EV_PERSIST, rspamadm_lua_accept_cb, - ctx); - event_base_set (ev_base, ev); - event_add (ev, NULL); + ev.data = ctx; + ev_io_init (&ev, rspamadm_lua_accept_cb, fd, EV_READ); + ev_io_start (ev_base, &ev); rspamd_printf ("listen on %s\n", rspamd_inet_address_to_string_pretty (addr)); } } - event_base_loop (ev_base, 0); + ev_loop (ev_base, 0); exit (EXIT_SUCCESS); } diff --git a/src/rspamadm/rspamadm.c b/src/rspamadm/rspamadm.c index 8096649f9..36b06ade5 100644 --- a/src/rspamadm/rspamadm.c +++ b/src/rspamadm/rspamadm.c @@ -21,6 +21,7 @@ #include "lua/lua_thread_pool.h" #include "lua_ucl.h" #include "unix-std.h" +#include "contrib/libev/ev.h" #ifdef HAVE_LIBUTIL_H #include <libutil.h> @@ -329,7 +330,7 @@ static void rspamadm_add_lua_globals (struct rspamd_dns_resolver *resolver) { struct rspamd_async_session **psession; - struct event_base **pev_base; + struct ev_loop **pev_base; struct rspamd_dns_resolver **presolver; rspamadm_session = rspamd_session_create (rspamd_main->cfg->cfg_pool, NULL, @@ -340,9 +341,9 @@ rspamadm_add_lua_globals (struct rspamd_dns_resolver *resolver) *psession = rspamadm_session; lua_setglobal (L, "rspamadm_session"); - pev_base = lua_newuserdata (L, sizeof (struct event_base *)); + pev_base = lua_newuserdata (L, sizeof (struct ev_loop *)); rspamd_lua_setclass (L, "rspamd{ev_base}", -1); - *pev_base = rspamd_main->ev_base; + *pev_base = rspamd_main->event_loop; lua_setglobal (L, "rspamadm_ev_base"); presolver = lua_newuserdata (L, sizeof (struct rspamd_dns_resolver *)); @@ -379,15 +380,6 @@ main (gint argc, gchar **argv, gchar **env) rspamd_main->server_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "rspamadm"); -#ifdef HAVE_EVENT_NO_CACHE_TIME_FLAG - struct event_config *ev_cfg; - ev_cfg = event_config_new (); - event_config_set_flag (ev_cfg, EVENT_BASE_FLAG_NO_CACHE_TIME); - rspamd_main->ev_base = event_base_new_with_config (ev_cfg); -#else - rspamd_main->ev_base = event_init (); -#endif - rspamadm_fill_internal_commands (all_commands); help_command.command_data = all_commands; @@ -443,10 +435,12 @@ main (gint argc, gchar **argv, gchar **env) rspamd_main->server_pool); (void) rspamd_log_open (rspamd_main->logger); + rspamd_main->event_loop = ev_default_loop (EVFLAG_SIGNALFD|EVBACKEND_ALL); + resolver = rspamd_dns_resolver_init (rspamd_main->logger, - rspamd_main->ev_base, + rspamd_main->event_loop, cfg); - rspamd_main->http_ctx = rspamd_http_context_create (cfg, rspamd_main->ev_base, + rspamd_main->http_ctx = rspamd_http_context_create (cfg, rspamd_main->event_loop, NULL); g_log_set_default_handler (rspamd_glib_log_function, rspamd_main->logger); @@ -481,7 +475,7 @@ main (gint argc, gchar **argv, gchar **env) rspamadm_add_lua_globals (resolver); #ifdef WITH_HIREDIS - rspamd_redis_pool_config (cfg->redis_pool, cfg, rspamd_main->ev_base); + rspamd_redis_pool_config (cfg->redis_pool, cfg, rspamd_main->event_loop); #endif /* Init rspamadm global */ @@ -565,10 +559,8 @@ main (gint argc, gchar **argv, gchar **env) cmd->run (0, NULL, cmd); } - event_base_loopexit (rspamd_main->ev_base, NULL); -#ifdef HAVE_EVENT_NO_CACHE_TIME_FLAG - event_config_free (ev_cfg); -#endif + ev_break (rspamd_main->event_loop, EVBREAK_ALL); + REF_RELEASE (rspamd_main->cfg); rspamd_log_close (rspamd_main->logger, TRUE); diff --git a/src/rspamd.c b/src/rspamd.c index 8b12fa48e..fb87df06f 100644 --- a/src/rspamd.c +++ b/src/rspamd.c @@ -59,9 +59,11 @@ #ifdef HAVE_OPENSSL #include <openssl/err.h> #include <openssl/evp.h> + #endif #include "sqlite3.h" +#include "contrib/libev/ev.h" /* 2 seconds to fork new process in place of dead one */ #define SOFT_FORK_TIME 2 @@ -70,13 +72,19 @@ #define TERMINATION_ATTEMPTS 50 static gboolean load_rspamd_config (struct rspamd_main *rspamd_main, - struct rspamd_config *cfg, - gboolean init_modules, - enum rspamd_post_load_options opts, - gboolean reload); + struct rspamd_config *cfg, + gboolean init_modules, + enum rspamd_post_load_options opts, + gboolean reload); +static void rspamd_cld_handler (EV_P_ ev_child *w, + struct rspamd_main *rspamd_main, + struct rspamd_worker *wrk); /* Control socket */ static gint control_fd; +static ev_io control_ev; + +static gboolean valgrind_mode = FALSE; /* Cmdline options */ static gboolean config_test = FALSE; @@ -100,9 +108,6 @@ static gboolean skip_template = FALSE; static gint term_attempts = 0; -/* List of unrelated forked processes */ -static GArray *other_workers = NULL; - /* List of active listen sockets indexed by worker type */ static GHashTable *listen_sockets = NULL; @@ -186,8 +191,7 @@ read_cmd_line (gint *argc, gchar ***argv, struct rspamd_config *cfg) { GError *error = NULL; GOptionContext *context; - guint i, cfg_num; - pid_t r; + guint cfg_num; context = g_option_context_new ("- run rspamd daemon"); #if defined(GIT_VERSION) && GIT_VERSION == 1 @@ -208,30 +212,13 @@ read_cmd_line (gint *argc, gchar ***argv, struct rspamd_config *cfg) cfg->rspamd_user = rspamd_user; cfg->rspamd_group = rspamd_group; cfg_num = cfg_names != NULL ? g_strv_length (cfg_names) : 0; + if (cfg_num == 0) { cfg->cfg_name = FIXED_CONFIG_FILE; } else { cfg->cfg_name = cfg_names[0]; - } - - for (i = 1; i < cfg_num; i++) { - r = fork (); - if (r == 0) { - /* Spawning new main process */ - cfg->cfg_name = cfg_names[i]; - (void)setsid (); - } - else if (r == -1) { - fprintf (stderr, - "fork failed while spawning process for %s configuration file: %s\n", - cfg_names[i], - strerror (errno)); - } - else { - /* Save pid to the list of other main processes, we need it to ignore SIGCHLD from them */ - g_array_append_val (other_workers, r); - } + g_assert (cfg_num == 1); } cfg->pid_file = rspamd_pidfile; @@ -358,21 +345,23 @@ reread_config (struct rspamd_main *rspamd_main) struct waiting_worker { struct rspamd_main *rspamd_main; - struct event wait_ev; + struct ev_timer wait_ev; struct rspamd_worker_conf *cf; guint oldindex; }; static void -rspamd_fork_delayed_cb (gint signo, short what, gpointer arg) +rspamd_fork_delayed_cb (EV_P_ ev_timer *w, int revents) { - struct waiting_worker *w = arg; - - event_del (&w->wait_ev); - rspamd_fork_worker (w->rspamd_main, w->cf, w->oldindex, - w->rspamd_main->ev_base); - REF_RELEASE (w->cf); - g_free (w); + struct waiting_worker *waiting_worker = (struct waiting_worker *)w->data; + + ev_timer_stop (EV_A_ &waiting_worker->wait_ev); + rspamd_fork_worker (waiting_worker->rspamd_main, waiting_worker->cf, + waiting_worker->oldindex, + waiting_worker->rspamd_main->event_loop, + rspamd_cld_handler); + REF_RELEASE (waiting_worker->cf); + g_free (waiting_worker); } static void @@ -390,9 +379,9 @@ rspamd_fork_delayed (struct rspamd_worker_conf *cf, tv.tv_sec = SOFT_FORK_TIME; tv.tv_usec = 0; REF_RETAIN (cf); - event_set (&nw->wait_ev, -1, EV_TIMEOUT, rspamd_fork_delayed_cb, nw); - event_base_set (rspamd_main->ev_base, &nw->wait_ev); - event_add (&nw->wait_ev, &tv); + nw->wait_ev.data = nw; + ev_timer_init (&nw->wait_ev, rspamd_fork_delayed_cb, SOFT_FORK_TIME, 0.0); + ev_timer_start (rspamd_main->event_loop, &nw->wait_ev); } static GList * @@ -552,7 +541,7 @@ make_listen_key (struct rspamd_worker_bind_conf *cf) } static void -spawn_worker_type (struct rspamd_main *rspamd_main, struct event_base *ev_base, +spawn_worker_type (struct rspamd_main *rspamd_main, struct ev_loop *event_loop, struct rspamd_worker_conf *cf) { gint i; @@ -569,20 +558,21 @@ spawn_worker_type (struct rspamd_main *rspamd_main, struct event_base *ev_base, "cannot spawn more than 1 %s worker, so spawn one", cf->worker->name); } - rspamd_fork_worker (rspamd_main, cf, 0, ev_base); + rspamd_fork_worker (rspamd_main, cf, 0, event_loop, rspamd_cld_handler); } else if (cf->worker->flags & RSPAMD_WORKER_THREADED) { - rspamd_fork_worker (rspamd_main, cf, 0, ev_base); + rspamd_fork_worker (rspamd_main, cf, 0, event_loop, rspamd_cld_handler); } else { for (i = 0; i < cf->count; i++) { - rspamd_fork_worker (rspamd_main, cf, i, ev_base); + rspamd_fork_worker (rspamd_main, cf, i, event_loop, + rspamd_cld_handler); } } } static void -spawn_workers (struct rspamd_main *rspamd_main, struct event_base *ev_base) +spawn_workers (struct rspamd_main *rspamd_main, struct ev_loop *ev_base) { GList *cur, *ls; struct rspamd_worker_conf *cf; @@ -718,6 +708,7 @@ kill_old_workers (gpointer key, gpointer value, gpointer unused) if (!w->wanna_die) { w->wanna_die = TRUE; kill (w->pid, SIGUSR2); + ev_io_stop (rspamd_main->event_loop, &w->srv_ev); msg_info_main ("send signal to worker %P", w->pid); } else { @@ -725,95 +716,55 @@ kill_old_workers (gpointer key, gpointer value, gpointer unused) } } -static gboolean -wait_for_workers (gpointer key, gpointer value, gpointer unused) +static void +rspamd_worker_wait (struct rspamd_worker *w) { - struct rspamd_worker *w = value; struct rspamd_main *rspamd_main; - gint res = 0; - gboolean nowait = FALSE; - rspamd_main = w->srv; - if (w->ppid != getpid ()) { - nowait = TRUE; - } - - if (nowait || waitpid (w->pid, &res, WNOHANG) <= 0) { - if (term_attempts < 0) { - if (w->cf->worker->flags & RSPAMD_WORKER_KILLABLE) { - msg_warn_main ("terminate worker %s(%P) with SIGKILL", - g_quark_to_string (w->type), w->pid); - if (kill (w->pid, SIGKILL) == -1) { - if (nowait && errno == ESRCH) { - /* We have actually killed the process */ - goto finished; - } + if (term_attempts < 0) { + if (w->cf->worker->flags & RSPAMD_WORKER_KILLABLE) { + msg_warn_main ("terminate worker %s(%P) with SIGKILL", + g_quark_to_string (w->type), w->pid); + if (kill (w->pid, SIGKILL) == -1) { + if (errno == ESRCH) { + /* We have actually killed the process */ + return; } } - else { - if (term_attempts > -(TERMINATION_ATTEMPTS * 2)) { - if (term_attempts % 10 == 0) { - msg_info_main ("waiting for worker %s(%P) to sync, " - "%d seconds remain", - g_quark_to_string (w->type), w->pid, - (TERMINATION_ATTEMPTS * 2 + term_attempts) / 5); - kill (w->pid, SIGTERM); - if (nowait && errno == ESRCH) { - /* We have actually killed the process */ - goto finished; - } - } - } - else { - msg_err_main ("data corruption warning: terminating " - "special worker %s(%P) with SIGKILL", - g_quark_to_string (w->type), w->pid); - kill (w->pid, SIGKILL); - if (nowait && errno == ESRCH) { + } + else { + if (term_attempts > -(TERMINATION_ATTEMPTS * 2)) { + if (term_attempts % 10 == 0) { + msg_info_main ("waiting for worker %s(%P) to sync, " + "%d seconds remain", + g_quark_to_string (w->type), w->pid, + (TERMINATION_ATTEMPTS * 2 + term_attempts) / 5); + kill (w->pid, SIGTERM); + if (errno == ESRCH) { /* We have actually killed the process */ - goto finished; + return; } } } - } - else if (nowait) { - kill (w->pid, 0); - - if (errno != ESRCH) { - return FALSE; - } else { - goto finished; + msg_err_main ("data corruption warning: terminating " + "special worker %s(%P) with SIGKILL", + g_quark_to_string (w->type), w->pid); + kill (w->pid, SIGKILL); + if (errno == ESRCH) { + /* We have actually killed the process */ + return; + } } } - - return FALSE; } +} - - - finished: - msg_info_main ("%s process %P terminated %s", - g_quark_to_string (w->type), w->pid, - nowait ? "with no result available" : - (WTERMSIG (res) == SIGKILL ? "hardly" : "softly")); - if (w->srv_pipe[0] != -1) { - /* Ugly workaround */ - if (w->tmp_data) { - g_free (w->tmp_data); - } - event_del (&w->srv_ev); - } - - if (w->finish_actions) { - g_ptr_array_free (w->finish_actions, TRUE); - } - - REF_RELEASE (w->cf); - g_free (w); - - return TRUE; +static void +hash_worker_wait_callback (gpointer key, gpointer value, gpointer unused) +{ + rspamd_worker_wait ((struct rspamd_worker *)value); } struct core_check_cbdata { @@ -982,213 +933,161 @@ do_encrypt_password (void) rspamd_fprintf (stderr, "use rspamadm pw for this operation\n"); } +static void +stop_srv_ev (gpointer key, gpointer value, gpointer ud) +{ + struct rspamd_worker *cur = (struct rspamd_worker *)value; + struct rspamd_main *rspamd_main = (struct rspamd_main *)ud; + + ev_io_stop (rspamd_main->event_loop, &cur->srv_ev); +} + +static void +rspamd_final_timer_handler (EV_P_ ev_timer *w, int revents) +{ + struct rspamd_main *rspamd_main = (struct rspamd_main *)w->data; + + term_attempts--; + + g_hash_table_foreach (rspamd_main->workers, hash_worker_wait_callback, NULL); + + if (g_hash_table_size (rspamd_main->workers) == 0) { + ev_break (rspamd_main->event_loop, EVBREAK_ALL); + } +} + /* Signal handlers */ static void -rspamd_term_handler (gint signo, short what, gpointer arg) +rspamd_term_handler (struct ev_loop *loop, ev_signal *w, int revents) { - struct rspamd_main *rspamd_main = arg; + struct rspamd_main *rspamd_main = (struct rspamd_main *)w->data; + static ev_timer ev_finale; + + if (!rspamd_main->wanna_die) { + rspamd_main->wanna_die = TRUE; + msg_info_main ("catch termination signal, waiting for children"); + rspamd_log_nolock (rspamd_main->logger); + /* Stop srv events to avoid false notifications */ + g_hash_table_foreach (rspamd_main->workers, stop_srv_ev, rspamd_main); + rspamd_pass_signal (rspamd_main->workers, SIGTERM); + + if (control_fd != -1) { + ev_io_stop (rspamd_main->event_loop, &control_ev); + close (control_fd); + } - msg_info_main ("catch termination signal, waiting for children"); - rspamd_log_nolock (rspamd_main->logger); - rspamd_pass_signal (rspamd_main->workers, signo); + if (valgrind_mode) { + /* Special case if we are likely running with valgrind */ + term_attempts = TERMINATION_ATTEMPTS * 10; + } + else { + term_attempts = TERMINATION_ATTEMPTS; + } - event_base_loopexit (rspamd_main->ev_base, NULL); + ev_finale.data = rspamd_main; + ev_timer_init (&ev_finale, rspamd_final_timer_handler, 0.2, 0.2); + ev_timer_start (rspamd_main->event_loop, &ev_finale); + } } static void -rspamd_usr1_handler (gint signo, short what, gpointer arg) +rspamd_usr1_handler (struct ev_loop *loop, ev_signal *w, int revents) { - struct rspamd_main *rspamd_main = arg; + struct rspamd_main *rspamd_main = (struct rspamd_main *)w->data; - rspamd_log_reopen_priv (rspamd_main->logger, - rspamd_main->workers_uid, - rspamd_main->workers_gid); - g_hash_table_foreach (rspamd_main->workers, reopen_log_handler, - NULL); + if (!rspamd_main->wanna_die) { + rspamd_log_reopen_priv (rspamd_main->logger, + rspamd_main->workers_uid, + rspamd_main->workers_gid); + g_hash_table_foreach (rspamd_main->workers, reopen_log_handler, + NULL); + } } static void -rspamd_hup_handler (gint signo, short what, gpointer arg) +rspamd_hup_handler (struct ev_loop *loop, ev_signal *w, int revents) { - struct rspamd_main *rspamd_main = arg; + struct rspamd_main *rspamd_main = (struct rspamd_main *)w->data; - msg_info_main ("rspamd " - RVERSION - " is restarting"); - g_hash_table_foreach (rspamd_main->workers, kill_old_workers, NULL); - rspamd_log_close_priv (rspamd_main->logger, + if (!rspamd_main->wanna_die) { + msg_info_main ("rspamd " + RVERSION + " is restarting"); + g_hash_table_foreach (rspamd_main->workers, kill_old_workers, NULL); + rspamd_log_close_priv (rspamd_main->logger, FALSE, rspamd_main->workers_uid, rspamd_main->workers_gid); - reread_config (rspamd_main); - rspamd_check_core_limits (rspamd_main); - spawn_workers (rspamd_main, rspamd_main->ev_base); + reread_config (rspamd_main); + rspamd_check_core_limits (rspamd_main); + spawn_workers (rspamd_main, rspamd_main->event_loop); + } } +/* Called when a dead child has been found */ + static void -rspamd_cld_handler (gint signo, short what, gpointer arg) +rspamd_cld_handler (EV_P_ ev_child *w, struct rspamd_main *rspamd_main, + struct rspamd_worker *wrk) { - struct rspamd_main *rspamd_main = arg; - guint i; - gint res = 0; - struct rspamd_worker *cur; - pid_t wrk; - gboolean need_refork = TRUE; + gboolean need_refork; /* Turn off locking for logger */ + ev_child_stop (EV_A_ w); rspamd_log_nolock (rspamd_main->logger); - msg_info_main ("catch SIGCHLD signal, finding terminated workers"); /* Remove dead child form children list */ - while ((wrk = waitpid (0, &res, WNOHANG)) > 0) { - if ((cur = - g_hash_table_lookup (rspamd_main->workers, - GSIZE_TO_POINTER (wrk))) != NULL) { - /* Unlink dead process from queue and hash table */ - - g_hash_table_remove (rspamd_main->workers, GSIZE_TO_POINTER ( - wrk)); - - if (cur->wanna_die) { - /* Do not refork workers that are intended to be terminated */ - need_refork = FALSE; - } - - if (WIFEXITED (res) && WEXITSTATUS (res) == 0) { - /* Normal worker termination, do not fork one more */ - msg_info_main ("%s process %P terminated normally", - g_quark_to_string (cur->type), - cur->pid); - } - else { - if (WIFSIGNALED (res)) { -#ifdef WCOREDUMP - if (WCOREDUMP (res)) { - msg_warn_main ( - "%s process %P terminated abnormally by signal: %s" - " and created core file", - g_quark_to_string (cur->type), - cur->pid, - g_strsignal (WTERMSIG (res))); - } - else { -#ifdef HAVE_SYS_RESOURCE_H - struct rlimit rlmt; - (void)getrlimit (RLIMIT_CORE, &rlmt); - - msg_warn_main ( - "%s process %P terminated abnormally by signal: %s" - " but NOT created core file (throttled=%s); " - "core file limits: %L current, %L max", - g_quark_to_string (cur->type), - cur->pid, - g_strsignal (WTERMSIG (res)), - cur->cores_throttled ? "yes" : "no", - (gint64)rlmt.rlim_cur, - (gint64)rlmt.rlim_max); -#else - msg_warn_main ( - "%s process %P terminated abnormally by signal: %s" - " but NOT created core file (throttled=%s); ", - g_quark_to_string (cur->type), - cur->pid, - g_strsignal (WTERMSIG (res)), - cur->cores_throttled ? "yes" : "no"); -#endif - } -#else - msg_warn_main ( - "%s process %P terminated abnormally by signal: %s", - g_quark_to_string (cur->type), - cur->pid, - g_strsignal (WTERMSIG (res))); -#endif - if (WTERMSIG (res) == SIGUSR2) { - /* - * It is actually race condition when not started process - * has been requested to be reloaded. - * - * We shouldn't refork on this - */ - need_refork = FALSE; - } - } - else { - msg_warn_main ("%s process %P terminated abnormally " - "with exit code %d", - g_quark_to_string (cur->type), - cur->pid, - WEXITSTATUS (res)); - } - - if (need_refork) { - /* Fork another worker in replace of dead one */ - rspamd_check_core_limits (rspamd_main); - - - rspamd_fork_delayed (cur->cf, cur->index, rspamd_main); - } - } - - if (cur->srv_pipe[0] != -1) { - /* Ugly workaround */ - if (cur->tmp_data) { - g_free (cur->tmp_data); - } - event_del (&cur->srv_ev); - } - - if (cur->control_pipe[0] != -1) { - /* We also need to clean descriptors left */ - close (cur->control_pipe[0]); - close (cur->srv_pipe[0]); - } - - REF_RELEASE (cur->cf); - - if (cur->finish_actions) { - g_ptr_array_free (cur->finish_actions, TRUE); - } - - g_free (cur); - } - else { - for (i = 0; i < other_workers->len; i++) { - if (g_array_index (other_workers, pid_t, i) == wrk) { - g_array_remove_index_fast (other_workers, i); - msg_info_main ("related process %P terminated", wrk); - } - } + g_hash_table_remove (rspamd_main->workers, GSIZE_TO_POINTER (wrk->pid)); + if (wrk->srv_pipe[0] != -1) { + /* Ugly workaround */ + if (wrk->tmp_data) { + g_free (wrk->tmp_data); } + ev_io_stop (rspamd_main->event_loop, &wrk->srv_ev); } - rspamd_log_lock (rspamd_main->logger); -} + if (wrk->control_pipe[0] != -1) { + /* We also need to clean descriptors left */ + close (wrk->control_pipe[0]); + close (wrk->srv_pipe[0]); + } -static void -rspamd_final_term_handler (gint signo, short what, gpointer arg) -{ - struct rspamd_main *rspamd_main = arg; + REF_RELEASE (wrk->cf); - term_attempts--; + if (wrk->finish_actions) { + g_ptr_array_free (wrk->finish_actions, TRUE); + } - g_hash_table_foreach_remove (rspamd_main->workers, wait_for_workers, NULL); + need_refork = rspamd_check_termination_clause (wrk->srv, wrk, w->rstatus); - if (g_hash_table_size (rspamd_main->workers) == 0) { - event_base_loopexit (rspamd_main->ev_base, NULL); + if (need_refork) { + /* Fork another worker in replace of dead one */ + msg_info_main ("respawn process %s in lieu of terminated process with pid %P", + g_quark_to_string (wrk->type), + wrk->pid); + rspamd_check_core_limits (rspamd_main); + rspamd_fork_delayed (wrk->cf, wrk->index, rspamd_main); + } + else { + msg_info_main ("do not respawn process %s after found terminated process with pid %P", + g_quark_to_string (wrk->type), + wrk->pid); } + + g_free (wrk); + rspamd_log_lock (rspamd_main->logger); } /* Control socket handler */ static void -rspamd_control_handler (gint fd, short what, gpointer arg) +rspamd_control_handler (EV_P_ ev_io *w, int revents) { - struct rspamd_main *rspamd_main = arg; + struct rspamd_main *rspamd_main = (struct rspamd_main *)w->data; rspamd_inet_addr_t *addr; gint nfd; if ((nfd = - rspamd_accept_from_socket (fd, &addr, NULL)) == -1) { + rspamd_accept_from_socket (w->fd, &addr, NULL, NULL)) == -1) { msg_warn_main ("accept failed: %s", strerror (errno)); return; } @@ -1243,11 +1142,9 @@ main (gint argc, gchar **argv, gchar **env) worker_t **pworker; GQuark type; rspamd_inet_addr_t *control_addr = NULL; - struct event_base *ev_base; - struct event term_ev, int_ev, cld_ev, hup_ev, usr1_ev, control_ev; - struct timeval term_tv; + struct ev_loop *event_loop; struct rspamd_main *rspamd_main; - gboolean skip_pid = FALSE, valgrind_mode = FALSE; + gboolean skip_pid = FALSE; #if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION <= 30)) g_thread_init (NULL); @@ -1273,7 +1170,6 @@ main (gint argc, gchar **argv, gchar **env) rspamd_main->cfg->libs_ctx = rspamd_init_libs (); memset (&signals, 0, sizeof (struct sigaction)); - other_workers = g_array_new (FALSE, TRUE, sizeof (pid_t)); read_cmd_line (&argc, &argv, rspamd_main->cfg); @@ -1462,7 +1358,6 @@ main (gint argc, gchar **argv, gchar **env) /* Set title */ setproctitle ("main process"); - /* Flush log */ rspamd_log_flush (rspamd_main->logger); @@ -1498,83 +1393,91 @@ main (gint argc, gchar **argv, gchar **env) rspamd_main->workers = g_hash_table_new (g_direct_hash, g_direct_equal); /* Init event base */ - ev_base = event_init (); - rspamd_main->ev_base = ev_base; + event_loop = ev_default_loop (EVFLAG_SIGNALFD|EVBACKEND_ALL); + rspamd_main->event_loop = event_loop; + + if (event_loop) { + unsigned loop_type = ev_backend (event_loop); + const gchar *loop_str = "unknown"; + gboolean poor_backend = TRUE; + + switch (loop_type) { + case EVBACKEND_EPOLL: + loop_str = "epoll"; + poor_backend = FALSE; + break; + case EVBACKEND_POLL: + loop_str = "poll"; + break; + case EVBACKEND_SELECT: + loop_str = "select"; + break; + case EVBACKEND_KQUEUE: + loop_str = "kqueue"; + poor_backend = FALSE; + break; + case EVBACKEND_PORT: + loop_str = "port"; + poor_backend = FALSE; + break; + case EVBACKEND_DEVPOLL: + loop_str = "/dev/poll"; + poor_backend = FALSE; + break; + default: + break; + } + + if (poor_backend) { + msg_warn_main ("event loop uses non-optimal backend: %s", loop_str); + } + else { + msg_info_main ("event loop initialised with backend: %s", loop_str); + } + } + else { + msg_err ("cannot init event loop! exiting"); + exit (EXIT_FAILURE); + } + /* Unblock signals */ sigemptyset (&signals.sa_mask); sigprocmask (SIG_SETMASK, &signals.sa_mask, NULL); /* Set events for signals */ - evsignal_set (&term_ev, SIGTERM, rspamd_term_handler, rspamd_main); - event_base_set (ev_base, &term_ev); - event_add (&term_ev, NULL); - evsignal_set (&int_ev, SIGINT, rspamd_term_handler, rspamd_main); - event_base_set (ev_base, &int_ev); - event_add (&int_ev, NULL); - evsignal_set (&hup_ev, SIGHUP, rspamd_hup_handler, rspamd_main); - event_base_set (ev_base, &hup_ev); - event_add (&hup_ev, NULL); - evsignal_set (&cld_ev, SIGCHLD, rspamd_cld_handler, rspamd_main); - event_base_set (ev_base, &cld_ev); - event_add (&cld_ev, NULL); - evsignal_set (&usr1_ev, SIGUSR1, rspamd_usr1_handler, rspamd_main); - event_base_set (ev_base, &usr1_ev); - event_add (&usr1_ev, NULL); + ev_signal_init (&rspamd_main->term_ev, rspamd_term_handler, SIGTERM); + rspamd_main->term_ev.data = rspamd_main; + ev_signal_start (event_loop, &rspamd_main->term_ev); + + ev_signal_init (&rspamd_main->int_ev, rspamd_term_handler, SIGINT); + rspamd_main->int_ev.data = rspamd_main; + ev_signal_start (event_loop, &rspamd_main->int_ev); + + ev_signal_init (&rspamd_main->hup_ev, rspamd_hup_handler, SIGHUP); + rspamd_main->hup_ev.data = rspamd_main; + ev_signal_start (event_loop, &rspamd_main->hup_ev); + + ev_signal_init (&rspamd_main->usr1_ev, rspamd_usr1_handler, SIGUSR1); + rspamd_main->usr1_ev.data = rspamd_main; + ev_signal_start (event_loop, &rspamd_main->usr1_ev); rspamd_check_core_limits (rspamd_main); rspamd_mempool_lock_mutex (rspamd_main->start_mtx); - spawn_workers (rspamd_main, ev_base); + spawn_workers (rspamd_main, event_loop); rspamd_mempool_unlock_mutex (rspamd_main->start_mtx); rspamd_main->http_ctx = rspamd_http_context_create (rspamd_main->cfg, - ev_base, rspamd_main->cfg->ups_ctx); + event_loop, rspamd_main->cfg->ups_ctx); if (control_fd != -1) { msg_info_main ("listening for control commands on %s", rspamd_inet_address_to_string (control_addr)); - event_set (&control_ev, control_fd, EV_READ|EV_PERSIST, - rspamd_control_handler, rspamd_main); - event_base_set (ev_base, &control_ev); - event_add (&control_ev, NULL); + ev_io_init (&control_ev, rspamd_control_handler, control_fd, EV_READ); + control_ev.data = rspamd_main; + ev_io_start (event_loop, &control_ev); } - event_base_loop (ev_base, 0); - /* We need to block signals unless children are waited for */ - rspamd_worker_block_signals (); - - event_del (&term_ev); - event_del (&int_ev); - event_del (&hup_ev); - event_del (&cld_ev); - event_del (&usr1_ev); - - if (control_fd != -1) { - event_del (&control_ev); - close (control_fd); - } - - if (valgrind_mode) { - /* Special case if we are likely running with valgrind */ - term_attempts = TERMINATION_ATTEMPTS * 10; - } - else { - term_attempts = TERMINATION_ATTEMPTS; - } - - /* Check each 200 ms */ - term_tv.tv_sec = 0; - term_tv.tv_usec = 200000; - - /* Wait for workers termination */ - g_hash_table_foreach_remove (rspamd_main->workers, wait_for_workers, NULL); - - event_set (&term_ev, -1, EV_TIMEOUT|EV_PERSIST, - rspamd_final_term_handler, rspamd_main); - event_base_set (ev_base, &term_ev); - event_add (&term_ev, &term_tv); - - event_base_loop (ev_base, 0); - event_del (&term_ev); + ev_loop (event_loop, 0); /* Maybe save roll history */ if (rspamd_main->cfg->history_file) { @@ -1595,7 +1498,7 @@ main (gint argc, gchar **argv, gchar **env) } g_free (rspamd_main); - event_base_free (ev_base); + ev_unref (event_loop); sqlite3_shutdown (); if (control_addr) { diff --git a/src/rspamd.h b/src/rspamd.h index 10d3be9fb..fff373397 100644 --- a/src/rspamd.h +++ b/src/rspamd.h @@ -27,7 +27,7 @@ #include "libutil/radix.h" #include "libserver/url.h" #include "libserver/protocol.h" -#include "libserver/events.h" +#include "libserver/async_session.h" #include "libserver/roll_history.h" #include "libserver/task.h" #include <openssl/ssl.h> @@ -62,6 +62,15 @@ enum rspamd_worker_flags { RSPAMD_WORKER_CONTROLLER = (1 << 6), }; +struct rspamd_worker_accept_event { + ev_io accept_ev; + ev_timer throttling_ev; + struct ev_loop *event_loop; + struct rspamd_worker_accept_event *prev, *next; +}; + +typedef void (*rspamd_worker_term_cb)(EV_P_ ev_child *, struct rspamd_main *, + struct rspamd_worker *); /** * Worker process structure @@ -77,7 +86,7 @@ struct rspamd_worker { struct rspamd_main *srv; /**< pointer to server structure */ GQuark type; /**< process type */ GHashTable *signal_events; /**< signal events */ - GList *accept_events; /**< socket events */ + struct rspamd_worker_accept_event *accept_events; /**< socket events */ struct rspamd_worker_conf *cf; /**< worker config data */ gpointer ctx; /**< worker's specific data */ enum rspamd_worker_flags flags; /**< worker's flags */ @@ -85,16 +94,18 @@ struct rspamd_worker { [1] is used by a worker */ gint srv_pipe[2]; /**< used by workers to request something from the main process. [0] - main, [1] - worker */ - struct event srv_ev; /**< used by main for read workers' requests */ + ev_io srv_ev; /**< used by main for read workers' requests */ gpointer control_data; /**< used by control protocol to handle commands */ gpointer tmp_data; /**< used to avoid race condition to deal with control messages */ GPtrArray *finish_actions; /**< called when worker is terminated */ + ev_child cld_ev; /**< to allow reaping */ + rspamd_worker_term_cb term_handler; /**< custom term handler */ }; struct rspamd_abstract_worker_ctx { guint64 magic; /* Events base */ - struct event_base *ev_base; + struct ev_loop *event_loop; /* DNS resolver */ struct rspamd_dns_resolver *resolver; /* Config */ @@ -115,8 +126,8 @@ struct rspamd_worker_signal_cb { struct rspamd_worker_signal_handler { gint signo; gboolean enabled; - struct event ev; - struct event_base *base; + ev_signal ev_sig; + struct ev_loop *event_loop; struct rspamd_worker *worker; struct rspamd_worker_signal_cb *cb; }; @@ -274,9 +285,11 @@ struct rspamd_main { uid_t workers_uid; /**< worker's uid running to */ gid_t workers_gid; /**< worker's gid running to */ gboolean is_privilleged; /**< true if run in privilleged mode */ + gboolean wanna_die; /**< no respawn of processes */ gboolean cores_throttling; /**< turn off cores when limits are exceeded */ struct roll_history *history; /**< rolling history */ - struct event_base *ev_base; + struct ev_loop *event_loop; + ev_signal term_ev, int_ev, hup_ev, usr1_ev; /**< signals */ struct rspamd_http_context *http_ctx; }; @@ -311,7 +324,7 @@ struct controller_session { GList *parts; /**< extracted mime parts */ struct rspamd_async_session * s; /**< async session object */ struct rspamd_dns_resolver *resolver; /**< DNS resolver */ - struct event_base *ev_base; /**< Event base */ + struct ev_loop *ev_base; /**< Event base */ }; struct zstd_dictionary { diff --git a/src/rspamd_proxy.c b/src/rspamd_proxy.c index deba18dab..9122df514 100644 --- a/src/rspamd_proxy.c +++ b/src/rspamd_proxy.c @@ -86,7 +86,6 @@ struct rspamd_http_upstream { struct upstream_list *u; struct rspamd_cryptobox_pubkey *key; gdouble timeout; - struct timeval io_tv; gint parser_from_ref; gint parser_to_ref; gboolean local; @@ -101,7 +100,6 @@ struct rspamd_http_mirror { struct rspamd_cryptobox_pubkey *key; gdouble prob; gdouble timeout; - struct timeval io_tv; gint parser_from_ref; gint parser_to_ref; gboolean local; @@ -113,14 +111,13 @@ static const guint64 rspamd_rspamd_proxy_magic = 0xcdeb4fd1fc351980ULL; struct rspamd_proxy_ctx { guint64 magic; /* Events base */ - struct event_base *ev_base; + struct ev_loop *event_loop; /* DNS resolver */ struct rspamd_dns_resolver *resolver; /* Config */ struct rspamd_config *cfg; /* END OF COMMON PART */ gdouble timeout; - struct timeval io_tv; /* Encryption key for clients */ struct rspamd_cryptobox_keypair *key; /* HTTP context */ @@ -174,8 +171,8 @@ struct rspamd_proxy_backend_connection { ucl_object_t *results; const gchar *err; struct rspamd_proxy_session *s; - struct timeval *io_tv; gint backend_sock; + ev_tstamp timeout; enum rspamd_backend_flags flags; gint parser_from_ref; gint parser_to_ref; @@ -464,8 +461,6 @@ rspamd_proxy_parse_upstream (rspamd_mempool_t *pool, rspamd_lua_add_ref_dtor (L, pool, up->parser_to_ref); } - double_to_tv (up->timeout, &up->io_tv); - g_hash_table_insert (ctx->upstreams, up->name, up); return TRUE; @@ -617,8 +612,6 @@ rspamd_proxy_parse_mirror (rspamd_mempool_t *pool, up->settings_id = rspamd_mempool_strdup (pool, ucl_object_tostring (elt)); } - double_to_tv (up->timeout, &up->io_tv); - g_ptr_array_add (ctx->mirrors, up); return TRUE; @@ -1144,8 +1137,6 @@ proxy_request_decompress (struct rspamd_http_message *msg) rspamd_http_message_set_body_from_fstring_steal (msg, body); rspamd_http_message_remove_header (msg, "Compression"); } - - return; } static struct rspamd_proxy_session * @@ -1350,7 +1341,7 @@ proxy_open_mirror_connections (struct rspamd_proxy_session *session) sizeof (*bk_conn)); bk_conn->s = session; bk_conn->name = m->name; - bk_conn->io_tv = &m->io_tv; + bk_conn->timeout = m->timeout; bk_conn->up = rspamd_upstream_get (m->u, RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0); @@ -1415,7 +1406,7 @@ proxy_open_mirror_connections (struct rspamd_proxy_session *session) msg->method = HTTP_GET; rspamd_http_connection_write_message_shared (bk_conn->backend_conn, msg, NULL, NULL, bk_conn, - bk_conn->io_tv); + bk_conn->timeout); } else { if (session->fname) { @@ -1442,7 +1433,7 @@ proxy_open_mirror_connections (struct rspamd_proxy_session *session) rspamd_http_connection_write_message (bk_conn->backend_conn, msg, NULL, NULL, bk_conn, - bk_conn->io_tv); + bk_conn->timeout); } g_ptr_array_add (session->mirror_conns, bk_conn); @@ -1468,7 +1459,7 @@ proxy_client_write_error (struct rspamd_proxy_session *session, gint code, reply->status = rspamd_fstring_new_init (status, strlen (status)); rspamd_http_connection_write_message (session->client_conn, reply, NULL, NULL, session, - &session->ctx->io_tv); + session->ctx->timeout); } } @@ -1566,7 +1557,7 @@ proxy_backend_master_finish_handler (struct rspamd_http_connection *conn, else { rspamd_http_connection_write_message (session->client_conn, msg, NULL, NULL, session, - bk_conn->io_tv); + bk_conn->timeout); } return 0; @@ -1625,7 +1616,7 @@ rspamd_proxy_scan_self_reply (struct rspamd_task *task) NULL, ctype, session, - NULL); + 0); } } @@ -1666,7 +1657,7 @@ rspamd_proxy_self_scan (struct rspamd_proxy_session *session) msg = session->client_message; task = rspamd_task_new (session->worker, session->ctx->cfg, session->pool, session->ctx->lang_det, - session->ctx->ev_base); + session->ctx->event_loop); task->flags |= RSPAMD_TASK_FLAG_MIME; task->sock = -1; @@ -1711,23 +1702,18 @@ rspamd_proxy_self_scan (struct rspamd_proxy_session *session) /* Set global timeout for the task */ if (session->ctx->default_upstream->timeout > 0.0) { - struct timeval task_tv; + task->timeout_ev.data = task; + ev_timer_init (&task->timeout_ev, rspamd_task_timeout, + session->ctx->default_upstream->timeout, 0.0); + ev_timer_start (task->event_loop, &task->timeout_ev); - event_set (&task->timeout_ev, -1, EV_TIMEOUT, rspamd_task_timeout, - task); - event_base_set (session->ctx->ev_base, &task->timeout_ev); - double_to_tv (session->ctx->default_upstream->timeout, &task_tv); - event_add (&task->timeout_ev, &task_tv); } else if (session->ctx->has_self_scan) { if (session->ctx->cfg->task_timeout > 0) { - struct timeval task_tv; - - event_set (&task->timeout_ev, -1, EV_TIMEOUT, rspamd_task_timeout, - task); - event_base_set (session->ctx->ev_base, &task->timeout_ev); - double_to_tv (session->ctx->cfg->task_timeout, &task_tv); - event_add (&task->timeout_ev, &task_tv); + task->timeout_ev.data = task; + ev_timer_init (&task->timeout_ev, rspamd_task_timeout, + session->ctx->cfg->task_timeout, 0.0); + ev_timer_start (task->event_loop, &task->timeout_ev); } } @@ -1783,7 +1769,7 @@ retry: session->master_conn->up = rspamd_upstream_get (backend->u, RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0); - session->master_conn->io_tv = &backend->io_tv; + session->master_conn->timeout = backend->timeout; if (session->master_conn->up == NULL) { msg_err_session ("cannot select upstream for %s", @@ -1853,7 +1839,7 @@ retry: rspamd_http_connection_write_message_shared ( session->master_conn->backend_conn, msg, NULL, NULL, session->master_conn, - session->master_conn->io_tv); + session->master_conn->timeout); } else { if (session->fname) { @@ -1881,7 +1867,7 @@ retry: rspamd_http_connection_write_message ( session->master_conn->backend_conn, msg, NULL, NULL, session->master_conn, - session->master_conn->io_tv); + session->master_conn->timeout); } } @@ -2031,9 +2017,9 @@ proxy_milter_error_handler (gint fd, } static void -proxy_accept_socket (gint fd, short what, void *arg) +proxy_accept_socket (EV_P_ ev_io *w, int revents) { - struct rspamd_worker *worker = (struct rspamd_worker *) arg; + struct rspamd_worker *worker = (struct rspamd_worker *)w->data; struct rspamd_proxy_ctx *ctx; rspamd_inet_addr_t *addr; struct rspamd_proxy_session *session; @@ -2042,7 +2028,8 @@ proxy_accept_socket (gint fd, short what, void *arg) ctx = worker->ctx; if ((nfd = - rspamd_accept_from_socket (fd, &addr, worker->accept_events)) == -1) { + rspamd_accept_from_socket (w->fd, &addr, + rspamd_worker_throttle_accept_events, worker->accept_events)) == -1) { msg_warn ("accept failed: %s", strerror (errno)); return; } @@ -2086,7 +2073,7 @@ proxy_accept_socket (gint fd, short what, void *arg) rspamd_http_connection_read_message_shared (session->client_conn, session, - &ctx->io_tv); + session->ctx->timeout); } else { msg_info_session ("accepted milter connection from %s port %d", @@ -2110,9 +2097,9 @@ proxy_accept_socket (gint fd, short what, void *arg) } #endif - rspamd_milter_handle_socket (nfd, NULL, + rspamd_milter_handle_socket (nfd, 0.0, session->pool, - ctx->ev_base, + ctx->event_loop, proxy_milter_finish_handler, proxy_milter_error_handler, session); @@ -2153,30 +2140,30 @@ start_rspamd_proxy (struct rspamd_worker *worker) struct rspamd_proxy_ctx *ctx = worker->ctx; ctx->cfg = worker->srv->cfg; - ctx->ev_base = rspamd_prepare_worker (worker, "rspamd_proxy", + ctx->event_loop = rspamd_prepare_worker (worker, "rspamd_proxy", proxy_accept_socket); ctx->resolver = rspamd_dns_resolver_init (worker->srv->logger, - ctx->ev_base, + ctx->event_loop, worker->srv->cfg); - double_to_tv (ctx->timeout, &ctx->io_tv); - rspamd_map_watch (worker->srv->cfg, ctx->ev_base, ctx->resolver, worker, 0); + rspamd_map_watch (worker->srv->cfg, ctx->event_loop, ctx->resolver, worker, 0); rspamd_upstreams_library_config (worker->srv->cfg, ctx->cfg->ups_ctx, - ctx->ev_base, ctx->resolver->r); + ctx->event_loop, ctx->resolver->r); - ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base, + ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->event_loop, ctx->cfg->ups_ctx); if (ctx->has_self_scan) { /* Additional initialisation needed */ - rspamd_worker_init_scanner (worker, ctx->ev_base, ctx->resolver, + rspamd_worker_init_scanner (worker, ctx->event_loop, ctx->resolver, &ctx->lang_det); + } if (worker->srv->cfg->enable_sessions_cache) { ctx->sessions_cache = rspamd_worker_session_cache_new (worker, - ctx->ev_base); + ctx->event_loop); } ctx->milter_ctx.spam_header = ctx->spam_header; @@ -2188,11 +2175,11 @@ start_rspamd_proxy (struct rspamd_worker *worker) ctx->milter_ctx.cfg = ctx->cfg; rspamd_milter_init_library (&ctx->milter_ctx); - rspamd_lua_run_postloads (ctx->cfg->lua_state, ctx->cfg, ctx->ev_base, + rspamd_lua_run_postloads (ctx->cfg->lua_state, ctx->cfg, ctx->event_loop, worker); adjust_upstreams_limits (ctx); - event_base_loop (ctx->ev_base, 0); + ev_loop (ctx->event_loop, 0); rspamd_worker_block_signals (); if (ctx->has_self_scan) { diff --git a/src/worker.c b/src/worker.c index 5a01c6bd3..446de0799 100644 --- a/src/worker.c +++ b/src/worker.c @@ -42,7 +42,7 @@ #include "lua/lua_common.h" /* 60 seconds for worker's IO */ -#define DEFAULT_WORKER_IO_TIMEOUT 60000 +#define DEFAULT_WORKER_IO_TIMEOUT 60.0 gpointer init_worker (struct rspamd_config *cfg); void start_worker (struct rspamd_worker *worker); @@ -73,11 +73,10 @@ static gboolean rspamd_worker_finalize (gpointer user_data) { struct rspamd_task *task = user_data; - struct timeval tv = {.tv_sec = 0, .tv_usec = 0}; if (!(task->flags & RSPAMD_TASK_FLAG_PROCESSING)) { msg_info_task ("finishing actions has been processed, terminating"); - event_base_loopexit (task->ev_base, &tv); + ev_break (task->event_loop, EVBREAK_ALL); rspamd_session_destroy (task->s); return TRUE; @@ -97,7 +96,7 @@ rspamd_worker_call_finish_handlers (struct rspamd_worker *worker) if (cfg->on_term_scripts) { ctx = worker->ctx; /* Create a fake task object for async events */ - task = rspamd_task_new (worker, cfg, NULL, NULL, ctx->ev_base); + task = rspamd_task_new (worker, cfg, NULL, NULL, ctx->event_loop); task->resolver = ctx->resolver; task->flags |= RSPAMD_TASK_FLAG_PROCESSING; task->s = rspamd_session_create (task->task_pool, @@ -137,9 +136,9 @@ reduce_tasks_count (gpointer arg) } void -rspamd_task_timeout (gint fd, short what, gpointer ud) +rspamd_task_timeout (EV_P_ ev_timer *w, int revents) { - struct rspamd_task *task = (struct rspamd_task *) ud; + struct rspamd_task *task = (struct rspamd_task *)w->data; if (!(task->processed_stages & RSPAMD_TASK_STAGE_FILTERS)) { msg_info_task ("processing of task timed out, forced processing"); @@ -176,32 +175,13 @@ rspamd_task_timeout (gint fd, short what, gpointer ud) } void -rspamd_worker_guard_handler (gint fd, short what, void *data) +rspamd_worker_guard_handler (EV_P_ ev_io *w, int revents) { - struct rspamd_task *task = data; + struct rspamd_task *task = (struct rspamd_task *)w->data; gchar fake_buf[1024]; gssize r; -#ifdef EV_CLOSED - if (what == EV_CLOSED) { - if (!(task->flags & RSPAMD_TASK_FLAG_JSON) && - task->cfg->enable_shutdown_workaround) { - msg_info_task ("workaround for shutdown enabled, please update " - "your client, this support might be removed in future"); - shutdown (fd, SHUT_RD); - event_del (task->guard_ev); - task->guard_ev = NULL; - } - else { - msg_err_task ("the peer has closed connection unexpectedly"); - rspamd_session_destroy (task->s); - } - - return; - } -#endif - - r = read (fd, fake_buf, sizeof (fake_buf)); + r = read (w->fd, fake_buf, sizeof (fake_buf)); if (r > 0) { msg_warn_task ("received extra data after task is loaded, ignoring"); @@ -218,9 +198,8 @@ rspamd_worker_guard_handler (gint fd, short what, void *data) task->cfg->enable_shutdown_workaround) { msg_info_task ("workaround for shutdown enabled, please update " "your client, this support might be removed in future"); - shutdown (fd, SHUT_RD); - event_del (task->guard_ev); - task->guard_ev = NULL; + shutdown (w->fd, SHUT_RD); + ev_io_stop (task->event_loop, &task->guard_ev); } else { msg_err_task ("the peer has closed connection unexpectedly"); @@ -245,8 +224,6 @@ rspamd_worker_body_handler (struct rspamd_http_connection *conn, { struct rspamd_task *task = (struct rspamd_task *) conn->ud; struct rspamd_worker_ctx *ctx; - struct timeval task_tv; - struct event *guard_ev; ctx = task->worker->ctx; @@ -268,25 +245,16 @@ rspamd_worker_body_handler (struct rspamd_http_connection *conn, /* Set global timeout for the task */ if (ctx->task_timeout > 0.0) { - event_set (&task->timeout_ev, -1, EV_TIMEOUT, rspamd_task_timeout, - task); - event_base_set (ctx->ev_base, &task->timeout_ev); - double_to_tv (ctx->task_timeout, &task_tv); - event_add (&task->timeout_ev, &task_tv); + task->timeout_ev.data = task; + ev_timer_init (&task->timeout_ev, rspamd_task_timeout, + ctx->task_timeout, 0.0); + ev_timer_start (task->event_loop, &task->timeout_ev); } /* Set socket guard */ - guard_ev = rspamd_mempool_alloc (task->task_pool, sizeof (*guard_ev)); -#ifdef EV_CLOSED - event_set (guard_ev, task->sock, EV_READ|EV_PERSIST|EV_CLOSED, - rspamd_worker_guard_handler, task); -#else - event_set (guard_ev, task->sock, EV_READ|EV_PERSIST, - rspamd_worker_guard_handler, task); -#endif - event_base_set (task->ev_base, guard_ev); - event_add (guard_ev, NULL); - task->guard_ev = guard_ev; + task->guard_ev.data = task; + ev_io_init (&task->guard_ev, rspamd_worker_guard_handler, task->sock, EV_READ); + ev_io_start (task->event_loop, &task->guard_ev); rspamd_task_process (task, RSPAMD_TASK_PROCESS_ALL); @@ -332,7 +300,7 @@ rspamd_worker_error_handler (struct rspamd_http_connection *conn, GError *err) NULL, "application/json", task, - &task->tv); + 1.0); } } @@ -359,9 +327,9 @@ rspamd_worker_finish_handler (struct rspamd_http_connection *conn, * Accept new connection and construct task */ static void -accept_socket (gint fd, short what, void *arg) +accept_socket (EV_P_ ev_io *w, int revents) { - struct rspamd_worker *worker = (struct rspamd_worker *) arg; + struct rspamd_worker *worker = (struct rspamd_worker *) w->data; struct rspamd_worker_ctx *ctx; struct rspamd_task *task; rspamd_inet_addr_t *addr; @@ -377,7 +345,8 @@ accept_socket (gint fd, short what, void *arg) } if ((nfd = - rspamd_accept_from_socket (fd, &addr, worker->accept_events)) == -1) { + rspamd_accept_from_socket (w->fd, &addr, + rspamd_worker_throttle_accept_events, worker->accept_events)) == -1) { msg_warn_ctx ("accept failed: %s", strerror (errno)); return; } @@ -386,7 +355,7 @@ accept_socket (gint fd, short what, void *arg) return; } - task = rspamd_task_new (worker, ctx->cfg, NULL, ctx->lang_det, ctx->ev_base); + task = rspamd_task_new (worker, ctx->cfg, NULL, ctx->lang_det, ctx->event_loop); msg_info_task ("accepted connection from %s port %d, task ptr: %p", rspamd_inet_address_to_string (addr), @@ -435,41 +404,9 @@ accept_socket (gint fd, short what, void *arg) rspamd_http_connection_read_message (task->http_conn, task, - &ctx->io_tv); + ctx->timeout); } -#ifdef WITH_HYPERSCAN -static gboolean -rspamd_worker_hyperscan_ready (struct rspamd_main *rspamd_main, - struct rspamd_worker *worker, gint fd, - gint attached_fd, - struct rspamd_control_command *cmd, - gpointer ud) -{ - struct rspamd_control_reply rep; - struct rspamd_re_cache *cache = worker->srv->cfg->re_cache; - - memset (&rep, 0, sizeof (rep)); - rep.type = RSPAMD_CONTROL_HYPERSCAN_LOADED; - - if (!rspamd_re_cache_is_hs_loaded (cache) || cmd->cmd.hs_loaded.forced) { - msg_info ("loading hyperscan expressions after receiving compilation " - "notice: %s", - (!rspamd_re_cache_is_hs_loaded (cache)) ? - "new db" : "forced update"); - rep.reply.hs_loaded.status = rspamd_re_cache_load_hyperscan ( - worker->srv->cfg->re_cache, cmd->cmd.hs_loaded.cache_dir); - } - - if (write (fd, &rep, sizeof (rep)) != sizeof (rep)) { - msg_err ("cannot write reply to the control socket: %s", - strerror (errno)); - } - - return TRUE; -} -#endif - static gboolean rspamd_worker_log_pipe_handler (struct rspamd_main *rspamd_main, struct rspamd_worker *worker, gint fd, @@ -587,7 +524,7 @@ init_worker (struct rspamd_config *cfg) ctx, G_STRUCT_OFFSET (struct rspamd_worker_ctx, timeout), - RSPAMD_CL_FLAG_TIME_INTEGER, + RSPAMD_CL_FLAG_TIME_FLOAT, "Protocol IO timeout"); rspamd_rcl_register_worker_option (cfg, @@ -638,7 +575,7 @@ rspamd_worker_on_terminate (struct rspamd_worker *worker) void rspamd_worker_init_scanner (struct rspamd_worker *worker, - struct event_base *ev_base, + struct ev_loop *ev_base, struct rspamd_dns_resolver *resolver, struct rspamd_lang_detector **plang_det) { @@ -672,9 +609,8 @@ start_worker (struct rspamd_worker *worker) struct rspamd_worker_ctx *ctx = worker->ctx; ctx->cfg = worker->srv->cfg; - ctx->ev_base = rspamd_prepare_worker (worker, "normal", accept_socket); - msec_to_tv (ctx->timeout, &ctx->io_tv); - rspamd_symcache_start_refresh (worker->srv->cfg->cache, ctx->ev_base, + ctx->event_loop = rspamd_prepare_worker (worker, "normal", accept_socket); + rspamd_symcache_start_refresh (worker->srv->cfg->cache, ctx->event_loop, worker); if (isnan (ctx->task_timeout)) { @@ -687,20 +623,20 @@ start_worker (struct rspamd_worker *worker) } ctx->resolver = rspamd_dns_resolver_init (worker->srv->logger, - ctx->ev_base, + ctx->event_loop, worker->srv->cfg); - rspamd_map_watch (worker->srv->cfg, ctx->ev_base, ctx->resolver, worker, 0); + rspamd_map_watch (worker->srv->cfg, ctx->event_loop, ctx->resolver, worker, 0); rspamd_upstreams_library_config (worker->srv->cfg, ctx->cfg->ups_ctx, - ctx->ev_base, ctx->resolver->r); + ctx->event_loop, ctx->resolver->r); - ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base, + ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->event_loop, ctx->cfg->ups_ctx); - rspamd_worker_init_scanner (worker, ctx->ev_base, ctx->resolver, + rspamd_worker_init_scanner (worker, ctx->event_loop, ctx->resolver, &ctx->lang_det); - rspamd_lua_run_postloads (ctx->cfg->lua_state, ctx->cfg, ctx->ev_base, + rspamd_lua_run_postloads (ctx->cfg->lua_state, ctx->cfg, ctx->event_loop, worker); - event_base_loop (ctx->ev_base, 0); + ev_loop (ctx->event_loop, 0); rspamd_worker_block_signals (); rspamd_stat_close (); diff --git a/src/worker_private.h b/src/worker_private.h index 398c5d23d..6d0e763aa 100644 --- a/src/worker_private.h +++ b/src/worker_private.h @@ -30,14 +30,13 @@ struct rspamd_lang_detector; struct rspamd_worker_ctx { guint64 magic; /* Events base */ - struct event_base *ev_base; + struct ev_loop *event_loop; /* DNS resolver */ struct rspamd_dns_resolver *resolver; /* Config */ struct rspamd_config *cfg; - guint32 timeout; - struct timeval io_tv; + ev_tstamp timeout; /* Detect whether this worker is mime worker */ gboolean is_mime; /* Allow encrypted requests only using network */ @@ -45,7 +44,7 @@ struct rspamd_worker_ctx { /* Limit of tasks */ guint32 max_tasks; /* Maximum time for task processing */ - gdouble task_timeout; + ev_tstamp task_timeout; /* Encryption key */ struct rspamd_cryptobox_keypair *key; /* Keys cache */ @@ -57,18 +56,18 @@ struct rspamd_worker_ctx { * Init scanning routines */ void rspamd_worker_init_scanner (struct rspamd_worker *worker, - struct event_base *ev_base, + struct ev_loop *ev_base, struct rspamd_dns_resolver *resolver, struct rspamd_lang_detector **plang_det); /* * Called on forced timeout */ -void rspamd_task_timeout (gint fd, short what, gpointer ud); +void rspamd_task_timeout (EV_P_ ev_timer *w, int revents); /* * Called on unexpected IO error (e.g. ECONNRESET) */ -void rspamd_worker_guard_handler (gint fd, short what, void *data); +void rspamd_worker_guard_handler (EV_P_ ev_io *w, int revents); #endif |