From b218ec15bdae237f5ec9f23821239c29e2d3757d Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Fri, 10 May 2019 17:40:39 +0100 Subject: [PATCH] [Minor] Allow to tune GC related options --- src/libserver/cfg_file.h | 3 +++ src/libserver/cfg_rcl.c | 18 ++++++++++++++++++ src/libserver/cfg_utils.c | 8 ++++++++ src/libserver/task.c | 6 +++--- src/lua/lua_cfg_file.c | 2 ++ src/lua/lua_common.c | 15 +++++++++++---- src/lua/lua_common.h | 2 +- src/rspamadm/rspamadm.c | 1 + test/rspamd_lua_test.c | 1 + 9 files changed, 48 insertions(+), 8 deletions(-) diff --git a/src/libserver/cfg_file.h b/src/libserver/cfg_file.h index 8a1798b97..eccab1217 100644 --- a/src/libserver/cfg_file.h +++ b/src/libserver/cfg_file.h @@ -432,6 +432,9 @@ struct rspamd_config { guint words_decay; /**< limit for words for starting adaptive ignoring */ guint history_rows; /**< number of history rows stored */ guint max_sessions_cache; /**< maximum number of sessions cache elts */ + guint lua_gc_step; /**< lua gc step */ + guint lua_gc_pause; /**< lua gc pause */ + guint full_gc_iters; /**< iterations between full gc cycle */ GList *classify_headers; /**< list of headers using for statistics */ struct module_s **compiled_modules; /**< list of compiled C modules */ diff --git a/src/libserver/cfg_rcl.c b/src/libserver/cfg_rcl.c index 20bf7a7c6..9df9e4aa1 100644 --- a/src/libserver/cfg_rcl.c +++ b/src/libserver/cfg_rcl.c @@ -2127,6 +2127,24 @@ rspamd_rcl_config_init (struct rspamd_config *cfg, GHashTable *skip_sections) G_STRUCT_OFFSET (struct rspamd_config, task_timeout), RSPAMD_CL_FLAG_TIME_FLOAT, "Maximum time for checking a message (alias for task_timeout)"); + rspamd_rcl_add_default_handler (sub, + "lua_gc_step", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, lua_gc_step), + RSPAMD_CL_FLAG_UINT, + "Lua garbage-collector step (default: 200)"); + rspamd_rcl_add_default_handler (sub, + "lua_gc_pause", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, lua_gc_pause), + RSPAMD_CL_FLAG_UINT, + "Lua garbage-collector pause (default: 200)"); + rspamd_rcl_add_default_handler (sub, + "full_gc_iters", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, full_gc_iters), + RSPAMD_CL_FLAG_UINT, + "Task scanned before memory gc is performed (default: 0 - disabled)"); /* Neighbours configuration */ rspamd_rcl_add_section_doc (&sub->subsections, "neighbours", "name", diff --git a/src/libserver/cfg_utils.c b/src/libserver/cfg_utils.c index 308fb654e..265e575a7 100644 --- a/src/libserver/cfg_utils.c +++ b/src/libserver/cfg_utils.c @@ -49,6 +49,9 @@ #define DEFAULT_MAX_WORKERS 4 /* Timeout for task processing */ #define DEFAULT_TASK_TIMEOUT 8.0 +#define DEFAULT_LUA_GC_STEP 200 +#define DEFAULT_LUA_GC_PAUSE 200 +#define DEFAULT_GC_MAXITERS 0 struct rspamd_ucl_map_cbdata { struct rspamd_config *cfg; @@ -200,6 +203,11 @@ rspamd_config_new (enum rspamd_config_init_flags flags) cfg->min_word_len = DEFAULT_MIN_WORD; cfg->max_word_len = DEFAULT_MAX_WORD; + /* GC limits */ + cfg->lua_gc_pause = DEFAULT_LUA_GC_PAUSE; + cfg->lua_gc_step = DEFAULT_LUA_GC_STEP; + cfg->full_gc_iters = DEFAULT_GC_MAXITERS; + if (!(flags & RSPAMD_CONFIG_INIT_SKIP_LUA)) { cfg->lua_state = rspamd_lua_init (flags & RSPAMD_CONFIG_INIT_WIPE_LUA_MEM); cfg->own_lua_state = TRUE; diff --git a/src/libserver/task.c b/src/libserver/task.c index abe8143bc..a7c1bc478 100644 --- a/src/libserver/task.c +++ b/src/libserver/task.c @@ -242,7 +242,6 @@ rspamd_task_free (struct rspamd_task *task) struct rspamd_email_address *addr; struct rspamd_lua_cached_entry *entry; static guint free_iters = 0; - const guint free_iters_limit = 5000; GHashTableIter it; gpointer k, v; guint i; @@ -351,7 +350,7 @@ rspamd_task_free (struct rspamd_task *task) g_hash_table_unref (task->lua_cache); } - if (++free_iters > free_iters_limit) { + if (task->cfg->full_gc_iters && (++free_iters > task->cfg->full_gc_iters)) { /* Perform more expensive cleanup cycle */ gsize allocated = 0, active = 0, metadata = 0, resident = 0, mapped = 0, old_lua_mem = 0; @@ -381,7 +380,8 @@ rspamd_task_free (struct rspamd_task *task) allocated, active, metadata, resident, mapped, old_lua_mem, lua_gc (task->cfg->lua_state, LUA_GCCOUNT, 0), (t2 - t1) * 1000.0); - free_iters = rspamd_time_jitter (0, (gdouble)free_iters_limit / 2); + free_iters = rspamd_time_jitter (0, + (gdouble)task->cfg->full_gc_iters / 2); } REF_RELEASE (task->cfg); diff --git a/src/lua/lua_cfg_file.c b/src/lua/lua_cfg_file.c index efd8c8820..26e9811bd 100644 --- a/src/lua/lua_cfg_file.c +++ b/src/lua/lua_cfg_file.c @@ -137,4 +137,6 @@ rspamd_lua_post_load_config (struct rspamd_config *cfg) } lua_settop (L, 0); + + rspamd_lua_start_gc (cfg); } diff --git a/src/lua/lua_common.c b/src/lua/lua_common.c index 1a5cf117f..6502acd68 100644 --- a/src/lua/lua_common.c +++ b/src/lua/lua_common.c @@ -995,13 +995,20 @@ rspamd_lua_init (bool wipe_mem) lua_setglobal (L, "get_traces"); #endif + return L; +} + +void +rspamd_lua_start_gc (struct rspamd_config *cfg) +{ + lua_State *L = (lua_State *)cfg->lua_state; + + lua_settop (L, 0); /* Set up GC */ lua_gc (L, LUA_GCCOLLECT, 0); - lua_gc (L, LUA_GCSETSTEPMUL, 50); - lua_gc (L, LUA_GCSETPAUSE, 400); + lua_gc (L, LUA_GCSETSTEPMUL, cfg->lua_gc_step); + lua_gc (L, LUA_GCSETPAUSE, cfg->lua_gc_pause); lua_gc (L, LUA_GCRESTART, 0); - - return L; } /** diff --git a/src/lua/lua_common.h b/src/lua/lua_common.h index 0745d2ac3..214fffca6 100644 --- a/src/lua/lua_common.h +++ b/src/lua/lua_common.h @@ -167,7 +167,7 @@ gpointer rspamd_lua_check_class (lua_State *L, gint index, const gchar *name); */ lua_State *rspamd_lua_init (bool wipe_mem); - +void rspamd_lua_start_gc (struct rspamd_config *cfg); /** * Sets field in a global variable * @param L diff --git a/src/rspamadm/rspamadm.c b/src/rspamadm/rspamadm.c index f4fabfc9f..8096649f9 100644 --- a/src/rspamadm/rspamadm.c +++ b/src/rspamadm/rspamadm.c @@ -498,6 +498,7 @@ main (gint argc, gchar **argv, gchar **env) lua_setglobal (L, "rspamadm"); rspamadm_fill_lua_commands (L, all_commands); + rspamd_lua_start_gc (cfg); g_ptr_array_sort (all_commands, rspamdadm_commands_sort_func); g_strfreev (nargv); diff --git a/test/rspamd_lua_test.c b/test/rspamd_lua_test.c index d2a5ba2f5..5b310a7db 100644 --- a/test/rspamd_lua_test.c +++ b/test/rspamd_lua_test.c @@ -66,6 +66,7 @@ rspamd_lua_test_func (void) rspamd_lua_set_env (L, NULL, NULL, NULL); rspamd_lua_set_globals (rspamd_main->cfg, L); + rspamd_lua_start_gc (rspamd_main->cfg); if (lua_test_case) { lua_pushstring (L, lua_test_case); -- 2.39.5