[Feature] Tune memory management in Rspamd and Lua

diff --git a/src/libserver/task.c b/src/libserver/task.c
index b69b963415de4f4f371a13b62ec71454ef59bf37..6ec3495804ca1330fd0a89810713e7cb92a74714 100644 (file)
--- a/src/libserver/task.c
+++ b/src/libserver/task.c
@@ -30,6 +30,14 @@
 #include "libmime/lang_detection.h"
 #include "libmime/filter_private.h"
 
+#ifdef WITH_JEMALLOC
+#include <jemalloc/jemalloc.h>
+#else
+# if defined(__GLIBC__) && defined(_GNU_SOURCE)
+#  include <malloc.h>
+# endif
+#endif
+
 #include <math.h>
 
 /*
@@ -233,6 +241,8 @@ rspamd_task_free (struct rspamd_task *task)
        struct rspamd_mime_text_part *tp;
        struct rspamd_email_address *addr;
        struct rspamd_lua_cached_entry *entry;
+       static guint free_iters = 0;
+       const guint free_iters_limit = 5000;
        GHashTableIter it;
        gpointer k, v;
        guint i;
@@ -341,6 +351,32 @@ rspamd_task_free (struct rspamd_task *task)
                                g_hash_table_unref (task->lua_cache);
                        }
 
+                       if (++free_iters > free_iters_limit) {
+                               /* Perform more expensive cleanup cycle */
+                               gsize allocated = 0, active = 0, metadata = 0,
+                                               resident = 0, mapped = 0;
+
+#ifdef WITH_JEMALLOC
+                               gsize sz = sizeof (gsize);
+                               mallctl ("stats.allocated", &allocated, &sz, NULL, 0);
+                               mallctl ("stats.active", &active, &sz, NULL, 0);
+                               mallctl ("stats.metadata", &metadata, &sz, NULL, 0);
+                               mallctl ("stats.resident", &resident, &sz, NULL, 0);
+                               mallctl ("stats.mapped", &mapped, &sz, NULL, 0);
+#else
+# if defined(__GLIBC__) && defined(_GNU_SOURCE)
+                               malloc_trim (0);
+# endif
+#endif
+                               msg_notice_task ("perform full gc cycle; memory stats: "
+                                                                "%z allocated, %z active, %z metadata, %z rezident, %z mapped;"
+                                                                " lua memory: %d kb",
+                                               allocated, active, metadata, resident, mapped,
+                                               lua_gc (task->cfg->lua_state, LUA_GCCOUNT, 0));
+                               free_iters = 0;
+                               lua_gc (task->cfg->lua_state, LUA_GCCOLLECT, 0);
+                       }
+
                        REF_RELEASE (task->cfg);
                }
 
@@ -369,7 +405,7 @@ rspamd_task_unmapper (gpointer ud)
 
 gboolean
 rspamd_task_load_message (struct rspamd_task *task,
-       struct rspamd_http_message *msg, const gchar *start, gsize len)
+                                                 struct rspamd_http_message *msg, const gchar *start, gsize len)
 {
        guint control_len, r;
        struct ucl_parser *parser;
@@ -446,7 +482,7 @@ rspamd_task_load_message (struct rspamd_task *task,
 
                        if (offset > (gulong)st.st_size) {
                                msg_err_task ("invalid offset %ul (%ul available) for shm "
-                                               "segment %s", offset, st.st_size, fp);
+                                                         "segment %s", offset, st.st_size, fp);
                                munmap (map, st.st_size);
                                close (fd);
 
@@ -463,7 +499,7 @@ rspamd_task_load_message (struct rspamd_task *task,
 
                        if (shmem_size > (gulong)st.st_size) {
                                msg_err_task ("invalid length %ul (%ul available) for %s "
-                                               "segment %s", shmem_size, st.st_size, ft, fp);
+                                                         "segment %s", shmem_size, st.st_size, ft, fp);
                                munmap (map, st.st_size);
                                close (fd);
 
@@ -655,7 +691,7 @@ rspamd_task_load_message (struct rspamd_task *task,
                        task->flags |= RSPAMD_TASK_FLAG_COMPRESSED;
 
                        msg_info_task ("loaded message from zstd compressed stream; "
-                                       "compressed: %ul; uncompressed: %ul",
+                                                  "compressed: %ul; uncompressed: %ul",
                                        (gulong)zin.size, (gulong)zout.pos);
 
                }

diff --git a/src/lua/lua_common.c b/src/lua/lua_common.c
index 1b3e0121d061047a7440cbbcc81397ffde6529d2..1a5cf117f113461464bac870730319f357fb76e5 100644 (file)
--- a/src/lua/lua_common.c
+++ b/src/lua/lua_common.c
@@ -898,6 +898,7 @@ rspamd_lua_init (bool wipe_mem)
                L = luaL_newstate ();
        }
 
+       lua_gc (L, LUA_GCSTOP, 0);
        luaL_openlibs (L);
        luaopen_logger (L);
        luaopen_mempool (L);
@@ -994,6 +995,12 @@ rspamd_lua_init (bool wipe_mem)
        lua_setglobal (L, "get_traces");
 #endif
 
+       /* Set up GC */
+       lua_gc (L, LUA_GCCOLLECT, 0);
+       lua_gc (L, LUA_GCSETSTEPMUL, 50);
+       lua_gc (L, LUA_GCSETPAUSE, 400);
+       lua_gc (L, LUA_GCRESTART, 0);
+
        return L;
 }