Merge pull request #4873 from rspamd/vstakhov-fix-multipattern-init

Do not save multipatterns to FS in certain cases

13 files changed, 66 insertions, 101 deletions

diff --git a/.github/workflows/ci_rspamd.yml b/.github/workflows/ci_rspamd.yml
index 59a6b4832..04a6eff34 100644
--- a/.github/workflows/ci_rspamd.yml
+++ b/.github/workflows/ci_rspamd.yml
@@ -17,9 +17,10 @@ env:
 
 jobs:
   test:
-    runs-on: ["ubuntu-latest"]
+    runs-on: [ "ubuntu-latest" ]
     container:
       image: ${{ inputs.image }}
+      options: --user root
     steps:
       - name: Create directories
         run: |
@@ -28,6 +29,8 @@ jobs:
 
       - name: Check out source code
         uses: actions/checkout@v4
+        with:
+          path: src
 
       - name: Set variables on ARM64
         if: runner.arch == 'ARM64'
@@ -64,7 +67,7 @@ jobs:
           ulimit -c unlimited
           ulimit -s unlimited
           set +e
-          RSPAMD_INSTALLROOT=${GITHUB_WORKSPACE}/install robot --removekeywords wuks --exclude isbroken ${GITHUB_WORKSPACE}/src/test/functional/cases; EXIT_CODE=$?
+          RSPAMD_INSTALLROOT=${GITHUB_WORKSPACE}/install robot  -v RSPAMD_USER:root -v RSPAMD_GROUP:root --removekeywords wuks --exclude isbroken ${GITHUB_WORKSPACE}/src/test/functional/cases; EXIT_CODE=$?
           set -e
           core_files=$(find /var/tmp/ -name '*.core')
           for core in $core_files; do exe=$(gdb --batch -ex 'info proc mappings' -c $core | tail -1 | awk '{print $5}'); gdb --batch -ex 'bt' -c $core $exe; echo '---'; done
diff --git a/src/libmime/lang_detection.c b/src/libmime/lang_detection.c
index bdd0aad74..c485de5ad 100644
--- a/src/libmime/lang_detection.c
+++ b/src/libmime/lang_detection.c
@@ -898,7 +898,7 @@ rspamd_language_detector_init(struct rspamd_config *cfg)
 			rspamd_language_detector_process_chain(cfg, chain);
 		});
 
-		if (!rspamd_multipattern_compile(ret->stop_words[i].mp, &err)) {
+		if (!rspamd_multipattern_compile(ret->stop_words[i].mp, 0, &err)) {
 			msg_err_config("cannot compile stop words for %z language group: %e",
 						   i, err);
 			g_error_free(err);
diff --git a/src/libmime/message.c b/src/libmime/message.c
index 4fb118e60..f73c1ee35 100644
--- a/src/libmime/message.c
+++ b/src/libmime/message.c
@@ -674,7 +674,7 @@ rspamd_check_gtube(struct rspamd_task *task, struct rspamd_mime_text_part *part)
 										RSPAMD_MULTIPATTERN_DEFAULT);
 
 		GError *err = NULL;
-		rspamd_multipattern_compile(gtube_matcher, &err);
+		rspamd_multipattern_compile(gtube_matcher, RSPAMD_MULTIPATTERN_COMPILE_NO_FS, &err);
 
 		if (err != NULL) {
 			/* It will be expensive, but I don't care, still better than to abort */
diff --git a/src/libmime/mime_parser.c b/src/libmime/mime_parser.c
index 217f0b87d..562a9d7de 100644
--- a/src/libmime/mime_parser.c
+++ b/src/libmime/mime_parser.c
@@ -1,11 +1,11 @@
-/*-
- * Copyright 2016 Vsevolod Stakhov
+/*
+ * Copyright 2024 Vsevolod Stakhov
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -169,7 +169,7 @@ rspamd_mime_parser_init_lib(void)
 	rspamd_multipattern_add_pattern(lib_ctx->mp_boundary, "\n--", 0);
 
 	GError *err = NULL;
-	if (!rspamd_multipattern_compile(lib_ctx->mp_boundary, &err)) {
+	if (!rspamd_multipattern_compile(lib_ctx->mp_boundary, RSPAMD_MULTIPATTERN_COMPILE_NO_FS, &err)) {
 		msg_err("fatal error: cannot compile multipattern for mime parser boundaries: %e", err);
 		g_error_free(err);
 		g_abort();
diff --git a/src/libserver/url.c b/src/libserver/url.c
index 0842a1ebd..ff75e3c76 100644
--- a/src/libserver/url.c
+++ b/src/libserver/url.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2023 Vsevolod Stakhov
+ * Copyright 2024 Vsevolod Stakhov
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -539,6 +539,7 @@ void rspamd_url_init(const gchar *tld_file)
 {
 	GError *err = NULL;
 	gboolean ret = TRUE;
+	int mp_compile_flags = 0;
 
 	if (url_scanner != NULL) {
 		rspamd_url_deinit();
@@ -564,6 +565,7 @@ void rspamd_url_init(const gchar *tld_file)
 		url_scanner->matchers_full = NULL;
 		url_scanner->search_trie_full = NULL;
 		url_scanner->has_tld_file = false;
+		mp_compile_flags |= RSPAMD_MULTIPATTERN_COMPILE_NO_FS;
 	}
 
 	rspamd_url_add_static_matchers(url_scanner);
@@ -577,13 +579,13 @@ void rspamd_url_init(const gchar *tld_file)
 				 url_scanner->matchers_full->len);
 	}
 
-	if (!rspamd_multipattern_compile(url_scanner->search_trie_strict, &err)) {
+	if (!rspamd_multipattern_compile(url_scanner->search_trie_strict, mp_compile_flags, &err)) {
 		msg_err("cannot compile url matcher static patterns, fatal error: %e", err);
 		abort();
 	}
 
 	if (url_scanner->search_trie_full) {
-		if (!rspamd_multipattern_compile(url_scanner->search_trie_full, &err)) {
+		if (!rspamd_multipattern_compile(url_scanner->search_trie_full, mp_compile_flags, &err)) {
 			msg_err("cannot compile tld patterns, url matching will be "
 					"incomplete: %e",
 					err);
diff --git a/src/libutil/multipattern.c b/src/libutil/multipattern.c
index 630b1f921..bf3c7ad9a 100644
--- a/src/libutil/multipattern.c
+++ b/src/libutil/multipattern.c
@@ -1,11 +1,11 @@
-/*-
- * Copyright 2016 Vsevolod Stakhov
+/*
+ * Copyright 2024 Vsevolod Stakhov
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -466,7 +466,7 @@ rspamd_multipattern_try_save_hs(struct rspamd_multipattern *mp,
 #endif
 
 gboolean
-rspamd_multipattern_compile(struct rspamd_multipattern *mp, GError **err)
+rspamd_multipattern_compile(struct rspamd_multipattern *mp, int flags, GError **err)
 {
 	g_assert(mp != NULL);
 	g_assert(!mp->compiled);
@@ -483,7 +483,7 @@ rspamd_multipattern_compile(struct rspamd_multipattern *mp, GError **err)
 			rspamd_cryptobox_hash_update(&mp->hash_state, (void *) &plt, sizeof(plt));
 			rspamd_cryptobox_hash_final(&mp->hash_state, hash);
 
-			if (!rspamd_multipattern_try_load_hs(mp, hash)) {
+			if ((flags & RSPAMD_MULTIPATTERN_COMPILE_NO_FS) || !rspamd_multipattern_try_load_hs(mp, hash)) {
 				hs_database_t *db = NULL;
 
 				if (hs_compile_multi((const char *const *) mp->hs_pats->data,
@@ -504,18 +504,23 @@ rspamd_multipattern_compile(struct rspamd_multipattern *mp, GError **err)
 					return FALSE;
 				}
 
-				if (hs_cache_dir != NULL) {
-					char fpath[PATH_MAX];
-					rspamd_snprintf(fpath, sizeof(fpath), "%s/%*xs.hsmp", hs_cache_dir,
-									(gint) rspamd_cryptobox_HASHBYTES / 2, hash);
-					mp->hs_db = rspamd_hyperscan_from_raw_db(db, fpath);
+				if (!(flags & RSPAMD_MULTIPATTERN_COMPILE_NO_FS)) {
+					if (hs_cache_dir != NULL) {
+						char fpath[PATH_MAX];
+						rspamd_snprintf(fpath, sizeof(fpath), "%s/%*xs.hsmp", hs_cache_dir,
+										(gint) rspamd_cryptobox_HASHBYTES / 2, hash);
+						mp->hs_db = rspamd_hyperscan_from_raw_db(db, fpath);
+					}
+					else {
+						/* Should not happen in the real life */
+						mp->hs_db = rspamd_hyperscan_from_raw_db(db, NULL);
+					}
+
+					rspamd_multipattern_try_save_hs(mp, hash);
 				}
 				else {
-					/* Should not happen in the real life */
 					mp->hs_db = rspamd_hyperscan_from_raw_db(db, NULL);
 				}
-
-				rspamd_multipattern_try_save_hs(mp, hash);
 			}
 
 			for (i = 0; i < MAX_SCRATCH; i++) {
diff --git a/src/libutil/multipattern.h b/src/libutil/multipattern.h
index 93027661d..15099aaca 100644
--- a/src/libutil/multipattern.h
+++ b/src/libutil/multipattern.h
@@ -1,11 +1,11 @@
-/*-
- * Copyright 2016 Vsevolod Stakhov
+/*
+ * Copyright 2024 Vsevolod Stakhov
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -117,12 +117,15 @@ void rspamd_multipattern_add_pattern(struct rspamd_multipattern *mp,
 void rspamd_multipattern_add_pattern_len(struct rspamd_multipattern *mp,
 										 const gchar *pattern, gsize patlen, gint flags);
 
+
+#define RSPAMD_MULTIPATTERN_COMPILE_NO_FS (0x1u << 0u)
 /**
  * Compiles multipattern structure
  * @param mp
  * @return
  */
 gboolean rspamd_multipattern_compile(struct rspamd_multipattern *mp,
+									 int flags,
 									 GError **err);
 
 /**
diff --git a/src/lua/lua_trie.c b/src/lua/lua_trie.c
index b92832928..70a685da2 100644
--- a/src/lua/lua_trie.c
+++ b/src/lua/lua_trie.c
@@ -1,11 +1,11 @@
-/*-
- * Copyright 2016 Vsevolod Stakhov
+/*
+ * Copyright 2024 Vsevolod Stakhov
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -142,7 +142,7 @@ lua_trie_create(lua_State *L)
 
 		lua_pop(L, 1); /* table */
 
-		if (!rspamd_multipattern_compile(trie, &err)) {
+		if (!rspamd_multipattern_compile(trie, 0, &err)) {
 			msg_err("cannot compile multipattern: %e", err);
 			g_error_free(err);
 			rspamd_multipattern_destroy(trie);
diff --git a/src/lua/lua_util.c b/src/lua/lua_util.c
index d1d64b79c..612017d51 100644
--- a/src/lua/lua_util.c
+++ b/src/lua/lua_util.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2023 Vsevolod Stakhov
+ * Copyright 2024 Vsevolod Stakhov
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -2469,7 +2469,7 @@ lua_util_readline(lua_State *L)
 {
 	LUA_TRACE_POINT;
 	const gchar *prompt = "";
-	gchar *input;
+	gchar *input = NULL;
 
 	if (lua_type(L, 1) == LUA_TSTRING) {
 		prompt = lua_tostring(L, 1);
diff --git a/src/rspamadm/lua_repl.c b/src/rspamadm/lua_repl.c
index d3f32c9db..a3d5541fd 100644
--- a/src/rspamadm/lua_repl.c
+++ b/src/rspamadm/lua_repl.c
@@ -604,7 +604,7 @@ lua_syntax_highlighter(const char *str, ReplxxColor *colours, int size, void *ud
 static void
 rspamadm_lua_run_repl(lua_State *L, bool is_batch)
 {
-	gchar *input;
+	gchar *input = NULL;
 #ifdef WITH_LUA_REPL
 	gboolean is_multiline = FALSE;
 	GString *tb = NULL;
diff --git a/test/functional/cases/150_rspamadm.robot b/test/functional/cases/150_rspamadm.robot
index ba9bef5bd..6bff14b2e 100644
--- a/test/functional/cases/150_rspamadm.robot
+++ b/test/functional/cases/150_rspamadm.robot
@@ -1,8 +1,8 @@
 *** Settings ***
 Suite Setup     Rspamadm Setup
 Suite Teardown  Rspamadm Teardown
-Library         Process
-Library         ../lib/rspamd.py
+Library         ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource        ${RSPAMD_TESTDIR}/lib/rspamd.robot
 
 *** Test Cases ***
 Config Test
@@ -46,20 +46,3 @@ Verbose mode
   Should Match Regexp  ${result.stderr}  ^$
   Should Match Regexp  ${result.stdout}  hello world\n
   Should Be Equal As Integers  ${result.rc}  0
-
-*** Keywords ***
-Rspamadm Setup
-  ${RSPAMADM_TMPDIR} =  Make Temporary Directory
-  Set Suite Variable  ${RSPAMADM_TMPDIR}
-
-Rspamadm Teardown
-  Cleanup Temporary Directory  ${RSPAMADM_TMPDIR}
-
-Rspamadm
-  [Arguments]  @{args}
-  ${result} =  Run Process  ${RSPAMADM}
-  ...  --var\=TMPDIR\=${RSPAMADM_TMPDIR}
-  ...  --var\=DBDIR\=${RSPAMADM_TMPDIR}
-  ...  --var\=LOCAL_CONFDIR\=/nonexistent
-  ...  @{args}
-  [Return]  ${result}
diff --git a/test/functional/lib/rspamd.robot b/test/functional/lib/rspamd.robot
index a5f897365..f9614c6fa 100644
--- a/test/functional/lib/rspamd.robot
+++ b/test/functional/lib/rspamd.robot
@@ -289,6 +289,7 @@ Run Rspamd
   ...  --var\=DBDIR\=${RSPAMD_TMPDIR}
   ...  --var\=LOCAL_CONFDIR\=/non-existent
   ...  --var\=CONFDIR\=${RSPAMD_TESTDIR}/../../conf/
+  ...  --insecure
   ...  env:RSPAMD_LOCAL_CONFDIR=/non-existent
   ...  env:RSPAMD_TMPDIR=${RSPAMD_TMPDIR}
   ...  env:RSPAMD_CONFDIR=${RSPAMD_TESTDIR}/../../conf/
@@ -302,6 +303,21 @@ Run Rspamd
   # Confirm worker is reachable
   Wait Until Keyword Succeeds  15x  1 sec  Ping Rspamd  ${RSPAMD_LOCAL_ADDR}  ${check_port}
 
+Rspamadm Setup
+  ${RSPAMADM_TMPDIR} =  Make Temporary Directory
+  Set Suite Variable  ${RSPAMADM_TMPDIR}
+
+Rspamadm Teardown
+  Cleanup Temporary Directory  ${RSPAMADM_TMPDIR}
+
+Rspamadm
+  [Arguments]  @{args}
+  ${result} =  Run Process  ${RSPAMADM}
+  ...  --var\=TMPDIR\=${RSPAMADM_TMPDIR}
+  ...  --var\=DBDIR\=${RSPAMADM_TMPDIR}
+  ...  --var\=LOCAL_CONFDIR\=/nonexistent
+  ...  @{args}
+  [Return]  ${result}
 
 Run Nginx
   ${template} =  Get File  ${RSPAMD_TESTDIR}/configs/nginx.conf
diff --git a/test/functional/lib/vars.py b/test/functional/lib/vars.py
index a4bcbadda..bee2e92d4 100644
--- a/test/functional/lib/vars.py
+++ b/test/functional/lib/vars.py
@@ -11,54 +11,7 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
+
 
 import shutil
 import socket