Merge branch 'master' into vstakhov-stringzilla

.gitignore

@@ -1,6 +1,8 @@
 # Code::TidyAll
 /.tidyall.d/
 .idea
+# Added by CLion
+cmake-build-debug/
 # Logs and databases #
 ######################
 *.log

CMakeLists.txt

@@ -8,8 +8,8 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 3.12 FATAL_ERROR)
 
 SET(RSPAMD_VERSION_MAJOR 3)
-SET(RSPAMD_VERSION_MINOR 8)
-SET(RSPAMD_VERSION_PATCH 2)
+SET(RSPAMD_VERSION_MINOR 9)
+SET(RSPAMD_VERSION_PATCH 0)
 
 # Keep two digits all the time
 SET(RSPAMD_VERSION_MAJOR_NUM ${RSPAMD_VERSION_MAJOR}0)
@@ -88,6 +88,7 @@ ENDIF ()
 FIND_PACKAGE(PkgConfig REQUIRED)
 FIND_PACKAGE(Perl REQUIRED)
 
+
 option(SANITIZE "Enable sanitizer: address, memory, undefined, leak (comma separated list)" "")
 INCLUDE(Toolset)
 INCLUDE(Sanitizer)
@@ -241,6 +242,8 @@ ProcessPackage(LIBZ LIBRARY z INCLUDE zlib.h INCLUDE_SUFFIXES include/zlib
 ProcessPackage(SODIUM LIBRARY sodium INCLUDE sodium.h
         INCLUDE_SUFFIXES include/libsodium include/sodium
         ROOT ${LIBSODIUM_ROOT_DIR} MODULES libsodium>=1.0.0)
+ProcessPackage(LIBARCHIVE LIBRARY archive INCLUDE archive.h
+        ROOT ${LIBARCHIVE_ROOT_DIR} MODULES libarchive>=3.0.0)
 
 if (ENABLE_FASTTEXT MATCHES "ON")
     ProcessPackage(FASTTEXT LIBRARY fasttext INCLUDE fasttext/fasttext.h
@@ -687,6 +690,7 @@ IF (ENABLE_CLANG_PLUGIN MATCHES "ON")
 ENDIF ()
 
 ADD_SUBDIRECTORY(src)
+enable_testing()
 ADD_SUBDIRECTORY(test)
 ADD_SUBDIRECTORY(utils)
 

ChangeLog

@@ -1,3 +1,18 @@
+3.8.2: 20 Feb 2024
+  * [Feature] Add extraction type for `from` maps
+  * [Feature] Allow to add templates to redis history prefix
+  * [Feature] Implement dynamic keys map in fuzzy storage
+  * [Feature] Lua_url: Add `to_http` method
+  * [Feature] Support JSON logging when in syslog mode
+  * [Fix] Deal with `Connection` and `Host` headers on proxying
+  * [Fix] Encode headers in metadata exporter
+  * [Fix] Fix initial maps load
+  * [Fix] Make stat tokens allocation consistent
+  * [Fix] Resolve issue with bayes stat in `rspamadm` mode
+  * [Fix] Try to fix url path issue
+  * [Rework] Breaking: Do not report module as action
+  * [Rework] Use khash instead of glib hashes for many reasons
+
 3.8.1: 25 Jan 2024
   * [Fix] Fix headers insertion in the ordered list
   * [Fix] Fix learn error propagation

conf/modules.d/rbl.conf

@@ -212,8 +212,10 @@ rbl {
       returnbits = {
         CRACKED_SURBL = 128;
         ABUSE_SURBL = 64;
+        CT_SURBL = 32;
         MW_SURBL_MULTI = 16;
         PH_SURBL_MULTI = 8;
+        DM_SURBL = 4;
         SURBL_BLOCKED = 1;
       }
     }

conf/scores.d/surbl_group.conf

@@ -50,6 +50,18 @@ symbols = {
         one_shot = true;
         groups = ["surblorg"];
     }
+    "CT_SURBL" {
+        weight = 0.0;
+        description = "A domain in the message is listed in SURBL as a clicktracker";
+        one_shot = true;
+        groups = ["surblorg"];
+    }
+    "DM_SURBL" {
+        weight = 0.0;
+        description = "A domain in the message is listed in SURBL as belonging to a disposable email service";
+        one_shot = true;
+        groups = ["surblorg"];
+    }
 
     "RSPAMD_URIBL" {
         weight = 4.5;

interface/css/rspamd.css

@@ -213,15 +213,11 @@ table#symbolsTable input[type="number"] {
     padding-bottom: 0.1rem;
 }
 
-/* widget */
 .card-header,
 .modal-header {
     background-color: #f3f3f3;
     background-image: linear-gradient(to bottom, #fdfdfd, #eaeaea);
 }
-.card-header > .icon > svg {
-    vertical-align: middle;
-}
 .card-header .h6 {
     font-size: 0.857rem;
 }
@@ -394,6 +390,8 @@ table#symbolsTable input[type="number"] {
     text-align: center;
 }
 
+.outline-dashed-primary { outline: 2px dashed var(--bs-primary); }
+
 .scorebar-spam {
     background-color: rgba(240 0 0 / 0.1) !important;
 }

interface/index.html

@@ -164,9 +164,9 @@
 				<div class="row">
 					<div class="col-lg-6">
 						<div class="card bg-light shadow my-3">
-							<div class="card-header text-secondary py-2">
+							<div class="card-header text-secondary py-2 d-flex align-items-center">
 								<span class="icon me-3"><i class="fas fa-server"></i></span>
-								<span class="h6 fw-bolder my-2">Servers</span>
+								<span class="h6 fw-bolder my-auto">Servers</span>
 							</div>
 								<div class="card-body p-0 table-responsive">
 									<table class="table status-table table-sm table-hover table-bordered text-nowrap mb-0" id="clusterTable">
@@ -188,9 +188,9 @@
 								</div>
 						</div>
 						<div class="card bg-light shadow my-3">
-							<div class="card-header text-secondary py-2">
+							<div class="card-header text-secondary py-2 d-flex align-items-center">
 								<span class="icon me-3"><i class="fas fa-dice"></i></span>
-								<span class="h6 fw-bolder my-2">Bayesian statistics</span>
+								<span class="h6 fw-bolder my-auto">Bayesian statistics</span>
 							</div>
 								<div class="card-body p-0 table-responsive">
 									<table class="table status-table table-sm table-bordered text-nowrap mb-0" id="bayesTable">
@@ -209,9 +209,9 @@
 								</div>
 						</div>
 						<div class="card bg-light shadow my-3">
-							<div class="card-header text-secondary py-2">
+							<div class="card-header text-secondary py-2 d-flex align-items-center">
 								<span class="icon me-3"><i class="fas fa-hashtag"></i></span>
-								<span class="h6 fw-bolder my-2">Fuzzy hashes</span>
+								<span class="h6 fw-bolder my-auto">Fuzzy hashes</span>
 							</div>
 								<div class="card-body p-0 table-responsive">
 									<table class="table status-table table-sm table-bordered text-nowrap mb-0" id="fuzzyTable">
@@ -230,9 +230,9 @@
 					</div>
 					<div class="col-lg-6">
 						<div class="card bg-light shadow my-3">
-							<div class="card-header text-secondary py-2">
+							<div class="card-header text-secondary py-2 d-flex align-items-center">
 								<span class="icon me-3"><i class="fas fa-chart-pie"></i></span>
-								<span class="h6 fw-bolder my-2">Statistics</span>
+								<span class="h6 fw-bolder my-auto">Statistics</span>
 							</div>
 							<div class="card-body">
 								<div class="row">
@@ -246,9 +246,9 @@
 
 			<div class="tab-pane" id="throughput">
 				<div class="card bg-light shadow my-3">
-					<div class="card-header text-secondary py-2">
+					<div class="card-header text-secondary py-2 d-flex align-items-center">
 						<span class="icon me-3"><i class="fas fa-chart-area"></i></span>
-						<span class="h6 fw-bolder my-2">Throughput</span>
+						<span class="h6 fw-bolder my-auto">Throughput</span>
 					</div>
 					<div class="card-body text-center">
 						<div class="d-inline-block bg-white">
@@ -289,7 +289,7 @@
 								<option value="line" selected>Line</option>
 								<option value="area">Stacked area</option>
 							</select>
-							<a title="&ldquo;Curves&rdquo; section of &ldquo;d3-shape&rdquo; library documentation" href="https://github.com/d3/d3-shape#curves" target="_blank">Interpolation mode</a>:
+							<a title="&ldquo;Curves&rdquo; section of &ldquo;d3-shape&rdquo; module documentation" href="https://d3js.org/d3-shape/curve" target="_blank">Interpolation mode</a>:
 							<select id="selInterpolate" class="form-select">
 								<option value="curveLinear" selected>linear</option>
 								<option value="curveStep">step</option>
@@ -309,9 +309,9 @@
 
 			<div class="tab-pane" id="configuration">
 				<div class="card bg-light shadow my-3">
-					<div class="card-header text-secondary py-2">
+					<div class="card-header text-secondary py-2 d-flex align-items-center">
 						<span class="icon me-3"><i class="fas fa-tasks"></i></span>
-						<span class="h6 fw-bolder my-2">Actions</span>
+						<span class="h6 fw-bolder my-auto">Actions</span>
 					</div>
 					<div class="card-body pb-2">
 						<form id="actionsForm">
@@ -326,10 +326,10 @@
 					</div>
 				</div>
 				<div class="card bg-light shadow my-3">
-					<div class="card-header text-secondary py-2 d-flex">
+					<div class="card-header text-secondary py-2 d-flex align-items-center">
 						<span class="icon me-3"><i class="fas fa-list"></i></span>
-						<span class="h6 fw-bolder my-2">Lists</span>
-						<div class="input-group-sm align-self-center ms-auto me-1">
+						<span class="h6 fw-bolder my-auto">Lists</span>
+						<div class="input-group-sm ms-auto me-1">
 							Editor:
 							<div id="btnGroupEditor" class="btn-group btn-group-xs ms-1">
 								<input type="radio" class="btn-check" name="editorMode" id="editorModeBascic" autocomplete="off" value="basic">
@@ -349,12 +349,12 @@
 
 			<div class="tab-pane" id="symbols">
 				<div class="card bg-light shadow my-3">
-					<div class="card-header text-secondary py-1 d-flex">
+					<div class="card-header text-secondary py-1 d-flex align-items-center">
 						<span class="icon me-3"><i class="fas fa-tasks"></i></span>
-						<span class="h6 fw-bolder my-2 ms-0">Symbols and rules</span>
-						<div class="align-self-center ms-auto me-1">
-							<button class="btn btn-info btn-sm" id="updateSymbols">
-								<i class="fas fa-redo-alt"></i> Update
+						<span class="h6 fw-bolder my-auto ms-0">Symbols and rules</span>
+						<div class="ms-auto me-1">
+							<button class="btn btn-info btn-sm d-flex align-items-center" id="updateSymbols">
+								<i class="fas fa-redo-alt me-1"></i>Update
 							</button>
 						</div>
 					</div>
@@ -378,16 +378,20 @@
 
 			<div class="tab-pane" id="scan">
 				<div class="card bg-light shadow my-3">
-					<div class="card-header text-secondary py-2">
+					<div class="card-header text-secondary py-1 d-flex align-items-center">
 						<span class="icon me-3"><i class="fas fa-envelope"></i></span>
-						<span class="h6 fw-bolder my-2">Scan suspected message</span>
+						<span class="h6 fw-bolder my-auto">Scan suspected message</span>
+						<div class="d-flex input-group-sm align-items-center ms-auto">
+							<label for="formFile" class="col-auto col-form-label-sm me-1">Choose a file:</label>
+							<input class="form-control form-control-sm btn btn-secondary" id="formFile" type="file">
+						</div>
 					</div>
 					<div class="card-body">
 						<div class="row">
 							<form class="col-lg-12" id="scanForm">
 								<div class="mb-0">
 									<label class="form-label" for="scanMsgSource">Message source:</label>
-									<textarea class="form-control" id="scanMsgSource" rows="10" placeholder="Paste raw message source"></textarea>
+									<textarea class="form-control" id="scanMsgSource" rows="10" placeholder='Paste raw message source, drag and drop files here or use "Browse..." button.'></textarea>
 								</div>
 								<div class="collapse row mt-3" id="scanOptions">
 									<div class="col-lg-6">
@@ -442,23 +446,23 @@
 					</div>
 					<div class="card-footer d-md-flex justify-content-between py-1">
 						<div class="input-group d-inline-flex w-auto my-1">
-							<button type="submit" class="btn btn-primary" data-upload="scan"><i class="fas fa-search"></i> Scan message</button>
-							<button class="btn btn-secondary d-inline-block" id="scanOptionsToggle" data-bs-toggle="collapse" data-bs-target="#scanOptions"><i class="fas 	fa-bars"></i> Options</button>
+							<button type="submit" class="btn btn-primary d-flex align-items-center" data-upload="scan"><i class="fas fa-search me-2"></i>Scan message</button>
+							<button class="btn btn-secondary d-flex align-items-center" id="scanOptionsToggle" data-bs-toggle="collapse" data-bs-target="#scanOptions"><i class="fas fa-bars me-2"></i>Options</button>
 						</div>
 						<div class="input-group d-inline-flex w-auto my-1">
 							<label for="fuzzy-flag" class="input-group-text">Flag</label>
 							<input id="fuzzy-flag" class="form-control" value="1" min="1" type="number">
-							<button class="btn btn-warning" data-upload="compute-fuzzy"><i class="fas fa-hashtag"></i> Compute fuzzy hashes</button>
+							<button class="btn btn-warning d-flex align-items-center" data-upload="compute-fuzzy"><i class="fas fa-hashtag me-2"></i>Compute fuzzy hashes</button>
 						</div>
 						<div class="float-end my-1">
-							<button class="btn btn-secondary" id="scanClean"><i class="fas fa-trash-alt"></i> Clean form</button>
+							<button class="btn btn-secondary d-flex align-items-center" id="scanClean"><i class="fas fa-trash-alt me-2"></i>Clean form</button>
 						</div>
 					</div>
 				</div>
 				<div class="card ro-hide" style="display: none;">
-					<div class="card-header text-secondary py-1 d-flex">
+					<div class="card-header text-secondary py-1 d-flex align-items-center">
 						<span class="icon me-3"><i class="fas fa-graduation-cap"></i></span>
-						<span class="h6 fw-bolder my-2">Learn Rspamd</span>
+						<span class="h6 fw-bolder my-auto">Learn Rspamd</span>
 						<div id="learnServers" class="input-group-sm align-items-center text-nowrap ms-auto me-1">
 							<label for="selLearnServers">Learn servers:</label>
 							<select id="selLearnServers" class="form-select ms-1">
@@ -474,8 +478,8 @@
 									<p>Learn Bayesian classifier:</p>
 									<form>
 										<div class="btn-group">
-											<button class="btn btn-success" type="button" data-upload="ham" disabled><i class="fas fa-thumbs-up"></i> Upload HAM</button>
-											<button class="btn btn-danger" type="button" data-upload="spam" disabled><i class="fas fa-thumbs-down"></i> Upload SPAM</button>
+											<button class="btn btn-success d-flex align-items-center" type="button" data-upload="ham" disabled><i class="fas fa-thumbs-up me-2"></i>Upload HAM</button>
+											<button class="btn btn-danger d-flex align-items-center" type="button" data-upload="spam" disabled><i class="fas fa-thumbs-down me-2"></i>Upload SPAM</button>
 										</div>
 									</form>
 								</div>
@@ -492,7 +496,7 @@
 											<label for="fuzzyWeightText">Weight:</label>
 											<input name="fuzzyWeightText" id="fuzzyWeightText" class="form-control ms-1" type="number" value="1"/>
 										</div>
-										<button class="btn btn-warning ms-2" data-upload="fuzzy" disabled><i class="fas fa-upload"></i> Upload FUZZY</button>
+										<button class="btn btn-warning ms-2 d-flex align-items-center" data-upload="fuzzy" disabled><i class="fas fa-upload me-2"></i>Upload FUZZY</button>
 									</form>
 								</div>
 							</div>
@@ -501,9 +505,9 @@
 				</div>
 
 				<div id="hash-card" class="card bg-light shadow my-3" style="display: none;">
-					<div class="card-header text-secondary py-2">
+					<div class="card-header text-secondary py-2 d-flex align-items-center">
 						<span class="icon me-3"><i class="fas fa-hashtag"></i></span>
-						<span class="h6 fw-bolder my-2">Fuzzy hashes</span>
+						<span class="h6 fw-bolder my-auto">Fuzzy hashes</span>
 						<button type="button" class="card-close-btn btn-close float-end" aria-label="Close"></button>
 					</div>
 						<div class="card-body p-0 table-responsive">
@@ -521,9 +525,9 @@
 				</div>
 
 				<div class="card bg-light shadow my-3">
-					<div class="card-header text-secondary py-1 d-flex">
+					<div class="card-header text-secondary py-1 d-flex align-items-center">
 						<span class="icon me-3"><i class="fas fa-eye"></i></span>
-						<span class="h6 fw-bolder my-2 ms-0">Scan results history</span>
+						<span class="h6 fw-bolder my-auto ms-0">Scan results history</span>
 						<div id="scanResult" class="d-flex input-group-sm align-items-center text-nowrap ms-auto me-1">
 								<label for="selSymOrder_scan">Symbols order:</label>
 								<select id="selSymOrder_scan" class="form-select ms-1">
@@ -533,8 +537,8 @@
 								</select>
 							<label for="scan_page_size" class="ms-2">Rows per page:</label>
 							<input id="scan_page_size" class="form-control ms-1" value="25" min="1" type="number">
-							<button class="btn btn-secondary btn-sm ms-2" id="cleanScanHistory" disabled>
-								<i class="fas fa-trash-alt"></i> Clean history
+							<button class="btn btn-secondary btn-sm ms-2 d-flex align-items-center" id="cleanScanHistory" disabled>
+								<i class="fas fa-trash-alt me-1"></i>Clean history
 							</button>
 						</div>
 					</div>
@@ -548,9 +552,9 @@
 
 			<div class="tab-pane" id="selectors">
 				<div class="card bg-light shadow my-3">
-					<div class="card-header text-secondary py-2">
+					<div class="card-header text-secondary py-2 d-flex align-items-center">
 						<span class="icon me-3"><i class="fas fa-envelope"></i></span>
-						<span class="h6 fw-bolder my-2">Test Rspamd selectors</span>
+						<span class="h6 fw-bolder my-auto">Test Rspamd selectors</span>
 					</div>
 					<div class="card-body p-0">
 						<div class="row h-100 m-0" id="row-main">
@@ -581,7 +585,7 @@
 															<label class="form-label" for="selectorsMsgArea">Message source:</label>
 															<textarea class="form-control" id="selectorsMsgArea" rows="9" placeholder="Paste raw message source"></textarea>
 														</div>
-														<button class="btn btn-secondary float-end" id="selectorsMsgClean"><i class="fas fa-trash-alt"></i> Clean form</button>
+														<button class="btn btn-secondary d-flex align-items-center float-end" id="selectorsMsgClean"><i class="fas fa-trash-alt me-2"></i>Clean form</button>
 													</div>
 												</div>
 												<div class="row pt-3">
@@ -590,8 +594,8 @@
 															<label class="form-label" for="selectorsSelArea">Selector(s):</label>
 															<textarea class="form-control" id="selectorsSelArea" rows="1" placeholder="extractor.transform(arg);extractor.transform(arg);..."></textarea>
 														</div>
-														<button type="submit" class="btn btn-primary" id="selectorsChkMsgBtn"><i class="fas fa-search"></i> Check message</button>
-														<button class="btn btn-secondary float-end" id="selectorsClean"><i class="fas fa-trash-alt"></i> Clean form</button>
+														<button type="submit" class="btn btn-primary d-inline-flex align-items-center" id="selectorsChkMsgBtn"><i class="fas fa-search me-2"></i>Check message</button>
+														<button class="btn btn-secondary d-inline-flex align-items-center float-end" id="selectorsClean"><i class="fas fa-trash-alt me-2"></i>Clean form</button>
 													</div>
 												</div>
 												<div class="row pt-3 flex-grow-1">
@@ -630,9 +634,9 @@
 			<div class="tab-pane" id="history">
 
 				<div class="card bg-light shadow my-3">
-					<div class="card-header text-secondary py-1 d-flex">
+					<div class="card-header text-secondary py-1 d-flex align-items-center">
 						<span class="icon me-3"><i class="fas fa-eye"></i></span>
-						<span class="h6 fw-bolder my-2 ms-0">History</span>
+						<span class="h6 fw-bolder my-auto ms-0">History</span>
 						<a href="https://rspamd.com/doc/modules/history_redis.html" target="_blank" rel="noopener noreferrer"
 							title="If you'd like to use the modern version of History, please enable History redis module."
 							 id="legacy-history-badge" class="my-2 ms-2 badge text-bg-info" style="display: none;">Legacy version</a>
@@ -645,11 +649,11 @@
 							</select>
 							<label for="history_page_size" class="ms-2">Rows per page:</label>
 							<input id="history_page_size" class="form-control ms-1" value="25" min="1" type="number">
-							<button class="btn btn-danger btn-sm ms-2 ro-hide" id="resetHistory">
-								<i class="fas fa-times-circle"></i> Reset
+							<button class="btn btn-danger btn-sm ms-2 d-flex align-items-center ro-hide" id="resetHistory">
+								<i class="fas fa-times-circle me-1"></i>Reset
 							</button>
-							<button class="btn btn-info btn-sm ms-2" id="updateHistory">
-								<i class="fas fa-redo-alt"></i> Update
+							<button class="btn btn-info btn-sm ms-2 d-flex align-items-center" id="updateHistory">
+								<i class="fas fa-redo-alt me-1"></i>Update
 							</button>
 						</div>
 					</div>
@@ -660,12 +664,12 @@
 					</div>
 				</div>
 				<div class="card bg-light shadow my-3 ro-hide" id="errors-history">
-					<div class="card-header text-secondary py-1 d-flex">
+					<div class="card-header text-secondary py-1 d-flex align-items-center">
 						<span class="icon me-3"><i class="fas fa-exclamation-triangle"></i></span>
-						<span class="h6 fw-bolder my-2 ms-0">Errors</span>
+						<span class="h6 fw-bolder my-auto ms-0">Errors</span>
 						<div class="align-self-center ms-auto me-1">
-							<button class="btn btn-info btn-sm" id="updateErrors">
-								<i class="fas fa-redo-alt"></i> Update
+							<button class="btn btn-info btn-sm d-flex align-items-center" id="updateErrors">
+								<i class="fas fa-redo-alt me-1"></i>Update
 							</button>
 						</div>
 					</div>

interface/js/app/history.js

@@ -151,6 +151,7 @@ define(["jquery", "app/common", "app/libft", "footable"],
         }
 
         ui.getHistory = function () {
+            $("#refresh, #updateHistory").attr("disabled", true);
             common.query("history", {
                 success: function (req_data) {
                     function differentVersions(neighbours_data) {
@@ -190,7 +191,8 @@ define(["jquery", "app/common", "app/libft", "footable"],
                             libft.destroyTable("history");
                             // Is there a way to get an event when the table is destroyed?
                             setTimeout(() => {
-                                libft.initHistoryTable(data, items, "history", get_history_columns(data), false);
+                                libft.initHistoryTable(data, items, "history", get_history_columns(data), false,
+                                    () => $("#refresh, #updateHistory").removeAttr("disabled"));
                             }, 200);
                         }
                         prevVersion = version;
@@ -198,7 +200,7 @@ define(["jquery", "app/common", "app/libft", "footable"],
                         libft.destroyTable("history");
                     }
                 },
-                complete: function () { $("#refresh").removeAttr("disabled").removeClass("disabled"); },
+                error: () => $("#refresh, #updateHistory").removeAttr("disabled"),
                 errorMessage: "Cannot receive history",
             });
         };

interface/js/app/libft.js

@@ -62,12 +62,17 @@ define(["jquery", "app/common", "footable"],
                     wordBreak: "break-all",
                     whiteSpace: "normal"
                 }
+            }, {
+                name: "file",
+                title: "File name",
+                breakpoints: "xs",
+                sortValue: (val) => ((typeof val === "undefined") ? "" : val)
             }, {
                 name: "ip",
                 title: "IP address",
                 breakpoints: "xs sm md",
                 style: {
-                    "minWidth": "calc(7.6em + 8px)",
+                    "minWidth": "calc(14ch + 8px)",
                     "word-break": "break-all"
                 },
                 // Normalize IPv4
@@ -171,7 +176,7 @@ define(["jquery", "app/common", "footable"],
             }].filter((col) => {
                 switch (table) {
                     case "history":
-                        return true;
+                        return (col.name !== "file");
                     case "scan":
                         return ["ip", "sender_mime", "rcpt_mime_short", "rcpt_mime", "subject", "size", "user"]
                             .every((name) => col.name !== name);
@@ -231,7 +236,7 @@ define(["jquery", "app/common", "footable"],
             }
         };
 
-        ui.initHistoryTable = function (data, items, table, columns, expandFirst) {
+        ui.initHistoryTable = function (data, items, table, columns, expandFirst, postdrawCallback) {
             /* eslint-disable no-underscore-dangle */
             FooTable.Cell.extend("collapse", function () {
                 // call the original method
@@ -339,7 +344,8 @@ define(["jquery", "app/common", "footable"],
                             detail_row.find(".btn-sym-" + table + "-" + order)
                                 .addClass("active").siblings().removeClass("active");
                         }, 5);
-                    }
+                    },
+                    "postdraw.ft.table": postdrawCallback
                 }
             });
         };
@@ -508,19 +514,5 @@ define(["jquery", "app/common", "footable"],
             return {items: items, symbols: unsorted_symbols};
         };
 
-        ui.waitForRowsDisplayed = function (table, rows_total, callback, iteration) {
-            let i = (typeof iteration === "undefined") ? 10 : iteration;
-            const num_rows = $("#historyTable_" + table + " > tbody > tr:not(.footable-detail-row)").length;
-            if (num_rows === common.page_size[table] ||
-                num_rows === rows_total) {
-                return callback();
-            } else if (--i) {
-                setTimeout(() => {
-                    ui.waitForRowsDisplayed(table, rows_total, callback, i);
-                }, 500);
-            }
-            return null;
-        };
-
         return ui;
     });

interface/js/app/rspamd.js

@@ -465,9 +465,9 @@ define(["jquery", "app/common", "stickytabs", "visibility",
         checked_server = this.value;
         $("#selSrv [value=\"" + checked_server + "\"]").prop("checked", true);
         if (checked_server === "All SERVERS") {
-            $("#learnServers").show();
+            $("#learnServers").removeClass("invisible");
         } else {
-            $("#learnServers").hide();
+            $("#learnServers").addClass("invisible");
         }
         tabClick("#" + $("#tablist > .nav-item > .nav-link.active").attr("id"));
     });

interface/js/app/symbols.js

@@ -46,7 +46,7 @@ define(["jquery", "app/common", "footable"],
                     clear_altered();
                     common.alertMessage("alert-modal alert-success", "Symbols successfully saved");
                 },
-                complete: () => $("#save-alert button").removeAttr("disabled", true),
+                complete: () => $("#save-alert button").removeAttr("disabled"),
                 errorMessage: "Save symbols error",
                 method: "POST",
                 params: {
@@ -123,6 +123,7 @@ define(["jquery", "app/common", "footable"],
         }
         // @get symbols into modal form
         ui.getSymbols = function () {
+            $("#refresh, #updateSymbols").attr("disabled", true);
             clear_altered();
             common.query("symbols", {
                 success: function (json) {
@@ -216,10 +217,13 @@ define(["jquery", "app/common", "footable"],
                                 if (common.read_only) {
                                     $(".mb-disabled").attr("disabled", true);
                                 }
-                            }
+                            },
+                            "postdraw.ft.table":
+                                () => $("#refresh, #updateSymbols").removeAttr("disabled")
                         }
                     });
                 },
+                error: () => $("#refresh, #updateSymbols").removeAttr("disabled"),
                 server: common.getServer()
             });
         };
@@ -227,12 +231,14 @@ define(["jquery", "app/common", "footable"],
 
         $("#updateSymbols").on("click", (e) => {
             e.preventDefault();
+            $("#refresh, #updateSymbols").attr("disabled", true);
             clear_altered();
             common.query("symbols", {
                 success: function (data) {
                     const [items] = process_symbols_data(data[0].data);
                     common.tables.symbols.rows.load(items);
                 },
+                error: () => $("#refresh, #updateSymbols").removeAttr("disabled"),
                 server: common.getServer()
             });
         });

interface/js/app/upload.js

@@ -28,12 +28,14 @@ define(["jquery", "app/common", "app/libft"],
     ($, common, libft) => {
         "use strict";
         const ui = {};
+        let files = null;
+        let filesIdx = null;
+        let scanTextHeaders = {};
 
         function cleanTextUpload(source) {
             $("#" + source + "TextSource").val("");
         }
 
-        // @upload text
         function uploadText(data, source, headers) {
             let url = null;
             if (source === "spam") {
@@ -73,52 +75,72 @@ define(["jquery", "app/common", "app/libft"],
             });
         }
 
-        // @upload text
-        function scanText(data, headers) {
+        function enable_disable_scan_btn(disable) {
+            $("#scan button:not(#cleanScanHistory, #scanOptionsToggle)")
+                .prop("disabled", (disable || $.trim($("textarea").val()).length === 0));
+        }
+
+        function setFileInputFiles(i) {
+            const dt = new DataTransfer();
+            if (arguments.length) dt.items.add(files[i]);
+            $("#formFile").prop("files", dt.files);
+        }
+
+        function readFile(callback, i) {
+            const reader = new FileReader();
+            reader.readAsText(files[(arguments.length === 1) ? 0 : i]);
+            reader.onload = () => callback(reader.result);
+        }
+
+        function scanText(data) {
+            enable_disable_scan_btn(true);
             common.query("checkv2", {
                 data: data,
                 params: {
                     processData: false,
                 },
                 method: "POST",
-                headers: headers,
+                headers: scanTextHeaders,
                 success: function (neighbours_status) {
-                    function scrollTop(rows_total) {
-                        // Is there a way to get an event when all rows are loaded?
-                        libft.waitForRowsDisplayed("scan", rows_total, () => {
-                            $("#cleanScanHistory").removeAttr("disabled", true);
-                            $("html, body").animate({
-                                scrollTop: $("#scanResult").offset().top
-                            }, 1000);
-                        });
-                    }
-
                     const json = neighbours_status[0].data;
                     if (json.action) {
                         common.alertMessage("alert-success", "Data successfully scanned");
 
-                        const rows_total = $("#historyTable_scan > tbody > tr:not(.footable-detail-row)").length + 1;
                         const o = libft.process_history_v2({rows: [json]}, "scan");
                         const {items} = o;
                         common.symbols.scan.push(o.symbols[0]);
 
+                        if (files) items[0].file = files[filesIdx].name;
+
                         if (Object.prototype.hasOwnProperty.call(common.tables, "scan")) {
                             common.tables.scan.rows.load(items, true);
-                            scrollTop(rows_total);
                         } else {
-                            libft.destroyTable("scan");
                             require(["footable"], () => {
-                                // Is there a way to get an event when the table is destroyed?
-                                setTimeout(() => {
-                                    libft.initHistoryTable(data, items, "scan", libft.columns_v2("scan"), true);
-                                    scrollTop(rows_total);
-                                }, 200);
+                                libft.initHistoryTable(data, items, "scan", libft.columns_v2("scan"), true,
+                                    () => {
+                                        if (files && filesIdx < files.length - 1) {
+                                            readFile((result) => {
+                                                if (filesIdx === files.length - 1) {
+                                                    $("#scanMsgSource").val(result);
+                                                    setFileInputFiles(filesIdx);
+                                                }
+                                                scanText(result);
+                                            }, ++filesIdx);
+                                        } else {
+                                            enable_disable_scan_btn();
+                                            $("#cleanScanHistory").removeAttr("disabled");
+                                            $("html, body").animate({
+                                                scrollTop: $("#scanResult").offset().top
+                                            }, 1000);
+                                        }
+                                    });
                             });
                         }
                     } else {
                         common.alertMessage("alert-error", "Cannot scan data");
                     }
                 },
+                error: enable_disable_scan_btn,
                 errorMessage: "Cannot upload data",
                 statusCode: {
                     404: function () {
@@ -182,20 +204,18 @@ define(["jquery", "app/common", "app/libft"],
             $("#cleanScanHistory").attr("disabled", true);
         });
 
-        function enable_disable_scan_btn() {
-            $("#scan button:not(#cleanScanHistory, #scanOptionsToggle)")
-                .prop("disabled", ($.trim($("textarea").val()).length === 0));
-        }
         enable_disable_scan_btn();
         $("textarea").on("input", () => {
             enable_disable_scan_btn();
+            if (files) {
+                files = null;
+                setFileInputFiles();
+            }
         });
 
         $("#scanClean").on("click", () => {
-            $("#scan button:not(#cleanScanHistory, #scanOptionsToggle)").attr("disabled", true);
+            enable_disable_scan_btn(true);
             $("#scanForm")[0].reset();
-            $("#scanResult").hide();
-            $("#scanOutput tbody").remove();
             $("html, body").animate({scrollTop: 0}, 1000);
             return false;
         });
@@ -204,22 +224,26 @@ define(["jquery", "app/common", "app/libft"],
             $(this).closest(".card").slideUp();
         });
 
+        function getScanTextHeaders() {
+            scanTextHeaders = ["IP", "User", "From", "Rcpt", "Helo", "Hostname"].reduce((o, header) => {
+                const value = $("#scan-opt-" + header.toLowerCase()).val();
+                if (value !== "") o[header] = value;
+                return o;
+            }, {});
+            if ($("#scan-opt-pass-all").prop("checked")) scanTextHeaders.Pass = "all";
+        }
+
         $("[data-upload]").on("click", function () {
             const source = $(this).data("upload");
             const data = $("#scanMsgSource").val();
-            let headers = {};
             if ($.trim(data).length > 0) {
                 if (source === "scan") {
-                    headers = ["IP", "User", "From", "Rcpt", "Helo", "Hostname"].reduce((o, header) => {
-                        const value = $("#scan-opt-" + header.toLowerCase()).val();
-                        if (value !== "") o[header] = value;
-                        return o;
-                    }, {});
-                    if ($("#scan-opt-pass-all").prop("checked")) headers.Pass = "all";
-                    scanText(data, headers);
+                    getScanTextHeaders();
+                    scanText(data);
                 } else if (source === "compute-fuzzy") {
                     getFuzzyHashes(data);
                 } else {
+                    let headers = {};
                     if (source === "fuzzy") {
                         headers = {
                             flag: $("#fuzzyFlagText").val(),
@@ -234,5 +258,35 @@ define(["jquery", "app/common", "app/libft"],
             return false;
         });
 
+        const dragoverClassList = "outline-dashed-primary bg-primary-subtle";
+        $("#scanMsgSource")
+            .on("dragenter dragover dragleave drop", (e) => {
+                e.preventDefault();
+                e.stopPropagation();
+            })
+            .on("dragenter dragover", () => {
+                $("#scanMsgSource").addClass(dragoverClassList);
+            })
+            .on("dragleave drop", () => {
+                $("#scanMsgSource").removeClass(dragoverClassList);
+            })
+            .on("drop", (e) => {
+                ({files} = e.originalEvent.dataTransfer);
+                filesIdx = 0;
+
+                if (files.length === 1) {
+                    setFileInputFiles(0);
+                    enable_disable_scan_btn();
+                    readFile((result) => {
+                        $("#scanMsgSource").val(result);
+                        enable_disable_scan_btn();
+                    });
+                // eslint-disable-next-line no-alert
+                } else if (files.length < 10 || confirm("Are you sure you want to scan " + files.length + " files?")) {
+                    getScanTextHeaders();
+                    readFile((result) => scanText(result));
+                }
+            });
+
         return ui;
     });

lualib/lua_scanners/kaspersky_se.lua

@@ -48,6 +48,8 @@ local function kaspersky_se_config(opts)
     scan_mime_parts = true,
     scan_text_mime = false,
     scan_image_mime = false,
+    keepalive = true,
+    auth_string = nil
   }
 
   default_conf = lua_util.override_defaults(default_conf, opts)
@@ -118,6 +120,15 @@ local function kaspersky_se_check(task, content, digest, rule, maybe_part)
       ['X-KAV-Timeout'] = tostring(rule.timeout * 1000),
     }
 
+    local ip = task:get_from_ip()
+    if ip and ip:is_valid() then
+      hdrs['X-KAV-HostIP'] = tostring(ip)
+    end
+
+    if rule.auth_string then
+      hdrs['Authorization'] = rule.auth_string
+    end
+
     if task:has_from() then
       hdrs['X-KAV-ObjectURL'] = string.format('[from:%s]', task:get_from()[1].addr)
     end
@@ -158,6 +169,7 @@ local function kaspersky_se_check(task, content, digest, rule, maybe_part)
       body = req_body,
       headers = hdrs,
       timeout = rule.timeout,
+      keepalive = rule.keepalive,
     }
 
     local function kas_callback(http_err, code, body, headers)

lualib/lua_util.lua

@@ -32,6 +32,37 @@ local nospace = 1 - space
 local ptrim = space ^ 0 * lpeg.C((space ^ 0 * nospace ^ 1) ^ 0)
 local match = lpeg.match
 
+local function shallowcopy(orig)
+  local orig_type = type(orig)
+  local copy
+  if orig_type == 'table' then
+    copy = {}
+    for orig_key, orig_value in pairs(orig) do
+      copy[orig_key] = orig_value
+    end
+  else
+    copy = orig
+  end
+  return copy
+end
+local function deepcopy(orig)
+  local orig_type = type(orig)
+  local copy
+  if orig_type == 'table' then
+    copy = {}
+    for orig_key, orig_value in next, orig, nil do
+      copy[deepcopy(orig_key)] = deepcopy(orig_value)
+    end
+    if getmetatable(orig) then
+      setmetatable(copy, deepcopy(getmetatable(orig)))
+    end
+  else
+    -- number, string, boolean, etc
+    copy = orig
+  end
+  return copy
+end
+
 lupa.configure('{%', '%}', '{=', '=}', '{#', '#}', {
   keep_trailing_newline = true,
   autoescape = false,
@@ -42,6 +73,10 @@ lupa.filters.pbkdf = function(s)
   return cr.pbkdf(s)
 end
 
+-- Dirty hacks to avoid shared state
+package.loaded['lupa'] = nil
+local lupa_orig = require "lupa"
+
 local function rspamd_str_split(s, sep)
   local gr
   if not sep then
@@ -157,48 +192,88 @@ exports.template = function(tmpl, keys)
 end
 
 local function enrich_template_with_globals(env)
-  local newenv = exports.shallowcopy(env)
+  local newenv = shallowcopy(env)
   newenv.paths = rspamd_paths
   newenv.env = rspamd_env
 
   return newenv
 end
 --[[[
+-- @function lua_util.jinja_template(text, env[, skip_global_env][, is_orig][, custom_filters])
 -- Replaces values in a text template according to jinja2 syntax
 -- @param {string} text text containing variables
 -- @param {table} replacements key/value pairs for replacements
 -- @param {boolean} skip_global_env don't export Rspamd superglobals
+-- @param {boolean} is_orig use the original lupa configuration with {% raw %}`{{`{% endraw %} for variables
+-- @param {table} custom_filters custom filters to use (or nil if not needed)
 -- @return {string} string containing replaced values
 -- @example
+-- lua_util.jinja_template("HELLO {=FOO=} {=BAR=}!", {['FOO'] = 'LUA', ['BAR'] = 'WORLD'})
 -- "HELLO LUA WORLD!"
 --]]
-exports.jinja_template = function(text, env, skip_global_env)
+exports.jinja_template = function(text, env, skip_global_env, is_orig, custom_filters)
+  local lupa_to_use = is_orig and lupa_orig or lupa
   if not skip_global_env then
     env = enrich_template_with_globals(env)
   end
 
-  return lupa.expand(text, env)
+  local orig_filters = {}
+  if type(custom_filters) == 'table' then
+    for k, v in pairs(custom_filters) do
+      orig_filters[k] = lupa_to_use.filters[k]
+      lupa_to_use.filters[k] = v
+    end
+  end
+
+  local result = lupa_to_use.expand(text, env)
+
+  -- Restore custom filters
+  if type(custom_filters) == 'table' then
+    for k, _ in pairs(custom_filters) do
+      lupa_to_use.filters[k] = orig_filters[k]
+    end
+  end
+
+  return result
 end
 
 --[[[
+-- @function lua_util.jinja_file(filename, env[, skip_global_env][, is_orig][, custom_filters])
 -- Replaces values in a text template according to jinja2 syntax
 -- @param {string} filename name of file to expand
 -- @param {table} replacements key/value pairs for replacements
 -- @param {boolean} skip_global_env don't export Rspamd superglobals
+-- @param {boolean} is_orig use the original lupa configuration with {% raw %}`{{`{% endraw %} for variables
+-- @param {table} custom_filters custom filters to use (or nil if not needed)
 -- @return {string} string containing replaced values
 -- @example
+-- lua_util.jinja_template("HELLO {=FOO=} {=BAR=}!", {['FOO'] = 'LUA', ['BAR'] = 'WORLD'})
 -- "HELLO LUA WORLD!"
 --]]
-exports.jinja_template_file = function(filename, env, skip_global_env)
+exports.jinja_template_file = function(filename, env, skip_global_env, is_orig, custom_filters)
+  local lupa_to_use = is_orig and lupa_orig or lupa
   if not skip_global_env then
     env = enrich_template_with_globals(env)
   end
 
-  return lupa.expand_file(filename, env)
+  local orig_filters = {}
+  if type(custom_filters) == 'table' then
+    for k, v in pairs(custom_filters) do
+      orig_filters[k] = lupa_to_use.filters[k]
+      lupa_to_use.filters[k] = v
+    end
+  end
+
+  local result = lupa_to_use.expand_file(filename, env)
+
+  -- Restore custom filters
+  if type(custom_filters) == 'table' then
+    for k, _ in pairs(custom_filters) do
+      lupa_to_use.filters[k] = orig_filters[k]
+    end
+  end
+
+  return result
 end
 
 exports.remove_email_aliases = function(email_addr)
@@ -1019,6 +1094,7 @@ exports.extract_specific_urls = function(params_or_task, lim, need_emails, filte
   return exports.filter_specific_urls(urls, params)
 end
 
+
 --[[[
 -- @function lua_util.deepcopy(table)
 -- params: {
@@ -1026,24 +1102,6 @@ end
 -- }
 -- Performs deep copy of the table. Including metatables
 --]]
-local function deepcopy(orig)
-  local orig_type = type(orig)
-  local copy
-  if orig_type == 'table' then
-    copy = {}
-    for orig_key, orig_value in next, orig, nil do
-      copy[deepcopy(orig_key)] = deepcopy(orig_value)
-    end
-    if getmetatable(orig) then
-      setmetatable(copy, deepcopy(getmetatable(orig)))
-    end
-  else
-    -- number, string, boolean, etc
-    copy = orig
-  end
-  return copy
-end
-
 exports.deepcopy = deepcopy
 
 --[[[
@@ -1077,19 +1135,7 @@ exports.deepsort = deepsort
 -- @function lua_util.shallowcopy(tbl)
 -- Performs shallow (and fast) copy of a table or another Lua type
 --]]
-exports.shallowcopy = function(orig)
-  local orig_type = type(orig)
-  local copy
-  if orig_type == 'table' then
-    copy = {}
-    for orig_key, orig_value in pairs(orig) do
-      copy[orig_key] = orig_value
-    end
-  else
-    copy = orig
-  end
-  return copy
-end
+exports.shallowcopy = shallowcopy
 
 -- Debugging support
 local logger = require "rspamd_logger"

lualib/rspamadm/dmarc_report.lua

@@ -19,7 +19,6 @@ local lua_util = require "lua_util"
 local logger = require "rspamd_logger"
 local lua_redis = require "lua_redis"
 local dmarc_common = require "plugins/dmarc"
-local lupa = require "lupa"
 local rspamd_mempool = require "rspamd_mempool"
 local rspamd_url = require "rspamd_url"
 local rspamd_text = require "rspamd_text"
@@ -176,8 +175,6 @@ local function escape_xml(input)
 
   return ''
 end
-lupa.filters.escape_xml = escape_xml
 
 -- Creates report XML header
 local function report_header(reporting_domain, report_start, report_end, domain_policy)
@@ -211,7 +208,10 @@ local function report_header(reporting_domain, report_start, report_end, domain_
     report_end = report_end,
     domain_policy = domain_policy,
     reporting_domain = reporting_domain,
-  }, true)
+  }, true, false,
+      {
+        escape_xml = escape_xml
+      })
 end
 
 -- Generate xml entry for a preprocessed redis row
@@ -248,7 +248,10 @@ local function entry_to_xml(data)
   </auth_results>
 </record>
 ]]
-  return lua_util.jinja_template(xml_template, { data = data }, true)
+  return lua_util.jinja_template(xml_template, { data = data }, true,
+      false, {
+        escape_xml = escape_xml
+      })
 end
 
 -- Process a report entry stored in Redis splitting it to a lua table
@@ -534,10 +537,15 @@ local function prepare_report(opts, start_time, end_time, rep_key)
     message_id = rspamd_util.random_hex(16) .. '@' .. report_settings.msgid_from,
     report_start = start_time,
     report_end = end_time
-  }, true)
+  }, true,
+      false, {
+        escape_xml = escape_xml
+      })
   local rfooter = lua_util.jinja_template(report_footer, {
     uuid = uuid,
-  }, true)
+  }, true, false, {
+    escape_xml = escape_xml
+  })
   local message = rspamd_text.fromtable {
     (rhead:gsub("\n", "\r\n")),
     rspamd_util.encode_base64(rspamd_util.gzip_compress(xml_to_compress), 73),

rules/regexp/headers.lua

@@ -938,6 +938,13 @@ reconf['HAS_GOOGLE_FIREBASE_URL'] = {
   group = 'url'
 }
 
+reconf['HAS_FILE_URL'] = {
+  re = '/^file:\\/\\//{url}i',
+  description = 'Contains file:// URL',
+  score = 2.0,
+  group = 'url'
+}
+
 reconf['XM_UA_NO_VERSION'] = {
   re = string.format('(!%s && !%s) && (%s || %s)',
       'X-Mailer=/https?:/H',

src/fuzzy_storage.c

@@ -1,5 +1,5 @@
 /*
- * Copyright 2023 Vsevolod Stakhov
+ * Copyright 2024 Vsevolod Stakhov
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -107,7 +107,38 @@ struct rspamd_leaky_bucket_elt {
 };
 
 static const guint64 rspamd_fuzzy_storage_magic = 0x291a3253eb1b3ea5ULL;
+
+static int64_t
+fuzzy_kp_hash(const unsigned char *p)
+{
+	int64_t res;
+
+	memcpy(&res, p, sizeof(res));
+	return res;
+}
+static bool
+fuzzy_kp_equal(gconstpointer a, gconstpointer b)
+{
+	const guchar *pa = a, *pb = b;
+
+	return (memcmp(pa, pb, RSPAMD_FUZZY_KEYLEN) == 0);
+}
+
 KHASH_SET_INIT_INT(fuzzy_key_ids_set);
+KHASH_INIT(fuzzy_key_flag_stat, int, struct fuzzy_key_stat, 1, kh_int_hash_func,
+		   kh_int_hash_equal);
+struct fuzzy_key {
+	struct rspamd_cryptobox_keypair *key;
+	struct rspamd_cryptobox_pubkey *pk;
+	struct fuzzy_key_stat *stat;
+	khash_t(fuzzy_key_flag_stat) * flags_stat;
+	khash_t(fuzzy_key_ids_set) * forbidden_ids;
+	ref_entry_t ref;
+};
+
+KHASH_INIT(rspamd_fuzzy_keys_hash,
+		   const unsigned char *, struct fuzzy_key *, 1,
+		   fuzzy_kp_hash, fuzzy_kp_equal);
 
 struct rspamd_fuzzy_storage_ctx {
 	guint64 magic;
@@ -133,6 +164,7 @@ struct rspamd_fuzzy_storage_ctx {
 	const ucl_object_t *delay_whitelist_map;
 	const ucl_object_t *blocked_map;
 	const ucl_object_t *ratelimit_whitelist_map;
+	const ucl_object_t *dynamic_keys_map;
 
 	guint keypair_cache_size;
 	ev_timer stat_ev;
@@ -141,7 +173,10 @@ struct rspamd_fuzzy_storage_ctx {
 	/* Local keypair */
 	struct rspamd_cryptobox_keypair *default_keypair; /* Bad clash, need for parse keypair */
 	struct fuzzy_key *default_key;
-	GHashTable *keys;
+	khash_t(rspamd_fuzzy_keys_hash) * keys;
+	/* Those are loaded via map */
+	khash_t(rspamd_fuzzy_keys_hash) * dynamic_keys;
+
 	gboolean encrypted_only;
 	gboolean read_only;
 	gboolean dedicated_update_worker;
@@ -207,16 +242,6 @@ struct fuzzy_peer_request {
 	struct fuzzy_peer_cmd cmd;
 };
 
-KHASH_INIT(fuzzy_key_flag_stat, int, struct fuzzy_key_stat, 1, kh_int_hash_func,
-		   kh_int_hash_equal);
-struct fuzzy_key {
-	struct rspamd_cryptobox_keypair *key;
-	struct rspamd_cryptobox_pubkey *pk;
-	struct fuzzy_key_stat *stat;
-	khash_t(fuzzy_key_flag_stat) * flags_stat;
-	khash_t(fuzzy_key_ids_set) * forbidden_ids;
-};
-
 struct rspamd_updates_cbdata {
 	GArray *updates_pending;
 	struct rspamd_fuzzy_storage_ctx *ctx;
@@ -233,6 +258,151 @@ static gboolean rspamd_fuzzy_check_client(struct rspamd_fuzzy_storage_ctx *ctx,
 static void rspamd_fuzzy_maybe_call_blacklisted(struct rspamd_fuzzy_storage_ctx *ctx,
 												rspamd_inet_addr_t *addr,
 												const gchar *reason);
+static struct fuzzy_key *fuzzy_add_keypair_from_ucl(const ucl_object_t *obj,
+													khash_t(rspamd_fuzzy_keys_hash) * target);
+
+struct fuzzy_keymap_ucl_buf {
+	rspamd_fstring_t *buf;
+	struct rspamd_fuzzy_storage_ctx *ctx;
+};
+
+/* Callbacks for reading json dynamic rules */
+static gchar *
+ucl_keymap_read_cb(gchar *chunk,
+				   gint len,
+				   struct map_cb_data *data,
+				   gboolean final)
+{
+	struct fuzzy_keymap_ucl_buf *jb, *pd;
+
+	pd = data->prev_data;
+
+	g_assert(pd != NULL);
+
+	if (data->cur_data == NULL) {
+		jb = g_malloc0(sizeof(*jb));
+		jb->ctx = pd->ctx;
+		data->cur_data = jb;
+	}
+	else {
+		jb = data->cur_data;
+	}
+
+	if (jb->buf == NULL) {
+		/* Allocate memory for buffer */
+		jb->buf = rspamd_fstring_sized_new(MAX(len, 4096));
+	}
+
+	jb->buf = rspamd_fstring_append(jb->buf, chunk, len);
+
+	return NULL;
+}
+
+static void
+ucl_keymap_fin_cb(struct map_cb_data *data, void **target)
+{
+	struct fuzzy_keymap_ucl_buf *jb;
+	ucl_object_t *top;
+	struct ucl_parser *parser;
+	struct rspamd_config *cfg;
+
+	/* Now parse ucl */
+	if (data->cur_data) {
+		jb = data->cur_data;
+		cfg = jb->ctx->cfg;
+	}
+	else {
+		msg_err("no cur data in the map! might be a bug");
+		return;
+	}
+
+	if (jb->buf->len == 0) {
+		msg_err_config("no data read");
+
+		return;
+	}
+
+	parser = ucl_parser_new(UCL_PARSER_NO_FILEVARS);
+
+	if (!ucl_parser_add_chunk(parser, jb->buf->str, jb->buf->len)) {
+		msg_err_config("cannot load ucl data: parse error %s",
+					   ucl_parser_get_error(parser));
+		ucl_parser_free(parser);
+		return;
+	}
+
+	top = ucl_parser_get_object(parser);
+	ucl_parser_free(parser);
+
+	if (ucl_object_type(top) != UCL_ARRAY) {
+		ucl_object_unref(top);
+		msg_err_config("loaded ucl is not an array");
+		return;
+	}
+
+	if (target) {
+		*target = data->cur_data;
+	}
+
+	if (data->prev_data) {
+		jb = data->prev_data;
+		/* Clean prev data */
+		if (jb->buf) {
+			rspamd_fstring_free(jb->buf);
+		}
+
+		/* Clean the existing keys */
+		struct fuzzy_key *key;
+		kh_foreach_value(jb->ctx->dynamic_keys, key, {
+			REF_RELEASE(key);
+		});
+		kh_clear(rspamd_fuzzy_keys_hash, jb->ctx->dynamic_keys);
+
+		/* Insert new keys */
+		const ucl_object_t *cur;
+		ucl_object_iter_t it = NULL;
+		int success = 0;
+
+		while ((cur = ucl_object_iterate(top, &it, true)) != NULL) {
+			struct fuzzy_key *nk;
+
+			nk = fuzzy_add_keypair_from_ucl(cur, jb->ctx->dynamic_keys);
+
+			if (nk == NULL) {
+				msg_warn_config("cannot add dynamic keypair");
+			}
+			success++;
+		}
+
+		msg_info_config("loaded %d dynamic keypairs", success);
+
+		g_free(jb);
+	}
+
+	ucl_object_unref(top);
+}
+
+static void
+ucl_keymap_dtor_cb(struct map_cb_data *data)
+{
+	struct fuzzy_keymap_ucl_buf *jb;
+
+	if (data->cur_data) {
+		jb = data->cur_data;
+		/* Clean prev data */
+		if (jb->buf) {
+			rspamd_fstring_free(jb->buf);
+		}
+
+		struct fuzzy_key *key;
+		kh_foreach_value(jb->ctx->dynamic_keys, key, {
+			REF_RELEASE(key);
+		});
+		kh_destroy(rspamd_fuzzy_keys_hash, jb->ctx->dynamic_keys);
+
+		g_free(jb);
+	}
+}
 
 static gboolean
 rspamd_fuzzy_check_ratelimit(struct fuzzy_session *session)
@@ -493,6 +663,16 @@ fuzzy_key_dtor(gpointer p)
 	}
 }
 
+static void
+fuzzy_hash_table_dtor(khash_t(rspamd_fuzzy_keys_hash) * hash)
+{
+	struct fuzzy_key *key;
+	kh_foreach_value(hash, key, {
+		REF_RELEASE(key);
+	});
+	kh_destroy(rspamd_fuzzy_keys_hash, hash);
+}
+
 static void
 fuzzy_count_callback(guint64 count, void *ud)
 {
@@ -1446,9 +1626,9 @@ rspamd_fuzzy_decrypt_command(struct fuzzy_session *s, guchar *buf, gsize buflen)
 {
 	struct rspamd_fuzzy_encrypted_req_hdr hdr;
 	struct rspamd_cryptobox_pubkey *rk;
-	struct fuzzy_key *key;
+	struct fuzzy_key *key = NULL;
 
-	if (s->ctx->default_key == NULL) {
+	if (s->ctx->default_key == NULL && s->ctx->dynamic_keys == NULL) {
 		msg_warn("received encrypted request when encryption is not enabled");
 		return FALSE;
 	}
@@ -1463,16 +1643,31 @@ rspamd_fuzzy_decrypt_command(struct fuzzy_session *s, guchar *buf, gsize buflen)
 	buflen -= sizeof(hdr);
 
 	/* Try to find the desired key */
-	key = g_hash_table_lookup(s->ctx->keys, hdr.key_id);
+	khiter_t k = kh_get(rspamd_fuzzy_keys_hash, s->ctx->keys, hdr.key_id);
+	if (k == kh_end(s->ctx->keys)) {
 
-	if (key == NULL) {
-		/* Unknown key, assume default one */
 		key = s->ctx->default_key;
+
+		/* Check dynamic keys */
+		if (s->ctx->dynamic_keys) {
+			k = kh_get(rspamd_fuzzy_keys_hash, s->ctx->dynamic_keys, hdr.key_id);
+
+			if (k != kh_end(s->ctx->keys)) {
+				key = kh_val(s->ctx->dynamic_keys, k);
+			}
+		}
+	}
+	else {
+		key = kh_val(s->ctx->keys, k);
 	}
 
-	s->key = key;
+	if (key == NULL) {
+		/* Cannot find any suitable decryption key */
+		msg_debug("cannot find suitable decryption key");
+		return FALSE;
+	}
 
-	/* Now process keypair */
+	/* Now process the remote pubkey */
 	rk = rspamd_pubkey_from_bin(hdr.pubkey, sizeof(hdr.pubkey),
 								RSPAMD_KEYPAIR_KEX, RSPAMD_CRYPTOBOX_MODE_25519);
 
@@ -1482,6 +1677,7 @@ rspamd_fuzzy_decrypt_command(struct fuzzy_session *s, guchar *buf, gsize buflen)
 		return FALSE;
 	}
 
+	/* Try to get the cached NM */
 	rspamd_keypair_cache_process(s->ctx->keypair_cache, key->key, rk);
 
 	/* Now decrypt request */
@@ -1495,6 +1691,9 @@ rspamd_fuzzy_decrypt_command(struct fuzzy_session *s, guchar *buf, gsize buflen)
 		return FALSE;
 	}
 
+	s->key = key;
+	REF_RETAIN(key);
+
 	memcpy(s->nm, rspamd_pubkey_get_nm(rk, key->key), sizeof(s->nm));
 	rspamd_pubkey_unref(rk);
 
@@ -1750,6 +1949,10 @@ fuzzy_session_destroy(gpointer d)
 		g_free(session->extensions);
 	}
 
+	if (session->key) {
+		REF_RELEASE(session->key);
+	}
+
 	g_free(session);
 }
 
@@ -2088,64 +2291,65 @@ rspamd_fuzzy_storage_stat_key(const struct fuzzy_key_stat *key_stat)
 	return res;
 }
 
-static ucl_object_t *
-rspamd_fuzzy_stat_to_ucl(struct rspamd_fuzzy_storage_ctx *ctx, gboolean ip_stat)
+static void
+rspamd_fuzzy_key_stat_iter(const unsigned char *pk_iter, struct fuzzy_key *fuzzy_key, ucl_object_t *keys_obj, gboolean ip_stat)
 {
-	struct fuzzy_key_stat *key_stat;
-	GHashTableIter it;
-	struct fuzzy_key *fuzzy_key;
-	ucl_object_t *obj, *keys_obj, *elt, *ip_elt, *ip_cur;
-	gpointer k, v;
-	gint i;
+	struct fuzzy_key_stat *key_stat = fuzzy_key->stat;
 	gchar keyname[17];
 
-	obj = ucl_object_typed_new(UCL_OBJECT);
+	if (key_stat) {
+		rspamd_snprintf(keyname, sizeof(keyname), "%8bs", pk_iter);
 
-	keys_obj = ucl_object_typed_new(UCL_OBJECT);
-	g_hash_table_iter_init(&it, ctx->keys);
+		ucl_object_t *elt = rspamd_fuzzy_storage_stat_key(key_stat);
 
-	while (g_hash_table_iter_next(&it, &k, &v)) {
-		fuzzy_key = v;
-		key_stat = fuzzy_key->stat;
+		if (key_stat->last_ips && ip_stat) {
+			int i = 0;
+			ucl_object_t *ip_elt = ucl_object_typed_new(UCL_OBJECT);
+			gpointer k, v;
 
-		if (key_stat) {
-			rspamd_snprintf(keyname, sizeof(keyname), "%8bs", k);
+			while ((i = rspamd_lru_hash_foreach(key_stat->last_ips,
+												i, &k, &v)) != -1) {
+				ucl_object_t *ip_cur = rspamd_fuzzy_storage_stat_key(v);
+				ucl_object_insert_key(ip_elt, ip_cur,
+									  rspamd_inet_address_to_string(k), 0, true);
+			}
+			ucl_object_insert_key(elt, ip_elt, "ips", 0, false);
+		}
 
-			elt = rspamd_fuzzy_storage_stat_key(key_stat);
+		int flag;
+		struct fuzzy_key_stat *flag_stat;
+		ucl_object_t *flags_ucl = ucl_object_typed_new(UCL_OBJECT);
 
-			if (key_stat->last_ips && ip_stat) {
-				i = 0;
+		kh_foreach_key_value_ptr(fuzzy_key->flags_stat, flag, flag_stat, {
+			char intbuf[16];
+			rspamd_snprintf(intbuf, sizeof(intbuf), "%d", flag);
+			ucl_object_insert_key(flags_ucl, rspamd_fuzzy_storage_stat_key(flag_stat),
+								  intbuf, 0, true);
+		});
 
-				ip_elt = ucl_object_typed_new(UCL_OBJECT);
+		ucl_object_insert_key(elt, flags_ucl, "flags", 0, false);
 
-				while ((i = rspamd_lru_hash_foreach(key_stat->last_ips,
-													i, &k, &v)) != -1) {
-					ip_cur = rspamd_fuzzy_storage_stat_key(v);
-					ucl_object_insert_key(ip_elt, ip_cur,
-										  rspamd_inet_address_to_string(k), 0, true);
-				}
-				ucl_object_insert_key(elt, ip_elt, "ips", 0, false);
-			}
+		ucl_object_insert_key(elt,
+							  rspamd_keypair_to_ucl(fuzzy_key->key, RSPAMD_KEYPAIR_DUMP_NO_SECRET | RSPAMD_KEYPAIR_DUMP_FLATTENED),
+							  "keypair", 0, false);
+		ucl_object_insert_key(keys_obj, elt, keyname, 0, true);
+	}
+}
 
-			int flag;
-			struct fuzzy_key_stat *flag_stat;
-			ucl_object_t *flags_ucl = ucl_object_typed_new(UCL_OBJECT);
+static ucl_object_t *
+rspamd_fuzzy_stat_to_ucl(struct rspamd_fuzzy_storage_ctx *ctx, gboolean ip_stat)
+{
+	struct fuzzy_key *fuzzy_key;
+	ucl_object_t *obj, *keys_obj, *elt, *ip_elt;
+	const unsigned char *pk_iter;
 
-			kh_foreach_key_value_ptr(fuzzy_key->flags_stat, flag, flag_stat, {
-				char intbuf[16];
-				rspamd_snprintf(intbuf, sizeof(intbuf), "%d", flag);
-				ucl_object_insert_key(flags_ucl, rspamd_fuzzy_storage_stat_key(flag_stat),
-									  intbuf, 0, true);
-			});
+	obj = ucl_object_typed_new(UCL_OBJECT);
 
-			ucl_object_insert_key(elt, flags_ucl, "flags", 0, false);
+	keys_obj = ucl_object_typed_new(UCL_OBJECT);
 
-			ucl_object_insert_key(elt,
-								  rspamd_keypair_to_ucl(fuzzy_key->key, RSPAMD_KEYPAIR_DUMP_NO_SECRET | RSPAMD_KEYPAIR_DUMP_FLATTENED),
-								  "keypair", 0, false);
-			ucl_object_insert_key(keys_obj, elt, keyname, 0, true);
-		}
-	}
+	kh_foreach(ctx->keys, pk_iter, fuzzy_key, {
+		rspamd_fuzzy_key_stat_iter(pk_iter, fuzzy_key, keys_obj, ip_stat);
+	});
 
 	ucl_object_insert_key(obj, keys_obj, "keys", 0, false);
 
@@ -2172,8 +2376,8 @@ rspamd_fuzzy_stat_to_ucl(struct rspamd_fuzzy_storage_ctx *ctx, gboolean ip_stat)
 						  false);
 
 	if (ctx->errors_ips && ip_stat) {
-		i = 0;
-
+		gpointer k, v;
+		int i = 0;
 		ip_elt = ucl_object_typed_new(UCL_OBJECT);
 
 		while ((i = rspamd_lru_hash_foreach(ctx->errors_ips, i, &k, &v)) != -1) {
@@ -2192,7 +2396,7 @@ rspamd_fuzzy_stat_to_ucl(struct rspamd_fuzzy_storage_ctx *ctx, gboolean ip_stat)
 	/* Checked by epoch */
 	elt = ucl_object_typed_new(UCL_ARRAY);
 
-	for (i = RSPAMD_FUZZY_EPOCH10; i < RSPAMD_FUZZY_EPOCH_MAX; i++) {
+	for (int i = RSPAMD_FUZZY_EPOCH10; i < RSPAMD_FUZZY_EPOCH_MAX; i++) {
 		ucl_array_append(elt,
 						 ucl_object_fromint(ctx->stat.fuzzy_hashes_checked[i]));
 	}
@@ -2202,7 +2406,7 @@ rspamd_fuzzy_stat_to_ucl(struct rspamd_fuzzy_storage_ctx *ctx, gboolean ip_stat)
 	/* Shingles by epoch */
 	elt = ucl_object_typed_new(UCL_ARRAY);
 
-	for (i = RSPAMD_FUZZY_EPOCH10; i < RSPAMD_FUZZY_EPOCH_MAX; i++) {
+	for (int i = RSPAMD_FUZZY_EPOCH10; i < RSPAMD_FUZZY_EPOCH_MAX; i++) {
 		ucl_array_append(elt,
 						 ucl_object_fromint(ctx->stat.fuzzy_shingles_checked[i]));
 	}
@@ -2212,7 +2416,7 @@ rspamd_fuzzy_stat_to_ucl(struct rspamd_fuzzy_storage_ctx *ctx, gboolean ip_stat)
 	/* Matched by epoch */
 	elt = ucl_object_typed_new(UCL_ARRAY);
 
-	for (i = RSPAMD_FUZZY_EPOCH10; i < RSPAMD_FUZZY_EPOCH_MAX; i++) {
+	for (int i = RSPAMD_FUZZY_EPOCH10; i < RSPAMD_FUZZY_EPOCH_MAX; i++) {
 		ucl_array_append(elt,
 						 ucl_object_fromint(ctx->stat.fuzzy_hashes_found[i]));
 	}
@@ -2558,6 +2762,87 @@ fuzzy_parse_ids(rspamd_mempool_t *pool,
 	return FALSE;
 }
 
+static struct fuzzy_key *
+fuzzy_add_keypair_from_ucl(const ucl_object_t *obj, khash_t(rspamd_fuzzy_keys_hash) * target)
+{
+	struct rspamd_cryptobox_keypair *kp = rspamd_keypair_from_ucl(obj);
+
+	if (kp == NULL) {
+		return NULL;
+	}
+
+	if (rspamd_keypair_alg(kp) != RSPAMD_CRYPTOBOX_MODE_25519 ||
+		rspamd_keypair_type(kp) != RSPAMD_KEYPAIR_KEX) {
+		return FALSE;
+	}
+
+	struct fuzzy_key *key = g_malloc0(sizeof(*key));
+	REF_INIT_RETAIN(key, fuzzy_key_dtor);
+	key->key = kp;
+	struct fuzzy_key_stat *keystat = g_malloc0(sizeof(*keystat));
+	REF_INIT_RETAIN(keystat, fuzzy_key_stat_dtor);
+	/* Hash of ip -> fuzzy_key_stat */
+	keystat->last_ips = rspamd_lru_hash_new_full(1024,
+												 (GDestroyNotify) rspamd_inet_address_free,
+												 fuzzy_key_stat_unref,
+												 rspamd_inet_address_hash, rspamd_inet_address_equal);
+	key->stat = keystat;
+	key->flags_stat = kh_init(fuzzy_key_flag_stat);
+	/* Preallocate some space for flags */
+	kh_resize(fuzzy_key_flag_stat, key->flags_stat, 8);
+	const guchar *pk = rspamd_keypair_component(kp, RSPAMD_KEYPAIR_COMPONENT_PK,
+												NULL);
+	keystat->keypair = rspamd_keypair_ref(kp);
+	/* We map entries by pubkey in binary form for faster lookup */
+	khiter_t k;
+	int r;
+
+	k = kh_put(rspamd_fuzzy_keys_hash, target, pk, &r);
+
+	if (r == 0) {
+		msg_err("duplicate keypair found: pk=%*bs",
+				32, pk);
+		REF_RELEASE(key);
+
+		return FALSE;
+	}
+	else if (r == -1) {
+		msg_err("hash insertion error: pk=%*bs",
+				32, pk);
+		REF_RELEASE(key);
+
+		return FALSE;
+	}
+
+	kh_val(target, k) = key;
+
+	const ucl_object_t *extensions = rspamd_keypair_get_extensions(kp);
+
+	if (extensions) {
+		const ucl_object_t *forbidden_ids = ucl_object_lookup(extensions, "forbidden_ids");
+
+		if (forbidden_ids && ucl_object_type(forbidden_ids) == UCL_ARRAY) {
+			key->forbidden_ids = kh_init(fuzzy_key_ids_set);
+			const ucl_object_t *cur;
+			ucl_object_iter_t it = NULL;
+
+			while ((cur = ucl_object_iterate(forbidden_ids, &it, true)) != NULL) {
+				if (ucl_object_type(cur) == UCL_INT || ucl_object_type(cur) == UCL_FLOAT) {
+					int id = ucl_object_toint(cur);
+					int r;
+
+					kh_put(fuzzy_key_ids_set, key->forbidden_ids, id, &r);
+				}
+			}
+		}
+	}
+
+	msg_debug("loaded keypair %*bs", rspamd_cryptobox_pk_bytes(RSPAMD_CRYPTOBOX_MODE_25519), pk);
+
+	return key;
+}
+
+
 static gboolean
 fuzzy_parse_keypair(rspamd_mempool_t *pool,
 					const ucl_object_t *obj,
@@ -2567,11 +2852,8 @@ fuzzy_parse_keypair(rspamd_mempool_t *pool,
 {
 	struct rspamd_rcl_struct_parser *pd = ud;
 	struct rspamd_fuzzy_storage_ctx *ctx;
-	struct rspamd_cryptobox_keypair *kp;
-	struct fuzzy_key_stat *keystat;
 	struct fuzzy_key *key;
 	const ucl_object_t *cur;
-	const guchar *pk;
 	ucl_object_iter_t it = NULL;
 	gboolean ret;
 
@@ -2588,57 +2870,14 @@ fuzzy_parse_keypair(rspamd_mempool_t *pool,
 			return ret;
 		}
 
-		/* Insert key to the hash table */
-		kp = ctx->default_keypair;
+		key = fuzzy_add_keypair_from_ucl(obj, ctx->keys);
 
-		if (kp == NULL) {
+		if (key == NULL) {
 			return FALSE;
 		}
 
-		if (rspamd_keypair_alg(kp) != RSPAMD_CRYPTOBOX_MODE_25519 ||
-			rspamd_keypair_type(kp) != RSPAMD_KEYPAIR_KEX) {
-			return FALSE;
-		}
-
-		key = g_malloc0(sizeof(*key));
-		key->key = kp;
-		keystat = g_malloc0(sizeof(*keystat));
-		REF_INIT_RETAIN(keystat, fuzzy_key_stat_dtor);
-		/* Hash of ip -> fuzzy_key_stat */
-		keystat->last_ips = rspamd_lru_hash_new_full(1024,
-													 (GDestroyNotify) rspamd_inet_address_free,
-													 fuzzy_key_stat_unref,
-													 rspamd_inet_address_hash, rspamd_inet_address_equal);
-		key->stat = keystat;
-		key->flags_stat = kh_init(fuzzy_key_flag_stat);
-		/* Preallocate some space for flags */
-		kh_resize(fuzzy_key_flag_stat, key->flags_stat, 8);
-		pk = rspamd_keypair_component(kp, RSPAMD_KEYPAIR_COMPONENT_PK,
-									  NULL);
-		keystat->keypair = rspamd_keypair_ref(kp);
-		/* We map entries by pubkey in binary form for faster lookup */
-		g_hash_table_insert(ctx->keys, (gpointer) pk, key);
+		/* Use the last one ? */
 		ctx->default_key = key;
-
-		const ucl_object_t *extensions = rspamd_keypair_get_extensions(kp);
-
-		if (extensions) {
-			const ucl_object_t *forbidden_ids = ucl_object_lookup(extensions, "forbidden_ids");
-
-			if (forbidden_ids && ucl_object_type(forbidden_ids) == UCL_ARRAY) {
-				key->forbidden_ids = kh_init(fuzzy_key_ids_set);
-				while ((cur = ucl_object_iterate(forbidden_ids, &it, true)) != NULL) {
-					if (ucl_object_type(cur) == UCL_INT || ucl_object_type(cur) == UCL_FLOAT) {
-						int id = ucl_object_toint(cur);
-						int r;
-
-						kh_put(fuzzy_key_ids_set, key->forbidden_ids, id, &r);
-					}
-				}
-			}
-		}
-
-		msg_debug_pool_check("loaded keypair %*xs", 8, pk);
 	}
 	else if (ucl_object_type(obj) == UCL_ARRAY) {
 		while ((cur = ucl_object_iterate(obj, &it, true)) != NULL) {
@@ -2651,20 +2890,6 @@ fuzzy_parse_keypair(rspamd_mempool_t *pool,
 	return TRUE;
 }
 
-static guint
-fuzzy_kp_hash(gconstpointer p)
-{
-	return *(guint *) p;
-}
-
-static gboolean
-fuzzy_kp_equal(gconstpointer a, gconstpointer b)
-{
-	const guchar *pa = a, *pb = b;
-
-	return (memcmp(pa, pb, RSPAMD_FUZZY_KEYLEN) == 0);
-}
-
 gpointer
 init_fuzzy(struct rspamd_config *cfg)
 {
@@ -2682,10 +2907,9 @@ init_fuzzy(struct rspamd_config *cfg)
 	ctx->lua_pre_handler_cbref = -1;
 	ctx->lua_post_handler_cbref = -1;
 	ctx->lua_blacklist_cbref = -1;
-	ctx->keys = g_hash_table_new_full(fuzzy_kp_hash, fuzzy_kp_equal,
-									  NULL, fuzzy_key_dtor);
+	ctx->keys = kh_init(rspamd_fuzzy_keys_hash);
 	rspamd_mempool_add_destructor(cfg->cfg_pool,
-								  (rspamd_mempool_destruct_t) g_hash_table_unref, ctx->keys);
+								  (rspamd_mempool_destruct_t) fuzzy_hash_table_dtor, ctx->keys);
 	ctx->errors_ips = rspamd_lru_hash_new_full(1024,
 											   (GDestroyNotify) rspamd_inet_address_free, g_free,
 											   rspamd_inet_address_hash, rspamd_inet_address_equal);
@@ -2768,6 +2992,15 @@ init_fuzzy(struct rspamd_config *cfg)
 									  RSPAMD_CL_FLAG_MULTIPLE,
 									  "Encryption keypair (can be repeated for different keys)");
 
+	rspamd_rcl_register_worker_option(cfg,
+									  type,
+									  "dynamic_keys_map",
+									  rspamd_rcl_parse_struct_ucl,
+									  ctx,
+									  G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, dynamic_keys_map),
+									  0,
+									  "Dynamic encryption keypairs (can be repeated for different keys)");
+
 	rspamd_rcl_register_worker_option(cfg,
 									  type,
 									  "forbidden_ids",
@@ -3145,6 +3378,31 @@ start_fuzzy(struct rspamd_worker *worker)
 									 worker, "fuzzy ratelimit whitelist");
 	}
 
+	if (ctx->dynamic_keys_map) {
+		struct fuzzy_keymap_ucl_buf *jb, **pjb;
+
+		ctx->dynamic_keys = kh_init(rspamd_fuzzy_keys_hash);
+		/* Now try to add map with ucl data */
+		jb = g_malloc(sizeof(struct fuzzy_keymap_ucl_buf));
+		pjb = g_malloc(sizeof(struct fuzzy_keymap_ucl_buf *));
+		jb->buf = NULL;
+		jb->ctx = ctx;
+		*pjb = jb;
+		rspamd_mempool_add_destructor(ctx->cfg->cfg_pool,
+									  (rspamd_mempool_destruct_t) g_free,
+									  pjb);
+
+		if (!rspamd_map_add_from_ucl(cfg,
+									 ctx->dynamic_keys_map,
+									 "Dynamic fuzzy keys map",
+									 ucl_keymap_read_cb,
+									 ucl_keymap_fin_cb,
+									 ucl_keymap_dtor_cb,
+									 (void **) pjb, worker, RSPAMD_MAP_DEFAULT)) {
+			msg_err("cannot add map for dynamic keys");
+		}
+	}
+
 	if (!isnan(ctx->delay) && ctx->delay_whitelist_map != NULL) {
 		rspamd_config_radix_from_ucl(worker->srv->cfg, ctx->delay_whitelist_map,
 									 "Skip delay from the following ips",

src/libmime/archives.c

@@ -1,11 +1,11 @@
-/*-
- * Copyright 2016 Vsevolod Stakhov
+/*
+ * Copyright 2024 Vsevolod Stakhov
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -24,6 +24,9 @@
 #include <unicode/utf16.h>
 #include <unicode/ucnv.h>
 
+#include <archive.h>
+#include <archive_entry.h>
+
 #define msg_debug_archive(...) rspamd_conditional_debug_fast(NULL, NULL,                                                 \
 															 rspamd_archive_log_id, "archive", task->task_pool->tag.uid, \
 															 G_STRFUNC,                                                  \
@@ -1132,6 +1135,7 @@ rspamd_7zip_read_folder(struct rspamd_task *task,
 		msg_debug_archive("7zip: read codec id: %L", tmp);
 
 		if (IS_SZ_ENCRYPTED(tmp)) {
+			msg_debug_archive("7zip: encrypted codec: %L", tmp);
 			arch->flags |= RSPAMD_ARCHIVE_ENCRYPTED;
 		}
 
@@ -1643,7 +1647,8 @@ end:
 static const guchar *
 rspamd_7zip_read_next_section(struct rspamd_task *task,
 							  const guchar *p, const guchar *end,
-							  struct rspamd_archive *arch)
+							  struct rspamd_archive *arch,
+							  struct rspamd_mime_part *part)
 {
 	guchar t = *p;
 
@@ -1660,10 +1665,43 @@ rspamd_7zip_read_next_section(struct rspamd_task *task,
 		 * In fact, headers are just packed, but we assume it as
 		 * encrypted to distinguish from the normal archives
 		 */
-		msg_debug_archive("7zip: encoded header, needs to be uncompressed");
-		arch->flags |= RSPAMD_ARCHIVE_CANNOT_READ;
-		p = NULL; /* Cannot get anything useful */
-		break;
+		{
+			msg_debug_archive("7zip: encoded header, needs to be uncompressed");
+			struct archive *a = archive_read_new();
+			archive_read_support_format_7zip(a);
+			int r = archive_read_open_memory(a, part->parsed_data.begin, part->parsed_data.len);
+			if (r != ARCHIVE_OK) {
+				msg_debug_archive("7zip: cannot open memory archive: %s", archive_error_string(a));
+				archive_read_free(a);
+				return NULL;
+			}
+
+			/* Clean the existing files if any */
+			rspamd_archive_dtor(arch);
+			arch->files = g_ptr_array_new();
+
+			struct archive_entry *ae;
+
+			while (archive_read_next_header(a, &ae) == ARCHIVE_OK) {
+				const char *name = archive_entry_pathname_utf8(ae);
+				if (name) {
+					msg_debug_archive("7zip: found file %s", name);
+					struct rspamd_archive_file *f = g_malloc0(sizeof(*f));
+					f->fname = g_string_new(name);
+					g_ptr_array_add(arch->files, f);
+				}
+				archive_read_data_skip(a);
+			}
+
+			if (archive_read_has_encrypted_entries(a) > 0) {
+				msg_debug_archive("7zip: found encrypted stuff");
+				arch->flags |= RSPAMD_ARCHIVE_ENCRYPTED;
+			}
+
+			archive_read_free(a);
+			p = NULL; /* Stop internal processor, as we rely on libarchive here */
+			break;
+		}
 	case kArchiveProperties:
 		p = rspamd_7zip_read_archive_props(task, p, end, arch);
 		break;
@@ -1739,7 +1777,7 @@ rspamd_archive_process_7zip(struct rspamd_task *task,
 		return;
 	}
 
-	while ((p = rspamd_7zip_read_next_section(task, p, end, arch)) != NULL)
+	while ((p = rspamd_7zip_read_next_section(task, p, end, arch, part)) != NULL)
 		;
 
 	part->part_type = RSPAMD_MIME_PART_ARCHIVE;

src/libserver/http/http_message.c

@@ -1,11 +1,11 @@
-/*-
- * Copyright 2019 Vsevolod Stakhov
+/*
+ * Copyright 2024 Vsevolod Stakhov
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -85,7 +85,21 @@ rspamd_http_message_from_url(const gchar *url)
 	}
 	else {
 		path = url + pu.field_data[UF_PATH].off;
-		pathlen = urllen - pu.field_data[UF_PATH].off;
+		pathlen = pu.field_data[UF_PATH].len;
+
+		if (path > url && *(path - 1) == '/') {
+			path--;
+			pathlen++;
+		}
+
+
+		/* Include query if needed */
+		if ((pu.field_set & (1 << UF_QUERY)) != 0) {
+			/* Include both ? and query */
+			pathlen += pu.field_data[UF_QUERY].len + 1;
+		}
+
+		/* Do not include fragment here! */
 	}
 
 	msg = rspamd_http_new_message(HTTP_REQUEST);
@@ -722,4 +736,14 @@ bool rspamd_http_message_is_standard_port(struct rspamd_http_message *msg)
 	}
 
 	return msg->port == 80;
+}
+
+const gchar *rspamd_http_message_get_url(struct rspamd_http_message *msg, gsize *len)
+{
+	if (msg->url) {
+		*len = msg->url->len;
+		return msg->url->str;
+	}
+
+	return NULL;
 }

src/libserver/http/http_message.h

@@ -1,11 +1,11 @@
-/*-
- * Copyright 2019 Vsevolod Stakhov
+/*
+ * Copyright 2024 Vsevolod Stakhov
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -247,6 +247,8 @@ const gchar *rspamd_http_message_get_http_host(struct rspamd_http_message *msg,
  */
 bool rspamd_http_message_is_standard_port(struct rspamd_http_message *msg);
 
+const gchar *rspamd_http_message_get_url(struct rspamd_http_message *msg, gsize *len);
+
 #ifdef __cplusplus
 }
 #endif

src/libserver/logger/logger_syslog.c

@@ -1,11 +1,11 @@
-/*-
- * Copyright 2020 Vsevolod Stakhov
+/*
+ * Copyright 2024 Vsevolod Stakhov
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -43,7 +43,7 @@ rspamd_log_syslog_init(rspamd_logger_t *logger, struct rspamd_config *cfg,
 	priv = g_malloc0(sizeof(*priv));
 
 	priv->log_facility = cfg->log_facility;
-	openlog("rspamd", LOG_NDELAY | LOG_PID, priv->log_facility);
+	openlog("rspamd", LOG_CONS | LOG_NDELAY | LOG_PID, priv->log_facility);
 
 	return priv;
 }
@@ -88,11 +88,148 @@ bool rspamd_log_syslog_log(const gchar *module, const gchar *id,
 		}
 	}
 
-	syslog(syslog_level, "<%.*s>; %s; %s: %.*s",
-		   RSPAMD_LOG_ID_LEN, id != NULL ? id : "",
-		   module != NULL ? module : "",
-		   function != NULL ? function : "",
-		   (gint) mlen, message);
+	bool log_json = (rspamd_log->flags & RSPAMD_LOG_FLAG_JSON);
+
+	/* Ensure safety as %.*s is used */
+	char idbuf[RSPAMD_LOG_ID_LEN + 1];
+
+	if (id != NULL) {
+		rspamd_strlcpy(idbuf, id, RSPAMD_LOG_ID_LEN + 1);
+	}
+	else {
+		idbuf[0] = '\0';
+	}
+
+	if (log_json) {
+		long now = rspamd_get_calendar_ticks();
+		if (rspamd_memcspn(message, "\"\\\r\n\b\t\v", mlen) == mlen) {
+			/* Fast path */
+			syslog(syslog_level, "{\"ts\": %ld, "
+								 "\"pid\": %d, "
+								 "\"severity\": \"%s\", "
+								 "\"worker_type\": \"%s\", "
+								 "\"id\": \"%s\", "
+								 "\"module\": \"%s\", "
+								 "\"function\": \"%s\", "
+								 "\"message\": \"%.*s\"}",
+				   now,
+				   (int) rspamd_log->pid,
+				   rspamd_get_log_severity_string(level_flags),
+				   rspamd_log->process_type,
+				   idbuf,
+				   module != NULL ? module : "",
+				   function != NULL ? function : "",
+				   (gint) mlen, message);
+		}
+		else {
+			/* Escaped version */
+			/* We need to do JSON escaping of the quotes */
+			const char *p, *end = message + mlen;
+			long escaped_len;
+
+			for (p = message, escaped_len = 0; p < end; p++, escaped_len++) {
+				switch (*p) {
+				case '\v':
+				case '\0':
+					escaped_len += 5;
+					break;
+				case '\\':
+				case '"':
+				case '\n':
+				case '\r':
+				case '\b':
+				case '\t':
+					escaped_len++;
+					break;
+				default:
+					break;
+				}
+			}
+
+
+			char *dst = g_malloc(escaped_len + 1);
+			char *d;
+
+			for (p = message, d = dst; p < end; p++, d++) {
+				switch (*p) {
+				case '\n':
+					*d++ = '\\';
+					*d = 'n';
+					break;
+				case '\r':
+					*d++ = '\\';
+					*d = 'r';
+					break;
+				case '\b':
+					*d++ = '\\';
+					*d = 'b';
+					break;
+				case '\t':
+					*d++ = '\\';
+					*d = 't';
+					break;
+				case '\f':
+					*d++ = '\\';
+					*d = 'f';
+					break;
+				case '\0':
+					*d++ = '\\';
+					*d++ = 'u';
+					*d++ = '0';
+					*d++ = '0';
+					*d++ = '0';
+					*d = '0';
+					break;
+				case '\v':
+					*d++ = '\\';
+					*d++ = 'u';
+					*d++ = '0';
+					*d++ = '0';
+					*d++ = '0';
+					*d = 'B';
+					break;
+				case '\\':
+					*d++ = '\\';
+					*d = '\\';
+					break;
+				case '"':
+					*d++ = '\\';
+					*d = '"';
+					break;
+				default:
+					*d = *p;
+					break;
+				}
+			}
+
+			*d = '\0';
+
+			syslog(syslog_level, "{\"ts\": %ld, "
+								 "\"pid\": %d, "
+								 "\"severity\": \"%s\", "
+								 "\"worker_type\": \"%s\", "
+								 "\"id\": \"%s\", "
+								 "\"module\": \"%s\", "
+								 "\"function\": \"%s\", "
+								 "\"message\": \"%s\"}",
+				   now,
+				   (int) rspamd_log->pid,
+				   rspamd_get_log_severity_string(level_flags),
+				   rspamd_log->process_type,
+				   idbuf,
+				   module != NULL ? module : "",
+				   function != NULL ? function : "",
+				   dst);
+			g_free(dst);
+		}
+	}
+	else {
+		syslog(syslog_level, "<%s>; %s; %s: %.*s",
+			   idbuf,
+			   module != NULL ? module : "",
+			   function != NULL ? function : "",
+			   (gint) mlen, message);
+	}
 
 	return true;
 }

src/libserver/maps/map.c

@@ -1,5 +1,5 @@
 /*
- * Copyright 2023 Vsevolod Stakhov
+ * Copyright 2024 Vsevolod Stakhov
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -995,7 +995,7 @@ rspamd_map_periodic_dtor(struct map_periodic_cbdata *periodic)
 	struct rspamd_map *map;
 
 	map = periodic->map;
-	msg_debug_map("periodic dtor %p", periodic);
+	msg_debug_map("periodic dtor %p; need_modify=%d", periodic, periodic->need_modify);
 
 	if (periodic->need_modify || periodic->cbdata.errored) {
 		/* Need to notify the real data structure */
@@ -1062,6 +1062,8 @@ rspamd_map_schedule_periodic(struct rspamd_map *map, int how)
 		return;
 	}
 
+	map->seen = true;
+
 	if (map->non_trivial && map->next_check != 0) {
 		timeout = map->next_check - rspamd_get_calendar_ticks();
 		map->next_check = 0;
@@ -1107,7 +1109,7 @@ rspamd_map_schedule_periodic(struct rspamd_map *map, int how)
 		timeout = map->poll_timeout;
 
 		if (how & RSPAMD_MAP_SCHEDULE_INIT) {
-			if (map->active_http) {
+			if (map->non_trivial && map->active_http) {
 				/* Spill maps load to get better chances to hit ssl cache */
 				timeout = rspamd_time_jitter(0.0, 2.0);
 			}
@@ -2189,7 +2191,7 @@ void rspamd_map_watch(struct rspamd_config *cfg,
 
 				data = bk->data.fd;
 
-				if (map->user_data == NULL || *map->user_data == NULL) {
+				if (!map->seen || map->user_data == NULL || *map->user_data == NULL) {
 					/* Map has not been read, init it's reading if possible */
 					struct stat st;
 
@@ -2317,6 +2319,8 @@ void rspamd_map_preload(struct rspamd_config *cfg)
 				if (map->on_load_function) {
 					map->on_load_function(map, map->on_load_ud);
 				}
+
+				map->seen = true;
 			}
 			else {
 				msg_info_map("preload of %s failed", map->name);

src/libserver/maps/map_private.h

@@ -1,11 +1,11 @@
-/*-
- * Copyright 2016 Vsevolod Stakhov
+/*
+ * Copyright 2024 Vsevolod Stakhov
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -166,6 +166,7 @@ struct rspamd_map {
 	bool file_only;    /* No HTTP backends found */
 	bool static_only;  /* No need to check */
 	bool no_file_read; /* Do not read files */
+	bool seen;         /* This map has already been watched or pre-loaded */
 	/* Shared lock for temporary disabling of map reading (e.g. when this map is written by UI) */
 	gint *locked;
 	gchar tag[MEMPOOL_UID_LEN];

src/libstat/stat_api.h

@@ -1,11 +1,11 @@
-/*-
- * Copyright 2016 Vsevolod Stakhov
+/*
+ * Copyright 2024 Vsevolod Stakhov
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -53,13 +53,14 @@ typedef struct rspamd_stat_token_s {
 	guint flags;
 } rspamd_stat_token_t;
 
+#define RSPAMD_TOKEN_VALUE_TYPE float
 typedef struct token_node_s {
 	guint64 data;
 	guint window_idx;
 	guint flags;
 	rspamd_stat_token_t *t1;
 	rspamd_stat_token_t *t2;
-	float values[];
+	RSPAMD_TOKEN_VALUE_TYPE values[0];
 } rspamd_token_t;
 
 struct rspamd_stat_ctx;

src/libstat/tokenizers/osb.c

@@ -1,11 +1,11 @@
-/*-
- * Copyright 2016 Vsevolod Stakhov
+/*
+ * Copyright 2024 Vsevolod Stakhov
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -302,9 +302,8 @@ gint rspamd_tokenizer_osb(struct rspamd_stat_ctx *ctx,
 		hashpipe[i].h = 0xfe;
 		hashpipe[i].t = NULL;
 	}
-
 	token_size = sizeof(rspamd_token_t) +
-				 sizeof(gdouble) * ctx->statfiles->len;
+				 sizeof(RSPAMD_TOKEN_VALUE_TYPE) * ctx->statfiles->len;
 	g_assert(token_size > 0);
 
 	for (w = 0; w < words->len; w++) {

src/lua/lua_url.c

@@ -1,5 +1,5 @@
 /*
- * Copyright 2023 Vsevolod Stakhov
+ * Copyright 2024 Vsevolod Stakhov
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -74,6 +74,7 @@ LUA_FUNCTION_DEF(url, lt);
 LUA_FUNCTION_DEF(url, eq);
 LUA_FUNCTION_DEF(url, get_order);
 LUA_FUNCTION_DEF(url, get_part_order);
+LUA_FUNCTION_DEF(url, to_http);
 
 static const struct luaL_reg urllib_m[] = {
 	LUA_INTERFACE_DEF(url, get_length),
@@ -101,6 +102,7 @@ static const struct luaL_reg urllib_m[] = {
 	LUA_INTERFACE_DEF(url, get_flags_num),
 	LUA_INTERFACE_DEF(url, get_order),
 	LUA_INTERFACE_DEF(url, get_part_order),
+	LUA_INTERFACE_DEF(url, to_http),
 	{"get_redirected", lua_url_get_phished},
 	LUA_INTERFACE_DEF(url, set_redirected),
 	{"__tostring", lua_url_tostring},
@@ -343,6 +345,89 @@ lua_url_tostring(lua_State *L)
 	return 1;
 }
 
+/***
+ * @method url:to_http()
+ * Get URL suitable for HTTP request (e.g. by trimming fragment and user parts)
+ * @return {string} url as a string
+ */
+static gint
+lua_url_to_http(lua_State *L)
+{
+	LUA_TRACE_POINT;
+	struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+	if (url != NULL && url->url != NULL) {
+		if (url->url->protocol == PROTOCOL_MAILTO) {
+			/* Nothing to do here */
+			lua_pushnil(L);
+		}
+		else {
+
+			if (url->url->userlen > 0) {
+				/* We need to reconstruct url :( */
+				gsize len = url->url->urllen - url->url->fragmentlen + 1;
+
+				/* Strip the # character */
+				if (url->url->fragmentlen > 0 && len > 0) {
+					while (url->url->string[len - 1] == '#' && len > 0) {
+						len--;
+					}
+				}
+				gchar *nstr = g_malloc(len);
+				gchar *d = nstr, *end = nstr + len;
+				memcpy(nstr, url->url->string, url->url->protocollen);
+				d += url->url->protocollen;
+				*d++ = ':';
+				*d++ = '/';
+				*d++ = '/';
+
+				/* Host part */
+				memcpy(d, rspamd_url_host(url->url), url->url->hostlen);
+				d += url->url->hostlen;
+
+				int port = rspamd_url_get_port_if_special(url->url);
+
+				if (port > 0) {
+					d += rspamd_snprintf(d, end - d, ":%d/", port);
+				}
+				else {
+					*d++ = '/';
+				}
+
+				if (url->url->datalen > 0) {
+					memcpy(d, rspamd_url_data_unsafe(url->url), url->url->datalen);
+					d += url->url->datalen;
+				}
+
+				if (url->url->querylen > 0) {
+					*d++ = '?';
+					memcpy(d, rspamd_url_query_unsafe(url->url), url->url->querylen);
+					d += url->url->querylen;
+				}
+
+				g_assert(d < end);
+				lua_pushlstring(L, nstr, d - nstr);
+			}
+			else {
+				gsize len = url->url->urllen - url->url->fragmentlen;
+
+				/* Strip the # character */
+				if (url->url->fragmentlen > 0 && len > 0) {
+					while (url->url->string[len - 1] == '#' && len > 0) {
+						len--;
+					}
+				}
+				lua_pushlstring(L, url->url->string, len);
+			}
+		}
+	}
+	else {
+		lua_pushnil(L);
+	}
+
+	return 1;
+}
+
 /***
  * @method url:get_raw()
  * Get full content of the url as it was parsed (e.g. with urldecode)
@@ -773,38 +858,41 @@ lua_url_create(lua_State *L)
 	}
 	else {
 		pool = static_lua_url_pool;
-		t = lua_check_text_or_string(L, 2);
+		t = lua_check_text_or_string(L, 1);
 	}
 
-	if (pool == NULL || t == NULL) {
-		return luaL_error(L, "invalid arguments");
+	if (pool == NULL) {
+		return luaL_error(L, "invalid arguments: mempool is expected as the second argument");
 	}
-	else {
-		rspamd_url_find_single(pool, t->start, t->len, RSPAMD_URL_FIND_ALL,
-							   lua_url_single_inserter, L);
 
-		if (lua_type(L, -1) != LUA_TUSERDATA) {
-			/* URL is actually not found */
-			lua_pushnil(L);
+	if (t == NULL) {
+		return luaL_error(L, "invalid arguments: string/text is expected as the first argument");
+	}
 
-			return 1;
-		}
+	rspamd_url_find_single(pool, t->start, t->len, RSPAMD_URL_FIND_ALL,
+						   lua_url_single_inserter, L);
 
-		u = (struct rspamd_lua_url *) lua_touserdata(L, -1);
+	if (lua_type(L, -1) != LUA_TUSERDATA) {
+		/* URL is actually not found */
+		lua_pushnil(L);
 
-		if (lua_type(L, 3) == LUA_TTABLE) {
-			/* Add flags */
-			for (lua_pushnil(L); lua_next(L, 3); lua_pop(L, 1)) {
-				int nmask = 0;
-				const gchar *fname = lua_tostring(L, -1);
+		return 1;
+	}
 
-				if (rspamd_url_flag_from_string(fname, &nmask)) {
-					u->url->flags |= nmask;
-				}
-				else {
-					lua_pop(L, 1);
-					return luaL_error(L, "invalid flag: %s", fname);
-				}
+	u = (struct rspamd_lua_url *) lua_touserdata(L, -1);
+
+	if (lua_type(L, 3) == LUA_TTABLE) {
+		/* Add flags */
+		for (lua_pushnil(L); lua_next(L, 3); lua_pop(L, 1)) {
+			int nmask = 0;
+			const gchar *fname = lua_tostring(L, -1);
+
+			if (rspamd_url_flag_from_string(fname, &nmask)) {
+				u->url->flags |= nmask;
+			}
+			else {
+				lua_pop(L, 1);
+				return luaL_error(L, "invalid flag: %s", fname);
 			}
 		}
 	}

src/plugins/lua/history_redis.lua

@@ -149,7 +149,7 @@ local function history_save(task)
   end
 
   local data = task:get_protocol_reply { 'metrics', 'basic' }
-  local prefix = lua_util.jinja_template(settings.key_prefix, template_env)
+  local prefix = lua_util.jinja_template(settings.key_prefix, template_env, false, true)
 
   if data then
     normalise_results(data, task)
@@ -183,7 +183,7 @@ local function history_save(task)
 end
 
 local function handle_history_request(task, conn, from, to, reset)
-  local prefix = lua_util.jinja_template(settings.key_prefix, template_env)
+  local prefix = lua_util.jinja_template(settings.key_prefix, template_env, false, true)
 
   if reset then
     local function redis_ltrim_cb(err, _)
@@ -305,7 +305,7 @@ if opts then
       flags = 'empty,explicit_disable,ignore_passthrough',
       augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) }
     })
-    lua_redis.register_prefix(lua_util.jinja_template(settings.key_prefix, template_env), N,
+    lua_redis.register_prefix(lua_util.jinja_template(settings.key_prefix, template_env, false, true), N,
         "Redis history", {
           type = 'list',
         })

src/plugins/lua/metadata_exporter.lua

@@ -323,10 +323,11 @@ local pushers = {
         if type(v) == 'table' then
           hdrs[pfx .. k] = ucl.to_format(v, 'json-compact')
         else
-          hdrs[pfx .. k] = v
+          hdrs[pfx .. k] = rspamd_util.mime_header_encode(v)
         end
       end
     end
+
     rspamd_http.request({
       task = task,
       url = rule.url,
@@ -366,12 +367,12 @@ local pushers = {
       return true
     end
     rspamd_tcp.request({
-      task=task,
-      host=rule.host,
-      port=rule.port,
-      data=formatted,
-      callback=json_raw_tcp_callback,
-      read=false,
+      task = task,
+      host = rule.host,
+      port = rule.port,
+      data = formatted,
+      callback = json_raw_tcp_callback,
+      read = false,
     })
   end,
 }

src/rspamd_proxy.c

@@ -1101,6 +1101,8 @@ proxy_session_dtor(struct rspamd_proxy_session *session)
 		rspamd_mempool_delete(session->pool);
 	}
 
+	session->worker->nconns--;
+
 	g_free(session);
 }
 
@@ -2275,6 +2277,8 @@ proxy_accept_socket(EV_P_ ev_io *w, int revents)
 		return;
 	}
 
+	worker->nconns++;
+
 	session = g_malloc0(sizeof(*session));
 	REF_INIT_RETAIN(session, proxy_session_dtor);
 	session->client_sock = nfd;

test/CMakeLists.txt

@@ -1,50 +1,56 @@
-SET(TESTSRC		rspamd_mem_pool_test.c
-				rspamd_statfile_test.c
-				rspamd_url_test.c
-				rspamd_dns_test.c
-				rspamd_dkim_test.c
-				rspamd_rrd_test.c
-				rspamd_radix_test.c
-				rspamd_shingles_test.c
-				rspamd_upstream_test.c
-				rspamd_lua_pcall_vs_resume_test.c
-				rspamd_lua_test.c
-				rspamd_cryptobox_test.c
-				rspamd_heap_test.c
-				rspamd_test_suite.c)
+include(CTest)
 
-ADD_EXECUTABLE(rspamd-test EXCLUDE_FROM_ALL ${TESTSRC})
-SET_TARGET_PROPERTIES(rspamd-test PROPERTIES COMPILE_FLAGS "-DRSPAMD_TEST")
-ADD_DEPENDENCIES(rspamd-test rspamd-server)
-SET_TARGET_PROPERTIES(rspamd-test PROPERTIES LINKER_LANGUAGE CXX)
-TARGET_LINK_LIBRARIES(rspamd-test rspamd-server)
+IF(BUILD_TESTING MATCHES "ON")
+	SET(TESTSRC		rspamd_mem_pool_test.c
+					rspamd_statfile_test.c
+					rspamd_url_test.c
+					rspamd_dns_test.c
+					rspamd_dkim_test.c
+					rspamd_rrd_test.c
+					rspamd_radix_test.c
+					rspamd_shingles_test.c
+					rspamd_upstream_test.c
+					rspamd_lua_pcall_vs_resume_test.c
+					rspamd_lua_test.c
+					rspamd_cryptobox_test.c
+					rspamd_heap_test.c
+					rspamd_test_suite.c)
 
-SET(CXXTESTSSRC		rspamd_cxx_unit.cxx)
+	ADD_EXECUTABLE(rspamd-test ${TESTSRC})
+	SET_TARGET_PROPERTIES(rspamd-test PROPERTIES COMPILE_FLAGS "-DRSPAMD_TEST")
+	ADD_DEPENDENCIES(rspamd-test rspamd-server)
+	SET_TARGET_PROPERTIES(rspamd-test PROPERTIES LINKER_LANGUAGE CXX)
+	TARGET_LINK_LIBRARIES(rspamd-test rspamd-server)
+	ADD_TEST(NAME rspamd-test COMMAND rspamd-test "-p" "/rspamd/lua")
 
-ADD_EXECUTABLE(rspamd-test-cxx EXCLUDE_FROM_ALL ${CXXTESTSSRC})
-SET_TARGET_PROPERTIES(rspamd-test-cxx PROPERTIES LINKER_LANGUAGE CXX)
-ADD_DEPENDENCIES(rspamd-test-cxx rspamd-server)
-TARGET_LINK_LIBRARIES(rspamd-test-cxx PRIVATE rspamd-server)
-SET_TARGET_PROPERTIES(rspamd-test-cxx PROPERTIES LINKER_LANGUAGE CXX)
+	SET(CXXTESTSSRC		rspamd_cxx_unit.cxx)
 
-IF(NOT "${CMAKE_CURRENT_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_BINARY_DIR}")
-	# Also add dependencies for convenience
-	FILE(GLOB_RECURSE LUA_TESTS CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/lua/*.*")
-	ADD_CUSTOM_TARGET(units-dir COMMAND
-		${CMAKE_COMMAND} -E make_directory "${CMAKE_CURRENT_BINARY_DIR}/lua/unit"
-	)
-	ADD_DEPENDENCIES(rspamd-test units-dir)
-	FOREACH(_LF IN LISTS LUA_TESTS)
-		GET_FILENAME_COMPONENT(_NM "${_LF}" NAME)
-		IF("${_LF}" MATCHES "^.*/unit/.*$")
-			SET(_DS "${CMAKE_CURRENT_BINARY_DIR}/lua/unit/${_NM}")
-		ELSE()
-			SET(_DS "${CMAKE_CURRENT_BINARY_DIR}/lua/${_NM}")
-		ENDIF()
-		ADD_CUSTOM_TARGET("${_NM}" COMMAND
-			${CMAKE_COMMAND} -E copy_if_different ${_LF} ${_DS}
-			SOURCES "${_LF}"
+	ADD_EXECUTABLE(rspamd-test-cxx ${CXXTESTSSRC})
+	SET_TARGET_PROPERTIES(rspamd-test-cxx PROPERTIES LINKER_LANGUAGE CXX)
+	ADD_DEPENDENCIES(rspamd-test-cxx rspamd-server)
+	TARGET_LINK_LIBRARIES(rspamd-test-cxx PRIVATE rspamd-server)
+	SET_TARGET_PROPERTIES(rspamd-test-cxx PROPERTIES LINKER_LANGUAGE CXX)
+	ADD_TEST(NAME rspamd-test-cxx COMMAND rspamd-test-cxx)
+
+	IF(NOT "${CMAKE_CURRENT_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_BINARY_DIR}")
+		# Also add dependencies for convenience
+		FILE(GLOB_RECURSE LUA_TESTS CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/lua/*.*")
+		ADD_CUSTOM_TARGET(units-dir COMMAND
+			${CMAKE_COMMAND} -E make_directory "${CMAKE_CURRENT_BINARY_DIR}/lua/unit"
 		)
-		ADD_DEPENDENCIES(rspamd-test "${_NM}")
-	ENDFOREACH()
+		ADD_DEPENDENCIES(rspamd-test units-dir)
+		FOREACH(_LF IN LISTS LUA_TESTS)
+			GET_FILENAME_COMPONENT(_NM "${_LF}" NAME)
+			IF("${_LF}" MATCHES "^.*/unit/.*$")
+				SET(_DS "${CMAKE_CURRENT_BINARY_DIR}/lua/unit/${_NM}")
+			ELSE()
+				SET(_DS "${CMAKE_CURRENT_BINARY_DIR}/lua/${_NM}")
+			ENDIF()
+			ADD_CUSTOM_TARGET("${_NM}" COMMAND
+				${CMAKE_COMMAND} -E copy_if_different ${_LF} ${_DS}
+				SOURCES "${_LF}"
+			)
+			ADD_DEPENDENCIES(rspamd-test "${_NM}")
+		ENDFOREACH()
+	ENDIF()
 ENDIF()

test/functional/cases/001_merged/160_antivirus.robot

@@ -1,5 +1,4 @@
 *** Settings ***
-Suite Teardown  Antivirus Teardown
 Library         Process
 Library         ${RSPAMD_TESTDIR}/lib/rspamd.py
 Resource        ${RSPAMD_TESTDIR}/lib/rspamd.robot
@@ -14,19 +13,19 @@ ${SETTINGS_FPROT}   {symbols_enabled = [FPROT_VIRUS, FPROT2_VIRUS_DUPLICATE_DEFA
 
 *** Test Cases ***
 CLAMAV MISS
-  Run Dummy Clam  ${RSPAMD_PORT_CLAM}
+  ${process} =  Run Dummy Clam  ${RSPAMD_PORT_CLAM}
   Scan File  ${MESSAGE}
   ...  Settings=${SETTINGS_CLAM}
   Do Not Expect Symbol  CLAM_VIRUS
-  Shutdown clamav
+  [Teardown]  Terminate Process  ${process}
 
 CLAMAV HIT
-  Run Dummy Clam  ${RSPAMD_PORT_CLAM}  1
+  ${process} =  Run Dummy Clam  ${RSPAMD_PORT_CLAM}  1
   Scan File  ${MESSAGE2}
   ...  Settings=${SETTINGS_CLAM}
   Expect Symbol  CLAM_VIRUS
   Do Not Expect Symbol  CLAMAV_VIRUS_FAIL
-  Shutdown clamav
+  [Teardown]  Terminate Process  ${process}
 
 CLAMAV CACHE HIT
   Scan File  ${MESSAGE2}
@@ -41,16 +40,16 @@ CLAMAV CACHE MISS
   Do Not Expect Symbol  CLAMAV_VIRUS_FAIL
 
 FPROT MISS
-  Run Dummy Fprot  ${RSPAMD_PORT_FPROT}
+  ${process} =  Run Dummy Fprot  ${RSPAMD_PORT_FPROT}
   Scan File  ${MESSAGE2}
   ...  Settings=${SETTINGS_FPROT}
   Do Not Expect Symbol  FPROT_VIRUS
   Do Not Expect Symbol  FPROT_EICAR
-  Shutdown fport
+  [Teardown]  Terminate Process  ${process}
 
 FPROT HIT - PATTERN
-  Run Dummy Fprot  ${RSPAMD_PORT_FPROT}  1
-  Run Dummy Fprot  ${RSPAMD_PORT_FPROT2_DUPLICATE}  1  /tmp/dummy_fprot_dupe.pid
+  ${process1} =  Run Dummy Fprot  ${RSPAMD_PORT_FPROT}  1
+  ${process2} =  Run Dummy Fprot  ${RSPAMD_PORT_FPROT2_DUPLICATE}  1  /tmp/dummy_fprot_dupe.pid
   Scan File  ${MESSAGE}
   ...  Settings=${SETTINGS_FPROT}
   Expect Symbol  FPROT_EICAR
@@ -58,8 +57,7 @@ FPROT HIT - PATTERN
   Expect Symbol  FPROT2_VIRUS_DUPLICATE_PATTERN
   Do Not Expect Symbol  FPROT2_VIRUS_DUPLICATE_DEFAULT
   Do Not Expect Symbol  FPROT2_VIRUS_DUPLICATE_NOPE
-  Shutdown fport
-  Shutdown fport duplicate
+  [Teardown]  Double FProt Teardown  ${process1}  ${process2}
 
 FPROT CACHE HIT
   Scan File  ${MESSAGE}
@@ -76,19 +74,19 @@ FPROT CACHE MISS
   Do Not Expect Symbol  FPROT_VIRUS
 
 AVAST MISS
-  Run Dummy Avast  ${RSPAMD_PORT_AVAST}
+  ${process} =  Run Dummy Avast  ${RSPAMD_PORT_AVAST}
   Scan File  ${MESSAGE}
   ...  Settings=${SETTINGS_AVAST}
   Do Not Expect Symbol  AVAST_VIRUS
-  Shutdown avast
+  [Teardown]  Terminate Process  ${process}
 
 AVAST HIT
-  Run Dummy Avast  ${RSPAMD_PORT_AVAST}  1
+  ${process} =  Run Dummy Avast  ${RSPAMD_PORT_AVAST}  1
   Scan File  ${MESSAGE2}
   ...  Settings=${SETTINGS_AVAST}
   Expect Symbol  AVAST_VIRUS
   Do Not Expect Symbol  AVAST_VIRUS_FAIL
-  Shutdown avast
+  [Teardown]  Terminate Process  ${process}
 
 AVAST CACHE HIT
   Scan File  ${MESSAGE2}
@@ -103,26 +101,10 @@ AVAST CACHE MISS
   Do Not Expect Symbol  AVAST_VIRUS_FAIL
 
 *** Keywords ***
-Antivirus Teardown
-  Shutdown clamav
-  Shutdown fport
-  Shutdown avast
-
-Shutdown clamav
-  ${clamav_pid} =  Get File if exists  /tmp/dummy_clamav.pid
-  Run Keyword if  ${clamav_pid}  Shutdown Process With Children  ${clamav_pid}
-
-Shutdown fport
-  ${fport_pid} =  Get File if exists  /tmp/dummy_fprot.pid
-  Run Keyword if  ${fport_pid}  Shutdown Process With Children  ${fport_pid}
-
-Shutdown fport duplicate
-  ${fport_pid} =  Get File if exists  /tmp/dummy_fprot_dupe.pid
-  Run Keyword if  ${fport_pid}  Shutdown Process With Children  ${fport_pid}
-
-Shutdown avast
-  ${avast_pid} =  Get File if exists  /tmp/dummy_avast.pid
-  Run Keyword if  ${avast_pid}  Shutdown Process With Children  ${avast_pid}
+Double FProt Teardown
+  [Arguments]  ${process1}  ${process2}
+  Terminate Process  ${process1}
+  Terminate Process  ${process2}
 
 Run Dummy
   [Arguments]  @{varargs}
@@ -137,15 +119,19 @@ Run Dummy
   Log To Console  ${res.stdout}
   Log To Console  ${res.stderr}
   Fail  Dummy server failed to start
+  [Return]  ${process}
 
 Run Dummy Clam
   [Arguments]  ${port}  ${found}=  ${pid}=/tmp/dummy_clamav.pid
-  Run Dummy  ${RSPAMD_TESTDIR}/util/dummy_clam.py  ${port}  ${found}  ${pid}
+  ${process} =  Run Dummy  ${RSPAMD_TESTDIR}/util/dummy_clam.py  ${port}  ${found}  ${pid}
+  [Return]  ${process}
 
 Run Dummy Fprot
   [Arguments]  ${port}  ${found}=  ${pid}=/tmp/dummy_fprot.pid
-  Run Dummy  ${RSPAMD_TESTDIR}/util/dummy_fprot.py  ${port}  ${found}  ${pid}
+  ${process} =  Run Dummy  ${RSPAMD_TESTDIR}/util/dummy_fprot.py  ${port}  ${found}  ${pid}
+  [Return]  ${process}
 
 Run Dummy Avast
   [Arguments]  ${port}  ${found}=  ${pid}=/tmp/dummy_avast.pid
-  Run Dummy  ${RSPAMD_TESTDIR}/util/dummy_avast.py  ${port}  ${found}  ${pid}
+  ${process} =  Run Dummy  ${RSPAMD_TESTDIR}/util/dummy_avast.py  ${port}  ${found}  ${pid}
+  [Return]  ${process}

test/functional/cases/001_merged/310_udp.robot

@@ -1,6 +1,6 @@
 *** Settings ***
-Test Setup      UDP Setup
-Test Teardown   UDP Teardown
+Suite Setup      UDP Setup
+Suite Teardown   UDP Teardown
 Library         Process
 Library         ${RSPAMD_TESTDIR}/lib/rspamd.py
 Resource        ${RSPAMD_TESTDIR}/lib/rspamd.robot
@@ -31,10 +31,11 @@ UDP Setup
   Run Dummy UDP
 
 UDP Teardown
-  ${udp_pid} =  Get File  /tmp/dummy_udp.pid
-  Shutdown Process With Children  ${udp_pid}
+  Terminate Process  ${DUMMY_UDP_PROC}
+  Wait For Process  ${DUMMY_UDP_PROC}
 
 Run Dummy UDP
   [Arguments]
   ${result} =  Start Process  ${RSPAMD_TESTDIR}/util/dummy_udp.py  5005
   Wait Until Created  /tmp/dummy_udp.pid
+  Set Suite Variable  ${DUMMY_UDP_PROC}  ${result}

test/functional/cases/001_merged/__init__.robot

@@ -1,6 +1,6 @@
 *** Settings ***
 Suite Setup     Multi Setup
-Suite Teardown  Rspamd Redis Teardown
+Suite Teardown  Multi Teardown
 Library         ${RSPAMD_TESTDIR}/lib/rspamd.py
 Resource        ${RSPAMD_TESTDIR}/lib/rspamd.robot
 Variables       ${RSPAMD_TESTDIR}/lib/vars.py
@@ -25,4 +25,5 @@ Multi Teardown
   Rspamd Teardown
   Dummy Http Teardown
   Dummy Https Teardown
-  Redis Teardown
+  Redis Teardown
+  Try Reap Zombies

test/functional/cases/120_fuzzy/encrypted-dyn1.robot

@@ -0,0 +1,14 @@
+*** Settings ***
+Suite Setup     Fuzzy Setup Encrypted Dyn1 Siphash
+Suite Teardown  Rspamd Redis Teardown
+Resource        lib.robot
+
+*** Test Cases ***
+Fuzzy Add
+  Fuzzy Multimessage Add Test
+
+Fuzzy Fuzzy
+  Fuzzy Multimessage Fuzzy Test
+
+Fuzzy Miss
+  Fuzzy Multimessage Miss Test

test/functional/cases/120_fuzzy/encrypted-dyn2.robot

@@ -0,0 +1,14 @@
+*** Settings ***
+Suite Setup     Fuzzy Setup Encrypted Dyn2 Siphash
+Suite Teardown  Rspamd Redis Teardown
+Resource        lib.robot
+
+*** Test Cases ***
+Fuzzy Add
+  Fuzzy Multimessage Add Test
+
+Fuzzy Fuzzy
+  Fuzzy Multimessage Fuzzy Test
+
+Fuzzy Miss
+  Fuzzy Multimessage Miss Test

test/functional/cases/120_fuzzy/lib.robot

@@ -75,6 +75,15 @@ Fuzzy Fuzzy Test
     Expect Symbol  ${FLAG1_SYMBOL}
   END
 
+Fuzzy Encrypted Test
+  [Arguments]  ${message}
+  @{path_info} =  Path Splitter  ${message}
+  @{fuzzy_files} =  List Files In Directory  ${pathinfo}[0]  pattern=${pathinfo}[1].fuzzy*  absolute=1
+  FOR  ${i}  IN  @{fuzzy_files}
+    ${result} =  Run Rspamc  -p  -h  ${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_NORMAL}  --key  ${RSPAMD_FUZZY_ENCRYPTION_KEY}  ${i}
+    Check Rspamc  ${result}  ${FLAG1_SYMBOL}
+  END
+
 Fuzzy Miss Test
   [Arguments]  ${message}
   Scan File  ${message}
@@ -98,15 +107,34 @@ Fuzzy Setup Encrypted
   Set Suite Variable  ${RSPAMD_FUZZY_ALGORITHM}  ${algorithm}
   Set Suite Variable  ${RSPAMD_FUZZY_ENCRYPTED_ONLY}  true
   Set Suite Variable  ${RSPAMD_FUZZY_ENCRYPTION_KEY}  ${RSPAMD_KEY_PUB1}
+  Set Suite Variable  ${RSPAMD_FUZZY_CLIENT_ENCRYPTION_KEY}  ${RSPAMD_KEY_PUB1}
   Set Suite Variable  ${RSPAMD_FUZZY_INCLUDE}  ${RSPAMD_TESTDIR}/configs/fuzzy-encryption-key.conf
   Rspamd Redis Setup
 
-Fuzzy Setup Encrypted Keyed
+Fuzzy Setup Encrypted Dyn1
   [Arguments]  ${algorithm}
   Set Suite Variable  ${RSPAMD_FUZZY_ALGORITHM}  ${algorithm}
   Set Suite Variable  ${RSPAMD_FUZZY_ENCRYPTED_ONLY}  true
-  Set Suite Variable  ${RSPAMD_FUZZY_ENCRYPTION_KEY}  ${RSPAMD_KEY_PUB1} 
+  Set Suite Variable  ${RSPAMD_FUZZY_ENCRYPTION_KEY}  ${RSPAMD_KEY_PUB1}
+  Set Suite Variable  ${RSPAMD_FUZZY_CLIENT_ENCRYPTION_KEY}  ${RSPAMD_KEY_PUB2}
+  Set Suite Variable  ${RSPAMD_FUZZY_INCLUDE}  ${RSPAMD_TESTDIR}/configs/fuzzy-encryption-key.conf
+  Rspamd Redis Setup
 
+Fuzzy Setup Encrypted Dyn2
+  [Arguments]  ${algorithm}
+  Set Suite Variable  ${RSPAMD_FUZZY_ALGORITHM}  ${algorithm}
+  Set Suite Variable  ${RSPAMD_FUZZY_ENCRYPTED_ONLY}  true
+  Set Suite Variable  ${RSPAMD_FUZZY_ENCRYPTION_KEY}  ${RSPAMD_KEY_PUB1}
+  Set Suite Variable  ${RSPAMD_FUZZY_CLIENT_ENCRYPTION_KEY}  ${RSPAMD_KEY_PUB3}
+  Set Suite Variable  ${RSPAMD_FUZZY_INCLUDE}  ${RSPAMD_TESTDIR}/configs/fuzzy-encryption-key.conf
+  Rspamd Redis Setup
+
+Fuzzy Setup Encrypted Keyed
+  [Arguments]  ${algorithm}
+  Set Suite Variable  ${RSPAMD_FUZZY_ALGORITHM}  ${algorithm}
+  Set Suite Variable  ${RSPAMD_FUZZY_ENCRYPTED_ONLY}  true
+  Set Suite Variable  ${RSPAMD_FUZZY_ENCRYPTION_KEY}  ${RSPAMD_KEY_PUB1}
+  Set Suite Variable  ${RSPAMD_FUZZY_CLIENT_ENCRYPTION_KEY}  ${RSPAMD_KEY_PUB1}
   Set Suite Variable  ${RSPAMD_FUZZY_KEY}  mYN888sydwLTfE32g2hN
   Set Suite Variable  ${RSPAMD_FUZZY_SHINGLES_KEY}  hXUCgul9yYY3Zlk1QIT2
   Rspamd Redis Setup
@@ -150,6 +178,12 @@ Fuzzy Setup Keyed Xxhash
 Fuzzy Setup Encrypted Siphash
   Fuzzy Setup Encrypted  siphash
 
+Fuzzy Setup Encrypted Dyn1 Siphash
+  Fuzzy Setup Encrypted Dyn1  siphash
+
+Fuzzy Setup Encrypted Dyn2 Siphash
+  Fuzzy Setup Encrypted Dyn2  siphash
+
 Fuzzy Skip Hash Test Message
   FOR  ${i}  IN  @{MESSAGES_SKIP}
     Fuzzy Skip Add Test Base  ${i}
@@ -165,6 +199,11 @@ Fuzzy Multimessage Fuzzy Test
     Fuzzy Fuzzy Test  ${i}
   END
 
+Fuzzy Multimessage Fuzzy Encrypted Test
+  FOR  ${i}  IN  @{MESSAGES}
+    Fuzzy Encrypted Test  ${i}
+  END
+
 Fuzzy Multimessage Miss Test
   FOR  ${i}  IN  @{RANDOM_MESSAGES}
     Fuzzy Miss Test  ${i}

test/functional/cases/140_proxy.robot

@@ -30,21 +30,21 @@ Proxy Setup
   # Run slave & copy variables
   Set Suite Variable  ${CONFIG}  ${RSPAMD_TESTDIR}/configs/lua_test.conf
   Rspamd Setup
-  Set Suite Variable  ${SLAVE_PID}  ${RSPAMD_PID}
+  Set Suite Variable  ${SLAVE_PROCESS}  ${RSPAMD_PROCESS}
   Set Suite Variable  ${SLAVE_TMPDIR}  ${RSPAMD_TMPDIR}
 
   # Run proxy & copy variables
   Set Suite Variable  ${CONFIG}  ${RSPAMD_TESTDIR}/configs/proxy.conf
-  Rspamd Setup
-  Set Suite Variable  ${PROXY_PID}  ${RSPAMD_PID}
+  Rspamd Setup  check_port=${RSPAMD_PORT_PROXY}
+  Set Suite Variable  ${PROXY_PROCESS}  ${RSPAMD_PROCESS}
   Set Suite Variable  ${PROXY_TMPDIR}  ${RSPAMD_TMPDIR}
 
 Proxy Teardown
   # Restore variables & run normal teardown
-  Set Suite Variable  ${RSPAMD_PID}  ${PROXY_PID}
+  Set Suite Variable  ${RSPAMD_PROCESS}  ${PROXY_PROCESS}
   Set Suite Variable  ${RSPAMD_TMPDIR}  ${PROXY_TMPDIR}
   Rspamd Teardown
   # Do it again for slave
-  Set Suite Variable  ${RSPAMD_PID}  ${SLAVE_PID}
+  Set Suite Variable  ${RSPAMD_PROCESS}  ${SLAVE_PROCESS}
   Set Suite Variable  ${RSPAMD_TMPDIR}  ${SLAVE_TMPDIR}
   Rspamd Teardown

test/functional/cases/150_rspamadm.robot

@@ -1,18 +1,18 @@
 *** Settings ***
+Suite Setup     Rspamadm Setup
+Suite Teardown  Rspamadm Teardown
 Library         Process
 Library         ../lib/rspamd.py
 
-Suite Teardown    Terminate All Processes    kill=True
-
 *** Test Cases ***
 Config Test
-  ${result} =  Run Process  ${RSPAMADM}  configtest
+  ${result} =  Rspamadm  configtest
   Should Match Regexp  ${result.stderr}  ^$
   Should Match Regexp  ${result.stdout}  ^syntax OK$
   Should Be Equal As Integers  ${result.rc}  0
 
 Config Help
-  ${result} =  Run Process  ${RSPAMADM}  confighelp
+  ${result} =  Rspamadm  confighelp
   Should Match Regexp  ${result.stderr}  ^$
   Should Be Equal As Integers  ${result.rc}  0
 
@@ -20,26 +20,46 @@ Simple interpreter
   ${handle} =  Start Process  ${RSPAMADM}  lua  stdin=PIPE
   ${result} =  Write to stdin  ${handle}  1+1
   Should Be Equal As Strings  ${result}  2\n
+  Wait For Process  ${handle}
 
 Simple interpreter, two results
   ${handle} =  Start Process  ${RSPAMADM}  lua  stdin=PIPE
   ${result} =  Write to stdin  ${handle}  1+1, 2 * 5
   Should Be Equal  ${result}  2\n10\n
+  Wait For Process  ${handle}
 
 Process message callback
   ${handle} =  Start Process  ${RSPAMADM}  lua  stdin=PIPE
   ${result} =  Write to stdin  ${handle}  .load ${RSPAMD_TESTDIR}/lua/rspamadm/test_message_callback.lua\n.message message_callback ${RSPAMD_TESTDIR}/messages/empty_part.eml
   Should Contain  ${result}  n parts = 2
   Should Contain  ${result}  1\n2\n4\n6
+  Wait For Process  ${handle}
 
 Lua batch mode
-  ${result} =  Run Process  ${RSPAMADM}  lua  -b  ${RSPAMD_TESTDIR}/lua/rspamadm/test_batch.lua
+  ${result} =  Rspamadm  lua  -b  ${RSPAMD_TESTDIR}/lua/rspamadm/test_batch.lua
   Should Be Equal  ${result.stderr}  hello world
   Should Match Regexp  ${result.stdout}  ^$
   Should Be Equal As Integers  ${result.rc}  0
 
 Verbose mode
-  ${result} =  Run Process  ${RSPAMADM}  -v  lua  ${RSPAMD_TESTDIR}/lua/rspamadm/test_verbose.lua
+  ${result} =  Rspamadm  -v  lua  ${RSPAMD_TESTDIR}/lua/rspamadm/test_verbose.lua
   Should Match Regexp  ${result.stderr}  ^$
   Should Match Regexp  ${result.stdout}  hello world\n
   Should Be Equal As Integers  ${result.rc}  0
+
+*** Keywords ***
+Rspamadm Setup
+  ${RSPAMADM_TMPDIR} =  Make Temporary Directory
+  Set Suite Variable  ${RSPAMADM_TMPDIR}
+
+Rspamadm Teardown
+  Cleanup Temporary Directory  ${RSPAMADM_TMPDIR}
+
+Rspamadm
+  [Arguments]  @{args}
+  ${result} =  Run Process  ${RSPAMADM}
+  ...  --var\=TMPDIR\=${RSPAMADM_TMPDIR}
+  ...  --var\=DBDIR\=${RSPAMADM_TMPDIR}
+  ...  --var\=LOCAL_CONFDIR\=/nonexistent
+  ...  @{args}
+  [Return]  ${result}

test/functional/cases/151_rspamadm_async.robot

@@ -9,6 +9,8 @@ Variables       ${RSPAMD_TESTDIR}/lib/vars.py
 *** Variables ***
 ${CONFIG}          ${RSPAMD_TESTDIR}/configs/plugins.conf
 ${REDIS_SCOPE}     Test
+# For dummy http
+${RSPAMD_SCOPE}    Test
 ${RSPAMD_URL_TLD}  ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
 
 *** Test Cases ***

test/functional/configs/composites.conf

@@ -1,5 +1,6 @@
 options = {
     pidfile = "{= env.TMPDIR =}/rspamd.pid"
+    url_tld = "{= env.TESTDIR =}/../lua/unit/test_tld.dat"
 }
 logging = {
     type = "file",

test/functional/configs/fuzzy-encryption-key.conf

@@ -1,2 +1,2 @@
 # Setting this to null does not work out so it's hidden in an include
-encryption_key = {= env.FUZZY_ENCRYPTION_KEY =};
+encryption_key = {= env.FUZZY_CLIENT_ENCRYPTION_KEY =};

test/functional/configs/fuzzy.conf

@@ -60,6 +60,7 @@ worker {
 		privkey = "{= env.KEY_PVT1 =}";
 		pubkey = "{= env.KEY_PUB1 =}";
 	}
+	dynamic_keys_map = "{= env.TESTDIR =}/configs/maps/fuzzy_keymap.map";
 }
 
 fuzzy_check {

test/functional/configs/maps/fuzzy_keymap.map

@@ -0,0 +1,16 @@
+[{
+    privkey = "achyfduzs74yc1p95bk9apoknhtzn596pzeai5ybi5tftencoray";
+    id = "xb66rsu7e5i3o95sr7ifd3rxgjruktn8ptsesdxrf4biyc5ckyu6zcye54pkw3cmkhbyoebow85bsqxhryfyy4eep5gai4x1a8s3u5d";
+    pubkey = "mbggdnw3tdx7r3ruakjecpf5hcqr4cb4nmdp1fxynx3drbyujb3y";
+    type = "kex";
+    algorithm = "curve25519";
+    encoding = "base32";
+},
+{
+    privkey = "y1z16mw4n8eaefgwhgneyrntb8rxx911r4q7pgweb7t8sj1q8goy";
+    id = "id8kmo7im37bszdoorpm6cjjg8saazz71bc9ijz974wip3gaockbpymb5e91r8cwsf7kmcbbbygap9bss8r3zkhth5i7pdnyazpkppy";
+    pubkey = "zhypei8sartqrtow84dddgp5exh3gsr65kbw88wj7ppot1bwmuiy";
+    type = "kex";
+    algorithm = "curve25519";
+    encoding = "base32";
+}]

test/functional/configs/redis-server.conf

@@ -1,5 +1,5 @@
 bind ${RSPAMD_REDIS_ADDR}
-daemonize yes
+daemonize no
 loglevel debug
 logfile ${RSPAMD_TMPDIR}/redis.log
 pidfile ${RSPAMD_TMPDIR}/redis.pid

test/functional/lib/rspamd.py

@@ -1,3 +1,29 @@
+#  Copyright 2024 Vsevolod Stakhov
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
 from urllib.request import urlopen
 import glob
 import grp
@@ -17,6 +43,7 @@ from robot.api import logger
 from robot.libraries.BuiltIn import BuiltIn
 import demjson
 
+
 def Check_JSON(j):
     d = demjson.decode(j, strict=True)
     logger.debug('got json %s' % d)
@@ -24,6 +51,7 @@ def Check_JSON(j):
     assert 'error' not in d
     return d
 
+
 def check_json_log(fn):
     line_count = 0
     f = open(fn, 'r')
@@ -33,9 +61,11 @@ def check_json_log(fn):
         line_count = line_count + 1
     assert line_count > 0
 
+
 def cleanup_temporary_directory(directory):
     shutil.rmtree(directory)
 
+
 def save_run_results(directory, filenames):
     current_directory = os.getcwd()
     suite_name = BuiltIn().get_variable_value("${SUITE_NAME}")
@@ -58,24 +88,29 @@ def save_run_results(directory, filenames):
                 shutil.copy(source_file, "%s/%s" % (destination_directory, file))
                 shutil.copy(source_file, "%s/robot-save/%s.last" % (current_directory, file))
 
+
 def encode_filename(filename):
     return "".join(['%%%0X' % ord(b) for b in filename])
 
+
 def get_test_directory():
     return os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "../../")
 
+
 def get_top_dir():
     if os.environ.get('RSPAMD_TOPDIR'):
         return os.environ['RSPAMD_TOPDIR']
 
     return get_test_directory() + "/../../"
 
+
 def get_install_root():
     if os.environ.get('RSPAMD_INSTALLROOT'):
         return os.path.abspath(os.environ['RSPAMD_INSTALLROOT'])
 
     return os.path.abspath("../install/")
 
+
 def get_rspamd():
     if os.environ.get('RSPAMD'):
         return os.environ['RSPAMD']
@@ -84,6 +119,7 @@ def get_rspamd():
     dname = get_top_dir()
     return dname + "/src/rspamd"
 
+
 def get_rspamc():
     if os.environ.get('RSPAMC'):
         return os.environ['RSPAMC']
@@ -92,6 +128,7 @@ def get_rspamc():
     dname = get_top_dir()
     return dname + "/src/client/rspamc"
 
+
 def get_rspamadm():
     if os.environ.get('RSPAMADM'):
         return os.environ['RSPAMADM']
@@ -100,6 +137,7 @@ def get_rspamadm():
     dname = get_top_dir()
     return dname + "/src/rspamadm/rspamadm"
 
+
 def HTTP(method, host, port, path, data=None, headers={}):
     c = http.client.HTTPConnection("%s:%s" % (host, port))
     c.request(method, path, data, headers)
@@ -109,9 +147,11 @@ def HTTP(method, host, port, path, data=None, headers={}):
     c.close()
     return [s, t]
 
+
 def hard_link(src, dst):
     os.link(src, dst)
 
+
 def make_temporary_directory():
     """Creates and returns a unique temporary directory
 
@@ -128,27 +168,31 @@ def make_temporary_directory():
              stat.S_IXOTH)
     return dirname
 
+
 def make_temporary_file():
     return tempfile.mktemp()
 
+
 def path_splitter(path):
     dirname = os.path.dirname(path)
     basename = os.path.basename(path)
     return [dirname, basename]
 
+
 def rspamc(addr, port, filename):
     mboxgoo = b"From MAILER-DAEMON Fri May 13 19:17:40 2016\r\n"
     goo = open(filename, 'rb').read()
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     s.connect((addr, port))
     s.send(b"CHECK RSPAMC/1.0\r\nContent-length: ")
-    s.send(str(len(goo+mboxgoo)).encode('utf-8'))
+    s.send(str(len(goo + mboxgoo)).encode('utf-8'))
     s.send(b"\r\n\r\n")
     s.send(mboxgoo)
     s.send(goo)
     r = s.recv(2048)
     return r.decode('utf-8')
 
+
 def Scan_File(filename, **headers):
     addr = BuiltIn().get_variable_value("${RSPAMD_LOCAL_ADDR}")
     port = BuiltIn().get_variable_value("${RSPAMD_PORT_NORMAL}")
@@ -162,16 +206,19 @@ def Scan_File(filename, **headers):
     BuiltIn().set_test_variable("${SCAN_RESULT}", d)
     return
 
+
 def Send_SIGUSR1(pid):
     pid = int(pid)
     os.kill(pid, signal.SIGUSR1)
 
+
 def set_directory_ownership(path, username, groupname):
     if os.getuid() == 0:
-        uid=pwd.getpwnam(username).pw_uid
-        gid=grp.getgrnam(groupname).gr_gid
+        uid = pwd.getpwnam(username).pw_uid
+        gid = grp.getgrnam(groupname).gr_gid
         os.chown(path, uid, gid)
 
+
 def spamc(addr, port, filename):
     goo = open(filename, 'rb').read()
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
@@ -184,6 +231,7 @@ def spamc(addr, port, filename):
     r = s.recv(2048)
     return r.decode('utf-8')
 
+
 def TCP_Connect(addr, port):
     """Attempts to open a TCP connection to specified address:port
 
@@ -191,13 +239,22 @@ def TCP_Connect(addr, port):
     | Wait Until Keyword Succeeds | 5s | 10ms | TCP Connect | localhost | 8080 |
     """
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    s.settimeout(5) # seconds
+    s.settimeout(5)  # seconds
     s.connect((addr, port))
     s.close()
 
+
+def try_reap_zombies():
+    try:
+        os.waitpid(-1, os.WNOHANG)
+    except ChildProcessError:
+        pass
+
+
 def ping_rspamd(addr, port):
     return str(urlopen("http://%s:%s/ping" % (addr, port)).read())
 
+
 def redis_check(addr, port):
     """Attempts to open a TCP connection to specified address:port
 
@@ -205,7 +262,7 @@ def redis_check(addr, port):
     | Wait Until Keyword Succeeds | 5s | 10ms | TCP Connect | localhost | 8080 |
     """
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    s.settimeout(1.0) # seconds
+    s.settimeout(1.0)  # seconds
     s.connect((addr, port))
     if s.sendall(b"ECHO TEST\n"):
         result = s.recv(128)
@@ -213,6 +270,7 @@ def redis_check(addr, port):
     else:
         return False
 
+
 def update_dictionary(a, b):
     a.update(b)
     return a
@@ -221,6 +279,7 @@ def update_dictionary(a, b):
 TERM_TIMEOUT = 10  # wait after sending a SIGTERM signal
 KILL_WAIT = 20  # additional wait after sending a SIGKILL signal
 
+
 def shutdown_process(process):
     # send SIGTERM
     process.terminate()
@@ -229,7 +288,7 @@ def shutdown_process(process):
         process.wait(TERM_TIMEOUT)
         return
     except psutil.TimeoutExpired:
-        logger.info( "PID {} is not terminated in {} seconds, sending SIGKILL...".format(process.pid, TERM_TIMEOUT))
+        logger.info("PID {} is not terminated in {} seconds, sending SIGKILL...".format(process.pid, TERM_TIMEOUT))
         try:
             # send SIGKILL
             process.kill()
@@ -258,6 +317,7 @@ def shutdown_process_with_children(pid):
             pass
     psutil.wait_procs(children, timeout=KILL_WAIT)
 
+
 def write_to_stdin(process_handle, text):
     if not isinstance(text, bytes):
         text = bytes(text, 'utf-8')
@@ -269,12 +329,14 @@ def write_to_stdin(process_handle, text):
     out = obj.stdout.read(4096)
     return out.decode('utf-8')
 
+
 def get_file_if_exists(file_path):
     if os.path.exists(file_path):
         with open(file_path, 'r') as myfile:
             return myfile.read()
     return None
 
+
 def _merge_luacov_stats(statsfile, coverage):
     """
     Reads a coverage stats file written by luacov and merges coverage data to
@@ -331,7 +393,7 @@ def collect_lua_coverage():
     | Collect Lua Coverage |
     """
     # decided not to do optional coverage so far
-    #if not 'ENABLE_LUA_COVERAGE' in os.environ['HOME']:
+    # if not 'ENABLE_LUA_COVERAGE' in os.environ['HOME']:
     #    logger.info("ENABLE_LUA_COVERAGE is not present in env, will not collect Lua coverage")
     #    return
 

test/functional/lib/rspamd.robot

@@ -204,11 +204,12 @@ Redis SET
   Should Be Equal As Integers  ${result.rc}  0
 
 Redis Teardown
-  ${redis_pid} =  Get Variable Value  ${REDIS_PID}
-  Shutdown Process With Children  ${redis_pid}
+  Terminate Process  ${REDIS_PROCESS}
+  Wait For Process  ${REDIS_PROCESS}
   Cleanup Temporary Directory  ${REDIS_TMPDIR}
 
 Rspamd Setup
+  [Arguments]  ${check_port}=${RSPAMD_PORT_NORMAL}
   # Create and chown temporary directory
   ${RSPAMD_TMPDIR} =  Make Temporary Directory
   Set Directory Ownership  ${RSPAMD_TMPDIR}  ${RSPAMD_USER}  ${RSPAMD_GROUP}
@@ -216,7 +217,7 @@ Rspamd Setup
   # Export ${RSPAMD_TMPDIR} to appropriate scope according to ${RSPAMD_SCOPE}
   Export Scoped Variables  ${RSPAMD_SCOPE}  RSPAMD_TMPDIR=${RSPAMD_TMPDIR}
 
-  Run Rspamd
+  Run Rspamd  check_port=${check_port}
 
 Rspamd Redis Setup
   Run Redis
@@ -226,7 +227,8 @@ Rspamd Teardown
   IF  '${CONTROLLER_ERRORS}' == 'True'
     Run Keyword And Warn On Failure  Check Controller Errors
   END
-  Shutdown Process With Children  ${RSPAMD_PID}
+  Terminate Process  ${RSPAMD_PROCESS}
+  Wait For Process  ${RSPAMD_PROCESS}
   Save Run Results  ${RSPAMD_TMPDIR}  configdump.stdout configdump.stderr rspamd.stderr rspamd.stdout rspamd.conf rspamd.log redis.log clickhouse-config.xml
   Log does not contain segfault record
   Collect Lua Coverage
@@ -242,20 +244,17 @@ Run Redis
   ${config} =  Replace Variables  ${template}
   Create File  ${RSPAMD_TMPDIR}/redis-server.conf  ${config}
   Log  ${config}
-  ${result} =  Run Process  redis-server  ${RSPAMD_TMPDIR}/redis-server.conf
-  IF  ${result.rc} != 0
-    Log  ${result.stderr}
-  END
-  Should Be Equal As Integers  ${result.rc}  0
+  ${result} =  Start Process  redis-server  ${RSPAMD_TMPDIR}/redis-server.conf
   Wait Until Keyword Succeeds  5x  1 sec  Check Pidfile  ${RSPAMD_TMPDIR}/redis.pid  timeout=0.5s
   Wait Until Keyword Succeeds  5x  1 sec  Redis Check  ${RSPAMD_REDIS_ADDR}  ${RSPAMD_REDIS_PORT}
   ${REDIS_PID} =  Get File  ${RSPAMD_TMPDIR}/redis.pid
   ${REDIS_PID} =  Convert To Number  ${REDIS_PID}
-  Export Scoped Variables  ${REDIS_SCOPE}  REDIS_PID=${REDIS_PID}  REDIS_TMPDIR=${RSPAMD_TMPDIR}
+  Export Scoped Variables  ${REDIS_SCOPE}  REDIS_PID=${REDIS_PID}  REDIS_PROCESS=${result}  REDIS_TMPDIR=${RSPAMD_TMPDIR}
   ${redis_log} =  Get File  ${RSPAMD_TMPDIR}/redis.log
   Log  ${redis_log}
 
 Run Rspamd
+  [Arguments]  ${check_port}=${RSPAMD_PORT_NORMAL}
   Export Rspamd Variables To Environment
 
   # Dump templated config or errors to log
@@ -284,7 +283,7 @@ Run Rspamd
   Set Directory Ownership  ${RSPAMD_TMPDIR}  ${RSPAMD_USER}  ${RSPAMD_GROUP}
 
   # Run Rspamd
-  ${result} =  Run Process  ${RSPAMD}  -u  ${RSPAMD_USER}  -g  ${RSPAMD_GROUP}
+  ${result} =  Start Process  ${RSPAMD}  -f  -u  ${RSPAMD_USER}  -g  ${RSPAMD_GROUP}
   ...  -c  ${CONFIG}
   ...  --var\=TMPDIR\=${RSPAMD_TMPDIR}
   ...  --var\=DBDIR\=${RSPAMD_TMPDIR}
@@ -298,24 +297,11 @@ Run Rspamd
   ...  env:ASAN_OPTIONS=quarantine_size_mb=2048:malloc_context_size=20:fast_unwind_on_malloc=0:log_path=${RSPAMD_TMPDIR}/rspamd-asan
   ...  stdout=${RSPAMD_TMPDIR}/rspamd.stdout  stderr=${RSPAMD_TMPDIR}/rspamd.stderr
 
-  # Log stdout/stderr
-  ${rspamd_stdout} =  Get File  ${RSPAMD_TMPDIR}/rspamd.stdout  encoding_errors=ignore
-  ${rspamd_stderror} =  Get File  ${RSPAMD_TMPDIR}/rspamd.stderr  encoding_errors=ignore
-  Log  ${rspamd_stdout}
-  Log  ${rspamd_stderror}
-
-  # Abort if it failed
-  Should Be Equal As Integers  ${result.rc}  0
-
-  # Wait for pid file to be written
-  Wait Until Keyword Succeeds  10x  1 sec  Check Pidfile  ${RSPAMD_TMPDIR}/rspamd.pid  timeout=0.5s
+  Export Scoped Variables  ${RSPAMD_SCOPE}  RSPAMD_PROCESS=${result}
 
   # Confirm worker is reachable
-  Wait Until Keyword Succeeds  5x  1 sec  Ping Rspamd  ${RSPAMD_LOCAL_ADDR}  ${RSPAMD_PORT_NORMAL}
+  Wait Until Keyword Succeeds  15x  1 sec  Ping Rspamd  ${RSPAMD_LOCAL_ADDR}  ${check_port}
 
-  # Read PID from PIDfile and export it to appropriate scope as ${RSPAMD_PID}
-  ${RSPAMD_PID} =  Get File  ${RSPAMD_TMPDIR}/rspamd.pid
-  Export Scoped Variables  ${RSPAMD_SCOPE}  RSPAMD_PID=${RSPAMD_PID}
 
 Run Nginx
   ${template} =  Get File  ${RSPAMD_TESTDIR}/configs/nginx.conf
@@ -370,29 +356,21 @@ Sync Fuzzy Storage
   Sleep  0.1s  Try give fuzzy storage time to sync
 
 Run Dummy Http
-  ${fileExists} =  File Exists  /tmp/dummy_http.pid
-  IF  ${fileExists} is True
-    ${http_pid} =  Get File  /tmp/dummy_http.pid
-    Shutdown Process With Children  ${http_pid}
-  END
   ${result} =  Start Process  ${RSPAMD_TESTDIR}/util/dummy_http.py  -pf  /tmp/dummy_http.pid
   Wait Until Created  /tmp/dummy_http.pid  timeout=2 second
+  Export Scoped Variables  ${RSPAMD_SCOPE}  DUMMY_HTTP_PROC=${result}
 
 Run Dummy Https
-  ${fileExists} =  File Exists  /tmp/dummy_https.pid
-  IF  ${fileExists} is True
-    ${http_pid} =  Get File  /tmp/dummy_https.pid
-    Shutdown Process With Children  ${http_pid}
-  END
   ${result} =  Start Process  ${RSPAMD_TESTDIR}/util/dummy_http.py
   ...  -c  ${RSPAMD_TESTDIR}/util/server.pem  -k  ${RSPAMD_TESTDIR}/util/server.pem
   ...  -pf  /tmp/dummy_https.pid  -p  18081
   Wait Until Created  /tmp/dummy_https.pid  timeout=2 second
+  Export Scoped Variables  ${RSPAMD_SCOPE}  DUMMY_HTTPS_PROC=${result}
 
 Dummy Http Teardown
-  ${http_pid} =  Get File  /tmp/dummy_http.pid
-  Shutdown Process With Children  ${http_pid}
+  Terminate Process  ${DUMMY_HTTP_PROC}
+  Wait For Process  ${DUMMY_HTTP_PROC}
 
 Dummy Https Teardown
-  ${https_pid} =  Get File  /tmp/dummy_https.pid
-  Shutdown Process With Children  ${https_pid}
+  Terminate Process  ${DUMMY_HTTPS_PROC}
+  Wait For Process  ${DUMMY_HTTPS_PROC}

test/functional/lib/vars.py

@@ -1,3 +1,65 @@
+#  Copyright 2024 Vsevolod Stakhov
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
 import shutil
 import socket
 
@@ -6,6 +68,8 @@ HAVE_MILTERTEST = shutil.which('miltertest') and True or False
 RSPAMD_EXTERNAL_RELAY_ENABLED = False
 RSPAMD_KEY_PVT1 = 'ekd3x36tfa5gd76t6pa8hqif3ott7n1siuux68exbkk7ukscte9y'
 RSPAMD_KEY_PUB1 = 'm8kneubpcjsb8sbsoj7jy7azj9fdd3xmj63txni86a8ye9ncomny'
+RSPAMD_KEY_PUB2 = 'mbggdnw3tdx7r3ruakjecpf5hcqr4cb4nmdp1fxynx3drbyujb3y'
+RSPAMD_KEY_PUB3 = 'zhypei8sartqrtow84dddgp5exh3gsr65kbw88wj7ppot1bwmuiy'
 RSPAMD_LOCAL_ADDR = '127.0.0.1'
 RSPAMD_MAP_WATCH_INTERVAL = '1min'
 RSPAMD_PORT_CONTROLLER = 56790

test/rspamd_cxx_unit_utils.hxx

@@ -1,5 +1,5 @@
 /*
- * Copyright 2023 Vsevolod Stakhov
+ * Copyright 2024 Vsevolod Stakhov
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -25,6 +25,7 @@
 #include "libmime/mime_headers.h"
 #include "contrib/libottery/ottery.h"
 #include "libcryptobox/cryptobox.h"
+#include "libserver/http/http_message.h"
 
 #include <vector>
 #include <utility>
@@ -204,6 +205,32 @@ TEST_SUITE("rspamd_utils")
 			}
 		}
 	}
+
+	TEST_CASE("rspamd_http_message_from_url")
+	{
+		std::vector<std::pair<std::string, std::string>> cases{
+			{"http://example.com", "/"},
+			{"http://example.com/", "/"},
+			{"http://example.com/lol", "/lol"},
+			{"http://example.com/lol#keke", "/lol"},
+			{"http://example.com/lol?omg=huh&oh", "/lol?omg=huh&oh"},
+			{"http://example.com/lol?omg=huh&oh#", "/lol?omg=huh&oh"},
+			{"http://example.com/lol?omg=huh&oh#keke", "/lol?omg=huh&oh"},
+			{"http://example.com/lol?", "/lol"},
+			{"http://example.com/lol?#", "/lol"},
+		};
+
+		for (const auto &c: cases) {
+			SUBCASE(("rspamd_http_message_from_url: " + c.first).c_str())
+			{
+				auto *msg = rspamd_http_message_from_url(c.first.c_str());
+				std::size_t nlen;
+				auto *path = rspamd_http_message_get_url(msg, &nlen);
+				CHECK(std::string{path, nlen} == c.second);
+				rspamd_http_message_unref(msg);
+			}
+		}
+	}
 }
 
 #endif