[Feature] Support reply in message pack format

Rspamd currently sends reply in JSON format. However, message pack seems to
be a better choice for a compatible client. It is faster and does not need
escaping or even UTF8 validation. This is a surface for further updates.

diff --git a/src/controller.c b/src/controller.c
index c9cda1d62e9a7af32af4132a09171ec49d89ea16..0ea1b0f046cd308e49b1da947baabc94fcbbe3b6 100644 (file)
--- a/src/controller.c
+++ b/src/controller.c
@@ -1574,7 +1574,6 @@ rspamd_controller_handle_lua_history(lua_State *L,
                                }
 
                                task->http_conn = rspamd_http_connection_ref(conn_ent->conn);
-                               ;
                                task->sock = -1;
                                session->task = task;
 
@@ -1904,7 +1903,6 @@ rspamd_controller_handle_lua(struct rspamd_http_connection_entry *conn_ent,
                                                                        task);
        task->fin_arg = conn_ent;
        task->http_conn = rspamd_http_connection_ref(conn_ent->conn);
-       ;
        task->sock = -1;
        session->task = task;
 
@@ -2005,16 +2003,25 @@ rspamd_controller_scan_reply(struct rspamd_task *task)
 {
        struct rspamd_http_message *msg;
        struct rspamd_http_connection_entry *conn_ent;
+       int out_type = UCL_EMIT_JSON_COMPACT;
+       const char *ctype = "application/json";
+       const rspamd_ftok_t *accept_hdr = rspamd_task_get_request_header(task, "Accept");
+
+       if (accept_hdr && rspamd_substring_search(accept_hdr->begin, accept_hdr->len,
+                                                                                         "application/msgpack", sizeof("application/msgpack") - 1)) {
+               ctype = "application/msgpack";
+               out_type = UCL_EMIT_MSGPACK;
+       }
 
        conn_ent = task->fin_arg;
        msg = rspamd_http_new_message(HTTP_RESPONSE);
        msg->date = time(NULL);
        msg->code = 200;
-       rspamd_protocol_http_reply(msg, task, NULL);
+       rspamd_protocol_http_reply(msg, task, NULL, out_type);
        rspamd_http_connection_reset(conn_ent->conn);
        rspamd_http_router_insert_headers(conn_ent->rt, msg);
        rspamd_http_connection_write_message(conn_ent->conn, msg, NULL,
-                                                                                "application/json", conn_ent, conn_ent->rt->timeout);
+                                                                                ctype, conn_ent, conn_ent->rt->timeout);
        conn_ent->is_reply = TRUE;
 }
 

diff --git a/src/libserver/protocol.c b/src/libserver/protocol.c
index 37a41c111a60a2dd8cf0f01341717bae0c460abb..5993da61ed28b767117f10c6e38dd1dc06ba7724 100644 (file)
--- a/src/libserver/protocol.c
+++ b/src/libserver/protocol.c
@@ -1636,7 +1636,7 @@ rspamd_protocol_write_ucl(struct rspamd_task *task,
 }
 
 void rspamd_protocol_http_reply(struct rspamd_http_message *msg,
-                                                               struct rspamd_task *task, ucl_object_t **pobj)
+                                                               struct rspamd_task *task, ucl_object_t **pobj, int how)
 {
        struct rspamd_scan_result *metric_res;
        const struct rspamd_re_cache_stat *restat;
@@ -1695,7 +1695,7 @@ void rspamd_protocol_http_reply(struct rspamd_http_message *msg,
 
        if (msg->method < HTTP_SYMBOLS && !RSPAMD_TASK_IS_SPAMC(task)) {
                msg_debug_protocol("writing json reply");
-               rspamd_ucl_emit_fstring(top, UCL_EMIT_JSON_COMPACT, &reply);
+               rspamd_ucl_emit_fstring(top, how, &reply);
        }
        else {
                if (RSPAMD_TASK_IS_SPAMC(task)) {
@@ -2111,6 +2111,16 @@ void rspamd_protocol_write_reply(struct rspamd_task *task, ev_tstamp timeout)
                                                  MESSAGE_FIELD_CHECK(task, message_id));
        }
 
+       const rspamd_ftok_t *accept_hdr;
+       int out_type = UCL_EMIT_JSON_COMPACT;
+       accept_hdr = rspamd_task_get_request_header(task, "Accept");
+
+       if (accept_hdr && rspamd_substring_search(accept_hdr->begin, accept_hdr->len,
+                                                                                         "application/msgpack", sizeof("application/msgpack") - 1)) {
+               ctype = "application/msgpack";
+               out_type = UCL_EMIT_MSGPACK;
+       }
+
        /* Compatibility */
        if (task->cmd == CMD_CHECK_RSPAMC) {
                msg->method = HTTP_SYMBOLS;
@@ -2134,15 +2144,15 @@ void rspamd_protocol_write_reply(struct rspamd_task *task, ev_tstamp timeout)
                                                          ucl_object_fromstring(g_quark_to_string(task->err->domain)),
                                                          "error_domain", 0, false);
                reply = rspamd_fstring_sized_new(256);
-               rspamd_ucl_emit_fstring(top, UCL_EMIT_JSON_COMPACT, &reply);
+               rspamd_ucl_emit_fstring(top, out_type, &reply);
                ucl_object_unref(top);
 
                /* We also need to validate utf8 */
-               if (rspamd_fast_utf8_validate(reply->str, reply->len) != 0) {
+               if (out_type != UCL_EMIT_MSGPACK && rspamd_fast_utf8_validate(reply->str, reply->len) != 0) {
                        gsize valid_len;
                        gchar *validated;
 
-                       /* We copy reply several times here but it should be a rare case */
+                       /* We copy reply several times here, but it should be a rare case */
                        validated = rspamd_str_make_utf_valid(reply->str, reply->len,
                                                                                                  &valid_len, task->task_pool);
                        rspamd_http_message_set_body(msg, validated, valid_len);
@@ -2161,7 +2171,7 @@ void rspamd_protocol_write_reply(struct rspamd_task *task, ev_tstamp timeout)
                case CMD_CHECK_SPAMC:
                case CMD_SKIP:
                case CMD_CHECK_V2:
-                       rspamd_protocol_http_reply(msg, task, NULL);
+                       rspamd_protocol_http_reply(msg, task, NULL, out_type);
                        rspamd_protocol_write_log_pipe(task);
                        break;
                case CMD_PING:

diff --git a/src/libserver/protocol.h b/src/libserver/protocol.h
index 0e3c18744750bd489941e689a41b31d70db55a96..38d9cef4b35fdeac68fe746bad0d6b1cde6fbe77 100644 (file)
--- a/src/libserver/protocol.h
+++ b/src/libserver/protocol.h
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2024 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 /**
  * @file protocol.h
  * Rspamd protocol definition
@@ -70,7 +86,8 @@ gboolean rspamd_protocol_handle_request(struct rspamd_task *task,
  * @param task
  */
 void rspamd_protocol_http_reply(struct rspamd_http_message *msg,
-                                                               struct rspamd_task *task, ucl_object_t **pobj);
+                                                               struct rspamd_task *task, ucl_object_t **pobj,
+                                                               int how);
 
 /**
  * Write data to log pipes

diff --git a/src/lua/lua_common.c b/src/lua/lua_common.c
index 92e26417af660ec75cea98f36232fd89fbcefd94..cf4ca8023d9da2cc857dcbd83874e5f8df5d1e8d 100644 (file)
--- a/src/lua/lua_common.c
+++ b/src/lua/lua_common.c
@@ -1974,8 +1974,7 @@ rspamd_lua_check_udata_common(lua_State *L, gint pos, const gchar *classname,
                                                          gboolean fatal)
 {
        void *p = lua_touserdata(L, pos);
-       guint i, top = lua_gettop(L);
-       khiter_t k;
+       gint i, top = lua_gettop(L);
 
        if (p == NULL) {
                goto err;

diff --git a/src/rspamd_proxy.c b/src/rspamd_proxy.c
index 9139866b5fd06cecf4b4a89028bf346067c3a578..ad6edd333872e5c308a81f5983c54ee2831f8296 100644 (file)
--- a/src/rspamd_proxy.c
+++ b/src/rspamd_proxy.c
@@ -1713,7 +1713,15 @@ rspamd_proxy_scan_self_reply(struct rspamd_task *task)
        struct rspamd_http_message *msg;
        struct rspamd_proxy_session *session = task->fin_arg, *nsession;
        ucl_object_t *rep = NULL;
+       int out_type = UCL_EMIT_JSON_COMPACT;
        const char *ctype = "application/json";
+       const rspamd_ftok_t *accept_hdr = rspamd_task_get_request_header(task, "Accept");
+
+       if (accept_hdr && rspamd_substring_search(accept_hdr->begin, accept_hdr->len,
+                                                                                         "application/msgpack", sizeof("application/msgpack") - 1)) {
+               ctype = "application/msgpack";
+               out_type = UCL_EMIT_MSGPACK;
+       }
 
        msg = rspamd_http_new_message(HTTP_RESPONSE);
        msg->date = time(NULL);
@@ -1726,7 +1734,7 @@ rspamd_proxy_scan_self_reply(struct rspamd_task *task)
        case CMD_CHECK_SPAMC:
        case CMD_CHECK_V2:
                rspamd_task_set_finish_time(task);
-               rspamd_protocol_http_reply(msg, task, &rep);
+               rspamd_protocol_http_reply(msg, task, &rep, out_type);
                rspamd_protocol_write_log_pipe(task);
                break;
        case CMD_PING: