/*
* Copyright 2023 Vsevolod Stakhov
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "config.h"
#include "rspamd.h"
#include "message.h"
#include "utlist.h"
#include "libserver/http/http_private.h"
#include "worker_private.h"
#include "libserver/cfg_file_private.h"
#include "libmime/scan_result_private.h"
#include "lua/lua_common.h"
#include "unix-std.h"
#include "protocol_internal.h"
#include "libserver/mempool_vars_internal.h"
#include "contrib/fastutf8/fastutf8.h"
#include "task.h"
#include <math.h>
#ifdef SYS_ZSTD
#include "zstd.h"
#else
#include "contrib/zstd/zstd.h"
#endif
INIT_LOG_MODULE(protocol)
#define msg_err_protocol(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
"protocol", task->task_pool->tag.uid, \
G_STRFUNC, \
__VA_ARGS__)
#define msg_warn_protocol(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
"protocol", task->task_pool->tag.uid, \
G_STRFUNC, \
__VA_ARGS__)
#define msg_info_protocol(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
"protocol", task->task_pool->tag.uid, \
G_STRFUNC, \
__VA_ARGS__)
#define msg_debug_protocol(...) rspamd_conditional_debug_fast(NULL, NULL, \
rspamd_protocol_log_id, "protocol", task->task_pool->tag.uid, \
G_STRFUNC, \
__VA_ARGS__)
static GQuark
rspamd_protocol_quark(void)
{
return g_quark_from_static_string("protocol-error");
}
/*
* Remove <> from the fixed string and copy it to the pool
*/
static gchar *
rspamd_protocol_escape_braces(struct rspamd_task *task, rspamd_ftok_t *in)
{
guint nchars = 0;
const gchar *p;
rspamd_ftok_t tok;
gboolean has_obrace = FALSE;
g_assert(in != NULL);
g_assert(in->len > 0);
p = in->begin;
while ((g_ascii_isspace(*p) || *p == '<') && nchars < in->len) {
if (*p == '<') {
has_obrace = TRUE;
}
p++;
nchars++;
}
tok.begin = p;
p = in->begin + in->len - 1;
tok.len = in->len - nchars;
while (g_ascii_isspace(*p) && tok.len > 0) {
p--;
tok.len--;
}
if (has_obrace && *p == '>') {
tok.len--;
}
return rspamd_mempool_ftokdup(task->task_pool, &tok);
}
#define COMPARE_CMD(str, cmd, len) (sizeof(cmd) - 1 == (len) && rspamd_lc_cmp((str), (cmd), (len)) == 0)
static gboolean
rspamd_protocol_handle_url(struct rspamd_task *task,
struct rspamd_http_message *msg)
{
GHashTable *query_args;
GHashTableIter it;
struct http_parser_url u;
const gchar *p;
gsize pathlen;
rspamd_ftok_t *key, *value;
gpointer k, v;
if (msg->url == NULL || msg->url->len == 0) {
g_set_error(&task->err, rspamd_protocol_quark(), 400, "missing command");
return FALSE;
}
if (http_parser_parse_url(msg->url->str, msg->url->len, 0, &u) != 0) {
g_set_error(&task->err, rspamd_protocol_quark(), 400, "bad request URL");
return FALSE;
}
if (!(u.field_set & (1 << UF_PATH))) {
g_set_error(&task->err, rspamd_protocol_quark(), 400,
"bad request URL: missing path");
return FALSE;
}
p = msg->url->str + u.field_data[UF_PATH].off;
pathlen = u.field_data[UF_PATH].len;
if (*p == '/') {
p++;
pathlen--;
}
switch (*p) {
case 'c':
case 'C':
/* check */
if (COMPARE_CMD(p, MSG_CMD_CHECK_V2, pathlen)) {
task->cmd = CMD_CHECK_V2;
msg_debug_protocol("got checkv2 command");
}
else if (COMPARE_CMD(p, MSG_CMD_CHECK, pathlen)) {
task->cmd = CMD_CHECK;
msg_debug_protocol("got check command");
}
else {
goto err;
}
break;
case 's':
case 'S':
/* symbols, skip */
if (COMPARE_CMD(p, MSG_CMD_SYMBOLS, pathlen)) {
task->cmd = CMD_CHECK;
msg_debug_protocol("got symbols -> old check command");
}
else if (COMPARE_CMD(p, MSG_CMD_SCAN, pathlen)) {
task->cmd = CMD_CHECK;
msg_debug_protocol("got scan -> old check command");
}
else if (COMPARE_CMD(p, MSG_CMD_SKIP, pathlen)) {
msg_debug_protocol("got skip command");
task->cmd = CMD_SKIP;
}
else {
goto err;
}
break;
case 'p':
case 'P':
/* ping, process */
if (COMPARE_CMD(p, MSG_CMD_PING, pathlen)) {
msg_debug_protocol("got ping command");
task->cmd = CMD_PING;
task->flags |= RSPAMD_TASK_FLAG_SKIP;
task->processed_stages |= RSPAMD_TASK_STAGE_DONE; /* Skip all */
}
else if (COMPARE_CMD(p, MSG_CMD_PROCESS, pathlen)) {
msg_debug_protocol("got process -> old check command");
task->cmd = CMD_CHECK;
}
else {
goto err;
}
break;
case 'r':
case 'R':
/* report, report_ifspam */
if (COMPARE_CMD(p, MSG_CMD_REPORT, pathlen)) {
msg_debug_protocol("got report -> old check command");
task->cmd = CMD_CHECK;
}
else if (COMPARE_CMD(p, MSG_CMD_REPORT_IFSPAM, pathlen)) {
msg_debug_protocol("got reportifspam -> old check command");
task->cmd = CMD_CHECK;
}
else {
goto err;
}
break;
default:
goto err;
}
if (u.field_set & (1u << UF_QUERY)) {
/* In case if we have a query, we need to store it somewhere */
query_args = rspamd_http_message_parse_query(msg);
/* Insert the rest of query params as HTTP headers */
g_hash_table_iter_init(&it, query_args);
while (g_hash_table_iter_next(&it, &k, &v)) {
gchar *key_cpy;
key = k;
value = v;
key_cpy = rspamd_mempool_ftokdup(task->task_pool, key);
rspamd_http_message_add_header_len(msg, key_cpy,
value->begin, value->len);
msg_debug_protocol("added header \"%T\" -> \"%T\" from HTTP query",
key, value);
}
g_hash_table_unref(query_args);
}
return TRUE;
err:
g_set_error(&task->err, rspamd_protocol_quark(), 400, "invalid command");
return FALSE;
}
static void
rspamd_protocol_process_recipients(struct rspamd_task *task,
const rspamd_ftok_t *hdr)
{
enum {
skip_spaces,
quoted_string,
normal_string,
} state = skip_spaces;
const gchar *p, *end, *start_addr;
struct rspamd_email_address *addr;
p = hdr->begin;
end = hdr->begin + hdr->len;
start_addr = NULL;
while (p < end) {
switch (state) {
case skip_spaces:
if (g_ascii_isspace(*p)) {
p++;
}
else if (*p == '"') {
start_addr = p;
p++;
state = quoted_string;
}
else {
state = normal_string;
start_addr = p;
}
break;
case quoted_string:
if (*p == '"') {
state = normal_string;
p++;
}
else if (*p == '\\') {
/* Quoted pair */
p += 2;
}
else {
p++;
}
break;
case normal_string:
if (*p == '"') {
state = quoted_string;
p++;
}
else if (*p == ',' && start_addr != NULL && p > start_addr) {
/* We have finished address, check what we have */
addr = rspamd_email_address_from_smtp(start_addr,
p - start_addr);
if (addr) {
if (task->rcpt_envelope == NULL) {
task->rcpt_envelope = g_ptr_array_sized_new(
2);
}
g_ptr_array_add(task->rcpt_envelope, addr);
}
else {
msg_err_protocol("bad rcpt address: '%*s'",
(int) (p - start_addr), start_addr);
task->flags |= RSPAMD_TASK_FLAG_BROKEN_HEADERS;
}
start_addr = NULL;
p++;
state = skip_spaces;
}
else {
p++;
}
break;
}
}
/* Check remainder */
if (start_addr && p > start_addr) {
switch (state) {
case normal_string:
addr = rspamd_email_address_from_smtp(start_addr, end - start_addr);
if (addr) {
if (task->rcpt_envelope == NULL) {
task->rcpt_envelope = g_ptr_array_sized_new(
2);
}
g_ptr_array_add(task->rcpt_envelope, addr);
}
else {
msg_err_protocol("bad rcpt address: '%*s'",
(int) (end - start_addr), start_addr);
task->flags |= RSPAMD_TASK_FLAG_BROKEN_HEADERS;
}
break;
case skip_spaces:
/* Do nothing */
break;
case quoted_string:
default:
msg_err_protocol("bad state when parsing rcpt address: '%*s'",
(int) (end - start_addr), start_addr);
task->flags |= RSPAMD_TASK_FLAG_BROKEN_HEADERS;
}
}
}
#define COMPARE_FLAG_LIT(lit) (len == sizeof(lit) - 1 && memcmp((lit), str, len) == 0)
#define CHECK_PROTOCOL_FLAG(lit, fl) \
do { \
if (!known && COMPARE_FLAG_LIT(lit)) { \
task->protocol_flags |= (fl); \
known = TRUE; \
msg_debug_protocol("add protocol flag %s", lit); \
} \
} while (0)
#define CHECK_TASK_FLAG(lit, fl) \
do { \
if (!known && COMPARE_FLAG_LIT(lit)) { \
task->flags |= (fl); \
known = TRUE; \
msg_debug_protocol("add task flag %s", lit); \
} \
} while (0)
static void
rspamd_protocol_handle_flag(struct rspamd_task *task, const gchar *str,
gsize len)
{
gboolean known = FALSE;
CHECK_TASK_FLAG("pass_all", RSPAMD_TASK_FLAG_PASS_ALL);
CHECK_TASK_FLAG("no_log", RSPAMD_TASK_FLAG_NO_LOG);
CHECK_TASK_FLAG("skip", RSPAMD_TASK_FLAG_SKIP);
CHECK_TASK_FLAG("skip_process", RSPAMD_TASK_FLAG_SKIP_PROCESS);
CHECK_TASK_FLAG("no_stat", RSPAMD_TASK_FLAG_NO_STAT);
CHECK_TASK_FLAG("ssl", RSPAMD_TASK_FLAG_SSL);
CHECK_TASK_FLAG("profile", RSPAMD_TASK_FLAG_PROFILE);
CHECK_PROTOCOL_FLAG("milter", RSPAMD_TASK_PROTOCOL_FLAG_MILTER);
CHECK_PROTOCOL_FLAG("zstd", RSPAMD_TASK_PROTOCOL_FLAG_COMPRESSED);
CHECK_PROTOCOL_FLAG("ext_urls", RSPAMD_TASK_PROTOCOL_FLAG_EXT_URLS);
CHECK_PROTOCOL_FLAG("body_block", RSPAMD_TASK_PROTOCOL_FLAG_BODY_BLOCK);
CHECK_PROTOCOL_FLAG("groups", RSPAMD_TASK_PROTOCOL_FLAG_GROUPS);
if (!known) {
msg_warn_protocol("unknown flag: %*s", (gint) len, str);
}
}
#undef COMPARE_FLAG
#undef CHECK_PROTOCOL_FLAG
static void
rspamd_protocol_process_flags(struct rspamd_task *task, const rspamd_ftok_t *hdr)
{
enum {
skip_spaces,
read_flag,
} state = skip_spaces;
const gchar *p, *end, *start;
p = hdr->begin;
end = hdr->begin + hdr->len;
start = NULL;
while (p < end) {
switch (state) {
case skip_spaces:
if (g_ascii_isspace(*p)) {
p++;
}
else {
state = read_flag;
start = p;
}
break;
case read_flag:
if (*p == ',') {
if (p > start) {
rspamd_protocol_handle_flag(task, start, p - start);
}
start = NULL;
state = skip_spaces;
p++;
}
else {
p++;
}
break;
}
}
/* Check remainder */
if (start && end > start && state == read_flag) {
rspamd_protocol_handle_flag(task, start, end - start);
}
}
#define IF_HEADER(name) \
srch.begin = (name); \
srch.len = sizeof(name) - 1; \
if (rspamd_ftok_casecmp(hn_tok, &srch) == 0)
gboolean
rspamd_protocol_handle_headers(struct rspamd_task *task,
struct rspamd_http_message *msg)
{
rspamd_ftok_t *hn_tok, *hv_tok, srch;
gboolean has_ip = FALSE, seen_settings_header = FALSE;
struct rspamd_http_header *header, *h;
gchar *ntok;
kh_foreach_value (msg->headers, header, {
DL_FOREACH (header, h) {
ntok = rspamd_mempool_ftokdup (task->task_pool, &h->name);
hn_tok = rspamd_mempool_alloc (task->task_pool, sizeof (*hn_tok));
hn_tok->begin = ntok;
hn_tok->len = h->name.len;
ntok = rspamd_mempool_ftokdup (task->task_pool, &h->value);
hv_tok = rspamd_mempool_alloc (task->task_pool, sizeof (*hv_tok));
hv_tok->begin = ntok;
hv_tok->len = h->value.len;
switch (*hn_tok->begin) {
case 'd':
case 'D':
IF_HEADER(DELIVER_TO_HEADER)
{
task->deliver_to = rspamd_protocol_escape_braces(task, hv_tok);
msg_debug_protocol("read deliver-to header, value: %s",
task->deliver_to);
}
else
{
msg_debug_protocol("wrong header: %T", hn_tok);
}
break;
case 'h':
case 'H':
IF_HEADER(HELO_HEADER)
{
task->helo = rspamd_mempool_ftokdup(task->task_pool, hv_tok);
msg_debug_protocol("read helo header, value: %s", task->helo);
}
IF_HEADER(HOSTNAME_HEADER)
{
task->hostname = rspamd_mempool_ftokdup(task->task_pool,
hv_tok);
msg_debug_protocol("read hostname header, value: %s", task->hostname);
}
break;
case 'f':
case 'F':
IF_HEADER(FROM_HEADER)
{
if (hv_tok->len == 0) {
/* Replace '' with '<>' to fix parsing issue */
RSPAMD_FTOK_ASSIGN(hv_tok, "<>");
}
task->from_envelope = rspamd_email_address_from_smtp(
hv_tok->begin,
hv_tok->len);
msg_debug_protocol("read from header, value: %T", hv_tok);
if (!task->from_envelope) {
msg_err_protocol("bad from header: '%T'", hv_tok);
task->flags |= RSPAMD_TASK_FLAG_BROKEN_HEADERS;
}
}
IF_HEADER(FILENAME_HEADER)
{
task->msg.fpath = rspamd_mempool_ftokdup(task->task_pool,
hv_tok);
msg_debug_protocol("read filename header, value: %s", task->msg.fpath);
}
IF_HEADER(FLAGS_HEADER)
{
msg_debug_protocol("read flags header, value: %T", hv_tok);
rspamd_protocol_process_flags(task, hv_tok);
}
break;
case 'q':
case 'Q':
IF_HEADER(QUEUE_ID_HEADER)
{
task->queue_id = rspamd_mempool_ftokdup(task->task_pool,
hv_tok);
msg_debug_protocol("read queue_id header, value: %s", task->queue_id);
}
else
{
msg_debug_protocol("wrong header: %T", hn_tok);
}
break;
case 'r':
case 'R':
IF_HEADER(RCPT_HEADER)
{
rspamd_protocol_process_recipients(task, hv_tok);
msg_debug_protocol("read rcpt header, value: %T", hv_tok);
}
IF_HEADER(RAW_DATA_HEADER)
{
srch.begin = "yes";
srch.len = 3;
msg_debug_protocol("read raw data header, value: %T", hv_tok);
if (rspamd_ftok_casecmp(hv_tok, &srch) == 0) {
task->flags &= ~RSPAMD_TASK_FLAG_MIME;
msg_debug_protocol("disable mime parsing");
}
}
break;
case 'i':
case 'I':
IF_HEADER(IP_ADDR_HEADER)
{
if (!rspamd_parse_inet_address(&task->from_addr,
hv_tok->begin, hv_tok->len,
RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
msg_err_protocol("bad ip header: '%T'", hv_tok);
}
else {
msg_debug_protocol("read IP header, value: %T", hv_tok);
has_ip = TRUE;
}
}
else
{
msg_debug_protocol("wrong header: %T", hn_tok);
}
break;
case 'p':
case 'P':
IF_HEADER(PASS_HEADER)
{
srch.begin = "all";
srch.len = 3;
msg_debug_protocol("read pass header, value: %T", hv_tok);
if (rspamd_ftok_casecmp(hv_tok, &srch) == 0) {
task->flags |= RSPAMD_TASK_FLAG_PASS_ALL;
msg_debug_protocol("pass all filters");
}
}
IF_HEADER(PROFILE_HEADER)
{
msg_debug_protocol("read profile header, value: %T", hv_tok);
task->flags |= RSPAMD_TASK_FLAG_PROFILE;
}
break;
case 's':
case 'S':
IF_HEADER(SETTINGS_ID_HEADER)
{
msg_debug_protocol("read settings-id header, value: %T", hv_tok);
task->settings_elt = rspamd_config_find_settings_name_ref(
task->cfg, hv_tok->begin, hv_tok->len);
if (task->settings_elt == NULL) {
GString *known_ids = g_string_new(NULL);
struct rspamd_config_settings_elt *cur;
DL_FOREACH(task->cfg->setting_ids, cur)
{
rspamd_printf_gstring(known_ids, "%s(%ud);",
cur->name, cur->id);
}
msg_warn_protocol("unknown settings id: %T(%d); known_ids: %v",
hv_tok,
rspamd_config_name_to_id(hv_tok->begin, hv_tok->len),
known_ids);
g_string_free(known_ids, TRUE);
}
else {
msg_debug_protocol("applied settings id %T -> %ud", hv_tok,
task->settings_elt->id);
}
}
IF_HEADER(SETTINGS_HEADER)
{
msg_debug_protocol("read settings header, value: %T", hv_tok);
seen_settings_header = TRUE;
}
break;
case 'u':
case 'U':
IF_HEADER(USER_HEADER)
{
/*
* We must ignore User header in case of spamc, as SA has
* different meaning of this header
*/
msg_debug_protocol("read user header, value: %T", hv_tok);
if (!RSPAMD_TASK_IS_SPAMC(task)) {
task->auth_user = rspamd_mempool_ftokdup(task->task_pool,
hv_tok);
}
else {
msg_info_protocol("ignore user header: legacy SA protocol");
}
}
IF_HEADER(URLS_HEADER)
{
msg_debug_protocol("read urls header, value: %T", hv_tok);
srch.begin = "extended";
srch.len = 8;
if (rspamd_ftok_casecmp(hv_tok, &srch) == 0) {
task->protocol_flags |= RSPAMD_TASK_PROTOCOL_FLAG_EXT_URLS;
msg_debug_protocol("extended urls information");
}
/* TODO: add more formats there */
}
IF_HEADER(USER_AGENT_HEADER)
{
msg_debug_protocol("read user-agent header, value: %T", hv_tok);
if (hv_tok->len == 6 &&
rspamd_lc_cmp(hv_tok->begin, "rspamc", 6) == 0) {
task->protocol_flags |= RSPAMD_TASK_PROTOCOL_FLAG_LOCAL_CLIENT;
}
}
break;
case 'l':
case 'L':
IF_HEADER(NO_LOG_HEADER)
{
msg_debug_protocol("read log header, value: %T", hv_tok);
srch.begin = "no";
srch.len = 2;
if (rspamd_ftok_casecmp(hv_tok, &srch) == 0) {
task->flags |= RSPAMD_TASK_FLAG_NO_LOG;
}
}
break;
case 'm':
case 'M':
IF_HEADER(MLEN_HEADER)
{
msg_debug_protocol("read message length header, value: %T",
hv_tok);
task->protocol_flags |= RSPAMD_TASK_PROTOCOL_FLAG_HAS_CONTROL;
}
IF_HEADER(MTA_TAG_HEADER)
{
gchar *mta_tag;
mta_tag = rspamd_mempool_ftokdup(task->task_pool, hv_tok);
rspamd_mempool_set_variable(task->task_pool,
RSPAMD_MEMPOOL_MTA_TAG,
mta_tag, NULL);
msg_debug_protocol("read MTA-Tag header, value: %s", mta_tag);
}
IF_HEADER(MTA_NAME_HEADER)
{
gchar *mta_name;
mta_name = rspamd_mempool_ftokdup(task->task_pool, hv_tok);
rspamd_mempool_set_variable(task->task_pool,
RSPAMD_MEMPOOL_MTA_NAME,
mta_name, NULL);
msg_debug_protocol("read MTA-Name header, value: %s", mta_name);
}
IF_HEADER(MILTER_HEADER)
{
task->protocol_flags |= RSPAMD_TASK_PROTOCOL_FLAG_MILTER;
msg_debug_protocol("read Milter header, value: %T", hv_tok);
}
break;
case 't':
case 'T':
IF_HEADER(TLS_CIPHER_HEADER)
{
task->flags |= RSPAMD_TASK_FLAG_SSL;
msg_debug_protocol("read TLS cipher header, value: %T", hv_tok);
}
break;
default:
msg_debug_protocol("generic header: %T", hn_tok);
break;
}
rspamd_task_add_request_header (task, hn_tok, hv_tok);
}
}); /* End of kh_foreach_value */
if (seen_settings_header && task->settings_elt) {
msg_warn_task("ignore settings id %s as settings header is also presented",
task->settings_elt->name);
REF_RELEASE(task->settings_elt);
task->settings_elt = NULL;
}
if (!has_ip) {
task->flags |= RSPAMD_TASK_FLAG_NO_IP;
}
return TRUE;
}
#define BOOL_TO_FLAG(val, flags, flag) \
do { \
if ((val)) (flags) |= (flag); \
else \
(flags) &= ~(flag); \
} while (0)
gboolean
rspamd_protocol_parse_task_flags(rspamd_mempool_t *pool,
const ucl_object_t *obj,
gpointer ud,
struct rspamd_rcl_section *section,
GError **err)
{
struct rspamd_rcl_struct_parser *pd = ud;
gint *target;
const gchar *key;
gboolean value;
target = (gint *) (((gchar *) pd->user_struct) + pd->offset);
key = ucl_object_key(obj);
value = ucl_object_toboolean(obj);
if (key != NULL) {
if (g_ascii_strcasecmp(key, "pass_all") == 0) {
BOOL_TO_FLAG(value, *target, RSPAMD_TASK_FLAG_PASS_ALL);
}
else if (g_ascii_strcasecmp(key, "no_log") == 0) {
BOOL_TO_FLAG(value, *target, RSPAMD_TASK_FLAG_NO_LOG);
}
}
return TRUE;
}
static struct rspamd_rcl_sections_map *control_parser = NULL;
RSPAMD_CONSTRUCTOR(rspamd_protocol_control_parser_ctor)
{
struct rspamd_rcl_section *sub = rspamd_rcl_add_section(&control_parser, NULL,
"*",
NULL,
NULL,
UCL_OBJECT,
FALSE,
TRUE);
/* Default handlers */
rspamd_rcl_add_default_handler(sub,
"ip",
rspamd_rcl_parse_struct_addr,
G_STRUCT_OFFSET(struct rspamd_task, from_addr),
0,
NULL);
rspamd_rcl_add_default_handler(sub,
"from",
rspamd_rcl_parse_struct_mime_addr,
G_STRUCT_OFFSET(struct rspamd_task, from_envelope),
0,
NULL);
rspamd_rcl_add_default_handler(sub,
"rcpt",
rspamd_rcl_parse_struct_mime_addr,
G_STRUCT_OFFSET(struct rspamd_task, rcpt_envelope),
0,
NULL);
rspamd_rcl_add_default_handler(sub,
"helo",
rspamd_rcl_parse_struct_string,
G_STRUCT_OFFSET(struct rspamd_task, helo),
0,
NULL);
rspamd_rcl_add_default_handler(sub,
"user",
rspamd_rcl_parse_struct_string,
G_STRUCT_OFFSET(struct rspamd_task, auth_user),
0,
NULL);
rspamd_rcl_add_default_handler(sub,
"pass_all",
rspamd_protocol_parse_task_flags,
G_STRUCT_OFFSET(struct rspamd_task, flags),
0,
NULL);
rspamd_rcl_add_default_handler(sub,
"json",
rspamd_protocol_parse_task_flags,
G_STRUCT_OFFSET(struct rspamd_task, flags),
0,
NULL);
}
RSPAMD_DESTRUCTOR(rspamd_protocol_control_parser_dtor)
{
rspamd_rcl_sections_free(control_parser);
}
gboolean
rspamd_protocol_handle_control(struct rspamd_task *task,
const ucl_object_t *control)
{
GError *err = NULL;
if (!rspamd_rcl_parse(control_parser, task->cfg, task, task->task_pool,
control, &err)) {
msg_warn_protocol("cannot parse control block: %e", err);
g_error_free(err);
return FALSE;
}
return TRUE;
}
gboolean
rspamd_protocol_handle_request(struct rspamd_task *task,
struct rspamd_http_message *msg)
{
gboolean ret = TRUE;
if (msg->method == HTTP_SYMBOLS) {
msg_debug_protocol("got legacy SYMBOLS method, enable rspamc protocol workaround");
task->cmd = CMD_CHECK_RSPAMC;
}
else if (msg->method == HTTP_CHECK) {
msg_debug_protocol("got legacy CHECK method, enable rspamc protocol workaround");
task->cmd = CMD_CHECK_RSPAMC;
}
else {
ret = rspamd_protocol_handle_url(task, msg);
}
if (msg->flags & RSPAMD_HTTP_FLAG_SPAMC) {
msg_debug_protocol("got legacy SA input, enable spamc protocol workaround");
task->cmd = CMD_CHECK_SPAMC;
}
return ret;
}
/* Structure for writing tree data */
struct tree_cb_data {
ucl_object_t *top;
khash_t(rspamd_url_host_hash) * seen;
struct rspamd_task *task;
};
static ucl_object_t *
rspamd_protocol_extended_url(struct rspamd_task *task,
struct rspamd_url *url,
const gchar *encoded, gsize enclen)
{
ucl_object_t *obj, *elt;
obj = ucl_object_typed_new(UCL_OBJECT);
elt = ucl_object_fromstring_common(encoded, enclen, 0);
ucl_object_insert_key(obj, elt, "url", 0, false);
if (url->tldlen > 0) {
elt = ucl_object_fromstring_common(rspamd_url_tld_unsafe(url),
url->tldlen, 0);
ucl_object_insert_key(obj, elt, "tld", 0, false);
}
if (url->hostlen > 0) {
elt = ucl_object_fromstring_common(rspamd_url_host_unsafe(url),
url->hostlen, 0);
ucl_object_insert_key(obj, elt, "host", 0, false);
}
ucl_object_t *flags = ucl_object_typed_new(UCL_ARRAY);
for (unsigned int i = 0; i < RSPAMD_URL_MAX_FLAG_SHIFT; i++) {
if (url->flags & (1u << i)) {
ucl_object_t *fl = ucl_object_fromstring(rspamd_url_flag_to_string(1u << i));
ucl_array_append(flags, fl);
}
}
ucl_object_insert_key(obj, flags, "flags", 0, false);
if (url->ext && url->ext->linked_url) {
encoded = rspamd_url_encode(url->ext->linked_url, &enclen, task->task_pool);
elt = rspamd_protocol_extended_url(task, url->ext->linked_url, encoded,
enclen);
ucl_object_insert_key(obj, elt, "linked_url", 0, false);
}
return obj;
}
/*
* Callback for writing urls
*/
static void
urls_protocol_cb(struct rspamd_url *url, struct tree_cb_data *cb)
{
ucl_object_t *obj;
struct rspamd_task *task = cb->task;
const gchar *user_field = "unknown", *encoded = NULL;
gboolean has_user = FALSE;
guint len = 0;
gsize enclen = 0;
if (!(task->protocol_flags & RSPAMD_TASK_PROTOCOL_FLAG_EXT_URLS)) {
if (url->hostlen > 0) {
if (rspamd_url_host_set_has(cb->seen, url)) {
return;
}
goffset err_offset;
if ((err_offset = rspamd_fast_utf8_validate(rspamd_url_host_unsafe(url),
url->hostlen)) == 0) {
obj = ucl_object_fromstring_common(rspamd_url_host_unsafe(url),
url->hostlen, 0);
}
else {
obj = ucl_object_fromstring_common(rspamd_url_host_unsafe(url),
err_offset - 1, 0);
}
}
else {
return;
}
rspamd_url_host_set_add(cb->seen, url);
}
else {
encoded = rspamd_url_encode(url, &enclen, task->task_pool);
obj = rspamd_protocol_extended_url(task, url, encoded, enclen);
}
ucl_array_append(cb->top, obj);
if (cb->task->cfg->log_urls) {
if (task->auth_user) {
user_field = task->auth_user;
len = strlen(task->auth_user);
has_user = TRUE;
}
else if (task->from_envelope) {
user_field = task->from_envelope->addr;
len = task->from_envelope->addr_len;
}
if (!encoded) {
encoded = rspamd_url_encode(url, &enclen, task->task_pool);
}
msg_notice_task_encrypted("<%s> %s: %*s; ip: %s; URL: %*s",
MESSAGE_FIELD_CHECK(task, message_id),
has_user ? "user" : "from",
len, user_field,
rspamd_inet_address_to_string(task->from_addr),
(gint) enclen, encoded);
}
}
static ucl_object_t *
rspamd_urls_tree_ucl(khash_t(rspamd_url_hash) * set,
struct rspamd_task *task)
{
struct tree_cb_data cb;
ucl_object_t *obj;
struct rspamd_url *u;
obj = ucl_object_typed_new(UCL_ARRAY);
cb.top = obj;
cb.task = task;
cb.seen = kh_init(rspamd_url_host_hash);
kh_foreach_key(set, u, {
if (!(u->protocol & PROTOCOL_MAILTO)) {
urls_protocol_cb(u, &cb);
}
});
kh_destroy(rspamd_url_host_hash, cb.seen);
return obj;
}
static void
emails_protocol_cb(struct rspamd_url *url, struct tree_cb_data *cb)
{
ucl_object_t *obj;
if (url->userlen > 0 && url->hostlen > 0) {
obj = ucl_object_fromlstring(rspamd_url_user_unsafe(url),
url->userlen + url->hostlen + 1);
ucl_array_append(cb->top, obj);
}
}
static ucl_object_t *
rspamd_emails_tree_ucl(khash_t(rspamd_url_hash) * set,
struct rspamd_task *task)
{
struct tree_cb_data cb;
ucl_object_t *obj;
struct rspamd_url *u;
obj = ucl_object_typed_new(UCL_ARRAY);
cb.top = obj;
cb.task = task;
kh_foreach_key(set, u, {
if ((u->protocol & PROTOCOL_MAILTO)) {
emails_protocol_cb(u, &cb);
}
});
return obj;
}
/* Write new subject */
static const gchar *
rspamd_protocol_rewrite_subject(struct rspamd_task *task)
{
GString *subj_buf;
gchar *res;
const gchar *s, *c, *p;
gsize slen = 0;
c = rspamd_mempool_get_variable(task->task_pool, "metric_subject");
if (c == NULL) {
c = task->cfg->subject;
}
if (c == NULL) {
c = SPAM_SUBJECT;
}
p = c;
s = MESSAGE_FIELD_CHECK(task, subject);
if (s) {
slen = strlen(s);
}
subj_buf = g_string_sized_new(strlen(c) + slen);
while (*p) {
if (*p == '%') {
switch (p[1]) {
case 's':
g_string_append_len(subj_buf, c, p - c);
if (s) {
g_string_append_len(subj_buf, s, slen);
}
c = p + 2;
p += 2;
break;
case 'd':
g_string_append_len(subj_buf, c, p - c);
rspamd_printf_gstring(subj_buf, "%.2f", task->result->score);
c = p + 2;
p += 2;
break;
case '%':
g_string_append_len(subj_buf, c, p - c);
g_string_append_c(subj_buf, '%');
c = p + 2;
p += 2;
break;
default:
p++; /* Just % something unknown */
break;
}
}
else {
p++;
}
}
if (p > c) {
g_string_append_len(subj_buf, c, p - c);
}
res = rspamd_mime_header_encode(subj_buf->str, subj_buf->len);
rspamd_mempool_add_destructor(task->task_pool,
(rspamd_mempool_destruct_t) g_free,
res);
g_string_free(subj_buf, TRUE);
return res;
}
static ucl_object_t *
rspamd_metric_symbol_ucl(struct rspamd_task *task, struct rspamd_symbol_result *sym)
{
ucl_object_t *obj = NULL, *ar;
const gchar *description = NULL;
struct rspamd_symbol_option *opt;
if (sym->sym != NULL) {
description = sym->sym->description;
}
obj = ucl_object_typed_new(UCL_OBJECT);
ucl_object_insert_key(obj, ucl_object_fromstring(sym->name), "name", 0, false);
ucl_object_insert_key(obj, ucl_object_fromdouble(sym->score), "score", 0, false);
if (task->cmd == CMD_CHECK_V2) {
if (sym->sym) {
ucl_object_insert_key(obj, ucl_object_fromdouble(sym->sym->score), "metric_score", 0, false);
}
else {
ucl_object_insert_key(obj, ucl_object_fromdouble(0.0),
"metric_score", 0, false);
}
}
if (description) {
ucl_object_insert_key(obj, ucl_object_fromstring(description),
"description", 0, false);
}
if (sym->options != NULL) {
ar = ucl_object_typed_new(UCL_ARRAY);
DL_FOREACH(sym->opts_head, opt)
{
ucl_array_append(ar, ucl_object_fromstring_common(opt->option,
opt->optlen, 0));
}
ucl_object_insert_key(obj, ar, "options", 0, false);
}
return obj;
}
static ucl_object_t *
rspamd_metric_group_ucl(struct rspamd_task *task,
struct rspamd_symbols_group *gr, gdouble score)
{
ucl_object_t *obj = NULL;
obj = ucl_object_typed_new(UCL_OBJECT);
ucl_object_insert_key(obj, ucl_object_fromdouble(score),
"score", 0, false);
if (gr->description) {
ucl_object_insert_key(obj, ucl_object_fromstring(gr->description),
"description", 0, false);
}
return obj;
}
static ucl_object_t *
rspamd_scan_result_ucl(struct rspamd_task *task,
struct rspamd_scan_result *mres, ucl_object_t *top)
{
struct rspamd_symbol_result *sym;
gboolean is_spam;
struct rspamd_action *action;
ucl_object_t *obj = NULL, *sobj;
const gchar *subject;
struct rspamd_passthrough_result *pr = NULL;
action = rspamd_check_action_metric(task, &pr, NULL);
is_spam = !(action->flags & RSPAMD_ACTION_HAM);
if (task->cmd == CMD_CHECK) {
obj = ucl_object_typed_new(UCL_OBJECT);
ucl_object_insert_key(obj,
ucl_object_frombool(is_spam),
"is_spam", 0, false);
}
else {
obj = top;
}
if (pr) {
if (pr->message && !(pr->flags & RSPAMD_PASSTHROUGH_NO_SMTP_MESSAGE)) {
/* Add smtp message if it does not exist: see #3269 for details */
if (ucl_object_lookup(task->messages, "smtp_message") == NULL) {
ucl_object_insert_key(task->messages,
ucl_object_fromstring_common(pr->message, 0, UCL_STRING_RAW),
"smtp_message", 0,
false);
}
}
ucl_object_insert_key(obj,
ucl_object_fromstring(pr->module),
"passthrough_module", 0, false);
}
ucl_object_insert_key(obj,
ucl_object_frombool(RSPAMD_TASK_IS_SKIPPED(task)),
"is_skipped", 0, false);
if (!isnan(mres->score)) {
ucl_object_insert_key(obj, ucl_object_fromdouble(mres->score),
"score", 0, false);
}
else {
ucl_object_insert_key(obj,
ucl_object_fromdouble(0.0), "score", 0, false);
}
ucl_object_insert_key(obj,
ucl_object_fromdouble(rspamd_task_get_required_score(task, mres)),
"required_score", 0, false);
ucl_object_insert_key(obj,
ucl_object_fromstring(action->name),
"action", 0, false);
if (action->action_type == METRIC_ACTION_REWRITE_SUBJECT) {
subject = rspamd_protocol_rewrite_subject(task);
if (subject) {
ucl_object_insert_key(obj, ucl_object_fromstring(subject),
"subject", 0, false);
}
}
if (action->flags & RSPAMD_ACTION_MILTER) {
/* Treat milter action specially */
if (action->action_type == METRIC_ACTION_DISCARD) {
ucl_object_insert_key(obj, ucl_object_fromstring("discard"),
"reject", 0, false);
}
else if (action->action_type == METRIC_ACTION_QUARANTINE) {
ucl_object_insert_key(obj, ucl_object_fromstring("quarantine"),
"reject", 0, false);
}
}
/* Now handle symbols */
if (task->cmd != CMD_CHECK) {
/* Insert actions thresholds */
ucl_object_t *actions_obj = ucl_object_typed_new(UCL_OBJECT);
for (int i = task->result->nactions - 1; i >= 0; i--) {
struct rspamd_action_config *action_lim = &task->result->actions_config[i];
if (!isnan(action_lim->cur_limit) &&
!(action_lim->action->flags & (RSPAMD_ACTION_NO_THRESHOLD | RSPAMD_ACTION_HAM))) {
ucl_object_insert_key(actions_obj, ucl_object_fromdouble(action_lim->cur_limit),
action_lim->action->name, 0, true);
}
}
ucl_object_insert_key(obj, actions_obj, "thresholds", 0, false);
/* For checkv2 we insert symbols as a separate object */
obj = ucl_object_typed_new(UCL_OBJECT);
}
kh_foreach_value(mres->symbols, sym, {
if (!(sym->flags & RSPAMD_SYMBOL_RESULT_IGNORED)) {
sobj = rspamd_metric_symbol_ucl(task, sym);
ucl_object_insert_key(obj, sobj, sym->name, 0, false);
}
})
if (task->cmd != CMD_CHECK)
{
/* For checkv2 we insert symbols as a separate object */
ucl_object_insert_key(top, obj, "symbols", 0, false);
}
else
{
/* For legacy check we just insert it as "default" all together */
ucl_object_insert_key(top, obj, DEFAULT_METRIC, 0, false);
}
/* Handle groups if needed */
if (task->protocol_flags & RSPAMD_TASK_PROTOCOL_FLAG_GROUPS) {
struct rspamd_symbols_group *gr;
gdouble gr_score;
obj = ucl_object_typed_new(UCL_OBJECT);
ucl_object_reserve(obj, kh_size(mres->sym_groups));
kh_foreach(mres->sym_groups, gr, gr_score, {
if (task->cfg->public_groups_only &&
!(gr->flags & RSPAMD_SYMBOL_GROUP_PUBLIC)) {
continue;
}
sobj = rspamd_metric_group_ucl(task, gr, gr_score);
ucl_object_insert_key(obj, sobj, gr->name, 0, false);
});
ucl_object_insert_key(top, obj, "groups", 0, false);
}
return obj;
}
void rspamd_ucl_torspamc_output(const ucl_object_t *top,
rspamd_fstring_t **out)
{
const ucl_object_t *symbols, *score,
*required_score, *is_spam, *elt, *cur;
ucl_object_iter_t iter = NULL;
score = ucl_object_lookup(top, "score");
required_score = ucl_object_lookup(top, "required_score");
is_spam = ucl_object_lookup(top, "is_spam");
rspamd_printf_fstring(out,
"Metric: default; %s; %.2f / %.2f / 0.0\r\n",
ucl_object_toboolean(is_spam) ? "True" : "False",
ucl_object_todouble(score),
ucl_object_todouble(required_score));
elt = ucl_object_lookup(top, "action");
if (elt != NULL) {
rspamd_printf_fstring(out, "Action: %s\r\n",
ucl_object_tostring(elt));
}
elt = ucl_object_lookup(top, "subject");
if (elt != NULL) {
rspamd_printf_fstring(out, "Subject: %s\r\n",
ucl_object_tostring(elt));
}
symbols = ucl_object_lookup(top, "symbols");
if (symbols != NULL) {
iter = NULL;
while ((elt = ucl_object_iterate(symbols, &iter, true)) != NULL) {
if (elt->type == UCL_OBJECT) {
const ucl_object_t *sym_score;
sym_score = ucl_object_lookup(elt, "score");
rspamd_printf_fstring(out, "Symbol: %s(%.2f)\r\n",
ucl_object_key(elt),
ucl_object_todouble(sym_score));
}
}
}
elt = ucl_object_lookup(top, "messages");
if (elt != NULL) {
iter = NULL;
while ((cur = ucl_object_iterate(elt, &iter, true)) != NULL) {
if (cur->type == UCL_STRING) {
rspamd_printf_fstring(out, "Message: %s\r\n",
ucl_object_tostring(cur));
}
}
}
elt = ucl_object_lookup(top, "message-id");
if (elt != NULL) {
rspamd_printf_fstring(out, "Message-ID: %s\r\n",
ucl_object_tostring(elt));
}
}
void rspamd_ucl_tospamc_output(const ucl_object_t *top,
rspamd_fstring_t **out)
{
const ucl_object_t *symbols, *score,
*required_score, *is_spam, *elt;
ucl_object_iter_t iter = NULL;
rspamd_fstring_t *f;
score = ucl_object_lookup(top, "score");
required_score = ucl_object_lookup(top, "required_score");
is_spam = ucl_object_lookup(top, "is_spam");
rspamd_printf_fstring(out,
"Spam: %s ; %.2f / %.2f\r\n\r\n",
ucl_object_toboolean(is_spam) ? "True" : "False",
ucl_object_todouble(score),
ucl_object_todouble(required_score));
symbols = ucl_object_lookup(top, "symbols");
if (symbols != NULL) {
while ((elt = ucl_object_iterate(symbols, &iter, true)) != NULL) {
if (elt->type == UCL_OBJECT) {
rspamd_printf_fstring(out, "%s,",
ucl_object_key(elt));
}
}
/* Ugly hack, but the whole spamc is ugly */
f = *out;
if (f->str[f->len - 1] == ',') {
f->len--;
*out = rspamd_fstring_append(*out, CRLF, 2);
}
}
}
static void
rspamd_protocol_output_profiling(struct rspamd_task *task,
ucl_object_t *top)
{
GHashTable *tbl;
GHashTableIter it;
gpointer k, v;
ucl_object_t *prof;
gdouble val;
prof = ucl_object_typed_new(UCL_OBJECT);
tbl = rspamd_mempool_get_variable(task->task_pool, "profile");
if (tbl) {
g_hash_table_iter_init(&it, tbl);
while (g_hash_table_iter_next(&it, &k, &v)) {
val = *(gdouble *) v;
ucl_object_insert_key(prof, ucl_object_fromdouble(val),
(const char *) k, 0, false);
}
}
ucl_object_insert_key(top, prof, "profile", 0, false);
}
ucl_object_t *
rspamd_protocol_write_ucl(struct rspamd_task *task,
enum rspamd_protocol_flags flags)
{
ucl_object_t *top = NULL;
GString *dkim_sig;
GList *dkim_sigs;
const ucl_object_t *milter_reply;
rspamd_task_set_finish_time(task);
top = ucl_object_typed_new(UCL_OBJECT);
rspamd_mempool_add_destructor(task->task_pool,
(rspamd_mempool_destruct_t) ucl_object_unref, top);
if (flags & RSPAMD_PROTOCOL_METRICS) {
rspamd_scan_result_ucl(task, task->result, top);
}
if (flags & RSPAMD_PROTOCOL_MESSAGES) {
if (G_UNLIKELY(task->cfg->compat_messages)) {
const ucl_object_t *cur;
ucl_object_t *msg_object;
ucl_object_iter_t iter = NULL;
msg_object = ucl_object_typed_new(UCL_ARRAY);
while ((cur = ucl_object_iterate(task->messages, &iter, true)) != NULL) {
if (cur->type == UCL_STRING) {
ucl_array_append(msg_object, ucl_object_ref(cur));
}
}
ucl_object_insert_key(top, msg_object, "messages", 0, false);
}
else {
ucl_object_insert_key(top, ucl_object_ref(task->messages),
"messages", 0, false);
}
}
if (flags & RSPAMD_PROTOCOL_URLS && task->message) {
if (kh_size(MESSAGE_FIELD(task, urls)) > 0) {
ucl_object_insert_key(top,
rspamd_urls_tree_ucl(MESSAGE_FIELD(task, urls), task),
"urls", 0, false);
ucl_object_insert_key(top,
rspamd_emails_tree_ucl(MESSAGE_FIELD(task, urls), task),
"emails", 0, false);
}
}
if (flags & RSPAMD_PROTOCOL_EXTRA) {
if (G_UNLIKELY(RSPAMD_TASK_IS_PROFILING(task))) {
rspamd_protocol_output_profiling(task, top);
}
}
if (flags & RSPAMD_PROTOCOL_BASIC) {
ucl_object_insert_key(top,
ucl_object_fromstring(MESSAGE_FIELD_CHECK(task, message_id)),
"message-id", 0, false);
ucl_object_insert_key(top,
ucl_object_fromdouble(task->time_real_finish - task->task_timestamp),
"time_real", 0, false);
}
if (flags & RSPAMD_PROTOCOL_DKIM) {
dkim_sigs = rspamd_mempool_get_variable(task->task_pool,
RSPAMD_MEMPOOL_DKIM_SIGNATURE);
if (dkim_sigs) {
if (dkim_sigs->next) {
/* Multiple DKIM signatures */
ucl_object_t *ar = ucl_object_typed_new(UCL_ARRAY);
for (; dkim_sigs != NULL; dkim_sigs = dkim_sigs->next) {
GString *folded_header;
dkim_sig = (GString *) dkim_sigs->data;
if (task->protocol_flags & RSPAMD_TASK_PROTOCOL_FLAG_MILTER ||
!task->message) {
folded_header = rspamd_header_value_fold(
"DKIM-Signature", strlen("DKIM-Signature"),
dkim_sig->str, dkim_sig->len,
80, RSPAMD_TASK_NEWLINES_LF, NULL);
}
else {
folded_header = rspamd_header_value_fold(
"DKIM-Signature", strlen("DKIM-Signature"),
dkim_sig->str, dkim_sig->len,
80,
MESSAGE_FIELD(task, nlines_type),
NULL);
}
ucl_array_append(ar,
ucl_object_fromstring_common(folded_header->str,
folded_header->len, UCL_STRING_RAW));
g_string_free(folded_header, TRUE);
}
ucl_object_insert_key(top,
ar,
"dkim-signature", 0,
false);
}
else {
/* Single DKIM signature */
GString *folded_header;
dkim_sig = (GString *) dkim_sigs->data;
if (task->protocol_flags & RSPAMD_TASK_PROTOCOL_FLAG_MILTER) {
folded_header = rspamd_header_value_fold(
"DKIM-Signature", strlen("DKIM-Signature"),
dkim_sig->str, dkim_sig->len,
80, RSPAMD_TASK_NEWLINES_LF, NULL);
}
else {
folded_header = rspamd_header_value_fold(
"DKIM-Signature", strlen("DKIM-Signature"),
dkim_sig->str, dkim_sig->len,
80, MESSAGE_FIELD(task, nlines_type),
NULL);
}
ucl_object_insert_key(top,
ucl_object_fromstring_common(folded_header->str,
folded_header->len, UCL_STRING_RAW),
"dkim-signature", 0, false);
g_string_free(folded_header, TRUE);
}
}
}
if (flags & RSPAMD_PROTOCOL_RMILTER) {
milter_reply = rspamd_mempool_get_variable(task->task_pool,
RSPAMD_MEMPOOL_MILTER_REPLY);
if (milter_reply) {
if (task->cmd != CMD_CHECK) {
ucl_object_insert_key(top, ucl_object_ref(milter_reply),
"milter", 0, false);
}
else {
ucl_object_insert_key(top, ucl_object_ref(milter_reply),
"rmilter", 0, false);
}
}
}
return top;
}
void rspamd_protocol_http_reply(struct rspamd_http_message *msg,
struct rspamd_task *task, ucl_object_t **pobj)
{
struct rspamd_scan_result *metric_res;
const struct rspamd_re_cache_stat *restat;
ucl_object_t *top = NULL;
rspamd_fstring_t *reply;
gint flags = RSPAMD_PROTOCOL_DEFAULT;
struct rspamd_action *action;
/* Removed in 2.0 */
#if 0
GHashTableIter hiter;
gpointer h, v;
/* Write custom headers */
g_hash_table_iter_init (&hiter, task->reply_headers);
while (g_hash_table_iter_next (&hiter, &h, &v)) {
rspamd_ftok_t *hn = h, *hv = v;
rspamd_http_message_add_header (msg, hn->begin, hv->begin);
}
#endif
flags |= RSPAMD_PROTOCOL_URLS;
top = rspamd_protocol_write_ucl(task, flags);
if (pobj) {
*pobj = top;
}
if (!(task->flags & RSPAMD_TASK_FLAG_NO_LOG)) {
rspamd_roll_history_update(task->worker->srv->history, task);
}
else {
msg_debug_protocol("skip history update due to no log flag");
}
rspamd_task_write_log(task);
if (task->cfg->log_flags & RSPAMD_LOG_FLAG_RE_CACHE) {
restat = rspamd_re_cache_get_stat(task->re_rt);
g_assert(restat != NULL);
msg_notice_task(
"regexp statistics: %ud pcre regexps scanned, %ud regexps matched,"
" %ud regexps total, %ud regexps cached,"
" %HL scanned using pcre, %HL scanned total",
restat->regexp_checked,
restat->regexp_matched,
restat->regexp_total,
restat->regexp_fast_cached,
restat->bytes_scanned_pcre,
restat->bytes_scanned);
}
reply = rspamd_fstring_sized_new(1000);
if (msg->method < HTTP_SYMBOLS && !RSPAMD_TASK_IS_SPAMC(task)) {
msg_debug_protocol("writing json reply");
rspamd_ucl_emit_fstring(top, UCL_EMIT_JSON_COMPACT, &reply);
}
else {
if (RSPAMD_TASK_IS_SPAMC(task)) {
msg_debug_protocol("writing spamc legacy reply to client");
rspamd_ucl_tospamc_output(top, &reply);
}
else {
msg_debug_protocol("writing rspamc legacy reply to client");
rspamd_ucl_torspamc_output(top, &reply);
}
}
if (task->protocol_flags & RSPAMD_TASK_PROTOCOL_FLAG_BODY_BLOCK) {
/* Check if we need to insert a body block */
if (task->flags & RSPAMD_TASK_FLAG_MESSAGE_REWRITE) {
GString *hdr_offset = g_string_sized_new(30);
rspamd_printf_gstring(hdr_offset, "%z", RSPAMD_FSTRING_LEN(reply));
rspamd_http_message_add_header(msg, MESSAGE_OFFSET_HEADER,
hdr_offset->str);
msg_debug_protocol("write body block at position %s",
hdr_offset->str);
g_string_free(hdr_offset, TRUE);
/* In case of milter, we append just body, otherwise - full message */
if (task->protocol_flags & RSPAMD_TASK_PROTOCOL_FLAG_MILTER) {
const gchar *start;
goffset len, hdr_off;
start = task->msg.begin;
len = task->msg.len;
hdr_off = MESSAGE_FIELD(task, raw_headers_content).len;
if (hdr_off < len) {
start += hdr_off;
len -= hdr_off;
/* The problem here is that we need not end of headers, we need
* start of body.
*
* Hence, we need to skip one \r\n till there is anything else in
* a line.
*/
if (*start == '\r' && len > 0) {
start++;
len--;
}
if (*start == '\n' && len > 0) {
start++;
len--;
}
msg_debug_protocol("milter version of body block size %d",
(int) len);
reply = rspamd_fstring_append(reply, start, len);
}
}
else {
msg_debug_protocol("general version of body block size %d",
(int) task->msg.len);
reply = rspamd_fstring_append(reply,
task->msg.begin, task->msg.len);
}
}
}
if ((task->protocol_flags & RSPAMD_TASK_PROTOCOL_FLAG_COMPRESSED) &&
rspamd_libs_reset_compression(task->cfg->libs_ctx)) {
/* We can compress output */
ZSTD_inBuffer zin;
ZSTD_outBuffer zout;
ZSTD_CStream *zstream;
rspamd_fstring_t *compressed_reply;
gsize r;
zstream = task->cfg->libs_ctx->out_zstream;
compressed_reply = rspamd_fstring_sized_new(ZSTD_compressBound(reply->len));
zin.pos = 0;
zin.src = reply->str;
zin.size = reply->len;
zout.pos = 0;
zout.dst = compressed_reply->str;
zout.size = compressed_reply->allocated;
while (zin.pos < zin.size) {
r = ZSTD_compressStream(zstream, &zout, &zin);
if (ZSTD_isError(r)) {
msg_err_protocol("cannot compress: %s", ZSTD_getErrorName(r));
rspamd_fstring_free(compressed_reply);
rspamd_http_message_set_body_from_fstring_steal(msg, reply);
goto end;
}
}
ZSTD_flushStream(zstream, &zout);
r = ZSTD_endStream(zstream, &zout);
if (ZSTD_isError(r)) {
msg_err_protocol("cannot finalize compress: %s", ZSTD_getErrorName(r));
rspamd_fstring_free(compressed_reply);
rspamd_http_message_set_body_from_fstring_steal(msg, reply);
goto end;
}
msg_info_protocol("writing compressed results: %z bytes before "
"%z bytes after",
zin.pos, zout.pos);
compressed_reply->len = zout.pos;
rspamd_fstring_free(reply);
rspamd_http_message_set_body_from_fstring_steal(msg, compressed_reply);
rspamd_http_message_add_header(msg, COMPRESSION_HEADER, "zstd");
if (task->cfg->libs_ctx->out_dict &&
task->cfg->libs_ctx->out_dict->id != 0) {
gchar dict_str[32];
rspamd_snprintf(dict_str, sizeof(dict_str), "%ud",
task->cfg->libs_ctx->out_dict->id);
rspamd_http_message_add_header(msg, "Dictionary", dict_str);
}
}
else {
rspamd_http_message_set_body_from_fstring_steal(msg, reply);
}
end:
if (!(task->flags & RSPAMD_TASK_FLAG_NO_STAT)) {
/* Update stat for default metric */
msg_debug_protocol("skip stats update due to no_stat flag");
metric_res = task->result;
if (metric_res != NULL) {
action = rspamd_check_action_metric(task, NULL, NULL);
/* TODO: handle custom actions in stats */
if (action->action_type == METRIC_ACTION_SOFT_REJECT &&
(task->flags & RSPAMD_TASK_FLAG_GREYLISTED)) {
/* Set stat action to greylist to display greylisted messages */
#ifndef HAVE_ATOMIC_BUILTINS
task->worker->srv->stat->actions_stat[METRIC_ACTION_GREYLIST]++;
#else
__atomic_add_fetch(&task->worker->srv->stat->actions_stat[METRIC_ACTION_GREYLIST],
1, __ATOMIC_RELEASE);
#endif
}
else if (action->action_type < METRIC_ACTION_MAX) {
#ifndef HAVE_ATOMIC_BUILTINS
task->worker->srv->stat->actions_stat[action->action_type]++;
#else
__atomic_add_fetch(&task->worker->srv->stat->actions_stat[action->action_type],
1, __ATOMIC_RELEASE);
#endif
}
}
/* Increase counters */
#ifndef HAVE_ATOMIC_BUILTINS
task->worker->srv->stat->messages_scanned++;
#else
__atomic_add_fetch(&task->worker->srv->stat->messages_scanned,
1, __ATOMIC_RELEASE);
#endif
/* Set average processing time */
guint32 slot;
float processing_time = task->time_real_finish - task->task_timestamp;
#ifndef HAVE_ATOMIC_BUILTINS
slot = task->worker->srv->stat->avg_time.cur_slot++;
#else
slot = __atomic_fetch_add(&task->worker->srv->stat->avg_time.cur_slot,
1, __ATOMIC_RELEASE);
#endif
slot = slot % MAX_AVG_TIME_SLOTS;
/* TODO: this should be atomic but it is not supported in C */
task->worker->srv->stat->avg_time.avg_time[slot] = processing_time;
}
}
void rspamd_protocol_write_log_pipe(struct rspamd_task *task)
{
struct rspamd_worker_log_pipe *lp;
struct rspamd_protocol_log_message_sum *ls;
lua_State *L = task->cfg->lua_state;
struct rspamd_scan_result *mres;
struct rspamd_symbol_result *sym;
gint id, i;
guint32 n = 0, nextra = 0;
gsize sz;
GArray *extra;
struct rspamd_protocol_log_symbol_result er;
struct rspamd_task **ptask;
/* Get extra results from lua plugins */
extra = g_array_new(FALSE, FALSE, sizeof(er));
lua_getglobal(L, "rspamd_plugins");
if (lua_istable(L, -1)) {
lua_pushnil(L);
while (lua_next(L, -2)) {
if (lua_istable(L, -1)) {
lua_pushvalue(L, -2);
/* stack:
* -1: copy of key
* -2: value (module table)
* -3: key (module name)
* -4: global
*/
lua_pushstring(L, "log_callback");
lua_gettable(L, -3);
/* stack:
* -1: func
* -2: copy of key
* -3: value (module table)
* -3: key (module name)
* -4: global
*/
if (lua_isfunction(L, -1)) {
ptask = lua_newuserdata(L, sizeof(*ptask));
*ptask = task;
rspamd_lua_setclass(L, "rspamd{task}", -1);
/* stack:
* -1: task
* -2: func
* -3: key copy
* -4: value (module table)
* -5: key (module name)
* -6: global
*/
msg_debug_protocol("calling for %s", lua_tostring(L, -3));
if (lua_pcall(L, 1, 1, 0) != 0) {
msg_info_protocol("call to log callback %s failed: %s",
lua_tostring(L, -2), lua_tostring(L, -1));
lua_pop(L, 1);
/* stack:
* -1: key copy
* -2: value
* -3: key
*/
}
else {
/* stack:
* -1: result
* -2: key copy
* -3: value
* -4: key
*/
if (lua_istable(L, -1)) {
/* Another iteration */
lua_pushnil(L);
while (lua_next(L, -2)) {
/* stack:
* -1: value
* -2: key
* -3: result table (pcall)
* -4: key copy (parent)
* -5: value (parent)
* -6: key (parent)
*/
if (lua_istable(L, -1)) {
er.id = 0;
er.score = 0.0;
lua_rawgeti(L, -1, 1);
if (lua_isnumber(L, -1)) {
er.id = lua_tonumber(L, -1);
}
lua_rawgeti(L, -2, 2);
if (lua_isnumber(L, -1)) {
er.score = lua_tonumber(L, -1);
}
/* stack:
* -1: value[2]
* -2: value[1]
* -3: values
* -4: key
* -5: result table (pcall)
* -6: key copy (parent)
* -7: value (parent)
* -8: key (parent)
*/
lua_pop(L, 2); /* Values */
g_array_append_val(extra, er);
}
lua_pop(L, 1); /* Value for lua_next */
}
lua_pop(L, 1); /* Table result of pcall */
}
else {
msg_info_protocol("call to log callback %s returned "
"wrong type: %s",
lua_tostring(L, -2),
lua_typename(L, lua_type(L, -1)));
lua_pop(L, 1); /* Returned error */
}
}
}
else {
lua_pop(L, 1);
/* stack:
* -1: key copy
* -2: value
* -3: key
*/
}
}
lua_pop(L, 2); /* Top table + key copy */
}
lua_pop(L, 1); /* rspamd_plugins global */
}
else {
lua_pop(L, 1);
}
nextra = extra->len;
LL_FOREACH(task->cfg->log_pipes, lp)
{
if (lp->fd != -1) {
switch (lp->type) {
case RSPAMD_LOG_PIPE_SYMBOLS:
mres = task->result;
if (mres) {
n = kh_size(mres->symbols);
sz = sizeof(*ls) +
sizeof(struct rspamd_protocol_log_symbol_result) *
(n + nextra);
ls = g_malloc0(sz);
/* Handle settings id */
if (task->settings_elt) {
ls->settings_id = task->settings_elt->id;
}
else {
ls->settings_id = 0;
}
ls->score = mres->score;
ls->required_score = rspamd_task_get_required_score(task,
mres);
ls->nresults = n;
ls->nextra = nextra;
i = 0;
kh_foreach_value(mres->symbols, sym, {
id = rspamd_symcache_find_symbol(task->cfg->cache,
sym->name);
if (id >= 0) {
ls->results[i].id = id;
ls->results[i].score = sym->score;
}
else {
ls->results[i].id = -1;
ls->results[i].score = 0.0;
}
i++;
});
memcpy(&ls->results[n], extra->data, nextra * sizeof(er));
}
else {
sz = sizeof(*ls);
ls = g_malloc0(sz);
ls->nresults = 0;
}
/* We don't really care about return value here */
if (write(lp->fd, ls, sz) == -1) {
msg_info_protocol("cannot write to log pipe: %s",
strerror(errno));
}
g_free(ls);
break;
default:
msg_err_protocol("unknown log format %d", lp->type);
break;
}
}
}
g_array_free(extra, TRUE);
}
void rspamd_protocol_write_reply(struct rspamd_task *task, ev_tstamp timeout)
{
struct rspamd_http_message *msg;
const gchar *ctype = "application/json";
rspamd_fstring_t *reply;
msg = rspamd_http_new_message(HTTP_RESPONSE);
if (rspamd_http_connection_is_encrypted(task->http_conn)) {
msg_info_protocol("<%s> writing encrypted reply",
MESSAGE_FIELD_CHECK(task, message_id));
}
/* Compatibility */
if (task->cmd == CMD_CHECK_RSPAMC) {
msg->method = HTTP_SYMBOLS;
}
else if (task->cmd == CMD_CHECK_SPAMC) {
msg->method = HTTP_SYMBOLS;
msg->flags |= RSPAMD_HTTP_FLAG_SPAMC;
}
if (task->err != NULL) {
msg_debug_protocol("writing error reply to client");
ucl_object_t *top = NULL;
top = ucl_object_typed_new(UCL_OBJECT);
msg->code = 500 + task->err->code % 100;
msg->status = rspamd_fstring_new_init(task->err->message,
strlen(task->err->message));
ucl_object_insert_key(top, ucl_object_fromstring(task->err->message),
"error", 0, false);
ucl_object_insert_key(top,
ucl_object_fromstring(g_quark_to_string(task->err->domain)),
"error_domain", 0, false);
reply = rspamd_fstring_sized_new(256);
rspamd_ucl_emit_fstring(top, UCL_EMIT_JSON_COMPACT, &reply);
ucl_object_unref(top);
/* We also need to validate utf8 */
if (rspamd_fast_utf8_validate(reply->str, reply->len) != 0) {
gsize valid_len;
gchar *validated;
/* We copy reply several times here but it should be a rare case */
validated = rspamd_str_make_utf_valid(reply->str, reply->len,
&valid_len, task->task_pool);
rspamd_http_message_set_body(msg, validated, valid_len);
rspamd_fstring_free(reply);
}
else {
rspamd_http_message_set_body_from_fstring_steal(msg, reply);
}
}
else {
msg->status = rspamd_fstring_new_init("OK", 2);
switch (task->cmd) {
case CMD_CHECK:
case CMD_CHECK_RSPAMC:
case CMD_CHECK_SPAMC:
case CMD_SKIP:
case CMD_CHECK_V2:
rspamd_protocol_http_reply(msg, task, NULL);
rspamd_protocol_write_log_pipe(task);
break;
case CMD_PING:
msg_debug_protocol("writing pong to client");
rspamd_http_message_set_body(msg, "pong" CRLF, 6);
ctype = "text/plain";
break;
default:
msg_err_protocol("BROKEN");
break;
}
}
ev_now_update(task->event_loop);
msg->date = ev_time();
rspamd_http_connection_reset(task->http_conn);
rspamd_http_connection_write_message(task->http_conn, msg, NULL,
ctype, task, timeout);
task->processed_stages |= RSPAMD_TASK_STAGE_REPLIED;
}