diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/fuzzy_storage.c | 443 | ||||
-rw-r--r-- | src/libserver/CMakeLists.txt | 1 | ||||
-rw-r--r-- | src/libserver/fuzzy_backend/fuzzy_backend.c | 21 | ||||
-rw-r--r-- | src/libserver/fuzzy_backend/fuzzy_backend_noop.c | 96 | ||||
-rw-r--r-- | src/libserver/fuzzy_backend/fuzzy_backend_noop.h | 66 | ||||
-rw-r--r-- | src/libserver/fuzzy_backend/fuzzy_backend_redis.h | 7 | ||||
-rw-r--r-- | src/libserver/logger/logger.c | 26 | ||||
-rw-r--r-- | src/libserver/redis_pool.cxx | 11 | ||||
-rw-r--r-- | src/lua/CMakeLists.txt | 3 | ||||
-rw-r--r-- | src/lua/lua_classnames.c | 4 | ||||
-rw-r--r-- | src/lua/lua_classnames.h | 5 | ||||
-rw-r--r-- | src/lua/lua_common.c | 3 | ||||
-rw-r--r-- | src/lua/lua_common.h | 10 | ||||
-rw-r--r-- | src/lua/lua_config.c | 2 | ||||
-rw-r--r-- | src/lua/lua_http.c | 2 | ||||
-rw-r--r-- | src/lua/lua_redis.c | 97 | ||||
-rw-r--r-- | src/lua/lua_shingles.cxx | 133 | ||||
-rw-r--r-- | src/lua/lua_task.c | 93 | ||||
-rw-r--r-- | src/plugins/lua/gpt.lua | 16 | ||||
-rw-r--r-- | src/plugins/lua/rbl.lua | 4 |
20 files changed, 783 insertions, 260 deletions
diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c index bc69be98c..270dd9c53 100644 --- a/src/fuzzy_storage.c +++ b/src/fuzzy_storage.c @@ -1,5 +1,5 @@ /* - * Copyright 2024 Vsevolod Stakhov + * Copyright 2025 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -124,6 +124,11 @@ fuzzy_kp_equal(gconstpointer a, gconstpointer b) return (memcmp(pa, pb, RSPAMD_FUZZY_KEYLEN) == 0); } +enum fuzzy_key_op { + FUZZY_KEY_READ = 0x1u << 0, + FUZZY_KEY_WRITE = 0x1u << 1, + FUZZY_KEY_DELETE = 0x1u << 2, +}; KHASH_SET_INIT_INT(fuzzy_key_ids_set); KHASH_INIT(fuzzy_key_flag_stat, int, struct fuzzy_key_stat, 1, kh_int_hash_func, kh_int_hash_equal); @@ -135,10 +140,12 @@ struct fuzzy_key { khash_t(fuzzy_key_flag_stat) * flags_stat; khash_t(fuzzy_key_ids_set) * forbidden_ids; struct rspamd_leaky_bucket_elt *rl_bucket; + ucl_object_t *extensions; double burst; double rate; ev_tstamp expire; bool expired; + int flags; /* enum fuzzy_key_op */ ref_entry_t ref; }; @@ -146,6 +153,11 @@ KHASH_INIT(rspamd_fuzzy_keys_hash, const unsigned char *, struct fuzzy_key *, 1, fuzzy_kp_hash, fuzzy_kp_equal); +struct rspamd_lua_fuzzy_script { + int cbref; + struct rspamd_lua_fuzzy_script *next; +}; + struct rspamd_fuzzy_storage_ctx { uint64_t magic; /* Events base */ @@ -208,9 +220,9 @@ struct rspamd_fuzzy_storage_ctx { struct rspamd_worker *worker; const ucl_object_t *skip_map; struct rspamd_hash_map_helper *skip_hashes; - int lua_pre_handler_cbref; - int lua_post_handler_cbref; - int lua_blacklist_cbref; + struct rspamd_lua_fuzzy_script *lua_pre_handlers; + struct rspamd_lua_fuzzy_script *lua_post_handlers; + struct rspamd_lua_fuzzy_script *lua_blacklist_handlers; khash_t(fuzzy_key_ids_set) * default_forbidden_ids; /* Ids that should not override other ids */ khash_t(fuzzy_key_ids_set) * weak_ids; @@ -585,25 +597,29 @@ rspamd_fuzzy_maybe_call_blacklisted(struct rspamd_fuzzy_storage_ctx *ctx, rspamd_inet_addr_t *addr, const char *reason) { - if (ctx->lua_blacklist_cbref != -1) { - lua_State *L = ctx->cfg->lua_state; - int err_idx, ret; + if (ctx->lua_blacklist_handlers != NULL) { + struct rspamd_lua_fuzzy_script *cur; + LL_FOREACH(ctx->lua_blacklist_handlers, cur) + { + lua_State *L = ctx->cfg->lua_state; + int err_idx, ret; - lua_pushcfunction(L, &rspamd_lua_traceback); - err_idx = lua_gettop(L); - lua_rawgeti(L, LUA_REGISTRYINDEX, ctx->lua_blacklist_cbref); - /* client IP */ - rspamd_lua_ip_push(L, addr); - /* block reason */ - lua_pushstring(L, reason); + lua_pushcfunction(L, &rspamd_lua_traceback); + err_idx = lua_gettop(L); + lua_rawgeti(L, LUA_REGISTRYINDEX, cur->cbref); + /* client IP */ + rspamd_lua_ip_push(L, addr); + /* block reason */ + lua_pushstring(L, reason); - if ((ret = lua_pcall(L, 2, 0, err_idx)) != 0) { - msg_err("call to lua_blacklist_cbref " - "script failed (%d): %s", - ret, lua_tostring(L, -1)); - } + if ((ret = lua_pcall(L, 2, 0, err_idx)) != 0) { + msg_err("call to lua_blacklist_cbref " + "script failed (%d): %s", + ret, lua_tostring(L, -1)); + } - lua_settop(L, 0); + lua_settop(L, 0); + } } } @@ -624,12 +640,15 @@ rspamd_fuzzy_check_client(struct rspamd_fuzzy_storage_ctx *ctx, } static gboolean -rspamd_fuzzy_check_write(struct fuzzy_session *session) +rspamd_fuzzy_check_write(struct fuzzy_session *session, uint8_t cmd) { if (session->ctx->read_only) { return FALSE; } + /* + * Check IP first + */ if (session->ctx->update_ips != NULL && session->addr) { if (rspamd_inet_address_get_af(session->addr) == AF_UNIX) { return TRUE; @@ -643,6 +662,9 @@ rspamd_fuzzy_check_write(struct fuzzy_session *session) } } + /* + * Check global list of the update keys + */ if (session->ctx->update_keys != NULL && session->key->stat && session->key->key) { static char base32_buf[rspamd_cryptobox_HASHBYTES * 2 + 1]; unsigned int raw_len; @@ -657,6 +679,15 @@ rspamd_fuzzy_check_write(struct fuzzy_session *session) } } + if (session->key) { + if (cmd == FUZZY_WRITE && session->key->flags & FUZZY_KEY_WRITE) { + return TRUE; + } + else if (cmd == FUZZY_DEL && session->key->flags & FUZZY_KEY_DELETE) { + return TRUE; + } + } + return FALSE; } @@ -711,6 +742,10 @@ fuzzy_key_dtor(gpointer p) g_free(key->name); } + if (key->extensions) { + ucl_object_unref(key->extensions); + } + g_free(key); } } @@ -1259,80 +1294,89 @@ rspamd_fuzzy_check_callback(struct rspamd_fuzzy_reply *result, void *ud) break; } - if (session->ctx->lua_post_handler_cbref != -1) { - /* Start lua post handler */ - lua_State *L = session->ctx->cfg->lua_state; - int err_idx, ret; - - lua_pushcfunction(L, &rspamd_lua_traceback); - err_idx = lua_gettop(L); - /* Preallocate stack (small opt) */ - lua_checkstack(L, err_idx + 9); - /* function */ - lua_rawgeti(L, LUA_REGISTRYINDEX, session->ctx->lua_post_handler_cbref); - /* client IP */ - if (session->addr) { - rspamd_lua_ip_push(L, session->addr); - } - else { - lua_pushnil(L); - } - /* client command */ - lua_pushinteger(L, cmd->cmd); - /* command value (push as rspamd_text) */ - (void) lua_new_text(L, result->digest, sizeof(result->digest), FALSE); - /* is shingle */ - lua_pushboolean(L, is_shingle); - /* result value */ - lua_pushinteger(L, result->v1.value); - /* result probability */ - lua_pushnumber(L, result->v1.prob); - /* result flag */ - lua_pushinteger(L, result->v1.flag); - /* result timestamp */ - lua_pushinteger(L, result->ts); - /* TODO: add additional data maybe (encryption, pubkey, etc) */ - rspamd_fuzzy_extensions_tolua(L, session); - - if ((ret = lua_pcall(L, 9, LUA_MULTRET, err_idx)) != 0) { - msg_err("call to lua_post_handler lua " - "script failed (%d): %s", - ret, lua_tostring(L, -1)); - } - else { - /* Return values order: - * the first reply will be on err_idx + 1 - * if it is true, then we need to read the former ones: - * 2-nd will be reply code - * 3-rd will be probability (or 0.0 if missing) - * 4-th value is flag (or default flag if missing) - */ - ret = lua_toboolean(L, err_idx + 1); + if (session->ctx->lua_post_handlers != NULL) { + struct rspamd_lua_fuzzy_script *cur; + LL_FOREACH(session->ctx->lua_post_handlers, cur) + { + /* Start lua post handler */ + lua_State *L = session->ctx->cfg->lua_state; + int err_idx, ret, nargs = 9; + + lua_pushcfunction(L, &rspamd_lua_traceback); + err_idx = lua_gettop(L); + /* Preallocate stack (small opt) */ + lua_checkstack(L, err_idx + 9); + /* function */ + lua_rawgeti(L, LUA_REGISTRYINDEX, cur->cbref); + /* client IP */ + if (session->addr) { + rspamd_lua_ip_push(L, session->addr); + } + else { + lua_pushnil(L); + } + /* client command */ + lua_pushinteger(L, cmd->cmd); + /* command value (push as rspamd_text) */ + (void) lua_new_text(L, result->digest, sizeof(result->digest), FALSE); + /* is shingle */ + lua_pushboolean(L, is_shingle); + /* result value */ + lua_pushinteger(L, result->v1.value); + /* result probability */ + lua_pushnumber(L, result->v1.prob); + /* result flag */ + lua_pushinteger(L, result->v1.flag); + /* result timestamp */ + lua_pushinteger(L, result->ts); + /* TODO: add additional data maybe (encryption, pubkey, etc) */ + rspamd_fuzzy_extensions_tolua(L, session); + /* We push shingles merely for commands that modify content to avoid extra work */ + if (is_shingle && cmd->cmd != FUZZY_CHECK) { + lua_newshingle(L, &session->cmd.sgl); + nargs++; + } + + if ((ret = lua_pcall(L, nargs, LUA_MULTRET, err_idx)) != 0) { + msg_err("call to lua_post_handler lua " + "script failed (%d): %s", + ret, lua_tostring(L, -1)); + } + else { + /* Return values order: + * the first reply will be on err_idx + 1 + * if it is true, then we need to read the former ones: + * 2-nd will be reply code + * 3-rd will be probability (or 0.0 if missing) + * 4-th value is flag (or default flag if missing) + */ + ret = lua_toboolean(L, err_idx + 1); - if (ret) { - /* Artificial reply */ - result->v1.value = lua_tointeger(L, err_idx + 2); + if (ret) { + /* Artificial reply */ + result->v1.value = lua_tointeger(L, err_idx + 2); - if (lua_isnumber(L, err_idx + 3)) { - result->v1.prob = lua_tonumber(L, err_idx + 3); - } - else { - result->v1.prob = 0.0f; - } + if (lua_isnumber(L, err_idx + 3)) { + result->v1.prob = lua_tonumber(L, err_idx + 3); + } + else { + result->v1.prob = 0.0f; + } - if (lua_isnumber(L, err_idx + 4)) { - result->v1.flag = lua_tointeger(L, err_idx + 4); - } + if (lua_isnumber(L, err_idx + 4)) { + result->v1.flag = lua_tointeger(L, err_idx + 4); + } - lua_settop(L, 0); - rspamd_fuzzy_make_reply(cmd, result, session, send_flags); - REF_RELEASE(session); + lua_settop(L, 0); + rspamd_fuzzy_make_reply(cmd, result, session, send_flags); + REF_RELEASE(session); - return; + return; + } } - } - lua_settop(L, 0); + lua_settop(L, 0); + } } if (!isnan(session->ctx->delay) && @@ -1449,61 +1493,72 @@ rspamd_fuzzy_process_command(struct fuzzy_session *session) result.v1.flag = cmd->flag; result.v1.tag = cmd->tag; - if (session->ctx->lua_pre_handler_cbref != -1) { - /* Start lua pre handler */ - lua_State *L = session->ctx->cfg->lua_state; - int err_idx, ret; - - lua_pushcfunction(L, &rspamd_lua_traceback); - err_idx = lua_gettop(L); - /* Preallocate stack (small opt) */ - lua_checkstack(L, err_idx + 5); - /* function */ - lua_rawgeti(L, LUA_REGISTRYINDEX, session->ctx->lua_pre_handler_cbref); - /* client IP */ - rspamd_lua_ip_push(L, session->addr); - /* client command */ - lua_pushinteger(L, cmd->cmd); - /* command value (push as rspamd_text) */ - (void) lua_new_text(L, cmd->digest, sizeof(cmd->digest), FALSE); - /* is shingle */ - lua_pushboolean(L, is_shingle); - /* TODO: add additional data maybe (encryption, pubkey, etc) */ - rspamd_fuzzy_extensions_tolua(L, session); - - if ((ret = lua_pcall(L, 5, LUA_MULTRET, err_idx)) != 0) { - msg_err("call to lua_pre_handler lua " - "script failed (%d): %s", - ret, lua_tostring(L, -1)); - } - else { - /* Return values order: - * the first reply will be on err_idx + 1 - * if it is true, then we need to read the former ones: - * 2-nd will be reply code - * 3-rd will be probability (or 0.0 if missing) - */ - ret = lua_toboolean(L, err_idx + 1); + if (session->ctx->lua_pre_handlers != NULL) { + struct rspamd_lua_fuzzy_script *cur; - if (ret) { - /* Artificial reply */ - result.v1.value = lua_tointeger(L, err_idx + 2); + LL_FOREACH(session->ctx->lua_pre_handlers, cur) + { + /* Start lua pre handler */ + lua_State *L = session->ctx->cfg->lua_state; + int err_idx, ret, nargs = 5; + + lua_pushcfunction(L, &rspamd_lua_traceback); + err_idx = lua_gettop(L); + /* Preallocate stack (small opt) */ + lua_checkstack(L, err_idx + 5); + /* function */ + lua_rawgeti(L, LUA_REGISTRYINDEX, cur->cbref); + /* client IP */ + rspamd_lua_ip_push(L, session->addr); + /* client command */ + lua_pushinteger(L, cmd->cmd); + /* command value (push as rspamd_text) */ + (void) lua_new_text(L, cmd->digest, sizeof(cmd->digest), FALSE); + /* is shingle */ + lua_pushboolean(L, is_shingle); + /* TODO: add additional data maybe (encryption, pubkey, etc) */ + rspamd_fuzzy_extensions_tolua(L, session); + + /* We push shingles merely for commands that modify content to avoid extra work */ + if (is_shingle && cmd->cmd != FUZZY_CHECK) { + lua_newshingle(L, &session->cmd.sgl); + nargs++; + } + + if ((ret = lua_pcall(L, nargs, LUA_MULTRET, err_idx)) != 0) { + msg_err("call to lua_pre_handler lua " + "script failed (%d): %s", + ret, lua_tostring(L, -1)); + } + else { + /* Return values order: + * the first reply will be on err_idx + 1 + * if it is true, then we need to read the former ones: + * 2-nd will be reply code + * 3-rd will be probability (or 0.0 if missing) + */ + ret = lua_toboolean(L, err_idx + 1); - if (lua_isnumber(L, err_idx + 3)) { - result.v1.prob = lua_tonumber(L, err_idx + 3); - } - else { - result.v1.prob = 0.0f; - } + if (ret) { + /* Artificial reply */ + result.v1.value = lua_tointeger(L, err_idx + 2); - lua_settop(L, 0); - rspamd_fuzzy_make_reply(cmd, &result, session, send_flags); + if (lua_isnumber(L, err_idx + 3)) { + result.v1.prob = lua_tonumber(L, err_idx + 3); + } + else { + result.v1.prob = 0.0f; + } - return; + lua_settop(L, 0); + rspamd_fuzzy_make_reply(cmd, &result, session, send_flags); + + return; + } } - } - lua_settop(L, 0); + lua_settop(L, 0); + } } @@ -1628,6 +1683,14 @@ rspamd_fuzzy_process_command(struct fuzzy_session *session) } } + /* Key is not allowed to read */ + if (session->key && !(session->key->flags & FUZZY_KEY_READ)) { + result.v1.value = 503; + result.v1.prob = 0.0f; + rspamd_fuzzy_make_reply(cmd, &result, session, send_flags); + return; + } + if (is_rate_allowed) { REF_RETAIN(session); rspamd_fuzzy_backend_check(session->ctx->backend, cmd, @@ -1655,7 +1718,7 @@ rspamd_fuzzy_process_command(struct fuzzy_session *session) rspamd_fuzzy_make_reply(cmd, &result, session, send_flags); } else { - if (rspamd_fuzzy_check_write(session)) { + if (rspamd_fuzzy_check_write(session, cmd->cmd)) { /* Check whitelist */ if (session->ctx->skip_hashes && cmd->cmd == FUZZY_WRITE) { rspamd_encode_hex_buf(cmd->digest, sizeof(cmd->digest), @@ -2712,14 +2775,12 @@ lua_fuzzy_add_pre_handler(lua_State *L) if (wrk && lua_isfunction(L, 2)) { ctx = (struct rspamd_fuzzy_storage_ctx *) wrk->ctx; + struct rspamd_lua_fuzzy_script *script; - if (ctx->lua_pre_handler_cbref != -1) { - /* Should not happen */ - luaL_unref(L, LUA_REGISTRYINDEX, ctx->lua_pre_handler_cbref); - } - + script = g_malloc0(sizeof(*script)); lua_pushvalue(L, 2); - ctx->lua_pre_handler_cbref = luaL_ref(L, LUA_REGISTRYINDEX); + script->cbref = luaL_ref(L, LUA_REGISTRYINDEX); + LL_APPEND(ctx->lua_pre_handlers, script); } else { return luaL_error(L, "invalid arguments, worker + function are expected"); @@ -2743,14 +2804,12 @@ lua_fuzzy_add_post_handler(lua_State *L) if (wrk && lua_isfunction(L, 2)) { ctx = (struct rspamd_fuzzy_storage_ctx *) wrk->ctx; + struct rspamd_lua_fuzzy_script *script; - if (ctx->lua_post_handler_cbref != -1) { - /* Should not happen */ - luaL_unref(L, LUA_REGISTRYINDEX, ctx->lua_post_handler_cbref); - } - + script = g_malloc0(sizeof(*script)); lua_pushvalue(L, 2); - ctx->lua_post_handler_cbref = luaL_ref(L, LUA_REGISTRYINDEX); + script->cbref = luaL_ref(L, LUA_REGISTRYINDEX); + LL_APPEND(ctx->lua_post_handlers, script); } else { return luaL_error(L, "invalid arguments, worker + function are expected"); @@ -2773,15 +2832,12 @@ lua_fuzzy_add_blacklist_handler(lua_State *L) wrk = *pwrk; if (wrk && lua_isfunction(L, 2)) { - ctx = (struct rspamd_fuzzy_storage_ctx *) wrk->ctx; - - if (ctx->lua_blacklist_cbref != -1) { - /* Should not happen */ - luaL_unref(L, LUA_REGISTRYINDEX, ctx->lua_blacklist_cbref); - } + struct rspamd_lua_fuzzy_script *script; + script = g_malloc0(sizeof(*script)); lua_pushvalue(L, 2); - ctx->lua_blacklist_cbref = luaL_ref(L, LUA_REGISTRYINDEX); + script->cbref = luaL_ref(L, LUA_REGISTRYINDEX); + LL_APPEND(ctx->lua_blacklist_handlers, script); } else { return luaL_error(L, "invalid arguments, worker + function are expected"); @@ -2942,6 +2998,8 @@ fuzzy_add_keypair_from_ucl(struct rspamd_config *cfg, const ucl_object_t *obj, key->rate = NAN; key->expire = NAN; key->rl_bucket = NULL; + /* Allow read by default */ + key->flags = FUZZY_KEY_READ; /* Preallocate some space for flags */ kh_resize(fuzzy_key_flag_stat, key->flags_stat, 8); const unsigned char *pk = rspamd_keypair_component(kp, RSPAMD_KEYPAIR_COMPONENT_PK, @@ -2973,6 +3031,7 @@ fuzzy_add_keypair_from_ucl(struct rspamd_config *cfg, const ucl_object_t *obj, const ucl_object_t *extensions = rspamd_keypair_get_extensions(kp); if (extensions) { + key->extensions = ucl_object_ref(extensions); lua_State *L = RSPAMD_LUA_CFG_STATE(cfg); const ucl_object_t *forbidden_ids = ucl_object_lookup(extensions, "forbidden_ids"); @@ -3052,9 +3111,48 @@ fuzzy_add_keypair_from_ucl(struct rspamd_config *cfg, const ucl_object_t *obj, if (name && ucl_object_type(name) == UCL_STRING) { key->name = g_strdup(ucl_object_tostring(name)); } + + /* Check permissions */ + const ucl_object_t *read_only = ucl_object_lookup(extensions, "read_only"); + if (read_only && ucl_object_type(read_only) == UCL_BOOLEAN) { + if (ucl_object_toboolean(read_only)) { + key->flags &= ~(FUZZY_KEY_WRITE | FUZZY_KEY_DELETE); + } + else { + key->flags |= (FUZZY_KEY_WRITE | FUZZY_KEY_DELETE); + } + } + + const ucl_object_t *allowed_ops = ucl_object_lookup(extensions, "allowed_ops"); + if (allowed_ops && ucl_object_type(allowed_ops) == UCL_ARRAY) { + const ucl_object_t *cur; + ucl_object_iter_t it = NULL; + /* Reset to only allowed */ + key->flags = 0; + + while ((cur = ucl_object_iterate(allowed_ops, &it, true)) != NULL) { + if (ucl_object_type(cur) == UCL_STRING) { + const char *op = ucl_object_tostring(cur); + + if (g_ascii_strcasecmp(op, "read") == 0) { + key->flags |= FUZZY_KEY_READ; + } + else if (g_ascii_strcasecmp(op, "write") == 0) { + key->flags |= FUZZY_KEY_WRITE; + } + else if (g_ascii_strcasecmp(op, "delete") == 0) { + key->flags |= FUZZY_KEY_DELETE; + } + else { + msg_warn_config("invalid operation: %s", op); + } + } + } + } } - msg_debug("loaded keypair %*bs; expire=%f; rate=%f; burst=%f; name=%s", (int) crypto_box_publickeybytes(), pk, + msg_debug("loaded keypair %*bs; expire=%f; rate=%f; burst=%f; name=%s", + (int) crypto_box_publickeybytes(), pk, key->expire, key->rate, key->burst, key->name); return key; @@ -3122,9 +3220,6 @@ init_fuzzy(struct rspamd_config *cfg) ctx->magic = rspamd_fuzzy_storage_magic; ctx->sync_timeout = DEFAULT_SYNC_TIMEOUT; ctx->keypair_cache_size = DEFAULT_KEYPAIR_CACHE_SIZE; - ctx->lua_pre_handler_cbref = -1; - ctx->lua_post_handler_cbref = -1; - ctx->lua_blacklist_cbref = -1; ctx->keys = kh_init(rspamd_fuzzy_keys_hash); rspamd_mempool_add_destructor(cfg->cfg_pool, (rspamd_mempool_destruct_t) fuzzy_hash_table_dtor, ctx->keys); @@ -3677,12 +3772,12 @@ start_fuzzy(struct rspamd_worker *worker) .func = lua_fuzzy_add_pre_handler, }; rspamd_lua_add_metamethod(ctx->cfg->lua_state, rspamd_worker_classname, &fuzzy_lua_reg); - fuzzy_lua_reg = (luaL_Reg){ + fuzzy_lua_reg = (luaL_Reg) { .name = "add_fuzzy_post_handler", .func = lua_fuzzy_add_post_handler, }; rspamd_lua_add_metamethod(ctx->cfg->lua_state, rspamd_worker_classname, &fuzzy_lua_reg); - fuzzy_lua_reg = (luaL_Reg){ + fuzzy_lua_reg = (luaL_Reg) { .name = "add_fuzzy_blacklist_handler", .func = lua_fuzzy_add_blacklist_handler, }; @@ -3735,16 +3830,22 @@ start_fuzzy(struct rspamd_worker *worker) rspamd_lru_hash_destroy(ctx->ratelimit_buckets); } - if (ctx->lua_pre_handler_cbref != -1) { - luaL_unref(ctx->cfg->lua_state, LUA_REGISTRYINDEX, ctx->lua_pre_handler_cbref); - } + struct rspamd_lua_fuzzy_script *cur, *tmp; - if (ctx->lua_post_handler_cbref != -1) { - luaL_unref(ctx->cfg->lua_state, LUA_REGISTRYINDEX, ctx->lua_post_handler_cbref); + LL_FOREACH_SAFE(ctx->lua_pre_handlers, cur, tmp) + { + luaL_unref(ctx->cfg->lua_state, LUA_REGISTRYINDEX, cur->cbref); + g_free(cur); } - - if (ctx->lua_blacklist_cbref != -1) { - luaL_unref(ctx->cfg->lua_state, LUA_REGISTRYINDEX, ctx->lua_blacklist_cbref); + LL_FOREACH_SAFE(ctx->lua_post_handlers, cur, tmp) + { + luaL_unref(ctx->cfg->lua_state, LUA_REGISTRYINDEX, cur->cbref); + g_free(cur); + } + LL_FOREACH_SAFE(ctx->lua_blacklist_handlers, cur, tmp) + { + luaL_unref(ctx->cfg->lua_state, LUA_REGISTRYINDEX, cur->cbref); + g_free(cur); } if (ctx->default_forbidden_ids) { diff --git a/src/libserver/CMakeLists.txt b/src/libserver/CMakeLists.txt index dd17865de..d3415bdb2 100644 --- a/src/libserver/CMakeLists.txt +++ b/src/libserver/CMakeLists.txt @@ -12,6 +12,7 @@ SET(LIBRSPAMDSERVERSRC ${CMAKE_CURRENT_SOURCE_DIR}/fuzzy_backend/fuzzy_backend.c ${CMAKE_CURRENT_SOURCE_DIR}/fuzzy_backend/fuzzy_backend_sqlite.c ${CMAKE_CURRENT_SOURCE_DIR}/fuzzy_backend/fuzzy_backend_redis.c + ${CMAKE_CURRENT_SOURCE_DIR}/fuzzy_backend/fuzzy_backend_noop.c ${CMAKE_CURRENT_SOURCE_DIR}/milter.c ${CMAKE_CURRENT_SOURCE_DIR}/monitored.c ${CMAKE_CURRENT_SOURCE_DIR}/protocol.c diff --git a/src/libserver/fuzzy_backend/fuzzy_backend.c b/src/libserver/fuzzy_backend/fuzzy_backend.c index c18463618..bab2895cd 100644 --- a/src/libserver/fuzzy_backend/fuzzy_backend.c +++ b/src/libserver/fuzzy_backend/fuzzy_backend.c @@ -1,11 +1,11 @@ -/*- - * Copyright 2016 Vsevolod Stakhov +/* + * Copyright 2025 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,6 +18,7 @@ #include "fuzzy_backend.h" #include "fuzzy_backend_sqlite.h" #include "fuzzy_backend_redis.h" +#include "fuzzy_backend_noop.h" #include "cfg_file.h" #include "fuzzy_wire.h" @@ -26,6 +27,7 @@ enum rspamd_fuzzy_backend_type { RSPAMD_FUZZY_BACKEND_SQLITE = 0, RSPAMD_FUZZY_BACKEND_REDIS = 1, + RSPAMD_FUZZY_BACKEND_NOOP = 2, }; static void *rspamd_fuzzy_backend_init_sqlite(struct rspamd_fuzzy_backend *bk, @@ -96,6 +98,16 @@ static const struct rspamd_fuzzy_backend_subr fuzzy_subrs[] = { .id = rspamd_fuzzy_backend_id_redis, .periodic = rspamd_fuzzy_backend_expire_redis, .close = rspamd_fuzzy_backend_close_redis, + }, + [RSPAMD_FUZZY_BACKEND_NOOP] = { + .init = rspamd_fuzzy_backend_init_noop, + .check = rspamd_fuzzy_backend_check_noop, + .update = rspamd_fuzzy_backend_update_noop, + .count = rspamd_fuzzy_backend_count_noop, + .version = rspamd_fuzzy_backend_version_noop, + .id = rspamd_fuzzy_backend_id_noop, + .periodic = rspamd_fuzzy_backend_expire_noop, + .close = rspamd_fuzzy_backend_close_noop, }}; struct rspamd_fuzzy_backend { @@ -288,6 +300,9 @@ rspamd_fuzzy_backend_create(struct ev_loop *ev_base, else if (strcmp(ucl_object_tostring(elt), "redis") == 0) { type = RSPAMD_FUZZY_BACKEND_REDIS; } + else if (strcmp(ucl_object_tostring(elt), "noop") == 0) { + type = RSPAMD_FUZZY_BACKEND_NOOP; + } else { g_set_error(err, rspamd_fuzzy_backend_quark(), EINVAL, "invalid backend type: %s", diff --git a/src/libserver/fuzzy_backend/fuzzy_backend_noop.c b/src/libserver/fuzzy_backend/fuzzy_backend_noop.c new file mode 100644 index 000000000..451a1921b --- /dev/null +++ b/src/libserver/fuzzy_backend/fuzzy_backend_noop.c @@ -0,0 +1,96 @@ +/* + * Copyright 2025 Vsevolod Stakhov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include "fuzzy_backend_noop.h" + +/* + * No operations backend (useful for scripts only stuff) + */ + +void *rspamd_fuzzy_backend_init_noop(struct rspamd_fuzzy_backend *bk, + const ucl_object_t *obj, + struct rspamd_config *cfg, + GError **err) +{ + return NULL; +} + +void rspamd_fuzzy_backend_check_noop(struct rspamd_fuzzy_backend *bk, + const struct rspamd_fuzzy_cmd *cmd, + rspamd_fuzzy_check_cb cb, void *ud, + void *subr_ud) +{ + struct rspamd_fuzzy_reply rep; + + if (cb) { + memset(&rep, 0, sizeof(rep)); + cb(&rep, ud); + } + + return; +} + +void rspamd_fuzzy_backend_update_noop(struct rspamd_fuzzy_backend *bk, + GArray *updates, const char *src, + rspamd_fuzzy_update_cb cb, void *ud, + void *subr_ud) +{ + if (cb) { + cb(FALSE, 0, 0, 0, 0, ud); + } + + return; +} + +void rspamd_fuzzy_backend_count_noop(struct rspamd_fuzzy_backend *bk, + rspamd_fuzzy_count_cb cb, void *ud, + void *subr_ud) +{ + if (cb) { + cb(0, ud); + } + + return; +} + +void rspamd_fuzzy_backend_version_noop(struct rspamd_fuzzy_backend *bk, + const char *src, + rspamd_fuzzy_version_cb cb, void *ud, + void *subr_ud) +{ + if (cb) { + cb(0, ud); + } + + return; +} + +const char *rspamd_fuzzy_backend_id_noop(struct rspamd_fuzzy_backend *bk, + void *subr_ud) +{ + return NULL; +} + +void rspamd_fuzzy_backend_expire_noop(struct rspamd_fuzzy_backend *bk, + void *subr_ud) +{ +} + +void rspamd_fuzzy_backend_close_noop(struct rspamd_fuzzy_backend *bk, + void *subr_ud) +{ +} diff --git a/src/libserver/fuzzy_backend/fuzzy_backend_noop.h b/src/libserver/fuzzy_backend/fuzzy_backend_noop.h new file mode 100644 index 000000000..ac063dc39 --- /dev/null +++ b/src/libserver/fuzzy_backend/fuzzy_backend_noop.h @@ -0,0 +1,66 @@ +/* + * Copyright 2025 Vsevolod Stakhov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef FUZZY_BACKEND_NOOP_H +#define FUZZY_BACKEND_NOOP_H + +#include "config.h" +#include "fuzzy_backend.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Subroutines for fuzzy_backend + */ +void *rspamd_fuzzy_backend_init_noop(struct rspamd_fuzzy_backend *bk, + const ucl_object_t *obj, + struct rspamd_config *cfg, + GError **err); + +void rspamd_fuzzy_backend_check_noop(struct rspamd_fuzzy_backend *bk, + const struct rspamd_fuzzy_cmd *cmd, + rspamd_fuzzy_check_cb cb, void *ud, + void *subr_ud); + +void rspamd_fuzzy_backend_update_noop(struct rspamd_fuzzy_backend *bk, + GArray *updates, const char *src, + rspamd_fuzzy_update_cb cb, void *ud, + void *subr_ud); + +void rspamd_fuzzy_backend_count_noop(struct rspamd_fuzzy_backend *bk, + rspamd_fuzzy_count_cb cb, void *ud, + void *subr_ud); + +void rspamd_fuzzy_backend_version_noop(struct rspamd_fuzzy_backend *bk, + const char *src, + rspamd_fuzzy_version_cb cb, void *ud, + void *subr_ud); + +const char *rspamd_fuzzy_backend_id_noop(struct rspamd_fuzzy_backend *bk, + void *subr_ud); + +void rspamd_fuzzy_backend_expire_noop(struct rspamd_fuzzy_backend *bk, + void *subr_ud); + +void rspamd_fuzzy_backend_close_noop(struct rspamd_fuzzy_backend *bk, + void *subr_ud); + +#ifdef __cplusplus +} +#endif + +#endif//FUZZY_BACKEND_NOOP_H diff --git a/src/libserver/fuzzy_backend/fuzzy_backend_redis.h b/src/libserver/fuzzy_backend/fuzzy_backend_redis.h index afeb1c573..0a536c2fa 100644 --- a/src/libserver/fuzzy_backend/fuzzy_backend_redis.h +++ b/src/libserver/fuzzy_backend/fuzzy_backend_redis.h @@ -1,11 +1,11 @@ -/*- - * Copyright 2016 Vsevolod Stakhov +/* + * Copyright 2025 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,7 +15,6 @@ */ #ifndef SRC_LIBSERVER_FUZZY_BACKEND_REDIS_H_ #define SRC_LIBSERVER_FUZZY_BACKEND_REDIS_H_ - #include "config.h" #include "fuzzy_backend.h" diff --git a/src/libserver/logger/logger.c b/src/libserver/logger/logger.c index 25818e7a5..dc0a85a05 100644 --- a/src/libserver/logger/logger.c +++ b/src/libserver/logger/logger.c @@ -1,5 +1,5 @@ /* - * Copyright 2024 Vsevolod Stakhov + * Copyright 2025 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -1026,6 +1026,18 @@ log_time(double now, rspamd_logger_t *rspamd_log, char *timebuf, } } +static inline int +rspamd_log_id_strlen(const char *id) +{ + for (int i = 0; i < RSPAMD_LOG_ID_LEN; i++) { + if (G_UNLIKELY(id[i] == '\0')) { + return i; + } + } + + return RSPAMD_LOG_ID_LEN; +} + void rspamd_log_fill_iov(struct rspamd_logger_iov_ctx *iov_ctx, double ts, const char *module, @@ -1235,9 +1247,7 @@ void rspamd_log_fill_iov(struct rspamd_logger_iov_ctx *iov_ctx, m = modulebuf; if (id != NULL) { - unsigned int slen = strlen(id); - slen = MIN(RSPAMD_LOG_ID_LEN, slen); - mr = rspamd_snprintf(m, mremain, "<%*.s>; ", slen, + mr = rspamd_snprintf(m, mremain, "<%*.s>; ", rspamd_log_id_strlen(id), id); m += mr; mremain -= mr; @@ -1289,6 +1299,14 @@ void rspamd_log_fill_iov(struct rspamd_logger_iov_ctx *iov_ctx, if (logger->log_level == G_LOG_LEVEL_DEBUG) { iov_ctx->iov[niov].iov_base = (void *) timebuf; iov_ctx->iov[niov++].iov_len = strlen(timebuf); + if (id != NULL) { + iov_ctx->iov[niov].iov_base = (void *) "; "; + iov_ctx->iov[niov++].iov_len = 2; + iov_ctx->iov[niov].iov_base = (void *) id; + iov_ctx->iov[niov++].iov_len = rspamd_log_id_strlen(id); + iov_ctx->iov[niov].iov_base = (void *) ";"; + iov_ctx->iov[niov++].iov_len = 1; + } iov_ctx->iov[niov].iov_base = (void *) " "; iov_ctx->iov[niov++].iov_len = 1; } diff --git a/src/libserver/redis_pool.cxx b/src/libserver/redis_pool.cxx index cea8d0c86..586260a6f 100644 --- a/src/libserver/redis_pool.cxx +++ b/src/libserver/redis_pool.cxx @@ -1,5 +1,5 @@ /* - * Copyright 2023 Vsevolod Stakhov + * Copyright 2025 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -465,6 +465,8 @@ auto redis_pool_elt::new_connection() -> redisAsyncContext * * We cannot reuse connection, so we just recursively call * this function one more time */ + msg_debug_rpool("cannot reuse the existing connection to %s:%d: %p; errno=%d", + ip.c_str(), port, conn->ctx, err); return new_connection(); } else { @@ -481,6 +483,9 @@ auto redis_pool_elt::new_connection() -> redisAsyncContext * } else { auto *nctx = redis_async_new(); + msg_debug_rpool("error in the inactive connection: %s; opened new connection to %s:%d: %p", + conn->ctx->errstr, ip.c_str(), port, nctx); + if (nctx) { active.emplace_front(std::make_unique<redis_pool_connection>(pool, this, db.c_str(), username.c_str(), password.c_str(), nctx)); @@ -492,10 +497,14 @@ auto redis_pool_elt::new_connection() -> redisAsyncContext * } else { auto *nctx = redis_async_new(); + if (nctx) { active.emplace_front(std::make_unique<redis_pool_connection>(pool, this, db.c_str(), username.c_str(), password.c_str(), nctx)); active.front()->elt_pos = active.begin(); + auto conn = active.front().get(); + msg_debug_rpool("no inactive connections; opened new connection to %s:%d: %p", + ip.c_str(), port, nctx); } return nctx; diff --git a/src/lua/CMakeLists.txt b/src/lua/CMakeLists.txt index 46de053ba..135a21da2 100644 --- a/src/lua/CMakeLists.txt +++ b/src/lua/CMakeLists.txt @@ -35,6 +35,7 @@ SET(LUASRC ${CMAKE_CURRENT_SOURCE_DIR}/lua_common.c ${CMAKE_CURRENT_SOURCE_DIR}/lua_tensor.c ${CMAKE_CURRENT_SOURCE_DIR}/lua_parsers.c ${CMAKE_CURRENT_SOURCE_DIR}/lua_compress.c - ${CMAKE_CURRENT_SOURCE_DIR}/lua_classnames.c) + ${CMAKE_CURRENT_SOURCE_DIR}/lua_classnames.c + ${CMAKE_CURRENT_SOURCE_DIR}/lua_shingles.cxx) SET(RSPAMD_LUA ${LUASRC} PARENT_SCOPE)
\ No newline at end of file diff --git a/src/lua/lua_classnames.c b/src/lua/lua_classnames.c index 7ce2f8abc..2b5a90fe0 100644 --- a/src/lua/lua_classnames.c +++ b/src/lua/lua_classnames.c @@ -1,5 +1,5 @@ /* - * Copyright 2024 Vsevolod Stakhov + * Copyright 2025 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -67,6 +67,7 @@ const char *rspamd_url_classname = "rspamd{url}"; const char *rspamd_worker_classname = "rspamd{worker}"; const char *rspamd_zstd_compress_classname = "rspamd{zstd_compress}"; const char *rspamd_zstd_decompress_classname = "rspamd{zstd_decompress}"; +const char *rspamd_shingle_classname = "rspamd{shingle}"; KHASH_INIT(rspamd_lua_static_classes, const char *, const char *, 1, rspamd_str_hash, rspamd_str_equal); @@ -133,6 +134,7 @@ RSPAMD_CONSTRUCTOR(rspamd_lua_init_classnames) CLASS_PUT_STR(worker); CLASS_PUT_STR(zstd_compress); CLASS_PUT_STR(zstd_decompress); + CLASS_PUT_STR(shingle); /* Check consistency */ g_assert(kh_size(lua_static_classes) == RSPAMD_MAX_LUA_CLASSES); diff --git a/src/lua/lua_classnames.h b/src/lua/lua_classnames.h index 53db5f8c2..6e3a6441f 100644 --- a/src/lua/lua_classnames.h +++ b/src/lua/lua_classnames.h @@ -1,5 +1,5 @@ /* - * Copyright 2024 Vsevolod Stakhov + * Copyright 2025 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -70,9 +70,10 @@ extern const char *rspamd_url_classname; extern const char *rspamd_worker_classname; extern const char *rspamd_zstd_compress_classname; extern const char *rspamd_zstd_decompress_classname; +extern const char *rspamd_shingle_classname; /* Keep it consistent when adding new classes */ -#define RSPAMD_MAX_LUA_CLASSES 48 +#define RSPAMD_MAX_LUA_CLASSES 49 /* * Return a static class name for a given name (only for known classes) or NULL diff --git a/src/lua/lua_common.c b/src/lua/lua_common.c index d79efc308..3a0f1a06c 100644 --- a/src/lua/lua_common.c +++ b/src/lua/lua_common.c @@ -1,5 +1,5 @@ /* - * Copyright 2024 Vsevolod Stakhov + * Copyright 2025 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -985,6 +985,7 @@ rspamd_lua_init(bool wipe_mem) luaopen_tensor(L); luaopen_parsers(L); luaopen_compress(L); + luaopen_shingle(L); #ifndef WITH_LUAJIT rspamd_lua_add_preload(L, "bit", luaopen_bit); lua_settop(L, 0); diff --git a/src/lua/lua_common.h b/src/lua/lua_common.h index 1d39d0c52..f5a4967ba 100644 --- a/src/lua/lua_common.h +++ b/src/lua/lua_common.h @@ -1,5 +1,5 @@ /* - * Copyright 2024 Vsevolod Stakhov + * Copyright 2025 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -421,6 +421,8 @@ void luaopen_tensor(lua_State *L); void luaopen_parsers(lua_State *L); +void luaopen_shingle(lua_State *L); + void rspamd_lua_dostring(const char *line); double rspamd_lua_normalize(struct rspamd_config *cfg, @@ -454,6 +456,12 @@ struct rspamd_dns_resolver *lua_check_dns_resolver(lua_State *L, int pos); struct rspamd_lua_url *lua_check_url(lua_State *L, int pos); +/** + * Creates a new shingle object from the existing shingle + */ +struct rspamd_shingle; +void lua_newshingle(lua_State *L, const struct rspamd_shingle *sh); + enum rspamd_lua_parse_arguments_flags { RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT = 0, RSPAMD_LUA_PARSE_ARGUMENTS_IGNORE_MISSING, diff --git a/src/lua/lua_config.c b/src/lua/lua_config.c index 0b4d208b4..07ed58ad5 100644 --- a/src/lua/lua_config.c +++ b/src/lua/lua_config.c @@ -73,7 +73,7 @@ LUA_FUNCTION_DEF(config, get_ucl); /*** * @method rspamd_config:get_mempool() * Returns static configuration memory pool. - * @return {mempool} [memory pool](mempool.md) object + * @return {mempool} [memory pool](rspamd_mempool.md) object */ LUA_FUNCTION_DEF(config, get_mempool); /*** diff --git a/src/lua/lua_http.c b/src/lua/lua_http.c index 904f1cbbf..7e9e7b1df 100644 --- a/src/lua/lua_http.c +++ b/src/lua/lua_http.c @@ -600,7 +600,7 @@ lua_http_push_headers(lua_State *L, struct rspamd_http_message *msg) * - `config` * * @param {string} url specifies URL for a request in the standard URI form (e.g. 'http://example.com/path') - * @param {function} callback specifies callback function in format `function (err_message, code, body, headers)` that is called on HTTP request completion. if this parameter is missing, the function performs "pseudo-synchronous" call (see [Synchronous and Asynchronous API overview](/doc/lua/sync_async.html#API-example-http-module) + * @param {function} callback specifies callback function in format `function (err_message, code, body, headers)` that is called on HTTP request completion. if this parameter is missing, the function performs "pseudo-synchronous" call (see [Synchronous and Asynchronous API overview](/doc/developers/sync_async.html#API-example-http-module) * @param {task} task if called from symbol handler it is generally a good idea to use the common task objects: event base, DNS resolver and events session * @param {table} headers optional headers in form `[name='value', name='value']` * @param {string} mime_type MIME type of the HTTP content (for example, `text/html`) diff --git a/src/lua/lua_redis.c b/src/lua/lua_redis.c index d20c496ed..491007df3 100644 --- a/src/lua/lua_redis.c +++ b/src/lua/lua_redis.c @@ -1,5 +1,5 @@ /* - * Copyright 2024 Vsevolod Stakhov + * Copyright 2025 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -130,7 +130,7 @@ struct lua_redis_request_specific_userdata { unsigned int nargs; char **args; gsize *arglens; - struct lua_redis_userdata *c; + struct lua_redis_userdata *common_ud; struct lua_redis_ctx *ctx; struct lua_redis_request_specific_userdata *next; ev_timer timeout_ev; @@ -262,7 +262,7 @@ lua_redis_fin(void *arg) struct lua_redis_ctx *ctx; ctx = sp_ud->ctx; - ud = sp_ud->c; + ud = sp_ud->common_ud; if (ev_can_stop(&sp_ud->timeout_ev)) { ev_timer_stop(sp_ud->ctx->async.event_loop, &sp_ud->timeout_ev); @@ -290,7 +290,7 @@ lua_redis_push_error(const char *err, gboolean connected, ...) { - struct lua_redis_userdata *ud = sp_ud->c; + struct lua_redis_userdata *ud = sp_ud->common_ud; struct lua_callback_state cbs; lua_State *L; @@ -390,7 +390,7 @@ static void lua_redis_push_data(const redisReply *r, struct lua_redis_ctx *ctx, struct lua_redis_request_specific_userdata *sp_ud) { - struct lua_redis_userdata *ud = sp_ud->c; + struct lua_redis_userdata *ud = sp_ud->common_ud; struct lua_callback_state cbs; lua_State *L; @@ -467,14 +467,14 @@ lua_redis_callback(redisAsyncContext *c, gpointer r, gpointer priv) redisAsyncContext *ac; ctx = sp_ud->ctx; - ud = sp_ud->c; + ud = sp_ud->common_ud; if (ud->terminated || !rspamd_lua_is_initialised()) { /* We are already at the termination stage, just go out */ return; } - msg_debug_lua_redis("got reply from redis %p for query %p", sp_ud->c->ctx, + msg_debug_lua_redis("got async reply from redis %p for query %p", sp_ud->common_ud->ctx, sp_ud); REDIS_RETAIN(ctx); @@ -601,7 +601,7 @@ lua_redis_callback_sync(redisAsyncContext *ac, gpointer r, gpointer priv) int results; ctx = sp_ud->ctx; - ud = sp_ud->c; + ud = sp_ud->common_ud; lua_State *L = ctx->async.cfg->lua_state; sp_ud->flags |= LUA_REDIS_SPECIFIC_REPLIED; @@ -620,7 +620,7 @@ lua_redis_callback_sync(redisAsyncContext *ac, gpointer r, gpointer priv) } if (!(sp_ud->flags & LUA_REDIS_SPECIFIC_FINISHED)) { - msg_debug_lua_redis("got reply from redis: %p for query %p", ac, sp_ud); + msg_debug_lua_redis("got sync reply from redis: %p for query %p", ac, sp_ud); struct lua_redis_result *result = g_malloc0(sizeof *result); @@ -653,17 +653,17 @@ lua_redis_callback_sync(redisAsyncContext *ac, gpointer r, gpointer priv) /* if error happened, we should terminate the connection, and release it */ - if (result->is_error && sp_ud->c->ctx) { - ac = sp_ud->c->ctx; + if (result->is_error && sp_ud->common_ud->ctx) { + ac = sp_ud->common_ud->ctx; /* Set to NULL to avoid double free in dtor */ - sp_ud->c->ctx = NULL; + sp_ud->common_ud->ctx = NULL; ctx->flags |= LUA_REDIS_TERMINATED; /* * This will call all callbacks pending so the entire context * will be destructed */ - rspamd_redis_pool_release_connection(sp_ud->c->pool, ac, + rspamd_redis_pool_release_connection(sp_ud->common_ud->pool, ac, RSPAMD_REDIS_RELEASE_FATAL); } @@ -679,6 +679,8 @@ lua_redis_callback_sync(redisAsyncContext *ac, gpointer r, gpointer priv) ctx->cmds_pending--; if (ctx->cmds_pending == 0) { + msg_debug_lua_redis("no more commands left for: %p for query %p", ac, sp_ud); + if (ctx->thread) { if (!(sp_ud->flags & LUA_REDIS_SPECIFIC_FINISHED)) { /* somebody yielded and waits for results */ @@ -717,16 +719,16 @@ lua_redis_timeout_sync(EV_P_ ev_timer *w, int revents) return; } - ud = sp_ud->c; + ud = sp_ud->common_ud; ctx = sp_ud->ctx; msg_debug_lua_redis("timeout while querying redis server: %p, redis: %p", sp_ud, - sp_ud->c->ctx); + sp_ud->common_ud->ctx); - if (sp_ud->c->ctx) { - ac = sp_ud->c->ctx; + if (sp_ud->common_ud->ctx) { + ac = sp_ud->common_ud->ctx; /* Set to NULL to avoid double free in dtor */ - sp_ud->c->ctx = NULL; + sp_ud->common_ud->ctx = NULL; ac->err = REDIS_ERR_IO; errno = ETIMEDOUT; ctx->flags |= LUA_REDIS_TERMINATED; @@ -735,7 +737,7 @@ lua_redis_timeout_sync(EV_P_ ev_timer *w, int revents) * This will call all callbacks pending so the entire context * will be destructed */ - rspamd_redis_pool_release_connection(sp_ud->c->pool, ac, + rspamd_redis_pool_release_connection(sp_ud->common_ud->pool, ac, RSPAMD_REDIS_RELEASE_FATAL); } } @@ -754,24 +756,24 @@ lua_redis_timeout(EV_P_ ev_timer *w, int revents) } ctx = sp_ud->ctx; - ud = sp_ud->c; + ud = sp_ud->common_ud; REDIS_RETAIN(ctx); msg_debug_lua_redis("timeout while querying redis server: %p, redis: %p", sp_ud, - sp_ud->c->ctx); + sp_ud->common_ud->ctx); lua_redis_push_error("timeout while connecting the server (%.2f sec)", ctx, sp_ud, TRUE, ud->timeout); - if (sp_ud->c->ctx) { - ac = sp_ud->c->ctx; + if (sp_ud->common_ud->ctx) { + ac = sp_ud->common_ud->ctx; /* Set to NULL to avoid double free in dtor */ - sp_ud->c->ctx = NULL; + sp_ud->common_ud->ctx = NULL; ac->err = REDIS_ERR_IO; errno = ETIMEDOUT; /* * This will call all callbacks pending so the entire context * will be destructed */ - rspamd_redis_pool_release_connection(sp_ud->c->pool, ac, + rspamd_redis_pool_release_connection(sp_ud->common_ud->pool, ac, RSPAMD_REDIS_RELEASE_FATAL); } @@ -1095,8 +1097,8 @@ rspamd_lua_redis_prepare_connection(lua_State *L, int *pcbref, gboolean is_async return NULL; } - msg_debug_lua_redis("opened redis connection host=%s; ctx=%p; ud=%p", - host, ctx, ud); + msg_debug_lua_redis("opened redis connection host=%s; lua_ctx=%p; redis_ctx=%p; ud=%p", + host, ctx, ud->ctx, ud); return ctx; } @@ -1137,7 +1139,7 @@ lua_redis_make_request(lua_State *L) ud = &ctx->async; sp_ud = g_malloc0(sizeof(*sp_ud)); sp_ud->cbref = cbref; - sp_ud->c = ud; + sp_ud->common_ud = ud; sp_ud->ctx = ctx; lua_pushstring(L, "cmd"); @@ -1501,21 +1503,18 @@ lua_redis_add_cmd(lua_State *L) } sp_ud = g_malloc0(sizeof(*sp_ud)); + sp_ud->common_ud = &ctx->async; + ud = &ctx->async; if (IS_ASYNC(ctx)) { - sp_ud->c = &ctx->async; - ud = &ctx->async; sp_ud->cbref = cbref; } - else { - sp_ud->c = &ctx->async; - ud = &ctx->async; - } + sp_ud->ctx = ctx; lua_redis_parse_args(L, args_pos, cmd, &sp_ud->args, &sp_ud->arglens, &sp_ud->nargs); - LL_PREPEND(sp_ud->c->specific, sp_ud); + LL_PREPEND(sp_ud->common_ud->specific, sp_ud); if (ud->s && rspamd_session_blocked(ud->s)) { lua_pushboolean(L, 0); @@ -1525,7 +1524,7 @@ lua_redis_add_cmd(lua_State *L) } if (IS_ASYNC(ctx)) { - ret = redisAsyncCommandArgv(sp_ud->c->ctx, + ret = redisAsyncCommandArgv(sp_ud->common_ud->ctx, lua_redis_callback, sp_ud, sp_ud->nargs, @@ -1533,7 +1532,7 @@ lua_redis_add_cmd(lua_State *L) sp_ud->arglens); } else { - ret = redisAsyncCommandArgv(sp_ud->c->ctx, + ret = redisAsyncCommandArgv(sp_ud->common_ud->ctx, lua_redis_callback_sync, sp_ud, sp_ud->nargs, @@ -1554,25 +1553,28 @@ lua_redis_add_cmd(lua_State *L) } sp_ud->timeout_ev.data = sp_ud; + ev_now_update_if_cheap(ud->event_loop); if (IS_ASYNC(ctx)) { ev_timer_init(&sp_ud->timeout_ev, lua_redis_timeout, - sp_ud->c->timeout, 0.0); + sp_ud->common_ud->timeout, 0.0); } else { ev_timer_init(&sp_ud->timeout_ev, lua_redis_timeout_sync, - sp_ud->c->timeout, 0.0); + sp_ud->common_ud->timeout, 0.0); } ev_timer_start(ud->event_loop, &sp_ud->timeout_ev); + msg_debug_lua_redis("added timeout %f for %p", sp_ud->common_ud->timeout, sp_ud); + REDIS_RETAIN(ctx); ctx->cmds_pending++; } else { msg_info("call to redis failed: %s", - sp_ud->c->ctx->errstr); + sp_ud->common_ud->ctx->errstr); lua_pushboolean(L, 0); - lua_pushstring(L, sp_ud->c->ctx->errstr); + lua_pushstring(L, sp_ud->common_ud->ctx->errstr); return 2; } @@ -1606,11 +1608,20 @@ lua_redis_exec(lua_State *L) return 0; } else { - if (ctx->cmds_pending == 0 && g_queue_get_length(ctx->replies) == 0) { + struct lua_redis_userdata *ud = &ctx->async; + int replies_pending = g_queue_get_length(ctx->replies); + + msg_debug_lua_redis("execute pending commands for %p; commands pending = %d; replies pending = %d", + ctx, + ctx->cmds_pending, + replies_pending); + + if (ctx->cmds_pending == 0 && replies_pending == 0) { lua_pushstring(L, "No pending commands to execute"); lua_error(L); } - if (ctx->cmds_pending == 0 && g_queue_get_length(ctx->replies) > 0) { + + if (ctx->cmds_pending == 0 && replies_pending > 0) { int results = lua_redis_push_results(ctx, L); return results; } diff --git a/src/lua/lua_shingles.cxx b/src/lua/lua_shingles.cxx new file mode 100644 index 000000000..8e14d8ba8 --- /dev/null +++ b/src/lua/lua_shingles.cxx @@ -0,0 +1,133 @@ +/* + * Copyright 2025 Vsevolod Stakhov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "lua_common.h" +#include "lua_classnames.h" +#include "shingles.h" +#include "fmt/format.h" + +/*** + * @module rspamd_shingle + * This module provides methods to work with text shingles + */ + +/*** + * @method shingle:to_table() + * Converts shingle to table of decimal strings + * @return {table} table of RSPAMD_SHINGLE_SIZE decimal strings + */ +LUA_FUNCTION_DEF(shingle, to_table); + +/*** + * @method shingle:get(index) + * Gets element at index as two lua_Integer values (high and low 32 bits) + * @param {number} index 1-based index + * @return {number,number} high and low 32-bit parts + */ +LUA_FUNCTION_DEF(shingle, get); + +/*** + * @method shingle:get_string(index) + * Gets element at index as decimal string + * @param {number} index 1-based index + * @return {string} decimal representation + */ +LUA_FUNCTION_DEF(shingle, get_string); + +static const struct luaL_reg shinglelib_m[] = { + LUA_INTERFACE_DEF(shingle, to_table), + LUA_INTERFACE_DEF(shingle, get), + LUA_INTERFACE_DEF(shingle, get_string), + {"__tostring", rspamd_lua_class_tostring}, + {nullptr, nullptr}}; + +static struct rspamd_shingle * +lua_check_shingle(lua_State *L, int pos) +{ + void *ud = rspamd_lua_check_udata(L, pos, rspamd_shingle_classname); + luaL_argcheck(L, ud != nullptr, pos, "'shingle' expected"); + return static_cast<struct rspamd_shingle *>(ud); +} + +void lua_newshingle(lua_State *L, const struct rspamd_shingle *sh) +{ + auto *nsh = static_cast<struct rspamd_shingle *>( + lua_newuserdata(L, sizeof(struct rspamd_shingle))); + + if (sh != nullptr) { + memcpy(nsh, sh, sizeof(struct rspamd_shingle)); + } + + rspamd_lua_setclass(L, rspamd_shingle_classname, -1); +} + +static int +lua_shingle_to_table(lua_State *L) +{ + LUA_TRACE_POINT; + auto *sh = lua_check_shingle(L, 1); + + lua_createtable(L, RSPAMD_SHINGLE_SIZE, 0); + + for (int i = 0; i < RSPAMD_SHINGLE_SIZE; i++) { + auto str = fmt::format("{}", sh->hashes[i]); + lua_pushstring(L, str.c_str()); + lua_rawseti(L, -2, i + 1); + } + + return 1; +} + +static int +lua_shingle_get(lua_State *L) +{ + LUA_TRACE_POINT; + auto *sh = lua_check_shingle(L, 1); + auto idx = luaL_checkinteger(L, 2) - 1; + + if (idx < 0 || idx >= RSPAMD_SHINGLE_SIZE) { + return luaL_error(L, "index out of bounds: %d", idx + 1); + } + + uint64_t val = sh->hashes[idx]; + lua_pushinteger(L, (lua_Integer) (val >> 32)); + lua_pushinteger(L, (lua_Integer) (val & 0xFFFFFFFF)); + + return 2; +} + +static int +lua_shingle_get_string(lua_State *L) +{ + LUA_TRACE_POINT; + auto *sh = lua_check_shingle(L, 1); + auto idx = luaL_checkinteger(L, 2) - 1; + + if (idx < 0 || idx >= RSPAMD_SHINGLE_SIZE) { + return luaL_error(L, "index out of bounds: %d", idx + 1); + } + + auto str = fmt::format("{}", sh->hashes[idx]); + lua_pushstring(L, str.c_str()); + + return 1; +} + +void luaopen_shingle(lua_State *L) +{ + rspamd_lua_new_class(L, rspamd_shingle_classname, shinglelib_m); + lua_pop(L, 1); +} diff --git a/src/lua/lua_task.c b/src/lua/lua_task.c index b368ad4e6..355680881 100644 --- a/src/lua/lua_task.c +++ b/src/lua/lua_task.c @@ -1233,6 +1233,8 @@ static const struct luaL_reg tasklib_f[] = { {NULL, NULL}}; static const struct luaL_reg tasklib_m[] = { + LUA_INTERFACE_DEF(task, load_from_file), + LUA_INTERFACE_DEF(task, load_from_string), LUA_INTERFACE_DEF(task, get_message), LUA_INTERFACE_DEF(task, set_message), LUA_INTERFACE_DEF(task, destroy), @@ -1724,20 +1726,32 @@ lua_task_load_from_file(lua_State *L) { LUA_TRACE_POINT; struct rspamd_task *task = NULL, **ptask; - const char *fname = luaL_checkstring(L, 1), *err = NULL; + const char *fname, *err = NULL; struct rspamd_config *cfg = NULL; - gboolean res = FALSE; + gboolean res = FALSE, new_task = FALSE; gpointer map; gsize sz; + if (lua_type(L, 1) == LUA_TSTRING) { + fname = luaL_checkstring(L, 1); + new_task = TRUE; + } + else { + /* Method */ + task = lua_check_task(L, 1); + fname = luaL_checkstring(L, 2); + } + if (fname) { - if (lua_type(L, 2) == LUA_TUSERDATA) { - gpointer p; - p = rspamd_lua_check_udata_maybe(L, 2, rspamd_config_classname); + if (!task) { + if (lua_type(L, 2) == LUA_TUSERDATA) { + gpointer p; + p = rspamd_lua_check_udata_maybe(L, 2, rspamd_config_classname); - if (p) { - cfg = *(struct rspamd_config **) p; + if (p) { + cfg = *(struct rspamd_config **) p; + } } } @@ -1763,11 +1777,17 @@ lua_task_load_from_file(lua_State *L) } } - task = rspamd_task_new(NULL, cfg, NULL, NULL, NULL, FALSE); + if (!task) { + task = rspamd_task_new(NULL, cfg, NULL, NULL, NULL, FALSE); + } + task->msg.begin = data->str; task->msg.len = data->len; rspamd_mempool_add_destructor(task->task_pool, lua_task_free_dtor, data->str); + if (data->len > 0) { + task->flags &= ~RSPAMD_TASK_FLAG_EMPTY; + } res = TRUE; g_string_free(data, FALSE); /* Buffer is still valid */ } @@ -1778,9 +1798,16 @@ lua_task_load_from_file(lua_State *L) err = strerror(errno); } else { - task = rspamd_task_new(NULL, cfg, NULL, NULL, NULL, FALSE); + if (!task) { + task = rspamd_task_new(NULL, cfg, NULL, NULL, NULL, FALSE); + } + task->msg.begin = map; task->msg.len = sz; + + if (sz > 0) { + task->flags &= ~RSPAMD_TASK_FLAG_EMPTY; + } rspamd_mempool_add_destructor(task->task_pool, lua_task_unmap_dtor, task); res = TRUE; @@ -1793,21 +1820,26 @@ lua_task_load_from_file(lua_State *L) lua_pushboolean(L, res); - if (res) { + if (res && new_task) { ptask = lua_newuserdata(L, sizeof(*ptask)); *ptask = task; rspamd_lua_setclass(L, rspamd_task_classname, -1); + + return 2; } - else { + else if (!res) { if (err) { lua_pushstring(L, err); } else { lua_pushnil(L); } + return 2; + } + else { + /* No new task */ + return 1; } - - return 2; } static int @@ -1816,14 +1848,23 @@ lua_task_load_from_string(lua_State *L) LUA_TRACE_POINT; struct rspamd_task *task = NULL, **ptask; const char *str_message; - gsize message_len; + gsize message_len = 0; struct rspamd_config *cfg = NULL; + bool new_task = false; - str_message = luaL_checklstring(L, 1, &message_len); + if (lua_type(L, 1) == LUA_TSTRING) { + str_message = luaL_checklstring(L, 1, &message_len); + new_task = true; + } + else { + /* Method */ + task = lua_check_task(L, 1); + str_message = luaL_checklstring(L, 2, &message_len); + } if (str_message) { - if (lua_type(L, 2) == LUA_TUSERDATA) { + if (!task && lua_type(L, 2) == LUA_TUSERDATA) { gpointer p; p = rspamd_lua_check_udata_maybe(L, 2, rspamd_config_classname); @@ -1832,10 +1873,15 @@ lua_task_load_from_string(lua_State *L) } } - task = rspamd_task_new(NULL, cfg, NULL, NULL, NULL, FALSE); + if (!task) { + task = rspamd_task_new(NULL, cfg, NULL, NULL, NULL, FALSE); + } task->msg.begin = g_malloc(message_len); memcpy((char *) task->msg.begin, str_message, message_len); task->msg.len = message_len; + if (message_len > 0) { + task->flags &= ~RSPAMD_TASK_FLAG_EMPTY; + } rspamd_mempool_add_destructor(task->task_pool, lua_task_free_dtor, (gpointer) task->msg.begin); } @@ -1845,11 +1891,16 @@ lua_task_load_from_string(lua_State *L) lua_pushboolean(L, true); - ptask = lua_newuserdata(L, sizeof(*ptask)); - *ptask = task; - rspamd_lua_setclass(L, rspamd_task_classname, -1); + if (new_task) { + ptask = lua_newuserdata(L, sizeof(*ptask)); + *ptask = task; + rspamd_lua_setclass(L, rspamd_task_classname, -1); - return 2; + return 2; + } + else { + return 1; + } } static int diff --git a/src/plugins/lua/gpt.lua b/src/plugins/lua/gpt.lua index feccae73f..e4a77c6dd 100644 --- a/src/plugins/lua/gpt.lua +++ b/src/plugins/lua/gpt.lua @@ -48,6 +48,8 @@ gpt { allow_passthrough = false; # Check messages that are apparent ham (no action and negative score) allow_ham = false; + # default send response_format field { type = "json_object" } + include_response_format = true, } ]]) return @@ -393,7 +395,6 @@ local function default_llm_check(task) model = settings.model, max_tokens = settings.max_tokens, temperature = settings.temperature, - response_format = { type = "json_object" }, messages = { { role = 'system', @@ -418,6 +419,11 @@ local function default_llm_check(task) } } + -- Conditionally add response_format + if settings.include_response_format then + body.response_format = { type = "json_object" } + end + upstream = settings.upstreams:get_upstream_round_robin() local http_params = { url = settings.url, @@ -498,7 +504,6 @@ local function ollama_check(task) model = settings.model, max_tokens = settings.max_tokens, temperature = settings.temperature, - response_format = { type = "json_object" }, messages = { { role = 'system', @@ -523,6 +528,11 @@ local function ollama_check(task) } } + -- Conditionally add response_format + if settings.include_response_format then + body.response_format = { type = "json_object" } + end + upstream = settings.upstreams:get_upstream_round_robin() local http_params = { url = settings.url, @@ -618,4 +628,4 @@ if opts then parent = id, score = -2.0, }) -end
\ No newline at end of file +end diff --git a/src/plugins/lua/rbl.lua b/src/plugins/lua/rbl.lua index 76c84f85d..2c2fe0071 100644 --- a/src/plugins/lua/rbl.lua +++ b/src/plugins/lua/rbl.lua @@ -983,7 +983,7 @@ local function gen_rbl_callback(rule) if req.resolve_ip then -- Deal with both ipv4 and ipv6 -- Resolve names first - if r:resolve_a({ + if (rule.ipv4 == nil or rule.ipv4) and r:resolve_a({ task = task, name = req.n, callback = gen_rbl_ip_dns_callback(req), @@ -991,7 +991,7 @@ local function gen_rbl_callback(rule) }) then nresolved = nresolved + 1 end - if r:resolve('aaaa', { + if (rule.ipv6 == nil or rule.ipv6) and r:resolve('aaaa', { task = task, name = req.n, callback = gen_rbl_ip_dns_callback(req), |