diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/CMakeLists.txt | 532 | ||||
| -rw-r--r-- | src/controller.c | 116 | ||||
| -rw-r--r-- | src/libserver/fuzzy_backend/fuzzy_backend_redis.c | 4 | ||||
| -rw-r--r-- | src/libserver/maps/map.c | 102 | ||||
| -rw-r--r-- | src/libserver/maps/map_private.h | 36 | ||||
| -rw-r--r-- | src/libserver/symcache/symcache_impl.cxx | 4 | ||||
| -rw-r--r-- | src/lua/lua_common.h | 3 | ||||
| -rw-r--r-- | src/lua/lua_config.c | 4 | ||||
| -rw-r--r-- | src/lua/lua_logger.c | 485 | ||||
| -rw-r--r-- | src/lua/lua_map.c | 7 | ||||
| -rw-r--r-- | src/plugins/lua/contextal.lua | 332 | ||||
| -rw-r--r-- | src/plugins/lua/greylist.lua | 23 | ||||
| -rw-r--r-- | src/plugins/lua/hfilter.lua | 13 | ||||
| -rw-r--r-- | src/rspamadm/lua_repl.c | 12 |
14 files changed, 1070 insertions, 603 deletions
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 92edb0b6a..6cc49e4e4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,246 +1,286 @@ -MACRO(_AddModulesForced MLIST) -# Generate unique string for this build - SET(MODULES_C "${CMAKE_CURRENT_BINARY_DIR}/modules.c") - FILE(WRITE "${MODULES_C}" - "#include \"rspamd.h\"\n") - - # Handle even old cmake - LIST(LENGTH ${MLIST} MLIST_COUNT) - MATH(EXPR MLIST_MAX ${MLIST_COUNT}-1) - - FOREACH(MOD_IDX RANGE ${MLIST_MAX}) - LIST(GET ${MLIST} ${MOD_IDX} MOD) - FILE(APPEND "${MODULES_C}" "extern module_t ${MOD}_module;\n") - ENDFOREACH(MOD_IDX RANGE ${MLIST_MAX}) - - FILE(APPEND "${MODULES_C}" "\n\nmodule_t *modules[] = {\n") - - FOREACH(MOD_IDX RANGE ${MLIST_MAX}) - LIST(GET ${MLIST} ${MOD_IDX} MOD) - FILE(APPEND "${MODULES_C}" "&${MOD}_module,\n") - ENDFOREACH(MOD_IDX RANGE ${MLIST_MAX}) - - FILE(APPEND "${MODULES_C}" "NULL\n};\n") -ENDMACRO(_AddModulesForced MLIST) - -MACRO(_AddWorkersForced WLIST) - SET(WORKERS_C "${CMAKE_CURRENT_BINARY_DIR}/workers.c") - FILE(WRITE "${WORKERS_C}" - "#include \"rspamd.h\"\n") - - # Handle even old cmake - LIST(LENGTH ${WLIST} WLIST_COUNT) - MATH(EXPR WLIST_MAX ${WLIST_COUNT}-1) - FOREACH(MOD_IDX RANGE ${WLIST_MAX}) - LIST(GET ${WLIST} ${MOD_IDX} WRK) - FILE(APPEND "${WORKERS_C}" "extern worker_t ${WRK}_worker;\n") - ENDFOREACH(MOD_IDX RANGE ${WLIST_MAX}) - - FILE(APPEND "${WORKERS_C}" "\n\nworker_t *workers[] = {\n") - - FOREACH(MOD_IDX RANGE ${WLIST_MAX}) - LIST(GET ${WLIST} ${MOD_IDX} WRK) - FILE(APPEND "${WORKERS_C}" "&${WRK}_worker,\n") - ENDFOREACH(MOD_IDX RANGE ${WLIST_MAX}) - FILE(APPEND "${WORKERS_C}" "NULL\n};\n") -ENDMACRO(_AddWorkersForced WLIST) - -MACRO(AddModules MLIST WLIST) - _AddModulesForced(${MLIST}) - _AddWorkersForced(${WLIST}) - #IF(NOT EXISTS "modules.c") - # _AddModulesForced(${MLIST} ${WLIST}) - #ELSE(NOT EXISTS "modules.c") - # FILE(STRINGS "modules.c" FILE_ID_RAW REGEX "^/.*[a-zA-Z0-9]+.*/$") - # STRING(REGEX MATCH "[a-zA-Z0-9]+" FILE_ID "${FILE_ID_RAW}") - # IF(NOT FILE_ID STREQUAL MODULES_ID) - # MESSAGE("Regenerate modules info") - # _AddModulesForced(${MLIST} ${WLIST}) - # ENDIF(NOT FILE_ID STREQUAL MODULES_ID) - #ENDIF(NOT EXISTS "modules.c") -ENDMACRO(AddModules MLIST WLIST) - -# Rspamd core components -IF (ENABLE_CLANG_PLUGIN MATCHES "ON") - SET(CMAKE_C_FLAGS - "${CMAKE_C_FLAGS} -Xclang -load -Xclang ${CMAKE_CURRENT_BINARY_DIR}/../clang-plugin/librspamd-clang${CMAKE_SHARED_LIBRARY_SUFFIX} -Xclang -add-plugin -Xclang rspamd-ast") - IF(CLANG_EXTRA_PLUGINS_LIBS) - FOREACH(_lib ${CLANG_EXTRA_PLUGINS_LIBS}) - SET(CMAKE_C_FLAGS - "${CMAKE_C_FLAGS} -Xclang -load -Xclang ${_lib}") - SET(CMAKE_CXX_FLAGS - "${CMAKE_CXX_FLAGS} -Xclang -load -Xclang ${_lib}") - ENDFOREACH() - ENDIF() - IF(CLANG_EXTRA_PLUGINS) - FOREACH(_plug ${CLANG_EXTRA_PLUGINS}) - SET(CMAKE_C_FLAGS - "${CMAKE_C_FLAGS} -Xclang -add-plugin -Xclang ${_plug}") - SET(CMAKE_CXX_FLAGS - "${CMAKE_C_FLAGS} -Xclang -add-plugin -Xclang ${_plug}") - ENDFOREACH() - ENDIF() -ENDIF () - -ADD_SUBDIRECTORY(lua) -ADD_SUBDIRECTORY(libcryptobox) -ADD_SUBDIRECTORY(libutil) -ADD_SUBDIRECTORY(libserver) -ADD_SUBDIRECTORY(libmime) -ADD_SUBDIRECTORY(libstat) -ADD_SUBDIRECTORY(client) -ADD_SUBDIRECTORY(rspamadm) - -SET(RSPAMDSRC controller.c - fuzzy_storage.c - rspamd.c - worker.c - rspamd_proxy.c) - -SET(PLUGINSSRC plugins/regexp.c - plugins/chartable.cxx - plugins/fuzzy_check.c - plugins/dkim_check.c - libserver/rspamd_control.c) - -SET(MODULES_LIST regexp chartable fuzzy_check dkim) -SET(WORKERS_LIST normal controller fuzzy rspamd_proxy) -IF (ENABLE_HYPERSCAN MATCHES "ON") - LIST(APPEND WORKERS_LIST "hs_helper") - LIST(APPEND RSPAMDSRC "hs_helper.c") -ENDIF() - -AddModules(MODULES_LIST WORKERS_LIST) -LIST(LENGTH PLUGINSSRC RSPAMD_MODULES_NUM) - -SET(RAGEL_DEPENDS "${CMAKE_SOURCE_DIR}/src/ragel/smtp_address.rl" - "${CMAKE_SOURCE_DIR}/src/ragel/smtp_date.rl" - "${CMAKE_SOURCE_DIR}/src/ragel/smtp_ip.rl" - "${CMAKE_SOURCE_DIR}/src/ragel/smtp_base.rl" - "${CMAKE_SOURCE_DIR}/src/ragel/content_disposition.rl") -RAGEL_TARGET(ragel_smtp_addr - INPUTS ${CMAKE_SOURCE_DIR}/src/ragel/smtp_addr_parser.rl - DEPENDS ${RAGEL_DEPENDS} - COMPILE_FLAGS -T1 - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/smtp_addr_parser.rl.c) -RAGEL_TARGET(ragel_content_disposition - INPUTS ${CMAKE_SOURCE_DIR}/src/ragel/content_disposition_parser.rl - DEPENDS ${RAGEL_DEPENDS} - COMPILE_FLAGS -G2 - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/content_disposition.rl.c) -RAGEL_TARGET(ragel_rfc2047 - INPUTS ${CMAKE_SOURCE_DIR}/src/ragel/rfc2047_parser.rl - DEPENDS ${RAGEL_DEPENDS} - COMPILE_FLAGS -G2 - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/rfc2047.rl.c) -RAGEL_TARGET(ragel_smtp_date - INPUTS ${CMAKE_SOURCE_DIR}/src/ragel/smtp_date_parser.rl - DEPENDS ${RAGEL_DEPENDS} - COMPILE_FLAGS -G2 - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/date_parser.rl.c) -RAGEL_TARGET(ragel_smtp_ip - INPUTS ${CMAKE_SOURCE_DIR}/src/ragel/smtp_ip_parser.rl - DEPENDS ${RAGEL_DEPENDS} - COMPILE_FLAGS -G2 - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ip_parser.rl.c) -# Fucking cmake... -FOREACH(_GEN ${LIBSERVER_GENERATED}) - set_source_files_properties(${_GEN} PROPERTIES GENERATED TRUE) -ENDFOREACH() -######################### LINK SECTION ############################### - -IF(ENABLE_STATIC MATCHES "ON") - ADD_LIBRARY(rspamd-server STATIC - ${RSPAMD_CRYPTOBOX} - ${RSPAMD_UTIL} - ${RSPAMD_LUA} - ${RSPAMD_SERVER} - ${RSPAMD_STAT} - ${RSPAMD_MIME} - ${CMAKE_CURRENT_BINARY_DIR}/modules.c - ${PLUGINSSRC} - "${RAGEL_ragel_smtp_addr_OUTPUTS}" - "${RAGEL_ragel_newlines_strip_OUTPUTS}" - "${RAGEL_ragel_content_type_OUTPUTS}" - "${RAGEL_ragel_content_disposition_OUTPUTS}" - "${RAGEL_ragel_rfc2047_OUTPUTS}" - "${RAGEL_ragel_smtp_date_OUTPUTS}" - "${RAGEL_ragel_smtp_ip_OUTPUTS}" - ${BACKWARD_ENABLE}) -ELSE() - ADD_LIBRARY(rspamd-server SHARED - ${RSPAMD_CRYPTOBOX} - ${RSPAMD_UTIL} - ${RSPAMD_SERVER} - ${RSPAMD_STAT} - ${RSPAMD_MIME} - ${RSPAMD_LUA} - ${CMAKE_CURRENT_BINARY_DIR}/modules.c - ${PLUGINSSRC} - "${RAGEL_ragel_smtp_addr_OUTPUTS}" - "${RAGEL_ragel_newlines_strip_OUTPUTS}" - "${RAGEL_ragel_content_type_OUTPUTS}" - "${RAGEL_ragel_content_disposition_OUTPUTS}" - "${RAGEL_ragel_rfc2047_OUTPUTS}" - "${RAGEL_ragel_smtp_date_OUTPUTS}" - "${RAGEL_ragel_smtp_ip_OUTPUTS}" - ${BACKWARD_ENABLE}) -ENDIF() - -FOREACH(_DEP ${LIBSERVER_DEPENDS}) - ADD_DEPENDENCIES(rspamd-server "${_DEP}") -ENDFOREACH() - -TARGET_LINK_LIBRARIES(rspamd-server rspamd-http-parser) -TARGET_LINK_LIBRARIES(rspamd-server rspamd-fpconv) -TARGET_LINK_LIBRARIES(rspamd-server rspamd-cdb) -TARGET_LINK_LIBRARIES(rspamd-server rspamd-lpeg) -TARGET_LINK_LIBRARIES(rspamd-server lcbtrie) -IF(SYSTEM_ZSTD MATCHES "OFF") - TARGET_LINK_LIBRARIES(rspamd-server rspamd-zstd) -ELSE() - TARGET_LINK_LIBRARIES(rspamd-server zstd) -ENDIF() -TARGET_LINK_LIBRARIES(rspamd-server rspamd-simdutf) - -IF (ENABLE_CLANG_PLUGIN MATCHES "ON") - ADD_DEPENDENCIES(rspamd-server rspamd-clang) -ENDIF() - -IF (NOT WITH_LUAJIT) - TARGET_LINK_LIBRARIES(rspamd-server rspamd-bit) -ENDIF() - -IF (ENABLE_SNOWBALL MATCHES "ON") - TARGET_LINK_LIBRARIES(rspamd-server stemmer) -ENDIF() -TARGET_LINK_LIBRARIES(rspamd-server rspamd-hiredis) - -IF (ENABLE_FANN MATCHES "ON") - TARGET_LINK_LIBRARIES(rspamd-server fann) -ENDIF () - -IF (ENABLE_HYPERSCAN MATCHES "ON") - TARGET_LINK_LIBRARIES(rspamd-server hs) -ENDIF() - -IF(WITH_BLAS) - TARGET_LINK_LIBRARIES(rspamd-server ${BLAS_REQUIRED_LIBRARIES}) -ENDIF() - -TARGET_LINK_LIBRARIES(rspamd-server ${RSPAMD_REQUIRED_LIBRARIES}) -ADD_BACKWARD(rspamd-server) - -ADD_EXECUTABLE(rspamd ${RSPAMDSRC} ${CMAKE_CURRENT_BINARY_DIR}/workers.c ${CMAKE_CURRENT_BINARY_DIR}/config.h) -ADD_BACKWARD(rspamd) -SET_TARGET_PROPERTIES(rspamd PROPERTIES LINKER_LANGUAGE CXX) -SET_TARGET_PROPERTIES(rspamd-server PROPERTIES LINKER_LANGUAGE CXX) -IF(NOT NO_TARGET_VERSIONS) - SET_TARGET_PROPERTIES(rspamd PROPERTIES VERSION ${RSPAMD_VERSION}) -ENDIF() - -#TARGET_LINK_LIBRARIES(rspamd ${RSPAMD_REQUIRED_LIBRARIES}) -TARGET_LINK_LIBRARIES(rspamd rspamd-server) - -INSTALL(TARGETS rspamd RUNTIME DESTINATION bin) -INSTALL(TARGETS rspamd-server LIBRARY DESTINATION ${RSPAMD_LIBDIR})
\ No newline at end of file +# Function to generate module registrations +function(generate_modules_list MODULE_LIST) + # Generate unique string for this build + set(MODULES_C "${CMAKE_CURRENT_BINARY_DIR}/modules.c") + file(WRITE "${MODULES_C}" + "#include \"rspamd.h\"\n") + + # Process each module + foreach (MOD IN LISTS ${MODULE_LIST}) + file(APPEND "${MODULES_C}" "extern module_t ${MOD}_module;\n") + endforeach () + + file(APPEND "${MODULES_C}" "\n\nmodule_t *modules[] = {\n") + + foreach (MOD IN LISTS ${MODULE_LIST}) + file(APPEND "${MODULES_C}" "&${MOD}_module,\n") + endforeach () + + file(APPEND "${MODULES_C}" "NULL\n};\n") + + # Return the generated file path + set(MODULES_C_PATH "${MODULES_C}" PARENT_SCOPE) +endfunction() + +# Function to generate worker registrations +function(generate_workers_list WORKER_LIST) + set(WORKERS_C "${CMAKE_CURRENT_BINARY_DIR}/workers.c") + file(WRITE "${WORKERS_C}" + "#include \"rspamd.h\"\n") + + # Process each worker + foreach (WRK IN LISTS ${WORKER_LIST}) + file(APPEND "${WORKERS_C}" "extern worker_t ${WRK}_worker;\n") + endforeach () + + file(APPEND "${WORKERS_C}" "\n\nworker_t *workers[] = {\n") + + foreach (WRK IN LISTS ${WORKER_LIST}) + file(APPEND "${WORKERS_C}" "&${WRK}_worker,\n") + endforeach () + + file(APPEND "${WORKERS_C}" "NULL\n};\n") + + # Return the generated file path + set(WORKERS_C_PATH "${WORKERS_C}" PARENT_SCOPE) +endfunction() + +# Function to generate both modules and workers +function(generate_registration_code MODULE_LIST WORKER_LIST) + generate_modules_list(${MODULE_LIST}) + generate_workers_list(${WORKER_LIST}) + + # Set parent scope variables + set(MODULES_C_PATH ${MODULES_C_PATH} PARENT_SCOPE) + set(WORKERS_C_PATH ${WORKERS_C_PATH} PARENT_SCOPE) +endfunction() + +# Configure Clang Plugin if enabled +if (ENABLE_CLANG_PLUGIN) + set(CLANG_PLUGIN_FLAGS "-Xclang -load -Xclang ${CMAKE_CURRENT_BINARY_DIR}/../clang-plugin/librspamd-clang${CMAKE_SHARED_LIBRARY_SUFFIX} -Xclang -add-plugin -Xclang rspamd-ast") + + # Apply to both C and C++ compiler flags + add_compile_options(${CLANG_PLUGIN_FLAGS}) + + # Add any extra clang plugins + if (CLANG_EXTRA_PLUGINS_LIBS) + foreach (lib ${CLANG_EXTRA_PLUGINS_LIBS}) + add_compile_options("-Xclang" "-load" "-Xclang" "${lib}") + endforeach () + endif () + + if (CLANG_EXTRA_PLUGINS) + foreach (plug ${CLANG_EXTRA_PLUGINS}) + add_compile_options("-Xclang" "-add-plugin" "-Xclang" "${plug}") + endforeach () + endif () +endif () + +# Add subdirectories for components +add_subdirectory(lua) +add_subdirectory(libcryptobox) +add_subdirectory(libutil) +add_subdirectory(libserver) +add_subdirectory(libmime) +add_subdirectory(libstat) +add_subdirectory(client) +add_subdirectory(rspamadm) + +# Define source files +set(RSPAMD_SOURCES + controller.c + fuzzy_storage.c + rspamd.c + worker.c + rspamd_proxy.c) + +set(PLUGIN_SOURCES + plugins/regexp.c + plugins/chartable.cxx + plugins/fuzzy_check.c + plugins/dkim_check.c + libserver/rspamd_control.c) + +# Define module and worker lists +set(MODULES_LIST regexp chartable fuzzy_check dkim) +set(WORKERS_LIST normal controller fuzzy rspamd_proxy) + +# Add hyperscan worker if enabled +if (ENABLE_HYPERSCAN) + list(APPEND WORKERS_LIST hs_helper) + list(APPEND RSPAMD_SOURCES hs_helper.c) +endif () + +# Generate modules and workers registration code +generate_registration_code(MODULES_LIST WORKERS_LIST) + +# Count the number of modules +list(LENGTH PLUGIN_SOURCES RSPAMD_MODULES_NUM) + +# Configure Ragel for parsers +set(RAGEL_DEPENDS + "${CMAKE_SOURCE_DIR}/src/ragel/smtp_address.rl" + "${CMAKE_SOURCE_DIR}/src/ragel/smtp_date.rl" + "${CMAKE_SOURCE_DIR}/src/ragel/smtp_ip.rl" + "${CMAKE_SOURCE_DIR}/src/ragel/smtp_base.rl" + "${CMAKE_SOURCE_DIR}/src/ragel/content_disposition.rl") + +# Generate parsers with Ragel +ragel_target(ragel_smtp_addr + INPUTS ${CMAKE_SOURCE_DIR}/src/ragel/smtp_addr_parser.rl + DEPENDS ${RAGEL_DEPENDS} + COMPILE_FLAGS -T1 + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/smtp_addr_parser.rl.c) + +ragel_target(ragel_content_disposition + INPUTS ${CMAKE_SOURCE_DIR}/src/ragel/content_disposition_parser.rl + DEPENDS ${RAGEL_DEPENDS} + COMPILE_FLAGS -G2 + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/content_disposition.rl.c) + +ragel_target(ragel_rfc2047 + INPUTS ${CMAKE_SOURCE_DIR}/src/ragel/rfc2047_parser.rl + DEPENDS ${RAGEL_DEPENDS} + COMPILE_FLAGS -G2 + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/rfc2047.rl.c) + +ragel_target(ragel_smtp_date + INPUTS ${CMAKE_SOURCE_DIR}/src/ragel/smtp_date_parser.rl + DEPENDS ${RAGEL_DEPENDS} + COMPILE_FLAGS -G2 + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/date_parser.rl.c) + +ragel_target(ragel_smtp_ip + INPUTS ${CMAKE_SOURCE_DIR}/src/ragel/smtp_ip_parser.rl + DEPENDS ${RAGEL_DEPENDS} + COMPILE_FLAGS -G2 + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ip_parser.rl.c) + +# Mark generated files correctly +foreach (_gen ${LIBSERVER_GENERATED}) + set_source_files_properties(${_gen} PROPERTIES GENERATED TRUE) +endforeach () + +# Collection of all generated Ragel outputs +set(RAGEL_OUTPUTS + ${RAGEL_ragel_smtp_addr_OUTPUTS} + ${RAGEL_ragel_newlines_strip_OUTPUTS} + ${RAGEL_ragel_content_type_OUTPUTS} + ${RAGEL_ragel_content_disposition_OUTPUTS} + ${RAGEL_ragel_rfc2047_OUTPUTS} + ${RAGEL_ragel_smtp_date_OUTPUTS} + ${RAGEL_ragel_smtp_ip_OUTPUTS}) + +# Common sources for rspamd-server +set(SERVER_COMMON_SOURCES + ${RSPAMD_CRYPTOBOX} + ${RSPAMD_UTIL} + ${RSPAMD_LUA} + ${RSPAMD_SERVER} + ${RSPAMD_STAT} + ${RSPAMD_MIME} + ${MODULES_C_PATH} + ${PLUGIN_SOURCES} + ${RAGEL_OUTPUTS} + ${BACKWARD_ENABLE}) + +# Build rspamd-server as static or shared library based on configuration +if (ENABLE_STATIC) + add_library(rspamd-server STATIC ${SERVER_COMMON_SOURCES}) +else () + add_library(rspamd-server SHARED ${SERVER_COMMON_SOURCES}) +endif () + +# Set dependencies for rspamd-server +foreach (_dep ${LIBSERVER_DEPENDS}) + add_dependencies(rspamd-server "${_dep}") +endforeach () + +# Link dependencies +target_link_libraries(rspamd-server + PRIVATE + rspamd-http-parser + rspamd-fpconv + rspamd-cdb + rspamd-lpeg + ottery + lcbtrie + rspamd-simdutf + rdns + ucl) + +# Handle xxhash dependency +if (SYSTEM_XXHASH) + target_link_libraries(rspamd-server PUBLIC xxhash) +else () + target_link_libraries(rspamd-server PUBLIC rspamd-xxhash) +endif () + +# Handle zstd dependency +if (SYSTEM_ZSTD) + target_link_libraries(rspamd-server PUBLIC zstd) +else () + target_link_libraries(rspamd-server PRIVATE rspamd-zstd) +endif () + +# Handle clang plugin dependency +if (ENABLE_CLANG_PLUGIN) + add_dependencies(rspamd-server rspamd-clang) +endif () + +# Handle Lua JIT/Lua dependency +if (NOT WITH_LUAJIT) + target_link_libraries(rspamd-server PRIVATE rspamd-bit) +endif () + +# Link additional optional dependencies +if (ENABLE_SNOWBALL) + target_link_libraries(rspamd-server PRIVATE stemmer) +endif () + +target_link_libraries(rspamd-server PRIVATE rspamd-hiredis) + +if (ENABLE_FANN) + target_link_libraries(rspamd-server PRIVATE fann) +endif () + +if (ENABLE_HYPERSCAN) + target_link_libraries(rspamd-server PUBLIC hs) +endif () + +if (WITH_BLAS) + target_link_libraries(rspamd-server PRIVATE ${BLAS_REQUIRED_LIBRARIES}) +endif () + +# Link all required system libraries +target_link_libraries(rspamd-server PUBLIC ${RSPAMD_REQUIRED_LIBRARIES}) + +# Add Backward support for stacktrace +add_backward(rspamd-server) + +# Build main rspamd executable +add_executable(rspamd + ${RSPAMD_SOURCES} + ${WORKERS_C_PATH} + ${CMAKE_CURRENT_BINARY_DIR}/config.h) + +# Configure rspamd executable +add_backward(rspamd) +set_target_properties(rspamd PROPERTIES LINKER_LANGUAGE CXX) +set_target_properties(rspamd-server PROPERTIES LINKER_LANGUAGE CXX) + +if (NOT NO_TARGET_VERSIONS) + set_target_properties(rspamd PROPERTIES VERSION ${RSPAMD_VERSION}) +endif () + +# Link rspamd executable with the server library +target_link_libraries(rspamd PRIVATE rspamd-server) + +# Install targets +install(TARGETS rspamd + RUNTIME + DESTINATION bin) + +install(TARGETS rspamd-server + LIBRARY + DESTINATION ${RSPAMD_LIBDIR}) diff --git a/src/controller.c b/src/controller.c index 386448f93..22423e999 100644 --- a/src/controller.c +++ b/src/controller.c @@ -1,5 +1,5 @@ /* - * Copyright 2024 Vsevolod Stakhov + * Copyright 2025 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -979,12 +979,6 @@ rspamd_controller_handle_maps(struct rspamd_http_connection_entry *conn_ent, if (bk->protocol == MAP_PROTO_FILE) { editable = rspamd_controller_can_edit_map(bk); - - if (!editable && access(bk->uri, R_OK) == -1) { - /* Skip unreadable and non-existing maps */ - continue; - } - obj = ucl_object_typed_new(UCL_OBJECT); ucl_object_insert_key(obj, ucl_object_fromint(bk->id), "map", 0, false); @@ -994,8 +988,34 @@ rspamd_controller_handle_maps(struct rspamd_http_connection_entry *conn_ent, } ucl_object_insert_key(obj, ucl_object_fromstring(bk->uri), "uri", 0, false); + ucl_object_insert_key(obj, ucl_object_fromstring("file"), + "type", 0, false); ucl_object_insert_key(obj, ucl_object_frombool(editable), "editable", 0, false); + ucl_object_insert_key(obj, ucl_object_frombool(bk->shared->loaded), + "loaded", 0, false); + ucl_object_insert_key(obj, ucl_object_frombool(bk->shared->cached), + "cached", 0, false); + ucl_array_append(top, obj); + } + else { + obj = ucl_object_typed_new(UCL_OBJECT); + ucl_object_insert_key(obj, ucl_object_fromint(bk->id), + "map", 0, false); + if (map->description) { + ucl_object_insert_key(obj, ucl_object_fromstring(map->description), + "description", 0, false); + } + ucl_object_insert_key(obj, ucl_object_fromstring(bk->uri), + "uri", 0, false); + ucl_object_insert_key(obj, ucl_object_fromstring(rspamd_map_fetch_protocol_name(bk->protocol)), + "type", 0, false); + ucl_object_insert_key(obj, ucl_object_frombool(false), + "editable", 0, false); + ucl_object_insert_key(obj, ucl_object_frombool(bk->shared->loaded), + "loaded", 0, false); + ucl_object_insert_key(obj, ucl_object_frombool(bk->shared->cached), + "cached", 0, false); ucl_array_append(top, obj); } } @@ -1008,6 +1028,21 @@ rspamd_controller_handle_maps(struct rspamd_http_connection_entry *conn_ent, return 0; } +gboolean +rspamd_controller_map_traverse_callback(gconstpointer key, gconstpointer value, gsize _hits, gpointer ud) +{ + rspamd_fstring_t **target = (rspamd_fstring_t **) ud; + + *target = rspamd_fstring_append(*target, key, strlen(key)); + + if (value) { + *target = rspamd_fstring_append(*target, " ", 1); + *target = rspamd_fstring_append(*target, value, strlen(value)); + } + *target = rspamd_fstring_append(*target, "\n", 1); + + return TRUE; +} /* * Get map command handler: * request: /getmap @@ -1020,7 +1055,7 @@ rspamd_controller_handle_get_map(struct rspamd_http_connection_entry *conn_ent, { struct rspamd_controller_session *session = conn_ent->ud; GList *cur; - struct rspamd_map *map; + struct rspamd_map *map = NULL; struct rspamd_map_backend *bk = NULL; const rspamd_ftok_t *idstr; struct stat st; @@ -1054,7 +1089,7 @@ rspamd_controller_handle_get_map(struct rspamd_http_connection_entry *conn_ent, PTR_ARRAY_FOREACH(map->backends, i, bk) { - if (bk->id == id && bk->protocol == MAP_PROTO_FILE) { + if (bk->id == id) { found = TRUE; break; } @@ -1069,32 +1104,53 @@ rspamd_controller_handle_get_map(struct rspamd_http_connection_entry *conn_ent, return 0; } - if (stat(bk->uri, &st) == -1 || (fd = open(bk->uri, O_RDONLY)) == -1) { + if (bk->protocol == MAP_PROTO_FILE) { + if (stat(bk->uri, &st) == -1 || (fd = open(bk->uri, O_RDONLY)) == -1) { + reply = rspamd_http_new_message(HTTP_RESPONSE); + reply->date = time(NULL); + reply->code = 200; + } + else { + + reply = rspamd_http_new_message(HTTP_RESPONSE); + reply->date = time(NULL); + reply->code = 200; + + if (st.st_size > 0) { + if (!rspamd_http_message_set_body_from_fd(reply, fd)) { + close(fd); + rspamd_http_message_unref(reply); + msg_err_session("cannot read map %s: %s", bk->uri, strerror(errno)); + rspamd_controller_send_error(conn_ent, 500, "Map read error"); + return 0; + } + } + else { + rspamd_fstring_t *empty_body = rspamd_fstring_new_init("", 0); + rspamd_http_message_set_body_from_fstring_steal(reply, empty_body); + } + + close(fd); + } + } + else if (bk->protocol == MAP_PROTO_STATIC) { + /* We can just traverse map and form reply */ reply = rspamd_http_new_message(HTTP_RESPONSE); - reply->date = time(NULL); reply->code = 200; + rspamd_fstring_t *map_body = rspamd_fstring_new(); + rspamd_map_traverse(bk->map, rspamd_controller_map_traverse_callback, &map_body, FALSE); + rspamd_http_message_set_body_from_fstring_steal(reply, map_body); } - else { - + else if (bk->shared->loaded) { reply = rspamd_http_new_message(HTTP_RESPONSE); - reply->date = time(NULL); reply->code = 200; - - if (st.st_size > 0) { - if (!rspamd_http_message_set_body_from_fd(reply, fd)) { - close(fd); - rspamd_http_message_unref(reply); - msg_err_session("cannot read map %s: %s", bk->uri, strerror(errno)); - rspamd_controller_send_error(conn_ent, 500, "Map read error"); - return 0; - } - } - else { - rspamd_fstring_t *empty_body = rspamd_fstring_new_init("", 0); - rspamd_http_message_set_body_from_fstring_steal(reply, empty_body); - } - - close(fd); + rspamd_fstring_t *map_body = rspamd_fstring_new(); + rspamd_map_traverse(bk->map, rspamd_controller_map_traverse_callback, &map_body, FALSE); + rspamd_http_message_set_body_from_fstring_steal(reply, map_body); + } + else { + reply = rspamd_http_new_message(HTTP_RESPONSE); + reply->code = 404; } rspamd_http_connection_reset(conn_ent->conn); diff --git a/src/libserver/fuzzy_backend/fuzzy_backend_redis.c b/src/libserver/fuzzy_backend/fuzzy_backend_redis.c index 27c663070..f150d48be 100644 --- a/src/libserver/fuzzy_backend/fuzzy_backend_redis.c +++ b/src/libserver/fuzzy_backend/fuzzy_backend_redis.c @@ -116,11 +116,9 @@ rspamd_redis_get_servers(struct rspamd_fuzzy_backend_redis *ctx, res = *((struct upstream_list **) lua_touserdata(L, -1)); } else { - struct lua_logger_trace tr; char outbuf[8192]; - memset(&tr, 0, sizeof(tr)); - lua_logger_out_type(L, -2, outbuf, sizeof(outbuf) - 1, &tr, + lua_logger_out(L, -2, outbuf, sizeof(outbuf), LUA_ESCAPE_UNPRINTABLE); msg_err("cannot get %s upstreams for Redis fuzzy storage %s; table content: %s", diff --git a/src/libserver/maps/map.c b/src/libserver/maps/map.c index 97130ad7c..51390f24b 100644 --- a/src/libserver/maps/map.c +++ b/src/libserver/maps/map.c @@ -339,6 +339,7 @@ http_map_finish(struct rspamd_http_connection *conn, cbd->periodic->cur_backend = 0; /* Reset cache, old cached data will be cleaned on timeout */ g_atomic_int_set(&data->cache->available, 0); + g_atomic_int_set(&bk->shared->loaded, 0); data->cur_cache_cbd = NULL; rspamd_map_process_periodic(cbd->periodic); @@ -424,6 +425,8 @@ http_map_finish(struct rspamd_http_connection *conn, * We know that a map is in the locked state */ g_atomic_int_set(&data->cache->available, 1); + g_atomic_int_set(&bk->shared->loaded, 1); + g_atomic_int_set(&bk->shared->cached, 0); /* Store cached data */ rspamd_strlcpy(data->cache->shmem_name, cbd->shmem_data->shm_name, sizeof(data->cache->shmem_name)); @@ -919,6 +922,8 @@ read_map_file(struct rspamd_map *map, struct file_map_data *data, map->read_callback(NULL, 0, &periodic->cbdata, TRUE); } + g_atomic_int_set(&bk->shared->loaded, 1); + return TRUE; } @@ -1003,6 +1008,7 @@ read_map_static(struct rspamd_map *map, struct static_map_data *data, } data->processed = TRUE; + g_atomic_int_set(&bk->shared->loaded, 1); return TRUE; } @@ -1011,6 +1017,7 @@ static void rspamd_map_periodic_dtor(struct map_periodic_cbdata *periodic) { struct rspamd_map *map; + struct rspamd_map_backend *bk; map = periodic->map; msg_debug_map("periodic dtor %p; need_modify=%d", periodic, periodic->need_modify); @@ -1028,8 +1035,11 @@ rspamd_map_periodic_dtor(struct map_periodic_cbdata *periodic) } if (periodic->locked) { - g_atomic_int_set(periodic->map->locked, 0); - msg_debug_map("unlocked map %s", periodic->map->name); + if (periodic->cur_backend < map->backends->len) { + bk = (struct rspamd_map_backend *) g_ptr_array_index(map->backends, periodic->cur_backend); + g_atomic_int_set(&bk->shared->locked, 0); + msg_debug_map("unlocked map %s", map->name); + } if (periodic->map->wrk->state == rspamd_worker_state_running) { rspamd_map_schedule_periodic(periodic->map, @@ -1438,6 +1448,9 @@ rspamd_map_read_cached(struct rspamd_map *map, struct rspamd_map_backend *bk, map->read_callback(in, len, &periodic->cbdata, TRUE); } + g_atomic_int_set(&bk->shared->loaded, 1); + g_atomic_int_set(&bk->shared->cached, 1); + munmap(in, mmap_len); return TRUE; @@ -1727,6 +1740,8 @@ rspamd_map_read_http_cached_file(struct rspamd_map *map, struct tm tm; char ncheck_buf[32], lm_buf[32]; + g_atomic_int_set(&bk->shared->loaded, 1); + g_atomic_int_set(&bk->shared->cached, 1); rspamd_localtime(map->next_check, &tm); strftime(ncheck_buf, sizeof(ncheck_buf) - 1, "%Y-%m-%d %H:%M:%S", &tm); rspamd_localtime(htdata->last_modified, &tm); @@ -2027,8 +2042,20 @@ rspamd_map_process_periodic(struct map_periodic_cbdata *cbd) map = cbd->map; map->scheduled_check = NULL; + /* For each backend we need to check for modifications */ + if (cbd->cur_backend >= cbd->map->backends->len) { + /* Last backend */ + msg_debug_map("finished map: %d of %d", cbd->cur_backend, + cbd->map->backends->len); + MAP_RELEASE(cbd, "periodic"); + + return; + } + + bk = g_ptr_array_index(map->backends, cbd->cur_backend); + if (!map->file_only && !cbd->locked) { - if (!g_atomic_int_compare_and_exchange(cbd->map->locked, + if (!g_atomic_int_compare_and_exchange(&bk->shared->locked, 0, 1)) { msg_debug_map( "don't try to reread map %s as it is locked by other process, " @@ -2040,7 +2067,7 @@ rspamd_map_process_periodic(struct map_periodic_cbdata *cbd) return; } else { - msg_debug_map("locked map %s", cbd->map->name); + msg_debug_map("locked map %s", map->name); cbd->locked = TRUE; } } @@ -2050,7 +2077,7 @@ rspamd_map_process_periodic(struct map_periodic_cbdata *cbd) rspamd_map_schedule_periodic(cbd->map, RSPAMD_MAP_SCHEDULE_ERROR); if (cbd->locked) { - g_atomic_int_set(cbd->map->locked, 0); + g_atomic_int_set(&bk->shared->locked, 0); cbd->locked = FALSE; } @@ -2064,19 +2091,7 @@ rspamd_map_process_periodic(struct map_periodic_cbdata *cbd) return; } - /* For each backend we need to check for modifications */ - if (cbd->cur_backend >= cbd->map->backends->len) { - /* Last backend */ - msg_debug_map("finished map: %d of %d", cbd->cur_backend, - cbd->map->backends->len); - MAP_RELEASE(cbd, "periodic"); - - return; - } - if (cbd->map->wrk && cbd->map->wrk->state == rspamd_worker_state_running) { - bk = g_ptr_array_index(cbd->map->backends, cbd->cur_backend); - g_assert(bk != NULL); if (cbd->need_modify) { /* Load data from the next backend */ @@ -2781,9 +2796,8 @@ rspamd_map_parse_backend(struct rspamd_config *cfg, const char *map_line) bk->data.sd = sdata; } - bk->id = rspamd_cryptobox_fast_hash_specific(RSPAMD_CRYPTOBOX_T1HA, - bk->uri, strlen(bk->uri), - 0xdeadbabe); + bk->shared = rspamd_mempool_alloc0_shared(cfg->cfg_pool, + sizeof(struct rspamd_map_shared_backend_data)); return bk; @@ -2815,6 +2829,13 @@ rspamd_map_calculate_hash(struct rspamd_map *map) rspamd_cryptobox_hash_init(&st, NULL, 0); + if (map->name) { + rspamd_cryptobox_hash_update(&st, map->name, strlen(map->name)); + } + if (map->description) { + rspamd_cryptobox_hash_update(&st, map->description, strlen(map->description)); + } + for (i = 0; i < map->backends->len; i++) { bk = g_ptr_array_index(map->backends, i); rspamd_cryptobox_hash_update(&st, bk->uri, strlen(bk->uri)); @@ -2823,6 +2844,26 @@ rspamd_map_calculate_hash(struct rspamd_map *map) rspamd_cryptobox_hash_final(&st, cksum); cksum_encoded = rspamd_encode_base32(cksum, sizeof(cksum), RSPAMD_BASE32_DEFAULT); rspamd_strlcpy(map->tag, cksum_encoded, sizeof(map->tag)); + + for (i = 0; i < map->backends->len; i++) { + bk = g_ptr_array_index(map->backends, i); + + /* Also update each backend */ + rspamd_cryptobox_fast_hash_state_t hst; + rspamd_cryptobox_fast_hash_init(&hst, 0); + rspamd_cryptobox_fast_hash_update(&hst, bk->uri, strlen(bk->uri)); + rspamd_cryptobox_fast_hash_update(&hst, map->tag, sizeof(map->tag)); + + if (bk->protocol == MAP_PROTO_STATIC) { + /* Static maps content is pre-defined */ + rspamd_cryptobox_fast_hash_update(&hst, bk->data.sd->data, + bk->data.sd->len); + } + + /* We use only 52 bits to be compatible with other numbers representation */ + bk->id = rspamd_cryptobox_fast_hash_final(&hst) & ~(0xFFFULL << 52); + } + g_free(cksum_encoded); } @@ -2888,8 +2929,6 @@ rspamd_map_add(struct rspamd_config *cfg, map->user_data = user_data; map->cfg = cfg; map->id = rspamd_random_uint64_fast(); - map->locked = - rspamd_mempool_alloc0_shared(cfg->cfg_pool, sizeof(int)); map->backends = g_ptr_array_sized_new(1); map->wrk = worker; rspamd_mempool_add_destructor(cfg->cfg_pool, rspamd_ptr_array_free_hard, @@ -2988,8 +3027,6 @@ rspamd_map_add_from_ucl(struct rspamd_config *cfg, map->user_data = user_data; map->cfg = cfg; map->id = rspamd_random_uint64_fast(); - map->locked = - rspamd_mempool_alloc0_shared(cfg->cfg_pool, sizeof(int)); map->backends = g_ptr_array_new(); map->wrk = worker; map->no_file_read = (flags & RSPAMD_MAP_FILE_NO_READ); @@ -3108,7 +3145,7 @@ rspamd_map_add_from_ucl(struct rspamd_config *cfg, goto err; } - gboolean all_local = TRUE; + gboolean all_local = TRUE, all_loaded = TRUE; PTR_ARRAY_FOREACH(map->backends, i, bk) { @@ -3127,9 +3164,8 @@ rspamd_map_add_from_ucl(struct rspamd_config *cfg, map_data = g_string_sized_new(32); if (rspamd_map_add_static_string(cfg, elt, map_data)) { - bk->data.sd->data = map_data->str; bk->data.sd->len = map_data->len; - g_string_free(map_data, FALSE); + bk->data.sd->data = (unsigned char *) g_string_free(map_data, FALSE); } else { g_string_free(map_data, TRUE); @@ -3152,13 +3188,16 @@ rspamd_map_add_from_ucl(struct rspamd_config *cfg, } ucl_object_iterate_free(it); - bk->data.sd->data = map_data->str; bk->data.sd->len = map_data->len; - g_string_free(map_data, FALSE); + bk->data.sd->data = (unsigned char *) g_string_free(map_data, FALSE); } } else if (bk->protocol != MAP_PROTO_FILE) { all_local = FALSE; + all_loaded = FALSE; /* Will be loaded later */ + } + else { + all_loaded = FALSE; /* Will be loaded later (even for files) */ } } @@ -3167,6 +3206,11 @@ rspamd_map_add_from_ucl(struct rspamd_config *cfg, cfg->map_file_watch_multiplier); } + if (all_loaded) { + /* Static map */ + g_atomic_int_set(&bk->shared->loaded, 1); + } + rspamd_map_calculate_hash(map); msg_debug_map("added map from ucl"); diff --git a/src/libserver/maps/map_private.h b/src/libserver/maps/map_private.h index d0b22fe36..66949f926 100644 --- a/src/libserver/maps/map_private.h +++ b/src/libserver/maps/map_private.h @@ -1,5 +1,5 @@ /* - * Copyright 2024 Vsevolod Stakhov + * Copyright 2025 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -54,6 +54,23 @@ enum fetch_proto { MAP_PROTO_STATIC }; +static const char * +rspamd_map_fetch_protocol_name(enum fetch_proto proto) +{ + switch (proto) { + case MAP_PROTO_FILE: + return "file"; + case MAP_PROTO_HTTP: + return "http"; + case MAP_PROTO_HTTPS: + return "https"; + case MAP_PROTO_STATIC: + return "static"; + default: + return "unknown"; + } +} + /** * Data specific to file maps */ @@ -76,7 +93,7 @@ struct rspamd_http_map_cached_cbdata { time_t last_checked; }; -struct rspamd_map_cachepoint { +struct rspamd_http_map_cache { int available; gsize len; time_t last_modified; @@ -88,7 +105,7 @@ struct rspamd_map_cachepoint { */ struct http_map_data { /* Shared cache data */ - struct rspamd_map_cachepoint *cache; + struct rspamd_http_map_cache *cache; /* Non-shared for cache owner, used to cleanup cache */ struct rspamd_http_map_cached_cbdata *cur_cache_cbd; char *userinfo; @@ -117,14 +134,23 @@ union rspamd_map_backend_data { struct rspamd_map; +/* + * Shared between workers + */ +struct rspamd_map_shared_backend_data { + int locked; + int loaded; + int cached; +}; struct rspamd_map_backend { enum fetch_proto protocol; gboolean is_signed; gboolean is_compressed; gboolean is_fallback; + struct rspamd_map_shared_backend_data *shared; struct rspamd_map *map; struct ev_loop *event_loop; - uint32_t id; + uint64_t id; struct rspamd_cryptobox_pubkey *trusted_pubkey; union rspamd_map_backend_data data; char *uri; @@ -167,8 +193,6 @@ struct rspamd_map { bool static_only; /* No need to check */ bool no_file_read; /* Do not read files */ bool seen; /* This map has already been watched or pre-loaded */ - /* Shared lock for temporary disabling of map reading (e.g. when this map is written by UI) */ - int *locked; char tag[MEMPOOL_UID_LEN]; }; diff --git a/src/libserver/symcache/symcache_impl.cxx b/src/libserver/symcache/symcache_impl.cxx index 4d17348c2..c0278cfc1 100644 --- a/src/libserver/symcache/symcache_impl.cxx +++ b/src/libserver/symcache/symcache_impl.cxx @@ -126,7 +126,7 @@ auto symcache::init() -> bool } else { msg_err_cache("cannot register delayed dependency %s -> %s: " - "destionation %s is missing", + "destination %s is missing", delayed_dep.from.data(), delayed_dep.to.data(), delayed_dep.to.data()); } @@ -1338,4 +1338,4 @@ auto symcache::get_max_timeout(std::vector<std::pair<double, const cache_item *> return accumulated_timeout; } -}// namespace rspamd::symcache
\ No newline at end of file +}// namespace rspamd::symcache diff --git a/src/lua/lua_common.h b/src/lua/lua_common.h index a29444394..5819da8cb 100644 --- a/src/lua/lua_common.h +++ b/src/lua/lua_common.h @@ -538,8 +538,7 @@ enum lua_logger_escape_type { * @param len * @return */ -gsize lua_logger_out_type(lua_State *L, int pos, char *outbuf, - gsize len, struct lua_logger_trace *trace, +gsize lua_logger_out(lua_State *L, int pos, char *outbuf, gsize len, enum lua_logger_escape_type esc_type); /** diff --git a/src/lua/lua_config.c b/src/lua/lua_config.c index 07ed58ad5..f52eae44f 100644 --- a/src/lua/lua_config.c +++ b/src/lua/lua_config.c @@ -1,5 +1,5 @@ /* - * Copyright 2024 Vsevolod Stakhov + * Copyright 2025 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -118,7 +118,7 @@ local function foo(task) end */ /*** -* @method rspamd_config:radix_from_ucl(obj) +* @method rspamd_config:radix_from_ucl(obj, description) * Creates new embedded map of IP/mask addresses from object. * @param {ucl} obj object * @return {map} radix tree object diff --git a/src/lua/lua_logger.c b/src/lua/lua_logger.c index 004b82e72..8f2aa5be1 100644 --- a/src/lua/lua_logger.c +++ b/src/lua/lua_logger.c @@ -174,6 +174,11 @@ static const struct luaL_reg loggerlib_f[] = { {"__tostring", rspamd_lua_class_tostring}, {NULL, NULL}}; +static gsize +lua_logger_out_type(lua_State *L, int pos, char *outbuf, + gsize len, struct lua_logger_trace *trace, + enum lua_logger_escape_type esc_type); + static void lua_common_log_line(GLogLevelFlags level, lua_State *L, @@ -203,23 +208,19 @@ lua_common_log_line(GLogLevelFlags level, d.currentline); } - rspamd_common_log_function(NULL, - level, - module, - uid, - func_buf, - "%s", - msg); + p = func_buf; } else { - rspamd_common_log_function(NULL, - level, - module, - uid, - G_STRFUNC, - "%s", - msg); + p = (char *) G_STRFUNC; } + + rspamd_common_log_function(NULL, + level, + module, + uid, + p, + "%s", + msg); } /*** Logger interface ***/ @@ -279,105 +280,139 @@ lua_logger_char_safe(int t, unsigned int esc_type) return true; } +/* Could return negative value in case of wrong argument number */ +static glong +lua_logger_log_format_str(lua_State *L, int offset, char *logbuf, gsize remain, + const char *fmt, + enum lua_logger_escape_type esc_type) +{ + const char *c; + gsize r; + int digit; + + char *d = logbuf; + unsigned int arg_num, cur_arg = 0, arg_max = lua_gettop(L) - offset; + + while (remain > 1 && *fmt) { + if (*fmt == '%') { + ++fmt; + c = fmt; + if (*fmt == 's') { + ++fmt; + ++cur_arg; + } else { + arg_num = 0; + while ((digit = g_ascii_digit_value(*fmt)) >= 0) { + ++fmt; + arg_num = arg_num * 10 + digit; + if (arg_num >= 100) { + /* Avoid ridiculously large numbers */ + fmt = c; + break; + } + } + + if (fmt > c) { + /* Update the current argument */ + cur_arg = arg_num; + } + } + + if (fmt > c) { + if (cur_arg < 1 || cur_arg > arg_max) { + *d = 0; + return -((glong) cur_arg + 1); /* wrong argument number */ + } + + r = lua_logger_out(L, offset + cur_arg, d, remain, esc_type); + g_assert(r < remain); + remain -= r; + d += r; + continue; + } + + /* Copy % */ + --fmt; + } + + *d++ = *fmt++; + --remain; + } + + *d = 0; + + return d - logbuf; +} + static gsize lua_logger_out_str(lua_State *L, int pos, char *outbuf, gsize len, - struct lua_logger_trace *trace, enum lua_logger_escape_type esc_type) { - gsize slen, flen; - const char *str = lua_tolstring(L, pos, &slen); static const char hexdigests[16] = "0123456789abcdef"; - gsize r = 0, s; + gsize slen; + const unsigned char *str = lua_tolstring(L, pos, &slen); + unsigned char c; + char *out = outbuf; if (str) { - gboolean normal = TRUE; - flen = MIN(slen, len - 1); - - for (r = 0; r < flen; r++) { - if (!lua_logger_char_safe(str[r], esc_type)) { - normal = FALSE; - break; + while (slen > 0 && len > 1) { + c = *str++; + if (lua_logger_char_safe(c, esc_type)) { + *out++ = c; } - } - - if (normal) { - r = rspamd_strlcpy(outbuf, str, flen + 1); - } - else { - /* Need to escape non-printed characters */ - r = 0; - s = 0; - - while (slen > 0 && len > 1) { - if (!lua_logger_char_safe(str[s], esc_type)) { - if (len >= 3) { - outbuf[r++] = '\\'; - outbuf[r++] = hexdigests[((str[s] >> 4) & 0xF)]; - outbuf[r++] = hexdigests[((str[s]) & 0xF)]; - - len -= 2; - } - else { - outbuf[r++] = '?'; - } - } - else { - outbuf[r++] = str[s]; - } - - s++; - slen--; - len--; + else if (len > 3) { + /* Need to escape non-printed characters */ + *out++ = '\\'; + *out++ = hexdigests[c >> 4]; + *out++ = hexdigests[c & 0xF]; + len -= 2; } - - outbuf[r] = '\0'; + else { + *out++ = '?'; + } + --slen; + --len; } } + *out = 0; - return r; + return out - outbuf; } static gsize -lua_logger_out_num(lua_State *L, int pos, char *outbuf, gsize len, - struct lua_logger_trace *trace) +lua_logger_out_num(lua_State *L, int pos, char *outbuf, gsize len) { double num = lua_tonumber(L, pos); - glong inum; - gsize r = 0; + glong inum = (glong) num; - if ((double) (glong) num == num) { - inum = num; - r = rspamd_snprintf(outbuf, len + 1, "%l", inum); - } - else { - r = rspamd_snprintf(outbuf, len + 1, "%f", num); + if ((double) inum == num) { + return rspamd_snprintf(outbuf, len, "%l", inum); } - return r; + return rspamd_snprintf(outbuf, len, "%f", num); } static gsize -lua_logger_out_boolean(lua_State *L, int pos, char *outbuf, gsize len, - struct lua_logger_trace *trace) +lua_logger_out_boolean(lua_State *L, int pos, char *outbuf, gsize len) { gboolean val = lua_toboolean(L, pos); - gsize r = 0; - - r = rspamd_strlcpy(outbuf, val ? "true" : "false", len + 1); - return r; + return rspamd_snprintf(outbuf, len, val ? "true" : "false"); } static gsize -lua_logger_out_userdata(lua_State *L, int pos, char *outbuf, gsize len, - struct lua_logger_trace *trace) +lua_logger_out_userdata(lua_State *L, int pos, char *outbuf, gsize len) { - int r = 0, top; + gsize r = 0; + int top; const char *str = NULL; gboolean converted_to_str = FALSE; top = lua_gettop(L); + if (pos < 0) { + pos += top + 1; /* Convert to absolute */ + } if (!lua_getmetatable(L, pos)) { return 0; @@ -396,26 +431,17 @@ lua_logger_out_userdata(lua_State *L, int pos, char *outbuf, gsize len, if (lua_isfunction(L, -1)) { lua_pushvalue(L, pos); - if (lua_pcall(L, 1, 1, 0) != 0) { - lua_settop(L, top); - - return 0; - } - - str = lua_tostring(L, -1); - - if (str) { - r = rspamd_snprintf(outbuf, len, "%s", str); + if (lua_pcall(L, 1, 1, 0) == 0) { + str = lua_tostring(L, -1); + if (str) { + r = rspamd_snprintf(outbuf, len, "%s", str); + } } - - lua_settop(L, top); - - return r; } } lua_settop(L, top); - return 0; + return r; } lua_pushstring(L, "__tostring"); @@ -460,12 +486,12 @@ lua_logger_out_userdata(lua_State *L, int pos, char *outbuf, gsize len, return r; } -#define MOVE_BUF(d, remain, r) \ +#define MOVE_BUF(d, remain, r) \ (d) += (r); \ (remain) -= (r); \ - if ((remain) == 0) { \ - lua_settop(L, old_top); \ - break; \ + if ((remain) <= 1) { \ + lua_settop(L, top); \ + goto table_oob; \ } static gsize @@ -473,169 +499,154 @@ lua_logger_out_table(lua_State *L, int pos, char *outbuf, gsize len, struct lua_logger_trace *trace, enum lua_logger_escape_type esc_type) { - char *d = outbuf; - gsize remain = len, r; + char *d = outbuf, *str; + gsize remain = len; + glong r; gboolean first = TRUE; gconstpointer self = NULL; - int i, tpos, last_seq = -1, old_top; + int i, last_seq = 0, top; + double num; + glong inum; - if (!lua_istable(L, pos) || remain == 0) { - return 0; - } + /* Type and length checks are done in logger_out_type() */ - old_top = lua_gettop(L); self = lua_topointer(L, pos); /* Check if we have seen this pointer */ for (i = 0; i < TRACE_POINTS; i++) { if (trace->traces[i] == self) { - r = rspamd_snprintf(d, remain + 1, "ref(%p)", self); - - d += r; - - return (d - outbuf); + if ((trace->cur_level + TRACE_POINTS - 1) % TRACE_POINTS == i) { + return rspamd_snprintf(d, remain, "__self"); + } + return rspamd_snprintf(d, remain, "ref(%p)", self); } } trace->traces[trace->cur_level % TRACE_POINTS] = self; + ++trace->cur_level; - lua_pushvalue(L, pos); - r = rspamd_snprintf(d, remain + 1, "{"); - remain -= r; - d += r; + top = lua_gettop(L); + if (pos < 0) { + pos += top + 1; /* Convert to absolute */ + } + + r = rspamd_snprintf(d, remain, "{"); + MOVE_BUF(d, remain, r); /* Get numeric keys (ipairs) */ for (i = 1;; i++) { - lua_rawgeti(L, -1, i); + lua_rawgeti(L, pos, i); if (lua_isnil(L, -1)) { lua_pop(L, 1); + last_seq = i; break; } - last_seq = i; - - if (!first) { - r = rspamd_snprintf(d, remain + 1, ", "); - MOVE_BUF(d, remain, r); + if (first) { + first = FALSE; + str = "[%d] = "; + } else { + str = ", [%d] = "; } - - r = rspamd_snprintf(d, remain + 1, "[%d] = ", i); + r = rspamd_snprintf(d, remain, str, i); MOVE_BUF(d, remain, r); - tpos = lua_gettop(L); - if (lua_topointer(L, tpos) == self) { - r = rspamd_snprintf(d, remain + 1, "__self"); - } - else { - r = lua_logger_out_type(L, tpos, d, remain, trace, esc_type); - } + r = lua_logger_out_type(L, -1, d, remain, trace, esc_type); MOVE_BUF(d, remain, r); - first = FALSE; lua_pop(L, 1); } /* Get string keys (pairs) */ - for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) { + for (lua_pushnil(L); lua_next(L, pos); lua_pop(L, 1)) { /* 'key' is at index -2 and 'value' is at index -1 */ - if (lua_type(L, -2) == LUA_TNUMBER) { - if (last_seq > 0) { - lua_pushvalue(L, -2); - if (lua_tonumber(L, -1) <= last_seq + 1) { - lua_pop(L, 1); + /* Preserve key */ + lua_pushvalue(L, -2); + if (last_seq > 0) { + if (lua_type(L, -1) == LUA_TNUMBER) { + num = lua_tonumber(L, -1); /* no conversion here */ + inum = (glong) num; + if ((double) inum == num && inum > 0 && inum < last_seq) { /* Already seen */ + lua_pop(L, 1); continue; } - - lua_pop(L, 1); } } - if (!first) { - r = rspamd_snprintf(d, remain + 1, ", "); - MOVE_BUF(d, remain, r); + if (first) { + first = FALSE; + str = "[%2] = %1"; + } else { + str = ", [%2] = %1"; } - - /* Preserve key */ - lua_pushvalue(L, -2); - r = rspamd_snprintf(d, remain + 1, "[%s] = ", - lua_tostring(L, -1)); - lua_pop(L, 1); /* Remove key */ - MOVE_BUF(d, remain, r); - tpos = lua_gettop(L); - - if (lua_topointer(L, tpos) == self) { - r = rspamd_snprintf(d, remain + 1, "__self"); - } - else { - r = lua_logger_out_type(L, tpos, d, remain, trace, esc_type); + r = lua_logger_log_format_str(L, top + 1, d, remain, str, esc_type); + if (r < 0) { + /* should not happen */ + goto table_oob; } MOVE_BUF(d, remain, r); - first = FALSE; + /* Remove key */ + lua_pop(L, 1); } - lua_settop(L, old_top); - - r = rspamd_snprintf(d, remain + 1, "}"); + r = rspamd_snprintf(d, remain, "}"); d += r; +table_oob: + --trace->cur_level; + return (d - outbuf); } #undef MOVE_BUF -gsize lua_logger_out_type(lua_State *L, int pos, +static gsize +lua_logger_out_type(lua_State *L, int pos, char *outbuf, gsize len, struct lua_logger_trace *trace, enum lua_logger_escape_type esc_type) { - int type; - gsize r = 0; - if (len == 0) { return 0; } - type = lua_type(L, pos); - trace->cur_level++; + int type = lua_type(L, pos); switch (type) { case LUA_TNUMBER: - r = lua_logger_out_num(L, pos, outbuf, len, trace); - break; + return lua_logger_out_num(L, pos, outbuf, len); case LUA_TBOOLEAN: - r = lua_logger_out_boolean(L, pos, outbuf, len, trace); - break; + return lua_logger_out_boolean(L, pos, outbuf, len); case LUA_TTABLE: - r = lua_logger_out_table(L, pos, outbuf, len, trace, esc_type); - break; + return lua_logger_out_table(L, pos, outbuf, len, trace, esc_type); case LUA_TUSERDATA: - r = lua_logger_out_userdata(L, pos, outbuf, len, trace); - break; + return lua_logger_out_userdata(L, pos, outbuf, len); case LUA_TFUNCTION: - r = rspamd_snprintf(outbuf, len + 1, "function"); - break; + return rspamd_snprintf(outbuf, len, "function"); case LUA_TLIGHTUSERDATA: - r = rspamd_snprintf(outbuf, len + 1, "0x%p", lua_topointer(L, pos)); - break; + return rspamd_snprintf(outbuf, len, "0x%p", lua_topointer(L, pos)); case LUA_TNIL: - r = rspamd_snprintf(outbuf, len + 1, "nil"); - break; + return rspamd_snprintf(outbuf, len, "nil"); case LUA_TNONE: - r = rspamd_snprintf(outbuf, len + 1, "no value"); - break; - default: - /* Try to push everything as string using tostring magic */ - r = lua_logger_out_str(L, pos, outbuf, len, trace, esc_type); - break; + return rspamd_snprintf(outbuf, len, "no value"); } - trace->cur_level--; + /* Try to push everything as string using tostring magic */ + return lua_logger_out_str(L, pos, outbuf, len, esc_type); +} - return r; +gsize lua_logger_out(lua_State *L, int pos, + char *outbuf, gsize len, + enum lua_logger_escape_type esc_type) +{ + struct lua_logger_trace tr; + memset(&tr, 0, sizeof(tr)); + + return lua_logger_out_type(L, pos, outbuf, len, &tr, esc_type); } static const char * @@ -731,72 +742,16 @@ static gboolean lua_logger_log_format(lua_State *L, int fmt_pos, gboolean is_string, char *logbuf, gsize remain) { - char *d; - const char *s, *c; - gsize r; - unsigned int arg_num, arg_max, cur_arg; - struct lua_logger_trace tr; - int digit; - - s = lua_tostring(L, fmt_pos); - if (s == NULL) { + const char *fmt = lua_tostring(L, fmt_pos); + if (fmt == NULL) { return FALSE; } - arg_max = (unsigned int) lua_gettop(L) - fmt_pos; - d = logbuf; - cur_arg = 0; - - while (remain > 0 && *s) { - if (*s == '%') { - ++s; - c = s; - if (*s == 's') { - ++s; - ++cur_arg; - } else { - arg_num = 0; - while ((digit = g_ascii_digit_value(*s)) >= 0) { - ++s; - arg_num = arg_num * 10 + digit; - if (arg_num >= 100) { - /* Avoid ridiculously large numbers */ - s = c; - break; - } - } - - if (s > c) { - /* Update the current argument */ - cur_arg = arg_num; - } - } - - if (s > c) { - if (cur_arg < 1 || cur_arg > arg_max) { - msg_err("wrong argument number: %ud", cur_arg); - return FALSE; - } - - memset(&tr, 0, sizeof(tr)); - r = lua_logger_out_type(L, fmt_pos + cur_arg, d, remain, &tr, - is_string ? LUA_ESCAPE_UNPRINTABLE : LUA_ESCAPE_LOG); - g_assert(r <= remain); - remain -= r; - d += r; - continue; - } - - /* Copy % */ - --s; - } - - *d++ = *s++; - --remain; + glong ret = lua_logger_log_format_str(L, fmt_pos, logbuf, remain, fmt, is_string ? LUA_ESCAPE_UNPRINTABLE : LUA_ESCAPE_LOG); + if (ret < 0) { + msg_err("wrong argument number: %ud", -((int) ret + 1)); + return FALSE; } - - *d = '\0'; - return TRUE; } @@ -808,15 +763,10 @@ lua_logger_do_log(lua_State *L, { char logbuf[RSPAMD_LOGBUF_SIZE - 128]; const char *uid = NULL; - int fmt_pos = start_pos; int ret; - GError *err = NULL; - if (lua_type(L, start_pos) == LUA_TSTRING) { - fmt_pos = start_pos; - } - else if (lua_type(L, start_pos) == LUA_TUSERDATA) { - fmt_pos = start_pos + 1; + if (lua_type(L, start_pos) == LUA_TUSERDATA) { + GError *err = NULL; uid = lua_logger_get_id(L, start_pos, &err); @@ -830,15 +780,17 @@ lua_logger_do_log(lua_State *L, return ret; } + + ++start_pos; } - else { + + if (lua_type(L, start_pos) != LUA_TSTRING) { /* Bad argument type */ return luaL_error(L, "bad format string type: %s", lua_typename(L, lua_type(L, start_pos))); } - ret = lua_logger_log_format(L, fmt_pos, is_string, - logbuf, sizeof(logbuf) - 1); + ret = lua_logger_log_format(L, start_pos, is_string, logbuf, sizeof(logbuf)); if (ret) { if (is_string) { @@ -849,12 +801,9 @@ lua_logger_do_log(lua_State *L, lua_common_log_line(level, L, logbuf, uid, "lua", 1); } } - else { - if (is_string) { - lua_pushnil(L); - - return 1; - } + else if (is_string) { + lua_pushnil(L); + return 1; } return 0; @@ -917,11 +866,11 @@ lua_logger_logx(lua_State *L) if (uid && modname) { if (lua_type(L, 4) == LUA_TSTRING) { - ret = lua_logger_log_format(L, 4, FALSE, logbuf, sizeof(logbuf) - 1); + ret = lua_logger_log_format(L, 4, FALSE, logbuf, sizeof(logbuf)); } else if (lua_type(L, 4) == LUA_TNUMBER) { stack_pos = lua_tonumber(L, 4); - ret = lua_logger_log_format(L, 5, FALSE, logbuf, sizeof(logbuf) - 1); + ret = lua_logger_log_format(L, 5, FALSE, logbuf, sizeof(logbuf)); } else { return luaL_error(L, "invalid argument on pos 4"); @@ -959,11 +908,11 @@ lua_logger_debugm(lua_State *L) if (uid && module) { if (lua_type(L, 3) == LUA_TSTRING) { - ret = lua_logger_log_format(L, 3, FALSE, logbuf, sizeof(logbuf) - 1); + ret = lua_logger_log_format(L, 3, FALSE, logbuf, sizeof(logbuf)); } else if (lua_type(L, 3) == LUA_TNUMBER) { stack_pos = lua_tonumber(L, 3); - ret = lua_logger_log_format(L, 4, FALSE, logbuf, sizeof(logbuf) - 1); + ret = lua_logger_log_format(L, 4, FALSE, logbuf, sizeof(logbuf)); } else { return luaL_error(L, "invalid argument on pos 3"); diff --git a/src/lua/lua_map.c b/src/lua/lua_map.c index 062613bd7..5f55ece06 100644 --- a/src/lua/lua_map.c +++ b/src/lua/lua_map.c @@ -1,5 +1,5 @@ /* - * Copyright 2024 Vsevolod Stakhov + * Copyright 2025 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -319,6 +319,11 @@ int lua_config_radix_from_ucl(lua_State *L) ucl_object_insert_key(fake_obj, ucl_object_fromstring("static"), "url", 0, false); + if (lua_type(L, 3) == LUA_TSTRING) { + ucl_object_insert_key(fake_obj, ucl_object_fromstring(lua_tostring(L, 3)), + "description", 0, false); + } + if ((m = rspamd_map_add_from_ucl(cfg, fake_obj, "static radix map", rspamd_radix_read, rspamd_radix_fin, diff --git a/src/plugins/lua/contextal.lua b/src/plugins/lua/contextal.lua new file mode 100644 index 000000000..f6202781a --- /dev/null +++ b/src/plugins/lua/contextal.lua @@ -0,0 +1,332 @@ +--[[ +Copyright (c) 2025, Vsevolod Stakhov <vsevolod@rspamd.com> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +local E = {} +local N = 'contextal' + +if confighelp then + return +end + +local opts = rspamd_config:get_all_opt(N) +if not opts then + return +end + +local lua_redis = require "lua_redis" +local lua_util = require "lua_util" +local redis_cache = require "lua_cache" +local rspamd_http = require "rspamd_http" +local rspamd_logger = require "rspamd_logger" +local rspamd_util = require "rspamd_util" +local ts = require("tableshape").types +local ucl = require "ucl" + +local cache_context, redis_params + +local contextal_actions = { + ['ALERT'] = true, + ['ALLOW'] = true, + ['BLOCK'] = true, + ['QUARANTINE'] = true, + ['SPAM'] = true, +} + +local config_schema = lua_redis.enrich_schema { + action_symbol_prefix = ts.string:is_optional(), + base_url = ts.string:is_optional(), + cache_prefix = ts.string:is_optional(), + cache_timeout = ts.number:is_optional(), + cache_ttl = ts.number:is_optional(), + custom_actions = ts.array_of(ts.string):is_optional(), + defer_if_no_result = ts.boolean:is_optional(), + defer_message = ts.string:is_optional(), + enabled = ts.boolean:is_optional(), + http_timeout = ts.number:is_optional(), + request_ttl = ts.number:is_optional(), + submission_symbol = ts.string:is_optional(), +} + +local settings = { + action_symbol_prefix = 'CONTEXTAL_ACTION', + base_url = 'http://localhost:8080', + cache_prefix = 'CXAL', + cache_timeout = 5, + cache_ttl = 3600, + custom_actions = {}, + defer_if_no_result = false, + defer_message = 'Awaiting deep scan - try again later', + http_timeout = 2, + request_ttl = 4, + submission_symbol = 'CONTEXTAL_SUBMIT', +} + +local static_boundary = rspamd_util.random_hex(32) +local use_request_ttl = true + +local function maybe_defer(task, obj) + if settings.defer_if_no_result and not ((obj or E)[1] or E).actions then + task:set_pre_result('soft reject', settings.defer_message) + end +end + +local function process_actions(task, obj, is_cached) + for _, match in ipairs((obj[1] or E).actions or E) do + local act = match.action + local scenario = match.scenario + if not (act and scenario) then + rspamd_logger.err(task, 'bad result: %s', match) + elseif contextal_actions[act] then + task:insert_result(settings.action_symbol_prefix .. '_' .. act, 1.0, scenario) + else + rspamd_logger.err(task, 'unknown action: %s', act) + end + end + + if not cache_context or is_cached then + maybe_defer(task, obj) + return + end + + local cache_obj + if (obj[1] or E).actions then + cache_obj = {[1] = {["actions"] = obj[1].actions}} + else + local work_id = task:get_mempool():get_variable('contextal_work_id', 'string') + if work_id then + cache_obj = {[1] = {["work_id"] = work_id}} + else + rspamd_logger.err(task, 'no work id found in mempool') + return + end + end + + redis_cache.cache_set(task, + task:get_digest(), + cache_obj, + cache_context) + + maybe_defer(task, obj) +end + +local function process_cached(task, obj) + if (obj[1] or E).actions then + task:disable_symbol(settings.action_symbol_prefix) + return process_actions(task, obj, true) + elseif (obj[1] or E).work_id then + task:get_mempool():set_variable('contextal_work_id', obj[1].work_id) + else + rspamd_logger.err(task, 'bad result (cached): %s', obj) + end +end + +local function action_cb(task) + local work_id = task:get_mempool():get_variable('contextal_work_id', 'string') + if not work_id then + rspamd_logger.err(task, 'no work id found in mempool') + return + end + + local function http_callback(err, code, body, hdrs) + if err then + rspamd_logger.err(task, 'http error: %s', err) + maybe_defer(task) + return + end + if code ~= 200 then + rspamd_logger.err(task, 'bad http code: %s', code) + maybe_defer(task) + return + end + local parser = ucl.parser() + local _, parse_err = parser:parse_string(body) + if parse_err then + rspamd_logger.err(task, 'cannot parse JSON: %s', err) + maybe_defer(task) + return + end + local obj = parser:get_object() + return process_actions(task, obj, false) + end + + rspamd_http.request({ + task = task, + url = settings.actions_url .. work_id, + callback = http_callback, + timeout = settings.http_timeout, + gzip = settings.gzip, + keepalive = settings.keepalive, + no_ssl_verify = settings.no_ssl_verify, + }) +end + +local function submit(task) + + local function http_callback(err, code, body, hdrs) + if err then + rspamd_logger.err(task, 'http error: %s', err) + maybe_defer(task) + return + end + if code ~= 201 then + rspamd_logger.err(task, 'bad http code: %s', code) + maybe_defer(task) + return + end + local parser = ucl.parser() + local _, parse_err = parser:parse_string(body) + if parse_err then + rspamd_logger.err(task, 'cannot parse JSON: %s', err) + maybe_defer(task) + return + end + local obj = parser:get_object() + local work_id = obj.work_id + if work_id then + task:get_mempool():set_variable('contextal_work_id', work_id) + end + task:insert_result(settings.submission_symbol, 1.0, + string.format('work_id=%s', work_id or 'nil')) + task:add_timer(settings.request_ttl, action_cb) + end + + local req = { + object_data = {['data'] = task:get_content()}, + } + if settings.request_ttl then + req.ttl = {['data'] = tostring(settings.request_ttl)} + end + if settings.max_recursion then + req.maxrec = {['data'] = tostring(settings.max_recursion)} + end + rspamd_http.request({ + task = task, + url = settings.submit_url, + body = lua_util.table_to_multipart_body(req, static_boundary), + callback = http_callback, + headers = { + ['Content-Type'] = string.format('multipart/form-data; boundary="%s"', static_boundary) + }, + timeout = settings.http_timeout, + gzip = settings.gzip, + keepalive = settings.keepalive, + no_ssl_verify = settings.no_ssl_verify, + }) +end + +local function cache_hit(task, err, data) + if err then + rspamd_logger.err(task, 'error getting cache: %s', err) + else + process_cached(task, data) + end +end + +local function submit_cb(task) + if cache_context then + redis_cache.cache_get(task, + task:get_digest(), + cache_context, + settings.cache_timeout, + submit, + cache_hit + ) + else + submit(task) + end +end + +local function set_url_path(base, path) + local slash = base:sub(#base) == '/' and '' or '/' + return base .. slash .. path +end + +settings = lua_util.override_defaults(settings, opts) + +local res, err = config_schema:transform(settings) +if not res then + rspamd_logger.warnx(rspamd_config, 'plugin %s is misconfigured: %s', N, err) + local err_msg = string.format("schema error: %s", res) + lua_util.config_utils.push_config_error(N, err_msg) + lua_util.disable_module(N, "failed", err_msg) + return +end + +for _, k in ipairs(settings.custom_actions) do + contextal_actions[k] = true +end + +if not settings.base_url then + if not (settings.submit_url and settings.actions_url) then + rspamd_logger.err(rspamd_config, 'no URL configured for contextal') + lua_util.disable_module(N, 'config') + return + end +else + if not settings.submit_url then + settings.submit_url = set_url_path(settings.base_url, 'api/v1/submit') + end + if not settings.actions_url then + settings.actions_url = set_url_path(settings.base_url, 'api/v1/actions/') + end +end + +redis_params = lua_redis.parse_redis_server(N) +if redis_params then + cache_context = redis_cache.create_cache_context(redis_params, { + cache_prefix = settings.cache_prefix, + cache_ttl = settings.cache_ttl, + cache_format = 'json', + cache_use_hashing = false + }) +end + +local submission_id = rspamd_config:register_symbol({ + name = settings.submission_symbol, + type = 'normal', + group = N, + callback = submit_cb +}) + +local top_options = rspamd_config:get_all_opt('options') +if settings.request_ttl and settings.request_ttl >= (top_options.task_timeout * 0.8) then + rspamd_logger.warn(rspamd_config, [[request ttl is >= 80% of task timeout, won't wait on processing]]) + use_request_ttl = false +elseif not settings.request_ttl then + use_request_ttl = false +end + +local parent_id +if use_request_ttl then + parent_id = submission_id +else + parent_id = rspamd_config:register_symbol({ + name = settings.action_symbol_prefix, + type = 'postfilter', + priority = lua_util.symbols_priorities.high - 1, + group = N, + callback = action_cb + }) +end + +for k in pairs(contextal_actions) do + rspamd_config:register_symbol({ + name = settings.action_symbol_prefix .. '_' .. k, + parent = parent_id, + type = 'virtual', + group = N, + }) +end diff --git a/src/plugins/lua/greylist.lua b/src/plugins/lua/greylist.lua index e4a633233..934e17bce 100644 --- a/src/plugins/lua/greylist.lua +++ b/src/plugins/lua/greylist.lua @@ -122,6 +122,29 @@ local function data_key(task) local h = hash.create() h:update(body, len) + local subject = task:get_subject() or '' + h:update(subject) + + -- Take recipients into account + local rcpt = task:get_recipients('smtp') + if rcpt then + table.sort(rcpt, function(r1, r2) + return r1['addr'] < r2['addr'] + end) + + fun.each(function(r) + h:update(r['addr']) + end, rcpt) + end + + -- Use from as well, but mime one + local from = task:get_from('mime') + + local addr = '<>' + if from and from[1] then + addr = from[1]['addr'] + end + h:update(addr) local b32 = settings['key_prefix'] .. 'b' .. h:base32():sub(1, 20) task:get_mempool():set_variable("grey_bodyhash", b32) diff --git a/src/plugins/lua/hfilter.lua b/src/plugins/lua/hfilter.lua index 6bc011b83..a783565ab 100644 --- a/src/plugins/lua/hfilter.lua +++ b/src/plugins/lua/hfilter.lua @@ -199,9 +199,10 @@ local function check_regexp(str, regexp_text) return re:match(str) end -local function add_static_map(data) +local function add_static_map(data, description) return rspamd_config:add_map { type = 'regexp_multi', + description = description, url = { upstreams = 'static', data = data, @@ -568,16 +569,16 @@ local function append_t(t, a) end end if config['helo_enabled'] then - checks_hello_bareip_map = add_static_map(checks_hello_bareip) - checks_hello_badip_map = add_static_map(checks_hello_badip) - checks_hellohost_map = add_static_map(checks_hellohost) - checks_hello_map = add_static_map(checks_hello) + checks_hello_bareip_map = add_static_map(checks_hello_bareip, 'Hfilter: HELO bare ip') + checks_hello_badip_map = add_static_map(checks_hello_badip, 'Hfilter: HELO bad ip') + checks_hellohost_map = add_static_map(checks_hellohost, 'Hfilter: HELO host') + checks_hello_map = add_static_map(checks_hello, 'Hfilter: HELO') append_t(symbols_enabled, symbols_helo) timeout = math.max(timeout, rspamd_config:get_dns_timeout() * 3) end if config['hostname_enabled'] then if not checks_hellohost_map then - checks_hellohost_map = add_static_map(checks_hellohost) + checks_hellohost_map = add_static_map(checks_hellohost, 'Hfilter: HOSTNAME') end append_t(symbols_enabled, symbols_hostname) timeout = math.max(timeout, rspamd_config:get_dns_timeout()) diff --git a/src/rspamadm/lua_repl.c b/src/rspamadm/lua_repl.c index 1d6da5aa9..9ad790d20 100644 --- a/src/rspamadm/lua_repl.c +++ b/src/rspamadm/lua_repl.c @@ -1,5 +1,5 @@ /* - * Copyright 2024 Vsevolod Stakhov + * Copyright 2025 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -232,7 +232,6 @@ rspamadm_exec_input(lua_State *L, const char *input) int i, cbref; int top = 0; char outbuf[8192]; - struct lua_logger_trace tr; struct thread_entry *thread = lua_thread_pool_get_for_config(rspamd_main->cfg); L = thread->lua_state; @@ -272,8 +271,7 @@ rspamadm_exec_input(lua_State *L, const char *input) rspamd_printf("local function: %d\n", cbref); } else { - memset(&tr, 0, sizeof(tr)); - lua_logger_out_type(L, i, outbuf, sizeof(outbuf) - 1, &tr, + lua_logger_out(L, i, outbuf, sizeof(outbuf), LUA_ESCAPE_UNPRINTABLE); rspamd_printf("%s\n", outbuf); } @@ -393,7 +391,6 @@ rspamadm_lua_message_handler(lua_State *L, int argc, char **argv) gpointer map; gsize len; char outbuf[8192]; - struct lua_logger_trace tr; if (argv[1] == NULL) { rspamd_printf("no callback is specified\n"); @@ -455,8 +452,7 @@ rspamadm_lua_message_handler(lua_State *L, int argc, char **argv) rspamd_printf("lua callback for %s returned:\n", argv[i]); for (j = old_top + 1; j <= lua_gettop(L); j++) { - memset(&tr, 0, sizeof(tr)); - lua_logger_out_type(L, j, outbuf, sizeof(outbuf), &tr, + lua_logger_out(L, j, outbuf, sizeof(outbuf), LUA_ESCAPE_UNPRINTABLE); rspamd_printf("%s\n", outbuf); } @@ -705,8 +701,8 @@ rspamadm_lua_run_repl(lua_State *L, bool is_batch) g_string_append(tb, " \n"); } } - } #endif + } } } |