} | } | ||||
timeout = 20; # Increase redis timeout | timeout = 20; # Increase redis timeout | ||||
enabled = ${HAS_TORCH}; # Explicitly disable module when torch is disabled | |||||
use_settings = false; # If enabled, then settings-id is used to dispatch networks | |||||
# Legacy support | # Legacy support | ||||
.include(try=true,priority=5) "${DBDIR}/dynamic/fann_redis.conf" | .include(try=true,priority=5) "${DBDIR}/dynamic/fann_redis.conf" |
--[[ | |||||
Copyright (c) 2017, Vsevolod Stakhov <vsevolod@highsecure.ru> | |||||
Licensed under the Apache License, Version 2.0 (the "License"); | |||||
you may not use this file except in compliance with the License. | |||||
You may obtain a copy of the License at | |||||
http://www.apache.org/licenses/LICENSE-2.0 | |||||
Unless required by applicable law or agreed to in writing, software | |||||
distributed under the License is distributed on an "AS IS" BASIS, | |||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
See the License for the specific language governing permissions and | |||||
limitations under the License. | |||||
]]-- | |||||
local rspamd_logger = require "rspamd_logger" | |||||
local torch | |||||
local exports = {} | |||||
local lua_nn_models = {} | |||||
local conf_section = rspamd_config:get_all_opt("nn_models") | |||||
if conf_section then | |||||
if rspamd_config:has_torch() then | |||||
torch = require "torch" | |||||
torch.setnumthreads(1) | |||||
end | |||||
end | |||||
if torch then | |||||
exports.load_rspamd_nn = function() | |||||
local function gen_process_callback(name) | |||||
return function(str) | |||||
if str then | |||||
local f = torch.MemoryFile(torch.CharStorage():string(str)) | |||||
local ret, tnn_or_err = pcall(function() f:readObject() end) | |||||
if not ret then | |||||
rspamd_logger.errx(rspamd_config, "cannot load neural net model %s: %s", | |||||
name, tnn_or_err) | |||||
else | |||||
rspamd_logger.infox(rspamd_config, "loaded NN model %s: %s bytes", | |||||
name, #str) | |||||
lua_nn_models[name] = tnn_or_err | |||||
end | |||||
end | |||||
end | |||||
end | |||||
if conf_section and type(conf_section) == 'table' then | |||||
for k,v in pairs(conf_section) do | |||||
if not rspamd_config:add_map(v, "nn map " .. k, gen_process_callback(k)) then | |||||
rspamd_logger.warnx(rspamd_config, 'cannot load NN map %1', k) | |||||
end | |||||
end | |||||
end | |||||
end | |||||
exports.try_rspamd_nn = function(name, input) | |||||
if not lua_nn_models.name then | |||||
return false, 0.0 | |||||
else | |||||
local ret, res_or_err = pcall(function() lua_nn_models.name:forward(input) end) | |||||
if not ret then | |||||
rspamd_logger.errx(rspamd_config, "cannot run neural net model %s: %s", | |||||
name, res_or_err) | |||||
else | |||||
return true, tonumber(res_or_err) | |||||
end | |||||
end | |||||
return false, 0.0 | |||||
end | |||||
else | |||||
exports.load_rspamd_nn = function() | |||||
end | |||||
exports.try_rspamd_nn = function(name, input) | |||||
return false,0.0 | |||||
end | |||||
end | |||||
return exports |
ucl_parser_register_variable (parser, RSPAMD_BRANCH_VERSION_MACRO, | ucl_parser_register_variable (parser, RSPAMD_BRANCH_VERSION_MACRO, | ||||
RSPAMD_VERSION_BRANCH); | RSPAMD_VERSION_BRANCH); | ||||
#if defined(WITH_TORCH) && defined(WITH_LUAJIT) && defined(__x86_64__) | |||||
ucl_parser_register_variable (parser, "HAS_TORCH", | |||||
"yes"); | |||||
#else | |||||
ucl_parser_register_variable (parser, "HAS_TORCH", | |||||
"no"); | |||||
#endif | |||||
hostlen = sysconf (_SC_HOST_NAME_MAX); | hostlen = sysconf (_SC_HOST_NAME_MAX); | ||||
if (hostlen <= 0) { | if (hostlen <= 0) { |
static gint | static gint | ||||
lua_config_has_torch (lua_State *L) | lua_config_has_torch (lua_State *L) | ||||
{ | { | ||||
LUA_TRACE_POINT; | |||||
struct rspamd_config *cfg = lua_check_config (L, 1); | |||||
struct rspamd_cryptobox_library_ctx *crypto_ctx; | |||||
if (cfg != NULL) { | |||||
crypto_ctx = cfg->libs_ctx->crypto_ctx; | |||||
#if !defined(WITH_TORCH) || !defined(WITH_LUAJIT) | |||||
lua_pushboolean (L, false); | |||||
(void)crypto_ctx; | |||||
#else | |||||
if (crypto_ctx->cpu_config & CPUID_SSE2) { | |||||
lua_pushboolean (L, true); | |||||
} | |||||
else { | |||||
lua_pushboolean (L, false); | |||||
} | |||||
#endif | |||||
} | |||||
else { | |||||
return luaL_error (L, "invalid arguments"); | |||||
} | |||||
msg_warn ("use of the obsoleted `has_torch` function"); | |||||
lua_pushboolean (L, false); | |||||
return 1; | return 1; | ||||
} | } |
local rspamd_logger = require "rspamd_logger" | local rspamd_logger = require "rspamd_logger" | ||||
local rspamd_util = require "rspamd_util" | local rspamd_util = require "rspamd_util" | ||||
local rspamd_kann = require "rspamd_kann" | |||||
local lua_redis = require "lua_redis" | local lua_redis = require "lua_redis" | ||||
local lua_util = require "lua_util" | local lua_util = require "lua_util" | ||||
local fun = require "fun" | local fun = require "fun" | ||||
local nn | local nn | ||||
local N = "neural" | local N = "neural" | ||||
if rspamd_config:has_torch() then | |||||
use_torch = true | |||||
end | |||||
-- Module vars | -- Module vars | ||||
local default_options = { | local default_options = { | ||||
train = { | train = { |