[Project] Move rspamadm libraries to a standard place

8 files changed, 10 insertions, 387 deletions

diff --git a/lualib/rspamadm/ansicolors.lua b/lualib/rspamadm/ansicolors.lua
deleted file mode 100644
index 81783f618..000000000
--- a/lualib/rspamadm/ansicolors.lua
+++ /dev/null
@@ -1,68 +0,0 @@
-local colormt = {}
-local ansicolors = {}
-
-local rspamd_util = require "rspamd_util"
-local isatty = rspamd_util.isatty()
-
-function colormt:__tostring()
-  return self.value
-end
-
-function colormt:__concat(other)
-  return tostring(self) .. tostring(other)
-end
-
-function colormt:__call(s)
-  return self .. s .. ansicolors.reset
-end
-
-colormt.__metatable = {}
-local function makecolor(value)
-  if isatty then
-    return setmetatable({
-      value = string.char(27) .. '[' .. tostring(value) .. 'm'
-    }, colormt)
-  else
-    return setmetatable({
-      value = ''
-    }, colormt)
-  end
-end
-
-local colors = {
-  -- attributes
-  reset = 0,
-  clear = 0,
-  bright = 1,
-  dim = 2,
-  underscore = 4,
-  blink = 5,
-  reverse = 7,
-  hidden = 8,
-
-  -- foreground
-  black = 30,
-  red = 31,
-  green = 32,
-  yellow = 33,
-  blue = 34,
-  magenta = 35,
-  cyan = 36,
-  white = 37,
-
-  -- background
-  onblack = 40,
-  onred = 41,
-  ongreen = 42,
-  onyellow = 43,
-  onblue = 44,
-  onmagenta = 45,
-  oncyan = 46,
-  onwhite = 47,
-}
-
-for c, v in pairs(colors) do
-  ansicolors[c] = makecolor(v)
-end
-
-return ansicolors
\ No newline at end of file
diff --git a/lualib/rspamadm/confighelp.lua b/lualib/rspamadm/confighelp.lua
index a03578b6e..8f0fd2b7d 100644
--- a/lualib/rspamadm/confighelp.lua
+++ b/lualib/rspamadm/confighelp.lua
@@ -7,8 +7,8 @@ local known_attrs = {
   default = 1,
 }
 
-local getopt = require "rspamadm/getopt"
-local ansicolors = require "rspamadm/ansicolors"
+local getopt = require "getopt"
+local ansicolors = require "ansicolors"
 
 local function maybe_print_color(key)
   if not opts['no-color'] then
diff --git a/lualib/rspamadm/configwizard.lua b/lualib/rspamadm/configwizard.lua
index 1cd275f66..542b61552 100644
--- a/lualib/rspamadm/configwizard.lua
+++ b/lualib/rspamadm/configwizard.lua
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ]]--
 
-local ansicolors = require "rspamadm/ansicolors"
+local ansicolors = require "ansicolors"
 local local_conf = rspamd_paths['CONFDIR']
 local rspamd_util = require "rspamd_util"
 local rspamd_logger = require "rspamd_logger"
@@ -23,7 +23,7 @@ local lua_stat_tools = require "lua_stat"
 local lua_redis = require "lua_redis"
 local ucl = require "ucl"
 
-local plugins_stat = require "rspamadm/plugins_stats"
+local plugins_stat = require "plugins_stats"
 
 local rspamd_logo = [[
   ____                                     _
diff --git a/lualib/rspamadm/corpus_test.lua b/lualib/rspamadm/corpus_test.lua
index cd9f66155..71d1eeaf1 100644
--- a/lualib/rspamadm/corpus_test.lua
+++ b/lualib/rspamadm/corpus_test.lua
@@ -1,7 +1,7 @@
 local rspamd_logger = require "rspamd_logger"
 local ucl = require "ucl"
 local lua_util = require "lua_util"
-local getopt = require "rspamadm/getopt"
+local getopt = require "getopt"
 
 local HAM = "HAM"
 local SPAM = "SPAM"
diff --git a/lualib/rspamadm/getopt.lua b/lualib/rspamadm/getopt.lua
deleted file mode 100644
index 20715c387..000000000
--- a/lualib/rspamadm/getopt.lua
+++ /dev/null
@@ -1,50 +0,0 @@
-local function insert_option(tab, name, value)
-  if tab[name] then
-    if type(tab[name]) == 'table' then
-      table.insert(tab[name], value)
-    else
-      local old_val = tab[name]
-      tab[name] = {
-        old_val,
-        value
-      }
-    end
-  else
-    tab[name] = value
-  end
-end
-
-local function getopt(arg, options)
-  local tab = {}
-  for k, v in ipairs(arg) do
-    if string.sub(v, 1, 2) == "--" then
-      local x = string.find(v, "=", 1, true)
-      if x then insert_option(tab, string.sub(v, 3, x - 1), string.sub(v, x + 1))
-      else tab[string.sub(v, 3)] = true
-      end
-    elseif string.sub(v, 1, 1) == "-" then
-      local y = 2
-      local l = string.len(v)
-      local jopt
-      while (y <= l) do
-        jopt = string.sub(v, y, y)
-        if string.find(options, jopt, 1, true) then
-          if y < l then
-            insert_option(tab, jopt, string.sub(v, y + 1))
-            y = l
-          else
-            insert_option(tab, jopt, arg[k + 1])
-          end
-        else
-          tab[jopt] = true
-        end
-        y = y + 1
-      end
-    end
-  end
-  return tab
-end
-
-return {
-  getopt = getopt
-}
diff --git a/lualib/rspamadm/plugins_stats.lua b/lualib/rspamadm/plugins_stats.lua
deleted file mode 100644
index 5c9797a9e..000000000
--- a/lualib/rspamadm/plugins_stats.lua
+++ /dev/null
@@ -1,48 +0,0 @@
---[[
-Copyright (c) 2017, Vsevolod Stakhov <vsevolod@highsecure.ru>
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-]]--
-
-local ansicolors = require "rspamadm/ansicolors"
-
-local function printf(fmt, ...)
-  print(string.format(fmt, ...))
-end
-
-local function highlight(str)
-  return ansicolors.white .. str .. ansicolors.reset
-end
-
-local function print_plugins_table(tbl, what)
-  local mods = {}
-  for k, _ in pairs(tbl) do
-    table.insert(mods, k)
-  end
-
-  printf("Modules %s: %s", highlight(what), table.concat(mods, ", "))
-end
-
-return function(args, _)
-  print_plugins_table(rspamd_plugins_state.enabled, "enabled")
-  print_plugins_table(rspamd_plugins_state.disabled_explicitly,
-      "disabled (explicitly)")
-  print_plugins_table(rspamd_plugins_state.disabled_unconfigured,
-      "disabled (unconfigured)")
-  print_plugins_table(rspamd_plugins_state.disabled_redis,
-      "disabled (no Redis)")
-  print_plugins_table(rspamd_plugins_state.disabled_experimental,
-      "disabled (experimental)")
-  print_plugins_table(rspamd_plugins_state.disabled_failed,
-      "disabled (failed)")
-end
\ No newline at end of file
diff --git a/lualib/rspamadm/rescore.lua b/lualib/rspamadm/rescore.lua
index 231cf2ea6..e6df3b364 100644
--- a/lualib/rspamadm/rescore.lua
+++ b/lualib/rspamadm/rescore.lua
@@ -3,11 +3,11 @@ local nn = require "nn"
 local lua_util = require "lua_util"
 local ucl = require "ucl"
 local logger = require "rspamd_logger"
-local getopt = require "rspamadm/getopt"
+local getopt = require "getopt"
 local optim = require "optim"
 local rspamd_util = require "rspamd_util"
 
-local rescore_utility = require "rspamadm/rescore_utility"
+local rescore_utility = require "rescore_utility"
 
 local opts
 local ignore_symbols = {
@@ -473,8 +473,8 @@ return function (args, cfg)
                      "NAME", "HITS", "HAM", "HAM%", "SPAM", "SPAM%", "S/O", "OVER%"))
       for _, symbol_stats in pairs(t) do
           logger.message(
-              string.format("%-40s %6d %6d %6.2f %6d %6.2f %6.2f %6.2f", 
-                  symbol_stats.name, 
+              string.format("%-40s %6d %6d %6.2f %6d %6.2f %6.2f %6.2f",
+                  symbol_stats.name,
                   symbol_stats.no_of_hits,
                   symbol_stats.ham_hits,
                   lua_util.round(symbol_stats.ham_percent,2),
@@ -503,7 +503,7 @@ return function (args, cfg)
           -- Calculate percentage of rules with no hits
           local nhpct = lua_util.round((#symbols_no_hits/total_symbols)*100,2)
           logger.message(
-              string.format('\nFound %s (%-.2f%%) symbols out of %s with no hits in corpus:', 
+              string.format('\nFound %s (%-.2f%%) symbols out of %s with no hits in corpus:',
                             #symbols_no_hits, nhpct, total_symbols
               )
           )
diff --git a/lualib/rspamadm/rescore_utility.lua b/lualib/rspamadm/rescore_utility.lua
deleted file mode 100644
index 2a9372d4e..000000000
--- a/lualib/rspamadm/rescore_utility.lua
+++ /dev/null
@@ -1,211 +0,0 @@
-local lua_util = require "lua_util"
-local rspamd_util = require "rspamd_util"
-local fun = require "fun"
-
-local utility = {}
-
-function utility.get_all_symbols(logs, ignore_symbols)
-  -- Returns a list of all symbols
-
-  local symbols_set = {}
-
-  for _, line in pairs(logs) do
-    line = lua_util.rspamd_str_split(line, " ")
-    for i=4,(#line-2) do
-      line[i] = line[i]:gsub("%s+", "")
-      if not symbols_set[line[i]] then
-        symbols_set[line[i]] = true
-      end
-    end
-  end
-
-  local all_symbols = {}
-
-  for symbol, _ in pairs(symbols_set) do
-    if not ignore_symbols[symbol] then
-      all_symbols[#all_symbols + 1] = symbol
-    end
-  end
-
-  table.sort(all_symbols)
-
-  return all_symbols
-end
-
-function utility.read_log_file(file)
-
-  local lines = {}
-
-  file = assert(io.open(file, "r"))
-
-  for line in file:lines() do
-    lines[#lines + 1] = line
-  end
-
-  io.close(file)
-
-  return lines
-end
-
-function utility.get_all_logs(dir_path)
-  -- Reads all log files in the directory and returns a list of logs.
-
-  if dir_path:sub(#dir_path, #dir_path) == "/" then
-    dir_path = dir_path:sub(1, #dir_path -1)
-  end
-
-  local files = rspamd_util.glob(dir_path .. "/*.log")
-  local all_logs = {}
-
-  for _, file in pairs(files) do
-    local logs = utility.read_log_file(file)
-    for _, log_line in pairs(logs) do
-      all_logs[#all_logs + 1] = log_line
-    end
-  end
-
-  return all_logs
-end
-
-function utility.get_all_symbol_scores(conf, ignore_symbols)
-  local counters = conf:get_symbols_counters()
-
-  return fun.tomap(fun.map(function(elt)
-    return elt['symbol'],elt['weight']
-  end, fun.filter(function(elt)
-    return not ignore_symbols[elt['symbol']]
-  end, counters)))
-end
-
-function utility.generate_statistics_from_logs(logs, threshold)
-
-  -- Returns file_stats table and list of symbol_stats table.
-
-  local file_stats = {
-    no_of_emails = 0,
-    no_of_spam = 0,
-    no_of_ham = 0,
-    spam_percent = 0,
-    ham_percent = 0,
-    true_positives = 0,
-    true_negatives = 0,
-    false_negative_rate = 0,
-    false_positive_rate = 0,
-    overall_accuracy = 0,
-    fscore = 0,
-    avg_scan_time = 0,
-    slowest_file = nil,
-    slowest = 0
-  }
-
-  local all_symbols_stats = {}
-  local all_fps = {}
-  local all_fns = {}
-
-  local false_positives = 0
-  local false_negatives = 0
-  local true_positives = 0
-  local true_negatives = 0
-  local no_of_emails = 0
-  local no_of_spam = 0
-  local no_of_ham = 0
-
-  for _, log in pairs(logs) do
-    log = lua_util.rspamd_str_trim(log)
-    log = lua_util.rspamd_str_split(log, " ")
-
-    local is_spam = (log[1] == "SPAM")
-    local score = tonumber(log[2])
-
-    no_of_emails = no_of_emails + 1
-
-    if is_spam then
-      no_of_spam = no_of_spam + 1
-    else
-      no_of_ham = no_of_ham + 1
-    end
-
-    if is_spam and (score >= threshold) then
-      true_positives = true_positives + 1
-    elseif is_spam and (score < threshold) then
-      false_negatives = false_negatives + 1
-      table.insert(all_fns, log[#log])
-    elseif not is_spam and (score >= threshold) then
-      false_positives = false_positives + 1
-      table.insert(all_fps, log[#log])
-    else
-      true_negatives = true_negatives + 1
-    end
-
-    for i=4, (#log-2) do
-      if all_symbols_stats[log[i]] == nil then
-        all_symbols_stats[log[i]] = {
-          name = log[i],
-          no_of_hits = 0,
-          spam_hits = 0,
-          ham_hits = 0,
-          spam_overall = 0
-        }
-      end
-
-      all_symbols_stats[log[i]].no_of_hits =
-      all_symbols_stats[log[i]].no_of_hits + 1
-
-      if is_spam then
-        all_symbols_stats[log[i]].spam_hits =
-        all_symbols_stats[log[i]].spam_hits + 1
-      else
-        all_symbols_stats[log[i]].ham_hits =
-        all_symbols_stats[log[i]].ham_hits + 1
-      end
-
-      -- Find slowest message
-      if (tonumber(log[#log-1]) > tonumber(file_stats.slowest)) then
-          file_stats.slowest = tostring(tonumber(log[#log-1]))
-          file_stats.slowest_file = log[#log]
-      end
-    end
-  end
-
-  -- Calculating file stats
-
-  file_stats.no_of_ham = no_of_ham
-  file_stats.no_of_spam = no_of_spam
-  file_stats.no_of_emails = no_of_emails
-  file_stats.true_positives = true_positives
-  file_stats.true_negatives = true_negatives
-
-  if no_of_emails > 0 then
-    file_stats.spam_percent = no_of_spam * 100 / no_of_emails
-    file_stats.ham_percent = no_of_ham * 100 / no_of_emails
-    file_stats.overall_accuracy = (true_positives + true_negatives) * 100 /
-        no_of_emails
-  end
-
-  if no_of_ham > 0 then
-    file_stats.false_positive_rate = false_positives * 100 / no_of_ham
-  end
-
-  if no_of_spam > 0 then
-    file_stats.false_negative_rate = false_negatives * 100 / no_of_spam
-  end
-
-  file_stats.fscore = 2 * true_positives / (2
-      * true_positives
-      + false_positives
-      + false_negatives)
-
-  -- Calculating symbol stats
-
-  for _, symbol_stats in pairs(all_symbols_stats) do
-    symbol_stats.spam_percent = symbol_stats.spam_hits * 100 / no_of_spam
-    symbol_stats.ham_percent = symbol_stats.ham_hits * 100 / no_of_ham
-    symbol_stats.overall = symbol_stats.no_of_hits * 100 / no_of_emails
-    symbol_stats.spam_overall = symbol_stats.spam_percent /
-        (symbol_stats.spam_percent + symbol_stats.ham_percent)
-  end
-
-  return file_stats, all_symbols_stats, all_fps, all_fns
-end
-
-return utility