From 5ad02a7ad991404072c674b1e8d5e946a899730d Mon Sep 17 00:00:00 2001
From: Andrew Lewis <nerf@judo.za.org>
Date: Tue, 31 Jan 2017 20:57:45 +0200
Subject: [Minor] Fixes for rspamdgrep

---
 contrib/rspamdgrep/rspamdgrep.lua | 51 ++++++++++++++++++++++-----------------
 1 file changed, 29 insertions(+), 22 deletions(-)

(limited to 'contrib')

diff --git a/contrib/rspamdgrep/rspamdgrep.lua b/contrib/rspamdgrep/rspamdgrep.lua
index 7529c7c9c..4a4341f5e 100644
--- a/contrib/rspamdgrep/rspamdgrep.lua
+++ b/contrib/rspamdgrep/rspamdgrep.lua
@@ -1,15 +1,12 @@
 local rspamd_regexp = require 'rspamd_regexp'
-local rspamd_logger = require 'rspamd_logger'
 
-local BUF_SIZE = 10240
 local E = {}
 
 local buffer = {}
 local matches = {}
-local count = 0
 
 if type(arg) ~= 'table' then
-  io.stderr:write('No files specified for search\n')
+  io.stderr:write('Syntax: rspamdgrep <pattern> [sources]\n')
   os.exit(1)
 end
 
@@ -20,6 +17,10 @@ if not re then
   os.exit(1)
 end
 
+if not arg[1] then
+  arg = {'stdin'}
+end
+
 for _, n in ipairs(arg) do
   local h, err
   if string.match(n, '%.xz$') then
@@ -28,14 +29,16 @@ for _, n in ipairs(arg) do
     h, err = io.popen('bzcat ' .. n, 'r')
   elseif string.match(n, '%.gz$') then
     h, err = io.popen('zcat ' .. n, 'r')
-  elseif string.match(n, '%.log$') then
-    h, err = io.open(n, 'r')
+  elseif n == 'stdin' then
+    h = io.input()
   else
-    io.stderr:write("Couldn't identify log format: " .. n .. '\n')
+    h, err = io.open(n, 'r')
   end
   if not h then
     if err then
       io.stderr:write("Couldn't open file (" .. n .. '): ' .. err .. '\n')
+    else
+      io.stderr:write("Couldn't open file (" .. n .. '): no error')
     end
   else
     for line in h:lines() do
@@ -49,35 +52,39 @@ for _, n in ipairs(arg) do
           else
             buffer[hash] = {line}
           end
-          count = count + 1
-          if count >= BUF_SIZE then
-            local k = next(buffer)
-            buffer[k] = nil
-            count = count - 1
-          end
         end
       end
       if re:match(line) then
         if not hash then
-          hash = 'orphaned'
-        end
-        if matches[hash] then
-          table.insert(matches[hash], line)
+          print('*** orphaned ***')
+          print(line)
+          print()
         else
-          local old = buffer[hash] or E
-          table.insert(old, line)
-          matches[hash] = old
+          if matches[hash] then
+            table.insert(matches[hash], line)
+          else
+            local old = buffer[hash] or E
+            table.insert(old, line)
+            matches[hash] = old
+          end
         end
       end
       local is_end = string.match(line, '^%d+-%d+-%d+ %d+:%d+:%d+ #%d+%(%a+%) <%x+>; task; rspamd_protocol_http_reply:')
       if is_end then
         buffer[hash] = nil
+        if matches[hash] then
+          for _, v in ipairs(matches[hash]) do
+            print(v)
+          end
+          print()
+          matches[hash] = nil
+        end
       end
     end
   end
 end
-
-for k, v in pairs(matches) do
+for _, v in pairs(matches) do
+  print('*** partial ***')
   for _, vv in ipairs(v) do
     print(vv)
   end
-- 
cgit v1.2.3


From 29802ac374a9b315423d7815eaedd81d23205545 Mon Sep 17 00:00:00 2001
From: Andrew Lewis <nerf@judo.za.org>
Date: Wed, 1 Feb 2017 17:31:08 +0200
Subject: [Minor] Recreate grep tool as `rspamadm grep`

---
 contrib/rspamdgrep/rspamdgrep.lua |  92 -------------------------
 contrib/rspamdgrep/rspamdgrep.sh  |  11 ---
 src/rspamadm/CMakeLists.txt       |   1 +
 src/rspamadm/commands.c           |   2 +
 src/rspamadm/grep.c               | 139 ++++++++++++++++++++++++++++++++++++++
 src/rspamadm/grep.lua             | 104 ++++++++++++++++++++++++++++
 6 files changed, 246 insertions(+), 103 deletions(-)
 delete mode 100644 contrib/rspamdgrep/rspamdgrep.lua
 delete mode 100755 contrib/rspamdgrep/rspamdgrep.sh
 create mode 100644 src/rspamadm/grep.c
 create mode 100644 src/rspamadm/grep.lua

(limited to 'contrib')

diff --git a/contrib/rspamdgrep/rspamdgrep.lua b/contrib/rspamdgrep/rspamdgrep.lua
deleted file mode 100644
index 4a4341f5e..000000000
--- a/contrib/rspamdgrep/rspamdgrep.lua
+++ /dev/null
@@ -1,92 +0,0 @@
-local rspamd_regexp = require 'rspamd_regexp'
-
-local E = {}
-
-local buffer = {}
-local matches = {}
-
-if type(arg) ~= 'table' then
-  io.stderr:write('Syntax: rspamdgrep <pattern> [sources]\n')
-  os.exit(1)
-end
-
-local pattern = table.remove(arg, 1)
-local re = rspamd_regexp.create(pattern)
-if not re then
-  io.stderr:write("Couldn't compile regex: " .. pattern .. '\n')
-  os.exit(1)
-end
-
-if not arg[1] then
-  arg = {'stdin'}
-end
-
-for _, n in ipairs(arg) do
-  local h, err
-  if string.match(n, '%.xz$') then
-    h, err = io.popen('xzcat ' .. n, 'r')
-  elseif string.match(n, '%.bz2$') then
-    h, err = io.popen('bzcat ' .. n, 'r')
-  elseif string.match(n, '%.gz$') then
-    h, err = io.popen('zcat ' .. n, 'r')
-  elseif n == 'stdin' then
-    h = io.input()
-  else
-    h, err = io.open(n, 'r')
-  end
-  if not h then
-    if err then
-      io.stderr:write("Couldn't open file (" .. n .. '): ' .. err .. '\n')
-    else
-      io.stderr:write("Couldn't open file (" .. n .. '): no error')
-    end
-  else
-    for line in h:lines() do
-      local hash = string.match(line, '^%d+-%d+-%d+ %d+:%d+:%d+ #%d+%(%a+%) <(%x+)>')
-      if hash then
-        if matches[hash] then
-          table.insert(matches[hash], line)
-        else
-          if buffer[hash] then
-            table.insert(buffer[hash], line)
-          else
-            buffer[hash] = {line}
-          end
-        end
-      end
-      if re:match(line) then
-        if not hash then
-          print('*** orphaned ***')
-          print(line)
-          print()
-        else
-          if matches[hash] then
-            table.insert(matches[hash], line)
-          else
-            local old = buffer[hash] or E
-            table.insert(old, line)
-            matches[hash] = old
-          end
-        end
-      end
-      local is_end = string.match(line, '^%d+-%d+-%d+ %d+:%d+:%d+ #%d+%(%a+%) <%x+>; task; rspamd_protocol_http_reply:')
-      if is_end then
-        buffer[hash] = nil
-        if matches[hash] then
-          for _, v in ipairs(matches[hash]) do
-            print(v)
-          end
-          print()
-          matches[hash] = nil
-        end
-      end
-    end
-  end
-end
-for _, v in pairs(matches) do
-  print('*** partial ***')
-  for _, vv in ipairs(v) do
-    print(vv)
-  end
-  print()
-end
diff --git a/contrib/rspamdgrep/rspamdgrep.sh b/contrib/rspamdgrep/rspamdgrep.sh
deleted file mode 100755
index 08a5fea2f..000000000
--- a/contrib/rspamdgrep/rspamdgrep.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/sh
-
-# Process command-line arguments
-LARG=""
-for i in "$@"
-do
-	LARG="$LARG -a $i"
-done
-
-# Call rspamadm lua
-rspamadm lua $LARG rspamdgrep.lua
diff --git a/src/rspamadm/CMakeLists.txt b/src/rspamadm/CMakeLists.txt
index 8a016a497..91e14d749 100644
--- a/src/rspamadm/CMakeLists.txt
+++ b/src/rspamadm/CMakeLists.txt
@@ -5,6 +5,7 @@ SET(RSPAMADMSRC rspamadm.c
         configtest.c
         fuzzy_convert.c
         fuzzy_merge.c
+        grep.c
         configdump.c
         control.c
         confighelp.c
diff --git a/src/rspamadm/commands.c b/src/rspamadm/commands.c
index 300fc3dbe..d344dce3c 100644
--- a/src/rspamadm/commands.c
+++ b/src/rspamadm/commands.c
@@ -24,6 +24,7 @@ extern struct rspamadm_command control_command;
 extern struct rspamadm_command confighelp_command;
 extern struct rspamadm_command statconvert_command;
 extern struct rspamadm_command fuzzyconvert_command;
+extern struct rspamadm_command grep_command;
 extern struct rspamadm_command signtool_command;
 extern struct rspamadm_command lua_command;
 extern struct rspamadm_command dkim_keygen_command;
@@ -39,6 +40,7 @@ const struct rspamadm_command *commands[] = {
 	&confighelp_command,
 	&statconvert_command,
 	&fuzzyconvert_command,
+	&grep_command,
 	&signtool_command,
 	&lua_command,
 	&dkim_keygen_command,
diff --git a/src/rspamadm/grep.c b/src/rspamadm/grep.c
new file mode 100644
index 000000000..aff986c68
--- /dev/null
+++ b/src/rspamadm/grep.c
@@ -0,0 +1,139 @@
+/*-
+ * Copyright (c) 2017, Andrew Lewis <nerf@judo.za.org>
+ * Copyright (c) 2017, Vsevolod Stakhov <vsevolod@highsecure.ru>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "rspamadm.h"
+#include "lua/lua_common.h"
+#include "grep.lua.h"
+
+static gchar *string = NULL;
+static gchar *pattern = NULL;
+static gchar **inputs = NULL;
+static gboolean sensitive = FALSE;
+
+static void rspamadm_grep (gint argc, gchar **argv);
+static const char *rspamadm_grep_help (gboolean full_help);
+
+struct rspamadm_command grep_command = {
+		.name = "grep",
+		.flags = 0,
+		.help = rspamadm_grep_help,
+		.run = rspamadm_grep
+};
+
+static GOptionEntry entries[] = {
+		{"string", 's', 0, G_OPTION_ARG_STRING, &string,
+				"Plain string to search (case-insensitive)", NULL},
+		{"sensitive", 'S', 0, G_OPTION_ARG_NONE, &sensitive,
+				"Enable case-sensitivity in string search", NULL},
+		{"pattern", 'p', 0, G_OPTION_ARG_STRING, &pattern,
+				"Pattern to search for (regex)", NULL},
+                {"input", 'i', 0, G_OPTION_ARG_STRING_ARRAY, &inputs,
+                                "Processed specified inputs", NULL},
+		{NULL,     0,   0, G_OPTION_ARG_NONE, NULL, NULL, NULL}
+};
+
+
+static const char *
+rspamadm_grep_help (gboolean full_help)
+{
+	const char *help_str;
+
+	if (full_help) {
+		help_str = "Search for patterns in rspamd logs\n\n"
+				"Usage: rspamadm grep <-s string | -p pattern> [-S] [-i input1 -i input2]\n"
+				"Where options are:\n\n"
+				"-s: Plain string to search (case-insensitive)\n"
+				"-S: Enable case-sensitivity in string search\n"
+				"-p: Pattern to search for (regex)\n"
+				"-i: Process specified inputs\n";
+	}
+	else {
+		help_str = "Search for patterns in rspamd logs";
+	}
+
+	return help_str;
+}
+
+static void
+rspamadm_grep (gint argc, gchar **argv)
+{
+	GOptionContext *context;
+	GError *error = NULL;
+	lua_State *L;
+	ucl_object_t *obj, *nobj;
+	gchar **elt;
+
+	context = g_option_context_new (
+			"grep - search for patterns in rspamd logs");
+	g_option_context_set_summary (context,
+			"Summary:\n  Rspamd administration utility version "
+					RVERSION
+					"\n  Release id: "
+					RID);
+	g_option_context_add_main_entries (context, entries, NULL);
+	g_option_context_set_ignore_unknown_options (context, TRUE);
+
+	if (!g_option_context_parse (context, &argc, &argv, &error)) {
+		rspamd_fprintf (stderr, "option parsing failed: %s\n", error->message);
+		g_error_free (error);
+		exit (1);
+	}
+
+	if (!pattern && !string) {
+		rspamd_fprintf (stderr, "no search pattern specified\n");
+		exit (1);
+	}
+	if (pattern && string) {
+		rspamd_fprintf (stderr, "-s and -p are mutually-exclusive\n");
+		exit (1);
+	}
+
+	L = rspamd_lua_init ();
+
+	obj = ucl_object_typed_new (UCL_OBJECT);
+	if (string) {
+		ucl_object_insert_key (obj, ucl_object_fromstring (string),
+				"string", 0, false);
+	}
+	if (pattern) {
+		ucl_object_insert_key (obj, ucl_object_fromstring (pattern),
+				"pattern", 0, false);
+	}
+	nobj = ucl_object_typed_new (UCL_ARRAY);
+	if (!inputs) {
+		ucl_array_append (nobj, ucl_object_fromstring ("stdin"));
+	}
+	else {
+		for (elt = inputs; *elt != NULL; elt ++) {
+			ucl_array_append (nobj, ucl_object_fromstring (*elt));
+		}
+	}
+	ucl_object_insert_key (obj, nobj, "inputs", 0, false);
+	if (sensitive) {
+		ucl_object_insert_key (obj, ucl_object_frombool (sensitive),
+				"sensitive", 0, false);
+	}
+
+	rspamadm_execute_lua_ucl_subr (L,
+			argc,
+			argv,
+			obj,
+			rspamadm_script_grep);
+
+	lua_close (L);
+	ucl_object_unref (obj);
+}
diff --git a/src/rspamadm/grep.lua b/src/rspamadm/grep.lua
new file mode 100644
index 000000000..4c77db73d
--- /dev/null
+++ b/src/rspamadm/grep.lua
@@ -0,0 +1,104 @@
+local rspamd_regexp = require 'rspamd_regexp'
+local E = {}
+
+return function(_, res)
+
+  local buffer = {}
+  local matches = {}
+
+  local pattern = res['pattern']
+  local re
+  if pattern then
+    re = rspamd_regexp.create(pattern)
+    if not re then
+      io.stderr:write("Couldn't compile regex: " .. pattern .. '\n')
+      os.exit(1)
+    end
+  end
+
+  local search_str = res['string']
+  local sensitive = res['sensitive']
+  if search_str and not sensitive then
+    search_str = string.lower(search_str)
+  end
+  local inputs = res['inputs']
+
+  for _, n in ipairs(inputs) do
+    local h, err
+    if string.match(n, '%.xz$') then
+      h, err = io.popen('xzcat ' .. n, 'r')
+    elseif string.match(n, '%.bz2$') then
+      h, err = io.popen('bzcat ' .. n, 'r')
+    elseif string.match(n, '%.gz$') then
+      h, err = io.popen('zcat ' .. n, 'r')
+    elseif n == 'stdin' then
+      h = io.input()
+    else
+      h, err = io.open(n, 'r')
+    end
+    if not h then
+      if err then
+        io.stderr:write("Couldn't open file (" .. n .. '): ' .. err .. '\n')
+      else
+        io.stderr:write("Couldn't open file (" .. n .. '): no error\n')
+      end
+    else
+      for line in h:lines() do
+        local hash = string.match(line, '^%d+-%d+-%d+ %d+:%d+:%d+ #%d+%(%a+%) <(%x+)>')
+        if hash then
+          if matches[hash] then
+            table.insert(matches[hash], line)
+          else
+            if buffer[hash] then
+              table.insert(buffer[hash], line)
+            else
+              buffer[hash] = {line}
+            end
+          end
+        end
+        local ismatch = false
+        if re then
+          ismatch = re:match(line)
+        elseif sensitive and search_str then
+          ismatch = string.find(line, search_str)
+        elseif search_str then
+          local lwr = string.lower(line)
+          ismatch = string.find(lwr, search_str)
+        end
+        if ismatch then
+          if not hash then
+            print('*** orphaned ***')
+            print(line)
+            print()
+          else
+            if matches[hash] then
+              table.insert(matches[hash], line)
+            else
+              local cur = buffer[hash] or E
+              table.insert(cur, line)
+              matches[hash] = cur
+            end
+          end
+        end
+        local is_end = string.match(line, '^%d+-%d+-%d+ %d+:%d+:%d+ #%d+%(%a+%) <%x+>; task; rspamd_protocol_http_reply:')
+        if is_end then
+          buffer[hash] = nil
+          if matches[hash] then
+            for _, v in ipairs(matches[hash]) do
+              print(v)
+            end
+            print()
+            matches[hash] = nil
+          end
+        end
+      end
+    end
+  end
+  for _, v in pairs(matches) do
+    print('*** partial ***')
+    for _, vv in ipairs(v) do
+      print(vv)
+    end
+    print()
+  end
+end
-- 
cgit v1.2.3