From 29802ac374a9b315423d7815eaedd81d23205545 Mon Sep 17 00:00:00 2001 From: Andrew Lewis Date: Wed, 1 Feb 2017 17:31:08 +0200 Subject: [PATCH] [Minor] Recreate grep tool as `rspamadm grep` --- contrib/rspamdgrep/rspamdgrep.lua | 92 -------------------- contrib/rspamdgrep/rspamdgrep.sh | 11 --- src/rspamadm/CMakeLists.txt | 1 + src/rspamadm/commands.c | 2 + src/rspamadm/grep.c | 139 ++++++++++++++++++++++++++++++ src/rspamadm/grep.lua | 104 ++++++++++++++++++++++ 6 files changed, 246 insertions(+), 103 deletions(-) delete mode 100644 contrib/rspamdgrep/rspamdgrep.lua delete mode 100755 contrib/rspamdgrep/rspamdgrep.sh create mode 100644 src/rspamadm/grep.c create mode 100644 src/rspamadm/grep.lua diff --git a/contrib/rspamdgrep/rspamdgrep.lua b/contrib/rspamdgrep/rspamdgrep.lua deleted file mode 100644 index 4a4341f5e..000000000 --- a/contrib/rspamdgrep/rspamdgrep.lua +++ /dev/null @@ -1,92 +0,0 @@ -local rspamd_regexp = require 'rspamd_regexp' - -local E = {} - -local buffer = {} -local matches = {} - -if type(arg) ~= 'table' then - io.stderr:write('Syntax: rspamdgrep [sources]\n') - os.exit(1) -end - -local pattern = table.remove(arg, 1) -local re = rspamd_regexp.create(pattern) -if not re then - io.stderr:write("Couldn't compile regex: " .. pattern .. '\n') - os.exit(1) -end - -if not arg[1] then - arg = {'stdin'} -end - -for _, n in ipairs(arg) do - local h, err - if string.match(n, '%.xz$') then - h, err = io.popen('xzcat ' .. n, 'r') - elseif string.match(n, '%.bz2$') then - h, err = io.popen('bzcat ' .. n, 'r') - elseif string.match(n, '%.gz$') then - h, err = io.popen('zcat ' .. n, 'r') - elseif n == 'stdin' then - h = io.input() - else - h, err = io.open(n, 'r') - end - if not h then - if err then - io.stderr:write("Couldn't open file (" .. n .. '): ' .. err .. '\n') - else - io.stderr:write("Couldn't open file (" .. n .. '): no error') - end - else - for line in h:lines() do - local hash = string.match(line, '^%d+-%d+-%d+ %d+:%d+:%d+ #%d+%(%a+%) <(%x+)>') - if hash then - if matches[hash] then - table.insert(matches[hash], line) - else - if buffer[hash] then - table.insert(buffer[hash], line) - else - buffer[hash] = {line} - end - end - end - if re:match(line) then - if not hash then - print('*** orphaned ***') - print(line) - print() - else - if matches[hash] then - table.insert(matches[hash], line) - else - local old = buffer[hash] or E - table.insert(old, line) - matches[hash] = old - end - end - end - local is_end = string.match(line, '^%d+-%d+-%d+ %d+:%d+:%d+ #%d+%(%a+%) <%x+>; task; rspamd_protocol_http_reply:') - if is_end then - buffer[hash] = nil - if matches[hash] then - for _, v in ipairs(matches[hash]) do - print(v) - end - print() - matches[hash] = nil - end - end - end - end -end -for _, v in pairs(matches) do - print('*** partial ***') - for _, vv in ipairs(v) do - print(vv) - end - print() -end diff --git a/contrib/rspamdgrep/rspamdgrep.sh b/contrib/rspamdgrep/rspamdgrep.sh deleted file mode 100755 index 08a5fea2f..000000000 --- a/contrib/rspamdgrep/rspamdgrep.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh - -# Process command-line arguments -LARG="" -for i in "$@" -do - LARG="$LARG -a $i" -done - -# Call rspamadm lua -rspamadm lua $LARG rspamdgrep.lua diff --git a/src/rspamadm/CMakeLists.txt b/src/rspamadm/CMakeLists.txt index 8a016a497..91e14d749 100644 --- a/src/rspamadm/CMakeLists.txt +++ b/src/rspamadm/CMakeLists.txt @@ -5,6 +5,7 @@ SET(RSPAMADMSRC rspamadm.c configtest.c fuzzy_convert.c fuzzy_merge.c + grep.c configdump.c control.c confighelp.c diff --git a/src/rspamadm/commands.c b/src/rspamadm/commands.c index 300fc3dbe..d344dce3c 100644 --- a/src/rspamadm/commands.c +++ b/src/rspamadm/commands.c @@ -24,6 +24,7 @@ extern struct rspamadm_command control_command; extern struct rspamadm_command confighelp_command; extern struct rspamadm_command statconvert_command; extern struct rspamadm_command fuzzyconvert_command; +extern struct rspamadm_command grep_command; extern struct rspamadm_command signtool_command; extern struct rspamadm_command lua_command; extern struct rspamadm_command dkim_keygen_command; @@ -39,6 +40,7 @@ const struct rspamadm_command *commands[] = { &confighelp_command, &statconvert_command, &fuzzyconvert_command, + &grep_command, &signtool_command, &lua_command, &dkim_keygen_command, diff --git a/src/rspamadm/grep.c b/src/rspamadm/grep.c new file mode 100644 index 000000000..aff986c68 --- /dev/null +++ b/src/rspamadm/grep.c @@ -0,0 +1,139 @@ +/*- + * Copyright (c) 2017, Andrew Lewis + * Copyright (c) 2017, Vsevolod Stakhov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "config.h" +#include "rspamadm.h" +#include "lua/lua_common.h" +#include "grep.lua.h" + +static gchar *string = NULL; +static gchar *pattern = NULL; +static gchar **inputs = NULL; +static gboolean sensitive = FALSE; + +static void rspamadm_grep (gint argc, gchar **argv); +static const char *rspamadm_grep_help (gboolean full_help); + +struct rspamadm_command grep_command = { + .name = "grep", + .flags = 0, + .help = rspamadm_grep_help, + .run = rspamadm_grep +}; + +static GOptionEntry entries[] = { + {"string", 's', 0, G_OPTION_ARG_STRING, &string, + "Plain string to search (case-insensitive)", NULL}, + {"sensitive", 'S', 0, G_OPTION_ARG_NONE, &sensitive, + "Enable case-sensitivity in string search", NULL}, + {"pattern", 'p', 0, G_OPTION_ARG_STRING, &pattern, + "Pattern to search for (regex)", NULL}, + {"input", 'i', 0, G_OPTION_ARG_STRING_ARRAY, &inputs, + "Processed specified inputs", NULL}, + {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL} +}; + + +static const char * +rspamadm_grep_help (gboolean full_help) +{ + const char *help_str; + + if (full_help) { + help_str = "Search for patterns in rspamd logs\n\n" + "Usage: rspamadm grep <-s string | -p pattern> [-S] [-i input1 -i input2]\n" + "Where options are:\n\n" + "-s: Plain string to search (case-insensitive)\n" + "-S: Enable case-sensitivity in string search\n" + "-p: Pattern to search for (regex)\n" + "-i: Process specified inputs\n"; + } + else { + help_str = "Search for patterns in rspamd logs"; + } + + return help_str; +} + +static void +rspamadm_grep (gint argc, gchar **argv) +{ + GOptionContext *context; + GError *error = NULL; + lua_State *L; + ucl_object_t *obj, *nobj; + gchar **elt; + + context = g_option_context_new ( + "grep - search for patterns in rspamd logs"); + g_option_context_set_summary (context, + "Summary:\n Rspamd administration utility version " + RVERSION + "\n Release id: " + RID); + g_option_context_add_main_entries (context, entries, NULL); + g_option_context_set_ignore_unknown_options (context, TRUE); + + if (!g_option_context_parse (context, &argc, &argv, &error)) { + rspamd_fprintf (stderr, "option parsing failed: %s\n", error->message); + g_error_free (error); + exit (1); + } + + if (!pattern && !string) { + rspamd_fprintf (stderr, "no search pattern specified\n"); + exit (1); + } + if (pattern && string) { + rspamd_fprintf (stderr, "-s and -p are mutually-exclusive\n"); + exit (1); + } + + L = rspamd_lua_init (); + + obj = ucl_object_typed_new (UCL_OBJECT); + if (string) { + ucl_object_insert_key (obj, ucl_object_fromstring (string), + "string", 0, false); + } + if (pattern) { + ucl_object_insert_key (obj, ucl_object_fromstring (pattern), + "pattern", 0, false); + } + nobj = ucl_object_typed_new (UCL_ARRAY); + if (!inputs) { + ucl_array_append (nobj, ucl_object_fromstring ("stdin")); + } + else { + for (elt = inputs; *elt != NULL; elt ++) { + ucl_array_append (nobj, ucl_object_fromstring (*elt)); + } + } + ucl_object_insert_key (obj, nobj, "inputs", 0, false); + if (sensitive) { + ucl_object_insert_key (obj, ucl_object_frombool (sensitive), + "sensitive", 0, false); + } + + rspamadm_execute_lua_ucl_subr (L, + argc, + argv, + obj, + rspamadm_script_grep); + + lua_close (L); + ucl_object_unref (obj); +} diff --git a/src/rspamadm/grep.lua b/src/rspamadm/grep.lua new file mode 100644 index 000000000..4c77db73d --- /dev/null +++ b/src/rspamadm/grep.lua @@ -0,0 +1,104 @@ +local rspamd_regexp = require 'rspamd_regexp' +local E = {} + +return function(_, res) + + local buffer = {} + local matches = {} + + local pattern = res['pattern'] + local re + if pattern then + re = rspamd_regexp.create(pattern) + if not re then + io.stderr:write("Couldn't compile regex: " .. pattern .. '\n') + os.exit(1) + end + end + + local search_str = res['string'] + local sensitive = res['sensitive'] + if search_str and not sensitive then + search_str = string.lower(search_str) + end + local inputs = res['inputs'] + + for _, n in ipairs(inputs) do + local h, err + if string.match(n, '%.xz$') then + h, err = io.popen('xzcat ' .. n, 'r') + elseif string.match(n, '%.bz2$') then + h, err = io.popen('bzcat ' .. n, 'r') + elseif string.match(n, '%.gz$') then + h, err = io.popen('zcat ' .. n, 'r') + elseif n == 'stdin' then + h = io.input() + else + h, err = io.open(n, 'r') + end + if not h then + if err then + io.stderr:write("Couldn't open file (" .. n .. '): ' .. err .. '\n') + else + io.stderr:write("Couldn't open file (" .. n .. '): no error\n') + end + else + for line in h:lines() do + local hash = string.match(line, '^%d+-%d+-%d+ %d+:%d+:%d+ #%d+%(%a+%) <(%x+)>') + if hash then + if matches[hash] then + table.insert(matches[hash], line) + else + if buffer[hash] then + table.insert(buffer[hash], line) + else + buffer[hash] = {line} + end + end + end + local ismatch = false + if re then + ismatch = re:match(line) + elseif sensitive and search_str then + ismatch = string.find(line, search_str) + elseif search_str then + local lwr = string.lower(line) + ismatch = string.find(lwr, search_str) + end + if ismatch then + if not hash then + print('*** orphaned ***') + print(line) + print() + else + if matches[hash] then + table.insert(matches[hash], line) + else + local cur = buffer[hash] or E + table.insert(cur, line) + matches[hash] = cur + end + end + end + local is_end = string.match(line, '^%d+-%d+-%d+ %d+:%d+:%d+ #%d+%(%a+%) <%x+>; task; rspamd_protocol_http_reply:') + if is_end then + buffer[hash] = nil + if matches[hash] then + for _, v in ipairs(matches[hash]) do + print(v) + end + print() + matches[hash] = nil + end + end + end + end + end + for _, v in pairs(matches) do + print('*** partial ***') + for _, vv in ipairs(v) do + print(vv) + end + print() + end +end -- 2.39.5