From 1498f2c5cbbc415c898621c233261ff5d41bc15d Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Sat, 25 Mar 2017 12:27:52 +0000 Subject: [PATCH] [Feature] Memoize LPEG grammars --- rules/global_functions.lua | 14 +++++--- src/plugins/lua/dkim_signing.lua | 13 +++++--- src/plugins/lua/metadata_exporter.lua | 13 +++++--- src/plugins/lua/multimap.lua | 47 ++++++++++++++------------ src/plugins/lua/phishing.lua | 13 +++++--- src/plugins/lua/ratelimit.lua | 48 +++++++++++++++------------ 6 files changed, 87 insertions(+), 61 deletions(-) diff --git a/rules/global_functions.lua b/rules/global_functions.lua index e3a167a07..3a60e5812 100644 --- a/rules/global_functions.lua +++ b/rules/global_functions.lua @@ -178,12 +178,18 @@ function rspamd_redis_make_request(task, redis_params, key, is_write, callback, return ret,conn,addr end +local split_grammar function rspamd_str_split(s, sep) local lpeg = require "lpeg" - sep = lpeg.P(sep) - local elem = lpeg.C((1 - sep)^0) - local p = lpeg.Ct(elem * (sep * elem)^0) -- make a table capture - return lpeg.match(p, s) + + if not split_grammar then + sep = lpeg.P(sep) + local elem = lpeg.C((1 - sep)^0) + local p = lpeg.Ct(elem * (sep * elem)^0) + split_grammar = p + end + + return lpeg.match(split_grammar, s) end -- Metafunctions diff --git a/src/plugins/lua/dkim_signing.lua b/src/plugins/lua/dkim_signing.lua index 8777afe93..c224c241a 100644 --- a/src/plugins/lua/dkim_signing.lua +++ b/src/plugins/lua/dkim_signing.lua @@ -40,16 +40,19 @@ local E = {} local N = 'dkim_signing' local redis_params +local template_grammar local function simple_template(tmpl, keys) local lpeg = require "lpeg" - local var_lit = lpeg.P { lpeg.R("az") + lpeg.R("AZ") + lpeg.R("09") + "_" } - local var = lpeg.P { (lpeg.P("$") / "") * ((var_lit^1) / keys) } - local var_braced = lpeg.P { (lpeg.P("${") / "") * ((var_lit^1) / keys) * (lpeg.P("}") / "") } + if not template_grammar then + local var_lit = lpeg.P { lpeg.R("az") + lpeg.R("AZ") + lpeg.R("09") + "_" } + local var = lpeg.P { (lpeg.P("$") / "") * ((var_lit^1) / keys) } + local var_braced = lpeg.P { (lpeg.P("${") / "") * ((var_lit^1) / keys) * (lpeg.P("}") / "") } - local rep = lpeg.Cs((var + var_braced + 1)^0) + template_grammar = lpeg.Cs((var + var_braced + 1)^0) + end - return lpeg.match(rep, tmpl) + return lpeg.match(template_grammar, tmpl) end local function dkim_signing_cb(task) diff --git a/src/plugins/lua/metadata_exporter.lua b/src/plugins/lua/metadata_exporter.lua index 36e2a819a..8d0778ad5 100644 --- a/src/plugins/lua/metadata_exporter.lua +++ b/src/plugins/lua/metadata_exporter.lua @@ -128,16 +128,19 @@ local function get_general_metadata(task, flatten, no_content) return r end +local template_grammar local function simple_template(tmpl, keys) local lpeg = require "lpeg" - local var_lit = lpeg.P { lpeg.R("az") + lpeg.R("AZ") + lpeg.R("09") + "_" } - local var = lpeg.P { (lpeg.P("$") / "") * ((var_lit^1) / keys) } - local var_braced = lpeg.P { (lpeg.P("${") / "") * ((var_lit^1) / keys) * (lpeg.P("}") / "") } + if not template_grammar then + local var_lit = lpeg.P { lpeg.R("az") + lpeg.R("AZ") + lpeg.R("09") + "_" } + local var = lpeg.P { (lpeg.P("$") / "") * ((var_lit^1) / keys) } + local var_braced = lpeg.P { (lpeg.P("${") / "") * ((var_lit^1) / keys) * (lpeg.P("}") / "") } - local rep = lpeg.Cs((var + var_braced + 1)^0) + template_grammar = lpeg.Cs((var + var_braced + 1)^0) + end - return lpeg.match(rep, tmpl) + return lpeg.match(template_grammar, tmpl) end local formatters = { diff --git a/src/plugins/lua/multimap.lua b/src/plugins/lua/multimap.lua index 843ca5515..c85919881 100644 --- a/src/plugins/lua/multimap.lua +++ b/src/plugins/lua/multimap.lua @@ -343,6 +343,8 @@ local multimap_filters = { --content = apply_content_filter, -- Content filters are special :( } +local multimap_grammar + local function multimap_callback(task, rule) local pre_filter = rule['prefilter'] @@ -396,36 +398,39 @@ local function multimap_callback(task, rule) local function parse_ret(parse_rule, p_ret) if p_ret and type(p_ret) == 'string' then local lpeg = require "lpeg" + + if not multimap_grammar then local number = {} - local digit = lpeg.R("09") - number.integer = + local digit = lpeg.R("09") + number.integer = (lpeg.S("+-") ^ -1) * - (digit ^ 1) + (digit ^ 1) - -- Matches: .6, .899, .9999873 - number.fractional = + -- Matches: .6, .899, .9999873 + number.fractional = (lpeg.P(".") ) * - (digit ^ 1) + (digit ^ 1) - -- Matches: 55.97, -90.8, .9 - number.decimal = + -- Matches: 55.97, -90.8, .9 + number.decimal = (number.integer * -- Integer - (number.fractional ^ -1)) + -- Fractional - (lpeg.S("+-") * number.fractional) -- Completely fractional number - - local sym_start = lpeg.R("az", "AZ") + lpeg.S("_") - local sym_elt = sym_start + lpeg.R("09") - local symbol = sym_start * sym_elt ^0 - local symbol_cap = lpeg.Cg(symbol, 'symbol') - local score_cap = lpeg.Cg(number.decimal, 'score') - local symscore_cap = (symbol_cap * lpeg.P(":") * score_cap) - local grammar = symscore_cap + symbol_cap + score_cap - local parser = lpeg.Ct(grammar) - local tbl = parser:match(p_ret) + (number.fractional ^ -1)) + -- Fractional + (lpeg.S("+-") * number.fractional) -- Completely fractional number + + local sym_start = lpeg.R("az", "AZ") + lpeg.S("_") + local sym_elt = sym_start + lpeg.R("09") + local symbol = sym_start * sym_elt ^0 + local symbol_cap = lpeg.Cg(symbol, 'symbol') + local score_cap = lpeg.Cg(number.decimal, 'score') + local symscore_cap = (symbol_cap * lpeg.P(":") * score_cap) + local grammar = symscore_cap + symbol_cap + score_cap + multimap_grammar = lpeg.Ct(grammar) + end + local tbl = multimap_grammar:match(p_ret) if tbl then - local sym = nil + local sym local score = 1.0 if tbl['symbol'] then diff --git a/src/plugins/lua/phishing.lua b/src/plugins/lua/phishing.lua index 6c42c96f2..d248c7513 100644 --- a/src/plugins/lua/phishing.lua +++ b/src/plugins/lua/phishing.lua @@ -239,12 +239,17 @@ local function phishing_map(mapname, phishmap, id) end end +local lpeg_grammar local function rspamd_str_split_fun(s, sep, func) local lpeg = require "lpeg" - sep = lpeg.P(sep) - local elem = lpeg.C((1 - sep)^0 / func) - local p = lpeg.C(elem * (sep * elem)^0) -- make a table capture - return lpeg.match(p, s) + + if not lpeg_grammar then + sep = lpeg.P(sep) + local elem = lpeg.C((1 - sep)^0 / func) + local p = lpeg.C(elem * (sep * elem)^0) -- make a table capture + lpeg_grammar = p + end + return lpeg.match(lpeg_grammar, s) end local function insert_url_from_string(pool, tbl, str, data) diff --git a/src/plugins/lua/ratelimit.lua b/src/plugins/lua/ratelimit.lua index 1576b4130..341f9e658 100644 --- a/src/plugins/lua/ratelimit.lua +++ b/src/plugins/lua/ratelimit.lua @@ -500,6 +500,7 @@ local function parse_limit(str) end end +local limit_parser local function parse_string_limit(lim) local function parse_time_suffix(s) if s == 's' then @@ -524,30 +525,33 @@ local function parse_string_limit(lim) end end local lpeg = require "lpeg" - local digit = lpeg.R("09") - local grammar = {} - grammar.integer = + + if not limit_parser then + local digit = lpeg.R("09") + limit_parser = {} + limit_parser.integer = (lpeg.S("+-") ^ -1) * - (digit ^ 1) - grammar.fractional = + (digit ^ 1) + limit_parser.fractional = (lpeg.P(".") ) * - (digit ^ 1) - grammar.number = - (grammar.integer * - (grammar.fractional ^ -1)) + - (lpeg.S("+-") * grammar.fractional) - grammar.time = lpeg.Cf(lpeg.Cc(1) * - (grammar.number / tonumber) * - ((lpeg.S("smhd") / parse_time_suffix) ^ -1), - function (acc, val) return acc * val end) - grammar.suffixed_number = lpeg.Cf(lpeg.Cc(1) * - (grammar.number / tonumber) * - ((lpeg.S("kmg") / parse_num_suffix) ^ -1), - function (acc, val) return acc * val end) - grammar.limit = lpeg.Ct(grammar.suffixed_number * - (lpeg.S(" ") ^ 0) * lpeg.S("/") * (lpeg.S(" ") ^ 0) * - grammar.time) - local t = lpeg.match(grammar.limit, lim) + (digit ^ 1) + limit_parser.number = + (limit_parser.integer * + (limit_parser.fractional ^ -1)) + + (lpeg.S("+-") * limit_parser.fractional) + limit_parser.time = lpeg.Cf(lpeg.Cc(1) * + (limit_parser.number / tonumber) * + ((lpeg.S("smhd") / parse_time_suffix) ^ -1), + function (acc, val) return acc * val end) + limit_parser.suffixed_number = lpeg.Cf(lpeg.Cc(1) * + (limit_parser.number / tonumber) * + ((lpeg.S("kmg") / parse_num_suffix) ^ -1), + function (acc, val) return acc * val end) + limit_parser.limit = lpeg.Ct(limit_parser.suffixed_number * + (lpeg.S(" ") ^ 0) * lpeg.S("/") * (lpeg.S(" ") ^ 0) * + limit_parser.time) + end + local t = lpeg.match(limit_parser.limit, lim) if t and t[1] and t[2] and t[2] ~= 0 then return t[1] / t[2], t[1] -- 2.39.5