From 6de2193c6839f6f5593feb66ce072c93c54622c6 Mon Sep 17 00:00:00 2001 From: Igor Ippolitov Date: Wed, 18 Jul 2018 13:10:00 +0300 Subject: [PATCH] Implement memory efficient stat convert --- lualib/lua_stat.lua | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/lualib/lua_stat.lua b/lualib/lua_stat.lua index 81b5d0ed8..b5eaafcd3 100644 --- a/lualib/lua_stat.lua +++ b/lualib/lua_stat.lua @@ -45,29 +45,31 @@ local function convert_bayes_schema(redis_params, symbol_spam, symbol_ham, expi -- KEYS[2]: hash key ('S' or 'H') -- KEYS[3]: expire local lua_script = [[ +redis.replicate_commands() local keys = redis.call('SMEMBERS', KEYS[1]..'_keys') local nconverted = 0 - for _,k in ipairs(keys) do - local elts = redis.call('HGETALL', k) + local cursor = redis.call('HSCAN', k, 0) local neutral_prefix = string.gsub(k, KEYS[1], 'RS') - local real_key - - for i,v in ipairs(elts) do - - if i % 2 ~= 0 then - real_key = v - else - local nkey = string.format('%s_%s', neutral_prefix, real_key) - redis.call('HSET', nkey, KEYS[2], v) - if KEYS[3] and tonumber(KEYS[3]) > 0 then - redis.call('EXPIRE', nkey, KEYS[3]) + local elts + while cursor[1] ~= "0" do + elts = cursor[2] + cursor = redis.call('HSCAN', k, cursor[1]) + local real_key + for i,v in ipairs(elts) do + if i % 2 ~= 0 then + real_key = v + else + local nkey = string.format('%s_%s', neutral_prefix, real_key) + redis.call('HSET', nkey, KEYS[2], v) + if KEYS[3] and tonumber(KEYS[3]) > 0 then + redis.call('EXPIRE', nkey, KEYS[3]) + end + nconverted = nconverted + 1 end - nconverted = nconverted + 1 end end end - return nconverted ]] -- 2.39.5