[Minor] lua_scanners - oletools - make sure to read the complete reply

1 files changed, 136 insertions, 127 deletions

diff --git a/lualib/lua_scanners/oletools.lua b/lualib/lua_scanners/oletools.lua
index 91094ccb3..a7d6170d5 100644
--- a/lualib/lua_scanners/oletools.lua
+++ b/lualib/lua_scanners/oletools.lua
@@ -36,8 +36,9 @@ local function oletools_check(task, content, digest, rule)
     local addr = upstream:get_addr()
     local retransmits = rule.retransmits
     local protocol = 'OLEFY/1.0\nMethod: oletools\nRspamd-ID: ' .. task:get_uid() .. '\n\n'
+    local json_response = ""
 
-    local function oletools_callback(err, data)
+    local function oletools_callback(err, data, conn)
 
       local function oletools_requery(error)
         -- set current upstream to fail because an error occurred
@@ -83,144 +84,152 @@ local function oletools_check(task, content, digest, rule)
         -- Parse the response
         if upstream then upstream:ok() end
 
-        data = tostring(data)
+        json_response = json_response .. tostring(data)
 
-        local ucl_parser = ucl.parser()
-        local ok, ucl_err = ucl_parser:parse_string(tostring(data))
-        if not ok then
-          rspamd_logger.errx(task, "%s: error parsing json response: %s",
-              rule.log_prefix, ucl_err)
-          return
-        end
+	      if not string.find(json_response, '\t\n\n\t') and #data == 8192 then
+          lua_util.debugm(rule.name, task, '%s: no stop word: add_read - #json: %s / current packet: %s',
+            rule.log_prefix, #json_response, #data)
+          conn:add_read(oletools_callback)
+        else
 
-        local result = ucl_parser:get_object()
-
-        local oletools_rc = {
-          [0] = 'RETURN_OK',
-          [1] = 'RETURN_WARNINGS',
-          [2] = 'RETURN_WRONG_ARGS',
-          [3] = 'RETURN_FILE_NOT_FOUND',
-          [4] = 'RETURN_XGLOB_ERR',
-          [5] = 'RETURN_OPEN_ERROR',
-          [6] = 'RETURN_PARSE_ERROR',
-          [7] = 'RETURN_SEVERAL_ERRS',
-          [8] = 'RETURN_UNEXPECTED',
-          [9] = 'RETURN_ENCRYPTED',
-        }
-
-        if result[1].error ~= nil then
-          rspamd_logger.errx(task, '%s: ERROR found: %s', rule.log_prefix,
-              result[1].error)
-          if result[1].error == 'File too small' then
-            common.save_av_cache(task, digest, rule, 'OK')
-            common.log_clean(task, rule, 'File too small to be scanned for macros')
-          else
-            oletools_requery(result[1].error)
+          local ucl_parser = ucl.parser()
+          local ok, ucl_err = ucl_parser:parse_string(tostring(json_response))
+          if not ok then
+            rspamd_logger.errx(task, "%s: error parsing json response, retry: %s",
+                rule.log_prefix, ucl_err)
+            oletools_requery(ucl_err)
+            return
           end
-        elseif result[3]['return_code'] == 9 then
-          rspamd_logger.warnx(task, '%s: File is encrypted.', rule.log_prefix)
-          common.yield_result(task, rule, 'failed - err: ' .. oletools_rc[result[3]['return_code']], 0.0, 'fail')
-        elseif result[3]['return_code'] > 6 then
-          rspamd_logger.errx(task, '%s: Error Returned: %s',
-              rule.log_prefix, oletools_rc[result[3]['return_code']])
-          rspamd_logger.errx(task, '%s: Error message: %s',
-              rule.log_prefix, result[2]['message'])
-          common.yield_result(task, rule, 'failed - err: ' .. oletools_rc[result[3]['return_code']], 0.0, 'fail')
-        elseif result[3]['return_code'] > 1 then
-          rspamd_logger.errx(task, '%s: Error message: %s',
-              rule.log_prefix, result[2]['message'])
-          oletools_requery(oletools_rc[result[3]['return_code']])
-        elseif type(result[2]['analysis']) == 'table' and #result[2]['analysis'] == 0 and #result[2]['macros'] == 0 then
-          rspamd_logger.warnx(task, '%s: maybe unhandled python or oletools error', rule.log_prefix)
-          common.yield_result(task, rule, 'oletools unhandled error', 0.0, 'fail')
-        elseif type(result[2]['analysis']) ~= 'table' and #result[2]['macros'] == 0 then
-          common.save_av_cache(task, digest, rule, 'OK')
-          common.log_clean(task, rule, 'No macro found')
-        elseif #result[2]['macros'] > 0 then
-          -- M=Macros, A=Auto-executable, S=Suspicious keywords, I=IOCs,
-          -- H=Hex strings, B=Base64 strings, D=Dridex strings, V=VBA strings
-          local m_exist = 'M'
-          local m_autoexec = '-'
-          local m_suspicious = '-'
-          local m_iocs = '-'
-          local m_hex = '-'
-          local m_base64 = '-'
-          local m_dridex = '-'
-          local m_vba = '-'
 
-          lua_util.debugm(rule.name, task,
-              '%s: filename: %s', rule.log_prefix, result[2]['file'])
-          lua_util.debugm(rule.name, task,
-              '%s: type: %s', rule.log_prefix, result[2]['type'])
+          local result = ucl_parser:get_object()
+
+          local oletools_rc = {
+            [0] = 'RETURN_OK',
+            [1] = 'RETURN_WARNINGS',
+            [2] = 'RETURN_WRONG_ARGS',
+            [3] = 'RETURN_FILE_NOT_FOUND',
+            [4] = 'RETURN_XGLOB_ERR',
+            [5] = 'RETURN_OPEN_ERROR',
+            [6] = 'RETURN_PARSE_ERROR',
+            [7] = 'RETURN_SEVERAL_ERRS',
+            [8] = 'RETURN_UNEXPECTED',
+            [9] = 'RETURN_ENCRYPTED',
+          }
+
+          if result[1].error ~= nil then
+            rspamd_logger.errx(task, '%s: ERROR found: %s', rule.log_prefix,
+                result[1].error)
+            if result[1].error == 'File too small' then
+              common.save_av_cache(task, digest, rule, 'OK')
+              common.log_clean(task, rule, 'File too small to be scanned for macros')
+            else
+              oletools_requery(result[1].error)
+            end
+          elseif result[3]['return_code'] == 9 then
+            rspamd_logger.warnx(task, '%s: File is encrypted.', rule.log_prefix)
+            common.yield_result(task, rule, 'failed - err: ' .. oletools_rc[result[3]['return_code']], 0.0, 'fail')
+          elseif result[3]['return_code'] > 6 then
+            rspamd_logger.errx(task, '%s: Error Returned: %s',
+                rule.log_prefix, oletools_rc[result[3]['return_code']])
+            rspamd_logger.errx(task, '%s: Error message: %s',
+                rule.log_prefix, result[2]['message'])
+            common.yield_result(task, rule, 'failed - err: ' .. oletools_rc[result[3]['return_code']], 0.0, 'fail')
+          elseif result[3]['return_code'] > 1 then
+            rspamd_logger.errx(task, '%s: Error message: %s',
+                rule.log_prefix, result[2]['message'])
+            oletools_requery(oletools_rc[result[3]['return_code']])
+          elseif type(result[2]['analysis']) == 'table' and #result[2]['analysis'] == 0 and #result[2]['macros'] == 0 then
+            rspamd_logger.warnx(task, '%s: maybe unhandled python or oletools error', rule.log_prefix)
+            common.yield_result(task, rule, 'oletools unhandled error', 0.0, 'fail')
+          elseif type(result[2]['analysis']) ~= 'table' and #result[2]['macros'] == 0 then
+            common.save_av_cache(task, digest, rule, 'OK')
+            common.log_clean(task, rule, 'No macro found')
+          elseif #result[2]['macros'] > 0 then
+            -- M=Macros, A=Auto-executable, S=Suspicious keywords, I=IOCs,
+            -- H=Hex strings, B=Base64 strings, D=Dridex strings, V=VBA strings
+            local m_exist = 'M'
+            local m_autoexec = '-'
+            local m_suspicious = '-'
+            local m_iocs = '-'
+            local m_hex = '-'
+            local m_base64 = '-'
+            local m_dridex = '-'
+            local m_vba = '-'
+
+            lua_util.debugm(rule.name, task,
+                '%s: filename: %s', rule.log_prefix, result[2]['file'])
+            lua_util.debugm(rule.name, task,
+                '%s: type: %s', rule.log_prefix, result[2]['type'])
+
+            for _,m in ipairs(result[2]['macros']) do
+              lua_util.debugm(rule.name, task, '%s: macros found - code: %s, ole_stream: %s, '..
+                  'vba_filename: %s', rule.log_prefix, m.code, m.ole_stream, m.vba_filename)
+            end
 
-          for _,m in ipairs(result[2]['macros']) do
-            lua_util.debugm(rule.name, task, '%s: macros found - code: %s, ole_stream: %s, '..
-                'vba_filename: %s', rule.log_prefix, m.code, m.ole_stream, m.vba_filename)
-          end
+            local analysis_keyword_table = {}
 
-          local analysis_keyword_table = {}
-
-          for _,a in ipairs(result[2]['analysis']) do
-            lua_util.debugm(rule.name, task, '%s: threat found - type: %s, keyword: %s, '..
-                'description: %s', rule.log_prefix, a.type, a.keyword, a.description)
-            if a.type == 'AutoExec' then
-              m_autoexec = 'A'
-              table.insert(analysis_keyword_table, a.keyword)
-            elseif a.type == 'Suspicious' then
-              if rule.extended == true or
-                (a.keyword ~= 'Base64 Strings' and a.keyword ~= 'Hex Strings')
-              then
-                m_suspicious = 'S'
+            for _,a in ipairs(result[2]['analysis']) do
+              lua_util.debugm(rule.name, task, '%s: threat found - type: %s, keyword: %s, '..
+                  'description: %s', rule.log_prefix, a.type, a.keyword, a.description)
+              if a.type == 'AutoExec' then
+                m_autoexec = 'A'
                 table.insert(analysis_keyword_table, a.keyword)
+              elseif a.type == 'Suspicious' then
+                if rule.extended == true or
+                  (a.keyword ~= 'Base64 Strings' and a.keyword ~= 'Hex Strings')
+                then
+                  m_suspicious = 'S'
+                  table.insert(analysis_keyword_table, a.keyword)
+                end
+              elseif a.type == 'IOC' then
+                m_iocs = 'I'
+              elseif a.type == 'Hex strings' then
+                m_hex = 'H'
+              elseif a.type == 'Base64 strings' then
+                m_base64 = 'B'
+              elseif a.type == 'Dridex strings' then
+                m_dridex = 'D'
+              elseif a.type == 'VBA strings' then
+                m_vba = 'V'
               end
-            elseif a.type == 'IOC' then
-              m_iocs = 'I'
-            elseif a.type == 'Hex strings' then
-              m_hex = 'H'
-            elseif a.type == 'Base64 strings' then
-              m_base64 = 'B'
-            elseif a.type == 'Dridex strings' then
-              m_dridex = 'D'
-            elseif a.type == 'VBA strings' then
-              m_vba = 'V'
             end
-          end
 
-          --lua_util.debugm(N, task, '%s: analysis_keyword_table: %s', rule.log_prefix, analysis_keyword_table)
-
-          if rule.extended == false and m_autoexec == 'A' and m_suspicious == 'S' then
-            -- use single string as virus name
-            local threat = 'AutoExec + Suspicious (' .. table.concat(analysis_keyword_table, ',') .. ')'
-            lua_util.debugm(rule.name, task, '%s: threat result: %s', rule.log_prefix, threat)
-            common.yield_result(task, rule, threat, rule.default_score)
-            common.save_av_cache(task, digest, rule, threat, rule.default_score)
-
-          elseif rule.extended == true and #analysis_keyword_table > 0 then
-            -- report any flags (types) and any most keywords as individual virus name
-
-            local flags = m_exist ..
-                m_autoexec ..
-                m_suspicious ..
-                m_iocs ..
-                m_hex ..
-                m_base64 ..
-                m_dridex ..
-                m_vba
-            table.insert(analysis_keyword_table, 1, flags)
-
-            lua_util.debugm(rule.name, task, '%s: extended threat result: %s',
-                rule.log_prefix, table.concat(analysis_keyword_table, ','))
-
-            common.yield_result(task, rule, analysis_keyword_table, rule.default_score)
-            common.save_av_cache(task, digest, rule, analysis_keyword_table, rule.default_score)
+            --lua_util.debugm(N, task, '%s: analysis_keyword_table: %s', rule.log_prefix, analysis_keyword_table)
+
+            if rule.extended == false and m_autoexec == 'A' and m_suspicious == 'S' then
+              -- use single string as virus name
+              local threat = 'AutoExec + Suspicious (' .. table.concat(analysis_keyword_table, ',') .. ')'
+              lua_util.debugm(rule.name, task, '%s: threat result: %s', rule.log_prefix, threat)
+              common.yield_result(task, rule, threat, rule.default_score)
+              common.save_av_cache(task, digest, rule, threat, rule.default_score)
+
+            elseif rule.extended == true and #analysis_keyword_table > 0 then
+              -- report any flags (types) and any most keywords as individual virus name
+
+              local flags = m_exist ..
+                  m_autoexec ..
+                  m_suspicious ..
+                  m_iocs ..
+                  m_hex ..
+                  m_base64 ..
+                  m_dridex ..
+                  m_vba
+              table.insert(analysis_keyword_table, 1, flags)
+
+              lua_util.debugm(rule.name, task, '%s: extended threat result: %s',
+                  rule.log_prefix, table.concat(analysis_keyword_table, ','))
+
+              common.yield_result(task, rule, analysis_keyword_table, rule.default_score)
+              common.save_av_cache(task, digest, rule, analysis_keyword_table, rule.default_score)
+            else
+              common.save_av_cache(task, digest, rule, 'OK')
+              common.log_clean(task, rule, 'Scanned Macro is OK')
+            end
+
           else
-            common.save_av_cache(task, digest, rule, 'OK')
-            common.log_clean(task, rule, 'Scanned Macro is OK')
+            rspamd_logger.warnx(task, '%s: unhandled response', rule.log_prefix)
+            common.yield_result(task, rule, 'unhandled error', 0.0, 'fail')
           end
-
-        else
-          rspamd_logger.warnx(task, '%s: unhandled response', rule.log_prefix)
-          common.yield_result(task, rule, 'unhandled error', 0.0, 'fail')
         end
       end
     end