[Test] More test cases for HTTP API

.circleci/config.yml

@@ -93,7 +93,7 @@ jobs:
       - run: sudo apt-get install -qq libluajit-5.1-dev libpcre3-dev luarocks opendkim-tools python-pip redis-server
       - run: sudo apt-get install clickhouse-server
 
-      - run: sudo pip install demjson psutil robotframework requests
+      - run: sudo pip install demjson psutil robotframework requests http
       - run: sudo luarocks install luacheck
 
       - run: cd ../build

test/functional/cases/210_clickhouse/001_migration.robot

@@ -24,7 +24,7 @@ Migration
 
   Prepare rspamd
 
-  Sleep  1  #TODO: replace this check with waiting until migration finishes
+  Sleep  2  #TODO: replace this check with waiting until migration finishes
 
   Column should exist  rspamd  Symbols.Scores
   Column should exist  rspamd  Attachments.Digest

test/functional/cases/220_http.robot

@@ -1,5 +1,5 @@
 *** Settings ***
-# Test Setup      Http Setup
+Test Setup      Http Setup
 Test Teardown   Http Teardown
 Library         Process
 Library         ${TESTDIR}/lib/rspamd.py
@@ -11,31 +11,55 @@ Variables       ${TESTDIR}/lib/vars.py
 ${URL_TLD}      ${TESTDIR}/../lua/unit/test_tld.dat
 ${CONFIG}       ${TESTDIR}/configs/lua_test.conf
 ${MESSAGE}      ${TESTDIR}/messages/spam_message.eml
-${MESSAGE2}     ${TESTDIR}/messages/freemail.eml
 ${REDIS_SCOPE}  Suite
 ${RSPAMD_SCOPE}  Suite
 
 *** Test Cases ***
-HTTP
-  Run Dummy Http
-  [Setup]  Lua Setup  ${TESTDIR}/lua/http.lua
-  ${result} =  Scan Message With Rspamc  ${MESSAGE}
-  Check Rspamc  ${result}  HTTP_DNS_200
-  Check Rspamc  ${result}  HTTP_200
+Simple HTTP request
+  Check url  /request  get  HTTP_DNS_200  HTTP_200  HTTP_CORO_DNS_200  HTTP_CORO_200  method_get  hello world  HTTP_CORO_200 (0.00)[hello world]
+  Check url  /request  post  HTTP_DNS_200  HTTP_200  HTTP_CORO_DNS_200  HTTP_CORO_200  method_post  hello post  HTTP_CORO_DNS_200 (0.00)[hello post]
+
+*** Test Cases ***
+HTTP request 403
+  Check url  /error_403  get  HTTP_DNS_403  HTTP_403  HTTP_CORO_DNS_403  HTTP_CORO_403  method_get
+  Check url  /error_403  post  HTTP_DNS_403  HTTP_403  HTTP_CORO_DNS_403  HTTP_CORO_403  method_post
+
+
+*** Test Cases ***
+HTTP timeout
+  Check url  /timeout  get  HTTP_ERROR  HTTP_ERROR  HTTP_CORO_DNS_ERROR  HTTP_CORO_ERROR  method_get  IO timeout
+  Check url  /timeout  post  HTTP_DNS_ERROR  HTTP_ERROR  HTTP_CORO_DNS_ERROR  HTTP_CORO_ERROR  method_post  IO timeout
+
+
+*** Test Cases ***
+HTTP empty response
+  Check url  /empty  get  HTTP_ERROR  HTTP_ERROR  HTTP_CORO_DNS_ERROR  HTTP_CORO_ERROR  method_get  IO read error: unexpected EOF
+  Check url  /empty  post  HTTP_DNS_ERROR  HTTP_ERROR  HTTP_CORO_DNS_ERROR  HTTP_CORO_ERROR  method_post  IO read error: unexpected EOF
 
 
 *** Keywords ***
 Lua Setup
   [Arguments]  ${LUA_SCRIPT}
-  Set Test Variable  ${LUA_SCRIPT}
+  Set Global Variable  ${LUA_SCRIPT}
   Generic Setup
 
+Http Setup
+  Run Dummy Http
+  Lua Setup  ${TESTDIR}/lua/http.lua
+
 Http Teardown
   ${http_pid} =  Get File  /tmp/dummy_http.pid
-  # Shutdown Process With Children  ${http_pid}
+  Shutdown Process With Children  ${http_pid}
   Normal Teardown
 
 Run Dummy Http
   [Arguments]
   ${result} =  Start Process  ${TESTDIR}/util/dummy_http.py
   Wait Until Created  /tmp/dummy_http.pid
+
+
+Check url
+  [Arguments]  ${url}  ${method}  @{expect_results}
+  ${result} =  Scan Message With Rspamc  --header=url:${url}  --header=method:${method}  ${MESSAGE}
+  : FOR  ${expect}  IN  @{expect_results}
+  \  Check Rspamc  ${result}  ${expect}

test/functional/configs/plugins.conf

@@ -421,6 +421,11 @@ options = {
           name = "rspamd.tk",
           type = "txt";
           replies = ["bio=a263adeab8acdcdb8b89e127b67d696061fdfbee"];
+        },
+        {
+          name = "fail1.org.org.za",
+          type = "txt";
+          replies = ["v=spf1 redirect=www.dnssec-failed.org"];
         }];
   }
 }

test/functional/lua/http.lua

@@ -1,32 +1,78 @@
 local rspamd_http = require "rspamd_http"
+local rspamd_logger = require "rspamd_logger"
 
 local function http_symbol(task)
+
+  local url = tostring(task:get_request_header('url'))
+  local method = tostring(task:get_request_header('method'))
+
+  task:insert_result('method_' .. method, 1.0)
+
   local function http_callback(err, code, body)
-    task:insert_result('HTTP_' .. code, 1.0)
+    if err then
+      rspamd_logger.errx('http_callback error: ' .. err)
+      task:insert_result('HTTP_ERROR', 1.0, err)
+    else
+      task:insert_result('HTTP_' .. code, 1.0, body)
+    end
   end
 
   local function http_dns_callback(err, code, body)
-    task:insert_result('HTTP_DNS_' .. code, 1.0)
+    if err then
+      rspamd_logger.errx('http_dns_callback error: ' .. err)
+      task:insert_result('HTTP_DNS_ERROR', 1.0, err)
+    else
+      task:insert_result('HTTP_DNS_' .. code, 1.0, body)
+    end
   end
 
   rspamd_http.request({
-    url = 'http://127.0.0.1:18080/request',
+    url = 'http://127.0.0.1:18080' .. url,
     task = task,
-    method = 'post',
+    method = method,
     callback = http_callback,
+    timeout = 1,
   })
 
   --[[ request to this address involved DNS resolver subsystem ]]
   rspamd_http.request({
-    url = 'http://site.resolveme:18080/request',
+    url = 'http://site.resolveme:18080' .. url,
     task = task,
-    method = 'post',
+    method = method,
     callback = http_dns_callback,
+    timeout = 1,
   })
+
+  local err, response = rspamd_http.request({
+    url = 'http://127.0.0.1:18080' .. url,
+    task = task,
+    method = method,
+    timeout = 1,
+  })
+
+  if not err then
+    task:insert_result('HTTP_CORO_' .. response.code, 1.0, response.content)
+  else
+    task:insert_result('HTTP_CORO_ERROR', 1.0, err)
+  end
+
+  err, response = rspamd_http.request({
+    url = 'http://site.resolveme:18080' .. url,
+    task = task,
+    method = method,
+    timeout = 1,
+  })
+
+  if not err then
+    task:insert_result('HTTP_CORO_DNS_' .. response.code, 1.0, response.content)
+  else
+    task:insert_result('HTTP_CORO_DNS_ERROR', 1.0, err)
+  end
 end
 
 rspamd_config:register_symbol({
   name = 'SIMPLE_TEST',
   score = 1.0,
-  callback = http_symbol
+  callback = http_symbol,
+  no_squeeze = true
 })

test/functional/util/dummy_http.py

@@ -21,14 +21,36 @@ class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
 
     def do_GET(self):
         """Respond to a GET request."""
-        self.send_response(200)
+        if self.path == "/empty":
+            self.finish()
+            return
+
+        if self.path == "/timeout":
+            time.sleep(2)
+
+        if self.path == "/error_403":
+            self.send_response(403)
+        else:
+            self.send_response(200)
+
         self.send_header("Content-type", "text/plain")
         self.end_headers()
         self.wfile.write("hello world")
 
     def do_POST(self):
         """Respond to a GET request."""
-        self.send_response(200)
+        if self.path == "/empty":
+            self.finish()
+            return
+
+        if self.path == "/timeout":
+            time.sleep(2)
+
+        if self.path == "/error_403":
+            self.send_response(403)
+        else:
+            self.send_response(200)
+
         self.send_header("Content-type", "text/plain")
         self.end_headers()
         self.wfile.write("hello post")
@@ -69,7 +91,7 @@ if __name__ == '__main__':
 
     signal.signal(signal.SIGALRM, alarm_handler)
     signal.signal(signal.SIGTERM, alarm_handler)
-    signal.alarm(5)
+    signal.alarm(10)
 
     try:
         httpd.run()