diff options
| -rw-r--r-- | conf/modules.d/p0f.conf | 46 | ||||
| -rw-r--r-- | lualib/lua_scanners/init.lua | 1 | ||||
| -rw-r--r-- | lualib/lua_scanners/p0f.lua | 210 | ||||
| -rw-r--r-- | src/plugins/lua/milter_headers.lua | 24 | ||||
| -rw-r--r-- | src/plugins/lua/p0f.lua | 117 | ||||
| -rw-r--r-- | test/functional/cases/161_p0f.robot | 88 | ||||
| -rw-r--r-- | test/functional/configs/p0f.conf | 11 | ||||
| -rw-r--r-- | test/functional/lib/vars.py | 1 | ||||
| -rwxr-xr-x | test/functional/util/dummy_p0f.py | 98 |
9 files changed, 596 insertions, 0 deletions
diff --git a/conf/modules.d/p0f.conf b/conf/modules.d/p0f.conf new file mode 100644 index 000000000..efeab1a40 --- /dev/null +++ b/conf/modules.d/p0f.conf @@ -0,0 +1,46 @@ +# Please don't modify this file as your changes might be overwritten with +# the next update. +# +# You can modify '$LOCAL_CONFDIR/rspamd.conf.local.override' to redefine +# parameters defined on the top level +# +# You can modify '$LOCAL_CONFDIR/rspamd.conf.local' to add +# parameters defined on the top level +# +# For specific modules or configuration you can also modify +# '$LOCAL_CONFDIR/local.d/file.conf' - to add your options or rewrite defaults +# '$LOCAL_CONFDIR/override.d/file.conf' - to override the defaults +# +# See https://rspamd.com/doc/tutorials/writing_rules.html for details + +p0f { + # Disable module by default + enabled = false; + + # Path to the unix socket that p0f listens on + socket = '/tmp/p0f.sock'; + + # Connection timeout + timeout = 5s; + + # If defined, insert symbol with lookup results + symbol = 'P0F'; + + # Patterns to match against results returned by p0f + # Symbol will be yielded on OS string, link type or distance matches + patterns = { + WINDOWS = '^Windows.*'; + #DSL = '^DSL$'; + #DISTANCE10 = '^distance:10$'; + } + + # Cache lifetime in seconds (default - 2 hours) + expire = 7200; + + # Cache key prefix + prefix = 'p0f'; + + .include(try=true,priority=5) "${DBDIR}/dynamic/p0f.conf" + .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/p0f.conf" + .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/p0f.conf" +} diff --git a/lualib/lua_scanners/init.lua b/lualib/lua_scanners/init.lua index 99cec68b3..4a10dc51b 100644 --- a/lualib/lua_scanners/init.lua +++ b/lualib/lua_scanners/init.lua @@ -43,6 +43,7 @@ require_scanner('oletools') require_scanner('icap') require_scanner('vadesecure') require_scanner('spamassassin') +require_scanner('p0f') exports.add_scanner = function(name, t, conf_func, check_func) assert(type(conf_func) == 'function' and type(check_func) == 'function', diff --git a/lualib/lua_scanners/p0f.lua b/lualib/lua_scanners/p0f.lua new file mode 100644 index 000000000..72093577b --- /dev/null +++ b/lualib/lua_scanners/p0f.lua @@ -0,0 +1,210 @@ +--[[ +Copyright (c) 2019, Vsevolod Stakhov <vsevolod@highsecure.ru> +Copyright (c) 2019, Denis Paavilainen <denpa@denpa.pro> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +--[[[ +-- @module p0f +-- This module contains p0f access functions +--]] + +local tcp = require "rspamd_tcp" +local rspamd_util = require "rspamd_util" +local rspamd_logger = require "rspamd_logger" +local lua_redis = require "lua_redis" +local lua_util = require "lua_util" +local common = require "lua_scanners/common" + +-- SEE: https://github.com/p0f/p0f/blob/v3.06b/docs/README#L317 +local S = { + BAD_QUERY = 0x0, + OK = 0x10, + NO_MATCH = 0x20 +} + +local N = 'p0f' + +local function p0f_check(task, ip, rule) + + local function ip2bin(addr) + addr = addr:to_table() + + for k, v in ipairs(addr) do + addr[k] = rspamd_util.pack('B', v) + end + + return table.concat(addr) + end + + local function trim(...) + local vars = {...} + + for k in pairs(vars) do + -- skip numbers, trim only strings + if tonumber(vars[k]) == nil then + vars[k] = string.gsub(vars[k], '[^%w-_\\.\\(\\) ]', '') + end + end + + return lua_util.unpack(vars) + end + + local function parse_p0f_response(data) + --[[ + p0f_api_response[232]: magic, status, first_seen, last_seen, total_conn, + uptime_min, up_mod_days, last_nat, last_chg, distance, bad_sw, os_match_q, + os_name, os_flavor, http_name, http_flavor, link_type, language + ]]-- + + data = tostring(data) + + -- API response must be 232 bytes long + if (#data < 232) then + rspamd_logger.errx(task, 'malformed response from p0f on %s, %s bytes', + rule.socket, #data) + + common.yield_result(task, rule, 'Malformed Response: ' .. rule.socket, + 0.0, 'fail') + return + end + + local _, status, _, _, _, uptime_min, _, _, _, distance, _, _, os_name, + os_flavor, _, _, link_type, _ = trim(rspamd_util.unpack( + 'I4I4I4I4I4I4I4I4I4hbbc32c32c32c32c32c32', data)) + + if status ~= S.OK then + if status == S.BAD_QUERY then + rspamd_logger.errx(task, 'malformed p0f query on %s', rule.socket) + common.yield_result(task, rule, 'Malformed Query: ' .. rule.socket, + 0.0, 'fail') + end + + return + end + + local os_string = #os_name == 0 and 'unknown' or os_name .. ' ' .. os_flavor + + task:get_mempool():set_variable('os_fingerprint', os_string, link_type, + uptime_min, distance) + + common.yield_result(task, rule, { + os_string, link_type, 'distance:' .. distance }, 0.0) + + return data + end + + local function make_p0f_request() + + local function check_p0f_cb(err, data) + + local function redis_set_cb(redis_set_err) + if redis_set_err then + rspamd_logger.errx(task, 'redis received an error: %s', redis_set_err) + return + end + end + + data = parse_p0f_response(data) + + if rule.redis_params then + local key = rule.prefix .. ip:to_string() + local ret = lua_redis.redis_make_request(task, + rule.redis_params, + key, + true, + redis_set_cb, + 'SETEX', + { key, tostring(rule.expire), data } + ) + + if not ret then + rspamd_logger.warnx(task, 'error connecting to redis') + end + end + end + + local query = rspamd_util.pack('I4 I1 c16', 0x50304601, + ip:get_version(), ip2bin(ip)) + + tcp.request({ + host = rule.socket, + callback = check_p0f_cb, + data = { query }, + task = task, + timeout = rule.timeout + }) + end + + local function redis_get_cb(err, data) + if err or type(data) ~= 'string' then + make_p0f_request() + else + parse_p0f_response(data) + end + end + + local ret = nil + if rule.redis_prams then + local key = rule.prefix .. ip:to_string() + ret = lua_redis.redis_make_request(task, + rule.redis_params, + key, + false, + redis_get_cb, + 'GET', + { key } + ) + end + + if not ret then + make_p0f_request() -- fallback to directly querying p0f + end +end + +local function p0f_config(opts) + local p0f_conf = { + name = N, + timeout = 5, + symbol = 'P0F', + symbol_fail = 'P0F_FAIL', + patterns = {}, + expire = 7200, + prefix = 'p0f', + detection_category = 'fingerprint', + message = '${SCANNER}: fingerprint matched: "${VIRUS}"' + } + + p0f_conf = lua_util.override_defaults(p0f_conf, opts) + p0f_conf.patterns = common.create_regex_table(p0f_conf.patterns) + + if not p0f_conf.log_prefix then + p0f_conf.log_prefix = p0f_conf.name + end + + if not p0f_conf.socket then + rspamd_logger.errx(rspamd_config, 'no servers defined') + return nil + end + + return p0f_conf +end + +return { + type = {N, 'fingerprint', 'scanner'}, + description = 'passive OS fingerprinter', + configure = p0f_config, + check = p0f_check, + name = N +} diff --git a/src/plugins/lua/milter_headers.lua b/src/plugins/lua/milter_headers.lua index 332625d7c..5d8d24c91 100644 --- a/src/plugins/lua/milter_headers.lua +++ b/src/plugins/lua/milter_headers.lua @@ -78,6 +78,10 @@ local settings = { symbols_fail = {}, symbols = {}, -- needs config }, + ['x-os-fingerprint'] = { + header = 'X-OS-Fingerprint', + remove = 0, + }, ['x-spamd-bar'] = { header = 'X-Spamd-Bar', positive = '+', @@ -413,6 +417,26 @@ local function milter_headers(task) end end + routines['x-os-fingerprint'] = function() + if skip_wanted('x-os-fingerprint') then return end + + local os_string, link_type, uptime_min, distance = + task:get_mempool():get_variable('os_fingerprint', + 'string, string, int, int'); + + if not os_string then return end + + local value = string.format('%s, (up: %u min), (distance %i, link: %s)', + os_string, uptime_min, distance, link_type) + + if settings.routines['x-os-fingerprint'].remove then + remove[settings.routines['x-os-fingerprint'].header] + = settings.routines['x-os-fingerprint'].remove + end + + add_header(settings.routines['x-os-fingerprint'].header, value) + end + routines['x-spam-status'] = function() if skip_wanted('x-spam-status') then return end if not common['metric_score'] then diff --git a/src/plugins/lua/p0f.lua b/src/plugins/lua/p0f.lua new file mode 100644 index 000000000..84c525536 --- /dev/null +++ b/src/plugins/lua/p0f.lua @@ -0,0 +1,117 @@ +--[[ +Copyright (c) 2019, Vsevolod Stakhov <vsevolod@highsecure.ru> +Copyright (c) 2019, Denis Paavilainen <denpa@denpa.pro> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +-- Detect remote OS via passive fingerprinting + +local lua_util = require "lua_util" +local lua_redis = require "lua_redis" +local rspamd_logger = require "rspamd_logger" +local p0f = require("lua_scanners").filter('p0f').p0f + +local N = 'p0f' + +if confighelp then + rspamd_config:add_example(nil, N, + 'Detect remote OS via passive fingerprinting', + [[ +p0f { + # Enable module + enabled = true + + # Path to the unix socket that p0f listens on + socket = '/tmp/p0f.sock'; + + # Connection timeout + timeout = 5s; + + # If defined, insert symbol with lookup results + symbol = 'P0F'; + + # Patterns to match against results returned by p0f + # Symbol will be yielded on OS string, link type or distance matches + patterns = { + WINDOWS = '^Windows.*'; + #DSL = '^DSL$'; + #DISTANCE10 = '^distance:10$'; + } + + # Cache lifetime in seconds (default - 2 hours) + expire = 7200; + + # Cache key prefix + prefix = 'p0f'; +} +]]) + return +end + +local rule + +local function check_p0f(task) + local ip = task:get_from_ip() + + if not (ip and ip:is_valid()) or ip:is_local() then + return + end + + p0f.check(task, ip, rule) +end + +local opts = rspamd_config:get_all_opt(N) + +rule = p0f.configure(opts) + +if rule then + rule.redis_params = lua_redis.parse_redis_server(N) + + local id = rspamd_config:register_symbol({ + name = 'P0F_CHECK', + type = 'prefilter,nostat', + callback = check_p0f, + priority = 8, + flags = 'empty', + group = N + }) + + if rule.symbol then + rspamd_config:register_symbol({ + name = rule.symbol, + parent = id, + type = 'virtual', + flags = 'empty', + group = N + }) + end + + for sym in pairs(rule.patterns) do + rspamd_logger.debugm(N, rspamd_config, 'registering: %1', { + type = 'virtual', + name = sym, + parent = id, + group = N + }) + rspamd_config:register_symbol({ + type = 'virtual', + name = sym, + parent = id, + group = N + }) + end +else + lua_util.disable_module(N, 'config') + rspamd_logger.infox('p0f module not configured'); +end diff --git a/test/functional/cases/161_p0f.robot b/test/functional/cases/161_p0f.robot new file mode 100644 index 000000000..9acbf7b2d --- /dev/null +++ b/test/functional/cases/161_p0f.robot @@ -0,0 +1,88 @@ +*** Settings *** +Suite Setup p0f Setup +Suite Teardown p0f Teardown +Library Process +Library ${TESTDIR}/lib/rspamd.py +Resource ${TESTDIR}/lib/rspamd.robot +Variables ${TESTDIR}/lib/vars.py + +*** Variables *** +${CONFIG} ${TESTDIR}/configs/plugins.conf +${MESSAGE} ${TESTDIR}/messages/spam_message.eml +${MESSAGE2} ${TESTDIR}/messages/freemail.eml +${REDIS_SCOPE} Suite +${RSPAMD_SCOPE} Suite +${URL_TLD} ${TESTDIR}/../lua/unit/test_tld.dat + +*** Test Cases *** +p0f MISS + Run Dummy p0f + ${result} = Scan Message With Rspamc ${MESSAGE} --ip 1.1.1.1 + Check Rspamc ${result} P0F + Check Rspamc ${result} WINDOWS inverse=1 + Check Rspamc ${result} P0F_FAIL inverse=1 + Shutdown p0f + +p0f HIT + Run Dummy p0f ${P0F_SOCKET} windows + ${result} = Scan Message With Rspamc ${MESSAGE} --ip 1.1.1.2 + Check Rspamc ${result} P0F inverse=1 + Check Rspamc ${result} ETHER + Check Rspamc ${result} DISTGE10 + Check Rspamc ${result} WINDOWS + Shutdown p0f + +p0f NOREDIS + Shutdown Process With Children ${REDIS_PID} + Run Dummy p0f + ${result} = Scan Message With Rspamc ${MESSAGE} --ip 1.1.1.3 + Check Rspamc ${result} P0F + Check Rspamc ${result} ETHER + Check Rspamc ${result} DISTGE10 + Check Rspamc ${result} P0F_FAIL inverse=1 + Shutdown p0f + +p0f NOMATCH + Run Dummy p0f ${P0F_SOCKET} windows no_match + ${result} = Scan Message With Rspamc ${MESSAGE} --ip 1.1.1.4 + Check Rspamc ${result} P0F inverse=1 + Check Rspamc ${result} WINDOWS inverse=1 + Shutdown p0f + +p0f BADQUERY + Run Dummy p0f ${P0F_SOCKET} windows bad_query + ${result} = Scan Message With Rspamc ${MESSAGE} --ip 1.1.1.5 + Check Rspamc ${result} P0F_FAIL + Check Rspamc ${result} Malformed Query + Check Rspamc ${result} WINDOWS inverse=1 + Shutdown p0f + +p0f FAILURE + Run Dummy p0f ${P0F_SOCKET} windows fail + ${result} = Scan Message With Rspamc ${MESSAGE} --ip 1.1.1.6 + Check Rspamc ${result} P0F_FAIL + Check Rspamc ${result} Malformed Response + Check Rspamc ${result} WINDOWS inverse=1 + Shutdown p0f + +*** Keywords *** +p0f Setup + ${PLUGIN_CONFIG} = Get File ${TESTDIR}/configs/p0f.conf + Set Suite Variable ${PLUGIN_CONFIG} + Generic Setup PLUGIN_CONFIG + Run Redis + +p0f Teardown + Normal Teardown + Shutdown Process With Children ${REDIS_PID} + Shutdown p0f + Terminate All Processes kill=True + +Shutdown p0f + ${p0f_pid} = Get File if exists /tmp/dummy_p0f.pid + Run Keyword if ${p0f_pid} Shutdown Process With Children ${p0f_pid} + +Run Dummy p0f + [Arguments] ${socket}=${P0F_SOCKET} ${os}=linux ${status}=ok + ${result} = Start Process ${TESTDIR}/util/dummy_p0f.py ${socket} ${os} ${status} + Wait Until Created /tmp/dummy_p0f.pid diff --git a/test/functional/configs/p0f.conf b/test/functional/configs/p0f.conf new file mode 100644 index 000000000..69303772a --- /dev/null +++ b/test/functional/configs/p0f.conf @@ -0,0 +1,11 @@ +redis { + servers = "${REDIS_ADDR}:${REDIS_PORT}"; +} +p0f { + socket = "${P0F_SOCKET}"; + patterns { + WINDOWS = '^Windows.*'; + ETHER = '^Ethernet.*'; + DISTGE10 = '^distance:[0-9]{2}$'; + } +} diff --git a/test/functional/lib/vars.py b/test/functional/lib/vars.py index 97b53b2e1..4559db205 100644 --- a/test/functional/lib/vars.py +++ b/test/functional/lib/vars.py @@ -15,6 +15,7 @@ PORT_PROXY = 56795 PORT_CLAM = 56796 PORT_FPROT = 56797 PORT_FPROT2_DUPLICATE = 56798 +P0F_SOCKET = '/tmp/p0f.sock' REDIS_ADDR = u'127.0.0.1' REDIS_PORT = 56379 NGINX_ADDR = u'127.0.0.1' diff --git a/test/functional/util/dummy_p0f.py b/test/functional/util/dummy_p0f.py new file mode 100755 index 000000000..e44844812 --- /dev/null +++ b/test/functional/util/dummy_p0f.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python + +PID = "/tmp/dummy_p0f.pid" + +import os +import sys +import struct +import socket +import dummy_killer +try: + import SocketServer as socketserver +except: + import socketserver + +class MyStreamHandler(socketserver.BaseRequestHandler): + + def handle(self): + S = { + 'bad_query' : 0x0, + 'ok' : 0x10, + 'no_match' : 0x20 + } + + OS = { + 'windows' : ('Windows', '7 or 8'), + 'linux' : ('Linux', '3.11 and newer') + } + + self.data = self.request.recv(21).strip() + + if self.server.p0f_status == 'fail': + response = 0 + else: + response = struct.pack( + "IbIIIIIIIhbb32s32s32s32s32s32s", + 0x50304602, # magic + S[self.server.p0f_status], # status + 1568493408, # first_seen + 1568493408, # last_seen + 1, # total_conn + 1, # uptime_min + 4, # up_mod_days + 1568493408, # last_nat + 1568493408, # last_chg + 10, # distance + 0, # bad_sw + 0, # os_match_q + OS[self.server.p0f_os][0], # os_name + OS[self.server.p0f_os][1], # os_flavor + '', # http_name + '', # http_flavor + 'Ethernet or modem', # link_type + '' # language + ) + + self.request.sendall(response) + self.request.close() + +def cleanup(SOCK): + if os.path.exists(SOCK): + try: + os.unlink(SOCK) + except OSError: + logging.warning("Could not unlink socket %s", SOCK) + +if __name__ == "__main__": + SOCK = '/tmp/p0f.sock' + p0f_status = 'ok' + p0f_os = 'linux' + + alen = len(sys.argv) + if alen > 1: + SOCK = sys.argv[1] + if alen >= 4: + p0f_os = sys.argv[2] + p0f_status = sys.argv[3] + elif alen >= 3: + p0f_os = sys.argv[2] + + cleanup(SOCK) + + server = socketserver.UnixStreamServer(SOCK, MyStreamHandler, bind_and_activate=False) + server.allow_reuse_address = True + server.p0f_status = p0f_status + server.p0f_os = p0f_os + server.server_bind() + server.server_activate() + + dummy_killer.setup_killer(server) + dummy_killer.write_pid(PID) + + try: + server.handle_request() + except socket.error: + print "Socket closed" + + server.server_close() + cleanup(SOCK) |