- attach_workspace:
at: *workspace_root
+ - run: echo 'deb http://repo.yandex.ru/clickhouse/deb/stable/ main/' | sudo tee /etc/apt/sources.list.d/clickhouse.list
+ - run: sudo apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4 # optional, clickhouse key
+
- run: sudo apt-get update -qq || true
- run: sudo apt-get install -qq libluajit-5.1-dev libpcre3-dev luarocks opendkim-tools python-pip redis-server
+ - run: sudo apt-get install clickhouse-server
- - run: sudo pip install demjson psutil robotframework
+ - run: sudo pip install demjson psutil robotframework requests
- run: sudo luarocks install luacheck
- run: cd ../build
--- /dev/null
+*** Settings ***
+Documentation Checks if rspamd is able to upgrade migration schema from v0 (very initial) to v2
+Variables ${TESTDIR}/lib/vars.py
+Library ${TESTDIR}/lib/rspamd.py
+Library clickhouse.py
+Resource ${TESTDIR}/lib/rspamd.robot
+
+Suite Setup Clickhouse Setup
+Suite Teardown Clickhosue Teardown
+
+*** Variables ***
+${CONFIG} ${TESTDIR}/configs/clickhouse.conf
+${RSPAMD_SCOPE} Suite
+
+*** Test Cases ***
+Migration
+ Upload new schema ${TESTDIR}/data/initial_schema/schema.sql
+ Insert data rspamd ${TESTDIR}/data/initial_schema/data.rspamd.sql
+ Insert data rspamd_asn ${TESTDIR}/data/initial_schema/data.rspamd_asn.sql
+ Insert data rspamd_emails ${TESTDIR}/data/initial_schema/data.rspamd_emails.sql
+ Insert data rspamd_urls ${TESTDIR}/data/initial_schema/data.rspamd_urls.sql
+ Insert data rspamd_attachments ${TESTDIR}/data/initial_schema/data.rspamd_attachments.sql
+ Insert data rspamd_symbols ${TESTDIR}/data/initial_schema/data.rspamd_symbols.sql
+
+ Prepare rspamd
+
+ Sleep 1 #TODO: replace this check with waiting until migration finishes
+
+ Column should exist rspamd Symbols.Scores
+ Column should exist rspamd Attachments.Digest
+ Column should exist rspamd Symbols.Scores
+ Schema version should be 2
+
+
+*** Keywords ***
+Clickhouse Setup
+ ${TMPDIR} = Make Temporary Directory
+ Set Global Variable ${TMPDIR}
+ Set Directory Ownership ${TMPDIR} ${RSPAMD_USER} ${RSPAMD_GROUP}
+ ${template} = Get File ${TESTDIR}/configs/clickhouse-config.xml
+ ${config} = Replace Variables ${template}
+ Create File ${TMPDIR}/clickhouse-config.xml ${config}
+ Copy File ${TESTDIR}/configs/clickhouse-users.xml ${TMPDIR}/users.xml
+ Create Directory ${TMPDIR}/metadata
+ Create Directory ${TMPDIR}/metadata/default
+ Create Directory ${TMPDIR}/data/default
+ ${result} = Run Process clickhouse-server --daemon --config-file\=${TMPDIR}/clickhouse-config.xml --pid-file\=${TMPDIR}/clickhouse.pid
+ Run Keyword If ${result.rc} != 0 Log ${result.stderr}
+ Should Be Equal As Integers ${result.rc} 0
+ Wait Until Keyword Succeeds 5 sec 1 sec Check Pidfile ${TMPDIR}/clickhouse.pid timeout=5 sec
+ Set Suite Variable ${TMPDIR} ${TMPDIR}
+
+
+
+Clickhosue Teardown
+ # Sleep 30
+ ${clickhouse_pid} = Get File ${TMPDIR}/clickhouse.pid
+ Shutdown Process With Children ${clickhouse_pid}
+ Simple Teardown
+
+
+Prepare rspamd
+ &{d} = Run Rspamd CONFIG=${TESTDIR}/configs/clickhouse.conf TMPDIR=${TMPDIR}
+ ${keys} = Get Dictionary Keys ${d}
+ : FOR ${i} IN @{keys}
+ \ Run Keyword If '${RSPAMD_SCOPE}' == 'Suite' Set Suite Variable ${${i}} &{d}[${i}]
+ \ ... ELSE IF '${RSPAMD_SCOPE}' == 'Test' Set Test Variable ${${i}} &{d}[${i}]
+ \ ... ELSE Fail 'RSPAMD_SCOPE must be Test or Suite'
--- /dev/null
+import requests
+import json
+from robot.libraries.BuiltIn import BuiltIn
+from robot.api import logger
+
+__client = None
+
+
+class Client:
+ def __init__(self):
+ self.port = 18123
+
+ def get_query_string(self):
+ return "http://localhost:%d/?default_format=JSONEachRow" % (self.port)
+
+ def execute(self, sql):
+ r = requests.post(self.get_query_string(), sql)
+ if r.status_code != 200:
+ raise Exception("Clickhouse request failed: " + r.content)
+ return r
+
+ def query(self, sql):
+ r = self.execute(sql)
+
+ # logger.console("decoding " + r.content)
+ # [logger.console("decoding " + _) for _ in r.content.strip().split("\n")]
+ response = [json.loads(_) for _ in r.content.strip().split("\n")]
+ return response
+
+
+def client():
+ global __client
+ if __client is None:
+ __client = Client()
+ return __client
+
+
+def upload_new_schema(schema_file):
+ with open(schema_file, 'r') as content_file:
+ content = content_file.read()
+
+ queries = content.split(";")
+ for q in queries:
+ if q.strip() == "":
+ continue
+ client().execute(q) # throws exception on error
+
+
+def insert_data(table_name, filename):
+ with open(filename, 'r') as content_file:
+ content = content_file.read()
+
+ client().execute("insert into %s format Values %s;" % (table_name, content)) # throws exception on error
+
+
+def column_should_exist(table_name, column_name):
+ sql = "select hasColumnInTable('default', '%s', '%s') as is_exist" % (table_name, column_name)
+ r = client().query(sql)
+ if r[0]['is_exist'] != 1:
+ raise Exception("Failed asseting that column '%s' exists in table 'default'.'%s'" % (column_name, table_name))
+
+
+def schema_version_should_be(version):
+ sql = "select max(Version) as version from rspamd_version"
+ r = client().query(sql)
+ if r[0]['version'] != 2:
+ raise Exception("Failed asseting that schema version is '%d'" % version)
\ No newline at end of file
--- /dev/null
+<?xml version="1.0"?>
+<yandex>
+ <logger>
+ <!-- Possible levels: https://github.com/pocoproject/poco/blob/develop/Foundation/include/Poco/Logger.h#L105 -->
+ <level>trace</level>
+ <log>${TMPDIR}/clickhouse-server.log</log>
+ <errorlog>${TMPDIR}/clickhouse-server.err.log</errorlog>
+ <size>1000M</size>
+ <count>10</count>
+ <!-- <console>1</console> --> <!-- Default behavior is autodetection (log to console if not daemon mode and is tty) -->
+ </logger>
+ <!--display_name>production</display_name--> <!-- It is the name that will be shown in the client -->
+ <http_port>18123</http_port>
+ <tcp_port>19000</tcp_port>
+
+ <!-- For HTTPS and SSL over native protocol. -->
+ <!--
+ <https_port>8443</https_port>
+ <tcp_port_secure>9440</tcp_port_secure>
+ -->
+
+ <!-- Default root page on http[s] server. For example load UI from https://tabix.io/ when opening http://localhost:8123 -->
+ <!--
+ <http_server_default_response><![CDATA[<html ng-app="SMI2"><head><base href="http://ui.tabix.io/"></head><body><div ui-view="" class="content-ui"></div><script src="http://loader.tabix.io/master.js"></script></body></html>]]></http_server_default_response>
+ -->
+
+ <!-- Port for communication between replicas. Used for data exchange. -->
+ <interserver_http_port>19009</interserver_http_port>
+
+ <!-- Hostname that is used by other replicas to request this server.
+ If not specified, than it is determined analoguous to 'hostname -f' command.
+ This setting could be used to switch replication to another network interface.
+ -->
+ <!--
+ <interserver_http_host>example.yandex.ru</interserver_http_host>
+ -->
+
+ <!-- Listen specified host. use :: (wildcard IPv6 address), if you want to accept connections both with IPv4 and IPv6 from everywhere. -->
+ <!-- <listen_host>::</listen_host> -->
+ <!-- Same for hosts with disabled ipv6: -->
+ <!-- <listen_host>0.0.0.0</listen_host> -->
+
+ <!-- Default values - try listen localhost on ipv4 and ipv6: -->
+ <!--
+ <listen_host>::1</listen_host>
+ <listen_host>127.0.0.1</listen_host>
+ -->
+ <!-- Don't exit if ipv6 or ipv4 unavailable, but listen_host with this protocol specified -->
+ <!-- <listen_try>0</listen_try> -->
+
+ <!-- Allow listen on same address:port -->
+ <!-- <listen_reuse_port>0</listen_reuse_port> -->
+
+ <!-- <listen_backlog>64</listen_backlog> -->
+
+ <max_connections>4096</max_connections>
+ <keep_alive_timeout>3</keep_alive_timeout>
+
+ <!-- Maximum number of concurrent queries. -->
+ <max_concurrent_queries>100</max_concurrent_queries>
+
+ <!-- Set limit on number of open files (default: maximum). This setting makes sense on Mac OS X because getrlimit() fails to retrieve
+ correct maximum value. -->
+ <max_open_files>256</max_open_files>
+
+ <!-- Size of cache of uncompressed blocks of data, used in tables of MergeTree family.
+ In bytes. Cache is single for server. Memory is allocated only on demand.
+ Cache is used when 'use_uncompressed_cache' user setting turned on (off by default).
+ Uncompressed cache is advantageous only for very short queries and in rare cases.
+ -->
+ <uncompressed_cache_size>8589934592</uncompressed_cache_size>
+
+ <!-- Approximate size of mark cache, used in tables of MergeTree family.
+ In bytes. Cache is single for server. Memory is allocated only on demand.
+ You should not lower this value.
+ -->
+ <mark_cache_size>5368709120</mark_cache_size>
+
+
+ <!-- Path to data directory, with trailing slash. -->
+ <path>${TMPDIR}/</path>
+
+ <!-- Path to temporary data for processing hard queries. -->
+ <tmp_path>${TMPDIR}</tmp_path>
+
+ <!-- Directory with user provided files that are accessible by 'file' table function. -->
+ <user_files_path>${TMPDIR}</user_files_path>
+
+ <!-- Path to configuration file with users, access rights, profiles of settings, quotas. -->
+ <users_config>${TMPDIR}/users.xml</users_config>
+
+ <!-- Default profile of settings. -->
+ <default_profile>default</default_profile>
+
+ <!-- System profile of settings. This settings are used by internal processes (Buffer storage, Distibuted DDL worker and so on). -->
+ <!-- <system_profile>default</system_profile> -->
+
+ <!-- Default database. -->
+ <default_database>default</default_database>
+
+ <!-- Server time zone could be set here.
+
+ Time zone is used when converting between String and DateTime types,
+ when printing DateTime in text formats and parsing DateTime from text,
+ it is used in date and time related functions, if specific time zone was not passed as an argument.
+
+ Time zone is specified as identifier from IANA time zone database, like UTC or Africa/Abidjan.
+ If not specified, system time zone at server startup is used.
+
+ Please note, that server could display time zone alias instead of specified name.
+ Example: W-SU is an alias for Europe/Moscow and Zulu is an alias for UTC.
+ -->
+ <!-- <timezone>Europe/Moscow</timezone> -->
+
+ <!-- You can specify umask here (see "man umask"). Server will apply it on startup.
+ Number is always parsed as octal. Default umask is 027 (other users cannot read logs, data files, etc; group can only read).
+ -->
+ <!-- <umask>022</umask> -->
+
+ <!-- Configuration of clusters that could be used in Distributed tables.
+ https://clickhouse.yandex/docs/en/table_engines/distributed/
+ -->
+<!--
+ <remote_servers incl="clickhouse_remote_servers" >
+ <!– Test only shard config for testing distributed storage –>
+ <test_shard_localhost>
+ <shard>
+ <replica>
+ <host>localhost</host>
+ <port>19000</port>
+ </replica>
+ </shard>
+ </test_shard_localhost>
+ <test_shard_localhost_secure>
+ <shard>
+ <replica>
+ <host>localhost</host>
+ <port>19440</port>
+ <secure>1</secure>
+ </replica>
+ </shard>
+ </test_shard_localhost_secure>
+ </remote_servers>
+-->
+
+
+ <!-- If element has 'incl' attribute, then for it's value will be used corresponding substitution from another file.
+ By default, path to file with substitutions is /etc/metrika.xml. It could be changed in config in 'include_from' element.
+ Values for substitutions are specified in /yandex/name_of_substitution elements in that file.
+ -->
+
+ <!-- ZooKeeper is used to store metadata about replicas, when using Replicated tables.
+ Optional. If you don't use replicated tables, you could omit that.
+
+ See https://clickhouse.yandex/docs/en/table_engines/replication/
+ -->
+ <zookeeper incl="zookeeper-servers" optional="true" />
+
+ <!-- Substitutions for parameters of replicated tables.
+ Optional. If you don't use replicated tables, you could omit that.
+
+ See https://clickhouse.yandex/docs/en/table_engines/replication/#creating-replicated-tables
+ -->
+ <macros incl="macros" optional="true" />
+
+
+ <!-- Reloading interval for embedded dictionaries, in seconds. Default: 3600. -->
+ <builtin_dictionaries_reload_interval>3600</builtin_dictionaries_reload_interval>
+
+
+ <!-- Maximum session timeout, in seconds. Default: 3600. -->
+ <max_session_timeout>3600</max_session_timeout>
+
+ <!-- Default session timeout, in seconds. Default: 60. -->
+ <default_session_timeout>60</default_session_timeout>
+
+ <!-- Sending data to Graphite for monitoring. Several sections can be defined. -->
+ <!--
+ interval - send every X second
+ root_path - prefix for keys
+ hostname_in_path - append hostname to root_path (default = true)
+ metrics - send data from table system.metrics
+ events - send data from table system.events
+ asynchronous_metrics - send data from table system.asynchronous_metrics
+ -->
+ <!--
+ <graphite>
+ <host>localhost</host>
+ <port>42000</port>
+ <timeout>0.1</timeout>
+ <interval>60</interval>
+ <root_path>one_min</root_path>
+ <hostname_in_path>true</hostname_in_path>
+
+ <metrics>true</metrics>
+ <events>true</events>
+ <asynchronous_metrics>true</asynchronous_metrics>
+ </graphite>
+ <graphite>
+ <host>localhost</host>
+ <port>42000</port>
+ <timeout>0.1</timeout>
+ <interval>1</interval>
+ <root_path>one_sec</root_path>
+
+ <metrics>true</metrics>
+ <events>true</events>
+ <asynchronous_metrics>false</asynchronous_metrics>
+ </graphite>
+ -->
+
+
+ <!-- Query log. Used only for queries with setting log_queries = 1. -->
+ <query_log>
+ <!-- What table to insert data. If table is not exist, it will be created.
+ When query log structure is changed after system update,
+ then old table will be renamed and new table will be created automatically.
+ -->
+ <database>system</database>
+ <table>query_log</table>
+ <!--
+ PARTITION BY expr https://clickhouse.yandex/docs/en/table_engines/custom_partitioning_key/
+ Example:
+ event_date
+ toMonday(event_date)
+ toYYYYMM(event_date)
+ toStartOfHour(event_time)
+ -->
+ <partition_by>toYYYYMM(event_date)</partition_by>
+ <!-- Interval of flushing data. -->
+ <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+ </query_log>
+
+
+ <!-- Uncomment if use part_log
+ <part_log>
+ <database>system</database>
+ <table>part_log</table>
+
+ <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+ </part_log>
+ -->
+
+
+ <!-- Parameters for embedded dictionaries, used in Yandex.Metrica.
+ See https://clickhouse.yandex/docs/en/dicts/internal_dicts/
+ -->
+
+ <!-- Path to file with region hierarchy. -->
+ <!-- <path_to_regions_hierarchy_file>/opt/geo/regions_hierarchy.txt</path_to_regions_hierarchy_file> -->
+
+ <!-- Path to directory with files containing names of regions -->
+ <!-- <path_to_regions_names_files>/opt/geo/</path_to_regions_names_files> -->
+
+
+ <!-- Configuration of external dictionaries. See:
+ https://clickhouse.yandex/docs/en/dicts/external_dicts/
+ -->
+ <dictionaries_config>*_dictionary.xml</dictionaries_config>
+
+ <!-- Uncomment if you want data to be compressed 30-100% better.
+ Don't do that if you just started using ClickHouse.
+ -->
+ <compression incl="clickhouse_compression">
+ <!--
+ <!- - Set of variants. Checked in order. Last matching case wins. If nothing matches, lz4 will be used. - ->
+ <case>
+
+ <!- - Conditions. All must be satisfied. Some conditions may be omitted. - ->
+ <min_part_size>10000000000</min_part_size> <!- - Min part size in bytes. - ->
+ <min_part_size_ratio>0.01</min_part_size_ratio> <!- - Min size of part relative to whole table size. - ->
+
+ <!- - What compression method to use. - ->
+ <method>zstd</method>
+ </case>
+ -->
+ </compression>
+
+ <!-- Allow to execute distributed DDL queries (CREATE, DROP, ALTER, RENAME) on cluster.
+ Works only if ZooKeeper is enabled. Comment it if such functionality isn't required. -->
+ <distributed_ddl>
+ <!-- Path in ZooKeeper to queue with DDL queries -->
+ <path>/clickhouse/task_queue/ddl</path>
+
+ <!-- Settings from this profile will be used to execute DDL queries -->
+ <!-- <profile>default</profile> -->
+ </distributed_ddl>
+
+ <!-- Settings to fine tune MergeTree tables. See documentation in source code, in MergeTreeSettings.h -->
+ <!--
+ <merge_tree>
+ <max_suspicious_broken_parts>5</max_suspicious_broken_parts>
+ </merge_tree>
+ -->
+
+ <!-- Protection from accidental DROP.
+ If size of a MergeTree table is greater than max_table_size_to_drop (in bytes) than table could not be dropped with any DROP query.
+ If you want do delete one table and don't want to restart clickhouse-server, you could create special file <clickhouse-path>/flags/force_drop_table and make DROP once.
+ By default max_table_size_to_drop is 50GB, max_table_size_to_drop=0 allows to DROP any tables.
+ Uncomment to disable protection.
+ -->
+ <!-- <max_table_size_to_drop>0</max_table_size_to_drop> -->
+
+ <!-- Example of parameters for GraphiteMergeTree table engine -->
+ <graphite_rollup_example>
+ <pattern>
+ <regexp>click_cost</regexp>
+ <function>any</function>
+ <retention>
+ <age>0</age>
+ <precision>3600</precision>
+ </retention>
+ <retention>
+ <age>86400</age>
+ <precision>60</precision>
+ </retention>
+ </pattern>
+ <default>
+ <function>max</function>
+ <retention>
+ <age>0</age>
+ <precision>60</precision>
+ </retention>
+ <retention>
+ <age>3600</age>
+ <precision>300</precision>
+ </retention>
+ <retention>
+ <age>86400</age>
+ <precision>3600</precision>
+ </retention>
+ </default>
+ </graphite_rollup_example>
+
+ <!-- Directory in <clickhouse-path> containing schema files for various input formats.
+ The directory will be created if it doesn't exist.
+ -->
+ <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+
+ <!-- Uncomment to disable ClickHouse internal DNS caching. -->
+ <!-- <disable_internal_dns_cache>1</disable_internal_dns_cache> -->
+</yandex>
--- /dev/null
+<?xml version="1.0"?>
+<yandex>
+ <!-- Profiles of settings. -->
+ <profiles>
+ <!-- Default settings. -->
+ <default>
+ <!-- Maximum memory usage for processing single query, in bytes. -->
+ <max_memory_usage>10000000000</max_memory_usage>
+
+ <!-- Use cache of uncompressed blocks of data. Meaningful only for processing many of very short queries. -->
+ <use_uncompressed_cache>0</use_uncompressed_cache>
+
+ <!-- How to choose between replicas during distributed query processing.
+ random - choose random replica from set of replicas with minimum number of errors
+ nearest_hostname - from set of replicas with minimum number of errors, choose replica
+ with minumum number of different symbols between replica's hostname and local hostname
+ (Hamming distance).
+ in_order - first live replica is choosen in specified order.
+ -->
+ <load_balancing>random</load_balancing>
+ </default>
+
+ <!-- Profile that allows only read queries. -->
+ <readonly>
+ <readonly>1</readonly>
+ </readonly>
+ </profiles>
+
+ <!-- Users and ACL. -->
+ <users>
+ <!-- If user name was not specified, 'default' user is used. -->
+ <default>
+ <!-- Password could be specified in plaintext or in SHA256 (in hex format).
+
+ If you want to specify password in plaintext (not recommended), place it in 'password' element.
+ Example: <password>qwerty</password>.
+ Password could be empty.
+
+ If you want to specify SHA256, place it in 'password_sha256_hex' element.
+ Example: <password_sha256_hex>65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5</password_sha256_hex>
+
+ How to generate decent password:
+ Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-'
+ In first line will be password and in second - corresponding SHA256.
+ -->
+ <password></password>
+
+ <!-- List of networks with open access.
+
+ To open access from everywhere, specify:
+ <ip>::/0</ip>
+
+ To open access only from localhost, specify:
+ <ip>::1</ip>
+ <ip>127.0.0.1</ip>
+
+ Each element of list has one of the following forms:
+ <ip> IP-address or network mask. Examples: 213.180.204.3 or 10.0.0.1/8 or 10.0.0.1/255.255.255.0
+ 2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::.
+ <host> Hostname. Example: server01.yandex.ru.
+ To check access, DNS query is performed, and all received addresses compared to peer address.
+ <host_regexp> Regular expression for host names. Example, ^server\d\d-\d\d-\d\.yandex\.ru$
+ To check access, DNS PTR query is performed for peer address and then regexp is applied.
+ Then, for result of PTR query, another DNS query is performed and all received addresses compared to peer address.
+ Strongly recommended that regexp is ends with $
+ All results of DNS requests are cached till server restart.
+ -->
+ <networks incl="networks" replace="replace">
+ <ip>::/0</ip>
+ </networks>
+
+ <!-- Settings profile for user. -->
+ <profile>default</profile>
+
+ <!-- Quota for user. -->
+ <quota>default</quota>
+ </default>
+
+ <!-- Example of user with readonly access. -->
+ <readonly>
+ <password></password>
+ <networks incl="networks" replace="replace">
+ <ip>::1</ip>
+ <ip>127.0.0.1</ip>
+ </networks>
+ <profile>readonly</profile>
+ <quota>default</quota>
+ </readonly>
+ </users>
+
+ <!-- Quotas. -->
+ <quotas>
+ <!-- Name of quota. -->
+ <default>
+ <!-- Limits for time interval. You could specify many intervals with different limits. -->
+ <interval>
+ <!-- Length of interval. -->
+ <duration>3600</duration>
+
+ <!-- No limits. Just calculate resource usage for time interval. -->
+ <queries>0</queries>
+ <errors>0</errors>
+ <result_rows>0</result_rows>
+ <read_rows>0</read_rows>
+ <execution_time>0</execution_time>
+ </interval>
+ </default>
+ </quotas>
+</yandex>
--- /dev/null
+options = {
+ filters = ["spf", "dkim", "regexp"]
+ pidfile = "${TMPDIR}/rspamd.pid"
+ lua_path = "${INSTALLROOT}/share/rspamd/lib/?.lua"
+ dns {
+ nameserver = ["8.8.8.8", "8.8.4.4"];
+ retransmits = 10;
+ timeout = 2s;
+ fake_records = [{ # ed25519
+ name = "test._domainkey.example.com";
+ type = txt;
+ replies = ["k=ed25519; p=yi50DjK5O9pqbFpNHklsv9lqaS0ArSYu02qp1S0DW1Y="];
+ }];
+ }
+}
+clickhouse {
+ # Push update when 1000 records are collected (1000 if unset)
+ limit = 1;
+ # IP:port of Clickhouse server
+ server = "localhost:18123";
+ allow_local = true;
+}
+logging = {
+ type = "file",
+ level = "debug"
+ filename = "${TMPDIR}/rspamd.log"
+}
+metric = {
+ name = "default",
+ actions = {
+ reject = 100500,
+ }
+ unknown_weight = 1
+}
+worker {
+ type = normal
+ bind_socket = ${LOCAL_ADDR}:${PORT_NORMAL}
+ count = 1
+ task_timeout = 60s;
+}
+worker {
+ type = controller
+ bind_socket = ${LOCAL_ADDR}:${PORT_CONTROLLER}
+ count = 1
+ secure_ip = ["127.0.0.1", "::1"];
+ stats_path = "${TMPDIR}/stats.ucl"
+}
+modules {
+ path = "${TESTDIR}/../../src/plugins/lua/"
+}
+lua = "${INSTALLROOT}/share/rspamd/rules/rspamd.lua"
--- /dev/null
+('2018-08-21','2018-08-21 11:24:36','xent.com','evergo.net','127.0.0.0',1.19,0,3248,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','johnhall','','','friends of rohit khare <fork.xent.com>','162bc579fb3145dc3c65669328fa39cb'),
+('2018-08-21','2018-08-21 11:24:46','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,2890,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','c772a14916e765a4593939a5991bb859'),
+('2018-08-21','2018-08-21 11:24:46','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,975,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','83cfbad1dbcb7234843ff01b810ea2c0'),
+('2018-08-21','2018-08-21 11:24:46','xent.com','cse.ucsc.edu','127.0.0.0',0.69,0,3225,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','elias','','','friends of rohit khare <fork.xent.com>','9fa1aa06c0afe91f1aad86a128c475f6'),
+('2018-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+('2018-08-21','2018-08-21 11:24:15','example.sourceforge.net','users.sourceforge.net','127.0.0.0',0.69,0,4198,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','spamassassin-commits-admin','yyyyason','','','<spamassassin-commits.example.sourceforge.net>','b8c4054a876b3e6593b0c7ea427045ba'),
+('2018-08-21','2018-08-21 11:24:15','freshrpms.net','camperquake.de','127.0.0.0',1.69,0,3663,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rpm-zzzlist-admin','ralf','','','freshrpms rpm discussion list <rpm-zzzlist.freshrpms.net>','bb4fcb9417cb59838b2c83c4b4342ffa'),
+('2018-08-21','2018-08-21 11:24:15','freshrpms.net','rpmforge.net','127.0.0.0',1.69,0,3934,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rpm-zzzlist-admin','matthias','','','freshrpms rpm discussion list <rpm-zzzlist.freshrpms.net>','2b1b347f25159ca1ea8d4a9ee5187ba8'),
+('2018-08-21','2018-08-21 11:24:15','linux.ie','steorn.com','127.0.0.0',1.19,0,2934,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','ilug-admin','carlos.luna','','','irish linux users\' group <ilug.linux.ie>','c95d8ffd4f70a91328eeec15c452ab6b'),
+('2018-08-21','2018-08-21 11:24:15','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,1078,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','5a46062bde8cce224ea90aa134e10da1'),
+('2018-08-21','2018-08-21 11:24:15','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,2088,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','cf0d79f76e51a3e6c5bcfdfdac094685'),
+('2018-08-21','2018-08-21 11:24:15','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,2388,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','2ab3790853d8bf1a90623b6d788abeb0'),
+('2018-08-21','2018-08-21 11:24:15','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,1124,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','47362127e1870aa9aaf3eef8ae5c749b'),
+('2018-08-21','2018-08-21 11:24:15','xent.com','mithral.com','127.0.0.0',0.69,0,3546,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','beberg','','','friends of rohit khare <fork.xent.com>','f4d74c809ba48b987fd6271baf82c12f'),
+('2018-08-21','2018-08-21 11:24:20','baesystems.com','baesystems.com','127.0.0.0',0.9,0,3341,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','robert.chambers','robert.chambers','','','','835aaf36a75a56ce93dfa3badaeb92eb'),
+('2018-08-21','2018-08-21 11:24:20','example.sourceforge.net','telus.net','127.0.0.0',1.69,0,3852,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','spamassassin-talk-admin','cfortune','','','talk about spamassassin <spamassassin-talk.example.sourceforge.net>','8e799460b63b7919b4c4587dadd7b063'),
+('2018-08-21','2018-08-21 11:24:20','freshrpms.net','egwn.net','127.0.0.0',1.69,0,5074,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rpm-zzzlist-admin','matthias','','','freshrpms rpm discussion list <rpm-zzzlist.freshrpms.net>','3950ac78c11ec35d55cb56c265d9474a'),
+('2018-08-21','2018-08-21 11:24:20','freshrpms.net','fluid.com','127.0.0.0',0.69,0,3712,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rpm-zzzlist-admin','tengel','','','freshrpms rpm discussion list <rpm-zzzlist.freshrpms.net>','61936d1c00246dc8410391c9e5f03f5a'),
+('2018-08-21','2018-08-21 11:24:20','linux.ie','eircom.net','127.0.0.0',1.3,0,2763,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','ilug-admin','cout','','','irish linux users\' group <ilug.linux.ie>','d1e819de70503d5ebfa49b6708703942'),
+('2018-08-21','2018-08-21 11:24:20','petting-zoo.net','','127.0.0.0',1.8900000000000001,0,2538,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','0xdeadbeef-request','','','','','c6046b370d90b6b13a4788557e77cf24'),
+('2018-08-21','2018-08-21 11:24:20','returns.groups.yahoo.com','hotmail.com','127.0.0.0',0.89,0,4135,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','sentto-2242572-56028-1034088521-zzzz=example.com','skitster','','','','c22813bc4c29a3f5fa0dd1cbd598c7c8'),
+('2018-08-21','2018-08-21 11:24:20','returns.groups.yahoo.com','srv0.ems.ed.ac.uk','127.0.0.0',1.3900000000000001,0,5651,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','sentto-2242572-53809-1031308404-zzzz=example.com','martin','','','','a10e11dcdc3d527ade0f5e85bc583f78'),
+('2018-08-21','2018-08-21 11:24:20','returns.groups.yahoo.com','earthlink.net','127.0.0.0',1.3900000000000001,0,2885,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','sentto-2242572-60166-1038789202-yyyy=spamassassin.taint.org','billjac','','','','5a914e03141f3f7099e50be362c007c2'),
+('2018-08-21','2018-08-21 11:24:20','returns.groups.yahoo.com','srv0.ems.ed.ac.uk','127.0.0.0',1.3900000000000001,0,5706,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','sentto-2242572-53809-1031308404-zzzz=spamassassin.taint.org','martin','','','','a10e11dcdc3d527ade0f5e85bc583f78'),
+('2018-08-21','2018-08-21 11:24:20','returns.groups.yahoo.com','bestirishmusic.com','127.0.0.0',1.3900000000000001,0,3709,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','sentto-2242572-55915-1033992114-zzzz=spamassassin.taint.org','webmaster','','','','5e3597c0f14b241c915c161c462c4804'),
+('2018-08-21','2018-08-21 11:24:20','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,2168,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','472774888e942d317fecc91724b304f8'),
+('2018-08-21','2018-08-21 11:24:20','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,1125,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','54b07928d68e59fb370ea89f2efdaabd'),
+('2018-08-21','2018-08-21 11:24:20','xent.com','ianbell.com','127.0.0.0',0.69,0,15012,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','fork','','','friends of rohit khare <fork.xent.com>','64216f07ed5995242c78fe27b537c30b'),
+('2018-08-21','2018-08-21 11:24:20','xent.com','techmonkeys.net','127.0.0.0',0.69,0,4501,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','cdale','','','friends of rohit khare <fork.xent.com>','a96c963bc4d5297e3ebdf4f2ea27927f'),
+('2018-08-21','2018-08-21 11:24:20','xent.com','hotmail.com','127.0.0.0',1.69,0,2606,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','fork_list','','','friends of rohit khare <fork.xent.com>','9d743f10d435a607f9179d9429eb0ea1'),
+('2018-08-21','2018-08-21 11:24:20','xent.com','vertexdev.com','127.0.0.0',1.19,0,3224,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','jeff','','','friends of rohit khare <fork.xent.com>','5008d567ff90facc36d94be316d4e0fe'),
+('2018-08-21','2018-08-21 11:24:20','xent.com','panix.com','127.0.0.0',2.69,0,2366,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','lgonze','','','friends of rohit khare <fork.xent.com>','154b831310873a800b92169b7e049a8f'),
+('2018-08-21','2018-08-21 11:24:20','xent.com','magnesium.net','127.0.0.0',0.69,0,6039,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','bitbitch','','','friends of rohit khare <fork.xent.com>','0fcc0fdd0995b10de6f07585354e2941'),
+('2018-08-21','2018-08-21 11:24:20','xent.com','canada.com','127.0.0.0',0.69,0,3685,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','garym','','','friends of rohit khare <fork.xent.com>','c0796f14caaf4e02ca683309021f6812'),
+('2018-08-21','2018-08-21 11:24:25','freshrpms.net','egwn.net','127.0.0.0',1.69,0,3500,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rpm-zzzlist-admin','matthias','','','freshrpms rpm discussion list <rpm-zzzlist.freshrpms.net>','9848f3ace4d8b54ed52061cf0330d64f'),
+('2018-08-21','2018-08-21 11:24:25','xent.com','endeavors.com','127.0.0.0',0.69,0,3561,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','gbolcer','','','friends of rohit khare <fork.xent.com>','a5109e184c129e5521438be5375e7804'),
+('2018-08-21','2018-08-21 11:24:26','comcast.net','','undefined',6.9,0,4682,'unknown','unknown','unknown','unknown','unknown','unknown',0,'add header','tim.one','','','','','9367812a3c0f279aa3dbf691e9d9ec11'),
+('2018-08-21','2018-08-21 11:24:26','example.sourceforge.net','pobox.com','127.0.0.0',1.19,0,4167,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','razor-users-admin','wstearns','','','<razor-users.example.sourceforge.net>','8dd6255d0616f90f16ec8cfbba496143'),
+('2018-08-21','2018-08-21 11:24:26','xent.com','canada.com','127.0.0.0',0.69,0,3031,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','garym','','','friends of rohit khare <fork.xent.com>','e5ccbf749700f93ece4a41907d81593b'),
+('2018-08-21','2018-08-21 11:24:36','example.com','example.com','127.0.0.0',0.9,0,2154,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','27e6cbfca0423d5c2aa6fa6731670699'),
+('2018-08-21','2018-08-21 11:24:36','example.com','example.com','127.0.0.0',0.9,0,1630,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','b35cf34eb7186872f7c035a7bd450e92'),
+('2018-08-21','2018-08-21 11:24:36','example.com','panasas.com','127.0.0.0',0.69,0,4910,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','exmh-users-admin','welch','','','discussion list for exmh users <exmh-users.example.com>','7987eec136f5391445df5bfbe64d4b33'),
+('2018-08-21','2018-08-21 11:24:36','example.sourceforge.net','gmx.net','127.0.0.0',1.69,0,5667,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','spamassassin-talk-admin','msquadrat.nospamplease','','','talk about spamassassin <spamassassin-talk.example.sourceforge.net>','2c6645233690c3bd7677fc5a6b637000'),
+('2018-08-21','2018-08-21 11:24:36','freshrpms.net','bellsouth.net','127.0.0.0',0.69,0,4817,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rpm-zzzlist-admin','lance_tt','','','freshrpms rpm discussion list <rpm-zzzlist.freshrpms.net>','7201979c9c64847048d2e20512a6b5d9'),
+('2018-08-21','2018-08-21 11:24:36','freshrpms.net','ckloiber.com','127.0.0.0',0.69,0,3161,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rpm-zzzlist-admin','ckloiber','','','freshrpms rpm discussion list <rpm-zzzlist.freshrpms.net>','52fee32a1ec50eb0d0dab4dada87c497'),
+('2018-08-21','2018-08-21 11:24:36','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,1474,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','e3a16ef77e2ee48767c499104f67270b'),
+('2018-08-21','2018-08-21 11:24:36','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,1053,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','c3a8d98433975fefb64f3c059d6046f0'),
+('2018-08-21','2018-08-21 11:24:36','xent.com','techmonkeys.net','127.0.0.0',0.69,0,2218,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','cdale','','','friends of rohit khare <fork.xent.com>','672fa037061f621085fbe997d236f374'),
+('2018-08-21','2018-08-21 11:24:36','xent.com','','127.0.0.0',0.69,0,5136,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','','','','friends of rohit khare <fork.xent.com>','3b86bc8189eba5d6ed9b633ce2995c6d'),
+('2018-08-21','2018-08-21 11:24:36','xent.com','permafrost.net','127.0.0.0',0.69,0,3388,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','owen','','','friends of rohit khare <fork.xent.com>','03190fd537aa97e9be7d4a75fed57d36'),
+('2018-08-21','2018-08-21 11:24:36','xent.com','slack.net','127.0.0.0',0.69,0,2554,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','tomwhore','','','friends of rohit khare <fork.xent.com>','2a1f34828b8a6a9750a1a09bc305983d'),
+('2018-08-21','2018-08-21 11:24:36','xent.com','permafrost.net','127.0.0.0',0.69,0,4069,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','owen','','','friends of rohit khare <fork.xent.com>','9167f411217b18af953884010508dc2f'),
+('2018-08-21','2018-08-21 11:24:36','xent.com','magnesium.net','127.0.0.0',0.69,0,2547,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','bitbitch','','','friends of rohit khare <fork.xent.com>','8d5d3caf91661a5b36f857a3010c4a48'),
+('2018-08-21','2018-08-21 11:24:36','xent.com','canada.com','127.0.0.0',0.69,0,2806,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','garym','','','friends of rohit khare <fork.xent.com>','236f08f521364e701ad8dc43afde4fc4'),
+('2018-08-21','2018-08-21 11:24:46','example.com','example.com','127.0.0.0',0.9,0,1639,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','a8f132bbb47ea2b4310e550d9617023f'),
+('2018-08-21','2018-08-21 11:24:46','xent.com','hotmail.com','127.0.0.0',1.69,0,3376,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','deafbox','','','friends of rohit khare <fork.xent.com>','8d7f99415bb42f744c9d06ccf8d5ccbf')
\ No newline at end of file
--- /dev/null
+('2018-08-21','43607711d98b62ce6f3d13013ce7241f','--','--','--'),
+('2018-08-21','4ddf3d1d31558d038d9e7bba5a862494','--','--','--'),
+('2018-08-21','8db807c0b535792ea4e2d9b7f69b113c','--','--','--'),
+('2018-08-21','905721f001ce446b2ad23d6833716bcb','--','--','--'),
+('2018-08-21','8d7f99415bb42f744c9d06ccf8d5ccbf','--','--','--'),
+('2018-08-21','a8f132bbb47ea2b4310e550d9617023f','--','--','--'),
+('2018-08-21','03190fd537aa97e9be7d4a75fed57d36','--','--','--'),
+('2018-08-21','0fcc0fdd0995b10de6f07585354e2941','--','--','--'),
+('2018-08-21','154b831310873a800b92169b7e049a8f','--','--','--'),
+('2018-08-21','162bc579fb3145dc3c65669328fa39cb','--','--','--'),
+('2018-08-21','236f08f521364e701ad8dc43afde4fc4','--','--','--'),
+('2018-08-21','27e6cbfca0423d5c2aa6fa6731670699','--','--','--'),
+('2018-08-21','28ebd9e7e8a4ea4d54f18c02305382a8','--','--','--'),
+('2018-08-21','2a1f34828b8a6a9750a1a09bc305983d','--','--','--'),
+('2018-08-21','2ab3790853d8bf1a90623b6d788abeb0','--','--','--'),
+('2018-08-21','2b1b347f25159ca1ea8d4a9ee5187ba8','--','--','--'),
+('2018-08-21','2c6645233690c3bd7677fc5a6b637000','--','--','--'),
+('2018-08-21','3950ac78c11ec35d55cb56c265d9474a','--','--','--'),
+('2018-08-21','3b86bc8189eba5d6ed9b633ce2995c6d','--','--','--'),
+('2018-08-21','472774888e942d317fecc91724b304f8','--','--','--'),
+('2018-08-21','47362127e1870aa9aaf3eef8ae5c749b','--','--','--'),
+('2018-08-21','5008d567ff90facc36d94be316d4e0fe','--','--','--'),
+('2018-08-21','51cd92041e96327f9fc502ee19990c04','--','--','--'),
+('2018-08-21','52fee32a1ec50eb0d0dab4dada87c497','--','--','--'),
+('2018-08-21','54b07928d68e59fb370ea89f2efdaabd','--','--','--'),
+('2018-08-21','5a46062bde8cce224ea90aa134e10da1','--','--','--'),
+('2018-08-21','5a914e03141f3f7099e50be362c007c2','--','--','--'),
+('2018-08-21','5e3597c0f14b241c915c161c462c4804','--','--','--'),
+('2018-08-21','61936d1c00246dc8410391c9e5f03f5a','--','--','--'),
+('2018-08-21','64216f07ed5995242c78fe27b537c30b','--','--','--'),
+('2018-08-21','672fa037061f621085fbe997d236f374','--','--','--'),
+('2018-08-21','69afbde36dc807ef7190ad549f94e697','--','--','--'),
+('2018-08-21','7201979c9c64847048d2e20512a6b5d9','--','--','--'),
+('2018-08-21','7987eec136f5391445df5bfbe64d4b33','--','--','--'),
+('2018-08-21','835aaf36a75a56ce93dfa3badaeb92eb','--','--','--'),
+('2018-08-21','8d5d3caf91661a5b36f857a3010c4a48','--','--','--'),
+('2018-08-21','8dd6255d0616f90f16ec8cfbba496143','--','--','--'),
+('2018-08-21','8e799460b63b7919b4c4587dadd7b063','--','--','--'),
+('2018-08-21','9367812a3c0f279aa3dbf691e9d9ec11','--','--','--'),
+('2018-08-21','9848f3ace4d8b54ed52061cf0330d64f','--','--','--'),
+('2018-08-21','9d743f10d435a607f9179d9429eb0ea1','--','--','--'),
+('2018-08-21','a10e11dcdc3d527ade0f5e85bc583f78','--','--','--'),
+('2018-08-21','a10e11dcdc3d527ade0f5e85bc583f78','--','--','--'),
+('2018-08-21','a5109e184c129e5521438be5375e7804','--','--','--'),
+('2018-08-21','a96c963bc4d5297e3ebdf4f2ea27927f','--','--','--'),
+('2018-08-21','b35cf34eb7186872f7c035a7bd450e92','--','--','--'),
+('2018-08-21','b3e5e01c86a8fd8a9bc5a66d033cc32e','--','--','--'),
+('2018-08-21','b8c4054a876b3e6593b0c7ea427045ba','--','--','--'),
+('2018-08-21','bb4fcb9417cb59838b2c83c4b4342ffa','--','--','--'),
+('2018-08-21','c0796f14caaf4e02ca683309021f6812','--','--','--'),
+('2018-08-21','c22813bc4c29a3f5fa0dd1cbd598c7c8','--','--','--'),
+('2018-08-21','c4bb938afcc5d6af78c0ea66145bbc2d','--','--','--'),
+('2018-08-21','c6046b370d90b6b13a4788557e77cf24','--','--','--'),
+('2018-08-21','c772a14916e765a4593939a5991bb859','--','--','--'),
+('2018-08-21','c95d8ffd4f70a91328eeec15c452ab6b','--','--','--'),
+('2018-08-21','cd6dea756b83f81172d07a1e0f5b08a9','--','--','--'),
+('2018-08-21','cf0d79f76e51a3e6c5bcfdfdac094685','--','--','--'),
+('2018-08-21','d0bbb7fbc8d4e31471a25456fcfaff64','--','--','--'),
+('2018-08-21','d10e5a118896958c447eaa343e3d2db8','--','--','--'),
+('2018-08-21','d1e819de70503d5ebfa49b6708703942','--','--','--'),
+('2018-08-21','d9ffc9704faf5057b1394c507153c277','--','--','--'),
+('2018-08-21','e3a16ef77e2ee48767c499104f67270b','--','--','--'),
+('2018-08-21','e5ccbf749700f93ece4a41907d81593b','--','--','--'),
+('2018-08-21','f4d74c809ba48b987fd6271baf82c12f','--','--','--'),
+('2018-08-21','faf1b3f92d8aa05c97fbc212f11afc54','--','--','--'),
+('2018-08-21','feae0c04282d5e27705366d3211bc69e','--','--','--')
\ No newline at end of file
--- /dev/null
+('2018-08-08', 'd1e819de70503d5ebfa49b6708703942', ['fsouoxtjvb.xls'], ['application/vnd.ms-excel'], [39424], ['d66d8dfcab696065']),
+('2018-08-08', '5e3597c0f14b241c915c161c462c4804', ['eybxf.gif','llfoe.gif'], ['image/gif','image/gif'], [6558,67], ['d3e6d83395091599','abfae96a0fc709e9']),
+('2018-08-08', '8dd6255d0616f90f16ec8cfbba496143', ['fake.xls'], ['application/vnd.ms-excel'], [361472], ['e0791f7961db0e00']),
+('2018-08-08', '8d7f99415bb42f744c9d06ccf8d5ccbf', ['dsn_status','header'], ['message/delivery-status','text/rfc822-headers'], [451,650], ['4e5127839ad4af96','ee1db81d572b606c']),
+('2018-08-08', '2b1b347f25159ca1ea8d4a9ee5187ba8', ['vdxcpjg.xls'], ['application/vnd.ms-excel'], [39424], ['163b098ba7aee8f9']),
+('2018-08-08', '2ab3790853d8bf1a90623b6d788abeb0', ['photo399.jpg'], ['image/jpeg'], [22495], ['7fa378464a00d645']),
+('2018-08-08', '0fcc0fdd0995b10de6f07585354e2941', ['photo399.jpg'], ['image/jpeg'], [22495], ['7fa378464a00d645']),
+('2018-08-08', '7987eec136f5391445df5bfbe64d4b33', ['photo399.jpg'], ['image/jpeg'], [22495], ['7fa378464a00d645']),
+('2018-08-08', '2c6645233690c3bd7677fc5a6b637000', ['photo399.jpg'], ['image/jpeg'], [22495], ['7fa378464a00d645']),
+('2018-08-08', '52fee32a1ec50eb0d0dab4dada87c497', ['photo399.jpg'], ['image/jpeg'], [22495], ['7fa378464a00d645'])
\ No newline at end of file
--- /dev/null
+('2018-08-08', '2b1b347f25159ca1ea8d4a9ee5187ba8', ['fake1@mail.ru']),
+('2018-08-08', '2ab3790853d8bf1a90623b6d788abeb0', ['fake2@mail.ru']),
+('2018-08-08', '0fcc0fdd0995b10de6f07585354e2941', ['info_stop_mailer@yahoo.co.jp']),
+('2018-08-08', '7987eec136f5391445df5bfbe64d4b33', ['info_stop_mailer@yahoo.co.jp']),
+('2018-08-08', '2c6645233690c3bd7677fc5a6b637000', ['info_stop_mailer@yahoo.co.jp']),
+('2018-08-08', '52fee32a1ec50eb0d0dab4dada87c497', ['fake3@g-mail.com','fake4@g-mail.com'])
\ No newline at end of file
--- /dev/null
+('2018-08-08', 'd1e819de70503d5ebfa49b6708703942', ['arc_na','rcvd_via_smtp_auth','xm_ua_no_version','from_has_dn','spamtrap','missing_mime_version','to_match_envrcpt_all','to_dn_none','rcpt_count_one','many_invisible_parts','freemail_envrcpt','mime_base64_text','subj_all_caps','mime_html_only','rcvd_count_one','freemail_to','rcvd_no_tls_last','from_eq_envfrom','has_x_prio_one','asn','mid_rhs_match_from'], [0,0,0.01,0,0,2,0,0,0,0.05,0,0.1,3,0.2,0,0,0,0,0,0,0], ['','','','','fakexx@qq.com','','','','1','1','qq.com','','68','','1','qq.com','','','1','asn:4134, ipnet:49.64.0.0/11, country:cn','']),
+('2018-08-08', '5e3597c0f14b241c915c161c462c4804', ['arc_na','rcvd_via_smtp_auth','from_has_dn','to_match_envrcpt_all','mv_case','mime_good','to_dn_none','rcpt_count_one','rcvd_count_one','spamtrap','rcvd_no_tls_last','from_eq_envfrom','asn','mid_rhs_match_from'], [0,0,0,0,0.5,-0.1,0,0,0,0,0,0,0,0], ['','','','','','text/plain','','1','1','fake@mailto.gq','','','asn:201725, ipnet:176.97.248.0/21, country:pl','']),
+('2018-08-08', '8dd6255d0616f90f16ec8cfbba496143', ['arc_na','rcvd_via_smtp_auth','xm_ua_no_version','from_has_dn','spamtrap','missing_mime_version','to_match_envrcpt_all','subject_needs_encoding','to_dn_none','from_needs_encoding','rcpt_count_one','many_invisible_parts','freemail_envrcpt','mime_base64_text','mime_html_only','rcvd_count_one','freemail_to','rcvd_no_tls_last','from_eq_envfrom','asn','has_x_prio_five','mid_rhs_match_from'], [0,0,0.01,0,0,2,0,1,0,1,0,0.05,0,0.1,0.2,0,0,0,0,0,0,0], ['','','','','fakeq3@qq.com','','','','','','1','1','qq.com','','','1','qq.com','','','asn:4134, ipnet:113.120.0.0/13, country:cn','5','']),
+('2018-08-08', '8d7f99415bb42f744c9d06ccf8d5ccbf', ['arc_na','rcvd_via_smtp_auth','from_has_dn','to_match_envrcpt_all','mv_case','mime_good','to_dn_none','rcpt_count_one','rcvd_count_one','spamtrap','rcvd_no_tls_last','from_eq_envfrom','asn','mid_rhs_match_from'], [0,0,0,0,0.5,-0.1,0,0,0,0,0,0,0,0], ['','','','','','text/plain','','1','1','fake@mailto.gq','','','asn:28187, ipnet:189.89.208.0/22, country:br','']),
+('2018-08-08', '2b1b347f25159ca1ea8d4a9ee5187ba8', ['arc_na','rcvd_via_smtp_auth','from_has_dn','to_match_envrcpt_all','mv_case','mime_good','to_dn_none','rcpt_count_one','rcvd_count_one','spamtrap','rcvd_no_tls_last','from_eq_envfrom','asn','mid_rhs_match_from'], [0,0,0,0,0.5,-0.1,0,0,0,0,0,0,0,0], ['','','','','','text/plain','','1','1','fake3@wowmail.cf','','','asn:52935, ipnet:191.5.108.0/23, country:br','']),
+('2018-08-08', '2ab3790853d8bf1a90623b6d788abeb0', ['arc_na','rcvd_via_smtp_auth','zetascan_uri','from_has_dn','missing_mime_version','to_match_envrcpt_all','mime_good','to_dn_none','freemail_envrcpt','rambler_uribl','has_x_prio_three','rcpt_count_seven','dbl_abuse','rcvd_count_one','freemail_to','fsl_single_url','rcvd_no_tls_last','invuri','asn','freemail_cc'], [0,0,0,0,2,0,-0.1,0,0,0,0,0,6.5,0,0,0.1,0,0,0,0], ['','','suportdesk.com.e93be9862ef26e5fccb23696d6707ffc.dblack.api.zetascan.com','','','','text/plain','','bellsouth.net,aol.com,gmx.de,hotmail.com,gmail.com,me.com','suportdesk.com','3','7','suportdesk.com.dbl.uri.fslupdate.com','1','bellsouth.net','','','suportdesk.com.invuri.uri.fslupdate.com','asn:10429, ipnet:189.56.0.0/15, country:br','aol.com']),
+('2018-08-08', '0fcc0fdd0995b10de6f07585354e2941', ['arc_na','rcvd_via_smtp_auth','xm_ua_no_version','from_has_dn','spamtrap','to_match_envrcpt_all','freemail_envrcpt','r_bad_cte_7bit','to_dn_none','subject_needs_encoding','rcpt_count_one','many_invisible_parts','has_x_prio_three','mime_html_only','rcvd_count_one','freemail_to','rcvd_no_tls_last','from_eq_envfrom','asn','mid_rhs_match_from'], [0,0,0.01,0,0,0,0,3.5,0,1,0,0.05,0,0.2,0,0,0,0,0,0], ['','','','','fake5@qq.com','','qq.com','7bit','','','1','1','3','','1','qq.com','','','asn:4134, ipnet:113.120.0.0/13, country:cn','']),
+('2018-08-08', '7987eec136f5391445df5bfbe64d4b33', ['arc_na','rcvd_via_smtp_auth','zetascan_uri','from_has_dn','to_match_envrcpt_all','sem_uribl','to_dn_none','rambler_uribl','many_invisible_parts','mime_base64_text','has_x_prio_three','rcpt_count_twelve','mime_html_only','rcvd_count_one','uribl_black','rcvd_no_tls_last','from_eq_envfrom','mid_rhs_not_fqdn','invuri','asn','dbl_spam','uribl_sbl','forged_outlook_html'], [0,0,0,0,0,3.5,0,0,0.7,0.1,0,0,0.2,0,7.5,0,0,0.5,0,0,6.5,6.5,5], ['','','gdguojian.net.e93be9862ef26e5fccb23696d6707ffc.dblack.api.zetascan.com','','','gdguojian.net.uribl.spameatingmonkey.net','','gdguojian.net','8','','3','21','','1','gdguojian.net.multi.uri.fslupdate.com','','','','gdguojian.net.invuri.uri.fslupdate.com','asn:4134, ipnet:182.96.0.0/12, country:cn','gdguojian.net.dbl.uri.fslupdate.com','gdguojian.net','']),
+('2018-08-08', '2c6645233690c3bd7677fc5a6b637000', ['arc_na','rcvd_via_smtp_auth','xm_ua_no_version','from_has_dn','spamtrap','missing_mime_version','to_match_envrcpt_all','to_dn_none','rcpt_count_one','many_invisible_parts','freemail_envrcpt','subj_all_caps','mime_html_only','rcvd_count_one','freemail_to','rcvd_no_tls_last','from_eq_envfrom','asn','has_x_prio_five','mid_rhs_match_from'], [0,0,0.01,0,0,2,0,0,0,0.05,0,3,0.2,0,0,0,0,0,0,0], ['','','','','fake9@qq.com','','','','1','1','qq.com','74','','1','qq.com','','','asn:4134, ipnet:182.32.0.0/12, country:cn','5','']),
+('2018-08-08', '52fee32a1ec50eb0d0dab4dada87c497', ['arc_na','rcvd_via_smtp_auth','from_has_dn','to_match_envrcpt_all','mv_case','mime_good','to_dn_none','rcpt_count_one','rcvd_count_one','spamtrap','rcvd_no_tls_last','from_eq_envfrom','asn','mid_rhs_match_from'], [0,0,0,0,0.5,-0.1,0,0,0,0,0,0,0,0], ['','','','','','text/plain','','1','1','fake10@mailto.gq','','','asn:262812, ipnet:200.66.120.0/23, country:br',''])
\ No newline at end of file
--- /dev/null
+('2018-08-08', 'd1e819de70503d5ebfa49b6708703942', ['gdguojian.net'], ['http://gdguojian.net/']),
+('2018-08-08', '5e3597c0f14b241c915c161c462c4804', ['gdguojian.net'], ['http://gdguojian.net/']),
+('2018-08-08', '8dd6255d0616f90f16ec8cfbba496143', ['gdguojian.net'], ['http://gdguojian.net/']),
+('2018-08-08', '8d7f99415bb42f744c9d06ccf8d5ccbf', ['gdguojian.net'], ['http://gdguojian.net/']),
+('2018-08-08', '2b1b347f25159ca1ea8d4a9ee5187ba8', ['gdguojian.net'], ['http://gdguojian.net/']),
+('2018-08-08', '2ab3790853d8bf1a90623b6d788abeb0', ['gdguojian.net'], ['http://gdguojian.net/']),
+('2018-08-08', '0fcc0fdd0995b10de6f07585354e2941', ['gdguojian.net'], ['http://gdguojian.net/']),
+('2018-08-08', '7987eec136f5391445df5bfbe64d4b33', ['gdguojian.net'], ['http://gdguojian.net/']),
+('2018-08-08', '2c6645233690c3bd7677fc5a6b637000', ['gdguojian.net'], ['http://gdguojian.net/']),
+('2018-08-08', '52fee32a1ec50eb0d0dab4dada87c497', ['gdguojian.net'], ['http://gdguojian.net/'])
\ No newline at end of file
--- /dev/null
+CREATE TABLE IF NOT EXISTS rspamd
+(
+ Date Date,
+ TS DateTime,
+ From String,
+ MimeFrom String,
+ IP String,
+ Score Float64,
+ NRcpt UInt8,
+ Size UInt32,
+ IsWhitelist Enum8('blacklist' = 0, 'whitelist' = 1, 'unknown' = 2) DEFAULT CAST('unknown' AS Enum8('blacklist' = 0, 'whitelist' = 1, 'unknown' = 2)),
+ IsBayes Enum8('ham' = 0, 'spam' = 1, 'unknown' = 2) DEFAULT CAST('unknown' AS Enum8('ham' = 0, 'spam' = 1, 'unknown' = 2)),
+ IsFuzzy Enum8('whitelist' = 0, 'deny' = 1, 'unknown' = 2) DEFAULT CAST('unknown' AS Enum8('whitelist' = 0, 'deny' = 1, 'unknown' = 2)),
+ IsFann Enum8('ham' = 0, 'spam' = 1, 'unknown' = 2) DEFAULT CAST('unknown' AS Enum8('ham' = 0, 'spam' = 1, 'unknown' = 2)),
+ IsDkim Enum8('reject' = 0, 'allow' = 1, 'unknown' = 2) DEFAULT CAST('unknown' AS Enum8('reject' = 0, 'allow' = 1, 'unknown' = 2)),
+ IsDmarc Enum8('reject' = 0, 'allow' = 1, 'unknown' = 2) DEFAULT CAST('unknown' AS Enum8('reject' = 0, 'allow' = 1, 'unknown' = 2)),
+ NUrls Int32,
+ Action Enum8('reject' = 0, 'rewrite subject' = 1, 'add header' = 2, 'greylist' = 3, 'no action' = 4, 'soft reject' = 5) DEFAULT CAST('no action' AS Enum8('reject' = 0, 'rewrite subject' = 1, 'add header' = 2, 'greylist' = 3, 'no action' = 4, 'soft reject' = 5)),
+ FromUser String,
+ MimeUser String,
+ RcptUser String,
+ RcptDomain String,
+ ListId String,
+ Digest FixedString(32)
+) ENGINE = MergeTree(Date, (TS, From), 8192);
+
+
+CREATE TABLE IF NOT EXISTS rspamd_attachments (
+ Date Date,
+ Digest FixedString(32),
+ `Attachments.FileName` Array(String),
+ `Attachments.ContentType` Array(String),
+ `Attachments.Length` Array(UInt32),
+ `Attachments.Digest` Array(FixedString(16))
+) ENGINE = MergeTree(Date, Digest, 8192);
+
+
+CREATE TABLE IF NOT EXISTS rspamd_urls (
+ Date Date,
+ Digest FixedString(32),
+ `Urls.Tld` Array(String),
+ `Urls.Url` Array(String)
+) ENGINE = MergeTree(Date, Digest, 8192);
+
+
+CREATE TABLE IF NOT EXISTS rspamd_emails (
+ Date Date,
+ Digest FixedString(32),
+ Emails Array(String)
+) ENGINE = MergeTree(Date, Digest, 8192);
+
+
+CREATE TABLE IF NOT EXISTS rspamd_asn (
+ Date Date,
+ Digest FixedString(32),
+ ASN String,
+ Country FixedString(2),
+ IPNet String
+) ENGINE = MergeTree(Date, Digest, 8192);
+
+
+CREATE TABLE IF NOT EXISTS rspamd_symbols (
+ Date Date,
+ Digest FixedString(32),
+ `Symbols.Names` Array(String),
+ `Symbols.Scores` Array(Float64),
+ `Symbols.Options` Array(String)
+) ENGINE = MergeTree(Date, Digest, 8192);
+
+
Log @{result}[1]
Check Pidfile
- [Arguments] ${pidfile}
- Wait Until Created ${pidfile}
+ [Arguments] ${pidfile} ${timeout}=1 min
+ Wait Until Created ${pidfile} timeout=${timeout}
${size} = Get File Size ${pidfile}
Should Not Be Equal As Integers ${size} 0
[Arguments] @{ports}
Run Keyword If '${CONTROLLER_ERRORS}' == 'True' Check Controller Errors
Shutdown Process With Children ${RSPAMD_PID}
- Save Run Results ${TMPDIR} rspamd.log redis.log rspamd.conf
+ Save Run Results ${TMPDIR} rspamd.log redis.log rspamd.conf clickhouse-server.log clickhouse-server.err.log clickhouse-config.xml
Cleanup Temporary Directory ${TMPDIR}
Log Logs
projects / rspamd.git / commitdiff