[Rework] Breaking: Actualize elastic module, support Elastic 8 & OpenSearch 2, add index policy with logs retention and many more

Signed-off-by: Dmytro Alieksieiev <1865999+dragoangel@users.noreply.github.com>

1 files changed, 90 insertions, 16 deletions

diff --git a/conf/modules.d/elastic.conf b/conf/modules.d/elastic.conf
index 80560237a..6255d528f 100644
--- a/conf/modules.d/elastic.conf
+++ b/conf/modules.d/elastic.conf
@@ -13,22 +13,96 @@
 # Module documentation can be found at  https://rspamd.com/doc/modules/elastic.html
 
 elastic {
-  # Push update when 10 records are collected (10 if unset)
-  limit = 10;
-  # IP:port of Elasticsearch server
-  #server = "localhost:9200";
-  # Timeout to wait for response (5 seconds if unset)
-  timeout = 5;
-  # Elasticsearch template file (json format)
-  #template_file = "${SHAREDIR}/elastic/rspamd_template.json";
-  # Kibana prebuild visualizations and dashboard template (json format)
-  #kibana_file = "${SHAREDIR}/elastic/kibana.json";
-  # Elasticsearch index name pattern
-  index_pattern = "rspamd-%Y.%m.%d";
-  # Dump debug information
-  debug = false;
-  # Import kibana template
-  import_kibana = false;
+  enabled = false;
+  # server = "localhost:9200";
+  # user = "";
+  # password = "";
+  use_https = false;
+  periodic_interval = 5.0;
+  timeout = 5.0;
+  no_ssl_verify = false;
+  use_gzip = true;
+  use_keepalive = true;
+  version = {
+    autodetect_enabled = true;
+    autodetect_max_fail = 12;
+    # override works only if autodetect is disabled
+    override = {
+      name = "opensearch";
+      version = "2.17";
+    }
+  };
+  limits = {
+    max_rows = 500; # max logs in one bulk req to elastic and first reason to flush buffer to elastic
+    max_interval = 60; # seconds, if first log in buffer older then interval - flush buffer
+    max_size = 5000000; # max symbols count in buffer, if reached - flush buffer, f.e: 5000000 ~= 10MB/normal-worker
+    max_fail = 3;
+  };
+  index_template = {
+    managed = true;
+    name = "rspamd";
+    priority = 0;
+    pattern = "%Y.%m.%d";
+    shards_count = 3;
+    replicas_count = 1;
+    refresh_interval = 5; # seconds
+    dynamic_keyword_ignore_above = 256;
+    headers_text_ignore_above = 2048; # strip headers value and add "..." to the end; set 0 to disable limit
+    symbols_nested = false;
+    empty_value = "unknown"; # empty numbers, ips and ipnets are not customizable they will be always 0, :: and ::/128 respectively
+  };
+  index_policy = {
+    enabled = true;
+    managed = true;
+    name = "rspamd"; # if you want use custom lifecycle policy, change name and set managed = false
+    hot = {
+      index_priority = 100;
+    };
+    warm = {
+      enabled = true;
+      after = "2d";
+      index_priority = 50;
+      migrate = true; # only supported with elastic distro, will not have impact elsewhere
+      read_only = true;
+      change_replicas = false;
+      replicas_count = 1;
+      shrink = false;
+      shards_count = 1;
+      max_gb_per_shard = 0; # zero - disabled by default, if enabled - shards_count is ignored
+      force_merge = true;
+      segments_count = 1;
+    };
+    cold = {
+      enabled = true;
+      after = "14d";
+      index_priority = 0;
+      migrate = true; # only supported with elastic distro, will not have impact elsewhere
+      read_only = true;
+      change_replicas = false;
+      replicas_count = 1;
+    };
+    delete = {
+      enabled = true;
+      after = "30d";
+    };
+  };
+  collect_headers = [
+    "From";
+    "To";
+    "Subject";
+    "Date";
+    "User-Agent";
+  ];
+  # extra headers to collect, f.e.:
+  # "Precedence";
+  # "List-Id";
+  extra_collect_headers = [];
+  geoip = {
+    enabled = true;
+    managed = true;
+    pipeline_name = "rspamd-geoip";
+  };
+
   .include(try=true,priority=5) "${DBDIR}/dynamic/elastic.conf"
   .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/elastic.conf"
   .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/elastic.conf"