aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--ui/ui.droppable.js6
-rw-r--r--ui/ui.sortable.js202
2 files changed, 89 insertions, 119 deletions
diff --git a/ui/ui.droppable.js b/ui/ui.droppable.js
index fc10476ad..4131a918b 100644
--- a/ui/ui.droppable.js
+++ b/ui/ui.droppable.js
@@ -127,9 +127,7 @@ $.widget("ui.droppable", {
draggable: (c.currentItem || c.element),
helper: c.helper,
position: c.position,
- absolutePosition: c.positionAbs,
- options: this.options,
- element: this.element
+ absolutePosition: c.positionAbs
};
}
@@ -214,7 +212,7 @@ $.ui.ddmanager = {
m[i].offset = m[i].element.offset();
m[i].proportions = { width: m[i].element[0].offsetWidth, height: m[i].element[0].offsetHeight };
- if(type == "dragstart" || type == "sortactivate") m[i]._activate.call(m[i], event); //Activate the droppable if used directly from draggables
+ if(type == "mousedown") m[i]._activate.call(m[i], event); //Activate the droppable if used directly from draggables
}
diff --git a/ui/ui.sortable.js b/ui/ui.sortable.js
index 77c2678e2..5e7ffa1ca 100644
--- a/ui/ui.sortable.js
+++ b/ui/ui.sortable.js
@@ -80,7 +80,7 @@ $.widget("ui.sortable", $.extend({}, $.ui.mouse, {
_mouseStart: function(event, overrideHandle, noActivation) {
- var o = this.options;
+ var o = this.options, self = this;
this.currentContainer = this;
//We only need to call refreshPositions, because the refreshItems call has been moved to mouseCapture
@@ -148,8 +148,27 @@ $.widget("ui.sortable", $.extend({}, $.ui.mouse, {
if(o.containment)
this._setContainment();
- //Call plugins and callbacks
- this._trigger("start", event);
+ if(o.cursor) { // cursor option
+ if ($('body').css("cursor")) this._storedCursor = $('body').css("cursor");
+ $('body').css("cursor", o.cursor);
+ }
+
+ if(o.opacity) { // opacity option
+ if (this.helper.css("opacity")) this._storedOpacity = this.helper.css("opacity");
+ this.helper.css("opacity", o.opacity);
+ }
+
+ if(o.zIndex) { // zIndex option
+ if (this.helper.css("zIndex")) this._storedZIndex = this.helper.css("zIndex");
+ this.helper.css("zIndex", o.zIndex);
+ }
+
+ //Prepare scrolling
+ if(this.scrollParent[0] != document && this.scrollParent[0].tagName != 'HTML')
+ this.overflowOffset = this.scrollParent.offset();
+
+ //Call callbacks
+ this._trigger("start", event, this._uiHash());
//Recache the helper size
if(!this._preserveHelperProportions)
@@ -158,7 +177,7 @@ $.widget("ui.sortable", $.extend({}, $.ui.mouse, {
//Post 'activate' events to possible containers
if(!noActivation) {
- for (var i = this.containers.length - 1; i >= 0; i--) { this.containers[i]._trigger("activate", event, this); }
+ for (var i = this.containers.length - 1; i >= 0; i--) { this.containers[i]._trigger("activate", event, self._uiHash(this)); }
}
//Prepare possible droppables
@@ -186,8 +205,38 @@ $.widget("ui.sortable", $.extend({}, $.ui.mouse, {
this.lastPositionAbs = this.positionAbs;
}
- //Call the internal plugins
- $.ui.plugin.call(this, "sort", [event, this._uiHash()]);
+ //Do scrolling
+ if(this.options.scroll) {
+ var o = this.options, scrolled = false;
+ if(this.scrollParent[0] != document && this.scrollParent[0].tagName != 'HTML') {
+
+ if((this.overflowOffset.top + this.scrollParent[0].offsetHeight) - event.pageY < o.scrollSensitivity)
+ this.scrollParent[0].scrollTop = scrolled = this.scrollParent[0].scrollTop + o.scrollSpeed;
+ else if(event.pageY - this.overflowOffset.top < o.scrollSensitivity)
+ this.scrollParent[0].scrollTop = scrolled = this.scrollParent[0].scrollTop - o.scrollSpeed;
+
+ if((this.overflowOffset.left + this.scrollParent[0].offsetWidth) - event.pageX < o.scrollSensitivity)
+ this.scrollParent[0].scrollLeft = scrolled = this.scrollParent[0].scrollLeft + o.scrollSpeed;
+ else if(event.pageX - this.overflowOffset.left < o.scrollSensitivity)
+ this.scrollParent[0].scrollLeft = scrolled = this.scrollParent[0].scrollLeft - o.scrollSpeed;
+
+ } else {
+
+ if(event.pageY - $(document).scrollTop() < o.scrollSensitivity)
+ scrolled = $(document).scrollTop($(document).scrollTop() - o.scrollSpeed);
+ else if($(window).height() - (event.pageY - $(document).scrollTop()) < o.scrollSensitivity)
+ scrolled = $(document).scrollTop($(document).scrollTop() + o.scrollSpeed);
+
+ if(event.pageX - $(document).scrollLeft() < o.scrollSensitivity)
+ scrolled = $(document).scrollLeft($(document).scrollLeft() - o.scrollSpeed);
+ else if($(window).width() - (event.pageX - $(document).scrollLeft()) < o.scrollSensitivity)
+ scrolled = $(document).scrollLeft($(document).scrollLeft() + o.scrollSpeed);
+
+ }
+
+ if(scrolled !== false && $.ui.ddmanager && !o.dropBehaviour)
+ $.ui.ddmanager.prepareOffsets(this, event);
+ }
//Regenerate the absolute position used for position checks
this.positionAbs = this._convertPositionTo("absolute");
@@ -217,7 +266,7 @@ $.widget("ui.sortable", $.extend({}, $.ui.mouse, {
break;
}
- this._trigger("change", event); //Call plugins and callbacks
+ this._trigger("change", event, this._uiHash());
break;
}
}
@@ -229,7 +278,7 @@ $.widget("ui.sortable", $.extend({}, $.ui.mouse, {
if($.ui.ddmanager) $.ui.ddmanager.drag(this, event);
//Call callbacks
- this._trigger('sort', event);
+ this._trigger('sort', event, this._uiHash());
this.lastPositionAbs = this.positionAbs;
return false;
@@ -266,6 +315,8 @@ $.widget("ui.sortable", $.extend({}, $.ui.mouse, {
cancel: function() {
+ var self = this;
+
if(this.dragging) {
this._mouseUp();
@@ -277,9 +328,9 @@ $.widget("ui.sortable", $.extend({}, $.ui.mouse, {
//Post deactivating events to containers
for (var i = this.containers.length - 1; i >= 0; i--){
- this.containers[i]._trigger("deactivate", null, this);
+ this.containers[i]._trigger("deactivate", null, self._uiHash(this));
if(this.containers[i].containerCache.over) {
- this.containers[i]._trigger("out", null, this);
+ this.containers[i]._trigger("out", null, self._uiHash(this));
this.containers[i].containerCache.over = 0;
}
}
@@ -608,20 +659,20 @@ $.widget("ui.sortable", $.extend({}, $.ui.mouse, {
this.currentContainer = this.containers[i];
itemWithLeastDistance ? this.options.sortIndicator.call(this, event, itemWithLeastDistance, null, true) : this.options.sortIndicator.call(this, event, null, this.containers[i].element, true);
- this._trigger("change", event); //Call plugins and callbacks
- this.containers[i]._trigger("change", event, this); //Call plugins and callbacks
+ this._trigger("change", event, this._uiHash());
+ this.containers[i]._trigger("change", event, this._uiHash(this));
//Update the placeholder
this.options.placeholder.update(this.currentContainer, this.placeholder);
}
- this.containers[i]._trigger("over", event, this);
+ this.containers[i]._trigger("over", event, this._uiHash(this));
this.containers[i].containerCache.over = 1;
}
} else {
if(this.containers[i].containerCache.over) {
- this.containers[i]._trigger("out", event, this);
+ this.containers[i]._trigger("out", event, this._uiHash(this));
this.containers[i].containerCache.over = 0;
}
}
@@ -851,58 +902,60 @@ $.widget("ui.sortable", $.extend({}, $.ui.mouse, {
this.currentItem.show();
}
- if(this.fromOutside) this._trigger("receive", event, this, noPropagation);
- if(this.fromOutside || this.domPosition.prev != this.currentItem.prev().not("."+this.options.cssNamespace+"-sortable-helper")[0] || this.domPosition.parent != this.currentItem.parent()[0]) this._trigger("update", event, null, noPropagation); //Trigger update callback if the DOM position has changed
+ if(this.fromOutside && !noPropagation) this._trigger("receive", event, this._uiHash(this));
+ if((this.fromOutside || this.domPosition.prev != this.currentItem.prev().not("."+this.options.cssNamespace+"-sortable-helper")[0] || this.domPosition.parent != this.currentItem.parent()[0]) && !noPropagation) this._trigger("update", event, this._uiHash()); //Trigger update callback if the DOM position has changed
if(!$.ui.contains(this.element[0], this.currentItem[0])) { //Node was moved out of the current element
- this._trigger("remove", event, null, noPropagation);
+ if(!noPropagation) this._trigger("remove", event, this._uiHash());
for (var i = this.containers.length - 1; i >= 0; i--){
- if($.ui.contains(this.containers[i].element[0], this.currentItem[0])) {
- this.containers[i]._trigger("receive", event, this, noPropagation);
- this.containers[i]._trigger("update", event, this, noPropagation);
+ if($.ui.contains(this.containers[i].element[0], this.currentItem[0]) && !noPropagation) {
+ this.containers[i]._trigger("receive", event, this._uiHash(this));
+ this.containers[i]._trigger("update", event, this._uiHash(this));
}
};
};
//Post events to containers
for (var i = this.containers.length - 1; i >= 0; i--){
- this.containers[i]._trigger("deactivate", event, this, noPropagation);
+ if(!noPropagation) this.containers[i]._trigger("deactivate", event, this._uiHash(this));
if(this.containers[i].containerCache.over) {
- this.containers[i]._trigger("out", event, this);
+ this.containers[i]._trigger("out", event, this._uiHash(this));
this.containers[i].containerCache.over = 0;
}
}
+ //Do what was originally in plugins
+ if(this._storedCursor) $('body').css("cursor", this._storedCursor); //Reset cursor
+ if(this._storedOpacity) this.helper.css("opacity", this._storedCursor); //Reset cursor
+ if(this._storedZIndex) this.helper.css("zIndex", this._storedZIndex == 'auto' ? '' : this._storedZIndex); //Reset z-index
+
this.dragging = false;
if(this.cancelHelperRemoval) {
- this._trigger("beforeStop", event, null, noPropagation);
- this._trigger("stop", event, null, noPropagation);
+ if(!noPropagation) {
+ this._trigger("beforeStop", event, this._uiHash());
+ this._trigger("stop", event, this._uiHash());
+ }
return false;
}
- this._trigger("beforeStop", event, null, noPropagation);
+ if(!noPropagation) this._trigger("beforeStop", event, this._uiHash());
//$(this.placeholder[0]).remove(); would have been the jQuery way - unfortunately, it unbinds ALL events from the original node!
this.placeholder[0].parentNode.removeChild(this.placeholder[0]);
if(this.helper[0] != this.currentItem[0]) this.helper.remove(); this.helper = null;
- this._trigger("stop", event, null, noPropagation);
+ if(!noPropagation) this._trigger("stop", event, this._uiHash());
this.fromOutside = false;
return true;
},
- _trigger: function(type, event, inst, noPropagation) {
- $.ui.plugin.call(this, type, [event, this._uiHash(inst)]);
- if(!noPropagation) {
- if ($.widget.prototype._trigger.call(this, type, event, this._uiHash(inst)) === false) {
- this.cancel();
- }
+ _trigger: function() {
+ if ($.widget.prototype._trigger.apply(this, arguments) === false) {
+ this.cancel();
}
},
- plugins: {},
-
_uiHash: function(inst) {
var self = inst || this;
return {
@@ -942,85 +995,4 @@ $.extend($.ui.sortable, {
}
});
-/*
- * Sortable Extensions
- */
-
-$.ui.plugin.add("sortable", "cursor", {
- start: function(event, ui) {
- var t = $('body'), i = $(this).data('sortable');
- if (t.css("cursor")) i.options._cursor = t.css("cursor");
- t.css("cursor", i.options.cursor);
- },
- beforeStop: function(event, ui) {
- var i = $(this).data('sortable');
- if (i.options._cursor) $('body').css("cursor", i.options._cursor);
- }
-});
-
-$.ui.plugin.add("sortable", "opacity", {
- start: function(event, ui) {
- var t = ui.helper, i = $(this).data('sortable');
- if(t.css("opacity")) i.options._opacity = t.css("opacity");
- t.css('opacity', i.options.opacity);
- },
- beforeStop: function(event, ui) {
- var i = $(this).data('sortable');
- if(i.options._opacity) $(ui.helper).css('opacity', i.options._opacity);
- }
-});
-
-$.ui.plugin.add("sortable", "scroll", {
- start: function(event, ui) {
- var i = $(this).data("sortable"), o = i.options;
- if(i.scrollParent[0] != document && i.scrollParent[0].tagName != 'HTML') i.overflowOffset = i.scrollParent.offset();
- },
- sort: function(event, ui) {
-
- var i = $(this).data("sortable"), o = i.options, scrolled = false;
-
- if(i.scrollParent[0] != document && i.scrollParent[0].tagName != 'HTML') {
-
- if((i.overflowOffset.top + i.scrollParent[0].offsetHeight) - event.pageY < o.scrollSensitivity)
- i.scrollParent[0].scrollTop = scrolled = i.scrollParent[0].scrollTop + o.scrollSpeed;
- else if(event.pageY - i.overflowOffset.top < o.scrollSensitivity)
- i.scrollParent[0].scrollTop = scrolled = i.scrollParent[0].scrollTop - o.scrollSpeed;
-
- if((i.overflowOffset.left + i.scrollParent[0].offsetWidth) - event.pageX < o.scrollSensitivity)
- i.scrollParent[0].scrollLeft = scrolled = i.scrollParent[0].scrollLeft + o.scrollSpeed;
- else if(event.pageX - i.overflowOffset.left < o.scrollSensitivity)
- i.scrollParent[0].scrollLeft = scrolled = i.scrollParent[0].scrollLeft - o.scrollSpeed;
-
- } else {
-
- if(event.pageY - $(document).scrollTop() < o.scrollSensitivity)
- scrolled = $(document).scrollTop($(document).scrollTop() - o.scrollSpeed);
- else if($(window).height() - (event.pageY - $(document).scrollTop()) < o.scrollSensitivity)
- scrolled = $(document).scrollTop($(document).scrollTop() + o.scrollSpeed);
-
- if(event.pageX - $(document).scrollLeft() < o.scrollSensitivity)
- scrolled = $(document).scrollLeft($(document).scrollLeft() - o.scrollSpeed);
- else if($(window).width() - (event.pageX - $(document).scrollLeft()) < o.scrollSensitivity)
- scrolled = $(document).scrollLeft($(document).scrollLeft() + o.scrollSpeed);
-
- }
-
- if(scrolled !== false && $.ui.ddmanager && !o.dropBehaviour)
- $.ui.ddmanager.prepareOffsets(i, event);
-
- }
-});
-
-$.ui.plugin.add("sortable", "zIndex", {
- start: function(event, ui) {
- var t = ui.helper, i = $(this).data('sortable');
- if(t.css("zIndex")) i.options._zIndex = t.css("zIndex");
- t.css('zIndex', i.options.zIndex);
- },
- beforeStop: function(event, ui) {
- var i = $(this).data('sortable');
- if(i.options._zIndex) $(ui.helper).css('zIndex', i.options._zIndex == 'auto' ? '' : i.options._zIndex);
- }
-});
-
})(jQuery);
641' href='#n641'>641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334
--[[
Copyright (c) 2017-2018, Vsevolod Stakhov <vsevolod@highsecure.ru>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
]]--

if confighelp then
  return
end

-- A generic plugin for reputation handling

local E = {}
local N = 'reputation'

local rspamd_logger = require "rspamd_logger"
local rspamd_util = require "rspamd_util"
local lua_util = require "lua_util"
local lua_maps = require "lua_maps"
local lua_maps_exprs = require "lua_maps_expressions"
local hash = require 'rspamd_cryptobox_hash'
local lua_redis = require "lua_redis"
local fun = require "fun"
local lua_selectors = require "lua_selectors"
local ts = require("tableshape").types

local redis_params = nil
local default_expiry = 864000 -- 10 day by default
local default_prefix = 'RR:' -- Rspamd Reputation

local tanh = math.tanh or rspamd_util.tanh

local reject_threshold = rspamd_config:get_action('reject') or 10.0

-- Get reputation from ham/spam/probable hits
local function generic_reputation_calc(token, rule, mult, task)
  local cfg = rule.selector.config or E

  if cfg.score_calc_func then
    return cfg.score_calc_func(rule, token, mult)
  end

  if tonumber(token[1]) < cfg.lower_bound then
    lua_util.debugm(N, task, "not enough matches %s < %s for rule %s",
        token[1], cfg.lower_bound, rule.symbol)
    return 0
  end

  -- Get average score
  local avg_score = fun.foldl(function(acc, v)
    return acc + v
  end, 0.0, fun.map(tonumber, token[2])) / #token[2]

  -- Apply function tanh(x / reject_score * atanh(0.95) - atanh(0.5))
  --                                        1.83178       0.5493
  local score = tanh(avg_score / reject_threshold * 1.83178 - 0.5493) * mult
  lua_util.debugm(N, task, "got generic average score %s -> %s for rule %s",
      avg_score, score, rule.symbol)
  return score
end

local function add_symbol_score(task, rule, mult, params)
  if not params then params = {tostring(mult)};

  end
  if rule.selector.config.split_symbols then
    if mult >= 0 then
      task:insert_result(rule.symbol .. '_SPAM', mult, params)
    else
      task:insert_result(rule.symbol .. '_HAM', mult, params)
    end
  else
    task:insert_result(rule.symbol, mult, params)
  end
end

local function sub_symbol_score(task, rule, score)
  local function sym_score(sym)
    local s = task:get_symbol(sym)[1]
    return s.score
  end
  if rule.selector.config.split_symbols then
    local spam_sym = rule.symbol .. '_SPAM'
    local ham_sym = rule.symbol .. '_HAM'

    if task:has_symbol(spam_sym) then
      score = score - sym_score(spam_sym)
    elseif task:has_symbol(ham_sym) then
      score = score - sym_score(ham_sym)
    end
  else
    if task:has_symbol(rule.symbol) then
      score = score - sym_score(rule.symbol)
    end
  end

  return score
end

-- Extracts task score and subtracts score of the rule itself
local function extract_task_score(task, rule)
  local lua_verdict = require "lua_verdict"
  local verdict,score = lua_verdict.get_specific_verdict(N, task)

  if not score or verdict == 'passthrough' then return nil end

  return sub_symbol_score(task, rule, score)
end

-- DKIM Selector functions
local gr
local function gen_dkim_queries(task, rule)
  local dkim_trace = (task:get_symbol('DKIM_TRACE') or E)[1]
  local lpeg = require 'lpeg'
  local ret = {}

  if not gr then
    local semicolon = lpeg.P(':')
    local domain = lpeg.C((1 - semicolon)^1)
    local res = lpeg.S'+-?~'

    local function res_to_label(ch)
      if ch == '+' then return 'a'
      elseif ch == '-' then return 'r'
      end

      return 'u'
    end

    gr = domain * semicolon * (lpeg.C(res^1) / res_to_label)
  end

  if dkim_trace and dkim_trace.options then
    for _,opt in ipairs(dkim_trace.options) do
      local dom,res = lpeg.match(gr, opt)

      if dom and res then
        local tld = rspamd_util.get_tld(dom)
        ret[tld] = res
      end
    end
  end

  return ret
end

local function dkim_reputation_filter(task, rule)
  local requests = gen_dkim_queries(task, rule)
  local results = {}
  local dkim_tlds = lua_util.keys(requests)
  local requests_left = #dkim_tlds
  local rep_accepted = 0.0
  local rep_rejected = 0.0

  lua_util.debugm(N, task, 'dkim reputation tokens: %s', requests)

  local function tokens_cb(err, token, values)
    requests_left = requests_left - 1

    if values then
      results[token] = values
    end

    if requests_left == 0 then
      for k,v in pairs(results) do
        -- `k` in results is a prefixed and suffixed tld, so we need to look through
        -- all requests to find any request with the matching tld
        local sel_tld
        for _,tld in ipairs(dkim_tlds) do
          if k:find(tld) then
            sel_tld = tld
            break
          end
        end

        if sel_tld and requests[sel_tld] then
          if requests[sel_tld] == 'a' then
            rep_accepted = rep_accepted + generic_reputation_calc(v, rule, 1.0, task)
          elseif requests[sel_tld] == 'r' then
            rep_rejected = rep_rejected + generic_reputation_calc(v, rule, 1.0, task)
          end
        else
          rspamd_logger.warnx(task, "cannot find the requested tld for a request: %s (%s tlds noticed)",
              k, dkim_tlds)
        end
      end

      -- Set local reputation symbol
      if rep_accepted > 0 or rep_rejected > 0 then
        if rep_accepted > rep_rejected then
          add_symbol_score(task, rule, -(rep_accepted - rep_rejected))
        else
          add_symbol_score(task, rule, (rep_rejected - rep_accepted))
        end

        -- Store results for future DKIM results adjustments
        task:get_mempool():set_variable("dkim_reputation_accept", tostring(rep_accepted))
        task:get_mempool():set_variable("dkim_reputation_reject", tostring(rep_rejected))
      end
    end
  end

  for dom,res in pairs(requests) do
    -- tld + "." + check_result, e.g. example.com.+ - reputation for valid sigs
    local query = string.format('%s.%s', dom, res)
    rule.backend.get_token(task, rule, nil, query, tokens_cb, 'string')
  end
end

local function dkim_reputation_idempotent(task, rule)
  local requests = gen_dkim_queries(task, rule)
  local sc = extract_task_score(task, rule)

  if sc then
    for dom,res in pairs(requests) do
      -- tld + "." + check_result, e.g. example.com.+ - reputation for valid sigs
      local query = string.format('%s.%s', dom, res)
      rule.backend.set_token(task, rule, nil, query, sc)
    end
  end
end

local function dkim_reputation_postfilter(task, rule)
  local sym_accepted = task:get_symbol('R_DKIM_ALLOW')
  local accept_adjustment = task:get_mempool():get_variable("dkim_reputation_accept")

  if sym_accepted and accept_adjustment then
    local final_adjustment = rule.config.max_accept_adjustment *
        rspamd_util.tanh(tonumber(accept_adjustment))
    task:adjust_result('R_DKIM_ALLOW', sym_accepted.score * final_adjustment)
  end

  local sym_rejected = task:get_symbol('R_DKIM_REJECT')
  local reject_adjustment = task:get_mempool():get_variable("dkim_reputation_reject")

  if sym_rejected and reject_adjustment then
    local final_adjustment = rule.config.max_reject_adjustment *
        rspamd_util.tanh(tonumber(reject_adjustment))
    task:adjust_result('R_DKIM_REJECT', sym_rejected.score * final_adjustment)
  end
end

local dkim_selector = {
  config = {
    symbol = 'DKIM_SCORE', -- symbol to be inserted
    lower_bound = 10, -- minimum number of messages to be scored
    min_score = nil,
    max_score = nil,
    outbound = true,
    inbound = true,
    max_accept_adjustment = 2.0, -- How to adjust accepted DKIM score
    max_reject_adjustment = 3.0 -- How to adjust rejected DKIM score
  },
  dependencies = {"DKIM_TRACE"},
  filter = dkim_reputation_filter, -- used to get scores
  postfilter = dkim_reputation_postfilter, -- used to adjust DKIM scores
  idempotent = dkim_reputation_idempotent, -- used to set scores
}

-- URL Selector functions

local function gen_url_queries(task, rule)
  local domains = {}

  fun.each(function(u)
    if u:is_redirected() then
      local redir = u:get_redirected() -- get the original url
      local redir_tld = redir:get_tld()
      if domains[redir_tld] then
        domains[redir_tld] = domains[redir_tld] - 1
      end
    end
    local dom = u:get_tld()
    if not domains[dom] then
      domains[dom] = 1
    else
      domains[dom] = domains[dom] + 1
    end
  end, fun.filter(function(u) return not u:is_html_displayed() end,
    task:get_urls(true)))

  local results = {}
  for k,v in lua_util.spairs(domains,
    function(t, a, b) return t[a] > t[b] end, rule.selector.config.max_urls) do
    if v > 0 then
      table.insert(results, {k,v})
    end
  end

  return results
end

local function url_reputation_filter(task, rule)
  local requests = gen_url_queries(task, rule)
  local results = {}
  local nchecked = 0

  local function indexed_tokens_cb(err, index, values)
    nchecked = nchecked + 1

    if values then
      results[index] = values
    end

    if nchecked == #requests then
      -- Check the url with maximum hits
      local mhits = 0
      for k,_ in pairs(results) do
        if requests[k][2] > mhits then
          mhits = requests[k][2]
        end
      end

      if mhits > 0 then
        local score = 0
        for k,v in pairs(results) do
          score = score + generic_reputation_calc(v, rule,
              requests[k][2] / mhits, task)
        end

        if math.abs(score) > 1e-3 then
          -- TODO: add description
          add_symbol_score(task, rule, score)
        end
      end
    end
  end

  for i,req in ipairs(requests) do
    local function tokens_cb(err, token, values)
      indexed_tokens_cb(err, i, values)
    end

    rule.backend.get_token(task, rule, nil, req[1], tokens_cb, 'string')
  end
end

local function url_reputation_idempotent(task, rule)
  local requests = gen_url_queries(task, rule)
  local sc = extract_task_score(task, rule)

  if sc then
    for _,tld in ipairs(requests) do
      rule.backend.set_token(task, rule, nil, tld[1], sc)
    end
  end
end

local url_selector = {
  config = {
    symbol = 'URL_SCORE', -- symbol to be inserted
    lower_bound = 10, -- minimum number of messages to be scored
    min_score = nil,
    max_score = nil,
    max_urls = 10,
    check_from = true,
    outbound = true,
    inbound = true,
  },
  filter = url_reputation_filter, -- used to get scores
  idempotent = url_reputation_idempotent -- used to set scores
}
-- IP Selector functions

local function ip_reputation_init(rule)
  local cfg = rule.selector.config

  if cfg.asn_cc_whitelist then
    cfg.asn_cc_whitelist = rspamd_map_add('reputation',
      'asn_cc_whitelist',
      'map',
      'IP score whitelisted ASNs/countries')
  end

  return true
end

local function ip_reputation_filter(task, rule)

  local ip = task:get_from_ip()

  if not ip or not ip:is_valid() then return end
  if lua_util.is_rspamc_or_controller(task) then return end

  local cfg = rule.selector.config

  if ip:get_version() == 4 and cfg.ipv4_mask then
    ip = ip:apply_mask(cfg.ipv4_mask)
  elseif cfg.ipv6_mask then
    ip = ip:apply_mask(cfg.ipv6_mask)
  end

  local pool = task:get_mempool()
  local asn = pool:get_variable("asn")
  local country = pool:get_variable("country")

  if country and cfg.asn_cc_whitelist then
    if cfg.asn_cc_whitelist:get_key(country) then
      return
    end
    if asn and cfg.asn_cc_whitelist:get_key(asn) then
      return
    end
  end

  -- These variables are used to define if we have some specific token
  local has_asn = not asn
  local has_country = not country
  local has_ip = false

  local asn_stats, country_stats, ip_stats

  local function ipstats_check()
    local score = 0.0
    local description_t = {}

    if asn_stats then
      local asn_score = generic_reputation_calc(asn_stats, rule, cfg.scores.asn, task)
      score = score + asn_score
      table.insert(description_t, string.format('asn: %s(%.2f)',
              asn, asn_score))
    end
    if country_stats then
      local country_score = generic_reputation_calc(country_stats, rule,
          cfg.scores.country, task)
      score = score + country_score
      table.insert(description_t, string.format('country: %s(%.2f)',
              country, country_score))
    end
    if ip_stats then
      local ip_score = generic_reputation_calc(ip_stats, rule, cfg.scores.ip,
        task)
      score = score + ip_score
      table.insert(description_t, string.format('ip: %s(%.2f)',
              tostring(ip), ip_score))
    end

    if math.abs(score) > 0.001 then
      add_symbol_score(task, rule, score, table.concat(description_t, ', '))
    end
  end

  local function gen_token_callback(what)
    return function(err, _, values)
      if not err and values then
        if what == 'asn' then
          has_asn = true
          asn_stats = values
        elseif what == 'country' then
          has_country = true
          country_stats = values
        elseif what == 'ip' then
          has_ip = true
          ip_stats = values
        end
      else
        if what == 'asn' then
          has_asn = true
        elseif what == 'country' then
          has_country = true
        elseif what == 'ip' then
          has_ip = true
        end
      end

      if has_asn and has_country and has_ip then
        -- Check reputation
        ipstats_check()
      end
    end
  end

  if asn then
    rule.backend.get_token(task, rule, cfg.asn_prefix, asn,
            gen_token_callback('asn'), 'string')
  end
  if country then
    rule.backend.get_token(task, rule, cfg.country_prefix, country,
            gen_token_callback('country'), 'string')
  end

  rule.backend.get_token(task, rule, cfg.ip_prefix, ip,
          gen_token_callback('ip'), 'ip')
end

-- Used to set scores
local function ip_reputation_idempotent(task, rule)
  if not rule.backend.set_token then return end -- Read only backend
  local ip = task:get_from_ip()
  local cfg = rule.selector.config

  if not ip or not ip:is_valid() then return end

  if lua_util.is_rspamc_or_controller(task) then return end

  if ip:get_version() == 4 and cfg.ipv4_mask then
    ip = ip:apply_mask(cfg.ipv4_mask)
  elseif cfg.ipv6_mask then
    ip = ip:apply_mask(cfg.ipv6_mask)
  end

  local pool = task:get_mempool()
  local asn = pool:get_variable("asn")
  local country = pool:get_variable("country")

  if country and cfg.asn_cc_whitelist then
    if cfg.asn_cc_whitelist:get_key(country) then
      return
    end
    if asn and cfg.asn_cc_whitelist:get_key(asn) then
      return
    end
  end
  local sc = extract_task_score(task, rule)
  if sc then
    if asn then
      rule.backend.set_token(task, rule, cfg.asn_prefix, asn, sc, nil, 'string')
    end
    if country then
      rule.backend.set_token(task, rule, cfg.country_prefix, country, sc, nil, 'string')
    end

    rule.backend.set_token(task, rule, cfg.ip_prefix, ip, sc, nil, 'ip')
  end
end

-- Selectors are used to extract reputation tokens
local ip_selector = {
  config = {
    scores = { -- how each component is evaluated
      ['asn'] = 0.4,
      ['country'] = 0.01,
      ['ip'] = 1.0
    },
    symbol = 'SENDER_REP', -- symbol to be inserted
    split_symbols = true,
    asn_prefix = 'a:', -- prefix for ASN hashes
    country_prefix = 'c:', -- prefix for country hashes
    ip_prefix = 'i:',
    lower_bound = 10, -- minimum number of messages to be scored
    min_score = nil,
    max_score = nil,
    score_divisor = 1,
    outbound = false,
    inbound = true,
    ipv4_mask = 32, -- Mask bits for ipv4
    ipv6_mask = 64, -- Mask bits for ipv6
  },
  --dependencies = {"ASN"}, -- ASN is a prefilter now...
  init = ip_reputation_init,
  filter = ip_reputation_filter, -- used to get scores
  idempotent = ip_reputation_idempotent, -- used to set scores
}

-- SPF Selector functions

local function spf_reputation_filter(task, rule)
  local spf_record = task:get_mempool():get_variable('spf_record')
  local spf_allow = task:has_symbol('R_SPF_ALLOW')

  -- Don't care about bad/missing spf
  if not spf_record or not spf_allow then return end

  local cr = require "rspamd_cryptobox_hash"
  local hkey = cr.create(spf_record):base32():sub(1, 32)

  lua_util.debugm(N, task, 'check spf record %s -> %s', spf_record, hkey)

  local function tokens_cb(err, token, values)
    if values then
      local score = generic_reputation_calc(values, rule, 1.0, task)

      if math.abs(score) > 1e-3 then
        -- TODO: add description
        add_symbol_score(task, rule, score)
      end
    end
  end

  rule.backend.get_token(task, rule, nil, hkey, tokens_cb, 'string')
end

local function spf_reputation_idempotent(task, rule)
  local sc = extract_task_score(task, rule)
  local spf_record = task:get_mempool():get_variable('spf_record')
  local spf_allow = task:has_symbol('R_SPF_ALLOW')

  if not spf_record or not spf_allow or not sc then return end

  local cr = require "rspamd_cryptobox_hash"
  local hkey = cr.create(spf_record):base32():sub(1, 32)

  lua_util.debugm(N, task, 'set spf record %s -> %s = %s',
      spf_record, hkey, sc)
  rule.backend.set_token(task, rule, nil, hkey, sc)
end


local spf_selector = {
  config = {
    symbol = 'SPF_REP', -- symbol to be inserted
    split_symbols = true,
    lower_bound = 10, -- minimum number of messages to be scored
    min_score = nil,
    max_score = nil,
    outbound = true,
    inbound = true,
    max_accept_adjustment = 2.0, -- How to adjust accepted DKIM score
    max_reject_adjustment = 3.0 -- How to adjust rejected DKIM score
  },
  dependencies = {"R_SPF_ALLOW"},
  filter = spf_reputation_filter, -- used to get scores
  idempotent = spf_reputation_idempotent, -- used to set scores
}

-- Generic selector based on lua_selectors framework

local function generic_reputation_init(rule)
  local cfg = rule.selector.config

  if not cfg.selector then
    rspamd_logger.errx(rspamd_config, 'cannot configure generic rule: no selector specified')
    return false
  end

  local selector = lua_selectors.create_selector_closure(rspamd_config,
      cfg.selector, cfg.delimiter)

  if not selector then
    rspamd_logger.errx(rspamd_config, 'cannot configure generic rule: bad selector: %s',
        cfg.selector)
    return false
  end

  cfg.selector = selector -- Replace with closure

  if cfg.whitelist then
    cfg.whitelist = lua_maps.map_add('reputation',
        'generic_whitelist',
        'map',
        'Whitelisted selectors')
  end

  return true
end

local function generic_reputation_filter(task, rule)
  local cfg = rule.selector.config
  local selector_res = cfg.selector(task)

  local function tokens_cb(err, token, values)
    if values then
      local score = generic_reputation_calc(values, rule, 1.0, task)

      if math.abs(score) > 1e-3 then
        -- TODO: add description
        add_symbol_score(task, rule, score)
      end
    end
  end

  if selector_res then
    if type(selector_res) == 'table' then
      fun.each(function(e)
        lua_util.debugm(N, task, 'check generic reputation (%s) %s',
          rule['symbol'], e)
        rule.backend.get_token(task, rule, nil, e, tokens_cb, 'string')
      end, selector_res)
    else
      lua_util.debugm(N, task, 'check generic reputation (%s) %s',
        rule['symbol'], selector_res)
      rule.backend.get_token(task, rule, nil, selector_res, tokens_cb, 'string')
    end
  end
end

local function generic_reputation_idempotent(task, rule)
  local sc = extract_task_score(task, rule)
  local cfg = rule.selector.config

  local selector_res = cfg.selector(task)
  if not selector_res then return end

  if sc then
    if type(selector_res) == 'table' then
      fun.each(function(e)
        lua_util.debugm(N, task, 'set generic selector (%s) %s = %s',
            rule['symbol'], e, sc)
        rule.backend.set_token(task, rule, nil, e, sc)
      end, selector_res)
    else
      lua_util.debugm(N, task, 'set generic selector (%s) %s = %s',
          rule['symbol'], selector_res, sc)
      rule.backend.set_token(task, rule, nil, selector_res, sc)
    end
  end
end


local generic_selector = {
  schema = ts.shape{
    lower_bound = ts.number + ts.string / tonumber,
    max_score = ts.number:is_optional(),
    min_score = ts.number:is_optional(),
    outbound = ts.boolean,
    inbound = ts.boolean,
    selector = ts.string,
    delimiter = ts.string,
    whitelist = ts.one_of(lua_maps.map_schema, lua_maps_exprs.schema):is_optional(),
  },
  config = {
    lower_bound = 10, -- minimum number of messages to be scored
    min_score = nil,
    max_score = nil,
    outbound = true,
    inbound = true,
    selector = nil,
    delimiter = ':',
    whitelist = nil
  },
  init = generic_reputation_init,
  filter = generic_reputation_filter, -- used to get scores
  idempotent = generic_reputation_idempotent -- used to set scores
}



local selectors = {
  ip = ip_selector,
  sender = ip_selector, -- Better name
  url = url_selector,
  dkim = dkim_selector,
  spf = spf_selector,
  generic = generic_selector
}

local function reputation_dns_init(rule, _, _, _)
  if not rule.backend.config.list then
    rspamd_logger.errx(rspamd_config, "rule %s with DNS backend has no `list` parameter defined",
      rule.symbol)
    return false
  end

  return true
end


local function gen_token_key(prefix, token, rule)
  if prefix then
    token = prefix .. token
  end
  local res = token
  if rule.backend.config.hashed then
    local hash_alg = rule.backend.config.hash_alg or "blake2"
    local encoding = "base32"

    if rule.backend.config.hash_encoding then
      encoding = rule.backend.config.hash_encoding
    end

    local h = hash.create_specific(hash_alg, res)
    if encoding == 'hex' then
      res = h:hex()
    elseif encoding == 'base64' then
      res = h:base64()
    else
      res = h:base32()
    end
  end

  if rule.backend.config.hashlen then
    res = string.sub(res, 1, rule.backend.config.hashlen)
  end

  if rule.backend.config.prefix then
    res = rule.backend.config.prefix .. res
  end

  return res
end

--[[
-- Generic interface for get and set tokens functions:
-- get_token(task, rule, prefix, token, continuation, token_type), where `continuation` is the following function:
--
-- function(err, token, values) ... end
-- `err`: string value for error (similar to redis or DNS callbacks)
-- `token`: string value of a token
-- `values`: table of key=number, parsed from backend. It is selector's duty
--  to deal with missing, invalid or other values
--
-- set_token(task, rule, token, values, continuation_cb)
-- This function takes values, encodes them using whatever suitable format
-- and calls for continuation:
--
-- function(err, token) ... end
-- `err`: string value for error (similar to redis or DNS callbacks)
-- `token`: string value of a token
--
-- example of tokens: {'s': 0, 'h': 0, 'p': 1}
--]]

local function reputation_dns_get_token(task, rule, prefix, token, continuation_cb, token_type)
  -- local r = task:get_resolver()
  -- In DNS we never ever use prefix as prefix, we use if as a suffix!
  if token_type == 'ip' then
    token = table.concat(token:inversed_str_octets(), '.')
  end

  local key = gen_token_key(nil, token, rule)
  local dns_name = key .. '.' .. rule.backend.config.list

  if prefix then
    dns_name = string.format('%s.%s.%s', key, prefix,
            rule.backend.config.list)
  else
    dns_name = string.format('%s.%s', key, rule.backend.config.list)
  end

  local function dns_cb(_, _, results, err)
    if err and (err ~= 'requested record is not found' and
        err ~= 'no records with this name') then
      rspamd_logger.warnx(task, 'error looking up %s: %s', dns_name, err)
    end

    lua_util.debugm(N, task, 'DNS RESPONSE: label=%1 results=%2 err=%3 list=%4',
        dns_name, results, err, rule.backend.config.list)

    -- Now split tokens to list of values
    if results and results[1]  then
      -- Format: num_messages;sc1;sc2...scn
      local dns_tokens = lua_util.rspamd_str_split(results[1], ";")
      -- Convert all to numbers excluding any possible non-numbers
      dns_tokens = fun.totable(fun.filter(function(e)
        return type(e) == 'number'
      end,
      fun.map(function(e)
        local n = tonumber(e)
        if n then return n end
        return "BAD"
      end, dns_tokens)))

      if #dns_tokens < 2 then
        rspamd_logger.warnx(task, 'cannot parse response for reputation token %s: %s',
                dns_name, results[1])
        continuation_cb(results, dns_name, nil)
      else
        local cnt = table.remove(dns_tokens, 1)
        continuation_cb(nil, dns_name, { cnt, dns_tokens })
      end
    else
      rspamd_logger.messagex(task, 'invalid response for reputation token %s: %s',
              dns_name, results[1])
      continuation_cb(results, dns_name, nil)
    end
  end

 task:get_resolver():resolve_a({
    task = task,
    name = dns_name,
    callback = dns_cb,
    forced = true,
  })
end

local function reputation_redis_init(rule, cfg, ev_base, worker)
  local our_redis_params = {}

  our_redis_params = lua_redis.try_load_redis_servers(rule.backend.config, rspamd_config,
      true)
  if not our_redis_params then
    our_redis_params = redis_params
  end
  if not our_redis_params then
    rspamd_logger.errx(rspamd_config, 'cannot init redis for reputation rule: %s',
        rule)
    return false
  end
  -- Init scripts for buckets
  -- Redis script to extract data from Redis buckets
  -- KEYS[1] - key to extract
  -- Value returned - table of scores as a strings vector + number of scores
  local redis_get_script_tpl = [[
  local cnt = redis.call('HGET', KEYS[1], 'n')
  local results = {}
  if cnt then
  {% for w in windows %}
  local sc = tonumber(redis.call('HGET', KEYS[1], 'v' .. '{= w.name =}'))
  table.insert(results, tostring(sc * {= w.mult =}))
  {% endfor %}
  else
  {% for w in windows %}
  table.insert(results, '0')
  {% endfor %}
  end

  return {cnt or 0, results}
  ]]

  local get_script = lua_util.jinja_template(redis_get_script_tpl,
      {windows = rule.backend.config.buckets})
  rspamd_logger.debugm(N, rspamd_config, 'added extraction script %s', get_script)
  rule.backend.script_get = lua_redis.add_redis_script(get_script, our_redis_params)

  -- Redis script to update Redis buckets
  -- KEYS[1] - key to update
  -- KEYS[2] - current time in milliseconds
  -- KEYS[3] - message score
  -- KEYS[4] - expire for a bucket
  -- Value returned - table of scores as a strings vector
  local redis_adaptive_emea_script_tpl = [[
  local last = redis.call('HGET', KEYS[1], 'l')
  local score = tonumber(KEYS[3])
  local now = tonumber(KEYS[2])
  local scores = {}

  if last then
    {% for w in windows %}
    local last_value = tonumber(redis.call('HGET', KEYS[1], 'v' .. '{= w.name =}'))
    local window = {= w.time =}
    -- Adjust alpha
    local time_diff = now - last
    if time_diff < 0 then
      time_diff = 0
    end
    local alpha = 1.0 - math.exp((-time_diff) / (1000 * window))
    local nscore = alpha * score + (1.0 - alpha) * last_value
    table.insert(scores, tostring(nscore * {= w.mult =}))
    {% endfor %}
  else
    {% for w in windows %}
    table.insert(scores, tostring(score * {= w.mult =}))
    {% endfor %}
  end

  local i = 1
  {% for w in windows %}
    redis.call('HSET', KEYS[1], 'v' .. '{= w.name =}', scores[i])
    i = i + 1
  {% endfor %}
  redis.call('HSET', KEYS[1], 'l', now)
  redis.call('HINCRBY', KEYS[1], 'n', 1)
  redis.call('EXPIRE', KEYS[1], tonumber(KEYS[4]))

  return scores
]]

  local set_script = lua_util.jinja_template(redis_adaptive_emea_script_tpl,
      {windows = rule.backend.config.buckets})
  rspamd_logger.debugm(N, rspamd_config, 'added emea update script %s', set_script)
  rule.backend.script_set = lua_redis.add_redis_script(set_script, our_redis_params)

  return true
end

local function reputation_redis_get_token(task, rule, prefix, token, continuation_cb, token_type)
  if token_type and token_type == 'ip' then
    token = tostring(token)
  end
  local key = gen_token_key(prefix, token, rule)

  local function redis_get_cb(err, data)
    if data then
      if type(data) == 'table' then
        lua_util.debugm(N, task, 'rule %s - got values for key %s -> %s',
            rule['symbol'], key, data)
        continuation_cb(nil, key, data)
      else
        rspamd_logger.errx(task, 'rule %s - invalid type while getting reputation keys %s: %s',
          rule['symbol'], key, type(data))
        continuation_cb("invalid type", key, nil)
      end

    elseif err then
      rspamd_logger.errx(task, 'rule %s - got error while getting reputation keys %s: %s',
        rule['symbol'], key, err)
      continuation_cb(err, key, nil)
    else
      rspamd_logger.errx(task, 'rule %s - got error while getting reputation keys %s: %s',
        rule['symbol'], key, "unknown error")
      continuation_cb("unknown error", key, nil)
    end
  end

  local ret = lua_redis.exec_redis_script(rule.backend.script_get,
      {task = task, is_write = false},
      redis_get_cb,
      {key})
  if not ret then
    rspamd_logger.errx(task, 'cannot make redis request to check results')
  end
end

local function reputation_redis_set_token(task, rule, prefix, token, sc, continuation_cb, token_type)
  if token_type and token_type == 'ip' then
    token = tostring(token)
  end
  local key = gen_token_key(prefix, token, rule)

  local function redis_set_cb(err, data)
    if err then
      rspamd_logger.errx(task, 'rule %s - got error while setting reputation keys %s: %s',
        rule['symbol'], key, err)
      if continuation_cb then
        continuation_cb(err, key)
      end
    else
      if continuation_cb then
        continuation_cb(nil, key)
      end
    end
  end

  lua_util.debugm(N, task, 'rule %s - set values for key %s -> %s',
      rule['symbol'], key, sc)
  local ret = lua_redis.exec_redis_script(rule.backend.script_set,
      {task = task, is_write = true},
      redis_set_cb,
      {key, tostring(os.time() * 1000),
       tostring(sc),
       tostring(rule.backend.config.expiry)})
  if not ret then
    rspamd_logger.errx(task, 'got error while connecting to redis')
  end
end

--[[ Backends are responsible for getting reputation tokens
  -- Common config options:
  -- `hashed`: if `true` then apply hash function to the key
  -- `hash_alg`: use specific hash type (`blake2` by default)
  -- `hash_len`: strip hash to this amount of bytes (no strip by default)
  -- `hash_encoding`: use specific hash encoding (base32 by default)
--]]
local backends = {
  redis = {
    schema = ts.shape({
      prefix = ts.string,
      expiry = ts.number + ts.string / lua_util.parse_time_interval,
      buckets = ts.array_of(ts.shape{
        time = ts.number + ts.string / lua_util.parse_time_interval,
        name = ts.string,
        mult = ts.number + ts.string / tonumber
      }),
    }, {extra_fields = lua_redis.config_schema}),
    config = {
      expiry = default_expiry,
      prefix = default_prefix,
      buckets = {
        {
          time = 60 * 60 * 24 * 30,
          name = '1m',
          mult = 1.0,
        }
      }, -- What buckets should be used, default 1h and 1month
    },
    init = reputation_redis_init,
    get_token = reputation_redis_get_token,
    set_token = reputation_redis_set_token,
  },
  dns = {
    schema = ts.shape{
      list = ts.string,
    },
    config = {
      -- list = rep.example.com
    },
    get_token = reputation_dns_get_token,
    -- No set token for DNS
    init = reputation_dns_init,
  }
}

local function is_rule_applicable(task, rule)
  local ip = task:get_from_ip()
  if not (rule.selector.config.outbound and rule.selector.config.inbound) then
    if rule.selector.config.outbound then
      if not (task:get_user() or (ip and ip:is_local())) then
        return false
      end
    elseif rule.selector.config.inbound then
      if task:get_user() or (ip and ip:is_local()) then
        return false
      end
    end
  end

  if rule.config.whitelist_map then
    if rule.config.whitelist_map:process(task) then
      return false
    end
  end

  return true
end

local function reputation_filter_cb(task, rule)
  if (is_rule_applicable(task, rule)) then
    rule.selector.filter(task, rule, rule.backend)
  end
end

local function reputation_postfilter_cb(task, rule)
  if (is_rule_applicable(task, rule)) then
    rule.selector.postfilter(task, rule, rule.backend)
  end
end

local function reputation_idempotent_cb(task, rule)
  if (is_rule_applicable(task, rule)) then
    rule.selector.idempotent(task, rule, rule.backend)
  end
end

local function callback_gen(cb, rule)
  return function(task)
    if rule.enabled then
      cb(task, rule)
    end
  end
end

local function parse_rule(name, tbl)
  local sel_type,sel_conf = fun.head(tbl.selector)
  local selector = selectors[sel_type]

  if not selector then
    rspamd_logger.errx(rspamd_config, "unknown selector defined for rule %s: %s", name,
        sel_type)
    return
  end

  local bk_type,bk_conf = fun.head(tbl.backend)

  local backend = backends[bk_type]
  if not backend then
    rspamd_logger.errx(rspamd_config, "unknown backend defined for rule %s: %s", name,
      tbl.backend.type)
    return
  end
  -- Allow config override
  local rule = {
    selector = lua_util.shallowcopy(selector),
    backend = lua_util.shallowcopy(backend),
    config = {}
  }

  -- Override default config params
  rule.backend.config = lua_util.override_defaults(rule.backend.config, bk_conf)
  if backend.schema then
    local checked,schema_err = backend.schema:transform(rule.backend.config)
    if not checked then
      rspamd_logger.errx(rspamd_config, "cannot parse backend config for %s: %s",
          sel_type, schema_err)

      return
    end

    rule.backend.config = checked
  end

  rule.selector.config = lua_util.override_defaults(rule.selector.config, sel_conf)
  if selector.schema then
    local checked,schema_err = selector.schema:transform(rule.selector.config)

    if not checked then
      rspamd_logger.errx(rspamd_config, "cannot parse selector config for %s: %s (%s)",
          sel_type,
          schema_err, sel_conf)
      return
    end

    rule.selector.config = checked
  end
  -- Generic options
  tbl.selector = nil
  tbl.backend = nil
  rule.config = lua_util.override_defaults(rule.config, tbl)

  if rule.config.whitelist then
    if lua_maps_exprs.schema(rule.config.whitelist) then
      rule.config.whitelist_map = lua_maps_exprs.create(rspamd_config,
          rule.config.whitelist, N)
    elseif lua_maps.map_schema(rule.config.whitelist) then
      local map = lua_maps.map_add_from_ucl(rule.config.whitelist,
          'radix',
          sel_type .. ' reputation whitelist')

      if not map then
        rspamd_logger.errx(rspamd_config, "cannot parse whitelist map config for %s: (%s)",
            sel_type,
            rule.config.whitelist)
        return
      end

      rule.config.whitelist_map = {
        process = function(_, task)
          -- Hack: we assume that it is an ip whitelist :(
          local ip = task:get_from_ip()

          if ip and map:get_key(ip) then return true end
          return false
        end
      }
    else
      rspamd_logger.errx(rspamd_config, "cannot parse whitelist map config for %s: (%s)",
          sel_type,
          rule.config.whitelist)
      return
    end
  end

  local symbol = rule.selector.config.symbol or name
  if tbl.symbol then
    symbol = tbl.symbol
  end

  rule.symbol = symbol
  rule.enabled = true
  if rule.selector.init then
    rule.enabled = false
  end
  if rule.backend.init then
    rule.enabled = false
  end
  -- Perform additional initialization if needed
  rspamd_config:add_on_load(function(cfg, ev_base, worker)
    if rule.selector.init then
      if not rule.selector.init(rule, cfg, ev_base, worker) then
        rule.enabled = false
        rspamd_logger.errx(rspamd_config, 'Cannot init selector %s (backend %s) for symbol %s',
            sel_type, bk_type, rule.symbol)
      else
        rule.enabled = true
      end
    end
    if rule.backend.init then
      if not rule.backend.init(rule, cfg, ev_base, worker) then
        rule.enabled = false
        rspamd_logger.errx(rspamd_config, 'Cannot init backend (%s) for rule %s for symbol %s',
            bk_type, sel_type, rule.symbol)
      else
        rule.enabled = true
      end
    end

    if rule.enabled then
      rspamd_logger.infox(rspamd_config, 'Enable %s (%s backend) rule for symbol %s (split symbols: %s)',
          sel_type, bk_type, rule.symbol,
          rule.selector.config.split_symbols)
    end
  end)

  -- We now generate symbol for checking
  local rule_type = 'normal'
  if rule.selector.config.split_symbols then
    rule_type = 'callback'
  end

  local id = rspamd_config:register_symbol{
    name = rule.symbol,
    type = rule_type,
    callback = callback_gen(reputation_filter_cb, rule),
  }

  if rule.selector.config.split_symbols then
    rspamd_config:register_symbol{
      name = rule.symbol .. '_HAM',
      type = 'virtual',
      parent = id,
    }
    rspamd_config:register_symbol{
      name = rule.symbol .. '_SPAM',
      type = 'virtual',
      parent = id,
    }
  end

  if rule.selector.dependencies then
    fun.each(function(d)
      rspamd_config:register_dependency(symbol, d)
    end, rule.selector.dependencies)
  end

  if rule.selector.postfilter then
    -- Also register a postfilter
    rspamd_config:register_symbol{
      name = rule.symbol .. '_POST',
      type = 'postfilter',
      flags = 'nostat',
      callback = callback_gen(reputation_postfilter_cb, rule),
    }
  end

  if rule.selector.idempotent then
    -- Has also idempotent component (e.g. saving data to the backend)
    rspamd_config:register_symbol{
      name = rule.symbol .. '_IDEMPOTENT',
      type = 'idempotent',
      callback = callback_gen(reputation_idempotent_cb, rule),
    }
  end

end

redis_params = lua_redis.parse_redis_server('reputation')
local opts = rspamd_config:get_all_opt("reputation")

-- Initialization part
if not (opts and type(opts) == 'table') then
  rspamd_logger.infox(rspamd_config, 'Module is unconfigured')
  return
end

if opts['rules'] then
  for k,v in pairs(opts['rules']) do
    if not ((v or E).selector) then
      rspamd_logger.errx(rspamd_config, "no selector defined for rule %s", k)
    else
      parse_rule(k, v)
    end
  end
else
  lua_util.disable_module(N, "config")
end