diff options
39 files changed, 836 insertions, 435 deletions
diff --git a/conf/composites.conf b/conf/composites.conf index 5125de45c..edb5d3205 100644 --- a/conf/composites.conf +++ b/conf/composites.conf @@ -33,5 +33,5 @@ composite "RBL_SPAMHAUS_XBL_ANY" { expression = "(-RBL_SPAMHAUS_XBL | -RBL_SPAMHAUS_XBL1 | -RBL_SPAMHAUS_XBL2 | -RBL_SPAMHAUS_XBL3) & RECEIVED_SPAMHAUS_XBL"; } -.include(try=true; priority=1) "$LOCAL_CONFDIR/local.d/composites.conf" +.include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/composites.conf" .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/composites.conf" diff --git a/conf/options.inc b/conf/options.inc index 318de7554..41a1c6ecb 100644 --- a/conf/options.inc +++ b/conf/options.inc @@ -33,6 +33,9 @@ allow_raw_input = true; # amount of words processed will not be *LIKELY more than the twice of that limit words_decay = 200; +# Write statistics about rspamd usage to the round-robin database +rrd = "${DBDIR}/rspamd.rrd"; + # Local networks local_addrs = "192.168.0.0/16, 10.0.0.0/8, 172.16.0.0/12, fd00::/8, 169.254.0.0/16, fe80::/10"; hs_cache_dir = "${DBDIR}/"; diff --git a/doc/markdown/tutorials/index.md b/doc/markdown/tutorials/index.md index 46d1cd727..4a3b0d5d9 100644 --- a/doc/markdown/tutorials/index.md +++ b/doc/markdown/tutorials/index.md @@ -1,8 +1,9 @@ -# rspamd tutorials +# Rspamd tutorials -In this section you can find the current step-by-step tutorials for rspamd. +In this section you can find the current step-by-step tutorials coverign various topics about rspamd. * [Migrating from SA](migrate_sa.md) - the guide for those who wants to migrate an existing SpamAssassin system to Rspamd * [Writing rspamd rules](writing_rules.md) - how to extend rspamd by writing your own rules +* [Creating your fuzzy storage](http://rspamd.com/doc/fuzzy_storage.html) - learn how to make your own fuzzy storage * [Training rspamd with dovecot antispam plugin, part 1](https://kaworu.ch/blog/2014/03/25/dovecot-antispam-with-rspamd/) - this tutorial describes how to train rspamd automatically using the `antispam` pluging of the `dovecot` IMAP server * [Training rspamd with dovecot antispam plugin, part 2](https://kaworu.ch/blog/2015/10/12/dovecot-antispam-with-rspamd-part2/) - continuation of the previous tutorial diff --git a/doc/markdown/tutorials/migrate_sa.md b/doc/markdown/tutorials/migrate_sa.md index 88b1d844f..6af9e8441 100644 --- a/doc/markdown/tutorials/migrate_sa.md +++ b/doc/markdown/tutorials/migrate_sa.md @@ -1,4 +1,4 @@ -# Migrating from SpamAssassin to rspamd +# Migrating from SpamAssassin to Rspamd This guide provides information for those who wants to migrate an existing system from [SpamAssassin](https://spamassassin.apache.org) to rspamd. You will find information about major differences between the spam filtering engines and how to deal with the transition process. diff --git a/interface/css/d3evolution.css b/interface/css/d3evolution.css new file mode 100644 index 000000000..49082e3e8 --- /dev/null +++ b/interface/css/d3evolution.css @@ -0,0 +1,22 @@ +.d3evolution svg { + background-color: white; +} +.d3evolution .chart-title { + font-size: 17px; +} +.d3evolution .grid .tick { + stroke: lightgrey; + shape-rendering: crispEdges; +} +.d3evolution .axis, +.d3evolution .legend { + font-size: 13px; +} +.d3evolution .axis path, +.d3evolution .axis line { + fill: none; + stroke: grey; +} +.d3evolution .legend circle { + stroke-width: 2px; +} diff --git a/interface/css/rspamd.css b/interface/css/rspamd.css index 87ecc3bb3..fd73d41ce 100644 --- a/interface/css/rspamd.css +++ b/interface/css/rspamd.css @@ -302,7 +302,7 @@ input::-webkit-inner-spin-button { height:200px; } .scan-textarea { - height:400px; + height:300px; } .stat-boxes { overflow:hidden !important; diff --git a/interface/index.html b/interface/index.html index c121f69e1..e155e6894 100644 --- a/interface/index.html +++ b/interface/index.html @@ -6,10 +6,9 @@ <meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="description" content=""> <meta name="author" content=""> - <link href="//cdnjs.cloudflare.com/ajax/libs/file-uploader/3.7.0/fineuploader.min.css" rel="stylesheet"> <link href="//maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css" rel="stylesheet"> <link rel="stylesheet" type="text/css" href="./css/datatables.min.css"/> - <link rel="stylesheet" type="text/css" href="//cdn.rawgit.com/moisseev/D3Evolution/fb6ea62c43e26e728b48a43012fb796c5ab6689c/d3evolution.css"> + <link rel="stylesheet" type="text/css" href="./css/d3evolution.css"> <link href="./css/rspamd.css" rel="stylesheet"> </head> @@ -26,9 +25,9 @@ </ul> <ul class="nav navbar-nav nav-pills" role="tablist"> <li role="presentation" class="active"><a id="status_nav" aria-controls="status" role="tab" href="#status" data-toggle="tab">Status</a></li> -<!-- + <li role="presentation"><a id="throughput_nav" aria-controls="throughput" role="tab" href="#throughput" data-toggle="tab">Throughput</a></li> ---> + <li role="presentation"><a id="configuration_nav" aria-controls="configuration" role="tab" href="#configuration" data-toggle="tab">Configuration</a></li> <li role="presentation"><a id="learning_nav" aria-controls="learning" role="tab" href="#learning" data-toggle="tab">Learning</a></li> <li role="presentation"><a id="scan_nav"aria-controls="scan" role="tab" href="#scan" data-toggle="tab">Scan</a></li> @@ -116,8 +115,7 @@ <div class="widget-title"> <span class="icon"><i class="glyphicon glyphicon-tasks"></i></span><h5>Actions</h5> </div> - <div class="widget-content actions-content" id="actionsBody"> - </div> + <div class="widget-content actions-content" id="actionsBody"></div> </div> <div class="widget-box"> <div class="widget-title"> @@ -150,20 +148,6 @@ </div> <div class="widget-content"> <div class="row"> - <form class="col-md-6 upload-form" id="uploadSpamForm"> - <h5>Upload SPAM examples:</h5> - <button id="uploadSpamTrigger" class="btn pull-right btn-upload-trigger"><i class="glyphicon glyphicon-upload"></i> Upload files</button> - <div id="uploadSpamFiles"></div> - </form> - <form class="col-md-6 upload-form" id="uploadHamForm"> - <h5>Upload HAM examples:</h5> - <button id="uploadHamTrigger" class="btn pull-right btn-upload-trigger"><i class="glyphicon glyphicon-upload"></i> Upload files</button> - <div id="uploadHamFiles"></div> - </form> - </div> - </div> - <div class="widget-content"> - <div class="row"> <form class="col-md-6"> <h5>Insert raw SPAM source:</h5> <textarea class="col-md-5 upload-textarea" id="spamTextSource" value=""></textarea> @@ -176,29 +160,6 @@ </form> </div> <div class="row"> - <form class="col-md-6 upload-form" id="uploadFuzzyForm"> - <h5>Upload Fuzzy examples:</h5> - <div class="row"> - <label class="pull-left"> - Flag - </label> - <div class="pull-right col-md-10"> - <input id="fuzzyFlagUpload" class="slider" type="slider" value="0"/> - </div> - </div> - <div class="row"> - <label class="pull-left"> - Weight - </label> - <div class="pull-right col-md-10"> - <input id="fuzzyWeightUpload" class="slider" type="slider" value="0"/> - </div> - </div> - <button id="uploadFuzzyTrigger" class="btn pull-right btn-upload-trigger"><i class="glyphicon glyphicon-upload"></i> Upload files</button> - <div id="uploadFuzzyFiles"></div> - </form> - </div> - <div class="row"> <form class="col-md-6"> <h5>Insert raw Fuzzy storage:</h5> <textarea class="col-md-5 upload-textarea" id="fuzzyTextSource" value=""></textarea> @@ -207,7 +168,7 @@ Flag </label> <div class="pull-right col-md-10"> - <input id="fuzzyFlagText" class="slider" type="slider" value="0"/> + <input id="fuzzyFlagText" class="slider" type="slider" value="1"/> </div> </div> <div class="row"> @@ -215,7 +176,7 @@ Weight </label> <div class="pull-right col-md-10"> - <input id="fuzzyWeightText" class="slider" type="slider" value="0"/> + <input id="fuzzyWeightText" class="slider" type="slider" value="1"/> </div> </div> <p><button class="btn btn-default pull-right" data-upload="fuzzy"><i class="glyphicon glyphicon-upload"></i> Upload text</button></p> @@ -357,13 +318,11 @@ <script src="//code.jquery.com/jquery-2.1.4.min.js"></script> <script src="//maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script> <script src="//cdnjs.cloudflare.com/ajax/libs/jquery-cookie/1.4.1/jquery.cookie.min.js"></script> -<script src="//cdnjs.cloudflare.com/ajax/libs/file-uploader/3.7.0/fineuploader.min.js"></script> <script src="//cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js"></script> <script src="./js/d3pie.min.js"></script> -<script src="//cdn.rawgit.com/moisseev/D3Evolution/fb6ea62c43e26e728b48a43012fb796c5ab6689c/d3evolution.js"></script> +<script src="./js/d3evolution.min.js"></script> <script src="./js/humanize.min.js"></script> +<script src="./js/datatables.min.js"></script> <script src="./js/rspamd.js"></script> -<script type="text/javascript" src="./js/datatables.min.js"></script> - </body> </html> diff --git a/interface/js/d3evolution.min.js b/interface/js/d3evolution.min.js new file mode 100644 index 000000000..cfcc917f6 --- /dev/null +++ b/interface/js/d3evolution.min.js @@ -0,0 +1 @@ +function D3Evolution(t,e){"use strict";var n,r,a=$.extend(!0,{title:"",width:800,height:400,margin:{top:80,right:60,bottom:40,left:60},type:"line",duration:1250,interpolate:"linear",legend:{buttonRadius:7,space:130,entries:[]}},e),i=a.width-a.margin.left-a.margin.right,l=a.height-a.margin.top-a.margin.bottom,o=d3.time.scale().range([0,i]),c=d3.scale.linear().range([l,0]),s=d3.svg.axis().scale(o),u=d3.svg.axis().scale(c).orient("left").ticks(5),d=d3.svg.axis().tickFormat("").scale(o).tickSize(-l,0),p=d3.svg.axis().tickFormat("").scale(c).orient("left").tickSize(-i,0),f=d3.svg.line().x(function(t){return o(t.x)}).y(function(t){return c(t.y)}).interpolate(a.interpolate),g=d3.svg.area().x(function(t){return o(t.x)}).y0(function(t){return c(t.y0)}).y1(function(t){return c(t.y0+t.y)}).interpolate(a.interpolate),y=d3.layout.stack(),h=d3.scale.category10(),m=function(t){return void 0!==a.legend.entries[t]&&void 0!==a.legend.entries[t].color?a.legend.entries[t].color:h(t)},x=function(t){return void 0!==a.legend.entries[t]&&void 0!==a.legend.entries[t].label?a.legend.entries[t].label:"path_"+t},v=d3.select("#"+t).append("svg").classed("d3evolution",!0).attr("width",a.width).attr("height",a.height),k=v.append("g").attr("class","legend"),b=v.append("g").attr("width",i).attr("height",l).attr("transform","translate("+a.margin.left+", "+a.margin.top+")");b.append("g").attr("class","x grid").attr("transform","translate(0,"+l+")").call(d),b.append("g").attr("class","y grid").attr("transform","translate(0,0)").call(p),b.append("g").attr("class","x axis").attr("transform","translate(0,"+l+")").call(s),b.append("g").attr("class","y axis").attr("transform","translate(0,0)").call(u),v.append("svg:text").attr("class","chart-title").attr("x",a.width/2).attr("y",a.margin.top/3).attr("text-anchor","middle").text(a.title),this.data=function(t){n=t;var e=[];r=a.width-a.margin.right-a.legend.space*n.length,n.forEach(function(t){t.forEach(function(t){t.x*=1e3})});var i,l=d3.extent(d3.merge(n),function(t){return t.x});"area"===a.type?(y(n),i=[0,d3.max(d3.merge(n),function(t){return t.y0+t.y})]):i=d3.extent(d3.merge(n),function(t){return t.y}),o.domain([l[0],l[1]]),c.domain([i[0]>0?0:i[0],i[1]]);var h=b.selectAll("path.path").data(n);h.enter().append("path").attr("class","path").attr("id",function(t,e){return"path_"+e}).style("area"===a.type?{fill:function(t,e){return m(e)},stroke:"none","fill-opacity":function(t,n){return e[n]}}:{fill:"none",stroke:function(t,e){return m(e)},opacity:function(t,n){return e[n]}}),h.transition().duration(a.duration).attr("d","area"===a.type?g:f),h.exit().remove(),d3.transition().duration(a.duration).each(function(){b.select(".x.grid").call(d.scale(o)),b.select(".y.grid").call(p.scale(c)),b.select(".x.axis").call(s.scale(o)),b.select(".y.axis").call(u.scale(c))});var v=function(t){e[t]=0!=e[t]?0:1,d3.select("#circle_"+t).transition().duration(a.duration).style("fill-opacity",e[t]+.2),d3.select("#path_"+t).transition().duration(a.duration).style("opacity",e[t])},_=function(t){d3.select("#circle_"+t).attr("r",1.3*a.legend.buttonRadius),d3.select("#path_"+t).style("opacity",.4).style("fill-opacity",.4)},A=function(t){d3.select("#circle_"+t).attr("r",a.legend.buttonRadius),d3.select("#path_"+t).style("opacity",e[t]).style("fill-opacity",e[t])},w=k.selectAll("circle").data(n);w.enter().append("circle").attr("id",function(t,e){return"circle_"+e}).attr("cy",2*a.margin.top/3).attr("r",a.legend.buttonRadius).style("fill",function(t,e){return m(e)}).style("stroke",function(t,e){return m(e)}).style("fill-opacity",function(t,n){return e[n]+.2}).on("click",function(t,e){v(e)}).on("mouseover",function(t,e){_(e)}).on("mouseout",function(t,e){A(e)}),w.exit().remove(),w.transition().duration(a.duration).attr("cx",function(t,e){return r+a.legend.space*e});var R=k.selectAll("text").data(n);return R.enter().append("text").attr("y",2*a.margin.top/3).attr("dy","0.3em").text(function(t,e){return x(e)}).on("click",function(t,e){v(e)}).on("mouseover",function(t,e){_(e)}).on("mouseout",function(t,e){A(e)}),R.exit().remove(),R.transition().duration(a.duration).attr("x",function(t,e){return r+a.legend.space*e+2*a.legend.buttonRadius}),this},this.legend=function(t){return $.extend(!0,a.legend.entries,t),k.selectAll("circle").attr("cx",function(t,e){return r+a.legend.space*e}).transition().duration(a.duration).style("fill",function(t,e){return m(e)}).style("stroke",function(t,e){return m(e)}),k.selectAll("text").text(function(t,e){return x(e)}),b.selectAll("path.path").transition().duration(a.duration).style("fill","area"===a.type?function(t,e){return m(e)}:"none").style("stroke","area"!==a.type?function(t,e){return m(e)}:"none"),this},this.interpolate=function(t){return a.interpolate=t,g.interpolate(a.interpolate),f.interpolate(a.interpolate),b.selectAll("path.path").attr("d","area"===a.type?g:f),this},this.type=function(t){a.type=t;var e;return"area"===a.type?(y(n),e=[0,d3.max(d3.merge(n),function(t){return t.y0+t.y})]):e=d3.extent(d3.merge(n),function(t){return t.y}),c.domain([e[0]>0?0:e[0],e[1]]),b.selectAll("path.path").style("stroke","area"!==a.type?function(t,e){return m(e)}:"none").style("fill","area"===a.type?function(t,e){return m(e)}:"none").transition().duration(a.duration).attr("d","area"===a.type?g:f),d3.transition().duration(a.duration).each(function(){b.select(".y.grid").call(p.scale(c)),b.select(".y.axis").call(u.scale(c))}),this},this.destroy=function(){d3.select("svg").remove()}}
\ No newline at end of file diff --git a/interface/js/rspamd.js b/interface/js/rspamd.js index fbb798b5f..a7972caa6 100644 --- a/interface/js/rspamd.js +++ b/interface/js/rspamd.js @@ -515,8 +515,8 @@ $.ajax({ dataType: 'json', type: 'GET', - url: 'graph?type=', - data: type, + url: 'graph', + data: {"type": type}, beforeSend: function (xhr) { xhr.setRequestHeader('Password', getPassword()); }, @@ -549,6 +549,12 @@ // } // @get history log function getHistory() { + + if (history) { + history.destroy(); + $('#historyLog').children('tbody').remove(); + } + var items = []; $.ajax({ dataType: 'json', @@ -671,8 +677,10 @@ } // @update history log $('#resetHistory').on('click', function () { - history.destroy(); - $('#historyLog').children('tbody').remove(); + if (history) { + history.destroy(); + $('#historyLog').children('tbody').remove(); + } $.ajax({ dataType: 'json', type: 'GET', @@ -688,143 +696,12 @@ } }); }); + // @reset history log $('#updateHistory').on('click', function () { - history.destroy(); - $('#historyLog').children('tbody').remove(); getHistory(); }); - // @spam upload form - function createUploaders() { - var spamUploader = new qq.FineUploader({ - element: $('#uploadSpamFiles')[0], - request: { - endpoint: 'learnspam', - customHeaders: { - 'Password': getPassword() - } - }, - validation: { - allowedExtensions: ['eml', 'msg', 'txt', 'html'], - sizeLimit: 52428800 - }, - autoUpload: false, - text: { - uploadButton: '<i class="glyphicon glyphicon-plus glyphicon-white"></i> Select Files' - }, - retry: { - enableAuto: false - }, - template: '<div class="qq-uploader">' + - '<pre class="qq-upload-drop-area span12"><span>{dragZoneText}</span></pre>' + - '<div class="qq-upload-button btn btn-danger">{uploadButtonText}</div>' + - '<span class="qq-drop-processing"><span>{dropProcessingText}</span><span class="qq-drop-processing-spinner"></span></span>' + - '<ul class="qq-upload-list"></ul>' + - '</div>', - classes: { - success: 'alert-success', - fail: 'alert-error' - }, - debug: true, - callbacks: { - onError: function () { - alertMessage('alert-error', 'Cannot upload data'); - } - } - }); - var hamUploader = new qq.FineUploader({ - element: $('#uploadHamFiles')[0], - request: { - endpoint: 'learnham', - customHeaders: { - 'Password': getPassword() - } - }, - validation: { - allowedExtensions: ['eml', 'msg', 'txt', 'html'], - sizeLimit: 52428800 - }, - autoUpload: false, - text: { - uploadButton: '<i class="glyphicon glyphicon-plus glyphicon-white"></i> Select Files' - }, - retry: { - enableAuto: true - }, - template: '<div class="qq-uploader">' + - '<pre class="qq-upload-drop-area span12"><span>{dragZoneText}</span></pre>' + - '<div class="qq-upload-button btn btn-success">{uploadButtonText}</div>' + - '<span class="qq-drop-processing"><span>{dropProcessingText}</span><span class="qq-drop-processing-spinner"></span></span>' + - '<ul class="qq-upload-list"></ul>' + - '</div>', - classes: { - success: 'alert-success', - fail: 'alert-error' - }, - debug: true, - callbacks: { - onError: function () { - alertMessage('alert-error', 'Cannot upload data'); - } - } - }); - var data = { - flag: $('#fuzzyFlagUpload').val(), - weight: $('#fuzzyWeightUpload').val() - }; - var fuzzyUploader = new qq.FineUploader({ - element: $('#uploadFuzzyFiles')[0], - request: { - endpoint: 'learnfuzzy', - customHeaders: { - 'Password': getPassword() - } - }, - validation: { - allowedExtensions: ['eml', 'msg', 'txt', 'html', 'pdf'], - sizeLimit: 52428800 - }, - autoUpload: false, - text: { - uploadButton: '<i class="glyphicon glyphicon-plus glyphicon-white"></i> Select Files' - }, - retry: { - enableAuto: true - }, - template: '<div class="qq-uploader">' + - '<pre class="qq-upload-drop-area span12"><span>{dragZoneText}</span></pre>' + - '<div class="qq-upload-button btn btn-success">{uploadButtonText}</div>' + - '<span class="qq-drop-processing"><span>{dropProcessingText}</span><span class="qq-drop-processing-spinner"></span></span>' + - '<ul class="qq-upload-list"></ul>' + - '</div>', - classes: { - success: 'alert-success', - fail: 'alert-error' - }, - debug: true, - callbacks: { - onError: function () { - alertMessage('alert-error', 'Cannot upload data'); - } - } - }); - // @upload spam button - $('#uploadSpamTrigger').on('click', function () { - spamUploader.uploadStoredFiles(); - return false; - }); - // @upload ham button - $('#uploadHamTrigger').on('click', function () { - hamUploader.uploadStoredFiles(); - return false; - }); - // @upload fuzzy button - $('#uploadFuzzyTrigger').on('click', function () { - fuzzyUploader.uploadStoredFiles(); - uploadText(data, 'fuzzy'); - return false; - }); - } + // @upload text function uploadText(data, source) { if (source === 'spam') { @@ -1033,10 +910,11 @@ min = item.value; } }); - $('<form/>', { id: 'actionsForm', class: '', html: items.join('') }).appendTo('#actionsBody'); - $('<br><div class="form-group">' + + + $('#actionsBody').html('<form id="actionsForm">' + items.join('') + + '<br><div class="form-group">' + '<button class="btn btn-primary" ' + - 'type="submit">Save actions</button></div>').appendTo('#actionsForm'); + 'type="submit">Save actions</button></div></form>'); } }); } @@ -1194,17 +1072,21 @@ statWidgets(); $('#mainUI').show(); $('#progress').show(); - getActions(); - getMaps(); - createUploaders(); - getSymbols(); - getHistory(); + getChart(); initGraph(); $('#progress').hide(); $(disconnect).show(); } + connectRSPAMD(); + + $('#configuration_nav').bind('click', function (e) { + getActions(); + getMaps(); + getSymbols(); + }); + $(document).ajaxStart(function () { $('#navBar').addClass('loading'); }); @@ -1217,5 +1099,8 @@ $('#throughput_nav').bind('click', function () { getGraphData(selected.selData); }); + $('#history_nav').bind('click', function() { + getHistory(); + }); }); })(); diff --git a/src/controller.c b/src/controller.c index 085eda5d4..b45e5add5 100644 --- a/src/controller.c +++ b/src/controller.c @@ -1022,6 +1022,45 @@ rspamd_controller_handle_pie_chart ( return 0; } +void +rspamd_controller_graph_point (gulong t, gulong step, + struct rspamd_rrd_query_result* rrd_result, + gdouble *acc, + ucl_object_t **elt) +{ + guint nan_cnt; + gdouble sum = 0.0, yval; + ucl_object_t* data_elt; + guint i, j; + + for (i = 0; i < rrd_result->ds_count; i++) { + sum = 0.0; + nan_cnt = 0; + data_elt = ucl_object_typed_new (UCL_OBJECT); + ucl_object_insert_key (data_elt, ucl_object_fromint (t), "x", 1, false); + + for (j = 0; j < step; j++) { + yval = acc[i + j * rrd_result->ds_count]; + if (isnan(yval)) { + nan_cnt++; + } + else { + sum += yval; + } + } + if (nan_cnt == step) { + ucl_object_insert_key (data_elt, ucl_object_typed_new (UCL_NULL), + "y", 1, false); + } + else { + ucl_object_insert_key (data_elt, + ucl_object_fromdouble (sum / (gdouble) step), "y", 1, + false); + } + ucl_array_append (elt[i], data_elt); + } +} + /* * Graph command handler: * request: /graph?type=<hourly|daily|weekly|monthly> @@ -1042,8 +1081,9 @@ rspamd_controller_handle_graph ( struct rspamd_controller_worker_ctx *ctx; rspamd_ftok_t srch, *value; struct rspamd_rrd_query_result *rrd_result; - gulong i, j, ts, start_row, cnt, t; - ucl_object_t *res, *elt[4], *data_elt; + gulong i, j, k, start_row, cnt, t, ts, step; + gdouble *acc; + ucl_object_t *res, *elt[4]; enum { rra_hourly = 0, rra_daily, @@ -1051,6 +1091,8 @@ rspamd_controller_handle_graph ( rra_monthly, rra_invalid } rra_num = rra_invalid; + /* How many points are we going to send to display */ + static const guint desired_points = 500; ctx = session->ctx; @@ -1080,16 +1122,16 @@ rspamd_controller_handle_graph ( return 0; } - if (strncmp (value->begin, "hourly", value->len) == 0) { + if (value->len == 6 && rspamd_lc_cmp (value->begin, "hourly", value->len) == 0) { rra_num = rra_hourly; } - else if (strncmp (value->begin, "daily", value->len) == 0) { - rra_num = rra_hourly; + else if (value->len == 5 && rspamd_lc_cmp (value->begin, "daily", value->len) == 0) { + rra_num = rra_daily; } - else if (strncmp (value->begin, "weekly", value->len) == 0) { - rra_num = rra_hourly; + else if (value->len == 6 && rspamd_lc_cmp (value->begin, "weekly", value->len) == 0) { + rra_num = rra_weekly; } - else if (strncmp (value->begin, "monthly", value->len) == 0) { + else if (value->len == 7 && rspamd_lc_cmp (value->begin, "monthly", value->len) == 0) { rra_num = rra_monthly; } @@ -1124,43 +1166,49 @@ rspamd_controller_handle_graph ( start_row = rrd_result->cur_row == rrd_result->rra_rows - 1 ? 0 : rrd_result->cur_row; - for (i = start_row, cnt = 0; cnt < rrd_result->rra_rows; cnt ++) { - for (j = 0; j < rrd_result->ds_count; j++) { - gdouble yval; - - data_elt = ucl_object_typed_new (UCL_OBJECT); - t = ts * rrd_result->pdp_per_cdp; - ucl_object_insert_key (data_elt, - ucl_object_fromint (t), - "x", 1, - false); - yval = rrd_result->data[i * rrd_result->ds_count + j]; + /* Create window */ + step = (rrd_result->rra_rows / desired_points + 0.5); + acc = g_malloc0 (sizeof (double) * rrd_result->ds_count * step); - if (!isnan (yval)) { - ucl_object_insert_key (data_elt, - ucl_object_fromdouble (yval), - "y", 1, - false); + for (i = start_row, cnt = 0; cnt < rrd_result->rra_rows; + cnt ++) { + for (j = 0; j < rrd_result->ds_count; j++) { + if (k < step) { + /* Just update window */ + acc[k * rrd_result->ds_count + j] = + rrd_result->data[i * rrd_result->ds_count + j]; + k ++; } else { - ucl_object_insert_key (data_elt, - ucl_object_typed_new (UCL_NULL), - "y", 1, - false); + t = ts * rrd_result->pdp_per_cdp; + + /* Need a fresh point */ + rspamd_controller_graph_point (t, step, rrd_result, acc, elt); + k = 0; } - ucl_array_append (elt[j], data_elt); } - i = start_row == 0 ? i + 1 : (i + 1) % start_row; + if (i == rrd_result->rra_rows - 1) { + i = 0; + } + else { + i ++; + } + ts ++; } + if (k > 0) { + rspamd_controller_graph_point (t, k, rrd_result, acc, elt); + } + for (i = 0; i < rrd_result->ds_count; i++) { ucl_array_append (res, elt[i]); } rspamd_controller_send_ucl (conn_ent, res); ucl_object_unref (res); + g_free (acc); return 0; } @@ -2619,7 +2667,9 @@ start_controller_worker (struct rspamd_worker *worker) /* RRD collector */ if (ctx->cfg->rrd_file && worker->index == 0) { - ctx->rrd = rspamd_rrd_file_default (ctx->cfg->rrd_file, NULL); + GError *rrd_err = NULL; + + ctx->rrd = rspamd_rrd_file_default (ctx->cfg->rrd_file, &rrd_err); if (ctx->rrd) { ctx->rrd_event = g_slice_alloc0 (sizeof (*ctx->rrd_event)); @@ -2627,6 +2677,14 @@ start_controller_worker (struct rspamd_worker *worker) event_base_set (ctx->ev_base, ctx->rrd_event); event_add (ctx->rrd_event, &rrd_update_time); } + else if (rrd_err) { + msg_err ("cannot load rrd from %s: %e", ctx->cfg->rrd_file, + rrd_err); + g_error_free (rrd_err); + } + else { + msg_err ("cannot load rrd from %s: unknown error", ctx->cfg->rrd_file); + } } else { ctx->rrd = NULL; diff --git a/src/libutil/map.c b/src/libutil/map.c index a29939f26..354ac722e 100644 --- a/src/libutil/map.c +++ b/src/libutil/map.c @@ -599,9 +599,8 @@ rspamd_map_periodic_dtor (struct map_periodic_cbdata *periodic) /* Not modified */ } - rspamd_map_schedule_periodic (periodic->map, FALSE, FALSE, FALSE); - if (periodic->locked) { + rspamd_map_schedule_periodic (periodic->map, FALSE, FALSE, FALSE); g_atomic_int_set (periodic->map->locked, 0); } @@ -627,7 +626,7 @@ rspamd_map_schedule_periodic (struct rspamd_map *map, timeout = map->poll_timeout * error_mult; } else if (locked) { - timeout = map->poll_timeout * lock_mult; + timeout = lock_mult; } cbd = g_slice_alloc0 (sizeof (*cbd)); @@ -638,8 +637,6 @@ rspamd_map_schedule_periodic (struct rspamd_map *map, cbd->map = map; REF_INIT_RETAIN (cbd, rspamd_map_periodic_dtor); - msg_debug_map ("schedule new periodic event %p in %.2f seconds", cbd, timeout); - if (initial) { evtimer_set (&map->ev, rspamd_map_periodic_callback, cbd); event_base_set (map->ev_base, &map->ev); @@ -647,9 +644,11 @@ rspamd_map_schedule_periodic (struct rspamd_map *map, else { evtimer_del (&map->ev); evtimer_set (&map->ev, rspamd_map_periodic_callback, cbd); + event_base_set (map->ev_base, &map->ev); } jittered_sec = rspamd_time_jitter (timeout, 0); + msg_debug_map ("schedule new periodic event %p in %.2f seconds", cbd, jittered_sec); double_to_tv (jittered_sec, &map->tv); evtimer_add (&map->ev, &map->tv); diff --git a/src/libutil/rrd.c b/src/libutil/rrd.c index 470896efa..a230faf2b 100644 --- a/src/libutil/rrd.c +++ b/src/libutil/rrd.c @@ -217,20 +217,30 @@ rspamd_rrd_check_file (const gchar *filename, gboolean need_data, GError **err) return FALSE; } /* Check magic */ - if (memcmp (head.cookie, RRD_COOKIE, sizeof (head.cookie)) != 0 || - memcmp (head.version, RRD_VERSION, sizeof (head.version)) != 0 || - head.float_cookie != RRD_FLOAT_COOKIE) { + if (memcmp (head.version, RRD_VERSION, sizeof (head.version)) != 0) { g_set_error (err, - rrd_error_quark (), EINVAL, "rrd head cookies error: %s", - strerror (errno)); + rrd_error_quark (), EINVAL, "rrd head error: bad cookie"); + close (fd); + return FALSE; + } + if (memcmp (head.version, RRD_VERSION, sizeof (head.version)) != 0) { + g_set_error (err, + rrd_error_quark (), EINVAL, "rrd head error: invalid version"); + close (fd); + return FALSE; + } + if (head.float_cookie != RRD_FLOAT_COOKIE) { + g_set_error (err, + rrd_error_quark (), EINVAL, "rrd head error: another architecture " + "(file cookie %g != our cookie %g)", + head.float_cookie, RRD_FLOAT_COOKIE); close (fd); return FALSE; } /* Check for other params */ if (head.ds_cnt <= 0 || head.rra_cnt <= 0) { g_set_error (err, - rrd_error_quark (), EINVAL, "rrd head cookies error: %s", - strerror (errno)); + rrd_error_quark (), EINVAL, "rrd head cookies error: bad rra or ds count"); close (fd); return FALSE; } diff --git a/src/plugins/lua/phishing.lua b/src/plugins/lua/phishing.lua index 2e2b92244..ec9b9d2ad 100644 --- a/src/plugins/lua/phishing.lua +++ b/src/plugins/lua/phishing.lua @@ -30,42 +30,88 @@ local phishtank_enabled = false local openphish_premium = false local openphish_hash local phishtank_hash -local openphish_json = {} +local openphish_data = {} local phishtank_data = {} local rspamd_logger = require "rspamd_logger" local util = require "rspamd_util" local opts = rspamd_config:get_all_opt('phishing') local function phishing_cb(task) - local urls = task:get_urls() + local function check_phishing_map(map, url, symbol) + local host = url:get_host() + + if host then + local elt = map[host] + local found_path = false + local found_query = false + local data = nil + + if elt then + local path = url:get_path() + local query = url:get_query() + + if path then + for _,d in ipairs(elt) do + if d['path'] == path then + found_path = true + data = d['data'] + + if query and d['query'] and query == d['query'] then + found_query = true + elseif not d['query'] then + found_query = true + end + end + end + else + if not d['path'] then + found_path = true + found_query = true + end + end - if urls then - for _,url in ipairs(urls) do - if openphish_hash then - local t = url:get_text() - - if openphish_premium then - local elt = openphish_json[t] - if elt then - task:insert_result(openphish_symbol, 1.0, { - elt['tld'], - elt['sector'], - elt['brand'], - }) + if found_path then + local args = nil + + if type(data) == 'table' then + args = { + data['tld'], + data['sector'], + data['brand'], + } + elseif type(data) == 'string' then + args = data + else + args = host + end + + if found_query then + -- Query + path match + task:insert_result(symbol, 1.0, args) + else + -- Host + path match + task:insert_result(symbol, 0.3, args) end else - if openphish_hash:get_key(t) then - task:insert_result(openphish_symbol, 1.0, url:get_tld()) + if url:is_phished() then + -- Only host matches + task:insert_result(symbol, 0.1, host) end end end + end + end + + local urls = task:get_urls() + + if urls then + for _,url in ipairs(urls) do + if openphish_hash then + check_phishing_map(openphish_data, url, openphish_symbol) + end if phishtank_hash then - local t = url:get_text() - local elt = phishtank_data[t] - if elt then - task:insert_result(phishtank_symbol, 1.0, elt) - end + check_phishing_map(phishtank_data, url, phishtank_symbol) end if url:is_phished() and not url:is_redirected() then @@ -158,12 +204,42 @@ local function rspamd_str_split_fun(s, sep, func) return lpeg.match(p, s) end +local function insert_url_from_string(pool, tbl, str, data) + local rspamd_url = require "rspamd_url" + + local u = rspamd_url.create(pool, str) + + if u then + local host = u:get_host() + if host then + local elt = { + data = data, + path = u:get_path(), + query = u:get_query() + } + + if tbl[host] then + table.insert(tbl[host], elt) + else + tbl[host] = {elt} + end + + return true + end + end + + return false +end + local function openphish_json_cb(string) local ucl = require "ucl" + local rspamd_mempool = require "rspamd_mempool" local nelts = 0 local new_json_map = {} local valid = true + local pool = rspamd_mempool.create() + local function openphish_elt_parser(cap) if valid then local parser = ucl.parser() @@ -175,8 +251,9 @@ local function openphish_json_cb(string) local obj = parser:get_object() if obj['url'] then - new_json_map[obj['url']] = obj - nelts = nelts + 1 + if insert_url_from_string(pool, new_json_map, obj['url'], obj) then + nelts = nelts + 1 + end end end end @@ -185,10 +262,32 @@ local function openphish_json_cb(string) rspamd_str_split_fun(string, '\n', openphish_elt_parser) if valid then - openphish_json = new_json_map + openphish_data = new_json_map rspamd_logger.infox(openphish_hash, "parsed %s elements from openphish feed", nelts) end + + pool:destroy() +end + +local function openphish_plain_cb(string) + local nelts = 0 + local new_data = {} + local rspamd_mempool = require "rspamd_mempool" + local pool = rspamd_mempool.create() + + local function openphish_elt_parser(cap) + if insert_url_from_string(pool, new_data, cap, nil) then + nelts = nelts + 1 + end + end + + rspamd_str_split_fun(string, '\n', openphish_elt_parser) + + openphish_data = new_data + rspamd_logger.infox(openphish_hash, "parsed %s elements from openphish feed", + nelts) + pool:destroy() end local function phishtank_json_cb(string) @@ -198,6 +297,8 @@ local function phishtank_json_cb(string) local valid = true local parser = ucl.parser() local res,err = parser:parse_string(string) + local rspamd_mempool = require "rspamd_mempool" + local pool = rspamd_mempool.create() if not res then valid = false @@ -207,8 +308,10 @@ local function phishtank_json_cb(string) for _,elt in ipairs(obj) do if elt['url'] then - new_data[elt['url']] = elt['phish_detail_url'] - nelts = nelts + 1 + if insert_url_from_string(pool, new_data, elt['url'], + elt['phish_detail_url']) then + nelts = nelts + 1 + end end end end @@ -218,6 +321,9 @@ local function phishtank_json_cb(string) rspamd_logger.infox(phishtank_hash, "parsed %s elements from phishtank feed", nelts) end + + + pool:destroy() end if opts then @@ -243,8 +349,9 @@ if opts then if opts['openphish_enabled'] then if not openphish_premium then openphish_hash = rspamd_config:add_map({ - type = 'set', + type = 'callback', url = openphish_map, + callback = openphish_plain_cb, description = 'Open phishing feed map (see https://www.openphish.com for details)' }) else diff --git a/test/functional/cases/__init__.robot b/test/functional/cases/__init__.robot index 46fdc8f2a..4b95b9217 100644 --- a/test/functional/cases/__init__.robot +++ b/test/functional/cases/__init__.robot @@ -1,18 +1,12 @@ *** Settings *** -Suite Setup Export Global Variables -Library ../lib/rspamd.py -Variables ../lib/vars.py +Suite Setup Export Global Variables +Library ../lib/rspamd.py +Variables ../lib/vars.py *** Keywords *** Export Global Variables - ${TESTDIR} = Get Test Directory - Set Global Variable ${KEY_PUB1} - Set Global Variable ${KEY_PVT1} - Set Global Variable ${LOCAL_ADDR} - Set Global Variable ${PORT_CONTROLLER} - Set Global Variable ${PORT_NORMAL} - Set Global Variable ${RSPAMC} ${TESTDIR}/../../src/client/rspamc - Set Global Variable ${RSPAMD} ${TESTDIR}/../../src/rspamd - Set Global Variable ${RSPAMD_GROUP} - Set Global Variable ${RSPAMD_USER} - Set Global Variable ${TESTDIR} + ${TESTDIR} = Get Test Directory + Set Global Variable ${RSPAMADM} ${TESTDIR}/../../src/rspamadm/rspamadm + Set Global Variable ${RSPAMC} ${TESTDIR}/../../src/client/rspamc + Set Global Variable ${RSPAMD} ${TESTDIR}/../../src/rspamd + Set Global Variable ${TESTDIR} diff --git a/test/functional/cases/fuzzy/encrypted.robot b/test/functional/cases/fuzzy/encrypted.robot new file mode 100644 index 000000000..45408d7f8 --- /dev/null +++ b/test/functional/cases/fuzzy/encrypted.robot @@ -0,0 +1,20 @@ +*** Settings *** +Suite Setup Encrypted Fuzzy Setup +Suite Teardown Generic Teardown +Resource lib.robot + +*** Test Cases *** +Fuzzy Add + Fuzzy Add Test + +Fuzzy Delete + Fuzzy Delete Test + +Fuzzy Overwrite + Fuzzy Overwrite Test + +*** Keywords *** +Encrypted Fuzzy Setup + Set Suite Variable ${SETTINGS_FUZZY_WORKER} "keypair": {"pubkey": "${KEY_PUB1}", "privkey": "${KEY_PVT1}"}; "encrypted_only": true; + Set Suite Variable ${SETTINGS_FUZZY_CHECK} encryption_key = "${KEY_PUB1}"; + Generic Setup diff --git a/test/functional/cases/fuzzy/lib.robot b/test/functional/cases/fuzzy/lib.robot new file mode 100644 index 000000000..e5e9cbcb7 --- /dev/null +++ b/test/functional/cases/fuzzy/lib.robot @@ -0,0 +1,48 @@ +*** Settings *** +Library ${TESTDIR}/lib/rspamd.py +Resource ${TESTDIR}/lib/rspamd.robot +Variables ${TESTDIR}/lib/vars.py + +*** Variables *** +${CONFIG} ${TESTDIR}/configs/fuzzy.conf +${FLAG1_NUMBER} 50 +${FLAG1_SYMBOL} R_TEST_FUZZY_DENIED +${FLAG2_NUMBER} 51 +${FLAG2_SYMBOL} R_TEST_FUZZY_WHITE +${MESSAGE} ${TESTDIR}/messages/bad_message.eml +${RSPAMD_SCOPE} Suite + +*** Keywords *** +Fuzzy Add Test + Set Suite Variable ${RSPAMD_FUZZY_ADD} 0 + ${result} = Run Rspamc -h ${LOCAL_ADDR}:${PORT_CONTROLLER} -w 10 -f + ... ${FLAG1_NUMBER} fuzzy_add ${MESSAGE} + Check Rspamc ${result} + Sync Fuzzy Storage + ${result} = Scan Message With Rspamc ${MESSAGE} + Check Rspamc ${result} ${FLAG1_SYMBOL} + Set Suite Variable ${RSPAMD_FUZZY_ADD} 1 + +Fuzzy Delete Test + Run Keyword If ${RSPAMD_FUZZY_ADD} == 0 Fail "Fuzzy Add was not run" + ${result} = Run Rspamc -h ${LOCAL_ADDR}:${PORT_CONTROLLER} -f ${FLAG1_NUMBER} fuzzy_del + ... ${MESSAGE} + Check Rspamc ${result} + Sync Fuzzy Storage + ${result} = Scan Message With Rspamc ${MESSAGE} + Follow Rspamd Log + Should Not Contain ${result.stdout} ${FLAG1_SYMBOL} + Should Be Equal As Integers ${result.rc} 0 + +Fuzzy Overwrite Test + ${flag_numbers} = Create List ${FLAG1_NUMBER} ${FLAG2_NUMBER} + : FOR ${i} IN @{flag_numbers} + \ ${result} = Run Rspamc -h ${LOCAL_ADDR}:${PORT_CONTROLLER} -w 10 + \ ... -f ${i} fuzzy_add ${MESSAGE} + \ Check Rspamc ${result} + Sync Fuzzy Storage + ${result} = Scan Message With Rspamc ${MESSAGE} + Follow Rspamd Log + Should Not Contain ${result.stdout} ${FLAG1_SYMBOL} + Should Contain ${result.stdout} ${FLAG2_SYMBOL} + Should Be Equal As Integers ${result.rc} 0 diff --git a/test/functional/cases/fuzzy/plain.robot b/test/functional/cases/fuzzy/plain.robot new file mode 100644 index 000000000..2fc2fd5ef --- /dev/null +++ b/test/functional/cases/fuzzy/plain.robot @@ -0,0 +1,18 @@ +*** Settings *** +Suite Setup Generic Setup +Suite Teardown Generic Teardown +Resource lib.robot + +*** Variables *** +${SETTINGS_FUZZY_WORKER} ${EMPTY} +${SETTINGS_FUZZY_CHECK} ${EMPTY} + +*** Test Cases *** +Fuzzy Add + Fuzzy Add Test + +Fuzzy Delete + Fuzzy Delete Test + +Fuzzy Overwrite + Fuzzy Overwrite Test diff --git a/test/functional/cases/general.robot b/test/functional/cases/general.robot index aa146b4a9..5fb80520d 100644 --- a/test/functional/cases/general.robot +++ b/test/functional/cases/general.robot @@ -1,14 +1,14 @@ *** Settings *** -Library ${TESTDIR}/lib/rspamd.py -Resource ${TESTDIR}/lib/rspamd.robot Suite Setup Generic Setup Suite Teardown Generic Teardown +Library ${TESTDIR}/lib/rspamd.py +Resource ${TESTDIR}/lib/rspamd.robot +Variables ${TESTDIR}/lib/vars.py *** Variables *** -${CONFIG} ${TESTDIR}/configs/trivial.conf -${GTUBE} ${TESTDIR}/messages/gtube.eml -&{RSPAMD_KEYWORDS} KEY_PUBLIC=${KEY_PUB1} KEY_PRIVATE=${KEY_PVT1} LOCAL_ADDR=${LOCAL_ADDR} PORT_NORMAL=${PORT_NORMAL} TESTDIR=${TESTDIR} -${RSPAMD_SCOPE} Suite +${CONFIG} ${TESTDIR}/configs/trivial.conf +${GTUBE} ${TESTDIR}/messages/gtube.eml +${RSPAMD_SCOPE} Suite *** Test Cases *** GTUBE @@ -16,7 +16,8 @@ GTUBE Check Rspamc ${result} GTUBE ( GTUBE - Encrypted - ${result} = Run Rspamc -p -h ${LOCAL_ADDR}:${PORT_NORMAL} --key ${KEY_PUB1} ${GTUBE} + ${result} = Run Rspamc -p -h ${LOCAL_ADDR}:${PORT_NORMAL} --key ${KEY_PUB1} + ... ${GTUBE} Check Rspamc ${result} GTUBE ( GTUBE - Scan File feature diff --git a/test/functional/cases/lua.robot b/test/functional/cases/lua.robot index 46dd15ea0..f3a3567ec 100644 --- a/test/functional/cases/lua.robot +++ b/test/functional/cases/lua.robot @@ -1,19 +1,14 @@ *** Settings *** +Test Teardown Generic Teardown Library ${TESTDIR}/lib/rspamd.py Resource ${TESTDIR}/lib/rspamd.robot +Variables ${TESTDIR}/lib/vars.py *** Variables *** -${CONFIG} ${TESTDIR}/configs/lua_test.conf -${MESSAGE} ${TESTDIR}/messages/spam_message.eml +${CONFIG} ${TESTDIR}/configs/lua_test.conf +${MESSAGE} ${TESTDIR}/messages/spam_message.eml ${RSPAMD_SCOPE} Test -*** Keywords *** -Lua Setup - [Arguments] ${lua_script} - &{RSPAMD_KEYWORDS} = Create Dictionary LOCAL_ADDR=${LOCAL_ADDR} LUA_SCRIPT=${lua_script} PORT_CONTROLLER=${PORT_CONTROLLER} PORT_NORMAL=${PORT_NORMAL} TESTDIR=${TESTDIR} - Set Test Variable &{RSPAMD_KEYWORDS} - Generic Setup - *** Test Cases *** Flags [Setup] Lua Setup ${TESTDIR}/lua/flags.lua @@ -21,10 +16,14 @@ Flags Follow Rspamd Log ${result} = Run Rspamc -h ${LOCAL_ADDR}:${PORT_CONTROLLER} stat Should Contain ${result.stdout} Messages scanned: 0 - [Teardown] Generic Teardown Dependencies [Setup] Lua Setup ${TESTDIR}/lua/deps.lua ${result} = Scan Message With Rspamc ${MESSAGE} Check Rspamc ${result} DEP10 - [Teardown] Generic Teardown + +*** Keywords *** +Lua Setup + [Arguments] ${LUA_SCRIPT} + Set Test Variable ${LUA_SCRIPT} + Generic Setup diff --git a/test/functional/cases/statistics.robot b/test/functional/cases/statistics.robot deleted file mode 100644 index 6d0218b56..000000000 --- a/test/functional/cases/statistics.robot +++ /dev/null @@ -1,83 +0,0 @@ -*** Settings *** -Library ${TESTDIR}/lib/rspamd.py -Resource ${TESTDIR}/lib/rspamd.robot - -*** Variables *** -@{ALIASES} STATSDIR -${CONFIG} ${TESTDIR}/configs/stats.conf -${MESSAGE} ${TESTDIR}/messages/spam_message.eml -${RSPAMD_SCOPE} Test - -*** Keywords *** -Statistics Setup - [Arguments] @{aliases} &{kw} - &{RSPAMD_KEYWORDS} = Create Dictionary KEY_PRIVATE=${KEY_PVT1} KEY_PUBLIC=${KEY_PUB1} LOCAL_ADDR=${LOCAL_ADDR} PORT_CONTROLLER=${PORT_CONTROLLER} PORT_NORMAL=${PORT_NORMAL} TESTDIR=${TESTDIR} - Update Dictionary ${RSPAMD_KEYWORDS} ${kw} - Set Test Variable &{RSPAMD_KEYWORDS} - ${TMPDIR} ${RSPAMD_PID} ${RSPAMD_LOGPOS} = Run Rspamd @{aliases} &{RSPAMD_KEYWORDS} - Export Rspamd Vars To Test ${TMPDIR} ${RSPAMD_LOGPOS} ${RSPAMD_PID} - -*** Test Cases *** -Sqlite Learn - Keyed, siphash - [Setup] Statistics Setup @{ALIASES} STATS_BACKEND=sqlite3 STATS_HASH=siphash STATS_KEY=${KEY_PVT1} - ${result} = Run Rspamc -h ${LOCAL_ADDR}:${PORT_CONTROLLER} learn_spam ${MESSAGE} - Check Rspamc ${result} - ${result} = Scan Message With Rspamc ${MESSAGE} - Check Rspamc ${result} BAYES_SPAM - [Teardown] Generic Teardown - -Sqlite Learn - Keyed, xxhash - [Setup] Statistics Setup @{ALIASES} STATS_BACKEND=sqlite3 STATS_HASH=xxh STATS_KEY=${KEY_PVT1} - ${result} = Run Rspamc -h ${LOCAL_ADDR}:${PORT_CONTROLLER} learn_spam ${MESSAGE} - Check Rspamc ${result} - ${result} = Scan Message With Rspamc ${MESSAGE} - Check Rspamc ${result} BAYES_SPAM - [Teardown] Generic Teardown - -Sqlite Learn - Broken Stats Directory - [Setup] Statistics Setup @{EMPTY} STATS_BACKEND=sqlite3 STATS_HASH=xxh STATS_KEY=${KEY_PVT1} STATSDIR=/does/not/exist - ${result} = Run Rspamc -h ${LOCAL_ADDR}:${PORT_CONTROLLER} learn_spam ${MESSAGE} - Follow Rspamd Log - Should Not Contain ${result.stdout} success = true - [Teardown] Generic Teardown - -Sqlite Learn - Empty part - [Setup] Statistics Setup @{ALIASES} STATS_BACKEND=sqlite3 STATS_HASH=xxh STATS_KEY=${KEY_PVT1} - Set Test Variable ${MESSAGE} ${TESTDIR}/messages/empty_part.eml - ${result} = Run Rspamc -h ${LOCAL_ADDR}:${PORT_CONTROLLER} learn_spam ${MESSAGE} - Check Rspamc ${result} - ${result} = Scan Message With Rspamc ${MESSAGE} - Check Rspamc ${result} BAYES_SPAM - [Teardown] Generic Teardown - -Sqlite Relearn - [Setup] Statistics Setup @{ALIASES} STATS_BACKEND=sqlite3 STATS_HASH=xxh STATS_KEY=${KEY_PVT1} - ${result} = Run Rspamc -h ${LOCAL_ADDR}:${PORT_CONTROLLER} learn_spam ${MESSAGE} - Check Rspamc ${result} - ${result} = Scan Message With Rspamc ${MESSAGE} - Check Rspamc ${result} BAYES_SPAM - ${result} = Run Rspamc -h ${LOCAL_ADDR}:${PORT_CONTROLLER} learn_ham ${MESSAGE} - Check Rspamc ${result} - ${result} = Scan Message With Rspamc ${MESSAGE} - Check Rspamc ${result} BAYES_HAM - [Teardown] Generic Teardown - -Mmap Learn - [Setup] Statistics Setup @{ALIASES} STATS_BACKEND=mmap STATS_HASH=compat STATS_KEY=${KEY_PVT1} - ${result} = Run Rspamc -h ${LOCAL_ADDR}:${PORT_CONTROLLER} learn_spam ${MESSAGE} - Check Rspamc ${result} - ${result} = Scan Message With Rspamc ${MESSAGE} - Check Rspamc ${result} BAYES_SPAM - [Teardown] Generic Teardown - -Mmap Relearn - [Setup] Statistics Setup @{ALIASES} STATS_BACKEND=mmap STATS_HASH=compat STATS_KEY=${KEY_PVT1} - ${result} = Run Rspamc -h ${LOCAL_ADDR}:${PORT_CONTROLLER} learn_spam ${MESSAGE} - Check Rspamc ${result} - ${result} = Scan Message With Rspamc ${MESSAGE} - Check Rspamc ${result} BAYES_SPAM - ${result} = Run Rspamc -h ${LOCAL_ADDR}:${PORT_CONTROLLER} learn_ham ${MESSAGE} - Check Rspamc ${result} - ${result} = Scan Message With Rspamc ${MESSAGE} - Check Rspamc ${result} BAYES_HAM - [Teardown] Generic Teardown diff --git a/test/functional/cases/statistics/compat-keyed.robot b/test/functional/cases/statistics/compat-keyed.robot new file mode 100644 index 000000000..054978745 --- /dev/null +++ b/test/functional/cases/statistics/compat-keyed.robot @@ -0,0 +1,19 @@ +*** Settings *** +Suite Setup Statistics Setup +Suite Teardown Statistics Teardown +Resource lib.robot + +*** Variables *** +${STATS_BACKEND} mmap +${STATS_HASH} hash = "compat"; +${STATS_KEY} key = "${KEY_PVT1}"; + +*** Test Cases *** +Learn + Learn Test + +Relearn + Relearn Test + +Empty Part + Empty Part Test diff --git a/test/functional/cases/statistics/compat-plain.robot b/test/functional/cases/statistics/compat-plain.robot new file mode 100644 index 000000000..246d5a5d5 --- /dev/null +++ b/test/functional/cases/statistics/compat-plain.robot @@ -0,0 +1,18 @@ +*** Settings *** +Suite Setup Statistics Setup +Suite Teardown Statistics Teardown +Resource lib.robot + +*** Variables *** +${STATS_BACKEND} mmap +${STATS_HASH} hash = "compat"; + +*** Test Cases *** +Learn + Learn Test + +Relearn + Relearn Test + +Empty Part + Empty Part Test diff --git a/test/functional/cases/statistics/lib.robot b/test/functional/cases/statistics/lib.robot new file mode 100644 index 000000000..f710a87d5 --- /dev/null +++ b/test/functional/cases/statistics/lib.robot @@ -0,0 +1,59 @@ +*** Settings *** +Library ${TESTDIR}/lib/rspamd.py +Resource ${TESTDIR}/lib/rspamd.robot +Variables ${TESTDIR}/lib/vars.py + +*** Variables *** +${CONFIG} ${TESTDIR}/configs/stats.conf +${MESSAGE} ${TESTDIR}/messages/spam_message.eml +${REDIS_SCOPE} Suite +${REDIS_SERVER} ${EMPTY} +${RSPAMD_SCOPE} Suite +${STATS_HASH} ${EMPTY} +${STATS_KEY} ${EMPTY} +${STATS_PATH_CACHE} path = "\${TMPDIR}/bayes-cache.sqlite"; +${STATS_PATH_HAM} path = "\${TMPDIR}/bayes-ham.sqlite"; +${STATS_PATH_SPAM} path = "\${TMPDIR}/bayes-spam.sqlite"; + +*** Keywords *** +Broken Learn Test + ${result} = Run Rspamc -h ${LOCAL_ADDR}:${PORT_CONTROLLER} learn_spam ${MESSAGE} + Follow Rspamd Log + Should Not Contain ${result.stdout} success = true + Should Not Equal As Integers ${result.rc} 0 + +Empty Part Test + Set Test Variable ${MESSAGE} ${TESTDIR}/messages/empty_part.eml + ${result} = Run Rspamc -h ${LOCAL_ADDR}:${PORT_CONTROLLER} learn_spam ${MESSAGE} + Check Rspamc ${result} + ${result} = Scan Message With Rspamc ${MESSAGE} + Check Rspamc ${result} BAYES_SPAM + +Learn Test + Set Suite Variable ${RSPAMD_STATS_LEARNTEST} 0 + ${result} = Run Rspamc -h ${LOCAL_ADDR}:${PORT_CONTROLLER} learn_spam ${MESSAGE} + Check Rspamc ${result} + ${result} = Scan Message With Rspamc ${MESSAGE} + Check Rspamc ${result} BAYES_SPAM + Set Suite Variable ${RSPAMD_STATS_LEARNTEST} 1 + +Relearn Test + Run Keyword If ${RSPAMD_STATS_LEARNTEST} == 0 Fail "Learn test was not run" + ${result} = Run Rspamc -h ${LOCAL_ADDR}:${PORT_CONTROLLER} learn_ham ${MESSAGE} + Check Rspamc ${result} + ${result} = Scan Message With Rspamc ${MESSAGE} + Check Rspamc ${result} BAYES_HAM + +Redis Statistics Setup + Generic Setup + Run Redis + +Redis Statistics Teardown + Generic Teardown + Shutdown Process ${REDIS_PID} + +Statistics Setup + Generic Setup STATS_PATH_CACHE STATS_PATH_HAM STATS_PATH_SPAM + +Statistics Teardown + Generic Teardown diff --git a/test/functional/cases/statistics/redis-keyed-siphash.robot b/test/functional/cases/statistics/redis-keyed-siphash.robot new file mode 100644 index 000000000..f598915cd --- /dev/null +++ b/test/functional/cases/statistics/redis-keyed-siphash.robot @@ -0,0 +1,20 @@ +*** Settings *** +Suite Setup Redis Statistics Setup +Suite Teardown Redis Statistics Teardown +Resource lib.robot + +*** Variables *** +${REDIS_SERVER} servers = "${LOCAL_ADDR}:${REDIS_PORT}" +${STATS_BACKEND} redis +${STATS_HASH} hash = "siphash"; +${STATS_KEY} key = "${KEY_PVT1}"; + +*** Test Cases *** +Learn + Learn Test + +Relearn + Relearn Test + +Empty Part + Empty Part Test diff --git a/test/functional/cases/statistics/redis-keyed-xxhash.robot b/test/functional/cases/statistics/redis-keyed-xxhash.robot new file mode 100644 index 000000000..ca87895c5 --- /dev/null +++ b/test/functional/cases/statistics/redis-keyed-xxhash.robot @@ -0,0 +1,20 @@ +*** Settings *** +Suite Setup Redis Statistics Setup +Suite Teardown Redis Statistics Teardown +Resource lib.robot + +*** Variables *** +${REDIS_SERVER} servers = "${LOCAL_ADDR}:${REDIS_PORT}" +${STATS_BACKEND} redis +${STATS_HASH} hash = "xxhash"; +${STATS_KEY} key = "${KEY_PVT1}"; + +*** Test Cases *** +Learn + Learn Test + +Relearn + Relearn Test + +Empty Part + Empty Part Test diff --git a/test/functional/cases/statistics/redis-plain-siphash.robot b/test/functional/cases/statistics/redis-plain-siphash.robot new file mode 100644 index 000000000..2946e8903 --- /dev/null +++ b/test/functional/cases/statistics/redis-plain-siphash.robot @@ -0,0 +1,19 @@ +*** Settings *** +Suite Setup Redis Statistics Setup +Suite Teardown Redis Statistics Teardown +Resource lib.robot + +*** Variables *** +${REDIS_SERVER} servers = "${LOCAL_ADDR}:${REDIS_PORT}" +${STATS_BACKEND} redis +${STATS_HASH} hash = "siphash"; + +*** Test Cases *** +Learn + Learn Test + +Relearn + Relearn Test + +Empty Part + Empty Part Test diff --git a/test/functional/cases/statistics/redis-plain-xxhash.robot b/test/functional/cases/statistics/redis-plain-xxhash.robot new file mode 100644 index 000000000..de69465f0 --- /dev/null +++ b/test/functional/cases/statistics/redis-plain-xxhash.robot @@ -0,0 +1,19 @@ +*** Settings *** +Suite Setup Redis Statistics Setup +Suite Teardown Redis Statistics Teardown +Resource lib.robot + +*** Variables *** +${REDIS_SERVER} servers = "${LOCAL_ADDR}:${REDIS_PORT}" +${STATS_BACKEND} redis +${STATS_HASH} hash = "xxhash"; + +*** Test Cases *** +Learn + Learn Test + +Relearn + Relearn Test + +Empty Part + Empty Part Test diff --git a/test/functional/cases/statistics/sqlite-broken-stats-dir.robot b/test/functional/cases/statistics/sqlite-broken-stats-dir.robot new file mode 100644 index 000000000..57d75c294 --- /dev/null +++ b/test/functional/cases/statistics/sqlite-broken-stats-dir.robot @@ -0,0 +1,15 @@ +*** Settings *** +Suite Setup Generic Setup +Suite Teardown Generic Teardown +Resource ${TESTDIR}/lib/rspamd.robot +Resource lib.robot + +*** Variables *** +${STATS_BACKEND} sqlite3 +${STATS_PATH_CACHE} path = "/does/not/exist/bayes-cache.sqlite"; +${STATS_PATH_HAM} path = "/does/not/exist/bayes-ham.sqlite"; +${STATS_PATH_SPAM} path = "/does/not/exist/bayes-spam.sqlite"; + +*** Test Cases *** +Broken Stats Directory + Broken Learn Test diff --git a/test/functional/cases/statistics/sqlite-keyed-siphash.robot b/test/functional/cases/statistics/sqlite-keyed-siphash.robot new file mode 100644 index 000000000..8b9661a9a --- /dev/null +++ b/test/functional/cases/statistics/sqlite-keyed-siphash.robot @@ -0,0 +1,19 @@ +*** Settings *** +Suite Setup Statistics Setup +Suite Teardown Statistics Teardown +Resource lib.robot + +*** Variables *** +${STATS_BACKEND} sqlite3 +${STATS_HASH} hash = "siphash"; +${STATS_KEY} key = "${KEY_PVT1}"; + +*** Test Cases *** +Learn + Learn Test + +Relearn + Relearn Test + +Empty Part + Empty Part Test diff --git a/test/functional/cases/statistics/sqlite-keyed-xxhash.robot b/test/functional/cases/statistics/sqlite-keyed-xxhash.robot new file mode 100644 index 000000000..7a51e93ae --- /dev/null +++ b/test/functional/cases/statistics/sqlite-keyed-xxhash.robot @@ -0,0 +1,19 @@ +*** Settings *** +Suite Setup Statistics Setup +Suite Teardown Statistics Teardown +Resource lib.robot + +*** Variables *** +${STATS_BACKEND} sqlite3 +${STATS_HASH} hash = "xxhash"; +${STATS_KEY} key = "${KEY_PVT1}"; + +*** Test Cases *** +Learn + Learn Test + +Relearn + Relearn Test + +Empty Part + Empty Part Test diff --git a/test/functional/cases/statistics/sqlite-plain-siphash.robot b/test/functional/cases/statistics/sqlite-plain-siphash.robot new file mode 100644 index 000000000..2f88e0a95 --- /dev/null +++ b/test/functional/cases/statistics/sqlite-plain-siphash.robot @@ -0,0 +1,18 @@ +*** Settings *** +Suite Setup Statistics Setup +Suite Teardown Statistics Teardown +Resource lib.robot + +*** Variables *** +${STATS_BACKEND} sqlite3 +${STATS_HASH} hash = "siphash"; + +*** Test Cases *** +Learn + Learn Test + +Relearn + Relearn Test + +Empty Part + Empty Part Test diff --git a/test/functional/cases/statistics/sqlite-plain-xxhash.robot b/test/functional/cases/statistics/sqlite-plain-xxhash.robot new file mode 100644 index 000000000..dd8198222 --- /dev/null +++ b/test/functional/cases/statistics/sqlite-plain-xxhash.robot @@ -0,0 +1,18 @@ +*** Settings *** +Suite Setup Statistics Setup +Suite Teardown Statistics Teardown +Resource lib.robot + +*** Variables *** +${STATS_BACKEND} sqlite3 +${STATS_HASH} hash = "xxhash"; + +*** Test Cases *** +Learn + Learn Test + +Relearn + Relearn Test + +Empty Part + Empty Part Test diff --git a/test/functional/configs/fuzzy.conf b/test/functional/configs/fuzzy.conf new file mode 100644 index 000000000..9a4ed32d3 --- /dev/null +++ b/test/functional/configs/fuzzy.conf @@ -0,0 +1,74 @@ +options = { + filters = "fuzzy_check"; + pidfile = "${TMPDIR}/rspamd.pid" + control_socket = "${TMPDIR}/rspamd.sock mode=0600"; +} +logging = { + type = "file", + level = "debug" + filename = "${TMPDIR}/rspamd.log" +} +metric = { + name = "default", + actions = { + reject = 100500, + } + unknown_weight = 1 + symbol { + weight = 0.0; + name = "R_TEST_FUZZY_UNKNOWN"; + } + symbol { + weight = 10.0; + name = "${FLAG1_SYMBOL}"; + } + symbol { + weight = -1.0; + name = "${FLAG2_SYMBOL}"; + } +} + +worker { + type = normal + bind_socket = "${LOCAL_ADDR}:${PORT_NORMAL}"; + count = 1 +} + +worker { + type = controller + bind_socket = "${LOCAL_ADDR}:${PORT_CONTROLLER}"; + count = 1 + secure_ip = ["${LOCAL_ADDR}"]; +} + +worker { + bind_socket = "${LOCAL_ADDR}:${PORT_FUZZY}"; + type = "fuzzy"; + hashfile = "${TMPDIR}/fuzzy.db"; + allow_update = ["${LOCAL_ADDR}"]; + ${SETTINGS_FUZZY_WORKER} +} + +fuzzy_check { + min_bytes = 100; + + rule { + servers = "${LOCAL_ADDR}:${PORT_FUZZY}"; + symbol = "R_TEST_FUZZY"; + max_score = 10.0; + mime_types = ["application/*"]; + read_only = false; + skip_unknown = true; + ${SETTINGS_FUZZY_CHECK} + fuzzy_map = { + ${FLAG1_SYMBOL} { + max_score = 10.0; + flag = ${FLAG1_NUMBER}; + } + ${FLAG2_SYMBOL} { + max_score = 1.0; + flag = ${FLAG2_NUMBER}; + } + } + } +} diff --git a/test/functional/configs/redis-server.conf b/test/functional/configs/redis-server.conf new file mode 100644 index 000000000..2973ae8d9 --- /dev/null +++ b/test/functional/configs/redis-server.conf @@ -0,0 +1,4 @@ +bind ${LOCAL_ADDR} +daemonize yes +pidfile ${TMPDIR}/redis.pid +port ${REDIS_PORT} diff --git a/test/functional/configs/stats.conf b/test/functional/configs/stats.conf index 144b7a886..216d7d8f3 100644 --- a/test/functional/configs/stats.conf +++ b/test/functional/configs/stats.conf @@ -21,8 +21,8 @@ worker { bind_socket = ${LOCAL_ADDR}:${PORT_NORMAL} count = 1 keypair { - pubkey = "${KEY_PUBLIC}"; - privkey = "${KEY_PRIVATE}"; + pubkey = "${KEY_PUB1}"; + privkey = "${KEY_PVT1}"; } } worker { @@ -30,8 +30,8 @@ worker { bind_socket = ${LOCAL_ADDR}:${PORT_CONTROLLER} count = 1 keypair { - pubkey = "${KEY_PUBLIC}"; - privkey = "${KEY_PRIVATE}"; + pubkey = "${KEY_PUB1}"; + privkey = "${KEY_PVT1}"; } secure_ip = ["127.0.0.1", "::1"]; } @@ -40,26 +40,26 @@ classifier { languages_enabled = true; tokenizer { name = "osb"; - hash = ${STATS_HASH} - key = ${STATS_KEY} + ${STATS_HASH} + ${STATS_KEY} } backend = ${STATS_BACKEND} statfile { symbol = BAYES_SPAM; - path = ${STATSDIR}/rspamd-bats-bayes.spam; + ${STATS_PATH_SPAM} size = 1M; - write_servers = ${REDIS_SERVER} + ${REDIS_SERVER} } statfile { symbol = BAYES_HAM; - path = ${STATSDIR}/rspamd-bats-bayes.ham; + ${STATS_PATH_HAM} size = 1M; - write_servers = ${REDIS_SERVER} + ${REDIS_SERVER} } cache { - path = ${STATSDIR}/rspamd-bats-cache.sqlite; - write_servers = ${REDIS_SERVER} + ${STATS_PATH_CACHE} + ${REDIS_SERVER} } } diff --git a/test/functional/configs/trivial.conf b/test/functional/configs/trivial.conf index 03fb11e0f..9bc07fa5c 100644 --- a/test/functional/configs/trivial.conf +++ b/test/functional/configs/trivial.conf @@ -21,7 +21,7 @@ worker { bind_socket = ${LOCAL_ADDR}:${PORT_NORMAL} count = 1 keypair { - pubkey = "${KEY_PUBLIC}"; - privkey = "${KEY_PRIVATE}"; + pubkey = "${KEY_PUB1}"; + privkey = "${KEY_PVT1}"; } } diff --git a/test/functional/lib/rspamd.py b/test/functional/lib/rspamd.py index 93ec4c430..10caab3ca 100644 --- a/test/functional/lib/rspamd.py +++ b/test/functional/lib/rspamd.py @@ -29,12 +29,6 @@ def get_test_directory(): def make_temporary_directory(): return tempfile.mkdtemp() -def populate_rspamd_config(template_file, temporary_dir, **config): - t = string.Template(open(template_file).read()) - f = open("%s/rspamd.conf" % temporary_dir, "w") - f.write(t.safe_substitute(config)) - f.close() - def process_should_exist(pid): pid = int(pid) os.kill(pid, 0) @@ -75,7 +69,7 @@ def update_dictionary(a, b): a.update(b) return a -def shutdown_rspamd(pid): +def shutdown_process(pid): pid = int(pid) process_should_exist(pid) i = 0 diff --git a/test/functional/lib/rspamd.robot b/test/functional/lib/rspamd.robot index b6180d40d..ded537e03 100644 --- a/test/functional/lib/rspamd.robot +++ b/test/functional/lib/rspamd.robot @@ -1,7 +1,7 @@ *** Settings *** -Library Collections -Library OperatingSystem -Library Process +Library Collections +Library OperatingSystem +Library Process *** Keywords *** Check Rspamc @@ -32,42 +32,64 @@ Follow Rspamd Log ... ELSE Fail 'RSPAMD_SCOPE must be Test or Suite' Generic Setup - ${TMPDIR} ${RSPAMD_PID} ${RSPAMD_LOGPOS} = Run Rspamd + [Arguments] @{vargs} + ${TMPDIR} ${RSPAMD_PID} ${RSPAMD_LOGPOS} = Run Rspamd @{vargs} Run Keyword If '${RSPAMD_SCOPE}' == 'Test' Export Rspamd Vars To Test ${TMPDIR} ${RSPAMD_LOGPOS} ${RSPAMD_PID} ... ELSE IF '${RSPAMD_SCOPE}' == 'Suite' Export Rspamd Vars To Suite ${TMPDIR} ${RSPAMD_LOGPOS} ${RSPAMD_PID} ... ELSE Fail 'RSPAMD_SCOPE must be Test or Suite' Generic Teardown - Shutdown Rspamd ${RSPAMD_PID} - Cleanup Temporary Directory ${TMPDIR} + Shutdown Process ${RSPAMD_PID} + Cleanup Temporary Directory ${TMPDIR} Log Logs - [Arguments] ${logfile} ${position} - ${the_log} ${position} = Read Log From Position ${logfile} ${position} - Log ${the_log} - [Return] ${position} + [Arguments] ${logfile} ${position} + ${the_log} ${position} = Read Log From Position ${logfile} ${position} + Log ${the_log} + [Return] ${position} + +Run Redis + ${template} = Get File ${TESTDIR}/configs/redis-server.conf + ${config} = Replace Variables ${template} + Create File ${TMPDIR}/redis-server.conf ${config} + ${result} = Run Process redis-server ${TMPDIR}/redis-server.conf + Should Be Equal As Integers ${result.rc} 0 + ${REDIS_PID} = Get File ${TMPDIR}/redis.pid + Run Keyword If '${REDIS_SCOPE}' == 'Test' Set Test Variable ${REDIS_PID} + ... ELSE IF '${REDIS_SCOPE}' == 'Suite' Set Suite Variable ${REDIS_PID} Run Rspamc - [Arguments] @{args} - ${result} = Run Process ${RSPAMC} @{args} - [Return] ${result} + [Arguments] @{args} + ${result} = Run Process ${RSPAMC} @{args} + [Return] ${result} Run Rspamd - [Arguments] @{args} &{kw} - ${tmpdir} = Make Temporary Directory - Set Directory Ownership ${tmpdir} ${RSPAMD_USER} ${RSPAMD_GROUP} - Set To Dictionary ${RSPAMD_KEYWORDS} TMPDIR=${tmpdir} - Update Dictionary ${RSPAMD_KEYWORDS} ${kw} - :FOR ${i} IN @{args} - \ Set To Dictionary ${RSPAMD_KEYWORDS} ${i} ${tmpdir} - Populate Rspamd Config ${CONFIG} ${tmpdir} &{RSPAMD_KEYWORDS} - ${result} = Run Process ${RSPAMD} -u ${RSPAMD_USER} -g ${RSPAMD_GROUP} -c ${tmpdir}/rspamd.conf - ${rspamd_logpos} = Log Logs ${tmpdir}/rspamd.log 0 - Should Be Equal As Integers ${result.rc} 0 - ${rspamd_pid} = Get File ${tmpdir}/rspamd.pid - [Return] ${tmpdir} ${rspamd_pid} ${rspamd_logpos} + [Arguments] @{vargs} + ${TMPDIR} = Make Temporary Directory + Set Directory Ownership ${TMPDIR} ${RSPAMD_USER} ${RSPAMD_GROUP} + ${template} = Get File ${CONFIG} + : FOR ${i} IN @{vargs} + \ ${newvalue} = Replace Variables ${${i}} + \ Set Suite Variable ${${i}} ${newvalue} + \ Run Keyword If '${RSPAMD_SCOPE}' == 'Test' Set Test Variable ${${i}} ${newvalue} + \ ... ELSE IF '${RSPAMD_SCOPE}' == 'Suite' Set Suite Variable ${${i}} ${newvalue} + \ ... ELSE Fail 'RSPAMD_SCOPE must be Test or Suite' + ${config} = Replace Variables ${template} + Log ${config} + Create File ${TMPDIR}/rspamd.conf ${config} + ${result} = Run Process ${RSPAMD} -u ${RSPAMD_USER} -g ${RSPAMD_GROUP} + ... -c ${TMPDIR}/rspamd.conf + ${rspamd_logpos} = Log Logs ${TMPDIR}/rspamd.log 0 + Should Be Equal As Integers ${result.rc} 0 + ${rspamd_pid} = Get File ${TMPDIR}/rspamd.pid + [Return] ${TMPDIR} ${rspamd_pid} ${rspamd_logpos} Scan Message With Rspamc - [Arguments] ${msg_file} - ${result} = Run Rspamc -p -h ${LOCAL_ADDR}:${PORT_NORMAL} ${msg_file} - [Return] ${result} + [Arguments] ${msg_file} + ${result} = Run Rspamc -p -h ${LOCAL_ADDR}:${PORT_NORMAL} ${msg_file} + [Return] ${result} + +Sync Fuzzy Storage + ${result} = Run Process ${RSPAMADM} control -s ${TMPDIR}/rspamd.sock fuzzy_sync + Log ${result.stdout} + Follow Rspamd Log diff --git a/test/functional/lib/vars.py b/test/functional/lib/vars.py index 78151bd37..316669911 100644 --- a/test/functional/lib/vars.py +++ b/test/functional/lib/vars.py @@ -2,6 +2,8 @@ KEY_PVT1 = 'ekd3x36tfa5gd76t6pa8hqif3ott7n1siuux68exbkk7ukscte9y' KEY_PUB1 = 'm8kneubpcjsb8sbsoj7jy7azj9fdd3xmj63txni86a8ye9ncomny' LOCAL_ADDR = 'localhost' PORT_CONTROLLER = 56790 +PORT_FUZZY = 56791 PORT_NORMAL = 56789 +REDIS_PORT = 56379 RSPAMD_GROUP = 'nogroup' RSPAMD_USER = 'nobody' |