1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
|
/*-
* Copyright 2016 Vsevolod Stakhov
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "config.h"
#include "rspamd.h"
#include "shingles.h"
#include "ottery.h"
#include <math.h>
static void
generate_random_string (char *begin, size_t len)
{
gsize i;
for (i = 0; i < len; i ++) {
begin[i] = ottery_rand_range ('z' - 'a') + 'a';
}
}
static GArray *
generate_fuzzy_words (gsize cnt, gsize max_len)
{
GArray *res;
gsize i, wlen;
rspamd_ftok_t w;
char *t;
res = g_array_sized_new (FALSE, FALSE, sizeof (rspamd_ftok_t), cnt);
for (i = 0; i < cnt; i ++) {
wlen = ottery_rand_range (max_len) + 1;
w.len = wlen;
t = g_malloc (wlen);
generate_random_string (t, wlen);
w.begin = t;
g_array_append_val (res, w);
}
return res;
}
static void
permute_vector (GArray *in, gdouble prob)
{
gsize i, total = 0;
rspamd_ftok_t *w;
for (i = 0; i < in->len; i ++) {
if (ottery_rand_unsigned () <= G_MAXUINT * prob) {
w = &g_array_index (in, rspamd_ftok_t, i);
generate_random_string ((gchar *)w->begin, w->len);
total ++;
}
}
msg_debug ("generated %z permutations of %ud words", total, in->len);
}
static void
free_fuzzy_words (GArray *ar)
{
gsize i;
rspamd_ftok_t *w;
for (i = 0; i < ar->len; i ++) {
w = &g_array_index (ar, rspamd_ftok_t, i);
g_free ((gpointer)w->begin);
}
}
static void
test_case (gsize cnt, gsize max_len, gdouble perm_factor,
enum rspamd_shingle_alg alg)
{
GArray *input;
struct rspamd_shingle *sgl, *sgl_permuted;
gdouble res;
guchar key[16];
gdouble ts1, ts2;
ottery_rand_bytes (key, sizeof (key));
input = generate_fuzzy_words (cnt, max_len);
ts1 = rspamd_get_ticks ();
sgl = rspamd_shingles_generate (input, key, NULL,
rspamd_shingles_default_filter, NULL, alg);
ts2 = rspamd_get_ticks ();
permute_vector (input, perm_factor);
sgl_permuted = rspamd_shingles_generate (input, key, NULL,
rspamd_shingles_default_filter, NULL, alg);
res = rspamd_shingles_compare (sgl, sgl_permuted);
msg_info ("%d (%z words of %z max len, %.2f perm factor):"
" percentage of common shingles: %.3f, generate time: %.4f sec",
alg, cnt, max_len, perm_factor, res, ts2 - ts1);
g_assert_cmpfloat (fabs ((1.0 - res) - sqrt (perm_factor)), <=, 0.25);
free_fuzzy_words (input);
g_free (sgl);
g_free (sgl_permuted);
}
void
rspamd_shingles_test_func (void)
{
enum rspamd_shingle_alg alg = RSPAMD_SHINGLES_OLD;
for (alg = RSPAMD_SHINGLES_OLD; alg <= RSPAMD_SHINGLES_MUMHASH; alg ++) {
test_case (200, 10, 0.1, alg);
test_case (500, 20, 0.01, alg);
test_case (5000, 20, 0.01, alg);
test_case (5000, 15, 0, alg);
test_case (5000, 30, 1.0, alg);
test_case (50000, 30, 0.02, alg);
test_case (50000, 5, 0.02, alg);
}
}
|
