123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845 |
- /*
- * Copyright 2023 Vsevolod Stakhov
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
- #include "map_helpers.h"
- #include "map_private.h"
- #include "khash.h"
- #include "radix.h"
- #include "rspamd.h"
- #include "cryptobox.h"
- #include "mempool_vars_internal.h"
- #include "contrib/fastutf8/fastutf8.h"
- #include "contrib/cdb/cdb.h"
-
- #ifdef WITH_HYPERSCAN
- #include "hs.h"
- #include "hyperscan_tools.h"
- #endif
- #ifndef WITH_PCRE2
- #include <pcre.h>
- #else
- #include <pcre2.h>
- #endif
-
-
- static const uint64_t map_hash_seed = 0xdeadbabeULL;
- static const char *const hash_fill = "1";
-
- struct rspamd_map_helper_value {
- gsize hits;
- gconstpointer key;
- char value[]; /* Null terminated */
- };
-
- #define rspamd_map_ftok_hash(t) (rspamd_icase_hash((t).begin, (t).len, rspamd_hash_seed()))
- #define rspamd_map_ftok_equal(a, b) ((a).len == (b).len && rspamd_lc_cmp((a).begin, (b).begin, (a).len) == 0)
-
- KHASH_INIT(rspamd_map_hash, rspamd_ftok_t,
- struct rspamd_map_helper_value *, true,
- rspamd_map_ftok_hash, rspamd_map_ftok_equal);
-
- struct rspamd_radix_map_helper {
- rspamd_mempool_t *pool;
- khash_t(rspamd_map_hash) * htb;
- radix_compressed_t *trie;
- struct rspamd_map *map;
- rspamd_cryptobox_fast_hash_state_t hst;
- };
-
- struct rspamd_hash_map_helper {
- rspamd_mempool_t *pool;
- khash_t(rspamd_map_hash) * htb;
- struct rspamd_map *map;
- rspamd_cryptobox_fast_hash_state_t hst;
- };
-
- struct rspamd_cdb_map_helper {
- GQueue cdbs;
- struct rspamd_map *map;
- rspamd_cryptobox_fast_hash_state_t hst;
- gsize total_size;
- };
-
- struct rspamd_regexp_map_helper {
- rspamd_cryptobox_hash_state_t hst;
- unsigned char re_digest[rspamd_cryptobox_HASHBYTES];
- rspamd_mempool_t *pool;
- struct rspamd_map *map;
- GPtrArray *regexps;
- GPtrArray *values;
- khash_t(rspamd_map_hash) * htb;
- enum rspamd_regexp_map_flags map_flags;
- #ifdef WITH_HYPERSCAN
- rspamd_hyperscan_t *hs_db;
- hs_scratch_t *hs_scratch;
- char **patterns;
- int *flags;
- int *ids;
- #endif
- };
-
- /**
- * FSM for parsing lists
- */
-
- #define MAP_STORE_KEY \
- do { \
- while (g_ascii_isspace(*c) && p > c) { c++; } \
- key = g_malloc(p - c + 1); \
- rspamd_strlcpy(key, c, p - c + 1); \
- stripped_key = g_strstrip(key); \
- } while (0)
-
- #define MAP_STORE_VALUE \
- do { \
- while (g_ascii_isspace(*c) && p > c) { c++; } \
- value = g_malloc(p - c + 1); \
- rspamd_strlcpy(value, c, p - c + 1); \
- stripped_value = g_strstrip(value); \
- } while (0)
-
- char *
- rspamd_parse_kv_list(
- char *chunk,
- int len,
- struct map_cb_data *data,
- rspamd_map_insert_func func,
- const char *default_value,
- gboolean final)
- {
- enum {
- map_skip_spaces_before_key = 0,
- map_read_key,
- map_read_key_quoted,
- map_read_key_slashed,
- map_skip_spaces_after_key,
- map_backslash_quoted,
- map_backslash_slashed,
- map_read_key_after_slash,
- map_read_value,
- map_read_comment_start,
- map_skip_comment,
- map_read_eol,
- };
-
- char *c, *p, *key = NULL, *value = NULL, *stripped_key, *stripped_value, *end;
- struct rspamd_map *map = data->map;
- unsigned int line_number = 0;
-
- p = chunk;
- c = p;
- end = p + len;
-
- while (p < end) {
- switch (data->state) {
- case map_skip_spaces_before_key:
- if (g_ascii_isspace(*p)) {
- p++;
- }
- else {
- if (*p == '"') {
- p++;
- c = p;
- data->state = map_read_key_quoted;
- }
- else if (*p == '/') {
- /* Note that c is on '/' here as '/' is a part of key */
- c = p;
- p++;
- data->state = map_read_key_slashed;
- }
- else {
- c = p;
- data->state = map_read_key;
- }
- }
- break;
- case map_read_key:
- /* read key */
- /* Check here comments, eol and end of buffer */
- if (*p == '#' && (p == c || *(p - 1) != '\\')) {
- if (p - c > 0) {
- /* Store a single key */
- MAP_STORE_KEY;
- func(data->cur_data, stripped_key, default_value);
- msg_debug_map("insert key only pair: %s -> %s; line: %d",
- stripped_key, default_value, line_number);
- g_free(key);
- }
-
- key = NULL;
- data->state = map_read_comment_start;
- }
- else if (*p == '\r' || *p == '\n') {
- if (p - c > 0) {
- /* Store a single key */
- MAP_STORE_KEY;
- func(data->cur_data, stripped_key, default_value);
- msg_debug_map("insert key only pair: %s -> %s; line: %d",
- stripped_key, default_value, line_number);
- g_free(key);
- }
-
- data->state = map_read_eol;
- key = NULL;
- }
- else if (g_ascii_isspace(*p)) {
- if (p - c > 0) {
- MAP_STORE_KEY;
- data->state = map_skip_spaces_after_key;
- }
- else {
- msg_err_map("empty or invalid key found on line %d", line_number);
- data->state = map_skip_comment;
- }
- }
- else {
- p++;
- }
- break;
- case map_read_key_quoted:
- if (*p == '\\') {
- data->state = map_backslash_quoted;
- p++;
- }
- else if (*p == '"') {
- /* Allow empty keys in this case */
- if (p - c >= 0) {
- MAP_STORE_KEY;
- data->state = map_skip_spaces_after_key;
- }
- else {
- g_assert_not_reached();
- }
- p++;
- }
- else {
- p++;
- }
- break;
- case map_read_key_slashed:
- if (*p == '\\') {
- data->state = map_backslash_slashed;
- p++;
- }
- else if (*p == '/') {
- /* Allow empty keys in this case */
- if (p - c >= 0) {
- data->state = map_read_key_after_slash;
- }
- else {
- g_assert_not_reached();
- }
- }
- else {
- p++;
- }
- break;
- case map_read_key_after_slash:
- /*
- * This state is equal to reading of key but '/' is not
- * treated specially
- */
- if (*p == '#') {
- if (p - c > 0) {
- /* Store a single key */
- MAP_STORE_KEY;
- func(data->cur_data, stripped_key, default_value);
- msg_debug_map("insert key only pair: %s -> %s; line: %d",
- stripped_key, default_value, line_number);
- g_free(key);
- key = NULL;
- }
-
- data->state = map_read_comment_start;
- }
- else if (*p == '\r' || *p == '\n') {
- if (p - c > 0) {
- /* Store a single key */
- MAP_STORE_KEY;
- func(data->cur_data, stripped_key, default_value);
-
- msg_debug_map("insert key only pair: %s -> %s; line: %d",
- stripped_key, default_value, line_number);
- g_free(key);
- key = NULL;
- }
-
- data->state = map_read_eol;
- key = NULL;
- }
- else if (g_ascii_isspace(*p)) {
- if (p - c > 0) {
- MAP_STORE_KEY;
- data->state = map_skip_spaces_after_key;
- }
- else {
- msg_err_map("empty or invalid key found on line %d", line_number);
- data->state = map_skip_comment;
- }
- }
- else {
- p++;
- }
- break;
- case map_backslash_quoted:
- p++;
- data->state = map_read_key_quoted;
- break;
- case map_backslash_slashed:
- p++;
- data->state = map_read_key_slashed;
- break;
- case map_skip_spaces_after_key:
- if (*p == ' ' || *p == '\t') {
- p++;
- }
- else {
- c = p;
- data->state = map_read_value;
- }
- break;
- case map_read_value:
- if (key == NULL) {
- /* Ignore line */
- msg_err_map("empty or invalid key found on line %d", line_number);
- data->state = map_skip_comment;
- }
- else {
- if (*p == '#') {
- if (p - c > 0) {
- /* Store a single key */
- MAP_STORE_VALUE;
- func(data->cur_data, stripped_key, stripped_value);
- msg_debug_map("insert key value pair: %s -> %s; line: %d",
- stripped_key, stripped_value, line_number);
- g_free(key);
- g_free(value);
- key = NULL;
- value = NULL;
- }
- else {
- func(data->cur_data, stripped_key, default_value);
- msg_debug_map("insert key only pair: %s -> %s; line: %d",
- stripped_key, default_value, line_number);
- g_free(key);
- key = NULL;
- }
-
- data->state = map_read_comment_start;
- }
- else if (*p == '\r' || *p == '\n') {
- if (p - c > 0) {
- /* Store a single key */
- MAP_STORE_VALUE;
- func(data->cur_data, stripped_key, stripped_value);
- msg_debug_map("insert key value pair: %s -> %s",
- stripped_key, stripped_value);
- g_free(key);
- g_free(value);
- key = NULL;
- value = NULL;
- }
- else {
- func(data->cur_data, stripped_key, default_value);
- msg_debug_map("insert key only pair: %s -> %s",
- stripped_key, default_value);
- g_free(key);
- key = NULL;
- }
-
- data->state = map_read_eol;
- key = NULL;
- }
- else {
- p++;
- }
- }
- break;
- case map_read_comment_start:
- if (*p == '#') {
- data->state = map_skip_comment;
- p++;
- key = NULL;
- value = NULL;
- }
- else {
- g_assert_not_reached();
- }
- break;
- case map_skip_comment:
- if (*p == '\r' || *p == '\n') {
- data->state = map_read_eol;
- }
- else {
- p++;
- }
- break;
- case map_read_eol:
- /* Skip \r\n and whitespaces */
- if (*p == '\r' || *p == '\n') {
- if (*p == '\n') {
- /* We don't care about \r only line separators, they are too rare */
- line_number++;
- }
- p++;
- }
- else {
- data->state = map_skip_spaces_before_key;
- }
- break;
- default:
- g_assert_not_reached();
- break;
- }
- }
-
- if (final) {
- /* Examine the state */
- switch (data->state) {
- case map_read_key:
- case map_read_key_slashed:
- case map_read_key_quoted:
- case map_read_key_after_slash:
- if (p - c > 0) {
- /* Store a single key */
- MAP_STORE_KEY;
- func(data->cur_data, stripped_key, default_value);
- msg_debug_map("insert key only pair: %s -> %s",
- stripped_key, default_value);
- g_free(key);
- key = NULL;
- }
- break;
- case map_read_value:
- if (key == NULL) {
- /* Ignore line */
- msg_err_map("empty or invalid key found on line %d", line_number);
- data->state = map_skip_comment;
- }
- else {
- if (p - c > 0) {
- /* Store a single key */
- MAP_STORE_VALUE;
- func(data->cur_data, stripped_key, stripped_value);
- msg_debug_map("insert key value pair: %s -> %s",
- stripped_key, stripped_value);
- g_free(key);
- g_free(value);
- key = NULL;
- value = NULL;
- }
- else {
- func(data->cur_data, stripped_key, default_value);
- msg_debug_map("insert key only pair: %s -> %s",
- stripped_key, default_value);
- g_free(key);
- key = NULL;
- }
- }
- break;
- }
-
- data->state = map_skip_spaces_before_key;
- }
-
- return c;
- }
-
- /**
- * Radix tree helper function
- */
- void rspamd_map_helper_insert_radix(gpointer st, gconstpointer key, gconstpointer value)
- {
- struct rspamd_radix_map_helper *r = (struct rspamd_radix_map_helper *) st;
- struct rspamd_map_helper_value *val;
- gsize vlen;
- khiter_t k;
- gconstpointer nk;
- rspamd_ftok_t tok;
- int res;
- struct rspamd_map *map;
-
- map = r->map;
- tok.begin = key;
- tok.len = strlen(key);
-
- k = kh_get(rspamd_map_hash, r->htb, tok);
-
- if (k == kh_end(r->htb)) {
- nk = rspamd_mempool_strdup(r->pool, key);
- tok.begin = nk;
- k = kh_put(rspamd_map_hash, r->htb, tok, &res);
- }
- else {
- val = kh_value(r->htb, k);
-
- if (strcmp(value, val->value) == 0) {
- /* Same element, skip */
- return;
- }
- else {
- msg_warn_map("duplicate radix entry found for map %s: %s (old value: '%s', new: '%s')",
- map->name, key, val->value, value);
- }
-
- nk = kh_key(r->htb, k).begin;
- val->key = nk;
- kh_value(r->htb, k) = val;
-
- return; /* do not touch radix in case of exact duplicate */
- }
-
- vlen = strlen(value);
- val = rspamd_mempool_alloc0(r->pool, sizeof(*val) +
- vlen + 1);
- memcpy(val->value, value, vlen);
-
- nk = kh_key(r->htb, k).begin;
- val->key = nk;
- kh_value(r->htb, k) = val;
- rspamd_radix_add_iplist(key, ",", r->trie, val, FALSE,
- r->map->name);
- rspamd_cryptobox_fast_hash_update(&r->hst, nk, tok.len);
- }
-
- void rspamd_map_helper_insert_radix_resolve(gpointer st, gconstpointer key, gconstpointer value)
- {
- struct rspamd_radix_map_helper *r = (struct rspamd_radix_map_helper *) st;
- struct rspamd_map_helper_value *val;
- gsize vlen;
- khiter_t k;
- gconstpointer nk;
- rspamd_ftok_t tok;
- int res;
- struct rspamd_map *map;
-
- map = r->map;
-
- if (!key) {
- msg_warn_map("cannot insert NULL value in the map: %s",
- map->name);
- return;
- }
-
- tok.begin = key;
- tok.len = strlen(key);
-
- k = kh_get(rspamd_map_hash, r->htb, tok);
-
- if (k == kh_end(r->htb)) {
- nk = rspamd_mempool_strdup(r->pool, key);
- tok.begin = nk;
- k = kh_put(rspamd_map_hash, r->htb, tok, &res);
- }
- else {
- val = kh_value(r->htb, k);
-
- if (strcmp(value, val->value) == 0) {
- /* Same element, skip */
- return;
- }
- else {
- msg_warn_map("duplicate radix entry found for map %s: %s (old value: '%s', new: '%s')",
- map->name, key, val->value, value);
- }
-
- nk = kh_key(r->htb, k).begin;
- val->key = nk;
- kh_value(r->htb, k) = val;
-
- return; /* do not touch radix in case of exact duplicate */
- }
-
- vlen = strlen(value);
- val = rspamd_mempool_alloc0(r->pool, sizeof(*val) +
- vlen + 1);
- memcpy(val->value, value, vlen);
- nk = kh_key(r->htb, k).begin;
- val->key = nk;
- kh_value(r->htb, k) = val;
- rspamd_radix_add_iplist(key, ",", r->trie, val, TRUE,
- r->map->name);
- rspamd_cryptobox_fast_hash_update(&r->hst, nk, tok.len);
- }
-
- void rspamd_map_helper_insert_hash(gpointer st, gconstpointer key, gconstpointer value)
- {
- struct rspamd_hash_map_helper *ht = st;
- struct rspamd_map_helper_value *val;
- khiter_t k;
- gconstpointer nk;
- gsize vlen;
- int r;
- rspamd_ftok_t tok;
- struct rspamd_map *map;
-
- tok.begin = key;
- tok.len = strlen(key);
- map = ht->map;
-
- k = kh_get(rspamd_map_hash, ht->htb, tok);
-
- if (k == kh_end(ht->htb)) {
- nk = rspamd_mempool_strdup(ht->pool, key);
- tok.begin = nk;
- k = kh_put(rspamd_map_hash, ht->htb, tok, &r);
- }
- else {
- val = kh_value(ht->htb, k);
-
- if (strcmp(value, val->value) == 0) {
- /* Same element, skip */
- return;
- }
- else {
- msg_warn_map("duplicate hash entry found for map %s: %s (old value: '%s', new: '%s')",
- map->name, key, val->value, value);
- }
- }
-
- /* Null termination due to alloc0 */
- vlen = strlen(value);
- val = rspamd_mempool_alloc0(ht->pool, sizeof(*val) + vlen + 1);
- memcpy(val->value, value, vlen);
-
- tok = kh_key(ht->htb, k);
- nk = tok.begin;
- val->key = nk;
- kh_value(ht->htb, k) = val;
-
- rspamd_cryptobox_fast_hash_update(&ht->hst, nk, tok.len);
- }
-
- void rspamd_map_helper_insert_re(gpointer st, gconstpointer key, gconstpointer value)
- {
- struct rspamd_regexp_map_helper *re_map = st;
- struct rspamd_map *map;
- rspamd_regexp_t *re;
- char *escaped;
- GError *err = NULL;
- int pcre_flags;
- gsize escaped_len;
- struct rspamd_map_helper_value *val;
- khiter_t k;
- rspamd_ftok_t tok;
- gconstpointer nk;
- gsize vlen;
- int r;
-
- map = re_map->map;
-
- tok.begin = key;
- tok.len = strlen(key);
-
- k = kh_get(rspamd_map_hash, re_map->htb, tok);
-
- if (k == kh_end(re_map->htb)) {
- nk = rspamd_mempool_strdup(re_map->pool, key);
- tok.begin = nk;
- k = kh_put(rspamd_map_hash, re_map->htb, tok, &r);
- }
- else {
- val = kh_value(re_map->htb, k);
-
- /* Always warn about regexp duplicate as it's likely a bad mistake */
- msg_warn_map("duplicate re entry found for map %s: %s (old value: '%s', new: '%s')",
- map->name, key, val->value, value);
-
- if (strcmp(val->value, value) == 0) {
- /* Same value, skip */
- return;
- }
-
- /* Replace value but do not touch regexp */
- nk = kh_key(re_map->htb, k).begin;
- val->key = nk;
- kh_value(re_map->htb, k) = val;
-
- return;
- }
-
- /* Check regexp stuff */
- if (re_map->map_flags & RSPAMD_REGEXP_MAP_FLAG_GLOB) {
- escaped = rspamd_str_regexp_escape(key, strlen(key), &escaped_len,
- RSPAMD_REGEXP_ESCAPE_GLOB | RSPAMD_REGEXP_ESCAPE_UTF);
- re = rspamd_regexp_new(escaped, NULL, &err);
- g_free(escaped);
- }
- else {
- re = rspamd_regexp_new(key, NULL, &err);
- }
-
- if (re == NULL) {
- msg_err_map("cannot parse regexp %s: %e", key, err);
-
- if (err) {
- g_error_free(err);
- }
-
- return;
- }
-
- vlen = strlen(value);
- val = rspamd_mempool_alloc0(re_map->pool, sizeof(*val) +
- vlen + 1);
- memcpy(val->value, value, vlen); /* Null terminated due to alloc0 previously */
- nk = kh_key(re_map->htb, k).begin;
- val->key = nk;
- kh_value(re_map->htb, k) = val;
- rspamd_cryptobox_hash_update(&re_map->hst, nk, tok.len);
-
- pcre_flags = rspamd_regexp_get_pcre_flags(re);
-
- #ifndef WITH_PCRE2
- if (pcre_flags & PCRE_FLAG(UTF8)) {
- re_map->map_flags |= RSPAMD_REGEXP_MAP_FLAG_UTF;
- }
- #else
- if (pcre_flags & PCRE_FLAG(UTF)) {
- re_map->map_flags |= RSPAMD_REGEXP_MAP_FLAG_UTF;
- }
- #endif
-
- g_ptr_array_add(re_map->regexps, re);
- g_ptr_array_add(re_map->values, val);
- }
-
- static void
- rspamd_map_helper_traverse_regexp(void *data,
- rspamd_map_traverse_cb cb,
- gpointer cbdata,
- gboolean reset_hits)
- {
- rspamd_ftok_t tok;
- struct rspamd_map_helper_value *val;
- struct rspamd_regexp_map_helper *re_map = data;
-
- kh_foreach(re_map->htb, tok, val, {
- if (!cb(tok.begin, val->value, val->hits, cbdata)) {
- break;
- }
-
- if (reset_hits) {
- val->hits = 0;
- }
- });
- }
-
- struct rspamd_hash_map_helper *
- rspamd_map_helper_new_hash(struct rspamd_map *map)
- {
- struct rspamd_hash_map_helper *htb;
- rspamd_mempool_t *pool;
-
- if (map) {
- pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
- map->tag, 0);
- }
- else {
- pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
- NULL, 0);
- }
-
- htb = rspamd_mempool_alloc0_type(pool, struct rspamd_hash_map_helper);
- htb->htb = kh_init(rspamd_map_hash);
- htb->pool = pool;
- htb->map = map;
- rspamd_cryptobox_fast_hash_init(&htb->hst, map_hash_seed);
-
- return htb;
- }
-
- void rspamd_map_helper_destroy_hash(struct rspamd_hash_map_helper *r)
- {
- if (r == NULL || r->pool == NULL) {
- return;
- }
-
- rspamd_mempool_t *pool = r->pool;
- kh_destroy(rspamd_map_hash, r->htb);
- memset(r, 0, sizeof(*r));
- rspamd_mempool_delete(pool);
- }
-
- static void
- rspamd_map_helper_traverse_hash(void *data,
- rspamd_map_traverse_cb cb,
- gpointer cbdata,
- gboolean reset_hits)
- {
- rspamd_ftok_t tok;
- struct rspamd_map_helper_value *val;
- struct rspamd_hash_map_helper *ht = data;
-
- kh_foreach(ht->htb, tok, val, {
- if (!cb(tok.begin, val->value, val->hits, cbdata)) {
- break;
- }
-
- if (reset_hits) {
- val->hits = 0;
- }
- });
- }
-
- struct rspamd_radix_map_helper *
- rspamd_map_helper_new_radix(struct rspamd_map *map)
- {
- struct rspamd_radix_map_helper *r;
- rspamd_mempool_t *pool;
- const char *name = "unnamed";
-
- if (map) {
- pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
- map->tag, 0);
- name = map->name;
- }
- else {
- pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
- NULL, 0);
- }
-
- r = rspamd_mempool_alloc0_type(pool, struct rspamd_radix_map_helper);
- r->trie = radix_create_compressed_with_pool(pool, name);
- r->htb = kh_init(rspamd_map_hash);
- r->pool = pool;
- r->map = map;
- rspamd_cryptobox_fast_hash_init(&r->hst, map_hash_seed);
-
- return r;
- }
-
- void rspamd_map_helper_destroy_radix(struct rspamd_radix_map_helper *r)
- {
- if (r == NULL || !r->pool) {
- return;
- }
-
- kh_destroy(rspamd_map_hash, r->htb);
- rspamd_mempool_t *pool = r->pool;
- memset(r, 0, sizeof(*r));
- rspamd_mempool_delete(pool);
- }
-
- static void
- rspamd_map_helper_traverse_radix(void *data,
- rspamd_map_traverse_cb cb,
- gpointer cbdata,
- gboolean reset_hits)
- {
- rspamd_ftok_t tok;
- struct rspamd_map_helper_value *val;
- struct rspamd_radix_map_helper *r = data;
-
- kh_foreach(r->htb, tok, val, {
- if (!cb(tok.begin, val->value, val->hits, cbdata)) {
- break;
- }
-
- if (reset_hits) {
- val->hits = 0;
- }
- });
- }
-
- struct rspamd_regexp_map_helper *
- rspamd_map_helper_new_regexp(struct rspamd_map *map,
- enum rspamd_regexp_map_flags flags)
- {
- struct rspamd_regexp_map_helper *re_map;
- rspamd_mempool_t *pool;
-
- pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
- map->tag, 0);
-
- re_map = rspamd_mempool_alloc0_type(pool, struct rspamd_regexp_map_helper);
- re_map->pool = pool;
- re_map->values = g_ptr_array_new();
- re_map->regexps = g_ptr_array_new();
- re_map->map = map;
- re_map->map_flags = flags;
- re_map->htb = kh_init(rspamd_map_hash);
- rspamd_cryptobox_hash_init(&re_map->hst, NULL, 0);
-
- return re_map;
- }
-
-
- void rspamd_map_helper_destroy_regexp(struct rspamd_regexp_map_helper *re_map)
- {
- rspamd_regexp_t *re;
- unsigned int i;
-
- if (!re_map || !re_map->regexps) {
- return;
- }
-
- #ifdef WITH_HYPERSCAN
- if (re_map->hs_scratch) {
- hs_free_scratch(re_map->hs_scratch);
- }
- if (re_map->hs_db) {
- rspamd_hyperscan_free(re_map->hs_db, false);
- }
- if (re_map->patterns) {
- for (i = 0; i < re_map->regexps->len; i++) {
- g_free(re_map->patterns[i]);
- }
-
- g_free(re_map->patterns);
- }
- if (re_map->flags) {
- g_free(re_map->flags);
- }
- if (re_map->ids) {
- g_free(re_map->ids);
- }
- #endif
-
- for (i = 0; i < re_map->regexps->len; i++) {
- re = g_ptr_array_index(re_map->regexps, i);
- rspamd_regexp_unref(re);
- }
-
- g_ptr_array_free(re_map->regexps, TRUE);
- g_ptr_array_free(re_map->values, TRUE);
- kh_destroy(rspamd_map_hash, re_map->htb);
-
- rspamd_mempool_t *pool = re_map->pool;
- memset(re_map, 0, sizeof(*re_map));
- rspamd_mempool_delete(pool);
- }
-
- char *
- rspamd_kv_list_read(
- char *chunk,
- int len,
- struct map_cb_data *data,
- gboolean final)
- {
- if (data->cur_data == NULL) {
- data->cur_data = rspamd_map_helper_new_hash(data->map);
- }
-
- return rspamd_parse_kv_list(
- chunk,
- len,
- data,
- rspamd_map_helper_insert_hash,
- "",
- final);
- }
-
- void rspamd_kv_list_fin(struct map_cb_data *data, void **target)
- {
- struct rspamd_map *map = data->map;
- struct rspamd_hash_map_helper *htb;
-
- if (data->errored) {
- /* Clean up the current data and do not touch prev data */
- if (data->cur_data) {
- msg_info_map("cleanup unfinished new data as error occurred for %s",
- map->name);
- htb = (struct rspamd_hash_map_helper *) data->cur_data;
- rspamd_map_helper_destroy_hash(htb);
- data->cur_data = NULL;
- }
- }
- else {
- if (data->cur_data) {
- htb = (struct rspamd_hash_map_helper *) data->cur_data;
- msg_info_map("read hash of %d elements from %s", kh_size(htb->htb),
- map->name);
- data->map->traverse_function = rspamd_map_helper_traverse_hash;
- data->map->nelts = kh_size(htb->htb);
- data->map->digest = rspamd_cryptobox_fast_hash_final(&htb->hst);
- }
-
- if (target) {
- *target = data->cur_data;
- }
-
- if (data->prev_data) {
- htb = (struct rspamd_hash_map_helper *) data->prev_data;
- rspamd_map_helper_destroy_hash(htb);
- }
- }
- }
-
- void rspamd_kv_list_dtor(struct map_cb_data *data)
- {
- struct rspamd_hash_map_helper *htb;
-
- if (data->cur_data) {
- htb = (struct rspamd_hash_map_helper *) data->cur_data;
- rspamd_map_helper_destroy_hash(htb);
- }
- }
-
- char *
- rspamd_radix_read(
- char *chunk,
- int len,
- struct map_cb_data *data,
- gboolean final)
- {
- struct rspamd_radix_map_helper *r;
- struct rspamd_map *map = data->map;
-
- if (data->cur_data == NULL) {
- r = rspamd_map_helper_new_radix(map);
- data->cur_data = r;
- }
-
- return rspamd_parse_kv_list(
- chunk,
- len,
- data,
- rspamd_map_helper_insert_radix,
- hash_fill,
- final);
- }
-
- void rspamd_radix_fin(struct map_cb_data *data, void **target)
- {
- struct rspamd_map *map = data->map;
- struct rspamd_radix_map_helper *r;
-
- if (data->errored) {
- /* Clean up the current data and do not touch prev data */
- if (data->cur_data) {
- msg_info_map("cleanup unfinished new data as error occurred for %s",
- map->name);
- r = (struct rspamd_radix_map_helper *) data->cur_data;
- rspamd_map_helper_destroy_radix(r);
- data->cur_data = NULL;
- }
- }
- else {
- if (data->cur_data) {
- r = (struct rspamd_radix_map_helper *) data->cur_data;
- msg_info_map("read radix trie of %z elements: %s",
- radix_get_size(r->trie), radix_get_info(r->trie));
- data->map->traverse_function = rspamd_map_helper_traverse_radix;
- data->map->nelts = kh_size(r->htb);
- data->map->digest = rspamd_cryptobox_fast_hash_final(&r->hst);
- }
-
- if (target) {
- *target = data->cur_data;
- }
-
- if (data->prev_data) {
- r = (struct rspamd_radix_map_helper *) data->prev_data;
- rspamd_map_helper_destroy_radix(r);
- }
- }
- }
-
- void rspamd_radix_dtor(struct map_cb_data *data)
- {
- struct rspamd_radix_map_helper *r;
-
- if (data->cur_data) {
- r = (struct rspamd_radix_map_helper *) data->cur_data;
- rspamd_map_helper_destroy_radix(r);
- }
- }
-
- #ifdef WITH_HYPERSCAN
-
- static gboolean
- rspamd_try_load_re_map_cache(struct rspamd_regexp_map_helper *re_map)
- {
- char fp[PATH_MAX];
- struct rspamd_map *map;
-
- map = re_map->map;
-
- if (!map->cfg->hs_cache_dir) {
- return FALSE;
- }
-
- rspamd_snprintf(fp, sizeof(fp), "%s/%*xs.hsmc",
- map->cfg->hs_cache_dir,
- (int) rspamd_cryptobox_HASHBYTES / 2, re_map->re_digest);
-
- re_map->hs_db = rspamd_hyperscan_maybe_load(fp, 0);
-
- return re_map->hs_db != NULL;
- }
-
- static gboolean
- rspamd_try_save_re_map_cache(struct rspamd_regexp_map_helper *re_map)
- {
- char fp[PATH_MAX], np[PATH_MAX];
- gsize len;
- int fd;
- char *bytes = NULL;
- struct rspamd_map *map;
-
- map = re_map->map;
-
- if (!map->cfg->hs_cache_dir) {
- return FALSE;
- }
-
- rspamd_snprintf(fp, sizeof(fp), "%s/hsmc-XXXXXXXXXXXXX",
- re_map->map->cfg->hs_cache_dir);
-
- if ((fd = g_mkstemp_full(fp, O_WRONLY | O_CREAT | O_EXCL, 00644)) != -1) {
- if (hs_serialize_database(rspamd_hyperscan_get_database(re_map->hs_db), &bytes, &len) == HS_SUCCESS) {
- if (write(fd, bytes, len) == -1) {
- msg_warn_map("cannot write hyperscan cache to %s: %s",
- fp, strerror(errno));
- unlink(fp);
- free(bytes);
- }
- else {
- free(bytes);
- fsync(fd);
-
- rspamd_snprintf(np, sizeof(np), "%s/%*xs.hsmc",
- re_map->map->cfg->hs_cache_dir,
- (int) rspamd_cryptobox_HASHBYTES / 2, re_map->re_digest);
-
- if (rename(fp, np) == -1) {
- msg_warn_map("cannot rename hyperscan cache from %s to %s: %s",
- fp, np, strerror(errno));
- unlink(fp);
- }
- else {
- msg_info_map("written cached hyperscan data for %s to %s (%Hz length)",
- map->name, np, len);
- rspamd_hyperscan_notice_known(np);
- }
- }
- }
- else {
- msg_warn_map("cannot serialize hyperscan cache to %s: %s",
- fp, strerror(errno));
- unlink(fp);
- }
-
-
- close(fd);
- }
-
- return FALSE;
- }
-
- #endif
-
- static void
- rspamd_re_map_finalize(struct rspamd_regexp_map_helper *re_map)
- {
- #ifdef WITH_HYPERSCAN
- unsigned int i;
- hs_platform_info_t plt;
- hs_compile_error_t *err;
- struct rspamd_map *map;
- rspamd_regexp_t *re;
- int pcre_flags;
-
- map = re_map->map;
-
- #if !defined(__aarch64__) && !defined(__powerpc64__)
- if (!(map->cfg->libs_ctx->crypto_ctx->cpu_config & CPUID_SSSE3)) {
- msg_info_map("disable hyperscan for map %s, ssse3 instructions are not supported by CPU",
- map->name);
- return;
- }
- #endif
-
- if (hs_populate_platform(&plt) != HS_SUCCESS) {
- msg_err_map("cannot populate hyperscan platform");
- return;
- }
-
- re_map->patterns = g_new(char *, re_map->regexps->len);
- re_map->flags = g_new(int, re_map->regexps->len);
- re_map->ids = g_new(int, re_map->regexps->len);
-
- for (i = 0; i < re_map->regexps->len; i++) {
- const char *pat;
- char *escaped;
- int pat_flags;
-
- re = g_ptr_array_index(re_map->regexps, i);
- pcre_flags = rspamd_regexp_get_pcre_flags(re);
- pat = rspamd_regexp_get_pattern(re);
- pat_flags = rspamd_regexp_get_flags(re);
-
- if (pat_flags & RSPAMD_REGEXP_FLAG_UTF) {
- escaped = rspamd_str_regexp_escape(pat, strlen(pat), NULL,
- RSPAMD_REGEXP_ESCAPE_RE | RSPAMD_REGEXP_ESCAPE_UTF);
- re_map->flags[i] |= HS_FLAG_UTF8;
- }
- else {
- escaped = rspamd_str_regexp_escape(pat, strlen(pat), NULL,
- RSPAMD_REGEXP_ESCAPE_RE);
- }
-
- re_map->patterns[i] = escaped;
- re_map->flags[i] = HS_FLAG_SINGLEMATCH;
-
- #ifndef WITH_PCRE2
- if (pcre_flags & PCRE_FLAG(UTF8)) {
- re_map->flags[i] |= HS_FLAG_UTF8;
- }
- #else
- if (pcre_flags & PCRE_FLAG(UTF)) {
- re_map->flags[i] |= HS_FLAG_UTF8;
- }
- #endif
- if (pcre_flags & PCRE_FLAG(CASELESS)) {
- re_map->flags[i] |= HS_FLAG_CASELESS;
- }
- if (pcre_flags & PCRE_FLAG(MULTILINE)) {
- re_map->flags[i] |= HS_FLAG_MULTILINE;
- }
- if (pcre_flags & PCRE_FLAG(DOTALL)) {
- re_map->flags[i] |= HS_FLAG_DOTALL;
- }
- if (rspamd_regexp_get_maxhits(re) == 1) {
- re_map->flags[i] |= HS_FLAG_SINGLEMATCH;
- }
-
- re_map->ids[i] = i;
- }
-
- if (re_map->regexps->len > 0 && re_map->patterns) {
-
- if (!rspamd_try_load_re_map_cache(re_map)) {
- double ts1 = rspamd_get_ticks(FALSE);
- hs_database_t *hs_db = NULL;
-
- if (hs_compile_multi((const char **) re_map->patterns,
- re_map->flags,
- re_map->ids,
- re_map->regexps->len,
- HS_MODE_BLOCK,
- &plt,
- &hs_db,
- &err) != HS_SUCCESS) {
-
- msg_err_map("cannot create tree of regexp when processing '%s': %s",
- err->expression >= 0 ? re_map->patterns[err->expression] : "unknown regexp", err->message);
- re_map->hs_db = NULL;
- hs_free_compile_error(err);
-
- return;
- }
-
- if (re_map->map->cfg->hs_cache_dir) {
- char fpath[PATH_MAX];
- rspamd_snprintf(fpath, sizeof(fpath), "%s/%*xs.hsmc",
- re_map->map->cfg->hs_cache_dir,
- (int) rspamd_cryptobox_HASHBYTES / 2, re_map->re_digest);
- re_map->hs_db = rspamd_hyperscan_from_raw_db(hs_db, fpath);
- }
- else {
- re_map->hs_db = rspamd_hyperscan_from_raw_db(hs_db, NULL);
- }
-
- ts1 = (rspamd_get_ticks(FALSE) - ts1) * 1000.0;
- msg_info_map("hyperscan compiled %d regular expressions from %s in %.1f ms",
- re_map->regexps->len, re_map->map->name, ts1);
- rspamd_try_save_re_map_cache(re_map);
- }
- else {
- msg_info_map("hyperscan read %d cached regular expressions from %s",
- re_map->regexps->len, re_map->map->name);
- }
-
- if (hs_alloc_scratch(rspamd_hyperscan_get_database(re_map->hs_db), &re_map->hs_scratch) != HS_SUCCESS) {
- msg_err_map("cannot allocate scratch space for hyperscan");
- rspamd_hyperscan_free(re_map->hs_db, true);
- re_map->hs_db = NULL;
- }
- }
- else {
- msg_err_map("regexp map is empty");
- }
- #endif
- }
-
- char *
- rspamd_regexp_list_read_single(
- char *chunk,
- int len,
- struct map_cb_data *data,
- gboolean final)
- {
- struct rspamd_regexp_map_helper *re_map;
-
- if (data->cur_data == NULL) {
- re_map = rspamd_map_helper_new_regexp(data->map, 0);
- data->cur_data = re_map;
- }
-
- return rspamd_parse_kv_list(
- chunk,
- len,
- data,
- rspamd_map_helper_insert_re,
- hash_fill,
- final);
- }
-
- char *
- rspamd_glob_list_read_single(
- char *chunk,
- int len,
- struct map_cb_data *data,
- gboolean final)
- {
- struct rspamd_regexp_map_helper *re_map;
-
- if (data->cur_data == NULL) {
- re_map = rspamd_map_helper_new_regexp(data->map, RSPAMD_REGEXP_MAP_FLAG_GLOB);
- data->cur_data = re_map;
- }
-
- return rspamd_parse_kv_list(
- chunk,
- len,
- data,
- rspamd_map_helper_insert_re,
- hash_fill,
- final);
- }
-
- char *
- rspamd_regexp_list_read_multiple(
- char *chunk,
- int len,
- struct map_cb_data *data,
- gboolean final)
- {
- struct rspamd_regexp_map_helper *re_map;
-
- if (data->cur_data == NULL) {
- re_map = rspamd_map_helper_new_regexp(data->map,
- RSPAMD_REGEXP_MAP_FLAG_MULTIPLE);
- data->cur_data = re_map;
- }
-
- return rspamd_parse_kv_list(
- chunk,
- len,
- data,
- rspamd_map_helper_insert_re,
- hash_fill,
- final);
- }
-
- char *
- rspamd_glob_list_read_multiple(
- char *chunk,
- int len,
- struct map_cb_data *data,
- gboolean final)
- {
- struct rspamd_regexp_map_helper *re_map;
-
- if (data->cur_data == NULL) {
- re_map = rspamd_map_helper_new_regexp(data->map,
- RSPAMD_REGEXP_MAP_FLAG_GLOB | RSPAMD_REGEXP_MAP_FLAG_MULTIPLE);
- data->cur_data = re_map;
- }
-
- return rspamd_parse_kv_list(
- chunk,
- len,
- data,
- rspamd_map_helper_insert_re,
- hash_fill,
- final);
- }
-
-
- void rspamd_regexp_list_fin(struct map_cb_data *data, void **target)
- {
- struct rspamd_regexp_map_helper *re_map = NULL, *old_re_map;
- struct rspamd_map *map = data->map;
-
- if (data->errored) {
- /* Clean up the current data and do not touch prev data */
- if (data->cur_data) {
- msg_info_map("cleanup unfinished new data as error occurred for %s",
- map->name);
- re_map = (struct rspamd_regexp_map_helper *) data->cur_data;
- rspamd_map_helper_destroy_regexp(re_map);
- data->cur_data = NULL;
- }
- }
- else {
- if (data->cur_data) {
- re_map = data->cur_data;
- rspamd_cryptobox_hash_final(&re_map->hst, re_map->re_digest);
- memcpy(&data->map->digest, re_map->re_digest, sizeof(data->map->digest));
- rspamd_re_map_finalize(re_map);
- msg_info_map("read regexp list of %ud elements",
- re_map->regexps->len);
- data->map->traverse_function = rspamd_map_helper_traverse_regexp;
- data->map->nelts = kh_size(re_map->htb);
- }
-
- if (target) {
- *target = data->cur_data;
- }
-
- if (data->prev_data) {
- old_re_map = data->prev_data;
- rspamd_map_helper_destroy_regexp(old_re_map);
- }
- }
- }
- void rspamd_regexp_list_dtor(struct map_cb_data *data)
- {
- if (data->cur_data) {
- rspamd_map_helper_destroy_regexp(data->cur_data);
- }
- }
-
- #ifdef WITH_HYPERSCAN
- static int
- rspamd_match_hs_single_handler(unsigned int id, unsigned long long from,
- unsigned long long to,
- unsigned int flags, void *context)
- {
- unsigned int *i = context;
- /* Always return non-zero as we need a single match here */
-
- *i = id;
-
- return 1;
- }
- #endif
-
- gconstpointer
- rspamd_match_regexp_map_single(struct rspamd_regexp_map_helper *map,
- const char *in, gsize len)
- {
- unsigned int i;
- rspamd_regexp_t *re;
- int res = 0;
- gpointer ret = NULL;
- struct rspamd_map_helper_value *val;
- gboolean validated = FALSE;
-
- g_assert(in != NULL);
-
- if (map == NULL || len == 0 || map->regexps == NULL) {
- return NULL;
- }
-
- if (map->map_flags & RSPAMD_REGEXP_MAP_FLAG_UTF) {
- if (rspamd_fast_utf8_validate(in, len) == 0) {
- validated = TRUE;
- }
- }
- else {
- validated = TRUE;
- }
-
- #ifdef WITH_HYPERSCAN
- if (map->hs_db && map->hs_scratch) {
-
- if (validated) {
-
- res = hs_scan(rspamd_hyperscan_get_database(map->hs_db), in, len, 0,
- map->hs_scratch,
- rspamd_match_hs_single_handler, (void *) &i);
-
- if (res == HS_SCAN_TERMINATED) {
- res = 1;
- val = g_ptr_array_index(map->values, i);
-
- ret = val->value;
- val->hits++;
- }
-
- return ret;
- }
- }
- #endif
-
- if (!res) {
- /* PCRE version */
- for (i = 0; i < map->regexps->len; i++) {
- re = g_ptr_array_index(map->regexps, i);
-
- if (rspamd_regexp_search(re, in, len, NULL, NULL, !validated, NULL)) {
- val = g_ptr_array_index(map->values, i);
-
- ret = val->value;
- val->hits++;
- break;
- }
- }
- }
-
- return ret;
- }
-
- #ifdef WITH_HYPERSCAN
- struct rspamd_multiple_cbdata {
- GPtrArray *ar;
- struct rspamd_regexp_map_helper *map;
- };
-
- static int
- rspamd_match_hs_multiple_handler(unsigned int id, unsigned long long from,
- unsigned long long to,
- unsigned int flags, void *context)
- {
- struct rspamd_multiple_cbdata *cbd = context;
- struct rspamd_map_helper_value *val;
-
-
- if (id < cbd->map->values->len) {
- val = g_ptr_array_index(cbd->map->values, id);
- val->hits++;
- g_ptr_array_add(cbd->ar, val->value);
- }
-
- /* Always return zero as we need all matches here */
- return 0;
- }
- #endif
-
- GPtrArray *
- rspamd_match_regexp_map_all(struct rspamd_regexp_map_helper *map,
- const char *in, gsize len)
- {
- unsigned int i;
- rspamd_regexp_t *re;
- GPtrArray *ret;
- int res = 0;
- gboolean validated = FALSE;
- struct rspamd_map_helper_value *val;
-
- if (map == NULL || map->regexps == NULL || len == 0) {
- return NULL;
- }
-
- g_assert(in != NULL);
-
- if (map->map_flags & RSPAMD_REGEXP_MAP_FLAG_UTF) {
- if (rspamd_fast_utf8_validate(in, len) == 0) {
- validated = TRUE;
- }
- }
- else {
- validated = TRUE;
- }
-
- ret = g_ptr_array_new();
-
- #ifdef WITH_HYPERSCAN
- if (map->hs_db && map->hs_scratch) {
-
- if (validated) {
- struct rspamd_multiple_cbdata cbd;
-
- cbd.ar = ret;
- cbd.map = map;
-
- if (hs_scan(rspamd_hyperscan_get_database(map->hs_db), in, len,
- 0, map->hs_scratch,
- rspamd_match_hs_multiple_handler, &cbd) == HS_SUCCESS) {
- res = 1;
- }
- }
- }
- #endif
-
- if (!res) {
- /* PCRE version */
- for (i = 0; i < map->regexps->len; i++) {
- re = g_ptr_array_index(map->regexps, i);
-
- if (rspamd_regexp_search(re, in, len, NULL, NULL,
- !validated, NULL)) {
- val = g_ptr_array_index(map->values, i);
- val->hits++;
- g_ptr_array_add(ret, val->value);
- }
- }
- }
-
- if (ret->len > 0) {
- return ret;
- }
-
- g_ptr_array_free(ret, TRUE);
-
- return NULL;
- }
-
- gconstpointer
- rspamd_match_hash_map(struct rspamd_hash_map_helper *map, const char *in,
- gsize len)
- {
- khiter_t k;
- struct rspamd_map_helper_value *val;
- rspamd_ftok_t tok;
-
- if (map == NULL || map->htb == NULL) {
- return NULL;
- }
-
- tok.begin = in;
- tok.len = len;
-
- k = kh_get(rspamd_map_hash, map->htb, tok);
-
- if (k != kh_end(map->htb)) {
- val = kh_value(map->htb, k);
- val->hits++;
-
- return val->value;
- }
-
- return NULL;
- }
-
- gconstpointer
- rspamd_match_radix_map(struct rspamd_radix_map_helper *map,
- const unsigned char *in, gsize inlen)
- {
- struct rspamd_map_helper_value *val;
-
- if (map == NULL || map->trie == NULL) {
- return NULL;
- }
-
- val = (struct rspamd_map_helper_value *) radix_find_compressed(map->trie,
- in, inlen);
-
- if (val != (gconstpointer) RADIX_NO_VALUE) {
- val->hits++;
-
- return val->value;
- }
-
- return NULL;
- }
-
- gconstpointer
- rspamd_match_radix_map_addr(struct rspamd_radix_map_helper *map,
- const rspamd_inet_addr_t *addr)
- {
- struct rspamd_map_helper_value *val;
-
- if (map == NULL || map->trie == NULL) {
- return NULL;
- }
-
- val = (struct rspamd_map_helper_value *) radix_find_compressed_addr(map->trie, addr);
-
- if (val != (gconstpointer) RADIX_NO_VALUE) {
- val->hits++;
-
- return val->value;
- }
-
- return NULL;
- }
-
-
- /*
- * CBD stuff
- */
-
- struct rspamd_cdb_map_helper *
- rspamd_map_helper_new_cdb(struct rspamd_map *map)
- {
- struct rspamd_cdb_map_helper *n;
-
- n = g_malloc0(sizeof(*n));
- n->cdbs = (GQueue) G_QUEUE_INIT;
- n->map = map;
-
- rspamd_cryptobox_fast_hash_init(&n->hst, map_hash_seed);
-
- return n;
- }
-
- void rspamd_map_helper_destroy_cdb(struct rspamd_cdb_map_helper *c)
- {
- if (c == NULL) {
- return;
- }
-
- GList *cur = c->cdbs.head;
-
- while (cur) {
- struct cdb *cdb = (struct cdb *) cur->data;
-
- cdb_free(cdb);
- g_free(cdb->filename);
- close(cdb->cdb_fd);
- g_free(cdb);
-
- cur = g_list_next(cur);
- }
-
- g_queue_clear(&c->cdbs);
-
- g_free(c);
- }
-
- char *
- rspamd_cdb_list_read(char *chunk,
- int len,
- struct map_cb_data *data,
- gboolean final)
- {
- struct rspamd_cdb_map_helper *cdb_data;
- struct cdb *found = NULL;
- struct rspamd_map *map = data->map;
-
- g_assert(map->no_file_read);
-
- if (data->cur_data == NULL) {
- cdb_data = rspamd_map_helper_new_cdb(data->map);
- data->cur_data = cdb_data;
- }
- else {
- cdb_data = (struct rspamd_cdb_map_helper *) data->cur_data;
- }
-
- GList *cur = cdb_data->cdbs.head;
-
- while (cur) {
- struct cdb *elt = (struct cdb *) cur->data;
-
- if (strcmp(elt->filename, chunk) == 0) {
- found = elt;
- break;
- }
-
- cur = g_list_next(cur);
- }
-
- if (found == NULL) {
- /* New cdb */
- int fd;
- struct cdb *cdb;
-
- fd = rspamd_file_xopen(chunk, O_RDONLY, 0, TRUE);
-
- if (fd == -1) {
- msg_err_map("cannot open cdb map from %s: %s", chunk, strerror(errno));
-
- return NULL;
- }
-
- cdb = g_malloc0(sizeof(struct cdb));
-
- if (cdb_init(cdb, fd) == -1) {
- g_free(cdb);
- msg_err_map("cannot init cdb map from %s: %s", chunk, strerror(errno));
-
- return NULL;
- }
-
- cdb->filename = g_strdup(chunk);
- g_queue_push_tail(&cdb_data->cdbs, cdb);
- cdb_data->total_size += cdb->cdb_fsize;
- rspamd_cryptobox_fast_hash_update(&cdb_data->hst, chunk, len);
- }
-
- return chunk + len;
- }
-
- void rspamd_cdb_list_fin(struct map_cb_data *data, void **target)
- {
- struct rspamd_map *map = data->map;
- struct rspamd_cdb_map_helper *cdb_data;
-
- if (data->errored) {
- /* Clean up the current data and do not touch prev data */
- if (data->cur_data) {
- msg_info_map("cleanup unfinished new data as error occurred for %s",
- map->name);
- cdb_data = (struct rspamd_cdb_map_helper *) data->cur_data;
- rspamd_map_helper_destroy_cdb(cdb_data);
- data->cur_data = NULL;
- }
- }
- else {
- if (data->cur_data) {
- cdb_data = (struct rspamd_cdb_map_helper *) data->cur_data;
- msg_info_map("read cdb of %Hz size", cdb_data->total_size);
- data->map->traverse_function = NULL;
- data->map->nelts = 0;
- data->map->digest = rspamd_cryptobox_fast_hash_final(&cdb_data->hst);
- }
-
- if (target) {
- *target = data->cur_data;
- }
-
- if (data->prev_data) {
- cdb_data = (struct rspamd_cdb_map_helper *) data->prev_data;
- rspamd_map_helper_destroy_cdb(cdb_data);
- }
- }
- }
- void rspamd_cdb_list_dtor(struct map_cb_data *data)
- {
- if (data->cur_data) {
- rspamd_map_helper_destroy_cdb(data->cur_data);
- }
- }
-
- gconstpointer
- rspamd_match_cdb_map(struct rspamd_cdb_map_helper *map,
- const char *in, gsize inlen)
- {
- if (map == NULL || map->cdbs.head == NULL) {
- return NULL;
- }
-
- GList *cur = map->cdbs.head;
- static rspamd_ftok_t found;
-
- while (cur) {
- struct cdb *cdb = (struct cdb *) cur->data;
-
- if (cdb_find(cdb, in, inlen) > 0) {
- /* Extract and push value to lua as string */
- unsigned vlen;
- gconstpointer vpos;
-
- vpos = cdb->cdb_mem + cdb_datapos(cdb);
- vlen = cdb_datalen(cdb);
- found.len = vlen;
- found.begin = vpos;
-
- return &found; /* Do not reuse! */
- }
-
- cur = g_list_next(cur);
- }
-
- return NULL;
- }
|