From c9af649b3316857e77ec3509ca93c4a4ff37f477 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Sat, 5 Mar 2011 22:47:28 +0300 Subject: [PATCH] * Add ability to lookup CDB maps from lua * Add cdb:// map to multimap plugin --- CMakeLists.txt | 2 + src/cdb/CMakeLists.txt | 7 + src/cdb/cdb.h | 149 ++++++++++ src/cdb/cdb_find.c | 147 ++++++++++ src/cdb/cdb_init.c | 105 +++++++ src/cdb/cdb_make.c | 524 +++++++++++++++++++++++++++++++++++ src/lua/CMakeLists.txt | 3 +- src/lua/lua_cdb.c | 152 ++++++++++ src/lua/lua_common.c | 1 + src/lua/lua_common.h | 3 +- src/plugins/lua/multimap.lua | 166 ++++++++--- 11 files changed, 1217 insertions(+), 42 deletions(-) create mode 100644 src/cdb/CMakeLists.txt create mode 100644 src/cdb/cdb.h create mode 100644 src/cdb/cdb_find.c create mode 100644 src/cdb/cdb_init.c create mode 100644 src/cdb/cdb_make.c create mode 100644 src/lua/lua_cdb.c diff --git a/CMakeLists.txt b/CMakeLists.txt index e93793fdf..eda84cd41 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -593,6 +593,7 @@ ADD_SUBDIRECTORY(lib) ADD_SUBDIRECTORY(src/client) ADD_SUBDIRECTORY(src/json) +ADD_SUBDIRECTORY(src/cdb) # ADD_SUBDIRECTORY(src/evdns) # ADD_SUBDIRECTORY(src/plugins/custom) @@ -705,6 +706,7 @@ ENDIF(LIBJUDY_LIBRARY) # TARGET_LINK_LIBRARIES(rspamd rspamd_evdns) TARGET_LINK_LIBRARIES(rspamd event) TARGET_LINK_LIBRARIES(rspamd rspamd_json) +TARGET_LINK_LIBRARIES(rspamd rspamd_cdb) TARGET_LINK_LIBRARIES(rspamd ${CMAKE_REQUIRED_LIBRARIES}) TARGET_LINK_LIBRARIES(rspamd ${GLIB2_LIBRARIES}) IF(GMIME2_FOUND) diff --git a/src/cdb/CMakeLists.txt b/src/cdb/CMakeLists.txt new file mode 100644 index 000000000..d0b56bc6c --- /dev/null +++ b/src/cdb/CMakeLists.txt @@ -0,0 +1,7 @@ +# CDB support makefile +SET(CDBSRC cdb_init.c + cdb_find.c + cdb_make.c) + +ADD_LIBRARY(rspamd_cdb STATIC ${CDBSRC}) +SET_TARGET_PROPERTIES(rspamd_cdb PROPERTIES COMPILE_FLAGS "-DRSPAMD_LIB") \ No newline at end of file diff --git a/src/cdb/cdb.h b/src/cdb/cdb.h new file mode 100644 index 000000000..5802da4d5 --- /dev/null +++ b/src/cdb/cdb.h @@ -0,0 +1,149 @@ +/* $Id: cdb.h,v 1.10 2009-01-31 17:12:22 mjt Exp $ + * public cdb include file + * + * This file is a part of tinycdb package by Michael Tokarev, mjt@corpit.ru. + * Public domain. + */ + +#ifndef TINYCDB_VERSION +#define TINYCDB_VERSION 0.77 + +#include "../config.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef unsigned int cdbi_t; /* compatibility */ + +/* common routines */ +unsigned cdb_hash(const void *buf, unsigned len); +unsigned cdb_unpack(const unsigned char buf[4]); +void cdb_pack(unsigned num, unsigned char buf[4]); + +struct cdb { + int cdb_fd; /* file descriptor */ + char *filename; /* file name */ + /* private members */ + unsigned cdb_fsize; /* datafile size */ + unsigned cdb_dend; /* end of data ptr */ + const unsigned char *cdb_mem; /* mmap'ed file memory */ + unsigned cdb_vpos, cdb_vlen; /* found data */ + unsigned cdb_kpos, cdb_klen; /* found key */ +}; + +#define CDB_STATIC_INIT {0,0,0,0,0,0,0,0} + +#define cdb_datapos(c) ((c)->cdb_vpos) +#define cdb_datalen(c) ((c)->cdb_vlen) +#define cdb_keypos(c) ((c)->cdb_kpos) +#define cdb_keylen(c) ((c)->cdb_klen) +#define cdb_fileno(c) ((c)->cdb_fd) + +int cdb_init(struct cdb *cdbp, int fd); +void cdb_free(struct cdb *cdbp); + +int cdb_read(const struct cdb *cdbp, + void *buf, unsigned len, unsigned pos); +#define cdb_readdata(cdbp, buf) \ + cdb_read((cdbp), (buf), cdb_datalen(cdbp), cdb_datapos(cdbp)) +#define cdb_readkey(cdbp, buf) \ + cdb_read((cdbp), (buf), cdb_keylen(cdbp), cdb_keypos(cdbp)) + +const void *cdb_get(const struct cdb *cdbp, unsigned len, unsigned pos); +#define cdb_getdata(cdbp) \ + cdb_get((cdbp), cdb_datalen(cdbp), cdb_datapos(cdbp)) +#define cdb_getkey(cdbp) \ + cdb_get((cdbp), cdb_keylen(cdbp), cdb_keypos(cdbp)) + +int cdb_find(struct cdb *cdbp, const void *key, unsigned klen); + +struct cdb_find { + struct cdb *cdb_cdbp; + unsigned cdb_hval; + const unsigned char *cdb_htp, *cdb_htab, *cdb_htend; + unsigned cdb_httodo; + const void *cdb_key; + unsigned cdb_klen; +}; + +int cdb_findinit(struct cdb_find *cdbfp, struct cdb *cdbp, + const void *key, unsigned klen); +int cdb_findnext(struct cdb_find *cdbfp); + +#define cdb_seqinit(cptr, cdbp) ((*(cptr))=2048) +int cdb_seqnext(unsigned *cptr, struct cdb *cdbp); + +/* old simple interface */ +/* open file using standard routine, then: */ +int cdb_seek(int fd, const void *key, unsigned klen, unsigned *dlenp); +int cdb_bread(int fd, void *buf, int len); + +/* cdb_make */ + +struct cdb_make { + int cdb_fd; /* file descriptor */ + /* private */ + unsigned cdb_dpos; /* data position so far */ + unsigned cdb_rcnt; /* record count so far */ + unsigned char cdb_buf[4096]; /* write buffer */ + unsigned char *cdb_bpos; /* current buf position */ + struct cdb_rl *cdb_rec[256]; /* list of arrays of record infos */ +}; + +enum cdb_put_mode { + CDB_PUT_ADD = 0, /* add unconditionnaly, like cdb_make_add() */ +#define CDB_PUT_ADD CDB_PUT_ADD + CDB_FIND = CDB_PUT_ADD, + CDB_PUT_REPLACE, /* replace: do not place to index OLD record */ +#define CDB_PUT_REPLACE CDB_PUT_REPLACE + CDB_FIND_REMOVE = CDB_PUT_REPLACE, + CDB_PUT_INSERT, /* add only if not already exists */ +#define CDB_PUT_INSERT CDB_PUT_INSERT + CDB_PUT_WARN, /* add unconditionally but ret. 1 if exists */ +#define CDB_PUT_WARN CDB_PUT_WARN + CDB_PUT_REPLACE0, /* if a record exists, fill old one with zeros */ +#define CDB_PUT_REPLACE0 CDB_PUT_REPLACE0 + CDB_FIND_FILL0 = CDB_PUT_REPLACE0 +}; + +int cdb_make_start(struct cdb_make *cdbmp, int fd); +int cdb_make_add(struct cdb_make *cdbmp, + const void *key, unsigned klen, + const void *val, unsigned vlen); +int cdb_make_exists(struct cdb_make *cdbmp, + const void *key, unsigned klen); +int cdb_make_find(struct cdb_make *cdbmp, + const void *key, unsigned klen, + enum cdb_put_mode mode); +int cdb_make_put(struct cdb_make *cdbmp, + const void *key, unsigned klen, + const void *val, unsigned vlen, + enum cdb_put_mode mode); +int cdb_make_finish(struct cdb_make *cdbmp); + +/** Private API **/ +struct cdb_rec { + unsigned hval; + unsigned rpos; +}; + +struct cdb_rl { + struct cdb_rl *next; + unsigned cnt; + struct cdb_rec rec[254]; +}; + +int _cdb_make_write(struct cdb_make *cdbmp, + const unsigned char *ptr, unsigned len); +int _cdb_make_fullwrite(int fd, const unsigned char *buf, unsigned len); +int _cdb_make_flush(struct cdb_make *cdbmp); +int _cdb_make_add(struct cdb_make *cdbmp, unsigned hval, + const void *key, unsigned klen, + const void *val, unsigned vlen); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* include guard */ diff --git a/src/cdb/cdb_find.c b/src/cdb/cdb_find.c new file mode 100644 index 000000000..cae0f184b --- /dev/null +++ b/src/cdb/cdb_find.c @@ -0,0 +1,147 @@ +/* $Id: cdb_init.c,v 1.12 2008-11-06 18:07:04 mjt Exp $ + * cdb_init, cdb_free and cdb_read routines + * + * This file is a part of tinycdb package by Michael Tokarev, mjt@corpit.ru. + * Public domain. + */ + +#include "cdb.h" + +int +cdb_find(struct cdb *cdbp, const void *key, unsigned klen) +{ + const unsigned char *htp; /* hash table pointer */ + const unsigned char *htab; /* hash table */ + const unsigned char *htend; /* end of hash table */ + unsigned httodo; /* ht bytes left to look */ + unsigned pos, n; + + unsigned hval; + + if (klen >= cdbp->cdb_dend) /* if key size is too large */ + return 0; + + hval = cdb_hash (key, klen); + + /* find (pos,n) hash table to use */ + /* first 2048 bytes (toc) are always available */ + /* (hval % 256) * 8 */ + htp = cdbp->cdb_mem + ((hval << 3) & 2047); /* index in toc (256x8) */ + n = cdb_unpack (htp + 4); /* table size */ + if (!n) /* empty table */ + return 0; /* not found */ + httodo = n << 3; /* bytes of htab to lookup */ + pos = cdb_unpack (htp); /* htab position */ + if (n > (cdbp->cdb_fsize >> 3) /* overflow of httodo ? */ + || pos < cdbp->cdb_dend /* is htab inside data section ? */ + || pos > cdbp->cdb_fsize /* htab start within file ? */ + || httodo > cdbp->cdb_fsize - pos) /* entrie htab within file ? */ + return errno = EPROTO, -1; + + htab = cdbp->cdb_mem + pos; /* htab pointer */ + htend = htab + httodo; /* after end of htab */ + /* htab starting position: rest of hval modulo htsize, 8bytes per elt */ + htp = htab + (((hval >> 8) % n) << 3); + + for (;;) { + pos = cdb_unpack (htp + 4); /* record position */ + if (!pos) + return 0; + if (cdb_unpack (htp) == hval) { + if (pos > cdbp->cdb_dend - 8) /* key+val lengths */ + return errno = EPROTO, -1; + if (cdb_unpack (cdbp->cdb_mem + pos) == klen) { + if (cdbp->cdb_dend - klen < pos + 8) + return errno = EPROTO, -1; + if (memcmp (key, cdbp->cdb_mem + pos + 8, klen) == 0) { + n = cdb_unpack (cdbp->cdb_mem + pos + 4); + pos += 8; + if (cdbp->cdb_dend < n || cdbp->cdb_dend - n < pos + klen) + return errno = EPROTO, -1; + cdbp->cdb_kpos = pos; + cdbp->cdb_klen = klen; + cdbp->cdb_vpos = pos + klen; + cdbp->cdb_vlen = n; + return 1; + } + } + } + httodo -= 8; + if (!httodo) + return 0; + if ((htp += 8) >= htend) + htp = htab; + } + +} + +int +cdb_findinit(struct cdb_find *cdbfp, struct cdb *cdbp, + const void *key, unsigned klen) +{ + unsigned n, pos; + + cdbfp->cdb_cdbp = cdbp; + cdbfp->cdb_key = key; + cdbfp->cdb_klen = klen; + cdbfp->cdb_hval = cdb_hash(key, klen); + + cdbfp->cdb_htp = cdbp->cdb_mem + ((cdbfp->cdb_hval << 3) & 2047); + n = cdb_unpack(cdbfp->cdb_htp + 4); + cdbfp->cdb_httodo = n << 3; + if (!n) + return 0; + pos = cdb_unpack(cdbfp->cdb_htp); + if (n > (cdbp->cdb_fsize >> 3) + || pos < cdbp->cdb_dend + || pos > cdbp->cdb_fsize + || cdbfp->cdb_httodo > cdbp->cdb_fsize - pos) + return errno = EPROTO, -1; + + cdbfp->cdb_htab = cdbp->cdb_mem + pos; + cdbfp->cdb_htend = cdbfp->cdb_htab + cdbfp->cdb_httodo; + cdbfp->cdb_htp = cdbfp->cdb_htab + (((cdbfp->cdb_hval >> 8) % n) << 3); + + return 1; +} + +int +cdb_findnext(struct cdb_find *cdbfp) { + struct cdb *cdbp = cdbfp->cdb_cdbp; + unsigned pos, n; + unsigned klen = cdbfp->cdb_klen; + + while(cdbfp->cdb_httodo) { + pos = cdb_unpack(cdbfp->cdb_htp + 4); + if (!pos) + return 0; + n = cdb_unpack(cdbfp->cdb_htp) == cdbfp->cdb_hval; + if ((cdbfp->cdb_htp += 8) >= cdbfp->cdb_htend) + cdbfp->cdb_htp = cdbfp->cdb_htab; + cdbfp->cdb_httodo -= 8; + if (n) { + if (pos > cdbp->cdb_fsize - 8) + return errno = EPROTO, -1; + if (cdb_unpack(cdbp->cdb_mem + pos) == klen) { + if (cdbp->cdb_fsize - klen < pos + 8) + return errno = EPROTO, -1; + if (memcmp(cdbfp->cdb_key, + cdbp->cdb_mem + pos + 8, klen) == 0) { + n = cdb_unpack(cdbp->cdb_mem + pos + 4); + pos += 8; + if (cdbp->cdb_fsize < n || + cdbp->cdb_fsize - n < pos + klen) + return errno = EPROTO, -1; + cdbp->cdb_kpos = pos; + cdbp->cdb_klen = klen; + cdbp->cdb_vpos = pos + klen; + cdbp->cdb_vlen = n; + return 1; + } + } + } + } + + return 0; + +} diff --git a/src/cdb/cdb_init.c b/src/cdb/cdb_init.c new file mode 100644 index 000000000..d3ae8c01b --- /dev/null +++ b/src/cdb/cdb_init.c @@ -0,0 +1,105 @@ +/* $Id: cdb_init.c,v 1.12 2008-11-06 18:07:04 mjt Exp $ + * cdb_init, cdb_free and cdb_read routines + * + * This file is a part of tinycdb package by Michael Tokarev, mjt@corpit.ru. + * Public domain. + */ + +#include "cdb.h" + +unsigned +cdb_hash(const void *buf, unsigned len) +{ + register const unsigned char *p = (const unsigned char *) buf; + register const unsigned char *end = p + len; + register unsigned hash = 5381; /* start value */ + while (p < end) + hash = (hash + (hash << 5)) ^ *p++; + return hash; +} + +int +cdb_init(struct cdb *cdbp, int fd) +{ + struct stat st; + unsigned char *mem; + unsigned fsize, dend; +#ifdef _WIN32 + HANDLE hFile, hMapping; +#endif + + /* get file size */ + if (fstat (fd, &st) < 0) + return -1; + /* trivial sanity check: at least toc should be here */ + if (st.st_size < 2048) + return errno = EPROTO, -1; + fsize = (unsigned) (st.st_size & 0xffffffffu); + /* memory-map file */ +#ifdef _WIN32 + hFile = (HANDLE) _get_osfhandle(fd); + if (hFile == (HANDLE) -1) + return -1; + hMapping = CreateFileMapping(hFile, NULL, PAGE_READONLY, 0, 0, NULL); + if (!hMapping) + return -1; + mem = (unsigned char *)MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0); + CloseHandle(hMapping); + if (!mem) + return -1; +#else + mem = (unsigned char*) mmap (NULL, fsize, PROT_READ, MAP_SHARED, fd, 0); + if (mem == MAP_FAILED) + return -1; +#endif /* _WIN32 */ + + cdbp->cdb_fd = fd; + cdbp->cdb_fsize = fsize; + cdbp->cdb_mem = mem; + + + cdbp->cdb_vpos = cdbp->cdb_vlen = 0; + cdbp->cdb_kpos = cdbp->cdb_klen = 0; + dend = cdb_unpack (mem); + if (dend < 2048) + dend = 2048; + else if (dend >= fsize) + dend = fsize; + cdbp->cdb_dend = dend; + + return 0; +} + +void +cdb_free(struct cdb *cdbp) +{ + if (cdbp->cdb_mem) { +#ifdef _WIN32 + UnmapViewOfFile((void*) cdbp->cdb_mem); +#else + munmap ((void*) cdbp->cdb_mem, cdbp->cdb_fsize); +#endif /* _WIN32 */ + cdbp->cdb_mem = NULL; + } + cdbp->cdb_fsize = 0; +} + +const void * +cdb_get(const struct cdb *cdbp, unsigned len, unsigned pos) +{ + if (pos > cdbp->cdb_fsize || cdbp->cdb_fsize - pos < len) { + errno = EPROTO; + return NULL; + } + return cdbp->cdb_mem + pos; +} + +int +cdb_read(const struct cdb *cdbp, void *buf, unsigned len, unsigned pos) +{ + const void *data = cdb_get (cdbp, len, pos); + if (!data) + return -1; + memcpy (buf, data, len); + return 0; +} diff --git a/src/cdb/cdb_make.c b/src/cdb/cdb_make.c new file mode 100644 index 000000000..b5ce28eeb --- /dev/null +++ b/src/cdb/cdb_make.c @@ -0,0 +1,524 @@ +/* $Id: cdb_init.c,v 1.12 2008-11-06 18:07:04 mjt Exp $ + * cdb_init, cdb_free and cdb_read routines + * + * This file is a part of tinycdb package by Michael Tokarev, mjt@corpit.ru. + * Public domain. + */ + +#include "cdb.h" + +void cdb_pack(unsigned num, unsigned char buf[4]) +{ + buf[0] = num & 255; + num >>= 8; + buf[1] = num & 255; + num >>= 8; + buf[2] = num & 255; + buf[3] = num >> 8; +} + +int cdb_make_start(struct cdb_make *cdbmp, int fd) +{ + memset (cdbmp, 0, sizeof(*cdbmp)); + cdbmp->cdb_fd = fd; + cdbmp->cdb_dpos = 2048; + cdbmp->cdb_bpos = cdbmp->cdb_buf + 2048; + return 0; +} + +int +_cdb_make_fullwrite(int fd, const unsigned char *buf, unsigned len) +{ + while (len) { + int l = write (fd, buf, len); + if (l > 0) { + len -= l; + buf += l; + } + else if (l < 0 && errno != EINTR) + return -1; + } + return 0; +} + +int +_cdb_make_flush(struct cdb_make *cdbmp) +{ + unsigned len = cdbmp->cdb_bpos - cdbmp->cdb_buf; + if (len) { + if (_cdb_make_fullwrite (cdbmp->cdb_fd, cdbmp->cdb_buf, len) < 0) + return -1; + cdbmp->cdb_bpos = cdbmp->cdb_buf; + } + return 0; +} + +int +_cdb_make_write(struct cdb_make *cdbmp, const unsigned char *ptr, unsigned len) +{ + unsigned l = sizeof(cdbmp->cdb_buf) - (cdbmp->cdb_bpos - cdbmp->cdb_buf); + cdbmp->cdb_dpos += len; + if (len > l) { + memcpy (cdbmp->cdb_bpos, ptr, l); + cdbmp->cdb_bpos += l; + if (_cdb_make_flush (cdbmp) < 0) + return -1; + ptr += l; + len -= l; + l = len / sizeof(cdbmp->cdb_buf); + if (l) { + l *= sizeof(cdbmp->cdb_buf); + if (_cdb_make_fullwrite (cdbmp->cdb_fd, ptr, l) < 0) + return -1; + ptr += l; + len -= l; + } + } + if (len) { + memcpy (cdbmp->cdb_bpos, ptr, len); + cdbmp->cdb_bpos += len; + } + return 0; +} + +static int cdb_make_finish_internal(struct cdb_make *cdbmp) +{ + unsigned hcnt[256]; /* hash table counts */ + unsigned hpos[256]; /* hash table positions */ + struct cdb_rec *htab; + unsigned char *p; + struct cdb_rl *rl; + unsigned hsize; + unsigned t, i; + + if (((0xffffffff - cdbmp->cdb_dpos) >> 3) < cdbmp->cdb_rcnt) + return errno = ENOMEM, -1; + + /* count htab sizes and reorder reclists */ + hsize = 0; + for (t = 0; t < 256; ++t) { + struct cdb_rl *rlt = NULL; + i = 0; + rl = cdbmp->cdb_rec[t]; + while (rl) { + struct cdb_rl *rln = rl->next; + rl->next = rlt; + rlt = rl; + i += rl->cnt; + rl = rln; + } + cdbmp->cdb_rec[t] = rlt; + if (hsize < (hcnt[t] = i << 1)) + hsize = hcnt[t]; + } + + /* allocate memory to hold max htable */ + htab = (struct cdb_rec*) malloc ((hsize + 2) * sizeof(struct cdb_rec)); + if (!htab) + return errno = ENOENT, -1; + p = (unsigned char *) htab; + htab += 2; + + /* build hash tables */ + for (t = 0; t < 256; ++t) { + unsigned len, hi; + hpos[t] = cdbmp->cdb_dpos; + if ((len = hcnt[t]) == 0) + continue; + for (i = 0; i < len; ++i) + htab[i].hval = htab[i].rpos = 0; + for (rl = cdbmp->cdb_rec[t]; rl; rl = rl->next) + for (i = 0; i < rl->cnt; ++i) { + hi = (rl->rec[i].hval >> 8) % len; + while (htab[hi].rpos) + if (++hi == len) + hi = 0; + htab[hi] = rl->rec[i]; + } + for (i = 0; i < len; ++i) { + cdb_pack (htab[i].hval, p + (i << 3)); + cdb_pack (htab[i].rpos, p + (i << 3) + 4); + } + if (_cdb_make_write (cdbmp, p, len << 3) < 0) { + free (p); + return -1; + } + } + free (p); + if (_cdb_make_flush (cdbmp) < 0) + return -1; + p = cdbmp->cdb_buf; + for (t = 0; t < 256; ++t) { + cdb_pack (hpos[t], p + (t << 3)); + cdb_pack (hcnt[t], p + (t << 3) + 4); + } + if (lseek (cdbmp->cdb_fd, 0, 0) != 0 || _cdb_make_fullwrite (cdbmp->cdb_fd, + p, 2048) != 0) + return -1; + + return 0; +} + +static void cdb_make_free(struct cdb_make *cdbmp) +{ + unsigned t; + for (t = 0; t < 256; ++t) { + struct cdb_rl *rl = cdbmp->cdb_rec[t]; + while (rl) { + struct cdb_rl *tm = rl; + rl = rl->next; + free (tm); + } + } +} + +int cdb_make_finish(struct cdb_make *cdbmp) +{ + int r = cdb_make_finish_internal (cdbmp); + cdb_make_free (cdbmp); + return r; +} + +int +_cdb_make_add(struct cdb_make *cdbmp, unsigned hval, const void *key, + unsigned klen, const void *val, unsigned vlen) +{ + unsigned char rlen[8]; + struct cdb_rl *rl; + unsigned i; + if (klen > 0xffffffff - (cdbmp->cdb_dpos + 8) || vlen > 0xffffffff + - (cdbmp->cdb_dpos + klen + 8)) + return errno = ENOMEM, -1; + i = hval & 255; + rl = cdbmp->cdb_rec[i]; + if (!rl || rl->cnt >= sizeof(rl->rec) / sizeof(rl->rec[0])) { + rl = (struct cdb_rl*) malloc (sizeof(struct cdb_rl)); + if (!rl) + return errno = ENOMEM, -1; + rl->cnt = 0; + rl->next = cdbmp->cdb_rec[i]; + cdbmp->cdb_rec[i] = rl; + } + i = rl->cnt++; + rl->rec[i].hval = hval; + rl->rec[i].rpos = cdbmp->cdb_dpos; + ++cdbmp->cdb_rcnt; + cdb_pack (klen, rlen); + cdb_pack (vlen, rlen + 4); + if (_cdb_make_write (cdbmp, rlen, 8) < 0 || _cdb_make_write (cdbmp, key, + klen) < 0 || _cdb_make_write (cdbmp, val, vlen) < 0) + return -1; + return 0; +} + +int cdb_make_add(struct cdb_make *cdbmp, const void *key, unsigned klen, + const void *val, unsigned vlen) +{ + return _cdb_make_add (cdbmp, cdb_hash (key, klen), key, klen, val, vlen); +} + +static void fixup_rpos(struct cdb_make *cdbmp, unsigned rpos, unsigned rlen) +{ + unsigned i; + struct cdb_rl *rl; + register struct cdb_rec *rp, *rs; + for (i = 0; i < 256; ++i) { + for (rl = cdbmp->cdb_rec[i]; rl; rl = rl->next) + for (rs = rl->rec, rp = rs + rl->cnt; --rp >= rs;) + if (rp->rpos <= rpos) + goto nexthash; + else + rp->rpos -= rlen; + nexthash: ; + } +} + +static int remove_record(struct cdb_make *cdbmp, unsigned rpos, unsigned rlen) +{ + unsigned pos, len; + int r, fd; + + len = cdbmp->cdb_dpos - rpos - rlen; + cdbmp->cdb_dpos -= rlen; + if (!len) + return 0; /* it was the last record, nothing to do */ + pos = rpos; + fd = cdbmp->cdb_fd; + do { + r = len > sizeof(cdbmp->cdb_buf) ? sizeof(cdbmp->cdb_buf) : len; + if (lseek (fd, pos + rlen, SEEK_SET) < 0 || (r = read (fd, + cdbmp->cdb_buf, r)) <= 0) + return -1; + if (lseek (fd, pos, SEEK_SET) < 0 || _cdb_make_fullwrite (fd, + cdbmp->cdb_buf, r) < 0) + return -1; + pos += r; + len -= r; + } while (len); + g_assert (cdbmp->cdb_dpos == pos); + fixup_rpos (cdbmp, rpos, rlen); + return 0; +} + +static int zerofill_record(struct cdb_make *cdbmp, unsigned rpos, unsigned rlen) +{ + if (rpos + rlen == cdbmp->cdb_dpos) { + cdbmp->cdb_dpos = rpos; + return 0; + } + if (lseek (cdbmp->cdb_fd, rpos, SEEK_SET) < 0) + return -1; + memset (cdbmp->cdb_buf, 0, sizeof(cdbmp->cdb_buf)); + cdb_pack (rlen - 8, cdbmp->cdb_buf + 4); + for (;;) { + rpos = rlen > sizeof(cdbmp->cdb_buf) ? sizeof(cdbmp->cdb_buf) : rlen; + if (_cdb_make_fullwrite (cdbmp->cdb_fd, cdbmp->cdb_buf, rpos) < 0) + return -1; + rlen -= rpos; + if (!rlen) + return 0; + memset (cdbmp->cdb_buf + 4, 0, 4); + } +} + +/* return: 0 = not found, 1 = error, or record length */ +static unsigned match(struct cdb_make *cdbmp, unsigned pos, const char *key, + unsigned klen) +{ + int len; + unsigned rlen; + if (lseek (cdbmp->cdb_fd, pos, SEEK_SET) < 0) + return 1; + if (read (cdbmp->cdb_fd, cdbmp->cdb_buf, 8) != 8) + return 1; + if (cdb_unpack (cdbmp->cdb_buf) != klen) + return 0; + + /* record length; check its validity */ + rlen = cdb_unpack (cdbmp->cdb_buf + 4); + if (rlen > cdbmp->cdb_dpos - pos - klen - 8) + return errno = EPROTO, 1; /* someone changed our file? */ + rlen += klen + 8; + + while (klen) { + len = klen > sizeof(cdbmp->cdb_buf) ? sizeof(cdbmp->cdb_buf) : klen; + len = read (cdbmp->cdb_fd, cdbmp->cdb_buf, len); + if (len <= 0) + return 1; + if (memcmp (cdbmp->cdb_buf, key, len) != 0) + return 0; + key += len; + klen -= len; + } + + return rlen; +} + +static int findrec(struct cdb_make *cdbmp, const void *key, unsigned klen, + unsigned hval, enum cdb_put_mode mode) +{ + struct cdb_rl *rl; + struct cdb_rec *rp, *rs; + unsigned r; + int seeked = 0; + int ret = 0; + for (rl = cdbmp->cdb_rec[hval & 255]; rl; rl = rl->next) + for (rs = rl->rec, rp = rs + rl->cnt; --rp >= rs;) { + if (rp->hval != hval) + continue; + /*XXX this explicit flush may be unnecessary having + * smarter match() that looks into cdb_buf too, but + * most of a time here spent in finding hash values + * (above), not keys */ + if (!seeked && _cdb_make_flush (cdbmp) < 0) + return -1; + seeked = 1; + r = match (cdbmp, rp->rpos, key, klen); + if (!r) + continue; + if (r == 1) + return -1; + ret = 1; + switch (mode) + { + case CDB_FIND_REMOVE: + if (remove_record (cdbmp, rp->rpos, r) < 0) + return -1; + break; + case CDB_FIND_FILL0: + if (zerofill_record (cdbmp, rp->rpos, r) < 0) + return -1; + break; + default: + goto finish; + } + memmove (rp, rp + 1, (rs + rl->cnt - 1 - rp) * sizeof(*rp)); + --rl->cnt; + --cdbmp->cdb_rcnt; + } + finish: if (seeked && lseek (cdbmp->cdb_fd, cdbmp->cdb_dpos, SEEK_SET) < 0) + return -1; + return ret; +} + +int cdb_make_find(struct cdb_make *cdbmp, const void *key, unsigned klen, + enum cdb_put_mode mode) +{ + return findrec (cdbmp, key, klen, cdb_hash (key, klen), mode); +} + +int cdb_make_exists(struct cdb_make *cdbmp, const void *key, unsigned klen) +{ + return cdb_make_find (cdbmp, key, klen, CDB_FIND); +} + +int cdb_make_put(struct cdb_make *cdbmp, const void *key, unsigned klen, + const void *val, unsigned vlen, enum cdb_put_mode mode) +{ + unsigned hval = cdb_hash (key, klen); + int r; + + switch (mode) + { + case CDB_PUT_REPLACE: + case CDB_PUT_INSERT: + case CDB_PUT_WARN: + case CDB_PUT_REPLACE0: + r = findrec (cdbmp, key, klen, hval, mode); + if (r < 0) + return -1; + if (r && mode == CDB_PUT_INSERT) + return errno = EEXIST, 1; + break; + + case CDB_PUT_ADD: + r = 0; + break; + + default: + return errno = EINVAL, -1; + } + + if (_cdb_make_add (cdbmp, hval, key, klen, val, vlen) < 0) + return -1; + + return r; +} + +unsigned +cdb_unpack(const unsigned char buf[4]) +{ + unsigned n = buf[3]; + n <<= 8; n |= buf[2]; + n <<= 8; n |= buf[1]; + n <<= 8; n |= buf[0]; + return n; +} + +int +cdb_seqnext(unsigned *cptr, struct cdb *cdbp) { + unsigned klen, vlen; + unsigned pos = *cptr; + unsigned dend = cdbp->cdb_dend; + const unsigned char *mem = cdbp->cdb_mem; + if (pos > dend - 8) + return 0; + klen = cdb_unpack(mem + pos); + vlen = cdb_unpack(mem + pos + 4); + pos += 8; + if (dend - klen < pos || dend - vlen < pos + klen) + return errno = EPROTO, -1; + cdbp->cdb_kpos = pos; + cdbp->cdb_klen = klen; + cdbp->cdb_vpos = pos + klen; + cdbp->cdb_vlen = vlen; + *cptr = pos + klen + vlen; + return 1; +} + +/* read a chunk from file, ignoring interrupts (EINTR) */ + +int +cdb_bread(int fd, void *buf, int len) +{ + int l; + while(len > 0) { + do l = read(fd, buf, len); + while(l < 0 && errno == EINTR); + if (l <= 0) { + if (!l) + errno = EIO; + return -1; + } + buf = (char*)buf + l; + len -= l; + } + return 0; +} + +/* find a given key in cdb file, seek a file pointer to it's value and + place data length to *dlenp. */ + +int +cdb_seek(int fd, const void *key, unsigned klen, unsigned *dlenp) +{ + unsigned htstart; /* hash table start position */ + unsigned htsize; /* number of elements in a hash table */ + unsigned httodo; /* hash table elements left to look */ + unsigned hti; /* hash table index */ + unsigned pos; /* position in a file */ + unsigned hval; /* key's hash value */ + unsigned char rbuf[64]; /* read buffer */ + int needseek = 1; /* if we should seek to a hash slot */ + + hval = cdb_hash(key, klen); + pos = (hval & 0xff) << 3; /* position in TOC */ + /* read the hash table parameters */ + if (lseek(fd, pos, SEEK_SET) < 0 || cdb_bread(fd, rbuf, 8) < 0) + return -1; + if ((htsize = cdb_unpack(rbuf + 4)) == 0) + return 0; + hti = (hval >> 8) % htsize; /* start position in hash table */ + httodo = htsize; + htstart = cdb_unpack(rbuf); + + for(;;) { + if (needseek && lseek(fd, htstart + (hti << 3), SEEK_SET) < 0) + return -1; + if (cdb_bread(fd, rbuf, 8) < 0) + return -1; + if ((pos = cdb_unpack(rbuf + 4)) == 0) /* not found */ + return 0; + + if (cdb_unpack(rbuf) != hval) /* hash value not matched */ + needseek = 0; + else { /* hash value matched */ + if (lseek(fd, pos, SEEK_SET) < 0 || cdb_bread(fd, rbuf, 8) < 0) + return -1; + if (cdb_unpack(rbuf) == klen) { /* key length matches */ + /* read the key from file and compare with wanted */ + unsigned l = klen, c; + const char *k = (const char*)key; + if (dlenp) + *dlenp = cdb_unpack(rbuf + 4); /* save value length */ + for(;;) { + if (!l) /* the whole key read and matches, return */ + return 1; + c = l > sizeof(rbuf) ? sizeof(rbuf) : l; + if (cdb_bread(fd, rbuf, c) < 0) + return -1; + if (memcmp(rbuf, k, c) != 0) /* no, it differs, stop here */ + break; + k += c; l -= c; + } + } + needseek = 1; /* we're looked to other place, should seek back */ + } + if (!--httodo) + return 0; + if (++hti == htsize) { + hti = 0; + needseek = 1; + } + } +} diff --git a/src/lua/CMakeLists.txt b/src/lua/CMakeLists.txt index 3ec763539..63ecfe42b 100644 --- a/src/lua/CMakeLists.txt +++ b/src/lua/CMakeLists.txt @@ -5,7 +5,8 @@ SET(LUASRC lua_common.c lua_config.c lua_classifier.c lua_cfg_file.c - lua_regexp.c) + lua_regexp.c + lua_cdb.c) ADD_LIBRARY(rspamd_lua STATIC ${LUASRC}) TARGET_LINK_LIBRARIES(rspamd_lua ${LUALIB}) diff --git a/src/lua/lua_cdb.c b/src/lua/lua_cdb.c new file mode 100644 index 000000000..ef6041d7c --- /dev/null +++ b/src/lua/lua_cdb.c @@ -0,0 +1,152 @@ +/* Copyright (c) 2010, Vsevolod Stakhov + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL Rambler BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "lua_common.h" +#include "../cdb/cdb.h" + +LUA_FUNCTION_DEF (cdb, create); +LUA_FUNCTION_DEF (cdb, lookup); +LUA_FUNCTION_DEF (cdb, get_name); +LUA_FUNCTION_DEF (cdb, destroy); + +static const struct luaL_reg cdblib_m[] = { + LUA_INTERFACE_DEF (cdb, lookup), + LUA_INTERFACE_DEF (cdb, get_name), + {"__tostring", lua_class_tostring}, + {"__gc", lua_cdb_destroy}, + {NULL, NULL} +}; +static const struct luaL_reg cdblib_f[] = { + LUA_INTERFACE_DEF (cdb, create), + {NULL, NULL} +}; + +static struct cdb * +lua_check_cdb (lua_State * L) +{ + void *ud = luaL_checkudata (L, 1, "rspamd{cdb}"); + + luaL_argcheck (L, ud != NULL, 1, "'cdb' expected"); + return *((struct cdb **)ud); +} + +static gint +lua_cdb_create (lua_State *L) +{ + struct cdb *cdb, **pcdb; + const gchar *filename; + gint fd; + + filename = luaL_checkstring (L, 1); + /* If file begins with cdb://, just skip it */ + if (g_ascii_strncasecmp (filename, "cdb://", sizeof ("cdb://") - 1) == 0) { + filename += sizeof ("cdb://") - 1; + } + + if ((fd = open (filename, O_RDONLY)) == -1) { + msg_warn ("cannot open cdb: %s, %s", filename, strerror (errno)); + lua_pushnil (L); + } + else { + cdb = g_malloc (sizeof (struct cdb)); + cdb->filename = g_strdup (filename); + if (cdb_init (cdb, fd) == -1) { + msg_warn ("cannot open cdb: %s, %s", filename, strerror (errno)); + lua_pushnil (L); + } + else { + pcdb = lua_newuserdata (L, sizeof (struct cdb*)); + lua_setclass (L, "rspamd{cdb}", -1); + *pcdb = cdb; + } + } + + return 1; +} + +static gint +lua_cdb_get_name (lua_State *L) +{ + struct cdb *cdb = lua_check_cdb (L); + + lua_pushstring (L, cdb->filename); + return 1; +} + +static gint +lua_cdb_lookup (lua_State *L) +{ + struct cdb *cdb = lua_check_cdb (L); + const gchar *what; + gchar *value; + gsize vlen; + goffset vpos; + + what = luaL_checkstring (L, 2); + if (cdb_find (cdb, what, strlen (what)) > 0) { + /* Extract and push value to lua as string */ + vpos = cdb_datapos (cdb); + vlen = cdb_datalen (cdb); + value = g_malloc (vlen); + cdb_read (cdb, value, vlen, vpos); + lua_pushlstring (L, value, vlen); + g_free (value); + } + else { + lua_pushnil (L); + } + + return 1; +} + +static gint +lua_cdb_destroy (lua_State *L) +{ + struct cdb *cdb = lua_check_cdb (L); + + if (cdb) { + cdb_free (cdb); + g_free (cdb->filename); + g_free (cdb); + } + + return 0; +} + +gint +luaopen_cdb (lua_State * L) +{ + luaL_newmetatable (L, "rspamd{cdb}"); + lua_pushstring (L, "__index"); + lua_pushvalue (L, -2); + lua_settable (L, -3); + + lua_pushstring (L, "class"); + lua_pushstring (L, "rspamd{cdb}"); + lua_rawset (L, -3); + + luaL_openlib (L, NULL, cdblib_m, 0); + luaL_openlib(L, "cdb", cdblib_f, 0); + + return 1; +} diff --git a/src/lua/lua_common.c b/src/lua/lua_common.c index 1f9f7285d..ada44981b 100644 --- a/src/lua/lua_common.c +++ b/src/lua/lua_common.c @@ -238,6 +238,7 @@ init_lua (struct config_file *cfg) (void)luaopen_classifier (L); (void)luaopen_statfile (L); (void)luaopen_glib_regexp (L); + (void)luaopen_cdb (L); cfg->lua_state = L; memory_pool_add_destructor (cfg->cfg_pool, (pool_destruct_func)lua_close, L); diff --git a/src/lua/lua_common.h b/src/lua/lua_common.h index f6b69e0e0..9b0c356f3 100644 --- a/src/lua/lua_common.h +++ b/src/lua/lua_common.h @@ -16,7 +16,7 @@ extern const luaL_reg null_reg[]; -#define RSPAMD_LUA_API_VERSION 2 +#define RSPAMD_LUA_API_VERSION 3 /* Common utility functions */ void lua_newclass (lua_State *L, const gchar *classname, const struct luaL_reg *func); @@ -36,6 +36,7 @@ gint luaopen_url (lua_State *L); gint luaopen_classifier (lua_State *L); gint luaopen_statfile (lua_State * L); gint luaopen_glib_regexp (lua_State *L); +gint luaopen_cdb (lua_State *L); void init_lua (struct config_file *cfg); gboolean init_lua_filters (struct config_file *cfg); diff --git a/src/plugins/lua/multimap.lua b/src/plugins/lua/multimap.lua index a4ef3c0a5..699d2cc75 100644 --- a/src/plugins/lua/multimap.lua +++ b/src/plugins/lua/multimap.lua @@ -43,9 +43,16 @@ end function check_multimap(task) for _,rule in ipairs(rules) do if rule['type'] == 'ip' then - local ip = task:get_from_ip_num() - if rule['ips']:get_key(ip) then - task:insert_result(rule['symbol'], 1) + if rule['cdb'] then + local ip = task:get_from_ip() + if rule['hash']:lookup(ip) then + task:insert_result(rule['symbol'], 1) + end + else + local ip = task:get_from_ip_num() + if rule['ips']:get_key(ip) then + task:insert_result(rule['symbol'], 1) + end end elseif rule['type'] == 'header' then local headers = task:get_message():get_header(rule['header']) @@ -55,13 +62,25 @@ function check_multimap(task) -- extract a part from header local _,_,ext = string.find(hv, rule['pattern']) if ext then - if rule['hash']:get_key(ext) then - task:insert_result(rule['symbol'], 1) + if rule['cdb'] then + if rule['hash']:lookup(ext) then + task:insert_result(rule['symbol'], 1) + end + else + if rule['hash']:get_key(ext) then + task:insert_result(rule['symbol'], 1) + end end end else - if rule['hash']:get_key(hv) then - task:insert_result(rule['symbol'], 1) + if rule['cdb'] then + if rule['hash']:lookup(hv) then + task:insert_result(rule['symbol'], 1) + end + else + if rule['hash']:get_key(hv) then + task:insert_result(rule['symbol'], 1) + end end end end @@ -83,13 +102,25 @@ function check_multimap(task) -- extract a part from header local _,_,ext = string.find(r['addr'], rule['pattern']) if ext then - if rule['hash']:get_key(ext) then - task:insert_result(rule['symbol'], 1) + if rule['cdb'] then + if rule['hash']:lookup(hv) then + task:insert_result(rule['symbol'], 1) + end + else + if rule['hash']:get_key(hv) then + task:insert_result(rule['symbol'], 1) + end end end else - if rule['hash']:get_key(r['addr']) then - task:insert_result(rule['symbol'], 1) + if rule['cdb'] then + if rule['hash']:lookup(r['addr']) then + task:insert_result(rule['symbol'], 1) + end + else + if rule['hash']:get_key(r['addr']) then + task:insert_result(rule['symbol'], 1) + end end end end @@ -104,13 +135,25 @@ function check_multimap(task) -- extract a part from header local _,_,ext = string.find(r['addr'], rule['pattern']) if ext then - if rule['hash']:get_key(ext) then - task:insert_result(rule['symbol'], 1) + if rule['cdb'] then + if rule['hash']:lookup(ext) then + task:insert_result(rule['symbol'], 1) + end + else + if rule['hash']:get_key(ext) then + task:insert_result(rule['symbol'], 1) + end end end else - if rule['hash']:get_key(r['addr']) then - task:insert_result(rule['symbol'], 1) + if rule['cdb'] then + if rule['hash']:lookup(r['addr']) then + task:insert_result(rule['symbol'], 1) + end + else + if rule['hash']:get_key(r['addr']) then + task:insert_result(rule['symbol'], 1) + end end end end @@ -127,13 +170,25 @@ function check_multimap(task) -- extract a part from header local _,_,ext = string.find(r['addr'], rule['pattern']) if ext then - if rule['hash']:get_key(ext) then - task:insert_result(rule['symbol'], 1) + if rule['cdb'] then + if rule['hash']:lookup(ext) then + task:insert_result(rule['symbol'], 1) + end + else + if rule['hash']:get_key(ext) then + task:insert_result(rule['symbol'], 1) + end end end else - if rule['hash']:get_key(r['addr']) then - task:insert_result(rule['symbol'], 1) + if rule['cdb'] then + if rule['hash']:lookup(r['addr']) then + task:insert_result(rule['symbol'], 1) + end + else + if rule['hash']:get_key(r['addr']) then + task:insert_result(rule['symbol'], 1) + end end end end @@ -148,13 +203,25 @@ function check_multimap(task) -- extract a part from header local _,_,ext = string.find(r['addr'], rule['pattern']) if ext then - if rule['hash']:get_key(ext) then - task:insert_result(rule['symbol'], 1) + if rule['cdb'] then + if rule['hash']:lookup(ext) then + task:insert_result(rule['symbol'], 1) + end + else + if rule['hash']:get_key(ext) then + task:insert_result(rule['symbol'], 1) + end end end else - if rule['hash']:get_key(r['addr']) then - task:insert_result(rule['symbol'], 1) + if rule['cdb'] then + if rule['hash']:lookup(r['addr']) then + task:insert_result(rule['symbol'], 1) + end + else + if rule['hash']:get_key(r['addr']) then + task:insert_result(rule['symbol'], 1) + end end end end @@ -176,7 +243,7 @@ local function add_multimap_rule(params) for _,param in ipairs(params) do local _,_,name,value = string.find(param, '(%w+)%s*=%s*(.+)') if not name or not value then - rspamd_logger:err('invalid rule: '..param) + rspamd_logger.err('invalid rule: '..param) return nil end if name == 'type' then @@ -191,7 +258,7 @@ local function add_multimap_rule(params) elseif value == 'from' then newrule['type'] = 'from' else - rspamd_logger:err('invalid rule type: '.. value) + rspamd_logger.err('invalid rule type: '.. value) return nil end elseif name == 'header' then @@ -202,32 +269,51 @@ local function add_multimap_rule(params) newrule['map'] = value elseif name == 'symbol' then newrule['symbol'] = value - else - rspamd_logger:err('invalid rule option: '.. name) + else + rspamd_logger.err('invalid rule option: '.. name) return nil end end if not newrule['symbol'] or not newrule['map'] then - rspamd_logger:err('incomplete rule') + rspamd_logger.err('incomplete rule') return nil end - if newrule['type'] == 'ip' then - newrule['ips'] = rspamd_config:add_radix_map (newrule['map']) - if newrule['ips'] then - table.insert(rules, newrule) - else - rspamd_logger.warn('Cannot add rule: map doesn\'t exists: ' .. newrule['map']) - end - elseif newrule['type'] == 'header' or newrule['type'] == 'rcpt' or newrule['type'] == 'from' then - newrule['hash'] = rspamd_config:add_hash_map (newrule['map']) + -- Check cdb flag + if string.find(newrule['map'], '^cdb://.*$') then + local test = cdb.create(newrule['map']) + newrule['hash'] = cdb.create(newrule['map']) if newrule['hash'] then table.insert(rules, newrule) else rspamd_logger.warn('Cannot add rule: map doesn\'t exists: ' .. newrule['map']) end + newrule['cdb'] = true else - table.insert(rules, newrule) + if newrule['type'] == 'ip' then + newrule['ips'] = rspamd_config:add_radix_map (newrule['map']) + if newrule['ips'] then + table.insert(rules, newrule) + else + rspamd_logger.warn('Cannot add rule: map doesn\'t exists: ' .. newrule['map']) + end + elseif newrule['type'] == 'header' or newrule['type'] == 'rcpt' or newrule['type'] == 'from' then + newrule['hash'] = rspamd_config:add_hash_map (newrule['map']) + if newrule['hash'] then + table.insert(rules, newrule) + else + rspamd_logger.warn('Cannot add rule: map doesn\'t exists: ' .. newrule['map']) + end + elseif newrule['type'] == 'cdb' then + newrule['hash'] = rspamd_cdb.create(newrule['map']) + if newrule['hash'] then + table.insert(rules, newrule) + else + rspamd_logger.warn('Cannot add rule: map doesn\'t exists: ' .. newrule['map']) + end + else + table.insert(rules, newrule) + end end return newrule end @@ -248,7 +334,7 @@ if opts then local params = split(value, ',') local rule = add_multimap_rule (params) if not rule then - rspamd_logger:err('cannot add rule: "'..value..'"') + rspamd_logger.err('cannot add rule: "'..value..'"') else if type(rspamd_config.get_api_version) ~= 'nil' then rspamd_config:register_virtual_symbol(rule['symbol'], 1.0) @@ -259,7 +345,7 @@ if opts then local params = split(strrules, ',') local rule = add_multimap_rule (params) if not rule then - rspamd_logger:err('cannot add rule: "'..strrules..'"') + rspamd_logger.err('cannot add rule: "'..strrules..'"') else if type(rspamd_config.get_api_version) ~= 'nil' then rspamd_config:register_virtual_symbol(rule['symbol'], 1.0) -- 2.39.5