Du kan inte välja fler än 25 ämnen Ämnen måste starta med en bokstav eller siffra, kan innehålla bindestreck ('-') och vara max 35 tecken långa.

fuzzy_storage.c 67KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683
  1. /*-
  2. * Copyright 2016 Vsevolod Stakhov
  3. *
  4. * Licensed under the Apache License, Version 2.0 (the "License");
  5. * you may not use this file except in compliance with the License.
  6. * You may obtain a copy of the License at
  7. *
  8. * http://www.apache.org/licenses/LICENSE-2.0
  9. *
  10. * Unless required by applicable law or agreed to in writing, software
  11. * distributed under the License is distributed on an "AS IS" BASIS,
  12. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. * See the License for the specific language governing permissions and
  14. * limitations under the License.
  15. */
  16. /*
  17. * Rspamd fuzzy storage server
  18. */
  19. #include "config.h"
  20. #include "libserver/fuzzy_wire.h"
  21. #include "util.h"
  22. #include "rspamd.h"
  23. #include "libserver/maps/map.h"
  24. #include "libserver/maps/map_helpers.h"
  25. #include "libserver/fuzzy_backend/fuzzy_backend.h"
  26. #include "ottery.h"
  27. #include "ref.h"
  28. #include "xxhash.h"
  29. #include "libserver/worker_util.h"
  30. #include "libserver/rspamd_control.h"
  31. #include "libcryptobox/cryptobox.h"
  32. #include "libcryptobox/keypairs_cache.h"
  33. #include "libcryptobox/keypair.h"
  34. #include "libutil/hash.h"
  35. #include "libserver/maps/map_private.h"
  36. #include "contrib/uthash/utlist.h"
  37. #include "lua/lua_common.h"
  38. #include "unix-std.h"
  39. #include <math.h>
  40. /* Resync value in seconds */
  41. #define DEFAULT_SYNC_TIMEOUT 60.0
  42. #define DEFAULT_KEYPAIR_CACHE_SIZE 512
  43. #define DEFAULT_MASTER_TIMEOUT 10.0
  44. #define DEFAULT_UPDATES_MAXFAIL 3
  45. #define COOKIE_SIZE 128
  46. #define DEFAULT_MAX_BUCKETS 2000
  47. #define DEFAULT_BUCKET_TTL 3600
  48. #define DEFAULT_BUCKET_MASK 24
  49. static const gchar *local_db_name = "local";
  50. #define msg_err_fuzzy_update(...) rspamd_default_log_function (G_LOG_LEVEL_CRITICAL, \
  51. session->name, session->uid, \
  52. RSPAMD_LOG_FUNC, \
  53. __VA_ARGS__)
  54. #define msg_warn_fuzzy_update(...) rspamd_default_log_function (G_LOG_LEVEL_WARNING, \
  55. session->name, session->uid, \
  56. RSPAMD_LOG_FUNC, \
  57. __VA_ARGS__)
  58. #define msg_info_fuzzy_update(...) rspamd_default_log_function (G_LOG_LEVEL_INFO, \
  59. session->name, session->uid, \
  60. RSPAMD_LOG_FUNC, \
  61. __VA_ARGS__)
  62. #define msg_err_fuzzy_collection(...) rspamd_default_log_function (G_LOG_LEVEL_CRITICAL, \
  63. "fuzzy_collection", session->uid, \
  64. RSPAMD_LOG_FUNC, \
  65. __VA_ARGS__)
  66. #define msg_warn_fuzzy_collection(...) rspamd_default_log_function (G_LOG_LEVEL_WARNING, \
  67. "fuzzy_collection", session->uid, \
  68. RSPAMD_LOG_FUNC, \
  69. __VA_ARGS__)
  70. #define msg_info_fuzzy_collection(...) rspamd_default_log_function (G_LOG_LEVEL_INFO, \
  71. "fuzzy_collection", session->uid, \
  72. RSPAMD_LOG_FUNC, \
  73. __VA_ARGS__)
  74. /* Init functions */
  75. gpointer init_fuzzy (struct rspamd_config *cfg);
  76. void start_fuzzy (struct rspamd_worker *worker);
  77. worker_t fuzzy_worker = {
  78. "fuzzy", /* Name */
  79. init_fuzzy, /* Init function */
  80. start_fuzzy, /* Start function */
  81. RSPAMD_WORKER_HAS_SOCKET,
  82. RSPAMD_WORKER_SOCKET_UDP, /* UDP socket */
  83. RSPAMD_WORKER_VER /* Version info */
  84. };
  85. struct fuzzy_global_stat {
  86. guint64 fuzzy_hashes;
  87. /**< number of fuzzy hashes stored */
  88. guint64 fuzzy_hashes_expired;
  89. /**< number of fuzzy hashes expired */
  90. guint64 fuzzy_hashes_checked[RSPAMD_FUZZY_EPOCH_MAX];
  91. /**< amount of check requests for each epoch */
  92. guint64 fuzzy_shingles_checked[RSPAMD_FUZZY_EPOCH_MAX];
  93. /**< amount of shingle check requests for each epoch */
  94. guint64 fuzzy_hashes_found[RSPAMD_FUZZY_EPOCH_MAX];
  95. /**< amount of invalid requests */
  96. guint64 invalid_requests;
  97. /**< amount of delayed hashes found */
  98. guint64 delayed_hashes;
  99. };
  100. struct fuzzy_key_stat {
  101. guint64 checked;
  102. guint64 matched;
  103. guint64 added;
  104. guint64 deleted;
  105. guint64 errors;
  106. rspamd_lru_hash_t *last_ips;
  107. ref_entry_t ref;
  108. };
  109. struct rspamd_fuzzy_mirror {
  110. gchar *name;
  111. struct upstream_list *u;
  112. struct rspamd_cryptobox_pubkey *key;
  113. };
  114. struct rspamd_leaky_bucket_elt {
  115. rspamd_inet_addr_t *addr;
  116. gdouble last;
  117. gdouble cur;
  118. };
  119. static const guint64 rspamd_fuzzy_storage_magic = 0x291a3253eb1b3ea5ULL;
  120. struct rspamd_fuzzy_storage_ctx {
  121. guint64 magic;
  122. /* Events base */
  123. struct ev_loop *event_loop;
  124. /* DNS resolver */
  125. struct rspamd_dns_resolver *resolver;
  126. /* Config */
  127. struct rspamd_config *cfg;
  128. /* END OF COMMON PART */
  129. struct fuzzy_global_stat stat;
  130. gdouble expire;
  131. gdouble sync_timeout;
  132. gdouble delay;
  133. struct rspamd_radix_map_helper *update_ips;
  134. struct rspamd_radix_map_helper *blocked_ips;
  135. struct rspamd_radix_map_helper *ratelimit_whitelist;
  136. struct rspamd_radix_map_helper *delay_whitelist;
  137. const ucl_object_t *update_map;
  138. const ucl_object_t *delay_whitelist_map;
  139. const ucl_object_t *blocked_map;
  140. const ucl_object_t *ratelimit_whitelist_map;
  141. guint keypair_cache_size;
  142. ev_timer stat_ev;
  143. ev_io peer_ev;
  144. /* Local keypair */
  145. struct rspamd_cryptobox_keypair *default_keypair; /* Bad clash, need for parse keypair */
  146. struct fuzzy_key *default_key;
  147. GHashTable *keys;
  148. gboolean encrypted_only;
  149. gboolean read_only;
  150. gboolean dedicated_update_worker;
  151. struct rspamd_keypair_cache *keypair_cache;
  152. struct rspamd_http_context *http_ctx;
  153. rspamd_lru_hash_t *errors_ips;
  154. rspamd_lru_hash_t *ratelimit_buckets;
  155. struct rspamd_fuzzy_backend *backend;
  156. GArray *updates_pending;
  157. guint updates_failed;
  158. guint updates_maxfail;
  159. /* Used to send data between workers */
  160. gint peer_fd;
  161. /* Ratelimits */
  162. guint leaky_bucket_ttl;
  163. guint leaky_bucket_mask;
  164. guint max_buckets;
  165. gboolean ratelimit_log_only;
  166. gdouble leaky_bucket_burst;
  167. gdouble leaky_bucket_rate;
  168. struct rspamd_worker *worker;
  169. const ucl_object_t *skip_map;
  170. struct rspamd_hash_map_helper *skip_hashes;
  171. gint lua_pre_handler_cbref;
  172. gint lua_post_handler_cbref;
  173. guchar cookie[COOKIE_SIZE];
  174. };
  175. enum fuzzy_cmd_type {
  176. CMD_NORMAL,
  177. CMD_SHINGLE,
  178. CMD_ENCRYPTED_NORMAL,
  179. CMD_ENCRYPTED_SHINGLE
  180. };
  181. struct fuzzy_session {
  182. struct rspamd_worker *worker;
  183. rspamd_inet_addr_t *addr;
  184. struct rspamd_fuzzy_storage_ctx *ctx;
  185. struct rspamd_fuzzy_shingle_cmd cmd; /* Can handle both shingles and non-shingles */
  186. struct rspamd_fuzzy_encrypted_reply reply; /* Again: contains everything */
  187. struct fuzzy_key_stat *ip_stat;
  188. enum rspamd_fuzzy_epoch epoch;
  189. enum fuzzy_cmd_type cmd_type;
  190. gint fd;
  191. guint64 time;
  192. struct ev_io io;
  193. ref_entry_t ref;
  194. struct fuzzy_key_stat *key_stat;
  195. struct rspamd_fuzzy_cmd_extension *extensions;
  196. guchar nm[rspamd_cryptobox_MAX_NMBYTES];
  197. };
  198. struct fuzzy_peer_request {
  199. ev_io io_ev;
  200. struct fuzzy_peer_cmd cmd;
  201. };
  202. struct fuzzy_key {
  203. struct rspamd_cryptobox_keypair *key;
  204. struct rspamd_cryptobox_pubkey *pk;
  205. struct fuzzy_key_stat *stat;
  206. };
  207. struct rspamd_updates_cbdata {
  208. GArray *updates_pending;
  209. struct rspamd_fuzzy_storage_ctx *ctx;
  210. gchar *source;
  211. gboolean final;
  212. };
  213. static void rspamd_fuzzy_write_reply (struct fuzzy_session *session);
  214. static gboolean rspamd_fuzzy_process_updates_queue (
  215. struct rspamd_fuzzy_storage_ctx *ctx,
  216. const gchar *source, gboolean final);
  217. static gboolean
  218. rspamd_fuzzy_check_ratelimit (struct fuzzy_session *session)
  219. {
  220. rspamd_inet_addr_t *masked;
  221. struct rspamd_leaky_bucket_elt *elt;
  222. ev_tstamp now;
  223. if (session->ctx->ratelimit_whitelist != NULL) {
  224. if (rspamd_match_radix_map_addr (session->ctx->ratelimit_whitelist,
  225. session->addr) != NULL) {
  226. return TRUE;
  227. }
  228. }
  229. /*
  230. if (rspamd_inet_address_is_local (session->addr, TRUE)) {
  231. return TRUE;
  232. }
  233. */
  234. masked = rspamd_inet_address_copy(session->addr, NULL);
  235. if (rspamd_inet_address_get_af (masked) == AF_INET) {
  236. rspamd_inet_address_apply_mask (masked,
  237. MIN (session->ctx->leaky_bucket_mask, 32));
  238. }
  239. else {
  240. /* Must be at least /64 */
  241. rspamd_inet_address_apply_mask (masked,
  242. MIN (MAX (session->ctx->leaky_bucket_mask * 4, 64), 128));
  243. }
  244. now = ev_now (session->ctx->event_loop);
  245. elt = rspamd_lru_hash_lookup (session->ctx->ratelimit_buckets, masked,
  246. now);
  247. if (elt) {
  248. gboolean ratelimited = FALSE;
  249. if (isnan (elt->cur)) {
  250. /* Ratelimit exceeded, preserve it for the whole ttl */
  251. ratelimited = TRUE;
  252. }
  253. else {
  254. /* Update bucket */
  255. if (elt->last < now) {
  256. elt->cur -= session->ctx->leaky_bucket_rate * (now - elt->last);
  257. elt->last = now;
  258. if (elt->cur < 0) {
  259. elt->cur = 0;
  260. }
  261. }
  262. else {
  263. elt->last = now;
  264. }
  265. /* Check bucket */
  266. if (elt->cur >= session->ctx->leaky_bucket_burst) {
  267. msg_info ("ratelimiting %s (%s), %.1f max elts",
  268. rspamd_inet_address_to_string (session->addr),
  269. rspamd_inet_address_to_string (masked),
  270. session->ctx->leaky_bucket_burst);
  271. elt->cur = NAN;
  272. }
  273. else {
  274. elt->cur ++; /* Allow one more request */
  275. }
  276. }
  277. rspamd_inet_address_free (masked);
  278. return !ratelimited;
  279. }
  280. else {
  281. /* New bucket */
  282. elt = g_malloc (sizeof (*elt));
  283. elt->addr = masked; /* transfer ownership */
  284. elt->cur = 1;
  285. elt->last = now;
  286. rspamd_lru_hash_insert (session->ctx->ratelimit_buckets,
  287. masked,
  288. elt,
  289. now,
  290. session->ctx->leaky_bucket_ttl);
  291. }
  292. return TRUE;
  293. }
  294. static gboolean
  295. rspamd_fuzzy_check_client (struct rspamd_fuzzy_storage_ctx *ctx,
  296. rspamd_inet_addr_t *addr)
  297. {
  298. if (ctx->blocked_ips != NULL) {
  299. if (rspamd_match_radix_map_addr (ctx->blocked_ips,
  300. addr) != NULL) {
  301. return FALSE;
  302. }
  303. }
  304. return TRUE;
  305. }
  306. static gboolean
  307. rspamd_fuzzy_check_write (struct fuzzy_session *session)
  308. {
  309. if (session->ctx->read_only) {
  310. return FALSE;
  311. }
  312. if (session->ctx->update_ips != NULL) {
  313. if (rspamd_match_radix_map_addr (session->ctx->update_ips,
  314. session->addr) == NULL) {
  315. return FALSE;
  316. }
  317. else {
  318. return TRUE;
  319. }
  320. }
  321. return FALSE;
  322. }
  323. static void
  324. fuzzy_key_stat_dtor (gpointer p)
  325. {
  326. struct fuzzy_key_stat *st = p;
  327. if (st->last_ips) {
  328. rspamd_lru_hash_destroy (st->last_ips);
  329. }
  330. g_free (st);
  331. }
  332. static void
  333. fuzzy_key_stat_unref (gpointer p)
  334. {
  335. struct fuzzy_key_stat *st = p;
  336. REF_RELEASE (st);
  337. }
  338. static void
  339. fuzzy_key_dtor (gpointer p)
  340. {
  341. struct fuzzy_key *key = p;
  342. if (key) {
  343. if (key->stat) {
  344. REF_RELEASE (key->stat);
  345. }
  346. g_free (key);
  347. }
  348. }
  349. static void
  350. fuzzy_count_callback (guint64 count, void *ud)
  351. {
  352. struct rspamd_fuzzy_storage_ctx *ctx = ud;
  353. ctx->stat.fuzzy_hashes = count;
  354. }
  355. static void
  356. fuzzy_rl_bucket_free (gpointer p)
  357. {
  358. struct rspamd_leaky_bucket_elt *elt = (struct rspamd_leaky_bucket_elt *)p;
  359. rspamd_inet_address_free (elt->addr);
  360. g_free (elt);
  361. }
  362. static void
  363. fuzzy_stat_count_callback (guint64 count, void *ud)
  364. {
  365. struct rspamd_fuzzy_storage_ctx *ctx = ud;
  366. ev_timer_again (ctx->event_loop, &ctx->stat_ev);
  367. ctx->stat.fuzzy_hashes = count;
  368. }
  369. static void
  370. rspamd_fuzzy_stat_callback (EV_P_ ev_timer *w, int revents)
  371. {
  372. struct rspamd_fuzzy_storage_ctx *ctx =
  373. (struct rspamd_fuzzy_storage_ctx *)w->data;
  374. rspamd_fuzzy_backend_count (ctx->backend, fuzzy_stat_count_callback, ctx);
  375. }
  376. static void
  377. fuzzy_update_version_callback (guint64 ver, void *ud)
  378. {
  379. }
  380. static void
  381. rspamd_fuzzy_updates_cb (gboolean success,
  382. guint nadded,
  383. guint ndeleted,
  384. guint nextended,
  385. guint nignored,
  386. void *ud)
  387. {
  388. struct rspamd_updates_cbdata *cbdata = ud;
  389. struct rspamd_fuzzy_storage_ctx *ctx;
  390. const gchar *source;
  391. ctx = cbdata->ctx;
  392. source = cbdata->source;
  393. if (success) {
  394. rspamd_fuzzy_backend_count(ctx->backend, fuzzy_count_callback, ctx);
  395. msg_info ("successfully updated fuzzy storage %s: %d updates in queue; "
  396. "%d pending currently; "
  397. "%d added; %d deleted; %d extended; %d duplicates",
  398. ctx->worker->cf->bind_conf ?
  399. ctx->worker->cf->bind_conf->bind_line :
  400. "unknown",
  401. cbdata->updates_pending->len,
  402. ctx->updates_pending->len,
  403. nadded, ndeleted, nextended, nignored);
  404. rspamd_fuzzy_backend_version(ctx->backend, source,
  405. fuzzy_update_version_callback, NULL);
  406. ctx->updates_failed = 0;
  407. if (cbdata->final || ctx->worker->state != rspamd_worker_state_running) {
  408. /* Plan exit */
  409. ev_break(ctx->event_loop, EVBREAK_ALL);
  410. }
  411. }
  412. else {
  413. if (++ctx->updates_failed > ctx->updates_maxfail) {
  414. msg_err ("cannot commit update transaction to fuzzy backend %s, discard "
  415. "%ud updates after %d retries",
  416. ctx->worker->cf->bind_conf ?
  417. ctx->worker->cf->bind_conf->bind_line :
  418. "unknown",
  419. cbdata->updates_pending->len,
  420. ctx->updates_maxfail);
  421. ctx->updates_failed = 0;
  422. if (cbdata->final || ctx->worker->state != rspamd_worker_state_running) {
  423. /* Plan exit */
  424. ev_break(ctx->event_loop, EVBREAK_ALL);
  425. }
  426. }
  427. else {
  428. if (ctx->updates_pending) {
  429. msg_err ("cannot commit update transaction to fuzzy backend %s; "
  430. "%ud updates are still left; %ud currently pending;"
  431. " %d retries remaining",
  432. ctx->worker->cf->bind_conf ?
  433. ctx->worker->cf->bind_conf->bind_line : "unknown",
  434. cbdata->updates_pending->len,
  435. ctx->updates_pending->len,
  436. ctx->updates_maxfail - ctx->updates_failed);
  437. /* Move the remaining updates to ctx queue */
  438. g_array_append_vals(ctx->updates_pending,
  439. cbdata->updates_pending->data,
  440. cbdata->updates_pending->len);
  441. if (cbdata->final) {
  442. /* Try one more time */
  443. rspamd_fuzzy_process_updates_queue(cbdata->ctx, cbdata->source,
  444. cbdata->final);
  445. }
  446. }
  447. else {
  448. /* We have lost our ctx, so it is a race condition case */
  449. msg_err ("cannot commit update transaction to fuzzy backend %s; "
  450. "%ud updates are still left; no more retries: a worker is terminated",
  451. ctx->worker->cf->bind_conf ?
  452. ctx->worker->cf->bind_conf->bind_line : "unknown",
  453. cbdata->updates_pending->len);
  454. }
  455. }
  456. }
  457. g_array_free (cbdata->updates_pending, TRUE);
  458. g_free (cbdata->source);
  459. g_free (cbdata);
  460. }
  461. static gboolean
  462. rspamd_fuzzy_process_updates_queue (struct rspamd_fuzzy_storage_ctx *ctx,
  463. const gchar *source, gboolean final)
  464. {
  465. struct rspamd_updates_cbdata *cbdata;
  466. if (ctx->updates_pending->len > 0) {
  467. cbdata = g_malloc (sizeof (*cbdata));
  468. cbdata->ctx = ctx;
  469. cbdata->final = final;
  470. cbdata->updates_pending = ctx->updates_pending;
  471. ctx->updates_pending = g_array_sized_new (FALSE, FALSE,
  472. sizeof (struct fuzzy_peer_cmd),
  473. MAX (cbdata->updates_pending->len, 1024));
  474. cbdata->source = g_strdup (source);
  475. rspamd_fuzzy_backend_process_updates (ctx->backend,
  476. cbdata->updates_pending,
  477. source, rspamd_fuzzy_updates_cb, cbdata);
  478. return TRUE;
  479. }
  480. else if (final) {
  481. /* No need to sync */
  482. ev_break (ctx->event_loop, EVBREAK_ALL);
  483. }
  484. return FALSE;
  485. }
  486. static void
  487. rspamd_fuzzy_reply_io (EV_P_ ev_io *w, int revents)
  488. {
  489. struct fuzzy_session *session = (struct fuzzy_session *)w->data;
  490. ev_io_stop (EV_A_ w);
  491. rspamd_fuzzy_write_reply (session);
  492. REF_RELEASE (session);
  493. }
  494. static void
  495. rspamd_fuzzy_write_reply (struct fuzzy_session *session)
  496. {
  497. gssize r;
  498. gsize len;
  499. gconstpointer data;
  500. if (session->cmd_type == CMD_ENCRYPTED_NORMAL ||
  501. session->cmd_type == CMD_ENCRYPTED_SHINGLE) {
  502. /* Encrypted reply */
  503. data = &session->reply;
  504. if (session->epoch > RSPAMD_FUZZY_EPOCH10) {
  505. len = sizeof (session->reply);
  506. }
  507. else {
  508. len = sizeof (session->reply.hdr) + sizeof (session->reply.rep.v1);
  509. }
  510. }
  511. else {
  512. data = &session->reply.rep;
  513. if (session->epoch > RSPAMD_FUZZY_EPOCH10) {
  514. len = sizeof (session->reply.rep);
  515. }
  516. else {
  517. len = sizeof (session->reply.rep.v1);
  518. }
  519. }
  520. r = rspamd_inet_address_sendto (session->fd, data, len, 0,
  521. session->addr);
  522. if (r == -1) {
  523. if (errno == EINTR || errno == EWOULDBLOCK || errno == EAGAIN) {
  524. /* Grab reference to avoid early destruction */
  525. REF_RETAIN (session);
  526. session->io.data = session;
  527. ev_io_init (&session->io,
  528. rspamd_fuzzy_reply_io, session->fd, EV_WRITE);
  529. ev_io_start (session->ctx->event_loop, &session->io);
  530. }
  531. else {
  532. msg_err ("error while writing reply: %s", strerror (errno));
  533. }
  534. }
  535. }
  536. static void
  537. rspamd_fuzzy_update_stats (struct rspamd_fuzzy_storage_ctx *ctx,
  538. enum rspamd_fuzzy_epoch epoch,
  539. gboolean matched,
  540. gboolean is_shingle,
  541. gboolean is_delayed,
  542. struct fuzzy_key_stat *key_stat,
  543. struct fuzzy_key_stat *ip_stat,
  544. guint cmd, guint reply)
  545. {
  546. ctx->stat.fuzzy_hashes_checked[epoch] ++;
  547. if (matched) {
  548. ctx->stat.fuzzy_hashes_found[epoch]++;
  549. }
  550. if (is_shingle) {
  551. ctx->stat.fuzzy_shingles_checked[epoch]++;
  552. }
  553. if (is_delayed) {
  554. ctx->stat.delayed_hashes ++;
  555. }
  556. if (key_stat) {
  557. if (!matched && reply != 0) {
  558. key_stat->errors ++;
  559. }
  560. else {
  561. if (cmd == FUZZY_CHECK) {
  562. key_stat->checked++;
  563. if (matched) {
  564. key_stat->matched ++;
  565. }
  566. }
  567. else if (cmd == FUZZY_WRITE) {
  568. key_stat->added++;
  569. }
  570. else if (cmd == FUZZY_DEL) {
  571. key_stat->deleted++;
  572. }
  573. }
  574. }
  575. if (ip_stat) {
  576. if (!matched && reply != 0) {
  577. ip_stat->errors++;
  578. }
  579. else {
  580. if (cmd == FUZZY_CHECK) {
  581. ip_stat->checked++;
  582. if (matched) {
  583. ip_stat->matched++;
  584. }
  585. }
  586. else if (cmd == FUZZY_WRITE) {
  587. ip_stat->added++;
  588. }
  589. else if (cmd == FUZZY_DEL) {
  590. ip_stat->deleted++;
  591. }
  592. }
  593. }
  594. }
  595. enum rspamd_fuzzy_reply_flags {
  596. RSPAMD_FUZZY_REPLY_ENCRYPTED = 0x1u << 0u,
  597. RSPAMD_FUZZY_REPLY_SHINGLE = 0x1u << 1u,
  598. RSPAMD_FUZZY_REPLY_DELAY = 0x1u << 2u,
  599. };
  600. static void
  601. rspamd_fuzzy_make_reply (struct rspamd_fuzzy_cmd *cmd,
  602. struct rspamd_fuzzy_reply *result,
  603. struct fuzzy_session *session,
  604. gint flags)
  605. {
  606. gsize len;
  607. if (cmd) {
  608. result->v1.tag = cmd->tag;
  609. memcpy (&session->reply.rep, result, sizeof (*result));
  610. rspamd_fuzzy_update_stats (session->ctx,
  611. session->epoch,
  612. result->v1.prob > 0.5,
  613. flags & RSPAMD_FUZZY_REPLY_SHINGLE,
  614. flags & RSPAMD_FUZZY_REPLY_DELAY,
  615. session->key_stat,
  616. session->ip_stat,
  617. cmd->cmd,
  618. result->v1.value);
  619. if (flags & RSPAMD_FUZZY_REPLY_DELAY) {
  620. /* Hash is too fresh, need to delay it */
  621. session->reply.rep.ts = 0;
  622. session->reply.rep.v1.prob = 0.0;
  623. session->reply.rep.v1.value = 0;
  624. }
  625. if (flags & RSPAMD_FUZZY_REPLY_ENCRYPTED) {
  626. /* We need also to encrypt reply */
  627. ottery_rand_bytes (session->reply.hdr.nonce,
  628. sizeof (session->reply.hdr.nonce));
  629. /*
  630. * For old replies we need to encrypt just old part, otherwise
  631. * decryption would fail due to mac verification mistake
  632. */
  633. if (session->epoch > RSPAMD_FUZZY_EPOCH10) {
  634. len = sizeof (session->reply.rep);
  635. }
  636. else {
  637. len = sizeof (session->reply.rep.v1);
  638. }
  639. rspamd_cryptobox_encrypt_nm_inplace ((guchar *)&session->reply.rep,
  640. len,
  641. session->reply.hdr.nonce,
  642. session->nm,
  643. session->reply.hdr.mac,
  644. RSPAMD_CRYPTOBOX_MODE_25519);
  645. }
  646. }
  647. rspamd_fuzzy_write_reply (session);
  648. }
  649. static gboolean
  650. fuzzy_peer_try_send (gint fd, struct fuzzy_peer_request *up_req)
  651. {
  652. gssize r;
  653. r = write (fd, &up_req->cmd, sizeof (up_req->cmd));
  654. if (r != sizeof (up_req->cmd)) {
  655. return FALSE;
  656. }
  657. return TRUE;
  658. }
  659. static void
  660. fuzzy_peer_send_io (EV_P_ ev_io *w, int revents)
  661. {
  662. struct fuzzy_peer_request *up_req = (struct fuzzy_peer_request *)w->data;
  663. if (!fuzzy_peer_try_send (w->fd, up_req)) {
  664. msg_err ("cannot send update request to the peer: %s", strerror (errno));
  665. }
  666. ev_io_stop (EV_A_ w);
  667. g_free (up_req);
  668. }
  669. static void
  670. rspamd_fuzzy_extensions_tolua (lua_State *L,
  671. struct fuzzy_session *session)
  672. {
  673. struct rspamd_fuzzy_cmd_extension *ext;
  674. rspamd_inet_addr_t *addr;
  675. lua_createtable (L, 0, 0);
  676. LL_FOREACH (session->extensions, ext) {
  677. switch (ext->ext) {
  678. case RSPAMD_FUZZY_EXT_SOURCE_DOMAIN:
  679. lua_pushlstring (L, ext->payload, ext->length);
  680. lua_setfield (L, -2, "domain");
  681. break;
  682. case RSPAMD_FUZZY_EXT_SOURCE_IP4:
  683. addr = rspamd_inet_address_new (AF_INET, ext->payload);
  684. rspamd_lua_ip_push (L, addr);
  685. rspamd_inet_address_free (addr);
  686. lua_setfield (L, -2, "ip");
  687. break;
  688. case RSPAMD_FUZZY_EXT_SOURCE_IP6:
  689. addr = rspamd_inet_address_new (AF_INET6, ext->payload);
  690. rspamd_lua_ip_push (L, addr);
  691. rspamd_inet_address_free (addr);
  692. lua_setfield (L, -2, "ip");
  693. break;
  694. }
  695. }
  696. }
  697. static void
  698. rspamd_fuzzy_check_callback (struct rspamd_fuzzy_reply *result, void *ud)
  699. {
  700. struct fuzzy_session *session = ud;
  701. gboolean is_shingle = FALSE, __attribute__ ((unused)) encrypted = FALSE;
  702. struct rspamd_fuzzy_cmd *cmd = NULL;
  703. const struct rspamd_shingle *shingle = NULL;
  704. struct rspamd_shingle sgl_cpy;
  705. gint send_flags = 0;
  706. switch (session->cmd_type) {
  707. case CMD_ENCRYPTED_NORMAL:
  708. encrypted = TRUE;
  709. send_flags |= RSPAMD_FUZZY_REPLY_ENCRYPTED;
  710. /* Fallthrough */
  711. case CMD_NORMAL:
  712. cmd = &session->cmd.basic;
  713. break;
  714. case CMD_ENCRYPTED_SHINGLE:
  715. encrypted = TRUE;
  716. send_flags |= RSPAMD_FUZZY_REPLY_ENCRYPTED;
  717. /* Fallthrough */
  718. case CMD_SHINGLE:
  719. cmd = &session->cmd.basic;
  720. memcpy (&sgl_cpy, &session->cmd.sgl, sizeof (sgl_cpy));
  721. shingle = &sgl_cpy;
  722. is_shingle = TRUE;
  723. send_flags |= RSPAMD_FUZZY_REPLY_SHINGLE;
  724. break;
  725. }
  726. if (session->ctx->lua_post_handler_cbref != -1) {
  727. /* Start lua post handler */
  728. lua_State *L = session->ctx->cfg->lua_state;
  729. gint err_idx, ret;
  730. lua_pushcfunction (L, &rspamd_lua_traceback);
  731. err_idx = lua_gettop (L);
  732. /* Preallocate stack (small opt) */
  733. lua_checkstack (L, err_idx + 9);
  734. /* function */
  735. lua_rawgeti (L, LUA_REGISTRYINDEX, session->ctx->lua_post_handler_cbref);
  736. /* client IP */
  737. rspamd_lua_ip_push (L, session->addr);
  738. /* client command */
  739. lua_pushinteger (L, cmd->cmd);
  740. /* command value (push as rspamd_text) */
  741. (void)lua_new_text (L, result->digest, sizeof (result->digest), FALSE);
  742. /* is shingle */
  743. lua_pushboolean (L, is_shingle);
  744. /* result value */
  745. lua_pushinteger (L, result->v1.value);
  746. /* result probability */
  747. lua_pushnumber (L, result->v1.prob);
  748. /* result flag */
  749. lua_pushinteger (L, result->v1.flag);
  750. /* result timestamp */
  751. lua_pushinteger (L, result->ts);
  752. /* TODO: add additional data maybe (encryption, pubkey, etc) */
  753. rspamd_fuzzy_extensions_tolua (L, session);
  754. if ((ret = lua_pcall (L, 9, LUA_MULTRET, err_idx)) != 0) {
  755. msg_err ("call to lua_post_handler lua "
  756. "script failed (%d): %s", ret, lua_tostring (L, -1));
  757. }
  758. else {
  759. /* Return values order:
  760. * the first reply will be on err_idx + 1
  761. * if it is true, then we need to read the former ones:
  762. * 2-nd will be reply code
  763. * 3-rd will be probability (or 0.0 if missing)
  764. * 4-th value is flag (or default flag if missing)
  765. */
  766. ret = lua_toboolean (L, err_idx + 1);
  767. if (ret) {
  768. /* Artificial reply */
  769. result->v1.value = lua_tointeger (L, err_idx + 2);
  770. if (lua_isnumber (L, err_idx + 3)) {
  771. result->v1.prob = lua_tonumber (L, err_idx + 3);
  772. }
  773. else {
  774. result->v1.prob = 0.0f;
  775. }
  776. if (lua_isnumber (L, err_idx + 4)) {
  777. result->v1.flag = lua_tointeger (L, err_idx + 4);
  778. }
  779. lua_settop (L, 0);
  780. rspamd_fuzzy_make_reply (cmd, result, session, send_flags);
  781. REF_RELEASE (session);
  782. return;
  783. }
  784. }
  785. lua_settop (L, 0);
  786. }
  787. if (!isnan (session->ctx->delay) &&
  788. rspamd_match_radix_map_addr (session->ctx->delay_whitelist,
  789. session->addr) == NULL) {
  790. gdouble hash_age = rspamd_get_calendar_ticks () - result->ts;
  791. gdouble jittered_age = rspamd_time_jitter (session->ctx->delay,
  792. session->ctx->delay / 2.0);
  793. if (hash_age < jittered_age) {
  794. send_flags |= RSPAMD_FUZZY_REPLY_DELAY;
  795. }
  796. }
  797. /* Refresh hash if found with strong confidence */
  798. if (result->v1.prob > 0.9 && !session->ctx->read_only) {
  799. struct fuzzy_peer_cmd up_cmd;
  800. struct fuzzy_peer_request *up_req;
  801. if (session->worker->index == 0) {
  802. /* Just add to the queue */
  803. memset (&up_cmd, 0, sizeof (up_cmd));
  804. up_cmd.is_shingle = is_shingle;
  805. memcpy (up_cmd.cmd.normal.digest, result->digest,
  806. sizeof (up_cmd.cmd.normal.digest));
  807. up_cmd.cmd.normal.flag = result->v1.flag;
  808. up_cmd.cmd.normal.cmd = FUZZY_REFRESH;
  809. up_cmd.cmd.normal.shingles_count = cmd->shingles_count;
  810. if (is_shingle && shingle) {
  811. memcpy (&up_cmd.cmd.shingle.sgl, shingle,
  812. sizeof (up_cmd.cmd.shingle.sgl));
  813. }
  814. g_array_append_val (session->ctx->updates_pending, up_cmd);
  815. }
  816. else {
  817. /* We need to send request to the peer */
  818. up_req = g_malloc0 (sizeof (*up_req));
  819. up_req->cmd.is_shingle = is_shingle;
  820. memcpy (up_req->cmd.cmd.normal.digest, result->digest,
  821. sizeof (up_req->cmd.cmd.normal.digest));
  822. up_req->cmd.cmd.normal.flag = result->v1.flag;
  823. up_req->cmd.cmd.normal.cmd = FUZZY_REFRESH;
  824. up_req->cmd.cmd.normal.shingles_count = cmd->shingles_count;
  825. if (is_shingle && shingle) {
  826. memcpy (&up_req->cmd.cmd.shingle.sgl, shingle,
  827. sizeof (up_req->cmd.cmd.shingle.sgl));
  828. }
  829. if (!fuzzy_peer_try_send (session->ctx->peer_fd, up_req)) {
  830. up_req->io_ev.data = up_req;
  831. ev_io_init (&up_req->io_ev, fuzzy_peer_send_io,
  832. session->ctx->peer_fd, EV_WRITE);
  833. ev_io_start (session->ctx->event_loop, &up_req->io_ev);
  834. }
  835. else {
  836. g_free (up_req);
  837. }
  838. }
  839. }
  840. rspamd_fuzzy_make_reply (cmd, result, session, send_flags);
  841. REF_RELEASE (session);
  842. }
  843. static void
  844. rspamd_fuzzy_process_command (struct fuzzy_session *session)
  845. {
  846. gboolean is_shingle = FALSE, __attribute__ ((unused)) encrypted = FALSE;
  847. struct rspamd_fuzzy_cmd *cmd = NULL;
  848. struct rspamd_fuzzy_reply result;
  849. struct fuzzy_peer_cmd up_cmd;
  850. struct fuzzy_peer_request *up_req;
  851. struct fuzzy_key_stat *ip_stat = NULL;
  852. gchar hexbuf[rspamd_cryptobox_HASHBYTES * 2 + 1];
  853. rspamd_inet_addr_t *naddr;
  854. gpointer ptr;
  855. gsize up_len = 0;
  856. gint send_flags = 0;
  857. cmd = &session->cmd.basic;
  858. switch (session->cmd_type) {
  859. case CMD_NORMAL:
  860. up_len = sizeof (session->cmd.basic);
  861. break;
  862. case CMD_SHINGLE:
  863. up_len = sizeof (session->cmd);
  864. is_shingle = TRUE;
  865. send_flags |= RSPAMD_FUZZY_REPLY_SHINGLE;
  866. break;
  867. case CMD_ENCRYPTED_NORMAL:
  868. up_len = sizeof (session->cmd.basic);
  869. encrypted = TRUE;
  870. send_flags |= RSPAMD_FUZZY_REPLY_ENCRYPTED;
  871. break;
  872. case CMD_ENCRYPTED_SHINGLE:
  873. up_len = sizeof (session->cmd);
  874. encrypted = TRUE;
  875. is_shingle = TRUE;
  876. send_flags |= RSPAMD_FUZZY_REPLY_SHINGLE|RSPAMD_FUZZY_REPLY_ENCRYPTED;
  877. break;
  878. default:
  879. msg_err ("invalid command type: %d", session->cmd_type);
  880. return;
  881. }
  882. memset (&result, 0, sizeof (result));
  883. memcpy (result.digest, cmd->digest, sizeof (result.digest));
  884. result.v1.flag = cmd->flag;
  885. result.v1.tag = cmd->tag;
  886. if (session->ctx->lua_pre_handler_cbref != -1) {
  887. /* Start lua pre handler */
  888. lua_State *L = session->ctx->cfg->lua_state;
  889. gint err_idx, ret;
  890. lua_pushcfunction (L, &rspamd_lua_traceback);
  891. err_idx = lua_gettop (L);
  892. /* Preallocate stack (small opt) */
  893. lua_checkstack (L, err_idx + 5);
  894. /* function */
  895. lua_rawgeti (L, LUA_REGISTRYINDEX, session->ctx->lua_pre_handler_cbref);
  896. /* client IP */
  897. rspamd_lua_ip_push (L, session->addr);
  898. /* client command */
  899. lua_pushinteger (L, cmd->cmd);
  900. /* command value (push as rspamd_text) */
  901. (void)lua_new_text (L, cmd->digest, sizeof (cmd->digest), FALSE);
  902. /* is shingle */
  903. lua_pushboolean (L, is_shingle);
  904. /* TODO: add additional data maybe (encryption, pubkey, etc) */
  905. rspamd_fuzzy_extensions_tolua (L, session);
  906. if ((ret = lua_pcall (L, 5, LUA_MULTRET, err_idx)) != 0) {
  907. msg_err ("call to lua_pre_handler lua "
  908. "script failed (%d): %s", ret, lua_tostring (L, -1));
  909. }
  910. else {
  911. /* Return values order:
  912. * the first reply will be on err_idx + 1
  913. * if it is true, then we need to read the former ones:
  914. * 2-nd will be reply code
  915. * 3-rd will be probability (or 0.0 if missing)
  916. */
  917. ret = lua_toboolean (L, err_idx + 1);
  918. if (ret) {
  919. /* Artificial reply */
  920. result.v1.value = lua_tointeger (L, err_idx + 2);
  921. if (lua_isnumber (L, err_idx + 3)) {
  922. result.v1.prob = lua_tonumber (L, err_idx + 3);
  923. }
  924. else {
  925. result.v1.prob = 0.0f;
  926. }
  927. lua_settop (L, 0);
  928. rspamd_fuzzy_make_reply (cmd, &result, session, send_flags);
  929. return;
  930. }
  931. }
  932. lua_settop (L, 0);
  933. }
  934. if (G_UNLIKELY (cmd == NULL || up_len == 0)) {
  935. result.v1.value = 500;
  936. result.v1.prob = 0.0f;
  937. rspamd_fuzzy_make_reply (cmd, &result, session, send_flags);
  938. return;
  939. }
  940. if (session->ctx->encrypted_only && !encrypted) {
  941. /* Do not accept unencrypted commands */
  942. result.v1.value = 403;
  943. result.v1.prob = 0.0f;
  944. rspamd_fuzzy_make_reply (cmd, &result, session, send_flags);
  945. return;
  946. }
  947. if (session->key_stat) {
  948. ip_stat = rspamd_lru_hash_lookup (session->key_stat->last_ips,
  949. session->addr, -1);
  950. if (ip_stat == NULL) {
  951. naddr = rspamd_inet_address_copy(session->addr, NULL);
  952. ip_stat = g_malloc0 (sizeof (*ip_stat));
  953. REF_INIT_RETAIN (ip_stat, fuzzy_key_stat_dtor);
  954. rspamd_lru_hash_insert (session->key_stat->last_ips,
  955. naddr, ip_stat, -1, 0);
  956. }
  957. REF_RETAIN (ip_stat);
  958. session->ip_stat = ip_stat;
  959. }
  960. if (cmd->cmd == FUZZY_CHECK) {
  961. bool can_continue = true;
  962. if (session->ctx->ratelimit_buckets) {
  963. if (session->ctx->ratelimit_log_only) {
  964. (void)rspamd_fuzzy_check_ratelimit (session); /* Check but ignore */
  965. }
  966. else {
  967. can_continue = rspamd_fuzzy_check_ratelimit (session);
  968. }
  969. }
  970. if (can_continue) {
  971. REF_RETAIN (session);
  972. rspamd_fuzzy_backend_check (session->ctx->backend, cmd,
  973. rspamd_fuzzy_check_callback, session);
  974. }
  975. else {
  976. result.v1.value = 403;
  977. result.v1.prob = 0.0f;
  978. result.v1.flag = 0;
  979. rspamd_fuzzy_make_reply (cmd, &result, session, send_flags);
  980. }
  981. }
  982. else if (cmd->cmd == FUZZY_STAT) {
  983. result.v1.prob = 1.0f;
  984. result.v1.value = 0;
  985. result.v1.flag = session->ctx->stat.fuzzy_hashes;
  986. rspamd_fuzzy_make_reply (cmd, &result, session, send_flags);
  987. }
  988. else {
  989. if (rspamd_fuzzy_check_write (session)) {
  990. /* Check whitelist */
  991. if (session->ctx->skip_hashes && cmd->cmd == FUZZY_WRITE) {
  992. rspamd_encode_hex_buf (cmd->digest, sizeof (cmd->digest),
  993. hexbuf, sizeof (hexbuf) - 1);
  994. hexbuf[sizeof (hexbuf) - 1] = '\0';
  995. if (rspamd_match_hash_map (session->ctx->skip_hashes,
  996. hexbuf, sizeof (hexbuf) - 1)) {
  997. result.v1.value = 401;
  998. result.v1.prob = 0.0f;
  999. goto reply;
  1000. }
  1001. }
  1002. if (session->worker->index == 0 || session->ctx->peer_fd == -1) {
  1003. /* Just add to the queue */
  1004. up_cmd.is_shingle = is_shingle;
  1005. ptr = is_shingle ?
  1006. (gpointer)&up_cmd.cmd.shingle :
  1007. (gpointer)&up_cmd.cmd.normal;
  1008. memcpy (ptr, cmd, up_len);
  1009. g_array_append_val (session->ctx->updates_pending, up_cmd);
  1010. }
  1011. else {
  1012. /* We need to send request to the peer */
  1013. up_req = g_malloc0 (sizeof (*up_req));
  1014. up_req->cmd.is_shingle = is_shingle;
  1015. ptr = is_shingle ?
  1016. (gpointer)&up_req->cmd.cmd.shingle :
  1017. (gpointer)&up_req->cmd.cmd.normal;
  1018. memcpy (ptr, cmd, up_len);
  1019. if (!fuzzy_peer_try_send (session->ctx->peer_fd, up_req)) {
  1020. up_req->io_ev.data = up_req;
  1021. ev_io_init (&up_req->io_ev, fuzzy_peer_send_io,
  1022. session->ctx->peer_fd, EV_WRITE);
  1023. ev_io_start (session->ctx->event_loop, &up_req->io_ev);
  1024. }
  1025. else {
  1026. g_free (up_req);
  1027. }
  1028. }
  1029. result.v1.value = 0;
  1030. result.v1.prob = 1.0f;
  1031. }
  1032. else {
  1033. result.v1.value = 403;
  1034. result.v1.prob = 0.0f;
  1035. }
  1036. reply:
  1037. rspamd_fuzzy_make_reply (cmd, &result, session, send_flags);
  1038. }
  1039. }
  1040. static enum rspamd_fuzzy_epoch
  1041. rspamd_fuzzy_command_valid (struct rspamd_fuzzy_cmd *cmd, gint r)
  1042. {
  1043. enum rspamd_fuzzy_epoch ret = RSPAMD_FUZZY_EPOCH_MAX;
  1044. switch (cmd->version) {
  1045. case 4:
  1046. if (cmd->shingles_count > 0) {
  1047. if (r >= sizeof (struct rspamd_fuzzy_shingle_cmd)) {
  1048. ret = RSPAMD_FUZZY_EPOCH11;
  1049. }
  1050. }
  1051. else {
  1052. if (r >= sizeof (*cmd)) {
  1053. ret = RSPAMD_FUZZY_EPOCH11;
  1054. }
  1055. }
  1056. break;
  1057. case 3:
  1058. if (cmd->shingles_count > 0) {
  1059. if (r == sizeof (struct rspamd_fuzzy_shingle_cmd)) {
  1060. ret = RSPAMD_FUZZY_EPOCH10;
  1061. }
  1062. }
  1063. else {
  1064. if (r == sizeof (*cmd)) {
  1065. ret = RSPAMD_FUZZY_EPOCH10;
  1066. }
  1067. }
  1068. break;
  1069. default:
  1070. break;
  1071. }
  1072. return ret;
  1073. }
  1074. static gboolean
  1075. rspamd_fuzzy_decrypt_command (struct fuzzy_session *s, guchar *buf, gsize buflen)
  1076. {
  1077. struct rspamd_fuzzy_encrypted_req_hdr hdr;
  1078. struct rspamd_cryptobox_pubkey *rk;
  1079. struct fuzzy_key *key;
  1080. if (s->ctx->default_key == NULL) {
  1081. msg_warn ("received encrypted request when encryption is not enabled");
  1082. return FALSE;
  1083. }
  1084. if (buflen < sizeof (hdr)) {
  1085. msg_warn ("XXX: should not be reached");
  1086. return FALSE;
  1087. }
  1088. memcpy (&hdr, buf, sizeof (hdr));
  1089. buf += sizeof (hdr);
  1090. buflen -= sizeof (hdr);
  1091. /* Try to find the desired key */
  1092. key = g_hash_table_lookup (s->ctx->keys, hdr.key_id);
  1093. if (key == NULL) {
  1094. /* Unknown key, assume default one */
  1095. key = s->ctx->default_key;
  1096. }
  1097. s->key_stat = key->stat;
  1098. /* Now process keypair */
  1099. rk = rspamd_pubkey_from_bin (hdr.pubkey, sizeof (hdr.pubkey),
  1100. RSPAMD_KEYPAIR_KEX, RSPAMD_CRYPTOBOX_MODE_25519);
  1101. if (rk == NULL) {
  1102. msg_err ("bad key; ip=%s",
  1103. rspamd_inet_address_to_string (s->addr));
  1104. return FALSE;
  1105. }
  1106. rspamd_keypair_cache_process (s->ctx->keypair_cache, key->key, rk);
  1107. /* Now decrypt request */
  1108. if (!rspamd_cryptobox_decrypt_nm_inplace (buf, buflen, hdr.nonce,
  1109. rspamd_pubkey_get_nm (rk, key->key),
  1110. hdr.mac, RSPAMD_CRYPTOBOX_MODE_25519)) {
  1111. msg_err ("decryption failed; ip=%s",
  1112. rspamd_inet_address_to_string (s->addr));
  1113. rspamd_pubkey_unref (rk);
  1114. return FALSE;
  1115. }
  1116. memcpy (s->nm, rspamd_pubkey_get_nm (rk, key->key), sizeof (s->nm));
  1117. rspamd_pubkey_unref (rk);
  1118. return TRUE;
  1119. }
  1120. static gboolean
  1121. rspamd_fuzzy_extensions_from_wire (struct fuzzy_session *s, guchar *buf, gsize buflen)
  1122. {
  1123. struct rspamd_fuzzy_cmd_extension *ext, *prev_ext;
  1124. guchar *storage, *p = buf, *end = buf + buflen;
  1125. gsize st_len = 0, n_ext = 0;
  1126. /* Read number of extensions to allocate array */
  1127. while (p < end) {
  1128. guchar cmd = *p++;
  1129. if (p < end) {
  1130. if (cmd == RSPAMD_FUZZY_EXT_SOURCE_DOMAIN) {
  1131. /* Next byte is buf length */
  1132. guchar dom_len = *p++;
  1133. if (dom_len <= (end - p)) {
  1134. st_len += dom_len;
  1135. n_ext ++;
  1136. }
  1137. else {
  1138. /* Truncation */
  1139. return FALSE;
  1140. }
  1141. p += dom_len;
  1142. }
  1143. else if (cmd == RSPAMD_FUZZY_EXT_SOURCE_IP4) {
  1144. if (end - p >= sizeof (in_addr_t)) {
  1145. n_ext ++;
  1146. st_len += sizeof (in_addr_t);
  1147. }
  1148. else {
  1149. /* Truncation */
  1150. return FALSE;
  1151. }
  1152. p += sizeof (in_addr_t);
  1153. }
  1154. else if (cmd == RSPAMD_FUZZY_EXT_SOURCE_IP6) {
  1155. if (end - p >= sizeof (struct in6_addr)) {
  1156. n_ext ++;
  1157. st_len += sizeof (struct in6_addr);
  1158. }
  1159. else {
  1160. /* Truncation */
  1161. return FALSE;
  1162. }
  1163. p += sizeof (struct in6_addr);
  1164. }
  1165. else {
  1166. /* Invalid command */
  1167. return FALSE;
  1168. }
  1169. }
  1170. else {
  1171. /* Truncated extension */
  1172. return FALSE;
  1173. }
  1174. }
  1175. if (n_ext > 0) {
  1176. p = buf;
  1177. /*
  1178. * Memory layout: n_ext of struct rspamd_fuzzy_cmd_extension
  1179. * payload for each extension in a continuous data segment
  1180. */
  1181. storage = g_malloc (n_ext * sizeof (struct rspamd_fuzzy_cmd_extension) +
  1182. st_len);
  1183. guchar *data_buf = storage +
  1184. n_ext * sizeof (struct rspamd_fuzzy_cmd_extension);
  1185. ext = (struct rspamd_fuzzy_cmd_extension *)storage;
  1186. /* All validation has been done, so we can just go further */
  1187. while (p < end) {
  1188. prev_ext = ext;
  1189. guchar cmd = *p++;
  1190. if (cmd == RSPAMD_FUZZY_EXT_SOURCE_DOMAIN) {
  1191. /* Next byte is buf length */
  1192. guchar dom_len = *p++;
  1193. guchar *dest = data_buf;
  1194. ext->ext = RSPAMD_FUZZY_EXT_SOURCE_DOMAIN;
  1195. ext->next = ext + 1;
  1196. ext->length = dom_len;
  1197. ext->payload = dest;
  1198. memcpy (dest, p, dom_len);
  1199. p += dom_len;
  1200. data_buf += dom_len;
  1201. ext = ext->next;
  1202. }
  1203. else if (cmd == RSPAMD_FUZZY_EXT_SOURCE_IP4) {
  1204. guchar *dest = data_buf;
  1205. ext->ext = RSPAMD_FUZZY_EXT_SOURCE_IP4;
  1206. ext->next = ext + 1;
  1207. ext->length = sizeof (in_addr_t);
  1208. ext->payload = dest;
  1209. memcpy (dest, p, sizeof (in_addr_t));
  1210. p += sizeof (in_addr_t);
  1211. data_buf += sizeof (in_addr_t);
  1212. ext = ext->next;
  1213. }
  1214. else if (cmd == RSPAMD_FUZZY_EXT_SOURCE_IP6) {
  1215. guchar *dest = data_buf;
  1216. ext->ext = RSPAMD_FUZZY_EXT_SOURCE_IP6;
  1217. ext->next = ext + 1;
  1218. ext->length = sizeof (struct in6_addr);
  1219. ext->payload = dest;
  1220. memcpy (dest, p, sizeof (struct in6_addr));
  1221. p += sizeof (struct in6_addr);
  1222. data_buf += sizeof (struct in6_addr);
  1223. ext = ext->next;
  1224. }
  1225. else {
  1226. g_assert_not_reached ();
  1227. }
  1228. }
  1229. /* Last next should be NULL */
  1230. prev_ext->next = NULL;
  1231. /* Rewind to the begin */
  1232. ext = (struct rspamd_fuzzy_cmd_extension *)storage;
  1233. s->extensions = ext;
  1234. }
  1235. return TRUE;
  1236. }
  1237. static gboolean
  1238. rspamd_fuzzy_cmd_from_wire (guchar *buf, guint buflen, struct fuzzy_session *s)
  1239. {
  1240. enum rspamd_fuzzy_epoch epoch;
  1241. gboolean encrypted = FALSE;
  1242. if (buflen < sizeof (struct rspamd_fuzzy_cmd)) {
  1243. msg_debug ("truncated fuzzy command of size %d received", buflen);
  1244. return FALSE;
  1245. }
  1246. /* Now check encryption */
  1247. if (buflen >= sizeof (struct rspamd_fuzzy_encrypted_cmd)) {
  1248. if (memcmp (buf, fuzzy_encrypted_magic, sizeof (fuzzy_encrypted_magic)) == 0) {
  1249. /* Encrypted command */
  1250. encrypted = TRUE;
  1251. }
  1252. }
  1253. if (encrypted) {
  1254. /* Decrypt first */
  1255. if (!rspamd_fuzzy_decrypt_command (s, buf, buflen)) {
  1256. return FALSE;
  1257. }
  1258. else {
  1259. /*
  1260. * Advance buffer to skip encrypted header.
  1261. * Note that after rspamd_fuzzy_decrypt_command buf is unencrypted
  1262. */
  1263. buf += sizeof (struct rspamd_fuzzy_encrypted_req_hdr);
  1264. buflen -= sizeof (struct rspamd_fuzzy_encrypted_req_hdr);
  1265. }
  1266. }
  1267. /* Fill the normal command */
  1268. if (buflen < sizeof (s->cmd.basic)) {
  1269. msg_debug ("truncated normal fuzzy command of size %d received", buflen);
  1270. return FALSE;
  1271. }
  1272. memcpy (&s->cmd.basic, buf, sizeof (s->cmd.basic));
  1273. epoch = rspamd_fuzzy_command_valid (&s->cmd.basic, buflen);
  1274. if (epoch == RSPAMD_FUZZY_EPOCH_MAX) {
  1275. msg_debug ("invalid fuzzy command of size %d received", buflen);
  1276. return FALSE;
  1277. }
  1278. s->epoch = epoch;
  1279. /* Advance buf */
  1280. buf += sizeof (s->cmd.basic);
  1281. buflen -= sizeof (s->cmd.basic);
  1282. if (s->cmd.basic.shingles_count > 0) {
  1283. if (buflen >= sizeof (s->cmd.sgl)) {
  1284. /* Copy the shingles part */
  1285. memcpy (&s->cmd.sgl, buf, sizeof (s->cmd.sgl));
  1286. }
  1287. else {
  1288. /* Truncated stuff */
  1289. msg_debug ("truncated fuzzy shingles command of size %d received", buflen);
  1290. return FALSE;
  1291. }
  1292. buf += sizeof (s->cmd.sgl);
  1293. buflen -= sizeof (s->cmd.sgl);
  1294. if (encrypted) {
  1295. s->cmd_type = CMD_ENCRYPTED_SHINGLE;
  1296. }
  1297. else {
  1298. s->cmd_type = CMD_SHINGLE;
  1299. }
  1300. }
  1301. else {
  1302. if (encrypted) {
  1303. s->cmd_type = CMD_ENCRYPTED_NORMAL;
  1304. }
  1305. else {
  1306. s->cmd_type = CMD_NORMAL;
  1307. }
  1308. }
  1309. if (buflen > 0) {
  1310. /* Process possible extensions */
  1311. if (!rspamd_fuzzy_extensions_from_wire (s, buf, buflen)) {
  1312. msg_debug ("truncated fuzzy shingles command of size %d received", buflen);
  1313. return FALSE;
  1314. }
  1315. }
  1316. return TRUE;
  1317. }
  1318. static void
  1319. fuzzy_session_destroy (gpointer d)
  1320. {
  1321. struct fuzzy_session *session = d;
  1322. rspamd_inet_address_free (session->addr);
  1323. rspamd_explicit_memzero (session->nm, sizeof (session->nm));
  1324. session->worker->nconns--;
  1325. if (session->ip_stat) {
  1326. REF_RELEASE (session->ip_stat);
  1327. }
  1328. if (session->extensions) {
  1329. g_free (session->extensions);
  1330. }
  1331. g_free (session);
  1332. }
  1333. #define FUZZY_INPUT_BUFLEN 1024
  1334. #ifdef HAVE_RECVMMSG
  1335. #define MSGVEC_LEN 16
  1336. #else
  1337. #define MSGVEC_LEN 1
  1338. #endif
  1339. /*
  1340. * Accept new connection and construct task
  1341. */
  1342. static void
  1343. accept_fuzzy_socket (EV_P_ ev_io *w, int revents)
  1344. {
  1345. struct rspamd_worker *worker = (struct rspamd_worker *)w->data;
  1346. struct fuzzy_session *session;
  1347. gssize r, msg_len;
  1348. guint64 *nerrors;
  1349. struct iovec iovs[MSGVEC_LEN];
  1350. guint8 bufs[MSGVEC_LEN][FUZZY_INPUT_BUFLEN];
  1351. struct sockaddr_storage peer_sa[MSGVEC_LEN];
  1352. socklen_t salen = sizeof (peer_sa[0]);
  1353. #ifdef HAVE_RECVMMSG
  1354. #define MSG_FIELD(msg, field) msg.msg_hdr.field
  1355. struct mmsghdr msg[MSGVEC_LEN];
  1356. #else
  1357. #define MSG_FIELD(msg, field) msg.field
  1358. struct msghdr msg[MSGVEC_LEN];
  1359. #endif
  1360. memset (msg, 0, sizeof (*msg) * MSGVEC_LEN);
  1361. /* Prepare messages to receive */
  1362. for (int i = 0; i < MSGVEC_LEN; i ++) {
  1363. /* Prepare msghdr structs */
  1364. iovs[i].iov_base = bufs[i];
  1365. iovs[i].iov_len = sizeof (bufs[i]);
  1366. MSG_FIELD(msg[i], msg_name) = (void *)&peer_sa[i];
  1367. MSG_FIELD(msg[i], msg_namelen) = salen;
  1368. MSG_FIELD(msg[i], msg_iov) = &iovs[i];
  1369. MSG_FIELD(msg[i], msg_iovlen) = 1;
  1370. }
  1371. /* Got some data */
  1372. if (revents == EV_READ) {
  1373. for (;;) {
  1374. #ifdef HAVE_RECVMMSG
  1375. r = recvmmsg (w->fd, msg, MSGVEC_LEN, 0, NULL);
  1376. #else
  1377. r = recvmsg (w->fd, msg, 0);
  1378. #endif
  1379. if (r == -1) {
  1380. if (errno == EINTR) {
  1381. continue;
  1382. }
  1383. else if (errno == EAGAIN || errno == EWOULDBLOCK) {
  1384. return;
  1385. }
  1386. msg_err ("got error while reading from socket: %d, %s",
  1387. errno,
  1388. strerror (errno));
  1389. return;
  1390. }
  1391. #ifndef HAVE_RECVMMSG
  1392. msg_len = r; /* Save real length in bytes here */
  1393. r = 1; /* Assume that we have received a single message */
  1394. #endif
  1395. for (int i = 0; i < r; i ++) {
  1396. rspamd_inet_addr_t *client_addr;
  1397. client_addr = rspamd_inet_address_from_sa (MSG_FIELD(msg[i], msg_name),
  1398. MSG_FIELD(msg[i], msg_namelen));
  1399. if (!rspamd_fuzzy_check_client (worker->ctx, client_addr)) {
  1400. /* Disallow forbidden clients silently */
  1401. rspamd_inet_address_free (client_addr);
  1402. continue;
  1403. }
  1404. session = g_malloc0 (sizeof (*session));
  1405. REF_INIT_RETAIN (session, fuzzy_session_destroy);
  1406. session->worker = worker;
  1407. session->fd = w->fd;
  1408. session->ctx = worker->ctx;
  1409. session->time = (guint64) time (NULL);
  1410. session->addr = client_addr;
  1411. worker->nconns++;
  1412. /* Each message can have its length in case of recvmmsg */
  1413. #ifdef HAVE_RECVMMSG
  1414. msg_len = msg[i].msg_len;
  1415. #endif
  1416. if (rspamd_fuzzy_cmd_from_wire (iovs[i].iov_base,
  1417. msg_len, session)) {
  1418. /* Check shingles count sanity */
  1419. rspamd_fuzzy_process_command (session);
  1420. }
  1421. else {
  1422. /* Discard input */
  1423. session->ctx->stat.invalid_requests ++;
  1424. msg_debug ("invalid fuzzy command of size %z received", r);
  1425. nerrors = rspamd_lru_hash_lookup (session->ctx->errors_ips,
  1426. session->addr, -1);
  1427. if (nerrors == NULL) {
  1428. nerrors = g_malloc (sizeof (*nerrors));
  1429. *nerrors = 1;
  1430. rspamd_lru_hash_insert (session->ctx->errors_ips,
  1431. rspamd_inet_address_copy(session->addr, NULL),
  1432. nerrors, -1, -1);
  1433. }
  1434. else {
  1435. *nerrors = *nerrors + 1;
  1436. }
  1437. }
  1438. REF_RELEASE (session);
  1439. }
  1440. #ifdef HAVE_RECVMMSG
  1441. /* Stop reading as we are using recvmmsg instead of recvmsg */
  1442. break;
  1443. #endif
  1444. }
  1445. }
  1446. }
  1447. static gboolean
  1448. rspamd_fuzzy_storage_periodic_callback (void *ud)
  1449. {
  1450. struct rspamd_fuzzy_storage_ctx *ctx = ud;
  1451. if (ctx->updates_pending->len > 0) {
  1452. rspamd_fuzzy_process_updates_queue (ctx, local_db_name, FALSE);
  1453. return TRUE;
  1454. }
  1455. return FALSE;
  1456. }
  1457. static gboolean
  1458. rspamd_fuzzy_storage_sync (struct rspamd_main *rspamd_main,
  1459. struct rspamd_worker *worker, gint fd,
  1460. gint attached_fd,
  1461. struct rspamd_control_command *cmd,
  1462. gpointer ud)
  1463. {
  1464. struct rspamd_fuzzy_storage_ctx *ctx = ud;
  1465. struct rspamd_control_reply rep;
  1466. rep.reply.fuzzy_sync.status = 0;
  1467. rep.type = RSPAMD_CONTROL_FUZZY_SYNC;
  1468. if (ctx->backend && worker->index == 0) {
  1469. rspamd_fuzzy_process_updates_queue (ctx, local_db_name, FALSE);
  1470. rspamd_fuzzy_backend_start_update (ctx->backend, ctx->sync_timeout,
  1471. rspamd_fuzzy_storage_periodic_callback, ctx);
  1472. }
  1473. if (write (fd, &rep, sizeof (rep)) != sizeof (rep)) {
  1474. msg_err ("cannot write reply to the control socket: %s",
  1475. strerror (errno));
  1476. }
  1477. return TRUE;
  1478. }
  1479. static gboolean
  1480. rspamd_fuzzy_storage_reload (struct rspamd_main *rspamd_main,
  1481. struct rspamd_worker *worker, gint fd,
  1482. gint attached_fd,
  1483. struct rspamd_control_command *cmd,
  1484. gpointer ud)
  1485. {
  1486. struct rspamd_fuzzy_storage_ctx *ctx = ud;
  1487. GError *err = NULL;
  1488. struct rspamd_control_reply rep;
  1489. msg_info ("reloading fuzzy storage after receiving reload command");
  1490. if (ctx->backend) {
  1491. /* Close backend and reopen it one more time */
  1492. rspamd_fuzzy_backend_close (ctx->backend);
  1493. }
  1494. memset (&rep, 0, sizeof (rep));
  1495. rep.type = RSPAMD_CONTROL_RELOAD;
  1496. if ((ctx->backend = rspamd_fuzzy_backend_create (ctx->event_loop,
  1497. worker->cf->options, rspamd_main->cfg,
  1498. &err)) == NULL) {
  1499. msg_err ("cannot open backend after reload: %e", err);
  1500. rep.reply.reload.status = err->code;
  1501. g_error_free (err);
  1502. }
  1503. else {
  1504. rep.reply.reload.status = 0;
  1505. }
  1506. if (ctx->backend && worker->index == 0) {
  1507. rspamd_fuzzy_backend_start_update (ctx->backend, ctx->sync_timeout,
  1508. rspamd_fuzzy_storage_periodic_callback, ctx);
  1509. }
  1510. if (write (fd, &rep, sizeof (rep)) != sizeof (rep)) {
  1511. msg_err ("cannot write reply to the control socket: %s",
  1512. strerror (errno));
  1513. }
  1514. return TRUE;
  1515. }
  1516. static ucl_object_t *
  1517. rspamd_fuzzy_storage_stat_key (struct fuzzy_key_stat *key_stat)
  1518. {
  1519. ucl_object_t *res;
  1520. res = ucl_object_typed_new (UCL_OBJECT);
  1521. ucl_object_insert_key (res, ucl_object_fromint (key_stat->checked),
  1522. "checked", 0, false);
  1523. ucl_object_insert_key (res, ucl_object_fromint (key_stat->matched),
  1524. "matched", 0, false);
  1525. ucl_object_insert_key (res, ucl_object_fromint (key_stat->added),
  1526. "added", 0, false);
  1527. ucl_object_insert_key (res, ucl_object_fromint (key_stat->deleted),
  1528. "deleted", 0, false);
  1529. ucl_object_insert_key (res, ucl_object_fromint (key_stat->errors),
  1530. "errors", 0, false);
  1531. return res;
  1532. }
  1533. static ucl_object_t *
  1534. rspamd_fuzzy_stat_to_ucl (struct rspamd_fuzzy_storage_ctx *ctx, gboolean ip_stat)
  1535. {
  1536. struct fuzzy_key_stat *key_stat;
  1537. GHashTableIter it;
  1538. struct fuzzy_key *key;
  1539. ucl_object_t *obj, *keys_obj, *elt, *ip_elt, *ip_cur;
  1540. gpointer k, v;
  1541. gint i;
  1542. gchar keyname[17];
  1543. obj = ucl_object_typed_new (UCL_OBJECT);
  1544. keys_obj = ucl_object_typed_new (UCL_OBJECT);
  1545. g_hash_table_iter_init (&it, ctx->keys);
  1546. while (g_hash_table_iter_next (&it, &k, &v)) {
  1547. key = v;
  1548. key_stat = key->stat;
  1549. if (key_stat) {
  1550. rspamd_snprintf (keyname, sizeof (keyname), "%8bs", k);
  1551. elt = rspamd_fuzzy_storage_stat_key (key_stat);
  1552. if (key_stat->last_ips && ip_stat) {
  1553. i = 0;
  1554. ip_elt = ucl_object_typed_new (UCL_OBJECT);
  1555. while ((i = rspamd_lru_hash_foreach (key_stat->last_ips,
  1556. i, &k, &v)) != -1) {
  1557. ip_cur = rspamd_fuzzy_storage_stat_key (v);
  1558. ucl_object_insert_key (ip_elt, ip_cur,
  1559. rspamd_inet_address_to_string (k), 0, true);
  1560. }
  1561. ucl_object_insert_key (elt, ip_elt, "ips", 0, false);
  1562. }
  1563. ucl_object_insert_key (keys_obj, elt, keyname, 0, true);
  1564. }
  1565. }
  1566. ucl_object_insert_key (obj, keys_obj, "keys", 0, false);
  1567. /* Now generic stats */
  1568. ucl_object_insert_key (obj,
  1569. ucl_object_fromint (ctx->stat.fuzzy_hashes),
  1570. "fuzzy_stored",
  1571. 0,
  1572. false);
  1573. ucl_object_insert_key (obj,
  1574. ucl_object_fromint (ctx->stat.fuzzy_hashes_expired),
  1575. "fuzzy_expired",
  1576. 0,
  1577. false);
  1578. ucl_object_insert_key (obj,
  1579. ucl_object_fromint (ctx->stat.invalid_requests),
  1580. "invalid_requests",
  1581. 0,
  1582. false);
  1583. ucl_object_insert_key (obj,
  1584. ucl_object_fromint (ctx->stat.delayed_hashes),
  1585. "delayed_hashes",
  1586. 0,
  1587. false);
  1588. if (ctx->errors_ips && ip_stat) {
  1589. i = 0;
  1590. ip_elt = ucl_object_typed_new (UCL_OBJECT);
  1591. while ((i = rspamd_lru_hash_foreach (ctx->errors_ips, i, &k, &v)) != -1) {
  1592. ucl_object_insert_key (ip_elt,
  1593. ucl_object_fromint (*(guint64 *)v),
  1594. rspamd_inet_address_to_string (k), 0, true);
  1595. }
  1596. ucl_object_insert_key (obj,
  1597. ip_elt,
  1598. "errors_ips",
  1599. 0,
  1600. false);
  1601. }
  1602. /* Checked by epoch */
  1603. elt = ucl_object_typed_new (UCL_ARRAY);
  1604. for (i = RSPAMD_FUZZY_EPOCH10; i < RSPAMD_FUZZY_EPOCH_MAX; i++) {
  1605. ucl_array_append (elt,
  1606. ucl_object_fromint (ctx->stat.fuzzy_hashes_checked[i]));
  1607. }
  1608. ucl_object_insert_key (obj, elt, "fuzzy_checked", 0, false);
  1609. /* Shingles by epoch */
  1610. elt = ucl_object_typed_new (UCL_ARRAY);
  1611. for (i = RSPAMD_FUZZY_EPOCH10; i < RSPAMD_FUZZY_EPOCH_MAX; i++) {
  1612. ucl_array_append (elt,
  1613. ucl_object_fromint (ctx->stat.fuzzy_shingles_checked[i]));
  1614. }
  1615. ucl_object_insert_key (obj, elt, "fuzzy_shingles", 0, false);
  1616. /* Matched by epoch */
  1617. elt = ucl_object_typed_new (UCL_ARRAY);
  1618. for (i = RSPAMD_FUZZY_EPOCH10; i < RSPAMD_FUZZY_EPOCH_MAX; i++) {
  1619. ucl_array_append (elt,
  1620. ucl_object_fromint (ctx->stat.fuzzy_hashes_found[i]));
  1621. }
  1622. ucl_object_insert_key (obj, elt, "fuzzy_found", 0, false);
  1623. return obj;
  1624. }
  1625. static int
  1626. lua_fuzzy_add_pre_handler (lua_State *L)
  1627. {
  1628. struct rspamd_worker *wrk, **pwrk = (struct rspamd_worker **)
  1629. rspamd_lua_check_udata (L, 1, "rspamd{worker}");
  1630. struct rspamd_fuzzy_storage_ctx *ctx;
  1631. if (!pwrk) {
  1632. return luaL_error (L, "invalid arguments, worker + function are expected");
  1633. }
  1634. wrk = *pwrk;
  1635. if (wrk && lua_isfunction (L, 2)) {
  1636. ctx = (struct rspamd_fuzzy_storage_ctx *)wrk->ctx;
  1637. if (ctx->lua_pre_handler_cbref != -1) {
  1638. /* Should not happen */
  1639. luaL_unref (L, LUA_REGISTRYINDEX, ctx->lua_pre_handler_cbref);
  1640. }
  1641. lua_pushvalue (L, 2);
  1642. ctx->lua_pre_handler_cbref = luaL_ref (L, LUA_REGISTRYINDEX);
  1643. }
  1644. else {
  1645. return luaL_error (L, "invalid arguments, worker + function are expected");
  1646. }
  1647. return 0;
  1648. }
  1649. static int
  1650. lua_fuzzy_add_post_handler (lua_State *L)
  1651. {
  1652. struct rspamd_worker *wrk, **pwrk = (struct rspamd_worker **)
  1653. rspamd_lua_check_udata (L, 1, "rspamd{worker}");
  1654. struct rspamd_fuzzy_storage_ctx *ctx;
  1655. if (!pwrk) {
  1656. return luaL_error (L, "invalid arguments, worker + function are expected");
  1657. }
  1658. wrk = *pwrk;
  1659. if (wrk && lua_isfunction (L, 2)) {
  1660. ctx = (struct rspamd_fuzzy_storage_ctx *)wrk->ctx;
  1661. if (ctx->lua_post_handler_cbref != -1) {
  1662. /* Should not happen */
  1663. luaL_unref (L, LUA_REGISTRYINDEX, ctx->lua_post_handler_cbref);
  1664. }
  1665. lua_pushvalue (L, 2);
  1666. ctx->lua_post_handler_cbref = luaL_ref (L, LUA_REGISTRYINDEX);
  1667. }
  1668. else {
  1669. return luaL_error (L, "invalid arguments, worker + function are expected");
  1670. }
  1671. return 0;
  1672. }
  1673. static gboolean
  1674. rspamd_fuzzy_storage_stat (struct rspamd_main *rspamd_main,
  1675. struct rspamd_worker *worker, gint fd,
  1676. gint attached_fd,
  1677. struct rspamd_control_command *cmd,
  1678. gpointer ud)
  1679. {
  1680. struct rspamd_fuzzy_storage_ctx *ctx = ud;
  1681. struct rspamd_control_reply rep;
  1682. ucl_object_t *obj;
  1683. struct ucl_emitter_functions *emit_subr;
  1684. guchar fdspace[CMSG_SPACE(sizeof (int))];
  1685. struct iovec iov;
  1686. struct msghdr msg;
  1687. struct cmsghdr *cmsg;
  1688. gint outfd = -1;
  1689. gchar tmppath[PATH_MAX];
  1690. memset (&rep, 0, sizeof (rep));
  1691. rep.type = RSPAMD_CONTROL_FUZZY_STAT;
  1692. rspamd_snprintf (tmppath, sizeof (tmppath), "%s%c%s-XXXXXXXXXX",
  1693. rspamd_main->cfg->temp_dir, G_DIR_SEPARATOR, "fuzzy-stat");
  1694. if ((outfd = mkstemp (tmppath)) == -1) {
  1695. rep.reply.fuzzy_stat.status = errno;
  1696. msg_info_main ("cannot make temporary stat file for fuzzy stat: %s",
  1697. strerror (errno));
  1698. }
  1699. else {
  1700. rep.reply.fuzzy_stat.status = 0;
  1701. memcpy (rep.reply.fuzzy_stat.storage_id,
  1702. rspamd_fuzzy_backend_id (ctx->backend),
  1703. sizeof (rep.reply.fuzzy_stat.storage_id));
  1704. obj = rspamd_fuzzy_stat_to_ucl (ctx, TRUE);
  1705. emit_subr = ucl_object_emit_fd_funcs (outfd);
  1706. ucl_object_emit_full (obj, UCL_EMIT_JSON_COMPACT, emit_subr, NULL);
  1707. ucl_object_emit_funcs_free (emit_subr);
  1708. ucl_object_unref (obj);
  1709. /* Rewind output file */
  1710. close (outfd);
  1711. outfd = open (tmppath, O_RDONLY);
  1712. unlink (tmppath);
  1713. }
  1714. /* Now we can send outfd and status message */
  1715. memset (&msg, 0, sizeof (msg));
  1716. /* Attach fd to the message */
  1717. if (outfd != -1) {
  1718. memset (fdspace, 0, sizeof (fdspace));
  1719. msg.msg_control = fdspace;
  1720. msg.msg_controllen = sizeof (fdspace);
  1721. cmsg = CMSG_FIRSTHDR (&msg);
  1722. if (cmsg) {
  1723. cmsg->cmsg_level = SOL_SOCKET;
  1724. cmsg->cmsg_type = SCM_RIGHTS;
  1725. cmsg->cmsg_len = CMSG_LEN (sizeof (int));
  1726. memcpy (CMSG_DATA (cmsg), &outfd, sizeof (int));
  1727. }
  1728. }
  1729. iov.iov_base = &rep;
  1730. iov.iov_len = sizeof (rep);
  1731. msg.msg_iov = &iov;
  1732. msg.msg_iovlen = 1;
  1733. if (sendmsg (fd, &msg, 0) == -1) {
  1734. msg_err_main ("cannot send fuzzy stat: %s", strerror (errno));
  1735. }
  1736. if (outfd != -1) {
  1737. close (outfd);
  1738. }
  1739. return TRUE;
  1740. }
  1741. static gboolean
  1742. fuzzy_parse_keypair (rspamd_mempool_t *pool,
  1743. const ucl_object_t *obj,
  1744. gpointer ud,
  1745. struct rspamd_rcl_section *section,
  1746. GError **err)
  1747. {
  1748. struct rspamd_rcl_struct_parser *pd = ud;
  1749. struct rspamd_fuzzy_storage_ctx *ctx;
  1750. struct rspamd_cryptobox_keypair *kp;
  1751. struct fuzzy_key_stat *keystat;
  1752. struct fuzzy_key *key;
  1753. const ucl_object_t *cur;
  1754. const guchar *pk;
  1755. ucl_object_iter_t it = NULL;
  1756. gboolean ret;
  1757. ctx = pd->user_struct;
  1758. pd->offset = G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, default_keypair);
  1759. /*
  1760. * Single key
  1761. */
  1762. if (ucl_object_type (obj) == UCL_STRING || ucl_object_type (obj)
  1763. == UCL_OBJECT) {
  1764. ret = rspamd_rcl_parse_struct_keypair (pool, obj, pd, section, err);
  1765. if (!ret) {
  1766. return ret;
  1767. }
  1768. /* Insert key to the hash table */
  1769. kp = ctx->default_keypair;
  1770. if (kp == NULL) {
  1771. return FALSE;
  1772. }
  1773. if (rspamd_keypair_alg (kp) != RSPAMD_CRYPTOBOX_MODE_25519 ||
  1774. rspamd_keypair_type (kp) != RSPAMD_KEYPAIR_KEX) {
  1775. return FALSE;
  1776. }
  1777. key = g_malloc0 (sizeof (*key));
  1778. key->key = kp;
  1779. keystat = g_malloc0 (sizeof (*keystat));
  1780. REF_INIT_RETAIN (keystat, fuzzy_key_stat_dtor);
  1781. /* Hash of ip -> fuzzy_key_stat */
  1782. keystat->last_ips = rspamd_lru_hash_new_full (1024,
  1783. (GDestroyNotify) rspamd_inet_address_free,
  1784. fuzzy_key_stat_unref,
  1785. rspamd_inet_address_hash, rspamd_inet_address_equal);
  1786. key->stat = keystat;
  1787. pk = rspamd_keypair_component (kp, RSPAMD_KEYPAIR_COMPONENT_PK,
  1788. NULL);
  1789. g_hash_table_insert (ctx->keys, (gpointer)pk, key);
  1790. ctx->default_key = key;
  1791. msg_info_pool ("loaded keypair %*xs", 8, pk);
  1792. }
  1793. else if (ucl_object_type (obj) == UCL_ARRAY) {
  1794. while ((cur = ucl_object_iterate (obj, &it, true)) != NULL) {
  1795. if (!fuzzy_parse_keypair (pool, cur, pd, section, err)) {
  1796. msg_err_pool ("cannot parse keypair");
  1797. }
  1798. }
  1799. }
  1800. return TRUE;
  1801. }
  1802. static guint
  1803. fuzzy_kp_hash (gconstpointer p)
  1804. {
  1805. return *(guint *)p;
  1806. }
  1807. static gboolean
  1808. fuzzy_kp_equal (gconstpointer a, gconstpointer b)
  1809. {
  1810. const guchar *pa = a, *pb = b;
  1811. return (memcmp (pa, pb, RSPAMD_FUZZY_KEYLEN) == 0);
  1812. }
  1813. gpointer
  1814. init_fuzzy (struct rspamd_config *cfg)
  1815. {
  1816. struct rspamd_fuzzy_storage_ctx *ctx;
  1817. GQuark type;
  1818. type = g_quark_try_string ("fuzzy");
  1819. ctx = rspamd_mempool_alloc0 (cfg->cfg_pool,
  1820. sizeof (struct rspamd_fuzzy_storage_ctx));
  1821. ctx->magic = rspamd_fuzzy_storage_magic;
  1822. ctx->sync_timeout = DEFAULT_SYNC_TIMEOUT;
  1823. ctx->keypair_cache_size = DEFAULT_KEYPAIR_CACHE_SIZE;
  1824. ctx->lua_pre_handler_cbref = -1;
  1825. ctx->lua_post_handler_cbref = -1;
  1826. ctx->keys = g_hash_table_new_full (fuzzy_kp_hash, fuzzy_kp_equal,
  1827. NULL, fuzzy_key_dtor);
  1828. rspamd_mempool_add_destructor (cfg->cfg_pool,
  1829. (rspamd_mempool_destruct_t)g_hash_table_unref, ctx->keys);
  1830. ctx->errors_ips = rspamd_lru_hash_new_full (1024,
  1831. (GDestroyNotify) rspamd_inet_address_free, g_free,
  1832. rspamd_inet_address_hash, rspamd_inet_address_equal);
  1833. rspamd_mempool_add_destructor (cfg->cfg_pool,
  1834. (rspamd_mempool_destruct_t)rspamd_lru_hash_destroy, ctx->errors_ips);
  1835. ctx->cfg = cfg;
  1836. ctx->updates_maxfail = DEFAULT_UPDATES_MAXFAIL;
  1837. ctx->leaky_bucket_mask = DEFAULT_BUCKET_MASK;
  1838. ctx->leaky_bucket_ttl = DEFAULT_BUCKET_TTL;
  1839. ctx->max_buckets = DEFAULT_MAX_BUCKETS;
  1840. ctx->leaky_bucket_burst = NAN;
  1841. ctx->leaky_bucket_rate = NAN;
  1842. ctx->delay = NAN;
  1843. rspamd_rcl_register_worker_option (cfg,
  1844. type,
  1845. "sync",
  1846. rspamd_rcl_parse_struct_time,
  1847. ctx,
  1848. G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx,
  1849. sync_timeout),
  1850. RSPAMD_CL_FLAG_TIME_FLOAT,
  1851. "Time to perform database sync, default: "
  1852. G_STRINGIFY (DEFAULT_SYNC_TIMEOUT) " seconds");
  1853. rspamd_rcl_register_worker_option (cfg,
  1854. type,
  1855. "expire",
  1856. rspamd_rcl_parse_struct_time,
  1857. ctx,
  1858. G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx,
  1859. expire),
  1860. RSPAMD_CL_FLAG_TIME_FLOAT,
  1861. "Default expire time for hashes, default: "
  1862. G_STRINGIFY (DEFAULT_EXPIRE) " seconds");
  1863. rspamd_rcl_register_worker_option (cfg,
  1864. type,
  1865. "delay",
  1866. rspamd_rcl_parse_struct_time,
  1867. ctx,
  1868. G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx,
  1869. delay),
  1870. RSPAMD_CL_FLAG_TIME_FLOAT,
  1871. "Default delay time for hashes, default: not enabled");
  1872. rspamd_rcl_register_worker_option (cfg,
  1873. type,
  1874. "allow_update",
  1875. rspamd_rcl_parse_struct_ucl,
  1876. ctx,
  1877. G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, update_map),
  1878. 0,
  1879. "Allow modifications from the following IP addresses");
  1880. rspamd_rcl_register_worker_option (cfg,
  1881. type,
  1882. "delay_whitelist",
  1883. rspamd_rcl_parse_struct_ucl,
  1884. ctx,
  1885. G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, delay_whitelist_map),
  1886. 0,
  1887. "Disable delay check for the following IP addresses");
  1888. rspamd_rcl_register_worker_option (cfg,
  1889. type,
  1890. "keypair",
  1891. fuzzy_parse_keypair,
  1892. ctx,
  1893. 0,
  1894. RSPAMD_CL_FLAG_MULTIPLE,
  1895. "Encryption keypair (can be repeated for different keys)");
  1896. rspamd_rcl_register_worker_option (cfg,
  1897. type,
  1898. "keypair_cache_size",
  1899. rspamd_rcl_parse_struct_integer,
  1900. ctx,
  1901. G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx,
  1902. keypair_cache_size),
  1903. RSPAMD_CL_FLAG_UINT,
  1904. "Size of keypairs cache, default: "
  1905. G_STRINGIFY (DEFAULT_KEYPAIR_CACHE_SIZE));
  1906. rspamd_rcl_register_worker_option (cfg,
  1907. type,
  1908. "encrypted_only",
  1909. rspamd_rcl_parse_struct_boolean,
  1910. ctx,
  1911. G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, encrypted_only),
  1912. 0,
  1913. "Allow encrypted requests only (and forbid all unknown keys or plaintext requests)");
  1914. rspamd_rcl_register_worker_option (cfg,
  1915. type,
  1916. "dedicated_update_worker",
  1917. rspamd_rcl_parse_struct_boolean,
  1918. ctx,
  1919. G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, dedicated_update_worker),
  1920. 0,
  1921. "Use worker 0 for updates only");
  1922. rspamd_rcl_register_worker_option (cfg,
  1923. type,
  1924. "read_only",
  1925. rspamd_rcl_parse_struct_boolean,
  1926. ctx,
  1927. G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, read_only),
  1928. 0,
  1929. "Work in read only mode");
  1930. rspamd_rcl_register_worker_option (cfg,
  1931. type,
  1932. "blocked",
  1933. rspamd_rcl_parse_struct_ucl,
  1934. ctx,
  1935. G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, blocked_map),
  1936. 0,
  1937. "Block requests from specific networks");
  1938. rspamd_rcl_register_worker_option (cfg,
  1939. type,
  1940. "updates_maxfail",
  1941. rspamd_rcl_parse_struct_integer,
  1942. ctx,
  1943. G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, updates_maxfail),
  1944. RSPAMD_CL_FLAG_UINT,
  1945. "Maximum number of updates to be failed before discarding");
  1946. rspamd_rcl_register_worker_option (cfg,
  1947. type,
  1948. "skip_hashes",
  1949. rspamd_rcl_parse_struct_ucl,
  1950. ctx,
  1951. G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, skip_map),
  1952. 0,
  1953. "Skip specific hashes from the map");
  1954. /* Ratelimits */
  1955. rspamd_rcl_register_worker_option (cfg,
  1956. type,
  1957. "ratelimit_whitelist",
  1958. rspamd_rcl_parse_struct_ucl,
  1959. ctx,
  1960. G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, ratelimit_whitelist_map),
  1961. 0,
  1962. "Skip specific addresses from rate limiting");
  1963. rspamd_rcl_register_worker_option (cfg,
  1964. type,
  1965. "ratelimit_max_buckets",
  1966. rspamd_rcl_parse_struct_integer,
  1967. ctx,
  1968. G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, max_buckets),
  1969. RSPAMD_CL_FLAG_UINT,
  1970. "Maximum number of leaky buckets (default: " G_STRINGIFY(DEFAULT_MAX_BUCKETS) ")");
  1971. rspamd_rcl_register_worker_option (cfg,
  1972. type,
  1973. "ratelimit_network_mask",
  1974. rspamd_rcl_parse_struct_integer,
  1975. ctx,
  1976. G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, leaky_bucket_mask),
  1977. RSPAMD_CL_FLAG_UINT,
  1978. "Network mask to apply for IPv4 rate addresses (default: " G_STRINGIFY(DEFAULT_BUCKET_MASK) ")");
  1979. rspamd_rcl_register_worker_option (cfg,
  1980. type,
  1981. "ratelimit_bucket_ttl",
  1982. rspamd_rcl_parse_struct_time,
  1983. ctx,
  1984. G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, leaky_bucket_ttl),
  1985. RSPAMD_CL_FLAG_TIME_INTEGER,
  1986. "Time to live for ratelimit element (default: " G_STRINGIFY(DEFAULT_BUCKET_TTL) ")");
  1987. rspamd_rcl_register_worker_option (cfg,
  1988. type,
  1989. "ratelimit_rate",
  1990. rspamd_rcl_parse_struct_double,
  1991. ctx,
  1992. G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, leaky_bucket_rate),
  1993. 0,
  1994. "Leak rate in requests per second");
  1995. rspamd_rcl_register_worker_option (cfg,
  1996. type,
  1997. "ratelimit_burst",
  1998. rspamd_rcl_parse_struct_double,
  1999. ctx,
  2000. G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, leaky_bucket_burst),
  2001. 0,
  2002. "Peak value for ratelimit bucket");
  2003. rspamd_rcl_register_worker_option (cfg,
  2004. type,
  2005. "ratelimit_log_only",
  2006. rspamd_rcl_parse_struct_boolean,
  2007. ctx,
  2008. G_STRUCT_OFFSET (struct rspamd_fuzzy_storage_ctx, ratelimit_log_only),
  2009. 0,
  2010. "Don't really ban on ratelimit reaching, just log");
  2011. return ctx;
  2012. }
  2013. static void
  2014. rspamd_fuzzy_peer_io (EV_P_ ev_io *w, int revents)
  2015. {
  2016. struct fuzzy_peer_cmd cmd;
  2017. struct rspamd_fuzzy_storage_ctx *ctx =
  2018. (struct rspamd_fuzzy_storage_ctx *)w->data;
  2019. gssize r;
  2020. for (;;) {
  2021. r = read (w->fd, &cmd, sizeof (cmd));
  2022. if (r != sizeof (cmd)) {
  2023. if (errno == EINTR) {
  2024. continue;
  2025. }
  2026. if (errno != EAGAIN) {
  2027. msg_err ("cannot read command from peers: %s", strerror (errno));
  2028. }
  2029. break;
  2030. }
  2031. else {
  2032. g_array_append_val (ctx->updates_pending, cmd);
  2033. }
  2034. }
  2035. }
  2036. static void
  2037. fuzzy_peer_rep (struct rspamd_worker *worker,
  2038. struct rspamd_srv_reply *rep, gint rep_fd,
  2039. gpointer ud)
  2040. {
  2041. struct rspamd_fuzzy_storage_ctx *ctx = ud;
  2042. GList *cur;
  2043. struct rspamd_worker_listen_socket *ls;
  2044. struct rspamd_worker_accept_event *ac_ev;
  2045. ctx->peer_fd = rep_fd;
  2046. if (rep_fd == -1) {
  2047. msg_err ("cannot receive peer fd from the main process");
  2048. exit (EXIT_FAILURE);
  2049. }
  2050. else {
  2051. rspamd_socket_nonblocking (rep_fd);
  2052. }
  2053. msg_info ("got peer fd reply from the main process");
  2054. /* Start listening */
  2055. cur = worker->cf->listen_socks;
  2056. while (cur) {
  2057. ls = cur->data;
  2058. if (ls->fd != -1) {
  2059. msg_info ("start listening on %s",
  2060. rspamd_inet_address_to_string_pretty (ls->addr));
  2061. if (ls->type == RSPAMD_WORKER_SOCKET_UDP) {
  2062. ac_ev = g_malloc0 (sizeof (*ac_ev));
  2063. ac_ev->accept_ev.data = worker;
  2064. ac_ev->event_loop = ctx->event_loop;
  2065. ev_io_init (&ac_ev->accept_ev, accept_fuzzy_socket, ls->fd,
  2066. EV_READ);
  2067. ev_io_start (ctx->event_loop, &ac_ev->accept_ev);
  2068. DL_APPEND (worker->accept_events, ac_ev);
  2069. }
  2070. else {
  2071. /* We allow TCP listeners only for a update worker */
  2072. g_assert_not_reached ();
  2073. }
  2074. }
  2075. cur = g_list_next (cur);
  2076. }
  2077. if (ctx->peer_fd != -1) {
  2078. if (worker->index == 0) {
  2079. /* Listen for peer requests */
  2080. shutdown (ctx->peer_fd, SHUT_WR);
  2081. ctx->peer_ev.data = ctx;
  2082. ev_io_init (&ctx->peer_ev, rspamd_fuzzy_peer_io, ctx->peer_fd, EV_READ);
  2083. ev_io_start (ctx->event_loop, &ctx->peer_ev);
  2084. }
  2085. else {
  2086. shutdown (ctx->peer_fd, SHUT_RD);
  2087. }
  2088. }
  2089. }
  2090. /*
  2091. * Start worker process
  2092. */
  2093. __attribute__((noreturn))
  2094. void
  2095. start_fuzzy (struct rspamd_worker *worker)
  2096. {
  2097. struct rspamd_fuzzy_storage_ctx *ctx = worker->ctx;
  2098. GError *err = NULL;
  2099. struct rspamd_srv_command srv_cmd;
  2100. struct rspamd_config *cfg = worker->srv->cfg;
  2101. g_assert (rspamd_worker_check_context (worker->ctx, rspamd_fuzzy_storage_magic));
  2102. ctx->event_loop = rspamd_prepare_worker (worker,
  2103. "fuzzy",
  2104. NULL);
  2105. ctx->peer_fd = -1;
  2106. ctx->worker = worker;
  2107. ctx->cfg = worker->srv->cfg;
  2108. ctx->resolver = rspamd_dns_resolver_init (worker->srv->logger,
  2109. ctx->event_loop,
  2110. worker->srv->cfg);
  2111. rspamd_upstreams_library_config (worker->srv->cfg, ctx->cfg->ups_ctx,
  2112. ctx->event_loop, ctx->resolver->r);
  2113. /* Since this worker uses maps it needs a valid HTTP context */
  2114. ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->event_loop,
  2115. ctx->cfg->ups_ctx);
  2116. if (ctx->keypair_cache_size > 0) {
  2117. /* Create keypairs cache */
  2118. ctx->keypair_cache = rspamd_keypair_cache_new (ctx->keypair_cache_size);
  2119. }
  2120. if ((ctx->backend = rspamd_fuzzy_backend_create (ctx->event_loop,
  2121. worker->cf->options, cfg, &err)) == NULL) {
  2122. msg_err ("cannot open backend: %e", err);
  2123. if (err) {
  2124. g_error_free (err);
  2125. }
  2126. exit (EXIT_SUCCESS);
  2127. }
  2128. rspamd_fuzzy_backend_count (ctx->backend, fuzzy_count_callback, ctx);
  2129. if (worker->index == 0) {
  2130. ctx->updates_pending = g_array_sized_new (FALSE, FALSE,
  2131. sizeof (struct fuzzy_peer_cmd), 1024);
  2132. rspamd_fuzzy_backend_start_update (ctx->backend, ctx->sync_timeout,
  2133. rspamd_fuzzy_storage_periodic_callback, ctx);
  2134. if (ctx->dedicated_update_worker && worker->cf->count > 1) {
  2135. msg_info_config ("stop serving clients request in dedicated update mode");
  2136. rspamd_worker_stop_accept (worker);
  2137. GList *cur = worker->cf->listen_socks;
  2138. while (cur) {
  2139. struct rspamd_worker_listen_socket *ls =
  2140. (struct rspamd_worker_listen_socket *)cur->data;
  2141. if (ls->fd != -1) {
  2142. close (ls->fd);
  2143. }
  2144. cur = g_list_next (cur);
  2145. }
  2146. }
  2147. }
  2148. ctx->stat_ev.data = ctx;
  2149. ev_timer_init (&ctx->stat_ev, rspamd_fuzzy_stat_callback, ctx->sync_timeout,
  2150. ctx->sync_timeout);
  2151. ev_timer_start (ctx->event_loop, &ctx->stat_ev);
  2152. /* Register custom reload and stat commands for the control socket */
  2153. rspamd_control_worker_add_cmd_handler (worker, RSPAMD_CONTROL_RELOAD,
  2154. rspamd_fuzzy_storage_reload, ctx);
  2155. rspamd_control_worker_add_cmd_handler (worker, RSPAMD_CONTROL_FUZZY_STAT,
  2156. rspamd_fuzzy_storage_stat, ctx);
  2157. rspamd_control_worker_add_cmd_handler (worker, RSPAMD_CONTROL_FUZZY_SYNC,
  2158. rspamd_fuzzy_storage_sync, ctx);
  2159. /* Create radix trees */
  2160. if (ctx->update_map != NULL) {
  2161. rspamd_config_radix_from_ucl (worker->srv->cfg, ctx->update_map,
  2162. "Allow fuzzy updates from specified addresses",
  2163. &ctx->update_ips, NULL, worker, "fuzzy update");
  2164. }
  2165. if (ctx->skip_map != NULL) {
  2166. struct rspamd_map *m;
  2167. if ((m = rspamd_map_add_from_ucl (cfg, ctx->skip_map,
  2168. "Skip hashes",
  2169. rspamd_kv_list_read,
  2170. rspamd_kv_list_fin,
  2171. rspamd_kv_list_dtor,
  2172. (void **)&ctx->skip_hashes,
  2173. worker, RSPAMD_MAP_DEFAULT)) == NULL) {
  2174. msg_warn_config ("cannot load hashes list from %s",
  2175. ucl_object_tostring (ctx->skip_map));
  2176. }
  2177. else {
  2178. m->active_http = TRUE;
  2179. }
  2180. }
  2181. if (ctx->blocked_map != NULL) {
  2182. rspamd_config_radix_from_ucl (worker->srv->cfg, ctx->blocked_map,
  2183. "Block fuzzy requests from the specific IPs",
  2184. &ctx->blocked_ips,
  2185. NULL,
  2186. worker, "fuzzy blocked");
  2187. }
  2188. /* Create radix trees */
  2189. if (ctx->ratelimit_whitelist_map != NULL) {
  2190. rspamd_config_radix_from_ucl (worker->srv->cfg, ctx->ratelimit_whitelist_map,
  2191. "Skip ratelimits from specific ip addresses/networks",
  2192. &ctx->ratelimit_whitelist,
  2193. NULL,
  2194. worker, "fuzzy ratelimit whitelist");
  2195. }
  2196. if (!isnan (ctx->delay) && ctx->delay_whitelist_map != NULL) {
  2197. rspamd_config_radix_from_ucl (worker->srv->cfg, ctx->delay_whitelist_map,
  2198. "Skip delay from the following ips",
  2199. &ctx->delay_whitelist, NULL, worker,
  2200. "fuzzy delayed whitelist");
  2201. }
  2202. /* Ratelimits */
  2203. if (!isnan (ctx->leaky_bucket_rate) && !isnan (ctx->leaky_bucket_burst)) {
  2204. ctx->ratelimit_buckets = rspamd_lru_hash_new_full (ctx->max_buckets,
  2205. NULL, fuzzy_rl_bucket_free,
  2206. rspamd_inet_address_hash, rspamd_inet_address_equal);
  2207. }
  2208. /* Maps events */
  2209. ctx->resolver = rspamd_dns_resolver_init (worker->srv->logger,
  2210. ctx->event_loop,
  2211. worker->srv->cfg);
  2212. rspamd_map_watch (worker->srv->cfg, ctx->event_loop,
  2213. ctx->resolver, worker, RSPAMD_MAP_WATCH_WORKER);
  2214. /* Get peer pipe */
  2215. memset (&srv_cmd, 0, sizeof (srv_cmd));
  2216. srv_cmd.type = RSPAMD_SRV_SOCKETPAIR;
  2217. srv_cmd.cmd.spair.af = SOCK_DGRAM;
  2218. srv_cmd.cmd.spair.pair_num = worker->index;
  2219. memset (srv_cmd.cmd.spair.pair_id, 0, sizeof (srv_cmd.cmd.spair.pair_id));
  2220. /* 6 bytes of id (including \0) and bind_conf id */
  2221. G_STATIC_ASSERT (sizeof (srv_cmd.cmd.spair.pair_id) >=
  2222. sizeof ("fuzzy") + sizeof (guint64));
  2223. memcpy (srv_cmd.cmd.spair.pair_id, "fuzzy", sizeof ("fuzzy"));
  2224. /* Distinguish workers from each others... */
  2225. if (worker->cf->bind_conf && worker->cf->bind_conf->bind_line) {
  2226. guint64 bind_hash = rspamd_cryptobox_fast_hash (worker->cf->bind_conf->bind_line,
  2227. strlen (worker->cf->bind_conf->bind_line), 0xdeadbabe);
  2228. /* 8 more bytes */
  2229. memcpy (srv_cmd.cmd.spair.pair_id + sizeof ("fuzzy"), &bind_hash,
  2230. sizeof (bind_hash));
  2231. }
  2232. rspamd_srv_send_command (worker, ctx->event_loop, &srv_cmd, -1,
  2233. fuzzy_peer_rep, ctx);
  2234. luaL_Reg fuzzy_lua_reg = {
  2235. .name = "add_fuzzy_pre_handler",
  2236. .func = lua_fuzzy_add_pre_handler,
  2237. };
  2238. rspamd_lua_add_metamethod (ctx->cfg->lua_state, "rspamd{worker}", &fuzzy_lua_reg);
  2239. fuzzy_lua_reg = (luaL_Reg){
  2240. .name = "add_fuzzy_post_handler",
  2241. .func = lua_fuzzy_add_post_handler,
  2242. };
  2243. rspamd_lua_add_metamethod (ctx->cfg->lua_state, "rspamd{worker}", &fuzzy_lua_reg);
  2244. rspamd_lua_run_postloads (ctx->cfg->lua_state, ctx->cfg, ctx->event_loop,
  2245. worker);
  2246. ev_loop (ctx->event_loop, 0);
  2247. rspamd_worker_block_signals ();
  2248. if (ctx->peer_fd != -1) {
  2249. if (worker->index == 0) {
  2250. ev_io_stop (ctx->event_loop, &ctx->peer_ev);
  2251. }
  2252. close (ctx->peer_fd);
  2253. }
  2254. if (worker->index == 0 && ctx->updates_pending->len > 0) {
  2255. msg_info_config ("start another event loop to sync fuzzy storage");
  2256. if (rspamd_fuzzy_process_updates_queue (ctx, local_db_name, TRUE)) {
  2257. ev_loop (ctx->event_loop, 0);
  2258. msg_info_config ("sync cycle is done");
  2259. }
  2260. else {
  2261. msg_info_config ("no need to sync");
  2262. }
  2263. }
  2264. rspamd_fuzzy_backend_close (ctx->backend);
  2265. if (worker->index == 0) {
  2266. g_array_free (ctx->updates_pending, TRUE);
  2267. ctx->updates_pending = NULL;
  2268. }
  2269. if (ctx->keypair_cache) {
  2270. rspamd_keypair_cache_destroy (ctx->keypair_cache);
  2271. }
  2272. if (ctx->ratelimit_buckets) {
  2273. rspamd_lru_hash_destroy (ctx->ratelimit_buckets);
  2274. }
  2275. if (ctx->lua_pre_handler_cbref != -1) {
  2276. luaL_unref (ctx->cfg->lua_state, LUA_REGISTRYINDEX, ctx->lua_pre_handler_cbref);
  2277. }
  2278. if (ctx->lua_post_handler_cbref != -1) {
  2279. luaL_unref (ctx->cfg->lua_state, LUA_REGISTRYINDEX, ctx->lua_post_handler_cbref);
  2280. }
  2281. REF_RELEASE (ctx->cfg);
  2282. rspamd_log_close (worker->srv->logger);
  2283. exit (EXIT_SUCCESS);
  2284. }