You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

lua_http.c 27KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157
  1. /*-
  2. * Copyright 2016 Vsevolod Stakhov
  3. *
  4. * Licensed under the Apache License, Version 2.0 (the "License");
  5. * you may not use this file except in compliance with the License.
  6. * You may obtain a copy of the License at
  7. *
  8. * http://www.apache.org/licenses/LICENSE-2.0
  9. *
  10. * Unless required by applicable law or agreed to in writing, software
  11. * distributed under the License is distributed on an "AS IS" BASIS,
  12. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. * See the License for the specific language governing permissions and
  14. * limitations under the License.
  15. */
  16. #include "lua_common.h"
  17. #include "lua_thread_pool.h"
  18. #include "libserver/http/http_private.h"
  19. #include "ref.h"
  20. #include "unix-std.h"
  21. #include "zlib.h"
  22. /***
  23. * @module rspamd_http
  24. * Rspamd HTTP module represents HTTP asynchronous client available from LUA code.
  25. * This module hides all complexity: DNS resolving, sessions management, zero-copy
  26. * text transfers and so on under the hood.
  27. * @example
  28. local rspamd_http = require "rspamd_http"
  29. local function symbol_callback(task)
  30. local function http_callback(err_message, code, body, headers)
  31. task:insert_result('SYMBOL', 1) -- task is available via closure
  32. end
  33. rspamd_http.request({
  34. task=task,
  35. url='http://example.com/data',
  36. body=task:get_content(),
  37. callback=http_callback,
  38. headers={Header='Value', OtherHeader='Value'},
  39. mime_type='text/plain',
  40. })
  41. end
  42. */
  43. #define MAX_HEADERS_SIZE 8192
  44. static const gchar *M = "rspamd lua http";
  45. LUA_FUNCTION_DEF (http, request);
  46. static const struct luaL_reg httplib_m[] = {
  47. LUA_INTERFACE_DEF (http, request),
  48. {"__tostring", rspamd_lua_class_tostring},
  49. {NULL, NULL}
  50. };
  51. #define RSPAMD_LUA_HTTP_FLAG_TEXT (1 << 0)
  52. #define RSPAMD_LUA_HTTP_FLAG_NOVERIFY (1 << 1)
  53. #define RSPAMD_LUA_HTTP_FLAG_RESOLVED (1 << 2)
  54. #define RSPAMD_LUA_HTTP_FLAG_KEEP_ALIVE (1 << 3)
  55. #define RSPAMD_LUA_HTTP_FLAG_YIELDED (1 << 4)
  56. struct lua_http_cbdata {
  57. struct rspamd_http_connection *conn;
  58. struct rspamd_async_session *session;
  59. struct rspamd_symcache_item *item;
  60. struct rspamd_http_message *msg;
  61. struct ev_loop *event_loop;
  62. struct rspamd_config *cfg;
  63. struct rspamd_task *task;
  64. ev_tstamp timeout;
  65. struct rspamd_cryptobox_keypair *local_kp;
  66. struct rspamd_cryptobox_pubkey *peer_pk;
  67. rspamd_inet_addr_t *addr;
  68. gchar *mime_type;
  69. gchar *host;
  70. gchar *auth;
  71. const gchar *url;
  72. gsize max_size;
  73. gint flags;
  74. gint fd;
  75. gint cbref;
  76. struct thread_entry *thread;
  77. ref_entry_t ref;
  78. };
  79. static const gdouble default_http_timeout = 5.0;
  80. static struct rspamd_dns_resolver *
  81. lua_http_global_resolver (struct ev_loop *ev_base)
  82. {
  83. static struct rspamd_dns_resolver *global_resolver;
  84. if (global_resolver == NULL) {
  85. global_resolver = rspamd_dns_resolver_init (NULL, ev_base, NULL);
  86. }
  87. return global_resolver;
  88. }
  89. static void
  90. lua_http_fin (gpointer arg)
  91. {
  92. struct lua_http_cbdata *cbd = (struct lua_http_cbdata *)arg;
  93. if (cbd->cbref != -1) {
  94. luaL_unref (cbd->cfg->lua_state, LUA_REGISTRYINDEX, cbd->cbref);
  95. }
  96. if (cbd->conn) {
  97. /* Here we already have a connection, so we need to unref it */
  98. rspamd_http_connection_unref (cbd->conn);
  99. }
  100. else if (cbd->msg != NULL) {
  101. /* We need to free message */
  102. rspamd_http_message_unref (cbd->msg);
  103. }
  104. if (cbd->fd != -1) {
  105. close (cbd->fd);
  106. }
  107. if (cbd->addr) {
  108. rspamd_inet_address_free (cbd->addr);
  109. }
  110. if (cbd->mime_type) {
  111. g_free (cbd->mime_type);
  112. }
  113. if (cbd->auth) {
  114. g_free (cbd->auth);
  115. }
  116. if (cbd->local_kp) {
  117. rspamd_keypair_unref (cbd->local_kp);
  118. }
  119. if (cbd->peer_pk) {
  120. rspamd_pubkey_unref (cbd->peer_pk);
  121. }
  122. g_free (cbd);
  123. }
  124. static void
  125. lua_http_cbd_dtor (struct lua_http_cbdata *cbd)
  126. {
  127. if (cbd->session) {
  128. if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_RESOLVED) {
  129. /* Event is added merely for resolved events */
  130. if (cbd->item) {
  131. rspamd_symcache_item_async_dec_check (cbd->task, cbd->item, M);
  132. }
  133. rspamd_session_remove_event (cbd->session, lua_http_fin, cbd);
  134. }
  135. }
  136. else {
  137. lua_http_fin (cbd);
  138. }
  139. }
  140. static void
  141. lua_http_push_error (struct lua_http_cbdata *cbd, const char *err)
  142. {
  143. struct lua_callback_state lcbd;
  144. lua_State *L;
  145. lua_thread_pool_prepare_callback (cbd->cfg->lua_thread_pool, &lcbd);
  146. L = lcbd.L;
  147. lua_rawgeti (L, LUA_REGISTRYINDEX, cbd->cbref);
  148. lua_pushstring (L, err);
  149. if (cbd->item) {
  150. rspamd_symcache_set_cur_item (cbd->task, cbd->item);
  151. }
  152. if (lua_pcall (L, 1, 0, 0) != 0) {
  153. msg_info ("callback call failed: %s", lua_tostring (L, -1));
  154. lua_pop (L, 1);
  155. }
  156. lua_thread_pool_restore_callback (&lcbd);
  157. }
  158. static void lua_http_resume_handler (struct rspamd_http_connection *conn,
  159. struct rspamd_http_message *msg, const char *err);
  160. static void
  161. lua_http_error_handler (struct rspamd_http_connection *conn, GError *err)
  162. {
  163. struct lua_http_cbdata *cbd = (struct lua_http_cbdata *)conn->ud;
  164. if (cbd->cbref == -1) {
  165. if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_YIELDED) {
  166. cbd->flags &= ~RSPAMD_LUA_HTTP_FLAG_YIELDED;
  167. lua_http_resume_handler (conn, NULL, err->message);
  168. }
  169. else {
  170. /* TODO: kill me please */
  171. msg_info ("lost HTTP error from %s in coroutines mess: %s",
  172. rspamd_inet_address_to_string_pretty (cbd->addr),
  173. err->message);
  174. }
  175. }
  176. else {
  177. lua_http_push_error (cbd, err->message);
  178. }
  179. REF_RELEASE (cbd);
  180. }
  181. static int
  182. lua_http_finish_handler (struct rspamd_http_connection *conn,
  183. struct rspamd_http_message *msg)
  184. {
  185. struct lua_http_cbdata *cbd = (struct lua_http_cbdata *)conn->ud;
  186. struct rspamd_http_header *h;
  187. const gchar *body;
  188. gsize body_len;
  189. struct lua_callback_state lcbd;
  190. lua_State *L;
  191. if (cbd->cbref == -1) {
  192. if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_YIELDED) {
  193. cbd->flags &= ~RSPAMD_LUA_HTTP_FLAG_YIELDED;
  194. lua_http_resume_handler (conn, msg, NULL);
  195. }
  196. else {
  197. /* TODO: kill me please */
  198. msg_err ("lost HTTP data from %s in coroutines mess",
  199. rspamd_inet_address_to_string_pretty (cbd->addr));
  200. }
  201. REF_RELEASE (cbd);
  202. return 0;
  203. }
  204. lua_thread_pool_prepare_callback (cbd->cfg->lua_thread_pool, &lcbd);
  205. L = lcbd.L;
  206. lua_rawgeti (L, LUA_REGISTRYINDEX, cbd->cbref);
  207. /* Error */
  208. lua_pushnil (L);
  209. /* Reply code */
  210. lua_pushinteger (L, msg->code);
  211. /* Body */
  212. body = rspamd_http_message_get_body (msg, &body_len);
  213. if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_TEXT) {
  214. struct rspamd_lua_text *t;
  215. t = lua_newuserdata (L, sizeof (*t));
  216. rspamd_lua_setclass (L, "rspamd{text}", -1);
  217. t->start = body;
  218. t->len = body_len;
  219. t->flags = 0;
  220. }
  221. else {
  222. if (body_len > 0) {
  223. lua_pushlstring (L, body, body_len);
  224. }
  225. else {
  226. lua_pushnil (L);
  227. }
  228. }
  229. /* Headers */
  230. lua_newtable (L);
  231. kh_foreach_value (msg->headers, h, {
  232. /*
  233. * Lowercase header name, as Lua cannot search in caseless matter
  234. */
  235. rspamd_str_lc (h->combined->str, h->name.len);
  236. lua_pushlstring (L, h->name.begin, h->name.len);
  237. lua_pushlstring (L, h->value.begin, h->value.len);
  238. lua_settable (L, -3);
  239. });
  240. if (cbd->item) {
  241. /* Replace watcher to deal with nested calls */
  242. rspamd_symcache_set_cur_item (cbd->task, cbd->item);
  243. }
  244. if (lua_pcall (L, 4, 0, 0) != 0) {
  245. msg_info ("callback call failed: %s", lua_tostring (L, -1));
  246. lua_pop (L, 1);
  247. }
  248. REF_RELEASE (cbd);
  249. lua_thread_pool_restore_callback (&lcbd);
  250. return 0;
  251. }
  252. /*
  253. * resumes yielded thread
  254. */
  255. static void
  256. lua_http_resume_handler (struct rspamd_http_connection *conn,
  257. struct rspamd_http_message *msg, const char *err)
  258. {
  259. struct lua_http_cbdata *cbd = (struct lua_http_cbdata *)conn->ud;
  260. lua_State *L = cbd->thread->lua_state;
  261. const gchar *body;
  262. gsize body_len;
  263. struct rspamd_http_header *h;
  264. if (err) {
  265. lua_pushstring (L, err);
  266. lua_pushnil (L);
  267. }
  268. else {
  269. /*
  270. * 1 - nil (error)
  271. * 2 - table:
  272. * code (int)
  273. * content (string)
  274. * headers (table: header -> value)
  275. */
  276. lua_pushnil (L); // error code
  277. lua_createtable (L, 0, 3);
  278. /* code */
  279. lua_pushliteral (L, "code");
  280. lua_pushinteger (L, msg->code);
  281. lua_settable (L, -3);
  282. /* content */
  283. lua_pushliteral (L, "content");
  284. body = rspamd_http_message_get_body (msg, &body_len);
  285. if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_TEXT) {
  286. struct rspamd_lua_text *t;
  287. t = lua_newuserdata (L, sizeof (*t));
  288. rspamd_lua_setclass (L, "rspamd{text}", -1);
  289. t->start = body;
  290. t->len = body_len;
  291. t->flags = 0;
  292. }
  293. else {
  294. if (body_len > 0) {
  295. lua_pushlstring (L, body, body_len);
  296. }
  297. else {
  298. lua_pushnil (L);
  299. }
  300. }
  301. lua_settable (L, -3);
  302. /* headers */
  303. lua_pushliteral (L, "headers");
  304. lua_newtable (L);
  305. kh_foreach_value (msg->headers, h, {
  306. /*
  307. * Lowercase header name, as Lua cannot search in caseless matter
  308. */
  309. rspamd_str_lc (h->combined->str, h->name.len);
  310. lua_pushlstring (L, h->name.begin, h->name.len);
  311. lua_pushlstring (L, h->value.begin, h->value.len);
  312. lua_settable (L, -3);
  313. });
  314. lua_settable (L, -3);
  315. }
  316. if (cbd->item) {
  317. /* Replace watcher to deal with nested calls */
  318. rspamd_symcache_set_cur_item (cbd->task, cbd->item);
  319. }
  320. lua_thread_resume (cbd->thread, 2);
  321. }
  322. static gboolean
  323. lua_http_make_connection (struct lua_http_cbdata *cbd)
  324. {
  325. rspamd_inet_address_set_port (cbd->addr, cbd->msg->port);
  326. if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_KEEP_ALIVE) {
  327. cbd->fd = -1; /* FD is owned by keepalive connection */
  328. cbd->conn = rspamd_http_connection_new_keepalive (
  329. NULL, /* Default context */
  330. NULL,
  331. lua_http_error_handler,
  332. lua_http_finish_handler,
  333. cbd->addr,
  334. cbd->host);
  335. }
  336. else {
  337. cbd->fd = -1;
  338. cbd->conn = rspamd_http_connection_new_client (
  339. NULL, /* Default context */
  340. NULL,
  341. lua_http_error_handler,
  342. lua_http_finish_handler,
  343. RSPAMD_HTTP_CLIENT_SIMPLE,
  344. cbd->addr);
  345. }
  346. if (cbd->conn) {
  347. if (cbd->local_kp) {
  348. rspamd_http_connection_set_key (cbd->conn, cbd->local_kp);
  349. }
  350. if (cbd->peer_pk) {
  351. rspamd_http_message_set_peer_key (cbd->msg, cbd->peer_pk);
  352. }
  353. if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_NOVERIFY) {
  354. cbd->msg->flags |= RSPAMD_HTTP_FLAG_SSL_NOVERIFY;
  355. }
  356. if (cbd->max_size) {
  357. rspamd_http_connection_set_max_size (cbd->conn, cbd->max_size);
  358. }
  359. if (cbd->auth) {
  360. rspamd_http_message_add_header (cbd->msg, "Authorization",
  361. cbd->auth);
  362. }
  363. if (cbd->session) {
  364. rspamd_session_add_event (cbd->session,
  365. (event_finalizer_t) lua_http_fin, cbd,
  366. M);
  367. cbd->flags |= RSPAMD_LUA_HTTP_FLAG_RESOLVED;
  368. }
  369. if (cbd->task) {
  370. cbd->conn->log_tag = cbd->task->task_pool->tag.uid;
  371. if (cbd->item) {
  372. rspamd_symcache_item_async_inc (cbd->task, cbd->item, M);
  373. }
  374. }
  375. else if (cbd->cfg) {
  376. cbd->conn->log_tag = cbd->cfg->cfg_pool->tag.uid;
  377. }
  378. struct rspamd_http_message *msg = cbd->msg;
  379. /* Message is now owned by a connection object */
  380. cbd->msg = NULL;
  381. return rspamd_http_connection_write_message (cbd->conn, msg,
  382. cbd->host, cbd->mime_type, cbd,
  383. cbd->timeout);
  384. }
  385. return FALSE;
  386. }
  387. static void
  388. lua_http_dns_handler (struct rdns_reply *reply, gpointer ud)
  389. {
  390. struct lua_http_cbdata *cbd = (struct lua_http_cbdata *)ud;
  391. struct rspamd_symcache_item *item;
  392. struct rspamd_task *task;
  393. task = cbd->task;
  394. item = cbd->item;
  395. if (reply->code != RDNS_RC_NOERROR) {
  396. lua_http_push_error (cbd, "unable to resolve host");
  397. REF_RELEASE (cbd);
  398. }
  399. else {
  400. if (reply->entries->type == RDNS_REQUEST_A) {
  401. cbd->addr = rspamd_inet_address_new (AF_INET,
  402. &reply->entries->content.a.addr);
  403. }
  404. else if (reply->entries->type == RDNS_REQUEST_AAAA) {
  405. cbd->addr = rspamd_inet_address_new (AF_INET6,
  406. &reply->entries->content.aaa.addr);
  407. }
  408. REF_RETAIN (cbd);
  409. if (!lua_http_make_connection (cbd)) {
  410. lua_http_push_error (cbd, "unable to make connection to the host");
  411. if (cbd->ref.refcount > 1) {
  412. REF_RELEASE (cbd);
  413. }
  414. REF_RELEASE (cbd);
  415. return;
  416. }
  417. REF_RELEASE (cbd);
  418. }
  419. if (item) {
  420. rspamd_symcache_item_async_dec_check (task, item, M);
  421. }
  422. }
  423. static void
  424. lua_http_push_headers (lua_State *L, struct rspamd_http_message *msg)
  425. {
  426. const char *name, *value;
  427. gint i, sz;
  428. lua_pushnil (L);
  429. while (lua_next (L, -2) != 0) {
  430. lua_pushvalue (L, -2);
  431. name = lua_tostring (L, -1);
  432. sz = rspamd_lua_table_size (L, -2);
  433. if (sz != 0 && name != NULL) {
  434. for (i = 1; i <= sz ; i++) {
  435. lua_rawgeti (L, -2, i);
  436. value = lua_tostring (L, -1);
  437. if (value != NULL) {
  438. rspamd_http_message_add_header (msg, name, value);
  439. }
  440. lua_pop (L, 1);
  441. }
  442. } else {
  443. value = lua_tostring (L, -2);
  444. if (name != NULL && value != NULL) {
  445. rspamd_http_message_add_header (msg, name, value);
  446. }
  447. }
  448. lua_pop (L, 2);
  449. }
  450. }
  451. /***
  452. * @function rspamd_http.request({params...})
  453. * This function creates HTTP request and accepts several parameters as a table using key=value syntax.
  454. * Required params are:
  455. *
  456. * - `url`
  457. * - `task`
  458. *
  459. * In taskless mode, instead of `task` required are:
  460. *
  461. * - `ev_base`
  462. * - `config`
  463. *
  464. * @param {string} url specifies URL for a request in the standard URI form (e.g. 'http://example.com/path')
  465. * @param {function} callback specifies callback function in format `function (err_message, code, body, headers)` that is called on HTTP request completion. if this parameter is missing, the function performs "pseudo-synchronous" call (see [Synchronous and Asynchronous API overview](/doc/lua/sync_async.html#API-example-http-module)
  466. * @param {task} task if called from symbol handler it is generally a good idea to use the common task objects: event base, DNS resolver and events session
  467. * @param {table} headers optional headers in form `[name='value', name='value']`
  468. * @param {string} mime_type MIME type of the HTTP content (for example, `text/html`)
  469. * @param {string/text} body full body content, can be opaque `rspamd{text}` to avoid data copying
  470. * @param {number} timeout floating point request timeout value in seconds (default is 5.0 seconds)
  471. * @param {resolver} resolver to perform DNS-requests. Usually got from either `task` or `config`
  472. * @param {boolean} gzip if true, body of the requests will be compressed
  473. * @param {boolean} no_ssl_verify disable SSL peer checks
  474. * @param {boolean} keepalive enable keep-alive pool
  475. * @param {string} user for HTTP authentication
  476. * @param {string} password for HTTP authentication, only if "user" present
  477. * @return {boolean} `true`, in **async** mode, if a request has been successfully scheduled. If this value is `false` then some error occurred, the callback thus will not be called.
  478. * @return In **sync** mode `string|nil, nil|table` In sync mode error message if any and response as table: `int` _code_, `string` _content_ and `table` _headers_ (header -> value)
  479. */
  480. static gint
  481. lua_http_request (lua_State *L)
  482. {
  483. LUA_TRACE_POINT;
  484. struct ev_loop *ev_base;
  485. struct rspamd_http_message *msg;
  486. struct lua_http_cbdata *cbd;
  487. struct rspamd_dns_resolver *resolver;
  488. struct rspamd_async_session *session = NULL;
  489. struct rspamd_lua_text *t;
  490. struct rspamd_task *task = NULL;
  491. struct rspamd_config *cfg = NULL;
  492. struct rspamd_cryptobox_pubkey *peer_key = NULL;
  493. struct rspamd_cryptobox_keypair *local_kp = NULL;
  494. const gchar *url, *lua_body;
  495. rspamd_fstring_t *body = NULL;
  496. gint cbref = -1;
  497. gsize bodylen;
  498. gdouble timeout = default_http_timeout;
  499. gint flags = 0;
  500. gchar *mime_type = NULL;
  501. gchar *auth = NULL;
  502. gsize max_size = 0;
  503. gboolean gzip = FALSE;
  504. if (lua_gettop (L) >= 2) {
  505. /* url, callback and event_base format */
  506. url = luaL_checkstring (L, 1);
  507. if (url == NULL || lua_type (L, 2) != LUA_TFUNCTION) {
  508. msg_err ("http request has bad params");
  509. lua_pushboolean (L, FALSE);
  510. return 1;
  511. }
  512. lua_pushvalue (L, 2);
  513. cbref = luaL_ref (L, LUA_REGISTRYINDEX);
  514. if (lua_gettop (L) >= 3 && rspamd_lua_check_udata_maybe (L, 3, "rspamd{ev_base}")) {
  515. ev_base = *(struct ev_loop **)lua_touserdata (L, 3);
  516. }
  517. else {
  518. ev_base = NULL;
  519. }
  520. if (lua_gettop (L) >= 4 && rspamd_lua_check_udata_maybe (L, 4, "rspamd{resolver}")) {
  521. resolver = *(struct rspamd_dns_resolver **)lua_touserdata (L, 4);
  522. }
  523. else {
  524. resolver = lua_http_global_resolver (ev_base);
  525. }
  526. if (lua_gettop (L) >= 5 && rspamd_lua_check_udata_maybe (L, 5, "rspamd{session}")) {
  527. session = *(struct rspamd_async_session **)lua_touserdata (L, 5);
  528. }
  529. else {
  530. session = NULL;
  531. }
  532. msg = rspamd_http_message_from_url (url);
  533. if (msg == NULL) {
  534. lua_pushboolean (L, FALSE);
  535. return 1;
  536. }
  537. }
  538. else if (lua_type (L, 1) == LUA_TTABLE) {
  539. lua_pushstring (L, "url");
  540. lua_gettable (L, 1);
  541. url = luaL_checkstring (L, -1);
  542. lua_pop (L, 1);
  543. lua_pushstring (L, "callback");
  544. lua_gettable (L, 1);
  545. if (url == NULL || lua_type (L, -1) != LUA_TFUNCTION) {
  546. lua_pop (L, 1);
  547. } else {
  548. cbref = luaL_ref (L, LUA_REGISTRYINDEX);
  549. }
  550. lua_pushstring (L, "task");
  551. lua_gettable (L, 1);
  552. if (lua_type (L, -1) == LUA_TUSERDATA) {
  553. task = lua_check_task (L, -1);
  554. if (task) {
  555. ev_base = task->event_loop;
  556. resolver = task->resolver;
  557. session = task->s;
  558. cfg = task->cfg;
  559. }
  560. }
  561. lua_pop (L, 1);
  562. if (task == NULL) {
  563. lua_pushstring (L, "ev_base");
  564. lua_gettable (L, 1);
  565. if (rspamd_lua_check_udata_maybe (L, -1, "rspamd{ev_base}")) {
  566. ev_base = *(struct ev_loop **)lua_touserdata (L, -1);
  567. }
  568. else {
  569. ev_base = NULL;
  570. }
  571. lua_pop (L, 1);
  572. lua_pushstring (L, "session");
  573. lua_gettable (L, 1);
  574. if (rspamd_lua_check_udata_maybe (L, -1, "rspamd{session}")) {
  575. session = *(struct rspamd_async_session **)lua_touserdata (L, -1);
  576. }
  577. else {
  578. session = NULL;
  579. }
  580. lua_pop (L, 1);
  581. lua_pushstring (L, "config");
  582. lua_gettable (L, 1);
  583. if (rspamd_lua_check_udata_maybe (L, -1, "rspamd{config}")) {
  584. cfg = *(struct rspamd_config **)lua_touserdata (L, -1);
  585. }
  586. else {
  587. cfg = NULL;
  588. }
  589. lua_pop (L, 1);
  590. lua_pushstring (L, "resolver");
  591. lua_gettable (L, 1);
  592. if (rspamd_lua_check_udata_maybe (L, -1, "rspamd{resolver}")) {
  593. resolver = *(struct rspamd_dns_resolver **)lua_touserdata (L, -1);
  594. }
  595. else {
  596. if (cfg && cfg->dns_resolver) {
  597. resolver = cfg->dns_resolver;
  598. }
  599. else {
  600. resolver = lua_http_global_resolver (ev_base);
  601. }
  602. }
  603. lua_pop (L, 1);
  604. }
  605. msg = rspamd_http_message_from_url (url);
  606. if (msg == NULL) {
  607. msg_err ("cannot create HTTP message from url %s", url);
  608. lua_pushboolean (L, FALSE);
  609. return 1;
  610. }
  611. lua_pushstring (L, "headers");
  612. lua_gettable (L, 1);
  613. if (lua_type (L, -1) == LUA_TTABLE) {
  614. lua_http_push_headers (L, msg);
  615. }
  616. lua_pop (L, 1);
  617. lua_pushstring (L, "timeout");
  618. lua_gettable (L, 1);
  619. if (lua_type (L, -1) == LUA_TNUMBER) {
  620. timeout = lua_tonumber (L, -1);
  621. }
  622. lua_pop (L, 1);
  623. lua_pushstring (L, "mime_type");
  624. lua_gettable (L, 1);
  625. if (lua_type (L, -1) == LUA_TSTRING) {
  626. mime_type = g_strdup (lua_tostring (L, -1));
  627. }
  628. lua_pop (L, 1);
  629. lua_pushstring (L, "body");
  630. lua_gettable (L, 1);
  631. if (lua_type (L, -1) == LUA_TSTRING) {
  632. lua_body = lua_tolstring (L, -1, &bodylen);
  633. body = rspamd_fstring_new_init (lua_body, bodylen);
  634. }
  635. else if (lua_type (L, -1) == LUA_TUSERDATA) {
  636. t = lua_check_text (L, -1);
  637. /* TODO: think about zero-copy possibilities */
  638. if (t) {
  639. body = rspamd_fstring_new_init (t->start, t->len);
  640. }
  641. else {
  642. rspamd_http_message_unref (msg);
  643. g_free (mime_type);
  644. return luaL_error (L, "invalid body argument type: %s",
  645. lua_typename (L, lua_type (L, -1)));
  646. }
  647. }
  648. else if (lua_type (L, -1) == LUA_TTABLE) {
  649. gsize total_len = 0, nelts = rspamd_lua_table_size (L, -1);
  650. /* Calculate length and check types */
  651. for (gsize i = 0; i < nelts; i ++) {
  652. lua_rawgeti (L, -1, i + 1);
  653. if (lua_type (L, -1) == LUA_TSTRING) {
  654. #if LUA_VERSION_NUM >= 502
  655. total_len += lua_rawlen (L, -1);
  656. #else
  657. total_len += lua_objlen (L, -1);
  658. #endif
  659. }
  660. else if (lua_type (L, -1) == LUA_TUSERDATA) {
  661. t = lua_check_text (L, -1);
  662. if (t) {
  663. total_len += t->len;
  664. }
  665. else {
  666. rspamd_http_message_unref (msg);
  667. if (mime_type) {
  668. g_free (mime_type);
  669. }
  670. return luaL_error (L, "invalid body argument: %s",
  671. lua_typename (L, lua_type (L, -1)));
  672. }
  673. }
  674. else {
  675. rspamd_http_message_unref (msg);
  676. if (mime_type) {
  677. g_free (mime_type);
  678. }
  679. return luaL_error (L, "invalid body argument type: %s",
  680. lua_typename (L, lua_type (L, -1)));
  681. }
  682. lua_pop (L, 1);
  683. }
  684. /* Preallocate body */
  685. if (total_len > 0) {
  686. body = rspamd_fstring_sized_new (total_len);
  687. }
  688. else {
  689. rspamd_http_message_unref (msg);
  690. if (mime_type) {
  691. g_free (mime_type);
  692. }
  693. return luaL_error (L, "empty body specified");
  694. }
  695. /* Fill elements */
  696. for (gsize i = 0; i < nelts; i ++) {
  697. lua_rawgeti (L, -1, i + 1);
  698. if (lua_type (L, -1) == LUA_TSTRING) {
  699. lua_body = lua_tolstring (L, -1, &bodylen);
  700. body = rspamd_fstring_append (body, lua_body, bodylen);
  701. }
  702. else {
  703. t = lua_check_text (L, -1);
  704. if (t) {
  705. body = rspamd_fstring_append(body, t->start, t->len);
  706. }
  707. }
  708. lua_pop (L, 1);
  709. }
  710. }
  711. else if (lua_type (L, -1) != LUA_TNONE && lua_type (L, -1) != LUA_TNIL) {
  712. rspamd_http_message_unref (msg);
  713. return luaL_error (L, "invalid body argument type: %s",
  714. lua_typename (L, lua_type (L, -1)));
  715. }
  716. lua_pop (L, 1);
  717. lua_pushstring (L, "peer_key");
  718. lua_gettable (L, 1);
  719. if (lua_type (L, -1) == LUA_TSTRING) {
  720. const gchar *in;
  721. gsize inlen;
  722. in = lua_tolstring (L, -1, &inlen);
  723. peer_key = rspamd_pubkey_from_base32 (in, inlen,
  724. RSPAMD_KEYPAIR_KEX, RSPAMD_CRYPTOBOX_MODE_25519);
  725. }
  726. lua_pop (L, 1);
  727. lua_pushstring (L, "keypair");
  728. lua_gettable (L, 1);
  729. if (lua_type (L, -1) == LUA_TTABLE) {
  730. ucl_object_t *kp_ucl = ucl_object_lua_import (L, -1);
  731. local_kp = rspamd_keypair_from_ucl (kp_ucl);
  732. ucl_object_unref (kp_ucl);
  733. }
  734. lua_pop (L, 1);
  735. lua_pushstring (L, "opaque_body");
  736. lua_gettable (L, 1);
  737. if (!!lua_toboolean (L, -1)) {
  738. flags |= RSPAMD_LUA_HTTP_FLAG_TEXT;
  739. }
  740. lua_pop (L, 1);
  741. lua_pushstring (L, "gzip");
  742. lua_gettable (L, 1);
  743. if (!!lua_toboolean (L, -1)) {
  744. gzip = TRUE;
  745. }
  746. lua_pop (L, 1);
  747. lua_pushstring (L, "no_ssl_verify");
  748. lua_gettable (L, 1);
  749. if (!!lua_toboolean (L, -1)) {
  750. flags |= RSPAMD_LUA_HTTP_FLAG_NOVERIFY;
  751. }
  752. lua_pop (L, 1);
  753. lua_pushstring (L, "keepalive");
  754. lua_gettable (L, 1);
  755. if (!!lua_toboolean (L, -1)) {
  756. flags |= RSPAMD_LUA_HTTP_FLAG_KEEP_ALIVE;
  757. }
  758. lua_pop (L, 1);
  759. lua_pushstring (L, "max_size");
  760. lua_gettable (L, 1);
  761. if (lua_type (L, -1) == LUA_TNUMBER) {
  762. max_size = lua_tointeger (L, -1);
  763. }
  764. lua_pop (L, 1);
  765. lua_pushstring (L, "method");
  766. lua_gettable (L, 1);
  767. if (lua_type (L, -1) == LUA_TSTRING) {
  768. rspamd_http_message_set_method (msg, lua_tostring (L, -1));
  769. }
  770. lua_pop (L, 1);
  771. lua_pushstring (L, "user");
  772. lua_gettable (L, 1);
  773. if (lua_type (L, -1) == LUA_TSTRING) {
  774. const gchar *user = lua_tostring (L, -1);
  775. lua_pushstring (L, "password");
  776. lua_gettable (L, 1);
  777. if (lua_type (L, -1) == LUA_TSTRING) {
  778. const gchar *password = lua_tostring (L, -1);
  779. gchar *tmpbuf;
  780. gsize tlen;
  781. tlen = strlen (user) + strlen (password) + 1;
  782. tmpbuf = g_malloc (tlen + 1);
  783. rspamd_snprintf (tmpbuf, tlen + 1, "%s:%s", user, password);
  784. tlen *= 2;
  785. tlen += sizeof ("Basic ") - 1;
  786. auth = g_malloc (tlen + 1);
  787. rspamd_snprintf (auth, tlen + 1, "Basic %Bs", tmpbuf);
  788. g_free (tmpbuf);
  789. }
  790. else {
  791. msg_warn ("HTTP user must have password, disabling auth");
  792. }
  793. lua_pop (L, 1); /* password */
  794. }
  795. lua_pop (L, 1); /* username */
  796. }
  797. else {
  798. msg_err ("http request has bad params");
  799. lua_pushboolean (L, FALSE);
  800. return 1;
  801. }
  802. if (session && rspamd_session_blocked (session)) {
  803. lua_pushboolean (L, FALSE);
  804. g_free (auth);
  805. rspamd_http_message_unref (msg);
  806. if (body) {
  807. rspamd_fstring_free (body);
  808. }
  809. if (local_kp) {
  810. rspamd_keypair_unref (local_kp);
  811. }
  812. return 1;
  813. }
  814. if (task == NULL && cfg == NULL) {
  815. g_free (auth);
  816. rspamd_http_message_unref (msg);
  817. if (body) {
  818. rspamd_fstring_free (body);
  819. }
  820. if (local_kp) {
  821. rspamd_keypair_unref (local_kp);
  822. }
  823. return luaL_error (L,
  824. "Bad params to rspamd_http:request(): either task or config should be set");
  825. }
  826. if (ev_base == NULL) {
  827. g_free (auth);
  828. rspamd_http_message_unref (msg);
  829. if (body) {
  830. rspamd_fstring_free (body);
  831. }
  832. if (local_kp) {
  833. rspamd_keypair_unref (local_kp);
  834. }
  835. return luaL_error (L,
  836. "Bad params to rspamd_http:request(): ev_base isn't passed");
  837. }
  838. cbd = g_malloc0 (sizeof (*cbd));
  839. cbd->cbref = cbref;
  840. cbd->msg = msg;
  841. cbd->event_loop = ev_base;
  842. cbd->mime_type = mime_type;
  843. cbd->timeout = timeout;
  844. cbd->fd = -1;
  845. cbd->cfg = cfg;
  846. cbd->peer_pk = peer_key;
  847. cbd->local_kp = local_kp;
  848. cbd->flags = flags;
  849. cbd->max_size = max_size;
  850. cbd->url = url;
  851. cbd->auth = auth;
  852. cbd->task = task;
  853. if (cbd->cbref == -1) {
  854. cbd->thread = lua_thread_pool_get_running_entry (cfg->lua_thread_pool);
  855. }
  856. REF_INIT_RETAIN (cbd, lua_http_cbd_dtor);
  857. if (task) {
  858. cbd->item = rspamd_symcache_get_cur_item (task);
  859. }
  860. const rspamd_ftok_t *host_header_tok = rspamd_http_message_find_header (msg, "Host");
  861. if (host_header_tok != NULL) {
  862. if (msg->host) {
  863. g_string_free (msg->host, true);
  864. }
  865. msg->host = g_string_new_len (host_header_tok->begin, host_header_tok->len);
  866. cbd->host = msg->host->str;
  867. }
  868. else {
  869. if (msg->host) {
  870. cbd->host = msg->host->str;
  871. }
  872. }
  873. if (body) {
  874. if (gzip) {
  875. if (rspamd_fstring_gzip (&body)) {
  876. rspamd_http_message_add_header (msg, "Content-Encoding", "gzip");
  877. }
  878. }
  879. rspamd_http_message_set_body_from_fstring_steal (msg, body);
  880. }
  881. if (session) {
  882. cbd->session = session;
  883. }
  884. if (msg->host && rspamd_parse_inet_address (&cbd->addr,
  885. msg->host->str, msg->host->len, RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
  886. /* Host is numeric IP, no need to resolve */
  887. gboolean ret;
  888. REF_RETAIN (cbd);
  889. ret = lua_http_make_connection (cbd);
  890. if (!ret) {
  891. if (cbd->ref.refcount > 1) {
  892. /* Not released by make_connection */
  893. REF_RELEASE (cbd);
  894. }
  895. REF_RELEASE (cbd);
  896. lua_pushboolean (L, FALSE);
  897. return 1;
  898. }
  899. REF_RELEASE (cbd);
  900. }
  901. else {
  902. if (!cbd->host) {
  903. REF_RELEASE (cbd);
  904. return luaL_error (L, "no host has been specified");
  905. }
  906. if (task == NULL) {
  907. REF_RETAIN (cbd);
  908. if (!rspamd_dns_resolver_request (resolver, session, NULL, lua_http_dns_handler, cbd,
  909. RDNS_REQUEST_A,
  910. cbd->host)) {
  911. if (cbd->ref.refcount > 1) {
  912. /* Not released by make_connection */
  913. REF_RELEASE (cbd);
  914. }
  915. REF_RELEASE (cbd);
  916. lua_pushboolean (L, FALSE);
  917. return 1;
  918. }
  919. REF_RELEASE (cbd);
  920. }
  921. else {
  922. REF_RETAIN (cbd);
  923. if (!rspamd_dns_resolver_request_task_forced (task, lua_http_dns_handler, cbd,
  924. RDNS_REQUEST_A, cbd->host)) {
  925. if (cbd->ref.refcount > 1) {
  926. /* Not released by make_connection */
  927. REF_RELEASE (cbd);
  928. }
  929. REF_RELEASE (cbd);
  930. lua_pushboolean (L, FALSE);
  931. return 1;
  932. }
  933. else if (cbd->item) {
  934. rspamd_symcache_item_async_inc (cbd->task, cbd->item, M);
  935. }
  936. REF_RELEASE (cbd);
  937. }
  938. }
  939. if (cbd->cbref == -1) {
  940. cbd->thread = lua_thread_pool_get_running_entry (cfg->lua_thread_pool);
  941. cbd->flags |= RSPAMD_LUA_HTTP_FLAG_YIELDED;
  942. return lua_thread_yield (cbd->thread, 0);
  943. }
  944. else {
  945. lua_pushboolean (L, TRUE);
  946. }
  947. return 1;
  948. }
  949. static gint
  950. lua_load_http (lua_State * L)
  951. {
  952. lua_newtable (L);
  953. luaL_register (L, NULL, httplib_m);
  954. return 1;
  955. }
  956. void
  957. luaopen_http (lua_State * L)
  958. {
  959. rspamd_lua_add_preload (L, "rspamd_http", lua_load_http);
  960. }