rspamd/src/libmime/content_type.c

/*-
 * Copyright 2016 Vsevolod Stakhov
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "libmime/content_type.h"
#include "smtp_parsers.h"
#include "utlist.h"
#include "libserver/url.h"
#include "libmime/mime_encoding.h"

static gboolean
rspamd_rfc2231_decode (rspamd_mempool_t *pool,
		struct rspamd_content_type_param *param,
		gchar *value_start, gchar *value_end)
{
	gchar *quote_pos;

	quote_pos = memchr (value_start, '\'', value_end - value_start);

	if (quote_pos == NULL) {
		/* Plain percent encoding */
		gsize r = rspamd_url_decode (value_start, value_start,
				value_end - value_start);
		param->value.begin = value_start;
		param->value.len = r;
	}
	else {
		/*
		 * We can have encoding'language'data, or
		 * encoding'data (in theory).
		 * Try to handle both...
		 */
		const gchar *charset = NULL;
		rspamd_ftok_t ctok;

		ctok.begin = value_start;
		ctok.len = quote_pos - value_start;

		if (ctok.len > 0) {
			charset = rspamd_mime_detect_charset (&ctok, pool);
		}

		/* Now, we can check for either next quote sign or, eh, ignore that */
		value_start = quote_pos + 1;

		quote_pos = memchr (value_start, '\'', value_end - value_start);

		if (quote_pos) {
			/* Ignore language */
			value_start = quote_pos + 1;
		}

		/* Perform percent decoding */
		gsize r = rspamd_url_decode (value_start, value_start,
				value_end - value_start);
		GError *err = NULL;

		if (charset == NULL) {
			/* Try heuristic */
			charset = rspamd_mime_charset_find_by_content (value_start, r);
		}

		if (charset == NULL) {
			msg_warn_pool ("cannot convert parameter from charset %T", &ctok);

			return FALSE;
		}

		param->value.begin = rspamd_mime_text_to_utf8 (pool,
				value_start, r,
				charset, &param->value.len, &err);

		if (param->value.begin == NULL) {
			msg_warn_pool ("cannot convert parameter from charset %s: %e",
					charset, err);

			if (err) {
				g_error_free (err);
			}

			return FALSE;
		}
	}

	param->flags |= RSPAMD_CONTENT_PARAM_RFC2231;

	return TRUE;
}

static gboolean
rspamd_param_maybe_rfc2231_process (rspamd_mempool_t *pool,
									struct rspamd_content_type_param *param,
									gchar *name_start, gchar *name_end,
									gchar *value_start, gchar *value_end)
{
	const gchar *star_pos;

	star_pos = memchr (name_start, '*', name_end - name_start);

	if (star_pos == NULL) {
		return FALSE;
	}

	/* We have three possibilities here:
	 * 1. name* (just name + 2231 encoding)
	 * 2. name*(\d+) (piecewise stuff but no rfc2231 encoding)
	 * 3. name*(\d+)* (piecewise stuff and rfc2231 encoding)
	 */

	if (star_pos == name_end - 1) {
		/* First */
		if (rspamd_rfc2231_decode (pool, param, value_start, value_end)) {
			param->name.begin = name_start;
			param->name.len = name_end - name_start - 1;
		}
	}
	else if (*(name_end - 1) == '*') {
		/* Third */
		/* Check number */
		gulong tmp;

		if (!rspamd_strtoul (star_pos + 1, name_end - star_pos - 2, &tmp)) {
			return FALSE;
		}

		param->flags |= RSPAMD_CONTENT_PARAM_PIECEWISE|RSPAMD_CONTENT_PARAM_RFC2231;
		param->rfc2231_id = tmp;
		param->name.begin = name_start;
		param->name.len = star_pos - name_start;
		param->value.begin = value_start;
		param->value.len = value_end - value_start;

		/* Deal with that later... */
	}
	else {
		/* Second case */
		gulong tmp;

		if (!rspamd_strtoul (star_pos + 1, name_end - star_pos - 1, &tmp)) {
			return FALSE;
		}

		param->flags |= RSPAMD_CONTENT_PARAM_PIECEWISE;
		param->rfc2231_id = tmp;
		param->name.begin = name_start;
		param->name.len = star_pos - name_start;
		param->value.begin = value_start;
		param->value.len = value_end - value_start;
	}

	return TRUE;
}

static gint32
rspamd_cmp_pieces (struct rspamd_content_type_param *p1, struct rspamd_content_type_param *p2)
{
	return p1->rfc2231_id - p2->rfc2231_id;
}

static void
rspamd_postprocess_ct_attributes (rspamd_mempool_t *pool,
								  GHashTable *htb,
								  void (*proc)(rspamd_mempool_t *, struct rspamd_content_type_param *, gpointer ud),
								  gpointer procd)
{
	GHashTableIter it;
	gpointer k, v;
	struct rspamd_content_type_param *param, *sorted, *cur;

	if (htb == NULL) {
		return;
	}

	g_hash_table_iter_init (&it, htb);

	while (g_hash_table_iter_next (&it, &k, &v)) {
		param = (struct rspamd_content_type_param *)v;

		if (param->flags & RSPAMD_CONTENT_PARAM_PIECEWISE) {
			/* Reconstruct param */
			gsize tlen = 0;
			gchar *ndata, *pos;

			sorted = param;
			DL_SORT (sorted, rspamd_cmp_pieces);

			DL_FOREACH (sorted, cur) {
				tlen += cur->value.len;
			}

			ndata = rspamd_mempool_alloc (pool, tlen);
			pos = ndata;

			DL_FOREACH (sorted, cur) {
				memcpy (pos, cur->value.begin, cur->value.len);
				pos += cur->value.len;
			}

			if (param->flags & RSPAMD_CONTENT_PARAM_RFC2231) {
				if (!rspamd_rfc2231_decode (pool, param,
						ndata, pos)) {
					param->flags |= RSPAMD_CONTENT_PARAM_BROKEN;
					param->value.begin = ndata;
					param->value.len = tlen;
				}
			}
			else {
				param->value.begin = ndata;
				param->value.len = tlen;
			}

			/* Detach from list */
			param->next = NULL;
			param->prev = param;
		}

		proc (pool, param, procd);
	}
}

static void
rspamd_content_type_postprocess (rspamd_mempool_t *pool,
								 struct rspamd_content_type_param *param,
								 gpointer ud)
{
	rspamd_ftok_t srch;
	struct rspamd_content_type_param *found = NULL;

	struct rspamd_content_type *ct = (struct rspamd_content_type *)ud;

	RSPAMD_FTOK_ASSIGN (&srch, "charset");

	if (rspamd_ftok_casecmp (&param->name, &srch) == 0) {
		/* Adjust charset */
		found = param;
		ct->charset.begin = param->value.begin;
		ct->charset.len = param->value.len;
	}

	RSPAMD_FTOK_ASSIGN (&srch, "boundary");

	if (rspamd_ftok_casecmp (&param->name, &srch) == 0) {
		found = param;
		gchar *lc_boundary;
		/* Adjust boundary */
		lc_boundary = rspamd_mempool_alloc (pool, param->value.len);
		memcpy (lc_boundary, param->value.begin, param->value.len);
		rspamd_str_lc (lc_boundary, param->value.len);
		ct->boundary.begin = lc_boundary;
		ct->boundary.len = param->value.len;
		/* Preserve original (case sensitive) boundary */
		ct->orig_boundary.begin = param->value.begin;
		ct->orig_boundary.len = param->value.len;
	}

	if (!found) {
		/* Just lowercase */
		rspamd_str_lc ((gchar *)param->value.begin, param->value.len);
	}
}

static void
rspamd_content_disposition_postprocess (rspamd_mempool_t *pool,
										struct rspamd_content_type_param *param,
										gpointer ud)
{
	rspamd_ftok_t srch;
	struct rspamd_content_disposition *cd = (struct rspamd_content_disposition *)ud;

	srch.begin = "filename";
	srch.len = 8;

	if (rspamd_ftok_casecmp (&param->name, &srch) == 0) {
		/* Adjust filename */
		cd->filename.begin = param->value.begin;
		cd->filename.len = param->value.len;
	}
}

void
rspamd_content_type_add_param (rspamd_mempool_t *pool,
		struct rspamd_content_type *ct,
		gchar *name_start, gchar *name_end,
		gchar *value_start, gchar *value_end)
{
	struct rspamd_content_type_param *nparam;
	rspamd_ftok_t srch;
	struct rspamd_content_type_param *found = NULL;

	g_assert (ct != NULL);

	nparam = rspamd_mempool_alloc0 (pool, sizeof (*nparam));
	rspamd_str_lc (name_start, name_end - name_start);

	if (!rspamd_param_maybe_rfc2231_process (pool, nparam, name_start,
			name_end, value_start, value_end)) {
		nparam->name.begin = name_start;
		nparam->name.len = name_end - name_start;
		nparam->value.begin = value_start;
		nparam->value.len = value_end - value_start;
	}

	srch.begin = nparam->name.begin;
	srch.len = nparam->name.len;

	if (ct->attrs) {
		found = g_hash_table_lookup (ct->attrs, &srch);
	} else {
		ct->attrs = g_hash_table_new (rspamd_ftok_icase_hash,
				rspamd_ftok_icase_equal);
	}

	if (!found) {
		DL_APPEND (found, nparam);
		g_hash_table_insert (ct->attrs, &nparam->name, nparam);
	}
	else {
		DL_APPEND (found, nparam);
	}
}

static struct rspamd_content_type *
rspamd_content_type_parser (gchar *in, gsize len, rspamd_mempool_t *pool)
{
	guint obraces = 0, ebraces = 0, qlen = 0;
	gchar *p, *c, *end, *pname_start = NULL, *pname_end = NULL;
	struct rspamd_content_type *res = NULL, val;
	gboolean eqsign_seen = FALSE;
	enum {
		parse_type,
		parse_subtype,
		parse_after_subtype,
		parse_param_name,
		parse_param_after_name,
		parse_param_value,
		parse_param_value_after_quote,
		parse_space,
		parse_quoted,
		parse_comment,
	} state = parse_space, next_state = parse_type;

	p = in;
	c = p;
	end = p + len;
	memset (&val, 0, sizeof (val));
	val.cpy = in;

	while (p < end) {
		switch (state) {
		case parse_type:
			if (g_ascii_isspace (*p) || *p == ';') {
				/* We have type without subtype */
				val.type.begin = c;
				val.type.len = p - c;
				state = parse_after_subtype;
			} else if (*p == '/') {
				val.type.begin = c;
				val.type.len = p - c;
				state = parse_space;
				next_state = parse_subtype;
				p++;
			} else {
				p++;
			}
			break;
		case parse_subtype:
			if (g_ascii_isspace (*p) || *p == ';') {
				val.subtype.begin = c;
				val.subtype.len = p - c;
				state = parse_after_subtype;
			} else {
				p++;
			}
			break;
		case parse_after_subtype:
			if (*p == ';' || g_ascii_isspace (*p)) {
				p++;
			} else if (*p == '(') {
				c = p;
				state = parse_comment;
				next_state = parse_param_name;
				obraces = 1;
				ebraces = 0;
				pname_start = NULL;
				pname_end = NULL;
				eqsign_seen = FALSE;
				p++;
			} else {
				c = p;
				state = parse_param_name;
				pname_start = NULL;
				pname_end = NULL;
				eqsign_seen = FALSE;
			}
			break;
		case parse_param_name:
			if (*p == '=') {
				pname_start = c;
				pname_end = p;
				state = parse_param_after_name;
				eqsign_seen = TRUE;
				p++;
			} else if (g_ascii_isspace (*p)) {
				pname_start = c;
				pname_end = p;
				state = parse_param_after_name;
			} else {
				p++;
			}
			break;
		case parse_param_after_name:
			if (g_ascii_isspace (*p)) {
				p++;
			} else if (*p == '=') {
				if (eqsign_seen) {
					/* Treat as value start */
					c = p;
					eqsign_seen = FALSE;
					state = parse_space;
					next_state = parse_param_value;
					p++;
				} else {
					eqsign_seen = TRUE;
					p++;
				}
			} else {
				if (eqsign_seen) {
					state = parse_param_value;
					c = p;
				} else {
					/* Invalid parameter without value */
					c = p;
					state = parse_param_name;
					pname_start = NULL;
					pname_end = NULL;
				}
			}
			break;
		case parse_param_value:
			if (*p == '"') {
				p++;
				c = p;
				state = parse_quoted;
				next_state = parse_param_value_after_quote;
			} else if (g_ascii_isspace (*p)) {
				if (pname_start && pname_end && pname_end > pname_start) {
					rspamd_content_type_add_param (pool, &val, pname_start,
							pname_end, c, p);

				}

				state = parse_space;
				next_state = parse_param_name;
				pname_start = NULL;
				pname_end = NULL;
			} else if (*p == '(') {
				if (pname_start && pname_end && pname_end > pname_start) {
					rspamd_content_type_add_param (pool, &val, pname_start,
							pname_end, c, p);
				}

				obraces = 1;
				ebraces = 0;
				p++;
				state = parse_comment;
				next_state = parse_param_name;
				pname_start = NULL;
				pname_end = NULL;
			}
			else if (*p == ';') {
				if (pname_start && pname_end && pname_end > pname_start) {
					rspamd_content_type_add_param (pool, &val, pname_start,
							pname_end, c, p);
				}

				p ++;
				state = parse_space;
				next_state = parse_param_name;
				pname_start = NULL;
				pname_end = NULL;
			}
			else {
				p++;
			}
			break;
		case parse_param_value_after_quote:
			if (pname_start && pname_end && pname_end > pname_start) {
				rspamd_content_type_add_param (pool, &val, pname_start,
						pname_end, c, c + qlen);
			}

			if (*p == '"') {
				p ++;

				if (p == end) {
					/* Last quote: done... */
					break;
				}

				if (*p == ';') {
					p ++;
					state = parse_space;
					next_state = parse_param_name;
					pname_start = NULL;
					pname_end = NULL;
					continue;
				}
			}

			/* We should not normally be here in fact */
			if (g_ascii_isspace (*p)) {
				state = parse_space;
				next_state = parse_param_name;
				pname_start = NULL;
				pname_end = NULL;
			} else if (*p == '(') {
				obraces = 1;
				ebraces = 0;
				p++;
				state = parse_comment;
				next_state = parse_param_name;
				pname_start = NULL;
				pname_end = NULL;
			} else {
				state = parse_param_name;
				pname_start = NULL;
				pname_end = NULL;
				c = p;
			}
			break;
		case parse_quoted:
			if (*p == '\\') {
				/* Quoted pair */
				if (p + 1 < end) {
					p += 2;
				} else {
					p++;
				}
			} else if (*p == '"') {
				qlen = p - c;
				state = next_state;
			} else {
				p++;
			}
			break;
		case parse_comment:
			if (*p == '(') {
				obraces++;
				p++;
			} else if (*p == ')') {
				ebraces++;
				p++;

				if (ebraces == obraces && p < end) {
					if (g_ascii_isspace (*p)) {
						state = parse_space;
					} else {
						c = p;
						state = next_state;
					}
				}
			} else {
				p++;
			}
			break;
		case parse_space:
			if (g_ascii_isspace (*p)) {
				p++;
			} else if (*p == '(') {
				obraces = 1;
				ebraces = 0;
				p++;
				state = parse_comment;
			} else {
				c = p;
				state = next_state;
			}
			break;
		}
	}

	/* Process leftover */
	switch (state) {
	case parse_type:
		val.type.begin = c;
		val.type.len = p - c;
		break;
	case parse_subtype:
		val.subtype.begin = c;
		val.subtype.len = p - c;
		break;
	case parse_param_value:
		if (pname_start && pname_end && pname_end > pname_start) {
			if (p > c && *(p - 1) == ';') {
				p --;
			}

			rspamd_content_type_add_param (pool, &val, pname_start,
					pname_end, c, p);

		}
		break;
	case parse_param_value_after_quote:
		if (pname_start && pname_end && pname_end > pname_start) {
			rspamd_content_type_add_param (pool, &val, pname_start,
					pname_end, c, c + qlen);
		}
		break;
	default:
		break;
	}

	if (val.type.len > 0) {
		gchar *tmp;

		res = rspamd_mempool_alloc (pool, sizeof (val));
		memcpy (res, &val, sizeof (val));

		/*
		 * Lowercase type and subtype as they are specified as case insensitive
		 * in rfc2045 section 5.1
		 */
		tmp = rspamd_mempool_alloc (pool, val.type.len);
		memcpy (tmp, val.type.begin, val.type.len);
		rspamd_str_lc (tmp, val.type.len);
		res->type.begin = tmp;

		if (val.subtype.len > 0) {
			tmp = rspamd_mempool_alloc (pool, val.subtype.len);
			memcpy (tmp, val.subtype.begin, val.subtype.len);
			rspamd_str_lc (tmp, val.subtype.len);
			res->subtype.begin = tmp;
		}
	}

	return res;
}

struct rspamd_content_type *
rspamd_content_type_parse (const gchar *in,
		gsize len, rspamd_mempool_t *pool)
{
	struct rspamd_content_type *res = NULL;
	rspamd_ftok_t srch;
	gchar *cpy;

	cpy = rspamd_mempool_alloc (pool, len + 1);
	rspamd_strlcpy (cpy, in, len + 1);

	if ((res = rspamd_content_type_parser (cpy, len, pool)) != NULL) {
		if (res->attrs) {
			rspamd_mempool_add_destructor (pool,
					(rspamd_mempool_destruct_t)g_hash_table_unref, res->attrs);

			rspamd_postprocess_ct_attributes (pool, res->attrs,
					rspamd_content_type_postprocess, res);
		}

		/* Now do some hacks to work with broken content types */
		if (res->subtype.len == 0) {
			res->flags |= RSPAMD_CONTENT_TYPE_BROKEN;
			RSPAMD_FTOK_ASSIGN (&srch, "text");

			if (rspamd_ftok_casecmp (&res->type, &srch) == 0) {
				/* Workaround for Content-Type: text */
				/* Assume text/plain */
				RSPAMD_FTOK_ASSIGN (&srch, "plain");
			}
			else {
				RSPAMD_FTOK_ASSIGN (&srch, "html");

				if (rspamd_ftok_casecmp (&res->type, &srch) == 0) {
					/* Workaround for Content-Type: html */
					RSPAMD_FTOK_ASSIGN (&res->type, "text");
					RSPAMD_FTOK_ASSIGN (&res->subtype, "html");
				}
				else {
					RSPAMD_FTOK_ASSIGN (&srch, "application");

					if (rspamd_ftok_casecmp (&res->type, &srch) == 0) {
						RSPAMD_FTOK_ASSIGN (&res->subtype, "octet-stream");
					}
				}
			}
		}
		else {
			/* Common mistake done by retards */
			RSPAMD_FTOK_ASSIGN (&srch, "alternate");

			if (rspamd_ftok_casecmp (&res->subtype, &srch) == 0) {
				res->flags |= RSPAMD_CONTENT_TYPE_BROKEN;
				RSPAMD_FTOK_ASSIGN (&res->subtype, "alternative");
			}
		}

		RSPAMD_FTOK_ASSIGN (&srch, "multipart");

		if (rspamd_ftok_casecmp (&res->type, &srch) == 0) {
			res->flags |= RSPAMD_CONTENT_TYPE_MULTIPART;

			RSPAMD_FTOK_ASSIGN (&srch, "encrypted");
			if (rspamd_ftok_casecmp (&res->subtype, &srch) == 0) {
				res->flags |= RSPAMD_CONTENT_TYPE_ENCRYPTED;
			}
		}
		else {
			RSPAMD_FTOK_ASSIGN (&srch, "text");

			if (rspamd_ftok_casecmp (&res->type, &srch) == 0) {
				res->flags |= RSPAMD_CONTENT_TYPE_TEXT;
			}
			else {
				RSPAMD_FTOK_ASSIGN (&srch, "message");

				if (rspamd_ftok_casecmp (&res->type, &srch) == 0) {
					RSPAMD_FTOK_ASSIGN (&srch, "delivery-status");

					if (rspamd_ftok_casecmp (&res->subtype, &srch) == 0) {
						res->flags |= RSPAMD_CONTENT_TYPE_TEXT|RSPAMD_CONTENT_TYPE_DSN;
					}
					else {
						RSPAMD_FTOK_ASSIGN (&srch, "notification");

						if (rspamd_substring_search_caseless (res->subtype.begin,
								res->subtype.len, srch.begin, srch.len) != -1) {
							res->flags |= RSPAMD_CONTENT_TYPE_TEXT|
									RSPAMD_CONTENT_TYPE_DSN;
						}
						else {
							res->flags |= RSPAMD_CONTENT_TYPE_MESSAGE;
						}
					}
				}
			}
		}
	}
	else {
		msg_warn_pool ("cannot parse content type: %*s", (gint)len, cpy);
	}

	return res;
}

void
rspamd_content_disposition_add_param (rspamd_mempool_t *pool,
		struct rspamd_content_disposition *cd,
		const gchar *name_start, const gchar *name_end,
		const gchar *value_start, const gchar *value_end)
{
	rspamd_ftok_t srch;
	gchar *name_cpy, *value_cpy, *name_cpy_end, *value_cpy_end;
	struct rspamd_content_type_param *found = NULL, *nparam;

	g_assert (cd != NULL);

	name_cpy = rspamd_mempool_alloc (pool, name_end - name_start);
	memcpy (name_cpy, name_start, name_end - name_start);
	name_cpy_end = name_cpy + (name_end - name_start);

	value_cpy = rspamd_mempool_alloc (pool, value_end - value_start);
	memcpy (value_cpy, value_start, value_end - value_start);
	value_cpy_end = value_cpy + (value_end - value_start);

	nparam = rspamd_mempool_alloc0 (pool, sizeof (*nparam));
	rspamd_str_lc (name_cpy, name_cpy_end - name_cpy);

	if (!rspamd_param_maybe_rfc2231_process (pool, nparam, name_cpy,
			name_cpy_end, value_cpy, value_cpy_end)) {
		nparam->name.begin = name_cpy;
		nparam->name.len = name_cpy_end - name_cpy;
		nparam->value.begin = value_cpy;
		nparam->value.len = value_cpy_end - value_cpy;
	}

	srch.begin = nparam->name.begin;
	srch.len = nparam->name.len;

	if (cd->attrs) {
		found = g_hash_table_lookup (cd->attrs, &srch);
	} else {
		cd->attrs = g_hash_table_new (rspamd_ftok_icase_hash,
				rspamd_ftok_icase_equal);
	}

	if (!found) {
		DL_APPEND (found, nparam);
		g_hash_table_insert (cd->attrs, &nparam->name, nparam);
	}
	else {
		DL_APPEND (found, nparam);
	}
}

struct rspamd_content_disposition *
rspamd_content_disposition_parse (const gchar *in,
		gsize len, rspamd_mempool_t *pool)
{
	struct rspamd_content_disposition *res = NULL, val;

	if (rspamd_content_disposition_parser (in, len, &val, pool)) {
		res = rspamd_mempool_alloc (pool, sizeof (val));
		memcpy (res, &val, sizeof (val));
		res->lc_data = rspamd_mempool_alloc (pool, len + 1);
		rspamd_strlcpy (res->lc_data, in, len + 1);
		rspamd_str_lc (res->lc_data, len);

		if (res->attrs) {
			rspamd_postprocess_ct_attributes (pool, res->attrs,
					rspamd_content_disposition_postprocess, res);
			rspamd_mempool_add_destructor (pool,
					(rspamd_mempool_destruct_t)g_hash_table_unref, res->attrs);
		}
	}
	else {
		msg_warn_pool ("cannot parse content disposition: %*s",
				(gint)len, in);
	}

	return res;
}