1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
|
/**
* @file message.h
* Message processing functions and structures
*/
#ifndef RSPAMD_MESSAGE_H
#define RSPAMD_MESSAGE_H
#include "config.h"
#include "libmime/email_addr.h"
#include "libutil/addr.h"
#include "libcryptobox/cryptobox.h"
#include "libmime/mime_headers.h"
#include "libmime/content_type.h"
#include "libserver/url.h"
#include "libutil/ref.h"
#include "libutil/str_util.h"
#include <unicode/uchar.h>
#include <unicode/utext.h>
#ifdef __cplusplus
extern "C" {
#endif
struct rspamd_task;
struct controller_session;
struct html_content;
struct rspamd_image;
struct rspamd_archive;
enum rspamd_mime_part_flags {
RSPAMD_MIME_PART_ATTACHEMENT = (1 << 1),
RSPAMD_MIME_PART_BAD_CTE = (1 << 4),
RSPAMD_MIME_PART_MISSING_CTE = (1 << 5),
};
enum rspamd_mime_part_type {
RSPAMD_MIME_PART_UNDEFINED = 0,
RSPAMD_MIME_PART_MULTIPART,
RSPAMD_MIME_PART_MESSAGE,
RSPAMD_MIME_PART_TEXT,
RSPAMD_MIME_PART_ARCHIVE,
RSPAMD_MIME_PART_IMAGE,
RSPAMD_MIME_PART_CUSTOM_LUA
};
#define IS_PART_MULTIPART(part) ((part) && ((part)->part_type == RSPAMD_MIME_PART_MULTIPART))
#define IS_PART_TEXT(part) ((part) && ((part)->part_type == RSPAMD_MIME_PART_TEXT))
#define IS_PART_MESSAGE(part) ((part) &&((part)->part_type == RSPAMD_MIME_PART_MESSAGE))
enum rspamd_cte {
RSPAMD_CTE_UNKNOWN = 0,
RSPAMD_CTE_7BIT = 1,
RSPAMD_CTE_8BIT = 2,
RSPAMD_CTE_QP = 3,
RSPAMD_CTE_B64 = 4,
RSPAMD_CTE_UUE = 5,
};
struct rspamd_mime_text_part;
struct rspamd_mime_multipart {
GPtrArray *children;
rspamd_ftok_t boundary;
};
enum rspamd_lua_specific_type {
RSPAMD_LUA_PART_TEXT,
RSPAMD_LUA_PART_STRING,
RSPAMD_LUA_PART_TABLE,
RSPAMD_LUA_PART_FUNCTION,
RSPAMD_LUA_PART_UNKNOWN,
};
struct rspamd_lua_specific_part {
gint cbref;
enum rspamd_lua_specific_type type;
};
struct rspamd_mime_part {
struct rspamd_content_type *ct;
struct rspamd_content_type *detected_ct;
gchar *detected_type;
gchar *detected_ext;
struct rspamd_content_disposition *cd;
rspamd_ftok_t raw_data;
rspamd_ftok_t parsed_data;
struct rspamd_mime_part *parent_part;
struct rspamd_mime_header *headers_order;
struct rspamd_mime_headers_table *raw_headers;
GPtrArray *urls;
gchar *raw_headers_str;
gsize raw_headers_len;
enum rspamd_cte cte;
guint flags;
enum rspamd_mime_part_type part_type;
guint part_number;
union {
struct rspamd_mime_multipart *mp;
struct rspamd_mime_text_part *txt;
struct rspamd_image *img;
struct rspamd_archive *arch;
struct rspamd_lua_specific_part lua_specific;
} specific;
guchar digest[rspamd_cryptobox_HASHBYTES];
};
#define RSPAMD_MIME_TEXT_PART_FLAG_UTF (1 << 0)
#define RSPAMD_MIME_TEXT_PART_FLAG_BALANCED (1 << 1)
#define RSPAMD_MIME_TEXT_PART_FLAG_EMPTY (1 << 2)
#define RSPAMD_MIME_TEXT_PART_FLAG_HTML (1 << 3)
#define RSPAMD_MIME_TEXT_PART_FLAG_8BIT_RAW (1 << 4)
#define RSPAMD_MIME_TEXT_PART_FLAG_8BIT_ENCODED (1 << 5)
#define RSPAMD_MIME_TEXT_PART_HAS_SUBNORMAL (1 << 6)
#define RSPAMD_MIME_TEXT_PART_NORMALISED (1 << 7)
#define RSPAMD_MIME_TEXT_PART_ATTACHMENT (1 << 8)
#define IS_TEXT_PART_EMPTY(part) ((part)->flags & RSPAMD_MIME_TEXT_PART_FLAG_EMPTY)
#define IS_TEXT_PART_UTF(part) ((part)->flags & RSPAMD_MIME_TEXT_PART_FLAG_UTF)
#define IS_TEXT_PART_HTML(part) ((part)->flags & RSPAMD_MIME_TEXT_PART_FLAG_HTML)
#define IS_TEXT_PART_ATTACHMENT(part) ((part)->flags & RSPAMD_MIME_TEXT_PART_ATTACHMENT)
struct rspamd_mime_text_part {
const gchar *language;
GPtrArray *languages;
const gchar *real_charset;
/* Raw data in native encoding */
rspamd_ftok_t raw;
rspamd_ftok_t parsed; /* decoded from mime encodings */
/* UTF8 content */
GByteArray *utf_content; /* utf8 encoded processed content */
GByteArray *utf_raw_content; /* utf raw content */
GByteArray *utf_stripped_content; /* utf content with no newlines */
GArray *normalized_hashes; /* Array of guint64 */
GArray *utf_words; /* Array of rspamd_stat_token_t */
UText utf_stripped_text; /* Used by libicu to represent the utf8 content */
GPtrArray *newlines; /**< positions of newlines in text, relative to content*/
struct html_content *html;
GList *exceptions; /**< list of offsets of urls */
struct rspamd_mime_part *mime_part;
guint flags;
guint nlines;
guint spaces;
guint nwords;
guint non_ascii_chars;
guint ascii_chars;
guint double_spaces;
guint non_spaces;
guint empty_lines;
guint capital_letters;
guint numeric_characters;
guint unicode_scripts;
};
struct rspamd_message_raw_headers_content {
const gchar *begin;
gsize len;
const gchar *body_start;
};
struct rspamd_message {
const gchar *message_id;
gchar *subject;
GPtrArray *parts; /**< list of parsed parts */
GPtrArray *text_parts; /**< list of text parts */
struct rspamd_message_raw_headers_content raw_headers_content;
struct rspamd_received_header *received; /**< list of received headers */
khash_t (rspamd_url_hash) *urls;
struct rspamd_mime_headers_table *raw_headers; /**< list of raw headers */
struct rspamd_mime_header *headers_order; /**< order of raw headers */
struct rspamd_task *task;
GPtrArray *rcpt_mime;
GPtrArray *from_mime;
guchar digest[16];
enum rspamd_newlines_type nlines_type; /**< type of newlines (detected on most of headers */
ref_entry_t ref;
};
#define MESSAGE_FIELD(task, field) ((task)->message->field)
#define MESSAGE_FIELD_CHECK(task, field) ((task)->message ? \
(task)->message->field : \
(__typeof__((task)->message->field))NULL)
/**
* Parse and pre-process mime message
* @param task worker_task object
* @return
*/
gboolean rspamd_message_parse (struct rspamd_task *task);
/**
* Process content in task (e.g. HTML parsing)
* @param task
*/
void rspamd_message_process (struct rspamd_task *task);
/**
* Converts string to cte
* @param str
* @return
*/
enum rspamd_cte rspamd_cte_from_string (const gchar *str);
/**
* Converts cte to string
* @param ct
* @return
*/
const gchar *rspamd_cte_to_string (enum rspamd_cte ct);
struct rspamd_message* rspamd_message_new (struct rspamd_task *task);
struct rspamd_message *rspamd_message_ref (struct rspamd_message *msg);
void rspamd_message_unref (struct rspamd_message *msg);
/**
* Updates digest of the message if modified
* @param msg
* @param input
* @param len
*/
void rspamd_message_update_digest (struct rspamd_message *msg,
const void *input, gsize len);
#ifdef __cplusplus
}
#endif
#endif
|
