1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
|
/**
* @file message.h
* Message processing functions and structures
*/
#ifndef RSPAMD_MESSAGE_H
#define RSPAMD_MESSAGE_H
#include "config.h"
#include "libmime/email_addr.h"
#include "libutil/addr.h"
#include "libcryptobox/cryptobox.h"
#include "libmime/mime_headers.h"
#include "libmime/content_type.h"
#include "libutil/ref.h"
#include "libutil/str_util.h"
#include <unicode/uchar.h>
#include <unicode/utext.h>
#ifdef __cplusplus
extern "C" {
#endif
struct rspamd_task;
struct controller_session;
struct html_content;
struct rspamd_image;
struct rspamd_archive;
enum rspamd_mime_part_flags {
RSPAMD_MIME_PART_TEXT = (1 << 0),
RSPAMD_MIME_PART_ATTACHEMENT = (1 << 1),
RSPAMD_MIME_PART_IMAGE = (1 << 2),
RSPAMD_MIME_PART_ARCHIVE = (1 << 3),
RSPAMD_MIME_PART_BAD_CTE = (1 << 4),
RSPAMD_MIME_PART_MISSING_CTE = (1 << 5)
};
enum rspamd_cte {
RSPAMD_CTE_UNKNOWN = 0,
RSPAMD_CTE_7BIT = 1,
RSPAMD_CTE_8BIT = 2,
RSPAMD_CTE_QP = 3,
RSPAMD_CTE_B64 = 4,
};
struct rspamd_mime_text_part;
struct rspamd_mime_multipart {
GPtrArray *children;
rspamd_ftok_t boundary;
};
struct rspamd_mime_part {
struct rspamd_content_type *ct;
struct rspamd_content_type *detected_ct;
struct rspamd_content_disposition *cd;
rspamd_ftok_t raw_data;
rspamd_ftok_t parsed_data;
struct rspamd_mime_part *parent_part;
struct rspamd_mime_header *headers_order;
khash_t(rspamd_mime_headers_htb) *raw_headers;
gchar *raw_headers_str;
gsize raw_headers_len;
enum rspamd_cte cte;
enum rspamd_mime_part_flags flags;
guint id;
union {
struct rspamd_mime_multipart *mp;
struct rspamd_mime_text_part *txt;
struct rspamd_image *img;
struct rspamd_archive *arch;
} specific;
guchar digest[rspamd_cryptobox_HASHBYTES];
};
#define RSPAMD_MIME_TEXT_PART_FLAG_UTF (1 << 0)
#define RSPAMD_MIME_TEXT_PART_FLAG_BALANCED (1 << 1)
#define RSPAMD_MIME_TEXT_PART_FLAG_EMPTY (1 << 2)
#define RSPAMD_MIME_TEXT_PART_FLAG_HTML (1 << 3)
#define RSPAMD_MIME_TEXT_PART_FLAG_8BIT (1 << 4)
#define RSPAMD_MIME_TEXT_PART_FLAG_8BIT_ENCODED (1 << 5)
#define RSPAMD_MIME_TEXT_PART_HAS_SUBNORMAL (1 << 6)
#define RSPAMD_MIME_TEXT_PART_NORMALISED (1 << 7)
#define IS_PART_EMPTY(part) ((part)->flags & RSPAMD_MIME_TEXT_PART_FLAG_EMPTY)
#define IS_PART_UTF(part) ((part)->flags & RSPAMD_MIME_TEXT_PART_FLAG_UTF)
#define IS_PART_RAW(part) (!((part)->flags & RSPAMD_MIME_TEXT_PART_FLAG_UTF))
#define IS_PART_HTML(part) ((part)->flags & RSPAMD_MIME_TEXT_PART_FLAG_HTML)
struct rspamd_mime_text_part {
const gchar *language;
GPtrArray *languages;
const gchar *real_charset;
/* Raw data in native encoding */
rspamd_ftok_t raw;
rspamd_ftok_t parsed; /* decoded from mime encodings */
/* UTF8 content */
GByteArray *utf_content; /* utf8 encoded processed content */
GByteArray *utf_raw_content; /* utf raw content */
GByteArray *utf_stripped_content; /* utf content with no newlines */
GArray *normalized_hashes;
GArray *utf_words;
UText utf_stripped_text; /* Used by libicu to represent the utf8 content */
GPtrArray *newlines; /**< positions of newlines in text, relative to content*/
struct html_content *html;
GList *exceptions; /**< list of offsets of urls */
struct rspamd_mime_part *mime_part;
guint flags;
guint nlines;
guint spaces;
guint nwords;
guint non_ascii_chars;
guint ascii_chars;
guint double_spaces;
guint non_spaces;
guint empty_lines;
guint capital_letters;
guint numeric_characters;
guint unicode_scripts;
};
struct rspamd_message {
const gchar *message_id;
gchar *subject;
GPtrArray *parts; /**< list of parsed parts */
GPtrArray *text_parts; /**< list of text parts */
struct {
const gchar *begin;
gsize len;
const gchar *body_start;
} raw_headers_content; /**< list of raw headers */
struct rspamd_received_header *received; /**< list of received headers */
GHashTable *urls; /**< list of parsed urls */
GHashTable *emails; /**< list of parsed emails */
khash_t(rspamd_mime_headers_htb) *raw_headers; /**< list of raw headers */
struct rspamd_mime_header *headers_order; /**< order of raw headers */
GPtrArray *rcpt_mime;
GPtrArray *from_mime;
guchar digest[16];
enum rspamd_newlines_type nlines_type; /**< type of newlines (detected on most of headers */
ref_entry_t ref;
};
#ifndef FULL_DEBUG
#define MESSAGE_FIELD(task, field) ((task)->message->field)
#else
#define MESSAGE_FIELD(task, field)
((!(task)->message) ? \
(__typeof__((task)->message->field))(msg_err_task("no message when getting field %s", #field), g_assert(0), NULL) : \
((task)->message->field))
#endif
/**
* Parse and pre-process mime message
* @param task worker_task object
* @return
*/
gboolean rspamd_message_parse (struct rspamd_task *task);
/**
* Process content in task (e.g. HTML parsing)
* @param task
*/
void rspamd_message_process (struct rspamd_task *task);
/**
* Converts string to cte
* @param str
* @return
*/
enum rspamd_cte rspamd_cte_from_string (const gchar *str);
/**
* Converts cte to string
* @param ct
* @return
*/
const gchar *rspamd_cte_to_string (enum rspamd_cte ct);
struct rspamd_message* rspamd_message_new (struct rspamd_task *task);
struct rspamd_message *rspamd_message_ref (struct rspamd_message *msg);
void rspamd_message_unref (struct rspamd_message *msg);
#ifdef __cplusplus
}
#endif
#endif
|
