1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
|
/**
* @file message.h
* Message processing functions and structures
*/
#ifndef RSPAMD_MESSAGE_H
#define RSPAMD_MESSAGE_H
#include "config.h"
#include "email_addr.h"
#include "addr.h"
#include "cryptobox.h"
#include "mime_headers.h"
#include "content_type.h"
#include <unicode/uchar.h>
#include <unicode/utext.h>
struct rspamd_task;
struct controller_session;
struct html_content;
struct rspamd_image;
struct rspamd_archive;
enum rspamd_mime_part_flags {
RSPAMD_MIME_PART_TEXT = (1 << 0),
RSPAMD_MIME_PART_ATTACHEMENT = (1 << 1),
RSPAMD_MIME_PART_IMAGE = (1 << 2),
RSPAMD_MIME_PART_ARCHIVE = (1 << 3),
RSPAMD_MIME_PART_BAD_CTE = (1 << 4),
RSPAMD_MIME_PART_MISSING_CTE = (1 << 5)
};
enum rspamd_cte {
RSPAMD_CTE_UNKNOWN = 0,
RSPAMD_CTE_7BIT = 1,
RSPAMD_CTE_8BIT = 2,
RSPAMD_CTE_QP = 3,
RSPAMD_CTE_B64 = 4,
};
struct rspamd_mime_text_part;
struct rspamd_mime_multipart {
GPtrArray *children;
};
struct rspamd_mime_part {
struct rspamd_content_type *ct;
struct rspamd_content_disposition *cd;
rspamd_ftok_t raw_data;
rspamd_ftok_t parsed_data;
struct rspamd_mime_part *parent_part;
GHashTable *raw_headers;
GQueue *headers_order;
gchar *raw_headers_str;
gsize raw_headers_len;
enum rspamd_cte cte;
union {
struct rspamd_mime_multipart mp;
struct rspamd_mime_text_part *txt;
struct rspamd_image *img;
struct rspamd_archive *arch;
} specific;
enum rspamd_mime_part_flags flags;
guchar digest[rspamd_cryptobox_HASHBYTES];
};
#define RSPAMD_MIME_TEXT_PART_FLAG_UTF (1 << 0)
#define RSPAMD_MIME_TEXT_PART_FLAG_BALANCED (1 << 1)
#define RSPAMD_MIME_TEXT_PART_FLAG_EMPTY (1 << 2)
#define RSPAMD_MIME_TEXT_PART_FLAG_HTML (1 << 3)
#define RSPAMD_MIME_TEXT_PART_FLAG_8BIT (1 << 4)
#define RSPAMD_MIME_TEXT_PART_FLAG_8BIT_ENCODED (1 << 5)
#define RSPAMD_MIME_TEXT_PART_HAS_SUBNORMAL (1 << 6)
#define RSPAMD_MIME_TEXT_PART_NORMALISED (1 << 7)
#define IS_PART_EMPTY(part) ((part)->flags & RSPAMD_MIME_TEXT_PART_FLAG_EMPTY)
#define IS_PART_UTF(part) ((part)->flags & RSPAMD_MIME_TEXT_PART_FLAG_UTF)
#define IS_PART_RAW(part) (!((part)->flags & RSPAMD_MIME_TEXT_PART_FLAG_UTF))
#define IS_PART_HTML(part) ((part)->flags & RSPAMD_MIME_TEXT_PART_FLAG_HTML)
struct rspamd_mime_text_part {
const gchar *language;
GPtrArray *languages;
const gchar *real_charset;
/* Raw data in native encoding */
rspamd_ftok_t raw;
rspamd_ftok_t parsed; /* decoded from mime encodings */
/* UTF8 content */
GByteArray *utf_content; /* utf8 encoded processed content */
GByteArray *utf_raw_content; /* utf raw content */
GByteArray *utf_stripped_content; /* utf content with no newlines */
GArray *normalized_hashes;
GArray *utf_words;
UText utf_stripped_text; /* Used by libicu to represent the utf8 content */
/* Unicode content, used by libicu */
GArray *unicode_raw_content; /* unicode raw content (of UChar) */
GArray *unicode_content; /* unicode processed content (of UChar) */
GPtrArray *newlines; /**< positions of newlines in text, relative to content*/
struct html_content *html;
GList *exceptions; /**< list of offsets of urls */
struct rspamd_mime_part *mime_part;
guint flags;
guint nlines;
guint spaces;
guint non_ascii_chars;
guint ascii_chars;
guint double_spaces;
guint non_spaces;
guint empty_lines;
guint capital_letters;
guint numeric_characters;
guint unicode_scripts;
};
enum rspamd_received_type {
RSPAMD_RECEIVED_SMTP = 0,
RSPAMD_RECEIVED_ESMTP,
RSPAMD_RECEIVED_ESMTPA,
RSPAMD_RECEIVED_ESMTPS,
RSPAMD_RECEIVED_ESMTPSA,
RSPAMD_RECEIVED_LMTP,
RSPAMD_RECEIVED_IMAP,
RSPAMD_RECEIVED_UNKNOWN
};
#define RSPAMD_RECEIVED_FLAG_ARTIFICIAL (1 << 0)
#define RSPAMD_RECEIVED_FLAG_SSL (1 << 1)
#define RSPAMD_RECEIVED_FLAG_AUTHENTICATED (1 << 2)
struct received_header {
gchar *from_hostname;
gchar *from_ip;
gchar *real_hostname;
gchar *real_ip;
gchar *by_hostname;
gchar *for_mbox;
gchar *comment_ip;
rspamd_inet_addr_t *addr;
struct rspamd_mime_header *hdr;
time_t timestamp;
enum rspamd_received_type type;
gint flags;
};
/**
* Parse and pre-process mime message
* @param task worker_task object
* @return
*/
gboolean rspamd_message_parse (struct rspamd_task *task);
/**
* Process content in task (e.g. HTML parsing)
* @param task
*/
void rspamd_message_process (struct rspamd_task *task);
/**
* Get an array of header's values with specified header's name using raw headers
* @param task worker task structure
* @param field header's name
* @param strong if this flag is TRUE header's name is case sensitive, otherwise it is not
* @return An array of header's values or NULL. It is NOT permitted to free array or values.
*/
GPtrArray *rspamd_message_get_header_array (struct rspamd_task *task,
const gchar *field,
gboolean strong);
/**
* Get an array of mime parts header's values with specified header's name using raw headers
* @param task worker task structure
* @param field header's name
* @param strong if this flag is TRUE header's name is case sensitive, otherwise it is not
* @return An array of header's values or NULL. It is NOT permitted to free array or values.
*/
GPtrArray *rspamd_message_get_mime_header_array (struct rspamd_task *task,
const gchar *field,
gboolean strong);
/**
* Get an array of header's values with specified header's name using raw headers
* @param htb hash table indexed by header name (caseless) with ptr arrays as elements
* @param field header's name
* @param strong if this flag is TRUE header's name is case sensitive, otherwise it is not
* @return An array of header's values or NULL. It is NOT permitted to free array or values.
*/
GPtrArray *rspamd_message_get_header_from_hash (GHashTable *htb,
rspamd_mempool_t *pool,
const gchar *field,
gboolean strong);
/**
* Converts string to cte
* @param str
* @return
*/
enum rspamd_cte rspamd_cte_from_string (const gchar *str);
/**
* Converts cte to string
* @param ct
* @return
*/
const gchar* rspamd_cte_to_string (enum rspamd_cte ct);
#endif
|
