From 8c18e5c97c77d73eda809ef474354ab0efbaf773 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Fri, 5 Oct 2018 20:01:28 +0100 Subject: [Fix] Slashing: backport chunk logic from libucl --- contrib/libucl/ucl.h | 1 + contrib/libucl/ucl_internal.h | 17 +++++- contrib/libucl/ucl_msgpack.c | 47 ++++++---------- contrib/libucl/ucl_parser.c | 128 ++++++++++++++++++++++++++++++++++++------ contrib/libucl/ucl_util.c | 9 ++- 5 files changed, 153 insertions(+), 49 deletions(-) (limited to 'contrib') diff --git a/contrib/libucl/ucl.h b/contrib/libucl/ucl.h index 34888b8c9..852a77cf1 100644 --- a/contrib/libucl/ucl.h +++ b/contrib/libucl/ucl.h @@ -105,6 +105,7 @@ typedef enum ucl_error { UCL_EIO, /**< IO error occurred during parsing */ UCL_ESTATE, /**< Invalid state machine state */ UCL_ENESTED, /**< Input has too many recursion levels */ + UCL_EUNPAIRED, /**< Input has too many recursion levels */ UCL_EMACRO, /**< Error processing a macro */ UCL_EINTERNAL, /**< Internal unclassified error */ UCL_ESSL, /**< SSL error */ diff --git a/contrib/libucl/ucl_internal.h b/contrib/libucl/ucl_internal.h index 4ddc713b5..8c16dce8b 100644 --- a/contrib/libucl/ucl_internal.h +++ b/contrib/libucl/ucl_internal.h @@ -148,6 +148,7 @@ enum ucl_parser_state { UCL_STATE_OBJECT, UCL_STATE_ARRAY, UCL_STATE_KEY, + UCL_STATE_KEY_OBRACE, UCL_STATE_VALUE, UCL_STATE_AFTER_VALUE, UCL_STATE_ARRAY_VALUE, @@ -185,16 +186,30 @@ struct ucl_macro { UT_hash_handle hh; }; +enum ucl_stack_flags { + UCL_STACK_HAS_OBRACE = (1u << 0), + UCL_STACK_MAX = (1u << 1), +}; + struct ucl_stack { ucl_object_t *obj; struct ucl_stack *next; - uint64_t level; + union { + struct { + uint16_t level; + uint16_t flags; + uint32_t line; + } params; + uint64_t len; + } e; + struct ucl_chunk *chunk; }; struct ucl_chunk { const unsigned char *begin; const unsigned char *end; const unsigned char *pos; + char *fname; size_t remain; unsigned int line; unsigned int column; diff --git a/contrib/libucl/ucl_msgpack.c b/contrib/libucl/ucl_msgpack.c index bd7c3a1ce..b075c9d7f 100644 --- a/contrib/libucl/ucl_msgpack.c +++ b/contrib/libucl/ucl_msgpack.c @@ -434,7 +434,6 @@ static ssize_t ucl_msgpack_parse_ignore (struct ucl_parser *parser, #define MSGPACK_FLAG_EXT (1 << 3) #define MSGPACK_FLAG_ASSOC (1 << 4) #define MSGPACK_FLAG_KEY (1 << 5) -#define MSGPACK_CONTAINER_BIT (1ULL << 62) /* * Search tree packed in array @@ -768,7 +767,6 @@ ucl_msgpack_get_container (struct ucl_parser *parser, assert (obj_parser != NULL); if (obj_parser->flags & MSGPACK_FLAG_CONTAINER) { - assert ((len & MSGPACK_CONTAINER_BIT) == 0); /* * Insert new container to the stack */ @@ -779,6 +777,8 @@ ucl_msgpack_get_container (struct ucl_parser *parser, ucl_create_err (&parser->err, "no memory"); return NULL; } + + parser->stack->chunk = parser->chunks; } else { stack = calloc (1, sizeof (struct ucl_stack)); @@ -788,11 +788,12 @@ ucl_msgpack_get_container (struct ucl_parser *parser, return NULL; } + stack->chunk = parser->chunks; stack->next = parser->stack; parser->stack = stack; } - parser->stack->level = len | MSGPACK_CONTAINER_BIT; + parser->stack->e.len = len; #ifdef MSGPACK_DEBUG_PARSER stack = parser->stack; @@ -823,16 +824,11 @@ ucl_msgpack_get_container (struct ucl_parser *parser, static bool ucl_msgpack_is_container_finished (struct ucl_stack *container) { - uint64_t level; - assert (container != NULL); - if (container->level & MSGPACK_CONTAINER_BIT) { - level = container->level & ~MSGPACK_CONTAINER_BIT; - if (level == 0) { - return true; - } + if (container->e.len == 0) { + return true; } return false; @@ -843,12 +839,11 @@ ucl_msgpack_insert_object (struct ucl_parser *parser, const unsigned char *key, size_t keylen, ucl_object_t *obj) { - uint64_t level; struct ucl_stack *container; container = parser->stack; assert (container != NULL); - assert (container->level > 0); + assert (container->e.len > 0); assert (obj != NULL); assert (container->obj != NULL); @@ -875,10 +870,7 @@ ucl_msgpack_insert_object (struct ucl_parser *parser, return false; } - if (container->level & MSGPACK_CONTAINER_BIT) { - level = container->level & ~MSGPACK_CONTAINER_BIT; - container->level = (level - 1) | MSGPACK_CONTAINER_BIT; - } + container->e.len--; return true; } @@ -887,7 +879,7 @@ static struct ucl_stack * ucl_msgpack_get_next_container (struct ucl_parser *parser) { struct ucl_stack *cur = NULL; - uint64_t level; + uint64_t len; cur = parser->stack; @@ -895,17 +887,16 @@ ucl_msgpack_get_next_container (struct ucl_parser *parser) return NULL; } - if (cur->level & MSGPACK_CONTAINER_BIT) { - level = cur->level & ~MSGPACK_CONTAINER_BIT; + len = cur->e.len; - if (level == 0) { - /* We need to switch to the previous container */ - parser->stack = cur->next; - parser->cur_obj = cur->obj; - free (cur); + if (len == 0) { + /* We need to switch to the previous container */ + parser->stack = cur->next; + parser->cur_obj = cur->obj; + free (cur); #ifdef MSGPACK_DEBUG_PARSER - cur = parser->stack; + cur = parser->stack; while (cur) { fprintf(stderr, "-"); cur = cur->next; @@ -913,8 +904,7 @@ ucl_msgpack_get_next_container (struct ucl_parser *parser) fprintf(stderr, "-%s -> %d\n", parser->cur_obj->type == UCL_OBJECT ? "object" : "array", (int)parser->cur_obj->len); #endif - return ucl_msgpack_get_next_container (parser); - } + return ucl_msgpack_get_next_container (parser); } /* @@ -1311,8 +1301,7 @@ ucl_msgpack_consume (struct ucl_parser *parser) /* Rewind to the top level container */ ucl_msgpack_get_next_container (parser); - assert (parser->stack == NULL || - (parser->stack->level & MSGPACK_CONTAINER_BIT) == 0); + assert (parser->stack == NULL); return true; } diff --git a/contrib/libucl/ucl_parser.c b/contrib/libucl/ucl_parser.c index 631bc7412..c4b390407 100644 --- a/contrib/libucl/ucl_parser.c +++ b/contrib/libucl/ucl_parser.c @@ -630,7 +630,7 @@ ucl_copy_or_store_ptr (struct ucl_parser *parser, */ static inline ucl_object_t * ucl_parser_add_container (ucl_object_t *obj, struct ucl_parser *parser, - bool is_array, int level) + bool is_array, uint32_t level, bool has_obrace) { struct ucl_stack *st; @@ -666,7 +666,27 @@ ucl_parser_add_container (ucl_object_t *obj, struct ucl_parser *parser, } st->obj = obj; - st->level = level; + + if (level >= UINT16_MAX) { + ucl_set_err (parser, UCL_ENESTED, + "objects are nesting too deep (over 65535 limit)", + &parser->err); + ucl_object_unref (obj); + return NULL; + } + + + st->e.params.level = level; + st->e.params.line = parser->chunks->line; + st->chunk = parser->chunks; + + if (has_obrace) { + st->e.params.flags = UCL_STACK_HAS_OBRACE; + } + else { + st->e.params.flags = 0; + } + LL_PREPEND (parser->stack, st); parser->cur_obj = obj; @@ -1014,7 +1034,8 @@ ucl_lex_json_string (struct ucl_parser *parser, ucl_chunk_skipc (chunk, p); } if (p >= chunk->end) { - ucl_set_err (parser, UCL_ESYNTAX, "unfinished escape character", + ucl_set_err (parser, UCL_ESYNTAX, + "unfinished escape character", &parser->err); return false; } @@ -1040,7 +1061,8 @@ ucl_lex_json_string (struct ucl_parser *parser, ucl_chunk_skipc (chunk, p); } - ucl_set_err (parser, UCL_ESYNTAX, "no quote at the end of json string", + ucl_set_err (parser, UCL_ESYNTAX, + "no quote at the end of json string", &parser->err); return false; } @@ -1065,7 +1087,8 @@ ucl_lex_squoted_string (struct ucl_parser *parser, ucl_chunk_skipc (chunk, p); if (p >= chunk->end) { - ucl_set_err (parser, UCL_ESYNTAX, "unfinished escape character", + ucl_set_err (parser, UCL_ESYNTAX, + "unfinished escape character", &parser->err); return false; } @@ -1084,7 +1107,8 @@ ucl_lex_squoted_string (struct ucl_parser *parser, ucl_chunk_skipc (chunk, p); } - ucl_set_err (parser, UCL_ESYNTAX, "no quote at the end of single quoted string", + ucl_set_err (parser, UCL_ESYNTAX, + "no quote at the end of single quoted string", &parser->err); return false; } @@ -1706,7 +1730,7 @@ ucl_parse_value (struct ucl_parser *parser, struct ucl_chunk *chunk) /* We have a new object */ if (parser->stack) { obj = ucl_parser_add_container (obj, parser, false, - parser->stack->level); + parser->stack->e.params.level, true); } else { return false; @@ -1727,7 +1751,7 @@ ucl_parse_value (struct ucl_parser *parser, struct ucl_chunk *chunk) /* We have a new array */ if (parser->stack) { obj = ucl_parser_add_container (obj, parser, true, - parser->stack->level); + parser->stack->e.params.level, true); } else { return false; @@ -1906,6 +1930,17 @@ ucl_parse_after_value (struct ucl_parser *parser, struct ucl_chunk *chunk) /* Pop all nested objects from a stack */ st = parser->stack; + + if (!(st->e.params.flags & UCL_STACK_HAS_OBRACE)) { + parser->err_code = UCL_EUNPAIRED; + ucl_create_err (&parser->err, + "%s:%d object closed with } is not opened with { at line %d", + chunk->fname ? chunk->fname : "memory", + parser->chunks->line, st->e.params.line); + + return false; + } + parser->stack = st->next; UCL_FREE (sizeof (struct ucl_stack), st); @@ -1916,9 +1951,13 @@ ucl_parse_after_value (struct ucl_parser *parser, struct ucl_chunk *chunk) while (parser->stack != NULL) { st = parser->stack; - if (st->next == NULL || st->next->level == st->level) { + if (st->next == NULL) { break; } + else if (st->next->e.params.level == st->e.params.level) { + break; + } + parser->stack = st->next; parser->cur_obj = st->obj; @@ -2294,6 +2333,8 @@ ucl_state_machine (struct ucl_parser *parser) return false; } else { + bool seen_obrace = false; + /* Skip any spaces */ while (p < chunk->end && ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) { @@ -2305,20 +2346,28 @@ ucl_state_machine (struct ucl_parser *parser) if (*p == '[') { parser->state = UCL_STATE_VALUE; ucl_chunk_skipc (chunk, p); + seen_obrace = true; } else { - parser->state = UCL_STATE_KEY; + if (*p == '{') { ucl_chunk_skipc (chunk, p); + parser->state = UCL_STATE_KEY_OBRACE; + seen_obrace = true; + } + else { + parser->state = UCL_STATE_KEY; } } if (parser->top_obj == NULL) { if (parser->state == UCL_STATE_VALUE) { - obj = ucl_parser_add_container (NULL, parser, true, 0); + obj = ucl_parser_add_container (NULL, parser, true, 0, + seen_obrace); } else { - obj = ucl_parser_add_container (NULL, parser, false, 0); + obj = ucl_parser_add_container (NULL, parser, false, 0, + seen_obrace); } if (obj == NULL) { @@ -2332,6 +2381,7 @@ ucl_state_machine (struct ucl_parser *parser) } break; case UCL_STATE_KEY: + case UCL_STATE_KEY_OBRACE: /* Skip any spaces */ while (p < chunk->end && ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) { ucl_chunk_skipc (chunk, p); @@ -2362,8 +2412,11 @@ ucl_state_machine (struct ucl_parser *parser) else if (parser->state != UCL_STATE_MACRO_NAME) { if (next_key && parser->stack->obj->type == UCL_OBJECT) { /* Parse more keys and nest objects accordingly */ - obj = ucl_parser_add_container (parser->cur_obj, parser, false, - parser->stack->level + 1); + obj = ucl_parser_add_container (parser->cur_obj, + parser, + false, + parser->stack->e.params.level + 1, + parser->state == UCL_STATE_KEY_OBRACE); if (obj == NULL) { return false; } @@ -2415,7 +2468,8 @@ ucl_state_machine (struct ucl_parser *parser) if (!ucl_skip_macro_as_comment (parser, chunk)) { /* We have invalid macro */ ucl_create_err (&parser->err, - "error on line %d at column %d: invalid macro", + "error at %s:%d at column %d: invalid macro", + chunk->fname ? chunk->fname : "memory", chunk->line, chunk->column); parser->state = UCL_STATE_ERROR; @@ -2438,8 +2492,9 @@ ucl_state_machine (struct ucl_parser *parser) HASH_FIND (hh, parser->macroes, c, macro_len, macro); if (macro == NULL) { ucl_create_err (&parser->err, - "error on line %d at column %d: " + "error at %s:%d at column %d: " "unknown macro: '%.*s', character: '%c'", + chunk->fname ? chunk->fname : "memory", chunk->line, chunk->column, (int) (p - c), @@ -2455,7 +2510,8 @@ ucl_state_machine (struct ucl_parser *parser) else { /* We have invalid macro name */ ucl_create_err (&parser->err, - "error on line %d at column %d: invalid macro name", + "error at %s:%d at column %d: invalid macro name", + chunk->fname ? chunk->fname : "memory", chunk->line, chunk->column); parser->state = UCL_STATE_ERROR; @@ -2554,6 +2610,35 @@ ucl_state_machine (struct ucl_parser *parser) } } + if (parser->stack != NULL) { + struct ucl_stack *st; + bool has_error = false; + + LL_FOREACH (parser->stack, st) { + if (st->chunk != parser->chunks) { + break; /* Not our chunk, give up */ + } + if (st->e.params.flags & UCL_STACK_HAS_OBRACE) { + if (parser->err == NULL) { + utstring_new (parser->err); + } + + utstring_printf (parser->err, "%s:%d unmatched open brace at %d; ", + chunk->fname ? chunk->fname : "memory", + parser->chunks->line, + st->e.params.line); + + has_error = true; + } + } + + if (has_error) { + parser->err_code = UCL_EUNPAIRED; + + return false; + } + } + return true; } @@ -2788,6 +2873,11 @@ ucl_parser_add_chunk_full (struct ucl_parser *parser, const unsigned char *data, chunk->priority = priority; chunk->strategy = strat; chunk->parse_type = parse_type; + + if (parser->cur_file) { + chunk->fname = strdup (parser->cur_file); + } + LL_PREPEND (parser->chunks, chunk); parser->recursion ++; @@ -2868,7 +2958,9 @@ ucl_parser_insert_chunk (struct ucl_parser *parser, const unsigned char *data, parser->state = UCL_STATE_INIT; /* Prevent inserted chunks from unintentionally closing the current object */ - if (parser->stack != NULL && parser->stack->next != NULL) parser->stack->level = parser->stack->next->level; + if (parser->stack != NULL && parser->stack->next != NULL) { + parser->stack->e.params.level = parser->stack->next->e.params.level; + } res = ucl_parser_add_chunk_full (parser, data, len, parser->chunks->priority, parser->chunks->strategy, parser->chunks->parse_type); diff --git a/contrib/libucl/ucl_util.c b/contrib/libucl/ucl_util.c index a65c30cec..b7b471205 100644 --- a/contrib/libucl/ucl_util.c +++ b/contrib/libucl/ucl_util.c @@ -532,6 +532,10 @@ ucl_chunk_free (struct ucl_chunk *chunk) } } + if (chunk->fname) { + free (chunk->fname); + } + UCL_FREE (sizeof (*chunk), chunk); } } @@ -1318,7 +1322,10 @@ ucl_include_file_single (const unsigned char *data, size_t len, return false; } st->obj = nest_obj; - st->level = parser->stack->level; + st->e.params.level = parser->stack->e.params.level; + st->e.params.flags = parser->stack->e.params.flags; + st->e.params.line = parser->stack->e.params.line; + st->chunk = parser->chunks; LL_PREPEND (parser->stack, st); parser->cur_obj = nest_obj; } -- cgit v1.2.3