1 files changed, 133 insertions, 85 deletions

diff --git a/contrib/libucl/ucl_util.c b/contrib/libucl/ucl_util.c
index 7833e1f1c..9ea151e37 100644
--- a/contrib/libucl/ucl_util.c
+++ b/contrib/libucl/ucl_util.c
@@ -410,11 +410,24 @@ ucl_copy_value_trash (const ucl_object_t *obj)
 		if (obj->type == UCL_STRING) {
 
 			/* Special case for strings */
-			deconst->trash_stack[UCL_TRASH_VALUE] = malloc (obj->len + 1);
-			if (deconst->trash_stack[UCL_TRASH_VALUE] != NULL) {
-				memcpy (deconst->trash_stack[UCL_TRASH_VALUE], obj->value.sv, obj->len);
-				deconst->trash_stack[UCL_TRASH_VALUE][obj->len] = '\0';
-				deconst->value.sv = obj->trash_stack[UCL_TRASH_VALUE];
+			if (obj->flags & UCL_OBJECT_BINARY) {
+				deconst->trash_stack[UCL_TRASH_VALUE] = malloc (obj->len);
+				if (deconst->trash_stack[UCL_TRASH_VALUE] != NULL) {
+					memcpy (deconst->trash_stack[UCL_TRASH_VALUE],
+							obj->value.sv,
+							obj->len);
+					deconst->value.sv = obj->trash_stack[UCL_TRASH_VALUE];
+				}
+			}
+			else {
+				deconst->trash_stack[UCL_TRASH_VALUE] = malloc (obj->len + 1);
+				if (deconst->trash_stack[UCL_TRASH_VALUE] != NULL) {
+					memcpy (deconst->trash_stack[UCL_TRASH_VALUE],
+							obj->value.sv,
+							obj->len);
+					deconst->trash_stack[UCL_TRASH_VALUE][obj->len] = '\0';
+					deconst->value.sv = obj->trash_stack[UCL_TRASH_VALUE];
+				}
 			}
 		}
 		else {
@@ -424,6 +437,7 @@ ucl_copy_value_trash (const ucl_object_t *obj)
 		}
 		deconst->flags |= UCL_OBJECT_ALLOCATED_VALUE;
 	}
+	
 	return obj->trash_stack[UCL_TRASH_VALUE];
 }
 
@@ -796,6 +810,20 @@ ucl_sig_check (const unsigned char *data, size_t datalen,
 }
 #endif
 
+struct ucl_include_params {
+	bool check_signature;
+	bool must_exist;
+	bool use_glob;
+	bool use_prefix;
+	bool soft_fail;
+	bool allow_glob;
+	unsigned priority;
+	enum ucl_duplicate_strategy strat;
+	enum ucl_parse_type parse_type;
+	const char *prefix;
+	const char *target;
+};
+
 /**
  * Include an url to configuration
  * @param data
@@ -806,8 +834,8 @@ ucl_sig_check (const unsigned char *data, size_t datalen,
  */
 static bool
 ucl_include_url (const unsigned char *data, size_t len,
-		struct ucl_parser *parser, bool check_signature, bool must_exist,
-		bool use_prefix, const char *prefix, const char *target, unsigned priority)
+		struct ucl_parser *parser,
+		struct ucl_include_params *params)
 {
 
 	bool res;
@@ -819,11 +847,11 @@ ucl_include_url (const unsigned char *data, size_t len,
 
 	snprintf (urlbuf, sizeof (urlbuf), "%.*s", (int)len, data);
 
-	if (!ucl_fetch_url (urlbuf, &buf, &buflen, &parser->err, must_exist)) {
-		return (!must_exist || false);
+	if (!ucl_fetch_url (urlbuf, &buf, &buflen, &parser->err, params->must_exist)) {
+		return (!params->must_exist || false);
 	}
 
-	if (check_signature) {
+	if (params->check_signature) {
 #if (defined(HAVE_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x10000000L)
 		unsigned char *sigbuf = NULL;
 		size_t siglen = 0;
@@ -850,7 +878,8 @@ ucl_include_url (const unsigned char *data, size_t len,
 	prev_state = parser->state;
 	parser->state = UCL_STATE_INIT;
 
-	res = ucl_parser_add_chunk_priority (parser, buf, buflen, priority);
+	res = ucl_parser_add_chunk_full (parser, buf, buflen, params->priority,
+			params->strat, params->parse_type);
 	if (res == true) {
 		/* Remove chunk from the stack */
 		chunk = parser->chunks;
@@ -879,9 +908,7 @@ ucl_include_url (const unsigned char *data, size_t len,
  */
 static bool
 ucl_include_file_single (const unsigned char *data, size_t len,
-		struct ucl_parser *parser, bool check_signature, bool must_exist,
-		bool use_prefix, const char *prefix, const char *target,
-		bool soft_fail, unsigned priority)
+		struct ucl_parser *parser, struct ucl_include_params *params)
 {
 	bool res;
 	struct ucl_chunk *chunk;
@@ -898,10 +925,10 @@ ucl_include_file_single (const unsigned char *data, size_t len,
 
 	snprintf (filebuf, sizeof (filebuf), "%.*s", (int)len, data);
 	if (ucl_realpath (filebuf, realbuf) == NULL) {
-		if (soft_fail) {
+		if (params->soft_fail) {
 			return false;
 		}
-		if (!must_exist) {
+		if (!params->must_exist) {
 			return true;
 		}
 		ucl_create_err (&parser->err, "cannot open file %s: %s",
@@ -912,22 +939,23 @@ ucl_include_file_single (const unsigned char *data, size_t len,
 
 	if (parser->cur_file && strcmp (realbuf, parser->cur_file) == 0) {
 		/* We are likely including the file itself */
-		if (soft_fail) {
+		if (params->soft_fail) {
 			return false;
 		}
+
 		ucl_create_err (&parser->err, "trying to include the file %s from itself",
 				realbuf);
 		return false;
 	}
 
-	if (!ucl_fetch_file (realbuf, &buf, &buflen, &parser->err, must_exist)) {
-		if (soft_fail) {
+	if (!ucl_fetch_file (realbuf, &buf, &buflen, &parser->err, params->must_exist)) {
+		if (params->soft_fail) {
 			return false;
 		}
-		return (!must_exist || false);
+		return (!params->must_exist || false);
 	}
 
-	if (check_signature) {
+	if (params->check_signature) {
 #if (defined(HAVE_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x10000000L)
 		unsigned char *sigbuf = NULL;
 		size_t siglen = 0;
@@ -971,26 +999,27 @@ ucl_include_file_single (const unsigned char *data, size_t len,
 	prev_state = parser->state;
 	parser->state = UCL_STATE_INIT;
 
-	if (use_prefix && prefix == NULL) {
+	if (params->use_prefix && params->prefix == NULL) {
 		/* Auto generate a key name based on the included filename */
-		prefix = basename (realbuf);
-		ext = strrchr(prefix, '.');
+		params->prefix = basename (realbuf);
+		ext = strrchr (params->prefix, '.');
 		if (ext != NULL && (strcmp (ext, ".conf") == 0 || strcmp (ext, ".ucl") == 0)) {
 			/* Strip off .conf or .ucl */
 			*ext = '\0';
 		}
 	}
-	if (prefix != NULL) {
+	if (params->prefix != NULL) {
 		/* This is a prefixed include */
 		container = parser->stack->obj->value.ov;
 
-		old_obj = __DECONST (ucl_object_t *, ucl_hash_search (container, prefix, strlen (prefix)));
+		old_obj = __DECONST (ucl_object_t *, ucl_hash_search (container,
+				params->prefix, strlen (params->prefix)));
 
-		if (strcasecmp (target, "array") == 0 && old_obj == NULL) {
+		if (strcasecmp (params->target, "array") == 0 && old_obj == NULL) {
 			/* Create an array with key: prefix */
-			old_obj = ucl_object_new_full (UCL_ARRAY, priority);
-			old_obj->key = prefix;
-			old_obj->keylen = strlen (prefix);
+			old_obj = ucl_object_new_full (UCL_ARRAY, params->priority);
+			old_obj->key = params->prefix;
+			old_obj->keylen = strlen (params->prefix);
 			ucl_copy_key_trash(old_obj);
 			old_obj->prev = old_obj;
 			old_obj->next = NULL;
@@ -999,7 +1028,7 @@ ucl_include_file_single (const unsigned char *data, size_t len,
 					parser->flags & UCL_PARSER_KEY_LOWERCASE);
 			parser->stack->obj->len ++;
 
-			nest_obj = ucl_object_new_full (UCL_OBJECT, priority);
+			nest_obj = ucl_object_new_full (UCL_OBJECT, params->priority);
 			nest_obj->prev = nest_obj;
 			nest_obj->next = NULL;
 
@@ -1007,9 +1036,9 @@ ucl_include_file_single (const unsigned char *data, size_t len,
 		}
 		else if (old_obj == NULL) {
 			/* Create an object with key: prefix */
-			nest_obj = ucl_object_new_full (UCL_OBJECT, priority);
-			nest_obj->key = prefix;
-			nest_obj->keylen = strlen (prefix);
+			nest_obj = ucl_object_new_full (UCL_OBJECT, params->priority);
+			nest_obj->key = params->prefix;
+			nest_obj->keylen = strlen (params->prefix);
 			ucl_copy_key_trash(nest_obj);
 			nest_obj->prev = nest_obj;
 			nest_obj->next = NULL;
@@ -1018,10 +1047,11 @@ ucl_include_file_single (const unsigned char *data, size_t len,
 					parser->flags & UCL_PARSER_KEY_LOWERCASE);
 			parser->stack->obj->len ++;
 		}
-		else if (strcasecmp (target, "array") == 0 || ucl_object_type(old_obj) == UCL_ARRAY) {
+		else if (strcasecmp (params->target, "array") == 0 ||
+				ucl_object_type(old_obj) == UCL_ARRAY) {
 			if (ucl_object_type(old_obj) == UCL_ARRAY) {
 				/* Append to the existing array */
-				nest_obj = ucl_object_new_full (UCL_OBJECT, priority);
+				nest_obj = ucl_object_new_full (UCL_OBJECT, params->priority);
 				nest_obj->prev = nest_obj;
 				nest_obj->next = NULL;
 
@@ -1036,7 +1066,7 @@ ucl_include_file_single (const unsigned char *data, size_t len,
 				new_obj->prev = new_obj;
 				new_obj->next = NULL;
 
-				nest_obj = ucl_object_new_full (UCL_OBJECT, priority);
+				nest_obj = ucl_object_new_full (UCL_OBJECT, params->priority);
 				nest_obj->prev = nest_obj;
 				nest_obj->next = NULL;
 
@@ -1054,7 +1084,7 @@ ucl_include_file_single (const unsigned char *data, size_t len,
 				/* The key is not an object */
 				ucl_create_err (&parser->err,
 						"Conflicting type for key: %s",
-						prefix);
+						params->prefix);
 				return false;
 			}
 		}
@@ -1076,8 +1106,9 @@ ucl_include_file_single (const unsigned char *data, size_t len,
 		}
 	}
 
-	res = ucl_parser_add_chunk_priority (parser, buf, buflen, priority);
-	if (!res && !must_exist) {
+	res = ucl_parser_add_chunk_full (parser, buf, buflen, params->priority,
+			params->strat, params->parse_type);
+	if (!res && !params->must_exist) {
 		/* Free error */
 		utstring_free (parser->err);
 		parser->err = NULL;
@@ -1085,7 +1116,7 @@ ucl_include_file_single (const unsigned char *data, size_t len,
 	}
 
 	/* Stop nesting the include, take 1 level off the stack */
-	if (prefix != NULL && nest_obj != NULL) {
+	if (params->prefix != NULL && nest_obj != NULL) {
 		parser->stack = st->next;
 		UCL_FREE (sizeof (struct ucl_stack), st);
 	}
@@ -1144,9 +1175,7 @@ ucl_include_file_single (const unsigned char *data, size_t len,
  */
 static bool
 ucl_include_file (const unsigned char *data, size_t len,
-		struct ucl_parser *parser, bool check_signature, bool must_exist,
-		bool allow_glob, bool use_prefix, const char *prefix,
-		const char *target, bool soft_fail, unsigned priority)
+		struct ucl_parser *parser, struct ucl_include_params *params)
 {
 	const unsigned char *p = data, *end = data + len;
 	bool need_glob = false;
@@ -1155,9 +1184,8 @@ ucl_include_file (const unsigned char *data, size_t len,
 	size_t i;
 
 #ifndef _WIN32
-	if (!allow_glob) {
-		return ucl_include_file_single (data, len, parser, check_signature,
-			must_exist, use_prefix, prefix, target, soft_fail, priority);
+	if (!params->allow_glob) {
+		return ucl_include_file_single (data, len, parser, params);
 	}
 	else {
 		/* Check for special symbols in a filename */
@@ -1174,13 +1202,12 @@ ucl_include_file (const unsigned char *data, size_t len,
 			ucl_strlcpy (glob_pattern, (const char *)data,
 				(len + 1 < sizeof (glob_pattern) ? len + 1 : sizeof (glob_pattern)));
 			if (glob (glob_pattern, 0, NULL, &globbuf) != 0) {
-				return (!must_exist || false);
+				return (!params->must_exist || false);
 			}
 			for (i = 0; i < globbuf.gl_pathc; i ++) {
 				if (!ucl_include_file_single ((unsigned char *)globbuf.gl_pathv[i],
-						strlen (globbuf.gl_pathv[i]), parser, check_signature,
-						must_exist, use_prefix, prefix, target, soft_fail, priority)) {
-					if (soft_fail) {
+						strlen (globbuf.gl_pathv[i]), parser, params)) {
+					if (params->soft_fail) {
 						continue;
 					}
 					globfree (&globbuf);
@@ -1190,22 +1217,20 @@ ucl_include_file (const unsigned char *data, size_t len,
 			}
 			globfree (&globbuf);
 
-			if (cnt == 0 && must_exist) {
+			if (cnt == 0 && params->must_exist) {
 				ucl_create_err (&parser->err, "cannot match any files for pattern %s",
 					glob_pattern);
 				return false;
 			}
 		}
 		else {
-			return ucl_include_file_single (data, len, parser, check_signature,
-				must_exist, use_prefix, prefix, target, soft_fail, priority);
+			return ucl_include_file_single (data, len, parser, params);
 		}
 	}
 #else
 	/* Win32 compilers do not support globbing. Therefore, for Win32,
 	   treat allow_glob/need_glob as a NOOP and just return */
-	return ucl_include_file_single (data, len, parser, check_signature,
-		must_exist, use_prefix, prefix, target, soft_fail, priority);
+	return ucl_include_file_single (data, len, parser, params);
 #endif
 	
 	return true;
@@ -1227,22 +1252,25 @@ ucl_include_common (const unsigned char *data, size_t len,
 		bool default_try,
 		bool default_sign)
 {
-	bool try_load, allow_glob, allow_url, need_sign, use_prefix, search;
-	const char *prefix, *target;
-	unsigned priority;
+	bool allow_url, search;
+	const char *duplicate;
 	const ucl_object_t *param;
 	ucl_object_iter_t it = NULL, ip = NULL;
 	char ipath[PATH_MAX];
+	struct ucl_include_params params;
 
 	/* Default values */
-	try_load = default_try;
-	allow_glob = false;
-	allow_url = true;
-	need_sign = default_sign;
-	use_prefix = false;
-	prefix = NULL;
-	target = "object";
-	priority = 0;
+	params.soft_fail = default_try;
+	params.allow_glob = false;
+	params.check_signature = default_sign;
+	params.use_prefix = false;
+	params.target = "object";
+	params.prefix = NULL;
+	params.priority = 0;
+	params.parse_type = UCL_PARSE_UCL;
+	params.strat = UCL_DUPLICATE_APPEND;
+	params.must_exist = !default_try;
+
 	search = false;
 
 	/* Process arguments */
@@ -1250,27 +1278,43 @@ ucl_include_common (const unsigned char *data, size_t len,
 		while ((param = ucl_iterate_object (args, &it, true)) != NULL) {
 			if (param->type == UCL_BOOLEAN) {
 				if (strncmp (param->key, "try", param->keylen) == 0) {
-					try_load = ucl_object_toboolean (param);
+					params.must_exist = !ucl_object_toboolean (param);
 				}
 				else if (strncmp (param->key, "sign", param->keylen) == 0) {
-					need_sign = ucl_object_toboolean (param);
+					params.check_signature = ucl_object_toboolean (param);
 				}
 				else if (strncmp (param->key, "glob", param->keylen) == 0) {
-					allow_glob = ucl_object_toboolean (param);
+					params.allow_glob = ucl_object_toboolean (param);
 				}
 				else if (strncmp (param->key, "url", param->keylen) == 0) {
 					allow_url = ucl_object_toboolean (param);
 				}
 				else if (strncmp (param->key, "prefix", param->keylen) == 0) {
-					use_prefix = ucl_object_toboolean (param);
+					params.use_prefix = ucl_object_toboolean (param);
 				}
 			}
 			else if (param->type == UCL_STRING) {
 				if (strncmp (param->key, "key", param->keylen) == 0) {
-					prefix = ucl_object_tostring (param);
+					params.prefix = ucl_object_tostring (param);
 				}
 				else if (strncmp (param->key, "target", param->keylen) == 0) {
-					target = ucl_object_tostring (param);
+					params.target = ucl_object_tostring (param);
+				}
+				else if (strncmp (param->key, "duplicate", param->keylen) == 0) {
+					duplicate = ucl_object_tostring (param);
+
+					if (strcmp (duplicate, "append") == 0) {
+						params.strat = UCL_DUPLICATE_APPEND;
+					}
+					else if (strcmp (duplicate, "merge") == 0) {
+						params.strat = UCL_DUPLICATE_MERGE;
+					}
+					else if (strcmp (duplicate, "rewrite") == 0) {
+						params.strat = UCL_DUPLICATE_REWRITE;
+					}
+					else if (strcmp (duplicate, "error") == 0) {
+						params.strat = UCL_DUPLICATE_ERROR;
+					}
 				}
 			}
 			else if (param->type == UCL_ARRAY) {
@@ -1280,7 +1324,7 @@ ucl_include_common (const unsigned char *data, size_t len,
 			}
 			else if (param->type == UCL_INT) {
 				if (strncmp (param->key, "priority", param->keylen) == 0) {
-					priority = ucl_object_toint (param);
+					params.priority = ucl_object_toint (param);
 				}
 			}
 		}
@@ -1289,20 +1333,17 @@ ucl_include_common (const unsigned char *data, size_t len,
 	if (parser->includepaths == NULL) {
 		if (allow_url && ucl_strnstr (data, "://", len) != NULL) {
 			/* Globbing is not used for URL's */
-			return ucl_include_url (data, len, parser, need_sign,
-					!try_load, use_prefix, prefix, target, priority);
+			return ucl_include_url (data, len, parser, &params);
 		}
 		else if (data != NULL) {
 			/* Try to load a file */
-			return ucl_include_file (data, len, parser, need_sign, !try_load,
-					allow_glob, use_prefix, prefix, target, false, priority);
+			return ucl_include_file (data, len, parser, &params);
 		}
 	}
 	else {
 		if (allow_url && ucl_strnstr (data, "://", len) != NULL) {
 			/* Globbing is not used for URL's */
-			return ucl_include_url (data, len, parser, need_sign,
-					!try_load, use_prefix, prefix, target, priority);
+			return ucl_include_url (data, len, parser, &params);
 		}
 
 		ip = ucl_object_iterate_new (parser->includepaths);
@@ -1310,9 +1351,9 @@ ucl_include_common (const unsigned char *data, size_t len,
 			if (ucl_object_type(param) == UCL_STRING) {
 				snprintf (ipath, sizeof (ipath), "%s/%.*s", ucl_object_tostring(param),
 						(int)len, data);
-				if ((search = ucl_include_file (ipath, strlen (ipath), parser, need_sign,
-						!try_load, allow_glob, use_prefix, prefix, target, true, priority))) {
-					if (!allow_glob) {
+				if ((search = ucl_include_file (ipath, strlen (ipath),
+						parser, &params))) {
+					if (!params.allow_glob) {
 						break;
 					}
 				}
@@ -2903,7 +2944,9 @@ ucl_object_tostring_safe (const ucl_object_t *obj, const char **target)
 
 	switch (obj->type) {
 	case UCL_STRING:
-		*target = ucl_copy_value_trash (obj);
+		if (!(obj->flags & UCL_OBJECT_BINARY)) {
+			*target = ucl_copy_value_trash (obj);
+		}
 		break;
 	default:
 		return false;
@@ -2924,7 +2967,12 @@ ucl_object_tostring (const ucl_object_t *obj)
 const char *
 ucl_object_tostring_forced (const ucl_object_t *obj)
 {
-	return ucl_copy_value_trash (obj);
+	/* TODO: For binary strings we might encode string here */
+	if (!(obj->flags & UCL_OBJECT_BINARY)) {
+		return ucl_copy_value_trash (obj);
+	}
+
+	return NULL;
 }
 
 bool