From 830131db46761dd7abdcda5734dc4d9e8aaf5125 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Wed, 7 Aug 2024 14:23:44 +0100 Subject: [PATCH] [Minor] Avoid some warnings --- CMakeLists.txt | 2 + contrib/kann/kann.c | 325 +++++++++++++++++++++++------------- contrib/zstd/CMakeLists.txt | 2 +- 3 files changed, 208 insertions(+), 121 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 75b70a42c..a9bb50642 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -267,6 +267,8 @@ ADD_DEFINITIONS(-DFMT_HEADER_ONLY) # Workaround for https://github.com/onqtam/doctest/issues/356 ADD_DEFINITIONS(-DDOCTEST_CONFIG_USE_STD_HEADERS) ADD_DEFINITIONS(-DU_CHARSET_IS_UTF8) +# Disable zstd deprecation warnings, as they are not relevant for us +ADD_DEFINITIONS(-DZSTD_DISABLE_DEPRECATE_WARNINGS) # Check platform specific includes CHECK_INCLUDE_FILES(sys/types.h HAVE_SYS_TYPES_H) diff --git a/contrib/kann/kann.c b/contrib/kann/kann.c index 70d1f02d6..658f98a44 100644 --- a/contrib/kann/kann.c +++ b/contrib/kann/kann.c @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Vsevolod Stakhov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #include "config.h" #include @@ -19,9 +35,9 @@ static void kad_ext_collate(int n, kad_node_t **a, float **_x, float **_g, float int i, j, k, l, n_var; float *x, *g, *c; n_var = kad_size_var(n, a); - x = *_x = (float*)realloc(*_x, n_var * sizeof(float)); - g = *_g = (float*)realloc(*_g, n_var * sizeof(float)); - c = *_c = (float*)realloc(*_c, kad_size_const(n, a) * sizeof(float)); + x = *_x = (float *) realloc(*_x, n_var * sizeof(float)); + g = *_g = (float *) realloc(*_g, n_var * sizeof(float)); + c = *_c = (float *) realloc(*_c, kad_size_const(n, a) * sizeof(float)); memset(g, 0, n_var * sizeof(float)); for (i = j = k = 0; i < n; ++i) { kad_node_t *v = a[i]; @@ -32,7 +48,8 @@ static void kad_ext_collate(int n, kad_node_t **a, float **_x, float **_g, float v->x = &x[j]; v->g = &g[j]; j += l; - } else if (kad_is_const(v)) { + } + else if (kad_is_const(v)) { l = kad_len(v); memcpy(&c[k], v->x, l * sizeof(float)); free(v->x); @@ -51,7 +68,8 @@ static void kad_ext_sync(int n, kad_node_t **a, float *x, float *g, float *c) v->x = &x[j]; v->g = &g[j]; j += kad_len(v); - } else if (kad_is_const(v)) { + } + else if (kad_is_const(v)) { v->x = &c[k]; k += kad_len(v); } @@ -68,14 +86,14 @@ kann_t *kann_new(kad_node_t *cost, int n_rest, ...) if (cost->n_d != 0) return 0; va_start(ap, n_rest); - roots = (kad_node_t**)malloc((n_roots + 1) * sizeof(kad_node_t*)); + roots = (kad_node_t **) malloc((n_roots + 1) * sizeof(kad_node_t *)); for (i = 0; i < n_rest; ++i) - roots[i] = va_arg(ap, kad_node_t*); + roots[i] = va_arg(ap, kad_node_t *); roots[i++] = cost; va_end(ap); cost->ext_flag |= KANN_F_COST; - a = (kann_t*)calloc(1, sizeof(kann_t)); + a = (kann_t *) calloc(1, sizeof(kann_t)); a->v = kad_compile_array(&a->n, n_roots, roots); for (i = 0; i < a->n; ++i) { @@ -84,7 +102,7 @@ kann_t *kann_new(kad_node_t *cost, int n_rest, ...) } if (has_recur && !has_pivot) { /* an RNN that doesn't have a pivot; then add a pivot on top of cost and recompile */ cost->ext_flag &= ~KANN_F_COST; - roots[n_roots-1] = cost = kad_avg(1, &cost), cost->ext_flag |= KANN_F_COST; + roots[n_roots - 1] = cost = kad_avg(1, &cost), cost->ext_flag |= KANN_F_COST; free(a->v); a->v = kad_compile_array(&a->n, n_roots, roots); } @@ -96,7 +114,7 @@ kann_t *kann_new(kad_node_t *cost, int n_rest, ...) kann_t *kann_clone(kann_t *a, int batch_size) { kann_t *b; - b = (kann_t*)calloc(1, sizeof(kann_t)); + b = (kann_t *) calloc(1, sizeof(kann_t)); b->n = a->n; b->v = kad_clone(a->n, a->v, batch_size); kad_ext_collate(b->n, b->v, &b->x, &b->g, &b->c); @@ -106,7 +124,7 @@ kann_t *kann_clone(kann_t *a, int batch_size) kann_t *kann_unroll_array(kann_t *a, int *len) { kann_t *b; - b = (kann_t*)calloc(1, sizeof(kann_t)); + b = (kann_t *) calloc(1, sizeof(kann_t)); b->x = a->x, b->g = a->g, b->c = a->c; /* these arrays are shared */ b->v = kad_unroll(a->n, a->v, &b->n, len); return b; @@ -118,7 +136,7 @@ kann_t *kann_unroll(kann_t *a, ...) va_list ap; int i, n_pivots, *len; n_pivots = kad_n_pivots(a->n, a->v); - len = (int*)calloc(n_pivots, sizeof(int)); + len = (int *) calloc(n_pivots, sizeof(int)); va_start(ap, a); for (i = 0; i < n_pivots; ++i) len[i] = va_arg(ap, int); va_end(ap); @@ -137,7 +155,9 @@ void kann_delete_unrolled(kann_t *a) void kann_delete(kann_t *a) { if (a == 0) return; - free(a->x); free(a->g); free(a->c); + free(a->x); + free(a->g); + free(a->c); kann_delete_unrolled(a); } @@ -146,7 +166,7 @@ static void kann_switch_core(kann_t *a, int is_train) int i; for (i = 0; i < a->n; ++i) if (a->v[i]->op == 12 && a->v[i]->n_child == 2) - *(int32_t*)a->v[i]->ptr = !!is_train; + *(int32_t *) a->v[i]->ptr = !!is_train; } #define chk_flg(flag, mask) ((mask) == 0 || ((flag) & (mask))) @@ -158,7 +178,8 @@ int kann_find(const kann_t *a, uint32_t ext_flag, int32_t ext_label) for (i = k = 0; i < a->n; ++i) if (chk_flg(a->v[i]->ext_flag, ext_flag) && chk_lbl(a->v[i]->ext_label, ext_label)) ++k, r = i; - return k == 1? r : k == 0? -1 : -2; + return k == 1 ? r : k == 0 ? -1 + : -2; } int kann_feed_bind(kann_t *a, uint32_t ext_flag, int32_t ext_label, float **x) @@ -176,8 +197,10 @@ int kann_feed_dim(const kann_t *a, uint32_t ext_flag, int32_t ext_label) int i, k, n = 0; for (i = k = 0; i < a->n; ++i) if (kad_is_feed(a->v[i]) && chk_flg(a->v[i]->ext_flag, ext_flag) && chk_lbl(a->v[i]->ext_label, ext_label)) - ++k, n = a->v[i]->n_d > 1? kad_len(a->v[i]) / a->v[i]->d[0] : a->v[i]->n_d == 1? a->v[i]->d[0] : 1; - return k == 1? n : k == 0? -1 : -2; + ++k, n = a->v[i]->n_d > 1 ? kad_len(a->v[i]) / a->v[i]->d[0] : a->v[i]->n_d == 1 ? a->v[i]->d[0] + : 1; + return k == 1 ? n : k == 0 ? -1 + : -2; } static float kann_cost_core(kann_t *a, int cost_label, int cal_grad) @@ -210,7 +233,8 @@ void kann_rnn_start(kann_t *a) if (p->pre) { /* NB: BE CAREFUL of the interaction between kann_rnn_start() and kann_set_batch_size() */ kad_node_t *q = p->pre; if (q->x) memcpy(p->x, q->x, kad_len(p) * sizeof(float)); - else memset(p->x, 0, kad_len(p) * sizeof(float)); + else + memset(p->x, 0, kad_len(p) * sizeof(float)); if (q->n_child > 0) free(q->x); q->x = p->x; } @@ -223,7 +247,7 @@ void kann_rnn_end(kann_t *a) kad_ext_sync(a->n, a->v, a->x, a->g, a->c); for (i = 0; i < a->n; ++i) if (a->v[i]->pre && a->v[i]->pre->n_child > 0) - a->v[i]->pre->x = (float*)calloc(kad_len(a->v[i]->pre), sizeof(float)); + a->v[i]->pre->x = (float *) calloc(kad_len(a->v[i]->pre), sizeof(float)); } static int kann_class_error_core(const kann_t *ann, int *base) @@ -238,10 +262,10 @@ static int kann_class_error_core(const kann_t *ann, int *base) float t_sum = 0.0f, t_min = 1.0f, t_max = 0.0f, x_max = 0.0f, x_min = 1.0f; int x_max_k = -1, t_max_k = -1; for (k = 0; k < n; ++k) { - float xk = x->x[off+k], tk = t->x[off+k]; + float xk = x->x[off + k], tk = t->x[off + k]; t_sum += tk; - t_min = t_min < tk? t_min : tk; - x_min = x_min < xk? x_min : xk; + t_min = t_min < tk ? t_min : tk; + x_min = x_min < xk ? x_min : xk; if (t_max < tk) t_max = tk, t_max_k = k; if (x_max < xk) x_max = xk, x_max_k = k; } @@ -283,7 +307,7 @@ typedef struct mtaux_t { /* cross-worker data */ static void *mt_worker(void *data) /* pthread worker */ { - mtaux1_t *mt1 = (mtaux1_t*)data; + mtaux1_t *mt1 = (mtaux1_t *) data; mtaux_t *mt = mt1->g; for (;;) { int action; @@ -297,7 +321,8 @@ static void *mt_worker(void *data) /* pthread worker */ if (action == -1) break; if (mt->eval_out) kann_eval(mt1->a, KANN_F_OUT, 0); - else mt1->cost = kann_cost_core(mt1->a, mt->cost_label, mt->cal_grad); + else + mt1->cost = kann_cost_core(mt1->a, mt->cost_label, mt->cal_grad); } pthread_exit(0); } @@ -324,18 +349,18 @@ void kann_mt(kann_t *ann, int n_threads, int max_batch_size) int i, k; if (n_threads <= 1) { - if (ann->mt) mt_destroy((mtaux_t*)ann->mt); + if (ann->mt) mt_destroy((mtaux_t *) ann->mt); ann->mt = 0; return; } if (n_threads > max_batch_size) n_threads = max_batch_size; if (n_threads <= 1) return; - mt = (mtaux_t*)calloc(1, sizeof(mtaux_t)); + mt = (mtaux_t *) calloc(1, sizeof(mtaux_t)); mt->n_threads = n_threads, mt->max_batch_size = max_batch_size; pthread_mutex_init(&mt->mtx, 0); pthread_cond_init(&mt->cv, 0); - mt->mt = (mtaux1_t*)calloc(n_threads, sizeof(mtaux1_t)); + mt->mt = (mtaux1_t *) calloc(n_threads, sizeof(mtaux1_t)); for (i = k = 0; i < n_threads; ++i) { int size = (max_batch_size - k) / (n_threads - i); mt->mt[i].a = kann_clone(ann, size); @@ -350,11 +375,11 @@ void kann_mt(kann_t *ann, int n_threads, int max_batch_size) static void mt_kickoff(kann_t *a, int cost_label, int cal_grad, int eval_out) { - mtaux_t *mt = (mtaux_t*)a->mt; + mtaux_t *mt = (mtaux_t *) a->mt; int i, j, k, B, n_var; B = kad_sync_dim(a->n, a->v, -1); /* get the current batch size */ - assert(B <= mt->max_batch_size); /* TODO: can be relaxed */ + assert(B <= mt->max_batch_size); /* TODO: can be relaxed */ n_var = kann_size_var(a); pthread_mutex_lock(&mt->mtx); @@ -376,7 +401,7 @@ static void mt_kickoff(kann_t *a, int cost_label, int cal_grad, int eval_out) float kann_cost(kann_t *a, int cost_label, int cal_grad) { - mtaux_t *mt = (mtaux_t*)a->mt; + mtaux_t *mt = (mtaux_t *) a->mt; int i, j, B, k, n_var; float cost; @@ -392,7 +417,7 @@ float kann_cost(kann_t *a, int cost_label, int cal_grad) for (i = k = 0, cost = 0.0f; i < mt->n_threads; ++i) { int size = (B - k) / (mt->n_threads - i); cost += mt->mt[i].cost * size / B; - kad_saxpy(n_var, (float)size / B, mt->mt[i].a->g, a->g); + kad_saxpy(n_var, (float) size / B, mt->mt[i].a->g, a->g); k += size; } for (j = 0; j < a->n; ++j) { /* copy values back at recurrent nodes (needed by textgen; TODO: temporary solution) */ @@ -410,14 +435,14 @@ float kann_cost(kann_t *a, int cost_label, int cal_grad) int kann_eval_out(kann_t *a) { - mtaux_t *mt = (mtaux_t*)a->mt; + mtaux_t *mt = (mtaux_t *) a->mt; int j, B, n_eval; if (mt == 0) return kann_eval(a, KANN_F_OUT, 0); B = kad_sync_dim(a->n, a->v, -1); /* get the current batch size */ mt_kickoff(a, 0, 0, 1); n_eval = kann_eval(mt->mt[0].a, KANN_F_OUT, 0); while (mt->n_idle < mt->n_threads - 1); /* busy waiting until all threads in sync */ - for (j = 0; j < a->n; ++j) { /* copy output values back */ + for (j = 0; j < a->n; ++j) { /* copy output values back */ kad_node_t *p = a->v[j]; if (p->ext_flag & KANN_F_OUT) { int i, t, k, d0 = p->d[0] / B, d1 = 1; /* for RNN, p->d[0] may equal unroll_len * batch_size */ @@ -438,7 +463,7 @@ int kann_eval_out(kann_t *a) int kann_class_error(const kann_t *ann, int *base) { - mtaux_t *mt = (mtaux_t*)ann->mt; + mtaux_t *mt = (mtaux_t *) ann->mt; int i, n_err = 0, b = 0; if (mt == 0) return kann_class_error_core(ann, base); for (i = 0; i < mt->n_threads; ++i) { @@ -450,7 +475,7 @@ int kann_class_error(const kann_t *ann, int *base) void kann_switch(kann_t *ann, int is_train) { - mtaux_t *mt = (mtaux_t*)ann->mt; + mtaux_t *mt = (mtaux_t *) ann->mt; int i; if (mt == 0) { kann_switch_core(ann, is_train); @@ -460,11 +485,25 @@ void kann_switch(kann_t *ann, int is_train) kann_switch_core(mt->mt[i].a, is_train); } #else -void kann_mt(kann_t *ann, int n_threads, int max_batch_size) {} -float kann_cost(kann_t *a, int cost_label, int cal_grad) { return kann_cost_core(a, cost_label, cal_grad); } -int kann_eval_out(kann_t *a) { return kann_eval(a, KANN_F_OUT, 0); } -int kann_class_error(const kann_t *a, int *base) { return kann_class_error_core(a, base); } -void kann_switch(kann_t *ann, int is_train) { return kann_switch_core(ann, is_train); } +void kann_mt(kann_t *ann, int n_threads, int max_batch_size) +{ +} +float kann_cost(kann_t *a, int cost_label, int cal_grad) +{ + return kann_cost_core(a, cost_label, cal_grad); +} +int kann_eval_out(kann_t *a) +{ + return kann_eval(a, KANN_F_OUT, 0); +} +int kann_class_error(const kann_t *a, int *base) +{ + return kann_class_error_core(a, base); +} +void kann_switch(kann_t *ann, int is_train) +{ + return kann_switch_core(ann, is_train); +} #endif /*********************** @@ -485,7 +524,7 @@ void kann_save_fp(FILE *fp, kann_t *ann) void kann_save(const char *fn, kann_t *ann) { FILE *fp; - fp = fn && strcmp(fn, "-")? fopen(fn, "wb") : stdout; + fp = fn && strcmp(fn, "-") ? fopen(fn, "wb") : stdout; kann_save_fp(fp, ann); fclose(fp); } @@ -500,13 +539,13 @@ kann_t *kann_load_fp(FILE *fp) if (strncmp(magic, KANN_MAGIC, 4) != 0) { return 0; } - ann = (kann_t*)calloc(1, sizeof(kann_t)); + ann = (kann_t *) calloc(1, sizeof(kann_t)); ann->v = kad_load(fp, &ann->n); n_var = kad_size_var(ann->n, ann->v); n_const = kad_size_const(ann->n, ann->v); - ann->x = (float*)malloc(n_var * sizeof(float)); - ann->g = (float*)calloc(n_var, sizeof(float)); - ann->c = (float*)malloc(n_const * sizeof(float)); + ann->x = (float *) malloc(n_var * sizeof(float)); + ann->g = (float *) calloc(n_var, sizeof(float)); + ann->c = (float *) malloc(n_const * sizeof(float)); (void) !fread(ann->x, sizeof(float), n_var, fp); (void) !fread(ann->c, sizeof(float), n_const, fp); kad_ext_sync(ann->n, ann->v, ann->x, ann->g, ann->c); @@ -517,7 +556,7 @@ kann_t *kann_load(const char *fn) { FILE *fp; kann_t *ann; - fp = fn && strcmp(fn, "-")? fopen(fn, "rb") : stdin; + fp = fn && strcmp(fn, "-") ? fopen(fn, "rb") : stdin; ann = kann_load_fp(fp); fclose(fp); return ann; @@ -531,23 +570,24 @@ kann_t *kann_load(const char *fn) kad_node_t *kann_new_leaf_array(int *offset, kad_node_p *par, uint8_t flag, float x0_01, int n_d, int32_t d[KAD_MAX_DIM]) { - int i, len, off = offset && par? *offset : -1; + int i, len, off = offset && par ? *offset : -1; kad_node_t *p; if (off >= 0 && par[off]) return par[(*offset)++]; - p = (kad_node_t*)calloc(1, sizeof(kad_node_t)); + p = (kad_node_t *) calloc(1, sizeof(kad_node_t)); p->n_d = n_d, p->flag = flag; memcpy(p->d, d, n_d * sizeof(int32_t)); len = kad_len(p); - p->x = (float*)calloc(len, sizeof(float)); + p->x = (float *) calloc(len, sizeof(float)); if (p->n_d <= 1) { for (i = 0; i < len; ++i) p->x[i] = x0_01; - } else { + } + else { double sdev_inv; - sdev_inv = 1.0 / sqrt((double)len / p->d[0]); + sdev_inv = 1.0 / sqrt((double) len / p->d[0]); for (i = 0; i < len; ++i) - p->x[i] = (float)(kad_drand_normal(0) * sdev_inv); + p->x[i] = (float) (kad_drand_normal(0) * sdev_inv); } if (off >= 0) par[off] = p, ++(*offset); return p; @@ -557,7 +597,9 @@ kad_node_t *kann_new_leaf2(int *offset, kad_node_p *par, uint8_t flag, float x0_ { int32_t i, d[KAD_MAX_DIM]; va_list ap; - va_start(ap, n_d); for (i = 0; i < n_d; ++i) d[i] = va_arg(ap, int); va_end(ap); + va_start(ap, n_d); + for (i = 0; i < n_d; ++i) d[i] = va_arg(ap, int); + va_end(ap); return kann_new_leaf_array(offset, par, flag, x0_01, n_d, d); } @@ -565,7 +607,7 @@ kad_node_t *kann_layer_dense2(int *offset, kad_node_p *par, kad_node_t *in, int { int n0; kad_node_t *w, *b; - n0 = in->n_d >= 2? kad_len(in) / in->d[0] : kad_len(in); + n0 = in->n_d >= 2 ? kad_len(in) / in->d[0] : kad_len(in); w = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 2, n1, n0); b = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 1, n1); return kad_add(kad_cmul(in, w), b); @@ -583,27 +625,27 @@ kad_node_t *kann_layer_layernorm2(int *offset, kad_node_t **par, kad_node_t *in) { int n0; kad_node_t *alpha, *beta; - n0 = in->n_d >= 2? kad_len(in) / in->d[0] : kad_len(in); + n0 = in->n_d >= 2 ? kad_len(in) / in->d[0] : kad_len(in); alpha = kann_new_leaf2(offset, par, KAD_VAR, 1.0f, 1, n0); - beta = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 1, n0); + beta = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 1, n0); return kad_add(kad_mul(kad_stdnorm(in), alpha), beta); } static inline kad_node_t *cmul_norm2(int *offset, kad_node_t **par, kad_node_t *x, kad_node_t *w, int use_norm) { - return use_norm? kann_layer_layernorm2(offset, par, kad_cmul(x, w)) : kad_cmul(x, w); + return use_norm ? kann_layer_layernorm2(offset, par, kad_cmul(x, w)) : kad_cmul(x, w); } kad_node_t *kann_layer_rnn2(int *offset, kad_node_t **par, kad_node_t *in, kad_node_t *h0, int rnn_flag) { - int n0, n1 = h0->d[h0->n_d-1], use_norm = !!(rnn_flag & KANN_RNN_NORM); + int n0, n1 = h0->d[h0->n_d - 1], use_norm = !!(rnn_flag & KANN_RNN_NORM); kad_node_t *t, *w, *u, *b, *out; u = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 2, n1, n1); b = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 1, n1); t = cmul_norm2(offset, par, h0, u, use_norm); if (in) { - n0 = in->n_d >= 2? kad_len(in) / in->d[0] : kad_len(in); + n0 = in->n_d >= 2 ? kad_len(in) / in->d[0] : kad_len(in); w = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 2, n1, n0); t = kad_add(cmul_norm2(offset, par, in, w, use_norm), t); } @@ -614,10 +656,10 @@ kad_node_t *kann_layer_rnn2(int *offset, kad_node_t **par, kad_node_t *in, kad_n kad_node_t *kann_layer_gru2(int *offset, kad_node_t **par, kad_node_t *in, kad_node_t *h0, int rnn_flag) { - int n0 = 0, n1 = h0->d[h0->n_d-1], use_norm = !!(rnn_flag & KANN_RNN_NORM); + int n0 = 0, n1 = h0->d[h0->n_d - 1], use_norm = !!(rnn_flag & KANN_RNN_NORM); kad_node_t *t, *r, *z, *w, *u, *b, *s, *out; - if (in) n0 = in->n_d >= 2? kad_len(in) / in->d[0] : kad_len(in); + if (in) n0 = in->n_d >= 2 ? kad_len(in) / in->d[0] : kad_len(in); /* z = sigm(x_t * W_z + h_{t-1} * U_z + b_z) */ u = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 2, n1, n1); b = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 1, n1); @@ -657,16 +699,36 @@ kad_node_t *kann_new_leaf(uint8_t flag, float x0_01, int n_d, ...) { int32_t i, d[KAD_MAX_DIM]; va_list ap; - va_start(ap, n_d); for (i = 0; i < n_d; ++i) d[i] = va_arg(ap, int); va_end(ap); + va_start(ap, n_d); + for (i = 0; i < n_d; ++i) d[i] = va_arg(ap, int); + va_end(ap); return kann_new_leaf_array(0, 0, flag, x0_01, n_d, d); } -kad_node_t *kann_new_scalar(uint8_t flag, float x) { return kann_new_leaf(flag, x, 0); } -kad_node_t *kann_new_weight(int n_row, int n_col) { return kann_new_leaf(KAD_VAR, 0.0f, 2, n_row, n_col); } -kad_node_t *kann_new_vec(int n, float x) { return kann_new_leaf(KAD_VAR, x, 1, n); } -kad_node_t *kann_new_bias(int n) { return kann_new_vec(n, 0.0f); } -kad_node_t *kann_new_weight_conv2d(int n_out, int n_in, int k_row, int k_col) { return kann_new_leaf(KAD_VAR, 0.0f, 4, n_out, n_in, k_row, k_col); } -kad_node_t *kann_new_weight_conv1d(int n_out, int n_in, int kernel_len) { return kann_new_leaf(KAD_VAR, 0.0f, 3, n_out, n_in, kernel_len); } +kad_node_t *kann_new_scalar(uint8_t flag, float x) +{ + return kann_new_leaf(flag, x, 0); +} +kad_node_t *kann_new_weight(int n_row, int n_col) +{ + return kann_new_leaf(KAD_VAR, 0.0f, 2, n_row, n_col); +} +kad_node_t *kann_new_vec(int n, float x) +{ + return kann_new_leaf(KAD_VAR, x, 1, n); +} +kad_node_t *kann_new_bias(int n) +{ + return kann_new_vec(n, 0.0f); +} +kad_node_t *kann_new_weight_conv2d(int n_out, int n_in, int k_row, int k_col) +{ + return kann_new_leaf(KAD_VAR, 0.0f, 4, n_out, n_in, k_row, k_col); +} +kad_node_t *kann_new_weight_conv1d(int n_out, int n_in, int kernel_len) +{ + return kann_new_leaf(KAD_VAR, 0.0f, 3, n_out, n_in, kernel_len); +} kad_node_t *kann_layer_input(int n1) { @@ -676,23 +738,32 @@ kad_node_t *kann_layer_input(int n1) return t; } -kad_node_t *kann_layer_dense(kad_node_t *in, int n1) { return kann_layer_dense2(0, 0, in, n1); } -kad_node_t *kann_layer_dropout(kad_node_t *t, float r) { return kann_layer_dropout2(0, 0, t, r); } -kad_node_t *kann_layer_layernorm(kad_node_t *in) { return kann_layer_layernorm2(0, 0, in); } +kad_node_t *kann_layer_dense(kad_node_t *in, int n1) +{ + return kann_layer_dense2(0, 0, in, n1); +} +kad_node_t *kann_layer_dropout(kad_node_t *t, float r) +{ + return kann_layer_dropout2(0, 0, t, r); +} +kad_node_t *kann_layer_layernorm(kad_node_t *in) +{ + return kann_layer_layernorm2(0, 0, in); +} kad_node_t *kann_layer_rnn(kad_node_t *in, int n1, int rnn_flag) { kad_node_t *h0; - h0 = (rnn_flag & KANN_RNN_VAR_H0)? kad_var(0, 0, 2, 1, n1) : kad_const(0, 2, 1, n1); - h0->x = (float*)calloc(n1, sizeof(float)); + h0 = (rnn_flag & KANN_RNN_VAR_H0) ? kad_var(0, 0, 2, 1, n1) : kad_const(0, 2, 1, n1); + h0->x = (float *) calloc(n1, sizeof(float)); return kann_layer_rnn2(0, 0, in, h0, rnn_flag); } kad_node_t *kann_layer_gru(kad_node_t *in, int n1, int rnn_flag) { kad_node_t *h0; - h0 = (rnn_flag & KANN_RNN_VAR_H0)? kad_var(0, 0, 2, 1, n1) : kad_const(0, 2, 1, n1); - h0->x = (float*)calloc(n1, sizeof(float)); + h0 = (rnn_flag & KANN_RNN_VAR_H0) ? kad_var(0, 0, 2, 1, n1) : kad_const(0, 2, 1, n1); + h0->x = (float *) calloc(n1, sizeof(float)); return kann_layer_gru2(0, 0, in, h0, rnn_flag); } @@ -705,13 +776,13 @@ kad_node_t *kann_layer_lstm(kad_node_t *in, int n1, int rnn_flag) { int n0; kad_node_t *i, *f, *o, *g, *w, *u, *b, *h0, *c0, *c, *out; - kad_node_t *(*cmul)(kad_node_t*, kad_node_t*) = (rnn_flag & KANN_RNN_NORM)? kann_cmul_norm : kad_cmul; + kad_node_t *(*cmul)(kad_node_t *, kad_node_t *) = (rnn_flag & KANN_RNN_NORM) ? kann_cmul_norm : kad_cmul; - n0 = in->n_d >= 2? kad_len(in) / in->d[0] : kad_len(in); - h0 = (rnn_flag & KANN_RNN_VAR_H0)? kad_var(0, 0, 2, 1, n1) : kad_const(0, 2, 1, n1); - h0->x = (float*)calloc(n1, sizeof(float)); - c0 = (rnn_flag & KANN_RNN_VAR_H0)? kad_var(0, 0, 2, 1, n1) : kad_const(0, 2, 1, n1); - c0->x = (float*)calloc(n1, sizeof(float)); + n0 = in->n_d >= 2 ? kad_len(in) / in->d[0] : kad_len(in); + h0 = (rnn_flag & KANN_RNN_VAR_H0) ? kad_var(0, 0, 2, 1, n1) : kad_const(0, 2, 1, n1); + h0->x = (float *) calloc(n1, sizeof(float)); + c0 = (rnn_flag & KANN_RNN_VAR_H0) ? kad_var(0, 0, 2, 1, n1) : kad_const(0, 2, 1, n1); + c0->x = (float *) calloc(n1, sizeof(float)); /* i = sigm(x_t * W_i + h_{t-1} * U_i + b_i) */ w = kann_new_weight(n1, n0); @@ -766,18 +837,21 @@ kad_node_t *kann_layer_cost(kad_node_t *t, int n_out, int cost_type) if (cost_type == KANN_C_MSE) { cost = kad_mse(t, truth); - } else if (cost_type == KANN_C_CEB) { + } + else if (cost_type == KANN_C_CEB) { t = kad_sigm(t); cost = kad_ce_bin(t, truth); - } else if (cost_type == KANN_C_CEB_NEG) { + } + else if (cost_type == KANN_C_CEB_NEG) { t = kad_tanh(t); cost = kad_ce_bin_neg(t, truth); - } else if (cost_type == KANN_C_CEM) { + } + else if (cost_type == KANN_C_CEM) { t = kad_softmax(t); cost = kad_ce_multi(t, truth); } else { - assert (0); + assert(0); } t->ext_flag |= KANN_F_OUT; @@ -791,8 +865,8 @@ void kann_shuffle(int n, int *s) int i, j, t; for (i = 0; i < n; ++i) s[i] = i; for (i = n; i > 0; --i) { - j = (int)(i * kad_drand(0)); - t = s[j], s[j] = s[i-1], s[i-1] = t; + j = (int) (i * kad_drand(0)); + t = s[j], s[j] = s[i - 1], s[i - 1] = t; } } @@ -805,7 +879,7 @@ void kann_shuffle(int n, int *s) void kann_RMSprop(int n, float h0, const float *h, float decay, const float *g, float *t, float *r) { - int i, n4 = n>>2<<2; + int i, n4 = n >> 2 << 2; __m128 vh, vg, vr, vt, vd, vd1, tmp, vtiny; vh = _mm_set1_ps(h0); vd = _mm_set1_ps(decay); @@ -823,7 +897,7 @@ void kann_RMSprop(int n, float h0, const float *h, float decay, const float *g, } for (; i < n; ++i) { r[i] = (1. - decay) * g[i] * g[i] + decay * r[i]; - t[i] -= (h? h[i] : h0) / sqrtf(1e-6f + r[i]) * g[i]; + t[i] -= (h ? h[i] : h0) / sqrtf(1e-6f + r[i]) * g[i]; } } #else @@ -831,7 +905,7 @@ void kann_RMSprop(int n, float h0, const float *h, float decay, const float *g, { int i; for (i = 0; i < n; ++i) { - float lr = h? h[i] : h0; + float lr = h ? h[i] : h0; r[i] = (1.0f - decay) * g[i] * g[i] + decay * r[i]; t[i] -= lr / sqrtf(1e-6f + r[i]) * g[i]; } @@ -847,8 +921,8 @@ float kann_grad_clip(float thres, int n, float *g) s2 = sqrt(s2); if (s2 > thres) for (i = 0, s2 = 1.0 / s2; i < n; ++i) - g[i] *= (float)s2; - return (float)s2 / thres; + g[i] *= (float) s2; + return (float) s2 / thres; } /**************************************************************** @@ -868,21 +942,21 @@ int kann_train_fnn1(kann_t *ann, float lr, int mini_size, int max_epoch, if (n_in < 0 || n_out < 0) return -1; n_var = kann_size_var(ann); n_const = kann_size_const(ann); - r = (float*)calloc(n_var, sizeof(float)); - shuf = (int*)malloc(n * sizeof(int)); - x = (float**)malloc(n * sizeof(float*)); - y = (float**)malloc(n * sizeof(float*)); + r = (float *) calloc(n_var, sizeof(float)); + shuf = (int *) malloc(n * sizeof(int)); + x = (float **) malloc(n * sizeof(float *)); + y = (float **) malloc(n * sizeof(float *)); kann_shuffle(n, shuf); for (j = 0; j < n; ++j) x[j] = _x[shuf[j]], y[j] = _y[shuf[j]]; - n_val = (int)(n * frac_val); + n_val = (int) (n * frac_val); n_train = n - n_val; - min_x = (float*)malloc(n_var * sizeof(float)); - min_c = (float*)malloc(n_const * sizeof(float)); + min_x = (float *) malloc(n_var * sizeof(float)); + min_c = (float *) malloc(n_const * sizeof(float)); - x1 = (float*)malloc(n_in * mini_size * sizeof(float)); - y1 = (float*)malloc(n_out * mini_size * sizeof(float)); - kann_feed_bind(ann, KANN_F_IN, 0, &x1); + x1 = (float *) malloc(n_in * mini_size * sizeof(float)); + y1 = (float *) malloc(n_out * mini_size * sizeof(float)); + kann_feed_bind(ann, KANN_F_IN, 0, &x1); kann_feed_bind(ann, KANN_F_TRUTH, 0, &y1); for (i = 0; i < max_epoch; ++i) { @@ -891,10 +965,10 @@ int kann_train_fnn1(kann_t *ann, float lr, int mini_size, int max_epoch, kann_shuffle(n_train, shuf); kann_switch(ann, 1); while (n_proc < n_train) { - int b, c, ms = n_train - n_proc < mini_size? n_train - n_proc : mini_size; + int b, c, ms = n_train - n_proc < mini_size ? n_train - n_proc : mini_size; for (b = 0; b < ms; ++b) { - memcpy(&x1[b*n_in], x[shuf[n_proc+b]], n_in * sizeof(float)); - memcpy(&y1[b*n_out], y[shuf[n_proc+b]], n_out * sizeof(float)); + memcpy(&x1[b * n_in], x[shuf[n_proc + b]], n_in * sizeof(float)); + memcpy(&y1[b * n_out], y[shuf[n_proc + b]], n_out * sizeof(float)); } kann_set_batch_size(ann, ms); train_cost += kann_cost(ann, 0, 1) * ms; @@ -907,10 +981,10 @@ int kann_train_fnn1(kann_t *ann, float lr, int mini_size, int max_epoch, kann_switch(ann, 0); n_proc = 0; while (n_proc < n_val) { - int b, c, ms = n_val - n_proc < mini_size? n_val - n_proc : mini_size; + int b, c, ms = n_val - n_proc < mini_size ? n_val - n_proc : mini_size; for (b = 0; b < ms; ++b) { - memcpy(&x1[b*n_in], x[n_train+n_proc+b], n_in * sizeof(float)); - memcpy(&y1[b*n_out], y[n_train+n_proc+b], n_out * sizeof(float)); + memcpy(&x1[b * n_in], x[n_train + n_proc + b], n_in * sizeof(float)); + memcpy(&y1[b * n_out], y[n_train + n_proc + b], n_out * sizeof(float)); } kann_set_batch_size(ann, ms); val_cost += kann_cost(ann, 0, 0) * ms; @@ -919,6 +993,8 @@ int kann_train_fnn1(kann_t *ann, float lr, int mini_size, int max_epoch, n_proc += ms; } if (n_val > 0) val_cost /= n_val; + (void) (n_train_err); + (void) (n_val_err); if (cb) { cb(i + 1, train_cost, val_cost, ud); #if 0 @@ -937,8 +1013,9 @@ int kann_train_fnn1(kann_t *ann, float lr, int mini_size, int max_epoch, memcpy(min_x, ann->x, n_var * sizeof(float)); memcpy(min_c, ann->c, n_const * sizeof(float)); drop_streak = 0; - min_val_cost = (float)val_cost; - } else if (++drop_streak >= max_drop_streak) + min_val_cost = (float) val_cost; + } + else if (++drop_streak >= max_drop_streak) break; } } @@ -947,13 +1024,20 @@ int kann_train_fnn1(kann_t *ann, float lr, int mini_size, int max_epoch, memcpy(ann->c, min_c, n_const * sizeof(float)); } - free(min_c); free(min_x); free(y1); free(x1); free(y); free(x); free(shuf); free(r); + free(min_c); + free(min_x); + free(y1); + free(x1); + free(y); + free(x); + free(shuf); + free(r); return i; } float kann_cost_fnn1(kann_t *ann, int n, float **x, float **y) { - int n_in, n_out, n_proc = 0, mini_size = 64 < n? 64 : n; + int n_in, n_out, n_proc = 0, mini_size = 64 < n ? 64 : n; float *x1, *y1; double cost = 0.0; @@ -961,23 +1045,24 @@ float kann_cost_fnn1(kann_t *ann, int n, float **x, float **y) n_out = kann_dim_out(ann); if (n <= 0 || n_in < 0 || n_out < 0) return 0.0; - x1 = (float*)malloc(n_in * mini_size * sizeof(float)); - y1 = (float*)malloc(n_out * mini_size * sizeof(float)); - kann_feed_bind(ann, KANN_F_IN, 0, &x1); + x1 = (float *) malloc(n_in * mini_size * sizeof(float)); + y1 = (float *) malloc(n_out * mini_size * sizeof(float)); + kann_feed_bind(ann, KANN_F_IN, 0, &x1); kann_feed_bind(ann, KANN_F_TRUTH, 0, &y1); kann_switch(ann, 0); while (n_proc < n) { - int b, ms = n - n_proc < mini_size? n - n_proc : mini_size; + int b, ms = n - n_proc < mini_size ? n - n_proc : mini_size; for (b = 0; b < ms; ++b) { - memcpy(&x1[b*n_in], x[n_proc+b], n_in * sizeof(float)); - memcpy(&y1[b*n_out], y[n_proc+b], n_out * sizeof(float)); + memcpy(&x1[b * n_in], x[n_proc + b], n_in * sizeof(float)); + memcpy(&y1[b * n_out], y[n_proc + b], n_out * sizeof(float)); } kann_set_batch_size(ann, ms); cost += kann_cost(ann, 0, 0) * ms; n_proc += ms; } - free(y1); free(x1); - return (float)(cost / n); + free(y1); + free(x1); + return (float) (cost / n); } const float *kann_apply1(kann_t *a, float *x) diff --git a/contrib/zstd/CMakeLists.txt b/contrib/zstd/CMakeLists.txt index 4601e53ce..9fba28fa4 100644 --- a/contrib/zstd/CMakeLists.txt +++ b/contrib/zstd/CMakeLists.txt @@ -24,4 +24,4 @@ SET(ZSTDSRC zstd_opt.c) ADD_LIBRARY(rspamd-zstd STATIC ${ZSTDSRC}) -ADD_DEFINITIONS(-DZSTD_DISABLE_ASM) \ No newline at end of file +ADD_DEFINITIONS(-DZSTD_DISABLE_ASM -DZSTD_DISABLE_DEPRECATE_WARNINGS) \ No newline at end of file -- 2.39.5