diff options
Diffstat (limited to 'src/tokenizers/tokenizers.h')
| -rw-r--r-- | src/tokenizers/tokenizers.h | 1 |
1 files changed, 1 insertions, 0 deletions
diff --git a/src/tokenizers/tokenizers.h b/src/tokenizers/tokenizers.h index fda5bded3..9a16e907c 100644 --- a/src/tokenizers/tokenizers.h +++ b/src/tokenizers/tokenizers.h @@ -18,6 +18,7 @@ typedef struct token_node_s { uint32_t h1; uint32_t h2; float value; + uintptr_t extra; } token_node_t; /* Common tokenizer structure */ |