diff options
author | heraklit256 <37872459+heraklit256@users.noreply.github.com> | 2018-06-09 07:58:21 +0000 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-06-09 07:58:21 +0000 |
commit | 728b3a44311cab0717fc1ef9d1576ed94f7118e4 (patch) | |
tree | 16f99393319ea9fd0cff9bb089cb20ce8eb4b80d | |
parent | 109730f3834c0aa7fedc8315406793b1d2a5f530 (diff) | |
parent | 85e5de3d81b8bf6ef7bf4592ff77d56cd59191c6 (diff) | |
download | rspamd-728b3a44311cab0717fc1ef9d1576ed94f7118e4.tar.gz rspamd-728b3a44311cab0717fc1ef9d1576ed94f7118e4.zip |
Merge pull request #2 from vstakhov/master
merge upstream into local master
-rw-r--r-- | CMakeLists.txt | 17 | ||||
-rw-r--r-- | clang-plugin/CMakeLists.txt | 2 | ||||
-rw-r--r-- | conf/composites.conf | 6 | ||||
-rw-r--r-- | conf/modules.d/ratelimit.conf | 4 | ||||
-rw-r--r-- | contrib/lc-btrie/btrie.c | 8 | ||||
-rw-r--r-- | contrib/lc-btrie/btrie.h | 2 | ||||
-rw-r--r-- | contrib/librdns/dns_private.h | 26 | ||||
-rw-r--r-- | contrib/librdns/rdns.h | 12 | ||||
-rw-r--r-- | contrib/librdns/resolver.c | 133 | ||||
-rw-r--r-- | contrib/librdns/util.c | 62 | ||||
-rw-r--r-- | contrib/lua-argparse/LICENSE | 20 | ||||
-rw-r--r-- | contrib/lua-argparse/argparse.lua | 1516 | ||||
-rw-r--r-- | contrib/lua-lpeg/CMakeLists.txt (renamed from contrib/lpeg/CMakeLists.txt) | 0 | ||||
-rw-r--r-- | contrib/lua-lpeg/LICENSE (renamed from contrib/lpeg/LICENSE) | 0 | ||||
-rw-r--r-- | contrib/lua-lpeg/lpcap.c (renamed from contrib/lpeg/lpcap.c) | 0 | ||||
-rw-r--r-- | contrib/lua-lpeg/lpcap.h (renamed from contrib/lpeg/lpcap.h) | 0 | ||||
-rw-r--r-- | contrib/lua-lpeg/lpcode.c (renamed from contrib/lpeg/lpcode.c) | 0 | ||||
-rw-r--r-- | contrib/lua-lpeg/lpcode.h (renamed from contrib/lpeg/lpcode.h) | 0 | ||||
-rw-r--r-- | contrib/lua-lpeg/lpprint.c (renamed from contrib/lpeg/lpprint.c) | 0 | ||||
-rw-r--r-- | contrib/lua-lpeg/lpprint.h (renamed from contrib/lpeg/lpprint.h) | 0 | ||||
-rw-r--r-- | contrib/lua-lpeg/lptree.c (renamed from contrib/lpeg/lptree.c) | 0 | ||||
-rw-r--r-- | contrib/lua-lpeg/lptree.h (renamed from contrib/lpeg/lptree.h) | 0 | ||||
-rw-r--r-- | contrib/lua-lpeg/lptypes.h (renamed from contrib/lpeg/lptypes.h) | 0 | ||||
-rw-r--r-- | contrib/lua-lpeg/lpvm.c (renamed from contrib/lpeg/lpvm.c) | 0 | ||||
-rw-r--r-- | contrib/lua-lpeg/lpvm.h (renamed from contrib/lpeg/lpvm.h) | 0 | ||||
-rw-r--r-- | contrib/lua-moses/LICENSE (renamed from contrib/moses/LICENSE) | 0 | ||||
-rw-r--r-- | contrib/lua-moses/moses.lua (renamed from contrib/moses/moses.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/CMakeLists.txt (renamed from contrib/torch/decisiontree/CMakeLists.txt) | 5 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/CartNode.lua (renamed from contrib/torch/decisiontree/CartNode.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/CartTrainer.lua (renamed from contrib/torch/decisiontree/CartTrainer.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/CartTree.lua (renamed from contrib/torch/decisiontree/CartTree.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/DFD.lua (renamed from contrib/torch/decisiontree/DFD.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/DataSet.lua (renamed from contrib/torch/decisiontree/DataSet.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/DecisionForest.lua (renamed from contrib/torch/decisiontree/DecisionForest.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/DecisionForestTrainer.lua (renamed from contrib/torch/decisiontree/DecisionForestTrainer.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/DecisionTree.lua (renamed from contrib/torch/decisiontree/DecisionTree.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/GBDT_common.h (renamed from contrib/torch/decisiontree/GBDT_common.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/GiniState.lua (renamed from contrib/torch/decisiontree/GiniState.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/GradientBoostState.lua (renamed from contrib/torch/decisiontree/GradientBoostState.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/GradientBoostTrainer.lua (renamed from contrib/torch/decisiontree/GradientBoostTrainer.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/LICENSE (renamed from contrib/torch/decisiontree/LICENSE) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/LogitBoostCriterion.lua (renamed from contrib/torch/decisiontree/LogitBoostCriterion.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/MSECriterion.lua (renamed from contrib/torch/decisiontree/MSECriterion.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/README.md (renamed from contrib/torch/decisiontree/README.md) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/RandomForestTrainer.lua (renamed from contrib/torch/decisiontree/RandomForestTrainer.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/Sparse2Dense.lua (renamed from contrib/torch/decisiontree/Sparse2Dense.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/SparseTensor.lua (renamed from contrib/torch/decisiontree/SparseTensor.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/TreeState.lua (renamed from contrib/torch/decisiontree/TreeState.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/WorkPool.lua (renamed from contrib/torch/decisiontree/WorkPool.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/_env.lua (renamed from contrib/torch/decisiontree/_env.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/benchmark.lua (renamed from contrib/torch/decisiontree/benchmark.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/doc/benchmark.md (renamed from contrib/torch/decisiontree/doc/benchmark.md) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/error.h (renamed from contrib/torch/decisiontree/error.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/generic/CartTree.c (renamed from contrib/torch/decisiontree/generic/CartTree.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/generic/DFD.c (renamed from contrib/torch/decisiontree/generic/DFD.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/generic/GBDT.c (renamed from contrib/torch/decisiontree/generic/GBDT.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/generic/GBDT_internal.c (renamed from contrib/torch/decisiontree/generic/GBDT_internal.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/generic/GBDT_internal.h (renamed from contrib/torch/decisiontree/generic/GBDT_internal.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/generic/LogitBoostCriterion.c (renamed from contrib/torch/decisiontree/generic/LogitBoostCriterion.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/generic/S2D.c (renamed from contrib/torch/decisiontree/generic/S2D.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/hash_map.c (renamed from contrib/torch/decisiontree/hash_map.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/hash_map.h (renamed from contrib/torch/decisiontree/hash_map.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/init.c (renamed from contrib/torch/decisiontree/init.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/init.lua (renamed from contrib/torch/decisiontree/init.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/internal_hash_map.h (renamed from contrib/torch/decisiontree/internal_hash_map.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/khash.h (renamed from contrib/torch/decisiontree/khash.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/math.lua (renamed from contrib/torch/decisiontree/math.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/rocks/decisiontree-scm-1.rockspec (renamed from contrib/torch/decisiontree/rocks/decisiontree-scm-1.rockspec) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/test.lua (renamed from contrib/torch/decisiontree/test.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/utils.h (renamed from contrib/torch/decisiontree/utils.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/decisiontree/utils.lua (renamed from contrib/torch/decisiontree/utils.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/.gitignore (renamed from contrib/torch/nn/.gitignore) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/.luacheckrc (renamed from contrib/torch/nn/.luacheckrc) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/.travis.yml (renamed from contrib/torch/nn/.travis.yml) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Abs.lua (renamed from contrib/torch/nn/Abs.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/AbsCriterion.lua (renamed from contrib/torch/nn/AbsCriterion.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Add.lua (renamed from contrib/torch/nn/Add.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/AddConstant.lua (renamed from contrib/torch/nn/AddConstant.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/BCECriterion.lua (renamed from contrib/torch/nn/BCECriterion.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/BatchNormalization.lua (renamed from contrib/torch/nn/BatchNormalization.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Bilinear.lua (renamed from contrib/torch/nn/Bilinear.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Bottle.lua (renamed from contrib/torch/nn/Bottle.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/CAdd.lua (renamed from contrib/torch/nn/CAdd.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/CAddTable.lua (renamed from contrib/torch/nn/CAddTable.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/CAddTensorTable.lua (renamed from contrib/torch/nn/CAddTensorTable.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/CDivTable.lua (renamed from contrib/torch/nn/CDivTable.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/CMakeLists.txt (renamed from contrib/torch/nn/CMakeLists.txt) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/CMaxTable.lua (renamed from contrib/torch/nn/CMaxTable.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/CMinTable.lua (renamed from contrib/torch/nn/CMinTable.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/CMul.lua (renamed from contrib/torch/nn/CMul.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/CMulTable.lua (renamed from contrib/torch/nn/CMulTable.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/CONTRIBUTING.md (renamed from contrib/torch/nn/CONTRIBUTING.md) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/COPYRIGHT.txt (renamed from contrib/torch/nn/COPYRIGHT.txt) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/CReLU.lua (renamed from contrib/torch/nn/CReLU.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/CSubTable.lua (renamed from contrib/torch/nn/CSubTable.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Clamp.lua (renamed from contrib/torch/nn/Clamp.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/ClassNLLCriterion.lua (renamed from contrib/torch/nn/ClassNLLCriterion.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/ClassSimplexCriterion.lua (renamed from contrib/torch/nn/ClassSimplexCriterion.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Collapse.lua (renamed from contrib/torch/nn/Collapse.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Concat.lua (renamed from contrib/torch/nn/Concat.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/ConcatTable.lua (renamed from contrib/torch/nn/ConcatTable.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Constant.lua (renamed from contrib/torch/nn/Constant.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Container.lua (renamed from contrib/torch/nn/Container.lua) | 0 | ||||
-rwxr-xr-x | contrib/lua-torch/nn/Contiguous.lua (renamed from contrib/torch/nn/Contiguous.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Convert.lua (renamed from contrib/torch/nn/Convert.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Copy.lua (renamed from contrib/torch/nn/Copy.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Cosine.lua (renamed from contrib/torch/nn/Cosine.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/CosineDistance.lua (renamed from contrib/torch/nn/CosineDistance.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/CosineEmbeddingCriterion.lua (renamed from contrib/torch/nn/CosineEmbeddingCriterion.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Criterion.lua (renamed from contrib/torch/nn/Criterion.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/CriterionTable.lua (renamed from contrib/torch/nn/CriterionTable.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/CrossEntropyCriterion.lua (renamed from contrib/torch/nn/CrossEntropyCriterion.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Decorator.lua (renamed from contrib/torch/nn/Decorator.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/DepthConcat.lua (renamed from contrib/torch/nn/DepthConcat.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/DistKLDivCriterion.lua (renamed from contrib/torch/nn/DistKLDivCriterion.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/DistanceRatioCriterion.lua (renamed from contrib/torch/nn/DistanceRatioCriterion.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/DontCast.lua (renamed from contrib/torch/nn/DontCast.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/DotProduct.lua (renamed from contrib/torch/nn/DotProduct.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Dropout.lua (renamed from contrib/torch/nn/Dropout.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/ELU.lua (renamed from contrib/torch/nn/ELU.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/ErrorMessages.lua (renamed from contrib/torch/nn/ErrorMessages.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Euclidean.lua (renamed from contrib/torch/nn/Euclidean.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Exp.lua (renamed from contrib/torch/nn/Exp.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/FlattenTable.lua (renamed from contrib/torch/nn/FlattenTable.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/GPU.lua (renamed from contrib/torch/nn/GPU.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/GatedLinearUnit.lua (renamed from contrib/torch/nn/GatedLinearUnit.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/GradientReversal.lua (renamed from contrib/torch/nn/GradientReversal.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/HardShrink.lua (renamed from contrib/torch/nn/HardShrink.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/HardTanh.lua (renamed from contrib/torch/nn/HardTanh.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/HingeEmbeddingCriterion.lua (renamed from contrib/torch/nn/HingeEmbeddingCriterion.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Identity.lua (renamed from contrib/torch/nn/Identity.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Index.lua (renamed from contrib/torch/nn/Index.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/IndexLinear.lua (renamed from contrib/torch/nn/IndexLinear.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Jacobian.lua (renamed from contrib/torch/nn/Jacobian.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/JoinTable.lua (renamed from contrib/torch/nn/JoinTable.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Kmeans.lua (renamed from contrib/torch/nn/Kmeans.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/L1Cost.lua (renamed from contrib/torch/nn/L1Cost.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/L1HingeEmbeddingCriterion.lua (renamed from contrib/torch/nn/L1HingeEmbeddingCriterion.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/L1Penalty.lua (renamed from contrib/torch/nn/L1Penalty.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/LayerNormalization.lua (renamed from contrib/torch/nn/LayerNormalization.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/LeakyReLU.lua (renamed from contrib/torch/nn/LeakyReLU.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Linear.lua (renamed from contrib/torch/nn/Linear.lua) | 0 | ||||
-rwxr-xr-x | contrib/lua-torch/nn/LinearWeightNorm.lua (renamed from contrib/torch/nn/LinearWeightNorm.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Log.lua (renamed from contrib/torch/nn/Log.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/LogSigmoid.lua (renamed from contrib/torch/nn/LogSigmoid.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/LogSoftMax.lua (renamed from contrib/torch/nn/LogSoftMax.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/LookupTable.lua (renamed from contrib/torch/nn/LookupTable.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/MM.lua (renamed from contrib/torch/nn/MM.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/MSECriterion.lua (renamed from contrib/torch/nn/MSECriterion.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/MV.lua (renamed from contrib/torch/nn/MV.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/MapTable.lua (renamed from contrib/torch/nn/MapTable.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/MarginCriterion.lua (renamed from contrib/torch/nn/MarginCriterion.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/MarginRankingCriterion.lua (renamed from contrib/torch/nn/MarginRankingCriterion.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/MaskedSelect.lua (renamed from contrib/torch/nn/MaskedSelect.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Max.lua (renamed from contrib/torch/nn/Max.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Maxout.lua (renamed from contrib/torch/nn/Maxout.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Mean.lua (renamed from contrib/torch/nn/Mean.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Min.lua (renamed from contrib/torch/nn/Min.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/MixtureTable.lua (renamed from contrib/torch/nn/MixtureTable.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Module.lua (renamed from contrib/torch/nn/Module.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/ModuleCriterion.lua (renamed from contrib/torch/nn/ModuleCriterion.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Mul.lua (renamed from contrib/torch/nn/Mul.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/MulConstant.lua (renamed from contrib/torch/nn/MulConstant.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/MultiCriterion.lua (renamed from contrib/torch/nn/MultiCriterion.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/MultiLabelMarginCriterion.lua (renamed from contrib/torch/nn/MultiLabelMarginCriterion.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/MultiLabelSoftMarginCriterion.lua (renamed from contrib/torch/nn/MultiLabelSoftMarginCriterion.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/MultiMarginCriterion.lua (renamed from contrib/torch/nn/MultiMarginCriterion.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/NaN.lua (renamed from contrib/torch/nn/NaN.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Narrow.lua (renamed from contrib/torch/nn/Narrow.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/NarrowTable.lua (renamed from contrib/torch/nn/NarrowTable.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Normalize.lua (renamed from contrib/torch/nn/Normalize.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/OneHot.lua (renamed from contrib/torch/nn/OneHot.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/PReLU.lua (renamed from contrib/torch/nn/PReLU.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Padding.lua (renamed from contrib/torch/nn/Padding.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/PairwiseDistance.lua (renamed from contrib/torch/nn/PairwiseDistance.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Parallel.lua (renamed from contrib/torch/nn/Parallel.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/ParallelCriterion.lua (renamed from contrib/torch/nn/ParallelCriterion.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/ParallelTable.lua (renamed from contrib/torch/nn/ParallelTable.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/PartialLinear.lua (renamed from contrib/torch/nn/PartialLinear.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/PixelShuffle.lua (renamed from contrib/torch/nn/PixelShuffle.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Power.lua (renamed from contrib/torch/nn/Power.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/PrintSize.lua (renamed from contrib/torch/nn/PrintSize.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Profile.lua (renamed from contrib/torch/nn/Profile.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/README.md (renamed from contrib/torch/nn/README.md) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/RReLU.lua (renamed from contrib/torch/nn/RReLU.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/ReLU.lua (renamed from contrib/torch/nn/ReLU.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/ReLU6.lua (renamed from contrib/torch/nn/ReLU6.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Replicate.lua (renamed from contrib/torch/nn/Replicate.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Reshape.lua (renamed from contrib/torch/nn/Reshape.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Select.lua (renamed from contrib/torch/nn/Select.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SelectTable.lua (renamed from contrib/torch/nn/SelectTable.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Sequential.lua (renamed from contrib/torch/nn/Sequential.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Sigmoid.lua (renamed from contrib/torch/nn/Sigmoid.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SmoothL1Criterion.lua (renamed from contrib/torch/nn/SmoothL1Criterion.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SoftMarginCriterion.lua (renamed from contrib/torch/nn/SoftMarginCriterion.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SoftMax.lua (renamed from contrib/torch/nn/SoftMax.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SoftMin.lua (renamed from contrib/torch/nn/SoftMin.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SoftPlus.lua (renamed from contrib/torch/nn/SoftPlus.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SoftShrink.lua (renamed from contrib/torch/nn/SoftShrink.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SoftSign.lua (renamed from contrib/torch/nn/SoftSign.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SparseJacobian.lua (renamed from contrib/torch/nn/SparseJacobian.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SparseLinear.lua (renamed from contrib/torch/nn/SparseLinear.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialAdaptiveAveragePooling.lua (renamed from contrib/torch/nn/SpatialAdaptiveAveragePooling.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialAdaptiveMaxPooling.lua (renamed from contrib/torch/nn/SpatialAdaptiveMaxPooling.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialAutoCropMSECriterion.lua (renamed from contrib/torch/nn/SpatialAutoCropMSECriterion.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialAveragePooling.lua (renamed from contrib/torch/nn/SpatialAveragePooling.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialBatchNormalization.lua (renamed from contrib/torch/nn/SpatialBatchNormalization.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialClassNLLCriterion.lua (renamed from contrib/torch/nn/SpatialClassNLLCriterion.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialContrastiveNormalization.lua (renamed from contrib/torch/nn/SpatialContrastiveNormalization.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialConvolution.lua (renamed from contrib/torch/nn/SpatialConvolution.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialConvolutionLocal.lua (renamed from contrib/torch/nn/SpatialConvolutionLocal.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialConvolutionMM.lua (renamed from contrib/torch/nn/SpatialConvolutionMM.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialConvolutionMap.lua (renamed from contrib/torch/nn/SpatialConvolutionMap.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialCrossMapLRN.lua (renamed from contrib/torch/nn/SpatialCrossMapLRN.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialDepthWiseConvolution.lua (renamed from contrib/torch/nn/SpatialDepthWiseConvolution.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialDilatedConvolution.lua (renamed from contrib/torch/nn/SpatialDilatedConvolution.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialDilatedMaxPooling.lua (renamed from contrib/torch/nn/SpatialDilatedMaxPooling.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialDivisiveNormalization.lua (renamed from contrib/torch/nn/SpatialDivisiveNormalization.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialDropout.lua (renamed from contrib/torch/nn/SpatialDropout.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialFractionalMaxPooling.lua (renamed from contrib/torch/nn/SpatialFractionalMaxPooling.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialFullConvolution.lua (renamed from contrib/torch/nn/SpatialFullConvolution.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialFullConvolutionMap.lua (renamed from contrib/torch/nn/SpatialFullConvolutionMap.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialLPPooling.lua (renamed from contrib/torch/nn/SpatialLPPooling.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialLogSoftMax.lua (renamed from contrib/torch/nn/SpatialLogSoftMax.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialMaxPooling.lua (renamed from contrib/torch/nn/SpatialMaxPooling.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialMaxUnpooling.lua (renamed from contrib/torch/nn/SpatialMaxUnpooling.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialReflectionPadding.lua (renamed from contrib/torch/nn/SpatialReflectionPadding.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialReplicationPadding.lua (renamed from contrib/torch/nn/SpatialReplicationPadding.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialSoftMax.lua (renamed from contrib/torch/nn/SpatialSoftMax.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialSubSampling.lua (renamed from contrib/torch/nn/SpatialSubSampling.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialSubtractiveNormalization.lua (renamed from contrib/torch/nn/SpatialSubtractiveNormalization.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialUpSamplingBilinear.lua (renamed from contrib/torch/nn/SpatialUpSamplingBilinear.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialUpSamplingNearest.lua (renamed from contrib/torch/nn/SpatialUpSamplingNearest.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SpatialZeroPadding.lua (renamed from contrib/torch/nn/SpatialZeroPadding.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/SplitTable.lua (renamed from contrib/torch/nn/SplitTable.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Sqrt.lua (renamed from contrib/torch/nn/Sqrt.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Square.lua (renamed from contrib/torch/nn/Square.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Squeeze.lua (renamed from contrib/torch/nn/Squeeze.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/StochasticGradient.lua (renamed from contrib/torch/nn/StochasticGradient.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Sum.lua (renamed from contrib/torch/nn/Sum.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/THNN.lua (renamed from contrib/torch/nn/THNN.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Tanh.lua (renamed from contrib/torch/nn/Tanh.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/TanhShrink.lua (renamed from contrib/torch/nn/TanhShrink.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/TemporalConvolution.lua (renamed from contrib/torch/nn/TemporalConvolution.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/TemporalDynamicKMaxPooling.lua (renamed from contrib/torch/nn/TemporalDynamicKMaxPooling.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/TemporalMaxPooling.lua (renamed from contrib/torch/nn/TemporalMaxPooling.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/TemporalRowConvolution.lua (renamed from contrib/torch/nn/TemporalRowConvolution.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/TemporalSubSampling.lua (renamed from contrib/torch/nn/TemporalSubSampling.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Threshold.lua (renamed from contrib/torch/nn/Threshold.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Transpose.lua (renamed from contrib/torch/nn/Transpose.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/Unsqueeze.lua (renamed from contrib/torch/nn/Unsqueeze.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/View.lua (renamed from contrib/torch/nn/View.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/VolumetricAveragePooling.lua (renamed from contrib/torch/nn/VolumetricAveragePooling.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/VolumetricBatchNormalization.lua (renamed from contrib/torch/nn/VolumetricBatchNormalization.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/VolumetricConvolution.lua (renamed from contrib/torch/nn/VolumetricConvolution.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/VolumetricDilatedConvolution.lua (renamed from contrib/torch/nn/VolumetricDilatedConvolution.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/VolumetricDilatedMaxPooling.lua (renamed from contrib/torch/nn/VolumetricDilatedMaxPooling.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/VolumetricDropout.lua (renamed from contrib/torch/nn/VolumetricDropout.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/VolumetricFractionalMaxPooling.lua (renamed from contrib/torch/nn/VolumetricFractionalMaxPooling.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/VolumetricFullConvolution.lua (renamed from contrib/torch/nn/VolumetricFullConvolution.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/VolumetricMaxPooling.lua (renamed from contrib/torch/nn/VolumetricMaxPooling.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/VolumetricMaxUnpooling.lua (renamed from contrib/torch/nn/VolumetricMaxUnpooling.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/VolumetricReplicationPadding.lua (renamed from contrib/torch/nn/VolumetricReplicationPadding.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/WeightNorm.lua (renamed from contrib/torch/nn/WeightNorm.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/WeightedEuclidean.lua (renamed from contrib/torch/nn/WeightedEuclidean.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/WeightedMSECriterion.lua (renamed from contrib/torch/nn/WeightedMSECriterion.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/WhiteNoise.lua (renamed from contrib/torch/nn/WhiteNoise.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/ZeroGrad.lua (renamed from contrib/torch/nn/ZeroGrad.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/ZipTable.lua (renamed from contrib/torch/nn/ZipTable.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/ZipTableOneToMany.lua (renamed from contrib/torch/nn/ZipTableOneToMany.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/hessian.lua (renamed from contrib/torch/nn/hessian.lua) | 0 | ||||
-rwxr-xr-x | contrib/lua-torch/nn/init.lua (renamed from contrib/torch/nn/init.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/CMakeLists.txt (renamed from contrib/torch/nn/lib/CMakeLists.txt) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/CMakeLists.txt (renamed from contrib/torch/nn/lib/THNN/CMakeLists.txt) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/README.md (renamed from contrib/torch/nn/lib/THNN/README.md) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/THNN.h (renamed from contrib/torch/nn/lib/THNN/THNN.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/Abs.c (renamed from contrib/torch/nn/lib/THNN/generic/Abs.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/AbsCriterion.c (renamed from contrib/torch/nn/lib/THNN/generic/AbsCriterion.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/BCECriterion.c (renamed from contrib/torch/nn/lib/THNN/generic/BCECriterion.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/BatchNormalization.c (renamed from contrib/torch/nn/lib/THNN/generic/BatchNormalization.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/ClassNLLCriterion.c (renamed from contrib/torch/nn/lib/THNN/generic/ClassNLLCriterion.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/DistKLDivCriterion.c (renamed from contrib/torch/nn/lib/THNN/generic/DistKLDivCriterion.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/ELU.c (renamed from contrib/torch/nn/lib/THNN/generic/ELU.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/FusedRNNKernel.c (renamed from contrib/torch/nn/lib/THNN/generic/FusedRNNKernel.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/GatedLinearUnit.c (renamed from contrib/torch/nn/lib/THNN/generic/GatedLinearUnit.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/HardShrink.c (renamed from contrib/torch/nn/lib/THNN/generic/HardShrink.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/HardTanh.c (renamed from contrib/torch/nn/lib/THNN/generic/HardTanh.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/IndexLinear.c (renamed from contrib/torch/nn/lib/THNN/generic/IndexLinear.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/L1Cost.c (renamed from contrib/torch/nn/lib/THNN/generic/L1Cost.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/LeakyReLU.c (renamed from contrib/torch/nn/lib/THNN/generic/LeakyReLU.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/Linear.c (renamed from contrib/torch/nn/lib/THNN/generic/Linear.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/LogSigmoid.c (renamed from contrib/torch/nn/lib/THNN/generic/LogSigmoid.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/LogSoftMax.c (renamed from contrib/torch/nn/lib/THNN/generic/LogSoftMax.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/LookupTable.c (renamed from contrib/torch/nn/lib/THNN/generic/LookupTable.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/MSECriterion.c (renamed from contrib/torch/nn/lib/THNN/generic/MSECriterion.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/MarginCriterion.c (renamed from contrib/torch/nn/lib/THNN/generic/MarginCriterion.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/MultiLabelMarginCriterion.c (renamed from contrib/torch/nn/lib/THNN/generic/MultiLabelMarginCriterion.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/MultiMarginCriterion.c (renamed from contrib/torch/nn/lib/THNN/generic/MultiMarginCriterion.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/PReLU.c (renamed from contrib/torch/nn/lib/THNN/generic/PReLU.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/RReLU.c (renamed from contrib/torch/nn/lib/THNN/generic/RReLU.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/Sigmoid.c (renamed from contrib/torch/nn/lib/THNN/generic/Sigmoid.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/SmoothL1Criterion.c (renamed from contrib/torch/nn/lib/THNN/generic/SmoothL1Criterion.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/SoftMarginCriterion.c (renamed from contrib/torch/nn/lib/THNN/generic/SoftMarginCriterion.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/SoftMax.c (renamed from contrib/torch/nn/lib/THNN/generic/SoftMax.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/SoftPlus.c (renamed from contrib/torch/nn/lib/THNN/generic/SoftPlus.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/SoftShrink.c (renamed from contrib/torch/nn/lib/THNN/generic/SoftShrink.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/SparseLinear.c (renamed from contrib/torch/nn/lib/THNN/generic/SparseLinear.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/SpatialAdaptiveAveragePooling.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialAdaptiveAveragePooling.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/SpatialAdaptiveMaxPooling.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialAdaptiveMaxPooling.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/SpatialAveragePooling.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialAveragePooling.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/SpatialClassNLLCriterion.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialClassNLLCriterion.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/SpatialConvolutionLocal.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialConvolutionLocal.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/SpatialConvolutionMM.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialConvolutionMM.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/SpatialConvolutionMap.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialConvolutionMap.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/SpatialDepthWiseConvolution.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialDepthWiseConvolution.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/SpatialDilatedConvolution.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialDilatedConvolution.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/SpatialDilatedMaxPooling.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialDilatedMaxPooling.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/SpatialFractionalMaxPooling.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialFractionalMaxPooling.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/SpatialFullConvolution.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialFullConvolution.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/SpatialFullConvolutionMap.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialFullConvolutionMap.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/SpatialMaxPooling.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialMaxPooling.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/SpatialMaxUnpooling.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialMaxUnpooling.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/SpatialReflectionPadding.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialReflectionPadding.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/SpatialReplicationPadding.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialReplicationPadding.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/SpatialSubSampling.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialSubSampling.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/SpatialUpSamplingBilinear.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialUpSamplingBilinear.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/SpatialUpSamplingNearest.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialUpSamplingNearest.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/Sqrt.c (renamed from contrib/torch/nn/lib/THNN/generic/Sqrt.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/Square.c (renamed from contrib/torch/nn/lib/THNN/generic/Square.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/THNN.h (renamed from contrib/torch/nn/lib/THNN/generic/THNN.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/Tanh.c (renamed from contrib/torch/nn/lib/THNN/generic/Tanh.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/TemporalConvolution.c (renamed from contrib/torch/nn/lib/THNN/generic/TemporalConvolution.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/TemporalMaxPooling.c (renamed from contrib/torch/nn/lib/THNN/generic/TemporalMaxPooling.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/TemporalRowConvolution.c (renamed from contrib/torch/nn/lib/THNN/generic/TemporalRowConvolution.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/TemporalSubSampling.c (renamed from contrib/torch/nn/lib/THNN/generic/TemporalSubSampling.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/Threshold.c (renamed from contrib/torch/nn/lib/THNN/generic/Threshold.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/VolumetricAveragePooling.c (renamed from contrib/torch/nn/lib/THNN/generic/VolumetricAveragePooling.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/VolumetricConvolution.c (renamed from contrib/torch/nn/lib/THNN/generic/VolumetricConvolution.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/VolumetricConvolutionMM.c (renamed from contrib/torch/nn/lib/THNN/generic/VolumetricConvolutionMM.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/VolumetricDilatedConvolution.c (renamed from contrib/torch/nn/lib/THNN/generic/VolumetricDilatedConvolution.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/VolumetricDilatedMaxPooling.c (renamed from contrib/torch/nn/lib/THNN/generic/VolumetricDilatedMaxPooling.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/VolumetricFractionalMaxPooling.c (renamed from contrib/torch/nn/lib/THNN/generic/VolumetricFractionalMaxPooling.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/VolumetricFullConvolution.c (renamed from contrib/torch/nn/lib/THNN/generic/VolumetricFullConvolution.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/VolumetricMaxPooling.c (renamed from contrib/torch/nn/lib/THNN/generic/VolumetricMaxPooling.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/VolumetricMaxUnpooling.c (renamed from contrib/torch/nn/lib/THNN/generic/VolumetricMaxUnpooling.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/VolumetricReplicationPadding.c (renamed from contrib/torch/nn/lib/THNN/generic/VolumetricReplicationPadding.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/VolumetricUpSamplingNearest.c (renamed from contrib/torch/nn/lib/THNN/generic/VolumetricUpSamplingNearest.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/VolumetricUpSamplingTrilinear.c (renamed from contrib/torch/nn/lib/THNN/generic/VolumetricUpSamplingTrilinear.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/generic/unfold.c (renamed from contrib/torch/nn/lib/THNN/generic/unfold.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/lib/THNN/init.c (renamed from contrib/torch/nn/lib/THNN/init.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/mkdocs.yml (renamed from contrib/torch/nn/mkdocs.yml) | 0 | ||||
-rwxr-xr-x | contrib/lua-torch/nn/test.lua (renamed from contrib/torch/nn/test.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/nn/utils.lua (renamed from contrib/torch/nn/utils.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/optim/CMakeLists.txt (renamed from contrib/torch/optim/CMakeLists.txt) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/optim/COPYRIGHT.txt (renamed from contrib/torch/optim/COPYRIGHT.txt) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/optim/ConfusionMatrix.lua (renamed from contrib/torch/optim/ConfusionMatrix.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/optim/Logger.lua (renamed from contrib/torch/optim/Logger.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/optim/adadelta.lua (renamed from contrib/torch/optim/adadelta.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/optim/adagrad.lua (renamed from contrib/torch/optim/adagrad.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/optim/adam.lua (renamed from contrib/torch/optim/adam.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/optim/adamax.lua (renamed from contrib/torch/optim/adamax.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/optim/asgd.lua (renamed from contrib/torch/optim/asgd.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/optim/cg.lua (renamed from contrib/torch/optim/cg.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/optim/checkgrad.lua (renamed from contrib/torch/optim/checkgrad.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/optim/cmaes.lua (renamed from contrib/torch/optim/cmaes.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/optim/de.lua (renamed from contrib/torch/optim/de.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/optim/fista.lua (renamed from contrib/torch/optim/fista.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/optim/init.lua (renamed from contrib/torch/optim/init.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/optim/lbfgs.lua (renamed from contrib/torch/optim/lbfgs.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/optim/lswolfe.lua (renamed from contrib/torch/optim/lswolfe.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/optim/nag.lua (renamed from contrib/torch/optim/nag.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/optim/polyinterp.lua (renamed from contrib/torch/optim/polyinterp.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/optim/rmsprop.lua (renamed from contrib/torch/optim/rmsprop.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/optim/rprop.lua (renamed from contrib/torch/optim/rprop.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/optim/sgd.lua (renamed from contrib/torch/optim/sgd.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/paths/CMakeLists.txt (renamed from contrib/torch/paths/CMakeLists.txt) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/paths/COPYRIGHT.txt (renamed from contrib/torch/paths/COPYRIGHT.txt) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/paths/README.md (renamed from contrib/torch/paths/README.md) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/paths/init.lua (renamed from contrib/torch/paths/init.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/paths/mkdocs.yml (renamed from contrib/torch/paths/mkdocs.yml) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/paths/paths.c (renamed from contrib/torch/paths/paths.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/paths/paths.h.in (renamed from contrib/torch/paths/paths.h.in) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/CMakeLists.txt (renamed from contrib/torch/torch7/CMakeLists.txt) | 20 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/COPYRIGHT.txt (renamed from contrib/torch/torch7/COPYRIGHT.txt) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/CmdLine.lua (renamed from contrib/torch/torch7/CmdLine.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/DiskFile.c (renamed from contrib/torch/torch7/DiskFile.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/FFInterface.lua (renamed from contrib/torch/torch7/FFInterface.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/File.c (renamed from contrib/torch/torch7/File.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/File.lua (renamed from contrib/torch/torch7/File.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/Generator.c (renamed from contrib/torch/torch7/Generator.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/MemoryFile.c (renamed from contrib/torch/torch7/MemoryFile.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/PipeFile.c (renamed from contrib/torch/torch7/PipeFile.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/README.md (renamed from contrib/torch/torch7/README.md) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/ROADMAP.md (renamed from contrib/torch/torch7/ROADMAP.md) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/Storage.c (renamed from contrib/torch/torch7/Storage.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/Tensor.c (renamed from contrib/torch/torch7/Tensor.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/Tensor.lua (renamed from contrib/torch/torch7/Tensor.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/TensorMath.c (renamed from contrib/torch/torch7/TensorMath.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/TensorMath.lua (renamed from contrib/torch/torch7/TensorMath.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/TensorOperator.c (renamed from contrib/torch/torch7/TensorOperator.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/TestSuite.lua (renamed from contrib/torch/torch7/TestSuite.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/Tester.lua (renamed from contrib/torch/torch7/Tester.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/Timer.c (renamed from contrib/torch/torch7/Timer.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/cmake/TorchConfig.cmake.in (renamed from contrib/torch/torch7/cmake/TorchConfig.cmake.in) | 2 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/cmake/TorchExports.cmake (renamed from contrib/torch/torch7/cmake/TorchExports.cmake) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/cmake/TorchPackage.cmake (renamed from contrib/torch/torch7/cmake/TorchPackage.cmake) | 8 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/cmake/TorchPaths.cmake (renamed from contrib/torch/torch7/cmake/TorchPaths.cmake) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/cmake/TorchPathsInit.cmake (renamed from contrib/torch/torch7/cmake/TorchPathsInit.cmake) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/cmake/TorchWrap.cmake (renamed from contrib/torch/torch7/cmake/TorchWrap.cmake) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/cmake/TorchWrap.cmake.in (renamed from contrib/torch/torch7/cmake/TorchWrap.cmake.in) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/general.h (renamed from contrib/torch/torch7/general.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/generic/Storage.c (renamed from contrib/torch/torch7/generic/Storage.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/generic/Tensor.c (renamed from contrib/torch/torch7/generic/Tensor.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/generic/TensorOperator.c (renamed from contrib/torch/torch7/generic/TensorOperator.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/generic/luaG.h (renamed from contrib/torch/torch7/generic/luaG.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/init.c (renamed from contrib/torch/torch7/init.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/init.lua (renamed from contrib/torch/torch7/init.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/CMakeLists.txt (renamed from contrib/torch/torch7/lib/CMakeLists.txt) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/CMakeLists.txt (renamed from contrib/torch/torch7/lib/TH/CMakeLists.txt) | 2 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/README.md (renamed from contrib/torch/torch7/lib/TH/README.md) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/TH.h (renamed from contrib/torch/torch7/lib/TH/TH.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THAllocator.c (renamed from contrib/torch/torch7/lib/TH/THAllocator.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THAllocator.h (renamed from contrib/torch/torch7/lib/TH/THAllocator.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THAtomic.c (renamed from contrib/torch/torch7/lib/TH/THAtomic.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THAtomic.h (renamed from contrib/torch/torch7/lib/TH/THAtomic.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THBlas.c (renamed from contrib/torch/torch7/lib/TH/THBlas.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THBlas.h (renamed from contrib/torch/torch7/lib/TH/THBlas.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THConfig.cmake.in (renamed from contrib/torch/torch7/lib/TH/THConfig.cmake.in) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THDiskFile.c (renamed from contrib/torch/torch7/lib/TH/THDiskFile.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THDiskFile.h (renamed from contrib/torch/torch7/lib/TH/THDiskFile.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THFile.c (renamed from contrib/torch/torch7/lib/TH/THFile.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THFile.h (renamed from contrib/torch/torch7/lib/TH/THFile.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THFilePrivate.h (renamed from contrib/torch/torch7/lib/TH/THFilePrivate.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THGeneral.c (renamed from contrib/torch/torch7/lib/TH/THGeneral.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THGeneral.h.in (renamed from contrib/torch/torch7/lib/TH/THGeneral.h.in) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THGenerateAllTypes.h (renamed from contrib/torch/torch7/lib/TH/THGenerateAllTypes.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THGenerateByteType.h (renamed from contrib/torch/torch7/lib/TH/THGenerateByteType.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THGenerateCharType.h (renamed from contrib/torch/torch7/lib/TH/THGenerateCharType.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THGenerateDoubleType.h (renamed from contrib/torch/torch7/lib/TH/THGenerateDoubleType.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THGenerateFloatType.h (renamed from contrib/torch/torch7/lib/TH/THGenerateFloatType.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THGenerateFloatTypes.h (renamed from contrib/torch/torch7/lib/TH/THGenerateFloatTypes.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THGenerateHalfType.h (renamed from contrib/torch/torch7/lib/TH/THGenerateHalfType.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THGenerateIntType.h (renamed from contrib/torch/torch7/lib/TH/THGenerateIntType.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THGenerateIntTypes.h (renamed from contrib/torch/torch7/lib/TH/THGenerateIntTypes.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THGenerateLongType.h (renamed from contrib/torch/torch7/lib/TH/THGenerateLongType.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THGenerateShortType.h (renamed from contrib/torch/torch7/lib/TH/THGenerateShortType.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THHalf.c (renamed from contrib/torch/torch7/lib/TH/THHalf.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THHalf.h (renamed from contrib/torch/torch7/lib/TH/THHalf.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THLapack.c (renamed from contrib/torch/torch7/lib/TH/THLapack.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THLapack.h (renamed from contrib/torch/torch7/lib/TH/THLapack.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THLogAdd.c (renamed from contrib/torch/torch7/lib/TH/THLogAdd.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THLogAdd.h (renamed from contrib/torch/torch7/lib/TH/THLogAdd.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THMath.h (renamed from contrib/torch/torch7/lib/TH/THMath.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THMemoryFile.c (renamed from contrib/torch/torch7/lib/TH/THMemoryFile.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THMemoryFile.h (renamed from contrib/torch/torch7/lib/TH/THMemoryFile.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THRandom.c (renamed from contrib/torch/torch7/lib/TH/THRandom.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THRandom.h (renamed from contrib/torch/torch7/lib/TH/THRandom.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THSize.c (renamed from contrib/torch/torch7/lib/TH/THSize.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THSize.h (renamed from contrib/torch/torch7/lib/TH/THSize.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THStorage.c (renamed from contrib/torch/torch7/lib/TH/THStorage.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THStorage.h (renamed from contrib/torch/torch7/lib/TH/THStorage.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THTensor.c (renamed from contrib/torch/torch7/lib/TH/THTensor.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THTensor.h (renamed from contrib/torch/torch7/lib/TH/THTensor.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THTensorApply.h (renamed from contrib/torch/torch7/lib/TH/THTensorApply.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THTensorDimApply.h (renamed from contrib/torch/torch7/lib/TH/THTensorDimApply.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THTensorMacros.h (renamed from contrib/torch/torch7/lib/TH/THTensorMacros.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THVector.c (renamed from contrib/torch/torch7/lib/TH/THVector.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/THVector.h (renamed from contrib/torch/torch7/lib/TH/THVector.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/cmake/FindARM.cmake (renamed from contrib/torch/torch7/lib/TH/cmake/FindARM.cmake) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/cmake/FindBLAS.cmake (renamed from contrib/torch/torch7/lib/TH/cmake/FindBLAS.cmake) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/cmake/FindLAPACK.cmake (renamed from contrib/torch/torch7/lib/TH/cmake/FindLAPACK.cmake) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/cmake/FindMKL.cmake (renamed from contrib/torch/torch7/lib/TH/cmake/FindMKL.cmake) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/cmake/FindSSE.cmake (renamed from contrib/torch/torch7/lib/TH/cmake/FindSSE.cmake) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/THBlas.c (renamed from contrib/torch/torch7/lib/TH/generic/THBlas.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/THBlas.h (renamed from contrib/torch/torch7/lib/TH/generic/THBlas.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/THLapack.c (renamed from contrib/torch/torch7/lib/TH/generic/THLapack.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/THLapack.h (renamed from contrib/torch/torch7/lib/TH/generic/THLapack.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/THStorage.c (renamed from contrib/torch/torch7/lib/TH/generic/THStorage.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/THStorage.h (renamed from contrib/torch/torch7/lib/TH/generic/THStorage.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/THStorageCopy.c (renamed from contrib/torch/torch7/lib/TH/generic/THStorageCopy.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/THStorageCopy.h (renamed from contrib/torch/torch7/lib/TH/generic/THStorageCopy.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/THTensor.c (renamed from contrib/torch/torch7/lib/TH/generic/THTensor.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/THTensor.h (renamed from contrib/torch/torch7/lib/TH/generic/THTensor.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/THTensorConv.c (renamed from contrib/torch/torch7/lib/TH/generic/THTensorConv.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/THTensorConv.h (renamed from contrib/torch/torch7/lib/TH/generic/THTensorConv.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/THTensorCopy.c (renamed from contrib/torch/torch7/lib/TH/generic/THTensorCopy.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/THTensorCopy.h (renamed from contrib/torch/torch7/lib/TH/generic/THTensorCopy.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/THTensorLapack.c (renamed from contrib/torch/torch7/lib/TH/generic/THTensorLapack.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/THTensorLapack.h (renamed from contrib/torch/torch7/lib/TH/generic/THTensorLapack.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/THTensorMath.c (renamed from contrib/torch/torch7/lib/TH/generic/THTensorMath.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/THTensorMath.h (renamed from contrib/torch/torch7/lib/TH/generic/THTensorMath.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/THTensorRandom.c (renamed from contrib/torch/torch7/lib/TH/generic/THTensorRandom.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/THTensorRandom.h (renamed from contrib/torch/torch7/lib/TH/generic/THTensorRandom.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/THVector.h (renamed from contrib/torch/torch7/lib/TH/generic/THVector.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/THVectorDefault.c (renamed from contrib/torch/torch7/lib/TH/generic/THVectorDefault.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/THVectorDispatch.c (renamed from contrib/torch/torch7/lib/TH/generic/THVectorDispatch.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/simd/common_simd.h (renamed from contrib/torch/torch7/lib/TH/generic/simd/common_simd.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/simd/convolve.c (renamed from contrib/torch/torch7/lib/TH/generic/simd/convolve.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/simd/convolve.h (renamed from contrib/torch/torch7/lib/TH/generic/simd/convolve.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/simd/convolve5x5_avx.c (renamed from contrib/torch/torch7/lib/TH/generic/simd/convolve5x5_avx.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/simd/convolve5x5_sse.c (renamed from contrib/torch/torch7/lib/TH/generic/simd/convolve5x5_sse.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/generic/simd/simd.h (renamed from contrib/torch/torch7/lib/TH/generic/simd/simd.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/vector/AVX.c (renamed from contrib/torch/torch7/lib/TH/vector/AVX.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/vector/AVX.h (renamed from contrib/torch/torch7/lib/TH/vector/AVX.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/vector/AVX2.c (renamed from contrib/torch/torch7/lib/TH/vector/AVX2.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/vector/AVX2.h (renamed from contrib/torch/torch7/lib/TH/vector/AVX2.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/vector/NEON.c (renamed from contrib/torch/torch7/lib/TH/vector/NEON.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/vector/SSE.c (renamed from contrib/torch/torch7/lib/TH/vector/SSE.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/TH/vector/VSX.c (renamed from contrib/torch/torch7/lib/TH/vector/VSX.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/luaT/CMakeLists.txt (renamed from contrib/torch/torch7/lib/luaT/CMakeLists.txt) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/luaT/README.md (renamed from contrib/torch/torch7/lib/luaT/README.md) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/luaT/luaT.c (renamed from contrib/torch/torch7/lib/luaT/luaT.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/luaT/luaT.h (renamed from contrib/torch/torch7/lib/luaT/luaT.h) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/lib/luaT/luaTConfig.cmake.in (renamed from contrib/torch/torch7/lib/luaT/luaTConfig.cmake.in) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/mkdocs.yml (renamed from contrib/torch/torch7/mkdocs.yml) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/paths.lua.in (renamed from contrib/torch/torch7/paths.lua.in) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/random.c (renamed from contrib/torch/torch7/random.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/random.lua (renamed from contrib/torch/torch7/random.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/test/longSize.lua (renamed from contrib/torch/torch7/test/longSize.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/test/test.lua (renamed from contrib/torch/torch7/test/test.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/test/test_Multinomial.lua (renamed from contrib/torch/torch7/test/test_Multinomial.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/test/test_Tester.lua (renamed from contrib/torch/torch7/test/test_Tester.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/test/test_half.lua (renamed from contrib/torch/torch7/test/test_half.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/test/test_qr.lua (renamed from contrib/torch/torch7/test/test_qr.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/test/test_sharedmem.lua (renamed from contrib/torch/torch7/test/test_sharedmem.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/test/test_timer.lua (renamed from contrib/torch/torch7/test/test_timer.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/test/test_writeObject.lua (renamed from contrib/torch/torch7/test/test_writeObject.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/test/timeSort.lua (renamed from contrib/torch/torch7/test/timeSort.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/torchcwrap.lua (renamed from contrib/torch/torch7/torchcwrap.lua) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/utils.c (renamed from contrib/torch/torch7/utils.c) | 0 | ||||
-rw-r--r-- | contrib/lua-torch/torch7/utils.h (renamed from contrib/torch/torch7/utils.h) | 0 | ||||
-rw-r--r-- | contrib/t1ha/t1ha1.c | 51 | ||||
-rw-r--r-- | contrib/t1ha/t1ha2.c | 95 | ||||
-rw-r--r-- | contrib/t1ha/t1ha_bits.h | 226 | ||||
-rw-r--r-- | interface/js/app/history.js | 25 | ||||
-rw-r--r-- | interface/js/app/symbols.js | 12 | ||||
-rw-r--r-- | lualib/ansicolors.lua (renamed from lualib/rspamadm/ansicolors.lua) | 0 | ||||
-rw-r--r-- | lualib/plugins_stats.lua (renamed from lualib/rspamadm/plugins_stats.lua) | 2 | ||||
-rw-r--r-- | lualib/rescore_utility.lua (renamed from lualib/rspamadm/rescore_utility.lua) | 99 | ||||
-rw-r--r-- | lualib/rspamadm/confighelp.lua | 18 | ||||
-rw-r--r-- | lualib/rspamadm/configwizard.lua | 210 | ||||
-rw-r--r-- | lualib/rspamadm/corpus_test.lua | 76 | ||||
-rw-r--r-- | lualib/rspamadm/fuzzy_stat.lua | 25 | ||||
-rw-r--r-- | lualib/rspamadm/getopt.lua | 50 | ||||
-rw-r--r-- | lualib/rspamadm/grep.lua | 66 | ||||
-rw-r--r-- | lualib/rspamadm/keypair.lua | 269 | ||||
-rw-r--r-- | lualib/rspamadm/rescore.lua | 251 | ||||
-rw-r--r-- | rules/regexp/headers.lua | 10 | ||||
-rw-r--r-- | src/libcryptobox/cryptobox.c | 9 | ||||
-rw-r--r-- | src/libcryptobox/cryptobox.h | 3 | ||||
-rw-r--r-- | src/libcryptobox/keypair.c | 2 | ||||
-rw-r--r-- | src/libmime/email_addr.c | 55 | ||||
-rw-r--r-- | src/libmime/message.c | 84 | ||||
-rw-r--r-- | src/libmime/message.h | 6 | ||||
-rw-r--r-- | src/libserver/cfg_file.h | 8 | ||||
-rw-r--r-- | src/libserver/cfg_rcl.c | 2028 | ||||
-rw-r--r-- | src/libserver/cfg_rcl.h | 39 | ||||
-rw-r--r-- | src/libserver/cfg_utils.c | 33 | ||||
-rw-r--r-- | src/libserver/composites.c | 2 | ||||
-rw-r--r-- | src/libserver/dkim.c | 128 | ||||
-rw-r--r-- | src/libserver/symbols_cache.c | 2 | ||||
-rw-r--r-- | src/libserver/task.c | 6 | ||||
-rw-r--r-- | src/libserver/task.h | 31 | ||||
-rw-r--r-- | src/libserver/url.c | 47 | ||||
-rw-r--r-- | src/libstat/stat_process.c | 9 | ||||
-rw-r--r-- | src/libutil/map.c | 2 | ||||
-rw-r--r-- | src/libutil/radix.c | 42 | ||||
-rw-r--r-- | src/libutil/str_util.c | 87 | ||||
-rw-r--r-- | src/libutil/util.c | 97 | ||||
-rw-r--r-- | src/libutil/util.h | 13 | ||||
-rw-r--r-- | src/lua/lua_common.c | 312 | ||||
-rw-r--r-- | src/lua/lua_common.h | 16 | ||||
-rw-r--r-- | src/lua/lua_config.c | 198 | ||||
-rw-r--r-- | src/lua/lua_cryptobox.c | 515 | ||||
-rw-r--r-- | src/lua/lua_mimepart.c | 29 | ||||
-rw-r--r-- | src/lua/lua_task.c | 10 | ||||
-rw-r--r-- | src/lua/lua_util.c | 197 | ||||
-rw-r--r-- | src/plugins/fuzzy_check.c | 2 | ||||
-rw-r--r-- | src/plugins/lua/dmarc.lua | 30 | ||||
-rw-r--r-- | src/plugins/lua/elastic.lua | 5 | ||||
-rw-r--r-- | src/plugins/lua/mime_types.lua | 5 | ||||
-rw-r--r-- | src/plugins/lua/phishing.lua | 11 | ||||
-rw-r--r-- | src/plugins/lua/ratelimit.lua | 12 | ||||
-rw-r--r-- | src/plugins/lua/settings.lua | 50 | ||||
-rw-r--r-- | src/rspamadm/CMakeLists.txt | 5 | ||||
-rw-r--r-- | src/rspamadm/commands.c | 250 | ||||
-rw-r--r-- | src/rspamadm/configdump.c | 12 | ||||
-rw-r--r-- | src/rspamadm/confighelp.c | 17 | ||||
-rw-r--r-- | src/rspamadm/configtest.c | 13 | ||||
-rw-r--r-- | src/rspamadm/configwizard.c | 165 | ||||
-rw-r--r-- | src/rspamadm/control.c | 11 | ||||
-rw-r--r-- | src/rspamadm/corpus_test.c | 125 | ||||
-rw-r--r-- | src/rspamadm/dkim_keygen.c | 10 | ||||
-rw-r--r-- | src/rspamadm/fuzzy_convert.c | 11 | ||||
-rw-r--r-- | src/rspamadm/fuzzy_merge.c | 10 | ||||
-rw-r--r-- | src/rspamadm/grep.c | 156 | ||||
-rw-r--r-- | src/rspamadm/keypair.c | 140 | ||||
-rw-r--r-- | src/rspamadm/lua_repl.c | 15 | ||||
-rw-r--r-- | src/rspamadm/pw.c | 10 | ||||
-rw-r--r-- | src/rspamadm/rescore.c | 199 | ||||
-rw-r--r-- | src/rspamadm/rspamadm.c | 193 | ||||
-rw-r--r-- | src/rspamadm/rspamadm.h | 22 | ||||
-rw-r--r-- | src/rspamadm/signtool.c | 14 | ||||
-rw-r--r-- | src/rspamadm/stat_convert.c | 11 | ||||
-rw-r--r-- | src/rspamd.c | 7 | ||||
-rw-r--r-- | test/functional/cases/106_url_tags.robot | 41 | ||||
-rw-r--r-- | test/rspamd_test_suite.c | 2 | ||||
-rw-r--r-- | test/rspamd_upstream_test.c | 2 | ||||
-rwxr-xr-x | utils/rspamd_stats.pl | 20 |
609 files changed, 5971 insertions, 2945 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index b88da4241..80a804e2e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -502,7 +502,7 @@ INCLUDE_DIRECTORIES("${CMAKE_SOURCE_DIR}/" "${CMAKE_SOURCE_DIR}/contrib/librdns" "${CMAKE_SOURCE_DIR}/contrib/aho-corasick" "${CMAKE_SOURCE_DIR}/contrib/lc-btrie" - "${CMAKE_SOURCE_DIR}/contrib/lpeg" + "${CMAKE_SOURCE_DIR}/contrib/lua-lpeg" "${CMAKE_BINARY_DIR}/src" #Stored in the binary dir "${CMAKE_BINARY_DIR}/src/libcryptobox") @@ -1240,7 +1240,7 @@ ENDIF() ADD_SUBDIRECTORY(contrib/libucl) ADD_SUBDIRECTORY(contrib/librdns) ADD_SUBDIRECTORY(contrib/aho-corasick) -ADD_SUBDIRECTORY(contrib/lpeg) +ADD_SUBDIRECTORY(contrib/lua-lpeg) ADD_SUBDIRECTORY(contrib/linenoise) ADD_SUBDIRECTORY(contrib/t1ha) @@ -1261,11 +1261,11 @@ ENDIF() IF(ENABLE_TORCH MATCHES "ON") IF(WITH_LUAJIT) - ADD_SUBDIRECTORY(contrib/torch/paths) - ADD_SUBDIRECTORY(contrib/torch/torch7) - ADD_SUBDIRECTORY(contrib/torch/nn) - ADD_SUBDIRECTORY(contrib/torch/optim) - ADD_SUBDIRECTORY(contrib/torch/decisiontree) + ADD_SUBDIRECTORY(contrib/lua-torch/paths) + ADD_SUBDIRECTORY(contrib/lua-torch/torch7) + ADD_SUBDIRECTORY(contrib/lua-torch/nn) + ADD_SUBDIRECTORY(contrib/lua-torch/optim) + ADD_SUBDIRECTORY(contrib/lua-torch/decisiontree) SET(WITH_TORCH 1) ELSE() MESSAGE(FATAL_ERROR "Cannot enable torch without luajit") @@ -1358,9 +1358,10 @@ ENDFOREACH(LUA_LIB) # Install lua fun library INSTALL(FILES "contrib/lua-fun/fun.lua" DESTINATION ${LUALIBDIR}) +INSTALL(FILES "contrib/lua-argparse/argparse.lua" DESTINATION ${LUALIBDIR}) IF(ENABLE_TORCH MATCHES "ON") - INSTALL(FILES "contrib/moses/moses.lua" DESTINATION ${LUALIBDIR}) + INSTALL(FILES "contrib/lua-moses/moses.lua" DESTINATION ${LUALIBDIR}) ENDIF() # systemd unit diff --git a/clang-plugin/CMakeLists.txt b/clang-plugin/CMakeLists.txt index eae764235..8b2def17e 100644 --- a/clang-plugin/CMakeLists.txt +++ b/clang-plugin/CMakeLists.txt @@ -4,7 +4,7 @@ IF (ENABLE_CLANG_PLUGIN MATCHES "ON") MESSAGE(FATAL_ERROR "Cannot build clang plugin when compiler is not clang") endif () - SET(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${CMAKE_CURRENT_SOURCE_DIR}") + LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}") ENABLE_LANGUAGE(CXX) FIND_PACKAGE(LLVM REQUIRED) diff --git a/conf/composites.conf b/conf/composites.conf index fb91d8dfd..a07b7020f 100644 --- a/conf/composites.conf +++ b/conf/composites.conf @@ -81,6 +81,12 @@ composites { score = 3.0; policy = "leave"; } + RCVD_UNAUTH_PBL { + expression = "RECEIVED_PBL & -RCVD_VIA_SMTP_AUTH"; + description = "Relayed through ZEN PBL IP without sufficient authentication"; + score = 2.0; + policy = "leave"; + } .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/composites.conf" .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/composites.conf" diff --git a/conf/modules.d/ratelimit.conf b/conf/modules.d/ratelimit.conf index 08109b820..521bdb1a8 100644 --- a/conf/modules.d/ratelimit.conf +++ b/conf/modules.d/ratelimit.conf @@ -30,6 +30,10 @@ ratelimit { #} # If symbol is specified, then it is inserted instead of setting result #symbol = "R_RATELIMIT"; + + # If info_symbol is specified, then it is inserted next to set the result + #info_symbol = "R_RATELIMIT_INFO"; + whitelisted_rcpts = "postmaster,mailer-daemon"; max_rcpt = 5; diff --git a/contrib/lc-btrie/btrie.c b/contrib/lc-btrie/btrie.c index fe768fc56..b1c66f1c3 100644 --- a/contrib/lc-btrie/btrie.c +++ b/contrib/lc-btrie/btrie.c @@ -1485,7 +1485,7 @@ static size_t count_free(const struct btrie *btrie) #endif /* not NDEBUG */ const char * -btrie_stats(const struct btrie *btrie) +btrie_stats(const struct btrie *btrie, guint duplicates) { static char buf[128]; size_t n_nodes = btrie->n_lc_nodes + btrie->n_tbm_nodes; @@ -1530,8 +1530,10 @@ btrie_stats(const struct btrie *btrie) , average_depth, (long unsigned)stats.max_depth); #else snprintf(buf, sizeof(buf), - "ents=%lu tbm=%lu lc=%lu mem=%.0fk free=%lu waste=%lu" - ,(long unsigned)btrie->n_entries, (long unsigned)btrie->n_tbm_nodes, + "ents=%lu dup=%u tbm=%lu lc=%lu mem=%.0fk free=%lu waste=%lu", + (long unsigned)btrie->n_entries, + duplicates, + (long unsigned)btrie->n_tbm_nodes, (long unsigned)btrie->n_lc_nodes, (double)btrie->alloc_total / 1024, (long unsigned)alloc_free, (long unsigned)btrie->alloc_waste ); diff --git a/contrib/lc-btrie/btrie.h b/contrib/lc-btrie/btrie.h index 2cc662961..370a03e51 100644 --- a/contrib/lc-btrie/btrie.h +++ b/contrib/lc-btrie/btrie.h @@ -70,7 +70,7 @@ enum btrie_result btrie_add_prefix(struct btrie *btrie, const void *btrie_lookup(const struct btrie *btrie, const btrie_oct_t *pfx, unsigned len); -const char *btrie_stats(const struct btrie *btrie); +const char *btrie_stats(const struct btrie *btrie, guint duplicates); #ifndef NO_MASTER_DUMP typedef void btrie_walk_cb_t(const btrie_oct_t *prefix, unsigned len, diff --git a/contrib/librdns/dns_private.h b/contrib/librdns/dns_private.h index 37d0240b6..44bb3dd84 100644 --- a/contrib/librdns/dns_private.h +++ b/contrib/librdns/dns_private.h @@ -56,6 +56,15 @@ struct rdns_server { upstream_entry_t up; }; +enum rdns_request_state { + RDNS_REQUEST_NEW = 0, + RDNS_REQUEST_REGISTERED = 1, + RDNS_REQUEST_WAIT_SEND, + RDNS_REQUEST_WAIT_REPLY, + RDNS_REQUEST_REPLIED, + RDNS_REQUEST_FAKE, +}; + struct rdns_request { struct rdns_resolver *resolver; struct rdns_async_context *async; @@ -69,14 +78,7 @@ struct rdns_request { int id; struct rdns_request_name *requested_names; unsigned int qcount; - - enum { - RDNS_REQUEST_NEW = 0, - RDNS_REQUEST_REGISTERED = 1, - RDNS_REQUEST_WAIT_SEND, - RDNS_REQUEST_WAIT_REPLY, - RDNS_REQUEST_REPLIED - } state; + enum rdns_request_state state; uint8_t *packet; off_t pos; @@ -109,6 +111,13 @@ struct rdns_io_channel { ref_entry_t ref; }; +struct rdns_fake_reply { + char *request; + enum dns_rcode rcode; + struct rdns_reply_entry *result; + UT_hash_handle hh; +}; + struct rdns_resolver { struct rdns_server *servers; @@ -117,6 +126,7 @@ struct rdns_resolver { void *periodic; /** periodic event for resolver */ struct rdns_upstream_context *ups; struct rdns_plugin *curve_plugin; + struct rdns_fake_reply *fake_elts; rdns_log_function logger; void *log_data; diff --git a/contrib/librdns/rdns.h b/contrib/librdns/rdns.h index 5c44900f1..545d9421f 100644 --- a/contrib/librdns/rdns.h +++ b/contrib/librdns/rdns.h @@ -324,6 +324,18 @@ void rdns_resolver_register_plugin (struct rdns_resolver *resolver, struct rdns_plugin *plugin); /** + * Add a fake reply for a specified name + * @param resolver + * @param type + * @param name + * @param reply + */ +void rdns_resolver_set_fake_reply (struct rdns_resolver *resolver, + const char *name, + enum dns_rcode rcode, + struct rdns_reply_entry *reply); + +/** * Init DNS resolver * @param resolver * @return diff --git a/contrib/librdns/resolver.c b/contrib/librdns/resolver.c index 7fa61a391..7afabb1ba 100644 --- a/contrib/librdns/resolver.c +++ b/contrib/librdns/resolver.c @@ -484,6 +484,14 @@ rdns_process_retransmit (int fd, void *arg) req->async_event); req->async_event = NULL; + if (req->state == RDNS_REQUEST_FAKE) { + /* Reply is ready */ + req->func (req->reply, req->arg); + REF_RELEASE (req); + + return; + } + r = rdns_send_request (req, fd, false); if (r == 0) { @@ -532,6 +540,7 @@ rdns_make_request_full ( size_t olen; const char *cur_name, *last_name = NULL; struct rdns_compression_entry *comp = NULL; + struct rdns_fake_reply *fake_rep = NULL; if (resolver == NULL || !resolver->initialized) { return NULL; @@ -569,6 +578,18 @@ rdns_make_request_full ( for (i = 0; i < queries * 2; i += 2) { cur = i / 2; cur_name = va_arg (args, const char *); + + if (last_name == NULL) { + HASH_FIND_STR (resolver->fake_elts, cur_name, fake_rep); + + if (fake_rep) { + /* We actually treat it as a short-circuit */ + req->reply = rdns_make_reply (req, fake_rep->rcode); + req->reply->entries = fake_rep->result; + req->state = RDNS_REQUEST_FAKE; + } + } + if (cur_name != NULL) { last_name = cur_name; clen = strlen (cur_name); @@ -585,7 +606,8 @@ rdns_make_request_full ( return NULL; } - if (!rdns_format_dns_name (resolver, last_name, clen, + if (req->state != RDNS_REQUEST_FAKE && + !rdns_format_dns_name (resolver, last_name, clen, &req->requested_names[cur].name, &olen)) { rdns_request_free (req); return NULL; @@ -597,37 +619,39 @@ rdns_make_request_full ( } va_end (args); - rdns_allocate_packet (req, tlen); - rdns_make_dns_header (req, queries); - - for (i = 0; i < queries; i ++) { - cur_name = req->requested_names[i].name; - clen = req->requested_names[i].len; - type = req->requested_names[i].type; - if (queries > 1) { - if (!rdns_add_rr (req, cur_name, clen, type, &comp)) { - REF_RELEASE (req); - rnds_compression_free (comp); - return NULL; - } - } - else { - if (!rdns_add_rr (req, cur_name, clen, type, NULL)) { - REF_RELEASE (req); - rnds_compression_free (comp); - return NULL; + if (req->state != RDNS_REQUEST_FAKE) { + rdns_allocate_packet (req, tlen); + rdns_make_dns_header (req, queries); + + for (i = 0; i < queries; i++) { + cur_name = req->requested_names[i].name; + clen = req->requested_names[i].len; + type = req->requested_names[i].type; + if (queries > 1) { + if (!rdns_add_rr (req, cur_name, clen, type, &comp)) { + REF_RELEASE (req); + rnds_compression_free (comp); + return NULL; + } + } else { + if (!rdns_add_rr (req, cur_name, clen, type, NULL)) { + REF_RELEASE (req); + rnds_compression_free (comp); + return NULL; + } } } - } - rnds_compression_free (comp); + rnds_compression_free (comp); - /* Add EDNS RR */ - rdns_add_edns0 (req); + /* Add EDNS RR */ + rdns_add_edns0 (req); + + req->retransmits = repeats; + req->timeout = timeout; + req->state = RDNS_REQUEST_NEW; + } - req->retransmits = repeats; - req->timeout = timeout; - req->state = RDNS_REQUEST_NEW; req->async = resolver->async; if (resolver->ups) { @@ -656,14 +680,21 @@ rdns_make_request_full ( /* Select random IO channel */ req->io = serv->io_channels[ottery_rand_uint32 () % serv->io_cnt]; - req->io->uses ++; - /* Now send request to server */ - r = rdns_send_request (req, req->io->sock, true); + if (req->state == RDNS_REQUEST_FAKE) { + req->async_event = resolver->async->add_write (resolver->async->data, + req->io->sock, req); + } + else { + req->io->uses++; - if (r == -1) { - REF_RELEASE (req); - return NULL; + /* Now send request to server */ + r = rdns_send_request (req, req->io->sock, true); + + if (r == -1) { + REF_RELEASE (req); + return NULL; + } } REF_RETAIN (req->io); @@ -889,3 +920,39 @@ rdns_resolver_set_dnssec (struct rdns_resolver *resolver, bool enabled) resolver->enable_dnssec = enabled; } } + + +void rdns_resolver_set_fake_reply (struct rdns_resolver *resolver, + const char *name, + enum dns_rcode rcode, + struct rdns_reply_entry *reply) +{ + struct rdns_fake_reply *fake_rep; + + HASH_FIND_STR (resolver->fake_elts, name, fake_rep); + + if (fake_rep) { + /* Append reply to the existing list */ + fake_rep->rcode = rcode; + DL_APPEND (fake_rep->result, reply); + } + else { + fake_rep = calloc (1, sizeof (*fake_rep)); + + if (fake_rep == NULL) { + abort (); + } + + fake_rep->request = strdup (name); + + if (fake_rep->request == NULL) { + abort (); + } + + fake_rep->rcode = rcode; + DL_APPEND (fake_rep->result, reply); + + HASH_ADD_KEYPTR (hh, resolver->fake_elts, fake_rep->request, + strlen (fake_rep->request), fake_rep); + } +}
\ No newline at end of file diff --git a/contrib/librdns/util.c b/contrib/librdns/util.c index b03128983..a4018cbd3 100644 --- a/contrib/librdns/util.c +++ b/contrib/librdns/util.c @@ -353,34 +353,38 @@ rdns_reply_free (struct rdns_reply *rep) { struct rdns_reply_entry *entry, *tmp; - LL_FOREACH_SAFE (rep->entries, entry, tmp) { - switch (entry->type) { - case RDNS_REQUEST_PTR: - free (entry->content.ptr.name); - break; - case RDNS_REQUEST_NS: - free (entry->content.ns.name); - break; - case RDNS_REQUEST_MX: - free (entry->content.mx.name); - break; - case RDNS_REQUEST_TXT: - case RDNS_REQUEST_SPF: - free (entry->content.txt.data); - break; - case RDNS_REQUEST_SRV: - free (entry->content.srv.target); - break; - case RDNS_REQUEST_TLSA: - free (entry->content.tlsa.data); - break; - case RDNS_REQUEST_SOA: - free (entry->content.soa.mname); - free (entry->content.soa.admin); - break; + /* We don't need to free data for faked replies */ + if (!rep->request || rep->request->state != RDNS_REQUEST_FAKE) { + LL_FOREACH_SAFE (rep->entries, entry, tmp) { + switch (entry->type) { + case RDNS_REQUEST_PTR: + free (entry->content.ptr.name); + break; + case RDNS_REQUEST_NS: + free (entry->content.ns.name); + break; + case RDNS_REQUEST_MX: + free (entry->content.mx.name); + break; + case RDNS_REQUEST_TXT: + case RDNS_REQUEST_SPF: + free (entry->content.txt.data); + break; + case RDNS_REQUEST_SRV: + free (entry->content.srv.target); + break; + case RDNS_REQUEST_TLSA: + free (entry->content.tlsa.data); + break; + case RDNS_REQUEST_SOA: + free (entry->content.soa.mname); + free (entry->content.soa.admin); + break; + } + free (entry); } - free (entry); } + free (rep); } @@ -418,7 +422,11 @@ rdns_request_free (struct rdns_request *req) HASH_DEL (req->io->requests, req); req->async_event = NULL; } - + else if (req->state == RDNS_REQUEST_FAKE) { + req->async->del_write (req->async->data, + req->async_event); + req->async_event = NULL; + } } #ifdef TWEETNACL if (req->curve_plugin_data != NULL) { diff --git a/contrib/lua-argparse/LICENSE b/contrib/lua-argparse/LICENSE new file mode 100644 index 000000000..87579acb0 --- /dev/null +++ b/contrib/lua-argparse/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2013 - 2018 Peter Melnichenko + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/contrib/lua-argparse/argparse.lua b/contrib/lua-argparse/argparse.lua new file mode 100644 index 000000000..48950a662 --- /dev/null +++ b/contrib/lua-argparse/argparse.lua @@ -0,0 +1,1516 @@ +-- The MIT License (MIT) + +-- Copyright (c) 2013 - 2018 Peter Melnichenko + +-- Permission is hereby granted, free of charge, to any person obtaining a copy of +-- this software and associated documentation files (the "Software"), to deal in +-- the Software without restriction, including without limitation the rights to +-- use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +-- the Software, and to permit persons to whom the Software is furnished to do so, +-- subject to the following conditions: + +-- The above copyright notice and this permission notice shall be included in all +-- copies or substantial portions of the Software. + +-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +-- FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +-- COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +-- IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +-- CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +local function deep_update(t1, t2) + for k, v in pairs(t2) do + if type(v) == "table" then + v = deep_update({}, v) + end + + t1[k] = v + end + + return t1 +end + +-- A property is a tuple {name, callback}. +-- properties.args is number of properties that can be set as arguments +-- when calling an object. +local function class(prototype, properties, parent) + -- Class is the metatable of its instances. + local cl = {} + cl.__index = cl + + if parent then + cl.__prototype = deep_update(deep_update({}, parent.__prototype), prototype) + else + cl.__prototype = prototype + end + + if properties then + local names = {} + + -- Create setter methods and fill set of property names. + for _, property in ipairs(properties) do + local name, callback = property[1], property[2] + + cl[name] = function(self, value) + if not callback(self, value) then + self["_" .. name] = value + end + + return self + end + + names[name] = true + end + + function cl.__call(self, ...) + -- When calling an object, if the first argument is a table, + -- interpret keys as property names, else delegate arguments + -- to corresponding setters in order. + if type((...)) == "table" then + for name, value in pairs((...)) do + if names[name] then + self[name](self, value) + end + end + else + local nargs = select("#", ...) + + for i, property in ipairs(properties) do + if i > nargs or i > properties.args then + break + end + + local arg = select(i, ...) + + if arg ~= nil then + self[property[1]](self, arg) + end + end + end + + return self + end + end + + -- If indexing class fails, fallback to its parent. + local class_metatable = {} + class_metatable.__index = parent + + function class_metatable.__call(self, ...) + -- Calling a class returns its instance. + -- Arguments are delegated to the instance. + local object = deep_update({}, self.__prototype) + setmetatable(object, self) + return object(...) + end + + return setmetatable(cl, class_metatable) +end + +local function typecheck(name, types, value) + for _, type_ in ipairs(types) do + if type(value) == type_ then + return true + end + end + + error(("bad property '%s' (%s expected, got %s)"):format(name, table.concat(types, " or "), type(value))) +end + +local function typechecked(name, ...) + local types = {...} + return {name, function(_, value) typecheck(name, types, value) end} +end + +local multiname = {"name", function(self, value) + typecheck("name", {"string"}, value) + + for alias in value:gmatch("%S+") do + self._name = self._name or alias + table.insert(self._aliases, alias) + end + + -- Do not set _name as with other properties. + return true +end} + +local function parse_boundaries(str) + if tonumber(str) then + return tonumber(str), tonumber(str) + end + + if str == "*" then + return 0, math.huge + end + + if str == "+" then + return 1, math.huge + end + + if str == "?" then + return 0, 1 + end + + if str:match "^%d+%-%d+$" then + local min, max = str:match "^(%d+)%-(%d+)$" + return tonumber(min), tonumber(max) + end + + if str:match "^%d+%+$" then + local min = str:match "^(%d+)%+$" + return tonumber(min), math.huge + end +end + +local function boundaries(name) + return {name, function(self, value) + typecheck(name, {"number", "string"}, value) + + local min, max = parse_boundaries(value) + + if not min then + error(("bad property '%s'"):format(name)) + end + + self["_min" .. name], self["_max" .. name] = min, max + end} +end + +local actions = {} + +local option_action = {"action", function(_, value) + typecheck("action", {"function", "string"}, value) + + if type(value) == "string" and not actions[value] then + error(("unknown action '%s'"):format(value)) + end +end} + +local option_init = {"init", function(self) + self._has_init = true +end} + +local option_default = {"default", function(self, value) + if type(value) ~= "string" then + self._init = value + self._has_init = true + return true + end +end} + +local add_help = {"add_help", function(self, value) + typecheck("add_help", {"boolean", "string", "table"}, value) + + if self._has_help then + table.remove(self._options) + self._has_help = false + end + + if value then + local help = self:flag() + :description "Show this help message and exit." + :action(function() + print(self:get_help()) + os.exit(0) + end) + + if value ~= true then + help = help(value) + end + + if not help._name then + help "-h" "--help" + end + + self._has_help = true + end +end} + +local Parser = class({ + _arguments = {}, + _options = {}, + _commands = {}, + _mutexes = {}, + _groups = {}, + _require_command = true, + _handle_options = true +}, { + args = 3, + typechecked("name", "string"), + typechecked("description", "string"), + typechecked("epilog", "string"), + typechecked("usage", "string"), + typechecked("help", "string"), + typechecked("require_command", "boolean"), + typechecked("handle_options", "boolean"), + typechecked("action", "function"), + typechecked("command_target", "string"), + typechecked("help_vertical_space", "number"), + typechecked("usage_margin", "number"), + typechecked("usage_max_width", "number"), + typechecked("help_usage_margin", "number"), + typechecked("help_description_margin", "number"), + typechecked("help_max_width", "number"), + add_help +}) + +local Command = class({ + _aliases = {} +}, { + args = 3, + multiname, + typechecked("description", "string"), + typechecked("epilog", "string"), + typechecked("target", "string"), + typechecked("usage", "string"), + typechecked("help", "string"), + typechecked("require_command", "boolean"), + typechecked("handle_options", "boolean"), + typechecked("action", "function"), + typechecked("command_target", "string"), + typechecked("help_vertical_space", "number"), + typechecked("usage_margin", "number"), + typechecked("usage_max_width", "number"), + typechecked("help_usage_margin", "number"), + typechecked("help_description_margin", "number"), + typechecked("help_max_width", "number"), + typechecked("hidden", "boolean"), + add_help +}, Parser) + +local Argument = class({ + _minargs = 1, + _maxargs = 1, + _mincount = 1, + _maxcount = 1, + _defmode = "unused", + _show_default = true +}, { + args = 5, + typechecked("name", "string"), + typechecked("description", "string"), + option_default, + typechecked("convert", "function", "table"), + boundaries("args"), + typechecked("target", "string"), + typechecked("defmode", "string"), + typechecked("show_default", "boolean"), + typechecked("argname", "string", "table"), + typechecked("hidden", "boolean"), + option_action, + option_init +}) + +local Option = class({ + _aliases = {}, + _mincount = 0, + _overwrite = true +}, { + args = 6, + multiname, + typechecked("description", "string"), + option_default, + typechecked("convert", "function", "table"), + boundaries("args"), + boundaries("count"), + typechecked("target", "string"), + typechecked("defmode", "string"), + typechecked("show_default", "boolean"), + typechecked("overwrite", "boolean"), + typechecked("argname", "string", "table"), + typechecked("hidden", "boolean"), + option_action, + option_init +}, Argument) + +function Parser:_inherit_property(name, default) + local element = self + + while true do + local value = element["_" .. name] + + if value ~= nil then + return value + end + + if not element._parent then + return default + end + + element = element._parent + end +end + +function Argument:_get_argument_list() + local buf = {} + local i = 1 + + while i <= math.min(self._minargs, 3) do + local argname = self:_get_argname(i) + + if self._default and self._defmode:find "a" then + argname = "[" .. argname .. "]" + end + + table.insert(buf, argname) + i = i+1 + end + + while i <= math.min(self._maxargs, 3) do + table.insert(buf, "[" .. self:_get_argname(i) .. "]") + i = i+1 + + if self._maxargs == math.huge then + break + end + end + + if i < self._maxargs then + table.insert(buf, "...") + end + + return buf +end + +function Argument:_get_usage() + local usage = table.concat(self:_get_argument_list(), " ") + + if self._default and self._defmode:find "u" then + if self._maxargs > 1 or (self._minargs == 1 and not self._defmode:find "a") then + usage = "[" .. usage .. "]" + end + end + + return usage +end + +function actions.store_true(result, target) + result[target] = true +end + +function actions.store_false(result, target) + result[target] = false +end + +function actions.store(result, target, argument) + result[target] = argument +end + +function actions.count(result, target, _, overwrite) + if not overwrite then + result[target] = result[target] + 1 + end +end + +function actions.append(result, target, argument, overwrite) + result[target] = result[target] or {} + table.insert(result[target], argument) + + if overwrite then + table.remove(result[target], 1) + end +end + +function actions.concat(result, target, arguments, overwrite) + if overwrite then + error("'concat' action can't handle too many invocations") + end + + result[target] = result[target] or {} + + for _, argument in ipairs(arguments) do + table.insert(result[target], argument) + end +end + +function Argument:_get_action() + local action, init + + if self._maxcount == 1 then + if self._maxargs == 0 then + action, init = "store_true", nil + else + action, init = "store", nil + end + else + if self._maxargs == 0 then + action, init = "count", 0 + else + action, init = "append", {} + end + end + + if self._action then + action = self._action + end + + if self._has_init then + init = self._init + end + + if type(action) == "string" then + action = actions[action] + end + + return action, init +end + +-- Returns placeholder for `narg`-th argument. +function Argument:_get_argname(narg) + local argname = self._argname or self:_get_default_argname() + + if type(argname) == "table" then + return argname[narg] + else + return argname + end +end + +function Argument:_get_default_argname() + return "<" .. self._name .. ">" +end + +function Option:_get_default_argname() + return "<" .. self:_get_default_target() .. ">" +end + +-- Returns labels to be shown in the help message. +function Argument:_get_label_lines() + return {self._name} +end + +function Option:_get_label_lines() + local argument_list = self:_get_argument_list() + + if #argument_list == 0 then + -- Don't put aliases for simple flags like `-h` on different lines. + return {table.concat(self._aliases, ", ")} + end + + local argument_list_repr = table.concat(argument_list, " ") + local lines = {} + + for i, alias in ipairs(self._aliases) do + local line = alias .. " " .. argument_list_repr + table.insert(lines, line) + end + + return {table.concat(lines, ", ")} +end + +function Command:_get_label_lines() + return {table.concat(self._aliases, ", ")} +end + +function Argument:_get_description() + if self._default and self._show_default then + if self._description then + return ("%s (default: %s)"):format(self._description, self._default) + else + return ("default: %s"):format(self._default) + end + else + return self._description or "" + end +end + +function Command:_get_description() + return self._description or "" +end + +function Option:_get_usage() + local usage = self:_get_argument_list() + table.insert(usage, 1, self._name) + usage = table.concat(usage, " ") + + if self._mincount == 0 or self._default then + usage = "[" .. usage .. "]" + end + + return usage +end + +function Argument:_get_default_target() + return self._name +end + +function Option:_get_default_target() + local res + + for _, alias in ipairs(self._aliases) do + if alias:sub(1, 1) == alias:sub(2, 2) then + res = alias:sub(3) + break + end + end + + res = res or self._name:sub(2) + return (res:gsub("-", "_")) +end + +function Option:_is_vararg() + return self._maxargs ~= self._minargs +end + +function Parser:_get_fullname() + local parent = self._parent + local buf = {self._name} + + while parent do + table.insert(buf, 1, parent._name) + parent = parent._parent + end + + return table.concat(buf, " ") +end + +function Parser:_update_charset(charset) + charset = charset or {} + + for _, command in ipairs(self._commands) do + command:_update_charset(charset) + end + + for _, option in ipairs(self._options) do + for _, alias in ipairs(option._aliases) do + charset[alias:sub(1, 1)] = true + end + end + + return charset +end + +function Parser:argument(...) + local argument = Argument(...) + table.insert(self._arguments, argument) + return argument +end + +function Parser:option(...) + local option = Option(...) + + if self._has_help then + table.insert(self._options, #self._options, option) + else + table.insert(self._options, option) + end + + return option +end + +function Parser:flag(...) + return self:option():args(0)(...) +end + +function Parser:command(...) + local command = Command():add_help(true)(...) + command._parent = self + table.insert(self._commands, command) + return command +end + +function Parser:mutex(...) + local elements = {...} + + for i, element in ipairs(elements) do + local mt = getmetatable(element) + assert(mt == Option or mt == Argument, ("bad argument #%d to 'mutex' (Option or Argument expected)"):format(i)) + end + + table.insert(self._mutexes, elements) + return self +end + +function Parser:group(name, ...) + assert(type(name) == "string", ("bad argument #1 to 'group' (string expected, got %s)"):format(type(name))) + + local group = {name = name, ...} + + for i, element in ipairs(group) do + local mt = getmetatable(element) + assert(mt == Option or mt == Argument or mt == Command, + ("bad argument #%d to 'group' (Option or Argument or Command expected)"):format(i + 1)) + end + + table.insert(self._groups, group) + return self +end + +local usage_welcome = "Usage: " + +function Parser:get_usage() + if self._usage then + return self._usage + end + + local usage_margin = self:_inherit_property("usage_margin", #usage_welcome) + local max_usage_width = self:_inherit_property("usage_max_width", 70) + local lines = {usage_welcome .. self:_get_fullname()} + + local function add(s) + if #lines[#lines]+1+#s <= max_usage_width then + lines[#lines] = lines[#lines] .. " " .. s + else + lines[#lines+1] = (" "):rep(usage_margin) .. s + end + end + + -- Normally options are before positional arguments in usage messages. + -- However, vararg options should be after, because they can't be reliable used + -- before a positional argument. + -- Mutexes come into play, too, and are shown as soon as possible. + -- Overall, output usages in the following order: + -- 1. Mutexes that don't have positional arguments or vararg options. + -- 2. Options that are not in any mutexes and are not vararg. + -- 3. Positional arguments - on their own or as a part of a mutex. + -- 4. Remaining mutexes. + -- 5. Remaining options. + + local elements_in_mutexes = {} + local added_elements = {} + local added_mutexes = {} + local argument_to_mutexes = {} + + local function add_mutex(mutex, main_argument) + if added_mutexes[mutex] then + return + end + + added_mutexes[mutex] = true + local buf = {} + + for _, element in ipairs(mutex) do + if not element._hidden and not added_elements[element] then + if getmetatable(element) == Option or element == main_argument then + table.insert(buf, element:_get_usage()) + added_elements[element] = true + end + end + end + + if #buf == 1 then + add(buf[1]) + elseif #buf > 1 then + add("(" .. table.concat(buf, " | ") .. ")") + end + end + + local function add_element(element) + if not element._hidden and not added_elements[element] then + add(element:_get_usage()) + added_elements[element] = true + end + end + + for _, mutex in ipairs(self._mutexes) do + local is_vararg = false + local has_argument = false + + for _, element in ipairs(mutex) do + if getmetatable(element) == Option then + if element:_is_vararg() then + is_vararg = true + end + else + has_argument = true + argument_to_mutexes[element] = argument_to_mutexes[element] or {} + table.insert(argument_to_mutexes[element], mutex) + end + + elements_in_mutexes[element] = true + end + + if not is_vararg and not has_argument then + add_mutex(mutex) + end + end + + for _, option in ipairs(self._options) do + if not elements_in_mutexes[option] and not option:_is_vararg() then + add_element(option) + end + end + + -- Add usages for positional arguments, together with one mutex containing them, if they are in a mutex. + for _, argument in ipairs(self._arguments) do + -- Pick a mutex as a part of which to show this argument, take the first one that's still available. + local mutex + + if elements_in_mutexes[argument] then + for _, argument_mutex in ipairs(argument_to_mutexes[argument]) do + if not added_mutexes[argument_mutex] then + mutex = argument_mutex + end + end + end + + if mutex then + add_mutex(mutex, argument) + else + add_element(argument) + end + end + + for _, mutex in ipairs(self._mutexes) do + add_mutex(mutex) + end + + for _, option in ipairs(self._options) do + add_element(option) + end + + if #self._commands > 0 then + if self._require_command then + add("<command>") + else + add("[<command>]") + end + + add("...") + end + + return table.concat(lines, "\n") +end + +local function split_lines(s) + if s == "" then + return {} + end + + local lines = {} + + if s:sub(-1) ~= "\n" then + s = s .. "\n" + end + + for line in s:gmatch("([^\n]*)\n") do + table.insert(lines, line) + end + + return lines +end + +local function autowrap_line(line, max_length) + -- Algorithm for splitting lines is simple and greedy. + local result_lines = {} + + -- Preserve original indentation of the line, put this at the beginning of each result line. + -- If the first word looks like a list marker ('*', '+', or '-'), add spaces so that starts + -- of the second and the following lines vertically align with the start of the second word. + local indentation = line:match("^ *") + + if line:find("^ *[%*%+%-]") then + indentation = indentation .. " " .. line:match("^ *[%*%+%-]( *)") + end + + -- Parts of the last line being assembled. + local line_parts = {} + + -- Length of the current line. + local line_length = 0 + + -- Index of the next character to consider. + local index = 1 + + while true do + local word_start, word_finish, word = line:find("([^ ]+)", index) + + if not word_start then + -- Ignore trailing spaces, if any. + break + end + + local preceding_spaces = line:sub(index, word_start - 1) + index = word_finish + 1 + + if (#line_parts == 0) or (line_length + #preceding_spaces + #word <= max_length) then + -- Either this is the very first word or it fits as an addition to the current line, add it. + table.insert(line_parts, preceding_spaces) -- For the very first word this adds the indentation. + table.insert(line_parts, word) + line_length = line_length + #preceding_spaces + #word + else + -- Does not fit, finish current line and put the word into a new one. + table.insert(result_lines, table.concat(line_parts)) + line_parts = {indentation, word} + line_length = #indentation + #word + end + end + + if #line_parts > 0 then + table.insert(result_lines, table.concat(line_parts)) + end + + if #result_lines == 0 then + -- Preserve empty lines. + result_lines[1] = "" + end + + return result_lines +end + +-- Automatically wraps lines within given array, +-- attempting to limit line length to `max_length`. +-- Existing line splits are preserved. +local function autowrap(lines, max_length) + local result_lines = {} + + for _, line in ipairs(lines) do + local autowrapped_lines = autowrap_line(line, max_length) + + for _, autowrapped_line in ipairs(autowrapped_lines) do + table.insert(result_lines, autowrapped_line) + end + end + + return result_lines +end + +function Parser:_get_element_help(element) + local label_lines = element:_get_label_lines() + local description_lines = split_lines(element:_get_description()) + + local result_lines = {} + + -- All label lines should have the same length (except the last one, it has no comma). + -- If too long, start description after all the label lines. + -- Otherwise, combine label and description lines. + + local usage_margin_len = self:_inherit_property("help_usage_margin", 3) + local usage_margin = (" "):rep(usage_margin_len) + local description_margin_len = self:_inherit_property("help_description_margin", 25) + local description_margin = (" "):rep(description_margin_len) + + local help_max_width = self:_inherit_property("help_max_width") + + if help_max_width then + local description_max_width = math.max(help_max_width - description_margin_len, 10) + description_lines = autowrap(description_lines, description_max_width) + end + + if #label_lines[1] >= (description_margin_len - usage_margin_len) then + for _, label_line in ipairs(label_lines) do + table.insert(result_lines, usage_margin .. label_line) + end + + for _, description_line in ipairs(description_lines) do + table.insert(result_lines, description_margin .. description_line) + end + else + for i = 1, math.max(#label_lines, #description_lines) do + local label_line = label_lines[i] + local description_line = description_lines[i] + + local line = "" + + if label_line then + line = usage_margin .. label_line + end + + if description_line and description_line ~= "" then + line = line .. (" "):rep(description_margin_len - #line) .. description_line + end + + table.insert(result_lines, line) + end + end + + return table.concat(result_lines, "\n") +end + +local function get_group_types(group) + local types = {} + + for _, element in ipairs(group) do + types[getmetatable(element)] = true + end + + return types +end + +function Parser:_add_group_help(blocks, added_elements, label, elements) + local buf = {label} + + for _, element in ipairs(elements) do + if not element._hidden and not added_elements[element] then + added_elements[element] = true + table.insert(buf, self:_get_element_help(element)) + end + end + + if #buf > 1 then + table.insert(blocks, table.concat(buf, ("\n"):rep(self:_inherit_property("help_vertical_space", 0) + 1))) + end +end + +function Parser:get_help() + if self._help then + return self._help + end + + local blocks = {self:get_usage()} + + local help_max_width = self:_inherit_property("help_max_width") + + if self._description then + local description = self._description + + if help_max_width then + description = table.concat(autowrap(split_lines(description), help_max_width), "\n") + end + + table.insert(blocks, description) + end + + -- 1. Put groups containing arguments first, then other arguments. + -- 2. Put remaining groups containing options, then other options. + -- 3. Put remaining groups containing commands, then other commands. + -- Assume that an element can't be in several groups. + local groups_by_type = { + [Argument] = {}, + [Option] = {}, + [Command] = {} + } + + for _, group in ipairs(self._groups) do + local group_types = get_group_types(group) + + for _, mt in ipairs({Argument, Option, Command}) do + if group_types[mt] then + table.insert(groups_by_type[mt], group) + break + end + end + end + + local default_groups = { + {name = "Arguments", type = Argument, elements = self._arguments}, + {name = "Options", type = Option, elements = self._options}, + {name = "Commands", type = Command, elements = self._commands} + } + + local added_elements = {} + + for _, default_group in ipairs(default_groups) do + local type_groups = groups_by_type[default_group.type] + + for _, group in ipairs(type_groups) do + self:_add_group_help(blocks, added_elements, group.name .. ":", group) + end + + local default_label = default_group.name .. ":" + + if #type_groups > 0 then + default_label = "Other " .. default_label:gsub("^.", string.lower) + end + + self:_add_group_help(blocks, added_elements, default_label, default_group.elements) + end + + if self._epilog then + local epilog = self._epilog + + if help_max_width then + epilog = table.concat(autowrap(split_lines(epilog), help_max_width), "\n") + end + + table.insert(blocks, epilog) + end + + return table.concat(blocks, "\n\n") +end + +local function get_tip(context, wrong_name) + local context_pool = {} + local possible_name + local possible_names = {} + + for name in pairs(context) do + if type(name) == "string" then + for i = 1, #name do + possible_name = name:sub(1, i - 1) .. name:sub(i + 1) + + if not context_pool[possible_name] then + context_pool[possible_name] = {} + end + + table.insert(context_pool[possible_name], name) + end + end + end + + for i = 1, #wrong_name + 1 do + possible_name = wrong_name:sub(1, i - 1) .. wrong_name:sub(i + 1) + + if context[possible_name] then + possible_names[possible_name] = true + elseif context_pool[possible_name] then + for _, name in ipairs(context_pool[possible_name]) do + possible_names[name] = true + end + end + end + + local first = next(possible_names) + + if first then + if next(possible_names, first) then + local possible_names_arr = {} + + for name in pairs(possible_names) do + table.insert(possible_names_arr, "'" .. name .. "'") + end + + table.sort(possible_names_arr) + return "\nDid you mean one of these: " .. table.concat(possible_names_arr, " ") .. "?" + else + return "\nDid you mean '" .. first .. "'?" + end + else + return "" + end +end + +local ElementState = class({ + invocations = 0 +}) + +function ElementState:__call(state, element) + self.state = state + self.result = state.result + self.element = element + self.target = element._target or element:_get_default_target() + self.action, self.result[self.target] = element:_get_action() + return self +end + +function ElementState:error(fmt, ...) + self.state:error(fmt, ...) +end + +function ElementState:convert(argument, index) + local converter = self.element._convert + + if converter then + local ok, err + + if type(converter) == "function" then + ok, err = converter(argument) + elseif type(converter[index]) == "function" then + ok, err = converter[index](argument) + else + ok = converter[argument] + end + + if ok == nil then + self:error(err and "%s" or "malformed argument '%s'", err or argument) + end + + argument = ok + end + + return argument +end + +function ElementState:default(mode) + return self.element._defmode:find(mode) and self.element._default +end + +local function bound(noun, min, max, is_max) + local res = "" + + if min ~= max then + res = "at " .. (is_max and "most" or "least") .. " " + end + + local number = is_max and max or min + return res .. tostring(number) .. " " .. noun .. (number == 1 and "" or "s") +end + +function ElementState:set_name(alias) + self.name = ("%s '%s'"):format(alias and "option" or "argument", alias or self.element._name) +end + +function ElementState:invoke() + self.open = true + self.overwrite = false + + if self.invocations >= self.element._maxcount then + if self.element._overwrite then + self.overwrite = true + else + local num_times_repr = bound("time", self.element._mincount, self.element._maxcount, true) + self:error("%s must be used %s", self.name, num_times_repr) + end + else + self.invocations = self.invocations + 1 + end + + self.args = {} + + if self.element._maxargs <= 0 then + self:close() + end + + return self.open +end + +function ElementState:pass(argument) + argument = self:convert(argument, #self.args + 1) + table.insert(self.args, argument) + + if #self.args >= self.element._maxargs then + self:close() + end + + return self.open +end + +function ElementState:complete_invocation() + while #self.args < self.element._minargs do + self:pass(self.element._default) + end +end + +function ElementState:close() + if self.open then + self.open = false + + if #self.args < self.element._minargs then + if self:default("a") then + self:complete_invocation() + else + if #self.args == 0 then + if getmetatable(self.element) == Argument then + self:error("missing %s", self.name) + elseif self.element._maxargs == 1 then + self:error("%s requires an argument", self.name) + end + end + + self:error("%s requires %s", self.name, bound("argument", self.element._minargs, self.element._maxargs)) + end + end + + local args + + if self.element._maxargs == 0 then + args = self.args[1] + elseif self.element._maxargs == 1 then + if self.element._minargs == 0 and self.element._mincount ~= self.element._maxcount then + args = self.args + else + args = self.args[1] + end + else + args = self.args + end + + self.action(self.result, self.target, args, self.overwrite) + end +end + +local ParseState = class({ + result = {}, + options = {}, + arguments = {}, + argument_i = 1, + element_to_mutexes = {}, + mutex_to_element_state = {}, + command_actions = {} +}) + +function ParseState:__call(parser, error_handler) + self.parser = parser + self.error_handler = error_handler + self.charset = parser:_update_charset() + self:switch(parser) + return self +end + +function ParseState:error(fmt, ...) + self.error_handler(self.parser, fmt:format(...)) +end + +function ParseState:switch(parser) + self.parser = parser + + if parser._action then + table.insert(self.command_actions, {action = parser._action, name = parser._name}) + end + + for _, option in ipairs(parser._options) do + option = ElementState(self, option) + table.insert(self.options, option) + + for _, alias in ipairs(option.element._aliases) do + self.options[alias] = option + end + end + + for _, mutex in ipairs(parser._mutexes) do + for _, element in ipairs(mutex) do + if not self.element_to_mutexes[element] then + self.element_to_mutexes[element] = {} + end + + table.insert(self.element_to_mutexes[element], mutex) + end + end + + for _, argument in ipairs(parser._arguments) do + argument = ElementState(self, argument) + table.insert(self.arguments, argument) + argument:set_name() + argument:invoke() + end + + self.handle_options = parser._handle_options + self.argument = self.arguments[self.argument_i] + self.commands = parser._commands + + for _, command in ipairs(self.commands) do + for _, alias in ipairs(command._aliases) do + self.commands[alias] = command + end + end +end + +function ParseState:get_option(name) + local option = self.options[name] + + if not option then + self:error("unknown option '%s'%s", name, get_tip(self.options, name)) + else + return option + end +end + +function ParseState:get_command(name) + local command = self.commands[name] + + if not command then + if #self.commands > 0 then + self:error("unknown command '%s'%s", name, get_tip(self.commands, name)) + else + self:error("too many arguments") + end + else + return command + end +end + +function ParseState:check_mutexes(element_state) + if self.element_to_mutexes[element_state.element] then + for _, mutex in ipairs(self.element_to_mutexes[element_state.element]) do + local used_element_state = self.mutex_to_element_state[mutex] + + if used_element_state and used_element_state ~= element_state then + self:error("%s can not be used together with %s", element_state.name, used_element_state.name) + else + self.mutex_to_element_state[mutex] = element_state + end + end + end +end + +function ParseState:invoke(option, name) + self:close() + option:set_name(name) + self:check_mutexes(option, name) + + if option:invoke() then + self.option = option + end +end + +function ParseState:pass(arg) + if self.option then + if not self.option:pass(arg) then + self.option = nil + end + elseif self.argument then + self:check_mutexes(self.argument) + + if not self.argument:pass(arg) then + self.argument_i = self.argument_i + 1 + self.argument = self.arguments[self.argument_i] + end + else + local command = self:get_command(arg) + self.result[command._target or command._name] = true + + if self.parser._command_target then + self.result[self.parser._command_target] = command._name + end + + self:switch(command) + end +end + +function ParseState:close() + if self.option then + self.option:close() + self.option = nil + end +end + +function ParseState:finalize() + self:close() + + for i = self.argument_i, #self.arguments do + local argument = self.arguments[i] + if #argument.args == 0 and argument:default("u") then + argument:complete_invocation() + else + argument:close() + end + end + + if self.parser._require_command and #self.commands > 0 then + self:error("a command is required") + end + + for _, option in ipairs(self.options) do + option.name = option.name or ("option '%s'"):format(option.element._name) + + if option.invocations == 0 then + if option:default("u") then + option:invoke() + option:complete_invocation() + option:close() + end + end + + local mincount = option.element._mincount + + if option.invocations < mincount then + if option:default("a") then + while option.invocations < mincount do + option:invoke() + option:close() + end + elseif option.invocations == 0 then + self:error("missing %s", option.name) + else + self:error("%s must be used %s", option.name, bound("time", mincount, option.element._maxcount)) + end + end + end + + for i = #self.command_actions, 1, -1 do + self.command_actions[i].action(self.result, self.command_actions[i].name) + end +end + +function ParseState:parse(args) + for _, arg in ipairs(args) do + local plain = true + + if self.handle_options then + local first = arg:sub(1, 1) + + if self.charset[first] then + if #arg > 1 then + plain = false + + if arg:sub(2, 2) == first then + if #arg == 2 then + if self.options[arg] then + local option = self:get_option(arg) + self:invoke(option, arg) + else + self:close() + end + + self.handle_options = false + else + local equals = arg:find "=" + if equals then + local name = arg:sub(1, equals - 1) + local option = self:get_option(name) + + if option.element._maxargs <= 0 then + self:error("option '%s' does not take arguments", name) + end + + self:invoke(option, name) + self:pass(arg:sub(equals + 1)) + else + local option = self:get_option(arg) + self:invoke(option, arg) + end + end + else + for i = 2, #arg do + local name = first .. arg:sub(i, i) + local option = self:get_option(name) + self:invoke(option, name) + + if i ~= #arg and option.element._maxargs > 0 then + self:pass(arg:sub(i + 1)) + break + end + end + end + end + end + end + + if plain then + self:pass(arg) + end + end + + self:finalize() + return self.result +end + +function Parser:error(msg) + io.stderr:write(("%s\n\nError: %s\n"):format(self:get_usage(), msg)) + os.exit(1) +end + +-- Compatibility with strict.lua and other checkers: +local default_cmdline = rawget(_G, "arg") or {} + +function Parser:_parse(args, error_handler) + return ParseState(self, error_handler):parse(args or default_cmdline) +end + +function Parser:parse(args) + return self:_parse(args, self.error) +end + +local function xpcall_error_handler(err) + return tostring(err) .. "\noriginal " .. debug.traceback("", 2):sub(2) +end + +function Parser:pparse(args) + local parse_error + + local ok, result = xpcall(function() + return self:_parse(args, function(_, err) + parse_error = err + error(err, 0) + end) + end, xpcall_error_handler) + + if ok then + return true, result + elseif not parse_error then + error(result, 0) + else + return false, parse_error + end +end + +local argparse = {} + +argparse.version = "0.6.0" + +setmetatable(argparse, {__call = function(_, ...) + return Parser(default_cmdline[0]):add_help(true)(...) +end}) + +return argparse diff --git a/contrib/lpeg/CMakeLists.txt b/contrib/lua-lpeg/CMakeLists.txt index 2362aac9c..2362aac9c 100644 --- a/contrib/lpeg/CMakeLists.txt +++ b/contrib/lua-lpeg/CMakeLists.txt diff --git a/contrib/lpeg/LICENSE b/contrib/lua-lpeg/LICENSE index cea2d8b5e..cea2d8b5e 100644 --- a/contrib/lpeg/LICENSE +++ b/contrib/lua-lpeg/LICENSE diff --git a/contrib/lpeg/lpcap.c b/contrib/lua-lpeg/lpcap.c index c9085de06..c9085de06 100644 --- a/contrib/lpeg/lpcap.c +++ b/contrib/lua-lpeg/lpcap.c diff --git a/contrib/lpeg/lpcap.h b/contrib/lua-lpeg/lpcap.h index d762fdcfa..d762fdcfa 100644 --- a/contrib/lpeg/lpcap.h +++ b/contrib/lua-lpeg/lpcap.h diff --git a/contrib/lpeg/lpcode.c b/contrib/lua-lpeg/lpcode.c index 6feefeb43..6feefeb43 100644 --- a/contrib/lpeg/lpcode.c +++ b/contrib/lua-lpeg/lpcode.c diff --git a/contrib/lpeg/lpcode.h b/contrib/lua-lpeg/lpcode.h index 896d3c79a..896d3c79a 100644 --- a/contrib/lpeg/lpcode.h +++ b/contrib/lua-lpeg/lpcode.h diff --git a/contrib/lpeg/lpprint.c b/contrib/lua-lpeg/lpprint.c index 174d1687b..174d1687b 100644 --- a/contrib/lpeg/lpprint.c +++ b/contrib/lua-lpeg/lpprint.c diff --git a/contrib/lpeg/lpprint.h b/contrib/lua-lpeg/lpprint.h index 632976076..632976076 100644 --- a/contrib/lpeg/lpprint.h +++ b/contrib/lua-lpeg/lpprint.h diff --git a/contrib/lpeg/lptree.c b/contrib/lua-lpeg/lptree.c index f1016c3db..f1016c3db 100644 --- a/contrib/lpeg/lptree.c +++ b/contrib/lua-lpeg/lptree.c diff --git a/contrib/lpeg/lptree.h b/contrib/lua-lpeg/lptree.h index 38a668e9f..38a668e9f 100644 --- a/contrib/lpeg/lptree.h +++ b/contrib/lua-lpeg/lptree.h diff --git a/contrib/lpeg/lptypes.h b/contrib/lua-lpeg/lptypes.h index 78dff6220..78dff6220 100644 --- a/contrib/lpeg/lptypes.h +++ b/contrib/lua-lpeg/lptypes.h diff --git a/contrib/lpeg/lpvm.c b/contrib/lua-lpeg/lpvm.c index eaf2ebfd7..eaf2ebfd7 100644 --- a/contrib/lpeg/lpvm.c +++ b/contrib/lua-lpeg/lpvm.c diff --git a/contrib/lpeg/lpvm.h b/contrib/lua-lpeg/lpvm.h index 757b9e135..757b9e135 100644 --- a/contrib/lpeg/lpvm.h +++ b/contrib/lua-lpeg/lpvm.h diff --git a/contrib/moses/LICENSE b/contrib/lua-moses/LICENSE index f06dce3e8..f06dce3e8 100644 --- a/contrib/moses/LICENSE +++ b/contrib/lua-moses/LICENSE diff --git a/contrib/moses/moses.lua b/contrib/lua-moses/moses.lua index bb67dccad..bb67dccad 100644 --- a/contrib/moses/moses.lua +++ b/contrib/lua-moses/moses.lua diff --git a/contrib/torch/decisiontree/CMakeLists.txt b/contrib/lua-torch/decisiontree/CMakeLists.txt index 6434eddd9..b94b4deb2 100644 --- a/contrib/torch/decisiontree/CMakeLists.txt +++ b/contrib/lua-torch/decisiontree/CMakeLists.txt @@ -1,7 +1,4 @@ -SET(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}) - -CMAKE_MINIMUM_REQUIRED(VERSION 2.6 FATAL_ERROR) -CMAKE_POLICY(VERSION 2.6) +LIST(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}") SET(src init.c diff --git a/contrib/torch/decisiontree/CartNode.lua b/contrib/lua-torch/decisiontree/CartNode.lua index e6a85006c..e6a85006c 100644 --- a/contrib/torch/decisiontree/CartNode.lua +++ b/contrib/lua-torch/decisiontree/CartNode.lua diff --git a/contrib/torch/decisiontree/CartTrainer.lua b/contrib/lua-torch/decisiontree/CartTrainer.lua index 63ae6c148..63ae6c148 100644 --- a/contrib/torch/decisiontree/CartTrainer.lua +++ b/contrib/lua-torch/decisiontree/CartTrainer.lua diff --git a/contrib/torch/decisiontree/CartTree.lua b/contrib/lua-torch/decisiontree/CartTree.lua index c74dfda9e..c74dfda9e 100644 --- a/contrib/torch/decisiontree/CartTree.lua +++ b/contrib/lua-torch/decisiontree/CartTree.lua diff --git a/contrib/torch/decisiontree/DFD.lua b/contrib/lua-torch/decisiontree/DFD.lua index e4746212a..e4746212a 100644 --- a/contrib/torch/decisiontree/DFD.lua +++ b/contrib/lua-torch/decisiontree/DFD.lua diff --git a/contrib/torch/decisiontree/DataSet.lua b/contrib/lua-torch/decisiontree/DataSet.lua index 15058a7c6..15058a7c6 100644 --- a/contrib/torch/decisiontree/DataSet.lua +++ b/contrib/lua-torch/decisiontree/DataSet.lua diff --git a/contrib/torch/decisiontree/DecisionForest.lua b/contrib/lua-torch/decisiontree/DecisionForest.lua index cac748e7e..cac748e7e 100644 --- a/contrib/torch/decisiontree/DecisionForest.lua +++ b/contrib/lua-torch/decisiontree/DecisionForest.lua diff --git a/contrib/torch/decisiontree/DecisionForestTrainer.lua b/contrib/lua-torch/decisiontree/DecisionForestTrainer.lua index fc903678b..fc903678b 100644 --- a/contrib/torch/decisiontree/DecisionForestTrainer.lua +++ b/contrib/lua-torch/decisiontree/DecisionForestTrainer.lua diff --git a/contrib/torch/decisiontree/DecisionTree.lua b/contrib/lua-torch/decisiontree/DecisionTree.lua index c61bc3757..c61bc3757 100644 --- a/contrib/torch/decisiontree/DecisionTree.lua +++ b/contrib/lua-torch/decisiontree/DecisionTree.lua diff --git a/contrib/torch/decisiontree/GBDT_common.h b/contrib/lua-torch/decisiontree/GBDT_common.h index eb993702d..eb993702d 100644 --- a/contrib/torch/decisiontree/GBDT_common.h +++ b/contrib/lua-torch/decisiontree/GBDT_common.h diff --git a/contrib/torch/decisiontree/GiniState.lua b/contrib/lua-torch/decisiontree/GiniState.lua index 6dfed2845..6dfed2845 100644 --- a/contrib/torch/decisiontree/GiniState.lua +++ b/contrib/lua-torch/decisiontree/GiniState.lua diff --git a/contrib/torch/decisiontree/GradientBoostState.lua b/contrib/lua-torch/decisiontree/GradientBoostState.lua index f268f3da8..f268f3da8 100644 --- a/contrib/torch/decisiontree/GradientBoostState.lua +++ b/contrib/lua-torch/decisiontree/GradientBoostState.lua diff --git a/contrib/torch/decisiontree/GradientBoostTrainer.lua b/contrib/lua-torch/decisiontree/GradientBoostTrainer.lua index 51299b109..51299b109 100644 --- a/contrib/torch/decisiontree/GradientBoostTrainer.lua +++ b/contrib/lua-torch/decisiontree/GradientBoostTrainer.lua diff --git a/contrib/torch/decisiontree/LICENSE b/contrib/lua-torch/decisiontree/LICENSE index 8dada3eda..8dada3eda 100644 --- a/contrib/torch/decisiontree/LICENSE +++ b/contrib/lua-torch/decisiontree/LICENSE diff --git a/contrib/torch/decisiontree/LogitBoostCriterion.lua b/contrib/lua-torch/decisiontree/LogitBoostCriterion.lua index 5b9eb6028..5b9eb6028 100644 --- a/contrib/torch/decisiontree/LogitBoostCriterion.lua +++ b/contrib/lua-torch/decisiontree/LogitBoostCriterion.lua diff --git a/contrib/torch/decisiontree/MSECriterion.lua b/contrib/lua-torch/decisiontree/MSECriterion.lua index 948c1a17e..948c1a17e 100644 --- a/contrib/torch/decisiontree/MSECriterion.lua +++ b/contrib/lua-torch/decisiontree/MSECriterion.lua diff --git a/contrib/torch/decisiontree/README.md b/contrib/lua-torch/decisiontree/README.md index db4622add..db4622add 100644 --- a/contrib/torch/decisiontree/README.md +++ b/contrib/lua-torch/decisiontree/README.md diff --git a/contrib/torch/decisiontree/RandomForestTrainer.lua b/contrib/lua-torch/decisiontree/RandomForestTrainer.lua index 41040b25b..41040b25b 100644 --- a/contrib/torch/decisiontree/RandomForestTrainer.lua +++ b/contrib/lua-torch/decisiontree/RandomForestTrainer.lua diff --git a/contrib/torch/decisiontree/Sparse2Dense.lua b/contrib/lua-torch/decisiontree/Sparse2Dense.lua index 4e5b79d2f..4e5b79d2f 100644 --- a/contrib/torch/decisiontree/Sparse2Dense.lua +++ b/contrib/lua-torch/decisiontree/Sparse2Dense.lua diff --git a/contrib/torch/decisiontree/SparseTensor.lua b/contrib/lua-torch/decisiontree/SparseTensor.lua index 4c620e618..4c620e618 100644 --- a/contrib/torch/decisiontree/SparseTensor.lua +++ b/contrib/lua-torch/decisiontree/SparseTensor.lua diff --git a/contrib/torch/decisiontree/TreeState.lua b/contrib/lua-torch/decisiontree/TreeState.lua index 3928649fd..3928649fd 100644 --- a/contrib/torch/decisiontree/TreeState.lua +++ b/contrib/lua-torch/decisiontree/TreeState.lua diff --git a/contrib/torch/decisiontree/WorkPool.lua b/contrib/lua-torch/decisiontree/WorkPool.lua index 8f473727e..8f473727e 100644 --- a/contrib/torch/decisiontree/WorkPool.lua +++ b/contrib/lua-torch/decisiontree/WorkPool.lua diff --git a/contrib/torch/decisiontree/_env.lua b/contrib/lua-torch/decisiontree/_env.lua index a92770152..a92770152 100644 --- a/contrib/torch/decisiontree/_env.lua +++ b/contrib/lua-torch/decisiontree/_env.lua diff --git a/contrib/torch/decisiontree/benchmark.lua b/contrib/lua-torch/decisiontree/benchmark.lua index 2b6a03dc6..2b6a03dc6 100644 --- a/contrib/torch/decisiontree/benchmark.lua +++ b/contrib/lua-torch/decisiontree/benchmark.lua diff --git a/contrib/torch/decisiontree/doc/benchmark.md b/contrib/lua-torch/decisiontree/doc/benchmark.md index cb8f905d6..cb8f905d6 100644 --- a/contrib/torch/decisiontree/doc/benchmark.md +++ b/contrib/lua-torch/decisiontree/doc/benchmark.md diff --git a/contrib/torch/decisiontree/error.h b/contrib/lua-torch/decisiontree/error.h index 18df3c939..18df3c939 100644 --- a/contrib/torch/decisiontree/error.h +++ b/contrib/lua-torch/decisiontree/error.h diff --git a/contrib/torch/decisiontree/generic/CartTree.c b/contrib/lua-torch/decisiontree/generic/CartTree.c index eb29fcf02..eb29fcf02 100644 --- a/contrib/torch/decisiontree/generic/CartTree.c +++ b/contrib/lua-torch/decisiontree/generic/CartTree.c diff --git a/contrib/torch/decisiontree/generic/DFD.c b/contrib/lua-torch/decisiontree/generic/DFD.c index 599c4d794..599c4d794 100644 --- a/contrib/torch/decisiontree/generic/DFD.c +++ b/contrib/lua-torch/decisiontree/generic/DFD.c diff --git a/contrib/torch/decisiontree/generic/GBDT.c b/contrib/lua-torch/decisiontree/generic/GBDT.c index 31f5b025d..31f5b025d 100644 --- a/contrib/torch/decisiontree/generic/GBDT.c +++ b/contrib/lua-torch/decisiontree/generic/GBDT.c diff --git a/contrib/torch/decisiontree/generic/GBDT_internal.c b/contrib/lua-torch/decisiontree/generic/GBDT_internal.c index 739aabf25..739aabf25 100644 --- a/contrib/torch/decisiontree/generic/GBDT_internal.c +++ b/contrib/lua-torch/decisiontree/generic/GBDT_internal.c diff --git a/contrib/torch/decisiontree/generic/GBDT_internal.h b/contrib/lua-torch/decisiontree/generic/GBDT_internal.h index 7119365cf..7119365cf 100644 --- a/contrib/torch/decisiontree/generic/GBDT_internal.h +++ b/contrib/lua-torch/decisiontree/generic/GBDT_internal.h diff --git a/contrib/torch/decisiontree/generic/LogitBoostCriterion.c b/contrib/lua-torch/decisiontree/generic/LogitBoostCriterion.c index f2ea1ef09..f2ea1ef09 100644 --- a/contrib/torch/decisiontree/generic/LogitBoostCriterion.c +++ b/contrib/lua-torch/decisiontree/generic/LogitBoostCriterion.c diff --git a/contrib/torch/decisiontree/generic/S2D.c b/contrib/lua-torch/decisiontree/generic/S2D.c index 2392ee7c8..2392ee7c8 100644 --- a/contrib/torch/decisiontree/generic/S2D.c +++ b/contrib/lua-torch/decisiontree/generic/S2D.c diff --git a/contrib/torch/decisiontree/hash_map.c b/contrib/lua-torch/decisiontree/hash_map.c index 2c679e207..2c679e207 100644 --- a/contrib/torch/decisiontree/hash_map.c +++ b/contrib/lua-torch/decisiontree/hash_map.c diff --git a/contrib/torch/decisiontree/hash_map.h b/contrib/lua-torch/decisiontree/hash_map.h index 5b215e4ca..5b215e4ca 100644 --- a/contrib/torch/decisiontree/hash_map.h +++ b/contrib/lua-torch/decisiontree/hash_map.h diff --git a/contrib/torch/decisiontree/init.c b/contrib/lua-torch/decisiontree/init.c index 276241e8e..276241e8e 100644 --- a/contrib/torch/decisiontree/init.c +++ b/contrib/lua-torch/decisiontree/init.c diff --git a/contrib/torch/decisiontree/init.lua b/contrib/lua-torch/decisiontree/init.lua index 26f790b60..26f790b60 100644 --- a/contrib/torch/decisiontree/init.lua +++ b/contrib/lua-torch/decisiontree/init.lua diff --git a/contrib/torch/decisiontree/internal_hash_map.h b/contrib/lua-torch/decisiontree/internal_hash_map.h index bc8c523ef..bc8c523ef 100644 --- a/contrib/torch/decisiontree/internal_hash_map.h +++ b/contrib/lua-torch/decisiontree/internal_hash_map.h diff --git a/contrib/torch/decisiontree/khash.h b/contrib/lua-torch/decisiontree/khash.h index 20e994063..20e994063 100644 --- a/contrib/torch/decisiontree/khash.h +++ b/contrib/lua-torch/decisiontree/khash.h diff --git a/contrib/torch/decisiontree/math.lua b/contrib/lua-torch/decisiontree/math.lua index eb71b31ed..eb71b31ed 100644 --- a/contrib/torch/decisiontree/math.lua +++ b/contrib/lua-torch/decisiontree/math.lua diff --git a/contrib/torch/decisiontree/rocks/decisiontree-scm-1.rockspec b/contrib/lua-torch/decisiontree/rocks/decisiontree-scm-1.rockspec index d5d916275..d5d916275 100644 --- a/contrib/torch/decisiontree/rocks/decisiontree-scm-1.rockspec +++ b/contrib/lua-torch/decisiontree/rocks/decisiontree-scm-1.rockspec diff --git a/contrib/torch/decisiontree/test.lua b/contrib/lua-torch/decisiontree/test.lua index 80510a4f6..80510a4f6 100644 --- a/contrib/torch/decisiontree/test.lua +++ b/contrib/lua-torch/decisiontree/test.lua diff --git a/contrib/torch/decisiontree/utils.h b/contrib/lua-torch/decisiontree/utils.h index 8a0196a58..8a0196a58 100644 --- a/contrib/torch/decisiontree/utils.h +++ b/contrib/lua-torch/decisiontree/utils.h diff --git a/contrib/torch/decisiontree/utils.lua b/contrib/lua-torch/decisiontree/utils.lua index c32c3d08b..c32c3d08b 100644 --- a/contrib/torch/decisiontree/utils.lua +++ b/contrib/lua-torch/decisiontree/utils.lua diff --git a/contrib/torch/nn/.gitignore b/contrib/lua-torch/nn/.gitignore index e0fa91eda..e0fa91eda 100644 --- a/contrib/torch/nn/.gitignore +++ b/contrib/lua-torch/nn/.gitignore diff --git a/contrib/torch/nn/.luacheckrc b/contrib/lua-torch/nn/.luacheckrc index 3d358e9c0..3d358e9c0 100644 --- a/contrib/torch/nn/.luacheckrc +++ b/contrib/lua-torch/nn/.luacheckrc diff --git a/contrib/torch/nn/.travis.yml b/contrib/lua-torch/nn/.travis.yml index 1d10e0fb5..1d10e0fb5 100644 --- a/contrib/torch/nn/.travis.yml +++ b/contrib/lua-torch/nn/.travis.yml diff --git a/contrib/torch/nn/Abs.lua b/contrib/lua-torch/nn/Abs.lua index b32b64f79..b32b64f79 100644 --- a/contrib/torch/nn/Abs.lua +++ b/contrib/lua-torch/nn/Abs.lua diff --git a/contrib/torch/nn/AbsCriterion.lua b/contrib/lua-torch/nn/AbsCriterion.lua index 65e2f8ae1..65e2f8ae1 100644 --- a/contrib/torch/nn/AbsCriterion.lua +++ b/contrib/lua-torch/nn/AbsCriterion.lua diff --git a/contrib/torch/nn/Add.lua b/contrib/lua-torch/nn/Add.lua index d071a15b3..d071a15b3 100644 --- a/contrib/torch/nn/Add.lua +++ b/contrib/lua-torch/nn/Add.lua diff --git a/contrib/torch/nn/AddConstant.lua b/contrib/lua-torch/nn/AddConstant.lua index b686d719c..b686d719c 100644 --- a/contrib/torch/nn/AddConstant.lua +++ b/contrib/lua-torch/nn/AddConstant.lua diff --git a/contrib/torch/nn/BCECriterion.lua b/contrib/lua-torch/nn/BCECriterion.lua index 8bb5f8178..8bb5f8178 100644 --- a/contrib/torch/nn/BCECriterion.lua +++ b/contrib/lua-torch/nn/BCECriterion.lua diff --git a/contrib/torch/nn/BatchNormalization.lua b/contrib/lua-torch/nn/BatchNormalization.lua index 8dfc576b3..8dfc576b3 100644 --- a/contrib/torch/nn/BatchNormalization.lua +++ b/contrib/lua-torch/nn/BatchNormalization.lua diff --git a/contrib/torch/nn/Bilinear.lua b/contrib/lua-torch/nn/Bilinear.lua index 9350b03ec..9350b03ec 100644 --- a/contrib/torch/nn/Bilinear.lua +++ b/contrib/lua-torch/nn/Bilinear.lua diff --git a/contrib/torch/nn/Bottle.lua b/contrib/lua-torch/nn/Bottle.lua index 6dee432f5..6dee432f5 100644 --- a/contrib/torch/nn/Bottle.lua +++ b/contrib/lua-torch/nn/Bottle.lua diff --git a/contrib/torch/nn/CAdd.lua b/contrib/lua-torch/nn/CAdd.lua index 1d7b45726..1d7b45726 100644 --- a/contrib/torch/nn/CAdd.lua +++ b/contrib/lua-torch/nn/CAdd.lua diff --git a/contrib/torch/nn/CAddTable.lua b/contrib/lua-torch/nn/CAddTable.lua index 79deb7e9b..79deb7e9b 100644 --- a/contrib/torch/nn/CAddTable.lua +++ b/contrib/lua-torch/nn/CAddTable.lua diff --git a/contrib/torch/nn/CAddTensorTable.lua b/contrib/lua-torch/nn/CAddTensorTable.lua index 16efe4450..16efe4450 100644 --- a/contrib/torch/nn/CAddTensorTable.lua +++ b/contrib/lua-torch/nn/CAddTensorTable.lua diff --git a/contrib/torch/nn/CDivTable.lua b/contrib/lua-torch/nn/CDivTable.lua index bf044c9af..bf044c9af 100644 --- a/contrib/torch/nn/CDivTable.lua +++ b/contrib/lua-torch/nn/CDivTable.lua diff --git a/contrib/torch/nn/CMakeLists.txt b/contrib/lua-torch/nn/CMakeLists.txt index cebddfbfc..cebddfbfc 100644 --- a/contrib/torch/nn/CMakeLists.txt +++ b/contrib/lua-torch/nn/CMakeLists.txt diff --git a/contrib/torch/nn/CMaxTable.lua b/contrib/lua-torch/nn/CMaxTable.lua index 845e38d23..845e38d23 100644 --- a/contrib/torch/nn/CMaxTable.lua +++ b/contrib/lua-torch/nn/CMaxTable.lua diff --git a/contrib/torch/nn/CMinTable.lua b/contrib/lua-torch/nn/CMinTable.lua index 25b9a19a2..25b9a19a2 100644 --- a/contrib/torch/nn/CMinTable.lua +++ b/contrib/lua-torch/nn/CMinTable.lua diff --git a/contrib/torch/nn/CMul.lua b/contrib/lua-torch/nn/CMul.lua index 890169761..890169761 100644 --- a/contrib/torch/nn/CMul.lua +++ b/contrib/lua-torch/nn/CMul.lua diff --git a/contrib/torch/nn/CMulTable.lua b/contrib/lua-torch/nn/CMulTable.lua index b47378e83..b47378e83 100644 --- a/contrib/torch/nn/CMulTable.lua +++ b/contrib/lua-torch/nn/CMulTable.lua diff --git a/contrib/torch/nn/CONTRIBUTING.md b/contrib/lua-torch/nn/CONTRIBUTING.md index cc800154e..cc800154e 100644 --- a/contrib/torch/nn/CONTRIBUTING.md +++ b/contrib/lua-torch/nn/CONTRIBUTING.md diff --git a/contrib/torch/nn/COPYRIGHT.txt b/contrib/lua-torch/nn/COPYRIGHT.txt index bc002b78a..bc002b78a 100644 --- a/contrib/torch/nn/COPYRIGHT.txt +++ b/contrib/lua-torch/nn/COPYRIGHT.txt diff --git a/contrib/torch/nn/CReLU.lua b/contrib/lua-torch/nn/CReLU.lua index 8da6e7974..8da6e7974 100644 --- a/contrib/torch/nn/CReLU.lua +++ b/contrib/lua-torch/nn/CReLU.lua diff --git a/contrib/torch/nn/CSubTable.lua b/contrib/lua-torch/nn/CSubTable.lua index eb7492055..eb7492055 100644 --- a/contrib/torch/nn/CSubTable.lua +++ b/contrib/lua-torch/nn/CSubTable.lua diff --git a/contrib/torch/nn/Clamp.lua b/contrib/lua-torch/nn/Clamp.lua index 36397a157..36397a157 100644 --- a/contrib/torch/nn/Clamp.lua +++ b/contrib/lua-torch/nn/Clamp.lua diff --git a/contrib/torch/nn/ClassNLLCriterion.lua b/contrib/lua-torch/nn/ClassNLLCriterion.lua index dae0e6685..dae0e6685 100644 --- a/contrib/torch/nn/ClassNLLCriterion.lua +++ b/contrib/lua-torch/nn/ClassNLLCriterion.lua diff --git a/contrib/torch/nn/ClassSimplexCriterion.lua b/contrib/lua-torch/nn/ClassSimplexCriterion.lua index 9cabc011f..9cabc011f 100644 --- a/contrib/torch/nn/ClassSimplexCriterion.lua +++ b/contrib/lua-torch/nn/ClassSimplexCriterion.lua diff --git a/contrib/torch/nn/Collapse.lua b/contrib/lua-torch/nn/Collapse.lua index a088608ca..a088608ca 100644 --- a/contrib/torch/nn/Collapse.lua +++ b/contrib/lua-torch/nn/Collapse.lua diff --git a/contrib/torch/nn/Concat.lua b/contrib/lua-torch/nn/Concat.lua index d7e3ee711..d7e3ee711 100644 --- a/contrib/torch/nn/Concat.lua +++ b/contrib/lua-torch/nn/Concat.lua diff --git a/contrib/torch/nn/ConcatTable.lua b/contrib/lua-torch/nn/ConcatTable.lua index 742719344..742719344 100644 --- a/contrib/torch/nn/ConcatTable.lua +++ b/contrib/lua-torch/nn/ConcatTable.lua diff --git a/contrib/torch/nn/Constant.lua b/contrib/lua-torch/nn/Constant.lua index 07773feb2..07773feb2 100644 --- a/contrib/torch/nn/Constant.lua +++ b/contrib/lua-torch/nn/Constant.lua diff --git a/contrib/torch/nn/Container.lua b/contrib/lua-torch/nn/Container.lua index 7e264bab9..7e264bab9 100644 --- a/contrib/torch/nn/Container.lua +++ b/contrib/lua-torch/nn/Container.lua diff --git a/contrib/torch/nn/Contiguous.lua b/contrib/lua-torch/nn/Contiguous.lua index f9974ce5a..f9974ce5a 100755 --- a/contrib/torch/nn/Contiguous.lua +++ b/contrib/lua-torch/nn/Contiguous.lua diff --git a/contrib/torch/nn/Convert.lua b/contrib/lua-torch/nn/Convert.lua index 855338dd6..855338dd6 100644 --- a/contrib/torch/nn/Convert.lua +++ b/contrib/lua-torch/nn/Convert.lua diff --git a/contrib/torch/nn/Copy.lua b/contrib/lua-torch/nn/Copy.lua index 9f83cf9b4..9f83cf9b4 100644 --- a/contrib/torch/nn/Copy.lua +++ b/contrib/lua-torch/nn/Copy.lua diff --git a/contrib/torch/nn/Cosine.lua b/contrib/lua-torch/nn/Cosine.lua index 19a9cba82..19a9cba82 100644 --- a/contrib/torch/nn/Cosine.lua +++ b/contrib/lua-torch/nn/Cosine.lua diff --git a/contrib/torch/nn/CosineDistance.lua b/contrib/lua-torch/nn/CosineDistance.lua index fe4e4b9f5..fe4e4b9f5 100644 --- a/contrib/torch/nn/CosineDistance.lua +++ b/contrib/lua-torch/nn/CosineDistance.lua diff --git a/contrib/torch/nn/CosineEmbeddingCriterion.lua b/contrib/lua-torch/nn/CosineEmbeddingCriterion.lua index d55e03130..d55e03130 100644 --- a/contrib/torch/nn/CosineEmbeddingCriterion.lua +++ b/contrib/lua-torch/nn/CosineEmbeddingCriterion.lua diff --git a/contrib/torch/nn/Criterion.lua b/contrib/lua-torch/nn/Criterion.lua index e48f06876..e48f06876 100644 --- a/contrib/torch/nn/Criterion.lua +++ b/contrib/lua-torch/nn/Criterion.lua diff --git a/contrib/torch/nn/CriterionTable.lua b/contrib/lua-torch/nn/CriterionTable.lua index 14f67bd39..14f67bd39 100644 --- a/contrib/torch/nn/CriterionTable.lua +++ b/contrib/lua-torch/nn/CriterionTable.lua diff --git a/contrib/torch/nn/CrossEntropyCriterion.lua b/contrib/lua-torch/nn/CrossEntropyCriterion.lua index 2f72cf87f..2f72cf87f 100644 --- a/contrib/torch/nn/CrossEntropyCriterion.lua +++ b/contrib/lua-torch/nn/CrossEntropyCriterion.lua diff --git a/contrib/torch/nn/Decorator.lua b/contrib/lua-torch/nn/Decorator.lua index 05fb4db92..05fb4db92 100644 --- a/contrib/torch/nn/Decorator.lua +++ b/contrib/lua-torch/nn/Decorator.lua diff --git a/contrib/torch/nn/DepthConcat.lua b/contrib/lua-torch/nn/DepthConcat.lua index f64a90eb8..f64a90eb8 100644 --- a/contrib/torch/nn/DepthConcat.lua +++ b/contrib/lua-torch/nn/DepthConcat.lua diff --git a/contrib/torch/nn/DistKLDivCriterion.lua b/contrib/lua-torch/nn/DistKLDivCriterion.lua index bfad57567..bfad57567 100644 --- a/contrib/torch/nn/DistKLDivCriterion.lua +++ b/contrib/lua-torch/nn/DistKLDivCriterion.lua diff --git a/contrib/torch/nn/DistanceRatioCriterion.lua b/contrib/lua-torch/nn/DistanceRatioCriterion.lua index 6b79d0620..6b79d0620 100644 --- a/contrib/torch/nn/DistanceRatioCriterion.lua +++ b/contrib/lua-torch/nn/DistanceRatioCriterion.lua diff --git a/contrib/torch/nn/DontCast.lua b/contrib/lua-torch/nn/DontCast.lua index b89f5436b..b89f5436b 100644 --- a/contrib/torch/nn/DontCast.lua +++ b/contrib/lua-torch/nn/DontCast.lua diff --git a/contrib/torch/nn/DotProduct.lua b/contrib/lua-torch/nn/DotProduct.lua index ccd347e6b..ccd347e6b 100644 --- a/contrib/torch/nn/DotProduct.lua +++ b/contrib/lua-torch/nn/DotProduct.lua diff --git a/contrib/torch/nn/Dropout.lua b/contrib/lua-torch/nn/Dropout.lua index 15f2f4699..15f2f4699 100644 --- a/contrib/torch/nn/Dropout.lua +++ b/contrib/lua-torch/nn/Dropout.lua diff --git a/contrib/torch/nn/ELU.lua b/contrib/lua-torch/nn/ELU.lua index 48a6caa2c..48a6caa2c 100644 --- a/contrib/torch/nn/ELU.lua +++ b/contrib/lua-torch/nn/ELU.lua diff --git a/contrib/torch/nn/ErrorMessages.lua b/contrib/lua-torch/nn/ErrorMessages.lua index a5cbed053..a5cbed053 100644 --- a/contrib/torch/nn/ErrorMessages.lua +++ b/contrib/lua-torch/nn/ErrorMessages.lua diff --git a/contrib/torch/nn/Euclidean.lua b/contrib/lua-torch/nn/Euclidean.lua index 509feff50..509feff50 100644 --- a/contrib/torch/nn/Euclidean.lua +++ b/contrib/lua-torch/nn/Euclidean.lua diff --git a/contrib/torch/nn/Exp.lua b/contrib/lua-torch/nn/Exp.lua index f41569026..f41569026 100644 --- a/contrib/torch/nn/Exp.lua +++ b/contrib/lua-torch/nn/Exp.lua diff --git a/contrib/torch/nn/FlattenTable.lua b/contrib/lua-torch/nn/FlattenTable.lua index 1c182557c..1c182557c 100644 --- a/contrib/torch/nn/FlattenTable.lua +++ b/contrib/lua-torch/nn/FlattenTable.lua diff --git a/contrib/torch/nn/GPU.lua b/contrib/lua-torch/nn/GPU.lua index 758618d8b..758618d8b 100644 --- a/contrib/torch/nn/GPU.lua +++ b/contrib/lua-torch/nn/GPU.lua diff --git a/contrib/torch/nn/GatedLinearUnit.lua b/contrib/lua-torch/nn/GatedLinearUnit.lua index 5273abfd4..5273abfd4 100644 --- a/contrib/torch/nn/GatedLinearUnit.lua +++ b/contrib/lua-torch/nn/GatedLinearUnit.lua diff --git a/contrib/torch/nn/GradientReversal.lua b/contrib/lua-torch/nn/GradientReversal.lua index c08b1dfb0..c08b1dfb0 100644 --- a/contrib/torch/nn/GradientReversal.lua +++ b/contrib/lua-torch/nn/GradientReversal.lua diff --git a/contrib/torch/nn/HardShrink.lua b/contrib/lua-torch/nn/HardShrink.lua index 85ff5909c..85ff5909c 100644 --- a/contrib/torch/nn/HardShrink.lua +++ b/contrib/lua-torch/nn/HardShrink.lua diff --git a/contrib/torch/nn/HardTanh.lua b/contrib/lua-torch/nn/HardTanh.lua index 07cfc6255..07cfc6255 100644 --- a/contrib/torch/nn/HardTanh.lua +++ b/contrib/lua-torch/nn/HardTanh.lua diff --git a/contrib/torch/nn/HingeEmbeddingCriterion.lua b/contrib/lua-torch/nn/HingeEmbeddingCriterion.lua index 13ad00f19..13ad00f19 100644 --- a/contrib/torch/nn/HingeEmbeddingCriterion.lua +++ b/contrib/lua-torch/nn/HingeEmbeddingCriterion.lua diff --git a/contrib/torch/nn/Identity.lua b/contrib/lua-torch/nn/Identity.lua index 5e6ccb624..5e6ccb624 100644 --- a/contrib/torch/nn/Identity.lua +++ b/contrib/lua-torch/nn/Identity.lua diff --git a/contrib/torch/nn/Index.lua b/contrib/lua-torch/nn/Index.lua index 6aa429708..6aa429708 100644 --- a/contrib/torch/nn/Index.lua +++ b/contrib/lua-torch/nn/Index.lua diff --git a/contrib/torch/nn/IndexLinear.lua b/contrib/lua-torch/nn/IndexLinear.lua index 928e5d3f2..928e5d3f2 100644 --- a/contrib/torch/nn/IndexLinear.lua +++ b/contrib/lua-torch/nn/IndexLinear.lua diff --git a/contrib/torch/nn/Jacobian.lua b/contrib/lua-torch/nn/Jacobian.lua index 4f728b18c..4f728b18c 100644 --- a/contrib/torch/nn/Jacobian.lua +++ b/contrib/lua-torch/nn/Jacobian.lua diff --git a/contrib/torch/nn/JoinTable.lua b/contrib/lua-torch/nn/JoinTable.lua index 6ab68e189..6ab68e189 100644 --- a/contrib/torch/nn/JoinTable.lua +++ b/contrib/lua-torch/nn/JoinTable.lua diff --git a/contrib/torch/nn/Kmeans.lua b/contrib/lua-torch/nn/Kmeans.lua index 56066b63d..56066b63d 100644 --- a/contrib/torch/nn/Kmeans.lua +++ b/contrib/lua-torch/nn/Kmeans.lua diff --git a/contrib/torch/nn/L1Cost.lua b/contrib/lua-torch/nn/L1Cost.lua index 6b58e0ec9..6b58e0ec9 100644 --- a/contrib/torch/nn/L1Cost.lua +++ b/contrib/lua-torch/nn/L1Cost.lua diff --git a/contrib/torch/nn/L1HingeEmbeddingCriterion.lua b/contrib/lua-torch/nn/L1HingeEmbeddingCriterion.lua index 6957278f5..6957278f5 100644 --- a/contrib/torch/nn/L1HingeEmbeddingCriterion.lua +++ b/contrib/lua-torch/nn/L1HingeEmbeddingCriterion.lua diff --git a/contrib/torch/nn/L1Penalty.lua b/contrib/lua-torch/nn/L1Penalty.lua index 9ee6b35ff..9ee6b35ff 100644 --- a/contrib/torch/nn/L1Penalty.lua +++ b/contrib/lua-torch/nn/L1Penalty.lua diff --git a/contrib/torch/nn/LayerNormalization.lua b/contrib/lua-torch/nn/LayerNormalization.lua index 722d7c802..722d7c802 100644 --- a/contrib/torch/nn/LayerNormalization.lua +++ b/contrib/lua-torch/nn/LayerNormalization.lua diff --git a/contrib/torch/nn/LeakyReLU.lua b/contrib/lua-torch/nn/LeakyReLU.lua index 56b7f2542..56b7f2542 100644 --- a/contrib/torch/nn/LeakyReLU.lua +++ b/contrib/lua-torch/nn/LeakyReLU.lua diff --git a/contrib/torch/nn/Linear.lua b/contrib/lua-torch/nn/Linear.lua index 09b5979ce..09b5979ce 100644 --- a/contrib/torch/nn/Linear.lua +++ b/contrib/lua-torch/nn/Linear.lua diff --git a/contrib/torch/nn/LinearWeightNorm.lua b/contrib/lua-torch/nn/LinearWeightNorm.lua index a712f5535..a712f5535 100755 --- a/contrib/torch/nn/LinearWeightNorm.lua +++ b/contrib/lua-torch/nn/LinearWeightNorm.lua diff --git a/contrib/torch/nn/Log.lua b/contrib/lua-torch/nn/Log.lua index e8f236bfb..e8f236bfb 100644 --- a/contrib/torch/nn/Log.lua +++ b/contrib/lua-torch/nn/Log.lua diff --git a/contrib/torch/nn/LogSigmoid.lua b/contrib/lua-torch/nn/LogSigmoid.lua index cab848f4d..cab848f4d 100644 --- a/contrib/torch/nn/LogSigmoid.lua +++ b/contrib/lua-torch/nn/LogSigmoid.lua diff --git a/contrib/torch/nn/LogSoftMax.lua b/contrib/lua-torch/nn/LogSoftMax.lua index 37c8acae4..37c8acae4 100644 --- a/contrib/torch/nn/LogSoftMax.lua +++ b/contrib/lua-torch/nn/LogSoftMax.lua diff --git a/contrib/torch/nn/LookupTable.lua b/contrib/lua-torch/nn/LookupTable.lua index 6cffc6c3e..6cffc6c3e 100644 --- a/contrib/torch/nn/LookupTable.lua +++ b/contrib/lua-torch/nn/LookupTable.lua diff --git a/contrib/torch/nn/MM.lua b/contrib/lua-torch/nn/MM.lua index cc978c8cb..cc978c8cb 100644 --- a/contrib/torch/nn/MM.lua +++ b/contrib/lua-torch/nn/MM.lua diff --git a/contrib/torch/nn/MSECriterion.lua b/contrib/lua-torch/nn/MSECriterion.lua index d38beb6bf..d38beb6bf 100644 --- a/contrib/torch/nn/MSECriterion.lua +++ b/contrib/lua-torch/nn/MSECriterion.lua diff --git a/contrib/torch/nn/MV.lua b/contrib/lua-torch/nn/MV.lua index a00478ef6..a00478ef6 100644 --- a/contrib/torch/nn/MV.lua +++ b/contrib/lua-torch/nn/MV.lua diff --git a/contrib/torch/nn/MapTable.lua b/contrib/lua-torch/nn/MapTable.lua index c79f1ea1d..c79f1ea1d 100644 --- a/contrib/torch/nn/MapTable.lua +++ b/contrib/lua-torch/nn/MapTable.lua diff --git a/contrib/torch/nn/MarginCriterion.lua b/contrib/lua-torch/nn/MarginCriterion.lua index 1ab8ad784..1ab8ad784 100644 --- a/contrib/torch/nn/MarginCriterion.lua +++ b/contrib/lua-torch/nn/MarginCriterion.lua diff --git a/contrib/torch/nn/MarginRankingCriterion.lua b/contrib/lua-torch/nn/MarginRankingCriterion.lua index 844d905d5..844d905d5 100644 --- a/contrib/torch/nn/MarginRankingCriterion.lua +++ b/contrib/lua-torch/nn/MarginRankingCriterion.lua diff --git a/contrib/torch/nn/MaskedSelect.lua b/contrib/lua-torch/nn/MaskedSelect.lua index c3f7834e1..c3f7834e1 100644 --- a/contrib/torch/nn/MaskedSelect.lua +++ b/contrib/lua-torch/nn/MaskedSelect.lua diff --git a/contrib/torch/nn/Max.lua b/contrib/lua-torch/nn/Max.lua index 8273e808c..8273e808c 100644 --- a/contrib/torch/nn/Max.lua +++ b/contrib/lua-torch/nn/Max.lua diff --git a/contrib/torch/nn/Maxout.lua b/contrib/lua-torch/nn/Maxout.lua index a797a9f43..a797a9f43 100644 --- a/contrib/torch/nn/Maxout.lua +++ b/contrib/lua-torch/nn/Maxout.lua diff --git a/contrib/torch/nn/Mean.lua b/contrib/lua-torch/nn/Mean.lua index 8087ac95e..8087ac95e 100644 --- a/contrib/torch/nn/Mean.lua +++ b/contrib/lua-torch/nn/Mean.lua diff --git a/contrib/torch/nn/Min.lua b/contrib/lua-torch/nn/Min.lua index 3a3e4a802..3a3e4a802 100644 --- a/contrib/torch/nn/Min.lua +++ b/contrib/lua-torch/nn/Min.lua diff --git a/contrib/torch/nn/MixtureTable.lua b/contrib/lua-torch/nn/MixtureTable.lua index dbe19742f..dbe19742f 100644 --- a/contrib/torch/nn/MixtureTable.lua +++ b/contrib/lua-torch/nn/MixtureTable.lua diff --git a/contrib/torch/nn/Module.lua b/contrib/lua-torch/nn/Module.lua index 3debc5789..3debc5789 100644 --- a/contrib/torch/nn/Module.lua +++ b/contrib/lua-torch/nn/Module.lua diff --git a/contrib/torch/nn/ModuleCriterion.lua b/contrib/lua-torch/nn/ModuleCriterion.lua index bfc79ef55..bfc79ef55 100644 --- a/contrib/torch/nn/ModuleCriterion.lua +++ b/contrib/lua-torch/nn/ModuleCriterion.lua diff --git a/contrib/torch/nn/Mul.lua b/contrib/lua-torch/nn/Mul.lua index efa1db656..efa1db656 100644 --- a/contrib/torch/nn/Mul.lua +++ b/contrib/lua-torch/nn/Mul.lua diff --git a/contrib/torch/nn/MulConstant.lua b/contrib/lua-torch/nn/MulConstant.lua index e8c473bee..e8c473bee 100644 --- a/contrib/torch/nn/MulConstant.lua +++ b/contrib/lua-torch/nn/MulConstant.lua diff --git a/contrib/torch/nn/MultiCriterion.lua b/contrib/lua-torch/nn/MultiCriterion.lua index 959317711..959317711 100644 --- a/contrib/torch/nn/MultiCriterion.lua +++ b/contrib/lua-torch/nn/MultiCriterion.lua diff --git a/contrib/torch/nn/MultiLabelMarginCriterion.lua b/contrib/lua-torch/nn/MultiLabelMarginCriterion.lua index 908b6133c..908b6133c 100644 --- a/contrib/torch/nn/MultiLabelMarginCriterion.lua +++ b/contrib/lua-torch/nn/MultiLabelMarginCriterion.lua diff --git a/contrib/torch/nn/MultiLabelSoftMarginCriterion.lua b/contrib/lua-torch/nn/MultiLabelSoftMarginCriterion.lua index 9d471d449..9d471d449 100644 --- a/contrib/torch/nn/MultiLabelSoftMarginCriterion.lua +++ b/contrib/lua-torch/nn/MultiLabelSoftMarginCriterion.lua diff --git a/contrib/torch/nn/MultiMarginCriterion.lua b/contrib/lua-torch/nn/MultiMarginCriterion.lua index e3122386a..e3122386a 100644 --- a/contrib/torch/nn/MultiMarginCriterion.lua +++ b/contrib/lua-torch/nn/MultiMarginCriterion.lua diff --git a/contrib/torch/nn/NaN.lua b/contrib/lua-torch/nn/NaN.lua index b80f6a04d..b80f6a04d 100644 --- a/contrib/torch/nn/NaN.lua +++ b/contrib/lua-torch/nn/NaN.lua diff --git a/contrib/torch/nn/Narrow.lua b/contrib/lua-torch/nn/Narrow.lua index a6ebaa321..a6ebaa321 100644 --- a/contrib/torch/nn/Narrow.lua +++ b/contrib/lua-torch/nn/Narrow.lua diff --git a/contrib/torch/nn/NarrowTable.lua b/contrib/lua-torch/nn/NarrowTable.lua index 17429f3b1..17429f3b1 100644 --- a/contrib/torch/nn/NarrowTable.lua +++ b/contrib/lua-torch/nn/NarrowTable.lua diff --git a/contrib/torch/nn/Normalize.lua b/contrib/lua-torch/nn/Normalize.lua index 0937ebba9..0937ebba9 100644 --- a/contrib/torch/nn/Normalize.lua +++ b/contrib/lua-torch/nn/Normalize.lua diff --git a/contrib/torch/nn/OneHot.lua b/contrib/lua-torch/nn/OneHot.lua index d1dc1b52d..d1dc1b52d 100644 --- a/contrib/torch/nn/OneHot.lua +++ b/contrib/lua-torch/nn/OneHot.lua diff --git a/contrib/torch/nn/PReLU.lua b/contrib/lua-torch/nn/PReLU.lua index 2e58fba4e..2e58fba4e 100644 --- a/contrib/torch/nn/PReLU.lua +++ b/contrib/lua-torch/nn/PReLU.lua diff --git a/contrib/torch/nn/Padding.lua b/contrib/lua-torch/nn/Padding.lua index d5f7771d0..d5f7771d0 100644 --- a/contrib/torch/nn/Padding.lua +++ b/contrib/lua-torch/nn/Padding.lua diff --git a/contrib/torch/nn/PairwiseDistance.lua b/contrib/lua-torch/nn/PairwiseDistance.lua index 99a502c16..99a502c16 100644 --- a/contrib/torch/nn/PairwiseDistance.lua +++ b/contrib/lua-torch/nn/PairwiseDistance.lua diff --git a/contrib/torch/nn/Parallel.lua b/contrib/lua-torch/nn/Parallel.lua index 58cb9748e..58cb9748e 100644 --- a/contrib/torch/nn/Parallel.lua +++ b/contrib/lua-torch/nn/Parallel.lua diff --git a/contrib/torch/nn/ParallelCriterion.lua b/contrib/lua-torch/nn/ParallelCriterion.lua index 45607d5c3..45607d5c3 100644 --- a/contrib/torch/nn/ParallelCriterion.lua +++ b/contrib/lua-torch/nn/ParallelCriterion.lua diff --git a/contrib/torch/nn/ParallelTable.lua b/contrib/lua-torch/nn/ParallelTable.lua index 2fe0899dd..2fe0899dd 100644 --- a/contrib/torch/nn/ParallelTable.lua +++ b/contrib/lua-torch/nn/ParallelTable.lua diff --git a/contrib/torch/nn/PartialLinear.lua b/contrib/lua-torch/nn/PartialLinear.lua index 6e92cfc08..6e92cfc08 100644 --- a/contrib/torch/nn/PartialLinear.lua +++ b/contrib/lua-torch/nn/PartialLinear.lua diff --git a/contrib/torch/nn/PixelShuffle.lua b/contrib/lua-torch/nn/PixelShuffle.lua index dd58ed948..dd58ed948 100644 --- a/contrib/torch/nn/PixelShuffle.lua +++ b/contrib/lua-torch/nn/PixelShuffle.lua diff --git a/contrib/torch/nn/Power.lua b/contrib/lua-torch/nn/Power.lua index 771183c48..771183c48 100644 --- a/contrib/torch/nn/Power.lua +++ b/contrib/lua-torch/nn/Power.lua diff --git a/contrib/torch/nn/PrintSize.lua b/contrib/lua-torch/nn/PrintSize.lua index d8dc91bff..d8dc91bff 100644 --- a/contrib/torch/nn/PrintSize.lua +++ b/contrib/lua-torch/nn/PrintSize.lua diff --git a/contrib/torch/nn/Profile.lua b/contrib/lua-torch/nn/Profile.lua index 36cd909cd..36cd909cd 100644 --- a/contrib/torch/nn/Profile.lua +++ b/contrib/lua-torch/nn/Profile.lua diff --git a/contrib/torch/nn/README.md b/contrib/lua-torch/nn/README.md index 6efd60962..6efd60962 100644 --- a/contrib/torch/nn/README.md +++ b/contrib/lua-torch/nn/README.md diff --git a/contrib/torch/nn/RReLU.lua b/contrib/lua-torch/nn/RReLU.lua index 843415f7e..843415f7e 100644 --- a/contrib/torch/nn/RReLU.lua +++ b/contrib/lua-torch/nn/RReLU.lua diff --git a/contrib/torch/nn/ReLU.lua b/contrib/lua-torch/nn/ReLU.lua index a6eb271ee..a6eb271ee 100644 --- a/contrib/torch/nn/ReLU.lua +++ b/contrib/lua-torch/nn/ReLU.lua diff --git a/contrib/torch/nn/ReLU6.lua b/contrib/lua-torch/nn/ReLU6.lua index 1cde00b46..1cde00b46 100644 --- a/contrib/torch/nn/ReLU6.lua +++ b/contrib/lua-torch/nn/ReLU6.lua diff --git a/contrib/torch/nn/Replicate.lua b/contrib/lua-torch/nn/Replicate.lua index c7dedd767..c7dedd767 100644 --- a/contrib/torch/nn/Replicate.lua +++ b/contrib/lua-torch/nn/Replicate.lua diff --git a/contrib/torch/nn/Reshape.lua b/contrib/lua-torch/nn/Reshape.lua index d508369fa..d508369fa 100644 --- a/contrib/torch/nn/Reshape.lua +++ b/contrib/lua-torch/nn/Reshape.lua diff --git a/contrib/torch/nn/Select.lua b/contrib/lua-torch/nn/Select.lua index be87c6465..be87c6465 100644 --- a/contrib/torch/nn/Select.lua +++ b/contrib/lua-torch/nn/Select.lua diff --git a/contrib/torch/nn/SelectTable.lua b/contrib/lua-torch/nn/SelectTable.lua index ef26f3507..ef26f3507 100644 --- a/contrib/torch/nn/SelectTable.lua +++ b/contrib/lua-torch/nn/SelectTable.lua diff --git a/contrib/torch/nn/Sequential.lua b/contrib/lua-torch/nn/Sequential.lua index 22b0886b8..22b0886b8 100644 --- a/contrib/torch/nn/Sequential.lua +++ b/contrib/lua-torch/nn/Sequential.lua diff --git a/contrib/torch/nn/Sigmoid.lua b/contrib/lua-torch/nn/Sigmoid.lua index 0126f6f8f..0126f6f8f 100644 --- a/contrib/torch/nn/Sigmoid.lua +++ b/contrib/lua-torch/nn/Sigmoid.lua diff --git a/contrib/torch/nn/SmoothL1Criterion.lua b/contrib/lua-torch/nn/SmoothL1Criterion.lua index be636a94c..be636a94c 100644 --- a/contrib/torch/nn/SmoothL1Criterion.lua +++ b/contrib/lua-torch/nn/SmoothL1Criterion.lua diff --git a/contrib/torch/nn/SoftMarginCriterion.lua b/contrib/lua-torch/nn/SoftMarginCriterion.lua index 96ccda8a4..96ccda8a4 100644 --- a/contrib/torch/nn/SoftMarginCriterion.lua +++ b/contrib/lua-torch/nn/SoftMarginCriterion.lua diff --git a/contrib/torch/nn/SoftMax.lua b/contrib/lua-torch/nn/SoftMax.lua index 23a444cf6..23a444cf6 100644 --- a/contrib/torch/nn/SoftMax.lua +++ b/contrib/lua-torch/nn/SoftMax.lua diff --git a/contrib/torch/nn/SoftMin.lua b/contrib/lua-torch/nn/SoftMin.lua index 7da2a6589..7da2a6589 100644 --- a/contrib/torch/nn/SoftMin.lua +++ b/contrib/lua-torch/nn/SoftMin.lua diff --git a/contrib/torch/nn/SoftPlus.lua b/contrib/lua-torch/nn/SoftPlus.lua index f77b25380..f77b25380 100644 --- a/contrib/torch/nn/SoftPlus.lua +++ b/contrib/lua-torch/nn/SoftPlus.lua diff --git a/contrib/torch/nn/SoftShrink.lua b/contrib/lua-torch/nn/SoftShrink.lua index 67af15a98..67af15a98 100644 --- a/contrib/torch/nn/SoftShrink.lua +++ b/contrib/lua-torch/nn/SoftShrink.lua diff --git a/contrib/torch/nn/SoftSign.lua b/contrib/lua-torch/nn/SoftSign.lua index ee72011f1..ee72011f1 100644 --- a/contrib/torch/nn/SoftSign.lua +++ b/contrib/lua-torch/nn/SoftSign.lua diff --git a/contrib/torch/nn/SparseJacobian.lua b/contrib/lua-torch/nn/SparseJacobian.lua index 7f4c02444..7f4c02444 100644 --- a/contrib/torch/nn/SparseJacobian.lua +++ b/contrib/lua-torch/nn/SparseJacobian.lua diff --git a/contrib/torch/nn/SparseLinear.lua b/contrib/lua-torch/nn/SparseLinear.lua index 9a50c6912..9a50c6912 100644 --- a/contrib/torch/nn/SparseLinear.lua +++ b/contrib/lua-torch/nn/SparseLinear.lua diff --git a/contrib/torch/nn/SpatialAdaptiveAveragePooling.lua b/contrib/lua-torch/nn/SpatialAdaptiveAveragePooling.lua index 2e223580a..2e223580a 100644 --- a/contrib/torch/nn/SpatialAdaptiveAveragePooling.lua +++ b/contrib/lua-torch/nn/SpatialAdaptiveAveragePooling.lua diff --git a/contrib/torch/nn/SpatialAdaptiveMaxPooling.lua b/contrib/lua-torch/nn/SpatialAdaptiveMaxPooling.lua index b78261c3d..b78261c3d 100644 --- a/contrib/torch/nn/SpatialAdaptiveMaxPooling.lua +++ b/contrib/lua-torch/nn/SpatialAdaptiveMaxPooling.lua diff --git a/contrib/torch/nn/SpatialAutoCropMSECriterion.lua b/contrib/lua-torch/nn/SpatialAutoCropMSECriterion.lua index 97206a062..97206a062 100644 --- a/contrib/torch/nn/SpatialAutoCropMSECriterion.lua +++ b/contrib/lua-torch/nn/SpatialAutoCropMSECriterion.lua diff --git a/contrib/torch/nn/SpatialAveragePooling.lua b/contrib/lua-torch/nn/SpatialAveragePooling.lua index 1e7666827..1e7666827 100644 --- a/contrib/torch/nn/SpatialAveragePooling.lua +++ b/contrib/lua-torch/nn/SpatialAveragePooling.lua diff --git a/contrib/torch/nn/SpatialBatchNormalization.lua b/contrib/lua-torch/nn/SpatialBatchNormalization.lua index c5004ce3a..c5004ce3a 100644 --- a/contrib/torch/nn/SpatialBatchNormalization.lua +++ b/contrib/lua-torch/nn/SpatialBatchNormalization.lua diff --git a/contrib/torch/nn/SpatialClassNLLCriterion.lua b/contrib/lua-torch/nn/SpatialClassNLLCriterion.lua index fbd367410..fbd367410 100644 --- a/contrib/torch/nn/SpatialClassNLLCriterion.lua +++ b/contrib/lua-torch/nn/SpatialClassNLLCriterion.lua diff --git a/contrib/torch/nn/SpatialContrastiveNormalization.lua b/contrib/lua-torch/nn/SpatialContrastiveNormalization.lua index 0ad251ae4..0ad251ae4 100644 --- a/contrib/torch/nn/SpatialContrastiveNormalization.lua +++ b/contrib/lua-torch/nn/SpatialContrastiveNormalization.lua diff --git a/contrib/torch/nn/SpatialConvolution.lua b/contrib/lua-torch/nn/SpatialConvolution.lua index 15a2b4b62..15a2b4b62 100644 --- a/contrib/torch/nn/SpatialConvolution.lua +++ b/contrib/lua-torch/nn/SpatialConvolution.lua diff --git a/contrib/torch/nn/SpatialConvolutionLocal.lua b/contrib/lua-torch/nn/SpatialConvolutionLocal.lua index 9494c2ffe..9494c2ffe 100644 --- a/contrib/torch/nn/SpatialConvolutionLocal.lua +++ b/contrib/lua-torch/nn/SpatialConvolutionLocal.lua diff --git a/contrib/torch/nn/SpatialConvolutionMM.lua b/contrib/lua-torch/nn/SpatialConvolutionMM.lua index f20734f9b..f20734f9b 100644 --- a/contrib/torch/nn/SpatialConvolutionMM.lua +++ b/contrib/lua-torch/nn/SpatialConvolutionMM.lua diff --git a/contrib/torch/nn/SpatialConvolutionMap.lua b/contrib/lua-torch/nn/SpatialConvolutionMap.lua index 9051c119e..9051c119e 100644 --- a/contrib/torch/nn/SpatialConvolutionMap.lua +++ b/contrib/lua-torch/nn/SpatialConvolutionMap.lua diff --git a/contrib/torch/nn/SpatialCrossMapLRN.lua b/contrib/lua-torch/nn/SpatialCrossMapLRN.lua index 088eb07f0..088eb07f0 100644 --- a/contrib/torch/nn/SpatialCrossMapLRN.lua +++ b/contrib/lua-torch/nn/SpatialCrossMapLRN.lua diff --git a/contrib/torch/nn/SpatialDepthWiseConvolution.lua b/contrib/lua-torch/nn/SpatialDepthWiseConvolution.lua index 1132f04cb..1132f04cb 100644 --- a/contrib/torch/nn/SpatialDepthWiseConvolution.lua +++ b/contrib/lua-torch/nn/SpatialDepthWiseConvolution.lua diff --git a/contrib/torch/nn/SpatialDilatedConvolution.lua b/contrib/lua-torch/nn/SpatialDilatedConvolution.lua index a0590c7e9..a0590c7e9 100644 --- a/contrib/torch/nn/SpatialDilatedConvolution.lua +++ b/contrib/lua-torch/nn/SpatialDilatedConvolution.lua diff --git a/contrib/torch/nn/SpatialDilatedMaxPooling.lua b/contrib/lua-torch/nn/SpatialDilatedMaxPooling.lua index 34525a4ad..34525a4ad 100644 --- a/contrib/torch/nn/SpatialDilatedMaxPooling.lua +++ b/contrib/lua-torch/nn/SpatialDilatedMaxPooling.lua diff --git a/contrib/torch/nn/SpatialDivisiveNormalization.lua b/contrib/lua-torch/nn/SpatialDivisiveNormalization.lua index dc2b8c530..dc2b8c530 100644 --- a/contrib/torch/nn/SpatialDivisiveNormalization.lua +++ b/contrib/lua-torch/nn/SpatialDivisiveNormalization.lua diff --git a/contrib/torch/nn/SpatialDropout.lua b/contrib/lua-torch/nn/SpatialDropout.lua index 4320061b7..4320061b7 100644 --- a/contrib/torch/nn/SpatialDropout.lua +++ b/contrib/lua-torch/nn/SpatialDropout.lua diff --git a/contrib/torch/nn/SpatialFractionalMaxPooling.lua b/contrib/lua-torch/nn/SpatialFractionalMaxPooling.lua index 884751d41..884751d41 100644 --- a/contrib/torch/nn/SpatialFractionalMaxPooling.lua +++ b/contrib/lua-torch/nn/SpatialFractionalMaxPooling.lua diff --git a/contrib/torch/nn/SpatialFullConvolution.lua b/contrib/lua-torch/nn/SpatialFullConvolution.lua index e6019bc18..e6019bc18 100644 --- a/contrib/torch/nn/SpatialFullConvolution.lua +++ b/contrib/lua-torch/nn/SpatialFullConvolution.lua diff --git a/contrib/torch/nn/SpatialFullConvolutionMap.lua b/contrib/lua-torch/nn/SpatialFullConvolutionMap.lua index 008f5e7cf..008f5e7cf 100644 --- a/contrib/torch/nn/SpatialFullConvolutionMap.lua +++ b/contrib/lua-torch/nn/SpatialFullConvolutionMap.lua diff --git a/contrib/torch/nn/SpatialLPPooling.lua b/contrib/lua-torch/nn/SpatialLPPooling.lua index 49a8493cf..49a8493cf 100644 --- a/contrib/torch/nn/SpatialLPPooling.lua +++ b/contrib/lua-torch/nn/SpatialLPPooling.lua diff --git a/contrib/torch/nn/SpatialLogSoftMax.lua b/contrib/lua-torch/nn/SpatialLogSoftMax.lua index 9c81d49e1..9c81d49e1 100644 --- a/contrib/torch/nn/SpatialLogSoftMax.lua +++ b/contrib/lua-torch/nn/SpatialLogSoftMax.lua diff --git a/contrib/torch/nn/SpatialMaxPooling.lua b/contrib/lua-torch/nn/SpatialMaxPooling.lua index 5c865c631..5c865c631 100644 --- a/contrib/torch/nn/SpatialMaxPooling.lua +++ b/contrib/lua-torch/nn/SpatialMaxPooling.lua diff --git a/contrib/torch/nn/SpatialMaxUnpooling.lua b/contrib/lua-torch/nn/SpatialMaxUnpooling.lua index 408bcc052..408bcc052 100644 --- a/contrib/torch/nn/SpatialMaxUnpooling.lua +++ b/contrib/lua-torch/nn/SpatialMaxUnpooling.lua diff --git a/contrib/torch/nn/SpatialReflectionPadding.lua b/contrib/lua-torch/nn/SpatialReflectionPadding.lua index 9ce4612ad..9ce4612ad 100644 --- a/contrib/torch/nn/SpatialReflectionPadding.lua +++ b/contrib/lua-torch/nn/SpatialReflectionPadding.lua diff --git a/contrib/torch/nn/SpatialReplicationPadding.lua b/contrib/lua-torch/nn/SpatialReplicationPadding.lua index 429763f9b..429763f9b 100644 --- a/contrib/torch/nn/SpatialReplicationPadding.lua +++ b/contrib/lua-torch/nn/SpatialReplicationPadding.lua diff --git a/contrib/torch/nn/SpatialSoftMax.lua b/contrib/lua-torch/nn/SpatialSoftMax.lua index 56f0b40e2..56f0b40e2 100644 --- a/contrib/torch/nn/SpatialSoftMax.lua +++ b/contrib/lua-torch/nn/SpatialSoftMax.lua diff --git a/contrib/torch/nn/SpatialSubSampling.lua b/contrib/lua-torch/nn/SpatialSubSampling.lua index 4e3fb8881..4e3fb8881 100644 --- a/contrib/torch/nn/SpatialSubSampling.lua +++ b/contrib/lua-torch/nn/SpatialSubSampling.lua diff --git a/contrib/torch/nn/SpatialSubtractiveNormalization.lua b/contrib/lua-torch/nn/SpatialSubtractiveNormalization.lua index d430083e9..d430083e9 100644 --- a/contrib/torch/nn/SpatialSubtractiveNormalization.lua +++ b/contrib/lua-torch/nn/SpatialSubtractiveNormalization.lua diff --git a/contrib/torch/nn/SpatialUpSamplingBilinear.lua b/contrib/lua-torch/nn/SpatialUpSamplingBilinear.lua index 12e1ce8f2..12e1ce8f2 100644 --- a/contrib/torch/nn/SpatialUpSamplingBilinear.lua +++ b/contrib/lua-torch/nn/SpatialUpSamplingBilinear.lua diff --git a/contrib/torch/nn/SpatialUpSamplingNearest.lua b/contrib/lua-torch/nn/SpatialUpSamplingNearest.lua index 362ae73a3..362ae73a3 100644 --- a/contrib/torch/nn/SpatialUpSamplingNearest.lua +++ b/contrib/lua-torch/nn/SpatialUpSamplingNearest.lua diff --git a/contrib/torch/nn/SpatialZeroPadding.lua b/contrib/lua-torch/nn/SpatialZeroPadding.lua index f19925841..f19925841 100644 --- a/contrib/torch/nn/SpatialZeroPadding.lua +++ b/contrib/lua-torch/nn/SpatialZeroPadding.lua diff --git a/contrib/torch/nn/SplitTable.lua b/contrib/lua-torch/nn/SplitTable.lua index 7c4f968e6..7c4f968e6 100644 --- a/contrib/torch/nn/SplitTable.lua +++ b/contrib/lua-torch/nn/SplitTable.lua diff --git a/contrib/torch/nn/Sqrt.lua b/contrib/lua-torch/nn/Sqrt.lua index df354a175..df354a175 100644 --- a/contrib/torch/nn/Sqrt.lua +++ b/contrib/lua-torch/nn/Sqrt.lua diff --git a/contrib/torch/nn/Square.lua b/contrib/lua-torch/nn/Square.lua index a6292afb9..a6292afb9 100644 --- a/contrib/torch/nn/Square.lua +++ b/contrib/lua-torch/nn/Square.lua diff --git a/contrib/torch/nn/Squeeze.lua b/contrib/lua-torch/nn/Squeeze.lua index 7d204a19d..7d204a19d 100644 --- a/contrib/torch/nn/Squeeze.lua +++ b/contrib/lua-torch/nn/Squeeze.lua diff --git a/contrib/torch/nn/StochasticGradient.lua b/contrib/lua-torch/nn/StochasticGradient.lua index a060371e8..a060371e8 100644 --- a/contrib/torch/nn/StochasticGradient.lua +++ b/contrib/lua-torch/nn/StochasticGradient.lua diff --git a/contrib/torch/nn/Sum.lua b/contrib/lua-torch/nn/Sum.lua index 7fe8a1ab8..7fe8a1ab8 100644 --- a/contrib/torch/nn/Sum.lua +++ b/contrib/lua-torch/nn/Sum.lua diff --git a/contrib/torch/nn/THNN.lua b/contrib/lua-torch/nn/THNN.lua index 0848e9ed2..0848e9ed2 100644 --- a/contrib/torch/nn/THNN.lua +++ b/contrib/lua-torch/nn/THNN.lua diff --git a/contrib/torch/nn/Tanh.lua b/contrib/lua-torch/nn/Tanh.lua index fc42cbbfd..fc42cbbfd 100644 --- a/contrib/torch/nn/Tanh.lua +++ b/contrib/lua-torch/nn/Tanh.lua diff --git a/contrib/torch/nn/TanhShrink.lua b/contrib/lua-torch/nn/TanhShrink.lua index 96df6c5b7..96df6c5b7 100644 --- a/contrib/torch/nn/TanhShrink.lua +++ b/contrib/lua-torch/nn/TanhShrink.lua diff --git a/contrib/torch/nn/TemporalConvolution.lua b/contrib/lua-torch/nn/TemporalConvolution.lua index 4b3a89eb6..4b3a89eb6 100644 --- a/contrib/torch/nn/TemporalConvolution.lua +++ b/contrib/lua-torch/nn/TemporalConvolution.lua diff --git a/contrib/torch/nn/TemporalDynamicKMaxPooling.lua b/contrib/lua-torch/nn/TemporalDynamicKMaxPooling.lua index 644a0fa9c..644a0fa9c 100644 --- a/contrib/torch/nn/TemporalDynamicKMaxPooling.lua +++ b/contrib/lua-torch/nn/TemporalDynamicKMaxPooling.lua diff --git a/contrib/torch/nn/TemporalMaxPooling.lua b/contrib/lua-torch/nn/TemporalMaxPooling.lua index 894f4a99f..894f4a99f 100644 --- a/contrib/torch/nn/TemporalMaxPooling.lua +++ b/contrib/lua-torch/nn/TemporalMaxPooling.lua diff --git a/contrib/torch/nn/TemporalRowConvolution.lua b/contrib/lua-torch/nn/TemporalRowConvolution.lua index 7c9d6a269..7c9d6a269 100644 --- a/contrib/torch/nn/TemporalRowConvolution.lua +++ b/contrib/lua-torch/nn/TemporalRowConvolution.lua diff --git a/contrib/torch/nn/TemporalSubSampling.lua b/contrib/lua-torch/nn/TemporalSubSampling.lua index e9287d63d..e9287d63d 100644 --- a/contrib/torch/nn/TemporalSubSampling.lua +++ b/contrib/lua-torch/nn/TemporalSubSampling.lua diff --git a/contrib/torch/nn/Threshold.lua b/contrib/lua-torch/nn/Threshold.lua index 6fdd26408..6fdd26408 100644 --- a/contrib/torch/nn/Threshold.lua +++ b/contrib/lua-torch/nn/Threshold.lua diff --git a/contrib/torch/nn/Transpose.lua b/contrib/lua-torch/nn/Transpose.lua index cceb2b643..cceb2b643 100644 --- a/contrib/torch/nn/Transpose.lua +++ b/contrib/lua-torch/nn/Transpose.lua diff --git a/contrib/torch/nn/Unsqueeze.lua b/contrib/lua-torch/nn/Unsqueeze.lua index 2e82a25a0..2e82a25a0 100644 --- a/contrib/torch/nn/Unsqueeze.lua +++ b/contrib/lua-torch/nn/Unsqueeze.lua diff --git a/contrib/torch/nn/View.lua b/contrib/lua-torch/nn/View.lua index 542e57e16..542e57e16 100644 --- a/contrib/torch/nn/View.lua +++ b/contrib/lua-torch/nn/View.lua diff --git a/contrib/torch/nn/VolumetricAveragePooling.lua b/contrib/lua-torch/nn/VolumetricAveragePooling.lua index df6d2c405..df6d2c405 100644 --- a/contrib/torch/nn/VolumetricAveragePooling.lua +++ b/contrib/lua-torch/nn/VolumetricAveragePooling.lua diff --git a/contrib/torch/nn/VolumetricBatchNormalization.lua b/contrib/lua-torch/nn/VolumetricBatchNormalization.lua index 6168a9245..6168a9245 100644 --- a/contrib/torch/nn/VolumetricBatchNormalization.lua +++ b/contrib/lua-torch/nn/VolumetricBatchNormalization.lua diff --git a/contrib/torch/nn/VolumetricConvolution.lua b/contrib/lua-torch/nn/VolumetricConvolution.lua index 329609aff..329609aff 100644 --- a/contrib/torch/nn/VolumetricConvolution.lua +++ b/contrib/lua-torch/nn/VolumetricConvolution.lua diff --git a/contrib/torch/nn/VolumetricDilatedConvolution.lua b/contrib/lua-torch/nn/VolumetricDilatedConvolution.lua index f1337ebaa..f1337ebaa 100644 --- a/contrib/torch/nn/VolumetricDilatedConvolution.lua +++ b/contrib/lua-torch/nn/VolumetricDilatedConvolution.lua diff --git a/contrib/torch/nn/VolumetricDilatedMaxPooling.lua b/contrib/lua-torch/nn/VolumetricDilatedMaxPooling.lua index 249b2b58e..249b2b58e 100644 --- a/contrib/torch/nn/VolumetricDilatedMaxPooling.lua +++ b/contrib/lua-torch/nn/VolumetricDilatedMaxPooling.lua diff --git a/contrib/torch/nn/VolumetricDropout.lua b/contrib/lua-torch/nn/VolumetricDropout.lua index 809e28afe..809e28afe 100644 --- a/contrib/torch/nn/VolumetricDropout.lua +++ b/contrib/lua-torch/nn/VolumetricDropout.lua diff --git a/contrib/torch/nn/VolumetricFractionalMaxPooling.lua b/contrib/lua-torch/nn/VolumetricFractionalMaxPooling.lua index f5ff58cf0..f5ff58cf0 100644 --- a/contrib/torch/nn/VolumetricFractionalMaxPooling.lua +++ b/contrib/lua-torch/nn/VolumetricFractionalMaxPooling.lua diff --git a/contrib/torch/nn/VolumetricFullConvolution.lua b/contrib/lua-torch/nn/VolumetricFullConvolution.lua index 0ce23401e..0ce23401e 100644 --- a/contrib/torch/nn/VolumetricFullConvolution.lua +++ b/contrib/lua-torch/nn/VolumetricFullConvolution.lua diff --git a/contrib/torch/nn/VolumetricMaxPooling.lua b/contrib/lua-torch/nn/VolumetricMaxPooling.lua index e25c5b31c..e25c5b31c 100644 --- a/contrib/torch/nn/VolumetricMaxPooling.lua +++ b/contrib/lua-torch/nn/VolumetricMaxPooling.lua diff --git a/contrib/torch/nn/VolumetricMaxUnpooling.lua b/contrib/lua-torch/nn/VolumetricMaxUnpooling.lua index 6291f5b85..6291f5b85 100644 --- a/contrib/torch/nn/VolumetricMaxUnpooling.lua +++ b/contrib/lua-torch/nn/VolumetricMaxUnpooling.lua diff --git a/contrib/torch/nn/VolumetricReplicationPadding.lua b/contrib/lua-torch/nn/VolumetricReplicationPadding.lua index 31a9503fd..31a9503fd 100644 --- a/contrib/torch/nn/VolumetricReplicationPadding.lua +++ b/contrib/lua-torch/nn/VolumetricReplicationPadding.lua diff --git a/contrib/torch/nn/WeightNorm.lua b/contrib/lua-torch/nn/WeightNorm.lua index 3ffcd90aa..3ffcd90aa 100644 --- a/contrib/torch/nn/WeightNorm.lua +++ b/contrib/lua-torch/nn/WeightNorm.lua diff --git a/contrib/torch/nn/WeightedEuclidean.lua b/contrib/lua-torch/nn/WeightedEuclidean.lua index dbf4158a9..dbf4158a9 100644 --- a/contrib/torch/nn/WeightedEuclidean.lua +++ b/contrib/lua-torch/nn/WeightedEuclidean.lua diff --git a/contrib/torch/nn/WeightedMSECriterion.lua b/contrib/lua-torch/nn/WeightedMSECriterion.lua index 933472937..933472937 100644 --- a/contrib/torch/nn/WeightedMSECriterion.lua +++ b/contrib/lua-torch/nn/WeightedMSECriterion.lua diff --git a/contrib/torch/nn/WhiteNoise.lua b/contrib/lua-torch/nn/WhiteNoise.lua index f1defb646..f1defb646 100644 --- a/contrib/torch/nn/WhiteNoise.lua +++ b/contrib/lua-torch/nn/WhiteNoise.lua diff --git a/contrib/torch/nn/ZeroGrad.lua b/contrib/lua-torch/nn/ZeroGrad.lua index 7c941ce1c..7c941ce1c 100644 --- a/contrib/torch/nn/ZeroGrad.lua +++ b/contrib/lua-torch/nn/ZeroGrad.lua diff --git a/contrib/torch/nn/ZipTable.lua b/contrib/lua-torch/nn/ZipTable.lua index 7b18619eb..7b18619eb 100644 --- a/contrib/torch/nn/ZipTable.lua +++ b/contrib/lua-torch/nn/ZipTable.lua diff --git a/contrib/torch/nn/ZipTableOneToMany.lua b/contrib/lua-torch/nn/ZipTableOneToMany.lua index d4a80fe0d..d4a80fe0d 100644 --- a/contrib/torch/nn/ZipTableOneToMany.lua +++ b/contrib/lua-torch/nn/ZipTableOneToMany.lua diff --git a/contrib/torch/nn/hessian.lua b/contrib/lua-torch/nn/hessian.lua index b841d8c59..b841d8c59 100644 --- a/contrib/torch/nn/hessian.lua +++ b/contrib/lua-torch/nn/hessian.lua diff --git a/contrib/torch/nn/init.lua b/contrib/lua-torch/nn/init.lua index 4319a8868..4319a8868 100755 --- a/contrib/torch/nn/init.lua +++ b/contrib/lua-torch/nn/init.lua diff --git a/contrib/torch/nn/lib/CMakeLists.txt b/contrib/lua-torch/nn/lib/CMakeLists.txt index de04595f6..de04595f6 100644 --- a/contrib/torch/nn/lib/CMakeLists.txt +++ b/contrib/lua-torch/nn/lib/CMakeLists.txt diff --git a/contrib/torch/nn/lib/THNN/CMakeLists.txt b/contrib/lua-torch/nn/lib/THNN/CMakeLists.txt index 00908a5b1..00908a5b1 100644 --- a/contrib/torch/nn/lib/THNN/CMakeLists.txt +++ b/contrib/lua-torch/nn/lib/THNN/CMakeLists.txt diff --git a/contrib/torch/nn/lib/THNN/README.md b/contrib/lua-torch/nn/lib/THNN/README.md index e6c61601d..e6c61601d 100644 --- a/contrib/torch/nn/lib/THNN/README.md +++ b/contrib/lua-torch/nn/lib/THNN/README.md diff --git a/contrib/torch/nn/lib/THNN/THNN.h b/contrib/lua-torch/nn/lib/THNN/THNN.h index 0019b7976..0019b7976 100644 --- a/contrib/torch/nn/lib/THNN/THNN.h +++ b/contrib/lua-torch/nn/lib/THNN/THNN.h diff --git a/contrib/torch/nn/lib/THNN/generic/Abs.c b/contrib/lua-torch/nn/lib/THNN/generic/Abs.c index 28721ec8e..28721ec8e 100644 --- a/contrib/torch/nn/lib/THNN/generic/Abs.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/Abs.c diff --git a/contrib/torch/nn/lib/THNN/generic/AbsCriterion.c b/contrib/lua-torch/nn/lib/THNN/generic/AbsCriterion.c index 9bee5de9e..9bee5de9e 100644 --- a/contrib/torch/nn/lib/THNN/generic/AbsCriterion.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/AbsCriterion.c diff --git a/contrib/torch/nn/lib/THNN/generic/BCECriterion.c b/contrib/lua-torch/nn/lib/THNN/generic/BCECriterion.c index 637a4067e..637a4067e 100644 --- a/contrib/torch/nn/lib/THNN/generic/BCECriterion.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/BCECriterion.c diff --git a/contrib/torch/nn/lib/THNN/generic/BatchNormalization.c b/contrib/lua-torch/nn/lib/THNN/generic/BatchNormalization.c index b8f462790..b8f462790 100644 --- a/contrib/torch/nn/lib/THNN/generic/BatchNormalization.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/BatchNormalization.c diff --git a/contrib/torch/nn/lib/THNN/generic/ClassNLLCriterion.c b/contrib/lua-torch/nn/lib/THNN/generic/ClassNLLCriterion.c index 4cf37aeaf..4cf37aeaf 100644 --- a/contrib/torch/nn/lib/THNN/generic/ClassNLLCriterion.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/ClassNLLCriterion.c diff --git a/contrib/torch/nn/lib/THNN/generic/DistKLDivCriterion.c b/contrib/lua-torch/nn/lib/THNN/generic/DistKLDivCriterion.c index 6bd6aa067..6bd6aa067 100644 --- a/contrib/torch/nn/lib/THNN/generic/DistKLDivCriterion.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/DistKLDivCriterion.c diff --git a/contrib/torch/nn/lib/THNN/generic/ELU.c b/contrib/lua-torch/nn/lib/THNN/generic/ELU.c index ddcfb9705..ddcfb9705 100644 --- a/contrib/torch/nn/lib/THNN/generic/ELU.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/ELU.c diff --git a/contrib/torch/nn/lib/THNN/generic/FusedRNNKernel.c b/contrib/lua-torch/nn/lib/THNN/generic/FusedRNNKernel.c index 30788b0a2..30788b0a2 100644 --- a/contrib/torch/nn/lib/THNN/generic/FusedRNNKernel.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/FusedRNNKernel.c diff --git a/contrib/torch/nn/lib/THNN/generic/GatedLinearUnit.c b/contrib/lua-torch/nn/lib/THNN/generic/GatedLinearUnit.c index 274a27e3b..274a27e3b 100644 --- a/contrib/torch/nn/lib/THNN/generic/GatedLinearUnit.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/GatedLinearUnit.c diff --git a/contrib/torch/nn/lib/THNN/generic/HardShrink.c b/contrib/lua-torch/nn/lib/THNN/generic/HardShrink.c index aaae85bac..aaae85bac 100644 --- a/contrib/torch/nn/lib/THNN/generic/HardShrink.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/HardShrink.c diff --git a/contrib/torch/nn/lib/THNN/generic/HardTanh.c b/contrib/lua-torch/nn/lib/THNN/generic/HardTanh.c index 589a66e15..589a66e15 100644 --- a/contrib/torch/nn/lib/THNN/generic/HardTanh.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/HardTanh.c diff --git a/contrib/torch/nn/lib/THNN/generic/IndexLinear.c b/contrib/lua-torch/nn/lib/THNN/generic/IndexLinear.c index 42d8368ba..42d8368ba 100644 --- a/contrib/torch/nn/lib/THNN/generic/IndexLinear.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/IndexLinear.c diff --git a/contrib/torch/nn/lib/THNN/generic/L1Cost.c b/contrib/lua-torch/nn/lib/THNN/generic/L1Cost.c index 53940e894..53940e894 100644 --- a/contrib/torch/nn/lib/THNN/generic/L1Cost.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/L1Cost.c diff --git a/contrib/torch/nn/lib/THNN/generic/LeakyReLU.c b/contrib/lua-torch/nn/lib/THNN/generic/LeakyReLU.c index 074047d83..074047d83 100644 --- a/contrib/torch/nn/lib/THNN/generic/LeakyReLU.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/LeakyReLU.c diff --git a/contrib/torch/nn/lib/THNN/generic/Linear.c b/contrib/lua-torch/nn/lib/THNN/generic/Linear.c index 8c5cd115e..8c5cd115e 100644 --- a/contrib/torch/nn/lib/THNN/generic/Linear.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/Linear.c diff --git a/contrib/torch/nn/lib/THNN/generic/LogSigmoid.c b/contrib/lua-torch/nn/lib/THNN/generic/LogSigmoid.c index 651d56002..651d56002 100644 --- a/contrib/torch/nn/lib/THNN/generic/LogSigmoid.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/LogSigmoid.c diff --git a/contrib/torch/nn/lib/THNN/generic/LogSoftMax.c b/contrib/lua-torch/nn/lib/THNN/generic/LogSoftMax.c index a7280422b..a7280422b 100644 --- a/contrib/torch/nn/lib/THNN/generic/LogSoftMax.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/LogSoftMax.c diff --git a/contrib/torch/nn/lib/THNN/generic/LookupTable.c b/contrib/lua-torch/nn/lib/THNN/generic/LookupTable.c index 46bc2c3c1..46bc2c3c1 100644 --- a/contrib/torch/nn/lib/THNN/generic/LookupTable.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/LookupTable.c diff --git a/contrib/torch/nn/lib/THNN/generic/MSECriterion.c b/contrib/lua-torch/nn/lib/THNN/generic/MSECriterion.c index 58911f6f0..58911f6f0 100644 --- a/contrib/torch/nn/lib/THNN/generic/MSECriterion.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/MSECriterion.c diff --git a/contrib/torch/nn/lib/THNN/generic/MarginCriterion.c b/contrib/lua-torch/nn/lib/THNN/generic/MarginCriterion.c index d6d9b60b9..d6d9b60b9 100644 --- a/contrib/torch/nn/lib/THNN/generic/MarginCriterion.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/MarginCriterion.c diff --git a/contrib/torch/nn/lib/THNN/generic/MultiLabelMarginCriterion.c b/contrib/lua-torch/nn/lib/THNN/generic/MultiLabelMarginCriterion.c index 16398c13c..16398c13c 100644 --- a/contrib/torch/nn/lib/THNN/generic/MultiLabelMarginCriterion.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/MultiLabelMarginCriterion.c diff --git a/contrib/torch/nn/lib/THNN/generic/MultiMarginCriterion.c b/contrib/lua-torch/nn/lib/THNN/generic/MultiMarginCriterion.c index 2f8f8ff58..2f8f8ff58 100644 --- a/contrib/torch/nn/lib/THNN/generic/MultiMarginCriterion.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/MultiMarginCriterion.c diff --git a/contrib/torch/nn/lib/THNN/generic/PReLU.c b/contrib/lua-torch/nn/lib/THNN/generic/PReLU.c index 488322fde..488322fde 100644 --- a/contrib/torch/nn/lib/THNN/generic/PReLU.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/PReLU.c diff --git a/contrib/torch/nn/lib/THNN/generic/RReLU.c b/contrib/lua-torch/nn/lib/THNN/generic/RReLU.c index 8fd46d3c2..8fd46d3c2 100644 --- a/contrib/torch/nn/lib/THNN/generic/RReLU.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/RReLU.c diff --git a/contrib/torch/nn/lib/THNN/generic/Sigmoid.c b/contrib/lua-torch/nn/lib/THNN/generic/Sigmoid.c index 17fb2cb4d..17fb2cb4d 100644 --- a/contrib/torch/nn/lib/THNN/generic/Sigmoid.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/Sigmoid.c diff --git a/contrib/torch/nn/lib/THNN/generic/SmoothL1Criterion.c b/contrib/lua-torch/nn/lib/THNN/generic/SmoothL1Criterion.c index d1928d11c..d1928d11c 100644 --- a/contrib/torch/nn/lib/THNN/generic/SmoothL1Criterion.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/SmoothL1Criterion.c diff --git a/contrib/torch/nn/lib/THNN/generic/SoftMarginCriterion.c b/contrib/lua-torch/nn/lib/THNN/generic/SoftMarginCriterion.c index bac0a3b53..bac0a3b53 100644 --- a/contrib/torch/nn/lib/THNN/generic/SoftMarginCriterion.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/SoftMarginCriterion.c diff --git a/contrib/torch/nn/lib/THNN/generic/SoftMax.c b/contrib/lua-torch/nn/lib/THNN/generic/SoftMax.c index 7b60d64c2..7b60d64c2 100644 --- a/contrib/torch/nn/lib/THNN/generic/SoftMax.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/SoftMax.c diff --git a/contrib/torch/nn/lib/THNN/generic/SoftPlus.c b/contrib/lua-torch/nn/lib/THNN/generic/SoftPlus.c index 6491e66d6..6491e66d6 100644 --- a/contrib/torch/nn/lib/THNN/generic/SoftPlus.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/SoftPlus.c diff --git a/contrib/torch/nn/lib/THNN/generic/SoftShrink.c b/contrib/lua-torch/nn/lib/THNN/generic/SoftShrink.c index e77950868..e77950868 100644 --- a/contrib/torch/nn/lib/THNN/generic/SoftShrink.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/SoftShrink.c diff --git a/contrib/torch/nn/lib/THNN/generic/SparseLinear.c b/contrib/lua-torch/nn/lib/THNN/generic/SparseLinear.c index 1cf712212..1cf712212 100644 --- a/contrib/torch/nn/lib/THNN/generic/SparseLinear.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/SparseLinear.c diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialAdaptiveAveragePooling.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialAdaptiveAveragePooling.c index 3675b42d7..3675b42d7 100644 --- a/contrib/torch/nn/lib/THNN/generic/SpatialAdaptiveAveragePooling.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialAdaptiveAveragePooling.c diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialAdaptiveMaxPooling.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialAdaptiveMaxPooling.c index fff716e67..fff716e67 100644 --- a/contrib/torch/nn/lib/THNN/generic/SpatialAdaptiveMaxPooling.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialAdaptiveMaxPooling.c diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialAveragePooling.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialAveragePooling.c index c063502e7..c063502e7 100644 --- a/contrib/torch/nn/lib/THNN/generic/SpatialAveragePooling.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialAveragePooling.c diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialClassNLLCriterion.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialClassNLLCriterion.c index d711c8590..d711c8590 100644 --- a/contrib/torch/nn/lib/THNN/generic/SpatialClassNLLCriterion.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialClassNLLCriterion.c diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialConvolutionLocal.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialConvolutionLocal.c index 6db5a5db9..6db5a5db9 100644 --- a/contrib/torch/nn/lib/THNN/generic/SpatialConvolutionLocal.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialConvolutionLocal.c diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialConvolutionMM.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialConvolutionMM.c index 28fea517c..28fea517c 100644 --- a/contrib/torch/nn/lib/THNN/generic/SpatialConvolutionMM.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialConvolutionMM.c diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialConvolutionMap.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialConvolutionMap.c index 142a03551..142a03551 100644 --- a/contrib/torch/nn/lib/THNN/generic/SpatialConvolutionMap.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialConvolutionMap.c diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialDepthWiseConvolution.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialDepthWiseConvolution.c index efb66a3e3..efb66a3e3 100644 --- a/contrib/torch/nn/lib/THNN/generic/SpatialDepthWiseConvolution.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialDepthWiseConvolution.c diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialDilatedConvolution.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialDilatedConvolution.c index 897cc0da4..897cc0da4 100644 --- a/contrib/torch/nn/lib/THNN/generic/SpatialDilatedConvolution.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialDilatedConvolution.c diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialDilatedMaxPooling.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialDilatedMaxPooling.c index 8f4ad13c3..8f4ad13c3 100644 --- a/contrib/torch/nn/lib/THNN/generic/SpatialDilatedMaxPooling.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialDilatedMaxPooling.c diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialFractionalMaxPooling.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialFractionalMaxPooling.c index a98954cc6..a98954cc6 100644 --- a/contrib/torch/nn/lib/THNN/generic/SpatialFractionalMaxPooling.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialFractionalMaxPooling.c diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialFullConvolution.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialFullConvolution.c index 2edc53b5a..2edc53b5a 100644 --- a/contrib/torch/nn/lib/THNN/generic/SpatialFullConvolution.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialFullConvolution.c diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialFullConvolutionMap.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialFullConvolutionMap.c index 6952fbe25..6952fbe25 100644 --- a/contrib/torch/nn/lib/THNN/generic/SpatialFullConvolutionMap.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialFullConvolutionMap.c diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialMaxPooling.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialMaxPooling.c index 88aaa40e1..88aaa40e1 100644 --- a/contrib/torch/nn/lib/THNN/generic/SpatialMaxPooling.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialMaxPooling.c diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialMaxUnpooling.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialMaxUnpooling.c index 320538686..320538686 100644 --- a/contrib/torch/nn/lib/THNN/generic/SpatialMaxUnpooling.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialMaxUnpooling.c diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialReflectionPadding.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialReflectionPadding.c index dcde660ea..dcde660ea 100644 --- a/contrib/torch/nn/lib/THNN/generic/SpatialReflectionPadding.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialReflectionPadding.c diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialReplicationPadding.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialReplicationPadding.c index 4e318aa70..4e318aa70 100644 --- a/contrib/torch/nn/lib/THNN/generic/SpatialReplicationPadding.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialReplicationPadding.c diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialSubSampling.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialSubSampling.c index 4c077bc64..4c077bc64 100644 --- a/contrib/torch/nn/lib/THNN/generic/SpatialSubSampling.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialSubSampling.c diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialUpSamplingBilinear.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialUpSamplingBilinear.c index 8bc487ead..8bc487ead 100644 --- a/contrib/torch/nn/lib/THNN/generic/SpatialUpSamplingBilinear.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialUpSamplingBilinear.c diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialUpSamplingNearest.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialUpSamplingNearest.c index b4699ff3e..b4699ff3e 100644 --- a/contrib/torch/nn/lib/THNN/generic/SpatialUpSamplingNearest.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialUpSamplingNearest.c diff --git a/contrib/torch/nn/lib/THNN/generic/Sqrt.c b/contrib/lua-torch/nn/lib/THNN/generic/Sqrt.c index 174884e34..174884e34 100644 --- a/contrib/torch/nn/lib/THNN/generic/Sqrt.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/Sqrt.c diff --git a/contrib/torch/nn/lib/THNN/generic/Square.c b/contrib/lua-torch/nn/lib/THNN/generic/Square.c index aad0a911c..aad0a911c 100644 --- a/contrib/torch/nn/lib/THNN/generic/Square.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/Square.c diff --git a/contrib/torch/nn/lib/THNN/generic/THNN.h b/contrib/lua-torch/nn/lib/THNN/generic/THNN.h index 76a28eb2d..76a28eb2d 100644 --- a/contrib/torch/nn/lib/THNN/generic/THNN.h +++ b/contrib/lua-torch/nn/lib/THNN/generic/THNN.h diff --git a/contrib/torch/nn/lib/THNN/generic/Tanh.c b/contrib/lua-torch/nn/lib/THNN/generic/Tanh.c index ecf0708c2..ecf0708c2 100644 --- a/contrib/torch/nn/lib/THNN/generic/Tanh.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/Tanh.c diff --git a/contrib/torch/nn/lib/THNN/generic/TemporalConvolution.c b/contrib/lua-torch/nn/lib/THNN/generic/TemporalConvolution.c index 8cfd97d85..8cfd97d85 100644 --- a/contrib/torch/nn/lib/THNN/generic/TemporalConvolution.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/TemporalConvolution.c diff --git a/contrib/torch/nn/lib/THNN/generic/TemporalMaxPooling.c b/contrib/lua-torch/nn/lib/THNN/generic/TemporalMaxPooling.c index 344c1b3fd..344c1b3fd 100644 --- a/contrib/torch/nn/lib/THNN/generic/TemporalMaxPooling.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/TemporalMaxPooling.c diff --git a/contrib/torch/nn/lib/THNN/generic/TemporalRowConvolution.c b/contrib/lua-torch/nn/lib/THNN/generic/TemporalRowConvolution.c index e3ae41e22..e3ae41e22 100644 --- a/contrib/torch/nn/lib/THNN/generic/TemporalRowConvolution.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/TemporalRowConvolution.c diff --git a/contrib/torch/nn/lib/THNN/generic/TemporalSubSampling.c b/contrib/lua-torch/nn/lib/THNN/generic/TemporalSubSampling.c index 68f35e28a..68f35e28a 100644 --- a/contrib/torch/nn/lib/THNN/generic/TemporalSubSampling.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/TemporalSubSampling.c diff --git a/contrib/torch/nn/lib/THNN/generic/Threshold.c b/contrib/lua-torch/nn/lib/THNN/generic/Threshold.c index 949c7a07c..949c7a07c 100644 --- a/contrib/torch/nn/lib/THNN/generic/Threshold.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/Threshold.c diff --git a/contrib/torch/nn/lib/THNN/generic/VolumetricAveragePooling.c b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricAveragePooling.c index 91c870e6f..91c870e6f 100644 --- a/contrib/torch/nn/lib/THNN/generic/VolumetricAveragePooling.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricAveragePooling.c diff --git a/contrib/torch/nn/lib/THNN/generic/VolumetricConvolution.c b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricConvolution.c index be1aa82e6..be1aa82e6 100644 --- a/contrib/torch/nn/lib/THNN/generic/VolumetricConvolution.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricConvolution.c diff --git a/contrib/torch/nn/lib/THNN/generic/VolumetricConvolutionMM.c b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricConvolutionMM.c index 00a121db6..00a121db6 100644 --- a/contrib/torch/nn/lib/THNN/generic/VolumetricConvolutionMM.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricConvolutionMM.c diff --git a/contrib/torch/nn/lib/THNN/generic/VolumetricDilatedConvolution.c b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricDilatedConvolution.c index ca740f78e..ca740f78e 100644 --- a/contrib/torch/nn/lib/THNN/generic/VolumetricDilatedConvolution.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricDilatedConvolution.c diff --git a/contrib/torch/nn/lib/THNN/generic/VolumetricDilatedMaxPooling.c b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricDilatedMaxPooling.c index 66c0f9531..66c0f9531 100644 --- a/contrib/torch/nn/lib/THNN/generic/VolumetricDilatedMaxPooling.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricDilatedMaxPooling.c diff --git a/contrib/torch/nn/lib/THNN/generic/VolumetricFractionalMaxPooling.c b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricFractionalMaxPooling.c index 236986bb9..236986bb9 100644 --- a/contrib/torch/nn/lib/THNN/generic/VolumetricFractionalMaxPooling.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricFractionalMaxPooling.c diff --git a/contrib/torch/nn/lib/THNN/generic/VolumetricFullConvolution.c b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricFullConvolution.c index c974fab50..c974fab50 100644 --- a/contrib/torch/nn/lib/THNN/generic/VolumetricFullConvolution.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricFullConvolution.c diff --git a/contrib/torch/nn/lib/THNN/generic/VolumetricMaxPooling.c b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricMaxPooling.c index a3601e0b6..a3601e0b6 100644 --- a/contrib/torch/nn/lib/THNN/generic/VolumetricMaxPooling.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricMaxPooling.c diff --git a/contrib/torch/nn/lib/THNN/generic/VolumetricMaxUnpooling.c b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricMaxUnpooling.c index d9d9e5951..d9d9e5951 100644 --- a/contrib/torch/nn/lib/THNN/generic/VolumetricMaxUnpooling.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricMaxUnpooling.c diff --git a/contrib/torch/nn/lib/THNN/generic/VolumetricReplicationPadding.c b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricReplicationPadding.c index 4d8993ec2..4d8993ec2 100644 --- a/contrib/torch/nn/lib/THNN/generic/VolumetricReplicationPadding.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricReplicationPadding.c diff --git a/contrib/torch/nn/lib/THNN/generic/VolumetricUpSamplingNearest.c b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricUpSamplingNearest.c index 9068fb58d..9068fb58d 100644 --- a/contrib/torch/nn/lib/THNN/generic/VolumetricUpSamplingNearest.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricUpSamplingNearest.c diff --git a/contrib/torch/nn/lib/THNN/generic/VolumetricUpSamplingTrilinear.c b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricUpSamplingTrilinear.c index f2b04dba9..f2b04dba9 100644 --- a/contrib/torch/nn/lib/THNN/generic/VolumetricUpSamplingTrilinear.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricUpSamplingTrilinear.c diff --git a/contrib/torch/nn/lib/THNN/generic/unfold.c b/contrib/lua-torch/nn/lib/THNN/generic/unfold.c index 14a73b567..14a73b567 100644 --- a/contrib/torch/nn/lib/THNN/generic/unfold.c +++ b/contrib/lua-torch/nn/lib/THNN/generic/unfold.c diff --git a/contrib/torch/nn/lib/THNN/init.c b/contrib/lua-torch/nn/lib/THNN/init.c index 5c8c023dc..5c8c023dc 100644 --- a/contrib/torch/nn/lib/THNN/init.c +++ b/contrib/lua-torch/nn/lib/THNN/init.c diff --git a/contrib/torch/nn/mkdocs.yml b/contrib/lua-torch/nn/mkdocs.yml index a37a34fb0..a37a34fb0 100644 --- a/contrib/torch/nn/mkdocs.yml +++ b/contrib/lua-torch/nn/mkdocs.yml diff --git a/contrib/torch/nn/test.lua b/contrib/lua-torch/nn/test.lua index 4e3f627fc..4e3f627fc 100755 --- a/contrib/torch/nn/test.lua +++ b/contrib/lua-torch/nn/test.lua diff --git a/contrib/torch/nn/utils.lua b/contrib/lua-torch/nn/utils.lua index 17b52afb3..17b52afb3 100644 --- a/contrib/torch/nn/utils.lua +++ b/contrib/lua-torch/nn/utils.lua diff --git a/contrib/torch/optim/CMakeLists.txt b/contrib/lua-torch/optim/CMakeLists.txt index b1c13e701..b1c13e701 100644 --- a/contrib/torch/optim/CMakeLists.txt +++ b/contrib/lua-torch/optim/CMakeLists.txt diff --git a/contrib/torch/optim/COPYRIGHT.txt b/contrib/lua-torch/optim/COPYRIGHT.txt index 2e4118c0f..2e4118c0f 100644 --- a/contrib/torch/optim/COPYRIGHT.txt +++ b/contrib/lua-torch/optim/COPYRIGHT.txt diff --git a/contrib/torch/optim/ConfusionMatrix.lua b/contrib/lua-torch/optim/ConfusionMatrix.lua index ec5302c24..ec5302c24 100644 --- a/contrib/torch/optim/ConfusionMatrix.lua +++ b/contrib/lua-torch/optim/ConfusionMatrix.lua diff --git a/contrib/torch/optim/Logger.lua b/contrib/lua-torch/optim/Logger.lua index 3c1da54d3..3c1da54d3 100644 --- a/contrib/torch/optim/Logger.lua +++ b/contrib/lua-torch/optim/Logger.lua diff --git a/contrib/torch/optim/adadelta.lua b/contrib/lua-torch/optim/adadelta.lua index 7cc058d29..7cc058d29 100644 --- a/contrib/torch/optim/adadelta.lua +++ b/contrib/lua-torch/optim/adadelta.lua diff --git a/contrib/torch/optim/adagrad.lua b/contrib/lua-torch/optim/adagrad.lua index 6860c4317..6860c4317 100644 --- a/contrib/torch/optim/adagrad.lua +++ b/contrib/lua-torch/optim/adagrad.lua diff --git a/contrib/torch/optim/adam.lua b/contrib/lua-torch/optim/adam.lua index 2e127e96a..2e127e96a 100644 --- a/contrib/torch/optim/adam.lua +++ b/contrib/lua-torch/optim/adam.lua diff --git a/contrib/torch/optim/adamax.lua b/contrib/lua-torch/optim/adamax.lua index 2b6487720..2b6487720 100644 --- a/contrib/torch/optim/adamax.lua +++ b/contrib/lua-torch/optim/adamax.lua diff --git a/contrib/torch/optim/asgd.lua b/contrib/lua-torch/optim/asgd.lua index cc1c459f3..cc1c459f3 100644 --- a/contrib/torch/optim/asgd.lua +++ b/contrib/lua-torch/optim/asgd.lua diff --git a/contrib/torch/optim/cg.lua b/contrib/lua-torch/optim/cg.lua index 842a7d569..842a7d569 100644 --- a/contrib/torch/optim/cg.lua +++ b/contrib/lua-torch/optim/cg.lua diff --git a/contrib/torch/optim/checkgrad.lua b/contrib/lua-torch/optim/checkgrad.lua index 0382b2132..0382b2132 100644 --- a/contrib/torch/optim/checkgrad.lua +++ b/contrib/lua-torch/optim/checkgrad.lua diff --git a/contrib/torch/optim/cmaes.lua b/contrib/lua-torch/optim/cmaes.lua index 74cd58a0c..74cd58a0c 100644 --- a/contrib/torch/optim/cmaes.lua +++ b/contrib/lua-torch/optim/cmaes.lua diff --git a/contrib/torch/optim/de.lua b/contrib/lua-torch/optim/de.lua index 1e8e8001d..1e8e8001d 100644 --- a/contrib/torch/optim/de.lua +++ b/contrib/lua-torch/optim/de.lua diff --git a/contrib/torch/optim/fista.lua b/contrib/lua-torch/optim/fista.lua index c8c6f5e43..c8c6f5e43 100644 --- a/contrib/torch/optim/fista.lua +++ b/contrib/lua-torch/optim/fista.lua diff --git a/contrib/torch/optim/init.lua b/contrib/lua-torch/optim/init.lua index a045bd8a2..a045bd8a2 100644 --- a/contrib/torch/optim/init.lua +++ b/contrib/lua-torch/optim/init.lua diff --git a/contrib/torch/optim/lbfgs.lua b/contrib/lua-torch/optim/lbfgs.lua index d850fcbb3..d850fcbb3 100644 --- a/contrib/torch/optim/lbfgs.lua +++ b/contrib/lua-torch/optim/lbfgs.lua diff --git a/contrib/torch/optim/lswolfe.lua b/contrib/lua-torch/optim/lswolfe.lua index 0afbdbe8b..0afbdbe8b 100644 --- a/contrib/torch/optim/lswolfe.lua +++ b/contrib/lua-torch/optim/lswolfe.lua diff --git a/contrib/torch/optim/nag.lua b/contrib/lua-torch/optim/nag.lua index 875d81e4c..875d81e4c 100644 --- a/contrib/torch/optim/nag.lua +++ b/contrib/lua-torch/optim/nag.lua diff --git a/contrib/torch/optim/polyinterp.lua b/contrib/lua-torch/optim/polyinterp.lua index c5026bf04..c5026bf04 100644 --- a/contrib/torch/optim/polyinterp.lua +++ b/contrib/lua-torch/optim/polyinterp.lua diff --git a/contrib/torch/optim/rmsprop.lua b/contrib/lua-torch/optim/rmsprop.lua index aa562006a..aa562006a 100644 --- a/contrib/torch/optim/rmsprop.lua +++ b/contrib/lua-torch/optim/rmsprop.lua diff --git a/contrib/torch/optim/rprop.lua b/contrib/lua-torch/optim/rprop.lua index d7af16429..d7af16429 100644 --- a/contrib/torch/optim/rprop.lua +++ b/contrib/lua-torch/optim/rprop.lua diff --git a/contrib/torch/optim/sgd.lua b/contrib/lua-torch/optim/sgd.lua index e21c696a6..e21c696a6 100644 --- a/contrib/torch/optim/sgd.lua +++ b/contrib/lua-torch/optim/sgd.lua diff --git a/contrib/torch/paths/CMakeLists.txt b/contrib/lua-torch/paths/CMakeLists.txt index c578b66e9..c578b66e9 100644 --- a/contrib/torch/paths/CMakeLists.txt +++ b/contrib/lua-torch/paths/CMakeLists.txt diff --git a/contrib/torch/paths/COPYRIGHT.txt b/contrib/lua-torch/paths/COPYRIGHT.txt index bc002b78a..bc002b78a 100644 --- a/contrib/torch/paths/COPYRIGHT.txt +++ b/contrib/lua-torch/paths/COPYRIGHT.txt diff --git a/contrib/torch/paths/README.md b/contrib/lua-torch/paths/README.md index 2464c090e..2464c090e 100644 --- a/contrib/torch/paths/README.md +++ b/contrib/lua-torch/paths/README.md diff --git a/contrib/torch/paths/init.lua b/contrib/lua-torch/paths/init.lua index 9078c3523..9078c3523 100644 --- a/contrib/torch/paths/init.lua +++ b/contrib/lua-torch/paths/init.lua diff --git a/contrib/torch/paths/mkdocs.yml b/contrib/lua-torch/paths/mkdocs.yml index 1fe86d1b0..1fe86d1b0 100644 --- a/contrib/torch/paths/mkdocs.yml +++ b/contrib/lua-torch/paths/mkdocs.yml diff --git a/contrib/torch/paths/paths.c b/contrib/lua-torch/paths/paths.c index 7c3a72e54..7c3a72e54 100644 --- a/contrib/torch/paths/paths.c +++ b/contrib/lua-torch/paths/paths.c diff --git a/contrib/torch/paths/paths.h.in b/contrib/lua-torch/paths/paths.h.in index fb5417a6c..fb5417a6c 100644 --- a/contrib/torch/paths/paths.h.in +++ b/contrib/lua-torch/paths/paths.h.in diff --git a/contrib/torch/torch7/CMakeLists.txt b/contrib/lua-torch/torch7/CMakeLists.txt index 7519e85d7..1c3995dbe 100644 --- a/contrib/torch/torch7/CMakeLists.txt +++ b/contrib/lua-torch/torch7/CMakeLists.txt @@ -1,14 +1,4 @@ -IF(APPLE) - CMAKE_MINIMUM_REQUIRED(VERSION 2.8.12 FATAL_ERROR) - CMAKE_POLICY(VERSION 2.8.12) -ELSE() - CMAKE_MINIMUM_REQUIRED(VERSION 2.8 FATAL_ERROR) - CMAKE_POLICY(VERSION 2.8) -ENDIF() - -SET(CMAKE_MODULE_PATH - "${CMAKE_CURRENT_SOURCE_DIR}/cmake" - "${CMAKE_MODULE_PATH}") +LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") IF (NOT MSVC) IF (MINGW) @@ -18,14 +8,6 @@ IF (NOT MSVC) ENDIF(MINGW) ENDIF(NOT MSVC) -# Flags -# When using MSVC -IF(MSVC) - # we want to respect the standard, and we are bored of those **** . - ADD_DEFINITIONS(-D_CRT_SECURE_NO_DEPRECATE=1) -ENDIF(MSVC) - - IF (WITH_OPENMP) FIND_PACKAGE(OpenMP) IF(OPENMP_FOUND) diff --git a/contrib/torch/torch7/COPYRIGHT.txt b/contrib/lua-torch/torch7/COPYRIGHT.txt index bc002b78a..bc002b78a 100644 --- a/contrib/torch/torch7/COPYRIGHT.txt +++ b/contrib/lua-torch/torch7/COPYRIGHT.txt diff --git a/contrib/torch/torch7/CmdLine.lua b/contrib/lua-torch/torch7/CmdLine.lua index 643635069..643635069 100644 --- a/contrib/torch/torch7/CmdLine.lua +++ b/contrib/lua-torch/torch7/CmdLine.lua diff --git a/contrib/torch/torch7/DiskFile.c b/contrib/lua-torch/torch7/DiskFile.c index c50b74f95..c50b74f95 100644 --- a/contrib/torch/torch7/DiskFile.c +++ b/contrib/lua-torch/torch7/DiskFile.c diff --git a/contrib/torch/torch7/FFInterface.lua b/contrib/lua-torch/torch7/FFInterface.lua index cb8bd3365..cb8bd3365 100644 --- a/contrib/torch/torch7/FFInterface.lua +++ b/contrib/lua-torch/torch7/FFInterface.lua diff --git a/contrib/torch/torch7/File.c b/contrib/lua-torch/torch7/File.c index e07bc4666..e07bc4666 100644 --- a/contrib/torch/torch7/File.c +++ b/contrib/lua-torch/torch7/File.c diff --git a/contrib/torch/torch7/File.lua b/contrib/lua-torch/torch7/File.lua index 62249a361..62249a361 100644 --- a/contrib/torch/torch7/File.lua +++ b/contrib/lua-torch/torch7/File.lua diff --git a/contrib/torch/torch7/Generator.c b/contrib/lua-torch/torch7/Generator.c index 8cf5ba66c..8cf5ba66c 100644 --- a/contrib/torch/torch7/Generator.c +++ b/contrib/lua-torch/torch7/Generator.c diff --git a/contrib/torch/torch7/MemoryFile.c b/contrib/lua-torch/torch7/MemoryFile.c index a22dc17ec..a22dc17ec 100644 --- a/contrib/torch/torch7/MemoryFile.c +++ b/contrib/lua-torch/torch7/MemoryFile.c diff --git a/contrib/torch/torch7/PipeFile.c b/contrib/lua-torch/torch7/PipeFile.c index a47c90d13..a47c90d13 100644 --- a/contrib/torch/torch7/PipeFile.c +++ b/contrib/lua-torch/torch7/PipeFile.c diff --git a/contrib/torch/torch7/README.md b/contrib/lua-torch/torch7/README.md index 1a51c314c..1a51c314c 100644 --- a/contrib/torch/torch7/README.md +++ b/contrib/lua-torch/torch7/README.md diff --git a/contrib/torch/torch7/ROADMAP.md b/contrib/lua-torch/torch7/ROADMAP.md index 01bc8b8ac..01bc8b8ac 100644 --- a/contrib/torch/torch7/ROADMAP.md +++ b/contrib/lua-torch/torch7/ROADMAP.md diff --git a/contrib/torch/torch7/Storage.c b/contrib/lua-torch/torch7/Storage.c index 730d97476..730d97476 100644 --- a/contrib/torch/torch7/Storage.c +++ b/contrib/lua-torch/torch7/Storage.c diff --git a/contrib/torch/torch7/Tensor.c b/contrib/lua-torch/torch7/Tensor.c index bf78d1aae..bf78d1aae 100644 --- a/contrib/torch/torch7/Tensor.c +++ b/contrib/lua-torch/torch7/Tensor.c diff --git a/contrib/torch/torch7/Tensor.lua b/contrib/lua-torch/torch7/Tensor.lua index 9a8215be1..9a8215be1 100644 --- a/contrib/torch/torch7/Tensor.lua +++ b/contrib/lua-torch/torch7/Tensor.lua diff --git a/contrib/torch/torch7/TensorMath.c b/contrib/lua-torch/torch7/TensorMath.c index 84a821812..84a821812 100644 --- a/contrib/torch/torch7/TensorMath.c +++ b/contrib/lua-torch/torch7/TensorMath.c diff --git a/contrib/torch/torch7/TensorMath.lua b/contrib/lua-torch/torch7/TensorMath.lua index 45e07c63e..45e07c63e 100644 --- a/contrib/torch/torch7/TensorMath.lua +++ b/contrib/lua-torch/torch7/TensorMath.lua diff --git a/contrib/torch/torch7/TensorOperator.c b/contrib/lua-torch/torch7/TensorOperator.c index 8986ff761..8986ff761 100644 --- a/contrib/torch/torch7/TensorOperator.c +++ b/contrib/lua-torch/torch7/TensorOperator.c diff --git a/contrib/torch/torch7/TestSuite.lua b/contrib/lua-torch/torch7/TestSuite.lua index 630c2c948..630c2c948 100644 --- a/contrib/torch/torch7/TestSuite.lua +++ b/contrib/lua-torch/torch7/TestSuite.lua diff --git a/contrib/torch/torch7/Tester.lua b/contrib/lua-torch/torch7/Tester.lua index 6509413c2..6509413c2 100644 --- a/contrib/torch/torch7/Tester.lua +++ b/contrib/lua-torch/torch7/Tester.lua diff --git a/contrib/torch/torch7/Timer.c b/contrib/lua-torch/torch7/Timer.c index 13865b590..13865b590 100644 --- a/contrib/torch/torch7/Timer.c +++ b/contrib/lua-torch/torch7/Timer.c diff --git a/contrib/torch/torch7/cmake/TorchConfig.cmake.in b/contrib/lua-torch/torch7/cmake/TorchConfig.cmake.in index 3d85eb117..bafa04f91 100644 --- a/contrib/torch/torch7/cmake/TorchConfig.cmake.in +++ b/contrib/lua-torch/torch7/cmake/TorchConfig.cmake.in @@ -18,7 +18,7 @@ SET(Torch_INSTALL_CMAKE_RIDBUS "@Torch_INSTALL_CMAKE_RIDBUS@") FILE(RELATIVE_PATH Torch_INSTALL_LUA_PATH_SUBDIR "${Torch_INSTALL_PREFIX}" "${CMAKE_INSTALL_PREFIX}/lua") FILE(RELATIVE_PATH Torch_INSTALL_LUA_CPATH_SUBDIR "${Torch_INSTALL_PREFIX}" "${CMAKE_INSTALL_PREFIX}/lib") -SET(CMAKE_MODULE_PATH "${Torch_INSTALL_PREFIX}/${Torch_INSTALL_CMAKE_SUBDIR}" "${CMAKE_MODULE_PATH}") +LIST(APPEND CMAKE_MODULE_PATH "${Torch_INSTALL_PREFIX}/${Torch_INSTALL_CMAKE_SUBDIR}") SET(CMAKE_INSTALL_PREFIX "${Torch_INSTALL_PREFIX}") # override INCLUDE(TorchPathsInit) diff --git a/contrib/torch/torch7/cmake/TorchExports.cmake b/contrib/lua-torch/torch7/cmake/TorchExports.cmake index f47fd6f59..f47fd6f59 100644 --- a/contrib/torch/torch7/cmake/TorchExports.cmake +++ b/contrib/lua-torch/torch7/cmake/TorchExports.cmake diff --git a/contrib/torch/torch7/cmake/TorchPackage.cmake b/contrib/lua-torch/torch7/cmake/TorchPackage.cmake index 9ffdcf788..846b263a4 100644 --- a/contrib/torch/torch7/cmake/TorchPackage.cmake +++ b/contrib/lua-torch/torch7/cmake/TorchPackage.cmake @@ -19,10 +19,10 @@ ENDMACRO() MACRO(ADD_TORCH_PACKAGE package src luasrc) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) - INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/contrib/torch/torch7/lib/TH) - INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/contrib/torch/torch7/lib/luaT) - INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/contrib/torch/torch7/lib/TH) - INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/contrib/torch/torch7/lib/luaT) + INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/contrib/lua-torch/torch7/lib/TH) + INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/contrib/lua-torch/torch7/lib/luaT) + INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/contrib/lua-torch/torch7/lib/TH) + INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/contrib/lua-torch/torch7/lib/luaT) INCLUDE_DIRECTORIES(${Torch_LUA_INCLUDE_DIR}) ### C/C++ sources diff --git a/contrib/torch/torch7/cmake/TorchPaths.cmake b/contrib/lua-torch/torch7/cmake/TorchPaths.cmake index 70c7a4fb8..70c7a4fb8 100644 --- a/contrib/torch/torch7/cmake/TorchPaths.cmake +++ b/contrib/lua-torch/torch7/cmake/TorchPaths.cmake diff --git a/contrib/torch/torch7/cmake/TorchPathsInit.cmake b/contrib/lua-torch/torch7/cmake/TorchPathsInit.cmake index 42f8ffb3f..42f8ffb3f 100644 --- a/contrib/torch/torch7/cmake/TorchPathsInit.cmake +++ b/contrib/lua-torch/torch7/cmake/TorchPathsInit.cmake diff --git a/contrib/torch/torch7/cmake/TorchWrap.cmake b/contrib/lua-torch/torch7/cmake/TorchWrap.cmake index b367b2402..b367b2402 100644 --- a/contrib/torch/torch7/cmake/TorchWrap.cmake +++ b/contrib/lua-torch/torch7/cmake/TorchWrap.cmake diff --git a/contrib/torch/torch7/cmake/TorchWrap.cmake.in b/contrib/lua-torch/torch7/cmake/TorchWrap.cmake.in index 5c20445d1..5c20445d1 100644 --- a/contrib/torch/torch7/cmake/TorchWrap.cmake.in +++ b/contrib/lua-torch/torch7/cmake/TorchWrap.cmake.in diff --git a/contrib/torch/torch7/general.h b/contrib/lua-torch/torch7/general.h index 3ccf4bdf0..3ccf4bdf0 100644 --- a/contrib/torch/torch7/general.h +++ b/contrib/lua-torch/torch7/general.h diff --git a/contrib/torch/torch7/generic/Storage.c b/contrib/lua-torch/torch7/generic/Storage.c index b936e5714..b936e5714 100644 --- a/contrib/torch/torch7/generic/Storage.c +++ b/contrib/lua-torch/torch7/generic/Storage.c diff --git a/contrib/torch/torch7/generic/Tensor.c b/contrib/lua-torch/torch7/generic/Tensor.c index 112a4bd63..112a4bd63 100644 --- a/contrib/torch/torch7/generic/Tensor.c +++ b/contrib/lua-torch/torch7/generic/Tensor.c diff --git a/contrib/torch/torch7/generic/TensorOperator.c b/contrib/lua-torch/torch7/generic/TensorOperator.c index 37b2a0889..37b2a0889 100644 --- a/contrib/torch/torch7/generic/TensorOperator.c +++ b/contrib/lua-torch/torch7/generic/TensorOperator.c diff --git a/contrib/torch/torch7/generic/luaG.h b/contrib/lua-torch/torch7/generic/luaG.h index f1ffce29e..f1ffce29e 100644 --- a/contrib/torch/torch7/generic/luaG.h +++ b/contrib/lua-torch/torch7/generic/luaG.h diff --git a/contrib/torch/torch7/init.c b/contrib/lua-torch/torch7/init.c index 3bdac176d..3bdac176d 100644 --- a/contrib/torch/torch7/init.c +++ b/contrib/lua-torch/torch7/init.c diff --git a/contrib/torch/torch7/init.lua b/contrib/lua-torch/torch7/init.lua index 0f3cfbb1e..0f3cfbb1e 100644 --- a/contrib/torch/torch7/init.lua +++ b/contrib/lua-torch/torch7/init.lua diff --git a/contrib/torch/torch7/lib/CMakeLists.txt b/contrib/lua-torch/torch7/lib/CMakeLists.txt index d6a0e2c9c..d6a0e2c9c 100644 --- a/contrib/torch/torch7/lib/CMakeLists.txt +++ b/contrib/lua-torch/torch7/lib/CMakeLists.txt diff --git a/contrib/torch/torch7/lib/TH/CMakeLists.txt b/contrib/lua-torch/torch7/lib/TH/CMakeLists.txt index ef5bdd08c..b4f115f39 100644 --- a/contrib/torch/torch7/lib/TH/CMakeLists.txt +++ b/contrib/lua-torch/torch7/lib/TH/CMakeLists.txt @@ -5,7 +5,7 @@ IF(POLICY CMP0026) CMAKE_POLICY(SET CMP0026 OLD) ENDIF() -SET(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH}) +LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") SET(CMAKE_LIBRARY_PATH /usr/lib/x86_64-linux-gnu/ ${CMAKE_LIBRARY_PATH}) ####################################################################### diff --git a/contrib/torch/torch7/lib/TH/README.md b/contrib/lua-torch/torch7/lib/TH/README.md index 4ac26c103..4ac26c103 100644 --- a/contrib/torch/torch7/lib/TH/README.md +++ b/contrib/lua-torch/torch7/lib/TH/README.md diff --git a/contrib/torch/torch7/lib/TH/TH.h b/contrib/lua-torch/torch7/lib/TH/TH.h index 11f208c4b..11f208c4b 100644 --- a/contrib/torch/torch7/lib/TH/TH.h +++ b/contrib/lua-torch/torch7/lib/TH/TH.h diff --git a/contrib/torch/torch7/lib/TH/THAllocator.c b/contrib/lua-torch/torch7/lib/TH/THAllocator.c index 51ac69b94..51ac69b94 100644 --- a/contrib/torch/torch7/lib/TH/THAllocator.c +++ b/contrib/lua-torch/torch7/lib/TH/THAllocator.c diff --git a/contrib/torch/torch7/lib/TH/THAllocator.h b/contrib/lua-torch/torch7/lib/TH/THAllocator.h index 18fc9ec0a..18fc9ec0a 100644 --- a/contrib/torch/torch7/lib/TH/THAllocator.h +++ b/contrib/lua-torch/torch7/lib/TH/THAllocator.h diff --git a/contrib/torch/torch7/lib/TH/THAtomic.c b/contrib/lua-torch/torch7/lib/TH/THAtomic.c index 714fc52db..714fc52db 100644 --- a/contrib/torch/torch7/lib/TH/THAtomic.c +++ b/contrib/lua-torch/torch7/lib/TH/THAtomic.c diff --git a/contrib/torch/torch7/lib/TH/THAtomic.h b/contrib/lua-torch/torch7/lib/TH/THAtomic.h index d77b20b24..d77b20b24 100644 --- a/contrib/torch/torch7/lib/TH/THAtomic.h +++ b/contrib/lua-torch/torch7/lib/TH/THAtomic.h diff --git a/contrib/torch/torch7/lib/TH/THBlas.c b/contrib/lua-torch/torch7/lib/TH/THBlas.c index 35618b26a..35618b26a 100644 --- a/contrib/torch/torch7/lib/TH/THBlas.c +++ b/contrib/lua-torch/torch7/lib/TH/THBlas.c diff --git a/contrib/torch/torch7/lib/TH/THBlas.h b/contrib/lua-torch/torch7/lib/TH/THBlas.h index 5fef0febc..5fef0febc 100644 --- a/contrib/torch/torch7/lib/TH/THBlas.h +++ b/contrib/lua-torch/torch7/lib/TH/THBlas.h diff --git a/contrib/torch/torch7/lib/TH/THConfig.cmake.in b/contrib/lua-torch/torch7/lib/TH/THConfig.cmake.in index 306cd878b..306cd878b 100644 --- a/contrib/torch/torch7/lib/TH/THConfig.cmake.in +++ b/contrib/lua-torch/torch7/lib/TH/THConfig.cmake.in diff --git a/contrib/torch/torch7/lib/TH/THDiskFile.c b/contrib/lua-torch/torch7/lib/TH/THDiskFile.c index 3f57b3b35..3f57b3b35 100644 --- a/contrib/torch/torch7/lib/TH/THDiskFile.c +++ b/contrib/lua-torch/torch7/lib/TH/THDiskFile.c diff --git a/contrib/torch/torch7/lib/TH/THDiskFile.h b/contrib/lua-torch/torch7/lib/TH/THDiskFile.h index bc5c001c7..bc5c001c7 100644 --- a/contrib/torch/torch7/lib/TH/THDiskFile.h +++ b/contrib/lua-torch/torch7/lib/TH/THDiskFile.h diff --git a/contrib/torch/torch7/lib/TH/THFile.c b/contrib/lua-torch/torch7/lib/TH/THFile.c index 3717b7b5c..3717b7b5c 100644 --- a/contrib/torch/torch7/lib/TH/THFile.c +++ b/contrib/lua-torch/torch7/lib/TH/THFile.c diff --git a/contrib/torch/torch7/lib/TH/THFile.h b/contrib/lua-torch/torch7/lib/TH/THFile.h index e097bdf34..e097bdf34 100644 --- a/contrib/torch/torch7/lib/TH/THFile.h +++ b/contrib/lua-torch/torch7/lib/TH/THFile.h diff --git a/contrib/torch/torch7/lib/TH/THFilePrivate.h b/contrib/lua-torch/torch7/lib/TH/THFilePrivate.h index 55169c3bc..55169c3bc 100644 --- a/contrib/torch/torch7/lib/TH/THFilePrivate.h +++ b/contrib/lua-torch/torch7/lib/TH/THFilePrivate.h diff --git a/contrib/torch/torch7/lib/TH/THGeneral.c b/contrib/lua-torch/torch7/lib/TH/THGeneral.c index 6ad96b51c..6ad96b51c 100644 --- a/contrib/torch/torch7/lib/TH/THGeneral.c +++ b/contrib/lua-torch/torch7/lib/TH/THGeneral.c diff --git a/contrib/torch/torch7/lib/TH/THGeneral.h.in b/contrib/lua-torch/torch7/lib/TH/THGeneral.h.in index 88a3934c8..88a3934c8 100644 --- a/contrib/torch/torch7/lib/TH/THGeneral.h.in +++ b/contrib/lua-torch/torch7/lib/TH/THGeneral.h.in diff --git a/contrib/torch/torch7/lib/TH/THGenerateAllTypes.h b/contrib/lua-torch/torch7/lib/TH/THGenerateAllTypes.h index 5b9508df7..5b9508df7 100644 --- a/contrib/torch/torch7/lib/TH/THGenerateAllTypes.h +++ b/contrib/lua-torch/torch7/lib/TH/THGenerateAllTypes.h diff --git a/contrib/torch/torch7/lib/TH/THGenerateByteType.h b/contrib/lua-torch/torch7/lib/TH/THGenerateByteType.h index 71ce7c405..71ce7c405 100644 --- a/contrib/torch/torch7/lib/TH/THGenerateByteType.h +++ b/contrib/lua-torch/torch7/lib/TH/THGenerateByteType.h diff --git a/contrib/torch/torch7/lib/TH/THGenerateCharType.h b/contrib/lua-torch/torch7/lib/TH/THGenerateCharType.h index 158dd0e80..158dd0e80 100644 --- a/contrib/torch/torch7/lib/TH/THGenerateCharType.h +++ b/contrib/lua-torch/torch7/lib/TH/THGenerateCharType.h diff --git a/contrib/torch/torch7/lib/TH/THGenerateDoubleType.h b/contrib/lua-torch/torch7/lib/TH/THGenerateDoubleType.h index fffee606d..fffee606d 100644 --- a/contrib/torch/torch7/lib/TH/THGenerateDoubleType.h +++ b/contrib/lua-torch/torch7/lib/TH/THGenerateDoubleType.h diff --git a/contrib/torch/torch7/lib/TH/THGenerateFloatType.h b/contrib/lua-torch/torch7/lib/TH/THGenerateFloatType.h index a31b50c55..a31b50c55 100644 --- a/contrib/torch/torch7/lib/TH/THGenerateFloatType.h +++ b/contrib/lua-torch/torch7/lib/TH/THGenerateFloatType.h diff --git a/contrib/torch/torch7/lib/TH/THGenerateFloatTypes.h b/contrib/lua-torch/torch7/lib/TH/THGenerateFloatTypes.h index be5ea8403..be5ea8403 100644 --- a/contrib/torch/torch7/lib/TH/THGenerateFloatTypes.h +++ b/contrib/lua-torch/torch7/lib/TH/THGenerateFloatTypes.h diff --git a/contrib/torch/torch7/lib/TH/THGenerateHalfType.h b/contrib/lua-torch/torch7/lib/TH/THGenerateHalfType.h index 47ff1e8d7..47ff1e8d7 100644 --- a/contrib/torch/torch7/lib/TH/THGenerateHalfType.h +++ b/contrib/lua-torch/torch7/lib/TH/THGenerateHalfType.h diff --git a/contrib/torch/torch7/lib/TH/THGenerateIntType.h b/contrib/lua-torch/torch7/lib/TH/THGenerateIntType.h index 1562b9e98..1562b9e98 100644 --- a/contrib/torch/torch7/lib/TH/THGenerateIntType.h +++ b/contrib/lua-torch/torch7/lib/TH/THGenerateIntType.h diff --git a/contrib/torch/torch7/lib/TH/THGenerateIntTypes.h b/contrib/lua-torch/torch7/lib/TH/THGenerateIntTypes.h index 9931fb1f5..9931fb1f5 100644 --- a/contrib/torch/torch7/lib/TH/THGenerateIntTypes.h +++ b/contrib/lua-torch/torch7/lib/TH/THGenerateIntTypes.h diff --git a/contrib/torch/torch7/lib/TH/THGenerateLongType.h b/contrib/lua-torch/torch7/lib/TH/THGenerateLongType.h index 75f90e1a6..75f90e1a6 100644 --- a/contrib/torch/torch7/lib/TH/THGenerateLongType.h +++ b/contrib/lua-torch/torch7/lib/TH/THGenerateLongType.h diff --git a/contrib/torch/torch7/lib/TH/THGenerateShortType.h b/contrib/lua-torch/torch7/lib/TH/THGenerateShortType.h index 047e51a8d..047e51a8d 100644 --- a/contrib/torch/torch7/lib/TH/THGenerateShortType.h +++ b/contrib/lua-torch/torch7/lib/TH/THGenerateShortType.h diff --git a/contrib/torch/torch7/lib/TH/THHalf.c b/contrib/lua-torch/torch7/lib/TH/THHalf.c index d7468ac3d..d7468ac3d 100644 --- a/contrib/torch/torch7/lib/TH/THHalf.c +++ b/contrib/lua-torch/torch7/lib/TH/THHalf.c diff --git a/contrib/torch/torch7/lib/TH/THHalf.h b/contrib/lua-torch/torch7/lib/TH/THHalf.h index 0f9807b50..0f9807b50 100644 --- a/contrib/torch/torch7/lib/TH/THHalf.h +++ b/contrib/lua-torch/torch7/lib/TH/THHalf.h diff --git a/contrib/torch/torch7/lib/TH/THLapack.c b/contrib/lua-torch/torch7/lib/TH/THLapack.c index bd4dc716b..bd4dc716b 100644 --- a/contrib/torch/torch7/lib/TH/THLapack.c +++ b/contrib/lua-torch/torch7/lib/TH/THLapack.c diff --git a/contrib/torch/torch7/lib/TH/THLapack.h b/contrib/lua-torch/torch7/lib/TH/THLapack.h index 614d15f94..614d15f94 100644 --- a/contrib/torch/torch7/lib/TH/THLapack.h +++ b/contrib/lua-torch/torch7/lib/TH/THLapack.h diff --git a/contrib/torch/torch7/lib/TH/THLogAdd.c b/contrib/lua-torch/torch7/lib/TH/THLogAdd.c index 4b14f8540..4b14f8540 100644 --- a/contrib/torch/torch7/lib/TH/THLogAdd.c +++ b/contrib/lua-torch/torch7/lib/TH/THLogAdd.c diff --git a/contrib/torch/torch7/lib/TH/THLogAdd.h b/contrib/lua-torch/torch7/lib/TH/THLogAdd.h index 9319b8f46..9319b8f46 100644 --- a/contrib/torch/torch7/lib/TH/THLogAdd.h +++ b/contrib/lua-torch/torch7/lib/TH/THLogAdd.h diff --git a/contrib/torch/torch7/lib/TH/THMath.h b/contrib/lua-torch/torch7/lib/TH/THMath.h index 004e4fe45..004e4fe45 100644 --- a/contrib/torch/torch7/lib/TH/THMath.h +++ b/contrib/lua-torch/torch7/lib/TH/THMath.h diff --git a/contrib/torch/torch7/lib/TH/THMemoryFile.c b/contrib/lua-torch/torch7/lib/TH/THMemoryFile.c index ecce6e1b1..ecce6e1b1 100644 --- a/contrib/torch/torch7/lib/TH/THMemoryFile.c +++ b/contrib/lua-torch/torch7/lib/TH/THMemoryFile.c diff --git a/contrib/torch/torch7/lib/TH/THMemoryFile.h b/contrib/lua-torch/torch7/lib/TH/THMemoryFile.h index b54cdcc2f..b54cdcc2f 100644 --- a/contrib/torch/torch7/lib/TH/THMemoryFile.h +++ b/contrib/lua-torch/torch7/lib/TH/THMemoryFile.h diff --git a/contrib/torch/torch7/lib/TH/THRandom.c b/contrib/lua-torch/torch7/lib/TH/THRandom.c index 86d721e7b..86d721e7b 100644 --- a/contrib/torch/torch7/lib/TH/THRandom.c +++ b/contrib/lua-torch/torch7/lib/TH/THRandom.c diff --git a/contrib/torch/torch7/lib/TH/THRandom.h b/contrib/lua-torch/torch7/lib/TH/THRandom.h index 28a14c0d7..28a14c0d7 100644 --- a/contrib/torch/torch7/lib/TH/THRandom.h +++ b/contrib/lua-torch/torch7/lib/TH/THRandom.h diff --git a/contrib/torch/torch7/lib/TH/THSize.c b/contrib/lua-torch/torch7/lib/TH/THSize.c index ccf1f61dd..ccf1f61dd 100644 --- a/contrib/torch/torch7/lib/TH/THSize.c +++ b/contrib/lua-torch/torch7/lib/TH/THSize.c diff --git a/contrib/torch/torch7/lib/TH/THSize.h b/contrib/lua-torch/torch7/lib/TH/THSize.h index 3d39696f6..3d39696f6 100644 --- a/contrib/torch/torch7/lib/TH/THSize.h +++ b/contrib/lua-torch/torch7/lib/TH/THSize.h diff --git a/contrib/torch/torch7/lib/TH/THStorage.c b/contrib/lua-torch/torch7/lib/TH/THStorage.c index f6b63f4a8..f6b63f4a8 100644 --- a/contrib/torch/torch7/lib/TH/THStorage.c +++ b/contrib/lua-torch/torch7/lib/TH/THStorage.c diff --git a/contrib/torch/torch7/lib/TH/THStorage.h b/contrib/lua-torch/torch7/lib/TH/THStorage.h index fb7946bd9..fb7946bd9 100644 --- a/contrib/torch/torch7/lib/TH/THStorage.h +++ b/contrib/lua-torch/torch7/lib/TH/THStorage.h diff --git a/contrib/torch/torch7/lib/TH/THTensor.c b/contrib/lua-torch/torch7/lib/TH/THTensor.c index 115e396a1..115e396a1 100644 --- a/contrib/torch/torch7/lib/TH/THTensor.c +++ b/contrib/lua-torch/torch7/lib/TH/THTensor.c diff --git a/contrib/torch/torch7/lib/TH/THTensor.h b/contrib/lua-torch/torch7/lib/TH/THTensor.h index d2a1c57e8..d2a1c57e8 100644 --- a/contrib/torch/torch7/lib/TH/THTensor.h +++ b/contrib/lua-torch/torch7/lib/TH/THTensor.h diff --git a/contrib/torch/torch7/lib/TH/THTensorApply.h b/contrib/lua-torch/torch7/lib/TH/THTensorApply.h index 7f48da47e..7f48da47e 100644 --- a/contrib/torch/torch7/lib/TH/THTensorApply.h +++ b/contrib/lua-torch/torch7/lib/TH/THTensorApply.h diff --git a/contrib/torch/torch7/lib/TH/THTensorDimApply.h b/contrib/lua-torch/torch7/lib/TH/THTensorDimApply.h index 6727e1f7f..6727e1f7f 100644 --- a/contrib/torch/torch7/lib/TH/THTensorDimApply.h +++ b/contrib/lua-torch/torch7/lib/TH/THTensorDimApply.h diff --git a/contrib/torch/torch7/lib/TH/THTensorMacros.h b/contrib/lua-torch/torch7/lib/TH/THTensorMacros.h index 15b67665e..15b67665e 100644 --- a/contrib/torch/torch7/lib/TH/THTensorMacros.h +++ b/contrib/lua-torch/torch7/lib/TH/THTensorMacros.h diff --git a/contrib/torch/torch7/lib/TH/THVector.c b/contrib/lua-torch/torch7/lib/TH/THVector.c index 441057884..441057884 100644 --- a/contrib/torch/torch7/lib/TH/THVector.c +++ b/contrib/lua-torch/torch7/lib/TH/THVector.c diff --git a/contrib/torch/torch7/lib/TH/THVector.h b/contrib/lua-torch/torch7/lib/TH/THVector.h index e29917b93..e29917b93 100644 --- a/contrib/torch/torch7/lib/TH/THVector.h +++ b/contrib/lua-torch/torch7/lib/TH/THVector.h diff --git a/contrib/torch/torch7/lib/TH/cmake/FindARM.cmake b/contrib/lua-torch/torch7/lib/TH/cmake/FindARM.cmake index 2dcb2a24f..2dcb2a24f 100644 --- a/contrib/torch/torch7/lib/TH/cmake/FindARM.cmake +++ b/contrib/lua-torch/torch7/lib/TH/cmake/FindARM.cmake diff --git a/contrib/torch/torch7/lib/TH/cmake/FindBLAS.cmake b/contrib/lua-torch/torch7/lib/TH/cmake/FindBLAS.cmake index 1f254d231..1f254d231 100644 --- a/contrib/torch/torch7/lib/TH/cmake/FindBLAS.cmake +++ b/contrib/lua-torch/torch7/lib/TH/cmake/FindBLAS.cmake diff --git a/contrib/torch/torch7/lib/TH/cmake/FindLAPACK.cmake b/contrib/lua-torch/torch7/lib/TH/cmake/FindLAPACK.cmake index 9eca0730f..9eca0730f 100644 --- a/contrib/torch/torch7/lib/TH/cmake/FindLAPACK.cmake +++ b/contrib/lua-torch/torch7/lib/TH/cmake/FindLAPACK.cmake diff --git a/contrib/torch/torch7/lib/TH/cmake/FindMKL.cmake b/contrib/lua-torch/torch7/lib/TH/cmake/FindMKL.cmake index 08b450985..08b450985 100644 --- a/contrib/torch/torch7/lib/TH/cmake/FindMKL.cmake +++ b/contrib/lua-torch/torch7/lib/TH/cmake/FindMKL.cmake diff --git a/contrib/torch/torch7/lib/TH/cmake/FindSSE.cmake b/contrib/lua-torch/torch7/lib/TH/cmake/FindSSE.cmake index a14abe8d4..a14abe8d4 100644 --- a/contrib/torch/torch7/lib/TH/cmake/FindSSE.cmake +++ b/contrib/lua-torch/torch7/lib/TH/cmake/FindSSE.cmake diff --git a/contrib/torch/torch7/lib/TH/generic/THBlas.c b/contrib/lua-torch/torch7/lib/TH/generic/THBlas.c index b04931f34..b04931f34 100644 --- a/contrib/torch/torch7/lib/TH/generic/THBlas.c +++ b/contrib/lua-torch/torch7/lib/TH/generic/THBlas.c diff --git a/contrib/torch/torch7/lib/TH/generic/THBlas.h b/contrib/lua-torch/torch7/lib/TH/generic/THBlas.h index 9e14f5a84..9e14f5a84 100644 --- a/contrib/torch/torch7/lib/TH/generic/THBlas.h +++ b/contrib/lua-torch/torch7/lib/TH/generic/THBlas.h diff --git a/contrib/torch/torch7/lib/TH/generic/THLapack.c b/contrib/lua-torch/torch7/lib/TH/generic/THLapack.c index 148ae26c4..148ae26c4 100644 --- a/contrib/torch/torch7/lib/TH/generic/THLapack.c +++ b/contrib/lua-torch/torch7/lib/TH/generic/THLapack.c diff --git a/contrib/torch/torch7/lib/TH/generic/THLapack.h b/contrib/lua-torch/torch7/lib/TH/generic/THLapack.h index b464dd2d2..b464dd2d2 100644 --- a/contrib/torch/torch7/lib/TH/generic/THLapack.h +++ b/contrib/lua-torch/torch7/lib/TH/generic/THLapack.h diff --git a/contrib/torch/torch7/lib/TH/generic/THStorage.c b/contrib/lua-torch/torch7/lib/TH/generic/THStorage.c index a592cfb62..a592cfb62 100644 --- a/contrib/torch/torch7/lib/TH/generic/THStorage.c +++ b/contrib/lua-torch/torch7/lib/TH/generic/THStorage.c diff --git a/contrib/torch/torch7/lib/TH/generic/THStorage.h b/contrib/lua-torch/torch7/lib/TH/generic/THStorage.h index 3dd214b33..3dd214b33 100644 --- a/contrib/torch/torch7/lib/TH/generic/THStorage.h +++ b/contrib/lua-torch/torch7/lib/TH/generic/THStorage.h diff --git a/contrib/torch/torch7/lib/TH/generic/THStorageCopy.c b/contrib/lua-torch/torch7/lib/TH/generic/THStorageCopy.c index ce4b57eaf..ce4b57eaf 100644 --- a/contrib/torch/torch7/lib/TH/generic/THStorageCopy.c +++ b/contrib/lua-torch/torch7/lib/TH/generic/THStorageCopy.c diff --git a/contrib/torch/torch7/lib/TH/generic/THStorageCopy.h b/contrib/lua-torch/torch7/lib/TH/generic/THStorageCopy.h index ce8a2a690..ce8a2a690 100644 --- a/contrib/torch/torch7/lib/TH/generic/THStorageCopy.h +++ b/contrib/lua-torch/torch7/lib/TH/generic/THStorageCopy.h diff --git a/contrib/torch/torch7/lib/TH/generic/THTensor.c b/contrib/lua-torch/torch7/lib/TH/generic/THTensor.c index e44e06ec3..e44e06ec3 100644 --- a/contrib/torch/torch7/lib/TH/generic/THTensor.c +++ b/contrib/lua-torch/torch7/lib/TH/generic/THTensor.c diff --git a/contrib/torch/torch7/lib/TH/generic/THTensor.h b/contrib/lua-torch/torch7/lib/TH/generic/THTensor.h index 9fb246c85..9fb246c85 100644 --- a/contrib/torch/torch7/lib/TH/generic/THTensor.h +++ b/contrib/lua-torch/torch7/lib/TH/generic/THTensor.h diff --git a/contrib/torch/torch7/lib/TH/generic/THTensorConv.c b/contrib/lua-torch/torch7/lib/TH/generic/THTensorConv.c index 684ff9db5..684ff9db5 100644 --- a/contrib/torch/torch7/lib/TH/generic/THTensorConv.c +++ b/contrib/lua-torch/torch7/lib/TH/generic/THTensorConv.c diff --git a/contrib/torch/torch7/lib/TH/generic/THTensorConv.h b/contrib/lua-torch/torch7/lib/TH/generic/THTensorConv.h index 79866f390..79866f390 100644 --- a/contrib/torch/torch7/lib/TH/generic/THTensorConv.h +++ b/contrib/lua-torch/torch7/lib/TH/generic/THTensorConv.h diff --git a/contrib/torch/torch7/lib/TH/generic/THTensorCopy.c b/contrib/lua-torch/torch7/lib/TH/generic/THTensorCopy.c index d9cd1c0d5..d9cd1c0d5 100644 --- a/contrib/torch/torch7/lib/TH/generic/THTensorCopy.c +++ b/contrib/lua-torch/torch7/lib/TH/generic/THTensorCopy.c diff --git a/contrib/torch/torch7/lib/TH/generic/THTensorCopy.h b/contrib/lua-torch/torch7/lib/TH/generic/THTensorCopy.h index b9e5bfc99..b9e5bfc99 100644 --- a/contrib/torch/torch7/lib/TH/generic/THTensorCopy.h +++ b/contrib/lua-torch/torch7/lib/TH/generic/THTensorCopy.h diff --git a/contrib/torch/torch7/lib/TH/generic/THTensorLapack.c b/contrib/lua-torch/torch7/lib/TH/generic/THTensorLapack.c index d4e52f6d7..d4e52f6d7 100644 --- a/contrib/torch/torch7/lib/TH/generic/THTensorLapack.c +++ b/contrib/lua-torch/torch7/lib/TH/generic/THTensorLapack.c diff --git a/contrib/torch/torch7/lib/TH/generic/THTensorLapack.h b/contrib/lua-torch/torch7/lib/TH/generic/THTensorLapack.h index 878594348..878594348 100644 --- a/contrib/torch/torch7/lib/TH/generic/THTensorLapack.h +++ b/contrib/lua-torch/torch7/lib/TH/generic/THTensorLapack.h diff --git a/contrib/torch/torch7/lib/TH/generic/THTensorMath.c b/contrib/lua-torch/torch7/lib/TH/generic/THTensorMath.c index db7a0cb19..db7a0cb19 100644 --- a/contrib/torch/torch7/lib/TH/generic/THTensorMath.c +++ b/contrib/lua-torch/torch7/lib/TH/generic/THTensorMath.c diff --git a/contrib/torch/torch7/lib/TH/generic/THTensorMath.h b/contrib/lua-torch/torch7/lib/TH/generic/THTensorMath.h index 17e54ccf6..17e54ccf6 100644 --- a/contrib/torch/torch7/lib/TH/generic/THTensorMath.h +++ b/contrib/lua-torch/torch7/lib/TH/generic/THTensorMath.h diff --git a/contrib/torch/torch7/lib/TH/generic/THTensorRandom.c b/contrib/lua-torch/torch7/lib/TH/generic/THTensorRandom.c index 514d3dd27..514d3dd27 100644 --- a/contrib/torch/torch7/lib/TH/generic/THTensorRandom.c +++ b/contrib/lua-torch/torch7/lib/TH/generic/THTensorRandom.c diff --git a/contrib/torch/torch7/lib/TH/generic/THTensorRandom.h b/contrib/lua-torch/torch7/lib/TH/generic/THTensorRandom.h index d20514242..d20514242 100644 --- a/contrib/torch/torch7/lib/TH/generic/THTensorRandom.h +++ b/contrib/lua-torch/torch7/lib/TH/generic/THTensorRandom.h diff --git a/contrib/torch/torch7/lib/TH/generic/THVector.h b/contrib/lua-torch/torch7/lib/TH/generic/THVector.h index 7d368541a..7d368541a 100644 --- a/contrib/torch/torch7/lib/TH/generic/THVector.h +++ b/contrib/lua-torch/torch7/lib/TH/generic/THVector.h diff --git a/contrib/torch/torch7/lib/TH/generic/THVectorDefault.c b/contrib/lua-torch/torch7/lib/TH/generic/THVectorDefault.c index 3388e0d9b..3388e0d9b 100644 --- a/contrib/torch/torch7/lib/TH/generic/THVectorDefault.c +++ b/contrib/lua-torch/torch7/lib/TH/generic/THVectorDefault.c diff --git a/contrib/torch/torch7/lib/TH/generic/THVectorDispatch.c b/contrib/lua-torch/torch7/lib/TH/generic/THVectorDispatch.c index 5b8885283..5b8885283 100644 --- a/contrib/torch/torch7/lib/TH/generic/THVectorDispatch.c +++ b/contrib/lua-torch/torch7/lib/TH/generic/THVectorDispatch.c diff --git a/contrib/torch/torch7/lib/TH/generic/simd/common_simd.h b/contrib/lua-torch/torch7/lib/TH/generic/simd/common_simd.h index 425b4b96e..425b4b96e 100644 --- a/contrib/torch/torch7/lib/TH/generic/simd/common_simd.h +++ b/contrib/lua-torch/torch7/lib/TH/generic/simd/common_simd.h diff --git a/contrib/torch/torch7/lib/TH/generic/simd/convolve.c b/contrib/lua-torch/torch7/lib/TH/generic/simd/convolve.c index da7a4bb20..da7a4bb20 100644 --- a/contrib/torch/torch7/lib/TH/generic/simd/convolve.c +++ b/contrib/lua-torch/torch7/lib/TH/generic/simd/convolve.c diff --git a/contrib/torch/torch7/lib/TH/generic/simd/convolve.h b/contrib/lua-torch/torch7/lib/TH/generic/simd/convolve.h index 7b9b04c50..7b9b04c50 100644 --- a/contrib/torch/torch7/lib/TH/generic/simd/convolve.h +++ b/contrib/lua-torch/torch7/lib/TH/generic/simd/convolve.h diff --git a/contrib/torch/torch7/lib/TH/generic/simd/convolve5x5_avx.c b/contrib/lua-torch/torch7/lib/TH/generic/simd/convolve5x5_avx.c index 52b6d0ffb..52b6d0ffb 100644 --- a/contrib/torch/torch7/lib/TH/generic/simd/convolve5x5_avx.c +++ b/contrib/lua-torch/torch7/lib/TH/generic/simd/convolve5x5_avx.c diff --git a/contrib/torch/torch7/lib/TH/generic/simd/convolve5x5_sse.c b/contrib/lua-torch/torch7/lib/TH/generic/simd/convolve5x5_sse.c index f34b79695..f34b79695 100644 --- a/contrib/torch/torch7/lib/TH/generic/simd/convolve5x5_sse.c +++ b/contrib/lua-torch/torch7/lib/TH/generic/simd/convolve5x5_sse.c diff --git a/contrib/torch/torch7/lib/TH/generic/simd/simd.h b/contrib/lua-torch/torch7/lib/TH/generic/simd/simd.h index 83c4c5681..83c4c5681 100644 --- a/contrib/torch/torch7/lib/TH/generic/simd/simd.h +++ b/contrib/lua-torch/torch7/lib/TH/generic/simd/simd.h diff --git a/contrib/torch/torch7/lib/TH/vector/AVX.c b/contrib/lua-torch/torch7/lib/TH/vector/AVX.c index 58c4e6d35..58c4e6d35 100644 --- a/contrib/torch/torch7/lib/TH/vector/AVX.c +++ b/contrib/lua-torch/torch7/lib/TH/vector/AVX.c diff --git a/contrib/torch/torch7/lib/TH/vector/AVX.h b/contrib/lua-torch/torch7/lib/TH/vector/AVX.h index bfaeaa6b0..bfaeaa6b0 100644 --- a/contrib/torch/torch7/lib/TH/vector/AVX.h +++ b/contrib/lua-torch/torch7/lib/TH/vector/AVX.h diff --git a/contrib/torch/torch7/lib/TH/vector/AVX2.c b/contrib/lua-torch/torch7/lib/TH/vector/AVX2.c index 082a680ea..082a680ea 100644 --- a/contrib/torch/torch7/lib/TH/vector/AVX2.c +++ b/contrib/lua-torch/torch7/lib/TH/vector/AVX2.c diff --git a/contrib/torch/torch7/lib/TH/vector/AVX2.h b/contrib/lua-torch/torch7/lib/TH/vector/AVX2.h index 85a9e93ee..85a9e93ee 100644 --- a/contrib/torch/torch7/lib/TH/vector/AVX2.h +++ b/contrib/lua-torch/torch7/lib/TH/vector/AVX2.h diff --git a/contrib/torch/torch7/lib/TH/vector/NEON.c b/contrib/lua-torch/torch7/lib/TH/vector/NEON.c index 7920fb13b..7920fb13b 100644 --- a/contrib/torch/torch7/lib/TH/vector/NEON.c +++ b/contrib/lua-torch/torch7/lib/TH/vector/NEON.c diff --git a/contrib/torch/torch7/lib/TH/vector/SSE.c b/contrib/lua-torch/torch7/lib/TH/vector/SSE.c index d026935ab..d026935ab 100644 --- a/contrib/torch/torch7/lib/TH/vector/SSE.c +++ b/contrib/lua-torch/torch7/lib/TH/vector/SSE.c diff --git a/contrib/torch/torch7/lib/TH/vector/VSX.c b/contrib/lua-torch/torch7/lib/TH/vector/VSX.c index 9ff984ad7..9ff984ad7 100644 --- a/contrib/torch/torch7/lib/TH/vector/VSX.c +++ b/contrib/lua-torch/torch7/lib/TH/vector/VSX.c diff --git a/contrib/torch/torch7/lib/luaT/CMakeLists.txt b/contrib/lua-torch/torch7/lib/luaT/CMakeLists.txt index ebe788cef..ebe788cef 100644 --- a/contrib/torch/torch7/lib/luaT/CMakeLists.txt +++ b/contrib/lua-torch/torch7/lib/luaT/CMakeLists.txt diff --git a/contrib/torch/torch7/lib/luaT/README.md b/contrib/lua-torch/torch7/lib/luaT/README.md index 235b8edc0..235b8edc0 100644 --- a/contrib/torch/torch7/lib/luaT/README.md +++ b/contrib/lua-torch/torch7/lib/luaT/README.md diff --git a/contrib/torch/torch7/lib/luaT/luaT.c b/contrib/lua-torch/torch7/lib/luaT/luaT.c index d87f5d54c..d87f5d54c 100644 --- a/contrib/torch/torch7/lib/luaT/luaT.c +++ b/contrib/lua-torch/torch7/lib/luaT/luaT.c diff --git a/contrib/torch/torch7/lib/luaT/luaT.h b/contrib/lua-torch/torch7/lib/luaT/luaT.h index 2479a1dc1..2479a1dc1 100644 --- a/contrib/torch/torch7/lib/luaT/luaT.h +++ b/contrib/lua-torch/torch7/lib/luaT/luaT.h diff --git a/contrib/torch/torch7/lib/luaT/luaTConfig.cmake.in b/contrib/lua-torch/torch7/lib/luaT/luaTConfig.cmake.in index bfb20b87a..bfb20b87a 100644 --- a/contrib/torch/torch7/lib/luaT/luaTConfig.cmake.in +++ b/contrib/lua-torch/torch7/lib/luaT/luaTConfig.cmake.in diff --git a/contrib/torch/torch7/mkdocs.yml b/contrib/lua-torch/torch7/mkdocs.yml index 39a34d7c2..39a34d7c2 100644 --- a/contrib/torch/torch7/mkdocs.yml +++ b/contrib/lua-torch/torch7/mkdocs.yml diff --git a/contrib/torch/torch7/paths.lua.in b/contrib/lua-torch/torch7/paths.lua.in index 287770b12..287770b12 100644 --- a/contrib/torch/torch7/paths.lua.in +++ b/contrib/lua-torch/torch7/paths.lua.in diff --git a/contrib/torch/torch7/random.c b/contrib/lua-torch/torch7/random.c index bbe6ba08e..bbe6ba08e 100644 --- a/contrib/torch/torch7/random.c +++ b/contrib/lua-torch/torch7/random.c diff --git a/contrib/torch/torch7/random.lua b/contrib/lua-torch/torch7/random.lua index 59bbd7b1a..59bbd7b1a 100644 --- a/contrib/torch/torch7/random.lua +++ b/contrib/lua-torch/torch7/random.lua diff --git a/contrib/torch/torch7/test/longSize.lua b/contrib/lua-torch/torch7/test/longSize.lua index f5eba865c..f5eba865c 100644 --- a/contrib/torch/torch7/test/longSize.lua +++ b/contrib/lua-torch/torch7/test/longSize.lua diff --git a/contrib/torch/torch7/test/test.lua b/contrib/lua-torch/torch7/test/test.lua index 6c5dc71b0..6c5dc71b0 100644 --- a/contrib/torch/torch7/test/test.lua +++ b/contrib/lua-torch/torch7/test/test.lua diff --git a/contrib/torch/torch7/test/test_Multinomial.lua b/contrib/lua-torch/torch7/test/test_Multinomial.lua index 9069ecbed..9069ecbed 100644 --- a/contrib/torch/torch7/test/test_Multinomial.lua +++ b/contrib/lua-torch/torch7/test/test_Multinomial.lua diff --git a/contrib/torch/torch7/test/test_Tester.lua b/contrib/lua-torch/torch7/test/test_Tester.lua index a28336088..a28336088 100644 --- a/contrib/torch/torch7/test/test_Tester.lua +++ b/contrib/lua-torch/torch7/test/test_Tester.lua diff --git a/contrib/torch/torch7/test/test_half.lua b/contrib/lua-torch/torch7/test/test_half.lua index bf3830b5e..bf3830b5e 100644 --- a/contrib/torch/torch7/test/test_half.lua +++ b/contrib/lua-torch/torch7/test/test_half.lua diff --git a/contrib/torch/torch7/test/test_qr.lua b/contrib/lua-torch/torch7/test/test_qr.lua index c850c3fe1..c850c3fe1 100644 --- a/contrib/torch/torch7/test/test_qr.lua +++ b/contrib/lua-torch/torch7/test/test_qr.lua diff --git a/contrib/torch/torch7/test/test_sharedmem.lua b/contrib/lua-torch/torch7/test/test_sharedmem.lua index 1230e5914..1230e5914 100644 --- a/contrib/torch/torch7/test/test_sharedmem.lua +++ b/contrib/lua-torch/torch7/test/test_sharedmem.lua diff --git a/contrib/torch/torch7/test/test_timer.lua b/contrib/lua-torch/torch7/test/test_timer.lua index ecf576a1a..ecf576a1a 100644 --- a/contrib/torch/torch7/test/test_timer.lua +++ b/contrib/lua-torch/torch7/test/test_timer.lua diff --git a/contrib/torch/torch7/test/test_writeObject.lua b/contrib/lua-torch/torch7/test/test_writeObject.lua index 52bcb7173..52bcb7173 100644 --- a/contrib/torch/torch7/test/test_writeObject.lua +++ b/contrib/lua-torch/torch7/test/test_writeObject.lua diff --git a/contrib/torch/torch7/test/timeSort.lua b/contrib/lua-torch/torch7/test/timeSort.lua index ad513b877..ad513b877 100644 --- a/contrib/torch/torch7/test/timeSort.lua +++ b/contrib/lua-torch/torch7/test/timeSort.lua diff --git a/contrib/torch/torch7/torchcwrap.lua b/contrib/lua-torch/torch7/torchcwrap.lua index 551bd05d2..551bd05d2 100644 --- a/contrib/torch/torch7/torchcwrap.lua +++ b/contrib/lua-torch/torch7/torchcwrap.lua diff --git a/contrib/torch/torch7/utils.c b/contrib/lua-torch/torch7/utils.c index 974d0ac08..974d0ac08 100644 --- a/contrib/torch/torch7/utils.c +++ b/contrib/lua-torch/torch7/utils.c diff --git a/contrib/torch/torch7/utils.h b/contrib/lua-torch/torch7/utils.h index f9654d4b7..f9654d4b7 100644 --- a/contrib/torch/torch7/utils.h +++ b/contrib/lua-torch/torch7/utils.h diff --git a/contrib/t1ha/t1ha1.c b/contrib/t1ha/t1ha1.c index 956f7e24e..6e25d37f4 100644 --- a/contrib/t1ha/t1ha1.c +++ b/contrib/t1ha/t1ha1.c @@ -58,21 +58,21 @@ static __inline uint64_t final_weak_avalanche(uint64_t a, uint64_t b) { return mux64(rot64(a + b, 17), prime_4) + mix64(a ^ b, prime_0); } -/* TODO C++ template in the next version */ -#define T1HA1_BODY(ENDIANNES, ALIGNESS, DOCOPY) \ +/* TODO: C++ template in the next version */ +#define T1HA1_BODY(ENDIANNES, ALIGNESS) \ + const uint64_t *v = (const uint64_t *)data; \ if (unlikely(len > 32)) { \ uint64_t c = rot64(len, 17) + seed; \ uint64_t d = len ^ rot64(seed, 17); \ - const void *detent = (const uint8_t *)data + len - 31; \ + const uint64_t *detent = \ + (const uint64_t *)((const uint8_t *)data + len - 31); \ do { \ - const uint64_t *v = (const uint64_t *)data; \ - if (DOCOPY) \ - memcpy((void *)(v = align), data, 32); \ - \ const uint64_t w0 = fetch64_##ENDIANNES##_##ALIGNESS(v + 0); \ const uint64_t w1 = fetch64_##ENDIANNES##_##ALIGNESS(v + 1); \ const uint64_t w2 = fetch64_##ENDIANNES##_##ALIGNESS(v + 2); \ const uint64_t w3 = fetch64_##ENDIANNES##_##ALIGNESS(v + 3); \ + v += 4; \ + prefetch(v); \ \ const uint64_t d02 = w0 ^ rot64(w2 + d, 17); \ const uint64_t c13 = w1 ^ rot64(w3 + c, 17); \ @@ -80,18 +80,13 @@ static __inline uint64_t final_weak_avalanche(uint64_t a, uint64_t b) { d -= b ^ rot64(w1, 31); \ a ^= prime_1 * (d02 + w3); \ b ^= prime_0 * (c13 + w2); \ - data = (const uint64_t *)data + 4; \ - } while (likely(data < detent)); \ + } while (likely(v < detent)); \ \ a ^= prime_6 * (rot64(c, 17) + d); \ b ^= prime_5 * (c + rot64(d, 17)); \ len &= 31; \ } \ \ - const uint64_t *v = (const uint64_t *)data; \ - if (unlikely(need_copy4align) && len > 8) \ - memcpy((void *)(v = align), data, len); \ - \ switch (len) { \ default: \ b += mux64(fetch64_##ENDIANNES##_##ALIGNESS(v++), prime_4); \ @@ -134,26 +129,30 @@ uint64_t t1ha1_le(const void *data, size_t len, uint64_t seed) { uint64_t a = seed; uint64_t b = len; - const bool need_copy4align = - (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0 && !UNALIGNED_OK; - uint64_t align[4]; - if (need_copy4align) { - T1HA1_BODY(le, aligned, true); +#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT + T1HA1_BODY(le, unaligned); +#else + const bool misaligned = (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0; + if (misaligned) { + T1HA1_BODY(le, unaligned); } else { - T1HA1_BODY(le, unaligned, false); + T1HA1_BODY(le, aligned); } +#endif } uint64_t t1ha1_be(const void *data, size_t len, uint64_t seed) { uint64_t a = seed; uint64_t b = len; - const bool need_copy4align = - (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0 && !UNALIGNED_OK; - uint64_t align[4]; - if (need_copy4align) { - T1HA1_BODY(be, aligned, true); +#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT + T1HA1_BODY(be, unaligned); +#else + const bool misaligned = (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0; + if (misaligned) { + T1HA1_BODY(be, unaligned); } else { - T1HA1_BODY(be, unaligned, false); + T1HA1_BODY(be, aligned); } -}
\ No newline at end of file +#endif +} diff --git a/contrib/t1ha/t1ha2.c b/contrib/t1ha/t1ha2.c index 95f646da4..4cb5281ee 100644 --- a/contrib/t1ha/t1ha2.c +++ b/contrib/t1ha/t1ha2.c @@ -56,7 +56,7 @@ static __always_inline void init_cd(t1ha_state256_t *s, uint64_t x, s->n.d = ~y + rot64(x, 19); } -/* TODO C++ template in the next version */ +/* TODO: C++ template in the next version */ #define T1HA2_UPDATE(ENDIANNES, ALIGNESS, state, v) \ do { \ t1ha_state256_t *const s = state; \ @@ -67,8 +67,8 @@ static __always_inline void init_cd(t1ha_state256_t *s, uint64_t x, \ const uint64_t d02 = w0 + rot64(w2 + s->n.d, 56); \ const uint64_t c13 = w1 + rot64(w3 + s->n.c, 19); \ - s->n.d ^= s->n.b + rot64(w1, 38); \ s->n.c ^= s->n.a + rot64(w0, 57); \ + s->n.d ^= s->n.b + rot64(w1, 38); \ s->n.b ^= prime_6 * (c13 + w2); \ s->n.a ^= prime_5 * (d02 + w3); \ } while (0) @@ -78,26 +78,23 @@ static __always_inline void squash(t1ha_state256_t *s) { s->n.b ^= prime_5 * (rot64(s->n.c, 19) + s->n.d); } -/* TODO C++ template in the next version */ -#define T1HA2_LOOP(ENDIANNES, ALIGNESS, BUFFER4COPY, state, data, len) \ +/* TODO: C++ template in the next version */ +#define T1HA2_LOOP(ENDIANNES, ALIGNESS, state, data, len) \ do { \ const void *detent = (const uint8_t *)data + len - 31; \ do { \ const uint64_t *v = (const uint64_t *)data; \ - if (BUFFER4COPY != NULL) \ - memcpy((void *)(v = BUFFER4COPY), data, 32); \ - T1HA2_UPDATE(le, unaligned, state, v); \ data = (const uint64_t *)data + 4; \ + prefetch(data); \ + T1HA2_UPDATE(le, ALIGNESS, state, v); \ } while (likely(data < detent)); \ } while (0) -/* TODO C++ template in the next version */ -#define T1HA2_TAIL_AB(ENDIANNES, ALIGNESS, BUFFER4COPY, state, data, len) \ +/* TODO: C++ template in the next version */ +#define T1HA2_TAIL_AB(ENDIANNES, ALIGNESS, state, data, len) \ do { \ t1ha_state256_t *const s = state; \ const uint64_t *v = (const uint64_t *)data; \ - if (BUFFER4COPY != NULL) \ - memcpy((void *)(v = BUFFER4COPY), data, len); \ switch (len) { \ default: \ mixup64(&s->n.a, &s->n.b, fetch64_##ENDIANNES##_##ALIGNESS(v++), \ @@ -141,13 +138,11 @@ static __always_inline void squash(t1ha_state256_t *s) { } \ } while (0) -/* TODO C++ template in the next version */ -#define T1HA2_TAIL_ABCD(ENDIANNES, ALIGNESS, BUFFER4COPY, state, data, len) \ +/* TODO: C++ template in the next version */ +#define T1HA2_TAIL_ABCD(ENDIANNES, ALIGNESS, state, data, len) \ do { \ t1ha_state256_t *const s = state; \ const uint64_t *v = (const uint64_t *)data; \ - if (BUFFER4COPY != NULL) \ - memcpy((void *)(v = BUFFER4COPY), data, len); \ switch (len) { \ default: \ mixup64(&s->n.a, &s->n.d, fetch64_##ENDIANNES##_##ALIGNESS(v++), \ @@ -207,26 +202,34 @@ uint64_t t1ha2_atonce(const void *data, size_t length, uint64_t seed) { t1ha_state256_t state; init_ab(&state, seed, length); - const bool need_copy4align = - (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0 && !UNALIGNED_OK; - if (need_copy4align) { - uint64_t buffer4align[4]; +#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT + if (unlikely(length > 32)) { + init_cd(&state, seed, length); + T1HA2_LOOP(le, unaligned, &state, data, length); + squash(&state); + length &= 31; + } + T1HA2_TAIL_AB(le, unaligned, &state, data, length); +#else + const bool misaligned = (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0; + if (misaligned) { if (unlikely(length > 32)) { init_cd(&state, seed, length); - T1HA2_LOOP(le, aligned, buffer4align, &state, data, length); + T1HA2_LOOP(le, unaligned, &state, data, length); squash(&state); length &= 31; } - T1HA2_TAIL_AB(le, aligned, buffer4align, &state, data, length); + T1HA2_TAIL_AB(le, unaligned, &state, data, length); } else { if (unlikely(length > 32)) { init_cd(&state, seed, length); - T1HA2_LOOP(le, unaligned, NULL, &state, data, length); + T1HA2_LOOP(le, aligned, &state, data, length); squash(&state); length &= 31; } - T1HA2_TAIL_AB(le, unaligned, NULL, &state, data, length); + T1HA2_TAIL_AB(le, aligned, &state, data, length); } +#endif } uint64_t t1ha2_atonce128(uint64_t *__restrict extra_result, @@ -236,22 +239,28 @@ uint64_t t1ha2_atonce128(uint64_t *__restrict extra_result, init_ab(&state, seed, length); init_cd(&state, seed, length); - const bool need_copy4align = - (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0 && !UNALIGNED_OK; - if (need_copy4align) { - uint64_t buffer4align[4]; +#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT + if (unlikely(length > 32)) { + T1HA2_LOOP(le, unaligned, &state, data, length); + length &= 31; + } + T1HA2_TAIL_ABCD(le, unaligned, &state, data, length); +#else + const bool misaligned = (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0; + if (misaligned) { if (unlikely(length > 32)) { - T1HA2_LOOP(le, aligned, buffer4align, &state, data, length); + T1HA2_LOOP(le, unaligned, &state, data, length); length &= 31; } - T1HA2_TAIL_ABCD(le, aligned, buffer4align, &state, data, length); + T1HA2_TAIL_ABCD(le, unaligned, &state, data, length); } else { if (unlikely(length > 32)) { - T1HA2_LOOP(le, unaligned, NULL, &state, data, length); + T1HA2_LOOP(le, aligned, &state, data, length); length &= 31; } - T1HA2_TAIL_ABCD(le, unaligned, NULL, &state, data, length); + T1HA2_TAIL_ABCD(le, aligned, &state, data, length); } +#endif } //------------------------------------------------------------------------------ @@ -283,13 +292,16 @@ void t1ha2_update(t1ha_context_t *__restrict ctx, const void *__restrict data, } if (length >= 32) { - const bool need_copy4align = - (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0 && !UNALIGNED_OK; - if (need_copy4align) { - T1HA2_LOOP(le, aligned, ctx->buffer.u64, &ctx->state, data, length); +#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT + T1HA2_LOOP(le, unaligned, &ctx->state, data, length); +#else + const bool misaligned = (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0; + if (misaligned) { + T1HA2_LOOP(le, unaligned, &ctx->state, data, length); } else { - T1HA2_LOOP(le, unaligned, NULL, &ctx->state, data, length); + T1HA2_LOOP(le, aligned, &ctx->state, data, length); } +#endif length &= 31; } @@ -307,13 +319,8 @@ uint64_t t1ha2_final(t1ha_context_t *__restrict ctx, if (likely(!extra_result)) { squash(&ctx->state); - T1HA2_TAIL_AB(le, aligned, NULL, &ctx->state, ctx->buffer.u64, - ctx->partial); - return final64(ctx->state.n.a, ctx->state.n.b); + T1HA2_TAIL_AB(le, aligned, &ctx->state, ctx->buffer.u64, ctx->partial); } - T1HA2_TAIL_ABCD(le, aligned, NULL, &ctx->state, ctx->buffer.u64, - ctx->partial); - return final128(ctx->state.n.a, ctx->state.n.b, ctx->state.n.c, - ctx->state.n.d, extra_result); -}
\ No newline at end of file + T1HA2_TAIL_ABCD(le, aligned, &ctx->state, ctx->buffer.u64, ctx->partial); +} diff --git a/contrib/t1ha/t1ha_bits.h b/contrib/t1ha/t1ha_bits.h index e3815a4e7..454e43aed 100644 --- a/contrib/t1ha/t1ha_bits.h +++ b/contrib/t1ha/t1ha_bits.h @@ -72,19 +72,23 @@ #error Unsupported byte order. #endif -#if !defined(UNALIGNED_OK) -#if (defined(__ia32__) || defined(__e2k__) || \ - defined(__ARM_FEATURE_UNALIGNED)) && \ - !defined(__ALIGNED__) -#define UNALIGNED_OK 1 +#define T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE 0 +#define T1HA_CONFIG_UNALIGNED_ACCESS__SLOW 1 +#define T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT 2 + +#ifndef T1HA_CONFIG_UNALIGNED_ACCESS +#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) +#define T1HA_CONFIG_UNALIGNED_ACCESS T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT +#elif defined(__ia32__) +#define T1HA_CONFIG_UNALIGNED_ACCESS T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT +#elif defined(__e2k__) +#define T1HA_CONFIG_UNALIGNED_ACCESS T1HA_CONFIG_UNALIGNED_ACCESS__SLOW +#elif defined(__ARM_FEATURE_UNALIGNED) +#define T1HA_CONFIG_UNALIGNED_ACCESS T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT #else -#define UNALIGNED_OK 0 +#define T1HA_CONFIG_UNALIGNED_ACCESS T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE #endif -#endif /* UNALIGNED_OK */ - -#if UNALIGNED_OK && !defined(PAGESIZE) -#define PAGESIZE 4096 -#endif /* PAGESIZE */ +#endif /* T1HA_CONFIG_UNALIGNED_ACCESS */ #define ALIGNMENT_16 2 #define ALIGNMENT_32 4 @@ -94,6 +98,10 @@ #define ALIGNMENT_64 4 #endif +#ifndef PAGESIZE +#define PAGESIZE 4096 +#endif /* PAGESIZE */ + /***************************************************************************/ #ifndef __has_builtin @@ -118,6 +126,13 @@ #if __GNUC_PREREQ(4, 4) || defined(__clang__) +#if defined(__ia32__) || defined(__e2k__) +#include <x86intrin.h> +#endif + +#if defined(__ia32__) && !defined(__cpuid_count) +#include <cpuid.h> +#endif #if defined(__e2k__) #include <e2kbuiltin.h> @@ -393,10 +408,10 @@ typedef struct { #endif /* read_unaligned */ #ifndef read_aligned -#if __has_builtin(assume_aligned) +#if __GNUC_PREREQ(4, 8) || __has_builtin(__builtin_assume_aligned) #define read_aligned(ptr, bits) \ (*(const uint##bits##_t *)__builtin_assume_aligned(ptr, ALIGNMENT_##bits)) -#elif __has_attribute(aligned) && !defined(__clang__) +#elif (__GNUC_PREREQ(3, 3) || __has_attribute(aligned)) && !defined(__clang__) #define read_aligned(ptr, bits) \ (*(const uint##bits##_t __attribute__((aligned(ALIGNMENT_##bits))) *)(ptr)) #elif __has_attribute(assume_aligned) @@ -427,6 +442,20 @@ static __always_inline const #endif #endif /* read_aligned */ +#ifndef prefetch +#if (__GNUC_PREREQ(4, 0) || __has_builtin(__builtin_prefetch)) && \ + !defined(__ia32__) +#define prefetch(ptr) __builtin_prefetch(ptr) +#elif defined(_M_ARM64) || defined(_M_ARM) +#define prefetch(ptr) __prefetch(ptr) +#else +#define prefetch(ptr) \ + do { \ + (void)(ptr); \ + } while (0) +#endif +#endif /* prefetch */ + #if __has_warning("-Wconstant-logical-operand") #if defined(__clang__) #pragma clang diagnostic ignored "-Wconstant-logical-operand" @@ -451,28 +480,33 @@ static __always_inline const /*---------------------------------------------------------- Little Endian */ -#ifndef fetch64_le_aligned -static __always_inline uint64_t fetch64_le_aligned(const void *v) { +#ifndef fetch16_le_aligned +static __always_inline uint16_t fetch16_le_aligned(const void *v) { + assert(((uintptr_t)v) % ALIGNMENT_16 == 0); #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - return read_aligned(v, 64); + return read_aligned(v, 16); #else - return bswap64(read_aligned(v, 64)); + return bswap16(read_aligned(v, 16)); #endif } -#endif /* fetch64_le_aligned */ +#endif /* fetch16_le_aligned */ -#ifndef fetch64_le_unaligned -static __always_inline uint64_t fetch64_le_unaligned(const void *v) { -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - return read_unaligned(v, 64); +#ifndef fetch16_le_unaligned +static __always_inline uint16_t fetch16_le_unaligned(const void *v) { +#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE + const uint8_t *p = (const uint8_t *)v; + return p[0] | (uint16_t)p[1] << 8; +#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + return read_unaligned(v, 16); #else - return bswap64(read_unaligned(v, 64)); + return bswap16(read_unaligned(v, 16)); #endif } -#endif /* fetch64_le_unaligned */ +#endif /* fetch16_le_unaligned */ #ifndef fetch32_le_aligned static __always_inline uint32_t fetch32_le_aligned(const void *v) { + assert(((uintptr_t)v) % ALIGNMENT_32 == 0); #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ return read_aligned(v, 32); #else @@ -483,7 +517,10 @@ static __always_inline uint32_t fetch32_le_aligned(const void *v) { #ifndef fetch32_le_unaligned static __always_inline uint32_t fetch32_le_unaligned(const void *v) { -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE + return fetch16_le_unaligned(v) | + (uint32_t)fetch16_le_unaligned((const uint8_t *)v + 2) << 16; +#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ return read_unaligned(v, 32); #else return bswap32(read_unaligned(v, 32)); @@ -491,25 +528,29 @@ static __always_inline uint32_t fetch32_le_unaligned(const void *v) { } #endif /* fetch32_le_unaligned */ -#ifndef fetch16_le_aligned -static __always_inline uint16_t fetch16_le_aligned(const void *v) { +#ifndef fetch64_le_aligned +static __always_inline uint64_t fetch64_le_aligned(const void *v) { + assert(((uintptr_t)v) % ALIGNMENT_64 == 0); #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - return read_aligned(v, 16); + return read_aligned(v, 64); #else - return bswap16(read_aligned(v, 16)); + return bswap64(read_aligned(v, 64)); #endif } -#endif /* fetch16_le_aligned */ +#endif /* fetch64_le_aligned */ -#ifndef fetch16_le_unaligned -static __always_inline uint16_t fetch16_le_unaligned(const void *v) { -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - return read_unaligned(v, 16); +#ifndef fetch64_le_unaligned +static __always_inline uint64_t fetch64_le_unaligned(const void *v) { +#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE + return fetch32_le_unaligned(v) | + (uint64_t)fetch32_le_unaligned((const uint8_t *)v + 4) << 32; +#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + return read_unaligned(v, 64); #else - return bswap16(read_unaligned(v, 16)); + return bswap64(read_unaligned(v, 64)); #endif } -#endif /* fetch16_le_unaligned */ +#endif /* fetch64_le_unaligned */ static __always_inline uint64_t tail64_le_aligned(const void *v, size_t tail) { const uint8_t *const p = (const uint8_t *)v; @@ -517,10 +558,12 @@ static __always_inline uint64_t tail64_le_aligned(const void *v, size_t tail) { /* We can perform a 'oneshot' read, which is little bit faster. */ const unsigned shift = ((8 - tail) & 7) << 3; return fetch64_le_aligned(p) & ((~UINT64_C(0)) >> shift); -#endif /* 'oneshot' read */ - +#else uint64_t r = 0; switch (tail & 7) { + default: + unreachable(); +/* fall through */ #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ /* For most CPUs this code is better when not needed byte reordering. */ case 0: @@ -577,14 +620,15 @@ static __always_inline uint64_t tail64_le_aligned(const void *v, size_t tail) { return r + p[0]; #endif } - unreachable(); +#endif /* T1HA_USE_FAST_ONESHOT_READ */ } -#if T1HA_USE_FAST_ONESHOT_READ && UNALIGNED_OK && defined(PAGESIZE) && \ - !defined(__SANITIZE_ADDRESS__) && !defined(__sun) +#if T1HA_USE_FAST_ONESHOT_READ && \ + T1HA_CONFIG_UNALIGNED_ACCESS != T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE && \ + defined(PAGESIZE) && !defined(__sun) && !defined(__SANITIZE_ADDRESS__) #define can_read_underside(ptr, size) \ ((size) <= sizeof(uintptr_t) && ((PAGESIZE - (size)) & (uintptr_t)(ptr)) != 0) -#endif /* can_fast_read */ +#endif /* T1HA_USE_FAST_ONESHOT_READ */ static __always_inline uint64_t tail64_le_unaligned(const void *v, size_t tail) { @@ -600,11 +644,14 @@ static __always_inline uint64_t tail64_le_unaligned(const void *v, return fetch64_le_unaligned(p) >> shift; } return fetch64_le_unaligned(p) & ((~UINT64_C(0)) >> shift); -#endif /* 'oneshot' read */ - +#else uint64_t r = 0; switch (tail & 7) { -#if UNALIGNED_OK && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + default: + unreachable(); +/* fall through */ +#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT && \ + __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ /* For most CPUs this code is better when not needed * copying for alignment or byte reordering. */ case 0: @@ -663,36 +710,41 @@ static __always_inline uint64_t tail64_le_unaligned(const void *v, return r + p[0]; #endif } - unreachable(); +#endif /* can_read_underside */ } /*------------------------------------------------------------- Big Endian */ -#ifndef fetch64_be_aligned -static __maybe_unused __always_inline uint64_t -fetch64_be_aligned(const void *v) { +#ifndef fetch16_be_aligned +static __maybe_unused __always_inline uint16_t +fetch16_be_aligned(const void *v) { + assert(((uintptr_t)v) % ALIGNMENT_16 == 0); #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - return read_aligned(v, 64); + return read_aligned(v, 16); #else - return bswap64(read_aligned(v, 64)); + return bswap16(read_aligned(v, 16)); #endif } -#endif /* fetch64_be_aligned */ +#endif /* fetch16_be_aligned */ -#ifndef fetch64_be_unaligned -static __maybe_unused __always_inline uint64_t -fetch64_be_unaligned(const void *v) { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - return read_unaligned(v, 64); +#ifndef fetch16_be_unaligned +static __maybe_unused __always_inline uint16_t +fetch16_be_unaligned(const void *v) { +#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE + const uint8_t *p = (const uint8_t *)v; + return (uint16_t)p[0] << 8 | p[1]; +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + return read_unaligned(v, 16); #else - return bswap64(read_unaligned(v, 64)); + return bswap16(read_unaligned(v, 16)); #endif } -#endif /* fetch64_be_unaligned */ +#endif /* fetch16_be_unaligned */ #ifndef fetch32_be_aligned static __maybe_unused __always_inline uint32_t fetch32_be_aligned(const void *v) { + assert(((uintptr_t)v) % ALIGNMENT_32 == 0); #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ return read_aligned(v, 32); #else @@ -704,7 +756,10 @@ fetch32_be_aligned(const void *v) { #ifndef fetch32_be_unaligned static __maybe_unused __always_inline uint32_t fetch32_be_unaligned(const void *v) { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE + return (uint32_t)fetch16_be_unaligned(v) << 16 | + fetch16_be_unaligned((const uint8_t *)v + 2); +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ return read_unaligned(v, 32); #else return bswap32(read_unaligned(v, 32)); @@ -712,27 +767,31 @@ fetch32_be_unaligned(const void *v) { } #endif /* fetch32_be_unaligned */ -#ifndef fetch16_be_aligned -static __maybe_unused __always_inline uint16_t -fetch16_be_aligned(const void *v) { +#ifndef fetch64_be_aligned +static __maybe_unused __always_inline uint64_t +fetch64_be_aligned(const void *v) { + assert(((uintptr_t)v) % ALIGNMENT_64 == 0); #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - return read_aligned(v, 16); + return read_aligned(v, 64); #else - return bswap16(read_aligned(v, 16)); + return bswap64(read_aligned(v, 64)); #endif } -#endif /* fetch16_be_aligned */ +#endif /* fetch64_be_aligned */ -#ifndef fetch16_be_unaligned -static __maybe_unused __always_inline uint16_t -fetch16_be_unaligned(const void *v) { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - return read_unaligned(v, 16); +#ifndef fetch64_be_unaligned +static __maybe_unused __always_inline uint64_t +fetch64_be_unaligned(const void *v) { +#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE + return (uint64_t)fetch32_be_unaligned(v) << 32 | + fetch32_be_unaligned((const uint8_t *)v + 4); +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + return read_unaligned(v, 64); #else - return bswap16(read_unaligned(v, 16)); + return bswap64(read_unaligned(v, 64)); #endif } -#endif /* fetch16_be_unaligned */ +#endif /* fetch64_be_unaligned */ static __maybe_unused __always_inline uint64_t tail64_be_aligned(const void *v, size_t tail) { @@ -741,9 +800,11 @@ static __maybe_unused __always_inline uint64_t tail64_be_aligned(const void *v, /* We can perform a 'oneshot' read, which is little bit faster. */ const unsigned shift = ((8 - tail) & 7) << 3; return fetch64_be_aligned(p) >> shift; -#endif /* 'oneshot' read */ - +#else switch (tail & 7) { + default: + unreachable(); +/* fall through */ #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ /* For most CPUs this code is better when not byte reordering. */ case 1: @@ -762,7 +823,7 @@ static __maybe_unused __always_inline uint64_t tail64_be_aligned(const void *v, return (uint64_t)fetch32_be_aligned(p) << 24 | (uint32_t)fetch16_be_aligned(p + 4) << 8 | p[6]; case 0: - return fetch64_be(p); + return fetch64_be_aligned(p); #else case 1: return p[0]; @@ -789,7 +850,7 @@ static __maybe_unused __always_inline uint64_t tail64_be_aligned(const void *v, (uint64_t)p[1] << 48 | (uint64_t)p[0] << 56; #endif } - unreachable(); +#endif /* T1HA_USE_FAST_ONESHOT_READ */ } static __maybe_unused __always_inline uint64_t @@ -806,10 +867,13 @@ tail64_be_unaligned(const void *v, size_t tail) { return fetch64_be_unaligned(p) & ((~UINT64_C(0)) >> shift); } return fetch64_be_unaligned(p) >> shift; -#endif /* 'oneshot' read */ - +#else switch (tail & 7) { -#if UNALIGNED_OK && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + default: + unreachable(); +/* fall through */ +#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT && \ + __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ /* For most CPUs this code is better when not needed * copying for alignment or byte reordering. */ case 1: @@ -858,7 +922,7 @@ tail64_be_unaligned(const void *v, size_t tail) { (uint64_t)p[1] << 48 | (uint64_t)p[0] << 56; #endif } - unreachable(); +#endif /* can_read_underside */ } /***************************************************************************/ diff --git a/interface/js/app/history.js b/interface/js/app/history.js index dc44a2ddf..c5cd0d37c 100644 --- a/interface/js/app/history.js +++ b/interface/js/app/history.js @@ -171,7 +171,14 @@ function($, _, Humanize) { "value": scan_time }; item.id = item['message-id']; - item.rcpt_mime = item.rcpt_mime.join(",​"); + if ($(item.rcpt_mime).not(item.rcpt_smtp).length !== 0 || $(item.rcpt_smtp).not(item.rcpt_mime).length !== 0) { + item.rcpt_mime = "[" + item.rcpt_smtp.join(",​") + "] " + item.rcpt_mime.join(",​"); + } else { + item.rcpt_mime = item.rcpt_mime.join(",​"); + } + if (item.sender_mime !== item.sender_smtp) { + item.sender_mime = "[" + item.sender_smtp + "] " + item.sender_mime; + } items.push(item); }); @@ -225,19 +232,23 @@ function($, _, Humanize) { } }, { "name": "sender_mime", - "title": "From", + "title": "[Envelope From] From", "breakpoints": "xs sm md", "style": { "font-size": "11px", - "minWidth": 100 + "minWidth": 100, + "maxWidth": 200, + "word-wrap": "break-word" } }, { "name": "rcpt_mime", - "title": "To", + "title": "[Envelope To] To/Cc/Bcc", "breakpoints": "xs sm md", "style": { "font-size": "11px", - "minWidth": 100 + "minWidth": 100, + "maxWidth": 200, + "word-wrap": "break-word" } }, { "name": "subject", @@ -305,7 +316,9 @@ function($, _, Humanize) { "breakpoints": "xs sm md", "style": { "font-size": "11px", - "minWidth": 100 + "minWidth": 100, + "maxWidth": 130, + "word-wrap": "break-word" } }]; } diff --git a/interface/js/app/symbols.js b/interface/js/app/symbols.js index b1745abf0..7983ccc7f 100644 --- a/interface/js/app/symbols.js +++ b/interface/js/app/symbols.js @@ -97,7 +97,7 @@ function($) { var label_class = ''; if (item.weight < 0) { label_class = 'scorebar-ham'; - } else { + } else if (item.weight > 0) { label_class = 'scorebar-spam'; } item.weight = '<input class="form-control input-sm mb-disabled ' + label_class + @@ -239,11 +239,15 @@ function($) { }, components: { filtering: FooTable.groupFilter + }, + "on": { + "ready.ft.table": function () { + if (rspamd.read_only) { + $(".mb-disabled").attr('disabled', true); + } + } } }); - if (rspamd.read_only) { - $( ".mb-disabled" ).attr('disabled', true); - } }, error: function (data) { rspamd.alertMessage('alert-modal alert-error', data.statusText); diff --git a/lualib/rspamadm/ansicolors.lua b/lualib/ansicolors.lua index 81783f618..81783f618 100644 --- a/lualib/rspamadm/ansicolors.lua +++ b/lualib/ansicolors.lua diff --git a/lualib/rspamadm/plugins_stats.lua b/lualib/plugins_stats.lua index 5c9797a9e..9bbebb1c8 100644 --- a/lualib/rspamadm/plugins_stats.lua +++ b/lualib/plugins_stats.lua @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. ]]-- -local ansicolors = require "rspamadm/ansicolors" +local ansicolors = require "ansicolors" local function printf(fmt, ...) print(string.format(fmt, ...)) diff --git a/lualib/rspamadm/rescore_utility.lua b/lualib/rescore_utility.lua index 2a9372d4e..195ae4364 100644 --- a/lualib/rspamadm/rescore_utility.lua +++ b/lualib/rescore_utility.lua @@ -11,7 +11,7 @@ function utility.get_all_symbols(logs, ignore_symbols) for _, line in pairs(logs) do line = lua_util.rspamd_str_split(line, " ") - for i=4,(#line-2) do + for i=4,(#line-1) do line[i] = line[i]:gsub("%s+", "") if not symbols_set[line[i]] then symbols_set[line[i]] = true @@ -35,49 +35,69 @@ end function utility.read_log_file(file) local lines = {} + local messages = {} - file = assert(io.open(file, "r")) + local fd = assert(io.open(file, "r")) + local fname = string.gsub(file, "(.*/)(.*)", "%2") - for line in file:lines() do - lines[#lines + 1] = line - end + for line in fd:lines() do + local start,stop = string.find(line, fname .. ':') + + if start and stop then + table.insert(lines, string.sub(line, 1, start)) + table.insert(messages, string.sub(line, stop + 1, -1)) + end +end - io.close(file) + io.close(fd) - return lines + return lines,messages end -function utility.get_all_logs(dir_path) +function utility.get_all_logs(dirs) -- Reads all log files in the directory and returns a list of logs. - if dir_path:sub(#dir_path, #dir_path) == "/" then - dir_path = dir_path:sub(1, #dir_path -1) + if type(dirs) == 'string' then + dirs = {dirs} end - local files = rspamd_util.glob(dir_path .. "/*.log") local all_logs = {} - - for _, file in pairs(files) do - local logs = utility.read_log_file(file) - for _, log_line in pairs(logs) do - all_logs[#all_logs + 1] = log_line + local all_messages = {} + + for _,dir in ipairs(dirs) do + if dir:sub(-1, -1) == "/" then + dir = dir:sub(1, -2) + local files = rspamd_util.glob(dir .. "/*.log") + for _, file in pairs(files) do + local logs,messages = utility.read_log_file(file) + for i=1,#logs do + table.insert(all_logs, logs[i]) + table.insert(all_messages, messages[i]) + end + end + else + local logs,messages = utility.read_log_file(dir) + for i=1,#logs do + table.insert(all_logs, logs[i]) + table.insert(all_messages, messages[i]) + end end end - return all_logs + return all_logs,all_messages end function utility.get_all_symbol_scores(conf, ignore_symbols) - local counters = conf:get_symbols_counters() + local symbols = conf:get_symbols_scores() - return fun.tomap(fun.map(function(elt) - return elt['symbol'],elt['weight'] - end, fun.filter(function(elt) - return not ignore_symbols[elt['symbol']] - end, counters))) + return fun.tomap(fun.map(function(name, elt) + return name,elt['score'] + end, fun.filter(function(name, elt) + return not ignore_symbols[name] + end, symbols))) end -function utility.generate_statistics_from_logs(logs, threshold) +function utility.generate_statistics_from_logs(logs, messages, threshold) -- Returns file_stats table and list of symbol_stats table. @@ -110,9 +130,10 @@ function utility.generate_statistics_from_logs(logs, threshold) local no_of_spam = 0 local no_of_ham = 0 - for _, log in pairs(logs) do + for i, log in ipairs(logs) do log = lua_util.rspamd_str_trim(log) log = lua_util.rspamd_str_split(log, " ") + local message = messages[i] local is_spam = (log[1] == "SPAM") local score = tonumber(log[2]) @@ -129,40 +150,38 @@ function utility.generate_statistics_from_logs(logs, threshold) true_positives = true_positives + 1 elseif is_spam and (score < threshold) then false_negatives = false_negatives + 1 - table.insert(all_fns, log[#log]) + table.insert(all_fns, message) elseif not is_spam and (score >= threshold) then false_positives = false_positives + 1 - table.insert(all_fps, log[#log]) + table.insert(all_fps, message) else true_negatives = true_negatives + 1 end - for i=4, (#log-2) do - if all_symbols_stats[log[i]] == nil then - all_symbols_stats[log[i]] = { - name = log[i], + for j=4, (#log-1) do + if all_symbols_stats[log[j]] == nil then + all_symbols_stats[log[j]] = { + name = message, no_of_hits = 0, spam_hits = 0, ham_hits = 0, spam_overall = 0 } end + local sym = log[j] - all_symbols_stats[log[i]].no_of_hits = - all_symbols_stats[log[i]].no_of_hits + 1 + all_symbols_stats[sym].no_of_hits = all_symbols_stats[sym].no_of_hits + 1 if is_spam then - all_symbols_stats[log[i]].spam_hits = - all_symbols_stats[log[i]].spam_hits + 1 + all_symbols_stats[sym].spam_hits = all_symbols_stats[sym].spam_hits + 1 else - all_symbols_stats[log[i]].ham_hits = - all_symbols_stats[log[i]].ham_hits + 1 + all_symbols_stats[sym].ham_hits = all_symbols_stats[sym].ham_hits + 1 end -- Find slowest message - if (tonumber(log[#log-1]) > tonumber(file_stats.slowest)) then - file_stats.slowest = tostring(tonumber(log[#log-1])) - file_stats.slowest_file = log[#log] + if ((tonumber(log[#log]) or 0) > file_stats.slowest) then + file_stats.slowest = tonumber(log[#log]) + file_stats.slowest_file = message end end end diff --git a/lualib/rspamadm/confighelp.lua b/lualib/rspamadm/confighelp.lua index a03578b6e..d477ff69b 100644 --- a/lualib/rspamadm/confighelp.lua +++ b/lualib/rspamadm/confighelp.lua @@ -1,4 +1,4 @@ -local opts = {} +local opts local known_attrs = { data = 1, example = 1, @@ -6,9 +6,19 @@ local known_attrs = { required = 1, default = 1, } +local argparse = require "argparse" +local ansicolors = require "ansicolors" -local getopt = require "rspamadm/getopt" -local ansicolors = require "rspamadm/ansicolors" +local parser = argparse() + :name "rspamadm confighelp" + :description "Shows help for the specified configuration options" + :help_description_margin(32) +parser:flag "--no-color" + :description "Disable coloured output" +parser:flag "--short" + :description "Show only option names" +parser:flag "--no-examples" + :description "Do not show examples (impied by --short)" local function maybe_print_color(key) if not opts['no-color'] then @@ -100,7 +110,7 @@ local function print_help(key, value, tabs) end return function(args, res) - opts = getopt.getopt(args, '') + opts = parser:parse(args) local sorted = sort_values(res) diff --git a/lualib/rspamadm/configwizard.lua b/lualib/rspamadm/configwizard.lua index 1cd275f66..10134c575 100644 --- a/lualib/rspamadm/configwizard.lua +++ b/lualib/rspamadm/configwizard.lua @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. ]]-- -local ansicolors = require "rspamadm/ansicolors" +local ansicolors = require "ansicolors" local local_conf = rspamd_paths['CONFDIR'] local rspamd_util = require "rspamd_util" local rspamd_logger = require "rspamd_logger" @@ -22,8 +22,9 @@ local lua_util = require "lua_util" local lua_stat_tools = require "lua_stat" local lua_redis = require "lua_redis" local ucl = require "ucl" +local argparse = require "argparse" -local plugins_stat = require "rspamadm/plugins_stats" +local plugins_stat = require "plugins_stats" local rspamd_logo = [[ ____ _ @@ -34,6 +35,19 @@ local rspamd_logo = [[ |_| ]] +local parser = argparse() + :name "rspamadm configwizard" + :description "Perform guided configuration for Rspamd daemon" + :help_description_margin(32) +parser:option "-c --config" + :description "Path to config file" + :argname("<file>") + :default(rspamd_paths["CONFDIR"] .. "/" .. "rspamd.conf") +parser:argument "checks" + :description "Checks to do (or 'list')" + :argname("<checks>") + :args "*" + local redis_params local function printf(fmt, ...) @@ -578,109 +592,135 @@ end -return function(args, cfg) - local changes = { - l = {}, -- local changes - o = {}, -- override changes - } - - local interactive_start = true - local checks = {} - local all_checks = { - 'controller', - 'redis', - 'dkim', - 'statistic', - } - - if #args > 0 then - interactive_start = false - - for _,arg in ipairs(args) do - if arg == 'all' then - checks = all_checks - elseif arg == 'list' then - printf(highlight(rspamd_logo)) - printf('Available modules') - for _,c in ipairs(all_checks) do - printf('- %s', c) +return { + handler = function(cmd_args) + local changes = { + l = {}, -- local changes + o = {}, -- override changes + } + + local interactive_start = true + local checks = {} + local all_checks = { + 'controller', + 'redis', + 'dkim', + 'statistic', + } + + local opts = parser:parse(cmd_args) + local args = opts['checks'] or {} + + local _r,err = rspamd_config:load_ucl(opts['config']) + local cfg = rspamd_config:get_ucl() + + if not _r then + rspamd_logger.errx('cannot parse %s: %s', opts['config'], err) + os.exit(1) + end + + _r,err = rspamd_config:parse_rcl({'logging', 'worker'}) + if not _r then + rspamd_logger.errx('cannot process %s: %s', opts['config'], err) + os.exit(1) + end + + if not rspamd_config:init_modules() then + rspamd_logger.errx('cannot init modules when parsing %s', opts['config']) + os.exit(1) + end + + if #args > 0 then + interactive_start = false + + for _,arg in ipairs(args) do + if arg == 'all' then + checks = all_checks + elseif arg == 'list' then + printf(highlight(rspamd_logo)) + printf('Available modules') + for _,c in ipairs(all_checks) do + printf('- %s', c) + end + return + else + table.insert(checks, arg) end - return - else - table.insert(checks, arg) end + else + checks = all_checks end - else - checks = all_checks - end - local function has_check(check) - for _,c in ipairs(checks) do - if c == check then - return true + local function has_check(check) + for _,c in ipairs(checks) do + if c == check then + return true + end end - end - return false - end + return false + end - rspamd_util.umask('022') - if interactive_start then - printf(highlight(rspamd_logo)) - printf("Welcome to the configuration tool") - printf("We use %s configuration file, writing results to %s", - highlight(cfg.config_path), highlight(local_conf)) - plugins_stat(nil, nil) - end + rspamd_util.umask('022') + if interactive_start then + printf(highlight(rspamd_logo)) + printf("Welcome to the configuration tool") + printf("We use %s configuration file, writing results to %s", + highlight(opts['config']), highlight(local_conf)) + plugins_stat(nil, nil) + end - if not interactive_start or - ask_yes_no("Do you wish to continue?", true) then + if not interactive_start or + ask_yes_no("Do you wish to continue?", true) then - if has_check('controller') then - local controller = find_worker(cfg, 'controller') - if controller then - setup_controller(controller, changes) + if has_check('controller') then + local controller = find_worker(cfg, 'controller') + if controller then + setup_controller(controller, changes) + end end - end - if has_check('redis') then - if not cfg.redis or (not cfg.redis.servers and not cfg.redis.read_servers) then - setup_redis(cfg, changes) + if has_check('redis') then + if not cfg.redis or (not cfg.redis.servers and not cfg.redis.read_servers) then + setup_redis(cfg, changes) + else + redis_params = cfg.redis + end else redis_params = cfg.redis end - else - redis_params = cfg.redis - end - if has_check('dkim') then - if cfg.dkim_signing and not cfg.dkim_signing.domain then - if ask_yes_no('Do you want to setup dkim signing feature?') then - setup_dkim_signing(cfg, changes) + if has_check('dkim') then + if cfg.dkim_signing and not cfg.dkim_signing.domain then + if ask_yes_no('Do you want to setup dkim signing feature?') then + setup_dkim_signing(cfg, changes) + end end end - end - - if has_check('statistic') or has_check('statistics') then - setup_statistic(cfg, changes) - end - local nchanges = 0 - for _,_ in pairs(changes.l) do nchanges = nchanges + 1 end - for _,_ in pairs(changes.o) do nchanges = nchanges + 1 end + if has_check('statistic') or has_check('statistics') then + setup_statistic(cfg, changes) + end - if nchanges > 0 then - print_changes(changes) - if ask_yes_no("Apply changes?", true) then - apply_changes(changes) - printf("%d changes applied, the wizard is finished now", nchanges) - printf("*** Please reload the Rspamd configuration ***") + local nchanges = 0 + for _,_ in pairs(changes.l) do nchanges = nchanges + 1 end + for _,_ in pairs(changes.o) do nchanges = nchanges + 1 end + + if nchanges > 0 then + print_changes(changes) + if ask_yes_no("Apply changes?", true) then + apply_changes(changes) + printf("%d changes applied, the wizard is finished now", nchanges) + printf("*** Please reload the Rspamd configuration ***") + else + printf("No changes applied, the wizard is finished now") + end else - printf("No changes applied, the wizard is finished now") + printf("No changes found, the wizard is finished now") end - else - printf("No changes found, the wizard is finished now") end - end -end + end, + name = 'configwizard', + description = parser._description, +} diff --git a/lualib/rspamadm/corpus_test.lua b/lualib/rspamadm/corpus_test.lua index cd9f66155..07051a196 100644 --- a/lualib/rspamadm/corpus_test.lua +++ b/lualib/rspamadm/corpus_test.lua @@ -1,19 +1,50 @@ local rspamd_logger = require "rspamd_logger" local ucl = require "ucl" local lua_util = require "lua_util" -local getopt = require "rspamadm/getopt" +local argparse = require "argparse" + +local parser = argparse() + :name "rspamadm corpus_test" + :description "Create logs files from email corpus" + :help_description_margin(32) + +parser:option "-H --ham" + :description("Ham directory") + :argname("<dir>") +parser:option "-S --spam" + :description("Spam directory") + :argname("<dir>") +parser:option "-n --conns" + :description("Number of parallel connections") + :argname("<N>") + :convert(tonumber) + :default(10) +parser:option "-o --output" + :description("Output file") + :argname("<file>") + :default('results.log') +parser:option "-t --timeout" + :description("Timeout for client connections") + :argname("<sec>") + :convert(tonumber) + :default(60) +parser:option "-c --connect" + :description("Connect to specific host") + :argname("<host>") + :default('localhost:11334') +parser:option "-r --rspamc" + :description("Use specific rspamc path") + :argname("<path>") + :default('rspamc') local HAM = "HAM" local SPAM = "SPAM" local opts -local default_opts = { - connect = 'localhost:11334', -} local function scan_email(n_parallel, path, timeout) - local rspamc_command = string.format("rspamc --connect %s -j --compact -n %s -t %.3f %s", - opts.connect, n_parallel, timeout, path) + local rspamc_command = string.format("%s --connect %s -j --compact -n %s -t %.3f %s", + opts.rspamc, opts.connect, n_parallel, timeout, path) local result = assert(io.popen(rspamc_command)) result = result:read("*all") return result @@ -24,7 +55,8 @@ local function write_results(results, file) local f = io.open(file, 'w') for _, result in pairs(results) do - local log_line = string.format("%s %.2f %s", result.type, result.score, result.action) + local log_line = string.format("%s %.2f %s", + result.type, result.score, result.action) for _, sym in pairs(result.symbols) do log_line = log_line .. " " .. sym @@ -44,16 +76,16 @@ local function encoded_json_to_log(result) -- Returns table containing score, action, list of symbols local filtered_result = {} - local parser = ucl.parser() + local ucl_parser = ucl.parser() - local is_good, err = parser:parse_string(result) + local is_good, err = ucl_parser:parse_string(result) if not is_good then rspamd_logger.errx("Parser error: %1", err) return nil end - result = parser:get_object() + result = ucl_parser:get_object() filtered_result.score = result.score if not result.action then @@ -96,13 +128,12 @@ local function scan_results_to_logs(results, actual_email_type) return logs end -return function(args, res) - opts = default_opts - opts = lua_util.override_defaults(opts, getopt.getopt(args, '')) - local ham_directory = res['ham_directory'] - local spam_directory = res['spam_directory'] - local connections = res["connections"] - local output = res["output_location"] +local function handler(args) + opts = parser:parse(args) + local ham_directory = opts['ham_directory'] + local spam_directory = opts['spam_directory'] + local connections = opts["connections"] + local output = opts["output"] local results = {} @@ -112,7 +143,7 @@ return function(args, res) if ham_directory then rspamd_logger.messagex("Scanning ham corpus...") - local ham_results = scan_email(connections, ham_directory, res["timeout"]) + local ham_results = scan_email(connections, ham_directory, opts["timeout"]) ham_results = scan_results_to_logs(ham_results, HAM) no_of_ham = #ham_results @@ -124,7 +155,7 @@ return function(args, res) if spam_directory then rspamd_logger.messagex("Scanning spam corpus...") - local spam_results = scan_email(connections, spam_directory, res.timeout) + local spam_results = scan_email(connections, spam_directory, opts.timeout) spam_results = scan_results_to_logs(spam_results, SPAM) no_of_spam = #spam_results @@ -145,3 +176,10 @@ return function(args, res) rspamd_logger.messagex("No of spam: %s", no_of_spam) rspamd_logger.messagex("Messages/sec: %s", (total_msgs / elapsed_time)) end + + +return { + name = 'corpus_test', + handler = handler, + description = parser._description +}
\ No newline at end of file diff --git a/lualib/rspamadm/fuzzy_stat.lua b/lualib/rspamadm/fuzzy_stat.lua index 05c2622bf..45e13bb60 100644 --- a/lualib/rspamadm/fuzzy_stat.lua +++ b/lualib/rspamadm/fuzzy_stat.lua @@ -1,6 +1,27 @@ local util = require "rspamd_util" local opts = {} +local argparse = require "argparse" +local parser = argparse() + :name "rspamadm confighelp" + :description "Shows help for the specified configuration options" + :help_description_margin(32) +parser:flag "--no-ips" + :description "No IPs stats" +parser:flag "--no-keys" + :description "No keys stats" +parser:flag "--short" + :description "Short output mode" +parser:flag "-n --number" + :description "Disable numbers humanization" +parser:option "-s --sort" + :description "Sort order" + :convert { + matched = "matched", + errors = "errors", + ip = "ip" + } + local function add_data(target, src) for k,v in pairs(src) do if k ~= 'ips' then @@ -143,13 +164,11 @@ local function print_result(r) return print_num(r) end -local getopt = require "rspamadm/getopt" - return function(args, res) local res_ips = {} local res_databases = {} local wrk = res['workers'] - opts = getopt.getopt(args, '') + opts = parser:parse(args) if wrk then for _,pr in pairs(wrk) do diff --git a/lualib/rspamadm/getopt.lua b/lualib/rspamadm/getopt.lua deleted file mode 100644 index 20715c387..000000000 --- a/lualib/rspamadm/getopt.lua +++ /dev/null @@ -1,50 +0,0 @@ -local function insert_option(tab, name, value) - if tab[name] then - if type(tab[name]) == 'table' then - table.insert(tab[name], value) - else - local old_val = tab[name] - tab[name] = { - old_val, - value - } - end - else - tab[name] = value - end -end - -local function getopt(arg, options) - local tab = {} - for k, v in ipairs(arg) do - if string.sub(v, 1, 2) == "--" then - local x = string.find(v, "=", 1, true) - if x then insert_option(tab, string.sub(v, 3, x - 1), string.sub(v, x + 1)) - else tab[string.sub(v, 3)] = true - end - elseif string.sub(v, 1, 1) == "-" then - local y = 2 - local l = string.len(v) - local jopt - while (y <= l) do - jopt = string.sub(v, y, y) - if string.find(options, jopt, 1, true) then - if y < l then - insert_option(tab, jopt, string.sub(v, y + 1)) - y = l - else - insert_option(tab, jopt, arg[k + 1]) - end - else - tab[jopt] = true - end - y = y + 1 - end - end - end - return tab -end - -return { - getopt = getopt -} diff --git a/lualib/rspamadm/grep.lua b/lualib/rspamadm/grep.lua index a9d4b084a..b149a0337 100644 --- a/lualib/rspamadm/grep.lua +++ b/lualib/rspamadm/grep.lua @@ -1,6 +1,60 @@ -return function(_, res) +--[[ +Copyright (c) 2018, Vsevolod Stakhov <vsevolod@highsecure.ru> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +local argparse = require "argparse" + + +-- Define command line options +local parser = argparse() + :name "rspamadm grep" + :description "Search for patterns in rspamd logs" + :help_description_margin(30) +parser:mutex( + parser:option "-s --string" + :description('Plain string to search (case-insensitive)') + :argname "<str>", + parser:option "-p --pattern" + :description('Pattern to search for (regex)') + :argname "<re>" +) +parser:flag "-l --lua" + :description('Use Lua patterns in string search') + +parser:argument "input":args "*" + :description('Process specified inputs') + :default("stdin") +parser:flag "-S --sensitive" + :description('Enable case-sensitivity in string search') +parser:flag "-o --orphans" + :description('Print orphaned logs') +parser:flag "-P --partial" + :description('Print partial logs') + +local function handler(args) local rspamd_regexp = require 'rspamd_regexp' + local res = parser:parse(args) + + if not res['string'] and not res['pattern'] then + parser:error('string or pattern options must be specified') + end + + if res['string'] and res['pattern'] then + parser:error('string and pattern options are mutually exclusive') + end local buffer = {} local matches = {} @@ -16,7 +70,7 @@ return function(_, res) end local plainm = true - if res['luapat'] then + if res['lua'] then plainm = false end local orphans = res['orphans'] @@ -26,7 +80,7 @@ return function(_, res) if search_str and not sensitive then search_str = string.lower(search_str) end - local inputs = res['inputs'] + local inputs = res['input'] or {'stdin'} for _, n in ipairs(inputs) do local h, err @@ -110,3 +164,9 @@ return function(_, res) end end end + +return { + handler = handler, + description = parser._description, + name = 'grep' +}
\ No newline at end of file diff --git a/lualib/rspamadm/keypair.lua b/lualib/rspamadm/keypair.lua new file mode 100644 index 000000000..518c9e65f --- /dev/null +++ b/lualib/rspamadm/keypair.lua @@ -0,0 +1,269 @@ +--[[ +Copyright (c) 2018, Vsevolod Stakhov <vsevolod@highsecure.ru> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +local argparse = require "argparse" +local rspamd_keypair = require "rspamd_cryptobox_keypair" +local rspamd_pubkey = require "rspamd_cryptobox_pubkey" +local rspamd_signature = require "rspamd_cryptobox_signature" +local rspamd_crypto = require "rspamd_cryptobox" +local ucl = require "ucl" +local logger = require "rspamd_logger" + +-- Define command line options +local parser = argparse() + :name "rspamadm keypair" + :description "Manages keypairs for Rspamd" + :help_description_margin(30) + :command_target("command") + :require_command(false) + +local generate = parser:command "generate gen g" + :description "Creates a new keypair" +generate:flag "-s --sign" + :description "Generates a sign keypair instead of the encryption one" +generate:flag "-n --nist" + :description "Uses nist encryption algorithm" +generate:option "-o --output" + :description "Write keypair to file" + :argname "<file>" +generate:mutex( + generate:flag "-j --json" + :description "Output JSON instead of UCL", + generate:flag "-u --ucl" + :description "Output UCL" + :default(true) +) + +local sign = parser:command "sign sig s" + :description "Signs a file using keypair" +sign:option "-k --keypair" + :description "Keypair to use" + :argname "<file>" +sign:option "-s --suffix" + :description "Suffix for signature" + :argname "<suffix>" + :default("sig") +sign:argument "file" + :description "File to sign" + :argname "<file>" + :args "*" + +local verify = parser:command "verify ver v" + :description "Verifies a file using keypair or a public key" +verify:mutex( + verify:option "-p --pubkey" + :description "Load pubkey from the specified file" + :argname "<file>", + verify:option "-P --pubstring" + :description "Load pubkey from the base32 encoded string" + :argname "<base32>", + verify:option "-k --keypair" + :description "Get pubkey from the keypair file" + :argname "<file>" +) +verify:argument "file" + :description "File to verify" + :argname "<file>" + :args "*" +verify:flag "-n --nist" + :description "Uses nistp curves (P256)" +verify:option "-s --suffix" + :description "Suffix for signature" + :argname "<suffix>" + :default("sig") + +-- Default command is generate, so duplicate options +parser:flag "-s --sign" + :description "Generates a sign keypair instead of the encryption one" +parser:flag "-n --nist" + :description "Uses nistp curves (P256)" +parser:mutex( + parser:flag "-j --json" + :description "Output JSON instead of UCL", + parser:flag "-u --ucl" + :description "Output UCL" + :default(true) +) +parser:option "-o --output" + :description "Write keypair to file" + :argname "<file>" + +local function fatal(...) + logger.errx(...) + os.exit(1) +end + +local function generate_handler(opts) + local mode = 'encryption' + if opts.sign then + mode = 'sign' + end + local alg = 'curve25519' + if opts.nist then + alg = 'nist' + end + -- TODO: probably, do it in a more safe way + local kp = rspamd_keypair.create(mode, alg):totable() + + local format = 'ucl' + + if opts.json then + format = 'json' + end + + if opts.output then + local out = io.open(opts.output, 'w') + if not out then + fatal('cannot open output to write: ' .. opts.output) + end + out:write(ucl.to_format(kp, format)) + out:close() + else + io.write(ucl.to_format(kp, format)) + end +end + +local function sign_handler(opts) + if opts.file then + if type(opts.file) == 'string' then + opts.file = {opts.file} + end + else + parser:error('no files to sign') + end + if not opts.keypair then + parser:error("no keypair specified") + end + + local ucl_parser = ucl.parser() + local res,err = ucl_parser:parse_file(opts.keypair) + + if not res then + fatal(string.format('cannot load %s: %s', opts.keypair, err)) + end + + local kp = rspamd_keypair.load(ucl_parser:get_object()) + + if not kp then + fatal("cannot load keypair: " .. opts.keypair) + end + + for _,fname in ipairs(opts.file) do + local sig = rspamd_crypto.sign_file(kp, fname) + + if not sig then + fatal(string.format("cannot sign %s\n", fname)) + end + + local out = string.format('%s.%s', fname, opts.suffix or 'sig') + local of = io.open(out, 'w') + if not of then + fatal('cannot open output to write: ' .. out) + end + of:write(sig:bin()) + of:close() + io.write(string.format('signed %s -> %s (%s)\n', fname, out, sig:hex())) + end +end + +local function verify_handler(opts) + if opts.file then + if type(opts.file) == 'string' then + opts.file = {opts.file} + end + else + parser:error('no files to verify') + end + + local pk + local alg = 'curve25519' + + if opts.keypair then + local ucl_parser = ucl.parser() + local res,err = ucl_parser:parse_file(opts.keypair) + + if not res then + fatal(string.format('cannot load %s: %s', opts.keypair, err)) + end + + local kp = rspamd_keypair.load(ucl_parser:get_object()) + + if not kp then + fatal("cannot load keypair: " .. opts.keypair) + end + + pk = kp:pk() + alg = kp:alg() + elseif opts.pubkey then + if opts.nist then alg = 'nist' end + pk = rspamd_pubkey.load(opts.pubkey, 'sign', alg) + elseif opts.pubstr then + if opts.nist then alg = 'nist' end + pk = rspamd_pubkey.create(opts.pubstr, 'sign', alg) + end + + if not pk then + fatal("cannot create pubkey") + end + + local valid = true + + for _,fname in ipairs(opts.file) do + + local sig_fname = string.format('%s.%s', fname, opts.suffix or 'sig') + local sig = rspamd_signature.load(sig_fname, alg) + + if not sig then + fatal(string.format("cannot load signature for %s -> %s", + fname, sig_fname)) + end + + if rspamd_crypto.verify_file(pk, sig, fname, alg) then + io.write(string.format('verified %s -> %s (%s)\n', fname, sig_fname, sig:hex())) + else + valid = false + io.write(string.format('FAILED to verify %s -> %s (%s)\n', fname, + sig_fname, sig:hex())) + end + end + + if not valid then + os.exit(1) + end +end + +local function handler(args) + local opts = parser:parse(args) + + local command = opts.command or "generate" + + if command == 'generate' then + generate_handler(opts) + elseif command == 'sign' then + sign_handler(opts) + elseif command == 'verify' then + verify_handler(opts) + else + parser:error('command %s is not yet implemented', command) + end +end + +return { + name = 'keypair', + aliases = {'kp', 'key'}, + handler = handler, + description = parser._description +}
\ No newline at end of file diff --git a/lualib/rspamadm/rescore.lua b/lualib/rspamadm/rescore.lua index 231cf2ea6..cc331c6e8 100644 --- a/lualib/rspamadm/rescore.lua +++ b/lualib/rspamadm/rescore.lua @@ -1,13 +1,32 @@ +--[[ +Copyright (c) 2018, Vsevolod Stakhov <vsevolod@highsecure.ru> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +if not rspamd_config:has_torch() then + return +end + local torch = require "torch" local nn = require "nn" local lua_util = require "lua_util" local ucl = require "ucl" local logger = require "rspamd_logger" -local getopt = require "rspamadm/getopt" local optim = require "optim" local rspamd_util = require "rspamd_util" - -local rescore_utility = require "rspamadm/rescore_utility" +local argparse = require "argparse" +local rescore_utility = require "rescore_utility" local opts local ignore_symbols = { @@ -15,6 +34,104 @@ local ignore_symbols = { ['DATE_IN_FUTURE'] = true, } +local parser = argparse() + :name "rspamadm rescore" + :description "Estimate optimal symbol weights from log files" + :help_description_margin(37) + +parser:option "-l --log" + :description "Log file or files (from rescore)" + :argname("<log>") + :args "*" +parser:option "-c --config" + :description "Path to config file" + :argname("<file>") + :default(rspamd_paths["CONFDIR"] .. "/" .. "rspamd.conf") +parser:option "-o --output" + :description "Output file" + :argname("<file>") + :default("new.scores") +parser:flag "-d --diff" + :description "Show differences in scores" +parser:flag "-v --verbose" + :description "Verbose output" +parser:flag "-z --freq" + :description "Display hit frequencies" +parser:option "-i --iters" + :description "Learn iterations" + :argname("<n>") + :convert(tonumber) + :default(10) +parser:option "-b --batch" + :description "Batch size" + :argname("<n>") + :convert(tonumber) + :default(100) +parser:option "-d --decay" + :description "Decay rate" + :argname("<n>") + :convert(tonumber) + :default(0.001) +parser:option "-m --momentum" + :description "Learn momentum" + :argname("<n>") + :convert(tonumber) + :default(0.1) +parser:option "-t --threads" + :description "Number of threads to use" + :argname("<n>") + :convert(tonumber) + :default(1) +parser:option "-o --optim" + :description "Optimisation algorithm" + :argname("<alg>") + :convert { + LBFGS = "LBFGS", + ADAM = "ADAM", + ADAGRAD = "ADAGRAD", + SGD = "SGD", + NAG = "NAG" + } + :default "ADAM" +parser:option "--ignore-symbol" + :description "Ignore symbol from logs" + :argname("<sym>") + :args "*" +parser:option "--penalty-weight" + :description "Add new penalty weight to test" + :argname("<n>") + :convert(tonumber) + :args "*" +parser:option "--learning-rate" + :description "Add new learning rate to test" + :argname("<n>") + :convert(tonumber) + :args "*" +parser:option "--spam_action" + :description "Spam action" + :argname("<act>") + :default("reject") +parser:option "--learning_rate_decay" + :description "Learn rate decay (for some algs)" + :argname("<n>") + :convert(tonumber) + :default(0.0) +parser:option "--weight_decay" + :description "Weight decay (for some algs)" + :argname("<n>") + :convert(tonumber) + :default(0.0) +parser:option "--l1" + :description "L1 regularization penalty" + :argname("<n>") + :convert(tonumber) + :default(0.0) +parser:option "--l2" + :description "L2 regularization penalty" + :argname("<n>") + :convert(tonumber) + :default(0.0) + local function make_dataset_from_logs(logs, all_symbols, spam_score) -- Returns a list of {input, output} for torch SGD train @@ -71,17 +188,18 @@ local function init_weights(all_symbols, original_symbol_scores) return weights end -local function shuffle(logs) +local function shuffle(logs, messages) local size = #logs for i = size, 1, -1 do local rand = math.random(size) logs[i], logs[rand] = logs[rand], logs[i] + messages[i], messages[rand] = messages[rand], messages[i] end end -local function split_logs(logs, split_percent) +local function split_logs(logs, messages, split_percent) if not split_percent then split_percent = 60 @@ -91,16 +209,20 @@ local function split_logs(logs, split_percent) local test_logs = {} local train_logs = {} + local test_messages = {} + local train_messages = {} for i=1,split_index do - train_logs[#train_logs + 1] = logs[i] + table.insert(train_logs, logs[i]) + table.insert(train_messages, messages[i]) end for i=split_index + 1, #logs do - test_logs[#test_logs + 1] = logs[i] + table.insert(test_logs, logs[i]) + table.insert(test_messages, messages[i]) end - return train_logs, test_logs + return {train_logs,train_messages}, {test_logs,test_messages} end local function stitch_new_scores(all_symbols, new_scores) @@ -174,7 +296,10 @@ local function print_score_diff(new_symbol_scores, original_symbol_scores) end -local function calculate_fscore_from_weights(logs, all_symbols, weights, threshold) +local function calculate_fscore_from_weights(logs, messages, + all_symbols, + weights, + threshold) local new_symbol_scores = weights:clone() @@ -182,14 +307,16 @@ local function calculate_fscore_from_weights(logs, all_symbols, weights, thresho logs = update_logs(logs, new_symbol_scores) - local file_stats, _, all_fps, all_fns = rescore_utility.generate_statistics_from_logs(logs, threshold) + local file_stats, _, all_fps, all_fns = + rescore_utility.generate_statistics_from_logs(logs, messages, threshold) return file_stats.fscore, all_fps, all_fns end -local function print_stats(logs, threshold) +local function print_stats(logs, messages, threshold) - local file_stats, _ = rescore_utility.generate_statistics_from_logs(logs, threshold) + local file_stats, _ = rescore_utility.generate_statistics_from_logs(logs, + messages, threshold) local file_stat_format = [[ F-score: %.2f @@ -226,7 +353,7 @@ local function train(dataset, opt, model, criterion, epoch, local lbfgsState local sgdState - local batch_size = opt.batch_size + local batch_size = opt.batch logger.messagex("trainer epoch #%s, %s batch", epoch, batch_size) @@ -300,7 +427,7 @@ local function train(dataset, opt, model, criterion, epoch, end -- optimize on current mini-batch - if opt.optimization == 'LBFGS' then + if opt.optim == 'LBFGS' then -- Perform LBFGS step: lbfgsState = lbfgsState or { @@ -315,7 +442,7 @@ local function train(dataset, opt, model, criterion, epoch, logger.messagex(' - nb of iterations: ' .. lbfgsState.nIter) logger.messagex(' - nb of function evalutions: ' .. lbfgsState.funcEval) - elseif opt.optimization == 'ADAM' then + elseif opt.optim == 'ADAM' then sgdState = sgdState or { learningRate = tonumber(opts.learning_rate),-- opt.learningRate, momentum = tonumber(opts.momentum), -- opt.momentum, @@ -323,7 +450,7 @@ local function train(dataset, opt, model, criterion, epoch, weightDecay = tonumber(opts.weight_decay), } optim.adam(feval, parameters, sgdState) - elseif opt.optimization == 'ADAGRAD' then + elseif opt.optim == 'ADAGRAD' then sgdState = sgdState or { learningRate = tonumber(opts.learning_rate),-- opt.learningRate, momentum = tonumber(opts.momentum), -- opt.momentum, @@ -331,7 +458,7 @@ local function train(dataset, opt, model, criterion, epoch, weightDecay = tonumber(opts.weight_decay), } optim.adagrad(feval, parameters, sgdState) - elseif opt.optimization == 'SGD' then + elseif opt.optim == 'SGD' then sgdState = sgdState or { learningRate = tonumber(opts.learning_rate),-- opt.learningRate, momentum = tonumber(opts.momentum), -- opt.momentum, @@ -339,7 +466,7 @@ local function train(dataset, opt, model, criterion, epoch, weightDecay = tonumber(opts.weight_decay), } optim.sgd(feval, parameters, sgdState) - elseif opt.optimization == 'NAG' then + elseif opt.optim == 'NAG' then sgdState = sgdState or { learningRate = tonumber(opts.learning_rate),-- opt.learningRate, momentum = tonumber(opts.momentum), -- opt.momentum, @@ -348,7 +475,8 @@ local function train(dataset, opt, model, criterion, epoch, } optim.nag(feval, parameters, sgdState) else - error('unknown optimization method') + logger.errx('unknown optimization method: %s', opt.optim) + os.exit(1) end end @@ -363,19 +491,6 @@ local function train(dataset, opt, model, criterion, epoch, confusion:zero() end - -local default_opts = { - verbose = true, - iters = 10, - threads = 1, - batch_size = 1000, - optimization = 'ADAM', - learning_rate_decay = 0.001, - momentum = 0.1, - l1 = 0.0, - l2 = 0.0, -} - local learning_rates = { 0.01 } @@ -393,11 +508,27 @@ local function get_threshold() or actions['reject']), actions['reject'] end -return function (args, cfg) - opts = default_opts - opts = lua_util.override_defaults(opts, getopt.getopt(args, 'i:')) +local function handler(args) + opts = parser:parse(args) + if not opts['log'] then + parser:error('no log specified') + end + + local _r,err = rspamd_config:load_ucl(opts['config']) + + if not _r then + logger.errx('cannot parse %s: %s', opts['config'], err) + os.exit(1) + end + + _r,err = rspamd_config:parse_rcl({'logging', 'worker'}) + if not _r then + logger.errx('cannot process %s: %s', opts['config'], err) + os.exit(1) + end + local threshold,reject_score = get_threshold() - local logs = rescore_utility.get_all_logs(cfg["logdir"]) + local logs,messages = rescore_utility.get_all_logs(opts['log']) if opts['ignore-symbol'] then local function add_ignore(s) @@ -446,15 +577,15 @@ return function (args, cfg) end end - if opts['i'] then opts['iters'] = opts['i'] end - local all_symbols = rescore_utility.get_all_symbols(logs, ignore_symbols) local original_symbol_scores = rescore_utility.get_all_symbol_scores(rspamd_config, ignore_symbols) -- Display hit frequencies - if opts['z'] then - local _, all_symbols_stats = rescore_utility.generate_statistics_from_logs(logs, threshold) + if opts['freq'] then + local _, all_symbols_stats = rescore_utility.generate_statistics_from_logs(logs, + messages, + threshold) local t = {} for _, symbol_stats in pairs(all_symbols_stats) do table.insert(t, symbol_stats) end @@ -473,8 +604,8 @@ return function (args, cfg) "NAME", "HITS", "HAM", "HAM%", "SPAM", "SPAM%", "S/O", "OVER%")) for _, symbol_stats in pairs(t) do logger.message( - string.format("%-40s %6d %6d %6.2f %6d %6.2f %6.2f %6.2f", - symbol_stats.name, + string.format("%-40s %6d %6d %6.2f %6d %6.2f %6.2f %6.2f", + symbol_stats.name, symbol_stats.no_of_hits, symbol_stats.ham_hits, lua_util.round(symbol_stats.ham_percent,2), @@ -487,7 +618,7 @@ return function (args, cfg) end -- Print file statistics - print_stats(logs, threshold) + print_stats(logs, messages, threshold) -- Work out how many symbols weren't seen in the corpus local symbols_no_hits = {} @@ -503,7 +634,7 @@ return function (args, cfg) -- Calculate percentage of rules with no hits local nhpct = lua_util.round((#symbols_no_hits/total_symbols)*100,2) logger.message( - string.format('\nFound %s (%-.2f%%) symbols out of %s with no hits in corpus:', + string.format('\nFound %s (%-.2f%%) symbols out of %s with no hits in corpus:', #symbols_no_hits, nhpct, total_symbols ) ) @@ -515,13 +646,13 @@ return function (args, cfg) return end - shuffle(logs) + shuffle(logs, messages) torch.setdefaulttensortype('torch.FloatTensor') - local train_logs, validation_logs = split_logs(logs, 70) - local cv_logs, test_logs = split_logs(validation_logs, 50) + local train_logs, validation_logs = split_logs(logs, messages,70) + local cv_logs, test_logs = split_logs(validation_logs[1], validation_logs[2], 50) - local dataset = make_dataset_from_logs(train_logs, all_symbols, reject_score) + local dataset = make_dataset_from_logs(train_logs[1], all_symbols, reject_score) -- Start of perceptron training local input_size = #all_symbols @@ -555,7 +686,8 @@ return function (args, cfg) initial_weights) end - local fscore, fps, fns = calculate_fscore_from_weights(cv_logs, + local fscore, fps, fns = calculate_fscore_from_weights(cv_logs[1], + cv_logs[2], all_symbols, linear_module.weight[1], threshold) @@ -580,23 +712,23 @@ return function (args, cfg) new_symbol_scores = stitch_new_scores(all_symbols, new_symbol_scores) - if cfg["output"] then - write_scores(new_symbol_scores, cfg["output"]) + if opts["output"] then + write_scores(new_symbol_scores, opts["output"]) end - if cfg["diff"] then + if opts["diff"] then print_score_diff(new_symbol_scores, original_symbol_scores) end -- Pre-rescore test stats logger.message("\n\nPre-rescore test stats\n") - test_logs = update_logs(test_logs, original_symbol_scores) - print_stats(test_logs, threshold) + test_logs[1] = update_logs(test_logs[1], original_symbol_scores) + print_stats(test_logs[1], test_logs[2], threshold) -- Post-rescore test stats - test_logs = update_logs(test_logs, new_symbol_scores) + test_logs[1] = update_logs(test_logs[1], new_symbol_scores) logger.message("\n\nPost-rescore test stats\n") - print_stats(test_logs, threshold) + print_stats(test_logs[1], test_logs[2], threshold) logger.messagex('Best fscore=%s, best learning rate=%s, best weight decay=%s', best_fscore, best_learning_rate, best_weight_decay) @@ -616,3 +748,10 @@ return function (args, cfg) end end end + + +return { + handler = handler, + description = parser._description, + name = 'rescore' +}
\ No newline at end of file diff --git a/rules/regexp/headers.lua b/rules/regexp/headers.lua index 5658608fb..a7300e2b8 100644 --- a/rules/regexp/headers.lua +++ b/rules/regexp/headers.lua @@ -589,11 +589,19 @@ reconf['YANDEX_RU_MAILER'] = { -- Detect 1C v8.2 and v8.3 mailers reconf['MAILER_1C_8'] = { re = 'X-Mailer=/^1C:Enterprise 8\\.[23]$/H', - score = 0, + score = 0.0, description = 'Sent with 1C:Enterprise 8', group = 'header' } +-- Detect rogue 'strongmail' MTA with IPv4 and '(-)' in Received line +reconf['STRONGMAIL'] = { + re = [[Received=/^from\s+strongmail\s+\(\[\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\]\) by \S+ \(-\); /mH]], + score = 6.0, + description = 'Sent via rogue "strongmail" MTA', + group = 'header' +} + -- Two received headers with ip addresses local double_ip_spam_1 = 'Received=/from \\[\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\] by \\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3} with/H' local double_ip_spam_2 = 'Received=/from\\s+\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s+by\\s+\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3};/H' diff --git a/src/libcryptobox/cryptobox.c b/src/libcryptobox/cryptobox.c index c0857b094..99c91e3dd 100644 --- a/src/libcryptobox/cryptobox.c +++ b/src/libcryptobox/cryptobox.c @@ -560,6 +560,10 @@ rspamd_cryptobox_sign (guchar *sig, gsize *siglen_p, &diglen, kinv, rp, lk) == 1); g_assert (diglen <= sizeof (rspamd_signature_t)); + if (siglen_p) { + *siglen_p = diglen; + } + EC_KEY_free (lk); EVP_MD_CTX_destroy (sha_ctx); BN_free (bn_sec); @@ -572,6 +576,7 @@ rspamd_cryptobox_sign (guchar *sig, gsize *siglen_p, bool rspamd_cryptobox_verify (const guchar *sig, + gsize siglen, const guchar *m, gsize mlen, const rspamd_pk_t pk, @@ -580,6 +585,7 @@ rspamd_cryptobox_verify (const guchar *sig, bool ret = false; if (G_LIKELY (mode == RSPAMD_CRYPTOBOX_MODE_25519)) { + g_assert (siglen == rspamd_cryptobox_signature_bytes (RSPAMD_CRYPTOBOX_MODE_25519)); ret = ed25519_verify (sig, m, mlen, pk); } else { @@ -608,8 +614,7 @@ rspamd_cryptobox_verify (const guchar *sig, g_assert (EC_KEY_set_public_key (lk, ec_pub) == 1); /* ECDSA */ - ret = ECDSA_verify (0, h, sizeof (h), sig, - rspamd_cryptobox_signature_bytes (mode), lk) == 1; + ret = ECDSA_verify (0, h, sizeof (h), sig, siglen, lk) == 1; EC_KEY_free (lk); EVP_MD_CTX_destroy (sha_ctx); diff --git a/src/libcryptobox/cryptobox.h b/src/libcryptobox/cryptobox.h index 1045547a2..1d48c06e5 100644 --- a/src/libcryptobox/cryptobox.h +++ b/src/libcryptobox/cryptobox.h @@ -40,7 +40,7 @@ struct rspamd_cryptobox_segment { #define rspamd_cryptobox_HASHSTATEBYTES 256 #define rspamd_cryptobox_MAX_SIGSKBYTES 64 #define rspamd_cryptobox_MAX_SIGPKBYTES 32 -#define rspamd_cryptobox_MAX_SIGBYTES 64 +#define rspamd_cryptobox_MAX_SIGBYTES 72 #define CPUID_AVX2 0x1 #define CPUID_AVX 0x2 @@ -214,6 +214,7 @@ void rspamd_cryptobox_sign (guchar *sig, gsize *siglen_p, * @return true if signature is valid, false otherwise */ bool rspamd_cryptobox_verify (const guchar *sig, + gsize siglen, const guchar *m, gsize mlen, const rspamd_pk_t pk, diff --git a/src/libcryptobox/keypair.c b/src/libcryptobox/keypair.c index 50e3614d9..21b497130 100644 --- a/src/libcryptobox/keypair.c +++ b/src/libcryptobox/keypair.c @@ -881,7 +881,7 @@ rspamd_keypair_verify (struct rspamd_cryptobox_pubkey *pk, return FALSE; } - if (!rspamd_cryptobox_verify (sig, data, len, + if (!rspamd_cryptobox_verify (sig, siglen, data, len, rspamd_cryptobox_pubkey_pk (pk, &pklen), pk->alg)) { g_set_error (err, rspamd_keypair_quark (), EPERM, "signature verification failed"); diff --git a/src/libmime/email_addr.c b/src/libmime/email_addr.c index 08071afc8..a0ad9ac19 100644 --- a/src/libmime/email_addr.c +++ b/src/libmime/email_addr.c @@ -203,10 +203,12 @@ rspamd_email_address_from_mime (rspamd_mempool_t *pool, struct rspamd_email_address addr; const gchar *p = hdr, *end = hdr + len, *c = hdr, *t; GString *ns; + gint obraces, ebraces; enum { parse_name = 0, parse_quoted, parse_addr, + skip_comment, skip_spaces } state = parse_name, next_state = parse_name; @@ -290,6 +292,23 @@ rspamd_email_address_from_mime (rspamd_mempool_t *pool, else if (*p == '@') { seen_at = TRUE; } + else if (*p == '(') { + if (p > c) { + t = p - 1; + + while (t > c && g_ascii_isspace (*t)) { + t --; + } + + g_string_append_len (ns, c, t - c + 1); + } + + c = p; + obraces = 1; + ebraces = 0; + state = skip_comment; + next_state = parse_name; + } p ++; break; case parse_quoted: @@ -331,6 +350,12 @@ rspamd_email_address_from_mime (rspamd_mempool_t *pool, else if (*p == '@') { seen_at = TRUE; } + else if (*p == '(') { + obraces = 1; + ebraces = 0; + state = skip_comment; + next_state = parse_addr; + } p ++; break; case skip_spaces: @@ -342,6 +367,33 @@ rspamd_email_address_from_mime (rspamd_mempool_t *pool, p ++; } break; + case skip_comment: + if (*p == '(') { + obraces ++; + } + else if (*p == ')') { + ebraces --; + } + + if (obraces == ebraces) { + if (next_state == parse_name) { + /* Include comment in name */ + if (p > c) { + t = p - 1; + + while (t > c && g_ascii_isspace (*t)) { + t --; + } + + g_string_append_len (ns, c, t - c + 1); + } + + c = p; + } + state = next_state; + } + p ++; + break; } } @@ -396,7 +448,8 @@ rspamd_email_address_from_mime (rspamd_mempool_t *pool, } break; case parse_quoted: - /* Unfinished quoted string */ + case skip_comment: + /* Unfinished quoted string or a comment */ break; default: /* Do nothing */ diff --git a/src/libmime/message.c b/src/libmime/message.c index 5681c3d00..e5e43c5be 100644 --- a/src/libmime/message.c +++ b/src/libmime/message.c @@ -907,13 +907,10 @@ rspamd_message_from_data (struct rspamd_task *task, const guchar *start, gboolean rspamd_message_parse (struct rspamd_task *task) { - struct rspamd_mime_text_part *p1, *p2; struct received_header *recv, *trecv; const gchar *p; gsize len; guint i; - gdouble diff, *pdiff; - guint tw, *ptw, dw; GError *err = NULL; rspamd_cryptobox_hash_state_t st; guchar digest_out[rspamd_cryptobox_HASHBYTES]; @@ -1016,16 +1013,6 @@ rspamd_message_parse (struct rspamd_task *task) task->queue_id = "undef"; } - for (i = 0; i < task->parts->len; i ++) { - struct rspamd_mime_part *part; - - part = g_ptr_array_index (task->parts, i); - rspamd_message_process_text_part (task, part); - } - - rspamd_images_process (task); - rspamd_archives_process (task); - if (task->received->len > 0) { gboolean need_recv_correction = FALSE; rspamd_inet_addr_t *raddr; @@ -1130,6 +1117,50 @@ rspamd_message_parse (struct rspamd_task *task) rspamd_url_task_subject_callback, task); } + for (i = 0; i < task->parts->len; i ++) { + struct rspamd_mime_part *part; + + part = g_ptr_array_index (task->parts, i); + rspamd_cryptobox_hash_update (&st, part->digest, sizeof (part->digest)); + } + + rspamd_cryptobox_hash_final (&st, digest_out); + memcpy (task->digest, digest_out, sizeof (task->digest)); + + if (task->queue_id) { + msg_info_task ("loaded message; id: <%s>; queue-id: <%s>; size: %z; " + "checksum: <%*xs>", + task->message_id, task->queue_id, task->msg.len, + (gint)sizeof (task->digest), task->digest); + } + else { + msg_info_task ("loaded message; id: <%s>; size: %z; " + "checksum: <%*xs>", + task->message_id, task->msg.len, + (gint)sizeof (task->digest), task->digest); + } + + return TRUE; +} + +void +rspamd_message_process (struct rspamd_task *task) +{ + guint i; + struct rspamd_mime_text_part *p1, *p2; + gdouble diff, *pdiff; + guint tw, *ptw, dw; + + for (i = 0; i < task->parts->len; i ++) { + struct rspamd_mime_part *part; + + part = g_ptr_array_index (task->parts, i); + rspamd_message_process_text_part (task, part); + } + + rspamd_images_process (task); + rspamd_archives_process (task); + /* Calculate distance for 2-parts messages */ if (task->text_parts->len == 2) { p1 = g_ptr_array_index (task->text_parts, 0); @@ -1144,7 +1175,7 @@ rspamd_message_parse (struct rspamd_task *task) if (rspamd_ftok_cmp (&p1->mime_part->parent_part->ct->subtype, &srch) == 0) { if (!IS_PART_EMPTY (p1) && !IS_PART_EMPTY (p2) && - p1->normalized_hashes && p2->normalized_hashes) { + p1->normalized_hashes && p2->normalized_hashes) { /* * We also detect language on one part and propagate it to * another one @@ -1219,13 +1250,6 @@ rspamd_message_parse (struct rspamd_task *task) } } - for (i = 0; i < task->parts->len; i ++) { - struct rspamd_mime_part *part; - - part = g_ptr_array_index (task->parts, i); - rspamd_cryptobox_hash_update (&st, part->digest, sizeof (part->digest)); - } - /* Calculate average words length and number of short words */ struct rspamd_mime_text_part *text_part; gdouble *var; @@ -1258,24 +1282,6 @@ rspamd_message_parse (struct rspamd_task *task) *var /= (double)total_words; } } - - rspamd_cryptobox_hash_final (&st, digest_out); - memcpy (task->digest, digest_out, sizeof (task->digest)); - - if (task->queue_id) { - msg_info_task ("loaded message; id: <%s>; queue-id: <%s>; size: %z; " - "checksum: <%*xs>", - task->message_id, task->queue_id, task->msg.len, - (gint)sizeof (task->digest), task->digest); - } - else { - msg_info_task ("loaded message; id: <%s>; size: %z; " - "checksum: <%*xs>", - task->message_id, task->msg.len, - (gint)sizeof (task->digest), task->digest); - } - - return TRUE; } diff --git a/src/libmime/message.h b/src/libmime/message.h index 0ed2a5c66..b16011666 100644 --- a/src/libmime/message.h +++ b/src/libmime/message.h @@ -143,6 +143,12 @@ struct received_header { gboolean rspamd_message_parse (struct rspamd_task *task); /** + * Process content in task (e.g. HTML parsing) + * @param task + */ +void rspamd_message_process (struct rspamd_task *task); + +/** * Get an array of header's values with specified header's name using raw headers * @param task worker task structure * @param field header's name diff --git a/src/libserver/cfg_file.h b/src/libserver/cfg_file.h index 520c847f3..777340e16 100644 --- a/src/libserver/cfg_file.h +++ b/src/libserver/cfg_file.h @@ -313,6 +313,7 @@ struct rspamd_config { gboolean enable_experimental; /**< Enable experimental plugins */ gboolean disable_pcre_jit; /**< Disable pcre JIT */ gboolean disable_lua_squeeze; /**< Disable lua rules squeezing */ + gboolean own_lua_state; /**< True if we have created lua_state internally */ gsize max_diff; /**< maximum diff size for text parts */ gsize max_cores_size; /**< maximum size occupied by rspamd core files */ @@ -445,11 +446,16 @@ struct rspamd_config { gboolean rspamd_parse_bind_line (struct rspamd_config *cfg, struct rspamd_worker_conf *cf, const gchar *str); + +enum rspamd_config_init_flags { + RSPAMD_CONFIG_INIT_DEFAULT = 0, + RSPAMD_CONFIG_INIT_SKIP_LUA = (1 << 0) +}; /** * Init default values * @param cfg config file */ -struct rspamd_config *rspamd_config_new (void); +struct rspamd_config *rspamd_config_new (enum rspamd_config_init_flags flags); /** * Free memory used by config structure diff --git a/src/libserver/cfg_rcl.c b/src/libserver/cfg_rcl.c index deb4a8ed3..20d51b31c 100644 --- a/src/libserver/cfg_rcl.c +++ b/src/libserver/cfg_rcl.c @@ -34,10 +34,6 @@ #include <syslog.h> #endif -#ifdef HAVE_GLOB_H -#include <glob.h> -#endif - #include <math.h> struct rspamd_rcl_default_handler_data { @@ -627,320 +623,6 @@ rspamd_rcl_worker_handler (rspamd_mempool_t *pool, const ucl_object_t *obj, return TRUE; } -#define RSPAMD_CONFDIR_INDEX "CONFDIR" -#define RSPAMD_RUNDIR_INDEX "RUNDIR" -#define RSPAMD_DBDIR_INDEX "DBDIR" -#define RSPAMD_LOGDIR_INDEX "LOGDIR" -#define RSPAMD_PLUGINSDIR_INDEX "PLUGINSDIR" -#define RSPAMD_RULESDIR_INDEX "RULESDIR" -#define RSPAMD_LUALIBDIR_INDEX "LUALIBDIR" -#define RSPAMD_WWWDIR_INDEX "WWWDIR" -#define RSPAMD_PREFIX_INDEX "PREFIX" -#define RSPAMD_VERSION_INDEX "VERSION" - -static gint -rspamd_rcl_cmp_components (const gchar *comp1, const gchar *comp2) -{ - guint v1, v2; - - v1 = strtoul (comp1, NULL, 10); - v2 = strtoul (comp2, NULL, 10); - - return v1 - v2; -} - -static int -rspamd_rcl_lua_version_cmp (lua_State *L) -{ - const gchar *ver; - gchar **components; - gint ret = 0; - - if (lua_type (L, 2) == LUA_TSTRING) { - ver = lua_tostring (L, 2); - - components = g_strsplit_set (ver, ".-_", -1); - - if (!components) { - return luaL_error (L, "invalid arguments to 'cmp': %s", ver); - } - - if (components[0]) { - ret = rspamd_rcl_cmp_components (components[0], RSPAMD_VERSION_MAJOR); - } - - if (ret) { - goto set; - } - - if (components[1]) { - ret = rspamd_rcl_cmp_components (components[1], RSPAMD_VERSION_MINOR); - } - - if (ret) { - goto set; - } - - if (components[2]) { - ret = rspamd_rcl_cmp_components (components[2], RSPAMD_VERSION_PATCH); - } - - /* - * XXX: we don't compare git releases assuming that it is meaningless - */ - } - else { - return luaL_error (L, "invalid arguments to 'cmp'"); - } - -set: - g_strfreev (components); - lua_pushnumber (L, ret); - - return 1; -} - -static int -rspamd_rcl_lua_version_numeric (lua_State *L) -{ - static gint64 version_num = RSPAMD_VERSION_NUM; - const gchar *type; - - if (lua_gettop (L) >= 2 && lua_type (L, 1) == LUA_TSTRING) { - type = lua_tostring (L, 1); - if (g_ascii_strcasecmp (type, "short") == 0) { - version_num = RSPAMD_VERSION_MAJOR_NUM * 1000 + - RSPAMD_VERSION_MINOR_NUM * 100 + RSPAMD_VERSION_PATCH_NUM * 10; - } - else if (g_ascii_strcasecmp (type, "main") == 0) { - version_num = RSPAMD_VERSION_MAJOR_NUM * 1000 + - RSPAMD_VERSION_MINOR_NUM * 100; - } - else if (g_ascii_strcasecmp (type, "major") == 0) { - version_num = RSPAMD_VERSION_MAJOR_NUM; - } - else if (g_ascii_strcasecmp (type, "minor") == 0) { - version_num = RSPAMD_VERSION_MINOR_NUM; - } - else if (g_ascii_strcasecmp (type, "patch") == 0) { - version_num = RSPAMD_VERSION_PATCH_NUM; - } - } - - lua_pushnumber (L, version_num); - - return 1; -} - -static int -rspamd_rcl_lua_version (lua_State *L) -{ - const gchar *result = NULL, *type; - - if (lua_gettop (L) == 0) { - result = RVERSION; - } - else if (lua_gettop (L) >= 1 && lua_type (L, 1) == LUA_TSTRING) { - /* We got something like string */ - type = lua_tostring (L, 1); - - if (g_ascii_strcasecmp (type, "short") == 0) { - result = RSPAMD_VERSION_MAJOR - "." RSPAMD_VERSION_MINOR - "." RSPAMD_VERSION_PATCH; - } - else if (g_ascii_strcasecmp (type, "main") == 0) { - result = RSPAMD_VERSION_MAJOR "." RSPAMD_VERSION_MINOR; - } - else if (g_ascii_strcasecmp (type, "major") == 0) { - result = RSPAMD_VERSION_MAJOR; - } - else if (g_ascii_strcasecmp (type, "minor") == 0) { - result = RSPAMD_VERSION_MINOR; - } - else if (g_ascii_strcasecmp (type, "patch") == 0) { - result = RSPAMD_VERSION_PATCH; - } - else if (g_ascii_strcasecmp (type, "id") == 0) { - result = RID; - } - else if (g_ascii_strcasecmp (type, "num") == 0) { - return rspamd_rcl_lua_version_numeric (L); - } - else if (g_ascii_strcasecmp (type, "cmp") == 0) { - return rspamd_rcl_lua_version_cmp (L); - } - } - - lua_pushstring (L, result); - - return 1; -} - -static void -rspamd_rcl_set_lua_globals (struct rspamd_config *cfg, lua_State *L, - GHashTable *vars) -{ - struct rspamd_config **pcfg; - - /* First check for global variable 'config' */ - lua_getglobal (L, "config"); - if (lua_isnil (L, -1)) { - /* Assign global table to set up attributes */ - lua_newtable (L); - lua_setglobal (L, "config"); - } - - lua_getglobal (L, "metrics"); - if (lua_isnil (L, -1)) { - lua_newtable (L); - lua_setglobal (L, "metrics"); - } - - lua_getglobal (L, "composites"); - if (lua_isnil (L, -1)) { - lua_newtable (L); - lua_setglobal (L, "composites"); - } - - lua_getglobal (L, "rspamd_classifiers"); - if (lua_isnil (L, -1)) { - lua_newtable (L); - lua_setglobal (L, "rspamd_classifiers"); - } - - lua_getglobal (L, "classifiers"); - if (lua_isnil (L, -1)) { - lua_newtable (L); - lua_setglobal (L, "classifiers"); - } - - lua_getglobal (L, "rspamd_version"); - if (lua_isnil (L, -1)) { - lua_pushcfunction (L, rspamd_rcl_lua_version); - lua_setglobal (L, "rspamd_version"); - } - - pcfg = lua_newuserdata (L, sizeof (struct rspamd_config *)); - rspamd_lua_setclass (L, "rspamd{config}", -1); - *pcfg = cfg; - lua_setglobal (L, "rspamd_config"); - - /* Clear stack from globals */ - lua_pop (L, 4); - - rspamd_lua_set_path (L, cfg->rcl_obj, vars); - - /* Set known paths as rspamd_paths global */ - lua_getglobal (L, "rspamd_paths"); - if (lua_isnil (L, -1)) { - const gchar *confdir = RSPAMD_CONFDIR, *rundir = RSPAMD_RUNDIR, - *dbdir = RSPAMD_DBDIR, *logdir = RSPAMD_LOGDIR, - *wwwdir = RSPAMD_WWWDIR, *pluginsdir = RSPAMD_PLUGINSDIR, - *rulesdir = RSPAMD_RULESDIR, *lualibdir = RSPAMD_LUALIBDIR, - *prefix = RSPAMD_PREFIX; - const gchar *t; - - /* Try environment */ - t = getenv ("PLUGINSDIR"); - if (t) { - pluginsdir = t; - } - - t = getenv ("RULESDIR"); - if (t) { - rulesdir = t; - } - - t = getenv ("DBDIR"); - if (t) { - dbdir = t; - } - - t = getenv ("RUNDIR"); - if (t) { - rundir = t; - } - - t = getenv ("LUALIBDIR"); - if (t) { - lualibdir = t; - } - - t = getenv ("LOGDIR"); - if (t) { - logdir = t; - } - - t = getenv ("WWWDIR"); - if (t) { - wwwdir = t; - } - - t = getenv ("CONFDIR"); - if (t) { - confdir = t; - } - - - if (vars) { - t = g_hash_table_lookup (vars, "PLUGINSDIR"); - if (t) { - pluginsdir = t; - } - - t = g_hash_table_lookup (vars, "RULESDIR"); - if (t) { - rulesdir = t; - } - - t = g_hash_table_lookup (vars, "LUALIBDIR"); - if (t) { - lualibdir = t; - } - - t = g_hash_table_lookup (vars, "RUNDIR"); - if (t) { - rundir = t; - } - - t = g_hash_table_lookup (vars, "WWWDIR"); - if (t) { - wwwdir = t; - } - - t = g_hash_table_lookup (vars, "CONFDIR"); - if (t) { - confdir = t; - } - - t = g_hash_table_lookup (vars, "DBDIR"); - if (t) { - dbdir = t; - } - - t = g_hash_table_lookup (vars, "LOGDIR"); - if (t) { - logdir = t; - } - } - - lua_createtable (L, 0, 9); - - rspamd_lua_table_set (L, RSPAMD_CONFDIR_INDEX, confdir); - rspamd_lua_table_set (L, RSPAMD_RUNDIR_INDEX, rundir); - rspamd_lua_table_set (L, RSPAMD_DBDIR_INDEX, dbdir); - rspamd_lua_table_set (L, RSPAMD_LOGDIR_INDEX, logdir); - rspamd_lua_table_set (L, RSPAMD_WWWDIR_INDEX, wwwdir); - rspamd_lua_table_set (L, RSPAMD_PLUGINSDIR_INDEX, pluginsdir); - rspamd_lua_table_set (L, RSPAMD_RULESDIR_INDEX, rulesdir); - rspamd_lua_table_set (L, RSPAMD_LUALIBDIR_INDEX, lualibdir); - rspamd_lua_table_set (L, RSPAMD_PREFIX_INDEX, prefix); - - lua_setglobal (L, "rspamd_paths"); - } -} - static gboolean rspamd_rcl_lua_handler (rspamd_mempool_t *pool, const ucl_object_t *obj, const gchar *key, gpointer ud, @@ -1047,9 +729,8 @@ rspamd_rcl_add_lua_plugins_path (struct rspamd_config *cfg, { struct stat st; struct script_module *cur_mod; - glob_t globbuf; - gchar *pattern, *ext_pos; - size_t len; + GPtrArray *paths; + gchar *fname, *ext_pos; guint i; if (stat (path, &st) == -1) { @@ -1064,52 +745,39 @@ rspamd_rcl_add_lua_plugins_path (struct rspamd_config *cfg, /* Handle directory */ if (S_ISDIR (st.st_mode)) { - globbuf.gl_offs = 0; - len = strlen (path) + sizeof ("*.lua"); - pattern = g_malloc (len); - rspamd_snprintf (pattern, len, "%s%s", path, "*.lua"); - - if (glob (pattern, GLOB_DOOFFS, NULL, &globbuf) == 0) { - for (i = 0; i < globbuf.gl_pathc; i++) { - cur_mod = + paths = rspamd_glob_path (path, "*.lua", TRUE, err); + + if (!paths) { + return FALSE; + } + + PTR_ARRAY_FOREACH (paths, i, fname) { + cur_mod = rspamd_mempool_alloc (cfg->cfg_pool, - sizeof (struct script_module)); - cur_mod->path = rspamd_mempool_strdup (cfg->cfg_pool, - globbuf.gl_pathv[i]); - cur_mod->name = g_path_get_basename (cur_mod->path); - rspamd_mempool_add_destructor (cfg->cfg_pool, g_free, - cur_mod->name); - ext_pos = strstr (cur_mod->name, ".lua"); - - if (ext_pos != NULL) { - *ext_pos = '\0'; - } + sizeof (struct script_module)); + cur_mod->path = rspamd_mempool_strdup (cfg->cfg_pool, fname); + cur_mod->name = g_path_get_basename (cur_mod->path); + rspamd_mempool_add_destructor (cfg->cfg_pool, g_free, + cur_mod->name); + ext_pos = strstr (cur_mod->name, ".lua"); + + if (ext_pos != NULL) { + *ext_pos = '\0'; + } - if (cfg->script_modules == NULL) { - cfg->script_modules = g_list_append (cfg->script_modules, - cur_mod); - rspamd_mempool_add_destructor (cfg->cfg_pool, - (rspamd_mempool_destruct_t)g_list_free, - cfg->script_modules); - } - else { - cfg->script_modules = g_list_append (cfg->script_modules, - cur_mod); - } + if (cfg->script_modules == NULL) { + cfg->script_modules = g_list_append (cfg->script_modules, + cur_mod); + rspamd_mempool_add_destructor (cfg->cfg_pool, + (rspamd_mempool_destruct_t) g_list_free, + cfg->script_modules); + } else { + cfg->script_modules = g_list_append (cfg->script_modules, + cur_mod); } - globfree (&globbuf); - g_free (pattern); - } - else { - g_set_error (err, - CFG_RCL_ERROR, - errno, - "glob failed for %s, %s", - pattern, - strerror (errno)); - g_free (pattern); - return FALSE; } + + g_ptr_array_free (paths, TRUE); } else { /* Handle single file */ @@ -1762,7 +1430,7 @@ rspamd_rcl_add_default_handler (struct rspamd_rcl_section *section, } struct rspamd_rcl_section * -rspamd_rcl_config_init (struct rspamd_config *cfg) +rspamd_rcl_config_init (struct rspamd_config *cfg, GHashTable *skip_sections) { struct rspamd_rcl_section *new = NULL, *sub, *ssub; @@ -1775,770 +1443,788 @@ rspamd_rcl_config_init (struct rspamd_config *cfg) /** * Logging section */ - sub = rspamd_rcl_add_section_doc (&new, - "logging", NULL, - rspamd_rcl_logging_handler, - UCL_OBJECT, - FALSE, - TRUE, - cfg->doc_strings, - "Configure rspamd logging"); - /* Default handlers */ - rspamd_rcl_add_default_handler (sub, - "log_buffer", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, log_buf_size), - RSPAMD_CL_FLAG_INT_32, - "Size of log buffer in bytes (for file logging)"); - rspamd_rcl_add_default_handler (sub, - "log_urls", - rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_config, log_urls), - 0, - "Write each URL found in a message to the log file"); - rspamd_rcl_add_default_handler (sub, - "debug_ip", - rspamd_rcl_parse_struct_ucl, - G_STRUCT_OFFSET (struct rspamd_config, debug_ip_map), - 0, - "Enable debugging log for the specified IP addresses"); - rspamd_rcl_add_default_handler (sub, - "debug_symbols", - rspamd_rcl_parse_struct_string_list, - G_STRUCT_OFFSET (struct rspamd_config, debug_symbols), - 0, - "Enable debug for the specified symbols"); - rspamd_rcl_add_default_handler (sub, - "debug_modules", - rspamd_rcl_parse_struct_string_list, - G_STRUCT_OFFSET (struct rspamd_config, debug_modules), - RSPAMD_CL_FLAG_STRING_LIST_HASH, - "Enable debugging for the specified modules"); - rspamd_rcl_add_default_handler (sub, - "log_format", - rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_config, log_format_str), - 0, - "Specify format string for the task logging output " - "(https://rspamd.com/doc/configuration/logging.html " - "for details)"); - rspamd_rcl_add_default_handler (sub, - "encryption_key", - rspamd_rcl_parse_struct_pubkey, - G_STRUCT_OFFSET (struct rspamd_config, log_encryption_key), - 0, - "Encrypt sensitive information in logs using this pubkey"); - rspamd_rcl_add_default_handler (sub, - "error_elts", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, log_error_elts), - RSPAMD_CL_FLAG_UINT, - "Size of circular buffer for last errors (10 by default)"); - rspamd_rcl_add_default_handler (sub, - "error_maxlen", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, log_error_elt_maxlen), - RSPAMD_CL_FLAG_UINT, - "Size of each element in error log buffer (1000 by default)"); - - /* Documentation only options, handled in log_handler to map flags */ - rspamd_rcl_add_doc_by_path (cfg, - "logging", - "Enable colored output (for console logging)", - "log_color", - UCL_BOOLEAN, - NULL, - 0, - NULL, - 0); - rspamd_rcl_add_doc_by_path (cfg, - "logging", - "Enable systemd compatible logging", - "systemd", - UCL_BOOLEAN, - NULL, - 0, - NULL, - 0); - rspamd_rcl_add_doc_by_path (cfg, - "logging", - "Write statistics of regexp processing to log (useful for hyperscan)", - "log_re_cache", - UCL_BOOLEAN, - NULL, - 0, - NULL, - 0); - rspamd_rcl_add_doc_by_path (cfg, - "logging", - "Use microseconds resolution for timestamps", - "log_usec", - UCL_BOOLEAN, - NULL, - 0, - NULL, - 0); - /** - * Options section - */ - sub = rspamd_rcl_add_section_doc (&new, - "options", NULL, - rspamd_rcl_options_handler, - UCL_OBJECT, - FALSE, - TRUE, - cfg->doc_strings, - "Global rspamd options"); - rspamd_rcl_add_default_handler (sub, - "cache_file", - rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_config, cache_filename), - RSPAMD_CL_FLAG_STRING_PATH, - "Path to the cache file"); - rspamd_rcl_add_default_handler (sub, - "cache_reload", - rspamd_rcl_parse_struct_time, - G_STRUCT_OFFSET (struct rspamd_config, cache_reload_time), - RSPAMD_CL_FLAG_TIME_FLOAT, - "How often cache reload should be performed"); - /* Old DNS configuration */ - rspamd_rcl_add_default_handler (sub, - "dns_nameserver", - rspamd_rcl_parse_struct_ucl, - G_STRUCT_OFFSET (struct rspamd_config, nameservers), - 0, - "Legacy option for DNS servers used"); - rspamd_rcl_add_default_handler (sub, - "dns_timeout", - rspamd_rcl_parse_struct_time, - G_STRUCT_OFFSET (struct rspamd_config, dns_timeout), - RSPAMD_CL_FLAG_TIME_FLOAT, - "Legacy option for DNS request timeout"); - rspamd_rcl_add_default_handler (sub, - "dns_retransmits", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, dns_retransmits), - RSPAMD_CL_FLAG_INT_32, - "Legacy option for DNS retransmits count"); - rspamd_rcl_add_default_handler (sub, - "dns_sockets", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, dns_io_per_server), - RSPAMD_CL_FLAG_INT_32, - "Legacy option for DNS sockets per server count"); - rspamd_rcl_add_default_handler (sub, - "dns_max_requests", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, dns_max_requests), - RSPAMD_CL_FLAG_INT_32, - "Legacy option for DNS maximum requests per task count"); - rspamd_rcl_add_default_handler (sub, - "classify_headers", - rspamd_rcl_parse_struct_string_list, - G_STRUCT_OFFSET (struct rspamd_config, classify_headers), - 0, - "List of headers used for classifiers"); - rspamd_rcl_add_default_handler (sub, - "control_socket", - rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_config, control_socket_path), - 0, - "Path to the control socket"); - rspamd_rcl_add_default_handler (sub, - "explicit_modules", - rspamd_rcl_parse_struct_string_list, - G_STRUCT_OFFSET (struct rspamd_config, explicit_modules), - RSPAMD_CL_FLAG_STRING_LIST_HASH, - "Always load these modules even if they are not configured explicitly"); - rspamd_rcl_add_default_handler (sub, - "allow_raw_input", - rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_config, allow_raw_input), - 0, - "Allow non MIME input for rspamd"); - rspamd_rcl_add_default_handler (sub, - "raw_mode", - rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_config, raw_mode), - 0, - "Don't try to convert all messages to utf8"); - rspamd_rcl_add_default_handler (sub, - "one_shot", - rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_config, one_shot_mode), - 0, - "Add all symbols only once per message"); - rspamd_rcl_add_default_handler (sub, - "check_attachements", - rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_config, check_text_attachements), - 0, - "Treat text attachments as normal text parts"); - rspamd_rcl_add_default_handler (sub, - "check_attachments", - rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_config, check_text_attachements), - 0, - "Treat text attachments as normal text parts"); - rspamd_rcl_add_default_handler (sub, - "tempdir", - rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_config, temp_dir), - RSPAMD_CL_FLAG_STRING_PATH, - "Directory for temporary files"); - rspamd_rcl_add_default_handler (sub, - "pidfile", - rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_config, pid_file), - RSPAMD_CL_FLAG_STRING_PATH, - "Path to the pid file"); - rspamd_rcl_add_default_handler (sub, - "filters", - rspamd_rcl_parse_struct_string_list, - G_STRUCT_OFFSET (struct rspamd_config, filters), - 0, - "List of internal filters enabled"); - rspamd_rcl_add_default_handler (sub, - "max_diff", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, max_diff), - RSPAMD_CL_FLAG_INT_SIZE, - "Legacy option, do not use"); - rspamd_rcl_add_default_handler (sub, - "map_watch_interval", - rspamd_rcl_parse_struct_time, - G_STRUCT_OFFSET (struct rspamd_config, map_timeout), - RSPAMD_CL_FLAG_TIME_FLOAT, - "Interval for checking maps"); - rspamd_rcl_add_default_handler (sub, - "map_file_watch_multiplier", - rspamd_rcl_parse_struct_double, - G_STRUCT_OFFSET (struct rspamd_config, map_file_watch_multiplier), - 0, - "Multiplier for map watch interval when map is file"); - rspamd_rcl_add_default_handler (sub, - "monitoring_watch_interval", - rspamd_rcl_parse_struct_time, - G_STRUCT_OFFSET (struct rspamd_config, monitored_interval), - RSPAMD_CL_FLAG_TIME_FLOAT, - "Interval for checking monitored instances"); - rspamd_rcl_add_default_handler (sub, - "disable_monitoring", - rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_config, disable_monitored), - 0, - "Disable monitoring completely"); - rspamd_rcl_add_default_handler (sub, - "dynamic_conf", - rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_config, dynamic_conf), - 0, - "Path to the dynamic configuration"); - rspamd_rcl_add_default_handler (sub, - "rrd", - rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_config, rrd_file), - RSPAMD_CL_FLAG_STRING_PATH, - "Path to RRD file"); - rspamd_rcl_add_default_handler (sub, - "history_file", - rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_config, history_file), - RSPAMD_CL_FLAG_STRING_PATH, - "Path to history file"); - rspamd_rcl_add_default_handler (sub, - "check_all_filters", - rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_config, check_all_filters), - 0, - "Always check all filters"); - rspamd_rcl_add_default_handler (sub, - "enable_experimental", - rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_config, enable_experimental), - 0, - "Enable experimental plugins"); - rspamd_rcl_add_default_handler (sub, - "disable_pcre_jit", - rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_config, disable_pcre_jit), - 0, - "Disable PCRE JIT"); - rspamd_rcl_add_default_handler (sub, - "disable_lua_squeeze", - rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_config, disable_lua_squeeze), - 0, - "Disable Lua rules squeezing"); - rspamd_rcl_add_default_handler (sub, - "min_word_len", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, min_word_len), - RSPAMD_CL_FLAG_UINT, - "Minimum length of the word to be considered in statistics/fuzzy"); - rspamd_rcl_add_default_handler (sub, - "max_word_len", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, max_word_len), - RSPAMD_CL_FLAG_UINT, - "Maximum length of the word to be considered in statistics/fuzzy"); - rspamd_rcl_add_default_handler (sub, - "words_decay", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, words_decay), - RSPAMD_CL_FLAG_UINT, - "Start skipping words at this amount"); - rspamd_rcl_add_default_handler (sub, - "url_tld", - rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_config, tld_file), - RSPAMD_CL_FLAG_STRING_PATH, - "Path to the TLD file for urls detector"); - rspamd_rcl_add_default_handler (sub, - "tld", - rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_config, tld_file), - RSPAMD_CL_FLAG_STRING_PATH, - "Path to the TLD file for urls detector"); - rspamd_rcl_add_default_handler (sub, - "hs_cache_dir", - rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_config, hs_cache_dir), - RSPAMD_CL_FLAG_STRING_PATH, - "Path directory where rspamd would save hyperscan cache"); - rspamd_rcl_add_default_handler (sub, - "history_rows", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, history_rows), - RSPAMD_CL_FLAG_UINT, - "Number of records in the history file"); - rspamd_rcl_add_default_handler (sub, - "disable_hyperscan", - rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_config, disable_hyperscan), - 0, - "Disable hyperscan optimizations for regular expressions"); - rspamd_rcl_add_default_handler (sub, - "vectorized_hyperscan", - rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_config, vectorized_hyperscan), - 0, - "Use hyperscan in vectorized mode (experimental)"); - rspamd_rcl_add_default_handler (sub, - "cores_dir", - rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_config, cores_dir), - RSPAMD_CL_FLAG_STRING_PATH, - "Path to the directory where rspamd core files are intended to be dumped"); - rspamd_rcl_add_default_handler (sub, - "max_cores_size", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, max_cores_size), - RSPAMD_CL_FLAG_INT_SIZE, - "Limit of joint size of all files in `cores_dir`"); - rspamd_rcl_add_default_handler (sub, - "max_cores_count", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, max_cores_count), - RSPAMD_CL_FLAG_INT_SIZE, - "Limit of files count in `cores_dir`"); - rspamd_rcl_add_default_handler (sub, - "local_addrs", - rspamd_rcl_parse_struct_ucl, - G_STRUCT_OFFSET (struct rspamd_config, local_addrs), - 0, - "Use the specified addresses as local ones"); - rspamd_rcl_add_default_handler (sub, - "local_networks", - rspamd_rcl_parse_struct_ucl, - G_STRUCT_OFFSET (struct rspamd_config, local_addrs), - 0, - "Use the specified addresses as local ones (alias for `local_addrs`)"); - rspamd_rcl_add_default_handler (sub, - "trusted_keys", - rspamd_rcl_parse_struct_string_list, - G_STRUCT_OFFSET (struct rspamd_config, trusted_keys), - RSPAMD_CL_FLAG_STRING_LIST_HASH, - "List of trusted public keys used for signatures in base32 encoding"); - rspamd_rcl_add_default_handler (sub, - "enable_shutdown_workaround", - rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_config, enable_shutdown_workaround), - 0, - "Enable workaround for legacy clients"); - rspamd_rcl_add_default_handler (sub, - "ignore_received", - rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_config, ignore_received), - 0, - "Ignore data from the first received header"); - rspamd_rcl_add_default_handler (sub, - "ssl_ca_path", - rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_config, ssl_ca_path), - RSPAMD_CL_FLAG_STRING_PATH, - "Path to ssl CA file"); - rspamd_rcl_add_default_handler (sub, - "ssl_ciphers", - rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_config, ssl_ciphers), - 0, - "List of ssl ciphers (e.g. HIGH:!aNULL:!kRSA:!PSK:!SRP:!MD5:!RC4)"); - rspamd_rcl_add_default_handler (sub, - "magic_file", - rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_config, magic_file), - 0, - "Path to a custom libmagic file"); - rspamd_rcl_add_default_handler (sub, - "max_message", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, max_message), - RSPAMD_CL_FLAG_INT_SIZE, - "Maximum size of the message to be scanned (50Mb by default)"); - rspamd_rcl_add_default_handler (sub, - "max_pic", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, max_pic_size), - RSPAMD_CL_FLAG_INT_SIZE, - "Maximum size of the picture to be normalized (1Mb by default)"); - rspamd_rcl_add_default_handler (sub, - "images_cache", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, max_pic_size), - RSPAMD_CL_FLAG_INT_SIZE, - "Size of DCT data cache for images (256 elements by default)"); - rspamd_rcl_add_default_handler (sub, - "zstd_input_dictionary", - rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_config, zstd_input_dictionary), - RSPAMD_CL_FLAG_STRING_PATH, - "Dictionary for zstd inbound protocol compression"); - rspamd_rcl_add_default_handler (sub, - "zstd_output_dictionary", - rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_config, zstd_output_dictionary), - RSPAMD_CL_FLAG_STRING_PATH, - "Dictionary for outbound zstd compression"); - rspamd_rcl_add_default_handler (sub, - "compat_messages", - rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_config, compat_messages), - 0, - "Use pre 1.4 style of messages in the protocol"); - rspamd_rcl_add_default_handler (sub, - "max_shots", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, default_max_shots), - 0, - "Maximum number of hits per a single symbol (default: 100)"); - rspamd_rcl_add_default_handler (sub, - "sessions_cache", - rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_config, enable_sessions_cache), - 0, - "Enable sessions cache to debug dangling sessions"); - rspamd_rcl_add_default_handler (sub, - "max_sessions_cache", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, max_sessions_cache), - 0, - "Maximum number of sessions in cache before warning (default: 100)"); - - /* Neighbours configuration */ - rspamd_rcl_add_section_doc (&sub->subsections, "neighbours", "name", - rspamd_rcl_neighbours_handler, - UCL_OBJECT, FALSE, TRUE, - cfg->doc_strings, - "List of members of Rspamd cluster"); - - /* New DNS configuration */ - ssub = rspamd_rcl_add_section_doc (&sub->subsections, "dns", NULL, NULL, - UCL_OBJECT, FALSE, TRUE, - cfg->doc_strings, - "Options for DNS resolver"); - rspamd_rcl_add_default_handler (ssub, - "nameserver", - rspamd_rcl_parse_struct_ucl, - G_STRUCT_OFFSET (struct rspamd_config, nameservers), - 0, - "List of DNS servers"); - rspamd_rcl_add_default_handler (ssub, - "server", - rspamd_rcl_parse_struct_ucl, - G_STRUCT_OFFSET (struct rspamd_config, nameservers), - 0, - "List of DNS servers"); - rspamd_rcl_add_default_handler (ssub, - "timeout", - rspamd_rcl_parse_struct_time, - G_STRUCT_OFFSET (struct rspamd_config, dns_timeout), - RSPAMD_CL_FLAG_TIME_FLOAT, - "DNS request timeout"); - rspamd_rcl_add_default_handler (ssub, - "retransmits", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, dns_retransmits), - RSPAMD_CL_FLAG_INT_32, - "DNS request retransmits"); - rspamd_rcl_add_default_handler (ssub, - "sockets", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, dns_io_per_server), - RSPAMD_CL_FLAG_INT_32, - "Number of sockets per DNS server"); - rspamd_rcl_add_default_handler (ssub, - "connections", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, dns_io_per_server), - RSPAMD_CL_FLAG_INT_32, - "Number of sockets per DNS server"); - rspamd_rcl_add_default_handler (ssub, - "enable_dnssec", - rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_config, enable_dnssec), - 0, - "Enable DNSSEC support in Rspamd"); - - - /* New upstreams configuration */ - ssub = rspamd_rcl_add_section_doc (&sub->subsections, "upstream", NULL, NULL, - UCL_OBJECT, FALSE, TRUE, - cfg->doc_strings, - "Upstreams configuration parameters"); - rspamd_rcl_add_default_handler (ssub, - "max_errors", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, upstream_max_errors), - RSPAMD_CL_FLAG_UINT, - "Maximum number of errors during `error_time` to consider upstream down"); - rspamd_rcl_add_default_handler (ssub, - "error_time", - rspamd_rcl_parse_struct_time, - G_STRUCT_OFFSET (struct rspamd_config, upstream_error_time), - RSPAMD_CL_FLAG_TIME_FLOAT, - "Time frame to check errors"); - rspamd_rcl_add_default_handler (ssub, - "revive_time", - rspamd_rcl_parse_struct_time, - G_STRUCT_OFFSET (struct rspamd_config, upstream_revive_time), - RSPAMD_CL_FLAG_TIME_FLOAT, - "Time before attempting to recover upstream after an error"); - - /** - * Symbols and actions sections - */ - sub = rspamd_rcl_add_section_doc (&new, - "actions", NULL, - rspamd_rcl_actions_handler, - UCL_OBJECT, - FALSE, - TRUE, - cfg->doc_strings, - "Actions configuration"); - rspamd_rcl_add_default_handler (sub, - "unknown_weight", - rspamd_rcl_parse_struct_double, - G_STRUCT_OFFSET (struct rspamd_config, unknown_weight), - 0, - "Accept unknown symbols with the specified weight"); - rspamd_rcl_add_default_handler (sub, - "grow_factor", - rspamd_rcl_parse_struct_double, - G_STRUCT_OFFSET (struct rspamd_config, grow_factor), - 0, - "Multiply the subsequent symbols by this number " - "(does not affect symbols with score less or " - "equal to zero)"); - rspamd_rcl_add_default_handler (sub, - "subject", - rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_config, subject), - 0, - "Rewrite subject with this value"); - - sub = rspamd_rcl_add_section_doc (&new, - "group", "name", - rspamd_rcl_group_handler, - UCL_OBJECT, - FALSE, - TRUE, - cfg->doc_strings, - "Symbol groups configuration"); - ssub = rspamd_rcl_add_section_doc (&sub->subsections, "symbols", "name", - rspamd_rcl_symbol_handler, - UCL_OBJECT, FALSE, TRUE, - cfg->doc_strings, - "Symbols configuration"); - - /* Group part */ - rspamd_rcl_add_default_handler (sub, - "disabled", - rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_symbols_group, disabled), - 0, - "Disable symbols group"); - rspamd_rcl_add_default_handler (sub, - "enabled", - rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_symbols_group, disabled), - RSPAMD_CL_FLAG_BOOLEAN_INVERSE, - "Enable or disable symbols group"); - rspamd_rcl_add_default_handler (sub, - "max_score", - rspamd_rcl_parse_struct_double, - G_STRUCT_OFFSET (struct rspamd_symbols_group, max_score), - 0, - "Maximum score that could be reached by this symbols group"); - - /** - * Worker section - */ - sub = rspamd_rcl_add_section_doc (&new, - "worker", "type", - rspamd_rcl_worker_handler, - UCL_OBJECT, - FALSE, - TRUE, - cfg->doc_strings, - "Workers common options"); - rspamd_rcl_add_default_handler (sub, - "count", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_worker_conf, count), - RSPAMD_CL_FLAG_INT_16, - "Number of workers to spawn"); - rspamd_rcl_add_default_handler (sub, - "max_files", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_worker_conf, rlimit_nofile), - RSPAMD_CL_FLAG_INT_32, - "Maximum number of opened files per worker"); - rspamd_rcl_add_default_handler (sub, - "max_core", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_worker_conf, rlimit_maxcore), - RSPAMD_CL_FLAG_INT_32, - "Max size of core file in bytes"); - rspamd_rcl_add_default_handler (sub, - "enabled", - rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_worker_conf, enabled), - 0, - "Enable or disable a worker (true by default)"); - - /** - * Modules handler - */ - sub = rspamd_rcl_add_section_doc (&new, - "modules", NULL, - rspamd_rcl_modules_handler, - UCL_OBJECT, - FALSE, - FALSE, - cfg->doc_strings, - "Lua plugins to load"); + if (!(skip_sections && g_hash_table_lookup (skip_sections, "logging"))) { + sub = rspamd_rcl_add_section_doc (&new, + "logging", NULL, + rspamd_rcl_logging_handler, + UCL_OBJECT, + FALSE, + TRUE, + cfg->doc_strings, + "Configure rspamd logging"); + /* Default handlers */ + rspamd_rcl_add_default_handler (sub, + "log_buffer", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, log_buf_size), + RSPAMD_CL_FLAG_INT_32, + "Size of log buffer in bytes (for file logging)"); + rspamd_rcl_add_default_handler (sub, + "log_urls", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_config, log_urls), + 0, + "Write each URL found in a message to the log file"); + rspamd_rcl_add_default_handler (sub, + "debug_ip", + rspamd_rcl_parse_struct_ucl, + G_STRUCT_OFFSET (struct rspamd_config, debug_ip_map), + 0, + "Enable debugging log for the specified IP addresses"); + rspamd_rcl_add_default_handler (sub, + "debug_symbols", + rspamd_rcl_parse_struct_string_list, + G_STRUCT_OFFSET (struct rspamd_config, debug_symbols), + 0, + "Enable debug for the specified symbols"); + rspamd_rcl_add_default_handler (sub, + "debug_modules", + rspamd_rcl_parse_struct_string_list, + G_STRUCT_OFFSET (struct rspamd_config, debug_modules), + RSPAMD_CL_FLAG_STRING_LIST_HASH, + "Enable debugging for the specified modules"); + rspamd_rcl_add_default_handler (sub, + "log_format", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_config, log_format_str), + 0, + "Specify format string for the task logging output " + "(https://rspamd.com/doc/configuration/logging.html " + "for details)"); + rspamd_rcl_add_default_handler (sub, + "encryption_key", + rspamd_rcl_parse_struct_pubkey, + G_STRUCT_OFFSET (struct rspamd_config, log_encryption_key), + 0, + "Encrypt sensitive information in logs using this pubkey"); + rspamd_rcl_add_default_handler (sub, + "error_elts", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, log_error_elts), + RSPAMD_CL_FLAG_UINT, + "Size of circular buffer for last errors (10 by default)"); + rspamd_rcl_add_default_handler (sub, + "error_maxlen", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, log_error_elt_maxlen), + RSPAMD_CL_FLAG_UINT, + "Size of each element in error log buffer (1000 by default)"); + + /* Documentation only options, handled in log_handler to map flags */ + rspamd_rcl_add_doc_by_path (cfg, + "logging", + "Enable colored output (for console logging)", + "log_color", + UCL_BOOLEAN, + NULL, + 0, + NULL, + 0); + rspamd_rcl_add_doc_by_path (cfg, + "logging", + "Enable systemd compatible logging", + "systemd", + UCL_BOOLEAN, + NULL, + 0, + NULL, + 0); + rspamd_rcl_add_doc_by_path (cfg, + "logging", + "Write statistics of regexp processing to log (useful for hyperscan)", + "log_re_cache", + UCL_BOOLEAN, + NULL, + 0, + NULL, + 0); + rspamd_rcl_add_doc_by_path (cfg, + "logging", + "Use microseconds resolution for timestamps", + "log_usec", + UCL_BOOLEAN, + NULL, + 0, + NULL, + 0); + } + if (!(skip_sections && g_hash_table_lookup (skip_sections, "options"))) { + /** + * Options section + */ + sub = rspamd_rcl_add_section_doc (&new, + "options", NULL, + rspamd_rcl_options_handler, + UCL_OBJECT, + FALSE, + TRUE, + cfg->doc_strings, + "Global rspamd options"); + rspamd_rcl_add_default_handler (sub, + "cache_file", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_config, cache_filename), + RSPAMD_CL_FLAG_STRING_PATH, + "Path to the cache file"); + rspamd_rcl_add_default_handler (sub, + "cache_reload", + rspamd_rcl_parse_struct_time, + G_STRUCT_OFFSET (struct rspamd_config, cache_reload_time), + RSPAMD_CL_FLAG_TIME_FLOAT, + "How often cache reload should be performed"); + /* Old DNS configuration */ + rspamd_rcl_add_default_handler (sub, + "dns_nameserver", + rspamd_rcl_parse_struct_ucl, + G_STRUCT_OFFSET (struct rspamd_config, nameservers), + 0, + "Legacy option for DNS servers used"); + rspamd_rcl_add_default_handler (sub, + "dns_timeout", + rspamd_rcl_parse_struct_time, + G_STRUCT_OFFSET (struct rspamd_config, dns_timeout), + RSPAMD_CL_FLAG_TIME_FLOAT, + "Legacy option for DNS request timeout"); + rspamd_rcl_add_default_handler (sub, + "dns_retransmits", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, dns_retransmits), + RSPAMD_CL_FLAG_INT_32, + "Legacy option for DNS retransmits count"); + rspamd_rcl_add_default_handler (sub, + "dns_sockets", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, dns_io_per_server), + RSPAMD_CL_FLAG_INT_32, + "Legacy option for DNS sockets per server count"); + rspamd_rcl_add_default_handler (sub, + "dns_max_requests", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, dns_max_requests), + RSPAMD_CL_FLAG_INT_32, + "Legacy option for DNS maximum requests per task count"); + rspamd_rcl_add_default_handler (sub, + "classify_headers", + rspamd_rcl_parse_struct_string_list, + G_STRUCT_OFFSET (struct rspamd_config, classify_headers), + 0, + "List of headers used for classifiers"); + rspamd_rcl_add_default_handler (sub, + "control_socket", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_config, control_socket_path), + 0, + "Path to the control socket"); + rspamd_rcl_add_default_handler (sub, + "explicit_modules", + rspamd_rcl_parse_struct_string_list, + G_STRUCT_OFFSET (struct rspamd_config, explicit_modules), + RSPAMD_CL_FLAG_STRING_LIST_HASH, + "Always load these modules even if they are not configured explicitly"); + rspamd_rcl_add_default_handler (sub, + "allow_raw_input", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_config, allow_raw_input), + 0, + "Allow non MIME input for rspamd"); + rspamd_rcl_add_default_handler (sub, + "raw_mode", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_config, raw_mode), + 0, + "Don't try to convert all messages to utf8"); + rspamd_rcl_add_default_handler (sub, + "one_shot", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_config, one_shot_mode), + 0, + "Add all symbols only once per message"); + rspamd_rcl_add_default_handler (sub, + "check_attachements", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_config, check_text_attachements), + 0, + "Treat text attachments as normal text parts"); + rspamd_rcl_add_default_handler (sub, + "check_attachments", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_config, check_text_attachements), + 0, + "Treat text attachments as normal text parts"); + rspamd_rcl_add_default_handler (sub, + "tempdir", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_config, temp_dir), + RSPAMD_CL_FLAG_STRING_PATH, + "Directory for temporary files"); + rspamd_rcl_add_default_handler (sub, + "pidfile", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_config, pid_file), + RSPAMD_CL_FLAG_STRING_PATH, + "Path to the pid file"); + rspamd_rcl_add_default_handler (sub, + "filters", + rspamd_rcl_parse_struct_string_list, + G_STRUCT_OFFSET (struct rspamd_config, filters), + 0, + "List of internal filters enabled"); + rspamd_rcl_add_default_handler (sub, + "max_diff", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, max_diff), + RSPAMD_CL_FLAG_INT_SIZE, + "Legacy option, do not use"); + rspamd_rcl_add_default_handler (sub, + "map_watch_interval", + rspamd_rcl_parse_struct_time, + G_STRUCT_OFFSET (struct rspamd_config, map_timeout), + RSPAMD_CL_FLAG_TIME_FLOAT, + "Interval for checking maps"); + rspamd_rcl_add_default_handler (sub, + "map_file_watch_multiplier", + rspamd_rcl_parse_struct_double, + G_STRUCT_OFFSET (struct rspamd_config, map_file_watch_multiplier), + 0, + "Multiplier for map watch interval when map is file"); + rspamd_rcl_add_default_handler (sub, + "monitoring_watch_interval", + rspamd_rcl_parse_struct_time, + G_STRUCT_OFFSET (struct rspamd_config, monitored_interval), + RSPAMD_CL_FLAG_TIME_FLOAT, + "Interval for checking monitored instances"); + rspamd_rcl_add_default_handler (sub, + "disable_monitoring", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_config, disable_monitored), + 0, + "Disable monitoring completely"); + rspamd_rcl_add_default_handler (sub, + "dynamic_conf", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_config, dynamic_conf), + 0, + "Path to the dynamic configuration"); + rspamd_rcl_add_default_handler (sub, + "rrd", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_config, rrd_file), + RSPAMD_CL_FLAG_STRING_PATH, + "Path to RRD file"); + rspamd_rcl_add_default_handler (sub, + "history_file", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_config, history_file), + RSPAMD_CL_FLAG_STRING_PATH, + "Path to history file"); + rspamd_rcl_add_default_handler (sub, + "check_all_filters", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_config, check_all_filters), + 0, + "Always check all filters"); + rspamd_rcl_add_default_handler (sub, + "enable_experimental", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_config, enable_experimental), + 0, + "Enable experimental plugins"); + rspamd_rcl_add_default_handler (sub, + "disable_pcre_jit", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_config, disable_pcre_jit), + 0, + "Disable PCRE JIT"); + rspamd_rcl_add_default_handler (sub, + "disable_lua_squeeze", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_config, disable_lua_squeeze), + 0, + "Disable Lua rules squeezing"); + rspamd_rcl_add_default_handler (sub, + "min_word_len", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, min_word_len), + RSPAMD_CL_FLAG_UINT, + "Minimum length of the word to be considered in statistics/fuzzy"); + rspamd_rcl_add_default_handler (sub, + "max_word_len", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, max_word_len), + RSPAMD_CL_FLAG_UINT, + "Maximum length of the word to be considered in statistics/fuzzy"); + rspamd_rcl_add_default_handler (sub, + "words_decay", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, words_decay), + RSPAMD_CL_FLAG_UINT, + "Start skipping words at this amount"); + rspamd_rcl_add_default_handler (sub, + "url_tld", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_config, tld_file), + RSPAMD_CL_FLAG_STRING_PATH, + "Path to the TLD file for urls detector"); + rspamd_rcl_add_default_handler (sub, + "tld", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_config, tld_file), + RSPAMD_CL_FLAG_STRING_PATH, + "Path to the TLD file for urls detector"); + rspamd_rcl_add_default_handler (sub, + "hs_cache_dir", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_config, hs_cache_dir), + RSPAMD_CL_FLAG_STRING_PATH, + "Path directory where rspamd would save hyperscan cache"); + rspamd_rcl_add_default_handler (sub, + "history_rows", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, history_rows), + RSPAMD_CL_FLAG_UINT, + "Number of records in the history file"); + rspamd_rcl_add_default_handler (sub, + "disable_hyperscan", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_config, disable_hyperscan), + 0, + "Disable hyperscan optimizations for regular expressions"); + rspamd_rcl_add_default_handler (sub, + "vectorized_hyperscan", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_config, vectorized_hyperscan), + 0, + "Use hyperscan in vectorized mode (experimental)"); + rspamd_rcl_add_default_handler (sub, + "cores_dir", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_config, cores_dir), + RSPAMD_CL_FLAG_STRING_PATH, + "Path to the directory where rspamd core files are intended to be dumped"); + rspamd_rcl_add_default_handler (sub, + "max_cores_size", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, max_cores_size), + RSPAMD_CL_FLAG_INT_SIZE, + "Limit of joint size of all files in `cores_dir`"); + rspamd_rcl_add_default_handler (sub, + "max_cores_count", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, max_cores_count), + RSPAMD_CL_FLAG_INT_SIZE, + "Limit of files count in `cores_dir`"); + rspamd_rcl_add_default_handler (sub, + "local_addrs", + rspamd_rcl_parse_struct_ucl, + G_STRUCT_OFFSET (struct rspamd_config, local_addrs), + 0, + "Use the specified addresses as local ones"); + rspamd_rcl_add_default_handler (sub, + "local_networks", + rspamd_rcl_parse_struct_ucl, + G_STRUCT_OFFSET (struct rspamd_config, local_addrs), + 0, + "Use the specified addresses as local ones (alias for `local_addrs`)"); + rspamd_rcl_add_default_handler (sub, + "trusted_keys", + rspamd_rcl_parse_struct_string_list, + G_STRUCT_OFFSET (struct rspamd_config, trusted_keys), + RSPAMD_CL_FLAG_STRING_LIST_HASH, + "List of trusted public keys used for signatures in base32 encoding"); + rspamd_rcl_add_default_handler (sub, + "enable_shutdown_workaround", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_config, enable_shutdown_workaround), + 0, + "Enable workaround for legacy clients"); + rspamd_rcl_add_default_handler (sub, + "ignore_received", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_config, ignore_received), + 0, + "Ignore data from the first received header"); + rspamd_rcl_add_default_handler (sub, + "ssl_ca_path", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_config, ssl_ca_path), + RSPAMD_CL_FLAG_STRING_PATH, + "Path to ssl CA file"); + rspamd_rcl_add_default_handler (sub, + "ssl_ciphers", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_config, ssl_ciphers), + 0, + "List of ssl ciphers (e.g. HIGH:!aNULL:!kRSA:!PSK:!SRP:!MD5:!RC4)"); + rspamd_rcl_add_default_handler (sub, + "magic_file", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_config, magic_file), + 0, + "Path to a custom libmagic file"); + rspamd_rcl_add_default_handler (sub, + "max_message", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, max_message), + RSPAMD_CL_FLAG_INT_SIZE, + "Maximum size of the message to be scanned (50Mb by default)"); + rspamd_rcl_add_default_handler (sub, + "max_pic", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, max_pic_size), + RSPAMD_CL_FLAG_INT_SIZE, + "Maximum size of the picture to be normalized (1Mb by default)"); + rspamd_rcl_add_default_handler (sub, + "images_cache", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, max_pic_size), + RSPAMD_CL_FLAG_INT_SIZE, + "Size of DCT data cache for images (256 elements by default)"); + rspamd_rcl_add_default_handler (sub, + "zstd_input_dictionary", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_config, zstd_input_dictionary), + RSPAMD_CL_FLAG_STRING_PATH, + "Dictionary for zstd inbound protocol compression"); + rspamd_rcl_add_default_handler (sub, + "zstd_output_dictionary", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_config, zstd_output_dictionary), + RSPAMD_CL_FLAG_STRING_PATH, + "Dictionary for outbound zstd compression"); + rspamd_rcl_add_default_handler (sub, + "compat_messages", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_config, compat_messages), + 0, + "Use pre 1.4 style of messages in the protocol"); + rspamd_rcl_add_default_handler (sub, + "max_shots", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, default_max_shots), + 0, + "Maximum number of hits per a single symbol (default: 100)"); + rspamd_rcl_add_default_handler (sub, + "sessions_cache", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_config, enable_sessions_cache), + 0, + "Enable sessions cache to debug dangling sessions"); + rspamd_rcl_add_default_handler (sub, + "max_sessions_cache", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, max_sessions_cache), + 0, + "Maximum number of sessions in cache before warning (default: 100)"); + + /* Neighbours configuration */ + rspamd_rcl_add_section_doc (&sub->subsections, "neighbours", "name", + rspamd_rcl_neighbours_handler, + UCL_OBJECT, FALSE, TRUE, + cfg->doc_strings, + "List of members of Rspamd cluster"); + + /* New DNS configuration */ + ssub = rspamd_rcl_add_section_doc (&sub->subsections, "dns", NULL, NULL, + UCL_OBJECT, FALSE, TRUE, + cfg->doc_strings, + "Options for DNS resolver"); + rspamd_rcl_add_default_handler (ssub, + "nameserver", + rspamd_rcl_parse_struct_ucl, + G_STRUCT_OFFSET (struct rspamd_config, nameservers), + 0, + "List of DNS servers"); + rspamd_rcl_add_default_handler (ssub, + "server", + rspamd_rcl_parse_struct_ucl, + G_STRUCT_OFFSET (struct rspamd_config, nameservers), + 0, + "List of DNS servers"); + rspamd_rcl_add_default_handler (ssub, + "timeout", + rspamd_rcl_parse_struct_time, + G_STRUCT_OFFSET (struct rspamd_config, dns_timeout), + RSPAMD_CL_FLAG_TIME_FLOAT, + "DNS request timeout"); + rspamd_rcl_add_default_handler (ssub, + "retransmits", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, dns_retransmits), + RSPAMD_CL_FLAG_INT_32, + "DNS request retransmits"); + rspamd_rcl_add_default_handler (ssub, + "sockets", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, dns_io_per_server), + RSPAMD_CL_FLAG_INT_32, + "Number of sockets per DNS server"); + rspamd_rcl_add_default_handler (ssub, + "connections", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, dns_io_per_server), + RSPAMD_CL_FLAG_INT_32, + "Number of sockets per DNS server"); + rspamd_rcl_add_default_handler (ssub, + "enable_dnssec", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_config, enable_dnssec), + 0, + "Enable DNSSEC support in Rspamd"); + + + /* New upstreams configuration */ + ssub = rspamd_rcl_add_section_doc (&sub->subsections, "upstream", NULL, NULL, + UCL_OBJECT, FALSE, TRUE, + cfg->doc_strings, + "Upstreams configuration parameters"); + rspamd_rcl_add_default_handler (ssub, + "max_errors", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, upstream_max_errors), + RSPAMD_CL_FLAG_UINT, + "Maximum number of errors during `error_time` to consider upstream down"); + rspamd_rcl_add_default_handler (ssub, + "error_time", + rspamd_rcl_parse_struct_time, + G_STRUCT_OFFSET (struct rspamd_config, upstream_error_time), + RSPAMD_CL_FLAG_TIME_FLOAT, + "Time frame to check errors"); + rspamd_rcl_add_default_handler (ssub, + "revive_time", + rspamd_rcl_parse_struct_time, + G_STRUCT_OFFSET (struct rspamd_config, upstream_revive_time), + RSPAMD_CL_FLAG_TIME_FLOAT, + "Time before attempting to recover upstream after an error"); + } + + if (!(skip_sections && g_hash_table_lookup (skip_sections, "actions"))) { + /** + * Symbols and actions sections + */ + sub = rspamd_rcl_add_section_doc (&new, + "actions", NULL, + rspamd_rcl_actions_handler, + UCL_OBJECT, + FALSE, + TRUE, + cfg->doc_strings, + "Actions configuration"); + rspamd_rcl_add_default_handler (sub, + "unknown_weight", + rspamd_rcl_parse_struct_double, + G_STRUCT_OFFSET (struct rspamd_config, unknown_weight), + 0, + "Accept unknown symbols with the specified weight"); + rspamd_rcl_add_default_handler (sub, + "grow_factor", + rspamd_rcl_parse_struct_double, + G_STRUCT_OFFSET (struct rspamd_config, grow_factor), + 0, + "Multiply the subsequent symbols by this number " + "(does not affect symbols with score less or " + "equal to zero)"); + rspamd_rcl_add_default_handler (sub, + "subject", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_config, subject), + 0, + "Rewrite subject with this value"); + } + + if (!(skip_sections && g_hash_table_lookup (skip_sections, "group"))) { + sub = rspamd_rcl_add_section_doc (&new, + "group", "name", + rspamd_rcl_group_handler, + UCL_OBJECT, + FALSE, + TRUE, + cfg->doc_strings, + "Symbol groups configuration"); + ssub = rspamd_rcl_add_section_doc (&sub->subsections, "symbols", "name", + rspamd_rcl_symbol_handler, + UCL_OBJECT, FALSE, TRUE, + cfg->doc_strings, + "Symbols configuration"); + + /* Group part */ + rspamd_rcl_add_default_handler (sub, + "disabled", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_symbols_group, disabled), + 0, + "Disable symbols group"); + rspamd_rcl_add_default_handler (sub, + "enabled", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_symbols_group, disabled), + RSPAMD_CL_FLAG_BOOLEAN_INVERSE, + "Enable or disable symbols group"); + rspamd_rcl_add_default_handler (sub, + "max_score", + rspamd_rcl_parse_struct_double, + G_STRUCT_OFFSET (struct rspamd_symbols_group, max_score), + 0, + "Maximum score that could be reached by this symbols group"); + } - /** - * Classifiers handler - */ - sub = rspamd_rcl_add_section_doc (&new, - "classifier", "type", - rspamd_rcl_classifier_handler, - UCL_OBJECT, - FALSE, - TRUE, - cfg->doc_strings, - "CLassifier options"); - /* Default classifier is 'bayes' for now */ - sub->default_key = "bayes"; - - rspamd_rcl_add_default_handler (sub, - "min_tokens", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_classifier_config, min_tokens), - RSPAMD_CL_FLAG_INT_32, - "Minimum count of tokens (words) to be considered for statistics"); - rspamd_rcl_add_default_handler (sub, - "max_tokens", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_classifier_config, max_tokens), - RSPAMD_CL_FLAG_INT_32, - "Maximum count of tokens (words) to be considered for statistics"); - rspamd_rcl_add_default_handler (sub, - "min_learns", - rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_classifier_config, min_learns), - RSPAMD_CL_FLAG_UINT, - "Minimum number of learns for each statfile to use this classifier"); - rspamd_rcl_add_default_handler (sub, - "backend", - rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_classifier_config, backend), - 0, - "Statfiles engine"); - rspamd_rcl_add_default_handler (sub, - "name", - rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_classifier_config, name), - 0, - "Name of classifier"); + if (!(skip_sections && g_hash_table_lookup (skip_sections, "worker"))) { + /** + * Worker section + */ + sub = rspamd_rcl_add_section_doc (&new, + "worker", "type", + rspamd_rcl_worker_handler, + UCL_OBJECT, + FALSE, + TRUE, + cfg->doc_strings, + "Workers common options"); + rspamd_rcl_add_default_handler (sub, + "count", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_worker_conf, count), + RSPAMD_CL_FLAG_INT_16, + "Number of workers to spawn"); + rspamd_rcl_add_default_handler (sub, + "max_files", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_worker_conf, rlimit_nofile), + RSPAMD_CL_FLAG_INT_32, + "Maximum number of opened files per worker"); + rspamd_rcl_add_default_handler (sub, + "max_core", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_worker_conf, rlimit_maxcore), + RSPAMD_CL_FLAG_INT_32, + "Max size of core file in bytes"); + rspamd_rcl_add_default_handler (sub, + "enabled", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_worker_conf, enabled), + 0, + "Enable or disable a worker (true by default)"); + } - /* - * Statfile defaults - */ - ssub = rspamd_rcl_add_section_doc (&sub->subsections, - "statfile", "symbol", - rspamd_rcl_statfile_handler, - UCL_OBJECT, - TRUE, - TRUE, - sub->doc_ref, - "Statfiles options"); - rspamd_rcl_add_default_handler (ssub, - "label", - rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_statfile_config, label), - 0, - "Statfile unique label"); - rspamd_rcl_add_default_handler (ssub, - "spam", - rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_statfile_config, is_spam), - 0, - "Sets if this statfile contains spam samples"); + if (!(skip_sections && g_hash_table_lookup (skip_sections, "modules"))) { + /** + * Modules handler + */ + sub = rspamd_rcl_add_section_doc (&new, + "modules", NULL, + rspamd_rcl_modules_handler, + UCL_OBJECT, + FALSE, + FALSE, + cfg->doc_strings, + "Lua plugins to load"); + } + + if (!(skip_sections && g_hash_table_lookup (skip_sections, "classifier"))) { + /** + * Classifiers handler + */ + sub = rspamd_rcl_add_section_doc (&new, + "classifier", "type", + rspamd_rcl_classifier_handler, + UCL_OBJECT, + FALSE, + TRUE, + cfg->doc_strings, + "CLassifier options"); + /* Default classifier is 'bayes' for now */ + sub->default_key = "bayes"; + + rspamd_rcl_add_default_handler (sub, + "min_tokens", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_classifier_config, min_tokens), + RSPAMD_CL_FLAG_INT_32, + "Minimum count of tokens (words) to be considered for statistics"); + rspamd_rcl_add_default_handler (sub, + "max_tokens", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_classifier_config, max_tokens), + RSPAMD_CL_FLAG_INT_32, + "Maximum count of tokens (words) to be considered for statistics"); + rspamd_rcl_add_default_handler (sub, + "min_learns", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_classifier_config, min_learns), + RSPAMD_CL_FLAG_UINT, + "Minimum number of learns for each statfile to use this classifier"); + rspamd_rcl_add_default_handler (sub, + "backend", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_classifier_config, backend), + 0, + "Statfiles engine"); + rspamd_rcl_add_default_handler (sub, + "name", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_classifier_config, name), + 0, + "Name of classifier"); - /** - * Composites handlers - */ - sub = rspamd_rcl_add_section_doc (&new, - "composite", "name", - rspamd_rcl_composite_handler, - UCL_OBJECT, - FALSE, - TRUE, - cfg->doc_strings, - "Rspamd composite symbols"); - sub = rspamd_rcl_add_section_doc (&new, - "composites", NULL, - rspamd_rcl_composites_handler, - UCL_OBJECT, - FALSE, - TRUE, - cfg->doc_strings, - "Rspamd composite symbols"); + /* + * Statfile defaults + */ + ssub = rspamd_rcl_add_section_doc (&sub->subsections, + "statfile", "symbol", + rspamd_rcl_statfile_handler, + UCL_OBJECT, + TRUE, + TRUE, + sub->doc_ref, + "Statfiles options"); + rspamd_rcl_add_default_handler (ssub, + "label", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_statfile_config, label), + 0, + "Statfile unique label"); + rspamd_rcl_add_default_handler (ssub, + "spam", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_statfile_config, is_spam), + 0, + "Sets if this statfile contains spam samples"); + } - /** - * Lua handler - */ - sub = rspamd_rcl_add_section_doc (&new, - "lua", NULL, - rspamd_rcl_lua_handler, - UCL_STRING, - FALSE, - TRUE, - cfg->doc_strings, - "Lua files to load"); + if (!(skip_sections && g_hash_table_lookup (skip_sections, "composite"))) { + /** + * Composites handlers + */ + sub = rspamd_rcl_add_section_doc (&new, + "composite", "name", + rspamd_rcl_composite_handler, + UCL_OBJECT, + FALSE, + TRUE, + cfg->doc_strings, + "Rspamd composite symbols"); + sub = rspamd_rcl_add_section_doc (&new, + "composites", NULL, + rspamd_rcl_composites_handler, + UCL_OBJECT, + FALSE, + TRUE, + cfg->doc_strings, + "Rspamd composite symbols"); + } + + if (!(skip_sections && g_hash_table_lookup (skip_sections, "lua"))) { + /** + * Lua handler + */ + sub = rspamd_rcl_add_section_doc (&new, + "lua", NULL, + rspamd_rcl_lua_handler, + UCL_STRING, + FALSE, + TRUE, + cfg->doc_strings, + "Lua files to load"); + } return new; } @@ -3518,7 +3204,7 @@ rspamd_rcl_emitter_append_double (double elt, void *ud) return 0; } -static void +void rspamd_rcl_section_free (gpointer p) { struct rspamd_rcl_section *top = p, *cur, *tmp; @@ -3561,7 +3247,7 @@ rspamd_rcl_section_free (gpointer p) * it is changed, then rcl_obj is imported from lua. Old config is dereferenced. * @param cfg */ -static void +void rspamd_rcl_maybe_apply_lua_transform (struct rspamd_config *cfg) { lua_State *L = cfg->lua_state; @@ -3650,37 +3336,56 @@ rspamd_rcl_decrypt_free (unsigned char *data, size_t len, void *user_data) g_free (data); } +void +rspamd_config_calculate_cksum (struct rspamd_config *cfg) +{ + rspamd_cryptobox_hash_state_t hs; + unsigned char cksumbuf[rspamd_cryptobox_HASHBYTES]; + struct ucl_emitter_functions f; + + /* Calculate checksum */ + rspamd_cryptobox_hash_init (&hs, NULL, 0); + f.ucl_emitter_append_character = rspamd_rcl_emitter_append_c; + f.ucl_emitter_append_double = rspamd_rcl_emitter_append_double; + f.ucl_emitter_append_int = rspamd_rcl_emitter_append_int; + f.ucl_emitter_append_len = rspamd_rcl_emitter_append_len; + f.ucl_emitter_free_func = NULL; + f.ud = &hs; + ucl_object_emit_full (cfg->rcl_obj, UCL_EMIT_MSGPACK, + &f, cfg->config_comments); + rspamd_cryptobox_hash_final (&hs, cksumbuf); + cfg->checksum = rspamd_encode_base32 (cksumbuf, sizeof (cksumbuf)); + /* Also change the tag of cfg pool to be equal to the checksum */ + rspamd_strlcpy (cfg->cfg_pool->tag.uid, cfg->checksum, + MIN (sizeof (cfg->cfg_pool->tag.uid), strlen (cfg->checksum))); +} + gboolean -rspamd_config_read (struct rspamd_config *cfg, const gchar *filename, - const gchar *convert_to, rspamd_rcl_section_fin_t logger_fin, - gpointer logger_ud, GHashTable *vars) +rspamd_config_parse_ucl (struct rspamd_config *cfg, const gchar *filename, + GHashTable *vars, GError **err) { struct stat st; gint fd; - gchar *data; - GError *err = NULL; - struct rspamd_rcl_section *top, *logger_section; struct ucl_parser *parser; - const ucl_object_t *logger_obj; - rspamd_cryptobox_hash_state_t hs; - unsigned char cksumbuf[rspamd_cryptobox_HASHBYTES]; gchar keypair_path[PATH_MAX]; struct rspamd_cryptobox_keypair *decrypt_keypair = NULL; - struct ucl_emitter_functions f; + gchar *data; if (stat (filename, &st) == -1) { - msg_err_config_forced ("cannot stat %s: %s", filename, strerror (errno)); + g_set_error (err, cfg_rcl_error_quark (), errno, + "cannot stat %s: %s", filename, strerror (errno)); return FALSE; } if ((fd = open (filename, O_RDONLY)) == -1) { - msg_err_config_forced ("cannot open %s: %s", filename, strerror (errno)); + g_set_error (err, cfg_rcl_error_quark (), errno, + "cannot open %s: %s", filename, strerror (errno)); return FALSE; } /* Now mmap this file to simplify reading process */ - if ((data = - mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) { - msg_err_config_forced ("cannot mmap %s: %s", filename, strerror (errno)); + if ((data = mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) { + g_set_error (err, cfg_rcl_error_quark (), errno, + "cannot mmap %s: %s", filename, strerror (errno)); close (fd); return FALSE; } @@ -3725,7 +3430,6 @@ rspamd_config_read (struct rspamd_config *cfg, const gchar *filename, ucl_parser_free (kp_parser); } - rspamd_cryptobox_hash_init (&hs, NULL, 0); parser = ucl_parser_new (UCL_PARSER_SAVE_COMMENTS); rspamd_ucl_add_conf_variables (parser, vars); rspamd_ucl_add_conf_macros (parser, cfg); @@ -3756,9 +3460,28 @@ rspamd_config_read (struct rspamd_config *cfg, const gchar *filename, cfg->config_comments = ucl_object_ref (ucl_parser_get_comments (parser)); ucl_parser_free (parser); - top = rspamd_rcl_config_init (cfg); + return TRUE; +} + +gboolean +rspamd_config_read (struct rspamd_config *cfg, const gchar *filename, + rspamd_rcl_section_fin_t logger_fin, + gpointer logger_ud, GHashTable *vars) +{ + GError *err = NULL; + struct rspamd_rcl_section *top, *logger_section; + const ucl_object_t *logger_obj; + + if (!rspamd_config_parse_ucl (cfg, filename, vars, &err)) { + msg_err_config_forced ("failed to load config: %e", err); + g_error_free (err); + + return FALSE; + } + + top = rspamd_rcl_config_init (cfg, NULL); rspamd_lua_set_path (cfg->lua_state, cfg->rcl_obj, vars); - rspamd_rcl_set_lua_globals (cfg, cfg->lua_state, vars); + rspamd_lua_set_globals (cfg, cfg->lua_state, vars); rspamd_mempool_add_destructor (cfg->cfg_pool, rspamd_rcl_section_free, top); err = NULL; @@ -3790,22 +3513,7 @@ rspamd_config_read (struct rspamd_config *cfg, const gchar *filename, /* Transform config if needed */ rspamd_rcl_maybe_apply_lua_transform (cfg); - - /* Calculate checksum */ - f.ucl_emitter_append_character = rspamd_rcl_emitter_append_c; - f.ucl_emitter_append_double = rspamd_rcl_emitter_append_double; - f.ucl_emitter_append_int = rspamd_rcl_emitter_append_int; - f.ucl_emitter_append_len = rspamd_rcl_emitter_append_len; - f.ucl_emitter_free_func = NULL; - f.ud = &hs; - ucl_object_emit_full (cfg->rcl_obj, UCL_EMIT_MSGPACK, - &f, cfg->config_comments); - rspamd_cryptobox_hash_final (&hs, cksumbuf); - cfg->checksum = rspamd_encode_base32 (cksumbuf, sizeof (cksumbuf)); - /* Also change the tag of cfg pool to be equal to the checksum */ - rspamd_strlcpy (cfg->cfg_pool->tag.uid, cfg->checksum, - MIN (sizeof (cfg->cfg_pool->tag.uid), strlen (cfg->checksum))); - + rspamd_config_calculate_cksum (cfg); if (!rspamd_rcl_parse (top, cfg, cfg, cfg->cfg_pool, cfg->rcl_obj, &err)) { msg_err_config ("rcl parse error: %e", err); diff --git a/src/libserver/cfg_rcl.h b/src/libserver/cfg_rcl.h index c1d958efc..b14b98f8b 100644 --- a/src/libserver/cfg_rcl.h +++ b/src/libserver/cfg_rcl.h @@ -134,7 +134,8 @@ struct rspamd_rcl_section *rspamd_rcl_add_section_doc ( * Init common sections known to rspamd * @return top section */ -struct rspamd_rcl_section * rspamd_rcl_config_init (struct rspamd_config *cfg); +struct rspamd_rcl_section * rspamd_rcl_config_init (struct rspamd_config *cfg, + GHashTable *skip_sections); /** * Get a section specified by path, it understand paths separated by '/' character @@ -446,11 +447,41 @@ gboolean rspamd_rcl_add_lua_plugins_path (struct rspamd_config *cfg, const gchar *path, GError **err); + +/** + * Calls for an external lua function to apply potential config transformations + * if needed. This function can change the cfg->rcl_obj. + * + * Example of transformation function: + * + * function(obj) + * if obj.something == 'foo' then + * obj.something = "bla" + * return true, obj + * end + * + * return false, nil + * end + * + * If function returns 'false' then rcl_obj is not touched. Otherwise, + * it is changed, then rcl_obj is imported from lua. Old config is dereferenced. + * @param cfg + */ +void rspamd_rcl_maybe_apply_lua_transform (struct rspamd_config *cfg); +void rspamd_rcl_section_free (gpointer p); + +void rspamd_config_calculate_cksum (struct rspamd_config *cfg); + /* * Read configuration file */ +gboolean rspamd_config_parse_ucl (struct rspamd_config *cfg, + const gchar *filename, + GHashTable *vars, + GError **err); gboolean rspamd_config_read (struct rspamd_config *cfg, - const gchar *filename, const gchar *convert_to, - rspamd_rcl_section_fin_t logger_fin, gpointer logger_ud, - GHashTable *vars); + const gchar *filename, + rspamd_rcl_section_fin_t logger_fin, + gpointer logger_ud, + GHashTable *vars); #endif /* CFG_RCL_H_ */ diff --git a/src/libserver/cfg_utils.c b/src/libserver/cfg_utils.c index bf63b2188..e58232a00 100644 --- a/src/libserver/cfg_utils.c +++ b/src/libserver/cfg_utils.c @@ -114,7 +114,7 @@ rspamd_parse_bind_line (struct rspamd_config *cfg, } struct rspamd_config * -rspamd_config_new (void) +rspamd_config_new (enum rspamd_config_init_flags flags) { struct rspamd_config *cfg; @@ -172,7 +172,11 @@ rspamd_config_new (void) cfg->min_word_len = DEFAULT_MIN_WORD; cfg->max_word_len = DEFAULT_MAX_WORD; - cfg->lua_state = rspamd_lua_init (); + if (!(flags & RSPAMD_CONFIG_INIT_SKIP_LUA)) { + cfg->lua_state = rspamd_lua_init (); + cfg->own_lua_state = TRUE; + } + cfg->cache = rspamd_symbols_cache_new (cfg); cfg->ups_ctx = rspamd_upstreams_library_init (); cfg->re_cache = rspamd_re_cache_new (); @@ -251,7 +255,10 @@ rspamd_config_free (struct rspamd_config *cfg) rspamd_re_cache_unref (cfg->re_cache); rspamd_upstreams_library_unref (cfg->ups_ctx); rspamd_mempool_delete (cfg->cfg_pool); - lua_close (cfg->lua_state); + + if (cfg->lua_state && cfg->own_lua_state) { + lua_close (cfg->lua_state); + } REF_RELEASE (cfg->libs_ctx); DL_FOREACH_SAFE (cfg->log_pipes, lp, ltmp) { @@ -1105,12 +1112,15 @@ rspamd_include_map_handler (const guchar *data, gsize len, #define RSPAMD_VERSION_MINOR_MACRO "VERSION_MINOR" #define RSPAMD_VERSION_PATCH_MACRO "VERSION_PATCH" #define RSPAMD_BRANCH_VERSION_MACRO "BRANCH_VERSION" +#define RSPAMD_HOSTNAME_MACRO "HOSTNAME" void rspamd_ucl_add_conf_variables (struct ucl_parser *parser, GHashTable *vars) { GHashTableIter it; gpointer k, v; + gchar *hostbuf; + gsize hostlen; ucl_parser_register_variable (parser, RSPAMD_CONFDIR_MACRO, @@ -1152,6 +1162,23 @@ rspamd_ucl_add_conf_variables (struct ucl_parser *parser, GHashTable *vars) "no"); #endif + hostlen = sysconf (_SC_HOST_NAME_MAX); + + if (hostlen <= 0) { + hostlen = 256; + } + else { + hostlen ++; + } + + hostbuf = g_alloca (hostlen); + memset (hostbuf, 0, hostlen); + gethostname (hostbuf, hostlen - 1); + + /* UCL copies variables, so it is safe to pass an ephemeral buffer here */ + ucl_parser_register_variable (parser, RSPAMD_HOSTNAME_MACRO, + hostbuf); + if (vars != NULL) { g_hash_table_iter_init (&it, vars); diff --git a/src/libserver/composites.c b/src/libserver/composites.c index a90b5c032..f3dc7a956 100644 --- a/src/libserver/composites.c +++ b/src/libserver/composites.c @@ -266,7 +266,7 @@ rspamd_composite_expr_process (gpointer input, rspamd_expression_atom_t *atom) t = *beg; if (t == '~') { - nrd->action &= ~RSPAMD_COMPOSITE_REMOVE_WEIGHT; + nrd->action &= ~RSPAMD_COMPOSITE_REMOVE_SYMBOL; } else if (t == '-') { nrd->action &= ~(RSPAMD_COMPOSITE_REMOVE_WEIGHT| diff --git a/src/libserver/dkim.c b/src/libserver/dkim.c index 5af61daf7..4aaa74a3b 100644 --- a/src/libserver/dkim.c +++ b/src/libserver/dkim.c @@ -65,12 +65,14 @@ enum rspamd_sign_type { DKIM_SIGN_RSASHA256, DKIM_SIGN_RSASHA512, DKIM_SIGN_ECDSASHA256, - DKIM_SIGN_ECDSASHA512 + DKIM_SIGN_ECDSASHA512, + DKIM_SIGN_EDDSASHA256, }; enum rspamd_dkim_key_type { RSPAMD_DKIM_KEY_RSA = 0, - RSPAMD_DKIM_KEY_ECDSA + RSPAMD_DKIM_KEY_ECDSA, + RSPAMD_DKIM_KEY_EDDSA }; #define RSPAMD_DKIM_MAX_ARC_IDX 10 @@ -152,6 +154,7 @@ struct rspamd_dkim_key_s { union { RSA *key_rsa; EC_KEY *key_ecdsa; + guchar *key_eddsa; } key; enum rspamd_dkim_key_type type; BIO *key_bio; @@ -315,6 +318,12 @@ rspamd_dkim_parse_signalg (rspamd_dkim_context_t * ctx, return TRUE; } } + else if (len == sizeof ("ed25519") - 1) { + if (memcmp (param, "ed25519", len) == 0) { + ctx->sig_alg = DKIM_SIGN_EDDSASHA256; + return TRUE; + } + } g_set_error (err, DKIM_ERROR, @@ -1219,7 +1228,10 @@ rspamd_dkim_make_key (rspamd_dkim_context_t *ctx, const gchar *keydata, rspamd_dkim_key_t *key = NULL; if (keylen < 3) { - msg_err_dkim ("DKIM key is too short to be valid"); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_KEYFAIL, + "DKIM key is too short to be valid"); return NULL; } @@ -1233,55 +1245,71 @@ rspamd_dkim_make_key (rspamd_dkim_context_t *ctx, const gchar *keydata, rspamd_cryptobox_base64_decode (keydata, keylen, key->keydata, &key->decoded_len); - key->key_bio = BIO_new_mem_buf (key->keydata, key->decoded_len); + if (key->type == RSPAMD_DKIM_KEY_EDDSA) { + key->key.key_eddsa = key->keydata; - if (key->key_bio == NULL) { - g_set_error (err, - DKIM_ERROR, - DKIM_SIGERROR_KEYFAIL, - "cannot make ssl bio from key"); - REF_RELEASE (key); - - return NULL; - } - - key->key_evp = d2i_PUBKEY_bio (key->key_bio, NULL); - - if (key->key_evp == NULL) { - g_set_error (err, - DKIM_ERROR, - DKIM_SIGERROR_KEYFAIL, - "cannot extract pubkey from bio"); - REF_RELEASE (key); + if (key->decoded_len != rspamd_cryptobox_pk_sig_bytes ( + RSPAMD_CRYPTOBOX_MODE_25519)) { + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_KEYFAIL, + "DKIM key is has invalid length %d for eddsa", + (gint)key->decoded_len); + REF_RELEASE (key); - return NULL; + return NULL; + } } + else { + key->key_bio = BIO_new_mem_buf (key->keydata, key->decoded_len); - if (type == RSPAMD_DKIM_KEY_RSA) { - key->key.key_rsa = EVP_PKEY_get1_RSA (key->key_evp); - - if (key->key.key_rsa == NULL) { + if (key->key_bio == NULL) { g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_KEYFAIL, - "cannot extract rsa key from evp key"); + "cannot make ssl bio from key"); REF_RELEASE (key); return NULL; } - } - else { - key->key.key_ecdsa = EVP_PKEY_get1_EC_KEY (key->key_evp); - if (key->key.key_ecdsa == NULL) { + key->key_evp = d2i_PUBKEY_bio (key->key_bio, NULL); + + if (key->key_evp == NULL) { g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_KEYFAIL, - "cannot extract ecdsa key from evp key"); + "cannot extract pubkey from bio"); REF_RELEASE (key); return NULL; } + + if (type == RSPAMD_DKIM_KEY_RSA) { + key->key.key_rsa = EVP_PKEY_get1_RSA (key->key_evp); + + if (key->key.key_rsa == NULL) { + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_KEYFAIL, + "cannot extract rsa key from evp key"); + REF_RELEASE (key); + + return NULL; + } + } else { + key->key.key_ecdsa = EVP_PKEY_get1_EC_KEY (key->key_evp); + + if (key->key.key_ecdsa == NULL) { + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_KEYFAIL, + "cannot extract ecdsa key from evp key"); + REF_RELEASE (key); + + return NULL; + } + } } return key; @@ -1303,11 +1331,12 @@ rspamd_dkim_key_free (rspamd_dkim_key_t *key) RSA_free (key->key.key_rsa); } } - else { + else if (key->type == RSPAMD_DKIM_KEY_ECDSA) { if (key->key.key_ecdsa) { EC_KEY_free (key->key.key_ecdsa); } } + /* Nothing in case of eddsa key */ if (key->key_bio) { BIO_free (key->key_bio); } @@ -1433,14 +1462,18 @@ rspamd_dkim_parse_key (rspamd_dkim_context_t *ctx, const gchar *txt, alglen = 3; } - if (alglen == 8 && rspamd_lc_cmp (alg, "ecdsa256", alglen) == 0) { - if (keylen) { - *keylen = klen; - } + if (keylen) { + *keylen = klen; + } + if (alglen == 8 && rspamd_lc_cmp (alg, "ecdsa256", alglen) == 0) { return rspamd_dkim_make_key (ctx, c, klen, RSPAMD_DKIM_KEY_ECDSA, err); } + else if (alglen == 7 && rspamd_lc_cmp (alg, "ed25519", alglen) == 0) { + return rspamd_dkim_make_key (ctx, c, klen, + RSPAMD_DKIM_KEY_EDDSA, err); + } else { /* We assume RSA default in all cases */ if (alglen != 3 || rspamd_lc_cmp (alg, "rsa", alglen) != 0) { @@ -2419,7 +2452,8 @@ rspamd_dkim_check (rspamd_dkim_context_t *ctx, nid = NID_sha1; } else if (ctx->sig_alg == DKIM_SIGN_RSASHA256 || - ctx->sig_alg == DKIM_SIGN_ECDSASHA256) { + ctx->sig_alg == DKIM_SIGN_ECDSASHA256 || + ctx->sig_alg == DKIM_SIGN_EDDSASHA256) { nid = NID_sha256; } else if (ctx->sig_alg == DKIM_SIGN_RSASHA512 || @@ -2431,21 +2465,31 @@ rspamd_dkim_check (rspamd_dkim_context_t *ctx, nid = NID_sha1; } - if (key->type == RSPAMD_DKIM_KEY_RSA) { + switch (key->type) { + case RSPAMD_DKIM_KEY_RSA: if (RSA_verify (nid, raw_digest, dlen, ctx->b, ctx->blen, key->key.key_rsa) != 1) { msg_debug_dkim ("rsa verify failed"); res = DKIM_REJECT; } - } - else { + break; + case RSPAMD_DKIM_KEY_ECDSA: if (ECDSA_verify (nid, raw_digest, dlen, ctx->b, ctx->blen, key->key.key_ecdsa) != 1) { msg_debug_dkim ("ecdsa verify failed"); res = DKIM_REJECT; } + break; + case RSPAMD_DKIM_KEY_EDDSA: + if (!rspamd_cryptobox_verify (ctx->b, ctx->blen, raw_digest, dlen, + key->key.key_eddsa, RSPAMD_CRYPTOBOX_MODE_25519)) { + msg_debug_dkim ("eddsa verify failed"); + res = DKIM_REJECT; + } + break; } + if (ctx->common.type == RSPAMD_DKIM_ARC_SEAL && res == DKIM_CONTINUE) { switch (ctx->cv) { case RSPAMD_ARC_INVALID: diff --git a/src/libserver/symbols_cache.c b/src/libserver/symbols_cache.c index abed26fc3..23ad15ed1 100644 --- a/src/libserver/symbols_cache.c +++ b/src/libserver/symbols_cache.c @@ -2037,7 +2037,7 @@ rspamd_symbols_cache_counters (struct symbols_cache * cache) top = ucl_object_typed_new (UCL_ARRAY); cbd.top = top; cbd.cache = cache; - g_ptr_array_foreach (cache->items_by_order->d, + g_ptr_array_foreach (cache->items_by_id, rspamd_symbols_cache_counters_cb, &cbd); return top; diff --git a/src/libserver/task.c b/src/libserver/task.c index b5594816b..9be780b1b 100644 --- a/src/libserver/task.c +++ b/src/libserver/task.c @@ -738,6 +738,12 @@ rspamd_task_process (struct rspamd_task *task, guint stages) RSPAMD_TASK_STAGE_PRE_FILTERS); break; + case RSPAMD_TASK_STAGE_PROCESS_MESSAGE: + if (!(task->flags & RSPAMD_TASK_FLAG_SKIP_PROCESS)) { + rspamd_message_process (task); + } + break; + case RSPAMD_TASK_STAGE_FILTERS: rspamd_symbols_cache_process_symbols (task, task->cfg->cache, RSPAMD_TASK_STAGE_FILTERS); diff --git a/src/libserver/task.h b/src/libserver/task.h index e3c0492f6..b9fd2f811 100644 --- a/src/libserver/task.h +++ b/src/libserver/task.h @@ -41,25 +41,27 @@ enum rspamd_task_stage { RSPAMD_TASK_STAGE_ENVELOPE = (1 << 1), RSPAMD_TASK_STAGE_READ_MESSAGE = (1 << 2), RSPAMD_TASK_STAGE_PRE_FILTERS = (1 << 3), - RSPAMD_TASK_STAGE_FILTERS = (1 << 4), - RSPAMD_TASK_STAGE_CLASSIFIERS_PRE = (1 << 5), - RSPAMD_TASK_STAGE_CLASSIFIERS = (1 << 6), - RSPAMD_TASK_STAGE_CLASSIFIERS_POST = (1 << 7), - RSPAMD_TASK_STAGE_COMPOSITES = (1 << 8), - RSPAMD_TASK_STAGE_POST_FILTERS = (1 << 9), - RSPAMD_TASK_STAGE_LEARN_PRE = (1 << 10), - RSPAMD_TASK_STAGE_LEARN = (1 << 11), - RSPAMD_TASK_STAGE_LEARN_POST = (1 << 12), - RSPAMD_TASK_STAGE_COMPOSITES_POST = (1 << 13), - RSPAMD_TASK_STAGE_IDEMPOTENT = (1 << 14), - RSPAMD_TASK_STAGE_DONE = (1 << 15), - RSPAMD_TASK_STAGE_REPLIED = (1 << 16) + RSPAMD_TASK_STAGE_PROCESS_MESSAGE = (1 << 4), + RSPAMD_TASK_STAGE_FILTERS = (1 << 5), + RSPAMD_TASK_STAGE_CLASSIFIERS_PRE = (1 << 6), + RSPAMD_TASK_STAGE_CLASSIFIERS = (1 << 7), + RSPAMD_TASK_STAGE_CLASSIFIERS_POST = (1 << 8), + RSPAMD_TASK_STAGE_COMPOSITES = (1 << 9), + RSPAMD_TASK_STAGE_POST_FILTERS = (1 << 10), + RSPAMD_TASK_STAGE_LEARN_PRE = (1 << 11), + RSPAMD_TASK_STAGE_LEARN = (1 << 12), + RSPAMD_TASK_STAGE_LEARN_POST = (1 << 13), + RSPAMD_TASK_STAGE_COMPOSITES_POST = (1 << 14), + RSPAMD_TASK_STAGE_IDEMPOTENT = (1 << 15), + RSPAMD_TASK_STAGE_DONE = (1 << 16), + RSPAMD_TASK_STAGE_REPLIED = (1 << 17) }; #define RSPAMD_TASK_PROCESS_ALL (RSPAMD_TASK_STAGE_CONNECT | \ RSPAMD_TASK_STAGE_ENVELOPE | \ RSPAMD_TASK_STAGE_READ_MESSAGE | \ RSPAMD_TASK_STAGE_PRE_FILTERS | \ + RSPAMD_TASK_STAGE_PROCESS_MESSAGE | \ RSPAMD_TASK_STAGE_FILTERS | \ RSPAMD_TASK_STAGE_CLASSIFIERS_PRE | \ RSPAMD_TASK_STAGE_CLASSIFIERS | \ @@ -75,6 +77,7 @@ enum rspamd_task_stage { #define RSPAMD_TASK_PROCESS_LEARN (RSPAMD_TASK_STAGE_CONNECT | \ RSPAMD_TASK_STAGE_ENVELOPE | \ RSPAMD_TASK_STAGE_READ_MESSAGE | \ + RSPAMD_TASK_STAGE_PROCESS_MESSAGE | \ RSPAMD_TASK_STAGE_CLASSIFIERS_PRE | \ RSPAMD_TASK_STAGE_CLASSIFIERS | \ RSPAMD_TASK_STAGE_CLASSIFIERS_POST | \ @@ -85,7 +88,7 @@ enum rspamd_task_stage { #define RSPAMD_TASK_FLAG_MIME (1 << 0) #define RSPAMD_TASK_FLAG_JSON (1 << 1) -#define RSPAMD_TASK_FLAG_SKIP_EXTRA (1 << 2) +#define RSPAMD_TASK_FLAG_SKIP_PROCESS (1 << 2) #define RSPAMD_TASK_FLAG_SKIP (1 << 3) #define RSPAMD_TASK_FLAG_EXT_URLS (1 << 4) #define RSPAMD_TASK_FLAG_SPAMC (1 << 5) diff --git a/src/libserver/url.c b/src/libserver/url.c index 23826d51b..650f86a28 100644 --- a/src/libserver/url.c +++ b/src/libserver/url.c @@ -1012,13 +1012,46 @@ rspamd_web_parse (struct http_parser_url *u, const gchar *str, gsize len, break; case parse_port_password: if (g_ascii_isdigit (t)) { - /* XXX: that breaks urls with passwords starting with number */ - st = parse_port; - c = slash; - p--; - SET_U (u, UF_HOST); - p++; - c = p; + const gchar *tmp = p; + + while (tmp < last) { + if (!g_ascii_isdigit (*tmp)) { + if (*tmp == '/' || *tmp == '#' || *tmp == '?' || + is_url_end (*tmp) || g_ascii_isspace (*tmp)) { + /* Port + something */ + st = parse_port; + c = slash; + p--; + SET_U (u, UF_HOST); + p++; + c = p; + break; + } + else { + /* Not a port, bad character at the end */ + break; + } + } + tmp ++; + } + + if (tmp == last) { + /* Host + port only */ + st = parse_port; + c = slash; + p--; + SET_U (u, UF_HOST); + p++; + c = p; + } + + if (st != parse_port) { + /* Fallback to user:password */ + p = slash; + c = slash; + user_seen = TRUE; + st = parse_user; + } } else { /* Rewind back */ diff --git a/src/libstat/stat_process.c b/src/libstat/stat_process.c index 070b9d6ca..4f7f4e703 100644 --- a/src/libstat/stat_process.c +++ b/src/libstat/stat_process.c @@ -190,13 +190,8 @@ rspamd_stat_tokenize_parts_metadata (struct rspamd_stat_ctx *st_ctx, hdr = cur->data; if (hdr->name && hdr->type != RSPAMD_HEADER_RECEIVED) { - /* We assume that headers count is not more than 10^10 */ - gsize nlen = strlen (hdr->name) + 1 + 10; - gchar *hdrbuf = rspamd_mempool_alloc (task->task_pool, nlen); - nlen = rspamd_snprintf (hdrbuf, nlen, "%s:%d", hdr->name, hdr->order); - rspamd_str_lc (hdrbuf, nlen); - elt.begin = hdrbuf; - elt.len = nlen; + elt.begin = hdr->name; + elt.len = strlen (hdr->name); g_array_append_val (ar, elt); } diff --git a/src/libutil/map.c b/src/libutil/map.c index 577933c25..2873b76f6 100644 --- a/src/libutil/map.c +++ b/src/libutil/map.c @@ -150,7 +150,7 @@ rspamd_map_check_sig_pk_mem (const guchar *sig, ret = FALSE; } - if (ret && !rspamd_cryptobox_verify (sig, input, inlen, + if (ret && !rspamd_cryptobox_verify (sig, siglen, input, inlen, rspamd_pubkey_get_pk (pk, NULL), RSPAMD_CRYPTOBOX_MODE_25519)) { msg_err_map ("can't verify signature for %s: incorrect signature", map->name); diff --git a/src/libutil/radix.c b/src/libutil/radix.c index a42be7032..3ea6c6b8c 100644 --- a/src/libutil/radix.c +++ b/src/libutil/radix.c @@ -42,6 +42,7 @@ struct radix_tree_compressed { rspamd_mempool_t *pool; struct btrie *tree; size_t size; + guint duplicates; gboolean own_pool; }; @@ -68,6 +69,7 @@ radix_insert_compressed (radix_compressed_t * tree, gsize masklen, uintptr_t value) { + static const guint max_duplicates = 32; guint keybits = keylen * NBBY; uintptr_t old; gchar ip_str[INET6_ADDRSTRLEN + 1]; @@ -85,23 +87,29 @@ radix_insert_compressed (radix_compressed_t * tree, (gconstpointer)value); if (ret != BTRIE_OKAY) { - memset (ip_str, 0, sizeof (ip_str)); + tree->duplicates++; - if (keybits == 32) { - msg_err_radix ("cannot insert %p, key: %s/%d, duplicate value", - (gpointer)value, - inet_ntop (AF_INET, key, ip_str, sizeof (ip_str) - 1), - (gint)(keybits - masklen)); + if (tree->duplicates == max_duplicates) { + msg_err_radix ("maximum duplicates limit reached: %d, " + "suppress further errors", max_duplicates); } - else if (keybits == 128) { - msg_err_radix ("cannot insert %p, key: [%s]/%d, duplicate value", - (gpointer)value, - inet_ntop (AF_INET6, key, ip_str, sizeof (ip_str) - 1), - (gint)(keybits - masklen)); - } - else { - msg_err_radix ("cannot insert %p with mask %z, key: %*xs, duplicate value", - (gpointer)value, keybits - masklen, (int)keylen, key); + else if (tree->duplicates < max_duplicates) { + memset (ip_str, 0, sizeof (ip_str)); + + if (keybits == 32) { + msg_err_radix ("cannot insert %p, key: %s/%d, duplicate value", + (gpointer) value, + inet_ntop (AF_INET, key, ip_str, sizeof (ip_str) - 1), + (gint) (keybits - masklen)); + } else if (keybits == 128) { + msg_err_radix ("cannot insert %p, key: [%s]/%d, duplicate value", + (gpointer) value, + inet_ntop (AF_INET6, key, ip_str, sizeof (ip_str) - 1), + (gint) (keybits - masklen)); + } else { + msg_err_radix ("cannot insert %p with mask %z, key: %*xs, duplicate value", + (gpointer) value, keybits - masklen, (int) keylen, key); + } } } else { @@ -124,6 +132,7 @@ radix_create_compressed (void) tree->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL); tree->size = 0; + tree->duplicates = 0; tree->tree = btrie_init (tree->pool); tree->own_pool = TRUE; @@ -138,6 +147,7 @@ radix_create_compressed_with_pool (rspamd_mempool_t *pool) tree = rspamd_mempool_alloc (pool, sizeof (*tree)); tree->pool = pool; tree->size = 0; + tree->duplicates = 0; tree->tree = btrie_init (tree->pool); tree->own_pool = FALSE; @@ -393,5 +403,5 @@ radix_get_info (radix_compressed_t *tree) return NULL; } - return btrie_stats (tree->tree); + return btrie_stats (tree->tree, tree->duplicates); } diff --git a/src/libutil/str_util.c b/src/libutil/str_util.c index fc8d6637b..026e331fe 100644 --- a/src/libutil/str_util.c +++ b/src/libutil/str_util.c @@ -866,9 +866,8 @@ rspamd_strings_levenshtein_distance (const gchar *s1, gsize s1len, const gchar *s2, gsize s2len, guint replace_cost) { - guint x, y, lastdiag, olddiag; - gchar c1, c2; - guint *column; + gchar c1, c2, last_c2, last_c1; + static GArray *current_row = NULL, *prev_row = NULL, *transp_row = NULL; gint eq; static const guint max_cmp = 8192; gint ret; @@ -885,31 +884,79 @@ rspamd_strings_levenshtein_distance (const gchar *s1, gsize s1len, if (MAX(s1len, s2len) > max_cmp) { /* Cannot compare too many characters */ - return 0; + return max_cmp; + } + + if (s1len > s2len) { + /* Exchange s1 and s2 */ + const gchar *tmp; + gsize tmplen; + + tmp = s2; + s2 = s1; + s1 = tmp; + + tmplen = s2len; + s2len = s1len; + s1len = tmplen; + } + + /* Adjust static space */ + if (current_row == NULL) { + current_row = g_array_sized_new (FALSE, FALSE, sizeof (gint), s1len + 1); + prev_row = g_array_sized_new (FALSE, FALSE, sizeof (gint), s1len + 1); + transp_row = g_array_sized_new (FALSE, FALSE, sizeof (gint), s1len + 1); + g_array_set_size (current_row, s1len + 1); + g_array_set_size (prev_row, s1len + 1); + g_array_set_size (transp_row, s1len + 1); + } + else if (current_row->len < s1len + 1) { + g_array_set_size (current_row, s1len + 1); + g_array_set_size (prev_row, s1len + 1); + g_array_set_size (transp_row, s1len + 1); } - column = g_malloc0 ((s1len + 1) * sizeof (guint)); + memset (current_row->data, 0, (s1len + 1) * sizeof (gint)); + memset (transp_row->data, 0, (s1len + 1) * sizeof (gint)); - for (y = 1; y <= s1len; y++) { - column[y] = y; + for (gint i = 0; i <= s1len; i++) { + g_array_index (prev_row, gint, i) = i; } - for (x = 1; x <= s2len; x++) { - column[0] = x; + last_c2 = '\0'; + + for (gint i = 1; i <= s2len; i++) { + c2 = s2[i - 1]; + g_array_index (current_row, gint, 0) = i; + last_c1 = '\0'; - for (y = 1, lastdiag = x - 1; y <= s1len; y++) { - olddiag = column[y]; - c1 = s1[y - 1]; - c2 = s2[x - 1]; - eq = (c1 == c2) ? 0 : replace_cost; - column[y] = MIN3 (column[y] + 1, column[y - 1] + 1, - lastdiag + (eq)); - lastdiag = olddiag; + for (gint j = 1; j <= s1len; j++) { + c1 = s1[j - 1]; + eq = c1 == c2 ? 0 : replace_cost; + ret = MIN3 (g_array_index (current_row, gint, j - 1) + 1, /* Insert */ + g_array_index (prev_row, gint, j) + 1, /* Remove */ + g_array_index (prev_row, gint, j - 1) + eq /* Replace */); + + /* Take reordering into account */ + if (c1 == last_c2 && c2 == last_c1 && j >= 2) { + ret = MIN (ret, g_array_index (transp_row, gint, j - 2) + eq); + } + + g_array_index (current_row, gint, j) = ret; + last_c1 = c1; } + + last_c2 = c2; + + /* Exchange pointers */ + GArray *tmp; + tmp = transp_row; + transp_row = prev_row; + prev_row = current_row; + current_row = tmp; } - ret = column[s1len]; - g_free (column); + ret = g_array_index (prev_row, gint, s1len); return ret; } @@ -1214,7 +1261,7 @@ rspamd_substring_preprocess_kmp (const gchar *pat, gsize len, goffset *fsm, i++; j++; - if (f(pat[i], pat[j])) { + if (i < len && j < len && f(pat[i], pat[j])) { fsm[i] = fsm[j]; } else { diff --git a/src/libutil/util.c b/src/libutil/util.c index d55a8e8ac..510b16045 100644 --- a/src/libutil/util.c +++ b/src/libutil/util.c @@ -80,6 +80,7 @@ #endif #endif #include <math.h> /* for pow */ +#include <glob.h> /* in fact, we require this file ultimately */ #include "cryptobox.h" #include "zlib.h" @@ -2846,4 +2847,100 @@ rspamd_fstring_gzip (rspamd_fstring_t **in) *in = comp; return TRUE; +} + +static gboolean +rspamd_glob_dir (const gchar *full_path, const gchar *pattern, + gboolean recursive, guint rec_len, + GPtrArray *res, GError **err) +{ + glob_t globbuf; + const gchar *path; + static gchar pathbuf[PATH_MAX]; /* Static to help recursion */ + guint i; + gint rc; + static const guint rec_lim = 16; + struct stat st; + + if (rec_len > rec_lim) { + g_set_error (err, g_quark_from_static_string ("glob"), EOVERFLOW, + "maximum nesting is reached: %d", rec_lim); + + return FALSE; + } + + memset (&globbuf, 0, sizeof (globbuf)); + + if ((rc = glob (full_path, 0, NULL, &globbuf)) != 0) { + + if (rc != GLOB_NOMATCH) { + g_set_error (err, g_quark_from_static_string ("glob"), errno, + "glob %s failed: %s", full_path, strerror (errno)); + globfree (&globbuf); + + return FALSE; + } + else { + globfree (&globbuf); + + return TRUE; + } + } + + for (i = 0; i < globbuf.gl_pathc; i ++) { + path = globbuf.gl_pathv[i]; + + if (stat (path, &st) == -1) { + if (errno == EPERM || errno == EACCES || errno == ELOOP) { + /* Silently ignore */ + continue; + } + + g_set_error (err, g_quark_from_static_string ("glob"), errno, + "stat %s failed: %s", path, strerror (errno)); + globfree (&globbuf); + + return FALSE; + } + + if (S_ISREG (st.st_mode)) { + g_ptr_array_add (res, g_strdup (path)); + } + else if (recursive && S_ISDIR (st.st_mode)) { + rspamd_snprintf (pathbuf, sizeof (pathbuf), "%s%c%s", + path, G_DIR_SEPARATOR, pattern); + + if (!rspamd_glob_dir (full_path, pattern, recursive, rec_len + 1, + res, err)) { + globfree (&globbuf); + + return FALSE; + } + } + } + + globfree (&globbuf); + + return TRUE; +} + +GPtrArray * +rspamd_glob_path (const gchar *dir, + const gchar *pattern, + gboolean recursive, + GError **err) +{ + gchar path[PATH_MAX]; + GPtrArray *res; + + res = g_ptr_array_new_full (32, (GDestroyNotify)g_free); + rspamd_snprintf (path, sizeof (path), "%s%c%s", dir, G_DIR_SEPARATOR, pattern); + + if (!rspamd_glob_dir (path, pattern, recursive, 0, res, err)) { + g_ptr_array_free (res, TRUE); + + return NULL; + } + + return res; }
\ No newline at end of file diff --git a/src/libutil/util.h b/src/libutil/util.h index 6470b5c45..96e85af30 100644 --- a/src/libutil/util.h +++ b/src/libutil/util.h @@ -498,4 +498,17 @@ void rspamd_localtime (gint64 ts, struct tm *dest); */ gboolean rspamd_fstring_gzip (rspamd_fstring_t **in); +/** + * Perform globbing searching for the specified path. Allow recursion, + * returns an error if maximum nesting is reached. + * @param pattern + * @param recursive + * @param err + * @return GPtrArray of gchar *, elements are freed when array is freed + */ +GPtrArray *rspamd_glob_path (const gchar *dir, + const gchar *pattern, + gboolean recursive, + GError **err); + #endif diff --git a/src/lua/lua_common.c b/src/lua/lua_common.c index fe715f393..323957086 100644 --- a/src/lua/lua_common.c +++ b/src/lua/lua_common.c @@ -375,6 +375,315 @@ rspamd_lua_set_path (lua_State *L, const ucl_object_t *cfg_obj, GHashTable *vars lua_pop (L, 1); } +static gint +rspamd_lua_cmp_version_components (const gchar *comp1, const gchar *comp2) +{ + guint v1, v2; + + v1 = strtoul (comp1, NULL, 10); + v2 = strtoul (comp2, NULL, 10); + + return v1 - v2; +} + +static int +rspamd_lua_rspamd_version_cmp (lua_State *L) +{ + const gchar *ver; + gchar **components; + gint ret = 0; + + if (lua_type (L, 2) == LUA_TSTRING) { + ver = lua_tostring (L, 2); + + components = g_strsplit_set (ver, ".-_", -1); + + if (!components) { + return luaL_error (L, "invalid arguments to 'cmp': %s", ver); + } + + if (components[0]) { + ret = rspamd_lua_cmp_version_components (components[0], + RSPAMD_VERSION_MAJOR); + } + + if (ret) { + goto set; + } + + if (components[1]) { + ret = rspamd_lua_cmp_version_components (components[1], + RSPAMD_VERSION_MINOR); + } + + if (ret) { + goto set; + } + + if (components[2]) { + ret = rspamd_lua_cmp_version_components (components[2], + RSPAMD_VERSION_PATCH); + } + + /* + * XXX: we don't compare git releases assuming that it is meaningless + */ + } + else { + return luaL_error (L, "invalid arguments to 'cmp'"); + } + +set: + g_strfreev (components); + lua_pushnumber (L, ret); + + return 1; +} + +static int +rspamd_lua_rspamd_version_numeric (lua_State *L) +{ + static gint64 version_num = RSPAMD_VERSION_NUM; + const gchar *type; + + if (lua_gettop (L) >= 2 && lua_type (L, 1) == LUA_TSTRING) { + type = lua_tostring (L, 1); + if (g_ascii_strcasecmp (type, "short") == 0) { + version_num = RSPAMD_VERSION_MAJOR_NUM * 1000 + + RSPAMD_VERSION_MINOR_NUM * 100 + + RSPAMD_VERSION_PATCH_NUM * 10; + } + else if (g_ascii_strcasecmp (type, "main") == 0) { + version_num = RSPAMD_VERSION_MAJOR_NUM * 1000 + + RSPAMD_VERSION_MINOR_NUM * 100; + } + else if (g_ascii_strcasecmp (type, "major") == 0) { + version_num = RSPAMD_VERSION_MAJOR_NUM; + } + else if (g_ascii_strcasecmp (type, "minor") == 0) { + version_num = RSPAMD_VERSION_MINOR_NUM; + } + else if (g_ascii_strcasecmp (type, "patch") == 0) { + version_num = RSPAMD_VERSION_PATCH_NUM; + } + } + + lua_pushnumber (L, version_num); + + return 1; +} + +static int +rspamd_lua_rspamd_version (lua_State *L) +{ + const gchar *result = NULL, *type; + + if (lua_gettop (L) == 0) { + result = RVERSION; + } + else if (lua_gettop (L) >= 1 && lua_type (L, 1) == LUA_TSTRING) { + /* We got something like string */ + type = lua_tostring (L, 1); + + if (g_ascii_strcasecmp (type, "short") == 0) { + result = RSPAMD_VERSION_MAJOR + "." RSPAMD_VERSION_MINOR + "." RSPAMD_VERSION_PATCH; + } + else if (g_ascii_strcasecmp (type, "main") == 0) { + result = RSPAMD_VERSION_MAJOR "." RSPAMD_VERSION_MINOR; + } + else if (g_ascii_strcasecmp (type, "major") == 0) { + result = RSPAMD_VERSION_MAJOR; + } + else if (g_ascii_strcasecmp (type, "minor") == 0) { + result = RSPAMD_VERSION_MINOR; + } + else if (g_ascii_strcasecmp (type, "patch") == 0) { + result = RSPAMD_VERSION_PATCH; + } + else if (g_ascii_strcasecmp (type, "id") == 0) { + result = RID; + } + else if (g_ascii_strcasecmp (type, "num") == 0) { + return rspamd_lua_rspamd_version_numeric (L); + } + else if (g_ascii_strcasecmp (type, "cmp") == 0) { + return rspamd_lua_rspamd_version_cmp (L); + } + } + + lua_pushstring (L, result); + + return 1; +} + +void +rspamd_lua_set_globals (struct rspamd_config *cfg, lua_State *L, + GHashTable *vars) +{ + struct rspamd_config **pcfg; + gint orig_top = lua_gettop (L); + + /* First check for global variable 'config' */ + lua_getglobal (L, "config"); + if (lua_isnil (L, -1)) { + /* Assign global table to set up attributes */ + lua_newtable (L); + lua_setglobal (L, "config"); + } + + lua_getglobal (L, "metrics"); + if (lua_isnil (L, -1)) { + lua_newtable (L); + lua_setglobal (L, "metrics"); + } + + lua_getglobal (L, "composites"); + if (lua_isnil (L, -1)) { + lua_newtable (L); + lua_setglobal (L, "composites"); + } + + lua_getglobal (L, "rspamd_classifiers"); + if (lua_isnil (L, -1)) { + lua_newtable (L); + lua_setglobal (L, "rspamd_classifiers"); + } + + lua_getglobal (L, "classifiers"); + if (lua_isnil (L, -1)) { + lua_newtable (L); + lua_setglobal (L, "classifiers"); + } + + lua_getglobal (L, "rspamd_version"); + if (lua_isnil (L, -1)) { + lua_pushcfunction (L, rspamd_lua_rspamd_version); + lua_setglobal (L, "rspamd_version"); + } + + if (cfg != NULL) { + pcfg = lua_newuserdata (L, sizeof (struct rspamd_config *)); + rspamd_lua_setclass (L, "rspamd{config}", -1); + *pcfg = cfg; + lua_setglobal (L, "rspamd_config"); + } + + lua_settop (L, orig_top); + + /* Set known paths as rspamd_paths global */ + lua_getglobal (L, "rspamd_paths"); + if (lua_isnil (L, -1)) { + const gchar *confdir = RSPAMD_CONFDIR, *rundir = RSPAMD_RUNDIR, + *dbdir = RSPAMD_DBDIR, *logdir = RSPAMD_LOGDIR, + *wwwdir = RSPAMD_WWWDIR, *pluginsdir = RSPAMD_PLUGINSDIR, + *rulesdir = RSPAMD_RULESDIR, *lualibdir = RSPAMD_LUALIBDIR, + *prefix = RSPAMD_PREFIX; + const gchar *t; + + /* Try environment */ + t = getenv ("PLUGINSDIR"); + if (t) { + pluginsdir = t; + } + + t = getenv ("RULESDIR"); + if (t) { + rulesdir = t; + } + + t = getenv ("DBDIR"); + if (t) { + dbdir = t; + } + + t = getenv ("RUNDIR"); + if (t) { + rundir = t; + } + + t = getenv ("LUALIBDIR"); + if (t) { + lualibdir = t; + } + + t = getenv ("LOGDIR"); + if (t) { + logdir = t; + } + + t = getenv ("WWWDIR"); + if (t) { + wwwdir = t; + } + + t = getenv ("CONFDIR"); + if (t) { + confdir = t; + } + + + if (vars) { + t = g_hash_table_lookup (vars, "PLUGINSDIR"); + if (t) { + pluginsdir = t; + } + + t = g_hash_table_lookup (vars, "RULESDIR"); + if (t) { + rulesdir = t; + } + + t = g_hash_table_lookup (vars, "LUALIBDIR"); + if (t) { + lualibdir = t; + } + + t = g_hash_table_lookup (vars, "RUNDIR"); + if (t) { + rundir = t; + } + + t = g_hash_table_lookup (vars, "WWWDIR"); + if (t) { + wwwdir = t; + } + + t = g_hash_table_lookup (vars, "CONFDIR"); + if (t) { + confdir = t; + } + + t = g_hash_table_lookup (vars, "DBDIR"); + if (t) { + dbdir = t; + } + + t = g_hash_table_lookup (vars, "LOGDIR"); + if (t) { + logdir = t; + } + } + + lua_createtable (L, 0, 9); + + rspamd_lua_table_set (L, RSPAMD_CONFDIR_INDEX, confdir); + rspamd_lua_table_set (L, RSPAMD_RUNDIR_INDEX, rundir); + rspamd_lua_table_set (L, RSPAMD_DBDIR_INDEX, dbdir); + rspamd_lua_table_set (L, RSPAMD_LOGDIR_INDEX, logdir); + rspamd_lua_table_set (L, RSPAMD_WWWDIR_INDEX, wwwdir); + rspamd_lua_table_set (L, RSPAMD_PLUGINSDIR_INDEX, pluginsdir); + rspamd_lua_table_set (L, RSPAMD_RULESDIR_INDEX, rulesdir); + rspamd_lua_table_set (L, RSPAMD_LUALIBDIR_INDEX, lualibdir); + rspamd_lua_table_set (L, RSPAMD_PREFIX_INDEX, prefix); + + lua_setglobal (L, "rspamd_paths"); + } + + lua_settop (L, orig_top); +} + lua_State * rspamd_lua_init () { @@ -587,9 +896,6 @@ rspamd_init_lua_filters (struct rspamd_config *cfg, gboolean force_load) cur = g_list_next (cur); } - /* Assign state */ - cfg->lua_state = L; - return TRUE; } diff --git a/src/lua/lua_common.h b/src/lua/lua_common.h index 457f470e9..6c64c76d4 100644 --- a/src/lua/lua_common.h +++ b/src/lua/lua_common.h @@ -298,6 +298,10 @@ void rspamd_lua_dumpstack (lua_State *L); void rspamd_lua_set_path (lua_State *L, const ucl_object_t *cfg_obj, GHashTable *vars); +/* Set some lua globals */ +void rspamd_lua_set_globals (struct rspamd_config *cfg, lua_State *L, + GHashTable *vars); + struct memory_pool_s * rspamd_lua_check_mempool (lua_State * L, gint pos); struct rspamd_config * lua_check_config (lua_State * L, gint pos); struct rspamd_async_session* lua_check_session (lua_State * L, gint pos); @@ -400,5 +404,17 @@ void rspamd_lua_add_ref_dtor (lua_State *L, rspamd_mempool_t *pool, gboolean rspamd_lua_require_function (lua_State *L, const gchar *modname, const gchar *funcname); +/* Paths defs */ +#define RSPAMD_CONFDIR_INDEX "CONFDIR" +#define RSPAMD_RUNDIR_INDEX "RUNDIR" +#define RSPAMD_DBDIR_INDEX "DBDIR" +#define RSPAMD_LOGDIR_INDEX "LOGDIR" +#define RSPAMD_PLUGINSDIR_INDEX "PLUGINSDIR" +#define RSPAMD_RULESDIR_INDEX "RULESDIR" +#define RSPAMD_LUALIBDIR_INDEX "LUALIBDIR" +#define RSPAMD_WWWDIR_INDEX "WWWDIR" +#define RSPAMD_PREFIX_INDEX "PREFIX" +#define RSPAMD_VERSION_INDEX "VERSION" + #endif /* WITH_LUA */ #endif /* RSPAMD_LUA_H */ diff --git a/src/lua/lua_config.c b/src/lua/lua_config.c index 2802ab6a4..37d122402 100644 --- a/src/lua/lua_config.c +++ b/src/lua/lua_config.c @@ -546,6 +546,13 @@ LUA_FUNCTION_DEF (config, get_symbols_cksum); LUA_FUNCTION_DEF (config, get_symbols_counters); /*** + * @method rspamd_config:get_symbols_scores() + * Returns table of all scores defined in config + * @return {table|tables} all symbols indexed by name + */ +LUA_FUNCTION_DEF (config, get_symbols_scores); + +/*** * @method rspamd_config:get_symbol_callback(name) * Returns callback function for the specified symbol if it is a lua registered callback * @return {function} callback function or nil @@ -680,6 +687,29 @@ LUA_FUNCTION_DEF (config, has_torch); */ LUA_FUNCTION_DEF (config, experimental_enabled); +/*** + * @method rspamd_config:load_ucl(filename) + * Loads config from the UCL file (but does not perform parsing using rcl) + * @param {string} filename file to load + * @return true or false + error message + */ +LUA_FUNCTION_DEF (config, load_ucl); + +/*** + * @method rspamd_config:parse_rcl([skip_sections]) + * Parses RCL using loaded ucl file + * @param {table|string} sections to skip + * @return true or false + error message + */ +LUA_FUNCTION_DEF (config, parse_rcl); + +/*** + * @method rspamd_config:init_modules() + * Initialize lua and internal modules + * @return true or false + */ +LUA_FUNCTION_DEF (config, init_modules); + static const struct luaL_reg configlib_m[] = { LUA_INTERFACE_DEF (config, get_module_opt), LUA_INTERFACE_DEF (config, get_mempool), @@ -726,6 +756,7 @@ static const struct luaL_reg configlib_m[] = { LUA_INTERFACE_DEF (config, get_symbols_count), LUA_INTERFACE_DEF (config, get_symbols_cksum), LUA_INTERFACE_DEF (config, get_symbols_counters), + LUA_INTERFACE_DEF (config, get_symbols_scores), LUA_INTERFACE_DEF (config, get_symbol_callback), LUA_INTERFACE_DEF (config, set_symbol_callback), LUA_INTERFACE_DEF (config, get_symbol_stat), @@ -737,6 +768,9 @@ static const struct luaL_reg configlib_m[] = { LUA_INTERFACE_DEF (config, get_cpu_flags), LUA_INTERFACE_DEF (config, has_torch), LUA_INTERFACE_DEF (config, experimental_enabled), + LUA_INTERFACE_DEF (config, load_ucl), + LUA_INTERFACE_DEF (config, parse_rcl), + LUA_INTERFACE_DEF (config, init_modules), {"__tostring", rspamd_lua_class_tostring}, {"__newindex", lua_config_newindex}, {NULL, NULL} @@ -2864,6 +2898,44 @@ lua_config_get_symbols_counters (lua_State *L) return 1; } +static void +lua_metric_symbol_inserter (gpointer k, gpointer v, gpointer ud) +{ + lua_State *L = (lua_State *) ud; + const gchar *sym = k; + struct rspamd_symbol *s = (struct rspamd_symbol *) v; + + lua_pushstring (L, sym); + + lua_createtable (L, 0, 3); /* TODO: add more if needed */ + lua_pushstring (L, "score"); + lua_pushnumber (L, s->score); + lua_settable (L, -3); + lua_pushstring (L, "description"); + lua_pushstring (L, s->description); + lua_settable (L, -3); + + lua_settable (L, -3); +} + +static gint +lua_config_get_symbols_scores (lua_State *L) +{ + struct rspamd_config *cfg = lua_check_config (L, 1); + + if (cfg != NULL) { + lua_createtable (L, 0, g_hash_table_size (cfg->symbols)); + g_hash_table_foreach (cfg->symbols, + lua_metric_symbol_inserter, + L); + } + else { + return luaL_error (L, "invalid arguments"); + } + + return 1; +} + static gint lua_config_get_symbol_callback (lua_State *L) @@ -3212,6 +3284,132 @@ lua_config_experimental_enabled (lua_State *L) return 1; } +#define LUA_TABLE_TO_HASH(htb, idx) do { \ + lua_pushstring (L, (idx)); \ + lua_gettable (L, -2); \ + if (lua_isstring (L, -1)) { \ + g_hash_table_insert ((htb), (idx), g_strdup (lua_tostring (L, -1))); \ + } \ + lua_pop (L, 1); \ +} while(0) + +static gint +lua_config_load_ucl (lua_State *L) +{ + struct rspamd_config *cfg = lua_check_config (L, 1); + const gchar *filename; + GHashTable *paths = g_hash_table_new_full (rspamd_str_hash, rspamd_str_equal, + NULL, g_free); + GError *err = NULL; + + if (cfg) { + if (lua_isstring (L, 2)) { + filename = lua_tostring (L, 2); + } + else { + filename = RSPAMD_CONFDIR "/rspamd.conf"; + } + + /* Convert rspamd_paths */ + lua_getglobal (L, "rspamd_paths"); + + if (lua_istable (L, -1)) { + LUA_TABLE_TO_HASH(paths, RSPAMD_CONFDIR_INDEX); + LUA_TABLE_TO_HASH(paths, RSPAMD_RUNDIR_INDEX); + LUA_TABLE_TO_HASH(paths, RSPAMD_DBDIR_INDEX); + LUA_TABLE_TO_HASH(paths, RSPAMD_LOGDIR_INDEX); + LUA_TABLE_TO_HASH(paths, RSPAMD_WWWDIR_INDEX); + LUA_TABLE_TO_HASH(paths, RSPAMD_PLUGINSDIR_INDEX); + LUA_TABLE_TO_HASH(paths, RSPAMD_RULESDIR_INDEX); + LUA_TABLE_TO_HASH(paths, RSPAMD_LUALIBDIR_INDEX); + LUA_TABLE_TO_HASH(paths, RSPAMD_PREFIX_INDEX); + } + + lua_pop (L, 1); + + if (!rspamd_config_parse_ucl (cfg, filename, paths, &err)) { + lua_pushboolean (L, false); + lua_pushfstring (L, "failed to load config: %s", err->message); + g_error_free (err); + g_hash_table_unref (paths); + + return 2; + } + + rspamd_rcl_maybe_apply_lua_transform (cfg); + rspamd_config_calculate_cksum (cfg); + } + else { + return luaL_error (L, "invalid arguments"); + } + + g_hash_table_unref (paths); + lua_pushboolean (L, true); + + return 1; +} + +#undef IDX_TO_HASH + +static gint +lua_config_parse_rcl (lua_State *L) +{ + struct rspamd_config *cfg = lua_check_config (L, 1); + GHashTable *excluded = g_hash_table_new_full (rspamd_str_hash, rspamd_str_equal, + g_free, NULL); + GError *err = NULL; + struct rspamd_rcl_section *top; + + if (cfg) { + if (lua_istable (L, 2)) { + lua_pushvalue (L, 2); + + for (lua_pushnil (L); lua_next (L, -2); lua_pop (L, 1)) { + g_hash_table_insert (excluded, g_strdup (lua_tostring (L, -1)), + GINT_TO_POINTER (-1)); + } + + lua_pop (L, 1); + } + + top = rspamd_rcl_config_init (cfg, excluded); + + if (!rspamd_rcl_parse (top, cfg, cfg, cfg->cfg_pool, cfg->rcl_obj, &err)) { + lua_pushboolean (L, false); + lua_pushfstring (L, "failed to load config: %s", err->message); + g_error_free (err); + g_hash_table_unref (excluded); + rspamd_rcl_section_free (top); + + return 2; + } + } + else { + return luaL_error (L, "invalid arguments"); + } + + g_hash_table_unref (excluded); + rspamd_rcl_section_free (top); + lua_pushboolean (L, true); + + return 1; +} + +static gint +lua_config_init_modules (lua_State *L) +{ + struct rspamd_config *cfg = lua_check_config (L, 1); + + if (cfg != NULL) { + rspamd_lua_post_load_config (cfg); + lua_pushboolean (L, rspamd_init_filters (cfg, FALSE)); + } + else { + return luaL_error (L, "invalid arguments"); + } + + return 1; +} static gint lua_monitored_alive (lua_State *L) diff --git a/src/lua/lua_cryptobox.c b/src/lua/lua_cryptobox.c index b646b0926..6fa2aa997 100644 --- a/src/lua/lua_cryptobox.c +++ b/src/lua/lua_cryptobox.c @@ -27,6 +27,9 @@ */ #include "lua_common.h" +#include "libcryptobox/cryptobox.h" +#include "libcryptobox/keypair.h" +#include "libcryptobox/keypair_private.h" #include "unix-std.h" struct rspamd_lua_cryptobox_hash { @@ -42,6 +45,10 @@ LUA_FUNCTION_DEF (cryptobox_pubkey, gc); LUA_FUNCTION_DEF (cryptobox_keypair, load); LUA_FUNCTION_DEF (cryptobox_keypair, create); LUA_FUNCTION_DEF (cryptobox_keypair, gc); +LUA_FUNCTION_DEF (cryptobox_keypair, totable); +LUA_FUNCTION_DEF (cryptobox_keypair, get_type); +LUA_FUNCTION_DEF (cryptobox_keypair, get_alg); +LUA_FUNCTION_DEF (cryptobox_keypair, get_pk); LUA_FUNCTION_DEF (cryptobox_signature, create); LUA_FUNCTION_DEF (cryptobox_signature, load); LUA_FUNCTION_DEF (cryptobox_signature, save); @@ -60,16 +67,24 @@ LUA_FUNCTION_DEF (cryptobox_hash, base32); LUA_FUNCTION_DEF (cryptobox_hash, base64); LUA_FUNCTION_DEF (cryptobox_hash, bin); LUA_FUNCTION_DEF (cryptobox_hash, gc); -LUA_FUNCTION_DEF (cryptobox, verify_memory); -LUA_FUNCTION_DEF (cryptobox, verify_file); -LUA_FUNCTION_DEF (cryptobox, sign_file); -LUA_FUNCTION_DEF (cryptobox, sign_memory); +LUA_FUNCTION_DEF (cryptobox, verify_memory); +LUA_FUNCTION_DEF (cryptobox, verify_file); +LUA_FUNCTION_DEF (cryptobox, sign_file); +LUA_FUNCTION_DEF (cryptobox, sign_memory); +LUA_FUNCTION_DEF (cryptobox, encrypt_memory); +LUA_FUNCTION_DEF (cryptobox, encrypt_file); +LUA_FUNCTION_DEF (cryptobox, decrypt_memory); +LUA_FUNCTION_DEF (cryptobox, decrypt_file); static const struct luaL_reg cryptoboxlib_f[] = { LUA_INTERFACE_DEF (cryptobox, verify_memory), LUA_INTERFACE_DEF (cryptobox, verify_file), LUA_INTERFACE_DEF (cryptobox, sign_memory), LUA_INTERFACE_DEF (cryptobox, sign_file), + LUA_INTERFACE_DEF (cryptobox, encrypt_memory), + LUA_INTERFACE_DEF (cryptobox, encrypt_file), + LUA_INTERFACE_DEF (cryptobox, decrypt_memory), + LUA_INTERFACE_DEF (cryptobox, decrypt_file), {NULL, NULL} }; @@ -93,6 +108,13 @@ static const struct luaL_reg cryptoboxkeypairlib_f[] = { static const struct luaL_reg cryptoboxkeypairlib_m[] = { {"__tostring", rspamd_lua_class_tostring}, + {"totable", lua_cryptobox_keypair_totable}, + {"get_type", lua_cryptobox_keypair_get_type}, + {"get_alg", lua_cryptobox_keypair_get_alg}, + {"type", lua_cryptobox_keypair_get_type}, + {"alg", lua_cryptobox_keypair_get_alg}, + {"pk", lua_cryptobox_keypair_get_pk}, + {"pubkey", lua_cryptobox_keypair_get_pk}, {"__gc", lua_cryptobox_keypair_gc}, {NULL, NULL} }; @@ -320,8 +342,8 @@ lua_cryptobox_pubkey_gc (lua_State *L) } /*** - * @function rspamd_cryptobox_keypair.load(file) - * Loads public key from UCL file + * @function rspamd_cryptobox_keypair.load(file|table) + * Loads public key from UCL file or directly from Lua * @param {string} file filename to load * @return {cryptobox_keypair} new keypair */ @@ -329,57 +351,6 @@ static gint lua_cryptobox_keypair_load (lua_State *L) { struct rspamd_cryptobox_keypair *kp, **pkp; - const gchar *filename; - struct ucl_parser *parser; - ucl_object_t *obj; - - filename = luaL_checkstring (L, 1); - if (filename != NULL) { - parser = ucl_parser_new (0); - - if (!ucl_parser_add_file (parser, filename)) { - msg_err ("cannot open keypair from file: %s, %s", - filename, - ucl_parser_get_error (parser)); - ucl_parser_free (parser); - lua_pushnil (L); - } - else { - obj = ucl_parser_get_object (parser); - kp = rspamd_keypair_from_ucl (obj); - ucl_parser_free (parser); - - if (kp == NULL) { - msg_err ("cannot open keypair from file: %s", - filename); - ucl_object_unref (obj); - lua_pushnil (L); - } - else { - pkp = lua_newuserdata (L, sizeof (gpointer)); - *pkp = kp; - rspamd_lua_setclass (L, "rspamd{cryptobox_keypair}", -1); - ucl_object_unref (obj); - } - } - } - else { - return luaL_error (L, "bad input arguments"); - } - - return 1; -} - -/*** - * @function rspamd_cryptobox_keypair.create(ucl_data) - * Loads public key from UCL data - * @param {string} ucl_data ucl to load - * @return {cryptobox_keypair} new keypair - */ -static gint -lua_cryptobox_keypair_create (lua_State *L) -{ - struct rspamd_cryptobox_keypair *kp, **pkp; const gchar *buf; gsize len; struct ucl_parser *parser; @@ -439,6 +410,57 @@ lua_cryptobox_keypair_create (lua_State *L) return 1; } +/*** + * @function rspamd_cryptobox_keypair.create([type='encryption'[, alg='curve25519']]) + * Generates new keypair + * @param {string} type type of keypair: 'encryption' (default) or 'sign' + * @param {string} alg algorithm of keypair: 'curve25519' (default) or 'nist' + * @return {cryptobox_keypair} new keypair + */ +static gint +lua_cryptobox_keypair_create (lua_State *L) +{ + struct rspamd_cryptobox_keypair *kp, **pkp; + enum rspamd_cryptobox_keypair_type type = RSPAMD_KEYPAIR_KEX; + enum rspamd_cryptobox_mode alg = RSPAMD_CRYPTOBOX_MODE_25519; + + if (lua_isstring (L, 1)) { + const gchar *str = lua_tostring (L, 1); + + if (strcmp (str, "sign") == 0) { + type = RSPAMD_KEYPAIR_SIGN; + } + else if (strcmp (str, "encryption") == 0) { + type = RSPAMD_KEYPAIR_KEX; + } + else { + return luaL_error (L, "invalid keypair type: %s", str); + } + } + + if (lua_isstring (L, 2)) { + const gchar *str = lua_tostring (L, 2); + + if (strcmp (str, "nist") == 0 || strcmp (str, "openssl") == 0) { + alg = RSPAMD_CRYPTOBOX_MODE_NIST; + } + else if (strcmp (str, "curve25519") == 0 || strcmp (str, "default") == 0) { + alg = RSPAMD_CRYPTOBOX_MODE_25519; + } + else { + return luaL_error (L, "invalid keypair algorithm: %s", str); + } + } + + kp = rspamd_keypair_new (type, alg); + + pkp = lua_newuserdata (L, sizeof (gpointer)); + *pkp = kp; + rspamd_lua_setclass (L, "rspamd{cryptobox_keypair}", -1); + + return 1; +} + static gint lua_cryptobox_keypair_gc (lua_State *L) { @@ -452,7 +474,118 @@ lua_cryptobox_keypair_gc (lua_State *L) } /*** - * @function rspamd_cryptobox_signature.load(file) + * @method keypair:totable([hex=false]]) + * Converts keypair to table (not very safe due to memory leftovers) + */ +static gint +lua_cryptobox_keypair_totable (lua_State *L) +{ + struct rspamd_cryptobox_keypair *kp = lua_check_cryptobox_keypair (L, 1); + ucl_object_t *obj; + gboolean hex = FALSE; + gint ret = 1; + + if (kp != NULL) { + + if (lua_isboolean (L, 2)) { + hex = lua_toboolean (L, 2); + } + + obj = rspamd_keypair_to_ucl (kp, hex); + + ret = ucl_object_push_lua (L, obj, true); + ucl_object_unref (obj); + } + else { + return luaL_error (L, "invalid arguments"); + } + + return ret; +} +/*** + * @method keypair:type() + * Returns type of keypair as a string: 'encryption' or 'sign' + * @return {string} type of keypair as a string + */ +static gint +lua_cryptobox_keypair_get_type (lua_State *L) +{ + struct rspamd_cryptobox_keypair *kp = lua_check_cryptobox_keypair (L, 1); + + if (kp) { + if (kp->type == RSPAMD_KEYPAIR_KEX) { + lua_pushstring (L, "encryption"); + } + else { + lua_pushstring (L, "sign"); + } + } + else { + return luaL_error (L, "invalid arguments"); + } + + return 1; +} + +/*** + * @method keypair:alg() + * Returns algorithm of keypair as a string: 'encryption' or 'sign' + * @return {string} type of keypair as a string + */ +static gint +lua_cryptobox_keypair_get_alg (lua_State *L) +{ + struct rspamd_cryptobox_keypair *kp = lua_check_cryptobox_keypair (L, 1); + + if (kp) { + if (kp->alg == RSPAMD_CRYPTOBOX_MODE_25519) { + lua_pushstring (L, "curve25519"); + } + else { + lua_pushstring (L, "nist"); + } + } + else { + return luaL_error (L, "invalid arguments"); + } + + return 1; +} + +/*** + * @method keypair:pk() + * Returns pubkey for a specific keypair + * @return {rspamd_pubkey} pubkey for a keypair + */ +static gint +lua_cryptobox_keypair_get_pk (lua_State *L) +{ + struct rspamd_cryptobox_keypair *kp = lua_check_cryptobox_keypair (L, 1); + struct rspamd_cryptobox_pubkey *pk, **ppk; + const guchar *data; + guint dlen; + + if (kp) { + data = rspamd_keypair_component (kp, RSPAMD_KEYPAIR_COMPONENT_PK, &dlen); + pk = rspamd_pubkey_from_bin (data, dlen, kp->type, kp->alg); + + if (pk == NULL) { + return luaL_error (L, "invalid keypair"); + } + + ppk = lua_newuserdata (L, sizeof (*ppk)); + *ppk = pk; + rspamd_lua_setclass (L, "rspamd{cryptobox_pubkey}", -1); + } + else { + return luaL_error (L, "invalid arguments"); + } + + return 1; +} + +/*** + * @function rspamd_cryptobox_signature.load(file, [alg = 'curve25519']) * Loads signature from raw file * @param {string} file filename to load * @return {cryptobox_signature} new signature @@ -465,6 +598,7 @@ lua_cryptobox_signature_load (lua_State *L) gpointer data; int fd; struct stat st; + enum rspamd_cryptobox_mode alg = RSPAMD_CRYPTOBOX_MODE_25519; filename = luaL_checkstring (L, 1); if (filename != NULL) { @@ -475,7 +609,6 @@ lua_cryptobox_signature_load (lua_State *L) lua_pushnil (L); } else { - sig = g_malloc (sizeof (rspamd_fstring_t)); if (fstat (fd, &st) == -1 || (data = mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) @@ -484,8 +617,20 @@ lua_cryptobox_signature_load (lua_State *L) lua_pushnil (L); } else { - if (st.st_size == rspamd_cryptobox_signature_bytes ( - RSPAMD_CRYPTOBOX_MODE_25519)) { + if (lua_isstring (L, 2)) { + const gchar *str = lua_tostring (L, 2); + + if (strcmp (str, "nist") == 0 || strcmp (str, "openssl") == 0) { + alg = RSPAMD_CRYPTOBOX_MODE_NIST; + } + else if (strcmp (str, "curve25519") == 0 || strcmp (str, "default") == 0) { + alg = RSPAMD_CRYPTOBOX_MODE_25519; + } + else { + return luaL_error (L, "invalid keypair algorithm: %s", str); + } + } + if (st.st_size > 0) { sig = rspamd_fstring_new_init (data, st.st_size); psig = lua_newuserdata (L, sizeof (rspamd_fstring_t *)); rspamd_lua_setclass (L, "rspamd{cryptobox_signature}", -1); @@ -494,7 +639,7 @@ lua_cryptobox_signature_load (lua_State *L) else { msg_err ("size of %s mismatches: %d while %d is expected", filename, (int)st.st_size, - rspamd_cryptobox_signature_bytes (RSPAMD_CRYPTOBOX_MODE_25519)); + rspamd_cryptobox_signature_bytes (alg)); lua_pushnil (L); } @@ -1156,7 +1301,7 @@ lua_cryptobox_hash_gc (lua_State *L) } /*** - * @function rspamd_cryptobox.verify_memory(pk, sig, data) + * @function rspamd_cryptobox.verify_memory(pk, sig, data, [alg = 'curve25519']) * Check memory using specified cryptobox key and signature * @param {pubkey} pk public key to verify * @param {sig} signature to check @@ -1170,6 +1315,7 @@ lua_cryptobox_verify_memory (lua_State *L) rspamd_fstring_t *signature; struct rspamd_lua_text *t; const gchar *data; + enum rspamd_cryptobox_mode alg = RSPAMD_CRYPTOBOX_MODE_25519; gsize len; gint ret; @@ -1190,9 +1336,23 @@ lua_cryptobox_verify_memory (lua_State *L) data = luaL_checklstring (L, 3, &len); } + if (lua_isstring (L, 4)) { + const gchar *str = lua_tostring (L, 4); + + if (strcmp (str, "nist") == 0 || strcmp (str, "openssl") == 0) { + alg = RSPAMD_CRYPTOBOX_MODE_NIST; + } + else if (strcmp (str, "curve25519") == 0 || strcmp (str, "default") == 0) { + alg = RSPAMD_CRYPTOBOX_MODE_25519; + } + else { + return luaL_error (L, "invalid algorithm: %s", str); + } + } + if (pk != NULL && signature != NULL && data != NULL) { - ret = rspamd_cryptobox_verify (signature->str, data, len, - rspamd_pubkey_get_pk (pk, NULL), RSPAMD_CRYPTOBOX_MODE_25519); + ret = rspamd_cryptobox_verify (signature->str, signature->len, data, len, + rspamd_pubkey_get_pk (pk, NULL), alg); if (ret) { lua_pushboolean (L, 1); @@ -1209,7 +1369,7 @@ lua_cryptobox_verify_memory (lua_State *L) } /*** - * @function rspamd_cryptobox.verify_file(pk, sig, file) + * @function rspamd_cryptobox.verify_file(pk, sig, file, [alg = 'curve25519']) * Check file using specified cryptobox key and signature * @param {pubkey} pk public key to verify * @param {sig} signature to check @@ -1223,6 +1383,7 @@ lua_cryptobox_verify_file (lua_State *L) struct rspamd_cryptobox_pubkey *pk; rspamd_fstring_t *signature; guchar *map = NULL; + enum rspamd_cryptobox_mode alg = RSPAMD_CRYPTOBOX_MODE_25519; gsize len; gint ret; @@ -1230,11 +1391,26 @@ lua_cryptobox_verify_file (lua_State *L) signature = lua_check_cryptobox_sign (L, 2); fname = luaL_checkstring (L, 3); + if (lua_isstring (L, 4)) { + const gchar *str = lua_tostring (L, 4); + + if (strcmp (str, "nist") == 0 || strcmp (str, "openssl") == 0) { + alg = RSPAMD_CRYPTOBOX_MODE_NIST; + } + else if (strcmp (str, "curve25519") == 0 || strcmp (str, "default") == 0) { + alg = RSPAMD_CRYPTOBOX_MODE_25519; + } + else { + return luaL_error (L, "invalid algorithm: %s", str); + } + } + map = rspamd_file_xmap (fname, PROT_READ, &len, TRUE); if (map != NULL && pk != NULL && signature != NULL) { - ret = rspamd_cryptobox_verify (signature->str, map, len, - rspamd_pubkey_get_pk (pk, NULL), RSPAMD_CRYPTOBOX_MODE_25519); + ret = rspamd_cryptobox_verify (signature->str, signature->len, + map, len, + rspamd_pubkey_get_pk (pk, NULL), alg); if (ret) { lua_pushboolean (L, 1); @@ -1291,7 +1467,7 @@ lua_cryptobox_sign_memory (lua_State *L) } - if (!kp || !data) { + if (!kp || !data || kp->type == RSPAMD_KEYPAIR_KEX) { return luaL_error (L, "invalid arguments"); } @@ -1353,6 +1529,207 @@ lua_cryptobox_sign_file (lua_State *L) return 1; } +/*** + * @function rspamd_cryptobox.encrypt_memory(kp, data) + * Encrypt data using specified keypair + * @param {keypair} kp keypair to use + * @param {string|text} data + * @return {rspamd_text} encrypted text + */ +static gint +lua_cryptobox_encrypt_memory (lua_State *L) +{ + struct rspamd_cryptobox_keypair *kp; + const gchar *data; + guchar *out; + struct rspamd_lua_text *t, *res; + gsize len = 0, outlen; + GError *err = NULL; + + kp = lua_check_cryptobox_keypair (L, 1); + + if (lua_isuserdata (L, 2)) { + t = lua_check_text (L, 2); + + if (!t) { + return luaL_error (L, "invalid arguments"); + } + + data = t->start; + len = t->len; + } + else { + data = luaL_checklstring (L, 2, &len); + } + + + if (!kp || !data) { + return luaL_error (L, "invalid arguments"); + } + + if (!rspamd_keypair_encrypt (kp, data, len, &out, &outlen, &err)) { + gint ret = luaL_error (L, "cannot encrypt data: %s", err->message); + g_error_free (err); + + return ret; + } + + res = lua_newuserdata (L, sizeof (*res)); + res->flags = RSPAMD_TEXT_FLAG_OWN; + res->start = out; + res->len = outlen; + rspamd_lua_setclass (L, "rspamd{text}", -1); + + return 1; +} + +/*** + * @function rspamd_cryptobox.encrypt_file(kp, filename) + * Encrypt data using specified keypair + * @param {keypair} kp keypair to use + * @param {string} filename + * @return {rspamd_text} encrypted text + */ +static gint +lua_cryptobox_encrypt_file (lua_State *L) +{ + struct rspamd_cryptobox_keypair *kp; + const gchar *filename; + gchar *data; + guchar *out; + struct rspamd_lua_text *res; + gsize len = 0, outlen; + GError *err = NULL; + + kp = lua_check_cryptobox_keypair (L, 1); + filename = luaL_checkstring (L, 2); + data = rspamd_file_xmap (filename, PROT_READ, &len, TRUE); + + + if (!kp || !data) { + return luaL_error (L, "invalid arguments"); + } + + if (!rspamd_keypair_encrypt (kp, data, len, &out, &outlen, &err)) { + gint ret = luaL_error (L, "cannot encrypt file %s: %s", filename, + err->message); + g_error_free (err); + munmap (data, len); + + return ret; + } + + res = lua_newuserdata (L, sizeof (*res)); + res->flags = RSPAMD_TEXT_FLAG_OWN; + res->start = out; + res->len = outlen; + rspamd_lua_setclass (L, "rspamd{text}", -1); + munmap (data, len); + + return 1; +} + +/*** + * @function rspamd_cryptobox.decrypt_memory(kp, data) + * Encrypt data using specified keypair + * @param {keypair} kp keypair to use + * @param {string} data + * @return status,{rspamd_text}|error status is boolean variable followed by either unencrypted data or an error message + */ +static gint +lua_cryptobox_decrypt_memory (lua_State *L) +{ + struct rspamd_cryptobox_keypair *kp; + const gchar *data; + guchar *out; + struct rspamd_lua_text *t, *res; + gsize len = 0, outlen; + GError *err = NULL; + + kp = lua_check_cryptobox_keypair (L, 1); + + if (lua_isuserdata (L, 2)) { + t = lua_check_text (L, 2); + + if (!t) { + return luaL_error (L, "invalid arguments"); + } + + data = t->start; + len = t->len; + } + else { + data = luaL_checklstring (L, 2, &len); + } + + + if (!kp || !data) { + return luaL_error (L, "invalid arguments"); + } + + if (!rspamd_keypair_decrypt (kp, data, len, &out, &outlen, &err)) { + lua_pushboolean (L, false); + lua_pushstring (L, err->message); + g_error_free (err); + } + else { + lua_pushboolean (L, true); + res = lua_newuserdata (L, sizeof (*res)); + res->flags = RSPAMD_TEXT_FLAG_OWN; + res->start = out; + res->len = outlen; + rspamd_lua_setclass (L, "rspamd{text}", -1); + } + + return 2; +} + +/*** + * @function rspamd_cryptobox.decrypt_file(kp, filename) + * Encrypt data using specified keypair + * @param {keypair} kp keypair to use + * @param {string} filename + * @return status,{rspamd_text}|error status is boolean variable followed by either unencrypted data or an error message + */ +static gint +lua_cryptobox_decrypt_file (lua_State *L) +{ + struct rspamd_cryptobox_keypair *kp; + const gchar *filename; + gchar *data; + guchar *out; + struct rspamd_lua_text *res; + gsize len = 0, outlen; + GError *err = NULL; + + kp = lua_check_cryptobox_keypair (L, 1); + filename = luaL_checkstring (L, 2); + data = rspamd_file_xmap (filename, PROT_READ, &len, TRUE); + + + if (!kp || !data) { + return luaL_error (L, "invalid arguments"); + } + + if (!rspamd_keypair_decrypt (kp, data, len, &out, &outlen, &err)) { + lua_pushboolean (L, false); + lua_pushstring (L, err->message); + g_error_free (err); + } + else { + lua_pushboolean (L, true); + res = lua_newuserdata (L, sizeof (*res)); + res->flags = RSPAMD_TEXT_FLAG_OWN; + res->start = out; + res->len = outlen; + rspamd_lua_setclass (L, "rspamd{text}", -1); + } + + munmap (data, len); + + return 2; +} + static gint lua_load_pubkey (lua_State * L) { diff --git a/src/lua/lua_mimepart.c b/src/lua/lua_mimepart.c index 9c1d826d7..080bf4662 100644 --- a/src/lua/lua_mimepart.c +++ b/src/lua/lua_mimepart.c @@ -264,11 +264,17 @@ end LUA_FUNCTION_DEF (mimepart, get_header_full); /*** * @method mime_part:get_content() - * Get the raw content of part + * Get the parsed content of part * @return {text} opaque text object (zero-copy if not casted to lua string) */ LUA_FUNCTION_DEF (mimepart, get_content); /*** + * @method mime_part:get_raw_content() + * Get the raw content of part + * @return {text} opaque text object (zero-copy if not casted to lua string) + */ +LUA_FUNCTION_DEF (mimepart, get_raw_content); +/*** * @method mime_part:get_length() * Get length of the content of the part * @return {integer} length of part in **bytes** @@ -395,6 +401,7 @@ LUA_FUNCTION_DEF (mimepart, headers_foreach); static const struct luaL_reg mimepartlib_m[] = { LUA_INTERFACE_DEF (mimepart, get_content), + LUA_INTERFACE_DEF (mimepart, get_raw_content), LUA_INTERFACE_DEF (mimepart, get_length), LUA_INTERFACE_DEF (mimepart, get_type), LUA_INTERFACE_DEF (mimepart, get_type_full), @@ -918,6 +925,26 @@ lua_mimepart_get_content (lua_State * L) } static gint +lua_mimepart_get_raw_content (lua_State * L) +{ + struct rspamd_mime_part *part = lua_check_mimepart (L); + struct rspamd_lua_text *t; + + if (part == NULL) { + lua_pushnil (L); + return 1; + } + + t = lua_newuserdata (L, sizeof (*t)); + rspamd_lua_setclass (L, "rspamd{text}", -1); + t->start = part->raw_data.begin; + t->len = part->raw_data.len; + t->flags = 0; + + return 1; +} + +static gint lua_mimepart_get_length (lua_State * L) { struct rspamd_mime_part *part = lua_check_mimepart (L); diff --git a/src/lua/lua_task.c b/src/lua/lua_task.c index 3aa24e88b..1807888c8 100644 --- a/src/lua/lua_task.c +++ b/src/lua/lua_task.c @@ -1096,8 +1096,9 @@ lua_task_process_message (lua_State *L) if (task != NULL) { if (task->msg.len > 0) { - if (rspamd_message_parse (task) == 0) { + if (rspamd_message_parse (task)) { lua_pushboolean (L, TRUE); + rspamd_message_process (task); } else { lua_pushboolean (L, FALSE); @@ -3606,6 +3607,7 @@ lua_task_set_flag (lua_State *L) LUA_TASK_SET_FLAG (flag, "broken_headers", RSPAMD_TASK_FLAG_BROKEN_HEADERS, set); LUA_TASK_SET_FLAG (flag, "greylisted", RSPAMD_TASK_FLAG_GREYLISTED, set); + LUA_TASK_SET_FLAG (flag, "skip_process", RSPAMD_TASK_FLAG_SKIP_PROCESS, set); if (!found) { msg_warn_task ("unknown flag requested: %s", flag); @@ -3636,6 +3638,8 @@ lua_task_has_flag (lua_State *L) LUA_TASK_GET_FLAG (flag, "greylisted", RSPAMD_TASK_FLAG_GREYLISTED); LUA_TASK_GET_FLAG (flag, "broken_headers", RSPAMD_TASK_FLAG_BROKEN_HEADERS); + LUA_TASK_GET_FLAG (flag, "skip_process", + RSPAMD_TASK_FLAG_SKIP_PROCESS); LUA_TASK_GET_FLAG (flag, "milter", RSPAMD_TASK_FLAG_MILTER); @@ -3704,6 +3708,10 @@ lua_task_get_flags (lua_State *L) lua_pushstring (L, "greylisted"); lua_rawseti (L, -2, idx++); break; + case RSPAMD_TASK_FLAG_SKIP_PROCESS: + lua_pushstring (L, "skip_process"); + lua_rawseti (L, -2, idx++); + break; case RSPAMD_TASK_FLAG_MILTER: lua_pushstring (L, "milter"); lua_rawseti (L, -2, idx++); diff --git a/src/lua/lua_util.c b/src/lua/lua_util.c index 6c200d1be..f99ae7d1f 100644 --- a/src/lua/lua_util.c +++ b/src/lua/lua_util.c @@ -22,6 +22,7 @@ #include "linenoise.h" #include <math.h> #include <glob.h> +#include <zlib.h> #include "unicode/uspoof.h" @@ -356,6 +357,24 @@ LUA_FUNCTION_DEF (util, zstd_compress); LUA_FUNCTION_DEF (util, zstd_decompress); /*** + * @function util.gzip_decompress(data) + * Decompresses input using gzip algorithm + * + * @param {string/rspamd_text} data compressed data + * @return {error,rspamd_text} pair of error + decompressed text + */ +LUA_FUNCTION_DEF (util, gzip_decompress); + +/*** + * @function util.gzip_compress(data) + * Compresses input using gzip compression + * + * @param {string/rspamd_text} data input data + * @return {rspamd_text} compressed data + */ +LUA_FUNCTION_DEF (util, gzip_compress); + +/*** * @function util.normalize_prob(prob, [bias = 0.5]) * Normalize probabilities using polynom * @@ -558,6 +577,8 @@ static const struct luaL_reg utillib_f[] = { LUA_INTERFACE_DEF (util, random_hex), LUA_INTERFACE_DEF (util, zstd_compress), LUA_INTERFACE_DEF (util, zstd_decompress), + LUA_INTERFACE_DEF (util, gzip_compress), + LUA_INTERFACE_DEF (util, gzip_decompress), LUA_INTERFACE_DEF (util, normalize_prob), LUA_INTERFACE_DEF (util, caseless_hash), LUA_INTERFACE_DEF (util, caseless_hash_fast), @@ -626,9 +647,10 @@ lua_util_load_rspamd_config (lua_State *L) cfg_name = luaL_checkstring (L, 1); if (cfg_name) { - cfg = rspamd_config_new (); + cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_SKIP_LUA); + cfg->lua_state = L; - if (rspamd_config_read (cfg, cfg_name, NULL, NULL, NULL, NULL)) { + if (rspamd_config_read (cfg, cfg_name, NULL, NULL, NULL)) { msg_err_config ("cannot load config from %s", cfg_name); lua_pushnil (L); } @@ -655,11 +677,12 @@ lua_util_config_from_ucl (lua_State *L) if (obj) { cfg = g_malloc0 (sizeof (struct rspamd_config)); - cfg = rspamd_config_new (); + cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_SKIP_LUA); + cfg->lua_state = L; cfg->rcl_obj = obj; cfg->cache = rspamd_symbols_cache_new (cfg); - top = rspamd_rcl_config_init (cfg); + top = rspamd_rcl_config_init (cfg, NULL); if (!rspamd_rcl_parse (top, cfg, cfg, cfg->cfg_pool, cfg->rcl_obj, &err)) { msg_err_config ("rcl parse error: %s", err->message); @@ -1944,6 +1967,172 @@ lua_util_zstd_decompress (lua_State *L) } static gint +lua_util_gzip_compress (lua_State *L) +{ + struct rspamd_lua_text *t = NULL, *res, tmp; + gsize sz; + z_stream strm; + gint rc; + guchar *p; + gsize remain; + + if (lua_type (L, 1) == LUA_TSTRING) { + t = &tmp; + t->start = lua_tolstring (L, 1, &sz); + t->len = sz; + } + else { + t = lua_check_text (L, 1); + } + + if (t == NULL || t->start == NULL) { + return luaL_error (L, "invalid arguments"); + } + + + + memset (&strm, 0, sizeof (strm)); + rc = deflateInit2 (&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, + MAX_WBITS + 16, MAX_MEM_LEVEL - 1, Z_DEFAULT_STRATEGY); + + if (rc != Z_OK) { + return luaL_error (L, "cannot init zlib: %s", zError (rc)); + } + + sz = deflateBound (&strm, t->len); + + strm.avail_in = t->len; + strm.next_in = (guchar *)t->start; + + res = lua_newuserdata (L, sizeof (*res)); + res->start = g_malloc (sz); + res->flags = RSPAMD_TEXT_FLAG_OWN; + rspamd_lua_setclass (L, "rspamd{text}", -1); + + p = (guchar *)res->start; + remain = sz; + + while (strm.avail_in != 0) { + strm.avail_out = remain; + strm.next_out = p; + + rc = deflate (&strm, Z_FINISH); + + if (rc != Z_OK && rc != Z_BUF_ERROR) { + if (rc == Z_STREAM_END) { + break; + } + else { + msg_err ("cannot compress data: %s", zError (rc)); + lua_pop (L, 1); /* Text will be freed here */ + lua_pushnil (L); + deflateEnd (&strm); + + return 1; + } + } + + res->len = strm.total_out; + + if (strm.avail_out == 0 && strm.avail_in != 0) { + /* Need to allocate more */ + remain = res->len; + res->start = g_realloc ((gpointer)res->start, strm.avail_in + sz); + sz = strm.avail_in + sz; + p = (guchar *)res->start + remain; + remain = sz - remain; + } + } + + deflateEnd (&strm); + res->len = strm.total_out; + + return 1; +} + + +static gint +lua_util_gzip_decompress (lua_State *L) +{ + struct rspamd_lua_text *t = NULL, *res, tmp; + gsize sz; + z_stream strm; + gint rc; + guchar *p; + gsize remain; + + if (lua_type (L, 1) == LUA_TSTRING) { + t = &tmp; + t->start = lua_tolstring (L, 1, &sz); + t->len = sz; + } + else { + t = lua_check_text (L, 1); + } + + if (t == NULL || t->start == NULL) { + return luaL_error (L, "invalid arguments"); + } + + memset (&strm, 0, sizeof (strm)); + /* windowBits +16 to decode gzip, zlib 1.2.0.4+ */ + rc = inflateInit2 (&strm, MAX_WBITS + 16); + + if (rc != Z_OK) { + return luaL_error (L, "cannot init zlib"); + } + + strm.avail_in = t->len; + strm.next_in = (guchar *)t->start; + + res = lua_newuserdata (L, sizeof (*res)); + res->start = g_malloc (sz); + res->flags = RSPAMD_TEXT_FLAG_OWN; + rspamd_lua_setclass (L, "rspamd{text}", -1); + + p = (guchar *)res->start; + remain = sz; + + while (strm.avail_in != 0) { + strm.avail_out = remain; + strm.next_out = p; + + rc = inflate (&strm, Z_FINISH); + + if (rc != Z_OK && rc != Z_BUF_ERROR) { + if (rc == Z_STREAM_END) { + break; + } + else { + msg_err ("cannot decompress data: %s", zError (rc)); + lua_pop (L, 1); /* Text will be freed here */ + lua_pushnil (L); + inflateEnd (&strm); + + return 1; + } + } + + res->len = strm.total_out; + + if (strm.avail_out == 0 && strm.avail_in != 0) { + /* Need to allocate more */ + remain = res->len; + res->start = g_realloc ((gpointer)res->start, strm.avail_in + sz); + sz = strm.avail_in + sz; + p = (guchar *)res->start + remain; + remain = sz - remain; + } + } + + inflateEnd (&strm); + res->len = strm.total_out; + + return 2; +} + + +static gint lua_util_normalize_prob (lua_State *L) { gdouble x, bias = 0.5; diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c index 0873d3c7a..c4318777f 100644 --- a/src/plugins/fuzzy_check.c +++ b/src/plugins/fuzzy_check.c @@ -3019,6 +3019,8 @@ fuzzy_process_handler (struct rspamd_http_connection_entry *conn_ent, "Message processing error"); return; } + + rspamd_message_process (task); } PTR_ARRAY_FOREACH (fuzzy_module_ctx->fuzzy_rules, i, rule) { diff --git a/src/plugins/lua/dmarc.lua b/src/plugins/lua/dmarc.lua index 58e48bd01..5c24bff53 100644 --- a/src/plugins/lua/dmarc.lua +++ b/src/plugins/lua/dmarc.lua @@ -66,10 +66,10 @@ Content-Transfer-Encoding: 7bit This is an aggregate report from %s. ------=_NextPart_000_024E_01CC9B0A.AFE54C00 -Content-Type: text/xml +Content-Type: application/gzip Content-Transfer-Encoding: base64 Content-Disposition: attachment; - filename="%s!%s!%s!%s.xml" + filename="%s!%s!%s!%s.xml.gz" ]] local report_footer = [[ @@ -744,10 +744,11 @@ if opts['reporting'] == true then for k in pairs(reporting_addr) do table.insert(tmp_addr, k) end - local encoded = rspamd_util.encode_base64(table.concat( + local encoded = rspamd_util.encode_base64(rspamd_util.gzip_compress( + table.concat( {xmlf('header'), xmlf('entries'), - xmlf('footer')}), 78) + xmlf('footer')})), 78) local function mail_cb(err, data, conn) local function no_error(merr, mdata, wantcode) wantcode = wantcode or '2' @@ -808,11 +809,22 @@ if opts['reporting'] == true then table.insert(atmp, k) end local addr_string = table.concat(atmp, ', ') - local rhead = string.format(report_template, report_settings.email, addr_string, - reporting_domain, report_settings.domain, report_id, rspamd_util.time_to_string(rspamd_util.get_time()), - rspamd_util.random_hex(12) .. '@rspamd', report_settings.domain, report_settings.domain, reporting_domain, - report_start, report_end) - conn:add_write(pre_quit_cb, {rhead, encoded, report_footer, '\r\n.\r\n'}) + local rhead = string.format(report_template, + report_settings.email, + addr_string, + reporting_domain, + report_settings.domain, + report_id, + rspamd_util.time_to_string(rspamd_util.get_time()), + rspamd_util.random_hex(12) .. '@rspamd', + report_settings.domain, + report_settings.domain, + reporting_domain, + report_start, report_end) + conn:add_write(pre_quit_cb, {rhead, + encoded, + report_footer, + '\r\n.\r\n'}) end end local function data_cb(merr, mdata) diff --git a/src/plugins/lua/elastic.lua b/src/plugins/lua/elastic.lua index 4caedd684..3a80256d2 100644 --- a/src/plugins/lua/elastic.lua +++ b/src/plugins/lua/elastic.lua @@ -46,6 +46,7 @@ local settings = { failover = false, import_kibana = false, use_https = false, + use_gzip = true, allow_local = false, } @@ -106,6 +107,7 @@ local function elastic_send_data(task) body = bulk_json, task = task, method = 'post', + gzip = settings.use_gzip, callback = http_index_data_callback }) @@ -305,6 +307,7 @@ local function initial_setup(cfg, ev_base, worker) }, body = table.concat(tbl, "\n"), method = 'post', + gzip = settings.use_gzip, callback = kibana_template_callback }) else @@ -341,6 +344,7 @@ local function initial_setup(cfg, ev_base, worker) headers = { ['Content-Type'] = 'application/json', }, + gzip = settings.use_gzip, body = ucl.to_format(template, 'json-compact'), method = 'put', }) @@ -366,6 +370,7 @@ local function initial_setup(cfg, ev_base, worker) headers = { ['Content-Type'] = 'application/json', }, + gzip = settings.use_gzip, callback = http_template_put_callback, }) else diff --git a/src/plugins/lua/mime_types.lua b/src/plugins/lua/mime_types.lua index 43142132c..f97f22d2a 100644 --- a/src/plugins/lua/mime_types.lua +++ b/src/plugins/lua/mime_types.lua @@ -344,7 +344,7 @@ local full_extensions_map = { {"group", "text/x-ms-group"}, {"gsm", "audio/x-gsm"}, {"gtar", "application/x-gtar"}, - {"gz", "application/x-gzip"}, + {"gz", {"application/gzip", "application/x-gzip"}}, {"h", "text/plain"}, {"hdf", "application/x-hdf"}, {"hdml", "text/x-hdml"}, @@ -773,7 +773,7 @@ local full_extensions_map = { {"xls", {"application/vnd.ms-excel", "application/vnd.ms-office", "application/x-excel", "application/octet-stream"}}, {"xlsb", "application/vnd.ms-excel.sheet.binary.macroEnabled.12"}, {"xlsm", "application/vnd.ms-excel.sheet.macroEnabled.12"}, - {"xlsx", {"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/octet-stream"}}, + {"xlsx", {"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/vnd.ms-excel.12", "application/octet-stream"}}, {"xlt", "application/vnd.ms-excel"}, {"xltm", "application/vnd.ms-excel.template.macroEnabled.12"}, {"xltx", "application/vnd.openxmlformats-officedocument.spreadsheetml.template"}, @@ -798,6 +798,7 @@ local full_extensions_map = { {"xwd", "image/x-xwindowdump"}, {"z", "application/x-compress"}, {"zip", {"application/zip", "application/x-zip-compressed", "application/octet-stream"}}, + {"zlib", "application/zlib"}, } local function check_mime_type(task) diff --git a/src/plugins/lua/phishing.lua b/src/plugins/lua/phishing.lua index e76ffc79a..69aec219f 100644 --- a/src/plugins/lua/phishing.lua +++ b/src/plugins/lua/phishing.lua @@ -75,8 +75,12 @@ local function phishing_cb(task) for _,d in ipairs(elt) do if not d['path'] then found_path = true + end + + if query and d['query'] and query == d['query'] then + found_query = true + elseif not d['query'] then found_query = true - break end end end @@ -101,7 +105,10 @@ local function phishing_cb(task) task:insert_result(phish_symbol, 1.0, args) else -- Host + path match - task:insert_result(phish_symbol, 0.3, args) + if path then + task:insert_result(phish_symbol, 0.3, args) + end + -- No path, no symbol end else if url:is_phished() then diff --git a/src/plugins/lua/ratelimit.lua b/src/plugins/lua/ratelimit.lua index e060fe084..1d22f8838 100644 --- a/src/plugins/lua/ratelimit.lua +++ b/src/plugins/lua/ratelimit.lua @@ -377,8 +377,16 @@ local function ratelimit_cb(task) if err then rspamd_logger.errx('cannot check limit %s: %s %s', prefix, err, data) elseif type(data) == 'table' and data[1] and data[1] == 1 then - if settings.info_symbol then - task:insert_result(settings.info_symbol, 1.0, prefix) + -- set symbol only and do NOT soft reject + if settings.symbol then + task:insert_result(settings.symbol, 0.0, lim_name .. "(" .. prefix .. ")") + rspamd_logger.infox(task, + 'set_symbol_only: ratelimit "%s(%s)" exceeded, (%s / %s): %s (%s:%s dyn)', + lim_name, prefix, bucket[2], bucket[1], data[2], data[3], data[4]) + return + -- set INFO symbol and soft reject + elseif settings.info_symbol then + task:insert_result(settings.info_symbol, 1.0, lim_name .. "(" .. prefix .. ")") end rspamd_logger.infox(task, 'ratelimit "%s(%s)" exceeded, (%s / %s): %s (%s:%s dyn)', diff --git a/src/plugins/lua/settings.lua b/src/plugins/lua/settings.lua index e1a4cef74..8e4009162 100644 --- a/src/plugins/lua/settings.lua +++ b/src/plugins/lua/settings.lua @@ -37,6 +37,26 @@ local rspamd_regexp = require "rspamd_regexp" local ucl = require "ucl" local fun = require "fun" +local function apply_settings(task, to_apply) + task:set_settings(to_apply) + task:cache_set('settings', to_apply) + lua_squeeze.handle_settings(task, to_apply) + + if to_apply['add_headers'] or to_apply['remove_headers'] then + local rep = { + add_headers = to_apply['add_headers'] or {}, + remove_headers = to_apply['remove_headers'] or {}, + } + task:set_rmilter_reply(rep) + end + + if to_apply.flags and type(to_apply.flags) == 'table' then + for _,fl in ipairs(to_apply.flags) do + task:set_flag(fl) + end + end +end + -- Checks for overridden settings within query params and returns 'true' if -- settings are overridden local function check_query_settings(task) @@ -47,9 +67,7 @@ local function check_query_settings(task) local res,err = parser:parse_string(tostring(query_set)) if res then local settings_obj = parser:get_object() - task:set_settings(settings_obj) - task:cache_set('settings', settings_obj) - lua_squeeze.handle_settings(task, settings_obj) + apply_settings(task, settings_obj) return true else @@ -76,10 +94,7 @@ local function check_query_settings(task) nset['default']['actions']['add header'] = ss end - task:set_settings(nset) - task:cache_set('settings', nset) - lua_squeeze.handle_settings(task, nset) - + apply_settings(task, nset) return true end end @@ -90,19 +105,8 @@ local function check_query_settings(task) local id_str = tostring(settings_id) local elt = settings_ids[id_str] if elt and elt['apply'] then - task:set_settings(elt['apply']) - lua_squeeze.handle_settings(task, elt['apply']) - task:cache_set('settings', elt['apply']) - - if elt.apply['add_headers'] or elt.apply['remove_headers'] then - local rep = { - add_headers = elt.apply['add_headers'] or {}, - remove_headers = elt.apply['remove_headers'] or {}, - } - task:set_rmilter_reply(rep) - end + apply_settings(task, elt['apply']) rspamd_logger.infox(task, "applying settings id %s", id_str) - return true end end @@ -340,9 +344,7 @@ local function check_settings(task) rspamd_logger.infox(task, "<%1> apply settings according to rule %2", task:get_message_id(), s.name) if rule['apply'] then - task:set_settings(rule['apply']) - lua_squeeze.handle_settings(task, rule['apply']) - task:cache_set('settings', rule['apply']) + apply_settings(task, rule['apply']) applied = true end if rule['symbols'] then @@ -653,9 +655,7 @@ local function gen_redis_callback(handler, id) local obj = parser:get_object() rspamd_logger.infox(task, "<%1> apply settings according to redis rule %2", task:get_message_id(), id) - task:set_settings(obj) - lua_squeeze.handle_settings(task, obj) - task:cache_set('settings', obj) + apply_settings(task, obj) break end end diff --git a/src/rspamadm/CMakeLists.txt b/src/rspamadm/CMakeLists.txt index fb3f25229..2105c16b3 100644 --- a/src/rspamadm/CMakeLists.txt +++ b/src/rspamadm/CMakeLists.txt @@ -1,21 +1,16 @@ SET(RSPAMADMSRC rspamadm.c commands.c pw.c - keypair.c configtest.c fuzzy_convert.c fuzzy_merge.c - grep.c configdump.c control.c confighelp.c - configwizard.c - corpus_test.c stat_convert.c signtool.c lua_repl.c dkim_keygen.c - rescore.c ${CMAKE_BINARY_DIR}/src/workers.c ${CMAKE_BINARY_DIR}/src/modules.c ${CMAKE_SOURCE_DIR}/src/controller.c diff --git a/src/rspamadm/commands.c b/src/rspamadm/commands.c index 410306fe3..9216c79b6 100644 --- a/src/rspamadm/commands.c +++ b/src/rspamadm/commands.c @@ -14,9 +14,11 @@ * limitations under the License. */ #include "rspamadm.h" +#include "libutil/util.h" +#include "libutil/logger.h" +#include "lua/lua_common.h" extern struct rspamadm_command pw_command; -extern struct rspamadm_command keypair_command; extern struct rspamadm_command configtest_command; extern struct rspamadm_command fuzzy_merge_command; extern struct rspamadm_command configdump_command; @@ -24,18 +26,13 @@ extern struct rspamadm_command control_command; extern struct rspamadm_command confighelp_command; extern struct rspamadm_command statconvert_command; extern struct rspamadm_command fuzzyconvert_command; -extern struct rspamadm_command grep_command; extern struct rspamadm_command signtool_command; extern struct rspamadm_command lua_command; extern struct rspamadm_command dkim_keygen_command; -extern struct rspamadm_command configwizard_command; -extern struct rspamadm_command corpus_test_command; -extern struct rspamadm_command rescore_command; const struct rspamadm_command *commands[] = { &help_command, &pw_command, - &keypair_command, &configtest_command, &fuzzy_merge_command, &configdump_command, @@ -43,31 +40,33 @@ const struct rspamadm_command *commands[] = { &confighelp_command, &statconvert_command, &fuzzyconvert_command, - &grep_command, &signtool_command, &lua_command, &dkim_keygen_command, - &configwizard_command, - &corpus_test_command, - &rescore_command, NULL }; const struct rspamadm_command * -rspamadm_search_command (const gchar *name) +rspamadm_search_command (const gchar *name, GPtrArray *all_commands) { - const struct rspamadm_command *ret = NULL; - guint i; + const struct rspamadm_command *ret = NULL, *cmd; + const gchar *alias; + guint i, j; if (name == NULL) { name = "help"; } - for (i = 0; i < G_N_ELEMENTS (commands); i ++) { - if (commands[i] != NULL) { - if (strcmp (name, commands[i]->name) == 0) { - ret = commands[i]; + PTR_ARRAY_FOREACH (all_commands, i, cmd) { + if (strcmp (name, cmd->name) == 0) { + ret = cmd; + break; + } + + PTR_ARRAY_FOREACH (cmd->aliases, j, alias) { + if (strcmp (name, alias) == 0) { + ret = cmd; break; } } @@ -75,3 +74,220 @@ rspamadm_search_command (const gchar *name) return ret; } + +void +rspamadm_fill_internal_commands (GPtrArray *dest) +{ + guint i; + + for (i = 0; i < G_N_ELEMENTS (commands); i ++) { + if (commands[i]) { + g_ptr_array_add (dest, (gpointer)commands[i]); + } + } +} + +static void +rspamadm_lua_command_run (gint argc, gchar **argv, + const struct rspamadm_command *cmd) +{ + gint table_idx = GPOINTER_TO_INT (cmd->command_data); + gint i, err_idx, ret; + GString *tb; + + lua_pushcfunction (L, &rspamd_lua_traceback); + err_idx = lua_gettop (L); + + /* Function */ + lua_rawgeti (L, LUA_REGISTRYINDEX, table_idx); + lua_pushstring (L, "handler"); + lua_gettable (L, -2); + + /* Args */ + lua_createtable (L, argc + 1, 0); + + for (i = 0; i < argc; i ++) { + lua_pushstring (L, argv[i]); + lua_rawseti (L, -2, i); /* Starting from zero ! */ + } + + if ((ret = lua_pcall (L, 1, 0, err_idx)) != 0) { + tb = lua_touserdata (L, -1); + msg_err ("call to rspamadm lua script %s failed (%d): %v", cmd->name, + ret, tb); + + if (tb) { + g_string_free (tb, TRUE); + } + + lua_settop (L, 0); + + exit (EXIT_FAILURE); + } + + lua_settop (L, 0); +} + +static const gchar * +rspamadm_lua_command_help (gboolean full_help, + const struct rspamadm_command *cmd) +{ + gint table_idx = GPOINTER_TO_INT (cmd->command_data); + gint err_idx, ret; + GString *tb; + + if (full_help) { + lua_pushcfunction (L, &rspamd_lua_traceback); + err_idx = lua_gettop (L); + + lua_rawgeti (L, LUA_REGISTRYINDEX, table_idx); + /* Function */ + lua_pushstring (L, "handler"); + lua_gettable (L, -2); + + /* Args */ + lua_createtable (L, 2, 0); + lua_pushstring (L, cmd->name); + lua_rawseti (L, -2, 0); /* Starting from zero ! */ + + lua_pushstring (L, "--help"); + lua_rawseti (L, -2, 1); + + if ((ret = lua_pcall (L, 1, 0, err_idx)) != 0) { + tb = lua_touserdata (L, -1); + msg_err ("call to rspamadm lua script %s failed (%d): %v", cmd->name, + ret, tb); + + if (tb) { + g_string_free (tb, TRUE); + } + + lua_settop (L, 0); + + exit (EXIT_FAILURE); + } + } + else { + lua_rawgeti (L, LUA_REGISTRYINDEX, table_idx); + lua_pushstring (L, "description"); + lua_gettable (L, -2); + + if (lua_isstring (L, -1)) { + printf (" %-18s %-60s\n", cmd->name, lua_tostring (L, -1)); + } + else { + printf (" %-18s %-60s\n", cmd->name, "no description available"); + } + } + + lua_settop (L, 0); + + return NULL; /* Must be handled in rspamadm itself */ +} + +void +rspamadm_fill_lua_commands (lua_State *L, GPtrArray *dest) +{ + gint i; + + GPtrArray *lua_paths; + GError *err = NULL; + const gchar *lualibdir = RSPAMD_LUALIBDIR, *path; + struct rspamadm_command *lua_cmd; + gchar search_dir[PATH_MAX]; + + if (g_hash_table_lookup (ucl_vars, "LUALIBDIR")) { + lualibdir = g_hash_table_lookup (ucl_vars, "LUALIBDIR"); + } + + rspamd_snprintf (search_dir, sizeof (search_dir), "%s%crspamadm%c", + lualibdir, G_DIR_SEPARATOR, G_DIR_SEPARATOR); + + if ((lua_paths = rspamd_glob_path (search_dir, "*.lua", FALSE, &err)) == NULL) { + msg_err ("cannot glob files in %s/*.lua: %e", search_dir, err); + g_error_free (err); + + return; + } + + PTR_ARRAY_FOREACH (lua_paths, i, path) { + if (luaL_dofile (L, path) != 0) { + msg_err ("cannot execute lua script %s: %s", + path, lua_tostring (L, -1)); + lua_settop (L, 0); + continue; + } else { + if (lua_type (L, -1) == LUA_TTABLE) { + lua_pushstring (L, "handler"); + lua_gettable (L, -2); + } + else { + continue; /* Something goes wrong, huh */ + } + + if (lua_type (L, -1) != LUA_TFUNCTION) { + msg_err ("rspamadm script %s does not have 'handler' field with type " + "function", + path); + continue; + } + + /* Pop handler */ + lua_pop (L, 1); + lua_cmd = g_malloc0 (sizeof (*lua_cmd)); + + lua_pushstring (L, "name"); + lua_gettable (L, -2); + + if (lua_type (L, -1) == LUA_TSTRING) { + lua_cmd->name = g_strdup (lua_tostring (L, -1)); + } + else { + goffset ext_pos; + gchar *name; + + name = g_path_get_basename (path); + /* Remove .lua */ + ext_pos = rspamd_substring_search (path, strlen (path), ".lua", 4); + + if (ext_pos != -1) { + name[ext_pos] = '\0'; + } + + lua_cmd->name = name; + } + + lua_pop (L, 1); + + lua_pushstring (L, "aliases"); + lua_gettable (L, -2); + + if (lua_type (L, -1) == LUA_TTABLE) { + lua_cmd->aliases = g_ptr_array_new_full ( + rspamd_lua_table_size (L, -1), + g_free); + + for (lua_pushnil (L); lua_next (L, -2); lua_pop (L, 1)) { + if (lua_isstring (L, -1)) { + g_ptr_array_add (lua_cmd->aliases, + g_strdup (lua_tostring (L, -1))); + } + } + } + + lua_pop (L, 1); + + lua_pushvalue (L, -1); + /* Reference table itself */ + lua_cmd->command_data = GINT_TO_POINTER (luaL_ref (L, LUA_REGISTRYINDEX)); + lua_cmd->flags |= RSPAMADM_FLAG_LUA; + lua_cmd->run = rspamadm_lua_command_run; + lua_cmd->help = rspamadm_lua_command_help; + + + g_ptr_array_add (dest, lua_cmd); + } + + lua_settop (L, 0); + } +} diff --git a/src/rspamadm/configdump.c b/src/rspamadm/configdump.c index d5380bc9d..cd9f6d378 100644 --- a/src/rspamadm/configdump.c +++ b/src/rspamadm/configdump.c @@ -33,8 +33,8 @@ extern struct rspamd_main *rspamd_main; extern module_t *modules[]; extern worker_t *workers[]; -static void rspamadm_configdump (gint argc, gchar **argv); -static const char *rspamadm_configdump_help (gboolean full_help); +static void rspamadm_configdump (gint argc, gchar **argv, const struct rspamadm_command *); +static const char *rspamadm_configdump_help (gboolean full_help, const struct rspamadm_command *); struct rspamadm_command configdump_command = { .name = "configdump", @@ -61,7 +61,7 @@ static GOptionEntry entries[] = { }; static const char * -rspamadm_configdump_help (gboolean full_help) +rspamadm_configdump_help (gboolean full_help, const struct rspamadm_command *cmd) { const char *help_str; @@ -237,7 +237,7 @@ rspamadm_dump_section_obj (struct rspamd_config *cfg, } static void -rspamadm_configdump (gint argc, gchar **argv) +rspamadm_configdump (gint argc, gchar **argv, const struct rspamadm_command *cmd) { GOptionContext *context; GError *error = NULL; @@ -278,13 +278,13 @@ rspamadm_configdump (gint argc, gchar **argv) (void) g_quark_from_static_string ((*pworker)->name); pworker++; } + cfg->cache = rspamd_symbols_cache_new (cfg); cfg->compiled_modules = modules; cfg->compiled_workers = workers; cfg->cfg_name = config; - if (!rspamd_config_read (cfg, cfg->cfg_name, NULL, - config_logger, rspamd_main, ucl_vars)) { + if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main, ucl_vars)) { ret = FALSE; } else { diff --git a/src/rspamadm/confighelp.c b/src/rspamadm/confighelp.c index 8f208b805..c77cb6091 100644 --- a/src/rspamadm/confighelp.c +++ b/src/rspamadm/confighelp.c @@ -30,9 +30,11 @@ extern struct rspamd_main *rspamd_main; extern module_t *modules[]; extern worker_t *workers[]; -static void rspamadm_confighelp (gint argc, gchar **argv); +static void rspamadm_confighelp (gint argc, gchar **argv, + const struct rspamadm_command *cmd); -static const char *rspamadm_confighelp_help (gboolean full_help); +static const char *rspamadm_confighelp_help (gboolean full_help, + const struct rspamadm_command *cmd); struct rspamadm_command confighelp_command = { .name = "confighelp", @@ -55,7 +57,7 @@ static GOptionEntry entries[] = { }; static const char * -rspamadm_confighelp_help (gboolean full_help) +rspamadm_confighelp_help (gboolean full_help, const struct rspamadm_command *cmd) { const char *help_str; @@ -67,7 +69,7 @@ rspamadm_confighelp_help (gboolean full_help) "-j: output pretty formatted JSON\n" "-k: search by keyword in doc string\n" "-P: use specific Lua plugins path\n" - "--no-color: show colored output\n" + "--no-color: disable coloured output\n" "--short: show only option names\n" "--no-examples: do not show examples (impied by --short)\n" "--help: shows available options and commands"; @@ -189,7 +191,7 @@ rspamadm_confighelp_search_word (const ucl_object_t *obj, const gchar *str) } static void -rspamadm_confighelp (gint argc, gchar **argv) +rspamadm_confighelp (gint argc, gchar **argv, const struct rspamadm_command *cmd) { struct rspamd_config *cfg; ucl_object_t *doc_obj; @@ -224,11 +226,12 @@ rspamadm_confighelp (gint argc, gchar **argv) pworker++; } - cfg = rspamd_config_new (); + cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_SKIP_LUA); + cfg->lua_state = L; cfg->compiled_modules = modules; cfg->compiled_workers = workers; - rspamd_rcl_config_init (cfg); + rspamd_rcl_config_init (cfg, NULL); lua_pushboolean (cfg->lua_state, true); lua_setglobal (cfg->lua_state, "confighelp"); rspamd_rcl_add_lua_plugins_path (cfg, plugins_path, NULL); diff --git a/src/rspamadm/configtest.c b/src/rspamadm/configtest.c index 18104c109..72a8f4945 100644 --- a/src/rspamadm/configtest.c +++ b/src/rspamadm/configtest.c @@ -28,8 +28,10 @@ extern struct rspamd_main *rspamd_main; extern module_t *modules[]; extern worker_t *workers[]; -static void rspamadm_configtest (gint argc, gchar **argv); -static const char *rspamadm_configtest_help (gboolean full_help); +static void rspamadm_configtest (gint argc, gchar **argv, + const struct rspamadm_command *cmd); +static const char *rspamadm_configtest_help (gboolean full_help, + const struct rspamadm_command *cmd); struct rspamadm_command configtest_command = { .name = "configtest", @@ -50,7 +52,7 @@ static GOptionEntry entries[] = { }; static const char * -rspamadm_configtest_help (gboolean full_help) +rspamadm_configtest_help (gboolean full_help, const struct rspamadm_command *cmd) { const char *help_str; @@ -94,7 +96,7 @@ config_logger (rspamd_mempool_t *pool, gpointer ud) } static void -rspamadm_configtest (gint argc, gchar **argv) +rspamadm_configtest (gint argc, gchar **argv, const struct rspamadm_command *cmd) { GOptionContext *context; GError *error = NULL; @@ -139,8 +141,7 @@ rspamadm_configtest (gint argc, gchar **argv) cfg->compiled_workers = workers; cfg->cfg_name = config; - if (!rspamd_config_read (cfg, cfg->cfg_name, NULL, - config_logger, rspamd_main, ucl_vars)) { + if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main, ucl_vars)) { ret = FALSE; } else { diff --git a/src/rspamadm/configwizard.c b/src/rspamadm/configwizard.c deleted file mode 100644 index 12c7b8cc1..000000000 --- a/src/rspamadm/configwizard.c +++ /dev/null @@ -1,165 +0,0 @@ -/*- - * Copyright 2017 Vsevolod Stakhov - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "config.h" -#include "rspamadm.h" -#include "cfg_file.h" -#include "cfg_rcl.h" -#include "utlist.h" -#include "rspamd.h" -#include "lua/lua_common.h" -#include "utlist.h" - -static gchar *config = NULL; -extern struct rspamd_main *rspamd_main; -/* Defined in modules.c */ -extern module_t *modules[]; -extern worker_t *workers[]; - -static void rspamadm_configwizard (gint argc, gchar **argv); -static const char *rspamadm_configwizard_help (gboolean full_help); - -struct rspamadm_command configwizard_command = { - .name = "configwizard", - .flags = 0, - .help = rspamadm_configwizard_help, - .run = rspamadm_configwizard, - .lua_subrs = NULL, -}; - -static GOptionEntry entries[] = { - {"config", 'c', 0, G_OPTION_ARG_STRING, &config, - "Config file to use", NULL}, - {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL} -}; - -static const char * -rspamadm_configwizard_help (gboolean full_help) -{ - const char *help_str; - - if (full_help) { - help_str = "Perform guided configuration for Rspamd daemon\n\n" - "Usage: rspamadm configwizard [-c <config_name>] [checks...]\n" - " rspamadm configwizard [-c <config_name>] list\n" - "Where options are:\n\n" - "--help: shows available options and commands"; - } - else { - help_str = "Perform guided configuration for Rspamd daemon"; - } - - return help_str; -} - -static void -config_logger (rspamd_mempool_t *pool, gpointer ud) -{ - struct rspamd_main *rm = ud; - - rm->cfg->log_type = RSPAMD_LOG_CONSOLE; - rm->cfg->log_level = G_LOG_LEVEL_MESSAGE; - - rspamd_set_logger (rm->cfg, g_quark_try_string ("main"), &rm->logger, - rm->server_pool); - - if (rspamd_log_open_priv (rm->logger, rm->workers_uid, rm->workers_gid) == - -1) { - fprintf (stderr, "Fatal error, cannot open logfile, exiting\n"); - exit (EXIT_FAILURE); - } -} - -static void -rspamadm_configwizard (gint argc, gchar **argv) -{ - GOptionContext *context; - GError *error = NULL; - const gchar *confdir; - struct rspamd_config *cfg = rspamd_main->cfg; - gboolean ret = TRUE; - worker_t **pworker; - lua_State *L; - - context = g_option_context_new ( - "configwizard - perform guided configuration"); - g_option_context_set_summary (context, - "Summary:\n Rspamd administration utility version " - RVERSION - "\n Release id: " - RID); - g_option_context_add_main_entries (context, entries, NULL); - - if (!g_option_context_parse (context, &argc, &argv, &error)) { - fprintf (stderr, "option parsing failed: %s\n", error->message); - g_error_free (error); - exit (1); - } - - if (config == NULL) { - if ((confdir = g_hash_table_lookup (ucl_vars, "CONFDIR")) == NULL) { - confdir = RSPAMD_CONFDIR; - } - - config = g_strdup_printf ("%s%c%s", confdir, G_DIR_SEPARATOR, - "rspamd.conf"); - } - - pworker = &workers[0]; - while (*pworker) { - /* Init string quarks */ - (void) g_quark_from_static_string ((*pworker)->name); - pworker++; - } - - cfg->cache = rspamd_symbols_cache_new (cfg); - cfg->compiled_modules = modules; - cfg->compiled_workers = workers; - cfg->cfg_name = config; - - if (!rspamd_config_read (cfg, cfg->cfg_name, NULL, - config_logger, rspamd_main, ucl_vars)) { - ret = FALSE; - } - else { - /* Do post-load actions */ - rspamd_lua_post_load_config (cfg); - - if (!rspamd_init_filters (cfg, FALSE)) { - ret = FALSE; - } - - if (ret) { - ret = rspamd_config_post_load (cfg, RSPAMD_CONFIG_INIT_SYMCACHE); - } - } - - if (ret) { - L = cfg->lua_state; - rspamd_lua_set_path (L, cfg->rcl_obj, ucl_vars); - ucl_object_insert_key (cfg->rcl_obj, ucl_object_fromstring (cfg->cfg_name), - "config_path", 0, false); - - rspamadm_execute_lua_ucl_subr (L, - argc, - argv, - cfg->rcl_obj, - "configwizard"); - - lua_close (L); - } - - exit (ret ? EXIT_SUCCESS : EXIT_FAILURE); -} diff --git a/src/rspamadm/control.c b/src/rspamadm/control.c index 35cecbc69..4d7806b84 100644 --- a/src/rspamadm/control.c +++ b/src/rspamadm/control.c @@ -31,8 +31,10 @@ static gboolean ucl = TRUE; static gboolean compact = FALSE; static gdouble timeout = 1.0; -static void rspamadm_control (gint argc, gchar **argv); -static const char *rspamadm_control_help (gboolean full_help); +static void rspamadm_control (gint argc, gchar **argv, + const struct rspamadm_command *cmd); +static const char *rspamadm_control_help (gboolean full_help, + const struct rspamadm_command *cmd); struct rspamadm_command control_command = { .name = "control", @@ -64,7 +66,7 @@ static GOptionEntry entries[] = { }; static const char * -rspamadm_control_help (gboolean full_help) +rspamadm_control_help (gboolean full_help, const struct rspamadm_command *cmd) { const char *help_str; @@ -158,7 +160,7 @@ rspamd_control_finish_handler (struct rspamd_http_connection *conn, } static void -rspamadm_control (gint argc, gchar **argv) +rspamadm_control (gint argc, gchar **argv, const struct rspamadm_command *_cmd) { GOptionContext *context; GError *error = NULL; @@ -260,6 +262,5 @@ rspamadm_control (gint argc, gchar **argv) rspamd_http_connection_unref (conn); rspamd_inet_address_free (addr); - lua_close (L); close (sock); } diff --git a/src/rspamadm/corpus_test.c b/src/rspamadm/corpus_test.c deleted file mode 100644 index d72788d21..000000000 --- a/src/rspamadm/corpus_test.c +++ /dev/null @@ -1,125 +0,0 @@ -/*- - * Copyright 2017 Pragadeesh C - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "rspamadm.h" -#include "config.h" -#include "lua/lua_common.h" - -static gchar *ham_directory = NULL; -static gchar *spam_directory = NULL; -static gchar *output_location = "results.log"; -static gint connections = 10; -static gdouble timeout = 60.0; - -static void rspamadm_corpus_test (gint argc, gchar **argv); -static const char *rspamadm_corpus_test_help (gboolean full_help); - -struct rspamadm_command corpus_test_command = { - .name = "corpus_test", - .flags = 0, - .help = rspamadm_corpus_test_help, - .run = rspamadm_corpus_test -}; - -static GOptionEntry entries[] = { - {"ham", 'h', 0, G_OPTION_ARG_FILENAME, &ham_directory, - "Ham directory", NULL}, - {"spam", 's', 0, G_OPTION_ARG_FILENAME, &spam_directory, - "Spam directory", NULL}, - {"output", 'o', 0, G_OPTION_ARG_FILENAME, &output_location, - "Log output location", NULL}, - {"connections", 'n', 0, G_OPTION_ARG_INT, &connections, - "Number of parellel connections [Default: 10]", NULL}, - {"timeout", 't', 0, G_OPTION_ARG_DOUBLE, &timeout, - "Timeout for connections [Default: 60]", NULL}, - {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL} -}; - -static const char * -rspamadm_corpus_test_help (gboolean full_help) -{ - const char *help_str; - - if (full_help) { - help_str = "Create logs files from email corpus\n\n" - "Usage: rspamadm corpus_test [-h <ham_directory>]" - " [-s <spam_directory>]\n" - "Where option are:\n\n" - "-h: path to ham directory\n" - "-s: path to spam directory\n" - "-n: maximum parallel connections\n" - "-o: log output file\n" - "-t: timeout for rspamc operations (default: 60)\n"; - - } - - else { - help_str = "Create logs files from email corpus"; - } - - return help_str; -} - -static void -rspamadm_corpus_test (gint argc, gchar **argv) -{ - GOptionContext *context; - GError *error = NULL; - lua_State *L; - ucl_object_t *obj; - - context = g_option_context_new ( - "corpus_test - create logs files from email corpus"); - - g_option_context_set_summary (context, - "Summary:\n Rspamd administration utility version " - RVERSION - "\n Release id: " - RID); - - g_option_context_add_main_entries (context, entries, NULL); - g_option_context_set_ignore_unknown_options (context, TRUE); - - if (!g_option_context_parse (context, &argc, &argv, &error)) { - rspamd_fprintf (stderr, "option parsing failed: %s\n", error->message); - g_error_free (error); - exit(1); - } - - L = rspamd_lua_init (); - rspamd_lua_set_path(L, NULL, ucl_vars); - - obj = ucl_object_typed_new (UCL_OBJECT); - ucl_object_insert_key (obj, ucl_object_fromstring (ham_directory), - "ham_directory", 0, false); - ucl_object_insert_key (obj, ucl_object_fromstring (spam_directory), - "spam_directory", 0, false); - ucl_object_insert_key (obj, ucl_object_fromstring (output_location), - "output_location", 0, false); - ucl_object_insert_key (obj, ucl_object_fromint (connections), - "connections", 0, false); - ucl_object_insert_key (obj, ucl_object_fromdouble (timeout), - "timeout", 0, false); - - rspamadm_execute_lua_ucl_subr (L, - argc, - argv, - obj, - "corpus_test"); - - lua_close (L); - ucl_object_unref (obj); -} diff --git a/src/rspamadm/dkim_keygen.c b/src/rspamadm/dkim_keygen.c index e32f7bf05..86b228d01 100644 --- a/src/rspamadm/dkim_keygen.c +++ b/src/rspamadm/dkim_keygen.c @@ -27,8 +27,10 @@ static gchar *selector = NULL; static gchar *domain = NULL; static guint bits = 1024; -static void rspamadm_dkim_keygen (gint argc, gchar **argv); -static const char *rspamadm_dkim_keygen_help (gboolean full_help); +static void rspamadm_dkim_keygen (gint argc, gchar **argv, + const struct rspamadm_command *cmd); +static const char *rspamadm_dkim_keygen_help (gboolean full_help, + const struct rspamadm_command *cmd); static void rspamadm_dkim_keygen_lua_subrs (gpointer pL); struct rspamadm_command dkim_keygen_command = { @@ -52,7 +54,7 @@ static GOptionEntry entries[] = { }; static const char * -rspamadm_dkim_keygen_help (gboolean full_help) +rspamadm_dkim_keygen_help (gboolean full_help, const struct rspamadm_command *cmd) { const char *help_str; @@ -221,7 +223,7 @@ rspamadm_dkim_keygen_lua_subrs (gpointer pL) } static void -rspamadm_dkim_keygen (gint argc, gchar **argv) +rspamadm_dkim_keygen (gint argc, gchar **argv, const struct rspamadm_command *cmd) { GOptionContext *context; GError *error = NULL; diff --git a/src/rspamadm/fuzzy_convert.c b/src/rspamadm/fuzzy_convert.c index 40757d2e4..7671678dc 100644 --- a/src/rspamadm/fuzzy_convert.c +++ b/src/rspamadm/fuzzy_convert.c @@ -24,8 +24,10 @@ static gchar *redis_db = NULL; static gchar *redis_password = NULL; static int64_t fuzzy_expiry = 0; -static void rspamadm_fuzzyconvert (gint argc, gchar **argv); -static const char *rspamadm_fuzzyconvert_help (gboolean full_help); +static void rspamadm_fuzzyconvert (gint argc, gchar **argv, + const struct rspamadm_command *cmd); +static const char *rspamadm_fuzzyconvert_help (gboolean full_help, + const struct rspamadm_command *cmd); struct rspamadm_command fuzzyconvert_command = { .name = "fuzzyconvert", @@ -51,7 +53,7 @@ static GOptionEntry entries[] = { static const char * -rspamadm_fuzzyconvert_help (gboolean full_help) +rspamadm_fuzzyconvert_help (gboolean full_help, const struct rspamadm_command *cmd) { const char *help_str; @@ -72,7 +74,7 @@ rspamadm_fuzzyconvert_help (gboolean full_help) } static void -rspamadm_fuzzyconvert (gint argc, gchar **argv) +rspamadm_fuzzyconvert (gint argc, gchar **argv, const struct rspamadm_command *cmd) { GOptionContext *context; GError *error = NULL; @@ -135,6 +137,5 @@ rspamadm_fuzzyconvert (gint argc, gchar **argv) obj, "fuzzy_convert"); - lua_close (L); ucl_object_unref (obj); } diff --git a/src/rspamadm/fuzzy_merge.c b/src/rspamadm/fuzzy_merge.c index 94631137c..f5e6847fa 100644 --- a/src/rspamadm/fuzzy_merge.c +++ b/src/rspamadm/fuzzy_merge.c @@ -22,8 +22,10 @@ static gchar *target = NULL; static gchar **sources = NULL; static gboolean quiet; -static void rspamadm_fuzzy_merge (gint argc, gchar **argv); -static const char *rspamadm_fuzzy_merge_help (gboolean full_help); +static void rspamadm_fuzzy_merge (gint argc, gchar **argv, + const struct rspamadm_command *cmd); +static const char *rspamadm_fuzzy_merge_help (gboolean full_help, + const struct rspamadm_command *cmd); struct rspamadm_command fuzzy_merge_command = { .name = "fuzzy_merge", @@ -157,7 +159,7 @@ static struct rspamd_sqlite3_prstmt prepared_stmts[STMAX] = { }; static const char * -rspamadm_fuzzy_merge_help (gboolean full_help) +rspamadm_fuzzy_merge_help (gboolean full_help, const struct rspamadm_command *cmd) { const char *help_str; @@ -218,7 +220,7 @@ rspamadm_op_equal (gconstpointer a, gconstpointer b) } static void -rspamadm_fuzzy_merge (gint argc, gchar **argv) +rspamadm_fuzzy_merge (gint argc, gchar **argv, const struct rspamadm_command *cmd) { GOptionContext *context; GError *error = NULL; diff --git a/src/rspamadm/grep.c b/src/rspamadm/grep.c deleted file mode 100644 index 5474ecf46..000000000 --- a/src/rspamadm/grep.c +++ /dev/null @@ -1,156 +0,0 @@ -/*- - * Copyright (c) 2017, Andrew Lewis <nerf@judo.za.org> - * Copyright (c) 2017, Vsevolod Stakhov <vsevolod@highsecure.ru> - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "config.h" -#include "rspamadm.h" -#include "lua/lua_common.h" - -static gchar *string = NULL; -static gchar *pattern = NULL; -static gchar **inputs = NULL; -static gboolean sensitive = FALSE; -static gboolean orphans = FALSE; -static gboolean partial = FALSE; -static gboolean luapat = FALSE; - -static void rspamadm_grep (gint argc, gchar **argv); -static const char *rspamadm_grep_help (gboolean full_help); - -struct rspamadm_command grep_command = { - .name = "grep", - .flags = 0, - .help = rspamadm_grep_help, - .run = rspamadm_grep, - .lua_subrs = NULL, -}; - -static GOptionEntry entries[] = { - {"string", 's', 0, G_OPTION_ARG_STRING, &string, - "Plain string to search (case-insensitive)", NULL}, - {"lua", 'l', 0, G_OPTION_ARG_NONE, &luapat, - "Use Lua patterns in string search", NULL}, - {"pattern", 'p', 0, G_OPTION_ARG_STRING, &pattern, - "Pattern to search for (regex)", NULL}, - {"input", 'i', 0, G_OPTION_ARG_STRING_ARRAY, &inputs, - "Process specified inputs (stdin if unspecified)", NULL}, - {"sensitive", 'S', 0, G_OPTION_ARG_NONE, &sensitive, - "Enable case-sensitivity in string search", NULL}, - {"orphans", 'o', 0, G_OPTION_ARG_NONE, &orphans, - "Print orphaned logs", NULL}, - {"partial", 'P', 0, G_OPTION_ARG_NONE, &partial, - "Print partial logs", NULL}, - {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL} -}; - - -static const char * -rspamadm_grep_help (gboolean full_help) -{ - const char *help_str; - - if (full_help) { - help_str = "Search for patterns in rspamd logs\n\n" - "Usage: rspamadm grep <-s string | -p pattern> [-i input1 -i input2 -S -o -P]\n" - "Where options are:\n\n" - "-s: Plain string to search (case-insensitive)\n" - "-l: Use Lua patterns in string search\n" - "-p: Pattern to search for (regex)\n" - "-i: Process specified inputs (stdin if unspecified)\n" - "-S: Enable case-sensitivity in string search\n" - "-o: Print orphaned logs\n" - "-P: Print partial logs\n"; - } - else { - help_str = "Search for patterns in rspamd logs"; - } - - return help_str; -} - -static void -rspamadm_grep (gint argc, gchar **argv) -{ - GOptionContext *context; - GError *error = NULL; - lua_State *L; - ucl_object_t *obj, *nobj; - gchar **elt; - - context = g_option_context_new ( - "grep - search for patterns in rspamd logs"); - g_option_context_set_summary (context, - "Summary:\n Rspamd administration utility version " - RVERSION - "\n Release id: " - RID); - g_option_context_add_main_entries (context, entries, NULL); - g_option_context_set_ignore_unknown_options (context, FALSE); - - if (!g_option_context_parse (context, &argc, &argv, &error)) { - rspamd_fprintf (stderr, "option parsing failed: %s\n", error->message); - g_error_free (error); - exit (1); - } - - if (!pattern && !string) { - rspamd_fprintf (stderr, "no search pattern specified\n"); - exit (1); - } - if (pattern && string) { - rspamd_fprintf (stderr, "-s and -p are mutually-exclusive\n"); - exit (1); - } - - L = rspamd_lua_init (); - rspamd_lua_set_path (L, NULL, ucl_vars); - - obj = ucl_object_typed_new (UCL_OBJECT); - if (string) { - ucl_object_insert_key (obj, ucl_object_fromstring (string), - "string", 0, false); - } - if (pattern) { - ucl_object_insert_key (obj, ucl_object_fromstring (pattern), - "pattern", 0, false); - } - nobj = ucl_object_typed_new (UCL_ARRAY); - if (!inputs) { - ucl_array_append (nobj, ucl_object_fromstring ("stdin")); - } - else { - for (elt = inputs; *elt != NULL; elt ++) { - ucl_array_append (nobj, ucl_object_fromstring (*elt)); - } - } - ucl_object_insert_key (obj, nobj, "inputs", 0, false); - ucl_object_insert_key (obj, ucl_object_frombool (sensitive), - "sensitive", 0, false); - ucl_object_insert_key (obj, ucl_object_frombool (orphans), - "orphans", 0, false); - ucl_object_insert_key (obj, ucl_object_frombool (partial), - "partial", 0, false); - ucl_object_insert_key (obj, ucl_object_frombool (luapat), - "luapat", 0, false); - - rspamadm_execute_lua_ucl_subr (L, - argc, - argv, - obj, - "grep"); - - lua_close (L); - ucl_object_unref (obj); -} diff --git a/src/rspamadm/keypair.c b/src/rspamadm/keypair.c deleted file mode 100644 index 4214e0052..000000000 --- a/src/rspamadm/keypair.c +++ /dev/null @@ -1,140 +0,0 @@ -/*- - * Copyright 2016 Vsevolod Stakhov - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "config.h" -#include "rspamadm.h" -#include "cryptobox.h" -#include "printf.h" -#include "http.h" - -static gboolean hex_encode = FALSE; -static gboolean raw = FALSE; -static gboolean openssl = FALSE; -static gboolean ucl = FALSE; -static gboolean sign = FALSE; - -static void rspamadm_keypair (gint argc, gchar **argv); -static const char *rspamadm_keypair_help (gboolean full_help); - -struct rspamadm_command keypair_command = { - .name = "keypair", - .flags = 0, - .help = rspamadm_keypair_help, - .run = rspamadm_keypair, - .lua_subrs = NULL, -}; - -static GOptionEntry entries[] = { - {"hex", 'x', 0, G_OPTION_ARG_NONE, &hex_encode, - "Use hex encoding", NULL}, - {"raw", 'r', 0, G_OPTION_ARG_NONE, &raw, - "Print just keys, no description", NULL}, - {"openssl", 'o', 0, G_OPTION_ARG_NONE, &openssl, - "Generate openssl nistp256 keypair not curve25519 one", NULL}, - {"sign", 's', 0, G_OPTION_ARG_NONE, &sign, - "Generate keypair for digital signing", NULL}, - {"ucl", 'u', 0, G_OPTION_ARG_NONE, &ucl, - "Generate ucl config", NULL}, - {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL} -}; - -static const char * -rspamadm_keypair_help (gboolean full_help) -{ - const char *help_str; - - if (full_help) { - help_str = "Create key pairs for httpcrypt\n\n" - "Usage: rspamadm keypair [-x -r]\n" - "Where options are:\n\n" - "-x: encode with hex instead of base32\n" - "-r: print raw base32/hex\n" - "-o: generate openssl nistp256 keypair\n" - "-s: generate keypair suitable for signatures\n" - "-u: generate ucl config for keypair\n" - "--help: shows available options and commands"; - } - else { - help_str = "Create encryption key pairs"; - } - - return help_str; -} - -static void -rspamadm_keypair (gint argc, gchar **argv) -{ - GOptionContext *context; - GError *error = NULL; - struct rspamd_cryptobox_keypair *kp; - gint how = 0; - ucl_object_t *ucl_out; - struct ucl_emitter_functions *ucl_emit_subr; - GString *out; - enum rspamd_cryptobox_keypair_type type = RSPAMD_KEYPAIR_KEX; - enum rspamd_cryptobox_mode mode = RSPAMD_CRYPTOBOX_MODE_25519; - - context = g_option_context_new ( - "keypair - create encryption keys"); - g_option_context_set_summary (context, - "Summary:\n Rspamd administration utility version " - RVERSION - "\n Release id: " - RID); - g_option_context_add_main_entries (context, entries, NULL); - - if (!g_option_context_parse (context, &argc, &argv, &error)) { - fprintf (stderr, "option parsing failed: %s\n", error->message); - g_error_free (error); - exit (1); - } - - if (openssl) { - mode = RSPAMD_CRYPTOBOX_MODE_NIST; - } - if (hex_encode) { - how |= RSPAMD_KEYPAIR_HEX; - } - else { - how |= RSPAMD_KEYPAIR_BASE32; - } - - if (sign) { - type = RSPAMD_KEYPAIR_SIGN; - } - - kp = rspamd_keypair_new (type, mode); - - if (ucl) { - ucl_out = rspamd_keypair_to_ucl (kp, hex_encode); - ucl_emit_subr = ucl_object_emit_file_funcs (stdout); - ucl_object_emit_full (ucl_out, UCL_EMIT_CONFIG, ucl_emit_subr, NULL); - ucl_object_emit_funcs_free (ucl_emit_subr); - ucl_object_unref (ucl_out); - } - else { - how |= RSPAMD_KEYPAIR_PUBKEY | RSPAMD_KEYPAIR_PRIVKEY; - - if (!raw) { - how |= RSPAMD_KEYPAIR_HUMAN|RSPAMD_KEYPAIR_ID; - } - - out = rspamd_keypair_print (kp, how); - rspamd_printf ("%v", out); - g_string_free (out, TRUE); - } - - rspamd_keypair_unref (kp); -} diff --git a/src/rspamadm/lua_repl.c b/src/rspamadm/lua_repl.c index e0ee7d9f1..a807101e0 100644 --- a/src/rspamadm/lua_repl.c +++ b/src/rspamadm/lua_repl.c @@ -47,8 +47,10 @@ static const char *default_history_file = ".rspamd_repl.hist"; #endif #define MULTILINE_PROMPT "... " -static void rspamadm_lua (gint argc, gchar **argv); -static const char *rspamadm_lua_help (gboolean full_help); +static void rspamadm_lua (gint argc, gchar **argv, + const struct rspamadm_command *cmd); +static const char *rspamadm_lua_help (gboolean full_help, + const struct rspamadm_command *cmd); struct rspamadm_command lua_command = { .name = "lua", @@ -121,7 +123,7 @@ static GOptionEntry entries[] = { }; static const char * -rspamadm_lua_help (gboolean full_help) +rspamadm_lua_help (gboolean full_help, const struct rspamadm_command *cmd) { const char *help_str; @@ -388,6 +390,7 @@ rspamadm_lua_message_handler (lua_State *L, gint argc, gchar **argv) continue; } + rspamd_message_process (task); lua_pushcfunction (L, &rspamd_lua_traceback); err_idx = lua_gettop (L); @@ -658,13 +661,12 @@ rspamadm_lua_handle_exec (struct rspamd_http_connection_entry *conn_ent, } static void -rspamadm_lua (gint argc, gchar **argv) +rspamadm_lua (gint argc, gchar **argv, const struct rspamadm_command *cmd) { GOptionContext *context; GError *error = NULL; gchar **elt; guint i; - lua_State *L; context = g_option_context_new ("lua - run lua interpreter"); g_option_context_set_summary (context, @@ -689,9 +691,6 @@ rspamadm_lua (gint argc, gchar **argv) } } - L = rspamd_lua_init (); - rspamd_lua_set_path (L, NULL, ucl_vars); - if (paths) { for (elt = paths; *elt != NULL; elt ++) { rspamadm_lua_add_path (L, *elt); diff --git a/src/rspamadm/pw.c b/src/rspamadm/pw.c index 3299c479d..b00aac223 100644 --- a/src/rspamadm/pw.c +++ b/src/rspamadm/pw.c @@ -21,8 +21,10 @@ #include "rspamadm.h" #include "unix-std.h" -static void rspamadm_pw (gint argc, gchar **argv); -static const char *rspamadm_pw_help (gboolean full_help); +static void rspamadm_pw (gint argc, gchar **argv, + const struct rspamadm_command *cmd); +static const char *rspamadm_pw_help (gboolean full_help, + const struct rspamadm_command *cmd); static void rspamadm_pw_lua_subrs (gpointer pL); static gboolean do_encrypt = FALSE; @@ -57,7 +59,7 @@ static GOptionEntry entries[] = { }; static const char * -rspamadm_pw_help (gboolean full_help) +rspamadm_pw_help (gboolean full_help, const struct rspamadm_command *cmd) { const char *help_str; @@ -354,7 +356,7 @@ rspamadm_alg_list (void) } static void -rspamadm_pw (gint argc, gchar **argv) +rspamadm_pw (gint argc, gchar **argv, const struct rspamadm_command *cmd) { GOptionContext *context; GError *error = NULL; diff --git a/src/rspamadm/rescore.c b/src/rspamadm/rescore.c deleted file mode 100644 index 926740023..000000000 --- a/src/rspamadm/rescore.c +++ /dev/null @@ -1,199 +0,0 @@ -/*- - * Copyright 2017 Pragadeesh C - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "config.h" -#include "rspamadm.h" -#include "lua/lua_common.h" - -#if !defined(WITH_TORCH) || !defined(WITH_LUAJIT) -#define HAS_TORCH false -#else -#define HAS_TORCH true -#endif - -static gchar *logdir = NULL; -static gchar *output = "new.scores"; -static gboolean score_diff = false; /* Print score diff flag */ -static gchar *config = NULL; -extern struct rspamd_main *rspamd_main; -/* Defined in modules.c */ -extern module_t *modules[]; -extern worker_t *workers[]; - -static void rspamadm_rescore (gint argc, gchar **argv); - -static const char *rspamadm_rescore_help (gboolean full_help); - -struct rspamadm_command rescore_command = { - .name = "rescore", - .flags = 0, - .help = rspamadm_rescore_help, - .run = rspamadm_rescore -}; - -static GOptionEntry entries[] = { - {"logdir", 'l', 0, G_OPTION_ARG_FILENAME, &logdir, - "Logs directory", NULL}, - {"output", 'o', 0, G_OPTION_ARG_FILENAME, &output, - "Scores output location", NULL}, - {"diff", 'd', 0, G_OPTION_ARG_NONE, &score_diff, - "Print score diff", NULL}, - {"config", 'c', 0, G_OPTION_ARG_STRING, &config, - "Config file to use", NULL}, - {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL} -}; - -static void -config_logger (rspamd_mempool_t *pool, gpointer ud) -{ - struct rspamd_main *rm = ud; - - rm->cfg->log_type = RSPAMD_LOG_CONSOLE; - rm->cfg->log_level = G_LOG_LEVEL_MESSAGE; - - rspamd_set_logger (rm->cfg, g_quark_try_string ("main"), &rm->logger, - rm->server_pool); - - if (rspamd_log_open_priv (rm->logger, rm->workers_uid, rm->workers_gid) == - -1) { - fprintf (stderr, "Fatal error, cannot open logfile, exiting\n"); - exit (EXIT_FAILURE); - } -} - -static const char * -rspamadm_rescore_help (gboolean full_help) { - - const char *help_str; - - if (full_help) { - help_str = "Estimate optimal symbol weights from log files\n\n" - "Usage: rspamadm rescore -l <log_directory>\n" - "Where options are:\n\n" - "-l: path to logs directory\n" - "-o: scores output file location\n" - "-d: print scores diff\n" - "-i: max iterations for perceptron\n"; - } else { - help_str = "Estimate optimal symbol weights from log files"; - } - - return help_str; -} - -static void -rspamadm_rescore (gint argc, gchar **argv) { - - GOptionContext *context; - GError *error = NULL; - lua_State *L; - struct rspamd_config *cfg = rspamd_main->cfg, **pcfg; - gboolean ret = TRUE; - worker_t **pworker; - const gchar *confdir; - - context = g_option_context_new ( - "rescore - estimate optimal symbol weights from log files"); - - g_option_context_set_summary (context, - "Summary:\n Rspamd administration utility version " - RVERSION - "\n Release id: " - RID); - - g_option_context_add_main_entries (context, entries, NULL); - g_option_context_set_ignore_unknown_options (context, TRUE); - - if (!g_option_context_parse (context, &argc, &argv, &error)) { - rspamd_fprintf (stderr, "option parsing failed: %s\n", error->message); - g_error_free (error); - exit (EXIT_FAILURE); - } - - if (!HAS_TORCH) { - rspamd_fprintf (stderr, "Torch is not enabled. " - "Use -DENABLE_TORCH=ON option while running cmake.\n"); - exit (EXIT_FAILURE); - } - - if (logdir == NULL) { - rspamd_fprintf (stderr, "Please specify log directory.\n"); - exit (EXIT_FAILURE); - } - - if (config == NULL) { - if ((confdir = g_hash_table_lookup (ucl_vars, "CONFDIR")) == NULL) { - confdir = RSPAMD_CONFDIR; - } - - config = g_strdup_printf ("%s%c%s", confdir, G_DIR_SEPARATOR, - "rspamd.conf"); - } - - pworker = &workers[0]; - while (*pworker) { - /* Init string quarks */ - (void) g_quark_from_static_string ((*pworker)->name); - pworker++; - } - - cfg->cache = rspamd_symbols_cache_new (cfg); - cfg->compiled_modules = modules; - cfg->compiled_workers = workers; - cfg->cfg_name = config; - - if (!rspamd_config_read (cfg, cfg->cfg_name, NULL, - config_logger, rspamd_main, ucl_vars)) { - ret = FALSE; - } - else { - /* Do post-load actions */ - rspamd_lua_post_load_config (cfg); - - if (!rspamd_init_filters (cfg, FALSE)) { - ret = FALSE; - } - - if (ret) { - ret = rspamd_config_post_load (cfg, RSPAMD_CONFIG_INIT_SYMCACHE); - rspamd_symbols_cache_validate (cfg->cache, - cfg, - FALSE); - } - } - - if (ret) { - L = cfg->lua_state; - rspamd_lua_set_path (L, cfg->rcl_obj, ucl_vars); - ucl_object_insert_key (cfg->rcl_obj, ucl_object_fromstring (cfg->cfg_name), - "config_path", 0, false); - ucl_object_insert_key (cfg->rcl_obj, ucl_object_fromstring (logdir), - "logdir", 0, false); - ucl_object_insert_key (cfg->rcl_obj, ucl_object_fromstring (output), - "output", 0, false); - ucl_object_insert_key (cfg->rcl_obj, ucl_object_frombool (score_diff), - "diff", 0, false); - pcfg = lua_newuserdata (L, sizeof (struct rspamd_config *)); - rspamd_lua_setclass (L, "rspamd{config}", -1); - *pcfg = cfg; - lua_setglobal (L, "rspamd_config"); - rspamadm_execute_lua_ucl_subr (L, - argc, - argv, - cfg->rcl_obj, - "rescore"); - } -} diff --git a/src/rspamadm/rspamadm.c b/src/rspamadm/rspamadm.c index b39c75897..68dbdeb85 100644 --- a/src/rspamadm/rspamadm.c +++ b/src/rspamadm/rspamadm.c @@ -31,9 +31,14 @@ static gboolean show_help = FALSE; static gboolean show_version = FALSE; GHashTable *ucl_vars = NULL; struct rspamd_main *rspamd_main = NULL; +lua_State *L = NULL; -static void rspamadm_help (gint argc, gchar **argv); -static const char* rspamadm_help_help (gboolean full_help); +/* Defined in modules.c */ +extern module_t *modules[]; +extern worker_t *workers[]; + +static void rspamadm_help (gint argc, gchar **argv, const struct rspamadm_command *); +static const char* rspamadm_help_help (gboolean full_help, const struct rspamadm_command *); struct rspamadm_command help_command = { .name = "help", @@ -70,7 +75,7 @@ rspamadm_error (void) static void rspamadm_version (void) { - printf ("Rspamadm %s\n", RVERSION); + rspamd_printf ("Rspamadm %s\n", RVERSION); } static void @@ -79,30 +84,28 @@ rspamadm_usage (GOptionContext *context) gchar *help_str; help_str = g_option_context_get_help (context, TRUE, NULL); - printf ("%s", help_str); + rspamd_printf ("%s", help_str); } static void -rspamadm_commands (void) +rspamadm_commands (GPtrArray *all_commands) { - const struct rspamadm_command **cmd; - - printf ("Rspamadm %s\n", RVERSION); - printf ("Usage: rspamadm [global_options] command [command_options]\n"); - printf ("\nAvailable commands:\n"); + const struct rspamadm_command *cmd; + guint i; - cmd = commands; + rspamd_printf ("Rspamadm %s\n", RVERSION); + rspamd_printf ("Usage: rspamadm [global_options] command [command_options]\n"); + rspamd_printf ("\nAvailable commands:\n"); - while (*cmd) { - if (!((*cmd)->flags & RSPAMADM_FLAG_NOHELP)) { - printf (" %-18s %-60s\n", (*cmd)->name, (*cmd)->help (FALSE)); + PTR_ARRAY_FOREACH (all_commands, i, cmd) { + if (!(cmd->flags & RSPAMADM_FLAG_NOHELP)) { + rspamd_printf (" %-18s %-60s\n", cmd->name, cmd->help (FALSE, cmd)); } - cmd ++; } } static const char * -rspamadm_help_help (gboolean full_help) +rspamadm_help_help (gboolean full_help, const struct rspamadm_command *cmd) { const char *help_str; @@ -118,23 +121,24 @@ rspamadm_help_help (gboolean full_help) } static void -rspamadm_help (gint argc, gchar **argv) +rspamadm_help (gint argc, gchar **argv, const struct rspamadm_command *command) { const gchar *cmd_name; - const struct rspamadm_command *cmd, **cmd_list; + const struct rspamadm_command *cmd; + GPtrArray *all_commands = (GPtrArray *)command->command_data; - printf ("Rspamadm %s\n", RVERSION); - printf ("Usage: rspamadm [global_options] command [command_options]\n\n"); + rspamd_printf ("Rspamadm %s\n", RVERSION); + rspamd_printf ("Usage: rspamadm [global_options] command [command_options]\n\n"); if (argc <= 1) { cmd_name = "help"; } else { cmd_name = argv[1]; - printf ("Showing help for %s command\n\n", cmd_name); + rspamd_printf ("Showing help for %s command\n\n", cmd_name); } - cmd = rspamadm_search_command (cmd_name); + cmd = rspamadm_search_command (cmd_name, all_commands); if (cmd == NULL) { fprintf (stderr, "Invalid command name: %s\n", cmd_name); @@ -142,20 +146,30 @@ rspamadm_help (gint argc, gchar **argv) } if (strcmp (cmd_name, "help") == 0) { - printf ("Available commands:\n"); - - cmd_list = commands; - - while (*cmd_list) { - if (!((*cmd_list)->flags & RSPAMADM_FLAG_NOHELP)) { - printf (" %-18s %-60s\n", (*cmd_list)->name, - (*cmd_list)->help (FALSE)); + guint i; + rspamd_printf ("Available commands:\n"); + + PTR_ARRAY_FOREACH (all_commands, i, cmd) { + if (!(cmd->flags & RSPAMADM_FLAG_NOHELP)) { + if (!(cmd->flags & RSPAMADM_FLAG_LUA)) { + printf (" %-18s %-60s\n", cmd->name, + cmd->help (FALSE, cmd)); + } + else { + /* Just call lua subr */ + (void)cmd->help (FALSE, cmd); + } } - cmd_list++; } } else { - printf ("%s\n", cmd->help (TRUE)); + if (!(cmd->flags & RSPAMADM_FLAG_LUA)) { + rspamd_printf ("%s\n", cmd->help (TRUE, cmd)); + } + else { + /* Just call lua subr */ + (void)cmd->help (TRUE, cmd); + } } } @@ -193,25 +207,12 @@ rspamadm_execute_lua_ucl_subr (gpointer pL, gint argc, gchar **argv, gint err_idx, i, ret; GString *tb; gchar str[PATH_MAX]; - const struct rspamadm_command **cmd; g_assert (script_name != NULL); g_assert (res != NULL); g_assert (L != NULL); /* Init internal rspamadm routines */ - lua_newtable (L); - cmd = commands; - - while (*cmd) { - if ((*cmd)->lua_subrs != NULL) { - (*cmd)->lua_subrs (L); - } - - cmd ++; - } - - lua_setglobal (L, "rspamadm"); rspamd_snprintf (str, sizeof (str), "return require \"%s.%s\"", "rspamadm", script_name); @@ -222,6 +223,11 @@ rspamadm_execute_lua_ucl_subr (gpointer pL, gint argc, gchar **argv, return FALSE; } else { + if (lua_type (L, -1) == LUA_TTABLE) { + lua_pushstring (L, "handler"); + lua_gettable (L, -2); + } + if (lua_type (L, -1) != LUA_TFUNCTION) { msg_err ("lua script must return " "function and not %s", @@ -267,6 +273,37 @@ rspamadm_execute_lua_ucl_subr (gpointer pL, gint argc, gchar **argv, return TRUE; } +static gint +rspamdadm_commands_sort_func (gconstpointer a, gconstpointer b) +{ + const struct rspamadm_command *cmda = *((struct rspamadm_command const **)a), + *cmdb = *((struct rspamadm_command const **)b); + + return strcmp (cmda->name, cmdb->name); +} + +static gboolean +rspamadm_command_maybe_match_name (const gchar *cmd, const gchar *input) +{ + gsize clen, inplen; + + clen = strlen (cmd); + inplen = strlen (input); + + if (rspamd_strings_levenshtein_distance (cmd, clen, + input, inplen, 1) == 1) { + return TRUE; + } + else if ((clen > inplen && + rspamd_substring_search (cmd, clen, input, inplen) != -1) || + (inplen > clen && + rspamd_substring_search (input, inplen, cmd, clen) != -1)) { + return TRUE; + } + + return FALSE; +} + gint main (gint argc, gchar **argv, gchar **env) { @@ -278,12 +315,14 @@ main (gint argc, gchar **argv, gchar **env) gchar **nargv, **targv; const gchar *cmd_name; const struct rspamadm_command *cmd; + GPtrArray *all_commands = g_ptr_array_new (); /* Discovered during check */ gint i, nargc, targc; + worker_t **pworker; ucl_vars = g_hash_table_new_full (rspamd_strcase_hash, rspamd_strcase_equal, g_free, g_free); process_quark = g_quark_from_static_string ("rspamadm"); - cfg = rspamd_config_new (); + cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_DEFAULT); cfg->libs_ctx = rspamd_init_libs (); rspamd_main = g_malloc0 (sizeof (*rspamd_main)); rspamd_main->cfg = cfg; @@ -291,6 +330,8 @@ main (gint argc, gchar **argv, gchar **env) rspamd_main->type = process_quark; rspamd_main->server_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "rspamadm"); + rspamadm_fill_internal_commands (all_commands); + help_command.command_data = all_commands; /* Setup logger */ if (verbose) { @@ -311,6 +352,16 @@ main (gint argc, gchar **argv, gchar **env) rspamd_config_post_load (cfg, RSPAMD_CONFIG_INIT_LIBS|RSPAMD_CONFIG_INIT_URL|RSPAMD_CONFIG_INIT_NO_TLD); + pworker = &workers[0]; + while (*pworker) { + /* Init string quarks */ + (void) g_quark_from_static_string ((*pworker)->name); + pworker++; + } + + cfg->compiled_modules = modules; + cfg->compiled_workers = workers; + gperf_profiler_init (cfg, "rspamadm"); setproctitle ("rspamdadm"); @@ -349,6 +400,26 @@ main (gint argc, gchar **argv, gchar **env) exit (1); } + L = cfg->lua_state; + rspamd_lua_set_path (L, NULL, ucl_vars); + rspamd_lua_set_globals (cfg, L, ucl_vars); + + /* Init rspamadm global */ + lua_newtable (L); + + PTR_ARRAY_FOREACH (all_commands, i, cmd) { + if (cmd->lua_subrs != NULL) { + cmd->lua_subrs (L); + } + + cmd ++; + } + + lua_setglobal (L, "rspamadm"); + + rspamadm_fill_lua_commands (L, all_commands); + g_ptr_array_sort (all_commands, rspamdadm_commands_sort_func); + g_strfreev (nargv); if (show_version) { @@ -360,7 +431,7 @@ main (gint argc, gchar **argv, gchar **env) exit (EXIT_SUCCESS); } if (list_commands) { - rspamadm_commands (); + rspamadm_commands (all_commands); exit (EXIT_SUCCESS); } @@ -370,10 +441,29 @@ main (gint argc, gchar **argv, gchar **env) cmd_name = "help"; } - cmd = rspamadm_search_command (cmd_name); + cmd = rspamadm_search_command (cmd_name, all_commands); if (cmd == NULL) { - fprintf (stderr, "Invalid command name: %s\n", cmd_name); + rspamd_fprintf (stderr, "Invalid command name: %s\n", cmd_name); + + /* Try fuzz search */ + rspamd_fprintf (stderr, "Suggested commands:\n"); + PTR_ARRAY_FOREACH (all_commands, i, cmd) { + guint j; + const gchar *alias; + + if (rspamadm_command_maybe_match_name (cmd->name, cmd_name)) { + rspamd_fprintf (stderr, "%s\n", cmd->name); + } + else { + PTR_ARRAY_FOREACH (cmd->aliases, j, alias) { + if (rspamadm_command_maybe_match_name (alias, cmd_name)) { + rspamd_fprintf (stderr, "%s\n", alias); + } + } + } + } + exit (EXIT_FAILURE); } @@ -387,16 +477,17 @@ main (gint argc, gchar **argv, gchar **env) targc = argc - nargc; targv = nargv; - cmd->run (targc, targv); + cmd->run (targc, targv, cmd); g_strfreev (nargv); } else { - cmd->run (0, NULL); + cmd->run (0, NULL, cmd); } rspamd_log_close (rspamd_main->logger); REF_RELEASE (rspamd_main->cfg); g_free (rspamd_main); + g_ptr_array_free (all_commands, TRUE); return 0; } diff --git a/src/rspamadm/rspamadm.h b/src/rspamadm/rspamadm.h index 9a209fd91..3d9799dd5 100644 --- a/src/rspamadm/rspamadm.h +++ b/src/rspamadm/rspamadm.h @@ -18,16 +18,24 @@ #include "config.h" #include "ucl.h" +#include <lua.h> +#include <lauxlib.h> +#include <lualib.h> extern GHashTable *ucl_vars; +extern lua_State *L; GQuark rspamadm_error (void); -typedef const gchar* (*rspamadm_help_func) (gboolean full_help); -typedef void (*rspamadm_run_func) (gint argc, gchar **argv); +struct rspamadm_command; +typedef const gchar* (*rspamadm_help_func) (gboolean full_help, + const struct rspamadm_command *cmd); +typedef void (*rspamadm_run_func) (gint argc, gchar **argv, + const struct rspamadm_command *cmd); typedef void (*rspamadm_lua_exports_func) (gpointer lua_state); #define RSPAMADM_FLAG_NOHELP (1 << 0) +#define RSPAMADM_FLAG_LUA (1 << 1) struct rspamadm_command { const gchar *name; @@ -35,14 +43,20 @@ struct rspamadm_command { rspamadm_help_func help; rspamadm_run_func run; rspamadm_lua_exports_func lua_subrs; + GPtrArray *aliases; + gpointer command_data; /* Opaque data */ }; extern const struct rspamadm_command *commands[]; extern struct rspamadm_command help_command; -const struct rspamadm_command *rspamadm_search_command (const gchar *name); +const struct rspamadm_command *rspamadm_search_command (const gchar *name, + GPtrArray *all_commands); +void rspamadm_fill_internal_commands (GPtrArray *dest); +void rspamadm_fill_lua_commands (lua_State *L, GPtrArray *dest); gboolean rspamadm_execute_lua_ucl_subr (gpointer L, gint argc, gchar **argv, - const ucl_object_t *res, const gchar *script_name); + const ucl_object_t *res, + const gchar *script_name); #endif diff --git a/src/rspamadm/signtool.c b/src/rspamadm/signtool.c index 0ae7e22c5..eae55b8bd 100644 --- a/src/rspamadm/signtool.c +++ b/src/rspamadm/signtool.c @@ -39,8 +39,10 @@ static gchar *editor = NULL; static gboolean edit = FALSE; enum rspamd_cryptobox_mode mode = RSPAMD_CRYPTOBOX_MODE_25519; -static void rspamadm_signtool (gint argc, gchar **argv); -static const char *rspamadm_signtool_help (gboolean full_help); +static void rspamadm_signtool (gint argc, gchar **argv, + const struct rspamadm_command *cmd); +static const char *rspamadm_signtool_help (gboolean full_help, + const struct rspamadm_command *cmd); struct rspamadm_command signtool_command = { .name = "signtool", @@ -75,7 +77,8 @@ static GOptionEntry entries[] = { }; static const char * -rspamadm_signtool_help (gboolean full_help) +rspamadm_signtool_help (gboolean full_help, + const struct rspamadm_command *cmd) { const char *help_str; @@ -444,7 +447,8 @@ rspamadm_verify_file (const gchar *fname, const guchar *pk) exit (errno); } - ret = rspamd_cryptobox_verify (map_sig, map, st.st_size, pk, mode); + ret = rspamd_cryptobox_verify (map_sig, st_sig.st_size, + map, st.st_size, pk, mode); munmap (map, st.st_size); munmap (map_sig, st_sig.st_size); @@ -462,7 +466,7 @@ rspamadm_verify_file (const gchar *fname, const guchar *pk) static void -rspamadm_signtool (gint argc, gchar **argv) +rspamadm_signtool (gint argc, gchar **argv, const struct rspamadm_command *cmd) { GOptionContext *context; GError *error = NULL; diff --git a/src/rspamadm/stat_convert.c b/src/rspamadm/stat_convert.c index e4cf24629..ef17194b0 100644 --- a/src/rspamadm/stat_convert.c +++ b/src/rspamadm/stat_convert.c @@ -37,8 +37,10 @@ static gchar *redis_db = NULL; static gchar *redis_password = NULL; static gboolean reset_previous = FALSE; -static void rspamadm_statconvert (gint argc, gchar **argv); -static const char *rspamadm_statconvert_help (gboolean full_help); +static void rspamadm_statconvert (gint argc, gchar **argv, + const struct rspamadm_command *cmd); +static const char *rspamadm_statconvert_help (gboolean full_help, + const struct rspamadm_command *cmd); struct rspamadm_command statconvert_command = { .name = "statconvert", @@ -77,7 +79,7 @@ static GOptionEntry entries[] = { static const char * -rspamadm_statconvert_help (gboolean full_help) +rspamadm_statconvert_help (gboolean full_help, const struct rspamadm_command *cmd) { const char *help_str; @@ -107,7 +109,7 @@ rspamadm_statconvert_help (gboolean full_help) } static void -rspamadm_statconvert (gint argc, gchar **argv) +rspamadm_statconvert (gint argc, gchar **argv, const struct rspamadm_command *cmd) { GOptionContext *context; GError *error = NULL; @@ -251,6 +253,5 @@ rspamadm_statconvert (gint argc, gchar **argv) obj, "stat_convert"); - lua_close (L); ucl_object_unref (obj); } diff --git a/src/rspamd.c b/src/rspamd.c index cc3be3357..643d249b1 100644 --- a/src/rspamd.c +++ b/src/rspamd.c @@ -276,7 +276,7 @@ reread_config (struct rspamd_main *rspamd_main) gchar *cfg_file; rspamd_symbols_cache_save (rspamd_main->cfg->cache); - tmp_cfg = rspamd_config_new (); + tmp_cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_DEFAULT); g_hash_table_unref (tmp_cfg->c_modules); tmp_cfg->c_modules = g_hash_table_ref (rspamd_main->cfg->c_modules); tmp_cfg->libs_ctx = rspamd_main->cfg->libs_ctx; @@ -874,8 +874,7 @@ load_rspamd_config (struct rspamd_main *rspamd_main, cfg->compiled_modules = modules; cfg->compiled_workers = workers; - if (!rspamd_config_read (cfg, cfg->cfg_name, NULL, - config_logger, rspamd_main, ucl_vars)) { + if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main, ucl_vars)) { return FALSE; } @@ -1180,7 +1179,7 @@ main (gint argc, gchar **argv, gchar **env) "main"); rspamd_main->stat = rspamd_mempool_alloc0_shared (rspamd_main->server_pool, sizeof (struct rspamd_stat)); - rspamd_main->cfg = rspamd_config_new (); + rspamd_main->cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_DEFAULT); rspamd_main->spairs = g_hash_table_new_full (rspamd_spair_hash, rspamd_spair_equal, g_free, rspamd_spair_close); rspamd_main->start_mtx = rspamd_mempool_get_mutex (rspamd_main->server_pool); diff --git a/test/functional/cases/106_url_tags.robot b/test/functional/cases/106_url_tags.robot deleted file mode 100644 index 7954d0aac..000000000 --- a/test/functional/cases/106_url_tags.robot +++ /dev/null @@ -1,41 +0,0 @@ -*** Settings *** -Suite Setup URL Tags Setup -Suite Teardown URL Tags Teardown -Library ${TESTDIR}/lib/rspamd.py -Resource ${TESTDIR}/lib/rspamd.robot -Variables ${TESTDIR}/lib/vars.py - -*** Variables *** -${ADDITIONAL} ${TESTDIR}/lua/url_tags.lua -${CONFIG} ${TESTDIR}/configs/pluginsplus.conf -${MESSAGE} ${TESTDIR}/messages/url1.eml -${REDIS_SCOPE} Suite -${RSPAMD_SCOPE} Suite -${URL_TLD} ${TESTDIR}/../lua/unit/test_tld.dat - -*** Test Cases *** -URL TAGS PERSISTENCE - ${result} = Scan Message With Rspamc --header=addtags=1 ${MESSAGE} - Check Rspamc ${result} ADDED_TAGS (1.00)[no worry] - ${result} = Scan Message With Rspamc ${MESSAGE} - Check Rspamc ${result} FOUND_TAGS (1.00)[no worry] - -*** Keywords *** -URL Tags Setup - ${TMPDIR} = Make Temporary Directory - Set Suite Variable ${TMPDIR} - Run Redis - ${LUA} = Make Temporary File - ${goop} = Get File ${INSTALLROOT}/share/rspamd/rules/rspamd.lua - ${goop2} = Get File ${ADDITIONAL} - ${goop_unesc} = Catenate ${goop} ${goop2} - ${PLUGIN_CONFIG} = Get File ${TESTDIR}/configs/url_tags.conf - Set Suite Variable ${LUA} - Set Suite Variable ${PLUGIN_CONFIG} - Create File ${LUA} ${goop_unesc} - Generic Setup TMPDIR=${TMPDIR} - -URL Tags Teardown - Normal Teardown - Remove File ${LUA} - Shutdown Process With Children ${REDIS_PID} diff --git a/test/rspamd_test_suite.c b/test/rspamd_test_suite.c index d6fa237a3..2f8e893de 100644 --- a/test/rspamd_test_suite.c +++ b/test/rspamd_test_suite.c @@ -15,7 +15,7 @@ main (int argc, char **argv) rspamd_main = (struct rspamd_main *)g_malloc (sizeof (struct rspamd_main)); memset (rspamd_main, 0, sizeof (struct rspamd_main)); rspamd_main->server_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL); - cfg = rspamd_config_new (); + cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_DEFAULT); rspamd_main->cfg = cfg; cfg->cfg_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL); cfg->log_type = RSPAMD_LOG_CONSOLE; diff --git a/test/rspamd_upstream_test.c b/test/rspamd_upstream_test.c index 47094398f..cce0008a9 100644 --- a/test/rspamd_upstream_test.c +++ b/test/rspamd_upstream_test.c @@ -64,7 +64,7 @@ rspamd_upstream_test_func (void) struct timeval tv; rspamd_inet_addr_t *addr, *next_addr, *paddr; - cfg = rspamd_config_new (); + cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_SKIP_LUA); cfg->dns_retransmits = 2; cfg->dns_timeout = 0.5; cfg->upstream_max_errors = 1; diff --git a/utils/rspamd_stats.pl b/utils/rspamd_stats.pl index f4f01dd85..b997ff525 100755 --- a/utils/rspamd_stats.pl +++ b/utils/rspamd_stats.pl @@ -460,7 +460,6 @@ sub ProcessRelated { sub ProcessLog { my ( $ts_format, @line ) = &log_time_format($rspamd_log); - my $is_syslog = defined $ts_format && $ts_format eq 'syslog'; while() { last if eof $rspamd_log; @@ -474,10 +473,15 @@ sub ProcessLog { if (/^.*rspamd_task_write_log.*$/) { &spinner; - my $ts = - ($is_syslog) - ? syslog2iso( join ' ', ( split /\s+/ )[ 0 .. 2 ] ) - : join ' ', ( split /\s+/ )[ 0 .. 1 ]; + my $ts; + if ( $ts_format eq 'syslog' ) { + $ts = syslog2iso( join ' ', ( split /\s+/ )[ 0 .. 2 ] ); + } elsif ( $ts_format eq 'syslog5424' ) { + /^([0-9-]+)T([0-9:]+)/; + $ts = "$1 $2"; + } else { + $ts = join ' ', ( split /\s+/ )[ 0 .. 1 ]; + } next if ( $ts lt $startTime ); next if ( defined $endTime && $ts gt $endTime ); @@ -718,6 +722,12 @@ sub log_time_format { last; } + # 2018-04-16T06:25:46.012590+02:00 rspamd rspamd[12968] + elsif(/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,6})?(Z|[-+]\d{2}:\d{2}) \S+ rspamd\[\d+\]/) { + $format = 'syslog5424'; + last; + } + # Skip newsyslog messages # Aug 8 00:00:00 hostname newsyslog[63284]: logfile turned over elsif ( /^\w{3} (?:\s?\d|\d\d) \d\d:\d\d:\d\d\ \S+ newsyslog\[\d+\]: logfile turned over$/ ) { |