aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorheraklit256 <37872459+heraklit256@users.noreply.github.com>2018-06-09 07:58:21 +0000
committerGitHub <noreply@github.com>2018-06-09 07:58:21 +0000
commit728b3a44311cab0717fc1ef9d1576ed94f7118e4 (patch)
tree16f99393319ea9fd0cff9bb089cb20ce8eb4b80d
parent109730f3834c0aa7fedc8315406793b1d2a5f530 (diff)
parent85e5de3d81b8bf6ef7bf4592ff77d56cd59191c6 (diff)
downloadrspamd-728b3a44311cab0717fc1ef9d1576ed94f7118e4.tar.gz
rspamd-728b3a44311cab0717fc1ef9d1576ed94f7118e4.zip
Merge pull request #2 from vstakhov/master
merge upstream into local master
-rw-r--r--CMakeLists.txt17
-rw-r--r--clang-plugin/CMakeLists.txt2
-rw-r--r--conf/composites.conf6
-rw-r--r--conf/modules.d/ratelimit.conf4
-rw-r--r--contrib/lc-btrie/btrie.c8
-rw-r--r--contrib/lc-btrie/btrie.h2
-rw-r--r--contrib/librdns/dns_private.h26
-rw-r--r--contrib/librdns/rdns.h12
-rw-r--r--contrib/librdns/resolver.c133
-rw-r--r--contrib/librdns/util.c62
-rw-r--r--contrib/lua-argparse/LICENSE20
-rw-r--r--contrib/lua-argparse/argparse.lua1516
-rw-r--r--contrib/lua-lpeg/CMakeLists.txt (renamed from contrib/lpeg/CMakeLists.txt)0
-rw-r--r--contrib/lua-lpeg/LICENSE (renamed from contrib/lpeg/LICENSE)0
-rw-r--r--contrib/lua-lpeg/lpcap.c (renamed from contrib/lpeg/lpcap.c)0
-rw-r--r--contrib/lua-lpeg/lpcap.h (renamed from contrib/lpeg/lpcap.h)0
-rw-r--r--contrib/lua-lpeg/lpcode.c (renamed from contrib/lpeg/lpcode.c)0
-rw-r--r--contrib/lua-lpeg/lpcode.h (renamed from contrib/lpeg/lpcode.h)0
-rw-r--r--contrib/lua-lpeg/lpprint.c (renamed from contrib/lpeg/lpprint.c)0
-rw-r--r--contrib/lua-lpeg/lpprint.h (renamed from contrib/lpeg/lpprint.h)0
-rw-r--r--contrib/lua-lpeg/lptree.c (renamed from contrib/lpeg/lptree.c)0
-rw-r--r--contrib/lua-lpeg/lptree.h (renamed from contrib/lpeg/lptree.h)0
-rw-r--r--contrib/lua-lpeg/lptypes.h (renamed from contrib/lpeg/lptypes.h)0
-rw-r--r--contrib/lua-lpeg/lpvm.c (renamed from contrib/lpeg/lpvm.c)0
-rw-r--r--contrib/lua-lpeg/lpvm.h (renamed from contrib/lpeg/lpvm.h)0
-rw-r--r--contrib/lua-moses/LICENSE (renamed from contrib/moses/LICENSE)0
-rw-r--r--contrib/lua-moses/moses.lua (renamed from contrib/moses/moses.lua)0
-rw-r--r--contrib/lua-torch/decisiontree/CMakeLists.txt (renamed from contrib/torch/decisiontree/CMakeLists.txt)5
-rw-r--r--contrib/lua-torch/decisiontree/CartNode.lua (renamed from contrib/torch/decisiontree/CartNode.lua)0
-rw-r--r--contrib/lua-torch/decisiontree/CartTrainer.lua (renamed from contrib/torch/decisiontree/CartTrainer.lua)0
-rw-r--r--contrib/lua-torch/decisiontree/CartTree.lua (renamed from contrib/torch/decisiontree/CartTree.lua)0
-rw-r--r--contrib/lua-torch/decisiontree/DFD.lua (renamed from contrib/torch/decisiontree/DFD.lua)0
-rw-r--r--contrib/lua-torch/decisiontree/DataSet.lua (renamed from contrib/torch/decisiontree/DataSet.lua)0
-rw-r--r--contrib/lua-torch/decisiontree/DecisionForest.lua (renamed from contrib/torch/decisiontree/DecisionForest.lua)0
-rw-r--r--contrib/lua-torch/decisiontree/DecisionForestTrainer.lua (renamed from contrib/torch/decisiontree/DecisionForestTrainer.lua)0
-rw-r--r--contrib/lua-torch/decisiontree/DecisionTree.lua (renamed from contrib/torch/decisiontree/DecisionTree.lua)0
-rw-r--r--contrib/lua-torch/decisiontree/GBDT_common.h (renamed from contrib/torch/decisiontree/GBDT_common.h)0
-rw-r--r--contrib/lua-torch/decisiontree/GiniState.lua (renamed from contrib/torch/decisiontree/GiniState.lua)0
-rw-r--r--contrib/lua-torch/decisiontree/GradientBoostState.lua (renamed from contrib/torch/decisiontree/GradientBoostState.lua)0
-rw-r--r--contrib/lua-torch/decisiontree/GradientBoostTrainer.lua (renamed from contrib/torch/decisiontree/GradientBoostTrainer.lua)0
-rw-r--r--contrib/lua-torch/decisiontree/LICENSE (renamed from contrib/torch/decisiontree/LICENSE)0
-rw-r--r--contrib/lua-torch/decisiontree/LogitBoostCriterion.lua (renamed from contrib/torch/decisiontree/LogitBoostCriterion.lua)0
-rw-r--r--contrib/lua-torch/decisiontree/MSECriterion.lua (renamed from contrib/torch/decisiontree/MSECriterion.lua)0
-rw-r--r--contrib/lua-torch/decisiontree/README.md (renamed from contrib/torch/decisiontree/README.md)0
-rw-r--r--contrib/lua-torch/decisiontree/RandomForestTrainer.lua (renamed from contrib/torch/decisiontree/RandomForestTrainer.lua)0
-rw-r--r--contrib/lua-torch/decisiontree/Sparse2Dense.lua (renamed from contrib/torch/decisiontree/Sparse2Dense.lua)0
-rw-r--r--contrib/lua-torch/decisiontree/SparseTensor.lua (renamed from contrib/torch/decisiontree/SparseTensor.lua)0
-rw-r--r--contrib/lua-torch/decisiontree/TreeState.lua (renamed from contrib/torch/decisiontree/TreeState.lua)0
-rw-r--r--contrib/lua-torch/decisiontree/WorkPool.lua (renamed from contrib/torch/decisiontree/WorkPool.lua)0
-rw-r--r--contrib/lua-torch/decisiontree/_env.lua (renamed from contrib/torch/decisiontree/_env.lua)0
-rw-r--r--contrib/lua-torch/decisiontree/benchmark.lua (renamed from contrib/torch/decisiontree/benchmark.lua)0
-rw-r--r--contrib/lua-torch/decisiontree/doc/benchmark.md (renamed from contrib/torch/decisiontree/doc/benchmark.md)0
-rw-r--r--contrib/lua-torch/decisiontree/error.h (renamed from contrib/torch/decisiontree/error.h)0
-rw-r--r--contrib/lua-torch/decisiontree/generic/CartTree.c (renamed from contrib/torch/decisiontree/generic/CartTree.c)0
-rw-r--r--contrib/lua-torch/decisiontree/generic/DFD.c (renamed from contrib/torch/decisiontree/generic/DFD.c)0
-rw-r--r--contrib/lua-torch/decisiontree/generic/GBDT.c (renamed from contrib/torch/decisiontree/generic/GBDT.c)0
-rw-r--r--contrib/lua-torch/decisiontree/generic/GBDT_internal.c (renamed from contrib/torch/decisiontree/generic/GBDT_internal.c)0
-rw-r--r--contrib/lua-torch/decisiontree/generic/GBDT_internal.h (renamed from contrib/torch/decisiontree/generic/GBDT_internal.h)0
-rw-r--r--contrib/lua-torch/decisiontree/generic/LogitBoostCriterion.c (renamed from contrib/torch/decisiontree/generic/LogitBoostCriterion.c)0
-rw-r--r--contrib/lua-torch/decisiontree/generic/S2D.c (renamed from contrib/torch/decisiontree/generic/S2D.c)0
-rw-r--r--contrib/lua-torch/decisiontree/hash_map.c (renamed from contrib/torch/decisiontree/hash_map.c)0
-rw-r--r--contrib/lua-torch/decisiontree/hash_map.h (renamed from contrib/torch/decisiontree/hash_map.h)0
-rw-r--r--contrib/lua-torch/decisiontree/init.c (renamed from contrib/torch/decisiontree/init.c)0
-rw-r--r--contrib/lua-torch/decisiontree/init.lua (renamed from contrib/torch/decisiontree/init.lua)0
-rw-r--r--contrib/lua-torch/decisiontree/internal_hash_map.h (renamed from contrib/torch/decisiontree/internal_hash_map.h)0
-rw-r--r--contrib/lua-torch/decisiontree/khash.h (renamed from contrib/torch/decisiontree/khash.h)0
-rw-r--r--contrib/lua-torch/decisiontree/math.lua (renamed from contrib/torch/decisiontree/math.lua)0
-rw-r--r--contrib/lua-torch/decisiontree/rocks/decisiontree-scm-1.rockspec (renamed from contrib/torch/decisiontree/rocks/decisiontree-scm-1.rockspec)0
-rw-r--r--contrib/lua-torch/decisiontree/test.lua (renamed from contrib/torch/decisiontree/test.lua)0
-rw-r--r--contrib/lua-torch/decisiontree/utils.h (renamed from contrib/torch/decisiontree/utils.h)0
-rw-r--r--contrib/lua-torch/decisiontree/utils.lua (renamed from contrib/torch/decisiontree/utils.lua)0
-rw-r--r--contrib/lua-torch/nn/.gitignore (renamed from contrib/torch/nn/.gitignore)0
-rw-r--r--contrib/lua-torch/nn/.luacheckrc (renamed from contrib/torch/nn/.luacheckrc)0
-rw-r--r--contrib/lua-torch/nn/.travis.yml (renamed from contrib/torch/nn/.travis.yml)0
-rw-r--r--contrib/lua-torch/nn/Abs.lua (renamed from contrib/torch/nn/Abs.lua)0
-rw-r--r--contrib/lua-torch/nn/AbsCriterion.lua (renamed from contrib/torch/nn/AbsCriterion.lua)0
-rw-r--r--contrib/lua-torch/nn/Add.lua (renamed from contrib/torch/nn/Add.lua)0
-rw-r--r--contrib/lua-torch/nn/AddConstant.lua (renamed from contrib/torch/nn/AddConstant.lua)0
-rw-r--r--contrib/lua-torch/nn/BCECriterion.lua (renamed from contrib/torch/nn/BCECriterion.lua)0
-rw-r--r--contrib/lua-torch/nn/BatchNormalization.lua (renamed from contrib/torch/nn/BatchNormalization.lua)0
-rw-r--r--contrib/lua-torch/nn/Bilinear.lua (renamed from contrib/torch/nn/Bilinear.lua)0
-rw-r--r--contrib/lua-torch/nn/Bottle.lua (renamed from contrib/torch/nn/Bottle.lua)0
-rw-r--r--contrib/lua-torch/nn/CAdd.lua (renamed from contrib/torch/nn/CAdd.lua)0
-rw-r--r--contrib/lua-torch/nn/CAddTable.lua (renamed from contrib/torch/nn/CAddTable.lua)0
-rw-r--r--contrib/lua-torch/nn/CAddTensorTable.lua (renamed from contrib/torch/nn/CAddTensorTable.lua)0
-rw-r--r--contrib/lua-torch/nn/CDivTable.lua (renamed from contrib/torch/nn/CDivTable.lua)0
-rw-r--r--contrib/lua-torch/nn/CMakeLists.txt (renamed from contrib/torch/nn/CMakeLists.txt)0
-rw-r--r--contrib/lua-torch/nn/CMaxTable.lua (renamed from contrib/torch/nn/CMaxTable.lua)0
-rw-r--r--contrib/lua-torch/nn/CMinTable.lua (renamed from contrib/torch/nn/CMinTable.lua)0
-rw-r--r--contrib/lua-torch/nn/CMul.lua (renamed from contrib/torch/nn/CMul.lua)0
-rw-r--r--contrib/lua-torch/nn/CMulTable.lua (renamed from contrib/torch/nn/CMulTable.lua)0
-rw-r--r--contrib/lua-torch/nn/CONTRIBUTING.md (renamed from contrib/torch/nn/CONTRIBUTING.md)0
-rw-r--r--contrib/lua-torch/nn/COPYRIGHT.txt (renamed from contrib/torch/nn/COPYRIGHT.txt)0
-rw-r--r--contrib/lua-torch/nn/CReLU.lua (renamed from contrib/torch/nn/CReLU.lua)0
-rw-r--r--contrib/lua-torch/nn/CSubTable.lua (renamed from contrib/torch/nn/CSubTable.lua)0
-rw-r--r--contrib/lua-torch/nn/Clamp.lua (renamed from contrib/torch/nn/Clamp.lua)0
-rw-r--r--contrib/lua-torch/nn/ClassNLLCriterion.lua (renamed from contrib/torch/nn/ClassNLLCriterion.lua)0
-rw-r--r--contrib/lua-torch/nn/ClassSimplexCriterion.lua (renamed from contrib/torch/nn/ClassSimplexCriterion.lua)0
-rw-r--r--contrib/lua-torch/nn/Collapse.lua (renamed from contrib/torch/nn/Collapse.lua)0
-rw-r--r--contrib/lua-torch/nn/Concat.lua (renamed from contrib/torch/nn/Concat.lua)0
-rw-r--r--contrib/lua-torch/nn/ConcatTable.lua (renamed from contrib/torch/nn/ConcatTable.lua)0
-rw-r--r--contrib/lua-torch/nn/Constant.lua (renamed from contrib/torch/nn/Constant.lua)0
-rw-r--r--contrib/lua-torch/nn/Container.lua (renamed from contrib/torch/nn/Container.lua)0
-rwxr-xr-xcontrib/lua-torch/nn/Contiguous.lua (renamed from contrib/torch/nn/Contiguous.lua)0
-rw-r--r--contrib/lua-torch/nn/Convert.lua (renamed from contrib/torch/nn/Convert.lua)0
-rw-r--r--contrib/lua-torch/nn/Copy.lua (renamed from contrib/torch/nn/Copy.lua)0
-rw-r--r--contrib/lua-torch/nn/Cosine.lua (renamed from contrib/torch/nn/Cosine.lua)0
-rw-r--r--contrib/lua-torch/nn/CosineDistance.lua (renamed from contrib/torch/nn/CosineDistance.lua)0
-rw-r--r--contrib/lua-torch/nn/CosineEmbeddingCriterion.lua (renamed from contrib/torch/nn/CosineEmbeddingCriterion.lua)0
-rw-r--r--contrib/lua-torch/nn/Criterion.lua (renamed from contrib/torch/nn/Criterion.lua)0
-rw-r--r--contrib/lua-torch/nn/CriterionTable.lua (renamed from contrib/torch/nn/CriterionTable.lua)0
-rw-r--r--contrib/lua-torch/nn/CrossEntropyCriterion.lua (renamed from contrib/torch/nn/CrossEntropyCriterion.lua)0
-rw-r--r--contrib/lua-torch/nn/Decorator.lua (renamed from contrib/torch/nn/Decorator.lua)0
-rw-r--r--contrib/lua-torch/nn/DepthConcat.lua (renamed from contrib/torch/nn/DepthConcat.lua)0
-rw-r--r--contrib/lua-torch/nn/DistKLDivCriterion.lua (renamed from contrib/torch/nn/DistKLDivCriterion.lua)0
-rw-r--r--contrib/lua-torch/nn/DistanceRatioCriterion.lua (renamed from contrib/torch/nn/DistanceRatioCriterion.lua)0
-rw-r--r--contrib/lua-torch/nn/DontCast.lua (renamed from contrib/torch/nn/DontCast.lua)0
-rw-r--r--contrib/lua-torch/nn/DotProduct.lua (renamed from contrib/torch/nn/DotProduct.lua)0
-rw-r--r--contrib/lua-torch/nn/Dropout.lua (renamed from contrib/torch/nn/Dropout.lua)0
-rw-r--r--contrib/lua-torch/nn/ELU.lua (renamed from contrib/torch/nn/ELU.lua)0
-rw-r--r--contrib/lua-torch/nn/ErrorMessages.lua (renamed from contrib/torch/nn/ErrorMessages.lua)0
-rw-r--r--contrib/lua-torch/nn/Euclidean.lua (renamed from contrib/torch/nn/Euclidean.lua)0
-rw-r--r--contrib/lua-torch/nn/Exp.lua (renamed from contrib/torch/nn/Exp.lua)0
-rw-r--r--contrib/lua-torch/nn/FlattenTable.lua (renamed from contrib/torch/nn/FlattenTable.lua)0
-rw-r--r--contrib/lua-torch/nn/GPU.lua (renamed from contrib/torch/nn/GPU.lua)0
-rw-r--r--contrib/lua-torch/nn/GatedLinearUnit.lua (renamed from contrib/torch/nn/GatedLinearUnit.lua)0
-rw-r--r--contrib/lua-torch/nn/GradientReversal.lua (renamed from contrib/torch/nn/GradientReversal.lua)0
-rw-r--r--contrib/lua-torch/nn/HardShrink.lua (renamed from contrib/torch/nn/HardShrink.lua)0
-rw-r--r--contrib/lua-torch/nn/HardTanh.lua (renamed from contrib/torch/nn/HardTanh.lua)0
-rw-r--r--contrib/lua-torch/nn/HingeEmbeddingCriterion.lua (renamed from contrib/torch/nn/HingeEmbeddingCriterion.lua)0
-rw-r--r--contrib/lua-torch/nn/Identity.lua (renamed from contrib/torch/nn/Identity.lua)0
-rw-r--r--contrib/lua-torch/nn/Index.lua (renamed from contrib/torch/nn/Index.lua)0
-rw-r--r--contrib/lua-torch/nn/IndexLinear.lua (renamed from contrib/torch/nn/IndexLinear.lua)0
-rw-r--r--contrib/lua-torch/nn/Jacobian.lua (renamed from contrib/torch/nn/Jacobian.lua)0
-rw-r--r--contrib/lua-torch/nn/JoinTable.lua (renamed from contrib/torch/nn/JoinTable.lua)0
-rw-r--r--contrib/lua-torch/nn/Kmeans.lua (renamed from contrib/torch/nn/Kmeans.lua)0
-rw-r--r--contrib/lua-torch/nn/L1Cost.lua (renamed from contrib/torch/nn/L1Cost.lua)0
-rw-r--r--contrib/lua-torch/nn/L1HingeEmbeddingCriterion.lua (renamed from contrib/torch/nn/L1HingeEmbeddingCriterion.lua)0
-rw-r--r--contrib/lua-torch/nn/L1Penalty.lua (renamed from contrib/torch/nn/L1Penalty.lua)0
-rw-r--r--contrib/lua-torch/nn/LayerNormalization.lua (renamed from contrib/torch/nn/LayerNormalization.lua)0
-rw-r--r--contrib/lua-torch/nn/LeakyReLU.lua (renamed from contrib/torch/nn/LeakyReLU.lua)0
-rw-r--r--contrib/lua-torch/nn/Linear.lua (renamed from contrib/torch/nn/Linear.lua)0
-rwxr-xr-xcontrib/lua-torch/nn/LinearWeightNorm.lua (renamed from contrib/torch/nn/LinearWeightNorm.lua)0
-rw-r--r--contrib/lua-torch/nn/Log.lua (renamed from contrib/torch/nn/Log.lua)0
-rw-r--r--contrib/lua-torch/nn/LogSigmoid.lua (renamed from contrib/torch/nn/LogSigmoid.lua)0
-rw-r--r--contrib/lua-torch/nn/LogSoftMax.lua (renamed from contrib/torch/nn/LogSoftMax.lua)0
-rw-r--r--contrib/lua-torch/nn/LookupTable.lua (renamed from contrib/torch/nn/LookupTable.lua)0
-rw-r--r--contrib/lua-torch/nn/MM.lua (renamed from contrib/torch/nn/MM.lua)0
-rw-r--r--contrib/lua-torch/nn/MSECriterion.lua (renamed from contrib/torch/nn/MSECriterion.lua)0
-rw-r--r--contrib/lua-torch/nn/MV.lua (renamed from contrib/torch/nn/MV.lua)0
-rw-r--r--contrib/lua-torch/nn/MapTable.lua (renamed from contrib/torch/nn/MapTable.lua)0
-rw-r--r--contrib/lua-torch/nn/MarginCriterion.lua (renamed from contrib/torch/nn/MarginCriterion.lua)0
-rw-r--r--contrib/lua-torch/nn/MarginRankingCriterion.lua (renamed from contrib/torch/nn/MarginRankingCriterion.lua)0
-rw-r--r--contrib/lua-torch/nn/MaskedSelect.lua (renamed from contrib/torch/nn/MaskedSelect.lua)0
-rw-r--r--contrib/lua-torch/nn/Max.lua (renamed from contrib/torch/nn/Max.lua)0
-rw-r--r--contrib/lua-torch/nn/Maxout.lua (renamed from contrib/torch/nn/Maxout.lua)0
-rw-r--r--contrib/lua-torch/nn/Mean.lua (renamed from contrib/torch/nn/Mean.lua)0
-rw-r--r--contrib/lua-torch/nn/Min.lua (renamed from contrib/torch/nn/Min.lua)0
-rw-r--r--contrib/lua-torch/nn/MixtureTable.lua (renamed from contrib/torch/nn/MixtureTable.lua)0
-rw-r--r--contrib/lua-torch/nn/Module.lua (renamed from contrib/torch/nn/Module.lua)0
-rw-r--r--contrib/lua-torch/nn/ModuleCriterion.lua (renamed from contrib/torch/nn/ModuleCriterion.lua)0
-rw-r--r--contrib/lua-torch/nn/Mul.lua (renamed from contrib/torch/nn/Mul.lua)0
-rw-r--r--contrib/lua-torch/nn/MulConstant.lua (renamed from contrib/torch/nn/MulConstant.lua)0
-rw-r--r--contrib/lua-torch/nn/MultiCriterion.lua (renamed from contrib/torch/nn/MultiCriterion.lua)0
-rw-r--r--contrib/lua-torch/nn/MultiLabelMarginCriterion.lua (renamed from contrib/torch/nn/MultiLabelMarginCriterion.lua)0
-rw-r--r--contrib/lua-torch/nn/MultiLabelSoftMarginCriterion.lua (renamed from contrib/torch/nn/MultiLabelSoftMarginCriterion.lua)0
-rw-r--r--contrib/lua-torch/nn/MultiMarginCriterion.lua (renamed from contrib/torch/nn/MultiMarginCriterion.lua)0
-rw-r--r--contrib/lua-torch/nn/NaN.lua (renamed from contrib/torch/nn/NaN.lua)0
-rw-r--r--contrib/lua-torch/nn/Narrow.lua (renamed from contrib/torch/nn/Narrow.lua)0
-rw-r--r--contrib/lua-torch/nn/NarrowTable.lua (renamed from contrib/torch/nn/NarrowTable.lua)0
-rw-r--r--contrib/lua-torch/nn/Normalize.lua (renamed from contrib/torch/nn/Normalize.lua)0
-rw-r--r--contrib/lua-torch/nn/OneHot.lua (renamed from contrib/torch/nn/OneHot.lua)0
-rw-r--r--contrib/lua-torch/nn/PReLU.lua (renamed from contrib/torch/nn/PReLU.lua)0
-rw-r--r--contrib/lua-torch/nn/Padding.lua (renamed from contrib/torch/nn/Padding.lua)0
-rw-r--r--contrib/lua-torch/nn/PairwiseDistance.lua (renamed from contrib/torch/nn/PairwiseDistance.lua)0
-rw-r--r--contrib/lua-torch/nn/Parallel.lua (renamed from contrib/torch/nn/Parallel.lua)0
-rw-r--r--contrib/lua-torch/nn/ParallelCriterion.lua (renamed from contrib/torch/nn/ParallelCriterion.lua)0
-rw-r--r--contrib/lua-torch/nn/ParallelTable.lua (renamed from contrib/torch/nn/ParallelTable.lua)0
-rw-r--r--contrib/lua-torch/nn/PartialLinear.lua (renamed from contrib/torch/nn/PartialLinear.lua)0
-rw-r--r--contrib/lua-torch/nn/PixelShuffle.lua (renamed from contrib/torch/nn/PixelShuffle.lua)0
-rw-r--r--contrib/lua-torch/nn/Power.lua (renamed from contrib/torch/nn/Power.lua)0
-rw-r--r--contrib/lua-torch/nn/PrintSize.lua (renamed from contrib/torch/nn/PrintSize.lua)0
-rw-r--r--contrib/lua-torch/nn/Profile.lua (renamed from contrib/torch/nn/Profile.lua)0
-rw-r--r--contrib/lua-torch/nn/README.md (renamed from contrib/torch/nn/README.md)0
-rw-r--r--contrib/lua-torch/nn/RReLU.lua (renamed from contrib/torch/nn/RReLU.lua)0
-rw-r--r--contrib/lua-torch/nn/ReLU.lua (renamed from contrib/torch/nn/ReLU.lua)0
-rw-r--r--contrib/lua-torch/nn/ReLU6.lua (renamed from contrib/torch/nn/ReLU6.lua)0
-rw-r--r--contrib/lua-torch/nn/Replicate.lua (renamed from contrib/torch/nn/Replicate.lua)0
-rw-r--r--contrib/lua-torch/nn/Reshape.lua (renamed from contrib/torch/nn/Reshape.lua)0
-rw-r--r--contrib/lua-torch/nn/Select.lua (renamed from contrib/torch/nn/Select.lua)0
-rw-r--r--contrib/lua-torch/nn/SelectTable.lua (renamed from contrib/torch/nn/SelectTable.lua)0
-rw-r--r--contrib/lua-torch/nn/Sequential.lua (renamed from contrib/torch/nn/Sequential.lua)0
-rw-r--r--contrib/lua-torch/nn/Sigmoid.lua (renamed from contrib/torch/nn/Sigmoid.lua)0
-rw-r--r--contrib/lua-torch/nn/SmoothL1Criterion.lua (renamed from contrib/torch/nn/SmoothL1Criterion.lua)0
-rw-r--r--contrib/lua-torch/nn/SoftMarginCriterion.lua (renamed from contrib/torch/nn/SoftMarginCriterion.lua)0
-rw-r--r--contrib/lua-torch/nn/SoftMax.lua (renamed from contrib/torch/nn/SoftMax.lua)0
-rw-r--r--contrib/lua-torch/nn/SoftMin.lua (renamed from contrib/torch/nn/SoftMin.lua)0
-rw-r--r--contrib/lua-torch/nn/SoftPlus.lua (renamed from contrib/torch/nn/SoftPlus.lua)0
-rw-r--r--contrib/lua-torch/nn/SoftShrink.lua (renamed from contrib/torch/nn/SoftShrink.lua)0
-rw-r--r--contrib/lua-torch/nn/SoftSign.lua (renamed from contrib/torch/nn/SoftSign.lua)0
-rw-r--r--contrib/lua-torch/nn/SparseJacobian.lua (renamed from contrib/torch/nn/SparseJacobian.lua)0
-rw-r--r--contrib/lua-torch/nn/SparseLinear.lua (renamed from contrib/torch/nn/SparseLinear.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialAdaptiveAveragePooling.lua (renamed from contrib/torch/nn/SpatialAdaptiveAveragePooling.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialAdaptiveMaxPooling.lua (renamed from contrib/torch/nn/SpatialAdaptiveMaxPooling.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialAutoCropMSECriterion.lua (renamed from contrib/torch/nn/SpatialAutoCropMSECriterion.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialAveragePooling.lua (renamed from contrib/torch/nn/SpatialAveragePooling.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialBatchNormalization.lua (renamed from contrib/torch/nn/SpatialBatchNormalization.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialClassNLLCriterion.lua (renamed from contrib/torch/nn/SpatialClassNLLCriterion.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialContrastiveNormalization.lua (renamed from contrib/torch/nn/SpatialContrastiveNormalization.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialConvolution.lua (renamed from contrib/torch/nn/SpatialConvolution.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialConvolutionLocal.lua (renamed from contrib/torch/nn/SpatialConvolutionLocal.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialConvolutionMM.lua (renamed from contrib/torch/nn/SpatialConvolutionMM.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialConvolutionMap.lua (renamed from contrib/torch/nn/SpatialConvolutionMap.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialCrossMapLRN.lua (renamed from contrib/torch/nn/SpatialCrossMapLRN.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialDepthWiseConvolution.lua (renamed from contrib/torch/nn/SpatialDepthWiseConvolution.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialDilatedConvolution.lua (renamed from contrib/torch/nn/SpatialDilatedConvolution.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialDilatedMaxPooling.lua (renamed from contrib/torch/nn/SpatialDilatedMaxPooling.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialDivisiveNormalization.lua (renamed from contrib/torch/nn/SpatialDivisiveNormalization.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialDropout.lua (renamed from contrib/torch/nn/SpatialDropout.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialFractionalMaxPooling.lua (renamed from contrib/torch/nn/SpatialFractionalMaxPooling.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialFullConvolution.lua (renamed from contrib/torch/nn/SpatialFullConvolution.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialFullConvolutionMap.lua (renamed from contrib/torch/nn/SpatialFullConvolutionMap.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialLPPooling.lua (renamed from contrib/torch/nn/SpatialLPPooling.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialLogSoftMax.lua (renamed from contrib/torch/nn/SpatialLogSoftMax.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialMaxPooling.lua (renamed from contrib/torch/nn/SpatialMaxPooling.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialMaxUnpooling.lua (renamed from contrib/torch/nn/SpatialMaxUnpooling.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialReflectionPadding.lua (renamed from contrib/torch/nn/SpatialReflectionPadding.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialReplicationPadding.lua (renamed from contrib/torch/nn/SpatialReplicationPadding.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialSoftMax.lua (renamed from contrib/torch/nn/SpatialSoftMax.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialSubSampling.lua (renamed from contrib/torch/nn/SpatialSubSampling.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialSubtractiveNormalization.lua (renamed from contrib/torch/nn/SpatialSubtractiveNormalization.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialUpSamplingBilinear.lua (renamed from contrib/torch/nn/SpatialUpSamplingBilinear.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialUpSamplingNearest.lua (renamed from contrib/torch/nn/SpatialUpSamplingNearest.lua)0
-rw-r--r--contrib/lua-torch/nn/SpatialZeroPadding.lua (renamed from contrib/torch/nn/SpatialZeroPadding.lua)0
-rw-r--r--contrib/lua-torch/nn/SplitTable.lua (renamed from contrib/torch/nn/SplitTable.lua)0
-rw-r--r--contrib/lua-torch/nn/Sqrt.lua (renamed from contrib/torch/nn/Sqrt.lua)0
-rw-r--r--contrib/lua-torch/nn/Square.lua (renamed from contrib/torch/nn/Square.lua)0
-rw-r--r--contrib/lua-torch/nn/Squeeze.lua (renamed from contrib/torch/nn/Squeeze.lua)0
-rw-r--r--contrib/lua-torch/nn/StochasticGradient.lua (renamed from contrib/torch/nn/StochasticGradient.lua)0
-rw-r--r--contrib/lua-torch/nn/Sum.lua (renamed from contrib/torch/nn/Sum.lua)0
-rw-r--r--contrib/lua-torch/nn/THNN.lua (renamed from contrib/torch/nn/THNN.lua)0
-rw-r--r--contrib/lua-torch/nn/Tanh.lua (renamed from contrib/torch/nn/Tanh.lua)0
-rw-r--r--contrib/lua-torch/nn/TanhShrink.lua (renamed from contrib/torch/nn/TanhShrink.lua)0
-rw-r--r--contrib/lua-torch/nn/TemporalConvolution.lua (renamed from contrib/torch/nn/TemporalConvolution.lua)0
-rw-r--r--contrib/lua-torch/nn/TemporalDynamicKMaxPooling.lua (renamed from contrib/torch/nn/TemporalDynamicKMaxPooling.lua)0
-rw-r--r--contrib/lua-torch/nn/TemporalMaxPooling.lua (renamed from contrib/torch/nn/TemporalMaxPooling.lua)0
-rw-r--r--contrib/lua-torch/nn/TemporalRowConvolution.lua (renamed from contrib/torch/nn/TemporalRowConvolution.lua)0
-rw-r--r--contrib/lua-torch/nn/TemporalSubSampling.lua (renamed from contrib/torch/nn/TemporalSubSampling.lua)0
-rw-r--r--contrib/lua-torch/nn/Threshold.lua (renamed from contrib/torch/nn/Threshold.lua)0
-rw-r--r--contrib/lua-torch/nn/Transpose.lua (renamed from contrib/torch/nn/Transpose.lua)0
-rw-r--r--contrib/lua-torch/nn/Unsqueeze.lua (renamed from contrib/torch/nn/Unsqueeze.lua)0
-rw-r--r--contrib/lua-torch/nn/View.lua (renamed from contrib/torch/nn/View.lua)0
-rw-r--r--contrib/lua-torch/nn/VolumetricAveragePooling.lua (renamed from contrib/torch/nn/VolumetricAveragePooling.lua)0
-rw-r--r--contrib/lua-torch/nn/VolumetricBatchNormalization.lua (renamed from contrib/torch/nn/VolumetricBatchNormalization.lua)0
-rw-r--r--contrib/lua-torch/nn/VolumetricConvolution.lua (renamed from contrib/torch/nn/VolumetricConvolution.lua)0
-rw-r--r--contrib/lua-torch/nn/VolumetricDilatedConvolution.lua (renamed from contrib/torch/nn/VolumetricDilatedConvolution.lua)0
-rw-r--r--contrib/lua-torch/nn/VolumetricDilatedMaxPooling.lua (renamed from contrib/torch/nn/VolumetricDilatedMaxPooling.lua)0
-rw-r--r--contrib/lua-torch/nn/VolumetricDropout.lua (renamed from contrib/torch/nn/VolumetricDropout.lua)0
-rw-r--r--contrib/lua-torch/nn/VolumetricFractionalMaxPooling.lua (renamed from contrib/torch/nn/VolumetricFractionalMaxPooling.lua)0
-rw-r--r--contrib/lua-torch/nn/VolumetricFullConvolution.lua (renamed from contrib/torch/nn/VolumetricFullConvolution.lua)0
-rw-r--r--contrib/lua-torch/nn/VolumetricMaxPooling.lua (renamed from contrib/torch/nn/VolumetricMaxPooling.lua)0
-rw-r--r--contrib/lua-torch/nn/VolumetricMaxUnpooling.lua (renamed from contrib/torch/nn/VolumetricMaxUnpooling.lua)0
-rw-r--r--contrib/lua-torch/nn/VolumetricReplicationPadding.lua (renamed from contrib/torch/nn/VolumetricReplicationPadding.lua)0
-rw-r--r--contrib/lua-torch/nn/WeightNorm.lua (renamed from contrib/torch/nn/WeightNorm.lua)0
-rw-r--r--contrib/lua-torch/nn/WeightedEuclidean.lua (renamed from contrib/torch/nn/WeightedEuclidean.lua)0
-rw-r--r--contrib/lua-torch/nn/WeightedMSECriterion.lua (renamed from contrib/torch/nn/WeightedMSECriterion.lua)0
-rw-r--r--contrib/lua-torch/nn/WhiteNoise.lua (renamed from contrib/torch/nn/WhiteNoise.lua)0
-rw-r--r--contrib/lua-torch/nn/ZeroGrad.lua (renamed from contrib/torch/nn/ZeroGrad.lua)0
-rw-r--r--contrib/lua-torch/nn/ZipTable.lua (renamed from contrib/torch/nn/ZipTable.lua)0
-rw-r--r--contrib/lua-torch/nn/ZipTableOneToMany.lua (renamed from contrib/torch/nn/ZipTableOneToMany.lua)0
-rw-r--r--contrib/lua-torch/nn/hessian.lua (renamed from contrib/torch/nn/hessian.lua)0
-rwxr-xr-xcontrib/lua-torch/nn/init.lua (renamed from contrib/torch/nn/init.lua)0
-rw-r--r--contrib/lua-torch/nn/lib/CMakeLists.txt (renamed from contrib/torch/nn/lib/CMakeLists.txt)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/CMakeLists.txt (renamed from contrib/torch/nn/lib/THNN/CMakeLists.txt)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/README.md (renamed from contrib/torch/nn/lib/THNN/README.md)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/THNN.h (renamed from contrib/torch/nn/lib/THNN/THNN.h)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/Abs.c (renamed from contrib/torch/nn/lib/THNN/generic/Abs.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/AbsCriterion.c (renamed from contrib/torch/nn/lib/THNN/generic/AbsCriterion.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/BCECriterion.c (renamed from contrib/torch/nn/lib/THNN/generic/BCECriterion.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/BatchNormalization.c (renamed from contrib/torch/nn/lib/THNN/generic/BatchNormalization.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/ClassNLLCriterion.c (renamed from contrib/torch/nn/lib/THNN/generic/ClassNLLCriterion.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/DistKLDivCriterion.c (renamed from contrib/torch/nn/lib/THNN/generic/DistKLDivCriterion.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/ELU.c (renamed from contrib/torch/nn/lib/THNN/generic/ELU.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/FusedRNNKernel.c (renamed from contrib/torch/nn/lib/THNN/generic/FusedRNNKernel.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/GatedLinearUnit.c (renamed from contrib/torch/nn/lib/THNN/generic/GatedLinearUnit.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/HardShrink.c (renamed from contrib/torch/nn/lib/THNN/generic/HardShrink.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/HardTanh.c (renamed from contrib/torch/nn/lib/THNN/generic/HardTanh.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/IndexLinear.c (renamed from contrib/torch/nn/lib/THNN/generic/IndexLinear.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/L1Cost.c (renamed from contrib/torch/nn/lib/THNN/generic/L1Cost.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/LeakyReLU.c (renamed from contrib/torch/nn/lib/THNN/generic/LeakyReLU.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/Linear.c (renamed from contrib/torch/nn/lib/THNN/generic/Linear.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/LogSigmoid.c (renamed from contrib/torch/nn/lib/THNN/generic/LogSigmoid.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/LogSoftMax.c (renamed from contrib/torch/nn/lib/THNN/generic/LogSoftMax.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/LookupTable.c (renamed from contrib/torch/nn/lib/THNN/generic/LookupTable.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/MSECriterion.c (renamed from contrib/torch/nn/lib/THNN/generic/MSECriterion.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/MarginCriterion.c (renamed from contrib/torch/nn/lib/THNN/generic/MarginCriterion.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/MultiLabelMarginCriterion.c (renamed from contrib/torch/nn/lib/THNN/generic/MultiLabelMarginCriterion.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/MultiMarginCriterion.c (renamed from contrib/torch/nn/lib/THNN/generic/MultiMarginCriterion.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/PReLU.c (renamed from contrib/torch/nn/lib/THNN/generic/PReLU.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/RReLU.c (renamed from contrib/torch/nn/lib/THNN/generic/RReLU.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/Sigmoid.c (renamed from contrib/torch/nn/lib/THNN/generic/Sigmoid.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/SmoothL1Criterion.c (renamed from contrib/torch/nn/lib/THNN/generic/SmoothL1Criterion.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/SoftMarginCriterion.c (renamed from contrib/torch/nn/lib/THNN/generic/SoftMarginCriterion.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/SoftMax.c (renamed from contrib/torch/nn/lib/THNN/generic/SoftMax.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/SoftPlus.c (renamed from contrib/torch/nn/lib/THNN/generic/SoftPlus.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/SoftShrink.c (renamed from contrib/torch/nn/lib/THNN/generic/SoftShrink.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/SparseLinear.c (renamed from contrib/torch/nn/lib/THNN/generic/SparseLinear.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/SpatialAdaptiveAveragePooling.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialAdaptiveAveragePooling.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/SpatialAdaptiveMaxPooling.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialAdaptiveMaxPooling.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/SpatialAveragePooling.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialAveragePooling.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/SpatialClassNLLCriterion.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialClassNLLCriterion.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/SpatialConvolutionLocal.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialConvolutionLocal.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/SpatialConvolutionMM.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialConvolutionMM.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/SpatialConvolutionMap.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialConvolutionMap.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/SpatialDepthWiseConvolution.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialDepthWiseConvolution.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/SpatialDilatedConvolution.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialDilatedConvolution.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/SpatialDilatedMaxPooling.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialDilatedMaxPooling.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/SpatialFractionalMaxPooling.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialFractionalMaxPooling.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/SpatialFullConvolution.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialFullConvolution.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/SpatialFullConvolutionMap.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialFullConvolutionMap.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/SpatialMaxPooling.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialMaxPooling.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/SpatialMaxUnpooling.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialMaxUnpooling.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/SpatialReflectionPadding.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialReflectionPadding.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/SpatialReplicationPadding.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialReplicationPadding.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/SpatialSubSampling.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialSubSampling.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/SpatialUpSamplingBilinear.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialUpSamplingBilinear.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/SpatialUpSamplingNearest.c (renamed from contrib/torch/nn/lib/THNN/generic/SpatialUpSamplingNearest.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/Sqrt.c (renamed from contrib/torch/nn/lib/THNN/generic/Sqrt.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/Square.c (renamed from contrib/torch/nn/lib/THNN/generic/Square.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/THNN.h (renamed from contrib/torch/nn/lib/THNN/generic/THNN.h)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/Tanh.c (renamed from contrib/torch/nn/lib/THNN/generic/Tanh.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/TemporalConvolution.c (renamed from contrib/torch/nn/lib/THNN/generic/TemporalConvolution.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/TemporalMaxPooling.c (renamed from contrib/torch/nn/lib/THNN/generic/TemporalMaxPooling.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/TemporalRowConvolution.c (renamed from contrib/torch/nn/lib/THNN/generic/TemporalRowConvolution.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/TemporalSubSampling.c (renamed from contrib/torch/nn/lib/THNN/generic/TemporalSubSampling.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/Threshold.c (renamed from contrib/torch/nn/lib/THNN/generic/Threshold.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/VolumetricAveragePooling.c (renamed from contrib/torch/nn/lib/THNN/generic/VolumetricAveragePooling.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/VolumetricConvolution.c (renamed from contrib/torch/nn/lib/THNN/generic/VolumetricConvolution.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/VolumetricConvolutionMM.c (renamed from contrib/torch/nn/lib/THNN/generic/VolumetricConvolutionMM.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/VolumetricDilatedConvolution.c (renamed from contrib/torch/nn/lib/THNN/generic/VolumetricDilatedConvolution.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/VolumetricDilatedMaxPooling.c (renamed from contrib/torch/nn/lib/THNN/generic/VolumetricDilatedMaxPooling.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/VolumetricFractionalMaxPooling.c (renamed from contrib/torch/nn/lib/THNN/generic/VolumetricFractionalMaxPooling.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/VolumetricFullConvolution.c (renamed from contrib/torch/nn/lib/THNN/generic/VolumetricFullConvolution.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/VolumetricMaxPooling.c (renamed from contrib/torch/nn/lib/THNN/generic/VolumetricMaxPooling.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/VolumetricMaxUnpooling.c (renamed from contrib/torch/nn/lib/THNN/generic/VolumetricMaxUnpooling.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/VolumetricReplicationPadding.c (renamed from contrib/torch/nn/lib/THNN/generic/VolumetricReplicationPadding.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/VolumetricUpSamplingNearest.c (renamed from contrib/torch/nn/lib/THNN/generic/VolumetricUpSamplingNearest.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/VolumetricUpSamplingTrilinear.c (renamed from contrib/torch/nn/lib/THNN/generic/VolumetricUpSamplingTrilinear.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/generic/unfold.c (renamed from contrib/torch/nn/lib/THNN/generic/unfold.c)0
-rw-r--r--contrib/lua-torch/nn/lib/THNN/init.c (renamed from contrib/torch/nn/lib/THNN/init.c)0
-rw-r--r--contrib/lua-torch/nn/mkdocs.yml (renamed from contrib/torch/nn/mkdocs.yml)0
-rwxr-xr-xcontrib/lua-torch/nn/test.lua (renamed from contrib/torch/nn/test.lua)0
-rw-r--r--contrib/lua-torch/nn/utils.lua (renamed from contrib/torch/nn/utils.lua)0
-rw-r--r--contrib/lua-torch/optim/CMakeLists.txt (renamed from contrib/torch/optim/CMakeLists.txt)0
-rw-r--r--contrib/lua-torch/optim/COPYRIGHT.txt (renamed from contrib/torch/optim/COPYRIGHT.txt)0
-rw-r--r--contrib/lua-torch/optim/ConfusionMatrix.lua (renamed from contrib/torch/optim/ConfusionMatrix.lua)0
-rw-r--r--contrib/lua-torch/optim/Logger.lua (renamed from contrib/torch/optim/Logger.lua)0
-rw-r--r--contrib/lua-torch/optim/adadelta.lua (renamed from contrib/torch/optim/adadelta.lua)0
-rw-r--r--contrib/lua-torch/optim/adagrad.lua (renamed from contrib/torch/optim/adagrad.lua)0
-rw-r--r--contrib/lua-torch/optim/adam.lua (renamed from contrib/torch/optim/adam.lua)0
-rw-r--r--contrib/lua-torch/optim/adamax.lua (renamed from contrib/torch/optim/adamax.lua)0
-rw-r--r--contrib/lua-torch/optim/asgd.lua (renamed from contrib/torch/optim/asgd.lua)0
-rw-r--r--contrib/lua-torch/optim/cg.lua (renamed from contrib/torch/optim/cg.lua)0
-rw-r--r--contrib/lua-torch/optim/checkgrad.lua (renamed from contrib/torch/optim/checkgrad.lua)0
-rw-r--r--contrib/lua-torch/optim/cmaes.lua (renamed from contrib/torch/optim/cmaes.lua)0
-rw-r--r--contrib/lua-torch/optim/de.lua (renamed from contrib/torch/optim/de.lua)0
-rw-r--r--contrib/lua-torch/optim/fista.lua (renamed from contrib/torch/optim/fista.lua)0
-rw-r--r--contrib/lua-torch/optim/init.lua (renamed from contrib/torch/optim/init.lua)0
-rw-r--r--contrib/lua-torch/optim/lbfgs.lua (renamed from contrib/torch/optim/lbfgs.lua)0
-rw-r--r--contrib/lua-torch/optim/lswolfe.lua (renamed from contrib/torch/optim/lswolfe.lua)0
-rw-r--r--contrib/lua-torch/optim/nag.lua (renamed from contrib/torch/optim/nag.lua)0
-rw-r--r--contrib/lua-torch/optim/polyinterp.lua (renamed from contrib/torch/optim/polyinterp.lua)0
-rw-r--r--contrib/lua-torch/optim/rmsprop.lua (renamed from contrib/torch/optim/rmsprop.lua)0
-rw-r--r--contrib/lua-torch/optim/rprop.lua (renamed from contrib/torch/optim/rprop.lua)0
-rw-r--r--contrib/lua-torch/optim/sgd.lua (renamed from contrib/torch/optim/sgd.lua)0
-rw-r--r--contrib/lua-torch/paths/CMakeLists.txt (renamed from contrib/torch/paths/CMakeLists.txt)0
-rw-r--r--contrib/lua-torch/paths/COPYRIGHT.txt (renamed from contrib/torch/paths/COPYRIGHT.txt)0
-rw-r--r--contrib/lua-torch/paths/README.md (renamed from contrib/torch/paths/README.md)0
-rw-r--r--contrib/lua-torch/paths/init.lua (renamed from contrib/torch/paths/init.lua)0
-rw-r--r--contrib/lua-torch/paths/mkdocs.yml (renamed from contrib/torch/paths/mkdocs.yml)0
-rw-r--r--contrib/lua-torch/paths/paths.c (renamed from contrib/torch/paths/paths.c)0
-rw-r--r--contrib/lua-torch/paths/paths.h.in (renamed from contrib/torch/paths/paths.h.in)0
-rw-r--r--contrib/lua-torch/torch7/CMakeLists.txt (renamed from contrib/torch/torch7/CMakeLists.txt)20
-rw-r--r--contrib/lua-torch/torch7/COPYRIGHT.txt (renamed from contrib/torch/torch7/COPYRIGHT.txt)0
-rw-r--r--contrib/lua-torch/torch7/CmdLine.lua (renamed from contrib/torch/torch7/CmdLine.lua)0
-rw-r--r--contrib/lua-torch/torch7/DiskFile.c (renamed from contrib/torch/torch7/DiskFile.c)0
-rw-r--r--contrib/lua-torch/torch7/FFInterface.lua (renamed from contrib/torch/torch7/FFInterface.lua)0
-rw-r--r--contrib/lua-torch/torch7/File.c (renamed from contrib/torch/torch7/File.c)0
-rw-r--r--contrib/lua-torch/torch7/File.lua (renamed from contrib/torch/torch7/File.lua)0
-rw-r--r--contrib/lua-torch/torch7/Generator.c (renamed from contrib/torch/torch7/Generator.c)0
-rw-r--r--contrib/lua-torch/torch7/MemoryFile.c (renamed from contrib/torch/torch7/MemoryFile.c)0
-rw-r--r--contrib/lua-torch/torch7/PipeFile.c (renamed from contrib/torch/torch7/PipeFile.c)0
-rw-r--r--contrib/lua-torch/torch7/README.md (renamed from contrib/torch/torch7/README.md)0
-rw-r--r--contrib/lua-torch/torch7/ROADMAP.md (renamed from contrib/torch/torch7/ROADMAP.md)0
-rw-r--r--contrib/lua-torch/torch7/Storage.c (renamed from contrib/torch/torch7/Storage.c)0
-rw-r--r--contrib/lua-torch/torch7/Tensor.c (renamed from contrib/torch/torch7/Tensor.c)0
-rw-r--r--contrib/lua-torch/torch7/Tensor.lua (renamed from contrib/torch/torch7/Tensor.lua)0
-rw-r--r--contrib/lua-torch/torch7/TensorMath.c (renamed from contrib/torch/torch7/TensorMath.c)0
-rw-r--r--contrib/lua-torch/torch7/TensorMath.lua (renamed from contrib/torch/torch7/TensorMath.lua)0
-rw-r--r--contrib/lua-torch/torch7/TensorOperator.c (renamed from contrib/torch/torch7/TensorOperator.c)0
-rw-r--r--contrib/lua-torch/torch7/TestSuite.lua (renamed from contrib/torch/torch7/TestSuite.lua)0
-rw-r--r--contrib/lua-torch/torch7/Tester.lua (renamed from contrib/torch/torch7/Tester.lua)0
-rw-r--r--contrib/lua-torch/torch7/Timer.c (renamed from contrib/torch/torch7/Timer.c)0
-rw-r--r--contrib/lua-torch/torch7/cmake/TorchConfig.cmake.in (renamed from contrib/torch/torch7/cmake/TorchConfig.cmake.in)2
-rw-r--r--contrib/lua-torch/torch7/cmake/TorchExports.cmake (renamed from contrib/torch/torch7/cmake/TorchExports.cmake)0
-rw-r--r--contrib/lua-torch/torch7/cmake/TorchPackage.cmake (renamed from contrib/torch/torch7/cmake/TorchPackage.cmake)8
-rw-r--r--contrib/lua-torch/torch7/cmake/TorchPaths.cmake (renamed from contrib/torch/torch7/cmake/TorchPaths.cmake)0
-rw-r--r--contrib/lua-torch/torch7/cmake/TorchPathsInit.cmake (renamed from contrib/torch/torch7/cmake/TorchPathsInit.cmake)0
-rw-r--r--contrib/lua-torch/torch7/cmake/TorchWrap.cmake (renamed from contrib/torch/torch7/cmake/TorchWrap.cmake)0
-rw-r--r--contrib/lua-torch/torch7/cmake/TorchWrap.cmake.in (renamed from contrib/torch/torch7/cmake/TorchWrap.cmake.in)0
-rw-r--r--contrib/lua-torch/torch7/general.h (renamed from contrib/torch/torch7/general.h)0
-rw-r--r--contrib/lua-torch/torch7/generic/Storage.c (renamed from contrib/torch/torch7/generic/Storage.c)0
-rw-r--r--contrib/lua-torch/torch7/generic/Tensor.c (renamed from contrib/torch/torch7/generic/Tensor.c)0
-rw-r--r--contrib/lua-torch/torch7/generic/TensorOperator.c (renamed from contrib/torch/torch7/generic/TensorOperator.c)0
-rw-r--r--contrib/lua-torch/torch7/generic/luaG.h (renamed from contrib/torch/torch7/generic/luaG.h)0
-rw-r--r--contrib/lua-torch/torch7/init.c (renamed from contrib/torch/torch7/init.c)0
-rw-r--r--contrib/lua-torch/torch7/init.lua (renamed from contrib/torch/torch7/init.lua)0
-rw-r--r--contrib/lua-torch/torch7/lib/CMakeLists.txt (renamed from contrib/torch/torch7/lib/CMakeLists.txt)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/CMakeLists.txt (renamed from contrib/torch/torch7/lib/TH/CMakeLists.txt)2
-rw-r--r--contrib/lua-torch/torch7/lib/TH/README.md (renamed from contrib/torch/torch7/lib/TH/README.md)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/TH.h (renamed from contrib/torch/torch7/lib/TH/TH.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THAllocator.c (renamed from contrib/torch/torch7/lib/TH/THAllocator.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THAllocator.h (renamed from contrib/torch/torch7/lib/TH/THAllocator.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THAtomic.c (renamed from contrib/torch/torch7/lib/TH/THAtomic.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THAtomic.h (renamed from contrib/torch/torch7/lib/TH/THAtomic.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THBlas.c (renamed from contrib/torch/torch7/lib/TH/THBlas.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THBlas.h (renamed from contrib/torch/torch7/lib/TH/THBlas.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THConfig.cmake.in (renamed from contrib/torch/torch7/lib/TH/THConfig.cmake.in)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THDiskFile.c (renamed from contrib/torch/torch7/lib/TH/THDiskFile.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THDiskFile.h (renamed from contrib/torch/torch7/lib/TH/THDiskFile.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THFile.c (renamed from contrib/torch/torch7/lib/TH/THFile.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THFile.h (renamed from contrib/torch/torch7/lib/TH/THFile.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THFilePrivate.h (renamed from contrib/torch/torch7/lib/TH/THFilePrivate.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THGeneral.c (renamed from contrib/torch/torch7/lib/TH/THGeneral.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THGeneral.h.in (renamed from contrib/torch/torch7/lib/TH/THGeneral.h.in)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THGenerateAllTypes.h (renamed from contrib/torch/torch7/lib/TH/THGenerateAllTypes.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THGenerateByteType.h (renamed from contrib/torch/torch7/lib/TH/THGenerateByteType.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THGenerateCharType.h (renamed from contrib/torch/torch7/lib/TH/THGenerateCharType.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THGenerateDoubleType.h (renamed from contrib/torch/torch7/lib/TH/THGenerateDoubleType.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THGenerateFloatType.h (renamed from contrib/torch/torch7/lib/TH/THGenerateFloatType.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THGenerateFloatTypes.h (renamed from contrib/torch/torch7/lib/TH/THGenerateFloatTypes.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THGenerateHalfType.h (renamed from contrib/torch/torch7/lib/TH/THGenerateHalfType.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THGenerateIntType.h (renamed from contrib/torch/torch7/lib/TH/THGenerateIntType.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THGenerateIntTypes.h (renamed from contrib/torch/torch7/lib/TH/THGenerateIntTypes.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THGenerateLongType.h (renamed from contrib/torch/torch7/lib/TH/THGenerateLongType.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THGenerateShortType.h (renamed from contrib/torch/torch7/lib/TH/THGenerateShortType.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THHalf.c (renamed from contrib/torch/torch7/lib/TH/THHalf.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THHalf.h (renamed from contrib/torch/torch7/lib/TH/THHalf.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THLapack.c (renamed from contrib/torch/torch7/lib/TH/THLapack.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THLapack.h (renamed from contrib/torch/torch7/lib/TH/THLapack.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THLogAdd.c (renamed from contrib/torch/torch7/lib/TH/THLogAdd.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THLogAdd.h (renamed from contrib/torch/torch7/lib/TH/THLogAdd.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THMath.h (renamed from contrib/torch/torch7/lib/TH/THMath.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THMemoryFile.c (renamed from contrib/torch/torch7/lib/TH/THMemoryFile.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THMemoryFile.h (renamed from contrib/torch/torch7/lib/TH/THMemoryFile.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THRandom.c (renamed from contrib/torch/torch7/lib/TH/THRandom.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THRandom.h (renamed from contrib/torch/torch7/lib/TH/THRandom.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THSize.c (renamed from contrib/torch/torch7/lib/TH/THSize.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THSize.h (renamed from contrib/torch/torch7/lib/TH/THSize.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THStorage.c (renamed from contrib/torch/torch7/lib/TH/THStorage.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THStorage.h (renamed from contrib/torch/torch7/lib/TH/THStorage.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THTensor.c (renamed from contrib/torch/torch7/lib/TH/THTensor.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THTensor.h (renamed from contrib/torch/torch7/lib/TH/THTensor.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THTensorApply.h (renamed from contrib/torch/torch7/lib/TH/THTensorApply.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THTensorDimApply.h (renamed from contrib/torch/torch7/lib/TH/THTensorDimApply.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THTensorMacros.h (renamed from contrib/torch/torch7/lib/TH/THTensorMacros.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THVector.c (renamed from contrib/torch/torch7/lib/TH/THVector.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/THVector.h (renamed from contrib/torch/torch7/lib/TH/THVector.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/cmake/FindARM.cmake (renamed from contrib/torch/torch7/lib/TH/cmake/FindARM.cmake)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/cmake/FindBLAS.cmake (renamed from contrib/torch/torch7/lib/TH/cmake/FindBLAS.cmake)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/cmake/FindLAPACK.cmake (renamed from contrib/torch/torch7/lib/TH/cmake/FindLAPACK.cmake)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/cmake/FindMKL.cmake (renamed from contrib/torch/torch7/lib/TH/cmake/FindMKL.cmake)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/cmake/FindSSE.cmake (renamed from contrib/torch/torch7/lib/TH/cmake/FindSSE.cmake)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/THBlas.c (renamed from contrib/torch/torch7/lib/TH/generic/THBlas.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/THBlas.h (renamed from contrib/torch/torch7/lib/TH/generic/THBlas.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/THLapack.c (renamed from contrib/torch/torch7/lib/TH/generic/THLapack.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/THLapack.h (renamed from contrib/torch/torch7/lib/TH/generic/THLapack.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/THStorage.c (renamed from contrib/torch/torch7/lib/TH/generic/THStorage.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/THStorage.h (renamed from contrib/torch/torch7/lib/TH/generic/THStorage.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/THStorageCopy.c (renamed from contrib/torch/torch7/lib/TH/generic/THStorageCopy.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/THStorageCopy.h (renamed from contrib/torch/torch7/lib/TH/generic/THStorageCopy.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/THTensor.c (renamed from contrib/torch/torch7/lib/TH/generic/THTensor.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/THTensor.h (renamed from contrib/torch/torch7/lib/TH/generic/THTensor.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/THTensorConv.c (renamed from contrib/torch/torch7/lib/TH/generic/THTensorConv.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/THTensorConv.h (renamed from contrib/torch/torch7/lib/TH/generic/THTensorConv.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/THTensorCopy.c (renamed from contrib/torch/torch7/lib/TH/generic/THTensorCopy.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/THTensorCopy.h (renamed from contrib/torch/torch7/lib/TH/generic/THTensorCopy.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/THTensorLapack.c (renamed from contrib/torch/torch7/lib/TH/generic/THTensorLapack.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/THTensorLapack.h (renamed from contrib/torch/torch7/lib/TH/generic/THTensorLapack.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/THTensorMath.c (renamed from contrib/torch/torch7/lib/TH/generic/THTensorMath.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/THTensorMath.h (renamed from contrib/torch/torch7/lib/TH/generic/THTensorMath.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/THTensorRandom.c (renamed from contrib/torch/torch7/lib/TH/generic/THTensorRandom.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/THTensorRandom.h (renamed from contrib/torch/torch7/lib/TH/generic/THTensorRandom.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/THVector.h (renamed from contrib/torch/torch7/lib/TH/generic/THVector.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/THVectorDefault.c (renamed from contrib/torch/torch7/lib/TH/generic/THVectorDefault.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/THVectorDispatch.c (renamed from contrib/torch/torch7/lib/TH/generic/THVectorDispatch.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/simd/common_simd.h (renamed from contrib/torch/torch7/lib/TH/generic/simd/common_simd.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/simd/convolve.c (renamed from contrib/torch/torch7/lib/TH/generic/simd/convolve.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/simd/convolve.h (renamed from contrib/torch/torch7/lib/TH/generic/simd/convolve.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/simd/convolve5x5_avx.c (renamed from contrib/torch/torch7/lib/TH/generic/simd/convolve5x5_avx.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/simd/convolve5x5_sse.c (renamed from contrib/torch/torch7/lib/TH/generic/simd/convolve5x5_sse.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/generic/simd/simd.h (renamed from contrib/torch/torch7/lib/TH/generic/simd/simd.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/vector/AVX.c (renamed from contrib/torch/torch7/lib/TH/vector/AVX.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/vector/AVX.h (renamed from contrib/torch/torch7/lib/TH/vector/AVX.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/vector/AVX2.c (renamed from contrib/torch/torch7/lib/TH/vector/AVX2.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/vector/AVX2.h (renamed from contrib/torch/torch7/lib/TH/vector/AVX2.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/vector/NEON.c (renamed from contrib/torch/torch7/lib/TH/vector/NEON.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/vector/SSE.c (renamed from contrib/torch/torch7/lib/TH/vector/SSE.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/TH/vector/VSX.c (renamed from contrib/torch/torch7/lib/TH/vector/VSX.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/luaT/CMakeLists.txt (renamed from contrib/torch/torch7/lib/luaT/CMakeLists.txt)0
-rw-r--r--contrib/lua-torch/torch7/lib/luaT/README.md (renamed from contrib/torch/torch7/lib/luaT/README.md)0
-rw-r--r--contrib/lua-torch/torch7/lib/luaT/luaT.c (renamed from contrib/torch/torch7/lib/luaT/luaT.c)0
-rw-r--r--contrib/lua-torch/torch7/lib/luaT/luaT.h (renamed from contrib/torch/torch7/lib/luaT/luaT.h)0
-rw-r--r--contrib/lua-torch/torch7/lib/luaT/luaTConfig.cmake.in (renamed from contrib/torch/torch7/lib/luaT/luaTConfig.cmake.in)0
-rw-r--r--contrib/lua-torch/torch7/mkdocs.yml (renamed from contrib/torch/torch7/mkdocs.yml)0
-rw-r--r--contrib/lua-torch/torch7/paths.lua.in (renamed from contrib/torch/torch7/paths.lua.in)0
-rw-r--r--contrib/lua-torch/torch7/random.c (renamed from contrib/torch/torch7/random.c)0
-rw-r--r--contrib/lua-torch/torch7/random.lua (renamed from contrib/torch/torch7/random.lua)0
-rw-r--r--contrib/lua-torch/torch7/test/longSize.lua (renamed from contrib/torch/torch7/test/longSize.lua)0
-rw-r--r--contrib/lua-torch/torch7/test/test.lua (renamed from contrib/torch/torch7/test/test.lua)0
-rw-r--r--contrib/lua-torch/torch7/test/test_Multinomial.lua (renamed from contrib/torch/torch7/test/test_Multinomial.lua)0
-rw-r--r--contrib/lua-torch/torch7/test/test_Tester.lua (renamed from contrib/torch/torch7/test/test_Tester.lua)0
-rw-r--r--contrib/lua-torch/torch7/test/test_half.lua (renamed from contrib/torch/torch7/test/test_half.lua)0
-rw-r--r--contrib/lua-torch/torch7/test/test_qr.lua (renamed from contrib/torch/torch7/test/test_qr.lua)0
-rw-r--r--contrib/lua-torch/torch7/test/test_sharedmem.lua (renamed from contrib/torch/torch7/test/test_sharedmem.lua)0
-rw-r--r--contrib/lua-torch/torch7/test/test_timer.lua (renamed from contrib/torch/torch7/test/test_timer.lua)0
-rw-r--r--contrib/lua-torch/torch7/test/test_writeObject.lua (renamed from contrib/torch/torch7/test/test_writeObject.lua)0
-rw-r--r--contrib/lua-torch/torch7/test/timeSort.lua (renamed from contrib/torch/torch7/test/timeSort.lua)0
-rw-r--r--contrib/lua-torch/torch7/torchcwrap.lua (renamed from contrib/torch/torch7/torchcwrap.lua)0
-rw-r--r--contrib/lua-torch/torch7/utils.c (renamed from contrib/torch/torch7/utils.c)0
-rw-r--r--contrib/lua-torch/torch7/utils.h (renamed from contrib/torch/torch7/utils.h)0
-rw-r--r--contrib/t1ha/t1ha1.c51
-rw-r--r--contrib/t1ha/t1ha2.c95
-rw-r--r--contrib/t1ha/t1ha_bits.h226
-rw-r--r--interface/js/app/history.js25
-rw-r--r--interface/js/app/symbols.js12
-rw-r--r--lualib/ansicolors.lua (renamed from lualib/rspamadm/ansicolors.lua)0
-rw-r--r--lualib/plugins_stats.lua (renamed from lualib/rspamadm/plugins_stats.lua)2
-rw-r--r--lualib/rescore_utility.lua (renamed from lualib/rspamadm/rescore_utility.lua)99
-rw-r--r--lualib/rspamadm/confighelp.lua18
-rw-r--r--lualib/rspamadm/configwizard.lua210
-rw-r--r--lualib/rspamadm/corpus_test.lua76
-rw-r--r--lualib/rspamadm/fuzzy_stat.lua25
-rw-r--r--lualib/rspamadm/getopt.lua50
-rw-r--r--lualib/rspamadm/grep.lua66
-rw-r--r--lualib/rspamadm/keypair.lua269
-rw-r--r--lualib/rspamadm/rescore.lua251
-rw-r--r--rules/regexp/headers.lua10
-rw-r--r--src/libcryptobox/cryptobox.c9
-rw-r--r--src/libcryptobox/cryptobox.h3
-rw-r--r--src/libcryptobox/keypair.c2
-rw-r--r--src/libmime/email_addr.c55
-rw-r--r--src/libmime/message.c84
-rw-r--r--src/libmime/message.h6
-rw-r--r--src/libserver/cfg_file.h8
-rw-r--r--src/libserver/cfg_rcl.c2028
-rw-r--r--src/libserver/cfg_rcl.h39
-rw-r--r--src/libserver/cfg_utils.c33
-rw-r--r--src/libserver/composites.c2
-rw-r--r--src/libserver/dkim.c128
-rw-r--r--src/libserver/symbols_cache.c2
-rw-r--r--src/libserver/task.c6
-rw-r--r--src/libserver/task.h31
-rw-r--r--src/libserver/url.c47
-rw-r--r--src/libstat/stat_process.c9
-rw-r--r--src/libutil/map.c2
-rw-r--r--src/libutil/radix.c42
-rw-r--r--src/libutil/str_util.c87
-rw-r--r--src/libutil/util.c97
-rw-r--r--src/libutil/util.h13
-rw-r--r--src/lua/lua_common.c312
-rw-r--r--src/lua/lua_common.h16
-rw-r--r--src/lua/lua_config.c198
-rw-r--r--src/lua/lua_cryptobox.c515
-rw-r--r--src/lua/lua_mimepart.c29
-rw-r--r--src/lua/lua_task.c10
-rw-r--r--src/lua/lua_util.c197
-rw-r--r--src/plugins/fuzzy_check.c2
-rw-r--r--src/plugins/lua/dmarc.lua30
-rw-r--r--src/plugins/lua/elastic.lua5
-rw-r--r--src/plugins/lua/mime_types.lua5
-rw-r--r--src/plugins/lua/phishing.lua11
-rw-r--r--src/plugins/lua/ratelimit.lua12
-rw-r--r--src/plugins/lua/settings.lua50
-rw-r--r--src/rspamadm/CMakeLists.txt5
-rw-r--r--src/rspamadm/commands.c250
-rw-r--r--src/rspamadm/configdump.c12
-rw-r--r--src/rspamadm/confighelp.c17
-rw-r--r--src/rspamadm/configtest.c13
-rw-r--r--src/rspamadm/configwizard.c165
-rw-r--r--src/rspamadm/control.c11
-rw-r--r--src/rspamadm/corpus_test.c125
-rw-r--r--src/rspamadm/dkim_keygen.c10
-rw-r--r--src/rspamadm/fuzzy_convert.c11
-rw-r--r--src/rspamadm/fuzzy_merge.c10
-rw-r--r--src/rspamadm/grep.c156
-rw-r--r--src/rspamadm/keypair.c140
-rw-r--r--src/rspamadm/lua_repl.c15
-rw-r--r--src/rspamadm/pw.c10
-rw-r--r--src/rspamadm/rescore.c199
-rw-r--r--src/rspamadm/rspamadm.c193
-rw-r--r--src/rspamadm/rspamadm.h22
-rw-r--r--src/rspamadm/signtool.c14
-rw-r--r--src/rspamadm/stat_convert.c11
-rw-r--r--src/rspamd.c7
-rw-r--r--test/functional/cases/106_url_tags.robot41
-rw-r--r--test/rspamd_test_suite.c2
-rw-r--r--test/rspamd_upstream_test.c2
-rwxr-xr-xutils/rspamd_stats.pl20
609 files changed, 5971 insertions, 2945 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt
index b88da4241..80a804e2e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -502,7 +502,7 @@ INCLUDE_DIRECTORIES("${CMAKE_SOURCE_DIR}/"
"${CMAKE_SOURCE_DIR}/contrib/librdns"
"${CMAKE_SOURCE_DIR}/contrib/aho-corasick"
"${CMAKE_SOURCE_DIR}/contrib/lc-btrie"
- "${CMAKE_SOURCE_DIR}/contrib/lpeg"
+ "${CMAKE_SOURCE_DIR}/contrib/lua-lpeg"
"${CMAKE_BINARY_DIR}/src" #Stored in the binary dir
"${CMAKE_BINARY_DIR}/src/libcryptobox")
@@ -1240,7 +1240,7 @@ ENDIF()
ADD_SUBDIRECTORY(contrib/libucl)
ADD_SUBDIRECTORY(contrib/librdns)
ADD_SUBDIRECTORY(contrib/aho-corasick)
-ADD_SUBDIRECTORY(contrib/lpeg)
+ADD_SUBDIRECTORY(contrib/lua-lpeg)
ADD_SUBDIRECTORY(contrib/linenoise)
ADD_SUBDIRECTORY(contrib/t1ha)
@@ -1261,11 +1261,11 @@ ENDIF()
IF(ENABLE_TORCH MATCHES "ON")
IF(WITH_LUAJIT)
- ADD_SUBDIRECTORY(contrib/torch/paths)
- ADD_SUBDIRECTORY(contrib/torch/torch7)
- ADD_SUBDIRECTORY(contrib/torch/nn)
- ADD_SUBDIRECTORY(contrib/torch/optim)
- ADD_SUBDIRECTORY(contrib/torch/decisiontree)
+ ADD_SUBDIRECTORY(contrib/lua-torch/paths)
+ ADD_SUBDIRECTORY(contrib/lua-torch/torch7)
+ ADD_SUBDIRECTORY(contrib/lua-torch/nn)
+ ADD_SUBDIRECTORY(contrib/lua-torch/optim)
+ ADD_SUBDIRECTORY(contrib/lua-torch/decisiontree)
SET(WITH_TORCH 1)
ELSE()
MESSAGE(FATAL_ERROR "Cannot enable torch without luajit")
@@ -1358,9 +1358,10 @@ ENDFOREACH(LUA_LIB)
# Install lua fun library
INSTALL(FILES "contrib/lua-fun/fun.lua" DESTINATION ${LUALIBDIR})
+INSTALL(FILES "contrib/lua-argparse/argparse.lua" DESTINATION ${LUALIBDIR})
IF(ENABLE_TORCH MATCHES "ON")
- INSTALL(FILES "contrib/moses/moses.lua" DESTINATION ${LUALIBDIR})
+ INSTALL(FILES "contrib/lua-moses/moses.lua" DESTINATION ${LUALIBDIR})
ENDIF()
# systemd unit
diff --git a/clang-plugin/CMakeLists.txt b/clang-plugin/CMakeLists.txt
index eae764235..8b2def17e 100644
--- a/clang-plugin/CMakeLists.txt
+++ b/clang-plugin/CMakeLists.txt
@@ -4,7 +4,7 @@ IF (ENABLE_CLANG_PLUGIN MATCHES "ON")
MESSAGE(FATAL_ERROR "Cannot build clang plugin when compiler is not clang")
endif ()
- SET(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${CMAKE_CURRENT_SOURCE_DIR}")
+ LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}")
ENABLE_LANGUAGE(CXX)
FIND_PACKAGE(LLVM REQUIRED)
diff --git a/conf/composites.conf b/conf/composites.conf
index fb91d8dfd..a07b7020f 100644
--- a/conf/composites.conf
+++ b/conf/composites.conf
@@ -81,6 +81,12 @@ composites {
score = 3.0;
policy = "leave";
}
+ RCVD_UNAUTH_PBL {
+ expression = "RECEIVED_PBL & -RCVD_VIA_SMTP_AUTH";
+ description = "Relayed through ZEN PBL IP without sufficient authentication";
+ score = 2.0;
+ policy = "leave";
+ }
.include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/composites.conf"
.include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/composites.conf"
diff --git a/conf/modules.d/ratelimit.conf b/conf/modules.d/ratelimit.conf
index 08109b820..521bdb1a8 100644
--- a/conf/modules.d/ratelimit.conf
+++ b/conf/modules.d/ratelimit.conf
@@ -30,6 +30,10 @@ ratelimit {
#}
# If symbol is specified, then it is inserted instead of setting result
#symbol = "R_RATELIMIT";
+
+ # If info_symbol is specified, then it is inserted next to set the result
+ #info_symbol = "R_RATELIMIT_INFO";
+
whitelisted_rcpts = "postmaster,mailer-daemon";
max_rcpt = 5;
diff --git a/contrib/lc-btrie/btrie.c b/contrib/lc-btrie/btrie.c
index fe768fc56..b1c66f1c3 100644
--- a/contrib/lc-btrie/btrie.c
+++ b/contrib/lc-btrie/btrie.c
@@ -1485,7 +1485,7 @@ static size_t count_free(const struct btrie *btrie)
#endif /* not NDEBUG */
const char *
-btrie_stats(const struct btrie *btrie)
+btrie_stats(const struct btrie *btrie, guint duplicates)
{
static char buf[128];
size_t n_nodes = btrie->n_lc_nodes + btrie->n_tbm_nodes;
@@ -1530,8 +1530,10 @@ btrie_stats(const struct btrie *btrie)
, average_depth, (long unsigned)stats.max_depth);
#else
snprintf(buf, sizeof(buf),
- "ents=%lu tbm=%lu lc=%lu mem=%.0fk free=%lu waste=%lu"
- ,(long unsigned)btrie->n_entries, (long unsigned)btrie->n_tbm_nodes,
+ "ents=%lu dup=%u tbm=%lu lc=%lu mem=%.0fk free=%lu waste=%lu",
+ (long unsigned)btrie->n_entries,
+ duplicates,
+ (long unsigned)btrie->n_tbm_nodes,
(long unsigned)btrie->n_lc_nodes, (double)btrie->alloc_total / 1024,
(long unsigned)alloc_free, (long unsigned)btrie->alloc_waste
);
diff --git a/contrib/lc-btrie/btrie.h b/contrib/lc-btrie/btrie.h
index 2cc662961..370a03e51 100644
--- a/contrib/lc-btrie/btrie.h
+++ b/contrib/lc-btrie/btrie.h
@@ -70,7 +70,7 @@ enum btrie_result btrie_add_prefix(struct btrie *btrie,
const void *btrie_lookup(const struct btrie *btrie, const btrie_oct_t *pfx,
unsigned len);
-const char *btrie_stats(const struct btrie *btrie);
+const char *btrie_stats(const struct btrie *btrie, guint duplicates);
#ifndef NO_MASTER_DUMP
typedef void btrie_walk_cb_t(const btrie_oct_t *prefix, unsigned len,
diff --git a/contrib/librdns/dns_private.h b/contrib/librdns/dns_private.h
index 37d0240b6..44bb3dd84 100644
--- a/contrib/librdns/dns_private.h
+++ b/contrib/librdns/dns_private.h
@@ -56,6 +56,15 @@ struct rdns_server {
upstream_entry_t up;
};
+enum rdns_request_state {
+ RDNS_REQUEST_NEW = 0,
+ RDNS_REQUEST_REGISTERED = 1,
+ RDNS_REQUEST_WAIT_SEND,
+ RDNS_REQUEST_WAIT_REPLY,
+ RDNS_REQUEST_REPLIED,
+ RDNS_REQUEST_FAKE,
+};
+
struct rdns_request {
struct rdns_resolver *resolver;
struct rdns_async_context *async;
@@ -69,14 +78,7 @@ struct rdns_request {
int id;
struct rdns_request_name *requested_names;
unsigned int qcount;
-
- enum {
- RDNS_REQUEST_NEW = 0,
- RDNS_REQUEST_REGISTERED = 1,
- RDNS_REQUEST_WAIT_SEND,
- RDNS_REQUEST_WAIT_REPLY,
- RDNS_REQUEST_REPLIED
- } state;
+ enum rdns_request_state state;
uint8_t *packet;
off_t pos;
@@ -109,6 +111,13 @@ struct rdns_io_channel {
ref_entry_t ref;
};
+struct rdns_fake_reply {
+ char *request;
+ enum dns_rcode rcode;
+ struct rdns_reply_entry *result;
+ UT_hash_handle hh;
+};
+
struct rdns_resolver {
struct rdns_server *servers;
@@ -117,6 +126,7 @@ struct rdns_resolver {
void *periodic; /** periodic event for resolver */
struct rdns_upstream_context *ups;
struct rdns_plugin *curve_plugin;
+ struct rdns_fake_reply *fake_elts;
rdns_log_function logger;
void *log_data;
diff --git a/contrib/librdns/rdns.h b/contrib/librdns/rdns.h
index 5c44900f1..545d9421f 100644
--- a/contrib/librdns/rdns.h
+++ b/contrib/librdns/rdns.h
@@ -324,6 +324,18 @@ void rdns_resolver_register_plugin (struct rdns_resolver *resolver,
struct rdns_plugin *plugin);
/**
+ * Add a fake reply for a specified name
+ * @param resolver
+ * @param type
+ * @param name
+ * @param reply
+ */
+void rdns_resolver_set_fake_reply (struct rdns_resolver *resolver,
+ const char *name,
+ enum dns_rcode rcode,
+ struct rdns_reply_entry *reply);
+
+/**
* Init DNS resolver
* @param resolver
* @return
diff --git a/contrib/librdns/resolver.c b/contrib/librdns/resolver.c
index 7fa61a391..7afabb1ba 100644
--- a/contrib/librdns/resolver.c
+++ b/contrib/librdns/resolver.c
@@ -484,6 +484,14 @@ rdns_process_retransmit (int fd, void *arg)
req->async_event);
req->async_event = NULL;
+ if (req->state == RDNS_REQUEST_FAKE) {
+ /* Reply is ready */
+ req->func (req->reply, req->arg);
+ REF_RELEASE (req);
+
+ return;
+ }
+
r = rdns_send_request (req, fd, false);
if (r == 0) {
@@ -532,6 +540,7 @@ rdns_make_request_full (
size_t olen;
const char *cur_name, *last_name = NULL;
struct rdns_compression_entry *comp = NULL;
+ struct rdns_fake_reply *fake_rep = NULL;
if (resolver == NULL || !resolver->initialized) {
return NULL;
@@ -569,6 +578,18 @@ rdns_make_request_full (
for (i = 0; i < queries * 2; i += 2) {
cur = i / 2;
cur_name = va_arg (args, const char *);
+
+ if (last_name == NULL) {
+ HASH_FIND_STR (resolver->fake_elts, cur_name, fake_rep);
+
+ if (fake_rep) {
+ /* We actually treat it as a short-circuit */
+ req->reply = rdns_make_reply (req, fake_rep->rcode);
+ req->reply->entries = fake_rep->result;
+ req->state = RDNS_REQUEST_FAKE;
+ }
+ }
+
if (cur_name != NULL) {
last_name = cur_name;
clen = strlen (cur_name);
@@ -585,7 +606,8 @@ rdns_make_request_full (
return NULL;
}
- if (!rdns_format_dns_name (resolver, last_name, clen,
+ if (req->state != RDNS_REQUEST_FAKE &&
+ !rdns_format_dns_name (resolver, last_name, clen,
&req->requested_names[cur].name, &olen)) {
rdns_request_free (req);
return NULL;
@@ -597,37 +619,39 @@ rdns_make_request_full (
}
va_end (args);
- rdns_allocate_packet (req, tlen);
- rdns_make_dns_header (req, queries);
-
- for (i = 0; i < queries; i ++) {
- cur_name = req->requested_names[i].name;
- clen = req->requested_names[i].len;
- type = req->requested_names[i].type;
- if (queries > 1) {
- if (!rdns_add_rr (req, cur_name, clen, type, &comp)) {
- REF_RELEASE (req);
- rnds_compression_free (comp);
- return NULL;
- }
- }
- else {
- if (!rdns_add_rr (req, cur_name, clen, type, NULL)) {
- REF_RELEASE (req);
- rnds_compression_free (comp);
- return NULL;
+ if (req->state != RDNS_REQUEST_FAKE) {
+ rdns_allocate_packet (req, tlen);
+ rdns_make_dns_header (req, queries);
+
+ for (i = 0; i < queries; i++) {
+ cur_name = req->requested_names[i].name;
+ clen = req->requested_names[i].len;
+ type = req->requested_names[i].type;
+ if (queries > 1) {
+ if (!rdns_add_rr (req, cur_name, clen, type, &comp)) {
+ REF_RELEASE (req);
+ rnds_compression_free (comp);
+ return NULL;
+ }
+ } else {
+ if (!rdns_add_rr (req, cur_name, clen, type, NULL)) {
+ REF_RELEASE (req);
+ rnds_compression_free (comp);
+ return NULL;
+ }
}
}
- }
- rnds_compression_free (comp);
+ rnds_compression_free (comp);
- /* Add EDNS RR */
- rdns_add_edns0 (req);
+ /* Add EDNS RR */
+ rdns_add_edns0 (req);
+
+ req->retransmits = repeats;
+ req->timeout = timeout;
+ req->state = RDNS_REQUEST_NEW;
+ }
- req->retransmits = repeats;
- req->timeout = timeout;
- req->state = RDNS_REQUEST_NEW;
req->async = resolver->async;
if (resolver->ups) {
@@ -656,14 +680,21 @@ rdns_make_request_full (
/* Select random IO channel */
req->io = serv->io_channels[ottery_rand_uint32 () % serv->io_cnt];
- req->io->uses ++;
- /* Now send request to server */
- r = rdns_send_request (req, req->io->sock, true);
+ if (req->state == RDNS_REQUEST_FAKE) {
+ req->async_event = resolver->async->add_write (resolver->async->data,
+ req->io->sock, req);
+ }
+ else {
+ req->io->uses++;
- if (r == -1) {
- REF_RELEASE (req);
- return NULL;
+ /* Now send request to server */
+ r = rdns_send_request (req, req->io->sock, true);
+
+ if (r == -1) {
+ REF_RELEASE (req);
+ return NULL;
+ }
}
REF_RETAIN (req->io);
@@ -889,3 +920,39 @@ rdns_resolver_set_dnssec (struct rdns_resolver *resolver, bool enabled)
resolver->enable_dnssec = enabled;
}
}
+
+
+void rdns_resolver_set_fake_reply (struct rdns_resolver *resolver,
+ const char *name,
+ enum dns_rcode rcode,
+ struct rdns_reply_entry *reply)
+{
+ struct rdns_fake_reply *fake_rep;
+
+ HASH_FIND_STR (resolver->fake_elts, name, fake_rep);
+
+ if (fake_rep) {
+ /* Append reply to the existing list */
+ fake_rep->rcode = rcode;
+ DL_APPEND (fake_rep->result, reply);
+ }
+ else {
+ fake_rep = calloc (1, sizeof (*fake_rep));
+
+ if (fake_rep == NULL) {
+ abort ();
+ }
+
+ fake_rep->request = strdup (name);
+
+ if (fake_rep->request == NULL) {
+ abort ();
+ }
+
+ fake_rep->rcode = rcode;
+ DL_APPEND (fake_rep->result, reply);
+
+ HASH_ADD_KEYPTR (hh, resolver->fake_elts, fake_rep->request,
+ strlen (fake_rep->request), fake_rep);
+ }
+} \ No newline at end of file
diff --git a/contrib/librdns/util.c b/contrib/librdns/util.c
index b03128983..a4018cbd3 100644
--- a/contrib/librdns/util.c
+++ b/contrib/librdns/util.c
@@ -353,34 +353,38 @@ rdns_reply_free (struct rdns_reply *rep)
{
struct rdns_reply_entry *entry, *tmp;
- LL_FOREACH_SAFE (rep->entries, entry, tmp) {
- switch (entry->type) {
- case RDNS_REQUEST_PTR:
- free (entry->content.ptr.name);
- break;
- case RDNS_REQUEST_NS:
- free (entry->content.ns.name);
- break;
- case RDNS_REQUEST_MX:
- free (entry->content.mx.name);
- break;
- case RDNS_REQUEST_TXT:
- case RDNS_REQUEST_SPF:
- free (entry->content.txt.data);
- break;
- case RDNS_REQUEST_SRV:
- free (entry->content.srv.target);
- break;
- case RDNS_REQUEST_TLSA:
- free (entry->content.tlsa.data);
- break;
- case RDNS_REQUEST_SOA:
- free (entry->content.soa.mname);
- free (entry->content.soa.admin);
- break;
+ /* We don't need to free data for faked replies */
+ if (!rep->request || rep->request->state != RDNS_REQUEST_FAKE) {
+ LL_FOREACH_SAFE (rep->entries, entry, tmp) {
+ switch (entry->type) {
+ case RDNS_REQUEST_PTR:
+ free (entry->content.ptr.name);
+ break;
+ case RDNS_REQUEST_NS:
+ free (entry->content.ns.name);
+ break;
+ case RDNS_REQUEST_MX:
+ free (entry->content.mx.name);
+ break;
+ case RDNS_REQUEST_TXT:
+ case RDNS_REQUEST_SPF:
+ free (entry->content.txt.data);
+ break;
+ case RDNS_REQUEST_SRV:
+ free (entry->content.srv.target);
+ break;
+ case RDNS_REQUEST_TLSA:
+ free (entry->content.tlsa.data);
+ break;
+ case RDNS_REQUEST_SOA:
+ free (entry->content.soa.mname);
+ free (entry->content.soa.admin);
+ break;
+ }
+ free (entry);
}
- free (entry);
}
+
free (rep);
}
@@ -418,7 +422,11 @@ rdns_request_free (struct rdns_request *req)
HASH_DEL (req->io->requests, req);
req->async_event = NULL;
}
-
+ else if (req->state == RDNS_REQUEST_FAKE) {
+ req->async->del_write (req->async->data,
+ req->async_event);
+ req->async_event = NULL;
+ }
}
#ifdef TWEETNACL
if (req->curve_plugin_data != NULL) {
diff --git a/contrib/lua-argparse/LICENSE b/contrib/lua-argparse/LICENSE
new file mode 100644
index 000000000..87579acb0
--- /dev/null
+++ b/contrib/lua-argparse/LICENSE
@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 - 2018 Peter Melnichenko
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/contrib/lua-argparse/argparse.lua b/contrib/lua-argparse/argparse.lua
new file mode 100644
index 000000000..48950a662
--- /dev/null
+++ b/contrib/lua-argparse/argparse.lua
@@ -0,0 +1,1516 @@
+-- The MIT License (MIT)
+
+-- Copyright (c) 2013 - 2018 Peter Melnichenko
+
+-- Permission is hereby granted, free of charge, to any person obtaining a copy of
+-- this software and associated documentation files (the "Software"), to deal in
+-- the Software without restriction, including without limitation the rights to
+-- use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+-- the Software, and to permit persons to whom the Software is furnished to do so,
+-- subject to the following conditions:
+
+-- The above copyright notice and this permission notice shall be included in all
+-- copies or substantial portions of the Software.
+
+-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+-- FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+-- COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+-- IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+-- CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+local function deep_update(t1, t2)
+ for k, v in pairs(t2) do
+ if type(v) == "table" then
+ v = deep_update({}, v)
+ end
+
+ t1[k] = v
+ end
+
+ return t1
+end
+
+-- A property is a tuple {name, callback}.
+-- properties.args is number of properties that can be set as arguments
+-- when calling an object.
+local function class(prototype, properties, parent)
+ -- Class is the metatable of its instances.
+ local cl = {}
+ cl.__index = cl
+
+ if parent then
+ cl.__prototype = deep_update(deep_update({}, parent.__prototype), prototype)
+ else
+ cl.__prototype = prototype
+ end
+
+ if properties then
+ local names = {}
+
+ -- Create setter methods and fill set of property names.
+ for _, property in ipairs(properties) do
+ local name, callback = property[1], property[2]
+
+ cl[name] = function(self, value)
+ if not callback(self, value) then
+ self["_" .. name] = value
+ end
+
+ return self
+ end
+
+ names[name] = true
+ end
+
+ function cl.__call(self, ...)
+ -- When calling an object, if the first argument is a table,
+ -- interpret keys as property names, else delegate arguments
+ -- to corresponding setters in order.
+ if type((...)) == "table" then
+ for name, value in pairs((...)) do
+ if names[name] then
+ self[name](self, value)
+ end
+ end
+ else
+ local nargs = select("#", ...)
+
+ for i, property in ipairs(properties) do
+ if i > nargs or i > properties.args then
+ break
+ end
+
+ local arg = select(i, ...)
+
+ if arg ~= nil then
+ self[property[1]](self, arg)
+ end
+ end
+ end
+
+ return self
+ end
+ end
+
+ -- If indexing class fails, fallback to its parent.
+ local class_metatable = {}
+ class_metatable.__index = parent
+
+ function class_metatable.__call(self, ...)
+ -- Calling a class returns its instance.
+ -- Arguments are delegated to the instance.
+ local object = deep_update({}, self.__prototype)
+ setmetatable(object, self)
+ return object(...)
+ end
+
+ return setmetatable(cl, class_metatable)
+end
+
+local function typecheck(name, types, value)
+ for _, type_ in ipairs(types) do
+ if type(value) == type_ then
+ return true
+ end
+ end
+
+ error(("bad property '%s' (%s expected, got %s)"):format(name, table.concat(types, " or "), type(value)))
+end
+
+local function typechecked(name, ...)
+ local types = {...}
+ return {name, function(_, value) typecheck(name, types, value) end}
+end
+
+local multiname = {"name", function(self, value)
+ typecheck("name", {"string"}, value)
+
+ for alias in value:gmatch("%S+") do
+ self._name = self._name or alias
+ table.insert(self._aliases, alias)
+ end
+
+ -- Do not set _name as with other properties.
+ return true
+end}
+
+local function parse_boundaries(str)
+ if tonumber(str) then
+ return tonumber(str), tonumber(str)
+ end
+
+ if str == "*" then
+ return 0, math.huge
+ end
+
+ if str == "+" then
+ return 1, math.huge
+ end
+
+ if str == "?" then
+ return 0, 1
+ end
+
+ if str:match "^%d+%-%d+$" then
+ local min, max = str:match "^(%d+)%-(%d+)$"
+ return tonumber(min), tonumber(max)
+ end
+
+ if str:match "^%d+%+$" then
+ local min = str:match "^(%d+)%+$"
+ return tonumber(min), math.huge
+ end
+end
+
+local function boundaries(name)
+ return {name, function(self, value)
+ typecheck(name, {"number", "string"}, value)
+
+ local min, max = parse_boundaries(value)
+
+ if not min then
+ error(("bad property '%s'"):format(name))
+ end
+
+ self["_min" .. name], self["_max" .. name] = min, max
+ end}
+end
+
+local actions = {}
+
+local option_action = {"action", function(_, value)
+ typecheck("action", {"function", "string"}, value)
+
+ if type(value) == "string" and not actions[value] then
+ error(("unknown action '%s'"):format(value))
+ end
+end}
+
+local option_init = {"init", function(self)
+ self._has_init = true
+end}
+
+local option_default = {"default", function(self, value)
+ if type(value) ~= "string" then
+ self._init = value
+ self._has_init = true
+ return true
+ end
+end}
+
+local add_help = {"add_help", function(self, value)
+ typecheck("add_help", {"boolean", "string", "table"}, value)
+
+ if self._has_help then
+ table.remove(self._options)
+ self._has_help = false
+ end
+
+ if value then
+ local help = self:flag()
+ :description "Show this help message and exit."
+ :action(function()
+ print(self:get_help())
+ os.exit(0)
+ end)
+
+ if value ~= true then
+ help = help(value)
+ end
+
+ if not help._name then
+ help "-h" "--help"
+ end
+
+ self._has_help = true
+ end
+end}
+
+local Parser = class({
+ _arguments = {},
+ _options = {},
+ _commands = {},
+ _mutexes = {},
+ _groups = {},
+ _require_command = true,
+ _handle_options = true
+}, {
+ args = 3,
+ typechecked("name", "string"),
+ typechecked("description", "string"),
+ typechecked("epilog", "string"),
+ typechecked("usage", "string"),
+ typechecked("help", "string"),
+ typechecked("require_command", "boolean"),
+ typechecked("handle_options", "boolean"),
+ typechecked("action", "function"),
+ typechecked("command_target", "string"),
+ typechecked("help_vertical_space", "number"),
+ typechecked("usage_margin", "number"),
+ typechecked("usage_max_width", "number"),
+ typechecked("help_usage_margin", "number"),
+ typechecked("help_description_margin", "number"),
+ typechecked("help_max_width", "number"),
+ add_help
+})
+
+local Command = class({
+ _aliases = {}
+}, {
+ args = 3,
+ multiname,
+ typechecked("description", "string"),
+ typechecked("epilog", "string"),
+ typechecked("target", "string"),
+ typechecked("usage", "string"),
+ typechecked("help", "string"),
+ typechecked("require_command", "boolean"),
+ typechecked("handle_options", "boolean"),
+ typechecked("action", "function"),
+ typechecked("command_target", "string"),
+ typechecked("help_vertical_space", "number"),
+ typechecked("usage_margin", "number"),
+ typechecked("usage_max_width", "number"),
+ typechecked("help_usage_margin", "number"),
+ typechecked("help_description_margin", "number"),
+ typechecked("help_max_width", "number"),
+ typechecked("hidden", "boolean"),
+ add_help
+}, Parser)
+
+local Argument = class({
+ _minargs = 1,
+ _maxargs = 1,
+ _mincount = 1,
+ _maxcount = 1,
+ _defmode = "unused",
+ _show_default = true
+}, {
+ args = 5,
+ typechecked("name", "string"),
+ typechecked("description", "string"),
+ option_default,
+ typechecked("convert", "function", "table"),
+ boundaries("args"),
+ typechecked("target", "string"),
+ typechecked("defmode", "string"),
+ typechecked("show_default", "boolean"),
+ typechecked("argname", "string", "table"),
+ typechecked("hidden", "boolean"),
+ option_action,
+ option_init
+})
+
+local Option = class({
+ _aliases = {},
+ _mincount = 0,
+ _overwrite = true
+}, {
+ args = 6,
+ multiname,
+ typechecked("description", "string"),
+ option_default,
+ typechecked("convert", "function", "table"),
+ boundaries("args"),
+ boundaries("count"),
+ typechecked("target", "string"),
+ typechecked("defmode", "string"),
+ typechecked("show_default", "boolean"),
+ typechecked("overwrite", "boolean"),
+ typechecked("argname", "string", "table"),
+ typechecked("hidden", "boolean"),
+ option_action,
+ option_init
+}, Argument)
+
+function Parser:_inherit_property(name, default)
+ local element = self
+
+ while true do
+ local value = element["_" .. name]
+
+ if value ~= nil then
+ return value
+ end
+
+ if not element._parent then
+ return default
+ end
+
+ element = element._parent
+ end
+end
+
+function Argument:_get_argument_list()
+ local buf = {}
+ local i = 1
+
+ while i <= math.min(self._minargs, 3) do
+ local argname = self:_get_argname(i)
+
+ if self._default and self._defmode:find "a" then
+ argname = "[" .. argname .. "]"
+ end
+
+ table.insert(buf, argname)
+ i = i+1
+ end
+
+ while i <= math.min(self._maxargs, 3) do
+ table.insert(buf, "[" .. self:_get_argname(i) .. "]")
+ i = i+1
+
+ if self._maxargs == math.huge then
+ break
+ end
+ end
+
+ if i < self._maxargs then
+ table.insert(buf, "...")
+ end
+
+ return buf
+end
+
+function Argument:_get_usage()
+ local usage = table.concat(self:_get_argument_list(), " ")
+
+ if self._default and self._defmode:find "u" then
+ if self._maxargs > 1 or (self._minargs == 1 and not self._defmode:find "a") then
+ usage = "[" .. usage .. "]"
+ end
+ end
+
+ return usage
+end
+
+function actions.store_true(result, target)
+ result[target] = true
+end
+
+function actions.store_false(result, target)
+ result[target] = false
+end
+
+function actions.store(result, target, argument)
+ result[target] = argument
+end
+
+function actions.count(result, target, _, overwrite)
+ if not overwrite then
+ result[target] = result[target] + 1
+ end
+end
+
+function actions.append(result, target, argument, overwrite)
+ result[target] = result[target] or {}
+ table.insert(result[target], argument)
+
+ if overwrite then
+ table.remove(result[target], 1)
+ end
+end
+
+function actions.concat(result, target, arguments, overwrite)
+ if overwrite then
+ error("'concat' action can't handle too many invocations")
+ end
+
+ result[target] = result[target] or {}
+
+ for _, argument in ipairs(arguments) do
+ table.insert(result[target], argument)
+ end
+end
+
+function Argument:_get_action()
+ local action, init
+
+ if self._maxcount == 1 then
+ if self._maxargs == 0 then
+ action, init = "store_true", nil
+ else
+ action, init = "store", nil
+ end
+ else
+ if self._maxargs == 0 then
+ action, init = "count", 0
+ else
+ action, init = "append", {}
+ end
+ end
+
+ if self._action then
+ action = self._action
+ end
+
+ if self._has_init then
+ init = self._init
+ end
+
+ if type(action) == "string" then
+ action = actions[action]
+ end
+
+ return action, init
+end
+
+-- Returns placeholder for `narg`-th argument.
+function Argument:_get_argname(narg)
+ local argname = self._argname or self:_get_default_argname()
+
+ if type(argname) == "table" then
+ return argname[narg]
+ else
+ return argname
+ end
+end
+
+function Argument:_get_default_argname()
+ return "<" .. self._name .. ">"
+end
+
+function Option:_get_default_argname()
+ return "<" .. self:_get_default_target() .. ">"
+end
+
+-- Returns labels to be shown in the help message.
+function Argument:_get_label_lines()
+ return {self._name}
+end
+
+function Option:_get_label_lines()
+ local argument_list = self:_get_argument_list()
+
+ if #argument_list == 0 then
+ -- Don't put aliases for simple flags like `-h` on different lines.
+ return {table.concat(self._aliases, ", ")}
+ end
+
+ local argument_list_repr = table.concat(argument_list, " ")
+ local lines = {}
+
+ for i, alias in ipairs(self._aliases) do
+ local line = alias .. " " .. argument_list_repr
+ table.insert(lines, line)
+ end
+
+ return {table.concat(lines, ", ")}
+end
+
+function Command:_get_label_lines()
+ return {table.concat(self._aliases, ", ")}
+end
+
+function Argument:_get_description()
+ if self._default and self._show_default then
+ if self._description then
+ return ("%s (default: %s)"):format(self._description, self._default)
+ else
+ return ("default: %s"):format(self._default)
+ end
+ else
+ return self._description or ""
+ end
+end
+
+function Command:_get_description()
+ return self._description or ""
+end
+
+function Option:_get_usage()
+ local usage = self:_get_argument_list()
+ table.insert(usage, 1, self._name)
+ usage = table.concat(usage, " ")
+
+ if self._mincount == 0 or self._default then
+ usage = "[" .. usage .. "]"
+ end
+
+ return usage
+end
+
+function Argument:_get_default_target()
+ return self._name
+end
+
+function Option:_get_default_target()
+ local res
+
+ for _, alias in ipairs(self._aliases) do
+ if alias:sub(1, 1) == alias:sub(2, 2) then
+ res = alias:sub(3)
+ break
+ end
+ end
+
+ res = res or self._name:sub(2)
+ return (res:gsub("-", "_"))
+end
+
+function Option:_is_vararg()
+ return self._maxargs ~= self._minargs
+end
+
+function Parser:_get_fullname()
+ local parent = self._parent
+ local buf = {self._name}
+
+ while parent do
+ table.insert(buf, 1, parent._name)
+ parent = parent._parent
+ end
+
+ return table.concat(buf, " ")
+end
+
+function Parser:_update_charset(charset)
+ charset = charset or {}
+
+ for _, command in ipairs(self._commands) do
+ command:_update_charset(charset)
+ end
+
+ for _, option in ipairs(self._options) do
+ for _, alias in ipairs(option._aliases) do
+ charset[alias:sub(1, 1)] = true
+ end
+ end
+
+ return charset
+end
+
+function Parser:argument(...)
+ local argument = Argument(...)
+ table.insert(self._arguments, argument)
+ return argument
+end
+
+function Parser:option(...)
+ local option = Option(...)
+
+ if self._has_help then
+ table.insert(self._options, #self._options, option)
+ else
+ table.insert(self._options, option)
+ end
+
+ return option
+end
+
+function Parser:flag(...)
+ return self:option():args(0)(...)
+end
+
+function Parser:command(...)
+ local command = Command():add_help(true)(...)
+ command._parent = self
+ table.insert(self._commands, command)
+ return command
+end
+
+function Parser:mutex(...)
+ local elements = {...}
+
+ for i, element in ipairs(elements) do
+ local mt = getmetatable(element)
+ assert(mt == Option or mt == Argument, ("bad argument #%d to 'mutex' (Option or Argument expected)"):format(i))
+ end
+
+ table.insert(self._mutexes, elements)
+ return self
+end
+
+function Parser:group(name, ...)
+ assert(type(name) == "string", ("bad argument #1 to 'group' (string expected, got %s)"):format(type(name)))
+
+ local group = {name = name, ...}
+
+ for i, element in ipairs(group) do
+ local mt = getmetatable(element)
+ assert(mt == Option or mt == Argument or mt == Command,
+ ("bad argument #%d to 'group' (Option or Argument or Command expected)"):format(i + 1))
+ end
+
+ table.insert(self._groups, group)
+ return self
+end
+
+local usage_welcome = "Usage: "
+
+function Parser:get_usage()
+ if self._usage then
+ return self._usage
+ end
+
+ local usage_margin = self:_inherit_property("usage_margin", #usage_welcome)
+ local max_usage_width = self:_inherit_property("usage_max_width", 70)
+ local lines = {usage_welcome .. self:_get_fullname()}
+
+ local function add(s)
+ if #lines[#lines]+1+#s <= max_usage_width then
+ lines[#lines] = lines[#lines] .. " " .. s
+ else
+ lines[#lines+1] = (" "):rep(usage_margin) .. s
+ end
+ end
+
+ -- Normally options are before positional arguments in usage messages.
+ -- However, vararg options should be after, because they can't be reliable used
+ -- before a positional argument.
+ -- Mutexes come into play, too, and are shown as soon as possible.
+ -- Overall, output usages in the following order:
+ -- 1. Mutexes that don't have positional arguments or vararg options.
+ -- 2. Options that are not in any mutexes and are not vararg.
+ -- 3. Positional arguments - on their own or as a part of a mutex.
+ -- 4. Remaining mutexes.
+ -- 5. Remaining options.
+
+ local elements_in_mutexes = {}
+ local added_elements = {}
+ local added_mutexes = {}
+ local argument_to_mutexes = {}
+
+ local function add_mutex(mutex, main_argument)
+ if added_mutexes[mutex] then
+ return
+ end
+
+ added_mutexes[mutex] = true
+ local buf = {}
+
+ for _, element in ipairs(mutex) do
+ if not element._hidden and not added_elements[element] then
+ if getmetatable(element) == Option or element == main_argument then
+ table.insert(buf, element:_get_usage())
+ added_elements[element] = true
+ end
+ end
+ end
+
+ if #buf == 1 then
+ add(buf[1])
+ elseif #buf > 1 then
+ add("(" .. table.concat(buf, " | ") .. ")")
+ end
+ end
+
+ local function add_element(element)
+ if not element._hidden and not added_elements[element] then
+ add(element:_get_usage())
+ added_elements[element] = true
+ end
+ end
+
+ for _, mutex in ipairs(self._mutexes) do
+ local is_vararg = false
+ local has_argument = false
+
+ for _, element in ipairs(mutex) do
+ if getmetatable(element) == Option then
+ if element:_is_vararg() then
+ is_vararg = true
+ end
+ else
+ has_argument = true
+ argument_to_mutexes[element] = argument_to_mutexes[element] or {}
+ table.insert(argument_to_mutexes[element], mutex)
+ end
+
+ elements_in_mutexes[element] = true
+ end
+
+ if not is_vararg and not has_argument then
+ add_mutex(mutex)
+ end
+ end
+
+ for _, option in ipairs(self._options) do
+ if not elements_in_mutexes[option] and not option:_is_vararg() then
+ add_element(option)
+ end
+ end
+
+ -- Add usages for positional arguments, together with one mutex containing them, if they are in a mutex.
+ for _, argument in ipairs(self._arguments) do
+ -- Pick a mutex as a part of which to show this argument, take the first one that's still available.
+ local mutex
+
+ if elements_in_mutexes[argument] then
+ for _, argument_mutex in ipairs(argument_to_mutexes[argument]) do
+ if not added_mutexes[argument_mutex] then
+ mutex = argument_mutex
+ end
+ end
+ end
+
+ if mutex then
+ add_mutex(mutex, argument)
+ else
+ add_element(argument)
+ end
+ end
+
+ for _, mutex in ipairs(self._mutexes) do
+ add_mutex(mutex)
+ end
+
+ for _, option in ipairs(self._options) do
+ add_element(option)
+ end
+
+ if #self._commands > 0 then
+ if self._require_command then
+ add("<command>")
+ else
+ add("[<command>]")
+ end
+
+ add("...")
+ end
+
+ return table.concat(lines, "\n")
+end
+
+local function split_lines(s)
+ if s == "" then
+ return {}
+ end
+
+ local lines = {}
+
+ if s:sub(-1) ~= "\n" then
+ s = s .. "\n"
+ end
+
+ for line in s:gmatch("([^\n]*)\n") do
+ table.insert(lines, line)
+ end
+
+ return lines
+end
+
+local function autowrap_line(line, max_length)
+ -- Algorithm for splitting lines is simple and greedy.
+ local result_lines = {}
+
+ -- Preserve original indentation of the line, put this at the beginning of each result line.
+ -- If the first word looks like a list marker ('*', '+', or '-'), add spaces so that starts
+ -- of the second and the following lines vertically align with the start of the second word.
+ local indentation = line:match("^ *")
+
+ if line:find("^ *[%*%+%-]") then
+ indentation = indentation .. " " .. line:match("^ *[%*%+%-]( *)")
+ end
+
+ -- Parts of the last line being assembled.
+ local line_parts = {}
+
+ -- Length of the current line.
+ local line_length = 0
+
+ -- Index of the next character to consider.
+ local index = 1
+
+ while true do
+ local word_start, word_finish, word = line:find("([^ ]+)", index)
+
+ if not word_start then
+ -- Ignore trailing spaces, if any.
+ break
+ end
+
+ local preceding_spaces = line:sub(index, word_start - 1)
+ index = word_finish + 1
+
+ if (#line_parts == 0) or (line_length + #preceding_spaces + #word <= max_length) then
+ -- Either this is the very first word or it fits as an addition to the current line, add it.
+ table.insert(line_parts, preceding_spaces) -- For the very first word this adds the indentation.
+ table.insert(line_parts, word)
+ line_length = line_length + #preceding_spaces + #word
+ else
+ -- Does not fit, finish current line and put the word into a new one.
+ table.insert(result_lines, table.concat(line_parts))
+ line_parts = {indentation, word}
+ line_length = #indentation + #word
+ end
+ end
+
+ if #line_parts > 0 then
+ table.insert(result_lines, table.concat(line_parts))
+ end
+
+ if #result_lines == 0 then
+ -- Preserve empty lines.
+ result_lines[1] = ""
+ end
+
+ return result_lines
+end
+
+-- Automatically wraps lines within given array,
+-- attempting to limit line length to `max_length`.
+-- Existing line splits are preserved.
+local function autowrap(lines, max_length)
+ local result_lines = {}
+
+ for _, line in ipairs(lines) do
+ local autowrapped_lines = autowrap_line(line, max_length)
+
+ for _, autowrapped_line in ipairs(autowrapped_lines) do
+ table.insert(result_lines, autowrapped_line)
+ end
+ end
+
+ return result_lines
+end
+
+function Parser:_get_element_help(element)
+ local label_lines = element:_get_label_lines()
+ local description_lines = split_lines(element:_get_description())
+
+ local result_lines = {}
+
+ -- All label lines should have the same length (except the last one, it has no comma).
+ -- If too long, start description after all the label lines.
+ -- Otherwise, combine label and description lines.
+
+ local usage_margin_len = self:_inherit_property("help_usage_margin", 3)
+ local usage_margin = (" "):rep(usage_margin_len)
+ local description_margin_len = self:_inherit_property("help_description_margin", 25)
+ local description_margin = (" "):rep(description_margin_len)
+
+ local help_max_width = self:_inherit_property("help_max_width")
+
+ if help_max_width then
+ local description_max_width = math.max(help_max_width - description_margin_len, 10)
+ description_lines = autowrap(description_lines, description_max_width)
+ end
+
+ if #label_lines[1] >= (description_margin_len - usage_margin_len) then
+ for _, label_line in ipairs(label_lines) do
+ table.insert(result_lines, usage_margin .. label_line)
+ end
+
+ for _, description_line in ipairs(description_lines) do
+ table.insert(result_lines, description_margin .. description_line)
+ end
+ else
+ for i = 1, math.max(#label_lines, #description_lines) do
+ local label_line = label_lines[i]
+ local description_line = description_lines[i]
+
+ local line = ""
+
+ if label_line then
+ line = usage_margin .. label_line
+ end
+
+ if description_line and description_line ~= "" then
+ line = line .. (" "):rep(description_margin_len - #line) .. description_line
+ end
+
+ table.insert(result_lines, line)
+ end
+ end
+
+ return table.concat(result_lines, "\n")
+end
+
+local function get_group_types(group)
+ local types = {}
+
+ for _, element in ipairs(group) do
+ types[getmetatable(element)] = true
+ end
+
+ return types
+end
+
+function Parser:_add_group_help(blocks, added_elements, label, elements)
+ local buf = {label}
+
+ for _, element in ipairs(elements) do
+ if not element._hidden and not added_elements[element] then
+ added_elements[element] = true
+ table.insert(buf, self:_get_element_help(element))
+ end
+ end
+
+ if #buf > 1 then
+ table.insert(blocks, table.concat(buf, ("\n"):rep(self:_inherit_property("help_vertical_space", 0) + 1)))
+ end
+end
+
+function Parser:get_help()
+ if self._help then
+ return self._help
+ end
+
+ local blocks = {self:get_usage()}
+
+ local help_max_width = self:_inherit_property("help_max_width")
+
+ if self._description then
+ local description = self._description
+
+ if help_max_width then
+ description = table.concat(autowrap(split_lines(description), help_max_width), "\n")
+ end
+
+ table.insert(blocks, description)
+ end
+
+ -- 1. Put groups containing arguments first, then other arguments.
+ -- 2. Put remaining groups containing options, then other options.
+ -- 3. Put remaining groups containing commands, then other commands.
+ -- Assume that an element can't be in several groups.
+ local groups_by_type = {
+ [Argument] = {},
+ [Option] = {},
+ [Command] = {}
+ }
+
+ for _, group in ipairs(self._groups) do
+ local group_types = get_group_types(group)
+
+ for _, mt in ipairs({Argument, Option, Command}) do
+ if group_types[mt] then
+ table.insert(groups_by_type[mt], group)
+ break
+ end
+ end
+ end
+
+ local default_groups = {
+ {name = "Arguments", type = Argument, elements = self._arguments},
+ {name = "Options", type = Option, elements = self._options},
+ {name = "Commands", type = Command, elements = self._commands}
+ }
+
+ local added_elements = {}
+
+ for _, default_group in ipairs(default_groups) do
+ local type_groups = groups_by_type[default_group.type]
+
+ for _, group in ipairs(type_groups) do
+ self:_add_group_help(blocks, added_elements, group.name .. ":", group)
+ end
+
+ local default_label = default_group.name .. ":"
+
+ if #type_groups > 0 then
+ default_label = "Other " .. default_label:gsub("^.", string.lower)
+ end
+
+ self:_add_group_help(blocks, added_elements, default_label, default_group.elements)
+ end
+
+ if self._epilog then
+ local epilog = self._epilog
+
+ if help_max_width then
+ epilog = table.concat(autowrap(split_lines(epilog), help_max_width), "\n")
+ end
+
+ table.insert(blocks, epilog)
+ end
+
+ return table.concat(blocks, "\n\n")
+end
+
+local function get_tip(context, wrong_name)
+ local context_pool = {}
+ local possible_name
+ local possible_names = {}
+
+ for name in pairs(context) do
+ if type(name) == "string" then
+ for i = 1, #name do
+ possible_name = name:sub(1, i - 1) .. name:sub(i + 1)
+
+ if not context_pool[possible_name] then
+ context_pool[possible_name] = {}
+ end
+
+ table.insert(context_pool[possible_name], name)
+ end
+ end
+ end
+
+ for i = 1, #wrong_name + 1 do
+ possible_name = wrong_name:sub(1, i - 1) .. wrong_name:sub(i + 1)
+
+ if context[possible_name] then
+ possible_names[possible_name] = true
+ elseif context_pool[possible_name] then
+ for _, name in ipairs(context_pool[possible_name]) do
+ possible_names[name] = true
+ end
+ end
+ end
+
+ local first = next(possible_names)
+
+ if first then
+ if next(possible_names, first) then
+ local possible_names_arr = {}
+
+ for name in pairs(possible_names) do
+ table.insert(possible_names_arr, "'" .. name .. "'")
+ end
+
+ table.sort(possible_names_arr)
+ return "\nDid you mean one of these: " .. table.concat(possible_names_arr, " ") .. "?"
+ else
+ return "\nDid you mean '" .. first .. "'?"
+ end
+ else
+ return ""
+ end
+end
+
+local ElementState = class({
+ invocations = 0
+})
+
+function ElementState:__call(state, element)
+ self.state = state
+ self.result = state.result
+ self.element = element
+ self.target = element._target or element:_get_default_target()
+ self.action, self.result[self.target] = element:_get_action()
+ return self
+end
+
+function ElementState:error(fmt, ...)
+ self.state:error(fmt, ...)
+end
+
+function ElementState:convert(argument, index)
+ local converter = self.element._convert
+
+ if converter then
+ local ok, err
+
+ if type(converter) == "function" then
+ ok, err = converter(argument)
+ elseif type(converter[index]) == "function" then
+ ok, err = converter[index](argument)
+ else
+ ok = converter[argument]
+ end
+
+ if ok == nil then
+ self:error(err and "%s" or "malformed argument '%s'", err or argument)
+ end
+
+ argument = ok
+ end
+
+ return argument
+end
+
+function ElementState:default(mode)
+ return self.element._defmode:find(mode) and self.element._default
+end
+
+local function bound(noun, min, max, is_max)
+ local res = ""
+
+ if min ~= max then
+ res = "at " .. (is_max and "most" or "least") .. " "
+ end
+
+ local number = is_max and max or min
+ return res .. tostring(number) .. " " .. noun .. (number == 1 and "" or "s")
+end
+
+function ElementState:set_name(alias)
+ self.name = ("%s '%s'"):format(alias and "option" or "argument", alias or self.element._name)
+end
+
+function ElementState:invoke()
+ self.open = true
+ self.overwrite = false
+
+ if self.invocations >= self.element._maxcount then
+ if self.element._overwrite then
+ self.overwrite = true
+ else
+ local num_times_repr = bound("time", self.element._mincount, self.element._maxcount, true)
+ self:error("%s must be used %s", self.name, num_times_repr)
+ end
+ else
+ self.invocations = self.invocations + 1
+ end
+
+ self.args = {}
+
+ if self.element._maxargs <= 0 then
+ self:close()
+ end
+
+ return self.open
+end
+
+function ElementState:pass(argument)
+ argument = self:convert(argument, #self.args + 1)
+ table.insert(self.args, argument)
+
+ if #self.args >= self.element._maxargs then
+ self:close()
+ end
+
+ return self.open
+end
+
+function ElementState:complete_invocation()
+ while #self.args < self.element._minargs do
+ self:pass(self.element._default)
+ end
+end
+
+function ElementState:close()
+ if self.open then
+ self.open = false
+
+ if #self.args < self.element._minargs then
+ if self:default("a") then
+ self:complete_invocation()
+ else
+ if #self.args == 0 then
+ if getmetatable(self.element) == Argument then
+ self:error("missing %s", self.name)
+ elseif self.element._maxargs == 1 then
+ self:error("%s requires an argument", self.name)
+ end
+ end
+
+ self:error("%s requires %s", self.name, bound("argument", self.element._minargs, self.element._maxargs))
+ end
+ end
+
+ local args
+
+ if self.element._maxargs == 0 then
+ args = self.args[1]
+ elseif self.element._maxargs == 1 then
+ if self.element._minargs == 0 and self.element._mincount ~= self.element._maxcount then
+ args = self.args
+ else
+ args = self.args[1]
+ end
+ else
+ args = self.args
+ end
+
+ self.action(self.result, self.target, args, self.overwrite)
+ end
+end
+
+local ParseState = class({
+ result = {},
+ options = {},
+ arguments = {},
+ argument_i = 1,
+ element_to_mutexes = {},
+ mutex_to_element_state = {},
+ command_actions = {}
+})
+
+function ParseState:__call(parser, error_handler)
+ self.parser = parser
+ self.error_handler = error_handler
+ self.charset = parser:_update_charset()
+ self:switch(parser)
+ return self
+end
+
+function ParseState:error(fmt, ...)
+ self.error_handler(self.parser, fmt:format(...))
+end
+
+function ParseState:switch(parser)
+ self.parser = parser
+
+ if parser._action then
+ table.insert(self.command_actions, {action = parser._action, name = parser._name})
+ end
+
+ for _, option in ipairs(parser._options) do
+ option = ElementState(self, option)
+ table.insert(self.options, option)
+
+ for _, alias in ipairs(option.element._aliases) do
+ self.options[alias] = option
+ end
+ end
+
+ for _, mutex in ipairs(parser._mutexes) do
+ for _, element in ipairs(mutex) do
+ if not self.element_to_mutexes[element] then
+ self.element_to_mutexes[element] = {}
+ end
+
+ table.insert(self.element_to_mutexes[element], mutex)
+ end
+ end
+
+ for _, argument in ipairs(parser._arguments) do
+ argument = ElementState(self, argument)
+ table.insert(self.arguments, argument)
+ argument:set_name()
+ argument:invoke()
+ end
+
+ self.handle_options = parser._handle_options
+ self.argument = self.arguments[self.argument_i]
+ self.commands = parser._commands
+
+ for _, command in ipairs(self.commands) do
+ for _, alias in ipairs(command._aliases) do
+ self.commands[alias] = command
+ end
+ end
+end
+
+function ParseState:get_option(name)
+ local option = self.options[name]
+
+ if not option then
+ self:error("unknown option '%s'%s", name, get_tip(self.options, name))
+ else
+ return option
+ end
+end
+
+function ParseState:get_command(name)
+ local command = self.commands[name]
+
+ if not command then
+ if #self.commands > 0 then
+ self:error("unknown command '%s'%s", name, get_tip(self.commands, name))
+ else
+ self:error("too many arguments")
+ end
+ else
+ return command
+ end
+end
+
+function ParseState:check_mutexes(element_state)
+ if self.element_to_mutexes[element_state.element] then
+ for _, mutex in ipairs(self.element_to_mutexes[element_state.element]) do
+ local used_element_state = self.mutex_to_element_state[mutex]
+
+ if used_element_state and used_element_state ~= element_state then
+ self:error("%s can not be used together with %s", element_state.name, used_element_state.name)
+ else
+ self.mutex_to_element_state[mutex] = element_state
+ end
+ end
+ end
+end
+
+function ParseState:invoke(option, name)
+ self:close()
+ option:set_name(name)
+ self:check_mutexes(option, name)
+
+ if option:invoke() then
+ self.option = option
+ end
+end
+
+function ParseState:pass(arg)
+ if self.option then
+ if not self.option:pass(arg) then
+ self.option = nil
+ end
+ elseif self.argument then
+ self:check_mutexes(self.argument)
+
+ if not self.argument:pass(arg) then
+ self.argument_i = self.argument_i + 1
+ self.argument = self.arguments[self.argument_i]
+ end
+ else
+ local command = self:get_command(arg)
+ self.result[command._target or command._name] = true
+
+ if self.parser._command_target then
+ self.result[self.parser._command_target] = command._name
+ end
+
+ self:switch(command)
+ end
+end
+
+function ParseState:close()
+ if self.option then
+ self.option:close()
+ self.option = nil
+ end
+end
+
+function ParseState:finalize()
+ self:close()
+
+ for i = self.argument_i, #self.arguments do
+ local argument = self.arguments[i]
+ if #argument.args == 0 and argument:default("u") then
+ argument:complete_invocation()
+ else
+ argument:close()
+ end
+ end
+
+ if self.parser._require_command and #self.commands > 0 then
+ self:error("a command is required")
+ end
+
+ for _, option in ipairs(self.options) do
+ option.name = option.name or ("option '%s'"):format(option.element._name)
+
+ if option.invocations == 0 then
+ if option:default("u") then
+ option:invoke()
+ option:complete_invocation()
+ option:close()
+ end
+ end
+
+ local mincount = option.element._mincount
+
+ if option.invocations < mincount then
+ if option:default("a") then
+ while option.invocations < mincount do
+ option:invoke()
+ option:close()
+ end
+ elseif option.invocations == 0 then
+ self:error("missing %s", option.name)
+ else
+ self:error("%s must be used %s", option.name, bound("time", mincount, option.element._maxcount))
+ end
+ end
+ end
+
+ for i = #self.command_actions, 1, -1 do
+ self.command_actions[i].action(self.result, self.command_actions[i].name)
+ end
+end
+
+function ParseState:parse(args)
+ for _, arg in ipairs(args) do
+ local plain = true
+
+ if self.handle_options then
+ local first = arg:sub(1, 1)
+
+ if self.charset[first] then
+ if #arg > 1 then
+ plain = false
+
+ if arg:sub(2, 2) == first then
+ if #arg == 2 then
+ if self.options[arg] then
+ local option = self:get_option(arg)
+ self:invoke(option, arg)
+ else
+ self:close()
+ end
+
+ self.handle_options = false
+ else
+ local equals = arg:find "="
+ if equals then
+ local name = arg:sub(1, equals - 1)
+ local option = self:get_option(name)
+
+ if option.element._maxargs <= 0 then
+ self:error("option '%s' does not take arguments", name)
+ end
+
+ self:invoke(option, name)
+ self:pass(arg:sub(equals + 1))
+ else
+ local option = self:get_option(arg)
+ self:invoke(option, arg)
+ end
+ end
+ else
+ for i = 2, #arg do
+ local name = first .. arg:sub(i, i)
+ local option = self:get_option(name)
+ self:invoke(option, name)
+
+ if i ~= #arg and option.element._maxargs > 0 then
+ self:pass(arg:sub(i + 1))
+ break
+ end
+ end
+ end
+ end
+ end
+ end
+
+ if plain then
+ self:pass(arg)
+ end
+ end
+
+ self:finalize()
+ return self.result
+end
+
+function Parser:error(msg)
+ io.stderr:write(("%s\n\nError: %s\n"):format(self:get_usage(), msg))
+ os.exit(1)
+end
+
+-- Compatibility with strict.lua and other checkers:
+local default_cmdline = rawget(_G, "arg") or {}
+
+function Parser:_parse(args, error_handler)
+ return ParseState(self, error_handler):parse(args or default_cmdline)
+end
+
+function Parser:parse(args)
+ return self:_parse(args, self.error)
+end
+
+local function xpcall_error_handler(err)
+ return tostring(err) .. "\noriginal " .. debug.traceback("", 2):sub(2)
+end
+
+function Parser:pparse(args)
+ local parse_error
+
+ local ok, result = xpcall(function()
+ return self:_parse(args, function(_, err)
+ parse_error = err
+ error(err, 0)
+ end)
+ end, xpcall_error_handler)
+
+ if ok then
+ return true, result
+ elseif not parse_error then
+ error(result, 0)
+ else
+ return false, parse_error
+ end
+end
+
+local argparse = {}
+
+argparse.version = "0.6.0"
+
+setmetatable(argparse, {__call = function(_, ...)
+ return Parser(default_cmdline[0]):add_help(true)(...)
+end})
+
+return argparse
diff --git a/contrib/lpeg/CMakeLists.txt b/contrib/lua-lpeg/CMakeLists.txt
index 2362aac9c..2362aac9c 100644
--- a/contrib/lpeg/CMakeLists.txt
+++ b/contrib/lua-lpeg/CMakeLists.txt
diff --git a/contrib/lpeg/LICENSE b/contrib/lua-lpeg/LICENSE
index cea2d8b5e..cea2d8b5e 100644
--- a/contrib/lpeg/LICENSE
+++ b/contrib/lua-lpeg/LICENSE
diff --git a/contrib/lpeg/lpcap.c b/contrib/lua-lpeg/lpcap.c
index c9085de06..c9085de06 100644
--- a/contrib/lpeg/lpcap.c
+++ b/contrib/lua-lpeg/lpcap.c
diff --git a/contrib/lpeg/lpcap.h b/contrib/lua-lpeg/lpcap.h
index d762fdcfa..d762fdcfa 100644
--- a/contrib/lpeg/lpcap.h
+++ b/contrib/lua-lpeg/lpcap.h
diff --git a/contrib/lpeg/lpcode.c b/contrib/lua-lpeg/lpcode.c
index 6feefeb43..6feefeb43 100644
--- a/contrib/lpeg/lpcode.c
+++ b/contrib/lua-lpeg/lpcode.c
diff --git a/contrib/lpeg/lpcode.h b/contrib/lua-lpeg/lpcode.h
index 896d3c79a..896d3c79a 100644
--- a/contrib/lpeg/lpcode.h
+++ b/contrib/lua-lpeg/lpcode.h
diff --git a/contrib/lpeg/lpprint.c b/contrib/lua-lpeg/lpprint.c
index 174d1687b..174d1687b 100644
--- a/contrib/lpeg/lpprint.c
+++ b/contrib/lua-lpeg/lpprint.c
diff --git a/contrib/lpeg/lpprint.h b/contrib/lua-lpeg/lpprint.h
index 632976076..632976076 100644
--- a/contrib/lpeg/lpprint.h
+++ b/contrib/lua-lpeg/lpprint.h
diff --git a/contrib/lpeg/lptree.c b/contrib/lua-lpeg/lptree.c
index f1016c3db..f1016c3db 100644
--- a/contrib/lpeg/lptree.c
+++ b/contrib/lua-lpeg/lptree.c
diff --git a/contrib/lpeg/lptree.h b/contrib/lua-lpeg/lptree.h
index 38a668e9f..38a668e9f 100644
--- a/contrib/lpeg/lptree.h
+++ b/contrib/lua-lpeg/lptree.h
diff --git a/contrib/lpeg/lptypes.h b/contrib/lua-lpeg/lptypes.h
index 78dff6220..78dff6220 100644
--- a/contrib/lpeg/lptypes.h
+++ b/contrib/lua-lpeg/lptypes.h
diff --git a/contrib/lpeg/lpvm.c b/contrib/lua-lpeg/lpvm.c
index eaf2ebfd7..eaf2ebfd7 100644
--- a/contrib/lpeg/lpvm.c
+++ b/contrib/lua-lpeg/lpvm.c
diff --git a/contrib/lpeg/lpvm.h b/contrib/lua-lpeg/lpvm.h
index 757b9e135..757b9e135 100644
--- a/contrib/lpeg/lpvm.h
+++ b/contrib/lua-lpeg/lpvm.h
diff --git a/contrib/moses/LICENSE b/contrib/lua-moses/LICENSE
index f06dce3e8..f06dce3e8 100644
--- a/contrib/moses/LICENSE
+++ b/contrib/lua-moses/LICENSE
diff --git a/contrib/moses/moses.lua b/contrib/lua-moses/moses.lua
index bb67dccad..bb67dccad 100644
--- a/contrib/moses/moses.lua
+++ b/contrib/lua-moses/moses.lua
diff --git a/contrib/torch/decisiontree/CMakeLists.txt b/contrib/lua-torch/decisiontree/CMakeLists.txt
index 6434eddd9..b94b4deb2 100644
--- a/contrib/torch/decisiontree/CMakeLists.txt
+++ b/contrib/lua-torch/decisiontree/CMakeLists.txt
@@ -1,7 +1,4 @@
-SET(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR})
-
-CMAKE_MINIMUM_REQUIRED(VERSION 2.6 FATAL_ERROR)
-CMAKE_POLICY(VERSION 2.6)
+LIST(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}")
SET(src
init.c
diff --git a/contrib/torch/decisiontree/CartNode.lua b/contrib/lua-torch/decisiontree/CartNode.lua
index e6a85006c..e6a85006c 100644
--- a/contrib/torch/decisiontree/CartNode.lua
+++ b/contrib/lua-torch/decisiontree/CartNode.lua
diff --git a/contrib/torch/decisiontree/CartTrainer.lua b/contrib/lua-torch/decisiontree/CartTrainer.lua
index 63ae6c148..63ae6c148 100644
--- a/contrib/torch/decisiontree/CartTrainer.lua
+++ b/contrib/lua-torch/decisiontree/CartTrainer.lua
diff --git a/contrib/torch/decisiontree/CartTree.lua b/contrib/lua-torch/decisiontree/CartTree.lua
index c74dfda9e..c74dfda9e 100644
--- a/contrib/torch/decisiontree/CartTree.lua
+++ b/contrib/lua-torch/decisiontree/CartTree.lua
diff --git a/contrib/torch/decisiontree/DFD.lua b/contrib/lua-torch/decisiontree/DFD.lua
index e4746212a..e4746212a 100644
--- a/contrib/torch/decisiontree/DFD.lua
+++ b/contrib/lua-torch/decisiontree/DFD.lua
diff --git a/contrib/torch/decisiontree/DataSet.lua b/contrib/lua-torch/decisiontree/DataSet.lua
index 15058a7c6..15058a7c6 100644
--- a/contrib/torch/decisiontree/DataSet.lua
+++ b/contrib/lua-torch/decisiontree/DataSet.lua
diff --git a/contrib/torch/decisiontree/DecisionForest.lua b/contrib/lua-torch/decisiontree/DecisionForest.lua
index cac748e7e..cac748e7e 100644
--- a/contrib/torch/decisiontree/DecisionForest.lua
+++ b/contrib/lua-torch/decisiontree/DecisionForest.lua
diff --git a/contrib/torch/decisiontree/DecisionForestTrainer.lua b/contrib/lua-torch/decisiontree/DecisionForestTrainer.lua
index fc903678b..fc903678b 100644
--- a/contrib/torch/decisiontree/DecisionForestTrainer.lua
+++ b/contrib/lua-torch/decisiontree/DecisionForestTrainer.lua
diff --git a/contrib/torch/decisiontree/DecisionTree.lua b/contrib/lua-torch/decisiontree/DecisionTree.lua
index c61bc3757..c61bc3757 100644
--- a/contrib/torch/decisiontree/DecisionTree.lua
+++ b/contrib/lua-torch/decisiontree/DecisionTree.lua
diff --git a/contrib/torch/decisiontree/GBDT_common.h b/contrib/lua-torch/decisiontree/GBDT_common.h
index eb993702d..eb993702d 100644
--- a/contrib/torch/decisiontree/GBDT_common.h
+++ b/contrib/lua-torch/decisiontree/GBDT_common.h
diff --git a/contrib/torch/decisiontree/GiniState.lua b/contrib/lua-torch/decisiontree/GiniState.lua
index 6dfed2845..6dfed2845 100644
--- a/contrib/torch/decisiontree/GiniState.lua
+++ b/contrib/lua-torch/decisiontree/GiniState.lua
diff --git a/contrib/torch/decisiontree/GradientBoostState.lua b/contrib/lua-torch/decisiontree/GradientBoostState.lua
index f268f3da8..f268f3da8 100644
--- a/contrib/torch/decisiontree/GradientBoostState.lua
+++ b/contrib/lua-torch/decisiontree/GradientBoostState.lua
diff --git a/contrib/torch/decisiontree/GradientBoostTrainer.lua b/contrib/lua-torch/decisiontree/GradientBoostTrainer.lua
index 51299b109..51299b109 100644
--- a/contrib/torch/decisiontree/GradientBoostTrainer.lua
+++ b/contrib/lua-torch/decisiontree/GradientBoostTrainer.lua
diff --git a/contrib/torch/decisiontree/LICENSE b/contrib/lua-torch/decisiontree/LICENSE
index 8dada3eda..8dada3eda 100644
--- a/contrib/torch/decisiontree/LICENSE
+++ b/contrib/lua-torch/decisiontree/LICENSE
diff --git a/contrib/torch/decisiontree/LogitBoostCriterion.lua b/contrib/lua-torch/decisiontree/LogitBoostCriterion.lua
index 5b9eb6028..5b9eb6028 100644
--- a/contrib/torch/decisiontree/LogitBoostCriterion.lua
+++ b/contrib/lua-torch/decisiontree/LogitBoostCriterion.lua
diff --git a/contrib/torch/decisiontree/MSECriterion.lua b/contrib/lua-torch/decisiontree/MSECriterion.lua
index 948c1a17e..948c1a17e 100644
--- a/contrib/torch/decisiontree/MSECriterion.lua
+++ b/contrib/lua-torch/decisiontree/MSECriterion.lua
diff --git a/contrib/torch/decisiontree/README.md b/contrib/lua-torch/decisiontree/README.md
index db4622add..db4622add 100644
--- a/contrib/torch/decisiontree/README.md
+++ b/contrib/lua-torch/decisiontree/README.md
diff --git a/contrib/torch/decisiontree/RandomForestTrainer.lua b/contrib/lua-torch/decisiontree/RandomForestTrainer.lua
index 41040b25b..41040b25b 100644
--- a/contrib/torch/decisiontree/RandomForestTrainer.lua
+++ b/contrib/lua-torch/decisiontree/RandomForestTrainer.lua
diff --git a/contrib/torch/decisiontree/Sparse2Dense.lua b/contrib/lua-torch/decisiontree/Sparse2Dense.lua
index 4e5b79d2f..4e5b79d2f 100644
--- a/contrib/torch/decisiontree/Sparse2Dense.lua
+++ b/contrib/lua-torch/decisiontree/Sparse2Dense.lua
diff --git a/contrib/torch/decisiontree/SparseTensor.lua b/contrib/lua-torch/decisiontree/SparseTensor.lua
index 4c620e618..4c620e618 100644
--- a/contrib/torch/decisiontree/SparseTensor.lua
+++ b/contrib/lua-torch/decisiontree/SparseTensor.lua
diff --git a/contrib/torch/decisiontree/TreeState.lua b/contrib/lua-torch/decisiontree/TreeState.lua
index 3928649fd..3928649fd 100644
--- a/contrib/torch/decisiontree/TreeState.lua
+++ b/contrib/lua-torch/decisiontree/TreeState.lua
diff --git a/contrib/torch/decisiontree/WorkPool.lua b/contrib/lua-torch/decisiontree/WorkPool.lua
index 8f473727e..8f473727e 100644
--- a/contrib/torch/decisiontree/WorkPool.lua
+++ b/contrib/lua-torch/decisiontree/WorkPool.lua
diff --git a/contrib/torch/decisiontree/_env.lua b/contrib/lua-torch/decisiontree/_env.lua
index a92770152..a92770152 100644
--- a/contrib/torch/decisiontree/_env.lua
+++ b/contrib/lua-torch/decisiontree/_env.lua
diff --git a/contrib/torch/decisiontree/benchmark.lua b/contrib/lua-torch/decisiontree/benchmark.lua
index 2b6a03dc6..2b6a03dc6 100644
--- a/contrib/torch/decisiontree/benchmark.lua
+++ b/contrib/lua-torch/decisiontree/benchmark.lua
diff --git a/contrib/torch/decisiontree/doc/benchmark.md b/contrib/lua-torch/decisiontree/doc/benchmark.md
index cb8f905d6..cb8f905d6 100644
--- a/contrib/torch/decisiontree/doc/benchmark.md
+++ b/contrib/lua-torch/decisiontree/doc/benchmark.md
diff --git a/contrib/torch/decisiontree/error.h b/contrib/lua-torch/decisiontree/error.h
index 18df3c939..18df3c939 100644
--- a/contrib/torch/decisiontree/error.h
+++ b/contrib/lua-torch/decisiontree/error.h
diff --git a/contrib/torch/decisiontree/generic/CartTree.c b/contrib/lua-torch/decisiontree/generic/CartTree.c
index eb29fcf02..eb29fcf02 100644
--- a/contrib/torch/decisiontree/generic/CartTree.c
+++ b/contrib/lua-torch/decisiontree/generic/CartTree.c
diff --git a/contrib/torch/decisiontree/generic/DFD.c b/contrib/lua-torch/decisiontree/generic/DFD.c
index 599c4d794..599c4d794 100644
--- a/contrib/torch/decisiontree/generic/DFD.c
+++ b/contrib/lua-torch/decisiontree/generic/DFD.c
diff --git a/contrib/torch/decisiontree/generic/GBDT.c b/contrib/lua-torch/decisiontree/generic/GBDT.c
index 31f5b025d..31f5b025d 100644
--- a/contrib/torch/decisiontree/generic/GBDT.c
+++ b/contrib/lua-torch/decisiontree/generic/GBDT.c
diff --git a/contrib/torch/decisiontree/generic/GBDT_internal.c b/contrib/lua-torch/decisiontree/generic/GBDT_internal.c
index 739aabf25..739aabf25 100644
--- a/contrib/torch/decisiontree/generic/GBDT_internal.c
+++ b/contrib/lua-torch/decisiontree/generic/GBDT_internal.c
diff --git a/contrib/torch/decisiontree/generic/GBDT_internal.h b/contrib/lua-torch/decisiontree/generic/GBDT_internal.h
index 7119365cf..7119365cf 100644
--- a/contrib/torch/decisiontree/generic/GBDT_internal.h
+++ b/contrib/lua-torch/decisiontree/generic/GBDT_internal.h
diff --git a/contrib/torch/decisiontree/generic/LogitBoostCriterion.c b/contrib/lua-torch/decisiontree/generic/LogitBoostCriterion.c
index f2ea1ef09..f2ea1ef09 100644
--- a/contrib/torch/decisiontree/generic/LogitBoostCriterion.c
+++ b/contrib/lua-torch/decisiontree/generic/LogitBoostCriterion.c
diff --git a/contrib/torch/decisiontree/generic/S2D.c b/contrib/lua-torch/decisiontree/generic/S2D.c
index 2392ee7c8..2392ee7c8 100644
--- a/contrib/torch/decisiontree/generic/S2D.c
+++ b/contrib/lua-torch/decisiontree/generic/S2D.c
diff --git a/contrib/torch/decisiontree/hash_map.c b/contrib/lua-torch/decisiontree/hash_map.c
index 2c679e207..2c679e207 100644
--- a/contrib/torch/decisiontree/hash_map.c
+++ b/contrib/lua-torch/decisiontree/hash_map.c
diff --git a/contrib/torch/decisiontree/hash_map.h b/contrib/lua-torch/decisiontree/hash_map.h
index 5b215e4ca..5b215e4ca 100644
--- a/contrib/torch/decisiontree/hash_map.h
+++ b/contrib/lua-torch/decisiontree/hash_map.h
diff --git a/contrib/torch/decisiontree/init.c b/contrib/lua-torch/decisiontree/init.c
index 276241e8e..276241e8e 100644
--- a/contrib/torch/decisiontree/init.c
+++ b/contrib/lua-torch/decisiontree/init.c
diff --git a/contrib/torch/decisiontree/init.lua b/contrib/lua-torch/decisiontree/init.lua
index 26f790b60..26f790b60 100644
--- a/contrib/torch/decisiontree/init.lua
+++ b/contrib/lua-torch/decisiontree/init.lua
diff --git a/contrib/torch/decisiontree/internal_hash_map.h b/contrib/lua-torch/decisiontree/internal_hash_map.h
index bc8c523ef..bc8c523ef 100644
--- a/contrib/torch/decisiontree/internal_hash_map.h
+++ b/contrib/lua-torch/decisiontree/internal_hash_map.h
diff --git a/contrib/torch/decisiontree/khash.h b/contrib/lua-torch/decisiontree/khash.h
index 20e994063..20e994063 100644
--- a/contrib/torch/decisiontree/khash.h
+++ b/contrib/lua-torch/decisiontree/khash.h
diff --git a/contrib/torch/decisiontree/math.lua b/contrib/lua-torch/decisiontree/math.lua
index eb71b31ed..eb71b31ed 100644
--- a/contrib/torch/decisiontree/math.lua
+++ b/contrib/lua-torch/decisiontree/math.lua
diff --git a/contrib/torch/decisiontree/rocks/decisiontree-scm-1.rockspec b/contrib/lua-torch/decisiontree/rocks/decisiontree-scm-1.rockspec
index d5d916275..d5d916275 100644
--- a/contrib/torch/decisiontree/rocks/decisiontree-scm-1.rockspec
+++ b/contrib/lua-torch/decisiontree/rocks/decisiontree-scm-1.rockspec
diff --git a/contrib/torch/decisiontree/test.lua b/contrib/lua-torch/decisiontree/test.lua
index 80510a4f6..80510a4f6 100644
--- a/contrib/torch/decisiontree/test.lua
+++ b/contrib/lua-torch/decisiontree/test.lua
diff --git a/contrib/torch/decisiontree/utils.h b/contrib/lua-torch/decisiontree/utils.h
index 8a0196a58..8a0196a58 100644
--- a/contrib/torch/decisiontree/utils.h
+++ b/contrib/lua-torch/decisiontree/utils.h
diff --git a/contrib/torch/decisiontree/utils.lua b/contrib/lua-torch/decisiontree/utils.lua
index c32c3d08b..c32c3d08b 100644
--- a/contrib/torch/decisiontree/utils.lua
+++ b/contrib/lua-torch/decisiontree/utils.lua
diff --git a/contrib/torch/nn/.gitignore b/contrib/lua-torch/nn/.gitignore
index e0fa91eda..e0fa91eda 100644
--- a/contrib/torch/nn/.gitignore
+++ b/contrib/lua-torch/nn/.gitignore
diff --git a/contrib/torch/nn/.luacheckrc b/contrib/lua-torch/nn/.luacheckrc
index 3d358e9c0..3d358e9c0 100644
--- a/contrib/torch/nn/.luacheckrc
+++ b/contrib/lua-torch/nn/.luacheckrc
diff --git a/contrib/torch/nn/.travis.yml b/contrib/lua-torch/nn/.travis.yml
index 1d10e0fb5..1d10e0fb5 100644
--- a/contrib/torch/nn/.travis.yml
+++ b/contrib/lua-torch/nn/.travis.yml
diff --git a/contrib/torch/nn/Abs.lua b/contrib/lua-torch/nn/Abs.lua
index b32b64f79..b32b64f79 100644
--- a/contrib/torch/nn/Abs.lua
+++ b/contrib/lua-torch/nn/Abs.lua
diff --git a/contrib/torch/nn/AbsCriterion.lua b/contrib/lua-torch/nn/AbsCriterion.lua
index 65e2f8ae1..65e2f8ae1 100644
--- a/contrib/torch/nn/AbsCriterion.lua
+++ b/contrib/lua-torch/nn/AbsCriterion.lua
diff --git a/contrib/torch/nn/Add.lua b/contrib/lua-torch/nn/Add.lua
index d071a15b3..d071a15b3 100644
--- a/contrib/torch/nn/Add.lua
+++ b/contrib/lua-torch/nn/Add.lua
diff --git a/contrib/torch/nn/AddConstant.lua b/contrib/lua-torch/nn/AddConstant.lua
index b686d719c..b686d719c 100644
--- a/contrib/torch/nn/AddConstant.lua
+++ b/contrib/lua-torch/nn/AddConstant.lua
diff --git a/contrib/torch/nn/BCECriterion.lua b/contrib/lua-torch/nn/BCECriterion.lua
index 8bb5f8178..8bb5f8178 100644
--- a/contrib/torch/nn/BCECriterion.lua
+++ b/contrib/lua-torch/nn/BCECriterion.lua
diff --git a/contrib/torch/nn/BatchNormalization.lua b/contrib/lua-torch/nn/BatchNormalization.lua
index 8dfc576b3..8dfc576b3 100644
--- a/contrib/torch/nn/BatchNormalization.lua
+++ b/contrib/lua-torch/nn/BatchNormalization.lua
diff --git a/contrib/torch/nn/Bilinear.lua b/contrib/lua-torch/nn/Bilinear.lua
index 9350b03ec..9350b03ec 100644
--- a/contrib/torch/nn/Bilinear.lua
+++ b/contrib/lua-torch/nn/Bilinear.lua
diff --git a/contrib/torch/nn/Bottle.lua b/contrib/lua-torch/nn/Bottle.lua
index 6dee432f5..6dee432f5 100644
--- a/contrib/torch/nn/Bottle.lua
+++ b/contrib/lua-torch/nn/Bottle.lua
diff --git a/contrib/torch/nn/CAdd.lua b/contrib/lua-torch/nn/CAdd.lua
index 1d7b45726..1d7b45726 100644
--- a/contrib/torch/nn/CAdd.lua
+++ b/contrib/lua-torch/nn/CAdd.lua
diff --git a/contrib/torch/nn/CAddTable.lua b/contrib/lua-torch/nn/CAddTable.lua
index 79deb7e9b..79deb7e9b 100644
--- a/contrib/torch/nn/CAddTable.lua
+++ b/contrib/lua-torch/nn/CAddTable.lua
diff --git a/contrib/torch/nn/CAddTensorTable.lua b/contrib/lua-torch/nn/CAddTensorTable.lua
index 16efe4450..16efe4450 100644
--- a/contrib/torch/nn/CAddTensorTable.lua
+++ b/contrib/lua-torch/nn/CAddTensorTable.lua
diff --git a/contrib/torch/nn/CDivTable.lua b/contrib/lua-torch/nn/CDivTable.lua
index bf044c9af..bf044c9af 100644
--- a/contrib/torch/nn/CDivTable.lua
+++ b/contrib/lua-torch/nn/CDivTable.lua
diff --git a/contrib/torch/nn/CMakeLists.txt b/contrib/lua-torch/nn/CMakeLists.txt
index cebddfbfc..cebddfbfc 100644
--- a/contrib/torch/nn/CMakeLists.txt
+++ b/contrib/lua-torch/nn/CMakeLists.txt
diff --git a/contrib/torch/nn/CMaxTable.lua b/contrib/lua-torch/nn/CMaxTable.lua
index 845e38d23..845e38d23 100644
--- a/contrib/torch/nn/CMaxTable.lua
+++ b/contrib/lua-torch/nn/CMaxTable.lua
diff --git a/contrib/torch/nn/CMinTable.lua b/contrib/lua-torch/nn/CMinTable.lua
index 25b9a19a2..25b9a19a2 100644
--- a/contrib/torch/nn/CMinTable.lua
+++ b/contrib/lua-torch/nn/CMinTable.lua
diff --git a/contrib/torch/nn/CMul.lua b/contrib/lua-torch/nn/CMul.lua
index 890169761..890169761 100644
--- a/contrib/torch/nn/CMul.lua
+++ b/contrib/lua-torch/nn/CMul.lua
diff --git a/contrib/torch/nn/CMulTable.lua b/contrib/lua-torch/nn/CMulTable.lua
index b47378e83..b47378e83 100644
--- a/contrib/torch/nn/CMulTable.lua
+++ b/contrib/lua-torch/nn/CMulTable.lua
diff --git a/contrib/torch/nn/CONTRIBUTING.md b/contrib/lua-torch/nn/CONTRIBUTING.md
index cc800154e..cc800154e 100644
--- a/contrib/torch/nn/CONTRIBUTING.md
+++ b/contrib/lua-torch/nn/CONTRIBUTING.md
diff --git a/contrib/torch/nn/COPYRIGHT.txt b/contrib/lua-torch/nn/COPYRIGHT.txt
index bc002b78a..bc002b78a 100644
--- a/contrib/torch/nn/COPYRIGHT.txt
+++ b/contrib/lua-torch/nn/COPYRIGHT.txt
diff --git a/contrib/torch/nn/CReLU.lua b/contrib/lua-torch/nn/CReLU.lua
index 8da6e7974..8da6e7974 100644
--- a/contrib/torch/nn/CReLU.lua
+++ b/contrib/lua-torch/nn/CReLU.lua
diff --git a/contrib/torch/nn/CSubTable.lua b/contrib/lua-torch/nn/CSubTable.lua
index eb7492055..eb7492055 100644
--- a/contrib/torch/nn/CSubTable.lua
+++ b/contrib/lua-torch/nn/CSubTable.lua
diff --git a/contrib/torch/nn/Clamp.lua b/contrib/lua-torch/nn/Clamp.lua
index 36397a157..36397a157 100644
--- a/contrib/torch/nn/Clamp.lua
+++ b/contrib/lua-torch/nn/Clamp.lua
diff --git a/contrib/torch/nn/ClassNLLCriterion.lua b/contrib/lua-torch/nn/ClassNLLCriterion.lua
index dae0e6685..dae0e6685 100644
--- a/contrib/torch/nn/ClassNLLCriterion.lua
+++ b/contrib/lua-torch/nn/ClassNLLCriterion.lua
diff --git a/contrib/torch/nn/ClassSimplexCriterion.lua b/contrib/lua-torch/nn/ClassSimplexCriterion.lua
index 9cabc011f..9cabc011f 100644
--- a/contrib/torch/nn/ClassSimplexCriterion.lua
+++ b/contrib/lua-torch/nn/ClassSimplexCriterion.lua
diff --git a/contrib/torch/nn/Collapse.lua b/contrib/lua-torch/nn/Collapse.lua
index a088608ca..a088608ca 100644
--- a/contrib/torch/nn/Collapse.lua
+++ b/contrib/lua-torch/nn/Collapse.lua
diff --git a/contrib/torch/nn/Concat.lua b/contrib/lua-torch/nn/Concat.lua
index d7e3ee711..d7e3ee711 100644
--- a/contrib/torch/nn/Concat.lua
+++ b/contrib/lua-torch/nn/Concat.lua
diff --git a/contrib/torch/nn/ConcatTable.lua b/contrib/lua-torch/nn/ConcatTable.lua
index 742719344..742719344 100644
--- a/contrib/torch/nn/ConcatTable.lua
+++ b/contrib/lua-torch/nn/ConcatTable.lua
diff --git a/contrib/torch/nn/Constant.lua b/contrib/lua-torch/nn/Constant.lua
index 07773feb2..07773feb2 100644
--- a/contrib/torch/nn/Constant.lua
+++ b/contrib/lua-torch/nn/Constant.lua
diff --git a/contrib/torch/nn/Container.lua b/contrib/lua-torch/nn/Container.lua
index 7e264bab9..7e264bab9 100644
--- a/contrib/torch/nn/Container.lua
+++ b/contrib/lua-torch/nn/Container.lua
diff --git a/contrib/torch/nn/Contiguous.lua b/contrib/lua-torch/nn/Contiguous.lua
index f9974ce5a..f9974ce5a 100755
--- a/contrib/torch/nn/Contiguous.lua
+++ b/contrib/lua-torch/nn/Contiguous.lua
diff --git a/contrib/torch/nn/Convert.lua b/contrib/lua-torch/nn/Convert.lua
index 855338dd6..855338dd6 100644
--- a/contrib/torch/nn/Convert.lua
+++ b/contrib/lua-torch/nn/Convert.lua
diff --git a/contrib/torch/nn/Copy.lua b/contrib/lua-torch/nn/Copy.lua
index 9f83cf9b4..9f83cf9b4 100644
--- a/contrib/torch/nn/Copy.lua
+++ b/contrib/lua-torch/nn/Copy.lua
diff --git a/contrib/torch/nn/Cosine.lua b/contrib/lua-torch/nn/Cosine.lua
index 19a9cba82..19a9cba82 100644
--- a/contrib/torch/nn/Cosine.lua
+++ b/contrib/lua-torch/nn/Cosine.lua
diff --git a/contrib/torch/nn/CosineDistance.lua b/contrib/lua-torch/nn/CosineDistance.lua
index fe4e4b9f5..fe4e4b9f5 100644
--- a/contrib/torch/nn/CosineDistance.lua
+++ b/contrib/lua-torch/nn/CosineDistance.lua
diff --git a/contrib/torch/nn/CosineEmbeddingCriterion.lua b/contrib/lua-torch/nn/CosineEmbeddingCriterion.lua
index d55e03130..d55e03130 100644
--- a/contrib/torch/nn/CosineEmbeddingCriterion.lua
+++ b/contrib/lua-torch/nn/CosineEmbeddingCriterion.lua
diff --git a/contrib/torch/nn/Criterion.lua b/contrib/lua-torch/nn/Criterion.lua
index e48f06876..e48f06876 100644
--- a/contrib/torch/nn/Criterion.lua
+++ b/contrib/lua-torch/nn/Criterion.lua
diff --git a/contrib/torch/nn/CriterionTable.lua b/contrib/lua-torch/nn/CriterionTable.lua
index 14f67bd39..14f67bd39 100644
--- a/contrib/torch/nn/CriterionTable.lua
+++ b/contrib/lua-torch/nn/CriterionTable.lua
diff --git a/contrib/torch/nn/CrossEntropyCriterion.lua b/contrib/lua-torch/nn/CrossEntropyCriterion.lua
index 2f72cf87f..2f72cf87f 100644
--- a/contrib/torch/nn/CrossEntropyCriterion.lua
+++ b/contrib/lua-torch/nn/CrossEntropyCriterion.lua
diff --git a/contrib/torch/nn/Decorator.lua b/contrib/lua-torch/nn/Decorator.lua
index 05fb4db92..05fb4db92 100644
--- a/contrib/torch/nn/Decorator.lua
+++ b/contrib/lua-torch/nn/Decorator.lua
diff --git a/contrib/torch/nn/DepthConcat.lua b/contrib/lua-torch/nn/DepthConcat.lua
index f64a90eb8..f64a90eb8 100644
--- a/contrib/torch/nn/DepthConcat.lua
+++ b/contrib/lua-torch/nn/DepthConcat.lua
diff --git a/contrib/torch/nn/DistKLDivCriterion.lua b/contrib/lua-torch/nn/DistKLDivCriterion.lua
index bfad57567..bfad57567 100644
--- a/contrib/torch/nn/DistKLDivCriterion.lua
+++ b/contrib/lua-torch/nn/DistKLDivCriterion.lua
diff --git a/contrib/torch/nn/DistanceRatioCriterion.lua b/contrib/lua-torch/nn/DistanceRatioCriterion.lua
index 6b79d0620..6b79d0620 100644
--- a/contrib/torch/nn/DistanceRatioCriterion.lua
+++ b/contrib/lua-torch/nn/DistanceRatioCriterion.lua
diff --git a/contrib/torch/nn/DontCast.lua b/contrib/lua-torch/nn/DontCast.lua
index b89f5436b..b89f5436b 100644
--- a/contrib/torch/nn/DontCast.lua
+++ b/contrib/lua-torch/nn/DontCast.lua
diff --git a/contrib/torch/nn/DotProduct.lua b/contrib/lua-torch/nn/DotProduct.lua
index ccd347e6b..ccd347e6b 100644
--- a/contrib/torch/nn/DotProduct.lua
+++ b/contrib/lua-torch/nn/DotProduct.lua
diff --git a/contrib/torch/nn/Dropout.lua b/contrib/lua-torch/nn/Dropout.lua
index 15f2f4699..15f2f4699 100644
--- a/contrib/torch/nn/Dropout.lua
+++ b/contrib/lua-torch/nn/Dropout.lua
diff --git a/contrib/torch/nn/ELU.lua b/contrib/lua-torch/nn/ELU.lua
index 48a6caa2c..48a6caa2c 100644
--- a/contrib/torch/nn/ELU.lua
+++ b/contrib/lua-torch/nn/ELU.lua
diff --git a/contrib/torch/nn/ErrorMessages.lua b/contrib/lua-torch/nn/ErrorMessages.lua
index a5cbed053..a5cbed053 100644
--- a/contrib/torch/nn/ErrorMessages.lua
+++ b/contrib/lua-torch/nn/ErrorMessages.lua
diff --git a/contrib/torch/nn/Euclidean.lua b/contrib/lua-torch/nn/Euclidean.lua
index 509feff50..509feff50 100644
--- a/contrib/torch/nn/Euclidean.lua
+++ b/contrib/lua-torch/nn/Euclidean.lua
diff --git a/contrib/torch/nn/Exp.lua b/contrib/lua-torch/nn/Exp.lua
index f41569026..f41569026 100644
--- a/contrib/torch/nn/Exp.lua
+++ b/contrib/lua-torch/nn/Exp.lua
diff --git a/contrib/torch/nn/FlattenTable.lua b/contrib/lua-torch/nn/FlattenTable.lua
index 1c182557c..1c182557c 100644
--- a/contrib/torch/nn/FlattenTable.lua
+++ b/contrib/lua-torch/nn/FlattenTable.lua
diff --git a/contrib/torch/nn/GPU.lua b/contrib/lua-torch/nn/GPU.lua
index 758618d8b..758618d8b 100644
--- a/contrib/torch/nn/GPU.lua
+++ b/contrib/lua-torch/nn/GPU.lua
diff --git a/contrib/torch/nn/GatedLinearUnit.lua b/contrib/lua-torch/nn/GatedLinearUnit.lua
index 5273abfd4..5273abfd4 100644
--- a/contrib/torch/nn/GatedLinearUnit.lua
+++ b/contrib/lua-torch/nn/GatedLinearUnit.lua
diff --git a/contrib/torch/nn/GradientReversal.lua b/contrib/lua-torch/nn/GradientReversal.lua
index c08b1dfb0..c08b1dfb0 100644
--- a/contrib/torch/nn/GradientReversal.lua
+++ b/contrib/lua-torch/nn/GradientReversal.lua
diff --git a/contrib/torch/nn/HardShrink.lua b/contrib/lua-torch/nn/HardShrink.lua
index 85ff5909c..85ff5909c 100644
--- a/contrib/torch/nn/HardShrink.lua
+++ b/contrib/lua-torch/nn/HardShrink.lua
diff --git a/contrib/torch/nn/HardTanh.lua b/contrib/lua-torch/nn/HardTanh.lua
index 07cfc6255..07cfc6255 100644
--- a/contrib/torch/nn/HardTanh.lua
+++ b/contrib/lua-torch/nn/HardTanh.lua
diff --git a/contrib/torch/nn/HingeEmbeddingCriterion.lua b/contrib/lua-torch/nn/HingeEmbeddingCriterion.lua
index 13ad00f19..13ad00f19 100644
--- a/contrib/torch/nn/HingeEmbeddingCriterion.lua
+++ b/contrib/lua-torch/nn/HingeEmbeddingCriterion.lua
diff --git a/contrib/torch/nn/Identity.lua b/contrib/lua-torch/nn/Identity.lua
index 5e6ccb624..5e6ccb624 100644
--- a/contrib/torch/nn/Identity.lua
+++ b/contrib/lua-torch/nn/Identity.lua
diff --git a/contrib/torch/nn/Index.lua b/contrib/lua-torch/nn/Index.lua
index 6aa429708..6aa429708 100644
--- a/contrib/torch/nn/Index.lua
+++ b/contrib/lua-torch/nn/Index.lua
diff --git a/contrib/torch/nn/IndexLinear.lua b/contrib/lua-torch/nn/IndexLinear.lua
index 928e5d3f2..928e5d3f2 100644
--- a/contrib/torch/nn/IndexLinear.lua
+++ b/contrib/lua-torch/nn/IndexLinear.lua
diff --git a/contrib/torch/nn/Jacobian.lua b/contrib/lua-torch/nn/Jacobian.lua
index 4f728b18c..4f728b18c 100644
--- a/contrib/torch/nn/Jacobian.lua
+++ b/contrib/lua-torch/nn/Jacobian.lua
diff --git a/contrib/torch/nn/JoinTable.lua b/contrib/lua-torch/nn/JoinTable.lua
index 6ab68e189..6ab68e189 100644
--- a/contrib/torch/nn/JoinTable.lua
+++ b/contrib/lua-torch/nn/JoinTable.lua
diff --git a/contrib/torch/nn/Kmeans.lua b/contrib/lua-torch/nn/Kmeans.lua
index 56066b63d..56066b63d 100644
--- a/contrib/torch/nn/Kmeans.lua
+++ b/contrib/lua-torch/nn/Kmeans.lua
diff --git a/contrib/torch/nn/L1Cost.lua b/contrib/lua-torch/nn/L1Cost.lua
index 6b58e0ec9..6b58e0ec9 100644
--- a/contrib/torch/nn/L1Cost.lua
+++ b/contrib/lua-torch/nn/L1Cost.lua
diff --git a/contrib/torch/nn/L1HingeEmbeddingCriterion.lua b/contrib/lua-torch/nn/L1HingeEmbeddingCriterion.lua
index 6957278f5..6957278f5 100644
--- a/contrib/torch/nn/L1HingeEmbeddingCriterion.lua
+++ b/contrib/lua-torch/nn/L1HingeEmbeddingCriterion.lua
diff --git a/contrib/torch/nn/L1Penalty.lua b/contrib/lua-torch/nn/L1Penalty.lua
index 9ee6b35ff..9ee6b35ff 100644
--- a/contrib/torch/nn/L1Penalty.lua
+++ b/contrib/lua-torch/nn/L1Penalty.lua
diff --git a/contrib/torch/nn/LayerNormalization.lua b/contrib/lua-torch/nn/LayerNormalization.lua
index 722d7c802..722d7c802 100644
--- a/contrib/torch/nn/LayerNormalization.lua
+++ b/contrib/lua-torch/nn/LayerNormalization.lua
diff --git a/contrib/torch/nn/LeakyReLU.lua b/contrib/lua-torch/nn/LeakyReLU.lua
index 56b7f2542..56b7f2542 100644
--- a/contrib/torch/nn/LeakyReLU.lua
+++ b/contrib/lua-torch/nn/LeakyReLU.lua
diff --git a/contrib/torch/nn/Linear.lua b/contrib/lua-torch/nn/Linear.lua
index 09b5979ce..09b5979ce 100644
--- a/contrib/torch/nn/Linear.lua
+++ b/contrib/lua-torch/nn/Linear.lua
diff --git a/contrib/torch/nn/LinearWeightNorm.lua b/contrib/lua-torch/nn/LinearWeightNorm.lua
index a712f5535..a712f5535 100755
--- a/contrib/torch/nn/LinearWeightNorm.lua
+++ b/contrib/lua-torch/nn/LinearWeightNorm.lua
diff --git a/contrib/torch/nn/Log.lua b/contrib/lua-torch/nn/Log.lua
index e8f236bfb..e8f236bfb 100644
--- a/contrib/torch/nn/Log.lua
+++ b/contrib/lua-torch/nn/Log.lua
diff --git a/contrib/torch/nn/LogSigmoid.lua b/contrib/lua-torch/nn/LogSigmoid.lua
index cab848f4d..cab848f4d 100644
--- a/contrib/torch/nn/LogSigmoid.lua
+++ b/contrib/lua-torch/nn/LogSigmoid.lua
diff --git a/contrib/torch/nn/LogSoftMax.lua b/contrib/lua-torch/nn/LogSoftMax.lua
index 37c8acae4..37c8acae4 100644
--- a/contrib/torch/nn/LogSoftMax.lua
+++ b/contrib/lua-torch/nn/LogSoftMax.lua
diff --git a/contrib/torch/nn/LookupTable.lua b/contrib/lua-torch/nn/LookupTable.lua
index 6cffc6c3e..6cffc6c3e 100644
--- a/contrib/torch/nn/LookupTable.lua
+++ b/contrib/lua-torch/nn/LookupTable.lua
diff --git a/contrib/torch/nn/MM.lua b/contrib/lua-torch/nn/MM.lua
index cc978c8cb..cc978c8cb 100644
--- a/contrib/torch/nn/MM.lua
+++ b/contrib/lua-torch/nn/MM.lua
diff --git a/contrib/torch/nn/MSECriterion.lua b/contrib/lua-torch/nn/MSECriterion.lua
index d38beb6bf..d38beb6bf 100644
--- a/contrib/torch/nn/MSECriterion.lua
+++ b/contrib/lua-torch/nn/MSECriterion.lua
diff --git a/contrib/torch/nn/MV.lua b/contrib/lua-torch/nn/MV.lua
index a00478ef6..a00478ef6 100644
--- a/contrib/torch/nn/MV.lua
+++ b/contrib/lua-torch/nn/MV.lua
diff --git a/contrib/torch/nn/MapTable.lua b/contrib/lua-torch/nn/MapTable.lua
index c79f1ea1d..c79f1ea1d 100644
--- a/contrib/torch/nn/MapTable.lua
+++ b/contrib/lua-torch/nn/MapTable.lua
diff --git a/contrib/torch/nn/MarginCriterion.lua b/contrib/lua-torch/nn/MarginCriterion.lua
index 1ab8ad784..1ab8ad784 100644
--- a/contrib/torch/nn/MarginCriterion.lua
+++ b/contrib/lua-torch/nn/MarginCriterion.lua
diff --git a/contrib/torch/nn/MarginRankingCriterion.lua b/contrib/lua-torch/nn/MarginRankingCriterion.lua
index 844d905d5..844d905d5 100644
--- a/contrib/torch/nn/MarginRankingCriterion.lua
+++ b/contrib/lua-torch/nn/MarginRankingCriterion.lua
diff --git a/contrib/torch/nn/MaskedSelect.lua b/contrib/lua-torch/nn/MaskedSelect.lua
index c3f7834e1..c3f7834e1 100644
--- a/contrib/torch/nn/MaskedSelect.lua
+++ b/contrib/lua-torch/nn/MaskedSelect.lua
diff --git a/contrib/torch/nn/Max.lua b/contrib/lua-torch/nn/Max.lua
index 8273e808c..8273e808c 100644
--- a/contrib/torch/nn/Max.lua
+++ b/contrib/lua-torch/nn/Max.lua
diff --git a/contrib/torch/nn/Maxout.lua b/contrib/lua-torch/nn/Maxout.lua
index a797a9f43..a797a9f43 100644
--- a/contrib/torch/nn/Maxout.lua
+++ b/contrib/lua-torch/nn/Maxout.lua
diff --git a/contrib/torch/nn/Mean.lua b/contrib/lua-torch/nn/Mean.lua
index 8087ac95e..8087ac95e 100644
--- a/contrib/torch/nn/Mean.lua
+++ b/contrib/lua-torch/nn/Mean.lua
diff --git a/contrib/torch/nn/Min.lua b/contrib/lua-torch/nn/Min.lua
index 3a3e4a802..3a3e4a802 100644
--- a/contrib/torch/nn/Min.lua
+++ b/contrib/lua-torch/nn/Min.lua
diff --git a/contrib/torch/nn/MixtureTable.lua b/contrib/lua-torch/nn/MixtureTable.lua
index dbe19742f..dbe19742f 100644
--- a/contrib/torch/nn/MixtureTable.lua
+++ b/contrib/lua-torch/nn/MixtureTable.lua
diff --git a/contrib/torch/nn/Module.lua b/contrib/lua-torch/nn/Module.lua
index 3debc5789..3debc5789 100644
--- a/contrib/torch/nn/Module.lua
+++ b/contrib/lua-torch/nn/Module.lua
diff --git a/contrib/torch/nn/ModuleCriterion.lua b/contrib/lua-torch/nn/ModuleCriterion.lua
index bfc79ef55..bfc79ef55 100644
--- a/contrib/torch/nn/ModuleCriterion.lua
+++ b/contrib/lua-torch/nn/ModuleCriterion.lua
diff --git a/contrib/torch/nn/Mul.lua b/contrib/lua-torch/nn/Mul.lua
index efa1db656..efa1db656 100644
--- a/contrib/torch/nn/Mul.lua
+++ b/contrib/lua-torch/nn/Mul.lua
diff --git a/contrib/torch/nn/MulConstant.lua b/contrib/lua-torch/nn/MulConstant.lua
index e8c473bee..e8c473bee 100644
--- a/contrib/torch/nn/MulConstant.lua
+++ b/contrib/lua-torch/nn/MulConstant.lua
diff --git a/contrib/torch/nn/MultiCriterion.lua b/contrib/lua-torch/nn/MultiCriterion.lua
index 959317711..959317711 100644
--- a/contrib/torch/nn/MultiCriterion.lua
+++ b/contrib/lua-torch/nn/MultiCriterion.lua
diff --git a/contrib/torch/nn/MultiLabelMarginCriterion.lua b/contrib/lua-torch/nn/MultiLabelMarginCriterion.lua
index 908b6133c..908b6133c 100644
--- a/contrib/torch/nn/MultiLabelMarginCriterion.lua
+++ b/contrib/lua-torch/nn/MultiLabelMarginCriterion.lua
diff --git a/contrib/torch/nn/MultiLabelSoftMarginCriterion.lua b/contrib/lua-torch/nn/MultiLabelSoftMarginCriterion.lua
index 9d471d449..9d471d449 100644
--- a/contrib/torch/nn/MultiLabelSoftMarginCriterion.lua
+++ b/contrib/lua-torch/nn/MultiLabelSoftMarginCriterion.lua
diff --git a/contrib/torch/nn/MultiMarginCriterion.lua b/contrib/lua-torch/nn/MultiMarginCriterion.lua
index e3122386a..e3122386a 100644
--- a/contrib/torch/nn/MultiMarginCriterion.lua
+++ b/contrib/lua-torch/nn/MultiMarginCriterion.lua
diff --git a/contrib/torch/nn/NaN.lua b/contrib/lua-torch/nn/NaN.lua
index b80f6a04d..b80f6a04d 100644
--- a/contrib/torch/nn/NaN.lua
+++ b/contrib/lua-torch/nn/NaN.lua
diff --git a/contrib/torch/nn/Narrow.lua b/contrib/lua-torch/nn/Narrow.lua
index a6ebaa321..a6ebaa321 100644
--- a/contrib/torch/nn/Narrow.lua
+++ b/contrib/lua-torch/nn/Narrow.lua
diff --git a/contrib/torch/nn/NarrowTable.lua b/contrib/lua-torch/nn/NarrowTable.lua
index 17429f3b1..17429f3b1 100644
--- a/contrib/torch/nn/NarrowTable.lua
+++ b/contrib/lua-torch/nn/NarrowTable.lua
diff --git a/contrib/torch/nn/Normalize.lua b/contrib/lua-torch/nn/Normalize.lua
index 0937ebba9..0937ebba9 100644
--- a/contrib/torch/nn/Normalize.lua
+++ b/contrib/lua-torch/nn/Normalize.lua
diff --git a/contrib/torch/nn/OneHot.lua b/contrib/lua-torch/nn/OneHot.lua
index d1dc1b52d..d1dc1b52d 100644
--- a/contrib/torch/nn/OneHot.lua
+++ b/contrib/lua-torch/nn/OneHot.lua
diff --git a/contrib/torch/nn/PReLU.lua b/contrib/lua-torch/nn/PReLU.lua
index 2e58fba4e..2e58fba4e 100644
--- a/contrib/torch/nn/PReLU.lua
+++ b/contrib/lua-torch/nn/PReLU.lua
diff --git a/contrib/torch/nn/Padding.lua b/contrib/lua-torch/nn/Padding.lua
index d5f7771d0..d5f7771d0 100644
--- a/contrib/torch/nn/Padding.lua
+++ b/contrib/lua-torch/nn/Padding.lua
diff --git a/contrib/torch/nn/PairwiseDistance.lua b/contrib/lua-torch/nn/PairwiseDistance.lua
index 99a502c16..99a502c16 100644
--- a/contrib/torch/nn/PairwiseDistance.lua
+++ b/contrib/lua-torch/nn/PairwiseDistance.lua
diff --git a/contrib/torch/nn/Parallel.lua b/contrib/lua-torch/nn/Parallel.lua
index 58cb9748e..58cb9748e 100644
--- a/contrib/torch/nn/Parallel.lua
+++ b/contrib/lua-torch/nn/Parallel.lua
diff --git a/contrib/torch/nn/ParallelCriterion.lua b/contrib/lua-torch/nn/ParallelCriterion.lua
index 45607d5c3..45607d5c3 100644
--- a/contrib/torch/nn/ParallelCriterion.lua
+++ b/contrib/lua-torch/nn/ParallelCriterion.lua
diff --git a/contrib/torch/nn/ParallelTable.lua b/contrib/lua-torch/nn/ParallelTable.lua
index 2fe0899dd..2fe0899dd 100644
--- a/contrib/torch/nn/ParallelTable.lua
+++ b/contrib/lua-torch/nn/ParallelTable.lua
diff --git a/contrib/torch/nn/PartialLinear.lua b/contrib/lua-torch/nn/PartialLinear.lua
index 6e92cfc08..6e92cfc08 100644
--- a/contrib/torch/nn/PartialLinear.lua
+++ b/contrib/lua-torch/nn/PartialLinear.lua
diff --git a/contrib/torch/nn/PixelShuffle.lua b/contrib/lua-torch/nn/PixelShuffle.lua
index dd58ed948..dd58ed948 100644
--- a/contrib/torch/nn/PixelShuffle.lua
+++ b/contrib/lua-torch/nn/PixelShuffle.lua
diff --git a/contrib/torch/nn/Power.lua b/contrib/lua-torch/nn/Power.lua
index 771183c48..771183c48 100644
--- a/contrib/torch/nn/Power.lua
+++ b/contrib/lua-torch/nn/Power.lua
diff --git a/contrib/torch/nn/PrintSize.lua b/contrib/lua-torch/nn/PrintSize.lua
index d8dc91bff..d8dc91bff 100644
--- a/contrib/torch/nn/PrintSize.lua
+++ b/contrib/lua-torch/nn/PrintSize.lua
diff --git a/contrib/torch/nn/Profile.lua b/contrib/lua-torch/nn/Profile.lua
index 36cd909cd..36cd909cd 100644
--- a/contrib/torch/nn/Profile.lua
+++ b/contrib/lua-torch/nn/Profile.lua
diff --git a/contrib/torch/nn/README.md b/contrib/lua-torch/nn/README.md
index 6efd60962..6efd60962 100644
--- a/contrib/torch/nn/README.md
+++ b/contrib/lua-torch/nn/README.md
diff --git a/contrib/torch/nn/RReLU.lua b/contrib/lua-torch/nn/RReLU.lua
index 843415f7e..843415f7e 100644
--- a/contrib/torch/nn/RReLU.lua
+++ b/contrib/lua-torch/nn/RReLU.lua
diff --git a/contrib/torch/nn/ReLU.lua b/contrib/lua-torch/nn/ReLU.lua
index a6eb271ee..a6eb271ee 100644
--- a/contrib/torch/nn/ReLU.lua
+++ b/contrib/lua-torch/nn/ReLU.lua
diff --git a/contrib/torch/nn/ReLU6.lua b/contrib/lua-torch/nn/ReLU6.lua
index 1cde00b46..1cde00b46 100644
--- a/contrib/torch/nn/ReLU6.lua
+++ b/contrib/lua-torch/nn/ReLU6.lua
diff --git a/contrib/torch/nn/Replicate.lua b/contrib/lua-torch/nn/Replicate.lua
index c7dedd767..c7dedd767 100644
--- a/contrib/torch/nn/Replicate.lua
+++ b/contrib/lua-torch/nn/Replicate.lua
diff --git a/contrib/torch/nn/Reshape.lua b/contrib/lua-torch/nn/Reshape.lua
index d508369fa..d508369fa 100644
--- a/contrib/torch/nn/Reshape.lua
+++ b/contrib/lua-torch/nn/Reshape.lua
diff --git a/contrib/torch/nn/Select.lua b/contrib/lua-torch/nn/Select.lua
index be87c6465..be87c6465 100644
--- a/contrib/torch/nn/Select.lua
+++ b/contrib/lua-torch/nn/Select.lua
diff --git a/contrib/torch/nn/SelectTable.lua b/contrib/lua-torch/nn/SelectTable.lua
index ef26f3507..ef26f3507 100644
--- a/contrib/torch/nn/SelectTable.lua
+++ b/contrib/lua-torch/nn/SelectTable.lua
diff --git a/contrib/torch/nn/Sequential.lua b/contrib/lua-torch/nn/Sequential.lua
index 22b0886b8..22b0886b8 100644
--- a/contrib/torch/nn/Sequential.lua
+++ b/contrib/lua-torch/nn/Sequential.lua
diff --git a/contrib/torch/nn/Sigmoid.lua b/contrib/lua-torch/nn/Sigmoid.lua
index 0126f6f8f..0126f6f8f 100644
--- a/contrib/torch/nn/Sigmoid.lua
+++ b/contrib/lua-torch/nn/Sigmoid.lua
diff --git a/contrib/torch/nn/SmoothL1Criterion.lua b/contrib/lua-torch/nn/SmoothL1Criterion.lua
index be636a94c..be636a94c 100644
--- a/contrib/torch/nn/SmoothL1Criterion.lua
+++ b/contrib/lua-torch/nn/SmoothL1Criterion.lua
diff --git a/contrib/torch/nn/SoftMarginCriterion.lua b/contrib/lua-torch/nn/SoftMarginCriterion.lua
index 96ccda8a4..96ccda8a4 100644
--- a/contrib/torch/nn/SoftMarginCriterion.lua
+++ b/contrib/lua-torch/nn/SoftMarginCriterion.lua
diff --git a/contrib/torch/nn/SoftMax.lua b/contrib/lua-torch/nn/SoftMax.lua
index 23a444cf6..23a444cf6 100644
--- a/contrib/torch/nn/SoftMax.lua
+++ b/contrib/lua-torch/nn/SoftMax.lua
diff --git a/contrib/torch/nn/SoftMin.lua b/contrib/lua-torch/nn/SoftMin.lua
index 7da2a6589..7da2a6589 100644
--- a/contrib/torch/nn/SoftMin.lua
+++ b/contrib/lua-torch/nn/SoftMin.lua
diff --git a/contrib/torch/nn/SoftPlus.lua b/contrib/lua-torch/nn/SoftPlus.lua
index f77b25380..f77b25380 100644
--- a/contrib/torch/nn/SoftPlus.lua
+++ b/contrib/lua-torch/nn/SoftPlus.lua
diff --git a/contrib/torch/nn/SoftShrink.lua b/contrib/lua-torch/nn/SoftShrink.lua
index 67af15a98..67af15a98 100644
--- a/contrib/torch/nn/SoftShrink.lua
+++ b/contrib/lua-torch/nn/SoftShrink.lua
diff --git a/contrib/torch/nn/SoftSign.lua b/contrib/lua-torch/nn/SoftSign.lua
index ee72011f1..ee72011f1 100644
--- a/contrib/torch/nn/SoftSign.lua
+++ b/contrib/lua-torch/nn/SoftSign.lua
diff --git a/contrib/torch/nn/SparseJacobian.lua b/contrib/lua-torch/nn/SparseJacobian.lua
index 7f4c02444..7f4c02444 100644
--- a/contrib/torch/nn/SparseJacobian.lua
+++ b/contrib/lua-torch/nn/SparseJacobian.lua
diff --git a/contrib/torch/nn/SparseLinear.lua b/contrib/lua-torch/nn/SparseLinear.lua
index 9a50c6912..9a50c6912 100644
--- a/contrib/torch/nn/SparseLinear.lua
+++ b/contrib/lua-torch/nn/SparseLinear.lua
diff --git a/contrib/torch/nn/SpatialAdaptiveAveragePooling.lua b/contrib/lua-torch/nn/SpatialAdaptiveAveragePooling.lua
index 2e223580a..2e223580a 100644
--- a/contrib/torch/nn/SpatialAdaptiveAveragePooling.lua
+++ b/contrib/lua-torch/nn/SpatialAdaptiveAveragePooling.lua
diff --git a/contrib/torch/nn/SpatialAdaptiveMaxPooling.lua b/contrib/lua-torch/nn/SpatialAdaptiveMaxPooling.lua
index b78261c3d..b78261c3d 100644
--- a/contrib/torch/nn/SpatialAdaptiveMaxPooling.lua
+++ b/contrib/lua-torch/nn/SpatialAdaptiveMaxPooling.lua
diff --git a/contrib/torch/nn/SpatialAutoCropMSECriterion.lua b/contrib/lua-torch/nn/SpatialAutoCropMSECriterion.lua
index 97206a062..97206a062 100644
--- a/contrib/torch/nn/SpatialAutoCropMSECriterion.lua
+++ b/contrib/lua-torch/nn/SpatialAutoCropMSECriterion.lua
diff --git a/contrib/torch/nn/SpatialAveragePooling.lua b/contrib/lua-torch/nn/SpatialAveragePooling.lua
index 1e7666827..1e7666827 100644
--- a/contrib/torch/nn/SpatialAveragePooling.lua
+++ b/contrib/lua-torch/nn/SpatialAveragePooling.lua
diff --git a/contrib/torch/nn/SpatialBatchNormalization.lua b/contrib/lua-torch/nn/SpatialBatchNormalization.lua
index c5004ce3a..c5004ce3a 100644
--- a/contrib/torch/nn/SpatialBatchNormalization.lua
+++ b/contrib/lua-torch/nn/SpatialBatchNormalization.lua
diff --git a/contrib/torch/nn/SpatialClassNLLCriterion.lua b/contrib/lua-torch/nn/SpatialClassNLLCriterion.lua
index fbd367410..fbd367410 100644
--- a/contrib/torch/nn/SpatialClassNLLCriterion.lua
+++ b/contrib/lua-torch/nn/SpatialClassNLLCriterion.lua
diff --git a/contrib/torch/nn/SpatialContrastiveNormalization.lua b/contrib/lua-torch/nn/SpatialContrastiveNormalization.lua
index 0ad251ae4..0ad251ae4 100644
--- a/contrib/torch/nn/SpatialContrastiveNormalization.lua
+++ b/contrib/lua-torch/nn/SpatialContrastiveNormalization.lua
diff --git a/contrib/torch/nn/SpatialConvolution.lua b/contrib/lua-torch/nn/SpatialConvolution.lua
index 15a2b4b62..15a2b4b62 100644
--- a/contrib/torch/nn/SpatialConvolution.lua
+++ b/contrib/lua-torch/nn/SpatialConvolution.lua
diff --git a/contrib/torch/nn/SpatialConvolutionLocal.lua b/contrib/lua-torch/nn/SpatialConvolutionLocal.lua
index 9494c2ffe..9494c2ffe 100644
--- a/contrib/torch/nn/SpatialConvolutionLocal.lua
+++ b/contrib/lua-torch/nn/SpatialConvolutionLocal.lua
diff --git a/contrib/torch/nn/SpatialConvolutionMM.lua b/contrib/lua-torch/nn/SpatialConvolutionMM.lua
index f20734f9b..f20734f9b 100644
--- a/contrib/torch/nn/SpatialConvolutionMM.lua
+++ b/contrib/lua-torch/nn/SpatialConvolutionMM.lua
diff --git a/contrib/torch/nn/SpatialConvolutionMap.lua b/contrib/lua-torch/nn/SpatialConvolutionMap.lua
index 9051c119e..9051c119e 100644
--- a/contrib/torch/nn/SpatialConvolutionMap.lua
+++ b/contrib/lua-torch/nn/SpatialConvolutionMap.lua
diff --git a/contrib/torch/nn/SpatialCrossMapLRN.lua b/contrib/lua-torch/nn/SpatialCrossMapLRN.lua
index 088eb07f0..088eb07f0 100644
--- a/contrib/torch/nn/SpatialCrossMapLRN.lua
+++ b/contrib/lua-torch/nn/SpatialCrossMapLRN.lua
diff --git a/contrib/torch/nn/SpatialDepthWiseConvolution.lua b/contrib/lua-torch/nn/SpatialDepthWiseConvolution.lua
index 1132f04cb..1132f04cb 100644
--- a/contrib/torch/nn/SpatialDepthWiseConvolution.lua
+++ b/contrib/lua-torch/nn/SpatialDepthWiseConvolution.lua
diff --git a/contrib/torch/nn/SpatialDilatedConvolution.lua b/contrib/lua-torch/nn/SpatialDilatedConvolution.lua
index a0590c7e9..a0590c7e9 100644
--- a/contrib/torch/nn/SpatialDilatedConvolution.lua
+++ b/contrib/lua-torch/nn/SpatialDilatedConvolution.lua
diff --git a/contrib/torch/nn/SpatialDilatedMaxPooling.lua b/contrib/lua-torch/nn/SpatialDilatedMaxPooling.lua
index 34525a4ad..34525a4ad 100644
--- a/contrib/torch/nn/SpatialDilatedMaxPooling.lua
+++ b/contrib/lua-torch/nn/SpatialDilatedMaxPooling.lua
diff --git a/contrib/torch/nn/SpatialDivisiveNormalization.lua b/contrib/lua-torch/nn/SpatialDivisiveNormalization.lua
index dc2b8c530..dc2b8c530 100644
--- a/contrib/torch/nn/SpatialDivisiveNormalization.lua
+++ b/contrib/lua-torch/nn/SpatialDivisiveNormalization.lua
diff --git a/contrib/torch/nn/SpatialDropout.lua b/contrib/lua-torch/nn/SpatialDropout.lua
index 4320061b7..4320061b7 100644
--- a/contrib/torch/nn/SpatialDropout.lua
+++ b/contrib/lua-torch/nn/SpatialDropout.lua
diff --git a/contrib/torch/nn/SpatialFractionalMaxPooling.lua b/contrib/lua-torch/nn/SpatialFractionalMaxPooling.lua
index 884751d41..884751d41 100644
--- a/contrib/torch/nn/SpatialFractionalMaxPooling.lua
+++ b/contrib/lua-torch/nn/SpatialFractionalMaxPooling.lua
diff --git a/contrib/torch/nn/SpatialFullConvolution.lua b/contrib/lua-torch/nn/SpatialFullConvolution.lua
index e6019bc18..e6019bc18 100644
--- a/contrib/torch/nn/SpatialFullConvolution.lua
+++ b/contrib/lua-torch/nn/SpatialFullConvolution.lua
diff --git a/contrib/torch/nn/SpatialFullConvolutionMap.lua b/contrib/lua-torch/nn/SpatialFullConvolutionMap.lua
index 008f5e7cf..008f5e7cf 100644
--- a/contrib/torch/nn/SpatialFullConvolutionMap.lua
+++ b/contrib/lua-torch/nn/SpatialFullConvolutionMap.lua
diff --git a/contrib/torch/nn/SpatialLPPooling.lua b/contrib/lua-torch/nn/SpatialLPPooling.lua
index 49a8493cf..49a8493cf 100644
--- a/contrib/torch/nn/SpatialLPPooling.lua
+++ b/contrib/lua-torch/nn/SpatialLPPooling.lua
diff --git a/contrib/torch/nn/SpatialLogSoftMax.lua b/contrib/lua-torch/nn/SpatialLogSoftMax.lua
index 9c81d49e1..9c81d49e1 100644
--- a/contrib/torch/nn/SpatialLogSoftMax.lua
+++ b/contrib/lua-torch/nn/SpatialLogSoftMax.lua
diff --git a/contrib/torch/nn/SpatialMaxPooling.lua b/contrib/lua-torch/nn/SpatialMaxPooling.lua
index 5c865c631..5c865c631 100644
--- a/contrib/torch/nn/SpatialMaxPooling.lua
+++ b/contrib/lua-torch/nn/SpatialMaxPooling.lua
diff --git a/contrib/torch/nn/SpatialMaxUnpooling.lua b/contrib/lua-torch/nn/SpatialMaxUnpooling.lua
index 408bcc052..408bcc052 100644
--- a/contrib/torch/nn/SpatialMaxUnpooling.lua
+++ b/contrib/lua-torch/nn/SpatialMaxUnpooling.lua
diff --git a/contrib/torch/nn/SpatialReflectionPadding.lua b/contrib/lua-torch/nn/SpatialReflectionPadding.lua
index 9ce4612ad..9ce4612ad 100644
--- a/contrib/torch/nn/SpatialReflectionPadding.lua
+++ b/contrib/lua-torch/nn/SpatialReflectionPadding.lua
diff --git a/contrib/torch/nn/SpatialReplicationPadding.lua b/contrib/lua-torch/nn/SpatialReplicationPadding.lua
index 429763f9b..429763f9b 100644
--- a/contrib/torch/nn/SpatialReplicationPadding.lua
+++ b/contrib/lua-torch/nn/SpatialReplicationPadding.lua
diff --git a/contrib/torch/nn/SpatialSoftMax.lua b/contrib/lua-torch/nn/SpatialSoftMax.lua
index 56f0b40e2..56f0b40e2 100644
--- a/contrib/torch/nn/SpatialSoftMax.lua
+++ b/contrib/lua-torch/nn/SpatialSoftMax.lua
diff --git a/contrib/torch/nn/SpatialSubSampling.lua b/contrib/lua-torch/nn/SpatialSubSampling.lua
index 4e3fb8881..4e3fb8881 100644
--- a/contrib/torch/nn/SpatialSubSampling.lua
+++ b/contrib/lua-torch/nn/SpatialSubSampling.lua
diff --git a/contrib/torch/nn/SpatialSubtractiveNormalization.lua b/contrib/lua-torch/nn/SpatialSubtractiveNormalization.lua
index d430083e9..d430083e9 100644
--- a/contrib/torch/nn/SpatialSubtractiveNormalization.lua
+++ b/contrib/lua-torch/nn/SpatialSubtractiveNormalization.lua
diff --git a/contrib/torch/nn/SpatialUpSamplingBilinear.lua b/contrib/lua-torch/nn/SpatialUpSamplingBilinear.lua
index 12e1ce8f2..12e1ce8f2 100644
--- a/contrib/torch/nn/SpatialUpSamplingBilinear.lua
+++ b/contrib/lua-torch/nn/SpatialUpSamplingBilinear.lua
diff --git a/contrib/torch/nn/SpatialUpSamplingNearest.lua b/contrib/lua-torch/nn/SpatialUpSamplingNearest.lua
index 362ae73a3..362ae73a3 100644
--- a/contrib/torch/nn/SpatialUpSamplingNearest.lua
+++ b/contrib/lua-torch/nn/SpatialUpSamplingNearest.lua
diff --git a/contrib/torch/nn/SpatialZeroPadding.lua b/contrib/lua-torch/nn/SpatialZeroPadding.lua
index f19925841..f19925841 100644
--- a/contrib/torch/nn/SpatialZeroPadding.lua
+++ b/contrib/lua-torch/nn/SpatialZeroPadding.lua
diff --git a/contrib/torch/nn/SplitTable.lua b/contrib/lua-torch/nn/SplitTable.lua
index 7c4f968e6..7c4f968e6 100644
--- a/contrib/torch/nn/SplitTable.lua
+++ b/contrib/lua-torch/nn/SplitTable.lua
diff --git a/contrib/torch/nn/Sqrt.lua b/contrib/lua-torch/nn/Sqrt.lua
index df354a175..df354a175 100644
--- a/contrib/torch/nn/Sqrt.lua
+++ b/contrib/lua-torch/nn/Sqrt.lua
diff --git a/contrib/torch/nn/Square.lua b/contrib/lua-torch/nn/Square.lua
index a6292afb9..a6292afb9 100644
--- a/contrib/torch/nn/Square.lua
+++ b/contrib/lua-torch/nn/Square.lua
diff --git a/contrib/torch/nn/Squeeze.lua b/contrib/lua-torch/nn/Squeeze.lua
index 7d204a19d..7d204a19d 100644
--- a/contrib/torch/nn/Squeeze.lua
+++ b/contrib/lua-torch/nn/Squeeze.lua
diff --git a/contrib/torch/nn/StochasticGradient.lua b/contrib/lua-torch/nn/StochasticGradient.lua
index a060371e8..a060371e8 100644
--- a/contrib/torch/nn/StochasticGradient.lua
+++ b/contrib/lua-torch/nn/StochasticGradient.lua
diff --git a/contrib/torch/nn/Sum.lua b/contrib/lua-torch/nn/Sum.lua
index 7fe8a1ab8..7fe8a1ab8 100644
--- a/contrib/torch/nn/Sum.lua
+++ b/contrib/lua-torch/nn/Sum.lua
diff --git a/contrib/torch/nn/THNN.lua b/contrib/lua-torch/nn/THNN.lua
index 0848e9ed2..0848e9ed2 100644
--- a/contrib/torch/nn/THNN.lua
+++ b/contrib/lua-torch/nn/THNN.lua
diff --git a/contrib/torch/nn/Tanh.lua b/contrib/lua-torch/nn/Tanh.lua
index fc42cbbfd..fc42cbbfd 100644
--- a/contrib/torch/nn/Tanh.lua
+++ b/contrib/lua-torch/nn/Tanh.lua
diff --git a/contrib/torch/nn/TanhShrink.lua b/contrib/lua-torch/nn/TanhShrink.lua
index 96df6c5b7..96df6c5b7 100644
--- a/contrib/torch/nn/TanhShrink.lua
+++ b/contrib/lua-torch/nn/TanhShrink.lua
diff --git a/contrib/torch/nn/TemporalConvolution.lua b/contrib/lua-torch/nn/TemporalConvolution.lua
index 4b3a89eb6..4b3a89eb6 100644
--- a/contrib/torch/nn/TemporalConvolution.lua
+++ b/contrib/lua-torch/nn/TemporalConvolution.lua
diff --git a/contrib/torch/nn/TemporalDynamicKMaxPooling.lua b/contrib/lua-torch/nn/TemporalDynamicKMaxPooling.lua
index 644a0fa9c..644a0fa9c 100644
--- a/contrib/torch/nn/TemporalDynamicKMaxPooling.lua
+++ b/contrib/lua-torch/nn/TemporalDynamicKMaxPooling.lua
diff --git a/contrib/torch/nn/TemporalMaxPooling.lua b/contrib/lua-torch/nn/TemporalMaxPooling.lua
index 894f4a99f..894f4a99f 100644
--- a/contrib/torch/nn/TemporalMaxPooling.lua
+++ b/contrib/lua-torch/nn/TemporalMaxPooling.lua
diff --git a/contrib/torch/nn/TemporalRowConvolution.lua b/contrib/lua-torch/nn/TemporalRowConvolution.lua
index 7c9d6a269..7c9d6a269 100644
--- a/contrib/torch/nn/TemporalRowConvolution.lua
+++ b/contrib/lua-torch/nn/TemporalRowConvolution.lua
diff --git a/contrib/torch/nn/TemporalSubSampling.lua b/contrib/lua-torch/nn/TemporalSubSampling.lua
index e9287d63d..e9287d63d 100644
--- a/contrib/torch/nn/TemporalSubSampling.lua
+++ b/contrib/lua-torch/nn/TemporalSubSampling.lua
diff --git a/contrib/torch/nn/Threshold.lua b/contrib/lua-torch/nn/Threshold.lua
index 6fdd26408..6fdd26408 100644
--- a/contrib/torch/nn/Threshold.lua
+++ b/contrib/lua-torch/nn/Threshold.lua
diff --git a/contrib/torch/nn/Transpose.lua b/contrib/lua-torch/nn/Transpose.lua
index cceb2b643..cceb2b643 100644
--- a/contrib/torch/nn/Transpose.lua
+++ b/contrib/lua-torch/nn/Transpose.lua
diff --git a/contrib/torch/nn/Unsqueeze.lua b/contrib/lua-torch/nn/Unsqueeze.lua
index 2e82a25a0..2e82a25a0 100644
--- a/contrib/torch/nn/Unsqueeze.lua
+++ b/contrib/lua-torch/nn/Unsqueeze.lua
diff --git a/contrib/torch/nn/View.lua b/contrib/lua-torch/nn/View.lua
index 542e57e16..542e57e16 100644
--- a/contrib/torch/nn/View.lua
+++ b/contrib/lua-torch/nn/View.lua
diff --git a/contrib/torch/nn/VolumetricAveragePooling.lua b/contrib/lua-torch/nn/VolumetricAveragePooling.lua
index df6d2c405..df6d2c405 100644
--- a/contrib/torch/nn/VolumetricAveragePooling.lua
+++ b/contrib/lua-torch/nn/VolumetricAveragePooling.lua
diff --git a/contrib/torch/nn/VolumetricBatchNormalization.lua b/contrib/lua-torch/nn/VolumetricBatchNormalization.lua
index 6168a9245..6168a9245 100644
--- a/contrib/torch/nn/VolumetricBatchNormalization.lua
+++ b/contrib/lua-torch/nn/VolumetricBatchNormalization.lua
diff --git a/contrib/torch/nn/VolumetricConvolution.lua b/contrib/lua-torch/nn/VolumetricConvolution.lua
index 329609aff..329609aff 100644
--- a/contrib/torch/nn/VolumetricConvolution.lua
+++ b/contrib/lua-torch/nn/VolumetricConvolution.lua
diff --git a/contrib/torch/nn/VolumetricDilatedConvolution.lua b/contrib/lua-torch/nn/VolumetricDilatedConvolution.lua
index f1337ebaa..f1337ebaa 100644
--- a/contrib/torch/nn/VolumetricDilatedConvolution.lua
+++ b/contrib/lua-torch/nn/VolumetricDilatedConvolution.lua
diff --git a/contrib/torch/nn/VolumetricDilatedMaxPooling.lua b/contrib/lua-torch/nn/VolumetricDilatedMaxPooling.lua
index 249b2b58e..249b2b58e 100644
--- a/contrib/torch/nn/VolumetricDilatedMaxPooling.lua
+++ b/contrib/lua-torch/nn/VolumetricDilatedMaxPooling.lua
diff --git a/contrib/torch/nn/VolumetricDropout.lua b/contrib/lua-torch/nn/VolumetricDropout.lua
index 809e28afe..809e28afe 100644
--- a/contrib/torch/nn/VolumetricDropout.lua
+++ b/contrib/lua-torch/nn/VolumetricDropout.lua
diff --git a/contrib/torch/nn/VolumetricFractionalMaxPooling.lua b/contrib/lua-torch/nn/VolumetricFractionalMaxPooling.lua
index f5ff58cf0..f5ff58cf0 100644
--- a/contrib/torch/nn/VolumetricFractionalMaxPooling.lua
+++ b/contrib/lua-torch/nn/VolumetricFractionalMaxPooling.lua
diff --git a/contrib/torch/nn/VolumetricFullConvolution.lua b/contrib/lua-torch/nn/VolumetricFullConvolution.lua
index 0ce23401e..0ce23401e 100644
--- a/contrib/torch/nn/VolumetricFullConvolution.lua
+++ b/contrib/lua-torch/nn/VolumetricFullConvolution.lua
diff --git a/contrib/torch/nn/VolumetricMaxPooling.lua b/contrib/lua-torch/nn/VolumetricMaxPooling.lua
index e25c5b31c..e25c5b31c 100644
--- a/contrib/torch/nn/VolumetricMaxPooling.lua
+++ b/contrib/lua-torch/nn/VolumetricMaxPooling.lua
diff --git a/contrib/torch/nn/VolumetricMaxUnpooling.lua b/contrib/lua-torch/nn/VolumetricMaxUnpooling.lua
index 6291f5b85..6291f5b85 100644
--- a/contrib/torch/nn/VolumetricMaxUnpooling.lua
+++ b/contrib/lua-torch/nn/VolumetricMaxUnpooling.lua
diff --git a/contrib/torch/nn/VolumetricReplicationPadding.lua b/contrib/lua-torch/nn/VolumetricReplicationPadding.lua
index 31a9503fd..31a9503fd 100644
--- a/contrib/torch/nn/VolumetricReplicationPadding.lua
+++ b/contrib/lua-torch/nn/VolumetricReplicationPadding.lua
diff --git a/contrib/torch/nn/WeightNorm.lua b/contrib/lua-torch/nn/WeightNorm.lua
index 3ffcd90aa..3ffcd90aa 100644
--- a/contrib/torch/nn/WeightNorm.lua
+++ b/contrib/lua-torch/nn/WeightNorm.lua
diff --git a/contrib/torch/nn/WeightedEuclidean.lua b/contrib/lua-torch/nn/WeightedEuclidean.lua
index dbf4158a9..dbf4158a9 100644
--- a/contrib/torch/nn/WeightedEuclidean.lua
+++ b/contrib/lua-torch/nn/WeightedEuclidean.lua
diff --git a/contrib/torch/nn/WeightedMSECriterion.lua b/contrib/lua-torch/nn/WeightedMSECriterion.lua
index 933472937..933472937 100644
--- a/contrib/torch/nn/WeightedMSECriterion.lua
+++ b/contrib/lua-torch/nn/WeightedMSECriterion.lua
diff --git a/contrib/torch/nn/WhiteNoise.lua b/contrib/lua-torch/nn/WhiteNoise.lua
index f1defb646..f1defb646 100644
--- a/contrib/torch/nn/WhiteNoise.lua
+++ b/contrib/lua-torch/nn/WhiteNoise.lua
diff --git a/contrib/torch/nn/ZeroGrad.lua b/contrib/lua-torch/nn/ZeroGrad.lua
index 7c941ce1c..7c941ce1c 100644
--- a/contrib/torch/nn/ZeroGrad.lua
+++ b/contrib/lua-torch/nn/ZeroGrad.lua
diff --git a/contrib/torch/nn/ZipTable.lua b/contrib/lua-torch/nn/ZipTable.lua
index 7b18619eb..7b18619eb 100644
--- a/contrib/torch/nn/ZipTable.lua
+++ b/contrib/lua-torch/nn/ZipTable.lua
diff --git a/contrib/torch/nn/ZipTableOneToMany.lua b/contrib/lua-torch/nn/ZipTableOneToMany.lua
index d4a80fe0d..d4a80fe0d 100644
--- a/contrib/torch/nn/ZipTableOneToMany.lua
+++ b/contrib/lua-torch/nn/ZipTableOneToMany.lua
diff --git a/contrib/torch/nn/hessian.lua b/contrib/lua-torch/nn/hessian.lua
index b841d8c59..b841d8c59 100644
--- a/contrib/torch/nn/hessian.lua
+++ b/contrib/lua-torch/nn/hessian.lua
diff --git a/contrib/torch/nn/init.lua b/contrib/lua-torch/nn/init.lua
index 4319a8868..4319a8868 100755
--- a/contrib/torch/nn/init.lua
+++ b/contrib/lua-torch/nn/init.lua
diff --git a/contrib/torch/nn/lib/CMakeLists.txt b/contrib/lua-torch/nn/lib/CMakeLists.txt
index de04595f6..de04595f6 100644
--- a/contrib/torch/nn/lib/CMakeLists.txt
+++ b/contrib/lua-torch/nn/lib/CMakeLists.txt
diff --git a/contrib/torch/nn/lib/THNN/CMakeLists.txt b/contrib/lua-torch/nn/lib/THNN/CMakeLists.txt
index 00908a5b1..00908a5b1 100644
--- a/contrib/torch/nn/lib/THNN/CMakeLists.txt
+++ b/contrib/lua-torch/nn/lib/THNN/CMakeLists.txt
diff --git a/contrib/torch/nn/lib/THNN/README.md b/contrib/lua-torch/nn/lib/THNN/README.md
index e6c61601d..e6c61601d 100644
--- a/contrib/torch/nn/lib/THNN/README.md
+++ b/contrib/lua-torch/nn/lib/THNN/README.md
diff --git a/contrib/torch/nn/lib/THNN/THNN.h b/contrib/lua-torch/nn/lib/THNN/THNN.h
index 0019b7976..0019b7976 100644
--- a/contrib/torch/nn/lib/THNN/THNN.h
+++ b/contrib/lua-torch/nn/lib/THNN/THNN.h
diff --git a/contrib/torch/nn/lib/THNN/generic/Abs.c b/contrib/lua-torch/nn/lib/THNN/generic/Abs.c
index 28721ec8e..28721ec8e 100644
--- a/contrib/torch/nn/lib/THNN/generic/Abs.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/Abs.c
diff --git a/contrib/torch/nn/lib/THNN/generic/AbsCriterion.c b/contrib/lua-torch/nn/lib/THNN/generic/AbsCriterion.c
index 9bee5de9e..9bee5de9e 100644
--- a/contrib/torch/nn/lib/THNN/generic/AbsCriterion.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/AbsCriterion.c
diff --git a/contrib/torch/nn/lib/THNN/generic/BCECriterion.c b/contrib/lua-torch/nn/lib/THNN/generic/BCECriterion.c
index 637a4067e..637a4067e 100644
--- a/contrib/torch/nn/lib/THNN/generic/BCECriterion.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/BCECriterion.c
diff --git a/contrib/torch/nn/lib/THNN/generic/BatchNormalization.c b/contrib/lua-torch/nn/lib/THNN/generic/BatchNormalization.c
index b8f462790..b8f462790 100644
--- a/contrib/torch/nn/lib/THNN/generic/BatchNormalization.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/BatchNormalization.c
diff --git a/contrib/torch/nn/lib/THNN/generic/ClassNLLCriterion.c b/contrib/lua-torch/nn/lib/THNN/generic/ClassNLLCriterion.c
index 4cf37aeaf..4cf37aeaf 100644
--- a/contrib/torch/nn/lib/THNN/generic/ClassNLLCriterion.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/ClassNLLCriterion.c
diff --git a/contrib/torch/nn/lib/THNN/generic/DistKLDivCriterion.c b/contrib/lua-torch/nn/lib/THNN/generic/DistKLDivCriterion.c
index 6bd6aa067..6bd6aa067 100644
--- a/contrib/torch/nn/lib/THNN/generic/DistKLDivCriterion.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/DistKLDivCriterion.c
diff --git a/contrib/torch/nn/lib/THNN/generic/ELU.c b/contrib/lua-torch/nn/lib/THNN/generic/ELU.c
index ddcfb9705..ddcfb9705 100644
--- a/contrib/torch/nn/lib/THNN/generic/ELU.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/ELU.c
diff --git a/contrib/torch/nn/lib/THNN/generic/FusedRNNKernel.c b/contrib/lua-torch/nn/lib/THNN/generic/FusedRNNKernel.c
index 30788b0a2..30788b0a2 100644
--- a/contrib/torch/nn/lib/THNN/generic/FusedRNNKernel.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/FusedRNNKernel.c
diff --git a/contrib/torch/nn/lib/THNN/generic/GatedLinearUnit.c b/contrib/lua-torch/nn/lib/THNN/generic/GatedLinearUnit.c
index 274a27e3b..274a27e3b 100644
--- a/contrib/torch/nn/lib/THNN/generic/GatedLinearUnit.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/GatedLinearUnit.c
diff --git a/contrib/torch/nn/lib/THNN/generic/HardShrink.c b/contrib/lua-torch/nn/lib/THNN/generic/HardShrink.c
index aaae85bac..aaae85bac 100644
--- a/contrib/torch/nn/lib/THNN/generic/HardShrink.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/HardShrink.c
diff --git a/contrib/torch/nn/lib/THNN/generic/HardTanh.c b/contrib/lua-torch/nn/lib/THNN/generic/HardTanh.c
index 589a66e15..589a66e15 100644
--- a/contrib/torch/nn/lib/THNN/generic/HardTanh.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/HardTanh.c
diff --git a/contrib/torch/nn/lib/THNN/generic/IndexLinear.c b/contrib/lua-torch/nn/lib/THNN/generic/IndexLinear.c
index 42d8368ba..42d8368ba 100644
--- a/contrib/torch/nn/lib/THNN/generic/IndexLinear.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/IndexLinear.c
diff --git a/contrib/torch/nn/lib/THNN/generic/L1Cost.c b/contrib/lua-torch/nn/lib/THNN/generic/L1Cost.c
index 53940e894..53940e894 100644
--- a/contrib/torch/nn/lib/THNN/generic/L1Cost.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/L1Cost.c
diff --git a/contrib/torch/nn/lib/THNN/generic/LeakyReLU.c b/contrib/lua-torch/nn/lib/THNN/generic/LeakyReLU.c
index 074047d83..074047d83 100644
--- a/contrib/torch/nn/lib/THNN/generic/LeakyReLU.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/LeakyReLU.c
diff --git a/contrib/torch/nn/lib/THNN/generic/Linear.c b/contrib/lua-torch/nn/lib/THNN/generic/Linear.c
index 8c5cd115e..8c5cd115e 100644
--- a/contrib/torch/nn/lib/THNN/generic/Linear.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/Linear.c
diff --git a/contrib/torch/nn/lib/THNN/generic/LogSigmoid.c b/contrib/lua-torch/nn/lib/THNN/generic/LogSigmoid.c
index 651d56002..651d56002 100644
--- a/contrib/torch/nn/lib/THNN/generic/LogSigmoid.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/LogSigmoid.c
diff --git a/contrib/torch/nn/lib/THNN/generic/LogSoftMax.c b/contrib/lua-torch/nn/lib/THNN/generic/LogSoftMax.c
index a7280422b..a7280422b 100644
--- a/contrib/torch/nn/lib/THNN/generic/LogSoftMax.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/LogSoftMax.c
diff --git a/contrib/torch/nn/lib/THNN/generic/LookupTable.c b/contrib/lua-torch/nn/lib/THNN/generic/LookupTable.c
index 46bc2c3c1..46bc2c3c1 100644
--- a/contrib/torch/nn/lib/THNN/generic/LookupTable.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/LookupTable.c
diff --git a/contrib/torch/nn/lib/THNN/generic/MSECriterion.c b/contrib/lua-torch/nn/lib/THNN/generic/MSECriterion.c
index 58911f6f0..58911f6f0 100644
--- a/contrib/torch/nn/lib/THNN/generic/MSECriterion.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/MSECriterion.c
diff --git a/contrib/torch/nn/lib/THNN/generic/MarginCriterion.c b/contrib/lua-torch/nn/lib/THNN/generic/MarginCriterion.c
index d6d9b60b9..d6d9b60b9 100644
--- a/contrib/torch/nn/lib/THNN/generic/MarginCriterion.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/MarginCriterion.c
diff --git a/contrib/torch/nn/lib/THNN/generic/MultiLabelMarginCriterion.c b/contrib/lua-torch/nn/lib/THNN/generic/MultiLabelMarginCriterion.c
index 16398c13c..16398c13c 100644
--- a/contrib/torch/nn/lib/THNN/generic/MultiLabelMarginCriterion.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/MultiLabelMarginCriterion.c
diff --git a/contrib/torch/nn/lib/THNN/generic/MultiMarginCriterion.c b/contrib/lua-torch/nn/lib/THNN/generic/MultiMarginCriterion.c
index 2f8f8ff58..2f8f8ff58 100644
--- a/contrib/torch/nn/lib/THNN/generic/MultiMarginCriterion.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/MultiMarginCriterion.c
diff --git a/contrib/torch/nn/lib/THNN/generic/PReLU.c b/contrib/lua-torch/nn/lib/THNN/generic/PReLU.c
index 488322fde..488322fde 100644
--- a/contrib/torch/nn/lib/THNN/generic/PReLU.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/PReLU.c
diff --git a/contrib/torch/nn/lib/THNN/generic/RReLU.c b/contrib/lua-torch/nn/lib/THNN/generic/RReLU.c
index 8fd46d3c2..8fd46d3c2 100644
--- a/contrib/torch/nn/lib/THNN/generic/RReLU.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/RReLU.c
diff --git a/contrib/torch/nn/lib/THNN/generic/Sigmoid.c b/contrib/lua-torch/nn/lib/THNN/generic/Sigmoid.c
index 17fb2cb4d..17fb2cb4d 100644
--- a/contrib/torch/nn/lib/THNN/generic/Sigmoid.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/Sigmoid.c
diff --git a/contrib/torch/nn/lib/THNN/generic/SmoothL1Criterion.c b/contrib/lua-torch/nn/lib/THNN/generic/SmoothL1Criterion.c
index d1928d11c..d1928d11c 100644
--- a/contrib/torch/nn/lib/THNN/generic/SmoothL1Criterion.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/SmoothL1Criterion.c
diff --git a/contrib/torch/nn/lib/THNN/generic/SoftMarginCriterion.c b/contrib/lua-torch/nn/lib/THNN/generic/SoftMarginCriterion.c
index bac0a3b53..bac0a3b53 100644
--- a/contrib/torch/nn/lib/THNN/generic/SoftMarginCriterion.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/SoftMarginCriterion.c
diff --git a/contrib/torch/nn/lib/THNN/generic/SoftMax.c b/contrib/lua-torch/nn/lib/THNN/generic/SoftMax.c
index 7b60d64c2..7b60d64c2 100644
--- a/contrib/torch/nn/lib/THNN/generic/SoftMax.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/SoftMax.c
diff --git a/contrib/torch/nn/lib/THNN/generic/SoftPlus.c b/contrib/lua-torch/nn/lib/THNN/generic/SoftPlus.c
index 6491e66d6..6491e66d6 100644
--- a/contrib/torch/nn/lib/THNN/generic/SoftPlus.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/SoftPlus.c
diff --git a/contrib/torch/nn/lib/THNN/generic/SoftShrink.c b/contrib/lua-torch/nn/lib/THNN/generic/SoftShrink.c
index e77950868..e77950868 100644
--- a/contrib/torch/nn/lib/THNN/generic/SoftShrink.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/SoftShrink.c
diff --git a/contrib/torch/nn/lib/THNN/generic/SparseLinear.c b/contrib/lua-torch/nn/lib/THNN/generic/SparseLinear.c
index 1cf712212..1cf712212 100644
--- a/contrib/torch/nn/lib/THNN/generic/SparseLinear.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/SparseLinear.c
diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialAdaptiveAveragePooling.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialAdaptiveAveragePooling.c
index 3675b42d7..3675b42d7 100644
--- a/contrib/torch/nn/lib/THNN/generic/SpatialAdaptiveAveragePooling.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialAdaptiveAveragePooling.c
diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialAdaptiveMaxPooling.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialAdaptiveMaxPooling.c
index fff716e67..fff716e67 100644
--- a/contrib/torch/nn/lib/THNN/generic/SpatialAdaptiveMaxPooling.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialAdaptiveMaxPooling.c
diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialAveragePooling.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialAveragePooling.c
index c063502e7..c063502e7 100644
--- a/contrib/torch/nn/lib/THNN/generic/SpatialAveragePooling.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialAveragePooling.c
diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialClassNLLCriterion.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialClassNLLCriterion.c
index d711c8590..d711c8590 100644
--- a/contrib/torch/nn/lib/THNN/generic/SpatialClassNLLCriterion.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialClassNLLCriterion.c
diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialConvolutionLocal.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialConvolutionLocal.c
index 6db5a5db9..6db5a5db9 100644
--- a/contrib/torch/nn/lib/THNN/generic/SpatialConvolutionLocal.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialConvolutionLocal.c
diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialConvolutionMM.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialConvolutionMM.c
index 28fea517c..28fea517c 100644
--- a/contrib/torch/nn/lib/THNN/generic/SpatialConvolutionMM.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialConvolutionMM.c
diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialConvolutionMap.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialConvolutionMap.c
index 142a03551..142a03551 100644
--- a/contrib/torch/nn/lib/THNN/generic/SpatialConvolutionMap.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialConvolutionMap.c
diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialDepthWiseConvolution.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialDepthWiseConvolution.c
index efb66a3e3..efb66a3e3 100644
--- a/contrib/torch/nn/lib/THNN/generic/SpatialDepthWiseConvolution.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialDepthWiseConvolution.c
diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialDilatedConvolution.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialDilatedConvolution.c
index 897cc0da4..897cc0da4 100644
--- a/contrib/torch/nn/lib/THNN/generic/SpatialDilatedConvolution.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialDilatedConvolution.c
diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialDilatedMaxPooling.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialDilatedMaxPooling.c
index 8f4ad13c3..8f4ad13c3 100644
--- a/contrib/torch/nn/lib/THNN/generic/SpatialDilatedMaxPooling.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialDilatedMaxPooling.c
diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialFractionalMaxPooling.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialFractionalMaxPooling.c
index a98954cc6..a98954cc6 100644
--- a/contrib/torch/nn/lib/THNN/generic/SpatialFractionalMaxPooling.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialFractionalMaxPooling.c
diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialFullConvolution.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialFullConvolution.c
index 2edc53b5a..2edc53b5a 100644
--- a/contrib/torch/nn/lib/THNN/generic/SpatialFullConvolution.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialFullConvolution.c
diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialFullConvolutionMap.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialFullConvolutionMap.c
index 6952fbe25..6952fbe25 100644
--- a/contrib/torch/nn/lib/THNN/generic/SpatialFullConvolutionMap.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialFullConvolutionMap.c
diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialMaxPooling.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialMaxPooling.c
index 88aaa40e1..88aaa40e1 100644
--- a/contrib/torch/nn/lib/THNN/generic/SpatialMaxPooling.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialMaxPooling.c
diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialMaxUnpooling.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialMaxUnpooling.c
index 320538686..320538686 100644
--- a/contrib/torch/nn/lib/THNN/generic/SpatialMaxUnpooling.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialMaxUnpooling.c
diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialReflectionPadding.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialReflectionPadding.c
index dcde660ea..dcde660ea 100644
--- a/contrib/torch/nn/lib/THNN/generic/SpatialReflectionPadding.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialReflectionPadding.c
diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialReplicationPadding.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialReplicationPadding.c
index 4e318aa70..4e318aa70 100644
--- a/contrib/torch/nn/lib/THNN/generic/SpatialReplicationPadding.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialReplicationPadding.c
diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialSubSampling.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialSubSampling.c
index 4c077bc64..4c077bc64 100644
--- a/contrib/torch/nn/lib/THNN/generic/SpatialSubSampling.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialSubSampling.c
diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialUpSamplingBilinear.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialUpSamplingBilinear.c
index 8bc487ead..8bc487ead 100644
--- a/contrib/torch/nn/lib/THNN/generic/SpatialUpSamplingBilinear.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialUpSamplingBilinear.c
diff --git a/contrib/torch/nn/lib/THNN/generic/SpatialUpSamplingNearest.c b/contrib/lua-torch/nn/lib/THNN/generic/SpatialUpSamplingNearest.c
index b4699ff3e..b4699ff3e 100644
--- a/contrib/torch/nn/lib/THNN/generic/SpatialUpSamplingNearest.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/SpatialUpSamplingNearest.c
diff --git a/contrib/torch/nn/lib/THNN/generic/Sqrt.c b/contrib/lua-torch/nn/lib/THNN/generic/Sqrt.c
index 174884e34..174884e34 100644
--- a/contrib/torch/nn/lib/THNN/generic/Sqrt.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/Sqrt.c
diff --git a/contrib/torch/nn/lib/THNN/generic/Square.c b/contrib/lua-torch/nn/lib/THNN/generic/Square.c
index aad0a911c..aad0a911c 100644
--- a/contrib/torch/nn/lib/THNN/generic/Square.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/Square.c
diff --git a/contrib/torch/nn/lib/THNN/generic/THNN.h b/contrib/lua-torch/nn/lib/THNN/generic/THNN.h
index 76a28eb2d..76a28eb2d 100644
--- a/contrib/torch/nn/lib/THNN/generic/THNN.h
+++ b/contrib/lua-torch/nn/lib/THNN/generic/THNN.h
diff --git a/contrib/torch/nn/lib/THNN/generic/Tanh.c b/contrib/lua-torch/nn/lib/THNN/generic/Tanh.c
index ecf0708c2..ecf0708c2 100644
--- a/contrib/torch/nn/lib/THNN/generic/Tanh.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/Tanh.c
diff --git a/contrib/torch/nn/lib/THNN/generic/TemporalConvolution.c b/contrib/lua-torch/nn/lib/THNN/generic/TemporalConvolution.c
index 8cfd97d85..8cfd97d85 100644
--- a/contrib/torch/nn/lib/THNN/generic/TemporalConvolution.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/TemporalConvolution.c
diff --git a/contrib/torch/nn/lib/THNN/generic/TemporalMaxPooling.c b/contrib/lua-torch/nn/lib/THNN/generic/TemporalMaxPooling.c
index 344c1b3fd..344c1b3fd 100644
--- a/contrib/torch/nn/lib/THNN/generic/TemporalMaxPooling.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/TemporalMaxPooling.c
diff --git a/contrib/torch/nn/lib/THNN/generic/TemporalRowConvolution.c b/contrib/lua-torch/nn/lib/THNN/generic/TemporalRowConvolution.c
index e3ae41e22..e3ae41e22 100644
--- a/contrib/torch/nn/lib/THNN/generic/TemporalRowConvolution.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/TemporalRowConvolution.c
diff --git a/contrib/torch/nn/lib/THNN/generic/TemporalSubSampling.c b/contrib/lua-torch/nn/lib/THNN/generic/TemporalSubSampling.c
index 68f35e28a..68f35e28a 100644
--- a/contrib/torch/nn/lib/THNN/generic/TemporalSubSampling.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/TemporalSubSampling.c
diff --git a/contrib/torch/nn/lib/THNN/generic/Threshold.c b/contrib/lua-torch/nn/lib/THNN/generic/Threshold.c
index 949c7a07c..949c7a07c 100644
--- a/contrib/torch/nn/lib/THNN/generic/Threshold.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/Threshold.c
diff --git a/contrib/torch/nn/lib/THNN/generic/VolumetricAveragePooling.c b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricAveragePooling.c
index 91c870e6f..91c870e6f 100644
--- a/contrib/torch/nn/lib/THNN/generic/VolumetricAveragePooling.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricAveragePooling.c
diff --git a/contrib/torch/nn/lib/THNN/generic/VolumetricConvolution.c b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricConvolution.c
index be1aa82e6..be1aa82e6 100644
--- a/contrib/torch/nn/lib/THNN/generic/VolumetricConvolution.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricConvolution.c
diff --git a/contrib/torch/nn/lib/THNN/generic/VolumetricConvolutionMM.c b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricConvolutionMM.c
index 00a121db6..00a121db6 100644
--- a/contrib/torch/nn/lib/THNN/generic/VolumetricConvolutionMM.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricConvolutionMM.c
diff --git a/contrib/torch/nn/lib/THNN/generic/VolumetricDilatedConvolution.c b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricDilatedConvolution.c
index ca740f78e..ca740f78e 100644
--- a/contrib/torch/nn/lib/THNN/generic/VolumetricDilatedConvolution.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricDilatedConvolution.c
diff --git a/contrib/torch/nn/lib/THNN/generic/VolumetricDilatedMaxPooling.c b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricDilatedMaxPooling.c
index 66c0f9531..66c0f9531 100644
--- a/contrib/torch/nn/lib/THNN/generic/VolumetricDilatedMaxPooling.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricDilatedMaxPooling.c
diff --git a/contrib/torch/nn/lib/THNN/generic/VolumetricFractionalMaxPooling.c b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricFractionalMaxPooling.c
index 236986bb9..236986bb9 100644
--- a/contrib/torch/nn/lib/THNN/generic/VolumetricFractionalMaxPooling.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricFractionalMaxPooling.c
diff --git a/contrib/torch/nn/lib/THNN/generic/VolumetricFullConvolution.c b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricFullConvolution.c
index c974fab50..c974fab50 100644
--- a/contrib/torch/nn/lib/THNN/generic/VolumetricFullConvolution.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricFullConvolution.c
diff --git a/contrib/torch/nn/lib/THNN/generic/VolumetricMaxPooling.c b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricMaxPooling.c
index a3601e0b6..a3601e0b6 100644
--- a/contrib/torch/nn/lib/THNN/generic/VolumetricMaxPooling.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricMaxPooling.c
diff --git a/contrib/torch/nn/lib/THNN/generic/VolumetricMaxUnpooling.c b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricMaxUnpooling.c
index d9d9e5951..d9d9e5951 100644
--- a/contrib/torch/nn/lib/THNN/generic/VolumetricMaxUnpooling.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricMaxUnpooling.c
diff --git a/contrib/torch/nn/lib/THNN/generic/VolumetricReplicationPadding.c b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricReplicationPadding.c
index 4d8993ec2..4d8993ec2 100644
--- a/contrib/torch/nn/lib/THNN/generic/VolumetricReplicationPadding.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricReplicationPadding.c
diff --git a/contrib/torch/nn/lib/THNN/generic/VolumetricUpSamplingNearest.c b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricUpSamplingNearest.c
index 9068fb58d..9068fb58d 100644
--- a/contrib/torch/nn/lib/THNN/generic/VolumetricUpSamplingNearest.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricUpSamplingNearest.c
diff --git a/contrib/torch/nn/lib/THNN/generic/VolumetricUpSamplingTrilinear.c b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricUpSamplingTrilinear.c
index f2b04dba9..f2b04dba9 100644
--- a/contrib/torch/nn/lib/THNN/generic/VolumetricUpSamplingTrilinear.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/VolumetricUpSamplingTrilinear.c
diff --git a/contrib/torch/nn/lib/THNN/generic/unfold.c b/contrib/lua-torch/nn/lib/THNN/generic/unfold.c
index 14a73b567..14a73b567 100644
--- a/contrib/torch/nn/lib/THNN/generic/unfold.c
+++ b/contrib/lua-torch/nn/lib/THNN/generic/unfold.c
diff --git a/contrib/torch/nn/lib/THNN/init.c b/contrib/lua-torch/nn/lib/THNN/init.c
index 5c8c023dc..5c8c023dc 100644
--- a/contrib/torch/nn/lib/THNN/init.c
+++ b/contrib/lua-torch/nn/lib/THNN/init.c
diff --git a/contrib/torch/nn/mkdocs.yml b/contrib/lua-torch/nn/mkdocs.yml
index a37a34fb0..a37a34fb0 100644
--- a/contrib/torch/nn/mkdocs.yml
+++ b/contrib/lua-torch/nn/mkdocs.yml
diff --git a/contrib/torch/nn/test.lua b/contrib/lua-torch/nn/test.lua
index 4e3f627fc..4e3f627fc 100755
--- a/contrib/torch/nn/test.lua
+++ b/contrib/lua-torch/nn/test.lua
diff --git a/contrib/torch/nn/utils.lua b/contrib/lua-torch/nn/utils.lua
index 17b52afb3..17b52afb3 100644
--- a/contrib/torch/nn/utils.lua
+++ b/contrib/lua-torch/nn/utils.lua
diff --git a/contrib/torch/optim/CMakeLists.txt b/contrib/lua-torch/optim/CMakeLists.txt
index b1c13e701..b1c13e701 100644
--- a/contrib/torch/optim/CMakeLists.txt
+++ b/contrib/lua-torch/optim/CMakeLists.txt
diff --git a/contrib/torch/optim/COPYRIGHT.txt b/contrib/lua-torch/optim/COPYRIGHT.txt
index 2e4118c0f..2e4118c0f 100644
--- a/contrib/torch/optim/COPYRIGHT.txt
+++ b/contrib/lua-torch/optim/COPYRIGHT.txt
diff --git a/contrib/torch/optim/ConfusionMatrix.lua b/contrib/lua-torch/optim/ConfusionMatrix.lua
index ec5302c24..ec5302c24 100644
--- a/contrib/torch/optim/ConfusionMatrix.lua
+++ b/contrib/lua-torch/optim/ConfusionMatrix.lua
diff --git a/contrib/torch/optim/Logger.lua b/contrib/lua-torch/optim/Logger.lua
index 3c1da54d3..3c1da54d3 100644
--- a/contrib/torch/optim/Logger.lua
+++ b/contrib/lua-torch/optim/Logger.lua
diff --git a/contrib/torch/optim/adadelta.lua b/contrib/lua-torch/optim/adadelta.lua
index 7cc058d29..7cc058d29 100644
--- a/contrib/torch/optim/adadelta.lua
+++ b/contrib/lua-torch/optim/adadelta.lua
diff --git a/contrib/torch/optim/adagrad.lua b/contrib/lua-torch/optim/adagrad.lua
index 6860c4317..6860c4317 100644
--- a/contrib/torch/optim/adagrad.lua
+++ b/contrib/lua-torch/optim/adagrad.lua
diff --git a/contrib/torch/optim/adam.lua b/contrib/lua-torch/optim/adam.lua
index 2e127e96a..2e127e96a 100644
--- a/contrib/torch/optim/adam.lua
+++ b/contrib/lua-torch/optim/adam.lua
diff --git a/contrib/torch/optim/adamax.lua b/contrib/lua-torch/optim/adamax.lua
index 2b6487720..2b6487720 100644
--- a/contrib/torch/optim/adamax.lua
+++ b/contrib/lua-torch/optim/adamax.lua
diff --git a/contrib/torch/optim/asgd.lua b/contrib/lua-torch/optim/asgd.lua
index cc1c459f3..cc1c459f3 100644
--- a/contrib/torch/optim/asgd.lua
+++ b/contrib/lua-torch/optim/asgd.lua
diff --git a/contrib/torch/optim/cg.lua b/contrib/lua-torch/optim/cg.lua
index 842a7d569..842a7d569 100644
--- a/contrib/torch/optim/cg.lua
+++ b/contrib/lua-torch/optim/cg.lua
diff --git a/contrib/torch/optim/checkgrad.lua b/contrib/lua-torch/optim/checkgrad.lua
index 0382b2132..0382b2132 100644
--- a/contrib/torch/optim/checkgrad.lua
+++ b/contrib/lua-torch/optim/checkgrad.lua
diff --git a/contrib/torch/optim/cmaes.lua b/contrib/lua-torch/optim/cmaes.lua
index 74cd58a0c..74cd58a0c 100644
--- a/contrib/torch/optim/cmaes.lua
+++ b/contrib/lua-torch/optim/cmaes.lua
diff --git a/contrib/torch/optim/de.lua b/contrib/lua-torch/optim/de.lua
index 1e8e8001d..1e8e8001d 100644
--- a/contrib/torch/optim/de.lua
+++ b/contrib/lua-torch/optim/de.lua
diff --git a/contrib/torch/optim/fista.lua b/contrib/lua-torch/optim/fista.lua
index c8c6f5e43..c8c6f5e43 100644
--- a/contrib/torch/optim/fista.lua
+++ b/contrib/lua-torch/optim/fista.lua
diff --git a/contrib/torch/optim/init.lua b/contrib/lua-torch/optim/init.lua
index a045bd8a2..a045bd8a2 100644
--- a/contrib/torch/optim/init.lua
+++ b/contrib/lua-torch/optim/init.lua
diff --git a/contrib/torch/optim/lbfgs.lua b/contrib/lua-torch/optim/lbfgs.lua
index d850fcbb3..d850fcbb3 100644
--- a/contrib/torch/optim/lbfgs.lua
+++ b/contrib/lua-torch/optim/lbfgs.lua
diff --git a/contrib/torch/optim/lswolfe.lua b/contrib/lua-torch/optim/lswolfe.lua
index 0afbdbe8b..0afbdbe8b 100644
--- a/contrib/torch/optim/lswolfe.lua
+++ b/contrib/lua-torch/optim/lswolfe.lua
diff --git a/contrib/torch/optim/nag.lua b/contrib/lua-torch/optim/nag.lua
index 875d81e4c..875d81e4c 100644
--- a/contrib/torch/optim/nag.lua
+++ b/contrib/lua-torch/optim/nag.lua
diff --git a/contrib/torch/optim/polyinterp.lua b/contrib/lua-torch/optim/polyinterp.lua
index c5026bf04..c5026bf04 100644
--- a/contrib/torch/optim/polyinterp.lua
+++ b/contrib/lua-torch/optim/polyinterp.lua
diff --git a/contrib/torch/optim/rmsprop.lua b/contrib/lua-torch/optim/rmsprop.lua
index aa562006a..aa562006a 100644
--- a/contrib/torch/optim/rmsprop.lua
+++ b/contrib/lua-torch/optim/rmsprop.lua
diff --git a/contrib/torch/optim/rprop.lua b/contrib/lua-torch/optim/rprop.lua
index d7af16429..d7af16429 100644
--- a/contrib/torch/optim/rprop.lua
+++ b/contrib/lua-torch/optim/rprop.lua
diff --git a/contrib/torch/optim/sgd.lua b/contrib/lua-torch/optim/sgd.lua
index e21c696a6..e21c696a6 100644
--- a/contrib/torch/optim/sgd.lua
+++ b/contrib/lua-torch/optim/sgd.lua
diff --git a/contrib/torch/paths/CMakeLists.txt b/contrib/lua-torch/paths/CMakeLists.txt
index c578b66e9..c578b66e9 100644
--- a/contrib/torch/paths/CMakeLists.txt
+++ b/contrib/lua-torch/paths/CMakeLists.txt
diff --git a/contrib/torch/paths/COPYRIGHT.txt b/contrib/lua-torch/paths/COPYRIGHT.txt
index bc002b78a..bc002b78a 100644
--- a/contrib/torch/paths/COPYRIGHT.txt
+++ b/contrib/lua-torch/paths/COPYRIGHT.txt
diff --git a/contrib/torch/paths/README.md b/contrib/lua-torch/paths/README.md
index 2464c090e..2464c090e 100644
--- a/contrib/torch/paths/README.md
+++ b/contrib/lua-torch/paths/README.md
diff --git a/contrib/torch/paths/init.lua b/contrib/lua-torch/paths/init.lua
index 9078c3523..9078c3523 100644
--- a/contrib/torch/paths/init.lua
+++ b/contrib/lua-torch/paths/init.lua
diff --git a/contrib/torch/paths/mkdocs.yml b/contrib/lua-torch/paths/mkdocs.yml
index 1fe86d1b0..1fe86d1b0 100644
--- a/contrib/torch/paths/mkdocs.yml
+++ b/contrib/lua-torch/paths/mkdocs.yml
diff --git a/contrib/torch/paths/paths.c b/contrib/lua-torch/paths/paths.c
index 7c3a72e54..7c3a72e54 100644
--- a/contrib/torch/paths/paths.c
+++ b/contrib/lua-torch/paths/paths.c
diff --git a/contrib/torch/paths/paths.h.in b/contrib/lua-torch/paths/paths.h.in
index fb5417a6c..fb5417a6c 100644
--- a/contrib/torch/paths/paths.h.in
+++ b/contrib/lua-torch/paths/paths.h.in
diff --git a/contrib/torch/torch7/CMakeLists.txt b/contrib/lua-torch/torch7/CMakeLists.txt
index 7519e85d7..1c3995dbe 100644
--- a/contrib/torch/torch7/CMakeLists.txt
+++ b/contrib/lua-torch/torch7/CMakeLists.txt
@@ -1,14 +1,4 @@
-IF(APPLE)
- CMAKE_MINIMUM_REQUIRED(VERSION 2.8.12 FATAL_ERROR)
- CMAKE_POLICY(VERSION 2.8.12)
-ELSE()
- CMAKE_MINIMUM_REQUIRED(VERSION 2.8 FATAL_ERROR)
- CMAKE_POLICY(VERSION 2.8)
-ENDIF()
-
-SET(CMAKE_MODULE_PATH
- "${CMAKE_CURRENT_SOURCE_DIR}/cmake"
- "${CMAKE_MODULE_PATH}")
+LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
IF (NOT MSVC)
IF (MINGW)
@@ -18,14 +8,6 @@ IF (NOT MSVC)
ENDIF(MINGW)
ENDIF(NOT MSVC)
-# Flags
-# When using MSVC
-IF(MSVC)
- # we want to respect the standard, and we are bored of those **** .
- ADD_DEFINITIONS(-D_CRT_SECURE_NO_DEPRECATE=1)
-ENDIF(MSVC)
-
-
IF (WITH_OPENMP)
FIND_PACKAGE(OpenMP)
IF(OPENMP_FOUND)
diff --git a/contrib/torch/torch7/COPYRIGHT.txt b/contrib/lua-torch/torch7/COPYRIGHT.txt
index bc002b78a..bc002b78a 100644
--- a/contrib/torch/torch7/COPYRIGHT.txt
+++ b/contrib/lua-torch/torch7/COPYRIGHT.txt
diff --git a/contrib/torch/torch7/CmdLine.lua b/contrib/lua-torch/torch7/CmdLine.lua
index 643635069..643635069 100644
--- a/contrib/torch/torch7/CmdLine.lua
+++ b/contrib/lua-torch/torch7/CmdLine.lua
diff --git a/contrib/torch/torch7/DiskFile.c b/contrib/lua-torch/torch7/DiskFile.c
index c50b74f95..c50b74f95 100644
--- a/contrib/torch/torch7/DiskFile.c
+++ b/contrib/lua-torch/torch7/DiskFile.c
diff --git a/contrib/torch/torch7/FFInterface.lua b/contrib/lua-torch/torch7/FFInterface.lua
index cb8bd3365..cb8bd3365 100644
--- a/contrib/torch/torch7/FFInterface.lua
+++ b/contrib/lua-torch/torch7/FFInterface.lua
diff --git a/contrib/torch/torch7/File.c b/contrib/lua-torch/torch7/File.c
index e07bc4666..e07bc4666 100644
--- a/contrib/torch/torch7/File.c
+++ b/contrib/lua-torch/torch7/File.c
diff --git a/contrib/torch/torch7/File.lua b/contrib/lua-torch/torch7/File.lua
index 62249a361..62249a361 100644
--- a/contrib/torch/torch7/File.lua
+++ b/contrib/lua-torch/torch7/File.lua
diff --git a/contrib/torch/torch7/Generator.c b/contrib/lua-torch/torch7/Generator.c
index 8cf5ba66c..8cf5ba66c 100644
--- a/contrib/torch/torch7/Generator.c
+++ b/contrib/lua-torch/torch7/Generator.c
diff --git a/contrib/torch/torch7/MemoryFile.c b/contrib/lua-torch/torch7/MemoryFile.c
index a22dc17ec..a22dc17ec 100644
--- a/contrib/torch/torch7/MemoryFile.c
+++ b/contrib/lua-torch/torch7/MemoryFile.c
diff --git a/contrib/torch/torch7/PipeFile.c b/contrib/lua-torch/torch7/PipeFile.c
index a47c90d13..a47c90d13 100644
--- a/contrib/torch/torch7/PipeFile.c
+++ b/contrib/lua-torch/torch7/PipeFile.c
diff --git a/contrib/torch/torch7/README.md b/contrib/lua-torch/torch7/README.md
index 1a51c314c..1a51c314c 100644
--- a/contrib/torch/torch7/README.md
+++ b/contrib/lua-torch/torch7/README.md
diff --git a/contrib/torch/torch7/ROADMAP.md b/contrib/lua-torch/torch7/ROADMAP.md
index 01bc8b8ac..01bc8b8ac 100644
--- a/contrib/torch/torch7/ROADMAP.md
+++ b/contrib/lua-torch/torch7/ROADMAP.md
diff --git a/contrib/torch/torch7/Storage.c b/contrib/lua-torch/torch7/Storage.c
index 730d97476..730d97476 100644
--- a/contrib/torch/torch7/Storage.c
+++ b/contrib/lua-torch/torch7/Storage.c
diff --git a/contrib/torch/torch7/Tensor.c b/contrib/lua-torch/torch7/Tensor.c
index bf78d1aae..bf78d1aae 100644
--- a/contrib/torch/torch7/Tensor.c
+++ b/contrib/lua-torch/torch7/Tensor.c
diff --git a/contrib/torch/torch7/Tensor.lua b/contrib/lua-torch/torch7/Tensor.lua
index 9a8215be1..9a8215be1 100644
--- a/contrib/torch/torch7/Tensor.lua
+++ b/contrib/lua-torch/torch7/Tensor.lua
diff --git a/contrib/torch/torch7/TensorMath.c b/contrib/lua-torch/torch7/TensorMath.c
index 84a821812..84a821812 100644
--- a/contrib/torch/torch7/TensorMath.c
+++ b/contrib/lua-torch/torch7/TensorMath.c
diff --git a/contrib/torch/torch7/TensorMath.lua b/contrib/lua-torch/torch7/TensorMath.lua
index 45e07c63e..45e07c63e 100644
--- a/contrib/torch/torch7/TensorMath.lua
+++ b/contrib/lua-torch/torch7/TensorMath.lua
diff --git a/contrib/torch/torch7/TensorOperator.c b/contrib/lua-torch/torch7/TensorOperator.c
index 8986ff761..8986ff761 100644
--- a/contrib/torch/torch7/TensorOperator.c
+++ b/contrib/lua-torch/torch7/TensorOperator.c
diff --git a/contrib/torch/torch7/TestSuite.lua b/contrib/lua-torch/torch7/TestSuite.lua
index 630c2c948..630c2c948 100644
--- a/contrib/torch/torch7/TestSuite.lua
+++ b/contrib/lua-torch/torch7/TestSuite.lua
diff --git a/contrib/torch/torch7/Tester.lua b/contrib/lua-torch/torch7/Tester.lua
index 6509413c2..6509413c2 100644
--- a/contrib/torch/torch7/Tester.lua
+++ b/contrib/lua-torch/torch7/Tester.lua
diff --git a/contrib/torch/torch7/Timer.c b/contrib/lua-torch/torch7/Timer.c
index 13865b590..13865b590 100644
--- a/contrib/torch/torch7/Timer.c
+++ b/contrib/lua-torch/torch7/Timer.c
diff --git a/contrib/torch/torch7/cmake/TorchConfig.cmake.in b/contrib/lua-torch/torch7/cmake/TorchConfig.cmake.in
index 3d85eb117..bafa04f91 100644
--- a/contrib/torch/torch7/cmake/TorchConfig.cmake.in
+++ b/contrib/lua-torch/torch7/cmake/TorchConfig.cmake.in
@@ -18,7 +18,7 @@ SET(Torch_INSTALL_CMAKE_RIDBUS "@Torch_INSTALL_CMAKE_RIDBUS@")
FILE(RELATIVE_PATH Torch_INSTALL_LUA_PATH_SUBDIR "${Torch_INSTALL_PREFIX}" "${CMAKE_INSTALL_PREFIX}/lua")
FILE(RELATIVE_PATH Torch_INSTALL_LUA_CPATH_SUBDIR "${Torch_INSTALL_PREFIX}" "${CMAKE_INSTALL_PREFIX}/lib")
-SET(CMAKE_MODULE_PATH "${Torch_INSTALL_PREFIX}/${Torch_INSTALL_CMAKE_SUBDIR}" "${CMAKE_MODULE_PATH}")
+LIST(APPEND CMAKE_MODULE_PATH "${Torch_INSTALL_PREFIX}/${Torch_INSTALL_CMAKE_SUBDIR}")
SET(CMAKE_INSTALL_PREFIX "${Torch_INSTALL_PREFIX}") # override
INCLUDE(TorchPathsInit)
diff --git a/contrib/torch/torch7/cmake/TorchExports.cmake b/contrib/lua-torch/torch7/cmake/TorchExports.cmake
index f47fd6f59..f47fd6f59 100644
--- a/contrib/torch/torch7/cmake/TorchExports.cmake
+++ b/contrib/lua-torch/torch7/cmake/TorchExports.cmake
diff --git a/contrib/torch/torch7/cmake/TorchPackage.cmake b/contrib/lua-torch/torch7/cmake/TorchPackage.cmake
index 9ffdcf788..846b263a4 100644
--- a/contrib/torch/torch7/cmake/TorchPackage.cmake
+++ b/contrib/lua-torch/torch7/cmake/TorchPackage.cmake
@@ -19,10 +19,10 @@ ENDMACRO()
MACRO(ADD_TORCH_PACKAGE package src luasrc)
INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR})
- INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/contrib/torch/torch7/lib/TH)
- INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/contrib/torch/torch7/lib/luaT)
- INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/contrib/torch/torch7/lib/TH)
- INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/contrib/torch/torch7/lib/luaT)
+ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/contrib/lua-torch/torch7/lib/TH)
+ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/contrib/lua-torch/torch7/lib/luaT)
+ INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/contrib/lua-torch/torch7/lib/TH)
+ INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/contrib/lua-torch/torch7/lib/luaT)
INCLUDE_DIRECTORIES(${Torch_LUA_INCLUDE_DIR})
### C/C++ sources
diff --git a/contrib/torch/torch7/cmake/TorchPaths.cmake b/contrib/lua-torch/torch7/cmake/TorchPaths.cmake
index 70c7a4fb8..70c7a4fb8 100644
--- a/contrib/torch/torch7/cmake/TorchPaths.cmake
+++ b/contrib/lua-torch/torch7/cmake/TorchPaths.cmake
diff --git a/contrib/torch/torch7/cmake/TorchPathsInit.cmake b/contrib/lua-torch/torch7/cmake/TorchPathsInit.cmake
index 42f8ffb3f..42f8ffb3f 100644
--- a/contrib/torch/torch7/cmake/TorchPathsInit.cmake
+++ b/contrib/lua-torch/torch7/cmake/TorchPathsInit.cmake
diff --git a/contrib/torch/torch7/cmake/TorchWrap.cmake b/contrib/lua-torch/torch7/cmake/TorchWrap.cmake
index b367b2402..b367b2402 100644
--- a/contrib/torch/torch7/cmake/TorchWrap.cmake
+++ b/contrib/lua-torch/torch7/cmake/TorchWrap.cmake
diff --git a/contrib/torch/torch7/cmake/TorchWrap.cmake.in b/contrib/lua-torch/torch7/cmake/TorchWrap.cmake.in
index 5c20445d1..5c20445d1 100644
--- a/contrib/torch/torch7/cmake/TorchWrap.cmake.in
+++ b/contrib/lua-torch/torch7/cmake/TorchWrap.cmake.in
diff --git a/contrib/torch/torch7/general.h b/contrib/lua-torch/torch7/general.h
index 3ccf4bdf0..3ccf4bdf0 100644
--- a/contrib/torch/torch7/general.h
+++ b/contrib/lua-torch/torch7/general.h
diff --git a/contrib/torch/torch7/generic/Storage.c b/contrib/lua-torch/torch7/generic/Storage.c
index b936e5714..b936e5714 100644
--- a/contrib/torch/torch7/generic/Storage.c
+++ b/contrib/lua-torch/torch7/generic/Storage.c
diff --git a/contrib/torch/torch7/generic/Tensor.c b/contrib/lua-torch/torch7/generic/Tensor.c
index 112a4bd63..112a4bd63 100644
--- a/contrib/torch/torch7/generic/Tensor.c
+++ b/contrib/lua-torch/torch7/generic/Tensor.c
diff --git a/contrib/torch/torch7/generic/TensorOperator.c b/contrib/lua-torch/torch7/generic/TensorOperator.c
index 37b2a0889..37b2a0889 100644
--- a/contrib/torch/torch7/generic/TensorOperator.c
+++ b/contrib/lua-torch/torch7/generic/TensorOperator.c
diff --git a/contrib/torch/torch7/generic/luaG.h b/contrib/lua-torch/torch7/generic/luaG.h
index f1ffce29e..f1ffce29e 100644
--- a/contrib/torch/torch7/generic/luaG.h
+++ b/contrib/lua-torch/torch7/generic/luaG.h
diff --git a/contrib/torch/torch7/init.c b/contrib/lua-torch/torch7/init.c
index 3bdac176d..3bdac176d 100644
--- a/contrib/torch/torch7/init.c
+++ b/contrib/lua-torch/torch7/init.c
diff --git a/contrib/torch/torch7/init.lua b/contrib/lua-torch/torch7/init.lua
index 0f3cfbb1e..0f3cfbb1e 100644
--- a/contrib/torch/torch7/init.lua
+++ b/contrib/lua-torch/torch7/init.lua
diff --git a/contrib/torch/torch7/lib/CMakeLists.txt b/contrib/lua-torch/torch7/lib/CMakeLists.txt
index d6a0e2c9c..d6a0e2c9c 100644
--- a/contrib/torch/torch7/lib/CMakeLists.txt
+++ b/contrib/lua-torch/torch7/lib/CMakeLists.txt
diff --git a/contrib/torch/torch7/lib/TH/CMakeLists.txt b/contrib/lua-torch/torch7/lib/TH/CMakeLists.txt
index ef5bdd08c..b4f115f39 100644
--- a/contrib/torch/torch7/lib/TH/CMakeLists.txt
+++ b/contrib/lua-torch/torch7/lib/TH/CMakeLists.txt
@@ -5,7 +5,7 @@ IF(POLICY CMP0026)
CMAKE_POLICY(SET CMP0026 OLD)
ENDIF()
-SET(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH})
+LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
SET(CMAKE_LIBRARY_PATH /usr/lib/x86_64-linux-gnu/ ${CMAKE_LIBRARY_PATH})
#######################################################################
diff --git a/contrib/torch/torch7/lib/TH/README.md b/contrib/lua-torch/torch7/lib/TH/README.md
index 4ac26c103..4ac26c103 100644
--- a/contrib/torch/torch7/lib/TH/README.md
+++ b/contrib/lua-torch/torch7/lib/TH/README.md
diff --git a/contrib/torch/torch7/lib/TH/TH.h b/contrib/lua-torch/torch7/lib/TH/TH.h
index 11f208c4b..11f208c4b 100644
--- a/contrib/torch/torch7/lib/TH/TH.h
+++ b/contrib/lua-torch/torch7/lib/TH/TH.h
diff --git a/contrib/torch/torch7/lib/TH/THAllocator.c b/contrib/lua-torch/torch7/lib/TH/THAllocator.c
index 51ac69b94..51ac69b94 100644
--- a/contrib/torch/torch7/lib/TH/THAllocator.c
+++ b/contrib/lua-torch/torch7/lib/TH/THAllocator.c
diff --git a/contrib/torch/torch7/lib/TH/THAllocator.h b/contrib/lua-torch/torch7/lib/TH/THAllocator.h
index 18fc9ec0a..18fc9ec0a 100644
--- a/contrib/torch/torch7/lib/TH/THAllocator.h
+++ b/contrib/lua-torch/torch7/lib/TH/THAllocator.h
diff --git a/contrib/torch/torch7/lib/TH/THAtomic.c b/contrib/lua-torch/torch7/lib/TH/THAtomic.c
index 714fc52db..714fc52db 100644
--- a/contrib/torch/torch7/lib/TH/THAtomic.c
+++ b/contrib/lua-torch/torch7/lib/TH/THAtomic.c
diff --git a/contrib/torch/torch7/lib/TH/THAtomic.h b/contrib/lua-torch/torch7/lib/TH/THAtomic.h
index d77b20b24..d77b20b24 100644
--- a/contrib/torch/torch7/lib/TH/THAtomic.h
+++ b/contrib/lua-torch/torch7/lib/TH/THAtomic.h
diff --git a/contrib/torch/torch7/lib/TH/THBlas.c b/contrib/lua-torch/torch7/lib/TH/THBlas.c
index 35618b26a..35618b26a 100644
--- a/contrib/torch/torch7/lib/TH/THBlas.c
+++ b/contrib/lua-torch/torch7/lib/TH/THBlas.c
diff --git a/contrib/torch/torch7/lib/TH/THBlas.h b/contrib/lua-torch/torch7/lib/TH/THBlas.h
index 5fef0febc..5fef0febc 100644
--- a/contrib/torch/torch7/lib/TH/THBlas.h
+++ b/contrib/lua-torch/torch7/lib/TH/THBlas.h
diff --git a/contrib/torch/torch7/lib/TH/THConfig.cmake.in b/contrib/lua-torch/torch7/lib/TH/THConfig.cmake.in
index 306cd878b..306cd878b 100644
--- a/contrib/torch/torch7/lib/TH/THConfig.cmake.in
+++ b/contrib/lua-torch/torch7/lib/TH/THConfig.cmake.in
diff --git a/contrib/torch/torch7/lib/TH/THDiskFile.c b/contrib/lua-torch/torch7/lib/TH/THDiskFile.c
index 3f57b3b35..3f57b3b35 100644
--- a/contrib/torch/torch7/lib/TH/THDiskFile.c
+++ b/contrib/lua-torch/torch7/lib/TH/THDiskFile.c
diff --git a/contrib/torch/torch7/lib/TH/THDiskFile.h b/contrib/lua-torch/torch7/lib/TH/THDiskFile.h
index bc5c001c7..bc5c001c7 100644
--- a/contrib/torch/torch7/lib/TH/THDiskFile.h
+++ b/contrib/lua-torch/torch7/lib/TH/THDiskFile.h
diff --git a/contrib/torch/torch7/lib/TH/THFile.c b/contrib/lua-torch/torch7/lib/TH/THFile.c
index 3717b7b5c..3717b7b5c 100644
--- a/contrib/torch/torch7/lib/TH/THFile.c
+++ b/contrib/lua-torch/torch7/lib/TH/THFile.c
diff --git a/contrib/torch/torch7/lib/TH/THFile.h b/contrib/lua-torch/torch7/lib/TH/THFile.h
index e097bdf34..e097bdf34 100644
--- a/contrib/torch/torch7/lib/TH/THFile.h
+++ b/contrib/lua-torch/torch7/lib/TH/THFile.h
diff --git a/contrib/torch/torch7/lib/TH/THFilePrivate.h b/contrib/lua-torch/torch7/lib/TH/THFilePrivate.h
index 55169c3bc..55169c3bc 100644
--- a/contrib/torch/torch7/lib/TH/THFilePrivate.h
+++ b/contrib/lua-torch/torch7/lib/TH/THFilePrivate.h
diff --git a/contrib/torch/torch7/lib/TH/THGeneral.c b/contrib/lua-torch/torch7/lib/TH/THGeneral.c
index 6ad96b51c..6ad96b51c 100644
--- a/contrib/torch/torch7/lib/TH/THGeneral.c
+++ b/contrib/lua-torch/torch7/lib/TH/THGeneral.c
diff --git a/contrib/torch/torch7/lib/TH/THGeneral.h.in b/contrib/lua-torch/torch7/lib/TH/THGeneral.h.in
index 88a3934c8..88a3934c8 100644
--- a/contrib/torch/torch7/lib/TH/THGeneral.h.in
+++ b/contrib/lua-torch/torch7/lib/TH/THGeneral.h.in
diff --git a/contrib/torch/torch7/lib/TH/THGenerateAllTypes.h b/contrib/lua-torch/torch7/lib/TH/THGenerateAllTypes.h
index 5b9508df7..5b9508df7 100644
--- a/contrib/torch/torch7/lib/TH/THGenerateAllTypes.h
+++ b/contrib/lua-torch/torch7/lib/TH/THGenerateAllTypes.h
diff --git a/contrib/torch/torch7/lib/TH/THGenerateByteType.h b/contrib/lua-torch/torch7/lib/TH/THGenerateByteType.h
index 71ce7c405..71ce7c405 100644
--- a/contrib/torch/torch7/lib/TH/THGenerateByteType.h
+++ b/contrib/lua-torch/torch7/lib/TH/THGenerateByteType.h
diff --git a/contrib/torch/torch7/lib/TH/THGenerateCharType.h b/contrib/lua-torch/torch7/lib/TH/THGenerateCharType.h
index 158dd0e80..158dd0e80 100644
--- a/contrib/torch/torch7/lib/TH/THGenerateCharType.h
+++ b/contrib/lua-torch/torch7/lib/TH/THGenerateCharType.h
diff --git a/contrib/torch/torch7/lib/TH/THGenerateDoubleType.h b/contrib/lua-torch/torch7/lib/TH/THGenerateDoubleType.h
index fffee606d..fffee606d 100644
--- a/contrib/torch/torch7/lib/TH/THGenerateDoubleType.h
+++ b/contrib/lua-torch/torch7/lib/TH/THGenerateDoubleType.h
diff --git a/contrib/torch/torch7/lib/TH/THGenerateFloatType.h b/contrib/lua-torch/torch7/lib/TH/THGenerateFloatType.h
index a31b50c55..a31b50c55 100644
--- a/contrib/torch/torch7/lib/TH/THGenerateFloatType.h
+++ b/contrib/lua-torch/torch7/lib/TH/THGenerateFloatType.h
diff --git a/contrib/torch/torch7/lib/TH/THGenerateFloatTypes.h b/contrib/lua-torch/torch7/lib/TH/THGenerateFloatTypes.h
index be5ea8403..be5ea8403 100644
--- a/contrib/torch/torch7/lib/TH/THGenerateFloatTypes.h
+++ b/contrib/lua-torch/torch7/lib/TH/THGenerateFloatTypes.h
diff --git a/contrib/torch/torch7/lib/TH/THGenerateHalfType.h b/contrib/lua-torch/torch7/lib/TH/THGenerateHalfType.h
index 47ff1e8d7..47ff1e8d7 100644
--- a/contrib/torch/torch7/lib/TH/THGenerateHalfType.h
+++ b/contrib/lua-torch/torch7/lib/TH/THGenerateHalfType.h
diff --git a/contrib/torch/torch7/lib/TH/THGenerateIntType.h b/contrib/lua-torch/torch7/lib/TH/THGenerateIntType.h
index 1562b9e98..1562b9e98 100644
--- a/contrib/torch/torch7/lib/TH/THGenerateIntType.h
+++ b/contrib/lua-torch/torch7/lib/TH/THGenerateIntType.h
diff --git a/contrib/torch/torch7/lib/TH/THGenerateIntTypes.h b/contrib/lua-torch/torch7/lib/TH/THGenerateIntTypes.h
index 9931fb1f5..9931fb1f5 100644
--- a/contrib/torch/torch7/lib/TH/THGenerateIntTypes.h
+++ b/contrib/lua-torch/torch7/lib/TH/THGenerateIntTypes.h
diff --git a/contrib/torch/torch7/lib/TH/THGenerateLongType.h b/contrib/lua-torch/torch7/lib/TH/THGenerateLongType.h
index 75f90e1a6..75f90e1a6 100644
--- a/contrib/torch/torch7/lib/TH/THGenerateLongType.h
+++ b/contrib/lua-torch/torch7/lib/TH/THGenerateLongType.h
diff --git a/contrib/torch/torch7/lib/TH/THGenerateShortType.h b/contrib/lua-torch/torch7/lib/TH/THGenerateShortType.h
index 047e51a8d..047e51a8d 100644
--- a/contrib/torch/torch7/lib/TH/THGenerateShortType.h
+++ b/contrib/lua-torch/torch7/lib/TH/THGenerateShortType.h
diff --git a/contrib/torch/torch7/lib/TH/THHalf.c b/contrib/lua-torch/torch7/lib/TH/THHalf.c
index d7468ac3d..d7468ac3d 100644
--- a/contrib/torch/torch7/lib/TH/THHalf.c
+++ b/contrib/lua-torch/torch7/lib/TH/THHalf.c
diff --git a/contrib/torch/torch7/lib/TH/THHalf.h b/contrib/lua-torch/torch7/lib/TH/THHalf.h
index 0f9807b50..0f9807b50 100644
--- a/contrib/torch/torch7/lib/TH/THHalf.h
+++ b/contrib/lua-torch/torch7/lib/TH/THHalf.h
diff --git a/contrib/torch/torch7/lib/TH/THLapack.c b/contrib/lua-torch/torch7/lib/TH/THLapack.c
index bd4dc716b..bd4dc716b 100644
--- a/contrib/torch/torch7/lib/TH/THLapack.c
+++ b/contrib/lua-torch/torch7/lib/TH/THLapack.c
diff --git a/contrib/torch/torch7/lib/TH/THLapack.h b/contrib/lua-torch/torch7/lib/TH/THLapack.h
index 614d15f94..614d15f94 100644
--- a/contrib/torch/torch7/lib/TH/THLapack.h
+++ b/contrib/lua-torch/torch7/lib/TH/THLapack.h
diff --git a/contrib/torch/torch7/lib/TH/THLogAdd.c b/contrib/lua-torch/torch7/lib/TH/THLogAdd.c
index 4b14f8540..4b14f8540 100644
--- a/contrib/torch/torch7/lib/TH/THLogAdd.c
+++ b/contrib/lua-torch/torch7/lib/TH/THLogAdd.c
diff --git a/contrib/torch/torch7/lib/TH/THLogAdd.h b/contrib/lua-torch/torch7/lib/TH/THLogAdd.h
index 9319b8f46..9319b8f46 100644
--- a/contrib/torch/torch7/lib/TH/THLogAdd.h
+++ b/contrib/lua-torch/torch7/lib/TH/THLogAdd.h
diff --git a/contrib/torch/torch7/lib/TH/THMath.h b/contrib/lua-torch/torch7/lib/TH/THMath.h
index 004e4fe45..004e4fe45 100644
--- a/contrib/torch/torch7/lib/TH/THMath.h
+++ b/contrib/lua-torch/torch7/lib/TH/THMath.h
diff --git a/contrib/torch/torch7/lib/TH/THMemoryFile.c b/contrib/lua-torch/torch7/lib/TH/THMemoryFile.c
index ecce6e1b1..ecce6e1b1 100644
--- a/contrib/torch/torch7/lib/TH/THMemoryFile.c
+++ b/contrib/lua-torch/torch7/lib/TH/THMemoryFile.c
diff --git a/contrib/torch/torch7/lib/TH/THMemoryFile.h b/contrib/lua-torch/torch7/lib/TH/THMemoryFile.h
index b54cdcc2f..b54cdcc2f 100644
--- a/contrib/torch/torch7/lib/TH/THMemoryFile.h
+++ b/contrib/lua-torch/torch7/lib/TH/THMemoryFile.h
diff --git a/contrib/torch/torch7/lib/TH/THRandom.c b/contrib/lua-torch/torch7/lib/TH/THRandom.c
index 86d721e7b..86d721e7b 100644
--- a/contrib/torch/torch7/lib/TH/THRandom.c
+++ b/contrib/lua-torch/torch7/lib/TH/THRandom.c
diff --git a/contrib/torch/torch7/lib/TH/THRandom.h b/contrib/lua-torch/torch7/lib/TH/THRandom.h
index 28a14c0d7..28a14c0d7 100644
--- a/contrib/torch/torch7/lib/TH/THRandom.h
+++ b/contrib/lua-torch/torch7/lib/TH/THRandom.h
diff --git a/contrib/torch/torch7/lib/TH/THSize.c b/contrib/lua-torch/torch7/lib/TH/THSize.c
index ccf1f61dd..ccf1f61dd 100644
--- a/contrib/torch/torch7/lib/TH/THSize.c
+++ b/contrib/lua-torch/torch7/lib/TH/THSize.c
diff --git a/contrib/torch/torch7/lib/TH/THSize.h b/contrib/lua-torch/torch7/lib/TH/THSize.h
index 3d39696f6..3d39696f6 100644
--- a/contrib/torch/torch7/lib/TH/THSize.h
+++ b/contrib/lua-torch/torch7/lib/TH/THSize.h
diff --git a/contrib/torch/torch7/lib/TH/THStorage.c b/contrib/lua-torch/torch7/lib/TH/THStorage.c
index f6b63f4a8..f6b63f4a8 100644
--- a/contrib/torch/torch7/lib/TH/THStorage.c
+++ b/contrib/lua-torch/torch7/lib/TH/THStorage.c
diff --git a/contrib/torch/torch7/lib/TH/THStorage.h b/contrib/lua-torch/torch7/lib/TH/THStorage.h
index fb7946bd9..fb7946bd9 100644
--- a/contrib/torch/torch7/lib/TH/THStorage.h
+++ b/contrib/lua-torch/torch7/lib/TH/THStorage.h
diff --git a/contrib/torch/torch7/lib/TH/THTensor.c b/contrib/lua-torch/torch7/lib/TH/THTensor.c
index 115e396a1..115e396a1 100644
--- a/contrib/torch/torch7/lib/TH/THTensor.c
+++ b/contrib/lua-torch/torch7/lib/TH/THTensor.c
diff --git a/contrib/torch/torch7/lib/TH/THTensor.h b/contrib/lua-torch/torch7/lib/TH/THTensor.h
index d2a1c57e8..d2a1c57e8 100644
--- a/contrib/torch/torch7/lib/TH/THTensor.h
+++ b/contrib/lua-torch/torch7/lib/TH/THTensor.h
diff --git a/contrib/torch/torch7/lib/TH/THTensorApply.h b/contrib/lua-torch/torch7/lib/TH/THTensorApply.h
index 7f48da47e..7f48da47e 100644
--- a/contrib/torch/torch7/lib/TH/THTensorApply.h
+++ b/contrib/lua-torch/torch7/lib/TH/THTensorApply.h
diff --git a/contrib/torch/torch7/lib/TH/THTensorDimApply.h b/contrib/lua-torch/torch7/lib/TH/THTensorDimApply.h
index 6727e1f7f..6727e1f7f 100644
--- a/contrib/torch/torch7/lib/TH/THTensorDimApply.h
+++ b/contrib/lua-torch/torch7/lib/TH/THTensorDimApply.h
diff --git a/contrib/torch/torch7/lib/TH/THTensorMacros.h b/contrib/lua-torch/torch7/lib/TH/THTensorMacros.h
index 15b67665e..15b67665e 100644
--- a/contrib/torch/torch7/lib/TH/THTensorMacros.h
+++ b/contrib/lua-torch/torch7/lib/TH/THTensorMacros.h
diff --git a/contrib/torch/torch7/lib/TH/THVector.c b/contrib/lua-torch/torch7/lib/TH/THVector.c
index 441057884..441057884 100644
--- a/contrib/torch/torch7/lib/TH/THVector.c
+++ b/contrib/lua-torch/torch7/lib/TH/THVector.c
diff --git a/contrib/torch/torch7/lib/TH/THVector.h b/contrib/lua-torch/torch7/lib/TH/THVector.h
index e29917b93..e29917b93 100644
--- a/contrib/torch/torch7/lib/TH/THVector.h
+++ b/contrib/lua-torch/torch7/lib/TH/THVector.h
diff --git a/contrib/torch/torch7/lib/TH/cmake/FindARM.cmake b/contrib/lua-torch/torch7/lib/TH/cmake/FindARM.cmake
index 2dcb2a24f..2dcb2a24f 100644
--- a/contrib/torch/torch7/lib/TH/cmake/FindARM.cmake
+++ b/contrib/lua-torch/torch7/lib/TH/cmake/FindARM.cmake
diff --git a/contrib/torch/torch7/lib/TH/cmake/FindBLAS.cmake b/contrib/lua-torch/torch7/lib/TH/cmake/FindBLAS.cmake
index 1f254d231..1f254d231 100644
--- a/contrib/torch/torch7/lib/TH/cmake/FindBLAS.cmake
+++ b/contrib/lua-torch/torch7/lib/TH/cmake/FindBLAS.cmake
diff --git a/contrib/torch/torch7/lib/TH/cmake/FindLAPACK.cmake b/contrib/lua-torch/torch7/lib/TH/cmake/FindLAPACK.cmake
index 9eca0730f..9eca0730f 100644
--- a/contrib/torch/torch7/lib/TH/cmake/FindLAPACK.cmake
+++ b/contrib/lua-torch/torch7/lib/TH/cmake/FindLAPACK.cmake
diff --git a/contrib/torch/torch7/lib/TH/cmake/FindMKL.cmake b/contrib/lua-torch/torch7/lib/TH/cmake/FindMKL.cmake
index 08b450985..08b450985 100644
--- a/contrib/torch/torch7/lib/TH/cmake/FindMKL.cmake
+++ b/contrib/lua-torch/torch7/lib/TH/cmake/FindMKL.cmake
diff --git a/contrib/torch/torch7/lib/TH/cmake/FindSSE.cmake b/contrib/lua-torch/torch7/lib/TH/cmake/FindSSE.cmake
index a14abe8d4..a14abe8d4 100644
--- a/contrib/torch/torch7/lib/TH/cmake/FindSSE.cmake
+++ b/contrib/lua-torch/torch7/lib/TH/cmake/FindSSE.cmake
diff --git a/contrib/torch/torch7/lib/TH/generic/THBlas.c b/contrib/lua-torch/torch7/lib/TH/generic/THBlas.c
index b04931f34..b04931f34 100644
--- a/contrib/torch/torch7/lib/TH/generic/THBlas.c
+++ b/contrib/lua-torch/torch7/lib/TH/generic/THBlas.c
diff --git a/contrib/torch/torch7/lib/TH/generic/THBlas.h b/contrib/lua-torch/torch7/lib/TH/generic/THBlas.h
index 9e14f5a84..9e14f5a84 100644
--- a/contrib/torch/torch7/lib/TH/generic/THBlas.h
+++ b/contrib/lua-torch/torch7/lib/TH/generic/THBlas.h
diff --git a/contrib/torch/torch7/lib/TH/generic/THLapack.c b/contrib/lua-torch/torch7/lib/TH/generic/THLapack.c
index 148ae26c4..148ae26c4 100644
--- a/contrib/torch/torch7/lib/TH/generic/THLapack.c
+++ b/contrib/lua-torch/torch7/lib/TH/generic/THLapack.c
diff --git a/contrib/torch/torch7/lib/TH/generic/THLapack.h b/contrib/lua-torch/torch7/lib/TH/generic/THLapack.h
index b464dd2d2..b464dd2d2 100644
--- a/contrib/torch/torch7/lib/TH/generic/THLapack.h
+++ b/contrib/lua-torch/torch7/lib/TH/generic/THLapack.h
diff --git a/contrib/torch/torch7/lib/TH/generic/THStorage.c b/contrib/lua-torch/torch7/lib/TH/generic/THStorage.c
index a592cfb62..a592cfb62 100644
--- a/contrib/torch/torch7/lib/TH/generic/THStorage.c
+++ b/contrib/lua-torch/torch7/lib/TH/generic/THStorage.c
diff --git a/contrib/torch/torch7/lib/TH/generic/THStorage.h b/contrib/lua-torch/torch7/lib/TH/generic/THStorage.h
index 3dd214b33..3dd214b33 100644
--- a/contrib/torch/torch7/lib/TH/generic/THStorage.h
+++ b/contrib/lua-torch/torch7/lib/TH/generic/THStorage.h
diff --git a/contrib/torch/torch7/lib/TH/generic/THStorageCopy.c b/contrib/lua-torch/torch7/lib/TH/generic/THStorageCopy.c
index ce4b57eaf..ce4b57eaf 100644
--- a/contrib/torch/torch7/lib/TH/generic/THStorageCopy.c
+++ b/contrib/lua-torch/torch7/lib/TH/generic/THStorageCopy.c
diff --git a/contrib/torch/torch7/lib/TH/generic/THStorageCopy.h b/contrib/lua-torch/torch7/lib/TH/generic/THStorageCopy.h
index ce8a2a690..ce8a2a690 100644
--- a/contrib/torch/torch7/lib/TH/generic/THStorageCopy.h
+++ b/contrib/lua-torch/torch7/lib/TH/generic/THStorageCopy.h
diff --git a/contrib/torch/torch7/lib/TH/generic/THTensor.c b/contrib/lua-torch/torch7/lib/TH/generic/THTensor.c
index e44e06ec3..e44e06ec3 100644
--- a/contrib/torch/torch7/lib/TH/generic/THTensor.c
+++ b/contrib/lua-torch/torch7/lib/TH/generic/THTensor.c
diff --git a/contrib/torch/torch7/lib/TH/generic/THTensor.h b/contrib/lua-torch/torch7/lib/TH/generic/THTensor.h
index 9fb246c85..9fb246c85 100644
--- a/contrib/torch/torch7/lib/TH/generic/THTensor.h
+++ b/contrib/lua-torch/torch7/lib/TH/generic/THTensor.h
diff --git a/contrib/torch/torch7/lib/TH/generic/THTensorConv.c b/contrib/lua-torch/torch7/lib/TH/generic/THTensorConv.c
index 684ff9db5..684ff9db5 100644
--- a/contrib/torch/torch7/lib/TH/generic/THTensorConv.c
+++ b/contrib/lua-torch/torch7/lib/TH/generic/THTensorConv.c
diff --git a/contrib/torch/torch7/lib/TH/generic/THTensorConv.h b/contrib/lua-torch/torch7/lib/TH/generic/THTensorConv.h
index 79866f390..79866f390 100644
--- a/contrib/torch/torch7/lib/TH/generic/THTensorConv.h
+++ b/contrib/lua-torch/torch7/lib/TH/generic/THTensorConv.h
diff --git a/contrib/torch/torch7/lib/TH/generic/THTensorCopy.c b/contrib/lua-torch/torch7/lib/TH/generic/THTensorCopy.c
index d9cd1c0d5..d9cd1c0d5 100644
--- a/contrib/torch/torch7/lib/TH/generic/THTensorCopy.c
+++ b/contrib/lua-torch/torch7/lib/TH/generic/THTensorCopy.c
diff --git a/contrib/torch/torch7/lib/TH/generic/THTensorCopy.h b/contrib/lua-torch/torch7/lib/TH/generic/THTensorCopy.h
index b9e5bfc99..b9e5bfc99 100644
--- a/contrib/torch/torch7/lib/TH/generic/THTensorCopy.h
+++ b/contrib/lua-torch/torch7/lib/TH/generic/THTensorCopy.h
diff --git a/contrib/torch/torch7/lib/TH/generic/THTensorLapack.c b/contrib/lua-torch/torch7/lib/TH/generic/THTensorLapack.c
index d4e52f6d7..d4e52f6d7 100644
--- a/contrib/torch/torch7/lib/TH/generic/THTensorLapack.c
+++ b/contrib/lua-torch/torch7/lib/TH/generic/THTensorLapack.c
diff --git a/contrib/torch/torch7/lib/TH/generic/THTensorLapack.h b/contrib/lua-torch/torch7/lib/TH/generic/THTensorLapack.h
index 878594348..878594348 100644
--- a/contrib/torch/torch7/lib/TH/generic/THTensorLapack.h
+++ b/contrib/lua-torch/torch7/lib/TH/generic/THTensorLapack.h
diff --git a/contrib/torch/torch7/lib/TH/generic/THTensorMath.c b/contrib/lua-torch/torch7/lib/TH/generic/THTensorMath.c
index db7a0cb19..db7a0cb19 100644
--- a/contrib/torch/torch7/lib/TH/generic/THTensorMath.c
+++ b/contrib/lua-torch/torch7/lib/TH/generic/THTensorMath.c
diff --git a/contrib/torch/torch7/lib/TH/generic/THTensorMath.h b/contrib/lua-torch/torch7/lib/TH/generic/THTensorMath.h
index 17e54ccf6..17e54ccf6 100644
--- a/contrib/torch/torch7/lib/TH/generic/THTensorMath.h
+++ b/contrib/lua-torch/torch7/lib/TH/generic/THTensorMath.h
diff --git a/contrib/torch/torch7/lib/TH/generic/THTensorRandom.c b/contrib/lua-torch/torch7/lib/TH/generic/THTensorRandom.c
index 514d3dd27..514d3dd27 100644
--- a/contrib/torch/torch7/lib/TH/generic/THTensorRandom.c
+++ b/contrib/lua-torch/torch7/lib/TH/generic/THTensorRandom.c
diff --git a/contrib/torch/torch7/lib/TH/generic/THTensorRandom.h b/contrib/lua-torch/torch7/lib/TH/generic/THTensorRandom.h
index d20514242..d20514242 100644
--- a/contrib/torch/torch7/lib/TH/generic/THTensorRandom.h
+++ b/contrib/lua-torch/torch7/lib/TH/generic/THTensorRandom.h
diff --git a/contrib/torch/torch7/lib/TH/generic/THVector.h b/contrib/lua-torch/torch7/lib/TH/generic/THVector.h
index 7d368541a..7d368541a 100644
--- a/contrib/torch/torch7/lib/TH/generic/THVector.h
+++ b/contrib/lua-torch/torch7/lib/TH/generic/THVector.h
diff --git a/contrib/torch/torch7/lib/TH/generic/THVectorDefault.c b/contrib/lua-torch/torch7/lib/TH/generic/THVectorDefault.c
index 3388e0d9b..3388e0d9b 100644
--- a/contrib/torch/torch7/lib/TH/generic/THVectorDefault.c
+++ b/contrib/lua-torch/torch7/lib/TH/generic/THVectorDefault.c
diff --git a/contrib/torch/torch7/lib/TH/generic/THVectorDispatch.c b/contrib/lua-torch/torch7/lib/TH/generic/THVectorDispatch.c
index 5b8885283..5b8885283 100644
--- a/contrib/torch/torch7/lib/TH/generic/THVectorDispatch.c
+++ b/contrib/lua-torch/torch7/lib/TH/generic/THVectorDispatch.c
diff --git a/contrib/torch/torch7/lib/TH/generic/simd/common_simd.h b/contrib/lua-torch/torch7/lib/TH/generic/simd/common_simd.h
index 425b4b96e..425b4b96e 100644
--- a/contrib/torch/torch7/lib/TH/generic/simd/common_simd.h
+++ b/contrib/lua-torch/torch7/lib/TH/generic/simd/common_simd.h
diff --git a/contrib/torch/torch7/lib/TH/generic/simd/convolve.c b/contrib/lua-torch/torch7/lib/TH/generic/simd/convolve.c
index da7a4bb20..da7a4bb20 100644
--- a/contrib/torch/torch7/lib/TH/generic/simd/convolve.c
+++ b/contrib/lua-torch/torch7/lib/TH/generic/simd/convolve.c
diff --git a/contrib/torch/torch7/lib/TH/generic/simd/convolve.h b/contrib/lua-torch/torch7/lib/TH/generic/simd/convolve.h
index 7b9b04c50..7b9b04c50 100644
--- a/contrib/torch/torch7/lib/TH/generic/simd/convolve.h
+++ b/contrib/lua-torch/torch7/lib/TH/generic/simd/convolve.h
diff --git a/contrib/torch/torch7/lib/TH/generic/simd/convolve5x5_avx.c b/contrib/lua-torch/torch7/lib/TH/generic/simd/convolve5x5_avx.c
index 52b6d0ffb..52b6d0ffb 100644
--- a/contrib/torch/torch7/lib/TH/generic/simd/convolve5x5_avx.c
+++ b/contrib/lua-torch/torch7/lib/TH/generic/simd/convolve5x5_avx.c
diff --git a/contrib/torch/torch7/lib/TH/generic/simd/convolve5x5_sse.c b/contrib/lua-torch/torch7/lib/TH/generic/simd/convolve5x5_sse.c
index f34b79695..f34b79695 100644
--- a/contrib/torch/torch7/lib/TH/generic/simd/convolve5x5_sse.c
+++ b/contrib/lua-torch/torch7/lib/TH/generic/simd/convolve5x5_sse.c
diff --git a/contrib/torch/torch7/lib/TH/generic/simd/simd.h b/contrib/lua-torch/torch7/lib/TH/generic/simd/simd.h
index 83c4c5681..83c4c5681 100644
--- a/contrib/torch/torch7/lib/TH/generic/simd/simd.h
+++ b/contrib/lua-torch/torch7/lib/TH/generic/simd/simd.h
diff --git a/contrib/torch/torch7/lib/TH/vector/AVX.c b/contrib/lua-torch/torch7/lib/TH/vector/AVX.c
index 58c4e6d35..58c4e6d35 100644
--- a/contrib/torch/torch7/lib/TH/vector/AVX.c
+++ b/contrib/lua-torch/torch7/lib/TH/vector/AVX.c
diff --git a/contrib/torch/torch7/lib/TH/vector/AVX.h b/contrib/lua-torch/torch7/lib/TH/vector/AVX.h
index bfaeaa6b0..bfaeaa6b0 100644
--- a/contrib/torch/torch7/lib/TH/vector/AVX.h
+++ b/contrib/lua-torch/torch7/lib/TH/vector/AVX.h
diff --git a/contrib/torch/torch7/lib/TH/vector/AVX2.c b/contrib/lua-torch/torch7/lib/TH/vector/AVX2.c
index 082a680ea..082a680ea 100644
--- a/contrib/torch/torch7/lib/TH/vector/AVX2.c
+++ b/contrib/lua-torch/torch7/lib/TH/vector/AVX2.c
diff --git a/contrib/torch/torch7/lib/TH/vector/AVX2.h b/contrib/lua-torch/torch7/lib/TH/vector/AVX2.h
index 85a9e93ee..85a9e93ee 100644
--- a/contrib/torch/torch7/lib/TH/vector/AVX2.h
+++ b/contrib/lua-torch/torch7/lib/TH/vector/AVX2.h
diff --git a/contrib/torch/torch7/lib/TH/vector/NEON.c b/contrib/lua-torch/torch7/lib/TH/vector/NEON.c
index 7920fb13b..7920fb13b 100644
--- a/contrib/torch/torch7/lib/TH/vector/NEON.c
+++ b/contrib/lua-torch/torch7/lib/TH/vector/NEON.c
diff --git a/contrib/torch/torch7/lib/TH/vector/SSE.c b/contrib/lua-torch/torch7/lib/TH/vector/SSE.c
index d026935ab..d026935ab 100644
--- a/contrib/torch/torch7/lib/TH/vector/SSE.c
+++ b/contrib/lua-torch/torch7/lib/TH/vector/SSE.c
diff --git a/contrib/torch/torch7/lib/TH/vector/VSX.c b/contrib/lua-torch/torch7/lib/TH/vector/VSX.c
index 9ff984ad7..9ff984ad7 100644
--- a/contrib/torch/torch7/lib/TH/vector/VSX.c
+++ b/contrib/lua-torch/torch7/lib/TH/vector/VSX.c
diff --git a/contrib/torch/torch7/lib/luaT/CMakeLists.txt b/contrib/lua-torch/torch7/lib/luaT/CMakeLists.txt
index ebe788cef..ebe788cef 100644
--- a/contrib/torch/torch7/lib/luaT/CMakeLists.txt
+++ b/contrib/lua-torch/torch7/lib/luaT/CMakeLists.txt
diff --git a/contrib/torch/torch7/lib/luaT/README.md b/contrib/lua-torch/torch7/lib/luaT/README.md
index 235b8edc0..235b8edc0 100644
--- a/contrib/torch/torch7/lib/luaT/README.md
+++ b/contrib/lua-torch/torch7/lib/luaT/README.md
diff --git a/contrib/torch/torch7/lib/luaT/luaT.c b/contrib/lua-torch/torch7/lib/luaT/luaT.c
index d87f5d54c..d87f5d54c 100644
--- a/contrib/torch/torch7/lib/luaT/luaT.c
+++ b/contrib/lua-torch/torch7/lib/luaT/luaT.c
diff --git a/contrib/torch/torch7/lib/luaT/luaT.h b/contrib/lua-torch/torch7/lib/luaT/luaT.h
index 2479a1dc1..2479a1dc1 100644
--- a/contrib/torch/torch7/lib/luaT/luaT.h
+++ b/contrib/lua-torch/torch7/lib/luaT/luaT.h
diff --git a/contrib/torch/torch7/lib/luaT/luaTConfig.cmake.in b/contrib/lua-torch/torch7/lib/luaT/luaTConfig.cmake.in
index bfb20b87a..bfb20b87a 100644
--- a/contrib/torch/torch7/lib/luaT/luaTConfig.cmake.in
+++ b/contrib/lua-torch/torch7/lib/luaT/luaTConfig.cmake.in
diff --git a/contrib/torch/torch7/mkdocs.yml b/contrib/lua-torch/torch7/mkdocs.yml
index 39a34d7c2..39a34d7c2 100644
--- a/contrib/torch/torch7/mkdocs.yml
+++ b/contrib/lua-torch/torch7/mkdocs.yml
diff --git a/contrib/torch/torch7/paths.lua.in b/contrib/lua-torch/torch7/paths.lua.in
index 287770b12..287770b12 100644
--- a/contrib/torch/torch7/paths.lua.in
+++ b/contrib/lua-torch/torch7/paths.lua.in
diff --git a/contrib/torch/torch7/random.c b/contrib/lua-torch/torch7/random.c
index bbe6ba08e..bbe6ba08e 100644
--- a/contrib/torch/torch7/random.c
+++ b/contrib/lua-torch/torch7/random.c
diff --git a/contrib/torch/torch7/random.lua b/contrib/lua-torch/torch7/random.lua
index 59bbd7b1a..59bbd7b1a 100644
--- a/contrib/torch/torch7/random.lua
+++ b/contrib/lua-torch/torch7/random.lua
diff --git a/contrib/torch/torch7/test/longSize.lua b/contrib/lua-torch/torch7/test/longSize.lua
index f5eba865c..f5eba865c 100644
--- a/contrib/torch/torch7/test/longSize.lua
+++ b/contrib/lua-torch/torch7/test/longSize.lua
diff --git a/contrib/torch/torch7/test/test.lua b/contrib/lua-torch/torch7/test/test.lua
index 6c5dc71b0..6c5dc71b0 100644
--- a/contrib/torch/torch7/test/test.lua
+++ b/contrib/lua-torch/torch7/test/test.lua
diff --git a/contrib/torch/torch7/test/test_Multinomial.lua b/contrib/lua-torch/torch7/test/test_Multinomial.lua
index 9069ecbed..9069ecbed 100644
--- a/contrib/torch/torch7/test/test_Multinomial.lua
+++ b/contrib/lua-torch/torch7/test/test_Multinomial.lua
diff --git a/contrib/torch/torch7/test/test_Tester.lua b/contrib/lua-torch/torch7/test/test_Tester.lua
index a28336088..a28336088 100644
--- a/contrib/torch/torch7/test/test_Tester.lua
+++ b/contrib/lua-torch/torch7/test/test_Tester.lua
diff --git a/contrib/torch/torch7/test/test_half.lua b/contrib/lua-torch/torch7/test/test_half.lua
index bf3830b5e..bf3830b5e 100644
--- a/contrib/torch/torch7/test/test_half.lua
+++ b/contrib/lua-torch/torch7/test/test_half.lua
diff --git a/contrib/torch/torch7/test/test_qr.lua b/contrib/lua-torch/torch7/test/test_qr.lua
index c850c3fe1..c850c3fe1 100644
--- a/contrib/torch/torch7/test/test_qr.lua
+++ b/contrib/lua-torch/torch7/test/test_qr.lua
diff --git a/contrib/torch/torch7/test/test_sharedmem.lua b/contrib/lua-torch/torch7/test/test_sharedmem.lua
index 1230e5914..1230e5914 100644
--- a/contrib/torch/torch7/test/test_sharedmem.lua
+++ b/contrib/lua-torch/torch7/test/test_sharedmem.lua
diff --git a/contrib/torch/torch7/test/test_timer.lua b/contrib/lua-torch/torch7/test/test_timer.lua
index ecf576a1a..ecf576a1a 100644
--- a/contrib/torch/torch7/test/test_timer.lua
+++ b/contrib/lua-torch/torch7/test/test_timer.lua
diff --git a/contrib/torch/torch7/test/test_writeObject.lua b/contrib/lua-torch/torch7/test/test_writeObject.lua
index 52bcb7173..52bcb7173 100644
--- a/contrib/torch/torch7/test/test_writeObject.lua
+++ b/contrib/lua-torch/torch7/test/test_writeObject.lua
diff --git a/contrib/torch/torch7/test/timeSort.lua b/contrib/lua-torch/torch7/test/timeSort.lua
index ad513b877..ad513b877 100644
--- a/contrib/torch/torch7/test/timeSort.lua
+++ b/contrib/lua-torch/torch7/test/timeSort.lua
diff --git a/contrib/torch/torch7/torchcwrap.lua b/contrib/lua-torch/torch7/torchcwrap.lua
index 551bd05d2..551bd05d2 100644
--- a/contrib/torch/torch7/torchcwrap.lua
+++ b/contrib/lua-torch/torch7/torchcwrap.lua
diff --git a/contrib/torch/torch7/utils.c b/contrib/lua-torch/torch7/utils.c
index 974d0ac08..974d0ac08 100644
--- a/contrib/torch/torch7/utils.c
+++ b/contrib/lua-torch/torch7/utils.c
diff --git a/contrib/torch/torch7/utils.h b/contrib/lua-torch/torch7/utils.h
index f9654d4b7..f9654d4b7 100644
--- a/contrib/torch/torch7/utils.h
+++ b/contrib/lua-torch/torch7/utils.h
diff --git a/contrib/t1ha/t1ha1.c b/contrib/t1ha/t1ha1.c
index 956f7e24e..6e25d37f4 100644
--- a/contrib/t1ha/t1ha1.c
+++ b/contrib/t1ha/t1ha1.c
@@ -58,21 +58,21 @@ static __inline uint64_t final_weak_avalanche(uint64_t a, uint64_t b) {
return mux64(rot64(a + b, 17), prime_4) + mix64(a ^ b, prime_0);
}
-/* TODO C++ template in the next version */
-#define T1HA1_BODY(ENDIANNES, ALIGNESS, DOCOPY) \
+/* TODO: C++ template in the next version */
+#define T1HA1_BODY(ENDIANNES, ALIGNESS) \
+ const uint64_t *v = (const uint64_t *)data; \
if (unlikely(len > 32)) { \
uint64_t c = rot64(len, 17) + seed; \
uint64_t d = len ^ rot64(seed, 17); \
- const void *detent = (const uint8_t *)data + len - 31; \
+ const uint64_t *detent = \
+ (const uint64_t *)((const uint8_t *)data + len - 31); \
do { \
- const uint64_t *v = (const uint64_t *)data; \
- if (DOCOPY) \
- memcpy((void *)(v = align), data, 32); \
- \
const uint64_t w0 = fetch64_##ENDIANNES##_##ALIGNESS(v + 0); \
const uint64_t w1 = fetch64_##ENDIANNES##_##ALIGNESS(v + 1); \
const uint64_t w2 = fetch64_##ENDIANNES##_##ALIGNESS(v + 2); \
const uint64_t w3 = fetch64_##ENDIANNES##_##ALIGNESS(v + 3); \
+ v += 4; \
+ prefetch(v); \
\
const uint64_t d02 = w0 ^ rot64(w2 + d, 17); \
const uint64_t c13 = w1 ^ rot64(w3 + c, 17); \
@@ -80,18 +80,13 @@ static __inline uint64_t final_weak_avalanche(uint64_t a, uint64_t b) {
d -= b ^ rot64(w1, 31); \
a ^= prime_1 * (d02 + w3); \
b ^= prime_0 * (c13 + w2); \
- data = (const uint64_t *)data + 4; \
- } while (likely(data < detent)); \
+ } while (likely(v < detent)); \
\
a ^= prime_6 * (rot64(c, 17) + d); \
b ^= prime_5 * (c + rot64(d, 17)); \
len &= 31; \
} \
\
- const uint64_t *v = (const uint64_t *)data; \
- if (unlikely(need_copy4align) && len > 8) \
- memcpy((void *)(v = align), data, len); \
- \
switch (len) { \
default: \
b += mux64(fetch64_##ENDIANNES##_##ALIGNESS(v++), prime_4); \
@@ -134,26 +129,30 @@ uint64_t t1ha1_le(const void *data, size_t len, uint64_t seed) {
uint64_t a = seed;
uint64_t b = len;
- const bool need_copy4align =
- (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0 && !UNALIGNED_OK;
- uint64_t align[4];
- if (need_copy4align) {
- T1HA1_BODY(le, aligned, true);
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT
+ T1HA1_BODY(le, unaligned);
+#else
+ const bool misaligned = (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0;
+ if (misaligned) {
+ T1HA1_BODY(le, unaligned);
} else {
- T1HA1_BODY(le, unaligned, false);
+ T1HA1_BODY(le, aligned);
}
+#endif
}
uint64_t t1ha1_be(const void *data, size_t len, uint64_t seed) {
uint64_t a = seed;
uint64_t b = len;
- const bool need_copy4align =
- (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0 && !UNALIGNED_OK;
- uint64_t align[4];
- if (need_copy4align) {
- T1HA1_BODY(be, aligned, true);
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT
+ T1HA1_BODY(be, unaligned);
+#else
+ const bool misaligned = (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0;
+ if (misaligned) {
+ T1HA1_BODY(be, unaligned);
} else {
- T1HA1_BODY(be, unaligned, false);
+ T1HA1_BODY(be, aligned);
}
-} \ No newline at end of file
+#endif
+}
diff --git a/contrib/t1ha/t1ha2.c b/contrib/t1ha/t1ha2.c
index 95f646da4..4cb5281ee 100644
--- a/contrib/t1ha/t1ha2.c
+++ b/contrib/t1ha/t1ha2.c
@@ -56,7 +56,7 @@ static __always_inline void init_cd(t1ha_state256_t *s, uint64_t x,
s->n.d = ~y + rot64(x, 19);
}
-/* TODO C++ template in the next version */
+/* TODO: C++ template in the next version */
#define T1HA2_UPDATE(ENDIANNES, ALIGNESS, state, v) \
do { \
t1ha_state256_t *const s = state; \
@@ -67,8 +67,8 @@ static __always_inline void init_cd(t1ha_state256_t *s, uint64_t x,
\
const uint64_t d02 = w0 + rot64(w2 + s->n.d, 56); \
const uint64_t c13 = w1 + rot64(w3 + s->n.c, 19); \
- s->n.d ^= s->n.b + rot64(w1, 38); \
s->n.c ^= s->n.a + rot64(w0, 57); \
+ s->n.d ^= s->n.b + rot64(w1, 38); \
s->n.b ^= prime_6 * (c13 + w2); \
s->n.a ^= prime_5 * (d02 + w3); \
} while (0)
@@ -78,26 +78,23 @@ static __always_inline void squash(t1ha_state256_t *s) {
s->n.b ^= prime_5 * (rot64(s->n.c, 19) + s->n.d);
}
-/* TODO C++ template in the next version */
-#define T1HA2_LOOP(ENDIANNES, ALIGNESS, BUFFER4COPY, state, data, len) \
+/* TODO: C++ template in the next version */
+#define T1HA2_LOOP(ENDIANNES, ALIGNESS, state, data, len) \
do { \
const void *detent = (const uint8_t *)data + len - 31; \
do { \
const uint64_t *v = (const uint64_t *)data; \
- if (BUFFER4COPY != NULL) \
- memcpy((void *)(v = BUFFER4COPY), data, 32); \
- T1HA2_UPDATE(le, unaligned, state, v); \
data = (const uint64_t *)data + 4; \
+ prefetch(data); \
+ T1HA2_UPDATE(le, ALIGNESS, state, v); \
} while (likely(data < detent)); \
} while (0)
-/* TODO C++ template in the next version */
-#define T1HA2_TAIL_AB(ENDIANNES, ALIGNESS, BUFFER4COPY, state, data, len) \
+/* TODO: C++ template in the next version */
+#define T1HA2_TAIL_AB(ENDIANNES, ALIGNESS, state, data, len) \
do { \
t1ha_state256_t *const s = state; \
const uint64_t *v = (const uint64_t *)data; \
- if (BUFFER4COPY != NULL) \
- memcpy((void *)(v = BUFFER4COPY), data, len); \
switch (len) { \
default: \
mixup64(&s->n.a, &s->n.b, fetch64_##ENDIANNES##_##ALIGNESS(v++), \
@@ -141,13 +138,11 @@ static __always_inline void squash(t1ha_state256_t *s) {
} \
} while (0)
-/* TODO C++ template in the next version */
-#define T1HA2_TAIL_ABCD(ENDIANNES, ALIGNESS, BUFFER4COPY, state, data, len) \
+/* TODO: C++ template in the next version */
+#define T1HA2_TAIL_ABCD(ENDIANNES, ALIGNESS, state, data, len) \
do { \
t1ha_state256_t *const s = state; \
const uint64_t *v = (const uint64_t *)data; \
- if (BUFFER4COPY != NULL) \
- memcpy((void *)(v = BUFFER4COPY), data, len); \
switch (len) { \
default: \
mixup64(&s->n.a, &s->n.d, fetch64_##ENDIANNES##_##ALIGNESS(v++), \
@@ -207,26 +202,34 @@ uint64_t t1ha2_atonce(const void *data, size_t length, uint64_t seed) {
t1ha_state256_t state;
init_ab(&state, seed, length);
- const bool need_copy4align =
- (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0 && !UNALIGNED_OK;
- if (need_copy4align) {
- uint64_t buffer4align[4];
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT
+ if (unlikely(length > 32)) {
+ init_cd(&state, seed, length);
+ T1HA2_LOOP(le, unaligned, &state, data, length);
+ squash(&state);
+ length &= 31;
+ }
+ T1HA2_TAIL_AB(le, unaligned, &state, data, length);
+#else
+ const bool misaligned = (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0;
+ if (misaligned) {
if (unlikely(length > 32)) {
init_cd(&state, seed, length);
- T1HA2_LOOP(le, aligned, buffer4align, &state, data, length);
+ T1HA2_LOOP(le, unaligned, &state, data, length);
squash(&state);
length &= 31;
}
- T1HA2_TAIL_AB(le, aligned, buffer4align, &state, data, length);
+ T1HA2_TAIL_AB(le, unaligned, &state, data, length);
} else {
if (unlikely(length > 32)) {
init_cd(&state, seed, length);
- T1HA2_LOOP(le, unaligned, NULL, &state, data, length);
+ T1HA2_LOOP(le, aligned, &state, data, length);
squash(&state);
length &= 31;
}
- T1HA2_TAIL_AB(le, unaligned, NULL, &state, data, length);
+ T1HA2_TAIL_AB(le, aligned, &state, data, length);
}
+#endif
}
uint64_t t1ha2_atonce128(uint64_t *__restrict extra_result,
@@ -236,22 +239,28 @@ uint64_t t1ha2_atonce128(uint64_t *__restrict extra_result,
init_ab(&state, seed, length);
init_cd(&state, seed, length);
- const bool need_copy4align =
- (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0 && !UNALIGNED_OK;
- if (need_copy4align) {
- uint64_t buffer4align[4];
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT
+ if (unlikely(length > 32)) {
+ T1HA2_LOOP(le, unaligned, &state, data, length);
+ length &= 31;
+ }
+ T1HA2_TAIL_ABCD(le, unaligned, &state, data, length);
+#else
+ const bool misaligned = (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0;
+ if (misaligned) {
if (unlikely(length > 32)) {
- T1HA2_LOOP(le, aligned, buffer4align, &state, data, length);
+ T1HA2_LOOP(le, unaligned, &state, data, length);
length &= 31;
}
- T1HA2_TAIL_ABCD(le, aligned, buffer4align, &state, data, length);
+ T1HA2_TAIL_ABCD(le, unaligned, &state, data, length);
} else {
if (unlikely(length > 32)) {
- T1HA2_LOOP(le, unaligned, NULL, &state, data, length);
+ T1HA2_LOOP(le, aligned, &state, data, length);
length &= 31;
}
- T1HA2_TAIL_ABCD(le, unaligned, NULL, &state, data, length);
+ T1HA2_TAIL_ABCD(le, aligned, &state, data, length);
}
+#endif
}
//------------------------------------------------------------------------------
@@ -283,13 +292,16 @@ void t1ha2_update(t1ha_context_t *__restrict ctx, const void *__restrict data,
}
if (length >= 32) {
- const bool need_copy4align =
- (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0 && !UNALIGNED_OK;
- if (need_copy4align) {
- T1HA2_LOOP(le, aligned, ctx->buffer.u64, &ctx->state, data, length);
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT
+ T1HA2_LOOP(le, unaligned, &ctx->state, data, length);
+#else
+ const bool misaligned = (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0;
+ if (misaligned) {
+ T1HA2_LOOP(le, unaligned, &ctx->state, data, length);
} else {
- T1HA2_LOOP(le, unaligned, NULL, &ctx->state, data, length);
+ T1HA2_LOOP(le, aligned, &ctx->state, data, length);
}
+#endif
length &= 31;
}
@@ -307,13 +319,8 @@ uint64_t t1ha2_final(t1ha_context_t *__restrict ctx,
if (likely(!extra_result)) {
squash(&ctx->state);
- T1HA2_TAIL_AB(le, aligned, NULL, &ctx->state, ctx->buffer.u64,
- ctx->partial);
- return final64(ctx->state.n.a, ctx->state.n.b);
+ T1HA2_TAIL_AB(le, aligned, &ctx->state, ctx->buffer.u64, ctx->partial);
}
- T1HA2_TAIL_ABCD(le, aligned, NULL, &ctx->state, ctx->buffer.u64,
- ctx->partial);
- return final128(ctx->state.n.a, ctx->state.n.b, ctx->state.n.c,
- ctx->state.n.d, extra_result);
-} \ No newline at end of file
+ T1HA2_TAIL_ABCD(le, aligned, &ctx->state, ctx->buffer.u64, ctx->partial);
+}
diff --git a/contrib/t1ha/t1ha_bits.h b/contrib/t1ha/t1ha_bits.h
index e3815a4e7..454e43aed 100644
--- a/contrib/t1ha/t1ha_bits.h
+++ b/contrib/t1ha/t1ha_bits.h
@@ -72,19 +72,23 @@
#error Unsupported byte order.
#endif
-#if !defined(UNALIGNED_OK)
-#if (defined(__ia32__) || defined(__e2k__) || \
- defined(__ARM_FEATURE_UNALIGNED)) && \
- !defined(__ALIGNED__)
-#define UNALIGNED_OK 1
+#define T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE 0
+#define T1HA_CONFIG_UNALIGNED_ACCESS__SLOW 1
+#define T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT 2
+
+#ifndef T1HA_CONFIG_UNALIGNED_ACCESS
+#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)
+#define T1HA_CONFIG_UNALIGNED_ACCESS T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT
+#elif defined(__ia32__)
+#define T1HA_CONFIG_UNALIGNED_ACCESS T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT
+#elif defined(__e2k__)
+#define T1HA_CONFIG_UNALIGNED_ACCESS T1HA_CONFIG_UNALIGNED_ACCESS__SLOW
+#elif defined(__ARM_FEATURE_UNALIGNED)
+#define T1HA_CONFIG_UNALIGNED_ACCESS T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT
#else
-#define UNALIGNED_OK 0
+#define T1HA_CONFIG_UNALIGNED_ACCESS T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE
#endif
-#endif /* UNALIGNED_OK */
-
-#if UNALIGNED_OK && !defined(PAGESIZE)
-#define PAGESIZE 4096
-#endif /* PAGESIZE */
+#endif /* T1HA_CONFIG_UNALIGNED_ACCESS */
#define ALIGNMENT_16 2
#define ALIGNMENT_32 4
@@ -94,6 +98,10 @@
#define ALIGNMENT_64 4
#endif
+#ifndef PAGESIZE
+#define PAGESIZE 4096
+#endif /* PAGESIZE */
+
/***************************************************************************/
#ifndef __has_builtin
@@ -118,6 +126,13 @@
#if __GNUC_PREREQ(4, 4) || defined(__clang__)
+#if defined(__ia32__) || defined(__e2k__)
+#include <x86intrin.h>
+#endif
+
+#if defined(__ia32__) && !defined(__cpuid_count)
+#include <cpuid.h>
+#endif
#if defined(__e2k__)
#include <e2kbuiltin.h>
@@ -393,10 +408,10 @@ typedef struct {
#endif /* read_unaligned */
#ifndef read_aligned
-#if __has_builtin(assume_aligned)
+#if __GNUC_PREREQ(4, 8) || __has_builtin(__builtin_assume_aligned)
#define read_aligned(ptr, bits) \
(*(const uint##bits##_t *)__builtin_assume_aligned(ptr, ALIGNMENT_##bits))
-#elif __has_attribute(aligned) && !defined(__clang__)
+#elif (__GNUC_PREREQ(3, 3) || __has_attribute(aligned)) && !defined(__clang__)
#define read_aligned(ptr, bits) \
(*(const uint##bits##_t __attribute__((aligned(ALIGNMENT_##bits))) *)(ptr))
#elif __has_attribute(assume_aligned)
@@ -427,6 +442,20 @@ static __always_inline const
#endif
#endif /* read_aligned */
+#ifndef prefetch
+#if (__GNUC_PREREQ(4, 0) || __has_builtin(__builtin_prefetch)) && \
+ !defined(__ia32__)
+#define prefetch(ptr) __builtin_prefetch(ptr)
+#elif defined(_M_ARM64) || defined(_M_ARM)
+#define prefetch(ptr) __prefetch(ptr)
+#else
+#define prefetch(ptr) \
+ do { \
+ (void)(ptr); \
+ } while (0)
+#endif
+#endif /* prefetch */
+
#if __has_warning("-Wconstant-logical-operand")
#if defined(__clang__)
#pragma clang diagnostic ignored "-Wconstant-logical-operand"
@@ -451,28 +480,33 @@ static __always_inline const
/*---------------------------------------------------------- Little Endian */
-#ifndef fetch64_le_aligned
-static __always_inline uint64_t fetch64_le_aligned(const void *v) {
+#ifndef fetch16_le_aligned
+static __always_inline uint16_t fetch16_le_aligned(const void *v) {
+ assert(((uintptr_t)v) % ALIGNMENT_16 == 0);
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
- return read_aligned(v, 64);
+ return read_aligned(v, 16);
#else
- return bswap64(read_aligned(v, 64));
+ return bswap16(read_aligned(v, 16));
#endif
}
-#endif /* fetch64_le_aligned */
+#endif /* fetch16_le_aligned */
-#ifndef fetch64_le_unaligned
-static __always_inline uint64_t fetch64_le_unaligned(const void *v) {
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
- return read_unaligned(v, 64);
+#ifndef fetch16_le_unaligned
+static __always_inline uint16_t fetch16_le_unaligned(const void *v) {
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE
+ const uint8_t *p = (const uint8_t *)v;
+ return p[0] | (uint16_t)p[1] << 8;
+#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ return read_unaligned(v, 16);
#else
- return bswap64(read_unaligned(v, 64));
+ return bswap16(read_unaligned(v, 16));
#endif
}
-#endif /* fetch64_le_unaligned */
+#endif /* fetch16_le_unaligned */
#ifndef fetch32_le_aligned
static __always_inline uint32_t fetch32_le_aligned(const void *v) {
+ assert(((uintptr_t)v) % ALIGNMENT_32 == 0);
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
return read_aligned(v, 32);
#else
@@ -483,7 +517,10 @@ static __always_inline uint32_t fetch32_le_aligned(const void *v) {
#ifndef fetch32_le_unaligned
static __always_inline uint32_t fetch32_le_unaligned(const void *v) {
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE
+ return fetch16_le_unaligned(v) |
+ (uint32_t)fetch16_le_unaligned((const uint8_t *)v + 2) << 16;
+#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
return read_unaligned(v, 32);
#else
return bswap32(read_unaligned(v, 32));
@@ -491,25 +528,29 @@ static __always_inline uint32_t fetch32_le_unaligned(const void *v) {
}
#endif /* fetch32_le_unaligned */
-#ifndef fetch16_le_aligned
-static __always_inline uint16_t fetch16_le_aligned(const void *v) {
+#ifndef fetch64_le_aligned
+static __always_inline uint64_t fetch64_le_aligned(const void *v) {
+ assert(((uintptr_t)v) % ALIGNMENT_64 == 0);
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
- return read_aligned(v, 16);
+ return read_aligned(v, 64);
#else
- return bswap16(read_aligned(v, 16));
+ return bswap64(read_aligned(v, 64));
#endif
}
-#endif /* fetch16_le_aligned */
+#endif /* fetch64_le_aligned */
-#ifndef fetch16_le_unaligned
-static __always_inline uint16_t fetch16_le_unaligned(const void *v) {
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
- return read_unaligned(v, 16);
+#ifndef fetch64_le_unaligned
+static __always_inline uint64_t fetch64_le_unaligned(const void *v) {
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE
+ return fetch32_le_unaligned(v) |
+ (uint64_t)fetch32_le_unaligned((const uint8_t *)v + 4) << 32;
+#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ return read_unaligned(v, 64);
#else
- return bswap16(read_unaligned(v, 16));
+ return bswap64(read_unaligned(v, 64));
#endif
}
-#endif /* fetch16_le_unaligned */
+#endif /* fetch64_le_unaligned */
static __always_inline uint64_t tail64_le_aligned(const void *v, size_t tail) {
const uint8_t *const p = (const uint8_t *)v;
@@ -517,10 +558,12 @@ static __always_inline uint64_t tail64_le_aligned(const void *v, size_t tail) {
/* We can perform a 'oneshot' read, which is little bit faster. */
const unsigned shift = ((8 - tail) & 7) << 3;
return fetch64_le_aligned(p) & ((~UINT64_C(0)) >> shift);
-#endif /* 'oneshot' read */
-
+#else
uint64_t r = 0;
switch (tail & 7) {
+ default:
+ unreachable();
+/* fall through */
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
/* For most CPUs this code is better when not needed byte reordering. */
case 0:
@@ -577,14 +620,15 @@ static __always_inline uint64_t tail64_le_aligned(const void *v, size_t tail) {
return r + p[0];
#endif
}
- unreachable();
+#endif /* T1HA_USE_FAST_ONESHOT_READ */
}
-#if T1HA_USE_FAST_ONESHOT_READ && UNALIGNED_OK && defined(PAGESIZE) && \
- !defined(__SANITIZE_ADDRESS__) && !defined(__sun)
+#if T1HA_USE_FAST_ONESHOT_READ && \
+ T1HA_CONFIG_UNALIGNED_ACCESS != T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE && \
+ defined(PAGESIZE) && !defined(__sun) && !defined(__SANITIZE_ADDRESS__)
#define can_read_underside(ptr, size) \
((size) <= sizeof(uintptr_t) && ((PAGESIZE - (size)) & (uintptr_t)(ptr)) != 0)
-#endif /* can_fast_read */
+#endif /* T1HA_USE_FAST_ONESHOT_READ */
static __always_inline uint64_t tail64_le_unaligned(const void *v,
size_t tail) {
@@ -600,11 +644,14 @@ static __always_inline uint64_t tail64_le_unaligned(const void *v,
return fetch64_le_unaligned(p) >> shift;
}
return fetch64_le_unaligned(p) & ((~UINT64_C(0)) >> shift);
-#endif /* 'oneshot' read */
-
+#else
uint64_t r = 0;
switch (tail & 7) {
-#if UNALIGNED_OK && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ default:
+ unreachable();
+/* fall through */
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT && \
+ __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
/* For most CPUs this code is better when not needed
* copying for alignment or byte reordering. */
case 0:
@@ -663,36 +710,41 @@ static __always_inline uint64_t tail64_le_unaligned(const void *v,
return r + p[0];
#endif
}
- unreachable();
+#endif /* can_read_underside */
}
/*------------------------------------------------------------- Big Endian */
-#ifndef fetch64_be_aligned
-static __maybe_unused __always_inline uint64_t
-fetch64_be_aligned(const void *v) {
+#ifndef fetch16_be_aligned
+static __maybe_unused __always_inline uint16_t
+fetch16_be_aligned(const void *v) {
+ assert(((uintptr_t)v) % ALIGNMENT_16 == 0);
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
- return read_aligned(v, 64);
+ return read_aligned(v, 16);
#else
- return bswap64(read_aligned(v, 64));
+ return bswap16(read_aligned(v, 16));
#endif
}
-#endif /* fetch64_be_aligned */
+#endif /* fetch16_be_aligned */
-#ifndef fetch64_be_unaligned
-static __maybe_unused __always_inline uint64_t
-fetch64_be_unaligned(const void *v) {
-#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
- return read_unaligned(v, 64);
+#ifndef fetch16_be_unaligned
+static __maybe_unused __always_inline uint16_t
+fetch16_be_unaligned(const void *v) {
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE
+ const uint8_t *p = (const uint8_t *)v;
+ return (uint16_t)p[0] << 8 | p[1];
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ return read_unaligned(v, 16);
#else
- return bswap64(read_unaligned(v, 64));
+ return bswap16(read_unaligned(v, 16));
#endif
}
-#endif /* fetch64_be_unaligned */
+#endif /* fetch16_be_unaligned */
#ifndef fetch32_be_aligned
static __maybe_unused __always_inline uint32_t
fetch32_be_aligned(const void *v) {
+ assert(((uintptr_t)v) % ALIGNMENT_32 == 0);
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
return read_aligned(v, 32);
#else
@@ -704,7 +756,10 @@ fetch32_be_aligned(const void *v) {
#ifndef fetch32_be_unaligned
static __maybe_unused __always_inline uint32_t
fetch32_be_unaligned(const void *v) {
-#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE
+ return (uint32_t)fetch16_be_unaligned(v) << 16 |
+ fetch16_be_unaligned((const uint8_t *)v + 2);
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
return read_unaligned(v, 32);
#else
return bswap32(read_unaligned(v, 32));
@@ -712,27 +767,31 @@ fetch32_be_unaligned(const void *v) {
}
#endif /* fetch32_be_unaligned */
-#ifndef fetch16_be_aligned
-static __maybe_unused __always_inline uint16_t
-fetch16_be_aligned(const void *v) {
+#ifndef fetch64_be_aligned
+static __maybe_unused __always_inline uint64_t
+fetch64_be_aligned(const void *v) {
+ assert(((uintptr_t)v) % ALIGNMENT_64 == 0);
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
- return read_aligned(v, 16);
+ return read_aligned(v, 64);
#else
- return bswap16(read_aligned(v, 16));
+ return bswap64(read_aligned(v, 64));
#endif
}
-#endif /* fetch16_be_aligned */
+#endif /* fetch64_be_aligned */
-#ifndef fetch16_be_unaligned
-static __maybe_unused __always_inline uint16_t
-fetch16_be_unaligned(const void *v) {
-#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
- return read_unaligned(v, 16);
+#ifndef fetch64_be_unaligned
+static __maybe_unused __always_inline uint64_t
+fetch64_be_unaligned(const void *v) {
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE
+ return (uint64_t)fetch32_be_unaligned(v) << 32 |
+ fetch32_be_unaligned((const uint8_t *)v + 4);
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ return read_unaligned(v, 64);
#else
- return bswap16(read_unaligned(v, 16));
+ return bswap64(read_unaligned(v, 64));
#endif
}
-#endif /* fetch16_be_unaligned */
+#endif /* fetch64_be_unaligned */
static __maybe_unused __always_inline uint64_t tail64_be_aligned(const void *v,
size_t tail) {
@@ -741,9 +800,11 @@ static __maybe_unused __always_inline uint64_t tail64_be_aligned(const void *v,
/* We can perform a 'oneshot' read, which is little bit faster. */
const unsigned shift = ((8 - tail) & 7) << 3;
return fetch64_be_aligned(p) >> shift;
-#endif /* 'oneshot' read */
-
+#else
switch (tail & 7) {
+ default:
+ unreachable();
+/* fall through */
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
/* For most CPUs this code is better when not byte reordering. */
case 1:
@@ -762,7 +823,7 @@ static __maybe_unused __always_inline uint64_t tail64_be_aligned(const void *v,
return (uint64_t)fetch32_be_aligned(p) << 24 |
(uint32_t)fetch16_be_aligned(p + 4) << 8 | p[6];
case 0:
- return fetch64_be(p);
+ return fetch64_be_aligned(p);
#else
case 1:
return p[0];
@@ -789,7 +850,7 @@ static __maybe_unused __always_inline uint64_t tail64_be_aligned(const void *v,
(uint64_t)p[1] << 48 | (uint64_t)p[0] << 56;
#endif
}
- unreachable();
+#endif /* T1HA_USE_FAST_ONESHOT_READ */
}
static __maybe_unused __always_inline uint64_t
@@ -806,10 +867,13 @@ tail64_be_unaligned(const void *v, size_t tail) {
return fetch64_be_unaligned(p) & ((~UINT64_C(0)) >> shift);
}
return fetch64_be_unaligned(p) >> shift;
-#endif /* 'oneshot' read */
-
+#else
switch (tail & 7) {
-#if UNALIGNED_OK && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ default:
+ unreachable();
+/* fall through */
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT && \
+ __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
/* For most CPUs this code is better when not needed
* copying for alignment or byte reordering. */
case 1:
@@ -858,7 +922,7 @@ tail64_be_unaligned(const void *v, size_t tail) {
(uint64_t)p[1] << 48 | (uint64_t)p[0] << 56;
#endif
}
- unreachable();
+#endif /* can_read_underside */
}
/***************************************************************************/
diff --git a/interface/js/app/history.js b/interface/js/app/history.js
index dc44a2ddf..c5cd0d37c 100644
--- a/interface/js/app/history.js
+++ b/interface/js/app/history.js
@@ -171,7 +171,14 @@ function($, _, Humanize) {
"value": scan_time
};
item.id = item['message-id'];
- item.rcpt_mime = item.rcpt_mime.join(",&#8203;");
+ if ($(item.rcpt_mime).not(item.rcpt_smtp).length !== 0 || $(item.rcpt_smtp).not(item.rcpt_mime).length !== 0) {
+ item.rcpt_mime = "[" + item.rcpt_smtp.join(",&#8203;") + "] " + item.rcpt_mime.join(",&#8203;");
+ } else {
+ item.rcpt_mime = item.rcpt_mime.join(",&#8203;");
+ }
+ if (item.sender_mime !== item.sender_smtp) {
+ item.sender_mime = "[" + item.sender_smtp + "] " + item.sender_mime;
+ }
items.push(item);
});
@@ -225,19 +232,23 @@ function($, _, Humanize) {
}
}, {
"name": "sender_mime",
- "title": "From",
+ "title": "[Envelope From] From",
"breakpoints": "xs sm md",
"style": {
"font-size": "11px",
- "minWidth": 100
+ "minWidth": 100,
+ "maxWidth": 200,
+ "word-wrap": "break-word"
}
}, {
"name": "rcpt_mime",
- "title": "To",
+ "title": "[Envelope To] To/Cc/Bcc",
"breakpoints": "xs sm md",
"style": {
"font-size": "11px",
- "minWidth": 100
+ "minWidth": 100,
+ "maxWidth": 200,
+ "word-wrap": "break-word"
}
}, {
"name": "subject",
@@ -305,7 +316,9 @@ function($, _, Humanize) {
"breakpoints": "xs sm md",
"style": {
"font-size": "11px",
- "minWidth": 100
+ "minWidth": 100,
+ "maxWidth": 130,
+ "word-wrap": "break-word"
}
}];
}
diff --git a/interface/js/app/symbols.js b/interface/js/app/symbols.js
index b1745abf0..7983ccc7f 100644
--- a/interface/js/app/symbols.js
+++ b/interface/js/app/symbols.js
@@ -97,7 +97,7 @@ function($) {
var label_class = '';
if (item.weight < 0) {
label_class = 'scorebar-ham';
- } else {
+ } else if (item.weight > 0) {
label_class = 'scorebar-spam';
}
item.weight = '<input class="form-control input-sm mb-disabled ' + label_class +
@@ -239,11 +239,15 @@ function($) {
},
components: {
filtering: FooTable.groupFilter
+ },
+ "on": {
+ "ready.ft.table": function () {
+ if (rspamd.read_only) {
+ $(".mb-disabled").attr('disabled', true);
+ }
+ }
}
});
- if (rspamd.read_only) {
- $( ".mb-disabled" ).attr('disabled', true);
- }
},
error: function (data) {
rspamd.alertMessage('alert-modal alert-error', data.statusText);
diff --git a/lualib/rspamadm/ansicolors.lua b/lualib/ansicolors.lua
index 81783f618..81783f618 100644
--- a/lualib/rspamadm/ansicolors.lua
+++ b/lualib/ansicolors.lua
diff --git a/lualib/rspamadm/plugins_stats.lua b/lualib/plugins_stats.lua
index 5c9797a9e..9bbebb1c8 100644
--- a/lualib/rspamadm/plugins_stats.lua
+++ b/lualib/plugins_stats.lua
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
]]--
-local ansicolors = require "rspamadm/ansicolors"
+local ansicolors = require "ansicolors"
local function printf(fmt, ...)
print(string.format(fmt, ...))
diff --git a/lualib/rspamadm/rescore_utility.lua b/lualib/rescore_utility.lua
index 2a9372d4e..195ae4364 100644
--- a/lualib/rspamadm/rescore_utility.lua
+++ b/lualib/rescore_utility.lua
@@ -11,7 +11,7 @@ function utility.get_all_symbols(logs, ignore_symbols)
for _, line in pairs(logs) do
line = lua_util.rspamd_str_split(line, " ")
- for i=4,(#line-2) do
+ for i=4,(#line-1) do
line[i] = line[i]:gsub("%s+", "")
if not symbols_set[line[i]] then
symbols_set[line[i]] = true
@@ -35,49 +35,69 @@ end
function utility.read_log_file(file)
local lines = {}
+ local messages = {}
- file = assert(io.open(file, "r"))
+ local fd = assert(io.open(file, "r"))
+ local fname = string.gsub(file, "(.*/)(.*)", "%2")
- for line in file:lines() do
- lines[#lines + 1] = line
- end
+ for line in fd:lines() do
+ local start,stop = string.find(line, fname .. ':')
+
+ if start and stop then
+ table.insert(lines, string.sub(line, 1, start))
+ table.insert(messages, string.sub(line, stop + 1, -1))
+ end
+end
- io.close(file)
+ io.close(fd)
- return lines
+ return lines,messages
end
-function utility.get_all_logs(dir_path)
+function utility.get_all_logs(dirs)
-- Reads all log files in the directory and returns a list of logs.
- if dir_path:sub(#dir_path, #dir_path) == "/" then
- dir_path = dir_path:sub(1, #dir_path -1)
+ if type(dirs) == 'string' then
+ dirs = {dirs}
end
- local files = rspamd_util.glob(dir_path .. "/*.log")
local all_logs = {}
-
- for _, file in pairs(files) do
- local logs = utility.read_log_file(file)
- for _, log_line in pairs(logs) do
- all_logs[#all_logs + 1] = log_line
+ local all_messages = {}
+
+ for _,dir in ipairs(dirs) do
+ if dir:sub(-1, -1) == "/" then
+ dir = dir:sub(1, -2)
+ local files = rspamd_util.glob(dir .. "/*.log")
+ for _, file in pairs(files) do
+ local logs,messages = utility.read_log_file(file)
+ for i=1,#logs do
+ table.insert(all_logs, logs[i])
+ table.insert(all_messages, messages[i])
+ end
+ end
+ else
+ local logs,messages = utility.read_log_file(dir)
+ for i=1,#logs do
+ table.insert(all_logs, logs[i])
+ table.insert(all_messages, messages[i])
+ end
end
end
- return all_logs
+ return all_logs,all_messages
end
function utility.get_all_symbol_scores(conf, ignore_symbols)
- local counters = conf:get_symbols_counters()
+ local symbols = conf:get_symbols_scores()
- return fun.tomap(fun.map(function(elt)
- return elt['symbol'],elt['weight']
- end, fun.filter(function(elt)
- return not ignore_symbols[elt['symbol']]
- end, counters)))
+ return fun.tomap(fun.map(function(name, elt)
+ return name,elt['score']
+ end, fun.filter(function(name, elt)
+ return not ignore_symbols[name]
+ end, symbols)))
end
-function utility.generate_statistics_from_logs(logs, threshold)
+function utility.generate_statistics_from_logs(logs, messages, threshold)
-- Returns file_stats table and list of symbol_stats table.
@@ -110,9 +130,10 @@ function utility.generate_statistics_from_logs(logs, threshold)
local no_of_spam = 0
local no_of_ham = 0
- for _, log in pairs(logs) do
+ for i, log in ipairs(logs) do
log = lua_util.rspamd_str_trim(log)
log = lua_util.rspamd_str_split(log, " ")
+ local message = messages[i]
local is_spam = (log[1] == "SPAM")
local score = tonumber(log[2])
@@ -129,40 +150,38 @@ function utility.generate_statistics_from_logs(logs, threshold)
true_positives = true_positives + 1
elseif is_spam and (score < threshold) then
false_negatives = false_negatives + 1
- table.insert(all_fns, log[#log])
+ table.insert(all_fns, message)
elseif not is_spam and (score >= threshold) then
false_positives = false_positives + 1
- table.insert(all_fps, log[#log])
+ table.insert(all_fps, message)
else
true_negatives = true_negatives + 1
end
- for i=4, (#log-2) do
- if all_symbols_stats[log[i]] == nil then
- all_symbols_stats[log[i]] = {
- name = log[i],
+ for j=4, (#log-1) do
+ if all_symbols_stats[log[j]] == nil then
+ all_symbols_stats[log[j]] = {
+ name = message,
no_of_hits = 0,
spam_hits = 0,
ham_hits = 0,
spam_overall = 0
}
end
+ local sym = log[j]
- all_symbols_stats[log[i]].no_of_hits =
- all_symbols_stats[log[i]].no_of_hits + 1
+ all_symbols_stats[sym].no_of_hits = all_symbols_stats[sym].no_of_hits + 1
if is_spam then
- all_symbols_stats[log[i]].spam_hits =
- all_symbols_stats[log[i]].spam_hits + 1
+ all_symbols_stats[sym].spam_hits = all_symbols_stats[sym].spam_hits + 1
else
- all_symbols_stats[log[i]].ham_hits =
- all_symbols_stats[log[i]].ham_hits + 1
+ all_symbols_stats[sym].ham_hits = all_symbols_stats[sym].ham_hits + 1
end
-- Find slowest message
- if (tonumber(log[#log-1]) > tonumber(file_stats.slowest)) then
- file_stats.slowest = tostring(tonumber(log[#log-1]))
- file_stats.slowest_file = log[#log]
+ if ((tonumber(log[#log]) or 0) > file_stats.slowest) then
+ file_stats.slowest = tonumber(log[#log])
+ file_stats.slowest_file = message
end
end
end
diff --git a/lualib/rspamadm/confighelp.lua b/lualib/rspamadm/confighelp.lua
index a03578b6e..d477ff69b 100644
--- a/lualib/rspamadm/confighelp.lua
+++ b/lualib/rspamadm/confighelp.lua
@@ -1,4 +1,4 @@
-local opts = {}
+local opts
local known_attrs = {
data = 1,
example = 1,
@@ -6,9 +6,19 @@ local known_attrs = {
required = 1,
default = 1,
}
+local argparse = require "argparse"
+local ansicolors = require "ansicolors"
-local getopt = require "rspamadm/getopt"
-local ansicolors = require "rspamadm/ansicolors"
+local parser = argparse()
+ :name "rspamadm confighelp"
+ :description "Shows help for the specified configuration options"
+ :help_description_margin(32)
+parser:flag "--no-color"
+ :description "Disable coloured output"
+parser:flag "--short"
+ :description "Show only option names"
+parser:flag "--no-examples"
+ :description "Do not show examples (impied by --short)"
local function maybe_print_color(key)
if not opts['no-color'] then
@@ -100,7 +110,7 @@ local function print_help(key, value, tabs)
end
return function(args, res)
- opts = getopt.getopt(args, '')
+ opts = parser:parse(args)
local sorted = sort_values(res)
diff --git a/lualib/rspamadm/configwizard.lua b/lualib/rspamadm/configwizard.lua
index 1cd275f66..10134c575 100644
--- a/lualib/rspamadm/configwizard.lua
+++ b/lualib/rspamadm/configwizard.lua
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
]]--
-local ansicolors = require "rspamadm/ansicolors"
+local ansicolors = require "ansicolors"
local local_conf = rspamd_paths['CONFDIR']
local rspamd_util = require "rspamd_util"
local rspamd_logger = require "rspamd_logger"
@@ -22,8 +22,9 @@ local lua_util = require "lua_util"
local lua_stat_tools = require "lua_stat"
local lua_redis = require "lua_redis"
local ucl = require "ucl"
+local argparse = require "argparse"
-local plugins_stat = require "rspamadm/plugins_stats"
+local plugins_stat = require "plugins_stats"
local rspamd_logo = [[
____ _
@@ -34,6 +35,19 @@ local rspamd_logo = [[
|_|
]]
+local parser = argparse()
+ :name "rspamadm configwizard"
+ :description "Perform guided configuration for Rspamd daemon"
+ :help_description_margin(32)
+parser:option "-c --config"
+ :description "Path to config file"
+ :argname("<file>")
+ :default(rspamd_paths["CONFDIR"] .. "/" .. "rspamd.conf")
+parser:argument "checks"
+ :description "Checks to do (or 'list')"
+ :argname("<checks>")
+ :args "*"
+
local redis_params
local function printf(fmt, ...)
@@ -578,109 +592,135 @@ end
-return function(args, cfg)
- local changes = {
- l = {}, -- local changes
- o = {}, -- override changes
- }
-
- local interactive_start = true
- local checks = {}
- local all_checks = {
- 'controller',
- 'redis',
- 'dkim',
- 'statistic',
- }
-
- if #args > 0 then
- interactive_start = false
-
- for _,arg in ipairs(args) do
- if arg == 'all' then
- checks = all_checks
- elseif arg == 'list' then
- printf(highlight(rspamd_logo))
- printf('Available modules')
- for _,c in ipairs(all_checks) do
- printf('- %s', c)
+return {
+ handler = function(cmd_args)
+ local changes = {
+ l = {}, -- local changes
+ o = {}, -- override changes
+ }
+
+ local interactive_start = true
+ local checks = {}
+ local all_checks = {
+ 'controller',
+ 'redis',
+ 'dkim',
+ 'statistic',
+ }
+
+ local opts = parser:parse(cmd_args)
+ local args = opts['checks'] or {}
+
+ local _r,err = rspamd_config:load_ucl(opts['config'])
+ local cfg = rspamd_config:get_ucl()
+
+ if not _r then
+ rspamd_logger.errx('cannot parse %s: %s', opts['config'], err)
+ os.exit(1)
+ end
+
+ _r,err = rspamd_config:parse_rcl({'logging', 'worker'})
+ if not _r then
+ rspamd_logger.errx('cannot process %s: %s', opts['config'], err)
+ os.exit(1)
+ end
+
+ if not rspamd_config:init_modules() then
+ rspamd_logger.errx('cannot init modules when parsing %s', opts['config'])
+ os.exit(1)
+ end
+
+ if #args > 0 then
+ interactive_start = false
+
+ for _,arg in ipairs(args) do
+ if arg == 'all' then
+ checks = all_checks
+ elseif arg == 'list' then
+ printf(highlight(rspamd_logo))
+ printf('Available modules')
+ for _,c in ipairs(all_checks) do
+ printf('- %s', c)
+ end
+ return
+ else
+ table.insert(checks, arg)
end
- return
- else
- table.insert(checks, arg)
end
+ else
+ checks = all_checks
end
- else
- checks = all_checks
- end
- local function has_check(check)
- for _,c in ipairs(checks) do
- if c == check then
- return true
+ local function has_check(check)
+ for _,c in ipairs(checks) do
+ if c == check then
+ return true
+ end
end
- end
- return false
- end
+ return false
+ end
- rspamd_util.umask('022')
- if interactive_start then
- printf(highlight(rspamd_logo))
- printf("Welcome to the configuration tool")
- printf("We use %s configuration file, writing results to %s",
- highlight(cfg.config_path), highlight(local_conf))
- plugins_stat(nil, nil)
- end
+ rspamd_util.umask('022')
+ if interactive_start then
+ printf(highlight(rspamd_logo))
+ printf("Welcome to the configuration tool")
+ printf("We use %s configuration file, writing results to %s",
+ highlight(opts['config']), highlight(local_conf))
+ plugins_stat(nil, nil)
+ end
- if not interactive_start or
- ask_yes_no("Do you wish to continue?", true) then
+ if not interactive_start or
+ ask_yes_no("Do you wish to continue?", true) then
- if has_check('controller') then
- local controller = find_worker(cfg, 'controller')
- if controller then
- setup_controller(controller, changes)
+ if has_check('controller') then
+ local controller = find_worker(cfg, 'controller')
+ if controller then
+ setup_controller(controller, changes)
+ end
end
- end
- if has_check('redis') then
- if not cfg.redis or (not cfg.redis.servers and not cfg.redis.read_servers) then
- setup_redis(cfg, changes)
+ if has_check('redis') then
+ if not cfg.redis or (not cfg.redis.servers and not cfg.redis.read_servers) then
+ setup_redis(cfg, changes)
+ else
+ redis_params = cfg.redis
+ end
else
redis_params = cfg.redis
end
- else
- redis_params = cfg.redis
- end
- if has_check('dkim') then
- if cfg.dkim_signing and not cfg.dkim_signing.domain then
- if ask_yes_no('Do you want to setup dkim signing feature?') then
- setup_dkim_signing(cfg, changes)
+ if has_check('dkim') then
+ if cfg.dkim_signing and not cfg.dkim_signing.domain then
+ if ask_yes_no('Do you want to setup dkim signing feature?') then
+ setup_dkim_signing(cfg, changes)
+ end
end
end
- end
-
- if has_check('statistic') or has_check('statistics') then
- setup_statistic(cfg, changes)
- end
- local nchanges = 0
- for _,_ in pairs(changes.l) do nchanges = nchanges + 1 end
- for _,_ in pairs(changes.o) do nchanges = nchanges + 1 end
+ if has_check('statistic') or has_check('statistics') then
+ setup_statistic(cfg, changes)
+ end
- if nchanges > 0 then
- print_changes(changes)
- if ask_yes_no("Apply changes?", true) then
- apply_changes(changes)
- printf("%d changes applied, the wizard is finished now", nchanges)
- printf("*** Please reload the Rspamd configuration ***")
+ local nchanges = 0
+ for _,_ in pairs(changes.l) do nchanges = nchanges + 1 end
+ for _,_ in pairs(changes.o) do nchanges = nchanges + 1 end
+
+ if nchanges > 0 then
+ print_changes(changes)
+ if ask_yes_no("Apply changes?", true) then
+ apply_changes(changes)
+ printf("%d changes applied, the wizard is finished now", nchanges)
+ printf("*** Please reload the Rspamd configuration ***")
+ else
+ printf("No changes applied, the wizard is finished now")
+ end
else
- printf("No changes applied, the wizard is finished now")
+ printf("No changes found, the wizard is finished now")
end
- else
- printf("No changes found, the wizard is finished now")
end
- end
-end
+ end,
+ name = 'configwizard',
+ description = parser._description,
+}
diff --git a/lualib/rspamadm/corpus_test.lua b/lualib/rspamadm/corpus_test.lua
index cd9f66155..07051a196 100644
--- a/lualib/rspamadm/corpus_test.lua
+++ b/lualib/rspamadm/corpus_test.lua
@@ -1,19 +1,50 @@
local rspamd_logger = require "rspamd_logger"
local ucl = require "ucl"
local lua_util = require "lua_util"
-local getopt = require "rspamadm/getopt"
+local argparse = require "argparse"
+
+local parser = argparse()
+ :name "rspamadm corpus_test"
+ :description "Create logs files from email corpus"
+ :help_description_margin(32)
+
+parser:option "-H --ham"
+ :description("Ham directory")
+ :argname("<dir>")
+parser:option "-S --spam"
+ :description("Spam directory")
+ :argname("<dir>")
+parser:option "-n --conns"
+ :description("Number of parallel connections")
+ :argname("<N>")
+ :convert(tonumber)
+ :default(10)
+parser:option "-o --output"
+ :description("Output file")
+ :argname("<file>")
+ :default('results.log')
+parser:option "-t --timeout"
+ :description("Timeout for client connections")
+ :argname("<sec>")
+ :convert(tonumber)
+ :default(60)
+parser:option "-c --connect"
+ :description("Connect to specific host")
+ :argname("<host>")
+ :default('localhost:11334')
+parser:option "-r --rspamc"
+ :description("Use specific rspamc path")
+ :argname("<path>")
+ :default('rspamc')
local HAM = "HAM"
local SPAM = "SPAM"
local opts
-local default_opts = {
- connect = 'localhost:11334',
-}
local function scan_email(n_parallel, path, timeout)
- local rspamc_command = string.format("rspamc --connect %s -j --compact -n %s -t %.3f %s",
- opts.connect, n_parallel, timeout, path)
+ local rspamc_command = string.format("%s --connect %s -j --compact -n %s -t %.3f %s",
+ opts.rspamc, opts.connect, n_parallel, timeout, path)
local result = assert(io.popen(rspamc_command))
result = result:read("*all")
return result
@@ -24,7 +55,8 @@ local function write_results(results, file)
local f = io.open(file, 'w')
for _, result in pairs(results) do
- local log_line = string.format("%s %.2f %s", result.type, result.score, result.action)
+ local log_line = string.format("%s %.2f %s",
+ result.type, result.score, result.action)
for _, sym in pairs(result.symbols) do
log_line = log_line .. " " .. sym
@@ -44,16 +76,16 @@ local function encoded_json_to_log(result)
-- Returns table containing score, action, list of symbols
local filtered_result = {}
- local parser = ucl.parser()
+ local ucl_parser = ucl.parser()
- local is_good, err = parser:parse_string(result)
+ local is_good, err = ucl_parser:parse_string(result)
if not is_good then
rspamd_logger.errx("Parser error: %1", err)
return nil
end
- result = parser:get_object()
+ result = ucl_parser:get_object()
filtered_result.score = result.score
if not result.action then
@@ -96,13 +128,12 @@ local function scan_results_to_logs(results, actual_email_type)
return logs
end
-return function(args, res)
- opts = default_opts
- opts = lua_util.override_defaults(opts, getopt.getopt(args, ''))
- local ham_directory = res['ham_directory']
- local spam_directory = res['spam_directory']
- local connections = res["connections"]
- local output = res["output_location"]
+local function handler(args)
+ opts = parser:parse(args)
+ local ham_directory = opts['ham_directory']
+ local spam_directory = opts['spam_directory']
+ local connections = opts["connections"]
+ local output = opts["output"]
local results = {}
@@ -112,7 +143,7 @@ return function(args, res)
if ham_directory then
rspamd_logger.messagex("Scanning ham corpus...")
- local ham_results = scan_email(connections, ham_directory, res["timeout"])
+ local ham_results = scan_email(connections, ham_directory, opts["timeout"])
ham_results = scan_results_to_logs(ham_results, HAM)
no_of_ham = #ham_results
@@ -124,7 +155,7 @@ return function(args, res)
if spam_directory then
rspamd_logger.messagex("Scanning spam corpus...")
- local spam_results = scan_email(connections, spam_directory, res.timeout)
+ local spam_results = scan_email(connections, spam_directory, opts.timeout)
spam_results = scan_results_to_logs(spam_results, SPAM)
no_of_spam = #spam_results
@@ -145,3 +176,10 @@ return function(args, res)
rspamd_logger.messagex("No of spam: %s", no_of_spam)
rspamd_logger.messagex("Messages/sec: %s", (total_msgs / elapsed_time))
end
+
+
+return {
+ name = 'corpus_test',
+ handler = handler,
+ description = parser._description
+} \ No newline at end of file
diff --git a/lualib/rspamadm/fuzzy_stat.lua b/lualib/rspamadm/fuzzy_stat.lua
index 05c2622bf..45e13bb60 100644
--- a/lualib/rspamadm/fuzzy_stat.lua
+++ b/lualib/rspamadm/fuzzy_stat.lua
@@ -1,6 +1,27 @@
local util = require "rspamd_util"
local opts = {}
+local argparse = require "argparse"
+local parser = argparse()
+ :name "rspamadm confighelp"
+ :description "Shows help for the specified configuration options"
+ :help_description_margin(32)
+parser:flag "--no-ips"
+ :description "No IPs stats"
+parser:flag "--no-keys"
+ :description "No keys stats"
+parser:flag "--short"
+ :description "Short output mode"
+parser:flag "-n --number"
+ :description "Disable numbers humanization"
+parser:option "-s --sort"
+ :description "Sort order"
+ :convert {
+ matched = "matched",
+ errors = "errors",
+ ip = "ip"
+ }
+
local function add_data(target, src)
for k,v in pairs(src) do
if k ~= 'ips' then
@@ -143,13 +164,11 @@ local function print_result(r)
return print_num(r)
end
-local getopt = require "rspamadm/getopt"
-
return function(args, res)
local res_ips = {}
local res_databases = {}
local wrk = res['workers']
- opts = getopt.getopt(args, '')
+ opts = parser:parse(args)
if wrk then
for _,pr in pairs(wrk) do
diff --git a/lualib/rspamadm/getopt.lua b/lualib/rspamadm/getopt.lua
deleted file mode 100644
index 20715c387..000000000
--- a/lualib/rspamadm/getopt.lua
+++ /dev/null
@@ -1,50 +0,0 @@
-local function insert_option(tab, name, value)
- if tab[name] then
- if type(tab[name]) == 'table' then
- table.insert(tab[name], value)
- else
- local old_val = tab[name]
- tab[name] = {
- old_val,
- value
- }
- end
- else
- tab[name] = value
- end
-end
-
-local function getopt(arg, options)
- local tab = {}
- for k, v in ipairs(arg) do
- if string.sub(v, 1, 2) == "--" then
- local x = string.find(v, "=", 1, true)
- if x then insert_option(tab, string.sub(v, 3, x - 1), string.sub(v, x + 1))
- else tab[string.sub(v, 3)] = true
- end
- elseif string.sub(v, 1, 1) == "-" then
- local y = 2
- local l = string.len(v)
- local jopt
- while (y <= l) do
- jopt = string.sub(v, y, y)
- if string.find(options, jopt, 1, true) then
- if y < l then
- insert_option(tab, jopt, string.sub(v, y + 1))
- y = l
- else
- insert_option(tab, jopt, arg[k + 1])
- end
- else
- tab[jopt] = true
- end
- y = y + 1
- end
- end
- end
- return tab
-end
-
-return {
- getopt = getopt
-}
diff --git a/lualib/rspamadm/grep.lua b/lualib/rspamadm/grep.lua
index a9d4b084a..b149a0337 100644
--- a/lualib/rspamadm/grep.lua
+++ b/lualib/rspamadm/grep.lua
@@ -1,6 +1,60 @@
-return function(_, res)
+--[[
+Copyright (c) 2018, Vsevolod Stakhov <vsevolod@highsecure.ru>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local argparse = require "argparse"
+
+
+-- Define command line options
+local parser = argparse()
+ :name "rspamadm grep"
+ :description "Search for patterns in rspamd logs"
+ :help_description_margin(30)
+parser:mutex(
+ parser:option "-s --string"
+ :description('Plain string to search (case-insensitive)')
+ :argname "<str>",
+ parser:option "-p --pattern"
+ :description('Pattern to search for (regex)')
+ :argname "<re>"
+)
+parser:flag "-l --lua"
+ :description('Use Lua patterns in string search')
+
+parser:argument "input":args "*"
+ :description('Process specified inputs')
+ :default("stdin")
+parser:flag "-S --sensitive"
+ :description('Enable case-sensitivity in string search')
+parser:flag "-o --orphans"
+ :description('Print orphaned logs')
+parser:flag "-P --partial"
+ :description('Print partial logs')
+
+local function handler(args)
local rspamd_regexp = require 'rspamd_regexp'
+ local res = parser:parse(args)
+
+ if not res['string'] and not res['pattern'] then
+ parser:error('string or pattern options must be specified')
+ end
+
+ if res['string'] and res['pattern'] then
+ parser:error('string and pattern options are mutually exclusive')
+ end
local buffer = {}
local matches = {}
@@ -16,7 +70,7 @@ return function(_, res)
end
local plainm = true
- if res['luapat'] then
+ if res['lua'] then
plainm = false
end
local orphans = res['orphans']
@@ -26,7 +80,7 @@ return function(_, res)
if search_str and not sensitive then
search_str = string.lower(search_str)
end
- local inputs = res['inputs']
+ local inputs = res['input'] or {'stdin'}
for _, n in ipairs(inputs) do
local h, err
@@ -110,3 +164,9 @@ return function(_, res)
end
end
end
+
+return {
+ handler = handler,
+ description = parser._description,
+ name = 'grep'
+} \ No newline at end of file
diff --git a/lualib/rspamadm/keypair.lua b/lualib/rspamadm/keypair.lua
new file mode 100644
index 000000000..518c9e65f
--- /dev/null
+++ b/lualib/rspamadm/keypair.lua
@@ -0,0 +1,269 @@
+--[[
+Copyright (c) 2018, Vsevolod Stakhov <vsevolod@highsecure.ru>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local argparse = require "argparse"
+local rspamd_keypair = require "rspamd_cryptobox_keypair"
+local rspamd_pubkey = require "rspamd_cryptobox_pubkey"
+local rspamd_signature = require "rspamd_cryptobox_signature"
+local rspamd_crypto = require "rspamd_cryptobox"
+local ucl = require "ucl"
+local logger = require "rspamd_logger"
+
+-- Define command line options
+local parser = argparse()
+ :name "rspamadm keypair"
+ :description "Manages keypairs for Rspamd"
+ :help_description_margin(30)
+ :command_target("command")
+ :require_command(false)
+
+local generate = parser:command "generate gen g"
+ :description "Creates a new keypair"
+generate:flag "-s --sign"
+ :description "Generates a sign keypair instead of the encryption one"
+generate:flag "-n --nist"
+ :description "Uses nist encryption algorithm"
+generate:option "-o --output"
+ :description "Write keypair to file"
+ :argname "<file>"
+generate:mutex(
+ generate:flag "-j --json"
+ :description "Output JSON instead of UCL",
+ generate:flag "-u --ucl"
+ :description "Output UCL"
+ :default(true)
+)
+
+local sign = parser:command "sign sig s"
+ :description "Signs a file using keypair"
+sign:option "-k --keypair"
+ :description "Keypair to use"
+ :argname "<file>"
+sign:option "-s --suffix"
+ :description "Suffix for signature"
+ :argname "<suffix>"
+ :default("sig")
+sign:argument "file"
+ :description "File to sign"
+ :argname "<file>"
+ :args "*"
+
+local verify = parser:command "verify ver v"
+ :description "Verifies a file using keypair or a public key"
+verify:mutex(
+ verify:option "-p --pubkey"
+ :description "Load pubkey from the specified file"
+ :argname "<file>",
+ verify:option "-P --pubstring"
+ :description "Load pubkey from the base32 encoded string"
+ :argname "<base32>",
+ verify:option "-k --keypair"
+ :description "Get pubkey from the keypair file"
+ :argname "<file>"
+)
+verify:argument "file"
+ :description "File to verify"
+ :argname "<file>"
+ :args "*"
+verify:flag "-n --nist"
+ :description "Uses nistp curves (P256)"
+verify:option "-s --suffix"
+ :description "Suffix for signature"
+ :argname "<suffix>"
+ :default("sig")
+
+-- Default command is generate, so duplicate options
+parser:flag "-s --sign"
+ :description "Generates a sign keypair instead of the encryption one"
+parser:flag "-n --nist"
+ :description "Uses nistp curves (P256)"
+parser:mutex(
+ parser:flag "-j --json"
+ :description "Output JSON instead of UCL",
+ parser:flag "-u --ucl"
+ :description "Output UCL"
+ :default(true)
+)
+parser:option "-o --output"
+ :description "Write keypair to file"
+ :argname "<file>"
+
+local function fatal(...)
+ logger.errx(...)
+ os.exit(1)
+end
+
+local function generate_handler(opts)
+ local mode = 'encryption'
+ if opts.sign then
+ mode = 'sign'
+ end
+ local alg = 'curve25519'
+ if opts.nist then
+ alg = 'nist'
+ end
+ -- TODO: probably, do it in a more safe way
+ local kp = rspamd_keypair.create(mode, alg):totable()
+
+ local format = 'ucl'
+
+ if opts.json then
+ format = 'json'
+ end
+
+ if opts.output then
+ local out = io.open(opts.output, 'w')
+ if not out then
+ fatal('cannot open output to write: ' .. opts.output)
+ end
+ out:write(ucl.to_format(kp, format))
+ out:close()
+ else
+ io.write(ucl.to_format(kp, format))
+ end
+end
+
+local function sign_handler(opts)
+ if opts.file then
+ if type(opts.file) == 'string' then
+ opts.file = {opts.file}
+ end
+ else
+ parser:error('no files to sign')
+ end
+ if not opts.keypair then
+ parser:error("no keypair specified")
+ end
+
+ local ucl_parser = ucl.parser()
+ local res,err = ucl_parser:parse_file(opts.keypair)
+
+ if not res then
+ fatal(string.format('cannot load %s: %s', opts.keypair, err))
+ end
+
+ local kp = rspamd_keypair.load(ucl_parser:get_object())
+
+ if not kp then
+ fatal("cannot load keypair: " .. opts.keypair)
+ end
+
+ for _,fname in ipairs(opts.file) do
+ local sig = rspamd_crypto.sign_file(kp, fname)
+
+ if not sig then
+ fatal(string.format("cannot sign %s\n", fname))
+ end
+
+ local out = string.format('%s.%s', fname, opts.suffix or 'sig')
+ local of = io.open(out, 'w')
+ if not of then
+ fatal('cannot open output to write: ' .. out)
+ end
+ of:write(sig:bin())
+ of:close()
+ io.write(string.format('signed %s -> %s (%s)\n', fname, out, sig:hex()))
+ end
+end
+
+local function verify_handler(opts)
+ if opts.file then
+ if type(opts.file) == 'string' then
+ opts.file = {opts.file}
+ end
+ else
+ parser:error('no files to verify')
+ end
+
+ local pk
+ local alg = 'curve25519'
+
+ if opts.keypair then
+ local ucl_parser = ucl.parser()
+ local res,err = ucl_parser:parse_file(opts.keypair)
+
+ if not res then
+ fatal(string.format('cannot load %s: %s', opts.keypair, err))
+ end
+
+ local kp = rspamd_keypair.load(ucl_parser:get_object())
+
+ if not kp then
+ fatal("cannot load keypair: " .. opts.keypair)
+ end
+
+ pk = kp:pk()
+ alg = kp:alg()
+ elseif opts.pubkey then
+ if opts.nist then alg = 'nist' end
+ pk = rspamd_pubkey.load(opts.pubkey, 'sign', alg)
+ elseif opts.pubstr then
+ if opts.nist then alg = 'nist' end
+ pk = rspamd_pubkey.create(opts.pubstr, 'sign', alg)
+ end
+
+ if not pk then
+ fatal("cannot create pubkey")
+ end
+
+ local valid = true
+
+ for _,fname in ipairs(opts.file) do
+
+ local sig_fname = string.format('%s.%s', fname, opts.suffix or 'sig')
+ local sig = rspamd_signature.load(sig_fname, alg)
+
+ if not sig then
+ fatal(string.format("cannot load signature for %s -> %s",
+ fname, sig_fname))
+ end
+
+ if rspamd_crypto.verify_file(pk, sig, fname, alg) then
+ io.write(string.format('verified %s -> %s (%s)\n', fname, sig_fname, sig:hex()))
+ else
+ valid = false
+ io.write(string.format('FAILED to verify %s -> %s (%s)\n', fname,
+ sig_fname, sig:hex()))
+ end
+ end
+
+ if not valid then
+ os.exit(1)
+ end
+end
+
+local function handler(args)
+ local opts = parser:parse(args)
+
+ local command = opts.command or "generate"
+
+ if command == 'generate' then
+ generate_handler(opts)
+ elseif command == 'sign' then
+ sign_handler(opts)
+ elseif command == 'verify' then
+ verify_handler(opts)
+ else
+ parser:error('command %s is not yet implemented', command)
+ end
+end
+
+return {
+ name = 'keypair',
+ aliases = {'kp', 'key'},
+ handler = handler,
+ description = parser._description
+} \ No newline at end of file
diff --git a/lualib/rspamadm/rescore.lua b/lualib/rspamadm/rescore.lua
index 231cf2ea6..cc331c6e8 100644
--- a/lualib/rspamadm/rescore.lua
+++ b/lualib/rspamadm/rescore.lua
@@ -1,13 +1,32 @@
+--[[
+Copyright (c) 2018, Vsevolod Stakhov <vsevolod@highsecure.ru>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+if not rspamd_config:has_torch() then
+ return
+end
+
local torch = require "torch"
local nn = require "nn"
local lua_util = require "lua_util"
local ucl = require "ucl"
local logger = require "rspamd_logger"
-local getopt = require "rspamadm/getopt"
local optim = require "optim"
local rspamd_util = require "rspamd_util"
-
-local rescore_utility = require "rspamadm/rescore_utility"
+local argparse = require "argparse"
+local rescore_utility = require "rescore_utility"
local opts
local ignore_symbols = {
@@ -15,6 +34,104 @@ local ignore_symbols = {
['DATE_IN_FUTURE'] = true,
}
+local parser = argparse()
+ :name "rspamadm rescore"
+ :description "Estimate optimal symbol weights from log files"
+ :help_description_margin(37)
+
+parser:option "-l --log"
+ :description "Log file or files (from rescore)"
+ :argname("<log>")
+ :args "*"
+parser:option "-c --config"
+ :description "Path to config file"
+ :argname("<file>")
+ :default(rspamd_paths["CONFDIR"] .. "/" .. "rspamd.conf")
+parser:option "-o --output"
+ :description "Output file"
+ :argname("<file>")
+ :default("new.scores")
+parser:flag "-d --diff"
+ :description "Show differences in scores"
+parser:flag "-v --verbose"
+ :description "Verbose output"
+parser:flag "-z --freq"
+ :description "Display hit frequencies"
+parser:option "-i --iters"
+ :description "Learn iterations"
+ :argname("<n>")
+ :convert(tonumber)
+ :default(10)
+parser:option "-b --batch"
+ :description "Batch size"
+ :argname("<n>")
+ :convert(tonumber)
+ :default(100)
+parser:option "-d --decay"
+ :description "Decay rate"
+ :argname("<n>")
+ :convert(tonumber)
+ :default(0.001)
+parser:option "-m --momentum"
+ :description "Learn momentum"
+ :argname("<n>")
+ :convert(tonumber)
+ :default(0.1)
+parser:option "-t --threads"
+ :description "Number of threads to use"
+ :argname("<n>")
+ :convert(tonumber)
+ :default(1)
+parser:option "-o --optim"
+ :description "Optimisation algorithm"
+ :argname("<alg>")
+ :convert {
+ LBFGS = "LBFGS",
+ ADAM = "ADAM",
+ ADAGRAD = "ADAGRAD",
+ SGD = "SGD",
+ NAG = "NAG"
+ }
+ :default "ADAM"
+parser:option "--ignore-symbol"
+ :description "Ignore symbol from logs"
+ :argname("<sym>")
+ :args "*"
+parser:option "--penalty-weight"
+ :description "Add new penalty weight to test"
+ :argname("<n>")
+ :convert(tonumber)
+ :args "*"
+parser:option "--learning-rate"
+ :description "Add new learning rate to test"
+ :argname("<n>")
+ :convert(tonumber)
+ :args "*"
+parser:option "--spam_action"
+ :description "Spam action"
+ :argname("<act>")
+ :default("reject")
+parser:option "--learning_rate_decay"
+ :description "Learn rate decay (for some algs)"
+ :argname("<n>")
+ :convert(tonumber)
+ :default(0.0)
+parser:option "--weight_decay"
+ :description "Weight decay (for some algs)"
+ :argname("<n>")
+ :convert(tonumber)
+ :default(0.0)
+parser:option "--l1"
+ :description "L1 regularization penalty"
+ :argname("<n>")
+ :convert(tonumber)
+ :default(0.0)
+parser:option "--l2"
+ :description "L2 regularization penalty"
+ :argname("<n>")
+ :convert(tonumber)
+ :default(0.0)
+
local function make_dataset_from_logs(logs, all_symbols, spam_score)
-- Returns a list of {input, output} for torch SGD train
@@ -71,17 +188,18 @@ local function init_weights(all_symbols, original_symbol_scores)
return weights
end
-local function shuffle(logs)
+local function shuffle(logs, messages)
local size = #logs
for i = size, 1, -1 do
local rand = math.random(size)
logs[i], logs[rand] = logs[rand], logs[i]
+ messages[i], messages[rand] = messages[rand], messages[i]
end
end
-local function split_logs(logs, split_percent)
+local function split_logs(logs, messages, split_percent)
if not split_percent then
split_percent = 60
@@ -91,16 +209,20 @@ local function split_logs(logs, split_percent)
local test_logs = {}
local train_logs = {}
+ local test_messages = {}
+ local train_messages = {}
for i=1,split_index do
- train_logs[#train_logs + 1] = logs[i]
+ table.insert(train_logs, logs[i])
+ table.insert(train_messages, messages[i])
end
for i=split_index + 1, #logs do
- test_logs[#test_logs + 1] = logs[i]
+ table.insert(test_logs, logs[i])
+ table.insert(test_messages, messages[i])
end
- return train_logs, test_logs
+ return {train_logs,train_messages}, {test_logs,test_messages}
end
local function stitch_new_scores(all_symbols, new_scores)
@@ -174,7 +296,10 @@ local function print_score_diff(new_symbol_scores, original_symbol_scores)
end
-local function calculate_fscore_from_weights(logs, all_symbols, weights, threshold)
+local function calculate_fscore_from_weights(logs, messages,
+ all_symbols,
+ weights,
+ threshold)
local new_symbol_scores = weights:clone()
@@ -182,14 +307,16 @@ local function calculate_fscore_from_weights(logs, all_symbols, weights, thresho
logs = update_logs(logs, new_symbol_scores)
- local file_stats, _, all_fps, all_fns = rescore_utility.generate_statistics_from_logs(logs, threshold)
+ local file_stats, _, all_fps, all_fns =
+ rescore_utility.generate_statistics_from_logs(logs, messages, threshold)
return file_stats.fscore, all_fps, all_fns
end
-local function print_stats(logs, threshold)
+local function print_stats(logs, messages, threshold)
- local file_stats, _ = rescore_utility.generate_statistics_from_logs(logs, threshold)
+ local file_stats, _ = rescore_utility.generate_statistics_from_logs(logs,
+ messages, threshold)
local file_stat_format = [[
F-score: %.2f
@@ -226,7 +353,7 @@ local function train(dataset, opt, model, criterion, epoch,
local lbfgsState
local sgdState
- local batch_size = opt.batch_size
+ local batch_size = opt.batch
logger.messagex("trainer epoch #%s, %s batch", epoch, batch_size)
@@ -300,7 +427,7 @@ local function train(dataset, opt, model, criterion, epoch,
end
-- optimize on current mini-batch
- if opt.optimization == 'LBFGS' then
+ if opt.optim == 'LBFGS' then
-- Perform LBFGS step:
lbfgsState = lbfgsState or {
@@ -315,7 +442,7 @@ local function train(dataset, opt, model, criterion, epoch,
logger.messagex(' - nb of iterations: ' .. lbfgsState.nIter)
logger.messagex(' - nb of function evalutions: ' .. lbfgsState.funcEval)
- elseif opt.optimization == 'ADAM' then
+ elseif opt.optim == 'ADAM' then
sgdState = sgdState or {
learningRate = tonumber(opts.learning_rate),-- opt.learningRate,
momentum = tonumber(opts.momentum), -- opt.momentum,
@@ -323,7 +450,7 @@ local function train(dataset, opt, model, criterion, epoch,
weightDecay = tonumber(opts.weight_decay),
}
optim.adam(feval, parameters, sgdState)
- elseif opt.optimization == 'ADAGRAD' then
+ elseif opt.optim == 'ADAGRAD' then
sgdState = sgdState or {
learningRate = tonumber(opts.learning_rate),-- opt.learningRate,
momentum = tonumber(opts.momentum), -- opt.momentum,
@@ -331,7 +458,7 @@ local function train(dataset, opt, model, criterion, epoch,
weightDecay = tonumber(opts.weight_decay),
}
optim.adagrad(feval, parameters, sgdState)
- elseif opt.optimization == 'SGD' then
+ elseif opt.optim == 'SGD' then
sgdState = sgdState or {
learningRate = tonumber(opts.learning_rate),-- opt.learningRate,
momentum = tonumber(opts.momentum), -- opt.momentum,
@@ -339,7 +466,7 @@ local function train(dataset, opt, model, criterion, epoch,
weightDecay = tonumber(opts.weight_decay),
}
optim.sgd(feval, parameters, sgdState)
- elseif opt.optimization == 'NAG' then
+ elseif opt.optim == 'NAG' then
sgdState = sgdState or {
learningRate = tonumber(opts.learning_rate),-- opt.learningRate,
momentum = tonumber(opts.momentum), -- opt.momentum,
@@ -348,7 +475,8 @@ local function train(dataset, opt, model, criterion, epoch,
}
optim.nag(feval, parameters, sgdState)
else
- error('unknown optimization method')
+ logger.errx('unknown optimization method: %s', opt.optim)
+ os.exit(1)
end
end
@@ -363,19 +491,6 @@ local function train(dataset, opt, model, criterion, epoch,
confusion:zero()
end
-
-local default_opts = {
- verbose = true,
- iters = 10,
- threads = 1,
- batch_size = 1000,
- optimization = 'ADAM',
- learning_rate_decay = 0.001,
- momentum = 0.1,
- l1 = 0.0,
- l2 = 0.0,
-}
-
local learning_rates = {
0.01
}
@@ -393,11 +508,27 @@ local function get_threshold()
or actions['reject']), actions['reject']
end
-return function (args, cfg)
- opts = default_opts
- opts = lua_util.override_defaults(opts, getopt.getopt(args, 'i:'))
+local function handler(args)
+ opts = parser:parse(args)
+ if not opts['log'] then
+ parser:error('no log specified')
+ end
+
+ local _r,err = rspamd_config:load_ucl(opts['config'])
+
+ if not _r then
+ logger.errx('cannot parse %s: %s', opts['config'], err)
+ os.exit(1)
+ end
+
+ _r,err = rspamd_config:parse_rcl({'logging', 'worker'})
+ if not _r then
+ logger.errx('cannot process %s: %s', opts['config'], err)
+ os.exit(1)
+ end
+
local threshold,reject_score = get_threshold()
- local logs = rescore_utility.get_all_logs(cfg["logdir"])
+ local logs,messages = rescore_utility.get_all_logs(opts['log'])
if opts['ignore-symbol'] then
local function add_ignore(s)
@@ -446,15 +577,15 @@ return function (args, cfg)
end
end
- if opts['i'] then opts['iters'] = opts['i'] end
-
local all_symbols = rescore_utility.get_all_symbols(logs, ignore_symbols)
local original_symbol_scores = rescore_utility.get_all_symbol_scores(rspamd_config,
ignore_symbols)
-- Display hit frequencies
- if opts['z'] then
- local _, all_symbols_stats = rescore_utility.generate_statistics_from_logs(logs, threshold)
+ if opts['freq'] then
+ local _, all_symbols_stats = rescore_utility.generate_statistics_from_logs(logs,
+ messages,
+ threshold)
local t = {}
for _, symbol_stats in pairs(all_symbols_stats) do table.insert(t, symbol_stats) end
@@ -473,8 +604,8 @@ return function (args, cfg)
"NAME", "HITS", "HAM", "HAM%", "SPAM", "SPAM%", "S/O", "OVER%"))
for _, symbol_stats in pairs(t) do
logger.message(
- string.format("%-40s %6d %6d %6.2f %6d %6.2f %6.2f %6.2f",
- symbol_stats.name,
+ string.format("%-40s %6d %6d %6.2f %6d %6.2f %6.2f %6.2f",
+ symbol_stats.name,
symbol_stats.no_of_hits,
symbol_stats.ham_hits,
lua_util.round(symbol_stats.ham_percent,2),
@@ -487,7 +618,7 @@ return function (args, cfg)
end
-- Print file statistics
- print_stats(logs, threshold)
+ print_stats(logs, messages, threshold)
-- Work out how many symbols weren't seen in the corpus
local symbols_no_hits = {}
@@ -503,7 +634,7 @@ return function (args, cfg)
-- Calculate percentage of rules with no hits
local nhpct = lua_util.round((#symbols_no_hits/total_symbols)*100,2)
logger.message(
- string.format('\nFound %s (%-.2f%%) symbols out of %s with no hits in corpus:',
+ string.format('\nFound %s (%-.2f%%) symbols out of %s with no hits in corpus:',
#symbols_no_hits, nhpct, total_symbols
)
)
@@ -515,13 +646,13 @@ return function (args, cfg)
return
end
- shuffle(logs)
+ shuffle(logs, messages)
torch.setdefaulttensortype('torch.FloatTensor')
- local train_logs, validation_logs = split_logs(logs, 70)
- local cv_logs, test_logs = split_logs(validation_logs, 50)
+ local train_logs, validation_logs = split_logs(logs, messages,70)
+ local cv_logs, test_logs = split_logs(validation_logs[1], validation_logs[2], 50)
- local dataset = make_dataset_from_logs(train_logs, all_symbols, reject_score)
+ local dataset = make_dataset_from_logs(train_logs[1], all_symbols, reject_score)
-- Start of perceptron training
local input_size = #all_symbols
@@ -555,7 +686,8 @@ return function (args, cfg)
initial_weights)
end
- local fscore, fps, fns = calculate_fscore_from_weights(cv_logs,
+ local fscore, fps, fns = calculate_fscore_from_weights(cv_logs[1],
+ cv_logs[2],
all_symbols,
linear_module.weight[1],
threshold)
@@ -580,23 +712,23 @@ return function (args, cfg)
new_symbol_scores = stitch_new_scores(all_symbols, new_symbol_scores)
- if cfg["output"] then
- write_scores(new_symbol_scores, cfg["output"])
+ if opts["output"] then
+ write_scores(new_symbol_scores, opts["output"])
end
- if cfg["diff"] then
+ if opts["diff"] then
print_score_diff(new_symbol_scores, original_symbol_scores)
end
-- Pre-rescore test stats
logger.message("\n\nPre-rescore test stats\n")
- test_logs = update_logs(test_logs, original_symbol_scores)
- print_stats(test_logs, threshold)
+ test_logs[1] = update_logs(test_logs[1], original_symbol_scores)
+ print_stats(test_logs[1], test_logs[2], threshold)
-- Post-rescore test stats
- test_logs = update_logs(test_logs, new_symbol_scores)
+ test_logs[1] = update_logs(test_logs[1], new_symbol_scores)
logger.message("\n\nPost-rescore test stats\n")
- print_stats(test_logs, threshold)
+ print_stats(test_logs[1], test_logs[2], threshold)
logger.messagex('Best fscore=%s, best learning rate=%s, best weight decay=%s',
best_fscore, best_learning_rate, best_weight_decay)
@@ -616,3 +748,10 @@ return function (args, cfg)
end
end
end
+
+
+return {
+ handler = handler,
+ description = parser._description,
+ name = 'rescore'
+} \ No newline at end of file
diff --git a/rules/regexp/headers.lua b/rules/regexp/headers.lua
index 5658608fb..a7300e2b8 100644
--- a/rules/regexp/headers.lua
+++ b/rules/regexp/headers.lua
@@ -589,11 +589,19 @@ reconf['YANDEX_RU_MAILER'] = {
-- Detect 1C v8.2 and v8.3 mailers
reconf['MAILER_1C_8'] = {
re = 'X-Mailer=/^1C:Enterprise 8\\.[23]$/H',
- score = 0,
+ score = 0.0,
description = 'Sent with 1C:Enterprise 8',
group = 'header'
}
+-- Detect rogue 'strongmail' MTA with IPv4 and '(-)' in Received line
+reconf['STRONGMAIL'] = {
+ re = [[Received=/^from\s+strongmail\s+\(\[\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\]\) by \S+ \(-\); /mH]],
+ score = 6.0,
+ description = 'Sent via rogue "strongmail" MTA',
+ group = 'header'
+}
+
-- Two received headers with ip addresses
local double_ip_spam_1 = 'Received=/from \\[\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\] by \\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3} with/H'
local double_ip_spam_2 = 'Received=/from\\s+\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s+by\\s+\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3};/H'
diff --git a/src/libcryptobox/cryptobox.c b/src/libcryptobox/cryptobox.c
index c0857b094..99c91e3dd 100644
--- a/src/libcryptobox/cryptobox.c
+++ b/src/libcryptobox/cryptobox.c
@@ -560,6 +560,10 @@ rspamd_cryptobox_sign (guchar *sig, gsize *siglen_p,
&diglen, kinv, rp, lk) == 1);
g_assert (diglen <= sizeof (rspamd_signature_t));
+ if (siglen_p) {
+ *siglen_p = diglen;
+ }
+
EC_KEY_free (lk);
EVP_MD_CTX_destroy (sha_ctx);
BN_free (bn_sec);
@@ -572,6 +576,7 @@ rspamd_cryptobox_sign (guchar *sig, gsize *siglen_p,
bool
rspamd_cryptobox_verify (const guchar *sig,
+ gsize siglen,
const guchar *m,
gsize mlen,
const rspamd_pk_t pk,
@@ -580,6 +585,7 @@ rspamd_cryptobox_verify (const guchar *sig,
bool ret = false;
if (G_LIKELY (mode == RSPAMD_CRYPTOBOX_MODE_25519)) {
+ g_assert (siglen == rspamd_cryptobox_signature_bytes (RSPAMD_CRYPTOBOX_MODE_25519));
ret = ed25519_verify (sig, m, mlen, pk);
}
else {
@@ -608,8 +614,7 @@ rspamd_cryptobox_verify (const guchar *sig,
g_assert (EC_KEY_set_public_key (lk, ec_pub) == 1);
/* ECDSA */
- ret = ECDSA_verify (0, h, sizeof (h), sig,
- rspamd_cryptobox_signature_bytes (mode), lk) == 1;
+ ret = ECDSA_verify (0, h, sizeof (h), sig, siglen, lk) == 1;
EC_KEY_free (lk);
EVP_MD_CTX_destroy (sha_ctx);
diff --git a/src/libcryptobox/cryptobox.h b/src/libcryptobox/cryptobox.h
index 1045547a2..1d48c06e5 100644
--- a/src/libcryptobox/cryptobox.h
+++ b/src/libcryptobox/cryptobox.h
@@ -40,7 +40,7 @@ struct rspamd_cryptobox_segment {
#define rspamd_cryptobox_HASHSTATEBYTES 256
#define rspamd_cryptobox_MAX_SIGSKBYTES 64
#define rspamd_cryptobox_MAX_SIGPKBYTES 32
-#define rspamd_cryptobox_MAX_SIGBYTES 64
+#define rspamd_cryptobox_MAX_SIGBYTES 72
#define CPUID_AVX2 0x1
#define CPUID_AVX 0x2
@@ -214,6 +214,7 @@ void rspamd_cryptobox_sign (guchar *sig, gsize *siglen_p,
* @return true if signature is valid, false otherwise
*/
bool rspamd_cryptobox_verify (const guchar *sig,
+ gsize siglen,
const guchar *m,
gsize mlen,
const rspamd_pk_t pk,
diff --git a/src/libcryptobox/keypair.c b/src/libcryptobox/keypair.c
index 50e3614d9..21b497130 100644
--- a/src/libcryptobox/keypair.c
+++ b/src/libcryptobox/keypair.c
@@ -881,7 +881,7 @@ rspamd_keypair_verify (struct rspamd_cryptobox_pubkey *pk,
return FALSE;
}
- if (!rspamd_cryptobox_verify (sig, data, len,
+ if (!rspamd_cryptobox_verify (sig, siglen, data, len,
rspamd_cryptobox_pubkey_pk (pk, &pklen), pk->alg)) {
g_set_error (err, rspamd_keypair_quark (), EPERM,
"signature verification failed");
diff --git a/src/libmime/email_addr.c b/src/libmime/email_addr.c
index 08071afc8..a0ad9ac19 100644
--- a/src/libmime/email_addr.c
+++ b/src/libmime/email_addr.c
@@ -203,10 +203,12 @@ rspamd_email_address_from_mime (rspamd_mempool_t *pool,
struct rspamd_email_address addr;
const gchar *p = hdr, *end = hdr + len, *c = hdr, *t;
GString *ns;
+ gint obraces, ebraces;
enum {
parse_name = 0,
parse_quoted,
parse_addr,
+ skip_comment,
skip_spaces
} state = parse_name, next_state = parse_name;
@@ -290,6 +292,23 @@ rspamd_email_address_from_mime (rspamd_mempool_t *pool,
else if (*p == '@') {
seen_at = TRUE;
}
+ else if (*p == '(') {
+ if (p > c) {
+ t = p - 1;
+
+ while (t > c && g_ascii_isspace (*t)) {
+ t --;
+ }
+
+ g_string_append_len (ns, c, t - c + 1);
+ }
+
+ c = p;
+ obraces = 1;
+ ebraces = 0;
+ state = skip_comment;
+ next_state = parse_name;
+ }
p ++;
break;
case parse_quoted:
@@ -331,6 +350,12 @@ rspamd_email_address_from_mime (rspamd_mempool_t *pool,
else if (*p == '@') {
seen_at = TRUE;
}
+ else if (*p == '(') {
+ obraces = 1;
+ ebraces = 0;
+ state = skip_comment;
+ next_state = parse_addr;
+ }
p ++;
break;
case skip_spaces:
@@ -342,6 +367,33 @@ rspamd_email_address_from_mime (rspamd_mempool_t *pool,
p ++;
}
break;
+ case skip_comment:
+ if (*p == '(') {
+ obraces ++;
+ }
+ else if (*p == ')') {
+ ebraces --;
+ }
+
+ if (obraces == ebraces) {
+ if (next_state == parse_name) {
+ /* Include comment in name */
+ if (p > c) {
+ t = p - 1;
+
+ while (t > c && g_ascii_isspace (*t)) {
+ t --;
+ }
+
+ g_string_append_len (ns, c, t - c + 1);
+ }
+
+ c = p;
+ }
+ state = next_state;
+ }
+ p ++;
+ break;
}
}
@@ -396,7 +448,8 @@ rspamd_email_address_from_mime (rspamd_mempool_t *pool,
}
break;
case parse_quoted:
- /* Unfinished quoted string */
+ case skip_comment:
+ /* Unfinished quoted string or a comment */
break;
default:
/* Do nothing */
diff --git a/src/libmime/message.c b/src/libmime/message.c
index 5681c3d00..e5e43c5be 100644
--- a/src/libmime/message.c
+++ b/src/libmime/message.c
@@ -907,13 +907,10 @@ rspamd_message_from_data (struct rspamd_task *task, const guchar *start,
gboolean
rspamd_message_parse (struct rspamd_task *task)
{
- struct rspamd_mime_text_part *p1, *p2;
struct received_header *recv, *trecv;
const gchar *p;
gsize len;
guint i;
- gdouble diff, *pdiff;
- guint tw, *ptw, dw;
GError *err = NULL;
rspamd_cryptobox_hash_state_t st;
guchar digest_out[rspamd_cryptobox_HASHBYTES];
@@ -1016,16 +1013,6 @@ rspamd_message_parse (struct rspamd_task *task)
task->queue_id = "undef";
}
- for (i = 0; i < task->parts->len; i ++) {
- struct rspamd_mime_part *part;
-
- part = g_ptr_array_index (task->parts, i);
- rspamd_message_process_text_part (task, part);
- }
-
- rspamd_images_process (task);
- rspamd_archives_process (task);
-
if (task->received->len > 0) {
gboolean need_recv_correction = FALSE;
rspamd_inet_addr_t *raddr;
@@ -1130,6 +1117,50 @@ rspamd_message_parse (struct rspamd_task *task)
rspamd_url_task_subject_callback, task);
}
+ for (i = 0; i < task->parts->len; i ++) {
+ struct rspamd_mime_part *part;
+
+ part = g_ptr_array_index (task->parts, i);
+ rspamd_cryptobox_hash_update (&st, part->digest, sizeof (part->digest));
+ }
+
+ rspamd_cryptobox_hash_final (&st, digest_out);
+ memcpy (task->digest, digest_out, sizeof (task->digest));
+
+ if (task->queue_id) {
+ msg_info_task ("loaded message; id: <%s>; queue-id: <%s>; size: %z; "
+ "checksum: <%*xs>",
+ task->message_id, task->queue_id, task->msg.len,
+ (gint)sizeof (task->digest), task->digest);
+ }
+ else {
+ msg_info_task ("loaded message; id: <%s>; size: %z; "
+ "checksum: <%*xs>",
+ task->message_id, task->msg.len,
+ (gint)sizeof (task->digest), task->digest);
+ }
+
+ return TRUE;
+}
+
+void
+rspamd_message_process (struct rspamd_task *task)
+{
+ guint i;
+ struct rspamd_mime_text_part *p1, *p2;
+ gdouble diff, *pdiff;
+ guint tw, *ptw, dw;
+
+ for (i = 0; i < task->parts->len; i ++) {
+ struct rspamd_mime_part *part;
+
+ part = g_ptr_array_index (task->parts, i);
+ rspamd_message_process_text_part (task, part);
+ }
+
+ rspamd_images_process (task);
+ rspamd_archives_process (task);
+
/* Calculate distance for 2-parts messages */
if (task->text_parts->len == 2) {
p1 = g_ptr_array_index (task->text_parts, 0);
@@ -1144,7 +1175,7 @@ rspamd_message_parse (struct rspamd_task *task)
if (rspamd_ftok_cmp (&p1->mime_part->parent_part->ct->subtype, &srch) == 0) {
if (!IS_PART_EMPTY (p1) && !IS_PART_EMPTY (p2) &&
- p1->normalized_hashes && p2->normalized_hashes) {
+ p1->normalized_hashes && p2->normalized_hashes) {
/*
* We also detect language on one part and propagate it to
* another one
@@ -1219,13 +1250,6 @@ rspamd_message_parse (struct rspamd_task *task)
}
}
- for (i = 0; i < task->parts->len; i ++) {
- struct rspamd_mime_part *part;
-
- part = g_ptr_array_index (task->parts, i);
- rspamd_cryptobox_hash_update (&st, part->digest, sizeof (part->digest));
- }
-
/* Calculate average words length and number of short words */
struct rspamd_mime_text_part *text_part;
gdouble *var;
@@ -1258,24 +1282,6 @@ rspamd_message_parse (struct rspamd_task *task)
*var /= (double)total_words;
}
}
-
- rspamd_cryptobox_hash_final (&st, digest_out);
- memcpy (task->digest, digest_out, sizeof (task->digest));
-
- if (task->queue_id) {
- msg_info_task ("loaded message; id: <%s>; queue-id: <%s>; size: %z; "
- "checksum: <%*xs>",
- task->message_id, task->queue_id, task->msg.len,
- (gint)sizeof (task->digest), task->digest);
- }
- else {
- msg_info_task ("loaded message; id: <%s>; size: %z; "
- "checksum: <%*xs>",
- task->message_id, task->msg.len,
- (gint)sizeof (task->digest), task->digest);
- }
-
- return TRUE;
}
diff --git a/src/libmime/message.h b/src/libmime/message.h
index 0ed2a5c66..b16011666 100644
--- a/src/libmime/message.h
+++ b/src/libmime/message.h
@@ -143,6 +143,12 @@ struct received_header {
gboolean rspamd_message_parse (struct rspamd_task *task);
/**
+ * Process content in task (e.g. HTML parsing)
+ * @param task
+ */
+void rspamd_message_process (struct rspamd_task *task);
+
+/**
* Get an array of header's values with specified header's name using raw headers
* @param task worker task structure
* @param field header's name
diff --git a/src/libserver/cfg_file.h b/src/libserver/cfg_file.h
index 520c847f3..777340e16 100644
--- a/src/libserver/cfg_file.h
+++ b/src/libserver/cfg_file.h
@@ -313,6 +313,7 @@ struct rspamd_config {
gboolean enable_experimental; /**< Enable experimental plugins */
gboolean disable_pcre_jit; /**< Disable pcre JIT */
gboolean disable_lua_squeeze; /**< Disable lua rules squeezing */
+ gboolean own_lua_state; /**< True if we have created lua_state internally */
gsize max_diff; /**< maximum diff size for text parts */
gsize max_cores_size; /**< maximum size occupied by rspamd core files */
@@ -445,11 +446,16 @@ struct rspamd_config {
gboolean rspamd_parse_bind_line (struct rspamd_config *cfg,
struct rspamd_worker_conf *cf, const gchar *str);
+
+enum rspamd_config_init_flags {
+ RSPAMD_CONFIG_INIT_DEFAULT = 0,
+ RSPAMD_CONFIG_INIT_SKIP_LUA = (1 << 0)
+};
/**
* Init default values
* @param cfg config file
*/
-struct rspamd_config *rspamd_config_new (void);
+struct rspamd_config *rspamd_config_new (enum rspamd_config_init_flags flags);
/**
* Free memory used by config structure
diff --git a/src/libserver/cfg_rcl.c b/src/libserver/cfg_rcl.c
index deb4a8ed3..20d51b31c 100644
--- a/src/libserver/cfg_rcl.c
+++ b/src/libserver/cfg_rcl.c
@@ -34,10 +34,6 @@
#include <syslog.h>
#endif
-#ifdef HAVE_GLOB_H
-#include <glob.h>
-#endif
-
#include <math.h>
struct rspamd_rcl_default_handler_data {
@@ -627,320 +623,6 @@ rspamd_rcl_worker_handler (rspamd_mempool_t *pool, const ucl_object_t *obj,
return TRUE;
}
-#define RSPAMD_CONFDIR_INDEX "CONFDIR"
-#define RSPAMD_RUNDIR_INDEX "RUNDIR"
-#define RSPAMD_DBDIR_INDEX "DBDIR"
-#define RSPAMD_LOGDIR_INDEX "LOGDIR"
-#define RSPAMD_PLUGINSDIR_INDEX "PLUGINSDIR"
-#define RSPAMD_RULESDIR_INDEX "RULESDIR"
-#define RSPAMD_LUALIBDIR_INDEX "LUALIBDIR"
-#define RSPAMD_WWWDIR_INDEX "WWWDIR"
-#define RSPAMD_PREFIX_INDEX "PREFIX"
-#define RSPAMD_VERSION_INDEX "VERSION"
-
-static gint
-rspamd_rcl_cmp_components (const gchar *comp1, const gchar *comp2)
-{
- guint v1, v2;
-
- v1 = strtoul (comp1, NULL, 10);
- v2 = strtoul (comp2, NULL, 10);
-
- return v1 - v2;
-}
-
-static int
-rspamd_rcl_lua_version_cmp (lua_State *L)
-{
- const gchar *ver;
- gchar **components;
- gint ret = 0;
-
- if (lua_type (L, 2) == LUA_TSTRING) {
- ver = lua_tostring (L, 2);
-
- components = g_strsplit_set (ver, ".-_", -1);
-
- if (!components) {
- return luaL_error (L, "invalid arguments to 'cmp': %s", ver);
- }
-
- if (components[0]) {
- ret = rspamd_rcl_cmp_components (components[0], RSPAMD_VERSION_MAJOR);
- }
-
- if (ret) {
- goto set;
- }
-
- if (components[1]) {
- ret = rspamd_rcl_cmp_components (components[1], RSPAMD_VERSION_MINOR);
- }
-
- if (ret) {
- goto set;
- }
-
- if (components[2]) {
- ret = rspamd_rcl_cmp_components (components[2], RSPAMD_VERSION_PATCH);
- }
-
- /*
- * XXX: we don't compare git releases assuming that it is meaningless
- */
- }
- else {
- return luaL_error (L, "invalid arguments to 'cmp'");
- }
-
-set:
- g_strfreev (components);
- lua_pushnumber (L, ret);
-
- return 1;
-}
-
-static int
-rspamd_rcl_lua_version_numeric (lua_State *L)
-{
- static gint64 version_num = RSPAMD_VERSION_NUM;
- const gchar *type;
-
- if (lua_gettop (L) >= 2 && lua_type (L, 1) == LUA_TSTRING) {
- type = lua_tostring (L, 1);
- if (g_ascii_strcasecmp (type, "short") == 0) {
- version_num = RSPAMD_VERSION_MAJOR_NUM * 1000 +
- RSPAMD_VERSION_MINOR_NUM * 100 + RSPAMD_VERSION_PATCH_NUM * 10;
- }
- else if (g_ascii_strcasecmp (type, "main") == 0) {
- version_num = RSPAMD_VERSION_MAJOR_NUM * 1000 +
- RSPAMD_VERSION_MINOR_NUM * 100;
- }
- else if (g_ascii_strcasecmp (type, "major") == 0) {
- version_num = RSPAMD_VERSION_MAJOR_NUM;
- }
- else if (g_ascii_strcasecmp (type, "minor") == 0) {
- version_num = RSPAMD_VERSION_MINOR_NUM;
- }
- else if (g_ascii_strcasecmp (type, "patch") == 0) {
- version_num = RSPAMD_VERSION_PATCH_NUM;
- }
- }
-
- lua_pushnumber (L, version_num);
-
- return 1;
-}
-
-static int
-rspamd_rcl_lua_version (lua_State *L)
-{
- const gchar *result = NULL, *type;
-
- if (lua_gettop (L) == 0) {
- result = RVERSION;
- }
- else if (lua_gettop (L) >= 1 && lua_type (L, 1) == LUA_TSTRING) {
- /* We got something like string */
- type = lua_tostring (L, 1);
-
- if (g_ascii_strcasecmp (type, "short") == 0) {
- result = RSPAMD_VERSION_MAJOR
- "." RSPAMD_VERSION_MINOR
- "." RSPAMD_VERSION_PATCH;
- }
- else if (g_ascii_strcasecmp (type, "main") == 0) {
- result = RSPAMD_VERSION_MAJOR "." RSPAMD_VERSION_MINOR;
- }
- else if (g_ascii_strcasecmp (type, "major") == 0) {
- result = RSPAMD_VERSION_MAJOR;
- }
- else if (g_ascii_strcasecmp (type, "minor") == 0) {
- result = RSPAMD_VERSION_MINOR;
- }
- else if (g_ascii_strcasecmp (type, "patch") == 0) {
- result = RSPAMD_VERSION_PATCH;
- }
- else if (g_ascii_strcasecmp (type, "id") == 0) {
- result = RID;
- }
- else if (g_ascii_strcasecmp (type, "num") == 0) {
- return rspamd_rcl_lua_version_numeric (L);
- }
- else if (g_ascii_strcasecmp (type, "cmp") == 0) {
- return rspamd_rcl_lua_version_cmp (L);
- }
- }
-
- lua_pushstring (L, result);
-
- return 1;
-}
-
-static void
-rspamd_rcl_set_lua_globals (struct rspamd_config *cfg, lua_State *L,
- GHashTable *vars)
-{
- struct rspamd_config **pcfg;
-
- /* First check for global variable 'config' */
- lua_getglobal (L, "config");
- if (lua_isnil (L, -1)) {
- /* Assign global table to set up attributes */
- lua_newtable (L);
- lua_setglobal (L, "config");
- }
-
- lua_getglobal (L, "metrics");
- if (lua_isnil (L, -1)) {
- lua_newtable (L);
- lua_setglobal (L, "metrics");
- }
-
- lua_getglobal (L, "composites");
- if (lua_isnil (L, -1)) {
- lua_newtable (L);
- lua_setglobal (L, "composites");
- }
-
- lua_getglobal (L, "rspamd_classifiers");
- if (lua_isnil (L, -1)) {
- lua_newtable (L);
- lua_setglobal (L, "rspamd_classifiers");
- }
-
- lua_getglobal (L, "classifiers");
- if (lua_isnil (L, -1)) {
- lua_newtable (L);
- lua_setglobal (L, "classifiers");
- }
-
- lua_getglobal (L, "rspamd_version");
- if (lua_isnil (L, -1)) {
- lua_pushcfunction (L, rspamd_rcl_lua_version);
- lua_setglobal (L, "rspamd_version");
- }
-
- pcfg = lua_newuserdata (L, sizeof (struct rspamd_config *));
- rspamd_lua_setclass (L, "rspamd{config}", -1);
- *pcfg = cfg;
- lua_setglobal (L, "rspamd_config");
-
- /* Clear stack from globals */
- lua_pop (L, 4);
-
- rspamd_lua_set_path (L, cfg->rcl_obj, vars);
-
- /* Set known paths as rspamd_paths global */
- lua_getglobal (L, "rspamd_paths");
- if (lua_isnil (L, -1)) {
- const gchar *confdir = RSPAMD_CONFDIR, *rundir = RSPAMD_RUNDIR,
- *dbdir = RSPAMD_DBDIR, *logdir = RSPAMD_LOGDIR,
- *wwwdir = RSPAMD_WWWDIR, *pluginsdir = RSPAMD_PLUGINSDIR,
- *rulesdir = RSPAMD_RULESDIR, *lualibdir = RSPAMD_LUALIBDIR,
- *prefix = RSPAMD_PREFIX;
- const gchar *t;
-
- /* Try environment */
- t = getenv ("PLUGINSDIR");
- if (t) {
- pluginsdir = t;
- }
-
- t = getenv ("RULESDIR");
- if (t) {
- rulesdir = t;
- }
-
- t = getenv ("DBDIR");
- if (t) {
- dbdir = t;
- }
-
- t = getenv ("RUNDIR");
- if (t) {
- rundir = t;
- }
-
- t = getenv ("LUALIBDIR");
- if (t) {
- lualibdir = t;
- }
-
- t = getenv ("LOGDIR");
- if (t) {
- logdir = t;
- }
-
- t = getenv ("WWWDIR");
- if (t) {
- wwwdir = t;
- }
-
- t = getenv ("CONFDIR");
- if (t) {
- confdir = t;
- }
-
-
- if (vars) {
- t = g_hash_table_lookup (vars, "PLUGINSDIR");
- if (t) {
- pluginsdir = t;
- }
-
- t = g_hash_table_lookup (vars, "RULESDIR");
- if (t) {
- rulesdir = t;
- }
-
- t = g_hash_table_lookup (vars, "LUALIBDIR");
- if (t) {
- lualibdir = t;
- }
-
- t = g_hash_table_lookup (vars, "RUNDIR");
- if (t) {
- rundir = t;
- }
-
- t = g_hash_table_lookup (vars, "WWWDIR");
- if (t) {
- wwwdir = t;
- }
-
- t = g_hash_table_lookup (vars, "CONFDIR");
- if (t) {
- confdir = t;
- }
-
- t = g_hash_table_lookup (vars, "DBDIR");
- if (t) {
- dbdir = t;
- }
-
- t = g_hash_table_lookup (vars, "LOGDIR");
- if (t) {
- logdir = t;
- }
- }
-
- lua_createtable (L, 0, 9);
-
- rspamd_lua_table_set (L, RSPAMD_CONFDIR_INDEX, confdir);
- rspamd_lua_table_set (L, RSPAMD_RUNDIR_INDEX, rundir);
- rspamd_lua_table_set (L, RSPAMD_DBDIR_INDEX, dbdir);
- rspamd_lua_table_set (L, RSPAMD_LOGDIR_INDEX, logdir);
- rspamd_lua_table_set (L, RSPAMD_WWWDIR_INDEX, wwwdir);
- rspamd_lua_table_set (L, RSPAMD_PLUGINSDIR_INDEX, pluginsdir);
- rspamd_lua_table_set (L, RSPAMD_RULESDIR_INDEX, rulesdir);
- rspamd_lua_table_set (L, RSPAMD_LUALIBDIR_INDEX, lualibdir);
- rspamd_lua_table_set (L, RSPAMD_PREFIX_INDEX, prefix);
-
- lua_setglobal (L, "rspamd_paths");
- }
-}
-
static gboolean
rspamd_rcl_lua_handler (rspamd_mempool_t *pool, const ucl_object_t *obj,
const gchar *key, gpointer ud,
@@ -1047,9 +729,8 @@ rspamd_rcl_add_lua_plugins_path (struct rspamd_config *cfg,
{
struct stat st;
struct script_module *cur_mod;
- glob_t globbuf;
- gchar *pattern, *ext_pos;
- size_t len;
+ GPtrArray *paths;
+ gchar *fname, *ext_pos;
guint i;
if (stat (path, &st) == -1) {
@@ -1064,52 +745,39 @@ rspamd_rcl_add_lua_plugins_path (struct rspamd_config *cfg,
/* Handle directory */
if (S_ISDIR (st.st_mode)) {
- globbuf.gl_offs = 0;
- len = strlen (path) + sizeof ("*.lua");
- pattern = g_malloc (len);
- rspamd_snprintf (pattern, len, "%s%s", path, "*.lua");
-
- if (glob (pattern, GLOB_DOOFFS, NULL, &globbuf) == 0) {
- for (i = 0; i < globbuf.gl_pathc; i++) {
- cur_mod =
+ paths = rspamd_glob_path (path, "*.lua", TRUE, err);
+
+ if (!paths) {
+ return FALSE;
+ }
+
+ PTR_ARRAY_FOREACH (paths, i, fname) {
+ cur_mod =
rspamd_mempool_alloc (cfg->cfg_pool,
- sizeof (struct script_module));
- cur_mod->path = rspamd_mempool_strdup (cfg->cfg_pool,
- globbuf.gl_pathv[i]);
- cur_mod->name = g_path_get_basename (cur_mod->path);
- rspamd_mempool_add_destructor (cfg->cfg_pool, g_free,
- cur_mod->name);
- ext_pos = strstr (cur_mod->name, ".lua");
-
- if (ext_pos != NULL) {
- *ext_pos = '\0';
- }
+ sizeof (struct script_module));
+ cur_mod->path = rspamd_mempool_strdup (cfg->cfg_pool, fname);
+ cur_mod->name = g_path_get_basename (cur_mod->path);
+ rspamd_mempool_add_destructor (cfg->cfg_pool, g_free,
+ cur_mod->name);
+ ext_pos = strstr (cur_mod->name, ".lua");
+
+ if (ext_pos != NULL) {
+ *ext_pos = '\0';
+ }
- if (cfg->script_modules == NULL) {
- cfg->script_modules = g_list_append (cfg->script_modules,
- cur_mod);
- rspamd_mempool_add_destructor (cfg->cfg_pool,
- (rspamd_mempool_destruct_t)g_list_free,
- cfg->script_modules);
- }
- else {
- cfg->script_modules = g_list_append (cfg->script_modules,
- cur_mod);
- }
+ if (cfg->script_modules == NULL) {
+ cfg->script_modules = g_list_append (cfg->script_modules,
+ cur_mod);
+ rspamd_mempool_add_destructor (cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) g_list_free,
+ cfg->script_modules);
+ } else {
+ cfg->script_modules = g_list_append (cfg->script_modules,
+ cur_mod);
}
- globfree (&globbuf);
- g_free (pattern);
- }
- else {
- g_set_error (err,
- CFG_RCL_ERROR,
- errno,
- "glob failed for %s, %s",
- pattern,
- strerror (errno));
- g_free (pattern);
- return FALSE;
}
+
+ g_ptr_array_free (paths, TRUE);
}
else {
/* Handle single file */
@@ -1762,7 +1430,7 @@ rspamd_rcl_add_default_handler (struct rspamd_rcl_section *section,
}
struct rspamd_rcl_section *
-rspamd_rcl_config_init (struct rspamd_config *cfg)
+rspamd_rcl_config_init (struct rspamd_config *cfg, GHashTable *skip_sections)
{
struct rspamd_rcl_section *new = NULL, *sub, *ssub;
@@ -1775,770 +1443,788 @@ rspamd_rcl_config_init (struct rspamd_config *cfg)
/**
* Logging section
*/
- sub = rspamd_rcl_add_section_doc (&new,
- "logging", NULL,
- rspamd_rcl_logging_handler,
- UCL_OBJECT,
- FALSE,
- TRUE,
- cfg->doc_strings,
- "Configure rspamd logging");
- /* Default handlers */
- rspamd_rcl_add_default_handler (sub,
- "log_buffer",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_config, log_buf_size),
- RSPAMD_CL_FLAG_INT_32,
- "Size of log buffer in bytes (for file logging)");
- rspamd_rcl_add_default_handler (sub,
- "log_urls",
- rspamd_rcl_parse_struct_boolean,
- G_STRUCT_OFFSET (struct rspamd_config, log_urls),
- 0,
- "Write each URL found in a message to the log file");
- rspamd_rcl_add_default_handler (sub,
- "debug_ip",
- rspamd_rcl_parse_struct_ucl,
- G_STRUCT_OFFSET (struct rspamd_config, debug_ip_map),
- 0,
- "Enable debugging log for the specified IP addresses");
- rspamd_rcl_add_default_handler (sub,
- "debug_symbols",
- rspamd_rcl_parse_struct_string_list,
- G_STRUCT_OFFSET (struct rspamd_config, debug_symbols),
- 0,
- "Enable debug for the specified symbols");
- rspamd_rcl_add_default_handler (sub,
- "debug_modules",
- rspamd_rcl_parse_struct_string_list,
- G_STRUCT_OFFSET (struct rspamd_config, debug_modules),
- RSPAMD_CL_FLAG_STRING_LIST_HASH,
- "Enable debugging for the specified modules");
- rspamd_rcl_add_default_handler (sub,
- "log_format",
- rspamd_rcl_parse_struct_string,
- G_STRUCT_OFFSET (struct rspamd_config, log_format_str),
- 0,
- "Specify format string for the task logging output "
- "(https://rspamd.com/doc/configuration/logging.html "
- "for details)");
- rspamd_rcl_add_default_handler (sub,
- "encryption_key",
- rspamd_rcl_parse_struct_pubkey,
- G_STRUCT_OFFSET (struct rspamd_config, log_encryption_key),
- 0,
- "Encrypt sensitive information in logs using this pubkey");
- rspamd_rcl_add_default_handler (sub,
- "error_elts",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_config, log_error_elts),
- RSPAMD_CL_FLAG_UINT,
- "Size of circular buffer for last errors (10 by default)");
- rspamd_rcl_add_default_handler (sub,
- "error_maxlen",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_config, log_error_elt_maxlen),
- RSPAMD_CL_FLAG_UINT,
- "Size of each element in error log buffer (1000 by default)");
-
- /* Documentation only options, handled in log_handler to map flags */
- rspamd_rcl_add_doc_by_path (cfg,
- "logging",
- "Enable colored output (for console logging)",
- "log_color",
- UCL_BOOLEAN,
- NULL,
- 0,
- NULL,
- 0);
- rspamd_rcl_add_doc_by_path (cfg,
- "logging",
- "Enable systemd compatible logging",
- "systemd",
- UCL_BOOLEAN,
- NULL,
- 0,
- NULL,
- 0);
- rspamd_rcl_add_doc_by_path (cfg,
- "logging",
- "Write statistics of regexp processing to log (useful for hyperscan)",
- "log_re_cache",
- UCL_BOOLEAN,
- NULL,
- 0,
- NULL,
- 0);
- rspamd_rcl_add_doc_by_path (cfg,
- "logging",
- "Use microseconds resolution for timestamps",
- "log_usec",
- UCL_BOOLEAN,
- NULL,
- 0,
- NULL,
- 0);
- /**
- * Options section
- */
- sub = rspamd_rcl_add_section_doc (&new,
- "options", NULL,
- rspamd_rcl_options_handler,
- UCL_OBJECT,
- FALSE,
- TRUE,
- cfg->doc_strings,
- "Global rspamd options");
- rspamd_rcl_add_default_handler (sub,
- "cache_file",
- rspamd_rcl_parse_struct_string,
- G_STRUCT_OFFSET (struct rspamd_config, cache_filename),
- RSPAMD_CL_FLAG_STRING_PATH,
- "Path to the cache file");
- rspamd_rcl_add_default_handler (sub,
- "cache_reload",
- rspamd_rcl_parse_struct_time,
- G_STRUCT_OFFSET (struct rspamd_config, cache_reload_time),
- RSPAMD_CL_FLAG_TIME_FLOAT,
- "How often cache reload should be performed");
- /* Old DNS configuration */
- rspamd_rcl_add_default_handler (sub,
- "dns_nameserver",
- rspamd_rcl_parse_struct_ucl,
- G_STRUCT_OFFSET (struct rspamd_config, nameservers),
- 0,
- "Legacy option for DNS servers used");
- rspamd_rcl_add_default_handler (sub,
- "dns_timeout",
- rspamd_rcl_parse_struct_time,
- G_STRUCT_OFFSET (struct rspamd_config, dns_timeout),
- RSPAMD_CL_FLAG_TIME_FLOAT,
- "Legacy option for DNS request timeout");
- rspamd_rcl_add_default_handler (sub,
- "dns_retransmits",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_config, dns_retransmits),
- RSPAMD_CL_FLAG_INT_32,
- "Legacy option for DNS retransmits count");
- rspamd_rcl_add_default_handler (sub,
- "dns_sockets",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_config, dns_io_per_server),
- RSPAMD_CL_FLAG_INT_32,
- "Legacy option for DNS sockets per server count");
- rspamd_rcl_add_default_handler (sub,
- "dns_max_requests",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_config, dns_max_requests),
- RSPAMD_CL_FLAG_INT_32,
- "Legacy option for DNS maximum requests per task count");
- rspamd_rcl_add_default_handler (sub,
- "classify_headers",
- rspamd_rcl_parse_struct_string_list,
- G_STRUCT_OFFSET (struct rspamd_config, classify_headers),
- 0,
- "List of headers used for classifiers");
- rspamd_rcl_add_default_handler (sub,
- "control_socket",
- rspamd_rcl_parse_struct_string,
- G_STRUCT_OFFSET (struct rspamd_config, control_socket_path),
- 0,
- "Path to the control socket");
- rspamd_rcl_add_default_handler (sub,
- "explicit_modules",
- rspamd_rcl_parse_struct_string_list,
- G_STRUCT_OFFSET (struct rspamd_config, explicit_modules),
- RSPAMD_CL_FLAG_STRING_LIST_HASH,
- "Always load these modules even if they are not configured explicitly");
- rspamd_rcl_add_default_handler (sub,
- "allow_raw_input",
- rspamd_rcl_parse_struct_boolean,
- G_STRUCT_OFFSET (struct rspamd_config, allow_raw_input),
- 0,
- "Allow non MIME input for rspamd");
- rspamd_rcl_add_default_handler (sub,
- "raw_mode",
- rspamd_rcl_parse_struct_boolean,
- G_STRUCT_OFFSET (struct rspamd_config, raw_mode),
- 0,
- "Don't try to convert all messages to utf8");
- rspamd_rcl_add_default_handler (sub,
- "one_shot",
- rspamd_rcl_parse_struct_boolean,
- G_STRUCT_OFFSET (struct rspamd_config, one_shot_mode),
- 0,
- "Add all symbols only once per message");
- rspamd_rcl_add_default_handler (sub,
- "check_attachements",
- rspamd_rcl_parse_struct_boolean,
- G_STRUCT_OFFSET (struct rspamd_config, check_text_attachements),
- 0,
- "Treat text attachments as normal text parts");
- rspamd_rcl_add_default_handler (sub,
- "check_attachments",
- rspamd_rcl_parse_struct_boolean,
- G_STRUCT_OFFSET (struct rspamd_config, check_text_attachements),
- 0,
- "Treat text attachments as normal text parts");
- rspamd_rcl_add_default_handler (sub,
- "tempdir",
- rspamd_rcl_parse_struct_string,
- G_STRUCT_OFFSET (struct rspamd_config, temp_dir),
- RSPAMD_CL_FLAG_STRING_PATH,
- "Directory for temporary files");
- rspamd_rcl_add_default_handler (sub,
- "pidfile",
- rspamd_rcl_parse_struct_string,
- G_STRUCT_OFFSET (struct rspamd_config, pid_file),
- RSPAMD_CL_FLAG_STRING_PATH,
- "Path to the pid file");
- rspamd_rcl_add_default_handler (sub,
- "filters",
- rspamd_rcl_parse_struct_string_list,
- G_STRUCT_OFFSET (struct rspamd_config, filters),
- 0,
- "List of internal filters enabled");
- rspamd_rcl_add_default_handler (sub,
- "max_diff",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_config, max_diff),
- RSPAMD_CL_FLAG_INT_SIZE,
- "Legacy option, do not use");
- rspamd_rcl_add_default_handler (sub,
- "map_watch_interval",
- rspamd_rcl_parse_struct_time,
- G_STRUCT_OFFSET (struct rspamd_config, map_timeout),
- RSPAMD_CL_FLAG_TIME_FLOAT,
- "Interval for checking maps");
- rspamd_rcl_add_default_handler (sub,
- "map_file_watch_multiplier",
- rspamd_rcl_parse_struct_double,
- G_STRUCT_OFFSET (struct rspamd_config, map_file_watch_multiplier),
- 0,
- "Multiplier for map watch interval when map is file");
- rspamd_rcl_add_default_handler (sub,
- "monitoring_watch_interval",
- rspamd_rcl_parse_struct_time,
- G_STRUCT_OFFSET (struct rspamd_config, monitored_interval),
- RSPAMD_CL_FLAG_TIME_FLOAT,
- "Interval for checking monitored instances");
- rspamd_rcl_add_default_handler (sub,
- "disable_monitoring",
- rspamd_rcl_parse_struct_boolean,
- G_STRUCT_OFFSET (struct rspamd_config, disable_monitored),
- 0,
- "Disable monitoring completely");
- rspamd_rcl_add_default_handler (sub,
- "dynamic_conf",
- rspamd_rcl_parse_struct_string,
- G_STRUCT_OFFSET (struct rspamd_config, dynamic_conf),
- 0,
- "Path to the dynamic configuration");
- rspamd_rcl_add_default_handler (sub,
- "rrd",
- rspamd_rcl_parse_struct_string,
- G_STRUCT_OFFSET (struct rspamd_config, rrd_file),
- RSPAMD_CL_FLAG_STRING_PATH,
- "Path to RRD file");
- rspamd_rcl_add_default_handler (sub,
- "history_file",
- rspamd_rcl_parse_struct_string,
- G_STRUCT_OFFSET (struct rspamd_config, history_file),
- RSPAMD_CL_FLAG_STRING_PATH,
- "Path to history file");
- rspamd_rcl_add_default_handler (sub,
- "check_all_filters",
- rspamd_rcl_parse_struct_boolean,
- G_STRUCT_OFFSET (struct rspamd_config, check_all_filters),
- 0,
- "Always check all filters");
- rspamd_rcl_add_default_handler (sub,
- "enable_experimental",
- rspamd_rcl_parse_struct_boolean,
- G_STRUCT_OFFSET (struct rspamd_config, enable_experimental),
- 0,
- "Enable experimental plugins");
- rspamd_rcl_add_default_handler (sub,
- "disable_pcre_jit",
- rspamd_rcl_parse_struct_boolean,
- G_STRUCT_OFFSET (struct rspamd_config, disable_pcre_jit),
- 0,
- "Disable PCRE JIT");
- rspamd_rcl_add_default_handler (sub,
- "disable_lua_squeeze",
- rspamd_rcl_parse_struct_boolean,
- G_STRUCT_OFFSET (struct rspamd_config, disable_lua_squeeze),
- 0,
- "Disable Lua rules squeezing");
- rspamd_rcl_add_default_handler (sub,
- "min_word_len",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_config, min_word_len),
- RSPAMD_CL_FLAG_UINT,
- "Minimum length of the word to be considered in statistics/fuzzy");
- rspamd_rcl_add_default_handler (sub,
- "max_word_len",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_config, max_word_len),
- RSPAMD_CL_FLAG_UINT,
- "Maximum length of the word to be considered in statistics/fuzzy");
- rspamd_rcl_add_default_handler (sub,
- "words_decay",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_config, words_decay),
- RSPAMD_CL_FLAG_UINT,
- "Start skipping words at this amount");
- rspamd_rcl_add_default_handler (sub,
- "url_tld",
- rspamd_rcl_parse_struct_string,
- G_STRUCT_OFFSET (struct rspamd_config, tld_file),
- RSPAMD_CL_FLAG_STRING_PATH,
- "Path to the TLD file for urls detector");
- rspamd_rcl_add_default_handler (sub,
- "tld",
- rspamd_rcl_parse_struct_string,
- G_STRUCT_OFFSET (struct rspamd_config, tld_file),
- RSPAMD_CL_FLAG_STRING_PATH,
- "Path to the TLD file for urls detector");
- rspamd_rcl_add_default_handler (sub,
- "hs_cache_dir",
- rspamd_rcl_parse_struct_string,
- G_STRUCT_OFFSET (struct rspamd_config, hs_cache_dir),
- RSPAMD_CL_FLAG_STRING_PATH,
- "Path directory where rspamd would save hyperscan cache");
- rspamd_rcl_add_default_handler (sub,
- "history_rows",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_config, history_rows),
- RSPAMD_CL_FLAG_UINT,
- "Number of records in the history file");
- rspamd_rcl_add_default_handler (sub,
- "disable_hyperscan",
- rspamd_rcl_parse_struct_boolean,
- G_STRUCT_OFFSET (struct rspamd_config, disable_hyperscan),
- 0,
- "Disable hyperscan optimizations for regular expressions");
- rspamd_rcl_add_default_handler (sub,
- "vectorized_hyperscan",
- rspamd_rcl_parse_struct_boolean,
- G_STRUCT_OFFSET (struct rspamd_config, vectorized_hyperscan),
- 0,
- "Use hyperscan in vectorized mode (experimental)");
- rspamd_rcl_add_default_handler (sub,
- "cores_dir",
- rspamd_rcl_parse_struct_string,
- G_STRUCT_OFFSET (struct rspamd_config, cores_dir),
- RSPAMD_CL_FLAG_STRING_PATH,
- "Path to the directory where rspamd core files are intended to be dumped");
- rspamd_rcl_add_default_handler (sub,
- "max_cores_size",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_config, max_cores_size),
- RSPAMD_CL_FLAG_INT_SIZE,
- "Limit of joint size of all files in `cores_dir`");
- rspamd_rcl_add_default_handler (sub,
- "max_cores_count",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_config, max_cores_count),
- RSPAMD_CL_FLAG_INT_SIZE,
- "Limit of files count in `cores_dir`");
- rspamd_rcl_add_default_handler (sub,
- "local_addrs",
- rspamd_rcl_parse_struct_ucl,
- G_STRUCT_OFFSET (struct rspamd_config, local_addrs),
- 0,
- "Use the specified addresses as local ones");
- rspamd_rcl_add_default_handler (sub,
- "local_networks",
- rspamd_rcl_parse_struct_ucl,
- G_STRUCT_OFFSET (struct rspamd_config, local_addrs),
- 0,
- "Use the specified addresses as local ones (alias for `local_addrs`)");
- rspamd_rcl_add_default_handler (sub,
- "trusted_keys",
- rspamd_rcl_parse_struct_string_list,
- G_STRUCT_OFFSET (struct rspamd_config, trusted_keys),
- RSPAMD_CL_FLAG_STRING_LIST_HASH,
- "List of trusted public keys used for signatures in base32 encoding");
- rspamd_rcl_add_default_handler (sub,
- "enable_shutdown_workaround",
- rspamd_rcl_parse_struct_boolean,
- G_STRUCT_OFFSET (struct rspamd_config, enable_shutdown_workaround),
- 0,
- "Enable workaround for legacy clients");
- rspamd_rcl_add_default_handler (sub,
- "ignore_received",
- rspamd_rcl_parse_struct_boolean,
- G_STRUCT_OFFSET (struct rspamd_config, ignore_received),
- 0,
- "Ignore data from the first received header");
- rspamd_rcl_add_default_handler (sub,
- "ssl_ca_path",
- rspamd_rcl_parse_struct_string,
- G_STRUCT_OFFSET (struct rspamd_config, ssl_ca_path),
- RSPAMD_CL_FLAG_STRING_PATH,
- "Path to ssl CA file");
- rspamd_rcl_add_default_handler (sub,
- "ssl_ciphers",
- rspamd_rcl_parse_struct_string,
- G_STRUCT_OFFSET (struct rspamd_config, ssl_ciphers),
- 0,
- "List of ssl ciphers (e.g. HIGH:!aNULL:!kRSA:!PSK:!SRP:!MD5:!RC4)");
- rspamd_rcl_add_default_handler (sub,
- "magic_file",
- rspamd_rcl_parse_struct_string,
- G_STRUCT_OFFSET (struct rspamd_config, magic_file),
- 0,
- "Path to a custom libmagic file");
- rspamd_rcl_add_default_handler (sub,
- "max_message",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_config, max_message),
- RSPAMD_CL_FLAG_INT_SIZE,
- "Maximum size of the message to be scanned (50Mb by default)");
- rspamd_rcl_add_default_handler (sub,
- "max_pic",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_config, max_pic_size),
- RSPAMD_CL_FLAG_INT_SIZE,
- "Maximum size of the picture to be normalized (1Mb by default)");
- rspamd_rcl_add_default_handler (sub,
- "images_cache",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_config, max_pic_size),
- RSPAMD_CL_FLAG_INT_SIZE,
- "Size of DCT data cache for images (256 elements by default)");
- rspamd_rcl_add_default_handler (sub,
- "zstd_input_dictionary",
- rspamd_rcl_parse_struct_string,
- G_STRUCT_OFFSET (struct rspamd_config, zstd_input_dictionary),
- RSPAMD_CL_FLAG_STRING_PATH,
- "Dictionary for zstd inbound protocol compression");
- rspamd_rcl_add_default_handler (sub,
- "zstd_output_dictionary",
- rspamd_rcl_parse_struct_string,
- G_STRUCT_OFFSET (struct rspamd_config, zstd_output_dictionary),
- RSPAMD_CL_FLAG_STRING_PATH,
- "Dictionary for outbound zstd compression");
- rspamd_rcl_add_default_handler (sub,
- "compat_messages",
- rspamd_rcl_parse_struct_boolean,
- G_STRUCT_OFFSET (struct rspamd_config, compat_messages),
- 0,
- "Use pre 1.4 style of messages in the protocol");
- rspamd_rcl_add_default_handler (sub,
- "max_shots",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_config, default_max_shots),
- 0,
- "Maximum number of hits per a single symbol (default: 100)");
- rspamd_rcl_add_default_handler (sub,
- "sessions_cache",
- rspamd_rcl_parse_struct_boolean,
- G_STRUCT_OFFSET (struct rspamd_config, enable_sessions_cache),
- 0,
- "Enable sessions cache to debug dangling sessions");
- rspamd_rcl_add_default_handler (sub,
- "max_sessions_cache",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_config, max_sessions_cache),
- 0,
- "Maximum number of sessions in cache before warning (default: 100)");
-
- /* Neighbours configuration */
- rspamd_rcl_add_section_doc (&sub->subsections, "neighbours", "name",
- rspamd_rcl_neighbours_handler,
- UCL_OBJECT, FALSE, TRUE,
- cfg->doc_strings,
- "List of members of Rspamd cluster");
-
- /* New DNS configuration */
- ssub = rspamd_rcl_add_section_doc (&sub->subsections, "dns", NULL, NULL,
- UCL_OBJECT, FALSE, TRUE,
- cfg->doc_strings,
- "Options for DNS resolver");
- rspamd_rcl_add_default_handler (ssub,
- "nameserver",
- rspamd_rcl_parse_struct_ucl,
- G_STRUCT_OFFSET (struct rspamd_config, nameservers),
- 0,
- "List of DNS servers");
- rspamd_rcl_add_default_handler (ssub,
- "server",
- rspamd_rcl_parse_struct_ucl,
- G_STRUCT_OFFSET (struct rspamd_config, nameservers),
- 0,
- "List of DNS servers");
- rspamd_rcl_add_default_handler (ssub,
- "timeout",
- rspamd_rcl_parse_struct_time,
- G_STRUCT_OFFSET (struct rspamd_config, dns_timeout),
- RSPAMD_CL_FLAG_TIME_FLOAT,
- "DNS request timeout");
- rspamd_rcl_add_default_handler (ssub,
- "retransmits",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_config, dns_retransmits),
- RSPAMD_CL_FLAG_INT_32,
- "DNS request retransmits");
- rspamd_rcl_add_default_handler (ssub,
- "sockets",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_config, dns_io_per_server),
- RSPAMD_CL_FLAG_INT_32,
- "Number of sockets per DNS server");
- rspamd_rcl_add_default_handler (ssub,
- "connections",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_config, dns_io_per_server),
- RSPAMD_CL_FLAG_INT_32,
- "Number of sockets per DNS server");
- rspamd_rcl_add_default_handler (ssub,
- "enable_dnssec",
- rspamd_rcl_parse_struct_boolean,
- G_STRUCT_OFFSET (struct rspamd_config, enable_dnssec),
- 0,
- "Enable DNSSEC support in Rspamd");
-
-
- /* New upstreams configuration */
- ssub = rspamd_rcl_add_section_doc (&sub->subsections, "upstream", NULL, NULL,
- UCL_OBJECT, FALSE, TRUE,
- cfg->doc_strings,
- "Upstreams configuration parameters");
- rspamd_rcl_add_default_handler (ssub,
- "max_errors",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_config, upstream_max_errors),
- RSPAMD_CL_FLAG_UINT,
- "Maximum number of errors during `error_time` to consider upstream down");
- rspamd_rcl_add_default_handler (ssub,
- "error_time",
- rspamd_rcl_parse_struct_time,
- G_STRUCT_OFFSET (struct rspamd_config, upstream_error_time),
- RSPAMD_CL_FLAG_TIME_FLOAT,
- "Time frame to check errors");
- rspamd_rcl_add_default_handler (ssub,
- "revive_time",
- rspamd_rcl_parse_struct_time,
- G_STRUCT_OFFSET (struct rspamd_config, upstream_revive_time),
- RSPAMD_CL_FLAG_TIME_FLOAT,
- "Time before attempting to recover upstream after an error");
-
- /**
- * Symbols and actions sections
- */
- sub = rspamd_rcl_add_section_doc (&new,
- "actions", NULL,
- rspamd_rcl_actions_handler,
- UCL_OBJECT,
- FALSE,
- TRUE,
- cfg->doc_strings,
- "Actions configuration");
- rspamd_rcl_add_default_handler (sub,
- "unknown_weight",
- rspamd_rcl_parse_struct_double,
- G_STRUCT_OFFSET (struct rspamd_config, unknown_weight),
- 0,
- "Accept unknown symbols with the specified weight");
- rspamd_rcl_add_default_handler (sub,
- "grow_factor",
- rspamd_rcl_parse_struct_double,
- G_STRUCT_OFFSET (struct rspamd_config, grow_factor),
- 0,
- "Multiply the subsequent symbols by this number "
- "(does not affect symbols with score less or "
- "equal to zero)");
- rspamd_rcl_add_default_handler (sub,
- "subject",
- rspamd_rcl_parse_struct_string,
- G_STRUCT_OFFSET (struct rspamd_config, subject),
- 0,
- "Rewrite subject with this value");
-
- sub = rspamd_rcl_add_section_doc (&new,
- "group", "name",
- rspamd_rcl_group_handler,
- UCL_OBJECT,
- FALSE,
- TRUE,
- cfg->doc_strings,
- "Symbol groups configuration");
- ssub = rspamd_rcl_add_section_doc (&sub->subsections, "symbols", "name",
- rspamd_rcl_symbol_handler,
- UCL_OBJECT, FALSE, TRUE,
- cfg->doc_strings,
- "Symbols configuration");
-
- /* Group part */
- rspamd_rcl_add_default_handler (sub,
- "disabled",
- rspamd_rcl_parse_struct_boolean,
- G_STRUCT_OFFSET (struct rspamd_symbols_group, disabled),
- 0,
- "Disable symbols group");
- rspamd_rcl_add_default_handler (sub,
- "enabled",
- rspamd_rcl_parse_struct_boolean,
- G_STRUCT_OFFSET (struct rspamd_symbols_group, disabled),
- RSPAMD_CL_FLAG_BOOLEAN_INVERSE,
- "Enable or disable symbols group");
- rspamd_rcl_add_default_handler (sub,
- "max_score",
- rspamd_rcl_parse_struct_double,
- G_STRUCT_OFFSET (struct rspamd_symbols_group, max_score),
- 0,
- "Maximum score that could be reached by this symbols group");
-
- /**
- * Worker section
- */
- sub = rspamd_rcl_add_section_doc (&new,
- "worker", "type",
- rspamd_rcl_worker_handler,
- UCL_OBJECT,
- FALSE,
- TRUE,
- cfg->doc_strings,
- "Workers common options");
- rspamd_rcl_add_default_handler (sub,
- "count",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_worker_conf, count),
- RSPAMD_CL_FLAG_INT_16,
- "Number of workers to spawn");
- rspamd_rcl_add_default_handler (sub,
- "max_files",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_worker_conf, rlimit_nofile),
- RSPAMD_CL_FLAG_INT_32,
- "Maximum number of opened files per worker");
- rspamd_rcl_add_default_handler (sub,
- "max_core",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_worker_conf, rlimit_maxcore),
- RSPAMD_CL_FLAG_INT_32,
- "Max size of core file in bytes");
- rspamd_rcl_add_default_handler (sub,
- "enabled",
- rspamd_rcl_parse_struct_boolean,
- G_STRUCT_OFFSET (struct rspamd_worker_conf, enabled),
- 0,
- "Enable or disable a worker (true by default)");
-
- /**
- * Modules handler
- */
- sub = rspamd_rcl_add_section_doc (&new,
- "modules", NULL,
- rspamd_rcl_modules_handler,
- UCL_OBJECT,
- FALSE,
- FALSE,
- cfg->doc_strings,
- "Lua plugins to load");
+ if (!(skip_sections && g_hash_table_lookup (skip_sections, "logging"))) {
+ sub = rspamd_rcl_add_section_doc (&new,
+ "logging", NULL,
+ rspamd_rcl_logging_handler,
+ UCL_OBJECT,
+ FALSE,
+ TRUE,
+ cfg->doc_strings,
+ "Configure rspamd logging");
+ /* Default handlers */
+ rspamd_rcl_add_default_handler (sub,
+ "log_buffer",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_config, log_buf_size),
+ RSPAMD_CL_FLAG_INT_32,
+ "Size of log buffer in bytes (for file logging)");
+ rspamd_rcl_add_default_handler (sub,
+ "log_urls",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET (struct rspamd_config, log_urls),
+ 0,
+ "Write each URL found in a message to the log file");
+ rspamd_rcl_add_default_handler (sub,
+ "debug_ip",
+ rspamd_rcl_parse_struct_ucl,
+ G_STRUCT_OFFSET (struct rspamd_config, debug_ip_map),
+ 0,
+ "Enable debugging log for the specified IP addresses");
+ rspamd_rcl_add_default_handler (sub,
+ "debug_symbols",
+ rspamd_rcl_parse_struct_string_list,
+ G_STRUCT_OFFSET (struct rspamd_config, debug_symbols),
+ 0,
+ "Enable debug for the specified symbols");
+ rspamd_rcl_add_default_handler (sub,
+ "debug_modules",
+ rspamd_rcl_parse_struct_string_list,
+ G_STRUCT_OFFSET (struct rspamd_config, debug_modules),
+ RSPAMD_CL_FLAG_STRING_LIST_HASH,
+ "Enable debugging for the specified modules");
+ rspamd_rcl_add_default_handler (sub,
+ "log_format",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET (struct rspamd_config, log_format_str),
+ 0,
+ "Specify format string for the task logging output "
+ "(https://rspamd.com/doc/configuration/logging.html "
+ "for details)");
+ rspamd_rcl_add_default_handler (sub,
+ "encryption_key",
+ rspamd_rcl_parse_struct_pubkey,
+ G_STRUCT_OFFSET (struct rspamd_config, log_encryption_key),
+ 0,
+ "Encrypt sensitive information in logs using this pubkey");
+ rspamd_rcl_add_default_handler (sub,
+ "error_elts",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_config, log_error_elts),
+ RSPAMD_CL_FLAG_UINT,
+ "Size of circular buffer for last errors (10 by default)");
+ rspamd_rcl_add_default_handler (sub,
+ "error_maxlen",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_config, log_error_elt_maxlen),
+ RSPAMD_CL_FLAG_UINT,
+ "Size of each element in error log buffer (1000 by default)");
+
+ /* Documentation only options, handled in log_handler to map flags */
+ rspamd_rcl_add_doc_by_path (cfg,
+ "logging",
+ "Enable colored output (for console logging)",
+ "log_color",
+ UCL_BOOLEAN,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path (cfg,
+ "logging",
+ "Enable systemd compatible logging",
+ "systemd",
+ UCL_BOOLEAN,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path (cfg,
+ "logging",
+ "Write statistics of regexp processing to log (useful for hyperscan)",
+ "log_re_cache",
+ UCL_BOOLEAN,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path (cfg,
+ "logging",
+ "Use microseconds resolution for timestamps",
+ "log_usec",
+ UCL_BOOLEAN,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ }
+ if (!(skip_sections && g_hash_table_lookup (skip_sections, "options"))) {
+ /**
+ * Options section
+ */
+ sub = rspamd_rcl_add_section_doc (&new,
+ "options", NULL,
+ rspamd_rcl_options_handler,
+ UCL_OBJECT,
+ FALSE,
+ TRUE,
+ cfg->doc_strings,
+ "Global rspamd options");
+ rspamd_rcl_add_default_handler (sub,
+ "cache_file",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET (struct rspamd_config, cache_filename),
+ RSPAMD_CL_FLAG_STRING_PATH,
+ "Path to the cache file");
+ rspamd_rcl_add_default_handler (sub,
+ "cache_reload",
+ rspamd_rcl_parse_struct_time,
+ G_STRUCT_OFFSET (struct rspamd_config, cache_reload_time),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "How often cache reload should be performed");
+ /* Old DNS configuration */
+ rspamd_rcl_add_default_handler (sub,
+ "dns_nameserver",
+ rspamd_rcl_parse_struct_ucl,
+ G_STRUCT_OFFSET (struct rspamd_config, nameservers),
+ 0,
+ "Legacy option for DNS servers used");
+ rspamd_rcl_add_default_handler (sub,
+ "dns_timeout",
+ rspamd_rcl_parse_struct_time,
+ G_STRUCT_OFFSET (struct rspamd_config, dns_timeout),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "Legacy option for DNS request timeout");
+ rspamd_rcl_add_default_handler (sub,
+ "dns_retransmits",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_config, dns_retransmits),
+ RSPAMD_CL_FLAG_INT_32,
+ "Legacy option for DNS retransmits count");
+ rspamd_rcl_add_default_handler (sub,
+ "dns_sockets",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_config, dns_io_per_server),
+ RSPAMD_CL_FLAG_INT_32,
+ "Legacy option for DNS sockets per server count");
+ rspamd_rcl_add_default_handler (sub,
+ "dns_max_requests",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_config, dns_max_requests),
+ RSPAMD_CL_FLAG_INT_32,
+ "Legacy option for DNS maximum requests per task count");
+ rspamd_rcl_add_default_handler (sub,
+ "classify_headers",
+ rspamd_rcl_parse_struct_string_list,
+ G_STRUCT_OFFSET (struct rspamd_config, classify_headers),
+ 0,
+ "List of headers used for classifiers");
+ rspamd_rcl_add_default_handler (sub,
+ "control_socket",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET (struct rspamd_config, control_socket_path),
+ 0,
+ "Path to the control socket");
+ rspamd_rcl_add_default_handler (sub,
+ "explicit_modules",
+ rspamd_rcl_parse_struct_string_list,
+ G_STRUCT_OFFSET (struct rspamd_config, explicit_modules),
+ RSPAMD_CL_FLAG_STRING_LIST_HASH,
+ "Always load these modules even if they are not configured explicitly");
+ rspamd_rcl_add_default_handler (sub,
+ "allow_raw_input",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET (struct rspamd_config, allow_raw_input),
+ 0,
+ "Allow non MIME input for rspamd");
+ rspamd_rcl_add_default_handler (sub,
+ "raw_mode",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET (struct rspamd_config, raw_mode),
+ 0,
+ "Don't try to convert all messages to utf8");
+ rspamd_rcl_add_default_handler (sub,
+ "one_shot",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET (struct rspamd_config, one_shot_mode),
+ 0,
+ "Add all symbols only once per message");
+ rspamd_rcl_add_default_handler (sub,
+ "check_attachements",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET (struct rspamd_config, check_text_attachements),
+ 0,
+ "Treat text attachments as normal text parts");
+ rspamd_rcl_add_default_handler (sub,
+ "check_attachments",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET (struct rspamd_config, check_text_attachements),
+ 0,
+ "Treat text attachments as normal text parts");
+ rspamd_rcl_add_default_handler (sub,
+ "tempdir",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET (struct rspamd_config, temp_dir),
+ RSPAMD_CL_FLAG_STRING_PATH,
+ "Directory for temporary files");
+ rspamd_rcl_add_default_handler (sub,
+ "pidfile",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET (struct rspamd_config, pid_file),
+ RSPAMD_CL_FLAG_STRING_PATH,
+ "Path to the pid file");
+ rspamd_rcl_add_default_handler (sub,
+ "filters",
+ rspamd_rcl_parse_struct_string_list,
+ G_STRUCT_OFFSET (struct rspamd_config, filters),
+ 0,
+ "List of internal filters enabled");
+ rspamd_rcl_add_default_handler (sub,
+ "max_diff",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_config, max_diff),
+ RSPAMD_CL_FLAG_INT_SIZE,
+ "Legacy option, do not use");
+ rspamd_rcl_add_default_handler (sub,
+ "map_watch_interval",
+ rspamd_rcl_parse_struct_time,
+ G_STRUCT_OFFSET (struct rspamd_config, map_timeout),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "Interval for checking maps");
+ rspamd_rcl_add_default_handler (sub,
+ "map_file_watch_multiplier",
+ rspamd_rcl_parse_struct_double,
+ G_STRUCT_OFFSET (struct rspamd_config, map_file_watch_multiplier),
+ 0,
+ "Multiplier for map watch interval when map is file");
+ rspamd_rcl_add_default_handler (sub,
+ "monitoring_watch_interval",
+ rspamd_rcl_parse_struct_time,
+ G_STRUCT_OFFSET (struct rspamd_config, monitored_interval),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "Interval for checking monitored instances");
+ rspamd_rcl_add_default_handler (sub,
+ "disable_monitoring",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET (struct rspamd_config, disable_monitored),
+ 0,
+ "Disable monitoring completely");
+ rspamd_rcl_add_default_handler (sub,
+ "dynamic_conf",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET (struct rspamd_config, dynamic_conf),
+ 0,
+ "Path to the dynamic configuration");
+ rspamd_rcl_add_default_handler (sub,
+ "rrd",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET (struct rspamd_config, rrd_file),
+ RSPAMD_CL_FLAG_STRING_PATH,
+ "Path to RRD file");
+ rspamd_rcl_add_default_handler (sub,
+ "history_file",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET (struct rspamd_config, history_file),
+ RSPAMD_CL_FLAG_STRING_PATH,
+ "Path to history file");
+ rspamd_rcl_add_default_handler (sub,
+ "check_all_filters",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET (struct rspamd_config, check_all_filters),
+ 0,
+ "Always check all filters");
+ rspamd_rcl_add_default_handler (sub,
+ "enable_experimental",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET (struct rspamd_config, enable_experimental),
+ 0,
+ "Enable experimental plugins");
+ rspamd_rcl_add_default_handler (sub,
+ "disable_pcre_jit",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET (struct rspamd_config, disable_pcre_jit),
+ 0,
+ "Disable PCRE JIT");
+ rspamd_rcl_add_default_handler (sub,
+ "disable_lua_squeeze",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET (struct rspamd_config, disable_lua_squeeze),
+ 0,
+ "Disable Lua rules squeezing");
+ rspamd_rcl_add_default_handler (sub,
+ "min_word_len",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_config, min_word_len),
+ RSPAMD_CL_FLAG_UINT,
+ "Minimum length of the word to be considered in statistics/fuzzy");
+ rspamd_rcl_add_default_handler (sub,
+ "max_word_len",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_config, max_word_len),
+ RSPAMD_CL_FLAG_UINT,
+ "Maximum length of the word to be considered in statistics/fuzzy");
+ rspamd_rcl_add_default_handler (sub,
+ "words_decay",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_config, words_decay),
+ RSPAMD_CL_FLAG_UINT,
+ "Start skipping words at this amount");
+ rspamd_rcl_add_default_handler (sub,
+ "url_tld",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET (struct rspamd_config, tld_file),
+ RSPAMD_CL_FLAG_STRING_PATH,
+ "Path to the TLD file for urls detector");
+ rspamd_rcl_add_default_handler (sub,
+ "tld",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET (struct rspamd_config, tld_file),
+ RSPAMD_CL_FLAG_STRING_PATH,
+ "Path to the TLD file for urls detector");
+ rspamd_rcl_add_default_handler (sub,
+ "hs_cache_dir",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET (struct rspamd_config, hs_cache_dir),
+ RSPAMD_CL_FLAG_STRING_PATH,
+ "Path directory where rspamd would save hyperscan cache");
+ rspamd_rcl_add_default_handler (sub,
+ "history_rows",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_config, history_rows),
+ RSPAMD_CL_FLAG_UINT,
+ "Number of records in the history file");
+ rspamd_rcl_add_default_handler (sub,
+ "disable_hyperscan",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET (struct rspamd_config, disable_hyperscan),
+ 0,
+ "Disable hyperscan optimizations for regular expressions");
+ rspamd_rcl_add_default_handler (sub,
+ "vectorized_hyperscan",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET (struct rspamd_config, vectorized_hyperscan),
+ 0,
+ "Use hyperscan in vectorized mode (experimental)");
+ rspamd_rcl_add_default_handler (sub,
+ "cores_dir",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET (struct rspamd_config, cores_dir),
+ RSPAMD_CL_FLAG_STRING_PATH,
+ "Path to the directory where rspamd core files are intended to be dumped");
+ rspamd_rcl_add_default_handler (sub,
+ "max_cores_size",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_config, max_cores_size),
+ RSPAMD_CL_FLAG_INT_SIZE,
+ "Limit of joint size of all files in `cores_dir`");
+ rspamd_rcl_add_default_handler (sub,
+ "max_cores_count",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_config, max_cores_count),
+ RSPAMD_CL_FLAG_INT_SIZE,
+ "Limit of files count in `cores_dir`");
+ rspamd_rcl_add_default_handler (sub,
+ "local_addrs",
+ rspamd_rcl_parse_struct_ucl,
+ G_STRUCT_OFFSET (struct rspamd_config, local_addrs),
+ 0,
+ "Use the specified addresses as local ones");
+ rspamd_rcl_add_default_handler (sub,
+ "local_networks",
+ rspamd_rcl_parse_struct_ucl,
+ G_STRUCT_OFFSET (struct rspamd_config, local_addrs),
+ 0,
+ "Use the specified addresses as local ones (alias for `local_addrs`)");
+ rspamd_rcl_add_default_handler (sub,
+ "trusted_keys",
+ rspamd_rcl_parse_struct_string_list,
+ G_STRUCT_OFFSET (struct rspamd_config, trusted_keys),
+ RSPAMD_CL_FLAG_STRING_LIST_HASH,
+ "List of trusted public keys used for signatures in base32 encoding");
+ rspamd_rcl_add_default_handler (sub,
+ "enable_shutdown_workaround",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET (struct rspamd_config, enable_shutdown_workaround),
+ 0,
+ "Enable workaround for legacy clients");
+ rspamd_rcl_add_default_handler (sub,
+ "ignore_received",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET (struct rspamd_config, ignore_received),
+ 0,
+ "Ignore data from the first received header");
+ rspamd_rcl_add_default_handler (sub,
+ "ssl_ca_path",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET (struct rspamd_config, ssl_ca_path),
+ RSPAMD_CL_FLAG_STRING_PATH,
+ "Path to ssl CA file");
+ rspamd_rcl_add_default_handler (sub,
+ "ssl_ciphers",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET (struct rspamd_config, ssl_ciphers),
+ 0,
+ "List of ssl ciphers (e.g. HIGH:!aNULL:!kRSA:!PSK:!SRP:!MD5:!RC4)");
+ rspamd_rcl_add_default_handler (sub,
+ "magic_file",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET (struct rspamd_config, magic_file),
+ 0,
+ "Path to a custom libmagic file");
+ rspamd_rcl_add_default_handler (sub,
+ "max_message",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_config, max_message),
+ RSPAMD_CL_FLAG_INT_SIZE,
+ "Maximum size of the message to be scanned (50Mb by default)");
+ rspamd_rcl_add_default_handler (sub,
+ "max_pic",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_config, max_pic_size),
+ RSPAMD_CL_FLAG_INT_SIZE,
+ "Maximum size of the picture to be normalized (1Mb by default)");
+ rspamd_rcl_add_default_handler (sub,
+ "images_cache",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_config, max_pic_size),
+ RSPAMD_CL_FLAG_INT_SIZE,
+ "Size of DCT data cache for images (256 elements by default)");
+ rspamd_rcl_add_default_handler (sub,
+ "zstd_input_dictionary",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET (struct rspamd_config, zstd_input_dictionary),
+ RSPAMD_CL_FLAG_STRING_PATH,
+ "Dictionary for zstd inbound protocol compression");
+ rspamd_rcl_add_default_handler (sub,
+ "zstd_output_dictionary",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET (struct rspamd_config, zstd_output_dictionary),
+ RSPAMD_CL_FLAG_STRING_PATH,
+ "Dictionary for outbound zstd compression");
+ rspamd_rcl_add_default_handler (sub,
+ "compat_messages",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET (struct rspamd_config, compat_messages),
+ 0,
+ "Use pre 1.4 style of messages in the protocol");
+ rspamd_rcl_add_default_handler (sub,
+ "max_shots",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_config, default_max_shots),
+ 0,
+ "Maximum number of hits per a single symbol (default: 100)");
+ rspamd_rcl_add_default_handler (sub,
+ "sessions_cache",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET (struct rspamd_config, enable_sessions_cache),
+ 0,
+ "Enable sessions cache to debug dangling sessions");
+ rspamd_rcl_add_default_handler (sub,
+ "max_sessions_cache",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_config, max_sessions_cache),
+ 0,
+ "Maximum number of sessions in cache before warning (default: 100)");
+
+ /* Neighbours configuration */
+ rspamd_rcl_add_section_doc (&sub->subsections, "neighbours", "name",
+ rspamd_rcl_neighbours_handler,
+ UCL_OBJECT, FALSE, TRUE,
+ cfg->doc_strings,
+ "List of members of Rspamd cluster");
+
+ /* New DNS configuration */
+ ssub = rspamd_rcl_add_section_doc (&sub->subsections, "dns", NULL, NULL,
+ UCL_OBJECT, FALSE, TRUE,
+ cfg->doc_strings,
+ "Options for DNS resolver");
+ rspamd_rcl_add_default_handler (ssub,
+ "nameserver",
+ rspamd_rcl_parse_struct_ucl,
+ G_STRUCT_OFFSET (struct rspamd_config, nameservers),
+ 0,
+ "List of DNS servers");
+ rspamd_rcl_add_default_handler (ssub,
+ "server",
+ rspamd_rcl_parse_struct_ucl,
+ G_STRUCT_OFFSET (struct rspamd_config, nameservers),
+ 0,
+ "List of DNS servers");
+ rspamd_rcl_add_default_handler (ssub,
+ "timeout",
+ rspamd_rcl_parse_struct_time,
+ G_STRUCT_OFFSET (struct rspamd_config, dns_timeout),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "DNS request timeout");
+ rspamd_rcl_add_default_handler (ssub,
+ "retransmits",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_config, dns_retransmits),
+ RSPAMD_CL_FLAG_INT_32,
+ "DNS request retransmits");
+ rspamd_rcl_add_default_handler (ssub,
+ "sockets",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_config, dns_io_per_server),
+ RSPAMD_CL_FLAG_INT_32,
+ "Number of sockets per DNS server");
+ rspamd_rcl_add_default_handler (ssub,
+ "connections",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_config, dns_io_per_server),
+ RSPAMD_CL_FLAG_INT_32,
+ "Number of sockets per DNS server");
+ rspamd_rcl_add_default_handler (ssub,
+ "enable_dnssec",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET (struct rspamd_config, enable_dnssec),
+ 0,
+ "Enable DNSSEC support in Rspamd");
+
+
+ /* New upstreams configuration */
+ ssub = rspamd_rcl_add_section_doc (&sub->subsections, "upstream", NULL, NULL,
+ UCL_OBJECT, FALSE, TRUE,
+ cfg->doc_strings,
+ "Upstreams configuration parameters");
+ rspamd_rcl_add_default_handler (ssub,
+ "max_errors",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_config, upstream_max_errors),
+ RSPAMD_CL_FLAG_UINT,
+ "Maximum number of errors during `error_time` to consider upstream down");
+ rspamd_rcl_add_default_handler (ssub,
+ "error_time",
+ rspamd_rcl_parse_struct_time,
+ G_STRUCT_OFFSET (struct rspamd_config, upstream_error_time),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "Time frame to check errors");
+ rspamd_rcl_add_default_handler (ssub,
+ "revive_time",
+ rspamd_rcl_parse_struct_time,
+ G_STRUCT_OFFSET (struct rspamd_config, upstream_revive_time),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "Time before attempting to recover upstream after an error");
+ }
+
+ if (!(skip_sections && g_hash_table_lookup (skip_sections, "actions"))) {
+ /**
+ * Symbols and actions sections
+ */
+ sub = rspamd_rcl_add_section_doc (&new,
+ "actions", NULL,
+ rspamd_rcl_actions_handler,
+ UCL_OBJECT,
+ FALSE,
+ TRUE,
+ cfg->doc_strings,
+ "Actions configuration");
+ rspamd_rcl_add_default_handler (sub,
+ "unknown_weight",
+ rspamd_rcl_parse_struct_double,
+ G_STRUCT_OFFSET (struct rspamd_config, unknown_weight),
+ 0,
+ "Accept unknown symbols with the specified weight");
+ rspamd_rcl_add_default_handler (sub,
+ "grow_factor",
+ rspamd_rcl_parse_struct_double,
+ G_STRUCT_OFFSET (struct rspamd_config, grow_factor),
+ 0,
+ "Multiply the subsequent symbols by this number "
+ "(does not affect symbols with score less or "
+ "equal to zero)");
+ rspamd_rcl_add_default_handler (sub,
+ "subject",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET (struct rspamd_config, subject),
+ 0,
+ "Rewrite subject with this value");
+ }
+
+ if (!(skip_sections && g_hash_table_lookup (skip_sections, "group"))) {
+ sub = rspamd_rcl_add_section_doc (&new,
+ "group", "name",
+ rspamd_rcl_group_handler,
+ UCL_OBJECT,
+ FALSE,
+ TRUE,
+ cfg->doc_strings,
+ "Symbol groups configuration");
+ ssub = rspamd_rcl_add_section_doc (&sub->subsections, "symbols", "name",
+ rspamd_rcl_symbol_handler,
+ UCL_OBJECT, FALSE, TRUE,
+ cfg->doc_strings,
+ "Symbols configuration");
+
+ /* Group part */
+ rspamd_rcl_add_default_handler (sub,
+ "disabled",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET (struct rspamd_symbols_group, disabled),
+ 0,
+ "Disable symbols group");
+ rspamd_rcl_add_default_handler (sub,
+ "enabled",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET (struct rspamd_symbols_group, disabled),
+ RSPAMD_CL_FLAG_BOOLEAN_INVERSE,
+ "Enable or disable symbols group");
+ rspamd_rcl_add_default_handler (sub,
+ "max_score",
+ rspamd_rcl_parse_struct_double,
+ G_STRUCT_OFFSET (struct rspamd_symbols_group, max_score),
+ 0,
+ "Maximum score that could be reached by this symbols group");
+ }
- /**
- * Classifiers handler
- */
- sub = rspamd_rcl_add_section_doc (&new,
- "classifier", "type",
- rspamd_rcl_classifier_handler,
- UCL_OBJECT,
- FALSE,
- TRUE,
- cfg->doc_strings,
- "CLassifier options");
- /* Default classifier is 'bayes' for now */
- sub->default_key = "bayes";
-
- rspamd_rcl_add_default_handler (sub,
- "min_tokens",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_classifier_config, min_tokens),
- RSPAMD_CL_FLAG_INT_32,
- "Minimum count of tokens (words) to be considered for statistics");
- rspamd_rcl_add_default_handler (sub,
- "max_tokens",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_classifier_config, max_tokens),
- RSPAMD_CL_FLAG_INT_32,
- "Maximum count of tokens (words) to be considered for statistics");
- rspamd_rcl_add_default_handler (sub,
- "min_learns",
- rspamd_rcl_parse_struct_integer,
- G_STRUCT_OFFSET (struct rspamd_classifier_config, min_learns),
- RSPAMD_CL_FLAG_UINT,
- "Minimum number of learns for each statfile to use this classifier");
- rspamd_rcl_add_default_handler (sub,
- "backend",
- rspamd_rcl_parse_struct_string,
- G_STRUCT_OFFSET (struct rspamd_classifier_config, backend),
- 0,
- "Statfiles engine");
- rspamd_rcl_add_default_handler (sub,
- "name",
- rspamd_rcl_parse_struct_string,
- G_STRUCT_OFFSET (struct rspamd_classifier_config, name),
- 0,
- "Name of classifier");
+ if (!(skip_sections && g_hash_table_lookup (skip_sections, "worker"))) {
+ /**
+ * Worker section
+ */
+ sub = rspamd_rcl_add_section_doc (&new,
+ "worker", "type",
+ rspamd_rcl_worker_handler,
+ UCL_OBJECT,
+ FALSE,
+ TRUE,
+ cfg->doc_strings,
+ "Workers common options");
+ rspamd_rcl_add_default_handler (sub,
+ "count",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_worker_conf, count),
+ RSPAMD_CL_FLAG_INT_16,
+ "Number of workers to spawn");
+ rspamd_rcl_add_default_handler (sub,
+ "max_files",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_worker_conf, rlimit_nofile),
+ RSPAMD_CL_FLAG_INT_32,
+ "Maximum number of opened files per worker");
+ rspamd_rcl_add_default_handler (sub,
+ "max_core",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_worker_conf, rlimit_maxcore),
+ RSPAMD_CL_FLAG_INT_32,
+ "Max size of core file in bytes");
+ rspamd_rcl_add_default_handler (sub,
+ "enabled",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET (struct rspamd_worker_conf, enabled),
+ 0,
+ "Enable or disable a worker (true by default)");
+ }
- /*
- * Statfile defaults
- */
- ssub = rspamd_rcl_add_section_doc (&sub->subsections,
- "statfile", "symbol",
- rspamd_rcl_statfile_handler,
- UCL_OBJECT,
- TRUE,
- TRUE,
- sub->doc_ref,
- "Statfiles options");
- rspamd_rcl_add_default_handler (ssub,
- "label",
- rspamd_rcl_parse_struct_string,
- G_STRUCT_OFFSET (struct rspamd_statfile_config, label),
- 0,
- "Statfile unique label");
- rspamd_rcl_add_default_handler (ssub,
- "spam",
- rspamd_rcl_parse_struct_boolean,
- G_STRUCT_OFFSET (struct rspamd_statfile_config, is_spam),
- 0,
- "Sets if this statfile contains spam samples");
+ if (!(skip_sections && g_hash_table_lookup (skip_sections, "modules"))) {
+ /**
+ * Modules handler
+ */
+ sub = rspamd_rcl_add_section_doc (&new,
+ "modules", NULL,
+ rspamd_rcl_modules_handler,
+ UCL_OBJECT,
+ FALSE,
+ FALSE,
+ cfg->doc_strings,
+ "Lua plugins to load");
+ }
+
+ if (!(skip_sections && g_hash_table_lookup (skip_sections, "classifier"))) {
+ /**
+ * Classifiers handler
+ */
+ sub = rspamd_rcl_add_section_doc (&new,
+ "classifier", "type",
+ rspamd_rcl_classifier_handler,
+ UCL_OBJECT,
+ FALSE,
+ TRUE,
+ cfg->doc_strings,
+ "CLassifier options");
+ /* Default classifier is 'bayes' for now */
+ sub->default_key = "bayes";
+
+ rspamd_rcl_add_default_handler (sub,
+ "min_tokens",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_classifier_config, min_tokens),
+ RSPAMD_CL_FLAG_INT_32,
+ "Minimum count of tokens (words) to be considered for statistics");
+ rspamd_rcl_add_default_handler (sub,
+ "max_tokens",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_classifier_config, max_tokens),
+ RSPAMD_CL_FLAG_INT_32,
+ "Maximum count of tokens (words) to be considered for statistics");
+ rspamd_rcl_add_default_handler (sub,
+ "min_learns",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET (struct rspamd_classifier_config, min_learns),
+ RSPAMD_CL_FLAG_UINT,
+ "Minimum number of learns for each statfile to use this classifier");
+ rspamd_rcl_add_default_handler (sub,
+ "backend",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET (struct rspamd_classifier_config, backend),
+ 0,
+ "Statfiles engine");
+ rspamd_rcl_add_default_handler (sub,
+ "name",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET (struct rspamd_classifier_config, name),
+ 0,
+ "Name of classifier");
- /**
- * Composites handlers
- */
- sub = rspamd_rcl_add_section_doc (&new,
- "composite", "name",
- rspamd_rcl_composite_handler,
- UCL_OBJECT,
- FALSE,
- TRUE,
- cfg->doc_strings,
- "Rspamd composite symbols");
- sub = rspamd_rcl_add_section_doc (&new,
- "composites", NULL,
- rspamd_rcl_composites_handler,
- UCL_OBJECT,
- FALSE,
- TRUE,
- cfg->doc_strings,
- "Rspamd composite symbols");
+ /*
+ * Statfile defaults
+ */
+ ssub = rspamd_rcl_add_section_doc (&sub->subsections,
+ "statfile", "symbol",
+ rspamd_rcl_statfile_handler,
+ UCL_OBJECT,
+ TRUE,
+ TRUE,
+ sub->doc_ref,
+ "Statfiles options");
+ rspamd_rcl_add_default_handler (ssub,
+ "label",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET (struct rspamd_statfile_config, label),
+ 0,
+ "Statfile unique label");
+ rspamd_rcl_add_default_handler (ssub,
+ "spam",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET (struct rspamd_statfile_config, is_spam),
+ 0,
+ "Sets if this statfile contains spam samples");
+ }
- /**
- * Lua handler
- */
- sub = rspamd_rcl_add_section_doc (&new,
- "lua", NULL,
- rspamd_rcl_lua_handler,
- UCL_STRING,
- FALSE,
- TRUE,
- cfg->doc_strings,
- "Lua files to load");
+ if (!(skip_sections && g_hash_table_lookup (skip_sections, "composite"))) {
+ /**
+ * Composites handlers
+ */
+ sub = rspamd_rcl_add_section_doc (&new,
+ "composite", "name",
+ rspamd_rcl_composite_handler,
+ UCL_OBJECT,
+ FALSE,
+ TRUE,
+ cfg->doc_strings,
+ "Rspamd composite symbols");
+ sub = rspamd_rcl_add_section_doc (&new,
+ "composites", NULL,
+ rspamd_rcl_composites_handler,
+ UCL_OBJECT,
+ FALSE,
+ TRUE,
+ cfg->doc_strings,
+ "Rspamd composite symbols");
+ }
+
+ if (!(skip_sections && g_hash_table_lookup (skip_sections, "lua"))) {
+ /**
+ * Lua handler
+ */
+ sub = rspamd_rcl_add_section_doc (&new,
+ "lua", NULL,
+ rspamd_rcl_lua_handler,
+ UCL_STRING,
+ FALSE,
+ TRUE,
+ cfg->doc_strings,
+ "Lua files to load");
+ }
return new;
}
@@ -3518,7 +3204,7 @@ rspamd_rcl_emitter_append_double (double elt, void *ud)
return 0;
}
-static void
+void
rspamd_rcl_section_free (gpointer p)
{
struct rspamd_rcl_section *top = p, *cur, *tmp;
@@ -3561,7 +3247,7 @@ rspamd_rcl_section_free (gpointer p)
* it is changed, then rcl_obj is imported from lua. Old config is dereferenced.
* @param cfg
*/
-static void
+void
rspamd_rcl_maybe_apply_lua_transform (struct rspamd_config *cfg)
{
lua_State *L = cfg->lua_state;
@@ -3650,37 +3336,56 @@ rspamd_rcl_decrypt_free (unsigned char *data, size_t len, void *user_data)
g_free (data);
}
+void
+rspamd_config_calculate_cksum (struct rspamd_config *cfg)
+{
+ rspamd_cryptobox_hash_state_t hs;
+ unsigned char cksumbuf[rspamd_cryptobox_HASHBYTES];
+ struct ucl_emitter_functions f;
+
+ /* Calculate checksum */
+ rspamd_cryptobox_hash_init (&hs, NULL, 0);
+ f.ucl_emitter_append_character = rspamd_rcl_emitter_append_c;
+ f.ucl_emitter_append_double = rspamd_rcl_emitter_append_double;
+ f.ucl_emitter_append_int = rspamd_rcl_emitter_append_int;
+ f.ucl_emitter_append_len = rspamd_rcl_emitter_append_len;
+ f.ucl_emitter_free_func = NULL;
+ f.ud = &hs;
+ ucl_object_emit_full (cfg->rcl_obj, UCL_EMIT_MSGPACK,
+ &f, cfg->config_comments);
+ rspamd_cryptobox_hash_final (&hs, cksumbuf);
+ cfg->checksum = rspamd_encode_base32 (cksumbuf, sizeof (cksumbuf));
+ /* Also change the tag of cfg pool to be equal to the checksum */
+ rspamd_strlcpy (cfg->cfg_pool->tag.uid, cfg->checksum,
+ MIN (sizeof (cfg->cfg_pool->tag.uid), strlen (cfg->checksum)));
+}
+
gboolean
-rspamd_config_read (struct rspamd_config *cfg, const gchar *filename,
- const gchar *convert_to, rspamd_rcl_section_fin_t logger_fin,
- gpointer logger_ud, GHashTable *vars)
+rspamd_config_parse_ucl (struct rspamd_config *cfg, const gchar *filename,
+ GHashTable *vars, GError **err)
{
struct stat st;
gint fd;
- gchar *data;
- GError *err = NULL;
- struct rspamd_rcl_section *top, *logger_section;
struct ucl_parser *parser;
- const ucl_object_t *logger_obj;
- rspamd_cryptobox_hash_state_t hs;
- unsigned char cksumbuf[rspamd_cryptobox_HASHBYTES];
gchar keypair_path[PATH_MAX];
struct rspamd_cryptobox_keypair *decrypt_keypair = NULL;
- struct ucl_emitter_functions f;
+ gchar *data;
if (stat (filename, &st) == -1) {
- msg_err_config_forced ("cannot stat %s: %s", filename, strerror (errno));
+ g_set_error (err, cfg_rcl_error_quark (), errno,
+ "cannot stat %s: %s", filename, strerror (errno));
return FALSE;
}
if ((fd = open (filename, O_RDONLY)) == -1) {
- msg_err_config_forced ("cannot open %s: %s", filename, strerror (errno));
+ g_set_error (err, cfg_rcl_error_quark (), errno,
+ "cannot open %s: %s", filename, strerror (errno));
return FALSE;
}
/* Now mmap this file to simplify reading process */
- if ((data =
- mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) {
- msg_err_config_forced ("cannot mmap %s: %s", filename, strerror (errno));
+ if ((data = mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) {
+ g_set_error (err, cfg_rcl_error_quark (), errno,
+ "cannot mmap %s: %s", filename, strerror (errno));
close (fd);
return FALSE;
}
@@ -3725,7 +3430,6 @@ rspamd_config_read (struct rspamd_config *cfg, const gchar *filename,
ucl_parser_free (kp_parser);
}
- rspamd_cryptobox_hash_init (&hs, NULL, 0);
parser = ucl_parser_new (UCL_PARSER_SAVE_COMMENTS);
rspamd_ucl_add_conf_variables (parser, vars);
rspamd_ucl_add_conf_macros (parser, cfg);
@@ -3756,9 +3460,28 @@ rspamd_config_read (struct rspamd_config *cfg, const gchar *filename,
cfg->config_comments = ucl_object_ref (ucl_parser_get_comments (parser));
ucl_parser_free (parser);
- top = rspamd_rcl_config_init (cfg);
+ return TRUE;
+}
+
+gboolean
+rspamd_config_read (struct rspamd_config *cfg, const gchar *filename,
+ rspamd_rcl_section_fin_t logger_fin,
+ gpointer logger_ud, GHashTable *vars)
+{
+ GError *err = NULL;
+ struct rspamd_rcl_section *top, *logger_section;
+ const ucl_object_t *logger_obj;
+
+ if (!rspamd_config_parse_ucl (cfg, filename, vars, &err)) {
+ msg_err_config_forced ("failed to load config: %e", err);
+ g_error_free (err);
+
+ return FALSE;
+ }
+
+ top = rspamd_rcl_config_init (cfg, NULL);
rspamd_lua_set_path (cfg->lua_state, cfg->rcl_obj, vars);
- rspamd_rcl_set_lua_globals (cfg, cfg->lua_state, vars);
+ rspamd_lua_set_globals (cfg, cfg->lua_state, vars);
rspamd_mempool_add_destructor (cfg->cfg_pool, rspamd_rcl_section_free, top);
err = NULL;
@@ -3790,22 +3513,7 @@ rspamd_config_read (struct rspamd_config *cfg, const gchar *filename,
/* Transform config if needed */
rspamd_rcl_maybe_apply_lua_transform (cfg);
-
- /* Calculate checksum */
- f.ucl_emitter_append_character = rspamd_rcl_emitter_append_c;
- f.ucl_emitter_append_double = rspamd_rcl_emitter_append_double;
- f.ucl_emitter_append_int = rspamd_rcl_emitter_append_int;
- f.ucl_emitter_append_len = rspamd_rcl_emitter_append_len;
- f.ucl_emitter_free_func = NULL;
- f.ud = &hs;
- ucl_object_emit_full (cfg->rcl_obj, UCL_EMIT_MSGPACK,
- &f, cfg->config_comments);
- rspamd_cryptobox_hash_final (&hs, cksumbuf);
- cfg->checksum = rspamd_encode_base32 (cksumbuf, sizeof (cksumbuf));
- /* Also change the tag of cfg pool to be equal to the checksum */
- rspamd_strlcpy (cfg->cfg_pool->tag.uid, cfg->checksum,
- MIN (sizeof (cfg->cfg_pool->tag.uid), strlen (cfg->checksum)));
-
+ rspamd_config_calculate_cksum (cfg);
if (!rspamd_rcl_parse (top, cfg, cfg, cfg->cfg_pool, cfg->rcl_obj, &err)) {
msg_err_config ("rcl parse error: %e", err);
diff --git a/src/libserver/cfg_rcl.h b/src/libserver/cfg_rcl.h
index c1d958efc..b14b98f8b 100644
--- a/src/libserver/cfg_rcl.h
+++ b/src/libserver/cfg_rcl.h
@@ -134,7 +134,8 @@ struct rspamd_rcl_section *rspamd_rcl_add_section_doc (
* Init common sections known to rspamd
* @return top section
*/
-struct rspamd_rcl_section * rspamd_rcl_config_init (struct rspamd_config *cfg);
+struct rspamd_rcl_section * rspamd_rcl_config_init (struct rspamd_config *cfg,
+ GHashTable *skip_sections);
/**
* Get a section specified by path, it understand paths separated by '/' character
@@ -446,11 +447,41 @@ gboolean rspamd_rcl_add_lua_plugins_path (struct rspamd_config *cfg,
const gchar *path,
GError **err);
+
+/**
+ * Calls for an external lua function to apply potential config transformations
+ * if needed. This function can change the cfg->rcl_obj.
+ *
+ * Example of transformation function:
+ *
+ * function(obj)
+ * if obj.something == 'foo' then
+ * obj.something = "bla"
+ * return true, obj
+ * end
+ *
+ * return false, nil
+ * end
+ *
+ * If function returns 'false' then rcl_obj is not touched. Otherwise,
+ * it is changed, then rcl_obj is imported from lua. Old config is dereferenced.
+ * @param cfg
+ */
+void rspamd_rcl_maybe_apply_lua_transform (struct rspamd_config *cfg);
+void rspamd_rcl_section_free (gpointer p);
+
+void rspamd_config_calculate_cksum (struct rspamd_config *cfg);
+
/*
* Read configuration file
*/
+gboolean rspamd_config_parse_ucl (struct rspamd_config *cfg,
+ const gchar *filename,
+ GHashTable *vars,
+ GError **err);
gboolean rspamd_config_read (struct rspamd_config *cfg,
- const gchar *filename, const gchar *convert_to,
- rspamd_rcl_section_fin_t logger_fin, gpointer logger_ud,
- GHashTable *vars);
+ const gchar *filename,
+ rspamd_rcl_section_fin_t logger_fin,
+ gpointer logger_ud,
+ GHashTable *vars);
#endif /* CFG_RCL_H_ */
diff --git a/src/libserver/cfg_utils.c b/src/libserver/cfg_utils.c
index bf63b2188..e58232a00 100644
--- a/src/libserver/cfg_utils.c
+++ b/src/libserver/cfg_utils.c
@@ -114,7 +114,7 @@ rspamd_parse_bind_line (struct rspamd_config *cfg,
}
struct rspamd_config *
-rspamd_config_new (void)
+rspamd_config_new (enum rspamd_config_init_flags flags)
{
struct rspamd_config *cfg;
@@ -172,7 +172,11 @@ rspamd_config_new (void)
cfg->min_word_len = DEFAULT_MIN_WORD;
cfg->max_word_len = DEFAULT_MAX_WORD;
- cfg->lua_state = rspamd_lua_init ();
+ if (!(flags & RSPAMD_CONFIG_INIT_SKIP_LUA)) {
+ cfg->lua_state = rspamd_lua_init ();
+ cfg->own_lua_state = TRUE;
+ }
+
cfg->cache = rspamd_symbols_cache_new (cfg);
cfg->ups_ctx = rspamd_upstreams_library_init ();
cfg->re_cache = rspamd_re_cache_new ();
@@ -251,7 +255,10 @@ rspamd_config_free (struct rspamd_config *cfg)
rspamd_re_cache_unref (cfg->re_cache);
rspamd_upstreams_library_unref (cfg->ups_ctx);
rspamd_mempool_delete (cfg->cfg_pool);
- lua_close (cfg->lua_state);
+
+ if (cfg->lua_state && cfg->own_lua_state) {
+ lua_close (cfg->lua_state);
+ }
REF_RELEASE (cfg->libs_ctx);
DL_FOREACH_SAFE (cfg->log_pipes, lp, ltmp) {
@@ -1105,12 +1112,15 @@ rspamd_include_map_handler (const guchar *data, gsize len,
#define RSPAMD_VERSION_MINOR_MACRO "VERSION_MINOR"
#define RSPAMD_VERSION_PATCH_MACRO "VERSION_PATCH"
#define RSPAMD_BRANCH_VERSION_MACRO "BRANCH_VERSION"
+#define RSPAMD_HOSTNAME_MACRO "HOSTNAME"
void
rspamd_ucl_add_conf_variables (struct ucl_parser *parser, GHashTable *vars)
{
GHashTableIter it;
gpointer k, v;
+ gchar *hostbuf;
+ gsize hostlen;
ucl_parser_register_variable (parser,
RSPAMD_CONFDIR_MACRO,
@@ -1152,6 +1162,23 @@ rspamd_ucl_add_conf_variables (struct ucl_parser *parser, GHashTable *vars)
"no");
#endif
+ hostlen = sysconf (_SC_HOST_NAME_MAX);
+
+ if (hostlen <= 0) {
+ hostlen = 256;
+ }
+ else {
+ hostlen ++;
+ }
+
+ hostbuf = g_alloca (hostlen);
+ memset (hostbuf, 0, hostlen);
+ gethostname (hostbuf, hostlen - 1);
+
+ /* UCL copies variables, so it is safe to pass an ephemeral buffer here */
+ ucl_parser_register_variable (parser, RSPAMD_HOSTNAME_MACRO,
+ hostbuf);
+
if (vars != NULL) {
g_hash_table_iter_init (&it, vars);
diff --git a/src/libserver/composites.c b/src/libserver/composites.c
index a90b5c032..f3dc7a956 100644
--- a/src/libserver/composites.c
+++ b/src/libserver/composites.c
@@ -266,7 +266,7 @@ rspamd_composite_expr_process (gpointer input, rspamd_expression_atom_t *atom)
t = *beg;
if (t == '~') {
- nrd->action &= ~RSPAMD_COMPOSITE_REMOVE_WEIGHT;
+ nrd->action &= ~RSPAMD_COMPOSITE_REMOVE_SYMBOL;
}
else if (t == '-') {
nrd->action &= ~(RSPAMD_COMPOSITE_REMOVE_WEIGHT|
diff --git a/src/libserver/dkim.c b/src/libserver/dkim.c
index 5af61daf7..4aaa74a3b 100644
--- a/src/libserver/dkim.c
+++ b/src/libserver/dkim.c
@@ -65,12 +65,14 @@ enum rspamd_sign_type {
DKIM_SIGN_RSASHA256,
DKIM_SIGN_RSASHA512,
DKIM_SIGN_ECDSASHA256,
- DKIM_SIGN_ECDSASHA512
+ DKIM_SIGN_ECDSASHA512,
+ DKIM_SIGN_EDDSASHA256,
};
enum rspamd_dkim_key_type {
RSPAMD_DKIM_KEY_RSA = 0,
- RSPAMD_DKIM_KEY_ECDSA
+ RSPAMD_DKIM_KEY_ECDSA,
+ RSPAMD_DKIM_KEY_EDDSA
};
#define RSPAMD_DKIM_MAX_ARC_IDX 10
@@ -152,6 +154,7 @@ struct rspamd_dkim_key_s {
union {
RSA *key_rsa;
EC_KEY *key_ecdsa;
+ guchar *key_eddsa;
} key;
enum rspamd_dkim_key_type type;
BIO *key_bio;
@@ -315,6 +318,12 @@ rspamd_dkim_parse_signalg (rspamd_dkim_context_t * ctx,
return TRUE;
}
}
+ else if (len == sizeof ("ed25519") - 1) {
+ if (memcmp (param, "ed25519", len) == 0) {
+ ctx->sig_alg = DKIM_SIGN_EDDSASHA256;
+ return TRUE;
+ }
+ }
g_set_error (err,
DKIM_ERROR,
@@ -1219,7 +1228,10 @@ rspamd_dkim_make_key (rspamd_dkim_context_t *ctx, const gchar *keydata,
rspamd_dkim_key_t *key = NULL;
if (keylen < 3) {
- msg_err_dkim ("DKIM key is too short to be valid");
+ g_set_error (err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_KEYFAIL,
+ "DKIM key is too short to be valid");
return NULL;
}
@@ -1233,55 +1245,71 @@ rspamd_dkim_make_key (rspamd_dkim_context_t *ctx, const gchar *keydata,
rspamd_cryptobox_base64_decode (keydata, keylen, key->keydata,
&key->decoded_len);
- key->key_bio = BIO_new_mem_buf (key->keydata, key->decoded_len);
+ if (key->type == RSPAMD_DKIM_KEY_EDDSA) {
+ key->key.key_eddsa = key->keydata;
- if (key->key_bio == NULL) {
- g_set_error (err,
- DKIM_ERROR,
- DKIM_SIGERROR_KEYFAIL,
- "cannot make ssl bio from key");
- REF_RELEASE (key);
-
- return NULL;
- }
-
- key->key_evp = d2i_PUBKEY_bio (key->key_bio, NULL);
-
- if (key->key_evp == NULL) {
- g_set_error (err,
- DKIM_ERROR,
- DKIM_SIGERROR_KEYFAIL,
- "cannot extract pubkey from bio");
- REF_RELEASE (key);
+ if (key->decoded_len != rspamd_cryptobox_pk_sig_bytes (
+ RSPAMD_CRYPTOBOX_MODE_25519)) {
+ g_set_error (err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_KEYFAIL,
+ "DKIM key is has invalid length %d for eddsa",
+ (gint)key->decoded_len);
+ REF_RELEASE (key);
- return NULL;
+ return NULL;
+ }
}
+ else {
+ key->key_bio = BIO_new_mem_buf (key->keydata, key->decoded_len);
- if (type == RSPAMD_DKIM_KEY_RSA) {
- key->key.key_rsa = EVP_PKEY_get1_RSA (key->key_evp);
-
- if (key->key.key_rsa == NULL) {
+ if (key->key_bio == NULL) {
g_set_error (err,
DKIM_ERROR,
DKIM_SIGERROR_KEYFAIL,
- "cannot extract rsa key from evp key");
+ "cannot make ssl bio from key");
REF_RELEASE (key);
return NULL;
}
- }
- else {
- key->key.key_ecdsa = EVP_PKEY_get1_EC_KEY (key->key_evp);
- if (key->key.key_ecdsa == NULL) {
+ key->key_evp = d2i_PUBKEY_bio (key->key_bio, NULL);
+
+ if (key->key_evp == NULL) {
g_set_error (err,
DKIM_ERROR,
DKIM_SIGERROR_KEYFAIL,
- "cannot extract ecdsa key from evp key");
+ "cannot extract pubkey from bio");
REF_RELEASE (key);
return NULL;
}
+
+ if (type == RSPAMD_DKIM_KEY_RSA) {
+ key->key.key_rsa = EVP_PKEY_get1_RSA (key->key_evp);
+
+ if (key->key.key_rsa == NULL) {
+ g_set_error (err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_KEYFAIL,
+ "cannot extract rsa key from evp key");
+ REF_RELEASE (key);
+
+ return NULL;
+ }
+ } else {
+ key->key.key_ecdsa = EVP_PKEY_get1_EC_KEY (key->key_evp);
+
+ if (key->key.key_ecdsa == NULL) {
+ g_set_error (err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_KEYFAIL,
+ "cannot extract ecdsa key from evp key");
+ REF_RELEASE (key);
+
+ return NULL;
+ }
+ }
}
return key;
@@ -1303,11 +1331,12 @@ rspamd_dkim_key_free (rspamd_dkim_key_t *key)
RSA_free (key->key.key_rsa);
}
}
- else {
+ else if (key->type == RSPAMD_DKIM_KEY_ECDSA) {
if (key->key.key_ecdsa) {
EC_KEY_free (key->key.key_ecdsa);
}
}
+ /* Nothing in case of eddsa key */
if (key->key_bio) {
BIO_free (key->key_bio);
}
@@ -1433,14 +1462,18 @@ rspamd_dkim_parse_key (rspamd_dkim_context_t *ctx, const gchar *txt,
alglen = 3;
}
- if (alglen == 8 && rspamd_lc_cmp (alg, "ecdsa256", alglen) == 0) {
- if (keylen) {
- *keylen = klen;
- }
+ if (keylen) {
+ *keylen = klen;
+ }
+ if (alglen == 8 && rspamd_lc_cmp (alg, "ecdsa256", alglen) == 0) {
return rspamd_dkim_make_key (ctx, c, klen,
RSPAMD_DKIM_KEY_ECDSA, err);
}
+ else if (alglen == 7 && rspamd_lc_cmp (alg, "ed25519", alglen) == 0) {
+ return rspamd_dkim_make_key (ctx, c, klen,
+ RSPAMD_DKIM_KEY_EDDSA, err);
+ }
else {
/* We assume RSA default in all cases */
if (alglen != 3 || rspamd_lc_cmp (alg, "rsa", alglen) != 0) {
@@ -2419,7 +2452,8 @@ rspamd_dkim_check (rspamd_dkim_context_t *ctx,
nid = NID_sha1;
}
else if (ctx->sig_alg == DKIM_SIGN_RSASHA256 ||
- ctx->sig_alg == DKIM_SIGN_ECDSASHA256) {
+ ctx->sig_alg == DKIM_SIGN_ECDSASHA256 ||
+ ctx->sig_alg == DKIM_SIGN_EDDSASHA256) {
nid = NID_sha256;
}
else if (ctx->sig_alg == DKIM_SIGN_RSASHA512 ||
@@ -2431,21 +2465,31 @@ rspamd_dkim_check (rspamd_dkim_context_t *ctx,
nid = NID_sha1;
}
- if (key->type == RSPAMD_DKIM_KEY_RSA) {
+ switch (key->type) {
+ case RSPAMD_DKIM_KEY_RSA:
if (RSA_verify (nid, raw_digest, dlen, ctx->b, ctx->blen,
key->key.key_rsa) != 1) {
msg_debug_dkim ("rsa verify failed");
res = DKIM_REJECT;
}
- }
- else {
+ break;
+ case RSPAMD_DKIM_KEY_ECDSA:
if (ECDSA_verify (nid, raw_digest, dlen, ctx->b, ctx->blen,
key->key.key_ecdsa) != 1) {
msg_debug_dkim ("ecdsa verify failed");
res = DKIM_REJECT;
}
+ break;
+ case RSPAMD_DKIM_KEY_EDDSA:
+ if (!rspamd_cryptobox_verify (ctx->b, ctx->blen, raw_digest, dlen,
+ key->key.key_eddsa, RSPAMD_CRYPTOBOX_MODE_25519)) {
+ msg_debug_dkim ("eddsa verify failed");
+ res = DKIM_REJECT;
+ }
+ break;
}
+
if (ctx->common.type == RSPAMD_DKIM_ARC_SEAL && res == DKIM_CONTINUE) {
switch (ctx->cv) {
case RSPAMD_ARC_INVALID:
diff --git a/src/libserver/symbols_cache.c b/src/libserver/symbols_cache.c
index abed26fc3..23ad15ed1 100644
--- a/src/libserver/symbols_cache.c
+++ b/src/libserver/symbols_cache.c
@@ -2037,7 +2037,7 @@ rspamd_symbols_cache_counters (struct symbols_cache * cache)
top = ucl_object_typed_new (UCL_ARRAY);
cbd.top = top;
cbd.cache = cache;
- g_ptr_array_foreach (cache->items_by_order->d,
+ g_ptr_array_foreach (cache->items_by_id,
rspamd_symbols_cache_counters_cb, &cbd);
return top;
diff --git a/src/libserver/task.c b/src/libserver/task.c
index b5594816b..9be780b1b 100644
--- a/src/libserver/task.c
+++ b/src/libserver/task.c
@@ -738,6 +738,12 @@ rspamd_task_process (struct rspamd_task *task, guint stages)
RSPAMD_TASK_STAGE_PRE_FILTERS);
break;
+ case RSPAMD_TASK_STAGE_PROCESS_MESSAGE:
+ if (!(task->flags & RSPAMD_TASK_FLAG_SKIP_PROCESS)) {
+ rspamd_message_process (task);
+ }
+ break;
+
case RSPAMD_TASK_STAGE_FILTERS:
rspamd_symbols_cache_process_symbols (task, task->cfg->cache,
RSPAMD_TASK_STAGE_FILTERS);
diff --git a/src/libserver/task.h b/src/libserver/task.h
index e3c0492f6..b9fd2f811 100644
--- a/src/libserver/task.h
+++ b/src/libserver/task.h
@@ -41,25 +41,27 @@ enum rspamd_task_stage {
RSPAMD_TASK_STAGE_ENVELOPE = (1 << 1),
RSPAMD_TASK_STAGE_READ_MESSAGE = (1 << 2),
RSPAMD_TASK_STAGE_PRE_FILTERS = (1 << 3),
- RSPAMD_TASK_STAGE_FILTERS = (1 << 4),
- RSPAMD_TASK_STAGE_CLASSIFIERS_PRE = (1 << 5),
- RSPAMD_TASK_STAGE_CLASSIFIERS = (1 << 6),
- RSPAMD_TASK_STAGE_CLASSIFIERS_POST = (1 << 7),
- RSPAMD_TASK_STAGE_COMPOSITES = (1 << 8),
- RSPAMD_TASK_STAGE_POST_FILTERS = (1 << 9),
- RSPAMD_TASK_STAGE_LEARN_PRE = (1 << 10),
- RSPAMD_TASK_STAGE_LEARN = (1 << 11),
- RSPAMD_TASK_STAGE_LEARN_POST = (1 << 12),
- RSPAMD_TASK_STAGE_COMPOSITES_POST = (1 << 13),
- RSPAMD_TASK_STAGE_IDEMPOTENT = (1 << 14),
- RSPAMD_TASK_STAGE_DONE = (1 << 15),
- RSPAMD_TASK_STAGE_REPLIED = (1 << 16)
+ RSPAMD_TASK_STAGE_PROCESS_MESSAGE = (1 << 4),
+ RSPAMD_TASK_STAGE_FILTERS = (1 << 5),
+ RSPAMD_TASK_STAGE_CLASSIFIERS_PRE = (1 << 6),
+ RSPAMD_TASK_STAGE_CLASSIFIERS = (1 << 7),
+ RSPAMD_TASK_STAGE_CLASSIFIERS_POST = (1 << 8),
+ RSPAMD_TASK_STAGE_COMPOSITES = (1 << 9),
+ RSPAMD_TASK_STAGE_POST_FILTERS = (1 << 10),
+ RSPAMD_TASK_STAGE_LEARN_PRE = (1 << 11),
+ RSPAMD_TASK_STAGE_LEARN = (1 << 12),
+ RSPAMD_TASK_STAGE_LEARN_POST = (1 << 13),
+ RSPAMD_TASK_STAGE_COMPOSITES_POST = (1 << 14),
+ RSPAMD_TASK_STAGE_IDEMPOTENT = (1 << 15),
+ RSPAMD_TASK_STAGE_DONE = (1 << 16),
+ RSPAMD_TASK_STAGE_REPLIED = (1 << 17)
};
#define RSPAMD_TASK_PROCESS_ALL (RSPAMD_TASK_STAGE_CONNECT | \
RSPAMD_TASK_STAGE_ENVELOPE | \
RSPAMD_TASK_STAGE_READ_MESSAGE | \
RSPAMD_TASK_STAGE_PRE_FILTERS | \
+ RSPAMD_TASK_STAGE_PROCESS_MESSAGE | \
RSPAMD_TASK_STAGE_FILTERS | \
RSPAMD_TASK_STAGE_CLASSIFIERS_PRE | \
RSPAMD_TASK_STAGE_CLASSIFIERS | \
@@ -75,6 +77,7 @@ enum rspamd_task_stage {
#define RSPAMD_TASK_PROCESS_LEARN (RSPAMD_TASK_STAGE_CONNECT | \
RSPAMD_TASK_STAGE_ENVELOPE | \
RSPAMD_TASK_STAGE_READ_MESSAGE | \
+ RSPAMD_TASK_STAGE_PROCESS_MESSAGE | \
RSPAMD_TASK_STAGE_CLASSIFIERS_PRE | \
RSPAMD_TASK_STAGE_CLASSIFIERS | \
RSPAMD_TASK_STAGE_CLASSIFIERS_POST | \
@@ -85,7 +88,7 @@ enum rspamd_task_stage {
#define RSPAMD_TASK_FLAG_MIME (1 << 0)
#define RSPAMD_TASK_FLAG_JSON (1 << 1)
-#define RSPAMD_TASK_FLAG_SKIP_EXTRA (1 << 2)
+#define RSPAMD_TASK_FLAG_SKIP_PROCESS (1 << 2)
#define RSPAMD_TASK_FLAG_SKIP (1 << 3)
#define RSPAMD_TASK_FLAG_EXT_URLS (1 << 4)
#define RSPAMD_TASK_FLAG_SPAMC (1 << 5)
diff --git a/src/libserver/url.c b/src/libserver/url.c
index 23826d51b..650f86a28 100644
--- a/src/libserver/url.c
+++ b/src/libserver/url.c
@@ -1012,13 +1012,46 @@ rspamd_web_parse (struct http_parser_url *u, const gchar *str, gsize len,
break;
case parse_port_password:
if (g_ascii_isdigit (t)) {
- /* XXX: that breaks urls with passwords starting with number */
- st = parse_port;
- c = slash;
- p--;
- SET_U (u, UF_HOST);
- p++;
- c = p;
+ const gchar *tmp = p;
+
+ while (tmp < last) {
+ if (!g_ascii_isdigit (*tmp)) {
+ if (*tmp == '/' || *tmp == '#' || *tmp == '?' ||
+ is_url_end (*tmp) || g_ascii_isspace (*tmp)) {
+ /* Port + something */
+ st = parse_port;
+ c = slash;
+ p--;
+ SET_U (u, UF_HOST);
+ p++;
+ c = p;
+ break;
+ }
+ else {
+ /* Not a port, bad character at the end */
+ break;
+ }
+ }
+ tmp ++;
+ }
+
+ if (tmp == last) {
+ /* Host + port only */
+ st = parse_port;
+ c = slash;
+ p--;
+ SET_U (u, UF_HOST);
+ p++;
+ c = p;
+ }
+
+ if (st != parse_port) {
+ /* Fallback to user:password */
+ p = slash;
+ c = slash;
+ user_seen = TRUE;
+ st = parse_user;
+ }
}
else {
/* Rewind back */
diff --git a/src/libstat/stat_process.c b/src/libstat/stat_process.c
index 070b9d6ca..4f7f4e703 100644
--- a/src/libstat/stat_process.c
+++ b/src/libstat/stat_process.c
@@ -190,13 +190,8 @@ rspamd_stat_tokenize_parts_metadata (struct rspamd_stat_ctx *st_ctx,
hdr = cur->data;
if (hdr->name && hdr->type != RSPAMD_HEADER_RECEIVED) {
- /* We assume that headers count is not more than 10^10 */
- gsize nlen = strlen (hdr->name) + 1 + 10;
- gchar *hdrbuf = rspamd_mempool_alloc (task->task_pool, nlen);
- nlen = rspamd_snprintf (hdrbuf, nlen, "%s:%d", hdr->name, hdr->order);
- rspamd_str_lc (hdrbuf, nlen);
- elt.begin = hdrbuf;
- elt.len = nlen;
+ elt.begin = hdr->name;
+ elt.len = strlen (hdr->name);
g_array_append_val (ar, elt);
}
diff --git a/src/libutil/map.c b/src/libutil/map.c
index 577933c25..2873b76f6 100644
--- a/src/libutil/map.c
+++ b/src/libutil/map.c
@@ -150,7 +150,7 @@ rspamd_map_check_sig_pk_mem (const guchar *sig,
ret = FALSE;
}
- if (ret && !rspamd_cryptobox_verify (sig, input, inlen,
+ if (ret && !rspamd_cryptobox_verify (sig, siglen, input, inlen,
rspamd_pubkey_get_pk (pk, NULL), RSPAMD_CRYPTOBOX_MODE_25519)) {
msg_err_map ("can't verify signature for %s: incorrect signature", map->name);
diff --git a/src/libutil/radix.c b/src/libutil/radix.c
index a42be7032..3ea6c6b8c 100644
--- a/src/libutil/radix.c
+++ b/src/libutil/radix.c
@@ -42,6 +42,7 @@ struct radix_tree_compressed {
rspamd_mempool_t *pool;
struct btrie *tree;
size_t size;
+ guint duplicates;
gboolean own_pool;
};
@@ -68,6 +69,7 @@ radix_insert_compressed (radix_compressed_t * tree,
gsize masklen,
uintptr_t value)
{
+ static const guint max_duplicates = 32;
guint keybits = keylen * NBBY;
uintptr_t old;
gchar ip_str[INET6_ADDRSTRLEN + 1];
@@ -85,23 +87,29 @@ radix_insert_compressed (radix_compressed_t * tree,
(gconstpointer)value);
if (ret != BTRIE_OKAY) {
- memset (ip_str, 0, sizeof (ip_str));
+ tree->duplicates++;
- if (keybits == 32) {
- msg_err_radix ("cannot insert %p, key: %s/%d, duplicate value",
- (gpointer)value,
- inet_ntop (AF_INET, key, ip_str, sizeof (ip_str) - 1),
- (gint)(keybits - masklen));
+ if (tree->duplicates == max_duplicates) {
+ msg_err_radix ("maximum duplicates limit reached: %d, "
+ "suppress further errors", max_duplicates);
}
- else if (keybits == 128) {
- msg_err_radix ("cannot insert %p, key: [%s]/%d, duplicate value",
- (gpointer)value,
- inet_ntop (AF_INET6, key, ip_str, sizeof (ip_str) - 1),
- (gint)(keybits - masklen));
- }
- else {
- msg_err_radix ("cannot insert %p with mask %z, key: %*xs, duplicate value",
- (gpointer)value, keybits - masklen, (int)keylen, key);
+ else if (tree->duplicates < max_duplicates) {
+ memset (ip_str, 0, sizeof (ip_str));
+
+ if (keybits == 32) {
+ msg_err_radix ("cannot insert %p, key: %s/%d, duplicate value",
+ (gpointer) value,
+ inet_ntop (AF_INET, key, ip_str, sizeof (ip_str) - 1),
+ (gint) (keybits - masklen));
+ } else if (keybits == 128) {
+ msg_err_radix ("cannot insert %p, key: [%s]/%d, duplicate value",
+ (gpointer) value,
+ inet_ntop (AF_INET6, key, ip_str, sizeof (ip_str) - 1),
+ (gint) (keybits - masklen));
+ } else {
+ msg_err_radix ("cannot insert %p with mask %z, key: %*xs, duplicate value",
+ (gpointer) value, keybits - masklen, (int) keylen, key);
+ }
}
}
else {
@@ -124,6 +132,7 @@ radix_create_compressed (void)
tree->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL);
tree->size = 0;
+ tree->duplicates = 0;
tree->tree = btrie_init (tree->pool);
tree->own_pool = TRUE;
@@ -138,6 +147,7 @@ radix_create_compressed_with_pool (rspamd_mempool_t *pool)
tree = rspamd_mempool_alloc (pool, sizeof (*tree));
tree->pool = pool;
tree->size = 0;
+ tree->duplicates = 0;
tree->tree = btrie_init (tree->pool);
tree->own_pool = FALSE;
@@ -393,5 +403,5 @@ radix_get_info (radix_compressed_t *tree)
return NULL;
}
- return btrie_stats (tree->tree);
+ return btrie_stats (tree->tree, tree->duplicates);
}
diff --git a/src/libutil/str_util.c b/src/libutil/str_util.c
index fc8d6637b..026e331fe 100644
--- a/src/libutil/str_util.c
+++ b/src/libutil/str_util.c
@@ -866,9 +866,8 @@ rspamd_strings_levenshtein_distance (const gchar *s1, gsize s1len,
const gchar *s2, gsize s2len,
guint replace_cost)
{
- guint x, y, lastdiag, olddiag;
- gchar c1, c2;
- guint *column;
+ gchar c1, c2, last_c2, last_c1;
+ static GArray *current_row = NULL, *prev_row = NULL, *transp_row = NULL;
gint eq;
static const guint max_cmp = 8192;
gint ret;
@@ -885,31 +884,79 @@ rspamd_strings_levenshtein_distance (const gchar *s1, gsize s1len,
if (MAX(s1len, s2len) > max_cmp) {
/* Cannot compare too many characters */
- return 0;
+ return max_cmp;
+ }
+
+ if (s1len > s2len) {
+ /* Exchange s1 and s2 */
+ const gchar *tmp;
+ gsize tmplen;
+
+ tmp = s2;
+ s2 = s1;
+ s1 = tmp;
+
+ tmplen = s2len;
+ s2len = s1len;
+ s1len = tmplen;
+ }
+
+ /* Adjust static space */
+ if (current_row == NULL) {
+ current_row = g_array_sized_new (FALSE, FALSE, sizeof (gint), s1len + 1);
+ prev_row = g_array_sized_new (FALSE, FALSE, sizeof (gint), s1len + 1);
+ transp_row = g_array_sized_new (FALSE, FALSE, sizeof (gint), s1len + 1);
+ g_array_set_size (current_row, s1len + 1);
+ g_array_set_size (prev_row, s1len + 1);
+ g_array_set_size (transp_row, s1len + 1);
+ }
+ else if (current_row->len < s1len + 1) {
+ g_array_set_size (current_row, s1len + 1);
+ g_array_set_size (prev_row, s1len + 1);
+ g_array_set_size (transp_row, s1len + 1);
}
- column = g_malloc0 ((s1len + 1) * sizeof (guint));
+ memset (current_row->data, 0, (s1len + 1) * sizeof (gint));
+ memset (transp_row->data, 0, (s1len + 1) * sizeof (gint));
- for (y = 1; y <= s1len; y++) {
- column[y] = y;
+ for (gint i = 0; i <= s1len; i++) {
+ g_array_index (prev_row, gint, i) = i;
}
- for (x = 1; x <= s2len; x++) {
- column[0] = x;
+ last_c2 = '\0';
+
+ for (gint i = 1; i <= s2len; i++) {
+ c2 = s2[i - 1];
+ g_array_index (current_row, gint, 0) = i;
+ last_c1 = '\0';
- for (y = 1, lastdiag = x - 1; y <= s1len; y++) {
- olddiag = column[y];
- c1 = s1[y - 1];
- c2 = s2[x - 1];
- eq = (c1 == c2) ? 0 : replace_cost;
- column[y] = MIN3 (column[y] + 1, column[y - 1] + 1,
- lastdiag + (eq));
- lastdiag = olddiag;
+ for (gint j = 1; j <= s1len; j++) {
+ c1 = s1[j - 1];
+ eq = c1 == c2 ? 0 : replace_cost;
+ ret = MIN3 (g_array_index (current_row, gint, j - 1) + 1, /* Insert */
+ g_array_index (prev_row, gint, j) + 1, /* Remove */
+ g_array_index (prev_row, gint, j - 1) + eq /* Replace */);
+
+ /* Take reordering into account */
+ if (c1 == last_c2 && c2 == last_c1 && j >= 2) {
+ ret = MIN (ret, g_array_index (transp_row, gint, j - 2) + eq);
+ }
+
+ g_array_index (current_row, gint, j) = ret;
+ last_c1 = c1;
}
+
+ last_c2 = c2;
+
+ /* Exchange pointers */
+ GArray *tmp;
+ tmp = transp_row;
+ transp_row = prev_row;
+ prev_row = current_row;
+ current_row = tmp;
}
- ret = column[s1len];
- g_free (column);
+ ret = g_array_index (prev_row, gint, s1len);
return ret;
}
@@ -1214,7 +1261,7 @@ rspamd_substring_preprocess_kmp (const gchar *pat, gsize len, goffset *fsm,
i++;
j++;
- if (f(pat[i], pat[j])) {
+ if (i < len && j < len && f(pat[i], pat[j])) {
fsm[i] = fsm[j];
}
else {
diff --git a/src/libutil/util.c b/src/libutil/util.c
index d55a8e8ac..510b16045 100644
--- a/src/libutil/util.c
+++ b/src/libutil/util.c
@@ -80,6 +80,7 @@
#endif
#endif
#include <math.h> /* for pow */
+#include <glob.h> /* in fact, we require this file ultimately */
#include "cryptobox.h"
#include "zlib.h"
@@ -2846,4 +2847,100 @@ rspamd_fstring_gzip (rspamd_fstring_t **in)
*in = comp;
return TRUE;
+}
+
+static gboolean
+rspamd_glob_dir (const gchar *full_path, const gchar *pattern,
+ gboolean recursive, guint rec_len,
+ GPtrArray *res, GError **err)
+{
+ glob_t globbuf;
+ const gchar *path;
+ static gchar pathbuf[PATH_MAX]; /* Static to help recursion */
+ guint i;
+ gint rc;
+ static const guint rec_lim = 16;
+ struct stat st;
+
+ if (rec_len > rec_lim) {
+ g_set_error (err, g_quark_from_static_string ("glob"), EOVERFLOW,
+ "maximum nesting is reached: %d", rec_lim);
+
+ return FALSE;
+ }
+
+ memset (&globbuf, 0, sizeof (globbuf));
+
+ if ((rc = glob (full_path, 0, NULL, &globbuf)) != 0) {
+
+ if (rc != GLOB_NOMATCH) {
+ g_set_error (err, g_quark_from_static_string ("glob"), errno,
+ "glob %s failed: %s", full_path, strerror (errno));
+ globfree (&globbuf);
+
+ return FALSE;
+ }
+ else {
+ globfree (&globbuf);
+
+ return TRUE;
+ }
+ }
+
+ for (i = 0; i < globbuf.gl_pathc; i ++) {
+ path = globbuf.gl_pathv[i];
+
+ if (stat (path, &st) == -1) {
+ if (errno == EPERM || errno == EACCES || errno == ELOOP) {
+ /* Silently ignore */
+ continue;
+ }
+
+ g_set_error (err, g_quark_from_static_string ("glob"), errno,
+ "stat %s failed: %s", path, strerror (errno));
+ globfree (&globbuf);
+
+ return FALSE;
+ }
+
+ if (S_ISREG (st.st_mode)) {
+ g_ptr_array_add (res, g_strdup (path));
+ }
+ else if (recursive && S_ISDIR (st.st_mode)) {
+ rspamd_snprintf (pathbuf, sizeof (pathbuf), "%s%c%s",
+ path, G_DIR_SEPARATOR, pattern);
+
+ if (!rspamd_glob_dir (full_path, pattern, recursive, rec_len + 1,
+ res, err)) {
+ globfree (&globbuf);
+
+ return FALSE;
+ }
+ }
+ }
+
+ globfree (&globbuf);
+
+ return TRUE;
+}
+
+GPtrArray *
+rspamd_glob_path (const gchar *dir,
+ const gchar *pattern,
+ gboolean recursive,
+ GError **err)
+{
+ gchar path[PATH_MAX];
+ GPtrArray *res;
+
+ res = g_ptr_array_new_full (32, (GDestroyNotify)g_free);
+ rspamd_snprintf (path, sizeof (path), "%s%c%s", dir, G_DIR_SEPARATOR, pattern);
+
+ if (!rspamd_glob_dir (path, pattern, recursive, 0, res, err)) {
+ g_ptr_array_free (res, TRUE);
+
+ return NULL;
+ }
+
+ return res;
} \ No newline at end of file
diff --git a/src/libutil/util.h b/src/libutil/util.h
index 6470b5c45..96e85af30 100644
--- a/src/libutil/util.h
+++ b/src/libutil/util.h
@@ -498,4 +498,17 @@ void rspamd_localtime (gint64 ts, struct tm *dest);
*/
gboolean rspamd_fstring_gzip (rspamd_fstring_t **in);
+/**
+ * Perform globbing searching for the specified path. Allow recursion,
+ * returns an error if maximum nesting is reached.
+ * @param pattern
+ * @param recursive
+ * @param err
+ * @return GPtrArray of gchar *, elements are freed when array is freed
+ */
+GPtrArray *rspamd_glob_path (const gchar *dir,
+ const gchar *pattern,
+ gboolean recursive,
+ GError **err);
+
#endif
diff --git a/src/lua/lua_common.c b/src/lua/lua_common.c
index fe715f393..323957086 100644
--- a/src/lua/lua_common.c
+++ b/src/lua/lua_common.c
@@ -375,6 +375,315 @@ rspamd_lua_set_path (lua_State *L, const ucl_object_t *cfg_obj, GHashTable *vars
lua_pop (L, 1);
}
+static gint
+rspamd_lua_cmp_version_components (const gchar *comp1, const gchar *comp2)
+{
+ guint v1, v2;
+
+ v1 = strtoul (comp1, NULL, 10);
+ v2 = strtoul (comp2, NULL, 10);
+
+ return v1 - v2;
+}
+
+static int
+rspamd_lua_rspamd_version_cmp (lua_State *L)
+{
+ const gchar *ver;
+ gchar **components;
+ gint ret = 0;
+
+ if (lua_type (L, 2) == LUA_TSTRING) {
+ ver = lua_tostring (L, 2);
+
+ components = g_strsplit_set (ver, ".-_", -1);
+
+ if (!components) {
+ return luaL_error (L, "invalid arguments to 'cmp': %s", ver);
+ }
+
+ if (components[0]) {
+ ret = rspamd_lua_cmp_version_components (components[0],
+ RSPAMD_VERSION_MAJOR);
+ }
+
+ if (ret) {
+ goto set;
+ }
+
+ if (components[1]) {
+ ret = rspamd_lua_cmp_version_components (components[1],
+ RSPAMD_VERSION_MINOR);
+ }
+
+ if (ret) {
+ goto set;
+ }
+
+ if (components[2]) {
+ ret = rspamd_lua_cmp_version_components (components[2],
+ RSPAMD_VERSION_PATCH);
+ }
+
+ /*
+ * XXX: we don't compare git releases assuming that it is meaningless
+ */
+ }
+ else {
+ return luaL_error (L, "invalid arguments to 'cmp'");
+ }
+
+set:
+ g_strfreev (components);
+ lua_pushnumber (L, ret);
+
+ return 1;
+}
+
+static int
+rspamd_lua_rspamd_version_numeric (lua_State *L)
+{
+ static gint64 version_num = RSPAMD_VERSION_NUM;
+ const gchar *type;
+
+ if (lua_gettop (L) >= 2 && lua_type (L, 1) == LUA_TSTRING) {
+ type = lua_tostring (L, 1);
+ if (g_ascii_strcasecmp (type, "short") == 0) {
+ version_num = RSPAMD_VERSION_MAJOR_NUM * 1000 +
+ RSPAMD_VERSION_MINOR_NUM * 100 +
+ RSPAMD_VERSION_PATCH_NUM * 10;
+ }
+ else if (g_ascii_strcasecmp (type, "main") == 0) {
+ version_num = RSPAMD_VERSION_MAJOR_NUM * 1000 +
+ RSPAMD_VERSION_MINOR_NUM * 100;
+ }
+ else if (g_ascii_strcasecmp (type, "major") == 0) {
+ version_num = RSPAMD_VERSION_MAJOR_NUM;
+ }
+ else if (g_ascii_strcasecmp (type, "minor") == 0) {
+ version_num = RSPAMD_VERSION_MINOR_NUM;
+ }
+ else if (g_ascii_strcasecmp (type, "patch") == 0) {
+ version_num = RSPAMD_VERSION_PATCH_NUM;
+ }
+ }
+
+ lua_pushnumber (L, version_num);
+
+ return 1;
+}
+
+static int
+rspamd_lua_rspamd_version (lua_State *L)
+{
+ const gchar *result = NULL, *type;
+
+ if (lua_gettop (L) == 0) {
+ result = RVERSION;
+ }
+ else if (lua_gettop (L) >= 1 && lua_type (L, 1) == LUA_TSTRING) {
+ /* We got something like string */
+ type = lua_tostring (L, 1);
+
+ if (g_ascii_strcasecmp (type, "short") == 0) {
+ result = RSPAMD_VERSION_MAJOR
+ "." RSPAMD_VERSION_MINOR
+ "." RSPAMD_VERSION_PATCH;
+ }
+ else if (g_ascii_strcasecmp (type, "main") == 0) {
+ result = RSPAMD_VERSION_MAJOR "." RSPAMD_VERSION_MINOR;
+ }
+ else if (g_ascii_strcasecmp (type, "major") == 0) {
+ result = RSPAMD_VERSION_MAJOR;
+ }
+ else if (g_ascii_strcasecmp (type, "minor") == 0) {
+ result = RSPAMD_VERSION_MINOR;
+ }
+ else if (g_ascii_strcasecmp (type, "patch") == 0) {
+ result = RSPAMD_VERSION_PATCH;
+ }
+ else if (g_ascii_strcasecmp (type, "id") == 0) {
+ result = RID;
+ }
+ else if (g_ascii_strcasecmp (type, "num") == 0) {
+ return rspamd_lua_rspamd_version_numeric (L);
+ }
+ else if (g_ascii_strcasecmp (type, "cmp") == 0) {
+ return rspamd_lua_rspamd_version_cmp (L);
+ }
+ }
+
+ lua_pushstring (L, result);
+
+ return 1;
+}
+
+void
+rspamd_lua_set_globals (struct rspamd_config *cfg, lua_State *L,
+ GHashTable *vars)
+{
+ struct rspamd_config **pcfg;
+ gint orig_top = lua_gettop (L);
+
+ /* First check for global variable 'config' */
+ lua_getglobal (L, "config");
+ if (lua_isnil (L, -1)) {
+ /* Assign global table to set up attributes */
+ lua_newtable (L);
+ lua_setglobal (L, "config");
+ }
+
+ lua_getglobal (L, "metrics");
+ if (lua_isnil (L, -1)) {
+ lua_newtable (L);
+ lua_setglobal (L, "metrics");
+ }
+
+ lua_getglobal (L, "composites");
+ if (lua_isnil (L, -1)) {
+ lua_newtable (L);
+ lua_setglobal (L, "composites");
+ }
+
+ lua_getglobal (L, "rspamd_classifiers");
+ if (lua_isnil (L, -1)) {
+ lua_newtable (L);
+ lua_setglobal (L, "rspamd_classifiers");
+ }
+
+ lua_getglobal (L, "classifiers");
+ if (lua_isnil (L, -1)) {
+ lua_newtable (L);
+ lua_setglobal (L, "classifiers");
+ }
+
+ lua_getglobal (L, "rspamd_version");
+ if (lua_isnil (L, -1)) {
+ lua_pushcfunction (L, rspamd_lua_rspamd_version);
+ lua_setglobal (L, "rspamd_version");
+ }
+
+ if (cfg != NULL) {
+ pcfg = lua_newuserdata (L, sizeof (struct rspamd_config *));
+ rspamd_lua_setclass (L, "rspamd{config}", -1);
+ *pcfg = cfg;
+ lua_setglobal (L, "rspamd_config");
+ }
+
+ lua_settop (L, orig_top);
+
+ /* Set known paths as rspamd_paths global */
+ lua_getglobal (L, "rspamd_paths");
+ if (lua_isnil (L, -1)) {
+ const gchar *confdir = RSPAMD_CONFDIR, *rundir = RSPAMD_RUNDIR,
+ *dbdir = RSPAMD_DBDIR, *logdir = RSPAMD_LOGDIR,
+ *wwwdir = RSPAMD_WWWDIR, *pluginsdir = RSPAMD_PLUGINSDIR,
+ *rulesdir = RSPAMD_RULESDIR, *lualibdir = RSPAMD_LUALIBDIR,
+ *prefix = RSPAMD_PREFIX;
+ const gchar *t;
+
+ /* Try environment */
+ t = getenv ("PLUGINSDIR");
+ if (t) {
+ pluginsdir = t;
+ }
+
+ t = getenv ("RULESDIR");
+ if (t) {
+ rulesdir = t;
+ }
+
+ t = getenv ("DBDIR");
+ if (t) {
+ dbdir = t;
+ }
+
+ t = getenv ("RUNDIR");
+ if (t) {
+ rundir = t;
+ }
+
+ t = getenv ("LUALIBDIR");
+ if (t) {
+ lualibdir = t;
+ }
+
+ t = getenv ("LOGDIR");
+ if (t) {
+ logdir = t;
+ }
+
+ t = getenv ("WWWDIR");
+ if (t) {
+ wwwdir = t;
+ }
+
+ t = getenv ("CONFDIR");
+ if (t) {
+ confdir = t;
+ }
+
+
+ if (vars) {
+ t = g_hash_table_lookup (vars, "PLUGINSDIR");
+ if (t) {
+ pluginsdir = t;
+ }
+
+ t = g_hash_table_lookup (vars, "RULESDIR");
+ if (t) {
+ rulesdir = t;
+ }
+
+ t = g_hash_table_lookup (vars, "LUALIBDIR");
+ if (t) {
+ lualibdir = t;
+ }
+
+ t = g_hash_table_lookup (vars, "RUNDIR");
+ if (t) {
+ rundir = t;
+ }
+
+ t = g_hash_table_lookup (vars, "WWWDIR");
+ if (t) {
+ wwwdir = t;
+ }
+
+ t = g_hash_table_lookup (vars, "CONFDIR");
+ if (t) {
+ confdir = t;
+ }
+
+ t = g_hash_table_lookup (vars, "DBDIR");
+ if (t) {
+ dbdir = t;
+ }
+
+ t = g_hash_table_lookup (vars, "LOGDIR");
+ if (t) {
+ logdir = t;
+ }
+ }
+
+ lua_createtable (L, 0, 9);
+
+ rspamd_lua_table_set (L, RSPAMD_CONFDIR_INDEX, confdir);
+ rspamd_lua_table_set (L, RSPAMD_RUNDIR_INDEX, rundir);
+ rspamd_lua_table_set (L, RSPAMD_DBDIR_INDEX, dbdir);
+ rspamd_lua_table_set (L, RSPAMD_LOGDIR_INDEX, logdir);
+ rspamd_lua_table_set (L, RSPAMD_WWWDIR_INDEX, wwwdir);
+ rspamd_lua_table_set (L, RSPAMD_PLUGINSDIR_INDEX, pluginsdir);
+ rspamd_lua_table_set (L, RSPAMD_RULESDIR_INDEX, rulesdir);
+ rspamd_lua_table_set (L, RSPAMD_LUALIBDIR_INDEX, lualibdir);
+ rspamd_lua_table_set (L, RSPAMD_PREFIX_INDEX, prefix);
+
+ lua_setglobal (L, "rspamd_paths");
+ }
+
+ lua_settop (L, orig_top);
+}
+
lua_State *
rspamd_lua_init ()
{
@@ -587,9 +896,6 @@ rspamd_init_lua_filters (struct rspamd_config *cfg, gboolean force_load)
cur = g_list_next (cur);
}
- /* Assign state */
- cfg->lua_state = L;
-
return TRUE;
}
diff --git a/src/lua/lua_common.h b/src/lua/lua_common.h
index 457f470e9..6c64c76d4 100644
--- a/src/lua/lua_common.h
+++ b/src/lua/lua_common.h
@@ -298,6 +298,10 @@ void rspamd_lua_dumpstack (lua_State *L);
void rspamd_lua_set_path (lua_State *L, const ucl_object_t *cfg_obj,
GHashTable *vars);
+/* Set some lua globals */
+void rspamd_lua_set_globals (struct rspamd_config *cfg, lua_State *L,
+ GHashTable *vars);
+
struct memory_pool_s * rspamd_lua_check_mempool (lua_State * L, gint pos);
struct rspamd_config * lua_check_config (lua_State * L, gint pos);
struct rspamd_async_session* lua_check_session (lua_State * L, gint pos);
@@ -400,5 +404,17 @@ void rspamd_lua_add_ref_dtor (lua_State *L, rspamd_mempool_t *pool,
gboolean rspamd_lua_require_function (lua_State *L, const gchar *modname,
const gchar *funcname);
+/* Paths defs */
+#define RSPAMD_CONFDIR_INDEX "CONFDIR"
+#define RSPAMD_RUNDIR_INDEX "RUNDIR"
+#define RSPAMD_DBDIR_INDEX "DBDIR"
+#define RSPAMD_LOGDIR_INDEX "LOGDIR"
+#define RSPAMD_PLUGINSDIR_INDEX "PLUGINSDIR"
+#define RSPAMD_RULESDIR_INDEX "RULESDIR"
+#define RSPAMD_LUALIBDIR_INDEX "LUALIBDIR"
+#define RSPAMD_WWWDIR_INDEX "WWWDIR"
+#define RSPAMD_PREFIX_INDEX "PREFIX"
+#define RSPAMD_VERSION_INDEX "VERSION"
+
#endif /* WITH_LUA */
#endif /* RSPAMD_LUA_H */
diff --git a/src/lua/lua_config.c b/src/lua/lua_config.c
index 2802ab6a4..37d122402 100644
--- a/src/lua/lua_config.c
+++ b/src/lua/lua_config.c
@@ -546,6 +546,13 @@ LUA_FUNCTION_DEF (config, get_symbols_cksum);
LUA_FUNCTION_DEF (config, get_symbols_counters);
/***
+ * @method rspamd_config:get_symbols_scores()
+ * Returns table of all scores defined in config
+ * @return {table|tables} all symbols indexed by name
+ */
+LUA_FUNCTION_DEF (config, get_symbols_scores);
+
+/***
* @method rspamd_config:get_symbol_callback(name)
* Returns callback function for the specified symbol if it is a lua registered callback
* @return {function} callback function or nil
@@ -680,6 +687,29 @@ LUA_FUNCTION_DEF (config, has_torch);
*/
LUA_FUNCTION_DEF (config, experimental_enabled);
+/***
+ * @method rspamd_config:load_ucl(filename)
+ * Loads config from the UCL file (but does not perform parsing using rcl)
+ * @param {string} filename file to load
+ * @return true or false + error message
+ */
+LUA_FUNCTION_DEF (config, load_ucl);
+
+/***
+ * @method rspamd_config:parse_rcl([skip_sections])
+ * Parses RCL using loaded ucl file
+ * @param {table|string} sections to skip
+ * @return true or false + error message
+ */
+LUA_FUNCTION_DEF (config, parse_rcl);
+
+/***
+ * @method rspamd_config:init_modules()
+ * Initialize lua and internal modules
+ * @return true or false
+ */
+LUA_FUNCTION_DEF (config, init_modules);
+
static const struct luaL_reg configlib_m[] = {
LUA_INTERFACE_DEF (config, get_module_opt),
LUA_INTERFACE_DEF (config, get_mempool),
@@ -726,6 +756,7 @@ static const struct luaL_reg configlib_m[] = {
LUA_INTERFACE_DEF (config, get_symbols_count),
LUA_INTERFACE_DEF (config, get_symbols_cksum),
LUA_INTERFACE_DEF (config, get_symbols_counters),
+ LUA_INTERFACE_DEF (config, get_symbols_scores),
LUA_INTERFACE_DEF (config, get_symbol_callback),
LUA_INTERFACE_DEF (config, set_symbol_callback),
LUA_INTERFACE_DEF (config, get_symbol_stat),
@@ -737,6 +768,9 @@ static const struct luaL_reg configlib_m[] = {
LUA_INTERFACE_DEF (config, get_cpu_flags),
LUA_INTERFACE_DEF (config, has_torch),
LUA_INTERFACE_DEF (config, experimental_enabled),
+ LUA_INTERFACE_DEF (config, load_ucl),
+ LUA_INTERFACE_DEF (config, parse_rcl),
+ LUA_INTERFACE_DEF (config, init_modules),
{"__tostring", rspamd_lua_class_tostring},
{"__newindex", lua_config_newindex},
{NULL, NULL}
@@ -2864,6 +2898,44 @@ lua_config_get_symbols_counters (lua_State *L)
return 1;
}
+static void
+lua_metric_symbol_inserter (gpointer k, gpointer v, gpointer ud)
+{
+ lua_State *L = (lua_State *) ud;
+ const gchar *sym = k;
+ struct rspamd_symbol *s = (struct rspamd_symbol *) v;
+
+ lua_pushstring (L, sym);
+
+ lua_createtable (L, 0, 3); /* TODO: add more if needed */
+ lua_pushstring (L, "score");
+ lua_pushnumber (L, s->score);
+ lua_settable (L, -3);
+ lua_pushstring (L, "description");
+ lua_pushstring (L, s->description);
+ lua_settable (L, -3);
+
+ lua_settable (L, -3);
+}
+
+static gint
+lua_config_get_symbols_scores (lua_State *L)
+{
+ struct rspamd_config *cfg = lua_check_config (L, 1);
+
+ if (cfg != NULL) {
+ lua_createtable (L, 0, g_hash_table_size (cfg->symbols));
+ g_hash_table_foreach (cfg->symbols,
+ lua_metric_symbol_inserter,
+ L);
+ }
+ else {
+ return luaL_error (L, "invalid arguments");
+ }
+
+ return 1;
+}
+
static gint
lua_config_get_symbol_callback (lua_State *L)
@@ -3212,6 +3284,132 @@ lua_config_experimental_enabled (lua_State *L)
return 1;
}
+#define LUA_TABLE_TO_HASH(htb, idx) do { \
+ lua_pushstring (L, (idx)); \
+ lua_gettable (L, -2); \
+ if (lua_isstring (L, -1)) { \
+ g_hash_table_insert ((htb), (idx), g_strdup (lua_tostring (L, -1))); \
+ } \
+ lua_pop (L, 1); \
+} while(0)
+
+static gint
+lua_config_load_ucl (lua_State *L)
+{
+ struct rspamd_config *cfg = lua_check_config (L, 1);
+ const gchar *filename;
+ GHashTable *paths = g_hash_table_new_full (rspamd_str_hash, rspamd_str_equal,
+ NULL, g_free);
+ GError *err = NULL;
+
+ if (cfg) {
+ if (lua_isstring (L, 2)) {
+ filename = lua_tostring (L, 2);
+ }
+ else {
+ filename = RSPAMD_CONFDIR "/rspamd.conf";
+ }
+
+ /* Convert rspamd_paths */
+ lua_getglobal (L, "rspamd_paths");
+
+ if (lua_istable (L, -1)) {
+ LUA_TABLE_TO_HASH(paths, RSPAMD_CONFDIR_INDEX);
+ LUA_TABLE_TO_HASH(paths, RSPAMD_RUNDIR_INDEX);
+ LUA_TABLE_TO_HASH(paths, RSPAMD_DBDIR_INDEX);
+ LUA_TABLE_TO_HASH(paths, RSPAMD_LOGDIR_INDEX);
+ LUA_TABLE_TO_HASH(paths, RSPAMD_WWWDIR_INDEX);
+ LUA_TABLE_TO_HASH(paths, RSPAMD_PLUGINSDIR_INDEX);
+ LUA_TABLE_TO_HASH(paths, RSPAMD_RULESDIR_INDEX);
+ LUA_TABLE_TO_HASH(paths, RSPAMD_LUALIBDIR_INDEX);
+ LUA_TABLE_TO_HASH(paths, RSPAMD_PREFIX_INDEX);
+ }
+
+ lua_pop (L, 1);
+
+ if (!rspamd_config_parse_ucl (cfg, filename, paths, &err)) {
+ lua_pushboolean (L, false);
+ lua_pushfstring (L, "failed to load config: %s", err->message);
+ g_error_free (err);
+ g_hash_table_unref (paths);
+
+ return 2;
+ }
+
+ rspamd_rcl_maybe_apply_lua_transform (cfg);
+ rspamd_config_calculate_cksum (cfg);
+ }
+ else {
+ return luaL_error (L, "invalid arguments");
+ }
+
+ g_hash_table_unref (paths);
+ lua_pushboolean (L, true);
+
+ return 1;
+}
+
+#undef IDX_TO_HASH
+
+static gint
+lua_config_parse_rcl (lua_State *L)
+{
+ struct rspamd_config *cfg = lua_check_config (L, 1);
+ GHashTable *excluded = g_hash_table_new_full (rspamd_str_hash, rspamd_str_equal,
+ g_free, NULL);
+ GError *err = NULL;
+ struct rspamd_rcl_section *top;
+
+ if (cfg) {
+ if (lua_istable (L, 2)) {
+ lua_pushvalue (L, 2);
+
+ for (lua_pushnil (L); lua_next (L, -2); lua_pop (L, 1)) {
+ g_hash_table_insert (excluded, g_strdup (lua_tostring (L, -1)),
+ GINT_TO_POINTER (-1));
+ }
+
+ lua_pop (L, 1);
+ }
+
+ top = rspamd_rcl_config_init (cfg, excluded);
+
+ if (!rspamd_rcl_parse (top, cfg, cfg, cfg->cfg_pool, cfg->rcl_obj, &err)) {
+ lua_pushboolean (L, false);
+ lua_pushfstring (L, "failed to load config: %s", err->message);
+ g_error_free (err);
+ g_hash_table_unref (excluded);
+ rspamd_rcl_section_free (top);
+
+ return 2;
+ }
+ }
+ else {
+ return luaL_error (L, "invalid arguments");
+ }
+
+ g_hash_table_unref (excluded);
+ rspamd_rcl_section_free (top);
+ lua_pushboolean (L, true);
+
+ return 1;
+}
+
+static gint
+lua_config_init_modules (lua_State *L)
+{
+ struct rspamd_config *cfg = lua_check_config (L, 1);
+
+ if (cfg != NULL) {
+ rspamd_lua_post_load_config (cfg);
+ lua_pushboolean (L, rspamd_init_filters (cfg, FALSE));
+ }
+ else {
+ return luaL_error (L, "invalid arguments");
+ }
+
+ return 1;
+}
static gint
lua_monitored_alive (lua_State *L)
diff --git a/src/lua/lua_cryptobox.c b/src/lua/lua_cryptobox.c
index b646b0926..6fa2aa997 100644
--- a/src/lua/lua_cryptobox.c
+++ b/src/lua/lua_cryptobox.c
@@ -27,6 +27,9 @@
*/
#include "lua_common.h"
+#include "libcryptobox/cryptobox.h"
+#include "libcryptobox/keypair.h"
+#include "libcryptobox/keypair_private.h"
#include "unix-std.h"
struct rspamd_lua_cryptobox_hash {
@@ -42,6 +45,10 @@ LUA_FUNCTION_DEF (cryptobox_pubkey, gc);
LUA_FUNCTION_DEF (cryptobox_keypair, load);
LUA_FUNCTION_DEF (cryptobox_keypair, create);
LUA_FUNCTION_DEF (cryptobox_keypair, gc);
+LUA_FUNCTION_DEF (cryptobox_keypair, totable);
+LUA_FUNCTION_DEF (cryptobox_keypair, get_type);
+LUA_FUNCTION_DEF (cryptobox_keypair, get_alg);
+LUA_FUNCTION_DEF (cryptobox_keypair, get_pk);
LUA_FUNCTION_DEF (cryptobox_signature, create);
LUA_FUNCTION_DEF (cryptobox_signature, load);
LUA_FUNCTION_DEF (cryptobox_signature, save);
@@ -60,16 +67,24 @@ LUA_FUNCTION_DEF (cryptobox_hash, base32);
LUA_FUNCTION_DEF (cryptobox_hash, base64);
LUA_FUNCTION_DEF (cryptobox_hash, bin);
LUA_FUNCTION_DEF (cryptobox_hash, gc);
-LUA_FUNCTION_DEF (cryptobox, verify_memory);
-LUA_FUNCTION_DEF (cryptobox, verify_file);
-LUA_FUNCTION_DEF (cryptobox, sign_file);
-LUA_FUNCTION_DEF (cryptobox, sign_memory);
+LUA_FUNCTION_DEF (cryptobox, verify_memory);
+LUA_FUNCTION_DEF (cryptobox, verify_file);
+LUA_FUNCTION_DEF (cryptobox, sign_file);
+LUA_FUNCTION_DEF (cryptobox, sign_memory);
+LUA_FUNCTION_DEF (cryptobox, encrypt_memory);
+LUA_FUNCTION_DEF (cryptobox, encrypt_file);
+LUA_FUNCTION_DEF (cryptobox, decrypt_memory);
+LUA_FUNCTION_DEF (cryptobox, decrypt_file);
static const struct luaL_reg cryptoboxlib_f[] = {
LUA_INTERFACE_DEF (cryptobox, verify_memory),
LUA_INTERFACE_DEF (cryptobox, verify_file),
LUA_INTERFACE_DEF (cryptobox, sign_memory),
LUA_INTERFACE_DEF (cryptobox, sign_file),
+ LUA_INTERFACE_DEF (cryptobox, encrypt_memory),
+ LUA_INTERFACE_DEF (cryptobox, encrypt_file),
+ LUA_INTERFACE_DEF (cryptobox, decrypt_memory),
+ LUA_INTERFACE_DEF (cryptobox, decrypt_file),
{NULL, NULL}
};
@@ -93,6 +108,13 @@ static const struct luaL_reg cryptoboxkeypairlib_f[] = {
static const struct luaL_reg cryptoboxkeypairlib_m[] = {
{"__tostring", rspamd_lua_class_tostring},
+ {"totable", lua_cryptobox_keypair_totable},
+ {"get_type", lua_cryptobox_keypair_get_type},
+ {"get_alg", lua_cryptobox_keypair_get_alg},
+ {"type", lua_cryptobox_keypair_get_type},
+ {"alg", lua_cryptobox_keypair_get_alg},
+ {"pk", lua_cryptobox_keypair_get_pk},
+ {"pubkey", lua_cryptobox_keypair_get_pk},
{"__gc", lua_cryptobox_keypair_gc},
{NULL, NULL}
};
@@ -320,8 +342,8 @@ lua_cryptobox_pubkey_gc (lua_State *L)
}
/***
- * @function rspamd_cryptobox_keypair.load(file)
- * Loads public key from UCL file
+ * @function rspamd_cryptobox_keypair.load(file|table)
+ * Loads public key from UCL file or directly from Lua
* @param {string} file filename to load
* @return {cryptobox_keypair} new keypair
*/
@@ -329,57 +351,6 @@ static gint
lua_cryptobox_keypair_load (lua_State *L)
{
struct rspamd_cryptobox_keypair *kp, **pkp;
- const gchar *filename;
- struct ucl_parser *parser;
- ucl_object_t *obj;
-
- filename = luaL_checkstring (L, 1);
- if (filename != NULL) {
- parser = ucl_parser_new (0);
-
- if (!ucl_parser_add_file (parser, filename)) {
- msg_err ("cannot open keypair from file: %s, %s",
- filename,
- ucl_parser_get_error (parser));
- ucl_parser_free (parser);
- lua_pushnil (L);
- }
- else {
- obj = ucl_parser_get_object (parser);
- kp = rspamd_keypair_from_ucl (obj);
- ucl_parser_free (parser);
-
- if (kp == NULL) {
- msg_err ("cannot open keypair from file: %s",
- filename);
- ucl_object_unref (obj);
- lua_pushnil (L);
- }
- else {
- pkp = lua_newuserdata (L, sizeof (gpointer));
- *pkp = kp;
- rspamd_lua_setclass (L, "rspamd{cryptobox_keypair}", -1);
- ucl_object_unref (obj);
- }
- }
- }
- else {
- return luaL_error (L, "bad input arguments");
- }
-
- return 1;
-}
-
-/***
- * @function rspamd_cryptobox_keypair.create(ucl_data)
- * Loads public key from UCL data
- * @param {string} ucl_data ucl to load
- * @return {cryptobox_keypair} new keypair
- */
-static gint
-lua_cryptobox_keypair_create (lua_State *L)
-{
- struct rspamd_cryptobox_keypair *kp, **pkp;
const gchar *buf;
gsize len;
struct ucl_parser *parser;
@@ -439,6 +410,57 @@ lua_cryptobox_keypair_create (lua_State *L)
return 1;
}
+/***
+ * @function rspamd_cryptobox_keypair.create([type='encryption'[, alg='curve25519']])
+ * Generates new keypair
+ * @param {string} type type of keypair: 'encryption' (default) or 'sign'
+ * @param {string} alg algorithm of keypair: 'curve25519' (default) or 'nist'
+ * @return {cryptobox_keypair} new keypair
+ */
+static gint
+lua_cryptobox_keypair_create (lua_State *L)
+{
+ struct rspamd_cryptobox_keypair *kp, **pkp;
+ enum rspamd_cryptobox_keypair_type type = RSPAMD_KEYPAIR_KEX;
+ enum rspamd_cryptobox_mode alg = RSPAMD_CRYPTOBOX_MODE_25519;
+
+ if (lua_isstring (L, 1)) {
+ const gchar *str = lua_tostring (L, 1);
+
+ if (strcmp (str, "sign") == 0) {
+ type = RSPAMD_KEYPAIR_SIGN;
+ }
+ else if (strcmp (str, "encryption") == 0) {
+ type = RSPAMD_KEYPAIR_KEX;
+ }
+ else {
+ return luaL_error (L, "invalid keypair type: %s", str);
+ }
+ }
+
+ if (lua_isstring (L, 2)) {
+ const gchar *str = lua_tostring (L, 2);
+
+ if (strcmp (str, "nist") == 0 || strcmp (str, "openssl") == 0) {
+ alg = RSPAMD_CRYPTOBOX_MODE_NIST;
+ }
+ else if (strcmp (str, "curve25519") == 0 || strcmp (str, "default") == 0) {
+ alg = RSPAMD_CRYPTOBOX_MODE_25519;
+ }
+ else {
+ return luaL_error (L, "invalid keypair algorithm: %s", str);
+ }
+ }
+
+ kp = rspamd_keypair_new (type, alg);
+
+ pkp = lua_newuserdata (L, sizeof (gpointer));
+ *pkp = kp;
+ rspamd_lua_setclass (L, "rspamd{cryptobox_keypair}", -1);
+
+ return 1;
+}
+
static gint
lua_cryptobox_keypair_gc (lua_State *L)
{
@@ -452,7 +474,118 @@ lua_cryptobox_keypair_gc (lua_State *L)
}
/***
- * @function rspamd_cryptobox_signature.load(file)
+ * @method keypair:totable([hex=false]])
+ * Converts keypair to table (not very safe due to memory leftovers)
+ */
+static gint
+lua_cryptobox_keypair_totable (lua_State *L)
+{
+ struct rspamd_cryptobox_keypair *kp = lua_check_cryptobox_keypair (L, 1);
+ ucl_object_t *obj;
+ gboolean hex = FALSE;
+ gint ret = 1;
+
+ if (kp != NULL) {
+
+ if (lua_isboolean (L, 2)) {
+ hex = lua_toboolean (L, 2);
+ }
+
+ obj = rspamd_keypair_to_ucl (kp, hex);
+
+ ret = ucl_object_push_lua (L, obj, true);
+ ucl_object_unref (obj);
+ }
+ else {
+ return luaL_error (L, "invalid arguments");
+ }
+
+ return ret;
+}
+/***
+ * @method keypair:type()
+ * Returns type of keypair as a string: 'encryption' or 'sign'
+ * @return {string} type of keypair as a string
+ */
+static gint
+lua_cryptobox_keypair_get_type (lua_State *L)
+{
+ struct rspamd_cryptobox_keypair *kp = lua_check_cryptobox_keypair (L, 1);
+
+ if (kp) {
+ if (kp->type == RSPAMD_KEYPAIR_KEX) {
+ lua_pushstring (L, "encryption");
+ }
+ else {
+ lua_pushstring (L, "sign");
+ }
+ }
+ else {
+ return luaL_error (L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method keypair:alg()
+ * Returns algorithm of keypair as a string: 'encryption' or 'sign'
+ * @return {string} type of keypair as a string
+ */
+static gint
+lua_cryptobox_keypair_get_alg (lua_State *L)
+{
+ struct rspamd_cryptobox_keypair *kp = lua_check_cryptobox_keypair (L, 1);
+
+ if (kp) {
+ if (kp->alg == RSPAMD_CRYPTOBOX_MODE_25519) {
+ lua_pushstring (L, "curve25519");
+ }
+ else {
+ lua_pushstring (L, "nist");
+ }
+ }
+ else {
+ return luaL_error (L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method keypair:pk()
+ * Returns pubkey for a specific keypair
+ * @return {rspamd_pubkey} pubkey for a keypair
+ */
+static gint
+lua_cryptobox_keypair_get_pk (lua_State *L)
+{
+ struct rspamd_cryptobox_keypair *kp = lua_check_cryptobox_keypair (L, 1);
+ struct rspamd_cryptobox_pubkey *pk, **ppk;
+ const guchar *data;
+ guint dlen;
+
+ if (kp) {
+ data = rspamd_keypair_component (kp, RSPAMD_KEYPAIR_COMPONENT_PK, &dlen);
+ pk = rspamd_pubkey_from_bin (data, dlen, kp->type, kp->alg);
+
+ if (pk == NULL) {
+ return luaL_error (L, "invalid keypair");
+ }
+
+ ppk = lua_newuserdata (L, sizeof (*ppk));
+ *ppk = pk;
+ rspamd_lua_setclass (L, "rspamd{cryptobox_pubkey}", -1);
+ }
+ else {
+ return luaL_error (L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_cryptobox_signature.load(file, [alg = 'curve25519'])
* Loads signature from raw file
* @param {string} file filename to load
* @return {cryptobox_signature} new signature
@@ -465,6 +598,7 @@ lua_cryptobox_signature_load (lua_State *L)
gpointer data;
int fd;
struct stat st;
+ enum rspamd_cryptobox_mode alg = RSPAMD_CRYPTOBOX_MODE_25519;
filename = luaL_checkstring (L, 1);
if (filename != NULL) {
@@ -475,7 +609,6 @@ lua_cryptobox_signature_load (lua_State *L)
lua_pushnil (L);
}
else {
- sig = g_malloc (sizeof (rspamd_fstring_t));
if (fstat (fd, &st) == -1 ||
(data =
mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0))
@@ -484,8 +617,20 @@ lua_cryptobox_signature_load (lua_State *L)
lua_pushnil (L);
}
else {
- if (st.st_size == rspamd_cryptobox_signature_bytes (
- RSPAMD_CRYPTOBOX_MODE_25519)) {
+ if (lua_isstring (L, 2)) {
+ const gchar *str = lua_tostring (L, 2);
+
+ if (strcmp (str, "nist") == 0 || strcmp (str, "openssl") == 0) {
+ alg = RSPAMD_CRYPTOBOX_MODE_NIST;
+ }
+ else if (strcmp (str, "curve25519") == 0 || strcmp (str, "default") == 0) {
+ alg = RSPAMD_CRYPTOBOX_MODE_25519;
+ }
+ else {
+ return luaL_error (L, "invalid keypair algorithm: %s", str);
+ }
+ }
+ if (st.st_size > 0) {
sig = rspamd_fstring_new_init (data, st.st_size);
psig = lua_newuserdata (L, sizeof (rspamd_fstring_t *));
rspamd_lua_setclass (L, "rspamd{cryptobox_signature}", -1);
@@ -494,7 +639,7 @@ lua_cryptobox_signature_load (lua_State *L)
else {
msg_err ("size of %s mismatches: %d while %d is expected",
filename, (int)st.st_size,
- rspamd_cryptobox_signature_bytes (RSPAMD_CRYPTOBOX_MODE_25519));
+ rspamd_cryptobox_signature_bytes (alg));
lua_pushnil (L);
}
@@ -1156,7 +1301,7 @@ lua_cryptobox_hash_gc (lua_State *L)
}
/***
- * @function rspamd_cryptobox.verify_memory(pk, sig, data)
+ * @function rspamd_cryptobox.verify_memory(pk, sig, data, [alg = 'curve25519'])
* Check memory using specified cryptobox key and signature
* @param {pubkey} pk public key to verify
* @param {sig} signature to check
@@ -1170,6 +1315,7 @@ lua_cryptobox_verify_memory (lua_State *L)
rspamd_fstring_t *signature;
struct rspamd_lua_text *t;
const gchar *data;
+ enum rspamd_cryptobox_mode alg = RSPAMD_CRYPTOBOX_MODE_25519;
gsize len;
gint ret;
@@ -1190,9 +1336,23 @@ lua_cryptobox_verify_memory (lua_State *L)
data = luaL_checklstring (L, 3, &len);
}
+ if (lua_isstring (L, 4)) {
+ const gchar *str = lua_tostring (L, 4);
+
+ if (strcmp (str, "nist") == 0 || strcmp (str, "openssl") == 0) {
+ alg = RSPAMD_CRYPTOBOX_MODE_NIST;
+ }
+ else if (strcmp (str, "curve25519") == 0 || strcmp (str, "default") == 0) {
+ alg = RSPAMD_CRYPTOBOX_MODE_25519;
+ }
+ else {
+ return luaL_error (L, "invalid algorithm: %s", str);
+ }
+ }
+
if (pk != NULL && signature != NULL && data != NULL) {
- ret = rspamd_cryptobox_verify (signature->str, data, len,
- rspamd_pubkey_get_pk (pk, NULL), RSPAMD_CRYPTOBOX_MODE_25519);
+ ret = rspamd_cryptobox_verify (signature->str, signature->len, data, len,
+ rspamd_pubkey_get_pk (pk, NULL), alg);
if (ret) {
lua_pushboolean (L, 1);
@@ -1209,7 +1369,7 @@ lua_cryptobox_verify_memory (lua_State *L)
}
/***
- * @function rspamd_cryptobox.verify_file(pk, sig, file)
+ * @function rspamd_cryptobox.verify_file(pk, sig, file, [alg = 'curve25519'])
* Check file using specified cryptobox key and signature
* @param {pubkey} pk public key to verify
* @param {sig} signature to check
@@ -1223,6 +1383,7 @@ lua_cryptobox_verify_file (lua_State *L)
struct rspamd_cryptobox_pubkey *pk;
rspamd_fstring_t *signature;
guchar *map = NULL;
+ enum rspamd_cryptobox_mode alg = RSPAMD_CRYPTOBOX_MODE_25519;
gsize len;
gint ret;
@@ -1230,11 +1391,26 @@ lua_cryptobox_verify_file (lua_State *L)
signature = lua_check_cryptobox_sign (L, 2);
fname = luaL_checkstring (L, 3);
+ if (lua_isstring (L, 4)) {
+ const gchar *str = lua_tostring (L, 4);
+
+ if (strcmp (str, "nist") == 0 || strcmp (str, "openssl") == 0) {
+ alg = RSPAMD_CRYPTOBOX_MODE_NIST;
+ }
+ else if (strcmp (str, "curve25519") == 0 || strcmp (str, "default") == 0) {
+ alg = RSPAMD_CRYPTOBOX_MODE_25519;
+ }
+ else {
+ return luaL_error (L, "invalid algorithm: %s", str);
+ }
+ }
+
map = rspamd_file_xmap (fname, PROT_READ, &len, TRUE);
if (map != NULL && pk != NULL && signature != NULL) {
- ret = rspamd_cryptobox_verify (signature->str, map, len,
- rspamd_pubkey_get_pk (pk, NULL), RSPAMD_CRYPTOBOX_MODE_25519);
+ ret = rspamd_cryptobox_verify (signature->str, signature->len,
+ map, len,
+ rspamd_pubkey_get_pk (pk, NULL), alg);
if (ret) {
lua_pushboolean (L, 1);
@@ -1291,7 +1467,7 @@ lua_cryptobox_sign_memory (lua_State *L)
}
- if (!kp || !data) {
+ if (!kp || !data || kp->type == RSPAMD_KEYPAIR_KEX) {
return luaL_error (L, "invalid arguments");
}
@@ -1353,6 +1529,207 @@ lua_cryptobox_sign_file (lua_State *L)
return 1;
}
+/***
+ * @function rspamd_cryptobox.encrypt_memory(kp, data)
+ * Encrypt data using specified keypair
+ * @param {keypair} kp keypair to use
+ * @param {string|text} data
+ * @return {rspamd_text} encrypted text
+ */
+static gint
+lua_cryptobox_encrypt_memory (lua_State *L)
+{
+ struct rspamd_cryptobox_keypair *kp;
+ const gchar *data;
+ guchar *out;
+ struct rspamd_lua_text *t, *res;
+ gsize len = 0, outlen;
+ GError *err = NULL;
+
+ kp = lua_check_cryptobox_keypair (L, 1);
+
+ if (lua_isuserdata (L, 2)) {
+ t = lua_check_text (L, 2);
+
+ if (!t) {
+ return luaL_error (L, "invalid arguments");
+ }
+
+ data = t->start;
+ len = t->len;
+ }
+ else {
+ data = luaL_checklstring (L, 2, &len);
+ }
+
+
+ if (!kp || !data) {
+ return luaL_error (L, "invalid arguments");
+ }
+
+ if (!rspamd_keypair_encrypt (kp, data, len, &out, &outlen, &err)) {
+ gint ret = luaL_error (L, "cannot encrypt data: %s", err->message);
+ g_error_free (err);
+
+ return ret;
+ }
+
+ res = lua_newuserdata (L, sizeof (*res));
+ res->flags = RSPAMD_TEXT_FLAG_OWN;
+ res->start = out;
+ res->len = outlen;
+ rspamd_lua_setclass (L, "rspamd{text}", -1);
+
+ return 1;
+}
+
+/***
+ * @function rspamd_cryptobox.encrypt_file(kp, filename)
+ * Encrypt data using specified keypair
+ * @param {keypair} kp keypair to use
+ * @param {string} filename
+ * @return {rspamd_text} encrypted text
+ */
+static gint
+lua_cryptobox_encrypt_file (lua_State *L)
+{
+ struct rspamd_cryptobox_keypair *kp;
+ const gchar *filename;
+ gchar *data;
+ guchar *out;
+ struct rspamd_lua_text *res;
+ gsize len = 0, outlen;
+ GError *err = NULL;
+
+ kp = lua_check_cryptobox_keypair (L, 1);
+ filename = luaL_checkstring (L, 2);
+ data = rspamd_file_xmap (filename, PROT_READ, &len, TRUE);
+
+
+ if (!kp || !data) {
+ return luaL_error (L, "invalid arguments");
+ }
+
+ if (!rspamd_keypair_encrypt (kp, data, len, &out, &outlen, &err)) {
+ gint ret = luaL_error (L, "cannot encrypt file %s: %s", filename,
+ err->message);
+ g_error_free (err);
+ munmap (data, len);
+
+ return ret;
+ }
+
+ res = lua_newuserdata (L, sizeof (*res));
+ res->flags = RSPAMD_TEXT_FLAG_OWN;
+ res->start = out;
+ res->len = outlen;
+ rspamd_lua_setclass (L, "rspamd{text}", -1);
+ munmap (data, len);
+
+ return 1;
+}
+
+/***
+ * @function rspamd_cryptobox.decrypt_memory(kp, data)
+ * Encrypt data using specified keypair
+ * @param {keypair} kp keypair to use
+ * @param {string} data
+ * @return status,{rspamd_text}|error status is boolean variable followed by either unencrypted data or an error message
+ */
+static gint
+lua_cryptobox_decrypt_memory (lua_State *L)
+{
+ struct rspamd_cryptobox_keypair *kp;
+ const gchar *data;
+ guchar *out;
+ struct rspamd_lua_text *t, *res;
+ gsize len = 0, outlen;
+ GError *err = NULL;
+
+ kp = lua_check_cryptobox_keypair (L, 1);
+
+ if (lua_isuserdata (L, 2)) {
+ t = lua_check_text (L, 2);
+
+ if (!t) {
+ return luaL_error (L, "invalid arguments");
+ }
+
+ data = t->start;
+ len = t->len;
+ }
+ else {
+ data = luaL_checklstring (L, 2, &len);
+ }
+
+
+ if (!kp || !data) {
+ return luaL_error (L, "invalid arguments");
+ }
+
+ if (!rspamd_keypair_decrypt (kp, data, len, &out, &outlen, &err)) {
+ lua_pushboolean (L, false);
+ lua_pushstring (L, err->message);
+ g_error_free (err);
+ }
+ else {
+ lua_pushboolean (L, true);
+ res = lua_newuserdata (L, sizeof (*res));
+ res->flags = RSPAMD_TEXT_FLAG_OWN;
+ res->start = out;
+ res->len = outlen;
+ rspamd_lua_setclass (L, "rspamd{text}", -1);
+ }
+
+ return 2;
+}
+
+/***
+ * @function rspamd_cryptobox.decrypt_file(kp, filename)
+ * Encrypt data using specified keypair
+ * @param {keypair} kp keypair to use
+ * @param {string} filename
+ * @return status,{rspamd_text}|error status is boolean variable followed by either unencrypted data or an error message
+ */
+static gint
+lua_cryptobox_decrypt_file (lua_State *L)
+{
+ struct rspamd_cryptobox_keypair *kp;
+ const gchar *filename;
+ gchar *data;
+ guchar *out;
+ struct rspamd_lua_text *res;
+ gsize len = 0, outlen;
+ GError *err = NULL;
+
+ kp = lua_check_cryptobox_keypair (L, 1);
+ filename = luaL_checkstring (L, 2);
+ data = rspamd_file_xmap (filename, PROT_READ, &len, TRUE);
+
+
+ if (!kp || !data) {
+ return luaL_error (L, "invalid arguments");
+ }
+
+ if (!rspamd_keypair_decrypt (kp, data, len, &out, &outlen, &err)) {
+ lua_pushboolean (L, false);
+ lua_pushstring (L, err->message);
+ g_error_free (err);
+ }
+ else {
+ lua_pushboolean (L, true);
+ res = lua_newuserdata (L, sizeof (*res));
+ res->flags = RSPAMD_TEXT_FLAG_OWN;
+ res->start = out;
+ res->len = outlen;
+ rspamd_lua_setclass (L, "rspamd{text}", -1);
+ }
+
+ munmap (data, len);
+
+ return 2;
+}
+
static gint
lua_load_pubkey (lua_State * L)
{
diff --git a/src/lua/lua_mimepart.c b/src/lua/lua_mimepart.c
index 9c1d826d7..080bf4662 100644
--- a/src/lua/lua_mimepart.c
+++ b/src/lua/lua_mimepart.c
@@ -264,11 +264,17 @@ end
LUA_FUNCTION_DEF (mimepart, get_header_full);
/***
* @method mime_part:get_content()
- * Get the raw content of part
+ * Get the parsed content of part
* @return {text} opaque text object (zero-copy if not casted to lua string)
*/
LUA_FUNCTION_DEF (mimepart, get_content);
/***
+ * @method mime_part:get_raw_content()
+ * Get the raw content of part
+ * @return {text} opaque text object (zero-copy if not casted to lua string)
+ */
+LUA_FUNCTION_DEF (mimepart, get_raw_content);
+/***
* @method mime_part:get_length()
* Get length of the content of the part
* @return {integer} length of part in **bytes**
@@ -395,6 +401,7 @@ LUA_FUNCTION_DEF (mimepart, headers_foreach);
static const struct luaL_reg mimepartlib_m[] = {
LUA_INTERFACE_DEF (mimepart, get_content),
+ LUA_INTERFACE_DEF (mimepart, get_raw_content),
LUA_INTERFACE_DEF (mimepart, get_length),
LUA_INTERFACE_DEF (mimepart, get_type),
LUA_INTERFACE_DEF (mimepart, get_type_full),
@@ -918,6 +925,26 @@ lua_mimepart_get_content (lua_State * L)
}
static gint
+lua_mimepart_get_raw_content (lua_State * L)
+{
+ struct rspamd_mime_part *part = lua_check_mimepart (L);
+ struct rspamd_lua_text *t;
+
+ if (part == NULL) {
+ lua_pushnil (L);
+ return 1;
+ }
+
+ t = lua_newuserdata (L, sizeof (*t));
+ rspamd_lua_setclass (L, "rspamd{text}", -1);
+ t->start = part->raw_data.begin;
+ t->len = part->raw_data.len;
+ t->flags = 0;
+
+ return 1;
+}
+
+static gint
lua_mimepart_get_length (lua_State * L)
{
struct rspamd_mime_part *part = lua_check_mimepart (L);
diff --git a/src/lua/lua_task.c b/src/lua/lua_task.c
index 3aa24e88b..1807888c8 100644
--- a/src/lua/lua_task.c
+++ b/src/lua/lua_task.c
@@ -1096,8 +1096,9 @@ lua_task_process_message (lua_State *L)
if (task != NULL) {
if (task->msg.len > 0) {
- if (rspamd_message_parse (task) == 0) {
+ if (rspamd_message_parse (task)) {
lua_pushboolean (L, TRUE);
+ rspamd_message_process (task);
}
else {
lua_pushboolean (L, FALSE);
@@ -3606,6 +3607,7 @@ lua_task_set_flag (lua_State *L)
LUA_TASK_SET_FLAG (flag, "broken_headers",
RSPAMD_TASK_FLAG_BROKEN_HEADERS, set);
LUA_TASK_SET_FLAG (flag, "greylisted", RSPAMD_TASK_FLAG_GREYLISTED, set);
+ LUA_TASK_SET_FLAG (flag, "skip_process", RSPAMD_TASK_FLAG_SKIP_PROCESS, set);
if (!found) {
msg_warn_task ("unknown flag requested: %s", flag);
@@ -3636,6 +3638,8 @@ lua_task_has_flag (lua_State *L)
LUA_TASK_GET_FLAG (flag, "greylisted", RSPAMD_TASK_FLAG_GREYLISTED);
LUA_TASK_GET_FLAG (flag, "broken_headers",
RSPAMD_TASK_FLAG_BROKEN_HEADERS);
+ LUA_TASK_GET_FLAG (flag, "skip_process",
+ RSPAMD_TASK_FLAG_SKIP_PROCESS);
LUA_TASK_GET_FLAG (flag, "milter",
RSPAMD_TASK_FLAG_MILTER);
@@ -3704,6 +3708,10 @@ lua_task_get_flags (lua_State *L)
lua_pushstring (L, "greylisted");
lua_rawseti (L, -2, idx++);
break;
+ case RSPAMD_TASK_FLAG_SKIP_PROCESS:
+ lua_pushstring (L, "skip_process");
+ lua_rawseti (L, -2, idx++);
+ break;
case RSPAMD_TASK_FLAG_MILTER:
lua_pushstring (L, "milter");
lua_rawseti (L, -2, idx++);
diff --git a/src/lua/lua_util.c b/src/lua/lua_util.c
index 6c200d1be..f99ae7d1f 100644
--- a/src/lua/lua_util.c
+++ b/src/lua/lua_util.c
@@ -22,6 +22,7 @@
#include "linenoise.h"
#include <math.h>
#include <glob.h>
+#include <zlib.h>
#include "unicode/uspoof.h"
@@ -356,6 +357,24 @@ LUA_FUNCTION_DEF (util, zstd_compress);
LUA_FUNCTION_DEF (util, zstd_decompress);
/***
+ * @function util.gzip_decompress(data)
+ * Decompresses input using gzip algorithm
+ *
+ * @param {string/rspamd_text} data compressed data
+ * @return {error,rspamd_text} pair of error + decompressed text
+ */
+LUA_FUNCTION_DEF (util, gzip_decompress);
+
+/***
+ * @function util.gzip_compress(data)
+ * Compresses input using gzip compression
+ *
+ * @param {string/rspamd_text} data input data
+ * @return {rspamd_text} compressed data
+ */
+LUA_FUNCTION_DEF (util, gzip_compress);
+
+/***
* @function util.normalize_prob(prob, [bias = 0.5])
* Normalize probabilities using polynom
*
@@ -558,6 +577,8 @@ static const struct luaL_reg utillib_f[] = {
LUA_INTERFACE_DEF (util, random_hex),
LUA_INTERFACE_DEF (util, zstd_compress),
LUA_INTERFACE_DEF (util, zstd_decompress),
+ LUA_INTERFACE_DEF (util, gzip_compress),
+ LUA_INTERFACE_DEF (util, gzip_decompress),
LUA_INTERFACE_DEF (util, normalize_prob),
LUA_INTERFACE_DEF (util, caseless_hash),
LUA_INTERFACE_DEF (util, caseless_hash_fast),
@@ -626,9 +647,10 @@ lua_util_load_rspamd_config (lua_State *L)
cfg_name = luaL_checkstring (L, 1);
if (cfg_name) {
- cfg = rspamd_config_new ();
+ cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_SKIP_LUA);
+ cfg->lua_state = L;
- if (rspamd_config_read (cfg, cfg_name, NULL, NULL, NULL, NULL)) {
+ if (rspamd_config_read (cfg, cfg_name, NULL, NULL, NULL)) {
msg_err_config ("cannot load config from %s", cfg_name);
lua_pushnil (L);
}
@@ -655,11 +677,12 @@ lua_util_config_from_ucl (lua_State *L)
if (obj) {
cfg = g_malloc0 (sizeof (struct rspamd_config));
- cfg = rspamd_config_new ();
+ cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_SKIP_LUA);
+ cfg->lua_state = L;
cfg->rcl_obj = obj;
cfg->cache = rspamd_symbols_cache_new (cfg);
- top = rspamd_rcl_config_init (cfg);
+ top = rspamd_rcl_config_init (cfg, NULL);
if (!rspamd_rcl_parse (top, cfg, cfg, cfg->cfg_pool, cfg->rcl_obj, &err)) {
msg_err_config ("rcl parse error: %s", err->message);
@@ -1944,6 +1967,172 @@ lua_util_zstd_decompress (lua_State *L)
}
static gint
+lua_util_gzip_compress (lua_State *L)
+{
+ struct rspamd_lua_text *t = NULL, *res, tmp;
+ gsize sz;
+ z_stream strm;
+ gint rc;
+ guchar *p;
+ gsize remain;
+
+ if (lua_type (L, 1) == LUA_TSTRING) {
+ t = &tmp;
+ t->start = lua_tolstring (L, 1, &sz);
+ t->len = sz;
+ }
+ else {
+ t = lua_check_text (L, 1);
+ }
+
+ if (t == NULL || t->start == NULL) {
+ return luaL_error (L, "invalid arguments");
+ }
+
+
+
+ memset (&strm, 0, sizeof (strm));
+ rc = deflateInit2 (&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED,
+ MAX_WBITS + 16, MAX_MEM_LEVEL - 1, Z_DEFAULT_STRATEGY);
+
+ if (rc != Z_OK) {
+ return luaL_error (L, "cannot init zlib: %s", zError (rc));
+ }
+
+ sz = deflateBound (&strm, t->len);
+
+ strm.avail_in = t->len;
+ strm.next_in = (guchar *)t->start;
+
+ res = lua_newuserdata (L, sizeof (*res));
+ res->start = g_malloc (sz);
+ res->flags = RSPAMD_TEXT_FLAG_OWN;
+ rspamd_lua_setclass (L, "rspamd{text}", -1);
+
+ p = (guchar *)res->start;
+ remain = sz;
+
+ while (strm.avail_in != 0) {
+ strm.avail_out = remain;
+ strm.next_out = p;
+
+ rc = deflate (&strm, Z_FINISH);
+
+ if (rc != Z_OK && rc != Z_BUF_ERROR) {
+ if (rc == Z_STREAM_END) {
+ break;
+ }
+ else {
+ msg_err ("cannot compress data: %s", zError (rc));
+ lua_pop (L, 1); /* Text will be freed here */
+ lua_pushnil (L);
+ deflateEnd (&strm);
+
+ return 1;
+ }
+ }
+
+ res->len = strm.total_out;
+
+ if (strm.avail_out == 0 && strm.avail_in != 0) {
+ /* Need to allocate more */
+ remain = res->len;
+ res->start = g_realloc ((gpointer)res->start, strm.avail_in + sz);
+ sz = strm.avail_in + sz;
+ p = (guchar *)res->start + remain;
+ remain = sz - remain;
+ }
+ }
+
+ deflateEnd (&strm);
+ res->len = strm.total_out;
+
+ return 1;
+}
+
+
+static gint
+lua_util_gzip_decompress (lua_State *L)
+{
+ struct rspamd_lua_text *t = NULL, *res, tmp;
+ gsize sz;
+ z_stream strm;
+ gint rc;
+ guchar *p;
+ gsize remain;
+
+ if (lua_type (L, 1) == LUA_TSTRING) {
+ t = &tmp;
+ t->start = lua_tolstring (L, 1, &sz);
+ t->len = sz;
+ }
+ else {
+ t = lua_check_text (L, 1);
+ }
+
+ if (t == NULL || t->start == NULL) {
+ return luaL_error (L, "invalid arguments");
+ }
+
+ memset (&strm, 0, sizeof (strm));
+ /* windowBits +16 to decode gzip, zlib 1.2.0.4+ */
+ rc = inflateInit2 (&strm, MAX_WBITS + 16);
+
+ if (rc != Z_OK) {
+ return luaL_error (L, "cannot init zlib");
+ }
+
+ strm.avail_in = t->len;
+ strm.next_in = (guchar *)t->start;
+
+ res = lua_newuserdata (L, sizeof (*res));
+ res->start = g_malloc (sz);
+ res->flags = RSPAMD_TEXT_FLAG_OWN;
+ rspamd_lua_setclass (L, "rspamd{text}", -1);
+
+ p = (guchar *)res->start;
+ remain = sz;
+
+ while (strm.avail_in != 0) {
+ strm.avail_out = remain;
+ strm.next_out = p;
+
+ rc = inflate (&strm, Z_FINISH);
+
+ if (rc != Z_OK && rc != Z_BUF_ERROR) {
+ if (rc == Z_STREAM_END) {
+ break;
+ }
+ else {
+ msg_err ("cannot decompress data: %s", zError (rc));
+ lua_pop (L, 1); /* Text will be freed here */
+ lua_pushnil (L);
+ inflateEnd (&strm);
+
+ return 1;
+ }
+ }
+
+ res->len = strm.total_out;
+
+ if (strm.avail_out == 0 && strm.avail_in != 0) {
+ /* Need to allocate more */
+ remain = res->len;
+ res->start = g_realloc ((gpointer)res->start, strm.avail_in + sz);
+ sz = strm.avail_in + sz;
+ p = (guchar *)res->start + remain;
+ remain = sz - remain;
+ }
+ }
+
+ inflateEnd (&strm);
+ res->len = strm.total_out;
+
+ return 2;
+}
+
+
+static gint
lua_util_normalize_prob (lua_State *L)
{
gdouble x, bias = 0.5;
diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c
index 0873d3c7a..c4318777f 100644
--- a/src/plugins/fuzzy_check.c
+++ b/src/plugins/fuzzy_check.c
@@ -3019,6 +3019,8 @@ fuzzy_process_handler (struct rspamd_http_connection_entry *conn_ent,
"Message processing error");
return;
}
+
+ rspamd_message_process (task);
}
PTR_ARRAY_FOREACH (fuzzy_module_ctx->fuzzy_rules, i, rule) {
diff --git a/src/plugins/lua/dmarc.lua b/src/plugins/lua/dmarc.lua
index 58e48bd01..5c24bff53 100644
--- a/src/plugins/lua/dmarc.lua
+++ b/src/plugins/lua/dmarc.lua
@@ -66,10 +66,10 @@ Content-Transfer-Encoding: 7bit
This is an aggregate report from %s.
------=_NextPart_000_024E_01CC9B0A.AFE54C00
-Content-Type: text/xml
+Content-Type: application/gzip
Content-Transfer-Encoding: base64
Content-Disposition: attachment;
- filename="%s!%s!%s!%s.xml"
+ filename="%s!%s!%s!%s.xml.gz"
]]
local report_footer = [[
@@ -744,10 +744,11 @@ if opts['reporting'] == true then
for k in pairs(reporting_addr) do
table.insert(tmp_addr, k)
end
- local encoded = rspamd_util.encode_base64(table.concat(
+ local encoded = rspamd_util.encode_base64(rspamd_util.gzip_compress(
+ table.concat(
{xmlf('header'),
xmlf('entries'),
- xmlf('footer')}), 78)
+ xmlf('footer')})), 78)
local function mail_cb(err, data, conn)
local function no_error(merr, mdata, wantcode)
wantcode = wantcode or '2'
@@ -808,11 +809,22 @@ if opts['reporting'] == true then
table.insert(atmp, k)
end
local addr_string = table.concat(atmp, ', ')
- local rhead = string.format(report_template, report_settings.email, addr_string,
- reporting_domain, report_settings.domain, report_id, rspamd_util.time_to_string(rspamd_util.get_time()),
- rspamd_util.random_hex(12) .. '@rspamd', report_settings.domain, report_settings.domain, reporting_domain,
- report_start, report_end)
- conn:add_write(pre_quit_cb, {rhead, encoded, report_footer, '\r\n.\r\n'})
+ local rhead = string.format(report_template,
+ report_settings.email,
+ addr_string,
+ reporting_domain,
+ report_settings.domain,
+ report_id,
+ rspamd_util.time_to_string(rspamd_util.get_time()),
+ rspamd_util.random_hex(12) .. '@rspamd',
+ report_settings.domain,
+ report_settings.domain,
+ reporting_domain,
+ report_start, report_end)
+ conn:add_write(pre_quit_cb, {rhead,
+ encoded,
+ report_footer,
+ '\r\n.\r\n'})
end
end
local function data_cb(merr, mdata)
diff --git a/src/plugins/lua/elastic.lua b/src/plugins/lua/elastic.lua
index 4caedd684..3a80256d2 100644
--- a/src/plugins/lua/elastic.lua
+++ b/src/plugins/lua/elastic.lua
@@ -46,6 +46,7 @@ local settings = {
failover = false,
import_kibana = false,
use_https = false,
+ use_gzip = true,
allow_local = false,
}
@@ -106,6 +107,7 @@ local function elastic_send_data(task)
body = bulk_json,
task = task,
method = 'post',
+ gzip = settings.use_gzip,
callback = http_index_data_callback
})
@@ -305,6 +307,7 @@ local function initial_setup(cfg, ev_base, worker)
},
body = table.concat(tbl, "\n"),
method = 'post',
+ gzip = settings.use_gzip,
callback = kibana_template_callback
})
else
@@ -341,6 +344,7 @@ local function initial_setup(cfg, ev_base, worker)
headers = {
['Content-Type'] = 'application/json',
},
+ gzip = settings.use_gzip,
body = ucl.to_format(template, 'json-compact'),
method = 'put',
})
@@ -366,6 +370,7 @@ local function initial_setup(cfg, ev_base, worker)
headers = {
['Content-Type'] = 'application/json',
},
+ gzip = settings.use_gzip,
callback = http_template_put_callback,
})
else
diff --git a/src/plugins/lua/mime_types.lua b/src/plugins/lua/mime_types.lua
index 43142132c..f97f22d2a 100644
--- a/src/plugins/lua/mime_types.lua
+++ b/src/plugins/lua/mime_types.lua
@@ -344,7 +344,7 @@ local full_extensions_map = {
{"group", "text/x-ms-group"},
{"gsm", "audio/x-gsm"},
{"gtar", "application/x-gtar"},
- {"gz", "application/x-gzip"},
+ {"gz", {"application/gzip", "application/x-gzip"}},
{"h", "text/plain"},
{"hdf", "application/x-hdf"},
{"hdml", "text/x-hdml"},
@@ -773,7 +773,7 @@ local full_extensions_map = {
{"xls", {"application/vnd.ms-excel", "application/vnd.ms-office", "application/x-excel", "application/octet-stream"}},
{"xlsb", "application/vnd.ms-excel.sheet.binary.macroEnabled.12"},
{"xlsm", "application/vnd.ms-excel.sheet.macroEnabled.12"},
- {"xlsx", {"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/octet-stream"}},
+ {"xlsx", {"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/vnd.ms-excel.12", "application/octet-stream"}},
{"xlt", "application/vnd.ms-excel"},
{"xltm", "application/vnd.ms-excel.template.macroEnabled.12"},
{"xltx", "application/vnd.openxmlformats-officedocument.spreadsheetml.template"},
@@ -798,6 +798,7 @@ local full_extensions_map = {
{"xwd", "image/x-xwindowdump"},
{"z", "application/x-compress"},
{"zip", {"application/zip", "application/x-zip-compressed", "application/octet-stream"}},
+ {"zlib", "application/zlib"},
}
local function check_mime_type(task)
diff --git a/src/plugins/lua/phishing.lua b/src/plugins/lua/phishing.lua
index e76ffc79a..69aec219f 100644
--- a/src/plugins/lua/phishing.lua
+++ b/src/plugins/lua/phishing.lua
@@ -75,8 +75,12 @@ local function phishing_cb(task)
for _,d in ipairs(elt) do
if not d['path'] then
found_path = true
+ end
+
+ if query and d['query'] and query == d['query'] then
+ found_query = true
+ elseif not d['query'] then
found_query = true
- break
end
end
end
@@ -101,7 +105,10 @@ local function phishing_cb(task)
task:insert_result(phish_symbol, 1.0, args)
else
-- Host + path match
- task:insert_result(phish_symbol, 0.3, args)
+ if path then
+ task:insert_result(phish_symbol, 0.3, args)
+ end
+ -- No path, no symbol
end
else
if url:is_phished() then
diff --git a/src/plugins/lua/ratelimit.lua b/src/plugins/lua/ratelimit.lua
index e060fe084..1d22f8838 100644
--- a/src/plugins/lua/ratelimit.lua
+++ b/src/plugins/lua/ratelimit.lua
@@ -377,8 +377,16 @@ local function ratelimit_cb(task)
if err then
rspamd_logger.errx('cannot check limit %s: %s %s', prefix, err, data)
elseif type(data) == 'table' and data[1] and data[1] == 1 then
- if settings.info_symbol then
- task:insert_result(settings.info_symbol, 1.0, prefix)
+ -- set symbol only and do NOT soft reject
+ if settings.symbol then
+ task:insert_result(settings.symbol, 0.0, lim_name .. "(" .. prefix .. ")")
+ rspamd_logger.infox(task,
+ 'set_symbol_only: ratelimit "%s(%s)" exceeded, (%s / %s): %s (%s:%s dyn)',
+ lim_name, prefix, bucket[2], bucket[1], data[2], data[3], data[4])
+ return
+ -- set INFO symbol and soft reject
+ elseif settings.info_symbol then
+ task:insert_result(settings.info_symbol, 1.0, lim_name .. "(" .. prefix .. ")")
end
rspamd_logger.infox(task,
'ratelimit "%s(%s)" exceeded, (%s / %s): %s (%s:%s dyn)',
diff --git a/src/plugins/lua/settings.lua b/src/plugins/lua/settings.lua
index e1a4cef74..8e4009162 100644
--- a/src/plugins/lua/settings.lua
+++ b/src/plugins/lua/settings.lua
@@ -37,6 +37,26 @@ local rspamd_regexp = require "rspamd_regexp"
local ucl = require "ucl"
local fun = require "fun"
+local function apply_settings(task, to_apply)
+ task:set_settings(to_apply)
+ task:cache_set('settings', to_apply)
+ lua_squeeze.handle_settings(task, to_apply)
+
+ if to_apply['add_headers'] or to_apply['remove_headers'] then
+ local rep = {
+ add_headers = to_apply['add_headers'] or {},
+ remove_headers = to_apply['remove_headers'] or {},
+ }
+ task:set_rmilter_reply(rep)
+ end
+
+ if to_apply.flags and type(to_apply.flags) == 'table' then
+ for _,fl in ipairs(to_apply.flags) do
+ task:set_flag(fl)
+ end
+ end
+end
+
-- Checks for overridden settings within query params and returns 'true' if
-- settings are overridden
local function check_query_settings(task)
@@ -47,9 +67,7 @@ local function check_query_settings(task)
local res,err = parser:parse_string(tostring(query_set))
if res then
local settings_obj = parser:get_object()
- task:set_settings(settings_obj)
- task:cache_set('settings', settings_obj)
- lua_squeeze.handle_settings(task, settings_obj)
+ apply_settings(task, settings_obj)
return true
else
@@ -76,10 +94,7 @@ local function check_query_settings(task)
nset['default']['actions']['add header'] = ss
end
- task:set_settings(nset)
- task:cache_set('settings', nset)
- lua_squeeze.handle_settings(task, nset)
-
+ apply_settings(task, nset)
return true
end
end
@@ -90,19 +105,8 @@ local function check_query_settings(task)
local id_str = tostring(settings_id)
local elt = settings_ids[id_str]
if elt and elt['apply'] then
- task:set_settings(elt['apply'])
- lua_squeeze.handle_settings(task, elt['apply'])
- task:cache_set('settings', elt['apply'])
-
- if elt.apply['add_headers'] or elt.apply['remove_headers'] then
- local rep = {
- add_headers = elt.apply['add_headers'] or {},
- remove_headers = elt.apply['remove_headers'] or {},
- }
- task:set_rmilter_reply(rep)
- end
+ apply_settings(task, elt['apply'])
rspamd_logger.infox(task, "applying settings id %s", id_str)
-
return true
end
end
@@ -340,9 +344,7 @@ local function check_settings(task)
rspamd_logger.infox(task, "<%1> apply settings according to rule %2",
task:get_message_id(), s.name)
if rule['apply'] then
- task:set_settings(rule['apply'])
- lua_squeeze.handle_settings(task, rule['apply'])
- task:cache_set('settings', rule['apply'])
+ apply_settings(task, rule['apply'])
applied = true
end
if rule['symbols'] then
@@ -653,9 +655,7 @@ local function gen_redis_callback(handler, id)
local obj = parser:get_object()
rspamd_logger.infox(task, "<%1> apply settings according to redis rule %2",
task:get_message_id(), id)
- task:set_settings(obj)
- lua_squeeze.handle_settings(task, obj)
- task:cache_set('settings', obj)
+ apply_settings(task, obj)
break
end
end
diff --git a/src/rspamadm/CMakeLists.txt b/src/rspamadm/CMakeLists.txt
index fb3f25229..2105c16b3 100644
--- a/src/rspamadm/CMakeLists.txt
+++ b/src/rspamadm/CMakeLists.txt
@@ -1,21 +1,16 @@
SET(RSPAMADMSRC rspamadm.c
commands.c
pw.c
- keypair.c
configtest.c
fuzzy_convert.c
fuzzy_merge.c
- grep.c
configdump.c
control.c
confighelp.c
- configwizard.c
- corpus_test.c
stat_convert.c
signtool.c
lua_repl.c
dkim_keygen.c
- rescore.c
${CMAKE_BINARY_DIR}/src/workers.c
${CMAKE_BINARY_DIR}/src/modules.c
${CMAKE_SOURCE_DIR}/src/controller.c
diff --git a/src/rspamadm/commands.c b/src/rspamadm/commands.c
index 410306fe3..9216c79b6 100644
--- a/src/rspamadm/commands.c
+++ b/src/rspamadm/commands.c
@@ -14,9 +14,11 @@
* limitations under the License.
*/
#include "rspamadm.h"
+#include "libutil/util.h"
+#include "libutil/logger.h"
+#include "lua/lua_common.h"
extern struct rspamadm_command pw_command;
-extern struct rspamadm_command keypair_command;
extern struct rspamadm_command configtest_command;
extern struct rspamadm_command fuzzy_merge_command;
extern struct rspamadm_command configdump_command;
@@ -24,18 +26,13 @@ extern struct rspamadm_command control_command;
extern struct rspamadm_command confighelp_command;
extern struct rspamadm_command statconvert_command;
extern struct rspamadm_command fuzzyconvert_command;
-extern struct rspamadm_command grep_command;
extern struct rspamadm_command signtool_command;
extern struct rspamadm_command lua_command;
extern struct rspamadm_command dkim_keygen_command;
-extern struct rspamadm_command configwizard_command;
-extern struct rspamadm_command corpus_test_command;
-extern struct rspamadm_command rescore_command;
const struct rspamadm_command *commands[] = {
&help_command,
&pw_command,
- &keypair_command,
&configtest_command,
&fuzzy_merge_command,
&configdump_command,
@@ -43,31 +40,33 @@ const struct rspamadm_command *commands[] = {
&confighelp_command,
&statconvert_command,
&fuzzyconvert_command,
- &grep_command,
&signtool_command,
&lua_command,
&dkim_keygen_command,
- &configwizard_command,
- &corpus_test_command,
- &rescore_command,
NULL
};
const struct rspamadm_command *
-rspamadm_search_command (const gchar *name)
+rspamadm_search_command (const gchar *name, GPtrArray *all_commands)
{
- const struct rspamadm_command *ret = NULL;
- guint i;
+ const struct rspamadm_command *ret = NULL, *cmd;
+ const gchar *alias;
+ guint i, j;
if (name == NULL) {
name = "help";
}
- for (i = 0; i < G_N_ELEMENTS (commands); i ++) {
- if (commands[i] != NULL) {
- if (strcmp (name, commands[i]->name) == 0) {
- ret = commands[i];
+ PTR_ARRAY_FOREACH (all_commands, i, cmd) {
+ if (strcmp (name, cmd->name) == 0) {
+ ret = cmd;
+ break;
+ }
+
+ PTR_ARRAY_FOREACH (cmd->aliases, j, alias) {
+ if (strcmp (name, alias) == 0) {
+ ret = cmd;
break;
}
}
@@ -75,3 +74,220 @@ rspamadm_search_command (const gchar *name)
return ret;
}
+
+void
+rspamadm_fill_internal_commands (GPtrArray *dest)
+{
+ guint i;
+
+ for (i = 0; i < G_N_ELEMENTS (commands); i ++) {
+ if (commands[i]) {
+ g_ptr_array_add (dest, (gpointer)commands[i]);
+ }
+ }
+}
+
+static void
+rspamadm_lua_command_run (gint argc, gchar **argv,
+ const struct rspamadm_command *cmd)
+{
+ gint table_idx = GPOINTER_TO_INT (cmd->command_data);
+ gint i, err_idx, ret;
+ GString *tb;
+
+ lua_pushcfunction (L, &rspamd_lua_traceback);
+ err_idx = lua_gettop (L);
+
+ /* Function */
+ lua_rawgeti (L, LUA_REGISTRYINDEX, table_idx);
+ lua_pushstring (L, "handler");
+ lua_gettable (L, -2);
+
+ /* Args */
+ lua_createtable (L, argc + 1, 0);
+
+ for (i = 0; i < argc; i ++) {
+ lua_pushstring (L, argv[i]);
+ lua_rawseti (L, -2, i); /* Starting from zero ! */
+ }
+
+ if ((ret = lua_pcall (L, 1, 0, err_idx)) != 0) {
+ tb = lua_touserdata (L, -1);
+ msg_err ("call to rspamadm lua script %s failed (%d): %v", cmd->name,
+ ret, tb);
+
+ if (tb) {
+ g_string_free (tb, TRUE);
+ }
+
+ lua_settop (L, 0);
+
+ exit (EXIT_FAILURE);
+ }
+
+ lua_settop (L, 0);
+}
+
+static const gchar *
+rspamadm_lua_command_help (gboolean full_help,
+ const struct rspamadm_command *cmd)
+{
+ gint table_idx = GPOINTER_TO_INT (cmd->command_data);
+ gint err_idx, ret;
+ GString *tb;
+
+ if (full_help) {
+ lua_pushcfunction (L, &rspamd_lua_traceback);
+ err_idx = lua_gettop (L);
+
+ lua_rawgeti (L, LUA_REGISTRYINDEX, table_idx);
+ /* Function */
+ lua_pushstring (L, "handler");
+ lua_gettable (L, -2);
+
+ /* Args */
+ lua_createtable (L, 2, 0);
+ lua_pushstring (L, cmd->name);
+ lua_rawseti (L, -2, 0); /* Starting from zero ! */
+
+ lua_pushstring (L, "--help");
+ lua_rawseti (L, -2, 1);
+
+ if ((ret = lua_pcall (L, 1, 0, err_idx)) != 0) {
+ tb = lua_touserdata (L, -1);
+ msg_err ("call to rspamadm lua script %s failed (%d): %v", cmd->name,
+ ret, tb);
+
+ if (tb) {
+ g_string_free (tb, TRUE);
+ }
+
+ lua_settop (L, 0);
+
+ exit (EXIT_FAILURE);
+ }
+ }
+ else {
+ lua_rawgeti (L, LUA_REGISTRYINDEX, table_idx);
+ lua_pushstring (L, "description");
+ lua_gettable (L, -2);
+
+ if (lua_isstring (L, -1)) {
+ printf (" %-18s %-60s\n", cmd->name, lua_tostring (L, -1));
+ }
+ else {
+ printf (" %-18s %-60s\n", cmd->name, "no description available");
+ }
+ }
+
+ lua_settop (L, 0);
+
+ return NULL; /* Must be handled in rspamadm itself */
+}
+
+void
+rspamadm_fill_lua_commands (lua_State *L, GPtrArray *dest)
+{
+ gint i;
+
+ GPtrArray *lua_paths;
+ GError *err = NULL;
+ const gchar *lualibdir = RSPAMD_LUALIBDIR, *path;
+ struct rspamadm_command *lua_cmd;
+ gchar search_dir[PATH_MAX];
+
+ if (g_hash_table_lookup (ucl_vars, "LUALIBDIR")) {
+ lualibdir = g_hash_table_lookup (ucl_vars, "LUALIBDIR");
+ }
+
+ rspamd_snprintf (search_dir, sizeof (search_dir), "%s%crspamadm%c",
+ lualibdir, G_DIR_SEPARATOR, G_DIR_SEPARATOR);
+
+ if ((lua_paths = rspamd_glob_path (search_dir, "*.lua", FALSE, &err)) == NULL) {
+ msg_err ("cannot glob files in %s/*.lua: %e", search_dir, err);
+ g_error_free (err);
+
+ return;
+ }
+
+ PTR_ARRAY_FOREACH (lua_paths, i, path) {
+ if (luaL_dofile (L, path) != 0) {
+ msg_err ("cannot execute lua script %s: %s",
+ path, lua_tostring (L, -1));
+ lua_settop (L, 0);
+ continue;
+ } else {
+ if (lua_type (L, -1) == LUA_TTABLE) {
+ lua_pushstring (L, "handler");
+ lua_gettable (L, -2);
+ }
+ else {
+ continue; /* Something goes wrong, huh */
+ }
+
+ if (lua_type (L, -1) != LUA_TFUNCTION) {
+ msg_err ("rspamadm script %s does not have 'handler' field with type "
+ "function",
+ path);
+ continue;
+ }
+
+ /* Pop handler */
+ lua_pop (L, 1);
+ lua_cmd = g_malloc0 (sizeof (*lua_cmd));
+
+ lua_pushstring (L, "name");
+ lua_gettable (L, -2);
+
+ if (lua_type (L, -1) == LUA_TSTRING) {
+ lua_cmd->name = g_strdup (lua_tostring (L, -1));
+ }
+ else {
+ goffset ext_pos;
+ gchar *name;
+
+ name = g_path_get_basename (path);
+ /* Remove .lua */
+ ext_pos = rspamd_substring_search (path, strlen (path), ".lua", 4);
+
+ if (ext_pos != -1) {
+ name[ext_pos] = '\0';
+ }
+
+ lua_cmd->name = name;
+ }
+
+ lua_pop (L, 1);
+
+ lua_pushstring (L, "aliases");
+ lua_gettable (L, -2);
+
+ if (lua_type (L, -1) == LUA_TTABLE) {
+ lua_cmd->aliases = g_ptr_array_new_full (
+ rspamd_lua_table_size (L, -1),
+ g_free);
+
+ for (lua_pushnil (L); lua_next (L, -2); lua_pop (L, 1)) {
+ if (lua_isstring (L, -1)) {
+ g_ptr_array_add (lua_cmd->aliases,
+ g_strdup (lua_tostring (L, -1)));
+ }
+ }
+ }
+
+ lua_pop (L, 1);
+
+ lua_pushvalue (L, -1);
+ /* Reference table itself */
+ lua_cmd->command_data = GINT_TO_POINTER (luaL_ref (L, LUA_REGISTRYINDEX));
+ lua_cmd->flags |= RSPAMADM_FLAG_LUA;
+ lua_cmd->run = rspamadm_lua_command_run;
+ lua_cmd->help = rspamadm_lua_command_help;
+
+
+ g_ptr_array_add (dest, lua_cmd);
+ }
+
+ lua_settop (L, 0);
+ }
+}
diff --git a/src/rspamadm/configdump.c b/src/rspamadm/configdump.c
index d5380bc9d..cd9f6d378 100644
--- a/src/rspamadm/configdump.c
+++ b/src/rspamadm/configdump.c
@@ -33,8 +33,8 @@ extern struct rspamd_main *rspamd_main;
extern module_t *modules[];
extern worker_t *workers[];
-static void rspamadm_configdump (gint argc, gchar **argv);
-static const char *rspamadm_configdump_help (gboolean full_help);
+static void rspamadm_configdump (gint argc, gchar **argv, const struct rspamadm_command *);
+static const char *rspamadm_configdump_help (gboolean full_help, const struct rspamadm_command *);
struct rspamadm_command configdump_command = {
.name = "configdump",
@@ -61,7 +61,7 @@ static GOptionEntry entries[] = {
};
static const char *
-rspamadm_configdump_help (gboolean full_help)
+rspamadm_configdump_help (gboolean full_help, const struct rspamadm_command *cmd)
{
const char *help_str;
@@ -237,7 +237,7 @@ rspamadm_dump_section_obj (struct rspamd_config *cfg,
}
static void
-rspamadm_configdump (gint argc, gchar **argv)
+rspamadm_configdump (gint argc, gchar **argv, const struct rspamadm_command *cmd)
{
GOptionContext *context;
GError *error = NULL;
@@ -278,13 +278,13 @@ rspamadm_configdump (gint argc, gchar **argv)
(void) g_quark_from_static_string ((*pworker)->name);
pworker++;
}
+
cfg->cache = rspamd_symbols_cache_new (cfg);
cfg->compiled_modules = modules;
cfg->compiled_workers = workers;
cfg->cfg_name = config;
- if (!rspamd_config_read (cfg, cfg->cfg_name, NULL,
- config_logger, rspamd_main, ucl_vars)) {
+ if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main, ucl_vars)) {
ret = FALSE;
}
else {
diff --git a/src/rspamadm/confighelp.c b/src/rspamadm/confighelp.c
index 8f208b805..c77cb6091 100644
--- a/src/rspamadm/confighelp.c
+++ b/src/rspamadm/confighelp.c
@@ -30,9 +30,11 @@ extern struct rspamd_main *rspamd_main;
extern module_t *modules[];
extern worker_t *workers[];
-static void rspamadm_confighelp (gint argc, gchar **argv);
+static void rspamadm_confighelp (gint argc, gchar **argv,
+ const struct rspamadm_command *cmd);
-static const char *rspamadm_confighelp_help (gboolean full_help);
+static const char *rspamadm_confighelp_help (gboolean full_help,
+ const struct rspamadm_command *cmd);
struct rspamadm_command confighelp_command = {
.name = "confighelp",
@@ -55,7 +57,7 @@ static GOptionEntry entries[] = {
};
static const char *
-rspamadm_confighelp_help (gboolean full_help)
+rspamadm_confighelp_help (gboolean full_help, const struct rspamadm_command *cmd)
{
const char *help_str;
@@ -67,7 +69,7 @@ rspamadm_confighelp_help (gboolean full_help)
"-j: output pretty formatted JSON\n"
"-k: search by keyword in doc string\n"
"-P: use specific Lua plugins path\n"
- "--no-color: show colored output\n"
+ "--no-color: disable coloured output\n"
"--short: show only option names\n"
"--no-examples: do not show examples (impied by --short)\n"
"--help: shows available options and commands";
@@ -189,7 +191,7 @@ rspamadm_confighelp_search_word (const ucl_object_t *obj, const gchar *str)
}
static void
-rspamadm_confighelp (gint argc, gchar **argv)
+rspamadm_confighelp (gint argc, gchar **argv, const struct rspamadm_command *cmd)
{
struct rspamd_config *cfg;
ucl_object_t *doc_obj;
@@ -224,11 +226,12 @@ rspamadm_confighelp (gint argc, gchar **argv)
pworker++;
}
- cfg = rspamd_config_new ();
+ cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_SKIP_LUA);
+ cfg->lua_state = L;
cfg->compiled_modules = modules;
cfg->compiled_workers = workers;
- rspamd_rcl_config_init (cfg);
+ rspamd_rcl_config_init (cfg, NULL);
lua_pushboolean (cfg->lua_state, true);
lua_setglobal (cfg->lua_state, "confighelp");
rspamd_rcl_add_lua_plugins_path (cfg, plugins_path, NULL);
diff --git a/src/rspamadm/configtest.c b/src/rspamadm/configtest.c
index 18104c109..72a8f4945 100644
--- a/src/rspamadm/configtest.c
+++ b/src/rspamadm/configtest.c
@@ -28,8 +28,10 @@ extern struct rspamd_main *rspamd_main;
extern module_t *modules[];
extern worker_t *workers[];
-static void rspamadm_configtest (gint argc, gchar **argv);
-static const char *rspamadm_configtest_help (gboolean full_help);
+static void rspamadm_configtest (gint argc, gchar **argv,
+ const struct rspamadm_command *cmd);
+static const char *rspamadm_configtest_help (gboolean full_help,
+ const struct rspamadm_command *cmd);
struct rspamadm_command configtest_command = {
.name = "configtest",
@@ -50,7 +52,7 @@ static GOptionEntry entries[] = {
};
static const char *
-rspamadm_configtest_help (gboolean full_help)
+rspamadm_configtest_help (gboolean full_help, const struct rspamadm_command *cmd)
{
const char *help_str;
@@ -94,7 +96,7 @@ config_logger (rspamd_mempool_t *pool, gpointer ud)
}
static void
-rspamadm_configtest (gint argc, gchar **argv)
+rspamadm_configtest (gint argc, gchar **argv, const struct rspamadm_command *cmd)
{
GOptionContext *context;
GError *error = NULL;
@@ -139,8 +141,7 @@ rspamadm_configtest (gint argc, gchar **argv)
cfg->compiled_workers = workers;
cfg->cfg_name = config;
- if (!rspamd_config_read (cfg, cfg->cfg_name, NULL,
- config_logger, rspamd_main, ucl_vars)) {
+ if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main, ucl_vars)) {
ret = FALSE;
}
else {
diff --git a/src/rspamadm/configwizard.c b/src/rspamadm/configwizard.c
deleted file mode 100644
index 12c7b8cc1..000000000
--- a/src/rspamadm/configwizard.c
+++ /dev/null
@@ -1,165 +0,0 @@
-/*-
- * Copyright 2017 Vsevolod Stakhov
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include "config.h"
-#include "rspamadm.h"
-#include "cfg_file.h"
-#include "cfg_rcl.h"
-#include "utlist.h"
-#include "rspamd.h"
-#include "lua/lua_common.h"
-#include "utlist.h"
-
-static gchar *config = NULL;
-extern struct rspamd_main *rspamd_main;
-/* Defined in modules.c */
-extern module_t *modules[];
-extern worker_t *workers[];
-
-static void rspamadm_configwizard (gint argc, gchar **argv);
-static const char *rspamadm_configwizard_help (gboolean full_help);
-
-struct rspamadm_command configwizard_command = {
- .name = "configwizard",
- .flags = 0,
- .help = rspamadm_configwizard_help,
- .run = rspamadm_configwizard,
- .lua_subrs = NULL,
-};
-
-static GOptionEntry entries[] = {
- {"config", 'c', 0, G_OPTION_ARG_STRING, &config,
- "Config file to use", NULL},
- {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL}
-};
-
-static const char *
-rspamadm_configwizard_help (gboolean full_help)
-{
- const char *help_str;
-
- if (full_help) {
- help_str = "Perform guided configuration for Rspamd daemon\n\n"
- "Usage: rspamadm configwizard [-c <config_name>] [checks...]\n"
- " rspamadm configwizard [-c <config_name>] list\n"
- "Where options are:\n\n"
- "--help: shows available options and commands";
- }
- else {
- help_str = "Perform guided configuration for Rspamd daemon";
- }
-
- return help_str;
-}
-
-static void
-config_logger (rspamd_mempool_t *pool, gpointer ud)
-{
- struct rspamd_main *rm = ud;
-
- rm->cfg->log_type = RSPAMD_LOG_CONSOLE;
- rm->cfg->log_level = G_LOG_LEVEL_MESSAGE;
-
- rspamd_set_logger (rm->cfg, g_quark_try_string ("main"), &rm->logger,
- rm->server_pool);
-
- if (rspamd_log_open_priv (rm->logger, rm->workers_uid, rm->workers_gid) ==
- -1) {
- fprintf (stderr, "Fatal error, cannot open logfile, exiting\n");
- exit (EXIT_FAILURE);
- }
-}
-
-static void
-rspamadm_configwizard (gint argc, gchar **argv)
-{
- GOptionContext *context;
- GError *error = NULL;
- const gchar *confdir;
- struct rspamd_config *cfg = rspamd_main->cfg;
- gboolean ret = TRUE;
- worker_t **pworker;
- lua_State *L;
-
- context = g_option_context_new (
- "configwizard - perform guided configuration");
- g_option_context_set_summary (context,
- "Summary:\n Rspamd administration utility version "
- RVERSION
- "\n Release id: "
- RID);
- g_option_context_add_main_entries (context, entries, NULL);
-
- if (!g_option_context_parse (context, &argc, &argv, &error)) {
- fprintf (stderr, "option parsing failed: %s\n", error->message);
- g_error_free (error);
- exit (1);
- }
-
- if (config == NULL) {
- if ((confdir = g_hash_table_lookup (ucl_vars, "CONFDIR")) == NULL) {
- confdir = RSPAMD_CONFDIR;
- }
-
- config = g_strdup_printf ("%s%c%s", confdir, G_DIR_SEPARATOR,
- "rspamd.conf");
- }
-
- pworker = &workers[0];
- while (*pworker) {
- /* Init string quarks */
- (void) g_quark_from_static_string ((*pworker)->name);
- pworker++;
- }
-
- cfg->cache = rspamd_symbols_cache_new (cfg);
- cfg->compiled_modules = modules;
- cfg->compiled_workers = workers;
- cfg->cfg_name = config;
-
- if (!rspamd_config_read (cfg, cfg->cfg_name, NULL,
- config_logger, rspamd_main, ucl_vars)) {
- ret = FALSE;
- }
- else {
- /* Do post-load actions */
- rspamd_lua_post_load_config (cfg);
-
- if (!rspamd_init_filters (cfg, FALSE)) {
- ret = FALSE;
- }
-
- if (ret) {
- ret = rspamd_config_post_load (cfg, RSPAMD_CONFIG_INIT_SYMCACHE);
- }
- }
-
- if (ret) {
- L = cfg->lua_state;
- rspamd_lua_set_path (L, cfg->rcl_obj, ucl_vars);
- ucl_object_insert_key (cfg->rcl_obj, ucl_object_fromstring (cfg->cfg_name),
- "config_path", 0, false);
-
- rspamadm_execute_lua_ucl_subr (L,
- argc,
- argv,
- cfg->rcl_obj,
- "configwizard");
-
- lua_close (L);
- }
-
- exit (ret ? EXIT_SUCCESS : EXIT_FAILURE);
-}
diff --git a/src/rspamadm/control.c b/src/rspamadm/control.c
index 35cecbc69..4d7806b84 100644
--- a/src/rspamadm/control.c
+++ b/src/rspamadm/control.c
@@ -31,8 +31,10 @@ static gboolean ucl = TRUE;
static gboolean compact = FALSE;
static gdouble timeout = 1.0;
-static void rspamadm_control (gint argc, gchar **argv);
-static const char *rspamadm_control_help (gboolean full_help);
+static void rspamadm_control (gint argc, gchar **argv,
+ const struct rspamadm_command *cmd);
+static const char *rspamadm_control_help (gboolean full_help,
+ const struct rspamadm_command *cmd);
struct rspamadm_command control_command = {
.name = "control",
@@ -64,7 +66,7 @@ static GOptionEntry entries[] = {
};
static const char *
-rspamadm_control_help (gboolean full_help)
+rspamadm_control_help (gboolean full_help, const struct rspamadm_command *cmd)
{
const char *help_str;
@@ -158,7 +160,7 @@ rspamd_control_finish_handler (struct rspamd_http_connection *conn,
}
static void
-rspamadm_control (gint argc, gchar **argv)
+rspamadm_control (gint argc, gchar **argv, const struct rspamadm_command *_cmd)
{
GOptionContext *context;
GError *error = NULL;
@@ -260,6 +262,5 @@ rspamadm_control (gint argc, gchar **argv)
rspamd_http_connection_unref (conn);
rspamd_inet_address_free (addr);
- lua_close (L);
close (sock);
}
diff --git a/src/rspamadm/corpus_test.c b/src/rspamadm/corpus_test.c
deleted file mode 100644
index d72788d21..000000000
--- a/src/rspamadm/corpus_test.c
+++ /dev/null
@@ -1,125 +0,0 @@
-/*-
- * Copyright 2017 Pragadeesh C
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "rspamadm.h"
-#include "config.h"
-#include "lua/lua_common.h"
-
-static gchar *ham_directory = NULL;
-static gchar *spam_directory = NULL;
-static gchar *output_location = "results.log";
-static gint connections = 10;
-static gdouble timeout = 60.0;
-
-static void rspamadm_corpus_test (gint argc, gchar **argv);
-static const char *rspamadm_corpus_test_help (gboolean full_help);
-
-struct rspamadm_command corpus_test_command = {
- .name = "corpus_test",
- .flags = 0,
- .help = rspamadm_corpus_test_help,
- .run = rspamadm_corpus_test
-};
-
-static GOptionEntry entries[] = {
- {"ham", 'h', 0, G_OPTION_ARG_FILENAME, &ham_directory,
- "Ham directory", NULL},
- {"spam", 's', 0, G_OPTION_ARG_FILENAME, &spam_directory,
- "Spam directory", NULL},
- {"output", 'o', 0, G_OPTION_ARG_FILENAME, &output_location,
- "Log output location", NULL},
- {"connections", 'n', 0, G_OPTION_ARG_INT, &connections,
- "Number of parellel connections [Default: 10]", NULL},
- {"timeout", 't', 0, G_OPTION_ARG_DOUBLE, &timeout,
- "Timeout for connections [Default: 60]", NULL},
- {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL}
-};
-
-static const char *
-rspamadm_corpus_test_help (gboolean full_help)
-{
- const char *help_str;
-
- if (full_help) {
- help_str = "Create logs files from email corpus\n\n"
- "Usage: rspamadm corpus_test [-h <ham_directory>]"
- " [-s <spam_directory>]\n"
- "Where option are:\n\n"
- "-h: path to ham directory\n"
- "-s: path to spam directory\n"
- "-n: maximum parallel connections\n"
- "-o: log output file\n"
- "-t: timeout for rspamc operations (default: 60)\n";
-
- }
-
- else {
- help_str = "Create logs files from email corpus";
- }
-
- return help_str;
-}
-
-static void
-rspamadm_corpus_test (gint argc, gchar **argv)
-{
- GOptionContext *context;
- GError *error = NULL;
- lua_State *L;
- ucl_object_t *obj;
-
- context = g_option_context_new (
- "corpus_test - create logs files from email corpus");
-
- g_option_context_set_summary (context,
- "Summary:\n Rspamd administration utility version "
- RVERSION
- "\n Release id: "
- RID);
-
- g_option_context_add_main_entries (context, entries, NULL);
- g_option_context_set_ignore_unknown_options (context, TRUE);
-
- if (!g_option_context_parse (context, &argc, &argv, &error)) {
- rspamd_fprintf (stderr, "option parsing failed: %s\n", error->message);
- g_error_free (error);
- exit(1);
- }
-
- L = rspamd_lua_init ();
- rspamd_lua_set_path(L, NULL, ucl_vars);
-
- obj = ucl_object_typed_new (UCL_OBJECT);
- ucl_object_insert_key (obj, ucl_object_fromstring (ham_directory),
- "ham_directory", 0, false);
- ucl_object_insert_key (obj, ucl_object_fromstring (spam_directory),
- "spam_directory", 0, false);
- ucl_object_insert_key (obj, ucl_object_fromstring (output_location),
- "output_location", 0, false);
- ucl_object_insert_key (obj, ucl_object_fromint (connections),
- "connections", 0, false);
- ucl_object_insert_key (obj, ucl_object_fromdouble (timeout),
- "timeout", 0, false);
-
- rspamadm_execute_lua_ucl_subr (L,
- argc,
- argv,
- obj,
- "corpus_test");
-
- lua_close (L);
- ucl_object_unref (obj);
-}
diff --git a/src/rspamadm/dkim_keygen.c b/src/rspamadm/dkim_keygen.c
index e32f7bf05..86b228d01 100644
--- a/src/rspamadm/dkim_keygen.c
+++ b/src/rspamadm/dkim_keygen.c
@@ -27,8 +27,10 @@ static gchar *selector = NULL;
static gchar *domain = NULL;
static guint bits = 1024;
-static void rspamadm_dkim_keygen (gint argc, gchar **argv);
-static const char *rspamadm_dkim_keygen_help (gboolean full_help);
+static void rspamadm_dkim_keygen (gint argc, gchar **argv,
+ const struct rspamadm_command *cmd);
+static const char *rspamadm_dkim_keygen_help (gboolean full_help,
+ const struct rspamadm_command *cmd);
static void rspamadm_dkim_keygen_lua_subrs (gpointer pL);
struct rspamadm_command dkim_keygen_command = {
@@ -52,7 +54,7 @@ static GOptionEntry entries[] = {
};
static const char *
-rspamadm_dkim_keygen_help (gboolean full_help)
+rspamadm_dkim_keygen_help (gboolean full_help, const struct rspamadm_command *cmd)
{
const char *help_str;
@@ -221,7 +223,7 @@ rspamadm_dkim_keygen_lua_subrs (gpointer pL)
}
static void
-rspamadm_dkim_keygen (gint argc, gchar **argv)
+rspamadm_dkim_keygen (gint argc, gchar **argv, const struct rspamadm_command *cmd)
{
GOptionContext *context;
GError *error = NULL;
diff --git a/src/rspamadm/fuzzy_convert.c b/src/rspamadm/fuzzy_convert.c
index 40757d2e4..7671678dc 100644
--- a/src/rspamadm/fuzzy_convert.c
+++ b/src/rspamadm/fuzzy_convert.c
@@ -24,8 +24,10 @@ static gchar *redis_db = NULL;
static gchar *redis_password = NULL;
static int64_t fuzzy_expiry = 0;
-static void rspamadm_fuzzyconvert (gint argc, gchar **argv);
-static const char *rspamadm_fuzzyconvert_help (gboolean full_help);
+static void rspamadm_fuzzyconvert (gint argc, gchar **argv,
+ const struct rspamadm_command *cmd);
+static const char *rspamadm_fuzzyconvert_help (gboolean full_help,
+ const struct rspamadm_command *cmd);
struct rspamadm_command fuzzyconvert_command = {
.name = "fuzzyconvert",
@@ -51,7 +53,7 @@ static GOptionEntry entries[] = {
static const char *
-rspamadm_fuzzyconvert_help (gboolean full_help)
+rspamadm_fuzzyconvert_help (gboolean full_help, const struct rspamadm_command *cmd)
{
const char *help_str;
@@ -72,7 +74,7 @@ rspamadm_fuzzyconvert_help (gboolean full_help)
}
static void
-rspamadm_fuzzyconvert (gint argc, gchar **argv)
+rspamadm_fuzzyconvert (gint argc, gchar **argv, const struct rspamadm_command *cmd)
{
GOptionContext *context;
GError *error = NULL;
@@ -135,6 +137,5 @@ rspamadm_fuzzyconvert (gint argc, gchar **argv)
obj,
"fuzzy_convert");
- lua_close (L);
ucl_object_unref (obj);
}
diff --git a/src/rspamadm/fuzzy_merge.c b/src/rspamadm/fuzzy_merge.c
index 94631137c..f5e6847fa 100644
--- a/src/rspamadm/fuzzy_merge.c
+++ b/src/rspamadm/fuzzy_merge.c
@@ -22,8 +22,10 @@ static gchar *target = NULL;
static gchar **sources = NULL;
static gboolean quiet;
-static void rspamadm_fuzzy_merge (gint argc, gchar **argv);
-static const char *rspamadm_fuzzy_merge_help (gboolean full_help);
+static void rspamadm_fuzzy_merge (gint argc, gchar **argv,
+ const struct rspamadm_command *cmd);
+static const char *rspamadm_fuzzy_merge_help (gboolean full_help,
+ const struct rspamadm_command *cmd);
struct rspamadm_command fuzzy_merge_command = {
.name = "fuzzy_merge",
@@ -157,7 +159,7 @@ static struct rspamd_sqlite3_prstmt prepared_stmts[STMAX] = {
};
static const char *
-rspamadm_fuzzy_merge_help (gboolean full_help)
+rspamadm_fuzzy_merge_help (gboolean full_help, const struct rspamadm_command *cmd)
{
const char *help_str;
@@ -218,7 +220,7 @@ rspamadm_op_equal (gconstpointer a, gconstpointer b)
}
static void
-rspamadm_fuzzy_merge (gint argc, gchar **argv)
+rspamadm_fuzzy_merge (gint argc, gchar **argv, const struct rspamadm_command *cmd)
{
GOptionContext *context;
GError *error = NULL;
diff --git a/src/rspamadm/grep.c b/src/rspamadm/grep.c
deleted file mode 100644
index 5474ecf46..000000000
--- a/src/rspamadm/grep.c
+++ /dev/null
@@ -1,156 +0,0 @@
-/*-
- * Copyright (c) 2017, Andrew Lewis <nerf@judo.za.org>
- * Copyright (c) 2017, Vsevolod Stakhov <vsevolod@highsecure.ru>
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include "config.h"
-#include "rspamadm.h"
-#include "lua/lua_common.h"
-
-static gchar *string = NULL;
-static gchar *pattern = NULL;
-static gchar **inputs = NULL;
-static gboolean sensitive = FALSE;
-static gboolean orphans = FALSE;
-static gboolean partial = FALSE;
-static gboolean luapat = FALSE;
-
-static void rspamadm_grep (gint argc, gchar **argv);
-static const char *rspamadm_grep_help (gboolean full_help);
-
-struct rspamadm_command grep_command = {
- .name = "grep",
- .flags = 0,
- .help = rspamadm_grep_help,
- .run = rspamadm_grep,
- .lua_subrs = NULL,
-};
-
-static GOptionEntry entries[] = {
- {"string", 's', 0, G_OPTION_ARG_STRING, &string,
- "Plain string to search (case-insensitive)", NULL},
- {"lua", 'l', 0, G_OPTION_ARG_NONE, &luapat,
- "Use Lua patterns in string search", NULL},
- {"pattern", 'p', 0, G_OPTION_ARG_STRING, &pattern,
- "Pattern to search for (regex)", NULL},
- {"input", 'i', 0, G_OPTION_ARG_STRING_ARRAY, &inputs,
- "Process specified inputs (stdin if unspecified)", NULL},
- {"sensitive", 'S', 0, G_OPTION_ARG_NONE, &sensitive,
- "Enable case-sensitivity in string search", NULL},
- {"orphans", 'o', 0, G_OPTION_ARG_NONE, &orphans,
- "Print orphaned logs", NULL},
- {"partial", 'P', 0, G_OPTION_ARG_NONE, &partial,
- "Print partial logs", NULL},
- {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL}
-};
-
-
-static const char *
-rspamadm_grep_help (gboolean full_help)
-{
- const char *help_str;
-
- if (full_help) {
- help_str = "Search for patterns in rspamd logs\n\n"
- "Usage: rspamadm grep <-s string | -p pattern> [-i input1 -i input2 -S -o -P]\n"
- "Where options are:\n\n"
- "-s: Plain string to search (case-insensitive)\n"
- "-l: Use Lua patterns in string search\n"
- "-p: Pattern to search for (regex)\n"
- "-i: Process specified inputs (stdin if unspecified)\n"
- "-S: Enable case-sensitivity in string search\n"
- "-o: Print orphaned logs\n"
- "-P: Print partial logs\n";
- }
- else {
- help_str = "Search for patterns in rspamd logs";
- }
-
- return help_str;
-}
-
-static void
-rspamadm_grep (gint argc, gchar **argv)
-{
- GOptionContext *context;
- GError *error = NULL;
- lua_State *L;
- ucl_object_t *obj, *nobj;
- gchar **elt;
-
- context = g_option_context_new (
- "grep - search for patterns in rspamd logs");
- g_option_context_set_summary (context,
- "Summary:\n Rspamd administration utility version "
- RVERSION
- "\n Release id: "
- RID);
- g_option_context_add_main_entries (context, entries, NULL);
- g_option_context_set_ignore_unknown_options (context, FALSE);
-
- if (!g_option_context_parse (context, &argc, &argv, &error)) {
- rspamd_fprintf (stderr, "option parsing failed: %s\n", error->message);
- g_error_free (error);
- exit (1);
- }
-
- if (!pattern && !string) {
- rspamd_fprintf (stderr, "no search pattern specified\n");
- exit (1);
- }
- if (pattern && string) {
- rspamd_fprintf (stderr, "-s and -p are mutually-exclusive\n");
- exit (1);
- }
-
- L = rspamd_lua_init ();
- rspamd_lua_set_path (L, NULL, ucl_vars);
-
- obj = ucl_object_typed_new (UCL_OBJECT);
- if (string) {
- ucl_object_insert_key (obj, ucl_object_fromstring (string),
- "string", 0, false);
- }
- if (pattern) {
- ucl_object_insert_key (obj, ucl_object_fromstring (pattern),
- "pattern", 0, false);
- }
- nobj = ucl_object_typed_new (UCL_ARRAY);
- if (!inputs) {
- ucl_array_append (nobj, ucl_object_fromstring ("stdin"));
- }
- else {
- for (elt = inputs; *elt != NULL; elt ++) {
- ucl_array_append (nobj, ucl_object_fromstring (*elt));
- }
- }
- ucl_object_insert_key (obj, nobj, "inputs", 0, false);
- ucl_object_insert_key (obj, ucl_object_frombool (sensitive),
- "sensitive", 0, false);
- ucl_object_insert_key (obj, ucl_object_frombool (orphans),
- "orphans", 0, false);
- ucl_object_insert_key (obj, ucl_object_frombool (partial),
- "partial", 0, false);
- ucl_object_insert_key (obj, ucl_object_frombool (luapat),
- "luapat", 0, false);
-
- rspamadm_execute_lua_ucl_subr (L,
- argc,
- argv,
- obj,
- "grep");
-
- lua_close (L);
- ucl_object_unref (obj);
-}
diff --git a/src/rspamadm/keypair.c b/src/rspamadm/keypair.c
deleted file mode 100644
index 4214e0052..000000000
--- a/src/rspamadm/keypair.c
+++ /dev/null
@@ -1,140 +0,0 @@
-/*-
- * Copyright 2016 Vsevolod Stakhov
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include "config.h"
-#include "rspamadm.h"
-#include "cryptobox.h"
-#include "printf.h"
-#include "http.h"
-
-static gboolean hex_encode = FALSE;
-static gboolean raw = FALSE;
-static gboolean openssl = FALSE;
-static gboolean ucl = FALSE;
-static gboolean sign = FALSE;
-
-static void rspamadm_keypair (gint argc, gchar **argv);
-static const char *rspamadm_keypair_help (gboolean full_help);
-
-struct rspamadm_command keypair_command = {
- .name = "keypair",
- .flags = 0,
- .help = rspamadm_keypair_help,
- .run = rspamadm_keypair,
- .lua_subrs = NULL,
-};
-
-static GOptionEntry entries[] = {
- {"hex", 'x', 0, G_OPTION_ARG_NONE, &hex_encode,
- "Use hex encoding", NULL},
- {"raw", 'r', 0, G_OPTION_ARG_NONE, &raw,
- "Print just keys, no description", NULL},
- {"openssl", 'o', 0, G_OPTION_ARG_NONE, &openssl,
- "Generate openssl nistp256 keypair not curve25519 one", NULL},
- {"sign", 's', 0, G_OPTION_ARG_NONE, &sign,
- "Generate keypair for digital signing", NULL},
- {"ucl", 'u', 0, G_OPTION_ARG_NONE, &ucl,
- "Generate ucl config", NULL},
- {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL}
-};
-
-static const char *
-rspamadm_keypair_help (gboolean full_help)
-{
- const char *help_str;
-
- if (full_help) {
- help_str = "Create key pairs for httpcrypt\n\n"
- "Usage: rspamadm keypair [-x -r]\n"
- "Where options are:\n\n"
- "-x: encode with hex instead of base32\n"
- "-r: print raw base32/hex\n"
- "-o: generate openssl nistp256 keypair\n"
- "-s: generate keypair suitable for signatures\n"
- "-u: generate ucl config for keypair\n"
- "--help: shows available options and commands";
- }
- else {
- help_str = "Create encryption key pairs";
- }
-
- return help_str;
-}
-
-static void
-rspamadm_keypair (gint argc, gchar **argv)
-{
- GOptionContext *context;
- GError *error = NULL;
- struct rspamd_cryptobox_keypair *kp;
- gint how = 0;
- ucl_object_t *ucl_out;
- struct ucl_emitter_functions *ucl_emit_subr;
- GString *out;
- enum rspamd_cryptobox_keypair_type type = RSPAMD_KEYPAIR_KEX;
- enum rspamd_cryptobox_mode mode = RSPAMD_CRYPTOBOX_MODE_25519;
-
- context = g_option_context_new (
- "keypair - create encryption keys");
- g_option_context_set_summary (context,
- "Summary:\n Rspamd administration utility version "
- RVERSION
- "\n Release id: "
- RID);
- g_option_context_add_main_entries (context, entries, NULL);
-
- if (!g_option_context_parse (context, &argc, &argv, &error)) {
- fprintf (stderr, "option parsing failed: %s\n", error->message);
- g_error_free (error);
- exit (1);
- }
-
- if (openssl) {
- mode = RSPAMD_CRYPTOBOX_MODE_NIST;
- }
- if (hex_encode) {
- how |= RSPAMD_KEYPAIR_HEX;
- }
- else {
- how |= RSPAMD_KEYPAIR_BASE32;
- }
-
- if (sign) {
- type = RSPAMD_KEYPAIR_SIGN;
- }
-
- kp = rspamd_keypair_new (type, mode);
-
- if (ucl) {
- ucl_out = rspamd_keypair_to_ucl (kp, hex_encode);
- ucl_emit_subr = ucl_object_emit_file_funcs (stdout);
- ucl_object_emit_full (ucl_out, UCL_EMIT_CONFIG, ucl_emit_subr, NULL);
- ucl_object_emit_funcs_free (ucl_emit_subr);
- ucl_object_unref (ucl_out);
- }
- else {
- how |= RSPAMD_KEYPAIR_PUBKEY | RSPAMD_KEYPAIR_PRIVKEY;
-
- if (!raw) {
- how |= RSPAMD_KEYPAIR_HUMAN|RSPAMD_KEYPAIR_ID;
- }
-
- out = rspamd_keypair_print (kp, how);
- rspamd_printf ("%v", out);
- g_string_free (out, TRUE);
- }
-
- rspamd_keypair_unref (kp);
-}
diff --git a/src/rspamadm/lua_repl.c b/src/rspamadm/lua_repl.c
index e0ee7d9f1..a807101e0 100644
--- a/src/rspamadm/lua_repl.c
+++ b/src/rspamadm/lua_repl.c
@@ -47,8 +47,10 @@ static const char *default_history_file = ".rspamd_repl.hist";
#endif
#define MULTILINE_PROMPT "... "
-static void rspamadm_lua (gint argc, gchar **argv);
-static const char *rspamadm_lua_help (gboolean full_help);
+static void rspamadm_lua (gint argc, gchar **argv,
+ const struct rspamadm_command *cmd);
+static const char *rspamadm_lua_help (gboolean full_help,
+ const struct rspamadm_command *cmd);
struct rspamadm_command lua_command = {
.name = "lua",
@@ -121,7 +123,7 @@ static GOptionEntry entries[] = {
};
static const char *
-rspamadm_lua_help (gboolean full_help)
+rspamadm_lua_help (gboolean full_help, const struct rspamadm_command *cmd)
{
const char *help_str;
@@ -388,6 +390,7 @@ rspamadm_lua_message_handler (lua_State *L, gint argc, gchar **argv)
continue;
}
+ rspamd_message_process (task);
lua_pushcfunction (L, &rspamd_lua_traceback);
err_idx = lua_gettop (L);
@@ -658,13 +661,12 @@ rspamadm_lua_handle_exec (struct rspamd_http_connection_entry *conn_ent,
}
static void
-rspamadm_lua (gint argc, gchar **argv)
+rspamadm_lua (gint argc, gchar **argv, const struct rspamadm_command *cmd)
{
GOptionContext *context;
GError *error = NULL;
gchar **elt;
guint i;
- lua_State *L;
context = g_option_context_new ("lua - run lua interpreter");
g_option_context_set_summary (context,
@@ -689,9 +691,6 @@ rspamadm_lua (gint argc, gchar **argv)
}
}
- L = rspamd_lua_init ();
- rspamd_lua_set_path (L, NULL, ucl_vars);
-
if (paths) {
for (elt = paths; *elt != NULL; elt ++) {
rspamadm_lua_add_path (L, *elt);
diff --git a/src/rspamadm/pw.c b/src/rspamadm/pw.c
index 3299c479d..b00aac223 100644
--- a/src/rspamadm/pw.c
+++ b/src/rspamadm/pw.c
@@ -21,8 +21,10 @@
#include "rspamadm.h"
#include "unix-std.h"
-static void rspamadm_pw (gint argc, gchar **argv);
-static const char *rspamadm_pw_help (gboolean full_help);
+static void rspamadm_pw (gint argc, gchar **argv,
+ const struct rspamadm_command *cmd);
+static const char *rspamadm_pw_help (gboolean full_help,
+ const struct rspamadm_command *cmd);
static void rspamadm_pw_lua_subrs (gpointer pL);
static gboolean do_encrypt = FALSE;
@@ -57,7 +59,7 @@ static GOptionEntry entries[] = {
};
static const char *
-rspamadm_pw_help (gboolean full_help)
+rspamadm_pw_help (gboolean full_help, const struct rspamadm_command *cmd)
{
const char *help_str;
@@ -354,7 +356,7 @@ rspamadm_alg_list (void)
}
static void
-rspamadm_pw (gint argc, gchar **argv)
+rspamadm_pw (gint argc, gchar **argv, const struct rspamadm_command *cmd)
{
GOptionContext *context;
GError *error = NULL;
diff --git a/src/rspamadm/rescore.c b/src/rspamadm/rescore.c
deleted file mode 100644
index 926740023..000000000
--- a/src/rspamadm/rescore.c
+++ /dev/null
@@ -1,199 +0,0 @@
-/*-
- * Copyright 2017 Pragadeesh C
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "config.h"
-#include "rspamadm.h"
-#include "lua/lua_common.h"
-
-#if !defined(WITH_TORCH) || !defined(WITH_LUAJIT)
-#define HAS_TORCH false
-#else
-#define HAS_TORCH true
-#endif
-
-static gchar *logdir = NULL;
-static gchar *output = "new.scores";
-static gboolean score_diff = false; /* Print score diff flag */
-static gchar *config = NULL;
-extern struct rspamd_main *rspamd_main;
-/* Defined in modules.c */
-extern module_t *modules[];
-extern worker_t *workers[];
-
-static void rspamadm_rescore (gint argc, gchar **argv);
-
-static const char *rspamadm_rescore_help (gboolean full_help);
-
-struct rspamadm_command rescore_command = {
- .name = "rescore",
- .flags = 0,
- .help = rspamadm_rescore_help,
- .run = rspamadm_rescore
-};
-
-static GOptionEntry entries[] = {
- {"logdir", 'l', 0, G_OPTION_ARG_FILENAME, &logdir,
- "Logs directory", NULL},
- {"output", 'o', 0, G_OPTION_ARG_FILENAME, &output,
- "Scores output location", NULL},
- {"diff", 'd', 0, G_OPTION_ARG_NONE, &score_diff,
- "Print score diff", NULL},
- {"config", 'c', 0, G_OPTION_ARG_STRING, &config,
- "Config file to use", NULL},
- {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL}
-};
-
-static void
-config_logger (rspamd_mempool_t *pool, gpointer ud)
-{
- struct rspamd_main *rm = ud;
-
- rm->cfg->log_type = RSPAMD_LOG_CONSOLE;
- rm->cfg->log_level = G_LOG_LEVEL_MESSAGE;
-
- rspamd_set_logger (rm->cfg, g_quark_try_string ("main"), &rm->logger,
- rm->server_pool);
-
- if (rspamd_log_open_priv (rm->logger, rm->workers_uid, rm->workers_gid) ==
- -1) {
- fprintf (stderr, "Fatal error, cannot open logfile, exiting\n");
- exit (EXIT_FAILURE);
- }
-}
-
-static const char *
-rspamadm_rescore_help (gboolean full_help) {
-
- const char *help_str;
-
- if (full_help) {
- help_str = "Estimate optimal symbol weights from log files\n\n"
- "Usage: rspamadm rescore -l <log_directory>\n"
- "Where options are:\n\n"
- "-l: path to logs directory\n"
- "-o: scores output file location\n"
- "-d: print scores diff\n"
- "-i: max iterations for perceptron\n";
- } else {
- help_str = "Estimate optimal symbol weights from log files";
- }
-
- return help_str;
-}
-
-static void
-rspamadm_rescore (gint argc, gchar **argv) {
-
- GOptionContext *context;
- GError *error = NULL;
- lua_State *L;
- struct rspamd_config *cfg = rspamd_main->cfg, **pcfg;
- gboolean ret = TRUE;
- worker_t **pworker;
- const gchar *confdir;
-
- context = g_option_context_new (
- "rescore - estimate optimal symbol weights from log files");
-
- g_option_context_set_summary (context,
- "Summary:\n Rspamd administration utility version "
- RVERSION
- "\n Release id: "
- RID);
-
- g_option_context_add_main_entries (context, entries, NULL);
- g_option_context_set_ignore_unknown_options (context, TRUE);
-
- if (!g_option_context_parse (context, &argc, &argv, &error)) {
- rspamd_fprintf (stderr, "option parsing failed: %s\n", error->message);
- g_error_free (error);
- exit (EXIT_FAILURE);
- }
-
- if (!HAS_TORCH) {
- rspamd_fprintf (stderr, "Torch is not enabled. "
- "Use -DENABLE_TORCH=ON option while running cmake.\n");
- exit (EXIT_FAILURE);
- }
-
- if (logdir == NULL) {
- rspamd_fprintf (stderr, "Please specify log directory.\n");
- exit (EXIT_FAILURE);
- }
-
- if (config == NULL) {
- if ((confdir = g_hash_table_lookup (ucl_vars, "CONFDIR")) == NULL) {
- confdir = RSPAMD_CONFDIR;
- }
-
- config = g_strdup_printf ("%s%c%s", confdir, G_DIR_SEPARATOR,
- "rspamd.conf");
- }
-
- pworker = &workers[0];
- while (*pworker) {
- /* Init string quarks */
- (void) g_quark_from_static_string ((*pworker)->name);
- pworker++;
- }
-
- cfg->cache = rspamd_symbols_cache_new (cfg);
- cfg->compiled_modules = modules;
- cfg->compiled_workers = workers;
- cfg->cfg_name = config;
-
- if (!rspamd_config_read (cfg, cfg->cfg_name, NULL,
- config_logger, rspamd_main, ucl_vars)) {
- ret = FALSE;
- }
- else {
- /* Do post-load actions */
- rspamd_lua_post_load_config (cfg);
-
- if (!rspamd_init_filters (cfg, FALSE)) {
- ret = FALSE;
- }
-
- if (ret) {
- ret = rspamd_config_post_load (cfg, RSPAMD_CONFIG_INIT_SYMCACHE);
- rspamd_symbols_cache_validate (cfg->cache,
- cfg,
- FALSE);
- }
- }
-
- if (ret) {
- L = cfg->lua_state;
- rspamd_lua_set_path (L, cfg->rcl_obj, ucl_vars);
- ucl_object_insert_key (cfg->rcl_obj, ucl_object_fromstring (cfg->cfg_name),
- "config_path", 0, false);
- ucl_object_insert_key (cfg->rcl_obj, ucl_object_fromstring (logdir),
- "logdir", 0, false);
- ucl_object_insert_key (cfg->rcl_obj, ucl_object_fromstring (output),
- "output", 0, false);
- ucl_object_insert_key (cfg->rcl_obj, ucl_object_frombool (score_diff),
- "diff", 0, false);
- pcfg = lua_newuserdata (L, sizeof (struct rspamd_config *));
- rspamd_lua_setclass (L, "rspamd{config}", -1);
- *pcfg = cfg;
- lua_setglobal (L, "rspamd_config");
- rspamadm_execute_lua_ucl_subr (L,
- argc,
- argv,
- cfg->rcl_obj,
- "rescore");
- }
-}
diff --git a/src/rspamadm/rspamadm.c b/src/rspamadm/rspamadm.c
index b39c75897..68dbdeb85 100644
--- a/src/rspamadm/rspamadm.c
+++ b/src/rspamadm/rspamadm.c
@@ -31,9 +31,14 @@ static gboolean show_help = FALSE;
static gboolean show_version = FALSE;
GHashTable *ucl_vars = NULL;
struct rspamd_main *rspamd_main = NULL;
+lua_State *L = NULL;
-static void rspamadm_help (gint argc, gchar **argv);
-static const char* rspamadm_help_help (gboolean full_help);
+/* Defined in modules.c */
+extern module_t *modules[];
+extern worker_t *workers[];
+
+static void rspamadm_help (gint argc, gchar **argv, const struct rspamadm_command *);
+static const char* rspamadm_help_help (gboolean full_help, const struct rspamadm_command *);
struct rspamadm_command help_command = {
.name = "help",
@@ -70,7 +75,7 @@ rspamadm_error (void)
static void
rspamadm_version (void)
{
- printf ("Rspamadm %s\n", RVERSION);
+ rspamd_printf ("Rspamadm %s\n", RVERSION);
}
static void
@@ -79,30 +84,28 @@ rspamadm_usage (GOptionContext *context)
gchar *help_str;
help_str = g_option_context_get_help (context, TRUE, NULL);
- printf ("%s", help_str);
+ rspamd_printf ("%s", help_str);
}
static void
-rspamadm_commands (void)
+rspamadm_commands (GPtrArray *all_commands)
{
- const struct rspamadm_command **cmd;
-
- printf ("Rspamadm %s\n", RVERSION);
- printf ("Usage: rspamadm [global_options] command [command_options]\n");
- printf ("\nAvailable commands:\n");
+ const struct rspamadm_command *cmd;
+ guint i;
- cmd = commands;
+ rspamd_printf ("Rspamadm %s\n", RVERSION);
+ rspamd_printf ("Usage: rspamadm [global_options] command [command_options]\n");
+ rspamd_printf ("\nAvailable commands:\n");
- while (*cmd) {
- if (!((*cmd)->flags & RSPAMADM_FLAG_NOHELP)) {
- printf (" %-18s %-60s\n", (*cmd)->name, (*cmd)->help (FALSE));
+ PTR_ARRAY_FOREACH (all_commands, i, cmd) {
+ if (!(cmd->flags & RSPAMADM_FLAG_NOHELP)) {
+ rspamd_printf (" %-18s %-60s\n", cmd->name, cmd->help (FALSE, cmd));
}
- cmd ++;
}
}
static const char *
-rspamadm_help_help (gboolean full_help)
+rspamadm_help_help (gboolean full_help, const struct rspamadm_command *cmd)
{
const char *help_str;
@@ -118,23 +121,24 @@ rspamadm_help_help (gboolean full_help)
}
static void
-rspamadm_help (gint argc, gchar **argv)
+rspamadm_help (gint argc, gchar **argv, const struct rspamadm_command *command)
{
const gchar *cmd_name;
- const struct rspamadm_command *cmd, **cmd_list;
+ const struct rspamadm_command *cmd;
+ GPtrArray *all_commands = (GPtrArray *)command->command_data;
- printf ("Rspamadm %s\n", RVERSION);
- printf ("Usage: rspamadm [global_options] command [command_options]\n\n");
+ rspamd_printf ("Rspamadm %s\n", RVERSION);
+ rspamd_printf ("Usage: rspamadm [global_options] command [command_options]\n\n");
if (argc <= 1) {
cmd_name = "help";
}
else {
cmd_name = argv[1];
- printf ("Showing help for %s command\n\n", cmd_name);
+ rspamd_printf ("Showing help for %s command\n\n", cmd_name);
}
- cmd = rspamadm_search_command (cmd_name);
+ cmd = rspamadm_search_command (cmd_name, all_commands);
if (cmd == NULL) {
fprintf (stderr, "Invalid command name: %s\n", cmd_name);
@@ -142,20 +146,30 @@ rspamadm_help (gint argc, gchar **argv)
}
if (strcmp (cmd_name, "help") == 0) {
- printf ("Available commands:\n");
-
- cmd_list = commands;
-
- while (*cmd_list) {
- if (!((*cmd_list)->flags & RSPAMADM_FLAG_NOHELP)) {
- printf (" %-18s %-60s\n", (*cmd_list)->name,
- (*cmd_list)->help (FALSE));
+ guint i;
+ rspamd_printf ("Available commands:\n");
+
+ PTR_ARRAY_FOREACH (all_commands, i, cmd) {
+ if (!(cmd->flags & RSPAMADM_FLAG_NOHELP)) {
+ if (!(cmd->flags & RSPAMADM_FLAG_LUA)) {
+ printf (" %-18s %-60s\n", cmd->name,
+ cmd->help (FALSE, cmd));
+ }
+ else {
+ /* Just call lua subr */
+ (void)cmd->help (FALSE, cmd);
+ }
}
- cmd_list++;
}
}
else {
- printf ("%s\n", cmd->help (TRUE));
+ if (!(cmd->flags & RSPAMADM_FLAG_LUA)) {
+ rspamd_printf ("%s\n", cmd->help (TRUE, cmd));
+ }
+ else {
+ /* Just call lua subr */
+ (void)cmd->help (TRUE, cmd);
+ }
}
}
@@ -193,25 +207,12 @@ rspamadm_execute_lua_ucl_subr (gpointer pL, gint argc, gchar **argv,
gint err_idx, i, ret;
GString *tb;
gchar str[PATH_MAX];
- const struct rspamadm_command **cmd;
g_assert (script_name != NULL);
g_assert (res != NULL);
g_assert (L != NULL);
/* Init internal rspamadm routines */
- lua_newtable (L);
- cmd = commands;
-
- while (*cmd) {
- if ((*cmd)->lua_subrs != NULL) {
- (*cmd)->lua_subrs (L);
- }
-
- cmd ++;
- }
-
- lua_setglobal (L, "rspamadm");
rspamd_snprintf (str, sizeof (str), "return require \"%s.%s\"", "rspamadm",
script_name);
@@ -222,6 +223,11 @@ rspamadm_execute_lua_ucl_subr (gpointer pL, gint argc, gchar **argv,
return FALSE;
}
else {
+ if (lua_type (L, -1) == LUA_TTABLE) {
+ lua_pushstring (L, "handler");
+ lua_gettable (L, -2);
+ }
+
if (lua_type (L, -1) != LUA_TFUNCTION) {
msg_err ("lua script must return "
"function and not %s",
@@ -267,6 +273,37 @@ rspamadm_execute_lua_ucl_subr (gpointer pL, gint argc, gchar **argv,
return TRUE;
}
+static gint
+rspamdadm_commands_sort_func (gconstpointer a, gconstpointer b)
+{
+ const struct rspamadm_command *cmda = *((struct rspamadm_command const **)a),
+ *cmdb = *((struct rspamadm_command const **)b);
+
+ return strcmp (cmda->name, cmdb->name);
+}
+
+static gboolean
+rspamadm_command_maybe_match_name (const gchar *cmd, const gchar *input)
+{
+ gsize clen, inplen;
+
+ clen = strlen (cmd);
+ inplen = strlen (input);
+
+ if (rspamd_strings_levenshtein_distance (cmd, clen,
+ input, inplen, 1) == 1) {
+ return TRUE;
+ }
+ else if ((clen > inplen &&
+ rspamd_substring_search (cmd, clen, input, inplen) != -1) ||
+ (inplen > clen &&
+ rspamd_substring_search (input, inplen, cmd, clen) != -1)) {
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
gint
main (gint argc, gchar **argv, gchar **env)
{
@@ -278,12 +315,14 @@ main (gint argc, gchar **argv, gchar **env)
gchar **nargv, **targv;
const gchar *cmd_name;
const struct rspamadm_command *cmd;
+ GPtrArray *all_commands = g_ptr_array_new (); /* Discovered during check */
gint i, nargc, targc;
+ worker_t **pworker;
ucl_vars = g_hash_table_new_full (rspamd_strcase_hash,
rspamd_strcase_equal, g_free, g_free);
process_quark = g_quark_from_static_string ("rspamadm");
- cfg = rspamd_config_new ();
+ cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_DEFAULT);
cfg->libs_ctx = rspamd_init_libs ();
rspamd_main = g_malloc0 (sizeof (*rspamd_main));
rspamd_main->cfg = cfg;
@@ -291,6 +330,8 @@ main (gint argc, gchar **argv, gchar **env)
rspamd_main->type = process_quark;
rspamd_main->server_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (),
"rspamadm");
+ rspamadm_fill_internal_commands (all_commands);
+ help_command.command_data = all_commands;
/* Setup logger */
if (verbose) {
@@ -311,6 +352,16 @@ main (gint argc, gchar **argv, gchar **env)
rspamd_config_post_load (cfg,
RSPAMD_CONFIG_INIT_LIBS|RSPAMD_CONFIG_INIT_URL|RSPAMD_CONFIG_INIT_NO_TLD);
+ pworker = &workers[0];
+ while (*pworker) {
+ /* Init string quarks */
+ (void) g_quark_from_static_string ((*pworker)->name);
+ pworker++;
+ }
+
+ cfg->compiled_modules = modules;
+ cfg->compiled_workers = workers;
+
gperf_profiler_init (cfg, "rspamadm");
setproctitle ("rspamdadm");
@@ -349,6 +400,26 @@ main (gint argc, gchar **argv, gchar **env)
exit (1);
}
+ L = cfg->lua_state;
+ rspamd_lua_set_path (L, NULL, ucl_vars);
+ rspamd_lua_set_globals (cfg, L, ucl_vars);
+
+ /* Init rspamadm global */
+ lua_newtable (L);
+
+ PTR_ARRAY_FOREACH (all_commands, i, cmd) {
+ if (cmd->lua_subrs != NULL) {
+ cmd->lua_subrs (L);
+ }
+
+ cmd ++;
+ }
+
+ lua_setglobal (L, "rspamadm");
+
+ rspamadm_fill_lua_commands (L, all_commands);
+ g_ptr_array_sort (all_commands, rspamdadm_commands_sort_func);
+
g_strfreev (nargv);
if (show_version) {
@@ -360,7 +431,7 @@ main (gint argc, gchar **argv, gchar **env)
exit (EXIT_SUCCESS);
}
if (list_commands) {
- rspamadm_commands ();
+ rspamadm_commands (all_commands);
exit (EXIT_SUCCESS);
}
@@ -370,10 +441,29 @@ main (gint argc, gchar **argv, gchar **env)
cmd_name = "help";
}
- cmd = rspamadm_search_command (cmd_name);
+ cmd = rspamadm_search_command (cmd_name, all_commands);
if (cmd == NULL) {
- fprintf (stderr, "Invalid command name: %s\n", cmd_name);
+ rspamd_fprintf (stderr, "Invalid command name: %s\n", cmd_name);
+
+ /* Try fuzz search */
+ rspamd_fprintf (stderr, "Suggested commands:\n");
+ PTR_ARRAY_FOREACH (all_commands, i, cmd) {
+ guint j;
+ const gchar *alias;
+
+ if (rspamadm_command_maybe_match_name (cmd->name, cmd_name)) {
+ rspamd_fprintf (stderr, "%s\n", cmd->name);
+ }
+ else {
+ PTR_ARRAY_FOREACH (cmd->aliases, j, alias) {
+ if (rspamadm_command_maybe_match_name (alias, cmd_name)) {
+ rspamd_fprintf (stderr, "%s\n", alias);
+ }
+ }
+ }
+ }
+
exit (EXIT_FAILURE);
}
@@ -387,16 +477,17 @@ main (gint argc, gchar **argv, gchar **env)
targc = argc - nargc;
targv = nargv;
- cmd->run (targc, targv);
+ cmd->run (targc, targv, cmd);
g_strfreev (nargv);
}
else {
- cmd->run (0, NULL);
+ cmd->run (0, NULL, cmd);
}
rspamd_log_close (rspamd_main->logger);
REF_RELEASE (rspamd_main->cfg);
g_free (rspamd_main);
+ g_ptr_array_free (all_commands, TRUE);
return 0;
}
diff --git a/src/rspamadm/rspamadm.h b/src/rspamadm/rspamadm.h
index 9a209fd91..3d9799dd5 100644
--- a/src/rspamadm/rspamadm.h
+++ b/src/rspamadm/rspamadm.h
@@ -18,16 +18,24 @@
#include "config.h"
#include "ucl.h"
+#include <lua.h>
+#include <lauxlib.h>
+#include <lualib.h>
extern GHashTable *ucl_vars;
+extern lua_State *L;
GQuark rspamadm_error (void);
-typedef const gchar* (*rspamadm_help_func) (gboolean full_help);
-typedef void (*rspamadm_run_func) (gint argc, gchar **argv);
+struct rspamadm_command;
+typedef const gchar* (*rspamadm_help_func) (gboolean full_help,
+ const struct rspamadm_command *cmd);
+typedef void (*rspamadm_run_func) (gint argc, gchar **argv,
+ const struct rspamadm_command *cmd);
typedef void (*rspamadm_lua_exports_func) (gpointer lua_state);
#define RSPAMADM_FLAG_NOHELP (1 << 0)
+#define RSPAMADM_FLAG_LUA (1 << 1)
struct rspamadm_command {
const gchar *name;
@@ -35,14 +43,20 @@ struct rspamadm_command {
rspamadm_help_func help;
rspamadm_run_func run;
rspamadm_lua_exports_func lua_subrs;
+ GPtrArray *aliases;
+ gpointer command_data; /* Opaque data */
};
extern const struct rspamadm_command *commands[];
extern struct rspamadm_command help_command;
-const struct rspamadm_command *rspamadm_search_command (const gchar *name);
+const struct rspamadm_command *rspamadm_search_command (const gchar *name,
+ GPtrArray *all_commands);
+void rspamadm_fill_internal_commands (GPtrArray *dest);
+void rspamadm_fill_lua_commands (lua_State *L, GPtrArray *dest);
gboolean rspamadm_execute_lua_ucl_subr (gpointer L, gint argc, gchar **argv,
- const ucl_object_t *res, const gchar *script_name);
+ const ucl_object_t *res,
+ const gchar *script_name);
#endif
diff --git a/src/rspamadm/signtool.c b/src/rspamadm/signtool.c
index 0ae7e22c5..eae55b8bd 100644
--- a/src/rspamadm/signtool.c
+++ b/src/rspamadm/signtool.c
@@ -39,8 +39,10 @@ static gchar *editor = NULL;
static gboolean edit = FALSE;
enum rspamd_cryptobox_mode mode = RSPAMD_CRYPTOBOX_MODE_25519;
-static void rspamadm_signtool (gint argc, gchar **argv);
-static const char *rspamadm_signtool_help (gboolean full_help);
+static void rspamadm_signtool (gint argc, gchar **argv,
+ const struct rspamadm_command *cmd);
+static const char *rspamadm_signtool_help (gboolean full_help,
+ const struct rspamadm_command *cmd);
struct rspamadm_command signtool_command = {
.name = "signtool",
@@ -75,7 +77,8 @@ static GOptionEntry entries[] = {
};
static const char *
-rspamadm_signtool_help (gboolean full_help)
+rspamadm_signtool_help (gboolean full_help,
+ const struct rspamadm_command *cmd)
{
const char *help_str;
@@ -444,7 +447,8 @@ rspamadm_verify_file (const gchar *fname, const guchar *pk)
exit (errno);
}
- ret = rspamd_cryptobox_verify (map_sig, map, st.st_size, pk, mode);
+ ret = rspamd_cryptobox_verify (map_sig, st_sig.st_size,
+ map, st.st_size, pk, mode);
munmap (map, st.st_size);
munmap (map_sig, st_sig.st_size);
@@ -462,7 +466,7 @@ rspamadm_verify_file (const gchar *fname, const guchar *pk)
static void
-rspamadm_signtool (gint argc, gchar **argv)
+rspamadm_signtool (gint argc, gchar **argv, const struct rspamadm_command *cmd)
{
GOptionContext *context;
GError *error = NULL;
diff --git a/src/rspamadm/stat_convert.c b/src/rspamadm/stat_convert.c
index e4cf24629..ef17194b0 100644
--- a/src/rspamadm/stat_convert.c
+++ b/src/rspamadm/stat_convert.c
@@ -37,8 +37,10 @@ static gchar *redis_db = NULL;
static gchar *redis_password = NULL;
static gboolean reset_previous = FALSE;
-static void rspamadm_statconvert (gint argc, gchar **argv);
-static const char *rspamadm_statconvert_help (gboolean full_help);
+static void rspamadm_statconvert (gint argc, gchar **argv,
+ const struct rspamadm_command *cmd);
+static const char *rspamadm_statconvert_help (gboolean full_help,
+ const struct rspamadm_command *cmd);
struct rspamadm_command statconvert_command = {
.name = "statconvert",
@@ -77,7 +79,7 @@ static GOptionEntry entries[] = {
static const char *
-rspamadm_statconvert_help (gboolean full_help)
+rspamadm_statconvert_help (gboolean full_help, const struct rspamadm_command *cmd)
{
const char *help_str;
@@ -107,7 +109,7 @@ rspamadm_statconvert_help (gboolean full_help)
}
static void
-rspamadm_statconvert (gint argc, gchar **argv)
+rspamadm_statconvert (gint argc, gchar **argv, const struct rspamadm_command *cmd)
{
GOptionContext *context;
GError *error = NULL;
@@ -251,6 +253,5 @@ rspamadm_statconvert (gint argc, gchar **argv)
obj,
"stat_convert");
- lua_close (L);
ucl_object_unref (obj);
}
diff --git a/src/rspamd.c b/src/rspamd.c
index cc3be3357..643d249b1 100644
--- a/src/rspamd.c
+++ b/src/rspamd.c
@@ -276,7 +276,7 @@ reread_config (struct rspamd_main *rspamd_main)
gchar *cfg_file;
rspamd_symbols_cache_save (rspamd_main->cfg->cache);
- tmp_cfg = rspamd_config_new ();
+ tmp_cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_DEFAULT);
g_hash_table_unref (tmp_cfg->c_modules);
tmp_cfg->c_modules = g_hash_table_ref (rspamd_main->cfg->c_modules);
tmp_cfg->libs_ctx = rspamd_main->cfg->libs_ctx;
@@ -874,8 +874,7 @@ load_rspamd_config (struct rspamd_main *rspamd_main,
cfg->compiled_modules = modules;
cfg->compiled_workers = workers;
- if (!rspamd_config_read (cfg, cfg->cfg_name, NULL,
- config_logger, rspamd_main, ucl_vars)) {
+ if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main, ucl_vars)) {
return FALSE;
}
@@ -1180,7 +1179,7 @@ main (gint argc, gchar **argv, gchar **env)
"main");
rspamd_main->stat = rspamd_mempool_alloc0_shared (rspamd_main->server_pool,
sizeof (struct rspamd_stat));
- rspamd_main->cfg = rspamd_config_new ();
+ rspamd_main->cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_DEFAULT);
rspamd_main->spairs = g_hash_table_new_full (rspamd_spair_hash,
rspamd_spair_equal, g_free, rspamd_spair_close);
rspamd_main->start_mtx = rspamd_mempool_get_mutex (rspamd_main->server_pool);
diff --git a/test/functional/cases/106_url_tags.robot b/test/functional/cases/106_url_tags.robot
deleted file mode 100644
index 7954d0aac..000000000
--- a/test/functional/cases/106_url_tags.robot
+++ /dev/null
@@ -1,41 +0,0 @@
-*** Settings ***
-Suite Setup URL Tags Setup
-Suite Teardown URL Tags Teardown
-Library ${TESTDIR}/lib/rspamd.py
-Resource ${TESTDIR}/lib/rspamd.robot
-Variables ${TESTDIR}/lib/vars.py
-
-*** Variables ***
-${ADDITIONAL} ${TESTDIR}/lua/url_tags.lua
-${CONFIG} ${TESTDIR}/configs/pluginsplus.conf
-${MESSAGE} ${TESTDIR}/messages/url1.eml
-${REDIS_SCOPE} Suite
-${RSPAMD_SCOPE} Suite
-${URL_TLD} ${TESTDIR}/../lua/unit/test_tld.dat
-
-*** Test Cases ***
-URL TAGS PERSISTENCE
- ${result} = Scan Message With Rspamc --header=addtags=1 ${MESSAGE}
- Check Rspamc ${result} ADDED_TAGS (1.00)[no worry]
- ${result} = Scan Message With Rspamc ${MESSAGE}
- Check Rspamc ${result} FOUND_TAGS (1.00)[no worry]
-
-*** Keywords ***
-URL Tags Setup
- ${TMPDIR} = Make Temporary Directory
- Set Suite Variable ${TMPDIR}
- Run Redis
- ${LUA} = Make Temporary File
- ${goop} = Get File ${INSTALLROOT}/share/rspamd/rules/rspamd.lua
- ${goop2} = Get File ${ADDITIONAL}
- ${goop_unesc} = Catenate ${goop} ${goop2}
- ${PLUGIN_CONFIG} = Get File ${TESTDIR}/configs/url_tags.conf
- Set Suite Variable ${LUA}
- Set Suite Variable ${PLUGIN_CONFIG}
- Create File ${LUA} ${goop_unesc}
- Generic Setup TMPDIR=${TMPDIR}
-
-URL Tags Teardown
- Normal Teardown
- Remove File ${LUA}
- Shutdown Process With Children ${REDIS_PID}
diff --git a/test/rspamd_test_suite.c b/test/rspamd_test_suite.c
index d6fa237a3..2f8e893de 100644
--- a/test/rspamd_test_suite.c
+++ b/test/rspamd_test_suite.c
@@ -15,7 +15,7 @@ main (int argc, char **argv)
rspamd_main = (struct rspamd_main *)g_malloc (sizeof (struct rspamd_main));
memset (rspamd_main, 0, sizeof (struct rspamd_main));
rspamd_main->server_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL);
- cfg = rspamd_config_new ();
+ cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_DEFAULT);
rspamd_main->cfg = cfg;
cfg->cfg_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL);
cfg->log_type = RSPAMD_LOG_CONSOLE;
diff --git a/test/rspamd_upstream_test.c b/test/rspamd_upstream_test.c
index 47094398f..cce0008a9 100644
--- a/test/rspamd_upstream_test.c
+++ b/test/rspamd_upstream_test.c
@@ -64,7 +64,7 @@ rspamd_upstream_test_func (void)
struct timeval tv;
rspamd_inet_addr_t *addr, *next_addr, *paddr;
- cfg = rspamd_config_new ();
+ cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_SKIP_LUA);
cfg->dns_retransmits = 2;
cfg->dns_timeout = 0.5;
cfg->upstream_max_errors = 1;
diff --git a/utils/rspamd_stats.pl b/utils/rspamd_stats.pl
index f4f01dd85..b997ff525 100755
--- a/utils/rspamd_stats.pl
+++ b/utils/rspamd_stats.pl
@@ -460,7 +460,6 @@ sub ProcessRelated {
sub ProcessLog {
my ( $ts_format, @line ) = &log_time_format($rspamd_log);
- my $is_syslog = defined $ts_format && $ts_format eq 'syslog';
while() {
last if eof $rspamd_log;
@@ -474,10 +473,15 @@ sub ProcessLog {
if (/^.*rspamd_task_write_log.*$/) {
&spinner;
- my $ts =
- ($is_syslog)
- ? syslog2iso( join ' ', ( split /\s+/ )[ 0 .. 2 ] )
- : join ' ', ( split /\s+/ )[ 0 .. 1 ];
+ my $ts;
+ if ( $ts_format eq 'syslog' ) {
+ $ts = syslog2iso( join ' ', ( split /\s+/ )[ 0 .. 2 ] );
+ } elsif ( $ts_format eq 'syslog5424' ) {
+ /^([0-9-]+)T([0-9:]+)/;
+ $ts = "$1 $2";
+ } else {
+ $ts = join ' ', ( split /\s+/ )[ 0 .. 1 ];
+ }
next if ( $ts lt $startTime );
next if ( defined $endTime && $ts gt $endTime );
@@ -718,6 +722,12 @@ sub log_time_format {
last;
}
+ # 2018-04-16T06:25:46.012590+02:00 rspamd rspamd[12968]
+ elsif(/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,6})?(Z|[-+]\d{2}:\d{2}) \S+ rspamd\[\d+\]/) {
+ $format = 'syslog5424';
+ last;
+ }
+
# Skip newsyslog messages
# Aug 8 00:00:00 hostname newsyslog[63284]: logfile turned over
elsif ( /^\w{3} (?:\s?\d|\d\d) \d\d:\d\d:\d\d\ \S+ newsyslog\[\d+\]: logfile turned over$/ ) {