From: Vsevolod Stakhov Date: Mon, 17 Oct 2022 10:07:58 +0000 (+0100) Subject: [Minor] Rename file to a more appropriate name X-Git-Tag: 3.4~47 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=14b52f4498c10625f3c16101cd34dec25a59bac6;p=rspamd.git [Minor] Rename file to a more appropriate name --- diff --git a/src/client/rspamc.cxx b/src/client/rspamc.cxx index e1699c200..973bb58a4 100644 --- a/src/client/rspamc.cxx +++ b/src/client/rspamc.cxx @@ -34,7 +34,7 @@ #include "frozen/unordered_map.h" #include "fmt/format.h" #include "fmt/color.h" -#include "libutil/cxx/locked_file.hxx" +#include "libutil/cxx/file_util.hxx" #include "libutil/cxx/util.hxx" #ifdef HAVE_SYS_WAIT_H diff --git a/src/libserver/hyperscan_tools.cxx b/src/libserver/hyperscan_tools.cxx index c861a4668..a2acc0c18 100644 --- a/src/libserver/hyperscan_tools.cxx +++ b/src/libserver/hyperscan_tools.cxx @@ -20,7 +20,7 @@ #include "contrib/ankerl/unordered_dense.h" #include "contrib/ankerl/svector.h" #include "fmt/core.h" -#include "libutil/cxx/locked_file.hxx" +#include "libutil/cxx/file_util.hxx" #include "hs.h" #include "logger.h" #include "worker_util.h" diff --git a/src/libserver/symcache/symcache_impl.cxx b/src/libserver/symcache/symcache_impl.cxx index 961f270ce..c29b9d6d4 100644 --- a/src/libserver/symcache/symcache_impl.cxx +++ b/src/libserver/symcache/symcache_impl.cxx @@ -19,7 +19,7 @@ #include "symcache_item.hxx" #include "symcache_runtime.hxx" #include "unix-std.h" -#include "libutil/cxx/locked_file.hxx" +#include "libutil/cxx/file_util.hxx" #include "libutil/cxx/util.hxx" #include "fmt/core.h" #include "contrib/t1ha/t1ha.h" diff --git a/src/libutil/CMakeLists.txt b/src/libutil/CMakeLists.txt index 7b3103720..d3dd26454 100644 --- a/src/libutil/CMakeLists.txt +++ b/src/libutil/CMakeLists.txt @@ -18,6 +18,6 @@ SET(LIBRSPAMDUTILSRC ${CMAKE_CURRENT_SOURCE_DIR}/heap.c ${CMAKE_CURRENT_SOURCE_DIR}/multipattern.c ${CMAKE_CURRENT_SOURCE_DIR}/cxx/utf8_util.cxx - ${CMAKE_CURRENT_SOURCE_DIR}/cxx/locked_file.cxx) + ${CMAKE_CURRENT_SOURCE_DIR}/cxx/file_util.cxx) # Rspamdutil SET(RSPAMD_UTIL ${LIBRSPAMDUTILSRC} PARENT_SCOPE) \ No newline at end of file diff --git a/src/libutil/cxx/file_util.cxx b/src/libutil/cxx/file_util.cxx new file mode 100644 index 000000000..10a91a251 --- /dev/null +++ b/src/libutil/cxx/file_util.cxx @@ -0,0 +1,407 @@ +/*- + * Copyright 2022 Vsevolod Stakhov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "file_util.hxx" +#include +#include "libutil/util.h" +#include "libutil/unix-std.h" + +#define DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL + +#include "doctest/doctest.h" + +namespace rspamd::util { + +auto raii_file::open(const char *fname, int flags) -> tl::expected +{ + int oflags = flags; +#ifdef O_CLOEXEC + oflags |= O_CLOEXEC; +#endif + + if (fname == nullptr) { + return tl::make_unexpected("cannot open file; filename is nullptr"); + } + + auto fd = ::open(fname, oflags); + + if (fd == -1) { + return tl::make_unexpected(fmt::format("cannot open file {}: {}", fname, ::strerror(errno))); + } + + auto ret = raii_file{fname, fd, false}; + + if (fstat(ret.fd, &ret.st) == -1) { + return tl::make_unexpected(fmt::format("cannot stat file {}: {}", fname, ::strerror(errno))); + } + + return ret; +} + +auto raii_file::create(const char *fname, int flags, int perms) -> tl::expected +{ + int oflags = flags; +#ifdef O_CLOEXEC + oflags |= O_CLOEXEC; +#endif + + if (fname == nullptr) { + return tl::make_unexpected("cannot open file; filename is nullptr"); + } + + auto fd = ::open(fname, oflags, perms); + + if (fd == -1) { + return tl::make_unexpected(fmt::format("cannot create file {}: {}", fname, ::strerror(errno))); + } + + auto ret = raii_file{fname, fd, false}; + + if (fstat(ret.fd, &ret.st) == -1) { + return tl::make_unexpected(fmt::format("cannot stat file {}: {}", fname, ::strerror(errno))); + } + + return ret; +} + +auto raii_file::create_temp(const char *fname, int flags, int perms) -> tl::expected +{ + int oflags = flags; +#ifdef O_CLOEXEC + oflags |= O_CLOEXEC | O_CREAT | O_EXCL; +#endif + if (fname == nullptr) { + return tl::make_unexpected("cannot open file; filename is nullptr"); + } + + auto fd = ::open(fname, oflags, perms); + + if (fd == -1) { + return tl::make_unexpected(fmt::format("cannot create file {}: {}", fname, ::strerror(errno))); + } + + auto ret = raii_file{fname, fd, true}; + + if (fstat(ret.fd, &ret.st) == -1) { + return tl::make_unexpected(fmt::format("cannot stat file {}: {}", fname, ::strerror(errno))); + } + + return ret; +} + +auto raii_file::mkstemp(const char *pattern, int flags, int perms) -> tl::expected +{ + int oflags = flags; +#ifdef O_CLOEXEC + oflags |= O_CLOEXEC | O_CREAT | O_EXCL; +#endif + if (pattern == nullptr) { + return tl::make_unexpected("cannot open file; pattern is nullptr"); + } + + std::string mutable_pattern = pattern; + + auto fd = g_mkstemp_full(mutable_pattern.data(), oflags, perms); + + if (fd == -1) { + return tl::make_unexpected(fmt::format("cannot create file {}: {}", pattern, ::strerror(errno))); + } + + auto ret = raii_file{mutable_pattern.c_str(), fd, true}; + + if (fstat(ret.fd, &ret.st) == -1) { + return tl::make_unexpected(fmt::format("cannot stat file {}: {}", + mutable_pattern.c_str(), ::strerror(errno))); + } + + return ret; +} + +raii_file::~raii_file() noexcept +{ + if (fd != -1) { + if (temp) { + (void)unlink(fname.c_str()); + } + close(fd); + } +} + +auto raii_file::update_stat() noexcept -> bool +{ + return fstat(fd, &st) != -1; +} + + +raii_locked_file::~raii_locked_file() noexcept +{ + if (fd != -1) { + (void) rspamd_file_unlock(fd, FALSE); + } +} + +auto raii_locked_file::lock_raii_file(raii_file &&unlocked) -> tl::expected +{ + if (!rspamd_file_lock(unlocked.get_fd(), TRUE)) { + return tl::make_unexpected(fmt::format("cannot lock file {}: {}", unlocked.get_name(), ::strerror(errno))); + } + + return raii_locked_file{std::move(unlocked)}; +} + +auto raii_locked_file::unlock() -> raii_file { + if (fd != -1) { + (void) rspamd_file_unlock(fd, FALSE); + } + + return raii_file{static_cast(std::move(*this))}; +} + +raii_mmaped_file::raii_mmaped_file(raii_file &&_file, void *_map) + : file(std::move(_file)), map(_map) +{ +} + +auto raii_mmaped_file::mmap_shared(raii_file &&file, + int flags) -> tl::expected +{ + void *map; + + /* Update stat on file to ensure it is up-to-date */ + file.update_stat(); + map = mmap(NULL, file.get_stat().st_size, flags, MAP_SHARED, file.get_fd(), 0); + + if (map == MAP_FAILED) { + return tl::make_unexpected(fmt::format("cannot mmap file at fd: {}: {}", + file.get_fd(), ::strerror(errno))); + + } + + return raii_mmaped_file{std::move(file), map}; +} + +auto raii_mmaped_file::mmap_shared(const char *fname, int open_flags, + int mmap_flags) -> tl::expected +{ + auto file = raii_file::open(fname, open_flags); + + if (!file.has_value()) { + return tl::make_unexpected(file.error()); + } + + return raii_mmaped_file::mmap_shared(std::move(file.value()), mmap_flags); +} + +raii_mmaped_file::~raii_mmaped_file() +{ + if (map != nullptr) { + munmap(map, file.get_stat().st_size); + } +} + +raii_mmaped_file::raii_mmaped_file(raii_mmaped_file &&other) noexcept + : file(std::move(other.file)) +{ + std::swap(map, other.map); +} + +auto raii_file_sink::create(const char *fname, int flags, int perms, + const char *suffix) -> tl::expected +{ + if (!fname || !suffix) { + return tl::make_unexpected("cannot create file sink: bad input arguments"); + } + + auto tmp_fname = fmt::format("{}.{}", fname, suffix); + auto file = raii_locked_file::create(tmp_fname.c_str(), flags, perms); + + if (!file.has_value()) { + return tl::make_unexpected(file.error()); + } + + return raii_file_sink{std::move(file.value()), fname, std::move(tmp_fname)}; +} + +auto raii_file_sink::write_output() -> bool +{ + if (success) { + /* We cannot write output twice */ + return false; + } + + if (rename(tmp_fname.c_str(), output_fname.c_str()) == -1) { + return false; + } + + success = true; + + return true; +} + +raii_file_sink::~raii_file_sink() +{ + if (!success) { + /* Unlink sink */ + unlink(tmp_fname.c_str()); + } +} + +raii_file_sink::raii_file_sink(raii_locked_file &&_file, const char *_output, std::string &&_tmp_fname) + : file(std::move(_file)), output_fname(_output), tmp_fname(std::move(_tmp_fname)), success(false) +{ +} + +raii_file_sink::raii_file_sink(raii_file_sink &&other) noexcept + : file(std::move(other.file)), + output_fname(std::move(other.output_fname)), + tmp_fname(std::move(other.tmp_fname)), + success(other.success) +{ +} + +namespace tests { +template +static auto test_read_file(const T& f) { + auto fd = f.get_fd(); + (void)::lseek(fd, 0, SEEK_SET); + std::string buf('\0', (std::size_t)f.get_size()); + ::read(fd, buf.data(), buf.size()); + return buf; +} +template +static auto test_write_file(const T& f, const std::string_view &buf) { + auto fd = f.get_fd(); + (void)::lseek(fd, 0, SEEK_SET); + return ::write(fd, buf.data(), buf.size()); +} +auto random_fname(std::string_view extension) { + const auto *tmpdir = getenv("TMPDIR"); + if (tmpdir == nullptr) { + tmpdir = G_DIR_SEPARATOR_S "tmp"; + } + + std::string out_fname{tmpdir}; + out_fname += G_DIR_SEPARATOR_S; + + unsigned char hexbuf[32]; + rspamd_random_hex(hexbuf, sizeof(hexbuf)); + out_fname.append((const char *)hexbuf, sizeof(hexbuf)); + if (!extension.empty()) { + out_fname.append("."); + out_fname.append(extension); + } + + return out_fname; +} +TEST_SUITE("loked files utils") { + +TEST_CASE("create and delete file") { + auto fname = random_fname("tmp"); + { + auto raii_locked_file = raii_locked_file::create_temp(fname.c_str(), O_RDONLY, 00600); + CHECK(raii_locked_file.has_value()); + CHECK(raii_locked_file.value().get_extension() == "tmp"); + CHECK(::access(fname.c_str(), R_OK) == 0); + } + // File must be deleted after this call + auto ret = ::access(fname.c_str(), R_OK); + auto serrno = errno; + CHECK(ret == -1); + CHECK(serrno == ENOENT); + // Create one more time + { + auto raii_locked_file = raii_locked_file::create_temp(fname.c_str(), O_RDONLY, 00600); + CHECK(raii_locked_file.has_value()); + CHECK(::access(fname.c_str(), R_OK) == 0); + } + ret = ::access(fname.c_str(), R_OK); + serrno = errno; + CHECK(ret == -1); + CHECK(serrno == ENOENT); +} + +TEST_CASE("check lock") { + auto fname = random_fname(""); + { + auto raii_locked_file = raii_locked_file::create_temp(fname.c_str(), O_RDONLY, 00600); + CHECK(raii_locked_file.has_value()); + CHECK(raii_locked_file.value().get_extension() == ""); + CHECK(::access(fname.c_str(), R_OK) == 0); + auto raii_locked_file2 = raii_locked_file::open(fname.c_str(), O_RDONLY); + CHECK(!raii_locked_file2.has_value()); + CHECK(::access(fname.c_str(), R_OK) == 0); + } + // File must be deleted after this call + auto ret = ::access(fname.c_str(), R_OK); + auto serrno = errno; + CHECK(ret == -1); + CHECK(serrno == ENOENT); +} + +TEST_CASE("tempfile") { + std::string tmpname; + { + auto raii_locked_file = raii_locked_file::mkstemp("/tmp//doctest-XXXXXXXX", + O_RDONLY, 00600); + CHECK(raii_locked_file.has_value()); + CHECK(raii_locked_file.value().get_dir() == "/tmp"); + CHECK(access(raii_locked_file.value().get_name().data(), R_OK) == 0); + auto raii_locked_file2 = raii_locked_file::open(raii_locked_file.value().get_name().data(), O_RDONLY); + CHECK(!raii_locked_file2.has_value()); + CHECK(access(raii_locked_file.value().get_name().data(), R_OK) == 0); + tmpname = raii_locked_file.value().get_name(); + } + // File must be deleted after this call + auto ret = ::access(tmpname.c_str(), R_OK); + auto serrno = errno; + CHECK(ret == -1); + CHECK(serrno == ENOENT); +} + +TEST_CASE("mmap") { + std::string tmpname; + { + auto raii_file = raii_file::mkstemp("/tmp//doctest-XXXXXXXX", + O_RDWR|O_CREAT|O_EXCL, 00600); + CHECK(raii_file.has_value()); + CHECK(raii_file->get_dir() == "/tmp"); + CHECK(access(raii_file->get_name().data(), R_OK) == 0); + tmpname = std::string{raii_file->get_name()}; + char payload[] = {'1', '2', '3'}; + CHECK(write(raii_file->get_fd(), payload, sizeof(payload)) == sizeof(payload)); + auto mmapped_file1 = raii_mmaped_file::mmap_shared(std::move(raii_file.value()), PROT_READ|PROT_WRITE); + CHECK(mmapped_file1.has_value()); + CHECK(!raii_file->is_valid()); + CHECK(mmapped_file1->get_size() == sizeof(payload)); + CHECK(memcmp(mmapped_file1->get_map(), payload, sizeof(payload)) == 0); + *(char *)mmapped_file1->get_map() = '2'; + auto mmapped_file2 = raii_mmaped_file::mmap_shared(tmpname.c_str(), O_RDONLY, PROT_READ); + CHECK(mmapped_file2.has_value()); + CHECK(mmapped_file2->get_size() == sizeof(payload)); + CHECK(memcmp(mmapped_file2->get_map(), payload, sizeof(payload)) != 0); + CHECK(memcmp(mmapped_file2->get_map(), mmapped_file1->get_map(), sizeof(payload)) == 0); + } + // File must be deleted after this call + auto ret = ::access(tmpname.c_str(), R_OK); + auto serrno = errno; + CHECK(ret == -1); + CHECK(serrno == ENOENT); +} + +} // TEST_SUITE + +} // namespace tests + +} // namespace rspamd::util diff --git a/src/libutil/cxx/file_util.hxx b/src/libutil/cxx/file_util.hxx new file mode 100644 index 000000000..c66fd17ef --- /dev/null +++ b/src/libutil/cxx/file_util.hxx @@ -0,0 +1,261 @@ +/*- + * Copyright 2022 Vsevolod Stakhov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RSPAMD_FILE_UTIL_HXX +#define RSPAMD_FILE_UTIL_HXX +#pragma once + +#include "config.h" +#include "contrib/expected/expected.hpp" +#include +#include + +namespace rspamd::util { +/** + * A simple RAII object to contain a move only file descriptor + * A file is unlocked and closed when not needed + */ +struct raii_file { +public: + virtual ~raii_file() noexcept; + + static auto open(const char *fname, int flags) -> tl::expected; + static auto create(const char *fname, int flags, int perms) -> tl::expected; + static auto create_temp(const char *fname, int flags, int perms) -> tl::expected; + static auto mkstemp(const char *pattern, int flags, int perms) -> tl::expected; + + auto get_fd() const -> int { + return fd; + } + + auto get_stat() const -> const struct stat& { + return st; + }; + + auto get_size() const -> std::size_t { + return st.st_size; + }; + + auto get_name() const -> std::string_view { + return std::string_view{fname}; + } + + auto get_dir() const -> std::string_view { + auto sep_pos = fname.rfind(G_DIR_SEPARATOR); + + if (sep_pos == std::string::npos) { + return std::string_view{fname}; + } + + while (sep_pos >= 1 && fname[sep_pos - 1] == G_DIR_SEPARATOR) { + sep_pos --; + } + + return std::string_view{fname.c_str(), sep_pos}; + } + + auto get_extension() const -> std::string_view { + auto sep_pos = fname.rfind(G_DIR_SEPARATOR); + + if (sep_pos == std::string::npos) { + sep_pos = 0; + } + + auto filename = std::string_view{fname.c_str() + sep_pos}; + auto dot_pos = filename.find('.'); + + if (dot_pos == std::string::npos) { + return std::string_view{}; + } + else { + return std::string_view{filename.data() + dot_pos + 1, filename.size() - dot_pos - 1}; + } + } + + raii_file& operator=(raii_file &&other) noexcept { + std::swap(fd, other.fd); + std::swap(temp, other.temp); + std::swap(fname, other.fname); + std::swap(st, other.st); + + return *this; + } + + raii_file(raii_file &&other) noexcept { + *this = std::move(other); + } + + /** + * Prevent file from being deleted + * @return + */ + auto make_immortal() noexcept { + temp = false; + } + + /** + * Performs fstat on an opened file to refresh internal stat + * @return + */ + auto update_stat() noexcept -> bool; + + auto is_valid() noexcept -> bool { + return fd != -1; + } + + /* Do not allow copy/default ctor */ + const raii_file& operator=(const raii_file &other) = delete; + raii_file() = delete; + raii_file(const raii_file &other) = delete; +protected: + int fd = -1; + bool temp; + std::string fname; + struct stat st; + + explicit raii_file(const char *fname, int fd, bool temp) : fd(fd), temp(temp), fname(fname) {} +}; +/** + * A simple RAII object to contain a file descriptor with an flock wrap + * A file is unlocked and closed when not needed + */ +struct raii_locked_file final : public raii_file { +public: + ~raii_locked_file() noexcept override; + + static auto open(const char *fname, int flags) -> tl::expected { + auto locked = raii_file::open(fname, flags).and_then([](T &&file) { + return lock_raii_file(std::forward(file)); + }); + + return locked; + } + static auto create(const char *fname, int flags, int perms) -> tl::expected { + auto locked = raii_file::create(fname, flags, perms).and_then([](T &&file) { + return lock_raii_file(std::forward(file)); + }); + + return locked; + } + static auto create_temp(const char *fname, int flags, int perms) -> tl::expected { + auto locked = raii_file::create_temp(fname, flags, perms).and_then([](T &&file) { + return lock_raii_file(std::forward(file)); + }); + + return locked; + } + static auto mkstemp(const char *pattern, int flags, int perms) -> tl::expected { + auto locked = raii_file::mkstemp(pattern, flags, perms).and_then([](T &&file) { + return lock_raii_file(std::forward(file)); + }); + + return locked; + } + + raii_locked_file& operator=(raii_locked_file &&other) noexcept { + std::swap(fd, other.fd); + std::swap(temp, other.temp); + std::swap(fname, other.fname); + std::swap(st, other.st); + + return *this; + } + + /** + * Unlock a locked file and return back unlocked file transferring ownership. + * A locked file cannot be used after this method. + */ + auto unlock() -> raii_file; + + raii_locked_file(raii_locked_file &&other) noexcept : raii_file(static_cast(std::move(other))) {} + /* Do not allow copy/default ctor */ + const raii_locked_file& operator=(const raii_locked_file &other) = delete; + raii_locked_file() = delete; + raii_locked_file(const raii_locked_file &other) = delete; +private: + static auto lock_raii_file(raii_file &&unlocked) -> tl::expected; + raii_locked_file(raii_file &&other) noexcept : raii_file(std::move(other)) {} + explicit raii_locked_file(const char *fname, int fd, bool temp) : raii_file(fname, fd, temp) {} +}; + +/** + * A mmap wrapper on top of a locked file + */ +struct raii_mmaped_file final { + ~raii_mmaped_file(); + static auto mmap_shared(raii_file &&file, int flags) -> tl::expected; + static auto mmap_shared(const char *fname, int open_flags, int mmap_flags) -> tl::expected; + // Returns a constant pointer to the underlying map + auto get_map() const -> void* {return map;} + auto get_file() const -> const raii_file& { return file; } + // Passes the ownership of the mmaped memory to the callee + auto steal_map() -> std::tuple { + auto ret = std::make_tuple(this->map, file.get_stat().st_size); + this->map = nullptr; + return ret; + } + + auto get_size() const -> std::size_t { return file.get_stat().st_size; } + + raii_mmaped_file& operator=(raii_mmaped_file &&other) noexcept { + std::swap(map, other.map); + file = std::move(other.file); + + return *this; + } + + raii_mmaped_file(raii_mmaped_file &&other) noexcept; + + /* Do not allow copy/default ctor */ + const raii_mmaped_file& operator=(const raii_mmaped_file &other) = delete; + raii_mmaped_file() = delete; + raii_mmaped_file(const raii_mmaped_file &other) = delete; +private: + /* Is intended to be used with map_shared */ + explicit raii_mmaped_file(raii_file &&_file, void *_map); + raii_file file; + void *map = nullptr; +}; + +/** + * A helper to have a file to write that will be renamed to the + * target file if successful or deleted in the case of failure + */ +struct raii_file_sink final { + static auto create(const char *fname, int flags, int perms, const char *suffix = "new") + -> tl::expected; + auto write_output() -> bool; + ~raii_file_sink(); + auto get_fd() const -> int + { + return file.get_fd(); + } + + raii_file_sink(raii_file_sink &&other) noexcept; + /* Do not allow copy/default ctor */ + const raii_file_sink& operator=(const raii_file_sink &other) = delete; + raii_file_sink() = delete; + raii_file_sink(const raii_file_sink &other) = delete; +private: + explicit raii_file_sink(raii_locked_file &&_file, const char *_output, std::string &&_tmp_fname); + raii_locked_file file; + std::string output_fname; + std::string tmp_fname; + bool success; +}; + +} + +#endif //RSPAMD_FILE_UTIL_HXX diff --git a/src/libutil/cxx/locked_file.cxx b/src/libutil/cxx/locked_file.cxx deleted file mode 100644 index c972e1de3..000000000 --- a/src/libutil/cxx/locked_file.cxx +++ /dev/null @@ -1,407 +0,0 @@ -/*- - * Copyright 2022 Vsevolod Stakhov - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "locked_file.hxx" -#include -#include "libutil/util.h" -#include "libutil/unix-std.h" - -#define DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL - -#include "doctest/doctest.h" - -namespace rspamd::util { - -auto raii_file::open(const char *fname, int flags) -> tl::expected -{ - int oflags = flags; -#ifdef O_CLOEXEC - oflags |= O_CLOEXEC; -#endif - - if (fname == nullptr) { - return tl::make_unexpected("cannot open file; filename is nullptr"); - } - - auto fd = ::open(fname, oflags); - - if (fd == -1) { - return tl::make_unexpected(fmt::format("cannot open file {}: {}", fname, ::strerror(errno))); - } - - auto ret = raii_file{fname, fd, false}; - - if (fstat(ret.fd, &ret.st) == -1) { - return tl::make_unexpected(fmt::format("cannot stat file {}: {}", fname, ::strerror(errno))); - } - - return ret; -} - -auto raii_file::create(const char *fname, int flags, int perms) -> tl::expected -{ - int oflags = flags; -#ifdef O_CLOEXEC - oflags |= O_CLOEXEC; -#endif - - if (fname == nullptr) { - return tl::make_unexpected("cannot open file; filename is nullptr"); - } - - auto fd = ::open(fname, oflags, perms); - - if (fd == -1) { - return tl::make_unexpected(fmt::format("cannot create file {}: {}", fname, ::strerror(errno))); - } - - auto ret = raii_file{fname, fd, false}; - - if (fstat(ret.fd, &ret.st) == -1) { - return tl::make_unexpected(fmt::format("cannot stat file {}: {}", fname, ::strerror(errno))); - } - - return ret; -} - -auto raii_file::create_temp(const char *fname, int flags, int perms) -> tl::expected -{ - int oflags = flags; -#ifdef O_CLOEXEC - oflags |= O_CLOEXEC | O_CREAT | O_EXCL; -#endif - if (fname == nullptr) { - return tl::make_unexpected("cannot open file; filename is nullptr"); - } - - auto fd = ::open(fname, oflags, perms); - - if (fd == -1) { - return tl::make_unexpected(fmt::format("cannot create file {}: {}", fname, ::strerror(errno))); - } - - auto ret = raii_file{fname, fd, true}; - - if (fstat(ret.fd, &ret.st) == -1) { - return tl::make_unexpected(fmt::format("cannot stat file {}: {}", fname, ::strerror(errno))); - } - - return ret; -} - -auto raii_file::mkstemp(const char *pattern, int flags, int perms) -> tl::expected -{ - int oflags = flags; -#ifdef O_CLOEXEC - oflags |= O_CLOEXEC | O_CREAT | O_EXCL; -#endif - if (pattern == nullptr) { - return tl::make_unexpected("cannot open file; pattern is nullptr"); - } - - std::string mutable_pattern = pattern; - - auto fd = g_mkstemp_full(mutable_pattern.data(), oflags, perms); - - if (fd == -1) { - return tl::make_unexpected(fmt::format("cannot create file {}: {}", pattern, ::strerror(errno))); - } - - auto ret = raii_file{mutable_pattern.c_str(), fd, true}; - - if (fstat(ret.fd, &ret.st) == -1) { - return tl::make_unexpected(fmt::format("cannot stat file {}: {}", - mutable_pattern.c_str(), ::strerror(errno))); - } - - return ret; -} - -raii_file::~raii_file() noexcept -{ - if (fd != -1) { - if (temp) { - (void)unlink(fname.c_str()); - } - close(fd); - } -} - -auto raii_file::update_stat() noexcept -> bool -{ - return fstat(fd, &st) != -1; -} - - -raii_locked_file::~raii_locked_file() noexcept -{ - if (fd != -1) { - (void) rspamd_file_unlock(fd, FALSE); - } -} - -auto raii_locked_file::lock_raii_file(raii_file &&unlocked) -> tl::expected -{ - if (!rspamd_file_lock(unlocked.get_fd(), TRUE)) { - return tl::make_unexpected(fmt::format("cannot lock file {}: {}", unlocked.get_name(), ::strerror(errno))); - } - - return raii_locked_file{std::move(unlocked)}; -} - -auto raii_locked_file::unlock() -> raii_file { - if (fd != -1) { - (void) rspamd_file_unlock(fd, FALSE); - } - - return raii_file{static_cast(std::move(*this))}; -} - -raii_mmaped_file::raii_mmaped_file(raii_file &&_file, void *_map) - : file(std::move(_file)), map(_map) -{ -} - -auto raii_mmaped_file::mmap_shared(raii_file &&file, - int flags) -> tl::expected -{ - void *map; - - /* Update stat on file to ensure it is up-to-date */ - file.update_stat(); - map = mmap(NULL, file.get_stat().st_size, flags, MAP_SHARED, file.get_fd(), 0); - - if (map == MAP_FAILED) { - return tl::make_unexpected(fmt::format("cannot mmap file at fd: {}: {}", - file.get_fd(), ::strerror(errno))); - - } - - return raii_mmaped_file{std::move(file), map}; -} - -auto raii_mmaped_file::mmap_shared(const char *fname, int open_flags, - int mmap_flags) -> tl::expected -{ - auto file = raii_file::open(fname, open_flags); - - if (!file.has_value()) { - return tl::make_unexpected(file.error()); - } - - return raii_mmaped_file::mmap_shared(std::move(file.value()), mmap_flags); -} - -raii_mmaped_file::~raii_mmaped_file() -{ - if (map != nullptr) { - munmap(map, file.get_stat().st_size); - } -} - -raii_mmaped_file::raii_mmaped_file(raii_mmaped_file &&other) noexcept - : file(std::move(other.file)) -{ - std::swap(map, other.map); -} - -auto raii_file_sink::create(const char *fname, int flags, int perms, - const char *suffix) -> tl::expected -{ - if (!fname || !suffix) { - return tl::make_unexpected("cannot create file sink: bad input arguments"); - } - - auto tmp_fname = fmt::format("{}.{}", fname, suffix); - auto file = raii_locked_file::create(tmp_fname.c_str(), flags, perms); - - if (!file.has_value()) { - return tl::make_unexpected(file.error()); - } - - return raii_file_sink{std::move(file.value()), fname, std::move(tmp_fname)}; -} - -auto raii_file_sink::write_output() -> bool -{ - if (success) { - /* We cannot write output twice */ - return false; - } - - if (rename(tmp_fname.c_str(), output_fname.c_str()) == -1) { - return false; - } - - success = true; - - return true; -} - -raii_file_sink::~raii_file_sink() -{ - if (!success) { - /* Unlink sink */ - unlink(tmp_fname.c_str()); - } -} - -raii_file_sink::raii_file_sink(raii_locked_file &&_file, const char *_output, std::string &&_tmp_fname) - : file(std::move(_file)), output_fname(_output), tmp_fname(std::move(_tmp_fname)), success(false) -{ -} - -raii_file_sink::raii_file_sink(raii_file_sink &&other) noexcept - : file(std::move(other.file)), - output_fname(std::move(other.output_fname)), - tmp_fname(std::move(other.tmp_fname)), - success(other.success) -{ -} - -namespace tests { -template -static auto test_read_file(const T& f) { - auto fd = f.get_fd(); - (void)::lseek(fd, 0, SEEK_SET); - std::string buf('\0', (std::size_t)f.get_size()); - ::read(fd, buf.data(), buf.size()); - return buf; -} -template -static auto test_write_file(const T& f, const std::string_view &buf) { - auto fd = f.get_fd(); - (void)::lseek(fd, 0, SEEK_SET); - return ::write(fd, buf.data(), buf.size()); -} -auto random_fname(std::string_view extension) { - const auto *tmpdir = getenv("TMPDIR"); - if (tmpdir == nullptr) { - tmpdir = G_DIR_SEPARATOR_S "tmp"; - } - - std::string out_fname{tmpdir}; - out_fname += G_DIR_SEPARATOR_S; - - unsigned char hexbuf[32]; - rspamd_random_hex(hexbuf, sizeof(hexbuf)); - out_fname.append((const char *)hexbuf, sizeof(hexbuf)); - if (!extension.empty()) { - out_fname.append("."); - out_fname.append(extension); - } - - return out_fname; -} -TEST_SUITE("loked files utils") { - -TEST_CASE("create and delete file") { - auto fname = random_fname("tmp"); - { - auto raii_locked_file = raii_locked_file::create_temp(fname.c_str(), O_RDONLY, 00600); - CHECK(raii_locked_file.has_value()); - CHECK(raii_locked_file.value().get_extension() == "tmp"); - CHECK(::access(fname.c_str(), R_OK) == 0); - } - // File must be deleted after this call - auto ret = ::access(fname.c_str(), R_OK); - auto serrno = errno; - CHECK(ret == -1); - CHECK(serrno == ENOENT); - // Create one more time - { - auto raii_locked_file = raii_locked_file::create_temp(fname.c_str(), O_RDONLY, 00600); - CHECK(raii_locked_file.has_value()); - CHECK(::access(fname.c_str(), R_OK) == 0); - } - ret = ::access(fname.c_str(), R_OK); - serrno = errno; - CHECK(ret == -1); - CHECK(serrno == ENOENT); -} - -TEST_CASE("check lock") { - auto fname = random_fname(""); - { - auto raii_locked_file = raii_locked_file::create_temp(fname.c_str(), O_RDONLY, 00600); - CHECK(raii_locked_file.has_value()); - CHECK(raii_locked_file.value().get_extension() == ""); - CHECK(::access(fname.c_str(), R_OK) == 0); - auto raii_locked_file2 = raii_locked_file::open(fname.c_str(), O_RDONLY); - CHECK(!raii_locked_file2.has_value()); - CHECK(::access(fname.c_str(), R_OK) == 0); - } - // File must be deleted after this call - auto ret = ::access(fname.c_str(), R_OK); - auto serrno = errno; - CHECK(ret == -1); - CHECK(serrno == ENOENT); -} - -TEST_CASE("tempfile") { - std::string tmpname; - { - auto raii_locked_file = raii_locked_file::mkstemp("/tmp//doctest-XXXXXXXX", - O_RDONLY, 00600); - CHECK(raii_locked_file.has_value()); - CHECK(raii_locked_file.value().get_dir() == "/tmp"); - CHECK(access(raii_locked_file.value().get_name().data(), R_OK) == 0); - auto raii_locked_file2 = raii_locked_file::open(raii_locked_file.value().get_name().data(), O_RDONLY); - CHECK(!raii_locked_file2.has_value()); - CHECK(access(raii_locked_file.value().get_name().data(), R_OK) == 0); - tmpname = raii_locked_file.value().get_name(); - } - // File must be deleted after this call - auto ret = ::access(tmpname.c_str(), R_OK); - auto serrno = errno; - CHECK(ret == -1); - CHECK(serrno == ENOENT); -} - -TEST_CASE("mmap") { - std::string tmpname; - { - auto raii_file = raii_file::mkstemp("/tmp//doctest-XXXXXXXX", - O_RDWR|O_CREAT|O_EXCL, 00600); - CHECK(raii_file.has_value()); - CHECK(raii_file->get_dir() == "/tmp"); - CHECK(access(raii_file->get_name().data(), R_OK) == 0); - tmpname = std::string{raii_file->get_name()}; - char payload[] = {'1', '2', '3'}; - CHECK(write(raii_file->get_fd(), payload, sizeof(payload)) == sizeof(payload)); - auto mmapped_file1 = raii_mmaped_file::mmap_shared(std::move(raii_file.value()), PROT_READ|PROT_WRITE); - CHECK(mmapped_file1.has_value()); - CHECK(!raii_file->is_valid()); - CHECK(mmapped_file1->get_size() == sizeof(payload)); - CHECK(memcmp(mmapped_file1->get_map(), payload, sizeof(payload)) == 0); - *(char *)mmapped_file1->get_map() = '2'; - auto mmapped_file2 = raii_mmaped_file::mmap_shared(tmpname.c_str(), O_RDONLY, PROT_READ); - CHECK(mmapped_file2.has_value()); - CHECK(mmapped_file2->get_size() == sizeof(payload)); - CHECK(memcmp(mmapped_file2->get_map(), payload, sizeof(payload)) != 0); - CHECK(memcmp(mmapped_file2->get_map(), mmapped_file1->get_map(), sizeof(payload)) == 0); - } - // File must be deleted after this call - auto ret = ::access(tmpname.c_str(), R_OK); - auto serrno = errno; - CHECK(ret == -1); - CHECK(serrno == ENOENT); -} - -} // TEST_SUITE - -} // namespace tests - -} // namespace rspamd::util diff --git a/src/libutil/cxx/locked_file.hxx b/src/libutil/cxx/locked_file.hxx deleted file mode 100644 index 507b02762..000000000 --- a/src/libutil/cxx/locked_file.hxx +++ /dev/null @@ -1,261 +0,0 @@ -/*- - * Copyright 2022 Vsevolod Stakhov - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef RSPAMD_LOCKED_FILE_HXX -#define RSPAMD_LOCKED_FILE_HXX -#pragma once - -#include "config.h" -#include "contrib/expected/expected.hpp" -#include -#include - -namespace rspamd::util { -/** - * A simple RAII object to contain a move only file descriptor - * A file is unlocked and closed when not needed - */ -struct raii_file { -public: - virtual ~raii_file() noexcept; - - static auto open(const char *fname, int flags) -> tl::expected; - static auto create(const char *fname, int flags, int perms) -> tl::expected; - static auto create_temp(const char *fname, int flags, int perms) -> tl::expected; - static auto mkstemp(const char *pattern, int flags, int perms) -> tl::expected; - - auto get_fd() const -> int { - return fd; - } - - auto get_stat() const -> const struct stat& { - return st; - }; - - auto get_size() const -> std::size_t { - return st.st_size; - }; - - auto get_name() const -> std::string_view { - return std::string_view{fname}; - } - - auto get_dir() const -> std::string_view { - auto sep_pos = fname.rfind(G_DIR_SEPARATOR); - - if (sep_pos == std::string::npos) { - return std::string_view{fname}; - } - - while (sep_pos >= 1 && fname[sep_pos - 1] == G_DIR_SEPARATOR) { - sep_pos --; - } - - return std::string_view{fname.c_str(), sep_pos}; - } - - auto get_extension() const -> std::string_view { - auto sep_pos = fname.rfind(G_DIR_SEPARATOR); - - if (sep_pos == std::string::npos) { - sep_pos = 0; - } - - auto filename = std::string_view{fname.c_str() + sep_pos}; - auto dot_pos = filename.find('.'); - - if (dot_pos == std::string::npos) { - return std::string_view{}; - } - else { - return std::string_view{filename.data() + dot_pos + 1, filename.size() - dot_pos - 1}; - } - } - - raii_file& operator=(raii_file &&other) noexcept { - std::swap(fd, other.fd); - std::swap(temp, other.temp); - std::swap(fname, other.fname); - std::swap(st, other.st); - - return *this; - } - - raii_file(raii_file &&other) noexcept { - *this = std::move(other); - } - - /** - * Prevent file from being deleted - * @return - */ - auto make_immortal() noexcept { - temp = false; - } - - /** - * Performs fstat on an opened file to refresh internal stat - * @return - */ - auto update_stat() noexcept -> bool; - - auto is_valid() noexcept -> bool { - return fd != -1; - } - - /* Do not allow copy/default ctor */ - const raii_file& operator=(const raii_file &other) = delete; - raii_file() = delete; - raii_file(const raii_file &other) = delete; -protected: - int fd = -1; - bool temp; - std::string fname; - struct stat st; - - explicit raii_file(const char *fname, int fd, bool temp) : fd(fd), temp(temp), fname(fname) {} -}; -/** - * A simple RAII object to contain a file descriptor with an flock wrap - * A file is unlocked and closed when not needed - */ -struct raii_locked_file final : public raii_file { -public: - ~raii_locked_file() noexcept override; - - static auto open(const char *fname, int flags) -> tl::expected { - auto locked = raii_file::open(fname, flags).and_then([](T &&file) { - return lock_raii_file(std::forward(file)); - }); - - return locked; - } - static auto create(const char *fname, int flags, int perms) -> tl::expected { - auto locked = raii_file::create(fname, flags, perms).and_then([](T &&file) { - return lock_raii_file(std::forward(file)); - }); - - return locked; - } - static auto create_temp(const char *fname, int flags, int perms) -> tl::expected { - auto locked = raii_file::create_temp(fname, flags, perms).and_then([](T &&file) { - return lock_raii_file(std::forward(file)); - }); - - return locked; - } - static auto mkstemp(const char *pattern, int flags, int perms) -> tl::expected { - auto locked = raii_file::mkstemp(pattern, flags, perms).and_then([](T &&file) { - return lock_raii_file(std::forward(file)); - }); - - return locked; - } - - raii_locked_file& operator=(raii_locked_file &&other) noexcept { - std::swap(fd, other.fd); - std::swap(temp, other.temp); - std::swap(fname, other.fname); - std::swap(st, other.st); - - return *this; - } - - /** - * Unlock a locked file and return back unlocked file transferring ownership. - * A locked file cannot be used after this method. - */ - auto unlock() -> raii_file; - - raii_locked_file(raii_locked_file &&other) noexcept : raii_file(static_cast(std::move(other))) {} - /* Do not allow copy/default ctor */ - const raii_locked_file& operator=(const raii_locked_file &other) = delete; - raii_locked_file() = delete; - raii_locked_file(const raii_locked_file &other) = delete; -private: - static auto lock_raii_file(raii_file &&unlocked) -> tl::expected; - raii_locked_file(raii_file &&other) noexcept : raii_file(std::move(other)) {} - explicit raii_locked_file(const char *fname, int fd, bool temp) : raii_file(fname, fd, temp) {} -}; - -/** - * A mmap wrapper on top of a locked file - */ -struct raii_mmaped_file final { - ~raii_mmaped_file(); - static auto mmap_shared(raii_file &&file, int flags) -> tl::expected; - static auto mmap_shared(const char *fname, int open_flags, int mmap_flags) -> tl::expected; - // Returns a constant pointer to the underlying map - auto get_map() const -> void* {return map;} - auto get_file() const -> const raii_file& { return file; } - // Passes the ownership of the mmaped memory to the callee - auto steal_map() -> std::tuple { - auto ret = std::make_tuple(this->map, file.get_stat().st_size); - this->map = nullptr; - return ret; - } - - auto get_size() const -> std::size_t { return file.get_stat().st_size; } - - raii_mmaped_file& operator=(raii_mmaped_file &&other) noexcept { - std::swap(map, other.map); - file = std::move(other.file); - - return *this; - } - - raii_mmaped_file(raii_mmaped_file &&other) noexcept; - - /* Do not allow copy/default ctor */ - const raii_mmaped_file& operator=(const raii_mmaped_file &other) = delete; - raii_mmaped_file() = delete; - raii_mmaped_file(const raii_mmaped_file &other) = delete; -private: - /* Is intended to be used with map_shared */ - explicit raii_mmaped_file(raii_file &&_file, void *_map); - raii_file file; - void *map = nullptr; -}; - -/** - * A helper to have a file to write that will be renamed to the - * target file if successful or deleted in the case of failure - */ -struct raii_file_sink final { - static auto create(const char *fname, int flags, int perms, const char *suffix = "new") - -> tl::expected; - auto write_output() -> bool; - ~raii_file_sink(); - auto get_fd() const -> int - { - return file.get_fd(); - } - - raii_file_sink(raii_file_sink &&other) noexcept; - /* Do not allow copy/default ctor */ - const raii_file_sink& operator=(const raii_file_sink &other) = delete; - raii_file_sink() = delete; - raii_file_sink(const raii_file_sink &other) = delete; -private: - explicit raii_file_sink(raii_locked_file &&_file, const char *_output, std::string &&_tmp_fname); - raii_locked_file file; - std::string output_fname; - std::string tmp_fname; - bool success; -}; - -} - -#endif //RSPAMD_LOCKED_FILE_HXX