[Feature] Print stack trace on crash

5 files changed, 124 insertions, 0 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6526996a3..a3fe5bc2c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -64,6 +64,7 @@ OPTION(ENABLE_COVERAGE     "Build rspamd with code coverage options [default: OF
 OPTION(ENABLE_FULL_DEBUG   "Build rspamd with all possible debug [default: OFF]" OFF)
 OPTION(ENABLE_UTILS        "Build rspamd internal utils [default: OFF]" OFF)
 OPTION(ENABLE_TORCH        "Install torch7 with Rspamd [default: ON]" ON)
+OPTION(ENABLE_LIBUNWIND    "Use libunwind to print crash traces [default: OFF]" OFF)
 
 INCLUDE(FindArch.cmake)
 TARGET_ARCHITECTURE(ARCH)
@@ -630,6 +631,12 @@ IF(ENABLE_JEMALLOC MATCHES "ON")
 	SET(WITH_JEMALLOC "1")
 	LIST(APPEND RSPAMD_REQUIRED_LIBRARIES "-lpthread")
 ENDIF()
+
+IF(ENABLE_LIBUNWIND MATCHES "ON")
+	ProcessPackage(LIBUNWIND LIBRARY unwind INCLUDE libunwind.h INCLUDE_SUFFIXES include/libunwind
+			ROOT ${LIBUNWIND_ROOT_DIR} MODULES libunwind)
+	SET(WITH_LIBUNWIND "1")
+ENDIF()
 ProcessPackage(GTHREAD2 LIBRARY gthread-2.0 INCLUDE glib.h
 	INCLUDE_SUFFIXES include/glib include/glib-2.0
 	ROOT ${GLIB_ROOT_DIR} MODULES gthread-2.0>=2.28)
@@ -984,6 +991,7 @@ LIST(APPEND CMAKE_REQUIRED_INCLUDES "${LIBSSL_INCLUDE}")
 CHECK_SYMBOL_EXISTS(SSL_set_tlsext_host_name "openssl/ssl.h" HAVE_SSL_TLSEXT_HOSTNAME)
 CHECK_SYMBOL_EXISTS(dirfd "sys/types.h;unistd.h;dirent.h" HAVE_DIRFD)
 CHECK_SYMBOL_EXISTS(fpathconf "sys/types.h;unistd.h" HAVE_FPATHCONF)
+CHECK_SYMBOL_EXISTS(sigaltstack "signal.h" HAVE_SIGALTSTACK)
 
 IF(ENABLE_PCRE2 MATCHES "ON")
 	IF(HAVE_PCRE_JIT)
diff --git a/config.h.in b/config.h.in
index 007586b8f..5b22d3df9 100644
--- a/config.h.in
+++ b/config.h.in
@@ -83,6 +83,7 @@
 #cmakedefine HAVE_SETITIMER      1
 #cmakedefine HAVE_SETPROCTITLE   1
 #cmakedefine HAVE_SETSIG         1
+#cmakedefine HAVE_SIGALTSTACK    1
 #cmakedefine HAVE_SIGINFO_H      1
 #cmakedefine HAVE_SOCK_SEQPACKET 1
 #cmakedefine HAVE_SSL_TLSEXT_HOSTNAME 1
@@ -141,6 +142,7 @@
 #cmakedefine WITH_SQLITE         1
 #cmakedefine WITH_SYSTEM_HIREDIS 1
 #cmakedefine WITH_TORCH          1
+#cmakedefine WITH_LIBUNWIND      1
 
 #cmakedefine DISABLE_PTHREAD_MUTEX 1
 
diff --git a/src/libserver/worker_util.c b/src/libserver/worker_util.c
index c60273dea..a1fa4048c 100644
--- a/src/libserver/worker_util.c
+++ b/src/libserver/worker_util.c
@@ -44,6 +44,12 @@
 #endif
 #include "zlib.h"
 
+#ifdef WITH_LIBUNWIND
+#define UNW_LOCAL_ONLY 1
+#include <libunwind.h>
+#define UNWIND_BACKTRACE_DEPTH 256
+#endif
+
 static void rspamd_worker_ignore_signal (int signo);
 /**
  * Return worker's control structure by its type
@@ -924,4 +930,106 @@ rspamd_worker_init_monitored (struct rspamd_worker *worker,
 	rspamd_monitored_ctx_config (worker->srv->cfg->monitored_ctx,
 			worker->srv->cfg, ev_base, resolver->r,
 			rspamd_worker_monitored_on_change, worker);
+}
+
+#ifdef HAVE_SA_SIGINFO
+
+#ifdef WITH_LIBUNWIND
+static void
+rspamd_print_crash (ucontext_t *uap)
+{
+	unw_cursor_t cursor;
+	unw_word_t ip, off;
+	guint level;
+	gint ret;
+
+	if ((ret = unw_init_local (&cursor, uap)) != 0) {
+		msg_err ("unw_init_local: %d", ret);
+
+		return;
+	}
+
+	level = 0;
+	ret = 0;
+
+	for (;;) {
+		char name[128];
+
+		if (level >= UNWIND_BACKTRACE_DEPTH) {
+			break;
+		}
+
+		unw_get_reg (&cursor, UNW_REG_IP, &ip);
+		ret = unw_get_proc_name(&cursor, name, sizeof (name), &off);
+
+		if (ret == 0) {
+			msg_err ("%d: %p: %s()+0x%xl",
+				level, ip, name, (uintptr_t)off);
+		} else {
+			msg_err ("%d: %p: <unknown>", level, ip);
+		}
+
+		level++;
+		ret = unw_step (&cursor);
+
+		if (ret <= 0) {
+			break;
+		}
+	}
+
+	if (ret < 0) {
+		msg_err ("unw_step_ptr: %d", ret);
+	}
+}
+#endif
+
+static void
+rspamd_crash_sig_handler (int sig, siginfo_t *info, void *ctx)
+{
+	struct sigaction sa;
+	ucontext_t *uap = ctx;
+
+	msg_err ("caught fatal signal %d(%s), "
+			 "pid: %P, trace: ",
+			sig, strsignal (sig), getpid ());
+	(void)uap;
+#ifdef WITH_LIBUNWIND
+	rspamd_print_crash (uap);
+#endif
+
+	/*
+	 * Invoke signal with the default handler
+	 */
+	sigemptyset (&sa.sa_mask);
+	sa.sa_handler = SIG_DFL;
+	sa.sa_flags = 0;
+	sigaction (sig, &sa, NULL);
+	kill (getpid (), sig);
+}
+#endif
+
+void
+rspamd_set_crash_handler (struct rspamd_main *main)
+{
+#ifdef HAVE_SA_SIGINFO
+	struct sigaction sa;
+
+#ifdef HAVE_SIGALTSTACK
+	stack_t ss;
+
+	/* Allocate special stack, NOT freed at the end so far */
+	ss.ss_size = MAX (SIGSTKSZ, 8192 * 4);
+	ss.ss_sp = g_malloc0 (ss.ss_size);
+	sigaltstack (&ss, NULL);
+#endif
+
+	sigemptyset(&sa.sa_mask);
+	sa.sa_sigaction = &rspamd_crash_sig_handler;
+	sa.sa_flags = SA_RESTART | SA_SIGINFO | SA_ONSTACK;
+	sigaction(SIGSEGV, &sa, NULL);
+	sigaction(SIGBUS, &sa, NULL);
+	sigaction(SIGABRT, &sa, NULL);
+	sigaction(SIGFPE, &sa, NULL);
+	sigaction(SIGSYS, &sa, NULL);
+#endif
 }
\ No newline at end of file
diff --git a/src/libserver/worker_util.h b/src/libserver/worker_util.h
index 2e3fd4458..dbcc8f8a2 100644
--- a/src/libserver/worker_util.h
+++ b/src/libserver/worker_util.h
@@ -188,6 +188,11 @@ struct rspamd_worker *rspamd_fork_worker (struct rspamd_main *,
 		struct rspamd_worker_conf *, guint idx, struct event_base *ev_base);
 
 /**
+ * Sets crash signals handlers if compiled with libunwind
+ */
+void rspamd_set_crash_handler (struct rspamd_main *);
+
+/**
  * Initialise the main monitoring worker
  * @param worker
  * @param ev_base
diff --git a/src/rspamd.c b/src/rspamd.c
index d7543643e..7c5450521 100644
--- a/src/rspamd.c
+++ b/src/rspamd.c
@@ -1242,6 +1242,7 @@ main (gint argc, gchar **argv, gchar **env)
 	}
 
 	type = g_quark_from_static_string ("main");
+	rspamd_set_crash_handler (rspamd_main);
 
 	/* First set logger to console logger */
 	rspamd_main->cfg->log_type = RSPAMD_LOG_CONSOLE;