[Minor] Rework replxx to make in compatible with Rspamd again

contrib/DEPENDENCY_INFO.md

 | libottery     | ?       | Public Domain / CC0 | YES     | many changes       |
 | librdns       | ?       | BSD-2-Clause        | YES     |                    |
 | libucl        | ?       | BSD-2-Clause        | YES     |                    |
-| replxx        | 0.0.2   | BSD-2-Clause        | YES     | libicu usage       |
+| replxx        |  6d93360 | BSD-2-Clause       | YES     | libicu usage       |
 | lua-argparse  | 0.7.0   | MIT                 | NO      |                    |
 | lua-bit       | 1.0.2   | MIT                 | YES     | build fixes        |
 | lua-fun       | ?       | MIT                 | YES     | rspamd text        |
 | frozen        | 1.0.1   | Apache 2            | NO      |                    |
 | fmt           | 7.1.3   | MIT                 | NO      |                    |
 | doctest       | 2.4.5   | MIT                 | NO      |                    |
-| function2     | 4.1.0   | Boost               | NO      |                    |
+| function2     | 4.1.0   | Boost               | NO      |                    |

contrib/replxx/CMakeLists.txt

 	src/escape.cxx
 	src/history.cxx
 	src/replxx_impl.cxx
-	src/io.cxx
 	src/prompt.cxx
 	src/replxx.cxx
 	src/util.cxx
 	src/wcwidth.cpp
+	src/terminal.cxx
 	src/windows.cxx
 )
 

contrib/replxx/src/conversion.cxx

 #include <string>
 #include <cstring>
 #include <cctype>
-#include <locale.h>
+#include <clocale>
 
+#include "unicode/utf8.h"
 #include "conversion.hxx"
 
 #ifdef _WIN32
 ConversionResult copyString8to32(char32_t* dst, int dstSize, int& dstCount, const char* src) {
 	ConversionResult res = ConversionResult::conversionOK;
 	if ( ! locale::is8BitEncoding ) {
-		const UTF8* sourceStart = reinterpret_cast<const UTF8*>(src);
-		const UTF8* sourceEnd = sourceStart + strlen(src);
-		UTF32* targetStart = reinterpret_cast<UTF32*>(dst);
-		UTF32* targetEnd = targetStart + dstSize;
-
-		res = ConvertUTF8toUTF32(
-				&sourceStart, sourceEnd, &targetStart, targetEnd, lenientConversion);
+		auto sourceStart = reinterpret_cast<const unsigned char*>(src);
+		auto slen = strlen(src);
+		auto targetStart = reinterpret_cast<UChar32*>(dst);
+		int i = 0, j = 0;
+
+		while (i < slen && j < dstSize) {
+			UChar32 uc;
+			auto prev_i = i;
+			U8_NEXT (sourceStart, i, slen, uc);
+
+			if (uc <= 0) {
+				if (U8_IS_LEAD (sourceStart[prev_i])) {
+					auto lead_byte = sourceStart[prev_i];
+					auto trailing_bytes = (((uint8_t)(lead_byte)>=0xc2)+
+										   ((uint8_t)(lead_byte)>=0xe0)+
+										   ((uint8_t)(lead_byte)>=0xf0));
+
+					if (trailing_bytes + i > slen) {
+						return ConversionResult::sourceExhausted;
+					}
+				}
+
+				/* Replace with 0xFFFD */
+				uc = 0x0000FFFD;
+			}
+			targetStart[j++] = uc;
+		}
 
-		if (res == conversionOK) {
-			dstCount = static_cast<int>( targetStart - reinterpret_cast<UTF32*>( dst ) );
+		dstCount = j;
 
-			if (dstCount < dstSize) {
-				*targetStart = 0;
-			}
+		if (j < dstSize) {
+			targetStart[j] = 0;
 		}
 	} else {
 		for ( dstCount = 0; ( dstCount < dstSize ) && src[dstCount]; ++ dstCount ) {
 
 ConversionResult copyString8to32(char32_t* dst, int dstSize, int& dstCount, const char8_t* src) {
 	return copyString8to32(
-		dst, dstSize, dstCount, reinterpret_cast<const char*>(src)
+			dst, dstSize, dstCount, reinterpret_cast<const char*>(src)
 	);
 }
 
-int copyString32to8( char* dst, int dstSize, const char32_t* src, int srcSize ) {
-	int resCount( 0 );
+int copyString32to8(
+		char* dst, int dstSize, const char32_t* src, int srcSize
+) {
+	int resCount = 0;
+
 	if ( ! locale::is8BitEncoding ) {
-		const UTF32* sourceStart = reinterpret_cast<const UTF32*>(src);
-		const UTF32* sourceEnd = sourceStart + srcSize;
-		UTF8* targetStart = reinterpret_cast<UTF8*>(dst);
-		UTF8* targetEnd = targetStart + dstSize;
-
-		ConversionResult res = ConvertUTF32toUTF8(
-			&sourceStart, sourceEnd, &targetStart, targetEnd, lenientConversion
-		);
-
-		if ( res == conversionOK ) {
-			resCount = static_cast<int>( targetStart - reinterpret_cast<UTF8*>( dst ) );
-			if ( resCount < dstSize ) {
-				*targetStart = 0;
+		int j = 0;
+		UBool is_error = 0;
+
+		for (auto i = 0; i < srcSize; i ++) {
+			U8_APPEND ((uint8_t *)dst, j, dstSize, src[i], is_error);
+
+			if (is_error) {
+				break;
+			}
+		}
+
+		if (!is_error) {
+			resCount = j;
+
+			if (j < dstSize) {
+				dst[j] = '\0';
 			}
 		}
 	} else {
 			dst[i] = 0;
 		}
 	}
-	return ( resCount );
+
+	return resCount;
 }
 
 }

contrib/replxx/src/io.cxx → contrib/replxx/src/terminal.cxx

 #define write _write
 #define STDIN_FILENO 0
 
+#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
+static DWORD const ENABLE_VIRTUAL_TERMINAL_PROCESSING = 4;
+#endif
+
 #include "windows.hxx"
 
 #else /* _WIN32 */
 #include <sys/ioctl.h>
 #include <sys/select.h>
 #include <fcntl.h>
+#include <signal.h>
 
 #endif /* _WIN32 */
 
-#include "io.hxx"
+#include "terminal.hxx"
 #include "conversion.hxx"
 #include "escape.hxx"
 #include "replxx.hxx"
 
 }
 
+#ifndef _WIN32
+Terminal* _terminal_ = nullptr;
+static void WindowSizeChanged( int ) {
+	if ( ! _terminal_ ) {
+		return;
+	}
+	_terminal_->notify_event( Terminal::EVENT_TYPE::RESIZE );
+}
+#endif
+
+
 Terminal::Terminal( void )
 #ifdef _WIN32
 	: _consoleOut( INVALID_HANDLE_VALUE )
 	, _consoleIn( INVALID_HANDLE_VALUE )
-	, _oldMode()
+	, _origOutMode()
+	, _origInMode()
 	, _oldDisplayAttribute()
 	, _inputCodePage( GetConsoleCP() )
 	, _outputCodePage( GetConsoleOutputCP() )
 	, _interrupt( INVALID_HANDLE_VALUE )
 	, _events()
+	, _empty()
 #else
 	: _origTermios()
 	, _interrupt()
 #endif
-	, _rawMode( false ) {
+	, _rawMode( false )
+	, _utf8() {
 #ifdef _WIN32
 	_interrupt = CreateEvent( nullptr, true, false, TEXT( "replxx_interrupt_event" ) );
 #else
 }
 
 void Terminal::write32( char32_t const* text32, int len32 ) {
-	int len8 = 4 * len32 + 1;
-	unique_ptr<char[]> text8(new char[len8]);
-	int count8 = 0;
-
-	copyString32to8(text8.get(), len8, text32, len32, &count8);
-	int nWritten( 0 );
-#ifdef _WIN32
-	nWritten = win_write( text8.get(), count8 );
-#else
-	nWritten = write( 1, text8.get(), count8 );
-#endif
-	if ( nWritten != count8 ) {
-		throw std::runtime_error( "write failed" );
-	}
+	_utf8.assign( text32, len32 );
+	write8( _utf8.get(), _utf8.size() );
 	return;
 }
 
 void Terminal::write8( char const* data_, int size_ ) {
 #ifdef _WIN32
-	int nWritten( win_write( data_, size_ ) );
+	if ( ! _rawMode ) {
+		enable_out();
+	}
+	int nWritten( win_write( _consoleOut, _autoEscape, data_, size_ ) );
+	if ( ! _rawMode ) {
+		disable_out();
+	}
 #else
 	int nWritten( write( 1, data_, size_ ) );
 #endif
 }
 }
 
+void Terminal::enable_out( void ) {
+#ifdef _WIN32
+	_consoleOut = GetStdHandle( STD_OUTPUT_HANDLE );
+	SetConsoleOutputCP( 65001 );
+	GetConsoleMode( _consoleOut, &_origOutMode );
+	_autoEscape = SetConsoleMode( _consoleOut, _origOutMode | ENABLE_VIRTUAL_TERMINAL_PROCESSING ) != 0;
+#endif
+}
+
+void Terminal::disable_out( void ) {
+#ifdef _WIN32
+	SetConsoleMode( _consoleOut, _origOutMode );
+	SetConsoleOutputCP( _outputCodePage );
+	_consoleOut = INVALID_HANDLE_VALUE;
+	_autoEscape = false;
+#endif
+}
+
+void Terminal::enable_bracketed_paste( void ) {
+	static char const  BRACK_PASTE_INIT[] = "\033[?2004h";
+	write8( BRACK_PASTE_INIT, sizeof ( BRACK_PASTE_INIT ) - 1 );
+}
+
+void Terminal::disable_bracketed_paste( void ) {
+	static char const  BRACK_PASTE_DISABLE[] = "\033[?2004l";
+	write8( BRACK_PASTE_DISABLE, sizeof ( BRACK_PASTE_DISABLE ) - 1 );
+}
+
 int Terminal::enable_raw_mode( void ) {
 	if ( ! _rawMode ) {
 #ifdef _WIN32
 		_consoleIn = GetStdHandle( STD_INPUT_HANDLE );
-		_consoleOut = GetStdHandle( STD_OUTPUT_HANDLE );
 		SetConsoleCP( 65001 );
-		SetConsoleOutputCP( 65001 );
-		GetConsoleMode( _consoleIn, &_oldMode );
+		GetConsoleMode( _consoleIn, &_origInMode );
 		SetConsoleMode(
 			_consoleIn,
-			_oldMode & ~( ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT | ENABLE_PROCESSED_INPUT )
+			_origInMode & ~( ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT | ENABLE_PROCESSED_INPUT )
 		);
+		enable_out();
 #else
 		struct termios raw;
 
 		if ( tcsetattr(0, TCSADRAIN, &raw) < 0 ) {
 			return ( notty() );
 		}
+		_terminal_ = this;
 #endif
 		_rawMode = true;
 	}
-	return 0;
+	return ( 0 );
 }
 
 void Terminal::disable_raw_mode(void) {
 	if ( _rawMode ) {
 #ifdef _WIN32
-		SetConsoleMode( _consoleIn, _oldMode );
+		disable_out();
+		SetConsoleMode( _consoleIn, _origInMode );
 		SetConsoleCP( _inputCodePage );
-		SetConsoleOutputCP( _outputCodePage );
 		_consoleIn = INVALID_HANDLE_VALUE;
-		_consoleOut = INVALID_HANDLE_VALUE;
 #else
+		_terminal_ = nullptr;
 		if ( tcsetattr( 0, TCSADRAIN, &_origTermios ) == -1 ) {
 			return;
 		}
 			}
 		}
 #endif
-		if (rec.EventType != KEY_EVENT) {
+		if ( rec.EventType != KEY_EVENT ) {
 			continue;
 		}
-		// Windows provides for entry of characters that are not on your keyboard by
-		// sending the
-		// Unicode characters as a "key up" with virtual keycode 0x12 (VK_MENU ==
-		// Alt key) ...
+		// Windows provides for entry of characters that are not on your keyboard by sending the
+		// Unicode characters as a "key up" with virtual keycode 0x12 (VK_MENU == Alt key) ...
 		// accept these characters, otherwise only process characters on "key down"
-		if (!rec.Event.KeyEvent.bKeyDown &&
-				rec.Event.KeyEvent.wVirtualKeyCode != VK_MENU) {
+		if ( !rec.Event.KeyEvent.bKeyDown && ( rec.Event.KeyEvent.wVirtualKeyCode != VK_MENU ) ) {
 			continue;
 		}
 		modifierKeys = 0;
-		// AltGr is encoded as ( LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED ), so don't
-		// treat this
-		// combination as either CTRL or META we just turn off those two bits, so it
-		// is still
-		// possible to combine CTRL and/or META with an AltGr key by using
-		// right-Ctrl and/or
+		// AltGr is encoded as ( LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED ), so don't treat this
+		// combination as either CTRL or META we just turn off those two bits, so it is still
+		// possible to combine CTRL and/or META with an AltGr key by using right-Ctrl and/or
 		// left-Alt
-		if ((rec.Event.KeyEvent.dwControlKeyState &
-				 (LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED)) ==
-				(LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED)) {
-			rec.Event.KeyEvent.dwControlKeyState &=
-					~(LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED);
+		DWORD const AltGr( LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED );
+		if ( ( rec.Event.KeyEvent.dwControlKeyState & AltGr ) == AltGr ) {
+			rec.Event.KeyEvent.dwControlKeyState &= ~( LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED );
 		}
 		if ( rec.Event.KeyEvent.dwControlKeyState & ( RIGHT_CTRL_PRESSED | LEFT_CTRL_PRESSED ) ) {
 			modifierKeys |= Replxx::KEY::BASE_CONTROL;
 		if ( rec.Event.KeyEvent.dwControlKeyState & ( RIGHT_ALT_PRESSED | LEFT_ALT_PRESSED ) ) {
 			modifierKeys |= Replxx::KEY::BASE_META;
 		}
-		if (escSeen) {
+		if ( escSeen ) {
 			modifierKeys |= Replxx::KEY::BASE_META;
 		}
 		int key( rec.Event.KeyEvent.uChar.UnicodeChar );
 				key += 0x10000;
 			}
 			if ( is_control_code( key ) ) {
-				key += 0x40;
+				key = control_to_human( key );
 				modifierKeys |= Replxx::KEY::BASE_CONTROL;
 			}
 			key |= modifierKeys;
 					friendlyTextPtr = const_cast<char*>("DEL");
 				} else {
 					friendlyTextBuf[0] = '^';
-					friendlyTextBuf[1] = keyCopy + 0x40;
+					friendlyTextBuf[1] = control_to_human( keyCopy );
 					friendlyTextBuf[2] = 0;
 					friendlyTextPtr = friendlyTextBuf;
 				}
 
 	c = EscapeSequenceProcessing::doDispatch(c);
 	if ( is_control_code( c ) ) {
-		c = Replxx::KEY::control( c + 0x40 );
+		c = Replxx::KEY::control( control_to_human( c ) );
 	}
 #endif // #_WIN32
 	return ( c );
 #ifdef _WIN32
 	std::array<HANDLE,2> handles = { _consoleIn, _interrupt };
 	while ( true ) {
-		DWORD event( WaitForMultipleObjects( handles.size (), handles.data(), false, timeout_ > 0 ? timeout_ : INFINITE ) );
+		DWORD event( WaitForMultipleObjects( static_cast<DWORD>( handles.size() ), handles.data(), false, timeout_ > 0 ? timeout_ : INFINITE ) );
 		switch ( event ) {
 			case ( WAIT_OBJECT_0 + 0 ): {
 				// peek events that will be skipped
 			if ( data == 'm' ) {
 				return ( EVENT_TYPE::MESSAGE );
 			}
+			if ( data == 'r' ) {
+				return ( EVENT_TYPE::RESIZE );
+			}
 		}
 		if ( FD_ISSET( 0, &fdSet ) ) {
 			return ( EVENT_TYPE::KEY_PRESS );
 	_events.push_back( eventType_ );
 	SetEvent( _interrupt );
 #else
-	char data( eventType_ == EVENT_TYPE::KEY_PRESS ? 'k' : 'm' );
+	char data( ( eventType_ == EVENT_TYPE::KEY_PRESS ) ? 'k' : ( eventType_ == EVENT_TYPE::MESSAGE ? 'm' : 'r' ) );
 	static_cast<void>( write( _interrupt[1], &data, 1 ) == 1 );
 #endif
 }
  */
 void Terminal::clear_screen( CLEAR_SCREEN clearScreen_ ) {
 #ifdef _WIN32
+	if ( _autoEscape ) {
+#endif
+		if ( clearScreen_ == CLEAR_SCREEN::WHOLE ) {
+			char const clearCode[] = "\033c\033[H\033[2J\033[0m";
+			static_cast<void>( write(1, clearCode, sizeof ( clearCode ) - 1) >= 0 );
+		} else {
+			char const clearCode[] = "\033[J";
+			static_cast<void>( write(1, clearCode, sizeof ( clearCode ) - 1) >= 0 );
+		}
+		return;
+#ifdef _WIN32
+	}
 	COORD coord = { 0, 0 };
 	CONSOLE_SCREEN_BUFFER_INFO inf;
-	bool toEnd( clearScreen_ == CLEAR_SCREEN::TO_END );
 	HANDLE consoleOut( _consoleOut != INVALID_HANDLE_VALUE ? _consoleOut : GetStdHandle( STD_OUTPUT_HANDLE ) );
 	GetConsoleScreenBufferInfo( consoleOut, &inf );
-	if ( ! toEnd ) {
-		SetConsoleCursorPosition( consoleOut, coord );
-	} else {
+	if ( clearScreen_ == CLEAR_SCREEN::TO_END ) {
 		coord = inf.dwCursorPosition;
-	}
-	DWORD nWritten( 0 );
-	DWORD toWrite(
-		toEnd
-			? ( inf.dwSize.Y - inf.dwCursorPosition.Y ) * inf.dwSize.X - inf.dwCursorPosition.X
-			: inf.dwSize.X * inf.dwSize.Y
-	);
-	FillConsoleOutputCharacterA( consoleOut, ' ', toWrite, coord, &nWritten );
-#else
-	if ( clearScreen_ == CLEAR_SCREEN::WHOLE ) {
-		char const clearCode[] = "\033c\033[H\033[2J\033[0m";
-		static_cast<void>( write(1, clearCode, sizeof ( clearCode ) - 1) >= 0 );
+		DWORD nWritten( 0 );
+		SHORT height( inf.srWindow.Bottom - inf.srWindow.Top );
+		DWORD yPos( inf.dwCursorPosition.Y - inf.srWindow.Top );
+		DWORD toWrite( ( height + 1 - yPos ) * inf.dwSize.X - inf.dwCursorPosition.X );
+//		FillConsoleOutputCharacterA( consoleOut, ' ', toWrite, coord, &nWritten );
+		_empty.resize( toWrite - 1, ' ' );
+		WriteConsoleA( consoleOut, _empty.data(), toWrite - 1, &nWritten, nullptr );
 	} else {
-		char const clearCode[] = "\033[J";
-		static_cast<void>( write(1, clearCode, sizeof ( clearCode ) - 1) >= 0 );
+		COORD scrollTarget = { 0, -inf.dwSize.Y };
+		CHAR_INFO fill{ TEXT( ' ' ), inf.wAttributes };
+		SMALL_RECT scrollRect = { 0, 0, inf.dwSize.X, inf.dwSize.Y };
+		ScrollConsoleScreenBuffer( consoleOut, &scrollRect, nullptr, scrollTarget, &fill );
 	}
+	SetConsoleCursorPosition( consoleOut, coord );
 #endif
 }
 
 #endif
 }
 
+#ifdef _WIN32
+void Terminal::set_cursor_visible( bool visible_ ) {
+	CONSOLE_CURSOR_INFO     cursorInfo;
+	GetConsoleCursorInfo( _consoleOut, &cursorInfo );
+	cursorInfo.bVisible = visible_;
+	SetConsoleCursorInfo( _consoleOut, &cursorInfo );
+	return;
+}
+#else
+void Terminal::set_cursor_visible( bool ) {}
+#endif
+
 #ifndef _WIN32
 int Terminal::read_verbatim( char32_t* buffer_, int size_ ) {
 	int len( 0 );
 	::fcntl( STDIN_FILENO, F_SETFL, statusFlags );
 	return ( len );
 }
+
+int Terminal::install_window_change_handler( void ) {
+	struct sigaction sa;
+	sigemptyset(&sa.sa_mask);
+	sa.sa_flags = 0;
+	sa.sa_handler = &WindowSizeChanged;
+
+	if (sigaction(SIGWINCH, &sa, nullptr) == -1) {
+		return errno;
+	}
+	return 0;
+}
 #endif
 
 }

contrib/replxx/src/io.hxx → contrib/replxx/src/terminal.hxx

 #include <deque>
 
 #ifdef _WIN32
+#include <vector>
 #include <windows.h>
 #else
 #include <termios.h>
 #endif
 
+#include "utf8string.hxx"
+
 namespace replxx {
 
 class Terminal {
 	enum class EVENT_TYPE {
 		KEY_PRESS,
 		MESSAGE,
-		TIMEOUT
+		TIMEOUT,
+		RESIZE
 	};
 private:
 #ifdef _WIN32
 	HANDLE _consoleOut;
 	HANDLE _consoleIn;
-	DWORD _oldMode;
+	DWORD _origOutMode;
+	DWORD _origInMode;
+	bool _autoEscape;
 	WORD _oldDisplayAttribute;
 	UINT const _inputCodePage;
 	UINT const _outputCodePage;
 	HANDLE _interrupt;
 	typedef std::deque<EVENT_TYPE> events_t;
 	events_t _events;
+	std::vector<char> _empty;
 #else
 	struct termios _origTermios; /* in order to restore at exit */
 	int _interrupt[2];
 #endif
 	bool _rawMode; /* for destructor to check if restore is needed */
+	Utf8String _utf8;
 public:
 	enum class CLEAR_SCREEN {
 		WHOLE,
 	void write8( char const*, int );
 	int get_screen_columns(void);
 	int get_screen_rows(void);
+	void enable_bracketed_paste( void );
+	void disable_bracketed_paste( void );
 	int enable_raw_mode(void);
 	void disable_raw_mode(void);
 	char32_t read_char(void);
 	EVENT_TYPE wait_for_input( int long = 0 );
 	void notify_event( EVENT_TYPE );
 	void jump_cursor( int, int );
+	void set_cursor_visible( bool );
 #ifndef _WIN32
 	int read_verbatim( char32_t*, int );
+	int install_window_change_handler( void );
 #endif
+private:
+	void enable_out( void );
+	void disable_out( void );
 private:
 	Terminal( Terminal const& ) = delete;
 	Terminal& operator = ( Terminal const& ) = delete;