Merged r4342 - r4359 from 1_1 branch.

git-svn-id: svn://svn.code.sf.net/p/tigervnc/code/trunk@4383 3789f03b-4d11-0410-bbf8-ca57d06f2519

7 files changed, 150 insertions, 71 deletions

diff --git a/BUILDING.txt b/BUILDING.txt
index 634ed55c..043ad2a4 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -37,7 +37,7 @@ Build Requirements
    * zlib v1.2 or later
    * OpenSSL v0.9.7 or later
 
--- If building VeNCrypt support:
+-- If building TLS support:
    * GnuTLS and its dependencies (libgcrypt, libtasn1, libgpg-error)
 
 
@@ -113,9 +113,9 @@ will clean both the Xvnc and vncviewer builds without destroying any of the
 build configuration or module dependencies.
 
 
-=========================
-Building VeNCrypt support
-=========================
+====================
+Building TLS support
+====================
 
 VeNCrypt (the TigerVNC security and authentication extensions) can be built
 with TLS support, which provides built-in encryption for VNC sessions.  This
@@ -268,7 +268,7 @@ Build Requirements
    C:\Program Files\Inno Setup 5) to the system or user PATH environment
    variable prior to building TigerVNC.
 
--- If building VeNCrypt support:
+-- If building TLS support:
    * GnuTLS and its dependencies (libgcrypt, libtasn1, libgpg-error)
 
 
@@ -331,7 +331,7 @@ WinVNC.
 Debug Build
 -----------
 
-Add "-DCMAKE_BUILD_TYPE=Debug" to the cmake command line.  Or, if building with
+Add "-DCMAKE_BUILD_TYPE=Debug" to the CMake command line.  Or, if building with
 NMake, remove "-DCMAKE_BUILD_TYPE=Release" (Debug builds are the default with
 NMake.)
 
@@ -344,46 +344,81 @@ To eliminate this dependency, add
 
   -DCMAKE_C_FLAGS=-static-libgcc -DCMAKE_CXX_FLAGS=-static-libgcc
 
-to the cmake command line.
+to the CMake command line.
 
 
-=========================
-Building VeNCrypt support
-=========================
+====================
+Building TLS support
+====================
 
 VeNCrypt (the TigerVNC security and authentication extensions) can be built
 with TLS support, which provides built-in encryption for VNC sessions.  This
-requires GnuTLS, which is not Microsoft-friendly.  There is generally no
-sane way to build GnuTLS and its dependencies using Visual C++.  Those with
-a lot of time on their hands can build the GnuTLS DLLs using MinGW (or download
-32-bit versions of these from the link below), generate Visual C++ import
-libraries from the DLLs, then link TigerVNC against the Visual C++ import
-libraries.  However, this creates a version of TigerVNC that depends on
-the GnuTLS DLLs.  The TigerVNC packaging system currently is not designed to
-handle DLL dependencies, so really the only way to build and package a
-self-contained, VeNCrypt-enabled version of TigerVNC for Windows is to use
-MinGW and the static GnuTLS libraries.  The use of MinGW means that only the
-viewer can be built, not the server.
+requires GnuTLS, which is not Microsoft-friendly.  This section describes the
+issues associated with building a Windows version of TigerVNC with TLS support
+and how to work around those issues.
+
+Building with MinGW
+-------------------
 
 An installer containing the GnuTLS header files, as well as static and dynamic
-link libraries for 32-bit MinGW, is available from the following site:
+link libraries for 32-bit MinGW, can be downloaded from the following site:
 
 http://josefsson.org/gnutls4win/
 
-Whether you choose to use the above installer or build GnuTLS from source,
-make sure that you install the libraries and headers into a pathname that
-doesn't contain spaces (the installer will try to install under
-c:\Program Files unless you tell it otherwise.)  If the GnuTLS include path
-contains spaces, then the MinGW resource compiler will barf.
+As of this writing, GnuTLS cannot be built cleanly with MinGW64 due to the fact
+that portions of the code assume an LP64 data model (Windows uses LLP64.)
+Thus, it is not possible at this time to produce a Win64 version of TigerVNC
+with TLS support.
+
+Whether you use the above installer or build GnuTLS from source, make sure that
+you install the libraries and headers into a pathname that doesn't contain
+spaces (the installer will try to install under c:\Program Files unless you
+tell it otherwise.)  If the GnuTLS include path contains spaces, then the MinGW
+resource compiler will barf when you try to build TigerVNC.
 
-You can manipulate the GNUTLS_INCLUDE_DIR and GNUTLS_LIBRARY cmake variables to
+You can manipulate the GNUTLS_INCLUDE_DIR and GNUTLS_LIBRARY CMake variables to
 specify the directory under which you installed GnuTLS.  For instance, adding
 
   -DGNUTLS_INCLUDE_DIR=/c/gnutls/include \
+    -DGNUTLS_LIBRARY=/c/gnutls/lib/libgnutls.dll.a
+
+to the CMake command line when using MinGW will cause TigerVNC to be linked
+against GnuTLS DLLs that are installed under c:\gnutls.
+
+Adding
+
+  -DGNUTLS_INCLUDE_DIR=/c/gnutls/include \
     -DGNUTLS_LIBRARY='/c/gnutls/lib/libgnutls.a;/c/gnutls/lib/libgcrypt.a;/c/gnutls/lib/libtasn1.a;/c/gnutls/lib/libgpg-error.a'
 
-to the cmake command line when using MinGW will cause TigerVNC to be statically
-linked against an installation of GnuTLS that resides under c:\gnutls.
+to the CMake command line will cause TigerVNC to be statically linked against
+GnuTLS libraries that are installed under c:\gnutls.
+
+Note that the use of MinGW means that only the TigerVNC viewer can be built,
+not the server.
+
+Visual C++
+----------
+
+There is generally no sane way to build GnuTLS and its dependencies using
+Visual C++.  Thus, it is necessary to either build the libraries with MinGW (or
+download 32-bit versions of these from the link above), generate Visual C++
+import libraries from the DLLs, then link TigerVNC against the Visual C++
+import libraries.
+
+In the instructions below, {gnutls_path} indicates the path under which GnuTLS
+is installed (Example: c:\Program Files\GnuTLS-2.10.1).
+
+To generate Visual C++ import libraries:
+
+cd {gnutls_path}\lib
+lib /def:..\bin\libgnutls-{version}.def /out:libgnutls.lib
+
+Now, you can add the following arguments to the CMake command line:
+
+  -DGNUTLS_INCLUDE_DIR={gnutls_path}\include \
+    -DGNUTLS_LIBRARY={gnutls_path}\lib\libgnutls.lib
+
+to build TigerVNC against the GnuTLS DLLs installed under {gnutls_path}.
 
 
 ===================
@@ -395,7 +430,7 @@ choosing (as opposed to creating an installer.)  To do this, add:
 
   -DCMAKE_INSTALL_PREFIX={install_directory}
 
-to the cmake command line.
+to the CMake command line.
 
 For example,
 
@@ -531,3 +566,7 @@ The installer package (TigerVNC[64].exe) will be located under
 package will be located in a subdirectory with the same name as the
 configuration you built (such as {build_directory}\Debug\ or
 {build_directory}\Release\).
+
+NOTE:  If TigerVNC is built with TLS support, then the build system will
+attempt to package the GnuTLS DLLs into the Windows installer.  It looks for
+these DLLs in a directory called "bin" one level up from GNUTLS_INCLUDE_DIR.
diff --git a/unix/vncserver b/unix/vncserver
index 0fbed190..7fe93bd1 100755
--- a/unix/vncserver
+++ b/unix/vncserver
@@ -160,17 +160,48 @@ if (!(-e $vncUserDir)) {
     }
 }
     
-# Make sure the user has a password.
-
-($z,$z,$mode) = stat("$vncUserDir/passwd");
-if (!(-e "$vncUserDir/passwd") || ($mode & 077)) {
-    warn "\nYou will require a password to access your desktops.\n\n";
-    system($exedir."vncpasswd -q $vncUserDir/passwd"); 
-    if (($? >> 8) != 0) {
-	exit 1;
+# Check whether VNC authentication is enabled, and if so, prompt the user to
+# create a VNC password if they don't already have one.
+
+$securityTypeArgSpecified = 0;
+$vncAuthEnabled = 0;
+$passwordArgSpecified = 0;
+
+for ($i = 0; $i < @ARGV; ++$i) {
+    # -SecurityTypes can be followed by a space or "="
+    my @splitargs = split('=', $ARGV[$i]);
+    if (@splitargs <= 1 && $i < @ARGV - 1) {
+        push(@splitargs, $ARGV[$i + 1]);
+    }
+    if (lc(@splitargs[0]) eq "-securitytypes") {
+        if (@splitargs > 1) {
+            $securityTypeArgSpecified = 1;
+        }
+        foreach $arg2 (split(',', @splitargs[1])) {
+            if (lc($arg2) eq "vncauth" || lc($arg2) eq "tlsvnc"
+             || lc($arg2) eq "x509vnc") {
+                $vncAuthEnabled = 1;
+            }
+        }
+    }
+    if ((lc(@splitargs[0]) eq "-password")
+     || (lc(@splitargs[0]) eq "-passwordfile"
+     || (lc(@splitargs[0]) eq "-rfbauth"))) {
+        $passwordArgSpecified = 1;
     }
 }
 
+if ((!$securityTypeArgSpecified || $vncAuthEnabled) && !$passwordArgSpecified) {
+    ($z,$z,$mode) = stat("$vncUserDir/passwd");
+    if (!(-e "$vncUserDir/passwd") || ($mode & 077)) {
+        warn "\nYou will require a password to access your desktops.\n\n";
+        system($exedir."vncpasswd -q $vncUserDir/passwd");
+        if (($? >> 8) != 0) {
+            exit 1;
+        }
+    }
+}
+ 
 # Find display number.
 
 if ((@ARGV > 0) && ($ARGV[0] =~ /^:(\d+)$/)) {
@@ -307,13 +338,10 @@ if (-e "/tmp/.X11-unix/X$displayNumber" ||
 $ENV{VNCDESKTOP}= $desktopName;
 
 if ($opt{'-fg'}) {
-    system("$vncUserDir/xstartup >> " . &quotedString($desktopLog) . " 2>&1");
-    if (kill 0, `cat $pidFile`) {
-        $opt{'-kill'} = ':'.$displayNumber;
-        &Kill();
-    }
+    close(STDIN);
+    system("($vncUserDir/xstartup; $0 -kill :$displayNumber) >> " . &quotedString($desktopLog) . " 2>&1");
 } else {
-    system("($vncUserDir/xstartup; $0 -kill :$displayNumber) >> " . &quotedString($desktopLog) . " 2>&1 &");
+    system("$vncUserDir/xstartup >> " . &quotedString($desktopLog) . " 2>&1 &");
 }
 
 exit;
diff --git a/unix/vncserver.man b/unix/vncserver.man
index 5127056e..c24225ae 100644
--- a/unix/vncserver.man
+++ b/unix/vncserver.man
@@ -1,4 +1,4 @@
-.TH vncserver 1 "15 Apr 2009" "TigerVNC" "Virtual Network Computing"
+.TH vncserver 1 "13 Mar 2011" "TigerVNC" "Virtual Network Computing"
 .SH NAME
 vncserver \- start or stop a VNC server
 .SH SYNOPSIS
@@ -108,11 +108,10 @@ path for Xvnc to use.
 
 .TP
 .B \-fg
-This version of vncserver will always launch Xvnc in such a way that, when the
-user exits the window manager in their VNC session, Xvnc will terminate.
-Specifying this option will additionally make Xvnc run as a foreground process,
-which means that it can be aborted with CTRL-C.  This may be necessary when
-launching TigerVNC from within certain grid computing environments.
+Runs Xvnc as a foreground process.  This has two effects: (1) The VNC server
+can be aborted with CTRL-C, and (2) the VNC server will exit as soon as the
+user logs out of the window manager in the VNC session.  This may be necessary
+when launching TigerVNC from within certain grid computing environments.
 
 .SH FILES
 Several VNC-related files are found in the directory $HOME/.vnc:
diff --git a/win/installer/CMakeLists.txt b/win/installer/CMakeLists.txt
index 91781303..56f52a21 100644
--- a/win/installer/CMakeLists.txt
+++ b/win/installer/CMakeLists.txt
@@ -21,8 +21,8 @@ if(BUILD_WINVNC)
   set(INST_DEPS ${INST_DEPS} winvnc4 wm_hooks vncconfig)
 endif()
 
-if(GNUTLS_FOUND AND NOT GNUTLS_STATIC)
-  set(INST_DEFS ${INST_DEFS})
+if(GNUTLS_FOUND)
+  set(INST_DEFS ${INST_DEFS} -DHAVE_GNUTLS)
 endif()
 
 configure_file(tigervnc.iss.in tigervnc.iss)
diff --git a/win/vncviewer/CConn.cxx b/win/vncviewer/CConn.cxx
index 3585966e..b067134a 100644
--- a/win/vncviewer/CConn.cxx
+++ b/win/vncviewer/CConn.cxx
@@ -80,7 +80,7 @@ CConn::CConn()
   : window(0), sameMachine(false), encodingChange(false), formatChange(false), 
     lastUsedEncoding_(encodingRaw), sock(0), sockEvent(CreateEvent(0, TRUE, FALSE, 0)), 
     reverseConnection(false), requestUpdate(false), firstUpdate(true),
-    isClosed_(false) {
+    pendingUpdate(false), isClosed_(false) {
 }
 
 CConn::~CConn() {
@@ -184,11 +184,6 @@ CConn::displayChanged() {
   calculateFullColourPF();
 }
 
-void
-CConn::paintCompleted() {
-  // A repaint message has just completed - request next update if necessary
-  requestNewUpdate();
-}
 
 bool
 CConn::sysCommand(WPARAM wParam, LPARAM lParam) {
@@ -362,10 +357,7 @@ CConn::blockCallback() {
 
     // Wait for socket data, or a message to process
     DWORD result = MsgWaitForMultipleObjects(1, &sockEvent.h, FALSE, INFINITE, QS_ALLINPUT);
-    if (result == WAIT_OBJECT_0) {
-      // - Network event notification.  Return control to I/O routine.
-      break;
-    } else if (result == WAIT_FAILED) {
+    if (result == WAIT_FAILED) {
       // - The wait operation failed - raise an exception
       throw rdr::SystemException("blockCallback wait error", GetLastError());
     }
@@ -380,6 +372,10 @@ CConn::blockCallback() {
       // ToAscii() internally).
       DispatchMessage(&msg);
     }
+
+    if (result == WAIT_OBJECT_0)
+      // - Network event notification.  Return control to I/O routine.
+      break;
   }
 
   // Before we return control to the InStream, reset the network event
@@ -522,6 +518,16 @@ CConn::showOptionsDialog() {
 
 
 void
+CConn::framebufferUpdateStart() {
+  if (!formatChange) {
+    requestUpdate = pendingUpdate = true;
+    requestNewUpdate();
+  } else
+    pendingUpdate = false;
+}
+
+
+void
 CConn::framebufferUpdateEnd() {
   if (debugDelay != 0) {
     vlog.debug("debug delay %d",(int)debugDelay);
@@ -571,12 +577,17 @@ CConn::framebufferUpdateEnd() {
     firstUpdate = false;
   }
 
-  if (options.autoSelect)
-    autoSelectFormatAndEncoding();
-
   // Always request the next update
   requestUpdate = true;
 
+  // A format change prevented us from sending this before the update,
+  // so make sure to send it now.
+  if (formatChange && !pendingUpdate)
+    requestNewUpdate();
+
+  if (options.autoSelect)
+    autoSelectFormatAndEncoding();
+
   // Check that at least part of the window has changed
   if (!GetUpdateRect(window->getHandle(), 0, FALSE)) {
     if (!(GetWindowLong(window->getHandle(), GWL_STYLE) & WS_MINIMIZE))
@@ -662,6 +673,10 @@ CConn::requestNewUpdate() {
   if (!requestUpdate) return;
 
   if (formatChange) {
+
+    /* Catch incorrect requestNewUpdate calls */
+    assert(pendingUpdate == false);
+
     // Select the required pixel format
     if (options.fullColour) {
       window->setPF(fullColourPF);
diff --git a/win/vncviewer/CConn.h b/win/vncviewer/CConn.h
index 62b131e1..d3b3b206 100644
--- a/win/vncviewer/CConn.h
+++ b/win/vncviewer/CConn.h
@@ -93,7 +93,7 @@ namespace rfb {
 
       // DesktopWindow::Callback interface
       void displayChanged();
-      void paintCompleted();
+      void paintCompleted() {}
       bool sysCommand(WPARAM wParam, LPARAM lParam);
       void closeWindow();
       void refreshMenu(bool enableSysCommands);
@@ -101,7 +101,7 @@ namespace rfb {
       // CConnection interface
       void setColourMapEntries(int firstColour, int nColours, rdr::U16* rgbs);
       void bell();
-      void framebufferUpdateStart() {}
+      void framebufferUpdateStart();
       void framebufferUpdateEnd();
       void setDesktopSize(int w, int h);
       void setExtendedDesktopSize(int reason, int result, int w, int h,
@@ -152,6 +152,7 @@ namespace rfb {
       bool reverseConnection;
       bool requestUpdate;
       bool firstUpdate;
+      bool pendingUpdate;
 
       // Debugging/logging
       std::list<Rect> debugRects;
diff --git a/win/vncviewer/DesktopWindow.cxx b/win/vncviewer/DesktopWindow.cxx
index d60ff8fe..f55eb72e 100644
--- a/win/vncviewer/DesktopWindow.cxx
+++ b/win/vncviewer/DesktopWindow.cxx
@@ -725,9 +725,6 @@ DesktopWindow::processFrameMessage(UINT msg, WPARAM wParam, LPARAM lParam) {
       }
 
       EndPaint(frameHandle, &ps);
-
-      // - Notify the callback that a paint message has finished processing
-      callback->paintCompleted();
     }
     return 0;