From: Marc Englund Date: Fri, 11 Sep 2009 06:55:22 +0000 (+0000) Subject: App Engine datastore cleanup, fixes #3320 X-Git-Tag: 6.7.0.beta1~2485 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=f587260938476fa09c95b9422f132ffe54ba32f5;p=vaadin-framework.git App Engine datastore cleanup, fixes #3320 Improved javadoc. Improved logging. svn changeset:8737/svn branch:6.1 --- diff --git a/src/com/vaadin/terminal/gwt/server/GAEApplicationServlet.java b/src/com/vaadin/terminal/gwt/server/GAEApplicationServlet.java index daab5fbbc4..d8f4363ae3 100644 --- a/src/com/vaadin/terminal/gwt/server/GAEApplicationServlet.java +++ b/src/com/vaadin/terminal/gwt/server/GAEApplicationServlet.java @@ -6,7 +6,12 @@ import java.io.IOException; import java.io.NotSerializableException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; import java.util.Date; +import java.util.List; +import java.util.logging.Logger; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; @@ -20,16 +25,78 @@ import com.google.appengine.api.datastore.Entity; import com.google.appengine.api.datastore.EntityNotFoundException; import com.google.appengine.api.datastore.Key; import com.google.appengine.api.datastore.KeyFactory; +import com.google.appengine.api.datastore.PreparedQuery; +import com.google.appengine.api.datastore.Query; +import com.google.appengine.api.datastore.FetchOptions.Builder; +import com.google.appengine.api.datastore.Query.FilterOperator; import com.google.appengine.api.memcache.Expiration; import com.google.appengine.api.memcache.MemcacheService; import com.google.appengine.api.memcache.MemcacheServiceFactory; import com.google.apphosting.api.DeadlineExceededException; import com.vaadin.service.ApplicationContext; +/** + * ApplicationServlet to be used when deploying to Google App Engine, in + * web.xml: + * + * 
+ *      <servlet>
+ *              <servlet-name>HelloWorld</servlet-name>
+ *              <servlet-class>com.vaadin.terminal.gwt.server.GAEApplicationServlet</servlet-class>
+ *              <init-param>
+ *                      <param-name>application</param-name>
+ *                      <param-value>com.vaadin.demo.HelloWorld</param-value>
+ *              </init-param>
+ *      </servlet>
+ *
+ * + * Session support must be enabled in appengine-web.xml: + * + * 
+ *      <sessions-enabled>true</sessions-enabled>
+ *
+ * + * Appengine datastore cleanup can be invoked by calling one of the applications + * with an additional path "/CLEAN". This can be set up as a cron-job in + * cron.xml (see appengine documentation for more information): + * + * 
+ * <cronentries>
+ *   <cron>
+ *     <url>/HelloWorld/CLEAN</url>
+ *     <description>Clean up sessions</description>
+ *     <schedule>every 2 hours</schedule>
+ *   </cron>
+ * </cronentries>
+ *
+ * + * It is recommended (but not mandatory) to extract themes and widgetsets and + * have App Engine server these statically. Extract VAADIN folder (and it's + * contents) 'next to' the WEB-INF folder, and add the following to + * appengine-web.xml: + * + * 
+ *      <static-files>
+ *           <include path="/VAADIN/**" />
+ *      </static-files>
+ *
+ * + * Additional limitations: + * + */ public class GAEApplicationServlet extends ApplicationServlet { private static final long serialVersionUID = 2179597952818898526L; + private static final Logger log = Logger + .getLogger(GAEApplicationServlet.class.getName()); + // memcache mutex is MUTEX_BASE + sessio id private static final String MUTEX_BASE = "_vmutex"; @@ -48,6 +115,15 @@ public class GAEApplicationServlet extends ApplicationServlet { private static final String PROPERTY_EXPIRES = "expires"; private static final String PROPERTY_DATA = "data"; + // path used for cleanup + private static final String CLEANUP_PATH = "/CLEAN"; + // max entities to clean at once + private static final int CLEANUP_LIMIT = 200; + // appengine session kind + private static final String APPENGINE_SESSION_KIND = "_ah_SESSION"; + // appengine session expires-parameter + private static final String PROPERTY_APPENGINE_EXPIRES = "_expires"; + protected void sendDeadlineExceededNotification(HttpServletRequest request, HttpServletResponse response) throws IOException { criticalNotification( @@ -69,10 +145,26 @@ public class GAEApplicationServlet extends ApplicationServlet { + "?restartApplication"); } + protected void sendCriticalErrorNotification(HttpServletRequest request, + HttpServletResponse response) throws IOException { + criticalNotification( + request, + response, + "Critical error", + "I'm sorry, but there seems to be a serious problem, please contact the administrator. And please take note of any unsaved data...", + "", getApplicationUrl(request).toString() + + "?restartApplication"); + } + @Override protected void service(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + if (isCleanupRequest(request)) { + cleanDatastore(); + return; + } + RequestType requestType = getRequestType(request); if (requestType == RequestType.STATIC_FILE) { @@ -117,9 +209,9 @@ public class GAEApplicationServlet extends ApplicationServlet { try { Thread.sleep(RETRY_AFTER_MILLISECONDS); } catch (InterruptedException e) { - System.err - .println("Thread.sleep() interrupted while waiting for lock. Trying again."); - e.printStackTrace(System.err); + log + .info("Thread.sleep() interrupted while waiting for lock. Trying again. " + + e); } } @@ -162,13 +254,16 @@ public class GAEApplicationServlet extends ApplicationServlet { ds.put(entity); } catch (DeadlineExceededException e) { - System.err.println("DeadlineExceeded for " + session.getId()); + log.severe("DeadlineExceeded for " + session.getId()); sendDeadlineExceededNotification(request, response); } catch (NotSerializableException e) { // TODO this notification is usually not shown - should we redirect // in some other way - can we? sendNotSerializableNotification(request, response); - e.printStackTrace(System.err); + log.severe("NotSerializableException: " + getStackTraceAsString(e)); + } catch (Exception e) { + sendCriticalErrorNotification(request, response); + log.severe(e + ": " + getStackTraceAsString(e)); } finally { // "Next, please!" if (locked) { @@ -212,17 +307,13 @@ public class GAEApplicationServlet extends ApplicationServlet { session.setAttribute(WebApplicationContext.class.getName(), applicationContext); } catch (IOException e) { - System.err - .println("Could not de-serialize ApplicationContext for " - + session.getId() - + " A new one will be created."); - e.printStackTrace(); + log.warning("Could not de-serialize ApplicationContext for " + + session.getId() + " A new one will be created. " + + getStackTraceAsString(e)); } catch (ClassNotFoundException e) { - System.err - .println("Could not de-serialize ApplicationContext for " - + session.getId() - + " A new one will be created."); - e.printStackTrace(); + log.warning("Could not de-serialize ApplicationContext for " + + session.getId() + " A new one will be created. " + + getStackTraceAsString(e)); } } // will create new context if the above did not @@ -230,6 +321,14 @@ public class GAEApplicationServlet extends ApplicationServlet { } + private boolean isCleanupRequest(HttpServletRequest request) { + String path = getRequestPathInfo(request); + if (path != null && path.equals(CLEANUP_PATH)) { + return true; + } + return false; + } + /** * Removes the ApplicationContext from the session in order to minimize the * data serialized to datastore and memcache. @@ -243,4 +342,69 @@ public class GAEApplicationServlet extends ApplicationServlet { } } + /** + * This will look at the timestamp and delete expired persisted Vaadin and + * appengine sessions from the datastore. + * + * TODO Possible improvements include: 1. Use transactions (requires entity + * groups - overkill?) 2. Delete one-at-a-time, catch possible exception, + * continue w/ next. + */ + private void cleanDatastore() { + long expire = new Date().getTime(); + try { + DatastoreService ds = DatastoreServiceFactory.getDatastoreService(); + // Vaadin stuff first + { + Query q = new Query(AC_BASE); + q.setKeysOnly(); + + q.addFilter(PROPERTY_EXPIRES, + FilterOperator.LESS_THAN_OR_EQUAL, expire); + PreparedQuery pq = ds.prepare(q); + List entities = pq.asList(Builder + .withLimit(CLEANUP_LIMIT)); + if (entities != null) { + log.info("Vaadin cleanup deleting " + entities.size() + + " expired Vaadin sessions."); + List keys = new ArrayList(); + for (Entity e : entities) { + keys.add(e.getKey()); + } + ds.delete(keys); + } + } + // Also cleanup GAE sessions + { + Query q = new Query(APPENGINE_SESSION_KIND); + q.setKeysOnly(); + q.addFilter(PROPERTY_APPENGINE_EXPIRES, + FilterOperator.LESS_THAN_OR_EQUAL, expire); + PreparedQuery pq = ds.prepare(q); + List entities = pq.asList(Builder + .withLimit(CLEANUP_LIMIT)); + if (entities != null) { + log.info("Vaadin cleanup deleting " + entities.size() + + " expired appengine sessions."); + List keys = new ArrayList(); + for (Entity e : entities) { + keys.add(e.getKey()); + } + ds.delete(keys); + } + } + } catch (Exception e) { + log + .warning("Exception while cleaning: " + + getStackTraceAsString(e)); + } + } + + private String getStackTraceAsString(Throwable t) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + t.printStackTrace(pw); + return sw.toString(); + } + }