Add DataKeyMapper to correctly keep track of active data

Added a simple test that checks transported data correctness. Also
provides clean up method to TypedDataGenerator, even though it's not
called actually yet.

Change-Id: Icef69790732922b63a9874c9b1a6b44d4d682887

8 files changed, 371 insertions, 18 deletions

diff --git a/server/src/com/vaadin/server/communication/data/typed/DataKeyMapper.java b/server/src/com/vaadin/server/communication/data/typed/DataKeyMapper.java
new file mode 100644
index 0000000000..14f65d0292
--- /dev/null
+++ b/server/src/com/vaadin/server/communication/data/typed/DataKeyMapper.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2000-2014 Vaadin Ltd.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.vaadin.server.communication.data.typed;
+
+import java.io.Serializable;
+
+/**
+ * DataKeyMapper to map data objects to key strings.
+ * 
+ * @since
+ * @param <T>
+ *            data type
+ */
+public interface DataKeyMapper<T> extends Serializable {
+
+    /**
+     * Gets the key for data object. If no key exists beforehand, a new key is
+     * created.
+     * 
+     * @param dataObject
+     *            data object for key mappin.
+     * @return key for given data object
+     */
+    String key(T dataObject);
+
+    /**
+     * Gets the data object identified by given key.
+     * 
+     * @param key
+     *            key of a data object
+     * @return identified data object; <code>null</code> if invalid key
+     */
+    T get(String key);
+
+    /**
+     * Removes a data object from the key mapping. The key is also dropped.
+     * Dropped keys are not reused.
+     * 
+     * @param dataObject
+     *            dropped data object
+     */
+    void remove(T bean);
+
+    /**
+     * Removes all data objects from the key mapping. The keys are also dropped.
+     * Dropped keys are not reused.
+     */
+    void removeAll();
+}
diff --git a/server/src/com/vaadin/server/communication/data/typed/DataProvider.java b/server/src/com/vaadin/server/communication/data/typed/DataProvider.java
index 41cdb95dba..a4e937cbdb 100644
--- a/server/src/com/vaadin/server/communication/data/typed/DataProvider.java
+++ b/server/src/com/vaadin/server/communication/data/typed/DataProvider.java
@@ -15,11 +15,15 @@
  */
 package com.vaadin.server.communication.data.typed;
 
+import java.io.Serializable;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.LinkedHashSet;
+import java.util.Set;
 
 import com.vaadin.server.AbstractExtension;
 import com.vaadin.shared.data.DataProviderClientRpc;
+import com.vaadin.shared.data.DataProviderConstants;
 import com.vaadin.shared.data.DataRequestRpc;
 import com.vaadin.ui.AbstractComponent;
 
@@ -28,7 +32,13 @@ import elemental.json.JsonArray;
 import elemental.json.JsonObject;
 
 /**
- * DataProvider for Collection "container".
+ * DataProvider for Collections. This class takes care of sending data objects
+ * stored in a Collection from the server-side to the client-side. It uses
+ * {@link TypedDataGenerator}s to write a {@link JsonObject} representing each
+ * data object.
+ * <p>
+ * This is an implementation that does not provide any kind of lazy loading. All
+ * data is sent to the client-side on the initial client response.
  * 
  * @since
  */
@@ -57,6 +67,112 @@ public class DataProvider<T> extends AbstractExtension {
     }
 
     /**
+     * A class for handling currently active data and dropping data that is no
+     * longer needed. Data tracking is based on key string provided by
+     * {@link DataKeyMapper}.
+     * <p>
+     * When the {@link DataProvider} is pushing new data to the client-side via
+     * {@link DataProvider#pushData(long, Collection)},
+     * {@link #addActiveData(Collection)} and {@link #cleanUp(Collection)} are
+     * called with the same parameter. In the clean up method any dropped data
+     * objects that are not in the given collection will be cleaned up and
+     * {@link TypedDataGenerator#destroyData(Object)} will be called for them.
+     */
+    protected class ActiveDataHandler implements Serializable,
+            TypedDataGenerator<T> {
+
+        /**
+         * Set of key strings for currently active data objects
+         */
+        private final Set<String> activeData = new HashSet<String>();
+
+        /**
+         * Set of key strings for data objects dropped on the client. This set
+         * is used to clean up old data when it's no longer needed.
+         */
+        private final Set<String> droppedData = new HashSet<String>();
+
+        /**
+         * Adds given objects as currently active objects.
+         * 
+         * @param dataObjects
+         *            collection of new active data objects
+         */
+        public void addActiveData(Collection<T> dataObjects) {
+            for (T data : dataObjects) {
+                if (!activeData.contains(getKeyMapper().key(data))) {
+                    activeData.add(getKeyMapper().key(data));
+                }
+            }
+        }
+
+        /**
+         * Executes the data destruction for dropped data that is not sent to
+         * the client. This method takes most recently sent data objects in a
+         * collection. Doing the clean up like this prevents the
+         * {@link ActiveDataHandler} from creating new keys for rows that were
+         * dropped but got re-requested by the client-side. In the case of
+         * having all data at the client, the collection should be all the data
+         * in the back end.
+         * 
+         * @see DataProvider#pushData(long, Collection)
+         * @param dataObjects
+         *            collection of most recently sent data to the client
+         */
+        public void cleanUp(Collection<T> dataObjects) {
+            Collection<String> keys = new HashSet<String>();
+            for (T data : dataObjects) {
+                keys.add(getKeyMapper().key(data));
+            }
+
+            // Remove still active rows that were dropped by the client
+            droppedData.removeAll(keys);
+            // Do data clean up for object no longer needed.
+            dropData(droppedData);
+            droppedData.clear();
+        }
+
+        /**
+         * Marks a data object identified by given key string to be dropped.
+         * 
+         * @param key
+         *            key string
+         */
+        public void dropActiveData(String key) {
+            if (activeData.contains(key)) {
+                droppedData.add(key);
+            }
+        }
+
+        /**
+         * Returns the collection of all currently active data.
+         * 
+         * @return collection of active data objects
+         */
+        public Collection<T> getActiveData() {
+            HashSet<T> hashSet = new HashSet<T>();
+            for (String key : activeData) {
+                hashSet.add(getKeyMapper().get(key));
+            }
+            return hashSet;
+        }
+
+        @Override
+        public void generateData(T data, JsonObject jsonObject) {
+            // Write the key string for given data object
+            jsonObject.put(DataProviderConstants.KEY, getKeyMapper().key(data));
+        }
+
+        @Override
+        public void destroyData(T data) {
+            // Remove from active data set
+            activeData.remove(getKeyMapper().key(data));
+            // Drop the registered key
+            getKeyMapper().remove(data);
+        }
+    }
+
+    /**
      * Simple implementation of collection data provider communication. All data
      * is sent by server automatically and no data is requested by client.
      */
@@ -74,12 +190,14 @@ public class DataProvider<T> extends AbstractExtension {
         public void dropRows(JsonArray rowKeys) {
             // FIXME: What should I do with these?
         }
-
     }
 
     private Collection<T> data;
     private Collection<TypedDataGenerator<T>> generators = new LinkedHashSet<TypedDataGenerator<T>>();
     private DataProviderClientRpc rpc;
+    // TODO: Allow customizing the used key mapper
+    private DataKeyMapper<T> keyMapper = new KeyMapper<T>();
+    private ActiveDataHandler handler;
 
     /**
      * Creates a new DataProvider with the given Collection.
@@ -92,6 +210,8 @@ public class DataProvider<T> extends AbstractExtension {
 
         rpc = getRpcProxy(DataProviderClientRpc.class);
         registerRpc(createRpc());
+        handler = new ActiveDataHandler();
+        addDataGenerator(handler);
     }
 
     /**
@@ -105,7 +225,7 @@ public class DataProvider<T> extends AbstractExtension {
 
         if (initial) {
             getRpcProxy(DataProviderClientRpc.class).resetSize(data.size());
-            pushRows(0, data);
+            pushData(0, data);
         }
     }
 
@@ -130,22 +250,24 @@ public class DataProvider<T> extends AbstractExtension {
     }
 
     /**
-     * Sends given row range to the client.
+     * Sends given data collection to the client-side.
      * 
      * @param firstIndex
-     *            first index
-     * @param items
-     *            items to send as an iterable
+     *            first index of pushed data
+     * @param data
+     *            data objects to send as an iterable
      */
-    protected void pushRows(long firstIndex, Iterable<T> items) {
-        JsonArray data = Json.createArray();
+    protected void pushData(long firstIndex, Collection<T> data) {
+        JsonArray dataArray = Json.createArray();
 
         int i = 0;
-        for (T item : items) {
-            data.set(i++, getDataObject(item));
+        for (T item : data) {
+            dataArray.set(i++, getDataObject(item));
         }
 
-        rpc.setData(firstIndex, data);
+        rpc.setData(firstIndex, dataArray);
+        handler.addActiveData(data);
+        handler.cleanUp(data);
     }
 
     /**
@@ -167,6 +289,35 @@ public class DataProvider<T> extends AbstractExtension {
     }
 
     /**
+     * Drops data objects identified by given keys from memory. This will invoke
+     * {@link TypedDataGenerator#destroyData} for each of those objects.
+     * 
+     * @param droppedKeys
+     *            collection of dropped keys
+     */
+    private void dropData(Collection<String> droppedKeys) {
+        for (String key : droppedKeys) {
+            assert key != null : "Bookkeepping failure. Dropping a null key";
+
+            T data = getKeyMapper().get(key);
+            assert data != null : "Bookkeepping failure. No data object to match key";
+
+            for (TypedDataGenerator<T> g : generators) {
+                g.destroyData(data);
+            }
+        }
+    }
+
+    /**
+     * Gets the {@link DataKeyMapper} instance used by this {@link DataProvider}
+     * 
+     * @return key mapper
+     */
+    public DataKeyMapper<T> getKeyMapper() {
+        return keyMapper;
+    }
+
+    /**
      * Creates an instance of DataRequestRpc. By default it is
      * {@link DataRequestRpcImpl}.
      * 
diff --git a/server/src/com/vaadin/server/communication/data/typed/DefaultKeyMapper.java b/server/src/com/vaadin/server/communication/data/typed/DefaultKeyMapper.java
new file mode 100644
index 0000000000..d27dea1557
--- /dev/null
+++ b/server/src/com/vaadin/server/communication/data/typed/DefaultKeyMapper.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2000-2014 Vaadin Ltd.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.vaadin.server.communication.data.typed;
+
+/**
+ * Generic {@link DataKeyMapper} implementation based on
+ * {@link com.vaadin.server.KeyMapper}. Provides the interface on top of super
+ * class.
+ * 
+ * @since
+ * @param <T>
+ *            data type
+ */
+class KeyMapper<T> extends com.vaadin.server.KeyMapper<T> implements
+        DataKeyMapper<T> {
+}
diff --git a/server/src/com/vaadin/server/communication/data/typed/TypedDataGenerator.java b/server/src/com/vaadin/server/communication/data/typed/TypedDataGenerator.java
index ed2e321df8..6a4b9f7da7 100644
--- a/server/src/com/vaadin/server/communication/data/typed/TypedDataGenerator.java
+++ b/server/src/com/vaadin/server/communication/data/typed/TypedDataGenerator.java
@@ -27,8 +27,8 @@ import elemental.json.JsonObject;
 public interface TypedDataGenerator<T> extends Serializable {
 
     /**
-     * Adds data for given object to JsonObject. This JsonObject will be sent to
-     * client-side DataSource.
+     * Adds data for given object to {@link JsonObject}. This JsonObject will be
+     * sent to client-side DataSource.
      * 
      * @param data
      *            data object
@@ -36,4 +36,14 @@ public interface TypedDataGenerator<T> extends Serializable {
      *            json object being sent to the client
      */
     void generateData(T data, JsonObject jsonObject);
+
+    /**
+     * Informs the {@link TypedDataGenerator} that given data has been dropped
+     * and is no longer needed. This method should clean up any unneeded
+     * information stored for this data.
+     * 
+     * @param data
+     *            dropped data
+     */
+    public void destroyData(T data);
 }
diff --git a/shared/src/com/vaadin/shared/data/DataProviderConstants.java b/shared/src/com/vaadin/shared/data/DataProviderConstants.java
new file mode 100644
index 0000000000..16a25ed32e
--- /dev/null
+++ b/shared/src/com/vaadin/shared/data/DataProviderConstants.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2000-2014 Vaadin Ltd.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.vaadin.shared.data;
+
+import java.io.Serializable;
+
+/**
+ * Set of contants used by DataProvider. These are commonly used JsonObject keys
+ * which are considered to be reserved for internal use.
+ * 
+ * @since
+ */
+public final class DataProviderConstants implements Serializable {
+    public static final String KEY = "k";
+}
diff --git a/shared/src/com/vaadin/shared/ui/grid/GridState.java b/shared/src/com/vaadin/shared/ui/grid/GridState.java
index 54ccc78daa..4bc1ffcff4 100644
--- a/shared/src/com/vaadin/shared/ui/grid/GridState.java
+++ b/shared/src/com/vaadin/shared/ui/grid/GridState.java
@@ -20,6 +20,7 @@ import java.util.ArrayList;
 import java.util.List;
 
 import com.vaadin.shared.annotations.DelegateToWidget;
+import com.vaadin.shared.data.DataProviderConstants;
 import com.vaadin.shared.data.sort.SortDirection;
 import com.vaadin.shared.ui.TabIndexState;
 
@@ -86,7 +87,7 @@ public class GridState extends TabIndexState {
      * 
      * @see com.vaadin.shared.data.DataProviderRpc#setRowData(int, String)
      */
-    public static final String JSONKEY_ROWKEY = "k";
+    public static final String JSONKEY_ROWKEY = DataProviderConstants.KEY;
 
     /**
      * The key in which a row's generated style can be found
diff --git a/uitest/src/com/vaadin/tests/dataprovider/DummyDataProviderTest.java b/uitest/src/com/vaadin/tests/dataprovider/DummyDataProviderTest.java
new file mode 100644
index 0000000000..9771f7b0f6
--- /dev/null
+++ b/uitest/src/com/vaadin/tests/dataprovider/DummyDataProviderTest.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2000-2014 Vaadin Ltd.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.vaadin.tests.dataprovider;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.List;
+import java.util.Random;
+
+import org.junit.Test;
+import org.openqa.selenium.By;
+import org.openqa.selenium.WebElement;
+
+import com.vaadin.shared.data.DataProviderConstants;
+import com.vaadin.tests.fieldgroup.ComplexPerson;
+import com.vaadin.tests.tb3.SingleBrowserTest;
+
+import elemental.json.Json;
+import elemental.json.JsonObject;
+
+public class DummyDataProviderTest extends SingleBrowserTest {
+
+    @Override
+    protected Class<?> getUIClass() {
+        return DummyDataProviderUI.class;
+    }
+
+    @Test
+    public void testVerifyJsonContent() {
+        Random r = new Random(DummyDataProviderUI.RANDOM_SEED);
+        List<ComplexPerson> persons = DummyDataProviderUI.createPersons(
+                DummyDataProviderUI.PERSON_COUNT, r);
+
+        openTestURL();
+
+        int size = DummyDataProviderUI.PERSON_COUNT + 1;
+        List<WebElement> labels = findElements(By.className("v-label"));
+
+        assertEquals("Label count did not match person count", size,
+                labels.size());
+
+        List<WebElement> personData = labels.subList(1, size);
+
+        int key = 0;
+        for (WebElement e : personData) {
+            JsonObject j = Json.createObject();
+            ComplexPerson p = persons.get(key);
+            j.put(DataProviderConstants.KEY, "" + (++key));
+            j.put("name", p.getLastName() + ", " + p.getFirstName());
+            assertEquals("Json did not match.", j.toJson(), e.getText());
+        }
+    }
+}
diff --git a/uitest/src/com/vaadin/tests/dataprovider/DummyDataProviderUI.java b/uitest/src/com/vaadin/tests/dataprovider/DummyDataProviderUI.java
index 62aa8c2241..26a0277274 100644
--- a/uitest/src/com/vaadin/tests/dataprovider/DummyDataProviderUI.java
+++ b/uitest/src/com/vaadin/tests/dataprovider/DummyDataProviderUI.java
@@ -56,6 +56,10 @@ public class DummyDataProviderUI extends AbstractTestUI {
                                     + data.getFirstName();
                             dataObject.put("name", name);
                         }
+
+                        @Override
+                        public void destroyData(ComplexPerson data) {
+                        }
                     });
         }
 
@@ -72,8 +76,10 @@ public class DummyDataProviderUI extends AbstractTestUI {
         }
     }
 
-    private Random r = new Random(1337);
-    private List<ComplexPerson> persons = getPersons(20);
+    public static final int RANDOM_SEED = 1337;
+    public static final int PERSON_COUNT = 20;
+    private Random r = new Random(RANDOM_SEED);
+    private List<ComplexPerson> persons = createPersons(PERSON_COUNT, r);
     private DummyDataComponent dummy;
 
     @Override
@@ -98,7 +104,7 @@ public class DummyDataProviderUI extends AbstractTestUI {
         addComponent(dummy);
     }
 
-    private List<ComplexPerson> getPersons(int count) {
+    public static List<ComplexPerson> createPersons(int count, Random r) {
         List<ComplexPerson> c = new ArrayList<ComplexPerson>();
         for (int i = 0; i < count; ++i) {
             c.add(ComplexPerson.create(r));