DHT: Drop leading hash digits from row keys

Originally I put the first two digits of the object SHA-1 into the
start of a row key to try and spread the load of objects around a DHT
service. Unfortunately this tends to not work as well as I had hoped.

Servers reading a repository need to contact every node in a DHT
cluster if the cluster tries to evenly distribute the object rows.
This is a lot of connections, especially if the cluster has many
backend storage servers.  If the library has an open connection
limit (possibly due to JVM file descriptor limitations) it may need
to open and close a lot of connections to access a repository,
rather than being able to reuse the same connection to a handful
of backend servers.  This results in a lot of connection thrashing
for some DHT type databases, and is inefficient.

Some DHTs are able to operate even if part of the database space
is currently unavailable.  For example, a DHT service might assign
some section of the key space to a node, and then fail that section
over to another node when the primary is noticed as being offline.
During that failover period that section of the key space is not
available, but other sections hosted by other backends are still
ready for service. Spreading keys all over the cluster makes it
likely that any single backend being temporarily down means the
entire cluster is down, rather than only some.

This is a massive schema change, but it should improve relability
and performance for any DHT system.

Change-Id: I6b65bfb4c14b6f7bd323c2bd0638b49d429245be
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>

diff --git a/org.eclipse.jgit.storage.dht.test/tst/org/eclipse/jgit/storage/dht/ChunkKeyTest.java b/org.eclipse.jgit.storage.dht.test/tst/org/eclipse/jgit/storage/dht/ChunkKeyTest.java
index 63cbf520c01899dc2ad96c65ec1c712a43fc84e7..921966366714b4fa2c080a4f2a4f46acea3e0fd9 100644 (file)
--- a/org.eclipse.jgit.storage.dht.test/tst/org/eclipse/jgit/storage/dht/ChunkKeyTest.java
+++ b/org.eclipse.jgit.storage.dht.test/tst/org/eclipse/jgit/storage/dht/ChunkKeyTest.java
@@ -43,7 +43,8 @@
 
 package org.eclipse.jgit.storage.dht;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 
 import org.eclipse.jgit.lib.ObjectId;
 import org.junit.Test;
@@ -59,19 +60,19 @@ public class ChunkKeyTest {
                ChunkKey key1 = ChunkKey.create(repo1, id);
                assertEquals(repo1.asInt(), key1.getRepositoryId());
                assertEquals(id, key1.getChunkHash());
-               assertEquals("3e.41234567.3e64b928d51b3a28e89cfe2a3f0eeae35ef07839",
+               assertEquals("41234567.3e64b928d51b3a28e89cfe2a3f0eeae35ef07839",
                                key1.asString());
 
                ChunkKey key2 = ChunkKey.fromBytes(key1.asBytes());
                assertEquals(repo1.asInt(), key2.getRepositoryId());
                assertEquals(id, key2.getChunkHash());
-               assertEquals("3e.41234567.3e64b928d51b3a28e89cfe2a3f0eeae35ef07839",
+               assertEquals("41234567.3e64b928d51b3a28e89cfe2a3f0eeae35ef07839",
                                key2.asString());
 
                ChunkKey key3 = ChunkKey.fromString(key1.asString());
                assertEquals(repo1.asInt(), key3.getRepositoryId());
                assertEquals(id, key3.getChunkHash());
-               assertEquals("3e.41234567.3e64b928d51b3a28e89cfe2a3f0eeae35ef07839",
+               assertEquals("41234567.3e64b928d51b3a28e89cfe2a3f0eeae35ef07839",
                                key3.asString());
 
                assertEquals(key1, key2);

diff --git a/org.eclipse.jgit.storage.dht.test/tst/org/eclipse/jgit/storage/dht/ObjectIndexKeyTest.java b/org.eclipse.jgit.storage.dht.test/tst/org/eclipse/jgit/storage/dht/ObjectIndexKeyTest.java
index ab3b423ede759351d8da3da7092dd0365d1d4ed8..d3419bd140d2221a1bb0936e7a651477c8cd2560 100644 (file)
--- a/org.eclipse.jgit.storage.dht.test/tst/org/eclipse/jgit/storage/dht/ObjectIndexKeyTest.java
+++ b/org.eclipse.jgit.storage.dht.test/tst/org/eclipse/jgit/storage/dht/ObjectIndexKeyTest.java
@@ -58,19 +58,19 @@ public class ObjectIndexKeyTest {
                ObjectIndexKey key1 = ObjectIndexKey.create(repo, id);
                assertEquals(repo.asInt(), key1.getRepositoryId());
                assertEquals(key1, id);
-               assertEquals("3e.41234567.3e64b928d51b3a28e89cfe2a3f0eeae35ef07839",
+               assertEquals("41234567.3e64b928d51b3a28e89cfe2a3f0eeae35ef07839",
                                key1.asString());
 
                ObjectIndexKey key2 = ObjectIndexKey.fromBytes(key1.asBytes());
                assertEquals(repo.asInt(), key2.getRepositoryId());
                assertEquals(key2, id);
-               assertEquals("3e.41234567.3e64b928d51b3a28e89cfe2a3f0eeae35ef07839",
+               assertEquals("41234567.3e64b928d51b3a28e89cfe2a3f0eeae35ef07839",
                                key2.asString());
 
                ObjectIndexKey key3 = ObjectIndexKey.fromString(key1.asString());
                assertEquals(repo.asInt(), key3.getRepositoryId());
                assertEquals(key3, id);
-               assertEquals("3e.41234567.3e64b928d51b3a28e89cfe2a3f0eeae35ef07839",
+               assertEquals("41234567.3e64b928d51b3a28e89cfe2a3f0eeae35ef07839",
                                key3.asString());
        }
 }

diff --git a/org.eclipse.jgit.storage.dht/src/org/eclipse/jgit/storage/dht/ChunkKey.java b/org.eclipse.jgit.storage.dht/src/org/eclipse/jgit/storage/dht/ChunkKey.java
index 272b5ea173571125879399f0feaa3558460dbb5f..11a151f6ba5c52ac3a9f99e863a2840241abcc6a 100644 (file)
--- a/org.eclipse.jgit.storage.dht/src/org/eclipse/jgit/storage/dht/ChunkKey.java
+++ b/org.eclipse.jgit.storage.dht/src/org/eclipse/jgit/storage/dht/ChunkKey.java
@@ -54,7 +54,7 @@ import org.eclipse.jgit.lib.ObjectId;
 
 /** Unique identifier of a {@link PackChunk} in the DHT. */
 public final class ChunkKey implements RowKey {
-       static final int KEYLEN = 52;
+       static final int KEYLEN = 49;
 
        /**
         * @param repo
@@ -84,8 +84,8 @@ public final class ChunkKey implements RowKey {
                        throw new IllegalArgumentException(MessageFormat.format(
                                        DhtText.get().invalidChunkKey, decode(key, ptr, ptr + len)));
 
-               int repo = parse32(key, ptr + 3);
-               ObjectId chunk = ObjectId.fromString(key, ptr + 12);
+               int repo = parse32(key, ptr);
+               ObjectId chunk = ObjectId.fromString(key, ptr + 9);
                return new ChunkKey(repo, chunk);
        }
 
@@ -122,13 +122,9 @@ public final class ChunkKey implements RowKey {
 
        public byte[] asBytes() {
                byte[] r = new byte[KEYLEN];
-               chunk.copyTo(r, 12);
-               format32(r, 3, repo);
-               // bucket is the leading 2 digits of the SHA-1.
-               r[11] = '.';
-               r[2] = '.';
-               r[1] = r[12 + 1];
-               r[0] = r[12 + 0];
+               format32(r, 0, repo);
+               r[8] = '.';
+               chunk.copyTo(r, 9);
                return r;
        }
 

diff --git a/org.eclipse.jgit.storage.dht/src/org/eclipse/jgit/storage/dht/ObjectIndexKey.java b/org.eclipse.jgit.storage.dht/src/org/eclipse/jgit/storage/dht/ObjectIndexKey.java
index b38fdcec22e11353d6f0f7369e3547f244aa0209..ab8f8352b0ceb28d0cfadb561c61aeb9d7f7b90e 100644 (file)
--- a/org.eclipse.jgit.storage.dht/src/org/eclipse/jgit/storage/dht/ObjectIndexKey.java
+++ b/org.eclipse.jgit.storage.dht/src/org/eclipse/jgit/storage/dht/ObjectIndexKey.java
@@ -55,7 +55,7 @@ import org.eclipse.jgit.lib.ObjectId;
 
 /** Identifies an ObjectId in the DHT. */
 public final class ObjectIndexKey extends ObjectId implements RowKey {
-       private static final int KEYLEN = 52;
+       private static final int KEYLEN = 49;
 
        /**
         * @param repo
@@ -75,8 +75,8 @@ public final class ObjectIndexKey extends ObjectId implements RowKey {
                        throw new IllegalArgumentException(MessageFormat.format(
                                        DhtText.get().invalidChunkKey, decode(key)));
 
-               int repo = parse32(key, 3);
-               ObjectId id = ObjectId.fromString(key, 12);
+               int repo = parse32(key, 0);
+               ObjectId id = ObjectId.fromString(key, 9);
                return new ObjectIndexKey(repo, id);
        }
 
@@ -106,13 +106,9 @@ public final class ObjectIndexKey extends ObjectId implements RowKey {
 
        public byte[] asBytes() {
                byte[] r = new byte[KEYLEN];
-               copyTo(r, 12);
-               format32(r, 3, repo);
-               // bucket is the leading 2 digits of the SHA-1.
-               r[11] = '.';
-               r[2] = '.';
-               r[1] = r[12 + 1];
-               r[0] = r[12 + 0];
+               format32(r, 0, repo);
+               r[8] = '.';
+               copyTo(r, 9);
                return r;
        }