Use versioned index directories for repository indices.

Change from the index version of a repository index being stored in a config
file to also using index directories with the version in the name. For that,
`LuceneRepoIndexStore` is added, which adds the fixed `lucene` part to the path.
It also gives out the location of the `lucene.conf` file, which is now stored in
the index directory. This way it is automatically deleted when the directory is
deleted.

I believe that it should also provide means to store branch aliases and tips,
i.e. hide the config file completely. But this isn't implemented with this
commit, the `LuceneService` is still aware that a config file is used.

3 files changed, 339 insertions, 48 deletions

diff --git a/src/main/java/com/gitblit/service/LuceneRepoIndexStore.java b/src/main/java/com/gitblit/service/LuceneRepoIndexStore.java
new file mode 100644
index 00000000..ff7d0885
--- /dev/null
+++ b/src/main/java/com/gitblit/service/LuceneRepoIndexStore.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2017 gitblit.com.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.gitblit.service;
+
+import java.io.File;
+
+import com.gitblit.utils.LuceneIndexStore;
+
+/**
+ * @author Florian Zschocke
+ *
+ * @since 1.9.0
+ */
+class LuceneRepoIndexStore extends LuceneIndexStore
+{
+
+	private static final String LUCENE_DIR = "lucene";
+	private static final String CONF_FILE = "gb_lucene.conf";
+
+
+	/**
+	 * @param repositoryFolder
+	 * 			The directory of the repository for this index
+	 * @param indexVersion
+	 * 			Version of the index definition
+	 */
+	public LuceneRepoIndexStore(File repositoryFolder, int indexVersion) {
+		super(new File(repositoryFolder, LUCENE_DIR), indexVersion);
+	}
+
+
+	/**
+	 * Get the index config File.
+	 *
+	 * @return	The index config File
+	 */
+	public File getConfigFile() {
+		return new File(this.indexFolder, CONF_FILE);
+	}
+
+}
diff --git a/src/main/java/com/gitblit/service/LuceneService.java b/src/main/java/com/gitblit/service/LuceneService.java
index 9b97ad5b..906a0b5e 100644
--- a/src/main/java/com/gitblit/service/LuceneService.java
+++ b/src/main/java/com/gitblit/service/LuceneService.java
@@ -117,10 +117,6 @@ public class LuceneService implements Runnable {
 	private static final String FIELD_DATE = "date";

 	private static final String FIELD_TAG = "tag";

 

-	private static final String CONF_FILE = "lucene.conf";

-	private static final String LUCENE_DIR = "lucene";

-	private static final String CONF_INDEX = "index";

-	private static final String CONF_VERSION = "version";

 	private static final String CONF_ALIAS = "aliases";

 	private static final String CONF_BRANCH = "branches";

 

@@ -290,26 +286,13 @@ public class LuceneService implements Runnable {
 	 * @return true, if successful

 	 */

 	public boolean deleteIndex(String repositoryName) {

-		try {

-			// close any open writer/searcher

-			close(repositoryName);

-

-			// delete the index folder

-			File repositoryFolder = FileKey.resolve(new File(repositoriesFolder, repositoryName), FS.DETECTED);

-			File luceneIndex = new File(repositoryFolder, LUCENE_DIR);

-			if (luceneIndex.exists()) {

-				org.eclipse.jgit.util.FileUtils.delete(luceneIndex,

-						org.eclipse.jgit.util.FileUtils.RECURSIVE);

-			}

-			// delete the config file

-			File luceneConfig = new File(repositoryFolder, CONF_FILE);

-			if (luceneConfig.exists()) {

-				luceneConfig.delete();

-			}

-			return true;

-		} catch (IOException e) {

-			throw new RuntimeException(e);

-		}

+		// close any open writer/searcher

+		close(repositoryName);

+

+		// delete the index folder

+		File repositoryFolder = FileKey.resolve(new File(repositoriesFolder, repositoryName), FS.DETECTED);

+		LuceneRepoIndexStore luceneIndex = new LuceneRepoIndexStore(repositoryFolder, INDEX_VERSION);

+		return luceneIndex.delete();

 	}

 

 	/**

@@ -383,29 +366,20 @@ public class LuceneService implements Runnable {
 	 * @return a config object

 	 */

 	private FileBasedConfig getConfig(Repository repository) {

-		File file = new File(repository.getDirectory(), CONF_FILE);

-		FileBasedConfig config = new FileBasedConfig(file, FS.detect());

+		LuceneRepoIndexStore luceneIndex = new LuceneRepoIndexStore(repository.getDirectory(), INDEX_VERSION);

+		FileBasedConfig config = new FileBasedConfig(luceneIndex.getConfigFile(), FS.detect());

 		return config;

 	}

 

 	/**

-	 * Reads the Lucene config file for the repository to check the index

-	 * version. If the index version is different, then rebuild the repository

-	 * index.

+	 * Checks if an index exists for the repository, that is compatible with

+	 * INDEX_VERSION and the Lucene version.

 	 *

 	 * @param repository

-	 * @return true of the on-disk index format is different than INDEX_VERSION

+	 * @return true if no index is found for the repository, false otherwise.

 	 */

 	private boolean shouldReindex(Repository repository) {

-		try {

-			FileBasedConfig config = getConfig(repository);

-			config.load();

-			int indexVersion = config.getInt(CONF_INDEX, CONF_VERSION, 0);

-			// reindex if versions do not match

-			return indexVersion != INDEX_VERSION;

-		} catch (Throwable t) {

-		}

-		return true;

+		return ! (new LuceneRepoIndexStore(repository.getDirectory(), INDEX_VERSION).hasIndex());

 	}

 

 

@@ -615,7 +589,6 @@ public class LuceneService implements Runnable {
 			reader.close();

 

 			// commit all changes and reset the searcher

-			config.setInt(CONF_INDEX, null, CONF_VERSION, INDEX_VERSION);

 			config.save();

 			writer.commit();

 			resetIndexSearcher(model.name);

@@ -844,7 +817,6 @@ public class LuceneService implements Runnable {
 				}

 

 				// update the config

-				config.setInt(CONF_INDEX, null, CONF_VERSION, INDEX_VERSION);

 				config.setString(CONF_ALIAS, null, keyName, branchName);

 				config.setString(CONF_BRANCH, null, keyName, branch.getObjectId().getName());

 				config.save();

@@ -962,14 +934,11 @@ public class LuceneService implements Runnable {
 	 */

 	private IndexWriter getIndexWriter(String repository) throws IOException {

 		IndexWriter indexWriter = writers.get(repository);

-		File repositoryFolder = FileKey.resolve(new File(repositoriesFolder, repository), FS.DETECTED);

-		File indexFolder = new File(repositoryFolder, LUCENE_DIR);

-		Directory directory = FSDirectory.open(indexFolder.toPath());

-

 		if (indexWriter == null) {

-			if (!indexFolder.exists()) {

-				indexFolder.mkdirs();

-			}

+			File repositoryFolder = FileKey.resolve(new File(repositoriesFolder, repository), FS.DETECTED);

+			LuceneRepoIndexStore indexStore = new LuceneRepoIndexStore(repositoryFolder, INDEX_VERSION);

+			indexStore.create();

+			Directory directory = FSDirectory.open(indexStore.getPath());

 			StandardAnalyzer analyzer = new StandardAnalyzer();

 			IndexWriterConfig config = new IndexWriterConfig(analyzer);

 			config.setOpenMode(OpenMode.CREATE_OR_APPEND);

diff --git a/src/test/java/com/gitblit/service/LuceneRepoIndexStoreTest.java b/src/test/java/com/gitblit/service/LuceneRepoIndexStoreTest.java
new file mode 100644
index 00000000..baac3516
--- /dev/null
+++ b/src/test/java/com/gitblit/service/LuceneRepoIndexStoreTest.java
@@ -0,0 +1,267 @@
+/*
+ * Copyright 2017 gitblit.com.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.gitblit.service;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import com.gitblit.utils.LuceneIndexStore;
+
+/**
+ * @author Florian Zschocke
+ *
+ */
+public class LuceneRepoIndexStoreTest
+{
+
+	private static final int LUCENE_VERSION = LuceneIndexStore.LUCENE_CODEC_VERSION;
+	private static final String LUCENE_DIR = "lucene";
+
+
+	@Rule
+	public TemporaryFolder baseFolder = new TemporaryFolder();
+
+
+	private String getIndexDir(int version)
+	{
+		return version + "_" + LUCENE_VERSION;
+	}
+
+
+	private String getLuceneIndexDir(int version)
+	{
+		return LUCENE_DIR + "/" + version + "_" + LUCENE_VERSION;
+	}
+
+
+	@Test
+	public void testGetConfigFile() throws IOException
+	{
+		int version = 1;
+		File repositoryFolder = baseFolder.getRoot();
+		LuceneRepoIndexStore li = new LuceneRepoIndexStore(repositoryFolder, version);
+		File confFile= li.getConfigFile();
+		File luceneDir = new File(repositoryFolder, getLuceneIndexDir(version) + "/gb_lucene.conf");
+		assertEquals(luceneDir, confFile);
+	}
+
+
+	@Test
+	public void testCreate()
+	{
+		int version = 0;
+		File repositoryFolder = baseFolder.getRoot();
+		File luceneDir = new File(repositoryFolder, getLuceneIndexDir(version));
+		assertFalse("Precondition failure: directory exists already", new File(repositoryFolder, LUCENE_DIR).exists());
+		assertFalse("Precondition failure: directory exists already", luceneDir.exists());
+
+		LuceneIndexStore li = new LuceneRepoIndexStore(repositoryFolder, version);
+		li.create();
+
+		assertTrue(luceneDir.exists());
+		assertTrue(luceneDir.isDirectory());
+	}
+
+	@Test
+	public void testCreateIndexDir()
+	{
+		int version = 7777;
+		File repositoryFolder = baseFolder.getRoot();
+		try {
+			baseFolder.newFolder(LUCENE_DIR);
+		} catch (IOException e) {
+			fail("Failed in setup of folder: " + e);
+		}
+		File luceneDir = new File(repositoryFolder, getLuceneIndexDir(version));
+
+		assertTrue("Precondition failure: directory does not exist", new File(repositoryFolder, LUCENE_DIR).exists());
+		assertFalse("Precondition failure: directory exists already", luceneDir.exists());
+
+		LuceneIndexStore li = new LuceneRepoIndexStore(repositoryFolder, version);
+		li.create();
+
+		assertTrue(luceneDir.exists());
+		assertTrue(luceneDir.isDirectory());
+
+		// Make sure nothing else was created.
+		assertEquals(0, luceneDir.list().length);
+		assertEquals(1, luceneDir.getParentFile().list().length);
+	}
+
+	@Test
+	public void testCreateIfNecessary()
+	{
+		int version = 7777888;
+		File repositoryFolder = baseFolder.getRoot();
+		File luceneDir = null;
+		try {
+			luceneDir = baseFolder.newFolder(LUCENE_DIR, getIndexDir(version));
+		} catch (IOException e) {
+			fail("Failed in setup of folder: " + e);
+		}
+		assertTrue("Precondition failure: directory does not exist", new File(repositoryFolder, LUCENE_DIR).exists());
+		assertTrue("Precondition failure: directory does not exist", luceneDir.exists());
+
+		LuceneIndexStore li = new LuceneRepoIndexStore(repositoryFolder, version);
+		li.create();
+
+		assertTrue(luceneDir.exists());
+		assertTrue(luceneDir.isDirectory());
+
+		// Make sure nothing else was created.
+		assertEquals(0, luceneDir.list().length);
+		assertEquals(1, luceneDir.getParentFile().list().length);
+	}
+
+
+	@Test
+	public void testDelete()
+	{
+		int version = 111222333;
+
+		File repositoryFolder = baseFolder.getRoot();
+		File luceneDir = null;
+		try {
+			luceneDir = baseFolder.newFolder(LUCENE_DIR, getIndexDir(version));
+		} catch (IOException e) {
+			fail("Failed in setup of folder: " + e);
+		}
+		assertTrue("Precondition failure: directory does not exist", luceneDir.exists());
+
+		LuceneIndexStore li = new LuceneRepoIndexStore(repositoryFolder, version);
+		assertTrue(li.delete());
+
+		assertFalse(luceneDir.exists());
+		assertTrue(new File(repositoryFolder, LUCENE_DIR).exists());
+	}
+
+
+	@Test
+	public void testDeleteNotExist()
+	{
+		int version = 0;
+
+		File repositoryFolder = baseFolder.getRoot();
+		try {
+			baseFolder.newFolder(LUCENE_DIR);
+		} catch (IOException e) {
+			fail("Failed in setup of folder: " + e);
+		}
+		File luceneDir = new File(repositoryFolder, getLuceneIndexDir(version));
+		assertTrue("Precondition failure: directory does not exist", new File(repositoryFolder, LUCENE_DIR).exists());
+		assertFalse("Precondition failure: directory does exist", luceneDir.exists());
+
+		LuceneIndexStore li = new LuceneRepoIndexStore(repositoryFolder, version);
+		assertTrue(li.delete());
+
+		assertFalse(luceneDir.exists());
+		assertTrue(new File(repositoryFolder, LUCENE_DIR).exists());
+	}
+
+
+	@Test
+	public void testDeleteWithFiles()
+	{
+		int version = 5;
+
+		File repositoryFolder = baseFolder.getRoot();
+		File luceneFolder = new File(baseFolder.getRoot(), LUCENE_DIR);
+		File luceneDir = null;
+
+		File otherDir = new File(luceneFolder, version + "_10");
+		File dbFile = null;
+		try {
+			luceneDir = baseFolder.newFolder(LUCENE_DIR, getIndexDir(version));
+			File file = new File(luceneDir, "_file1");
+			file.createNewFile();
+			file = new File(luceneDir, "_file2.db");
+			file.createNewFile();
+			file = new File(luceneDir, "conf.conf");
+			file.createNewFile();
+
+			otherDir.mkdirs();
+			dbFile = new File(otherDir, "_file2.db");
+			dbFile.createNewFile();
+			file = new File(otherDir, "conf.conf");
+			file.createNewFile();
+		}
+		catch (IOException e) {
+			fail("Failed in setup of folder: " + e);
+		}
+		assertTrue("Precondition failure: index directory does not exist", luceneDir.exists());
+		assertTrue("Precondition failure: other index directory does not exist", otherDir.exists());
+
+		LuceneIndexStore li = new LuceneRepoIndexStore(repositoryFolder, version);
+		li.delete();
+
+		assertFalse(luceneDir.exists());
+		assertTrue(luceneFolder.exists());
+		assertTrue(otherDir.exists());
+		assertTrue(dbFile.exists());
+	}
+
+
+
+	@Test
+	public void testGetPath() throws IOException
+	{
+		int version = 7;
+		File repositoryFolder = baseFolder.getRoot();
+		LuceneIndexStore li = new LuceneRepoIndexStore(repositoryFolder, version);
+		Path dir = li.getPath();
+		File luceneDir = new File(repositoryFolder, getLuceneIndexDir(version));
+		assertEquals(luceneDir.toPath(), dir);
+	}
+
+
+	@Test
+	public void testHasIndex() throws IOException
+	{
+		int version = 0;
+		File luceneFolder = new File(baseFolder.getRoot(), "lucene");
+
+		LuceneIndexStore li = new LuceneRepoIndexStore(luceneFolder, version);
+		assertFalse(li.hasIndex());
+
+		baseFolder.newFolder("lucene");
+		li = new LuceneIndexStore(luceneFolder, version);
+		assertFalse(li.hasIndex());
+
+		File luceneDir = baseFolder.newFolder("lucene", getIndexDir(version));
+		li = new LuceneIndexStore(luceneFolder, version);
+		assertFalse(li.hasIndex());
+
+		new File(luceneDir, "write.lock").createNewFile();
+		li = new LuceneIndexStore(luceneFolder, version);
+		assertFalse(li.hasIndex());
+
+		new File(luceneDir, "segments_1").createNewFile();
+		li = new LuceneIndexStore(luceneFolder, version);
+		System.out.println("Check " + luceneDir);
+		assertTrue(li.hasIndex());
+
+	}
+
+
+}