From f44f45a15e10a87bcb62a6a3a1bb0f7cbc282478 Mon Sep 17 00:00:00 2001 From: Florian Zschocke Date: Sun, 5 Mar 2017 17:11:50 +0100 Subject: [PATCH] Use versioned index directories for repository indices. Change from the index version of a repository index being stored in a config file to also using index directories with the version in the name. For that, `LuceneRepoIndexStore` is added, which adds the fixed `lucene` part to the path. It also gives out the location of the `lucene.conf` file, which is now stored in the index directory. This way it is automatically deleted when the directory is deleted. I believe that it should also provide means to store branch aliases and tips, i.e. hide the config file completely. But this isn't implemented with this commit, the `LuceneService` is still aware that a config file is used. --- .../gitblit/service/LuceneRepoIndexStore.java | 55 ++++ .../com/gitblit/service/LuceneService.java | 65 ++--- .../service/LuceneRepoIndexStoreTest.java | 267 ++++++++++++++++++ 3 files changed, 339 insertions(+), 48 deletions(-) create mode 100644 src/main/java/com/gitblit/service/LuceneRepoIndexStore.java create mode 100644 src/test/java/com/gitblit/service/LuceneRepoIndexStoreTest.java diff --git a/src/main/java/com/gitblit/service/LuceneRepoIndexStore.java b/src/main/java/com/gitblit/service/LuceneRepoIndexStore.java new file mode 100644 index 00000000..ff7d0885 --- /dev/null +++ b/src/main/java/com/gitblit/service/LuceneRepoIndexStore.java @@ -0,0 +1,55 @@ +/* + * Copyright 2017 gitblit.com. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.gitblit.service; + +import java.io.File; + +import com.gitblit.utils.LuceneIndexStore; + +/** + * @author Florian Zschocke + * + * @since 1.9.0 + */ +class LuceneRepoIndexStore extends LuceneIndexStore +{ + + private static final String LUCENE_DIR = "lucene"; + private static final String CONF_FILE = "gb_lucene.conf"; + + + /** + * @param repositoryFolder + * The directory of the repository for this index + * @param indexVersion + * Version of the index definition + */ + public LuceneRepoIndexStore(File repositoryFolder, int indexVersion) { + super(new File(repositoryFolder, LUCENE_DIR), indexVersion); + } + + + /** + * Get the index config File. + * + * @return The index config File + */ + public File getConfigFile() { + return new File(this.indexFolder, CONF_FILE); + } + +} diff --git a/src/main/java/com/gitblit/service/LuceneService.java b/src/main/java/com/gitblit/service/LuceneService.java index 9b97ad5b..906a0b5e 100644 --- a/src/main/java/com/gitblit/service/LuceneService.java +++ b/src/main/java/com/gitblit/service/LuceneService.java @@ -117,10 +117,6 @@ public class LuceneService implements Runnable { private static final String FIELD_DATE = "date"; private static final String FIELD_TAG = "tag"; - private static final String CONF_FILE = "lucene.conf"; - private static final String LUCENE_DIR = "lucene"; - private static final String CONF_INDEX = "index"; - private static final String CONF_VERSION = "version"; private static final String CONF_ALIAS = "aliases"; private static final String CONF_BRANCH = "branches"; @@ -290,26 +286,13 @@ public class LuceneService implements Runnable { * @return true, if successful */ public boolean deleteIndex(String repositoryName) { - try { - // close any open writer/searcher - close(repositoryName); - - // delete the index folder - File repositoryFolder = FileKey.resolve(new File(repositoriesFolder, repositoryName), FS.DETECTED); - File luceneIndex = new File(repositoryFolder, LUCENE_DIR); - if (luceneIndex.exists()) { - org.eclipse.jgit.util.FileUtils.delete(luceneIndex, - org.eclipse.jgit.util.FileUtils.RECURSIVE); - } - // delete the config file - File luceneConfig = new File(repositoryFolder, CONF_FILE); - if (luceneConfig.exists()) { - luceneConfig.delete(); - } - return true; - } catch (IOException e) { - throw new RuntimeException(e); - } + // close any open writer/searcher + close(repositoryName); + + // delete the index folder + File repositoryFolder = FileKey.resolve(new File(repositoriesFolder, repositoryName), FS.DETECTED); + LuceneRepoIndexStore luceneIndex = new LuceneRepoIndexStore(repositoryFolder, INDEX_VERSION); + return luceneIndex.delete(); } /** @@ -383,29 +366,20 @@ public class LuceneService implements Runnable { * @return a config object */ private FileBasedConfig getConfig(Repository repository) { - File file = new File(repository.getDirectory(), CONF_FILE); - FileBasedConfig config = new FileBasedConfig(file, FS.detect()); + LuceneRepoIndexStore luceneIndex = new LuceneRepoIndexStore(repository.getDirectory(), INDEX_VERSION); + FileBasedConfig config = new FileBasedConfig(luceneIndex.getConfigFile(), FS.detect()); return config; } /** - * Reads the Lucene config file for the repository to check the index - * version. If the index version is different, then rebuild the repository - * index. + * Checks if an index exists for the repository, that is compatible with + * INDEX_VERSION and the Lucene version. * * @param repository - * @return true of the on-disk index format is different than INDEX_VERSION + * @return true if no index is found for the repository, false otherwise. */ private boolean shouldReindex(Repository repository) { - try { - FileBasedConfig config = getConfig(repository); - config.load(); - int indexVersion = config.getInt(CONF_INDEX, CONF_VERSION, 0); - // reindex if versions do not match - return indexVersion != INDEX_VERSION; - } catch (Throwable t) { - } - return true; + return ! (new LuceneRepoIndexStore(repository.getDirectory(), INDEX_VERSION).hasIndex()); } @@ -615,7 +589,6 @@ public class LuceneService implements Runnable { reader.close(); // commit all changes and reset the searcher - config.setInt(CONF_INDEX, null, CONF_VERSION, INDEX_VERSION); config.save(); writer.commit(); resetIndexSearcher(model.name); @@ -844,7 +817,6 @@ public class LuceneService implements Runnable { } // update the config - config.setInt(CONF_INDEX, null, CONF_VERSION, INDEX_VERSION); config.setString(CONF_ALIAS, null, keyName, branchName); config.setString(CONF_BRANCH, null, keyName, branch.getObjectId().getName()); config.save(); @@ -962,14 +934,11 @@ public class LuceneService implements Runnable { */ private IndexWriter getIndexWriter(String repository) throws IOException { IndexWriter indexWriter = writers.get(repository); - File repositoryFolder = FileKey.resolve(new File(repositoriesFolder, repository), FS.DETECTED); - File indexFolder = new File(repositoryFolder, LUCENE_DIR); - Directory directory = FSDirectory.open(indexFolder.toPath()); - if (indexWriter == null) { - if (!indexFolder.exists()) { - indexFolder.mkdirs(); - } + File repositoryFolder = FileKey.resolve(new File(repositoriesFolder, repository), FS.DETECTED); + LuceneRepoIndexStore indexStore = new LuceneRepoIndexStore(repositoryFolder, INDEX_VERSION); + indexStore.create(); + Directory directory = FSDirectory.open(indexStore.getPath()); StandardAnalyzer analyzer = new StandardAnalyzer(); IndexWriterConfig config = new IndexWriterConfig(analyzer); config.setOpenMode(OpenMode.CREATE_OR_APPEND); diff --git a/src/test/java/com/gitblit/service/LuceneRepoIndexStoreTest.java b/src/test/java/com/gitblit/service/LuceneRepoIndexStoreTest.java new file mode 100644 index 00000000..baac3516 --- /dev/null +++ b/src/test/java/com/gitblit/service/LuceneRepoIndexStoreTest.java @@ -0,0 +1,267 @@ +/* + * Copyright 2017 gitblit.com. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.gitblit.service; + +import static org.junit.Assert.*; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import com.gitblit.utils.LuceneIndexStore; + +/** + * @author Florian Zschocke + * + */ +public class LuceneRepoIndexStoreTest +{ + + private static final int LUCENE_VERSION = LuceneIndexStore.LUCENE_CODEC_VERSION; + private static final String LUCENE_DIR = "lucene"; + + + @Rule + public TemporaryFolder baseFolder = new TemporaryFolder(); + + + private String getIndexDir(int version) + { + return version + "_" + LUCENE_VERSION; + } + + + private String getLuceneIndexDir(int version) + { + return LUCENE_DIR + "/" + version + "_" + LUCENE_VERSION; + } + + + @Test + public void testGetConfigFile() throws IOException + { + int version = 1; + File repositoryFolder = baseFolder.getRoot(); + LuceneRepoIndexStore li = new LuceneRepoIndexStore(repositoryFolder, version); + File confFile= li.getConfigFile(); + File luceneDir = new File(repositoryFolder, getLuceneIndexDir(version) + "/gb_lucene.conf"); + assertEquals(luceneDir, confFile); + } + + + @Test + public void testCreate() + { + int version = 0; + File repositoryFolder = baseFolder.getRoot(); + File luceneDir = new File(repositoryFolder, getLuceneIndexDir(version)); + assertFalse("Precondition failure: directory exists already", new File(repositoryFolder, LUCENE_DIR).exists()); + assertFalse("Precondition failure: directory exists already", luceneDir.exists()); + + LuceneIndexStore li = new LuceneRepoIndexStore(repositoryFolder, version); + li.create(); + + assertTrue(luceneDir.exists()); + assertTrue(luceneDir.isDirectory()); + } + + @Test + public void testCreateIndexDir() + { + int version = 7777; + File repositoryFolder = baseFolder.getRoot(); + try { + baseFolder.newFolder(LUCENE_DIR); + } catch (IOException e) { + fail("Failed in setup of folder: " + e); + } + File luceneDir = new File(repositoryFolder, getLuceneIndexDir(version)); + + assertTrue("Precondition failure: directory does not exist", new File(repositoryFolder, LUCENE_DIR).exists()); + assertFalse("Precondition failure: directory exists already", luceneDir.exists()); + + LuceneIndexStore li = new LuceneRepoIndexStore(repositoryFolder, version); + li.create(); + + assertTrue(luceneDir.exists()); + assertTrue(luceneDir.isDirectory()); + + // Make sure nothing else was created. + assertEquals(0, luceneDir.list().length); + assertEquals(1, luceneDir.getParentFile().list().length); + } + + @Test + public void testCreateIfNecessary() + { + int version = 7777888; + File repositoryFolder = baseFolder.getRoot(); + File luceneDir = null; + try { + luceneDir = baseFolder.newFolder(LUCENE_DIR, getIndexDir(version)); + } catch (IOException e) { + fail("Failed in setup of folder: " + e); + } + assertTrue("Precondition failure: directory does not exist", new File(repositoryFolder, LUCENE_DIR).exists()); + assertTrue("Precondition failure: directory does not exist", luceneDir.exists()); + + LuceneIndexStore li = new LuceneRepoIndexStore(repositoryFolder, version); + li.create(); + + assertTrue(luceneDir.exists()); + assertTrue(luceneDir.isDirectory()); + + // Make sure nothing else was created. + assertEquals(0, luceneDir.list().length); + assertEquals(1, luceneDir.getParentFile().list().length); + } + + + @Test + public void testDelete() + { + int version = 111222333; + + File repositoryFolder = baseFolder.getRoot(); + File luceneDir = null; + try { + luceneDir = baseFolder.newFolder(LUCENE_DIR, getIndexDir(version)); + } catch (IOException e) { + fail("Failed in setup of folder: " + e); + } + assertTrue("Precondition failure: directory does not exist", luceneDir.exists()); + + LuceneIndexStore li = new LuceneRepoIndexStore(repositoryFolder, version); + assertTrue(li.delete()); + + assertFalse(luceneDir.exists()); + assertTrue(new File(repositoryFolder, LUCENE_DIR).exists()); + } + + + @Test + public void testDeleteNotExist() + { + int version = 0; + + File repositoryFolder = baseFolder.getRoot(); + try { + baseFolder.newFolder(LUCENE_DIR); + } catch (IOException e) { + fail("Failed in setup of folder: " + e); + } + File luceneDir = new File(repositoryFolder, getLuceneIndexDir(version)); + assertTrue("Precondition failure: directory does not exist", new File(repositoryFolder, LUCENE_DIR).exists()); + assertFalse("Precondition failure: directory does exist", luceneDir.exists()); + + LuceneIndexStore li = new LuceneRepoIndexStore(repositoryFolder, version); + assertTrue(li.delete()); + + assertFalse(luceneDir.exists()); + assertTrue(new File(repositoryFolder, LUCENE_DIR).exists()); + } + + + @Test + public void testDeleteWithFiles() + { + int version = 5; + + File repositoryFolder = baseFolder.getRoot(); + File luceneFolder = new File(baseFolder.getRoot(), LUCENE_DIR); + File luceneDir = null; + + File otherDir = new File(luceneFolder, version + "_10"); + File dbFile = null; + try { + luceneDir = baseFolder.newFolder(LUCENE_DIR, getIndexDir(version)); + File file = new File(luceneDir, "_file1"); + file.createNewFile(); + file = new File(luceneDir, "_file2.db"); + file.createNewFile(); + file = new File(luceneDir, "conf.conf"); + file.createNewFile(); + + otherDir.mkdirs(); + dbFile = new File(otherDir, "_file2.db"); + dbFile.createNewFile(); + file = new File(otherDir, "conf.conf"); + file.createNewFile(); + } + catch (IOException e) { + fail("Failed in setup of folder: " + e); + } + assertTrue("Precondition failure: index directory does not exist", luceneDir.exists()); + assertTrue("Precondition failure: other index directory does not exist", otherDir.exists()); + + LuceneIndexStore li = new LuceneRepoIndexStore(repositoryFolder, version); + li.delete(); + + assertFalse(luceneDir.exists()); + assertTrue(luceneFolder.exists()); + assertTrue(otherDir.exists()); + assertTrue(dbFile.exists()); + } + + + + @Test + public void testGetPath() throws IOException + { + int version = 7; + File repositoryFolder = baseFolder.getRoot(); + LuceneIndexStore li = new LuceneRepoIndexStore(repositoryFolder, version); + Path dir = li.getPath(); + File luceneDir = new File(repositoryFolder, getLuceneIndexDir(version)); + assertEquals(luceneDir.toPath(), dir); + } + + + @Test + public void testHasIndex() throws IOException + { + int version = 0; + File luceneFolder = new File(baseFolder.getRoot(), "lucene"); + + LuceneIndexStore li = new LuceneRepoIndexStore(luceneFolder, version); + assertFalse(li.hasIndex()); + + baseFolder.newFolder("lucene"); + li = new LuceneIndexStore(luceneFolder, version); + assertFalse(li.hasIndex()); + + File luceneDir = baseFolder.newFolder("lucene", getIndexDir(version)); + li = new LuceneIndexStore(luceneFolder, version); + assertFalse(li.hasIndex()); + + new File(luceneDir, "write.lock").createNewFile(); + li = new LuceneIndexStore(luceneFolder, version); + assertFalse(li.hasIndex()); + + new File(luceneDir, "segments_1").createNewFile(); + li = new LuceneIndexStore(luceneFolder, version); + System.out.println("Check " + luceneDir); + assertTrue(li.hasIndex()); + + } + + +} -- 2.39.5