From 6ba7b83f65cf70889873f802ae6cc9a87f72cedb Mon Sep 17 00:00:00 2001 From: Stephane Gamard Date: Mon, 26 May 2014 13:00:10 +0200 Subject: [PATCH] DAOv.2 - Added IndexStat --- .../org/sonar/server/search/BaseIndex.java | 108 +++++++++++------ .../java/org/sonar/server/search/Index.java | 4 +- .../org/sonar/server/search/IndexClient.java | 9 -- .../sonar/server/search/IndexDefinition.java | 18 ++- .../org/sonar/server/search/IndexStat.java | 56 +++++++++ .../sonar/server/search/BaseIndexTest.java | 111 ++++++++++++++++++ 6 files changed, 261 insertions(+), 45 deletions(-) create mode 100644 sonar-server/src/main/java/org/sonar/server/search/IndexStat.java create mode 100644 sonar-server/src/test/java/org/sonar/server/search/BaseIndexTest.java diff --git a/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java b/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java index 670374438a1..3f15c875e4c 100644 --- a/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java +++ b/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java @@ -26,6 +26,7 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.Client; +import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.BoolFilterBuilder; import org.elasticsearch.index.query.FilterBuilder; @@ -41,6 +42,7 @@ import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; import java.util.Collection; +import java.util.Date; import java.util.Map; import java.util.concurrent.ExecutionException; @@ -50,11 +52,11 @@ public abstract class BaseIndex, K extends Serializable> private static final Logger LOG = LoggerFactory.getLogger(BaseIndex.class); private final ESNode node; - protected BaseNormalizer normalizer; + protected final BaseNormalizer normalizer; protected final IndexDefinition indexDefinition; - public BaseIndex(IndexDefinition indexDefinition, BaseNormalizer normalizer, - WorkQueue workQueue, ESNode node) { + protected BaseIndex(IndexDefinition indexDefinition, BaseNormalizer normalizer, + WorkQueue workQueue, ESNode node) { this.normalizer = normalizer; this.node = node; this.indexDefinition = indexDefinition; @@ -94,7 +96,29 @@ public abstract class BaseIndex, K extends Serializable> /* Cluster And ES Stats/Client methods */ + private void initializeManagementIndex() { + LOG.info("Setup of Management Index for ES"); + + String index = indexDefinition.getManagementIndex(); + + IndicesExistsResponse indexExistsResponse = getClient().admin().indices() + .prepareExists(index).execute().actionGet(); + + if (!indexExistsResponse.isExists()) { + getClient().admin().indices().prepareCreate(index) + .setSettings(ImmutableSettings.builder() + .put("mapper.dynamic", true) + .put("number_of_replicas", 1) + .put("number_of_shards", 1) + .build()) + .get(); + } + } + protected void initializeIndex() { + + initializeManagementIndex(); + String index = this.getIndexName(); IndicesExistsResponse indexExistsResponse = getClient().admin().indices() @@ -105,22 +129,62 @@ public abstract class BaseIndex, K extends Serializable> LOG.info("Setup of {} for type {}", this.getIndexName(), this.getIndexType()); getClient().admin().indices().prepareCreate(index) .setSettings(getIndexSettings()) - .addMapping(getIndexType(), getMapping()) .execute().actionGet(); - } else { + } + + if (getMapping() != null) { LOG.info("Update of index {} for type {}", this.getIndexName(), this.getIndexType()); getClient().admin().indices().preparePutMapping(index) .setType(getIndexType()) .setIgnoreConflicts(true) .setSource(getMapping()) - .execute().actionGet(); + .get(); } } catch (Exception e) { throw new IllegalStateException("Invalid configuration for index " + this.getIndexName(), e); } } + public IndexStat getIndexStat() { + IndexStat stat = new IndexStat(); + + /** get total document count */ + stat.setDocumentCount( + getClient().prepareCount(this.getIndexName()) + .setQuery(QueryBuilders.matchAllQuery()) + .get().getCount() + ); + + /** get Management information */ + stat.setLastUpdate(getLastSynchronization()); + return stat; + } + + /* Synchronization methods */ + + private void setLastSynchronization() { + Date time = new Date(); + if (time.after(getLastSynchronization())) { + LOG.info("Updating synchTime updating"); + getClient().prepareUpdate() + .setId(indexDefinition.getIndexName()) + .setType(indexDefinition.getManagementType()) + .setIndex(indexDefinition.getManagementIndex()) + .setDoc("updatedAt", time) + .get(); + } + } + + @Override + public Date getLastSynchronization() { + return (java.util.Date) getClient().prepareGet() + .setIndex(indexDefinition.getManagementIndex()) + .setId(this.getIndexName()) + .setType(indexDefinition.getManagementType()) + .get().getField("updatedAt").getValue(); + } + /* Index management methods */ protected abstract String getKeyValue(K key); @@ -151,7 +215,7 @@ public abstract class BaseIndex, K extends Serializable> /* Base CRUD methods */ - protected abstract D toDoc(Map fields); + protected abstract D toDoc(Map fields); public D getByKey(K key) { GetResponse response = getClient().prepareGet() @@ -169,11 +233,11 @@ public abstract class BaseIndex, K extends Serializable> protected void updateDocument(Collection requests, K key) throws Exception { LOG.debug("UPDATE _id:{} in index {}", key, this.getIndexName()); BulkRequestBuilder bulkRequest = getClient().prepareBulk(); - for(UpdateRequest request : requests){ + for (UpdateRequest request : requests) { bulkRequest.add(request - .index(this.getIndexName()) - .id(this.getKeyValue(key)) - .type(this.getIndexType())); + .index(this.getIndexName()) + .id(this.getKeyValue(key)) + .type(this.getIndexType())); } bulkRequest.get(); } @@ -249,28 +313,6 @@ public abstract class BaseIndex, K extends Serializable> } } - /* Synchronization methods */ - - Long lastSynch = 0L; - Long cooldown = 30000L; - - @Override - public void setLastSynchronization(Long time) { - if (time > (getLastSynchronization() + cooldown)) { - LOG.trace("Updating synchTime updating"); - lastSynch = time; - } else { - LOG.trace("Not updating synchTime, still cooling down"); - } - - } - - @Override - public Long getLastSynchronization() { - //TODO need to read that in the admin index; - return 0L; - } - /* ES QueryHelper Methods */ diff --git a/sonar-server/src/main/java/org/sonar/server/search/Index.java b/sonar-server/src/main/java/org/sonar/server/search/Index.java index b59563ada4c..4997ace8bd2 100644 --- a/sonar-server/src/main/java/org/sonar/server/search/Index.java +++ b/sonar-server/src/main/java/org/sonar/server/search/Index.java @@ -56,8 +56,8 @@ public interface Index, K extends Serializable> extends Star void deleteByDto(E dto); - Long getLastSynchronization(); + java.util.Date getLastSynchronization(); - void setLastSynchronization(Long time); + IndexStat getIndexStat(); } diff --git a/sonar-server/src/main/java/org/sonar/server/search/IndexClient.java b/sonar-server/src/main/java/org/sonar/server/search/IndexClient.java index 6586e37618d..87813d21580 100644 --- a/sonar-server/src/main/java/org/sonar/server/search/IndexClient.java +++ b/sonar-server/src/main/java/org/sonar/server/search/IndexClient.java @@ -40,15 +40,6 @@ public class IndexClient implements ServerComponent { } } - public Long getLastUpdate(String indexType){ - for(Index index:indexComponents.values()){ - if(index.getIndexType().equals(indexType)){ - return index.getLastSynchronization(); - } - } - throw new IllegalStateException("no index for type '"+indexType+"' is registered"); - } - public K get(Class clazz){ return (K) this.indexComponents.get(clazz); } diff --git a/sonar-server/src/main/java/org/sonar/server/search/IndexDefinition.java b/sonar-server/src/main/java/org/sonar/server/search/IndexDefinition.java index 64f128fcea2..d171be8a4dd 100644 --- a/sonar-server/src/main/java/org/sonar/server/search/IndexDefinition.java +++ b/sonar-server/src/main/java/org/sonar/server/search/IndexDefinition.java @@ -19,15 +19,20 @@ */ package org.sonar.server.search; +import com.google.common.annotations.VisibleForTesting; + public class IndexDefinition { private final String indexName; private final String indexType; + private final String MANAGEMENT_INDEX = "sonarindex"; + private final String MANAGEMENT_TYPE = "index"; + private IndexDefinition(String indexName, String indexType) { this.indexName = indexName; this.indexType = indexType; - } +} public String getIndexName() { return indexName; @@ -37,6 +42,17 @@ public class IndexDefinition { return indexType; } + public String getManagementIndex() { + return MANAGEMENT_INDEX; + } + + public String getManagementType() { + return MANAGEMENT_TYPE; + } + public static IndexDefinition RULE = new IndexDefinition("rules","rules"); public static IndexDefinition ACTIVE_RULE = new IndexDefinition("rules","activeRules"); + + @VisibleForTesting + protected static IndexDefinition TEST = new IndexDefinition("test","test"); } diff --git a/sonar-server/src/main/java/org/sonar/server/search/IndexStat.java b/sonar-server/src/main/java/org/sonar/server/search/IndexStat.java new file mode 100644 index 00000000000..131b8c03bdc --- /dev/null +++ b/sonar-server/src/main/java/org/sonar/server/search/IndexStat.java @@ -0,0 +1,56 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.search; + +import java.util.Date; + +/** + * @since 4.4 + */ +public class IndexStat { + + private final Date statTime; + private Date lastUpdate; + private Long documentCount; + + public IndexStat() { + this.statTime = new Date(); + } + + public Date getStatTime() { + return statTime; + } + + public Date getLastUpdate() { + return lastUpdate; + } + + public void setLastUpdate(Date lastUpdate) { + this.lastUpdate = lastUpdate; + } + + public Long getDocumentCount() { + return documentCount; + } + + public void setDocumentCount(Long documentCount) { + this.documentCount = documentCount; + } +} diff --git a/sonar-server/src/test/java/org/sonar/server/search/BaseIndexTest.java b/sonar-server/src/test/java/org/sonar/server/search/BaseIndexTest.java new file mode 100644 index 00000000000..9887e52808f --- /dev/null +++ b/sonar-server/src/test/java/org/sonar/server/search/BaseIndexTest.java @@ -0,0 +1,111 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.search; + + +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.sonar.api.config.Settings; +import org.sonar.api.platform.ServerFileSystem; +import org.sonar.core.cluster.NullQueue; + +import java.io.File; +import java.io.IOException; +import java.io.Serializable; +import java.util.Map; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.fest.assertions.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class BaseIndexTest { + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + ESNode node; + + @Before + public void setup() throws IOException { + File homeDir = temp.newFolder(); + ServerFileSystem fs = mock(ServerFileSystem.class); + when(fs.getHomeDir()).thenReturn(homeDir); + node = new ESNode(fs, new Settings()); + node.start(); + } + + @After + public void tearDown(){ + node.stop(); + } + + @Test + public void can_load(){ + BaseIndex index = getIndex(this.node); + assertThat(index).isNotNull(); + } + + @Test + public void creates_management_index(){ + BaseIndex index = getIndex(this.node); + + IndicesExistsResponse indexExistsResponse = index.getClient().admin().indices() + .prepareExists(IndexDefinition.TEST.getManagementIndex()).execute().actionGet(); + + assertThat(indexExistsResponse.isExists()).isTrue(); + } + + + + + private BaseIndex getIndex(final ESNode esNode){ + BaseIndex index = new BaseIndex( + IndexDefinition.TEST, + null, new NullQueue(), esNode){ + @Override + protected String getKeyValue(Serializable key) { + return null; + } + + @Override + protected XContentBuilder getIndexSettings() throws IOException { + return jsonBuilder().startObject().endObject(); + } + + @Override + protected XContentBuilder getMapping() throws IOException { + return null; + } + + @Override + protected Object toDoc(Map fields) { + return null; + } + }; + index.start(); + return index; + } +} \ No newline at end of file -- 2.39.5