diff options
8 files changed, 195 insertions, 25 deletions
diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java b/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java index 35cb739359a..9e8a9566d69 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java @@ -36,6 +36,7 @@ import org.elasticsearch.action.count.CountRequestBuilder; import org.elasticsearch.action.deletebyquery.DeleteByQueryRequestBuilder; import org.elasticsearch.action.get.GetRequestBuilder; import org.elasticsearch.action.get.MultiGetRequestBuilder; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchScrollRequestBuilder; import org.elasticsearch.client.Client; @@ -46,23 +47,7 @@ import org.picocontainer.Startable; import org.sonar.core.profiling.Profiling; import org.sonar.server.search.ClusterHealth; import org.sonar.server.search.SearchClient; -import org.sonar.server.search.request.ProxyBulkRequestBuilder; -import org.sonar.server.search.request.ProxyClusterHealthRequestBuilder; -import org.sonar.server.search.request.ProxyClusterStateRequestBuilder; -import org.sonar.server.search.request.ProxyClusterStatsRequestBuilder; -import org.sonar.server.search.request.ProxyCountRequestBuilder; -import org.sonar.server.search.request.ProxyCreateIndexRequestBuilder; -import org.sonar.server.search.request.ProxyDeleteByQueryRequestBuilder; -import org.sonar.server.search.request.ProxyFlushRequestBuilder; -import org.sonar.server.search.request.ProxyGetRequestBuilder; -import org.sonar.server.search.request.ProxyIndicesExistsRequestBuilder; -import org.sonar.server.search.request.ProxyIndicesStatsRequestBuilder; -import org.sonar.server.search.request.ProxyMultiGetRequestBuilder; -import org.sonar.server.search.request.ProxyNodesStatsRequestBuilder; -import org.sonar.server.search.request.ProxyPutMappingRequestBuilder; -import org.sonar.server.search.request.ProxyRefreshRequestBuilder; -import org.sonar.server.search.request.ProxySearchRequestBuilder; -import org.sonar.server.search.request.ProxySearchScrollRequestBuilder; +import org.sonar.server.search.request.*; /** * Facade to connect to Elasticsearch node. Handles correctly errors (logging + exceptions @@ -169,6 +154,10 @@ public class EsClient implements Startable { return new ProxyDeleteByQueryRequestBuilder(client, profiling).setIndices(indices); } + public IndexRequestBuilder prepareIndex(String index, String type) { + return new ProxyIndexRequestBuilder(client, profiling).setIndex(index).setType(type); + } + public long getLastUpdatedAt(String indexName, String typeName) { SearchRequestBuilder request = prepareSearch(indexName) .setTypes(typeName) diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java b/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java index 2dc9b64f54d..b7a594aa4f8 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java +++ b/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java @@ -159,6 +159,7 @@ import org.sonar.server.rule.index.RuleNormalizer; import org.sonar.server.rule.ws.*; import org.sonar.server.search.*; import org.sonar.server.source.*; +import org.sonar.server.source.index.SourceLineIndex; import org.sonar.server.source.index.SourceLineIndexDefinition; import org.sonar.server.source.index.SourceLineIndexer; import org.sonar.server.source.ws.*; @@ -548,6 +549,7 @@ class ServerComponents { pico.addSingleton(RawAction.class); pico.addSingleton(ScmAction.class); pico.addSingleton(SourceLineIndexDefinition.class); + pico.addSingleton(SourceLineIndex.class); pico.addSingleton(SourceLineIndexer.class); pico.addSingleton(IndexSourceLinesStep.class); diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/request/ProxyIndexRequestBuilder.java b/server/sonar-server/src/main/java/org/sonar/server/search/request/ProxyIndexRequestBuilder.java new file mode 100644 index 00000000000..26751f0e004 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/search/request/ProxyIndexRequestBuilder.java @@ -0,0 +1,76 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.search.request; + +import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.unit.TimeValue; +import org.sonar.core.profiling.Profiling; +import org.sonar.core.profiling.StopWatch; + +public class ProxyIndexRequestBuilder extends IndexRequestBuilder { + + private final Profiling profiling; + + public ProxyIndexRequestBuilder(Client client, Profiling profiling) { + super(client); + this.profiling = profiling; + } + + @Override + public IndexResponse get() { + StopWatch fullProfile = profiling.start("index", Profiling.Level.FULL); + try { + return super.execute().actionGet(); + } catch (Exception e) { + throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); + } finally { + if (profiling.isProfilingEnabled(Profiling.Level.FULL)) { + fullProfile.stop("%s", toString()); + } + } + } + + @Override + public IndexResponse get(TimeValue timeout) { + throw new IllegalStateException("Not yet implemented"); + } + + @Override + public IndexResponse get(String timeout) { + throw new IllegalStateException("Not yet implemented"); + } + + @Override + public ListenableActionFuture<IndexResponse> execute() { + throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); + } + + @Override + public String toString() { + StringBuilder message = new StringBuilder().append("ES index request"); + message.append(String.format(" for key '%s'", request.id())); + message.append(String.format(" on index '%s'", request.index())); + message.append(String.format(" on type '%s'", request.type())); + return message.toString(); + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndex.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndex.java new file mode 100644 index 00000000000..36b39b968e2 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndex.java @@ -0,0 +1,42 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.source.index; + +import org.elasticsearch.index.query.QueryBuilders; +import org.sonar.api.ServerComponent; +import org.sonar.server.es.EsClient; + +public class SourceLineIndex implements ServerComponent { + + private final EsClient esClient; + + public SourceLineIndex(EsClient esClient) { + this.esClient = esClient; + } + + public void deleteLinesFromFileAbove(String fileUuid, int lastLine) { + esClient.prepareDeleteByQuery(SourceLineIndexDefinition.INDEX_SOURCE_LINES) + .setTypes(SourceLineIndexDefinition.TYPE_SOURCE_LINE) + .setQuery(QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery(SourceLineIndexDefinition.FIELD_FILE_UUID, fileUuid)) + .must(QueryBuilders.rangeQuery(SourceLineIndexDefinition.FIELD_LINE).gt(lastLine)) + ).get(); + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java index 8dfa2cedf2a..2895663d2cc 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java @@ -37,11 +37,13 @@ public class SourceLineIndexer implements ServerComponent { private final DbClient dbClient; private final EsClient esClient; + private final SourceLineIndex index; private long lastUpdatedAt = 0L; - public SourceLineIndexer(DbClient dbClient, EsClient esClient) { + public SourceLineIndexer(DbClient dbClient, EsClient esClient, SourceLineIndex index) { this.dbClient = dbClient; this.esClient = esClient; + this.index = index; } public void indexSourceLines(boolean large) { @@ -65,14 +67,18 @@ public class SourceLineIndexer implements ServerComponent { bulk.start(); while (sourceLines.hasNext()) { Collection<SourceLineDoc> lineDocs = sourceLines.next(); + String fileUuid = null; + int lastLine = 0; for (SourceLineDoc sourceLine: lineDocs) { + lastLine ++; + fileUuid = sourceLine.fileUuid(); bulk.add(newUpsertRequest(sourceLine)); long dtoUpdatedAt = sourceLine.updateDate().getTime(); if (lastUpdatedAt < dtoUpdatedAt) { lastUpdatedAt = dtoUpdatedAt; } } - + index.deleteLinesFromFileAbove(fileUuid, lastLine); } bulk.stop(); } diff --git a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java index 3cc23526c6f..43ebbfcb8bd 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java @@ -22,6 +22,10 @@ package org.sonar.server.source.index; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import org.apache.commons.io.IOUtils; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.index.query.QueryBuilders; +import org.fest.assertions.MapAssert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -30,14 +34,15 @@ import org.sonar.server.db.DbClient; import org.sonar.server.es.BulkIndexer; import org.sonar.server.es.EsTester; import org.sonar.server.search.BaseNormalizer; -import org.sonar.server.source.index.SourceLineDoc; -import org.sonar.server.source.index.SourceLineIndexDefinition; -import org.sonar.server.source.index.SourceLineIndexer; +import org.sonar.test.TestUtils; +import java.io.FileInputStream; import java.util.Collection; import java.util.Date; import java.util.List; +import java.util.Map; +import static org.fest.assertions.Assertions.assertThat; import static org.mockito.Mockito.mock; public class SourceLineIndexerTest { @@ -45,21 +50,32 @@ public class SourceLineIndexerTest { @Rule public EsTester es = new EsTester().addDefinitions(new SourceLineIndexDefinition(new Settings())); + private SourceLineIndex index; + private SourceLineIndexer indexer; @Before public void setUp() { - indexer = new SourceLineIndexer(mock(DbClient.class), es.client()); + index = new SourceLineIndex(es.client()); + indexer = new SourceLineIndexer(mock(DbClient.class), es.client(), index); } @Test - public void should_index_source_lines() { + public void should_index_source_lines() throws Exception { + es.client().prepareIndex(SourceLineIndexDefinition.INDEX_SOURCE_LINES, SourceLineIndexDefinition.TYPE_SOURCE_LINE) + .setSource(IOUtils.toString(new FileInputStream(TestUtils.getResource(this.getClass(), "line2.json")))) + .get(); + es.client().prepareIndex(SourceLineIndexDefinition.INDEX_SOURCE_LINES, SourceLineIndexDefinition.TYPE_SOURCE_LINE) + .setSource(IOUtils.toString(new FileInputStream(TestUtils.getResource(this.getClass(), "line2_other_file.json")))) + .setRefresh(true) + .get(); + BulkIndexer bulk = new BulkIndexer(es.client(), SourceLineIndexDefinition.INDEX_SOURCE_LINES); SourceLineDoc line1 = new SourceLineDoc(ImmutableMap.<String, Object>builder() .put(SourceLineIndexDefinition.FIELD_PROJECT_UUID, "abcd") .put(SourceLineIndexDefinition.FIELD_FILE_UUID, "efgh") - .put(SourceLineIndexDefinition.FIELD_LINE, 42) + .put(SourceLineIndexDefinition.FIELD_LINE, 1) .put(SourceLineIndexDefinition.FIELD_SCM_REVISION, "cafebabe") .put(SourceLineIndexDefinition.FIELD_SCM_DATE, "2014-01-01T12:34:56.7+0100") .put(SourceLineIndexDefinition.FIELD_SCM_AUTHOR, "polop") @@ -71,5 +87,24 @@ public class SourceLineIndexerTest { List<Collection<SourceLineDoc>> sourceLineContainer = Lists.newArrayList(); sourceLineContainer.add(sourceLines); indexer.indexSourceLines(bulk, sourceLineContainer.iterator()); + + assertThat(es.countDocuments(SourceLineIndexDefinition.INDEX_SOURCE_LINES, SourceLineIndexDefinition.TYPE_SOURCE_LINE)).isEqualTo(2L); + + SearchResponse fileSearch = es.client().prepareSearch(SourceLineIndexDefinition.INDEX_SOURCE_LINES) + .setTypes(SourceLineIndexDefinition.TYPE_SOURCE_LINE) + .setQuery(QueryBuilders.termQuery(SourceLineIndexDefinition.FIELD_FILE_UUID, "efgh")) + .get(); + assertThat(fileSearch.getHits().getTotalHits()).isEqualTo(1L); + Map<String, Object> fields = fileSearch.getHits().getHits()[0].sourceAsMap(); + assertThat(fields).hasSize(8); + assertThat(fields).includes( + MapAssert.entry(SourceLineIndexDefinition.FIELD_PROJECT_UUID, "abcd"), + MapAssert.entry(SourceLineIndexDefinition.FIELD_FILE_UUID, "efgh"), + MapAssert.entry(SourceLineIndexDefinition.FIELD_LINE, 1), + MapAssert.entry(SourceLineIndexDefinition.FIELD_SCM_REVISION, "cafebabe"), + MapAssert.entry(SourceLineIndexDefinition.FIELD_SCM_DATE, "2014-01-01T12:34:56.7+0100"), + MapAssert.entry(SourceLineIndexDefinition.FIELD_SCM_AUTHOR, "polop"), + MapAssert.entry(SourceLineIndexDefinition.FIELD_SOURCE, "package org.sonar.server.source;") + ); } } diff --git a/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineIndexerTest/line2.json b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineIndexerTest/line2.json new file mode 100644 index 00000000000..422dc023480 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineIndexerTest/line2.json @@ -0,0 +1,10 @@ +{ + "projectUuid": "abcd", + "fileUuid": "efgh", + "line": 2, + "scmAuthor": "polop", + "scmDate": "2014-01-01T12:34:56.7+01:00", + "scmRevision": "cafebabe", + "source": "// Empty", + "updatedAt": "2014-01-01T23:45:01.8+01:00" +}
\ No newline at end of file diff --git a/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineIndexerTest/line2_other_file.json b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineIndexerTest/line2_other_file.json new file mode 100644 index 00000000000..495d2684e51 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineIndexerTest/line2_other_file.json @@ -0,0 +1,10 @@ +{ + "projectUuid": "abcd", + "fileUuid": "fdsq", + "line": 2, + "scmAuthor": "polop", + "scmDate": "2014-01-01T12:34:56.7+01:00", + "scmRevision": "cafebabe", + "source": "// Empty", + "updatedAt": "2014-01-01T23:45:01.8+01:00" +}
\ No newline at end of file |