From 4820fe06b2666ba383083378da79e56c42706725 Mon Sep 17 00:00:00 2001 From: Julien Lancelot Date: Mon, 10 Nov 2014 14:59:19 +0100 Subject: [PATCH] Add periodic rate when inserting issues in E/S --- .../org/sonar/data/issues/AbstractTest.java | 32 -------------- .../data/issues/IssuesDbExtractionTest.java | 28 ++++++++++-- .../data/issues/IssuesIndexInjectionTest.java | 43 +++++++++++++++---- 3 files changed, 59 insertions(+), 44 deletions(-) diff --git a/server/sonar-data-test/src/test/java/org/sonar/data/issues/AbstractTest.java b/server/sonar-data-test/src/test/java/org/sonar/data/issues/AbstractTest.java index 404eaf4aa20..b8932803c88 100644 --- a/server/sonar-data-test/src/test/java/org/sonar/data/issues/AbstractTest.java +++ b/server/sonar-data-test/src/test/java/org/sonar/data/issues/AbstractTest.java @@ -23,12 +23,9 @@ package org.sonar.data.issues; import com.google.common.collect.Iterables; import org.apache.commons.lang.StringUtils; import org.apache.ibatis.io.Resources; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Rule; import org.junit.rules.TestName; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.sonar.api.issue.Issue; import org.sonar.api.rule.RuleKey; import org.sonar.api.rule.Severity; @@ -43,8 +40,6 @@ import java.io.FileReader; import java.util.Iterator; import java.util.List; import java.util.Properties; -import java.util.TimerTask; -import java.util.concurrent.atomic.AtomicLong; import static com.google.common.collect.Lists.newArrayList; import static org.fest.assertions.Assertions.assertThat; @@ -66,8 +61,6 @@ public class AbstractTest { static Properties properties = new Properties(); @Rule public TestName testName = new TestName(); - protected AtomicLong counter; - protected ProgressTask progressTask; @BeforeClass public static void loadProperties() { @@ -105,12 +98,6 @@ public class AbstractTest { return users; } - @Before - public void initCounter() throws Exception { - counter = new AtomicLong(0L); - progressTask = new ProgressTask(counter); - } - protected String getProperty(String test) { String currentUser = StringUtils.defaultString(properties.getProperty("user"), "default"); String property = currentUser + "." + test; @@ -146,23 +133,4 @@ public class AbstractTest { .setResolution(resolution); } - protected static class ProgressTask extends TimerTask { - public static final long PERIOD_MS = 60000L; - private static final Logger LOGGER = LoggerFactory.getLogger("PerformanceTests"); - private final AtomicLong counter; - - public ProgressTask(AtomicLong counter) { - this.counter = counter; - } - - @Override - public void run() { - log(); - } - - public void log() { - LOGGER.info(String.format("%d issues processed", counter.get())); - } - } - } diff --git a/server/sonar-data-test/src/test/java/org/sonar/data/issues/IssuesDbExtractionTest.java b/server/sonar-data-test/src/test/java/org/sonar/data/issues/IssuesDbExtractionTest.java index 96effcfc67c..5b395ae5886 100644 --- a/server/sonar-data-test/src/test/java/org/sonar/data/issues/IssuesDbExtractionTest.java +++ b/server/sonar-data-test/src/test/java/org/sonar/data/issues/IssuesDbExtractionTest.java @@ -38,22 +38,21 @@ import org.sonar.server.search.DbSynchronizationHandler; import java.util.Map; import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.atomic.AtomicLong; import static org.fest.assertions.Assertions.assertThat; public class IssuesDbExtractionTest extends AbstractTest { static final Logger LOGGER = LoggerFactory.getLogger(IssuesDbExtractionTest.class); - final static int PROJECTS_NUMBER = 100; final static int NUMBER_FILES_PER_PROJECT = 100; final static int NUMBER_ISSUES_PER_FILE = 100; - final static int ISSUE_COUNT = PROJECTS_NUMBER * NUMBER_FILES_PER_PROJECT * NUMBER_ISSUES_PER_FILE; - @Rule public TestDatabase db = new TestDatabase(); - + AtomicLong counter = new AtomicLong(0L); DbSession session; ProxyIssueDao issueDao; @@ -78,6 +77,7 @@ public class IssuesDbExtractionTest extends AbstractTest { public void extract_issues() throws Exception { int issueInsertCount = ISSUE_COUNT; + ProgressTask progressTask = new ProgressTask(counter); Timer timer = new Timer("Extract Issues"); timer.schedule(progressTask, ProgressTask.PERIOD_MS, ProgressTask.PERIOD_MS); try { @@ -136,6 +136,26 @@ public class IssuesDbExtractionTest extends AbstractTest { return (int) Math.round(ISSUE_COUNT / (time / 1000.0)); } + protected static class ProgressTask extends TimerTask { + + public static final long PERIOD_MS = 60000L; + private static final Logger LOGGER = LoggerFactory.getLogger("PerformanceTests"); + private final AtomicLong counter; + + public ProgressTask(AtomicLong counter) { + this.counter = counter; + } + + @Override + public void run() { + log(); + } + + public void log() { + LOGGER.info(String.format("%d issues processed", counter.get())); + } + } + class ProxyIssueDao extends IssueDao { public Integer synchronizedIssues = 0; diff --git a/server/sonar-data-test/src/test/java/org/sonar/data/issues/IssuesIndexInjectionTest.java b/server/sonar-data-test/src/test/java/org/sonar/data/issues/IssuesIndexInjectionTest.java index ea9b96fe89a..1347b24266d 100644 --- a/server/sonar-data-test/src/test/java/org/sonar/data/issues/IssuesIndexInjectionTest.java +++ b/server/sonar-data-test/src/test/java/org/sonar/data/issues/IssuesIndexInjectionTest.java @@ -40,6 +40,8 @@ import org.sonar.server.tester.ServerTester; import java.util.List; import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.atomic.AtomicLong; import static com.google.common.collect.Lists.newArrayList; import static org.fest.assertions.Assertions.assertThat; @@ -47,16 +49,14 @@ import static org.fest.assertions.Assertions.assertThat; public class IssuesIndexInjectionTest extends AbstractTest { static final Logger LOGGER = LoggerFactory.getLogger(IssuesIndexInjectionTest.class); - final static int PROJECTS_NUMBER = 100; final static int NUMBER_FILES_PER_PROJECT = 100; final static int NUMBER_ISSUES_PER_FILE = 100; - final static int ISSUE_COUNT = PROJECTS_NUMBER * NUMBER_FILES_PER_PROJECT * NUMBER_ISSUES_PER_FILE; @ClassRule public static ServerTester tester = new ServerTester(); - + AtomicLong counter = new AtomicLong(0L); DbSession batchSession; IssueIndex issueIndex; @@ -64,6 +64,10 @@ public class IssuesIndexInjectionTest extends AbstractTest { List projects = newArrayList(); ArrayListMultimap componentsByProjectId = ArrayListMultimap.create(); + protected static int documentPerSecond(long nbIssues, long time) { + return (int) Math.round(nbIssues / (time / 1000.0)); + } + @Before public void setUp() throws Exception { issueIndex = tester.get(IssueIndex.class); @@ -80,6 +84,7 @@ public class IssuesIndexInjectionTest extends AbstractTest { public void inject_issues() throws Exception { generateData(); + ProgressTask progressTask = new ProgressTask(counter); Timer timer = new Timer("Inject Issues"); timer.schedule(progressTask, ProgressTask.PERIOD_MS, ProgressTask.PERIOD_MS); try { @@ -99,9 +104,9 @@ public class IssuesIndexInjectionTest extends AbstractTest { assertThat(issueIndex.countAll()).isEqualTo(ISSUE_COUNT); - long time = stop - start; - LOGGER.info("Inserted {} Issues in {} ms with avg {} Issue/second", ISSUE_COUNT, time, documentPerSecond(time)); - assertDurationAround(time, Long.parseLong(getProperty("IssuesIndexInjectionTest.inject_issues"))); + long totalTime = stop - start; + LOGGER.info("Inserted {} Issues in {} ms with avg {} Issue/second", ISSUE_COUNT, totalTime, documentPerSecond(ISSUE_COUNT, totalTime)); + assertDurationAround(totalTime, Long.parseLong(getProperty("IssuesIndexInjectionTest.inject_issues"))); } finally { timer.cancel(); @@ -138,8 +143,30 @@ public class IssuesIndexInjectionTest extends AbstractTest { LOGGER.info("Generated data in {} ms", System.currentTimeMillis() - start); } - protected int documentPerSecond(long time) { - return (int) Math.round(ISSUE_COUNT / (time / 1000.0)); + protected static class ProgressTask extends TimerTask { + + public static final long PERIOD_MS = 60000L; + private static final Logger LOGGER = LoggerFactory.getLogger("PerformanceTests"); + private final AtomicLong counter; + private long currentStart; + private long previousTotal = 0; + + public ProgressTask(AtomicLong counter) { + this.counter = counter; + this.currentStart = System.currentTimeMillis(); + } + + @Override + public void run() { + log(); + } + + public void log() { + long currentNumberOfIssues = counter.get() - this.previousTotal; + LOGGER.info("{} issues inserted with avg {} issue/second", currentNumberOfIssues, documentPerSecond(currentNumberOfIssues, System.currentTimeMillis() - this.currentStart)); + this.previousTotal = counter.get(); + this.currentStart = System.currentTimeMillis(); + } } } -- 2.39.5