From: olivier lamy Date: Fri, 28 Jul 2017 23:22:14 +0000 (+1000) Subject: trying more memory too see what happen on Jenkins X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=d93ae430d5b2d21f98500669eb79b64a6c38c56a;p=archiva.git trying more memory too see what happen on Jenkins Signed-off-by: olivier lamy --- diff --git a/archiva-modules/archiva-web/archiva-rest/archiva-rest-services/pom.xml b/archiva-modules/archiva-web/archiva-rest/archiva-rest-services/pom.xml index 56c3cde39..f69d776d2 100644 --- a/archiva-modules/archiva-web/archiva-rest/archiva-rest-services/pom.xml +++ b/archiva-modules/archiva-web/archiva-rest/archiva-rest-services/pom.xml @@ -439,7 +439,7 @@ **/*Tests.java **/*Test.java - -Xmx512m -Xms512m -server -XX:MaxPermSize=256m @{jacocoproperty} + -Xms1024m -Xmx2048m -server -XX:MaxPermSize=256m @{jacocoproperty} ${project.build.directory}/appserver-base ${project.build.directory}/appserver-base diff --git a/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrRepositorySessionFactory.java b/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrRepositorySessionFactory.java index fa502798d..0b775c796 100644 --- a/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrRepositorySessionFactory.java +++ b/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrRepositorySessionFactory.java @@ -33,6 +33,7 @@ import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Service; import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; import javax.inject.Inject; import javax.jcr.Repository; import javax.jcr.RepositoryException; @@ -44,7 +45,7 @@ import java.util.Map; /** * */ -@Service("repositorySessionFactory#jcr") +@Service( "repositorySessionFactory#jcr" ) public class JcrRepositorySessionFactory implements RepositorySessionFactory { @@ -64,6 +65,8 @@ public class JcrRepositorySessionFactory @Inject private RepositorySessionFactoryBean repositorySessionFactoryBean; + private RepositoryFactory repositoryFactory; + @Override public RepositorySession createSession() { @@ -86,8 +89,10 @@ public class JcrRepositorySessionFactory } // Lazy evaluation to avoid problems with circular dependencies during initialization - private MetadataResolver getMetadataResolver() { - if (this.metadataResolver==null) { + private MetadataResolver getMetadataResolver() + { + if ( this.metadataResolver == null ) + { this.metadataResolver = applicationContext.getBean( MetadataResolver.class ); } return this.metadataResolver; @@ -124,11 +129,12 @@ public class JcrRepositorySessionFactory JcrMetadataRepository metadataRepository = null; try { - RepositoryFactory factory = new RepositoryFactory(); + + repositoryFactory = new RepositoryFactory(); // FIXME this need to be configurable - Path directoryPath = Paths.get(System.getProperty( "appserver.base" ), "data/jcr"); - factory.setRepositoryPath( directoryPath ); - repository = factory.createRepository(); + Path directoryPath = Paths.get( System.getProperty( "appserver.base" ), "data/jcr" ); + repositoryFactory.setRepositoryPath( directoryPath ); + repository = repositoryFactory.createRepository(); metadataRepository = new JcrMetadataRepository( metadataFacetFactories, repository ); JcrMetadataRepository.initialize( metadataRepository.getJcrSession() ); } @@ -147,4 +153,10 @@ public class JcrRepositorySessionFactory stopWatch.stop(); logger.info( "time to initialize JcrRepositorySessionFactory: {}", stopWatch.getTime() ); } + + @PreDestroy + public void close() + { + repositoryFactory.close(); + } } diff --git a/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/RepositoryFactory.java b/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/RepositoryFactory.java index 097838001..e2be03f40 100644 --- a/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/RepositoryFactory.java +++ b/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/RepositoryFactory.java @@ -20,6 +20,7 @@ package org.apache.archiva.metadata.repository.jcr; */ import com.google.common.collect.ImmutableSet; +import org.apache.commons.lang.time.StopWatch; import org.apache.jackrabbit.JcrConstants; import org.apache.jackrabbit.oak.Oak; import org.apache.jackrabbit.oak.api.Type; @@ -57,12 +58,10 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.Executors; +import static org.apache.archiva.metadata.repository.jcr.RepositoryFactory.StoreType.IN_MEMORY_TYPE; +import static org.apache.archiva.metadata.repository.jcr.RepositoryFactory.StoreType.SEGMENT_FILE_TYPE; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INCLUDE_PROPERTY_TYPES; /** @@ -74,109 +73,127 @@ import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstant public class RepositoryFactory { - Logger log = LoggerFactory.getLogger( RepositoryFactory.class ); + private Logger log = LoggerFactory.getLogger( RepositoryFactory.class ); - public static final String SEGMENT_FILE_TYPE = "oak-segment-tar"; - public static final String IN_MEMORY_TYPE = "oak-memory"; + private FileStore fileStore; + + public enum StoreType + { + SEGMENT_FILE_TYPE, + IN_MEMORY_TYPE; + } - String storeType = SEGMENT_FILE_TYPE; + + private StoreType storeType = SEGMENT_FILE_TYPE; Path repositoryPath = Paths.get( "repository" ); - public Repository createRepository( ) throws IOException, InvalidFileStoreVersionException + public Repository createRepository() + throws IOException, InvalidFileStoreVersionException { NodeStore nodeStore; - if ( SEGMENT_FILE_TYPE.equals( storeType ) ) + if ( SEGMENT_FILE_TYPE == storeType ) + { + fileStore = FileStoreBuilder.fileStoreBuilder( repositoryPath.toFile() ).build(); + nodeStore = SegmentNodeStoreBuilders.builder( fileStore ).build(); + } + else if ( IN_MEMORY_TYPE == storeType ) { - FileStore fs = FileStoreBuilder.fileStoreBuilder( repositoryPath.toFile( ) ).build( ); - nodeStore = SegmentNodeStoreBuilders.builder( fs ).build( ); - } else if (IN_MEMORY_TYPE.equals(storeType)) { nodeStore = null; - } else { - throw new IllegalArgumentException( "Store type "+storeType+" not recognized" ); + } + else + { + throw new IllegalArgumentException( "Store type " + storeType + " not recognized" ); } - Oak oak = nodeStore==null ? new Oak() : new Oak(nodeStore); - oak.with( new RepositoryInitializer( ) + Oak oak = nodeStore == null ? new Oak() : new Oak( nodeStore ); + oak.with( new RepositoryInitializer() { @Override public void initialize( @Nonnull NodeBuilder root ) { - log.info("Creating index "); + log.info( "Creating index " ); - NodeBuilder lucene = IndexUtils.getOrCreateOakIndex( root ).child("lucene"); - lucene.setProperty( JcrConstants.JCR_PRIMARYTYPE, "oak:QueryIndexDefinition", Type.NAME); + NodeBuilder lucene = IndexUtils.getOrCreateOakIndex( root ).child( "lucene" ); + lucene.setProperty( JcrConstants.JCR_PRIMARYTYPE, "oak:QueryIndexDefinition", Type.NAME ); - lucene.setProperty("compatVersion", 2); - lucene.setProperty("type", "lucene"); + lucene.setProperty( "compatVersion", 2 ); + lucene.setProperty( "type", "lucene" ); // lucene.setProperty("async", "async"); - lucene.setProperty(INCLUDE_PROPERTY_TYPES, - ImmutableSet.of("String"), Type.STRINGS); + lucene.setProperty( INCLUDE_PROPERTY_TYPES, ImmutableSet.of( "String" ), Type.STRINGS ); // lucene.setProperty("refresh",true); - lucene.setProperty("async",ImmutableSet.of("async", "sync"), Type.STRINGS); - NodeBuilder rules = lucene.child("indexRules"). - setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME); - rules.setProperty(":childOrder",ImmutableSet.of("archiva:projectVersion","archiva:artifact", - "archiva:facet","archiva:namespace", "archiva:project"), Type.STRINGS); - NodeBuilder allProps = rules.child("archiva:projectVersion") - .child("properties").setProperty( JcrConstants.JCR_PRIMARYTYPE, - "nt:unstructured", Type.NAME) - .setProperty( ":childOrder", ImmutableSet.of("allProps"), Type.STRINGS ) - .setProperty("indexNodeName",true) - .child("allProps").setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME); - allProps.setProperty("name", ".*"); - allProps.setProperty("isRegexp", true); - allProps.setProperty("nodeScopeIndex", true); - allProps.setProperty("index",true); - allProps.setProperty("analyzed",true); + lucene.setProperty( "async", ImmutableSet.of( "async", "sync" ), Type.STRINGS ); + NodeBuilder rules = lucene.child( "indexRules" ). + setProperty( JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME ); + rules.setProperty( ":childOrder", ImmutableSet.of( "archiva:projectVersion", // + "archiva:artifact", // + "archiva:facet", // + "archiva:namespace", // + "archiva:project" ), // + Type.STRINGS ); + NodeBuilder allProps = rules.child( "archiva:projectVersion" ) // + .child( "properties" ) // + .setProperty( JcrConstants.JCR_PRIMARYTYPE, "nt:unstructured", Type.NAME ) // + .setProperty( ":childOrder", ImmutableSet.of( "allProps" ), Type.STRINGS ) // + .setProperty( "indexNodeName", true ) // + .child( "allProps" ) // + .setProperty( JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME ); + allProps.setProperty( "name", ".*" ); + allProps.setProperty( "isRegexp", true ); + allProps.setProperty( "nodeScopeIndex", true ); + allProps.setProperty( "index", true ); + allProps.setProperty( "analyzed", true ); // allProps.setProperty("propertyIndex",true); - allProps = rules.child("archiva:artifact") - .child("properties").setProperty( JcrConstants.JCR_PRIMARYTYPE, - "nt:unstructured", Type.NAME) - .setProperty( ":childOrder", ImmutableSet.of("allProps"), Type.STRINGS ) - .setProperty("indexNodeName",true) - .child("allProps").setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME); - allProps.setProperty("name", ".*"); - allProps.setProperty("isRegexp", true); - allProps.setProperty("nodeScopeIndex", true); - allProps.setProperty("index",true); - allProps.setProperty("analyzed",true); - allProps = rules.child("archiva:facet") - .child("properties").setProperty( JcrConstants.JCR_PRIMARYTYPE, - "nt:unstructured", Type.NAME) - .setProperty( ":childOrder", ImmutableSet.of("allProps"), Type.STRINGS ) - .setProperty("indexNodeName",true) - .child("allProps").setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME); - allProps.setProperty("name", ".*"); - allProps.setProperty("isRegexp", true); - allProps.setProperty("nodeScopeIndex", true); - allProps.setProperty("index",true); - allProps.setProperty("analyzed",true); - allProps = rules.child("archiva:namespace") - .child("properties").setProperty( JcrConstants.JCR_PRIMARYTYPE, - "nt:unstructured", Type.NAME) - .setProperty( ":childOrder", ImmutableSet.of("allProps"), Type.STRINGS ) - .setProperty("indexNodeName",true) - .child("allProps").setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME); - allProps.setProperty("name", ".*"); - allProps.setProperty("isRegexp", true); - allProps.setProperty("nodeScopeIndex", true); - allProps.setProperty("index",true); - allProps.setProperty("analyzed",true); - allProps = rules.child("archiva:project") - .child("properties").setProperty( JcrConstants.JCR_PRIMARYTYPE, - "nt:unstructured", Type.NAME) - .setProperty( ":childOrder", ImmutableSet.of("allProps"), Type.STRINGS ) - .setProperty("indexNodeName",true) - .child("allProps").setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME); - allProps.setProperty("name", ".*"); - allProps.setProperty("isRegexp", true); - allProps.setProperty("nodeScopeIndex", true); - allProps.setProperty("index",true); - allProps.setProperty("analyzed",true); + allProps = rules.child( "archiva:artifact" ) // + .child( "properties" ) // + .setProperty( JcrConstants.JCR_PRIMARYTYPE, "nt:unstructured", Type.NAME ) // + .setProperty( ":childOrder", ImmutableSet.of( "allProps" ), Type.STRINGS ) // + .setProperty( "indexNodeName", true ).child( "allProps" ) // + .setProperty( JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME ); + allProps.setProperty( "name", ".*" ); + allProps.setProperty( "isRegexp", true ); + allProps.setProperty( "nodeScopeIndex", true ); + allProps.setProperty( "index", true ); + allProps.setProperty( "analyzed", true ); + allProps = rules.child( "archiva:facet" ) // + .child( "properties" ) // + .setProperty( JcrConstants.JCR_PRIMARYTYPE, "nt:unstructured", Type.NAME ) // + .setProperty( ":childOrder", ImmutableSet.of( "allProps" ), Type.STRINGS ) // + .setProperty( "indexNodeName", true ) // + .child( "allProps" ) // + .setProperty( JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME ); + allProps.setProperty( "name", ".*" ); + allProps.setProperty( "isRegexp", true ); + allProps.setProperty( "nodeScopeIndex", true ); + allProps.setProperty( "index", true ); + allProps.setProperty( "analyzed", true ); + allProps = rules.child( "archiva:namespace" ) // + .child( "properties" ) // + .setProperty( JcrConstants.JCR_PRIMARYTYPE, "nt:unstructured", Type.NAME ) // + .setProperty( ":childOrder", ImmutableSet.of( "allProps" ), Type.STRINGS ) // + .setProperty( "indexNodeName", true ) // + .child( "allProps" ) // + .setProperty( JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME ); + allProps.setProperty( "name", ".*" ); + allProps.setProperty( "isRegexp", true ); + allProps.setProperty( "nodeScopeIndex", true ); + allProps.setProperty( "index", true ); + allProps.setProperty( "analyzed", true ); + allProps = rules.child( "archiva:project" ) // + .child( "properties" ) // + .setProperty( JcrConstants.JCR_PRIMARYTYPE, "nt:unstructured", Type.NAME ) // + .setProperty( ":childOrder", ImmutableSet.of( "allProps" ), Type.STRINGS ) // + .setProperty( "indexNodeName", true ) // + .child( "allProps" ) // + .setProperty( JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME ); + allProps.setProperty( "name", ".*" ); + allProps.setProperty( "isRegexp", true ); + allProps.setProperty( "nodeScopeIndex", true ); + allProps.setProperty( "index", true ); + allProps.setProperty( "analyzed", true ); - log.info("Index: "+lucene+" myIndex "+lucene.getChildNode( "myIndex" )); - log.info("myIndex "+lucene.getChildNode( "myIndex" ).getProperties()); + log.info( "Index: {} myIndex {}", lucene, lucene.getChildNode( "myIndex" ) ); + log.info( "myIndex {}", lucene.getChildNode( "myIndex" ).getProperties() ); // IndexUtils.createIndexDefinition( ) } @@ -184,58 +201,73 @@ public class RepositoryFactory ExecutorService executorService = createExecutor(); StatisticsProvider statsProvider = StatisticsProvider.NOOP; - int queueSize = Integer.getInteger("queueSize", 10000); + int queueSize = Integer.getInteger( "queueSize", 10000 ); File indexDir = Files.createTempDirectory( "archiva_index" ).toFile(); - log.info("Queue Index "+indexDir.toString()); + log.info( "Queue Index {}", indexDir.toString() ); IndexCopier indexCopier = new IndexCopier( executorService, indexDir, true ); - NRTIndexFactory nrtIndexFactory = new NRTIndexFactory( indexCopier, statsProvider); - MountInfoProvider mountInfoProvider = Mounts.defaultMountInfoProvider( ); - IndexTracker tracker = new IndexTracker(new DefaultIndexReaderFactory( mountInfoProvider, indexCopier ), nrtIndexFactory); - DocumentQueue queue = new DocumentQueue(queueSize, tracker, executorService, statsProvider); - LocalIndexObserver localIndexObserver = new LocalIndexObserver( queue, statsProvider); - LuceneIndexProvider provider = new LuceneIndexProvider(tracker); + NRTIndexFactory nrtIndexFactory = new NRTIndexFactory( indexCopier, statsProvider ); + MountInfoProvider mountInfoProvider = Mounts.defaultMountInfoProvider(); + IndexTracker tracker = + new IndexTracker( new DefaultIndexReaderFactory( mountInfoProvider, indexCopier ), nrtIndexFactory ); + DocumentQueue queue = new DocumentQueue( queueSize, tracker, executorService, statsProvider ); + LocalIndexObserver localIndexObserver = new LocalIndexObserver( queue, statsProvider ); + LuceneIndexProvider provider = new LuceneIndexProvider( tracker ); // ExternalObserverBuilder builder = new ExternalObserverBuilder(queue, tracker, statsProvider, // executorService, queueSize); // Observer observer = builder.build(); // builder.getBackgroundObserver(); - LuceneIndexEditorProvider editorProvider = new LuceneIndexEditorProvider(null, - tracker, new ExtractedTextCache(0, 0), null, mountInfoProvider); - editorProvider.setIndexingQueue(queue); - + LuceneIndexEditorProvider editorProvider = // + new LuceneIndexEditorProvider( null, tracker, // + new ExtractedTextCache( 0, 0 ), // + null, mountInfoProvider ); + editorProvider.setIndexingQueue( queue ); - log.info("Oak: "+oak+" with nodeStore "+nodeStore); - Jcr jcr = new Jcr(oak).with( editorProvider ).with( (Observer) provider ) - .with(localIndexObserver) + log.info( "Oak: {} with nodeStore {}", oak, nodeStore ); + Jcr jcr = new Jcr( oak ).with( editorProvider ) // + .with( (Observer) provider ) // + .with( localIndexObserver ) // .with(observer) - .with( ( QueryIndexProvider) provider ) - .withAsyncIndexing("async",5 ); + .with( (QueryIndexProvider) provider ); // + //.withAsyncIndexing( "async", 5 ); + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); Repository r = jcr.createRepository(); - try - { - Thread.currentThread().sleep(1000); - } - catch ( InterruptedException e ) - { - e.printStackTrace( ); - } + stopWatch.stop(); + log.info( "time to create jcr repository: {} ms", stopWatch.getTime() ); +// try +// { +// Thread.currentThread().sleep( 1000 ); +// } +// catch ( InterruptedException e ) +// { +// log.error( e.getMessage(), e ); +// } return r; } - public String getStoreType( ) + public void close() + { + if ( fileStore != null ) + { + fileStore.close(); + } + } + + public StoreType getStoreType() { return storeType; } - public void setStoreType( String storeType ) + public void setStoreType( StoreType storeType ) { this.storeType = storeType; } - public Path getRepositoryPath( ) + public Path getRepositoryPath() { return repositoryPath; } @@ -256,37 +288,50 @@ public class RepositoryFactory } catch ( IOException e ) { - e.printStackTrace( ); + log.error( e.getMessage(), e ); + throw new IllegalArgumentException( "cannot create directory:" + repositoryPath, e ); } } } - private ExecutorService createExecutor() { - ThreadPoolExecutor executor = new ThreadPoolExecutor(0, 5, 60L, TimeUnit.SECONDS, - new LinkedBlockingQueue(), new ThreadFactory() { - private final AtomicInteger counter = new AtomicInteger(); - private final Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() { - @Override - public void uncaughtException(Thread t, Throwable e) { - log.warn("Error occurred in asynchronous processing ", e); - } - }; - @Override - public Thread newThread(@Nonnull Runnable r) { - Thread thread = new Thread(r, createName()); - thread.setDaemon(true); - thread.setPriority(Thread.MIN_PRIORITY); - thread.setUncaughtExceptionHandler(handler); - return thread; - } - - private String createName() { - return "oak-lucene-" + counter.getAndIncrement(); - } - }); - executor.setKeepAliveTime(1, TimeUnit.MINUTES); - executor.allowCoreThreadTimeOut(true); - return executor; + private ExecutorService createExecutor() + { + return Executors.newCachedThreadPool(); +// +// ThreadPoolExecutor executor = +// new ThreadPoolExecutor( 0, 5, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), +// new ThreadFactory() +// { +// private final AtomicInteger counter = new AtomicInteger(); +// +// private final Thread.UncaughtExceptionHandler handler = +// new Thread.UncaughtExceptionHandler() +// { +// @Override +// public void uncaughtException( Thread t, Throwable e ) +// { +// log.warn( "Error occurred in asynchronous processing ", e ); +// } +// }; +// +// @Override +// public Thread newThread( @Nonnull Runnable r ) +// { +// Thread thread = new Thread( r, createName() ); +// thread.setDaemon( true ); +// thread.setPriority( Thread.MIN_PRIORITY ); +// thread.setUncaughtExceptionHandler( handler ); +// return thread; +// } +// +// private String createName() +// { +// return "oak-lucene-" + counter.getAndIncrement(); +// } +// } ); +// executor.setKeepAliveTime( 1, TimeUnit.MINUTES ); +// executor.allowCoreThreadTimeOut( true ); +// return executor; } } diff --git a/pom.xml b/pom.xml index 96218db5a..4dc86ac77 100644 --- a/pom.xml +++ b/pom.xml @@ -71,7 +71,7 @@ 1.7.25 2.8.2 - 4.3.9.RELEASE + 4.3.10.RELEASE 2.0 UTF-8