From 7d2cc26b4bf41a5e9429aba3a54665669cf6cf79 Mon Sep 17 00:00:00 2001 From: Martin Stockhammer Date: Wed, 21 Jun 2017 23:52:17 +0200 Subject: [PATCH] Adding jcr oak implementation --- .../archiva-lucene-consumers/pom.xml | 35 ++- .../src/test/resources/spring-context.xml | 3 +- .../archiva-base/archiva-indexer/pom.xml | 10 + .../archiva-plexus-bridge/pom.xml | 10 + .../archiva-repository-admin-api/pom.xml | 8 + .../archiva-scheduler-indexing/pom.xml | 10 + .../archiva-rest-services/pom.xml | 16 - .../archiva-web/archiva-web-common/pom.xml | 15 - .../archiva-web/archiva-webapp/pom.xml | 23 -- .../resources/META-INF/spring-context.xml | 3 +- .../archiva-web/archiva-webdav/pom.xml | 12 - .../plugins/metadata-store-jcr/pom.xml | 85 +++++- .../jcr/ArchivaJcrRepositoryConfig.java | 64 ---- .../repository/jcr/JcrMetadataRepository.java | 22 +- .../repository/jcr/RepositoryFactory.java | 285 ++++++++++++++++++ .../src/main/resources/log4j2-test.xml | 42 +++ .../META-INF/spring-context.xml | 14 +- .../src/test/resources/log4j2-test.xml | 42 +++ .../plugins/repository-statistics/pom.xml | 16 - pom.xml | 31 +- 20 files changed, 537 insertions(+), 209 deletions(-) delete mode 100644 archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/ArchivaJcrRepositoryConfig.java create mode 100644 archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/RepositoryFactory.java create mode 100644 archiva-modules/plugins/metadata-store-jcr/src/main/resources/log4j2-test.xml create mode 100644 archiva-modules/plugins/metadata-store-jcr/src/test/resources/log4j2-test.xml diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/pom.xml b/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/pom.xml index 6eed0b863..e8b970b3e 100644 --- a/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/pom.xml +++ b/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/pom.xml @@ -66,10 +66,30 @@ commons-io commons-io + + + org.apache.lucene + lucene-queryparser + + + org.apache.lucene + lucene-analyzers-common + + org.apache.maven.indexer indexer-core + + + org.apache.lucene + lucene-queryparser + + + org.apache.lucene + lucene-analyzers-common + + javax.inject @@ -110,21 +130,6 @@ metadata-store-jcr test - - org.apache.jackrabbit - jackrabbit-core - test - - - commons-logging - commons-logging - - - org.slf4j - jcl-over-slf4j - - - xerces xercesImpl diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/test/resources/spring-context.xml b/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/test/resources/spring-context.xml index db01db735..a89137295 100644 --- a/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/test/resources/spring-context.xml +++ b/archiva-modules/archiva-base/archiva-consumers/archiva-lucene-consumers/src/test/resources/spring-context.xml @@ -59,6 +59,7 @@ + \ No newline at end of file diff --git a/archiva-modules/archiva-base/archiva-indexer/pom.xml b/archiva-modules/archiva-base/archiva-indexer/pom.xml index 5695593b7..eae5642a9 100644 --- a/archiva-modules/archiva-base/archiva-indexer/pom.xml +++ b/archiva-modules/archiva-base/archiva-indexer/pom.xml @@ -85,6 +85,16 @@ org.apache.maven.indexer indexer-core + + + org.apache.lucene + lucene-queryparser + + + org.apache.lucene + lucene-analyzers-common + + org.eclipse.sisu diff --git a/archiva-modules/archiva-base/archiva-plexus-bridge/pom.xml b/archiva-modules/archiva-base/archiva-plexus-bridge/pom.xml index fe40f6284..d8b477e3f 100644 --- a/archiva-modules/archiva-base/archiva-plexus-bridge/pom.xml +++ b/archiva-modules/archiva-base/archiva-plexus-bridge/pom.xml @@ -58,6 +58,16 @@ org.apache.maven.indexer indexer-core + + + org.apache.lucene + lucene-queryparser + + + org.apache.lucene + lucene-analyzers-common + + org.codehaus.plexus diff --git a/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-api/pom.xml b/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-api/pom.xml index a57c21fb2..d1ca7d768 100644 --- a/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-api/pom.xml +++ b/archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-api/pom.xml @@ -73,6 +73,14 @@ com.google.code.atinject atinject + + org.apache.lucene + lucene-queryparser + + + org.apache.lucene + lucene-analyzers-common + diff --git a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/pom.xml b/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/pom.xml index 5aca8e1ad..58a5f5bb1 100644 --- a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/pom.xml +++ b/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/pom.xml @@ -44,6 +44,16 @@ org.apache.maven.indexer indexer-core + + + org.apache.lucene + lucene-queryparser + + + org.apache.lucene + lucene-analyzers-common + + com.google.guava diff --git a/archiva-modules/archiva-web/archiva-rest/archiva-rest-services/pom.xml b/archiva-modules/archiva-web/archiva-rest/archiva-rest-services/pom.xml index 882cd6f25..56c3cde39 100644 --- a/archiva-modules/archiva-web/archiva-rest/archiva-rest-services/pom.xml +++ b/archiva-modules/archiva-web/archiva-rest/archiva-rest-services/pom.xml @@ -328,22 +328,6 @@ test - - org.apache.jackrabbit - jackrabbit-core - test - - - commons-logging - commons-logging - - - org.slf4j - jcl-over-slf4j - - - - org.apache.lucene lucene-core diff --git a/archiva-modules/archiva-web/archiva-web-common/pom.xml b/archiva-modules/archiva-web/archiva-web-common/pom.xml index 71ac238ad..645419ffb 100644 --- a/archiva-modules/archiva-web/archiva-web-common/pom.xml +++ b/archiva-modules/archiva-web/archiva-web-common/pom.xml @@ -368,21 +368,6 @@ test - - org.apache.jackrabbit - jackrabbit-core - test - - - commons-logging - commons-logging - - - org.slf4j - jcl-over-slf4j - - - org.assertj assertj-core diff --git a/archiva-modules/archiva-web/archiva-webapp/pom.xml b/archiva-modules/archiva-web/archiva-webapp/pom.xml index 1d3c53ed1..bdb0b6b68 100644 --- a/archiva-modules/archiva-web/archiva-webapp/pom.xml +++ b/archiva-modules/archiva-web/archiva-webapp/pom.xml @@ -130,29 +130,6 @@ metadata-store-cassandra - - org.apache.jackrabbit - jackrabbit-jcr-commons - - - org.apache.jackrabbit - jackrabbit-core - - - commons-logging - commons-logging - - - org.slf4j - jcl-over-slf4j - - - commons-dbcp - commons-dbcp - - - - commons-configuration commons-configuration diff --git a/archiva-modules/archiva-web/archiva-webapp/src/main/resources/META-INF/spring-context.xml b/archiva-modules/archiva-web/archiva-webapp/src/main/resources/META-INF/spring-context.xml index b0b3840b6..a66c9253b 100755 --- a/archiva-modules/archiva-web/archiva-webapp/src/main/resources/META-INF/spring-context.xml +++ b/archiva-modules/archiva-web/archiva-webapp/src/main/resources/META-INF/spring-context.xml @@ -37,6 +37,7 @@ + diff --git a/archiva-modules/archiva-web/archiva-webdav/pom.xml b/archiva-modules/archiva-web/archiva-webdav/pom.xml index 2b08b3f29..044b964e8 100644 --- a/archiva-modules/archiva-web/archiva-webdav/pom.xml +++ b/archiva-modules/archiva-web/archiva-webdav/pom.xml @@ -94,16 +94,6 @@ org.apache.archiva archiva-filelock - - org.apache.jackrabbit - jackrabbit-webdav - - - org.slf4j - jcl-over-slf4j - - - joda-time joda-time @@ -298,8 +288,6 @@ org.apache.archiva.metadata.repository.storage;version=${project.version}, org.apache.commons.io;version="[1.4,2)", org.apache.commons.lang;version="[2.4,3)", - org.apache.jackrabbit.util;version="[1,3)", - org.apache.jackrabbit.webdav*;version="[1,3)", org.joda.time*;version="[1.5,2)", org.springframework*;version="[3,4)", org.apache.maven.index*, diff --git a/archiva-modules/plugins/metadata-store-jcr/pom.xml b/archiva-modules/plugins/metadata-store-jcr/pom.xml index 8fa30860c..831b639b0 100644 --- a/archiva-modules/plugins/metadata-store-jcr/pom.xml +++ b/archiva-modules/plugins/metadata-store-jcr/pom.xml @@ -28,6 +28,9 @@ metadata-store-jcr bundle Archiva Core Plugins :: JCR Storage for Metadata + + 1.7.1 + org.apache.archiva @@ -48,6 +51,12 @@ archiva-test-utils ${project.version} test + + + org.slf4j + slf4j-simple + + javax.jcr @@ -55,24 +64,41 @@ org.apache.jackrabbit - jackrabbit-jcr-commons + oak-jcr + ${jcr-oak.version} org.apache.jackrabbit - jackrabbit-core - provided - + oak-segment-tar + ${jcr-oak.version} + + + org.apache.jackrabbit + oak-lucene + ${jcr-oak.version} - org.slf4j - jcl-over-slf4j + org.apache.lucene + lucene-misc + + + org.apache.lucene + lucene-facet + + org.apache.logging.log4j + log4j-core + org.apache.logging.log4j log4j-jcl + + org.apache.logging.log4j + log4j-slf4j-impl + org.springframework spring-test @@ -98,6 +124,51 @@ xercesImpl test + + org.apache.lucene + lucene-core + ${lucene.version} + + + org.apache.lucene + lucene-misc + ${lucene.version} + + + org.apache.lucene + lucene-facet + ${lucene.version} + + + org.apache.lucene + lucene-codecs + ${lucene.version} + + + org.apache.lucene + lucene-queryparser + ${lucene.version} + + + org.apache.lucene + lucene-sandbox + ${lucene.version} + + + org.apache.lucene + lucene-highlighter + ${lucene.version} + + + org.apache.lucene + lucene-memory + ${lucene.version} + + + org.apache.lucene + lucene-analyzers-common + ${lucene.version} + @@ -126,8 +197,6 @@ org.apache.archiva.metadata.model;version=${project.version}, org.apache.archiva.metadata.repository;version=${project.version}, org.apache.commons.lang;version="[2.4,3)", - org.apache.jackrabbit.commons;version="[2.3,3)", - org.apache.jackrabbit.core.config, org.springframework*;version="[3,4)", org.slf4j;resolution:=optional, org.apache.commons.lang.time diff --git a/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/ArchivaJcrRepositoryConfig.java b/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/ArchivaJcrRepositoryConfig.java deleted file mode 100644 index 8271e2bec..000000000 --- a/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/ArchivaJcrRepositoryConfig.java +++ /dev/null @@ -1,64 +0,0 @@ -package org.apache.archiva.metadata.repository.jcr; -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import org.apache.jackrabbit.core.config.ConfigurationException; -import org.apache.jackrabbit.core.config.RepositoryConfig; -import org.slf4j.LoggerFactory; - -import java.io.File; - -/** - * @author Olivier Lamy - * @since 1.4-M1 - */ -public class ArchivaJcrRepositoryConfig -{ - // Recommended to avoid inconsistencies caused by race conditions. - static - { - if ( !System.getProperties( ).containsKey( "org.apache.jackrabbit.core.state.validatehierarchy" ) ) - { - System.setProperty( "org.apache.jackrabbit.core.state.validatehierarchy", "true" ); - } - } - - public static RepositoryConfig create( String file, String home ) - throws ConfigurationException - { - File homeFile = new File( home ); - if ( !homeFile.exists( ) ) - { - homeFile.mkdirs( ); - } - - File configurationFile = new File( file ); - if ( !configurationFile.exists( ) ) - { - String resourcePath = "org/apache/archiva/metadata/repository/jcr/repository.xml"; - LoggerFactory.getLogger( ArchivaJcrRepositoryConfig.class ).info( - "no repository.xml file in path {} so use default from resources path {}", file, resourcePath ); - // use bundled repository.xml - return RepositoryConfig.create( - Thread.currentThread( ).getContextClassLoader( ).getResourceAsStream( resourcePath ), home ); - } - - return RepositoryConfig.create( file, home ); - } -} diff --git a/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java b/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java index 364930998..2bf94063f 100644 --- a/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java +++ b/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java @@ -37,6 +37,8 @@ import org.apache.archiva.metadata.repository.MetadataRepositoryException; import org.apache.archiva.metadata.repository.MetadataResolutionException; import org.apache.commons.lang.StringUtils; import org.apache.jackrabbit.commons.JcrUtils; +import org.apache.jackrabbit.oak.plugins.index.IndexUtils; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -139,6 +141,8 @@ public class JcrMetadataRepository registerMixinNodeType( nodeTypeManager, JcrMetadataRepository.FACET_NODE_TYPE ); registerMixinNodeType( nodeTypeManager, JcrMetadataRepository.DEPENDENCY_NODE_TYPE ); + + } private static void registerMixinNodeType( NodeTypeManager nodeTypeManager, String name ) @@ -156,6 +160,8 @@ public class JcrMetadataRepository } } + + @Override public void updateProject( String repositoryId, ProjectMetadata project ) throws MetadataRepositoryException @@ -755,6 +761,10 @@ public class JcrMetadataRepository { throw new MetadataRepositoryException( e.getMessage(), e ); } + log.info("Artifacts found {}", artifacts.size()); + for (ArtifactMetadata meta : artifacts) { + log.info("Artifact: "+meta.getVersion()+" "+meta.getFacetList()); + } return artifacts; } @@ -1418,11 +1428,9 @@ public class JcrMetadataRepository } @Override - public List searchArtifacts( String key, String text, String repositoryId, boolean exact ) + public List searchArtifacts( String key, String text, String repositoryId, boolean e ) throws MetadataRepositoryException { - // we can't do exact search in any property (*), we need a key - boolean e = exact && key != null; String theKey = key == null ? "*" : "[" + key + "]"; String projectVersionCondition = e ? "(projectVersion." + theKey + " = $value)" : "contains([projectVersion]." + theKey + ", $value)"; @@ -1432,7 +1440,6 @@ public class JcrMetadataRepository + ARTIFACT_NODE_TYPE + "] AS artifact ON ISCHILDNODE(artifact, projectVersion) LEFT OUTER JOIN [" + FACET_NODE_TYPE + "] AS facet ON ISCHILDNODE(facet, projectVersion) WHERE (" + projectVersionCondition + " OR " + facetCondition + ")"; - return runJcrQuery( repositoryId, q, ImmutableMap.of( "value", text ) ); } @@ -1555,7 +1562,7 @@ public class JcrMetadataRepository private static String getRepositoryContentPath( String repositoryId ) { - return getRepositoryPath( repositoryId ) + "/content/"; + return getRepositoryPath( repositoryId ) + "/content"; } private static String getFacetPath( String repositoryId, String facetId ) @@ -1565,7 +1572,7 @@ public class JcrMetadataRepository private static String getNamespacePath( String repositoryId, String namespace ) { - return getRepositoryContentPath( repositoryId ) + namespace.replace( '.', '/' ); + return getRepositoryContentPath( repositoryId ) + "/" + namespace.replace( '.', '/' ); } private static String getProjectPath( String repositoryId, String namespace, String projectId ) @@ -1594,6 +1601,7 @@ public class JcrMetadataRepository private Node getOrAddNodeByPath( Node baseNode, String name, String nodeType ) throws RepositoryException { + log.debug("getOrAddNodeByPath"+baseNode+" "+name+" "+nodeType); Node node = baseNode; for ( String n : name.split( "/" ) ) { @@ -1614,8 +1622,10 @@ public class JcrMetadataRepository private Node getOrAddRepositoryNode( String repositoryId ) throws RepositoryException { + log.debug("getOrAddRepositoryNode "+repositoryId); Node root = getJcrSession().getRootNode(); Node node = JcrUtils.getOrAddNode( root, "repositories" ); + log.debug("Repositories "+node); node = JcrUtils.getOrAddNode( node, repositoryId ); return node; } diff --git a/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/RepositoryFactory.java b/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/RepositoryFactory.java new file mode 100644 index 000000000..65e1acec8 --- /dev/null +++ b/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/RepositoryFactory.java @@ -0,0 +1,285 @@ +package org.apache.archiva.metadata.repository.jcr; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.MoreExecutors; +import org.apache.jackrabbit.JcrConstants; +import org.apache.jackrabbit.oak.Oak; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.jcr.Jcr; +import org.apache.jackrabbit.oak.plugins.index.IndexConstants; +import org.apache.jackrabbit.oak.plugins.index.IndexUtils; +import org.apache.jackrabbit.oak.plugins.index.lucene.ExtractedTextCache; +import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier; +import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker; +import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider; +import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider; +import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.DocumentQueue; +import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.ExternalObserverBuilder; +import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.LocalIndexObserver; +import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.NRTIndexFactory; +import org.apache.jackrabbit.oak.plugins.index.lucene.reader.DefaultIndexReaderFactory; +import org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneInitializerHelper; +import org.apache.jackrabbit.oak.segment.SegmentNodeStore; +import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders; +import org.apache.jackrabbit.oak.segment.file.FileStore; +import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder; +import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException; +import org.apache.jackrabbit.oak.spi.commit.Observer; +import org.apache.jackrabbit.oak.spi.lifecycle.RepositoryInitializer; +import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider; +import org.apache.jackrabbit.oak.spi.mount.Mounts; +import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.apache.jackrabbit.oak.stats.StatisticsProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nonnull; +import javax.jcr.Repository; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME; +import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INCLUDE_PROPERTY_TYPES; + +/** + * Created by martin on 14.06.17. + * + * @author Martin Stockhammer + * @since 3.0.0 + */ +public class RepositoryFactory +{ + + Logger log = LoggerFactory.getLogger( RepositoryFactory.class ); + + public static final String SEGMENT_FILE_TYPE = "oak-segment-tar"; + public static final String IN_MEMORY_TYPE = "oak-memory"; + + String storeType = SEGMENT_FILE_TYPE; + + Path repositoryPath = Paths.get( "repository" ); + + public Repository createRepository( ) throws IOException, InvalidFileStoreVersionException + { + NodeStore nodeStore; + if ( SEGMENT_FILE_TYPE.equals( storeType ) ) + { + FileStore fs = FileStoreBuilder.fileStoreBuilder( repositoryPath.toFile( ) ).build( ); + nodeStore = SegmentNodeStoreBuilders.builder( fs ).build( ); + } else if (IN_MEMORY_TYPE.equals(storeType)) { + nodeStore = null; + } else { + throw new IllegalArgumentException( "Store type "+storeType+" not recognized" ); + } + + Oak oak = nodeStore==null ? new Oak() : new Oak(nodeStore); + oak.with( new RepositoryInitializer( ) + { + @Override + public void initialize( @Nonnull NodeBuilder root ) + { + log.info("Creating index "); + + NodeBuilder lucene = IndexUtils.getOrCreateOakIndex( root ).child("lucene"); + lucene.setProperty( JcrConstants.JCR_PRIMARYTYPE, "oak:QueryIndexDefinition", Type.NAME); + + lucene.setProperty("compatVersion", 2); + lucene.setProperty("type", "lucene"); + // lucene.setProperty("async", "async"); + lucene.setProperty(INCLUDE_PROPERTY_TYPES, + ImmutableSet.of("String"), Type.STRINGS); + // lucene.setProperty("refresh",true); + lucene.setProperty("async",ImmutableSet.of("async", "sync"), Type.STRINGS); + NodeBuilder rules = lucene.child("indexRules"). + setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME); + rules.setProperty(":childOrder",ImmutableSet.of("archiva:projectVersion","archiva:artifact", + "archiva:facet","archiva:namespace", "archiva:project"), Type.STRINGS); + NodeBuilder allProps = rules.child("archiva:projectVersion") + .child("properties").setProperty( JcrConstants.JCR_PRIMARYTYPE, + "nt:unstructured", Type.NAME) + .setProperty( ":childOrder", ImmutableSet.of("allProps"), Type.STRINGS ) + .setProperty("indexNodeName",true) + .child("allProps").setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME); + allProps.setProperty("name", ".*"); + allProps.setProperty("isRegexp", true); + allProps.setProperty("nodeScopeIndex", true); + allProps.setProperty("index",true); + allProps.setProperty("analyzed",true); + // allProps.setProperty("propertyIndex",true); + allProps = rules.child("archiva:artifact") + .child("properties").setProperty( JcrConstants.JCR_PRIMARYTYPE, + "nt:unstructured", Type.NAME) + .setProperty( ":childOrder", ImmutableSet.of("allProps"), Type.STRINGS ) + .setProperty("indexNodeName",true) + .child("allProps").setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME); + allProps.setProperty("name", ".*"); + allProps.setProperty("isRegexp", true); + allProps.setProperty("nodeScopeIndex", true); + allProps.setProperty("index",true); + allProps.setProperty("analyzed",true); + allProps = rules.child("archiva:facet") + .child("properties").setProperty( JcrConstants.JCR_PRIMARYTYPE, + "nt:unstructured", Type.NAME) + .setProperty( ":childOrder", ImmutableSet.of("allProps"), Type.STRINGS ) + .setProperty("indexNodeName",true) + .child("allProps").setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME); + allProps.setProperty("name", ".*"); + allProps.setProperty("isRegexp", true); + allProps.setProperty("nodeScopeIndex", true); + allProps.setProperty("index",true); + allProps.setProperty("analyzed",true); + allProps = rules.child("archiva:namespace") + .child("properties").setProperty( JcrConstants.JCR_PRIMARYTYPE, + "nt:unstructured", Type.NAME) + .setProperty( ":childOrder", ImmutableSet.of("allProps"), Type.STRINGS ) + .setProperty("indexNodeName",true) + .child("allProps").setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME); + allProps.setProperty("name", ".*"); + allProps.setProperty("isRegexp", true); + allProps.setProperty("nodeScopeIndex", true); + allProps.setProperty("index",true); + allProps.setProperty("analyzed",true); + allProps = rules.child("archiva:project") + .child("properties").setProperty( JcrConstants.JCR_PRIMARYTYPE, + "nt:unstructured", Type.NAME) + .setProperty( ":childOrder", ImmutableSet.of("allProps"), Type.STRINGS ) + .setProperty("indexNodeName",true) + .child("allProps").setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME); + allProps.setProperty("name", ".*"); + allProps.setProperty("isRegexp", true); + allProps.setProperty("nodeScopeIndex", true); + allProps.setProperty("index",true); + allProps.setProperty("analyzed",true); + + log.info("Index: "+lucene+" myIndex "+lucene.getChildNode( "myIndex" )); + log.info("myIndex "+lucene.getChildNode( "myIndex" ).getProperties()); + // IndexUtils.createIndexDefinition( ) + + } + } ); + + ExecutorService executorService = createExecutor(); + StatisticsProvider statsProvider = StatisticsProvider.NOOP; + int queueSize = Integer.getInteger("queueSize", 10000); + File indexDir = Files.createTempDirectory( "archiva_index" ).toFile(); + log.info("Queue Index "+indexDir.toString()); + IndexCopier indexCopier = new IndexCopier( executorService, indexDir, true ); + NRTIndexFactory nrtIndexFactory = new NRTIndexFactory( indexCopier, statsProvider); + MountInfoProvider mountInfoProvider = Mounts.defaultMountInfoProvider( ); + IndexTracker tracker = new IndexTracker(new DefaultIndexReaderFactory( mountInfoProvider, indexCopier ), nrtIndexFactory); + DocumentQueue queue = new DocumentQueue(queueSize, tracker, executorService, statsProvider); + LocalIndexObserver localIndexObserver = new LocalIndexObserver( queue, statsProvider); + LuceneIndexProvider provider = new LuceneIndexProvider(tracker); + + // ExternalObserverBuilder builder = new ExternalObserverBuilder(queue, tracker, statsProvider, +// executorService, queueSize); +// Observer observer = builder.build(); +// builder.getBackgroundObserver(); + + LuceneIndexEditorProvider editorProvider = new LuceneIndexEditorProvider(null, + tracker, new ExtractedTextCache(0, 0), null, mountInfoProvider); + editorProvider.setIndexingQueue(queue); + + + log.info("Oak: "+oak+" with nodeStore "+nodeStore); + Jcr jcr = new Jcr(oak).with( editorProvider ).with( (Observer) provider ) + .with(localIndexObserver) + // .with(observer) + .with( ( QueryIndexProvider) provider ) + .withAsyncIndexing("async",5 ); + Repository r = jcr.createRepository(); + try + { + Thread.currentThread().sleep(1000); + } + catch ( InterruptedException e ) + { + e.printStackTrace( ); + } + return r; + + + } + + public String getStoreType( ) + { + return storeType; + } + + public void setStoreType( String storeType ) + { + this.storeType = storeType; + } + + public Path getRepositoryPath( ) + { + return repositoryPath; + } + + public void setRepositoryPath( Path repositoryPath ) + { + this.repositoryPath = repositoryPath; + } + + public void setRepositoryPath( String repositoryPath ) + { + this.repositoryPath = Paths.get( repositoryPath ); + if ( !Files.exists( this.repositoryPath ) ) + { + try + { + Files.createDirectories( this.repositoryPath ); + } + catch ( IOException e ) + { + e.printStackTrace( ); + } + } + } + + private ExecutorService createExecutor() { + ThreadPoolExecutor executor = new ThreadPoolExecutor(0, 5, 60L, TimeUnit.SECONDS, + new LinkedBlockingQueue(), new ThreadFactory() { + private final AtomicInteger counter = new AtomicInteger(); + private final Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() { + @Override + public void uncaughtException(Thread t, Throwable e) { + log.warn("Error occurred in asynchronous processing ", e); + } + }; + @Override + public Thread newThread(@Nonnull Runnable r) { + Thread thread = new Thread(r, createName()); + thread.setDaemon(true); + thread.setPriority(Thread.MIN_PRIORITY); + thread.setUncaughtExceptionHandler(handler); + return thread; + } + + private String createName() { + return "oak-lucene-" + counter.getAndIncrement(); + } + }); + executor.setKeepAliveTime(1, TimeUnit.MINUTES); + executor.allowCoreThreadTimeOut(true); + return executor; + } + +} diff --git a/archiva-modules/plugins/metadata-store-jcr/src/main/resources/log4j2-test.xml b/archiva-modules/plugins/metadata-store-jcr/src/main/resources/log4j2-test.xml new file mode 100644 index 000000000..74dd0773f --- /dev/null +++ b/archiva-modules/plugins/metadata-store-jcr/src/main/resources/log4j2-test.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/archiva-modules/plugins/metadata-store-jcr/src/test/filtered-resources/META-INF/spring-context.xml b/archiva-modules/plugins/metadata-store-jcr/src/test/filtered-resources/META-INF/spring-context.xml index 43f6d0829..f98b79bd1 100644 --- a/archiva-modules/plugins/metadata-store-jcr/src/test/filtered-resources/META-INF/spring-context.xml +++ b/archiva-modules/plugins/metadata-store-jcr/src/test/filtered-resources/META-INF/spring-context.xml @@ -19,14 +19,14 @@ --> + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:util="http://www.springframework.org/schema/util" + xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd"> - - + + - - - + + \ No newline at end of file diff --git a/archiva-modules/plugins/metadata-store-jcr/src/test/resources/log4j2-test.xml b/archiva-modules/plugins/metadata-store-jcr/src/test/resources/log4j2-test.xml new file mode 100644 index 000000000..74dd0773f --- /dev/null +++ b/archiva-modules/plugins/metadata-store-jcr/src/test/resources/log4j2-test.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/archiva-modules/plugins/repository-statistics/pom.xml b/archiva-modules/plugins/repository-statistics/pom.xml index 2d20acacb..ccbdbfc49 100644 --- a/archiva-modules/plugins/repository-statistics/pom.xml +++ b/archiva-modules/plugins/repository-statistics/pom.xml @@ -44,10 +44,6 @@ javax.jcr jcr - - org.apache.jackrabbit - jackrabbit-jcr-commons - org.apache.archiva archiva-test-utils @@ -59,17 +55,6 @@ log4j-jcl test - - org.apache.jackrabbit - jackrabbit-core - test - - - org.slf4j - jcl-over-slf4j - - - org.mockito mockito-all @@ -96,7 +81,6 @@ javax.jcr, javax.jcr.query, - org.apache.jackrabbit.commons;version="[2.3,3)", org.apache.archiva.metadata.model;version=${project.version}, org.apache.archiva.metadata.repository;version=${project.version}, org.apache.archiva.metadata.repository.storage.maven2;version=${project.version}, diff --git a/pom.xml b/pom.xml index 8e12a4c54..f335da7f4 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,7 @@ 1.6 3.2.1 1.2.2 - 2.4 + 2.5 2.6 3.0.5 @@ -59,7 +59,8 @@ 1.13.1 2.1 - 5.1.1 + + 6.0-SNAPSHOT 2.9 3.0.0-SNAPSHOT 2.2 @@ -75,7 +76,7 @@ 2.0 UTF-8 true - 3.6.2 + 4.10.4 2.14.1 3.0.3 @@ -126,17 +127,6 @@ 1.7.1 test - - org.apache.jackrabbit - jackrabbit-webdav - ${jackrabbit.version} - - - commons-logging - commons-logging - - - xerces xercesImpl @@ -1317,16 +1307,7 @@ jcr ${javax.jcr.version} - - org.apache.jackrabbit - jackrabbit-jcr-commons - ${jackrabbit.version} - - - org.apache.jackrabbit - jackrabbit-core - ${jackrabbit.version} - + @@ -1623,7 +1604,7 @@ org.apache.maven.plugins maven-surefire-plugin - @{jacocoproperty} + ${surefire.redirectTestOutputToFile} alphabetical -- 2.39.5