@@ -470,6 +470,15 @@ | |||
The directory for the indexes of this repository. | |||
</description> | |||
</field> | |||
<field> | |||
<name>packedIndexDir</name> | |||
<version>3.0.0+</version> | |||
<type>String</type> | |||
<defaultValue></defaultValue> | |||
<description> | |||
The directory for the packed indexes of this repository. | |||
</description> | |||
</field> | |||
<field> | |||
<name>description</name> | |||
<version>1.0.0+</version> |
@@ -123,10 +123,9 @@ public class MavenIndexManager implements ArchivaIndexManager { | |||
@Inject | |||
private ArtifactContextProducer artifactContextProducer; | |||
@Inject | |||
RepositoryRegistry repositoryRegistry; | |||
public static final String DEFAULT_INDEXER_DIR = ".indexer"; | |||
public static final String DEFAULT_PACKED_INDEX_DIR = ".index"; | |||
private ConcurrentSkipListSet<Path> activeContexts = new ConcurrentSkipListSet<>( ); | |||
@@ -523,6 +522,7 @@ public class MavenIndexManager implements ArchivaIndexManager { | |||
IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get(); | |||
try { | |||
icf.setLocalIndexPath(getIndexPath(repo)); | |||
icf.setLocalPackedIndexPath(getPackedIndexPath(repo)); | |||
} catch (IOException e) { | |||
log.error("Could not set local index path for {}. New URI: {}", repo.getId(), icf.getIndexPath()); | |||
} | |||
@@ -556,6 +556,33 @@ public class MavenIndexManager implements ArchivaIndexManager { | |||
return indexDirectory; | |||
} | |||
private Path getPackedIndexPath(Repository repo) throws IOException { | |||
IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get(); | |||
Path repoDir = repo.getLocalPath(); | |||
URI indexDir = icf.getPackedIndexPath(); | |||
Path indexDirectory = null; | |||
if ( ! StringUtils.isEmpty(indexDir.toString( ) ) ) | |||
{ | |||
indexDirectory = PathUtil.getPathFromUri( indexDir ); | |||
// not absolute so create it in repository directory | |||
if ( !indexDirectory.isAbsolute( ) ) | |||
{ | |||
indexDirectory = repoDir.resolve( indexDirectory ); | |||
} | |||
} | |||
else | |||
{ | |||
indexDirectory = repoDir.resolve( DEFAULT_PACKED_INDEX_DIR ); | |||
} | |||
if ( !Files.exists( indexDirectory ) ) | |||
{ | |||
Files.createDirectories( indexDirectory ); | |||
} | |||
return indexDirectory; | |||
} | |||
private IndexingContext createRemoteContext(RemoteRepository remoteRepository ) throws IOException | |||
{ | |||
Path appServerBase = archivaConfiguration.getAppServerBaseDir( ); |
@@ -20,27 +20,19 @@ package org.apache.archiva.proxy; | |||
*/ | |||
import net.sf.ehcache.CacheManager; | |||
import org.apache.archiva.admin.model.beans.ManagedRepository; | |||
import org.apache.archiva.admin.model.managed.ManagedRepositoryAdmin; | |||
import org.apache.archiva.admin.repository.managed.DefaultManagedRepositoryAdmin; | |||
import org.apache.archiva.common.plexusbridge.PlexusSisuBridge; | |||
import org.apache.archiva.configuration.ArchivaConfiguration; | |||
import org.apache.archiva.configuration.ManagedRepositoryConfiguration; | |||
import org.apache.archiva.configuration.ProxyConnectorConfiguration; | |||
import org.apache.archiva.configuration.RemoteRepositoryConfiguration; | |||
import org.apache.archiva.policies.CachedFailuresPolicy; | |||
import org.apache.archiva.policies.ChecksumPolicy; | |||
import org.apache.archiva.policies.PropagateErrorsDownloadPolicy; | |||
import org.apache.archiva.policies.PropagateErrorsOnUpdateDownloadPolicy; | |||
import org.apache.archiva.policies.ReleasesPolicy; | |||
import org.apache.archiva.policies.SnapshotsPolicy; | |||
import org.apache.archiva.policies.*; | |||
import org.apache.archiva.proxy.model.RepositoryProxyConnectors; | |||
import org.apache.archiva.repository.ManagedRepositoryContent; | |||
import org.apache.archiva.repository.RepositoryContentProvider; | |||
import org.apache.archiva.repository.RepositoryRegistry; | |||
import org.apache.archiva.repository.maven2.MavenManagedRepository; | |||
import org.apache.archiva.test.utils.ArchivaSpringJUnit4ClassRunner; | |||
import org.apache.commons.io.FileUtils; | |||
import org.apache.maven.index.NexusIndexer; | |||
import org.apache.maven.index.context.IndexingContext; | |||
import org.apache.maven.wagon.Wagon; | |||
@@ -124,10 +116,6 @@ public abstract class AbstractProxyTestCase | |||
WagonDelegate delegate; | |||
@Inject | |||
protected NexusIndexer nexusIndexer; | |||
@Before | |||
public void setUp() | |||
throws Exception | |||
@@ -155,9 +143,6 @@ public abstract class AbstractProxyTestCase | |||
applicationContext.getBean( RepositoryRegistry.class ).putRepository( repoConfig ); | |||
// to prevent windauze file leaking | |||
removeMavenIndexes(); | |||
repositoryRegistry.setArchivaConfiguration( config ); | |||
// Setup target (proxied to) repository. | |||
@@ -199,25 +184,6 @@ public abstract class AbstractProxyTestCase | |||
log.info( "\n.\\ {}() \\._________________________________________\n", name ); | |||
} | |||
@After | |||
public void shutdown() | |||
throws Exception | |||
{ | |||
removeMavenIndexes(); | |||
} | |||
protected void removeMavenIndexes() | |||
throws Exception | |||
{ | |||
for ( IndexingContext indexingContext : nexusIndexer.getIndexingContexts().values() ) | |||
{ | |||
nexusIndexer.removeIndexingContext( indexingContext, false ); | |||
} | |||
} | |||
protected void assertChecksums( Path expectedFile, String expectedSha1Contents, String expectedMd5Contents ) | |||
throws Exception | |||
{ |
@@ -54,6 +54,11 @@ public class AbstractRepository | |||
private String indexDirectory; | |||
/* | |||
* @since 3.0.0 | |||
*/ | |||
private String packedIndexDirectory; | |||
private String toStringCache = null; | |||
@@ -136,6 +141,7 @@ public class AbstractRepository | |||
public void setIndexDirectory( String indexDirectory ) | |||
{ | |||
this.toStringCache=null; | |||
this.indexDirectory = indexDirectory; | |||
} | |||
@@ -202,9 +208,19 @@ public class AbstractRepository | |||
} | |||
public void setType(String type) { | |||
toStringCache=null; | |||
this.type = type; | |||
} | |||
public String getPackedIndexDirectory() { | |||
return packedIndexDirectory; | |||
} | |||
public void setPackedIndexDirectory(String packedIndexDirectory) { | |||
toStringCache=null; | |||
this.packedIndexDirectory = packedIndexDirectory; | |||
} | |||
@Override | |||
public String toString() | |||
{ | |||
@@ -214,12 +230,13 @@ public class AbstractRepository | |||
{ | |||
final StringBuilder sb = new StringBuilder( ); | |||
sb.append( "AbstractRepository" ); | |||
sb.append( "{id='" ).append( id ).append( '\'' ); | |||
sb.append(", type='").append(type).append('\''); | |||
sb.append( ", name='" ).append( getLocaleString( name ) ).append( '\'' ); | |||
sb.append( ", layout='" ).append( layout ).append( '\'' ); | |||
sb.append( ", indexDirectory='" ).append( indexDirectory ).append( '\'' ); | |||
sb.append( ", description='" ).append( getLocaleString( description ) ).append( '\'' ); | |||
sb.append( "{ id=\"" ).append( id ).append( '"' ); | |||
sb.append( ", type=\"").append(type).append('"'); | |||
sb.append( ", name=\"" ).append( getLocaleString( name ) ).append( '"' ); | |||
sb.append( ", layout=\"" ).append( layout ).append( '"' ); | |||
sb.append( ", indexDirectory=\"" ).append( indexDirectory ).append( '"' ); | |||
sb.append( ", packedIndexDirectory=\"").append(packedIndexDirectory).append('"'); | |||
sb.append( ", description=\"" ).append( getLocaleString( description ) ).append( '"' ); | |||
sb.append( '}' ); | |||
toStringCache=sb.toString( ); | |||
return toStringCache; |
@@ -125,6 +125,7 @@ public abstract class AbstractRepositoryAdmin | |||
if (repo.supportsFeature( IndexCreationFeature.class )) { | |||
IndexCreationFeature icf = repo.getFeature( IndexCreationFeature.class ).get(); | |||
adminRepo.setIndexDirectory( convertUriToString( icf.getIndexPath() ) ); | |||
adminRepo.setPackedIndexDirectory(convertUriToString(icf.getPackedIndexPath())); | |||
} | |||
} | |||
@@ -134,6 +135,7 @@ public abstract class AbstractRepositoryAdmin | |||
repoConfig.setLayout( repo.getLayout() ); | |||
repoConfig.setDescription( repo.getDescription() ); | |||
repoConfig.setIndexDir( repo.getIndexDirectory() ); | |||
repoConfig.setPackedIndexDir(repo.getPackedIndexDirectory()); | |||
repoConfig.setType( StringUtils.isEmpty( repo.getType() ) ? "MAVEN" : repo.getType() ); | |||
} | |||
@@ -145,7 +145,6 @@ public class DefaultManagedRepositoryAdmin | |||
adminRepo.setCronExpression( repo.getSchedulingDefinition() ); | |||
if (repo.supportsFeature( IndexCreationFeature.class )) { | |||
IndexCreationFeature icf = repo.getFeature( IndexCreationFeature.class ).get(); | |||
adminRepo.setIndexDirectory(convertUriToString( icf.getIndexPath() )); | |||
adminRepo.setSkipPackedIndexCreation( icf.isSkipPackedIndexCreation() ); | |||
} | |||
adminRepo.setScanned( repo.isScanned() ); | |||
@@ -180,6 +179,7 @@ public class DefaultManagedRepositoryAdmin | |||
repoConfig.setDeleteReleasedSnapshots( repo.isDeleteReleasedSnapshots() ); | |||
repoConfig.setSkipPackedIndexCreation( repo.isSkipPackedIndexCreation()); | |||
repoConfig.setStageRepoNeeded( repo.isStageRepoNeeded() ); | |||
return repoConfig; | |||
} | |||
@@ -216,6 +216,7 @@ public class DefaultRemoteRepositoryAdmin | |||
Configuration configuration = getArchivaConfiguration().getConfiguration(); | |||
RemoteRepositoryConfiguration remoteRepositoryConfiguration = | |||
getRepositoryConfiguration( remoteRepository ); | |||
log.debug("Adding remote repo {}", remoteRepositoryConfiguration); | |||
try | |||
{ |
@@ -133,4 +133,13 @@ public class RemoteIndexFeature implements RepositoryFeature<RemoteIndexFeature> | |||
public boolean hasIndex() { | |||
return this.indexUri!=null && !StringUtils.isEmpty( this.indexUri.getPath() ); | |||
} | |||
@Override | |||
public String toString() { | |||
StringBuilder str = new StringBuilder(); | |||
return str.append("RemoteIndexFeature:{downloadRemoteIndex=").append(downloadRemoteIndex) | |||
.append(",indexURI=").append(indexUri) | |||
.append(",downloadOnStartup=").append(downloadRemoteIndexOnStartup) | |||
.append(",timeout=").append(downloadTimeout).append("}").toString(); | |||
} | |||
} |
@@ -151,4 +151,10 @@ public abstract class AbstractRemoteRepository extends AbstractRepository implem | |||
return repositoryBase.resolve(getId()); | |||
} | |||
@Override | |||
public String toString() { | |||
StringBuilder str = new StringBuilder(); | |||
return str.append("checkPath=").append(checkPath) | |||
.append(",creds:").append(credentials).toString(); | |||
} | |||
} |
@@ -21,7 +21,6 @@ package org.apache.archiva.repository; | |||
import org.apache.archiva.configuration.*; | |||
import org.apache.archiva.indexer.*; | |||
import org.apache.archiva.metadata.model.facets.AuditEvent; | |||
import org.apache.archiva.redback.components.registry.RegistryException; | |||
import org.apache.archiva.repository.features.IndexCreationEvent; | |||
import org.apache.archiva.repository.features.IndexCreationFeature; | |||
@@ -1017,7 +1016,7 @@ public class RepositoryRegistry implements ConfigurationListener, RepositoryEven | |||
@Override | |||
public <T> void raise(RepositoryEvent<T> event) { | |||
if (event.getType().equals(IndexCreationEvent.Index.URI_CHANGE)) { | |||
if (event instanceof IndexCreationEvent ) { | |||
if (managedRepositories.containsKey(event.getRepository().getId()) || | |||
remoteRepositories.containsKey(event.getRepository().getId())) { | |||
EditableRepository repo = (EditableRepository) event.getRepository(); |
@@ -66,9 +66,6 @@ public class ArchivaIndexingTaskExecutor | |||
@Inject | |||
private ArtifactContextProducer artifactContextProducer; | |||
@Inject | |||
private ManagedRepositoryAdmin managedRepositoryAdmin; | |||
@Inject | |||
private NexusIndexer nexusIndexer; | |||
@@ -241,23 +238,25 @@ public class ArchivaIndexingTaskExecutor | |||
try | |||
{ | |||
log.debug("Finishing indexing"); | |||
context.optimize( ); | |||
if ( repository.supportsFeature( IndexCreationFeature.class ) ) | |||
{ | |||
IndexCreationFeature icf = repository.getFeature( IndexCreationFeature.class ).get( ); | |||
if ( !icf.isSkipPackedIndexCreation( ) ) | |||
if ( !icf.isSkipPackedIndexCreation( ) && icf.getLocalPackedIndexPath()!=null) | |||
{ | |||
log.debug("Creating packed index from {} on {}", context.getIndexDirectoryFile(), icf.getLocalPackedIndexPath()); | |||
IndexPackingRequest request = new IndexPackingRequest( context, // | |||
context.acquireIndexSearcher( ).getIndexReader( ), | |||
// | |||
context.getIndexDirectoryFile( ) ); | |||
icf.getLocalPackedIndexPath().toFile() ); | |||
indexPacker.packIndex( request ); | |||
context.updateTimestamp( true ); | |||
log.debug( "Index file packaged at '{}'.", context.getIndexDirectoryFile( ) ); | |||
log.debug( "Index file packed at '{}'.", icf.getLocalPackedIndexPath() ); | |||
} else { | |||
log.debug( "skip packed index creation" ); | |||
} |
@@ -20,10 +20,7 @@ package org.apache.archiva.scheduler.indexing; | |||
import org.apache.archiva.admin.model.RepositoryAdminException; | |||
import org.apache.archiva.admin.model.beans.NetworkProxy; | |||
import org.apache.archiva.admin.model.beans.RemoteRepository; | |||
import org.apache.archiva.admin.model.networkproxy.NetworkProxyAdmin; | |||
import org.apache.archiva.admin.model.proxyconnector.ProxyConnectorAdmin; | |||
import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin; | |||
import org.apache.archiva.common.ArchivaException; | |||
import org.apache.archiva.common.plexusbridge.PlexusSisuBridgeException; | |||
import org.apache.archiva.configuration.ArchivaConfiguration; | |||
@@ -34,7 +31,6 @@ import org.apache.archiva.proxy.common.WagonFactory; | |||
import org.apache.archiva.repository.RepositoryRegistry; | |||
import org.apache.archiva.repository.features.RemoteIndexFeature; | |||
import org.apache.commons.lang.StringUtils; | |||
import org.apache.maven.index.NexusIndexer; | |||
import org.apache.maven.index.context.IndexingContext; | |||
import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException; | |||
import org.apache.maven.index.packer.IndexPacker; | |||
@@ -46,7 +42,6 @@ import org.springframework.scheduling.support.CronTrigger; | |||
import org.springframework.stereotype.Service; | |||
import javax.annotation.PostConstruct; | |||
import javax.annotation.PreDestroy; | |||
import javax.inject.Inject; | |||
import javax.inject.Named; | |||
import java.io.IOException; | |||
@@ -78,18 +73,9 @@ public class DefaultDownloadRemoteIndexScheduler | |||
@Inject | |||
private WagonFactory wagonFactory; | |||
@Inject | |||
private RemoteRepositoryAdmin remoteRepositoryAdmin; | |||
@Inject | |||
private ProxyConnectorAdmin proxyConnectorAdmin; | |||
@Inject | |||
private NetworkProxyAdmin networkProxyAdmin; | |||
@Inject | |||
private NexusIndexer nexusIndexer; | |||
@Inject | |||
private IndexUpdater indexUpdater; | |||
@@ -101,8 +87,8 @@ public class DefaultDownloadRemoteIndexScheduler | |||
@PostConstruct | |||
public void startup() | |||
throws ArchivaException, RepositoryAdminException, PlexusSisuBridgeException, IOException, | |||
UnsupportedExistingLuceneIndexException, DownloadRemoteIndexException, UnsupportedBaseContextException { | |||
throws | |||
DownloadRemoteIndexException, UnsupportedBaseContextException { | |||
archivaConfiguration.addListener( this ); | |||
// TODO add indexContexts even if null | |||
@@ -129,22 +115,6 @@ public class DefaultDownloadRemoteIndexScheduler | |||
} | |||
@PreDestroy | |||
public void shutdown() | |||
throws RepositoryAdminException, IOException | |||
{ | |||
for ( RemoteRepository remoteRepository : remoteRepositoryAdmin.getRemoteRepositories() ) | |||
{ | |||
String contextKey = "remote-" + remoteRepository.getId(); | |||
IndexingContext context = nexusIndexer.getIndexingContexts().get( contextKey ); | |||
if ( context == null ) | |||
{ | |||
continue; | |||
} | |||
nexusIndexer.removeIndexingContext( context, false ); | |||
} | |||
} | |||
@Override | |||
public void configurationEvent( ConfigurationEvent event ) | |||
{ | |||
@@ -158,36 +128,41 @@ public class DefaultDownloadRemoteIndexScheduler | |||
{ | |||
try | |||
{ | |||
RemoteRepository remoteRepository = remoteRepositoryAdmin.getRemoteRepository( repositoryId ); | |||
if ( remoteRepository == null ) | |||
org.apache.archiva.repository.RemoteRepository remoteRepo = repositoryRegistry.getRemoteRepository(repositoryId); | |||
if ( remoteRepo == null ) | |||
{ | |||
log.warn( "ignore scheduleDownloadRemote for repo with id {} as not exists", repositoryId ); | |||
return; | |||
} | |||
if (!remoteRepo.supportsFeature(RemoteIndexFeature.class)) { | |||
log.warn("ignore scheduleDownloadRemote for repo with id {}. Does not support remote index.", repositoryId); | |||
return; | |||
} | |||
RemoteIndexFeature rif = remoteRepo.getFeature(RemoteIndexFeature.class).get(); | |||
NetworkProxy networkProxy = null; | |||
if ( StringUtils.isNotBlank( remoteRepository.getRemoteDownloadNetworkProxyId() ) ) | |||
if ( StringUtils.isNotBlank( rif.getProxyId() ) ) | |||
{ | |||
networkProxy = networkProxyAdmin.getNetworkProxy( remoteRepository.getRemoteDownloadNetworkProxyId() ); | |||
networkProxy = networkProxyAdmin.getNetworkProxy( rif.getProxyId() ); | |||
if ( networkProxy == null ) | |||
{ | |||
log.warn( | |||
"your remote repository is configured to download remote index trought a proxy we cannot find id:{}", | |||
remoteRepository.getRemoteDownloadNetworkProxyId() ); | |||
rif.getProxyId() ); | |||
} | |||
} | |||
DownloadRemoteIndexTaskRequest downloadRemoteIndexTaskRequest = new DownloadRemoteIndexTaskRequest() // | |||
.setRemoteRepository( remoteRepository ) // | |||
.setRemoteRepository( remoteRepo ) // | |||
.setNetworkProxy( networkProxy ) // | |||
.setFullDownload( fullDownload ) // | |||
.setWagonFactory( wagonFactory ) // | |||
.setRemoteRepositoryAdmin( remoteRepositoryAdmin ) // | |||
.setIndexUpdater( indexUpdater ) // | |||
.setIndexPacker( this.indexPacker ); | |||
if ( now ) | |||
{ | |||
log.info( "schedule download remote index for repository {}", remoteRepository.getId() ); | |||
log.info( "schedule download remote index for repository {}", remoteRepo.getId() ); | |||
// do it now | |||
taskScheduler.schedule( | |||
new DownloadRemoteIndexTask( downloadRemoteIndexTaskRequest, this.runningRemoteDownloadIds ), | |||
@@ -196,10 +171,10 @@ public class DefaultDownloadRemoteIndexScheduler | |||
else | |||
{ | |||
log.info( "schedule download remote index for repository {} with cron expression {}", | |||
remoteRepository.getId(), remoteRepository.getCronExpression() ); | |||
remoteRepo.getId(), remoteRepo.getSchedulingDefinition()); | |||
try | |||
{ | |||
CronTrigger cronTrigger = new CronTrigger( remoteRepository.getCronExpression() ); | |||
CronTrigger cronTrigger = new CronTrigger( remoteRepo.getSchedulingDefinition()); | |||
taskScheduler.schedule( | |||
new DownloadRemoteIndexTask( downloadRemoteIndexTaskRequest, this.runningRemoteDownloadIds ), | |||
cronTrigger ); | |||
@@ -209,11 +184,11 @@ public class DefaultDownloadRemoteIndexScheduler | |||
log.warn( "Unable to schedule remote index download: {}", e.getLocalizedMessage() ); | |||
} | |||
if ( remoteRepository.isDownloadRemoteIndexOnStartup() ) | |||
if ( rif.isDownloadRemoteIndexOnStartup() ) | |||
{ | |||
log.info( | |||
"remote repository {} configured with downloadRemoteIndexOnStartup schedule now a download", | |||
remoteRepository.getId() ); | |||
remoteRepo.getId() ); | |||
taskScheduler.schedule( | |||
new DownloadRemoteIndexTask( downloadRemoteIndexTaskRequest, this.runningRemoteDownloadIds ), | |||
new Date() ); |
@@ -19,15 +19,22 @@ package org.apache.archiva.scheduler.indexing; | |||
*/ | |||
import org.apache.archiva.admin.model.beans.NetworkProxy; | |||
import org.apache.archiva.admin.model.beans.RemoteRepository; | |||
import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin; | |||
import org.apache.archiva.proxy.common.WagonFactory; | |||
import org.apache.archiva.proxy.common.WagonFactoryRequest; | |||
import org.apache.archiva.repository.PasswordCredentials; | |||
import org.apache.archiva.repository.RemoteRepository; | |||
import org.apache.archiva.repository.RepositoryException; | |||
import org.apache.archiva.repository.RepositoryType; | |||
import org.apache.archiva.repository.features.RemoteIndexFeature; | |||
import org.apache.commons.lang.time.StopWatch; | |||
import org.apache.http.auth.UsernamePasswordCredentials; | |||
import org.apache.maven.index.context.IndexingContext; | |||
import org.apache.maven.index.updater.IndexUpdateRequest; | |||
import org.apache.maven.index.updater.IndexUpdateResult; | |||
import org.apache.maven.index.updater.IndexUpdater; | |||
import org.apache.maven.index.updater.ResourceFetcher; | |||
import org.apache.maven.index_shaded.lucene.index.IndexNotFoundException; | |||
import org.apache.maven.wagon.ResourceDoesNotExistException; | |||
import org.apache.maven.wagon.StreamWagon; | |||
import org.apache.maven.wagon.TransferFailedException; | |||
@@ -65,8 +72,6 @@ public class DownloadRemoteIndexTask | |||
private RemoteRepository remoteRepository; | |||
private RemoteRepositoryAdmin remoteRepositoryAdmin; | |||
private WagonFactory wagonFactory; | |||
private NetworkProxy networkProxy; | |||
@@ -87,7 +92,6 @@ public class DownloadRemoteIndexTask | |||
this.fullDownload = downloadRemoteIndexTaskRequest.isFullDownload(); | |||
this.runningRemoteDownloadIds = runningRemoteDownloadIds; | |||
this.indexUpdater = downloadRemoteIndexTaskRequest.getIndexUpdater(); | |||
this.remoteRepositoryAdmin = downloadRemoteIndexTaskRequest.getRemoteRepositoryAdmin(); | |||
} | |||
@Override | |||
@@ -112,8 +116,17 @@ public class DownloadRemoteIndexTask | |||
try | |||
{ | |||
log.info( "start download remote index for remote repository {}", this.remoteRepository.getId() ); | |||
IndexingContext indexingContext = remoteRepositoryAdmin.createIndexContext( this.remoteRepository ); | |||
if (this.remoteRepository.getIndexingContext()==null) { | |||
throw new IndexNotFoundException("No index context set for repository "+remoteRepository.getId()); | |||
} | |||
if (this.remoteRepository.getType()!= RepositoryType.MAVEN) { | |||
throw new RepositoryException("Bad repository type"); | |||
} | |||
if (!this.remoteRepository.supportsFeature(RemoteIndexFeature.class)) { | |||
throw new RepositoryException("Repository does not support RemotIndexFeature "+remoteRepository.getId()); | |||
} | |||
RemoteIndexFeature rif = this.remoteRepository.getFeature(RemoteIndexFeature.class).get(); | |||
IndexingContext indexingContext = this.remoteRepository.getIndexingContext().getBaseContext(IndexingContext.class); | |||
// create a temp directory to download files | |||
tempIndexDirectory = Paths.get(indexingContext.getIndexDirectoryFile().getParent(), ".tmpIndex" ); | |||
Path indexCacheDirectory = Paths.get( indexingContext.getIndexDirectoryFile().getParent(), ".indexCache" ); | |||
@@ -126,22 +139,22 @@ public class DownloadRemoteIndexTask | |||
tempIndexDirectory.toFile().deleteOnExit(); | |||
String baseIndexUrl = indexingContext.getIndexUpdateUrl(); | |||
String wagonProtocol = new URL( this.remoteRepository.getUrl() ).getProtocol(); | |||
String wagonProtocol = this.remoteRepository.getLocation().getScheme(); | |||
final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon( | |||
new WagonFactoryRequest( wagonProtocol, this.remoteRepository.getExtraHeaders() ).networkProxy( | |||
this.networkProxy ) | |||
); | |||
// FIXME olamy having 2 config values | |||
wagon.setReadTimeout( remoteRepository.getRemoteDownloadTimeout() * 1000 ); | |||
wagon.setTimeout( remoteRepository.getTimeout() * 1000 ); | |||
wagon.setReadTimeout( (int)rif.getDownloadTimeout().toMillis()); | |||
wagon.setTimeout( (int)remoteRepository.getTimeout().toMillis()); | |||
if ( wagon instanceof AbstractHttpClientWagon ) | |||
{ | |||
HttpConfiguration httpConfiguration = new HttpConfiguration(); | |||
HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration(); | |||
httpMethodConfiguration.setUsePreemptive( true ); | |||
httpMethodConfiguration.setReadTimeout( remoteRepository.getRemoteDownloadTimeout() * 1000 ); | |||
httpMethodConfiguration.setReadTimeout( (int)rif.getDownloadTimeout().toMillis() ); | |||
httpConfiguration.setGet( httpMethodConfiguration ); | |||
AbstractHttpClientWagon.class.cast( wagon ).setHttpConfiguration( httpConfiguration ); | |||
} | |||
@@ -158,12 +171,14 @@ public class DownloadRemoteIndexTask | |||
proxyInfo.setPassword( this.networkProxy.getPassword() ); | |||
} | |||
AuthenticationInfo authenticationInfo = null; | |||
if ( this.remoteRepository.getUserName() != null ) | |||
if ( this.remoteRepository.getLoginCredentials()!=null && this.remoteRepository.getLoginCredentials() instanceof PasswordCredentials ) | |||
{ | |||
PasswordCredentials creds = (PasswordCredentials) this.remoteRepository.getLoginCredentials(); | |||
authenticationInfo = new AuthenticationInfo(); | |||
authenticationInfo.setUserName( this.remoteRepository.getUserName() ); | |||
authenticationInfo.setPassword( this.remoteRepository.getPassword() ); | |||
authenticationInfo.setUserName( creds.getUsername()); | |||
authenticationInfo.setPassword( new String(creds.getPassword()) ); | |||
} | |||
log.debug("Connection to {}, authInfo={}", this.remoteRepository.getId(), authenticationInfo); | |||
wagon.connect( new Repository( this.remoteRepository.getId(), baseIndexUrl ), authenticationInfo, | |||
proxyInfo ); | |||
@@ -172,6 +187,8 @@ public class DownloadRemoteIndexTask | |||
{ | |||
Files.createDirectories( indexDirectory ); | |||
} | |||
log.debug("Downloading index file to {}", indexDirectory); | |||
log.debug("Index cache dir {}", indexCacheDirectory); | |||
ResourceFetcher resourceFetcher = | |||
new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository ); | |||
@@ -179,10 +196,11 @@ public class DownloadRemoteIndexTask | |||
request.setForceFullUpdate( this.fullDownload ); | |||
request.setLocalIndexCacheDir( indexCacheDirectory.toFile() ); | |||
this.indexUpdater.fetchAndUpdateIndex( request ); | |||
IndexUpdateResult result = this.indexUpdater.fetchAndUpdateIndex(request); | |||
log.debug("Update result success: {}", result.isSuccessful()); | |||
stopWatch.stop(); | |||
log.info( "time update index from remote for repository {}: {} s", this.remoteRepository.getId(), | |||
( stopWatch.getTime() / 1000 ) ); | |||
log.info( "time update index from remote for repository {}: {}ms", this.remoteRepository.getId(), | |||
( stopWatch.getTime() ) ); | |||
// index packing optionnal ?? | |||
//IndexPackingRequest indexPackingRequest = | |||
@@ -241,6 +259,7 @@ public class DownloadRemoteIndexTask | |||
{ | |||
this.totalLength = 0; | |||
resourceName = transferEvent.getResource().getName(); | |||
log.info("Transferring: {}, {}", transferEvent.getResource().getContentLength(), transferEvent.getLocalFile().toString()); | |||
log.info( "start transfer of {}", transferEvent.getResource().getName() ); | |||
} | |||
@@ -256,8 +275,8 @@ public class DownloadRemoteIndexTask | |||
{ | |||
resourceName = transferEvent.getResource().getName(); | |||
long endTime = System.currentTimeMillis(); | |||
log.info( "end of transfer file {} {} kb: {}s", transferEvent.getResource().getName(), | |||
this.totalLength / 1024, ( endTime - startTime ) / 1000 ); | |||
log.info( "end of transfer file {}: {}b, {}ms", transferEvent.getResource().getName(), | |||
this.totalLength, ( endTime - startTime ) ); | |||
} | |||
@Override |
@@ -19,9 +19,9 @@ package org.apache.archiva.scheduler.indexing; | |||
*/ | |||
import org.apache.archiva.admin.model.beans.NetworkProxy; | |||
import org.apache.archiva.admin.model.beans.RemoteRepository; | |||
import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin; | |||
import org.apache.archiva.proxy.common.WagonFactory; | |||
import org.apache.archiva.repository.RemoteRepository; | |||
import org.apache.maven.index.packer.IndexPacker; | |||
import org.apache.maven.index.updater.IndexUpdater; | |||
@@ -33,8 +33,6 @@ public class DownloadRemoteIndexTaskRequest | |||
{ | |||
private RemoteRepository remoteRepository; | |||
private RemoteRepositoryAdmin remoteRepositoryAdmin; | |||
private WagonFactory wagonFactory; | |||
private NetworkProxy networkProxy; | |||
@@ -106,17 +104,6 @@ public class DownloadRemoteIndexTaskRequest | |||
return this; | |||
} | |||
public RemoteRepositoryAdmin getRemoteRepositoryAdmin() | |||
{ | |||
return remoteRepositoryAdmin; | |||
} | |||
public DownloadRemoteIndexTaskRequest setRemoteRepositoryAdmin( RemoteRepositoryAdmin remoteRepositoryAdmin ) | |||
{ | |||
this.remoteRepositoryAdmin = remoteRepositoryAdmin; | |||
return this; | |||
} | |||
public IndexPacker getIndexPacker() | |||
{ | |||
return indexPacker; |
@@ -160,7 +160,7 @@ public class ArchivaIndexingTaskExecutorTest | |||
FlatSearchResponse response = indexer.searchFlat( request ); | |||
assertTrue( Files.exists(basePath.resolve( ".indexer" )) ); | |||
assertFalse( Files.exists(basePath.resolve(".index" )) ); | |||
assertTrue( Files.exists(basePath.resolve(".index" )) ); | |||
assertEquals( 1, response.getTotalHits() ); | |||
Set<ArtifactInfo> results = response.getResults(); | |||
@@ -204,7 +204,7 @@ public class ArchivaIndexingTaskExecutorTest | |||
ctx.releaseIndexSearcher( searcher ); | |||
assertTrue( Files.exists(basePath.resolve(".indexer" )) ); | |||
assertFalse( Files.exists(basePath.resolve(".index" )) ); | |||
assertTrue( Files.exists(basePath.resolve(".index" )) ); | |||
// should only return 1 hit! | |||
assertEquals( 1, topDocs.totalHits ); | |||
@@ -239,7 +239,7 @@ public class ArchivaIndexingTaskExecutorTest | |||
FlatSearchResponse response = indexer.searchFlat( flatSearchRequest ); | |||
assertTrue( Files.exists(basePath.resolve(".indexer" )) ); | |||
assertFalse( Files.exists(basePath.resolve( ".index" )) ); | |||
assertTrue( Files.exists(basePath.resolve( ".index" )) ); | |||
// should return 1 hit | |||
assertEquals( 1, response.getTotalHitsCount() ); | |||
@@ -261,7 +261,7 @@ public class ArchivaIndexingTaskExecutorTest | |||
BooleanClause.Occur.SHOULD ); | |||
assertTrue( Files.exists(basePath.resolve( ".indexer" )) ); | |||
assertFalse( Files.exists(basePath.resolve(".index" )) ); | |||
assertTrue( Files.exists(basePath.resolve(".index" )) ); | |||
flatSearchRequest = new FlatSearchRequest( q, getIndexingContext() ); | |||
@@ -278,9 +278,9 @@ public class ArchivaIndexingTaskExecutorTest | |||
{ | |||
Path basePath = PathUtil.getPathFromUri( repositoryConfig.getLocation()); | |||
Path indexerDirectory =basePath.resolve( ".indexer" ); | |||
Path indexDirectory = basePath.resolve(".index"); | |||
Files.list(indexerDirectory).filter( path -> path.getFileName().toString().startsWith("nexus-maven-repository-index") ) | |||
Files.list(indexDirectory).filter( path -> path.getFileName().toString().startsWith("nexus-maven-repository-index") ) | |||
.forEach( path -> | |||
{ | |||
try | |||
@@ -311,19 +311,19 @@ public class ArchivaIndexingTaskExecutorTest | |||
indexingExecutor.executeTask( task ); | |||
assertTrue( Files.exists(indexerDirectory) ); | |||
assertTrue( Files.exists(indexDirectory) ); | |||
// test packed index file creation | |||
//no more zip | |||
//Assertions.assertThat(new File( indexerDirectory, "nexus-maven-repository-index.zip" )).exists(); | |||
Assertions.assertThat( Files.exists(indexerDirectory.resolve("nexus-maven-repository-index.properties" ) )); | |||
Assertions.assertThat( Files.exists(indexerDirectory.resolve("nexus-maven-repository-index.gz" ) )); | |||
Assertions.assertThat( Files.exists(indexDirectory.resolve("nexus-maven-repository-index.properties" ) )); | |||
Assertions.assertThat( Files.exists(indexDirectory.resolve("nexus-maven-repository-index.gz" ) )); | |||
// unpack .zip index | |||
Path destDir = basePath.resolve( ".indexer/tmp" ); | |||
Path destDir = basePath.resolve( ".index/tmp" ); | |||
//unzipIndex( indexerDirectory.getPath(), destDir.getPath() ); | |||
DefaultIndexUpdater.FileFetcher fetcher = new DefaultIndexUpdater.FileFetcher( indexerDirectory.toFile() ); | |||
DefaultIndexUpdater.FileFetcher fetcher = new DefaultIndexUpdater.FileFetcher( indexDirectory.toFile() ); | |||
IndexUpdateRequest updateRequest = new IndexUpdateRequest( getIndexingContext(), fetcher ); | |||
//updateRequest.setLocalIndexCacheDir( indexerDirectory ); | |||
indexUpdater.fetchAndUpdateIndex( updateRequest ); |
@@ -447,8 +447,11 @@ public abstract class AbstractArchivaRestTest | |||
} | |||
managedRepository.setLocation( Paths.get( repoPath ).toString() ); | |||
String suffix = Long.toString( new Date().getTime() ); | |||
String baseDir = System.getProperty("java.io.tmpdir"); | |||
managedRepository.setIndexDirectory( | |||
System.getProperty( "java.io.tmpdir" ) + "/.index-" + Long.toString( new Date().getTime() ) ); | |||
baseDir + "/.indexer-" + suffix ); | |||
managedRepository.setPackedIndexDirectory(baseDir + "/.index-" + suffix); | |||
managedRepository.setStageRepoNeeded( stageNeeded ); | |||
managedRepository.setSnapshots( true ); |
@@ -161,7 +161,7 @@ public class DownloadMergedIndexTest | |||
remoteRepository.setName( "all-merged" ); | |||
remoteRepository.setDownloadRemoteIndex( true ); | |||
remoteRepository.setUrl( "http://localhost:" + port + "/repository/test-group" ); | |||
remoteRepository.setRemoteIndexUrl( "http://localhost:" + port + "/repository/test-group/.indexer" ); | |||
remoteRepository.setRemoteIndexUrl( "http://localhost:" + port + "/repository/test-group/.index" ); | |||
remoteRepository.setUserName( RedbackRoleConstants.ADMINISTRATOR_ACCOUNT_NAME ); | |||
remoteRepository.setPassword( FakeCreateAdminService.ADMIN_TEST_PWD ); | |||
@@ -108,5 +108,8 @@ public class MavenRemoteRepository extends AbstractRemoteRepository | |||
} | |||
} | |||
@Override | |||
public String toString() { | |||
return super.toString()+", remoteIndexFeature="+remoteIndexFeature.toString()+", indexCreationFeature="+indexCreationFeature.toString(); | |||
} | |||
} |
@@ -150,6 +150,9 @@ public class MavenRepositoryProvider implements RepositoryProvider { | |||
IndexCreationFeature indexCreationFeature = repo.getFeature(IndexCreationFeature.class).get(); | |||
indexCreationFeature.setSkipPackedIndexCreation(cfg.isSkipPackedIndexCreation()); | |||
indexCreationFeature.setIndexPath(getURIFromString(cfg.getIndexDir())); | |||
indexCreationFeature.setPackedIndexPath(getURIFromString(cfg.getPackedIndexDir())); | |||
/* -> Should be created by MavenIndexProvider | |||
Path indexPath; | |||
if (indexCreationFeature.getIndexPath().getScheme() == null) { | |||
indexPath = Paths.get(indexCreationFeature.getIndexPath().getPath()); | |||
@@ -167,7 +170,7 @@ public class MavenRepositoryProvider implements RepositoryProvider { | |||
} catch (IOException e) { | |||
log.error("Could not create index directory {}", absoluteIndexPath); | |||
throw new RepositoryException("Could not create index directory " + absoluteIndexPath); | |||
} | |||
}*/ | |||
ArtifactCleanupFeature artifactCleanupFeature = repo.getFeature(ArtifactCleanupFeature.class).get(); | |||
@@ -237,10 +240,14 @@ public class MavenRepositoryProvider implements RepositoryProvider { | |||
} else { | |||
credentials.setPassword(new char[0]); | |||
} | |||
IndexCreationFeature indexCreationFeature = repo.getFeature(IndexCreationFeature.class).get(); | |||
if (cfg.getIndexDir() != null) { | |||
IndexCreationFeature indexCreationFeature = repo.getFeature(IndexCreationFeature.class).get(); | |||
indexCreationFeature.setIndexPath(getURIFromString(cfg.getIndexDir())); | |||
} | |||
if (cfg.getPackedIndexDir() != null) { | |||
indexCreationFeature.setPackedIndexPath(getURIFromString(cfg.getPackedIndexDir())); | |||
} | |||
log.debug("Updated remote instance {}", repo); | |||
} | |||
@Override | |||
@@ -272,6 +279,7 @@ public class MavenRepositoryProvider implements RepositoryProvider { | |||
IndexCreationFeature indexCreationFeature = remoteRepository.getFeature(IndexCreationFeature.class).get(); | |||
cfg.setIndexDir(convertUriToPath(indexCreationFeature.getIndexPath())); | |||
cfg.setPackedIndexDir(convertUriToPath(indexCreationFeature.getPackedIndexPath())); | |||
RemoteIndexFeature remoteIndexFeature = remoteRepository.getFeature(RemoteIndexFeature.class).get(); | |||
cfg.setRemoteIndexUrl(remoteIndexFeature.getIndexUri().toString()); | |||
@@ -305,6 +313,7 @@ public class MavenRepositoryProvider implements RepositoryProvider { | |||
cfg.setStageRepoNeeded(stagingRepositoryFeature.isStageRepoNeeded()); | |||
IndexCreationFeature indexCreationFeature = managedRepository.getFeature(IndexCreationFeature.class).get(); | |||
cfg.setIndexDir(convertUriToPath(indexCreationFeature.getIndexPath())); | |||
cfg.setPackedIndexDir(convertUriToPath(indexCreationFeature.getPackedIndexPath())); | |||
cfg.setSkipPackedIndexCreation(indexCreationFeature.isSkipPackedIndexCreation()); | |||
ArtifactCleanupFeature artifactCleanupFeature = managedRepository.getFeature(ArtifactCleanupFeature.class).get(); | |||
@@ -345,7 +354,7 @@ public class MavenRepositoryProvider implements RepositoryProvider { | |||
try { | |||
indexDir = Paths.get(new URI(repository.getIndexDir().startsWith("file://") ? repository.getIndexDir() : "file://" + repository.getIndexDir())); | |||
if (indexDir.isAbsolute()) { | |||
Path newDir = Paths.get(new URI(stagingRepository.getLocation().startsWith("file://") ? stagingRepository.getLocation() : "file://" + stagingRepository.getLocation())).resolve(".index"); | |||
Path newDir = indexDir.getParent().resolve(indexDir.getFileName() + StagingRepositoryFeature.STAGING_REPO_POSTFIX); | |||
log.debug("Changing index directory {} -> {}", indexDir, newDir); | |||
stagingRepository.setIndexDir(newDir.toString()); | |||
} else { | |||
@@ -358,6 +367,24 @@ public class MavenRepositoryProvider implements RepositoryProvider { | |||
} | |||
// in case of absolute dir do not use the same | |||
} | |||
if (StringUtils.isNotBlank(repository.getPackedIndexDir())) { | |||
Path packedIndexDir = null; | |||
try { | |||
packedIndexDir = Paths.get(new URI(repository.getPackedIndexDir().startsWith("file://") ? repository.getPackedIndexDir() : "file://" + repository.getPackedIndexDir())); | |||
if (packedIndexDir.isAbsolute()) { | |||
Path newDir = packedIndexDir.getParent().resolve(packedIndexDir.getFileName() + StagingRepositoryFeature.STAGING_REPO_POSTFIX); | |||
log.debug("Changing index directory {} -> {}", packedIndexDir, newDir); | |||
stagingRepository.setPackedIndexDir(newDir.toString()); | |||
} else { | |||
log.debug("Keeping index directory {}", repository.getPackedIndexDir()); | |||
stagingRepository.setPackedIndexDir(repository.getPackedIndexDir()); | |||
} | |||
} catch (URISyntaxException e) { | |||
log.error("Could not parse index path as uri {}", repository.getPackedIndexDir()); | |||
stagingRepository.setPackedIndexDir(""); | |||
} | |||
// in case of absolute dir do not use the same | |||
} | |||
stagingRepository.setRefreshCronExpression(repository.getRefreshCronExpression()); | |||
stagingRepository.setReleases(repository.isReleases()); | |||
stagingRepository.setRetentionCount(repository.getRetentionCount()); |