source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.indexer.maven;
   2
   3 /*
   4  * Licensed to the Apache Software Foundation (ASF) under one
   5  * or more contributor license agreements.  See the NOTICE file
   6  * distributed with this work for additional information
   7  * regarding copyright ownership.  The ASF licenses this file
   8  * to you under the Apache License, Version 2.0 (the
   9  * "License"); you may not use this file except in compliance
  10  * with the License.  You may obtain a copy of the License at
  11  *
  12  *  http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing,
  15  * software distributed under the License is distributed on an
  16  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  17  * KIND, either express or implied.  See the License for the
  18  * specific language governing permissions and limitations
  19  * under the License.
  20  */
  21
  22 import org.apache.archiva.admin.model.RepositoryAdminException;
  23 import org.apache.archiva.common.utils.FileUtils;
  24 import org.apache.archiva.common.utils.PathUtil;
  25 import org.apache.archiva.configuration.ArchivaConfiguration;
  26 import org.apache.archiva.indexer.ArchivaIndexManager;
  27 import org.apache.archiva.indexer.ArchivaIndexingContext;
  28 import org.apache.archiva.indexer.IndexCreationFailedException;
  29 import org.apache.archiva.indexer.IndexUpdateFailedException;
  30 import org.apache.archiva.indexer.UnsupportedBaseContextException;
  31 import org.apache.archiva.indexer.merger.IndexMergerException;
  32 import org.apache.archiva.indexer.merger.TemporaryGroupIndex;
  33 import org.apache.archiva.proxy.ProxyRegistry;
  34 import org.apache.archiva.proxy.maven.WagonFactory;
  35 import org.apache.archiva.proxy.maven.WagonFactoryException;
  36 import org.apache.archiva.proxy.maven.WagonFactoryRequest;
  37 import org.apache.archiva.proxy.model.NetworkProxy;
  38 import org.apache.archiva.repository.EditableRepository;
  39 import org.apache.archiva.repository.ManagedRepository;
  40 import org.apache.archiva.repository.PasswordCredentials;
  41 import org.apache.archiva.repository.RemoteRepository;
  42 import org.apache.archiva.repository.Repository;
  43 import org.apache.archiva.repository.RepositoryType;
  44 import org.apache.archiva.repository.UnsupportedRepositoryTypeException;
  45 import org.apache.archiva.repository.content.FilesystemAsset;
  46 import org.apache.archiva.repository.content.StorageAsset;
  47 import org.apache.archiva.repository.features.IndexCreationFeature;
  48 import org.apache.archiva.repository.features.RemoteIndexFeature;
  49 import org.apache.commons.lang.StringUtils;
  50 import org.apache.maven.index.ArtifactContext;
  51 import org.apache.maven.index.ArtifactContextProducer;
  52 import org.apache.maven.index.DefaultScannerListener;
  53 import org.apache.maven.index.Indexer;
  54 import org.apache.maven.index.IndexerEngine;
  55 import org.apache.maven.index.Scanner;
  56 import org.apache.maven.index.ScanningRequest;
  57 import org.apache.maven.index.ScanningResult;
  58 import org.apache.maven.index.context.ContextMemberProvider;
  59 import org.apache.maven.index.context.IndexCreator;
  60 import org.apache.maven.index.context.IndexingContext;
  61 import org.apache.maven.index.context.StaticContextMemberProvider;
  62 import org.apache.maven.index.packer.IndexPacker;
  63 import org.apache.maven.index.packer.IndexPackingRequest;
  64 import org.apache.maven.index.updater.IndexUpdateRequest;
  65 import org.apache.maven.index.updater.IndexUpdater;
  66 import org.apache.maven.index.updater.ResourceFetcher;
  67 import org.apache.maven.index_shaded.lucene.index.IndexFormatTooOldException;
  68 import org.apache.maven.wagon.ConnectionException;
  69 import org.apache.maven.wagon.ResourceDoesNotExistException;
  70 import org.apache.maven.wagon.StreamWagon;
  71 import org.apache.maven.wagon.TransferFailedException;
  72 import org.apache.maven.wagon.Wagon;
  73 import org.apache.maven.wagon.authentication.AuthenticationException;
  74 import org.apache.maven.wagon.authentication.AuthenticationInfo;
  75 import org.apache.maven.wagon.authorization.AuthorizationException;
  76 import org.apache.maven.wagon.events.TransferEvent;
  77 import org.apache.maven.wagon.events.TransferListener;
  78 import org.apache.maven.wagon.proxy.ProxyInfo;
  79 import org.apache.maven.wagon.shared.http.AbstractHttpClientWagon;
  80 import org.apache.maven.wagon.shared.http.HttpConfiguration;
  81 import org.apache.maven.wagon.shared.http.HttpMethodConfiguration;
  82 import org.slf4j.Logger;
  83 import org.slf4j.LoggerFactory;
  84 import org.springframework.stereotype.Service;
  85
  86 import javax.inject.Inject;
  87 import java.io.FileNotFoundException;
  88 import java.io.IOException;
  89 import java.io.InputStream;
  90 import java.net.MalformedURLException;
  91 import java.net.URI;
  92 import java.nio.file.Files;
  93 import java.nio.file.Path;
  94 import java.nio.file.Paths;
  95 import java.util.Collection;
  96 import java.util.List;
  97 import java.util.Map;
  98 import java.util.Objects;
  99 import java.util.concurrent.ConcurrentSkipListSet;
 100 import java.util.stream.Collectors;
 101
 102 /**
 103  * Maven implementation of index manager.
 104  * The index manager is a singleton, so we try to make sure, that index operations are not running
 105  * parallel by synchronizing on the index path.
 106  * A update operation waits for parallel running methods to finish before starting, but after a certain
 107  * time of retries a IndexUpdateFailedException is thrown.
 108  */
 109 @Service( "archivaIndexManager#maven" )
 110 public class MavenIndexManager implements ArchivaIndexManager {
 111
 112     private static final Logger log = LoggerFactory.getLogger( MavenIndexManager.class );
 113
 114     @Inject
 115     private Indexer indexer;
 116
 117     @Inject
 118     private IndexerEngine indexerEngine;
 119
 120     @Inject
 121     private List<? extends IndexCreator> indexCreators;
 122
 123     @Inject
 124     private IndexPacker indexPacker;
 125
 126     @Inject
 127     private Scanner scanner;
 128
 129     @Inject
 130     private ArchivaConfiguration archivaConfiguration;
 131
 132     @Inject
 133     private WagonFactory wagonFactory;
 134
 135     @Inject
 136     private IndexUpdater indexUpdater;
 137
 138     @Inject
 139     private ArtifactContextProducer artifactContextProducer;
 140
 141     @Inject
 142     private ProxyRegistry proxyRegistry;
 143
 144
 145     private ConcurrentSkipListSet<Path> activeContexts = new ConcurrentSkipListSet<>( );
 146
 147     private static final int WAIT_TIME = 100;
 148     private static final int MAX_WAIT = 10;
 149
 150
 151     public static IndexingContext getMvnContext( ArchivaIndexingContext context ) throws UnsupportedBaseContextException
 152     {
 153         if ( !context.supports( IndexingContext.class ) )
 154         {
 155             log.error( "The provided archiva index context does not support the maven IndexingContext" );
 156             throw new UnsupportedBaseContextException( "The context does not support the Maven IndexingContext" );
 157         }
 158         return context.getBaseContext( IndexingContext.class );
 159     }
 160
 161     private Path getIndexPath( ArchivaIndexingContext ctx )
 162     {
 163         return PathUtil.getPathFromUri( ctx.getPath( ) );
 164     }
 165
 166     @FunctionalInterface
 167     interface IndexUpdateConsumer
 168     {
 169
 170         void accept( IndexingContext indexingContext ) throws IndexUpdateFailedException;
 171     }
 172
 173     /*
 174      * This method is used to do some actions around the update execution code. And to make sure, that no other
 175      * method is running on the same index.
 176      */
 177     private void executeUpdateFunction( ArchivaIndexingContext context, IndexUpdateConsumer function ) throws IndexUpdateFailedException
 178     {
 179         IndexingContext indexingContext = null;
 180         try
 181         {
 182             indexingContext = getMvnContext( context );
 183         }
 184         catch ( UnsupportedBaseContextException e )
 185         {
 186             throw new IndexUpdateFailedException( "Maven index is not supported by this context", e );
 187         }
 188         final Path ctxPath = getIndexPath( context );
 189         int loop = MAX_WAIT;
 190         boolean active = false;
 191         while ( loop-- > 0 && !active )
 192         {
 193             active = activeContexts.add( ctxPath );
 194             try
 195             {
 196                 Thread.currentThread( ).sleep( WAIT_TIME );
 197             }
 198             catch ( InterruptedException e )
 199             {
 200                 // Ignore this
 201             }
 202         }
 203         if ( active )
 204         {
 205             try
 206             {
 207                 function.accept( indexingContext );
 208             }
 209             finally
 210             {
 211                 activeContexts.remove( ctxPath );
 212             }
 213         }
 214         else
 215         {
 216             throw new IndexUpdateFailedException( "Timeout while waiting for index release on context " + context.getId( ) );
 217         }
 218     }
 219
 220     @Override
 221     public void pack( final ArchivaIndexingContext context ) throws IndexUpdateFailedException
 222     {
 223         executeUpdateFunction( context, indexingContext -> {
 224                 try
 225                 {
 226                     IndexPackingRequest request = new IndexPackingRequest( indexingContext,
 227                         indexingContext.acquireIndexSearcher( ).getIndexReader( ),
 228                         indexingContext.getIndexDirectoryFile( ) );
 229                     indexPacker.packIndex( request );
 230                     indexingContext.updateTimestamp( true );
 231                 }
 232                 catch ( IOException e )
 233                 {
 234                     log.error( "IOException while packing index of context " + context.getId( ) + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ) );
 235                     throw new IndexUpdateFailedException( "IOException during update of " + context.getId( ), e );
 236                 }
 237             }
 238         );
 239
 240     }
 241
 242     @Override
 243     public void scan(final ArchivaIndexingContext context) throws IndexUpdateFailedException
 244     {
 245         executeUpdateFunction( context, indexingContext -> {
 246             DefaultScannerListener listener = new DefaultScannerListener( indexingContext, indexerEngine, true, null );
 247             ScanningRequest request = new ScanningRequest( indexingContext, listener );
 248             ScanningResult result = scanner.scan( request );
 249             if ( result.hasExceptions( ) )
 250             {
 251                 log.error( "Exceptions occured during index scan of " + context.getId( ) );
 252                 result.getExceptions( ).stream( ).map( e -> e.getMessage( ) ).distinct( ).limit( 5 ).forEach(
 253                     s -> log.error( "Message: " + s )
 254                 );
 255             }
 256
 257         } );
 258     }
 259
 260     @Override
 261     public void update(final ArchivaIndexingContext context, final boolean fullUpdate) throws IndexUpdateFailedException
 262     {
 263         log.info( "start download remote index for remote repository {}", context.getRepository( ).getId( ) );
 264         URI remoteUpdateUri;
 265         if ( !( context.getRepository( ) instanceof RemoteRepository ) || !(context.getRepository().supportsFeature(RemoteIndexFeature.class)) )
 266         {
 267             throw new IndexUpdateFailedException( "The context is not associated to a remote repository with remote index " + context.getId( ) );
 268         } else {
 269             RemoteIndexFeature rif = context.getRepository().getFeature(RemoteIndexFeature.class).get();
 270             remoteUpdateUri = context.getRepository().getLocation().resolve(rif.getIndexUri());
 271         }
 272         final RemoteRepository remoteRepository = (RemoteRepository) context.getRepository( );
 273
 274         executeUpdateFunction( context,
 275             indexingContext -> {
 276                 try
 277                 {
 278                     // create a temp directory to download files
 279                     Path tempIndexDirectory = Paths.get( indexingContext.getIndexDirectoryFile( ).getParent( ), ".tmpIndex" );
 280                     Path indexCacheDirectory = Paths.get( indexingContext.getIndexDirectoryFile( ).getParent( ), ".indexCache" );
 281                     Files.createDirectories( indexCacheDirectory );
 282                     if ( Files.exists( tempIndexDirectory ) )
 283                     {
 284                         org.apache.archiva.common.utils.FileUtils.deleteDirectory( tempIndexDirectory );
 285                     }
 286                     Files.createDirectories( tempIndexDirectory );
 287                     tempIndexDirectory.toFile( ).deleteOnExit( );
 288                     String baseIndexUrl = indexingContext.getIndexUpdateUrl( );
 289
 290                     String wagonProtocol = remoteUpdateUri.toURL( ).getProtocol( );
 291
 292                     NetworkProxy networkProxy = null;
 293                     if ( remoteRepository.supportsFeature( RemoteIndexFeature.class ) )
 294                     {
 295                         RemoteIndexFeature rif = remoteRepository.getFeature( RemoteIndexFeature.class ).get( );
 296                         if ( StringUtils.isNotBlank( rif.getProxyId( ) ) )
 297                         {
 298                             networkProxy = proxyRegistry.getNetworkProxy( rif.getProxyId( ) );
 299                             if ( networkProxy == null )
 300                             {
 301                                 log.warn(
 302                                     "your remote repository is configured to download remote index trought a proxy we cannot find id:{}",
 303                                     rif.getProxyId( ) );
 304                             }
 305                         }
 306
 307                         final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon(
 308                             new WagonFactoryRequest( wagonProtocol, remoteRepository.getExtraHeaders( ) ).networkProxy(
 309                                 networkProxy )
 310                         );
 311                         int readTimeout = (int) rif.getDownloadTimeout( ).toMillis( ) * 1000;
 312                         wagon.setReadTimeout( readTimeout );
 313                         wagon.setTimeout( (int) remoteRepository.getTimeout( ).toMillis( ) * 1000 );
 314
 315                         if ( wagon instanceof AbstractHttpClientWagon )
 316                         {
 317                             HttpConfiguration httpConfiguration = new HttpConfiguration( );
 318                             HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration( );
 319                             httpMethodConfiguration.setUsePreemptive( true );
 320                             httpMethodConfiguration.setReadTimeout( readTimeout );
 321                             httpConfiguration.setGet( httpMethodConfiguration );
 322                             AbstractHttpClientWagon.class.cast( wagon ).setHttpConfiguration( httpConfiguration );
 323                         }
 324
 325                         wagon.addTransferListener( new DownloadListener( ) );
 326                         ProxyInfo proxyInfo = null;
 327                         if ( networkProxy != null )
 328                         {
 329                             proxyInfo = new ProxyInfo( );
 330                             proxyInfo.setType( networkProxy.getProtocol( ) );
 331                             proxyInfo.setHost( networkProxy.getHost( ) );
 332                             proxyInfo.setPort( networkProxy.getPort( ) );
 333                             proxyInfo.setUserName( networkProxy.getUsername( ) );
 334                             proxyInfo.setPassword( networkProxy.getPassword( ) );
 335                         }
 336                         AuthenticationInfo authenticationInfo = null;
 337                         if ( remoteRepository.getLoginCredentials( ) != null && ( remoteRepository.getLoginCredentials( ) instanceof PasswordCredentials ) )
 338                         {
 339                             PasswordCredentials creds = (PasswordCredentials) remoteRepository.getLoginCredentials( );
 340                             authenticationInfo = new AuthenticationInfo( );
 341                             authenticationInfo.setUserName( creds.getUsername( ) );
 342                             authenticationInfo.setPassword( new String( creds.getPassword( ) ) );
 343                         }
 344                         wagon.connect( new org.apache.maven.wagon.repository.Repository( remoteRepository.getId( ), baseIndexUrl ), authenticationInfo,
 345                             proxyInfo );
 346
 347                         Path indexDirectory = indexingContext.getIndexDirectoryFile( ).toPath( );
 348                         if ( !Files.exists( indexDirectory ) )
 349                         {
 350                             Files.createDirectories( indexDirectory );
 351                         }
 352
 353                         ResourceFetcher resourceFetcher =
 354                             new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository );
 355                         IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
 356                         request.setForceFullUpdate( fullUpdate );
 357                         request.setLocalIndexCacheDir( indexCacheDirectory.toFile( ) );
 358
 359                         indexUpdater.fetchAndUpdateIndex( request );
 360
 361                         indexingContext.updateTimestamp( true );
 362                     }
 363
 364                 }
 365                 catch ( AuthenticationException e )
 366                 {
 367                     log.error( "Could not login to the remote proxy for updating index of {}", remoteRepository.getId( ), e );
 368                     throw new IndexUpdateFailedException( "Login in to proxy failed while updating remote repository " + remoteRepository.getId( ), e );
 369                 }
 370                 catch ( ConnectionException e )
 371                 {
 372                     log.error( "Connection error during index update for remote repository {}", remoteRepository.getId( ), e );
 373                     throw new IndexUpdateFailedException( "Connection error during index update for remote repository " + remoteRepository.getId( ), e );
 374                 }
 375                 catch ( MalformedURLException e )
 376                 {
 377                     log.error( "URL for remote index update of remote repository {} is not correct {}", remoteRepository.getId( ), remoteUpdateUri, e );
 378                     throw new IndexUpdateFailedException( "URL for remote index update of repository is not correct " + remoteUpdateUri, e );
 379                 }
 380                 catch ( IOException e )
 381                 {
 382                     log.error( "IOException during index update of remote repository {}: {}", remoteRepository.getId( ), e.getMessage( ), e );
 383                     throw new IndexUpdateFailedException( "IOException during index update of remote repository " + remoteRepository.getId( )
 384                         + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ), e );
 385                 }
 386                 catch ( WagonFactoryException e )
 387                 {
 388                     log.error( "Wagon for remote index download of {} could not be created: {}", remoteRepository.getId( ), e.getMessage( ), e );
 389                     throw new IndexUpdateFailedException( "Error while updating the remote index of " + remoteRepository.getId( ), e );
 390                 }
 391             } );
 392
 393     }
 394
 395     @Override
 396     public void addArtifactsToIndex( final ArchivaIndexingContext context, final Collection<URI> artifactReference ) throws IndexUpdateFailedException
 397     {
 398         final URI ctxUri = context.getPath();
 399         executeUpdateFunction(context, indexingContext -> {
 400             Collection<ArtifactContext> artifacts = artifactReference.stream().map(r -> artifactContextProducer.getArtifactContext(indexingContext, Paths.get(ctxUri.resolve(r)).toFile())).collect(Collectors.toList());
 401             try {
 402                 indexer.addArtifactsToIndex(artifacts, indexingContext);
 403             } catch (IOException e) {
 404                 log.error("IOException while adding artifact {}", e.getMessage(), e);
 405                 throw new IndexUpdateFailedException("Error occured while adding artifact to index of "+context.getId()
 406                 + (StringUtils.isNotEmpty(e.getMessage()) ? ": "+e.getMessage() : ""));
 407             }
 408         });
 409     }
 410
 411     @Override
 412     public void removeArtifactsFromIndex( ArchivaIndexingContext context, Collection<URI> artifactReference ) throws IndexUpdateFailedException
 413     {
 414         final URI ctxUri = context.getPath();
 415         executeUpdateFunction(context, indexingContext -> {
 416             Collection<ArtifactContext> artifacts = artifactReference.stream().map(r -> artifactContextProducer.getArtifactContext(indexingContext, Paths.get(ctxUri.resolve(r)).toFile())).collect(Collectors.toList());
 417             try {
 418                 indexer.deleteArtifactsFromIndex(artifacts, indexingContext);
 419             } catch (IOException e) {
 420                 log.error("IOException while removing artifact {}", e.getMessage(), e);
 421                 throw new IndexUpdateFailedException("Error occured while removing artifact from index of "+context.getId()
 422                         + (StringUtils.isNotEmpty(e.getMessage()) ? ": "+e.getMessage() : ""));
 423             }
 424         });
 425
 426     }
 427
 428     @Override
 429     public boolean supportsRepository( RepositoryType type )
 430     {
 431         return type == RepositoryType.MAVEN;
 432     }
 433
 434     @Override
 435     public ArchivaIndexingContext createContext( Repository repository ) throws IndexCreationFailedException
 436     {
 437         log.debug("Creating context for repo {}, type: {}", repository.getId(), repository.getType());
 438         if ( repository.getType( ) != RepositoryType.MAVEN )
 439         {
 440             throw new UnsupportedRepositoryTypeException( repository.getType( ) );
 441         }
 442         IndexingContext mvnCtx = null;
 443         try
 444         {
 445             if ( repository instanceof RemoteRepository )
 446             {
 447                 mvnCtx = createRemoteContext( (RemoteRepository) repository );
 448             }
 449             else if ( repository instanceof ManagedRepository )
 450             {
 451                 mvnCtx = createManagedContext( (ManagedRepository) repository );
 452             }
 453         }
 454         catch ( IOException e )
 455         {
 456             log.error( "IOException during context creation " + e.getMessage( ), e );
 457             throw new IndexCreationFailedException( "Could not create index context for repository " + repository.getId( )
 458                 + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ), e );
 459         }
 460         MavenIndexContext context = new MavenIndexContext( repository, mvnCtx );
 461
 462         return context;
 463     }
 464
 465     @Override
 466     public ArchivaIndexingContext reset(ArchivaIndexingContext context) throws IndexUpdateFailedException {
 467         ArchivaIndexingContext ctx;
 468         executeUpdateFunction(context, indexingContext -> {
 469             try {
 470                 indexingContext.close(true);
 471             } catch (IOException e) {
 472                 log.warn("Index close failed");
 473             }
 474             try {
 475                 FileUtils.deleteDirectory(Paths.get(context.getPath()));
 476             } catch (IOException e) {
 477                 throw new IndexUpdateFailedException("Could not delete index files");
 478             }
 479         });
 480         try {
 481             Repository repo = context.getRepository();
 482             ctx = createContext(context.getRepository());
 483             if (repo instanceof EditableRepository) {
 484                 ((EditableRepository)repo).setIndexingContext(ctx);
 485             }
 486         } catch (IndexCreationFailedException e) {
 487             throw new IndexUpdateFailedException("Could not create index");
 488         }
 489         return ctx;
 490     }
 491
 492     @Override
 493     public ArchivaIndexingContext move(ArchivaIndexingContext context, Repository repo) throws IndexCreationFailedException {
 494         if (context==null) {
 495             return null;
 496         }
 497         if (context.supports(IndexingContext.class)) {
 498             try {
 499                 StorageAsset newPath = getIndexPath(repo);
 500                 IndexingContext ctx = context.getBaseContext(IndexingContext.class);
 501                 Path oldPath = ctx.getIndexDirectoryFile().toPath();
 502                 if (oldPath.equals(newPath)) {
 503                     // Nothing to do, if path does not change
 504                     return context;
 505                 }
 506                 if (!Files.exists(oldPath)) {
 507                     return createContext(repo);
 508                 } else if (context.isEmpty()) {
 509                     context.close();
 510                     return createContext(repo);
 511                 } else {
 512                     context.close(false);
 513                     Files.move(oldPath, newPath.getFilePath());
 514                     return createContext(repo);
 515                 }
 516             } catch (IOException e) {
 517                 log.error("IOException while moving index directory {}", e.getMessage(), e);
 518                 throw new IndexCreationFailedException("Could not recreated the index.", e);
 519             } catch (UnsupportedBaseContextException e) {
 520                 throw new IndexCreationFailedException("The given context, is not a maven context.");
 521             }
 522         } else {
 523             throw new IndexCreationFailedException("Bad context type. This is not a maven context.");
 524         }
 525     }
 526
 527     @Override
 528     public void updateLocalIndexPath(Repository repo) {
 529         if (repo.supportsFeature(IndexCreationFeature.class)) {
 530             IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
 531             try {
 532                 icf.setLocalIndexPath(getIndexPath(repo));
 533                 icf.setLocalPackedIndexPath(getPackedIndexPath(repo));
 534             } catch (IOException e) {
 535                 log.error("Could not set local index path for {}. New URI: {}", repo.getId(), icf.getIndexPath());
 536             }
 537         }
 538     }
 539
 540     @Override
 541     public ArchivaIndexingContext mergeContexts(Repository destinationRepo, List<ArchivaIndexingContext> contexts,
 542                                                 boolean packIndex) throws UnsupportedOperationException,
 543             IndexCreationFailedException, IllegalArgumentException {
 544         if (!destinationRepo.supportsFeature(IndexCreationFeature.class)) {
 545             throw new IllegalArgumentException("The given repository does not support the indexcreation feature");
 546         }
 547         Path mergedIndexDirectory = null;
 548         try {
 549             mergedIndexDirectory = Files.createTempDirectory("archivaMergedIndex");
 550         } catch (IOException e) {
 551             log.error("Could not create temporary directory for merged index: {}", e.getMessage(), e);
 552             throw new IndexCreationFailedException("IO error while creating temporary directory for merged index: "+e.getMessage(), e);
 553         }
 554         IndexCreationFeature indexCreationFeature = destinationRepo.getFeature(IndexCreationFeature.class).get();
 555         if (indexCreationFeature.getLocalIndexPath()== null) {
 556             throw new IllegalArgumentException("The given repository does not have a local index path");
 557         }
 558         StorageAsset destinationPath = indexCreationFeature.getLocalIndexPath();
 559
 560         String tempRepoId = mergedIndexDirectory.getFileName().toString();
 561
 562         try
 563         {
 564             Path indexLocation = destinationPath.getFilePath();
 565
 566             List<IndexingContext> members = contexts.stream( ).filter(ctx -> ctx.supports(IndexingContext.class)).map( ctx ->
 567             {
 568                 try {
 569                     return ctx.getBaseContext(IndexingContext.class);
 570                 } catch (UnsupportedBaseContextException e) {
 571                     // does not happen here
 572                     return null;
 573                 }
 574             }).filter( Objects::nonNull ).collect( Collectors.toList() );
 575             ContextMemberProvider memberProvider = new StaticContextMemberProvider(members);
 576             IndexingContext mergedCtx = indexer.createMergedIndexingContext( tempRepoId, tempRepoId, mergedIndexDirectory.toFile(),
 577                     indexLocation.toFile(), true, memberProvider);
 578             mergedCtx.optimize();
 579
 580             if ( packIndex )
 581             {
 582                 IndexPackingRequest request = new IndexPackingRequest( mergedCtx, //
 583                         mergedCtx.acquireIndexSearcher().getIndexReader(), //
 584                         indexLocation.toFile() );
 585                 indexPacker.packIndex( request );
 586             }
 587
 588             return new MavenIndexContext(destinationRepo, mergedCtx);
 589         }
 590         catch ( IOException e)
 591         {
 592             throw new IndexCreationFailedException( "IO Error during index merge: "+ e.getMessage(), e );
 593         }
 594     }
 595
 596     private StorageAsset getIndexPath(URI indexDir, Path repoDir, String defaultDir) throws IOException
 597     {
 598         String indexPath = indexDir.getPath();
 599         Path indexDirectory = null;
 600         if ( ! StringUtils.isEmpty(indexDir.toString( ) ) )
 601         {
 602
 603             indexDirectory = PathUtil.getPathFromUri( indexDir );
 604             // not absolute so create it in repository directory
 605             if ( indexDirectory.isAbsolute( ) )
 606             {
 607                 indexPath = indexDirectory.getFileName().toString();
 608             }
 609             else
 610             {
 611                 indexDirectory = repoDir.resolve( indexDirectory );
 612             }
 613         }
 614         else
 615         {
 616             indexDirectory = repoDir.resolve( defaultDir );
 617             indexPath = defaultDir;
 618         }
 619
 620         if ( !Files.exists( indexDirectory ) )
 621         {
 622             Files.createDirectories( indexDirectory );
 623         }
 624         return new FilesystemAsset( indexPath, indexDirectory);
 625     }
 626
 627     private StorageAsset getIndexPath( Repository repo) throws IOException {
 628         IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
 629         return getIndexPath( icf.getIndexPath(), repo.getAsset( "" ).getFilePath(), DEFAULT_INDEX_PATH);
 630     }
 631
 632     private StorageAsset getPackedIndexPath(Repository repo) throws IOException {
 633         IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
 634         return getIndexPath(icf.getPackedIndexPath(), repo.getAsset( "" ).getFilePath(), DEFAULT_PACKED_INDEX_PATH);
 635     }
 636
 637     private IndexingContext createRemoteContext(RemoteRepository remoteRepository ) throws IOException
 638     {
 639         Path appServerBase = archivaConfiguration.getAppServerBaseDir( );
 640
 641         String contextKey = "remote-" + remoteRepository.getId( );
 642
 643
 644         // create remote repository path
 645         Path repoDir = remoteRepository.getAsset( "" ).getFilePath();
 646         if ( !Files.exists( repoDir ) )
 647         {
 648             Files.createDirectories( repoDir );
 649         }
 650
 651         StorageAsset indexDirectory = null;
 652
 653         // is there configured indexDirectory ?
 654         if ( remoteRepository.supportsFeature( RemoteIndexFeature.class ) )
 655         {
 656             RemoteIndexFeature rif = remoteRepository.getFeature( RemoteIndexFeature.class ).get( );
 657             indexDirectory = getIndexPath(remoteRepository);
 658             String remoteIndexUrl = calculateIndexRemoteUrl( remoteRepository.getLocation( ), rif );
 659             try
 660             {
 661
 662                 return getIndexingContext( remoteRepository, contextKey, repoDir, indexDirectory, remoteIndexUrl );
 663             }
 664             catch ( IndexFormatTooOldException e )
 665             {
 666                 // existing index with an old lucene format so we need to delete it!!!
 667                 // delete it first then recreate it.
 668                 log.warn( "the index of repository {} is too old we have to delete and recreate it", //
 669                     remoteRepository.getId( ) );
 670                 org.apache.archiva.common.utils.FileUtils.deleteDirectory( indexDirectory.getFilePath() );
 671                 return getIndexingContext( remoteRepository, contextKey, repoDir, indexDirectory, remoteIndexUrl );
 672
 673             }
 674         }
 675         else
 676         {
 677             throw new IOException( "No remote index defined" );
 678         }
 679     }
 680
 681     private IndexingContext getIndexingContext( Repository repository, String contextKey, Path repoDir, StorageAsset indexDirectory, String indexUrl ) throws IOException
 682     {
 683         try
 684         {
 685             if (!Files.exists(indexDirectory.getFilePath())) {
 686                 Files.createDirectories(indexDirectory.getFilePath());
 687             }
 688             return indexer.createIndexingContext( contextKey, repository.getId( ), repoDir.toFile( ), indexDirectory.getFilePath( ).toFile( ),
 689                 repository.getLocation( ) == null ? null : repository.getLocation( ).toString( ),
 690                 indexUrl,
 691                 true, false,
 692                 indexCreators );
 693         } catch (Exception e) {
 694             log.error("Could not create index for asset {}", indexDirectory);
 695             throw new IOException(e);
 696         }
 697     }
 698
 699     private IndexingContext createManagedContext( ManagedRepository repository ) throws IOException
 700     {
 701
 702         IndexingContext context;
 703         // take care first about repository location as can be relative
 704         Path repositoryDirectory = repository.getAsset( "" ).getFilePath();
 705
 706         if ( !Files.exists( repositoryDirectory ) )
 707         {
 708             try
 709             {
 710                 Files.createDirectories( repositoryDirectory );
 711             }
 712             catch ( IOException e )
 713             {
 714                 log.error( "Could not create directory {}", repositoryDirectory );
 715             }
 716         }
 717
 718         StorageAsset indexDirectory = null;
 719
 720         if ( repository.supportsFeature( IndexCreationFeature.class ) )
 721         {
 722             indexDirectory = getIndexPath(repository);
 723             log.debug( "Preparing index at {}", indexDirectory );
 724
 725             String indexUrl = repositoryDirectory.toUri( ).toURL( ).toExternalForm( );
 726             try
 727             {
 728                 context = getIndexingContext( repository, repository.getId( ), repositoryDirectory, indexDirectory, indexUrl );
 729                 context.setSearchable( repository.isScanned( ) );
 730             }
 731             catch ( IndexFormatTooOldException e )
 732             {
 733                 // existing index with an old lucene format so we need to delete it!!!
 734                 // delete it first then recreate it.
 735                 log.warn( "the index of repository {} is too old we have to delete and recreate it", //
 736                     repository.getId( ) );
 737                 org.apache.archiva.common.utils.FileUtils.deleteDirectory( indexDirectory.getFilePath() );
 738                 context = getIndexingContext( repository, repository.getId( ), repositoryDirectory, indexDirectory, indexUrl );
 739                 context.setSearchable( repository.isScanned( ) );
 740             }
 741             return context;
 742         }
 743         else
 744         {
 745             throw new IOException( "No repository index defined" );
 746         }
 747     }
 748
 749     private String calculateIndexRemoteUrl( URI baseUri, RemoteIndexFeature rif )
 750     {
 751         if ( rif.getIndexUri( ) == null )
 752         {
 753             return baseUri.resolve( DEFAULT_INDEX_PATH ).toString( );
 754         }
 755         else
 756         {
 757             return baseUri.resolve( rif.getIndexUri( ) ).toString( );
 758         }
 759     }
 760
 761     private static final class DownloadListener
 762         implements TransferListener
 763     {
 764         private Logger log = LoggerFactory.getLogger( getClass( ) );
 765
 766         private String resourceName;
 767
 768         private long startTime;
 769
 770         private int totalLength = 0;
 771
 772         @Override
 773         public void transferInitiated( TransferEvent transferEvent )
 774         {
 775             startTime = System.currentTimeMillis( );
 776             resourceName = transferEvent.getResource( ).getName( );
 777             log.debug( "initiate transfer of {}", resourceName );
 778         }
 779
 780         @Override
 781         public void transferStarted( TransferEvent transferEvent )
 782         {
 783             this.totalLength = 0;
 784             resourceName = transferEvent.getResource( ).getName( );
 785             log.info( "start transfer of {}", transferEvent.getResource( ).getName( ) );
 786         }
 787
 788         @Override
 789         public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
 790         {
 791             log.debug( "transfer of {} : {}/{}", transferEvent.getResource( ).getName( ), buffer.length, length );
 792             this.totalLength += length;
 793         }
 794
 795         @Override
 796         public void transferCompleted( TransferEvent transferEvent )
 797         {
 798             resourceName = transferEvent.getResource( ).getName( );
 799             long endTime = System.currentTimeMillis( );
 800             log.info( "end of transfer file {} {} kb: {}s", transferEvent.getResource( ).getName( ),
 801                 this.totalLength / 1024, ( endTime - startTime ) / 1000 );
 802         }
 803
 804         @Override
 805         public void transferError( TransferEvent transferEvent )
 806         {
 807             log.info( "error of transfer file {}: {}", transferEvent.getResource( ).getName( ),
 808                 transferEvent.getException( ).getMessage( ), transferEvent.getException( ) );
 809         }
 810
 811         @Override
 812         public void debug( String message )
 813         {
 814             log.debug( "transfer debug {}", message );
 815         }
 816     }
 817
 818     private static class WagonResourceFetcher
 819         implements ResourceFetcher
 820     {
 821
 822         Logger log;
 823
 824         Path tempIndexDirectory;
 825
 826         Wagon wagon;
 827
 828         RemoteRepository remoteRepository;
 829
 830         private WagonResourceFetcher( Logger log, Path tempIndexDirectory, Wagon wagon,
 831                                       RemoteRepository remoteRepository )
 832         {
 833             this.log = log;
 834             this.tempIndexDirectory = tempIndexDirectory;
 835             this.wagon = wagon;
 836             this.remoteRepository = remoteRepository;
 837         }
 838
 839         @Override
 840         public void connect( String id, String url )
 841             throws IOException
 842         {
 843             //no op
 844         }
 845
 846         @Override
 847         public void disconnect( )
 848             throws IOException
 849         {
 850             // no op
 851         }
 852
 853         @Override
 854         public InputStream retrieve( String name )
 855             throws IOException, FileNotFoundException
 856         {
 857             try
 858             {
 859                 log.info( "index update retrieve file, name:{}", name );
 860                 Path file = tempIndexDirectory.resolve( name );
 861                 Files.deleteIfExists( file );
 862                 file.toFile( ).deleteOnExit( );
 863                 wagon.get( addParameters( name, remoteRepository ), file.toFile( ) );
 864                 return Files.newInputStream( file );
 865             }
 866             catch ( AuthorizationException | TransferFailedException e )
 867             {
 868                 throw new IOException( e.getMessage( ), e );
 869             }
 870             catch ( ResourceDoesNotExistException e )
 871             {
 872                 FileNotFoundException fnfe = new FileNotFoundException( e.getMessage( ) );
 873                 fnfe.initCause( e );
 874                 throw fnfe;
 875             }
 876         }
 877
 878         // FIXME remove crappy copy/paste
 879         protected String addParameters( String path, RemoteRepository remoteRepository )
 880         {
 881             if ( remoteRepository.getExtraParameters( ).isEmpty( ) )
 882             {
 883                 return path;
 884             }
 885
 886             boolean question = false;
 887
 888             StringBuilder res = new StringBuilder( path == null ? "" : path );
 889
 890             for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters( ).entrySet( ) )
 891             {
 892                 if ( !question )
 893                 {
 894                     res.append( '?' ).append( entry.getKey( ) ).append( '=' ).append( entry.getValue( ) );
 895                 }
 896             }
 897
 898             return res.toString( );
 899         }
 900
 901     }
 902 }