source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.indexer.maven;
   2
   3 /*
   4  * Licensed to the Apache Software Foundation (ASF) under one
   5  * or more contributor license agreements.  See the NOTICE file
   6  * distributed with this work for additional information
   7  * regarding copyright ownership.  The ASF licenses this file
   8  * to you under the Apache License, Version 2.0 (the
   9  * "License"); you may not use this file except in compliance
  10  * with the License.  You may obtain a copy of the License at
  11  *
  12  *  http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing,
  15  * software distributed under the License is distributed on an
  16  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  17  * KIND, either express or implied.  See the License for the
  18  * specific language governing permissions and limitations
  19  * under the License.
  20  */
  21
  22 import org.apache.archiva.common.utils.FileUtils;
  23 import org.apache.archiva.common.utils.PathUtil;
  24 import org.apache.archiva.configuration.ArchivaConfiguration;
  25 import org.apache.archiva.indexer.ArchivaIndexManager;
  26 import org.apache.archiva.indexer.ArchivaIndexingContext;
  27 import org.apache.archiva.indexer.IndexCreationFailedException;
  28 import org.apache.archiva.indexer.IndexUpdateFailedException;
  29 import org.apache.archiva.indexer.UnsupportedBaseContextException;
  30 import org.apache.archiva.proxy.ProxyRegistry;
  31 import org.apache.archiva.proxy.maven.WagonFactory;
  32 import org.apache.archiva.proxy.maven.WagonFactoryException;
  33 import org.apache.archiva.proxy.maven.WagonFactoryRequest;
  34 import org.apache.archiva.proxy.model.NetworkProxy;
  35 import org.apache.archiva.repository.EditableRepository;
  36 import org.apache.archiva.repository.ManagedRepository;
  37 import org.apache.archiva.repository.PasswordCredentials;
  38 import org.apache.archiva.repository.RemoteRepository;
  39 import org.apache.archiva.repository.Repository;
  40 import org.apache.archiva.repository.RepositoryType;
  41 import org.apache.archiva.repository.UnsupportedRepositoryTypeException;
  42 import org.apache.archiva.repository.storage.FilesystemStorage;
  43 import org.apache.archiva.repository.storage.RepositoryStorage;
  44 import org.apache.archiva.repository.storage.StorageAsset;
  45 import org.apache.archiva.repository.features.IndexCreationFeature;
  46 import org.apache.archiva.repository.features.RemoteIndexFeature;
  47 import org.apache.archiva.repository.storage.StorageUtil;
  48 import org.apache.commons.lang3.StringUtils;
  49 import org.apache.maven.index.ArtifactContext;
  50 import org.apache.maven.index.ArtifactContextProducer;
  51 import org.apache.maven.index.DefaultScannerListener;
  52 import org.apache.maven.index.Indexer;
  53 import org.apache.maven.index.IndexerEngine;
  54 import org.apache.maven.index.Scanner;
  55 import org.apache.maven.index.ScanningRequest;
  56 import org.apache.maven.index.ScanningResult;
  57 import org.apache.maven.index.context.ContextMemberProvider;
  58 import org.apache.maven.index.context.IndexCreator;
  59 import org.apache.maven.index.context.IndexingContext;
  60 import org.apache.maven.index.context.StaticContextMemberProvider;
  61 import org.apache.maven.index.packer.IndexPacker;
  62 import org.apache.maven.index.packer.IndexPackingRequest;
  63 import org.apache.maven.index.updater.IndexUpdateRequest;
  64 import org.apache.maven.index.updater.IndexUpdater;
  65 import org.apache.maven.index.updater.ResourceFetcher;
  66 import org.apache.maven.index_shaded.lucene.index.IndexFormatTooOldException;
  67 import org.apache.maven.wagon.ConnectionException;
  68 import org.apache.maven.wagon.ResourceDoesNotExistException;
  69 import org.apache.maven.wagon.StreamWagon;
  70 import org.apache.maven.wagon.TransferFailedException;
  71 import org.apache.maven.wagon.Wagon;
  72 import org.apache.maven.wagon.authentication.AuthenticationException;
  73 import org.apache.maven.wagon.authentication.AuthenticationInfo;
  74 import org.apache.maven.wagon.authorization.AuthorizationException;
  75 import org.apache.maven.wagon.events.TransferEvent;
  76 import org.apache.maven.wagon.events.TransferListener;
  77 import org.apache.maven.wagon.proxy.ProxyInfo;
  78 import org.apache.maven.wagon.shared.http.AbstractHttpClientWagon;
  79 import org.apache.maven.wagon.shared.http.HttpConfiguration;
  80 import org.apache.maven.wagon.shared.http.HttpMethodConfiguration;
  81 import org.slf4j.Logger;
  82 import org.slf4j.LoggerFactory;
  83 import org.springframework.stereotype.Service;
  84
  85 import javax.inject.Inject;
  86 import java.io.FileNotFoundException;
  87 import java.io.IOException;
  88 import java.io.InputStream;
  89 import java.net.MalformedURLException;
  90 import java.net.URI;
  91 import java.nio.file.Files;
  92 import java.nio.file.Path;
  93 import java.nio.file.Paths;
  94 import java.util.Collection;
  95 import java.util.List;
  96 import java.util.Map;
  97 import java.util.Objects;
  98 import java.util.concurrent.ConcurrentSkipListSet;
  99 import java.util.stream.Collectors;
 100
 101 /**
 102  * Maven implementation of index manager.
 103  * The index manager is a singleton, so we try to make sure, that index operations are not running
 104  * parallel by synchronizing on the index path.
 105  * A update operation waits for parallel running methods to finish before starting, but after a certain
 106  * time of retries a IndexUpdateFailedException is thrown.
 107  */
 108 @Service( "archivaIndexManager#maven" )
 109 public class MavenIndexManager implements ArchivaIndexManager {
 110
 111     private static final Logger log = LoggerFactory.getLogger( MavenIndexManager.class );
 112
 113     @Inject
 114     private Indexer indexer;
 115
 116     @Inject
 117     private IndexerEngine indexerEngine;
 118
 119     @Inject
 120     private List<? extends IndexCreator> indexCreators;
 121
 122     @Inject
 123     private IndexPacker indexPacker;
 124
 125     @Inject
 126     private Scanner scanner;
 127
 128     @Inject
 129     private ArchivaConfiguration archivaConfiguration;
 130
 131     @Inject
 132     private WagonFactory wagonFactory;
 133
 134     @Inject
 135     private IndexUpdater indexUpdater;
 136
 137     @Inject
 138     private ArtifactContextProducer artifactContextProducer;
 139
 140     @Inject
 141     private ProxyRegistry proxyRegistry;
 142
 143
 144     private ConcurrentSkipListSet<StorageAsset> activeContexts = new ConcurrentSkipListSet<>( );
 145
 146     private static final int WAIT_TIME = 100;
 147     private static final int MAX_WAIT = 10;
 148
 149
 150     public static IndexingContext getMvnContext( ArchivaIndexingContext context ) throws UnsupportedBaseContextException
 151     {
 152         if ( !context.supports( IndexingContext.class ) )
 153         {
 154             log.error( "The provided archiva index context does not support the maven IndexingContext" );
 155             throw new UnsupportedBaseContextException( "The context does not support the Maven IndexingContext" );
 156         }
 157         return context.getBaseContext( IndexingContext.class );
 158     }
 159
 160     private StorageAsset getIndexPath( ArchivaIndexingContext ctx )
 161     {
 162         return ctx.getPath( );
 163     }
 164
 165     @FunctionalInterface
 166     interface IndexUpdateConsumer
 167     {
 168
 169         void accept( IndexingContext indexingContext ) throws IndexUpdateFailedException;
 170     }
 171
 172     /*
 173      * This method is used to do some actions around the update execution code. And to make sure, that no other
 174      * method is running on the same index.
 175      */
 176     private void executeUpdateFunction( ArchivaIndexingContext context, IndexUpdateConsumer function ) throws IndexUpdateFailedException
 177     {
 178         IndexingContext indexingContext = null;
 179         try
 180         {
 181             indexingContext = getMvnContext( context );
 182         }
 183         catch ( UnsupportedBaseContextException e )
 184         {
 185             throw new IndexUpdateFailedException( "Maven index is not supported by this context", e );
 186         }
 187         final StorageAsset ctxPath = getIndexPath( context );
 188         int loop = MAX_WAIT;
 189         boolean active = false;
 190         while ( loop-- > 0 && !active )
 191         {
 192             active = activeContexts.add( ctxPath );
 193             try
 194             {
 195                 Thread.currentThread( ).sleep( WAIT_TIME );
 196             }
 197             catch ( InterruptedException e )
 198             {
 199                 // Ignore this
 200             }
 201         }
 202         if ( active )
 203         {
 204             try
 205             {
 206                 function.accept( indexingContext );
 207             }
 208             finally
 209             {
 210                 activeContexts.remove( ctxPath );
 211             }
 212         }
 213         else
 214         {
 215             throw new IndexUpdateFailedException( "Timeout while waiting for index release on context " + context.getId( ) );
 216         }
 217     }
 218
 219     @Override
 220     public void pack( final ArchivaIndexingContext context ) throws IndexUpdateFailedException
 221     {
 222         executeUpdateFunction( context, indexingContext -> {
 223                 try
 224                 {
 225                     IndexPackingRequest request = new IndexPackingRequest( indexingContext,
 226                         indexingContext.acquireIndexSearcher( ).getIndexReader( ),
 227                         indexingContext.getIndexDirectoryFile( ) );
 228                     indexPacker.packIndex( request );
 229                     indexingContext.updateTimestamp( true );
 230                 }
 231                 catch ( IOException e )
 232                 {
 233                     log.error( "IOException while packing index of context " + context.getId( ) + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ) );
 234                     throw new IndexUpdateFailedException( "IOException during update of " + context.getId( ), e );
 235                 }
 236             }
 237         );
 238
 239     }
 240
 241     @Override
 242     public void scan(final ArchivaIndexingContext context) throws IndexUpdateFailedException
 243     {
 244         executeUpdateFunction( context, indexingContext -> {
 245             DefaultScannerListener listener = new DefaultScannerListener( indexingContext, indexerEngine, true, null );
 246             ScanningRequest request = new ScanningRequest( indexingContext, listener );
 247             ScanningResult result = scanner.scan( request );
 248             if ( result.hasExceptions( ) )
 249             {
 250                 log.error( "Exceptions occured during index scan of " + context.getId( ) );
 251                 result.getExceptions( ).stream( ).map( e -> e.getMessage( ) ).distinct( ).limit( 5 ).forEach(
 252                     s -> log.error( "Message: " + s )
 253                 );
 254             }
 255
 256         } );
 257     }
 258
 259     @Override
 260     public void update(final ArchivaIndexingContext context, final boolean fullUpdate) throws IndexUpdateFailedException
 261     {
 262         log.info( "start download remote index for remote repository {}", context.getRepository( ).getId( ) );
 263         URI remoteUpdateUri;
 264         if ( !( context.getRepository( ) instanceof RemoteRepository ) || !(context.getRepository().supportsFeature(RemoteIndexFeature.class)) )
 265         {
 266             throw new IndexUpdateFailedException( "The context is not associated to a remote repository with remote index " + context.getId( ) );
 267         } else {
 268             RemoteIndexFeature rif = context.getRepository().getFeature(RemoteIndexFeature.class).get();
 269             remoteUpdateUri = context.getRepository().getLocation().resolve(rif.getIndexUri());
 270         }
 271         final RemoteRepository remoteRepository = (RemoteRepository) context.getRepository( );
 272
 273         executeUpdateFunction( context,
 274             indexingContext -> {
 275                 try
 276                 {
 277                     // create a temp directory to download files
 278                     Path tempIndexDirectory = Paths.get( indexingContext.getIndexDirectoryFile( ).getParent( ), ".tmpIndex" );
 279                     Path indexCacheDirectory = Paths.get( indexingContext.getIndexDirectoryFile( ).getParent( ), ".indexCache" );
 280                     Files.createDirectories( indexCacheDirectory );
 281                     if ( Files.exists( tempIndexDirectory ) )
 282                     {
 283                         org.apache.archiva.common.utils.FileUtils.deleteDirectory( tempIndexDirectory );
 284                     }
 285                     Files.createDirectories( tempIndexDirectory );
 286                     tempIndexDirectory.toFile( ).deleteOnExit( );
 287                     String baseIndexUrl = indexingContext.getIndexUpdateUrl( );
 288
 289                     String wagonProtocol = remoteUpdateUri.toURL( ).getProtocol( );
 290
 291                     NetworkProxy networkProxy = null;
 292                     if ( remoteRepository.supportsFeature( RemoteIndexFeature.class ) )
 293                     {
 294                         RemoteIndexFeature rif = remoteRepository.getFeature( RemoteIndexFeature.class ).get( );
 295                         if ( StringUtils.isNotBlank( rif.getProxyId( ) ) )
 296                         {
 297                             networkProxy = proxyRegistry.getNetworkProxy( rif.getProxyId( ) );
 298                             if ( networkProxy == null )
 299                             {
 300                                 log.warn(
 301                                     "your remote repository is configured to download remote index trought a proxy we cannot find id:{}",
 302                                     rif.getProxyId( ) );
 303                             }
 304                         }
 305
 306                         final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon(
 307                             new WagonFactoryRequest( wagonProtocol, remoteRepository.getExtraHeaders( ) ).networkProxy(
 308                                 networkProxy )
 309                         );
 310                         int readTimeout = (int) rif.getDownloadTimeout( ).toMillis( ) * 1000;
 311                         wagon.setReadTimeout( readTimeout );
 312                         wagon.setTimeout( (int) remoteRepository.getTimeout( ).toMillis( ) * 1000 );
 313
 314                         if ( wagon instanceof AbstractHttpClientWagon )
 315                         {
 316                             HttpConfiguration httpConfiguration = new HttpConfiguration( );
 317                             HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration( );
 318                             httpMethodConfiguration.setUsePreemptive( true );
 319                             httpMethodConfiguration.setReadTimeout( readTimeout );
 320                             httpConfiguration.setGet( httpMethodConfiguration );
 321                             AbstractHttpClientWagon.class.cast( wagon ).setHttpConfiguration( httpConfiguration );
 322                         }
 323
 324                         wagon.addTransferListener( new DownloadListener( ) );
 325                         ProxyInfo proxyInfo = null;
 326                         if ( networkProxy != null )
 327                         {
 328                             proxyInfo = new ProxyInfo( );
 329                             proxyInfo.setType( networkProxy.getProtocol( ) );
 330                             proxyInfo.setHost( networkProxy.getHost( ) );
 331                             proxyInfo.setPort( networkProxy.getPort( ) );
 332                             proxyInfo.setUserName( networkProxy.getUsername( ) );
 333                             proxyInfo.setPassword( new String(networkProxy.getPassword( )) );
 334                         }
 335                         AuthenticationInfo authenticationInfo = null;
 336                         if ( remoteRepository.getLoginCredentials( ) != null && ( remoteRepository.getLoginCredentials( ) instanceof PasswordCredentials ) )
 337                         {
 338                             PasswordCredentials creds = (PasswordCredentials) remoteRepository.getLoginCredentials( );
 339                             authenticationInfo = new AuthenticationInfo( );
 340                             authenticationInfo.setUserName( creds.getUsername( ) );
 341                             authenticationInfo.setPassword( new String( creds.getPassword( ) ) );
 342                         }
 343                         wagon.connect( new org.apache.maven.wagon.repository.Repository( remoteRepository.getId( ), baseIndexUrl ), authenticationInfo,
 344                             proxyInfo );
 345
 346                         Path indexDirectory = indexingContext.getIndexDirectoryFile( ).toPath( );
 347                         if ( !Files.exists( indexDirectory ) )
 348                         {
 349                             Files.createDirectories( indexDirectory );
 350                         }
 351
 352                         ResourceFetcher resourceFetcher =
 353                             new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository );
 354                         IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
 355                         request.setForceFullUpdate( fullUpdate );
 356                         request.setLocalIndexCacheDir( indexCacheDirectory.toFile( ) );
 357
 358                         indexUpdater.fetchAndUpdateIndex( request );
 359
 360                         indexingContext.updateTimestamp( true );
 361                     }
 362
 363                 }
 364                 catch ( AuthenticationException e )
 365                 {
 366                     log.error( "Could not login to the remote proxy for updating index of {}", remoteRepository.getId( ), e );
 367                     throw new IndexUpdateFailedException( "Login in to proxy failed while updating remote repository " + remoteRepository.getId( ), e );
 368                 }
 369                 catch ( ConnectionException e )
 370                 {
 371                     log.error( "Connection error during index update for remote repository {}", remoteRepository.getId( ), e );
 372                     throw new IndexUpdateFailedException( "Connection error during index update for remote repository " + remoteRepository.getId( ), e );
 373                 }
 374                 catch ( MalformedURLException e )
 375                 {
 376                     log.error( "URL for remote index update of remote repository {} is not correct {}", remoteRepository.getId( ), remoteUpdateUri, e );
 377                     throw new IndexUpdateFailedException( "URL for remote index update of repository is not correct " + remoteUpdateUri, e );
 378                 }
 379                 catch ( IOException e )
 380                 {
 381                     log.error( "IOException during index update of remote repository {}: {}", remoteRepository.getId( ), e.getMessage( ), e );
 382                     throw new IndexUpdateFailedException( "IOException during index update of remote repository " + remoteRepository.getId( )
 383                         + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ), e );
 384                 }
 385                 catch ( WagonFactoryException e )
 386                 {
 387                     log.error( "Wagon for remote index download of {} could not be created: {}", remoteRepository.getId( ), e.getMessage( ), e );
 388                     throw new IndexUpdateFailedException( "Error while updating the remote index of " + remoteRepository.getId( ), e );
 389                 }
 390             } );
 391
 392     }
 393
 394     @Override
 395     public void addArtifactsToIndex( final ArchivaIndexingContext context, final Collection<URI> artifactReference ) throws IndexUpdateFailedException
 396     {
 397         final StorageAsset ctxUri = context.getPath();
 398         executeUpdateFunction(context, indexingContext -> {
 399             Collection<ArtifactContext> artifacts = artifactReference.stream().map(r -> artifactContextProducer.getArtifactContext(indexingContext, Paths.get(ctxUri.getFilePath().toUri().resolve(r)).toFile())).collect(Collectors.toList());
 400             try {
 401                 indexer.addArtifactsToIndex(artifacts, indexingContext);
 402             } catch (IOException e) {
 403                 log.error("IOException while adding artifact {}", e.getMessage(), e);
 404                 throw new IndexUpdateFailedException("Error occured while adding artifact to index of "+context.getId()
 405                 + (StringUtils.isNotEmpty(e.getMessage()) ? ": "+e.getMessage() : ""));
 406             }
 407         });
 408     }
 409
 410     @Override
 411     public void removeArtifactsFromIndex( ArchivaIndexingContext context, Collection<URI> artifactReference ) throws IndexUpdateFailedException
 412     {
 413         final StorageAsset ctxUri = context.getPath();
 414         executeUpdateFunction(context, indexingContext -> {
 415             Collection<ArtifactContext> artifacts = artifactReference.stream().map(r -> artifactContextProducer.getArtifactContext(indexingContext, Paths.get(ctxUri.getFilePath().toUri().resolve(r)).toFile())).collect(Collectors.toList());
 416             try {
 417                 indexer.deleteArtifactsFromIndex(artifacts, indexingContext);
 418             } catch (IOException e) {
 419                 log.error("IOException while removing artifact {}", e.getMessage(), e);
 420                 throw new IndexUpdateFailedException("Error occured while removing artifact from index of "+context.getId()
 421                         + (StringUtils.isNotEmpty(e.getMessage()) ? ": "+e.getMessage() : ""));
 422             }
 423         });
 424
 425     }
 426
 427     @Override
 428     public boolean supportsRepository( RepositoryType type )
 429     {
 430         return type == RepositoryType.MAVEN;
 431     }
 432
 433     @Override
 434     public ArchivaIndexingContext createContext( Repository repository ) throws IndexCreationFailedException
 435     {
 436         log.debug("Creating context for repo {}, type: {}", repository.getId(), repository.getType());
 437         if ( repository.getType( ) != RepositoryType.MAVEN )
 438         {
 439             throw new UnsupportedRepositoryTypeException( repository.getType( ) );
 440         }
 441         IndexingContext mvnCtx = null;
 442         try
 443         {
 444             if ( repository instanceof RemoteRepository )
 445             {
 446                 mvnCtx = createRemoteContext( (RemoteRepository) repository );
 447             }
 448             else if ( repository instanceof ManagedRepository )
 449             {
 450                 mvnCtx = createManagedContext( (ManagedRepository) repository );
 451             }
 452         }
 453         catch ( IOException e )
 454         {
 455             log.error( "IOException during context creation " + e.getMessage( ), e );
 456             throw new IndexCreationFailedException( "Could not create index context for repository " + repository.getId( )
 457                 + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ), e );
 458         }
 459
 460         return new MavenIndexContext( repository, mvnCtx );
 461     }
 462
 463     @Override
 464     public ArchivaIndexingContext reset(ArchivaIndexingContext context) throws IndexUpdateFailedException {
 465         ArchivaIndexingContext ctx;
 466         executeUpdateFunction(context, indexingContext -> {
 467             try {
 468                 indexingContext.close(true);
 469             } catch (IOException e) {
 470                 log.warn("Index close failed");
 471             }
 472             try {
 473                 StorageUtil.deleteRecursively(context.getPath());
 474             } catch (IOException e) {
 475                 throw new IndexUpdateFailedException("Could not delete index files");
 476             }
 477         });
 478         try {
 479             Repository repo = context.getRepository();
 480             ctx = createContext(context.getRepository());
 481             if (repo instanceof EditableRepository) {
 482                 ((EditableRepository)repo).setIndexingContext(ctx);
 483             }
 484         } catch (IndexCreationFailedException e) {
 485             throw new IndexUpdateFailedException("Could not create index");
 486         }
 487         return ctx;
 488     }
 489
 490     @Override
 491     public ArchivaIndexingContext move(ArchivaIndexingContext context, Repository repo) throws IndexCreationFailedException {
 492         if (context==null) {
 493             return null;
 494         }
 495         if (context.supports(IndexingContext.class)) {
 496             try {
 497                 StorageAsset newPath = getIndexPath(repo);
 498                 IndexingContext ctx = context.getBaseContext(IndexingContext.class);
 499                 Path oldPath = ctx.getIndexDirectoryFile().toPath();
 500                 if (oldPath.equals(newPath)) {
 501                     // Nothing to do, if path does not change
 502                     return context;
 503                 }
 504                 if (!Files.exists(oldPath)) {
 505                     return createContext(repo);
 506                 } else if (context.isEmpty()) {
 507                     context.close();
 508                     return createContext(repo);
 509                 } else {
 510                     context.close(false);
 511                     Files.move(oldPath, newPath.getFilePath());
 512                     return createContext(repo);
 513                 }
 514             } catch (IOException e) {
 515                 log.error("IOException while moving index directory {}", e.getMessage(), e);
 516                 throw new IndexCreationFailedException("Could not recreated the index.", e);
 517             } catch (UnsupportedBaseContextException e) {
 518                 throw new IndexCreationFailedException("The given context, is not a maven context.");
 519             }
 520         } else {
 521             throw new IndexCreationFailedException("Bad context type. This is not a maven context.");
 522         }
 523     }
 524
 525     @Override
 526     public void updateLocalIndexPath(Repository repo) {
 527         if (repo.supportsFeature(IndexCreationFeature.class)) {
 528             IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
 529             try {
 530                 icf.setLocalIndexPath(getIndexPath(repo));
 531                 icf.setLocalPackedIndexPath(getPackedIndexPath(repo));
 532             } catch (IOException e) {
 533                 log.error("Could not set local index path for {}. New URI: {}", repo.getId(), icf.getIndexPath());
 534             }
 535         }
 536     }
 537
 538     @Override
 539     public ArchivaIndexingContext mergeContexts(Repository destinationRepo, List<ArchivaIndexingContext> contexts,
 540                                                 boolean packIndex) throws UnsupportedOperationException,
 541             IndexCreationFailedException, IllegalArgumentException {
 542         if (!destinationRepo.supportsFeature(IndexCreationFeature.class)) {
 543             throw new IllegalArgumentException("The given repository does not support the indexcreation feature");
 544         }
 545         Path mergedIndexDirectory = null;
 546         try {
 547             mergedIndexDirectory = Files.createTempDirectory("archivaMergedIndex");
 548         } catch (IOException e) {
 549             log.error("Could not create temporary directory for merged index: {}", e.getMessage(), e);
 550             throw new IndexCreationFailedException("IO error while creating temporary directory for merged index: "+e.getMessage(), e);
 551         }
 552         IndexCreationFeature indexCreationFeature = destinationRepo.getFeature(IndexCreationFeature.class).get();
 553         if (indexCreationFeature.getLocalIndexPath()== null) {
 554             throw new IllegalArgumentException("The given repository does not have a local index path");
 555         }
 556         StorageAsset destinationPath = indexCreationFeature.getLocalIndexPath();
 557
 558         String tempRepoId = mergedIndexDirectory.getFileName().toString();
 559
 560         try
 561         {
 562             Path indexLocation = destinationPath.getFilePath();
 563
 564             List<IndexingContext> members = contexts.stream( ).filter(ctx -> ctx.supports(IndexingContext.class)).map( ctx ->
 565             {
 566                 try {
 567                     return ctx.getBaseContext(IndexingContext.class);
 568                 } catch (UnsupportedBaseContextException e) {
 569                     // does not happen here
 570                     return null;
 571                 }
 572             }).filter( Objects::nonNull ).collect( Collectors.toList() );
 573             ContextMemberProvider memberProvider = new StaticContextMemberProvider(members);
 574             IndexingContext mergedCtx = indexer.createMergedIndexingContext( tempRepoId, tempRepoId, mergedIndexDirectory.toFile(),
 575                     indexLocation.toFile(), true, memberProvider);
 576             mergedCtx.optimize();
 577
 578             if ( packIndex )
 579             {
 580                 IndexPackingRequest request = new IndexPackingRequest( mergedCtx, //
 581                         mergedCtx.acquireIndexSearcher().getIndexReader(), //
 582                         indexLocation.toFile() );
 583                 indexPacker.packIndex( request );
 584             }
 585
 586             return new MavenIndexContext(destinationRepo, mergedCtx);
 587         }
 588         catch ( IOException e)
 589         {
 590             throw new IndexCreationFailedException( "IO Error during index merge: "+ e.getMessage(), e );
 591         }
 592     }
 593
 594     private StorageAsset getIndexPath(URI indexDirUri, RepositoryStorage repoStorage, String defaultDir) throws IOException
 595     {
 596         StorageAsset rootAsset = repoStorage.getAsset("");
 597         RepositoryStorage storage = rootAsset.getStorage();
 598         Path indexDirectory;
 599         Path repositoryPath = rootAsset.getFilePath().toAbsolutePath();
 600         StorageAsset indexDir;
 601         if ( ! StringUtils.isEmpty(indexDirUri.toString( ) ) )
 602         {
 603
 604             indexDirectory = PathUtil.getPathFromUri( indexDirUri );
 605             // not absolute so create it in repository directory
 606             if ( indexDirectory.isAbsolute( ) && !indexDirectory.startsWith(repositoryPath))
 607             {
 608                 if (storage instanceof FilesystemStorage) {
 609                     FilesystemStorage fsStorage = (FilesystemStorage) storage;
 610                     FilesystemStorage indexStorage = new FilesystemStorage(indexDirectory.getParent(), fsStorage.getFileLockManager());
 611                     indexDir = indexStorage.getAsset(indexDirectory.getFileName().toString());
 612                 } else {
 613                     throw new IOException("The given storage is not file based.");
 614                 }
 615             } else if (indexDirectory.isAbsolute()) {
 616                 indexDir = storage.getAsset(repositoryPath.relativize(indexDirectory).toString());
 617             }
 618             else
 619             {
 620                 indexDir = storage.getAsset(indexDirectory.toString());
 621             }
 622         }
 623         else
 624         {
 625             indexDir = storage.getAsset( defaultDir );
 626         }
 627
 628         if ( !indexDir.exists() )
 629         {
 630             indexDir = storage.addAsset(indexDir.getPath(), true);
 631         }
 632         return indexDir;
 633     }
 634
 635     private StorageAsset getIndexPath( Repository repo) throws IOException {
 636         IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
 637         return getIndexPath( icf.getIndexPath(), repo, DEFAULT_INDEX_PATH);
 638     }
 639
 640     private StorageAsset getPackedIndexPath(Repository repo) throws IOException {
 641         IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
 642         return getIndexPath(icf.getPackedIndexPath(), repo, DEFAULT_PACKED_INDEX_PATH);
 643     }
 644
 645     private IndexingContext createRemoteContext(RemoteRepository remoteRepository ) throws IOException
 646     {
 647         String contextKey = "remote-" + remoteRepository.getId( );
 648
 649
 650         // create remote repository path
 651         Path repoDir = remoteRepository.getAsset( "" ).getFilePath();
 652         if ( !Files.exists( repoDir ) )
 653         {
 654             Files.createDirectories( repoDir );
 655         }
 656
 657         StorageAsset indexDirectory;
 658
 659         // is there configured indexDirectory ?
 660         if ( remoteRepository.supportsFeature( RemoteIndexFeature.class ) )
 661         {
 662             RemoteIndexFeature rif = remoteRepository.getFeature( RemoteIndexFeature.class ).get( );
 663             indexDirectory = getIndexPath(remoteRepository);
 664             String remoteIndexUrl = calculateIndexRemoteUrl( remoteRepository.getLocation( ), rif );
 665             try
 666             {
 667
 668                 return getIndexingContext( remoteRepository, contextKey, repoDir, indexDirectory, remoteIndexUrl );
 669             }
 670             catch ( IndexFormatTooOldException e )
 671             {
 672                 // existing index with an old lucene format so we need to delete it!!!
 673                 // delete it first then recreate it.
 674                 log.warn( "the index of repository {} is too old we have to delete and recreate it", //
 675                     remoteRepository.getId( ) );
 676                 org.apache.archiva.common.utils.FileUtils.deleteDirectory( indexDirectory.getFilePath() );
 677                 return getIndexingContext( remoteRepository, contextKey, repoDir, indexDirectory, remoteIndexUrl );
 678
 679             }
 680         }
 681         else
 682         {
 683             throw new IOException( "No remote index defined" );
 684         }
 685     }
 686
 687     private IndexingContext getIndexingContext( Repository repository, String contextKey, Path repoDir, StorageAsset indexDirectory, String indexUrl ) throws IOException
 688     {
 689         try
 690         {
 691             if (!Files.exists(indexDirectory.getFilePath())) {
 692                 Files.createDirectories(indexDirectory.getFilePath());
 693             }
 694             return indexer.createIndexingContext( contextKey, repository.getId( ), repoDir.toFile( ), indexDirectory.getFilePath( ).toFile( ),
 695                 repository.getLocation( ) == null ? null : repository.getLocation( ).toString( ),
 696                 indexUrl,
 697                 true, false,
 698                 indexCreators );
 699         } catch (Exception e) {
 700             log.error("Could not create index for asset {}", indexDirectory);
 701             throw new IOException(e);
 702         }
 703     }
 704
 705     private IndexingContext createManagedContext( ManagedRepository repository ) throws IOException
 706     {
 707
 708         IndexingContext context;
 709         // take care first about repository location as can be relative
 710         Path repositoryDirectory = repository.getAsset( "" ).getFilePath();
 711
 712         if ( !Files.exists( repositoryDirectory ) )
 713         {
 714             try
 715             {
 716                 Files.createDirectories( repositoryDirectory );
 717             }
 718             catch ( IOException e )
 719             {
 720                 log.error( "Could not create directory {}", repositoryDirectory );
 721             }
 722         }
 723
 724         StorageAsset indexDirectory;
 725
 726         if ( repository.supportsFeature( IndexCreationFeature.class ) )
 727         {
 728             indexDirectory = getIndexPath(repository);
 729             log.debug( "Preparing index at {}", indexDirectory );
 730
 731             String indexUrl = repositoryDirectory.toUri( ).toURL( ).toExternalForm( );
 732             try
 733             {
 734                 context = getIndexingContext( repository, repository.getId( ), repositoryDirectory, indexDirectory, indexUrl );
 735                 context.setSearchable( repository.isScanned( ) );
 736             }
 737             catch ( IndexFormatTooOldException e )
 738             {
 739                 // existing index with an old lucene format so we need to delete it!!!
 740                 // delete it first then recreate it.
 741                 log.warn( "the index of repository {} is too old we have to delete and recreate it", //
 742                     repository.getId( ) );
 743                 org.apache.archiva.common.utils.FileUtils.deleteDirectory( indexDirectory.getFilePath() );
 744                 context = getIndexingContext( repository, repository.getId( ), repositoryDirectory, indexDirectory, indexUrl );
 745                 context.setSearchable( repository.isScanned( ) );
 746             }
 747             return context;
 748         }
 749         else
 750         {
 751             throw new IOException( "No repository index defined" );
 752         }
 753     }
 754
 755     private String calculateIndexRemoteUrl( URI baseUri, RemoteIndexFeature rif )
 756     {
 757         if ( rif.getIndexUri( ) == null )
 758         {
 759             return baseUri.resolve( DEFAULT_INDEX_PATH ).toString( );
 760         }
 761         else
 762         {
 763             return baseUri.resolve( rif.getIndexUri( ) ).toString( );
 764         }
 765     }
 766
 767     private static final class DownloadListener
 768         implements TransferListener
 769     {
 770         private Logger log = LoggerFactory.getLogger( getClass( ) );
 771
 772         private String resourceName;
 773
 774         private long startTime;
 775
 776         private int totalLength = 0;
 777
 778         @Override
 779         public void transferInitiated( TransferEvent transferEvent )
 780         {
 781             startTime = System.currentTimeMillis( );
 782             resourceName = transferEvent.getResource( ).getName( );
 783             log.debug( "initiate transfer of {}", resourceName );
 784         }
 785
 786         @Override
 787         public void transferStarted( TransferEvent transferEvent )
 788         {
 789             this.totalLength = 0;
 790             resourceName = transferEvent.getResource( ).getName( );
 791             log.info( "start transfer of {}", transferEvent.getResource( ).getName( ) );
 792         }
 793
 794         @Override
 795         public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
 796         {
 797             log.debug( "transfer of {} : {}/{}", transferEvent.getResource( ).getName( ), buffer.length, length );
 798             this.totalLength += length;
 799         }
 800
 801         @Override
 802         public void transferCompleted( TransferEvent transferEvent )
 803         {
 804             resourceName = transferEvent.getResource( ).getName( );
 805             long endTime = System.currentTimeMillis( );
 806             log.info( "end of transfer file {} {} kb: {}s", transferEvent.getResource( ).getName( ),
 807                 this.totalLength / 1024, ( endTime - startTime ) / 1000 );
 808         }
 809
 810         @Override
 811         public void transferError( TransferEvent transferEvent )
 812         {
 813             log.info( "error of transfer file {}: {}", transferEvent.getResource( ).getName( ),
 814                 transferEvent.getException( ).getMessage( ), transferEvent.getException( ) );
 815         }
 816
 817         @Override
 818         public void debug( String message )
 819         {
 820             log.debug( "transfer debug {}", message );
 821         }
 822     }
 823
 824     private static class WagonResourceFetcher
 825         implements ResourceFetcher
 826     {
 827
 828         Logger log;
 829
 830         Path tempIndexDirectory;
 831
 832         Wagon wagon;
 833
 834         RemoteRepository remoteRepository;
 835
 836         private WagonResourceFetcher( Logger log, Path tempIndexDirectory, Wagon wagon,
 837                                       RemoteRepository remoteRepository )
 838         {
 839             this.log = log;
 840             this.tempIndexDirectory = tempIndexDirectory;
 841             this.wagon = wagon;
 842             this.remoteRepository = remoteRepository;
 843         }
 844
 845         @Override
 846         public void connect( String id, String url ) {
 847             //no op
 848         }
 849
 850         @Override
 851         public void disconnect( ) {
 852             // no op
 853         }
 854
 855         @Override
 856         public InputStream retrieve( String name )
 857             throws IOException {
 858             try
 859             {
 860                 log.info( "index update retrieve file, name:{}", name );
 861                 Path file = tempIndexDirectory.resolve( name );
 862                 Files.deleteIfExists( file );
 863                 file.toFile( ).deleteOnExit( );
 864                 wagon.get( addParameters( name, remoteRepository ), file.toFile( ) );
 865                 return Files.newInputStream( file );
 866             }
 867             catch ( AuthorizationException | TransferFailedException e )
 868             {
 869                 throw new IOException( e.getMessage( ), e );
 870             }
 871             catch ( ResourceDoesNotExistException e )
 872             {
 873                 FileNotFoundException fnfe = new FileNotFoundException( e.getMessage( ) );
 874                 fnfe.initCause( e );
 875                 throw fnfe;
 876             }
 877         }
 878
 879         // FIXME remove crappy copy/paste
 880         protected String addParameters( String path, RemoteRepository remoteRepository )
 881         {
 882             if ( remoteRepository.getExtraParameters( ).isEmpty( ) )
 883             {
 884                 return path;
 885             }
 886
 887             boolean question = false;
 888
 889             StringBuilder res = new StringBuilder( path == null ? "" : path );
 890
 891             for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters( ).entrySet( ) )
 892             {
 893                 if ( !question )
 894                 {
 895                     res.append( '?' ).append( entry.getKey( ) ).append( '=' ).append( entry.getValue( ) );
 896                 }
 897             }
 898
 899             return res.toString( );
 900         }
 901
 902     }
 903 }