source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.indexer.maven;
   2
   3 /*
   4  * Licensed to the Apache Software Foundation (ASF) under one
   5  * or more contributor license agreements.  See the NOTICE file
   6  * distributed with this work for additional information
   7  * regarding copyright ownership.  The ASF licenses this file
   8  * to you under the Apache License, Version 2.0 (the
   9  * "License"); you may not use this file except in compliance
  10  * with the License.  You may obtain a copy of the License at
  11  *
  12  *  http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing,
  15  * software distributed under the License is distributed on an
  16  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  17  * KIND, either express or implied.  See the License for the
  18  * specific language governing permissions and limitations
  19  * under the License.
  20  */
  21
  22 import org.apache.archiva.common.utils.PathUtil;
  23 import org.apache.archiva.configuration.ArchivaConfiguration;
  24 import org.apache.archiva.indexer.ArchivaIndexManager;
  25 import org.apache.archiva.indexer.ArchivaIndexingContext;
  26 import org.apache.archiva.indexer.IndexCreationFailedException;
  27 import org.apache.archiva.indexer.IndexUpdateFailedException;
  28 import org.apache.archiva.indexer.UnsupportedBaseContextException;
  29 import org.apache.archiva.proxy.ProxyRegistry;
  30 import org.apache.archiva.proxy.maven.WagonFactory;
  31 import org.apache.archiva.proxy.maven.WagonFactoryException;
  32 import org.apache.archiva.proxy.maven.WagonFactoryRequest;
  33 import org.apache.archiva.proxy.model.NetworkProxy;
  34 import org.apache.archiva.repository.EditableRepository;
  35 import org.apache.archiva.repository.ManagedRepository;
  36 import org.apache.archiva.repository.base.PasswordCredentials;
  37 import org.apache.archiva.repository.RemoteRepository;
  38 import org.apache.archiva.repository.Repository;
  39 import org.apache.archiva.repository.RepositoryType;
  40 import org.apache.archiva.repository.UnsupportedRepositoryTypeException;
  41 import org.apache.archiva.repository.storage.fs.FilesystemStorage;
  42 import org.apache.archiva.repository.storage.RepositoryStorage;
  43 import org.apache.archiva.repository.storage.StorageAsset;
  44 import org.apache.archiva.repository.features.IndexCreationFeature;
  45 import org.apache.archiva.repository.features.RemoteIndexFeature;
  46 import org.apache.commons.lang3.StringUtils;
  47 import org.apache.maven.index.ArtifactContext;
  48 import org.apache.maven.index.ArtifactContextProducer;
  49 import org.apache.maven.index.DefaultScannerListener;
  50 import org.apache.maven.index.Indexer;
  51 import org.apache.maven.index.IndexerEngine;
  52 import org.apache.maven.index.Scanner;
  53 import org.apache.maven.index.ScanningRequest;
  54 import org.apache.maven.index.ScanningResult;
  55 import org.apache.maven.index.context.ContextMemberProvider;
  56 import org.apache.maven.index.context.IndexCreator;
  57 import org.apache.maven.index.context.IndexingContext;
  58 import org.apache.maven.index.context.StaticContextMemberProvider;
  59 import org.apache.maven.index.packer.IndexPacker;
  60 import org.apache.maven.index.packer.IndexPackingRequest;
  61 import org.apache.maven.index.updater.IndexUpdateRequest;
  62 import org.apache.maven.index.updater.IndexUpdater;
  63 import org.apache.maven.index.updater.ResourceFetcher;
  64 import org.apache.maven.index_shaded.lucene.index.IndexFormatTooOldException;
  65 import org.apache.maven.wagon.ConnectionException;
  66 import org.apache.maven.wagon.ResourceDoesNotExistException;
  67 import org.apache.maven.wagon.StreamWagon;
  68 import org.apache.maven.wagon.TransferFailedException;
  69 import org.apache.maven.wagon.Wagon;
  70 import org.apache.maven.wagon.authentication.AuthenticationException;
  71 import org.apache.maven.wagon.authentication.AuthenticationInfo;
  72 import org.apache.maven.wagon.authorization.AuthorizationException;
  73 import org.apache.maven.wagon.events.TransferEvent;
  74 import org.apache.maven.wagon.events.TransferListener;
  75 import org.apache.maven.wagon.proxy.ProxyInfo;
  76 import org.apache.maven.wagon.shared.http.AbstractHttpClientWagon;
  77 import org.apache.maven.wagon.shared.http.HttpConfiguration;
  78 import org.apache.maven.wagon.shared.http.HttpMethodConfiguration;
  79 import org.slf4j.Logger;
  80 import org.slf4j.LoggerFactory;
  81 import org.springframework.stereotype.Service;
  82
  83 import javax.inject.Inject;
  84 import java.io.FileNotFoundException;
  85 import java.io.IOException;
  86 import java.io.InputStream;
  87 import java.net.MalformedURLException;
  88 import java.net.URI;
  89 import java.nio.file.Files;
  90 import java.nio.file.Path;
  91 import java.nio.file.Paths;
  92 import java.util.Collection;
  93 import java.util.List;
  94 import java.util.Map;
  95 import java.util.Objects;
  96 import java.util.concurrent.ConcurrentSkipListSet;
  97 import java.util.stream.Collectors;
  98
  99 /**
 100  * Maven implementation of index manager.
 101  * The index manager is a singleton, so we try to make sure, that index operations are not running
 102  * parallel by synchronizing on the index path.
 103  * A update operation waits for parallel running methods to finish before starting, but after a certain
 104  * time of retries a IndexUpdateFailedException is thrown.
 105  */
 106 @Service( "archivaIndexManager#maven" )
 107 public class MavenIndexManager implements ArchivaIndexManager {
 108
 109     private static final Logger log = LoggerFactory.getLogger( MavenIndexManager.class );
 110
 111     @Inject
 112     private Indexer indexer;
 113
 114     @Inject
 115     private IndexerEngine indexerEngine;
 116
 117     @Inject
 118     private List<? extends IndexCreator> indexCreators;
 119
 120     @Inject
 121     private IndexPacker indexPacker;
 122
 123     @Inject
 124     private Scanner scanner;
 125
 126     @Inject
 127     private ArchivaConfiguration archivaConfiguration;
 128
 129     @Inject
 130     private WagonFactory wagonFactory;
 131
 132     @Inject
 133     private IndexUpdater indexUpdater;
 134
 135     @Inject
 136     private ArtifactContextProducer artifactContextProducer;
 137
 138     @Inject
 139     private ProxyRegistry proxyRegistry;
 140
 141
 142     private ConcurrentSkipListSet<StorageAsset> activeContexts = new ConcurrentSkipListSet<>( );
 143
 144     private static final int WAIT_TIME = 100;
 145     private static final int MAX_WAIT = 10;
 146
 147
 148     public static IndexingContext getMvnContext( ArchivaIndexingContext context ) throws UnsupportedBaseContextException
 149     {
 150         if (context!=null)
 151         {
 152             if ( !context.supports( IndexingContext.class ) )
 153             {
 154                 log.error( "The provided archiva index context does not support the maven IndexingContext" );
 155                 throw new UnsupportedBaseContextException( "The context does not support the Maven IndexingContext" );
 156             }
 157             return context.getBaseContext( IndexingContext.class );
 158         } else {
 159             return null;
 160         }
 161     }
 162
 163     private StorageAsset getIndexPath( ArchivaIndexingContext ctx )
 164     {
 165         return ctx.getPath( );
 166     }
 167
 168     @FunctionalInterface
 169     interface IndexUpdateConsumer
 170     {
 171
 172         void accept( IndexingContext indexingContext ) throws IndexUpdateFailedException;
 173     }
 174
 175     /*
 176      * This method is used to do some actions around the update execution code. And to make sure, that no other
 177      * method is running on the same index.
 178      */
 179     private void executeUpdateFunction( ArchivaIndexingContext context, IndexUpdateConsumer function ) throws IndexUpdateFailedException
 180     {
 181         if (context==null) {
 182             throw new IndexUpdateFailedException( "Given context is null" );
 183         }
 184         IndexingContext indexingContext = null;
 185         try
 186         {
 187             indexingContext = getMvnContext( context );
 188         }
 189         catch ( UnsupportedBaseContextException e )
 190         {
 191             throw new IndexUpdateFailedException( "Maven index is not supported by this context", e );
 192         }
 193         final StorageAsset ctxPath = getIndexPath( context );
 194         int loop = MAX_WAIT;
 195         boolean active = false;
 196         while ( loop-- > 0 && !active )
 197         {
 198             active = activeContexts.add( ctxPath );
 199             try
 200             {
 201                 Thread.currentThread( ).sleep( WAIT_TIME );
 202             }
 203             catch ( InterruptedException e )
 204             {
 205                 // Ignore this
 206             }
 207         }
 208         if ( active )
 209         {
 210             try
 211             {
 212                 function.accept( indexingContext );
 213             }
 214             finally
 215             {
 216                 activeContexts.remove( ctxPath );
 217             }
 218         }
 219         else
 220         {
 221             throw new IndexUpdateFailedException( "Timeout while waiting for index release on context " + context.getId( ) );
 222         }
 223     }
 224
 225     @Override
 226     public void pack( final ArchivaIndexingContext context ) throws IndexUpdateFailedException
 227     {
 228         executeUpdateFunction( context, indexingContext -> {
 229                 try
 230                 {
 231                     IndexPackingRequest request = new IndexPackingRequest( indexingContext,
 232                         indexingContext.acquireIndexSearcher( ).getIndexReader( ),
 233                         indexingContext.getIndexDirectoryFile( ) );
 234                     indexPacker.packIndex( request );
 235                     indexingContext.updateTimestamp( true );
 236                 }
 237                 catch ( IOException e )
 238                 {
 239                     log.error( "IOException while packing index of context " + context.getId( ) + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ) );
 240                     throw new IndexUpdateFailedException( "IOException during update of " + context.getId( ), e );
 241                 }
 242             }
 243         );
 244
 245     }
 246
 247     @Override
 248     public void scan(final ArchivaIndexingContext context) throws IndexUpdateFailedException
 249     {
 250         executeUpdateFunction( context, indexingContext -> {
 251             DefaultScannerListener listener = new DefaultScannerListener( indexingContext, indexerEngine, true, null );
 252             ScanningRequest request = new ScanningRequest( indexingContext, listener );
 253             ScanningResult result = scanner.scan( request );
 254             if ( result.hasExceptions( ) )
 255             {
 256                 log.error( "Exceptions occured during index scan of " + context.getId( ) );
 257                 result.getExceptions( ).stream( ).map( e -> e.getMessage( ) ).distinct( ).limit( 5 ).forEach(
 258                     s -> log.error( "Message: " + s )
 259                 );
 260             }
 261
 262         } );
 263     }
 264
 265     @Override
 266     public void update(final ArchivaIndexingContext context, final boolean fullUpdate) throws IndexUpdateFailedException
 267     {
 268         log.info( "start download remote index for remote repository {}", context.getRepository( ).getId( ) );
 269         URI remoteUpdateUri;
 270         if ( !( context.getRepository( ) instanceof RemoteRepository ) || !(context.getRepository().supportsFeature(RemoteIndexFeature.class)) )
 271         {
 272             throw new IndexUpdateFailedException( "The context is not associated to a remote repository with remote index " + context.getId( ) );
 273         } else {
 274             RemoteIndexFeature rif = context.getRepository().getFeature(RemoteIndexFeature.class).get();
 275             remoteUpdateUri = context.getRepository().getLocation().resolve(rif.getIndexUri());
 276         }
 277         final RemoteRepository remoteRepository = (RemoteRepository) context.getRepository( );
 278
 279         executeUpdateFunction( context,
 280             indexingContext -> {
 281                 try
 282                 {
 283                     // create a temp directory to download files
 284                     Path tempIndexDirectory = Paths.get( indexingContext.getIndexDirectoryFile( ).getParent( ), ".tmpIndex" );
 285                     Path indexCacheDirectory = Paths.get( indexingContext.getIndexDirectoryFile( ).getParent( ), ".indexCache" );
 286                     Files.createDirectories( indexCacheDirectory );
 287                     if ( Files.exists( tempIndexDirectory ) )
 288                     {
 289                         org.apache.archiva.common.utils.FileUtils.deleteDirectory( tempIndexDirectory );
 290                     }
 291                     Files.createDirectories( tempIndexDirectory );
 292                     tempIndexDirectory.toFile( ).deleteOnExit( );
 293                     String baseIndexUrl = indexingContext.getIndexUpdateUrl( );
 294
 295                     String wagonProtocol = remoteUpdateUri.toURL( ).getProtocol( );
 296
 297                     NetworkProxy networkProxy = null;
 298                     if ( remoteRepository.supportsFeature( RemoteIndexFeature.class ) )
 299                     {
 300                         RemoteIndexFeature rif = remoteRepository.getFeature( RemoteIndexFeature.class ).get( );
 301                         if ( StringUtils.isNotBlank( rif.getProxyId( ) ) )
 302                         {
 303                             networkProxy = proxyRegistry.getNetworkProxy( rif.getProxyId( ) );
 304                             if ( networkProxy == null )
 305                             {
 306                                 log.warn(
 307                                     "your remote repository is configured to download remote index trought a proxy we cannot find id:{}",
 308                                     rif.getProxyId( ) );
 309                             }
 310                         }
 311
 312                         final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon(
 313                             new WagonFactoryRequest( wagonProtocol, remoteRepository.getExtraHeaders( ) ).networkProxy(
 314                                 networkProxy )
 315                         );
 316                         int readTimeout = (int) rif.getDownloadTimeout( ).toMillis( ) * 1000;
 317                         wagon.setReadTimeout( readTimeout );
 318                         wagon.setTimeout( (int) remoteRepository.getTimeout( ).toMillis( ) * 1000 );
 319
 320                         if ( wagon instanceof AbstractHttpClientWagon )
 321                         {
 322                             HttpConfiguration httpConfiguration = new HttpConfiguration( );
 323                             HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration( );
 324                             httpMethodConfiguration.setUsePreemptive( true );
 325                             httpMethodConfiguration.setReadTimeout( readTimeout );
 326                             httpConfiguration.setGet( httpMethodConfiguration );
 327                             AbstractHttpClientWagon.class.cast( wagon ).setHttpConfiguration( httpConfiguration );
 328                         }
 329
 330                         wagon.addTransferListener( new DownloadListener( ) );
 331                         ProxyInfo proxyInfo = null;
 332                         if ( networkProxy != null )
 333                         {
 334                             proxyInfo = new ProxyInfo( );
 335                             proxyInfo.setType( networkProxy.getProtocol( ) );
 336                             proxyInfo.setHost( networkProxy.getHost( ) );
 337                             proxyInfo.setPort( networkProxy.getPort( ) );
 338                             proxyInfo.setUserName( networkProxy.getUsername( ) );
 339                             proxyInfo.setPassword( new String(networkProxy.getPassword( )) );
 340                         }
 341                         AuthenticationInfo authenticationInfo = null;
 342                         if ( remoteRepository.getLoginCredentials( ) != null && ( remoteRepository.getLoginCredentials( ) instanceof PasswordCredentials ) )
 343                         {
 344                             PasswordCredentials creds = (PasswordCredentials) remoteRepository.getLoginCredentials( );
 345                             authenticationInfo = new AuthenticationInfo( );
 346                             authenticationInfo.setUserName( creds.getUsername( ) );
 347                             authenticationInfo.setPassword( new String( creds.getPassword( ) ) );
 348                         }
 349                         wagon.connect( new org.apache.maven.wagon.repository.Repository( remoteRepository.getId( ), baseIndexUrl ), authenticationInfo,
 350                             proxyInfo );
 351
 352                         Path indexDirectory = indexingContext.getIndexDirectoryFile( ).toPath( );
 353                         if ( !Files.exists( indexDirectory ) )
 354                         {
 355                             Files.createDirectories( indexDirectory );
 356                         }
 357
 358                         ResourceFetcher resourceFetcher =
 359                             new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository );
 360                         IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
 361                         request.setForceFullUpdate( fullUpdate );
 362                         request.setLocalIndexCacheDir( indexCacheDirectory.toFile( ) );
 363
 364                         indexUpdater.fetchAndUpdateIndex( request );
 365
 366                         indexingContext.updateTimestamp( true );
 367                     }
 368
 369                 }
 370                 catch ( AuthenticationException e )
 371                 {
 372                     log.error( "Could not login to the remote proxy for updating index of {}", remoteRepository.getId( ), e );
 373                     throw new IndexUpdateFailedException( "Login in to proxy failed while updating remote repository " + remoteRepository.getId( ), e );
 374                 }
 375                 catch ( ConnectionException e )
 376                 {
 377                     log.error( "Connection error during index update for remote repository {}", remoteRepository.getId( ), e );
 378                     throw new IndexUpdateFailedException( "Connection error during index update for remote repository " + remoteRepository.getId( ), e );
 379                 }
 380                 catch ( MalformedURLException e )
 381                 {
 382                     log.error( "URL for remote index update of remote repository {} is not correct {}", remoteRepository.getId( ), remoteUpdateUri, e );
 383                     throw new IndexUpdateFailedException( "URL for remote index update of repository is not correct " + remoteUpdateUri, e );
 384                 }
 385                 catch ( IOException e )
 386                 {
 387                     log.error( "IOException during index update of remote repository {}: {}", remoteRepository.getId( ), e.getMessage( ), e );
 388                     throw new IndexUpdateFailedException( "IOException during index update of remote repository " + remoteRepository.getId( )
 389                         + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ), e );
 390                 }
 391                 catch ( WagonFactoryException e )
 392                 {
 393                     log.error( "Wagon for remote index download of {} could not be created: {}", remoteRepository.getId( ), e.getMessage( ), e );
 394                     throw new IndexUpdateFailedException( "Error while updating the remote index of " + remoteRepository.getId( ), e );
 395                 }
 396             } );
 397
 398     }
 399
 400     @Override
 401     public void addArtifactsToIndex( final ArchivaIndexingContext context, final Collection<URI> artifactReference ) throws IndexUpdateFailedException
 402     {
 403         final StorageAsset ctxUri = context.getPath();
 404         executeUpdateFunction(context, indexingContext -> {
 405             Collection<ArtifactContext> artifacts = artifactReference.stream().map(r -> artifactContextProducer.getArtifactContext(indexingContext, Paths.get(ctxUri.getFilePath().toUri().resolve(r)).toFile())).collect(Collectors.toList());
 406             try {
 407                 indexer.addArtifactsToIndex(artifacts, indexingContext);
 408             } catch (IOException e) {
 409                 log.error("IOException while adding artifact {}", e.getMessage(), e);
 410                 throw new IndexUpdateFailedException("Error occured while adding artifact to index of "+context.getId()
 411                 + (StringUtils.isNotEmpty(e.getMessage()) ? ": "+e.getMessage() : ""));
 412             }
 413         });
 414     }
 415
 416     @Override
 417     public void removeArtifactsFromIndex( ArchivaIndexingContext context, Collection<URI> artifactReference ) throws IndexUpdateFailedException
 418     {
 419         final StorageAsset ctxUri = context.getPath();
 420         executeUpdateFunction(context, indexingContext -> {
 421             Collection<ArtifactContext> artifacts = artifactReference.stream().map(r -> artifactContextProducer.getArtifactContext(indexingContext, Paths.get(ctxUri.getFilePath().toUri().resolve(r)).toFile())).collect(Collectors.toList());
 422             try {
 423                 indexer.deleteArtifactsFromIndex(artifacts, indexingContext);
 424             } catch (IOException e) {
 425                 log.error("IOException while removing artifact {}", e.getMessage(), e);
 426                 throw new IndexUpdateFailedException("Error occured while removing artifact from index of "+context.getId()
 427                         + (StringUtils.isNotEmpty(e.getMessage()) ? ": "+e.getMessage() : ""));
 428             }
 429         });
 430
 431     }
 432
 433     @Override
 434     public boolean supportsRepository( RepositoryType type )
 435     {
 436         return type == RepositoryType.MAVEN;
 437     }
 438
 439     @Override
 440     public ArchivaIndexingContext createContext( Repository repository ) throws IndexCreationFailedException
 441     {
 442         log.debug("Creating context for repo {}, type: {}", repository.getId(), repository.getType());
 443         if ( repository.getType( ) != RepositoryType.MAVEN )
 444         {
 445             throw new UnsupportedRepositoryTypeException( repository.getType( ) );
 446         }
 447         IndexingContext mvnCtx = null;
 448         try
 449         {
 450             if ( repository instanceof RemoteRepository )
 451             {
 452                 mvnCtx = createRemoteContext( (RemoteRepository) repository );
 453             }
 454             else if ( repository instanceof ManagedRepository )
 455             {
 456                 mvnCtx = createManagedContext( (ManagedRepository) repository );
 457             }
 458         }
 459         catch ( IOException e )
 460         {
 461             log.error( "IOException during context creation " + e.getMessage( ), e );
 462             throw new IndexCreationFailedException( "Could not create index context for repository " + repository.getId( )
 463                 + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ), e );
 464         }
 465
 466         return new MavenIndexContext( repository, mvnCtx );
 467     }
 468
 469     @Override
 470     public ArchivaIndexingContext reset(ArchivaIndexingContext context) throws IndexUpdateFailedException {
 471         ArchivaIndexingContext ctx;
 472         executeUpdateFunction(context, indexingContext -> {
 473             try {
 474                 indexingContext.close(true);
 475             } catch (IOException e) {
 476                 log.warn("Index close failed");
 477             }
 478             org.apache.archiva.repository.storage.util.StorageUtil.deleteRecursively(context.getPath());
 479         });
 480         try {
 481             Repository repo = context.getRepository();
 482             ctx = createContext(context.getRepository());
 483             if (repo instanceof EditableRepository) {
 484                 ((EditableRepository)repo).setIndexingContext(ctx);
 485             }
 486         } catch (IndexCreationFailedException e) {
 487             throw new IndexUpdateFailedException("Could not create index");
 488         }
 489         return ctx;
 490     }
 491
 492     @Override
 493     public ArchivaIndexingContext move(ArchivaIndexingContext context, Repository repo) throws IndexCreationFailedException {
 494         if (context==null) {
 495             return null;
 496         }
 497         if (context.supports(IndexingContext.class)) {
 498             try {
 499                 StorageAsset newPath = getIndexPath(repo);
 500                 IndexingContext ctx = context.getBaseContext(IndexingContext.class);
 501                 Path oldPath = ctx.getIndexDirectoryFile().toPath();
 502                 if (oldPath.equals(newPath)) {
 503                     // Nothing to do, if path does not change
 504                     return context;
 505                 }
 506                 if (!Files.exists(oldPath)) {
 507                     return createContext(repo);
 508                 } else if (context.isEmpty()) {
 509                     context.close();
 510                     return createContext(repo);
 511                 } else {
 512                     context.close(false);
 513                     Files.move(oldPath, newPath.getFilePath());
 514                     return createContext(repo);
 515                 }
 516             } catch (IOException e) {
 517                 log.error("IOException while moving index directory {}", e.getMessage(), e);
 518                 throw new IndexCreationFailedException("Could not recreated the index.", e);
 519             } catch (UnsupportedBaseContextException e) {
 520                 throw new IndexCreationFailedException("The given context, is not a maven context.");
 521             }
 522         } else {
 523             throw new IndexCreationFailedException("Bad context type. This is not a maven context.");
 524         }
 525     }
 526
 527     @Override
 528     public void updateLocalIndexPath(Repository repo) {
 529         if (repo.supportsFeature(IndexCreationFeature.class)) {
 530             IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
 531             try {
 532                 icf.setLocalIndexPath(getIndexPath(repo));
 533                 icf.setLocalPackedIndexPath(getPackedIndexPath(repo));
 534             } catch (IOException e) {
 535                 log.error("Could not set local index path for {}. New URI: {}", repo.getId(), icf.getIndexPath());
 536             }
 537         }
 538     }
 539
 540     @Override
 541     public ArchivaIndexingContext mergeContexts(Repository destinationRepo, List<ArchivaIndexingContext> contexts,
 542                                                 boolean packIndex) throws UnsupportedOperationException,
 543             IndexCreationFailedException, IllegalArgumentException {
 544         if (!destinationRepo.supportsFeature(IndexCreationFeature.class)) {
 545             throw new IllegalArgumentException("The given repository does not support the indexcreation feature");
 546         }
 547         Path mergedIndexDirectory = null;
 548         try {
 549             mergedIndexDirectory = Files.createTempDirectory("archivaMergedIndex");
 550         } catch (IOException e) {
 551             log.error("Could not create temporary directory for merged index: {}", e.getMessage(), e);
 552             throw new IndexCreationFailedException("IO error while creating temporary directory for merged index: "+e.getMessage(), e);
 553         }
 554         IndexCreationFeature indexCreationFeature = destinationRepo.getFeature(IndexCreationFeature.class).get();
 555         if (indexCreationFeature.getLocalIndexPath()== null) {
 556             throw new IllegalArgumentException("The given repository does not have a local index path");
 557         }
 558         StorageAsset destinationPath = indexCreationFeature.getLocalIndexPath();
 559
 560         String tempRepoId = mergedIndexDirectory.getFileName().toString();
 561
 562         try
 563         {
 564             Path indexLocation = destinationPath.getFilePath();
 565
 566             List<IndexingContext> members = contexts.stream( ).filter(ctx -> ctx.supports(IndexingContext.class)).map( ctx ->
 567             {
 568                 try {
 569                     return ctx.getBaseContext(IndexingContext.class);
 570                 } catch (UnsupportedBaseContextException e) {
 571                     // does not happen here
 572                     return null;
 573                 }
 574             }).filter( Objects::nonNull ).collect( Collectors.toList() );
 575             ContextMemberProvider memberProvider = new StaticContextMemberProvider(members);
 576             IndexingContext mergedCtx = indexer.createMergedIndexingContext( tempRepoId, tempRepoId, mergedIndexDirectory.toFile(),
 577                     indexLocation.toFile(), true, memberProvider);
 578             mergedCtx.optimize();
 579
 580             if ( packIndex )
 581             {
 582                 IndexPackingRequest request = new IndexPackingRequest( mergedCtx, //
 583                         mergedCtx.acquireIndexSearcher().getIndexReader(), //
 584                         indexLocation.toFile() );
 585                 indexPacker.packIndex( request );
 586             }
 587
 588             return new MavenIndexContext(destinationRepo, mergedCtx);
 589         }
 590         catch ( IOException e)
 591         {
 592             throw new IndexCreationFailedException( "IO Error during index merge: "+ e.getMessage(), e );
 593         }
 594     }
 595
 596     private StorageAsset getIndexPath(URI indexDirUri, RepositoryStorage repoStorage, String defaultDir) throws IOException
 597     {
 598         StorageAsset rootAsset = repoStorage.getAsset("");
 599         RepositoryStorage storage = rootAsset.getStorage();
 600         Path indexDirectory;
 601         Path repositoryPath = rootAsset.getFilePath().toAbsolutePath();
 602         StorageAsset indexDir;
 603         if ( ! StringUtils.isEmpty(indexDirUri.toString( ) ) )
 604         {
 605
 606             indexDirectory = PathUtil.getPathFromUri( indexDirUri );
 607             // not absolute so create it in repository directory
 608             if ( indexDirectory.isAbsolute( ) && !indexDirectory.startsWith(repositoryPath))
 609             {
 610                 if (storage instanceof FilesystemStorage) {
 611                     FilesystemStorage fsStorage = (FilesystemStorage) storage;
 612                     FilesystemStorage indexStorage = new FilesystemStorage(indexDirectory.getParent(), fsStorage.getFileLockManager());
 613                     indexDir = indexStorage.getAsset(indexDirectory.getFileName().toString());
 614                 } else {
 615                     throw new IOException("The given storage is not file based.");
 616                 }
 617             } else if (indexDirectory.isAbsolute()) {
 618                 indexDir = storage.getAsset(repositoryPath.relativize(indexDirectory).toString());
 619             }
 620             else
 621             {
 622                 indexDir = storage.getAsset(indexDirectory.toString());
 623             }
 624         }
 625         else
 626         {
 627             indexDir = storage.getAsset( defaultDir );
 628         }
 629
 630         if ( !indexDir.exists() )
 631         {
 632             indexDir = storage.addAsset(indexDir.getPath(), true);
 633         }
 634         return indexDir;
 635     }
 636
 637     private StorageAsset getIndexPath( Repository repo) throws IOException {
 638         IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
 639         return getIndexPath( icf.getIndexPath(), repo, DEFAULT_INDEX_PATH);
 640     }
 641
 642     private StorageAsset getPackedIndexPath(Repository repo) throws IOException {
 643         IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
 644         return getIndexPath(icf.getPackedIndexPath(), repo, DEFAULT_PACKED_INDEX_PATH);
 645     }
 646
 647     private IndexingContext createRemoteContext(RemoteRepository remoteRepository ) throws IOException
 648     {
 649         String contextKey = "remote-" + remoteRepository.getId( );
 650
 651
 652         // create remote repository path
 653         Path repoDir = remoteRepository.getAsset( "" ).getFilePath();
 654         if ( !Files.exists( repoDir ) )
 655         {
 656             Files.createDirectories( repoDir );
 657         }
 658
 659         StorageAsset indexDirectory;
 660
 661         // is there configured indexDirectory ?
 662         if ( remoteRepository.supportsFeature( RemoteIndexFeature.class ) )
 663         {
 664             RemoteIndexFeature rif = remoteRepository.getFeature( RemoteIndexFeature.class ).get( );
 665             indexDirectory = getIndexPath(remoteRepository);
 666             String remoteIndexUrl = calculateIndexRemoteUrl( remoteRepository.getLocation( ), rif );
 667             try
 668             {
 669
 670                 return getIndexingContext( remoteRepository, contextKey, repoDir, indexDirectory, remoteIndexUrl );
 671             }
 672             catch ( IndexFormatTooOldException e )
 673             {
 674                 // existing index with an old lucene format so we need to delete it!!!
 675                 // delete it first then recreate it.
 676                 log.warn( "the index of repository {} is too old we have to delete and recreate it", //
 677                     remoteRepository.getId( ) );
 678                 org.apache.archiva.common.utils.FileUtils.deleteDirectory( indexDirectory.getFilePath() );
 679                 return getIndexingContext( remoteRepository, contextKey, repoDir, indexDirectory, remoteIndexUrl );
 680
 681             }
 682         }
 683         else
 684         {
 685             throw new IOException( "No remote index defined" );
 686         }
 687     }
 688
 689     private IndexingContext getIndexingContext( Repository repository, String contextKey, Path repoDir, StorageAsset indexDirectory, String indexUrl ) throws IOException
 690     {
 691         try
 692         {
 693             if (!Files.exists(indexDirectory.getFilePath())) {
 694                 Files.createDirectories(indexDirectory.getFilePath());
 695             }
 696             return indexer.createIndexingContext( contextKey, repository.getId( ), repoDir.toFile( ), indexDirectory.getFilePath( ).toFile( ),
 697                 repository.getLocation( ) == null ? null : repository.getLocation( ).toString( ),
 698                 indexUrl,
 699                 true, false,
 700                 indexCreators );
 701         } catch (Exception e) {
 702             log.error("Could not create index for asset {}", indexDirectory);
 703             throw new IOException(e);
 704         }
 705     }
 706
 707     private IndexingContext createManagedContext( ManagedRepository repository ) throws IOException
 708     {
 709
 710         IndexingContext context;
 711         // take care first about repository location as can be relative
 712         Path repositoryDirectory = repository.getAsset( "" ).getFilePath();
 713
 714         if ( !Files.exists( repositoryDirectory ) )
 715         {
 716             try
 717             {
 718                 Files.createDirectories( repositoryDirectory );
 719             }
 720             catch ( IOException e )
 721             {
 722                 log.error( "Could not create directory {}", repositoryDirectory );
 723             }
 724         }
 725
 726         StorageAsset indexDirectory;
 727
 728         if ( repository.supportsFeature( IndexCreationFeature.class ) )
 729         {
 730             indexDirectory = getIndexPath(repository);
 731             log.debug( "Preparing index at {}", indexDirectory );
 732
 733             String indexUrl = repositoryDirectory.toUri( ).toURL( ).toExternalForm( );
 734             try
 735             {
 736                 context = getIndexingContext( repository, repository.getId( ), repositoryDirectory, indexDirectory, indexUrl );
 737                 context.setSearchable( repository.isScanned( ) );
 738             }
 739             catch ( IndexFormatTooOldException e )
 740             {
 741                 // existing index with an old lucene format so we need to delete it!!!
 742                 // delete it first then recreate it.
 743                 log.warn( "the index of repository {} is too old we have to delete and recreate it", //
 744                     repository.getId( ) );
 745                 org.apache.archiva.common.utils.FileUtils.deleteDirectory( indexDirectory.getFilePath() );
 746                 context = getIndexingContext( repository, repository.getId( ), repositoryDirectory, indexDirectory, indexUrl );
 747                 context.setSearchable( repository.isScanned( ) );
 748             }
 749             return context;
 750         }
 751         else
 752         {
 753             throw new IOException( "No repository index defined" );
 754         }
 755     }
 756
 757     private String calculateIndexRemoteUrl( URI baseUri, RemoteIndexFeature rif )
 758     {
 759         if ( rif.getIndexUri( ) == null )
 760         {
 761             return baseUri.resolve( DEFAULT_INDEX_PATH ).toString( );
 762         }
 763         else
 764         {
 765             return baseUri.resolve( rif.getIndexUri( ) ).toString( );
 766         }
 767     }
 768
 769     private static final class DownloadListener
 770         implements TransferListener
 771     {
 772         private Logger log = LoggerFactory.getLogger( getClass( ) );
 773
 774         private String resourceName;
 775
 776         private long startTime;
 777
 778         private int totalLength = 0;
 779
 780         @Override
 781         public void transferInitiated( TransferEvent transferEvent )
 782         {
 783             startTime = System.currentTimeMillis( );
 784             resourceName = transferEvent.getResource( ).getName( );
 785             log.debug( "initiate transfer of {}", resourceName );
 786         }
 787
 788         @Override
 789         public void transferStarted( TransferEvent transferEvent )
 790         {
 791             this.totalLength = 0;
 792             resourceName = transferEvent.getResource( ).getName( );
 793             log.info( "start transfer of {}", transferEvent.getResource( ).getName( ) );
 794         }
 795
 796         @Override
 797         public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
 798         {
 799             log.debug( "transfer of {} : {}/{}", transferEvent.getResource( ).getName( ), buffer.length, length );
 800             this.totalLength += length;
 801         }
 802
 803         @Override
 804         public void transferCompleted( TransferEvent transferEvent )
 805         {
 806             resourceName = transferEvent.getResource( ).getName( );
 807             long endTime = System.currentTimeMillis( );
 808             log.info( "end of transfer file {} {} kb: {}s", transferEvent.getResource( ).getName( ),
 809                 this.totalLength / 1024, ( endTime - startTime ) / 1000 );
 810         }
 811
 812         @Override
 813         public void transferError( TransferEvent transferEvent )
 814         {
 815             log.info( "error of transfer file {}: {}", transferEvent.getResource( ).getName( ),
 816                 transferEvent.getException( ).getMessage( ), transferEvent.getException( ) );
 817         }
 818
 819         @Override
 820         public void debug( String message )
 821         {
 822             log.debug( "transfer debug {}", message );
 823         }
 824     }
 825
 826     private static class WagonResourceFetcher
 827         implements ResourceFetcher
 828     {
 829
 830         Logger log;
 831
 832         Path tempIndexDirectory;
 833
 834         Wagon wagon;
 835
 836         RemoteRepository remoteRepository;
 837
 838         private WagonResourceFetcher( Logger log, Path tempIndexDirectory, Wagon wagon,
 839                                       RemoteRepository remoteRepository )
 840         {
 841             this.log = log;
 842             this.tempIndexDirectory = tempIndexDirectory;
 843             this.wagon = wagon;
 844             this.remoteRepository = remoteRepository;
 845         }
 846
 847         @Override
 848         public void connect( String id, String url ) {
 849             //no op
 850         }
 851
 852         @Override
 853         public void disconnect( ) {
 854             // no op
 855         }
 856
 857         @Override
 858         public InputStream retrieve( String name )
 859             throws IOException {
 860             try
 861             {
 862                 log.info( "index update retrieve file, name:{}", name );
 863                 Path file = tempIndexDirectory.resolve( name );
 864                 Files.deleteIfExists( file );
 865                 file.toFile( ).deleteOnExit( );
 866                 wagon.get( addParameters( name, remoteRepository ), file.toFile( ) );
 867                 return Files.newInputStream( file );
 868             }
 869             catch ( AuthorizationException | TransferFailedException e )
 870             {
 871                 throw new IOException( e.getMessage( ), e );
 872             }
 873             catch ( ResourceDoesNotExistException e )
 874             {
 875                 FileNotFoundException fnfe = new FileNotFoundException( e.getMessage( ) );
 876                 fnfe.initCause( e );
 877                 throw fnfe;
 878             }
 879         }
 880
 881         // FIXME remove crappy copy/paste
 882         protected String addParameters( String path, RemoteRepository remoteRepository )
 883         {
 884             if ( remoteRepository.getExtraParameters( ).isEmpty( ) )
 885             {
 886                 return path;
 887             }
 888
 889             boolean question = false;
 890
 891             StringBuilder res = new StringBuilder( path == null ? "" : path );
 892
 893             for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters( ).entrySet( ) )
 894             {
 895                 if ( !question )
 896                 {
 897                     res.append( '?' ).append( entry.getKey( ) ).append( '=' ).append( entry.getValue( ) );
 898                 }
 899             }
 900
 901             return res.toString( );
 902         }
 903
 904     }
 905 }