source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.maven.indexer;
   2
   3 /*
   4  * Licensed to the Apache Software Foundation (ASF) under one
   5  * or more contributor license agreements.  See the NOTICE file
   6  * distributed with this work for additional information
   7  * regarding copyright ownership.  The ASF licenses this file
   8  * to you under the Apache License, Version 2.0 (the
   9  * "License"); you may not use this file except in compliance
  10  * with the License.  You may obtain a copy of the License at
  11  *
  12  * http://www.apache.org/licenses/LICENSE-2.0
  13  * Unless required by applicable law or agreed to in writing,
  14  * software distributed under the License is distributed on an
  15  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  16  * KIND, either express or implied.  See the License for the
  17  * specific language governing permissions and limitations
  18  * under the License.
  19  */
  20
  21 import org.apache.archiva.common.utils.FileUtils;
  22 import org.apache.archiva.common.utils.PathUtil;
  23 import org.apache.archiva.configuration.ArchivaConfiguration;
  24 import org.apache.archiva.indexer.ArchivaIndexManager;
  25 import org.apache.archiva.indexer.ArchivaIndexingContext;
  26 import org.apache.archiva.indexer.IndexCreationFailedException;
  27 import org.apache.archiva.indexer.IndexUpdateFailedException;
  28 import org.apache.archiva.indexer.UnsupportedBaseContextException;
  29 import org.apache.archiva.proxy.ProxyRegistry;
  30 import org.apache.archiva.maven.common.proxy.WagonFactory;
  31 import org.apache.archiva.maven.common.proxy.WagonFactoryException;
  32 import org.apache.archiva.maven.common.proxy.WagonFactoryRequest;
  33 import org.apache.archiva.proxy.model.NetworkProxy;
  34 import org.apache.archiva.repository.EditableRepository;
  35 import org.apache.archiva.repository.ManagedRepository;
  36 import org.apache.archiva.repository.base.PasswordCredentials;
  37 import org.apache.archiva.repository.RemoteRepository;
  38 import org.apache.archiva.repository.Repository;
  39 import org.apache.archiva.repository.RepositoryType;
  40 import org.apache.archiva.repository.UnsupportedRepositoryTypeException;
  41 import org.apache.archiva.repository.storage.AssetType;
  42 import org.apache.archiva.repository.storage.fs.FilesystemStorage;
  43 import org.apache.archiva.repository.storage.RepositoryStorage;
  44 import org.apache.archiva.repository.storage.StorageAsset;
  45 import org.apache.archiva.repository.features.IndexCreationFeature;
  46 import org.apache.archiva.repository.features.RemoteIndexFeature;
  47 import org.apache.commons.lang3.StringUtils;
  48 import org.apache.maven.index.ArtifactContext;
  49 import org.apache.maven.index.ArtifactContextProducer;
  50 import org.apache.maven.index.DefaultScannerListener;
  51 import org.apache.maven.index.Indexer;
  52 import org.apache.maven.index.IndexerEngine;
  53 import org.apache.maven.index.Scanner;
  54 import org.apache.maven.index.ScanningRequest;
  55 import org.apache.maven.index.ScanningResult;
  56 import org.apache.maven.index.context.ContextMemberProvider;
  57 import org.apache.maven.index.context.IndexCreator;
  58 import org.apache.maven.index.context.IndexingContext;
  59 import org.apache.maven.index.context.StaticContextMemberProvider;
  60 import org.apache.maven.index.packer.IndexPacker;
  61 import org.apache.maven.index.packer.IndexPackingRequest;
  62 import org.apache.maven.index.updater.IndexUpdateRequest;
  63 import org.apache.maven.index.updater.IndexUpdater;
  64 import org.apache.maven.index.updater.ResourceFetcher;
  65 import org.apache.maven.index_shaded.lucene.index.IndexFormatTooOldException;
  66 import org.apache.maven.wagon.ConnectionException;
  67 import org.apache.maven.wagon.ResourceDoesNotExistException;
  68 import org.apache.maven.wagon.StreamWagon;
  69 import org.apache.maven.wagon.TransferFailedException;
  70 import org.apache.maven.wagon.Wagon;
  71 import org.apache.maven.wagon.authentication.AuthenticationException;
  72 import org.apache.maven.wagon.authentication.AuthenticationInfo;
  73 import org.apache.maven.wagon.authorization.AuthorizationException;
  74 import org.apache.maven.wagon.events.TransferEvent;
  75 import org.apache.maven.wagon.events.TransferListener;
  76 import org.apache.maven.wagon.proxy.ProxyInfo;
  77 import org.apache.maven.wagon.shared.http.AbstractHttpClientWagon;
  78 import org.apache.maven.wagon.shared.http.HttpConfiguration;
  79 import org.apache.maven.wagon.shared.http.HttpMethodConfiguration;
  80 import org.slf4j.Logger;
  81 import org.slf4j.LoggerFactory;
  82 import org.springframework.stereotype.Service;
  83
  84 import javax.inject.Inject;
  85 import java.io.FileNotFoundException;
  86 import java.io.IOException;
  87 import java.io.InputStream;
  88 import java.net.MalformedURLException;
  89 import java.net.URI;
  90 import java.nio.file.Files;
  91 import java.nio.file.Path;
  92 import java.nio.file.Paths;
  93 import java.util.Collection;
  94 import java.util.List;
  95 import java.util.Map;
  96 import java.util.Objects;
  97 import java.util.concurrent.ConcurrentSkipListSet;
  98 import java.util.stream.Collectors;
  99
 100 /**
 101  * Maven implementation of index manager.
 102  * The index manager is a singleton, so we try to make sure, that index operations are not running
 103  * parallel by synchronizing on the index path.
 104  * A update operation waits for parallel running methods to finish before starting, but after a certain
 105  * time of retries a IndexUpdateFailedException is thrown.
 106  */
 107 @Service( "archivaIndexManager#maven" )
 108 public class MavenIndexManager implements ArchivaIndexManager {
 109
 110     private static final Logger log = LoggerFactory.getLogger( MavenIndexManager.class );
 111
 112     @Inject
 113     private Indexer indexer;
 114
 115     @Inject
 116     private IndexerEngine indexerEngine;
 117
 118     @Inject
 119     private List<? extends IndexCreator> indexCreators;
 120
 121     @Inject
 122     private IndexPacker indexPacker;
 123
 124     @Inject
 125     private Scanner scanner;
 126
 127     @Inject
 128     private ArchivaConfiguration archivaConfiguration;
 129
 130     @Inject
 131     private WagonFactory wagonFactory;
 132
 133     @Inject
 134     private IndexUpdater indexUpdater;
 135
 136     @Inject
 137     private ArtifactContextProducer artifactContextProducer;
 138
 139     @Inject
 140     private ProxyRegistry proxyRegistry;
 141
 142
 143     private ConcurrentSkipListSet<StorageAsset> activeContexts = new ConcurrentSkipListSet<>( );
 144
 145     private static final int WAIT_TIME = 100;
 146     private static final int MAX_WAIT = 10;
 147
 148
 149     public static IndexingContext getMvnContext( ArchivaIndexingContext context ) throws UnsupportedBaseContextException
 150     {
 151         if (context!=null)
 152         {
 153             if ( !context.supports( IndexingContext.class ) )
 154             {
 155                 log.error( "The provided archiva index context does not support the maven IndexingContext" );
 156                 throw new UnsupportedBaseContextException( "The context does not support the Maven IndexingContext" );
 157             }
 158             return context.getBaseContext( IndexingContext.class );
 159         } else {
 160             return null;
 161         }
 162     }
 163
 164     private StorageAsset getIndexPath( ArchivaIndexingContext ctx )
 165     {
 166         return ctx.getPath( );
 167     }
 168
 169     @FunctionalInterface
 170     interface IndexUpdateConsumer
 171     {
 172
 173         void accept( IndexingContext indexingContext ) throws IndexUpdateFailedException;
 174     }
 175
 176     /*
 177      * This method is used to do some actions around the update execution code. And to make sure, that no other
 178      * method is running on the same index.
 179      */
 180     private void executeUpdateFunction( ArchivaIndexingContext context, IndexUpdateConsumer function ) throws IndexUpdateFailedException
 181     {
 182         if (context==null) {
 183             throw new IndexUpdateFailedException( "Given context is null" );
 184         }
 185         IndexingContext indexingContext = null;
 186         try
 187         {
 188             indexingContext = getMvnContext( context );
 189         }
 190         catch ( UnsupportedBaseContextException e )
 191         {
 192             throw new IndexUpdateFailedException( "Maven index is not supported by this context", e );
 193         }
 194         final StorageAsset ctxPath = getIndexPath( context );
 195         int loop = MAX_WAIT;
 196         boolean active = false;
 197         while ( loop-- > 0 && !active )
 198         {
 199             active = activeContexts.add( ctxPath );
 200             try
 201             {
 202                 Thread.currentThread( ).sleep( WAIT_TIME );
 203             }
 204             catch ( InterruptedException e )
 205             {
 206                 // Ignore this
 207             }
 208         }
 209         if ( active )
 210         {
 211             try
 212             {
 213                 function.accept( indexingContext );
 214             }
 215             finally
 216             {
 217                 activeContexts.remove( ctxPath );
 218             }
 219         }
 220         else
 221         {
 222             throw new IndexUpdateFailedException( "Timeout while waiting for index release on context " + context.getId( ) );
 223         }
 224     }
 225
 226     @Override
 227     public void pack( final ArchivaIndexingContext context ) throws IndexUpdateFailedException
 228     {
 229         executeUpdateFunction( context, indexingContext -> {
 230                 try
 231                 {
 232                     IndexPackingRequest request = new IndexPackingRequest( indexingContext,
 233                         indexingContext.acquireIndexSearcher( ).getIndexReader( ),
 234                         indexingContext.getIndexDirectoryFile( ) );
 235                     indexPacker.packIndex( request );
 236                     indexingContext.updateTimestamp( true );
 237                 }
 238                 catch ( IOException e )
 239                 {
 240                     log.error( "IOException while packing index of context " + context.getId( ) + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ) );
 241                     throw new IndexUpdateFailedException( "IOException during update of " + context.getId( ), e );
 242                 }
 243             }
 244         );
 245
 246     }
 247
 248     @Override
 249     public void scan(final ArchivaIndexingContext context) throws IndexUpdateFailedException
 250     {
 251         executeUpdateFunction( context, indexingContext -> {
 252             DefaultScannerListener listener = new DefaultScannerListener( indexingContext, indexerEngine, true, null );
 253             ScanningRequest request = new ScanningRequest( indexingContext, listener );
 254             ScanningResult result = scanner.scan( request );
 255             if ( result.hasExceptions( ) )
 256             {
 257                 log.error( "Exceptions occured during index scan of " + context.getId( ) );
 258                 result.getExceptions( ).stream( ).map( e -> e.getMessage( ) ).distinct( ).limit( 5 ).forEach(
 259                     s -> log.error( "Message: " + s )
 260                 );
 261             }
 262
 263         } );
 264     }
 265
 266     @Override
 267     public void update(final ArchivaIndexingContext context, final boolean fullUpdate) throws IndexUpdateFailedException
 268     {
 269         log.info( "start download remote index for remote repository {}", context.getRepository( ).getId( ) );
 270         URI remoteUpdateUri;
 271         if ( !( context.getRepository( ) instanceof RemoteRepository ) || !(context.getRepository().supportsFeature(RemoteIndexFeature.class)) )
 272         {
 273             throw new IndexUpdateFailedException( "The context is not associated to a remote repository with remote index " + context.getId( ) );
 274         } else {
 275             RemoteIndexFeature rif = context.getRepository().getFeature(RemoteIndexFeature.class).get();
 276             remoteUpdateUri = context.getRepository().getLocation().resolve(rif.getIndexUri());
 277         }
 278         final RemoteRepository remoteRepository = (RemoteRepository) context.getRepository( );
 279
 280         executeUpdateFunction( context,
 281             indexingContext -> {
 282                 try
 283                 {
 284                     // create a temp directory to download files
 285                     Path tempIndexDirectory = Paths.get( indexingContext.getIndexDirectoryFile( ).getParent( ), ".tmpIndex" );
 286                     Path indexCacheDirectory = Paths.get( indexingContext.getIndexDirectoryFile( ).getParent( ), ".indexCache" );
 287                     Files.createDirectories( indexCacheDirectory );
 288                     if ( Files.exists( tempIndexDirectory ) )
 289                     {
 290                         org.apache.archiva.common.utils.FileUtils.deleteDirectory( tempIndexDirectory );
 291                     }
 292                     Files.createDirectories( tempIndexDirectory );
 293                     tempIndexDirectory.toFile( ).deleteOnExit( );
 294                     String baseIndexUrl = indexingContext.getIndexUpdateUrl( );
 295
 296                     String wagonProtocol = remoteUpdateUri.toURL( ).getProtocol( );
 297
 298                     NetworkProxy networkProxy = null;
 299                     if ( remoteRepository.supportsFeature( RemoteIndexFeature.class ) )
 300                     {
 301                         RemoteIndexFeature rif = remoteRepository.getFeature( RemoteIndexFeature.class ).get( );
 302                         if ( StringUtils.isNotBlank( rif.getProxyId( ) ) )
 303                         {
 304                             networkProxy = proxyRegistry.getNetworkProxy( rif.getProxyId( ) );
 305                             if ( networkProxy == null )
 306                             {
 307                                 log.warn(
 308                                     "your remote repository is configured to download remote index trought a proxy we cannot find id:{}",
 309                                     rif.getProxyId( ) );
 310                             }
 311                         }
 312
 313                         final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon(
 314                             new WagonFactoryRequest( wagonProtocol, remoteRepository.getExtraHeaders( ) ).networkProxy(
 315                                 networkProxy )
 316                         );
 317                         int readTimeout = (int) rif.getDownloadTimeout( ).toMillis( ) * 1000;
 318                         wagon.setReadTimeout( readTimeout );
 319                         wagon.setTimeout( (int) remoteRepository.getTimeout( ).toMillis( ) * 1000 );
 320
 321                         if ( wagon instanceof AbstractHttpClientWagon )
 322                         {
 323                             HttpConfiguration httpConfiguration = new HttpConfiguration( );
 324                             HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration( );
 325                             httpMethodConfiguration.setUsePreemptive( true );
 326                             httpMethodConfiguration.setReadTimeout( readTimeout );
 327                             httpConfiguration.setGet( httpMethodConfiguration );
 328                             AbstractHttpClientWagon.class.cast( wagon ).setHttpConfiguration( httpConfiguration );
 329                         }
 330
 331                         wagon.addTransferListener( new DownloadListener( ) );
 332                         ProxyInfo proxyInfo = null;
 333                         if ( networkProxy != null )
 334                         {
 335                             proxyInfo = new ProxyInfo( );
 336                             proxyInfo.setType( networkProxy.getProtocol( ) );
 337                             proxyInfo.setHost( networkProxy.getHost( ) );
 338                             proxyInfo.setPort( networkProxy.getPort( ) );
 339                             proxyInfo.setUserName( networkProxy.getUsername( ) );
 340                             proxyInfo.setPassword( new String(networkProxy.getPassword( )) );
 341                         }
 342                         AuthenticationInfo authenticationInfo = null;
 343                         if ( remoteRepository.getLoginCredentials( ) != null && ( remoteRepository.getLoginCredentials( ) instanceof PasswordCredentials ) )
 344                         {
 345                             PasswordCredentials creds = (PasswordCredentials) remoteRepository.getLoginCredentials( );
 346                             authenticationInfo = new AuthenticationInfo( );
 347                             authenticationInfo.setUserName( creds.getUsername( ) );
 348                             authenticationInfo.setPassword( new String( creds.getPassword( ) ) );
 349                         }
 350                         wagon.connect( new org.apache.maven.wagon.repository.Repository( remoteRepository.getId( ), baseIndexUrl ), authenticationInfo,
 351                             proxyInfo );
 352
 353                         Path indexDirectory = indexingContext.getIndexDirectoryFile( ).toPath( );
 354                         if ( !Files.exists( indexDirectory ) )
 355                         {
 356                             Files.createDirectories( indexDirectory );
 357                         }
 358
 359                         ResourceFetcher resourceFetcher =
 360                             new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository );
 361                         IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
 362                         request.setForceFullUpdate( fullUpdate );
 363                         request.setLocalIndexCacheDir( indexCacheDirectory.toFile( ) );
 364
 365                         indexUpdater.fetchAndUpdateIndex( request );
 366
 367                         indexingContext.updateTimestamp( true );
 368                     }
 369
 370                 }
 371                 catch ( AuthenticationException e )
 372                 {
 373                     log.error( "Could not login to the remote proxy for updating index of {}", remoteRepository.getId( ), e );
 374                     throw new IndexUpdateFailedException( "Login in to proxy failed while updating remote repository " + remoteRepository.getId( ), e );
 375                 }
 376                 catch ( ConnectionException e )
 377                 {
 378                     log.error( "Connection error during index update for remote repository {}", remoteRepository.getId( ), e );
 379                     throw new IndexUpdateFailedException( "Connection error during index update for remote repository " + remoteRepository.getId( ), e );
 380                 }
 381                 catch ( MalformedURLException e )
 382                 {
 383                     log.error( "URL for remote index update of remote repository {} is not correct {}", remoteRepository.getId( ), remoteUpdateUri, e );
 384                     throw new IndexUpdateFailedException( "URL for remote index update of repository is not correct " + remoteUpdateUri, e );
 385                 }
 386                 catch ( IOException e )
 387                 {
 388                     log.error( "IOException during index update of remote repository {}: {}", remoteRepository.getId( ), e.getMessage( ), e );
 389                     throw new IndexUpdateFailedException( "IOException during index update of remote repository " + remoteRepository.getId( )
 390                         + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ), e );
 391                 }
 392                 catch ( WagonFactoryException e )
 393                 {
 394                     log.error( "Wagon for remote index download of {} could not be created: {}", remoteRepository.getId( ), e.getMessage( ), e );
 395                     throw new IndexUpdateFailedException( "Error while updating the remote index of " + remoteRepository.getId( ), e );
 396                 }
 397             } );
 398
 399     }
 400
 401     @Override
 402     public void addArtifactsToIndex( final ArchivaIndexingContext context, final Collection<URI> artifactReference ) throws IndexUpdateFailedException
 403     {
 404         final StorageAsset ctxUri = context.getPath();
 405         executeUpdateFunction(context, indexingContext -> {
 406             Collection<ArtifactContext> artifacts = artifactReference.stream().map(r -> artifactContextProducer.getArtifactContext(indexingContext, Paths.get(ctxUri.getFilePath().toUri().resolve(r)).toFile())).collect(Collectors.toList());
 407             try {
 408                 indexer.addArtifactsToIndex(artifacts, indexingContext);
 409             } catch (IOException e) {
 410                 log.error("IOException while adding artifact {}", e.getMessage(), e);
 411                 throw new IndexUpdateFailedException("Error occured while adding artifact to index of "+context.getId()
 412                 + (StringUtils.isNotEmpty(e.getMessage()) ? ": "+e.getMessage() : ""));
 413             }
 414         });
 415     }
 416
 417     @Override
 418     public void removeArtifactsFromIndex( ArchivaIndexingContext context, Collection<URI> artifactReference ) throws IndexUpdateFailedException
 419     {
 420         final StorageAsset ctxUri = context.getPath();
 421         executeUpdateFunction(context, indexingContext -> {
 422             Collection<ArtifactContext> artifacts = artifactReference.stream().map(r -> artifactContextProducer.getArtifactContext(indexingContext, Paths.get(ctxUri.getFilePath().toUri().resolve(r)).toFile())).collect(Collectors.toList());
 423             try {
 424                 indexer.deleteArtifactsFromIndex(artifacts, indexingContext);
 425             } catch (IOException e) {
 426                 log.error("IOException while removing artifact {}", e.getMessage(), e);
 427                 throw new IndexUpdateFailedException("Error occured while removing artifact from index of "+context.getId()
 428                         + (StringUtils.isNotEmpty(e.getMessage()) ? ": "+e.getMessage() : ""));
 429             }
 430         });
 431
 432     }
 433
 434     @Override
 435     public boolean supportsRepository( RepositoryType type )
 436     {
 437         return type == RepositoryType.MAVEN;
 438     }
 439
 440     @Override
 441     public ArchivaIndexingContext createContext( Repository repository ) throws IndexCreationFailedException
 442     {
 443         log.debug("Creating context for repo {}, type: {}", repository.getId(), repository.getType());
 444         if ( repository.getType( ) != RepositoryType.MAVEN )
 445         {
 446             throw new UnsupportedRepositoryTypeException( repository.getType( ) );
 447         }
 448         IndexingContext mvnCtx = null;
 449         try
 450         {
 451             if ( repository instanceof RemoteRepository )
 452             {
 453                 mvnCtx = createRemoteContext( (RemoteRepository) repository );
 454             }
 455             else if ( repository instanceof ManagedRepository )
 456             {
 457                 mvnCtx = createManagedContext( (ManagedRepository) repository );
 458             }
 459         }
 460         catch ( IOException e )
 461         {
 462             log.error( "IOException during context creation " + e.getMessage( ), e );
 463             throw new IndexCreationFailedException( "Could not create index context for repository " + repository.getId( )
 464                 + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ), e );
 465         }
 466
 467         return new MavenIndexContext( repository, mvnCtx );
 468     }
 469
 470     @Override
 471     public ArchivaIndexingContext reset(ArchivaIndexingContext context) throws IndexUpdateFailedException {
 472         ArchivaIndexingContext ctx;
 473         executeUpdateFunction(context, indexingContext -> {
 474             try {
 475                 indexingContext.close(true);
 476             } catch (IOException e) {
 477                 log.warn("Index close failed");
 478             }
 479             org.apache.archiva.repository.storage.util.StorageUtil.deleteRecursively(context.getPath());
 480         });
 481         try {
 482             Repository repo = context.getRepository();
 483             ctx = createContext(context.getRepository());
 484             if (repo instanceof EditableRepository) {
 485                 ((EditableRepository)repo).setIndexingContext(ctx);
 486             }
 487         } catch (IndexCreationFailedException e) {
 488             throw new IndexUpdateFailedException("Could not create index");
 489         }
 490         return ctx;
 491     }
 492
 493     @Override
 494     public ArchivaIndexingContext move(ArchivaIndexingContext context, Repository repo) throws IndexCreationFailedException {
 495         if (context==null) {
 496             return null;
 497         }
 498         if (context.supports(IndexingContext.class)) {
 499             try {
 500                 StorageAsset newPath = getIndexPath(repo);
 501                 IndexingContext ctx = context.getBaseContext(IndexingContext.class);
 502                 Path oldPath = ctx.getIndexDirectoryFile().toPath();
 503                 Path newFilePath = newPath.getFilePath( );
 504                 if (oldPath.equals(newFilePath)) {
 505                     // Nothing to do, if path does not change
 506                     return context;
 507                 }
 508                 if (!Files.exists(oldPath)) {
 509                     return createContext(repo);
 510                 } else if (context.isEmpty()) {
 511                     context.close();
 512                     return createContext(repo);
 513                 } else {
 514                     context.close(false);
 515                     if (Files.exists( newFilePath )) {
 516                         FileUtils.copyContent( oldPath, newFilePath );
 517                         FileUtils.deleteDirectory( oldPath );
 518                     } else
 519                     {
 520                         Files.move( oldPath, newFilePath );
 521                     }
 522                     return createContext(repo);
 523                 }
 524             } catch (IOException e) {
 525                 log.error("IOException while moving index directory {}", e.getMessage(), e);
 526                 throw new IndexCreationFailedException("Could not recreated the index.", e);
 527             } catch (UnsupportedBaseContextException e) {
 528                 throw new IndexCreationFailedException("The given context, is not a maven context.");
 529             }
 530         } else {
 531             throw new IndexCreationFailedException("Bad context type. This is not a maven context.");
 532         }
 533     }
 534
 535     @Override
 536     public void updateLocalIndexPath(Repository repo) {
 537         if (repo.supportsFeature(IndexCreationFeature.class)) {
 538             IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
 539             try {
 540                 icf.setLocalIndexPath(getIndexPath(repo));
 541                 icf.setLocalPackedIndexPath(getPackedIndexPath(repo));
 542             } catch (IOException e) {
 543                 log.error("Could not set local index path for {}. New URI: {}", repo.getId(), icf.getIndexPath());
 544             }
 545         }
 546     }
 547
 548     @Override
 549     public ArchivaIndexingContext mergeContexts(Repository destinationRepo, List<ArchivaIndexingContext> contexts,
 550                                                 boolean packIndex) throws UnsupportedOperationException,
 551             IndexCreationFailedException, IllegalArgumentException {
 552         if (!destinationRepo.supportsFeature(IndexCreationFeature.class)) {
 553             throw new IllegalArgumentException("The given repository does not support the indexcreation feature");
 554         }
 555         Path mergedIndexDirectory = null;
 556         try {
 557             mergedIndexDirectory = Files.createTempDirectory("archivaMergedIndex");
 558         } catch (IOException e) {
 559             log.error("Could not create temporary directory for merged index: {}", e.getMessage(), e);
 560             throw new IndexCreationFailedException("IO error while creating temporary directory for merged index: "+e.getMessage(), e);
 561         }
 562         IndexCreationFeature indexCreationFeature = destinationRepo.getFeature(IndexCreationFeature.class).get();
 563         if (indexCreationFeature.getLocalIndexPath()== null) {
 564             throw new IllegalArgumentException("The given repository does not have a local index path");
 565         }
 566         StorageAsset destinationPath = indexCreationFeature.getLocalIndexPath();
 567
 568         String tempRepoId = mergedIndexDirectory.getFileName().toString();
 569
 570         try
 571         {
 572             Path indexLocation = destinationPath.getFilePath();
 573
 574             List<IndexingContext> members = contexts.stream( ).filter(ctx -> ctx.supports(IndexingContext.class)).map( ctx ->
 575             {
 576                 try {
 577                     return ctx.getBaseContext(IndexingContext.class);
 578                 } catch (UnsupportedBaseContextException e) {
 579                     // does not happen here
 580                     return null;
 581                 }
 582             }).filter( Objects::nonNull ).collect( Collectors.toList() );
 583             ContextMemberProvider memberProvider = new StaticContextMemberProvider(members);
 584             IndexingContext mergedCtx = indexer.createMergedIndexingContext( tempRepoId, tempRepoId, mergedIndexDirectory.toFile(),
 585                     indexLocation.toFile(), true, memberProvider);
 586             mergedCtx.optimize();
 587
 588             if ( packIndex )
 589             {
 590                 IndexPackingRequest request = new IndexPackingRequest( mergedCtx, //
 591                         mergedCtx.acquireIndexSearcher().getIndexReader(), //
 592                         indexLocation.toFile() );
 593                 indexPacker.packIndex( request );
 594             }
 595
 596             return new MavenIndexContext(destinationRepo, mergedCtx);
 597         }
 598         catch ( IOException e)
 599         {
 600             throw new IndexCreationFailedException( "IO Error during index merge: "+ e.getMessage(), e );
 601         }
 602     }
 603
 604     private StorageAsset getIndexPath(URI indexDirUri, RepositoryStorage repoStorage, String defaultDir) throws IOException
 605     {
 606         StorageAsset rootAsset = repoStorage.getRoot();
 607         RepositoryStorage storage = rootAsset.getStorage();
 608         Path indexDirectory;
 609         Path repositoryPath = rootAsset.getFilePath().toAbsolutePath();
 610         StorageAsset indexDir;
 611         if ( ! StringUtils.isEmpty(indexDirUri.toString( ) ) )
 612         {
 613
 614             indexDirectory = PathUtil.getPathFromUri( indexDirUri );
 615             // not absolute so create it in repository directory
 616             if ( indexDirectory.isAbsolute( ) && !indexDirectory.startsWith(repositoryPath))
 617             {
 618                 if (storage instanceof FilesystemStorage) {
 619                     FilesystemStorage fsStorage = (FilesystemStorage) storage;
 620                     FilesystemStorage indexStorage = new FilesystemStorage(indexDirectory.getParent(), fsStorage.getFileLockManager());
 621                     indexDir = indexStorage.getAsset(indexDirectory.getFileName().toString());
 622                 } else {
 623                     throw new IOException("The given storage is not file based.");
 624                 }
 625             } else if (indexDirectory.isAbsolute()) {
 626                 indexDir = storage.getAsset(repositoryPath.relativize(indexDirectory).toString());
 627             }
 628             else
 629             {
 630                 indexDir = storage.getAsset(indexDirectory.toString());
 631             }
 632         }
 633         else
 634         {
 635             indexDir = storage.getAsset( defaultDir );
 636         }
 637
 638         if ( !indexDir.exists() )
 639         {
 640             indexDir.create( AssetType.CONTAINER );
 641         }
 642         return indexDir;
 643     }
 644
 645     private StorageAsset getIndexPath( Repository repo) throws IOException {
 646         IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
 647         return getIndexPath( icf.getIndexPath(), repo, DEFAULT_INDEX_PATH);
 648     }
 649
 650     private StorageAsset getPackedIndexPath(Repository repo) throws IOException {
 651         IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
 652         return getIndexPath(icf.getPackedIndexPath(), repo, DEFAULT_PACKED_INDEX_PATH);
 653     }
 654
 655     private IndexingContext createRemoteContext(RemoteRepository remoteRepository ) throws IOException
 656     {
 657         String contextKey = "remote-" + remoteRepository.getId( );
 658
 659
 660         // create remote repository path
 661         Path repoDir = remoteRepository.getRoot().getFilePath();
 662         if ( !Files.exists( repoDir ) )
 663         {
 664             Files.createDirectories( repoDir );
 665         }
 666
 667         StorageAsset indexDirectory;
 668
 669         // is there configured indexDirectory ?
 670         if ( remoteRepository.supportsFeature( RemoteIndexFeature.class ) )
 671         {
 672             RemoteIndexFeature rif = remoteRepository.getFeature( RemoteIndexFeature.class ).get( );
 673             indexDirectory = getIndexPath(remoteRepository);
 674             String remoteIndexUrl = calculateIndexRemoteUrl( remoteRepository.getLocation( ), rif );
 675             try
 676             {
 677
 678                 return getIndexingContext( remoteRepository, contextKey, repoDir, indexDirectory, remoteIndexUrl );
 679             }
 680             catch ( IndexFormatTooOldException e )
 681             {
 682                 // existing index with an old lucene format so we need to delete it!!!
 683                 // delete it first then recreate it.
 684                 log.warn( "the index of repository {} is too old we have to delete and recreate it", //
 685                     remoteRepository.getId( ) );
 686                 org.apache.archiva.common.utils.FileUtils.deleteDirectory( indexDirectory.getFilePath() );
 687                 return getIndexingContext( remoteRepository, contextKey, repoDir, indexDirectory, remoteIndexUrl );
 688
 689             }
 690         }
 691         else
 692         {
 693             throw new IOException( "No remote index defined" );
 694         }
 695     }
 696
 697     private IndexingContext getIndexingContext( Repository repository, String contextKey, Path repoDir, StorageAsset indexDirectory, String indexUrl ) throws IOException
 698     {
 699         try
 700         {
 701             if (!Files.exists(indexDirectory.getFilePath())) {
 702                 Files.createDirectories(indexDirectory.getFilePath());
 703             }
 704             return indexer.createIndexingContext( contextKey, repository.getId( ), repoDir.toFile( ), indexDirectory.getFilePath( ).toFile( ),
 705                 repository.getLocation( ) == null ? null : repository.getLocation( ).toString( ),
 706                 indexUrl,
 707                 true, false,
 708                 indexCreators );
 709         } catch (Exception e) {
 710             log.error("Could not create index for asset {}", indexDirectory);
 711             throw new IOException(e);
 712         }
 713     }
 714
 715     private IndexingContext createManagedContext( ManagedRepository repository ) throws IOException
 716     {
 717
 718         IndexingContext context;
 719         // take care first about repository location as can be relative
 720         Path repositoryDirectory = repository.getRoot().getFilePath();
 721
 722         if ( !Files.exists( repositoryDirectory ) )
 723         {
 724             try
 725             {
 726                 Files.createDirectories( repositoryDirectory );
 727             }
 728             catch ( IOException e )
 729             {
 730                 log.error( "Could not create directory {}", repositoryDirectory );
 731             }
 732         }
 733
 734         StorageAsset indexDirectory;
 735
 736         if ( repository.supportsFeature( IndexCreationFeature.class ) )
 737         {
 738             indexDirectory = getIndexPath(repository);
 739             log.debug( "Preparing index at {}", indexDirectory );
 740
 741             String indexUrl = repositoryDirectory.toUri( ).toURL( ).toExternalForm( );
 742             try
 743             {
 744                 context = getIndexingContext( repository, repository.getId( ), repositoryDirectory, indexDirectory, indexUrl );
 745                 context.setSearchable( repository.isScanned( ) );
 746             }
 747             catch ( IndexFormatTooOldException e )
 748             {
 749                 // existing index with an old lucene format so we need to delete it!!!
 750                 // delete it first then recreate it.
 751                 log.warn( "the index of repository {} is too old we have to delete and recreate it", //
 752                     repository.getId( ) );
 753                 org.apache.archiva.common.utils.FileUtils.deleteDirectory( indexDirectory.getFilePath() );
 754                 context = getIndexingContext( repository, repository.getId( ), repositoryDirectory, indexDirectory, indexUrl );
 755                 context.setSearchable( repository.isScanned( ) );
 756             }
 757             return context;
 758         }
 759         else
 760         {
 761             throw new IOException( "No repository index defined" );
 762         }
 763     }
 764
 765     private String calculateIndexRemoteUrl( URI baseUri, RemoteIndexFeature rif )
 766     {
 767         if ( rif.getIndexUri( ) == null )
 768         {
 769             return baseUri.resolve( "/"+DEFAULT_INDEX_PATH ).toString( );
 770         }
 771         else
 772         {
 773             URI rifUri = rif.getIndexUri( );
 774             if (rifUri.isAbsolute()) {
 775                 return rifUri.toString( );
 776             } else
 777             {
 778                 if (baseUri.getScheme().toLowerCase().equals( "file" )) {
 779                     return Paths.get( baseUri ).resolve( rifUri.getPath() ).toUri( ).toString( );
 780                 } else
 781                 {
 782                     String pathString = rifUri.getPath( ).startsWith( "/" ) ? rifUri.getPath( ) : "/" + rifUri.getPath( );
 783                     return baseUri.resolve( pathString ).toString( );
 784                 }
 785             }
 786         }
 787     }
 788
 789     private static final class DownloadListener
 790         implements TransferListener
 791     {
 792         private Logger log = LoggerFactory.getLogger( getClass( ) );
 793
 794         private String resourceName;
 795
 796         private long startTime;
 797
 798         private int totalLength = 0;
 799
 800         @Override
 801         public void transferInitiated( TransferEvent transferEvent )
 802         {
 803             startTime = System.currentTimeMillis( );
 804             resourceName = transferEvent.getResource( ).getName( );
 805             log.debug( "initiate transfer of {}", resourceName );
 806         }
 807
 808         @Override
 809         public void transferStarted( TransferEvent transferEvent )
 810         {
 811             this.totalLength = 0;
 812             resourceName = transferEvent.getResource( ).getName( );
 813             log.info( "start transfer of {}", transferEvent.getResource( ).getName( ) );
 814         }
 815
 816         @Override
 817         public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
 818         {
 819             log.debug( "transfer of {} : {}/{}", transferEvent.getResource( ).getName( ), buffer.length, length );
 820             this.totalLength += length;
 821         }
 822
 823         @Override
 824         public void transferCompleted( TransferEvent transferEvent )
 825         {
 826             resourceName = transferEvent.getResource( ).getName( );
 827             long endTime = System.currentTimeMillis( );
 828             log.info( "end of transfer file {} {} kb: {}s", transferEvent.getResource( ).getName( ),
 829                 this.totalLength / 1024, ( endTime - startTime ) / 1000 );
 830         }
 831
 832         @Override
 833         public void transferError( TransferEvent transferEvent )
 834         {
 835             log.info( "error of transfer file {}: {}", transferEvent.getResource( ).getName( ),
 836                 transferEvent.getException( ).getMessage( ), transferEvent.getException( ) );
 837         }
 838
 839         @Override
 840         public void debug( String message )
 841         {
 842             log.debug( "transfer debug {}", message );
 843         }
 844     }
 845
 846     private static class WagonResourceFetcher
 847         implements ResourceFetcher
 848     {
 849
 850         Logger log;
 851
 852         Path tempIndexDirectory;
 853
 854         Wagon wagon;
 855
 856         RemoteRepository remoteRepository;
 857
 858         private WagonResourceFetcher( Logger log, Path tempIndexDirectory, Wagon wagon,
 859                                       RemoteRepository remoteRepository )
 860         {
 861             this.log = log;
 862             this.tempIndexDirectory = tempIndexDirectory;
 863             this.wagon = wagon;
 864             this.remoteRepository = remoteRepository;
 865         }
 866
 867         @Override
 868         public void connect( String id, String url ) {
 869             //no op
 870         }
 871
 872         @Override
 873         public void disconnect( ) {
 874             // no op
 875         }
 876
 877         @Override
 878         public InputStream retrieve( String name )
 879             throws IOException {
 880             try
 881             {
 882                 log.info( "index update retrieve file, name:{}", name );
 883                 Path file = tempIndexDirectory.resolve( name );
 884                 Files.deleteIfExists( file );
 885                 file.toFile( ).deleteOnExit( );
 886                 wagon.get( addParameters( name, remoteRepository ), file.toFile( ) );
 887                 return Files.newInputStream( file );
 888             }
 889             catch ( AuthorizationException | TransferFailedException e )
 890             {
 891                 throw new IOException( e.getMessage( ), e );
 892             }
 893             catch ( ResourceDoesNotExistException e )
 894             {
 895                 FileNotFoundException fnfe = new FileNotFoundException( e.getMessage( ) );
 896                 fnfe.initCause( e );
 897                 throw fnfe;
 898             }
 899         }
 900
 901         // FIXME remove crappy copy/paste
 902         protected String addParameters( String path, RemoteRepository remoteRepository )
 903         {
 904             if ( remoteRepository.getExtraParameters( ).isEmpty( ) )
 905             {
 906                 return path;
 907             }
 908
 909             boolean question = false;
 910
 911             StringBuilder res = new StringBuilder( path == null ? "" : path );
 912
 913             for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters( ).entrySet( ) )
 914             {
 915                 if ( !question )
 916                 {
 917                     res.append( '?' ).append( entry.getKey( ) ).append( '=' ).append( entry.getValue( ) );
 918                 }
 919             }
 920
 921             return res.toString( );
 922         }
 923
 924     }
 925 }