source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.indexer.maven;
   2
   3 /*
   4  * Licensed to the Apache Software Foundation (ASF) under one
   5  * or more contributor license agreements.  See the NOTICE file
   6  * distributed with this work for additional information
   7  * regarding copyright ownership.  The ASF licenses this file
   8  * to you under the Apache License, Version 2.0 (the
   9  * "License"); you may not use this file except in compliance
  10  * with the License.  You may obtain a copy of the License at
  11  *
  12  *  http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing,
  15  * software distributed under the License is distributed on an
  16  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  17  * KIND, either express or implied.  See the License for the
  18  * specific language governing permissions and limitations
  19  * under the License.
  20  */
  21
  22 import org.apache.archiva.common.utils.FileUtils;
  23 import org.apache.archiva.common.utils.PathUtil;
  24 import org.apache.archiva.configuration.ArchivaConfiguration;
  25 import org.apache.archiva.indexer.ArchivaIndexManager;
  26 import org.apache.archiva.indexer.ArchivaIndexingContext;
  27 import org.apache.archiva.indexer.IndexCreationFailedException;
  28 import org.apache.archiva.indexer.IndexUpdateFailedException;
  29 import org.apache.archiva.indexer.UnsupportedBaseContextException;
  30 import org.apache.archiva.proxy.ProxyRegistry;
  31 import org.apache.archiva.proxy.maven.WagonFactory;
  32 import org.apache.archiva.proxy.maven.WagonFactoryException;
  33 import org.apache.archiva.proxy.maven.WagonFactoryRequest;
  34 import org.apache.archiva.proxy.model.NetworkProxy;
  35 import org.apache.archiva.repository.EditableRepository;
  36 import org.apache.archiva.repository.ManagedRepository;
  37 import org.apache.archiva.repository.base.PasswordCredentials;
  38 import org.apache.archiva.repository.RemoteRepository;
  39 import org.apache.archiva.repository.Repository;
  40 import org.apache.archiva.repository.RepositoryType;
  41 import org.apache.archiva.repository.UnsupportedRepositoryTypeException;
  42 import org.apache.archiva.repository.storage.AssetType;
  43 import org.apache.archiva.repository.storage.fs.FilesystemStorage;
  44 import org.apache.archiva.repository.storage.RepositoryStorage;
  45 import org.apache.archiva.repository.storage.StorageAsset;
  46 import org.apache.archiva.repository.features.IndexCreationFeature;
  47 import org.apache.archiva.repository.features.RemoteIndexFeature;
  48 import org.apache.commons.lang3.StringUtils;
  49 import org.apache.maven.index.ArtifactContext;
  50 import org.apache.maven.index.ArtifactContextProducer;
  51 import org.apache.maven.index.DefaultScannerListener;
  52 import org.apache.maven.index.Indexer;
  53 import org.apache.maven.index.IndexerEngine;
  54 import org.apache.maven.index.Scanner;
  55 import org.apache.maven.index.ScanningRequest;
  56 import org.apache.maven.index.ScanningResult;
  57 import org.apache.maven.index.context.ContextMemberProvider;
  58 import org.apache.maven.index.context.IndexCreator;
  59 import org.apache.maven.index.context.IndexingContext;
  60 import org.apache.maven.index.context.StaticContextMemberProvider;
  61 import org.apache.maven.index.packer.IndexPacker;
  62 import org.apache.maven.index.packer.IndexPackingRequest;
  63 import org.apache.maven.index.updater.IndexUpdateRequest;
  64 import org.apache.maven.index.updater.IndexUpdater;
  65 import org.apache.maven.index.updater.ResourceFetcher;
  66 import org.apache.maven.index_shaded.lucene.index.IndexFormatTooOldException;
  67 import org.apache.maven.wagon.ConnectionException;
  68 import org.apache.maven.wagon.ResourceDoesNotExistException;
  69 import org.apache.maven.wagon.StreamWagon;
  70 import org.apache.maven.wagon.TransferFailedException;
  71 import org.apache.maven.wagon.Wagon;
  72 import org.apache.maven.wagon.authentication.AuthenticationException;
  73 import org.apache.maven.wagon.authentication.AuthenticationInfo;
  74 import org.apache.maven.wagon.authorization.AuthorizationException;
  75 import org.apache.maven.wagon.events.TransferEvent;
  76 import org.apache.maven.wagon.events.TransferListener;
  77 import org.apache.maven.wagon.proxy.ProxyInfo;
  78 import org.apache.maven.wagon.shared.http.AbstractHttpClientWagon;
  79 import org.apache.maven.wagon.shared.http.HttpConfiguration;
  80 import org.apache.maven.wagon.shared.http.HttpMethodConfiguration;
  81 import org.slf4j.Logger;
  82 import org.slf4j.LoggerFactory;
  83 import org.springframework.stereotype.Service;
  84
  85 import javax.inject.Inject;
  86 import java.io.FileNotFoundException;
  87 import java.io.IOException;
  88 import java.io.InputStream;
  89 import java.net.MalformedURLException;
  90 import java.net.URI;
  91 import java.nio.file.Files;
  92 import java.nio.file.Path;
  93 import java.nio.file.Paths;
  94 import java.util.Collection;
  95 import java.util.List;
  96 import java.util.Map;
  97 import java.util.Objects;
  98 import java.util.concurrent.ConcurrentSkipListSet;
  99 import java.util.stream.Collectors;
 100
 101 /**
 102  * Maven implementation of index manager.
 103  * The index manager is a singleton, so we try to make sure, that index operations are not running
 104  * parallel by synchronizing on the index path.
 105  * A update operation waits for parallel running methods to finish before starting, but after a certain
 106  * time of retries a IndexUpdateFailedException is thrown.
 107  */
 108 @Service( "archivaIndexManager#maven" )
 109 public class MavenIndexManager implements ArchivaIndexManager {
 110
 111     private static final Logger log = LoggerFactory.getLogger( MavenIndexManager.class );
 112
 113     @Inject
 114     private Indexer indexer;
 115
 116     @Inject
 117     private IndexerEngine indexerEngine;
 118
 119     @Inject
 120     private List<? extends IndexCreator> indexCreators;
 121
 122     @Inject
 123     private IndexPacker indexPacker;
 124
 125     @Inject
 126     private Scanner scanner;
 127
 128     @Inject
 129     private ArchivaConfiguration archivaConfiguration;
 130
 131     @Inject
 132     private WagonFactory wagonFactory;
 133
 134     @Inject
 135     private IndexUpdater indexUpdater;
 136
 137     @Inject
 138     private ArtifactContextProducer artifactContextProducer;
 139
 140     @Inject
 141     private ProxyRegistry proxyRegistry;
 142
 143
 144     private ConcurrentSkipListSet<StorageAsset> activeContexts = new ConcurrentSkipListSet<>( );
 145
 146     private static final int WAIT_TIME = 100;
 147     private static final int MAX_WAIT = 10;
 148
 149
 150     public static IndexingContext getMvnContext( ArchivaIndexingContext context ) throws UnsupportedBaseContextException
 151     {
 152         if (context!=null)
 153         {
 154             if ( !context.supports( IndexingContext.class ) )
 155             {
 156                 log.error( "The provided archiva index context does not support the maven IndexingContext" );
 157                 throw new UnsupportedBaseContextException( "The context does not support the Maven IndexingContext" );
 158             }
 159             return context.getBaseContext( IndexingContext.class );
 160         } else {
 161             return null;
 162         }
 163     }
 164
 165     private StorageAsset getIndexPath( ArchivaIndexingContext ctx )
 166     {
 167         return ctx.getPath( );
 168     }
 169
 170     @FunctionalInterface
 171     interface IndexUpdateConsumer
 172     {
 173
 174         void accept( IndexingContext indexingContext ) throws IndexUpdateFailedException;
 175     }
 176
 177     /*
 178      * This method is used to do some actions around the update execution code. And to make sure, that no other
 179      * method is running on the same index.
 180      */
 181     private void executeUpdateFunction( ArchivaIndexingContext context, IndexUpdateConsumer function ) throws IndexUpdateFailedException
 182     {
 183         if (context==null) {
 184             throw new IndexUpdateFailedException( "Given context is null" );
 185         }
 186         IndexingContext indexingContext = null;
 187         try
 188         {
 189             indexingContext = getMvnContext( context );
 190         }
 191         catch ( UnsupportedBaseContextException e )
 192         {
 193             throw new IndexUpdateFailedException( "Maven index is not supported by this context", e );
 194         }
 195         final StorageAsset ctxPath = getIndexPath( context );
 196         int loop = MAX_WAIT;
 197         boolean active = false;
 198         while ( loop-- > 0 && !active )
 199         {
 200             active = activeContexts.add( ctxPath );
 201             try
 202             {
 203                 Thread.currentThread( ).sleep( WAIT_TIME );
 204             }
 205             catch ( InterruptedException e )
 206             {
 207                 // Ignore this
 208             }
 209         }
 210         if ( active )
 211         {
 212             try
 213             {
 214                 function.accept( indexingContext );
 215             }
 216             finally
 217             {
 218                 activeContexts.remove( ctxPath );
 219             }
 220         }
 221         else
 222         {
 223             throw new IndexUpdateFailedException( "Timeout while waiting for index release on context " + context.getId( ) );
 224         }
 225     }
 226
 227     @Override
 228     public void pack( final ArchivaIndexingContext context ) throws IndexUpdateFailedException
 229     {
 230         executeUpdateFunction( context, indexingContext -> {
 231                 try
 232                 {
 233                     IndexPackingRequest request = new IndexPackingRequest( indexingContext,
 234                         indexingContext.acquireIndexSearcher( ).getIndexReader( ),
 235                         indexingContext.getIndexDirectoryFile( ) );
 236                     indexPacker.packIndex( request );
 237                     indexingContext.updateTimestamp( true );
 238                 }
 239                 catch ( IOException e )
 240                 {
 241                     log.error( "IOException while packing index of context " + context.getId( ) + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ) );
 242                     throw new IndexUpdateFailedException( "IOException during update of " + context.getId( ), e );
 243                 }
 244             }
 245         );
 246
 247     }
 248
 249     @Override
 250     public void scan(final ArchivaIndexingContext context) throws IndexUpdateFailedException
 251     {
 252         executeUpdateFunction( context, indexingContext -> {
 253             DefaultScannerListener listener = new DefaultScannerListener( indexingContext, indexerEngine, true, null );
 254             ScanningRequest request = new ScanningRequest( indexingContext, listener );
 255             ScanningResult result = scanner.scan( request );
 256             if ( result.hasExceptions( ) )
 257             {
 258                 log.error( "Exceptions occured during index scan of " + context.getId( ) );
 259                 result.getExceptions( ).stream( ).map( e -> e.getMessage( ) ).distinct( ).limit( 5 ).forEach(
 260                     s -> log.error( "Message: " + s )
 261                 );
 262             }
 263
 264         } );
 265     }
 266
 267     @Override
 268     public void update(final ArchivaIndexingContext context, final boolean fullUpdate) throws IndexUpdateFailedException
 269     {
 270         log.info( "start download remote index for remote repository {}", context.getRepository( ).getId( ) );
 271         URI remoteUpdateUri;
 272         if ( !( context.getRepository( ) instanceof RemoteRepository ) || !(context.getRepository().supportsFeature(RemoteIndexFeature.class)) )
 273         {
 274             throw new IndexUpdateFailedException( "The context is not associated to a remote repository with remote index " + context.getId( ) );
 275         } else {
 276             RemoteIndexFeature rif = context.getRepository().getFeature(RemoteIndexFeature.class).get();
 277             remoteUpdateUri = context.getRepository().getLocation().resolve(rif.getIndexUri());
 278         }
 279         final RemoteRepository remoteRepository = (RemoteRepository) context.getRepository( );
 280
 281         executeUpdateFunction( context,
 282             indexingContext -> {
 283                 try
 284                 {
 285                     // create a temp directory to download files
 286                     Path tempIndexDirectory = Paths.get( indexingContext.getIndexDirectoryFile( ).getParent( ), ".tmpIndex" );
 287                     Path indexCacheDirectory = Paths.get( indexingContext.getIndexDirectoryFile( ).getParent( ), ".indexCache" );
 288                     Files.createDirectories( indexCacheDirectory );
 289                     if ( Files.exists( tempIndexDirectory ) )
 290                     {
 291                         org.apache.archiva.common.utils.FileUtils.deleteDirectory( tempIndexDirectory );
 292                     }
 293                     Files.createDirectories( tempIndexDirectory );
 294                     tempIndexDirectory.toFile( ).deleteOnExit( );
 295                     String baseIndexUrl = indexingContext.getIndexUpdateUrl( );
 296
 297                     String wagonProtocol = remoteUpdateUri.toURL( ).getProtocol( );
 298
 299                     NetworkProxy networkProxy = null;
 300                     if ( remoteRepository.supportsFeature( RemoteIndexFeature.class ) )
 301                     {
 302                         RemoteIndexFeature rif = remoteRepository.getFeature( RemoteIndexFeature.class ).get( );
 303                         if ( StringUtils.isNotBlank( rif.getProxyId( ) ) )
 304                         {
 305                             networkProxy = proxyRegistry.getNetworkProxy( rif.getProxyId( ) );
 306                             if ( networkProxy == null )
 307                             {
 308                                 log.warn(
 309                                     "your remote repository is configured to download remote index trought a proxy we cannot find id:{}",
 310                                     rif.getProxyId( ) );
 311                             }
 312                         }
 313
 314                         final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon(
 315                             new WagonFactoryRequest( wagonProtocol, remoteRepository.getExtraHeaders( ) ).networkProxy(
 316                                 networkProxy )
 317                         );
 318                         int readTimeout = (int) rif.getDownloadTimeout( ).toMillis( ) * 1000;
 319                         wagon.setReadTimeout( readTimeout );
 320                         wagon.setTimeout( (int) remoteRepository.getTimeout( ).toMillis( ) * 1000 );
 321
 322                         if ( wagon instanceof AbstractHttpClientWagon )
 323                         {
 324                             HttpConfiguration httpConfiguration = new HttpConfiguration( );
 325                             HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration( );
 326                             httpMethodConfiguration.setUsePreemptive( true );
 327                             httpMethodConfiguration.setReadTimeout( readTimeout );
 328                             httpConfiguration.setGet( httpMethodConfiguration );
 329                             AbstractHttpClientWagon.class.cast( wagon ).setHttpConfiguration( httpConfiguration );
 330                         }
 331
 332                         wagon.addTransferListener( new DownloadListener( ) );
 333                         ProxyInfo proxyInfo = null;
 334                         if ( networkProxy != null )
 335                         {
 336                             proxyInfo = new ProxyInfo( );
 337                             proxyInfo.setType( networkProxy.getProtocol( ) );
 338                             proxyInfo.setHost( networkProxy.getHost( ) );
 339                             proxyInfo.setPort( networkProxy.getPort( ) );
 340                             proxyInfo.setUserName( networkProxy.getUsername( ) );
 341                             proxyInfo.setPassword( new String(networkProxy.getPassword( )) );
 342                         }
 343                         AuthenticationInfo authenticationInfo = null;
 344                         if ( remoteRepository.getLoginCredentials( ) != null && ( remoteRepository.getLoginCredentials( ) instanceof PasswordCredentials ) )
 345                         {
 346                             PasswordCredentials creds = (PasswordCredentials) remoteRepository.getLoginCredentials( );
 347                             authenticationInfo = new AuthenticationInfo( );
 348                             authenticationInfo.setUserName( creds.getUsername( ) );
 349                             authenticationInfo.setPassword( new String( creds.getPassword( ) ) );
 350                         }
 351                         wagon.connect( new org.apache.maven.wagon.repository.Repository( remoteRepository.getId( ), baseIndexUrl ), authenticationInfo,
 352                             proxyInfo );
 353
 354                         Path indexDirectory = indexingContext.getIndexDirectoryFile( ).toPath( );
 355                         if ( !Files.exists( indexDirectory ) )
 356                         {
 357                             Files.createDirectories( indexDirectory );
 358                         }
 359
 360                         ResourceFetcher resourceFetcher =
 361                             new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository );
 362                         IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
 363                         request.setForceFullUpdate( fullUpdate );
 364                         request.setLocalIndexCacheDir( indexCacheDirectory.toFile( ) );
 365
 366                         indexUpdater.fetchAndUpdateIndex( request );
 367
 368                         indexingContext.updateTimestamp( true );
 369                     }
 370
 371                 }
 372                 catch ( AuthenticationException e )
 373                 {
 374                     log.error( "Could not login to the remote proxy for updating index of {}", remoteRepository.getId( ), e );
 375                     throw new IndexUpdateFailedException( "Login in to proxy failed while updating remote repository " + remoteRepository.getId( ), e );
 376                 }
 377                 catch ( ConnectionException e )
 378                 {
 379                     log.error( "Connection error during index update for remote repository {}", remoteRepository.getId( ), e );
 380                     throw new IndexUpdateFailedException( "Connection error during index update for remote repository " + remoteRepository.getId( ), e );
 381                 }
 382                 catch ( MalformedURLException e )
 383                 {
 384                     log.error( "URL for remote index update of remote repository {} is not correct {}", remoteRepository.getId( ), remoteUpdateUri, e );
 385                     throw new IndexUpdateFailedException( "URL for remote index update of repository is not correct " + remoteUpdateUri, e );
 386                 }
 387                 catch ( IOException e )
 388                 {
 389                     log.error( "IOException during index update of remote repository {}: {}", remoteRepository.getId( ), e.getMessage( ), e );
 390                     throw new IndexUpdateFailedException( "IOException during index update of remote repository " + remoteRepository.getId( )
 391                         + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ), e );
 392                 }
 393                 catch ( WagonFactoryException e )
 394                 {
 395                     log.error( "Wagon for remote index download of {} could not be created: {}", remoteRepository.getId( ), e.getMessage( ), e );
 396                     throw new IndexUpdateFailedException( "Error while updating the remote index of " + remoteRepository.getId( ), e );
 397                 }
 398             } );
 399
 400     }
 401
 402     @Override
 403     public void addArtifactsToIndex( final ArchivaIndexingContext context, final Collection<URI> artifactReference ) throws IndexUpdateFailedException
 404     {
 405         final StorageAsset ctxUri = context.getPath();
 406         executeUpdateFunction(context, indexingContext -> {
 407             Collection<ArtifactContext> artifacts = artifactReference.stream().map(r -> artifactContextProducer.getArtifactContext(indexingContext, Paths.get(ctxUri.getFilePath().toUri().resolve(r)).toFile())).collect(Collectors.toList());
 408             try {
 409                 indexer.addArtifactsToIndex(artifacts, indexingContext);
 410             } catch (IOException e) {
 411                 log.error("IOException while adding artifact {}", e.getMessage(), e);
 412                 throw new IndexUpdateFailedException("Error occured while adding artifact to index of "+context.getId()
 413                 + (StringUtils.isNotEmpty(e.getMessage()) ? ": "+e.getMessage() : ""));
 414             }
 415         });
 416     }
 417
 418     @Override
 419     public void removeArtifactsFromIndex( ArchivaIndexingContext context, Collection<URI> artifactReference ) throws IndexUpdateFailedException
 420     {
 421         final StorageAsset ctxUri = context.getPath();
 422         executeUpdateFunction(context, indexingContext -> {
 423             Collection<ArtifactContext> artifacts = artifactReference.stream().map(r -> artifactContextProducer.getArtifactContext(indexingContext, Paths.get(ctxUri.getFilePath().toUri().resolve(r)).toFile())).collect(Collectors.toList());
 424             try {
 425                 indexer.deleteArtifactsFromIndex(artifacts, indexingContext);
 426             } catch (IOException e) {
 427                 log.error("IOException while removing artifact {}", e.getMessage(), e);
 428                 throw new IndexUpdateFailedException("Error occured while removing artifact from index of "+context.getId()
 429                         + (StringUtils.isNotEmpty(e.getMessage()) ? ": "+e.getMessage() : ""));
 430             }
 431         });
 432
 433     }
 434
 435     @Override
 436     public boolean supportsRepository( RepositoryType type )
 437     {
 438         return type == RepositoryType.MAVEN;
 439     }
 440
 441     @Override
 442     public ArchivaIndexingContext createContext( Repository repository ) throws IndexCreationFailedException
 443     {
 444         log.debug("Creating context for repo {}, type: {}", repository.getId(), repository.getType());
 445         if ( repository.getType( ) != RepositoryType.MAVEN )
 446         {
 447             throw new UnsupportedRepositoryTypeException( repository.getType( ) );
 448         }
 449         IndexingContext mvnCtx = null;
 450         try
 451         {
 452             if ( repository instanceof RemoteRepository )
 453             {
 454                 mvnCtx = createRemoteContext( (RemoteRepository) repository );
 455             }
 456             else if ( repository instanceof ManagedRepository )
 457             {
 458                 mvnCtx = createManagedContext( (ManagedRepository) repository );
 459             }
 460         }
 461         catch ( IOException e )
 462         {
 463             log.error( "IOException during context creation " + e.getMessage( ), e );
 464             throw new IndexCreationFailedException( "Could not create index context for repository " + repository.getId( )
 465                 + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ), e );
 466         }
 467
 468         return new MavenIndexContext( repository, mvnCtx );
 469     }
 470
 471     @Override
 472     public ArchivaIndexingContext reset(ArchivaIndexingContext context) throws IndexUpdateFailedException {
 473         ArchivaIndexingContext ctx;
 474         executeUpdateFunction(context, indexingContext -> {
 475             try {
 476                 indexingContext.close(true);
 477             } catch (IOException e) {
 478                 log.warn("Index close failed");
 479             }
 480             org.apache.archiva.repository.storage.util.StorageUtil.deleteRecursively(context.getPath());
 481         });
 482         try {
 483             Repository repo = context.getRepository();
 484             ctx = createContext(context.getRepository());
 485             if (repo instanceof EditableRepository) {
 486                 ((EditableRepository)repo).setIndexingContext(ctx);
 487             }
 488         } catch (IndexCreationFailedException e) {
 489             throw new IndexUpdateFailedException("Could not create index");
 490         }
 491         return ctx;
 492     }
 493
 494     @Override
 495     public ArchivaIndexingContext move(ArchivaIndexingContext context, Repository repo) throws IndexCreationFailedException {
 496         if (context==null) {
 497             return null;
 498         }
 499         if (context.supports(IndexingContext.class)) {
 500             try {
 501                 StorageAsset newPath = getIndexPath(repo);
 502                 IndexingContext ctx = context.getBaseContext(IndexingContext.class);
 503                 Path oldPath = ctx.getIndexDirectoryFile().toPath();
 504                 Path newFilePath = newPath.getFilePath( );
 505                 if (oldPath.equals(newFilePath)) {
 506                     // Nothing to do, if path does not change
 507                     return context;
 508                 }
 509                 if (!Files.exists(oldPath)) {
 510                     return createContext(repo);
 511                 } else if (context.isEmpty()) {
 512                     context.close();
 513                     return createContext(repo);
 514                 } else {
 515                     context.close(false);
 516                     if (Files.exists( newFilePath )) {
 517                         FileUtils.copyContent( oldPath, newFilePath );
 518                         FileUtils.deleteDirectory( oldPath );
 519                     } else
 520                     {
 521                         Files.move( oldPath, newFilePath );
 522                     }
 523                     return createContext(repo);
 524                 }
 525             } catch (IOException e) {
 526                 log.error("IOException while moving index directory {}", e.getMessage(), e);
 527                 throw new IndexCreationFailedException("Could not recreated the index.", e);
 528             } catch (UnsupportedBaseContextException e) {
 529                 throw new IndexCreationFailedException("The given context, is not a maven context.");
 530             }
 531         } else {
 532             throw new IndexCreationFailedException("Bad context type. This is not a maven context.");
 533         }
 534     }
 535
 536     @Override
 537     public void updateLocalIndexPath(Repository repo) {
 538         if (repo.supportsFeature(IndexCreationFeature.class)) {
 539             IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
 540             try {
 541                 icf.setLocalIndexPath(getIndexPath(repo));
 542                 icf.setLocalPackedIndexPath(getPackedIndexPath(repo));
 543             } catch (IOException e) {
 544                 log.error("Could not set local index path for {}. New URI: {}", repo.getId(), icf.getIndexPath());
 545             }
 546         }
 547     }
 548
 549     @Override
 550     public ArchivaIndexingContext mergeContexts(Repository destinationRepo, List<ArchivaIndexingContext> contexts,
 551                                                 boolean packIndex) throws UnsupportedOperationException,
 552             IndexCreationFailedException, IllegalArgumentException {
 553         if (!destinationRepo.supportsFeature(IndexCreationFeature.class)) {
 554             throw new IllegalArgumentException("The given repository does not support the indexcreation feature");
 555         }
 556         Path mergedIndexDirectory = null;
 557         try {
 558             mergedIndexDirectory = Files.createTempDirectory("archivaMergedIndex");
 559         } catch (IOException e) {
 560             log.error("Could not create temporary directory for merged index: {}", e.getMessage(), e);
 561             throw new IndexCreationFailedException("IO error while creating temporary directory for merged index: "+e.getMessage(), e);
 562         }
 563         IndexCreationFeature indexCreationFeature = destinationRepo.getFeature(IndexCreationFeature.class).get();
 564         if (indexCreationFeature.getLocalIndexPath()== null) {
 565             throw new IllegalArgumentException("The given repository does not have a local index path");
 566         }
 567         StorageAsset destinationPath = indexCreationFeature.getLocalIndexPath();
 568
 569         String tempRepoId = mergedIndexDirectory.getFileName().toString();
 570
 571         try
 572         {
 573             Path indexLocation = destinationPath.getFilePath();
 574
 575             List<IndexingContext> members = contexts.stream( ).filter(ctx -> ctx.supports(IndexingContext.class)).map( ctx ->
 576             {
 577                 try {
 578                     return ctx.getBaseContext(IndexingContext.class);
 579                 } catch (UnsupportedBaseContextException e) {
 580                     // does not happen here
 581                     return null;
 582                 }
 583             }).filter( Objects::nonNull ).collect( Collectors.toList() );
 584             ContextMemberProvider memberProvider = new StaticContextMemberProvider(members);
 585             IndexingContext mergedCtx = indexer.createMergedIndexingContext( tempRepoId, tempRepoId, mergedIndexDirectory.toFile(),
 586                     indexLocation.toFile(), true, memberProvider);
 587             mergedCtx.optimize();
 588
 589             if ( packIndex )
 590             {
 591                 IndexPackingRequest request = new IndexPackingRequest( mergedCtx, //
 592                         mergedCtx.acquireIndexSearcher().getIndexReader(), //
 593                         indexLocation.toFile() );
 594                 indexPacker.packIndex( request );
 595             }
 596
 597             return new MavenIndexContext(destinationRepo, mergedCtx);
 598         }
 599         catch ( IOException e)
 600         {
 601             throw new IndexCreationFailedException( "IO Error during index merge: "+ e.getMessage(), e );
 602         }
 603     }
 604
 605     private StorageAsset getIndexPath(URI indexDirUri, RepositoryStorage repoStorage, String defaultDir) throws IOException
 606     {
 607         StorageAsset rootAsset = repoStorage.getRoot();
 608         RepositoryStorage storage = rootAsset.getStorage();
 609         Path indexDirectory;
 610         Path repositoryPath = rootAsset.getFilePath().toAbsolutePath();
 611         StorageAsset indexDir;
 612         if ( ! StringUtils.isEmpty(indexDirUri.toString( ) ) )
 613         {
 614
 615             indexDirectory = PathUtil.getPathFromUri( indexDirUri );
 616             // not absolute so create it in repository directory
 617             if ( indexDirectory.isAbsolute( ) && !indexDirectory.startsWith(repositoryPath))
 618             {
 619                 if (storage instanceof FilesystemStorage) {
 620                     FilesystemStorage fsStorage = (FilesystemStorage) storage;
 621                     FilesystemStorage indexStorage = new FilesystemStorage(indexDirectory.getParent(), fsStorage.getFileLockManager());
 622                     indexDir = indexStorage.getAsset(indexDirectory.getFileName().toString());
 623                 } else {
 624                     throw new IOException("The given storage is not file based.");
 625                 }
 626             } else if (indexDirectory.isAbsolute()) {
 627                 indexDir = storage.getAsset(repositoryPath.relativize(indexDirectory).toString());
 628             }
 629             else
 630             {
 631                 indexDir = storage.getAsset(indexDirectory.toString());
 632             }
 633         }
 634         else
 635         {
 636             indexDir = storage.getAsset( defaultDir );
 637         }
 638
 639         if ( !indexDir.exists() )
 640         {
 641             indexDir.create( AssetType.CONTAINER );
 642         }
 643         return indexDir;
 644     }
 645
 646     private StorageAsset getIndexPath( Repository repo) throws IOException {
 647         IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
 648         return getIndexPath( icf.getIndexPath(), repo, DEFAULT_INDEX_PATH);
 649     }
 650
 651     private StorageAsset getPackedIndexPath(Repository repo) throws IOException {
 652         IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
 653         return getIndexPath(icf.getPackedIndexPath(), repo, DEFAULT_PACKED_INDEX_PATH);
 654     }
 655
 656     private IndexingContext createRemoteContext(RemoteRepository remoteRepository ) throws IOException
 657     {
 658         String contextKey = "remote-" + remoteRepository.getId( );
 659
 660
 661         // create remote repository path
 662         Path repoDir = remoteRepository.getRoot().getFilePath();
 663         if ( !Files.exists( repoDir ) )
 664         {
 665             Files.createDirectories( repoDir );
 666         }
 667
 668         StorageAsset indexDirectory;
 669
 670         // is there configured indexDirectory ?
 671         if ( remoteRepository.supportsFeature( RemoteIndexFeature.class ) )
 672         {
 673             RemoteIndexFeature rif = remoteRepository.getFeature( RemoteIndexFeature.class ).get( );
 674             indexDirectory = getIndexPath(remoteRepository);
 675             String remoteIndexUrl = calculateIndexRemoteUrl( remoteRepository.getLocation( ), rif );
 676             try
 677             {
 678
 679                 return getIndexingContext( remoteRepository, contextKey, repoDir, indexDirectory, remoteIndexUrl );
 680             }
 681             catch ( IndexFormatTooOldException e )
 682             {
 683                 // existing index with an old lucene format so we need to delete it!!!
 684                 // delete it first then recreate it.
 685                 log.warn( "the index of repository {} is too old we have to delete and recreate it", //
 686                     remoteRepository.getId( ) );
 687                 org.apache.archiva.common.utils.FileUtils.deleteDirectory( indexDirectory.getFilePath() );
 688                 return getIndexingContext( remoteRepository, contextKey, repoDir, indexDirectory, remoteIndexUrl );
 689
 690             }
 691         }
 692         else
 693         {
 694             throw new IOException( "No remote index defined" );
 695         }
 696     }
 697
 698     private IndexingContext getIndexingContext( Repository repository, String contextKey, Path repoDir, StorageAsset indexDirectory, String indexUrl ) throws IOException
 699     {
 700         try
 701         {
 702             if (!Files.exists(indexDirectory.getFilePath())) {
 703                 Files.createDirectories(indexDirectory.getFilePath());
 704             }
 705             return indexer.createIndexingContext( contextKey, repository.getId( ), repoDir.toFile( ), indexDirectory.getFilePath( ).toFile( ),
 706                 repository.getLocation( ) == null ? null : repository.getLocation( ).toString( ),
 707                 indexUrl,
 708                 true, false,
 709                 indexCreators );
 710         } catch (Exception e) {
 711             log.error("Could not create index for asset {}", indexDirectory);
 712             throw new IOException(e);
 713         }
 714     }
 715
 716     private IndexingContext createManagedContext( ManagedRepository repository ) throws IOException
 717     {
 718
 719         IndexingContext context;
 720         // take care first about repository location as can be relative
 721         Path repositoryDirectory = repository.getRoot().getFilePath();
 722
 723         if ( !Files.exists( repositoryDirectory ) )
 724         {
 725             try
 726             {
 727                 Files.createDirectories( repositoryDirectory );
 728             }
 729             catch ( IOException e )
 730             {
 731                 log.error( "Could not create directory {}", repositoryDirectory );
 732             }
 733         }
 734
 735         StorageAsset indexDirectory;
 736
 737         if ( repository.supportsFeature( IndexCreationFeature.class ) )
 738         {
 739             indexDirectory = getIndexPath(repository);
 740             log.debug( "Preparing index at {}", indexDirectory );
 741
 742             String indexUrl = repositoryDirectory.toUri( ).toURL( ).toExternalForm( );
 743             try
 744             {
 745                 context = getIndexingContext( repository, repository.getId( ), repositoryDirectory, indexDirectory, indexUrl );
 746                 context.setSearchable( repository.isScanned( ) );
 747             }
 748             catch ( IndexFormatTooOldException e )
 749             {
 750                 // existing index with an old lucene format so we need to delete it!!!
 751                 // delete it first then recreate it.
 752                 log.warn( "the index of repository {} is too old we have to delete and recreate it", //
 753                     repository.getId( ) );
 754                 org.apache.archiva.common.utils.FileUtils.deleteDirectory( indexDirectory.getFilePath() );
 755                 context = getIndexingContext( repository, repository.getId( ), repositoryDirectory, indexDirectory, indexUrl );
 756                 context.setSearchable( repository.isScanned( ) );
 757             }
 758             return context;
 759         }
 760         else
 761         {
 762             throw new IOException( "No repository index defined" );
 763         }
 764     }
 765
 766     private String calculateIndexRemoteUrl( URI baseUri, RemoteIndexFeature rif )
 767     {
 768         if ( rif.getIndexUri( ) == null )
 769         {
 770             return baseUri.resolve( "/"+DEFAULT_INDEX_PATH ).toString( );
 771         }
 772         else
 773         {
 774             URI rifUri = rif.getIndexUri( );
 775             if (rifUri.isAbsolute()) {
 776                 return rifUri.toString( );
 777             } else
 778             {
 779                 if (baseUri.getScheme().toLowerCase().equals( "file" )) {
 780                     return Paths.get( baseUri ).resolve( rifUri.getPath() ).toUri( ).toString( );
 781                 } else
 782                 {
 783                     String pathString = rifUri.getPath( ).startsWith( "/" ) ? rifUri.getPath( ) : "/" + rifUri.getPath( );
 784                     return baseUri.resolve( pathString ).toString( );
 785                 }
 786             }
 787         }
 788     }
 789
 790     private static final class DownloadListener
 791         implements TransferListener
 792     {
 793         private Logger log = LoggerFactory.getLogger( getClass( ) );
 794
 795         private String resourceName;
 796
 797         private long startTime;
 798
 799         private int totalLength = 0;
 800
 801         @Override
 802         public void transferInitiated( TransferEvent transferEvent )
 803         {
 804             startTime = System.currentTimeMillis( );
 805             resourceName = transferEvent.getResource( ).getName( );
 806             log.debug( "initiate transfer of {}", resourceName );
 807         }
 808
 809         @Override
 810         public void transferStarted( TransferEvent transferEvent )
 811         {
 812             this.totalLength = 0;
 813             resourceName = transferEvent.getResource( ).getName( );
 814             log.info( "start transfer of {}", transferEvent.getResource( ).getName( ) );
 815         }
 816
 817         @Override
 818         public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
 819         {
 820             log.debug( "transfer of {} : {}/{}", transferEvent.getResource( ).getName( ), buffer.length, length );
 821             this.totalLength += length;
 822         }
 823
 824         @Override
 825         public void transferCompleted( TransferEvent transferEvent )
 826         {
 827             resourceName = transferEvent.getResource( ).getName( );
 828             long endTime = System.currentTimeMillis( );
 829             log.info( "end of transfer file {} {} kb: {}s", transferEvent.getResource( ).getName( ),
 830                 this.totalLength / 1024, ( endTime - startTime ) / 1000 );
 831         }
 832
 833         @Override
 834         public void transferError( TransferEvent transferEvent )
 835         {
 836             log.info( "error of transfer file {}: {}", transferEvent.getResource( ).getName( ),
 837                 transferEvent.getException( ).getMessage( ), transferEvent.getException( ) );
 838         }
 839
 840         @Override
 841         public void debug( String message )
 842         {
 843             log.debug( "transfer debug {}", message );
 844         }
 845     }
 846
 847     private static class WagonResourceFetcher
 848         implements ResourceFetcher
 849     {
 850
 851         Logger log;
 852
 853         Path tempIndexDirectory;
 854
 855         Wagon wagon;
 856
 857         RemoteRepository remoteRepository;
 858
 859         private WagonResourceFetcher( Logger log, Path tempIndexDirectory, Wagon wagon,
 860                                       RemoteRepository remoteRepository )
 861         {
 862             this.log = log;
 863             this.tempIndexDirectory = tempIndexDirectory;
 864             this.wagon = wagon;
 865             this.remoteRepository = remoteRepository;
 866         }
 867
 868         @Override
 869         public void connect( String id, String url ) {
 870             //no op
 871         }
 872
 873         @Override
 874         public void disconnect( ) {
 875             // no op
 876         }
 877
 878         @Override
 879         public InputStream retrieve( String name )
 880             throws IOException {
 881             try
 882             {
 883                 log.info( "index update retrieve file, name:{}", name );
 884                 Path file = tempIndexDirectory.resolve( name );
 885                 Files.deleteIfExists( file );
 886                 file.toFile( ).deleteOnExit( );
 887                 wagon.get( addParameters( name, remoteRepository ), file.toFile( ) );
 888                 return Files.newInputStream( file );
 889             }
 890             catch ( AuthorizationException | TransferFailedException e )
 891             {
 892                 throw new IOException( e.getMessage( ), e );
 893             }
 894             catch ( ResourceDoesNotExistException e )
 895             {
 896                 FileNotFoundException fnfe = new FileNotFoundException( e.getMessage( ) );
 897                 fnfe.initCause( e );
 898                 throw fnfe;
 899             }
 900         }
 901
 902         // FIXME remove crappy copy/paste
 903         protected String addParameters( String path, RemoteRepository remoteRepository )
 904         {
 905             if ( remoteRepository.getExtraParameters( ).isEmpty( ) )
 906             {
 907                 return path;
 908             }
 909
 910             boolean question = false;
 911
 912             StringBuilder res = new StringBuilder( path == null ? "" : path );
 913
 914             for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters( ).entrySet( ) )
 915             {
 916                 if ( !question )
 917                 {
 918                     res.append( '?' ).append( entry.getKey( ) ).append( '=' ).append( entry.getValue( ) );
 919                 }
 920             }
 921
 922             return res.toString( );
 923         }
 924
 925     }
 926 }