source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.indexer.maven;
   2
   3 /*
   4  * Licensed to the Apache Software Foundation (ASF) under one
   5  * or more contributor license agreements.  See the NOTICE file
   6  * distributed with this work for additional information
   7  * regarding copyright ownership.  The ASF licenses this file
   8  * to you under the Apache License, Version 2.0 (the
   9  * "License"); you may not use this file except in compliance
  10  * with the License.  You may obtain a copy of the License at
  11  *
  12  *  http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing,
  15  * software distributed under the License is distributed on an
  16  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  17  * KIND, either express or implied.  See the License for the
  18  * specific language governing permissions and limitations
  19  * under the License.
  20  */
  21
  22 import org.apache.archiva.admin.model.RepositoryAdminException;
  23 import org.apache.archiva.admin.model.beans.NetworkProxy;
  24 import org.apache.archiva.admin.model.networkproxy.NetworkProxyAdmin;
  25 import org.apache.archiva.common.utils.FileUtils;
  26 import org.apache.archiva.common.utils.PathUtil;
  27 import org.apache.archiva.configuration.ArchivaConfiguration;
  28 import org.apache.archiva.indexer.ArchivaIndexManager;
  29 import org.apache.archiva.indexer.ArchivaIndexingContext;
  30 import org.apache.archiva.indexer.IndexCreationFailedException;
  31 import org.apache.archiva.indexer.IndexUpdateFailedException;
  32 import org.apache.archiva.indexer.UnsupportedBaseContextException;
  33 import org.apache.archiva.proxy.common.WagonFactory;
  34 import org.apache.archiva.proxy.common.WagonFactoryException;
  35 import org.apache.archiva.proxy.common.WagonFactoryRequest;
  36 import org.apache.archiva.repository.*;
  37 import org.apache.archiva.repository.features.IndexCreationFeature;
  38 import org.apache.archiva.repository.features.RemoteIndexFeature;
  39 import org.apache.commons.lang.StringUtils;
  40 import org.apache.maven.index.*;
  41 import org.apache.maven.index.context.IndexCreator;
  42 import org.apache.maven.index.context.IndexingContext;
  43 import org.apache.maven.index.packer.IndexPacker;
  44 import org.apache.maven.index.packer.IndexPackingRequest;
  45 import org.apache.maven.index.updater.IndexUpdateRequest;
  46 import org.apache.maven.index.updater.IndexUpdater;
  47 import org.apache.maven.index.updater.ResourceFetcher;
  48 import org.apache.maven.index_shaded.lucene.index.IndexFormatTooOldException;
  49 import org.apache.maven.wagon.ConnectionException;
  50 import org.apache.maven.wagon.ResourceDoesNotExistException;
  51 import org.apache.maven.wagon.StreamWagon;
  52 import org.apache.maven.wagon.TransferFailedException;
  53 import org.apache.maven.wagon.Wagon;
  54 import org.apache.maven.wagon.authentication.AuthenticationException;
  55 import org.apache.maven.wagon.authentication.AuthenticationInfo;
  56 import org.apache.maven.wagon.authorization.AuthorizationException;
  57 import org.apache.maven.wagon.events.TransferEvent;
  58 import org.apache.maven.wagon.events.TransferListener;
  59 import org.apache.maven.wagon.proxy.ProxyInfo;
  60 import org.apache.maven.wagon.shared.http.AbstractHttpClientWagon;
  61 import org.apache.maven.wagon.shared.http.HttpConfiguration;
  62 import org.apache.maven.wagon.shared.http.HttpMethodConfiguration;
  63 import org.slf4j.Logger;
  64 import org.slf4j.LoggerFactory;
  65 import org.springframework.stereotype.Service;
  66
  67 import javax.inject.Inject;
  68 import java.io.FileNotFoundException;
  69 import java.io.IOException;
  70 import java.io.InputStream;
  71 import java.net.MalformedURLException;
  72 import java.net.URI;
  73 import java.nio.file.Files;
  74 import java.nio.file.Path;
  75 import java.nio.file.Paths;
  76 import java.util.Collection;
  77 import java.util.List;
  78 import java.util.Map;
  79 import java.util.concurrent.ConcurrentSkipListSet;
  80 import java.util.stream.Collectors;
  81
  82 /**
  83  * Maven implementation of index manager.
  84  * The index manager is a singleton, so we try to make sure, that index operations are not running
  85  * parallel by synchronizing on the index path.
  86  * A update operation waits for parallel running methods to finish before starting, but after a certain
  87  * time of retries a IndexUpdateFailedException is thrown.
  88  */
  89 @Service( "archivaIndexManager#maven" )
  90 public class MavenIndexManager implements ArchivaIndexManager
  91 {
  92
  93     private static final Logger log = LoggerFactory.getLogger( MavenIndexManager.class );
  94
  95     @Inject
  96     private Indexer indexer;
  97
  98     @Inject
  99     private IndexerEngine indexerEngine;
 100
 101     @Inject
 102     private List<? extends IndexCreator> indexCreators;
 103
 104     @Inject
 105     private IndexPacker indexPacker;
 106
 107     @Inject
 108     private Scanner scanner;
 109
 110     @Inject
 111     private ArchivaConfiguration archivaConfiguration;
 112
 113     @Inject
 114     private WagonFactory wagonFactory;
 115
 116     @Inject
 117     private NetworkProxyAdmin networkProxyAdmin;
 118
 119     @Inject
 120     private IndexUpdater indexUpdater;
 121
 122     @Inject
 123     private ArtifactContextProducer artifactContextProducer;
 124
 125     private ConcurrentSkipListSet<Path> activeContexts = new ConcurrentSkipListSet<>( );
 126
 127     private static final int WAIT_TIME = 100;
 128     private static final int MAX_WAIT = 10;
 129
 130
 131     public static IndexingContext getMvnContext( ArchivaIndexingContext context ) throws UnsupportedBaseContextException
 132     {
 133         if ( !context.supports( IndexingContext.class ) )
 134         {
 135             log.error( "The provided archiva index context does not support the maven IndexingContext" );
 136             throw new UnsupportedBaseContextException( "The context does not support the Maven IndexingContext" );
 137         }
 138         return context.getBaseContext( IndexingContext.class );
 139     }
 140
 141     private Path getIndexPath( ArchivaIndexingContext ctx )
 142     {
 143         return PathUtil.getPathFromUri( ctx.getPath( ) );
 144     }
 145
 146     @FunctionalInterface
 147     interface IndexUpdateConsumer
 148     {
 149
 150         void accept( IndexingContext indexingContext ) throws IndexUpdateFailedException;
 151     }
 152
 153     /*
 154      * This method is used to do some actions around the update execution code. And to make sure, that no other
 155      * method is running on the same index.
 156      */
 157     private void executeUpdateFunction( ArchivaIndexingContext context, IndexUpdateConsumer function ) throws IndexUpdateFailedException
 158     {
 159         IndexingContext indexingContext = null;
 160         try
 161         {
 162             indexingContext = getMvnContext( context );
 163         }
 164         catch ( UnsupportedBaseContextException e )
 165         {
 166             throw new IndexUpdateFailedException( "Maven index is not supported by this context", e );
 167         }
 168         final Path ctxPath = getIndexPath( context );
 169         int loop = MAX_WAIT;
 170         boolean active = false;
 171         while ( loop-- > 0 && !active )
 172         {
 173             active = activeContexts.add( ctxPath );
 174             try
 175             {
 176                 Thread.currentThread( ).sleep( WAIT_TIME );
 177             }
 178             catch ( InterruptedException e )
 179             {
 180                 // Ignore this
 181             }
 182         }
 183         if ( active )
 184         {
 185             try
 186             {
 187                 function.accept( indexingContext );
 188             }
 189             finally
 190             {
 191                 activeContexts.remove( ctxPath );
 192             }
 193         }
 194         else
 195         {
 196             throw new IndexUpdateFailedException( "Timeout while waiting for index release on context " + context.getId( ) );
 197         }
 198     }
 199
 200     @Override
 201     public void pack( final ArchivaIndexingContext context ) throws IndexUpdateFailedException
 202     {
 203         executeUpdateFunction( context, indexingContext -> {
 204                 try
 205                 {
 206                     IndexPackingRequest request = new IndexPackingRequest( indexingContext,
 207                         indexingContext.acquireIndexSearcher( ).getIndexReader( ),
 208                         indexingContext.getIndexDirectoryFile( ) );
 209                     indexPacker.packIndex( request );
 210                     indexingContext.updateTimestamp( true );
 211                 }
 212                 catch ( IOException e )
 213                 {
 214                     log.error( "IOException while packing index of context " + context.getId( ) + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ) );
 215                     throw new IndexUpdateFailedException( "IOException during update of " + context.getId( ), e );
 216                 }
 217             }
 218         );
 219
 220     }
 221
 222     @Override
 223     public void scan(final ArchivaIndexingContext context) throws IndexUpdateFailedException
 224     {
 225         executeUpdateFunction( context, indexingContext -> {
 226             DefaultScannerListener listener = new DefaultScannerListener( indexingContext, indexerEngine, true, null );
 227             ScanningRequest request = new ScanningRequest( indexingContext, listener );
 228             ScanningResult result = scanner.scan( request );
 229             if ( result.hasExceptions( ) )
 230             {
 231                 log.error( "Exceptions occured during index scan of " + context.getId( ) );
 232                 result.getExceptions( ).stream( ).map( e -> e.getMessage( ) ).distinct( ).limit( 5 ).forEach(
 233                     s -> log.error( "Message: " + s )
 234                 );
 235             }
 236
 237         } );
 238     }
 239
 240     @Override
 241     public void update(final ArchivaIndexingContext context, final boolean fullUpdate) throws IndexUpdateFailedException
 242     {
 243         log.info( "start download remote index for remote repository {}", context.getRepository( ).getId( ) );
 244         URI remoteUpdateUri;
 245         if ( !( context.getRepository( ) instanceof RemoteRepository ) || !(context.getRepository().supportsFeature(RemoteIndexFeature.class)) )
 246         {
 247             throw new IndexUpdateFailedException( "The context is not associated to a remote repository with remote index " + context.getId( ) );
 248         } else {
 249             RemoteIndexFeature rif = context.getRepository().getFeature(RemoteIndexFeature.class).get();
 250             remoteUpdateUri = context.getRepository().getLocation().resolve(rif.getIndexUri());
 251         }
 252         final RemoteRepository remoteRepository = (RemoteRepository) context.getRepository( );
 253
 254         executeUpdateFunction( context,
 255             indexingContext -> {
 256                 try
 257                 {
 258                     // create a temp directory to download files
 259                     Path tempIndexDirectory = Paths.get( indexingContext.getIndexDirectoryFile( ).getParent( ), ".tmpIndex" );
 260                     Path indexCacheDirectory = Paths.get( indexingContext.getIndexDirectoryFile( ).getParent( ), ".indexCache" );
 261                     Files.createDirectories( indexCacheDirectory );
 262                     if ( Files.exists( tempIndexDirectory ) )
 263                     {
 264                         org.apache.archiva.common.utils.FileUtils.deleteDirectory( tempIndexDirectory );
 265                     }
 266                     Files.createDirectories( tempIndexDirectory );
 267                     tempIndexDirectory.toFile( ).deleteOnExit( );
 268                     String baseIndexUrl = indexingContext.getIndexUpdateUrl( );
 269
 270                     String wagonProtocol = remoteUpdateUri.toURL( ).getProtocol( );
 271
 272                     NetworkProxy networkProxy = null;
 273                     if ( remoteRepository.supportsFeature( RemoteIndexFeature.class ) )
 274                     {
 275                         RemoteIndexFeature rif = remoteRepository.getFeature( RemoteIndexFeature.class ).get( );
 276                         if ( StringUtils.isNotBlank( rif.getProxyId( ) ) )
 277                         {
 278                             try
 279                             {
 280                                 networkProxy = networkProxyAdmin.getNetworkProxy( rif.getProxyId( ) );
 281                             }
 282                             catch ( RepositoryAdminException e )
 283                             {
 284                                 log.error( "Error occured while retrieving proxy {}", e.getMessage( ) );
 285                             }
 286                             if ( networkProxy == null )
 287                             {
 288                                 log.warn(
 289                                     "your remote repository is configured to download remote index trought a proxy we cannot find id:{}",
 290                                     rif.getProxyId( ) );
 291                             }
 292                         }
 293
 294                         final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon(
 295                             new WagonFactoryRequest( wagonProtocol, remoteRepository.getExtraHeaders( ) ).networkProxy(
 296                                 networkProxy )
 297                         );
 298                         int readTimeout = (int) rif.getDownloadTimeout( ).toMillis( ) * 1000;
 299                         wagon.setReadTimeout( readTimeout );
 300                         wagon.setTimeout( (int) remoteRepository.getTimeout( ).toMillis( ) * 1000 );
 301
 302                         if ( wagon instanceof AbstractHttpClientWagon )
 303                         {
 304                             HttpConfiguration httpConfiguration = new HttpConfiguration( );
 305                             HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration( );
 306                             httpMethodConfiguration.setUsePreemptive( true );
 307                             httpMethodConfiguration.setReadTimeout( readTimeout );
 308                             httpConfiguration.setGet( httpMethodConfiguration );
 309                             AbstractHttpClientWagon.class.cast( wagon ).setHttpConfiguration( httpConfiguration );
 310                         }
 311
 312                         wagon.addTransferListener( new DownloadListener( ) );
 313                         ProxyInfo proxyInfo = null;
 314                         if ( networkProxy != null )
 315                         {
 316                             proxyInfo = new ProxyInfo( );
 317                             proxyInfo.setType( networkProxy.getProtocol( ) );
 318                             proxyInfo.setHost( networkProxy.getHost( ) );
 319                             proxyInfo.setPort( networkProxy.getPort( ) );
 320                             proxyInfo.setUserName( networkProxy.getUsername( ) );
 321                             proxyInfo.setPassword( networkProxy.getPassword( ) );
 322                         }
 323                         AuthenticationInfo authenticationInfo = null;
 324                         if ( remoteRepository.getLoginCredentials( ) != null && ( remoteRepository.getLoginCredentials( ) instanceof PasswordCredentials ) )
 325                         {
 326                             PasswordCredentials creds = (PasswordCredentials) remoteRepository.getLoginCredentials( );
 327                             authenticationInfo = new AuthenticationInfo( );
 328                             authenticationInfo.setUserName( creds.getUsername( ) );
 329                             authenticationInfo.setPassword( new String( creds.getPassword( ) ) );
 330                         }
 331                         wagon.connect( new org.apache.maven.wagon.repository.Repository( remoteRepository.getId( ), baseIndexUrl ), authenticationInfo,
 332                             proxyInfo );
 333
 334                         Path indexDirectory = indexingContext.getIndexDirectoryFile( ).toPath( );
 335                         if ( !Files.exists( indexDirectory ) )
 336                         {
 337                             Files.createDirectories( indexDirectory );
 338                         }
 339
 340                         ResourceFetcher resourceFetcher =
 341                             new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository );
 342                         IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
 343                         request.setForceFullUpdate( fullUpdate );
 344                         request.setLocalIndexCacheDir( indexCacheDirectory.toFile( ) );
 345
 346                         indexUpdater.fetchAndUpdateIndex( request );
 347
 348                         indexingContext.updateTimestamp( true );
 349                     }
 350
 351                 }
 352                 catch ( AuthenticationException e )
 353                 {
 354                     log.error( "Could not login to the remote proxy for updating index of {}", remoteRepository.getId( ), e );
 355                     throw new IndexUpdateFailedException( "Login in to proxy failed while updating remote repository " + remoteRepository.getId( ), e );
 356                 }
 357                 catch ( ConnectionException e )
 358                 {
 359                     log.error( "Connection error during index update for remote repository {}", remoteRepository.getId( ), e );
 360                     throw new IndexUpdateFailedException( "Connection error during index update for remote repository " + remoteRepository.getId( ), e );
 361                 }
 362                 catch ( MalformedURLException e )
 363                 {
 364                     log.error( "URL for remote index update of remote repository {} is not correct {}", remoteRepository.getId( ), remoteUpdateUri, e );
 365                     throw new IndexUpdateFailedException( "URL for remote index update of repository is not correct " + remoteUpdateUri, e );
 366                 }
 367                 catch ( IOException e )
 368                 {
 369                     log.error( "IOException during index update of remote repository {}: {}", remoteRepository.getId( ), e.getMessage( ), e );
 370                     throw new IndexUpdateFailedException( "IOException during index update of remote repository " + remoteRepository.getId( )
 371                         + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ), e );
 372                 }
 373                 catch ( WagonFactoryException e )
 374                 {
 375                     log.error( "Wagon for remote index download of {} could not be created: {}", remoteRepository.getId( ), e.getMessage( ), e );
 376                     throw new IndexUpdateFailedException( "Error while updating the remote index of " + remoteRepository.getId( ), e );
 377                 }
 378             } );
 379
 380     }
 381
 382     @Override
 383     public void addArtifactsToIndex( final ArchivaIndexingContext context, final Collection<URI> artifactReference ) throws IndexUpdateFailedException
 384     {
 385         final URI ctxUri = context.getPath();
 386         executeUpdateFunction(context, indexingContext -> {
 387             Collection<ArtifactContext> artifacts = artifactReference.stream().map(r -> artifactContextProducer.getArtifactContext(indexingContext, Paths.get(ctxUri.resolve(r)).toFile())).collect(Collectors.toList());
 388             try {
 389                 indexer.addArtifactsToIndex(artifacts, indexingContext);
 390             } catch (IOException e) {
 391                 log.error("IOException while adding artifact {}", e.getMessage(), e);
 392                 throw new IndexUpdateFailedException("Error occured while adding artifact to index of "+context.getId()
 393                 + (StringUtils.isNotEmpty(e.getMessage()) ? ": "+e.getMessage() : ""));
 394             }
 395         });
 396     }
 397
 398     @Override
 399     public void removeArtifactsFromIndex( ArchivaIndexingContext context, Collection<URI> artifactReference ) throws IndexUpdateFailedException
 400     {
 401         final URI ctxUri = context.getPath();
 402         executeUpdateFunction(context, indexingContext -> {
 403             Collection<ArtifactContext> artifacts = artifactReference.stream().map(r -> artifactContextProducer.getArtifactContext(indexingContext, Paths.get(ctxUri.resolve(r)).toFile())).collect(Collectors.toList());
 404             try {
 405                 indexer.deleteArtifactsFromIndex(artifacts, indexingContext);
 406             } catch (IOException e) {
 407                 log.error("IOException while removing artifact {}", e.getMessage(), e);
 408                 throw new IndexUpdateFailedException("Error occured while removing artifact from index of "+context.getId()
 409                         + (StringUtils.isNotEmpty(e.getMessage()) ? ": "+e.getMessage() : ""));
 410             }
 411         });
 412
 413     }
 414
 415     @Override
 416     public boolean supportsRepository( RepositoryType type )
 417     {
 418         return type == RepositoryType.MAVEN;
 419     }
 420
 421     @Override
 422     public ArchivaIndexingContext createContext( Repository repository ) throws IndexCreationFailedException
 423     {
 424         log.debug("Creating context for repo {}, type: {}", repository.getId(), repository.getType());
 425         if ( repository.getType( ) != RepositoryType.MAVEN )
 426         {
 427             throw new UnsupportedRepositoryTypeException( repository.getType( ) );
 428         }
 429         IndexingContext mvnCtx = null;
 430         try
 431         {
 432             if ( repository instanceof RemoteRepository )
 433             {
 434                 mvnCtx = createRemoteContext( (RemoteRepository) repository );
 435             }
 436             else if ( repository instanceof ManagedRepository )
 437             {
 438                 mvnCtx = createManagedContext( (ManagedRepository) repository );
 439             }
 440         }
 441         catch ( IOException e )
 442         {
 443             log.error( "IOException during context creation " + e.getMessage( ), e );
 444             throw new IndexCreationFailedException( "Could not create index context for repository " + repository.getId( )
 445                 + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ), e );
 446         }
 447         MavenIndexContext context = new MavenIndexContext( repository, mvnCtx );
 448
 449         return context;
 450     }
 451
 452     @Override
 453     public ArchivaIndexingContext reset(ArchivaIndexingContext context) throws IndexUpdateFailedException {
 454         ArchivaIndexingContext ctx;
 455         executeUpdateFunction(context, indexingContext -> {
 456             try {
 457                 indexingContext.close(true);
 458             } catch (IOException e) {
 459                 log.warn("Index close failed");
 460             }
 461             try {
 462                 FileUtils.deleteDirectory(Paths.get(context.getPath()));
 463             } catch (IOException e) {
 464                 throw new IndexUpdateFailedException("Could not delete index files");
 465             }
 466         });
 467         try {
 468             Repository repo = context.getRepository();
 469             ctx = createContext(context.getRepository());
 470             if (repo instanceof EditableRepository) {
 471                 ((EditableRepository)repo).setIndexingContext(ctx);
 472             }
 473         } catch (IndexCreationFailedException e) {
 474             throw new IndexUpdateFailedException("Could not create index");
 475         }
 476         return ctx;
 477     }
 478
 479     @Override
 480     public ArchivaIndexingContext move(ArchivaIndexingContext context, Repository repo) throws IndexCreationFailedException {
 481         if (context==null) {
 482             return null;
 483         }
 484         if (context.supports(IndexingContext.class)) {
 485             try {
 486                 Path newPath = getIndexPath(repo);
 487                 IndexingContext ctx = context.getBaseContext(IndexingContext.class);
 488                 Path oldPath = ctx.getIndexDirectoryFile().toPath();
 489                 if (oldPath.equals(newPath)) {
 490                     // Nothing to do, if path does not change
 491                     return context;
 492                 }
 493                 if (!Files.exists(oldPath)) {
 494                     return createContext(repo);
 495                 } else if (context.isEmpty()) {
 496                     context.close();
 497                     return createContext(repo);
 498                 } else {
 499                     context.close(false);
 500                     Files.move(oldPath, newPath);
 501                     return createContext(repo);
 502                 }
 503             } catch (IOException e) {
 504                 log.error("IOException while moving index directory {}", e.getMessage(), e);
 505                 throw new IndexCreationFailedException("Could not recreated the index.", e);
 506             } catch (UnsupportedBaseContextException e) {
 507                 throw new IndexCreationFailedException("The given context, is not a maven context.");
 508             }
 509         } else {
 510             throw new IndexCreationFailedException("Bad context type. This is not a maven context.");
 511         }
 512     }
 513
 514     private Path getIndexPath(Repository repo) throws IOException {
 515         IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
 516         Path repoDir = repo.getLocalPath();
 517         URI indexDir = icf.getIndexPath();
 518         Path indexDirectory = null;
 519         if ( ! StringUtils.isEmpty(indexDir.toString( ) ) )
 520         {
 521
 522             indexDirectory = PathUtil.getPathFromUri( indexDir );
 523             // not absolute so create it in repository directory
 524             if ( !indexDirectory.isAbsolute( ) )
 525             {
 526                 indexDirectory = repoDir.resolve( indexDirectory );
 527             }
 528         }
 529         else
 530         {
 531             indexDirectory = repoDir.resolve( ".index" );
 532         }
 533
 534         if ( !Files.exists( indexDirectory ) )
 535         {
 536             Files.createDirectories( indexDirectory );
 537         }
 538         return indexDirectory;
 539     }
 540
 541     private IndexingContext createRemoteContext(RemoteRepository remoteRepository ) throws IOException
 542     {
 543         Path appServerBase = archivaConfiguration.getAppServerBaseDir( );
 544
 545         String contextKey = "remote-" + remoteRepository.getId( );
 546
 547
 548         // create remote repository path
 549         Path repoDir = remoteRepository.getLocalPath();
 550         if ( !Files.exists( repoDir ) )
 551         {
 552             Files.createDirectories( repoDir );
 553         }
 554
 555         Path indexDirectory = null;
 556
 557         // is there configured indexDirectory ?
 558         if ( remoteRepository.supportsFeature( RemoteIndexFeature.class ) )
 559         {
 560             RemoteIndexFeature rif = remoteRepository.getFeature( RemoteIndexFeature.class ).get( );
 561             indexDirectory = getIndexPath(remoteRepository);
 562             String remoteIndexUrl = calculateIndexRemoteUrl( remoteRepository.getLocation( ), rif );
 563             try
 564             {
 565
 566                 return getIndexingContext( remoteRepository, contextKey, repoDir, indexDirectory, remoteIndexUrl );
 567             }
 568             catch ( IndexFormatTooOldException e )
 569             {
 570                 // existing index with an old lucene format so we need to delete it!!!
 571                 // delete it first then recreate it.
 572                 log.warn( "the index of repository {} is too old we have to delete and recreate it", //
 573                     remoteRepository.getId( ) );
 574                 org.apache.archiva.common.utils.FileUtils.deleteDirectory( indexDirectory );
 575                 return getIndexingContext( remoteRepository, contextKey, repoDir, indexDirectory, remoteIndexUrl );
 576
 577             }
 578         }
 579         else
 580         {
 581             throw new IOException( "No remote index defined" );
 582         }
 583     }
 584
 585     private IndexingContext getIndexingContext( Repository repository, String contextKey, Path repoDir, Path indexDirectory, String indexUrl ) throws IOException
 586     {
 587         return indexer.createIndexingContext( contextKey, repository.getId( ), repoDir.toFile( ), indexDirectory.toFile( ),
 588             repository.getLocation( ) == null ? null : repository.getLocation( ).toString( ),
 589             indexUrl,
 590             true, false,
 591             indexCreators );
 592     }
 593
 594     private IndexingContext createManagedContext( ManagedRepository repository ) throws IOException
 595     {
 596
 597         IndexingContext context;
 598         // take care first about repository location as can be relative
 599         Path repositoryDirectory = repository.getLocalPath();
 600
 601         if ( !Files.exists( repositoryDirectory ) )
 602         {
 603             try
 604             {
 605                 Files.createDirectories( repositoryDirectory );
 606             }
 607             catch ( IOException e )
 608             {
 609                 log.error( "Could not create directory {}", repositoryDirectory );
 610             }
 611         }
 612
 613         Path indexDirectory = null;
 614
 615         if ( repository.supportsFeature( IndexCreationFeature.class ) )
 616         {
 617             indexDirectory = getIndexPath(repository);
 618
 619             String indexUrl = repositoryDirectory.toUri( ).toURL( ).toExternalForm( );
 620             try
 621             {
 622                 context = getIndexingContext( repository, repository.getId( ), repositoryDirectory, indexDirectory, indexUrl );
 623                 context.setSearchable( repository.isScanned( ) );
 624             }
 625             catch ( IndexFormatTooOldException e )
 626             {
 627                 // existing index with an old lucene format so we need to delete it!!!
 628                 // delete it first then recreate it.
 629                 log.warn( "the index of repository {} is too old we have to delete and recreate it", //
 630                     repository.getId( ) );
 631                 org.apache.archiva.common.utils.FileUtils.deleteDirectory( indexDirectory );
 632                 context = getIndexingContext( repository, repository.getId( ), repositoryDirectory, indexDirectory, indexUrl );
 633                 context.setSearchable( repository.isScanned( ) );
 634             }
 635             return context;
 636         }
 637         else
 638         {
 639             throw new IOException( "No repository index defined" );
 640         }
 641     }
 642
 643     private String calculateIndexRemoteUrl( URI baseUri, RemoteIndexFeature rif )
 644     {
 645         if ( rif.getIndexUri( ) == null )
 646         {
 647             return baseUri.resolve( ".index" ).toString( );
 648         }
 649         else
 650         {
 651             return baseUri.resolve( rif.getIndexUri( ) ).toString( );
 652         }
 653     }
 654
 655     private static final class DownloadListener
 656         implements TransferListener
 657     {
 658         private Logger log = LoggerFactory.getLogger( getClass( ) );
 659
 660         private String resourceName;
 661
 662         private long startTime;
 663
 664         private int totalLength = 0;
 665
 666         @Override
 667         public void transferInitiated( TransferEvent transferEvent )
 668         {
 669             startTime = System.currentTimeMillis( );
 670             resourceName = transferEvent.getResource( ).getName( );
 671             log.debug( "initiate transfer of {}", resourceName );
 672         }
 673
 674         @Override
 675         public void transferStarted( TransferEvent transferEvent )
 676         {
 677             this.totalLength = 0;
 678             resourceName = transferEvent.getResource( ).getName( );
 679             log.info( "start transfer of {}", transferEvent.getResource( ).getName( ) );
 680         }
 681
 682         @Override
 683         public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
 684         {
 685             log.debug( "transfer of {} : {}/{}", transferEvent.getResource( ).getName( ), buffer.length, length );
 686             this.totalLength += length;
 687         }
 688
 689         @Override
 690         public void transferCompleted( TransferEvent transferEvent )
 691         {
 692             resourceName = transferEvent.getResource( ).getName( );
 693             long endTime = System.currentTimeMillis( );
 694             log.info( "end of transfer file {} {} kb: {}s", transferEvent.getResource( ).getName( ),
 695                 this.totalLength / 1024, ( endTime - startTime ) / 1000 );
 696         }
 697
 698         @Override
 699         public void transferError( TransferEvent transferEvent )
 700         {
 701             log.info( "error of transfer file {}: {}", transferEvent.getResource( ).getName( ),
 702                 transferEvent.getException( ).getMessage( ), transferEvent.getException( ) );
 703         }
 704
 705         @Override
 706         public void debug( String message )
 707         {
 708             log.debug( "transfer debug {}", message );
 709         }
 710     }
 711
 712     private static class WagonResourceFetcher
 713         implements ResourceFetcher
 714     {
 715
 716         Logger log;
 717
 718         Path tempIndexDirectory;
 719
 720         Wagon wagon;
 721
 722         RemoteRepository remoteRepository;
 723
 724         private WagonResourceFetcher( Logger log, Path tempIndexDirectory, Wagon wagon,
 725                                       RemoteRepository remoteRepository )
 726         {
 727             this.log = log;
 728             this.tempIndexDirectory = tempIndexDirectory;
 729             this.wagon = wagon;
 730             this.remoteRepository = remoteRepository;
 731         }
 732
 733         @Override
 734         public void connect( String id, String url )
 735             throws IOException
 736         {
 737             //no op
 738         }
 739
 740         @Override
 741         public void disconnect( )
 742             throws IOException
 743         {
 744             // no op
 745         }
 746
 747         @Override
 748         public InputStream retrieve( String name )
 749             throws IOException, FileNotFoundException
 750         {
 751             try
 752             {
 753                 log.info( "index update retrieve file, name:{}", name );
 754                 Path file = tempIndexDirectory.resolve( name );
 755                 Files.deleteIfExists( file );
 756                 file.toFile( ).deleteOnExit( );
 757                 wagon.get( addParameters( name, remoteRepository ), file.toFile( ) );
 758                 return Files.newInputStream( file );
 759             }
 760             catch ( AuthorizationException | TransferFailedException e )
 761             {
 762                 throw new IOException( e.getMessage( ), e );
 763             }
 764             catch ( ResourceDoesNotExistException e )
 765             {
 766                 FileNotFoundException fnfe = new FileNotFoundException( e.getMessage( ) );
 767                 fnfe.initCause( e );
 768                 throw fnfe;
 769             }
 770         }
 771
 772         // FIXME remove crappy copy/paste
 773         protected String addParameters( String path, RemoteRepository remoteRepository )
 774         {
 775             if ( remoteRepository.getExtraParameters( ).isEmpty( ) )
 776             {
 777                 return path;
 778             }
 779
 780             boolean question = false;
 781
 782             StringBuilder res = new StringBuilder( path == null ? "" : path );
 783
 784             for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters( ).entrySet( ) )
 785             {
 786                 if ( !question )
 787                 {
 788                     res.append( '?' ).append( entry.getKey( ) ).append( '=' ).append( entry.getValue( ) );
 789                 }
 790             }
 791
 792             return res.toString( );
 793         }
 794
 795     }
 796 }