source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.admin.mock;
   2
   3 /*
   4  * Licensed to the Apache Software Foundation (ASF) under one
   5  * or more contributor license agreements.  See the NOTICE file
   6  * distributed with this work for additional information
   7  * regarding copyright ownership.  The ASF licenses this file
   8  * to you under the Apache License, Version 2.0 (the
   9  * "License"); you may not use this file except in compliance
  10  * with the License.  You may obtain a copy of the License at
  11  *
  12  *   http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing,
  15  * software distributed under the License is distributed on an
  16  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  17  * KIND, either express or implied.  See the License for the
  18  * specific language governing permissions and limitations
  19  * under the License.
  20  */
  21
  22 import org.apache.archiva.common.filelock.DefaultFileLockManager;
  23 import org.apache.archiva.common.utils.PathUtil;
  24 import org.apache.archiva.configuration.ArchivaConfiguration;
  25 import org.apache.archiva.indexer.ArchivaIndexManager;
  26 import org.apache.archiva.indexer.ArchivaIndexingContext;
  27 import org.apache.archiva.indexer.IndexCreationFailedException;
  28 import org.apache.archiva.indexer.IndexUpdateFailedException;
  29 import org.apache.archiva.indexer.UnsupportedBaseContextException;
  30 import org.apache.archiva.proxy.ProxyRegistry;
  31 import org.apache.archiva.maven.proxy.WagonFactory;
  32 import org.apache.archiva.maven.proxy.WagonFactoryException;
  33 import org.apache.archiva.maven.proxy.WagonFactoryRequest;
  34 import org.apache.archiva.proxy.model.NetworkProxy;
  35 import org.apache.archiva.repository.EditableRepository;
  36 import org.apache.archiva.repository.ManagedRepository;
  37 import org.apache.archiva.repository.base.PasswordCredentials;
  38 import org.apache.archiva.repository.RemoteRepository;
  39 import org.apache.archiva.repository.Repository;
  40 import org.apache.archiva.repository.RepositoryType;
  41 import org.apache.archiva.repository.UnsupportedRepositoryTypeException;
  42 import org.apache.archiva.repository.storage.fs.FilesystemAsset;
  43 import org.apache.archiva.repository.storage.fs.FilesystemStorage;
  44 import org.apache.archiva.repository.storage.StorageAsset;
  45 import org.apache.archiva.repository.features.IndexCreationFeature;
  46 import org.apache.archiva.repository.features.RemoteIndexFeature;
  47 import org.apache.commons.lang3.StringUtils;
  48 import org.apache.maven.index.ArtifactContext;
  49 import org.apache.maven.index.ArtifactContextProducer;
  50 import org.apache.maven.index.DefaultScannerListener;
  51 import org.apache.maven.index.Indexer;
  52 import org.apache.maven.index.IndexerEngine;
  53 import org.apache.maven.index.Scanner;
  54 import org.apache.maven.index.ScanningRequest;
  55 import org.apache.maven.index.ScanningResult;
  56 import org.apache.maven.index.context.IndexCreator;
  57 import org.apache.maven.index.context.IndexingContext;
  58 import org.apache.maven.index.packer.IndexPacker;
  59 import org.apache.maven.index.packer.IndexPackingRequest;
  60 import org.apache.maven.index.updater.IndexUpdateRequest;
  61 import org.apache.maven.index.updater.ResourceFetcher;
  62 import org.apache.maven.index_shaded.lucene.index.IndexFormatTooOldException;
  63 import org.apache.maven.wagon.ConnectionException;
  64 import org.apache.maven.wagon.ResourceDoesNotExistException;
  65 import org.apache.maven.wagon.StreamWagon;
  66 import org.apache.maven.wagon.TransferFailedException;
  67 import org.apache.maven.wagon.Wagon;
  68 import org.apache.maven.wagon.authentication.AuthenticationException;
  69 import org.apache.maven.wagon.authentication.AuthenticationInfo;
  70 import org.apache.maven.wagon.authorization.AuthorizationException;
  71 import org.apache.maven.wagon.events.TransferEvent;
  72 import org.apache.maven.wagon.events.TransferListener;
  73 import org.apache.maven.wagon.proxy.ProxyInfo;
  74 import org.apache.maven.wagon.shared.http.AbstractHttpClientWagon;
  75 import org.apache.maven.wagon.shared.http.HttpConfiguration;
  76 import org.apache.maven.wagon.shared.http.HttpMethodConfiguration;
  77 import org.slf4j.Logger;
  78 import org.slf4j.LoggerFactory;
  79 import org.springframework.stereotype.Service;
  80
  81 import javax.inject.Inject;
  82 import java.io.FileNotFoundException;
  83 import java.io.IOException;
  84 import java.io.InputStream;
  85 import java.net.MalformedURLException;
  86 import java.net.URI;
  87 import java.nio.file.Files;
  88 import java.nio.file.Path;
  89 import java.nio.file.Paths;
  90 import java.util.Collection;
  91 import java.util.List;
  92 import java.util.Map;
  93 import java.util.concurrent.ConcurrentSkipListSet;
  94 import java.util.stream.Collectors;
  95
  96 @Service("archivaIndexManager#maven")
  97 public class ArchivaIndexManagerMock implements ArchivaIndexManager {
  98
  99     private static final Logger log = LoggerFactory.getLogger( ArchivaIndexManagerMock.class );
 100
 101     @Inject
 102     private Indexer indexer;
 103
 104     @Inject
 105     private IndexerEngine indexerEngine;
 106
 107     @Inject
 108     private List<? extends IndexCreator> indexCreators;
 109
 110     @Inject
 111     private IndexPacker indexPacker;
 112
 113     @Inject
 114     private Scanner scanner;
 115
 116     @Inject
 117     private ArchivaConfiguration archivaConfiguration;
 118
 119     @Inject
 120     private WagonFactory wagonFactory;
 121
 122     @Inject
 123     ProxyRegistry proxyRegistry;
 124
 125     @Inject
 126     private ArtifactContextProducer artifactContextProducer;
 127
 128     private ConcurrentSkipListSet<Path> activeContexts = new ConcurrentSkipListSet<>( );
 129
 130     private static final int WAIT_TIME = 100;
 131     private static final int MAX_WAIT = 10;
 132
 133
 134     public static IndexingContext getMvnContext(ArchivaIndexingContext context ) throws UnsupportedBaseContextException
 135     {
 136         if ( !context.supports( IndexingContext.class ) )
 137         {
 138             log.error( "The provided archiva index context does not support the maven IndexingContext" );
 139             throw new UnsupportedBaseContextException( "The context does not support the Maven IndexingContext" );
 140         }
 141         return context.getBaseContext( IndexingContext.class );
 142     }
 143
 144     private Path getIndexPath( ArchivaIndexingContext ctx )
 145     {
 146         return ctx.getPath( ).getFilePath();
 147     }
 148
 149     @FunctionalInterface
 150     interface IndexUpdateConsumer
 151     {
 152
 153         void accept( IndexingContext indexingContext ) throws IndexUpdateFailedException;
 154     }
 155
 156     /*
 157      * This method is used to do some actions around the update execution code. And to make sure, that no other
 158      * method is running on the same index.
 159      */
 160     private void executeUpdateFunction( ArchivaIndexingContext context, IndexUpdateConsumer function ) throws IndexUpdateFailedException
 161     {
 162         IndexingContext indexingContext = null;
 163         try
 164         {
 165             indexingContext = getMvnContext( context );
 166         }
 167         catch ( UnsupportedBaseContextException e )
 168         {
 169             throw new IndexUpdateFailedException( "Maven index is not supported by this context", e );
 170         }
 171         final Path ctxPath = getIndexPath( context );
 172         int loop = MAX_WAIT;
 173         boolean active = false;
 174         while ( loop-- > 0 && !active )
 175         {
 176             active = activeContexts.add( ctxPath );
 177             try
 178             {
 179                 Thread.currentThread( ).sleep( WAIT_TIME );
 180             }
 181             catch ( InterruptedException e )
 182             {
 183                 // Ignore this
 184             }
 185         }
 186         if ( active )
 187         {
 188             try
 189             {
 190                 function.accept( indexingContext );
 191             }
 192             finally
 193             {
 194                 activeContexts.remove( ctxPath );
 195             }
 196         }
 197         else
 198         {
 199             throw new IndexUpdateFailedException( "Timeout while waiting for index release on context " + context.getId( ) );
 200         }
 201     }
 202
 203     @Override
 204     public void pack( final ArchivaIndexingContext context ) throws IndexUpdateFailedException
 205     {
 206         executeUpdateFunction( context, indexingContext -> {
 207                     try
 208                     {
 209                         IndexPackingRequest request = new IndexPackingRequest( indexingContext,
 210                                 indexingContext.acquireIndexSearcher( ).getIndexReader( ),
 211                                 indexingContext.getIndexDirectoryFile( ) );
 212                         indexPacker.packIndex( request );
 213                         indexingContext.updateTimestamp( true );
 214                     }
 215                     catch ( IOException e )
 216                     {
 217                         log.error( "IOException while packing index of context " + context.getId( ) + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ) );
 218                         throw new IndexUpdateFailedException( "IOException during update of " + context.getId( ), e );
 219                     }
 220                 }
 221         );
 222
 223     }
 224
 225     @Override
 226     public void scan(final ArchivaIndexingContext context) throws IndexUpdateFailedException
 227     {
 228         executeUpdateFunction( context, indexingContext -> {
 229             DefaultScannerListener listener = new DefaultScannerListener( indexingContext, indexerEngine, true, null );
 230             ScanningRequest request = new ScanningRequest( indexingContext, listener );
 231             ScanningResult result = scanner.scan( request );
 232             if ( result.hasExceptions( ) )
 233             {
 234                 log.error( "Exceptions occured during index scan of " + context.getId( ) );
 235                 result.getExceptions( ).stream( ).map( e -> e.getMessage( ) ).distinct( ).limit( 5 ).forEach(
 236                         s -> log.error( "Message: " + s )
 237                 );
 238             }
 239
 240         } );
 241     }
 242
 243     @Override
 244     public void update(final ArchivaIndexingContext context, final boolean fullUpdate) throws IndexUpdateFailedException
 245     {
 246         log.info( "start download remote index for remote repository {}", context.getRepository( ).getId( ) );
 247         URI remoteUpdateUri;
 248         if ( !( context.getRepository( ) instanceof RemoteRepository) || !(context.getRepository().supportsFeature(RemoteIndexFeature.class)) )
 249         {
 250             throw new IndexUpdateFailedException( "The context is not associated to a remote repository with remote index " + context.getId( ) );
 251         } else {
 252             RemoteIndexFeature rif = context.getRepository().getFeature(RemoteIndexFeature.class).get();
 253             remoteUpdateUri = context.getRepository().getLocation().resolve(rif.getIndexUri());
 254         }
 255         final RemoteRepository remoteRepository = (RemoteRepository) context.getRepository( );
 256
 257         executeUpdateFunction( context,
 258                 indexingContext -> {
 259                     try
 260                     {
 261                         // create a temp directory to download files
 262                         Path tempIndexDirectory = Paths.get( indexingContext.getIndexDirectoryFile( ).getParent( ), ".tmpIndex" );
 263                         Path indexCacheDirectory = Paths.get( indexingContext.getIndexDirectoryFile( ).getParent( ), ".indexCache" );
 264                         Files.createDirectories( indexCacheDirectory );
 265                         if ( Files.exists( tempIndexDirectory ) )
 266                         {
 267                             org.apache.archiva.common.utils.FileUtils.deleteDirectory( tempIndexDirectory );
 268                         }
 269                         Files.createDirectories( tempIndexDirectory );
 270                         tempIndexDirectory.toFile( ).deleteOnExit( );
 271                         String baseIndexUrl = indexingContext.getIndexUpdateUrl( );
 272
 273                         String wagonProtocol = remoteUpdateUri.toURL( ).getProtocol( );
 274
 275                         NetworkProxy networkProxy = null;
 276                         if ( remoteRepository.supportsFeature( RemoteIndexFeature.class ) )
 277                         {
 278                             RemoteIndexFeature rif = remoteRepository.getFeature( RemoteIndexFeature.class ).get( );
 279                             if ( StringUtils.isNotBlank( rif.getProxyId( ) ) )
 280                             {
 281                                 networkProxy = proxyRegistry.getNetworkProxy( rif.getProxyId( ) );
 282                                 if ( networkProxy == null )
 283                                 {
 284                                     log.warn(
 285                                             "your remote repository is configured to download remote index trought a proxy we cannot find id:{}",
 286                                             rif.getProxyId( ) );
 287                                 }
 288                             }
 289
 290                             final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon(
 291                                     new WagonFactoryRequest( wagonProtocol, remoteRepository.getExtraHeaders( ) ).networkProxy(
 292                                             networkProxy )
 293                             );
 294                             int readTimeout = (int) rif.getDownloadTimeout( ).toMillis( ) * 1000;
 295                             wagon.setReadTimeout( readTimeout );
 296                             wagon.setTimeout( (int) remoteRepository.getTimeout( ).toMillis( ) * 1000 );
 297
 298                             if ( wagon instanceof AbstractHttpClientWagon)
 299                             {
 300                                 HttpConfiguration httpConfiguration = new HttpConfiguration( );
 301                                 HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration( );
 302                                 httpMethodConfiguration.setUsePreemptive( true );
 303                                 httpMethodConfiguration.setReadTimeout( readTimeout );
 304                                 httpConfiguration.setGet( httpMethodConfiguration );
 305                                 AbstractHttpClientWagon.class.cast( wagon ).setHttpConfiguration( httpConfiguration );
 306                             }
 307
 308                             wagon.addTransferListener( new DownloadListener( ) );
 309                             ProxyInfo proxyInfo = null;
 310                             if ( networkProxy != null )
 311                             {
 312                                 proxyInfo = new ProxyInfo( );
 313                                 proxyInfo.setType( networkProxy.getProtocol( ) );
 314                                 proxyInfo.setHost( networkProxy.getHost( ) );
 315                                 proxyInfo.setPort( networkProxy.getPort( ) );
 316                                 proxyInfo.setUserName( networkProxy.getUsername( ) );
 317                                 proxyInfo.setPassword(new String(networkProxy.getPassword()));
 318                             }
 319                             AuthenticationInfo authenticationInfo = null;
 320                             if ( remoteRepository.getLoginCredentials( ) != null && ( remoteRepository.getLoginCredentials( ) instanceof PasswordCredentials) )
 321                             {
 322                                 PasswordCredentials creds = (PasswordCredentials) remoteRepository.getLoginCredentials( );
 323                                 authenticationInfo = new AuthenticationInfo( );
 324                                 authenticationInfo.setUserName( creds.getUsername( ) );
 325                                 authenticationInfo.setPassword( new String( creds.getPassword( ) ) );
 326                             }
 327                             wagon.connect( new org.apache.maven.wagon.repository.Repository( remoteRepository.getId( ), baseIndexUrl ), authenticationInfo,
 328                                     proxyInfo );
 329
 330                             Path indexDirectory = indexingContext.getIndexDirectoryFile( ).toPath( );
 331                             if ( !Files.exists( indexDirectory ) )
 332                             {
 333                                 Files.createDirectories( indexDirectory );
 334                             }
 335
 336                             ResourceFetcher resourceFetcher =
 337                                     new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository );
 338                             IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
 339                             request.setForceFullUpdate( fullUpdate );
 340                             request.setLocalIndexCacheDir( indexCacheDirectory.toFile( ) );
 341
 342                             // indexUpdater.fetchAndUpdateIndex( request );
 343
 344                             indexingContext.updateTimestamp( true );
 345                         }
 346
 347                     }
 348                     catch ( AuthenticationException e )
 349                     {
 350                         log.error( "Could not login to the remote proxy for updating index of {}", remoteRepository.getId( ), e );
 351                         throw new IndexUpdateFailedException( "Login in to proxy failed while updating remote repository " + remoteRepository.getId( ), e );
 352                     }
 353                     catch ( ConnectionException e )
 354                     {
 355                         log.error( "Connection error during index update for remote repository {}", remoteRepository.getId( ), e );
 356                         throw new IndexUpdateFailedException( "Connection error during index update for remote repository " + remoteRepository.getId( ), e );
 357                     }
 358                     catch ( MalformedURLException e )
 359                     {
 360                         log.error( "URL for remote index update of remote repository {} is not correct {}", remoteRepository.getId( ), remoteUpdateUri, e );
 361                         throw new IndexUpdateFailedException( "URL for remote index update of repository is not correct " + remoteUpdateUri, e );
 362                     }
 363                     catch ( IOException e )
 364                     {
 365                         log.error( "IOException during index update of remote repository {}: {}", remoteRepository.getId( ), e.getMessage( ), e );
 366                         throw new IndexUpdateFailedException( "IOException during index update of remote repository " + remoteRepository.getId( )
 367                                 + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ), e );
 368                     }
 369                     catch ( WagonFactoryException e )
 370                     {
 371                         log.error( "Wagon for remote index download of {} could not be created: {}", remoteRepository.getId( ), e.getMessage( ), e );
 372                         throw new IndexUpdateFailedException( "Error while updating the remote index of " + remoteRepository.getId( ), e );
 373                     }
 374                 } );
 375
 376     }
 377
 378     @Override
 379     public void addArtifactsToIndex( final ArchivaIndexingContext context, final Collection<URI> artifactReference ) throws IndexUpdateFailedException
 380     {
 381         final StorageAsset ctxUri = context.getPath();
 382         executeUpdateFunction(context, indexingContext -> {
 383             Collection<ArtifactContext> artifacts = artifactReference.stream().map(r -> artifactContextProducer.getArtifactContext(indexingContext, Paths.get(ctxUri.getFilePath().toUri().resolve(r)).toFile())).collect(Collectors.toList());
 384             try {
 385                 indexer.addArtifactsToIndex(artifacts, indexingContext);
 386             } catch (IOException e) {
 387                 log.error("IOException while adding artifact {}", e.getMessage(), e);
 388                 throw new IndexUpdateFailedException("Error occured while adding artifact to index of "+context.getId()
 389                         + (StringUtils.isNotEmpty(e.getMessage()) ? ": "+e.getMessage() : ""));
 390             }
 391         });
 392     }
 393
 394     @Override
 395     public void removeArtifactsFromIndex( ArchivaIndexingContext context, Collection<URI> artifactReference ) throws IndexUpdateFailedException
 396     {
 397         final StorageAsset ctxUri = context.getPath();
 398         executeUpdateFunction(context, indexingContext -> {
 399             Collection<ArtifactContext> artifacts = artifactReference.stream().map(r -> artifactContextProducer.getArtifactContext(indexingContext, Paths.get(ctxUri.getFilePath().toUri().resolve(r)).toFile())).collect(Collectors.toList());
 400             try {
 401                 indexer.deleteArtifactsFromIndex(artifacts, indexingContext);
 402             } catch (IOException e) {
 403                 log.error("IOException while removing artifact {}", e.getMessage(), e);
 404                 throw new IndexUpdateFailedException("Error occured while removing artifact from index of "+context.getId()
 405                         + (StringUtils.isNotEmpty(e.getMessage()) ? ": "+e.getMessage() : ""));
 406             }
 407         });
 408
 409     }
 410
 411     @Override
 412     public boolean supportsRepository( RepositoryType type )
 413     {
 414         return type == RepositoryType.MAVEN;
 415     }
 416
 417     @Override
 418     public ArchivaIndexingContext createContext( Repository repository ) throws IndexCreationFailedException
 419     {
 420         log.debug("Creating context for repo {}, type: {}", repository.getId(), repository.getType());
 421         if ( repository.getType( ) != RepositoryType.MAVEN )
 422         {
 423             throw new UnsupportedRepositoryTypeException( repository.getType( ) );
 424         }
 425         IndexingContext mvnCtx = null;
 426         try
 427         {
 428             if ( repository instanceof RemoteRepository )
 429             {
 430                 mvnCtx = createRemoteContext( (RemoteRepository) repository );
 431             }
 432             else if ( repository instanceof ManagedRepository )
 433             {
 434                 mvnCtx = createManagedContext( (ManagedRepository) repository );
 435             }
 436         }
 437         catch ( IOException e )
 438         {
 439             log.error( "IOException during context creation " + e.getMessage( ), e );
 440             throw new IndexCreationFailedException( "Could not create index context for repository " + repository.getId( )
 441                     + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ), e );
 442         }
 443         MavenIndexContextMock context = new MavenIndexContextMock( repository, mvnCtx );
 444
 445         return context;
 446     }
 447
 448     @Override
 449     public ArchivaIndexingContext reset(ArchivaIndexingContext context) throws IndexUpdateFailedException {
 450         ArchivaIndexingContext ctx;
 451         executeUpdateFunction(context, indexingContext -> {
 452             try {
 453                 indexingContext.close(true);
 454             } catch (IOException e) {
 455                 log.warn("Index close failed");
 456             }
 457             org.apache.archiva.repository.storage.util.StorageUtil.deleteRecursively(context.getPath());
 458         });
 459         try {
 460             Repository repo = context.getRepository();
 461             ctx = createContext(context.getRepository());
 462             if (repo instanceof EditableRepository) {
 463                 ((EditableRepository)repo).setIndexingContext(ctx);
 464             }
 465         } catch (IndexCreationFailedException e) {
 466             throw new IndexUpdateFailedException("Could not create index");
 467         }
 468         return ctx;
 469     }
 470
 471     @Override
 472     public ArchivaIndexingContext move(ArchivaIndexingContext context, Repository repo) throws IndexCreationFailedException {
 473         if (context==null) {
 474             return null;
 475         }
 476         if (context.supports(IndexingContext.class)) {
 477             try {
 478                 StorageAsset newPath = getIndexPath(repo);
 479                 IndexingContext ctx = context.getBaseContext(IndexingContext.class);
 480                 Path oldPath = ctx.getIndexDirectoryFile().toPath();
 481                 if (oldPath.equals(newPath)) {
 482                     // Nothing to do, if path does not change
 483                     return context;
 484                 }
 485                 if (!Files.exists(oldPath)) {
 486                     return createContext(repo);
 487                 } else if (context.isEmpty()) {
 488                     context.close();
 489                     return createContext(repo);
 490                 } else {
 491                     context.close(false);
 492                     Files.move(oldPath, newPath.getFilePath());
 493                     return createContext(repo);
 494                 }
 495             } catch (IOException e) {
 496                 log.error("IOException while moving index directory {}", e.getMessage(), e);
 497                 throw new IndexCreationFailedException("Could not recreated the index.", e);
 498             } catch (UnsupportedBaseContextException e) {
 499                 throw new IndexCreationFailedException("The given context, is not a maven context.");
 500             }
 501         } else {
 502             throw new IndexCreationFailedException("Bad context type. This is not a maven context.");
 503         }
 504     }
 505
 506     @Override
 507     public void updateLocalIndexPath(Repository repo) {
 508         if (repo.supportsFeature(IndexCreationFeature.class)) {
 509             IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
 510             try {
 511                 icf.setLocalIndexPath(getIndexPath(repo));
 512             } catch (IOException e) {
 513                 log.error("Could not set local index path for {}. New URI: {}", repo.getId(), icf.getIndexPath());
 514             }
 515         }
 516     }
 517
 518     @Override
 519     public ArchivaIndexingContext mergeContexts(Repository destinationRepo, List<ArchivaIndexingContext> contexts, boolean packIndex) throws UnsupportedOperationException, IndexCreationFailedException {
 520         return null;
 521     }
 522
 523     private StorageAsset getIndexPath( Repository repo) throws IOException {
 524         IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
 525         Path repoDir = repo.getRoot().getFilePath();
 526         URI indexDir = icf.getIndexPath();
 527         String indexPath = indexDir.getPath();
 528         Path indexDirectory = null;
 529         FilesystemStorage filesystemStorage = (FilesystemStorage) repo.getRoot().getStorage();
 530         if ( ! StringUtils.isEmpty(indexDir.toString( ) ) )
 531         {
 532
 533             indexDirectory = PathUtil.getPathFromUri( indexDir );
 534             // not absolute so create it in repository directory
 535             if ( indexDirectory.isAbsolute( ) )
 536             {
 537                 indexPath = indexDirectory.getFileName().toString();
 538                 filesystemStorage = new FilesystemStorage(indexDirectory.getParent(), new DefaultFileLockManager());
 539             }
 540             else
 541             {
 542                 indexDirectory = repoDir.resolve( indexDirectory );
 543             }
 544         }
 545         else
 546         {
 547             indexDirectory = repoDir.resolve( ".index" );
 548             indexPath = ".index";
 549         }
 550
 551         if ( !Files.exists( indexDirectory ) )
 552         {
 553             Files.createDirectories( indexDirectory );
 554         }
 555         return new FilesystemAsset( filesystemStorage, indexPath, indexDirectory);
 556     }
 557
 558     private IndexingContext createRemoteContext(RemoteRepository remoteRepository ) throws IOException
 559     {
 560         Path appServerBase = archivaConfiguration.getAppServerBaseDir( );
 561
 562         String contextKey = "remote-" + remoteRepository.getId( );
 563
 564
 565         // create remote repository path
 566         Path repoDir = remoteRepository.getRoot().getFilePath();
 567         if ( !Files.exists( repoDir ) )
 568         {
 569             Files.createDirectories( repoDir );
 570         }
 571
 572         StorageAsset indexDirectory = null;
 573
 574         // is there configured indexDirectory ?
 575         if ( remoteRepository.supportsFeature( RemoteIndexFeature.class ) )
 576         {
 577             RemoteIndexFeature rif = remoteRepository.getFeature( RemoteIndexFeature.class ).get( );
 578             indexDirectory = getIndexPath(remoteRepository);
 579             String remoteIndexUrl = calculateIndexRemoteUrl( remoteRepository.getLocation( ), rif );
 580             try
 581             {
 582
 583                 return getIndexingContext( remoteRepository, contextKey, repoDir, indexDirectory, remoteIndexUrl );
 584             }
 585             catch ( IndexFormatTooOldException e )
 586             {
 587                 // existing index with an old lucene format so we need to delete it!!!
 588                 // delete it first then recreate it.
 589                 log.warn( "the index of repository {} is too old we have to delete and recreate it", //
 590                         remoteRepository.getId( ) );
 591                 org.apache.archiva.common.utils.FileUtils.deleteDirectory( indexDirectory.getFilePath() );
 592                 return getIndexingContext( remoteRepository, contextKey, repoDir, indexDirectory, remoteIndexUrl );
 593
 594             }
 595         }
 596         else
 597         {
 598             throw new IOException( "No remote index defined" );
 599         }
 600     }
 601
 602     private IndexingContext getIndexingContext( Repository repository, String contextKey, Path repoDir, StorageAsset indexDirectory, String indexUrl ) throws IOException
 603     {
 604         return indexer.createIndexingContext( contextKey, repository.getId( ), repoDir.toFile( ), indexDirectory.getFilePath().toFile( ),
 605                 repository.getLocation( ) == null ? null : repository.getLocation( ).toString( ),
 606                 indexUrl,
 607                 true, false,
 608                 indexCreators );
 609     }
 610
 611     private IndexingContext createManagedContext( ManagedRepository repository ) throws IOException
 612     {
 613
 614         IndexingContext context;
 615         // take care first about repository location as can be relative
 616         Path repositoryDirectory = repository.getRoot().getFilePath();
 617
 618         if ( !Files.exists( repositoryDirectory ) )
 619         {
 620             try
 621             {
 622                 Files.createDirectories( repositoryDirectory );
 623             }
 624             catch ( IOException e )
 625             {
 626                 log.error( "Could not create directory {}", repositoryDirectory );
 627             }
 628         }
 629
 630         StorageAsset indexDirectory = null;
 631
 632         if ( repository.supportsFeature( IndexCreationFeature.class ) )
 633         {
 634             indexDirectory = getIndexPath(repository);
 635
 636             String indexUrl = repositoryDirectory.toUri( ).toURL( ).toExternalForm( );
 637             try
 638             {
 639                 context = getIndexingContext( repository, repository.getId( ), repositoryDirectory, indexDirectory, indexUrl );
 640                 context.setSearchable( repository.isScanned( ) );
 641             }
 642             catch ( IndexFormatTooOldException e )
 643             {
 644                 // existing index with an old lucene format so we need to delete it!!!
 645                 // delete it first then recreate it.
 646                 log.warn( "the index of repository {} is too old we have to delete and recreate it", //
 647                         repository.getId( ) );
 648                 org.apache.archiva.common.utils.FileUtils.deleteDirectory( indexDirectory.getFilePath() );
 649                 context = getIndexingContext( repository, repository.getId( ), repositoryDirectory, indexDirectory, indexUrl );
 650                 context.setSearchable( repository.isScanned( ) );
 651             }
 652             return context;
 653         }
 654         else
 655         {
 656             throw new IOException( "No repository index defined" );
 657         }
 658     }
 659
 660     private String calculateIndexRemoteUrl( URI baseUri, RemoteIndexFeature rif )
 661     {
 662         if ( rif.getIndexUri( ) == null )
 663         {
 664             return baseUri.resolve( ".index" ).toString( );
 665         }
 666         else
 667         {
 668             return baseUri.resolve( rif.getIndexUri( ) ).toString( );
 669         }
 670     }
 671
 672     private static final class DownloadListener
 673             implements TransferListener
 674     {
 675         private Logger log = LoggerFactory.getLogger( getClass( ) );
 676
 677         private String resourceName;
 678
 679         private long startTime;
 680
 681         private int totalLength = 0;
 682
 683         @Override
 684         public void transferInitiated( TransferEvent transferEvent )
 685         {
 686             startTime = System.currentTimeMillis( );
 687             resourceName = transferEvent.getResource( ).getName( );
 688             log.debug( "initiate transfer of {}", resourceName );
 689         }
 690
 691         @Override
 692         public void transferStarted( TransferEvent transferEvent )
 693         {
 694             this.totalLength = 0;
 695             resourceName = transferEvent.getResource( ).getName( );
 696             log.info( "start transfer of {}", transferEvent.getResource( ).getName( ) );
 697         }
 698
 699         @Override
 700         public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
 701         {
 702             log.debug( "transfer of {} : {}/{}", transferEvent.getResource( ).getName( ), buffer.length, length );
 703             this.totalLength += length;
 704         }
 705
 706         @Override
 707         public void transferCompleted( TransferEvent transferEvent )
 708         {
 709             resourceName = transferEvent.getResource( ).getName( );
 710             long endTime = System.currentTimeMillis( );
 711             log.info( "end of transfer file {} {} kb: {}s", transferEvent.getResource( ).getName( ),
 712                     this.totalLength / 1024, ( endTime - startTime ) / 1000 );
 713         }
 714
 715         @Override
 716         public void transferError( TransferEvent transferEvent )
 717         {
 718             log.info( "error of transfer file {}: {}", transferEvent.getResource( ).getName( ),
 719                     transferEvent.getException( ).getMessage( ), transferEvent.getException( ) );
 720         }
 721
 722         @Override
 723         public void debug( String message )
 724         {
 725             log.debug( "transfer debug {}", message );
 726         }
 727     }
 728
 729     private static class WagonResourceFetcher
 730             implements ResourceFetcher
 731     {
 732
 733         Logger log;
 734
 735         Path tempIndexDirectory;
 736
 737         Wagon wagon;
 738
 739         RemoteRepository remoteRepository;
 740
 741         private WagonResourceFetcher( Logger log, Path tempIndexDirectory, Wagon wagon,
 742                                       RemoteRepository remoteRepository )
 743         {
 744             this.log = log;
 745             this.tempIndexDirectory = tempIndexDirectory;
 746             this.wagon = wagon;
 747             this.remoteRepository = remoteRepository;
 748         }
 749
 750         @Override
 751         public void connect( String id, String url )
 752                 throws IOException
 753         {
 754             //no op
 755         }
 756
 757         @Override
 758         public void disconnect( )
 759                 throws IOException
 760         {
 761             // no op
 762         }
 763
 764         @Override
 765         public InputStream retrieve(String name )
 766                 throws IOException, FileNotFoundException
 767         {
 768             try
 769             {
 770                 log.info( "index update retrieve file, name:{}", name );
 771                 Path file = tempIndexDirectory.resolve( name );
 772                 Files.deleteIfExists( file );
 773                 file.toFile( ).deleteOnExit( );
 774                 wagon.get( addParameters( name, remoteRepository ), file.toFile( ) );
 775                 return Files.newInputStream( file );
 776             }
 777             catch ( AuthorizationException | TransferFailedException e )
 778             {
 779                 throw new IOException( e.getMessage( ), e );
 780             }
 781             catch ( ResourceDoesNotExistException e )
 782             {
 783                 FileNotFoundException fnfe = new FileNotFoundException( e.getMessage( ) );
 784                 fnfe.initCause( e );
 785                 throw fnfe;
 786             }
 787         }
 788
 789         // FIXME remove crappy copy/paste
 790         protected String addParameters( String path, RemoteRepository remoteRepository )
 791         {
 792             if ( remoteRepository.getExtraParameters( ).isEmpty( ) )
 793             {
 794                 return path;
 795             }
 796
 797             boolean question = false;
 798
 799             StringBuilder res = new StringBuilder( path == null ? "" : path );
 800
 801             for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters( ).entrySet( ) )
 802             {
 803                 if ( !question )
 804                 {
 805                     res.append( '?' ).append( entry.getKey( ) ).append( '=' ).append( entry.getValue( ) );
 806                 }
 807             }
 808
 809             return res.toString( );
 810         }
 811
 812     }
 813 }