source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.scheduler.repository.mock;
   2
   3 /*
   4  * Licensed to the Apache Software Foundation (ASF) under one
   5  * or more contributor license agreements.  See the NOTICE file
   6  * distributed with this work for additional information
   7  * regarding copyright ownership.  The ASF licenses this file
   8  * to you under the Apache License, Version 2.0 (the
   9  * "License"); you may not use this file except in compliance
  10  * with the License.  You may obtain a copy of the License at
  11  *
  12  * http://www.apache.org/licenses/LICENSE-2.0
  13  * Unless required by applicable law or agreed to in writing,
  14  * software distributed under the License is distributed on an
  15  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  16  * KIND, either express or implied.  See the License for the
  17  * specific language governing permissions and limitations
  18  * under the License.
  19  */
  20
  21 import org.apache.archiva.common.filelock.DefaultFileLockManager;
  22 import org.apache.archiva.common.utils.FileUtils;
  23 import org.apache.archiva.common.utils.PathUtil;
  24 import org.apache.archiva.configuration.ArchivaConfiguration;
  25 import org.apache.archiva.indexer.ArchivaIndexManager;
  26 import org.apache.archiva.indexer.ArchivaIndexingContext;
  27 import org.apache.archiva.indexer.IndexCreationFailedException;
  28 import org.apache.archiva.indexer.IndexUpdateFailedException;
  29 import org.apache.archiva.indexer.UnsupportedBaseContextException;
  30 import org.apache.archiva.maven.common.proxy.WagonFactory;
  31 import org.apache.archiva.maven.common.proxy.WagonFactoryException;
  32 import org.apache.archiva.maven.common.proxy.WagonFactoryRequest;
  33 import org.apache.archiva.proxy.model.NetworkProxy;
  34 import org.apache.archiva.repository.EditableRepository;
  35 import org.apache.archiva.repository.ManagedRepository;
  36 import org.apache.archiva.repository.base.PasswordCredentials;
  37 import org.apache.archiva.repository.RemoteRepository;
  38 import org.apache.archiva.repository.Repository;
  39 import org.apache.archiva.repository.RepositoryType;
  40 import org.apache.archiva.repository.UnsupportedRepositoryTypeException;
  41 import org.apache.archiva.repository.storage.fs.FilesystemAsset;
  42 import org.apache.archiva.repository.storage.fs.FilesystemStorage;
  43 import org.apache.archiva.repository.storage.StorageAsset;
  44 import org.apache.archiva.repository.features.IndexCreationFeature;
  45 import org.apache.archiva.repository.features.RemoteIndexFeature;
  46 import org.apache.commons.lang3.StringUtils;
  47 import org.apache.maven.index.ArtifactContext;
  48 import org.apache.maven.index.ArtifactContextProducer;
  49 import org.apache.maven.index.DefaultScannerListener;
  50 import org.apache.maven.index.Indexer;
  51 import org.apache.maven.index.IndexerEngine;
  52 import org.apache.maven.index.Scanner;
  53 import org.apache.maven.index.ScanningRequest;
  54 import org.apache.maven.index.ScanningResult;
  55 import org.apache.maven.index.context.IndexCreator;
  56 import org.apache.maven.index.context.IndexingContext;
  57 import org.apache.maven.index.packer.IndexPacker;
  58 import org.apache.maven.index.packer.IndexPackingRequest;
  59 import org.apache.maven.index.updater.IndexUpdateRequest;
  60 import org.apache.maven.index.updater.ResourceFetcher;
  61 import org.apache.maven.index_shaded.lucene.index.IndexFormatTooOldException;
  62 import org.apache.maven.wagon.ConnectionException;
  63 import org.apache.maven.wagon.ResourceDoesNotExistException;
  64 import org.apache.maven.wagon.StreamWagon;
  65 import org.apache.maven.wagon.TransferFailedException;
  66 import org.apache.maven.wagon.Wagon;
  67 import org.apache.maven.wagon.authentication.AuthenticationException;
  68 import org.apache.maven.wagon.authentication.AuthenticationInfo;
  69 import org.apache.maven.wagon.authorization.AuthorizationException;
  70 import org.apache.maven.wagon.events.TransferEvent;
  71 import org.apache.maven.wagon.events.TransferListener;
  72 import org.apache.maven.wagon.proxy.ProxyInfo;
  73 import org.apache.maven.wagon.shared.http.AbstractHttpClientWagon;
  74 import org.apache.maven.wagon.shared.http.HttpConfiguration;
  75 import org.apache.maven.wagon.shared.http.HttpMethodConfiguration;
  76 import org.slf4j.Logger;
  77 import org.slf4j.LoggerFactory;
  78 import org.springframework.stereotype.Service;
  79
  80 import javax.inject.Inject;
  81 import java.io.FileNotFoundException;
  82 import java.io.IOException;
  83 import java.io.InputStream;
  84 import java.net.MalformedURLException;
  85 import java.net.URI;
  86 import java.nio.file.Files;
  87 import java.nio.file.Path;
  88 import java.nio.file.Paths;
  89 import java.util.Collection;
  90 import java.util.List;
  91 import java.util.Map;
  92 import java.util.concurrent.ConcurrentSkipListSet;
  93 import java.util.stream.Collectors;
  94
  95 @Service("archivaIndexManager#maven")
  96 public class ArchivaIndexManagerMock implements ArchivaIndexManager {
  97
  98     private static final Logger log = LoggerFactory.getLogger( ArchivaIndexManagerMock.class );
  99
 100     @Inject
 101     private Indexer indexer;
 102
 103     @Inject
 104     private IndexerEngine indexerEngine;
 105
 106     @Inject
 107     private List<? extends IndexCreator> indexCreators;
 108
 109     @Inject
 110     private IndexPacker indexPacker;
 111
 112     @Inject
 113     private Scanner scanner;
 114
 115     @Inject
 116     private ArchivaConfiguration archivaConfiguration;
 117
 118     @Inject
 119     private WagonFactory wagonFactory;
 120
 121
 122     @Inject
 123     private ArtifactContextProducer artifactContextProducer;
 124
 125     private ConcurrentSkipListSet<Path> activeContexts = new ConcurrentSkipListSet<>( );
 126
 127     private static final int WAIT_TIME = 100;
 128     private static final int MAX_WAIT = 10;
 129
 130
 131     public static IndexingContext getMvnContext(ArchivaIndexingContext context ) throws UnsupportedBaseContextException
 132     {
 133         if ( !context.supports( IndexingContext.class ) )
 134         {
 135             log.error( "The provided archiva index context does not support the maven IndexingContext" );
 136             throw new UnsupportedBaseContextException( "The context does not support the Maven IndexingContext" );
 137         }
 138         return context.getBaseContext( IndexingContext.class );
 139     }
 140
 141     private Path getIndexPath( ArchivaIndexingContext ctx )
 142     {
 143         return ctx.getPath().getFilePath();
 144     }
 145
 146     @FunctionalInterface
 147     interface IndexUpdateConsumer
 148     {
 149
 150         void accept( IndexingContext indexingContext ) throws IndexUpdateFailedException;
 151     }
 152
 153     /*
 154      * This method is used to do some actions around the update execution code. And to make sure, that no other
 155      * method is running on the same index.
 156      */
 157     private void executeUpdateFunction( ArchivaIndexingContext context, IndexUpdateConsumer function ) throws IndexUpdateFailedException
 158     {
 159         IndexingContext indexingContext = null;
 160         try
 161         {
 162             indexingContext = getMvnContext( context );
 163         }
 164         catch ( UnsupportedBaseContextException e )
 165         {
 166             throw new IndexUpdateFailedException( "Maven index is not supported by this context", e );
 167         }
 168         final Path ctxPath = getIndexPath( context );
 169         int loop = MAX_WAIT;
 170         boolean active = false;
 171         while ( loop-- > 0 && !active )
 172         {
 173             active = activeContexts.add( ctxPath );
 174             try
 175             {
 176                 Thread.currentThread( ).sleep( WAIT_TIME );
 177             }
 178             catch ( InterruptedException e )
 179             {
 180                 // Ignore this
 181             }
 182         }
 183         if ( active )
 184         {
 185             try
 186             {
 187                 function.accept( indexingContext );
 188             }
 189             finally
 190             {
 191                 activeContexts.remove( ctxPath );
 192             }
 193         }
 194         else
 195         {
 196             throw new IndexUpdateFailedException( "Timeout while waiting for index release on context " + context.getId( ) );
 197         }
 198     }
 199
 200     @Override
 201     public void pack( final ArchivaIndexingContext context ) throws IndexUpdateFailedException
 202     {
 203         executeUpdateFunction( context, indexingContext -> {
 204                     try
 205                     {
 206                         IndexPackingRequest request = new IndexPackingRequest( indexingContext,
 207                                 indexingContext.acquireIndexSearcher( ).getIndexReader( ),
 208                                 indexingContext.getIndexDirectoryFile( ) );
 209                         indexPacker.packIndex( request );
 210                         indexingContext.updateTimestamp( true );
 211                     }
 212                     catch ( IOException e )
 213                     {
 214                         log.error( "IOException while packing index of context " + context.getId( ) + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ) );
 215                         throw new IndexUpdateFailedException( "IOException during update of " + context.getId( ), e );
 216                     }
 217                 }
 218         );
 219
 220     }
 221
 222     @Override
 223     public void scan(final ArchivaIndexingContext context) throws IndexUpdateFailedException
 224     {
 225         executeUpdateFunction( context, indexingContext -> {
 226             DefaultScannerListener listener = new DefaultScannerListener( indexingContext, indexerEngine, true, null );
 227             ScanningRequest request = new ScanningRequest( indexingContext, listener );
 228             ScanningResult result = scanner.scan( request );
 229             if ( result.hasExceptions( ) )
 230             {
 231                 log.error( "Exceptions occured during index scan of " + context.getId( ) );
 232                 result.getExceptions( ).stream( ).map( e -> e.getMessage( ) ).distinct( ).limit( 5 ).forEach(
 233                         s -> log.error( "Message: " + s )
 234                 );
 235             }
 236
 237         } );
 238     }
 239
 240     @Override
 241     public void update(final ArchivaIndexingContext context, final boolean fullUpdate) throws IndexUpdateFailedException
 242     {
 243         log.info( "start download remote index for remote repository {}", context.getRepository( ).getId( ) );
 244         URI remoteUpdateUri;
 245         if ( !( context.getRepository( ) instanceof RemoteRepository) || !(context.getRepository().supportsFeature(RemoteIndexFeature.class)) )
 246         {
 247             throw new IndexUpdateFailedException( "The context is not associated to a remote repository with remote index " + context.getId( ) );
 248         } else {
 249             RemoteIndexFeature rif = context.getRepository().getFeature(RemoteIndexFeature.class).get();
 250             remoteUpdateUri = context.getRepository().getLocation().resolve(rif.getIndexUri());
 251         }
 252         final RemoteRepository remoteRepository = (RemoteRepository) context.getRepository( );
 253
 254         executeUpdateFunction( context,
 255                 indexingContext -> {
 256                     try
 257                     {
 258                         // create a temp directory to download files
 259                         Path tempIndexDirectory = Paths.get( indexingContext.getIndexDirectoryFile( ).getParent( ), ".tmpIndex" );
 260                         Path indexCacheDirectory = Paths.get( indexingContext.getIndexDirectoryFile( ).getParent( ), ".indexCache" );
 261                         Files.createDirectories( indexCacheDirectory );
 262                         if ( Files.exists( tempIndexDirectory ) )
 263                         {
 264                             FileUtils.deleteDirectory( tempIndexDirectory );
 265                         }
 266                         Files.createDirectories( tempIndexDirectory );
 267                         tempIndexDirectory.toFile( ).deleteOnExit( );
 268                         String baseIndexUrl = indexingContext.getIndexUpdateUrl( );
 269
 270                         String wagonProtocol = remoteUpdateUri.toURL( ).getProtocol( );
 271
 272                         NetworkProxy networkProxy = null;
 273                         if ( remoteRepository.supportsFeature( RemoteIndexFeature.class ) )
 274                         {
 275                             RemoteIndexFeature rif = remoteRepository.getFeature( RemoteIndexFeature.class ).get( );
 276
 277                             final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon(
 278                                     new WagonFactoryRequest( wagonProtocol, remoteRepository.getExtraHeaders( ) ).networkProxy(
 279                                             networkProxy )
 280                             );
 281                             int readTimeout = (int) rif.getDownloadTimeout( ).toMillis( ) * 1000;
 282                             wagon.setReadTimeout( readTimeout );
 283                             wagon.setTimeout( (int) remoteRepository.getTimeout( ).toMillis( ) * 1000 );
 284
 285                             if ( wagon instanceof AbstractHttpClientWagon)
 286                             {
 287                                 HttpConfiguration httpConfiguration = new HttpConfiguration( );
 288                                 HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration( );
 289                                 httpMethodConfiguration.setUsePreemptive( true );
 290                                 httpMethodConfiguration.setReadTimeout( readTimeout );
 291                                 httpConfiguration.setGet( httpMethodConfiguration );
 292                                 AbstractHttpClientWagon.class.cast( wagon ).setHttpConfiguration( httpConfiguration );
 293                             }
 294
 295                             wagon.addTransferListener( new DownloadListener( ) );
 296                             ProxyInfo proxyInfo = null;
 297                             if ( networkProxy != null )
 298                             {
 299                                 proxyInfo = new ProxyInfo( );
 300                                 proxyInfo.setType( networkProxy.getProtocol( ) );
 301                                 proxyInfo.setHost( networkProxy.getHost( ) );
 302                                 proxyInfo.setPort( networkProxy.getPort( ) );
 303                                 proxyInfo.setUserName( networkProxy.getUsername( ) );
 304                                 proxyInfo.setPassword(new String(networkProxy.getPassword()));
 305                             }
 306                             AuthenticationInfo authenticationInfo = null;
 307                             if ( remoteRepository.getLoginCredentials( ) != null && ( remoteRepository.getLoginCredentials( ) instanceof PasswordCredentials) )
 308                             {
 309                                 PasswordCredentials creds = (PasswordCredentials) remoteRepository.getLoginCredentials( );
 310                                 authenticationInfo = new AuthenticationInfo( );
 311                                 authenticationInfo.setUserName( creds.getUsername( ) );
 312                                 authenticationInfo.setPassword( new String( creds.getPassword( ) ) );
 313                             }
 314                             wagon.connect( new org.apache.maven.wagon.repository.Repository( remoteRepository.getId( ), baseIndexUrl ), authenticationInfo,
 315                                     proxyInfo );
 316
 317                             Path indexDirectory = indexingContext.getIndexDirectoryFile( ).toPath( );
 318                             if ( !Files.exists( indexDirectory ) )
 319                             {
 320                                 Files.createDirectories( indexDirectory );
 321                             }
 322
 323                             ResourceFetcher resourceFetcher =
 324                                     new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository );
 325                             IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
 326                             request.setForceFullUpdate( fullUpdate );
 327                             request.setLocalIndexCacheDir( indexCacheDirectory.toFile( ) );
 328
 329                             // indexUpdater.fetchAndUpdateIndex( request );
 330
 331                             indexingContext.updateTimestamp( true );
 332                         }
 333
 334                     }
 335                     catch ( AuthenticationException e )
 336                     {
 337                         log.error( "Could not login to the remote proxy for updating index of {}", remoteRepository.getId( ), e );
 338                         throw new IndexUpdateFailedException( "Login in to proxy failed while updating remote repository " + remoteRepository.getId( ), e );
 339                     }
 340                     catch ( ConnectionException e )
 341                     {
 342                         log.error( "Connection error during index update for remote repository {}", remoteRepository.getId( ), e );
 343                         throw new IndexUpdateFailedException( "Connection error during index update for remote repository " + remoteRepository.getId( ), e );
 344                     }
 345                     catch ( MalformedURLException e )
 346                     {
 347                         log.error( "URL for remote index update of remote repository {} is not correct {}", remoteRepository.getId( ), remoteUpdateUri, e );
 348                         throw new IndexUpdateFailedException( "URL for remote index update of repository is not correct " + remoteUpdateUri, e );
 349                     }
 350                     catch ( IOException e )
 351                     {
 352                         log.error( "IOException during index update of remote repository {}: {}", remoteRepository.getId( ), e.getMessage( ), e );
 353                         throw new IndexUpdateFailedException( "IOException during index update of remote repository " + remoteRepository.getId( )
 354                                 + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ), e );
 355                     }
 356                     catch ( WagonFactoryException e )
 357                     {
 358                         log.error( "Wagon for remote index download of {} could not be created: {}", remoteRepository.getId( ), e.getMessage( ), e );
 359                         throw new IndexUpdateFailedException( "Error while updating the remote index of " + remoteRepository.getId( ), e );
 360                     }
 361                 } );
 362
 363     }
 364
 365     @Override
 366     public void addArtifactsToIndex( final ArchivaIndexingContext context, final Collection<URI> artifactReference ) throws IndexUpdateFailedException
 367     {
 368         StorageAsset ctxUri = context.getPath();
 369         executeUpdateFunction(context, indexingContext -> {
 370             Collection<ArtifactContext> artifacts = artifactReference.stream().map(r -> artifactContextProducer.getArtifactContext(indexingContext, Paths.get(ctxUri.getFilePath().toUri().resolve(r)).toFile())).collect(Collectors.toList());
 371             try {
 372                 indexer.addArtifactsToIndex(artifacts, indexingContext);
 373             } catch (IOException e) {
 374                 log.error("IOException while adding artifact {}", e.getMessage(), e);
 375                 throw new IndexUpdateFailedException("Error occured while adding artifact to index of "+context.getId()
 376                         + (StringUtils.isNotEmpty(e.getMessage()) ? ": "+e.getMessage() : ""));
 377             }
 378         });
 379     }
 380
 381     @Override
 382     public void removeArtifactsFromIndex( ArchivaIndexingContext context, Collection<URI> artifactReference ) throws IndexUpdateFailedException
 383     {
 384         final StorageAsset ctxUri = context.getPath();
 385         executeUpdateFunction(context, indexingContext -> {
 386             Collection<ArtifactContext> artifacts = artifactReference.stream().map(r -> artifactContextProducer.getArtifactContext(indexingContext, Paths.get(ctxUri.getFilePath().toUri().resolve(r)).toFile())).collect(Collectors.toList());
 387             try {
 388                 indexer.deleteArtifactsFromIndex(artifacts, indexingContext);
 389             } catch (IOException e) {
 390                 log.error("IOException while removing artifact {}", e.getMessage(), e);
 391                 throw new IndexUpdateFailedException("Error occured while removing artifact from index of "+context.getId()
 392                         + (StringUtils.isNotEmpty(e.getMessage()) ? ": "+e.getMessage() : ""));
 393             }
 394         });
 395
 396     }
 397
 398     @Override
 399     public boolean supportsRepository( RepositoryType type )
 400     {
 401         return type == RepositoryType.MAVEN;
 402     }
 403
 404     @Override
 405     public ArchivaIndexingContext createContext( Repository repository ) throws IndexCreationFailedException
 406     {
 407         log.debug("Creating context for repo {}, type: {}", repository.getId(), repository.getType());
 408         if ( repository.getType( ) != RepositoryType.MAVEN )
 409         {
 410             throw new UnsupportedRepositoryTypeException( repository.getType( ) );
 411         }
 412         IndexingContext mvnCtx = null;
 413         try
 414         {
 415             if ( repository instanceof RemoteRepository )
 416             {
 417                 mvnCtx = createRemoteContext( (RemoteRepository) repository );
 418             }
 419             else if ( repository instanceof ManagedRepository )
 420             {
 421                 mvnCtx = createManagedContext( (ManagedRepository) repository );
 422             }
 423         }
 424         catch ( IOException e )
 425         {
 426             log.error( "IOException during context creation " + e.getMessage( ), e );
 427             throw new IndexCreationFailedException( "Could not create index context for repository " + repository.getId( )
 428                     + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ), e );
 429         }
 430         MavenIndexContextMock context = new MavenIndexContextMock( repository, mvnCtx );
 431
 432         return context;
 433     }
 434
 435     @Override
 436     public ArchivaIndexingContext reset(ArchivaIndexingContext context) throws IndexUpdateFailedException {
 437         ArchivaIndexingContext ctx;
 438         executeUpdateFunction(context, indexingContext -> {
 439             try {
 440                 indexingContext.close(true);
 441             } catch (IOException e) {
 442                 log.warn("Index close failed");
 443             }
 444             org.apache.archiva.repository.storage.util.StorageUtil.deleteRecursively(context.getPath());
 445         });
 446         try {
 447             Repository repo = context.getRepository();
 448             ctx = createContext(context.getRepository());
 449             if (repo instanceof EditableRepository) {
 450                 ((EditableRepository)repo).setIndexingContext(ctx);
 451             }
 452         } catch (IndexCreationFailedException e) {
 453             throw new IndexUpdateFailedException("Could not create index");
 454         }
 455         return ctx;
 456     }
 457
 458     @Override
 459     public ArchivaIndexingContext move(ArchivaIndexingContext context, Repository repo) throws IndexCreationFailedException {
 460         if (context==null) {
 461             return null;
 462         }
 463         if (context.supports(IndexingContext.class)) {
 464             try {
 465                 StorageAsset newPath = getIndexPath(repo);
 466                 IndexingContext ctx = context.getBaseContext(IndexingContext.class);
 467                 Path oldPath = ctx.getIndexDirectoryFile().toPath();
 468                 if (oldPath.equals(newPath)) {
 469                     // Nothing to do, if path does not change
 470                     return context;
 471                 }
 472                 if (!Files.exists(oldPath)) {
 473                     return createContext(repo);
 474                 } else if (context.isEmpty()) {
 475                     context.close();
 476                     return createContext(repo);
 477                 } else {
 478                     context.close(false);
 479                     Files.move(oldPath, newPath.getFilePath());
 480                     return createContext(repo);
 481                 }
 482             } catch (IOException e) {
 483                 log.error("IOException while moving index directory {}", e.getMessage(), e);
 484                 throw new IndexCreationFailedException("Could not recreated the index.", e);
 485             } catch (UnsupportedBaseContextException e) {
 486                 throw new IndexCreationFailedException("The given context, is not a maven context.");
 487             }
 488         } else {
 489             throw new IndexCreationFailedException("Bad context type. This is not a maven context.");
 490         }
 491     }
 492
 493     @Override
 494     public void updateLocalIndexPath(Repository repo) {
 495         if (repo.supportsFeature(IndexCreationFeature.class)) {
 496             IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
 497             try {
 498                 icf.setLocalIndexPath(getIndexPath(repo));
 499             } catch (IOException e) {
 500                 log.error("Could not set local index path for {}. New URI: {}", repo.getId(), icf.getIndexPath());
 501             }
 502         }
 503     }
 504
 505     @Override
 506     public ArchivaIndexingContext mergeContexts(Repository destinationRepo, List<ArchivaIndexingContext> contexts, boolean packIndex) throws UnsupportedOperationException, IndexCreationFailedException {
 507         return null;
 508     }
 509
 510     private StorageAsset getIndexPath( Repository repo) throws IOException {
 511         IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
 512         Path repoDir = repo.getRoot().getFilePath();
 513         URI indexDir = icf.getIndexPath();
 514         String indexPath = indexDir.getPath();
 515         Path indexDirectory = null;
 516         FilesystemStorage filesystemStorage = (FilesystemStorage) repo.getRoot().getStorage();
 517         if ( ! StringUtils.isEmpty(indexDir.toString( ) ) )
 518         {
 519
 520             indexDirectory = PathUtil.getPathFromUri( indexDir );
 521             // not absolute so create it in repository directory
 522             if ( indexDirectory.isAbsolute( ) )
 523             {
 524                 indexPath = indexDirectory.getFileName().toString();
 525                 filesystemStorage = new FilesystemStorage(indexDirectory, new DefaultFileLockManager());
 526             }
 527             else
 528             {
 529                 indexDirectory = repoDir.resolve( indexDirectory );
 530             }
 531         }
 532         else
 533         {
 534             indexDirectory = repoDir.resolve( ".index" );
 535             indexPath = ".index";
 536         }
 537
 538         if ( !Files.exists( indexDirectory ) )
 539         {
 540             Files.createDirectories( indexDirectory );
 541         }
 542         return new FilesystemAsset( filesystemStorage, indexPath, indexDirectory);
 543     }
 544
 545     private IndexingContext createRemoteContext(RemoteRepository remoteRepository ) throws IOException
 546     {
 547         Path appServerBase = archivaConfiguration.getAppServerBaseDir( );
 548
 549         String contextKey = "remote-" + remoteRepository.getId( );
 550
 551
 552         // create remote repository path
 553         Path repoDir = remoteRepository.getRoot().getFilePath();
 554         if ( !Files.exists( repoDir ) )
 555         {
 556             Files.createDirectories( repoDir );
 557         }
 558
 559         StorageAsset indexDirectory = null;
 560
 561         // is there configured indexDirectory ?
 562         if ( remoteRepository.supportsFeature( RemoteIndexFeature.class ) )
 563         {
 564             RemoteIndexFeature rif = remoteRepository.getFeature( RemoteIndexFeature.class ).get( );
 565             indexDirectory = getIndexPath(remoteRepository);
 566             String remoteIndexUrl = calculateIndexRemoteUrl( remoteRepository.getLocation( ), rif );
 567             try
 568             {
 569
 570                 return getIndexingContext( remoteRepository, contextKey, repoDir, indexDirectory, remoteIndexUrl );
 571             }
 572             catch ( IndexFormatTooOldException e )
 573             {
 574                 // existing index with an old lucene format so we need to delete it!!!
 575                 // delete it first then recreate it.
 576                 log.warn( "the index of repository {} is too old we have to delete and recreate it", //
 577                         remoteRepository.getId( ) );
 578                 FileUtils.deleteDirectory( indexDirectory.getFilePath() );
 579                 return getIndexingContext( remoteRepository, contextKey, repoDir, indexDirectory, remoteIndexUrl );
 580
 581             }
 582         }
 583         else
 584         {
 585             throw new IOException( "No remote index defined" );
 586         }
 587     }
 588
 589     private IndexingContext getIndexingContext( Repository repository, String contextKey, Path repoDir, StorageAsset indexDirectory, String indexUrl ) throws IOException
 590     {
 591         return indexer.createIndexingContext( contextKey, repository.getId( ), repoDir.toFile( ), indexDirectory.getFilePath().toFile( ),
 592                 repository.getLocation( ) == null ? null : repository.getLocation( ).toString( ),
 593                 indexUrl,
 594                 true, false,
 595                 indexCreators );
 596     }
 597
 598     private IndexingContext createManagedContext( ManagedRepository repository ) throws IOException
 599     {
 600
 601         IndexingContext context;
 602         // take care first about repository location as can be relative
 603         Path repositoryDirectory = repository.getRoot().getFilePath();
 604
 605         if ( !Files.exists( repositoryDirectory ) )
 606         {
 607             try
 608             {
 609                 Files.createDirectories( repositoryDirectory );
 610             }
 611             catch ( IOException e )
 612             {
 613                 log.error( "Could not create directory {}", repositoryDirectory );
 614             }
 615         }
 616
 617         StorageAsset indexDirectory = null;
 618
 619         if ( repository.supportsFeature( IndexCreationFeature.class ) )
 620         {
 621             indexDirectory = getIndexPath(repository);
 622
 623             String indexUrl = repositoryDirectory.toUri( ).toURL( ).toExternalForm( );
 624             try
 625             {
 626                 context = getIndexingContext( repository, repository.getId( ), repositoryDirectory, indexDirectory, indexUrl );
 627                 context.setSearchable( repository.isScanned( ) );
 628             }
 629             catch ( IndexFormatTooOldException e )
 630             {
 631                 // existing index with an old lucene format so we need to delete it!!!
 632                 // delete it first then recreate it.
 633                 log.warn( "the index of repository {} is too old we have to delete and recreate it", //
 634                         repository.getId( ) );
 635                 FileUtils.deleteDirectory( indexDirectory.getFilePath() );
 636                 context = getIndexingContext( repository, repository.getId( ), repositoryDirectory, indexDirectory, indexUrl );
 637                 context.setSearchable( repository.isScanned( ) );
 638             }
 639             return context;
 640         }
 641         else
 642         {
 643             throw new IOException( "No repository index defined" );
 644         }
 645     }
 646
 647     private String calculateIndexRemoteUrl( URI baseUri, RemoteIndexFeature rif )
 648     {
 649         if ( rif.getIndexUri( ) == null )
 650         {
 651             return baseUri.resolve( ".index" ).toString( );
 652         }
 653         else
 654         {
 655             return baseUri.resolve( rif.getIndexUri( ) ).toString( );
 656         }
 657     }
 658
 659     private static final class DownloadListener
 660             implements TransferListener
 661     {
 662         private Logger log = LoggerFactory.getLogger( getClass( ) );
 663
 664         private String resourceName;
 665
 666         private long startTime;
 667
 668         private int totalLength = 0;
 669
 670         @Override
 671         public void transferInitiated( TransferEvent transferEvent )
 672         {
 673             startTime = System.currentTimeMillis( );
 674             resourceName = transferEvent.getResource( ).getName( );
 675             log.debug( "initiate transfer of {}", resourceName );
 676         }
 677
 678         @Override
 679         public void transferStarted( TransferEvent transferEvent )
 680         {
 681             this.totalLength = 0;
 682             resourceName = transferEvent.getResource( ).getName( );
 683             log.info( "start transfer of {}", transferEvent.getResource( ).getName( ) );
 684         }
 685
 686         @Override
 687         public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
 688         {
 689             log.debug( "transfer of {} : {}/{}", transferEvent.getResource( ).getName( ), buffer.length, length );
 690             this.totalLength += length;
 691         }
 692
 693         @Override
 694         public void transferCompleted( TransferEvent transferEvent )
 695         {
 696             resourceName = transferEvent.getResource( ).getName( );
 697             long endTime = System.currentTimeMillis( );
 698             log.info( "end of transfer file {} {} kb: {}s", transferEvent.getResource( ).getName( ),
 699                     this.totalLength / 1024, ( endTime - startTime ) / 1000 );
 700         }
 701
 702         @Override
 703         public void transferError( TransferEvent transferEvent )
 704         {
 705             log.info( "error of transfer file {}: {}", transferEvent.getResource( ).getName( ),
 706                     transferEvent.getException( ).getMessage( ), transferEvent.getException( ) );
 707         }
 708
 709         @Override
 710         public void debug( String message )
 711         {
 712             log.debug( "transfer debug {}", message );
 713         }
 714     }
 715
 716     private static class WagonResourceFetcher
 717             implements ResourceFetcher
 718     {
 719
 720         Logger log;
 721
 722         Path tempIndexDirectory;
 723
 724         Wagon wagon;
 725
 726         RemoteRepository remoteRepository;
 727
 728         private WagonResourceFetcher( Logger log, Path tempIndexDirectory, Wagon wagon,
 729                                       RemoteRepository remoteRepository )
 730         {
 731             this.log = log;
 732             this.tempIndexDirectory = tempIndexDirectory;
 733             this.wagon = wagon;
 734             this.remoteRepository = remoteRepository;
 735         }
 736
 737         @Override
 738         public void connect( String id, String url )
 739                 throws IOException
 740         {
 741             //no op
 742         }
 743
 744         @Override
 745         public void disconnect( )
 746                 throws IOException
 747         {
 748             // no op
 749         }
 750
 751         @Override
 752         public InputStream retrieve(String name )
 753                 throws IOException, FileNotFoundException
 754         {
 755             try
 756             {
 757                 log.info( "index update retrieve file, name:{}", name );
 758                 Path file = tempIndexDirectory.resolve( name );
 759                 Files.deleteIfExists( file );
 760                 file.toFile( ).deleteOnExit( );
 761                 wagon.get( addParameters( name, remoteRepository ), file.toFile( ) );
 762                 return Files.newInputStream( file );
 763             }
 764             catch ( AuthorizationException | TransferFailedException e )
 765             {
 766                 throw new IOException( e.getMessage( ), e );
 767             }
 768             catch ( ResourceDoesNotExistException e )
 769             {
 770                 FileNotFoundException fnfe = new FileNotFoundException( e.getMessage( ) );
 771                 fnfe.initCause( e );
 772                 throw fnfe;
 773             }
 774         }
 775
 776         // FIXME remove crappy copy/paste
 777         protected String addParameters( String path, RemoteRepository remoteRepository )
 778         {
 779             if ( remoteRepository.getExtraParameters( ).isEmpty( ) )
 780             {
 781                 return path;
 782             }
 783
 784             boolean question = false;
 785
 786             StringBuilder res = new StringBuilder( path == null ? "" : path );
 787
 788             for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters( ).entrySet( ) )
 789             {
 790                 if ( !question )
 791                 {
 792                     res.append( '?' ).append( entry.getKey( ) ).append( '=' ).append( entry.getValue( ) );
 793                 }
 794             }
 795
 796             return res.toString( );
 797         }
 798
 799     }
 800 }