source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.indexer.maven;
   2
   3 /*
   4  * Licensed to the Apache Software Foundation (ASF) under one
   5  * or more contributor license agreements.  See the NOTICE file
   6  * distributed with this work for additional information
   7  * regarding copyright ownership.  The ASF licenses this file
   8  * to you under the Apache License, Version 2.0 (the
   9  * "License"); you may not use this file except in compliance
  10  * with the License.  You may obtain a copy of the License at
  11  *
  12  *  http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing,
  15  * software distributed under the License is distributed on an
  16  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  17  * KIND, either express or implied.  See the License for the
  18  * specific language governing permissions and limitations
  19  * under the License.
  20  */
  21
  22 import org.apache.archiva.admin.model.RepositoryAdminException;
  23 import org.apache.archiva.admin.model.beans.NetworkProxy;
  24 import org.apache.archiva.admin.model.networkproxy.NetworkProxyAdmin;
  25 import org.apache.archiva.common.utils.FileUtils;
  26 import org.apache.archiva.common.utils.PathUtil;
  27 import org.apache.archiva.configuration.ArchivaConfiguration;
  28 import org.apache.archiva.indexer.ArchivaIndexManager;
  29 import org.apache.archiva.indexer.ArchivaIndexingContext;
  30 import org.apache.archiva.indexer.IndexCreationFailedException;
  31 import org.apache.archiva.indexer.IndexUpdateFailedException;
  32 import org.apache.archiva.indexer.UnsupportedBaseContextException;
  33 import org.apache.archiva.proxy.common.WagonFactory;
  34 import org.apache.archiva.proxy.common.WagonFactoryException;
  35 import org.apache.archiva.proxy.common.WagonFactoryRequest;
  36 import org.apache.archiva.repository.*;
  37 import org.apache.archiva.repository.features.IndexCreationEvent;
  38 import org.apache.archiva.repository.features.IndexCreationFeature;
  39 import org.apache.archiva.repository.features.RemoteIndexFeature;
  40 import org.apache.commons.lang.StringUtils;
  41 import org.apache.maven.index.*;
  42 import org.apache.maven.index.context.IndexCreator;
  43 import org.apache.maven.index.context.IndexingContext;
  44 import org.apache.maven.index.packer.IndexPacker;
  45 import org.apache.maven.index.packer.IndexPackingRequest;
  46 import org.apache.maven.index.updater.IndexUpdateRequest;
  47 import org.apache.maven.index.updater.IndexUpdater;
  48 import org.apache.maven.index.updater.ResourceFetcher;
  49 import org.apache.maven.index_shaded.lucene.index.IndexFormatTooOldException;
  50 import org.apache.maven.wagon.ConnectionException;
  51 import org.apache.maven.wagon.ResourceDoesNotExistException;
  52 import org.apache.maven.wagon.StreamWagon;
  53 import org.apache.maven.wagon.TransferFailedException;
  54 import org.apache.maven.wagon.Wagon;
  55 import org.apache.maven.wagon.authentication.AuthenticationException;
  56 import org.apache.maven.wagon.authentication.AuthenticationInfo;
  57 import org.apache.maven.wagon.authorization.AuthorizationException;
  58 import org.apache.maven.wagon.events.TransferEvent;
  59 import org.apache.maven.wagon.events.TransferListener;
  60 import org.apache.maven.wagon.proxy.ProxyInfo;
  61 import org.apache.maven.wagon.shared.http.AbstractHttpClientWagon;
  62 import org.apache.maven.wagon.shared.http.HttpConfiguration;
  63 import org.apache.maven.wagon.shared.http.HttpMethodConfiguration;
  64 import org.slf4j.Logger;
  65 import org.slf4j.LoggerFactory;
  66 import org.springframework.stereotype.Service;
  67
  68 import javax.annotation.PostConstruct;
  69 import javax.inject.Inject;
  70 import java.io.FileNotFoundException;
  71 import java.io.IOException;
  72 import java.io.InputStream;
  73 import java.net.MalformedURLException;
  74 import java.net.URI;
  75 import java.nio.file.Files;
  76 import java.nio.file.Path;
  77 import java.nio.file.Paths;
  78 import java.util.Collection;
  79 import java.util.List;
  80 import java.util.Map;
  81 import java.util.concurrent.ConcurrentSkipListSet;
  82 import java.util.stream.Collectors;
  83
  84 /**
  85  * Maven implementation of index manager.
  86  * The index manager is a singleton, so we try to make sure, that index operations are not running
  87  * parallel by synchronizing on the index path.
  88  * A update operation waits for parallel running methods to finish before starting, but after a certain
  89  * time of retries a IndexUpdateFailedException is thrown.
  90  */
  91 @Service( "archivaIndexManager#maven" )
  92 public class MavenIndexManager implements ArchivaIndexManager {
  93
  94     private static final Logger log = LoggerFactory.getLogger( MavenIndexManager.class );
  95
  96     @Inject
  97     private Indexer indexer;
  98
  99     @Inject
 100     private IndexerEngine indexerEngine;
 101
 102     @Inject
 103     private List<? extends IndexCreator> indexCreators;
 104
 105     @Inject
 106     private IndexPacker indexPacker;
 107
 108     @Inject
 109     private Scanner scanner;
 110
 111     @Inject
 112     private ArchivaConfiguration archivaConfiguration;
 113
 114     @Inject
 115     private WagonFactory wagonFactory;
 116
 117     @Inject
 118     private NetworkProxyAdmin networkProxyAdmin;
 119
 120     @Inject
 121     private IndexUpdater indexUpdater;
 122
 123     @Inject
 124     private ArtifactContextProducer artifactContextProducer;
 125
 126     @Inject
 127     RepositoryRegistry repositoryRegistry;
 128
 129     public static final String DEFAULT_INDEXER_DIR = ".indexer";
 130
 131     private ConcurrentSkipListSet<Path> activeContexts = new ConcurrentSkipListSet<>( );
 132
 133     private static final int WAIT_TIME = 100;
 134     private static final int MAX_WAIT = 10;
 135
 136
 137     public static IndexingContext getMvnContext( ArchivaIndexingContext context ) throws UnsupportedBaseContextException
 138     {
 139         if ( !context.supports( IndexingContext.class ) )
 140         {
 141             log.error( "The provided archiva index context does not support the maven IndexingContext" );
 142             throw new UnsupportedBaseContextException( "The context does not support the Maven IndexingContext" );
 143         }
 144         return context.getBaseContext( IndexingContext.class );
 145     }
 146
 147     private Path getIndexPath( ArchivaIndexingContext ctx )
 148     {
 149         return PathUtil.getPathFromUri( ctx.getPath( ) );
 150     }
 151
 152     @FunctionalInterface
 153     interface IndexUpdateConsumer
 154     {
 155
 156         void accept( IndexingContext indexingContext ) throws IndexUpdateFailedException;
 157     }
 158
 159     /*
 160      * This method is used to do some actions around the update execution code. And to make sure, that no other
 161      * method is running on the same index.
 162      */
 163     private void executeUpdateFunction( ArchivaIndexingContext context, IndexUpdateConsumer function ) throws IndexUpdateFailedException
 164     {
 165         IndexingContext indexingContext = null;
 166         try
 167         {
 168             indexingContext = getMvnContext( context );
 169         }
 170         catch ( UnsupportedBaseContextException e )
 171         {
 172             throw new IndexUpdateFailedException( "Maven index is not supported by this context", e );
 173         }
 174         final Path ctxPath = getIndexPath( context );
 175         int loop = MAX_WAIT;
 176         boolean active = false;
 177         while ( loop-- > 0 && !active )
 178         {
 179             active = activeContexts.add( ctxPath );
 180             try
 181             {
 182                 Thread.currentThread( ).sleep( WAIT_TIME );
 183             }
 184             catch ( InterruptedException e )
 185             {
 186                 // Ignore this
 187             }
 188         }
 189         if ( active )
 190         {
 191             try
 192             {
 193                 function.accept( indexingContext );
 194             }
 195             finally
 196             {
 197                 activeContexts.remove( ctxPath );
 198             }
 199         }
 200         else
 201         {
 202             throw new IndexUpdateFailedException( "Timeout while waiting for index release on context " + context.getId( ) );
 203         }
 204     }
 205
 206     @Override
 207     public void pack( final ArchivaIndexingContext context ) throws IndexUpdateFailedException
 208     {
 209         executeUpdateFunction( context, indexingContext -> {
 210                 try
 211                 {
 212                     IndexPackingRequest request = new IndexPackingRequest( indexingContext,
 213                         indexingContext.acquireIndexSearcher( ).getIndexReader( ),
 214                         indexingContext.getIndexDirectoryFile( ) );
 215                     indexPacker.packIndex( request );
 216                     indexingContext.updateTimestamp( true );
 217                 }
 218                 catch ( IOException e )
 219                 {
 220                     log.error( "IOException while packing index of context " + context.getId( ) + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ) );
 221                     throw new IndexUpdateFailedException( "IOException during update of " + context.getId( ), e );
 222                 }
 223             }
 224         );
 225
 226     }
 227
 228     @Override
 229     public void scan(final ArchivaIndexingContext context) throws IndexUpdateFailedException
 230     {
 231         executeUpdateFunction( context, indexingContext -> {
 232             DefaultScannerListener listener = new DefaultScannerListener( indexingContext, indexerEngine, true, null );
 233             ScanningRequest request = new ScanningRequest( indexingContext, listener );
 234             ScanningResult result = scanner.scan( request );
 235             if ( result.hasExceptions( ) )
 236             {
 237                 log.error( "Exceptions occured during index scan of " + context.getId( ) );
 238                 result.getExceptions( ).stream( ).map( e -> e.getMessage( ) ).distinct( ).limit( 5 ).forEach(
 239                     s -> log.error( "Message: " + s )
 240                 );
 241             }
 242
 243         } );
 244     }
 245
 246     @Override
 247     public void update(final ArchivaIndexingContext context, final boolean fullUpdate) throws IndexUpdateFailedException
 248     {
 249         log.info( "start download remote index for remote repository {}", context.getRepository( ).getId( ) );
 250         URI remoteUpdateUri;
 251         if ( !( context.getRepository( ) instanceof RemoteRepository ) || !(context.getRepository().supportsFeature(RemoteIndexFeature.class)) )
 252         {
 253             throw new IndexUpdateFailedException( "The context is not associated to a remote repository with remote index " + context.getId( ) );
 254         } else {
 255             RemoteIndexFeature rif = context.getRepository().getFeature(RemoteIndexFeature.class).get();
 256             remoteUpdateUri = context.getRepository().getLocation().resolve(rif.getIndexUri());
 257         }
 258         final RemoteRepository remoteRepository = (RemoteRepository) context.getRepository( );
 259
 260         executeUpdateFunction( context,
 261             indexingContext -> {
 262                 try
 263                 {
 264                     // create a temp directory to download files
 265                     Path tempIndexDirectory = Paths.get( indexingContext.getIndexDirectoryFile( ).getParent( ), ".tmpIndex" );
 266                     Path indexCacheDirectory = Paths.get( indexingContext.getIndexDirectoryFile( ).getParent( ), ".indexCache" );
 267                     Files.createDirectories( indexCacheDirectory );
 268                     if ( Files.exists( tempIndexDirectory ) )
 269                     {
 270                         org.apache.archiva.common.utils.FileUtils.deleteDirectory( tempIndexDirectory );
 271                     }
 272                     Files.createDirectories( tempIndexDirectory );
 273                     tempIndexDirectory.toFile( ).deleteOnExit( );
 274                     String baseIndexUrl = indexingContext.getIndexUpdateUrl( );
 275
 276                     String wagonProtocol = remoteUpdateUri.toURL( ).getProtocol( );
 277
 278                     NetworkProxy networkProxy = null;
 279                     if ( remoteRepository.supportsFeature( RemoteIndexFeature.class ) )
 280                     {
 281                         RemoteIndexFeature rif = remoteRepository.getFeature( RemoteIndexFeature.class ).get( );
 282                         if ( StringUtils.isNotBlank( rif.getProxyId( ) ) )
 283                         {
 284                             try
 285                             {
 286                                 networkProxy = networkProxyAdmin.getNetworkProxy( rif.getProxyId( ) );
 287                             }
 288                             catch ( RepositoryAdminException e )
 289                             {
 290                                 log.error( "Error occured while retrieving proxy {}", e.getMessage( ) );
 291                             }
 292                             if ( networkProxy == null )
 293                             {
 294                                 log.warn(
 295                                     "your remote repository is configured to download remote index trought a proxy we cannot find id:{}",
 296                                     rif.getProxyId( ) );
 297                             }
 298                         }
 299
 300                         final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon(
 301                             new WagonFactoryRequest( wagonProtocol, remoteRepository.getExtraHeaders( ) ).networkProxy(
 302                                 networkProxy )
 303                         );
 304                         int readTimeout = (int) rif.getDownloadTimeout( ).toMillis( ) * 1000;
 305                         wagon.setReadTimeout( readTimeout );
 306                         wagon.setTimeout( (int) remoteRepository.getTimeout( ).toMillis( ) * 1000 );
 307
 308                         if ( wagon instanceof AbstractHttpClientWagon )
 309                         {
 310                             HttpConfiguration httpConfiguration = new HttpConfiguration( );
 311                             HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration( );
 312                             httpMethodConfiguration.setUsePreemptive( true );
 313                             httpMethodConfiguration.setReadTimeout( readTimeout );
 314                             httpConfiguration.setGet( httpMethodConfiguration );
 315                             AbstractHttpClientWagon.class.cast( wagon ).setHttpConfiguration( httpConfiguration );
 316                         }
 317
 318                         wagon.addTransferListener( new DownloadListener( ) );
 319                         ProxyInfo proxyInfo = null;
 320                         if ( networkProxy != null )
 321                         {
 322                             proxyInfo = new ProxyInfo( );
 323                             proxyInfo.setType( networkProxy.getProtocol( ) );
 324                             proxyInfo.setHost( networkProxy.getHost( ) );
 325                             proxyInfo.setPort( networkProxy.getPort( ) );
 326                             proxyInfo.setUserName( networkProxy.getUsername( ) );
 327                             proxyInfo.setPassword( networkProxy.getPassword( ) );
 328                         }
 329                         AuthenticationInfo authenticationInfo = null;
 330                         if ( remoteRepository.getLoginCredentials( ) != null && ( remoteRepository.getLoginCredentials( ) instanceof PasswordCredentials ) )
 331                         {
 332                             PasswordCredentials creds = (PasswordCredentials) remoteRepository.getLoginCredentials( );
 333                             authenticationInfo = new AuthenticationInfo( );
 334                             authenticationInfo.setUserName( creds.getUsername( ) );
 335                             authenticationInfo.setPassword( new String( creds.getPassword( ) ) );
 336                         }
 337                         wagon.connect( new org.apache.maven.wagon.repository.Repository( remoteRepository.getId( ), baseIndexUrl ), authenticationInfo,
 338                             proxyInfo );
 339
 340                         Path indexDirectory = indexingContext.getIndexDirectoryFile( ).toPath( );
 341                         if ( !Files.exists( indexDirectory ) )
 342                         {
 343                             Files.createDirectories( indexDirectory );
 344                         }
 345
 346                         ResourceFetcher resourceFetcher =
 347                             new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository );
 348                         IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
 349                         request.setForceFullUpdate( fullUpdate );
 350                         request.setLocalIndexCacheDir( indexCacheDirectory.toFile( ) );
 351
 352                         indexUpdater.fetchAndUpdateIndex( request );
 353
 354                         indexingContext.updateTimestamp( true );
 355                     }
 356
 357                 }
 358                 catch ( AuthenticationException e )
 359                 {
 360                     log.error( "Could not login to the remote proxy for updating index of {}", remoteRepository.getId( ), e );
 361                     throw new IndexUpdateFailedException( "Login in to proxy failed while updating remote repository " + remoteRepository.getId( ), e );
 362                 }
 363                 catch ( ConnectionException e )
 364                 {
 365                     log.error( "Connection error during index update for remote repository {}", remoteRepository.getId( ), e );
 366                     throw new IndexUpdateFailedException( "Connection error during index update for remote repository " + remoteRepository.getId( ), e );
 367                 }
 368                 catch ( MalformedURLException e )
 369                 {
 370                     log.error( "URL for remote index update of remote repository {} is not correct {}", remoteRepository.getId( ), remoteUpdateUri, e );
 371                     throw new IndexUpdateFailedException( "URL for remote index update of repository is not correct " + remoteUpdateUri, e );
 372                 }
 373                 catch ( IOException e )
 374                 {
 375                     log.error( "IOException during index update of remote repository {}: {}", remoteRepository.getId( ), e.getMessage( ), e );
 376                     throw new IndexUpdateFailedException( "IOException during index update of remote repository " + remoteRepository.getId( )
 377                         + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ), e );
 378                 }
 379                 catch ( WagonFactoryException e )
 380                 {
 381                     log.error( "Wagon for remote index download of {} could not be created: {}", remoteRepository.getId( ), e.getMessage( ), e );
 382                     throw new IndexUpdateFailedException( "Error while updating the remote index of " + remoteRepository.getId( ), e );
 383                 }
 384             } );
 385
 386     }
 387
 388     @Override
 389     public void addArtifactsToIndex( final ArchivaIndexingContext context, final Collection<URI> artifactReference ) throws IndexUpdateFailedException
 390     {
 391         final URI ctxUri = context.getPath();
 392         executeUpdateFunction(context, indexingContext -> {
 393             Collection<ArtifactContext> artifacts = artifactReference.stream().map(r -> artifactContextProducer.getArtifactContext(indexingContext, Paths.get(ctxUri.resolve(r)).toFile())).collect(Collectors.toList());
 394             try {
 395                 indexer.addArtifactsToIndex(artifacts, indexingContext);
 396             } catch (IOException e) {
 397                 log.error("IOException while adding artifact {}", e.getMessage(), e);
 398                 throw new IndexUpdateFailedException("Error occured while adding artifact to index of "+context.getId()
 399                 + (StringUtils.isNotEmpty(e.getMessage()) ? ": "+e.getMessage() : ""));
 400             }
 401         });
 402     }
 403
 404     @Override
 405     public void removeArtifactsFromIndex( ArchivaIndexingContext context, Collection<URI> artifactReference ) throws IndexUpdateFailedException
 406     {
 407         final URI ctxUri = context.getPath();
 408         executeUpdateFunction(context, indexingContext -> {
 409             Collection<ArtifactContext> artifacts = artifactReference.stream().map(r -> artifactContextProducer.getArtifactContext(indexingContext, Paths.get(ctxUri.resolve(r)).toFile())).collect(Collectors.toList());
 410             try {
 411                 indexer.deleteArtifactsFromIndex(artifacts, indexingContext);
 412             } catch (IOException e) {
 413                 log.error("IOException while removing artifact {}", e.getMessage(), e);
 414                 throw new IndexUpdateFailedException("Error occured while removing artifact from index of "+context.getId()
 415                         + (StringUtils.isNotEmpty(e.getMessage()) ? ": "+e.getMessage() : ""));
 416             }
 417         });
 418
 419     }
 420
 421     @Override
 422     public boolean supportsRepository( RepositoryType type )
 423     {
 424         return type == RepositoryType.MAVEN;
 425     }
 426
 427     @Override
 428     public ArchivaIndexingContext createContext( Repository repository ) throws IndexCreationFailedException
 429     {
 430         log.debug("Creating context for repo {}, type: {}", repository.getId(), repository.getType());
 431         if ( repository.getType( ) != RepositoryType.MAVEN )
 432         {
 433             throw new UnsupportedRepositoryTypeException( repository.getType( ) );
 434         }
 435         IndexingContext mvnCtx = null;
 436         try
 437         {
 438             if ( repository instanceof RemoteRepository )
 439             {
 440                 mvnCtx = createRemoteContext( (RemoteRepository) repository );
 441             }
 442             else if ( repository instanceof ManagedRepository )
 443             {
 444                 mvnCtx = createManagedContext( (ManagedRepository) repository );
 445             }
 446         }
 447         catch ( IOException e )
 448         {
 449             log.error( "IOException during context creation " + e.getMessage( ), e );
 450             throw new IndexCreationFailedException( "Could not create index context for repository " + repository.getId( )
 451                 + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ), e );
 452         }
 453         MavenIndexContext context = new MavenIndexContext( repository, mvnCtx );
 454
 455         return context;
 456     }
 457
 458     @Override
 459     public ArchivaIndexingContext reset(ArchivaIndexingContext context) throws IndexUpdateFailedException {
 460         ArchivaIndexingContext ctx;
 461         executeUpdateFunction(context, indexingContext -> {
 462             try {
 463                 indexingContext.close(true);
 464             } catch (IOException e) {
 465                 log.warn("Index close failed");
 466             }
 467             try {
 468                 FileUtils.deleteDirectory(Paths.get(context.getPath()));
 469             } catch (IOException e) {
 470                 throw new IndexUpdateFailedException("Could not delete index files");
 471             }
 472         });
 473         try {
 474             Repository repo = context.getRepository();
 475             ctx = createContext(context.getRepository());
 476             if (repo instanceof EditableRepository) {
 477                 ((EditableRepository)repo).setIndexingContext(ctx);
 478             }
 479         } catch (IndexCreationFailedException e) {
 480             throw new IndexUpdateFailedException("Could not create index");
 481         }
 482         return ctx;
 483     }
 484
 485     @Override
 486     public ArchivaIndexingContext move(ArchivaIndexingContext context, Repository repo) throws IndexCreationFailedException {
 487         if (context==null) {
 488             return null;
 489         }
 490         if (context.supports(IndexingContext.class)) {
 491             try {
 492                 Path newPath = getIndexPath(repo);
 493                 IndexingContext ctx = context.getBaseContext(IndexingContext.class);
 494                 Path oldPath = ctx.getIndexDirectoryFile().toPath();
 495                 if (oldPath.equals(newPath)) {
 496                     // Nothing to do, if path does not change
 497                     return context;
 498                 }
 499                 if (!Files.exists(oldPath)) {
 500                     return createContext(repo);
 501                 } else if (context.isEmpty()) {
 502                     context.close();
 503                     return createContext(repo);
 504                 } else {
 505                     context.close(false);
 506                     Files.move(oldPath, newPath);
 507                     return createContext(repo);
 508                 }
 509             } catch (IOException e) {
 510                 log.error("IOException while moving index directory {}", e.getMessage(), e);
 511                 throw new IndexCreationFailedException("Could not recreated the index.", e);
 512             } catch (UnsupportedBaseContextException e) {
 513                 throw new IndexCreationFailedException("The given context, is not a maven context.");
 514             }
 515         } else {
 516             throw new IndexCreationFailedException("Bad context type. This is not a maven context.");
 517         }
 518     }
 519
 520     @Override
 521     public void updateLocalIndexPath(Repository repo) {
 522         if (repo.supportsFeature(IndexCreationFeature.class)) {
 523             IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
 524             try {
 525                 icf.setLocalIndexPath(getIndexPath(repo));
 526             } catch (IOException e) {
 527                 log.error("Could not set local index path for {}. New URI: {}", repo.getId(), icf.getIndexPath());
 528             }
 529         }
 530     }
 531
 532     private Path getIndexPath(Repository repo) throws IOException {
 533         IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
 534         Path repoDir = repo.getLocalPath();
 535         URI indexDir = icf.getIndexPath();
 536         Path indexDirectory = null;
 537         if ( ! StringUtils.isEmpty(indexDir.toString( ) ) )
 538         {
 539
 540             indexDirectory = PathUtil.getPathFromUri( indexDir );
 541             // not absolute so create it in repository directory
 542             if ( !indexDirectory.isAbsolute( ) )
 543             {
 544                 indexDirectory = repoDir.resolve( indexDirectory );
 545             }
 546         }
 547         else
 548         {
 549             indexDirectory = repoDir.resolve( DEFAULT_INDEXER_DIR );
 550         }
 551
 552         if ( !Files.exists( indexDirectory ) )
 553         {
 554             Files.createDirectories( indexDirectory );
 555         }
 556         return indexDirectory;
 557     }
 558
 559     private IndexingContext createRemoteContext(RemoteRepository remoteRepository ) throws IOException
 560     {
 561         Path appServerBase = archivaConfiguration.getAppServerBaseDir( );
 562
 563         String contextKey = "remote-" + remoteRepository.getId( );
 564
 565
 566         // create remote repository path
 567         Path repoDir = remoteRepository.getLocalPath();
 568         if ( !Files.exists( repoDir ) )
 569         {
 570             Files.createDirectories( repoDir );
 571         }
 572
 573         Path indexDirectory = null;
 574
 575         // is there configured indexDirectory ?
 576         if ( remoteRepository.supportsFeature( RemoteIndexFeature.class ) )
 577         {
 578             RemoteIndexFeature rif = remoteRepository.getFeature( RemoteIndexFeature.class ).get( );
 579             indexDirectory = getIndexPath(remoteRepository);
 580             String remoteIndexUrl = calculateIndexRemoteUrl( remoteRepository.getLocation( ), rif );
 581             try
 582             {
 583
 584                 return getIndexingContext( remoteRepository, contextKey, repoDir, indexDirectory, remoteIndexUrl );
 585             }
 586             catch ( IndexFormatTooOldException e )
 587             {
 588                 // existing index with an old lucene format so we need to delete it!!!
 589                 // delete it first then recreate it.
 590                 log.warn( "the index of repository {} is too old we have to delete and recreate it", //
 591                     remoteRepository.getId( ) );
 592                 org.apache.archiva.common.utils.FileUtils.deleteDirectory( indexDirectory );
 593                 return getIndexingContext( remoteRepository, contextKey, repoDir, indexDirectory, remoteIndexUrl );
 594
 595             }
 596         }
 597         else
 598         {
 599             throw new IOException( "No remote index defined" );
 600         }
 601     }
 602
 603     private IndexingContext getIndexingContext( Repository repository, String contextKey, Path repoDir, Path indexDirectory, String indexUrl ) throws IOException
 604     {
 605         return indexer.createIndexingContext( contextKey, repository.getId( ), repoDir.toFile( ), indexDirectory.toFile( ),
 606             repository.getLocation( ) == null ? null : repository.getLocation( ).toString( ),
 607             indexUrl,
 608             true, false,
 609             indexCreators );
 610     }
 611
 612     private IndexingContext createManagedContext( ManagedRepository repository ) throws IOException
 613     {
 614
 615         IndexingContext context;
 616         // take care first about repository location as can be relative
 617         Path repositoryDirectory = repository.getLocalPath();
 618
 619         if ( !Files.exists( repositoryDirectory ) )
 620         {
 621             try
 622             {
 623                 Files.createDirectories( repositoryDirectory );
 624             }
 625             catch ( IOException e )
 626             {
 627                 log.error( "Could not create directory {}", repositoryDirectory );
 628             }
 629         }
 630
 631         Path indexDirectory = null;
 632
 633         if ( repository.supportsFeature( IndexCreationFeature.class ) )
 634         {
 635             indexDirectory = getIndexPath(repository);
 636
 637             String indexUrl = repositoryDirectory.toUri( ).toURL( ).toExternalForm( );
 638             try
 639             {
 640                 context = getIndexingContext( repository, repository.getId( ), repositoryDirectory, indexDirectory, indexUrl );
 641                 context.setSearchable( repository.isScanned( ) );
 642             }
 643             catch ( IndexFormatTooOldException e )
 644             {
 645                 // existing index with an old lucene format so we need to delete it!!!
 646                 // delete it first then recreate it.
 647                 log.warn( "the index of repository {} is too old we have to delete and recreate it", //
 648                     repository.getId( ) );
 649                 org.apache.archiva.common.utils.FileUtils.deleteDirectory( indexDirectory );
 650                 context = getIndexingContext( repository, repository.getId( ), repositoryDirectory, indexDirectory, indexUrl );
 651                 context.setSearchable( repository.isScanned( ) );
 652             }
 653             return context;
 654         }
 655         else
 656         {
 657             throw new IOException( "No repository index defined" );
 658         }
 659     }
 660
 661     private String calculateIndexRemoteUrl( URI baseUri, RemoteIndexFeature rif )
 662     {
 663         if ( rif.getIndexUri( ) == null )
 664         {
 665             return baseUri.resolve( DEFAULT_INDEXER_DIR ).toString( );
 666         }
 667         else
 668         {
 669             return baseUri.resolve( rif.getIndexUri( ) ).toString( );
 670         }
 671     }
 672
 673     private static final class DownloadListener
 674         implements TransferListener
 675     {
 676         private Logger log = LoggerFactory.getLogger( getClass( ) );
 677
 678         private String resourceName;
 679
 680         private long startTime;
 681
 682         private int totalLength = 0;
 683
 684         @Override
 685         public void transferInitiated( TransferEvent transferEvent )
 686         {
 687             startTime = System.currentTimeMillis( );
 688             resourceName = transferEvent.getResource( ).getName( );
 689             log.debug( "initiate transfer of {}", resourceName );
 690         }
 691
 692         @Override
 693         public void transferStarted( TransferEvent transferEvent )
 694         {
 695             this.totalLength = 0;
 696             resourceName = transferEvent.getResource( ).getName( );
 697             log.info( "start transfer of {}", transferEvent.getResource( ).getName( ) );
 698         }
 699
 700         @Override
 701         public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
 702         {
 703             log.debug( "transfer of {} : {}/{}", transferEvent.getResource( ).getName( ), buffer.length, length );
 704             this.totalLength += length;
 705         }
 706
 707         @Override
 708         public void transferCompleted( TransferEvent transferEvent )
 709         {
 710             resourceName = transferEvent.getResource( ).getName( );
 711             long endTime = System.currentTimeMillis( );
 712             log.info( "end of transfer file {} {} kb: {}s", transferEvent.getResource( ).getName( ),
 713                 this.totalLength / 1024, ( endTime - startTime ) / 1000 );
 714         }
 715
 716         @Override
 717         public void transferError( TransferEvent transferEvent )
 718         {
 719             log.info( "error of transfer file {}: {}", transferEvent.getResource( ).getName( ),
 720                 transferEvent.getException( ).getMessage( ), transferEvent.getException( ) );
 721         }
 722
 723         @Override
 724         public void debug( String message )
 725         {
 726             log.debug( "transfer debug {}", message );
 727         }
 728     }
 729
 730     private static class WagonResourceFetcher
 731         implements ResourceFetcher
 732     {
 733
 734         Logger log;
 735
 736         Path tempIndexDirectory;
 737
 738         Wagon wagon;
 739
 740         RemoteRepository remoteRepository;
 741
 742         private WagonResourceFetcher( Logger log, Path tempIndexDirectory, Wagon wagon,
 743                                       RemoteRepository remoteRepository )
 744         {
 745             this.log = log;
 746             this.tempIndexDirectory = tempIndexDirectory;
 747             this.wagon = wagon;
 748             this.remoteRepository = remoteRepository;
 749         }
 750
 751         @Override
 752         public void connect( String id, String url )
 753             throws IOException
 754         {
 755             //no op
 756         }
 757
 758         @Override
 759         public void disconnect( )
 760             throws IOException
 761         {
 762             // no op
 763         }
 764
 765         @Override
 766         public InputStream retrieve( String name )
 767             throws IOException, FileNotFoundException
 768         {
 769             try
 770             {
 771                 log.info( "index update retrieve file, name:{}", name );
 772                 Path file = tempIndexDirectory.resolve( name );
 773                 Files.deleteIfExists( file );
 774                 file.toFile( ).deleteOnExit( );
 775                 wagon.get( addParameters( name, remoteRepository ), file.toFile( ) );
 776                 return Files.newInputStream( file );
 777             }
 778             catch ( AuthorizationException | TransferFailedException e )
 779             {
 780                 throw new IOException( e.getMessage( ), e );
 781             }
 782             catch ( ResourceDoesNotExistException e )
 783             {
 784                 FileNotFoundException fnfe = new FileNotFoundException( e.getMessage( ) );
 785                 fnfe.initCause( e );
 786                 throw fnfe;
 787             }
 788         }
 789
 790         // FIXME remove crappy copy/paste
 791         protected String addParameters( String path, RemoteRepository remoteRepository )
 792         {
 793             if ( remoteRepository.getExtraParameters( ).isEmpty( ) )
 794             {
 795                 return path;
 796             }
 797
 798             boolean question = false;
 799
 800             StringBuilder res = new StringBuilder( path == null ? "" : path );
 801
 802             for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters( ).entrySet( ) )
 803             {
 804                 if ( !question )
 805                 {
 806                     res.append( '?' ).append( entry.getKey( ) ).append( '=' ).append( entry.getValue( ) );
 807                 }
 808             }
 809
 810             return res.toString( );
 811         }
 812
 813     }
 814 }