source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.scheduler.indexing.maven;
   2 /*
   3  * Licensed to the Apache Software Foundation (ASF) under one
   4  * or more contributor license agreements.  See the NOTICE file
   5  * distributed with this work for additional information
   6  * regarding copyright ownership.  The ASF licenses this file
   7  * to you under the Apache License, Version 2.0 (the
   8  * "License"); you may not use this file except in compliance
   9  * with the License.  You may obtain a copy of the License at
  10  *
  11  *   http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing,
  14  * software distributed under the License is distributed on an
  15  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  16  * KIND, either express or implied.  See the License for the
  17  * specific language governing permissions and limitations
  18  * under the License.
  19  */
  20
  21 import org.apache.archiva.proxy.maven.WagonFactory;
  22 import org.apache.archiva.proxy.maven.WagonFactoryRequest;
  23 import org.apache.archiva.proxy.model.NetworkProxy;
  24 import org.apache.archiva.repository.base.PasswordCredentials;
  25 import org.apache.archiva.repository.RemoteRepository;
  26 import org.apache.archiva.repository.RepositoryException;
  27 import org.apache.archiva.repository.RepositoryType;
  28 import org.apache.archiva.repository.features.RemoteIndexFeature;
  29 import org.apache.commons.lang3.time.StopWatch;
  30 import org.apache.maven.index.context.IndexingContext;
  31 import org.apache.maven.index.updater.IndexUpdateRequest;
  32 import org.apache.maven.index.updater.IndexUpdateResult;
  33 import org.apache.maven.index.updater.IndexUpdater;
  34 import org.apache.maven.index.updater.ResourceFetcher;
  35 import org.apache.maven.index_shaded.lucene.index.IndexNotFoundException;
  36 import org.apache.maven.wagon.ResourceDoesNotExistException;
  37 import org.apache.maven.wagon.StreamWagon;
  38 import org.apache.maven.wagon.TransferFailedException;
  39 import org.apache.maven.wagon.Wagon;
  40 import org.apache.maven.wagon.authentication.AuthenticationInfo;
  41 import org.apache.maven.wagon.authorization.AuthorizationException;
  42 import org.apache.maven.wagon.events.TransferEvent;
  43 import org.apache.maven.wagon.events.TransferListener;
  44 import org.apache.maven.wagon.proxy.ProxyInfo;
  45 import org.apache.maven.wagon.repository.Repository;
  46 import org.apache.maven.wagon.shared.http.AbstractHttpClientWagon;
  47 import org.apache.maven.wagon.shared.http.HttpConfiguration;
  48 import org.apache.maven.wagon.shared.http.HttpMethodConfiguration;
  49 import org.slf4j.Logger;
  50 import org.slf4j.LoggerFactory;
  51
  52 import java.io.FileNotFoundException;
  53 import java.io.IOException;
  54 import java.io.InputStream;
  55 import java.nio.file.Files;
  56 import java.nio.file.Path;
  57 import java.nio.file.Paths;
  58 import java.util.List;
  59 import java.util.Map;
  60
  61 /**
  62  * @author Olivier Lamy
  63  * @since 1.4-M1
  64  */
  65 public class DownloadRemoteIndexTask
  66     implements Runnable
  67 {
  68     private Logger log = LoggerFactory.getLogger( getClass() );
  69
  70     private RemoteRepository remoteRepository;
  71
  72     private WagonFactory wagonFactory;
  73
  74     private NetworkProxy networkProxy;
  75
  76     private boolean fullDownload;
  77
  78     private List<String> runningRemoteDownloadIds;
  79
  80     private IndexUpdater indexUpdater;
  81
  82
  83     public DownloadRemoteIndexTask( DownloadRemoteIndexTaskRequest downloadRemoteIndexTaskRequest,
  84                                     List<String> runningRemoteDownloadIds )
  85     {
  86         this.remoteRepository = downloadRemoteIndexTaskRequest.getRemoteRepository();
  87         this.wagonFactory = downloadRemoteIndexTaskRequest.getWagonFactory();
  88         this.networkProxy = downloadRemoteIndexTaskRequest.getNetworkProxy();
  89         this.fullDownload = downloadRemoteIndexTaskRequest.isFullDownload();
  90         this.runningRemoteDownloadIds = runningRemoteDownloadIds;
  91         this.indexUpdater = downloadRemoteIndexTaskRequest.getIndexUpdater();
  92     }
  93
  94     @Override
  95     public void run()
  96     {
  97
  98         // so short lock : not sure we need it
  99         synchronized ( this.runningRemoteDownloadIds )
 100         {
 101             if ( this.runningRemoteDownloadIds.contains( this.remoteRepository.getId() ) )
 102             {
 103                 // skip it as it's running
 104                 log.info( "skip download index remote for repo {} it's already running",
 105                           this.remoteRepository.getId() );
 106                 return;
 107             }
 108             this.runningRemoteDownloadIds.add( this.remoteRepository.getId() );
 109         }
 110         Path tempIndexDirectory = null;
 111         StopWatch stopWatch = new StopWatch();
 112         stopWatch.start();
 113         try
 114         {
 115             log.info( "start download remote index for remote repository {}", this.remoteRepository.getId() );
 116             if (this.remoteRepository.getIndexingContext()==null) {
 117                 throw new IndexNotFoundException("No index context set for repository "+remoteRepository.getId());
 118             }
 119             if (this.remoteRepository.getType()!= RepositoryType.MAVEN) {
 120                 throw new RepositoryException("Bad repository type");
 121             }
 122             if (!this.remoteRepository.supportsFeature(RemoteIndexFeature.class)) {
 123                 throw new RepositoryException("Repository does not support RemotIndexFeature "+remoteRepository.getId());
 124             }
 125             RemoteIndexFeature rif = this.remoteRepository.getFeature(RemoteIndexFeature.class).get();
 126             IndexingContext indexingContext = this.remoteRepository.getIndexingContext().getBaseContext(IndexingContext.class);
 127             // create a temp directory to download files
 128             tempIndexDirectory = Paths.get(indexingContext.getIndexDirectoryFile().getParent(), ".tmpIndex" );
 129             Path indexCacheDirectory = Paths.get( indexingContext.getIndexDirectoryFile().getParent(), ".indexCache" );
 130             Files.createDirectories( indexCacheDirectory );
 131             if ( Files.exists(tempIndexDirectory) )
 132             {
 133                 org.apache.archiva.common.utils.FileUtils.deleteDirectory( tempIndexDirectory );
 134             }
 135             Files.createDirectories( tempIndexDirectory );
 136             tempIndexDirectory.toFile().deleteOnExit();
 137             String baseIndexUrl = indexingContext.getIndexUpdateUrl();
 138
 139             String wagonProtocol = this.remoteRepository.getLocation().getScheme();
 140
 141             final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon(
 142                 new WagonFactoryRequest( wagonProtocol, this.remoteRepository.getExtraHeaders() ).networkProxy(
 143                     this.networkProxy )
 144             );
 145             // FIXME olamy having 2 config values
 146             wagon.setReadTimeout( (int)rif.getDownloadTimeout().toMillis());
 147             wagon.setTimeout( (int)remoteRepository.getTimeout().toMillis());
 148
 149             if ( wagon instanceof AbstractHttpClientWagon )
 150             {
 151                 HttpConfiguration httpConfiguration = new HttpConfiguration();
 152                 HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration();
 153                 httpMethodConfiguration.setUsePreemptive( true );
 154                 httpMethodConfiguration.setReadTimeout( (int)rif.getDownloadTimeout().toMillis() );
 155                 httpConfiguration.setGet( httpMethodConfiguration );
 156                 AbstractHttpClientWagon.class.cast( wagon ).setHttpConfiguration( httpConfiguration );
 157             }
 158
 159             wagon.addTransferListener( new DownloadListener() );
 160             ProxyInfo proxyInfo = null;
 161             if ( this.networkProxy != null )
 162             {
 163                 proxyInfo = new ProxyInfo();
 164                 proxyInfo.setType( this.networkProxy.getProtocol() );
 165                 proxyInfo.setHost( this.networkProxy.getHost() );
 166                 proxyInfo.setPort( this.networkProxy.getPort() );
 167                 proxyInfo.setUserName( this.networkProxy.getUsername() );
 168                 proxyInfo.setPassword( new String(this.networkProxy.getPassword()) );
 169             }
 170             AuthenticationInfo authenticationInfo = null;
 171             if ( this.remoteRepository.getLoginCredentials()!=null && this.remoteRepository.getLoginCredentials() instanceof PasswordCredentials )
 172             {
 173                 PasswordCredentials creds = (PasswordCredentials) this.remoteRepository.getLoginCredentials();
 174                 authenticationInfo = new AuthenticationInfo();
 175                 authenticationInfo.setUserName( creds.getUsername());
 176                 authenticationInfo.setPassword( new String(creds.getPassword()) );
 177             }
 178             log.debug("Connection to {}, authInfo={}", this.remoteRepository.getId(), authenticationInfo);
 179             wagon.connect( new Repository( this.remoteRepository.getId(), baseIndexUrl ), authenticationInfo,
 180                            proxyInfo );
 181
 182             Path indexDirectory = indexingContext.getIndexDirectoryFile().toPath();
 183             if ( !Files.exists(indexDirectory) )
 184             {
 185                 Files.createDirectories( indexDirectory );
 186             }
 187             log.debug("Downloading index file to {}", indexDirectory);
 188             log.debug("Index cache dir {}", indexCacheDirectory);
 189
 190             ResourceFetcher resourceFetcher =
 191                 new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository );
 192             IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
 193             request.setForceFullUpdate( this.fullDownload );
 194             request.setLocalIndexCacheDir( indexCacheDirectory.toFile() );
 195
 196             IndexUpdateResult result = this.indexUpdater.fetchAndUpdateIndex(request);
 197             log.debug("Update result success: {}", result.isSuccessful());
 198             stopWatch.stop();
 199             log.info( "time update index from remote for repository {}: {}ms", this.remoteRepository.getId(),
 200                       ( stopWatch.getTime() ) );
 201
 202             // index packing optionnal ??
 203             //IndexPackingRequest indexPackingRequest =
 204             //    new IndexPackingRequest( indexingContext, indexingContext.getIndexDirectoryFile() );
 205             //indexPacker.packIndex( indexPackingRequest );
 206             indexingContext.updateTimestamp( true );
 207
 208         }
 209         catch ( Exception e )
 210         {
 211             log.error( e.getMessage(), e );
 212             throw new RuntimeException( e.getMessage(), e );
 213         }
 214         finally
 215         {
 216             deleteDirectoryQuiet( tempIndexDirectory );
 217             this.runningRemoteDownloadIds.remove( this.remoteRepository.getId() );
 218         }
 219         log.info( "end download remote index for remote repository {}", this.remoteRepository.getId() );
 220     }
 221
 222     private void deleteDirectoryQuiet( Path f )
 223     {
 224         try
 225         {
 226             org.apache.archiva.common.utils.FileUtils.deleteDirectory( f );
 227         }
 228         catch ( IOException e )
 229         {
 230             log.warn( "skip error delete {} : {}", f, e.getMessage() );
 231         }
 232     }
 233
 234
 235     private static final class DownloadListener
 236         implements TransferListener
 237     {
 238         private Logger log = LoggerFactory.getLogger( getClass() );
 239
 240         private String resourceName;
 241
 242         private long startTime;
 243
 244         private int totalLength = 0;
 245
 246         @Override
 247         public void transferInitiated( TransferEvent transferEvent )
 248         {
 249             startTime = System.currentTimeMillis();
 250             resourceName = transferEvent.getResource().getName();
 251             log.debug( "initiate transfer of {}", resourceName );
 252         }
 253
 254         @Override
 255         public void transferStarted( TransferEvent transferEvent )
 256         {
 257             this.totalLength = 0;
 258             resourceName = transferEvent.getResource().getName();
 259             log.info("Transferring: {}, {}",  transferEvent.getResource().getContentLength(), transferEvent.getLocalFile().toString());
 260             log.info( "start transfer of {}", transferEvent.getResource().getName() );
 261         }
 262
 263         @Override
 264         public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
 265         {
 266             log.debug( "transfer of {} : {}/{}", transferEvent.getResource().getName(), buffer.length, length );
 267             this.totalLength += length;
 268         }
 269
 270         @Override
 271         public void transferCompleted( TransferEvent transferEvent )
 272         {
 273             resourceName = transferEvent.getResource().getName();
 274             long endTime = System.currentTimeMillis();
 275             log.info( "end of transfer file {}: {}b, {}ms", transferEvent.getResource().getName(),
 276                       this.totalLength, ( endTime - startTime ) );
 277         }
 278
 279         @Override
 280         public void transferError( TransferEvent transferEvent )
 281         {
 282             log.info( "error of transfer file {}: {}", transferEvent.getResource().getName(),
 283                       transferEvent.getException().getMessage(), transferEvent.getException() );
 284         }
 285
 286         @Override
 287         public void debug( String message )
 288         {
 289             log.debug( "transfer debug {}", message );
 290         }
 291     }
 292
 293     private static class WagonResourceFetcher
 294         implements ResourceFetcher
 295     {
 296
 297         Logger log;
 298
 299         Path tempIndexDirectory;
 300
 301         Wagon wagon;
 302
 303         RemoteRepository remoteRepository;
 304
 305         private WagonResourceFetcher( Logger log, Path tempIndexDirectory, Wagon wagon,
 306                                       RemoteRepository remoteRepository )
 307         {
 308             this.log = log;
 309             this.tempIndexDirectory = tempIndexDirectory;
 310             this.wagon = wagon;
 311             this.remoteRepository = remoteRepository;
 312         }
 313
 314         @Override
 315         public void connect( String id, String url )
 316             throws IOException
 317         {
 318             //no op
 319         }
 320
 321         @Override
 322         public void disconnect()
 323             throws IOException
 324         {
 325             // no op
 326         }
 327
 328         @Override
 329         public InputStream retrieve( String name )
 330             throws IOException, FileNotFoundException
 331         {
 332             try
 333             {
 334                 log.info( "index update retrieve file, name:{}", name );
 335                 Path file = tempIndexDirectory.resolve( name );
 336                 Files.deleteIfExists( file );
 337                 file.toFile().deleteOnExit();
 338                 wagon.get( addParameters( name, this.remoteRepository ), file.toFile() );
 339                 return Files.newInputStream( file );
 340             }
 341             catch ( AuthorizationException | TransferFailedException e )
 342             {
 343                 throw new IOException( e.getMessage(), e );
 344             }
 345             catch ( ResourceDoesNotExistException e )
 346             {
 347                 FileNotFoundException fnfe = new FileNotFoundException( e.getMessage() );
 348                 fnfe.initCause( e );
 349                 throw fnfe;
 350             }
 351         }
 352
 353         // FIXME remove crappy copy/paste
 354         protected String addParameters( String path, RemoteRepository remoteRepository )
 355         {
 356             if ( remoteRepository.getExtraParameters().isEmpty() )
 357             {
 358                 return path;
 359             }
 360
 361             boolean question = false;
 362
 363             StringBuilder res = new StringBuilder( path == null ? "" : path );
 364
 365             for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() )
 366             {
 367                 if ( !question )
 368                 {
 369                     res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() );
 370                 }
 371             }
 372
 373             return res.toString();
 374         }
 375
 376     }
 377
 378
 379 }
 380