source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.scheduler.indexing;
   2 /*
   3  * Licensed to the Apache Software Foundation (ASF) under one
   4  * or more contributor license agreements.  See the NOTICE file
   5  * distributed with this work for additional information
   6  * regarding copyright ownership.  The ASF licenses this file
   7  * to you under the Apache License, Version 2.0 (the
   8  * "License"); you may not use this file except in compliance
   9  * with the License.  You may obtain a copy of the License at
  10  *
  11  *   http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing,
  14  * software distributed under the License is distributed on an
  15  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  16  * KIND, either express or implied.  See the License for the
  17  * specific language governing permissions and limitations
  18  * under the License.
  19  */
  20
  21 import org.apache.archiva.admin.model.RepositoryAdminException;
  22 import org.apache.archiva.admin.model.beans.NetworkProxy;
  23 import org.apache.archiva.admin.model.beans.RemoteRepository;
  24 import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin;
  25 import org.apache.archiva.proxy.common.WagonFactory;
  26 import org.apache.archiva.proxy.common.WagonFactoryException;
  27 import org.apache.archiva.proxy.common.WagonFactoryRequest;
  28 import org.apache.commons.io.FileUtils;
  29 import org.apache.commons.lang.time.StopWatch;
  30 import org.apache.maven.index.context.IndexingContext;
  31 import org.apache.maven.index.updater.IndexUpdateRequest;
  32 import org.apache.maven.index.updater.IndexUpdater;
  33 import org.apache.maven.index.updater.ResourceFetcher;
  34 import org.apache.maven.wagon.ConnectionException;
  35 import org.apache.maven.wagon.ResourceDoesNotExistException;
  36 import org.apache.maven.wagon.StreamWagon;
  37 import org.apache.maven.wagon.TransferFailedException;
  38 import org.apache.maven.wagon.Wagon;
  39 import org.apache.maven.wagon.authentication.AuthenticationException;
  40 import org.apache.maven.wagon.authentication.AuthenticationInfo;
  41 import org.apache.maven.wagon.authorization.AuthorizationException;
  42 import org.apache.maven.wagon.events.TransferEvent;
  43 import org.apache.maven.wagon.events.TransferListener;
  44 import org.apache.maven.wagon.providers.http.AbstractHttpClientWagon;
  45 import org.apache.maven.wagon.providers.http.HttpConfiguration;
  46 import org.apache.maven.wagon.providers.http.HttpMethodConfiguration;
  47 import org.apache.maven.wagon.proxy.ProxyInfo;
  48 import org.apache.maven.wagon.repository.Repository;
  49 import org.slf4j.Logger;
  50 import org.slf4j.LoggerFactory;
  51
  52 import java.io.File;
  53 import java.io.FileInputStream;
  54 import java.io.FileNotFoundException;
  55 import java.io.IOException;
  56 import java.io.InputStream;
  57 import java.net.MalformedURLException;
  58 import java.net.URL;
  59 import java.util.List;
  60 import java.util.Map;
  61
  62 /**
  63  * @author Olivier Lamy
  64  * @since 1.4-M1
  65  */
  66 public class DownloadRemoteIndexTask
  67     implements Runnable
  68 {
  69     private Logger log = LoggerFactory.getLogger( getClass() );
  70
  71     private RemoteRepository remoteRepository;
  72
  73     private RemoteRepositoryAdmin remoteRepositoryAdmin;
  74
  75     private WagonFactory wagonFactory;
  76
  77     private NetworkProxy networkProxy;
  78
  79     private boolean fullDownload;
  80
  81     private List<String> runningRemoteDownloadIds;
  82
  83     private IndexUpdater indexUpdater;
  84
  85
  86     public DownloadRemoteIndexTask( DownloadRemoteIndexTaskRequest downloadRemoteIndexTaskRequest,
  87                                     List<String> runningRemoteDownloadIds )
  88     {
  89         this.remoteRepository = downloadRemoteIndexTaskRequest.getRemoteRepository();
  90         this.wagonFactory = downloadRemoteIndexTaskRequest.getWagonFactory();
  91         this.networkProxy = downloadRemoteIndexTaskRequest.getNetworkProxy();
  92         this.fullDownload = downloadRemoteIndexTaskRequest.isFullDownload();
  93         this.runningRemoteDownloadIds = runningRemoteDownloadIds;
  94         this.indexUpdater = downloadRemoteIndexTaskRequest.getIndexUpdater();
  95         this.remoteRepositoryAdmin = downloadRemoteIndexTaskRequest.getRemoteRepositoryAdmin();
  96     }
  97
  98     public void run()
  99     {
 100
 101         // so short lock : not sure we need it
 102         synchronized ( this.runningRemoteDownloadIds )
 103         {
 104             if ( this.runningRemoteDownloadIds.contains( this.remoteRepository.getId() ) )
 105             {
 106                 // skip it as it's running
 107                 log.info( "skip download index remote for repo {} it's already running",
 108                           this.remoteRepository.getId() );
 109                 return;
 110             }
 111             this.runningRemoteDownloadIds.add( this.remoteRepository.getId() );
 112         }
 113         File tempIndexDirectory = null;
 114         StopWatch stopWatch = new StopWatch();
 115         stopWatch.start();
 116         try
 117         {
 118             log.info( "start download remote index for remote repository {}", this.remoteRepository.getId() );
 119             IndexingContext indexingContext = remoteRepositoryAdmin.createIndexContext( this.remoteRepository );
 120
 121             // create a temp directory to download files
 122             tempIndexDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".tmpIndex" );
 123             File indexCacheDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".indexCache" );
 124             indexCacheDirectory.mkdirs();
 125             if ( tempIndexDirectory.exists() )
 126             {
 127                 FileUtils.deleteDirectory( tempIndexDirectory );
 128             }
 129             tempIndexDirectory.mkdirs();
 130             tempIndexDirectory.deleteOnExit();
 131             String baseIndexUrl = indexingContext.getIndexUpdateUrl();
 132
 133             String wagonProtocol = new URL( this.remoteRepository.getUrl() ).getProtocol();
 134
 135             final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon(
 136                 new WagonFactoryRequest( wagonProtocol, this.remoteRepository.getExtraHeaders() ).networkProxy(
 137                     this.networkProxy ) );
 138             int timeoutInMilliseconds = remoteRepository.getTimeout() * 1000;
 139             // FIXME olamy having 2 config values
 140             wagon.setReadTimeout( timeoutInMilliseconds );
 141             wagon.setTimeout( timeoutInMilliseconds );
 142
 143             if ( wagon instanceof AbstractHttpClientWagon )
 144             {
 145                 HttpConfiguration httpConfiguration = new HttpConfiguration();
 146                 HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration();
 147                 httpMethodConfiguration.setUsePreemptive( true );
 148                 httpMethodConfiguration.setReadTimeout( timeoutInMilliseconds );
 149                 httpConfiguration.setGet( httpMethodConfiguration );
 150                 ( (AbstractHttpClientWagon) wagon ).setHttpConfiguration( httpConfiguration );
 151             }
 152
 153             wagon.addTransferListener( new DownloadListener() );
 154             ProxyInfo proxyInfo = null;
 155             if ( this.networkProxy != null )
 156             {
 157                 proxyInfo = new ProxyInfo();
 158                 proxyInfo.setHost( this.networkProxy.getHost() );
 159                 proxyInfo.setPort( this.networkProxy.getPort() );
 160                 proxyInfo.setUserName( this.networkProxy.getUsername() );
 161                 proxyInfo.setPassword( this.networkProxy.getPassword() );
 162             }
 163             AuthenticationInfo authenticationInfo = null;
 164             if ( this.remoteRepository.getUserName() != null )
 165             {
 166                 authenticationInfo = new AuthenticationInfo();
 167                 authenticationInfo.setUserName( this.remoteRepository.getUserName() );
 168                 authenticationInfo.setPassword( this.remoteRepository.getPassword() );
 169             }
 170             wagon.connect( new Repository( this.remoteRepository.getId(), baseIndexUrl ), authenticationInfo,
 171                            proxyInfo );
 172
 173             File indexDirectory = indexingContext.getIndexDirectoryFile();
 174             if ( !indexDirectory.exists() )
 175             {
 176                 indexDirectory.mkdirs();
 177             }
 178
 179             ResourceFetcher resourceFetcher =
 180                 new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository );
 181             IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
 182             request.setForceFullUpdate( this.fullDownload );
 183             request.setLocalIndexCacheDir( indexCacheDirectory );
 184
 185             this.indexUpdater.fetchAndUpdateIndex( request );
 186             stopWatch.stop();
 187             log.info( "time update index from remote for repository {}: {} s", this.remoteRepository.getId(),
 188                       ( stopWatch.getTime() / 1000 ) );
 189
 190             // index packing optionnal ??
 191             //IndexPackingRequest indexPackingRequest =
 192             //    new IndexPackingRequest( indexingContext, indexingContext.getIndexDirectoryFile() );
 193             //indexPacker.packIndex( indexPackingRequest );
 194             indexingContext.updateTimestamp( true );
 195
 196         }
 197         catch ( MalformedURLException e )
 198         {
 199             log.error( e.getMessage(), e );
 200             throw new RuntimeException( e.getMessage(), e );
 201         }
 202         catch ( WagonFactoryException e )
 203         {
 204             log.error( e.getMessage(), e );
 205             throw new RuntimeException( e.getMessage(), e );
 206         }
 207         catch ( ConnectionException e )
 208         {
 209             log.error( e.getMessage(), e );
 210             throw new RuntimeException( e.getMessage(), e );
 211         }
 212         catch ( AuthenticationException e )
 213         {
 214             log.error( e.getMessage(), e );
 215             throw new RuntimeException( e.getMessage(), e );
 216         }
 217         catch ( IOException e )
 218         {
 219             log.error( e.getMessage(), e );
 220             throw new RuntimeException( e.getMessage(), e );
 221         }
 222         catch ( RepositoryAdminException e )
 223         {
 224             log.error( e.getMessage(), e );
 225             throw new RuntimeException( e.getMessage(), e );
 226         }
 227         finally
 228         {
 229             deleteDirectoryQuiet( tempIndexDirectory );
 230             this.runningRemoteDownloadIds.remove( this.remoteRepository.getId() );
 231         }
 232         log.info( "end download remote index for remote repository {}", this.remoteRepository.getId() );
 233     }
 234
 235     private void deleteDirectoryQuiet( File f )
 236     {
 237         try
 238         {
 239             FileUtils.deleteDirectory( f );
 240         }
 241         catch ( IOException e )
 242         {
 243             log.warn( "skip error delete {} : {}", f, e.getMessage() );
 244         }
 245     }
 246
 247
 248     private static final class DownloadListener
 249         implements TransferListener
 250     {
 251         private Logger log = LoggerFactory.getLogger( getClass() );
 252
 253         private String resourceName;
 254
 255         private long startTime;
 256
 257         private int totalLength = 0;
 258
 259         public void transferInitiated( TransferEvent transferEvent )
 260         {
 261             startTime = System.currentTimeMillis();
 262             resourceName = transferEvent.getResource().getName();
 263             log.debug( "initiate transfer of {}", resourceName );
 264         }
 265
 266         public void transferStarted( TransferEvent transferEvent )
 267         {
 268             this.totalLength = 0;
 269             resourceName = transferEvent.getResource().getName();
 270             log.info( "start transfer of {}", transferEvent.getResource().getName() );
 271         }
 272
 273         public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
 274         {
 275             log.debug( "transfer of {} : {}/{}", transferEvent.getResource().getName(), buffer.length, length );
 276             this.totalLength += length;
 277         }
 278
 279         public void transferCompleted( TransferEvent transferEvent )
 280         {
 281             resourceName = transferEvent.getResource().getName();
 282             long endTime = System.currentTimeMillis();
 283             log.info( "end of transfer file {} {} kb: {}s", transferEvent.getResource().getName(),
 284                       this.totalLength / 1024, ( endTime - startTime ) / 1000 );
 285         }
 286
 287         public void transferError( TransferEvent transferEvent )
 288         {
 289             log.info( "error of transfer file {}: {}", transferEvent.getResource().getName(),
 290                       transferEvent.getException().getMessage(), transferEvent.getException() );
 291         }
 292
 293         public void debug( String message )
 294         {
 295             log.debug( "transfer debug {}", message );
 296         }
 297     }
 298
 299     private static class WagonResourceFetcher
 300         implements ResourceFetcher
 301     {
 302
 303         Logger log;
 304
 305         File tempIndexDirectory;
 306
 307         Wagon wagon;
 308
 309         RemoteRepository remoteRepository;
 310
 311         private WagonResourceFetcher( Logger log, File tempIndexDirectory, Wagon wagon,
 312                                       RemoteRepository remoteRepository )
 313         {
 314             this.log = log;
 315             this.tempIndexDirectory = tempIndexDirectory;
 316             this.wagon = wagon;
 317             this.remoteRepository = remoteRepository;
 318         }
 319
 320         public void connect( String id, String url )
 321             throws IOException
 322         {
 323             //no op
 324         }
 325
 326         public void disconnect()
 327             throws IOException
 328         {
 329             // no op
 330         }
 331
 332         public InputStream retrieve( String name )
 333             throws IOException, FileNotFoundException
 334         {
 335             try
 336             {
 337                 log.info( "index update retrieve file, name:{}", name );
 338                 File file = new File( tempIndexDirectory, name );
 339                 if ( file.exists() )
 340                 {
 341                     file.delete();
 342                 }
 343                 file.deleteOnExit();
 344                 wagon.get( addParameters( name, this.remoteRepository ), file );
 345                 return new FileInputStream( file );
 346             }
 347             catch ( AuthorizationException e )
 348             {
 349                 throw new IOException( e.getMessage(), e );
 350             }
 351             catch ( TransferFailedException e )
 352             {
 353                 throw new IOException( e.getMessage(), e );
 354             }
 355             catch ( ResourceDoesNotExistException e )
 356             {
 357                 FileNotFoundException fnfe = new FileNotFoundException( e.getMessage() );
 358                 fnfe.initCause( e );
 359                 throw fnfe;
 360             }
 361         }
 362
 363         // FIXME remove crappy copy/paste
 364         protected String addParameters( String path, RemoteRepository remoteRepository )
 365         {
 366             if ( remoteRepository.getExtraParameters().isEmpty() )
 367             {
 368                 return path;
 369             }
 370
 371             boolean question = false;
 372
 373             StringBuilder res = new StringBuilder( path == null ? "" : path );
 374
 375             for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() )
 376             {
 377                 if ( !question )
 378                 {
 379                     res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() );
 380                 }
 381             }
 382
 383             return res.toString();
 384         }
 385
 386     }
 387
 388
 389 }
 390