source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.scheduler.indexing;
   2 /*
   3  * Licensed to the Apache Software Foundation (ASF) under one
   4  * or more contributor license agreements.  See the NOTICE file
   5  * distributed with this work for additional information
   6  * regarding copyright ownership.  The ASF licenses this file
   7  * to you under the Apache License, Version 2.0 (the
   8  * "License"); you may not use this file except in compliance
   9  * with the License.  You may obtain a copy of the License at
  10  *
  11  *   http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing,
  14  * software distributed under the License is distributed on an
  15  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  16  * KIND, either express or implied.  See the License for the
  17  * specific language governing permissions and limitations
  18  * under the License.
  19  */
  20
  21 import org.apache.archiva.admin.model.RepositoryAdminException;
  22 import org.apache.archiva.admin.model.beans.NetworkProxy;
  23 import org.apache.archiva.admin.model.beans.RemoteRepository;
  24 import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin;
  25 import org.apache.archiva.proxy.common.WagonFactory;
  26 import org.apache.archiva.proxy.common.WagonFactoryException;
  27 import org.apache.commons.io.FileUtils;
  28 import org.apache.commons.lang.time.StopWatch;
  29 import org.apache.maven.index.context.IndexingContext;
  30 import org.apache.maven.index.updater.IndexUpdateRequest;
  31 import org.apache.maven.index.updater.IndexUpdater;
  32 import org.apache.maven.index.updater.ResourceFetcher;
  33 import org.apache.maven.wagon.ConnectionException;
  34 import org.apache.maven.wagon.ResourceDoesNotExistException;
  35 import org.apache.maven.wagon.TransferFailedException;
  36 import org.apache.maven.wagon.Wagon;
  37 import org.apache.maven.wagon.authentication.AuthenticationException;
  38 import org.apache.maven.wagon.authentication.AuthenticationInfo;
  39 import org.apache.maven.wagon.authorization.AuthorizationException;
  40 import org.apache.maven.wagon.events.TransferEvent;
  41 import org.apache.maven.wagon.events.TransferListener;
  42 import org.apache.maven.wagon.proxy.ProxyInfo;
  43 import org.apache.maven.wagon.repository.Repository;
  44 import org.apache.maven.wagon.shared.http.HttpConfiguration;
  45 import org.apache.maven.wagon.shared.http.HttpMethodConfiguration;
  46 import org.slf4j.Logger;
  47 import org.slf4j.LoggerFactory;
  48
  49 import java.io.File;
  50 import java.io.FileInputStream;
  51 import java.io.FileNotFoundException;
  52 import java.io.IOException;
  53 import java.io.InputStream;
  54 import java.lang.reflect.Method;
  55 import java.net.MalformedURLException;
  56 import java.net.URL;
  57 import java.util.Arrays;
  58 import java.util.List;
  59
  60 /**
  61  * @author Olivier Lamy
  62  * @since 1.4-M1
  63  */
  64 public class DownloadRemoteIndexTask
  65     implements Runnable
  66 {
  67     private Logger log = LoggerFactory.getLogger( getClass() );
  68
  69     private RemoteRepository remoteRepository;
  70
  71     private RemoteRepositoryAdmin remoteRepositoryAdmin;
  72
  73     private WagonFactory wagonFactory;
  74
  75     private NetworkProxy networkProxy;
  76
  77     private boolean fullDownload;
  78
  79     private List<String> runningRemoteDownloadIds;
  80
  81     private IndexUpdater indexUpdater;
  82
  83     public DownloadRemoteIndexTask( DownloadRemoteIndexTaskRequest downloadRemoteIndexTaskRequest,
  84                                     List<String> runningRemoteDownloadIds )
  85     {
  86         this.remoteRepository = downloadRemoteIndexTaskRequest.getRemoteRepository();
  87         this.wagonFactory = downloadRemoteIndexTaskRequest.getWagonFactory();
  88         this.networkProxy = downloadRemoteIndexTaskRequest.getNetworkProxy();
  89         this.fullDownload = downloadRemoteIndexTaskRequest.isFullDownload();
  90         this.runningRemoteDownloadIds = runningRemoteDownloadIds;
  91         this.indexUpdater = downloadRemoteIndexTaskRequest.getIndexUpdater();
  92         this.remoteRepositoryAdmin = downloadRemoteIndexTaskRequest.getRemoteRepositoryAdmin();
  93     }
  94
  95     public void run()
  96     {
  97
  98         // so short lock : not sure we need it
  99         synchronized ( this.runningRemoteDownloadIds )
 100         {
 101             if ( this.runningRemoteDownloadIds.contains( this.remoteRepository.getId() ) )
 102             {
 103                 // skip it as it's running
 104                 log.info( "skip download index remote for repo {} it's already running",
 105                           this.remoteRepository.getId() );
 106                 return;
 107             }
 108             this.runningRemoteDownloadIds.add( this.remoteRepository.getId() );
 109         }
 110         File tempIndexDirectory = null;
 111         StopWatch stopWatch = new StopWatch();
 112         stopWatch.start();
 113         try
 114         {
 115             log.info( "start download remote index for remote repository " + this.remoteRepository.getId() );
 116             IndexingContext indexingContext = remoteRepositoryAdmin.createIndexContext( this.remoteRepository );
 117
 118             // create a temp directory to download files
 119             tempIndexDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".tmpIndex" );
 120             File indexCacheDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".indexCache" );
 121             indexCacheDirectory.mkdirs();
 122             if ( tempIndexDirectory.exists() )
 123             {
 124                 FileUtils.deleteDirectory( tempIndexDirectory );
 125             }
 126             tempIndexDirectory.mkdirs();
 127             tempIndexDirectory.deleteOnExit();
 128             String baseIndexUrl = indexingContext.getIndexUpdateUrl();
 129
 130             final Wagon wagon = wagonFactory.getWagon( new URL( this.remoteRepository.getUrl() ).getProtocol() );
 131             setupWagonReadTimeout( wagon );
 132
 133             wagon.addTransferListener( new DownloadListener() );
 134             ProxyInfo proxyInfo = null;
 135             if ( this.networkProxy != null )
 136             {
 137                 proxyInfo = new ProxyInfo();
 138                 proxyInfo.setHost( this.networkProxy.getHost() );
 139                 proxyInfo.setPort( this.networkProxy.getPort() );
 140                 proxyInfo.setUserName( this.networkProxy.getUsername() );
 141                 proxyInfo.setPassword( this.networkProxy.getPassword() );
 142             }
 143             AuthenticationInfo authenticationInfo = null;
 144             if ( this.remoteRepository.getUserName() != null )
 145             {
 146                 authenticationInfo = new AuthenticationInfo();
 147                 authenticationInfo.setUserName( this.remoteRepository.getUserName() );
 148                 authenticationInfo.setPassword( this.remoteRepository.getPassword() );
 149             }
 150             wagon.connect( new Repository( this.remoteRepository.getId(), baseIndexUrl ), authenticationInfo,
 151                            proxyInfo );
 152
 153             File indexDirectory = indexingContext.getIndexDirectoryFile();
 154             if ( !indexDirectory.exists() )
 155             {
 156                 indexDirectory.mkdirs();
 157             }
 158
 159             ResourceFetcher resourceFetcher = new WagonResourceFetcher( log, tempIndexDirectory, wagon );
 160             IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
 161             request.setForceFullUpdate( this.fullDownload );
 162             request.setLocalIndexCacheDir( indexCacheDirectory );
 163
 164             this.indexUpdater.fetchAndUpdateIndex( request );
 165             stopWatch.stop();
 166             log.info( "time to download remote repository index for repository {}: {} s", this.remoteRepository.getId(),
 167                       ( stopWatch.getTime() / 1000 ) );
 168         }
 169         catch ( MalformedURLException e )
 170         {
 171             log.error( e.getMessage(), e );
 172             throw new RuntimeException( e.getMessage(), e );
 173         }
 174         catch ( WagonFactoryException e )
 175         {
 176             log.error( e.getMessage(), e );
 177             throw new RuntimeException( e.getMessage(), e );
 178         }
 179         catch ( ConnectionException e )
 180         {
 181             log.error( e.getMessage(), e );
 182             throw new RuntimeException( e.getMessage(), e );
 183         }
 184         catch ( AuthenticationException e )
 185         {
 186             log.error( e.getMessage(), e );
 187             throw new RuntimeException( e.getMessage(), e );
 188         }
 189         catch ( IOException e )
 190         {
 191             log.error( e.getMessage(), e );
 192             throw new RuntimeException( e.getMessage(), e );
 193         }
 194         catch ( RepositoryAdminException e )
 195         {
 196             log.error( e.getMessage(), e );
 197             throw new RuntimeException( e.getMessage(), e );
 198         }
 199         finally
 200         {
 201             deleteDirectoryQuiet( tempIndexDirectory );
 202             this.runningRemoteDownloadIds.remove( this.remoteRepository.getId() );
 203         }
 204         log.info( "end download remote index for remote repository " + this.remoteRepository.getId() );
 205     }
 206
 207     private void deleteDirectoryQuiet( File f )
 208     {
 209         try
 210         {
 211             FileUtils.deleteDirectory( f );
 212         }
 213         catch ( IOException e )
 214         {
 215             log.warn( "skip error delete " + f + ": " + e.getMessage() );
 216         }
 217     }
 218
 219     private void setupWagonReadTimeout( Wagon wagon )
 220     {
 221         try
 222         {
 223             HttpConfiguration httpConfiguration = new HttpConfiguration().setAll(
 224                 new HttpMethodConfiguration().setReadTimeout( remoteRepository.getRemoteDownloadTimeout() * 1000 ) );
 225             Method setHttpConfigurationMethod =
 226                 wagon.getClass().getMethod( "setHttpConfiguration", HttpConfiguration.class );
 227             setHttpConfigurationMethod.invoke( wagon, httpConfiguration );
 228         }
 229         catch ( Exception e )
 230         {
 231             log.debug( "unable to set download remote time out for index {}", e.getMessage(), e );
 232         }
 233     }
 234
 235
 236     public static class DownloadListener
 237         implements TransferListener
 238     {
 239         private Logger log = LoggerFactory.getLogger( getClass() );
 240
 241         String reourceName;
 242
 243         long startTime;
 244
 245         public void transferInitiated( TransferEvent transferEvent )
 246         {
 247             reourceName = transferEvent.getResource().getName();
 248             log.debug( "initiate transfer of {}", reourceName );
 249         }
 250
 251         public void transferStarted( TransferEvent transferEvent )
 252         {
 253             reourceName = transferEvent.getResource().getName();
 254             startTime = System.currentTimeMillis();
 255             log.info( "start transfer of {}", transferEvent.getResource().getName() );
 256         }
 257
 258         public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
 259         {
 260             log.debug( "transfer of {} : {}/{}",
 261                        Arrays.asList( transferEvent.getResource().getName(), buffer.length, length ).toArray() );
 262         }
 263
 264         public void transferCompleted( TransferEvent transferEvent )
 265         {
 266             reourceName = transferEvent.getResource().getName();
 267             long endTime = System.currentTimeMillis();
 268             log.info( "end of transfer file {}: {}s", transferEvent.getResource().getName(),
 269                       ( endTime - startTime ) / 1000 );
 270         }
 271
 272         public void transferError( TransferEvent transferEvent )
 273         {
 274             log.info( "error of transfer file {}: {}", Arrays.asList( transferEvent.getResource().getName(),
 275                                                                       transferEvent.getException().getMessage() ).toArray(
 276                 new Object[2] ), transferEvent.getException() );
 277         }
 278
 279         public void debug( String message )
 280         {
 281             log.debug( "transfer debug {}", message );
 282         }
 283     }
 284
 285     private static class WagonResourceFetcher
 286         implements ResourceFetcher
 287     {
 288
 289         Logger log;
 290
 291         File tempIndexDirectory;
 292
 293         Wagon wagon;
 294
 295         private WagonResourceFetcher( Logger log, File tempIndexDirectory, Wagon wagon )
 296         {
 297             this.log = log;
 298             this.tempIndexDirectory = tempIndexDirectory;
 299             this.wagon = wagon;
 300         }
 301
 302         public void connect( String id, String url )
 303             throws IOException
 304         {
 305             //no op
 306         }
 307
 308         public void disconnect()
 309             throws IOException
 310         {
 311             // no op
 312         }
 313
 314         public InputStream retrieve( String name )
 315             throws IOException, FileNotFoundException
 316         {
 317             try
 318             {
 319                 log.info( "index update retrieve file, name:{}", name );
 320                 File file = new File( tempIndexDirectory, name );
 321                 if ( file.exists() )
 322                 {
 323                     file.delete();
 324                 }
 325                 file.deleteOnExit();
 326                 wagon.get( name, file );
 327                 return new FileInputStream( file );
 328             }
 329             catch ( AuthorizationException e )
 330             {
 331                 throw new IOException( e.getMessage() );
 332             }
 333             catch ( TransferFailedException e )
 334             {
 335                 throw new IOException( e.getMessage() );
 336             }
 337             catch ( ResourceDoesNotExistException e )
 338             {
 339                 throw new FileNotFoundException( e.getMessage() );
 340             }
 341         }
 342     }
 343 }
 344
 345