source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.scheduler.indexing;
   2 /*
   3  * Licensed to the Apache Software Foundation (ASF) under one
   4  * or more contributor license agreements.  See the NOTICE file
   5  * distributed with this work for additional information
   6  * regarding copyright ownership.  The ASF licenses this file
   7  * to you under the Apache License, Version 2.0 (the
   8  * "License"); you may not use this file except in compliance
   9  * with the License.  You may obtain a copy of the License at
  10  *
  11  *   http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing,
  14  * software distributed under the License is distributed on an
  15  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  16  * KIND, either express or implied.  See the License for the
  17  * specific language governing permissions and limitations
  18  * under the License.
  19  */
  20
  21 import org.apache.archiva.admin.model.RepositoryAdminException;
  22 import org.apache.archiva.admin.model.beans.NetworkProxy;
  23 import org.apache.archiva.admin.model.beans.RemoteRepository;
  24 import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin;
  25 import org.apache.archiva.proxy.common.WagonFactory;
  26 import org.apache.archiva.proxy.common.WagonFactoryException;
  27 import org.apache.archiva.proxy.common.WagonFactoryRequest;
  28 import org.apache.commons.io.FileUtils;
  29 import org.apache.commons.lang.time.StopWatch;
  30 import org.apache.maven.index.context.IndexingContext;
  31 import org.apache.maven.index.packer.IndexPacker;
  32 import org.apache.maven.index.packer.IndexPackingRequest;
  33 import org.apache.maven.index.updater.IndexUpdateRequest;
  34 import org.apache.maven.index.updater.IndexUpdater;
  35 import org.apache.maven.index.updater.ResourceFetcher;
  36 import org.apache.maven.wagon.ConnectionException;
  37 import org.apache.maven.wagon.ResourceDoesNotExistException;
  38 import org.apache.maven.wagon.StreamWagon;
  39 import org.apache.maven.wagon.TransferFailedException;
  40 import org.apache.maven.wagon.Wagon;
  41 import org.apache.maven.wagon.authentication.AuthenticationException;
  42 import org.apache.maven.wagon.authentication.AuthenticationInfo;
  43 import org.apache.maven.wagon.authorization.AuthorizationException;
  44 import org.apache.maven.wagon.events.TransferEvent;
  45 import org.apache.maven.wagon.events.TransferListener;
  46 import org.apache.maven.wagon.proxy.ProxyInfo;
  47 import org.apache.maven.wagon.repository.Repository;
  48 import org.slf4j.Logger;
  49 import org.slf4j.LoggerFactory;
  50
  51 import java.io.File;
  52 import java.io.FileInputStream;
  53 import java.io.FileNotFoundException;
  54 import java.io.IOException;
  55 import java.io.InputStream;
  56 import java.net.MalformedURLException;
  57 import java.net.URL;
  58 import java.util.Arrays;
  59 import java.util.List;
  60 import java.util.Map;
  61
  62 /**
  63  * @author Olivier Lamy
  64  * @since 1.4-M1
  65  */
  66 public class DownloadRemoteIndexTask
  67     implements Runnable
  68 {
  69     private Logger log = LoggerFactory.getLogger( getClass() );
  70
  71     private RemoteRepository remoteRepository;
  72
  73     private RemoteRepositoryAdmin remoteRepositoryAdmin;
  74
  75     private WagonFactory wagonFactory;
  76
  77     private NetworkProxy networkProxy;
  78
  79     private boolean fullDownload;
  80
  81     private List<String> runningRemoteDownloadIds;
  82
  83     private IndexUpdater indexUpdater;
  84
  85     private IndexPacker indexPacker;
  86
  87     public DownloadRemoteIndexTask( DownloadRemoteIndexTaskRequest downloadRemoteIndexTaskRequest,
  88                                     List<String> runningRemoteDownloadIds )
  89     {
  90         this.remoteRepository = downloadRemoteIndexTaskRequest.getRemoteRepository();
  91         this.wagonFactory = downloadRemoteIndexTaskRequest.getWagonFactory();
  92         this.networkProxy = downloadRemoteIndexTaskRequest.getNetworkProxy();
  93         this.fullDownload = downloadRemoteIndexTaskRequest.isFullDownload();
  94         this.runningRemoteDownloadIds = runningRemoteDownloadIds;
  95         this.indexUpdater = downloadRemoteIndexTaskRequest.getIndexUpdater();
  96         this.remoteRepositoryAdmin = downloadRemoteIndexTaskRequest.getRemoteRepositoryAdmin();
  97         this.indexPacker = downloadRemoteIndexTaskRequest.getIndexPacker();
  98     }
  99
 100     public void run()
 101     {
 102
 103         // so short lock : not sure we need it
 104         synchronized ( this.runningRemoteDownloadIds )
 105         {
 106             if ( this.runningRemoteDownloadIds.contains( this.remoteRepository.getId() ) )
 107             {
 108                 // skip it as it's running
 109                 log.info( "skip download index remote for repo {} it's already running",
 110                           this.remoteRepository.getId() );
 111                 return;
 112             }
 113             this.runningRemoteDownloadIds.add( this.remoteRepository.getId() );
 114         }
 115         File tempIndexDirectory = null;
 116         StopWatch stopWatch = new StopWatch();
 117         stopWatch.start();
 118         try
 119         {
 120             log.info( "start download remote index for remote repository {}", this.remoteRepository.getId() );
 121             IndexingContext indexingContext = remoteRepositoryAdmin.createIndexContext( this.remoteRepository );
 122
 123             // create a temp directory to download files
 124             tempIndexDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".tmpIndex" );
 125             File indexCacheDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".indexCache" );
 126             indexCacheDirectory.mkdirs();
 127             if ( tempIndexDirectory.exists() )
 128             {
 129                 FileUtils.deleteDirectory( tempIndexDirectory );
 130             }
 131             tempIndexDirectory.mkdirs();
 132             tempIndexDirectory.deleteOnExit();
 133             String baseIndexUrl = indexingContext.getIndexUpdateUrl();
 134
 135             String wagonProtocol =
 136                 new URL( this.remoteRepository.getUrl() ).getProtocol() + ( ( this.networkProxy != null
 137                     && this.networkProxy.isUseNtlm() ) ? "-ntlm" : "" );
 138
 139             final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon(
 140                 new WagonFactoryRequest( wagonProtocol, this.remoteRepository.getExtraHeaders() ) );
 141             int timeoutInMilliseconds = remoteRepository.getTimeout() * 1000;
 142             // FIXME olamy having 2 config values
 143             wagon.setReadTimeout( timeoutInMilliseconds );
 144             wagon.setTimeout( timeoutInMilliseconds );
 145
 146             wagon.addTransferListener( new DownloadListener() );
 147             ProxyInfo proxyInfo = null;
 148             if ( this.networkProxy != null )
 149             {
 150                 proxyInfo = new ProxyInfo();
 151                 proxyInfo.setHost( this.networkProxy.getHost() );
 152                 proxyInfo.setPort( this.networkProxy.getPort() );
 153                 proxyInfo.setUserName( this.networkProxy.getUsername() );
 154                 proxyInfo.setPassword( this.networkProxy.getPassword() );
 155             }
 156             AuthenticationInfo authenticationInfo = null;
 157             if ( this.remoteRepository.getUserName() != null )
 158             {
 159                 authenticationInfo = new AuthenticationInfo();
 160                 authenticationInfo.setUserName( this.remoteRepository.getUserName() );
 161                 authenticationInfo.setPassword( this.remoteRepository.getPassword() );
 162             }
 163             wagon.connect( new Repository( this.remoteRepository.getId(), baseIndexUrl ), authenticationInfo,
 164                            proxyInfo );
 165
 166             File indexDirectory = indexingContext.getIndexDirectoryFile();
 167             if ( !indexDirectory.exists() )
 168             {
 169                 indexDirectory.mkdirs();
 170             }
 171
 172             ResourceFetcher resourceFetcher =
 173                 new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository );
 174             IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
 175             request.setForceFullUpdate( this.fullDownload );
 176             request.setLocalIndexCacheDir( indexCacheDirectory );
 177
 178             this.indexUpdater.fetchAndUpdateIndex( request );
 179             stopWatch.stop();
 180             log.info( "time to download remote repository index for repository {}: {} s", this.remoteRepository.getId(),
 181                       ( stopWatch.getTime() / 1000 ) );
 182
 183             // index packing optionnal ??
 184             IndexPackingRequest indexPackingRequest =
 185                 new IndexPackingRequest( indexingContext, indexingContext.getIndexDirectoryFile() );
 186             indexPacker.packIndex( indexPackingRequest );
 187             indexingContext.updateTimestamp( true );
 188
 189         }
 190         catch ( MalformedURLException e )
 191         {
 192             log.error( e.getMessage(), e );
 193             throw new RuntimeException( e.getMessage(), e );
 194         }
 195         catch ( WagonFactoryException e )
 196         {
 197             log.error( e.getMessage(), e );
 198             throw new RuntimeException( e.getMessage(), e );
 199         }
 200         catch ( ConnectionException e )
 201         {
 202             log.error( e.getMessage(), e );
 203             throw new RuntimeException( e.getMessage(), e );
 204         }
 205         catch ( AuthenticationException e )
 206         {
 207             log.error( e.getMessage(), e );
 208             throw new RuntimeException( e.getMessage(), e );
 209         }
 210         catch ( IOException e )
 211         {
 212             log.error( e.getMessage(), e );
 213             throw new RuntimeException( e.getMessage(), e );
 214         }
 215         catch ( RepositoryAdminException e )
 216         {
 217             log.error( e.getMessage(), e );
 218             throw new RuntimeException( e.getMessage(), e );
 219         }
 220         finally
 221         {
 222             deleteDirectoryQuiet( tempIndexDirectory );
 223             this.runningRemoteDownloadIds.remove( this.remoteRepository.getId() );
 224         }
 225         log.info( "end download remote index for remote repository " + this.remoteRepository.getId() );
 226     }
 227
 228     private void deleteDirectoryQuiet( File f )
 229     {
 230         try
 231         {
 232             FileUtils.deleteDirectory( f );
 233         }
 234         catch ( IOException e )
 235         {
 236             log.warn( "skip error delete {} : {}", f, e.getMessage() );
 237         }
 238     }
 239
 240
 241     public static class DownloadListener
 242         implements TransferListener
 243     {
 244         private Logger log = LoggerFactory.getLogger( getClass() );
 245
 246         String resourceName;
 247
 248         long startTime;
 249
 250         public void transferInitiated( TransferEvent transferEvent )
 251         {
 252             resourceName = transferEvent.getResource().getName();
 253             log.debug( "initiate transfer of {}", resourceName );
 254         }
 255
 256         public void transferStarted( TransferEvent transferEvent )
 257         {
 258             resourceName = transferEvent.getResource().getName();
 259             startTime = System.currentTimeMillis();
 260             log.info( "start transfer of {}", transferEvent.getResource().getName() );
 261         }
 262
 263         public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
 264         {
 265             log.debug( "transfer of {} : {}/{}",
 266                        Arrays.asList( transferEvent.getResource().getName(), buffer.length, length ).toArray() );
 267         }
 268
 269         public void transferCompleted( TransferEvent transferEvent )
 270         {
 271             resourceName = transferEvent.getResource().getName();
 272             long endTime = System.currentTimeMillis();
 273             log.info( "end of transfer file {}: {}s", transferEvent.getResource().getName(),
 274                       ( endTime - startTime ) / 1000 );
 275         }
 276
 277         public void transferError( TransferEvent transferEvent )
 278         {
 279             log.info( "error of transfer file {}: {}", Arrays.asList( transferEvent.getResource().getName(),
 280                                                                       transferEvent.getException().getMessage() ).toArray(
 281                 new Object[2] ), transferEvent.getException() );
 282         }
 283
 284         public void debug( String message )
 285         {
 286             log.debug( "transfer debug {}", message );
 287         }
 288     }
 289
 290     private static class WagonResourceFetcher
 291         implements ResourceFetcher
 292     {
 293
 294         Logger log;
 295
 296         File tempIndexDirectory;
 297
 298         Wagon wagon;
 299
 300         RemoteRepository remoteRepository;
 301
 302         private WagonResourceFetcher( Logger log, File tempIndexDirectory, Wagon wagon,
 303                                       RemoteRepository remoteRepository )
 304         {
 305             this.log = log;
 306             this.tempIndexDirectory = tempIndexDirectory;
 307             this.wagon = wagon;
 308             this.remoteRepository = remoteRepository;
 309         }
 310
 311         public void connect( String id, String url )
 312             throws IOException
 313         {
 314             //no op
 315         }
 316
 317         public void disconnect()
 318             throws IOException
 319         {
 320             // no op
 321         }
 322
 323         public InputStream retrieve( String name )
 324             throws IOException, FileNotFoundException
 325         {
 326             try
 327             {
 328                 log.info( "index update retrieve file, name:{}", name );
 329                 File file = new File( tempIndexDirectory, name );
 330                 if ( file.exists() )
 331                 {
 332                     file.delete();
 333                 }
 334                 file.deleteOnExit();
 335                 wagon.get( addParameters( name, this.remoteRepository ), file );
 336                 return new FileInputStream( file );
 337             }
 338             catch ( AuthorizationException e )
 339             {
 340                 throw new IOException( e.getMessage() );
 341             }
 342             catch ( TransferFailedException e )
 343             {
 344                 throw new IOException( e.getMessage() );
 345             }
 346             catch ( ResourceDoesNotExistException e )
 347             {
 348                 throw new FileNotFoundException( e.getMessage() );
 349             }
 350         }
 351
 352         // FIXME remove crappy copy/paste
 353         protected String addParameters( String path, RemoteRepository remoteRepository )
 354         {
 355             if ( remoteRepository.getExtraParameters().isEmpty() )
 356             {
 357                 return path;
 358             }
 359
 360             boolean question = false;
 361
 362             StringBuilder res = new StringBuilder( path == null ? "" : path );
 363
 364             for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() )
 365             {
 366                 if ( !question )
 367                 {
 368                     res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() );
 369                 }
 370             }
 371
 372             return res.toString();
 373         }
 374
 375     }
 376
 377
 378 }
 379
 380