source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.scheduler.indexing;
   2 /*
   3  * Licensed to the Apache Software Foundation (ASF) under one
   4  * or more contributor license agreements.  See the NOTICE file
   5  * distributed with this work for additional information
   6  * regarding copyright ownership.  The ASF licenses this file
   7  * to you under the Apache License, Version 2.0 (the
   8  * "License"); you may not use this file except in compliance
   9  * with the License.  You may obtain a copy of the License at
  10  *
  11  *   http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing,
  14  * software distributed under the License is distributed on an
  15  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  16  * KIND, either express or implied.  See the License for the
  17  * specific language governing permissions and limitations
  18  * under the License.
  19  */
  20
  21 import org.apache.archiva.admin.model.RepositoryAdminException;
  22 import org.apache.archiva.admin.model.beans.NetworkProxy;
  23 import org.apache.archiva.admin.model.beans.RemoteRepository;
  24 import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin;
  25 import org.apache.archiva.proxy.common.WagonFactory;
  26 import org.apache.archiva.proxy.common.WagonFactoryException;
  27 import org.apache.archiva.proxy.common.WagonFactoryRequest;
  28 import org.apache.commons.io.FileUtils;
  29 import org.apache.commons.lang.time.StopWatch;
  30 import org.apache.maven.index.context.IndexingContext;
  31 import org.apache.maven.index.updater.IndexUpdateRequest;
  32 import org.apache.maven.index.updater.IndexUpdater;
  33 import org.apache.maven.index.updater.ResourceFetcher;
  34 import org.apache.maven.wagon.ConnectionException;
  35 import org.apache.maven.wagon.ResourceDoesNotExistException;
  36 import org.apache.maven.wagon.StreamWagon;
  37 import org.apache.maven.wagon.TransferFailedException;
  38 import org.apache.maven.wagon.Wagon;
  39 import org.apache.maven.wagon.authentication.AuthenticationException;
  40 import org.apache.maven.wagon.authentication.AuthenticationInfo;
  41 import org.apache.maven.wagon.authorization.AuthorizationException;
  42 import org.apache.maven.wagon.events.TransferEvent;
  43 import org.apache.maven.wagon.events.TransferListener;
  44 import org.apache.maven.wagon.providers.http.AbstractHttpClientWagon;
  45 import org.apache.maven.wagon.providers.http.HttpConfiguration;
  46 import org.apache.maven.wagon.providers.http.HttpMethodConfiguration;
  47 import org.apache.maven.wagon.proxy.ProxyInfo;
  48 import org.apache.maven.wagon.repository.Repository;
  49 import org.slf4j.Logger;
  50 import org.slf4j.LoggerFactory;
  51
  52 import java.io.File;
  53 import java.io.FileInputStream;
  54 import java.io.FileNotFoundException;
  55 import java.io.IOException;
  56 import java.io.InputStream;
  57 import java.net.MalformedURLException;
  58 import java.net.URL;
  59 import java.nio.file.Files;
  60 import java.util.List;
  61 import java.util.Map;
  62
  63 /**
  64  * @author Olivier Lamy
  65  * @since 1.4-M1
  66  */
  67 public class DownloadRemoteIndexTask
  68     implements Runnable
  69 {
  70     private Logger log = LoggerFactory.getLogger( getClass() );
  71
  72     private RemoteRepository remoteRepository;
  73
  74     private RemoteRepositoryAdmin remoteRepositoryAdmin;
  75
  76     private WagonFactory wagonFactory;
  77
  78     private NetworkProxy networkProxy;
  79
  80     private boolean fullDownload;
  81
  82     private List<String> runningRemoteDownloadIds;
  83
  84     private IndexUpdater indexUpdater;
  85
  86
  87     public DownloadRemoteIndexTask( DownloadRemoteIndexTaskRequest downloadRemoteIndexTaskRequest,
  88                                     List<String> runningRemoteDownloadIds )
  89     {
  90         this.remoteRepository = downloadRemoteIndexTaskRequest.getRemoteRepository();
  91         this.wagonFactory = downloadRemoteIndexTaskRequest.getWagonFactory();
  92         this.networkProxy = downloadRemoteIndexTaskRequest.getNetworkProxy();
  93         this.fullDownload = downloadRemoteIndexTaskRequest.isFullDownload();
  94         this.runningRemoteDownloadIds = runningRemoteDownloadIds;
  95         this.indexUpdater = downloadRemoteIndexTaskRequest.getIndexUpdater();
  96         this.remoteRepositoryAdmin = downloadRemoteIndexTaskRequest.getRemoteRepositoryAdmin();
  97     }
  98
  99     @Override
 100     public void run()
 101     {
 102
 103         // so short lock : not sure we need it
 104         synchronized ( this.runningRemoteDownloadIds )
 105         {
 106             if ( this.runningRemoteDownloadIds.contains( this.remoteRepository.getId() ) )
 107             {
 108                 // skip it as it's running
 109                 log.info( "skip download index remote for repo {} it's already running",
 110                           this.remoteRepository.getId() );
 111                 return;
 112             }
 113             this.runningRemoteDownloadIds.add( this.remoteRepository.getId() );
 114         }
 115         File tempIndexDirectory = null;
 116         StopWatch stopWatch = new StopWatch();
 117         stopWatch.start();
 118         try
 119         {
 120             log.info( "start download remote index for remote repository {}", this.remoteRepository.getId() );
 121             IndexingContext indexingContext = remoteRepositoryAdmin.createIndexContext( this.remoteRepository );
 122
 123             // create a temp directory to download files
 124             tempIndexDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".tmpIndex" );
 125             File indexCacheDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".indexCache" );
 126             indexCacheDirectory.mkdirs();
 127             if ( tempIndexDirectory.exists() )
 128             {
 129                 FileUtils.deleteDirectory( tempIndexDirectory );
 130             }
 131             tempIndexDirectory.mkdirs();
 132             tempIndexDirectory.deleteOnExit();
 133             String baseIndexUrl = indexingContext.getIndexUpdateUrl();
 134
 135             String wagonProtocol = new URL( this.remoteRepository.getUrl() ).getProtocol();
 136
 137             final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon(
 138                 new WagonFactoryRequest( wagonProtocol, this.remoteRepository.getExtraHeaders() ).networkProxy(
 139                     this.networkProxy )
 140             );
 141             // FIXME olamy having 2 config values
 142             wagon.setReadTimeout( remoteRepository.getRemoteDownloadTimeout() * 1000 );
 143             wagon.setTimeout( remoteRepository.getTimeout() * 1000 );
 144
 145             if ( wagon instanceof AbstractHttpClientWagon )
 146             {
 147                 HttpConfiguration httpConfiguration = new HttpConfiguration();
 148                 HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration();
 149                 httpMethodConfiguration.setUsePreemptive( true );
 150                 httpMethodConfiguration.setReadTimeout( remoteRepository.getRemoteDownloadTimeout() * 1000 );
 151                 httpConfiguration.setGet( httpMethodConfiguration );
 152                 AbstractHttpClientWagon.class.cast( wagon ).setHttpConfiguration( httpConfiguration );
 153             }
 154
 155             wagon.addTransferListener( new DownloadListener() );
 156             ProxyInfo proxyInfo = null;
 157             if ( this.networkProxy != null )
 158             {
 159                 proxyInfo = new ProxyInfo();
 160                 proxyInfo.setHost( this.networkProxy.getHost() );
 161                 proxyInfo.setPort( this.networkProxy.getPort() );
 162                 proxyInfo.setUserName( this.networkProxy.getUsername() );
 163                 proxyInfo.setPassword( this.networkProxy.getPassword() );
 164             }
 165             AuthenticationInfo authenticationInfo = null;
 166             if ( this.remoteRepository.getUserName() != null )
 167             {
 168                 authenticationInfo = new AuthenticationInfo();
 169                 authenticationInfo.setUserName( this.remoteRepository.getUserName() );
 170                 authenticationInfo.setPassword( this.remoteRepository.getPassword() );
 171             }
 172             wagon.connect( new Repository( this.remoteRepository.getId(), baseIndexUrl ), authenticationInfo,
 173                            proxyInfo );
 174
 175             File indexDirectory = indexingContext.getIndexDirectoryFile();
 176             if ( !indexDirectory.exists() )
 177             {
 178                 indexDirectory.mkdirs();
 179             }
 180
 181             ResourceFetcher resourceFetcher =
 182                 new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository );
 183             IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
 184             request.setForceFullUpdate( this.fullDownload );
 185             request.setLocalIndexCacheDir( indexCacheDirectory );
 186
 187             this.indexUpdater.fetchAndUpdateIndex( request );
 188             stopWatch.stop();
 189             log.info( "time update index from remote for repository {}: {} s", this.remoteRepository.getId(),
 190                       ( stopWatch.getTime() / 1000 ) );
 191
 192             // index packing optionnal ??
 193             //IndexPackingRequest indexPackingRequest =
 194             //    new IndexPackingRequest( indexingContext, indexingContext.getIndexDirectoryFile() );
 195             //indexPacker.packIndex( indexPackingRequest );
 196             indexingContext.updateTimestamp( true );
 197
 198         }
 199         catch ( MalformedURLException e )
 200         {
 201             log.error( e.getMessage(), e );
 202             throw new RuntimeException( e.getMessage(), e );
 203         }
 204         catch ( WagonFactoryException e )
 205         {
 206             log.error( e.getMessage(), e );
 207             throw new RuntimeException( e.getMessage(), e );
 208         }
 209         catch ( ConnectionException e )
 210         {
 211             log.error( e.getMessage(), e );
 212             throw new RuntimeException( e.getMessage(), e );
 213         }
 214         catch ( AuthenticationException e )
 215         {
 216             log.error( e.getMessage(), e );
 217             throw new RuntimeException( e.getMessage(), e );
 218         }
 219         catch ( IOException e )
 220         {
 221             log.error( e.getMessage(), e );
 222             throw new RuntimeException( e.getMessage(), e );
 223         }
 224         catch ( RepositoryAdminException e )
 225         {
 226             log.error( e.getMessage(), e );
 227             throw new RuntimeException( e.getMessage(), e );
 228         }
 229         finally
 230         {
 231             deleteDirectoryQuiet( tempIndexDirectory );
 232             this.runningRemoteDownloadIds.remove( this.remoteRepository.getId() );
 233         }
 234         log.info( "end download remote index for remote repository {}", this.remoteRepository.getId() );
 235     }
 236
 237     private void deleteDirectoryQuiet( File f )
 238     {
 239         try
 240         {
 241             FileUtils.deleteDirectory( f );
 242         }
 243         catch ( IOException e )
 244         {
 245             log.warn( "skip error delete {} : {}", f, e.getMessage() );
 246         }
 247     }
 248
 249
 250     private static final class DownloadListener
 251         implements TransferListener
 252     {
 253         private Logger log = LoggerFactory.getLogger( getClass() );
 254
 255         private String resourceName;
 256
 257         private long startTime;
 258
 259         private int totalLength = 0;
 260
 261         @Override
 262         public void transferInitiated( TransferEvent transferEvent )
 263         {
 264             startTime = System.currentTimeMillis();
 265             resourceName = transferEvent.getResource().getName();
 266             log.debug( "initiate transfer of {}", resourceName );
 267         }
 268
 269         @Override
 270         public void transferStarted( TransferEvent transferEvent )
 271         {
 272             this.totalLength = 0;
 273             resourceName = transferEvent.getResource().getName();
 274             log.info( "start transfer of {}", transferEvent.getResource().getName() );
 275         }
 276
 277         @Override
 278         public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
 279         {
 280             log.debug( "transfer of {} : {}/{}", transferEvent.getResource().getName(), buffer.length, length );
 281             this.totalLength += length;
 282         }
 283
 284         @Override
 285         public void transferCompleted( TransferEvent transferEvent )
 286         {
 287             resourceName = transferEvent.getResource().getName();
 288             long endTime = System.currentTimeMillis();
 289             log.info( "end of transfer file {} {} kb: {}s", transferEvent.getResource().getName(),
 290                       this.totalLength / 1024, ( endTime - startTime ) / 1000 );
 291         }
 292
 293         @Override
 294         public void transferError( TransferEvent transferEvent )
 295         {
 296             log.info( "error of transfer file {}: {}", transferEvent.getResource().getName(),
 297                       transferEvent.getException().getMessage(), transferEvent.getException() );
 298         }
 299
 300         @Override
 301         public void debug( String message )
 302         {
 303             log.debug( "transfer debug {}", message );
 304         }
 305     }
 306
 307     private static class WagonResourceFetcher
 308         implements ResourceFetcher
 309     {
 310
 311         Logger log;
 312
 313         File tempIndexDirectory;
 314
 315         Wagon wagon;
 316
 317         RemoteRepository remoteRepository;
 318
 319         private WagonResourceFetcher( Logger log, File tempIndexDirectory, Wagon wagon,
 320                                       RemoteRepository remoteRepository )
 321         {
 322             this.log = log;
 323             this.tempIndexDirectory = tempIndexDirectory;
 324             this.wagon = wagon;
 325             this.remoteRepository = remoteRepository;
 326         }
 327
 328         @Override
 329         public void connect( String id, String url )
 330             throws IOException
 331         {
 332             //no op
 333         }
 334
 335         @Override
 336         public void disconnect()
 337             throws IOException
 338         {
 339             // no op
 340         }
 341
 342         @Override
 343         public InputStream retrieve( String name )
 344             throws IOException, FileNotFoundException
 345         {
 346             try
 347             {
 348                 log.info( "index update retrieve file, name:{}", name );
 349                 File file = new File( tempIndexDirectory, name );
 350                 Files.deleteIfExists( file.toPath() );
 351                 file.deleteOnExit();
 352                 wagon.get( addParameters( name, this.remoteRepository ), file );
 353                 return Files.newInputStream( file.toPath() );
 354             }
 355             catch ( AuthorizationException e )
 356             {
 357                 throw new IOException( e.getMessage(), e );
 358             }
 359             catch ( TransferFailedException e )
 360             {
 361                 throw new IOException( e.getMessage(), e );
 362             }
 363             catch ( ResourceDoesNotExistException e )
 364             {
 365                 FileNotFoundException fnfe = new FileNotFoundException( e.getMessage() );
 366                 fnfe.initCause( e );
 367                 throw fnfe;
 368             }
 369         }
 370
 371         // FIXME remove crappy copy/paste
 372         protected String addParameters( String path, RemoteRepository remoteRepository )
 373         {
 374             if ( remoteRepository.getExtraParameters().isEmpty() )
 375             {
 376                 return path;
 377             }
 378
 379             boolean question = false;
 380
 381             StringBuilder res = new StringBuilder( path == null ? "" : path );
 382
 383             for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() )
 384             {
 385                 if ( !question )
 386                 {
 387                     res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() );
 388                 }
 389             }
 390
 391             return res.toString();
 392         }
 393
 394     }
 395
 396
 397 }
 398