source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.scheduler.indexing;
   2 /*
   3  * Licensed to the Apache Software Foundation (ASF) under one
   4  * or more contributor license agreements.  See the NOTICE file
   5  * distributed with this work for additional information
   6  * regarding copyright ownership.  The ASF licenses this file
   7  * to you under the Apache License, Version 2.0 (the
   8  * "License"); you may not use this file except in compliance
   9  * with the License.  You may obtain a copy of the License at
  10  *
  11  *   http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing,
  14  * software distributed under the License is distributed on an
  15  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  16  * KIND, either express or implied.  See the License for the
  17  * specific language governing permissions and limitations
  18  * under the License.
  19  */
  20
  21 import org.apache.archiva.admin.model.beans.NetworkProxy;
  22 import org.apache.archiva.admin.model.beans.RemoteRepository;
  23 import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin;
  24 import org.apache.archiva.proxy.common.WagonFactory;
  25 import org.apache.archiva.proxy.common.WagonFactoryRequest;
  26 import org.apache.commons.io.FileUtils;
  27 import org.apache.commons.lang.time.StopWatch;
  28 import org.apache.maven.index.context.IndexingContext;
  29 import org.apache.maven.index.updater.IndexUpdateRequest;
  30 import org.apache.maven.index.updater.IndexUpdater;
  31 import org.apache.maven.index.updater.ResourceFetcher;
  32 import org.apache.maven.wagon.ResourceDoesNotExistException;
  33 import org.apache.maven.wagon.StreamWagon;
  34 import org.apache.maven.wagon.TransferFailedException;
  35 import org.apache.maven.wagon.Wagon;
  36 import org.apache.maven.wagon.authentication.AuthenticationInfo;
  37 import org.apache.maven.wagon.authorization.AuthorizationException;
  38 import org.apache.maven.wagon.events.TransferEvent;
  39 import org.apache.maven.wagon.events.TransferListener;
  40 import org.apache.maven.wagon.proxy.ProxyInfo;
  41 import org.apache.maven.wagon.repository.Repository;
  42 import org.apache.maven.wagon.shared.http.AbstractHttpClientWagon;
  43 import org.apache.maven.wagon.shared.http.HttpConfiguration;
  44 import org.apache.maven.wagon.shared.http.HttpMethodConfiguration;
  45 import org.slf4j.Logger;
  46 import org.slf4j.LoggerFactory;
  47
  48 import java.io.File;
  49 import java.io.FileNotFoundException;
  50 import java.io.IOException;
  51 import java.io.InputStream;
  52 import java.net.URL;
  53 import java.nio.file.Files;
  54 import java.util.List;
  55 import java.util.Map;
  56
  57 /**
  58  * @author Olivier Lamy
  59  * @since 1.4-M1
  60  */
  61 public class DownloadRemoteIndexTask
  62     implements Runnable
  63 {
  64     private Logger log = LoggerFactory.getLogger( getClass() );
  65
  66     private RemoteRepository remoteRepository;
  67
  68     private RemoteRepositoryAdmin remoteRepositoryAdmin;
  69
  70     private WagonFactory wagonFactory;
  71
  72     private NetworkProxy networkProxy;
  73
  74     private boolean fullDownload;
  75
  76     private List<String> runningRemoteDownloadIds;
  77
  78     private IndexUpdater indexUpdater;
  79
  80
  81     public DownloadRemoteIndexTask( DownloadRemoteIndexTaskRequest downloadRemoteIndexTaskRequest,
  82                                     List<String> runningRemoteDownloadIds )
  83     {
  84         this.remoteRepository = downloadRemoteIndexTaskRequest.getRemoteRepository();
  85         this.wagonFactory = downloadRemoteIndexTaskRequest.getWagonFactory();
  86         this.networkProxy = downloadRemoteIndexTaskRequest.getNetworkProxy();
  87         this.fullDownload = downloadRemoteIndexTaskRequest.isFullDownload();
  88         this.runningRemoteDownloadIds = runningRemoteDownloadIds;
  89         this.indexUpdater = downloadRemoteIndexTaskRequest.getIndexUpdater();
  90         this.remoteRepositoryAdmin = downloadRemoteIndexTaskRequest.getRemoteRepositoryAdmin();
  91     }
  92
  93     @Override
  94     public void run()
  95     {
  96
  97         // so short lock : not sure we need it
  98         synchronized ( this.runningRemoteDownloadIds )
  99         {
 100             if ( this.runningRemoteDownloadIds.contains( this.remoteRepository.getId() ) )
 101             {
 102                 // skip it as it's running
 103                 log.info( "skip download index remote for repo {} it's already running",
 104                           this.remoteRepository.getId() );
 105                 return;
 106             }
 107             this.runningRemoteDownloadIds.add( this.remoteRepository.getId() );
 108         }
 109         File tempIndexDirectory = null;
 110         StopWatch stopWatch = new StopWatch();
 111         stopWatch.start();
 112         try
 113         {
 114             log.info( "start download remote index for remote repository {}", this.remoteRepository.getId() );
 115             IndexingContext indexingContext = remoteRepositoryAdmin.createIndexContext( this.remoteRepository );
 116
 117             // create a temp directory to download files
 118             tempIndexDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".tmpIndex" );
 119             File indexCacheDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".indexCache" );
 120             indexCacheDirectory.mkdirs();
 121             if ( tempIndexDirectory.exists() )
 122             {
 123                 FileUtils.deleteDirectory( tempIndexDirectory );
 124             }
 125             tempIndexDirectory.mkdirs();
 126             tempIndexDirectory.deleteOnExit();
 127             String baseIndexUrl = indexingContext.getIndexUpdateUrl();
 128
 129             String wagonProtocol = new URL( this.remoteRepository.getUrl() ).getProtocol();
 130
 131             final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon(
 132                 new WagonFactoryRequest( wagonProtocol, this.remoteRepository.getExtraHeaders() ).networkProxy(
 133                     this.networkProxy )
 134             );
 135             // FIXME olamy having 2 config values
 136             wagon.setReadTimeout( remoteRepository.getRemoteDownloadTimeout() * 1000 );
 137             wagon.setTimeout( remoteRepository.getTimeout() * 1000 );
 138
 139             if ( wagon instanceof AbstractHttpClientWagon )
 140             {
 141                 HttpConfiguration httpConfiguration = new HttpConfiguration();
 142                 HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration();
 143                 httpMethodConfiguration.setUsePreemptive( true );
 144                 httpMethodConfiguration.setReadTimeout( remoteRepository.getRemoteDownloadTimeout() * 1000 );
 145                 httpConfiguration.setGet( httpMethodConfiguration );
 146                 AbstractHttpClientWagon.class.cast( wagon ).setHttpConfiguration( httpConfiguration );
 147             }
 148
 149             wagon.addTransferListener( new DownloadListener() );
 150             ProxyInfo proxyInfo = null;
 151             if ( this.networkProxy != null )
 152             {
 153                 proxyInfo = new ProxyInfo();
 154                 proxyInfo.setType( this.networkProxy.getProtocol() );
 155                 proxyInfo.setHost( this.networkProxy.getHost() );
 156                 proxyInfo.setPort( this.networkProxy.getPort() );
 157                 proxyInfo.setUserName( this.networkProxy.getUsername() );
 158                 proxyInfo.setPassword( this.networkProxy.getPassword() );
 159             }
 160             AuthenticationInfo authenticationInfo = null;
 161             if ( this.remoteRepository.getUserName() != null )
 162             {
 163                 authenticationInfo = new AuthenticationInfo();
 164                 authenticationInfo.setUserName( this.remoteRepository.getUserName() );
 165                 authenticationInfo.setPassword( this.remoteRepository.getPassword() );
 166             }
 167             wagon.connect( new Repository( this.remoteRepository.getId(), baseIndexUrl ), authenticationInfo,
 168                            proxyInfo );
 169
 170             File indexDirectory = indexingContext.getIndexDirectoryFile();
 171             if ( !indexDirectory.exists() )
 172             {
 173                 indexDirectory.mkdirs();
 174             }
 175
 176             ResourceFetcher resourceFetcher =
 177                 new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository );
 178             IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
 179             request.setForceFullUpdate( this.fullDownload );
 180             request.setLocalIndexCacheDir( indexCacheDirectory );
 181
 182             this.indexUpdater.fetchAndUpdateIndex( request );
 183             stopWatch.stop();
 184             log.info( "time update index from remote for repository {}: {} s", this.remoteRepository.getId(),
 185                       ( stopWatch.getTime() / 1000 ) );
 186
 187             // index packing optionnal ??
 188             //IndexPackingRequest indexPackingRequest =
 189             //    new IndexPackingRequest( indexingContext, indexingContext.getIndexDirectoryFile() );
 190             //indexPacker.packIndex( indexPackingRequest );
 191             indexingContext.updateTimestamp( true );
 192
 193         }
 194         catch ( Exception e )
 195         {
 196             log.error( e.getMessage(), e );
 197             throw new RuntimeException( e.getMessage(), e );
 198         }
 199         finally
 200         {
 201             deleteDirectoryQuiet( tempIndexDirectory );
 202             this.runningRemoteDownloadIds.remove( this.remoteRepository.getId() );
 203         }
 204         log.info( "end download remote index for remote repository {}", this.remoteRepository.getId() );
 205     }
 206
 207     private void deleteDirectoryQuiet( File f )
 208     {
 209         try
 210         {
 211             FileUtils.deleteDirectory( f );
 212         }
 213         catch ( IOException e )
 214         {
 215             log.warn( "skip error delete {} : {}", f, e.getMessage() );
 216         }
 217     }
 218
 219
 220     private static final class DownloadListener
 221         implements TransferListener
 222     {
 223         private Logger log = LoggerFactory.getLogger( getClass() );
 224
 225         private String resourceName;
 226
 227         private long startTime;
 228
 229         private int totalLength = 0;
 230
 231         @Override
 232         public void transferInitiated( TransferEvent transferEvent )
 233         {
 234             startTime = System.currentTimeMillis();
 235             resourceName = transferEvent.getResource().getName();
 236             log.debug( "initiate transfer of {}", resourceName );
 237         }
 238
 239         @Override
 240         public void transferStarted( TransferEvent transferEvent )
 241         {
 242             this.totalLength = 0;
 243             resourceName = transferEvent.getResource().getName();
 244             log.info( "start transfer of {}", transferEvent.getResource().getName() );
 245         }
 246
 247         @Override
 248         public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
 249         {
 250             log.debug( "transfer of {} : {}/{}", transferEvent.getResource().getName(), buffer.length, length );
 251             this.totalLength += length;
 252         }
 253
 254         @Override
 255         public void transferCompleted( TransferEvent transferEvent )
 256         {
 257             resourceName = transferEvent.getResource().getName();
 258             long endTime = System.currentTimeMillis();
 259             log.info( "end of transfer file {} {} kb: {}s", transferEvent.getResource().getName(),
 260                       this.totalLength / 1024, ( endTime - startTime ) / 1000 );
 261         }
 262
 263         @Override
 264         public void transferError( TransferEvent transferEvent )
 265         {
 266             log.info( "error of transfer file {}: {}", transferEvent.getResource().getName(),
 267                       transferEvent.getException().getMessage(), transferEvent.getException() );
 268         }
 269
 270         @Override
 271         public void debug( String message )
 272         {
 273             log.debug( "transfer debug {}", message );
 274         }
 275     }
 276
 277     private static class WagonResourceFetcher
 278         implements ResourceFetcher
 279     {
 280
 281         Logger log;
 282
 283         File tempIndexDirectory;
 284
 285         Wagon wagon;
 286
 287         RemoteRepository remoteRepository;
 288
 289         private WagonResourceFetcher( Logger log, File tempIndexDirectory, Wagon wagon,
 290                                       RemoteRepository remoteRepository )
 291         {
 292             this.log = log;
 293             this.tempIndexDirectory = tempIndexDirectory;
 294             this.wagon = wagon;
 295             this.remoteRepository = remoteRepository;
 296         }
 297
 298         @Override
 299         public void connect( String id, String url )
 300             throws IOException
 301         {
 302             //no op
 303         }
 304
 305         @Override
 306         public void disconnect()
 307             throws IOException
 308         {
 309             // no op
 310         }
 311
 312         @Override
 313         public InputStream retrieve( String name )
 314             throws IOException, FileNotFoundException
 315         {
 316             try
 317             {
 318                 log.info( "index update retrieve file, name:{}", name );
 319                 File file = new File( tempIndexDirectory, name );
 320                 Files.deleteIfExists( file.toPath() );
 321                 file.deleteOnExit();
 322                 wagon.get( addParameters( name, this.remoteRepository ), file );
 323                 return Files.newInputStream( file.toPath() );
 324             }
 325             catch ( AuthorizationException | TransferFailedException e )
 326             {
 327                 throw new IOException( e.getMessage(), e );
 328             }
 329             catch ( ResourceDoesNotExistException e )
 330             {
 331                 FileNotFoundException fnfe = new FileNotFoundException( e.getMessage() );
 332                 fnfe.initCause( e );
 333                 throw fnfe;
 334             }
 335         }
 336
 337         // FIXME remove crappy copy/paste
 338         protected String addParameters( String path, RemoteRepository remoteRepository )
 339         {
 340             if ( remoteRepository.getExtraParameters().isEmpty() )
 341             {
 342                 return path;
 343             }
 344
 345             boolean question = false;
 346
 347             StringBuilder res = new StringBuilder( path == null ? "" : path );
 348
 349             for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() )
 350             {
 351                 if ( !question )
 352                 {
 353                     res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() );
 354                 }
 355             }
 356
 357             return res.toString();
 358         }
 359
 360     }
 361
 362
 363 }
 364