source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.scheduler.indexing;
   2 /*
   3  * Licensed to the Apache Software Foundation (ASF) under one
   4  * or more contributor license agreements.  See the NOTICE file
   5  * distributed with this work for additional information
   6  * regarding copyright ownership.  The ASF licenses this file
   7  * to you under the Apache License, Version 2.0 (the
   8  * "License"); you may not use this file except in compliance
   9  * with the License.  You may obtain a copy of the License at
  10  *
  11  *   http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing,
  14  * software distributed under the License is distributed on an
  15  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  16  * KIND, either express or implied.  See the License for the
  17  * specific language governing permissions and limitations
  18  * under the License.
  19  */
  20
  21 import org.apache.archiva.admin.model.RepositoryAdminException;
  22 import org.apache.archiva.admin.model.beans.NetworkProxy;
  23 import org.apache.archiva.admin.model.beans.RemoteRepository;
  24 import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin;
  25 import org.apache.archiva.proxy.common.WagonFactory;
  26 import org.apache.archiva.proxy.common.WagonFactoryException;
  27 import org.apache.archiva.proxy.common.WagonFactoryRequest;
  28 import org.apache.commons.io.FileUtils;
  29 import org.apache.commons.lang.time.StopWatch;
  30 import org.apache.maven.index.context.IndexingContext;
  31 import org.apache.maven.index.packer.IndexPacker;
  32 import org.apache.maven.index.packer.IndexPackingRequest;
  33 import org.apache.maven.index.updater.IndexUpdateRequest;
  34 import org.apache.maven.index.updater.IndexUpdater;
  35 import org.apache.maven.index.updater.ResourceFetcher;
  36 import org.apache.maven.wagon.ConnectionException;
  37 import org.apache.maven.wagon.ResourceDoesNotExistException;
  38 import org.apache.maven.wagon.StreamWagon;
  39 import org.apache.maven.wagon.TransferFailedException;
  40 import org.apache.maven.wagon.Wagon;
  41 import org.apache.maven.wagon.authentication.AuthenticationException;
  42 import org.apache.maven.wagon.authentication.AuthenticationInfo;
  43 import org.apache.maven.wagon.authorization.AuthorizationException;
  44 import org.apache.maven.wagon.events.TransferEvent;
  45 import org.apache.maven.wagon.events.TransferListener;
  46 import org.apache.maven.wagon.proxy.ProxyInfo;
  47 import org.apache.maven.wagon.repository.Repository;
  48 import org.slf4j.Logger;
  49 import org.slf4j.LoggerFactory;
  50
  51 import java.io.File;
  52 import java.io.FileInputStream;
  53 import java.io.FileNotFoundException;
  54 import java.io.IOException;
  55 import java.io.InputStream;
  56 import java.net.MalformedURLException;
  57 import java.net.URL;
  58 import java.util.List;
  59 import java.util.Map;
  60
  61 /**
  62  * @author Olivier Lamy
  63  * @since 1.4-M1
  64  */
  65 public class DownloadRemoteIndexTask
  66     implements Runnable
  67 {
  68     private Logger log = LoggerFactory.getLogger( getClass() );
  69
  70     private RemoteRepository remoteRepository;
  71
  72     private RemoteRepositoryAdmin remoteRepositoryAdmin;
  73
  74     private WagonFactory wagonFactory;
  75
  76     private NetworkProxy networkProxy;
  77
  78     private boolean fullDownload;
  79
  80     private List<String> runningRemoteDownloadIds;
  81
  82     private IndexUpdater indexUpdater;
  83
  84     private IndexPacker indexPacker;
  85
  86     public DownloadRemoteIndexTask( DownloadRemoteIndexTaskRequest downloadRemoteIndexTaskRequest,
  87                                     List<String> runningRemoteDownloadIds )
  88     {
  89         this.remoteRepository = downloadRemoteIndexTaskRequest.getRemoteRepository();
  90         this.wagonFactory = downloadRemoteIndexTaskRequest.getWagonFactory();
  91         this.networkProxy = downloadRemoteIndexTaskRequest.getNetworkProxy();
  92         this.fullDownload = downloadRemoteIndexTaskRequest.isFullDownload();
  93         this.runningRemoteDownloadIds = runningRemoteDownloadIds;
  94         this.indexUpdater = downloadRemoteIndexTaskRequest.getIndexUpdater();
  95         this.remoteRepositoryAdmin = downloadRemoteIndexTaskRequest.getRemoteRepositoryAdmin();
  96         this.indexPacker = downloadRemoteIndexTaskRequest.getIndexPacker();
  97     }
  98
  99     public void run()
 100     {
 101
 102         // so short lock : not sure we need it
 103         synchronized ( this.runningRemoteDownloadIds )
 104         {
 105             if ( this.runningRemoteDownloadIds.contains( this.remoteRepository.getId() ) )
 106             {
 107                 // skip it as it's running
 108                 log.info( "skip download index remote for repo {} it's already running",
 109                           this.remoteRepository.getId() );
 110                 return;
 111             }
 112             this.runningRemoteDownloadIds.add( this.remoteRepository.getId() );
 113         }
 114         File tempIndexDirectory = null;
 115         StopWatch stopWatch = new StopWatch();
 116         stopWatch.start();
 117         try
 118         {
 119             log.info( "start download remote index for remote repository {}", this.remoteRepository.getId() );
 120             IndexingContext indexingContext = remoteRepositoryAdmin.createIndexContext( this.remoteRepository );
 121
 122             // create a temp directory to download files
 123             tempIndexDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".tmpIndex" );
 124             File indexCacheDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".indexCache" );
 125             indexCacheDirectory.mkdirs();
 126             if ( tempIndexDirectory.exists() )
 127             {
 128                 FileUtils.deleteDirectory( tempIndexDirectory );
 129             }
 130             tempIndexDirectory.mkdirs();
 131             tempIndexDirectory.deleteOnExit();
 132             String baseIndexUrl = indexingContext.getIndexUpdateUrl();
 133
 134             String wagonProtocol = new URL( this.remoteRepository.getUrl() ).getProtocol();
 135
 136             final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon(
 137                 new WagonFactoryRequest( wagonProtocol, this.remoteRepository.getExtraHeaders() ).networkProxy(
 138                     this.networkProxy ) );
 139             int timeoutInMilliseconds = remoteRepository.getTimeout() * 1000;
 140             // FIXME olamy having 2 config values
 141             wagon.setReadTimeout( timeoutInMilliseconds );
 142             wagon.setTimeout( timeoutInMilliseconds );
 143
 144             wagon.addTransferListener( new DownloadListener() );
 145             ProxyInfo proxyInfo = null;
 146             if ( this.networkProxy != null )
 147             {
 148                 proxyInfo = new ProxyInfo();
 149                 proxyInfo.setHost( this.networkProxy.getHost() );
 150                 proxyInfo.setPort( this.networkProxy.getPort() );
 151                 proxyInfo.setUserName( this.networkProxy.getUsername() );
 152                 proxyInfo.setPassword( this.networkProxy.getPassword() );
 153             }
 154             AuthenticationInfo authenticationInfo = null;
 155             if ( this.remoteRepository.getUserName() != null )
 156             {
 157                 authenticationInfo = new AuthenticationInfo();
 158                 authenticationInfo.setUserName( this.remoteRepository.getUserName() );
 159                 authenticationInfo.setPassword( this.remoteRepository.getPassword() );
 160             }
 161             wagon.connect( new Repository( this.remoteRepository.getId(), baseIndexUrl ), authenticationInfo,
 162                            proxyInfo );
 163
 164             File indexDirectory = indexingContext.getIndexDirectoryFile();
 165             if ( !indexDirectory.exists() )
 166             {
 167                 indexDirectory.mkdirs();
 168             }
 169
 170             ResourceFetcher resourceFetcher =
 171                 new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository );
 172             IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
 173             request.setForceFullUpdate( this.fullDownload );
 174             request.setLocalIndexCacheDir( indexCacheDirectory );
 175
 176             this.indexUpdater.fetchAndUpdateIndex( request );
 177             stopWatch.stop();
 178             log.info( "time to download remote repository index for repository {}: {} s", this.remoteRepository.getId(),
 179                       ( stopWatch.getTime() / 1000 ) );
 180
 181             // index packing optionnal ??
 182             IndexPackingRequest indexPackingRequest =
 183                 new IndexPackingRequest( indexingContext, indexingContext.getIndexDirectoryFile() );
 184             indexPacker.packIndex( indexPackingRequest );
 185             indexingContext.updateTimestamp( true );
 186
 187         }
 188         catch ( MalformedURLException e )
 189         {
 190             log.error( e.getMessage(), e );
 191             throw new RuntimeException( e.getMessage(), e );
 192         }
 193         catch ( WagonFactoryException e )
 194         {
 195             log.error( e.getMessage(), e );
 196             throw new RuntimeException( e.getMessage(), e );
 197         }
 198         catch ( ConnectionException e )
 199         {
 200             log.error( e.getMessage(), e );
 201             throw new RuntimeException( e.getMessage(), e );
 202         }
 203         catch ( AuthenticationException e )
 204         {
 205             log.error( e.getMessage(), e );
 206             throw new RuntimeException( e.getMessage(), e );
 207         }
 208         catch ( IOException e )
 209         {
 210             log.error( e.getMessage(), e );
 211             throw new RuntimeException( e.getMessage(), e );
 212         }
 213         catch ( RepositoryAdminException e )
 214         {
 215             log.error( e.getMessage(), e );
 216             throw new RuntimeException( e.getMessage(), e );
 217         }
 218         finally
 219         {
 220             deleteDirectoryQuiet( tempIndexDirectory );
 221             this.runningRemoteDownloadIds.remove( this.remoteRepository.getId() );
 222         }
 223         log.info( "end download remote index for remote repository " + this.remoteRepository.getId() );
 224     }
 225
 226     private void deleteDirectoryQuiet( File f )
 227     {
 228         try
 229         {
 230             FileUtils.deleteDirectory( f );
 231         }
 232         catch ( IOException e )
 233         {
 234             log.warn( "skip error delete {} : {}", f, e.getMessage() );
 235         }
 236     }
 237
 238
 239     public static class DownloadListener
 240         implements TransferListener
 241     {
 242         private Logger log = LoggerFactory.getLogger( getClass() );
 243
 244         String resourceName;
 245
 246         long startTime;
 247
 248         public void transferInitiated( TransferEvent transferEvent )
 249         {
 250             resourceName = transferEvent.getResource().getName();
 251             log.debug( "initiate transfer of {}", resourceName );
 252         }
 253
 254         public void transferStarted( TransferEvent transferEvent )
 255         {
 256             resourceName = transferEvent.getResource().getName();
 257             startTime = System.currentTimeMillis();
 258             log.info( "start transfer of {}", transferEvent.getResource().getName() );
 259         }
 260
 261         public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
 262         {
 263             log.debug( "transfer of {} : {}/{}", transferEvent.getResource().getName(), buffer.length, length );
 264         }
 265
 266         public void transferCompleted( TransferEvent transferEvent )
 267         {
 268             resourceName = transferEvent.getResource().getName();
 269             long endTime = System.currentTimeMillis();
 270             log.info( "end of transfer file {}: {}s", transferEvent.getResource().getName(),
 271                       ( endTime - startTime ) / 1000 );
 272         }
 273
 274         public void transferError( TransferEvent transferEvent )
 275         {
 276             log.info( "error of transfer file {}: {}", transferEvent.getResource().getName(),
 277                       transferEvent.getException().getMessage(), transferEvent.getException() );
 278         }
 279
 280         public void debug( String message )
 281         {
 282             log.debug( "transfer debug {}", message );
 283         }
 284     }
 285
 286     private static class WagonResourceFetcher
 287         implements ResourceFetcher
 288     {
 289
 290         Logger log;
 291
 292         File tempIndexDirectory;
 293
 294         Wagon wagon;
 295
 296         RemoteRepository remoteRepository;
 297
 298         private WagonResourceFetcher( Logger log, File tempIndexDirectory, Wagon wagon,
 299                                       RemoteRepository remoteRepository )
 300         {
 301             this.log = log;
 302             this.tempIndexDirectory = tempIndexDirectory;
 303             this.wagon = wagon;
 304             this.remoteRepository = remoteRepository;
 305         }
 306
 307         public void connect( String id, String url )
 308             throws IOException
 309         {
 310             //no op
 311         }
 312
 313         public void disconnect()
 314             throws IOException
 315         {
 316             // no op
 317         }
 318
 319         public InputStream retrieve( String name )
 320             throws IOException, FileNotFoundException
 321         {
 322             try
 323             {
 324                 log.info( "index update retrieve file, name:{}", name );
 325                 File file = new File( tempIndexDirectory, name );
 326                 if ( file.exists() )
 327                 {
 328                     file.delete();
 329                 }
 330                 file.deleteOnExit();
 331                 wagon.get( addParameters( name, this.remoteRepository ), file );
 332                 return new FileInputStream( file );
 333             }
 334             catch ( AuthorizationException e )
 335             {
 336                 throw new IOException( e.getMessage(), e );
 337             }
 338             catch ( TransferFailedException e )
 339             {
 340                 throw new IOException( e.getMessage(), e );
 341             }
 342             catch ( ResourceDoesNotExistException e )
 343             {
 344                 FileNotFoundException fnfe = new FileNotFoundException( e.getMessage() );
 345                 fnfe.initCause( e );
 346                 throw fnfe;
 347             }
 348         }
 349
 350         // FIXME remove crappy copy/paste
 351         protected String addParameters( String path, RemoteRepository remoteRepository )
 352         {
 353             if ( remoteRepository.getExtraParameters().isEmpty() )
 354             {
 355                 return path;
 356             }
 357
 358             boolean question = false;
 359
 360             StringBuilder res = new StringBuilder( path == null ? "" : path );
 361
 362             for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() )
 363             {
 364                 if ( !question )
 365                 {
 366                     res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() );
 367                 }
 368             }
 369
 370             return res.toString();
 371         }
 372
 373     }
 374
 375
 376 }
 377
 378