source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.scheduler.indexing;
   2 /*
   3  * Licensed to the Apache Software Foundation (ASF) under one
   4  * or more contributor license agreements.  See the NOTICE file
   5  * distributed with this work for additional information
   6  * regarding copyright ownership.  The ASF licenses this file
   7  * to you under the Apache License, Version 2.0 (the
   8  * "License"); you may not use this file except in compliance
   9  * with the License.  You may obtain a copy of the License at
  10  *
  11  *   http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing,
  14  * software distributed under the License is distributed on an
  15  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  16  * KIND, either express or implied.  See the License for the
  17  * specific language governing permissions and limitations
  18  * under the License.
  19  */
  20
  21 import org.apache.archiva.admin.model.beans.NetworkProxy;
  22 import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin;
  23 import org.apache.archiva.proxy.common.WagonFactory;
  24 import org.apache.archiva.proxy.common.WagonFactoryRequest;
  25 import org.apache.archiva.repository.PasswordCredentials;
  26 import org.apache.archiva.repository.RemoteRepository;
  27 import org.apache.archiva.repository.RepositoryException;
  28 import org.apache.archiva.repository.RepositoryType;
  29 import org.apache.archiva.repository.features.RemoteIndexFeature;
  30 import org.apache.commons.lang.time.StopWatch;
  31 import org.apache.http.auth.UsernamePasswordCredentials;
  32 import org.apache.maven.index.context.IndexingContext;
  33 import org.apache.maven.index.updater.IndexUpdateRequest;
  34 import org.apache.maven.index.updater.IndexUpdateResult;
  35 import org.apache.maven.index.updater.IndexUpdater;
  36 import org.apache.maven.index.updater.ResourceFetcher;
  37 import org.apache.maven.index_shaded.lucene.index.IndexNotFoundException;
  38 import org.apache.maven.wagon.ResourceDoesNotExistException;
  39 import org.apache.maven.wagon.StreamWagon;
  40 import org.apache.maven.wagon.TransferFailedException;
  41 import org.apache.maven.wagon.Wagon;
  42 import org.apache.maven.wagon.authentication.AuthenticationInfo;
  43 import org.apache.maven.wagon.authorization.AuthorizationException;
  44 import org.apache.maven.wagon.events.TransferEvent;
  45 import org.apache.maven.wagon.events.TransferListener;
  46 import org.apache.maven.wagon.proxy.ProxyInfo;
  47 import org.apache.maven.wagon.repository.Repository;
  48 import org.apache.maven.wagon.shared.http.AbstractHttpClientWagon;
  49 import org.apache.maven.wagon.shared.http.HttpConfiguration;
  50 import org.apache.maven.wagon.shared.http.HttpMethodConfiguration;
  51 import org.slf4j.Logger;
  52 import org.slf4j.LoggerFactory;
  53
  54 import java.io.FileNotFoundException;
  55 import java.io.IOException;
  56 import java.io.InputStream;
  57 import java.net.URL;
  58 import java.nio.file.Files;
  59 import java.nio.file.Path;
  60 import java.nio.file.Paths;
  61 import java.util.List;
  62 import java.util.Map;
  63
  64 /**
  65  * @author Olivier Lamy
  66  * @since 1.4-M1
  67  */
  68 public class DownloadRemoteIndexTask
  69     implements Runnable
  70 {
  71     private Logger log = LoggerFactory.getLogger( getClass() );
  72
  73     private RemoteRepository remoteRepository;
  74
  75     private WagonFactory wagonFactory;
  76
  77     private NetworkProxy networkProxy;
  78
  79     private boolean fullDownload;
  80
  81     private List<String> runningRemoteDownloadIds;
  82
  83     private IndexUpdater indexUpdater;
  84
  85
  86     public DownloadRemoteIndexTask( DownloadRemoteIndexTaskRequest downloadRemoteIndexTaskRequest,
  87                                     List<String> runningRemoteDownloadIds )
  88     {
  89         this.remoteRepository = downloadRemoteIndexTaskRequest.getRemoteRepository();
  90         this.wagonFactory = downloadRemoteIndexTaskRequest.getWagonFactory();
  91         this.networkProxy = downloadRemoteIndexTaskRequest.getNetworkProxy();
  92         this.fullDownload = downloadRemoteIndexTaskRequest.isFullDownload();
  93         this.runningRemoteDownloadIds = runningRemoteDownloadIds;
  94         this.indexUpdater = downloadRemoteIndexTaskRequest.getIndexUpdater();
  95     }
  96
  97     @Override
  98     public void run()
  99     {
 100
 101         // so short lock : not sure we need it
 102         synchronized ( this.runningRemoteDownloadIds )
 103         {
 104             if ( this.runningRemoteDownloadIds.contains( this.remoteRepository.getId() ) )
 105             {
 106                 // skip it as it's running
 107                 log.info( "skip download index remote for repo {} it's already running",
 108                           this.remoteRepository.getId() );
 109                 return;
 110             }
 111             this.runningRemoteDownloadIds.add( this.remoteRepository.getId() );
 112         }
 113         Path tempIndexDirectory = null;
 114         StopWatch stopWatch = new StopWatch();
 115         stopWatch.start();
 116         try
 117         {
 118             log.info( "start download remote index for remote repository {}", this.remoteRepository.getId() );
 119             if (this.remoteRepository.getIndexingContext()==null) {
 120                 throw new IndexNotFoundException("No index context set for repository "+remoteRepository.getId());
 121             }
 122             if (this.remoteRepository.getType()!= RepositoryType.MAVEN) {
 123                 throw new RepositoryException("Bad repository type");
 124             }
 125             if (!this.remoteRepository.supportsFeature(RemoteIndexFeature.class)) {
 126                 throw new RepositoryException("Repository does not support RemotIndexFeature "+remoteRepository.getId());
 127             }
 128             RemoteIndexFeature rif = this.remoteRepository.getFeature(RemoteIndexFeature.class).get();
 129             IndexingContext indexingContext = this.remoteRepository.getIndexingContext().getBaseContext(IndexingContext.class);
 130             // create a temp directory to download files
 131             tempIndexDirectory = Paths.get(indexingContext.getIndexDirectoryFile().getParent(), ".tmpIndex" );
 132             Path indexCacheDirectory = Paths.get( indexingContext.getIndexDirectoryFile().getParent(), ".indexCache" );
 133             Files.createDirectories( indexCacheDirectory );
 134             if ( Files.exists(tempIndexDirectory) )
 135             {
 136                 org.apache.archiva.common.utils.FileUtils.deleteDirectory( tempIndexDirectory );
 137             }
 138             Files.createDirectories( tempIndexDirectory );
 139             tempIndexDirectory.toFile().deleteOnExit();
 140             String baseIndexUrl = indexingContext.getIndexUpdateUrl();
 141
 142             String wagonProtocol = this.remoteRepository.getLocation().getScheme();
 143
 144             final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon(
 145                 new WagonFactoryRequest( wagonProtocol, this.remoteRepository.getExtraHeaders() ).networkProxy(
 146                     this.networkProxy )
 147             );
 148             // FIXME olamy having 2 config values
 149             wagon.setReadTimeout( (int)rif.getDownloadTimeout().toMillis());
 150             wagon.setTimeout( (int)remoteRepository.getTimeout().toMillis());
 151
 152             if ( wagon instanceof AbstractHttpClientWagon )
 153             {
 154                 HttpConfiguration httpConfiguration = new HttpConfiguration();
 155                 HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration();
 156                 httpMethodConfiguration.setUsePreemptive( true );
 157                 httpMethodConfiguration.setReadTimeout( (int)rif.getDownloadTimeout().toMillis() );
 158                 httpConfiguration.setGet( httpMethodConfiguration );
 159                 AbstractHttpClientWagon.class.cast( wagon ).setHttpConfiguration( httpConfiguration );
 160             }
 161
 162             wagon.addTransferListener( new DownloadListener() );
 163             ProxyInfo proxyInfo = null;
 164             if ( this.networkProxy != null )
 165             {
 166                 proxyInfo = new ProxyInfo();
 167                 proxyInfo.setType( this.networkProxy.getProtocol() );
 168                 proxyInfo.setHost( this.networkProxy.getHost() );
 169                 proxyInfo.setPort( this.networkProxy.getPort() );
 170                 proxyInfo.setUserName( this.networkProxy.getUsername() );
 171                 proxyInfo.setPassword( this.networkProxy.getPassword() );
 172             }
 173             AuthenticationInfo authenticationInfo = null;
 174             if ( this.remoteRepository.getLoginCredentials()!=null && this.remoteRepository.getLoginCredentials() instanceof PasswordCredentials )
 175             {
 176                 PasswordCredentials creds = (PasswordCredentials) this.remoteRepository.getLoginCredentials();
 177                 authenticationInfo = new AuthenticationInfo();
 178                 authenticationInfo.setUserName( creds.getUsername());
 179                 authenticationInfo.setPassword( new String(creds.getPassword()) );
 180             }
 181             log.debug("Connection to {}, authInfo={}", this.remoteRepository.getId(), authenticationInfo);
 182             wagon.connect( new Repository( this.remoteRepository.getId(), baseIndexUrl ), authenticationInfo,
 183                            proxyInfo );
 184
 185             Path indexDirectory = indexingContext.getIndexDirectoryFile().toPath();
 186             if ( !Files.exists(indexDirectory) )
 187             {
 188                 Files.createDirectories( indexDirectory );
 189             }
 190             log.debug("Downloading index file to {}", indexDirectory);
 191             log.debug("Index cache dir {}", indexCacheDirectory);
 192
 193             ResourceFetcher resourceFetcher =
 194                 new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository );
 195             IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
 196             request.setForceFullUpdate( this.fullDownload );
 197             request.setLocalIndexCacheDir( indexCacheDirectory.toFile() );
 198
 199             IndexUpdateResult result = this.indexUpdater.fetchAndUpdateIndex(request);
 200             log.debug("Update result success: {}", result.isSuccessful());
 201             stopWatch.stop();
 202             log.info( "time update index from remote for repository {}: {}ms", this.remoteRepository.getId(),
 203                       ( stopWatch.getTime() ) );
 204
 205             // index packing optionnal ??
 206             //IndexPackingRequest indexPackingRequest =
 207             //    new IndexPackingRequest( indexingContext, indexingContext.getIndexDirectoryFile() );
 208             //indexPacker.packIndex( indexPackingRequest );
 209             indexingContext.updateTimestamp( true );
 210
 211         }
 212         catch ( Exception e )
 213         {
 214             log.error( e.getMessage(), e );
 215             throw new RuntimeException( e.getMessage(), e );
 216         }
 217         finally
 218         {
 219             deleteDirectoryQuiet( tempIndexDirectory );
 220             this.runningRemoteDownloadIds.remove( this.remoteRepository.getId() );
 221         }
 222         log.info( "end download remote index for remote repository {}", this.remoteRepository.getId() );
 223     }
 224
 225     private void deleteDirectoryQuiet( Path f )
 226     {
 227         try
 228         {
 229             org.apache.archiva.common.utils.FileUtils.deleteDirectory( f );
 230         }
 231         catch ( IOException e )
 232         {
 233             log.warn( "skip error delete {} : {}", f, e.getMessage() );
 234         }
 235     }
 236
 237
 238     private static final class DownloadListener
 239         implements TransferListener
 240     {
 241         private Logger log = LoggerFactory.getLogger( getClass() );
 242
 243         private String resourceName;
 244
 245         private long startTime;
 246
 247         private int totalLength = 0;
 248
 249         @Override
 250         public void transferInitiated( TransferEvent transferEvent )
 251         {
 252             startTime = System.currentTimeMillis();
 253             resourceName = transferEvent.getResource().getName();
 254             log.debug( "initiate transfer of {}", resourceName );
 255         }
 256
 257         @Override
 258         public void transferStarted( TransferEvent transferEvent )
 259         {
 260             this.totalLength = 0;
 261             resourceName = transferEvent.getResource().getName();
 262             log.info("Transferring: {}, {}",  transferEvent.getResource().getContentLength(), transferEvent.getLocalFile().toString());
 263             log.info( "start transfer of {}", transferEvent.getResource().getName() );
 264         }
 265
 266         @Override
 267         public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
 268         {
 269             log.debug( "transfer of {} : {}/{}", transferEvent.getResource().getName(), buffer.length, length );
 270             this.totalLength += length;
 271         }
 272
 273         @Override
 274         public void transferCompleted( TransferEvent transferEvent )
 275         {
 276             resourceName = transferEvent.getResource().getName();
 277             long endTime = System.currentTimeMillis();
 278             log.info( "end of transfer file {}: {}b, {}ms", transferEvent.getResource().getName(),
 279                       this.totalLength, ( endTime - startTime ) );
 280         }
 281
 282         @Override
 283         public void transferError( TransferEvent transferEvent )
 284         {
 285             log.info( "error of transfer file {}: {}", transferEvent.getResource().getName(),
 286                       transferEvent.getException().getMessage(), transferEvent.getException() );
 287         }
 288
 289         @Override
 290         public void debug( String message )
 291         {
 292             log.debug( "transfer debug {}", message );
 293         }
 294     }
 295
 296     private static class WagonResourceFetcher
 297         implements ResourceFetcher
 298     {
 299
 300         Logger log;
 301
 302         Path tempIndexDirectory;
 303
 304         Wagon wagon;
 305
 306         RemoteRepository remoteRepository;
 307
 308         private WagonResourceFetcher( Logger log, Path tempIndexDirectory, Wagon wagon,
 309                                       RemoteRepository remoteRepository )
 310         {
 311             this.log = log;
 312             this.tempIndexDirectory = tempIndexDirectory;
 313             this.wagon = wagon;
 314             this.remoteRepository = remoteRepository;
 315         }
 316
 317         @Override
 318         public void connect( String id, String url )
 319             throws IOException
 320         {
 321             //no op
 322         }
 323
 324         @Override
 325         public void disconnect()
 326             throws IOException
 327         {
 328             // no op
 329         }
 330
 331         @Override
 332         public InputStream retrieve( String name )
 333             throws IOException, FileNotFoundException
 334         {
 335             try
 336             {
 337                 log.info( "index update retrieve file, name:{}", name );
 338                 Path file = tempIndexDirectory.resolve( name );
 339                 Files.deleteIfExists( file );
 340                 file.toFile().deleteOnExit();
 341                 wagon.get( addParameters( name, this.remoteRepository ), file.toFile() );
 342                 return Files.newInputStream( file );
 343             }
 344             catch ( AuthorizationException | TransferFailedException e )
 345             {
 346                 throw new IOException( e.getMessage(), e );
 347             }
 348             catch ( ResourceDoesNotExistException e )
 349             {
 350                 FileNotFoundException fnfe = new FileNotFoundException( e.getMessage() );
 351                 fnfe.initCause( e );
 352                 throw fnfe;
 353             }
 354         }
 355
 356         // FIXME remove crappy copy/paste
 357         protected String addParameters( String path, RemoteRepository remoteRepository )
 358         {
 359             if ( remoteRepository.getExtraParameters().isEmpty() )
 360             {
 361                 return path;
 362             }
 363
 364             boolean question = false;
 365
 366             StringBuilder res = new StringBuilder( path == null ? "" : path );
 367
 368             for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() )
 369             {
 370                 if ( !question )
 371                 {
 372                     res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() );
 373                 }
 374             }
 375
 376             return res.toString();
 377         }
 378
 379     }
 380
 381
 382 }
 383