source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.scheduler.indexing;
   2 /*
   3  * Licensed to the Apache Software Foundation (ASF) under one
   4  * or more contributor license agreements.  See the NOTICE file
   5  * distributed with this work for additional information
   6  * regarding copyright ownership.  The ASF licenses this file
   7  * to you under the Apache License, Version 2.0 (the
   8  * "License"); you may not use this file except in compliance
   9  * with the License.  You may obtain a copy of the License at
  10  *
  11  *   http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing,
  14  * software distributed under the License is distributed on an
  15  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  16  * KIND, either express or implied.  See the License for the
  17  * specific language governing permissions and limitations
  18  * under the License.
  19  */
  20
  21 import org.apache.archiva.admin.model.beans.NetworkProxy;
  22 import org.apache.archiva.admin.model.beans.RemoteRepository;
  23 import org.apache.archiva.proxy.common.WagonFactory;
  24 import org.apache.archiva.proxy.common.WagonFactoryException;
  25 import org.apache.commons.compress.compressors.CompressorException;
  26 import org.apache.commons.io.FileUtils;
  27 import org.apache.maven.index.NexusIndexer;
  28 import org.apache.maven.index.context.IndexingContext;
  29 import org.apache.maven.index.updater.IndexUpdateRequest;
  30 import org.apache.maven.index.updater.IndexUpdater;
  31 import org.apache.maven.index.updater.ResourceFetcher;
  32 import org.apache.maven.wagon.ConnectionException;
  33 import org.apache.maven.wagon.ResourceDoesNotExistException;
  34 import org.apache.maven.wagon.TransferFailedException;
  35 import org.apache.maven.wagon.Wagon;
  36 import org.apache.maven.wagon.authentication.AuthenticationException;
  37 import org.apache.maven.wagon.authentication.AuthenticationInfo;
  38 import org.apache.maven.wagon.authorization.AuthorizationException;
  39 import org.apache.maven.wagon.events.TransferEvent;
  40 import org.apache.maven.wagon.events.TransferListener;
  41 import org.apache.maven.wagon.proxy.ProxyInfo;
  42 import org.apache.maven.wagon.repository.Repository;
  43 import org.slf4j.Logger;
  44 import org.slf4j.LoggerFactory;
  45
  46 import java.io.File;
  47 import java.io.FileInputStream;
  48 import java.io.FileNotFoundException;
  49 import java.io.IOException;
  50 import java.io.InputStream;
  51 import java.net.MalformedURLException;
  52 import java.net.URL;
  53 import java.util.Arrays;
  54 import java.util.List;
  55
  56 /**
  57  * @author Olivier Lamy
  58  * @since 1.4
  59  */
  60 public class DownloadRemoteIndexTask
  61     implements Runnable
  62 {
  63     private Logger log = LoggerFactory.getLogger( getClass() );
  64
  65     private RemoteRepository remoteRepository;
  66
  67     private NexusIndexer nexusIndexer;
  68
  69     private WagonFactory wagonFactory;
  70
  71     private NetworkProxy networkProxy;
  72
  73     private boolean fullDownload;
  74
  75     private List<String> runningRemoteDownloadIds;
  76
  77     private IndexUpdater indexUpdater;
  78
  79     public DownloadRemoteIndexTask( DownloadRemoteIndexTaskRequest downloadRemoteIndexTaskRequest,
  80                                     List<String> runningRemoteDownloadIds )
  81     {
  82         this.remoteRepository = downloadRemoteIndexTaskRequest.getRemoteRepository();
  83         this.nexusIndexer = downloadRemoteIndexTaskRequest.getNexusIndexer();
  84         this.wagonFactory = downloadRemoteIndexTaskRequest.getWagonFactory();
  85         this.networkProxy = downloadRemoteIndexTaskRequest.getNetworkProxy();
  86         this.fullDownload = downloadRemoteIndexTaskRequest.isFullDownload();
  87         this.runningRemoteDownloadIds = runningRemoteDownloadIds;
  88         this.indexUpdater = downloadRemoteIndexTaskRequest.getIndexUpdater();
  89     }
  90
  91     public void run()
  92     {
  93
  94         // so short lock : not sure we need it
  95         synchronized ( this.runningRemoteDownloadIds )
  96         {
  97             if ( this.runningRemoteDownloadIds.contains( this.remoteRepository.getId() ) )
  98             {
  99                 // skip it as it's running
 100                 log.info( "skip download index remote for repo {} it's already running",
 101                           this.remoteRepository.getId() );
 102                 return;
 103             }
 104             log.info( "start download remote index for remote repository " + this.remoteRepository.getId() );
 105             this.runningRemoteDownloadIds.add( this.remoteRepository.getId() );
 106         }
 107         IndexingContext indexingContext =
 108             nexusIndexer.getIndexingContexts().get( "remote-" + remoteRepository.getId() );
 109
 110         // TODO check if null ? normally not as created by DefaultDownloadRemoteIndexScheduler#startup
 111
 112         // create a temp directory to download files
 113         final File tempIndexDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".tmpIndex" );
 114         try
 115         {
 116             if ( tempIndexDirectory.exists() )
 117             {
 118                 FileUtils.deleteDirectory( tempIndexDirectory );
 119             }
 120             tempIndexDirectory.mkdirs();
 121             String baseIndexUrl = indexingContext.getIndexUpdateUrl();
 122
 123             final Wagon wagon = wagonFactory.getWagon( new URL( this.remoteRepository.getUrl() ).getProtocol() );
 124             // TODO transferListener
 125             wagon.addTransferListener( new DownloadListener() );
 126             ProxyInfo proxyInfo = null;
 127             if ( this.networkProxy != null )
 128             {
 129                 proxyInfo = new ProxyInfo();
 130                 proxyInfo.setHost( this.networkProxy.getHost() );
 131                 proxyInfo.setPort( this.networkProxy.getPort() );
 132                 proxyInfo.setUserName( this.networkProxy.getUsername() );
 133                 proxyInfo.setPassword( this.networkProxy.getPassword() );
 134             }
 135             AuthenticationInfo authenticationInfo = null;
 136             if ( this.remoteRepository.getUserName() != null )
 137             {
 138                 authenticationInfo = new AuthenticationInfo();
 139                 authenticationInfo.setUserName( this.remoteRepository.getUserName() );
 140                 authenticationInfo.setPassword( this.remoteRepository.getPassword() );
 141             }
 142             wagon.connect( new Repository( this.remoteRepository.getId(), baseIndexUrl ), authenticationInfo,
 143                            proxyInfo );
 144
 145             File indexDirectory = indexingContext.getIndexDirectoryFile();
 146             if ( !indexDirectory.exists() )
 147             {
 148                 indexDirectory.mkdirs();
 149             }
 150
 151             /*
 152             File[] indexFiles = indexDirectory.listFiles( new FileFilter()
 153             {
 154                 public boolean accept( File file )
 155                 {
 156                     return !file.isDirectory();
 157                 }
 158             } );
 159
 160             List<String> indexFileNames = new ArrayList<String>( indexFiles == null ? 0 : indexFiles.length );
 161
 162             for ( File f : indexFiles == null ? new File[0] : indexFiles )
 163             {
 164                 indexFileNames.add( f.getName() );
 165             }
 166             */
 167
 168             //List<String> files = wagon.getFileList( "" );
 169
 170             // take care about time stamp : no need to rebuild index
 171             // TODO incremental honor fullDownload true !!
 172             // FIXME dont fail all if one file fail ?
 173             /*
 174             for ( String file : files )
 175             {
 176                 if ( !indexFileNames.contains( file ) && StringUtils.endsWith( file, ".gz" ) )
 177                 {
 178                     downloadFile( wagon, file, tempIndexDirectory );
 179                     File compressIndexUpdate = new File( tempIndexDirectory, file );
 180                     mergeCompressIndex( indexingContext, compressIndexUpdate, wagon );
 181                 }
 182             }*/
 183             ResourceFetcher resourceFetcher = new ResourceFetcher()
 184             {
 185                 public void connect( String id, String url )
 186                     throws IOException
 187                 {
 188                     //no op
 189                 }
 190
 191                 public void disconnect()
 192                     throws IOException
 193                 {
 194                     // no op
 195                 }
 196
 197                 public InputStream retrieve( String name )
 198                     throws IOException, FileNotFoundException
 199                 {
 200                     try
 201                     {
 202                         log.debug( "resourceFetcher#retrieve, name:{}", name );
 203                         //TODO check those files are deleted !!
 204                         File file = new File( tempIndexDirectory, name );
 205                         if ( file.exists() )
 206                         {
 207                             file.delete();
 208                         }
 209                         //file.deleteOnExit();
 210                         wagon.get( name, file );
 211                         return new FileInputStream( file );
 212                     }
 213                     catch ( AuthorizationException e )
 214                     {
 215                         throw new IOException( e.getMessage(), e );
 216                     }
 217                     catch ( TransferFailedException e )
 218                     {
 219                         throw new IOException( e.getMessage(), e );
 220                     }
 221                     catch ( ResourceDoesNotExistException e )
 222                     {
 223                         throw new FileNotFoundException( e.getMessage() );
 224                     }
 225                 }
 226             };
 227
 228             IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
 229
 230             this.indexUpdater.fetchAndUpdateIndex( request );
 231
 232
 233         }
 234         catch ( MalformedURLException e )
 235         {
 236             log.error( e.getMessage(), e );
 237             throw new RuntimeException( e.getMessage(), e );
 238         }
 239         catch ( WagonFactoryException e )
 240         {
 241             log.error( e.getMessage(), e );
 242             throw new RuntimeException( e.getMessage(), e );
 243         }
 244         catch ( ConnectionException e )
 245         {
 246             log.error( e.getMessage(), e );
 247             throw new RuntimeException( e.getMessage(), e );
 248         }
 249         catch ( AuthenticationException e )
 250         {
 251             log.error( e.getMessage(), e );
 252             throw new RuntimeException( e.getMessage(), e );
 253         }
 254         catch ( IOException e )
 255         {
 256             log.error( e.getMessage(), e );
 257             throw new RuntimeException( e.getMessage(), e );
 258         }
 259         finally
 260         {
 261             //deleteDirectoryQuiet( tempIndexDirectory );
 262             this.runningRemoteDownloadIds.remove( this.remoteRepository.getId() );
 263         }
 264         log.info( "end download remote index for remote repository " + this.remoteRepository.getId() );
 265     }
 266
 267     private void deleteDirectoryQuiet( File f )
 268     {
 269         try
 270         {
 271             FileUtils.deleteDirectory( f );
 272         }
 273         catch ( IOException e )
 274         {
 275             log.warn( "skip error delete " + f + ": " + e.getMessage() );
 276         }
 277     }
 278
 279     protected void mergeCompressIndex( IndexingContext context, final File compressedIndexUpdate, final Wagon wagon )
 280         throws IOException, CompressorException
 281     {
 282
 283         /*
 284
 285         final File tmpUncompressDirectory = new File( compressedIndexUpdate.getParent(),
 286                                                       StringUtils.substringBeforeLast( compressedIndexUpdate.getName(),
 287                                                                                        "." ) );
 288         tmpUncompressDirectory.deleteOnExit();
 289         final FileOutputStream fos =
 290             new FileOutputStream( new File( tmpUncompressDirectory, compressedIndexUpdate.getName() ) );
 291         try
 292         {
 293             if ( tmpUncompressDirectory.exists() )
 294             {
 295                 tmpUncompressDirectory.delete();
 296             }
 297             tmpUncompressDirectory.mkdirs();
 298
 299             // gunzip  the file to a directory and merge
 300
 301             // gunzip
 302             final InputStream in = new FileInputStream( compressedIndexUpdate );
 303             CompressorInputStream cis =
 304                 new CompressorStreamFactory().createCompressorInputStream( CompressorStreamFactory.GZIP, in );
 305             IOUtils.copy( cis, fos );
 306             in.close();
 307             fos.flush();
 308             fos.close();
 309             // merge
 310
 311         }
 312         finally
 313         {
 314             IOUtils.closeQuietly( fos );
 315             //deleteDirectoryQuiet( tmpUncompressDirectory );
 316             //FileUtils.deleteQuietly( tmpUncompressDirectory );
 317         }
 318         */
 319     }
 320
 321     protected void downloadFile( Wagon wagon, String file, File tempIndexDirectory )
 322     {
 323         try
 324         {
 325             wagon.get( file, new File( tempIndexDirectory, file ) );
 326         }
 327         catch ( Exception e )
 328         {
 329             log.warn( "skip fail to download " + file + ": " + e.getMessage() );
 330         }
 331     }
 332
 333
 334     public static class DownloadListener
 335         implements TransferListener
 336     {
 337         private Logger log = LoggerFactory.getLogger( getClass() );
 338
 339         public void transferInitiated( TransferEvent transferEvent )
 340         {
 341             log.debug( "initiate transfer of {}", transferEvent.getResource().getName() );
 342         }
 343
 344         public void transferStarted( TransferEvent transferEvent )
 345         {
 346             log.debug( "start transfer of {}", transferEvent.getResource().getName() );
 347         }
 348
 349         public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
 350         {
 351             log.debug( "transfer of {} : {}/{}",
 352                        Arrays.asList( transferEvent.getResource().getName(), buffer.length, length ).toArray() );
 353         }
 354
 355         public void transferCompleted( TransferEvent transferEvent )
 356         {
 357             log.info( "end of transfer file " + transferEvent.getResource().getName() );
 358         }
 359
 360         public void transferError( TransferEvent transferEvent )
 361         {
 362             log.info( "error of transfer file {}: {}", Arrays.asList( transferEvent.getResource().getName(),
 363                                                                       transferEvent.getException().getMessage() ).toArray(
 364                 new Object[2] ), transferEvent.getException() );
 365         }
 366
 367         public void debug( String message )
 368         {
 369             log.debug( "transfer debug {}", message );
 370         }
 371     }
 372
 373 }