source.dussan.org Git - archiva.git/blob

   1 package org.apache.maven.archiva.proxy;
   2
   3 /*
   4  * Licensed to the Apache Software Foundation (ASF) under one
   5  * or more contributor license agreements.  See the NOTICE file
   6  * distributed with this work for additional information
   7  * regarding copyright ownership.  The ASF licenses this file
   8  * to you under the Apache License, Version 2.0 (the
   9  * "License"); you may not use this file except in compliance
  10  * with the License.  You may obtain a copy of the License at
  11  *
  12  *  http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing,
  15  * software distributed under the License is distributed on an
  16  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  17  * KIND, either express or implied.  See the License for the
  18  * specific language governing permissions and limitations
  19  * under the License.
  20  */
  21
  22 import java.io.File;
  23 import java.io.IOException;
  24 import java.util.ArrayList;
  25 import java.util.Collections;
  26 import java.util.HashMap;
  27 import java.util.LinkedHashMap;
  28 import java.util.List;
  29 import java.util.Map;
  30 import java.util.Properties;
  31 import java.util.Map.Entry;
  32
  33 import org.apache.commons.collections.CollectionUtils;
  34 import org.apache.commons.io.FileUtils;
  35 import org.apache.commons.lang.StringUtils;
  36 import org.apache.maven.archiva.configuration.ArchivaConfiguration;
  37 import org.apache.maven.archiva.configuration.ConfigurationNames;
  38 import org.apache.maven.archiva.configuration.NetworkProxyConfiguration;
  39 import org.apache.maven.archiva.configuration.ProxyConnectorConfiguration;
  40 import org.apache.maven.archiva.model.ArtifactReference;
  41 import org.apache.maven.archiva.model.Keys;
  42 import org.apache.maven.archiva.model.RepositoryURL;
  43 import org.apache.maven.archiva.policies.DownloadErrorPolicy;
  44 import org.apache.maven.archiva.policies.DownloadPolicy;
  45 import org.apache.maven.archiva.policies.PolicyConfigurationException;
  46 import org.apache.maven.archiva.policies.PolicyViolationException;
  47 import org.apache.maven.archiva.policies.PostDownloadPolicy;
  48 import org.apache.maven.archiva.policies.PreDownloadPolicy;
  49 import org.apache.maven.archiva.policies.ProxyDownloadException;
  50 import org.apache.maven.archiva.policies.urlcache.UrlFailureCache;
  51 import org.apache.maven.archiva.repository.ManagedRepositoryContent;
  52 import org.apache.maven.archiva.repository.RemoteRepositoryContent;
  53 import org.apache.maven.archiva.repository.RepositoryContentFactory;
  54 import org.apache.maven.archiva.repository.RepositoryException;
  55 import org.apache.maven.archiva.repository.RepositoryNotFoundException;
  56 import org.apache.maven.archiva.repository.metadata.MetadataTools;
  57 import org.apache.maven.archiva.repository.metadata.RepositoryMetadataException;
  58 import org.apache.maven.archiva.repository.scanner.RepositoryContentConsumers;
  59 import org.apache.maven.wagon.ConnectionException;
  60 import org.apache.maven.wagon.ResourceDoesNotExistException;
  61 import org.apache.maven.wagon.Wagon;
  62 import org.apache.maven.wagon.WagonException;
  63 import org.apache.maven.wagon.authentication.AuthenticationException;
  64 import org.apache.maven.wagon.authentication.AuthenticationInfo;
  65 import org.apache.maven.wagon.proxy.ProxyInfo;
  66 import org.apache.maven.wagon.repository.Repository;
  67 import org.codehaus.plexus.personality.plexus.lifecycle.phase.Initializable;
  68 import org.codehaus.plexus.personality.plexus.lifecycle.phase.InitializationException;
  69 import org.codehaus.plexus.registry.Registry;
  70 import org.codehaus.plexus.registry.RegistryListener;
  71 import org.codehaus.plexus.util.SelectorUtils;
  72 import org.slf4j.Logger;
  73 import org.slf4j.LoggerFactory;
  74
  75 /**
  76  * DefaultRepositoryProxyConnectors
  77  *
  78  * @version $Id$
  79  * @todo exception handling needs work - "not modified" is not really an exceptional case, and it has more layers than your average brown onion
  80  * @plexus.component role-hint="default"
  81  */
  82 public class DefaultRepositoryProxyConnectors
  83     implements RepositoryProxyConnectors, RegistryListener, Initializable
  84 {
  85     private Logger log = LoggerFactory.getLogger( DefaultRepositoryProxyConnectors.class );
  86
  87     /**
  88      * @plexus.requirement
  89      */
  90     private ArchivaConfiguration archivaConfiguration;
  91
  92     /**
  93      * @plexus.requirement
  94      */
  95     private RepositoryContentFactory repositoryFactory;
  96
  97     /**
  98      * @plexus.requirement
  99      */
 100     private MetadataTools metadataTools;
 101
 102     /**
 103      * @plexus.requirement role="org.apache.maven.archiva.policies.PreDownloadPolicy"
 104      */
 105     private Map<String, PreDownloadPolicy> preDownloadPolicies;
 106
 107     /**
 108      * @plexus.requirement role="org.apache.maven.archiva.policies.PostDownloadPolicy"
 109      */
 110     private Map<String, PostDownloadPolicy> postDownloadPolicies;
 111
 112     /**
 113      * @plexus.requirement role="org.apache.maven.archiva.policies.DownloadErrorPolicy"
 114      */
 115     private Map<String, DownloadErrorPolicy> downloadErrorPolicies;
 116
 117     /**
 118      * @plexus.requirement role-hint="default"
 119      */
 120     private UrlFailureCache urlFailureCache;
 121
 122     private Map<String, List<ProxyConnector>> proxyConnectorMap = new HashMap<String, List<ProxyConnector>>();
 123
 124     private Map<String, ProxyInfo> networkProxyMap = new HashMap<String, ProxyInfo>();
 125
 126     /**
 127      * @plexus.requirement
 128      */
 129     private RepositoryContentConsumers consumers;
 130
 131     /**
 132      * @plexus.requirement
 133      */
 134     private WagonFactory wagonFactory;
 135
 136     public File fetchFromProxies( ManagedRepositoryContent repository, ArtifactReference artifact )
 137         throws ProxyDownloadException
 138     {
 139         File workingDirectory = createWorkingDirectory(repository);
 140         try
 141         {
 142             File localFile = toLocalFile( repository, artifact );
 143
 144             Properties requestProperties = new Properties();
 145             requestProperties.setProperty( "filetype", "artifact" );
 146             requestProperties.setProperty( "version", artifact.getVersion() );
 147             requestProperties.setProperty( "managedRepositoryId", repository.getId() );
 148
 149             List<ProxyConnector> connectors = getProxyConnectors( repository );
 150             Map<String, Exception> previousExceptions = new LinkedHashMap<String, Exception>();
 151             for ( ProxyConnector connector : connectors )
 152             {
 153                 if (connector.isDisabled())
 154                 {
 155                     continue;
 156                 }
 157
 158                 RemoteRepositoryContent targetRepository = connector.getTargetRepository();
 159                 requestProperties.setProperty( "remoteRepositoryId", targetRepository.getId() );
 160
 161                 String targetPath = targetRepository.toPath( artifact );
 162
 163                 try
 164                 {
 165                     File downloadedFile =
 166                         transferFile( connector, targetRepository, targetPath, repository, workingDirectory, localFile, requestProperties,
 167                                       true );
 168
 169                     if ( fileExists( downloadedFile ) )
 170                     {
 171                         log.debug( "Successfully transferred: " + downloadedFile.getAbsolutePath() );
 172                         return downloadedFile;
 173                     }
 174                 }
 175                 catch ( NotFoundException e )
 176                 {
 177                     log.debug( "Artifact " + Keys.toKey( artifact ) + " not found on repository \""
 178                                            + targetRepository.getRepository().getId() + "\"." );
 179                 }
 180                 catch ( NotModifiedException e )
 181                 {
 182                     log.debug( "Artifact " + Keys.toKey( artifact ) + " not updated on repository \""
 183                                            + targetRepository.getRepository().getId() + "\"." );
 184                 }
 185                 catch ( ProxyException e )
 186                 {
 187                     validatePolicies( this.downloadErrorPolicies, connector.getPolicies(), requestProperties, artifact,
 188                                       targetRepository, localFile, e, previousExceptions );
 189                 }
 190             }
 191
 192             if ( !previousExceptions.isEmpty() )
 193             {
 194                 throw new ProxyDownloadException( "Failures occurred downloading from some remote repositories",
 195                                                   previousExceptions );
 196             }
 197
 198             log.debug( "Exhausted all target repositories, artifact " + Keys.toKey( artifact ) + " not found." );
 199         }
 200         finally
 201         {
 202             FileUtils.deleteQuietly(workingDirectory);
 203         }
 204
 205         return null;
 206     }
 207
 208     public File fetchFromProxies( ManagedRepositoryContent repository, String path )
 209     {
 210         File workingDir = createWorkingDirectory(repository);
 211         try
 212         {
 213             File localFile = new File( repository.getRepoRoot(), path );
 214
 215             // no update policies for these paths
 216             if ( localFile.exists() )
 217             {
 218                 return null;
 219             }
 220
 221             Properties requestProperties = new Properties();
 222             requestProperties.setProperty( "filetype", "resource" );
 223             requestProperties.setProperty( "managedRepositoryId", repository.getId() );
 224
 225             List<ProxyConnector> connectors = getProxyConnectors( repository );
 226             for ( ProxyConnector connector : connectors )
 227             {
 228                 if (connector.isDisabled())
 229                 {
 230                     continue;
 231                 }
 232
 233                 RemoteRepositoryContent targetRepository = connector.getTargetRepository();
 234                 requestProperties.setProperty( "remoteRepositoryId", targetRepository.getId() );
 235
 236                 String targetPath = path;
 237
 238                 try
 239                 {
 240                     File downloadedFile =
 241                         transferFile( connector, targetRepository, targetPath, repository, workingDir, localFile, requestProperties, false );
 242
 243                     if ( fileExists( downloadedFile ) )
 244                     {
 245                         log.debug( "Successfully transferred: " + downloadedFile.getAbsolutePath() );
 246                         return downloadedFile;
 247                     }
 248                 }
 249                 catch ( NotFoundException e )
 250                 {
 251                     log.debug( "Resource " + path + " not found on repository \""
 252                                            + targetRepository.getRepository().getId() + "\"." );
 253                 }
 254                 catch ( NotModifiedException e )
 255                 {
 256                     log.debug( "Resource " + path + " not updated on repository \""
 257                                            + targetRepository.getRepository().getId() + "\"." );
 258                 }
 259                 catch ( ProxyException e )
 260                 {
 261                     log.warn( "Transfer error from repository \"" + targetRepository.getRepository().getId()
 262                         + "\" for resource " + path + ", continuing to next repository. Error message: " + e.getMessage() );
 263                     log.debug( "Full stack trace", e );
 264                 }
 265             }
 266
 267             log.debug( "Exhausted all target repositories, resource " + path + " not found." );
 268         }
 269         finally
 270         {
 271             FileUtils.deleteQuietly(workingDir);
 272         }
 273
 274         return null;
 275     }
 276
 277     public File fetchMetatadaFromProxies(ManagedRepositoryContent repository, String logicalPath)
 278     {
 279         File workingDir = createWorkingDirectory(repository);
 280         try
 281         {
 282             File localFile = new File(repository.getRepoRoot(), logicalPath);
 283
 284             Properties requestProperties = new Properties();
 285             requestProperties.setProperty( "filetype", "metadata" );
 286             boolean metadataNeedsUpdating = false;
 287             long originalTimestamp = getLastModified( localFile );
 288
 289             List<ProxyConnector> connectors = getProxyConnectors( repository );
 290             for ( ProxyConnector connector : connectors )
 291             {
 292                 if (connector.isDisabled())
 293                 {
 294                     continue;
 295                 }
 296
 297                 RemoteRepositoryContent targetRepository = connector.getTargetRepository();
 298
 299                 File localRepoFile = toLocalRepoFile( repository, targetRepository, logicalPath );
 300                 long originalMetadataTimestamp = getLastModified( localRepoFile );
 301
 302                 try
 303                 {
 304                     transferFile( connector, targetRepository, logicalPath, repository, workingDir, localRepoFile, requestProperties, true );
 305
 306                     if ( hasBeenUpdated( localRepoFile, originalMetadataTimestamp ) )
 307                     {
 308                         metadataNeedsUpdating = true;
 309                     }
 310                 }
 311                 catch ( NotFoundException e )
 312                 {
 313                     log.debug( "Metadata " + logicalPath
 314                                            + " not found on remote repository \""
 315                                            + targetRepository.getRepository().getId() + "\".", e );
 316                 }
 317                 catch ( NotModifiedException e )
 318                 {
 319                     log.debug( "Metadata " + logicalPath
 320                                            + " not updated on remote repository \""
 321                                            + targetRepository.getRepository().getId() + "\".", e );
 322                 }
 323                 catch ( ProxyException e )
 324                 {
 325                     log.warn( "Transfer error from repository \"" + targetRepository.getRepository().getId() +
 326                         "\" for versioned Metadata " + logicalPath +
 327                         ", continuing to next repository. Error message: " + e.getMessage() );
 328                     log.debug( "Full stack trace", e );
 329                 }
 330             }
 331
 332             if ( hasBeenUpdated( localFile, originalTimestamp ) )
 333             {
 334                 metadataNeedsUpdating = true;
 335             }
 336
 337             if ( metadataNeedsUpdating || !localFile.exists())
 338             {
 339                 try
 340                 {
 341                     metadataTools.updateMetadata( repository, logicalPath );
 342                 }
 343                 catch ( RepositoryMetadataException e )
 344                 {
 345                     log.warn( "Unable to update metadata " + localFile.getAbsolutePath() + ": " + e.getMessage(), e );
 346                 }
 347             }
 348
 349             if ( fileExists( localFile ) )
 350             {
 351                 return localFile;
 352             }
 353         }
 354         finally
 355         {
 356             FileUtils.deleteQuietly(workingDir);
 357         }
 358
 359         return null;
 360     }
 361
 362     private long getLastModified( File file )
 363     {
 364         if ( !file.exists() || !file.isFile() )
 365         {
 366             return 0;
 367         }
 368
 369         return file.lastModified();
 370     }
 371
 372     private boolean hasBeenUpdated( File file, long originalLastModified )
 373     {
 374         if ( !file.exists() || !file.isFile() )
 375         {
 376             return false;
 377         }
 378
 379         long currentLastModified = getLastModified( file );
 380         return ( currentLastModified > originalLastModified );
 381     }
 382
 383     private File toLocalRepoFile( ManagedRepositoryContent repository, RemoteRepositoryContent targetRepository,
 384                                   String targetPath )
 385     {
 386         String repoPath = metadataTools.getRepositorySpecificName( targetRepository, targetPath );
 387         return new File( repository.getRepoRoot(), repoPath );
 388     }
 389
 390     /**
 391      * Test if the provided ManagedRepositoryContent has any proxies configured for it.
 392      */
 393     public boolean hasProxies( ManagedRepositoryContent repository )
 394     {
 395         synchronized ( this.proxyConnectorMap )
 396         {
 397             return this.proxyConnectorMap.containsKey( repository.getId() );
 398         }
 399     }
 400
 401     private File toLocalFile( ManagedRepositoryContent repository, ArtifactReference artifact )
 402     {
 403         return repository.toFile( artifact );
 404     }
 405
 406     /**
 407      * Simple method to test if the file exists on the local disk.
 408      *
 409      * @param file the file to test. (may be null)
 410      * @return true if file exists. false if the file param is null, doesn't exist, or is not of type File.
 411      */
 412     private boolean fileExists( File file )
 413     {
 414         if ( file == null )
 415         {
 416             return false;
 417         }
 418
 419         if ( !file.exists() )
 420         {
 421             return false;
 422         }
 423
 424         if ( !file.isFile() )
 425         {
 426             return false;
 427         }
 428
 429         return true;
 430     }
 431
 432     /**
 433      * Perform the transfer of the file.
 434      *
 435      * @param connector         the connector configuration to use.
 436      * @param remoteRepository  the remote repository get the resource from.
 437      * @param remotePath        the path in the remote repository to the resource to get.
 438      * @param repository        the managed repository that will hold the file
 439      * @param resource          the local file to place the downloaded resource into
 440      * @param requestProperties the request properties to utilize for policy handling.
 441      * @param executeConsumers  whether to execute the consumers after proxying
 442      * @return the local file that was downloaded, or null if not downloaded.
 443      * @throws NotFoundException    if the file was not found on the remote repository.
 444      * @throws NotModifiedException if the localFile was present, and the resource was present on remote repository,
 445      *                              but the remote resource is not newer than the local File.
 446      * @throws ProxyException       if transfer was unsuccessful.
 447      */
 448     private File transferFile( ProxyConnector connector, RemoteRepositoryContent remoteRepository, String remotePath,
 449                                ManagedRepositoryContent repository, File workingDirectory, File resource, Properties requestProperties,
 450                                boolean executeConsumers )
 451         throws ProxyException, NotModifiedException
 452     {
 453         String url = remoteRepository.getURL().getUrl();
 454         if ( !url.endsWith( "/" ) )
 455         {
 456             url = url + "/";
 457         }
 458         url = url + remotePath;
 459         requestProperties.setProperty( "url", url );
 460
 461         // Is a whitelist defined?
 462         if ( CollectionUtils.isNotEmpty( connector.getWhitelist() ) )
 463         {
 464             // Path must belong to whitelist.
 465             if ( !matchesPattern( remotePath, connector.getWhitelist() ) )
 466             {
 467                 log.debug( "Path [" + remotePath +
 468                     "] is not part of defined whitelist (skipping transfer from repository [" +
 469                     remoteRepository.getRepository().getName() + "])." );
 470                 return null;
 471             }
 472         }
 473
 474         // Is target path part of blacklist?
 475         if ( matchesPattern( remotePath, connector.getBlacklist() ) )
 476         {
 477             log.debug( "Path [" + remotePath + "] is part of blacklist (skipping transfer from repository [" +
 478                 remoteRepository.getRepository().getName() + "])." );
 479             return null;
 480         }
 481
 482         // Handle pre-download policy
 483         try
 484         {
 485             validatePolicies( this.preDownloadPolicies, connector.getPolicies(), requestProperties, resource );
 486         }
 487         catch ( PolicyViolationException e )
 488         {
 489             String emsg = "Transfer not attempted on " + url + " : " + e.getMessage();
 490             if ( fileExists( resource ) )
 491             {
 492                 log.info( emsg + ": using already present local file." );
 493                 return resource;
 494             }
 495
 496             log.info( emsg );
 497             return null;
 498         }
 499
 500         // MRM-631 - the lightweight wagon does not reset these - remove if we switch to httpclient based wagon
 501         String previousHttpProxyHost = System.getProperty( "http.proxyHost" );
 502         String previousHttpProxyPort = System.getProperty( "http.proxyPort" );
 503         String previousProxyExclusions = System.getProperty( "http.nonProxyHosts" );
 504
 505         File tmpMd5 = null;
 506         File tmpSha1 = null;
 507         File tmpResource = null;
 508
 509         Wagon wagon = null;
 510         try
 511         {
 512             RepositoryURL repoUrl = remoteRepository.getURL();
 513             String protocol = repoUrl.getProtocol();
 514             wagon = (Wagon) wagonFactory.getWagon( "wagon#" + protocol );
 515             if ( wagon == null )
 516             {
 517                 throw new ProxyException( "Unsupported target repository protocol: " + protocol );
 518             }
 519
 520             boolean connected = connectToRepository( connector, wagon, remoteRepository );
 521             if ( connected )
 522             {
 523                 tmpResource = transferSimpleFile( wagon, remoteRepository, remotePath, repository, workingDirectory, resource );
 524
 525                 // TODO: these should be used to validate the download based on the policies, not always downloaded to
 526                 //   save on connections since md5 is rarely used
 527                 tmpSha1 = transferChecksum( wagon, remoteRepository, remotePath, repository, workingDirectory, resource, ".sha1" );
 528                 tmpMd5 = transferChecksum( wagon, remoteRepository, remotePath, repository, workingDirectory, resource, ".md5" );
 529             }
 530         }
 531         catch ( NotFoundException e )
 532         {
 533             urlFailureCache.cacheFailure( url );
 534             throw e;
 535         }
 536         catch ( NotModifiedException e )
 537         {
 538             // Do not cache url here.
 539             throw e;
 540         }
 541         catch ( ProxyException e )
 542         {
 543             urlFailureCache.cacheFailure( url );
 544             throw e;
 545         }
 546         finally
 547         {
 548             if ( wagon != null )
 549             {
 550                 try
 551                 {
 552                     wagon.disconnect();
 553
 554                     // MRM-631 - the lightweight wagon does not reset these - remove if we switch to httpclient based wagon
 555                     if ( previousHttpProxyHost != null )
 556                     {
 557                         System.setProperty( "http.proxyHost", previousHttpProxyHost );
 558                     }
 559                     else
 560                     {
 561                         System.getProperties().remove( "http.proxyHost" );
 562                     }
 563                     if ( previousHttpProxyPort != null )
 564                     {
 565                         System.setProperty( "http.proxyPort", previousHttpProxyPort );
 566                     }
 567                     else
 568                     {
 569                         System.getProperties().remove( "http.proxyPort" );
 570                     }
 571                     if ( previousProxyExclusions != null )
 572                     {
 573                         System.setProperty( "http.nonProxyHosts", previousProxyExclusions );
 574                     }
 575                     else
 576                     {
 577                         System.getProperties().remove( "http.nonProxyHosts" );
 578                     }
 579                 }
 580                 catch ( ConnectionException e )
 581                 {
 582                     log.warn( "Unable to disconnect wagon.", e );
 583                 }
 584             }
 585         }
 586
 587         // Handle post-download policies.
 588         try
 589         {
 590             validatePolicies( this.postDownloadPolicies, connector.getPolicies(), requestProperties, tmpResource );
 591         }
 592         catch ( PolicyViolationException e )
 593         {
 594             log.info( "Transfer invalidated from " + url + " : " + e.getMessage() );
 595             executeConsumers = false;
 596             if ( !fileExists( tmpResource ) )
 597             {
 598                 resource = null;
 599             }
 600         }
 601
 602         if (resource != null)
 603         {
 604             synchronized (resource.getAbsolutePath().intern())
 605             {
 606                 File directory = resource.getParentFile();
 607                 moveFileIfExists(tmpMd5, directory);
 608                 moveFileIfExists(tmpSha1, directory);
 609                 moveFileIfExists(tmpResource, directory);
 610             }
 611         }
 612
 613         if ( executeConsumers )
 614         {
 615             // Just-in-time update of the index and database by executing the consumers for this artifact
 616             consumers.executeConsumers( connector.getSourceRepository().getRepository(), resource );
 617         }
 618
 619         return resource;
 620     }
 621
 622
 623
 624     /**
 625      * Moves the file into repository location if it exists
 626      *
 627      * @param fileToMove this could be either the main artifact, sha1 or md5 checksum file.
 628      * @param directory directory to write files to
 629      */
 630     private void moveFileIfExists(File fileToMove, File directory) throws ProxyException
 631     {
 632         if (fileToMove != null && fileToMove.exists())
 633         {
 634             File newLocation = new File(directory, fileToMove.getName());
 635             moveTempToTarget(fileToMove, newLocation);
 636         }
 637     }
 638
 639     /**
 640      * <p>
 641      * Quietly transfer the checksum file from the remote repository to the local file.
 642      * </p>
 643      *
 644      * @param wagon            the wagon instance (should already be connected) to use.
 645      * @param remoteRepository the remote repository to transfer from.
 646      * @param remotePath       the remote path to the resource to get.
 647      * @param repository       the managed repository that will hold the file
 648      * @param localFile        the local file that should contain the downloaded contents
 649      * @param type             the type of checksum to transfer (example: ".md5" or ".sha1")
 650      * @throws ProxyException if copying the downloaded file into place did not succeed.
 651      */
 652     private File transferChecksum( Wagon wagon, RemoteRepositoryContent remoteRepository, String remotePath,
 653                                    ManagedRepositoryContent repository, File workingDirectory, File localFile, String type )
 654         throws ProxyException
 655     {
 656         File hashFile = new File( localFile.getAbsolutePath() + type );
 657         File tmpChecksum = new File(workingDirectory, hashFile.getName());
 658         String url = remoteRepository.getURL().getUrl() + remotePath;
 659
 660         // Transfer checksum does not use the policy.
 661         if ( urlFailureCache.hasFailedBefore( url + type ) )
 662         {
 663             return null;
 664         }
 665
 666         try
 667         {
 668             transferSimpleFile( wagon, remoteRepository, remotePath + type, repository, workingDirectory, hashFile );
 669             log.debug( "Checksum" + type + " Downloaded: " + hashFile );
 670         }
 671         catch ( NotFoundException e )
 672         {
 673             urlFailureCache.cacheFailure( url + type );
 674             log.debug( "Transfer failed, checksum not found: " + url );
 675             // Consume it, do not pass this on.
 676         }
 677         catch ( NotModifiedException e )
 678         {
 679             log.debug( "Transfer skipped, checksum not modified: " + url );
 680             // Consume it, do not pass this on.
 681         }
 682         catch ( ProxyException e )
 683         {
 684             urlFailureCache.cacheFailure( url + type );
 685             log.warn( "Transfer failed on checksum: " + url + " : " + e.getMessage(), e );
 686             // Critical issue, pass it on.
 687             throw e;
 688         }
 689         return tmpChecksum;
 690     }
 691
 692     /**
 693      * Perform the transfer of the remote file to the local file specified.
 694      *
 695      * @param wagon            the wagon instance to use.
 696      * @param remoteRepository the remote repository to use
 697      * @param remotePath       the remote path to attempt to get
 698      * @param repository       the managed repository that will hold the file
 699      * @param localFile        the local file to save to
 700      * @return The local file that was transfered.
 701      * @throws ProxyException if there was a problem moving the downloaded file into place.
 702      * @throws WagonException if there was a problem tranfering the file.
 703      */
 704     private File transferSimpleFile( Wagon wagon, RemoteRepositoryContent remoteRepository, String remotePath,
 705                                      ManagedRepositoryContent repository, File workingDirectory, File localFile )
 706         throws ProxyException
 707     {
 708         assert ( remotePath != null );
 709
 710         // Transfer the file.
 711         File temp = null;
 712
 713         try
 714         {
 715             temp = new File(workingDirectory, localFile.getName());
 716
 717             boolean success = false;
 718
 719             if ( !localFile.exists() )
 720             {
 721                 log.debug( "Retrieving " + remotePath + " from " + remoteRepository.getRepository().getName() );
 722                 wagon.get( remotePath, temp );
 723                 success = true;
 724
 725                 // You wouldn't get here on failure, a WagonException would have been thrown.
 726                 log.debug( "Downloaded successfully." );
 727             }
 728             else
 729             {
 730                 log.debug( "Retrieving " + remotePath + " from " + remoteRepository.getRepository().getName()
 731                                        + " if updated" );
 732                 success = wagon.getIfNewer( remotePath, temp, localFile.lastModified() );
 733                 if ( !success )
 734                 {
 735                     throw new NotModifiedException(
 736                         "Not downloaded, as local file is newer than remote side: " + localFile.getAbsolutePath() );
 737                 }
 738
 739                 if ( temp.exists() )
 740                 {
 741                     log.debug( "Downloaded successfully." );
 742                 }
 743             }
 744
 745             return temp;
 746         }
 747         catch ( ResourceDoesNotExistException e )
 748         {
 749             throw new NotFoundException(
 750                 "Resource [" + remoteRepository.getURL() + "/" + remotePath + "] does not exist: " + e.getMessage(),
 751                 e );
 752         }
 753         catch ( WagonException e )
 754         {
 755             // TODO: shouldn't have to drill into the cause, but TransferFailedException is often not descriptive enough
 756
 757             String msg =
 758                 "Download failure on resource [" + remoteRepository.getURL() + "/" + remotePath + "]:" + e.getMessage();
 759             if ( e.getCause() != null )
 760             {
 761                 msg += " (cause: " + e.getCause() + ")";
 762             }
 763             throw new ProxyException( msg, e );
 764         }
 765     }
 766
 767     /**
 768      * Apply the policies.
 769      *
 770      * @param policies  the map of policies to execute. (Map of String policy keys, to {@link DownloadPolicy} objects)
 771      * @param settings  the map of settings for the policies to execute. (Map of String policy keys, to String policy setting)
 772      * @param request   the request properties (utilized by the {@link DownloadPolicy#applyPolicy(String,Properties,File)})
 773      * @param localFile the local file (utilized by the {@link DownloadPolicy#applyPolicy(String,Properties,File)})
 774      */
 775     private void validatePolicies( Map<String, ? extends DownloadPolicy> policies, Map<String, String> settings,
 776                                    Properties request, File localFile )
 777         throws PolicyViolationException
 778     {
 779         for ( Entry<String, ? extends DownloadPolicy> entry : policies.entrySet() )
 780         {
 781             String key = entry.getKey();
 782             DownloadPolicy policy = entry.getValue();
 783             String defaultSetting = policy.getDefaultOption();
 784             String setting = StringUtils.defaultString( settings.get( key ), defaultSetting );
 785
 786             log.debug( "Applying [" + key + "] policy with [" + setting + "]" );
 787             try
 788             {
 789                 policy.applyPolicy( setting, request, localFile );
 790             }
 791             catch ( PolicyConfigurationException e )
 792             {
 793                 log.error( e.getMessage(), e );
 794             }
 795         }
 796     }
 797
 798     private void validatePolicies( Map<String, DownloadErrorPolicy> policies, Map<String, String> settings,
 799                                    Properties request, ArtifactReference artifact, RemoteRepositoryContent content,
 800                                    File localFile, ProxyException exception, Map<String, Exception> previousExceptions )
 801         throws ProxyDownloadException
 802     {
 803         boolean process = true;
 804         for ( Entry<String, ? extends DownloadErrorPolicy> entry : policies.entrySet() )
 805         {
 806             String key = entry.getKey();
 807             DownloadErrorPolicy policy = entry.getValue();
 808             String defaultSetting = policy.getDefaultOption();
 809             String setting = StringUtils.defaultString( settings.get( key ), defaultSetting );
 810
 811             log.debug( "Applying [" + key + "] policy with [" + setting + "]" );
 812             try
 813             {
 814                 // all policies must approve the exception, any can cancel
 815                 process = policy.applyPolicy( setting, request, localFile, exception, previousExceptions );
 816                 if ( !process )
 817                 {
 818                     break;
 819                 }
 820             }
 821             catch ( PolicyConfigurationException e )
 822             {
 823                 log.error( e.getMessage(), e );
 824             }
 825         }
 826
 827         if ( process )
 828         {
 829             // if the exception was queued, don't throw it
 830             if ( !previousExceptions.containsKey( content.getId() ) )
 831             {
 832                 throw new ProxyDownloadException(
 833                     "An error occurred in downloading from the remote repository, and the policy is to fail immediately",
 834                     content.getId(), exception );
 835             }
 836         }
 837         else
 838         {
 839             // if the exception was queued, but cancelled, remove it
 840             previousExceptions.remove( content.getId() );
 841         }
 842
 843         log.warn( "Transfer error from repository \"" + content.getRepository().getId() + "\" for artifact " +
 844             Keys.toKey( artifact ) + ", continuing to next repository. Error message: " + exception.getMessage() );
 845         log.debug( "Full stack trace", exception );
 846     }
 847
 848     /**
 849      * Creates a working directory in the repository root for this request
 850      * @param repository
 851      * @return file location of working directory
 852      */
 853     private File createWorkingDirectory(ManagedRepositoryContent repository)
 854     {
 855         //TODO: This is ugly - lets actually clean this up when we get the new repository api
 856         try
 857         {
 858             File tmpDir = File.createTempFile(".workingdirectory", null, new File(repository.getRepoRoot()));
 859             tmpDir.delete();
 860             tmpDir.mkdirs();
 861             return tmpDir;
 862         }
 863         catch (IOException e)
 864         {
 865             throw new RuntimeException("Could not create working directory for this request", e);
 866         }
 867     }
 868
 869     /**
 870      * Used to move the temporary file to its real destination.  This is patterned from the way WagonManager handles
 871      * its downloaded files.
 872      *
 873      * @param temp   The completed download file
 874      * @param target The final location of the downloaded file
 875      * @throws ProxyException when the temp file cannot replace the target file
 876      */
 877     private void moveTempToTarget( File temp, File target )
 878         throws ProxyException
 879     {
 880         if ( target.exists() && !target.delete() )
 881         {
 882             throw new ProxyException( "Unable to overwrite existing target file: " + target.getAbsolutePath() );
 883         }
 884
 885         target.getParentFile().mkdirs();
 886         if ( !temp.renameTo( target ) )
 887         {
 888             log.warn( "Unable to rename tmp file to its final name... resorting to copy command." );
 889
 890             try
 891             {
 892                 FileUtils.copyFile( temp, target );
 893             }
 894             catch ( IOException e )
 895             {
 896                 if (target.exists())
 897                 {
 898                     log.debug("Tried to copy file " + temp.getName() + " to " + target.getAbsolutePath() + " but file with this name already exists.");
 899                 }
 900                 else
 901                 {
 902                     throw new ProxyException( "Cannot copy tmp file " + temp.getAbsolutePath() + " to its final location", e );
 903                 }
 904             }
 905             finally
 906             {
 907                 FileUtils.deleteQuietly(temp);
 908             }
 909         }
 910     }
 911
 912     /**
 913      * Using wagon, connect to the remote repository.
 914      *
 915      * @param connector        the connector configuration to utilize (for obtaining network proxy configuration from)
 916      * @param wagon            the wagon instance to establish the connection on.
 917      * @param remoteRepository the remote repository to connect to.
 918      * @return true if the connection was successful. false if not connected.
 919      */
 920     private boolean connectToRepository( ProxyConnector connector, Wagon wagon,
 921                                          RemoteRepositoryContent remoteRepository )
 922     {
 923         boolean connected = false;
 924
 925         final ProxyInfo networkProxy;
 926         synchronized ( this.networkProxyMap )
 927         {
 928             networkProxy = (ProxyInfo) this.networkProxyMap.get( connector.getProxyId() );
 929         }
 930
 931         if ( log.isDebugEnabled() )
 932         {
 933             if ( networkProxy != null )
 934             {
 935                 // TODO: move to proxyInfo.toString()
 936                 String msg =
 937                     "Using network proxy " + networkProxy.getHost() + ":" + networkProxy.getPort()
 938                         + " to connect to remote repository " + remoteRepository.getURL();
 939                 if ( networkProxy.getNonProxyHosts() != null )
 940                 {
 941                     msg += "; excluding hosts: " + networkProxy.getNonProxyHosts();
 942                 }
 943                 if ( StringUtils.isNotBlank( networkProxy.getUserName() ) )
 944                 {
 945                     msg += "; as user: " + networkProxy.getUserName();
 946                 }
 947                 log.debug( msg );
 948             }
 949         }
 950
 951         AuthenticationInfo authInfo = null;
 952         String username = remoteRepository.getRepository().getUsername();
 953         String password = remoteRepository.getRepository().getPassword();
 954
 955         if ( StringUtils.isNotBlank( username ) && StringUtils.isNotBlank( password ) )
 956         {
 957             log.debug( "Using username " + username + " to connect to remote repository "
 958                 + remoteRepository.getURL() );
 959             authInfo = new AuthenticationInfo();
 960             authInfo.setUserName( username );
 961             authInfo.setPassword( password );
 962         }
 963
 964         //Convert seconds to milliseconds
 965         int timeoutInMilliseconds = remoteRepository.getRepository().getTimeout() * 1000;
 966
 967         //Set timeout
 968         wagon.setTimeout(timeoutInMilliseconds);
 969
 970         try
 971         {
 972             Repository wagonRepository = new Repository( remoteRepository.getId(), remoteRepository.getURL().toString() );
 973             wagon.connect( wagonRepository, authInfo, networkProxy );
 974             connected = true;
 975         }
 976         catch ( ConnectionException e )
 977         {
 978             log.warn(
 979                 "Could not connect to " + remoteRepository.getRepository().getName() + ": " + e.getMessage() );
 980             connected = false;
 981         }
 982         catch ( AuthenticationException e )
 983         {
 984             log.warn(
 985                 "Could not connect to " + remoteRepository.getRepository().getName() + ": " + e.getMessage() );
 986             connected = false;
 987         }
 988
 989         return connected;
 990     }
 991
 992     /**
 993      * Tests whitelist and blacklist patterns against path.
 994      *
 995      * @param path     the path to test.
 996      * @param patterns the list of patterns to check.
 997      * @return true if the path matches at least 1 pattern in the provided patterns list.
 998      */
 999     private boolean matchesPattern( String path, List<String> patterns )
1000     {
1001         if ( CollectionUtils.isEmpty( patterns ) )
1002         {
1003             return false;
1004         }
1005
1006         for ( String pattern : patterns )
1007         {
1008             if ( SelectorUtils.matchPath( pattern, path, false ) )
1009             {
1010                 return true;
1011             }
1012         }
1013
1014         return false;
1015     }
1016
1017     /**
1018      * TODO: Ensure that list is correctly ordered based on configuration. See MRM-477
1019      */
1020     public List<ProxyConnector> getProxyConnectors( ManagedRepositoryContent repository )
1021     {
1022         synchronized ( this.proxyConnectorMap )
1023         {
1024             List<ProxyConnector> ret = (List<ProxyConnector>) this.proxyConnectorMap.get( repository.getId() );
1025             if ( ret == null )
1026             {
1027                 return Collections.emptyList();
1028             }
1029
1030             Collections.sort( ret, ProxyConnectorOrderComparator.getInstance() );
1031             return ret;
1032         }
1033     }
1034
1035     public void afterConfigurationChange( Registry registry, String propertyName, Object propertyValue )
1036     {
1037         if ( ConfigurationNames.isNetworkProxy( propertyName ) ||
1038             ConfigurationNames.isManagedRepositories( propertyName ) ||
1039             ConfigurationNames.isRemoteRepositories( propertyName ) ||
1040             ConfigurationNames.isProxyConnector( propertyName ) )
1041         {
1042             initConnectorsAndNetworkProxies();
1043         }
1044     }
1045
1046     public void beforeConfigurationChange( Registry registry, String propertyName, Object propertyValue )
1047     {
1048         /* do nothing */
1049     }
1050
1051     @SuppressWarnings("unchecked")
1052     private void initConnectorsAndNetworkProxies()
1053     {
1054         synchronized ( this.proxyConnectorMap )
1055         {
1056             ProxyConnectorOrderComparator proxyOrderSorter = new ProxyConnectorOrderComparator();
1057             this.proxyConnectorMap.clear();
1058
1059             List<ProxyConnectorConfiguration> proxyConfigs = archivaConfiguration.getConfiguration()
1060                 .getProxyConnectors();
1061             for ( ProxyConnectorConfiguration proxyConfig : proxyConfigs )
1062             {
1063                 String key = proxyConfig.getSourceRepoId();
1064
1065                 try
1066                 {
1067                     // Create connector object.
1068                     ProxyConnector connector = new ProxyConnector();
1069
1070                     connector.setSourceRepository( repositoryFactory.getManagedRepositoryContent( proxyConfig
1071                         .getSourceRepoId() ) );
1072                     connector.setTargetRepository( repositoryFactory.getRemoteRepositoryContent( proxyConfig
1073                         .getTargetRepoId() ) );
1074
1075                     connector.setProxyId( proxyConfig.getProxyId() );
1076                     connector.setPolicies( proxyConfig.getPolicies() );
1077                     connector.setOrder( proxyConfig.getOrder() );
1078                     connector.setDisabled( proxyConfig.isDisabled() );
1079
1080                     // Copy any blacklist patterns.
1081                     List<String> blacklist = new ArrayList<String>();
1082                     if ( CollectionUtils.isNotEmpty( proxyConfig.getBlackListPatterns() ) )
1083                     {
1084                         blacklist.addAll( proxyConfig.getBlackListPatterns() );
1085                     }
1086                     connector.setBlacklist( blacklist );
1087
1088                     // Copy any whitelist patterns.
1089                     List<String> whitelist = new ArrayList<String>();
1090                     if ( CollectionUtils.isNotEmpty( proxyConfig.getWhiteListPatterns() ) )
1091                     {
1092                         whitelist.addAll( proxyConfig.getWhiteListPatterns() );
1093                     }
1094                     connector.setWhitelist( whitelist );
1095
1096                     // Get other connectors
1097                     List<ProxyConnector> connectors = this.proxyConnectorMap.get( key );
1098                     if ( connectors == null )
1099                     {
1100                         // Create if we are the first.
1101                         connectors = new ArrayList<ProxyConnector>();
1102                     }
1103
1104                     // Add the connector.
1105                     connectors.add( connector );
1106
1107                     // Ensure the list is sorted.
1108                     Collections.sort( connectors, proxyOrderSorter );
1109
1110                     // Set the key to the list of connectors.
1111                     this.proxyConnectorMap.put( key, connectors );
1112                 }
1113                 catch ( RepositoryNotFoundException e )
1114                 {
1115                     log.warn( "Unable to use proxy connector: " + e.getMessage(), e );
1116                 }
1117                 catch ( RepositoryException e )
1118                 {
1119                     log.warn( "Unable to use proxy connector: " + e.getMessage(), e );
1120                 }
1121             }
1122
1123         }
1124
1125         synchronized ( this.networkProxyMap )
1126         {
1127             this.networkProxyMap.clear();
1128
1129             List<NetworkProxyConfiguration> networkProxies = archivaConfiguration.getConfiguration().getNetworkProxies();
1130             for ( NetworkProxyConfiguration networkProxyConfig : networkProxies )
1131             {
1132                 String key = networkProxyConfig.getId();
1133
1134                 ProxyInfo proxy = new ProxyInfo();
1135
1136                 proxy.setType( networkProxyConfig.getProtocol() );
1137                 proxy.setHost( networkProxyConfig.getHost() );
1138                 proxy.setPort( networkProxyConfig.getPort() );
1139                 proxy.setUserName( networkProxyConfig.getUsername() );
1140                 proxy.setPassword( networkProxyConfig.getPassword() );
1141
1142                 this.networkProxyMap.put( key, proxy );
1143             }
1144         }
1145     }
1146
1147     public void initialize()
1148         throws InitializationException
1149     {
1150         initConnectorsAndNetworkProxies();
1151         archivaConfiguration.addChangeListener( this );
1152     }
1153 }