1 package org.apache.archiva.proxy;
4 * Licensed to the Apache Software Foundation (ASF) under one
5 * or more contributor license agreements. See the NOTICE file
6 * distributed with this work for additional information
7 * regarding copyright ownership. The ASF licenses this file
8 * to you under the Apache License, Version 2.0 (the
9 * "License"); you may not use this file except in compliance
10 * with the License. You may obtain a copy of the License at
12 * http://www.apache.org/licenses/LICENSE-2.0
14 * Unless required by applicable law or agreed to in writing,
15 * software distributed under the License is distributed on an
16 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17 * KIND, either express or implied. See the License for the
18 * specific language governing permissions and limitations
22 import org.apache.archiva.admin.model.RepositoryAdminException;
23 import org.apache.archiva.admin.model.beans.NetworkProxy;
24 import org.apache.archiva.admin.model.beans.ProxyConnectorRuleType;
25 import org.apache.archiva.admin.model.beans.RemoteRepository;
26 import org.apache.archiva.admin.model.networkproxy.NetworkProxyAdmin;
27 import org.apache.archiva.common.filelock.FileLockException;
28 import org.apache.archiva.common.filelock.FileLockManager;
29 import org.apache.archiva.common.filelock.FileLockTimeoutException;
30 import org.apache.archiva.common.filelock.Lock;
31 import org.apache.archiva.configuration.ArchivaConfiguration;
32 import org.apache.archiva.configuration.Configuration;
33 import org.apache.archiva.configuration.ConfigurationNames;
34 import org.apache.archiva.configuration.NetworkProxyConfiguration;
35 import org.apache.archiva.configuration.ProxyConnectorConfiguration;
36 import org.apache.archiva.configuration.ProxyConnectorRuleConfiguration;
37 import org.apache.archiva.model.ArtifactReference;
38 import org.apache.archiva.model.Keys;
39 import org.apache.archiva.model.RepositoryURL;
40 import org.apache.archiva.policies.DownloadErrorPolicy;
41 import org.apache.archiva.policies.DownloadPolicy;
42 import org.apache.archiva.policies.PolicyConfigurationException;
43 import org.apache.archiva.policies.PolicyViolationException;
44 import org.apache.archiva.policies.PostDownloadPolicy;
45 import org.apache.archiva.policies.PreDownloadPolicy;
46 import org.apache.archiva.policies.ProxyDownloadException;
47 import org.apache.archiva.policies.urlcache.UrlFailureCache;
48 import org.apache.archiva.proxy.common.WagonFactory;
49 import org.apache.archiva.proxy.common.WagonFactoryException;
50 import org.apache.archiva.proxy.common.WagonFactoryRequest;
51 import org.apache.archiva.proxy.model.ProxyConnector;
52 import org.apache.archiva.proxy.model.RepositoryProxyConnectors;
53 import org.apache.archiva.redback.components.registry.Registry;
54 import org.apache.archiva.redback.components.registry.RegistryListener;
55 import org.apache.archiva.redback.components.taskqueue.TaskQueueException;
56 import org.apache.archiva.repository.ManagedRepositoryContent;
57 import org.apache.archiva.repository.RemoteRepositoryContent;
58 import org.apache.archiva.repository.RepositoryContentFactory;
59 import org.apache.archiva.repository.RepositoryException;
60 import org.apache.archiva.repository.RepositoryNotFoundException;
61 import org.apache.archiva.repository.metadata.MetadataTools;
62 import org.apache.archiva.repository.metadata.RepositoryMetadataException;
63 import org.apache.archiva.scheduler.ArchivaTaskScheduler;
64 import org.apache.archiva.scheduler.repository.model.RepositoryTask;
65 import org.apache.commons.collections.CollectionUtils;
66 import org.apache.commons.io.FileUtils;
67 import org.apache.commons.io.FilenameUtils;
68 import org.apache.commons.lang.StringUtils;
69 import org.apache.commons.lang.SystemUtils;
70 import org.apache.maven.wagon.ConnectionException;
71 import org.apache.maven.wagon.ResourceDoesNotExistException;
72 import org.apache.maven.wagon.Wagon;
73 import org.apache.maven.wagon.WagonException;
74 import org.apache.maven.wagon.authentication.AuthenticationException;
75 import org.apache.maven.wagon.authentication.AuthenticationInfo;
76 import org.apache.maven.wagon.proxy.ProxyInfo;
77 import org.apache.maven.wagon.repository.Repository;
78 import org.apache.tools.ant.types.selectors.SelectorUtils;
79 import org.slf4j.Logger;
80 import org.slf4j.LoggerFactory;
81 import org.slf4j.MarkerFactory;
82 import org.springframework.stereotype.Service;
84 import javax.annotation.PostConstruct;
85 import javax.inject.Inject;
86 import javax.inject.Named;
88 import java.io.IOException;
89 import java.nio.file.Files;
90 import java.util.ArrayList;
91 import java.util.Collections;
92 import java.util.HashMap;
93 import java.util.LinkedHashMap;
94 import java.util.List;
96 import java.util.Map.Entry;
97 import java.util.Properties;
98 import java.util.concurrent.ConcurrentHashMap;
101 * DefaultRepositoryProxyConnectors
103 * TODO exception handling needs work - "not modified" is not really an exceptional case, and it has more layers than
104 * your average brown onion
106 @Service("repositoryProxyConnectors#default")
107 public class DefaultRepositoryProxyConnectors
108 implements RepositoryProxyConnectors, RegistryListener
110 private Logger log = LoggerFactory.getLogger( DefaultRepositoryProxyConnectors.class );
113 @Named(value = "archivaConfiguration#default")
114 private ArchivaConfiguration archivaConfiguration;
117 @Named(value = "repositoryContentFactory#default")
118 private RepositoryContentFactory repositoryFactory;
121 @Named(value = "metadataTools#default")
122 private MetadataTools metadataTools;
125 private Map<String, PreDownloadPolicy> preDownloadPolicies;
128 private Map<String, PostDownloadPolicy> postDownloadPolicies;
131 private Map<String, DownloadErrorPolicy> downloadErrorPolicies;
134 private UrlFailureCache urlFailureCache;
136 private Map<String, List<ProxyConnector>> proxyConnectorMap = new HashMap<>();
138 private Map<String, ProxyInfo> networkProxyMap = new ConcurrentHashMap<>();
141 private WagonFactory wagonFactory;
144 @Named(value = "archivaTaskScheduler#repository")
145 private ArchivaTaskScheduler scheduler;
148 private NetworkProxyAdmin networkProxyAdmin;
151 @Named(value = "fileLockManager#default")
152 private FileLockManager fileLockManager;
155 public void initialize()
157 initConnectorsAndNetworkProxies();
158 archivaConfiguration.addChangeListener( this );
162 @SuppressWarnings("unchecked")
163 private void initConnectorsAndNetworkProxies()
166 ProxyConnectorOrderComparator proxyOrderSorter = new ProxyConnectorOrderComparator();
167 this.proxyConnectorMap.clear();
169 Configuration configuration = archivaConfiguration.getConfiguration();
171 List<ProxyConnectorRuleConfiguration> allProxyConnectorRuleConfigurations =
172 configuration.getProxyConnectorRuleConfigurations();
174 List<ProxyConnectorConfiguration> proxyConfigs = configuration.getProxyConnectors();
175 for ( ProxyConnectorConfiguration proxyConfig : proxyConfigs )
177 String key = proxyConfig.getSourceRepoId();
181 // Create connector object.
182 ProxyConnector connector = new ProxyConnector();
184 connector.setSourceRepository(
185 repositoryFactory.getManagedRepositoryContent( proxyConfig.getSourceRepoId() ) );
186 connector.setTargetRepository(
187 repositoryFactory.getRemoteRepositoryContent( proxyConfig.getTargetRepoId() ) );
189 connector.setProxyId( proxyConfig.getProxyId() );
190 connector.setPolicies( proxyConfig.getPolicies() );
191 connector.setOrder( proxyConfig.getOrder() );
192 connector.setDisabled( proxyConfig.isDisabled() );
194 // Copy any blacklist patterns.
195 List<String> blacklist = new ArrayList<>( 0 );
196 if ( CollectionUtils.isNotEmpty( proxyConfig.getBlackListPatterns() ) )
198 blacklist.addAll( proxyConfig.getBlackListPatterns() );
200 connector.setBlacklist( blacklist );
202 // Copy any whitelist patterns.
203 List<String> whitelist = new ArrayList<>( 0 );
204 if ( CollectionUtils.isNotEmpty( proxyConfig.getWhiteListPatterns() ) )
206 whitelist.addAll( proxyConfig.getWhiteListPatterns() );
208 connector.setWhitelist( whitelist );
210 List<ProxyConnectorRuleConfiguration> proxyConnectorRuleConfigurations =
211 findProxyConnectorRules( connector.getSourceRepository().getId(),
212 connector.getTargetRepository().getId(),
213 allProxyConnectorRuleConfigurations );
215 if ( !proxyConnectorRuleConfigurations.isEmpty() )
217 for ( ProxyConnectorRuleConfiguration proxyConnectorRuleConfiguration : proxyConnectorRuleConfigurations )
219 if ( StringUtils.equals( proxyConnectorRuleConfiguration.getRuleType(),
220 ProxyConnectorRuleType.BLACK_LIST.getRuleType() ) )
222 connector.getBlacklist().add( proxyConnectorRuleConfiguration.getPattern() );
225 if ( StringUtils.equals( proxyConnectorRuleConfiguration.getRuleType(),
226 ProxyConnectorRuleType.WHITE_LIST.getRuleType() ) )
228 connector.getWhitelist().add( proxyConnectorRuleConfiguration.getPattern() );
233 // Get other connectors
234 List<ProxyConnector> connectors = this.proxyConnectorMap.get( key );
235 if ( connectors == null )
237 // Create if we are the first.
238 connectors = new ArrayList<>( 1 );
241 // Add the connector.
242 connectors.add( connector );
244 // Ensure the list is sorted.
245 Collections.sort( connectors, proxyOrderSorter );
247 // Set the key to the list of connectors.
248 this.proxyConnectorMap.put( key, connectors );
250 catch ( RepositoryNotFoundException e )
252 log.warn( "Unable to use proxy connector: {}", e.getMessage(), e );
254 catch ( RepositoryException e )
256 log.warn( "Unable to use proxy connector: {}", e.getMessage(), e );
262 this.networkProxyMap.clear();
264 List<NetworkProxyConfiguration> networkProxies = archivaConfiguration.getConfiguration().getNetworkProxies();
265 for ( NetworkProxyConfiguration networkProxyConfig : networkProxies )
267 String key = networkProxyConfig.getId();
269 ProxyInfo proxy = new ProxyInfo();
271 proxy.setType( networkProxyConfig.getProtocol() );
272 proxy.setHost( networkProxyConfig.getHost() );
273 proxy.setPort( networkProxyConfig.getPort() );
274 proxy.setUserName( networkProxyConfig.getUsername() );
275 proxy.setPassword( networkProxyConfig.getPassword() );
277 this.networkProxyMap.put( key, proxy );
282 private List<ProxyConnectorRuleConfiguration> findProxyConnectorRules( String sourceRepository,
283 String targetRepository,
284 List<ProxyConnectorRuleConfiguration> all )
286 List<ProxyConnectorRuleConfiguration> proxyConnectorRuleConfigurations = new ArrayList<>();
288 for ( ProxyConnectorRuleConfiguration proxyConnectorRuleConfiguration : all )
290 for ( ProxyConnectorConfiguration proxyConnector : proxyConnectorRuleConfiguration.getProxyConnectors() )
292 if ( StringUtils.equals( sourceRepository, proxyConnector.getSourceRepoId() ) && StringUtils.equals(
293 targetRepository, proxyConnector.getTargetRepoId() ) )
295 proxyConnectorRuleConfigurations.add( proxyConnectorRuleConfiguration );
300 return proxyConnectorRuleConfigurations;
304 public File fetchFromProxies( ManagedRepositoryContent repository, ArtifactReference artifact )
305 throws ProxyDownloadException
307 File localFile = toLocalFile( repository, artifact );
309 Properties requestProperties = new Properties();
310 requestProperties.setProperty( "filetype", "artifact" );
311 requestProperties.setProperty( "version", artifact.getVersion() );
312 requestProperties.setProperty( "managedRepositoryId", repository.getId() );
314 List<ProxyConnector> connectors = getProxyConnectors( repository );
315 Map<String, Exception> previousExceptions = new LinkedHashMap<>();
316 for ( ProxyConnector connector : connectors )
318 if ( connector.isDisabled() )
323 RemoteRepositoryContent targetRepository = connector.getTargetRepository();
324 requestProperties.setProperty( "remoteRepositoryId", targetRepository.getId() );
326 String targetPath = targetRepository.toPath( artifact );
328 if ( SystemUtils.IS_OS_WINDOWS )
330 // toPath use system PATH_SEPARATOR so on windows url are \ which doesn't work very well :-)
331 targetPath = FilenameUtils.separatorsToUnix( targetPath );
336 File downloadedFile =
337 transferFile( connector, targetRepository, targetPath, repository, localFile, requestProperties,
340 if ( fileExists( downloadedFile ) )
342 log.debug( "Successfully transferred: {}", downloadedFile.getAbsolutePath() );
343 return downloadedFile;
346 catch ( NotFoundException e )
348 log.debug( "Artifact {} not found on repository \"{}\".", Keys.toKey( artifact ),
349 targetRepository.getRepository().getId() );
351 catch ( NotModifiedException e )
353 log.debug( "Artifact {} not updated on repository \"{}\".", Keys.toKey( artifact ),
354 targetRepository.getRepository().getId() );
356 catch ( ProxyException | RepositoryAdminException e )
358 validatePolicies( this.downloadErrorPolicies, connector.getPolicies(), requestProperties, artifact,
359 targetRepository, localFile, e, previousExceptions );
363 if ( !previousExceptions.isEmpty() )
365 throw new ProxyDownloadException( "Failures occurred downloading from some remote repositories",
366 previousExceptions );
369 log.debug( "Exhausted all target repositories, artifact {} not found.", Keys.toKey( artifact ) );
375 public File fetchFromProxies( ManagedRepositoryContent repository, String path )
377 File localFile = new File( repository.getRepoRoot(), path );
379 // no update policies for these paths
380 if ( localFile.exists() )
385 Properties requestProperties = new Properties();
386 requestProperties.setProperty( "filetype", "resource" );
387 requestProperties.setProperty( "managedRepositoryId", repository.getId() );
389 List<ProxyConnector> connectors = getProxyConnectors( repository );
390 for ( ProxyConnector connector : connectors )
392 if ( connector.isDisabled() )
397 RemoteRepositoryContent targetRepository = connector.getTargetRepository();
398 requestProperties.setProperty( "remoteRepositoryId", targetRepository.getId() );
400 String targetPath = path;
404 File downloadedFile =
405 transferFile( connector, targetRepository, targetPath, repository, localFile, requestProperties,
408 if ( fileExists( downloadedFile ) )
410 log.debug( "Successfully transferred: {}", downloadedFile.getAbsolutePath() );
411 return downloadedFile;
414 catch ( NotFoundException e )
416 log.debug( "Resource {} not found on repository \"{}\".", path,
417 targetRepository.getRepository().getId() );
419 catch ( NotModifiedException e )
421 log.debug( "Resource {} not updated on repository \"{}\".", path,
422 targetRepository.getRepository().getId() );
424 catch ( ProxyException e )
427 "Transfer error from repository {} for resource {}, continuing to next repository. Error message: {}",
428 targetRepository.getRepository().getId(), path, e.getMessage() );
429 log.debug( MarkerFactory.getDetachedMarker( "transfer.error" ),
430 "Transfer error from repository \"" + targetRepository.getRepository().getId()
431 + "\" for resource " + path + ", continuing to next repository. Error message: {}",
435 catch ( RepositoryAdminException e )
437 log.debug( MarkerFactory.getDetachedMarker( "transfer.error" ),
438 "Transfer error from repository {} for resource {}, continuing to next repository. Error message: {}",
439 targetRepository.getRepository().getId(), path, e.getMessage(), e );
440 log.debug( MarkerFactory.getDetachedMarker( "transfer.error" ), "Full stack trace", e );
444 log.debug( "Exhausted all target repositories, resource {} not found.", path );
450 public File fetchMetatadaFromProxies( ManagedRepositoryContent repository, String logicalPath )
452 File localFile = new File( repository.getRepoRoot(), logicalPath );
454 Properties requestProperties = new Properties();
455 requestProperties.setProperty( "filetype", "metadata" );
456 boolean metadataNeedsUpdating = false;
457 long originalTimestamp = getLastModified( localFile );
459 List<ProxyConnector> connectors = getProxyConnectors( repository );
460 for ( ProxyConnector connector : connectors )
462 if ( connector.isDisabled() )
467 RemoteRepositoryContent targetRepository = connector.getTargetRepository();
469 File localRepoFile = toLocalRepoFile( repository, targetRepository, logicalPath );
470 long originalMetadataTimestamp = getLastModified( localRepoFile );
474 transferFile( connector, targetRepository, logicalPath, repository, localRepoFile, requestProperties,
477 if ( hasBeenUpdated( localRepoFile, originalMetadataTimestamp ) )
479 metadataNeedsUpdating = true;
482 catch ( NotFoundException e )
485 log.debug( "Metadata {} not found on remote repository '{}'.", logicalPath,
486 targetRepository.getRepository().getId(), e );
489 catch ( NotModifiedException e )
492 log.debug( "Metadata {} not updated on remote repository '{}'.", logicalPath,
493 targetRepository.getRepository().getId(), e );
496 catch ( ProxyException | RepositoryAdminException e )
499 "Transfer error from repository {} for versioned Metadata {}, continuing to next repository. Error message: {}",
500 targetRepository.getRepository().getId(), logicalPath, e.getMessage() );
501 log.debug( "Full stack trace", e );
505 if ( hasBeenUpdated( localFile, originalTimestamp ) )
507 metadataNeedsUpdating = true;
510 if ( metadataNeedsUpdating || !localFile.exists() )
514 metadataTools.updateMetadata( repository, logicalPath );
516 catch ( RepositoryMetadataException e )
518 log.warn( "Unable to update metadata {}:{}", localFile.getAbsolutePath(), e.getMessage(), e );
522 if ( fileExists( localFile ) )
532 * @param remoteRepository
539 * @param workingDirectory
541 * @throws ProxyException
542 * @throws NotModifiedException
543 * @throws org.apache.archiva.admin.model.RepositoryAdminException
545 protected void transferResources( ProxyConnector connector, RemoteRepositoryContent remoteRepository, File tmpMd5,
546 File tmpSha1, File tmpResource, String url, String remotePath, File resource,
547 File workingDirectory, ManagedRepositoryContent repository )
548 throws ProxyException, NotModifiedException, RepositoryAdminException
553 RepositoryURL repoUrl = remoteRepository.getURL();
554 String protocol = repoUrl.getProtocol();
555 NetworkProxy networkProxy = null;
556 if ( StringUtils.isNotBlank( connector.getProxyId() ) )
558 networkProxy = networkProxyAdmin.getNetworkProxy( connector.getProxyId() );
560 WagonFactoryRequest wagonFactoryRequest = new WagonFactoryRequest( "wagon#" + protocol,
561 remoteRepository.getRepository().getExtraHeaders() ).networkProxy(
563 wagon = wagonFactory.getWagon( wagonFactoryRequest );
566 throw new ProxyException( "Unsupported target repository protocol: " + protocol );
571 throw new ProxyException( "Unsupported target repository protocol: " + protocol );
574 boolean connected = connectToRepository( connector, wagon, remoteRepository );
577 transferArtifact( wagon, remoteRepository, remotePath, repository, resource, workingDirectory,
580 // TODO: these should be used to validate the download based on the policies, not always downloaded
582 // save on connections since md5 is rarely used
583 transferChecksum( wagon, remoteRepository, remotePath, repository, resource, workingDirectory, ".sha1",
585 transferChecksum( wagon, remoteRepository, remotePath, repository, resource, workingDirectory, ".md5",
589 catch ( NotFoundException e )
591 urlFailureCache.cacheFailure( url );
594 catch ( NotModifiedException e )
596 // Do not cache url here.
599 catch ( ProxyException e )
601 urlFailureCache.cacheFailure( url );
604 catch ( WagonFactoryException e )
606 throw new ProxyException( e.getMessage(), e );
616 catch ( ConnectionException e )
618 log.warn( "Unable to disconnect wagon.", e );
624 private void transferArtifact( Wagon wagon, RemoteRepositoryContent remoteRepository, String remotePath,
625 ManagedRepositoryContent repository, File resource, File tmpDirectory,
627 throws ProxyException
629 transferSimpleFile( wagon, remoteRepository, remotePath, repository, resource, destFile );
632 private long getLastModified( File file )
634 if ( !file.exists() || !file.isFile() )
639 return file.lastModified();
642 private boolean hasBeenUpdated( File file, long originalLastModified )
644 if ( !file.exists() || !file.isFile() )
649 long currentLastModified = getLastModified( file );
650 return ( currentLastModified > originalLastModified );
653 private File toLocalRepoFile( ManagedRepositoryContent repository, RemoteRepositoryContent targetRepository,
656 String repoPath = metadataTools.getRepositorySpecificName( targetRepository, targetPath );
657 return new File( repository.getRepoRoot(), repoPath );
661 * Test if the provided ManagedRepositoryContent has any proxies configured for it.
664 public boolean hasProxies( ManagedRepositoryContent repository )
666 synchronized ( this.proxyConnectorMap )
668 return this.proxyConnectorMap.containsKey( repository.getId() );
672 private File toLocalFile( ManagedRepositoryContent repository, ArtifactReference artifact )
674 return repository.toFile( artifact );
678 * Simple method to test if the file exists on the local disk.
680 * @param file the file to test. (may be null)
681 * @return true if file exists. false if the file param is null, doesn't exist, or is not of type File.
683 private boolean fileExists( File file )
690 if ( !file.exists() )
695 return file.isFile();
699 * Perform the transfer of the file.
701 * @param connector the connector configuration to use.
702 * @param remoteRepository the remote repository get the resource from.
703 * @param remotePath the path in the remote repository to the resource to get.
704 * @param repository the managed repository that will hold the file
705 * @param resource the local file to place the downloaded resource into
706 * @param requestProperties the request properties to utilize for policy handling.
707 * @param executeConsumers whether to execute the consumers after proxying
708 * @return the local file that was downloaded, or null if not downloaded.
709 * @throws NotFoundException if the file was not found on the remote repository.
710 * @throws NotModifiedException if the localFile was present, and the resource was present on remote repository, but
711 * the remote resource is not newer than the local File.
712 * @throws ProxyException if transfer was unsuccessful.
714 private File transferFile( ProxyConnector connector, RemoteRepositoryContent remoteRepository, String remotePath,
715 ManagedRepositoryContent repository, File resource, Properties requestProperties,
716 boolean executeConsumers )
717 throws ProxyException, NotModifiedException, RepositoryAdminException
719 String url = remoteRepository.getURL().getUrl();
720 if ( !url.endsWith( "/" ) )
724 url = url + remotePath;
725 requestProperties.setProperty( "url", url );
727 // Is a whitelist defined?
728 if ( CollectionUtils.isNotEmpty( connector.getWhitelist() ) )
730 // Path must belong to whitelist.
731 if ( !matchesPattern( remotePath, connector.getWhitelist() ) )
733 log.debug( "Path [{}] is not part of defined whitelist (skipping transfer from repository [{}]).",
734 remotePath, remoteRepository.getRepository().getName() );
739 // Is target path part of blacklist?
740 if ( matchesPattern( remotePath, connector.getBlacklist() ) )
742 log.debug( "Path [{}] is part of blacklist (skipping transfer from repository [{}]).", remotePath,
743 remoteRepository.getRepository().getName() );
747 // Handle pre-download policy
750 validatePolicies( this.preDownloadPolicies, connector.getPolicies(), requestProperties, resource );
752 catch ( PolicyViolationException e )
754 String emsg = "Transfer not attempted on " + url + " : " + e.getMessage();
755 if ( fileExists( resource ) )
757 log.debug( "{} : using already present local file.", emsg );
765 File workingDirectory = createWorkingDirectory( repository );
766 File tmpResource = new File( workingDirectory, resource.getName() );
767 File tmpMd5 = new File( workingDirectory, resource.getName() + ".md5" );
768 File tmpSha1 = new File( workingDirectory, resource.getName() + ".sha1" );
773 transferResources( connector, remoteRepository, tmpMd5, tmpSha1, tmpResource, url, remotePath, resource,
774 workingDirectory, repository );
776 // Handle post-download policies.
779 validatePolicies( this.postDownloadPolicies, connector.getPolicies(), requestProperties, tmpResource );
781 catch ( PolicyViolationException e )
783 log.warn( "Transfer invalidated from {} : {}", url, e.getMessage() );
784 executeConsumers = false;
785 if ( !fileExists( tmpResource ) )
791 if ( resource != null )
793 synchronized ( resource.getAbsolutePath().intern() )
795 File directory = resource.getParentFile();
796 moveFileIfExists( tmpMd5, directory );
797 moveFileIfExists( tmpSha1, directory );
798 moveFileIfExists( tmpResource, directory );
804 FileUtils.deleteQuietly( workingDirectory );
807 if ( executeConsumers )
809 // Just-in-time update of the index and database by executing the consumers for this artifact
810 //consumers.executeConsumers( connector.getSourceRepository().getRepository(), resource );
811 queueRepositoryTask( connector.getSourceRepository().getRepository().getId(), resource );
817 private void queueRepositoryTask( String repositoryId, File localFile )
819 RepositoryTask task = new RepositoryTask();
820 task.setRepositoryId( repositoryId );
821 task.setResourceFile( localFile );
822 task.setUpdateRelatedArtifacts( true );
823 task.setScanAll( true );
827 scheduler.queueTask( task );
829 catch ( TaskQueueException e )
831 log.error( "Unable to queue repository task to execute consumers on resource file ['" + localFile.getName()
837 * Moves the file into repository location if it exists
839 * @param fileToMove this could be either the main artifact, sha1 or md5 checksum file.
840 * @param directory directory to write files to
842 private void moveFileIfExists( File fileToMove, File directory )
843 throws ProxyException
845 if ( fileToMove != null && fileToMove.exists() )
847 File newLocation = new File( directory, fileToMove.getName() );
848 moveTempToTarget( fileToMove, newLocation );
854 * Quietly transfer the checksum file from the remote repository to the local file.
857 * @param wagon the wagon instance (should already be connected) to use.
858 * @param remoteRepository the remote repository to transfer from.
859 * @param remotePath the remote path to the resource to get.
860 * @param repository the managed repository that will hold the file
861 * @param resource the local file that should contain the downloaded contents
862 * @param tmpDirectory the temporary directory to download to
863 * @param ext the type of checksum to transfer (example: ".md5" or ".sha1")
864 * @throws ProxyException if copying the downloaded file into place did not succeed.
866 private void transferChecksum( Wagon wagon, RemoteRepositoryContent remoteRepository, String remotePath,
867 ManagedRepositoryContent repository, File resource, File tmpDirectory, String ext,
869 throws ProxyException
871 String url = remoteRepository.getURL().getUrl() + remotePath + ext;
873 // Transfer checksum does not use the policy.
874 if ( urlFailureCache.hasFailedBefore( url ) )
881 transferSimpleFile( wagon, remoteRepository, remotePath + ext, repository, resource, destFile );
882 log.debug( "Checksum {} Downloaded: {} to move to {}", url, destFile, resource );
884 catch ( NotFoundException e )
886 urlFailureCache.cacheFailure( url );
887 log.debug( "Transfer failed, checksum not found: {}", url );
888 // Consume it, do not pass this on.
890 catch ( NotModifiedException e )
892 log.debug( "Transfer skipped, checksum not modified: {}", url );
893 // Consume it, do not pass this on.
895 catch ( ProxyException e )
897 urlFailureCache.cacheFailure( url );
898 log.warn( "Transfer failed on checksum: {} : {}", url, e.getMessage(), e );
899 // Critical issue, pass it on.
905 * Perform the transfer of the remote file to the local file specified.
907 * @param wagon the wagon instance to use.
908 * @param remoteRepository the remote repository to use
909 * @param remotePath the remote path to attempt to get
910 * @param repository the managed repository that will hold the file
911 * @param origFile the local file to save to
912 * @throws ProxyException if there was a problem moving the downloaded file into place.
914 private void transferSimpleFile( Wagon wagon, RemoteRepositoryContent remoteRepository, String remotePath,
915 ManagedRepositoryContent repository, File origFile, File destFile )
916 throws ProxyException
918 assert ( remotePath != null );
920 // Transfer the file.
923 boolean success = false;
925 if ( !origFile.exists() )
927 log.debug( "Retrieving {} from {}", remotePath, remoteRepository.getRepository().getName() );
928 wagon.get( addParameters( remotePath, remoteRepository.getRepository() ), destFile );
931 // You wouldn't get here on failure, a WagonException would have been thrown.
932 log.debug( "Downloaded successfully." );
936 log.debug( "Retrieving {} from {} if updated", remotePath, remoteRepository.getRepository().getName() );
937 success = wagon.getIfNewer( addParameters( remotePath, remoteRepository.getRepository() ), destFile,
938 origFile.lastModified() );
941 throw new NotModifiedException(
942 "Not downloaded, as local file is newer than remote side: " + origFile.getAbsolutePath() );
945 if ( destFile.exists() )
947 log.debug( "Downloaded successfully." );
951 catch ( ResourceDoesNotExistException e )
953 throw new NotFoundException(
954 "Resource [" + remoteRepository.getURL() + "/" + remotePath + "] does not exist: " + e.getMessage(),
957 catch ( WagonException e )
959 // TODO: shouldn't have to drill into the cause, but TransferFailedException is often not descriptive enough
962 "Download failure on resource [" + remoteRepository.getURL() + "/" + remotePath + "]:" + e.getMessage();
963 if ( e.getCause() != null )
965 msg += " (cause: " + e.getCause() + ")";
967 throw new ProxyException( msg, e );
972 * Apply the policies.
974 * @param policies the map of policies to execute. (Map of String policy keys, to {@link DownloadPolicy} objects)
975 * @param settings the map of settings for the policies to execute. (Map of String policy keys, to String policy
977 * @param request the request properties (utilized by the {@link DownloadPolicy#applyPolicy(String, Properties, File)}
979 * @param localFile the local file (utilized by the {@link DownloadPolicy#applyPolicy(String, Properties, File)})
980 * @throws PolicyViolationException
982 private void validatePolicies( Map<String, ? extends DownloadPolicy> policies, Map<String, String> settings,
983 Properties request, File localFile )
984 throws PolicyViolationException
986 for ( Entry<String, ? extends DownloadPolicy> entry : policies.entrySet() )
988 // olamy with spring rolehint is now downloadPolicy#hint
989 // so substring after last # to get the hint as with plexus
990 String key = StringUtils.substringAfterLast( entry.getKey(), "#" );
991 DownloadPolicy policy = entry.getValue();
992 String defaultSetting = policy.getDefaultOption();
994 String setting = StringUtils.defaultString( settings.get( key ), defaultSetting );
996 log.debug( "Applying [{}] policy with [{}]", key, setting );
999 policy.applyPolicy( setting, request, localFile );
1001 catch ( PolicyConfigurationException e )
1003 log.error( e.getMessage(), e );
1008 private void validatePolicies( Map<String, DownloadErrorPolicy> policies, Map<String, String> settings,
1009 Properties request, ArtifactReference artifact, RemoteRepositoryContent content,
1010 File localFile, Exception exception, Map<String, Exception> previousExceptions )
1011 throws ProxyDownloadException
1013 boolean process = true;
1014 for ( Entry<String, ? extends DownloadErrorPolicy> entry : policies.entrySet() )
1017 // olamy with spring rolehint is now downloadPolicy#hint
1018 // so substring after last # to get the hint as with plexus
1019 String key = StringUtils.substringAfterLast( entry.getKey(), "#" );
1020 DownloadErrorPolicy policy = entry.getValue();
1021 String defaultSetting = policy.getDefaultOption();
1022 String setting = StringUtils.defaultString( settings.get( key ), defaultSetting );
1024 log.debug( "Applying [{}] policy with [{}]", key, setting );
1027 // all policies must approve the exception, any can cancel
1028 process = policy.applyPolicy( setting, request, localFile, exception, previousExceptions );
1034 catch ( PolicyConfigurationException e )
1036 log.error( e.getMessage(), e );
1042 // if the exception was queued, don't throw it
1043 if ( !previousExceptions.containsKey( content.getId() ) )
1045 throw new ProxyDownloadException(
1046 "An error occurred in downloading from the remote repository, and the policy is to fail immediately",
1047 content.getId(), exception );
1052 // if the exception was queued, but cancelled, remove it
1053 previousExceptions.remove( content.getId() );
1057 "Transfer error from repository {} for artifact {} , continuing to next repository. Error message: {}",
1058 content.getRepository().getId(), Keys.toKey( artifact ), exception.getMessage() );
1059 log.debug( "Full stack trace", exception );
1063 * Creates a working directory
1066 * @return file location of working directory
1068 private File createWorkingDirectory( ManagedRepositoryContent repository )
1072 return Files.createTempDirectory( "temp" ).toFile();
1074 catch ( IOException e )
1076 throw new RuntimeException( e.getMessage(), e );
1082 * Used to move the temporary file to its real destination. This is patterned from the way WagonManager handles its
1085 * @param temp The completed download file
1086 * @param target The final location of the downloaded file
1087 * @throws ProxyException when the temp file cannot replace the target file
1089 private void moveTempToTarget( File temp, File target )
1090 throws ProxyException
1093 // TODO file lock library
1097 lock = fileLockManager.writeFileLock( target );
1098 if ( lock.getFile().exists() && !lock.getFile().delete() )
1100 throw new ProxyException( "Unable to overwrite existing target file: " + target.getAbsolutePath() );
1103 lock.getFile().getParentFile().mkdirs();
1105 if ( !temp.renameTo( lock.getFile() ) )
1107 log.warn( "Unable to rename tmp file to its final name... resorting to copy command." );
1111 FileUtils.copyFile( temp, lock.getFile() );
1113 catch ( IOException e )
1115 if ( lock.getFile().exists() )
1117 log.debug( "Tried to copy file {} to {} but file with this name already exists.",
1118 temp.getName(), lock.getFile().getAbsolutePath() );
1122 throw new ProxyException(
1123 "Cannot copy tmp file " + temp.getAbsolutePath() + " to its final location", e );
1128 FileUtils.deleteQuietly( temp );
1132 catch ( FileLockException e )
1134 throw new ProxyException( e.getMessage(), e );
1136 catch ( FileLockTimeoutException e )
1138 throw new ProxyException( e.getMessage(), e );
1143 * Using wagon, connect to the remote repository.
1145 * @param connector the connector configuration to utilize (for obtaining network proxy configuration from)
1146 * @param wagon the wagon instance to establish the connection on.
1147 * @param remoteRepository the remote repository to connect to.
1148 * @return true if the connection was successful. false if not connected.
1150 private boolean connectToRepository( ProxyConnector connector, Wagon wagon,
1151 RemoteRepositoryContent remoteRepository )
1153 boolean connected = false;
1155 final ProxyInfo networkProxy =
1156 connector.getProxyId() == null ? null : this.networkProxyMap.get( connector.getProxyId() );
1158 if ( log.isDebugEnabled() )
1160 if ( networkProxy != null )
1162 // TODO: move to proxyInfo.toString()
1163 String msg = "Using network proxy " + networkProxy.getHost() + ":" + networkProxy.getPort()
1164 + " to connect to remote repository " + remoteRepository.getURL();
1165 if ( networkProxy.getNonProxyHosts() != null )
1167 msg += "; excluding hosts: " + networkProxy.getNonProxyHosts();
1169 if ( StringUtils.isNotBlank( networkProxy.getUserName() ) )
1171 msg += "; as user: " + networkProxy.getUserName();
1177 AuthenticationInfo authInfo = null;
1178 String username = remoteRepository.getRepository().getUserName();
1179 String password = remoteRepository.getRepository().getPassword();
1181 if ( StringUtils.isNotBlank( username ) && StringUtils.isNotBlank( password ) )
1183 log.debug( "Using username {} to connect to remote repository {}", username, remoteRepository.getURL() );
1184 authInfo = new AuthenticationInfo();
1185 authInfo.setUserName( username );
1186 authInfo.setPassword( password );
1189 // Convert seconds to milliseconds
1190 int timeoutInMilliseconds = remoteRepository.getRepository().getTimeout() * 1000;
1192 // Set timeout read and connect
1193 // FIXME olamy having 2 config values
1194 wagon.setReadTimeout( timeoutInMilliseconds );
1195 wagon.setTimeout( timeoutInMilliseconds );
1199 Repository wagonRepository =
1200 new Repository( remoteRepository.getId(), remoteRepository.getURL().toString() );
1201 wagon.connect( wagonRepository, authInfo, networkProxy );
1204 catch ( ConnectionException | AuthenticationException e )
1206 log.warn( "Could not connect to {}: {}", remoteRepository.getRepository().getName(), e.getMessage() );
1214 * Tests whitelist and blacklist patterns against path.
1216 * @param path the path to test.
1217 * @param patterns the list of patterns to check.
1218 * @return true if the path matches at least 1 pattern in the provided patterns list.
1220 private boolean matchesPattern( String path, List<String> patterns )
1222 if ( CollectionUtils.isEmpty( patterns ) )
1227 if ( !path.startsWith( "/" ) )
1232 for ( String pattern : patterns )
1234 if ( !pattern.startsWith( "/" ) )
1236 pattern = "/" + pattern;
1239 if ( SelectorUtils.matchPath( pattern, path, false ) )
1249 * TODO: Ensure that list is correctly ordered based on configuration. See MRM-477
1252 public List<ProxyConnector> getProxyConnectors( ManagedRepositoryContent repository )
1254 synchronized ( this.proxyConnectorMap )
1256 List<ProxyConnector> ret = this.proxyConnectorMap.get( repository.getId() );
1259 return Collections.emptyList();
1262 Collections.sort( ret, ProxyConnectorOrderComparator.getInstance() );
1268 public void afterConfigurationChange( Registry registry, String propertyName, Object propertyValue )
1270 if ( ConfigurationNames.isNetworkProxy( propertyName ) || ConfigurationNames.isManagedRepositories(
1271 propertyName ) || ConfigurationNames.isRemoteRepositories( propertyName )
1272 || ConfigurationNames.isProxyConnector( propertyName ) )
1274 initConnectorsAndNetworkProxies();
1278 protected String addParameters( String path, RemoteRepository remoteRepository )
1280 if ( remoteRepository.getExtraParameters().isEmpty() )
1285 boolean question = false;
1287 StringBuilder res = new StringBuilder( path == null ? "" : path );
1289 for ( Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() )
1293 res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() );
1297 return res.toString();
1302 public void beforeConfigurationChange( Registry registry, String propertyName, Object propertyValue )
1307 public ArchivaConfiguration getArchivaConfiguration()
1309 return archivaConfiguration;
1312 public void setArchivaConfiguration( ArchivaConfiguration archivaConfiguration )
1314 this.archivaConfiguration = archivaConfiguration;
1317 public RepositoryContentFactory getRepositoryFactory()
1319 return repositoryFactory;
1322 public void setRepositoryFactory( RepositoryContentFactory repositoryFactory )
1324 this.repositoryFactory = repositoryFactory;
1327 public MetadataTools getMetadataTools()
1329 return metadataTools;
1332 public void setMetadataTools( MetadataTools metadataTools )
1334 this.metadataTools = metadataTools;
1337 public UrlFailureCache getUrlFailureCache()
1339 return urlFailureCache;
1342 public void setUrlFailureCache( UrlFailureCache urlFailureCache )
1344 this.urlFailureCache = urlFailureCache;
1347 public WagonFactory getWagonFactory()
1349 return wagonFactory;
1352 public void setWagonFactory( WagonFactory wagonFactory )
1354 this.wagonFactory = wagonFactory;
1357 public Map<String, PreDownloadPolicy> getPreDownloadPolicies()
1359 return preDownloadPolicies;
1362 public void setPreDownloadPolicies( Map<String, PreDownloadPolicy> preDownloadPolicies )
1364 this.preDownloadPolicies = preDownloadPolicies;
1367 public Map<String, PostDownloadPolicy> getPostDownloadPolicies()
1369 return postDownloadPolicies;
1372 public void setPostDownloadPolicies( Map<String, PostDownloadPolicy> postDownloadPolicies )
1374 this.postDownloadPolicies = postDownloadPolicies;
1377 public Map<String, DownloadErrorPolicy> getDownloadErrorPolicies()
1379 return downloadErrorPolicies;
1382 public void setDownloadErrorPolicies( Map<String, DownloadErrorPolicy> downloadErrorPolicies )
1384 this.downloadErrorPolicies = downloadErrorPolicies;