1 package org.apache.archiva.proxy;
4 * Licensed to the Apache Software Foundation (ASF) under one
5 * or more contributor license agreements. See the NOTICE file
6 * distributed with this work for additional information
7 * regarding copyright ownership. The ASF licenses this file
8 * to you under the Apache License, Version 2.0 (the
9 * "License"); you may not use this file except in compliance
10 * with the License. You may obtain a copy of the License at
12 * http://www.apache.org/licenses/LICENSE-2.0
14 * Unless required by applicable law or agreed to in writing,
15 * software distributed under the License is distributed on an
16 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17 * KIND, either express or implied. See the License for the
18 * specific language governing permissions and limitations
22 import org.apache.archiva.admin.model.RepositoryAdminException;
23 import org.apache.archiva.admin.model.beans.NetworkProxy;
24 import org.apache.archiva.admin.model.beans.ProxyConnectorRuleType;
25 import org.apache.archiva.admin.model.beans.RemoteRepository;
26 import org.apache.archiva.admin.model.networkproxy.NetworkProxyAdmin;
27 import org.apache.archiva.common.filelock.FileLockException;
28 import org.apache.archiva.common.filelock.FileLockManager;
29 import org.apache.archiva.common.filelock.FileLockTimeoutException;
30 import org.apache.archiva.common.filelock.Lock;
31 import org.apache.archiva.configuration.ArchivaConfiguration;
32 import org.apache.archiva.configuration.Configuration;
33 import org.apache.archiva.configuration.ConfigurationNames;
34 import org.apache.archiva.configuration.NetworkProxyConfiguration;
35 import org.apache.archiva.configuration.ProxyConnectorConfiguration;
36 import org.apache.archiva.configuration.ProxyConnectorRuleConfiguration;
37 import org.apache.archiva.model.ArtifactReference;
38 import org.apache.archiva.model.Keys;
39 import org.apache.archiva.model.RepositoryURL;
40 import org.apache.archiva.policies.DownloadErrorPolicy;
41 import org.apache.archiva.policies.DownloadPolicy;
42 import org.apache.archiva.policies.PolicyConfigurationException;
43 import org.apache.archiva.policies.PolicyViolationException;
44 import org.apache.archiva.policies.PostDownloadPolicy;
45 import org.apache.archiva.policies.PreDownloadPolicy;
46 import org.apache.archiva.policies.ProxyDownloadException;
47 import org.apache.archiva.policies.urlcache.UrlFailureCache;
48 import org.apache.archiva.proxy.common.WagonFactory;
49 import org.apache.archiva.proxy.common.WagonFactoryException;
50 import org.apache.archiva.proxy.common.WagonFactoryRequest;
51 import org.apache.archiva.proxy.model.ProxyFetchResult;
52 import org.apache.archiva.proxy.model.ProxyConnector;
53 import org.apache.archiva.proxy.model.RepositoryProxyConnectors;
54 import org.apache.archiva.redback.components.registry.Registry;
55 import org.apache.archiva.redback.components.registry.RegistryListener;
56 import org.apache.archiva.redback.components.taskqueue.TaskQueueException;
57 import org.apache.archiva.repository.ManagedRepositoryContent;
58 import org.apache.archiva.repository.RemoteRepositoryContent;
59 import org.apache.archiva.repository.RepositoryContentFactory;
60 import org.apache.archiva.repository.RepositoryException;
61 import org.apache.archiva.repository.RepositoryNotFoundException;
62 import org.apache.archiva.repository.metadata.MetadataTools;
63 import org.apache.archiva.repository.metadata.RepositoryMetadataException;
64 import org.apache.archiva.scheduler.ArchivaTaskScheduler;
65 import org.apache.archiva.scheduler.repository.model.RepositoryTask;
66 import org.apache.commons.collections.CollectionUtils;
67 import org.apache.commons.io.FileUtils;
68 import org.apache.commons.io.FilenameUtils;
69 import org.apache.commons.lang.StringUtils;
70 import org.apache.commons.lang.SystemUtils;
71 import org.apache.maven.wagon.ConnectionException;
72 import org.apache.maven.wagon.ResourceDoesNotExistException;
73 import org.apache.maven.wagon.Wagon;
74 import org.apache.maven.wagon.WagonException;
75 import org.apache.maven.wagon.authentication.AuthenticationException;
76 import org.apache.maven.wagon.authentication.AuthenticationInfo;
77 import org.apache.maven.wagon.proxy.ProxyInfo;
78 import org.apache.maven.wagon.repository.Repository;
79 import org.apache.tools.ant.types.selectors.SelectorUtils;
80 import org.slf4j.Logger;
81 import org.slf4j.LoggerFactory;
82 import org.slf4j.MarkerFactory;
83 import org.springframework.stereotype.Service;
85 import javax.annotation.PostConstruct;
86 import javax.inject.Inject;
87 import javax.inject.Named;
89 import java.io.IOException;
90 import java.nio.file.Files;
91 import java.util.ArrayList;
92 import java.util.Collections;
93 import java.util.HashMap;
94 import java.util.LinkedHashMap;
95 import java.util.List;
97 import java.util.Map.Entry;
98 import java.util.Properties;
99 import java.util.concurrent.ConcurrentHashMap;
100 import java.util.concurrent.ConcurrentMap;
103 * DefaultRepositoryProxyConnectors
104 * TODO exception handling needs work - "not modified" is not really an exceptional case, and it has more layers than
105 * your average brown onion
107 @Service("repositoryProxyConnectors#default")
108 public class DefaultRepositoryProxyConnectors
109 implements RepositoryProxyConnectors, RegistryListener
111 private Logger log = LoggerFactory.getLogger( DefaultRepositoryProxyConnectors.class );
114 @Named(value = "archivaConfiguration#default")
115 private ArchivaConfiguration archivaConfiguration;
118 @Named(value = "repositoryContentFactory#default")
119 private RepositoryContentFactory repositoryFactory;
122 @Named(value = "metadataTools#default")
123 private MetadataTools metadataTools;
126 private Map<String, PreDownloadPolicy> preDownloadPolicies;
129 private Map<String, PostDownloadPolicy> postDownloadPolicies;
132 private Map<String, DownloadErrorPolicy> downloadErrorPolicies;
135 private UrlFailureCache urlFailureCache;
137 private ConcurrentMap<String, List<ProxyConnector>> proxyConnectorMap = new ConcurrentHashMap<>();
139 private ConcurrentMap<String, ProxyInfo> networkProxyMap = new ConcurrentHashMap<>();
142 private WagonFactory wagonFactory;
145 @Named(value = "archivaTaskScheduler#repository")
146 private ArchivaTaskScheduler scheduler;
149 private NetworkProxyAdmin networkProxyAdmin;
152 @Named(value = "fileLockManager#default")
153 private FileLockManager fileLockManager;
156 public void initialize()
158 initConnectorsAndNetworkProxies();
159 archivaConfiguration.addChangeListener( this );
163 @SuppressWarnings("unchecked")
164 private void initConnectorsAndNetworkProxies()
167 ProxyConnectorOrderComparator proxyOrderSorter = new ProxyConnectorOrderComparator();
168 this.proxyConnectorMap.clear();
170 Configuration configuration = archivaConfiguration.getConfiguration();
172 List<ProxyConnectorRuleConfiguration> allProxyConnectorRuleConfigurations =
173 configuration.getProxyConnectorRuleConfigurations();
175 List<ProxyConnectorConfiguration> proxyConfigs = configuration.getProxyConnectors();
176 for ( ProxyConnectorConfiguration proxyConfig : proxyConfigs )
178 String key = proxyConfig.getSourceRepoId();
182 // Create connector object.
183 ProxyConnector connector = new ProxyConnector();
185 connector.setSourceRepository(
186 repositoryFactory.getManagedRepositoryContent( proxyConfig.getSourceRepoId() ) );
187 connector.setTargetRepository(
188 repositoryFactory.getRemoteRepositoryContent( proxyConfig.getTargetRepoId() ) );
190 connector.setProxyId( proxyConfig.getProxyId() );
191 connector.setPolicies( proxyConfig.getPolicies() );
192 connector.setOrder( proxyConfig.getOrder() );
193 connector.setDisabled( proxyConfig.isDisabled() );
195 // Copy any blacklist patterns.
196 List<String> blacklist = new ArrayList<>( 0 );
197 if ( CollectionUtils.isNotEmpty( proxyConfig.getBlackListPatterns() ) )
199 blacklist.addAll( proxyConfig.getBlackListPatterns() );
201 connector.setBlacklist( blacklist );
203 // Copy any whitelist patterns.
204 List<String> whitelist = new ArrayList<>( 0 );
205 if ( CollectionUtils.isNotEmpty( proxyConfig.getWhiteListPatterns() ) )
207 whitelist.addAll( proxyConfig.getWhiteListPatterns() );
209 connector.setWhitelist( whitelist );
211 List<ProxyConnectorRuleConfiguration> proxyConnectorRuleConfigurations =
212 findProxyConnectorRules( connector.getSourceRepository().getId(),
213 connector.getTargetRepository().getId(),
214 allProxyConnectorRuleConfigurations );
216 if ( !proxyConnectorRuleConfigurations.isEmpty() )
218 for ( ProxyConnectorRuleConfiguration proxyConnectorRuleConfiguration : proxyConnectorRuleConfigurations )
220 if ( StringUtils.equals( proxyConnectorRuleConfiguration.getRuleType(),
221 ProxyConnectorRuleType.BLACK_LIST.getRuleType() ) )
223 connector.getBlacklist().add( proxyConnectorRuleConfiguration.getPattern() );
226 if ( StringUtils.equals( proxyConnectorRuleConfiguration.getRuleType(),
227 ProxyConnectorRuleType.WHITE_LIST.getRuleType() ) )
229 connector.getWhitelist().add( proxyConnectorRuleConfiguration.getPattern() );
234 // Get other connectors
235 List<ProxyConnector> connectors = this.proxyConnectorMap.get( key );
236 if ( connectors == null )
238 // Create if we are the first.
239 connectors = new ArrayList<>( 1 );
242 // Add the connector.
243 connectors.add( connector );
245 // Ensure the list is sorted.
246 Collections.sort( connectors, proxyOrderSorter );
248 // Set the key to the list of connectors.
249 this.proxyConnectorMap.put( key, connectors );
251 catch ( RepositoryNotFoundException e )
253 log.warn( "Unable to use proxy connector: {}", e.getMessage(), e );
255 catch ( RepositoryException e )
257 log.warn( "Unable to use proxy connector: {}", e.getMessage(), e );
263 this.networkProxyMap.clear();
265 List<NetworkProxyConfiguration> networkProxies = archivaConfiguration.getConfiguration().getNetworkProxies();
266 for ( NetworkProxyConfiguration networkProxyConfig : networkProxies )
268 String key = networkProxyConfig.getId();
270 ProxyInfo proxy = new ProxyInfo();
272 proxy.setType( networkProxyConfig.getProtocol() );
273 proxy.setHost( networkProxyConfig.getHost() );
274 proxy.setPort( networkProxyConfig.getPort() );
275 proxy.setUserName( networkProxyConfig.getUsername() );
276 proxy.setPassword( networkProxyConfig.getPassword() );
278 this.networkProxyMap.put( key, proxy );
283 private List<ProxyConnectorRuleConfiguration> findProxyConnectorRules( String sourceRepository,
284 String targetRepository,
285 List<ProxyConnectorRuleConfiguration> all )
287 List<ProxyConnectorRuleConfiguration> proxyConnectorRuleConfigurations = new ArrayList<>();
289 for ( ProxyConnectorRuleConfiguration proxyConnectorRuleConfiguration : all )
291 for ( ProxyConnectorConfiguration proxyConnector : proxyConnectorRuleConfiguration.getProxyConnectors() )
293 if ( StringUtils.equals( sourceRepository, proxyConnector.getSourceRepoId() ) && StringUtils.equals(
294 targetRepository, proxyConnector.getTargetRepoId() ) )
296 proxyConnectorRuleConfigurations.add( proxyConnectorRuleConfiguration );
301 return proxyConnectorRuleConfigurations;
305 public File fetchFromProxies( ManagedRepositoryContent repository, ArtifactReference artifact )
306 throws ProxyDownloadException
308 File localFile = toLocalFile( repository, artifact );
310 Properties requestProperties = new Properties();
311 requestProperties.setProperty( "filetype", "artifact" );
312 requestProperties.setProperty( "version", artifact.getVersion() );
313 requestProperties.setProperty( "managedRepositoryId", repository.getId() );
315 List<ProxyConnector> connectors = getProxyConnectors( repository );
316 Map<String, Exception> previousExceptions = new LinkedHashMap<>();
317 for ( ProxyConnector connector : connectors )
319 if ( connector.isDisabled() )
324 RemoteRepositoryContent targetRepository = connector.getTargetRepository();
325 requestProperties.setProperty( "remoteRepositoryId", targetRepository.getId() );
327 String targetPath = targetRepository.toPath( artifact );
329 if ( SystemUtils.IS_OS_WINDOWS )
331 // toPath use system PATH_SEPARATOR so on windows url are \ which doesn't work very well :-)
332 targetPath = FilenameUtils.separatorsToUnix( targetPath );
337 File downloadedFile =
338 transferFile( connector, targetRepository, targetPath, repository, localFile, requestProperties,
341 if ( fileExists( downloadedFile ) )
343 log.debug( "Successfully transferred: {}", downloadedFile.getAbsolutePath() );
344 return downloadedFile;
347 catch ( NotFoundException e )
349 log.debug( "Artifact {} not found on repository \"{}\".", Keys.toKey( artifact ),
350 targetRepository.getRepository().getId() );
352 catch ( NotModifiedException e )
354 log.debug( "Artifact {} not updated on repository \"{}\".", Keys.toKey( artifact ),
355 targetRepository.getRepository().getId() );
357 catch ( ProxyException | RepositoryAdminException e )
359 validatePolicies( this.downloadErrorPolicies, connector.getPolicies(), requestProperties, artifact,
360 targetRepository, localFile, e, previousExceptions );
364 if ( !previousExceptions.isEmpty() )
366 throw new ProxyDownloadException( "Failures occurred downloading from some remote repositories",
367 previousExceptions );
370 log.debug( "Exhausted all target repositories, artifact {} not found.", Keys.toKey( artifact ) );
376 public File fetchFromProxies( ManagedRepositoryContent repository, String path )
378 File localFile = new File( repository.getRepoRoot(), path );
380 // no update policies for these paths
381 if ( localFile.exists() )
386 Properties requestProperties = new Properties();
387 requestProperties.setProperty( "filetype", "resource" );
388 requestProperties.setProperty( "managedRepositoryId", repository.getId() );
390 List<ProxyConnector> connectors = getProxyConnectors( repository );
391 for ( ProxyConnector connector : connectors )
393 if ( connector.isDisabled() )
398 RemoteRepositoryContent targetRepository = connector.getTargetRepository();
399 requestProperties.setProperty( "remoteRepositoryId", targetRepository.getId() );
401 String targetPath = path;
405 File downloadedFile =
406 transferFile( connector, targetRepository, targetPath, repository, localFile, requestProperties,
409 if ( fileExists( downloadedFile ) )
411 log.debug( "Successfully transferred: {}", downloadedFile.getAbsolutePath() );
412 return downloadedFile;
415 catch ( NotFoundException e )
417 log.debug( "Resource {} not found on repository \"{}\".", path,
418 targetRepository.getRepository().getId() );
420 catch ( NotModifiedException e )
422 log.debug( "Resource {} not updated on repository \"{}\".", path,
423 targetRepository.getRepository().getId() );
425 catch ( ProxyException e )
428 "Transfer error from repository {} for resource {}, continuing to next repository. Error message: {}",
429 targetRepository.getRepository().getId(), path, e.getMessage() );
430 log.debug( MarkerFactory.getDetachedMarker( "transfer.error" ),
431 "Transfer error from repository \"" + targetRepository.getRepository().getId()
432 + "\" for resource " + path + ", continuing to next repository. Error message: {}",
436 catch ( RepositoryAdminException e )
438 log.debug( MarkerFactory.getDetachedMarker( "transfer.error" ),
439 "Transfer error from repository {} for resource {}, continuing to next repository. Error message: {}",
440 targetRepository.getRepository().getId(), path, e.getMessage(), e );
441 log.debug( MarkerFactory.getDetachedMarker( "transfer.error" ), "Full stack trace", e );
445 log.debug( "Exhausted all target repositories, resource {} not found.", path );
451 public ProxyFetchResult fetchMetadataFromProxies( ManagedRepositoryContent repository, String logicalPath )
453 File localFile = new File( repository.getRepoRoot(), logicalPath );
455 Properties requestProperties = new Properties();
456 requestProperties.setProperty( "filetype", "metadata" );
457 boolean metadataNeedsUpdating = false;
458 long originalTimestamp = getLastModified( localFile );
460 List<ProxyConnector> connectors = new ArrayList<>( getProxyConnectors( repository ) );
461 for ( ProxyConnector connector : connectors )
463 if ( connector.isDisabled() )
468 RemoteRepositoryContent targetRepository = connector.getTargetRepository();
470 File localRepoFile = toLocalRepoFile( repository, targetRepository, logicalPath );
471 long originalMetadataTimestamp = getLastModified( localRepoFile );
475 transferFile( connector, targetRepository, logicalPath, repository, localRepoFile, requestProperties,
478 if ( hasBeenUpdated( localRepoFile, originalMetadataTimestamp ) )
480 metadataNeedsUpdating = true;
483 catch ( NotFoundException e )
486 log.debug( "Metadata {} not found on remote repository '{}'.", logicalPath,
487 targetRepository.getRepository().getId(), e );
490 catch ( NotModifiedException e )
493 log.debug( "Metadata {} not updated on remote repository '{}'.", logicalPath,
494 targetRepository.getRepository().getId(), e );
497 catch ( ProxyException | RepositoryAdminException e )
500 "Transfer error from repository {} for versioned Metadata {}, continuing to next repository. Error message: {}",
501 targetRepository.getRepository().getId(), logicalPath, e.getMessage() );
502 log.debug( "Full stack trace", e );
506 if ( hasBeenUpdated( localFile, originalTimestamp ) )
508 metadataNeedsUpdating = true;
511 if ( metadataNeedsUpdating || !localFile.exists() )
515 metadataTools.updateMetadata( repository, logicalPath );
517 catch ( RepositoryMetadataException e )
519 log.warn( "Unable to update metadata {}:{}", localFile.getAbsolutePath(), e.getMessage(), e );
524 if ( fileExists( localFile ) )
526 return new ProxyFetchResult( localFile, metadataNeedsUpdating );
529 return new ProxyFetchResult( null, false );
534 * @param remoteRepository
541 * @param workingDirectory
543 * @throws ProxyException
544 * @throws NotModifiedException
545 * @throws org.apache.archiva.admin.model.RepositoryAdminException
547 protected void transferResources( ProxyConnector connector, RemoteRepositoryContent remoteRepository, File tmpMd5,
548 File tmpSha1, File tmpResource, String url, String remotePath, File resource,
549 File workingDirectory, ManagedRepositoryContent repository )
550 throws ProxyException, NotModifiedException, RepositoryAdminException
555 RepositoryURL repoUrl = remoteRepository.getURL();
556 String protocol = repoUrl.getProtocol();
557 NetworkProxy networkProxy = null;
558 if ( StringUtils.isNotBlank( connector.getProxyId() ) )
560 networkProxy = networkProxyAdmin.getNetworkProxy( connector.getProxyId() );
562 WagonFactoryRequest wagonFactoryRequest = new WagonFactoryRequest( "wagon#" + protocol,
563 remoteRepository.getRepository().getExtraHeaders() ).networkProxy(
565 wagon = wagonFactory.getWagon( wagonFactoryRequest );
568 throw new ProxyException( "Unsupported target repository protocol: " + protocol );
573 throw new ProxyException( "Unsupported target repository protocol: " + protocol );
576 boolean connected = connectToRepository( connector, wagon, remoteRepository );
579 transferArtifact( wagon, remoteRepository, remotePath, repository, resource, workingDirectory,
582 // TODO: these should be used to validate the download based on the policies, not always downloaded
584 // save on connections since md5 is rarely used
585 transferChecksum( wagon, remoteRepository, remotePath, repository, resource, workingDirectory, ".sha1",
587 transferChecksum( wagon, remoteRepository, remotePath, repository, resource, workingDirectory, ".md5",
591 catch ( NotFoundException e )
593 urlFailureCache.cacheFailure( url );
596 catch ( NotModifiedException e )
598 // Do not cache url here.
601 catch ( ProxyException e )
603 urlFailureCache.cacheFailure( url );
606 catch ( WagonFactoryException e )
608 throw new ProxyException( e.getMessage(), e );
618 catch ( ConnectionException e )
620 log.warn( "Unable to disconnect wagon.", e );
626 private void transferArtifact( Wagon wagon, RemoteRepositoryContent remoteRepository, String remotePath,
627 ManagedRepositoryContent repository, File resource, File tmpDirectory,
629 throws ProxyException
631 transferSimpleFile( wagon, remoteRepository, remotePath, repository, resource, destFile );
634 private long getLastModified( File file )
636 if ( !file.exists() || !file.isFile() )
641 return file.lastModified();
644 private boolean hasBeenUpdated( File file, long originalLastModified )
646 if ( !file.exists() || !file.isFile() )
651 long currentLastModified = getLastModified( file );
652 return ( currentLastModified > originalLastModified );
655 private File toLocalRepoFile( ManagedRepositoryContent repository, RemoteRepositoryContent targetRepository,
658 String repoPath = metadataTools.getRepositorySpecificName( targetRepository, targetPath );
659 return new File( repository.getRepoRoot(), repoPath );
663 * Test if the provided ManagedRepositoryContent has any proxies configured for it.
666 public boolean hasProxies( ManagedRepositoryContent repository )
668 synchronized ( this.proxyConnectorMap )
670 return this.proxyConnectorMap.containsKey( repository.getId() );
674 private File toLocalFile( ManagedRepositoryContent repository, ArtifactReference artifact )
676 return repository.toFile( artifact );
680 * Simple method to test if the file exists on the local disk.
682 * @param file the file to test. (may be null)
683 * @return true if file exists. false if the file param is null, doesn't exist, or is not of type File.
685 private boolean fileExists( File file )
692 if ( !file.exists() )
697 return file.isFile();
701 * Perform the transfer of the file.
703 * @param connector the connector configuration to use.
704 * @param remoteRepository the remote repository get the resource from.
705 * @param remotePath the path in the remote repository to the resource to get.
706 * @param repository the managed repository that will hold the file
707 * @param resource the local file to place the downloaded resource into
708 * @param requestProperties the request properties to utilize for policy handling.
709 * @param executeConsumers whether to execute the consumers after proxying
710 * @return the local file that was downloaded, or null if not downloaded.
711 * @throws NotFoundException if the file was not found on the remote repository.
712 * @throws NotModifiedException if the localFile was present, and the resource was present on remote repository, but
713 * the remote resource is not newer than the local File.
714 * @throws ProxyException if transfer was unsuccessful.
716 private File transferFile( ProxyConnector connector, RemoteRepositoryContent remoteRepository, String remotePath,
717 ManagedRepositoryContent repository, File resource, Properties requestProperties,
718 boolean executeConsumers )
719 throws ProxyException, NotModifiedException, RepositoryAdminException
721 String url = remoteRepository.getURL().getUrl();
722 if ( !url.endsWith( "/" ) )
726 url = url + remotePath;
727 requestProperties.setProperty( "url", url );
729 // Is a whitelist defined?
730 if ( CollectionUtils.isNotEmpty( connector.getWhitelist() ) )
732 // Path must belong to whitelist.
733 if ( !matchesPattern( remotePath, connector.getWhitelist() ) )
735 log.debug( "Path [{}] is not part of defined whitelist (skipping transfer from repository [{}]).",
736 remotePath, remoteRepository.getRepository().getName() );
741 // Is target path part of blacklist?
742 if ( matchesPattern( remotePath, connector.getBlacklist() ) )
744 log.debug( "Path [{}] is part of blacklist (skipping transfer from repository [{}]).", remotePath,
745 remoteRepository.getRepository().getName() );
749 // Handle pre-download policy
752 validatePolicies( this.preDownloadPolicies, connector.getPolicies(), requestProperties, resource );
754 catch ( PolicyViolationException e )
756 String emsg = "Transfer not attempted on " + url + " : " + e.getMessage();
757 if ( fileExists( resource ) )
759 log.debug( "{} : using already present local file.", emsg );
767 File workingDirectory = createWorkingDirectory( repository );
768 File tmpResource = new File( workingDirectory, resource.getName() );
769 File tmpMd5 = new File( workingDirectory, resource.getName() + ".md5" );
770 File tmpSha1 = new File( workingDirectory, resource.getName() + ".sha1" );
775 transferResources( connector, remoteRepository, tmpMd5, tmpSha1, tmpResource, url, remotePath, resource,
776 workingDirectory, repository );
778 // Handle post-download policies.
781 validatePolicies( this.postDownloadPolicies, connector.getPolicies(), requestProperties, tmpResource );
783 catch ( PolicyViolationException e )
785 log.warn( "Transfer invalidated from {} : {}", url, e.getMessage() );
786 executeConsumers = false;
787 if ( !fileExists( tmpResource ) )
793 if ( resource != null )
795 synchronized ( resource.getAbsolutePath().intern() )
797 File directory = resource.getParentFile();
798 moveFileIfExists( tmpMd5, directory );
799 moveFileIfExists( tmpSha1, directory );
800 moveFileIfExists( tmpResource, directory );
806 FileUtils.deleteQuietly( workingDirectory );
809 if ( executeConsumers )
811 // Just-in-time update of the index and database by executing the consumers for this artifact
812 //consumers.executeConsumers( connector.getSourceRepository().getRepository(), resource );
813 queueRepositoryTask( connector.getSourceRepository().getRepository().getId(), resource );
819 private void queueRepositoryTask( String repositoryId, File localFile )
821 RepositoryTask task = new RepositoryTask();
822 task.setRepositoryId( repositoryId );
823 task.setResourceFile( localFile );
824 task.setUpdateRelatedArtifacts( true );
825 task.setScanAll( true );
829 scheduler.queueTask( task );
831 catch ( TaskQueueException e )
833 log.error( "Unable to queue repository task to execute consumers on resource file ['" + localFile.getName()
839 * Moves the file into repository location if it exists
841 * @param fileToMove this could be either the main artifact, sha1 or md5 checksum file.
842 * @param directory directory to write files to
844 private void moveFileIfExists( File fileToMove, File directory )
845 throws ProxyException
847 if ( fileToMove != null && fileToMove.exists() )
849 File newLocation = new File( directory, fileToMove.getName() );
850 moveTempToTarget( fileToMove, newLocation );
856 * Quietly transfer the checksum file from the remote repository to the local file.
859 * @param wagon the wagon instance (should already be connected) to use.
860 * @param remoteRepository the remote repository to transfer from.
861 * @param remotePath the remote path to the resource to get.
862 * @param repository the managed repository that will hold the file
863 * @param resource the local file that should contain the downloaded contents
864 * @param tmpDirectory the temporary directory to download to
865 * @param ext the type of checksum to transfer (example: ".md5" or ".sha1")
866 * @throws ProxyException if copying the downloaded file into place did not succeed.
868 private void transferChecksum( Wagon wagon, RemoteRepositoryContent remoteRepository, String remotePath,
869 ManagedRepositoryContent repository, File resource, File tmpDirectory, String ext,
871 throws ProxyException
873 String url = remoteRepository.getURL().getUrl() + remotePath + ext;
875 // Transfer checksum does not use the policy.
876 if ( urlFailureCache.hasFailedBefore( url ) )
883 transferSimpleFile( wagon, remoteRepository, remotePath + ext, repository, resource, destFile );
884 log.debug( "Checksum {} Downloaded: {} to move to {}", url, destFile, resource );
886 catch ( NotFoundException e )
888 urlFailureCache.cacheFailure( url );
889 log.debug( "Transfer failed, checksum not found: {}", url );
890 // Consume it, do not pass this on.
892 catch ( NotModifiedException e )
894 log.debug( "Transfer skipped, checksum not modified: {}", url );
895 // Consume it, do not pass this on.
897 catch ( ProxyException e )
899 urlFailureCache.cacheFailure( url );
900 log.warn( "Transfer failed on checksum: {} : {}", url, e.getMessage(), e );
901 // Critical issue, pass it on.
907 * Perform the transfer of the remote file to the local file specified.
909 * @param wagon the wagon instance to use.
910 * @param remoteRepository the remote repository to use
911 * @param remotePath the remote path to attempt to get
912 * @param repository the managed repository that will hold the file
913 * @param origFile the local file to save to
914 * @throws ProxyException if there was a problem moving the downloaded file into place.
916 private void transferSimpleFile( Wagon wagon, RemoteRepositoryContent remoteRepository, String remotePath,
917 ManagedRepositoryContent repository, File origFile, File destFile )
918 throws ProxyException
920 assert ( remotePath != null );
922 // Transfer the file.
925 boolean success = false;
927 if ( !origFile.exists() )
929 log.debug( "Retrieving {} from {}", remotePath, remoteRepository.getRepository().getName() );
930 wagon.get( addParameters( remotePath, remoteRepository.getRepository() ), destFile );
933 // You wouldn't get here on failure, a WagonException would have been thrown.
934 log.debug( "Downloaded successfully." );
938 log.debug( "Retrieving {} from {} if updated", remotePath, remoteRepository.getRepository().getName() );
939 success = wagon.getIfNewer( addParameters( remotePath, remoteRepository.getRepository() ), destFile,
940 origFile.lastModified() );
943 throw new NotModifiedException(
944 "Not downloaded, as local file is newer than remote side: " + origFile.getAbsolutePath() );
947 if ( destFile.exists() )
949 log.debug( "Downloaded successfully." );
953 catch ( ResourceDoesNotExistException e )
955 throw new NotFoundException(
956 "Resource [" + remoteRepository.getURL() + "/" + remotePath + "] does not exist: " + e.getMessage(),
959 catch ( WagonException e )
961 // TODO: shouldn't have to drill into the cause, but TransferFailedException is often not descriptive enough
964 "Download failure on resource [" + remoteRepository.getURL() + "/" + remotePath + "]:" + e.getMessage();
965 if ( e.getCause() != null )
967 msg += " (cause: " + e.getCause() + ")";
969 throw new ProxyException( msg, e );
974 * Apply the policies.
976 * @param policies the map of policies to execute. (Map of String policy keys, to {@link DownloadPolicy} objects)
977 * @param settings the map of settings for the policies to execute. (Map of String policy keys, to String policy
979 * @param request the request properties (utilized by the {@link DownloadPolicy#applyPolicy(String, Properties, File)}
981 * @param localFile the local file (utilized by the {@link DownloadPolicy#applyPolicy(String, Properties, File)})
982 * @throws PolicyViolationException
984 private void validatePolicies( Map<String, ? extends DownloadPolicy> policies, Map<String, String> settings,
985 Properties request, File localFile )
986 throws PolicyViolationException
988 for ( Entry<String, ? extends DownloadPolicy> entry : policies.entrySet() )
990 // olamy with spring rolehint is now downloadPolicy#hint
991 // so substring after last # to get the hint as with plexus
992 String key = StringUtils.substringAfterLast( entry.getKey(), "#" );
993 DownloadPolicy policy = entry.getValue();
994 String defaultSetting = policy.getDefaultOption();
996 String setting = StringUtils.defaultString( settings.get( key ), defaultSetting );
998 log.debug( "Applying [{}] policy with [{}]", key, setting );
1001 policy.applyPolicy( setting, request, localFile );
1003 catch ( PolicyConfigurationException e )
1005 log.error( e.getMessage(), e );
1010 private void validatePolicies( Map<String, DownloadErrorPolicy> policies, Map<String, String> settings,
1011 Properties request, ArtifactReference artifact, RemoteRepositoryContent content,
1012 File localFile, Exception exception, Map<String, Exception> previousExceptions )
1013 throws ProxyDownloadException
1015 boolean process = true;
1016 for ( Entry<String, ? extends DownloadErrorPolicy> entry : policies.entrySet() )
1019 // olamy with spring rolehint is now downloadPolicy#hint
1020 // so substring after last # to get the hint as with plexus
1021 String key = StringUtils.substringAfterLast( entry.getKey(), "#" );
1022 DownloadErrorPolicy policy = entry.getValue();
1023 String defaultSetting = policy.getDefaultOption();
1024 String setting = StringUtils.defaultString( settings.get( key ), defaultSetting );
1026 log.debug( "Applying [{}] policy with [{}]", key, setting );
1029 // all policies must approve the exception, any can cancel
1030 process = policy.applyPolicy( setting, request, localFile, exception, previousExceptions );
1036 catch ( PolicyConfigurationException e )
1038 log.error( e.getMessage(), e );
1044 // if the exception was queued, don't throw it
1045 if ( !previousExceptions.containsKey( content.getId() ) )
1047 throw new ProxyDownloadException(
1048 "An error occurred in downloading from the remote repository, and the policy is to fail immediately",
1049 content.getId(), exception );
1054 // if the exception was queued, but cancelled, remove it
1055 previousExceptions.remove( content.getId() );
1059 "Transfer error from repository {} for artifact {} , continuing to next repository. Error message: {}",
1060 content.getRepository().getId(), Keys.toKey( artifact ), exception.getMessage() );
1061 log.debug( "Full stack trace", exception );
1065 * Creates a working directory
1068 * @return file location of working directory
1070 private File createWorkingDirectory( ManagedRepositoryContent repository )
1074 return Files.createTempDirectory( "temp" ).toFile();
1076 catch ( IOException e )
1078 throw new RuntimeException( e.getMessage(), e );
1084 * Used to move the temporary file to its real destination. This is patterned from the way WagonManager handles its
1087 * @param temp The completed download file
1088 * @param target The final location of the downloaded file
1089 * @throws ProxyException when the temp file cannot replace the target file
1091 private void moveTempToTarget( File temp, File target )
1092 throws ProxyException
1095 // TODO file lock library
1099 lock = fileLockManager.writeFileLock( target );
1100 if ( lock.getFile().exists() && !lock.getFile().delete() )
1102 throw new ProxyException( "Unable to overwrite existing target file: " + target.getAbsolutePath() );
1105 lock.getFile().getParentFile().mkdirs();
1107 if ( !temp.renameTo( lock.getFile() ) )
1109 log.warn( "Unable to rename tmp file to its final name... resorting to copy command." );
1113 FileUtils.copyFile( temp, lock.getFile() );
1115 catch ( IOException e )
1117 if ( lock.getFile().exists() )
1119 log.debug( "Tried to copy file {} to {} but file with this name already exists.",
1120 temp.getName(), lock.getFile().getAbsolutePath() );
1124 throw new ProxyException(
1125 "Cannot copy tmp file " + temp.getAbsolutePath() + " to its final location", e );
1130 FileUtils.deleteQuietly( temp );
1134 catch ( FileLockException e )
1136 throw new ProxyException( e.getMessage(), e );
1138 catch ( FileLockTimeoutException e )
1140 throw new ProxyException( e.getMessage(), e );
1145 * Using wagon, connect to the remote repository.
1147 * @param connector the connector configuration to utilize (for obtaining network proxy configuration from)
1148 * @param wagon the wagon instance to establish the connection on.
1149 * @param remoteRepository the remote repository to connect to.
1150 * @return true if the connection was successful. false if not connected.
1152 private boolean connectToRepository( ProxyConnector connector, Wagon wagon,
1153 RemoteRepositoryContent remoteRepository )
1155 boolean connected = false;
1157 final ProxyInfo networkProxy =
1158 connector.getProxyId() == null ? null : this.networkProxyMap.get( connector.getProxyId() );
1160 if ( log.isDebugEnabled() )
1162 if ( networkProxy != null )
1164 // TODO: move to proxyInfo.toString()
1165 String msg = "Using network proxy " + networkProxy.getHost() + ":" + networkProxy.getPort()
1166 + " to connect to remote repository " + remoteRepository.getURL();
1167 if ( networkProxy.getNonProxyHosts() != null )
1169 msg += "; excluding hosts: " + networkProxy.getNonProxyHosts();
1171 if ( StringUtils.isNotBlank( networkProxy.getUserName() ) )
1173 msg += "; as user: " + networkProxy.getUserName();
1179 AuthenticationInfo authInfo = null;
1180 String username = remoteRepository.getRepository().getUserName();
1181 String password = remoteRepository.getRepository().getPassword();
1183 if ( StringUtils.isNotBlank( username ) && StringUtils.isNotBlank( password ) )
1185 log.debug( "Using username {} to connect to remote repository {}", username, remoteRepository.getURL() );
1186 authInfo = new AuthenticationInfo();
1187 authInfo.setUserName( username );
1188 authInfo.setPassword( password );
1191 // Convert seconds to milliseconds
1192 int timeoutInMilliseconds = remoteRepository.getRepository().getTimeout() * 1000;
1194 // Set timeout read and connect
1195 // FIXME olamy having 2 config values
1196 wagon.setReadTimeout( timeoutInMilliseconds );
1197 wagon.setTimeout( timeoutInMilliseconds );
1201 Repository wagonRepository =
1202 new Repository( remoteRepository.getId(), remoteRepository.getURL().toString() );
1203 wagon.connect( wagonRepository, authInfo, networkProxy );
1206 catch ( ConnectionException | AuthenticationException e )
1208 log.warn( "Could not connect to {}: {}", remoteRepository.getRepository().getName(), e.getMessage() );
1216 * Tests whitelist and blacklist patterns against path.
1218 * @param path the path to test.
1219 * @param patterns the list of patterns to check.
1220 * @return true if the path matches at least 1 pattern in the provided patterns list.
1222 private boolean matchesPattern( String path, List<String> patterns )
1224 if ( CollectionUtils.isEmpty( patterns ) )
1229 if ( !path.startsWith( "/" ) )
1234 for ( String pattern : patterns )
1236 if ( !pattern.startsWith( "/" ) )
1238 pattern = "/" + pattern;
1241 if ( SelectorUtils.matchPath( pattern, path, false ) )
1251 * TODO: Ensure that list is correctly ordered based on configuration. See MRM-477
1254 public List<ProxyConnector> getProxyConnectors( ManagedRepositoryContent repository )
1257 if ( !this.proxyConnectorMap.containsKey( repository.getId() ) )
1259 return Collections.emptyList();
1261 List<ProxyConnector> ret = new ArrayList<>( this.proxyConnectorMap.get( repository.getId() ) );
1263 Collections.sort( ret, ProxyConnectorOrderComparator.getInstance() );
1269 public void afterConfigurationChange( Registry registry, String propertyName, Object propertyValue )
1271 if ( ConfigurationNames.isNetworkProxy( propertyName ) || ConfigurationNames.isManagedRepositories(
1272 propertyName ) || ConfigurationNames.isRemoteRepositories( propertyName )
1273 || ConfigurationNames.isProxyConnector( propertyName ) )
1275 initConnectorsAndNetworkProxies();
1279 protected String addParameters( String path, RemoteRepository remoteRepository )
1281 if ( remoteRepository.getExtraParameters().isEmpty() )
1286 boolean question = false;
1288 StringBuilder res = new StringBuilder( path == null ? "" : path );
1290 for ( Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() )
1294 res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() );
1298 return res.toString();
1303 public void beforeConfigurationChange( Registry registry, String propertyName, Object propertyValue )
1308 public ArchivaConfiguration getArchivaConfiguration()
1310 return archivaConfiguration;
1313 public void setArchivaConfiguration( ArchivaConfiguration archivaConfiguration )
1315 this.archivaConfiguration = archivaConfiguration;
1318 public RepositoryContentFactory getRepositoryFactory()
1320 return repositoryFactory;
1323 public void setRepositoryFactory( RepositoryContentFactory repositoryFactory )
1325 this.repositoryFactory = repositoryFactory;
1328 public MetadataTools getMetadataTools()
1330 return metadataTools;
1333 public void setMetadataTools( MetadataTools metadataTools )
1335 this.metadataTools = metadataTools;
1338 public UrlFailureCache getUrlFailureCache()
1340 return urlFailureCache;
1343 public void setUrlFailureCache( UrlFailureCache urlFailureCache )
1345 this.urlFailureCache = urlFailureCache;
1348 public WagonFactory getWagonFactory()
1350 return wagonFactory;
1353 public void setWagonFactory( WagonFactory wagonFactory )
1355 this.wagonFactory = wagonFactory;
1358 public Map<String, PreDownloadPolicy> getPreDownloadPolicies()
1360 return preDownloadPolicies;
1363 public void setPreDownloadPolicies( Map<String, PreDownloadPolicy> preDownloadPolicies )
1365 this.preDownloadPolicies = preDownloadPolicies;
1368 public Map<String, PostDownloadPolicy> getPostDownloadPolicies()
1370 return postDownloadPolicies;
1373 public void setPostDownloadPolicies( Map<String, PostDownloadPolicy> postDownloadPolicies )
1375 this.postDownloadPolicies = postDownloadPolicies;
1378 public Map<String, DownloadErrorPolicy> getDownloadErrorPolicies()
1380 return downloadErrorPolicies;
1383 public void setDownloadErrorPolicies( Map<String, DownloadErrorPolicy> downloadErrorPolicies )
1385 this.downloadErrorPolicies = downloadErrorPolicies;