1 package org.apache.archiva.proxy;
4 * Licensed to the Apache Software Foundation (ASF) under one
5 * or more contributor license agreements. See the NOTICE file
6 * distributed with this work for additional information
7 * regarding copyright ownership. The ASF licenses this file
8 * to you under the Apache License, Version 2.0 (the
9 * "License"); you may not use this file except in compliance
10 * with the License. You may obtain a copy of the License at
12 * http://www.apache.org/licenses/LICENSE-2.0
14 * Unless required by applicable law or agreed to in writing,
15 * software distributed under the License is distributed on an
16 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17 * KIND, either express or implied. See the License for the
18 * specific language governing permissions and limitations
22 import org.apache.archiva.admin.model.RepositoryAdminException;
23 import org.apache.archiva.admin.model.beans.NetworkProxy;
24 import org.apache.archiva.admin.model.beans.ProxyConnectorRuleType;
25 import org.apache.archiva.admin.model.beans.RemoteRepository;
26 import org.apache.archiva.admin.model.networkproxy.NetworkProxyAdmin;
27 import org.apache.archiva.common.filelock.FileLockException;
28 import org.apache.archiva.common.filelock.FileLockManager;
29 import org.apache.archiva.common.filelock.FileLockTimeoutException;
30 import org.apache.archiva.common.filelock.Lock;
31 import org.apache.archiva.configuration.ArchivaConfiguration;
32 import org.apache.archiva.configuration.Configuration;
33 import org.apache.archiva.configuration.ConfigurationNames;
34 import org.apache.archiva.configuration.NetworkProxyConfiguration;
35 import org.apache.archiva.configuration.ProxyConnectorConfiguration;
36 import org.apache.archiva.configuration.ProxyConnectorRuleConfiguration;
37 import org.apache.archiva.model.ArtifactReference;
38 import org.apache.archiva.model.Keys;
39 import org.apache.archiva.model.RepositoryURL;
40 import org.apache.archiva.policies.DownloadErrorPolicy;
41 import org.apache.archiva.policies.DownloadPolicy;
42 import org.apache.archiva.policies.PolicyConfigurationException;
43 import org.apache.archiva.policies.PolicyViolationException;
44 import org.apache.archiva.policies.PostDownloadPolicy;
45 import org.apache.archiva.policies.PreDownloadPolicy;
46 import org.apache.archiva.policies.ProxyDownloadException;
47 import org.apache.archiva.policies.urlcache.UrlFailureCache;
48 import org.apache.archiva.proxy.common.WagonFactory;
49 import org.apache.archiva.proxy.common.WagonFactoryException;
50 import org.apache.archiva.proxy.common.WagonFactoryRequest;
51 import org.apache.archiva.proxy.model.ProxyFetchResult;
52 import org.apache.archiva.proxy.model.ProxyConnector;
53 import org.apache.archiva.proxy.model.RepositoryProxyConnectors;
54 import org.apache.archiva.redback.components.registry.Registry;
55 import org.apache.archiva.redback.components.registry.RegistryListener;
56 import org.apache.archiva.redback.components.taskqueue.TaskQueueException;
57 import org.apache.archiva.repository.ManagedRepositoryContent;
58 import org.apache.archiva.repository.RemoteRepositoryContent;
59 import org.apache.archiva.repository.RepositoryContentFactory;
60 import org.apache.archiva.repository.RepositoryException;
61 import org.apache.archiva.repository.RepositoryNotFoundException;
62 import org.apache.archiva.repository.metadata.MetadataTools;
63 import org.apache.archiva.repository.metadata.RepositoryMetadataException;
64 import org.apache.archiva.scheduler.ArchivaTaskScheduler;
65 import org.apache.archiva.scheduler.repository.model.RepositoryTask;
66 import org.apache.commons.collections.CollectionUtils;
67 import org.apache.commons.io.FileUtils;
68 import org.apache.commons.io.FilenameUtils;
69 import org.apache.commons.lang.StringUtils;
70 import org.apache.commons.lang.SystemUtils;
71 import org.apache.maven.wagon.ConnectionException;
72 import org.apache.maven.wagon.ResourceDoesNotExistException;
73 import org.apache.maven.wagon.Wagon;
74 import org.apache.maven.wagon.WagonException;
75 import org.apache.maven.wagon.authentication.AuthenticationException;
76 import org.apache.maven.wagon.authentication.AuthenticationInfo;
77 import org.apache.maven.wagon.proxy.ProxyInfo;
78 import org.apache.maven.wagon.repository.Repository;
79 import org.apache.tools.ant.types.selectors.SelectorUtils;
80 import org.slf4j.Logger;
81 import org.slf4j.LoggerFactory;
82 import org.slf4j.MarkerFactory;
83 import org.springframework.stereotype.Service;
85 import javax.annotation.PostConstruct;
86 import javax.inject.Inject;
87 import javax.inject.Named;
89 import java.io.IOException;
90 import java.nio.file.Files;
91 import java.util.ArrayList;
92 import java.util.Collections;
93 import java.util.HashMap;
94 import java.util.LinkedHashMap;
95 import java.util.List;
97 import java.util.Map.Entry;
98 import java.util.Properties;
99 import java.util.concurrent.ConcurrentHashMap;
100 import java.util.concurrent.ConcurrentMap;
103 * DefaultRepositoryProxyConnectors
105 * TODO exception handling needs work - "not modified" is not really an exceptional case, and it has more layers than
106 * your average brown onion
108 @Service("repositoryProxyConnectors#default")
109 public class DefaultRepositoryProxyConnectors
110 implements RepositoryProxyConnectors, RegistryListener
112 private Logger log = LoggerFactory.getLogger( DefaultRepositoryProxyConnectors.class );
115 @Named(value = "archivaConfiguration#default")
116 private ArchivaConfiguration archivaConfiguration;
119 @Named(value = "repositoryContentFactory#default")
120 private RepositoryContentFactory repositoryFactory;
123 @Named(value = "metadataTools#default")
124 private MetadataTools metadataTools;
127 private Map<String, PreDownloadPolicy> preDownloadPolicies;
130 private Map<String, PostDownloadPolicy> postDownloadPolicies;
133 private Map<String, DownloadErrorPolicy> downloadErrorPolicies;
136 private UrlFailureCache urlFailureCache;
138 private ConcurrentMap<String, List<ProxyConnector>> proxyConnectorMap = new ConcurrentHashMap<>();
140 private ConcurrentMap<String, ProxyInfo> networkProxyMap = new ConcurrentHashMap<>();
143 private WagonFactory wagonFactory;
146 @Named(value = "archivaTaskScheduler#repository")
147 private ArchivaTaskScheduler scheduler;
150 private NetworkProxyAdmin networkProxyAdmin;
153 @Named(value = "fileLockManager#default")
154 private FileLockManager fileLockManager;
157 public void initialize()
159 initConnectorsAndNetworkProxies();
160 archivaConfiguration.addChangeListener( this );
164 @SuppressWarnings("unchecked")
165 private void initConnectorsAndNetworkProxies()
168 ProxyConnectorOrderComparator proxyOrderSorter = new ProxyConnectorOrderComparator();
169 this.proxyConnectorMap.clear();
171 Configuration configuration = archivaConfiguration.getConfiguration();
173 List<ProxyConnectorRuleConfiguration> allProxyConnectorRuleConfigurations =
174 configuration.getProxyConnectorRuleConfigurations();
176 List<ProxyConnectorConfiguration> proxyConfigs = configuration.getProxyConnectors();
177 for ( ProxyConnectorConfiguration proxyConfig : proxyConfigs )
179 String key = proxyConfig.getSourceRepoId();
183 // Create connector object.
184 ProxyConnector connector = new ProxyConnector();
186 connector.setSourceRepository(
187 repositoryFactory.getManagedRepositoryContent( proxyConfig.getSourceRepoId() ) );
188 connector.setTargetRepository(
189 repositoryFactory.getRemoteRepositoryContent( proxyConfig.getTargetRepoId() ) );
191 connector.setProxyId( proxyConfig.getProxyId() );
192 connector.setPolicies( proxyConfig.getPolicies() );
193 connector.setOrder( proxyConfig.getOrder() );
194 connector.setDisabled( proxyConfig.isDisabled() );
196 // Copy any blacklist patterns.
197 List<String> blacklist = new ArrayList<>( 0 );
198 if ( CollectionUtils.isNotEmpty( proxyConfig.getBlackListPatterns() ) )
200 blacklist.addAll( proxyConfig.getBlackListPatterns() );
202 connector.setBlacklist( blacklist );
204 // Copy any whitelist patterns.
205 List<String> whitelist = new ArrayList<>( 0 );
206 if ( CollectionUtils.isNotEmpty( proxyConfig.getWhiteListPatterns() ) )
208 whitelist.addAll( proxyConfig.getWhiteListPatterns() );
210 connector.setWhitelist( whitelist );
212 List<ProxyConnectorRuleConfiguration> proxyConnectorRuleConfigurations =
213 findProxyConnectorRules( connector.getSourceRepository().getId(),
214 connector.getTargetRepository().getId(),
215 allProxyConnectorRuleConfigurations );
217 if ( !proxyConnectorRuleConfigurations.isEmpty() )
219 for ( ProxyConnectorRuleConfiguration proxyConnectorRuleConfiguration : proxyConnectorRuleConfigurations )
221 if ( StringUtils.equals( proxyConnectorRuleConfiguration.getRuleType(),
222 ProxyConnectorRuleType.BLACK_LIST.getRuleType() ) )
224 connector.getBlacklist().add( proxyConnectorRuleConfiguration.getPattern() );
227 if ( StringUtils.equals( proxyConnectorRuleConfiguration.getRuleType(),
228 ProxyConnectorRuleType.WHITE_LIST.getRuleType() ) )
230 connector.getWhitelist().add( proxyConnectorRuleConfiguration.getPattern() );
235 // Get other connectors
236 List<ProxyConnector> connectors = this.proxyConnectorMap.get( key );
237 if ( connectors == null )
239 // Create if we are the first.
240 connectors = new ArrayList<>( 1 );
243 // Add the connector.
244 connectors.add( connector );
246 // Ensure the list is sorted.
247 Collections.sort( connectors, proxyOrderSorter );
249 // Set the key to the list of connectors.
250 this.proxyConnectorMap.put( key, connectors );
252 catch ( RepositoryNotFoundException e )
254 log.warn( "Unable to use proxy connector: {}", e.getMessage(), e );
256 catch ( RepositoryException e )
258 log.warn( "Unable to use proxy connector: {}", e.getMessage(), e );
264 this.networkProxyMap.clear();
266 List<NetworkProxyConfiguration> networkProxies = archivaConfiguration.getConfiguration().getNetworkProxies();
267 for ( NetworkProxyConfiguration networkProxyConfig : networkProxies )
269 String key = networkProxyConfig.getId();
271 ProxyInfo proxy = new ProxyInfo();
273 proxy.setType( networkProxyConfig.getProtocol() );
274 proxy.setHost( networkProxyConfig.getHost() );
275 proxy.setPort( networkProxyConfig.getPort() );
276 proxy.setUserName( networkProxyConfig.getUsername() );
277 proxy.setPassword( networkProxyConfig.getPassword() );
279 this.networkProxyMap.put( key, proxy );
284 private List<ProxyConnectorRuleConfiguration> findProxyConnectorRules( String sourceRepository,
285 String targetRepository,
286 List<ProxyConnectorRuleConfiguration> all )
288 List<ProxyConnectorRuleConfiguration> proxyConnectorRuleConfigurations = new ArrayList<>();
290 for ( ProxyConnectorRuleConfiguration proxyConnectorRuleConfiguration : all )
292 for ( ProxyConnectorConfiguration proxyConnector : proxyConnectorRuleConfiguration.getProxyConnectors() )
294 if ( StringUtils.equals( sourceRepository, proxyConnector.getSourceRepoId() ) && StringUtils.equals(
295 targetRepository, proxyConnector.getTargetRepoId() ) )
297 proxyConnectorRuleConfigurations.add( proxyConnectorRuleConfiguration );
302 return proxyConnectorRuleConfigurations;
306 public File fetchFromProxies( ManagedRepositoryContent repository, ArtifactReference artifact )
307 throws ProxyDownloadException
309 File localFile = toLocalFile( repository, artifact );
311 Properties requestProperties = new Properties();
312 requestProperties.setProperty( "filetype", "artifact" );
313 requestProperties.setProperty( "version", artifact.getVersion() );
314 requestProperties.setProperty( "managedRepositoryId", repository.getId() );
316 List<ProxyConnector> connectors = getProxyConnectors( repository );
317 Map<String, Exception> previousExceptions = new LinkedHashMap<>();
318 for ( ProxyConnector connector : connectors )
320 if ( connector.isDisabled() )
325 RemoteRepositoryContent targetRepository = connector.getTargetRepository();
326 requestProperties.setProperty( "remoteRepositoryId", targetRepository.getId() );
328 String targetPath = targetRepository.toPath( artifact );
330 if ( SystemUtils.IS_OS_WINDOWS )
332 // toPath use system PATH_SEPARATOR so on windows url are \ which doesn't work very well :-)
333 targetPath = FilenameUtils.separatorsToUnix( targetPath );
338 File downloadedFile =
339 transferFile( connector, targetRepository, targetPath, repository, localFile, requestProperties,
342 if ( fileExists( downloadedFile ) )
344 log.debug( "Successfully transferred: {}", downloadedFile.getAbsolutePath() );
345 return downloadedFile;
348 catch ( NotFoundException e )
350 log.debug( "Artifact {} not found on repository \"{}\".", Keys.toKey( artifact ),
351 targetRepository.getRepository().getId() );
353 catch ( NotModifiedException e )
355 log.debug( "Artifact {} not updated on repository \"{}\".", Keys.toKey( artifact ),
356 targetRepository.getRepository().getId() );
358 catch ( ProxyException | RepositoryAdminException e )
360 validatePolicies( this.downloadErrorPolicies, connector.getPolicies(), requestProperties, artifact,
361 targetRepository, localFile, e, previousExceptions );
365 if ( !previousExceptions.isEmpty() )
367 throw new ProxyDownloadException( "Failures occurred downloading from some remote repositories",
368 previousExceptions );
371 log.debug( "Exhausted all target repositories, artifact {} not found.", Keys.toKey( artifact ) );
377 public File fetchFromProxies( ManagedRepositoryContent repository, String path )
379 File localFile = new File( repository.getRepoRoot(), path );
381 // no update policies for these paths
382 if ( localFile.exists() )
387 Properties requestProperties = new Properties();
388 requestProperties.setProperty( "filetype", "resource" );
389 requestProperties.setProperty( "managedRepositoryId", repository.getId() );
391 List<ProxyConnector> connectors = getProxyConnectors( repository );
392 for ( ProxyConnector connector : connectors )
394 if ( connector.isDisabled() )
399 RemoteRepositoryContent targetRepository = connector.getTargetRepository();
400 requestProperties.setProperty( "remoteRepositoryId", targetRepository.getId() );
402 String targetPath = path;
406 File downloadedFile =
407 transferFile( connector, targetRepository, targetPath, repository, localFile, requestProperties,
410 if ( fileExists( downloadedFile ) )
412 log.debug( "Successfully transferred: {}", downloadedFile.getAbsolutePath() );
413 return downloadedFile;
416 catch ( NotFoundException e )
418 log.debug( "Resource {} not found on repository \"{}\".", path,
419 targetRepository.getRepository().getId() );
421 catch ( NotModifiedException e )
423 log.debug( "Resource {} not updated on repository \"{}\".", path,
424 targetRepository.getRepository().getId() );
426 catch ( ProxyException e )
429 "Transfer error from repository {} for resource {}, continuing to next repository. Error message: {}",
430 targetRepository.getRepository().getId(), path, e.getMessage() );
431 log.debug( MarkerFactory.getDetachedMarker( "transfer.error" ),
432 "Transfer error from repository \"" + targetRepository.getRepository().getId()
433 + "\" for resource " + path + ", continuing to next repository. Error message: {}",
437 catch ( RepositoryAdminException e )
439 log.debug( MarkerFactory.getDetachedMarker( "transfer.error" ),
440 "Transfer error from repository {} for resource {}, continuing to next repository. Error message: {}",
441 targetRepository.getRepository().getId(), path, e.getMessage(), e );
442 log.debug( MarkerFactory.getDetachedMarker( "transfer.error" ), "Full stack trace", e );
446 log.debug( "Exhausted all target repositories, resource {} not found.", path );
452 public ProxyFetchResult fetchMetadataFromProxies( ManagedRepositoryContent repository, String logicalPath )
454 File localFile = new File( repository.getRepoRoot(), logicalPath );
456 Properties requestProperties = new Properties();
457 requestProperties.setProperty( "filetype", "metadata" );
458 boolean metadataNeedsUpdating = false;
459 long originalTimestamp = getLastModified( localFile );
461 List<ProxyConnector> connectors = new ArrayList<>( getProxyConnectors( repository ) );
462 for ( ProxyConnector connector : connectors )
464 if ( connector.isDisabled() )
469 RemoteRepositoryContent targetRepository = connector.getTargetRepository();
471 File localRepoFile = toLocalRepoFile( repository, targetRepository, logicalPath );
472 long originalMetadataTimestamp = getLastModified( localRepoFile );
476 transferFile( connector, targetRepository, logicalPath, repository, localRepoFile, requestProperties,
479 if ( hasBeenUpdated( localRepoFile, originalMetadataTimestamp ) )
481 metadataNeedsUpdating = true;
484 catch ( NotFoundException e )
487 log.debug( "Metadata {} not found on remote repository '{}'.", logicalPath,
488 targetRepository.getRepository().getId(), e );
491 catch ( NotModifiedException e )
494 log.debug( "Metadata {} not updated on remote repository '{}'.", logicalPath,
495 targetRepository.getRepository().getId(), e );
498 catch ( ProxyException | RepositoryAdminException e )
501 "Transfer error from repository {} for versioned Metadata {}, continuing to next repository. Error message: {}",
502 targetRepository.getRepository().getId(), logicalPath, e.getMessage() );
503 log.debug( "Full stack trace", e );
507 if ( hasBeenUpdated( localFile, originalTimestamp ) )
509 metadataNeedsUpdating = true;
512 if ( metadataNeedsUpdating || !localFile.exists() )
516 metadataTools.updateMetadata( repository, logicalPath );
518 catch ( RepositoryMetadataException e )
520 log.warn( "Unable to update metadata {}:{}", localFile.getAbsolutePath(), e.getMessage(), e );
525 if ( fileExists( localFile ) )
527 return new ProxyFetchResult( localFile, metadataNeedsUpdating );
530 return new ProxyFetchResult( null, false );
535 * @param remoteRepository
542 * @param workingDirectory
544 * @throws ProxyException
545 * @throws NotModifiedException
546 * @throws org.apache.archiva.admin.model.RepositoryAdminException
548 protected void transferResources( ProxyConnector connector, RemoteRepositoryContent remoteRepository, File tmpMd5,
549 File tmpSha1, File tmpResource, String url, String remotePath, File resource,
550 File workingDirectory, ManagedRepositoryContent repository )
551 throws ProxyException, NotModifiedException, RepositoryAdminException
556 RepositoryURL repoUrl = remoteRepository.getURL();
557 String protocol = repoUrl.getProtocol();
558 NetworkProxy networkProxy = null;
559 if ( StringUtils.isNotBlank( connector.getProxyId() ) )
561 networkProxy = networkProxyAdmin.getNetworkProxy( connector.getProxyId() );
563 WagonFactoryRequest wagonFactoryRequest = new WagonFactoryRequest( "wagon#" + protocol,
564 remoteRepository.getRepository().getExtraHeaders() ).networkProxy(
566 wagon = wagonFactory.getWagon( wagonFactoryRequest );
569 throw new ProxyException( "Unsupported target repository protocol: " + protocol );
574 throw new ProxyException( "Unsupported target repository protocol: " + protocol );
577 boolean connected = connectToRepository( connector, wagon, remoteRepository );
580 transferArtifact( wagon, remoteRepository, remotePath, repository, resource, workingDirectory,
583 // TODO: these should be used to validate the download based on the policies, not always downloaded
585 // save on connections since md5 is rarely used
586 transferChecksum( wagon, remoteRepository, remotePath, repository, resource, workingDirectory, ".sha1",
588 transferChecksum( wagon, remoteRepository, remotePath, repository, resource, workingDirectory, ".md5",
592 catch ( NotFoundException e )
594 urlFailureCache.cacheFailure( url );
597 catch ( NotModifiedException e )
599 // Do not cache url here.
602 catch ( ProxyException e )
604 urlFailureCache.cacheFailure( url );
607 catch ( WagonFactoryException e )
609 throw new ProxyException( e.getMessage(), e );
619 catch ( ConnectionException e )
621 log.warn( "Unable to disconnect wagon.", e );
627 private void transferArtifact( Wagon wagon, RemoteRepositoryContent remoteRepository, String remotePath,
628 ManagedRepositoryContent repository, File resource, File tmpDirectory,
630 throws ProxyException
632 transferSimpleFile( wagon, remoteRepository, remotePath, repository, resource, destFile );
635 private long getLastModified( File file )
637 if ( !file.exists() || !file.isFile() )
642 return file.lastModified();
645 private boolean hasBeenUpdated( File file, long originalLastModified )
647 if ( !file.exists() || !file.isFile() )
652 long currentLastModified = getLastModified( file );
653 return ( currentLastModified > originalLastModified );
656 private File toLocalRepoFile( ManagedRepositoryContent repository, RemoteRepositoryContent targetRepository,
659 String repoPath = metadataTools.getRepositorySpecificName( targetRepository, targetPath );
660 return new File( repository.getRepoRoot(), repoPath );
664 * Test if the provided ManagedRepositoryContent has any proxies configured for it.
667 public boolean hasProxies( ManagedRepositoryContent repository )
669 synchronized ( this.proxyConnectorMap )
671 return this.proxyConnectorMap.containsKey( repository.getId() );
675 private File toLocalFile( ManagedRepositoryContent repository, ArtifactReference artifact )
677 return repository.toFile( artifact );
681 * Simple method to test if the file exists on the local disk.
683 * @param file the file to test. (may be null)
684 * @return true if file exists. false if the file param is null, doesn't exist, or is not of type File.
686 private boolean fileExists( File file )
693 if ( !file.exists() )
698 return file.isFile();
702 * Perform the transfer of the file.
704 * @param connector the connector configuration to use.
705 * @param remoteRepository the remote repository get the resource from.
706 * @param remotePath the path in the remote repository to the resource to get.
707 * @param repository the managed repository that will hold the file
708 * @param resource the local file to place the downloaded resource into
709 * @param requestProperties the request properties to utilize for policy handling.
710 * @param executeConsumers whether to execute the consumers after proxying
711 * @return the local file that was downloaded, or null if not downloaded.
712 * @throws NotFoundException if the file was not found on the remote repository.
713 * @throws NotModifiedException if the localFile was present, and the resource was present on remote repository, but
714 * the remote resource is not newer than the local File.
715 * @throws ProxyException if transfer was unsuccessful.
717 private File transferFile( ProxyConnector connector, RemoteRepositoryContent remoteRepository, String remotePath,
718 ManagedRepositoryContent repository, File resource, Properties requestProperties,
719 boolean executeConsumers )
720 throws ProxyException, NotModifiedException, RepositoryAdminException
722 String url = remoteRepository.getURL().getUrl();
723 if ( !url.endsWith( "/" ) )
727 url = url + remotePath;
728 requestProperties.setProperty( "url", url );
730 // Is a whitelist defined?
731 if ( CollectionUtils.isNotEmpty( connector.getWhitelist() ) )
733 // Path must belong to whitelist.
734 if ( !matchesPattern( remotePath, connector.getWhitelist() ) )
736 log.debug( "Path [{}] is not part of defined whitelist (skipping transfer from repository [{}]).",
737 remotePath, remoteRepository.getRepository().getName() );
742 // Is target path part of blacklist?
743 if ( matchesPattern( remotePath, connector.getBlacklist() ) )
745 log.debug( "Path [{}] is part of blacklist (skipping transfer from repository [{}]).", remotePath,
746 remoteRepository.getRepository().getName() );
750 // Handle pre-download policy
753 validatePolicies( this.preDownloadPolicies, connector.getPolicies(), requestProperties, resource );
755 catch ( PolicyViolationException e )
757 String emsg = "Transfer not attempted on " + url + " : " + e.getMessage();
758 if ( fileExists( resource ) )
760 log.debug( "{} : using already present local file.", emsg );
768 File workingDirectory = createWorkingDirectory( repository );
769 File tmpResource = new File( workingDirectory, resource.getName() );
770 File tmpMd5 = new File( workingDirectory, resource.getName() + ".md5" );
771 File tmpSha1 = new File( workingDirectory, resource.getName() + ".sha1" );
776 transferResources( connector, remoteRepository, tmpMd5, tmpSha1, tmpResource, url, remotePath, resource,
777 workingDirectory, repository );
779 // Handle post-download policies.
782 validatePolicies( this.postDownloadPolicies, connector.getPolicies(), requestProperties, tmpResource );
784 catch ( PolicyViolationException e )
786 log.warn( "Transfer invalidated from {} : {}", url, e.getMessage() );
787 executeConsumers = false;
788 if ( !fileExists( tmpResource ) )
794 if ( resource != null )
796 synchronized ( resource.getAbsolutePath().intern() )
798 File directory = resource.getParentFile();
799 moveFileIfExists( tmpMd5, directory );
800 moveFileIfExists( tmpSha1, directory );
801 moveFileIfExists( tmpResource, directory );
807 FileUtils.deleteQuietly( workingDirectory );
810 if ( executeConsumers )
812 // Just-in-time update of the index and database by executing the consumers for this artifact
813 //consumers.executeConsumers( connector.getSourceRepository().getRepository(), resource );
814 queueRepositoryTask( connector.getSourceRepository().getRepository().getId(), resource );
820 private void queueRepositoryTask( String repositoryId, File localFile )
822 RepositoryTask task = new RepositoryTask();
823 task.setRepositoryId( repositoryId );
824 task.setResourceFile( localFile );
825 task.setUpdateRelatedArtifacts( true );
826 task.setScanAll( true );
830 scheduler.queueTask( task );
832 catch ( TaskQueueException e )
834 log.error( "Unable to queue repository task to execute consumers on resource file ['" + localFile.getName()
840 * Moves the file into repository location if it exists
842 * @param fileToMove this could be either the main artifact, sha1 or md5 checksum file.
843 * @param directory directory to write files to
845 private void moveFileIfExists( File fileToMove, File directory )
846 throws ProxyException
848 if ( fileToMove != null && fileToMove.exists() )
850 File newLocation = new File( directory, fileToMove.getName() );
851 moveTempToTarget( fileToMove, newLocation );
857 * Quietly transfer the checksum file from the remote repository to the local file.
860 * @param wagon the wagon instance (should already be connected) to use.
861 * @param remoteRepository the remote repository to transfer from.
862 * @param remotePath the remote path to the resource to get.
863 * @param repository the managed repository that will hold the file
864 * @param resource the local file that should contain the downloaded contents
865 * @param tmpDirectory the temporary directory to download to
866 * @param ext the type of checksum to transfer (example: ".md5" or ".sha1")
867 * @throws ProxyException if copying the downloaded file into place did not succeed.
869 private void transferChecksum( Wagon wagon, RemoteRepositoryContent remoteRepository, String remotePath,
870 ManagedRepositoryContent repository, File resource, File tmpDirectory, String ext,
872 throws ProxyException
874 String url = remoteRepository.getURL().getUrl() + remotePath + ext;
876 // Transfer checksum does not use the policy.
877 if ( urlFailureCache.hasFailedBefore( url ) )
884 transferSimpleFile( wagon, remoteRepository, remotePath + ext, repository, resource, destFile );
885 log.debug( "Checksum {} Downloaded: {} to move to {}", url, destFile, resource );
887 catch ( NotFoundException e )
889 urlFailureCache.cacheFailure( url );
890 log.debug( "Transfer failed, checksum not found: {}", url );
891 // Consume it, do not pass this on.
893 catch ( NotModifiedException e )
895 log.debug( "Transfer skipped, checksum not modified: {}", url );
896 // Consume it, do not pass this on.
898 catch ( ProxyException e )
900 urlFailureCache.cacheFailure( url );
901 log.warn( "Transfer failed on checksum: {} : {}", url, e.getMessage(), e );
902 // Critical issue, pass it on.
908 * Perform the transfer of the remote file to the local file specified.
910 * @param wagon the wagon instance to use.
911 * @param remoteRepository the remote repository to use
912 * @param remotePath the remote path to attempt to get
913 * @param repository the managed repository that will hold the file
914 * @param origFile the local file to save to
915 * @throws ProxyException if there was a problem moving the downloaded file into place.
917 private void transferSimpleFile( Wagon wagon, RemoteRepositoryContent remoteRepository, String remotePath,
918 ManagedRepositoryContent repository, File origFile, File destFile )
919 throws ProxyException
921 assert ( remotePath != null );
923 // Transfer the file.
926 boolean success = false;
928 if ( !origFile.exists() )
930 log.debug( "Retrieving {} from {}", remotePath, remoteRepository.getRepository().getName() );
931 wagon.get( addParameters( remotePath, remoteRepository.getRepository() ), destFile );
934 // You wouldn't get here on failure, a WagonException would have been thrown.
935 log.debug( "Downloaded successfully." );
939 log.debug( "Retrieving {} from {} if updated", remotePath, remoteRepository.getRepository().getName() );
940 success = wagon.getIfNewer( addParameters( remotePath, remoteRepository.getRepository() ), destFile,
941 origFile.lastModified() );
944 throw new NotModifiedException(
945 "Not downloaded, as local file is newer than remote side: " + origFile.getAbsolutePath() );
948 if ( destFile.exists() )
950 log.debug( "Downloaded successfully." );
954 catch ( ResourceDoesNotExistException e )
956 throw new NotFoundException(
957 "Resource [" + remoteRepository.getURL() + "/" + remotePath + "] does not exist: " + e.getMessage(),
960 catch ( WagonException e )
962 // TODO: shouldn't have to drill into the cause, but TransferFailedException is often not descriptive enough
965 "Download failure on resource [" + remoteRepository.getURL() + "/" + remotePath + "]:" + e.getMessage();
966 if ( e.getCause() != null )
968 msg += " (cause: " + e.getCause() + ")";
970 throw new ProxyException( msg, e );
975 * Apply the policies.
977 * @param policies the map of policies to execute. (Map of String policy keys, to {@link DownloadPolicy} objects)
978 * @param settings the map of settings for the policies to execute. (Map of String policy keys, to String policy
980 * @param request the request properties (utilized by the {@link DownloadPolicy#applyPolicy(String, Properties, File)}
982 * @param localFile the local file (utilized by the {@link DownloadPolicy#applyPolicy(String, Properties, File)})
983 * @throws PolicyViolationException
985 private void validatePolicies( Map<String, ? extends DownloadPolicy> policies, Map<String, String> settings,
986 Properties request, File localFile )
987 throws PolicyViolationException
989 for ( Entry<String, ? extends DownloadPolicy> entry : policies.entrySet() )
991 // olamy with spring rolehint is now downloadPolicy#hint
992 // so substring after last # to get the hint as with plexus
993 String key = StringUtils.substringAfterLast( entry.getKey(), "#" );
994 DownloadPolicy policy = entry.getValue();
995 String defaultSetting = policy.getDefaultOption();
997 String setting = StringUtils.defaultString( settings.get( key ), defaultSetting );
999 log.debug( "Applying [{}] policy with [{}]", key, setting );
1002 policy.applyPolicy( setting, request, localFile );
1004 catch ( PolicyConfigurationException e )
1006 log.error( e.getMessage(), e );
1011 private void validatePolicies( Map<String, DownloadErrorPolicy> policies, Map<String, String> settings,
1012 Properties request, ArtifactReference artifact, RemoteRepositoryContent content,
1013 File localFile, Exception exception, Map<String, Exception> previousExceptions )
1014 throws ProxyDownloadException
1016 boolean process = true;
1017 for ( Entry<String, ? extends DownloadErrorPolicy> entry : policies.entrySet() )
1020 // olamy with spring rolehint is now downloadPolicy#hint
1021 // so substring after last # to get the hint as with plexus
1022 String key = StringUtils.substringAfterLast( entry.getKey(), "#" );
1023 DownloadErrorPolicy policy = entry.getValue();
1024 String defaultSetting = policy.getDefaultOption();
1025 String setting = StringUtils.defaultString( settings.get( key ), defaultSetting );
1027 log.debug( "Applying [{}] policy with [{}]", key, setting );
1030 // all policies must approve the exception, any can cancel
1031 process = policy.applyPolicy( setting, request, localFile, exception, previousExceptions );
1037 catch ( PolicyConfigurationException e )
1039 log.error( e.getMessage(), e );
1045 // if the exception was queued, don't throw it
1046 if ( !previousExceptions.containsKey( content.getId() ) )
1048 throw new ProxyDownloadException(
1049 "An error occurred in downloading from the remote repository, and the policy is to fail immediately",
1050 content.getId(), exception );
1055 // if the exception was queued, but cancelled, remove it
1056 previousExceptions.remove( content.getId() );
1060 "Transfer error from repository {} for artifact {} , continuing to next repository. Error message: {}",
1061 content.getRepository().getId(), Keys.toKey( artifact ), exception.getMessage() );
1062 log.debug( "Full stack trace", exception );
1066 * Creates a working directory
1069 * @return file location of working directory
1071 private File createWorkingDirectory( ManagedRepositoryContent repository )
1075 return Files.createTempDirectory( "temp" ).toFile();
1077 catch ( IOException e )
1079 throw new RuntimeException( e.getMessage(), e );
1085 * Used to move the temporary file to its real destination. This is patterned from the way WagonManager handles its
1088 * @param temp The completed download file
1089 * @param target The final location of the downloaded file
1090 * @throws ProxyException when the temp file cannot replace the target file
1092 private void moveTempToTarget( File temp, File target )
1093 throws ProxyException
1096 // TODO file lock library
1100 lock = fileLockManager.writeFileLock( target );
1101 if ( lock.getFile().exists() && !lock.getFile().delete() )
1103 throw new ProxyException( "Unable to overwrite existing target file: " + target.getAbsolutePath() );
1106 lock.getFile().getParentFile().mkdirs();
1108 if ( !temp.renameTo( lock.getFile() ) )
1110 log.warn( "Unable to rename tmp file to its final name... resorting to copy command." );
1114 FileUtils.copyFile( temp, lock.getFile() );
1116 catch ( IOException e )
1118 if ( lock.getFile().exists() )
1120 log.debug( "Tried to copy file {} to {} but file with this name already exists.",
1121 temp.getName(), lock.getFile().getAbsolutePath() );
1125 throw new ProxyException(
1126 "Cannot copy tmp file " + temp.getAbsolutePath() + " to its final location", e );
1131 FileUtils.deleteQuietly( temp );
1135 catch ( FileLockException e )
1137 throw new ProxyException( e.getMessage(), e );
1139 catch ( FileLockTimeoutException e )
1141 throw new ProxyException( e.getMessage(), e );
1146 * Using wagon, connect to the remote repository.
1148 * @param connector the connector configuration to utilize (for obtaining network proxy configuration from)
1149 * @param wagon the wagon instance to establish the connection on.
1150 * @param remoteRepository the remote repository to connect to.
1151 * @return true if the connection was successful. false if not connected.
1153 private boolean connectToRepository( ProxyConnector connector, Wagon wagon,
1154 RemoteRepositoryContent remoteRepository )
1156 boolean connected = false;
1158 final ProxyInfo networkProxy =
1159 connector.getProxyId() == null ? null : this.networkProxyMap.get( connector.getProxyId() );
1161 if ( log.isDebugEnabled() )
1163 if ( networkProxy != null )
1165 // TODO: move to proxyInfo.toString()
1166 String msg = "Using network proxy " + networkProxy.getHost() + ":" + networkProxy.getPort()
1167 + " to connect to remote repository " + remoteRepository.getURL();
1168 if ( networkProxy.getNonProxyHosts() != null )
1170 msg += "; excluding hosts: " + networkProxy.getNonProxyHosts();
1172 if ( StringUtils.isNotBlank( networkProxy.getUserName() ) )
1174 msg += "; as user: " + networkProxy.getUserName();
1180 AuthenticationInfo authInfo = null;
1181 String username = remoteRepository.getRepository().getUserName();
1182 String password = remoteRepository.getRepository().getPassword();
1184 if ( StringUtils.isNotBlank( username ) && StringUtils.isNotBlank( password ) )
1186 log.debug( "Using username {} to connect to remote repository {}", username, remoteRepository.getURL() );
1187 authInfo = new AuthenticationInfo();
1188 authInfo.setUserName( username );
1189 authInfo.setPassword( password );
1192 // Convert seconds to milliseconds
1193 int timeoutInMilliseconds = remoteRepository.getRepository().getTimeout() * 1000;
1195 // Set timeout read and connect
1196 // FIXME olamy having 2 config values
1197 wagon.setReadTimeout( timeoutInMilliseconds );
1198 wagon.setTimeout( timeoutInMilliseconds );
1202 Repository wagonRepository =
1203 new Repository( remoteRepository.getId(), remoteRepository.getURL().toString() );
1204 wagon.connect( wagonRepository, authInfo, networkProxy );
1207 catch ( ConnectionException | AuthenticationException e )
1209 log.warn( "Could not connect to {}: {}", remoteRepository.getRepository().getName(), e.getMessage() );
1217 * Tests whitelist and blacklist patterns against path.
1219 * @param path the path to test.
1220 * @param patterns the list of patterns to check.
1221 * @return true if the path matches at least 1 pattern in the provided patterns list.
1223 private boolean matchesPattern( String path, List<String> patterns )
1225 if ( CollectionUtils.isEmpty( patterns ) )
1230 if ( !path.startsWith( "/" ) )
1235 for ( String pattern : patterns )
1237 if ( !pattern.startsWith( "/" ) )
1239 pattern = "/" + pattern;
1242 if ( SelectorUtils.matchPath( pattern, path, false ) )
1252 * TODO: Ensure that list is correctly ordered based on configuration. See MRM-477
1255 public List<ProxyConnector> getProxyConnectors( ManagedRepositoryContent repository )
1258 if ( !this.proxyConnectorMap.containsKey( repository.getId() ) )
1260 return Collections.emptyList();
1262 List<ProxyConnector> ret = new ArrayList<>( this.proxyConnectorMap.get( repository.getId() ) );
1264 Collections.sort( ret, ProxyConnectorOrderComparator.getInstance() );
1270 public void afterConfigurationChange( Registry registry, String propertyName, Object propertyValue )
1272 if ( ConfigurationNames.isNetworkProxy( propertyName ) || ConfigurationNames.isManagedRepositories(
1273 propertyName ) || ConfigurationNames.isRemoteRepositories( propertyName )
1274 || ConfigurationNames.isProxyConnector( propertyName ) )
1276 initConnectorsAndNetworkProxies();
1280 protected String addParameters( String path, RemoteRepository remoteRepository )
1282 if ( remoteRepository.getExtraParameters().isEmpty() )
1287 boolean question = false;
1289 StringBuilder res = new StringBuilder( path == null ? "" : path );
1291 for ( Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() )
1295 res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() );
1299 return res.toString();
1304 public void beforeConfigurationChange( Registry registry, String propertyName, Object propertyValue )
1309 public ArchivaConfiguration getArchivaConfiguration()
1311 return archivaConfiguration;
1314 public void setArchivaConfiguration( ArchivaConfiguration archivaConfiguration )
1316 this.archivaConfiguration = archivaConfiguration;
1319 public RepositoryContentFactory getRepositoryFactory()
1321 return repositoryFactory;
1324 public void setRepositoryFactory( RepositoryContentFactory repositoryFactory )
1326 this.repositoryFactory = repositoryFactory;
1329 public MetadataTools getMetadataTools()
1331 return metadataTools;
1334 public void setMetadataTools( MetadataTools metadataTools )
1336 this.metadataTools = metadataTools;
1339 public UrlFailureCache getUrlFailureCache()
1341 return urlFailureCache;
1344 public void setUrlFailureCache( UrlFailureCache urlFailureCache )
1346 this.urlFailureCache = urlFailureCache;
1349 public WagonFactory getWagonFactory()
1351 return wagonFactory;
1354 public void setWagonFactory( WagonFactory wagonFactory )
1356 this.wagonFactory = wagonFactory;
1359 public Map<String, PreDownloadPolicy> getPreDownloadPolicies()
1361 return preDownloadPolicies;
1364 public void setPreDownloadPolicies( Map<String, PreDownloadPolicy> preDownloadPolicies )
1366 this.preDownloadPolicies = preDownloadPolicies;
1369 public Map<String, PostDownloadPolicy> getPostDownloadPolicies()
1371 return postDownloadPolicies;
1374 public void setPostDownloadPolicies( Map<String, PostDownloadPolicy> postDownloadPolicies )
1376 this.postDownloadPolicies = postDownloadPolicies;
1379 public Map<String, DownloadErrorPolicy> getDownloadErrorPolicies()
1381 return downloadErrorPolicies;
1384 public void setDownloadErrorPolicies( Map<String, DownloadErrorPolicy> downloadErrorPolicies )
1386 this.downloadErrorPolicies = downloadErrorPolicies;