archiva/archiva-modules/archiva-base/archiva-proxy/src/main/java/org/apache/archiva/proxy/base/DefaultRepositoryProxyHandler.java

package org.apache.archiva.proxy.base;

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

import org.apache.archiva.checksum.ChecksumAlgorithm;
import org.apache.archiva.checksum.ChecksumUtil;
import org.apache.archiva.common.filelock.FileLockManager;
import org.apache.archiva.common.utils.PathUtil;
import org.apache.archiva.components.taskqueue.TaskQueueException;
import org.apache.archiva.configuration.provider.ArchivaConfiguration;
import org.apache.archiva.configuration.model.ProxyConnectorConfiguration;
import org.apache.archiva.configuration.model.ProxyConnectorRuleConfiguration;
import org.apache.archiva.policies.DownloadErrorPolicy;
import org.apache.archiva.policies.DownloadPolicy;
import org.apache.archiva.policies.Policy;
import org.apache.archiva.policies.PolicyConfigurationException;
import org.apache.archiva.policies.PolicyOption;
import org.apache.archiva.policies.PolicyViolationException;
import org.apache.archiva.policies.PostDownloadPolicy;
import org.apache.archiva.policies.PreDownloadPolicy;
import org.apache.archiva.policies.ProxyDownloadException;
import org.apache.archiva.policies.urlcache.UrlFailureCache;
import org.apache.archiva.proxy.model.NetworkProxy;
import org.apache.archiva.proxy.model.ProxyConnector;
import org.apache.archiva.proxy.model.ProxyFetchResult;
import org.apache.archiva.proxy.model.RepositoryProxyHandler;
import org.apache.archiva.repository.ManagedRepository;
import org.apache.archiva.repository.RemoteRepository;
import org.apache.archiva.repository.RemoteRepositoryContent;
import org.apache.archiva.repository.RepositoryType;
import org.apache.archiva.repository.content.Artifact;
import org.apache.archiva.repository.content.ContentItem;
import org.apache.archiva.repository.content.ItemSelector;
import org.apache.archiva.repository.metadata.RepositoryMetadataException;
import org.apache.archiva.repository.metadata.base.MetadataTools;
import org.apache.archiva.repository.storage.StorageAsset;
import org.apache.archiva.repository.storage.fs.FilesystemStorage;
import org.apache.archiva.repository.storage.fs.FsStorageUtil;
import org.apache.archiva.scheduler.ArchivaTaskScheduler;
import org.apache.archiva.scheduler.repository.model.RepositoryTask;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MarkerFactory;

import javax.annotation.PostConstruct;
import javax.inject.Inject;
import javax.inject.Named;
import java.io.IOException;
import java.net.MalformedURLException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;

public abstract class DefaultRepositoryProxyHandler implements RepositoryProxyHandler {

    protected Logger log = LoggerFactory.getLogger( DefaultRepositoryProxyHandler.class );
    @Inject
    protected UrlFailureCache urlFailureCache;

    @Inject
    @Named(value = "metadataTools#default")
    private MetadataTools metadataTools;

    private Map<String, PreDownloadPolicy> preDownloadPolicies = new HashMap<>(  );
    private Map<String, PostDownloadPolicy> postDownloadPolicies = new HashMap<>(  );
    private Map<String, DownloadErrorPolicy> downloadErrorPolicies = new HashMap<>(  );
    private ConcurrentMap<String, List<ProxyConnector>> proxyConnectorMap = new ConcurrentHashMap<>();

    @Inject
    @Named(value = "archivaTaskScheduler#repository")
    private ArchivaTaskScheduler<RepositoryTask> scheduler;

    @Inject
    private ArchivaConfiguration archivaConfiguration;

    @Inject
    @Named(value = "fileLockManager#default")
    private FileLockManager fileLockManager;

    private Map<String, NetworkProxy> networkProxyMap = new ConcurrentHashMap<>();
    private List<ChecksumAlgorithm> checksumAlgorithms;

    @PostConstruct
    public void initialize()
    {
        checksumAlgorithms = ChecksumUtil.getAlgorithms(archivaConfiguration.getConfiguration().getArchivaRuntimeConfiguration().getChecksumTypes());
    }

    private List<ProxyConnectorRuleConfiguration> findProxyConnectorRules(String sourceRepository,
                                                                          String targetRepository,
                                                                          List<ProxyConnectorRuleConfiguration> all )
    {
        List<ProxyConnectorRuleConfiguration> proxyConnectorRuleConfigurations = new ArrayList<>();

        for ( ProxyConnectorRuleConfiguration proxyConnectorRuleConfiguration : all )
        {
            for ( ProxyConnectorConfiguration proxyConnector : proxyConnectorRuleConfiguration.getProxyConnectors() )
            {
                if ( StringUtils.equals( sourceRepository, proxyConnector.getSourceRepoId() ) && StringUtils.equals(
                    targetRepository, proxyConnector.getTargetRepoId() ) )
                {
                    proxyConnectorRuleConfigurations.add( proxyConnectorRuleConfiguration );
                }
            }
        }

        return proxyConnectorRuleConfigurations;
    }

    @Override
    public StorageAsset fetchFromProxies( ManagedRepository repository, Artifact artifact )
        throws ProxyDownloadException
    {
        Map<String, Exception> previousExceptions = new LinkedHashMap<>();
        StorageAsset localFile = artifact.getAsset( );

        Properties requestProperties = new Properties();
        requestProperties.setProperty( "filetype", "artifact" );
        requestProperties.setProperty( "version", artifact.getVersion().getId() );
        requestProperties.setProperty( "managedRepositoryId", repository.getId() );

        List<ProxyConnector> connectors = getProxyConnectors( repository );
        for ( ProxyConnector connector : connectors )
        {
            if ( !connector.isEnabled() )
            {
                continue;
            }

            RemoteRepository targetRepository = connector.getTargetRepository();
            requestProperties.setProperty( "remoteRepositoryId", targetRepository.getId() );

            StorageAsset targetFile = targetRepository.getAsset( localFile.getPath( ) );
            // Removing the leading '/' from the path
            String targetPath = targetFile.getPath( ).substring( 1 );
            try
            {
                StorageAsset downloadedFile =
                    transferFile( connector, targetRepository, targetPath, repository, localFile, requestProperties,
                        true );

                if ( fileExists(downloadedFile) )
                {
                    log.debug( "Successfully transferred: {}", downloadedFile.getPath() );
                    return downloadedFile;
                }
            }
            catch ( NotFoundException e )
            {
                log.debug( "Artifact {} not found on repository \"{}\".", artifact.getId(),
                    targetRepository.getId() );
            }
            catch ( NotModifiedException e )
            {
                log.debug( "Artifact {} not updated on repository \"{}\".", artifact.getId(),
                    targetRepository.getId() );
            }
            catch ( ProxyException e )
            {
                validatePolicies( this.downloadErrorPolicies, connector.getPolicies(), requestProperties, artifact,
                    targetRepository.getContent(), localFile, e, previousExceptions );
            }
        }

        if ( !previousExceptions.isEmpty() )
        {
            throw new ProxyDownloadException( "Failures occurred downloading from some remote repositories",
                previousExceptions );
        }

        log.debug( "Exhausted all target repositories, artifact {} not found.", artifact.getId() );

        return null;
    }

    @Override
    public StorageAsset fetchFromProxies( ManagedRepository repository, ItemSelector artifactSelector )
        throws ProxyDownloadException
    {
        Map<String, Exception> previousExceptions = new LinkedHashMap<>();
        ContentItem item = repository.getContent( ).getItem( artifactSelector );
        StorageAsset localFile = item.getAsset( );

        Properties requestProperties = new Properties();
        requestProperties.setProperty( "filetype", "artifact" );
        requestProperties.setProperty( "version", artifactSelector.getVersion() );
        requestProperties.setProperty( "managedRepositoryId", repository.getId() );

        List<ProxyConnector> connectors = getProxyConnectors( repository );
        for ( ProxyConnector connector : connectors )
        {
            if ( !connector.isEnabled() )
            {
                continue;
            }

            RemoteRepository targetRepository = connector.getTargetRepository();
            requestProperties.setProperty( "remoteRepositoryId", targetRepository.getId() );

            StorageAsset targetFile = targetRepository.getAsset( localFile.getPath( ) );
            // Removing the leading '/' from the path
            String targetPath = targetFile.getPath( ).substring( 1 );
            try
            {
                StorageAsset downloadedFile =
                    transferFile( connector, targetRepository, targetPath, repository, localFile, requestProperties,
                        true );

                if ( fileExists(downloadedFile) )
                {
                    log.debug( "Successfully transferred: {}", downloadedFile.getPath() );
                    return downloadedFile;
                }
            }
            catch ( NotFoundException e )
            {
                log.debug( "Artifact {} not found on repository \"{}\".", item,
                    targetRepository.getId() );
            }
            catch ( NotModifiedException e )
            {
                log.debug( "Artifact {} not updated on repository \"{}\".", item,
                    targetRepository.getId() );
            }
            catch ( ProxyException e )
            {
                validatePolicies( this.downloadErrorPolicies, connector.getPolicies(), requestProperties, item,
                    targetRepository.getContent(), localFile, e, previousExceptions );
            }
        }

        if ( !previousExceptions.isEmpty() )
        {
            throw new ProxyDownloadException( "Failures occurred downloading from some remote repositories",
                previousExceptions );
        }

        log.debug( "Exhausted all target repositories, artifact {} not found.", item );

        return null;
    }

    @Override
    public StorageAsset fetchFromProxies( ManagedRepository repository, String path )
    {
        StorageAsset localFile = repository.getAsset( path );

        // no update policies for these paths
        if ( localFile.exists() )
        {
            return null;
        }

        Properties requestProperties = new Properties();
        requestProperties.setProperty( "filetype", "resource" );
        requestProperties.setProperty( "managedRepositoryId", repository.getId() );

        List<ProxyConnector> connectors = getProxyConnectors( repository );
        for ( ProxyConnector connector : connectors )
        {
            if ( !connector.isEnabled() )
            {
                continue;
            }

            RemoteRepository targetRepository = connector.getTargetRepository();
            requestProperties.setProperty( "remoteRepositoryId", targetRepository.getId() );

            String targetPath = path;

            try
            {
                StorageAsset downloadedFile =
                    transferFile( connector, targetRepository, targetPath, repository, localFile, requestProperties,
                                  false );

                if ( fileExists( downloadedFile ) )
                {
                    log.debug( "Successfully transferred: {}", downloadedFile.getPath() );
                    return downloadedFile;
                }
            }
            catch ( NotFoundException e )
            {
                log.debug( "Resource {} not found on repository \"{}\".", path,
                           targetRepository.getId() );
            }
            catch ( NotModifiedException e )
            {
                log.debug( "Resource {} not updated on repository \"{}\".", path,
                           targetRepository.getId() );
            }
            catch ( ProxyException e )
            {
                log.warn(
                    "Transfer error from repository {} for resource {}, continuing to next repository. Error message: {}",
                    targetRepository.getId(), path, e.getMessage() );
                log.debug( MarkerFactory.getDetachedMarker( "transfer.error" ),
                           "Transfer error from repository \"{}"
                               + "\" for resource {}, continuing to next repository. Error message: {}",
                           targetRepository.getId(), path, e.getMessage(), e );
            }

        }

        log.debug( "Exhausted all target repositories, resource {} not found.", path );

        return null;
    }

    @Override
    public ProxyFetchResult fetchMetadataFromProxies( ManagedRepository repository, String rawLogicalPath )
    {
        String logicalPath;
        if (rawLogicalPath.startsWith( "/" )){
            logicalPath = rawLogicalPath.substring( 1 );
        } else {
            logicalPath = rawLogicalPath;
        }
        StorageAsset localFile = repository.getAsset( logicalPath );

        Properties requestProperties = new Properties();
        requestProperties.setProperty( "filetype", "metadata" );
        boolean metadataNeedsUpdating = false;
        long originalTimestamp = getLastModified( localFile );

        List<ProxyConnector> connectors = new ArrayList<>( getProxyConnectors( repository ) );
        for ( ProxyConnector connector : connectors )
        {
            if ( !connector.isEnabled() )
            {
                continue;
            }

            RemoteRepository targetRepository = connector.getTargetRepository();

            StorageAsset localRepoFile = toLocalRepoFile( repository, targetRepository.getContent(), logicalPath );
            long originalMetadataTimestamp = getLastModified( localRepoFile );

            try
            {
                transferFile( connector, targetRepository, logicalPath, repository, localRepoFile, requestProperties,
                              true );

                if ( hasBeenUpdated( localRepoFile, originalMetadataTimestamp ) )
                {
                    metadataNeedsUpdating = true;
                }
            }
            catch ( NotFoundException e )
            {

                log.debug( "Metadata {} not found on remote repository '{}'.", logicalPath,
                           targetRepository.getId(), e );

            }
            catch ( NotModifiedException e )
            {

                log.debug( "Metadata {} not updated on remote repository '{}'.", logicalPath,
                           targetRepository.getId(), e );

            }
            catch ( ProxyException e )
            {
                log.warn(
                    "Transfer error from repository {} for versioned Metadata {}, continuing to next repository. Error message: {}",
                    targetRepository.getId(), logicalPath, e.getMessage() );
                log.debug( "Full stack trace", e );
            }
        }

        if ( hasBeenUpdated( localFile, originalTimestamp ) )
        {
            metadataNeedsUpdating = true;
        }

        if ( metadataNeedsUpdating || !localFile.exists())
        {
            try
            {
                metadataTools.updateMetadata( repository.getContent(), logicalPath );
            }
            catch ( RepositoryMetadataException e )
            {
                log.warn( "Unable to update metadata {}:{}", localFile.getPath(), e.getMessage(), e );
            }

        }

        if ( fileExists( localFile ) )
        {
            return new ProxyFetchResult( localFile, metadataNeedsUpdating );
        }

        return new ProxyFetchResult( null, false );
    }

    private long getLastModified(StorageAsset file )
    {
        if ( !file.exists() || file.isContainer() )
        {
            return 0;
        }

        return file.getModificationTime().toEpochMilli();
    }

    private boolean hasBeenUpdated(StorageAsset file, long originalLastModified )
    {
        if ( !file.exists() || file.isContainer() )
        {
            return false;
        }

        long currentLastModified = getLastModified( file );
        return ( currentLastModified > originalLastModified );
    }

    private StorageAsset toLocalRepoFile( ManagedRepository repository, RemoteRepositoryContent targetRepository,
                                          String targetPath )
    {
        String repoPath = metadataTools.getRepositorySpecificName( targetRepository, targetPath );
        return repository.getAsset( repoPath );
    }

    /**
     * Test if the provided ManagedRepositoryContent has any proxies configured for it.
     * @param repository
     */
    @Override
    public boolean hasProxies( ManagedRepository repository )
    {
        synchronized ( this.proxyConnectorMap )
        {
            return this.proxyConnectorMap.containsKey( repository.getId() );
        }
    }

    /**
     * Simple method to test if the file exists on the local disk.
     *
     * @param file the file to test. (may be null)
     * @return true if file exists. false if the file param is null, doesn't exist, or is not of type File.
     */
    private boolean fileExists( StorageAsset file )
    {
        if ( file == null )
        {
            return false;
        }

        if ( !file.exists())
        {
            return false;
        }

        return !file.isContainer();
    }

    /**
     * Perform the transfer of the file.
     *
     * @param connector         the connector configuration to use.
     * @param remoteRepository  the remote repository get the resource from.
     * @param remotePath        the path in the remote repository to the resource to get.
     * @param repository        the managed repository that will hold the file
     * @param resource          the path relative to the repository storage where the file should be downloaded to
     * @param requestProperties the request properties to utilize for policy handling.
     * @param executeConsumers  whether to execute the consumers after proxying
     * @return the local file that was downloaded, or null if not downloaded.
     * @throws NotFoundException    if the file was not found on the remote repository.
     * @throws NotModifiedException if the localFile was present, and the resource was present on remote repository, but
     *                              the remote resource is not newer than the local File.
     * @throws ProxyException       if transfer was unsuccessful.
     */
    protected StorageAsset transferFile( ProxyConnector connector, RemoteRepository remoteRepository, String remotePath,
                                         ManagedRepository repository, StorageAsset resource, Properties requestProperties,
                                         boolean executeConsumers )
        throws ProxyException, NotModifiedException
    {
        String url = null;
        try
        {
            url = remoteRepository.getLocation().toURL().toString();
        }
        catch ( MalformedURLException e )
        {
            throw new ProxyException( e.getMessage(), e );
        }
        if ( !url.endsWith( "/" ) )
        {
            url = url + "/";
        }
        if (remotePath.startsWith( "/" )) {
            url = url + remotePath.substring( 1 );
        } else {
            url = url + remotePath;
        }
        requestProperties.setProperty( "url", url );

        // Is a whitelist defined?
        if ( CollectionUtils.isNotEmpty( connector.getWhitelist() ) )
        {
            // Path must belong to whitelist.
            if ( !matchesPattern( remotePath, connector.getWhitelist() ) )
            {
                log.debug( "Path [{}] is not part of defined whitelist (skipping transfer from repository [{}]).",
                           remotePath, remoteRepository.getId() );
                return null;
            }
        }

        // Is target path part of blacklist?
        if ( matchesPattern( remotePath, connector.getBlacklist() ) )
        {
            log.debug( "Path [{}] is part of blacklist (skipping transfer from repository [{}]).", remotePath,
                       remoteRepository.getId() );
            return null;
        }

        // Handle pre-download policy
        try
        {
            validatePolicies( this.preDownloadPolicies, connector.getPolicies(), requestProperties, resource );
        }
        catch ( PolicyViolationException e )
        {
            String emsg = "Transfer not attempted on " + url + " : " + e.getMessage();
            if ( resource.exists() )
            {
                log.debug( "{} : using already present local file.", emsg );
                return resource;
            }

            log.debug( emsg );
            return null;
        }

        Path workingDirectory = createWorkingDirectory( repository );
        FilesystemStorage tmpStorage = null;
        try
        {
            tmpStorage = new FilesystemStorage( workingDirectory, fileLockManager );
        }
        catch ( IOException e )
        {
            throw new ProxyException( "Could not create tmp storage" );
        }
        StorageAsset tmpResource = tmpStorage.getAsset( resource.getName( ) );
        StorageAsset[] tmpChecksumFiles = new StorageAsset[checksumAlgorithms.size()];
        for(int i=0; i<checksumAlgorithms.size(); i++) {
            ChecksumAlgorithm alg = checksumAlgorithms.get( i );
            tmpChecksumFiles[i] = tmpStorage.getAsset( resource.getName() + "." + alg.getDefaultExtension() );
        }

        try
        {

            transferResources( connector, remoteRepository, tmpResource,tmpChecksumFiles , url, remotePath,
                resource, workingDirectory, repository );

            // Handle post-download policies.
            try
            {
                validatePolicies( this.postDownloadPolicies, connector.getPolicies(), requestProperties, tmpResource );
            }
            catch ( PolicyViolationException e )
            {
                log.warn( "Transfer invalidated from {} : {}", url, e.getMessage() );
                executeConsumers = false;
                if ( !fileExists( tmpResource ) )
                {
                    resource = null;
                }
            }

            if ( resource != null )
            {
                synchronized ( resource.getPath().intern() )
                {
                    StorageAsset directory = resource.getParent();
                    for (int i=0; i<tmpChecksumFiles.length; i++) {
                        moveFileIfExists( tmpChecksumFiles[i], directory );
                    }
                    moveFileIfExists( tmpResource, directory );
                }
            }
        }
        finally
        {
            org.apache.archiva.common.utils.FileUtils.deleteQuietly( workingDirectory );
        }

        if ( executeConsumers )
        {
            // Just-in-time update of the index and database by executing the consumers for this artifact
            //consumers.executeConsumers( connector.getSourceRepository().getRepository(), resource );
            queueRepositoryTask( connector.getSourceRepository().getId(), resource );
        }

        return resource;
    }

    protected abstract void transferResources( ProxyConnector connector, RemoteRepository remoteRepository,
                                               StorageAsset tmpResource, StorageAsset[] checksumFiles, String url, String remotePath, StorageAsset resource, Path workingDirectory,
                                               ManagedRepository repository ) throws ProxyException;

    private void queueRepositoryTask(String repositoryId, StorageAsset localFile )
    {
        RepositoryTask task = new RepositoryTask();
        task.setRepositoryId( repositoryId );
        task.setResourceFile( localFile );
        task.setUpdateRelatedArtifacts( true );
        task.setScanAll( true );

        try
        {
            scheduler.queueTask( task );
        }
        catch ( TaskQueueException e )
        {
            log.error( "Unable to queue repository task to execute consumers on resource file ['{}"
                           + "'].", localFile.getName() );
        }
    }

    /**
     * Moves the file into repository location if it exists
     *
     * @param fileToMove this could be either the main artifact, sha1 or md5 checksum file.
     * @param directory  directory to write files to
     */
    private void moveFileIfExists( StorageAsset fileToMove, StorageAsset directory )
        throws ProxyException
    {
        if ( fileToMove != null && fileToMove.exists() )
        {
            StorageAsset newLocation = directory.getStorage().getAsset( directory.getPath()+ "/" + fileToMove.getName());
            moveTempToTarget( fileToMove, newLocation );
        }
    }

    /**
     * Apply the policies.
     *
     * @param policies  the map of policies to execute. (Map of String policy keys, to {@link DownloadPolicy} objects)
     * @param settings  the map of settings for the policies to execute. (Map of String policy keys, to String policy
     *                  setting)
     * @param request   the request properties (utilized by the {@link DownloadPolicy#applyPolicy(PolicyOption, Properties, StorageAsset)}
     *                  )
     * @param localFile the local file (utilized by the {@link DownloadPolicy#applyPolicy(PolicyOption, Properties, StorageAsset)})
     * @throws PolicyViolationException
     */
    private void validatePolicies( Map<String, ? extends DownloadPolicy> policies, Map<Policy, PolicyOption> settings,
                                   Properties request, StorageAsset localFile )
        throws PolicyViolationException
    {
        for ( Map.Entry<String, ? extends DownloadPolicy> entry : policies.entrySet() )
        {
            // olamy with spring rolehint is now downloadPolicy#hint
            // so substring after last # to get the hint as with plexus
            String key = entry.getValue( ).getId( );
            DownloadPolicy policy = entry.getValue();
            PolicyOption option = settings.containsKey(policy ) ? settings.get(policy) : policy.getDefaultOption();

            log.debug( "Applying [{}] policy with [{}]", key, option );
            try
            {
                policy.applyPolicy( option, request, localFile );
            }
            catch ( PolicyConfigurationException e )
            {
                log.error( e.getMessage(), e );
            }
        }
    }

    private void validatePolicies( Map<String, DownloadErrorPolicy> policies, Map<Policy, PolicyOption> settings,
                                   Properties request, ContentItem artifact, RemoteRepositoryContent content,
                                   StorageAsset localFile, Exception exception, Map<String, Exception> previousExceptions )
        throws ProxyDownloadException
    {
        boolean process = true;
        for ( Map.Entry<String, ? extends DownloadErrorPolicy> entry : policies.entrySet() )
        {

            // olamy with spring rolehint is now downloadPolicy#hint
            // so substring after last # to get the hint as with plexus
            String key = entry.getValue( ).getId( );
            DownloadErrorPolicy policy = entry.getValue();
            PolicyOption option = settings.containsKey( policy ) ? settings.get(policy) : policy.getDefaultOption();

            log.debug( "Applying [{}] policy with [{}]", key, option );
            try
            {
                // all policies must approve the exception, any can cancel
                process = policy.applyPolicy( option, request, localFile, exception, previousExceptions );
                if ( !process )
                {
                    break;
                }
            }
            catch ( PolicyConfigurationException e )
            {
                log.error( e.getMessage(), e );
            }
        }

        if ( process )
        {
            // if the exception was queued, don't throw it
            if ( !previousExceptions.containsKey( content.getId() ) )
            {
                throw new ProxyDownloadException(
                    "An error occurred in downloading from the remote repository, and the policy is to fail immediately",
                    content.getId(), exception );
            }
        }
        else
        {
            // if the exception was queued, but cancelled, remove it
            previousExceptions.remove( content.getId() );
        }

        log.warn(
            "Transfer error from repository {} for artifact {} , continuing to next repository. Error message: {}",
            content.getRepository().getId(), artifact, exception.getMessage() );
        log.debug( "Full stack trace", exception );
    }

    /**
     * Creates a working directory
     *
     * @param repository
     * @return file location of working directory
     */
    private Path createWorkingDirectory( ManagedRepository repository )
    {
        try
        {
            return Files.createTempDirectory( "temp" );
        }
        catch ( IOException e )
        {
            throw new RuntimeException( e.getMessage(), e );
        }

    }

    /**
     * Used to move the temporary file to its real destination. This is patterned from the way WagonManager handles its
     * downloaded files.
     *
     * @param temp   The completed download file
     * @param target The final location of the downloaded file
     * @throws ProxyException when the temp file cannot replace the target file
     */
    private void moveTempToTarget( StorageAsset temp, StorageAsset target )
        throws ProxyException
    {

        try
        {
            org.apache.archiva.repository.storage.util.StorageUtil.moveAsset( temp, target, true , StandardCopyOption.REPLACE_EXISTING);
        }
        catch ( IOException e )
        {
            log.error( "Move failed from {} to {}, trying copy.", temp, target );
            try
            {
                FsStorageUtil.copyAsset( temp, target, true );
                if (temp.exists()) {
                    temp.getStorage( ).removeAsset( temp );
                }
            }
            catch ( IOException ex )
            {
                log.error("Copy failed from {} to {}: ({}) {}", temp, target, e.getClass(), e.getMessage());
                throw new ProxyException("Could not move temp file "+temp.getPath()+" to target "+target.getPath()+": ("+e.getClass()+") "+e.getMessage(), e);
            }
        }
    }

    /**
     * Tests whitelist and blacklist patterns against path.
     *
     * @param path     the path to test.
     * @param patterns the list of patterns to check.
     * @return true if the path matches at least 1 pattern in the provided patterns list.
     */
    private boolean matchesPattern( String path, List<String> patterns )
    {
        if ( CollectionUtils.isEmpty( patterns ) )
        {
            return false;
        }

        if ( !path.startsWith( "/" ) )
        {
            path = "/" + path;
        }

        for ( String pattern : patterns )
        {
            if ( !pattern.startsWith( "/" ) )
            {
                pattern = "/" + pattern;
            }

            if ( PathUtil.matchPath( pattern, path, false ) )
            {
                return true;
            }
        }

        return false;
    }

    /**
     * TODO: Ensure that list is correctly ordered based on configuration. See MRM-477
     * @param repository
     */
    @Override
    public List<ProxyConnector> getProxyConnectors( ManagedRepository repository )
    {

        if ( !this.proxyConnectorMap.containsKey( repository.getId() ) )
        {
            return Collections.emptyList();
        }
        List<ProxyConnector> ret = new ArrayList<>( this.proxyConnectorMap.get( repository.getId() ) );

        Collections.sort( ret, ProxyConnectorOrderComparator.getInstance() );
        return ret;

    }


    protected String addParameters(String path, RemoteRepository remoteRepository )
    {
        if ( remoteRepository.getExtraParameters().isEmpty() )
        {
            return path;
        }

        boolean question = false;

        StringBuilder res = new StringBuilder( path == null ? "" : path );

        for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() )
        {
            if ( !question )
            {
                res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() );
            }
        }

        return res.toString();
    }

    public void setArchivaConfiguration(ArchivaConfiguration archivaConfiguration )
    {
        this.archivaConfiguration = archivaConfiguration;
    }

    public MetadataTools getMetadataTools()
    {
        return metadataTools;
    }

    public void setMetadataTools(MetadataTools metadataTools )
    {
        this.metadataTools = metadataTools;
    }

    public UrlFailureCache getUrlFailureCache()
    {
        return urlFailureCache;
    }

    public void setUrlFailureCache(UrlFailureCache urlFailureCache )
    {
        this.urlFailureCache = urlFailureCache;
    }

    public Map<String, PreDownloadPolicy> getPreDownloadPolicies()
    {
        return preDownloadPolicies;
    }

    public void setPreDownloadPolicies(Map<String, PreDownloadPolicy> preDownloadPolicies )
    {
        this.preDownloadPolicies = preDownloadPolicies;
    }

    public Map<String, PostDownloadPolicy> getPostDownloadPolicies()
    {
        return postDownloadPolicies;
    }

    public void setPostDownloadPolicies(Map<String, PostDownloadPolicy> postDownloadPolicies )
    {
        this.postDownloadPolicies = postDownloadPolicies;
    }

    public Map<String, DownloadErrorPolicy> getDownloadErrorPolicies()
    {
        return downloadErrorPolicies;
    }

    public void setDownloadErrorPolicies(Map<String, DownloadErrorPolicy> downloadErrorPolicies )
    {
        this.downloadErrorPolicies = downloadErrorPolicies;
    }

    @Override
    public void setNetworkProxies(Map<String, NetworkProxy> networkProxies ) {
        this.networkProxyMap.clear();
        this.networkProxyMap.putAll( networkProxies );
    }

    @Override
    public NetworkProxy getNetworkProxy(String id) {
        return this.networkProxyMap.get(id);
    }

    @Override
    public Map<String, NetworkProxy> getNetworkProxies() {
        return this.networkProxyMap;
    }

    @Override
    public abstract List<RepositoryType> supports();

    @Override
    public void setPolicies( List<Policy> policyList )
    {
        preDownloadPolicies.clear();
        postDownloadPolicies.clear();
        downloadErrorPolicies.clear();
        for (Policy policy : policyList) {
            addPolicy( policy );
        }
    }

    void addPolicy(PreDownloadPolicy policy) {
        preDownloadPolicies.put( policy.getId( ), policy );
    }

    void addPolicy(PostDownloadPolicy policy) {
        postDownloadPolicies.put( policy.getId( ), policy );
    }
    void addPolicy(DownloadErrorPolicy policy) {
        downloadErrorPolicies.put( policy.getId( ), policy );
    }

    @Override
    public void addPolicy( Policy policy )
    {
        if (policy instanceof PreDownloadPolicy) {
            addPolicy( (PreDownloadPolicy)policy );
        } else if (policy instanceof PostDownloadPolicy) {
            addPolicy( (PostDownloadPolicy) policy );
        } else if (policy instanceof DownloadErrorPolicy) {
            addPolicy( (DownloadErrorPolicy) policy );
        } else {
            log.warn( "Policy not known: {}, {}", policy.getId( ), policy.getClass( ).getName( ) );
        }
    }

    @Override
    public void removePolicy( Policy policy )
    {
        final String id = policy.getId();
        if (preDownloadPolicies.containsKey( id )) {
            preDownloadPolicies.remove( id );
        } else if (postDownloadPolicies.containsKey( id )) {
            postDownloadPolicies.remove( id );
        } else if (downloadErrorPolicies.containsKey( id )) {
            downloadErrorPolicies.remove( id );
        }
    }

    @Override
    public void addProxyConnector( ProxyConnector connector )
    {
        final String sourceId = connector.getSourceRepository( ).getId( );
        List<ProxyConnector> connectors;
        if (proxyConnectorMap.containsKey( sourceId )) {
            connectors = proxyConnectorMap.get( sourceId );
        } else {
            connectors = new ArrayList<>( );
            proxyConnectorMap.put( sourceId, connectors );
        }
        connectors.add( connector );
    }

    @Override
    public void setProxyConnectors( List<ProxyConnector> proxyConnectors )
    {
        proxyConnectorMap.clear();
        for ( ProxyConnector connector : proxyConnectors )
        {
            addProxyConnector( connector );
        }
    }
}