back to use wagon for remote download index to prevent unit test failure

git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@1541818 13f79535-47bb-0310-9956-ffa450edef68

archiva-modules/archiva-scheduler/archiva-scheduler-indexing/pom.xml

@@ -204,19 +204,14 @@
               org.springframework*;version="[3,4)",
               org.apache.archiva.redback.components.taskqueue*,
               org.apache.maven.index*,
-              org.apache.http,
-              org.apache.http.auth,
-              org.apache.http.client,
-              org.apache.http.client.methods,
-              org.apache.http.concurrent,
-              org.apache.http.entity,
-              org.apache.http.impl.client,
-              org.apache.http.impl.nio.client,
-              org.apache.http.nio,
-              org.apache.http.nio.client.methods,
-              org.apache.http.nio.protocol,
-              org.apache.http.protocol,
-              org.apache.http.impl.nio.codecs,
+              org.apache.maven.wagon,
+              org.apache.maven.wagon.authentication,
+              org.apache.maven.wagon.authorization,
+              org.apache.maven.wagon.events,
+              org.apache.maven.wagon.providers.http,
+              org.apache.maven.wagon.proxy,
+              org.apache.maven.wagon.repository,
+              org.apache.maven.wagon.resource,
               org.slf4j;resolution:=optional
             </Import-Package>
           </instructions>

archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DownloadRemoteIndexTask.java

@@ -23,34 +23,29 @@ import org.apache.archiva.admin.model.beans.NetworkProxy;
 import org.apache.archiva.admin.model.beans.RemoteRepository;
 import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin;
 import org.apache.archiva.proxy.common.WagonFactory;
+import org.apache.archiva.proxy.common.WagonFactoryException;
+import org.apache.archiva.proxy.common.WagonFactoryRequest;
 import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang.time.StopWatch;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpException;
-import org.apache.http.HttpHost;
-import org.apache.http.HttpRequest;
-import org.apache.http.HttpResponse;
-import org.apache.http.HttpStatus;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.ClientProtocolException;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.entity.ContentType;
-import org.apache.http.impl.client.BasicCredentialsProvider;
-import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
-import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
-import org.apache.http.impl.nio.codecs.LengthDelimitedDecoder;
-import org.apache.http.nio.ContentDecoder;
-import org.apache.http.nio.ContentEncoder;
-import org.apache.http.nio.IOControl;
-import org.apache.http.nio.client.methods.ZeroCopyConsumer;
-import org.apache.http.nio.protocol.HttpAsyncRequestProducer;
-import org.apache.http.protocol.HttpContext;
 import org.apache.maven.index.context.IndexingContext;
 import org.apache.maven.index.updater.IndexUpdateRequest;
 import org.apache.maven.index.updater.IndexUpdater;
 import org.apache.maven.index.updater.ResourceFetcher;
+import org.apache.maven.wagon.ConnectionException;
+import org.apache.maven.wagon.ResourceDoesNotExistException;
+import org.apache.maven.wagon.StreamWagon;
+import org.apache.maven.wagon.TransferFailedException;
+import org.apache.maven.wagon.Wagon;
+import org.apache.maven.wagon.authentication.AuthenticationException;
+import org.apache.maven.wagon.authentication.AuthenticationInfo;
+import org.apache.maven.wagon.authorization.AuthorizationException;
+import org.apache.maven.wagon.events.TransferEvent;
+import org.apache.maven.wagon.events.TransferListener;
+import org.apache.maven.wagon.providers.http.AbstractHttpClientWagon;
+import org.apache.maven.wagon.providers.http.HttpConfiguration;
+import org.apache.maven.wagon.providers.http.HttpMethodConfiguration;
+import org.apache.maven.wagon.proxy.ProxyInfo;
+import org.apache.maven.wagon.repository.Repository;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -59,15 +54,10 @@ import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
-import java.lang.reflect.Field;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
 
 /**
  * @author Olivier Lamy
@@ -140,9 +130,6 @@ public class DownloadRemoteIndexTask
             tempIndexDirectory.deleteOnExit();
             String baseIndexUrl = indexingContext.getIndexUpdateUrl();
 
-            URL indexUrl = new URL( baseIndexUrl );
-
-            /*
             String wagonProtocol = new URL( this.remoteRepository.getUrl() ).getProtocol();
 
             final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon(
@@ -182,38 +169,6 @@ public class DownloadRemoteIndexTask
             }
             wagon.connect( new Repository( this.remoteRepository.getId(), baseIndexUrl ), authenticationInfo,
                            proxyInfo );
-            */
-            //---------------------------------------------
-
-            HttpAsyncClientBuilder builder = HttpAsyncClientBuilder.create();
-
-            BasicCredentialsProvider basicCredentialsProvider = new BasicCredentialsProvider();
-
-            if ( this.networkProxy != null )
-            {
-                HttpHost httpHost = new HttpHost( this.networkProxy.getHost(), this.networkProxy.getPort() );
-                builder = builder.setProxy( httpHost );
-
-                if ( this.networkProxy.getUsername() != null )
-                {
-                    basicCredentialsProvider.setCredentials(
-                        new AuthScope( this.networkProxy.getHost(), this.networkProxy.getPort(), null, null ),
-                        new UsernamePasswordCredentials( this.networkProxy.getUsername(),
-                                                         this.networkProxy.getPassword() ) );
-                }
-
-            }
-
-            if ( this.remoteRepository.getUserName() != null )
-            {
-                basicCredentialsProvider.setCredentials(
-                    new AuthScope( indexUrl.getHost(), indexUrl.getPort(), null, null ),
-                    new UsernamePasswordCredentials( this.remoteRepository.getUserName(),
-                                                     this.remoteRepository.getPassword() ) );
-
-            }
-
-            builder = builder.setDefaultCredentialsProvider( basicCredentialsProvider );
 
             File indexDirectory = indexingContext.getIndexDirectoryFile();
             if ( !indexDirectory.exists() )
@@ -221,36 +176,44 @@ public class DownloadRemoteIndexTask
                 indexDirectory.mkdirs();
             }
 
-            CloseableHttpAsyncClient closeableHttpAsyncClient = builder.build();
-            closeableHttpAsyncClient.start();
             ResourceFetcher resourceFetcher =
-                new ZeroCopyResourceFetcher( log, tempIndexDirectory, remoteRepository, closeableHttpAsyncClient,
-                                             baseIndexUrl );
-
+                new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository );
             IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
             request.setForceFullUpdate( this.fullDownload );
             request.setLocalIndexCacheDir( indexCacheDirectory );
 
             this.indexUpdater.fetchAndUpdateIndex( request );
+            stopWatch.stop();
+            log.info( "time update index from remote for repository {}: {} s", this.remoteRepository.getId(),
+                      ( stopWatch.getTime() / 1000 ) );
 
             // index packing optionnal ??
             //IndexPackingRequest indexPackingRequest =
             //    new IndexPackingRequest( indexingContext, indexingContext.getIndexDirectoryFile() );
             //indexPacker.packIndex( indexPackingRequest );
-
             indexingContext.updateTimestamp( true );
 
-            stopWatch.stop();
-            log.info( "time update index from remote for repository {}: {} s", this.remoteRepository.getId(),
-                      ( stopWatch.getTime() / 1000 ) );
-
-
         }
         catch ( MalformedURLException e )
         {
             log.error( e.getMessage(), e );
             throw new RuntimeException( e.getMessage(), e );
         }
+        catch ( WagonFactoryException e )
+        {
+            log.error( e.getMessage(), e );
+            throw new RuntimeException( e.getMessage(), e );
+        }
+        catch ( ConnectionException e )
+        {
+            log.error( e.getMessage(), e );
+            throw new RuntimeException( e.getMessage(), e );
+        }
+        catch ( AuthenticationException e )
+        {
+            log.error( e.getMessage(), e );
+            throw new RuntimeException( e.getMessage(), e );
+        }
         catch ( IOException e )
         {
             log.error( e.getMessage(), e );
@@ -266,7 +229,7 @@ public class DownloadRemoteIndexTask
             deleteDirectoryQuiet( tempIndexDirectory );
             this.runningRemoteDownloadIds.remove( this.remoteRepository.getId() );
         }
-        log.info( "end download remote index for remote repository {}", this.remoteRepository.getId() );
+        log.info( "end download remote index for remote repository " + this.remoteRepository.getId() );
     }
 
     private void deleteDirectoryQuiet( File f )
@@ -281,8 +244,9 @@ public class DownloadRemoteIndexTask
         }
     }
 
-    private static class ZeroCopyConsumerListener
-        extends ZeroCopyConsumer
+
+    private static final class DownloadListener
+        implements TransferListener
     {
         private Logger log = LoggerFactory.getLogger( getClass() );
 
@@ -290,87 +254,49 @@ public class DownloadRemoteIndexTask
 
         private long startTime;
 
-        private long totalLength = 0;
-
-        //private long currentLength = 0;
+        private int totalLength = 0;
 
-        private ZeroCopyConsumerListener( File file, String resourceName )
-            throws FileNotFoundException
+        public void transferInitiated( TransferEvent transferEvent )
         {
-            super( file );
-            this.resourceName = resourceName;
+            startTime = System.currentTimeMillis();
+            resourceName = transferEvent.getResource().getName();
+            log.debug( "initiate transfer of {}", resourceName );
         }
 
-        @Override
-        protected File process( final HttpResponse response, final File file, final ContentType contentType )
-            throws Exception
+        public void transferStarted( TransferEvent transferEvent )
         {
-            if ( response.getStatusLine().getStatusCode() != HttpStatus.SC_OK )
-            {
-                throw new ClientProtocolException( "Download failed: " + response.getStatusLine() );
-            }
-            long endTime = System.currentTimeMillis();
-            log.info( "end of transfer file {} {} kb: {}s", resourceName, this.totalLength / 1024,
-                      ( endTime - startTime ) / 1000 );
-            return file;
+            this.totalLength = 0;
+            resourceName = transferEvent.getResource().getName();
+            log.info( "start transfer of {}", transferEvent.getResource().getName() );
         }
 
-        @Override
-        protected void onContentReceived( ContentDecoder decoder, IOControl ioControl )
-            throws IOException
+        public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
         {
-            if ( decoder instanceof LengthDelimitedDecoder )
-            {
-                LengthDelimitedDecoder ldl = LengthDelimitedDecoder.class.cast( decoder );
-                long len = getLen( ldl );
-                if ( len > -1 )
-                {
-                    log.debug( "transfer of {} : {}/{}", resourceName, len / 1024, this.totalLength / 1024 );
-                }
-            }
-
-            super.onContentReceived( decoder, ioControl );
+            log.debug( "transfer of {} : {}/{}", transferEvent.getResource().getName(), buffer.length, length );
+            this.totalLength += length;
         }
 
-        @Override
-        protected void onResponseReceived( HttpResponse response )
+        public void transferCompleted( TransferEvent transferEvent )
         {
-            this.startTime = System.currentTimeMillis();
-            super.onResponseReceived( response );
-            this.totalLength = response.getEntity().getContentLength();
-            log.info( "start transfer of {}, contentLength: {}", resourceName, this.totalLength / 1024 );
+            resourceName = transferEvent.getResource().getName();
+            long endTime = System.currentTimeMillis();
+            log.info( "end of transfer file {} {} kb: {}s", transferEvent.getResource().getName(),
+                      this.totalLength / 1024, ( endTime - startTime ) / 1000 );
         }
 
-        @Override
-        protected void onEntityEnclosed( HttpEntity entity, ContentType contentType )
-            throws IOException
+        public void transferError( TransferEvent transferEvent )
         {
-            super.onEntityEnclosed( entity, contentType );
+            log.info( "error of transfer file {}: {}", transferEvent.getResource().getName(),
+                      transferEvent.getException().getMessage(), transferEvent.getException() );
         }
 
-        private long getLen( LengthDelimitedDecoder ldl )
+        public void debug( String message )
         {
-            try
-            {
-                Field lenField = LengthDelimitedDecoder.class.getDeclaredField( "len" );
-                lenField.setAccessible( true );
-                long len = (Long) lenField.get( ldl );
-                return len;
-            }
-            catch ( NoSuchFieldException e )
-            {
-                log.debug( e.getMessage(), e );
-                return -1;
-            }
-            catch ( IllegalAccessException e )
-            {
-                log.debug( e.getMessage(), e );
-                return -1;
-            }
+            log.debug( "transfer debug {}", message );
         }
     }
 
-    private static class ZeroCopyResourceFetcher
+    private static class WagonResourceFetcher
         implements ResourceFetcher
     {
 
@@ -378,26 +304,23 @@ public class DownloadRemoteIndexTask
 
         File tempIndexDirectory;
 
-        final RemoteRepository remoteRepository;
+        Wagon wagon;
 
-        CloseableHttpAsyncClient httpclient;
+        RemoteRepository remoteRepository;
 
-        String baseIndexUrl;
-
-        private ZeroCopyResourceFetcher( Logger log, File tempIndexDirectory, RemoteRepository remoteRepository,
-                                         CloseableHttpAsyncClient httpclient, String baseIndexUrl )
+        private WagonResourceFetcher( Logger log, File tempIndexDirectory, Wagon wagon,
+                                      RemoteRepository remoteRepository )
         {
             this.log = log;
             this.tempIndexDirectory = tempIndexDirectory;
+            this.wagon = wagon;
             this.remoteRepository = remoteRepository;
-            this.httpclient = httpclient;
-            this.baseIndexUrl = baseIndexUrl;
         }
 
         public void connect( String id, String url )
             throws IOException
         {
-            //no op
+            //no op  
         }
 
         public void disconnect()
@@ -406,131 +329,62 @@ public class DownloadRemoteIndexTask
             // no op
         }
 
-        public InputStream retrieve( final String name )
-            throws IOException
+        public InputStream retrieve( String name )
+            throws IOException, FileNotFoundException
         {
-
-            log.info( "index update retrieve file, name:{}", name );
-            File file = new File( tempIndexDirectory, name );
-            if ( file.exists() )
-            {
-                file.delete();
-            }
-            file.deleteOnExit();
-
-            ZeroCopyConsumer<File> consumer = new ZeroCopyConsumerListener( file, name );
-
-            URL targetUrl = new URL( this.baseIndexUrl );
-            final HttpHost targetHost = new HttpHost( targetUrl.getHost(), targetUrl.getPort() );
-
-            Future<File> httpResponseFuture = httpclient.execute( new HttpAsyncRequestProducer()
+            try
             {
-                @Override
-                public HttpHost getTarget()
-                {
-                    return targetHost;
-                }
-
-                @Override
-                public HttpRequest generateRequest()
-                    throws IOException, HttpException
-                {
-                    StringBuilder url = new StringBuilder( baseIndexUrl );
-                    if ( !StringUtils.endsWith( baseIndexUrl, "/" ) )
-                    {
-                        url.append( '/' );
-                    }
-                    HttpGet httpGet = new HttpGet( url.append( addParameters( name, remoteRepository ) ).toString() );
-                    return httpGet;
-                }
-
-                @Override
-                public void produceContent( ContentEncoder encoder, IOControl ioctrl )
-                    throws IOException
-                {
-                    // no op
-                }
-
-                @Override
-                public void requestCompleted( HttpContext context )
-                {
-                    log.debug( "requestCompleted" );
-                }
-
-                @Override
-                public void failed( Exception ex )
+                log.info( "index update retrieve file, name:{}", name );
+                File file = new File( tempIndexDirectory, name );
+                if ( file.exists() )
                 {
-                    log.error( "http request failed", ex );
+                    file.delete();
                 }
-
-                @Override
-                public boolean isRepeatable()
-                {
-                    log.debug( "isRepeatable" );
-                    return true;
-                }
-
-                @Override
-                public void resetRequest()
-                    throws IOException
-                {
-                    log.debug( "resetRequest" );
-                }
-
-                @Override
-                public void close()
-                    throws IOException
-                {
-                    log.debug( "close" );
-                }
-
-            }, consumer, null );
-            try
-            {
-                int timeOut = this.remoteRepository.getRemoteDownloadTimeout();
-                file = timeOut > 0 ? httpResponseFuture.get( timeOut, TimeUnit.SECONDS ) : httpResponseFuture.get();
+                file.deleteOnExit();
+                wagon.get( addParameters( name, this.remoteRepository ), file );
+                return new FileInputStream( file );
             }
-            catch ( InterruptedException e )
+            catch ( AuthorizationException e )
             {
                 throw new IOException( e.getMessage(), e );
             }
-            catch ( ExecutionException e )
+            catch ( TransferFailedException e )
             {
                 throw new IOException( e.getMessage(), e );
             }
-            catch ( TimeoutException e )
+            catch ( ResourceDoesNotExistException e )
             {
-                throw new IOException( e.getMessage(), e );
+                FileNotFoundException fnfe = new FileNotFoundException( e.getMessage() );
+                fnfe.initCause( e );
+                throw fnfe;
             }
-            return new FileInputStream( file );
         }
 
-    }
-
-    // FIXME remove crappy copy/paste
-    protected static String addParameters( String path, RemoteRepository remoteRepository )
-    {
-        if ( remoteRepository.getExtraParameters().isEmpty() )
+        // FIXME remove crappy copy/paste
+        protected String addParameters( String path, RemoteRepository remoteRepository )
         {
-            return path;
-        }
+            if ( remoteRepository.getExtraParameters().isEmpty() )
+            {
+                return path;
+            }
 
-        boolean question = false;
+            boolean question = false;
 
-        StringBuilder res = new StringBuilder( path == null ? "" : path );
+            StringBuilder res = new StringBuilder( path == null ? "" : path );
 
-        for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() )
-        {
-            if ( !question )
+            for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() )
             {
-                res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() );
+                if ( !question )
+                {
+                    res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() );
+                }
             }
+
+            return res.toString();
         }
 
-        return res.toString();
     }
 
 
 }
 
-

archiva-modules/archiva-web/archiva-web-common/src/test/java/org/apache/archiva/AbstractDownloadTest.java → archiva-modules/archiva-web/archiva-web-common/src/test/java/org/apache/archiva/remotedownload/AbstractDownloadTest.java

@@ -1,4 +1,4 @@
-package org.apache.archiva;
+package org.apache.archiva.remotedownload;
 /*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file

archiva-modules/archiva-web/archiva-web-common/src/test/java/org/apache/archiva/DownloadArtifactsTest.java → archiva-modules/archiva-web/archiva-web-common/src/test/java/org/apache/archiva/remotedownload/DownloadArtifactsTest.java

@@ -1,4 +1,4 @@
-package org.apache.archiva;
+package org.apache.archiva.remotedownload;
 /*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file

archiva-modules/archiva-web/archiva-web-common/src/test/java/org/apache/archiva/DownloadMergedIndexNonDefaultPathTest.java → archiva-modules/archiva-web/archiva-web-common/src/test/java/org/apache/archiva/remotedownload/DownloadMergedIndexNonDefaultPathTest.java

@@ -1,4 +1,4 @@
-package org.apache.archiva;
+package org.apache.archiva.remotedownload;
 /*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -186,6 +186,7 @@ public class DownloadMergedIndexNonDefaultPathTest
         while ( !repositoriesService.getRunningRemoteDownloadIds().getStrings().isEmpty() )
         {
             Thread.sleep( 500 );
+            log.debug( "still running remote download" );
         }
 
         SearchService searchService = getSearchService();

archiva-modules/archiva-web/archiva-web-common/src/test/java/org/apache/archiva/DownloadMergedIndexTest.java → archiva-modules/archiva-web/archiva-web-common/src/test/java/org/apache/archiva/remotedownload/DownloadMergedIndexTest.java

@@ -1,4 +1,4 @@
-package org.apache.archiva;
+package org.apache.archiva.remotedownload;
 /*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file

archiva-modules/archiva-web/archiva-web-common/src/test/resources/log4j2-test.xml

@@ -35,6 +35,8 @@
 
     <logger name="org.apache.archiva.scheduler.indexing" level="debug"/>
 
+    <logger name="org.apache.archiva.remotedownload" level="debug"/>
+
     <root level="info">
       <appender-ref ref="console"/>
     </root>