]> source.dussan.org Git - archiva.git/blob
7e007f04bcc89a453a0d4d27edba82934c55073f
[archiva.git] /
1 package org.apache.archiva.indexer.maven;
2
3 /*
4  * Licensed to the Apache Software Foundation (ASF) under one
5  * or more contributor license agreements.  See the NOTICE file
6  * distributed with this work for additional information
7  * regarding copyright ownership.  The ASF licenses this file
8  * to you under the Apache License, Version 2.0 (the
9  * "License"); you may not use this file except in compliance
10  * with the License.  You may obtain a copy of the License at
11  *
12  *  http://www.apache.org/licenses/LICENSE-2.0
13  *
14  * Unless required by applicable law or agreed to in writing,
15  * software distributed under the License is distributed on an
16  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17  * KIND, either express or implied.  See the License for the
18  * specific language governing permissions and limitations
19  * under the License.
20  */
21
22 import org.apache.archiva.common.utils.PathUtil;
23 import org.apache.archiva.configuration.ArchivaConfiguration;
24 import org.apache.archiva.indexer.ArchivaIndexManager;
25 import org.apache.archiva.indexer.ArchivaIndexingContext;
26 import org.apache.archiva.indexer.IndexCreationFailedException;
27 import org.apache.archiva.indexer.IndexUpdateFailedException;
28 import org.apache.archiva.indexer.UnsupportedBaseContextException;
29 import org.apache.archiva.proxy.ProxyRegistry;
30 import org.apache.archiva.proxy.maven.WagonFactory;
31 import org.apache.archiva.proxy.maven.WagonFactoryException;
32 import org.apache.archiva.proxy.maven.WagonFactoryRequest;
33 import org.apache.archiva.proxy.model.NetworkProxy;
34 import org.apache.archiva.repository.EditableRepository;
35 import org.apache.archiva.repository.ManagedRepository;
36 import org.apache.archiva.repository.base.PasswordCredentials;
37 import org.apache.archiva.repository.RemoteRepository;
38 import org.apache.archiva.repository.Repository;
39 import org.apache.archiva.repository.RepositoryType;
40 import org.apache.archiva.repository.UnsupportedRepositoryTypeException;
41 import org.apache.archiva.repository.storage.FilesystemStorage;
42 import org.apache.archiva.repository.storage.RepositoryStorage;
43 import org.apache.archiva.repository.storage.StorageAsset;
44 import org.apache.archiva.repository.features.IndexCreationFeature;
45 import org.apache.archiva.repository.features.RemoteIndexFeature;
46 import org.apache.archiva.repository.storage.StorageUtil;
47 import org.apache.commons.lang3.StringUtils;
48 import org.apache.maven.index.ArtifactContext;
49 import org.apache.maven.index.ArtifactContextProducer;
50 import org.apache.maven.index.DefaultScannerListener;
51 import org.apache.maven.index.Indexer;
52 import org.apache.maven.index.IndexerEngine;
53 import org.apache.maven.index.Scanner;
54 import org.apache.maven.index.ScanningRequest;
55 import org.apache.maven.index.ScanningResult;
56 import org.apache.maven.index.context.ContextMemberProvider;
57 import org.apache.maven.index.context.IndexCreator;
58 import org.apache.maven.index.context.IndexingContext;
59 import org.apache.maven.index.context.StaticContextMemberProvider;
60 import org.apache.maven.index.packer.IndexPacker;
61 import org.apache.maven.index.packer.IndexPackingRequest;
62 import org.apache.maven.index.updater.IndexUpdateRequest;
63 import org.apache.maven.index.updater.IndexUpdater;
64 import org.apache.maven.index.updater.ResourceFetcher;
65 import org.apache.maven.index_shaded.lucene.index.IndexFormatTooOldException;
66 import org.apache.maven.wagon.ConnectionException;
67 import org.apache.maven.wagon.ResourceDoesNotExistException;
68 import org.apache.maven.wagon.StreamWagon;
69 import org.apache.maven.wagon.TransferFailedException;
70 import org.apache.maven.wagon.Wagon;
71 import org.apache.maven.wagon.authentication.AuthenticationException;
72 import org.apache.maven.wagon.authentication.AuthenticationInfo;
73 import org.apache.maven.wagon.authorization.AuthorizationException;
74 import org.apache.maven.wagon.events.TransferEvent;
75 import org.apache.maven.wagon.events.TransferListener;
76 import org.apache.maven.wagon.proxy.ProxyInfo;
77 import org.apache.maven.wagon.shared.http.AbstractHttpClientWagon;
78 import org.apache.maven.wagon.shared.http.HttpConfiguration;
79 import org.apache.maven.wagon.shared.http.HttpMethodConfiguration;
80 import org.slf4j.Logger;
81 import org.slf4j.LoggerFactory;
82 import org.springframework.stereotype.Service;
83
84 import javax.inject.Inject;
85 import java.io.FileNotFoundException;
86 import java.io.IOException;
87 import java.io.InputStream;
88 import java.net.MalformedURLException;
89 import java.net.URI;
90 import java.nio.file.Files;
91 import java.nio.file.Path;
92 import java.nio.file.Paths;
93 import java.util.Collection;
94 import java.util.List;
95 import java.util.Map;
96 import java.util.Objects;
97 import java.util.concurrent.ConcurrentSkipListSet;
98 import java.util.stream.Collectors;
99
100 /**
101  * Maven implementation of index manager.
102  * The index manager is a singleton, so we try to make sure, that index operations are not running
103  * parallel by synchronizing on the index path.
104  * A update operation waits for parallel running methods to finish before starting, but after a certain
105  * time of retries a IndexUpdateFailedException is thrown.
106  */
107 @Service( "archivaIndexManager#maven" )
108 public class MavenIndexManager implements ArchivaIndexManager {
109
110     private static final Logger log = LoggerFactory.getLogger( MavenIndexManager.class );
111
112     @Inject
113     private Indexer indexer;
114
115     @Inject
116     private IndexerEngine indexerEngine;
117
118     @Inject
119     private List<? extends IndexCreator> indexCreators;
120
121     @Inject
122     private IndexPacker indexPacker;
123
124     @Inject
125     private Scanner scanner;
126
127     @Inject
128     private ArchivaConfiguration archivaConfiguration;
129
130     @Inject
131     private WagonFactory wagonFactory;
132
133     @Inject
134     private IndexUpdater indexUpdater;
135
136     @Inject
137     private ArtifactContextProducer artifactContextProducer;
138
139     @Inject
140     private ProxyRegistry proxyRegistry;
141
142
143     private ConcurrentSkipListSet<StorageAsset> activeContexts = new ConcurrentSkipListSet<>( );
144
145     private static final int WAIT_TIME = 100;
146     private static final int MAX_WAIT = 10;
147
148
149     public static IndexingContext getMvnContext( ArchivaIndexingContext context ) throws UnsupportedBaseContextException
150     {
151         if (context!=null)
152         {
153             if ( !context.supports( IndexingContext.class ) )
154             {
155                 log.error( "The provided archiva index context does not support the maven IndexingContext" );
156                 throw new UnsupportedBaseContextException( "The context does not support the Maven IndexingContext" );
157             }
158             return context.getBaseContext( IndexingContext.class );
159         } else {
160             return null;
161         }
162     }
163
164     private StorageAsset getIndexPath( ArchivaIndexingContext ctx )
165     {
166         return ctx.getPath( );
167     }
168
169     @FunctionalInterface
170     interface IndexUpdateConsumer
171     {
172
173         void accept( IndexingContext indexingContext ) throws IndexUpdateFailedException;
174     }
175
176     /*
177      * This method is used to do some actions around the update execution code. And to make sure, that no other
178      * method is running on the same index.
179      */
180     private void executeUpdateFunction( ArchivaIndexingContext context, IndexUpdateConsumer function ) throws IndexUpdateFailedException
181     {
182         if (context==null) {
183             throw new IndexUpdateFailedException( "Given context is null" );
184         }
185         IndexingContext indexingContext = null;
186         try
187         {
188             indexingContext = getMvnContext( context );
189         }
190         catch ( UnsupportedBaseContextException e )
191         {
192             throw new IndexUpdateFailedException( "Maven index is not supported by this context", e );
193         }
194         final StorageAsset ctxPath = getIndexPath( context );
195         int loop = MAX_WAIT;
196         boolean active = false;
197         while ( loop-- > 0 && !active )
198         {
199             active = activeContexts.add( ctxPath );
200             try
201             {
202                 Thread.currentThread( ).sleep( WAIT_TIME );
203             }
204             catch ( InterruptedException e )
205             {
206                 // Ignore this
207             }
208         }
209         if ( active )
210         {
211             try
212             {
213                 function.accept( indexingContext );
214             }
215             finally
216             {
217                 activeContexts.remove( ctxPath );
218             }
219         }
220         else
221         {
222             throw new IndexUpdateFailedException( "Timeout while waiting for index release on context " + context.getId( ) );
223         }
224     }
225
226     @Override
227     public void pack( final ArchivaIndexingContext context ) throws IndexUpdateFailedException
228     {
229         executeUpdateFunction( context, indexingContext -> {
230                 try
231                 {
232                     IndexPackingRequest request = new IndexPackingRequest( indexingContext,
233                         indexingContext.acquireIndexSearcher( ).getIndexReader( ),
234                         indexingContext.getIndexDirectoryFile( ) );
235                     indexPacker.packIndex( request );
236                     indexingContext.updateTimestamp( true );
237                 }
238                 catch ( IOException e )
239                 {
240                     log.error( "IOException while packing index of context " + context.getId( ) + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ) );
241                     throw new IndexUpdateFailedException( "IOException during update of " + context.getId( ), e );
242                 }
243             }
244         );
245
246     }
247
248     @Override
249     public void scan(final ArchivaIndexingContext context) throws IndexUpdateFailedException
250     {
251         executeUpdateFunction( context, indexingContext -> {
252             DefaultScannerListener listener = new DefaultScannerListener( indexingContext, indexerEngine, true, null );
253             ScanningRequest request = new ScanningRequest( indexingContext, listener );
254             ScanningResult result = scanner.scan( request );
255             if ( result.hasExceptions( ) )
256             {
257                 log.error( "Exceptions occured during index scan of " + context.getId( ) );
258                 result.getExceptions( ).stream( ).map( e -> e.getMessage( ) ).distinct( ).limit( 5 ).forEach(
259                     s -> log.error( "Message: " + s )
260                 );
261             }
262
263         } );
264     }
265
266     @Override
267     public void update(final ArchivaIndexingContext context, final boolean fullUpdate) throws IndexUpdateFailedException
268     {
269         log.info( "start download remote index for remote repository {}", context.getRepository( ).getId( ) );
270         URI remoteUpdateUri;
271         if ( !( context.getRepository( ) instanceof RemoteRepository ) || !(context.getRepository().supportsFeature(RemoteIndexFeature.class)) )
272         {
273             throw new IndexUpdateFailedException( "The context is not associated to a remote repository with remote index " + context.getId( ) );
274         } else {
275             RemoteIndexFeature rif = context.getRepository().getFeature(RemoteIndexFeature.class).get();
276             remoteUpdateUri = context.getRepository().getLocation().resolve(rif.getIndexUri());
277         }
278         final RemoteRepository remoteRepository = (RemoteRepository) context.getRepository( );
279
280         executeUpdateFunction( context,
281             indexingContext -> {
282                 try
283                 {
284                     // create a temp directory to download files
285                     Path tempIndexDirectory = Paths.get( indexingContext.getIndexDirectoryFile( ).getParent( ), ".tmpIndex" );
286                     Path indexCacheDirectory = Paths.get( indexingContext.getIndexDirectoryFile( ).getParent( ), ".indexCache" );
287                     Files.createDirectories( indexCacheDirectory );
288                     if ( Files.exists( tempIndexDirectory ) )
289                     {
290                         org.apache.archiva.common.utils.FileUtils.deleteDirectory( tempIndexDirectory );
291                     }
292                     Files.createDirectories( tempIndexDirectory );
293                     tempIndexDirectory.toFile( ).deleteOnExit( );
294                     String baseIndexUrl = indexingContext.getIndexUpdateUrl( );
295
296                     String wagonProtocol = remoteUpdateUri.toURL( ).getProtocol( );
297
298                     NetworkProxy networkProxy = null;
299                     if ( remoteRepository.supportsFeature( RemoteIndexFeature.class ) )
300                     {
301                         RemoteIndexFeature rif = remoteRepository.getFeature( RemoteIndexFeature.class ).get( );
302                         if ( StringUtils.isNotBlank( rif.getProxyId( ) ) )
303                         {
304                             networkProxy = proxyRegistry.getNetworkProxy( rif.getProxyId( ) );
305                             if ( networkProxy == null )
306                             {
307                                 log.warn(
308                                     "your remote repository is configured to download remote index trought a proxy we cannot find id:{}",
309                                     rif.getProxyId( ) );
310                             }
311                         }
312
313                         final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon(
314                             new WagonFactoryRequest( wagonProtocol, remoteRepository.getExtraHeaders( ) ).networkProxy(
315                                 networkProxy )
316                         );
317                         int readTimeout = (int) rif.getDownloadTimeout( ).toMillis( ) * 1000;
318                         wagon.setReadTimeout( readTimeout );
319                         wagon.setTimeout( (int) remoteRepository.getTimeout( ).toMillis( ) * 1000 );
320
321                         if ( wagon instanceof AbstractHttpClientWagon )
322                         {
323                             HttpConfiguration httpConfiguration = new HttpConfiguration( );
324                             HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration( );
325                             httpMethodConfiguration.setUsePreemptive( true );
326                             httpMethodConfiguration.setReadTimeout( readTimeout );
327                             httpConfiguration.setGet( httpMethodConfiguration );
328                             AbstractHttpClientWagon.class.cast( wagon ).setHttpConfiguration( httpConfiguration );
329                         }
330
331                         wagon.addTransferListener( new DownloadListener( ) );
332                         ProxyInfo proxyInfo = null;
333                         if ( networkProxy != null )
334                         {
335                             proxyInfo = new ProxyInfo( );
336                             proxyInfo.setType( networkProxy.getProtocol( ) );
337                             proxyInfo.setHost( networkProxy.getHost( ) );
338                             proxyInfo.setPort( networkProxy.getPort( ) );
339                             proxyInfo.setUserName( networkProxy.getUsername( ) );
340                             proxyInfo.setPassword( new String(networkProxy.getPassword( )) );
341                         }
342                         AuthenticationInfo authenticationInfo = null;
343                         if ( remoteRepository.getLoginCredentials( ) != null && ( remoteRepository.getLoginCredentials( ) instanceof PasswordCredentials ) )
344                         {
345                             PasswordCredentials creds = (PasswordCredentials) remoteRepository.getLoginCredentials( );
346                             authenticationInfo = new AuthenticationInfo( );
347                             authenticationInfo.setUserName( creds.getUsername( ) );
348                             authenticationInfo.setPassword( new String( creds.getPassword( ) ) );
349                         }
350                         wagon.connect( new org.apache.maven.wagon.repository.Repository( remoteRepository.getId( ), baseIndexUrl ), authenticationInfo,
351                             proxyInfo );
352
353                         Path indexDirectory = indexingContext.getIndexDirectoryFile( ).toPath( );
354                         if ( !Files.exists( indexDirectory ) )
355                         {
356                             Files.createDirectories( indexDirectory );
357                         }
358
359                         ResourceFetcher resourceFetcher =
360                             new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository );
361                         IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
362                         request.setForceFullUpdate( fullUpdate );
363                         request.setLocalIndexCacheDir( indexCacheDirectory.toFile( ) );
364
365                         indexUpdater.fetchAndUpdateIndex( request );
366
367                         indexingContext.updateTimestamp( true );
368                     }
369
370                 }
371                 catch ( AuthenticationException e )
372                 {
373                     log.error( "Could not login to the remote proxy for updating index of {}", remoteRepository.getId( ), e );
374                     throw new IndexUpdateFailedException( "Login in to proxy failed while updating remote repository " + remoteRepository.getId( ), e );
375                 }
376                 catch ( ConnectionException e )
377                 {
378                     log.error( "Connection error during index update for remote repository {}", remoteRepository.getId( ), e );
379                     throw new IndexUpdateFailedException( "Connection error during index update for remote repository " + remoteRepository.getId( ), e );
380                 }
381                 catch ( MalformedURLException e )
382                 {
383                     log.error( "URL for remote index update of remote repository {} is not correct {}", remoteRepository.getId( ), remoteUpdateUri, e );
384                     throw new IndexUpdateFailedException( "URL for remote index update of repository is not correct " + remoteUpdateUri, e );
385                 }
386                 catch ( IOException e )
387                 {
388                     log.error( "IOException during index update of remote repository {}: {}", remoteRepository.getId( ), e.getMessage( ), e );
389                     throw new IndexUpdateFailedException( "IOException during index update of remote repository " + remoteRepository.getId( )
390                         + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ), e );
391                 }
392                 catch ( WagonFactoryException e )
393                 {
394                     log.error( "Wagon for remote index download of {} could not be created: {}", remoteRepository.getId( ), e.getMessage( ), e );
395                     throw new IndexUpdateFailedException( "Error while updating the remote index of " + remoteRepository.getId( ), e );
396                 }
397             } );
398
399     }
400
401     @Override
402     public void addArtifactsToIndex( final ArchivaIndexingContext context, final Collection<URI> artifactReference ) throws IndexUpdateFailedException
403     {
404         final StorageAsset ctxUri = context.getPath();
405         executeUpdateFunction(context, indexingContext -> {
406             Collection<ArtifactContext> artifacts = artifactReference.stream().map(r -> artifactContextProducer.getArtifactContext(indexingContext, Paths.get(ctxUri.getFilePath().toUri().resolve(r)).toFile())).collect(Collectors.toList());
407             try {
408                 indexer.addArtifactsToIndex(artifacts, indexingContext);
409             } catch (IOException e) {
410                 log.error("IOException while adding artifact {}", e.getMessage(), e);
411                 throw new IndexUpdateFailedException("Error occured while adding artifact to index of "+context.getId()
412                 + (StringUtils.isNotEmpty(e.getMessage()) ? ": "+e.getMessage() : ""));
413             }
414         });
415     }
416
417     @Override
418     public void removeArtifactsFromIndex( ArchivaIndexingContext context, Collection<URI> artifactReference ) throws IndexUpdateFailedException
419     {
420         final StorageAsset ctxUri = context.getPath();
421         executeUpdateFunction(context, indexingContext -> {
422             Collection<ArtifactContext> artifacts = artifactReference.stream().map(r -> artifactContextProducer.getArtifactContext(indexingContext, Paths.get(ctxUri.getFilePath().toUri().resolve(r)).toFile())).collect(Collectors.toList());
423             try {
424                 indexer.deleteArtifactsFromIndex(artifacts, indexingContext);
425             } catch (IOException e) {
426                 log.error("IOException while removing artifact {}", e.getMessage(), e);
427                 throw new IndexUpdateFailedException("Error occured while removing artifact from index of "+context.getId()
428                         + (StringUtils.isNotEmpty(e.getMessage()) ? ": "+e.getMessage() : ""));
429             }
430         });
431
432     }
433
434     @Override
435     public boolean supportsRepository( RepositoryType type )
436     {
437         return type == RepositoryType.MAVEN;
438     }
439
440     @Override
441     public ArchivaIndexingContext createContext( Repository repository ) throws IndexCreationFailedException
442     {
443         log.debug("Creating context for repo {}, type: {}", repository.getId(), repository.getType());
444         if ( repository.getType( ) != RepositoryType.MAVEN )
445         {
446             throw new UnsupportedRepositoryTypeException( repository.getType( ) );
447         }
448         IndexingContext mvnCtx = null;
449         try
450         {
451             if ( repository instanceof RemoteRepository )
452             {
453                 mvnCtx = createRemoteContext( (RemoteRepository) repository );
454             }
455             else if ( repository instanceof ManagedRepository )
456             {
457                 mvnCtx = createManagedContext( (ManagedRepository) repository );
458             }
459         }
460         catch ( IOException e )
461         {
462             log.error( "IOException during context creation " + e.getMessage( ), e );
463             throw new IndexCreationFailedException( "Could not create index context for repository " + repository.getId( )
464                 + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ), e );
465         }
466
467         return new MavenIndexContext( repository, mvnCtx );
468     }
469
470     @Override
471     public ArchivaIndexingContext reset(ArchivaIndexingContext context) throws IndexUpdateFailedException {
472         ArchivaIndexingContext ctx;
473         executeUpdateFunction(context, indexingContext -> {
474             try {
475                 indexingContext.close(true);
476             } catch (IOException e) {
477                 log.warn("Index close failed");
478             }
479             try {
480                 StorageUtil.deleteRecursively(context.getPath());
481             } catch (IOException e) {
482                 throw new IndexUpdateFailedException("Could not delete index files");
483             }
484         });
485         try {
486             Repository repo = context.getRepository();
487             ctx = createContext(context.getRepository());
488             if (repo instanceof EditableRepository) {
489                 ((EditableRepository)repo).setIndexingContext(ctx);
490             }
491         } catch (IndexCreationFailedException e) {
492             throw new IndexUpdateFailedException("Could not create index");
493         }
494         return ctx;
495     }
496
497     @Override
498     public ArchivaIndexingContext move(ArchivaIndexingContext context, Repository repo) throws IndexCreationFailedException {
499         if (context==null) {
500             return null;
501         }
502         if (context.supports(IndexingContext.class)) {
503             try {
504                 StorageAsset newPath = getIndexPath(repo);
505                 IndexingContext ctx = context.getBaseContext(IndexingContext.class);
506                 Path oldPath = ctx.getIndexDirectoryFile().toPath();
507                 if (oldPath.equals(newPath)) {
508                     // Nothing to do, if path does not change
509                     return context;
510                 }
511                 if (!Files.exists(oldPath)) {
512                     return createContext(repo);
513                 } else if (context.isEmpty()) {
514                     context.close();
515                     return createContext(repo);
516                 } else {
517                     context.close(false);
518                     Files.move(oldPath, newPath.getFilePath());
519                     return createContext(repo);
520                 }
521             } catch (IOException e) {
522                 log.error("IOException while moving index directory {}", e.getMessage(), e);
523                 throw new IndexCreationFailedException("Could not recreated the index.", e);
524             } catch (UnsupportedBaseContextException e) {
525                 throw new IndexCreationFailedException("The given context, is not a maven context.");
526             }
527         } else {
528             throw new IndexCreationFailedException("Bad context type. This is not a maven context.");
529         }
530     }
531
532     @Override
533     public void updateLocalIndexPath(Repository repo) {
534         if (repo.supportsFeature(IndexCreationFeature.class)) {
535             IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
536             try {
537                 icf.setLocalIndexPath(getIndexPath(repo));
538                 icf.setLocalPackedIndexPath(getPackedIndexPath(repo));
539             } catch (IOException e) {
540                 log.error("Could not set local index path for {}. New URI: {}", repo.getId(), icf.getIndexPath());
541             }
542         }
543     }
544
545     @Override
546     public ArchivaIndexingContext mergeContexts(Repository destinationRepo, List<ArchivaIndexingContext> contexts,
547                                                 boolean packIndex) throws UnsupportedOperationException,
548             IndexCreationFailedException, IllegalArgumentException {
549         if (!destinationRepo.supportsFeature(IndexCreationFeature.class)) {
550             throw new IllegalArgumentException("The given repository does not support the indexcreation feature");
551         }
552         Path mergedIndexDirectory = null;
553         try {
554             mergedIndexDirectory = Files.createTempDirectory("archivaMergedIndex");
555         } catch (IOException e) {
556             log.error("Could not create temporary directory for merged index: {}", e.getMessage(), e);
557             throw new IndexCreationFailedException("IO error while creating temporary directory for merged index: "+e.getMessage(), e);
558         }
559         IndexCreationFeature indexCreationFeature = destinationRepo.getFeature(IndexCreationFeature.class).get();
560         if (indexCreationFeature.getLocalIndexPath()== null) {
561             throw new IllegalArgumentException("The given repository does not have a local index path");
562         }
563         StorageAsset destinationPath = indexCreationFeature.getLocalIndexPath();
564
565         String tempRepoId = mergedIndexDirectory.getFileName().toString();
566
567         try
568         {
569             Path indexLocation = destinationPath.getFilePath();
570
571             List<IndexingContext> members = contexts.stream( ).filter(ctx -> ctx.supports(IndexingContext.class)).map( ctx ->
572             {
573                 try {
574                     return ctx.getBaseContext(IndexingContext.class);
575                 } catch (UnsupportedBaseContextException e) {
576                     // does not happen here
577                     return null;
578                 }
579             }).filter( Objects::nonNull ).collect( Collectors.toList() );
580             ContextMemberProvider memberProvider = new StaticContextMemberProvider(members);
581             IndexingContext mergedCtx = indexer.createMergedIndexingContext( tempRepoId, tempRepoId, mergedIndexDirectory.toFile(),
582                     indexLocation.toFile(), true, memberProvider);
583             mergedCtx.optimize();
584
585             if ( packIndex )
586             {
587                 IndexPackingRequest request = new IndexPackingRequest( mergedCtx, //
588                         mergedCtx.acquireIndexSearcher().getIndexReader(), //
589                         indexLocation.toFile() );
590                 indexPacker.packIndex( request );
591             }
592
593             return new MavenIndexContext(destinationRepo, mergedCtx);
594         }
595         catch ( IOException e)
596         {
597             throw new IndexCreationFailedException( "IO Error during index merge: "+ e.getMessage(), e );
598         }
599     }
600
601     private StorageAsset getIndexPath(URI indexDirUri, RepositoryStorage repoStorage, String defaultDir) throws IOException
602     {
603         StorageAsset rootAsset = repoStorage.getAsset("");
604         RepositoryStorage storage = rootAsset.getStorage();
605         Path indexDirectory;
606         Path repositoryPath = rootAsset.getFilePath().toAbsolutePath();
607         StorageAsset indexDir;
608         if ( ! StringUtils.isEmpty(indexDirUri.toString( ) ) )
609         {
610
611             indexDirectory = PathUtil.getPathFromUri( indexDirUri );
612             // not absolute so create it in repository directory
613             if ( indexDirectory.isAbsolute( ) && !indexDirectory.startsWith(repositoryPath))
614             {
615                 if (storage instanceof FilesystemStorage) {
616                     FilesystemStorage fsStorage = (FilesystemStorage) storage;
617                     FilesystemStorage indexStorage = new FilesystemStorage(indexDirectory.getParent(), fsStorage.getFileLockManager());
618                     indexDir = indexStorage.getAsset(indexDirectory.getFileName().toString());
619                 } else {
620                     throw new IOException("The given storage is not file based.");
621                 }
622             } else if (indexDirectory.isAbsolute()) {
623                 indexDir = storage.getAsset(repositoryPath.relativize(indexDirectory).toString());
624             }
625             else
626             {
627                 indexDir = storage.getAsset(indexDirectory.toString());
628             }
629         }
630         else
631         {
632             indexDir = storage.getAsset( defaultDir );
633         }
634
635         if ( !indexDir.exists() )
636         {
637             indexDir = storage.addAsset(indexDir.getPath(), true);
638         }
639         return indexDir;
640     }
641
642     private StorageAsset getIndexPath( Repository repo) throws IOException {
643         IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
644         return getIndexPath( icf.getIndexPath(), repo, DEFAULT_INDEX_PATH);
645     }
646
647     private StorageAsset getPackedIndexPath(Repository repo) throws IOException {
648         IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
649         return getIndexPath(icf.getPackedIndexPath(), repo, DEFAULT_PACKED_INDEX_PATH);
650     }
651
652     private IndexingContext createRemoteContext(RemoteRepository remoteRepository ) throws IOException
653     {
654         String contextKey = "remote-" + remoteRepository.getId( );
655
656
657         // create remote repository path
658         Path repoDir = remoteRepository.getAsset( "" ).getFilePath();
659         if ( !Files.exists( repoDir ) )
660         {
661             Files.createDirectories( repoDir );
662         }
663
664         StorageAsset indexDirectory;
665
666         // is there configured indexDirectory ?
667         if ( remoteRepository.supportsFeature( RemoteIndexFeature.class ) )
668         {
669             RemoteIndexFeature rif = remoteRepository.getFeature( RemoteIndexFeature.class ).get( );
670             indexDirectory = getIndexPath(remoteRepository);
671             String remoteIndexUrl = calculateIndexRemoteUrl( remoteRepository.getLocation( ), rif );
672             try
673             {
674
675                 return getIndexingContext( remoteRepository, contextKey, repoDir, indexDirectory, remoteIndexUrl );
676             }
677             catch ( IndexFormatTooOldException e )
678             {
679                 // existing index with an old lucene format so we need to delete it!!!
680                 // delete it first then recreate it.
681                 log.warn( "the index of repository {} is too old we have to delete and recreate it", //
682                     remoteRepository.getId( ) );
683                 org.apache.archiva.common.utils.FileUtils.deleteDirectory( indexDirectory.getFilePath() );
684                 return getIndexingContext( remoteRepository, contextKey, repoDir, indexDirectory, remoteIndexUrl );
685
686             }
687         }
688         else
689         {
690             throw new IOException( "No remote index defined" );
691         }
692     }
693
694     private IndexingContext getIndexingContext( Repository repository, String contextKey, Path repoDir, StorageAsset indexDirectory, String indexUrl ) throws IOException
695     {
696         try
697         {
698             if (!Files.exists(indexDirectory.getFilePath())) {
699                 Files.createDirectories(indexDirectory.getFilePath());
700             }
701             return indexer.createIndexingContext( contextKey, repository.getId( ), repoDir.toFile( ), indexDirectory.getFilePath( ).toFile( ),
702                 repository.getLocation( ) == null ? null : repository.getLocation( ).toString( ),
703                 indexUrl,
704                 true, false,
705                 indexCreators );
706         } catch (Exception e) {
707             log.error("Could not create index for asset {}", indexDirectory);
708             throw new IOException(e);
709         }
710     }
711
712     private IndexingContext createManagedContext( ManagedRepository repository ) throws IOException
713     {
714
715         IndexingContext context;
716         // take care first about repository location as can be relative
717         Path repositoryDirectory = repository.getAsset( "" ).getFilePath();
718
719         if ( !Files.exists( repositoryDirectory ) )
720         {
721             try
722             {
723                 Files.createDirectories( repositoryDirectory );
724             }
725             catch ( IOException e )
726             {
727                 log.error( "Could not create directory {}", repositoryDirectory );
728             }
729         }
730
731         StorageAsset indexDirectory;
732
733         if ( repository.supportsFeature( IndexCreationFeature.class ) )
734         {
735             indexDirectory = getIndexPath(repository);
736             log.debug( "Preparing index at {}", indexDirectory );
737
738             String indexUrl = repositoryDirectory.toUri( ).toURL( ).toExternalForm( );
739             try
740             {
741                 context = getIndexingContext( repository, repository.getId( ), repositoryDirectory, indexDirectory, indexUrl );
742                 context.setSearchable( repository.isScanned( ) );
743             }
744             catch ( IndexFormatTooOldException e )
745             {
746                 // existing index with an old lucene format so we need to delete it!!!
747                 // delete it first then recreate it.
748                 log.warn( "the index of repository {} is too old we have to delete and recreate it", //
749                     repository.getId( ) );
750                 org.apache.archiva.common.utils.FileUtils.deleteDirectory( indexDirectory.getFilePath() );
751                 context = getIndexingContext( repository, repository.getId( ), repositoryDirectory, indexDirectory, indexUrl );
752                 context.setSearchable( repository.isScanned( ) );
753             }
754             return context;
755         }
756         else
757         {
758             throw new IOException( "No repository index defined" );
759         }
760     }
761
762     private String calculateIndexRemoteUrl( URI baseUri, RemoteIndexFeature rif )
763     {
764         if ( rif.getIndexUri( ) == null )
765         {
766             return baseUri.resolve( DEFAULT_INDEX_PATH ).toString( );
767         }
768         else
769         {
770             return baseUri.resolve( rif.getIndexUri( ) ).toString( );
771         }
772     }
773
774     private static final class DownloadListener
775         implements TransferListener
776     {
777         private Logger log = LoggerFactory.getLogger( getClass( ) );
778
779         private String resourceName;
780
781         private long startTime;
782
783         private int totalLength = 0;
784
785         @Override
786         public void transferInitiated( TransferEvent transferEvent )
787         {
788             startTime = System.currentTimeMillis( );
789             resourceName = transferEvent.getResource( ).getName( );
790             log.debug( "initiate transfer of {}", resourceName );
791         }
792
793         @Override
794         public void transferStarted( TransferEvent transferEvent )
795         {
796             this.totalLength = 0;
797             resourceName = transferEvent.getResource( ).getName( );
798             log.info( "start transfer of {}", transferEvent.getResource( ).getName( ) );
799         }
800
801         @Override
802         public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
803         {
804             log.debug( "transfer of {} : {}/{}", transferEvent.getResource( ).getName( ), buffer.length, length );
805             this.totalLength += length;
806         }
807
808         @Override
809         public void transferCompleted( TransferEvent transferEvent )
810         {
811             resourceName = transferEvent.getResource( ).getName( );
812             long endTime = System.currentTimeMillis( );
813             log.info( "end of transfer file {} {} kb: {}s", transferEvent.getResource( ).getName( ),
814                 this.totalLength / 1024, ( endTime - startTime ) / 1000 );
815         }
816
817         @Override
818         public void transferError( TransferEvent transferEvent )
819         {
820             log.info( "error of transfer file {}: {}", transferEvent.getResource( ).getName( ),
821                 transferEvent.getException( ).getMessage( ), transferEvent.getException( ) );
822         }
823
824         @Override
825         public void debug( String message )
826         {
827             log.debug( "transfer debug {}", message );
828         }
829     }
830
831     private static class WagonResourceFetcher
832         implements ResourceFetcher
833     {
834
835         Logger log;
836
837         Path tempIndexDirectory;
838
839         Wagon wagon;
840
841         RemoteRepository remoteRepository;
842
843         private WagonResourceFetcher( Logger log, Path tempIndexDirectory, Wagon wagon,
844                                       RemoteRepository remoteRepository )
845         {
846             this.log = log;
847             this.tempIndexDirectory = tempIndexDirectory;
848             this.wagon = wagon;
849             this.remoteRepository = remoteRepository;
850         }
851
852         @Override
853         public void connect( String id, String url ) {
854             //no op
855         }
856
857         @Override
858         public void disconnect( ) {
859             // no op
860         }
861
862         @Override
863         public InputStream retrieve( String name )
864             throws IOException {
865             try
866             {
867                 log.info( "index update retrieve file, name:{}", name );
868                 Path file = tempIndexDirectory.resolve( name );
869                 Files.deleteIfExists( file );
870                 file.toFile( ).deleteOnExit( );
871                 wagon.get( addParameters( name, remoteRepository ), file.toFile( ) );
872                 return Files.newInputStream( file );
873             }
874             catch ( AuthorizationException | TransferFailedException e )
875             {
876                 throw new IOException( e.getMessage( ), e );
877             }
878             catch ( ResourceDoesNotExistException e )
879             {
880                 FileNotFoundException fnfe = new FileNotFoundException( e.getMessage( ) );
881                 fnfe.initCause( e );
882                 throw fnfe;
883             }
884         }
885
886         // FIXME remove crappy copy/paste
887         protected String addParameters( String path, RemoteRepository remoteRepository )
888         {
889             if ( remoteRepository.getExtraParameters( ).isEmpty( ) )
890             {
891                 return path;
892             }
893
894             boolean question = false;
895
896             StringBuilder res = new StringBuilder( path == null ? "" : path );
897
898             for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters( ).entrySet( ) )
899             {
900                 if ( !question )
901                 {
902                     res.append( '?' ).append( entry.getKey( ) ).append( '=' ).append( entry.getValue( ) );
903                 }
904             }
905
906             return res.toString( );
907         }
908
909     }
910 }