]> source.dussan.org Git - archiva.git/blob
ed85005a8842a9bafba84e9bed22111712288005
[archiva.git] /
1 package org.apache.maven.archiva.proxy;
2
3 /*
4  * Licensed to the Apache Software Foundation (ASF) under one
5  * or more contributor license agreements.  See the NOTICE file
6  * distributed with this work for additional information
7  * regarding copyright ownership.  The ASF licenses this file
8  * to you under the Apache License, Version 2.0 (the
9  * "License"); you may not use this file except in compliance
10  * with the License.  You may obtain a copy of the License at
11  *
12  *  http://www.apache.org/licenses/LICENSE-2.0
13  *
14  * Unless required by applicable law or agreed to in writing,
15  * software distributed under the License is distributed on an
16  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17  * KIND, either express or implied.  See the License for the
18  * specific language governing permissions and limitations
19  * under the License.
20  */
21
22 import org.apache.commons.io.FileUtils;
23 import org.apache.maven.archiva.configuration.ArchivaConfiguration;
24 import org.apache.maven.archiva.configuration.NetworkProxyConfiguration;
25 import org.apache.maven.archiva.configuration.ProxyConnectorConfiguration;
26 import org.apache.maven.archiva.configuration.RepositoryConfiguration;
27 import org.apache.maven.archiva.model.ArchivaRepository;
28 import org.apache.maven.archiva.model.ArtifactReference;
29 import org.apache.maven.archiva.model.ProjectReference;
30 import org.apache.maven.archiva.policies.DownloadPolicy;
31 import org.apache.maven.archiva.policies.urlcache.UrlFailureCache;
32 import org.apache.maven.archiva.repository.layout.BidirectionalRepositoryLayout;
33 import org.apache.maven.archiva.repository.layout.BidirectionalRepositoryLayoutFactory;
34 import org.apache.maven.archiva.repository.layout.LayoutException;
35 import org.apache.maven.wagon.ConnectionException;
36 import org.apache.maven.wagon.ResourceDoesNotExistException;
37 import org.apache.maven.wagon.Wagon;
38 import org.apache.maven.wagon.WagonException;
39 import org.apache.maven.wagon.authentication.AuthenticationException;
40 import org.apache.maven.wagon.proxy.ProxyInfo;
41 import org.apache.maven.wagon.repository.Repository;
42 import org.codehaus.plexus.logging.AbstractLogEnabled;
43 import org.codehaus.plexus.personality.plexus.lifecycle.phase.Initializable;
44 import org.codehaus.plexus.personality.plexus.lifecycle.phase.InitializationException;
45 import org.codehaus.plexus.registry.Registry;
46 import org.codehaus.plexus.registry.RegistryListener;
47 import org.codehaus.plexus.util.SelectorUtils;
48
49 import java.io.File;
50 import java.io.IOException;
51 import java.util.ArrayList;
52 import java.util.Collection;
53 import java.util.Collections;
54 import java.util.HashMap;
55 import java.util.Iterator;
56 import java.util.List;
57 import java.util.Map;
58 import java.util.Properties;
59 import java.util.Map.Entry;
60
61 /**
62  * DefaultRepositoryProxyConnectors 
63  *
64  * @author <a href="mailto:joakim@erdfelt.com">Joakim Erdfelt</a>
65  * @version $Id$
66  * 
67  * @plexus.component role-hint="default"
68  */
69 public class DefaultRepositoryProxyConnectors
70     extends AbstractLogEnabled
71     implements RepositoryProxyConnectors, RegistryListener, Initializable
72 {
73     private static final String FILENAME_MAVEN_METADATA = "maven-metadata.xml";
74
75     /**
76      * @plexus.requirement
77      */
78     private ArchivaConfiguration archivaConfiguration;
79
80     /**
81      * @plexus.requirement role="org.apache.maven.wagon.Wagon"
82      */
83     private Map/*<String,Wagon>*/wagons;
84
85     /**
86      * @plexus.requirement
87      */
88     private BidirectionalRepositoryLayoutFactory layoutFactory;
89
90     /**
91      * @plexus.requirement role="org.apache.maven.archiva.policies.PreDownloadPolicy"
92      */
93     private Map preDownloadPolicies;
94
95     /**
96      * @plexus.requirement role="org.apache.maven.archiva.policies.PostDownloadPolicy"
97      */
98     private Map postDownloadPolicies;
99
100     /**
101      * @plexus.requirement role-hint="default"
102      */
103     private UrlFailureCache urlFailureCache;
104
105     private Map proxyConnectorMap = new HashMap();
106
107     private Map networkProxyMap = new HashMap();
108
109     private List propertyNameTriggers = new ArrayList();
110
111     public File fetchFromProxies( ArchivaRepository repository, ArtifactReference artifact )
112         throws ProxyException
113     {
114         if ( !repository.isManaged() )
115         {
116             throw new ProxyException( "Can only proxy managed repositories." );
117         }
118
119         File localFile;
120         try
121         {
122             BidirectionalRepositoryLayout sourceLayout = layoutFactory.getLayout( repository.getLayoutType() );
123             String sourcePath = sourceLayout.toPath( artifact );
124             localFile = new File( repository.getUrl().getPath(), sourcePath );
125         }
126         catch ( LayoutException e )
127         {
128             throw new ProxyException( "Unable to proxy due to bad source repository layout definition: "
129                 + e.getMessage(), e );
130         }
131
132         Properties requestProperties = new Properties();
133         requestProperties.setProperty( "version", artifact.getVersion() );
134
135         List connectors = getProxyConnectors( repository );
136         Iterator it = connectors.iterator();
137         while ( it.hasNext() )
138         {
139             ProxyConnector connector = (ProxyConnector) it.next();
140             getLogger().debug( "Attempting connector: " + connector );
141             ArchivaRepository targetRepository = connector.getTargetRepository();
142             try
143             {
144                 BidirectionalRepositoryLayout targetLayout = layoutFactory.getLayout( targetRepository.getLayoutType() );
145                 String targetPath = targetLayout.toPath( artifact );
146
147                 getLogger().debug(
148                                    "Using target repository: " + targetRepository.getId() + " - layout: "
149                                        + targetRepository.getLayoutType() + " - targetPath: " + targetPath );
150
151                 File downloadedFile = transferFile( connector, targetRepository, targetPath, localFile,
152                                                     requestProperties );
153
154                 if ( fileExists( downloadedFile ) )
155                 {
156                     getLogger().info( "Successfully transfered: " + downloadedFile.getAbsolutePath() );
157                     return downloadedFile;
158                 }
159             }
160             catch ( LayoutException e )
161             {
162                 getLogger().error( "Unable to proxy due to bad layout definition: " + e.getMessage(), e );
163                 return null;
164             }
165         }
166
167         return null;
168     }
169
170     public File fetchFromProxies( ArchivaRepository repository, ProjectReference metadata )
171         throws ProxyException
172     {
173         if ( !repository.isManaged() )
174         {
175             throw new ProxyException( "Can only proxy managed repositories." );
176         }
177
178         File localFile;
179         try
180         {
181             BidirectionalRepositoryLayout sourceLayout = layoutFactory.getLayout( repository.getLayoutType() );
182             String sourcePath = sourceLayout.toPath( metadata ) + FILENAME_MAVEN_METADATA;
183             localFile = new File( repository.getUrl().getPath(), sourcePath );
184         }
185         catch ( LayoutException e )
186         {
187             throw new ProxyException( "Unable to proxy due to bad source repository layout definition: "
188                 + e.getMessage(), e );
189         }
190
191         Properties requestProperties = new Properties();
192
193         List connectors = getProxyConnectors( repository );
194         Iterator it = connectors.iterator();
195         while ( it.hasNext() )
196         {
197             ProxyConnector connector = (ProxyConnector) it.next();
198             ArchivaRepository targetRepository = connector.getTargetRepository();
199             try
200             {
201                 BidirectionalRepositoryLayout targetLayout = layoutFactory.getLayout( targetRepository.getLayoutType() );
202                 String targetPath = targetLayout.toPath( metadata ) + FILENAME_MAVEN_METADATA;
203
204                 File downloadedFile = transferFile( connector, targetRepository, targetPath, localFile,
205                                                     requestProperties );
206
207                 if ( fileExists( downloadedFile ) )
208                 {
209                     getLogger().info( "Successfully transfered: " + downloadedFile.getAbsolutePath() );
210                     return downloadedFile;
211                 }
212             }
213             catch ( LayoutException e )
214             {
215                 getLogger().error( "Unable to proxy due to bad layout definition: " + e.getMessage(), e );
216                 return null;
217             }
218         }
219
220         return null;
221     }
222
223     private boolean fileExists( File file )
224     {
225         if ( file == null )
226         {
227             return false;
228         }
229
230         if ( !file.exists() )
231         {
232             return false;
233         }
234
235         if ( !file.isFile() )
236         {
237             return false;
238         }
239
240         return true;
241     }
242
243     /**
244      * Perform the transfer of the file.
245      * 
246      * @param connector
247      * @param targetRepository
248      * @param targetPath
249      * @param localFile
250      * @param requestProperties
251      * @return
252      * @throws ProxyException 
253      */
254     private File transferFile( ProxyConnector connector, ArchivaRepository targetRepository, String targetPath,
255                                File localFile, Properties requestProperties )
256         throws ProxyException
257     {
258         String url = targetRepository.getUrl().toString() + targetPath;
259         requestProperties.setProperty( "url", url );
260
261         // Handle pre-download policy
262         if ( !applyPolicies( connector.getPolicies(), this.preDownloadPolicies, requestProperties, localFile ) )
263         {
264             getLogger().info( "Failed pre-download policies - " + localFile.getAbsolutePath() );
265
266             if ( fileExists( localFile ) )
267             {
268                 return localFile;
269             }
270
271             return null;
272         }
273
274         // Is a whitelist defined?
275         if ( !isEmpty( connector.getWhitelist() ) )
276         {
277             // Path must belong to whitelist.
278             if ( !matchesPattern( targetPath, connector.getWhitelist() ) )
279             {
280                 getLogger().debug( "Path [" + targetPath + "] is not part of defined whitelist (skipping transfer)." );
281                 return null;
282             }
283         }
284
285         // Is target path part of blacklist?
286         if ( matchesPattern( targetPath, connector.getBlacklist() ) )
287         {
288             getLogger().debug( "Path [" + targetPath + "] is part of blacklist (skipping transfer)." );
289             return null;
290         }
291
292         Wagon wagon = null;
293         try
294         {
295             String protocol = targetRepository.getUrl().getProtocol();
296             wagon = (Wagon) wagons.get( protocol );
297             if ( wagon == null )
298             {
299                 throw new ProxyException( "Unsupported target repository protocol: " + protocol );
300             }
301
302             boolean connected = connectToRepository( connector, wagon, targetRepository );
303             if ( connected )
304             {
305                 localFile = transferSimpleFile( wagon, targetRepository, targetPath, localFile );
306
307                 transferChecksum( wagon, targetRepository, targetPath, localFile, ".sha1" );
308                 transferChecksum( wagon, targetRepository, targetPath, localFile, ".md5" );
309             }
310         }
311         catch ( ResourceDoesNotExistException e )
312         {
313             // Do not cache url here.
314             return null;
315         }
316         catch ( WagonException e )
317         {
318             urlFailureCache.cacheFailure( url );
319             return null;
320         }
321         finally
322         {
323             if ( wagon != null )
324             {
325                 try
326                 {
327                     wagon.disconnect();
328                 }
329                 catch ( ConnectionException e )
330                 {
331                     getLogger().warn( "Unable to disconnect wagon.", e );
332                 }
333             }
334         }
335
336         // Handle post-download policies.
337         if ( !applyPolicies( connector.getPolicies(), this.postDownloadPolicies, requestProperties, localFile ) )
338         {
339             getLogger().info( "Failed post-download policies - " + localFile.getAbsolutePath() );
340
341             if ( fileExists( localFile ) )
342             {
343                 return localFile;
344             }
345
346             return null;
347         }
348
349         // Everything passes.
350         return localFile;
351     }
352
353     private void transferChecksum( Wagon wagon, ArchivaRepository targetRepository, String targetPath, File localFile,
354                                    String type )
355         throws ProxyException
356     {
357         String url = targetRepository.getUrl().toString() + targetPath;
358
359         // Transfer checksum does not use the policy. 
360         if ( urlFailureCache.hasFailedBefore( url + type ) )
361         {
362             return;
363         }
364
365         try
366         {
367             File hashFile = new File( localFile.getAbsolutePath() + type );
368             transferSimpleFile( wagon, targetRepository, targetPath + type, hashFile );
369             getLogger().debug( "Checksum" + type + " Downloaded: " + hashFile );
370         }
371         catch ( ResourceDoesNotExistException e )
372         {
373             getLogger().debug( "Checksum" + type + " Not Download: " + e.getMessage() );
374         }
375         catch ( WagonException e )
376         {
377             urlFailureCache.cacheFailure( url + type );
378             getLogger().warn( "Transfer failed on checksum: " + url + " : " + e.getMessage(), e );
379         }
380     }
381
382     private File transferSimpleFile( Wagon wagon, ArchivaRepository targetRepository, String targetPath, File localFile )
383         throws ProxyException, WagonException
384     {
385         // Transfer the file.
386         File temp = null;
387
388         try
389         {
390             temp = new File( localFile.getAbsolutePath() + ".tmp" );
391
392             boolean success = false;
393
394             if ( localFile.exists() )
395             {
396                 getLogger().debug( "Retrieving " + targetPath + " from " + targetRepository.getName() );
397                 wagon.get( targetPath, temp );
398                 success = true;
399
400                 if ( temp.exists() )
401                 {
402                     moveTempToTarget( temp, localFile );
403                 }
404
405                 // You wouldn't get here on failure, a WagonException would have been thrown.
406                 getLogger().debug( "Downloaded successfully." );
407             }
408             else
409             {
410                 getLogger().debug( "Retrieving " + targetPath + " from " + targetRepository.getName() + " if updated" );
411                 success = wagon.getIfNewer( targetPath, temp, localFile.lastModified() );
412                 if ( !success )
413                 {
414                     getLogger().debug(
415                                        "Not downloaded, as local file is newer than remote side: "
416                                            + localFile.getAbsolutePath() );
417                 }
418                 else if ( temp.exists() )
419                 {
420                     getLogger().debug( "Downloaded successfully." );
421                     moveTempToTarget( temp, localFile );
422                 }
423             }
424
425             return localFile;
426         }
427         catch ( ResourceDoesNotExistException e )
428         {
429             getLogger().warn( "Resource does not exist: " + e.getMessage() );
430             throw e;
431         }
432         catch ( WagonException e )
433         {
434             getLogger().warn( "Download failure:" + e.getMessage(), e );
435             throw e;
436         }
437         finally
438         {
439             if ( temp != null )
440             {
441                 temp.delete();
442             }
443         }
444     }
445
446     private boolean applyPolicies( Properties policySettings, Map downloadPolicies, Properties request, File localFile )
447     {
448         Iterator it = downloadPolicies.entrySet().iterator();
449         while ( it.hasNext() )
450         {
451             Map.Entry entry = (Entry) it.next();
452             String key = (String) entry.getKey();
453             DownloadPolicy policy = (DownloadPolicy) entry.getValue();
454             String defaultSetting = policy.getDefaultPolicySetting();
455             String setting = policySettings.getProperty( key, defaultSetting );
456
457             getLogger().debug( "Applying [" + key + "] policy with [" + setting + "]" );
458             if ( !policy.applyPolicy( setting, request, localFile ) )
459             {
460                 getLogger().debug( "Didn't pass the [" + key + "] policy." );
461                 return false;
462             }
463         }
464         return true;
465     }
466
467     /**
468      * Used to move the temporary file to its real destination.  This is patterned from the way WagonManager handles
469      * its downloaded files.
470      *
471      * @param temp   The completed download file
472      * @param target The final location of the downloaded file
473      * @throws ProxyException when the temp file cannot replace the target file
474      */
475     private void moveTempToTarget( File temp, File target )
476         throws ProxyException
477     {
478         if ( target.exists() && !target.delete() )
479         {
480             throw new ProxyException( "Unable to overwrite existing target file: " + target.getAbsolutePath() );
481         }
482
483         if ( !temp.renameTo( target ) )
484         {
485             getLogger().warn( "Unable to rename tmp file to its final name... resorting to copy command." );
486
487             try
488             {
489                 FileUtils.copyFile( temp, target );
490             }
491             catch ( IOException e )
492             {
493                 throw new ProxyException( "Cannot copy tmp file to its final location", e );
494             }
495             finally
496             {
497                 temp.delete();
498             }
499         }
500     }
501
502     private boolean connectToRepository( ProxyConnector connector, Wagon wagon, ArchivaRepository targetRepository )
503     {
504         boolean connected = false;
505
506         ProxyInfo networkProxy = null;
507         synchronized ( this.networkProxyMap )
508         {
509             networkProxy = (ProxyInfo) this.networkProxyMap.get( connector.getProxyId() );
510         }
511
512         try
513         {
514             Repository wagonRepository = new Repository( targetRepository.getId(), targetRepository.getUrl().toString() );
515             if ( networkProxy != null )
516             {
517                 wagon.connect( wagonRepository, networkProxy );
518             }
519             else
520             {
521                 wagon.connect( wagonRepository );
522             }
523             connected = true;
524         }
525         catch ( ConnectionException e )
526         {
527             getLogger().info( "Could not connect to " + targetRepository.getName() + ": " + e.getMessage() );
528         }
529         catch ( AuthenticationException e )
530         {
531             getLogger().info( "Could not connect to " + targetRepository.getName() + ": " + e.getMessage() );
532         }
533
534         return connected;
535     }
536
537     private boolean matchesPattern( String path, List patterns )
538     {
539         if ( isEmpty( patterns ) )
540         {
541             return false;
542         }
543
544         Iterator it = patterns.iterator();
545         while ( it.hasNext() )
546         {
547             String pattern = (String) it.next();
548             if ( SelectorUtils.matchPath( pattern, path, false ) )
549             {
550                 return true;
551             }
552         }
553
554         return false;
555     }
556
557     public List getProxyConnectors( ArchivaRepository repository )
558     {
559         synchronized ( this.proxyConnectorMap )
560         {
561             List ret = (List) this.proxyConnectorMap.get( repository.getId() );
562             if ( ret == null )
563             {
564                 return Collections.EMPTY_LIST;
565             }
566             return ret;
567         }
568     }
569
570     public boolean hasProxies( ArchivaRepository repository )
571     {
572         synchronized ( this.proxyConnectorMap )
573         {
574             return this.proxyConnectorMap.containsKey( repository.getId() );
575         }
576     }
577
578     public void afterConfigurationChange( Registry registry, String propertyName, Object propertyValue )
579     {
580         if ( propertyNameTriggers.contains( propertyName ) )
581         {
582             initConnectorsAndNetworkProxies();
583         }
584     }
585
586     public void beforeConfigurationChange( Registry registry, String propertyName, Object propertyValue )
587     {
588         /* do nothing */
589     }
590
591     private void initConnectorsAndNetworkProxies()
592     {
593         Iterator it;
594
595         synchronized ( this.proxyConnectorMap )
596         {
597             this.proxyConnectorMap.clear();
598
599             List proxyConfigs = archivaConfiguration.getConfiguration().getProxyConnectors();
600             it = proxyConfigs.iterator();
601             while ( it.hasNext() )
602             {
603                 ProxyConnectorConfiguration proxyConfig = (ProxyConnectorConfiguration) it.next();
604                 String key = proxyConfig.getSourceRepoId();
605
606                 // Create connector object.
607                 ProxyConnector connector = new ProxyConnector();
608                 connector.setSourceRepository( getRepository( proxyConfig.getSourceRepoId() ) );
609                 connector.setTargetRepository( getRepository( proxyConfig.getTargetRepoId() ) );
610                 connector.setPolicies( proxyConfig.getPolicies() );
611
612                 // Copy any blacklist patterns.
613                 List blacklist = new ArrayList();
614                 if ( !isEmpty( proxyConfig.getBlackListPatterns() ) )
615                 {
616                     blacklist.addAll( proxyConfig.getBlackListPatterns() );
617                 }
618                 connector.setBlacklist( blacklist );
619
620                 // Copy any whitelist patterns.
621                 List whitelist = new ArrayList();
622                 if ( !isEmpty( proxyConfig.getWhiteListPatterns() ) )
623                 {
624                     whitelist.addAll( proxyConfig.getWhiteListPatterns() );
625                 }
626                 connector.setWhitelist( whitelist );
627
628                 // Get other connectors
629                 List connectors = (List) this.proxyConnectorMap.get( key );
630                 if ( connectors == null )
631                 {
632                     // Create if we are the first.
633                     connectors = new ArrayList();
634                 }
635
636                 // Add the connector.
637                 connectors.add( connector );
638
639                 // Set the key to the list of connectors.
640                 this.proxyConnectorMap.put( key, connectors );
641             }
642         }
643
644         synchronized ( this.networkProxyMap )
645         {
646             this.networkProxyMap.clear();
647
648             List networkProxies = archivaConfiguration.getConfiguration().getNetworkProxies();
649             it = networkProxies.iterator();
650             while ( it.hasNext() )
651             {
652                 NetworkProxyConfiguration networkProxyConfig = (NetworkProxyConfiguration) it.next();
653                 String key = networkProxyConfig.getId();
654
655                 ProxyInfo proxy = new ProxyInfo();
656
657                 proxy.setType( networkProxyConfig.getProtocol() );
658                 proxy.setHost( networkProxyConfig.getHost() );
659                 proxy.setPort( networkProxyConfig.getPort() );
660                 proxy.setUserName( networkProxyConfig.getUsername() );
661                 proxy.setPassword( networkProxyConfig.getPassword() );
662
663                 this.networkProxyMap.put( key, proxy );
664             }
665         }
666     }
667
668     private boolean isEmpty( Collection collection )
669     {
670         if ( collection == null )
671         {
672             return true;
673         }
674
675         return collection.isEmpty();
676     }
677
678     private ArchivaRepository getRepository( String repoId )
679     {
680         RepositoryConfiguration repoConfig = archivaConfiguration.getConfiguration().findRepositoryById( repoId );
681         if ( repoConfig == null )
682         {
683             return null;
684         }
685
686         ArchivaRepository repo = new ArchivaRepository( repoConfig.getId(), repoConfig.getName(), repoConfig.getUrl() );
687         repo.getModel().setLayoutName( repoConfig.getLayout() );
688         return repo;
689     }
690
691     public void initialize()
692         throws InitializationException
693     {
694         propertyNameTriggers.add( "repositories" );
695         propertyNameTriggers.add( "repository" );
696         propertyNameTriggers.add( "id" );
697         propertyNameTriggers.add( "name" );
698         propertyNameTriggers.add( "url" );
699         propertyNameTriggers.add( "layout" );
700         propertyNameTriggers.add( "releases" );
701         propertyNameTriggers.add( "snapshots" );
702         propertyNameTriggers.add( "indexed" );
703
704         propertyNameTriggers.add( "proxyConnectors" );
705         propertyNameTriggers.add( "proxyConnector" );
706         propertyNameTriggers.add( "sourceRepoId" );
707         propertyNameTriggers.add( "targetRepoId" );
708         propertyNameTriggers.add( "proxyId" );
709         propertyNameTriggers.add( "snapshotsPolicy" );
710         propertyNameTriggers.add( "releasePolicy" );
711         propertyNameTriggers.add( "checksumPolicy" );
712         propertyNameTriggers.add( "whiteListPatterns" );
713         propertyNameTriggers.add( "whiteListPattern" );
714         propertyNameTriggers.add( "blackListPatterns" );
715         propertyNameTriggers.add( "blackListPattern" );
716
717         propertyNameTriggers.add( "networkProxies" );
718         propertyNameTriggers.add( "networkProxy" );
719         propertyNameTriggers.add( "protocol" );
720         propertyNameTriggers.add( "host" );
721         propertyNameTriggers.add( "port" );
722         propertyNameTriggers.add( "username" );
723         propertyNameTriggers.add( "password" );
724
725         archivaConfiguration.addChangeListener( this );
726         initConnectorsAndNetworkProxies();
727     }
728 }