1 package org.apache.maven.archiva.repository.metadata;
4 * Licensed to the Apache Software Foundation (ASF) under one
5 * or more contributor license agreements. See the NOTICE file
6 * distributed with this work for additional information
7 * regarding copyright ownership. The ASF licenses this file
8 * to you under the Apache License, Version 2.0 (the
9 * "License"); you may not use this file except in compliance
10 * with the License. You may obtain a copy of the License at
12 * http://www.apache.org/licenses/LICENSE-2.0
14 * Unless required by applicable law or agreed to in writing,
15 * software distributed under the License is distributed on an
16 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17 * KIND, either express or implied. See the License for the
18 * specific language governing permissions and limitations
22 import org.apache.archiva.checksum.ChecksumAlgorithm;
23 import org.apache.archiva.checksum.ChecksummedFile;
24 import org.apache.commons.collections.CollectionUtils;
25 import org.apache.commons.lang.StringUtils;
26 import org.apache.commons.lang.math.NumberUtils;
27 import org.apache.commons.lang.time.DateUtils;
28 import org.apache.maven.archiva.common.utils.PathUtil;
29 import org.apache.maven.archiva.common.utils.VersionComparator;
30 import org.apache.maven.archiva.common.utils.VersionUtil;
31 import org.apache.maven.archiva.configuration.ArchivaConfiguration;
32 import org.apache.maven.archiva.configuration.ConfigurationNames;
33 import org.apache.maven.archiva.configuration.FileTypes;
34 import org.apache.maven.archiva.configuration.ProxyConnectorConfiguration;
35 import org.apache.maven.archiva.model.ArchivaRepositoryMetadata;
36 import org.apache.maven.archiva.model.ArtifactReference;
37 import org.apache.maven.archiva.model.Plugin;
38 import org.apache.maven.archiva.model.ProjectReference;
39 import org.apache.maven.archiva.model.SnapshotVersion;
40 import org.apache.maven.archiva.model.VersionedReference;
41 import org.apache.maven.archiva.repository.ContentNotFoundException;
42 import org.apache.maven.archiva.repository.ManagedRepositoryContent;
43 import org.apache.maven.archiva.repository.RemoteRepositoryContent;
44 import org.apache.maven.archiva.repository.layout.LayoutException;
45 import org.codehaus.plexus.personality.plexus.lifecycle.phase.Initializable;
46 import org.codehaus.plexus.personality.plexus.lifecycle.phase.InitializationException;
47 import org.codehaus.plexus.registry.Registry;
48 import org.codehaus.plexus.registry.RegistryListener;
49 import org.slf4j.Logger;
50 import org.slf4j.LoggerFactory;
53 import java.io.IOException;
54 import java.text.ParseException;
55 import java.text.SimpleDateFormat;
56 import java.util.ArrayList;
57 import java.util.Calendar;
58 import java.util.Collections;
59 import java.util.Date;
60 import java.util.HashMap;
61 import java.util.HashSet;
62 import java.util.Iterator;
63 import java.util.LinkedHashSet;
64 import java.util.List;
67 import java.util.regex.Matcher;
72 * @author <a href="mailto:joakime@apache.org">Joakim Erdfelt</a>
75 * @plexus.component role="org.apache.maven.archiva.repository.metadata.MetadataTools"
77 public class MetadataTools
78 implements RegistryListener, Initializable
80 private static Logger log = LoggerFactory.getLogger( MetadataTools.class );
82 public static final String MAVEN_METADATA = "maven-metadata.xml";
84 private static final char PATH_SEPARATOR = '/';
86 private static final char GROUP_SEPARATOR = '.';
91 private ArchivaConfiguration configuration;
96 private FileTypes filetypes;
98 private ChecksumAlgorithm[] algorithms = new ChecksumAlgorithm[] { ChecksumAlgorithm.SHA1, ChecksumAlgorithm.MD5 };
100 private List<String> artifactPatterns;
102 private Map<String, Set<String>> proxies;
104 private static final char NUMS[] = new char[] { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' };
106 private SimpleDateFormat lastUpdatedFormat;
108 public MetadataTools()
110 lastUpdatedFormat = new SimpleDateFormat( "yyyyMMddHHmmss" );
111 lastUpdatedFormat.setTimeZone( DateUtils.UTC_TIME_ZONE );
114 public void afterConfigurationChange( Registry registry, String propertyName, Object propertyValue )
116 if ( ConfigurationNames.isProxyConnector( propertyName ) )
118 initConfigVariables();
122 public void beforeConfigurationChange( Registry registry, String propertyName, Object propertyValue )
128 * Gather the set of snapshot versions found in a particular versioned reference.
130 * @return the Set of snapshot artifact versions found.
131 * @throws LayoutException
132 * @throws ContentNotFoundException
134 public Set<String> gatherSnapshotVersions( ManagedRepositoryContent managedRepository, VersionedReference reference )
135 throws LayoutException, IOException, ContentNotFoundException
137 Set<String> foundVersions = managedRepository.getVersions( reference );
139 // Next gather up the referenced 'latest' versions found in any proxied repositories
140 // maven-metadata-${proxyId}.xml files that may be present.
142 // Does this repository have a set of remote proxied repositories?
143 Set<String> proxiedRepoIds = this.proxies.get( managedRepository.getId() );
145 if ( CollectionUtils.isNotEmpty( proxiedRepoIds ) )
147 String baseVersion = VersionUtil.getBaseVersion( reference.getVersion() );
148 baseVersion = baseVersion.substring( 0, baseVersion.indexOf( VersionUtil.SNAPSHOT ) - 1 );
150 // Add in the proxied repo version ids too.
151 Iterator<String> it = proxiedRepoIds.iterator();
152 while ( it.hasNext() )
154 String proxyId = it.next();
156 ArchivaRepositoryMetadata proxyMetadata = readProxyMetadata( managedRepository, reference, proxyId );
157 if ( proxyMetadata == null )
159 // There is no proxy metadata, skip it.
163 // Is there some snapshot info?
164 SnapshotVersion snapshot = proxyMetadata.getSnapshotVersion();
165 if ( snapshot != null )
167 String timestamp = snapshot.getTimestamp();
168 int buildNumber = snapshot.getBuildNumber();
170 // Only interested in the timestamp + buildnumber.
171 if ( StringUtils.isNotBlank( timestamp ) && ( buildNumber > 0 ) )
173 foundVersions.add( baseVersion + "-" + timestamp + "-" + buildNumber );
179 return foundVersions;
183 * Take a path to a maven-metadata.xml, and attempt to translate it to a VersionedReference.
188 public VersionedReference toVersionedReference( String path )
189 throws RepositoryMetadataException
191 if ( !path.endsWith( "/" + MAVEN_METADATA ) )
193 throw new RepositoryMetadataException( "Cannot convert to versioned reference, not a metadata file. " );
196 VersionedReference reference = new VersionedReference();
198 String normalizedPath = StringUtils.replace( path, "\\", "/" );
199 String pathParts[] = StringUtils.split( normalizedPath, '/' );
201 int versionOffset = pathParts.length - 2;
202 int artifactIdOffset = versionOffset - 1;
203 int groupIdEnd = artifactIdOffset - 1;
205 reference.setVersion( pathParts[versionOffset] );
207 if ( !hasNumberAnywhere( reference.getVersion() ) )
209 // Scary check, but without it, all paths are version references;
210 throw new RepositoryMetadataException(
211 "Not a versioned reference, as version id on path has no number in it." );
214 reference.setArtifactId( pathParts[artifactIdOffset] );
216 StringBuffer gid = new StringBuffer();
217 for ( int i = 0; i <= groupIdEnd; i++ )
223 gid.append( pathParts[i] );
226 reference.setGroupId( gid.toString() );
231 private boolean hasNumberAnywhere( String version )
233 return StringUtils.indexOfAny( version, NUMS ) != ( -1 );
236 public ProjectReference toProjectReference( String path )
237 throws RepositoryMetadataException
239 if ( !path.endsWith( "/" + MAVEN_METADATA ) )
241 throw new RepositoryMetadataException( "Cannot convert to versioned reference, not a metadata file. " );
244 ProjectReference reference = new ProjectReference();
246 String normalizedPath = StringUtils.replace( path, "\\", "/" );
247 String pathParts[] = StringUtils.split( normalizedPath, '/' );
249 // Assume last part of the path is the version.
251 int artifactIdOffset = pathParts.length - 2;
252 int groupIdEnd = artifactIdOffset - 1;
254 reference.setArtifactId( pathParts[artifactIdOffset] );
256 StringBuffer gid = new StringBuffer();
257 for ( int i = 0; i <= groupIdEnd; i++ )
263 gid.append( pathParts[i] );
266 reference.setGroupId( gid.toString() );
271 public String toPath( ProjectReference reference )
273 StringBuffer path = new StringBuffer();
275 path.append( formatAsDirectory( reference.getGroupId() ) ).append( PATH_SEPARATOR );
276 path.append( reference.getArtifactId() ).append( PATH_SEPARATOR );
277 path.append( MAVEN_METADATA );
279 return path.toString();
282 public String toPath( VersionedReference reference )
284 StringBuffer path = new StringBuffer();
286 path.append( formatAsDirectory( reference.getGroupId() ) ).append( PATH_SEPARATOR );
287 path.append( reference.getArtifactId() ).append( PATH_SEPARATOR );
288 if ( reference.getVersion() != null )
290 // add the version only if it is present
291 path.append( VersionUtil.getBaseVersion( reference.getVersion() ) ).append( PATH_SEPARATOR );
293 path.append( MAVEN_METADATA );
295 return path.toString();
298 private String formatAsDirectory( String directory )
300 return directory.replace( GROUP_SEPARATOR, PATH_SEPARATOR );
304 * Adjusts a path for a metadata.xml file to its repository specific path.
306 * @param repository the repository to base new path off of.
307 * @param path the path to the metadata.xml file to adjust the name of.
308 * @return the newly adjusted path reference to the repository specific metadata path.
310 public String getRepositorySpecificName( RemoteRepositoryContent repository, String path )
312 return getRepositorySpecificName( repository.getId(), path );
316 * Adjusts a path for a metadata.xml file to its repository specific path.
318 * @param proxyId the repository id to base new path off of.
319 * @param path the path to the metadata.xml file to adjust the name of.
320 * @return the newly adjusted path reference to the repository specific metadata path.
322 public String getRepositorySpecificName( String proxyId, String path )
324 StringBuffer ret = new StringBuffer();
326 int idx = path.lastIndexOf( "/" );
329 ret.append( path.substring( 0, idx + 1 ) );
332 // TODO: need to filter out 'bad' characters from the proxy id.
333 ret.append( "maven-metadata-" ).append( proxyId ).append( ".xml" );
335 return ret.toString();
338 public void initialize()
339 throws InitializationException
341 this.artifactPatterns = new ArrayList<String>();
342 this.proxies = new HashMap<String, Set<String>>();
343 initConfigVariables();
345 configuration.addChangeListener( this );
348 public ArchivaRepositoryMetadata readProxyMetadata( ManagedRepositoryContent managedRepository,
349 ProjectReference reference, String proxyId )
351 String metadataPath = getRepositorySpecificName( proxyId, toPath( reference ) );
352 File metadataFile = new File( managedRepository.getRepoRoot(), metadataPath );
354 if ( !metadataFile.exists() || !metadataFile.isFile() )
356 // Nothing to do. return null.
362 return RepositoryMetadataReader.read( metadataFile );
364 catch ( RepositoryMetadataException e )
366 // TODO: [monitor] consider a monitor for this event.
367 // TODO: consider a read-redo on monitor return code?
368 log.warn( "Unable to read metadata: " + metadataFile.getAbsolutePath(), e );
373 public ArchivaRepositoryMetadata readProxyMetadata( ManagedRepositoryContent managedRepository,
374 VersionedReference reference, String proxyId )
376 String metadataPath = getRepositorySpecificName( proxyId, toPath( reference ) );
377 File metadataFile = new File( managedRepository.getRepoRoot(), metadataPath );
379 if ( !metadataFile.exists() || !metadataFile.isFile() )
381 // Nothing to do. return null.
387 return RepositoryMetadataReader.read( metadataFile );
389 catch ( RepositoryMetadataException e )
391 // TODO: [monitor] consider a monitor for this event.
392 // TODO: consider a read-redo on monitor return code?
393 log.warn( "Unable to read metadata: " + metadataFile.getAbsolutePath(), e );
399 * Update the metadata to represent the all versions/plugins of
400 * the provided groupId:artifactId project or group reference,
401 * based off of information present in the repository,
402 * the maven-metadata.xml files, and the proxy/repository specific
403 * metadata file contents.
405 * We must treat this as a group or a project metadata file as there is no way to know in advance
407 * @param managedRepository the managed repository where the metadata is kept.
408 * @param reference the reference to update.
409 * @throws LayoutException
410 * @throws RepositoryMetadataException
411 * @throws IOException
412 * @throws ContentNotFoundException
414 public void updateMetadata( ManagedRepositoryContent managedRepository, ProjectReference reference )
415 throws LayoutException, RepositoryMetadataException, IOException, ContentNotFoundException
417 File metadataFile = new File( managedRepository.getRepoRoot(), toPath( reference ) );
419 long lastUpdated = getExistingLastUpdated( metadataFile );
421 ArchivaRepositoryMetadata metadata = new ArchivaRepositoryMetadata();
422 metadata.setGroupId( reference.getGroupId() );
423 metadata.setArtifactId( reference.getArtifactId() );
425 // Gather up all versions found in the managed repository.
426 Set<String> allVersions = managedRepository.getVersions( reference );
428 // Gather up all plugins found in the managed repository.
429 // TODO: do we know this information instead?
430 // Set<Plugin> allPlugins = managedRepository.getPlugins( reference );
431 Set<Plugin> allPlugins;
432 if ( metadataFile.exists() )
434 allPlugins = new LinkedHashSet<Plugin>( RepositoryMetadataReader.read( metadataFile ).getPlugins() );
438 allPlugins = new LinkedHashSet<Plugin>();
441 // Does this repository have a set of remote proxied repositories?
442 Set<String> proxiedRepoIds = this.proxies.get( managedRepository.getId() );
444 if ( CollectionUtils.isNotEmpty( proxiedRepoIds ) )
446 // Add in the proxied repo version ids too.
447 Iterator<String> it = proxiedRepoIds.iterator();
448 while ( it.hasNext() )
450 String proxyId = it.next();
452 ArchivaRepositoryMetadata proxyMetadata = readProxyMetadata( managedRepository, reference, proxyId );
453 if ( proxyMetadata != null )
455 allVersions.addAll( proxyMetadata.getAvailableVersions() );
456 allPlugins.addAll( proxyMetadata.getPlugins() );
457 long proxyLastUpdated = getLastUpdated( proxyMetadata );
459 lastUpdated = Math.max( lastUpdated, proxyLastUpdated );
464 if ( !allVersions.isEmpty() )
467 List<String> sortedVersions = new ArrayList<String>( allVersions );
468 Collections.sort( sortedVersions, VersionComparator.getInstance() );
470 // Split the versions into released and snapshots.
471 List<String> releasedVersions = new ArrayList<String>();
472 List<String> snapshotVersions = new ArrayList<String>();
474 for ( String version : sortedVersions )
476 if ( VersionUtil.isSnapshot( version ) )
478 snapshotVersions.add( version );
482 releasedVersions.add( version );
486 Collections.sort( releasedVersions, VersionComparator.getInstance() );
487 Collections.sort( snapshotVersions, VersionComparator.getInstance() );
489 String latestVersion = sortedVersions.get( sortedVersions.size() - 1 );
490 String releaseVersion = null;
492 if ( CollectionUtils.isNotEmpty( releasedVersions ) )
494 releaseVersion = releasedVersions.get( releasedVersions.size() - 1 );
497 // Add the versions to the metadata model.
498 metadata.setAvailableVersions( sortedVersions );
500 metadata.setLatestVersion( latestVersion );
501 metadata.setReleasedVersion( releaseVersion );
505 // Add the plugins to the metadata model.
506 metadata.setPlugins( new ArrayList<Plugin>( allPlugins ) );
508 // artifact ID was actually the last part of the group
509 metadata.setGroupId( metadata.getGroupId() + "." + metadata.getArtifactId() );
510 metadata.setArtifactId( null );
513 if ( lastUpdated > 0 )
515 metadata.setLastUpdatedTimestamp( toLastUpdatedDate( lastUpdated ) );
518 // Save the metadata model to disk.
519 RepositoryMetadataWriter.write( metadata, metadataFile );
520 ChecksummedFile checksum = new ChecksummedFile( metadataFile );
521 checksum.fixChecksums( algorithms );
524 private Date toLastUpdatedDate( long lastUpdated )
526 Calendar cal = Calendar.getInstance( DateUtils.UTC_TIME_ZONE );
527 cal.setTimeInMillis( lastUpdated );
529 return cal.getTime();
532 private long toLastUpdatedLong( String timestampString )
536 Date date = lastUpdatedFormat.parse( timestampString );
537 Calendar cal = Calendar.getInstance( DateUtils.UTC_TIME_ZONE );
540 return cal.getTimeInMillis();
542 catch ( ParseException e )
548 private long getLastUpdated( ArchivaRepositoryMetadata metadata )
550 if ( metadata == null )
558 String lastUpdated = metadata.getLastUpdated();
559 if ( StringUtils.isBlank( lastUpdated ) )
565 Date lastUpdatedDate = lastUpdatedFormat.parse( lastUpdated );
566 return lastUpdatedDate.getTime();
568 catch ( ParseException e )
570 // Bad format on the last updated string.
575 private long getExistingLastUpdated( File metadataFile )
577 if ( !metadataFile.exists() )
585 ArchivaRepositoryMetadata metadata = RepositoryMetadataReader.read( metadataFile );
587 return getLastUpdated( metadata );
589 catch ( RepositoryMetadataException e )
597 * Update the metadata based on the following rules.
599 * 1) If this is a SNAPSHOT reference, then utilize the proxy/repository specific
600 * metadata files to represent the current / latest SNAPSHOT available.
601 * 2) If this is a RELEASE reference, and the metadata file does not exist, then
602 * create the metadata file with contents required of the VersionedReference
604 * @param managedRepository the managed repository where the metadata is kept.
605 * @param reference the versioned reference to update
606 * @throws LayoutException
607 * @throws RepositoryMetadataException
608 * @throws IOException
609 * @throws ContentNotFoundException
611 public void updateMetadata( ManagedRepositoryContent managedRepository, VersionedReference reference )
612 throws LayoutException, RepositoryMetadataException, IOException, ContentNotFoundException
614 File metadataFile = new File( managedRepository.getRepoRoot(), toPath( reference ) );
616 long lastUpdated = getExistingLastUpdated( metadataFile );
618 ArchivaRepositoryMetadata metadata = new ArchivaRepositoryMetadata();
619 metadata.setGroupId( reference.getGroupId() );
620 metadata.setArtifactId( reference.getArtifactId() );
622 if ( VersionUtil.isSnapshot( reference.getVersion() ) )
624 // Do SNAPSHOT handling.
625 metadata.setVersion( VersionUtil.getBaseVersion( reference.getVersion() ) );
627 // Gather up all of the versions found in the reference dir, and any
628 // proxied maven-metadata.xml files.
629 Set<String> snapshotVersions = gatherSnapshotVersions( managedRepository, reference );
631 if ( snapshotVersions.isEmpty() )
633 throw new ContentNotFoundException( "No snapshot versions found on reference ["
634 + VersionedReference.toKey( reference ) + "]." );
637 // sort the list to determine to aide in determining the Latest version.
638 List<String> sortedVersions = new ArrayList<String>();
639 sortedVersions.addAll( snapshotVersions );
640 Collections.sort( sortedVersions, new VersionComparator() );
642 String latestVersion = sortedVersions.get( sortedVersions.size() - 1 );
644 if ( VersionUtil.isUniqueSnapshot( latestVersion ) )
646 // The latestVersion will contain the full version string "1.0-alpha-5-20070821.213044-8"
647 // This needs to be broken down into ${base}-${timestamp}-${build_number}
649 Matcher m = VersionUtil.UNIQUE_SNAPSHOT_PATTERN.matcher( latestVersion );
652 metadata.setSnapshotVersion( new SnapshotVersion() );
653 int buildNumber = NumberUtils.toInt( m.group( 3 ), -1 );
654 metadata.getSnapshotVersion().setBuildNumber( buildNumber );
656 Matcher mtimestamp = VersionUtil.TIMESTAMP_PATTERN.matcher( m.group( 2 ) );
657 if ( mtimestamp.matches() )
659 String tsDate = mtimestamp.group( 1 );
660 String tsTime = mtimestamp.group( 2 );
662 long snapshotLastUpdated = toLastUpdatedLong( tsDate + tsTime );
664 lastUpdated = Math.max( lastUpdated, snapshotLastUpdated );
666 metadata.getSnapshotVersion().setTimestamp( m.group( 2 ) );
670 else if ( VersionUtil.isGenericSnapshot( latestVersion ) )
672 // The latestVersion ends with the generic version string.
673 // Example: 1.0-alpha-5-SNAPSHOT
675 metadata.setSnapshotVersion( new SnapshotVersion() );
677 /* Disabled due to decision in [MRM-535].
678 * Do not set metadata.lastUpdated to file.lastModified.
680 * Should this be the last updated timestamp of the file, or in the case of an
681 * archive, the most recent timestamp in the archive?
683 ArtifactReference artifact = getFirstArtifact( managedRepository, reference );
685 if ( artifact == null )
687 throw new IOException( "Not snapshot artifact found to reference in " + reference );
690 File artifactFile = managedRepository.toFile( artifact );
692 if ( artifactFile.exists() )
694 Date lastModified = new Date( artifactFile.lastModified() );
695 metadata.setLastUpdatedTimestamp( lastModified );
701 throw new RepositoryMetadataException( "Unable to process snapshot version <" + latestVersion
702 + "> reference <" + reference + ">" );
707 // Do RELEASE handling.
708 metadata.setVersion( reference.getVersion() );
712 if ( lastUpdated > 0 )
714 metadata.setLastUpdatedTimestamp( toLastUpdatedDate( lastUpdated ) );
717 // Save the metadata model to disk.
718 RepositoryMetadataWriter.write( metadata, metadataFile );
719 ChecksummedFile checksum = new ChecksummedFile( metadataFile );
720 checksum.fixChecksums( algorithms );
723 private void initConfigVariables()
725 synchronized ( this.artifactPatterns )
727 this.artifactPatterns.clear();
729 this.artifactPatterns.addAll( filetypes.getFileTypePatterns( FileTypes.ARTIFACTS ) );
732 synchronized ( proxies )
734 this.proxies.clear();
736 List<ProxyConnectorConfiguration> proxyConfigs = configuration.getConfiguration().getProxyConnectors();
737 for( ProxyConnectorConfiguration proxyConfig: proxyConfigs )
739 String key = proxyConfig.getSourceRepoId();
741 Set<String> remoteRepoIds = this.proxies.get( key );
743 if ( remoteRepoIds == null )
745 remoteRepoIds = new HashSet<String>();
748 remoteRepoIds.add( proxyConfig.getTargetRepoId() );
750 this.proxies.put( key, remoteRepoIds );
756 * Get the first Artifact found in the provided VersionedReference location.
758 * @param managedRepository the repository to search within.
759 * @param reference the reference to the versioned reference to search within
760 * @return the ArtifactReference to the first artifact located within the versioned reference. or null if
761 * no artifact was found within the versioned reference.
762 * @throws IOException if the versioned reference is invalid (example: doesn't exist, or isn't a directory)
763 * @throws LayoutException
765 public ArtifactReference getFirstArtifact( ManagedRepositoryContent managedRepository, VersionedReference reference )
766 throws LayoutException, IOException
768 String path = toPath( reference );
770 int idx = path.lastIndexOf( '/' );
773 path = path.substring( 0, idx );
776 File repoDir = new File( managedRepository.getRepoRoot(), path );
778 if ( !repoDir.exists() )
780 throw new IOException( "Unable to gather the list of snapshot versions on a non-existant directory: "
781 + repoDir.getAbsolutePath() );
784 if ( !repoDir.isDirectory() )
786 throw new IOException( "Unable to gather the list of snapshot versions on a non-directory: "
787 + repoDir.getAbsolutePath() );
790 File repoFiles[] = repoDir.listFiles();
791 for ( int i = 0; i < repoFiles.length; i++ )
793 if ( repoFiles[i].isDirectory() )
795 // Skip it. it's a directory.
799 String relativePath = PathUtil.getRelative( managedRepository.getRepoRoot(), repoFiles[i] );
801 if ( filetypes.matchesArtifactPattern( relativePath ) )
803 ArtifactReference artifact = managedRepository.toArtifactReference( relativePath );
809 // No artifact was found.