source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.repository.maven.metadata.storage;
   2
   3 /*
   4  * Licensed to the Apache Software Foundation (ASF) under one
   5  * or more contributor license agreements.  See the NOTICE file
   6  * distributed with this work for additional information
   7  * regarding copyright ownership.  The ASF licenses this file
   8  * to you under the Apache License, Version 2.0 (the
   9  * "License"); you may not use this file except in compliance
  10  * with the License.  You may obtain a copy of the License at
  11  *
  12  * http://www.apache.org/licenses/LICENSE-2.0
  13  * Unless required by applicable law or agreed to in writing,
  14  * software distributed under the License is distributed on an
  15  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  16  * KIND, either express or implied.  See the License for the
  17  * specific language governing permissions and limitations
  18  * under the License.
  19  */
  20
  21 import org.apache.archiva.checksum.ChecksumAlgorithm;
  22 import org.apache.archiva.checksum.ChecksummedFile;
  23 import org.apache.archiva.common.Try;
  24 import org.apache.archiva.common.utils.VersionUtil;
  25 import org.apache.archiva.filter.Filter;
  26 import org.apache.archiva.metadata.maven.MavenMetadataReader;
  27 import org.apache.archiva.metadata.model.ArtifactMetadata;
  28 import org.apache.archiva.metadata.model.ProjectMetadata;
  29 import org.apache.archiva.metadata.model.ProjectVersionMetadata;
  30 import org.apache.archiva.metadata.model.facets.RepositoryProblemFacet;
  31 import org.apache.archiva.metadata.repository.storage.*;
  32 import org.apache.archiva.model.ArchivaRepositoryMetadata;
  33 import org.apache.archiva.model.ArtifactReference;
  34 import org.apache.archiva.model.SnapshotVersion;
  35 import org.apache.archiva.policies.ProxyDownloadException;
  36 import org.apache.archiva.proxy.ProxyRegistry;
  37 import org.apache.archiva.proxy.maven.WagonFactory;
  38 import org.apache.archiva.proxy.model.NetworkProxy;
  39 import org.apache.archiva.proxy.model.ProxyConnector;
  40 import org.apache.archiva.proxy.model.RepositoryProxyHandler;
  41 import org.apache.archiva.repository.*;
  42 import org.apache.archiva.repository.content.PathParser;
  43 import org.apache.archiva.repository.maven.MavenSystemManager;
  44 import org.apache.archiva.repository.metadata.RepositoryMetadataException;
  45 import org.apache.archiva.repository.storage.StorageAsset;
  46 import org.apache.commons.lang3.ArrayUtils;
  47 import org.apache.commons.lang3.StringUtils;
  48 import org.apache.maven.model.*;
  49 import org.apache.maven.model.building.*;
  50 import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
  51 import org.codehaus.plexus.util.xml.pull.XmlPullParserException;
  52 import org.slf4j.Logger;
  53 import org.slf4j.LoggerFactory;
  54 import org.springframework.context.ApplicationContext;
  55 import org.springframework.stereotype.Service;
  56
  57 import javax.annotation.PostConstruct;
  58 import javax.inject.Inject;
  59 import javax.inject.Named;
  60 import java.io.FileNotFoundException;
  61 import java.io.IOException;
  62 import java.io.Reader;
  63 import java.nio.channels.Channels;
  64 import java.nio.charset.Charset;
  65 import java.nio.file.NoSuchFileException;
  66 import java.time.ZoneId;
  67 import java.time.ZonedDateTime;
  68 import java.util.*;
  69 import java.util.function.Predicate;
  70 import java.util.stream.Collectors;
  71
  72 // import java.io.FileNotFoundException;
  73
  74 /**
  75  * <p>
  76  * Maven 2 repository format storage implementation. This class currently takes parameters to indicate the repository to
  77  * deal with rather than being instantiated per-repository.
  78  * FIXME: instantiate one per repository and allocate permanently from a factory (which can be obtained within the session).
  79  * </p>
  80  * <p>
  81  * The session is passed in as an argument to obtain any necessary resources, rather than the class being instantiated
  82  * within the session in the context of a single managed repository's resolution needs.
  83  * </p>
  84  */
  85 @Service("repositoryStorage#maven2")
  86 public class Maven2RepositoryStorage
  87         implements RepositoryStorage {
  88
  89     private static final Logger log = LoggerFactory.getLogger(Maven2RepositoryStorage.class);
  90
  91     private ModelBuilder builder;
  92
  93     @Inject
  94     RepositoryRegistry repositoryRegistry;
  95
  96     @Inject
  97     @Named( "metadataReader#maven" )
  98     MavenMetadataReader metadataReader;
  99
 100     @Inject
 101     @Named("repositoryPathTranslator#maven2")
 102     private RepositoryPathTranslator pathTranslator;
 103
 104     @Inject
 105     private WagonFactory wagonFactory;
 106
 107     @Inject
 108     private ApplicationContext applicationContext;
 109
 110     @Inject
 111     @Named("pathParser#default")
 112     private PathParser pathParser;
 113
 114     @Inject
 115     private ProxyRegistry proxyRegistry;
 116
 117     @Inject
 118     private MavenSystemManager mavenSystemManager;
 119
 120     private static final String METADATA_FILENAME_START = "maven-metadata";
 121
 122     private static final String METADATA_FILENAME = METADATA_FILENAME_START + ".xml";
 123
 124     // This array must be lexically sorted
 125     private static final String[] IGNORED_FILES = {METADATA_FILENAME, "resolver-status.properties"};
 126
 127     private static final MavenXpp3Reader MAVEN_XPP_3_READER = new MavenXpp3Reader();
 128
 129
 130     @PostConstruct
 131     public void initialize() {
 132         builder = new DefaultModelBuilderFactory().newInstance();
 133
 134     }
 135
 136     @Override
 137     public ProjectMetadata readProjectMetadata(String repoId, String namespace, String projectId) {
 138         // TODO: could natively implement the "shared model" concept from the browse action to avoid needing it there?
 139         return null;
 140     }
 141
 142     @Override
 143     public ProjectVersionMetadata readProjectVersionMetadata(ReadMetadataRequest readMetadataRequest)
 144             throws RepositoryStorageMetadataNotFoundException, RepositoryStorageMetadataInvalidException,
 145             RepositoryStorageRuntimeException {
 146
 147         ManagedRepository managedRepository = repositoryRegistry.getManagedRepository(readMetadataRequest.getRepositoryId());
 148         boolean isReleases = managedRepository.getActiveReleaseSchemes().contains(ReleaseScheme.RELEASE);
 149         boolean isSnapshots = managedRepository.getActiveReleaseSchemes().contains(ReleaseScheme.SNAPSHOT);
 150         String artifactVersion = readMetadataRequest.getProjectVersion();
 151         // olamy: in case of browsing via the ui we can mix repos (parent of a SNAPSHOT can come from release repo)
 152         if (!readMetadataRequest.isBrowsingRequest()) {
 153             if (VersionUtil.isSnapshot(artifactVersion)) {
 154                 // skygo trying to improve speed by honoring managed configuration MRM-1658
 155                 if (isReleases && !isSnapshots) {
 156                     throw new RepositoryStorageRuntimeException("lookforsnaponreleaseonly",
 157                             "managed repo is configured for release only");
 158                 }
 159             } else {
 160                 if (!isReleases && isSnapshots) {
 161                     throw new RepositoryStorageRuntimeException("lookforsreleaseonsneponly",
 162                             "managed repo is configured for snapshot only");
 163                 }
 164             }
 165         }
 166         StorageAsset basedir = managedRepository.getAsset("");
 167         if (VersionUtil.isSnapshot(artifactVersion)) {
 168             StorageAsset metadataFile = pathTranslator.toFile(basedir, readMetadataRequest.getNamespace(),
 169                     readMetadataRequest.getProjectId(), artifactVersion,
 170                     METADATA_FILENAME);
 171             try {
 172                 ArchivaRepositoryMetadata metadata = metadataReader.read(metadataFile);
 173
 174                 // re-adjust to timestamp if present, otherwise retain the original -SNAPSHOT filename
 175                 SnapshotVersion snapshotVersion = metadata.getSnapshotVersion();
 176                 if (snapshotVersion != null) {
 177                     artifactVersion =
 178                             artifactVersion.substring(0, artifactVersion.length() - 8); // remove SNAPSHOT from end
 179                     artifactVersion =
 180                             artifactVersion + snapshotVersion.getTimestamp() + "-" + snapshotVersion.getBuildNumber();
 181                 }
 182             } catch ( RepositoryMetadataException e) {
 183                 // unable to parse metadata - LOGGER it, and continue with the version as the original SNAPSHOT version
 184                 log.warn("Invalid metadata: {} - {}", metadataFile, e.getMessage());
 185             }
 186         }
 187
 188         // TODO: won't work well with some other layouts, might need to convert artifact parts to ID by path translator
 189         String id = readMetadataRequest.getProjectId() + "-" + artifactVersion + ".pom";
 190         StorageAsset file =
 191                 pathTranslator.toFile(basedir, readMetadataRequest.getNamespace(), readMetadataRequest.getProjectId(),
 192                         readMetadataRequest.getProjectVersion(), id);
 193
 194         if (!file.exists()) {
 195             // metadata could not be resolved
 196             throw new RepositoryStorageMetadataNotFoundException(
 197                     "The artifact's POM file '" + file.getPath() + "' was missing");
 198         }
 199
 200         // TODO: this is a workaround until we can properly resolve using proxies as well - this doesn't cache
 201         //       anything locally!
 202         List<RemoteRepository> remoteRepositories = new ArrayList<>();
 203         Map<String, NetworkProxy> networkProxies = new HashMap<>();
 204
 205         Map<String, List<ProxyConnector>> proxyConnectorsMap = proxyRegistry.getProxyConnectorAsMap();
 206         List<ProxyConnector> proxyConnectors = proxyConnectorsMap.get(readMetadataRequest.getRepositoryId());
 207         if (proxyConnectors != null) {
 208             for (ProxyConnector proxyConnector : proxyConnectors) {
 209                 RemoteRepository remoteRepoConfig =
 210                         repositoryRegistry.getRemoteRepository(proxyConnector.getTargetRepository().getId());
 211
 212                 if (remoteRepoConfig != null) {
 213                     remoteRepositories.add(remoteRepoConfig);
 214
 215                     NetworkProxy networkProxyConfig =
 216                             proxyRegistry.getNetworkProxy(proxyConnector.getProxyId());
 217
 218                     if (networkProxyConfig != null) {
 219                         // key/value: remote repo ID/proxy info
 220                         networkProxies.put(proxyConnector.getTargetRepository().getId(), networkProxyConfig);
 221                     }
 222                 }
 223             }
 224         }
 225
 226         // That's a browsing request so we can a mix of SNAPSHOT and release artifacts (especially with snapshots which
 227         // can have released parent pom
 228         if (readMetadataRequest.isBrowsingRequest()) {
 229             remoteRepositories.addAll(repositoryRegistry.getRemoteRepositories());
 230         }
 231
 232         ModelBuildingRequest req =
 233                 new DefaultModelBuildingRequest().setProcessPlugins(false).setPomFile(file.getFilePath().toFile()).setTwoPhaseBuilding(
 234                         false).setValidationLevel(ModelBuildingRequest.VALIDATION_LEVEL_MINIMAL);
 235
 236         //MRM-1607. olamy this will resolve jdk profiles on the current running archiva jvm
 237         req.setSystemProperties(System.getProperties());
 238
 239         // MRM-1411
 240         req.setModelResolver(
 241                 new RepositoryModelResolver(managedRepository, pathTranslator, wagonFactory, remoteRepositories,
 242                         networkProxies, managedRepository, mavenSystemManager, metadataReader));
 243
 244         Model model;
 245         try {
 246             model = builder.build(req).getEffectiveModel();
 247         } catch (ModelBuildingException e) {
 248             String msg = "The artifact's POM file '" + file + "' was invalid: " + e.getMessage();
 249
 250             List<ModelProblem> modelProblems = e.getProblems();
 251             for (ModelProblem problem : modelProblems) {
 252                 // MRM-1411, related to MRM-1335
 253                 // this means that the problem was that the parent wasn't resolved!
 254                 // olamy really hackhish but fail with java profile so use error message
 255                 // || ( StringUtils.startsWith( problem.getMessage(), "Failed to determine Java version for profile" ) )
 256                 // but setTwoPhaseBuilding(true) fix that
 257                 if (((problem.getException() instanceof FileNotFoundException
 258                         || problem.getException() instanceof NoSuchFileException
 259                 ) && e.getModelId() != null &&
 260                         !e.getModelId().equals(problem.getModelId()))) {
 261                     log.warn("The artifact's parent POM file '{}' cannot be resolved. "
 262                             + "Using defaults for project version metadata..", file);
 263
 264                     ProjectVersionMetadata metadata = new ProjectVersionMetadata();
 265                     metadata.setId(readMetadataRequest.getProjectVersion());
 266
 267                     MavenProjectFacet facet = new MavenProjectFacet();
 268                     facet.setGroupId(readMetadataRequest.getNamespace());
 269                     facet.setArtifactId(readMetadataRequest.getProjectId());
 270                     facet.setPackaging("jar");
 271                     metadata.addFacet(facet);
 272
 273                     String errMsg =
 274                             "Error in resolving artifact's parent POM file. " + (problem.getException() == null
 275                                     ? problem.getMessage()
 276                                     : problem.getException().getMessage());
 277                     RepositoryProblemFacet repoProblemFacet = new RepositoryProblemFacet();
 278                     repoProblemFacet.setRepositoryId(readMetadataRequest.getRepositoryId());
 279                     repoProblemFacet.setId(readMetadataRequest.getRepositoryId());
 280                     repoProblemFacet.setMessage(errMsg);
 281                     repoProblemFacet.setProblem(errMsg);
 282                     repoProblemFacet.setProject(readMetadataRequest.getProjectId());
 283                     repoProblemFacet.setVersion(readMetadataRequest.getProjectVersion());
 284                     repoProblemFacet.setNamespace(readMetadataRequest.getNamespace());
 285
 286                     metadata.addFacet(repoProblemFacet);
 287
 288                     return metadata;
 289                 }
 290             }
 291
 292             throw new RepositoryStorageMetadataInvalidException("invalid-pom", msg, e);
 293         }
 294
 295         // Check if the POM is in the correct location
 296         boolean correctGroupId = readMetadataRequest.getNamespace().equals(model.getGroupId());
 297         boolean correctArtifactId = readMetadataRequest.getProjectId().equals(model.getArtifactId());
 298         boolean correctVersion = readMetadataRequest.getProjectVersion().equals(model.getVersion());
 299         if (!correctGroupId || !correctArtifactId || !correctVersion) {
 300             StringBuilder message = new StringBuilder("Incorrect POM coordinates in '" + file + "':");
 301             if (!correctGroupId) {
 302                 message.append("\nIncorrect group ID: ").append(model.getGroupId());
 303             }
 304             if (!correctArtifactId) {
 305                 message.append("\nIncorrect artifact ID: ").append(model.getArtifactId());
 306             }
 307             if (!correctVersion) {
 308                 message.append("\nIncorrect version: ").append(model.getVersion());
 309             }
 310
 311             throw new RepositoryStorageMetadataInvalidException("mislocated-pom", message.toString());
 312         }
 313
 314         ProjectVersionMetadata metadata = new ProjectVersionMetadata();
 315         metadata.setCiManagement(convertCiManagement(model.getCiManagement()));
 316         metadata.setDescription(model.getDescription());
 317         metadata.setId(readMetadataRequest.getProjectVersion());
 318         metadata.setIssueManagement(convertIssueManagement(model.getIssueManagement()));
 319         metadata.setLicenses(convertLicenses(model.getLicenses()));
 320         metadata.setMailingLists(convertMailingLists(model.getMailingLists()));
 321         metadata.setDependencies(convertDependencies(model.getDependencies()));
 322         metadata.setName(model.getName());
 323         metadata.setOrganization(convertOrganization(model.getOrganization()));
 324         metadata.setScm(convertScm(model.getScm()));
 325         metadata.setUrl(model.getUrl());
 326         metadata.setProperties(model.getProperties());
 327
 328         MavenProjectFacet facet = new MavenProjectFacet();
 329         facet.setGroupId(model.getGroupId() != null ? model.getGroupId() : model.getParent().getGroupId());
 330         facet.setArtifactId(model.getArtifactId());
 331         facet.setPackaging(model.getPackaging());
 332         if (model.getParent() != null) {
 333             MavenProjectParent parent = new MavenProjectParent();
 334             parent.setGroupId(model.getParent().getGroupId());
 335             parent.setArtifactId(model.getParent().getArtifactId());
 336             parent.setVersion(model.getParent().getVersion());
 337             facet.setParent(parent);
 338         }
 339         metadata.addFacet(facet);
 340
 341         return metadata;
 342
 343
 344     }
 345
 346     public void setWagonFactory(WagonFactory wagonFactory) {
 347         this.wagonFactory = wagonFactory;
 348     }
 349
 350     private List<org.apache.archiva.metadata.model.Dependency> convertDependencies(List<Dependency> dependencies) {
 351         List<org.apache.archiva.metadata.model.Dependency> l = new ArrayList<>();
 352         for (Dependency dependency : dependencies) {
 353             org.apache.archiva.metadata.model.Dependency newDependency =
 354                     new org.apache.archiva.metadata.model.Dependency();
 355             newDependency.setArtifactId(dependency.getArtifactId());
 356             newDependency.setClassifier(dependency.getClassifier());
 357             newDependency.setNamespace(dependency.getGroupId());
 358             newDependency.setOptional(dependency.isOptional());
 359             newDependency.setScope(dependency.getScope());
 360             newDependency.setSystemPath(dependency.getSystemPath());
 361             newDependency.setType(dependency.getType());
 362             newDependency.setVersion(dependency.getVersion());
 363             l.add(newDependency);
 364         }
 365         return l;
 366     }
 367
 368     private org.apache.archiva.metadata.model.Scm convertScm(Scm scm) {
 369         org.apache.archiva.metadata.model.Scm newScm = null;
 370         if (scm != null) {
 371             newScm = new org.apache.archiva.metadata.model.Scm();
 372             newScm.setConnection(scm.getConnection());
 373             newScm.setDeveloperConnection(scm.getDeveloperConnection());
 374             newScm.setUrl(scm.getUrl());
 375         }
 376         return newScm;
 377     }
 378
 379     private org.apache.archiva.metadata.model.Organization convertOrganization(Organization organization) {
 380         org.apache.archiva.metadata.model.Organization org = null;
 381         if (organization != null) {
 382             org = new org.apache.archiva.metadata.model.Organization();
 383             org.setName(organization.getName());
 384             org.setUrl(organization.getUrl());
 385         }
 386         return org;
 387     }
 388
 389     private List<org.apache.archiva.metadata.model.License> convertLicenses(List<License> licenses) {
 390         List<org.apache.archiva.metadata.model.License> l = new ArrayList<>();
 391         for (License license : licenses) {
 392             org.apache.archiva.metadata.model.License newLicense = new org.apache.archiva.metadata.model.License();
 393             newLicense.setName(license.getName());
 394             newLicense.setUrl(license.getUrl());
 395             l.add(newLicense);
 396         }
 397         return l;
 398     }
 399
 400     private List<org.apache.archiva.metadata.model.MailingList> convertMailingLists(List<MailingList> mailingLists) {
 401         List<org.apache.archiva.metadata.model.MailingList> l = new ArrayList<>();
 402         for (MailingList mailingList : mailingLists) {
 403             org.apache.archiva.metadata.model.MailingList newMailingList =
 404                     new org.apache.archiva.metadata.model.MailingList();
 405             newMailingList.setName(mailingList.getName());
 406             newMailingList.setMainArchiveUrl(mailingList.getArchive());
 407             newMailingList.setPostAddress(mailingList.getPost());
 408             newMailingList.setSubscribeAddress(mailingList.getSubscribe());
 409             newMailingList.setUnsubscribeAddress(mailingList.getUnsubscribe());
 410             newMailingList.setOtherArchives(mailingList.getOtherArchives());
 411             l.add(newMailingList);
 412         }
 413         return l;
 414     }
 415
 416     private org.apache.archiva.metadata.model.IssueManagement convertIssueManagement(IssueManagement issueManagement) {
 417         org.apache.archiva.metadata.model.IssueManagement im = null;
 418         if (issueManagement != null) {
 419             im = new org.apache.archiva.metadata.model.IssueManagement();
 420             im.setSystem(issueManagement.getSystem());
 421             im.setUrl(issueManagement.getUrl());
 422         }
 423         return im;
 424     }
 425
 426     private org.apache.archiva.metadata.model.CiManagement convertCiManagement(CiManagement ciManagement) {
 427         org.apache.archiva.metadata.model.CiManagement ci = null;
 428         if (ciManagement != null) {
 429             ci = new org.apache.archiva.metadata.model.CiManagement();
 430             ci.setSystem(ciManagement.getSystem());
 431             ci.setUrl(ciManagement.getUrl());
 432         }
 433         return ci;
 434     }
 435
 436     @Override
 437     public Collection<String> listRootNamespaces(String repoId, Filter<String> filter)
 438             throws RepositoryStorageRuntimeException {
 439         StorageAsset dir = getRepositoryBasedir(repoId);
 440
 441         return getSortedFiles(dir, filter);
 442     }
 443
 444     private static Collection<String> getSortedFiles(StorageAsset dir, Filter<String> filter) {
 445
 446         final Predicate<StorageAsset> dFilter = new DirectoryFilter(filter);
 447         return dir.list().stream().filter(f -> f.isContainer())
 448                 .filter(dFilter)
 449                 .map(path -> path.getName().toString())
 450                 .sorted().collect(Collectors.toList());
 451
 452     }
 453
 454     private StorageAsset getRepositoryBasedir(String repoId)
 455             throws RepositoryStorageRuntimeException {
 456         ManagedRepository repositoryConfiguration = repositoryRegistry.getManagedRepository(repoId);
 457
 458         return repositoryConfiguration.getAsset("");
 459     }
 460
 461     @Override
 462     public Collection<String> listNamespaces(String repoId, String namespace, Filter<String> filter)
 463             throws RepositoryStorageRuntimeException {
 464         StorageAsset dir = pathTranslator.toFile(getRepositoryBasedir(repoId), namespace);
 465         if (!(dir.exists()) && !dir.isContainer()) {
 466             return Collections.emptyList();
 467         }
 468         // scan all the directories which are potential namespaces. Any directories known to be projects are excluded
 469         Predicate<StorageAsset> dFilter = new DirectoryFilter(filter);
 470         return dir.list().stream().filter(dFilter).filter(path -> !isProject(path, filter)).map(path -> path.getName().toString())
 471                 .sorted().collect(Collectors.toList());
 472     }
 473
 474     @Override
 475     public Collection<String> listProjects(String repoId, String namespace, Filter<String> filter)
 476             throws RepositoryStorageRuntimeException {
 477         StorageAsset dir = pathTranslator.toFile(getRepositoryBasedir(repoId), namespace);
 478         if (!(dir.exists() && dir.isContainer())) {
 479             return Collections.emptyList();
 480         }
 481         // scan all directories in the namespace, and only include those that are known to be projects
 482         final Predicate<StorageAsset> dFilter = new DirectoryFilter(filter);
 483         return dir.list().stream().filter(dFilter).filter(path -> isProject(path, filter)).map(path -> path.getName().toString())
 484                 .sorted().collect(Collectors.toList());
 485
 486     }
 487
 488     @Override
 489     public Collection<String> listProjectVersions(String repoId, String namespace, String projectId,
 490                                                   Filter<String> filter)
 491             throws RepositoryStorageRuntimeException {
 492         StorageAsset dir = pathTranslator.toFile(getRepositoryBasedir(repoId), namespace, projectId);
 493         if (!(dir.exists() && dir.isContainer())) {
 494             return Collections.emptyList();
 495         }
 496
 497         // all directories in a project directory can be considered a version
 498         return getSortedFiles(dir, filter);
 499     }
 500
 501     @Override
 502     public Collection<ArtifactMetadata> readArtifactsMetadata(ReadMetadataRequest readMetadataRequest)
 503             throws RepositoryStorageRuntimeException {
 504         StorageAsset dir = pathTranslator.toFile(getRepositoryBasedir(readMetadataRequest.getRepositoryId()),
 505                 readMetadataRequest.getNamespace(), readMetadataRequest.getProjectId(),
 506                 readMetadataRequest.getProjectVersion());
 507         if (!(dir.exists() && dir.isContainer())) {
 508             return Collections.emptyList();
 509         }
 510
 511         // all files that are not metadata and not a checksum / signature are considered artifacts
 512         final Predicate<StorageAsset> dFilter = new ArtifactDirectoryFilter(readMetadataRequest.getFilter());
 513         // Returns a map TRUE -> (success values), FALSE -> (Exceptions)
 514         Map<Boolean, List<Try<ArtifactMetadata>>> result = dir.list().stream().filter(dFilter).map(path -> {
 515                     try {
 516                         return Try.success(getArtifactFromFile(readMetadataRequest.getRepositoryId(), readMetadataRequest.getNamespace(),
 517                                 readMetadataRequest.getProjectId(), readMetadataRequest.getProjectVersion(),
 518                                 path));
 519                     } catch (Exception e) {
 520                         log.debug("Could not create metadata for {}:  {}", path, e.getMessage(), e);
 521                         return Try.<ArtifactMetadata>failure(e);
 522                     }
 523                 }
 524         ).collect(Collectors.groupingBy(Try::isSuccess));
 525         if (result.containsKey(Boolean.FALSE) && result.get(Boolean.FALSE).size() > 0 && (!result.containsKey(Boolean.TRUE) || result.get(Boolean.TRUE).size() == 0)) {
 526             log.error("Could not get artifact metadata. Directory: {}. Number of errors {}.", dir, result.get(Boolean.FALSE).size());
 527             Try<ArtifactMetadata> failure = result.get(Boolean.FALSE).get(0);
 528             log.error("Sample exception {}", failure.getError().getMessage(), failure.getError());
 529             throw new RepositoryStorageRuntimeException(readMetadataRequest.getRepositoryId(), "Could not retrieve metadata of the files");
 530         } else {
 531             if (!result.containsKey(Boolean.TRUE) || result.get(Boolean.TRUE) == null) {
 532                 return Collections.emptyList();
 533             }
 534             return result.get(Boolean.TRUE).stream().map(tr -> tr.get()).collect(Collectors.toList());
 535         }
 536
 537     }
 538
 539     @Override
 540     public ArtifactMetadata readArtifactMetadataFromPath(String repoId, String path)
 541             throws RepositoryStorageRuntimeException {
 542         ArtifactMetadata metadata = pathTranslator.getArtifactForPath(repoId, path);
 543
 544         try {
 545             populateArtifactMetadataFromFile(metadata, getRepositoryBasedir(repoId).resolve(path));
 546         } catch (IOException e) {
 547             throw new RepositoryStorageRuntimeException(repoId, "Error during metadata retrieval of " + path + " :" + e.getMessage(), e);
 548         }
 549
 550         return metadata;
 551     }
 552
 553     private ArtifactMetadata getArtifactFromFile(String repoId, String namespace, String projectId,
 554                                                  String projectVersion, StorageAsset file) throws IOException {
 555         ArtifactMetadata metadata =
 556                 pathTranslator.getArtifactFromId(repoId, namespace, projectId, projectVersion, file.getName());
 557
 558         populateArtifactMetadataFromFile(metadata, file);
 559
 560         return metadata;
 561     }
 562
 563     @Override
 564     public void applyServerSideRelocation(ManagedRepository managedRepository, ArtifactReference artifact)
 565             throws ProxyDownloadException {
 566         if ("pom".equals(artifact.getType())) {
 567             return;
 568         }
 569
 570         // Build the artifact POM reference
 571         ArtifactReference pomReference = new ArtifactReference();
 572         pomReference.setGroupId(artifact.getGroupId());
 573         pomReference.setArtifactId(artifact.getArtifactId());
 574         pomReference.setVersion(artifact.getVersion());
 575         pomReference.setType("pom");
 576
 577         RepositoryType repositoryType = managedRepository.getType();
 578         if (!proxyRegistry.hasHandler(repositoryType)) {
 579             throw new ProxyDownloadException("No proxy handler found for repository type " + repositoryType, new HashMap<>());
 580         }
 581
 582         RepositoryProxyHandler proxyHandler = proxyRegistry.getHandler(repositoryType).get(0);
 583
 584         // Get the artifact POM from proxied repositories if needed
 585         proxyHandler.fetchFromProxies(managedRepository, pomReference);
 586
 587         // Open and read the POM from the managed repo
 588         StorageAsset pom = null;
 589         try
 590         {
 591             pom = managedRepository.getContent().getLayout( BaseRepositoryContentLayout.class ).toFile(pomReference);
 592         }
 593         catch ( LayoutException e )
 594         {
 595             throw new ProxyDownloadException( "Cannot convert layout ", new HashMap<>( ) );
 596         }
 597
 598         if (!pom.exists()) {
 599             return;
 600         }
 601
 602         try {
 603             // MavenXpp3Reader leaves the file open, so we need to close it ourselves.
 604
 605             Model model;
 606             try (Reader reader = Channels.newReader(pom.getReadChannel(), Charset.defaultCharset().name())) {
 607                 model = MAVEN_XPP_3_READER.read(reader);
 608             }
 609
 610             DistributionManagement dist = model.getDistributionManagement();
 611             if (dist != null) {
 612                 Relocation relocation = dist.getRelocation();
 613                 if (relocation != null) {
 614                     // artifact is relocated : update the repositoryPath
 615                     if (relocation.getGroupId() != null) {
 616                         artifact.setGroupId(relocation.getGroupId());
 617                     }
 618                     if (relocation.getArtifactId() != null) {
 619                         artifact.setArtifactId(relocation.getArtifactId());
 620                     }
 621                     if (relocation.getVersion() != null) {
 622                         artifact.setVersion(relocation.getVersion());
 623                     }
 624                 }
 625             }
 626         } catch (IOException e) {
 627             // Unable to read POM : ignore.
 628         } catch (XmlPullParserException e) {
 629             // Invalid POM : ignore
 630         }
 631     }
 632
 633
 634     @Override
 635     public String getFilePath(String requestPath, org.apache.archiva.repository.ManagedRepository managedRepository) {
 636         // managedRepository can be null
 637         // extract artifact reference from url
 638         // groupId:artifactId:version:packaging:classifier
 639         //org/apache/archiva/archiva-checksum/1.4-M4-SNAPSHOT/archiva-checksum-1.4-M4-SNAPSHOT.jar
 640         String logicalResource = null;
 641         String requestPathInfo = StringUtils.defaultString(requestPath);
 642
 643         //remove prefix ie /repository/blah becomes /blah
 644         requestPathInfo = removePrefix(requestPathInfo);
 645
 646         // Remove prefixing slash as the repository id doesn't contain it;
 647         if (requestPathInfo.startsWith("/")) {
 648             requestPathInfo = requestPathInfo.substring(1);
 649         }
 650
 651         int slash = requestPathInfo.indexOf('/');
 652         if (slash > 0) {
 653             logicalResource = requestPathInfo.substring(slash);
 654
 655             if (logicalResource.endsWith("/..")) {
 656                 logicalResource += "/";
 657             }
 658
 659             if (logicalResource != null && logicalResource.startsWith("//")) {
 660                 logicalResource = logicalResource.substring(1);
 661             }
 662
 663             if (logicalResource == null) {
 664                 logicalResource = "/";
 665             }
 666         } else {
 667             logicalResource = "/";
 668         }
 669         return logicalResource;
 670
 671     }
 672
 673     @Override
 674     public String getFilePathWithVersion(final String requestPath, ManagedRepositoryContent managedRepositoryContent)
 675             throws RelocationException
 676     {
 677
 678         if (StringUtils.endsWith(requestPath, METADATA_FILENAME)) {
 679             return getFilePath(requestPath, managedRepositoryContent.getRepository());
 680         }
 681
 682         String filePath = getFilePath(requestPath, managedRepositoryContent.getRepository());
 683
 684         ArtifactReference artifactReference = null;
 685         try {
 686             artifactReference = pathParser.toArtifactReference(filePath);
 687         } catch (LayoutException e) {
 688             return filePath;
 689         }
 690
 691         if (StringUtils.endsWith(artifactReference.getVersion(), VersionUtil.SNAPSHOT)) {
 692             // read maven metadata to get last timestamp
 693             StorageAsset metadataDir = managedRepositoryContent.getRepository().getAsset(filePath).getParent();
 694             if (!metadataDir.exists()) {
 695                 return filePath;
 696             }
 697             StorageAsset metadataFile = metadataDir.resolve(METADATA_FILENAME);
 698             if (!metadataFile.exists()) {
 699                 return filePath;
 700             }
 701             ArchivaRepositoryMetadata archivaRepositoryMetadata = null;
 702             try
 703             {
 704                 archivaRepositoryMetadata = metadataReader.read(metadataFile);
 705             }
 706             catch ( RepositoryMetadataException e )
 707             {
 708                 log.error( "Could not read metadata {}", e.getMessage( ), e );
 709                 return filePath;
 710             }
 711             int buildNumber = archivaRepositoryMetadata.getSnapshotVersion().getBuildNumber();
 712             String timestamp = archivaRepositoryMetadata.getSnapshotVersion().getTimestamp();
 713
 714             // MRM-1846
 715             if (buildNumber < 1 && timestamp == null) {
 716                 return filePath;
 717             }
 718
 719             // org/apache/archiva/archiva-checksum/1.4-M4-SNAPSHOT/archiva-checksum-1.4-M4-SNAPSHOT.jar
 720             // ->  archiva-checksum-1.4-M4-20130425.081822-1.jar
 721
 722             filePath = StringUtils.replace(filePath, //
 723                     artifactReference.getArtifactId() //
 724                             + "-" + artifactReference.getVersion(), //
 725                     artifactReference.getArtifactId() //
 726                             + "-" + StringUtils.remove(artifactReference.getVersion(),
 727                             "-" + VersionUtil.SNAPSHOT) //
 728                             + "-" + timestamp //
 729                             + "-" + buildNumber);
 730
 731             throw new RelocationException("/repository/" + managedRepositoryContent.getRepository().getId() +
 732                     (StringUtils.startsWith(filePath, "/") ? "" : "/") + filePath,
 733                     RelocationException.RelocationType.TEMPORARY);
 734
 735         }
 736
 737         return filePath;
 738     }
 739
 740     //-----------------------------
 741     // internal
 742     //-----------------------------
 743
 744     /**
 745      * FIXME remove
 746      *
 747      * @param href
 748      * @return
 749      */
 750     private static String removePrefix(final String href) {
 751         String[] parts = StringUtils.split(href, '/');
 752         parts = (String[]) ArrayUtils.subarray(parts, 1, parts.length);
 753         if (parts == null || parts.length == 0) {
 754             return "/";
 755         }
 756
 757         String joinedString = StringUtils.join(parts, '/');
 758         if (href.endsWith("/")) {
 759             joinedString = joinedString + "/";
 760         }
 761
 762         return joinedString;
 763     }
 764
 765     private static void populateArtifactMetadataFromFile(ArtifactMetadata metadata, StorageAsset file) throws IOException {
 766         metadata.setWhenGathered(ZonedDateTime.now(ZoneId.of("GMT")));
 767         metadata.setFileLastModified(file.getModificationTime().toEpochMilli());
 768         ChecksummedFile checksummedFile = new ChecksummedFile(file.getFilePath());
 769         try {
 770             metadata.setMd5(checksummedFile.calculateChecksum(ChecksumAlgorithm.MD5));
 771         } catch (IOException e) {
 772             log.error("Unable to checksum file {}: {},MD5", file, e.getMessage());
 773         }
 774         try {
 775             metadata.setSha1(checksummedFile.calculateChecksum(ChecksumAlgorithm.SHA1));
 776         } catch (IOException e) {
 777             log.error("Unable to checksum file {}: {},SHA1", file, e.getMessage());
 778         }
 779         metadata.setSize(file.getSize());
 780     }
 781
 782     private boolean isProject(StorageAsset dir, Filter<String> filter) {
 783         // scan directories for a valid project version subdirectory, meaning this must be a project directory
 784         final Predicate<StorageAsset> dFilter = new DirectoryFilter(filter);
 785         boolean projFound = dir.list().stream().filter(dFilter)
 786                 .anyMatch(path -> isProjectVersion(path));
 787         if (projFound) {
 788             return true;
 789         }
 790
 791         // if a metadata file is present, check if this is the "artifactId" directory, marking it as a project
 792         ArchivaRepositoryMetadata metadata = readMetadata(dir);
 793         if (metadata != null && dir.getName().toString().equals(metadata.getArtifactId())) {
 794             return true;
 795         }
 796
 797         return false;
 798     }
 799
 800     private boolean isProjectVersion(StorageAsset dir) {
 801         final String artifactId = dir.getParent().getName();
 802         final String projectVersion = dir.getName();
 803
 804         // check if there is a POM artifact file to ensure it is a version directory
 805
 806         Predicate<StorageAsset> filter;
 807         if (VersionUtil.isSnapshot(projectVersion)) {
 808             filter = new PomFilenameFilter(artifactId, projectVersion);
 809         } else {
 810             final String pomFile = artifactId + "-" + projectVersion + ".pom";
 811             filter = new PomFileFilter(pomFile);
 812         }
 813         if (dir.list().stream().filter(f -> !f.isContainer()).anyMatch(filter)) {
 814             return true;
 815         }
 816         // if a metadata file is present, check if this is the "version" directory, marking it as a project version
 817         ArchivaRepositoryMetadata metadata = readMetadata(dir);
 818         if (metadata != null && projectVersion.equals(metadata.getVersion())) {
 819             return true;
 820         }
 821
 822         return false;
 823     }
 824
 825     private ArchivaRepositoryMetadata readMetadata(StorageAsset directory) {
 826         ArchivaRepositoryMetadata metadata = null;
 827         StorageAsset metadataFile = directory.resolve(METADATA_FILENAME);
 828         if (metadataFile.exists()) {
 829             try {
 830                 metadata = metadataReader.read(metadataFile);
 831             } catch ( RepositoryMetadataException e )
 832             {
 833                 // Ignore missing or invalid metadata
 834             }
 835         }
 836         return metadata;
 837     }
 838
 839     private static class DirectoryFilter
 840             implements Predicate<StorageAsset> {
 841         private final Filter<String> filter;
 842
 843         public DirectoryFilter(Filter<String> filter) {
 844             this.filter = filter;
 845         }
 846
 847         @Override
 848         public boolean test(StorageAsset dir) {
 849             final String name = dir.getName();
 850             if (!filter.accept(name)) {
 851                 return false;
 852             } else if (name.startsWith(".")) {
 853                 return false;
 854             } else if (!dir.isContainer()) {
 855                 return false;
 856             }
 857             return true;
 858         }
 859     }
 860
 861     private static class ArtifactDirectoryFilter
 862             implements Predicate<StorageAsset> {
 863         private final Filter<String> filter;
 864
 865         private ArtifactDirectoryFilter(Filter<String> filter) {
 866             this.filter = filter;
 867         }
 868
 869         @Override
 870         public boolean test(StorageAsset file) {
 871             final Set<String> checksumExts = ChecksumAlgorithm.getAllExtensions();
 872             final String path = file.getPath();
 873             final String name = file.getName();
 874             final String extension = StringUtils.substringAfterLast(name, ".").toLowerCase();
 875             // TODO compare to logic in maven-repository-layer
 876             if (file.isContainer()) {
 877                 return false;
 878             } else if (!filter.accept(name)) {
 879                 return false;
 880             } else if (name.startsWith(".") || path.contains("/.") ) {
 881                 return false;
 882             } else if (checksumExts.contains(extension)) {
 883                 return false;
 884             } else if (Arrays.binarySearch(IGNORED_FILES, name) >= 0) {
 885                 return false;
 886             }
 887             // some files from remote repositories can have name like maven-metadata-archiva-vm-all-public.xml
 888             else if (StringUtils.startsWith(name, METADATA_FILENAME_START) && StringUtils.endsWith(name, ".xml")) {
 889                 return false;
 890             }
 891
 892             return true;
 893
 894         }
 895     }
 896
 897
 898     private static final class PomFilenameFilter
 899             implements Predicate<StorageAsset> {
 900
 901         private final String artifactId, projectVersion;
 902
 903         private PomFilenameFilter(String artifactId, String projectVersion) {
 904             this.artifactId = artifactId;
 905             this.projectVersion = projectVersion;
 906         }
 907
 908         @Override
 909         public boolean test(StorageAsset dir) {
 910             final String name = dir.getName();
 911             if (name.startsWith(artifactId + "-") && name.endsWith(".pom")) {
 912                 String v = name.substring(artifactId.length() + 1, name.length() - 4);
 913                 v = VersionUtil.getBaseVersion(v);
 914                 if (v.equals(projectVersion)) {
 915                     return true;
 916                 }
 917             }
 918             return false;
 919         }
 920
 921     }
 922
 923     private static class PomFileFilter
 924             implements Predicate<StorageAsset> {
 925         private final String pomFile;
 926
 927         private PomFileFilter(String pomFile) {
 928             this.pomFile = pomFile;
 929         }
 930
 931         @Override
 932         public boolean test(StorageAsset dir) {
 933             return pomFile.equals(dir.getName());
 934         }
 935     }
 936
 937
 938     public PathParser getPathParser() {
 939         return pathParser;
 940     }
 941
 942     public void setPathParser(PathParser pathParser) {
 943         this.pathParser = pathParser;
 944     }
 945 }