source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.repository.maven.metadata.storage;
   2
   3 /*
   4  * Licensed to the Apache Software Foundation (ASF) under one
   5  * or more contributor license agreements.  See the NOTICE file
   6  * distributed with this work for additional information
   7  * regarding copyright ownership.  The ASF licenses this file
   8  * to you under the Apache License, Version 2.0 (the
   9  * "License"); you may not use this file except in compliance
  10  * with the License.  You may obtain a copy of the License at
  11  *
  12  * http://www.apache.org/licenses/LICENSE-2.0
  13  * Unless required by applicable law or agreed to in writing,
  14  * software distributed under the License is distributed on an
  15  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  16  * KIND, either express or implied.  See the License for the
  17  * specific language governing permissions and limitations
  18  * under the License.
  19  */
  20
  21 import org.apache.archiva.checksum.ChecksumAlgorithm;
  22 import org.apache.archiva.checksum.ChecksummedFile;
  23 import org.apache.archiva.common.Try;
  24 import org.apache.archiva.common.utils.VersionUtil;
  25 import org.apache.archiva.filter.Filter;
  26 import org.apache.archiva.metadata.maven.MavenMetadataReader;
  27 import org.apache.archiva.metadata.model.ArtifactMetadata;
  28 import org.apache.archiva.metadata.model.ProjectMetadata;
  29 import org.apache.archiva.metadata.model.ProjectVersionMetadata;
  30 import org.apache.archiva.metadata.model.facets.RepositoryProblemFacet;
  31 import org.apache.archiva.metadata.repository.storage.*;
  32 import org.apache.archiva.model.ArchivaRepositoryMetadata;
  33 import org.apache.archiva.model.ArtifactReference;
  34 import org.apache.archiva.model.SnapshotVersion;
  35 import org.apache.archiva.policies.ProxyDownloadException;
  36 import org.apache.archiva.proxy.ProxyRegistry;
  37 import org.apache.archiva.proxy.maven.WagonFactory;
  38 import org.apache.archiva.proxy.model.NetworkProxy;
  39 import org.apache.archiva.proxy.model.ProxyConnector;
  40 import org.apache.archiva.proxy.model.RepositoryProxyHandler;
  41 import org.apache.archiva.repository.*;
  42 import org.apache.archiva.repository.content.PathParser;
  43 import org.apache.archiva.repository.maven.MavenSystemManager;
  44 import org.apache.archiva.repository.metadata.RepositoryMetadataException;
  45 import org.apache.archiva.repository.storage.StorageAsset;
  46 import org.apache.commons.lang3.ArrayUtils;
  47 import org.apache.commons.lang3.StringUtils;
  48 import org.apache.maven.model.*;
  49 import org.apache.maven.model.building.*;
  50 import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
  51 import org.codehaus.plexus.util.xml.pull.XmlPullParserException;
  52 import org.slf4j.Logger;
  53 import org.slf4j.LoggerFactory;
  54 import org.springframework.context.ApplicationContext;
  55 import org.springframework.stereotype.Service;
  56
  57 import javax.annotation.PostConstruct;
  58 import javax.inject.Inject;
  59 import javax.inject.Named;
  60 import java.io.FileNotFoundException;
  61 import java.io.IOException;
  62 import java.io.Reader;
  63 import java.nio.channels.Channels;
  64 import java.nio.charset.Charset;
  65 import java.nio.file.NoSuchFileException;
  66 import java.time.ZoneId;
  67 import java.time.ZonedDateTime;
  68 import java.util.*;
  69 import java.util.function.Predicate;
  70 import java.util.stream.Collectors;
  71
  72 // import java.io.FileNotFoundException;
  73
  74 /**
  75  * <p>
  76  * Maven 2 repository format storage implementation. This class currently takes parameters to indicate the repository to
  77  * deal with rather than being instantiated per-repository.
  78  * FIXME: instantiate one per repository and allocate permanently from a factory (which can be obtained within the session).
  79  * </p>
  80  * <p>
  81  * The session is passed in as an argument to obtain any necessary resources, rather than the class being instantiated
  82  * within the session in the context of a single managed repository's resolution needs.
  83  * </p>
  84  */
  85 @Service("repositoryStorage#maven2")
  86 public class Maven2RepositoryStorage
  87         implements RepositoryStorage {
  88
  89     private static final Logger log = LoggerFactory.getLogger(Maven2RepositoryStorage.class);
  90
  91     private ModelBuilder builder;
  92
  93     @Inject
  94     RepositoryRegistry repositoryRegistry;
  95
  96     @Inject
  97     @Named( "metadataReader#maven" )
  98     MavenMetadataReader metadataReader;
  99
 100     @Inject
 101     @Named("repositoryPathTranslator#maven2")
 102     private RepositoryPathTranslator pathTranslator;
 103
 104     @Inject
 105     private WagonFactory wagonFactory;
 106
 107     @Inject
 108     private ApplicationContext applicationContext;
 109
 110     @Inject
 111     @Named("pathParser#default")
 112     private PathParser pathParser;
 113
 114     @Inject
 115     private ProxyRegistry proxyRegistry;
 116
 117     @Inject
 118     private MavenSystemManager mavenSystemManager;
 119
 120     private static final String METADATA_FILENAME_START = "maven-metadata";
 121
 122     private static final String METADATA_FILENAME = METADATA_FILENAME_START + ".xml";
 123
 124     // This array must be lexically sorted
 125     private static final String[] IGNORED_FILES = {METADATA_FILENAME, "resolver-status.properties"};
 126
 127     private static final MavenXpp3Reader MAVEN_XPP_3_READER = new MavenXpp3Reader();
 128
 129
 130     @PostConstruct
 131     public void initialize() {
 132         builder = new DefaultModelBuilderFactory().newInstance();
 133
 134     }
 135
 136     @Override
 137     public ProjectMetadata readProjectMetadata(String repoId, String namespace, String projectId) {
 138         // TODO: could natively implement the "shared model" concept from the browse action to avoid needing it there?
 139         return null;
 140     }
 141
 142     @Override
 143     public ProjectVersionMetadata readProjectVersionMetadata(ReadMetadataRequest readMetadataRequest)
 144             throws RepositoryStorageMetadataNotFoundException, RepositoryStorageMetadataInvalidException,
 145             RepositoryStorageRuntimeException {
 146
 147         ManagedRepository managedRepository = repositoryRegistry.getManagedRepository(readMetadataRequest.getRepositoryId());
 148         boolean isReleases = managedRepository.getActiveReleaseSchemes().contains(ReleaseScheme.RELEASE);
 149         boolean isSnapshots = managedRepository.getActiveReleaseSchemes().contains(ReleaseScheme.SNAPSHOT);
 150         String artifactVersion = readMetadataRequest.getProjectVersion();
 151         // olamy: in case of browsing via the ui we can mix repos (parent of a SNAPSHOT can come from release repo)
 152         if (!readMetadataRequest.isBrowsingRequest()) {
 153             if (VersionUtil.isSnapshot(artifactVersion)) {
 154                 // skygo trying to improve speed by honoring managed configuration MRM-1658
 155                 if (isReleases && !isSnapshots) {
 156                     throw new RepositoryStorageRuntimeException("lookforsnaponreleaseonly",
 157                             "managed repo is configured for release only");
 158                 }
 159             } else {
 160                 if (!isReleases && isSnapshots) {
 161                     throw new RepositoryStorageRuntimeException("lookforsreleaseonsneponly",
 162                             "managed repo is configured for snapshot only");
 163                 }
 164             }
 165         }
 166         StorageAsset basedir = managedRepository.getAsset("");
 167         if (VersionUtil.isSnapshot(artifactVersion)) {
 168             StorageAsset metadataFile = pathTranslator.toFile(basedir, readMetadataRequest.getNamespace(),
 169                     readMetadataRequest.getProjectId(), artifactVersion,
 170                     METADATA_FILENAME);
 171             try {
 172                 ArchivaRepositoryMetadata metadata = metadataReader.read(metadataFile);
 173
 174                 // re-adjust to timestamp if present, otherwise retain the original -SNAPSHOT filename
 175                 SnapshotVersion snapshotVersion = metadata.getSnapshotVersion();
 176                 if (snapshotVersion != null) {
 177                     artifactVersion =
 178                             artifactVersion.substring(0, artifactVersion.length() - 8); // remove SNAPSHOT from end
 179                     artifactVersion =
 180                             artifactVersion + snapshotVersion.getTimestamp() + "-" + snapshotVersion.getBuildNumber();
 181                 }
 182             } catch ( RepositoryMetadataException e) {
 183                 // unable to parse metadata - LOGGER it, and continue with the version as the original SNAPSHOT version
 184                 log.warn("Invalid metadata: {} - {}", metadataFile, e.getMessage());
 185             }
 186         }
 187
 188         // TODO: won't work well with some other layouts, might need to convert artifact parts to ID by path translator
 189         String id = readMetadataRequest.getProjectId() + "-" + artifactVersion + ".pom";
 190         StorageAsset file =
 191                 pathTranslator.toFile(basedir, readMetadataRequest.getNamespace(), readMetadataRequest.getProjectId(),
 192                         readMetadataRequest.getProjectVersion(), id);
 193
 194         if (!file.exists()) {
 195             // metadata could not be resolved
 196             throw new RepositoryStorageMetadataNotFoundException(
 197                     "The artifact's POM file '" + file.getPath() + "' was missing");
 198         }
 199
 200         // TODO: this is a workaround until we can properly resolve using proxies as well - this doesn't cache
 201         //       anything locally!
 202         List<RemoteRepository> remoteRepositories = new ArrayList<>();
 203         Map<String, NetworkProxy> networkProxies = new HashMap<>();
 204
 205         Map<String, List<ProxyConnector>> proxyConnectorsMap = proxyRegistry.getProxyConnectorAsMap();
 206         List<ProxyConnector> proxyConnectors = proxyConnectorsMap.get(readMetadataRequest.getRepositoryId());
 207         if (proxyConnectors != null) {
 208             for (ProxyConnector proxyConnector : proxyConnectors) {
 209                 RemoteRepository remoteRepoConfig =
 210                         repositoryRegistry.getRemoteRepository(proxyConnector.getTargetRepository().getId());
 211
 212                 if (remoteRepoConfig != null) {
 213                     remoteRepositories.add(remoteRepoConfig);
 214
 215                     NetworkProxy networkProxyConfig =
 216                             proxyRegistry.getNetworkProxy(proxyConnector.getProxyId());
 217
 218                     if (networkProxyConfig != null) {
 219                         // key/value: remote repo ID/proxy info
 220                         networkProxies.put(proxyConnector.getTargetRepository().getId(), networkProxyConfig);
 221                     }
 222                 }
 223             }
 224         }
 225
 226         // That's a browsing request so we can a mix of SNAPSHOT and release artifacts (especially with snapshots which
 227         // can have released parent pom
 228         if (readMetadataRequest.isBrowsingRequest()) {
 229             remoteRepositories.addAll(repositoryRegistry.getRemoteRepositories());
 230         }
 231
 232         ModelBuildingRequest req =
 233                 new DefaultModelBuildingRequest().setProcessPlugins(false).setPomFile(file.getFilePath().toFile()).setTwoPhaseBuilding(
 234                         false).setValidationLevel(ModelBuildingRequest.VALIDATION_LEVEL_MINIMAL);
 235
 236         //MRM-1607. olamy this will resolve jdk profiles on the current running archiva jvm
 237         req.setSystemProperties(System.getProperties());
 238
 239         // MRM-1411
 240         req.setModelResolver(
 241                 new RepositoryModelResolver(managedRepository, pathTranslator, wagonFactory, remoteRepositories,
 242                         networkProxies, managedRepository, mavenSystemManager, metadataReader));
 243
 244         Model model;
 245         try {
 246             model = builder.build(req).getEffectiveModel();
 247         } catch (ModelBuildingException e) {
 248             String msg = "The artifact's POM file '" + file + "' was invalid: " + e.getMessage();
 249
 250             List<ModelProblem> modelProblems = e.getProblems();
 251             for (ModelProblem problem : modelProblems) {
 252                 // MRM-1411, related to MRM-1335
 253                 // this means that the problem was that the parent wasn't resolved!
 254                 // olamy really hackhish but fail with java profile so use error message
 255                 // || ( StringUtils.startsWith( problem.getMessage(), "Failed to determine Java version for profile" ) )
 256                 // but setTwoPhaseBuilding(true) fix that
 257                 if (((problem.getException() instanceof FileNotFoundException
 258                         || problem.getException() instanceof NoSuchFileException
 259                 ) && e.getModelId() != null &&
 260                         !e.getModelId().equals(problem.getModelId()))) {
 261                     log.warn("The artifact's parent POM file '{}' cannot be resolved. "
 262                             + "Using defaults for project version metadata..", file);
 263
 264                     ProjectVersionMetadata metadata = new ProjectVersionMetadata();
 265                     metadata.setId(readMetadataRequest.getProjectVersion());
 266
 267                     MavenProjectFacet facet = new MavenProjectFacet();
 268                     facet.setGroupId(readMetadataRequest.getNamespace());
 269                     facet.setArtifactId(readMetadataRequest.getProjectId());
 270                     facet.setPackaging("jar");
 271                     metadata.addFacet(facet);
 272
 273                     String errMsg =
 274                             "Error in resolving artifact's parent POM file. " + (problem.getException() == null
 275                                     ? problem.getMessage()
 276                                     : problem.getException().getMessage());
 277                     RepositoryProblemFacet repoProblemFacet = new RepositoryProblemFacet();
 278                     repoProblemFacet.setRepositoryId(readMetadataRequest.getRepositoryId());
 279                     repoProblemFacet.setId(readMetadataRequest.getRepositoryId());
 280                     repoProblemFacet.setMessage(errMsg);
 281                     repoProblemFacet.setProblem(errMsg);
 282                     repoProblemFacet.setProject(readMetadataRequest.getProjectId());
 283                     repoProblemFacet.setVersion(readMetadataRequest.getProjectVersion());
 284                     repoProblemFacet.setNamespace(readMetadataRequest.getNamespace());
 285
 286                     metadata.addFacet(repoProblemFacet);
 287
 288                     return metadata;
 289                 }
 290             }
 291
 292             throw new RepositoryStorageMetadataInvalidException("invalid-pom", msg, e);
 293         }
 294
 295         // Check if the POM is in the correct location
 296         boolean correctGroupId = readMetadataRequest.getNamespace().equals(model.getGroupId());
 297         boolean correctArtifactId = readMetadataRequest.getProjectId().equals(model.getArtifactId());
 298         boolean correctVersion = readMetadataRequest.getProjectVersion().equals(model.getVersion());
 299         if (!correctGroupId || !correctArtifactId || !correctVersion) {
 300             StringBuilder message = new StringBuilder("Incorrect POM coordinates in '" + file + "':");
 301             if (!correctGroupId) {
 302                 message.append("\nIncorrect group ID: ").append(model.getGroupId());
 303             }
 304             if (!correctArtifactId) {
 305                 message.append("\nIncorrect artifact ID: ").append(model.getArtifactId());
 306             }
 307             if (!correctVersion) {
 308                 message.append("\nIncorrect version: ").append(model.getVersion());
 309             }
 310
 311             throw new RepositoryStorageMetadataInvalidException("mislocated-pom", message.toString());
 312         }
 313
 314         ProjectVersionMetadata metadata = new ProjectVersionMetadata();
 315         metadata.setCiManagement(convertCiManagement(model.getCiManagement()));
 316         metadata.setDescription(model.getDescription());
 317         metadata.setId(readMetadataRequest.getProjectVersion());
 318         metadata.setIssueManagement(convertIssueManagement(model.getIssueManagement()));
 319         metadata.setLicenses(convertLicenses(model.getLicenses()));
 320         metadata.setMailingLists(convertMailingLists(model.getMailingLists()));
 321         metadata.setDependencies(convertDependencies(model.getDependencies()));
 322         metadata.setName(model.getName());
 323         metadata.setOrganization(convertOrganization(model.getOrganization()));
 324         metadata.setScm(convertScm(model.getScm()));
 325         metadata.setUrl(model.getUrl());
 326         metadata.setProperties(model.getProperties());
 327
 328         MavenProjectFacet facet = new MavenProjectFacet();
 329         facet.setGroupId(model.getGroupId() != null ? model.getGroupId() : model.getParent().getGroupId());
 330         facet.setArtifactId(model.getArtifactId());
 331         facet.setPackaging(model.getPackaging());
 332         if (model.getParent() != null) {
 333             MavenProjectParent parent = new MavenProjectParent();
 334             parent.setGroupId(model.getParent().getGroupId());
 335             parent.setArtifactId(model.getParent().getArtifactId());
 336             parent.setVersion(model.getParent().getVersion());
 337             facet.setParent(parent);
 338         }
 339         metadata.addFacet(facet);
 340
 341         return metadata;
 342
 343
 344     }
 345
 346     public void setWagonFactory(WagonFactory wagonFactory) {
 347         this.wagonFactory = wagonFactory;
 348     }
 349
 350     private List<org.apache.archiva.metadata.model.Dependency> convertDependencies(List<Dependency> dependencies) {
 351         List<org.apache.archiva.metadata.model.Dependency> l = new ArrayList<>();
 352         for (Dependency dependency : dependencies) {
 353             org.apache.archiva.metadata.model.Dependency newDependency =
 354                     new org.apache.archiva.metadata.model.Dependency();
 355             newDependency.setArtifactId(dependency.getArtifactId());
 356             newDependency.setClassifier(dependency.getClassifier());
 357             newDependency.setNamespace(dependency.getGroupId());
 358             newDependency.setOptional(dependency.isOptional());
 359             newDependency.setScope(dependency.getScope());
 360             newDependency.setSystemPath(dependency.getSystemPath());
 361             newDependency.setType(dependency.getType());
 362             newDependency.setVersion(dependency.getVersion());
 363             l.add(newDependency);
 364         }
 365         return l;
 366     }
 367
 368     private org.apache.archiva.metadata.model.Scm convertScm(Scm scm) {
 369         org.apache.archiva.metadata.model.Scm newScm = null;
 370         if (scm != null) {
 371             newScm = new org.apache.archiva.metadata.model.Scm();
 372             newScm.setConnection(scm.getConnection());
 373             newScm.setDeveloperConnection(scm.getDeveloperConnection());
 374             newScm.setUrl(scm.getUrl());
 375         }
 376         return newScm;
 377     }
 378
 379     private org.apache.archiva.metadata.model.Organization convertOrganization(Organization organization) {
 380         org.apache.archiva.metadata.model.Organization org = null;
 381         if (organization != null) {
 382             org = new org.apache.archiva.metadata.model.Organization();
 383             org.setName(organization.getName());
 384             org.setUrl(organization.getUrl());
 385         }
 386         return org;
 387     }
 388
 389     private List<org.apache.archiva.metadata.model.License> convertLicenses(List<License> licenses) {
 390         List<org.apache.archiva.metadata.model.License> l = new ArrayList<>();
 391         for (License license : licenses) {
 392             org.apache.archiva.metadata.model.License newLicense = new org.apache.archiva.metadata.model.License();
 393             newLicense.setName(license.getName());
 394             newLicense.setUrl(license.getUrl());
 395             l.add(newLicense);
 396         }
 397         return l;
 398     }
 399
 400     private List<org.apache.archiva.metadata.model.MailingList> convertMailingLists(List<MailingList> mailingLists) {
 401         List<org.apache.archiva.metadata.model.MailingList> l = new ArrayList<>();
 402         for (MailingList mailingList : mailingLists) {
 403             org.apache.archiva.metadata.model.MailingList newMailingList =
 404                     new org.apache.archiva.metadata.model.MailingList();
 405             newMailingList.setName(mailingList.getName());
 406             newMailingList.setMainArchiveUrl(mailingList.getArchive());
 407             newMailingList.setPostAddress(mailingList.getPost());
 408             newMailingList.setSubscribeAddress(mailingList.getSubscribe());
 409             newMailingList.setUnsubscribeAddress(mailingList.getUnsubscribe());
 410             newMailingList.setOtherArchives(mailingList.getOtherArchives());
 411             l.add(newMailingList);
 412         }
 413         return l;
 414     }
 415
 416     private org.apache.archiva.metadata.model.IssueManagement convertIssueManagement(IssueManagement issueManagement) {
 417         org.apache.archiva.metadata.model.IssueManagement im = null;
 418         if (issueManagement != null) {
 419             im = new org.apache.archiva.metadata.model.IssueManagement();
 420             im.setSystem(issueManagement.getSystem());
 421             im.setUrl(issueManagement.getUrl());
 422         }
 423         return im;
 424     }
 425
 426     private org.apache.archiva.metadata.model.CiManagement convertCiManagement(CiManagement ciManagement) {
 427         org.apache.archiva.metadata.model.CiManagement ci = null;
 428         if (ciManagement != null) {
 429             ci = new org.apache.archiva.metadata.model.CiManagement();
 430             ci.setSystem(ciManagement.getSystem());
 431             ci.setUrl(ciManagement.getUrl());
 432         }
 433         return ci;
 434     }
 435
 436     @Override
 437     public Collection<String> listRootNamespaces(String repoId, Filter<String> filter)
 438             throws RepositoryStorageRuntimeException {
 439         StorageAsset dir = getRepositoryBasedir(repoId);
 440
 441         return getSortedFiles(dir, filter);
 442     }
 443
 444     private static Collection<String> getSortedFiles(StorageAsset dir, Filter<String> filter) {
 445
 446         final Predicate<StorageAsset> dFilter = new DirectoryFilter(filter);
 447         return dir.list().stream().filter(f -> f.isContainer())
 448                 .filter(dFilter)
 449                 .map(path -> path.getName().toString())
 450                 .sorted().collect(Collectors.toList());
 451
 452     }
 453
 454     private StorageAsset getRepositoryBasedir(String repoId)
 455             throws RepositoryStorageRuntimeException {
 456         ManagedRepository repositoryConfiguration = repositoryRegistry.getManagedRepository(repoId);
 457
 458         return repositoryConfiguration.getAsset("");
 459     }
 460
 461     @Override
 462     public Collection<String> listNamespaces(String repoId, String namespace, Filter<String> filter)
 463             throws RepositoryStorageRuntimeException {
 464         StorageAsset dir = pathTranslator.toFile(getRepositoryBasedir(repoId), namespace);
 465         if (!(dir.exists()) && !dir.isContainer()) {
 466             return Collections.emptyList();
 467         }
 468         // scan all the directories which are potential namespaces. Any directories known to be projects are excluded
 469         Predicate<StorageAsset> dFilter = new DirectoryFilter(filter);
 470         return dir.list().stream().filter(dFilter).filter(path -> !isProject(path, filter)).map(path -> path.getName().toString())
 471                 .sorted().collect(Collectors.toList());
 472     }
 473
 474     @Override
 475     public Collection<String> listProjects(String repoId, String namespace, Filter<String> filter)
 476             throws RepositoryStorageRuntimeException {
 477         StorageAsset dir = pathTranslator.toFile(getRepositoryBasedir(repoId), namespace);
 478         if (!(dir.exists() && dir.isContainer())) {
 479             return Collections.emptyList();
 480         }
 481         // scan all directories in the namespace, and only include those that are known to be projects
 482         final Predicate<StorageAsset> dFilter = new DirectoryFilter(filter);
 483         return dir.list().stream().filter(dFilter).filter(path -> isProject(path, filter)).map(path -> path.getName().toString())
 484                 .sorted().collect(Collectors.toList());
 485
 486     }
 487
 488     @Override
 489     public Collection<String> listProjectVersions(String repoId, String namespace, String projectId,
 490                                                   Filter<String> filter)
 491             throws RepositoryStorageRuntimeException {
 492         StorageAsset dir = pathTranslator.toFile(getRepositoryBasedir(repoId), namespace, projectId);
 493         if (!(dir.exists() && dir.isContainer())) {
 494             return Collections.emptyList();
 495         }
 496
 497         // all directories in a project directory can be considered a version
 498         return getSortedFiles(dir, filter);
 499     }
 500
 501     @Override
 502     public Collection<ArtifactMetadata> readArtifactsMetadata(ReadMetadataRequest readMetadataRequest)
 503             throws RepositoryStorageRuntimeException {
 504         StorageAsset dir = pathTranslator.toFile(getRepositoryBasedir(readMetadataRequest.getRepositoryId()),
 505                 readMetadataRequest.getNamespace(), readMetadataRequest.getProjectId(),
 506                 readMetadataRequest.getProjectVersion());
 507         if (!(dir.exists() && dir.isContainer())) {
 508             return Collections.emptyList();
 509         }
 510
 511         // all files that are not metadata and not a checksum / signature are considered artifacts
 512         final Predicate<StorageAsset> dFilter = new ArtifactDirectoryFilter(readMetadataRequest.getFilter());
 513         // Returns a map TRUE -> (success values), FALSE -> (Exceptions)
 514         Map<Boolean, List<Try<ArtifactMetadata>>> result = dir.list().stream().filter(dFilter).map(path -> {
 515                     try {
 516                         return Try.success(getArtifactFromFile(readMetadataRequest.getRepositoryId(), readMetadataRequest.getNamespace(),
 517                                 readMetadataRequest.getProjectId(), readMetadataRequest.getProjectVersion(),
 518                                 path));
 519                     } catch (Exception e) {
 520                         log.debug("Could not create metadata for {}:  {}", path, e.getMessage(), e);
 521                         return Try.<ArtifactMetadata>failure(e);
 522                     }
 523                 }
 524         ).collect(Collectors.groupingBy(Try::isSuccess));
 525         if (result.containsKey(Boolean.FALSE) && result.get(Boolean.FALSE).size() > 0 && (!result.containsKey(Boolean.TRUE) || result.get(Boolean.TRUE).size() == 0)) {
 526             log.error("Could not get artifact metadata. Directory: {}. Number of errors {}.", dir, result.get(Boolean.FALSE).size());
 527             Try<ArtifactMetadata> failure = result.get(Boolean.FALSE).get(0);
 528             log.error("Sample exception {}", failure.getError().getMessage(), failure.getError());
 529             throw new RepositoryStorageRuntimeException(readMetadataRequest.getRepositoryId(), "Could not retrieve metadata of the files");
 530         } else {
 531             if (!result.containsKey(Boolean.TRUE) || result.get(Boolean.TRUE) == null) {
 532                 return Collections.emptyList();
 533             }
 534             return result.get(Boolean.TRUE).stream().map(tr -> tr.get()).collect(Collectors.toList());
 535         }
 536
 537     }
 538
 539     @Override
 540     public ArtifactMetadata readArtifactMetadataFromPath(String repoId, String path)
 541             throws RepositoryStorageRuntimeException {
 542         ArtifactMetadata metadata = pathTranslator.getArtifactForPath(repoId, path);
 543
 544         try {
 545             populateArtifactMetadataFromFile(metadata, getRepositoryBasedir(repoId).resolve(path));
 546         } catch (IOException e) {
 547             throw new RepositoryStorageRuntimeException(repoId, "Error during metadata retrieval of " + path + " :" + e.getMessage(), e);
 548         }
 549
 550         return metadata;
 551     }
 552
 553     private ArtifactMetadata getArtifactFromFile(String repoId, String namespace, String projectId,
 554                                                  String projectVersion, StorageAsset file) throws IOException {
 555         ArtifactMetadata metadata =
 556                 pathTranslator.getArtifactFromId(repoId, namespace, projectId, projectVersion, file.getName());
 557
 558         populateArtifactMetadataFromFile(metadata, file);
 559
 560         return metadata;
 561     }
 562
 563     @Override
 564     public void applyServerSideRelocation(ManagedRepository managedRepository, ArtifactReference artifact)
 565             throws ProxyDownloadException {
 566         if ("pom".equals(artifact.getType())) {
 567             return;
 568         }
 569
 570         // Build the artifact POM reference
 571         ArtifactReference pomReference = new ArtifactReference();
 572         pomReference.setGroupId(artifact.getGroupId());
 573         pomReference.setArtifactId(artifact.getArtifactId());
 574         pomReference.setVersion(artifact.getVersion());
 575         pomReference.setType("pom");
 576
 577         RepositoryType repositoryType = managedRepository.getType();
 578         if (!proxyRegistry.hasHandler(repositoryType)) {
 579             throw new ProxyDownloadException("No proxy handler found for repository type " + repositoryType, new HashMap<>());
 580         }
 581
 582         RepositoryProxyHandler proxyHandler = proxyRegistry.getHandler(repositoryType).get(0);
 583
 584         // Get the artifact POM from proxied repositories if needed
 585         proxyHandler.fetchFromProxies(managedRepository, pomReference);
 586
 587         // Open and read the POM from the managed repo
 588         StorageAsset pom = managedRepository.getContent().toFile(pomReference);
 589
 590         if (!pom.exists()) {
 591             return;
 592         }
 593
 594         try {
 595             // MavenXpp3Reader leaves the file open, so we need to close it ourselves.
 596
 597             Model model;
 598             try (Reader reader = Channels.newReader(pom.getReadChannel(), Charset.defaultCharset().name())) {
 599                 model = MAVEN_XPP_3_READER.read(reader);
 600             }
 601
 602             DistributionManagement dist = model.getDistributionManagement();
 603             if (dist != null) {
 604                 Relocation relocation = dist.getRelocation();
 605                 if (relocation != null) {
 606                     // artifact is relocated : update the repositoryPath
 607                     if (relocation.getGroupId() != null) {
 608                         artifact.setGroupId(relocation.getGroupId());
 609                     }
 610                     if (relocation.getArtifactId() != null) {
 611                         artifact.setArtifactId(relocation.getArtifactId());
 612                     }
 613                     if (relocation.getVersion() != null) {
 614                         artifact.setVersion(relocation.getVersion());
 615                     }
 616                 }
 617             }
 618         } catch (IOException e) {
 619             // Unable to read POM : ignore.
 620         } catch (XmlPullParserException e) {
 621             // Invalid POM : ignore
 622         }
 623     }
 624
 625
 626     @Override
 627     public String getFilePath(String requestPath, org.apache.archiva.repository.ManagedRepository managedRepository) {
 628         // managedRepository can be null
 629         // extract artifact reference from url
 630         // groupId:artifactId:version:packaging:classifier
 631         //org/apache/archiva/archiva-checksum/1.4-M4-SNAPSHOT/archiva-checksum-1.4-M4-SNAPSHOT.jar
 632         String logicalResource = null;
 633         String requestPathInfo = StringUtils.defaultString(requestPath);
 634
 635         //remove prefix ie /repository/blah becomes /blah
 636         requestPathInfo = removePrefix(requestPathInfo);
 637
 638         // Remove prefixing slash as the repository id doesn't contain it;
 639         if (requestPathInfo.startsWith("/")) {
 640             requestPathInfo = requestPathInfo.substring(1);
 641         }
 642
 643         int slash = requestPathInfo.indexOf('/');
 644         if (slash > 0) {
 645             logicalResource = requestPathInfo.substring(slash);
 646
 647             if (logicalResource.endsWith("/..")) {
 648                 logicalResource += "/";
 649             }
 650
 651             if (logicalResource != null && logicalResource.startsWith("//")) {
 652                 logicalResource = logicalResource.substring(1);
 653             }
 654
 655             if (logicalResource == null) {
 656                 logicalResource = "/";
 657             }
 658         } else {
 659             logicalResource = "/";
 660         }
 661         return logicalResource;
 662
 663     }
 664
 665     @Override
 666     public String getFilePathWithVersion(final String requestPath, ManagedRepositoryContent managedRepositoryContent)
 667             throws RelocationException
 668     {
 669
 670         if (StringUtils.endsWith(requestPath, METADATA_FILENAME)) {
 671             return getFilePath(requestPath, managedRepositoryContent.getRepository());
 672         }
 673
 674         String filePath = getFilePath(requestPath, managedRepositoryContent.getRepository());
 675
 676         ArtifactReference artifactReference = null;
 677         try {
 678             artifactReference = pathParser.toArtifactReference(filePath);
 679         } catch (LayoutException e) {
 680             return filePath;
 681         }
 682
 683         if (StringUtils.endsWith(artifactReference.getVersion(), VersionUtil.SNAPSHOT)) {
 684             // read maven metadata to get last timestamp
 685             StorageAsset metadataDir = managedRepositoryContent.getRepository().getAsset(filePath).getParent();
 686             if (!metadataDir.exists()) {
 687                 return filePath;
 688             }
 689             StorageAsset metadataFile = metadataDir.resolve(METADATA_FILENAME);
 690             if (!metadataFile.exists()) {
 691                 return filePath;
 692             }
 693             ArchivaRepositoryMetadata archivaRepositoryMetadata = null;
 694             try
 695             {
 696                 archivaRepositoryMetadata = metadataReader.read(metadataFile);
 697             }
 698             catch ( RepositoryMetadataException e )
 699             {
 700                 log.error( "Could not read metadata {}", e.getMessage( ), e );
 701                 return filePath;
 702             }
 703             int buildNumber = archivaRepositoryMetadata.getSnapshotVersion().getBuildNumber();
 704             String timestamp = archivaRepositoryMetadata.getSnapshotVersion().getTimestamp();
 705
 706             // MRM-1846
 707             if (buildNumber < 1 && timestamp == null) {
 708                 return filePath;
 709             }
 710
 711             // org/apache/archiva/archiva-checksum/1.4-M4-SNAPSHOT/archiva-checksum-1.4-M4-SNAPSHOT.jar
 712             // ->  archiva-checksum-1.4-M4-20130425.081822-1.jar
 713
 714             filePath = StringUtils.replace(filePath, //
 715                     artifactReference.getArtifactId() //
 716                             + "-" + artifactReference.getVersion(), //
 717                     artifactReference.getArtifactId() //
 718                             + "-" + StringUtils.remove(artifactReference.getVersion(),
 719                             "-" + VersionUtil.SNAPSHOT) //
 720                             + "-" + timestamp //
 721                             + "-" + buildNumber);
 722
 723             throw new RelocationException("/repository/" + managedRepositoryContent.getRepository().getId() +
 724                     (StringUtils.startsWith(filePath, "/") ? "" : "/") + filePath,
 725                     RelocationException.RelocationType.TEMPORARY);
 726
 727         }
 728
 729         return filePath;
 730     }
 731
 732     //-----------------------------
 733     // internal
 734     //-----------------------------
 735
 736     /**
 737      * FIXME remove
 738      *
 739      * @param href
 740      * @return
 741      */
 742     private static String removePrefix(final String href) {
 743         String[] parts = StringUtils.split(href, '/');
 744         parts = (String[]) ArrayUtils.subarray(parts, 1, parts.length);
 745         if (parts == null || parts.length == 0) {
 746             return "/";
 747         }
 748
 749         String joinedString = StringUtils.join(parts, '/');
 750         if (href.endsWith("/")) {
 751             joinedString = joinedString + "/";
 752         }
 753
 754         return joinedString;
 755     }
 756
 757     private static void populateArtifactMetadataFromFile(ArtifactMetadata metadata, StorageAsset file) throws IOException {
 758         metadata.setWhenGathered(ZonedDateTime.now(ZoneId.of("GMT")));
 759         metadata.setFileLastModified(file.getModificationTime().toEpochMilli());
 760         ChecksummedFile checksummedFile = new ChecksummedFile(file.getFilePath());
 761         try {
 762             metadata.setMd5(checksummedFile.calculateChecksum(ChecksumAlgorithm.MD5));
 763         } catch (IOException e) {
 764             log.error("Unable to checksum file {}: {},MD5", file, e.getMessage());
 765         }
 766         try {
 767             metadata.setSha1(checksummedFile.calculateChecksum(ChecksumAlgorithm.SHA1));
 768         } catch (IOException e) {
 769             log.error("Unable to checksum file {}: {},SHA1", file, e.getMessage());
 770         }
 771         metadata.setSize(file.getSize());
 772     }
 773
 774     private boolean isProject(StorageAsset dir, Filter<String> filter) {
 775         // scan directories for a valid project version subdirectory, meaning this must be a project directory
 776         final Predicate<StorageAsset> dFilter = new DirectoryFilter(filter);
 777         boolean projFound = dir.list().stream().filter(dFilter)
 778                 .anyMatch(path -> isProjectVersion(path));
 779         if (projFound) {
 780             return true;
 781         }
 782
 783         // if a metadata file is present, check if this is the "artifactId" directory, marking it as a project
 784         ArchivaRepositoryMetadata metadata = readMetadata(dir);
 785         if (metadata != null && dir.getName().toString().equals(metadata.getArtifactId())) {
 786             return true;
 787         }
 788
 789         return false;
 790     }
 791
 792     private boolean isProjectVersion(StorageAsset dir) {
 793         final String artifactId = dir.getParent().getName();
 794         final String projectVersion = dir.getName();
 795
 796         // check if there is a POM artifact file to ensure it is a version directory
 797
 798         Predicate<StorageAsset> filter;
 799         if (VersionUtil.isSnapshot(projectVersion)) {
 800             filter = new PomFilenameFilter(artifactId, projectVersion);
 801         } else {
 802             final String pomFile = artifactId + "-" + projectVersion + ".pom";
 803             filter = new PomFileFilter(pomFile);
 804         }
 805         if (dir.list().stream().filter(f -> !f.isContainer()).anyMatch(filter)) {
 806             return true;
 807         }
 808         // if a metadata file is present, check if this is the "version" directory, marking it as a project version
 809         ArchivaRepositoryMetadata metadata = readMetadata(dir);
 810         if (metadata != null && projectVersion.equals(metadata.getVersion())) {
 811             return true;
 812         }
 813
 814         return false;
 815     }
 816
 817     private ArchivaRepositoryMetadata readMetadata(StorageAsset directory) {
 818         ArchivaRepositoryMetadata metadata = null;
 819         StorageAsset metadataFile = directory.resolve(METADATA_FILENAME);
 820         if (metadataFile.exists()) {
 821             try {
 822                 metadata = metadataReader.read(metadataFile);
 823             } catch ( RepositoryMetadataException e )
 824             {
 825                 // Ignore missing or invalid metadata
 826             }
 827         }
 828         return metadata;
 829     }
 830
 831     private static class DirectoryFilter
 832             implements Predicate<StorageAsset> {
 833         private final Filter<String> filter;
 834
 835         public DirectoryFilter(Filter<String> filter) {
 836             this.filter = filter;
 837         }
 838
 839         @Override
 840         public boolean test(StorageAsset dir) {
 841             final String name = dir.getName();
 842             if (!filter.accept(name)) {
 843                 return false;
 844             } else if (name.startsWith(".")) {
 845                 return false;
 846             } else if (!dir.isContainer()) {
 847                 return false;
 848             }
 849             return true;
 850         }
 851     }
 852
 853     private static class ArtifactDirectoryFilter
 854             implements Predicate<StorageAsset> {
 855         private final Filter<String> filter;
 856
 857         private ArtifactDirectoryFilter(Filter<String> filter) {
 858             this.filter = filter;
 859         }
 860
 861         @Override
 862         public boolean test(StorageAsset file) {
 863             final Set<String> checksumExts = ChecksumAlgorithm.getAllExtensions();
 864             final String path = file.getPath();
 865             final String name = file.getName();
 866             final String extension = StringUtils.substringAfterLast(name, ".").toLowerCase();
 867             // TODO compare to logic in maven-repository-layer
 868             if (file.isContainer()) {
 869                 return false;
 870             } else if (!filter.accept(name)) {
 871                 return false;
 872             } else if (name.startsWith(".") || path.contains("/.") ) {
 873                 return false;
 874             } else if (checksumExts.contains(extension)) {
 875                 return false;
 876             } else if (Arrays.binarySearch(IGNORED_FILES, name) >= 0) {
 877                 return false;
 878             }
 879             // some files from remote repositories can have name like maven-metadata-archiva-vm-all-public.xml
 880             else if (StringUtils.startsWith(name, METADATA_FILENAME_START) && StringUtils.endsWith(name, ".xml")) {
 881                 return false;
 882             }
 883
 884             return true;
 885
 886         }
 887     }
 888
 889
 890     private static final class PomFilenameFilter
 891             implements Predicate<StorageAsset> {
 892
 893         private final String artifactId, projectVersion;
 894
 895         private PomFilenameFilter(String artifactId, String projectVersion) {
 896             this.artifactId = artifactId;
 897             this.projectVersion = projectVersion;
 898         }
 899
 900         @Override
 901         public boolean test(StorageAsset dir) {
 902             final String name = dir.getName();
 903             if (name.startsWith(artifactId + "-") && name.endsWith(".pom")) {
 904                 String v = name.substring(artifactId.length() + 1, name.length() - 4);
 905                 v = VersionUtil.getBaseVersion(v);
 906                 if (v.equals(projectVersion)) {
 907                     return true;
 908                 }
 909             }
 910             return false;
 911         }
 912
 913     }
 914
 915     private static class PomFileFilter
 916             implements Predicate<StorageAsset> {
 917         private final String pomFile;
 918
 919         private PomFileFilter(String pomFile) {
 920             this.pomFile = pomFile;
 921         }
 922
 923         @Override
 924         public boolean test(StorageAsset dir) {
 925             return pomFile.equals(dir.getName());
 926         }
 927     }
 928
 929
 930     public PathParser getPathParser() {
 931         return pathParser;
 932     }
 933
 934     public void setPathParser(PathParser pathParser) {
 935         this.pathParser = pathParser;
 936     }
 937 }